From fc14ac4de3106228ca7e084ad24a46b8eccf8761 Mon Sep 17 00:00:00 2001 From: TGooch44 Date: Thu, 14 Mar 2019 17:25:55 -0700 Subject: [PATCH 001/642] Initial python implementation (#54) --- CHANGELOG.md | 4 + README.md | 3 + iceberg/__init__.py | 17 + iceberg/api/__init__.py | 41 + iceberg/api/append_files.py | 24 + iceberg/api/combined_scan_task.py | 27 + iceberg/api/data_file.py | 98 +++ iceberg/api/delete_files.py | 34 + iceberg/api/exceptions/__init__.py | 19 + iceberg/api/exceptions/already_exists.py | 19 + .../api/exceptions/validation_exception.py | 26 + iceberg/api/expire_snapshots.py | 30 + iceberg/api/expressions/__init__.py | 97 +++ iceberg/api/expressions/binder.py | 90 +++ iceberg/api/expressions/evaluator.py | 90 +++ iceberg/api/expressions/expression.py | 234 ++++++ iceberg/api/expressions/expressions.py | 262 +++++++ .../inclusive_metrics_evaluator.py | 185 +++++ .../expressions/java_variables/__init__.py | 21 + iceberg/api/expressions/literals.py | 585 +++++++++++++++ iceberg/api/expressions/predicate.py | 137 ++++ iceberg/api/expressions/reference.py | 84 +++ .../expressions/strict_metrics_evaluator.py | 215 ++++++ iceberg/api/file_format.py | 42 ++ iceberg/api/file_scan_task.py | 42 ++ iceberg/api/files.py | 126 ++++ iceberg/api/filterable.py | 32 + iceberg/api/filtered_snapshot.py | 40 + iceberg/api/io/__init__.py | 30 + iceberg/api/io/closeable_group.py | 31 + iceberg/api/io/closeable_iterable.py | 30 + iceberg/api/io/delegating_input_stream.py | 22 + iceberg/api/io/delegating_output_stream.py | 22 + iceberg/api/io/file_appender.py | 29 + iceberg/api/io/input_file.py | 28 + iceberg/api/io/output_file.py | 28 + iceberg/api/io/position_output_stream.py | 32 + iceberg/api/io/seekable_input_stream.py | 46 ++ iceberg/api/manifest_file.py | 74 ++ iceberg/api/metrics.py | 30 + iceberg/api/overwrite_files.py | 30 + iceberg/api/partition_field.py | 41 + iceberg/api/partition_spec.py | 308 ++++++++ iceberg/api/pending_update.py | 25 + iceberg/api/replace_partitions.py | 30 + iceberg/api/rewrite_files.py | 30 + iceberg/api/rollback.py | 33 + iceberg/api/scan_task.py | 28 + iceberg/api/schema.py | 136 ++++ iceberg/api/snapshot.py | 37 + iceberg/api/snapshot_iterable.py | 21 + iceberg/api/struct_like.py | 28 + iceberg/api/table.py | 76 ++ iceberg/api/table_scan.py | 40 + iceberg/api/tables.py | 25 + iceberg/api/transaction.py | 55 ++ iceberg/api/transforms/__init__.py | 48 ++ iceberg/api/transforms/bucket.py | 185 +++++ iceberg/api/transforms/dates.py | 76 ++ iceberg/api/transforms/identity.py | 91 +++ iceberg/api/transforms/projection_util.py | 66 ++ iceberg/api/transforms/timestamps.py | 72 ++ iceberg/api/transforms/transform.py | 40 + iceberg/api/transforms/transform_util.py | 84 +++ iceberg/api/transforms/transforms.py | 111 +++ iceberg/api/transforms/truncate.py | 231 ++++++ iceberg/api/types/#conversions.py# | 87 +++ iceberg/api/types/__init__.py | 134 ++++ iceberg/api/types/check_compatibility.py | 0 iceberg/api/types/conversions.py | 101 +++ iceberg/api/types/type.py | 119 +++ iceberg/api/types/type_util.py | 528 +++++++++++++ iceberg/api/types/types.py | 706 ++++++++++++++++++ iceberg/api/update_properties.py | 30 + iceberg/api/update_schema.py | 33 + iceberg/core/__init__.py | 59 ++ iceberg/core/avro/__init__.py | 21 + iceberg/core/avro/avro_schema_util.py | 34 + iceberg/core/avro/avro_to_iceberg.py | 289 +++++++ iceberg/core/avro/iceberg_to_avro.py | 90 +++ .../core/base_metastore_table_operations.py | 113 +++ iceberg/core/base_metastore_tables.py | 53 ++ iceberg/core/base_snapshot.py | 125 ++++ iceberg/core/base_table.py | 109 +++ iceberg/core/base_table_scan.py | 36 + iceberg/core/base_transaction.py | 176 +++++ iceberg/core/config_properties.py | 26 + iceberg/core/data_files.py | 225 ++++++ iceberg/core/generic_data_file.py | 68 ++ iceberg/core/generic_manifest_file.py | 179 +++++ .../core/generic_partition_field_summary.py | 55 ++ iceberg/core/hadoop/__init__.py | 22 + iceberg/core/hadoop/file_status.py | 34 + iceberg/core/hadoop/file_system.py | 25 + iceberg/core/hadoop/hadoop_input_file.py | 61 ++ iceberg/core/hadoop/hadoop_output_file.py | 49 ++ .../core/hadoop/hadoop_table_operations.py | 49 ++ iceberg/core/hadoop/local_filesystem.py | 84 +++ iceberg/core/hadoop/s3_filesystem_wrapper.py | 54 ++ iceberg/core/hadoop/util.py | 53 ++ iceberg/core/manifest_entry.py | 141 ++++ iceberg/core/manifest_list_writer.py | 55 ++ iceberg/core/manifest_reader.py | 106 +++ iceberg/core/partition_data.py | 91 +++ iceberg/core/partition_spec_parser.py | 89 +++ iceberg/core/partition_summary.py | 31 + iceberg/core/schema_parser.py | 176 +++++ iceberg/core/schema_update.py | 40 + iceberg/core/snapshot_parser.py | 72 ++ iceberg/core/table_metadata.py | 189 +++++ iceberg/core/table_metadata_parser.py | 124 +++ iceberg/core/table_operations.py | 42 ++ iceberg/core/util/__init__.py | 20 + iceberg/core/util/atomic_integer.py | 36 + iceberg/exceptions/__init__.py | 26 + iceberg/exceptions/exceptions.py | 39 + iceberg/spark/__init__.py | 19 + iceberg/spark/source/__init__.py | 19 + iceberg/spark/source/spark_catalog.py | 53 ++ iceberg/spark/table_identifier.py | 21 + setup.py | 39 + tests/__init__.py | 17 + tests/api/__init__.py | 17 + tests/api/expressions/__init__.py | 17 + tests/api/expressions/conftest.py | 307 ++++++++ tests/api/expressions/test_evaluator.py | 130 ++++ .../expressions/test_expression_binding.py | 131 ++++ .../expressions/test_expression_helpers.py | 47 ++ .../test_expression_serializations.py | 21 + .../test_inclusive_metrics_evaluator.py | 118 +++ .../expressions/test_literal_serialization.py | 20 + .../test_misc_literal_conversions.py | 263 +++++++ .../test_numeric_literal_conversions.py | 116 +++ .../api/expressions/test_predicate_binding.py | 199 +++++ .../test_strict_metrics_evaluator.py | 159 ++++ .../test_string_literal_conversions.py | 102 +++ tests/api/test_conversions.py | 34 + tests/api/test_helpers.py | 79 ++ tests/api/transforms/__init__.py | 17 + tests/api/transforms/test_bucketing.py | 64 ++ tests/api/transforms/test_dates.py | 44 ++ tests/api/transforms/test_identity.py | 93 +++ tests/api/transforms/test_timestamps.py | 46 ++ tests/api/transforms/test_truncate.py | 59 ++ tests/api/types/__init__.py | 17 + tests/api/types/test_binary_comparator.py | 41 + tests/api/types/test_char_seq_comparator.py | 43 ++ tests/api/types/test_comparable_comparator.py | 51 ++ tests/api/types/test_readabilty_checks.py | 57 ++ tests/core/__init__.py | 17 + tests/core/avro/__init__.py | 17 + tests/core/avro/test_avro.py | 7 + tests/core/conftest.py | 259 +++++++ tests/core/test_snapshot_json.py | 26 + tests/core/test_table_metadata_json.py | 111 +++ tests/core/test_table_metadata_parser.py | 44 ++ tox.ini | 83 ++ 157 files changed, 13101 insertions(+) create mode 100644 CHANGELOG.md create mode 100644 README.md create mode 100644 iceberg/__init__.py create mode 100644 iceberg/api/__init__.py create mode 100644 iceberg/api/append_files.py create mode 100644 iceberg/api/combined_scan_task.py create mode 100644 iceberg/api/data_file.py create mode 100644 iceberg/api/delete_files.py create mode 100644 iceberg/api/exceptions/__init__.py create mode 100644 iceberg/api/exceptions/already_exists.py create mode 100644 iceberg/api/exceptions/validation_exception.py create mode 100644 iceberg/api/expire_snapshots.py create mode 100644 iceberg/api/expressions/__init__.py create mode 100644 iceberg/api/expressions/binder.py create mode 100644 iceberg/api/expressions/evaluator.py create mode 100644 iceberg/api/expressions/expression.py create mode 100644 iceberg/api/expressions/expressions.py create mode 100644 iceberg/api/expressions/inclusive_metrics_evaluator.py create mode 100644 iceberg/api/expressions/java_variables/__init__.py create mode 100644 iceberg/api/expressions/literals.py create mode 100644 iceberg/api/expressions/predicate.py create mode 100644 iceberg/api/expressions/reference.py create mode 100644 iceberg/api/expressions/strict_metrics_evaluator.py create mode 100644 iceberg/api/file_format.py create mode 100644 iceberg/api/file_scan_task.py create mode 100644 iceberg/api/files.py create mode 100644 iceberg/api/filterable.py create mode 100644 iceberg/api/filtered_snapshot.py create mode 100644 iceberg/api/io/__init__.py create mode 100644 iceberg/api/io/closeable_group.py create mode 100644 iceberg/api/io/closeable_iterable.py create mode 100644 iceberg/api/io/delegating_input_stream.py create mode 100644 iceberg/api/io/delegating_output_stream.py create mode 100644 iceberg/api/io/file_appender.py create mode 100644 iceberg/api/io/input_file.py create mode 100644 iceberg/api/io/output_file.py create mode 100644 iceberg/api/io/position_output_stream.py create mode 100644 iceberg/api/io/seekable_input_stream.py create mode 100644 iceberg/api/manifest_file.py create mode 100644 iceberg/api/metrics.py create mode 100644 iceberg/api/overwrite_files.py create mode 100644 iceberg/api/partition_field.py create mode 100644 iceberg/api/partition_spec.py create mode 100644 iceberg/api/pending_update.py create mode 100644 iceberg/api/replace_partitions.py create mode 100644 iceberg/api/rewrite_files.py create mode 100644 iceberg/api/rollback.py create mode 100644 iceberg/api/scan_task.py create mode 100644 iceberg/api/schema.py create mode 100644 iceberg/api/snapshot.py create mode 100644 iceberg/api/snapshot_iterable.py create mode 100644 iceberg/api/struct_like.py create mode 100644 iceberg/api/table.py create mode 100644 iceberg/api/table_scan.py create mode 100644 iceberg/api/tables.py create mode 100644 iceberg/api/transaction.py create mode 100644 iceberg/api/transforms/__init__.py create mode 100644 iceberg/api/transforms/bucket.py create mode 100644 iceberg/api/transforms/dates.py create mode 100644 iceberg/api/transforms/identity.py create mode 100644 iceberg/api/transforms/projection_util.py create mode 100644 iceberg/api/transforms/timestamps.py create mode 100644 iceberg/api/transforms/transform.py create mode 100644 iceberg/api/transforms/transform_util.py create mode 100644 iceberg/api/transforms/transforms.py create mode 100644 iceberg/api/transforms/truncate.py create mode 100644 iceberg/api/types/#conversions.py# create mode 100644 iceberg/api/types/__init__.py create mode 100644 iceberg/api/types/check_compatibility.py create mode 100644 iceberg/api/types/conversions.py create mode 100644 iceberg/api/types/type.py create mode 100644 iceberg/api/types/type_util.py create mode 100644 iceberg/api/types/types.py create mode 100644 iceberg/api/update_properties.py create mode 100644 iceberg/api/update_schema.py create mode 100644 iceberg/core/__init__.py create mode 100644 iceberg/core/avro/__init__.py create mode 100644 iceberg/core/avro/avro_schema_util.py create mode 100644 iceberg/core/avro/avro_to_iceberg.py create mode 100644 iceberg/core/avro/iceberg_to_avro.py create mode 100644 iceberg/core/base_metastore_table_operations.py create mode 100644 iceberg/core/base_metastore_tables.py create mode 100644 iceberg/core/base_snapshot.py create mode 100644 iceberg/core/base_table.py create mode 100644 iceberg/core/base_table_scan.py create mode 100644 iceberg/core/base_transaction.py create mode 100644 iceberg/core/config_properties.py create mode 100644 iceberg/core/data_files.py create mode 100644 iceberg/core/generic_data_file.py create mode 100644 iceberg/core/generic_manifest_file.py create mode 100644 iceberg/core/generic_partition_field_summary.py create mode 100644 iceberg/core/hadoop/__init__.py create mode 100644 iceberg/core/hadoop/file_status.py create mode 100644 iceberg/core/hadoop/file_system.py create mode 100644 iceberg/core/hadoop/hadoop_input_file.py create mode 100644 iceberg/core/hadoop/hadoop_output_file.py create mode 100644 iceberg/core/hadoop/hadoop_table_operations.py create mode 100644 iceberg/core/hadoop/local_filesystem.py create mode 100644 iceberg/core/hadoop/s3_filesystem_wrapper.py create mode 100644 iceberg/core/hadoop/util.py create mode 100644 iceberg/core/manifest_entry.py create mode 100644 iceberg/core/manifest_list_writer.py create mode 100644 iceberg/core/manifest_reader.py create mode 100644 iceberg/core/partition_data.py create mode 100644 iceberg/core/partition_spec_parser.py create mode 100644 iceberg/core/partition_summary.py create mode 100644 iceberg/core/schema_parser.py create mode 100644 iceberg/core/schema_update.py create mode 100644 iceberg/core/snapshot_parser.py create mode 100644 iceberg/core/table_metadata.py create mode 100644 iceberg/core/table_metadata_parser.py create mode 100644 iceberg/core/table_operations.py create mode 100644 iceberg/core/util/__init__.py create mode 100644 iceberg/core/util/atomic_integer.py create mode 100644 iceberg/exceptions/__init__.py create mode 100644 iceberg/exceptions/exceptions.py create mode 100644 iceberg/spark/__init__.py create mode 100644 iceberg/spark/source/__init__.py create mode 100644 iceberg/spark/source/spark_catalog.py create mode 100644 iceberg/spark/table_identifier.py create mode 100755 setup.py create mode 100644 tests/__init__.py create mode 100644 tests/api/__init__.py create mode 100644 tests/api/expressions/__init__.py create mode 100644 tests/api/expressions/conftest.py create mode 100644 tests/api/expressions/test_evaluator.py create mode 100644 tests/api/expressions/test_expression_binding.py create mode 100644 tests/api/expressions/test_expression_helpers.py create mode 100644 tests/api/expressions/test_expression_serializations.py create mode 100644 tests/api/expressions/test_inclusive_metrics_evaluator.py create mode 100644 tests/api/expressions/test_literal_serialization.py create mode 100644 tests/api/expressions/test_misc_literal_conversions.py create mode 100644 tests/api/expressions/test_numeric_literal_conversions.py create mode 100644 tests/api/expressions/test_predicate_binding.py create mode 100644 tests/api/expressions/test_strict_metrics_evaluator.py create mode 100644 tests/api/expressions/test_string_literal_conversions.py create mode 100644 tests/api/test_conversions.py create mode 100644 tests/api/test_helpers.py create mode 100644 tests/api/transforms/__init__.py create mode 100644 tests/api/transforms/test_bucketing.py create mode 100644 tests/api/transforms/test_dates.py create mode 100644 tests/api/transforms/test_identity.py create mode 100644 tests/api/transforms/test_timestamps.py create mode 100644 tests/api/transforms/test_truncate.py create mode 100644 tests/api/types/__init__.py create mode 100644 tests/api/types/test_binary_comparator.py create mode 100644 tests/api/types/test_char_seq_comparator.py create mode 100644 tests/api/types/test_comparable_comparator.py create mode 100644 tests/api/types/test_readabilty_checks.py create mode 100644 tests/core/__init__.py create mode 100644 tests/core/avro/__init__.py create mode 100644 tests/core/avro/test_avro.py create mode 100644 tests/core/conftest.py create mode 100644 tests/core/test_snapshot_json.py create mode 100644 tests/core/test_table_metadata_json.py create mode 100644 tests/core/test_table_metadata_parser.py create mode 100644 tox.ini diff --git a/CHANGELOG.md b/CHANGELOG.md new file mode 100644 index 0000000000..2ac6aeaac2 --- /dev/null +++ b/CHANGELOG.md @@ -0,0 +1,4 @@ +# iceberg Changelog + +## iceberg 0.0.1 (2019-02-08) +* Library creation. - [Ted Gooch] diff --git a/README.md b/README.md new file mode 100644 index 0000000000..2b30b5c00f --- /dev/null +++ b/README.md @@ -0,0 +1,3 @@ +#Iceberg +A python implementation of the Iceberg table format. +See the project level README for more details: https://github.com/apache/incubator-iceberg \ No newline at end of file diff --git a/iceberg/__init__.py b/iceberg/__init__.py new file mode 100644 index 0000000000..245692337b --- /dev/null +++ b/iceberg/__init__.py @@ -0,0 +1,17 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + diff --git a/iceberg/api/__init__.py b/iceberg/api/__init__.py new file mode 100644 index 0000000000..9586e3603f --- /dev/null +++ b/iceberg/api/__init__.py @@ -0,0 +1,41 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +__all__ = ["DataFile", "FileFormat", "Files", "Filterable", + "FilteredSnapshot", "ManifestFile", "PartitionFieldSummary", + "Metrics", "PartitionSpec", "PartitionSpecBuilder", + "Schema", "Snapshot", "SnapshotIterable", "StructLike", + "Table", "Tables", "TableScan", "Transaction", "UpdateSchema"] + +from .data_file import DataFile +from .file_format import FileFormat +from .files import Files +from .filterable import Filterable +from .filtered_snapshot import FilteredSnapshot +from .manifest_file import ManifestFile, PartitionFieldSummary +from .metrics import Metrics +from .partition_spec import (PartitionSpec, + PartitionSpecBuilder) +from .schema import Schema +from .snapshot import Snapshot +from .snapshot_iterable import SnapshotIterable +from .struct_like import StructLike +from .table import Table +from .table_scan import TableScan +from .tables import Tables +from .transaction import Transaction +from .update_schema import UpdateSchema diff --git a/iceberg/api/append_files.py b/iceberg/api/append_files.py new file mode 100644 index 0000000000..435f2f825b --- /dev/null +++ b/iceberg/api/append_files.py @@ -0,0 +1,24 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +from .pending_update import PendingUpdate + + +class AppendFiles(PendingUpdate): + + def append_file(self, file): + raise NotImplementedError() diff --git a/iceberg/api/combined_scan_task.py b/iceberg/api/combined_scan_task.py new file mode 100644 index 0000000000..e385ba524e --- /dev/null +++ b/iceberg/api/combined_scan_task.py @@ -0,0 +1,27 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +from .scan_task import ScanTask + + +class CombinedScanTask(ScanTask): + + def files(self): + raise NotImplementedError() + + def as_combined_scan_task(self): + return self diff --git a/iceberg/api/data_file.py b/iceberg/api/data_file.py new file mode 100644 index 0000000000..7f99ffa563 --- /dev/null +++ b/iceberg/api/data_file.py @@ -0,0 +1,98 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +from iceberg.api.types import (BinaryType, + IntegerType, + ListType, + LongType, + MapType, + NestedField, + StringType, + StructType) + + +class DataFile(object): + + @staticmethod + def get_type(partition_type): + return StructType.of([NestedField.required(100, "file_path", StringType.get()), + NestedField.required(101, "file_format", StringType.get()), + NestedField.required(102, "partition", partition_type), + NestedField.required(103, "record_count", LongType.get()), + NestedField.required(104, "file_size_in_bytes", LongType.get()), + NestedField.required(105, "block_size_in_bytes", LongType.get()), + NestedField.optional(106, "file_ordinal", IntegerType.get()), + NestedField.optional(107, "sort_columns", ListType.of_required(112, IntegerType.get())), + NestedField.optional(108, "column_sizes", MapType.of_required(117, 118, + IntegerType.get(), + LongType.get())), + NestedField.optional(109, "value_counts", MapType.of_required(119, 120, + IntegerType.get(), + LongType.get())), + NestedField.optional(110, "null_value_counts", MapType.of_required(121, 122, + IntegerType.get(), + LongType.get())), + NestedField.optional(125, "lower_bounds", MapType.of_required(126, 127, + IntegerType.get(), + BinaryType.get())), + NestedField.optional(128, "upper_bounds", MapType.of_required(129, 130, + IntegerType.get(), + BinaryType.get()))] + # NEXT ID TO ASSIGN: 131 + ) + + def path(self): + raise NotImplementedError() + + def format(self): + raise NotImplementedError() + + def partition(self): + raise NotImplementedError() + + def record_count(self): + raise NotImplementedError() + + def file_size_in_bytes(self): + raise NotImplementedError() + + def block_size_in_bytes(self): + raise NotImplementedError() + + def file_ordinal(self): + raise NotImplementedError() + + def sort_columns(self): + raise NotImplementedError() + + def column_size(self): + raise NotImplementedError() + + def value_counts(self): + raise NotImplementedError() + + def null_value_counts(self): + raise NotImplementedError() + + def lower_bounds(self): + raise NotImplementedError() + + def upper_bounds(self): + raise NotImplementedError() + + def copy(self): + raise NotImplementedError() diff --git a/iceberg/api/delete_files.py b/iceberg/api/delete_files.py new file mode 100644 index 0000000000..9b6c01846b --- /dev/null +++ b/iceberg/api/delete_files.py @@ -0,0 +1,34 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +from .pending_update import PendingUpdate + + +class DeleteFiles(PendingUpdate): + + def delete_files(self, path=None, datafile=None): + if datafile is not None: + path = datafile.path + + self._delete_from_path(path) + return self + + def _delete_from_path(self): + raise NotImplementedError() + + def delete_from_row_filter(self, expr): + raise NotImplementedError() diff --git a/iceberg/api/exceptions/__init__.py b/iceberg/api/exceptions/__init__.py new file mode 100644 index 0000000000..d0d7056ef9 --- /dev/null +++ b/iceberg/api/exceptions/__init__.py @@ -0,0 +1,19 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +from .already_exists import AlreadyExists # noqa +from .validation_exception import ValidationException # noqa \ No newline at end of file diff --git a/iceberg/api/exceptions/already_exists.py b/iceberg/api/exceptions/already_exists.py new file mode 100644 index 0000000000..e92de04b35 --- /dev/null +++ b/iceberg/api/exceptions/already_exists.py @@ -0,0 +1,19 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +class AlreadyExists(Exception): + pass diff --git a/iceberg/api/exceptions/validation_exception.py b/iceberg/api/exceptions/validation_exception.py new file mode 100644 index 0000000000..7261e6bd8c --- /dev/null +++ b/iceberg/api/exceptions/validation_exception.py @@ -0,0 +1,26 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +class ValidationException(RuntimeError): + + @staticmethod + def check(test, message, args): + if not test: + raise ValidationException(message, args) + + def __init__(self, message, args): + super(ValidationException, self).__init__(message % args) diff --git a/iceberg/api/expire_snapshots.py b/iceberg/api/expire_snapshots.py new file mode 100644 index 0000000000..0406844097 --- /dev/null +++ b/iceberg/api/expire_snapshots.py @@ -0,0 +1,30 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +from .pending_update import PendingUpdate + + +class ExpireSnapshots(PendingUpdate): + + def expire_snapshot_id(self, snapshot_id): + raise NotImplementedError() + + def expire_older_than(self, timestamp_millis): + raise NotImplementedError() + + def delete_with(self, delete_funct): + raise NotImplementedError() diff --git a/iceberg/api/expressions/__init__.py b/iceberg/api/expressions/__init__.py new file mode 100644 index 0000000000..126bdd703a --- /dev/null +++ b/iceberg/api/expressions/__init__.py @@ -0,0 +1,97 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +from __future__ import absolute_import + +__all__ = ["ABOVE_MAX", + "And", + "BELOW_MIN", + "BinaryLiteral", + "Binder", + "BooleanLiteral", + "BoundPredicate", + "BoundReference", + "Evaluator", + "Expression", + "ExpressionVisitors", + "Expressions", + "DateLiteral", + "DecimalLiteral", + "DoubleLiteral", + "FALSE", + "FalseExp", + "FixedLiteral", + "FloatLiteral", + "InclusiveMetricsEvaluator", + "IntegerLiteral", + "JAVA_MAX_FLOAT", + "JAVA_MAX_INT", + "JAVA_MIN_FLOAT", + "JAVA_MIN_INT", + "Literal", + "Literals", + "NamedReference", + "Not", + "Operation", + "Or", + "Predicate", + "Reference", + "StrictMetricsEvaluator", + "StringLiteral", + "TRUE", + "TrueExp", + "UUIDLiteral", + "UnboundPredicate"] + +from .evaluator import (Binder, + Evaluator) +from .expression import (And, + Expression, + FALSE, + FalseExp, + Not, + Operation, + Or, + TRUE, + TrueExp) +from .expressions import Expressions, ExpressionVisitors +from .inclusive_metrics_evaluator import InclusiveMetricsEvaluator +from .java_variables import (JAVA_MAX_FLOAT, + JAVA_MAX_INT, + JAVA_MIN_FLOAT, + JAVA_MIN_INT) +from .literals import (ABOVE_MAX, + BELOW_MIN, + BinaryLiteral, + BooleanLiteral, + DateLiteral, + DecimalLiteral, + DoubleLiteral, + FixedLiteral, + FloatLiteral, + IntegerLiteral, + Literal, + Literals, + StringLiteral, + UUIDLiteral) +from .predicate import (BoundPredicate, + Predicate, + UnboundPredicate) +from .reference import (BoundReference, + NamedReference, + Reference) +from .strict_metrics_evaluator import StrictMetricsEvaluator diff --git a/iceberg/api/expressions/binder.py b/iceberg/api/expressions/binder.py new file mode 100644 index 0000000000..ad5921ca9a --- /dev/null +++ b/iceberg/api/expressions/binder.py @@ -0,0 +1,90 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +from .expressions import Expressions, ExpressionVisitors +from .predicate import BoundPredicate + + +class Binder(object): + + @staticmethod + def bind(struct, expr): + return ExpressionVisitors.visit(expr, Binder.BindVisitor(struct)) + + @staticmethod + def bound_references(struct, exprs): + if exprs is None: + return set() + visitor = Binder.ReferenceVisitor() + for expr in exprs: + ExpressionVisitors.visit(Binder.bind(struct, expr), visitor) + + return visitor.references + + def __init__(self): + pass + + class BindVisitor(ExpressionVisitors.ExpressionVisitor): + + def __init__(self, struct): + self.struct = struct + + def always_true(self): + return Expressions.always_true() + + def always_false(self): + return Expressions.always_false() + + def not_(self, result): + return Expressions.not_(result) + + def and_(self, left_result, right_result): + return Expressions.and_(left_result, right_result) + + def or_(self, left_result, right_result): + return Expressions.or_(left_result, right_result) + + def predicate(self, pred): + if isinstance(pred, BoundPredicate): + raise RuntimeError("Found already bound predicate: {}".format(pred)) + + return pred.bind(self.struct) + + class ReferenceVisitor(ExpressionVisitors.ExpressionVisitor): + + def __init__(self): + self.references = set() + + def always_true(self): + return self.references + + def always_false(self): + return self.references + + def not_(self, result): + return self.references + + def and_(self, left_result, right_result): + return self.references + + def or_(self, left_result, right_result): + return self.references + + def predicate(self, pred): + if isinstance(pred, BoundPredicate): + self.references.add(pred.ref.field_id) + return self.references diff --git a/iceberg/api/expressions/evaluator.py b/iceberg/api/expressions/evaluator.py new file mode 100644 index 0000000000..d4bf26933a --- /dev/null +++ b/iceberg/api/expressions/evaluator.py @@ -0,0 +1,90 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +from .binder import Binder +from .expressions import ExpressionVisitors + + +class Evaluator(object): + + def __init__(self, struct, unbound): + self.expr = Binder.bind(struct, unbound) + self.visitors = None + + def visitor(self): + if self.visitors is None: + self.visitors = Evaluator.EvalVisitor() + + return self.visitors + + def eval(self, data): + return self.visitor().eval(data, self.expr) + + class EvalVisitor(ExpressionVisitors.BoundExpressionVisitor): + + def __init__(self): + super(Evaluator.EvalVisitor, self).__init__() + self.struct = None + + def eval(self, row, expr): + self.struct = row + return ExpressionVisitors.visit(expr, self) + + def always_true(self): + return True + + def always_false(self): + return False + + def not_(self, result): + return not result + + def and_(self, left_result, right_result): + return left_result and right_result + + def or_(self, left_result, right_result): + return left_result or right_result + + def is_null(self, ref): + return ref.get(self.struct) is None + + def not_null(self, ref): + return not (ref.get(self.struct) is None) + + def lt(self, ref, lit): + return ref.get(self.struct) < lit.value + + def lt_eq(self, ref, lit): + return ref.get(self.struct) <= lit.value + + def gt(self, ref, lit): + return ref.get(self.struct) > lit.value + + def gt_eq(self, ref, lit): + return ref.get(self.struct) >= lit.value + + def eq(self, ref, lit): + return ref.get(self.struct) == lit.value + + def not_eq(self, ref, lit): + return ref.get(self.struct) != lit.value + + def in_(self, ref, lit): + raise NotImplementedError() + + def not_in(self, ref, lit): + return not self.in_(ref, lit.value) diff --git a/iceberg/api/expressions/expression.py b/iceberg/api/expressions/expression.py new file mode 100644 index 0000000000..1aedf5ccb2 --- /dev/null +++ b/iceberg/api/expressions/expression.py @@ -0,0 +1,234 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +from enum import Enum + + +class Expression(object): + def __init__(self): + pass + + def negate(self): + raise RuntimeError("%s cannot be negated" % self) + + +class Operation(Enum): + TRUE = "TRUE" + FALSE = "FALSE" + IS_NULL = "IS_NULL" + NOT_NULL = "NOT_NULL" + LT = "LT" + LT_EQ = "LT_EQ" + GT = "GT" + GT_EQ = "GT_EQ" + EQ = "EQ" + NOT_EQ = "NOT_EQ" + IN = "IN" + NOT_IN = "NOT_IN" + NOT = "NOT" + AND = "AND" + OR = "OR" + + def negate(self): # noqa + if self == Operation.IS_NULL: + return Operation.NOT_NULL + elif self == Operation.NOT_NULL: + return Operation.IS_NULL + elif self == Operation.LT: + return Operation.GT_EQ + elif self == Operation.LT_EQ: + return Operation.GT + elif self == Operation.GT: + return Operation.LT_EQ + elif self == Operation.GT_EQ: + return Operation.LT + elif self == Operation.EQ: + return Operation.NOT_EQ + elif self == Operation.NOT_EQ: + return Operation.EQ + elif self == Operation.IN: + return Operation.NOT_IN + elif self == Operation.NOT_IN: + return Operation.IN + else: + raise RuntimeError("No negation for operation: %s" % self) + + def flipLR(self): + if self == Operation.LT: + return Operation.GT + elif self == Operation.LT_EQ: + return Operation.GT_EQ + elif self == Operation.GT: + return Operation.LT + elif self == Operation.GT_EQ: + return Operation.LT_EQ + elif self == Operation.EQ: + return Operation.EQ + elif self == Operation.NOT_EQ: + return Operation.NOT_EQ + elif self == Operation.AND: + return Operation.AND + elif self == Operation.OR: + return Operation.OR + else: + raise RuntimeError("No left-right flip for operation: %s" % self) + + def op(self): + pass + + +class And(Expression): + + def __init__(self, left, right): + self.left = left + self.right = right + + def __eq__(self, other): + if id(self) == id(other): + return True + elif other is None or not isinstance(other, And): + return False + + return self.left == other.left and self.right == other.right + + def __ne__(self, other): + return not self.__eq__(other) + + def op(self): + return Operation.AND + + def negate(self): + from .expressions import Expressions + return Expressions.or_(self.left.negate(), self.right.negate()) # noqa + + def __repr__(self): + return "And({},{})".format(self.left, self.right) + + def __str__(self): + return '({} and {})'.format(self.left, self.right) + + +class FalseExp(Expression): + + def op(self): + return Operation.FALSE + + def negate(self): + return TRUE + + def __repr__(self): + return "false" + + def __str__(self): + return self.__repr__() + + def __eq__(self, other): + if isinstance(other, FalseExp): + return True + + return False + + def __ne__(self, other): + return not self.__eq__(other) + + +class Or(Expression): + + def __init__(self, left, right): + self.left = left + self.right = right + + def __eq__(self, other): + if id(self) == id(other): + return True + elif other is None or not isinstance(other, Or): + return False + + return self.left == other.left and self.right == other.right + + def __ne__(self, other): + return not self.__eq__(other) + + def op(self): + return Operation.OR + + def negate(self): + from .expressions import Expressions + return Expressions.and_(self.left.negate(), self.right.negate()) # noqa + + def __repr__(self): + return "Or({},{})".format(self.left, self.right) + + def __str__(self): + return '({} or {})'.format(self.left, self.right) + + +class Not(Expression): + + def __init__(self, child): + self.child = child + + def __eq__(self, other): + if id(self) == id(other): + return True + elif other is None or not isinstance(other, Not): + return False + + return self.child == other.child + + def __ne__(self, other): + return not self.__eq__(other) + + def op(self): + return Operation.NOT + + def negate(self): + return self.child + + def __repr__(self): + return "Not({})".format(self.child) + + def __str__(self): + return 'not({})'.format(self.child) + + +class TrueExp(Expression): + + def op(self): + return Operation.TRUE + + def negate(self): + return False + + def __repr__(self): + return "true" + + def __str__(self): + return self.__repr__() + + def __eq__(self, other): + if isinstance(other, TrueExp): + return True + + return False + + def __ne__(self, other): + return not self.__eq__(other) + + +TRUE = TrueExp() +FALSE = FalseExp() diff --git a/iceberg/api/expressions/expressions.py b/iceberg/api/expressions/expressions.py new file mode 100644 index 0000000000..72bb5621f2 --- /dev/null +++ b/iceberg/api/expressions/expressions.py @@ -0,0 +1,262 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +from .expression import (And, + FALSE, + Not, + Operation, + Or, + TRUE) +from .predicate import (Predicate, + UnboundPredicate) +from .reference import NamedReference + + +class Expressions(object): + + @staticmethod + def and_(left, right): + if left == Expressions.always_false() or right == Expressions.always_false(): + return Expressions.always_false() + elif left == Expressions.always_true(): + return right + elif right == Expressions.always_true(): + return left + + return And(left, right) + + @staticmethod + def or_(left, right): + if left == Expressions.always_true() or right == Expressions.always_true(): + return Expressions.always_true() + elif left == Expressions.always_false(): + return right + elif right == Expressions.always_false(): + return left + + return Or(left, right) + + @staticmethod + def not_(child): + if child == Expressions.always_true(): + return Expressions.always_false() + elif child == Expressions.always_false(): + return Expressions.always_true() + elif isinstance(child, Not): + return child.child + + return Not(child) + + @staticmethod + def is_null(name): + return UnboundPredicate(Operation.IS_NULL, Expressions.ref(name)) + + @staticmethod + def not_null(name): + return UnboundPredicate(Operation.NOT_NULL, Expressions.ref(name)) + + @staticmethod + def less_than(name, value): + return UnboundPredicate(Operation.LT, Expressions.ref(name), value) + + @staticmethod + def less_than_or_equal(name, value): + return UnboundPredicate(Operation.LT_EQ, Expressions.ref(name), value) + + @staticmethod + def greater_than(name, value): + return UnboundPredicate(Operation.GT, Expressions.ref(name), value) + + @staticmethod + def greater_than_or_equal(name, value): + return UnboundPredicate(Operation.GT_EQ, Expressions.ref(name), value) + + @staticmethod + def equal(name, value): + return UnboundPredicate(Operation.EQ, Expressions.ref(name), value) + + @staticmethod + def not_equal(name, value): + return UnboundPredicate(Operation.NOT_EQ, Expressions.ref(name), value) + + @staticmethod + def predicate(op, name, value=None, lit=None): + if value is not None and op not in (Operation.IS_NULL, Operation.NOT_NULL): + return UnboundPredicate(op, Expressions.ref(name), value) + elif lit is not None and op not in (Operation.IS_NULL, Operation.NOT_NULL): + return UnboundPredicate(op, Expressions.ref(name), value) + elif op in (Operation.IS_NULL, Operation.NOT_NULL): + if value is not None or lit is not None: + raise RuntimeError("Cannot create {} predicate inclusive a value".format(op)) + return UnboundPredicate(op, Expressions.ref(name)) + else: + raise RuntimeError("Cannot create {} predicate without a value".format(op)) + + @staticmethod + def always_true(): + return TRUE + + @staticmethod + def always_false(): + return FALSE + + @staticmethod + def rewrite_not(expr): + return ExpressionVisitors.visit(expr, RewriteNot.get()) # noqa + + @staticmethod + def ref(name): + return NamedReference(name) + + +class ExpressionVisitors(object): + + @staticmethod + def visit(expr, visitor): + if isinstance(expr, Predicate): + return visitor.predicate(expr) + + if expr.op() == Operation.TRUE: + return visitor.always_true() + elif expr.op() == Operation.FALSE: + return visitor.always_false() + elif expr.op() == Operation.NOT: + return visitor.not_(ExpressionVisitors.visit(expr.child, visitor)) + elif expr.op() == Operation.AND: + return visitor.and_(ExpressionVisitors.visit(expr.left, visitor), + ExpressionVisitors.visit(expr.right, visitor)) + elif expr.op() == Operation.OR: + return visitor.or_(ExpressionVisitors.visit(expr.left, visitor), + ExpressionVisitors.visit(expr.right, visitor)) + else: + raise RuntimeError("Unknown operation: {}".format(expr.op())) + + class ExpressionVisitor(object): + + def always_true(self): + return NotImplementedError() + + def always_false(self): + return NotImplementedError() + + def not_(self, result): + return NotImplementedError() + + def and_(self, left_result, right_result): + return NotImplementedError() + + def or_(self, left_result, right_result): + return NotImplementedError() + + def predicate(self, pred): + return NotImplementedError() + + class BoundExpressionVisitor(ExpressionVisitor): + + def __init__(self): + super(ExpressionVisitors.BoundExpressionVisitor, self).__init__() + + def is_null(self, ref): + return NotImplementedError() + + def not_null(self, ref): + return NotImplementedError() + + def lt(self, ref, lit): + return NotImplementedError() + + def lt_eq(self, ref, lit): + return NotImplementedError() + + def gt(self, ref, lit): + return NotImplementedError() + + def gt_eq(self, ref, lit): + return None + + def eq(self, ref, lit): + return None + + def not_eq(self, ref, lit): + return None + + def in_(self, ref, lit): + return None + + def not_in(self, ref, lit): + return None + + def predicate(self, pred): # noqa + + if isinstance(pred, UnboundPredicate): + raise RuntimeError("Not a bound Predicate: {}".format(pred)) + + if pred.op == Operation.IS_NULL: + return self.is_null(pred.ref) + elif pred.op == Operation.NOT_NULL: + return self.not_null(pred.ref) + elif pred.op == Operation.LT: + return self.lt(pred.ref, pred.lit) + elif pred.op == Operation.LT_EQ: + return self.lt_eq(pred.ref, pred.lit) + elif pred.op == Operation.GT: + return self.gt(pred.ref, pred.lit) + elif pred.op == Operation.GT_EQ: + return self.gt_eq(pred.ref, pred.lit) + elif pred.op == Operation.EQ: + return self.eq(pred.ref, pred.lit) + elif pred.op == Operation.NOT_EQ: + return self.not_eq(pred.ref, pred.lit) + elif pred.op == Operation.IN: + return self.in_(pred.ref, pred.lit) + elif pred.op == Operation.NOT_IN: + return self.not_in(pred.ref, pred.lit) + else: + raise RuntimeError("Unknown operation for Predicate: {}".format(pred.op)) + + +class RewriteNot(ExpressionVisitors.ExpressionVisitor): + __instance = None + + @staticmethod + def get(): + if RewriteNot.__instance is None: + RewriteNot() + return RewriteNot.__instance + + def __init__(self): + if RewriteNot.__instance is not None: + raise Exception("Multiple RewriteNot Types created") + RewriteNot.__instance = self + + def always_true(self): + return Expressions.always_true() + + def always_false(self): + return Expressions.always_false() + + def not_(self, result): + return result.negate() + + def and_(self, left_result, right_result): + return Expressions.and_(left_result, right_result) + + def or_(self, left_result, right_result): + return Expressions.or_(left_result, right_result) + + def predicate(self, pred): + return pred diff --git a/iceberg/api/expressions/inclusive_metrics_evaluator.py b/iceberg/api/expressions/inclusive_metrics_evaluator.py new file mode 100644 index 0000000000..5b943e2ebf --- /dev/null +++ b/iceberg/api/expressions/inclusive_metrics_evaluator.py @@ -0,0 +1,185 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +from .expressions import Expressions, ExpressionVisitors +from ..expressions.binder import Binder +from ..types import Conversions + + +class InclusiveMetricsEvaluator(object): + + def __init__(self, schema, unbound): + self.schema = schema + self.struct = schema.as_struct() + self.expr = Binder.bind(self.struct, Expressions.rewrite_not(unbound)) + self._visitors = None + + def _visitor(self): + if self._visitors is None: + self._visitors = MetricsEvalVisitor(self.expr, self.schema, self.struct) + + return self._visitors + + def eval(self, file): + return self._visitor().eval(file) + + +class MetricsEvalVisitor(ExpressionVisitors.BoundExpressionVisitor): + ROWS_MIGHT_MATCH = True + ROWS_CANNOT_MATCH = False + + def __init__(self, expr, schema, struct): + self.expr = expr + self.value_counts = None + self.null_counts = None + self.lower_bounds = None + self.upper_bounds = None + self.schema = schema + self.struct = struct + + def eval(self, file): + if file.record_count <= 0: + return MetricsEvalVisitor.ROWS_CANNOT_MATCH + + self.value_counts = file.value_counts + self.null_counts = file.null_value_counts + self.lower_bounds = file.lower_bounds + self.upper_bounds = file.upper_bounds + + return ExpressionVisitors.visit(self.expr, self) + + def always_true(self): + return MetricsEvalVisitor.ROWS_MIGHT_MATCH + + def always_false(self): + return MetricsEvalVisitor.ROWS_CANNOT_MATCH + + def not_(self, result): + return not result + + def and_(self, left_result, right_result): + return left_result and right_result + + def or_(self, left_result, right_result): + return left_result or right_result + + def is_null(self, ref): + id = ref.field_id + + if self.struct.field(id=id) is None: + raise RuntimeError("Cannot filter by nested column: %s" % self.schema.find_field(id)) + + if self.null_counts is not None and self.null_counts.get(id, -1) == 0: + return MetricsEvalVisitor.ROWS_CANNOT_MATCH + + return MetricsEvalVisitor.ROWS_MIGHT_MATCH + + def not_null(self, ref): + id = ref.field_id + + if self.struct.field(id=id) is None: + raise RuntimeError("Cannot filter by nested column: %s" % self.schema.find_field(id)) + + if self.value_counts is not None and id in self.value_counts and id in self.null_counts \ + and self.value_counts.get(id) - self.null_counts.get(id) == 0: + return MetricsEvalVisitor.ROWS_CANNOT_MATCH + + return MetricsEvalVisitor.ROWS_MIGHT_MATCH + + def lt(self, ref, lit): + id = ref.field_id + field = self.struct.field(id=id) + + if field is None: + raise RuntimeError("Cannot filter by nested column: %s" % self.schema.find_field(id)) + + if self.lower_bounds is not None and id in self.lower_bounds: + lower = Conversions.from_byte_buffer(field.type, self.lower_bounds.get(id)) + if lower >= lit.value: + return MetricsEvalVisitor.ROWS_CANNOT_MATCH + + return MetricsEvalVisitor.ROWS_MIGHT_MATCH + + def lt_eq(self, ref, lit): + id = ref.field_id + field = self.struct.field(id=id) + + if field is None: + raise RuntimeError("Cannot filter by nested column: %s" % self.schema.find_field(id)) + + if self.lower_bounds is not None and id in self.lower_bounds: + lower = Conversions.from_byte_buffer(field.type, self.lower_bounds.get(id)) + if lower > lit.value: + return MetricsEvalVisitor.ROWS_CANNOT_MATCH + + return MetricsEvalVisitor.ROWS_MIGHT_MATCH + + def gt(self, ref, lit): + id = ref.field_id + field = self.struct.field(id=id) + + if field is None: + raise RuntimeError("Cannot filter by nested column: %s" % self.schema.find_field(id)) + + if self.upper_bounds is not None and id in self.upper_bounds: + upper = Conversions.from_byte_buffer(field.type, self.upper_bounds.get(id)) + if upper <= lit.value: + return MetricsEvalVisitor.ROWS_CANNOT_MATCH + + return MetricsEvalVisitor.ROWS_MIGHT_MATCH + + def gt_eq(self, ref, lit): + id = ref.field_id + field = self.struct.field(id=id) + + if field is None: + raise RuntimeError("Cannot filter by nested column: %s" % self.schema.find_field(id)) + + if self.upper_bounds is not None and id in self.upper_bounds: + upper = Conversions.from_byte_buffer(field.type, self.upper_bounds.get(id)) + if upper < lit.value: + return MetricsEvalVisitor.ROWS_CANNOT_MATCH + + return MetricsEvalVisitor.ROWS_MIGHT_MATCH + + def eq(self, ref, lit): + id = ref.field_id + field = self.struct.field(id=id) + + if field is None: + raise RuntimeError("Cannot filter by nested column: %s" % self.schema.find_field(id)) + + if self.lower_bounds is not None and id in self.lower_bounds: + lower = Conversions.from_byte_buffer(field.type, self.lower_bounds.get(id)) + if lower > lit.value: + return MetricsEvalVisitor.ROWS_CANNOT_MATCH + + if self.upper_bounds is not None and id in self.upper_bounds: + upper = Conversions.from_byte_buffer(field.type, self.upper_bounds.get(id)) + if upper < lit.value: + return MetricsEvalVisitor.ROWS_CANNOT_MATCH + + return MetricsEvalVisitor.ROWS_MIGHT_MATCH + + def not_eq(self, ref, lit): + return MetricsEvalVisitor.ROWS_MIGHT_MATCH + + def in_(self, ref, lit): + return MetricsEvalVisitor.ROWS_MIGHT_MATCH + + def not_in(self, ref, lit): + return MetricsEvalVisitor.ROWS_MIGHT_MATCH diff --git a/iceberg/api/expressions/java_variables/__init__.py b/iceberg/api/expressions/java_variables/__init__.py new file mode 100644 index 0000000000..1ee068e2f7 --- /dev/null +++ b/iceberg/api/expressions/java_variables/__init__.py @@ -0,0 +1,21 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +JAVA_MAX_INT = 2147483647 +JAVA_MIN_INT = -2147483648 +JAVA_MAX_FLOAT = 3.4028235E38 +JAVA_MIN_FLOAT = -3.4028235E38 diff --git a/iceberg/api/expressions/literals.py b/iceberg/api/expressions/literals.py new file mode 100644 index 0000000000..d340172e23 --- /dev/null +++ b/iceberg/api/expressions/literals.py @@ -0,0 +1,585 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +import datetime +from decimal import (Decimal, + ROUND_HALF_UP) +import sys +import uuid + +import pytz + + +from .expression import (FALSE, + TRUE) +from .java_variables import (JAVA_MAX_FLOAT, + JAVA_MIN_FLOAT) +from ..types.type import TypeID + +# to-do un-wind python 2 switches +if sys.version_info >= (3, 0): + unicode = str + long = int + + +class Literals(object): + + EPOCH = datetime.datetime.utcfromtimestamp(0) + EPOCH_DAY = EPOCH.date() + + @staticmethod # noqa: C901 + def from_(value): + if value is None: + raise RuntimeError("Cannot create an expression literal from None") + if isinstance(value, bool): + return BooleanLiteral(value) + elif isinstance(value, int): + if Literal.JAVA_MIN_INT < value < Literal.JAVA_MAX_INT: + return IntegerLiteral(value) + return LongLiteral(value) + elif isinstance(value, float): + if Literal.JAVA_MIN_FLOAT < value < Literal.JAVA_MAX_FLOAT: + return FloatLiteral(value) + return DoubleLiteral(value) + elif isinstance(value, (str, unicode)): + return StringLiteral(value) + elif isinstance(value, uuid.UUID): + return UUIDLiteral(value) + elif isinstance(value, bytearray): + return BinaryLiteral(value) + elif isinstance(value, bytes): + return FixedLiteral(value) + elif isinstance(value, Decimal): + return DecimalLiteral(value) + else: + raise RuntimeError("Unimplemented Type Literal") + + @staticmethod + def above_max(): + return ABOVE_MAX + + @staticmethod + def below_min(): + return BELOW_MIN + + +class Literal(object): + JAVA_MAX_INT = 2147483647 + JAVA_MIN_INT = -2147483648 + JAVA_MAX_FLOAT = 3.4028235E38 + JAVA_MIN_FLOAT = -3.4028235E38 + + @staticmethod # noqa: C901 + def of(value): + + if isinstance(value, bool): + return BooleanLiteral(value) + elif isinstance(value, int): + if value < Literal.JAVA_MIN_INT or value > Literal.JAVA_MAX_INT: + return LongLiteral(value) + return IntegerLiteral(value) + elif isinstance(value, float): + if value < Literal.JAVA_MIN_FLOAT or value > Literal.JAVA_MAX_FLOAT: + return DoubleLiteral(value) + return FloatLiteral(value) + elif isinstance(value, (str, unicode)): + return StringLiteral(value) + elif isinstance(value, uuid.UUID): + return UUIDLiteral(value) + elif isinstance(value, bytes): + return FixedLiteral(value) + elif isinstance(value, bytearray): + return BinaryLiteral(value) + elif isinstance(value, Decimal): + return DecimalLiteral(value) + + def to(self, type): + raise NotImplementedError() + + +class BaseLiteral(Literal): + def __init__(self, value): + self.value = value + + def to(self, type): + raise NotImplementedError() + + def __eq__(self, other): + if id(self) == id(other): + return True + elif other is None or not isinstance(other, BaseLiteral): + return False + + return self.value == other.value + + def __ne__(self, other): + return not self.__eq__(other) + + def __repr__(self): + return "BaseLiteral(%s)" % str(self.value) + + def __str__(self): + return str(self.value) + + +class ComparableLiteral(BaseLiteral): + + def __init__(self, value): + super(ComparableLiteral, self).__init__(value) + + def to(self, type): + raise NotImplementedError() + + def __eq__(self, other): + return self.value == other.value + + def __ne__(self, other): + return not self.__eq__(other) + + def __lt__(self, other): + if self.value is None: + return True + + if other is None or other.value is None: + return False + + return self.value < other.value + + def __gt__(self, other): + if self.value is None: + return False + + if other is None or other.value is None: + return True + + return self.value > other.value + + def __le__(self, other): + if self.value is None: + return True + + if other is None or other.value is None: + return False + + return self.value <= other.value + + def __ge__(self, other): + if self.value is None: + return False + + if other is None or other.value is None: + return True + + return self.value >= other.value + + +class AboveMax(Literal): + def __init__(self): + super(AboveMax, self).__init__() + + def value(self): + raise RuntimeError("AboveMax has no value") + + def to(self, type): + raise RuntimeError("Cannot change the type of AboveMax") + + def __str__(self): + return "aboveMax" + + +class BelowMin(Literal): + def __init__(self): + super(BelowMin, self).__init__() + + def value(self): + raise RuntimeError("BelowMin has no value") + + def to(self, type): + raise RuntimeError("Cannot change the type of BelowMin") + + def __str__(self): + return "belowMin" + + +class BooleanLiteral(ComparableLiteral): + + def __init__(self, value): + super(BooleanLiteral, self).__init__(value) + + def to(self, type_var): + if type_var.type_id == TypeID.BOOLEAN: + return self + + +class IntegerLiteral(ComparableLiteral): + + def __init__(self, value): + super(IntegerLiteral, self).__init__(value) + + def to(self, type_var): + if type_var.type_id == TypeID.INTEGER: + return self + elif type_var.type_id == TypeID.LONG: + return LongLiteral(self.value) + elif type_var.type_id == TypeID.FLOAT: + return FloatLiteral(float(self.value)) + elif type_var.type_id == TypeID.DOUBLE: + return DoubleLiteral(float(self.value)) + elif type_var.type_id == TypeID.DATE: + return DateLiteral(self.value) + elif type_var.type_id == TypeID.DECIMAL: + if type_var.scale == 0: + return DecimalLiteral(Decimal(self.value)) + else: + return DecimalLiteral(Decimal(self.value) + .quantize(Decimal("." + "".join(["0" for i in range(1, type_var.scale)]) + "1"), + rounding=ROUND_HALF_UP)) + + +class LongLiteral(ComparableLiteral): + + def __init__(self, value): + super(LongLiteral, self).__init__(value) + + def to(self, type_var): # noqa: C901 + if type_var.type_id == TypeID.INTEGER: + if Literal.JAVA_MAX_INT < self.value: + return ABOVE_MAX + elif Literal.JAVA_MIN_INT > self.value: + return BELOW_MIN + + return IntegerLiteral(self.value) + elif type_var.type_id == TypeID.LONG: + return self + elif type_var.type_id == TypeID.FLOAT: + return FloatLiteral(float(self.value)) + elif type_var.type_id == TypeID.DOUBLE: + return DoubleLiteral(float(self.value)) + elif type_var.type_id == TypeID.TIME: + return TimeLiteral(self.value) + elif type_var.type_id == TypeID.TIMESTAMP: + return TimestampLiteral(self.value) + elif type_var.type_id == TypeID.DECIMAL: + if type_var.scale == 0: + return DecimalLiteral(Decimal(self.value)) + else: + return DecimalLiteral(Decimal(self.value) + .quantize(Decimal("." + "".join(["0" for i in range(1, type_var.scale)]) + "1"), + rounding=ROUND_HALF_UP)) + + +class FloatLiteral(ComparableLiteral): + + def __init__(self, value): + super(FloatLiteral, self).__init__(value) + + def to(self, type_var): + if type_var.type_id == TypeID.FLOAT: + return self + elif type_var.type_id == TypeID.DOUBLE: + return DoubleLiteral(self.value) + elif type_var.type_id == TypeID.DECIMAL: + if type_var.scale == 0: + return DecimalLiteral(Decimal(self.value) + .quantize(Decimal('1.'), + rounding=ROUND_HALF_UP)) + else: + return DecimalLiteral(Decimal(self.value) + .quantize(Decimal("." + "".join(["0" for i in range(1, type_var.scale)]) + "1"), + rounding=ROUND_HALF_UP)) + + +class DoubleLiteral(ComparableLiteral): + + def __init__(self, value): + super(DoubleLiteral, self).__init__(value) + + def to(self, type_var): + if type_var.type_id == TypeID.FLOAT: + if JAVA_MAX_FLOAT < self.value: + return ABOVE_MAX + elif JAVA_MIN_FLOAT > self.value: + return BELOW_MIN + + return FloatLiteral(self.value) + elif type_var.type_id == TypeID.DOUBLE: + return self + elif type_var.type_id == TypeID.DECIMAL: + if type_var.scale == 0: + return DecimalLiteral(Decimal(self.value) + .quantize(Decimal('1.'), + rounding=ROUND_HALF_UP)) + else: + return DecimalLiteral(Decimal(self.value) + .quantize(Decimal("." + "".join(["0" for i in range(1, type_var.scale)]) + "1"), + rounding=ROUND_HALF_UP)) + + +class DateLiteral(ComparableLiteral): + + def __init__(self, value): + super(DateLiteral, self).__init__(value) + + def to(self, type_var): + if type_var.type_id == TypeID.DATE: + return self + + +class TimeLiteral(ComparableLiteral): + + def __init__(self, value): + super(TimeLiteral, self).__init__(value) + + def to(self, type_var): + if type_var.type_id == TypeID.TIME: + return self + + +class TimestampLiteral(ComparableLiteral): + + def __init__(self, value): + super(TimestampLiteral, self).__init__(value) + + def to(self, type_var): + if type_var.type_id == TypeID.TIMESTAMP: + return self + elif type_var.type_id == TypeID.DATE: + return DateLiteral((datetime.datetime.fromtimestamp(self.value / 1000000) - Literals.EPOCH).days) + + +class DecimalLiteral(ComparableLiteral): + + def __init__(self, value): + super(DecimalLiteral, self).__init__(value) + + def to(self, type_var): + if type_var.type_id == TypeID.DECIMAL and type_var.scale == abs(self.value.as_tuple().exponent): + return self + + +class StringLiteral(BaseLiteral): + def __init__(self, value): + super(StringLiteral, self).__init__(value) + + def to(self, type_var): # noqa: C901 + import dateutil.parser + if type_var.type_id == TypeID.DATE: + return DateLiteral((dateutil.parser.parse(self.value) - Literals.EPOCH).days) + elif type_var.type_id == TypeID.TIME: + return TimeLiteral( + int((dateutil.parser.parse(Literals.EPOCH.strftime("%Y-%m-%d ") + self.value) - Literals.EPOCH) + .total_seconds() * 1000000)) + elif type_var.type_id == TypeID.TIMESTAMP: + timestamp = dateutil.parser.parse(self.value) + EPOCH = Literals.EPOCH + if bool(timestamp.tzinfo) != bool(type_var.adjust_to_utc): + raise RuntimeError("Cannot convert to %s when string is: %s" % (type_var, self.value)) + + if timestamp.tzinfo is not None: + EPOCH = EPOCH.replace(tzinfo=pytz.UTC) + + return TimestampLiteral(int((timestamp - EPOCH).total_seconds() * 1000000)) + elif type_var.type_id == TypeID.STRING: + return self + elif type_var.type_id == TypeID.UUID: + return UUIDLiteral(uuid.UUID(self.value)) + elif type_var.type_id == TypeID.DECIMAL: + dec_val = Decimal(str(self.value)) + if abs(dec_val.as_tuple().exponent) == type_var.scale: + if type_var.scale == 0: + return DecimalLiteral(Decimal(str(self.value)) + .quantize(Decimal('1.'), + rounding=ROUND_HALF_UP)) + else: + return DecimalLiteral(Decimal(str(self.value)) + .quantize(Decimal("." + "".join(["0" for i in range(1, type_var.scale)]) + "1"), + rounding=ROUND_HALF_UP)) + + def __eq__(self, other): + if id(self) == id(other): + return True + + if other is None or not isinstance(other, StringLiteral): + return False + + return self.value == other.value + + def __ne__(self, other): + return not self.__eq__(other) + + def __lt__(self, other): + if other is None: + return False + + return self.value < other.value + + def __gt__(self, other): + if other is None: + return True + + return self.value > other.value + + def __le__(self, other): + if other is None: + return False + + return self.value <= other.value + + def __ge__(self, other): + if other is None: + return True + + return self.value >= other.value + + def __str__(self): + return '"' + self.value + '"' + + +class UUIDLiteral(ComparableLiteral): + def __init__(self, value): + super(UUIDLiteral, self).__init__(value) + + def to(self, type_var): + if type_var.type_id == TypeID.UUID: + return self + + +class FixedLiteral(BaseLiteral): + def __init__(self, value): + super(FixedLiteral, self).__init__(value) + + def to(self, type_var): + if type_var.type_id == TypeID.FIXED: + if len(self.value) == type_var.length: + return self + elif type_var.type_id == TypeID.BINARY: + return BinaryLiteral(self.value) + + def write_replace(self): + return FixedLiteralProxy(self.value) + + def __eq__(self, other): + return self.value == other.value + + def __ne__(self, other): + return not self.__eq__(other) + + def __lt__(self, other): + if other is None: + return False + + return self.value < other.value + + def __gt__(self, other): + if other is None: + return True + + return self.value > other.value + + def __le__(self, other): + if other is None: + return False + + return self.value <= other.value + + def __ge__(self, other): + if other is None: + return True + + return self.value >= other.value + + +class BinaryLiteral(BaseLiteral): + def __init__(self, value): + super(BinaryLiteral, self).__init__(value) + + def to(self, type_var): + if type_var.type_id == TypeID.FIXED: + if type_var.length == len(self.value): + return FixedLiteral(self.value) + return None + elif type_var.type_id == TypeID.BINARY: + return self + + def write_replace(self): + return BinaryLiteralProxy(self.value) + + def __eq__(self, other): + return self.value == other.value + + def __ne__(self, other): + return not self.__eq__(other) + + def __lt__(self, other): + if other is None: + return False + + return self.value < other.value + + def __gt__(self, other): + if other is None: + return True + + return self.value > other.value + + def __le__(self, other): + if other is None: + return False + + return self.value <= other.value + + def __ge__(self, other): + if other is None: + return True + + return self.value >= other.value + + +class FixedLiteralProxy(object): + + def __init__(self, buffer=None): + if buffer is not None: + self.bytes = list(buffer) + + def read_resolve(self): + return FixedLiteral(self.bytes) + + +class ConstantExpressionProxy(object): + + def __init__(self, true_or_false=None): + if true_or_false is not None: + self.true_or_false = true_or_false + + def read_resolve(self): + if self.true_or_false: + return TRUE + else: + return FALSE + + +class BinaryLiteralProxy(FixedLiteralProxy): + + def __init__(self, buffer=None): + super(BinaryLiteralProxy, self).__init__(buffer) + + def read_resolve(self): + return BinaryLiteral(self.bytes) + + +ABOVE_MAX = AboveMax() +BELOW_MIN = BelowMin() diff --git a/iceberg/api/expressions/predicate.py b/iceberg/api/expressions/predicate.py new file mode 100644 index 0000000000..55625dcc7e --- /dev/null +++ b/iceberg/api/expressions/predicate.py @@ -0,0 +1,137 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +from iceberg.exceptions import ValidationException + +from .expression import (Expression, + FALSE, + Operation, + TRUE) +from .literals import (Literal, + Literals) +from .reference import BoundReference + + +class Predicate(Expression): + + def __init__(self, op, ref, lit): + self.op = op + self.ref = ref + self.lit = lit + + def __eq__(self, other): + if id(self) == id(other): + return True + elif other is None or not isinstance(other, Predicate): + return False + + return self.op == other.op and self.ref == other.ref and self.lit == other.lit + + def __ne__(self, other): + return not self.__eq__(other) + + def __repr__(self): + return "Predicate({},{},{})".format(self.op, self.ref, self.lit) + + def __str__(self): + if self.op == Operation.IS_NULL: + return "is_null({})".format(self.ref) + elif self.op == Operation.NOT_NULL: + return "not_null({})".format(self.ref) + elif self.op == Operation.LT: + return "less_than({})".format(self.ref) + elif self.op == Operation.LT_EQ: + return "less_than_equal({})".format(self.ref) + elif self.op == Operation.GT: + return "greater_than({})".format(self.ref) + elif self.op == Operation.GT_EQ: + return "greater_than_equal({})".format(self.ref) + elif self.op == Operation.EQ: + return "equal({})".format(self.ref) + elif self.op == Operation.NOT_EQ: + return "not_equal({})".format(self.ref) + else: + return "invalid predicate: operation = {}".format(self.op) + + +class BoundPredicate(Predicate): + + def __init__(self, op, ref, lit=None): + super(BoundPredicate, self).__init__(op, ref, lit) + + def negate(self): + return BoundPredicate(self.op.negate(), self.ref, self.lit) + + +class UnboundPredicate(Predicate): + + def __init__(self, op, named_ref, value=None, lit=None): + if value is None and lit is None: + super(UnboundPredicate, self).__init__(op, named_ref, None) + if isinstance(value, Literal): + lit = value + value = None + if value is not None: + super(UnboundPredicate, self).__init__(op, named_ref, Literals.from_(value)) + elif lit is not None: + super(UnboundPredicate, self).__init__(op, named_ref, lit) + + def negate(self): + return UnboundPredicate(self.op.negate(), self.ref, self.lit) + + def bind(self, struct): # noqa: C901 + field = struct.field(self.ref.name) + ValidationException.check(field is not None, + "Cannot find field '%s' in struct %s", (self.ref.name, struct)) + + if self.lit is None: + if self.op == Operation.IS_NULL: + if field.is_required: + return FALSE + return BoundPredicate(Operation.IS_NULL, BoundReference(struct, field.field_id)) + elif self.op == Operation.NOT_NULL: + if field.is_required: + return TRUE + return BoundPredicate(Operation.NOT_NULL, BoundReference(struct, field.field_id)) + else: + raise ValidationException("Operation must be IS_NULL or NOT_NULL", None) + + literal = self.lit.to(field.type) + if literal is None: + raise ValidationException("Invalid value for comparison inclusive type %s: %s (%s)", + (field.type, self.lit.value, type(self.lit.value))) + elif literal == Literals.above_max(): + if self.op in (Operation.LT, + Operation.LT_EQ, + Operation.NOT_EQ): + return TRUE + elif self.op in (Operation.GT, + Operation.GT_EQ, + Operation.EQ): + return FALSE + + elif literal == Literals.below_min(): + if self.op in (Operation.LT, + Operation.LT_EQ, + Operation.NOT_EQ): + return FALSE + elif self.op in (Operation.GT, + Operation.GT_EQ, + Operation.EQ): + return TRUE + + return BoundPredicate(self.op, BoundReference(struct, field.field_id), literal) diff --git a/iceberg/api/expressions/reference.py b/iceberg/api/expressions/reference.py new file mode 100644 index 0000000000..1ceeb3a150 --- /dev/null +++ b/iceberg/api/expressions/reference.py @@ -0,0 +1,84 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +from iceberg.exceptions import ValidationException + + +class Reference(object): + pass + + +class BoundReference(Reference): + + def __init__(self, struct, field_id): + self.field_id = field_id + self.pos = self.find(field_id, struct) + self._type = struct.fields[self.pos].type + + def __eq__(self, other): + if id(self) == id(other): + return True + elif other is None or not isinstance(other, BoundReference): + return False + + return self.field_id == other.field_id and self.pos == other.pos and self._type == other._type + + def __ne__(self, other): + return not self.__eq__(other) + + def find(self, field_id, struct): + fields = struct.fields + for i, field in enumerate(fields): + if field.field_id == self.field_id: + return i + + raise ValidationException("Cannot find top-level field id %d in struct: %s", (field_id, struct)) + + def get(self, struct): + return struct.get(self.pos) + + def __str__(self): + return "ref(id={id}, pos={pos}, type={_type})".format(id=self.field_id, + pos=self.pos, + _type=self._type) + + +class NamedReference(Reference): + + def __init__(self, name): + super(NamedReference, self).__init__() + if name is None: + raise RuntimeError("Name cannot be null") + + self.name = name + + def __eq__(self, other): + if id(self) == id(other): + return True + elif other is None or not isinstance(other, NamedReference): + return False + + return self.name == other.name + + def __ne__(self, other): + return not self.__eq__(other) + + def __repr__(self): + return "NamedReference({})".format(self.name) + + def __str__(self): + return 'ref(name="{}")'.format(self.name) diff --git a/iceberg/api/expressions/strict_metrics_evaluator.py b/iceberg/api/expressions/strict_metrics_evaluator.py new file mode 100644 index 0000000000..1997bdd321 --- /dev/null +++ b/iceberg/api/expressions/strict_metrics_evaluator.py @@ -0,0 +1,215 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +from .expressions import Expressions, ExpressionVisitors +from ..expressions.binder import Binder +from ..types import Conversions + + +class StrictMetricsEvaluator(object): + + def __init__(self, schema, unbound): + self.schema = schema + self.struct = schema.as_struct() + self.expr = Binder.bind(self.struct, Expressions.rewrite_not(unbound)) + self._visitors = None + + def _visitor(self): + if self._visitors is None: + self._visitors = StrictMetricsEvaluator.MetricsEvalVisitor(self.expr, self.schema, self.struct) + + return self._visitors + + def eval(self, file): + return self._visitor().eval(file) + + class MetricsEvalVisitor(ExpressionVisitors.BoundExpressionVisitor): + ROWS_MUST_MATCH = True + ROWS_MIGHT_NOT_MATCH = False + + def __init__(self, expr, schema, struct): + self.expr = expr + self.schema = schema + self.struct = struct + self.value_counts = None + self.null_counts = None + self.lower_bounds = None + self.upper_bounds = None + + def eval(self, file): + if file.record_count <= 0: + return StrictMetricsEvaluator.MetricsEvalVisitor.ROWS_MUST_MATCH + + self.value_counts = file.value_counts + self.null_counts = file.null_value_counts + self.lower_bounds = file.lower_bounds + self.upper_bounds = file.upper_bounds + + return ExpressionVisitors.visit(self.expr, self) + + def always_true(self): + return StrictMetricsEvaluator.MetricsEvalVisitor.ROWS_MUST_MATCH + + def always_false(self): + return StrictMetricsEvaluator.MetricsEvalVisitor.ROWS_MIGHT_NOT_MATCH + + def not_(self, result): + return not result + + def and_(self, left_result, right_result): + return left_result and right_result + + def or_(self, left_result, right_result): + return left_result or right_result + + def is_null(self, ref): + id = ref.field_id + if self.struct.field(id=id) is None: + raise RuntimeError("Cannot filter by nested column: %s" % self.schema.find_field(id)) + + if self.value_counts is not None and self.value_counts.get(id) is not None \ + and self.null_counts is not None and self.null_counts.get(id) is not None \ + and self.value_counts.get(id) - self.null_counts.get(id) == 0: + return StrictMetricsEvaluator.MetricsEvalVisitor.ROWS_MUST_MATCH + + return StrictMetricsEvaluator.MetricsEvalVisitor.ROWS_MIGHT_NOT_MATCH + + def not_null(self, ref): + id = ref.field_id + if self.struct.field(id=id) is None: + raise RuntimeError("Cannot filter by nested column: %s" % self.schema.find_field(id)) + + if self.null_counts is not None and self.null_counts.get(id, -1) == 0: + return StrictMetricsEvaluator.MetricsEvalVisitor.ROWS_MUST_MATCH + + return StrictMetricsEvaluator.MetricsEvalVisitor.ROWS_MIGHT_NOT_MATCH + + def lt(self, ref, lit): + # Rows must match when: <----------Min----Max---X-------> + id = ref.field_id + + field = self.struct.field(id=id) + + if field is None: + raise RuntimeError("Cannot filter by nested column: %s" % self.schema.find_field(id)) + + if self.upper_bounds is not None and id in self.upper_bounds: + upper = Conversions.from_byte_buffer(field.type, self.upper_bounds.get(id)) + if upper < lit.value: + return StrictMetricsEvaluator.MetricsEvalVisitor.ROWS_MUST_MATCH + + return StrictMetricsEvaluator.MetricsEvalVisitor.ROWS_MIGHT_NOT_MATCH + + def lt_eq(self, ref, lit): + # Rows must match when: <----------Min----Max---X-------> + id = ref.field_id + + field = self.struct.field(id=id) + + if field is None: + raise RuntimeError("Cannot filter by nested column: %s" % self.schema.find_field(id)) + + if self.upper_bounds is not None and id in self.upper_bounds: + upper = Conversions.from_byte_buffer(field.type, self.upper_bounds.get(id)) + if upper <= lit.value: + return StrictMetricsEvaluator.MetricsEvalVisitor.ROWS_MUST_MATCH + + return StrictMetricsEvaluator.MetricsEvalVisitor.ROWS_MIGHT_NOT_MATCH + + def gt(self, ref, lit): + # Rows must match when: <-------X---Min----Max----------> + id = ref.field_id + + field = self.struct.field(id=id) + + if field is None: + raise RuntimeError("Cannot filter by nested column: %s" % self.schema.find_field(id)) + + if self.lower_bounds is not None and id in self.lower_bounds: + lower = Conversions.from_byte_buffer(field.type, self.lower_bounds.get(id)) + if lower > lit.value: + return StrictMetricsEvaluator.MetricsEvalVisitor.ROWS_MUST_MATCH + + return StrictMetricsEvaluator.MetricsEvalVisitor.ROWS_MIGHT_NOT_MATCH + + def gt_eq(self, ref, lit): + # Rows must match when: <-------X---Min----Max----------> + id = ref.field_id + + field = self.struct.field(id=id) + + if field is None: + raise RuntimeError("Cannot filter by nested column: %s" % self.schema.find_field(id)) + + if self.lower_bounds is not None and id in self.lower_bounds: + lower = Conversions.from_byte_buffer(field.type, self.lower_bounds.get(id)) + if lower >= lit.value: + return StrictMetricsEvaluator.MetricsEvalVisitor.ROWS_MUST_MATCH + + return StrictMetricsEvaluator.MetricsEvalVisitor.ROWS_MIGHT_NOT_MATCH + + def eq(self, ref, lit): + # Rows must match when Min == X == Max + id = ref.field_id + + field = self.struct.field(id=id) + + if field is None: + raise RuntimeError("Cannot filter by nested column: %s" % self.schema.find_field(id)) + + if self.lower_bounds is not None and id in self.lower_bounds \ + and self.upper_bounds is not None and id in self.upper_bounds: + lower = Conversions.from_byte_buffer(field.type, self.lower_bounds.get(id)) + if lower != lit.value: + return StrictMetricsEvaluator.MetricsEvalVisitor.ROWS_MIGHT_NOT_MATCH + + upper = Conversions.from_byte_buffer(field.type, self.upper_bounds.get(id)) + + if upper != lit.value: + return StrictMetricsEvaluator.MetricsEvalVisitor.ROWS_MIGHT_NOT_MATCH + + return StrictMetricsEvaluator.MetricsEvalVisitor.ROWS_MUST_MATCH + + return StrictMetricsEvaluator.MetricsEvalVisitor.ROWS_MIGHT_NOT_MATCH + + def not_eq(self, ref, lit): + # Rows must match when X < Min or Max < X because it is not in the range + id = ref.field_id + + field = self.struct.field(id=id) + + if field is None: + raise RuntimeError("Cannot filter by nested column: %s" % self.schema.find_field(id)) + + if self.lower_bounds is not None and id in self.lower_bounds: + lower = Conversions.from_byte_buffer(field.type, self.lower_bounds.get(id)) + if lower > lit.value: + return StrictMetricsEvaluator.MetricsEvalVisitor.ROWS_MUST_MATCH + + if self.upper_bounds is not None and id in self.upper_bounds: + upper = Conversions.from_byte_buffer(field.type, self.upper_bounds.get(id)) + + if upper < lit.value: + return StrictMetricsEvaluator.MetricsEvalVisitor.ROWS_MUST_MATCH + + return StrictMetricsEvaluator.MetricsEvalVisitor.ROWS_MIGHT_NOT_MATCH + + def in_(self, ref, lit): + return StrictMetricsEvaluator.MetricsEvalVisitor.ROWS_MIGHT_NOT_MATCH + + def not_in(self, ref, lit): + return StrictMetricsEvaluator.MetricsEvalVisitor.ROWS_MIGHT_NOT_MATCH diff --git a/iceberg/api/file_format.py b/iceberg/api/file_format.py new file mode 100644 index 0000000000..0665e938b0 --- /dev/null +++ b/iceberg/api/file_format.py @@ -0,0 +1,42 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +from enum import Enum, unique + + +@unique +class FileFormat(Enum): + + ORC = "orc" + PARQUET = "parquet" + AVRO = "avro" + + def add_extension(self, filename): + if filename.endswith(self.value): + return filename + else: + return filename + "." + self.value + + @staticmethod + def from_file_name(filename): + last_index_of = filename.rfind('.') + if last_index_of < 0: + return None + ext = filename[last_index_of + 1:] + for fmt in FileFormat: + if ext == fmt.value: + return fmt diff --git a/iceberg/api/file_scan_task.py b/iceberg/api/file_scan_task.py new file mode 100644 index 0000000000..20865f0c74 --- /dev/null +++ b/iceberg/api/file_scan_task.py @@ -0,0 +1,42 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +from .scan_task import ScanTask + + +class FileScanTask(ScanTask): + + def file(self): + raise NotImplementedError() + + def spec(self): + raise NotImplementedError() + + def start(self): + raise NotImplementedError() + + def length(self): + raise NotImplementedError() + + def residual(self): + raise NotImplementedError() + + def is_file_scan_task(self): + return True + + def as_file_scan_task(self): + return self diff --git a/iceberg/api/files.py b/iceberg/api/files.py new file mode 100644 index 0000000000..6ec90bed3c --- /dev/null +++ b/iceberg/api/files.py @@ -0,0 +1,126 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +import gzip +import os + +from iceberg.exceptions import AlreadyExistsException + +from .expressions import JAVA_MAX_INT +from .io import (InputFile, + OutputFile, + PositionOutputStream, + SeekableInputStream) + + +class Files(object): + + @staticmethod + def local_output(file): + return LocalOutputFile(file) + + @staticmethod + def local_input(file): + return LocalInputFile(file) + + +class LocalOutputFile(OutputFile): + + def __init__(self, file): + self.file = file + + def create(self): + if os.path.exists(self.file): + raise AlreadyExistsException("File already exists: %s" % self.file) + + return PositionOutputStream(open(self.file, "rw")) + + def create_or_overwrite(self): + if os.path.exists(self.file): + if not os.remove(self.file.name): + raise RuntimeError("File deleting file: %s" % self.file) + + return self.create() + + def location(self): + return self.file + + def __str__(self): + return self.location() + + +class LocalInputFile(InputFile): + + def __init__(self, file): + self.file = file + + def get_length(self): + return os.path.getsize(self.file) + + def new_stream(self, gzipped=False): + with open(self.file, "rb") as fo: + if gzipped: + fo = gzip.GzipFile(fileobj=fo) + return fo + + def location(self): + return self.file + + def __str__(self): + return self.location() + + +class SeekableFileInputStream(SeekableInputStream): + + def __init__(self, stream): + self.stream = stream + + def get_pos(self): + return self.stream.tell() + + def seek(self, new_pos): + return self.stream.seek(new_pos) + + def read(self, b=None, off=None, read_len=None): + if b is None and off is None and read_len is None: + return None, self.stream.read() + if b is None: + raise RuntimeError("supplied byte field is None") + + if off is None and read_len is None: + new_b = self.stream.read(b.length) + return len(new_b), new_b + + if off is not None and read_len is None or off is None and read_len is not None: + raise RuntimeError("Invalid args: read_len(%s), off(%s)" % (read_len, off)) + + if read_len < 0 or off < 0 or (len(b) - off < read_len): + raise RuntimeError("Invalid args: read_len(%s), off(%s), len_b_offset(%s)" % (read_len, off, len(b) - off)) + + new_b = bytes(self.stream.read(read_len), "utf8") + + if off > 0: + new_b = b[0:off] + new_b + if off + read_len < len(b): + new_b = new_b + b[off + read_len:] + return read_len, new_b + + def skip(self, n): + if n > JAVA_MAX_INT: + return self.stream.seek(JAVA_MAX_INT) + else: + return self.stream.seek(n) diff --git a/iceberg/api/filterable.py b/iceberg/api/filterable.py new file mode 100644 index 0000000000..beb8d48767 --- /dev/null +++ b/iceberg/api/filterable.py @@ -0,0 +1,32 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + + +class Filterable(object): + ALL_COLUMNS = ("*",) + + def select(self, columns): + raise NotImplementedError() + + def filter_partitions(self, expr): + raise NotImplementedError() + + def filter_rows(self, expr): + raise NotImplementedError() + + def iterator(self): + raise NotImplementedError() diff --git a/iceberg/api/filtered_snapshot.py b/iceberg/api/filtered_snapshot.py new file mode 100644 index 0000000000..055480a283 --- /dev/null +++ b/iceberg/api/filtered_snapshot.py @@ -0,0 +1,40 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +from .expressions import Expressions +from .filterable import Filterable + + +class FilteredSnapshot(Filterable): + + def __init__(self, snapshot, part_filter, row_filter, columns): + self.snapshot = snapshot + self.part_filter = part_filter + self.row_filter = row_filter + self.columns = columns + + def select(self, columns): + return FilteredSnapshot(self.snapshot, self.part_filter, self.row_filter, columns) + + def filter_partitions(self, expr): + return FilteredSnapshot(self.snapshot, Expressions.and_(self.part_filter, expr), self.row_filter, self.columns) + + def filter_rows(self, expr): + return FilteredSnapshot(self.snapshot, self.part_filter, Expressions.and_(self.row_filter, expr), self.columns) + + def iterator(self): + return self.snapshot.iterator(self.part_filter, self.row_filter, self.columns) diff --git a/iceberg/api/io/__init__.py b/iceberg/api/io/__init__.py new file mode 100644 index 0000000000..0fdac354b9 --- /dev/null +++ b/iceberg/api/io/__init__.py @@ -0,0 +1,30 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +__all__ = ["CloseableGroup", + "FileAppender", + "InputFile", + "OutputFile", + "PositionOutputStream", + "SeekableInputStream"] + +from .closeable_group import CloseableGroup +from .file_appender import FileAppender +from .input_file import InputFile +from .output_file import OutputFile +from .position_output_stream import PositionOutputStream +from .seekable_input_stream import SeekableInputStream diff --git a/iceberg/api/io/closeable_group.py b/iceberg/api/io/closeable_group.py new file mode 100644 index 0000000000..4abe1b2fd1 --- /dev/null +++ b/iceberg/api/io/closeable_group.py @@ -0,0 +1,31 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + + +class CloseableGroup(object): + + def __init__(self): + self.closeables = list() + + def add_closeable(self, closeable): + self.closeables.append(closeable) + + def close(self): + while self.closeables: + to_close = self.closeables.pop(0) + if to_close is not None: + to_close.close() diff --git a/iceberg/api/io/closeable_iterable.py b/iceberg/api/io/closeable_iterable.py new file mode 100644 index 0000000000..1d62e2db41 --- /dev/null +++ b/iceberg/api/io/closeable_iterable.py @@ -0,0 +1,30 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +import collections + + +class CloseableIterable(collections.Iterator): + + def next(self): + raise NotImplementedError() + + def __next__(self): + return self.next() + + def close(self): + raise NotImplementedError() diff --git a/iceberg/api/io/delegating_input_stream.py b/iceberg/api/io/delegating_input_stream.py new file mode 100644 index 0000000000..40ebe34dc2 --- /dev/null +++ b/iceberg/api/io/delegating_input_stream.py @@ -0,0 +1,22 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + + +class DelegatingInputStream(object): + + def get_delegate(self): + raise NotImplementedError() diff --git a/iceberg/api/io/delegating_output_stream.py b/iceberg/api/io/delegating_output_stream.py new file mode 100644 index 0000000000..84064d0f3b --- /dev/null +++ b/iceberg/api/io/delegating_output_stream.py @@ -0,0 +1,22 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + + +class DelegatingOutputStream(object): + + def get_delegate(self): + raise NotImplementedError() diff --git a/iceberg/api/io/file_appender.py b/iceberg/api/io/file_appender.py new file mode 100644 index 0000000000..fabb05d068 --- /dev/null +++ b/iceberg/api/io/file_appender.py @@ -0,0 +1,29 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + + +class FileAppender(object): + + def add(self, d): + raise NotImplementedError() + + def add_all(self, values): + for value in values: + self.add(value) + + def metrics(self): + raise NotImplementedError() diff --git a/iceberg/api/io/input_file.py b/iceberg/api/io/input_file.py new file mode 100644 index 0000000000..67277f79f5 --- /dev/null +++ b/iceberg/api/io/input_file.py @@ -0,0 +1,28 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + + +class InputFile(object): + + def get_length(self): + raise NotImplementedError() + + def new_stream(self): + raise NotImplementedError() + + def location(self): + raise NotImplementedError() diff --git a/iceberg/api/io/output_file.py b/iceberg/api/io/output_file.py new file mode 100644 index 0000000000..f091459b82 --- /dev/null +++ b/iceberg/api/io/output_file.py @@ -0,0 +1,28 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + + +class OutputFile(object): + + def create(self): + raise NotImplementedError() + + def create_or_overwrite(self): + raise NotImplementedError() + + def location(self): + raise NotImplementedError() diff --git a/iceberg/api/io/position_output_stream.py b/iceberg/api/io/position_output_stream.py new file mode 100644 index 0000000000..fb1e758043 --- /dev/null +++ b/iceberg/api/io/position_output_stream.py @@ -0,0 +1,32 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + + +class PositionOutputStream(object): + + # OutputStream methods + def close(self): + raise NotImplementedError() + + def flush(self): + raise NotImplementedError() + + def write(self, b, off=None, len=None): + raise NotImplementedError() + + def get_pos(self): + raise NotImplementedError() diff --git a/iceberg/api/io/seekable_input_stream.py b/iceberg/api/io/seekable_input_stream.py new file mode 100644 index 0000000000..60808a8e38 --- /dev/null +++ b/iceberg/api/io/seekable_input_stream.py @@ -0,0 +1,46 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + + +class SeekableInputStream(object): + + def available(self): + raise NotImplementedError() + + def close(self): + raise NotImplementedError() + + def mark(self, read_limit): + raise NotImplementedError() + + def mark_supported(self): + raise NotImplementedError() + + def read(self, b=None, off=None, len=None): + raise NotImplementedError() + + def reset(self): + raise NotImplementedError() + + def skip(self, n): + raise NotImplementedError() + + def get_pos(self): + raise NotImplementedError() + + def seek(self, new_pos): + raise NotImplementedError() diff --git a/iceberg/api/manifest_file.py b/iceberg/api/manifest_file.py new file mode 100644 index 0000000000..e8fa5c9900 --- /dev/null +++ b/iceberg/api/manifest_file.py @@ -0,0 +1,74 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +from .schema import Schema +from .types import (BinaryType, + BooleanType, + IntegerType, + ListType, + LongType, + NestedField, + StringType, + StructType) + + +class ManifestFile(object): + SCHEMA = Schema(NestedField.required(500, "manifest_path", StringType.get()), + NestedField.required(501, "manifest_length", LongType.get()), + NestedField.required(502, "partition_spec_id", IntegerType.get()), + NestedField.optional(503, "added_snapshot_id", LongType.get()), + NestedField.optional(504, "added_data_files_count", IntegerType.get()), + NestedField.optional(505, "existing_data_files_count", IntegerType.get()), + NestedField.optional(506, "deleted_data_files_count", IntegerType.get()), + NestedField + .optional(507, "partitions", + ListType.of_required(508, StructType.of([NestedField.required(509, + "contains_null", + BooleanType.get()), + NestedField.optional(510, + "lower_bound", + BinaryType.get()), + NestedField.optional(511, + "upper_bound", + BinaryType.get())])))) + + @staticmethod + def schema(): + return ManifestFile.SCHEMA + + def copy(self): + raise NotImplementedError() + + +class PartitionFieldSummary(object): + TYPE = ManifestFile.schema().find_type("partitions").as_list_type().element_type.as_struct_type() + + @staticmethod + def get_type(): + return PartitionFieldSummary.TYPE + + def contains_null(self): + raise NotImplementedError() + + def lower_bound(self): + raise NotImplementedError() + + def upper_bound(self): + raise NotImplementedError() + + def copy(self): + raise NotImplementedError() diff --git a/iceberg/api/metrics.py b/iceberg/api/metrics.py new file mode 100644 index 0000000000..231388a256 --- /dev/null +++ b/iceberg/api/metrics.py @@ -0,0 +1,30 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + + +class Metrics(object): + + def __init__(self, row_count=None, + column_sizes=None, + value_counts=None, null_value_counts=None, + lower_bounds=None, upper_bounds=None): + self.row_count = row_count + self.column_sizes = column_sizes + self.value_counts = value_counts + self.null_value_counts = null_value_counts + self.lower_bounds = lower_bounds + self.upper_bounds = upper_bounds diff --git a/iceberg/api/overwrite_files.py b/iceberg/api/overwrite_files.py new file mode 100644 index 0000000000..3bcf0cec88 --- /dev/null +++ b/iceberg/api/overwrite_files.py @@ -0,0 +1,30 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +from .pending_update import PendingUpdate + + +class OverwriteFiles(PendingUpdate): + + def overwrite_by_row_filter(self, expr): + raise NotImplementedError() + + def add_file(self, file): + raise NotImplementedError() + + def validate_added_files(self): + raise NotImplementedError() diff --git a/iceberg/api/partition_field.py b/iceberg/api/partition_field.py new file mode 100644 index 0000000000..adaca28943 --- /dev/null +++ b/iceberg/api/partition_field.py @@ -0,0 +1,41 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + + +class PartitionField(object): + + def __init__(self, source_id, name, transform): + self.source_id = source_id + self.name = name + self.transform = transform + + def __eq__(self, other): + if id(self) == id(other): + return True + elif other is None or not isinstance(other, PartitionField): + return False + + return self.source_id == other.source_id and self.name == other.name and self.transform == other.transform + + def __ne__(self, other): + return not self.__eq__(other) + + def __hash__(self): + return hash(self.__key()) + + def __key(self): + return PartitionField.__class__, self.source_id, self.name, self.transform diff --git a/iceberg/api/partition_spec.py b/iceberg/api/partition_spec.py new file mode 100644 index 0000000000..272f49536e --- /dev/null +++ b/iceberg/api/partition_spec.py @@ -0,0 +1,308 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +from urllib.parse import quote_plus + +from iceberg.exceptions import ValidationException + +from .partition_field import PartitionField +from .schema import Schema +from .transforms import Transforms +from .types import (NestedField, + StructType) + +""" +TO-DO: Needs some work, please review + +""" + + +class PartitionSpec(object): + + PARTITION_DATA_ID_START = 1000 + + @staticmethod + def UNPARTITIONED_SPEC(): + return PartitionSpec(Schema(), 0, []) + + @staticmethod + def unpartitioned(): + return PartitionSpec.UNPARTITIONED_SPEC() + + def __init__(self, schema, spec_id, fields): + self.fields_by_source_id = None + self.fields_by_name = None + self.__java_classes = None + self.field_list = None + + self.schema = schema + self.spec_id = spec_id + self.__fields = list() + for field in fields: + self.__fields.append(field) + + @property + def fields(self): + return self.lazy_field_list() + + @property + def java_classes(self): + if self.__java_classes is None: + self.__java_classes + for i, field in enumerate(self.__fields): + source_type = self.schema.find_type(field.source_id) + result = field.transform().get_result_by_type(source_type) + self.__java_classes.append(result.type_id.java_class()) + + return self.__java_classes + + def get_field_by_source_id(self, field_id): + return self.lazy_fields_by_source_id().get(field_id) + + def partition_type(self): + struct_fields = list() + for i, field in enumerate(self.__fields): + source_type = self.schema.find_type(field.source_id) + result_type = field.transform.get_result_type(source_type) + struct_fields.append(NestedField.optional(PartitionSpec.PARTITION_DATA_ID_START + i, + field.name, + result_type)) + + return StructType.of(struct_fields) + + def get(self, data, pos, java_class): + data.get(pos, java_class) + + def escape(self, string): + return quote_plus(string, encoding="UTF-8") + + def partition_to_path(self, data): + sb = list() + java_classes = self.java_classes + for i, jclass in enumerate(java_classes): + field = self.__fields[i] + value_string = field.transform().to_human_string(self.get(data, i, jclass)) + + if i > 0: + sb.append("/") + sb.append(field.name) + sb.append("=") + sb.append(self.escape(value_string)) + + return "".join(sb) + + def compatible_with(self, other): + if self.__eq__(other): + return True + + if len(self.__fields) != len(other.__fields): + return False + + for i, field in enumerate(self.__fields): + that_field = other.__fields[i] + if field.source_id != that_field.source_id or str(field.transform) != str(that_field.transform): + return False + + return True + + def lazy_fields_by_source_id(self): + if self.fields_by_source_id is None: + self.fields_by_source_id = dict() + for field in self.fields: + self.fields_by_source_id[field.source_id] = field + + return self.fields_by_source_id + + def identity_source_ids(self): + source_ids = set() + fields = self.fields + for field in fields: + if "identity" == str(field.transform()): + source_ids.add(field) + + return source_ids + + def lazy_field_list(self): + if self.field_list is None: + self.field_list = list(self.__fields) + + return self.field_list + + def lazy_fields_by_source_name(self): + if self.fields_by_name is None: + self.fields_by_name = dict() + for field in self.__fields: + self.fields_by_name[field.name] = field + + return self.fields_by_name + + def __eq__(self, other): + if id(self) == id(other): + return True + + if other is None or not isinstance(other, PartitionSpec): + return False + + return self.__fields == other.__fields + + def __ne__(self, other): + return not self.__eq__(other) + + def __hash__(self): + return hash(self.__key()) + + def __key(self): + return PartitionSpec.__class__, self.fields + + def __str__(self): + return self.__repr__() + + def __repr__(self): + sb = ["["] + + for field in self.__fields: + sb.append("\n {name}: {transform}({source_id})".format(name=field.name, + transform=str(field.transform), + source_id=field.source_id)) + + if len(self.__fields) > 0: + sb.append("\n") + sb.append("]") + + return "".join(sb) + + @staticmethod + def builder_for(schema): + return PartitionSpecBuilder(schema) + + @staticmethod + def check_compatibility(spec, schema): + for field in spec.fields: + src_type = schema.find_type(field.source_id) + if not src_type.is_primitive_type(): + raise ValidationException("Cannot partition by non-primitive source field: %s" % src_type) + if not field.transform.can_transform(src_type): + ValidationException("Invalid source type %s for transform: %s" % (src_type, field.transform)) + + +class PartitionSpecBuilder(object): + + def __init__(self, schema): + self.schema = schema + self.fields = list() + self.partition_names = set() + self.spec_id = 0 + + def with_spec_id(self, spec_id): + self.spec_id = spec_id + return self + + def check_and_add_partition_name(self, name): + if name is None or name == "": + raise RuntimeError("Cannot use empty or null partition name") + if name in self.partition_names: + raise RuntimeError("Cannot use partition names more than once: %s" % name) + + self.partition_names.add(name) + return self + + def find_source_column(self, source_name): + source_column = self.schema.find_field(source_name) + if source_column is None: + raise RuntimeError("Cannot find source column: %s" % source_name) + + return source_column + + def identity(self, source_name): + self.check_and_add_partition_name(source_name) + source_column = self.find_source_column(source_name) + self.fields.append(PartitionField(source_column.field_id, + source_name, + Transforms.identity(source_column.type))) + return self + + def year(self, source_name): + name = "%s_year".format(source_name) + self.check_and_add_partition_name(name) + source_column = self.find_source_column(source_name) + self.fields.append(PartitionField(source_column.field_id, + source_name, + Transforms.year(source_column.types))) + return self + + def month(self, source_name): + name = "%s_month".format(source_name) + self.check_and_add_partition_name(name) + source_column = self.find_source_column(source_name) + self.fields.append(PartitionField(source_column.field_id, + source_name, + Transforms.month(source_column.types))) + return self + + def day(self, source_name): + name = "%s_day".format(source_name) + self.check_and_add_partition_name(name) + source_column = self.find_source_column(source_name) + self.fields.append(PartitionField(source_column.field_id, + source_name, + Transforms.day(source_column.types))) + return self + + def hour(self, source_name): + name = "%s_hour".format(source_name) + self.check_and_add_partition_name(name) + source_column = self.find_source_column(source_name) + self.fields.append(PartitionField(source_column.field_id, + source_name, + Transforms.hour(source_column.types))) + return self + + def bucket(self, source_name, num_buckets): + name = "%s_bucket".format(source_name) + self.check_and_add_partition_name(name) + source_column = self.find_source_column(source_name) + self.fields.append(PartitionField(source_column.field_id, + source_name, + Transforms.bucket(source_column.types, num_buckets))) + return self + + def truncate(self, source_name, width): + name = "%s_truncate".format(source_name) + self.check_and_add_partition_name(name) + source_column = self.find_source_column(source_name) + self.fields.append(PartitionField(source_column.field_id, + source_name, + Transforms.truncate(source_column.types, width))) + return self + + def add(self, source_id, name, transform): + self.check_and_add_partition_name(name) + column = self.schema.find_field(source_id) + if column is None: + raise RuntimeError("Cannot find source column: %s" % source_id) + + transform_obj = Transforms.from_string(column.type, transform) + field = PartitionField(source_id, + name, + transform_obj) + self.fields.append(field) + return self + + def build(self): + spec = PartitionSpec(self.schema, self.spec_id, self.fields) + PartitionSpec.check_compatibility(spec, self.schema) + return spec diff --git a/iceberg/api/pending_update.py b/iceberg/api/pending_update.py new file mode 100644 index 0000000000..6569895e90 --- /dev/null +++ b/iceberg/api/pending_update.py @@ -0,0 +1,25 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + + +class PendingUpdate(object): + + def apply(self): + raise NotImplementedError() + + def commit(self): + raise NotImplementedError() diff --git a/iceberg/api/replace_partitions.py b/iceberg/api/replace_partitions.py new file mode 100644 index 0000000000..ca7199daba --- /dev/null +++ b/iceberg/api/replace_partitions.py @@ -0,0 +1,30 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +from .pending_update import PendingUpdate + + +class ReplacePartitions(PendingUpdate): + + def __init__(self): + raise NotImplementedError() + + def apply(self): + raise NotImplementedError() + + def commit(self): + raise NotImplementedError() diff --git a/iceberg/api/rewrite_files.py b/iceberg/api/rewrite_files.py new file mode 100644 index 0000000000..fd120985bc --- /dev/null +++ b/iceberg/api/rewrite_files.py @@ -0,0 +1,30 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +from .pending_update import PendingUpdate + + +class RewriteFiles(PendingUpdate): + + def rewrite_files(self, files_to_delete=None, files_to_add=None): + raise NotImplementedError() + + def apply(self): + raise NotImplementedError() + + def commit(self): + raise NotImplementedError() diff --git a/iceberg/api/rollback.py b/iceberg/api/rollback.py new file mode 100644 index 0000000000..5b19261238 --- /dev/null +++ b/iceberg/api/rollback.py @@ -0,0 +1,33 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +from .pending_update import PendingUpdate + + +class Rollback(PendingUpdate): + + def to_snapshot_id(self, snapshot_id): + raise NotImplementedError() + + def to_snapshot_at_time(self, timestamp_millis): + raise NotImplementedError() + + def apply(self): + raise NotImplementedError() + + def commit(self): + raise NotImplementedError() diff --git a/iceberg/api/scan_task.py b/iceberg/api/scan_task.py new file mode 100644 index 0000000000..7b13560def --- /dev/null +++ b/iceberg/api/scan_task.py @@ -0,0 +1,28 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + + +class ScanTask(object): + + def is_file_scan_task(self): + return False + + def as_file_scan_task(self): + raise RuntimeError("Not a FileScanTask: %s" % self) + + def as_combined_scan_task(self): + RuntimeError("Not a CombinedScanTask: %s" % self) diff --git a/iceberg/api/schema.py b/iceberg/api/schema.py new file mode 100644 index 0000000000..19a154d96b --- /dev/null +++ b/iceberg/api/schema.py @@ -0,0 +1,136 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +from .types import StructType + +""" + The schema of a data table. + +""" + + +class Schema(object): + NEWLINE = '\n' + ALL_COLUMNS = "*" + + def __init__(self, *argv): + aliases = dict() + if len(argv) == 1 and isinstance(argv[0], (list, tuple)): + columns = argv[0] + elif len(argv) == 2 and isinstance(argv[0], list) and isinstance(argv[1], dict): + columns = argv[0] + aliases = argv[1] + else: + columns = argv + + self.struct = StructType.of(columns) + self._alias_to_id = None + self._id_to_alias = None + if aliases is not None: + self._alias_to_id = dict(aliases) + self._id_to_alias = {v: k for k, v in self._alias_to_id.items()} + + self._id_to_field = None + self._name_to_id = None + self._id_to_name = None + + def as_struct(self): + return self.struct + + def get_aliases(self): + return self._alias_to_id + + def lazy_id_to_field(self): + from .types import index_by_id + if self._id_to_field is None: + self._id_to_field = index_by_id(self.struct) # noqa + + return self._id_to_field + + def lazy_name_to_id(self): + from .types import index_by_name + if self._name_to_id is None: + self._name_to_id = index_by_name(self.struct) + self._id_to_name = {v: k for k, v in self._name_to_id.items()} + return self._name_to_id + + def columns(self): + return self.struct.fields + + def find_type(self, name): + if not name: + raise RuntimeError("Invalid Column Name (empty)") + + if isinstance(name, int): + field = self.lazy_id_to_field().get(name) + if field: + return field.type + + id = self.lazy_name_to_id().get(name) + if id: + return self.find_type(id) + + raise RuntimeError("Invalid Column (could not find): %s" % name) + + def find_field(self, id): + if isinstance(id, int): + return self.lazy_id_to_field().get(id) + + if not id: + raise RuntimeError("Invalid Column Name (empty)") + + id = self.lazy_name_to_id().get(id) + if id: + return self.lazy_id_to_field().get(id) + + def find_column_name(self, id): + if isinstance(id, int): + return self._id_to_name.get(id) + + def alias_to_id(self, alias): + if self._alias_to_id: + return self._alias_to_id.get(alias) + + def id_to_alias(self, field_id): + if self._id_to_alias: + return self._id_to_alias.get(field_id) + + def select(self, *argv): + from .types import select + if not(len(argv) == 1 and isinstance(argv[0], list)): + return self.select(argv) + + if len(argv) == 1: + names = argv[0] + if Schema.ALL_COLUMNS in names: + return self + else: + selected = list() + for name in names: + id = self.lazy_name_to_id().get(name) + if id: + selected.append(id) + + return select(self, selected) # noqa + + raise RuntimeError("Illegal argument for select %s", argv) + + def __repr__(self): + return "Schema(%s)" % self.struct.fields + + def __str__(self): + return "table {\n%s\n}" % Schema.NEWLINE.join([" " + str(field) for field in self.struct.fields]) diff --git a/iceberg/api/snapshot.py b/iceberg/api/snapshot.py new file mode 100644 index 0000000000..a112b0e5b7 --- /dev/null +++ b/iceberg/api/snapshot.py @@ -0,0 +1,37 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + + +class Snapshot(object): + + @property + def snapshot_id(self): + raise NotImplementedError() + + @property + def timestamp_millis(self): + raise NotImplementedError() + + @property + def manifests(self): + raise NotImplementedError() + + def added_files(self): + raise NotImplementedError() + + def deleted_files(self): + raise NotImplementedError() diff --git a/iceberg/api/snapshot_iterable.py b/iceberg/api/snapshot_iterable.py new file mode 100644 index 0000000000..ef8a0fb0c7 --- /dev/null +++ b/iceberg/api/snapshot_iterable.py @@ -0,0 +1,21 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + + +class SnapshotIterable(object): + def iterator(self, part_filter, row_filter, columns): + raise RuntimeError("Interface Implementation") diff --git a/iceberg/api/struct_like.py b/iceberg/api/struct_like.py new file mode 100644 index 0000000000..e421c2eb9f --- /dev/null +++ b/iceberg/api/struct_like.py @@ -0,0 +1,28 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + + +class StructLike(object): + + def __init__(self): + raise NotImplementedError() + + def get(self, pos): + raise NotImplementedError() + + def set(self, pos, value): + raise NotImplementedError() diff --git a/iceberg/api/table.py b/iceberg/api/table.py new file mode 100644 index 0000000000..2152121a50 --- /dev/null +++ b/iceberg/api/table.py @@ -0,0 +1,76 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + + +class Table(object): + + def __init__(self): + raise NotImplementedError() + + def refresh(self): + raise NotImplementedError() + + def new_scan(self): + raise NotImplementedError() + + def schema(self): + raise NotImplementedError() + + def spec(self): + raise NotImplementedError() + + def properties(self): + raise NotImplementedError() + + def location(self): + raise NotImplementedError() + + def snapshots(self): + raise NotImplementedError() + + def update_schema(self): + raise NotImplementedError() + + def update_properties(self): + raise NotImplementedError() + + def new_append(self): + raise NotImplementedError() + + def new_fast_append(self): + return self.new_append() + + def new_rewrite(self): + raise NotImplementedError() + + def new_overwrite(self): + raise NotImplementedError() + + def new_replace_partitions(self): + raise NotImplementedError() + + def new_delete(self): + raise NotImplementedError() + + def expire_snapshots(self): + raise NotImplementedError() + + def rollback(self): + raise NotImplementedError() + + def new_transaction(self): + raise NotImplementedError() diff --git a/iceberg/api/table_scan.py b/iceberg/api/table_scan.py new file mode 100644 index 0000000000..e457b9a73f --- /dev/null +++ b/iceberg/api/table_scan.py @@ -0,0 +1,40 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + + +class TableScan(object): + + def __init__(self): + raise NotImplementedError() + + def use_snapshot(self, snapshot_id): + raise NotImplementedError() + + def as_of_time(self, timestamp_millis): + raise NotImplementedError() + + def select(self, columns): + raise NotImplementedError() + + def filter(self, expr=None): + raise NotImplementedError() + + def plan_file(self): + raise NotImplementedError() + + def plan_tasks(self): + raise NotImplementedError() diff --git a/iceberg/api/tables.py b/iceberg/api/tables.py new file mode 100644 index 0000000000..7cd2e0501e --- /dev/null +++ b/iceberg/api/tables.py @@ -0,0 +1,25 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + + +class Tables(object): + + def create(self, schema, spec, table_identifier): + raise NotImplementedError() + + def load(self, table_identifier): + raise NotImplementedError() diff --git a/iceberg/api/transaction.py b/iceberg/api/transaction.py new file mode 100644 index 0000000000..1860c8af88 --- /dev/null +++ b/iceberg/api/transaction.py @@ -0,0 +1,55 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + + +class Transaction(object): + + def table(self): + raise NotImplementedError() + + def update_schema(self): + raise NotImplementedError() + + def update_properties(self): + raise NotImplementedError() + + def update_location(self): + raise NotImplementedError() + + def new_append(self): + raise NotImplementedError() + + def new_fast_append(self): + raise NotImplementedError() + + def new_rewrite(self): + raise NotImplementedError() + + def new_overwrite(self): + raise NotImplementedError() + + def new_replace_partitions(self): + raise NotImplementedError() + + def new_delete(self): + raise RuntimeError("Interface implementation") + + def expire_snapshots(self): + raise RuntimeError("Interface implementation") + + def commit_transaction(self): + raise RuntimeError("Interface implementation") diff --git a/iceberg/api/transforms/__init__.py b/iceberg/api/transforms/__init__.py new file mode 100644 index 0000000000..c0f29d778e --- /dev/null +++ b/iceberg/api/transforms/__init__.py @@ -0,0 +1,48 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +__all__ = ["Bucket", + "BucketByteBuffer", + "BucketDecimal", + "BucketDouble", + "BucketFloat", + "BucketInteger", + "BucketLong", + "BucketString", + "BucketUUID", + "Dates", + "Identity", + "Timestamps", + "Transform", + "Transforms", + "Truncate"] + +from .bucket import (Bucket, + BucketByteBuffer, + BucketDecimal, + BucketDouble, + BucketFloat, + BucketInteger, + BucketLong, + BucketString, + BucketUUID) +from .dates import Dates +from .identity import Identity +from .timestamps import Timestamps +from .transform import Transform +from .transforms import Transforms +from .truncate import Truncate diff --git a/iceberg/api/transforms/bucket.py b/iceberg/api/transforms/bucket.py new file mode 100644 index 0000000000..c1672da4a4 --- /dev/null +++ b/iceberg/api/transforms/bucket.py @@ -0,0 +1,185 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +import math +import struct +import sys + +import mmh3 + +from .transform import Transform +from .transform_util import TransformUtil +from ..expressions import (Expressions, + JAVA_MAX_INT, + Operation) +from ..types.types import (IntegerType, + TypeID) + + +class Bucket(Transform): + MURMUR3 = mmh3 + + BUCKET_TYPE = {TypeID.DATE: lambda n: BucketInteger(n), + TypeID.INTEGER: lambda n: BucketInteger(n), + TypeID.TIME: lambda n: BucketLong(n), + TypeID.TIMESTAMP: lambda n: BucketLong(n), + TypeID.LONG: lambda n: BucketLong(n), + TypeID.DECIMAL: lambda n: BucketDecimal(n), + TypeID.STRING: lambda n: BucketString(n), + TypeID.FIXED: lambda n: BucketByteBuffer(n), + TypeID.BINARY: lambda n: BucketByteBuffer(n), + TypeID.UUID: lambda n: BucketUUID(n)} + + @staticmethod + def get(type_var, n): + bucket_type_func = Bucket.BUCKET_TYPE.get(type_var.type_id) + if not bucket_type_func: + raise RuntimeError("Cannot bucket by type: %s" % type_var) + return bucket_type_func(n) + + def __init__(self, n): + self.n = n + + def __eq__(self, other): + if id(self) == id(other): + return True + if other is None or not isinstance(other, Bucket): + return False + + return self.n == other.n + + def __hash__(self): + return hash(self.__key()) + + def __key(self): + return Bucket.__class__, self.n + + def __repr__(self): + return "Bucket(n)" % self.n + + def __str__(self): + return "bucket[%s]" % self.n + + def apply(self, value): + return (self.hash(value) & JAVA_MAX_INT) % self.n + + def hash(self): + raise NotImplementedError() + + def project(self, name, predicate): + if predicate.op == Operation.EQ: + return Expressions.predicate(predicate.op, name, self.apply(predicate.lit.value)) + + def project_strict(self, name, predicate): + if predicate.op == Operation.NOT_EQ: + return Expressions.predicate(predicate.op, name, self.apply(predicate.lit.value)) + + def get_result_type(self, source_type): + return IntegerType.get() + + +class BucketInteger(Bucket): + + def __init__(self, n): + super(BucketInteger, self).__init__(n) + + def hash(self, value): + return Bucket.MURMUR3.hash(struct.pack("q", value)) + + def can_transform(self, type_var): + return type_var.type_id() in [TypeID.INTEGER, TypeID.DATE] + + +class BucketLong(Bucket): + def __init__(self, n): + super(BucketLong, self).__init__(n) + + def hash(self, value): + return Bucket.MURMUR3.hash(struct.pack("q", value)) + + def can_transform(self, type_var): + return type_var.type_id() in [TypeID.LONG, + TypeID.TIME, + TypeID.TIMESTAMP] + + +class BucketFloat(Bucket): + def __init__(self, n): + super(BucketFloat, self).__init__(n) + + def hash(self, value): + return Bucket.MURMUR3.hash(struct.pack("d", value)) + + def can_transform(self, type_var): + return type_var.type_id() == TypeID.FLOAT + + +class BucketDouble(Bucket): + def __init__(self, n): + super(BucketDouble, self).__init__(n) + + def hash(self, value): + return Bucket.MURMUR3.hash(struct.pack("d", value)) + + def can_transform(self, type_var): + return type_var.type_id() == TypeID.DOUBLE + + +class BucketDecimal(Bucket): + + def __init__(self, n): + super(BucketDecimal, self).__init__(n) + + def hash(self, value): + # to-do: unwrap to_bytes func since python2 support is being removed + unscaled_value = TransformUtil.unscale_decimal(value) + number_of_bytes = int(math.ceil(unscaled_value.bit_length() / 8)) + return Bucket.MURMUR3.hash(to_bytes(unscaled_value, number_of_bytes, byteorder='big')) + + def can_transform(self, type_var): + return type_var.type_id == TypeID.DECIMAL + + +class BucketString(Bucket): + def __init__(self, n): + super(BucketString, self).__init__(n) + + def hash(self, value): + return Bucket.MURMUR3.hash(value) + + def can_transform(self, type_var): + return type_var.type_id == TypeID.STRING + + +class BucketByteBuffer(Bucket): + def __init__(self, n): + # super(BucketByteBuffer, self).__init__(n) + raise NotImplementedError() + + +class BucketUUID(Bucket): + def __init__(self, n): + # super(BucketUUID, self).__init__(n) + raise NotImplementedError() + + +def to_bytes(n, length, byteorder='big'): + if sys.version_info >= (3, 0): + return n.to_bytes(length, byteorder=byteorder) + h = '%x' % n + s = ('0' * (len(h) % 2) + h).zfill(length * 2).decode('hex') + return s if byteorder == 'big' else s[::-1] diff --git a/iceberg/api/transforms/dates.py b/iceberg/api/transforms/dates.py new file mode 100644 index 0000000000..628281b20b --- /dev/null +++ b/iceberg/api/transforms/dates.py @@ -0,0 +1,76 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +import datetime + +from .projection_util import ProjectionUtil +from .transform import Transform +from .transform_util import TransformUtil +from ..expressions import (Expressions, + Operation) +from ..types.types import (IntegerType, + TypeID) + + +class Dates(Transform): + YEAR = "year" + MONTH = "month" + DAY = "day" + + EPOCH = datetime.datetime.utcfromtimestamp(0) + SECONDS_IN_DAY = 86400 + + HUMAN_FUNCS = {"year": lambda x: TransformUtil.human_year(x), + "month": lambda x: TransformUtil.human_month(x), + "day": lambda x: TransformUtil.human_day(x)} + + def __init__(self, granularity, name): + if granularity not in (Dates.YEAR, Dates.MONTH, Dates.DAY): + raise RuntimeError("Invalid Granularity: %s" % granularity) + self.granularity = granularity + self.name = name + + def apply(self, days): + if self.granularity == Dates.DAY: + return days + else: + apply_func = getattr(TransformUtil, "diff_{}".format(self.granularity)) + return apply_func(datetime.datetime.utcfromtimestamp(days * Dates.SECONDS_IN_DAY), Dates.EPOCH) + + def can_transform(self, type): + return type.type_id() == TypeID.DATE + + def get_result_type(self, source_type): + return IntegerType.get() + + def project(self, name, predicate): + if predicate.op == Operation.NOT_NULL or predicate.op == Operation.IS_NULL: + return Expressions.predicate(predicate.op, name) + + return ProjectionUtil.truncate_integer(name, predicate, self) + + def project_strict(self, name, predicate): + return None + + def to_human_string(self, value): + if value is None: + return "null" + + return Dates.HUMAN_FUNCS[self.granularity](value) + + def __str__(self): + return "%s" % self diff --git a/iceberg/api/transforms/identity.py b/iceberg/api/transforms/identity.py new file mode 100644 index 0000000000..f57524eb8a --- /dev/null +++ b/iceberg/api/transforms/identity.py @@ -0,0 +1,91 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +from .transform import Transform +from .transform_util import TransformUtil +from ..expressions import Expressions +from ..types import TypeID + + +class Identity(Transform): + + @staticmethod + def get(type_var): + return Identity(type_var) + + def __init__(self, type_var): + self.type_var = type_var + + def apply(self, value): + return value + + def can_transform(self, type_var): + return type_var.is_primitive_type() + + def get_result_type(self, source_type): + return source_type + + def project(self, name, predicate): + return self.project_strict(name, predicate) + + def project_strict(self, name, predicate): + if predicate.lit is not None: + return Expressions.predicate(predicate.op, name, predicate.lit.value) + else: + return Expressions.predicate(predicate.op, name) + + def to_human_string(self, value): + if value is None: + return "null" + + if self.type_var.type_id == TypeID.DATE: + return TransformUtil.human_day(value) + elif self.type_var.type_id == TypeID.TIME: + return TransformUtil.human_time(value) + elif self.type_var.type_id == TypeID.TIMESTAMP: + if self.type_var.adjust_to_utc: + return TransformUtil.human_timestamp_with_timezone(value) + else: + return TransformUtil.human_timestamp_without_timezone(value) + elif self.type_var.type_id in (TypeID.BINARY, TypeID.FIXED): + raise NotImplementedError() + # if isinstance(value, bytearray): + # return base64.b64encode(value) + # elif isinstance(value, bytes): + # return base64.b64encode(bytes(value)) + # else: + # raise RuntimeError("Unsupported binary type: %s" % value.__class__.__name__) + else: + return str(value) + + def __str__(self): + return "identity" + + def __eq__(self, other): + if id(self) == id(other): + return True + + if other is None or not isinstance(other, Identity): + return False + + return self.type_var == other.type_var + + def __hash__(self): + return hash(self.__key()) + + def __key(self): + return Identity.__class__, self.type_var diff --git a/iceberg/api/transforms/projection_util.py b/iceberg/api/transforms/projection_util.py new file mode 100644 index 0000000000..f9b0bb547e --- /dev/null +++ b/iceberg/api/transforms/projection_util.py @@ -0,0 +1,66 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + + +import decimal + +from iceberg.api.expressions import Expressions, Operation + + +class ProjectionUtil(object): + @staticmethod + def truncate_integer(name, pred, transform): + boundary = pred.lit.value + if pred.op == Operation.LT: + return Expressions.predicate(Operation.LT_EQ, name, transform.apply(boundary - 1)) + elif pred.op == Operation.LT_EQ: + return Expressions.predicate(Operation.LT_EQ, name, transform.apply(boundary)) + elif pred.op == Operation.GT: + return Expressions.predicate(Operation.GT_EQ, name, transform.apply(boundary + 1)) + elif pred.op == Operation.GT_EQ: + return Expressions.predicate(Operation.GT_EQ, name, transform.apply(boundary)) + elif pred.op == Operation.EQ: + return Expressions.predicate(pred.op, name, transform.apply(boundary)) + + def truncate_long(name, pred, transform): + return ProjectionUtil.truncate_integer(name, pred, transform) + + def truncate_decimal(name, pred, transform): + boundary = pred.lit.value + + if pred.op == Operation.LT: + minus_one = boundary - decimal.Decimal(1) + return Expressions.predicate(Operation.LT_EQ, name, transform.apply(minus_one)) + elif pred.op == Operation.LT_EQ: + return Expressions.predicate(Operation.LT_EQ, name, transform.apply(boundary)) + elif pred.op == Operation.GT: + plus_one = boundary + decimal.Decimal(1) + return Expressions.predicate(Operation.GT_EQ, name, transform.apply(plus_one)) + elif pred.op == Operation.GT_EQ: + return Expressions.predicate(Operation.GT_EQ, name, transform.apply(boundary)) + elif pred.op == Operation.EQ: + return Expressions.predicate(pred.op, name, transform.apply(boundary)) + + def truncate_array(name, pred, transform): + boundary = pred.lit.value + + if pred.op == Operation.LT or pred.op == Operation.LT_EQ: + return Expressions.predicate(Operation.LT_EQ, name, transform.apply(boundary)) + elif pred.op == Operation.GT or pred.op == Operation.GT_EQ: + return Expressions.predicate(Operation.GT_EQ, name, transform.apply(boundary)) + elif pred.op == Operation.EQ: + return Expressions.predicate(pred.op, name, transform.apply(boundary)) diff --git a/iceberg/api/transforms/timestamps.py b/iceberg/api/transforms/timestamps.py new file mode 100644 index 0000000000..697cec67e5 --- /dev/null +++ b/iceberg/api/transforms/timestamps.py @@ -0,0 +1,72 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + + +import datetime + +from .transform import Transform +from .transform_util import TransformUtil +from ..expressions import (Expressions, + Operation) +from ..types.types import (IntegerType, + TypeID) + + +class Timestamps(Transform): + YEAR = "year" + MONTH = "month" + DAY = "day" + HOUR = "hour" + + EPOCH = datetime.datetime.utcfromtimestamp(0) + HUMAN_FUNCS = {"year": lambda x: TransformUtil.human_year(x), + "month": lambda x: TransformUtil.human_month(x), + "day": lambda x: TransformUtil.human_day(x), + "hour": lambda x: TransformUtil.human_hour(x)} + + def __init__(self, granularity, name): + if granularity not in (Timestamps.YEAR, Timestamps.MONTH, Timestamps.DAY, Timestamps.HOUR): + raise RuntimeError("Invalid Granularity: %s" % granularity) + + self.granularity = granularity + self.name = name + + def apply(self, value): + apply_func = getattr(TransformUtil, "diff_{}".format(self.granularity)) + return apply_func(datetime.datetime.utcfromtimestamp(value / 1000000), Timestamps.EPOCH) + + def can_transform(self, type_var): + return type_var == TypeID.TIMESTAMP + + def get_result_type(self, source_type): + return IntegerType.get() + + def project(self, name, predicate): + if predicate.op == Operation.NOT_NULL or predicate.op == Operation.IS_NULL: + return Expressions.predicate(predicate.op, self.name) + + def project_strict(self, name, predicate): + return None + + def to_human_string(self, value): + if value is None: + return "null" + + return Timestamps.HUMAN_FUNCS[self.granularity](value) + + def __str__(self): + return self.name diff --git a/iceberg/api/transforms/transform.py b/iceberg/api/transforms/transform.py new file mode 100644 index 0000000000..776b0a507b --- /dev/null +++ b/iceberg/api/transforms/transform.py @@ -0,0 +1,40 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + + +class Transform(object): + + def __init__(self): + raise NotImplementedError() + + def apply(self, value): + raise NotImplementedError() + + def can_transform(self, type_var): + raise NotImplementedError() + + def get_result_type(self, source_type): + raise NotImplementedError() + + def project(self, name, predicate): + raise NotImplementedError() + + def project_strict(self, name, predicate): + raise NotImplementedError() + + def to_human_string(self, value): + return str(value) diff --git a/iceberg/api/transforms/transform_util.py b/iceberg/api/transforms/transform_util.py new file mode 100644 index 0000000000..9042fcc40f --- /dev/null +++ b/iceberg/api/transforms/transform_util.py @@ -0,0 +1,84 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +from datetime import datetime, timedelta + +import pytz + + +class TransformUtil(object): + EPOCH = datetime.utcfromtimestamp(0) + EPOCH_YEAR = datetime.utcfromtimestamp(0).year + + @staticmethod + def human_year(year_ordinal): + return "{0:0=4d}".format(TransformUtil.EPOCH_YEAR + year_ordinal) + + @staticmethod + def human_month(month_ordinal): + return "{0:0=4d}-{1:0=2d}".format(TransformUtil.EPOCH_YEAR + int(month_ordinal / 12), 1 + int(month_ordinal % 12)) + + @staticmethod + def human_day(day_ordinal): + day = TransformUtil.EPOCH + timedelta(days=day_ordinal) + return "{0:0=4d}-{1:0=2d}-{2:0=2d}".format(day.year, day.month, day.day) + + @staticmethod + def human_time(micros_from_midnight): + day = TransformUtil.EPOCH + timedelta(microseconds=micros_from_midnight) + return "{}".format(day.time()) + + @staticmethod + def human_timestamp_with_timezone(timestamp_micros): + day = TransformUtil.EPOCH + timedelta(microseconds=timestamp_micros) + return pytz.timezone("UTC").localize(day).strftime("%Y-%m-%dT%H:%M:%S.%fZ") + + @staticmethod + def human_timestamp_without_timezone(timestamp_micros): + day = TransformUtil.EPOCH + timedelta(microseconds=timestamp_micros) + return day.isoformat() + + @staticmethod + def human_hour(hour_ordinal): + time = TransformUtil.EPOCH + timedelta(hours=hour_ordinal) + return "{0:0=4d}-{1:0=2d}-{2:0=2d}-{3:0=2d}".format(time.year, time.month, time.day, time.hour) + + @staticmethod + def base_64_encode(buffer): + raise NotImplementedError() + + @staticmethod + def diff_hour(date1, date2): + return int((date1 - date2).total_seconds() / 3600) + + @staticmethod + def diff_day(date1, date2): + return (date1 - date2).days + + @staticmethod + def diff_month(date1, date2): + return (date1.year - date2.year) * 12 + (date1.month - date2.month) - (1 if date1.day < date2.day else 0) + + @staticmethod + def diff_year(date1, date2): + return (date1.year - date2.year) - \ + (1 if date1.month < date2.month or (date1.month == date2.month and date1.day < date2.day) else 0) + + @staticmethod + def unscale_decimal(decimal_value): + value_tuple = decimal_value.as_tuple() + return int(("-" if value_tuple.sign else "") + "".join([str(d) for d in value_tuple.digits])) diff --git a/iceberg/api/transforms/transforms.py b/iceberg/api/transforms/transforms.py new file mode 100644 index 0000000000..c14d84930f --- /dev/null +++ b/iceberg/api/transforms/transforms.py @@ -0,0 +1,111 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +import re + +from .bucket import Bucket +from .dates import Dates +from .identity import Identity +from .timestamps import Timestamps +from .truncate import Truncate +from ..types import (TypeID) + + +""" + Factory methods for transforms. +

+ Most users should create transforms using a + {@link PartitionSpec.Builder#builderFor(Schema)} partition spec builder}. + + @see PartitionSpec#builderFor(Schema) The partition spec builder. +""" + + +class Transforms(object): + HAS_WIDTH = re.compile("(\\w+)\\[(\\d+)\\]") + + def __init__(self): + pass + + @staticmethod + def from_string(type, transform): + match = Transforms.HAS_WIDTH.match(transform) + + if match is not None: + name = match.group(1) + w = match.group(2) + if name.lower() == "truncate": + return Truncate.get(type, w) + elif name.lower() == "bucket": + return Bucket.get(type, w) + + if transform.lower() == "identity": + return Identity.get(type) + elif type.type_id() == TypeID.TIMESTAMP: + return Timestamps(transform.lower(), transform.lower()) + elif type.type_id() == TypeID.DATE: + return Dates(transform.lower(), transform.lower()) + + raise RuntimeError("Unknown transform: %s" % transform) + + @staticmethod + def identity(type_var): + return Identity.get(type_var) + + @staticmethod + def year(type_var): + if type_var.type_id == TypeID.DATE: + return Dates("year", "year") + elif type_var.type_id == TypeID.TIMESTAMP: + return Timestamps("year", "year") + else: + raise RuntimeError("Cannot partition type %s by year" % type_var) + + @staticmethod + def month(type_var): + if type_var.type_id == TypeID.DATE: + return Dates("month", "month") + elif type_var.type_id == TypeID.TIMESTAMP: + return Timestamps("month", "month") + else: + raise RuntimeError("Cannot partition type %s by month" % type_var) + + @staticmethod + def day(type_var): + if type_var.type_id == TypeID.DATE: + return Dates("day", "day") + elif type_var.type_id == TypeID.TIMESTAMP: + return Timestamps("day", "day") + else: + raise RuntimeError("Cannot partition type %s by day" % type_var) + + @staticmethod + def hour(type_var): + if type_var.type_id == TypeID.DATE: + return Dates("hour", "hour") + elif type_var.type_id == TypeID.TIMESTAMP: + return Timestamps("hour", "hour") + else: + raise RuntimeError("Cannot partition type %s by hour" % type_var) + + @staticmethod + def bucket(type_var, num_buckets): + return Bucket.get(type_var, num_buckets) + + @staticmethod + def truncate(type_var, width): + return Truncate.get(type, width) diff --git a/iceberg/api/transforms/truncate.py b/iceberg/api/transforms/truncate.py new file mode 100644 index 0000000000..cd001ad44e --- /dev/null +++ b/iceberg/api/transforms/truncate.py @@ -0,0 +1,231 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +from decimal import Decimal + +from .projection_util import ProjectionUtil +from .transform import Transform +from .transform_util import TransformUtil +from ..expressions import (Expressions, + Operation) +from ..types import TypeID + + +class Truncate(Transform): + + @staticmethod + def get(type_var, width): + if type_var.type_id == TypeID.INTEGER: + return TruncateInteger(width) + elif type_var.type_id == TypeID.LONG: + return TruncateInteger(width) + elif type_var.type_id == TypeID.DECIMAL: + return TruncateDecimal(width) + elif type_var.type_id == TypeID.STRING: + return TruncateString(width) + + def __init__(self): + raise NotImplementedError() + + def apply(self, value): + raise NotImplementedError() + + def can_transform(self, type_var): + raise NotImplementedError() + + def get_result_type(self, source_type): + return type(source_type) + + def project(self, name, predicate): + raise NotImplementedError() + + def project_strict(self, name, predicate): + raise NotImplementedError() + + +class TruncateInteger(Truncate): + + def __init__(self, width): + self.W = width + + def apply(self, value): + return value - (((value % self.W) + self.W) % self.W) + + def can_transform(self, type_var): + return type_var.type_id == TypeID.INTEGER + + def project(self, name, predicate): + if predicate.op == Operation.NOT_NULL or predicate.op == Operation.IS_NULL: + return Expressions.predicate(predicate.op, name) + + return ProjectionUtil.truncate_integer(name, predicate, self) + + def project_strict(self, name, predicate): + if predicate.op == Operation.LT: + _in = predicate.lit.value - 1 + _out = predicate.lit.value + in_image = self.apply(_in) + out_image = self.apply(_out) + if in_image != out_image: + return Expressions.predicate(Operation.LT_EQ, name, in_image) + else: + return Expressions.predicate(Operation.LT, name, in_image) + elif predicate.op == Operation.LT_EQ: + _in = predicate.lit.value + _out = predicate.lit.value + 1 + in_image = self.apply(_in) + out_image = self.apply(_out) + if in_image != out_image: + return Expressions.predicate(Operation.LT_EQ, name, in_image) + else: + return Expressions.predicate(Operation.LT, name, in_image) + + def __eq__(self, other): + if id(self) == id(other): + return True + + if other is None or not isinstance(other, TruncateInteger): + return False + + return self.W == other.W + + def __hash__(self): + return hash(self.__key()) + + def __key(self): + return TruncateInteger.__class__, self.W + + def __str__(self): + return "truncate[%s]" % self.W + + +class TruncateLong(Truncate): + + def __init__(self, width): + self.W = width + + def apply(self, value): + return value - (((value % self.W) + self.W) % self.W) + + def can_transform(self, type_var): + return type_var.type_id == TypeID.LONG + + def project(self, name, predicate): + if predicate.op == Operation.NOT_NULL or predicate.op == Operation.IS_NULL: + return Expressions.predicate(predicate.op, name) + + return ProjectionUtil.truncate_long(name, predicate, self) + + def project_strict(self, name, predicate): + return None + + def __eq__(self, other): + if id(self) == id(other): + return True + + if other is None or not isinstance(other, TruncateLong): + return False + + return self.W == other.W + + def __hash__(self): + return hash(self.__key()) + + def __key(self): + return TruncateLong.__class__, self.W + + def __str__(self): + return "truncate[%s]" % self.W + + +class TruncateDecimal(Truncate): + + def __init__(self, unscaled_width): + self.unscaled_width = unscaled_width + + def apply(self, value): + unscaled_value = TransformUtil.unscale_decimal(value) + applied_value = unscaled_value - (((unscaled_value % self.unscaled_width) + self.unscaled_width) % self.unscaled_width) + return Decimal("{}e{}".format(applied_value, value.as_tuple().exponent)) + + def can_transform(self, type_var): + return type_var.type_id == TypeID.DECIMAL + + def project(self, name, predicate): + if predicate.op == Operation.NOT_NULL or predicate.op == Operation.IS_NULL: + return Expressions.predicate(predicate.op, name) + + return ProjectionUtil.truncate_decimal(name, predicate, self) + + def project_strict(self, name, predicate): + return None + + def __eq__(self, other): + if id(self) == id(other): + return True + + if other is None or not isinstance(other, TruncateDecimal): + return False + + return self.unscaled_width == other.unscaled_width + + def __hash__(self): + return hash(self.__key()) + + def __key(self): + return TruncateDecimal.__class__, self.unscaled_width + + def __str__(self): + return "truncate[%s]" % self.unscaled_width + + +class TruncateString(Truncate): + def __init__(self, length): + self.L = length + + def apply(self, value): + return value[0:min(self.L, len(value))] + + def can_transform(self, type_var): + return type_var.type_id == TypeID.STRING + + def project(self, name, predicate): + if predicate.op == Operation.NOT_NULL or predicate.op == Operation.IS_NULL: + return Expressions.predicate(predicate.op, name) + + return ProjectionUtil.truncate_array(name, predicate, self) + + def project_strict(self, name, predicate): + return None + + def __eq__(self, other): + if id(self) == id(other): + return True + + if other is None or not isinstance(other, TruncateString): + return False + + return self.L == other.L + + def __hash__(self): + return hash(self.__key()) + + def __key(self): + return TruncateString.__class__, self.L + + def __str__(self): + return "truncate[%s]" % self.L diff --git a/iceberg/api/types/#conversions.py# b/iceberg/api/types/#conversions.py# new file mode 100644 index 0000000000..3e8031da7d --- /dev/null +++ b/iceberg/api/types/#conversions.py# @@ -0,0 +1,87 @@ +1from decimal import Decimal +import struct +import sys +import uuid + +from .type import TypeID + + +class Conversions(object): + HIVE_NULL = "__HIVE_DEFAULT_PARTITION__" + value_mapping = {TypeID.BOOLEAN: lambda as_str: as_str.lower() == "true" if as_str is not None else False, + TypeID.INTEGER: lambda as_str: int(float(as_str)), + TypeID.LONG: lambda as_str: int(float(as_str)), + TypeID.FLOAT: lambda as_str: float(as_str), + TypeID.DOUBLE: lambda as_str: float(as_str), + TypeID.STRING: lambda as_str: as_str, + TypeID.UUID: lambda as_str: uuid.UUID(as_str), + TypeID.FIXED: lambda as_str: bytearray(bytes(as_str, "UTF-8") + if sys.version_info >= (3, 0) + else bytes(as_str)), + TypeID.BINARY: lambda as_str: bytes(as_str, "UTF-8") if sys.version_info >= (3, 0) else bytes(as_str), + TypeID.DECIMAL: lambda as_str: Decimal(as_str), + } + + to_byte_buff_mapping = {TypeID.BOOLEAN: lambda type_var, value: struct.pack("QQ', (value.int >> 64) & 0xFFFFFFFFFFFFFFFF, + value.int & 0xFFFFFFFFFFFFFFFF), + # TypeId.FIXED: lambda as_str: None, + # TypeId.BINARY: lambda as_str: None, + # TypeId.DECIMAL: lambda type_var, value: struct.pack(value.quantize( + # Decimal('.' + "".join(['0' for x in range(0, type_var.scale)]) + '1')) + } + + from_byte_buff_mapping = {TypeID.BOOLEAN: lambda type_var, value: struct.unpack('QQ', +value)[0] << 64 | + struct.unpack('>QQ', value)[1]), + TypeID.FIXED: lambda type_var, value: value, + TypeID.BINARY: lambda type_var, value: value} + + @staticmethod + def from_partition_string(type_var, as_string): + if as_string is None or Conversions.HIVE_NULL == as_string: + return None + part_func = Conversions.value_mapping.get(type_var) + if part_func is None: + raise RuntimeError("Unsupported type for fromPartitionString: %s" % type_var) + + return part_func(as_string) + + @staticmethod + def to_byte_buffer(type_var, value): + byte_buf_func = Conversions.to_byte_buff_mapping.get(type_var) + if byte_buf_func is None: + raise RuntimeError("Cannot Serialize Type: %s" % type_var) + + return byte_buf_func(type_var, value) + + @staticmethod + def from_byte_buffer(type_var, buffer_var): + return Conversions.internal_from_byte_buffer(type_var, buffer_var) + + @staticmethod + def internal_from_byte_buffer(type_var, buffer_var): + tmp = bytearray(len(buffer_var)) + tmp[:] = buffer_var + byte_buf_func = Conversions.from_byte_buff_mapping.get(type_var) + if byte_buf_func is None: + raise RuntimeError("Cannot Serialize Type: %s" % type_var) + + return byte_buf_func(type_var, tmp) diff --git a/iceberg/api/types/__init__.py b/iceberg/api/types/__init__.py new file mode 100644 index 0000000000..cf3fe0167b --- /dev/null +++ b/iceberg/api/types/__init__.py @@ -0,0 +1,134 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +__all__ = ["AssignFreshIds", + "assign_fresh_ids", + "from_primitive_string", + "get_projected_ids", + "Conversions", + "CustomOrderSchemaVisitor", + "NestedType", + "PrimitiveType", + "Type", + "TypeID", + "BinaryType", + "BooleanType", + "DateType", + "DecimalType", + "DoubleType", + "FixedType", + "FloatType", + "GetProjectedIds", + "IntegerType", + "IndexById", + "IndexByName", + "index_by_id", + "index_by_name", + "join", + "ListType", + "LongType", + "MapType", + "NestedField", + "PruneColumns", + "SchemaVisitor", + "select", + "select_not", + "StringType", + "StructType", + "TimeType", + "TimestampType", + "UUIDType", + "visit", + "visit_custom_order", + "VisitFieldFuture", + "VisitFuture" + ] + +import re + +from .conversions import Conversions +from .type import (NestedType, + PrimitiveType, + Type, + TypeID) +from .type_util import (assign_fresh_ids, + AssignFreshIds, + CustomOrderSchemaVisitor, + get_projected_ids, + GetProjectedIds, + index_by_id, + index_by_name, + IndexById, + IndexByName, + join, + PruneColumns, + SchemaVisitor, + select, + select_not, + visit, + visit_custom_order, + VisitFieldFuture, + VisitFuture) +from .types import (BinaryType, + BooleanType, + DateType, + DecimalType, + DoubleType, + FixedType, + FloatType, + IntegerType, + ListType, + LongType, + MapType, + NestedField, + StringType, + StructType, + TimestampType, + TimeType, + UUIDType) + +TYPES = {str(BooleanType.get()): BooleanType.get(), + str(IntegerType.get()): IntegerType.get(), + str(LongType.get()): LongType.get(), + str(FloatType.get()): FloatType.get(), + str(DoubleType.get()): DoubleType.get(), + str(DateType.get()): DateType.get(), + str(TimeType.get()): TimeType.get(), + str(TimestampType.with_timezone()): TimestampType.with_timezone(), + str(TimestampType.without_timezone()): TimestampType.without_timezone(), + str(StringType.get()): StringType.get(), + str(UUIDType.get()): UUIDType.get(), + str(BinaryType.get()): BinaryType.get()} + +FIXED = re.compile("fixed\\[(\\d+)\\]") +DECIMAL = re.compile("decimal\\((\\d+),\\s+(\\d+)\\)") + + +def from_primitive_string(type_string): + lower_type_string = type_string.lower() + if lower_type_string in TYPES.keys(): + return TYPES[lower_type_string] + + matches = FIXED.match(type_string) + if matches: + return FixedType.of_length(matches.group(1)) + + matches = DECIMAL.match(type_string) + if matches: + return DecimalType.of(matches.group(1), matches.group(2)) + + raise RuntimeError("Cannot parse type string to primitive: %s", type_string) diff --git a/iceberg/api/types/check_compatibility.py b/iceberg/api/types/check_compatibility.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/iceberg/api/types/conversions.py b/iceberg/api/types/conversions.py new file mode 100644 index 0000000000..a8bc3f5f25 --- /dev/null +++ b/iceberg/api/types/conversions.py @@ -0,0 +1,101 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +from decimal import Decimal +import struct +import sys +import uuid + +from .type import TypeID + + +class Conversions(object): + HIVE_NULL = "__HIVE_DEFAULT_PARTITION__" + value_mapping = {TypeID.BOOLEAN: lambda as_str: as_str.lower() == "true" if as_str is not None else False, + TypeID.INTEGER: lambda as_str: int(float(as_str)), + TypeID.LONG: lambda as_str: int(float(as_str)), + TypeID.FLOAT: lambda as_str: float(as_str), + TypeID.DOUBLE: lambda as_str: float(as_str), + TypeID.STRING: lambda as_str: as_str, + TypeID.UUID: lambda as_str: uuid.UUID(as_str), + TypeID.FIXED: lambda as_str: bytearray(bytes(as_str, "UTF-8") + if sys.version_info >= (3, 0) + else bytes(as_str)), + TypeID.BINARY: lambda as_str: bytes(as_str, "UTF-8") if sys.version_info >= (3, 0) else bytes(as_str), + TypeID.DECIMAL: lambda as_str: Decimal(as_str), + } + + to_byte_buff_mapping = {TypeID.BOOLEAN: lambda type_var, value: struct.pack("QQ', (value.int >> 64) & 0xFFFFFFFFFFFFFFFF, + value.int & 0xFFFFFFFFFFFFFFFF), + # TypeId.FIXED: lambda as_str: None, + # TypeId.BINARY: lambda as_str: None, + # TypeId.DECIMAL: lambda type_var, value: struct.pack(value.quantize( + # Decimal('.' + "".join(['0' for x in range(0, type_var.scale)]) + '1')) + } + + from_byte_buff_mapping = {TypeID.BOOLEAN: lambda type_var, value: struct.unpack('QQ', value)[0] << 64 | struct.unpack('>QQ', value)[1]), + TypeID.FIXED: lambda type_var, value: value, + TypeID.BINARY: lambda type_var, value: value} + + @staticmethod + def from_partition_string(type_var, as_string): + if as_string is None or Conversions.HIVE_NULL == as_string: + return None + part_func = Conversions.value_mapping.get(type_var.type_id) + if part_func is None: + raise RuntimeError("Unsupported type for fromPartitionString: %s" % type_var) + + return part_func(as_string) + + @staticmethod + def to_byte_buffer(type_var, value): + byte_buf_func = Conversions.to_byte_buff_mapping.get(type_var.type_id) + if byte_buf_func is None: + raise RuntimeError("Cannot Serialize Type: %s" % type_var) + + return byte_buf_func(type_var, value) + + @staticmethod + def from_byte_buffer(type_var, buffer_var): + return Conversions.internal_from_byte_buffer(type_var, buffer_var) + + @staticmethod + def internal_from_byte_buffer(type_var, buffer_var): + byte_buf_func = Conversions.from_byte_buff_mapping.get(type_var.type_id) + if byte_buf_func is None: + raise RuntimeError("Cannot Serialize Type: %s" % type_var) + + return byte_buf_func(type_var.type_id, buffer_var) diff --git a/iceberg/api/types/type.py b/iceberg/api/types/type.py new file mode 100644 index 0000000000..c855637820 --- /dev/null +++ b/iceberg/api/types/type.py @@ -0,0 +1,119 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +from decimal import Decimal +from enum import Enum, unique +import uuid + + +@unique +class TypeID(Enum): + BOOLEAN = {"java_class": "Boolean.class", "python_class": bool, "id": 1} + INTEGER = {"java_class": "Integer.class", "python_class": int, "id": 2} + LONG = {"java_class": "Long.class", "python_class": int, "id": 3} + FLOAT = {"java_class": "Float.class", "python_class": float, "id": 4} + DOUBLE = {"java_class": "Double.class", "python_class": float, "id": 5} + DATE = {"java_class": "Integer.class", "python_class": int, "id": 6} + TIME = {"java_class": "Long.class", "python_class": int, "id": 7} + TIMESTAMP = {"java_class": "Long.class", "python_class": int, "id": 8} + STRING = {"java_class": "CharSequence.class", "python_class": str, "id": 9} + UUID = {"java_class": "java.util.UUID.class", "python_class": uuid.UUID, "id": 10} + FIXED = {"java_class": "ByteBuffer.class", "python_class": bytes, "id": 11} + BINARY = {"java_class": "ByteBuffer.class", "python_class": bytearray, "id": 12} + DECIMAL = {"java_class": "BigDecimal.class", "python_class": Decimal, "id": 13} + STRUCT = {"java_class": "Void.class", "python_class": None, "id": 14} + LIST = {"java_class": "Void.class", "python_class": None, "id": 15} + MAP = {"java_class": "Void.class", "python_class": None, "id": 16} + + +class Type(object): + def __init__(self): + pass + + def type_id(self): + pass + + def is_primitive_type(self): + return False + + def as_primitive_type(self): + raise ValueError("Not a primitive type: " + self) + + def as_struct_type(self): + raise ValueError("Not a struct type: " + self) + + def as_list_type(self): + raise ValueError("Not a list type: " + self) + + def asMapType(self): + raise ValueError("Not a map type: " + self) + + def is_nested_type(self): + return False + + def is_struct_type(self): + return False + + def is_list_type(self): + return False + + def is_map_type(self): + return False + + def as_nested_type(self): + raise ValueError("Not a nested type: " + self) + + +class PrimitiveType(Type): + + def __eq__(self, other): + if id(self) == id(other): + return True + elif other is None or not isinstance(other, PrimitiveType): + return False + + return True + + def __ne__(self, other): + return not self.__eq__(other) + + def is_primitive_type(self): + return True + + def as_primitive_type(self): + return self + + +class NestedType(Type): + + def __init__(self): + super(NestedType, self).__init__() + + def is_nested_type(self): + return True + + def as_nested_type(self): + return self + + def fields(self): + pass + + def field_type(self, name): + pass + + def field(self, id): + pass diff --git a/iceberg/api/types/type_util.py b/iceberg/api/types/type_util.py new file mode 100644 index 0000000000..7898262a06 --- /dev/null +++ b/iceberg/api/types/type_util.py @@ -0,0 +1,528 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +import math + +from .type import (Type, + TypeID) +from .types import (ListType, + MapType, + NestedField, + StructType) + +MAX_PRECISION = list() +REQUIRED_LENGTH = [-1 for item in range(40)] + +MAX_PRECISION.append(0) +for i in range(1, 24): + MAX_PRECISION.append(int(math.floor(math.log10(math.pow(2, 8 * i - 1) - 1)))) + +for i in range(len(REQUIRED_LENGTH)): + for j in range(len(MAX_PRECISION)): + if i <= MAX_PRECISION[j]: + REQUIRED_LENGTH[i] = j + break + if REQUIRED_LENGTH[i] < 0: + raise RuntimeError("Could not find required length for precision %s" % i) + + +def select(schema, field_ids): + import iceberg.api.schema + if schema is None: + raise RuntimeError("Schema cannot be None") + if field_ids is None: + raise RuntimeError("Field ids cannot be None") + + result = visit(schema, PruneColumns(field_ids)) + if schema.as_struct() == result: + return schema + elif result is not None: + if schema.get_aliases() is not None: + return iceberg.api.schema.Schema(result.as_nested_type(), schema.get_aliases()) + else: + return iceberg.api.schema.Schema(result.as_nested_type()) + + return iceberg.api.schema.Schema(list(), schema.get_aliases()) + + +def get_projected_ids(schema): + import iceberg.api.schema + if isinstance(schema, iceberg.api.schema.Schema): + return visit(schema, GetProjectedIds()) + elif isinstance(schema, Type): + if schema.is_primitive_type(): + return set() + + return set(visit(schema, GetProjectedIds)) + + else: + raise RuntimeError("Argument %s must be Schema or a Type" % schema) + + +def select_not(schema, field_ids): + projected_ids = get_projected_ids(schema) + projected_ids.difference(field_ids) + + return select(schema, projected_ids) + + +def join(left, right): + import iceberg.api.schema + return iceberg.api.schema.Schema(left + right) + + +def index_by_name(struct): + return visit(struct, IndexByName()) + + +def index_by_id(struct): + return visit(struct, IndexById()) + + +def assign_fresh_ids(type_var, next_id): + from ..schema import Schema + if isinstance(type_var, Type): + return visit(type_var, AssignFreshIds(next_id)) + elif isinstance(type_var, Schema): + schema = type_var + return Schema(list(visit(schema.as_struct(), AssignFreshIds(next_id)) + .as_nested_type().fields)) + + +def visit(arg, visitor): # noqa: ignore=C901 + from ..schema import Schema + if isinstance(visitor, CustomOrderSchemaVisitor): + return visit_custom_order(arg, visitor) + elif isinstance(arg, Schema): + return visitor.schema(arg, visit(arg.as_struct(), visitor)) + elif isinstance(arg, Type): + type_var = arg + if type_var.type_id == TypeID.STRUCT: + struct = type_var.as_nested_type().as_struct_type() + results = list() + for field in struct.fields: + visitor.field_ids.append(field.field_id) + visitor.field_names.append(field.name) + result = None + try: + result = visit(field.type, visitor) + except RuntimeError: + pass + finally: + visitor.field_ids.pop() + visitor.field_names.pop() + results.append(visitor.field(field, result)) + return visitor.struct(struct, results) + elif type_var.type_id == TypeID.LIST: + list_var = type_var.as_nested_type().as_list_type() + visitor.field_ids.append(list_var.element_id) + try: + element_result = visit(list_var.element_type, visitor) + except RuntimeError: + pass + finally: + visitor.field_ids.pop() + + return visitor.list(list_var, element_result) + elif type_var.type_id == TypeID.MAP: + raise NotImplementedError() + else: + return visitor.primitive(arg.as_primitive_type()) + else: + raise RuntimeError("Invalid type for arg: %s" % arg) + + +def visit_custom_order(arg, visitor): + from ..schema import Schema + if isinstance(arg, Schema): + schema = arg + return visitor.schema(arg, VisitFuture(schema.as_struct(), visitor)) + elif isinstance(arg, Type): + type_var = arg + if type_var.type_id == TypeID.STRUCT: + struct = type_var.as_nested_type().as_struct_type() + results = list() + fields = struct.fields + for field in fields: + results.append(VisitFieldFuture(field, visitor)) + struct = visitor.struct(struct, [x.get() for x in results]) + return struct + + return visitor.primitive(type_var.as_primitive_type()) + + +class SchemaVisitor(object): + + def __init__(self): + self.field_names = list() + self.field_ids = list() + + def schema(self, schema, struct_result): + return NotImplementedError() + + def struct(self, struct, field_results): + return NotImplementedError() + + def field(self, field, field_result): + return NotImplementedError() + + def list(self, list_var, element_result): + return NotImplementedError() + + def map(self, map_var, key_result, value_result): + return NotImplementedError() + + def primitive(self, primitive_var): + return NotImplementedError() + + +class CustomOrderSchemaVisitor(object): + def __init__(self): + super(CustomOrderSchemaVisitor, self).__init__() + + def schema(self, schema, struct_result): + return NotImplementedError() + + def struct(self, struct, field_results): + return NotImplementedError() + + def field(self, field, field_result): + return NotImplementedError() + + def list(self, list_var, element_result): + return NotImplementedError() + + def map(self, map_var, key_result, value_result): + return NotImplementedError() + + def primitive(self, primitive_var): + return NotImplementedError() + + +class VisitFuture(object): + + def __init__(self, type, visitor): + self.type = type + self.visitor = visitor + + def get(self): + return visit(self.type, self.visitor) + + +class VisitFieldFuture(object): + + def __init__(self, field, visitor): + self.field = field + self.visitor = visitor + + def get(self): + return self.visitor.field(self.field, VisitFuture(self.field.type, self.visitor).get) + + +@staticmethod +def decimal_required_bytes(precision): + if precision < 0 or precision > 40: + raise RuntimeError("Unsupported decimal precision: %s" % precision) + + return REQUIRED_LENGTH[precision] + + +class GetProjectedIds(SchemaVisitor): + + def __init__(self): + super(GetProjectedIds, self).__init__() + self.field_ids = set() + + def schema(self, schema, struct_result): + return self.field_ids + + def struct(self, struct, field_results): + return self.field_ids + + def field(self, field, field_result): + if field_result is not None: + self.field_ids.add(field.field_id) + + return self.field_ids + + def list(self, list_var, element_result): + if element_result is None: + for field in list_var.fields: + self.field_ids.add(field.field_id) + + return self.field_ids + + def map(self, map_var, key_result, value_result): + if value_result is None: + for field in map_var.fields: + self.field_ids.add(field.field_id) + + return self.field_ids + + +class PruneColumns(SchemaVisitor): + + def __init__(self, selected): + super(PruneColumns, self).__init__() + self.selected = selected + + def schema(self, schema, struct_result): + return struct_result + + def struct(self, struct, field_results): + fields = struct.fields + selected_fields = list() + same_types = True + + for i, field in enumerate(field_results): + projected_type = field_results[i] + if projected_type is not None: + if field.type_id == projected_type.type_id: + selected_fields.append(field) + elif projected_type is not None: + same_types = False + if field.is_optional: + selected_fields.append(NestedField.optional(field.field_id, + field.name, + projected_type)) + else: + selected_fields.append(NestedField.required(field.field_id, + field.name, + projected_type)) + + if len(selected_fields) != 0: + if len(selected_fields) == len(fields) and same_types: + return struct + else: + return StructType.of(selected_fields) + + def field(self, field, field_result): + if field.field_id in self.selected: + return field.type + elif field_result is not None: + return field_result + + +class IndexByName(SchemaVisitor): + + DOT = "." + + def __init__(self): + super(IndexByName, self).__init__() + self.name_to_id = dict() + + def schema(self, schema, struct_result): + return self.name_to_id + + def struct(self, struct, field_results): + return self.name_to_id + + def field(self, field, field_result): + self.add_field(field.name, field.field_id) + + def list(self, list_var, element_result): + for field in list_var.fields(): + self.add_field(field.name, field.field_id) + + def map(self, map_var, key_result, value_result): + for field in map_var.fields(): + self.add_field(field.name, field.field_id) + + def add_field(self, name, field_id): + full_name = name + if not self.field_names and len(self.field_names) > 0: + full_name = IndexByName.DOT.join([IndexByName.DOT.join(reversed(self.field_names)), name]) + + self.name_to_id[full_name] = field_id + + +class IndexById(SchemaVisitor): + + def __init__(self): + super(IndexById, self).__init__() + self.index = dict() + + def schema(self, schema, struct_result): + return self.index + + def struct(self, struct, field_results): + return self.index + + def field(self, field, field_result): + self.index[field.field_id] = field + + def list(self, list_var, element_result): + for field in list_var.fields(): + self.index[field.field_id] = field + + def map(self, map_var, key_result, value_result): + for field in map_var.fields: + self.index[field.field_id] = field + + +class NextID(object): + def __init__(self): + raise NotImplementedError() + + def get(self): + raise NotImplementedError() + + +class AssignFreshIds(CustomOrderSchemaVisitor): + def __init__(self, next_id): + super(AssignFreshIds, self).__init__() + self.next_id = next_id + + def schema(self, schema, struct_result): + return self.next_id() + + def struct(self, struct, field_results): + fields = struct.fields + length = len(struct.fields) + new_ids = list() + + for i in range(length): + new_ids.append(self.next_id()) + + new_fields = list() + types = iter(field_results) + for i in range(length): + field = fields[i] + type = next(types) + if field.is_optional: + new_fields.append(NestedField.optional(new_ids[i], field.name, type)) + else: + new_fields.append(NestedField.required(new_ids[i], field.name, type)) + + return StructType.of(new_fields) + + def field(self, field, field_result): + return field_result() + + def list(self, list_var, element_result): + new_id = self.next_id() + if list_var.is_element_optional(): + return ListType.of_optional(new_id, element_result()) + else: + return ListType.of_required(new_id, element_result()) + + def map(self, map_var, key_result, value_result): + new_key_id = self.next_id() + new_value_id = self.next_id() + + if map_var.is_value_optional(): + return MapType.of_optional(new_key_id, new_value_id, key_result(), value_result()) + else: + return MapType.of_required(new_key_id, new_value_id, key_result(), value_result()) + + def primitive(self, primitive_var): + return primitive_var + + +class CheckCompatibility(CustomOrderSchemaVisitor): + + @staticmethod + def write_compatibility_errors(read_schema, write_schema): + visit(read_schema, CheckCompatibility(write_schema, True)) + + @staticmethod + def read_compatibility_errors(read_schema, write_schema): + visit(write_schema, CheckCompatibility(read_schema, False)) + + NO_ERRORS = [] + + def __init__(self, schema, check_ordering): + self.schema = schema + self.check_ordering + self.current_type = None + + def schema(self, schema, struct_result): + self.current_type = self.schema.as_struct() + try: + struct_result.get() + finally: + self.current_type = None + + def struct(self, struct, field_results): + if struct is None: + raise RuntimeError("Evaluation must start with a schema.") + + if not self.current_type.is_struct_type(): + return [": %s cannot be read as a struct" % self.current_type] + + errors = [] + + for field_errors in field_results: + errors = errors + field_errors + + if self.check_ordering: + new_struct = self.current_type.as_struct_type() + id_to_ord = {} + for i, val in enumerate(new_struct.fields): + id_to_ord[val.field_id] = i + + last_ordinal = -1 + + for read_field in self.struct.fields: + id_var = read_field.field_id + + field = struct.field(id=id_var) + if field is not None: + ordinal = id_to_ord[id] + if last_ordinal >= ordinal: + errors.append("%s is out of order before %s" % (read_field.name, + new_struct.fields[last_ordinal].name)) + last_ordinal = ordinal + + return errors + + def field(self, field, field_result): + struct = self.current_type.as_struct_type() + curr_field = struct.field(field.field_id) + errors = list() + + if curr_field is None: + if not field.is_optional: + errors.add("{} is required, but is missing".format(field.name)) + return self.NO_ERRORS + + self.current_type = curr_field.type + + try: + if not field.is_optional and curr_field.is_optional: + errors.add(field.name + " should be required, but is optional") + + for error in field_result: + if error.startswith(":"): + errors.append("{}{}".format(field.field_name, error)) + else: + errors.append("{}.{}".format(field.field_name, error)) + + return errors + except RuntimeError: + pass + finally: + self.current_type = struct + + def list(self, list_var, element_result): + raise NotImplementedError() + + def map(self, map_var, key_result, value_result): + raise NotImplementedError() + + def primitive(self, primitive_var): + raise NotImplementedError() diff --git a/iceberg/api/types/types.py b/iceberg/api/types/types.py new file mode 100644 index 0000000000..40e1d88ca8 --- /dev/null +++ b/iceberg/api/types/types.py @@ -0,0 +1,706 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +from .type import (NestedType, + PrimitiveType, + TypeID) + + +class BooleanType(PrimitiveType): + __instance = None + + @staticmethod + def get(): + if BooleanType.__instance is None: + BooleanType() + return BooleanType.__instance + + def __init__(self): + if BooleanType.__instance is not None: + raise Exception("Multiple Boolean Types created") + BooleanType.__instance = self + + @property + def type_id(self): + return TypeID.BOOLEAN + + def __repr__(self): + return "boolean" + + def __str__(self): + return "boolean" + + +class IntegerType(PrimitiveType): + __instance = None + + @staticmethod + def get(): + if IntegerType.__instance is None: + IntegerType() + return IntegerType.__instance + + def __init__(self): + if IntegerType.__instance is not None: + raise Exception("Multiple Integer Types created") + IntegerType.__instance = self + + @property + def type_id(self): + return TypeID.INTEGER + + def __repr__(self): + return "int" + + def __str__(self): + return "int" + + +class LongType(PrimitiveType): + __instance = None + + @staticmethod + def get(): + if LongType.__instance is None: + LongType() + return LongType.__instance + + def __init__(self): + if LongType.__instance is not None: + raise Exception("Multiple Long Types created") + LongType.__instance = self + + @property + def type_id(self): + return TypeID.LONG + + def __repr__(self): + return "long" + + def __str__(self): + return "long" + + +class FloatType(PrimitiveType): + __instance = None + + @staticmethod + def get(): + if FloatType.__instance is None: + FloatType() + return FloatType.__instance + + def __init__(self): + if FloatType.__instance is not None: + raise Exception("Multiple Float Types created") + FloatType.__instance = self + + @property + def type_id(self): + return TypeID.FLOAT + + def __repr__(self): + return "float" + + def __str__(self): + return "float" + + +class DoubleType(PrimitiveType): + __instance = None + + @staticmethod + def get(): + if DoubleType.__instance is None: + DoubleType() + return DoubleType.__instance + + def __init__(self): + if DoubleType.__instance is not None: + raise Exception("Multiple Double Types created") + DoubleType.__instance = self + + @property + def type_id(self): + return TypeID.DOUBLE + + def __repr__(self): + return "double" + + def __str__(self): + return "double" + + +class DateType(PrimitiveType): + __instance = None + + @staticmethod + def get(): + if DateType.__instance is None: + DateType() + return DateType.__instance + + def __init__(self): + if DateType.__instance is not None: + raise Exception("Multiple Date Types created") + DateType.__instance = self + + @property + def type_id(self): + return TypeID.DATE + + def __repr__(self): + return "date" + + def __str__(self): + return "date" + + +class TimeType(PrimitiveType): + __instance = None + + @staticmethod + def get(): + if TimeType.__instance is None: + TimeType() + return TimeType.__instance + + def __init__(self): + if TimeType.__instance is not None: + raise Exception("Multiple Time Types created") + TimeType.__instance = self + + @property + def type_id(self): + return TypeID.TIME + + def __repr__(self): + return "time" + + def __str__(self): + return "time" + + +class TimestampType(PrimitiveType): + __instance_with_tz = None + __instance_without_tz = None + + @staticmethod + def with_timezone(): + if not TimestampType.__instance_with_tz: + TimestampType() + return TimestampType.__instance_with_tz + + @staticmethod + def without_timezone(): + if not TimestampType.__instance_without_tz: + TimestampType(False) + return TimestampType.__instance_without_tz + + def __init__(self, with_timezone=True): + self.adjust_to_utc = with_timezone + if (with_timezone and TimestampType.__instance_with_tz is not None)\ + or (not with_timezone and TimestampType.__instance_without_tz is not None): + raise Exception("Multiple Timestamp Types created") + + if with_timezone: + TimestampType.__instance_with_tz = self + else: + TimestampType.__instance_without_tz = self + + @property + def type_id(self): + return TypeID.TIMESTAMP + + def __eq__(self, other): + if id(self) == id(other): + return True + elif other is None or not isinstance(other, TimestampType): + return False + + return self.adjust_to_utc == other.adjust_to_utc + + def __hash__(self): + return hash(self.__key()) + + def __key(self): + return TimestampType.__class__, self.adjust_to_utc + + def __ne__(self, other): + return not self.__eq__(other) + + def __repr__(self): + if self.adjust_to_utc: + return "timestamptz" + else: + return "timestamp" + + def __str__(self): + return self.__repr__() + + +class StringType(PrimitiveType): + __instance = None + + @staticmethod + def get(): + if StringType.__instance is None: + StringType() + return StringType.__instance + + def __init__(self): + if StringType.__instance is not None: + raise Exception("Multiple String Types created") + StringType.__instance = self + + @property + def type_id(self): + return TypeID.STRING + + def __repr__(self): + return "string" + + def __str__(self): + return "string" + + +class UUIDType(PrimitiveType): + __instance = None + + @staticmethod + def get(): + if UUIDType.__instance is None: + UUIDType() + return UUIDType.__instance + + def __init__(self): + if UUIDType.__instance is not None: + raise Exception("Multiple UUID Types created") + UUIDType.__instance = self + + @property + def type_id(self): + return TypeID.UUID + + def __repr__(self): + return "uuid" + + def __str__(self): + return "uuid" + + +class FixedType(PrimitiveType): + + @staticmethod + def of_length(length): + return FixedType(length) + + def __init__(self, length): + self.length = length + + def length(self): + return self.length + + @property + def type_id(self): + return TypeID.FIXED + + def __hash__(self): + return hash(self.__key()) + + def __eq__(self, other): + if id(self) == id(other): + return True + elif other is None or not isinstance(other, FixedType): + return False + + return self.length == other.length + + def __key(self): + return FixedType.__class__, self.length + + def __ne__(self, other): + return not self.__eq__(other) + + def __repr__(self): + return "fixed[%s]" % (self.length()) + + def __str__(self): + return self.__repr__() + + +class BinaryType(PrimitiveType): + __instance = None + + @staticmethod + def get(): + if BinaryType.__instance is None: + BinaryType() + return BinaryType.__instance + + def __init__(self): + if BinaryType.__instance is not None: + raise Exception("Multiple Binary Types created") + BinaryType.__instance = self + + @property + def type_id(self): + return TypeID.BINARY + + def __repr__(self): + return "binary" + + def __str__(self): + return "binary" + + +class DecimalType(PrimitiveType): + + @staticmethod + def of(precison, scale): + return DecimalType(precison, scale) + + def __init__(self, precision, scale): + if precision > 38: + raise RuntimeError("Decimals with precision larger than 38 are not supported: %s", precision) + self.precision = precision + self.scale = scale + + @property + def type_id(self): + return TypeID.DECIMAL + + def __repr__(self): + return "decimal(%s,%s)" % (self.precision, self.scale) + + def __str__(self): + return self.__repr__() + + def __eq__(self, other): + if id(self) == id(other): + return True + elif other is None or not isinstance(other, DecimalType): + return False + + return self.precision == other.precison and self.scale == other.scale + + def __ne__(self, other): + return not self.__eq__(other) + + def __hash__(self): + return hash(self.__key()) + + def __key(self): + return DecimalType.__class__, self.precision, self.scale + + +class NestedField(): + @staticmethod + def optional(id, name, type_var): + return NestedField(True, id, name, type_var) + + @staticmethod + def required(id, name, type): + return NestedField(False, id, name, type) + + def __init__(self, is_optional, id, name, type): + self.is_optional = is_optional + self.id = id + self.name = name + self.type = type + + @property + def is_required(self): + return not self.is_optional + + @property + def field_id(self): + return self.id + + def __repr__(self): + return "%s: %s: %s %s" % (self.id, self.name, "optional" if self.is_optional else "required", self.type) + + def __str__(self): + return self.__repr__() + + def __eq__(self, other): + if id(self) == id(other): + return True + elif other is None or not isinstance(other, NestedField): + return False + + return self.is_optional == other.is_optional \ + and self.id == other.id \ + and self.name == other.name and self.type == other.type + + def __ne__(self, other): + return not self.__eq__(other) + + def __hash__(self): + return hash(self.__key()) + + def __key(self): + type_name = self.type.type_id.name + return NestedField.__class__, self.is_optional, self.id, self.name, type_name + + +class StructType(NestedType): + FIELD_SEP = ", " + + @staticmethod + def of(fields): + return StructType(fields) + + def __init__(self, fields): + if fields is None: + raise RuntimeError("Field list cannot be None") + + self._fields = list() + for i in range(0, len(fields)): + self._fields.append(fields[i]) + + self._fieldList = None + self._fieldsByName = None + self._fieldsById = None + + def __eq__(self, other): + if id(self) == id(other): + return True + elif other is None or not isinstance(other, StructType): + return False + + return self._fields == other._fields + + def __ne__(self, other): + return not self.__eq__(other) + + @property + def fields(self): + return self._lazyFieldList() + + def field(self, name=None, id=None): + if name: + return self._lazyFieldsByName().get(name) + elif id: + return self._lazyFieldsById()[id] + + raise RuntimeError("No valid field info passed in ") + + @property + def type_id(self): + return TypeID.STRUCT + + def is_struct_type(self): + return True + + def as_struct_type(self): + return self + + def __str__(self): + return "struct<{}>".format(StructType.FIELD_SEP.join(str(x) for x in self.fields)) + + def __hash__(self): + return hash(self.__key()) + + def __key(self): + return StructType.__class__, self.fields + + def _lazyFieldList(self): + if self._fieldList is None: + self._fieldList = tuple(self._fields) + return self._fieldList + + def _lazyFieldsByName(self): + if self._fieldsByName is None: + self.indexFields() + return self._fieldsByName + + def _lazyFieldsById(self): + if self._fieldsById is None: + self.indexFields() + return self._fieldsById + + def indexFields(self): + self._fieldsByName = dict() + self._fieldsById = dict() + + for field in self.fields: + self._fieldsByName[field.name] = field + self._fieldsById[field.id] = field + + +class ListType(NestedType): + @staticmethod + def of_optional(element_id, element_type): + if element_type is None: + raise RuntimeError("Element type cannot be null") + return ListType(NestedField.optional(element_id, "element", element_type)) + + @staticmethod + def of_required(element_id, element_type): + if element_type is None: + raise RuntimeError("Element type cannot be null") + return ListType(NestedField.required(element_id, "element", element_type)) + + def __init__(self, element_field): + self.element_field = element_field + self._fields = None + + @property + def type_id(self): + return TypeID.LIST + + @property + def element_type(self): + return self.element_field.type + + def field_type(self, name): + if "element" == name: + return self.element_type + + def field(self, id): + if self.element_field.id == id: + return self.element_field + + def fields(self): + return self._lazyFieldsList() + + @property + def element_id(self): + return self.element_field.id + + def is_element_required(self): + return not self.element_field.is_optional + + def is_element_optional(self): + return self.element_field.is_optional + + def is_list_type(self): + return True + + def as_list_type(self): + return self + + def __str__(self): + return "list<%s>" % self.element_field.type + + def __eq__(self, other): + if id(self) == id(other): + return True + elif other is None or not isinstance(other, ListType): + return False + + return self.element_field == other.element_field + + def __ne__(self, other): + return self.__eq__(other) + + def __hash__(self): + return hash(self.__key()) + + def __key(self): + return StructType.__class__, self.element_field + + def _lazyFieldsList(self): + if self._fields is None: + self._fields = [self.element_field] + + return self._fields + + +class MapType(NestedType): + + @staticmethod + def of_optional(key_id, value_id, key_type, value_type): + if value_type is None: + raise RuntimeError("Value type cannot be null") + + return MapType(NestedField.required(key_id, 'key', key_type), + NestedField.optional(value_id, 'value', value_type)) + + @staticmethod + def of_required(key_id, value_id, key_type, value_type): + if value_type is None: + raise RuntimeError("Value type cannot be null") + + return MapType(NestedField.required(key_id, 'key', key_type), + NestedField.required(value_id, 'value', value_type)) + + def __init__(self, key_field, value_field): + self.key_field = key_field + self.value_field = value_field + self._fields = None + + @property + def type_id(self): + return TypeID.MAP + + def key_type(self): + return self.key_field.type + + def value_type(self): + return self.value_field.type + + def field_type(self, name): + if "key" == name: + return self.key_field.type + elif "value" == name: + return self.value_field.type + + def field(self, id): + if self.key_field.id == id: + return self.key_field + elif self.value_field.id == id: + return self.value_field + + def fields(self): + return self._lazyFieldList() + + def key_id(self): + return self.key_field.field_id + + def value_id(self): + return self.value_field.field_id + + def is_value_optional(self): + return self.value_field.is_optional + + def is_value_required(self): + return not self.is_value_optional() + + def __str__(self): + return "map<%s, %s>" % (self.key_field.type, self.value_field.type) + + def __eq__(self, other): + if id(self) == id(other): + return True + elif other is None or not isinstance(other, MapType): + return False + + return self.key_field == other.key_field and self.value_field == other.value_field + + def __ne__(self, other): + return not self.__eq__(other) + + def __hash__(self): + return hash(self.__key()) + + def __key(self): + return MapType.__class__, self.key_field, self.value_field + + def _lazyFieldList(self): + return tuple(self.key_field, self.value_field) diff --git a/iceberg/api/update_properties.py b/iceberg/api/update_properties.py new file mode 100644 index 0000000000..763bb06bcd --- /dev/null +++ b/iceberg/api/update_properties.py @@ -0,0 +1,30 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +from .pending_update import PendingUpdate + + +class UpdateProperties(PendingUpdate): + + def set(self, key, value): + raise NotImplementedError() + + def remove(self, key): + raise NotImplementedError() + + def default_format(self, file_format): + raise NotImplementedError() diff --git a/iceberg/api/update_schema.py b/iceberg/api/update_schema.py new file mode 100644 index 0000000000..22ef05e64a --- /dev/null +++ b/iceberg/api/update_schema.py @@ -0,0 +1,33 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +from .pending_update import PendingUpdate + + +class UpdateSchema(PendingUpdate): + + def add_column(self, name, type, parent=None): + raise NotImplementedError() + + def rename_column(self, name, new_name): + raise NotImplementedError() + + def update_column(self, name, new_type): + raise NotImplementedError() + + def delete_column(self, name): + raise NotImplementedError() diff --git a/iceberg/core/__init__.py b/iceberg/core/__init__.py new file mode 100644 index 0000000000..83fb8fc5b7 --- /dev/null +++ b/iceberg/core/__init__.py @@ -0,0 +1,59 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +__all__ = ["BaseMetastoreTableOperations", + "BaseMetastoreTables", + "BaseSnapshot", + "BaseTable", + "ConfigProperties", + "DataFiles", + "GenericDataFile", + "GenericManifestFile", + "ManifestEntry", + "ManifestListWriter", + "ManifestReader", + "PartitionSpecParser", + "PartitionData", + "SchemaParser", + "SchemaUpdate", + "SnapshotParser", + "TableOperations", + "SnapshotLogEntry", + "TableMetadata", + "TableMetadataParser", + "TableOperations"] + +from .base_metastore_table_operations import BaseMetastoreTableOperations +from .base_metastore_tables import BaseMetastoreTables +from .base_snapshot import BaseSnapshot +from .base_table import BaseTable +from .config_properties import ConfigProperties +from .data_files import DataFiles +from .generic_data_file import GenericDataFile +from .generic_manifest_file import GenericManifestFile +from .manifest_entry import ManifestEntry +from .manifest_list_writer import ManifestListWriter +from .manifest_reader import ManifestReader +from .partition_data import PartitionData +from .partition_spec_parser import PartitionSpecParser +from .schema_parser import SchemaParser +from .schema_update import SchemaUpdate +from .snapshot_parser import SnapshotParser +from .table_metadata import (SnapshotLogEntry, + TableMetadata) +from .table_metadata_parser import TableMetadataParser +from .table_operations import TableOperations diff --git a/iceberg/core/avro/__init__.py b/iceberg/core/avro/__init__.py new file mode 100644 index 0000000000..3d07a5b387 --- /dev/null +++ b/iceberg/core/avro/__init__.py @@ -0,0 +1,21 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +__all__ = ["AvroToIceberg", "IcebergToAvro"] + +from .avro_to_iceberg import AvroToIceberg +from .iceberg_to_avro import IcebergToAvro diff --git a/iceberg/core/avro/avro_schema_util.py b/iceberg/core/avro/avro_schema_util.py new file mode 100644 index 0000000000..efe729987b --- /dev/null +++ b/iceberg/core/avro/avro_schema_util.py @@ -0,0 +1,34 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +class AvroSchemaUtil(object): + FIELD_ID_PROP = "field-id" + KEY_ID_PROP = "key-id" + VALUE_ID_PROP = "value-id" + ELEMENT_ID_PROP = "element-id" + ADJUST_TO_UTC_PROP = "adjust-to_utc" + + # NULL = PrimitiveSchema(NULL) + + @staticmethod + def convert(iceberg_schema=None, avro_schema=None, table_name=None, names=None, + type_var=None, name=None): + if iceberg_schema is not None and table_name is not None: + return AvroSchemaUtil.convert(iceberg_schema=iceberg_schema, + names={iceberg_schema.as_struct(): table_name}) + elif iceberg_schema is not None and names is not None: + raise RuntimeError("Not yet implemented") diff --git a/iceberg/core/avro/avro_to_iceberg.py b/iceberg/core/avro/avro_to_iceberg.py new file mode 100644 index 0000000000..8e08f1ff05 --- /dev/null +++ b/iceberg/core/avro/avro_to_iceberg.py @@ -0,0 +1,289 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +from iceberg.api import Schema +from iceberg.api.types import (BinaryType, + BooleanType, + DateType, + DoubleType, + FloatType, + IntegerType, + ListType, + LongType, + MapType, + NestedField, + StringType, + StructType, + TimestampType, + TimeType, + TypeID) + + +class AvroToIceberg(object): + FIELD_ID_PROP = "field-id" + FIELD_TYPE_PROP = "type" + FIELD_NAME_PROP = "name" + FIELD_LOGICAL_TYPE_PROP = "logicalType" + FIELD_FIELDS_PROP = "fields" + FIELD_ITEMS_PROP = "items" + FIELD_ELEMENT_ID_PROP = "element-id" + + AVRO_JSON_PRIMITIVE_TYPES = ("boolean", "int", "long", "float", "double", "bytes", "string") + AVRO_JSON_COMPLEX_TYPES = ("record", "array", "enum", "fixed") + + TYPE_PROCESSING_MAP = {str: lambda x, y: AvroToIceberg.convert_str_type(x, y), + dict: lambda x, y: AvroToIceberg.convert_complex_type(x, y), + list: lambda x, y: AvroToIceberg.convert_union_type(x, y)} + + COMPLEX_TYPE_PROCESSING_MAP = {"record": lambda x, y: AvroToIceberg.convert_record_type(x, y), + "array": lambda x, y: AvroToIceberg.convert_array_type(x, y), + "map": lambda x, y: AvroToIceberg.convert_map_type(x, y)} + + PRIMITIVE_FIELD_TYPE_MAP = {"boolean": BooleanType.get(), + "bytes": BinaryType.get(), + "date": DateType.get(), + "double": DoubleType.get(), + "float": FloatType.get(), + "int": IntegerType.get(), + "long": LongType.get(), + "string": StringType.get(), + "time-millis": TimeType.get(), + "timestamp-millis": TimestampType.without_timezone() + } + + @staticmethod + def convert_avro_schema_to_iceberg(avro_schema): + if avro_schema.get(AvroToIceberg.FIELD_TYPE_PROP) != "record": + raise RuntimeError("Cannot convert avro schema to iceberg %s" % avro_schema) + + struct = AvroToIceberg.convert_type(avro_schema, None) + + return Schema(struct[0].fields) + + @staticmethod + def convert_record_type(avro_field, next_id=None): + avro_field_type = avro_field.get(AvroToIceberg.FIELD_TYPE_PROP) + + if avro_field_type != "record": + raise RuntimeError("Field type muse be 'record': %s" % avro_field_type) + + fields = avro_field.get(AvroToIceberg.FIELD_FIELDS_PROP) + + iceberg_fields = [] + if next_id is None: + next_id = len(fields) + for field in fields: + iceberg_field, next_id = AvroToIceberg.convert_avro_field_to_iceberg(field, next_id=next_id) + iceberg_fields.append(iceberg_field) + + return StructType.of(iceberg_fields), next_id + + @staticmethod + def convert_avro_field_to_iceberg(field, next_id): + field_type, is_optional, next_id = AvroToIceberg.convert_type(field, next_id) + + if field.get(AvroToIceberg.FIELD_ID_PROP) is None: + return field_type, next_id + + if is_optional: + return NestedField.optional(field.get(AvroToIceberg.FIELD_ID_PROP), + field.get(AvroToIceberg.FIELD_NAME_PROP), + field_type), next_id + else: + return NestedField.required(field.get(AvroToIceberg.FIELD_ID_PROP), + field.get(AvroToIceberg.FIELD_NAME_PROP), + field_type), next_id + + @staticmethod + def convert_type(field, next_id=None): + avro_field_type = field.get(AvroToIceberg.FIELD_TYPE_PROP) + + optional = AvroToIceberg.is_option_schema(avro_field_type) + + processing_func = AvroToIceberg.TYPE_PROCESSING_MAP.get(type(avro_field_type)) + if processing_func is None: + raise RuntimeError("No function found to process %s" % avro_field_type) + + iceberg_type, next_id = processing_func(field, next_id) + + return iceberg_type, optional, next_id + + @staticmethod + def convert_str_type(avro_field, next_id=None): + avro_field_type = avro_field.get(AvroToIceberg.FIELD_TYPE_PROP) + logical_type = avro_field.get(AvroToIceberg.FIELD_LOGICAL_TYPE_PROP) + if not isinstance(avro_field_type, str): + raise RuntimeError("Field type must be of type str: %s" % avro_field_type) + + if avro_field_type in AvroToIceberg.AVRO_JSON_PRIMITIVE_TYPES: + if logical_type is not None: + return AvroToIceberg.PRIMITIVE_FIELD_TYPE_MAP.get(logical_type), next_id + else: + return AvroToIceberg.PRIMITIVE_FIELD_TYPE_MAP.get(avro_field_type), next_id + + elif avro_field_type in AvroToIceberg.AVRO_JSON_COMPLEX_TYPES: + if logical_type is not None: + processing_func = AvroToIceberg.COMPLEX_TYPE_PROCESSING_MAP.get(logical_type) + else: + processing_func = AvroToIceberg.COMPLEX_TYPE_PROCESSING_MAP.get(avro_field_type) + + if processing_func is None: + raise RuntimeError("No function found to process %s" % avro_field_type) + + return processing_func(avro_field, next_id) + else: + raise RuntimeError("Unknown type %s" % avro_field_type) + + @staticmethod + def convert_complex_type(avro_field, next_id=None): + avro_field_type = avro_field.get(AvroToIceberg.FIELD_TYPE_PROP) + if not isinstance(avro_field_type, dict): + raise RuntimeError("Complex field type must be of type dict: %s" % avro_field_type) + + return AvroToIceberg.convert_avro_field_to_iceberg(avro_field_type, next_id) + + @staticmethod + def convert_union_type(avro_field, next_id=None): + avro_field_type = avro_field.get(AvroToIceberg.FIELD_TYPE_PROP) + if not isinstance(avro_field_type, list): + raise RuntimeError("Union field type must be of type list: %s" % avro_field_type) + + if len(avro_field_type) > 2: + raise RuntimeError("Cannot process unions larger than 2 items: %s" % avro_field_type) + for item in avro_field_type: + if isinstance(item, str) and item == "null": + continue + avro_field_type = item + avro_field[AvroToIceberg.FIELD_TYPE_PROP] = avro_field_type + items = AvroToIceberg.convert_type(avro_field, next_id) + return items[0], items[2] + + @staticmethod + def convert_array_type(avro_field, next_id=None): + avro_field_type = avro_field.get(AvroToIceberg.FIELD_TYPE_PROP) + if avro_field_type != "array": + raise RuntimeError("Avro type must be array: %s" % avro_field_type) + element_id = avro_field.get(AvroToIceberg.FIELD_ELEMENT_ID_PROP) + items = avro_field.get(AvroToIceberg.FIELD_ITEMS_PROP) + + is_optional = AvroToIceberg.is_option_schema(items) + + if isinstance(items, str) and items in AvroToIceberg.PRIMITIVE_FIELD_TYPE_MAP: + item_type = AvroToIceberg.PRIMITIVE_FIELD_TYPE_MAP.get(items) + if item_type is None: + raise RuntimeError("No mapping found for type %s" % items) + else: + raise RuntimeError("Complex list types not yet implemented") + + if is_optional: + return ListType.of_optional(element_id, item_type), next_id + else: + return ListType.of_required(element_id, item_type), next_id + + @staticmethod + def convert_map_type(avro_field, next_id=None): + avro_field_type = avro_field.get(AvroToIceberg.FIELD_TYPE_PROP) + avro_logical_type = avro_field.get(AvroToIceberg.FIELD_LOGICAL_TYPE_PROP) + if avro_field_type != "array" or avro_logical_type != "map": + raise RuntimeError("Avro type must be array and logical type must be map: %s" % (avro_field_type, + avro_logical_type)) + is_optional = False + items = avro_field.get(AvroToIceberg.FIELD_ITEMS_PROP) + for field in items.get(AvroToIceberg.FIELD_FIELDS_PROP, list()): + if field.get(AvroToIceberg.FIELD_NAME_PROP) == "key": + key_id = field.get(AvroToIceberg.FIELD_ID_PROP) + if not isinstance(field.get(AvroToIceberg.FIELD_TYPE_PROP), str): + raise RuntimeError("Support for complex map keys not yet implemented") + key_type = AvroToIceberg.PRIMITIVE_FIELD_TYPE_MAP.get(field.get(AvroToIceberg.FIELD_TYPE_PROP)) + elif field.get(AvroToIceberg.FIELD_NAME_PROP) == "value": + value_id = field.get(AvroToIceberg.FIELD_ID_PROP) + if not isinstance(field.get(AvroToIceberg.FIELD_TYPE_PROP), str): + raise RuntimeError("Support for complex map values not yet imeplemented") + value_type = AvroToIceberg.PRIMITIVE_FIELD_TYPE_MAP.get(field.get(AvroToIceberg.FIELD_TYPE_PROP)) + + if is_optional: + return MapType.of_optional(key_id, value_id, key_type, value_type), next_id + else: + return MapType.of_required(key_id, value_id, key_type, value_type), next_id + + @staticmethod + def is_option_schema(field_type): + if isinstance(field_type, list) and len(field_type) == 2 and "null" in field_type: + return True + + return False + + @staticmethod + def read_avro_row(iceberg_schema, avro_reader): + try: + avro_row = avro_reader.__next__() + iceberg_row = dict() + for field in iceberg_schema.as_struct().fields: + iceberg_row[field.name] = AvroToIceberg.get_field_from_avro(avro_row, field) + yield iceberg_row + except StopIteration: + return + + @staticmethod + def get_field_from_avro(avro_row, field): + process_funcs = {TypeID.STRUCT: lambda avro_row, field: AvroToIceberg.get_field_from_struct(avro_row, field), + TypeID.LIST: lambda avro_row, field: AvroToIceberg.get_field_from_list(avro_row, field), + TypeID.MAP: lambda avro_row, field: AvroToIceberg.get_field_from_map(avro_row, field)} + if field.type.is_primitive_type(): + processing_func = AvroToIceberg.get_field_from_primitive + else: + processing_func = process_funcs.get(field.type.type_id) + + if processing_func is None: + raise RuntimeError("Don't know how to get field of type: %s" % field.type.type_id) + + return processing_func(avro_row, field) + + @staticmethod + def get_field_from_primitive(avro_row, field): + avro_value = avro_row.get(field.name) + if avro_row is None and field.is_required: + raise RuntimeError("Field is required but missing in source %s\n%s:" % (field, avro_row)) + return avro_value + + @staticmethod + def get_field_from_struct(avro_row, field): + field_obj = {} + for nested_field in field.type.fields: + field_obj[nested_field.name] = AvroToIceberg.get_field_from_avro(avro_row[field.name], nested_field) + return field_obj + + @staticmethod + def get_field_from_list(avro_row, field): + avro_value = avro_row.get(field.name) + if avro_value is None: + if field.is_required: + raise RuntimeError("Field is required but missing in source %s\n%s:" % (field, avro_row)) + return None + + return avro_value + + @staticmethod + def get_field_from_map(avro_row, field): + val_map = dict() + avro_value = avro_row.get(field.name) + if avro_value is None and field.is_required: + raise RuntimeError("Field is required but missing in source %s\n%s:" % (field, avro_row)) + + for val in avro_value: + val_map[val['key']] = val['value'] + return val_map diff --git a/iceberg/core/avro/iceberg_to_avro.py b/iceberg/core/avro/iceberg_to_avro.py new file mode 100644 index 0000000000..29e4330b1b --- /dev/null +++ b/iceberg/core/avro/iceberg_to_avro.py @@ -0,0 +1,90 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +from iceberg.api.types import BinaryType, TypeID + + +class IcebergToAvro(object): + + @staticmethod + def type_to_schema(struct_type, name): + struct_fields = list() + for field in struct_type.fields: + struct_fields.append({"field-id": field.id, + "name": field.name, + "type": IcebergToAvro.get_field(field)}) + return {"type": "record", + "name": name, + "fields": struct_fields} + + @staticmethod + def get_field(field): + if field.type.is_primitive_type(): + return IcebergToAvro.to_option(field) + + elif field.type.type_id == TypeID.STRUCT: + struct_fields = list() + for struct_field in field.type.fields: + field_dict = {"field-id": struct_field.id, + "name": struct_field.name, + "type": IcebergToAvro.get_field(struct_field)} + if struct_field.is_optional: + field_dict["default"] = None + + struct_fields.append(field_dict) + + return {"fields": struct_fields, + "name": field.name, + "type": "record"} + + elif field.type.type_id == TypeID.LIST: + array_obj = {'element-id': field.type.element_id, + "items": IcebergToAvro.get_field(field.type.element_field), + "type": 'array'} + if field.is_optional: + return ['null', array_obj] + return array_obj + + elif field.type.type_id == TypeID.MAP: + key = field.type.key_field + value = field.type.value_field + array_obj = {"items": {"fields": [{"field-id": key.field_id, + "name": key.name, + "type": IcebergToAvro.get_field(key)}, + {"field-id": value.field_id, + "name": value.name, + "type": IcebergToAvro.get_field(value)}], + "name": "k{}_v{}".format(key.field_id, value.field_id), + "type": "record"}, + "logicalType": "map", + "type": "array"} + if field.is_optional: + return ["null", array_obj] + + return array_obj + + @staticmethod + def to_option(field): + if field.type == BinaryType.get(): + type_name = "bytes" + else: + type_name = str(field.type) + + if field.is_optional: + return ["null", type_name] + else: + return type_name diff --git a/iceberg/core/base_metastore_table_operations.py b/iceberg/core/base_metastore_table_operations.py new file mode 100644 index 0000000000..b536110e78 --- /dev/null +++ b/iceberg/core/base_metastore_table_operations.py @@ -0,0 +1,113 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +import logging +import uuid + +from iceberg.core.hadoop import (get_fs, + HadoopInputFile, + HadoopOutputFile) +from retrying import retry + +from .table_metadata_parser import TableMetadataParser +from .table_operations import TableOperations + +_logger = logging.getLogger(__name__) + + +class BaseMetastoreTableOperations(TableOperations): + + TABLE_TYPE_PROP = "table_type" + ICEBERG_TABLE_TYPE_VALUE = "iceberg" + METADATA_LOCATION_PROP = "metadata_location" + PREVIOUS_METADATA_LOCATION_PROP = "previous_metadata_location" + + METADATA_FOLDER_NAME = "metadata" + DATA_FOLDER_NAME = "data" + HIVE_LOCATION_FOLDER_NAME = "empty" + + def __init__(self, conf): + self.conf = conf + + self.current_metadata = None + self.current_metadata_location = None + self.base_location = None + self.version = -1 + + def current(self): + return self.current_metadata + + def hive_table_location(self): + return "{base_location}/{hive}".format(base_location=self.base_location, + hive=BaseMetastoreTableOperations.HIVE_LOCATION_FOLDER_NAME) + + def data_location(self): + return "{base_location}/{data}".format(base_location=self.base_location, + data=BaseMetastoreTableOperations.DATA_FOLDER_NAME) + + def write_new_metadata(self, metadata, version): + if self.base_location is None: + self.base_location = metadata.location + + new_filename = BaseMetastoreTableOperations.new_table_metadata_filename(self.base_location, + version) + new_metadata_location = HadoopOutputFile.from_path(new_filename, self.conf) + + TableMetadataParser.write(metadata, new_metadata_location) + return new_filename + + def refresh_from_metadata_location(self, new_location, num_retries=20): + if not self.current_metadata_location == new_location: + _logger.info("Refreshing table metadata from new version: %s" % new_location) + + self.retryable_refresh(new_location) + + def new_input_file(self, path): + return HadoopInputFile.from_location(path, self.conf) + + def new_metadata_file(self, filename): + return HadoopOutputFile.from_path(BaseMetastoreTableOperations.new_metadata_location(self.base_location, + filename), + self.conf) + + def delete_file(self, path): + get_fs(path, self.conf).delete(path, False) + + @retry(wait_incrementing_start=100, wait_exponential_multiplier=4, + wait_exponential_max=5000, stop_max_delay=600000, stop_max_attempt_number=2) + def retryable_refresh(self, location): + self.current_metadata = TableMetadataParser.read(self, HadoopInputFile.from_location(location, self.conf)) + self.current_metadata_location = location + self.base_location = self.current_metadata.location + self.version = BaseMetastoreTableOperations.parse_version(location) + + @staticmethod + def parse_version(metadata_location): + version_start = metadata_location.rfind("/") + 1 + version_end = version_start + metadata_location[version_start:].find("-") + return int(metadata_location[version_start:version_end]) + + @staticmethod + def new_metadata_location(base_location, filename): + return "{}/{}/{}".format(base_location, BaseMetastoreTableOperations.METADATA_FOLDER_NAME, filename) + + @staticmethod + def new_table_metadata_filename(base_location, new_version): + return "{}/{}/{}-{}.metadata.json".format(base_location, + BaseMetastoreTableOperations.METADATA_FOLDER_NAME, + '%05d' % new_version, + uuid.uuid4()) diff --git a/iceberg/core/base_metastore_tables.py b/iceberg/core/base_metastore_tables.py new file mode 100644 index 0000000000..bdb91de5ce --- /dev/null +++ b/iceberg/core/base_metastore_tables.py @@ -0,0 +1,53 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +from iceberg.api import Tables +from iceberg.exceptions import NoSuchTableException + +from .base_table import BaseTable + + +class BaseMetastoreTables(Tables): + + def __init__(self, conf): + self.conf = conf + + def new_table_ops(self, conf, database, table): + raise RuntimeError("Abstract Implementation") + + def load(self, database, table): + ops = self.new_table_ops(self.conf, database, table) + if ops.current() is None: + raise NoSuchTableException("Table does not exist: {}.{}".format(database, table)) + + return BaseTable(ops, "{}.{}".format(database, table)) + + def create(self, schema, spec, table_identifier=None, database=None, table=None): + raise RuntimeError("Not Yet Implemented") + + def begin_create(self, schema, spec, database, table_name, properties=None): + raise RuntimeError("Not Yet Implemented") + + def begin_replace(self, schema, spec, database, table, properties=None): + raise RuntimeError("Not Yet Implemented") + + def default_warehouse_location(self, conf, database, table): + warehouse_location = conf.get("hive.metastore.warehouse.dir") + if warehouse_location is None: + raise RuntimeError("Warehouse location is not set: hive.metastore.warehouse.dir=null") + + return "%s/%s.db/%s".format(warehouse_location, database, table) diff --git a/iceberg/core/base_snapshot.py b/iceberg/core/base_snapshot.py new file mode 100644 index 0000000000..b1c222c55a --- /dev/null +++ b/iceberg/core/base_snapshot.py @@ -0,0 +1,125 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +import time + +import fastavro +from iceberg.api import (Filterable, + FilteredSnapshot, + ManifestFile, + Snapshot, + SnapshotIterable) +from iceberg.api.expressions import Expressions +from iceberg.api.io import CloseableGroup +from iceberg.core.avro import AvroToIceberg + +from .generic_manifest_file import GenericManifestFile +from .manifest_reader import ManifestReader + + +class BaseSnapshot(Snapshot, SnapshotIterable, CloseableGroup): + + @staticmethod + def snapshot_from_files(ops, snapshot_id, files): + return BaseSnapshot(ops, snapshot_id, None, + manifests=[GenericManifestFile(file=ops.new_input_file(path), spec_id=0) + for path in files]) + + def __init__(self, ops, snapshot_id, parent_id=None, manifests=None, manifest_list=None, timestamp_millis=None, + operation=None, summary=None): + super(BaseSnapshot, self).__init__() + if timestamp_millis is None: + timestamp_millis = int(time.time() * 1000) + + self._ops = ops + self._snapshot_id = snapshot_id + self._parent_id = parent_id + self._timestamp_millis = timestamp_millis + if manifests is not None: + self._manifests = [manifest if isinstance(manifest, GenericManifestFile) + else GenericManifestFile(file=ops.new_input_file(manifest), spec_id=0) + for manifest in manifests] + else: + self._manifests = None + self._manifest_list = manifest_list + self.operation = operation + self.summary = summary + + self._adds = None + self._deletes = None + + @property + def snapshot_id(self): + return self._snapshot_id + + @property + def timestamp_millis(self): + return self._timestamp_millis + + @property + def parent_id(self): + return self._parent_id + + @property + def manifests(self): + if self._manifests is None: + # if manifest isn't set then the snapshot_file is set and should be read to get the list + with self._manifest_list.new_fo() as fo: + avro_reader = fastavro.reader(fo) + + self._manifests = [GenericManifestFile.from_avro_record_json(manifest) + for manifest in AvroToIceberg.read_avro_row(ManifestFile.schema(), avro_reader)] + + return self._manifests + + def select(self, columns): + return FilteredSnapshot(self, Expressions.always_true(), Expressions.always_true(), columns) + + def filter_partitions(self, expr): + return FilteredSnapshot(self, expr, Expressions.always_true(), Filterable.ALL_COLUMNS) + + def filter_rows(self, expr): + return FilteredSnapshot(self, Expressions.always_true(), expr, Filterable.ALL_COLUMNS) + + def iterator(self, part_filter=None, row_filter=None, columns=None): + if part_filter is None and row_filter is None and columns is None: + return self.iterator(Expressions.always_true(), Expressions.always_true(), Filterable.ALL_COLUMNS) + + return iter([self.get_filtered_manifest(path, part_filter, row_filter, columns) + for path in self._manifest_files]) + + def added_files(self): + raise NotImplementedError() + + def deleted_files(self): + raise NotImplementedError() + + def cache_changes(self): + raise NotImplementedError + + def __repr__(self): + return "BaseSnapshot(id={id},timestamp_ms={ts_ms},manifests={manifests}".format(id=self._snapshot_id, + ts_ms=self._timestamp_millis, + manifests=self._manifests) + + def __str__(self): + return self.__repr__() + + def get_filtered_manifest(self, path, part_filter, row_filter, columns): + reader = ManifestReader.read(self.ops.new_input_file(path)) + self.add_closeable(reader) + return reader diff --git a/iceberg/core/base_table.py b/iceberg/core/base_table.py new file mode 100644 index 0000000000..6c1d3b5971 --- /dev/null +++ b/iceberg/core/base_table.py @@ -0,0 +1,109 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +from iceberg.api import Table + +from .schema_update import SchemaUpdate + + +class BaseTable(Table): + + def __init__(self, ops, name): + self.ops = ops + self.name = name + + def refresh(self): + self.ops.refresh() + + def new_scan(self): + raise NotImplementedError() + + def schema(self): + return self.ops.current().schema + + def spec(self): + return self.ops.current().spec + + def properties(self): + return self.ops.current().properties + + def location(self): + return self.ops.current().location + + def current_snapshot(self): + return self.ops.current().current_snapshot() + + def snapshots(self): + return self.ops.current().snapshots + + def snapshots_with_summary_property(self, prop_key, prop_val): + if prop_key is None: + raise RuntimeError("Property Key cannot be None: (%s, %s)" % (prop_key, prop_val)) + + for snapshot in self.ops.current().snapshots: + if prop_key in snapshot.summary.keys() and snapshot.summary.get(prop_key) == prop_val: + yield snapshot + + def update_schema(self): + return SchemaUpdate(self.ops) + + def update_properties(self): + # PropertiesUpdate(self.ops) + raise NotImplementedError() + + def update_location(self): + # SetLocation(self.ops) + raise NotImplementedError() + + def new_append(self): + # MergeAppend(ops) + raise NotImplementedError() + + def new_fast_append(self): + # FastAppend(ops) + raise NotImplementedError() + + def new_rewrite(self): + # ReplaceFiles(ops) + raise NotImplementedError() + + def new_overwrite(self): + # OverwriteData(ops) + raise NotImplementedError() + + def new_replace_partitions(self): + # ReplacePartitionsOperation(ops) + raise NotImplementedError() + + def new_delete(self): + # StreamingDelete(ops) + raise NotImplementedError() + + def expire_snapshots(self): + # RemoveSnapshots(ops) + raise NotImplementedError() + + def rollback(self): + # RollbackToSnapshot(ops) + raise NotImplementedError() + + def new_transaction(self): + # BaseTransaction.newTransaction(ops) + raise NotImplementedError() + + def __str__(self): + return self.name diff --git a/iceberg/core/base_table_scan.py b/iceberg/core/base_table_scan.py new file mode 100644 index 0000000000..254b95f7f6 --- /dev/null +++ b/iceberg/core/base_table_scan.py @@ -0,0 +1,36 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + + +from iceberg.api import Filterable +from iceberg.api import TableScan +from iceberg.api.expressions import Expressions +from iceberg.api.io import CloseableGroup + + +class BaseTableScan(CloseableGroup, TableScan): + + def __init__(self, ops, table, snapshot_id=None, columns=None, row_filter=None): + self.ops = ops + self.table = table + self.snapshot_id = snapshot_id + self.columns = columns + self.row_filter = row_filter + + if self.columns is None and self.row_filter is None: + self.columns = Filterable.ALL_COLUMNS + self.row_filter = Expressions.always_true() diff --git a/iceberg/core/base_transaction.py b/iceberg/core/base_transaction.py new file mode 100644 index 0000000000..3d4f711b56 --- /dev/null +++ b/iceberg/core/base_transaction.py @@ -0,0 +1,176 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + + +import enum + + +from iceberg.api import (Table, + Transaction) +from iceberg.core import TableOperations +from iceberg.exceptions import CommitFailedException + + +class BaseTransaction(Transaction): + + @staticmethod + def replace_table_transaction(ops, start): + return BaseTransaction(ops, start) + + @staticmethod + def create_table_transaction(ops, start): + if ops.current() is not None: + raise RuntimeError("Cannot start create table transaction: table already exists") + + @staticmethod + def new_transaction(ops): + return BaseTransaction(ops, ops.refesh()) + + def __init__(self, ops, start): + self.ops = ops + self.updates = list() + self.intermediate_snapshot_ids = set() + self.base = ops.current + if self.base is None and start is None: + self.type = TransactionType.CREATE_TABLE + elif self.base is not None and start != self.base: + self.type = TransactionType.REPLACE_TABLE + else: + self.type = TransactionType.SIMPLE + + self.last_base = None + self.current = start + self.transaction_table = TransactionTable(self, self.current) + self.transaction_ops = TransactionTableOperations + + def table(self): + return self.transaction_table + + def check_last_operation_commited(self, operation): + if self.last_base == self.current: + raise RuntimeError("Cannot create new %s: last operation has not committed" % operation) + self.last_base = self.current + + def update_schema(self): + self.check_last_operation_commited("UpdateSchema") + + @staticmethod + def current_id(meta): + if meta is not None and meta.current_snapshot() is not None: + return meta.current_snapshot().snapshot_id + + +class TransactionType(enum.Enum): + + CREATE_TABLE = 0 + REPLACE_TABLE = 1 + SIMPLE = 1 + + +class TransactionTableOperations(TableOperations): + + def __init__(self, bt): + self._bt = bt + + def current(self): + return self._bt.current + + def refresh(self): + return self._bt.current + + def commit(self, base, metadata): + if base != self._current: + raise CommitFailedException("Table metadata refresh is required") + + old_id = BaseTransaction.current_id(self._bt.current) + if old_id is not None and old_id not in (BaseTransaction.current_id(metadata), + BaseTransaction.current_id(base)): + self._bt.intermediate_snapshot_ids.add(old_id) + + self._bt.current = metadata + + def io(self): + return self._bt.ops.io() + + def metadata_file_location(self, file): + return self._bt.ops.metadata_file_location(file) + + def new_snapshot_id(self): + return self._bt.ops.new_snapshot_id() + + +class TransactionTable(Table): + def __init__(self, bt, current): + self.bt = bt + self.current = current + + def refresh(self): + pass + + def new_scan(self): + raise RuntimeError("Transaction tables do not support scans") + + def schema(self): + return self.current.schema + + def spec(self): + return self.current.spec + + def properties(self): + return self.current.properties + + def location(self): + return self.current.location + + def current_snapshot(self): + return self.current.current_snapshot() + + def snapshots(self): + return self.current.snapshots + + def update_schema(self): + return self.bt.update_schema() + + def update_properties(self): + return self.bt.update_properties() + + def update_location(self): + return self.bt.update_location() + + def new_append(self): + return self.bt.new_append() + + def new_rewrite(self): + return self.bt.new_rewrite() + + def new_overwrite(self): + return self.bt.new_overwrite() + + def new_replace_partitions(self): + return self.bt.new_replace_partitions() + + def new_delete(self): + return self.bt.new_delete() + + def expire_snapshots(self): + return self.bt.expire_snapshots() + + def rollback(self): + raise RuntimeError("Transaction tables do not support rollback") + + def new_transaction(self): + raise RuntimeError("Cannot create a transaction within a transaction") diff --git a/iceberg/core/config_properties.py b/iceberg/core/config_properties.py new file mode 100644 index 0000000000..1cb2ecfb67 --- /dev/null +++ b/iceberg/core/config_properties.py @@ -0,0 +1,26 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + + +class ConfigProperties(object): + + COMPRESS_METADATA = "iceberg.compress.metadata" + COMPRESS_METADATA_DEFAULT = False + + @staticmethod + def should_compress(config): + return config.get(ConfigProperties.COMPRESS_METADATA, ConfigProperties.COMPRESS_METADATA_DEFAULT) diff --git a/iceberg/core/data_files.py b/iceberg/core/data_files.py new file mode 100644 index 0000000000..34d4422ac0 --- /dev/null +++ b/iceberg/core/data_files.py @@ -0,0 +1,225 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + + +from iceberg.api import (FileFormat, + Metrics) +from iceberg.api.types import Conversions + +from .generic_data_file import GenericDataFile +from .hadoop import HadoopInputFile +from .partition_data import PartitionData + + +class DataFiles(object): + DEFAULT_BLOCK_SIZE = 64 * 1024 * 1024 + + @staticmethod + def new_partition_data(spec): + return PartitionData(spec.partition_type()) + + @staticmethod + def copy_partition_data(spec, partition_data, reuse): + data = reuse + if data is None: + data = DataFiles.new_partition_data(spec) + + for i, field in enumerate(spec.fields): + data.set(i, partition_data.get(i)) + + return data + + @staticmethod + def fill_from_path(spec, partition_path, reuse): + data = reuse + if data is None: + data = DataFiles.new_partition_data(spec) + + partitions = partition_path.split("/") + if len(partitions) > len(spec.fields): + raise RuntimeError("Invalid partition data, too many fields (expecting %s): %s" % (len(spec.fields), + partition_path)) + + if len(partitions) < len(spec.fields): + raise RuntimeError("Invalid partition data, not enough fields(expecting %s): %s" % (len(spec.fields), + partition_path)) + + for i, part in enumerate(partitions): + field = spec.fields[i] + parts = part.split("=") + if len(parts) != 2 or parts[0] is None or parts[0] != field.name: + raise RuntimeError("Invalid partition: %s" % part) + + data.set(i, Conversions.from_partition_string(data.get_type(i), parts[1])) + + return data + + @staticmethod + def data(spec, partition_path): + return DataFiles.fill_from_path(spec, partition_path, None) + + @staticmethod + def copy(spec, partition): + return DataFiles.copy_partition_data(spec, partition, None) + + @staticmethod + def from_input_file(input_file, row_count, partition_data=None, metrics=None): + if isinstance(input_file, HadoopInputFile): + return DataFiles.from_stat(input_file.get_stat(), row_count, + partition_data=partition_data, metrics=metrics) + + @staticmethod + def from_stat(stat, row_count, partition_data=None, metrics=None): + location = stat.path + format = FileFormat.from_file_name(location) + return GenericDataFile(location, format, stat.length, stat.block_size, + row_count=row_count, partition=partition_data, metrics=metrics) + + @staticmethod + def builder(spec=None): + return DataFileBuilder(spec) + + +class DataFileBuilder(object): + + def __init__(self, spec=None): + self.spec = spec + self.partition_data = DataFiles.new_partition_data(spec) if spec is not None else None + self.is_partitioned = spec is not None + self.file_path = None + self.format = None + self.record_count = -1 + self.file_size_in_bytes = -1 + self.block_size_in_bytes = -1 + self.column_sizes = None + self.value_counts = None + self.null_value_counts = None + self.lower_bounds = None + self.upper_bounds = None + + def clear(self): + if self.is_partitioned: + self.partition_data.clear() + self.file_path = None + self.format = None + self.record_count = -1 + self.file_size_in_bytes = -1 + self.block_size_in_bytes = -1 + self.column_sizes = None + self.value_counts = None + self.null_value_counts = None + self.lower_bounds = None + self.upper_bounds = None + + def copy(self, to_copy): + if self.is_partitioned: + self.partition_data = DataFiles.copy_partition_data(self.spec, to_copy.partition, self.partition_data) + + self.file_path = str(to_copy.path()) + self.format = to_copy.format + self.record_count = to_copy.record_count + self.file_size_in_bytes = to_copy.file_size_in_bytes + self.block_size_in_bytes = to_copy.block_size_in_bytes + self.column_sizes = to_copy.column_sizes + self.value_counts = to_copy.value_counts + self.null_value_counts = to_copy.null_value_counts + self.lower_bounds = to_copy.lower_bounds + self.upper_bounds = to_copy.upper_bounds + return self + + def with_status(self, stat): + self.file_path = stat.path + self.file_size_in_bytes = stat.length + self.block_size_in_bytes = stat.blocksize + return self + + def with_input_file(self, input_file): + if isinstance(input_file, HadoopInputFile): + self.with_status(input_file.get_stat()) + + self.file_path = self.location() + self.file_size_in_bytes = self.get_length() + + return self + + def with_path(self, path): + self.file_path = path + + def with_format(self, fmt): + if isinstance(fmt, FileFormat): + self.format = fmt + else: + self.format = FileFormat[str(fmt).upper()] + + return self + + def with_partition(self, partition): + self.partition_data = DataFiles.copy_partition_data(self.spec, partition, self.partition_data) + return self + + def with_record_count(self, record_count): + self.record_count = record_count + return self + + def with_file_size_in_bytes(self, file_size_in_bytes): + self.file_size_in_bytes = file_size_in_bytes + return self + + def with_block_size_in_bytes(self, block_size_in_bytes): + self.block_size_in_bytes = block_size_in_bytes + return self + + def with_partition_path(self, partition_path): + if not self.is_partitioned: + raise RuntimeError("Cannot add partition data for an unpartitioned table") + + self.partition_data = DataFiles.fill_from_path(self.spec, partition_path, self.partition_data) + return self + + def with_metrics(self, metrics): + self.record_count = metrics.row_count if metrics.row_count is not None else -1 + self.column_sizes = metrics.column_sizes + self.value_counts = metrics.value_counts + self.null_value_counts = metrics.null_value_counts + self.lower_bounds = metrics.lower_bounds + self.upper_bounds = metrics.upper_bounds + return self + + def build(self): + if self.file_path is None: + raise RuntimeError("File path is required") + if self.format is None: + self.format = FileFormat.from_file_name(self.file_path) + + if self.format is None: + raise RuntimeError("File format is required") + + if self.file_size_in_bytes < 0: + raise RuntimeError("File size is required") + + if self.record_count < 0: + raise RuntimeError("Record count is required") + + if self.block_size_in_bytes is None: + self.block_size_in_bytes = DataFiles.DEFAULT_BLOCK_SIZE + + return GenericDataFile(self.file_path, self.format, self.file_size_in_bytes, self.block_size_in_bytes, + partition=self.partition_data.copy() if self.is_partitioned else None, + metrics=Metrics(row_count=self.record_count, column_sizes=self.column_sizes, + value_counts=self.value_counts, + null_value_counts=self.null_value_counts, + lower_bounds=self.lower_bounds, upper_bounds=self.upper_bounds)) diff --git a/iceberg/core/generic_data_file.py b/iceberg/core/generic_data_file.py new file mode 100644 index 0000000000..1f5e84f7a5 --- /dev/null +++ b/iceberg/core/generic_data_file.py @@ -0,0 +1,68 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + + +from iceberg.api import (DataFile, + StructLike) +from iceberg.api.types import StructType + +from .avro.iceberg_to_avro import IcebergToAvro +from .partition_data import PartitionData + + +class GenericDataFile(DataFile, StructLike): + + EMPTY_STRUCT_TYPE = StructType.of([]) + EMPTY_PARTITION_DATA = PartitionData(EMPTY_STRUCT_TYPE) + + def __init__(self, file_path, format, file_size_in_bytes, block_size_in_bytes, + row_count=None, partition=None, metrics=None): + self.file_path = file_path + self.format = format + self.row_count = row_count + self.file_size_in_bytes = file_size_in_bytes + self.block_size_in_bytes = block_size_in_bytes + self.file_ordinal = None + self.sort_columns = None + + if partition is None: + self.partition_data = GenericDataFile.EMPTY_PARTITION_DATA + self.partition_type = GenericDataFile.EMPTY_PARTITION_DATA.partition_type + else: + self.partition_data = partition + self.partition_type = partition.get_partition_type() + if metrics is None: + self.row_count = row_count + self.column_sizes = None + self.value_counts = None + self.null_value_counts = None + self.lower_bounds = None + self.upper_bounds = None + else: + self.row_count = metrics.row_count + self.column_sizes = metrics.column_sizes + self.value_counts = metrics.value_counts + self.null_value_counts = metrics.null_value_counts + self.lower_bounds = metrics.lower_bounds + self.upper_bounds = metrics.upper_bounds + + def partition(self): + return self.partition_data + + @staticmethod + def get_avro_schema(partition_type): + return IcebergToAvro.type_to_schema(DataFile.get_type(partition_type), DataFile.__class__.__name__) diff --git a/iceberg/core/generic_manifest_file.py b/iceberg/core/generic_manifest_file.py new file mode 100644 index 0000000000..73e669b3d0 --- /dev/null +++ b/iceberg/core/generic_manifest_file.py @@ -0,0 +1,179 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + + +import json + +from iceberg.api import (ManifestFile, + StructLike) +from iceberg.core.avro import IcebergToAvro + +from .generic_partition_field_summary import GenericPartitionFieldSummary + + +class GenericManifestFile(ManifestFile, StructLike): + AVRO_SCHEMA = IcebergToAvro.type_to_schema(ManifestFile.schema().as_struct(), "manifest_file") + + GET_POS_MAP = {0: lambda curr_manifest: curr_manifest.manifest_path, + 1: lambda curr_manifest: curr_manifest.lazy_length(), + 2: lambda curr_manifest: curr_manifest.spec_id, + 3: lambda curr_manifest: curr_manifest.snapshot_id, + 4: lambda curr_manifest: curr_manifest.added_files_count, + 5: lambda curr_manifest: curr_manifest.existing_files_count, + 6: lambda curr_manifest: curr_manifest.deleted_files_count, + 7: lambda curr_manifest: curr_manifest.partitions} + + @staticmethod + def generic_manifest_from_file(file, spec_id): + return GenericManifestFile(file=file, spec_id=spec_id) + + @staticmethod + def generic_manifest_from_path(path, length, spec_id, snapshot_id, + added_files_count, existing_files_count, deleted_files_count, + partitions): + return GenericManifestFile(path, length, spec_id, snapshot_id, + added_files_count, existing_files_count, deleted_files_count, + partitions) + + def __init__(self, path=None, file=None, spec_id=None, length=None, snapshot_id=None, + added_files_count=None, existing_files_count=None, deleted_files_count=None, + partitions=None): + if file is not None: + self.file = file + self.manifest_path = file.location() + else: + self.manifest_path = path + + self._length = length + self.spec_id = spec_id + self.snapshot_id = snapshot_id + self.added_files_count = added_files_count + self.existing_files_count = existing_files_count + self.deleted_files_count = deleted_files_count + self.partitions = partitions if partitions is not None else list() + self.from_projection_pos = None + + @property + def length(self): + return self.lazy_length() + + def lazy_length(self): + if self._length is None: + if self.file is not None: + self._length = self.file.get_length() + else: + return None + else: + return self._length + + def size(self): + return len(ManifestFile.schema().columns()) + + def get(self, pos, cast_type=None): + if self.from_projection_pos: + pos = self.from_projection_pos[pos] + + get_func = GenericManifestFile.GET_POS_MAP.get(pos) + if get_func is None: + raise RuntimeError("Unknown field ordinal: %s" % pos) + + if cast_type is not None: + return cast_type(get_func(self)) + + return get_func(self) + + def set(self, pos, value): + if self.from_projection_pos: + pos = self.from_projection_pos[pos] + + if pos == 0: + self.manifest_path = str(value) + elif pos == 1: + self.length = int(value) + elif pos == 2: + self.spec_id = int(value) + elif pos == 3: + self.snapshot_id = int(value) + elif pos == 4: + self.added_files_count = int(value) + elif pos == 5: + self.existing_files_count = int(value) + elif pos == 6: + self.deleted_files_count = int(value) + elif pos == 7: + self.partitions = value + + def copy(self): + return GenericManifestFile(path=self.manifest_path, spec_id=self.spec_id, length=self.length, + snapshot_id=self.snapshot_id, added_files_count=self.added_files_count, + existing_files_count=self.existing_files_count, + deleted_files_count=self.deleted_files_count, + partitions=list(self.partitions), avro_schema=self.avro_schema) + + @staticmethod + def get_schema(): + return GenericManifestFile.AVRO_SCHEMA + + @staticmethod + def to_avro_record_json(manifest): + return json.dumps(GenericManifestFile.to_avro_record_dict(manifest)) + + @staticmethod + def to_avro_record_dict(manifest): + return {"manifest_path": manifest.manifest_path, + "manifest_length": manifest._length, + "partition_spec_id": manifest.spec_id, + "added_snapshot_id": manifest.snapshot_id, + "added_data_files_count": manifest.added_files_count, + "existing_data_files_count": manifest.existing_files_count, + "deleted_data_files_count": manifest.deleted_files_count, + "partitions": manifest.partitions} + + @staticmethod + def from_avro_record_json(row): + return GenericManifestFile(path=row.get("manifest_path"), + length=row.get("manifest_length"), + spec_id=row.get("partition_spec_id"), + added_files_count=row.get("added_data_files_count"), + existing_files_count=row.get("existing_data_files_count"), + deleted_files_count=row.get("existing_data_files_count"), + partitions=[GenericPartitionFieldSummary(contains_null=partition["contains_null"], + lower_bound=partition["lower_bound"], + upper_bound=partition["upper_bound"]) + for partition in row.get("partitions")]) + + def __eq__(self, other): + if id(self) == id(other): + return True + + if other is None or not isinstance(other, GenericManifestFile): + return False + + return self.__key() == other.__key() + + def __hash__(self): + return hash(self.__key()) + + def __str__(self): + return self.__repr__() + + def __repr__(self): + return "GenericManifestFile({})".format(self.manifest_path) + + def __key(self): + return (GenericManifestFile.__class__, + self.manifest_path) diff --git a/iceberg/core/generic_partition_field_summary.py b/iceberg/core/generic_partition_field_summary.py new file mode 100644 index 0000000000..8fa054bd21 --- /dev/null +++ b/iceberg/core/generic_partition_field_summary.py @@ -0,0 +1,55 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + + +from iceberg.api import PartitionFieldSummary, StructLike + + +class GenericPartitionFieldSummary(PartitionFieldSummary, StructLike): + + AVRO_SCHEMA = None # IcebergToAvro.type_to_schema(PartitionFieldSummary.get_type()) + + def __init__(self, avro_schema=None, contains_null=False, lower_bound=None, upper_bound=None): + if avro_schema is None: + avro_schema = GenericPartitionFieldSummary.AVRO_SCHEMA + + self.avro_schema = avro_schema + self.contains_null = contains_null + self.lower_bound = lower_bound + self.upper_bound = upper_bound + + def __str__(self): + return ("GenericPartitionFieldSummary(contains_null={},lower_bound={}, upper_bound={}" + .format(self.contains_null, self.lower_bound, self.upper_bound)) + + def contains_null(self): + raise NotImplementedError() + + def get(self, pos): + raise NotImplementedError() + + def set(self, pos, value): + raise NotImplementedError() + + def lower_bound(self): + raise NotImplementedError() + + def upper_bound(self): + raise NotImplementedError() + + def copy(self): + raise NotImplementedError() diff --git a/iceberg/core/hadoop/__init__.py b/iceberg/core/hadoop/__init__.py new file mode 100644 index 0000000000..2ac455f573 --- /dev/null +++ b/iceberg/core/hadoop/__init__.py @@ -0,0 +1,22 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +__all__ = ["get_fs", "HadoopInputFile", "HadoopOutputFile"] + +from .hadoop_input_file import HadoopInputFile +from .hadoop_output_file import HadoopOutputFile +from .util import get_fs diff --git a/iceberg/core/hadoop/file_status.py b/iceberg/core/hadoop/file_status.py new file mode 100644 index 0000000000..bea4e1b0b3 --- /dev/null +++ b/iceberg/core/hadoop/file_status.py @@ -0,0 +1,34 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + + +class FileStatus(): + + def __init__(self, path=None, length=None, is_dir=None, block_replication=None, + blocksize=None, modification_time=None, access_time=None, permission=None, + owner=None, group=None, symlink=None): + self.path = path + self.length = length + self.is_dir = is_dir + self.block_replication = block_replication + self.block_size = blocksize + self.modification_time = modification_time + self.access_time = access_time + self.permission = permission + self.owner = owner + self.group = group + self.symlink = symlink diff --git a/iceberg/core/hadoop/file_system.py b/iceberg/core/hadoop/file_system.py new file mode 100644 index 0000000000..2630227688 --- /dev/null +++ b/iceberg/core/hadoop/file_system.py @@ -0,0 +1,25 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + + +class FileSystem(object): + + def open(self, path, mode='rb'): + raise NotImplementedError() + + def stat(self, path): + raise NotImplementedError() diff --git a/iceberg/core/hadoop/hadoop_input_file.py b/iceberg/core/hadoop/hadoop_input_file.py new file mode 100644 index 0000000000..66049dd11b --- /dev/null +++ b/iceberg/core/hadoop/hadoop_input_file.py @@ -0,0 +1,61 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +import gzip + +from iceberg.api.io import InputFile + +from .util import get_fs + + +class HadoopInputFile(InputFile): + + def __init__(self, fs, path, conf, length=None, stat=None): + self.fs = fs + self.path = path + self.conf = conf + self.length = length + self.stat = stat + + @staticmethod + def from_location(location, conf): + fs = get_fs(location, conf) + return HadoopInputFile(fs, location, conf) + + def location(self): + return self.path + + def get_length(self): + return self.get_stat().length + + def get_stat(self): + return self.lazy_stat() + + def lazy_stat(self): + if self.stat is None: + self.stat = self.fs.stat(self.path) + return self.stat + + def new_stream(self, gzipped=False): + with self.fs.open(self.location()) as fo: + if gzipped: + fo = gzip.GzipFile(fileobj=fo) + for line in fo: + yield line + + def new_fo(self): + return self.fs.open(self.location()) diff --git a/iceberg/core/hadoop/hadoop_output_file.py b/iceberg/core/hadoop/hadoop_output_file.py new file mode 100644 index 0000000000..4c0e2436d5 --- /dev/null +++ b/iceberg/core/hadoop/hadoop_output_file.py @@ -0,0 +1,49 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +from iceberg.api.io import OutputFile + +from .util import get_fs + + +class HadoopOutputFile(OutputFile): + + @staticmethod + def from_path(path, conf): + return HadoopOutputFile(path, conf) + + def __init__(self, path, conf): + self.path = path + self.conf = conf + + def create(self): + fs = get_fs(self.path, self.conf) + if fs.exists(): + raise RuntimeError("File %s already exists" % self.path) + + return fs.open(self.path) + + def create_or_overwrite(self): + fs = get_fs(self.path, self.conf) + + return fs.open(self.path, "wb") + + def location(self): + return str(self.path) + + def __str__(self): + return self.location() diff --git a/iceberg/core/hadoop/hadoop_table_operations.py b/iceberg/core/hadoop/hadoop_table_operations.py new file mode 100644 index 0000000000..e771084c4b --- /dev/null +++ b/iceberg/core/hadoop/hadoop_table_operations.py @@ -0,0 +1,49 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +import iceberg.core as core + + +class HadoopTableOperations(core.TableOperations): + + def __init__(self, location, conf): + self.conf = conf + self.location = location + self.should_refresh = True + self.verison = None + self.current_metadata = None + + def current(self): + pass + + def refresh(self): + raise RuntimeError("Not yet implemented") + + def commit(self, base, metadata): + raise RuntimeError("Not yet implemented") + + def new_input_file(self, path): + raise RuntimeError("Not yet implemented") + + def new_metadata_file(self, filename): + raise RuntimeError("Not yet implemented") + + def delete_file(self, path): + raise RuntimeError("Not yet implemented") + + def new_snapshot_id(self): + raise RuntimeError("Not yet implemented") diff --git a/iceberg/core/hadoop/local_filesystem.py b/iceberg/core/hadoop/local_filesystem.py new file mode 100644 index 0000000000..aa55de9cec --- /dev/null +++ b/iceberg/core/hadoop/local_filesystem.py @@ -0,0 +1,84 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +import os +import stat + +from .file_status import FileStatus +from .file_system import FileSystem + + +class LocalFileSystem(FileSystem): + fs_inst = None + + @staticmethod + def get_instance(): + if LocalFileSystem.fs_inst is None: + LocalFileSystem() + return LocalFileSystem.fs_inst + + def __init__(self): + if LocalFileSystem.fs_inst is None: + LocalFileSystem.fs_inst = self + + def open(self, path, mode='rb'): + + return open(LocalFileSystem.fix_path(path), mode=mode) + + def stat(self, path): + st = os.stat(LocalFileSystem.fix_path(path)) + return FileStatus(path=path, length=st.st_size, is_dir=stat.S_ISDIR(st.st_mode), + blocksize=st.st_blksize, modification_time=st.st_mtime, access_time=st.st_atime, + permission=st.st_mode, owner=st.st_uid, group=st.st_gid) + + @staticmethod + def fix_path(path): + if path.startswith("file://"): + path = str(path[7:]) + return path + + # def ls(self, path): + # return os.listdir(path) + # + # def delete(self, path, recursive=False): + # return os.remove(path) + # + + # + # def rename(self, path, new_path): + # return os.rename(path, new_path) + # + # def mkdir(self, path, create_parents=True): + # if create_parents: + # return os.makedirs(path) + # + # return os.mkdir(path) + # + # def exists(self, path): + # return os.path.isfile(path) + # + # def isdir(self, path): + # return os.path.isdir(path) + # + # def isfile(self, path): + # return os.path.isfile(path) + # + # def _isfilestore(self): + # return True + # + # def open(self, path, mode='rb'): + # return open(path, mode=mode) diff --git a/iceberg/core/hadoop/s3_filesystem_wrapper.py b/iceberg/core/hadoop/s3_filesystem_wrapper.py new file mode 100644 index 0000000000..f1fd713b95 --- /dev/null +++ b/iceberg/core/hadoop/s3_filesystem_wrapper.py @@ -0,0 +1,54 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +import s3fs + +from .file_status import FileStatus +from .file_system import FileSystem + + +class S3FileSystemWrapper(FileSystem): + + def __init__(self, anon=False, key=None, secret=None, token=None): + self._fs = s3fs.S3FileSystem(anon=anon, key=key, secret=secret, token=token) + + def open(self, path, mode='rb'): + return self._fs.open(S3FileSystemWrapper.fix_s3_path(path), mode=mode) + + def stat(self, path): + is_dir = False + st = dict() + try: + st = self._fs.info(S3FileSystemWrapper.fix_s3_path(path), refresh=True) + except RuntimeError: + # must be a directory or subsequent du will fail + du = self._fs.du(S3FileSystemWrapper.fix_s3_path(path), refresh=True) + if len(du) > 0: + is_dir = True + length = 0 + for key, val in du.items(): + length += val + + return FileStatus(path=path, length=st.get("Size", length), is_dir=is_dir, + blocksize=None, modification_time=st.get("LastModified"), access_time=None, + permission=None, owner=None, group=None) + + @staticmethod + def fix_s3_path(path): + if path.startswith("s3n"): + path = "s3" + str(path[3:]) + return path diff --git a/iceberg/core/hadoop/util.py b/iceberg/core/hadoop/util.py new file mode 100644 index 0000000000..8516e1eb37 --- /dev/null +++ b/iceberg/core/hadoop/util.py @@ -0,0 +1,53 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +import time +from urllib.parse import urlparse + +import boto3 + +from .local_filesystem import LocalFileSystem +from .s3_filesystem_wrapper import S3FileSystemWrapper + + +def get_fs(path, conf, local_only=False): + if local_only: + return LocalFileSystem.get_instance() + else: + parsed_path = urlparse(path) + + if parsed_path.scheme in ["", "file"]: + return LocalFileSystem.get_instance() + elif parsed_path.scheme in ["s3", "s3n"]: + if conf.get("hive.aws_iam_role") is not None: + key, secret, token = get_sts_session_keys(conf.get("hive.aws_iam_role")) + return S3FileSystemWrapper(key=key, secret=secret, token=token) + return S3FileSystemWrapper() + elif parsed_path.scheme in ["hdfs"]: + raise RuntimeError("Hadoop FS not implemented") + + raise RuntimeError("No filesystem found for this location: %s" % path) + + +def get_sts_session_keys(role_arn): + client = boto3.client('sts') + response = client.assume_role( + RoleArn=role_arn, + RoleSessionName='iceberg_python_client_{}'.format(int(time.time()))) + return (response["Credentials"]["AccessKeyId"], + response["Credentials"]["SecretAccessKey"], + response["Credentials"]["SessionToken"]) diff --git a/iceberg/core/manifest_entry.py b/iceberg/core/manifest_entry.py new file mode 100644 index 0000000000..b0ac5e9af9 --- /dev/null +++ b/iceberg/core/manifest_entry.py @@ -0,0 +1,141 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +from enum import Enum + +from iceberg.api import (DataFile, + Metrics, + Schema) +from iceberg.api.types import (IntegerType, + LongType, + NestedField) +from iceberg.core.avro import IcebergToAvro + +from .generic_data_file import GenericDataFile +from .partition_data import PartitionData + + +class ManifestEntry(): + AVRO_NAME = "manifest_entry" + + def __init__(self, schema=None, partition_type=None, to_copy=None): + self.schema = schema + self.snapshot_id = 0 + self.file = None + self.status = Status.EXISTING + + if self.schema is None and partition_type is not None: + self.schema = IcebergToAvro.type_to_schema(ManifestEntry.get_schema(partition_type)) + + elif self.schema is None and partition_type is None and to_copy is not None: + self.schema = to_copy.schema + self.status = to_copy.status + self.snapshot_id = to_copy.snapshot_id + self.file = to_copy.file.copy() + + if self.schema is None: + raise RuntimeError("Invalid arguments schema=%s, partition_type=%s, to_copy=%s" % (self.schema, + partition_type, + to_copy)) + + def wrap_existing(self, snapshot_id, file): + self.status = Status.EXISTING + self.snapshot_id = snapshot_id + self.file = file + return self + + def wrap_append(self, snapshot_id, file): + self.status = Status.ADDED + self.snapshot_id = snapshot_id + self.file = file + return self + + def wrap_delete(self, snapshot_id, file): + self.status = Status.DELETED + self.snapshot_id = snapshot_id + self.file = file + return self + + def copy(self): + return ManifestEntry(to_copy=self) + + def put(self, i, v): + if i == 0: + self.status = Status.from_id(i) + elif i == 1: + self.snapshot_id = v + elif i == 2: + if isinstance(v, dict): + metrics = Metrics(row_count=v.get("record_count"), + column_sizes=v.get("column_sizes"), + value_counts=v.get("value_counts"), + null_value_counts=v.get("null_value_counts"), + lower_bounds=v.get("lower_bounds"), + upper_bounds=v.get("upper_bounds")) + + data_file_schema = self.schema.as_struct().field(name="data_file") + part_data = PartitionData.from_json(data_file_schema + .type + .field(name="partition").type, v.get("partition")) + + v = GenericDataFile(v.get("file_path"), + v.get("file_format"), + v.get("file_size_in_bytes"), + v.get("block_size_in_byte"), + row_count=v.get("record_count"), + partition=part_data, + metrics=metrics + ) + self.file = v + + def get(self, i): + if i == 0: + return self.status.value + elif i == 1: + return self.snapshot_id + elif i == 2: + return self.file + + def __str__(self): + return "ManifestEntry(status=%s, snapshot_id=%s, file=%s" % (self.status, self.snapshot_id, self.file) + + @staticmethod + def project_schema(part_type, columns): + return ManifestEntry.wrap_file_schema(Schema(DataFile.get_type(part_type).fields) + .select(columns) + .as_struct()) + + @staticmethod + def get_schema(partition_type): + return ManifestEntry.wrap_file_schema(DataFile.get_type(partition_type)) + + @staticmethod + def wrap_file_schema(file_struct): + return Schema(NestedField.required(0, "status", IntegerType.get()), + NestedField.required(1, "snapshot_id", LongType.get()), + NestedField.required(2, "data_file", file_struct)) + + +class Status(Enum): + + EXISTING = 0 + ADDED = 1 + DELETED = 2 + + @staticmethod + def from_id(id): + return Status(id) diff --git a/iceberg/core/manifest_list_writer.py b/iceberg/core/manifest_list_writer.py new file mode 100644 index 0000000000..b3078edf53 --- /dev/null +++ b/iceberg/core/manifest_list_writer.py @@ -0,0 +1,55 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +import json + +from fastavro import parse_schema, writer +from iceberg.api import ManifestFile +from iceberg.api.io import FileAppender +from iceberg.core import GenericManifestFile +from iceberg.core.avro import IcebergToAvro + + +class ManifestListWriter(FileAppender): + + def __init__(self, snapshot_file, snapshot_id, parent_snapshot_id): + self.file = snapshot_file + self.meta = {"snapshot-id": str(snapshot_id), + "parent-snapshot-id": str(parent_snapshot_id)} + tmp_schema = IcebergToAvro.type_to_schema(ManifestFile.SCHEMA.as_struct(), + "manifest_file") + + self.schema = parse_schema(json.dumps(tmp_schema)) + + def add(self, d): + writer(self.file, + self.schema, + d, + metadata=self.meta) + + def add_all(self, values): + manifest_records = [GenericManifestFile.to_avro_record_dict(value) + for value in values if not isinstance(value, str)] + writer(self.file, + self.schema, + manifest_records) + + def close(self): + self.writer.flush() + + def metrics(self): + raise RuntimeError("Metrics not available") diff --git a/iceberg/core/manifest_reader.py b/iceberg/core/manifest_reader.py new file mode 100644 index 0000000000..812681e1f8 --- /dev/null +++ b/iceberg/core/manifest_reader.py @@ -0,0 +1,106 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +import fastavro +from iceberg.api import FileFormat, Filterable +from iceberg.api.expressions import Expressions +from iceberg.api.io import CloseableGroup + +from .avro import AvroToIceberg +from .manifest_entry import ManifestEntry, Status +from .partition_spec_parser import PartitionSpecParser +from .schema_parser import SchemaParser +from .table_metadata import TableMetadata + + +class ManifestReader(CloseableGroup, Filterable): + ALL_COLUMNS = ["*"] + CHANGE_COLUMNS = ["file_path", "file_format", "partition", "record_count", "file_size_in_bytes"] + + @staticmethod + def read(file): + return ManifestReader(file=file) + + def select(self, columns): + raise NotImplementedError() + + def filter_partitions(self, expr): + raise NotImplementedError() + + def filter_rows(self, expr): + raise NotImplementedError() + + @staticmethod + def in_memory(spec, entries): + return ManifestReader(spec=spec, entries=entries) + + def __init__(self, file=None, spec=None, entries=None): + self.file = file + self.schema = None + self.metadata = None + self.spec = spec + self._entries = entries + if self.file is not None: + self.__init_from_file() + else: + self.__init_from_spec() + + def __init_from_file(self): + try: + with self.file.new_fo() as fo: + avro_reader = fastavro.reader(fo) + self.metadata = avro_reader.metadata + except Exception as e: + raise e + + self.schema = SchemaParser.from_json(self.metadata.get("schema")) + spec_id = int(self.metadata.get("partition-spec-id", TableMetadata.INITIAL_SPEC_ID)) + self.spec = PartitionSpecParser.from_json_fields(self.schema, spec_id, self.metadata.get("partition-spec")) + + def __init_from_spec(self): + self.metadata = dict() + self.schema = self.spec.schema + + def entries(self, columns=None): + if columns is None: + columns = ManifestReader.ALL_COLUMNS + + format = FileFormat.from_file_name(self.file.location()) + if format is None: + raise RuntimeError("Unable to determine format of manifest: " + self.file) + + proj_schema = ManifestEntry.project_schema(self.spec.partition_type(), columns) + read_entries = list() + if format == FileFormat.AVRO: + with self.file.new_fo() as fo: + avro_reader = fastavro.reader(fo) + + for read_entry in AvroToIceberg.read_avro_row(proj_schema, avro_reader): + entry = ManifestEntry(schema=proj_schema, partition_type=self.spec.partition_type()) + for i, key in enumerate(read_entry.keys()): + entry.put(i, read_entry[key]) + read_entries.append(entry) + else: + raise RuntimeError("Invalid format for manifest file: " + format) + + return read_entries + + def iterator(self, part_filter=None, columns=None): + if part_filter is None and columns is None: + return self.iterator(Expressions.always_true(), Filterable.ALL_COLUMNS) + + return (entry.file for entry in self.entries if entry.status != Status.DELETED) diff --git a/iceberg/core/partition_data.py b/iceberg/core/partition_data.py new file mode 100644 index 0000000000..87f98895b7 --- /dev/null +++ b/iceberg/core/partition_data.py @@ -0,0 +1,91 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +import copy +import json + +from iceberg.api.struct_like import StructLike + + +class PartitionData(StructLike): + + def __init__(self, schema=None, partition_type=None): + if partition_type is None: + self.partition_type = schema.as_nested_type().as_struct_type() + else: + for field in partition_type.fields: + if not field.type.is_primitive_type(): + raise RuntimeError("Partitions cannot contain nested types: %s" % field.type) + self.partition_type = partition_type + # schema = PartitionData.get_schema(self.partition_type) + self.size = len(self.partition_type.fields) + self.data = list() + self.schema = schema + self.string_schema = str(schema) + + def clear(self): + self.data = list() + + def copy(self): + return copy.deepcopy(self) + + def get_partition_type(self): + return self.partition_type + + def __eq__(self, other): + if id(self) == id(other): + return True + + if other is None or not isinstance(other, PartitionData): + return False + + return self.partition_type == other.partition_type and self.data == other.data + + def __hash__(self): + return hash(self.__key()) + + def __key(self): + return PartitionData.__class__, self.partition_type, self.data + + def __ne__(self, other): + return not self.__eq__(other) + + def __str__(self): + return "PartitionData{%s}" % (",".join(["{}={}".format(self.partition_type.fields[i], + datum) for i, datum in enumerate(self.data)])) + + def put(self, i, v): + self.data.insert(i, v) + + def get(self, pos): + if pos > len(self.data): + return None + + return self.data[pos][1] + + @staticmethod + def from_json(schema, json_obj): + if isinstance(json_obj, str): + json_obj = json.loads(json_obj) + + data = PartitionData(partition_type=schema) + for i, kv_tuple in enumerate(json_obj.items()): + data.put(i, kv_tuple) + return data + + def set(self, pos, value): + raise NotImplementedError() diff --git a/iceberg/core/partition_spec_parser.py b/iceberg/core/partition_spec_parser.py new file mode 100644 index 0000000000..600904d53a --- /dev/null +++ b/iceberg/core/partition_spec_parser.py @@ -0,0 +1,89 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +import json + +from iceberg.api import PartitionSpec + + +class PartitionSpecParser(object): + + SPEC_ID = "spec-id" + FIELDS = "fields" + SOURCE_ID = "source-id" + TRANSFORM = "transform" + NAME = "name" + + @staticmethod + def to_json(spec, indent=None): + return json.dumps(PartitionSpecParser.to_dict(spec), indent=indent) + + @staticmethod + def to_dict(spec): + return {PartitionSpecParser.SPEC_ID: spec.spec_id, + PartitionSpecParser.FIELDS: PartitionSpecParser.to_json_fields(spec)} + + @staticmethod + def to_json_fields(spec): + return [{PartitionSpecParser.NAME: field.name, + PartitionSpecParser.TRANSFORM: str(field.transform), + PartitionSpecParser.SOURCE_ID: field.source_id} + for field in spec.fields] + + @staticmethod + def from_json(schema, json_obj): + if isinstance(json_obj, str): + json_obj = json.loads(json_obj) + + if not isinstance(json_obj, dict): + raise RuntimeError("Cannot parse partition spec, not an object: %s" % json_obj) + + spec_id = json_obj.get(PartitionSpecParser.SPEC_ID) + + builder = PartitionSpec.builder_for(schema).with_spec_id(spec_id) + fields = json_obj.get(PartitionSpecParser.FIELDS) + if not isinstance(fields, (list, tuple)): + raise RuntimeError("Cannot parse partition spec fields, not an array: %s" % fields) + + for element in fields: + if not isinstance(element, dict): + raise RuntimeError("Cannot parse partition field, not an object: %s" % element) + + builder.add(element.get(PartitionSpecParser.SOURCE_ID), + element.get(PartitionSpecParser.NAME), + element.get(PartitionSpecParser.TRANSFORM)) + + return builder.build() + + @staticmethod + def from_json_fields(schema, spec_id, json_obj): + builder = PartitionSpec.builder_for(schema).with_spec_id(spec_id) + + if isinstance(json_obj, str): + json_obj = json.loads(json_obj) + + if not isinstance(json_obj, list): + raise RuntimeError("Cannot parse partition spec fields, not an array: %s" % json_obj) + + for item in json_obj: + if not isinstance(item, dict): + raise RuntimeError("Cannot parse partition field, not an object: %s" % json_obj) + builder.add(item.get(PartitionSpecParser.SOURCE_ID), + item.get(PartitionSpecParser.NAME), + item.get(PartitionSpecParser.TRANSFORM)) + + return builder.build() diff --git a/iceberg/core/partition_summary.py b/iceberg/core/partition_summary.py new file mode 100644 index 0000000000..d7fbdb6324 --- /dev/null +++ b/iceberg/core/partition_summary.py @@ -0,0 +1,31 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + + +class PartitionSummary(object): + + def __init__(self, spec): + raise NotImplementedError() + + +class PartitionFieldStats(object): + + def __init__(self, type): + self.contains_null = False + self.type = type + self.min = None + self.max = None diff --git a/iceberg/core/schema_parser.py b/iceberg/core/schema_parser.py new file mode 100644 index 0000000000..0ab4fe8f25 --- /dev/null +++ b/iceberg/core/schema_parser.py @@ -0,0 +1,176 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +import json + +from iceberg.api import Schema +from iceberg.api.types import (from_primitive_string, + ListType, + MapType, + NestedField, + StructType, + Type, + TypeID) + + +class SchemaParser(object): + + TYPE = "type" + STRUCT = "struct" + LIST = "list" + MAP = "map" + FIELDS = "fields" + ELEMENT = "element" + KEY = "key" + VALUE = "value" + NAME = "name" + ID = "id" + ELEMENT_ID = "element-id" + KEY_ID = "key-id" + VALUE_ID = "value-id" + REQUIRED = "required" + ELEMENT_REQUIRED = "element-required" + VALUE_REQUIRED = "value-required" + + @staticmethod + def to_json(type_var, indent=None): + return json.dumps(SchemaParser.to_dict(type_var), indent=indent) + + @staticmethod + def to_dict(type_var): + if isinstance(type_var, Schema): + struct = type_var.as_struct() + elif isinstance(type_var, Type): + struct = type_var + + return SchemaParser._type_to_dict(struct) + + @staticmethod + def _type_to_dict(type_var): + if type_var.is_primitive_type(): + return SchemaParser._primitive_to_dict(type_var) + else: + nested = type_var.as_nested_type() + if type_var.type_id == TypeID.STRUCT: + return SchemaParser._struct_to_dict(nested) + elif type_var.type_id == TypeID.LIST: + return SchemaParser._list_to_dict(nested) + elif type_var.type_id == TypeID.MAP: + return SchemaParser._map_to_dict(nested) + else: + raise RuntimeError("Cannot write unknown type: %s" % type) + + @staticmethod + def _primitive_to_dict(type_var): + return str(type_var) + + @staticmethod + def _struct_to_dict(struct): + return {SchemaParser.TYPE: SchemaParser.STRUCT, + SchemaParser.FIELDS: [SchemaParser._struct_field_to_dict(field) for field in struct.fields]} + + @staticmethod + def _struct_field_to_dict(field): + return {SchemaParser.ID: field.field_id, + SchemaParser.NAME: field.name, + SchemaParser.REQUIRED: field.is_required, + SchemaParser.TYPE: SchemaParser._type_to_dict(field.type)} + + @staticmethod + def _list_to_dict(list_type): + return {SchemaParser.TYPE: SchemaParser.LIST, + SchemaParser.ELEMENT_ID: list_type.element_id, + SchemaParser.ELEMENT: SchemaParser._type_to_dict(list_type.element_type), + SchemaParser.ELEMENT_REQUIRED: list_type.is_element_required()} + + @staticmethod + def _map_to_dict(map_type): + return {SchemaParser.TYPE: SchemaParser.MAP, + SchemaParser.KEY_ID: map_type.key_id(), + SchemaParser.KEY: SchemaParser._type_to_dict(map_type.key_type()), + SchemaParser.VALUE_ID: map_type.value_id(), + SchemaParser.VALUE: SchemaParser._type_to_dict(map_type.value_type()), + SchemaParser.VALUE_REQUIRED: map_type.is_value_required()} + + @staticmethod + def type_from_dict(dict_obj): + if isinstance(dict_obj, (str, bool, int, float)): + return from_primitive_string(dict_obj) + else: + type_str = dict_obj.get(SchemaParser.TYPE) + if type_str.upper() == "STRUCT": + return SchemaParser.struct_from_dict(dict_obj) + elif type_str.upper() == "LIST": + return SchemaParser.list_from_dict(dict_obj) + elif type_str.upper() == "MAP": + return SchemaParser.map_from_dict(dict_obj) + else: + raise RuntimeError("Cannot parse type from dict: %s" % dict_obj) + + @staticmethod + def struct_from_dict(dict_obj): + struct_fields = list() + fields = dict_obj.get(SchemaParser.FIELDS) + for field in fields: + field_id = field.get(SchemaParser.ID) + field_name = field.get(SchemaParser.NAME) + field_type = SchemaParser.type_from_dict(field.get(SchemaParser.TYPE)) + + if field.get(SchemaParser.REQUIRED): + struct_fields.append(NestedField.required(field_id, field_name, field_type)) + else: + struct_fields.append(NestedField.optional(field_id, field_name, field_type)) + + return StructType.of(struct_fields) + + @staticmethod + def list_from_dict(dict_obj): + elem_id = dict_obj.get(SchemaParser.ELEMENT_ID) + elem_type = SchemaParser.type_from_dict(dict_obj.get(SchemaParser.ELEMENT)) + is_required = dict_obj.get(SchemaParser.REQUIRED) + + if is_required: + return ListType.of_required(elem_id, elem_type) + else: + return ListType.of_optional(elem_id, elem_type) + + @staticmethod + def map_from_dict(dict_obj): + key_id = dict_obj.get(SchemaParser.KEY_ID) + key_type = SchemaParser.type_from_dict(dict_obj.get(SchemaParser.KEY)) + + value_id = dict_obj.get(SchemaParser.VALUE_ID) + value_type = SchemaParser.type_from_dict(dict_obj.get(SchemaParser.VALUE)) + + is_required = dict_obj.get(SchemaParser.VALUE_REQUIRED) + + if is_required: + return MapType.of_required(key_id, value_id, key_type, value_type) + else: + return MapType.of_optional(key_id, value_id, key_type, value_type) + + @staticmethod + def from_json(json_obj): + if isinstance(json_obj, str): + type_var = SchemaParser.type_from_dict(json.loads(json_obj)) + else: + type_var = SchemaParser.type_from_dict(json_obj) + + if type_var is not None and type_var.is_nested_type() and type_var.as_nested_type().is_struct_type(): + return Schema(type_var.as_nested_type().as_struct_type().fields) + else: + raise RuntimeError("Cannot create schema, not a struct type: %s" % type_var) diff --git a/iceberg/core/schema_update.py b/iceberg/core/schema_update.py new file mode 100644 index 0000000000..8363801291 --- /dev/null +++ b/iceberg/core/schema_update.py @@ -0,0 +1,40 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +from iceberg.api import UpdateSchema + + +class SchemaUpdate(UpdateSchema): + TABLE_ROOT_ID = -1 + + def add_column(self, name, type, parent=None): + raise NotImplementedError() + + def rename_column(self, name, new_name): + raise NotImplementedError() + + def update_column(self, name, new_type): + raise NotImplementedError() + + def delete_column(self, name): + raise NotImplementedError() + + def apply(self): + raise NotImplementedError() + + def commit(self): + raise NotImplementedError() diff --git a/iceberg/core/snapshot_parser.py b/iceberg/core/snapshot_parser.py new file mode 100644 index 0000000000..a1b7c96f11 --- /dev/null +++ b/iceberg/core/snapshot_parser.py @@ -0,0 +1,72 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +import json + +from .base_snapshot import BaseSnapshot +from .generic_manifest_file import GenericManifestFile + + +class SnapshotParser(object): + SNAPSHOT_ID = "snapshot-id" + PARENT_SNAPSHOT_ID = "parent-snapshot-id" + TIMESTAMP_MS = "timestamp-ms" + SUMMARY = "summary" + OPERATION = "operation" + MANIFESTS = "manifests" + MANIFEST_LIST = "manifest-list" + + @staticmethod + def to_json(snapshot): + return json.dumps(SnapshotParser.to_dict(snapshot)) + + @staticmethod + def to_dict(snapshot): + return {SnapshotParser.SNAPSHOT_ID: snapshot.snapshot_id, + SnapshotParser.TIMESTAMP_MS: snapshot.timestamp_millis, + SnapshotParser.PARENT_SNAPSHOT_ID: snapshot.parent_id, + SnapshotParser.MANIFESTS: [manifest.manifest_path for manifest in snapshot.manifests]} + + @staticmethod + def from_json(ops, json_obj): + if isinstance(json_obj, str): + json_obj = json.loads(json_obj) + + if not isinstance(json_obj, dict): + raise RuntimeError("Cannot parse table version from a non-object: %s" % json_obj) + + version_id = json_obj.get(SnapshotParser.SNAPSHOT_ID) + parent_id = json_obj.get(SnapshotParser.PARENT_SNAPSHOT_ID) + timestamp_millis = json_obj.get(SnapshotParser.TIMESTAMP_MS) + operation = json_obj.get(SnapshotParser.OPERATION) + summary = json_obj.get(SnapshotParser.SUMMARY) + + if SnapshotParser.MANIFEST_LIST in json_obj: + # the manifest list is stored in a manifest list file + return BaseSnapshot(ops, version_id, parent_id, + manifest_list=ops.new_input_file(json_obj.get(SnapshotParser.MANIFEST_LIST)), + timestamp_millis=timestamp_millis, + operation=operation, + summary=summary) + else: + manifests = [GenericManifestFile.generic_manifest_from_file(ops.new_input_file(location), 0) # noqa: F841 + for location in json_obj.get(SnapshotParser.MANIFESTS, list())] + return BaseSnapshot(ops, version_id, parent_id, + manifests=manifests, + timestamp_millis=timestamp_millis, + operation=operation, + summary=summary) diff --git a/iceberg/core/table_metadata.py b/iceberg/core/table_metadata.py new file mode 100644 index 0000000000..b816f90a50 --- /dev/null +++ b/iceberg/core/table_metadata.py @@ -0,0 +1,189 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +import time + +from iceberg.api import PartitionSpec +from iceberg.api.types import assign_fresh_ids +from iceberg.core.util import AtomicInteger +from iceberg.exceptions import ValidationException + + +class TableMetadata(object): + INITIAL_SPEC_ID = 0 + TABLE_FORMAT_VERSION = 1 + + @staticmethod + def new_table_metadata(ops, schema, spec, location): + last_column_id = AtomicInteger(0) + fresh_schema = assign_fresh_ids(schema, last_column_id.increment_and_get) + + spec_builder = PartitionSpec.builder_for(fresh_schema) + for field in spec.fields: + src_name = schema.find_column_name(field.source_id) + spec_builder.add(fresh_schema.find_field(src_name), + field, + str(field.fransform())) + + fresh_spec = spec_builder.build() + return TableMetadata(ops, None, location, + int(time.time() * 1000), + last_column_id.get(), fresh_schema, TableMetadata.INITIAL_SPEC_ID, [fresh_spec], + dict(), -1, list(), list()) + + def __init__(self, ops, file, location, last_updated_millis, + last_column_id, schema, default_spec_id, specs, properties, + current_snapshot_id, snapshots, snapshot_log): + self.ops = ops + self.file = file + self.location = location + self.last_updated_millis = last_updated_millis + self.last_column_id = last_column_id + self.schema = schema + self.default_spec_id = default_spec_id + self.specs = specs + self.properties = properties + self.current_snapshot_id = current_snapshot_id + self.snapshots = snapshots + self.snapshot_log = snapshot_log + + self.snapshot_by_id = {version.snapshot_id: version for version in self.snapshots} + self.specs_by_id = {spec.spec_id: spec for spec in self.specs} + + last = None + for log_entry in snapshot_log: + if last is not None: + if not (log_entry.timestamp_millis - last.timestamp_millis > 0): + raise RuntimeError("[BUG] Expected sorted snapshot log entries.") + last = log_entry + + if not (len(self.snapshot_by_id) == 0 or self.current_snapshot_id in self.snapshot_by_id): + raise RuntimeError("Invalid table metadata: Cannot find current version") + + @property + def spec(self): + return self.specs_by_id[self.default_spec_id] + + def spec_id(self, spec_id): + if spec_id is None: + spec_id = self.default_spec_id + + return self.specs_by_id[spec_id] + + def property_as_int(self, property_name, default_value): + return int(self.properties.get(property_name, default_value)) + + def current_snapshot(self): + return self.snapshot_by_id[self.current_snapshot_id] + + def snapshot(self, snapshot_id): + return self.snapshot_by_id[snapshot_id] + + def update_metadata_location(self, new_location): + return TableMetadata(self.ops, None, new_location, + int(time.time() * 1000), self.last_column_id, self.schema, self.spec, self.properties, + self.current_snapshot_id, self.snapshots, self.snapshot_log) + + def update_schema(self, schema, last_column_id): + PartitionSpec.check_compatibility(self.spec, schema) + return TableMetadata(self.ops, None, self.location, + int(time.time() * 1000), last_column_id, schema, self.spec, self.properties, + self.current_snapshot_id, self.snapshots, self.snapshot_log) + + def add_snapshot(self, snapshot): + new_snapshots = self.snapshots + snapshot + new_snapshot_log = self.snapshot_log + [SnapshotLogEntry(snapshot.timestamp_millis, snapshot.snapshot_id)] + + return TableMetadata(self.ops, None, self.location, + int(time.time() * 1000), self.last_column_id, self.schema, self.spec, self.properties, + self.current_snapshot_id, new_snapshots, new_snapshot_log) + + def remove_snapshots_if(self, remove_if): + filtered = list() + + for snapshot in self.snapshots: + if snapshot.snapshot_id == self.current_snapshot_id or not remove_if(snapshot): + filtered.append(snapshot) + + valid_ids = [snapshot.snapshot_id for snapshot in filtered] + new_snapshot_log = list() + for log_entry in self.snapshot_log: + if log_entry.snapshot_id in valid_ids: + new_snapshot_log.append(log_entry) + else: + new_snapshot_log.clear() + + return TableMetadata(self.ops, None, self.location, + int(time.time() * 1000), self.last_column_id, self.schema, self.spec, self.properties, + self.current_snapshot_id, filtered, new_snapshot_log) + + def rollback_to(self, snapshot): + ValidationException.check(snapshot.snapshot_id not in self.snapshot_by_id, + "Cannot set current snapshot to unknown: %s", (snapshot.snapshot_id,)) + + now_millis = int(time.time() * 1000) + new_snapshot_log = self.snapshot_log + [SnapshotLogEntry(now_millis, snapshot.snapshot_id)] + + return TableMetadata(self.ops, None, self.location, + now_millis, self.last_column_id, self.schema, self.spec, self.properties, + snapshot.snapshot_id, self.snapshots, new_snapshot_log) + + def replace_properties(self, new_properties): + ValidationException.check(new_properties is not None, "Cannot set properties to null") + + return TableMetadata(self.ops, None, self.location, + int(time.time() * 1000), self.last_column_id, self.schema, self.spec, new_properties, + self.current_snapshot_id, self.snapshots, self.snapshot_log) + + def remove_snapshot_log_entries(self, snapshot_ids): + new_snapshot_log = list() + + for entry in self.snapshot_log: + if entry.snapshot_id not in snapshot_ids: + new_snapshot_log.append(entry) + + check_snapshot = self.current_snapshot_id < 0 or new_snapshot_log[-1].snapshot_id == self.current_snapshot_id + ValidationException.check(check_snapshot, + "Cannot set invalid snapshot log: latest entry is not the current snapshot") + + return TableMetadata(self.ops, None, self.location, + int(time.time() * 1000), self.last_column_id, self.schema, self.spec, self.properties, + self.current_snapshot_id, self.snapshots, new_snapshot_log) + + +class SnapshotLogEntry(object): + + def __init__(self, timestamp_millis, snapshot_id): + self.timestamp_millis = timestamp_millis + self.snapshot_id = snapshot_id + + def __hash__(self): + return hash(self.__key()) + + def __key(self): + return SnapshotLogEntry.__class__, self.snapshot_id, self.timestamp_millis + + def __eq__(self, other): + if id(self) == id(other): + return True + if other is None or not isinstance(other, SnapshotLogEntry): + return False + + return self.snapshot_id == other.snapshot_id and self.timestamp_millis == other.timestamp_millis + + def __ne__(self, other): + return not self.__eq__() diff --git a/iceberg/core/table_metadata_parser.py b/iceberg/core/table_metadata_parser.py new file mode 100644 index 0000000000..1f364c5d6a --- /dev/null +++ b/iceberg/core/table_metadata_parser.py @@ -0,0 +1,124 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +import gzip +import json + +from .config_properties import ConfigProperties +from .partition_spec_parser import PartitionSpecParser +from .schema_parser import SchemaParser +from .snapshot_parser import SnapshotParser +from .table_metadata import (SnapshotLogEntry, + TableMetadata) + + +class TableMetadataParser(object): + FORMAT_VERSION = "format-version" + LOCATION = "location" + LAST_UPDATED_MILLIS = "last-updated-ms" + LAST_COLUMN_ID = "last-column-id" + SCHEMA = "schema" + PARTITION_SPEC = "partition-spec" + PARTITION_SPECS = "partition-specs" + DEFAULT_SPEC_ID = "default-spec-id" + PROPERTIES = "properties" + CURRENT_SNAPSHOT_ID = "current-snapshot-id" + SNAPSHOTS = "snapshots" + SNAPSHOT_ID = "snapshot-id" + TIMESTAMP_MS = "timestamp-ms" + LOG = "snapshot-log" + + @staticmethod + def to_json(metadata): + return json.dumps({TableMetadataParser.FORMAT_VERSION: TableMetadata.TABLE_FORMAT_VERSION, + TableMetadataParser.LOCATION: metadata.location, + TableMetadataParser.LAST_UPDATED_MILLIS: metadata.last_updated_millis, + TableMetadataParser.LAST_COLUMN_ID: metadata.last_column_id, + TableMetadataParser.SCHEMA: SchemaParser.to_dict(metadata.schema), + TableMetadataParser.PARTITION_SPEC: PartitionSpecParser.to_json_fields(metadata.spec), + TableMetadataParser.DEFAULT_SPEC_ID: int(metadata.default_spec_id), + TableMetadataParser.PARTITION_SPECS: [PartitionSpecParser.to_dict(spec) + for spec in metadata.specs], + TableMetadataParser.PROPERTIES: metadata.properties, + TableMetadataParser.CURRENT_SNAPSHOT_ID: (metadata.current_snapshot_id + if metadata.current_snapshot_id is not None + else -1), + TableMetadataParser.SNAPSHOTS: [SnapshotParser.to_dict(snapshot) + for snapshot in metadata.snapshots], + TableMetadataParser.LOG: [{TableMetadataParser.TIMESTAMP_MS: log_entry.timestamp_millis, + TableMetadataParser.SNAPSHOT_ID: log_entry.snapshot_id} + for log_entry in metadata.snapshot_log]}) + + @staticmethod + def write(metadata, metadata_location): + if metadata_location.location().endswith(".gz"): + output_file = gzip.open(metadata_location.location(), "wt") + else: + output_file = open(metadata_location.location(), "w") + + json_str = TableMetadataParser.to_json(metadata) + output_file.write(json_str) + output_file.close() + + @staticmethod + def get_file_extension(config): + return ".metadata.json.gz" if ConfigProperties.should_compress(config) else ".metadata.json" + + @staticmethod + def read(ops, file): + metadata = "".join([line.decode("utf-8") for line in file.new_stream(gzipped=file.location().endswith("gz"))]) + return TableMetadataParser.from_json(ops, file.location(), metadata) + + @staticmethod + def from_json(ops, file, json_obj): + if isinstance(json_obj, str): + json_obj = json.loads(json_obj) + + if not isinstance(json_obj, dict): + raise RuntimeError("Cannot parse metadata from non-object: %s" % json_obj) + + format_version = json_obj.get(TableMetadataParser.FORMAT_VERSION) + if format_version != TableMetadata.TABLE_FORMAT_VERSION: + raise RuntimeError("Cannot read unsupported version: %s" % format_version) + + location = json_obj.get(TableMetadataParser.LOCATION) + last_assigned_column = json_obj.get(TableMetadataParser.LAST_COLUMN_ID) + schema = SchemaParser.from_json(json_obj.get(TableMetadataParser.SCHEMA)) + + spec_array = json_obj.get(TableMetadataParser.PARTITION_SPECS) + if spec_array is not None: + default_spec_id = json_obj.get(TableMetadataParser.DEFAULT_SPEC_ID) + specs = [PartitionSpecParser.from_json(schema, spec) + for spec in spec_array] + else: + default_spec_id = TableMetadata.INITIAL_SPEC_ID + specs = (PartitionSpecParser.from_json_fields(schema, + default_spec_id, + json_obj.get(TableMetadataParser.PARTITION_SPEC)),) + + props = json_obj.get(TableMetadataParser.PROPERTIES) + current_version_id = json_obj.get(TableMetadataParser.CURRENT_SNAPSHOT_ID) + last_updated_millis = json_obj.get(TableMetadataParser.LAST_UPDATED_MILLIS) + snapshots = [SnapshotParser.from_json(ops, snapshot) for snapshot in json_obj.get(TableMetadataParser.SNAPSHOTS)] + entries = [SnapshotLogEntry(log_entry.get(TableMetadataParser.TIMESTAMP_MS), + log_entry.get(TableMetadataParser.SNAPSHOT_ID)) + for log_entry in sorted(json_obj.get(TableMetadataParser.LOG, []), + key=lambda x: x.get(TableMetadataParser.TIMESTAMP_MS))] + + return TableMetadata(ops, file, location, + last_updated_millis, last_assigned_column, schema, default_spec_id, specs, props, current_version_id, + snapshots, entries) diff --git a/iceberg/core/table_operations.py b/iceberg/core/table_operations.py new file mode 100644 index 0000000000..659b9b8b05 --- /dev/null +++ b/iceberg/core/table_operations.py @@ -0,0 +1,42 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +import struct +import uuid + + +class TableOperations(object): + + def current(self): + raise NotImplementedError() + + def refresh(self): + raise NotImplementedError() + + def commit(self, base, metadata): + raise NotImplementedError() + + def io(self): + raise NotImplementedError() + + def metadata_file_location(self, file): + raise NotImplementedError() + + def new_snapshot_id(self): + new_uuid = uuid.uuid4() + msb, lsb = struct.unpack(">qq", new_uuid.bytes) + return abs(msb ^ lsb) diff --git a/iceberg/core/util/__init__.py b/iceberg/core/util/__init__.py new file mode 100644 index 0000000000..5df92bc141 --- /dev/null +++ b/iceberg/core/util/__init__.py @@ -0,0 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +__all__ = ["AtomicInteger"] + +from .atomic_integer import AtomicInteger diff --git a/iceberg/core/util/atomic_integer.py b/iceberg/core/util/atomic_integer.py new file mode 100644 index 0000000000..87d70236a6 --- /dev/null +++ b/iceberg/core/util/atomic_integer.py @@ -0,0 +1,36 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +from threading import RLock + + +class AtomicInteger(object): + + def __init__(self, value): + if not isinstance(value, int): + raise RuntimeError("Value %s is not an int" % value) + self.value = value + self._lock = RLock() + + def increment_and_get(self): + with self._lock: + self.value += 1 + return self.value + + def get(self): + with self._lock: + return self.value diff --git a/iceberg/exceptions/__init__.py b/iceberg/exceptions/__init__.py new file mode 100644 index 0000000000..e6810ef4e9 --- /dev/null +++ b/iceberg/exceptions/__init__.py @@ -0,0 +1,26 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +__all__ = ["AlreadyExistsException", + "CommitFailedException", + "NoSuchTableException", + "ValidationException"] + +from .exceptions import (AlreadyExistsException, + CommitFailedException, + NoSuchTableException, + ValidationException) diff --git a/iceberg/exceptions/exceptions.py b/iceberg/exceptions/exceptions.py new file mode 100644 index 0000000000..3a1072e7bb --- /dev/null +++ b/iceberg/exceptions/exceptions.py @@ -0,0 +1,39 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + + +class AlreadyExistsException(RuntimeError): + pass + + +class CommitFailedException(RuntimeError): + pass + + +class NoSuchTableException(RuntimeError): + pass + + +class ValidationException(RuntimeError): + + @staticmethod + def check(test, message, args): + if not test: + raise ValidationException(message, args) + + def __init__(self, message, args): + super(ValidationException, self).__init__(message % args) diff --git a/iceberg/spark/__init__.py b/iceberg/spark/__init__.py new file mode 100644 index 0000000000..ede72c06ae --- /dev/null +++ b/iceberg/spark/__init__.py @@ -0,0 +1,19 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# flake8: noqa + +from .table_identifier import TableIdentifier diff --git a/iceberg/spark/source/__init__.py b/iceberg/spark/source/__init__.py new file mode 100644 index 0000000000..cc91ef472b --- /dev/null +++ b/iceberg/spark/source/__init__.py @@ -0,0 +1,19 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# flake8: noqa + +from .spark_catalog import SparkCatalog diff --git a/iceberg/spark/source/spark_catalog.py b/iceberg/spark/source/spark_catalog.py new file mode 100644 index 0000000000..858be62ce5 --- /dev/null +++ b/iceberg/spark/source/spark_catalog.py @@ -0,0 +1,53 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +class SparkCatalog(object): + + def create(self, ident, schema, spec, properties): + pass + + def load(self, ident): + pass + + def drop(self, ident): + pass + + def load_table(self, ident): + pass + + def create_table(self, ident, table_type, partitions, properties): + pass + + def apply_property_changes(self, table, changes): + pass + + def apply_schema_changes(self, table, changes): + pass + + def drop_table(self, ident): + pass + + def initialize(self, name, options): + self.name = name + self.options = options + + def convert(self, schema, partitioning): + pass + + def __init__(self): + self.name = "" + self.options = None diff --git a/iceberg/spark/table_identifier.py b/iceberg/spark/table_identifier.py new file mode 100644 index 0000000000..84d04f1309 --- /dev/null +++ b/iceberg/spark/table_identifier.py @@ -0,0 +1,21 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +class TableIdentifier(object): + + def __init__(self): + pass diff --git a/setup.py b/setup.py new file mode 100755 index 0000000000..af55914a2d --- /dev/null +++ b/setup.py @@ -0,0 +1,39 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +from setuptools import setup + + +setup( + name='iceberg', + maintainer='Apache Incubator Iceberg Devs', + author_email='dev@iceberg.apache.org', + description='Iceberg is a new table format for storing large, slow-moving tabular data', + keywords='iceberg', + url='https://github.com/apache/incubator-iceberg/blob/master/README.md', + python_requires='>=3.4', + install_requires=['boto3', + 'fastavro', + 'mmh3', + "pytest", + 'python-dateutil', + 'pytz', + 'retrying', + 's3fs', + 'six'], + setup_requires=['setupmeta'] +) diff --git a/tests/__init__.py b/tests/__init__.py new file mode 100644 index 0000000000..245692337b --- /dev/null +++ b/tests/__init__.py @@ -0,0 +1,17 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + diff --git a/tests/api/__init__.py b/tests/api/__init__.py new file mode 100644 index 0000000000..245692337b --- /dev/null +++ b/tests/api/__init__.py @@ -0,0 +1,17 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + diff --git a/tests/api/expressions/__init__.py b/tests/api/expressions/__init__.py new file mode 100644 index 0000000000..245692337b --- /dev/null +++ b/tests/api/expressions/__init__.py @@ -0,0 +1,17 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + diff --git a/tests/api/expressions/conftest.py b/tests/api/expressions/conftest.py new file mode 100644 index 0000000000..54039b209f --- /dev/null +++ b/tests/api/expressions/conftest.py @@ -0,0 +1,307 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +from decimal import Decimal +import io +import pickle +import uuid + +from iceberg.api import DataFile +from iceberg.api.expressions import (BoundPredicate, + Expressions, + ExpressionVisitors, + Literal, + Operation, + UnboundPredicate) +from iceberg.api.schema import Schema +from iceberg.api.struct_like import StructLike +from iceberg.api.types import (BinaryType, + Conversions, + DateType, + DecimalType, + FixedType, + IntegerType, + NestedField, + StringType, + TimestampType, + TimeType) +import pytest + +exp_schema = Schema(NestedField.optional(34, "a", IntegerType.get())) + + +class TestHelpers(object): + + @staticmethod + def assert_all_references_bound(message, expr): + ExpressionVisitors.visit(expr, TestHelpers.CheckReferencesBound(message)) + + @staticmethod + def assert_and_unwrap(expr, expected=None): + if expected is not None: + assert isinstance(expr, expected) + else: + assert isinstance(expr, BoundPredicate) + + return expr + + @staticmethod + def round_trip_serialize(type_var): + stream = io.BytesIO() + pickle.dump(type_var, stream, pickle.HIGHEST_PROTOCOL) + stream.seek(0) + + return pickle.load(stream) + + class Row(StructLike): + + @staticmethod + def of(values=None): + return TestHelpers.Row(values) + + def __init__(self, values): + self.values = values + + def get(self, pos): + return self.values[pos] + + def set(self, pos, value): + raise RuntimeError("Setting values is not supported") + + class CheckReferencesBound(ExpressionVisitors.ExpressionVisitor): + + def __init__(self, message): + self.message = message + + def predicate(self, pred): + if isinstance(pred, UnboundPredicate): + assert False + + +class TestDataFile(DataFile): + + def __init__(self, path, partition, record_count, value_counts=None, null_value_counts=None, + lower_bounds=None, upper_bounds=None): + self.path = path + self.partition = partition + self.record_count = record_count + self.value_counts = value_counts + self.null_value_counts = null_value_counts + self.lower_bounds = lower_bounds + self.upper_bounds = upper_bounds + self.file_size_in_bytes = 0 + self.block_size_in_bytes = 0 + self.file_ordinal = None + self.column_size = None + + def copy(self): + return self + + +@pytest.fixture(scope="session") +def schema(): + return Schema(NestedField.required(1, "id", IntegerType.get()), + NestedField.optional(2, "no_stats", IntegerType.get()), + NestedField.required(3, "required", StringType.get()), + NestedField.optional(4, "all_nulls", StringType.get()), + NestedField.optional(5, "some_nulls", StringType.get()), + NestedField.optional(6, "no_nulls", StringType.get())) + + +@pytest.fixture(scope="session") +def strict_schema(): + return Schema(NestedField.required(1, "id", IntegerType.get()), + NestedField.optional(2, "no_stats", IntegerType.get()), + NestedField.required(3, "required", StringType.get()), + NestedField.optional(4, "all_nulls", StringType.get()), + NestedField.optional(5, "some_nulls", StringType.get()), + NestedField.optional(6, "no_nulls", StringType.get()), + NestedField.required(7, "always_5", IntegerType.get())) + + +@pytest.fixture(scope="session") +def file(): + return TestDataFile("file.avro", TestHelpers.Row.of(), 50, + # value counts + {4: 50, 5: 50, 6: 50}, + # null value counts + {4: 50, 5: 10, 6: 0}, + # lower bounds + {1: Conversions.to_byte_buffer(IntegerType.get(), 30)}, + # upper bounds + {1: Conversions.to_byte_buffer(IntegerType.get(), 79)}) + + +@pytest.fixture(scope="session") +def strict_file(): + return TestDataFile("file.avro", + TestHelpers.Row.of(), + 50, + {4: 50, 5: 50, 6: 50}, + {4: 50, 5: 10, 6: 0}, + {1: Conversions.to_byte_buffer(IntegerType.get(), 30), + 7: Conversions.to_byte_buffer(IntegerType.get(), 5)}, + {1: Conversions.to_byte_buffer(IntegerType.get(), 79), + 7: Conversions.to_byte_buffer(IntegerType.get(), 5)} + ) + + +@pytest.fixture(scope="session") +def missing_stats(): + return TestDataFile("file.parquet", TestHelpers.Row.of(), 50) + + +@pytest.fixture(scope="session") +def empty(): + return TestDataFile("file.parquet", TestHelpers.Row.of(), record_count=0) + + +@pytest.fixture(scope="session") +def assert_and_unwrap(): + return lambda x, y=None: TestHelpers.assert_and_unwrap(x, y) + + +@pytest.fixture(scope="session") +def assert_all_bound(): + return lambda msg, expr: TestHelpers.assert_all_references_bound(msg, expr) + + +@pytest.fixture(scope="session") +def round_trip_serialize(): + return lambda x: TestHelpers.round_trip_serialize(x) + + +@pytest.fixture(scope="session") +def row_of(): + return lambda x: TestHelpers.Row.of(x) + + +@pytest.fixture(scope="session", + params=[Operation.LT, + Operation.LT_EQ, + Operation.GT, + Operation.GT_EQ, + Operation.EQ, + Operation.NOT_EQ]) +def op(request): + yield request.param + + +@pytest.fixture(scope="session", + params=[Expressions.always_false(), + Expressions.always_true(), + Expressions.less_than("x", 5), + Expressions.less_than_or_equal("y", -3), + Expressions.greater_than("z", 0), + Expressions.greater_than_or_equal("t", 129), + Expressions.equal("col", "data"), + Expressions.not_equal("col", "abc"), + Expressions.not_null("maybeNull"), + Expressions.is_null("maybeNull2"), + Expressions.not_(Expressions.greater_than("a", 10)), + Expressions.and_(Expressions.greater_than_or_equal("a", 0), + Expressions.less_than("a", 3)), + Expressions.or_(Expressions.less_than("a", 0), + Expressions.greater_than("a", 10)), + Expressions.equal("a", 5).bind(exp_schema.as_struct())]) +def expression(request): + yield request.param + + +@pytest.fixture(scope="session", + params=[Expressions.less_than("no_stats", 5), + Expressions.less_than_or_equal("no_stats", 30), + Expressions.equal("no_stats", 70), + Expressions.greater_than("no_stats", 78), + Expressions.greater_than_or_equal("no_stats", 90), + Expressions.not_equal("no_stats", 101), + Expressions.is_null("no_stats"), + Expressions.not_null("no_stats")]) +def missing_stats_exprs(request): + yield request.param + + +@pytest.fixture(scope="session", + params=[Expressions.less_than("id", 5), + Expressions.less_than_or_equal("id", 30), + Expressions.equal("id", 70), + Expressions.greater_than("id", 78), + Expressions.greater_than_or_equal("id", 90), + Expressions.not_equal("id", 101), + Expressions.is_null("some_nulls"), + Expressions.not_null("some_nulls")]) +def zero_rows_exprs(request): + yield request.param + + +@pytest.fixture(scope="session", + params=[Expressions.not_equal("id", 5), + Expressions.not_equal("id", 29), + Expressions.not_equal("id", 30), + Expressions.not_equal("id", 75), + Expressions.not_equal("id", 79), + Expressions.not_equal("id", 80), + Expressions.not_equal("id", 85)]) +def not_eq(request): + yield request.param + + +@pytest.fixture(scope="session", + params=[Expressions.equal("id", 5), + Expressions.equal("id", 29), + Expressions.equal("id", 30), + Expressions.equal("id", 75), + Expressions.equal("id", 79), + Expressions.equal("id", 80), + Expressions.equal("id", 85)]) +def not_eq_rewrite(request): + yield request.param + + +@pytest.fixture(scope="session", + params=[Literal.of(False), + Literal.of(34), + Literal.of(35), + Literal.of(36.75), + Literal.of(8.75), + Literal.of("2017-11-29").to(DateType.get()), + Literal.of("11:30:0").to(TimeType.get()), + Literal.of("2017-11-29T11:30:07.123").to(TimestampType.without_timezone()), + Literal.of("2017-11-29T11:30:07.123+01:00").to(TimestampType.with_timezone()), + Literal.of("abc"), + Literal.of(uuid.uuid4()), + Literal.of(bytes([0x01, 0x02, 0x03])).to(FixedType.of_length(3)), + Literal.of(bytes([0x03, 0x04, 0x05, 0x06])).to(BinaryType.get()), + Literal.of(Decimal(122.50).quantize(Decimal(".01")))]) +def literal(request): + yield request.param + + +@pytest.fixture(scope="session", + params=[(DecimalType.of(9, 0), "34"), + (DecimalType.of(9, 2), "34.00"), + (DecimalType.of(9, 4), "34.0000")]) +def type_val_tuples(request): + yield request.param + + +@pytest.fixture(scope="session", + params=[(DecimalType.of(9, 1), "34.6"), + (DecimalType.of(9, 2), "34.56"), + (DecimalType.of(9, 4), "34.5600")]) +def float_type_val_tuples(request): + yield request.param diff --git a/tests/api/expressions/test_evaluator.py b/tests/api/expressions/test_evaluator.py new file mode 100644 index 0000000000..ec3d08f672 --- /dev/null +++ b/tests/api/expressions/test_evaluator.py @@ -0,0 +1,130 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +import iceberg.api.expressions as exp +from iceberg.api.types import (IntegerType, + NestedField, + StringType, + StructType) + +STRUCT = StructType.of([NestedField.required(13, "x", IntegerType.get()), + NestedField.required(14, "y", IntegerType.get()), + NestedField.optional(15, "z", IntegerType.get())]) + + +def test_less_than(row_of): + evaluator = exp.evaluator.Evaluator(STRUCT, + exp.expressions.Expressions.less_than("x", 7)) + assert not evaluator.eval(row_of((7, 8, None))) + assert evaluator.eval(row_of((6, 8, None))) + + +def test_less_than_or_equal(row_of): + evaluator = exp.evaluator.Evaluator(STRUCT, + exp.expressions.Expressions.less_than_or_equal("x", 7)) + assert evaluator.eval(row_of((7, 8, None))) + assert evaluator.eval(row_of((6, 8, None))) + assert not evaluator.eval(row_of((8, 8, None))) + + +def test_greater_than(row_of): + evaluator = exp.evaluator.Evaluator(STRUCT, + exp.expressions.Expressions.greater_than("x", 7)) + assert not evaluator.eval(row_of((7, 8, None))) + assert not evaluator.eval(row_of((6, 8, None))) + assert evaluator.eval(row_of((8, 8, None))) + + +def test_greater_than_or_equal(row_of): + evaluator = exp.evaluator.Evaluator(STRUCT, + exp.expressions.Expressions.greater_than_or_equal("x", 7)) + assert evaluator.eval(row_of((7, 8, None))) + assert not evaluator.eval(row_of((6, 8, None))) + assert evaluator.eval(row_of((8, 8, None))) + + +def test_equal(row_of): + evaluator = exp.evaluator.Evaluator(STRUCT, + exp.expressions.Expressions.equal("x", 7)) + assert evaluator.eval(row_of((7, 8, None))) + + +def test_not_equal(row_of): + evaluator = exp.evaluator.Evaluator(STRUCT, + exp.expressions.Expressions.not_equal("x", 7)) + assert not evaluator.eval(row_of((7, 8, None))) + assert evaluator.eval(row_of((6, 8, None))) + + +def test_always_true(row_of): + evaluator = exp.evaluator.Evaluator(STRUCT, + exp.expressions.Expressions.always_true()) + assert evaluator.eval(row_of(())) + + +def test_always_false(row_of): + evaluator = exp.evaluator.Evaluator(STRUCT, + exp.expressions.Expressions.always_false()) + assert not evaluator.eval(row_of(())) + + +def test_is_null(row_of): + evaluator = exp.evaluator.Evaluator(STRUCT, + exp.expressions.Expressions.is_null("z")) + assert evaluator.eval(row_of((1, 2, None))) + assert not evaluator.eval(row_of((1, 2, 3))) + + +def test_is_not_null(row_of): + evaluator = exp.evaluator.Evaluator(STRUCT, + exp.expressions.Expressions.not_null("z")) + assert not evaluator.eval(row_of((1, 2, None))) + assert evaluator.eval(row_of((1, 2, 3))) + + +def test_and(row_of): + evaluator = exp.evaluator.Evaluator(STRUCT, + exp.expressions.Expressions.and_(exp.expressions.Expressions.equal("x", 7), + exp.expressions.Expressions.not_null("z"))) + assert evaluator.eval(row_of((7, 0, 3))) + assert not evaluator.eval(row_of((8, 0, 3))) + assert not evaluator.eval(row_of((7, 0, None))) + assert not evaluator.eval(row_of((8, 0, None))) + + +def test_or(row_of): + evaluator = exp.evaluator.Evaluator(STRUCT, + exp.expressions.Expressions.or_(exp.expressions.Expressions.equal("x", 7), + exp.expressions.Expressions.not_null("z"))) + assert evaluator.eval(row_of((7, 0, 3))) + assert evaluator.eval(row_of((8, 0, 3))) + assert evaluator.eval(row_of((7, 0, None))) + assert not evaluator.eval(row_of((8, 0, None))) + + +def test_not(row_of): + evaluator = exp.evaluator.Evaluator(STRUCT, + exp.expressions.Expressions.not_(exp.expressions.Expressions.equal("x", 7))) + assert not evaluator.eval(row_of((7,))) + assert evaluator.eval(row_of((8,))) + + +def test_char_seq_value(row_of): + struct = StructType.of([NestedField.required(34, "s", StringType.get())]) + evaluator = exp.evaluator.Evaluator(struct, exp.expressions.Expressions.equal("s", "abc")) + assert evaluator.eval(row_of(("abc",))) + assert not evaluator.eval(row_of(("abcd",))) diff --git a/tests/api/expressions/test_expression_binding.py b/tests/api/expressions/test_expression_binding.py new file mode 100644 index 0000000000..47f6fead84 --- /dev/null +++ b/tests/api/expressions/test_expression_binding.py @@ -0,0 +1,131 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +from iceberg.api.expressions import (And, + Binder, + Expressions, + Not, + Or) +from iceberg.api.types import (IntegerType, + NestedField, + StructType) +import iceberg.exceptions as ice_ex +from pytest import raises + +STRUCT = StructType.of([NestedField.required(0, "x", IntegerType.get()), + NestedField.required(1, "y", IntegerType.get()), + NestedField.required(2, "z", IntegerType.get())]) + + +def test_missing_reference(): + expr = Expressions.and_(Expressions.equal("t", 5), + Expressions.equal("x", 7)) + try: + Binder.bind(STRUCT, expr) + except ice_ex.ValidationException as e: + assert "Cannot find field 't' in struct" in "{}".format(e) + + +def test_bound_expression_fails(): + with raises(RuntimeError): + expr = Expressions.not_(Expressions.equal("x", 7)) + Binder.bind(STRUCT, + Binder.bind(STRUCT, expr)) + + +def test_single_reference(assert_all_bound): + expr = Expressions.not_(Expressions.equal("x", 7)) + assert_all_bound("Single reference", + Binder.bind(STRUCT, expr)) + + +def test_multiple_references(assert_all_bound): + expr = Expressions.or_(Expressions.and_(Expressions.equal("x", 7), + Expressions.less_than("y", 100)), + Expressions.greater_than("z", -100)) + + assert_all_bound("Multiple references", Binder.bind(STRUCT, expr)) + + +def test_and(assert_all_bound, assert_and_unwrap): + expr = Expressions.and_(Expressions.equal("x", 7), + Expressions.less_than("y", 100)) + bound_expr = Binder.bind(STRUCT, expr) + assert_all_bound("And", bound_expr) + + and_ = assert_and_unwrap(bound_expr, And) + + left = assert_and_unwrap(and_.left, None) + # should bind x correctly + assert 0 == left.ref.field_id + right = assert_and_unwrap(and_.right, None) + # should bind y correctly + assert 1 == right.ref.field_id + + +def test_or(assert_all_bound, assert_and_unwrap): + expr = Expressions.or_(Expressions.greater_than("z", -100), + Expressions.less_than("y", 100)) + bound_expr = Binder.bind(STRUCT, expr) + assert_all_bound("Or", bound_expr) + + or_ = assert_and_unwrap(bound_expr, Or) + + left = assert_and_unwrap(or_.left, None) + # should bind z correctly + assert 2 == left.ref.field_id + right = assert_and_unwrap(or_.right, None) + # should bind y correctly + assert 1 == right.ref.field_id + + +def test_not(assert_all_bound, assert_and_unwrap): + expr = Expressions.not_(Expressions.equal("x", 7)) + bound_expr = Binder.bind(STRUCT, expr) + assert_all_bound("Not", bound_expr) + + not_ = assert_and_unwrap(bound_expr, Not) + + child = assert_and_unwrap(not_.child, None) + # should bind x correctly + assert 0 == child.ref.field_id + + +def test_always_true(): + assert Expressions.always_true() == Binder.bind(STRUCT, + Expressions.always_true()) + + +def test_always_false(): + assert Expressions.always_false() == Binder.bind(STRUCT, + Expressions.always_false()) + + +def test_basic_simplification(assert_and_unwrap): + # Should simplify or expression to alwaysTrue + assert Expressions.always_true() == Binder.bind(STRUCT, + Expressions.or_(Expressions.less_than("y", 100), + Expressions.greater_than("z", -9999999999))) + # Should simplify or expression to alwaysfalse + assert Expressions.always_false() == Binder.bind(STRUCT, + Expressions.and_(Expressions.less_than("y", 100), + Expressions.less_than("z", -9999999999))) + + bound = Binder.bind(STRUCT, + Expressions.not_(Expressions.not_(Expressions.less_than("y", 100)))) + pred = assert_and_unwrap(bound, None) + assert 1 == pred.ref.field_id diff --git a/tests/api/expressions/test_expression_helpers.py b/tests/api/expressions/test_expression_helpers.py new file mode 100644 index 0000000000..02f0b7db39 --- /dev/null +++ b/tests/api/expressions/test_expression_helpers.py @@ -0,0 +1,47 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +from iceberg.api.expressions.expressions import Expressions +from pytest import raises + +pred = Expressions.less_than("x", 7) + + +def test_simplify_or(): + assert Expressions.always_true() == Expressions.or_(Expressions.always_true(), pred) + assert Expressions.always_true() == Expressions.or_(pred, Expressions.always_true()) + assert pred == Expressions.or_(Expressions.always_false(), pred) + assert pred == Expressions.or_(pred, Expressions.always_false()) + + +def test_simplify_and(): + assert pred == Expressions.and_(Expressions.always_true(), pred) + assert pred == Expressions.and_(pred, Expressions.always_true()) + + assert Expressions.always_false() == Expressions.and_(Expressions.always_false(), pred) + assert Expressions.always_false() == Expressions.and_(pred, Expressions.always_false()) + + +def test_simplify_not(): + assert Expressions.always_false() == Expressions.not_(Expressions.always_true()) + assert Expressions.always_true() == Expressions.not_(Expressions.always_false()) + assert pred == Expressions.not_(Expressions.not_(pred)) + + +def test_null_name(): + with raises(RuntimeError): + Expressions.equal(None, 5) diff --git a/tests/api/expressions/test_expression_serializations.py b/tests/api/expressions/test_expression_serializations.py new file mode 100644 index 0000000000..1ada7e9447 --- /dev/null +++ b/tests/api/expressions/test_expression_serializations.py @@ -0,0 +1,21 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + + +def test_expressions(expression, round_trip_serialize): + copy = round_trip_serialize(expression) + assert copy == expression diff --git a/tests/api/expressions/test_inclusive_metrics_evaluator.py b/tests/api/expressions/test_inclusive_metrics_evaluator.py new file mode 100644 index 0000000000..fcbef1e213 --- /dev/null +++ b/tests/api/expressions/test_inclusive_metrics_evaluator.py @@ -0,0 +1,118 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +from iceberg.api.expressions import (Expressions, + InclusiveMetricsEvaluator) +from pytest import raises + + +def test_all_nulls(schema, file): + # Should skip: no non-null value in all null column + assert not InclusiveMetricsEvaluator(schema, Expressions.not_null("all_nulls")).eval(file) + # Should read: column with some nulls contains a non-null value + assert InclusiveMetricsEvaluator(schema, Expressions.not_null("some_nulls")).eval(file) + # Should read: non-null column contains a non-null value + assert InclusiveMetricsEvaluator(schema, Expressions.not_null("no_nulls")).eval(file) + + +def test_no_nulls(schema, file): + # Should read: at least one null value in all null column + assert InclusiveMetricsEvaluator(schema, Expressions.is_null("all_nulls")).eval(file) + # Should read: column with some nulls contains a null value + assert InclusiveMetricsEvaluator(schema, Expressions.is_null("some_nulls")).eval(file) + # Should skip: non-null column contains no null values + assert not InclusiveMetricsEvaluator(schema, Expressions.is_null("no_nulls")).eval(file) + + +def test_required_column(schema, file): + assert InclusiveMetricsEvaluator(schema, Expressions.not_null("required")).eval(file) + assert not InclusiveMetricsEvaluator(schema, Expressions.is_null("required")).eval(file) + + +def test_missing_column(schema, file): + with raises(RuntimeError): + InclusiveMetricsEvaluator(schema, Expressions.less_than("missing", 5)).eval(file) + + +def test_missing_stats(schema, missing_stats, missing_stats_exprs): + assert InclusiveMetricsEvaluator(schema, missing_stats_exprs).eval(missing_stats) + + +def test_zero_record_file(schema, empty, zero_rows_exprs): + assert not InclusiveMetricsEvaluator(schema, zero_rows_exprs).eval(empty) + + +def test_not(schema, file): + assert InclusiveMetricsEvaluator(schema, Expressions.not_(Expressions.less_than("id", 5))).eval(file) + assert not InclusiveMetricsEvaluator(schema, + Expressions.not_(Expressions.greater_than("id", 5))).eval(file) + + +def test_and(schema, file): + assert not InclusiveMetricsEvaluator(schema, + Expressions.and_(Expressions.less_than("id", 5), + Expressions.greater_than_or_equal("id", 0))).eval(file) + assert InclusiveMetricsEvaluator(schema, Expressions.and_(Expressions.greater_than("id", 5), + Expressions.less_than_or_equal("id", 30))).eval(file) + + +def test_or(schema, file): + assert not InclusiveMetricsEvaluator(schema, + Expressions.or_(Expressions.less_than("id", 5), + Expressions.greater_than_or_equal("id", 80))).eval(file) + assert InclusiveMetricsEvaluator(schema, + Expressions.or_(Expressions.less_than("id", 5), + Expressions.greater_than_or_equal("id", 60))).eval(file) + + +def test_integer_lt(schema, file): + assert not InclusiveMetricsEvaluator(schema, Expressions.less_than("id", 5)).eval(file) + assert not InclusiveMetricsEvaluator(schema, Expressions.less_than("id", 30)).eval(file) + assert InclusiveMetricsEvaluator(schema, Expressions.less_than("id", 31)).eval(file) + assert InclusiveMetricsEvaluator(schema, Expressions.less_than("id", 79)).eval(file) + + +def test_integer_gt(schema, file): + assert not InclusiveMetricsEvaluator(schema, Expressions.greater_than("id", 85)).eval(file) + assert not InclusiveMetricsEvaluator(schema, Expressions.greater_than("id", 79)).eval(file) + assert InclusiveMetricsEvaluator(schema, Expressions.greater_than("id", 78)).eval(file) + assert InclusiveMetricsEvaluator(schema, Expressions.greater_than("id", 75)).eval(file) + + +def test_integer_gt_eq(schema, file): + assert not InclusiveMetricsEvaluator(schema, Expressions.greater_than_or_equal("id", 85)).eval(file) + assert not InclusiveMetricsEvaluator(schema, Expressions.greater_than_or_equal("id", 80)).eval(file) + assert InclusiveMetricsEvaluator(schema, Expressions.greater_than_or_equal("id", 79)).eval(file) + assert InclusiveMetricsEvaluator(schema, Expressions.greater_than_or_equal("id", 75)).eval(file) + + +def test_integer_eq(schema, file): + assert not InclusiveMetricsEvaluator(schema, Expressions.equal("id", 5)).eval(file) + assert not InclusiveMetricsEvaluator(schema, Expressions.equal("id", 29)).eval(file) + assert not InclusiveMetricsEvaluator(schema, Expressions.equal("id", 80)).eval(file) + assert not InclusiveMetricsEvaluator(schema, Expressions.equal("id", 85)).eval(file) + assert InclusiveMetricsEvaluator(schema, Expressions.equal("id", 30)).eval(file) + assert InclusiveMetricsEvaluator(schema, Expressions.equal("id", 75)).eval(file) + assert InclusiveMetricsEvaluator(schema, Expressions.equal("id", 79)).eval(file) + + +def test_integer_not_eq(schema, file, not_eq): + assert InclusiveMetricsEvaluator(schema, not_eq).eval(file) + + +def test_not_eq_rewritten(schema, file, not_eq_rewrite): + assert InclusiveMetricsEvaluator(schema, Expressions.not_(not_eq_rewrite)).eval(file) diff --git a/tests/api/expressions/test_literal_serialization.py b/tests/api/expressions/test_literal_serialization.py new file mode 100644 index 0000000000..3e0a2f8aa3 --- /dev/null +++ b/tests/api/expressions/test_literal_serialization.py @@ -0,0 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + + +def test_literals(round_trip_serialize, literal): + assert round_trip_serialize(literal) == literal diff --git a/tests/api/expressions/test_misc_literal_conversions.py b/tests/api/expressions/test_misc_literal_conversions.py new file mode 100644 index 0000000000..d87ca8a8d3 --- /dev/null +++ b/tests/api/expressions/test_misc_literal_conversions.py @@ -0,0 +1,263 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +from decimal import Decimal +import sys +import uuid + +from iceberg.api.expressions import Literal +from iceberg.api.types import (BinaryType, + BooleanType, + DateType, + DecimalType, + DoubleType, + FixedType, + FloatType, + IntegerType, + LongType, + StringType, + TimestampType, + TimeType, + UUIDType) +import pytest + + +def test_identity_conversions(): + pairs = [(Literal.of(True), BooleanType.get()), + (Literal.of(34), IntegerType.get()), + (Literal.of(3400000000), LongType.get()), + (Literal.of(34.11), FloatType.get()), + (Literal.of(34.11), DoubleType.get()), + (Literal.of(Decimal(34.55).quantize(Decimal("0.01"))), DecimalType.of(9, 2)), + (Literal.of("2017-08-18"), DateType.get()), + (Literal.of("14:21:01.919"), TimeType.get()), + (Literal.of("2017-08-18T14:21:01.919"), TimestampType.without_timezone()), + (Literal.of("abc"), StringType.get()), + (Literal.of(uuid.uuid4()), UUIDType.get()) + ] + + if sys.version_info[0] >= 3: + pairs = pairs + [(Literal.of(bytes([0x01, 0x02, 0x03])), FixedType.of_length(3)), + (Literal.of(bytearray([0x03, 0x04, 0x05, 0x06])), BinaryType.get())] + + for pair in pairs: + expected = pair[0].to(pair[1]) + assert expected is expected.to(pair[1]) + + +def test_binary_to_fixed(): + if sys.version_info[0] >= 3: + lit = Literal.of(bytearray([0x00, 0x01, 0x02])) + fixed_lit = lit.to(FixedType.of_length(3)) + assert fixed_lit is not None + assert lit.value == fixed_lit.value + assert lit.to(FixedType.of_length(4)) is None + assert lit.to(FixedType.of_length(2)) is None + + +def test_fixed_to_binary(): + if sys.version_info[0] >= 3: + lit = Literal.of(bytes([0x00, 0x01, 0x02])) + binary_lit = lit.to(BinaryType.get()) + assert binary_lit is not None + assert lit.value == binary_lit.value + + +def test_invalid_boolean_conversions(): + assert_invalid_conversions(Literal.of(True), [IntegerType.get(), + LongType.get(), + FloatType.get(), + DoubleType.get(), + DateType.get(), + TimeType.get(), + TimestampType.with_timezone(), + TimestampType.without_timezone(), + DecimalType.of(9, 2), + StringType.get(), + UUIDType.get(), + FixedType.of_length(1), + BinaryType.get()]) + + +def test_invalid_integer_conversions(): + assert_invalid_conversions(Literal.of(34), [BooleanType.get(), + TimeType.get(), + TimestampType.with_timezone(), + TimestampType.without_timezone(), + StringType.get(), + UUIDType.get(), + FixedType.of_length(1), + BinaryType.get()]) + + +def test_invalid_long_conversions(): + assert_invalid_conversions(Literal.of(34).to(LongType.get()), [BooleanType.get(), + DateType.get(), + StringType.get(), + UUIDType.get(), + FixedType.of_length(1), + BinaryType.get()]) + + +@pytest.mark.parametrize("lit", [Literal.of(34.11), + # double + Literal.of(34.11).to(DoubleType.get())]) +@pytest.mark.parametrize("test_type", [BooleanType.get(), + IntegerType.get(), + LongType.get(), + DateType.get(), + TimeType.get(), + TimestampType.with_timezone(), + TimestampType.without_timezone(), + StringType.get(), + UUIDType.get(), + FixedType.of_length(1), + BinaryType.get()]) +def test_invalid_float_conversions(lit, test_type): + assert lit.to(test_type) is None + + +@pytest.mark.parametrize("lit", [Literal.of("2017-08-18").to(DateType.get())]) +@pytest.mark.parametrize("test_type", [BooleanType.get(), + IntegerType.get(), + LongType.get(), + FloatType.get(), + DoubleType.get(), + TimeType.get(), + TimestampType.with_timezone(), + TimestampType.without_timezone(), + DecimalType.of(9, 2), + StringType.get(), + UUIDType.get(), + FixedType.of_length(1), + BinaryType.get()]) +def test_invalid_datetime_conversions(lit, test_type): + assert_invalid_conversions(lit, (test_type,)) + + +def test_invalid_time_conversions(): + assert_invalid_conversions(Literal.of("14:21:01.919") + .to(TimeType.get()), [BooleanType.get(), + IntegerType.get(), + LongType.get(), + FloatType.get(), + DoubleType.get(), + DateType.get(), + TimestampType.with_timezone(), + TimestampType.without_timezone(), + DecimalType.of(9, 2), + StringType.get(), + UUIDType.get(), + FixedType.of_length(1), + BinaryType.get()]) + + +def test_invalid_timestamp_conversions(): + assert_invalid_conversions(Literal.of("2017-08-18T14:21:01.919") + .to(TimestampType.without_timezone()), [BooleanType.get(), + IntegerType.get(), + LongType.get(), + FloatType.get(), + DoubleType.get(), + TimeType.get(), + DecimalType.of(9, 2), + StringType.get(), + UUIDType.get(), + FixedType.of_length(1), + BinaryType.get()]) + + +def test_invalid_decimal_conversions(): + assert_invalid_conversions(Literal.of(Decimal("34.11")), [BooleanType.get(), + IntegerType.get(), + LongType.get(), + FloatType.get(), + DoubleType.get(), + DateType.get(), + TimeType.get(), + TimestampType.with_timezone(), + TimestampType.without_timezone(), + DecimalType.of(9, 4), + StringType.get(), + UUIDType.get(), + FixedType.of_length(1), + BinaryType.get()]) + + +def test_invalid_string_conversions(): + assert_invalid_conversions(Literal.of("abc"), [BooleanType.get(), + IntegerType.get(), + LongType.get(), + FloatType.get(), + DoubleType.get(), + FixedType.of_length(1), + BinaryType.get()]) + + +def test_invalid_uuid_conversions(): + assert_invalid_conversions(Literal.of(uuid.uuid4()), [BooleanType.get(), + IntegerType.get(), + LongType.get(), + FloatType.get(), + DoubleType.get(), + DateType.get(), + TimeType.get(), + TimestampType.with_timezone(), + TimestampType.without_timezone(), + DecimalType.of(9, 2), + StringType.get(), + FixedType.of_length(1), + BinaryType.get()]) + + +def test_invalid_fixed_conversions(): + if sys.version_info[0] >= 3: + assert_invalid_conversions(Literal.of(bytes([0x00, 0x01, 0x02])), [BooleanType.get(), + IntegerType.get(), + LongType.get(), + FloatType.get(), + DoubleType.get(), + DateType.get(), + TimeType.get(), + TimestampType.with_timezone(), + TimestampType.without_timezone(), + DecimalType.of(9, 2), + StringType.get(), + UUIDType.get(), + FixedType.of_length(1)]) + + +def test_invalid_binary_conversions(): + if sys.version_info[0] >= 3: + assert_invalid_conversions(Literal.of(bytearray([0x00, 0x01, 0x02])), [BooleanType.get(), + IntegerType.get(), + LongType.get(), + FloatType.get(), + DoubleType.get(), + DateType.get(), + TimeType.get(), + TimestampType.with_timezone(), + TimestampType.without_timezone(), + DecimalType.of(9, 2), + StringType.get(), + UUIDType.get(), + FixedType.of_length(1)]) + + +def assert_invalid_conversions(lit, types=None): + for type_var in types: + assert lit.to(type_var) is None diff --git a/tests/api/expressions/test_numeric_literal_conversions.py b/tests/api/expressions/test_numeric_literal_conversions.py new file mode 100644 index 0000000000..7c1dd2a9fd --- /dev/null +++ b/tests/api/expressions/test_numeric_literal_conversions.py @@ -0,0 +1,116 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +from decimal import Decimal +import math + +from iceberg.api.expressions import Literal +from iceberg.api.types import (DecimalType, + DoubleType, + FloatType, + IntegerType, + LongType) + + +def test_integer_to_long_conversion(): + lit = Literal.of(34) + long_lit = lit.to(LongType.get()) + + assert lit.value == long_lit.value + + +def test_integer_to_float_conversion(): + lit = Literal.of(34) + float_lit = lit.to(FloatType.get()) + + assert math.isclose(lit.value, float_lit.value) + + +def test_integer_to_double_conversion(): + lit = Literal.of(34) + dbl_lit = lit.to(DoubleType.get()) + + assert math.isclose(lit.value, dbl_lit.value) + + +def test_integer_to_decimal_conversion(type_val_tuples): + lit = Literal.of(34) + + assert lit.to(type_val_tuples[0]).value.as_tuple() == Decimal(type_val_tuples[1]).as_tuple() + + +def test_long_to_integer(): + lit = Literal.of(34).to(LongType.get()) + int_lit = lit.to(IntegerType.get()) + + assert lit.value == int_lit.value + + +def test_long_to_float_conversion(): + lit = Literal.of(34).to(LongType.get()) + float_lit = lit.to(FloatType.get()) + + assert math.isclose(lit.value, float_lit.value) + + +def test_long_to_double_conversion(): + lit = Literal.of(34).to(LongType.get()) + dbl_lit = lit.to(DoubleType.get()) + + assert math.isclose(lit.value, dbl_lit.value) + + +def test_long_to_decimal_conversion(type_val_tuples): + lit = Literal.of(34).to(LongType.get()) + + assert lit.to(type_val_tuples[0]).value.as_tuple() == Decimal(type_val_tuples[1]).as_tuple() + + +def test_float_to_double(): + lit = Literal.of(34.56) + dbl_lit = lit.to(DoubleType.get()) + + assert math.isclose(lit.value, dbl_lit.value) + + +def test_float_to_decimal_conversion(float_type_val_tuples): + lit = Literal.of(34.56) + + assert lit.to(float_type_val_tuples[0]).value.as_tuple() == Decimal(float_type_val_tuples[1]).as_tuple() + + +def test_double_to_float(): + lit = Literal.of(34.56).to(DoubleType.get()) + float_lit = lit.to(FloatType.get()) + + assert math.isclose(lit.value, float_lit.value) + + +def test_double_to_decimal_conversion(float_type_val_tuples): + lit = Literal.of(34.56).to(DoubleType.get()) + + assert lit.to(float_type_val_tuples[0]).value.as_tuple() == Decimal(float_type_val_tuples[1]).as_tuple() + + +def test_decimal_to_decimal_conversion(): + lit = Literal.of(Decimal("34.11").quantize(Decimal(".01"))) + + assert lit.value.as_tuple() == lit.to(DecimalType.of(9, 2)).value.as_tuple() + assert lit.value.as_tuple() == lit.to(DecimalType.of(11, 2)).value.as_tuple() + assert lit.to(DecimalType.of(9, 0)) is None + assert lit.to(DecimalType.of(9, 1)) is None + assert lit.to(DecimalType.of(9, 3)) is None diff --git a/tests/api/expressions/test_predicate_binding.py b/tests/api/expressions/test_predicate_binding.py new file mode 100644 index 0000000000..44d45167a1 --- /dev/null +++ b/tests/api/expressions/test_predicate_binding.py @@ -0,0 +1,199 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +from decimal import Decimal + +from iceberg.api.expressions import (Expressions, + Literal, + Operation, + UnboundPredicate) +from iceberg.api.types import (DecimalType, + FloatType, + IntegerType, + NestedField, + StringType, + StructType) +from iceberg.exceptions import ValidationException + + +def test_multiple_fields(assert_and_unwrap): + struct = StructType.of([NestedField.required(10, 'x', IntegerType.get()), + NestedField.required(11, 'y', IntegerType.get()), + NestedField.required(12, 'z', IntegerType.get())]) + + unbound = UnboundPredicate(Operation.LT, Expressions.ref("y"), 6) + expr = unbound.bind(struct) + + bound = assert_and_unwrap(expr) + assert 11 == bound.ref.field_id + assert Operation.LT == bound.op + assert 6 == bound.lit.value + + +def test_missing_field(): + struct = StructType.of([NestedField.required(13, "x", IntegerType.get())]) + + unbound = UnboundPredicate(Operation.LT, Expressions.ref("missing"), 6) + try: + unbound.bind(struct) + except ValidationException as e: + assert e.args[0].startswith("Cannot find field 'missing' in struct") + + +def test_comparison_predicate_binding(op, assert_and_unwrap): + struct = StructType.of([NestedField.required(14, "x", IntegerType.get())]) + unbound = UnboundPredicate(op, Expressions.ref("x"), 5) + bound = assert_and_unwrap(unbound.bind(struct)) + + assert 5 == bound.lit.value + assert 14 == bound.ref.field_id + assert op == bound.op + + +def test_literal_converison(op, assert_and_unwrap): + struct = StructType.of([NestedField.required(15, "d", DecimalType.of(9, 2))]) + unbound = UnboundPredicate(op, Expressions.ref("d"), "12.40") + bound = assert_and_unwrap(unbound.bind(struct)) + + assert Decimal(12.40).quantize(Decimal(".01")).as_tuple() == bound.lit.value.as_tuple() + assert 15 == bound.ref.field_id + assert op == bound.op + + +def test_invalid_conversions(op): + struct = StructType.of([NestedField.required(16, "f", FloatType.get())]) + unbound = UnboundPredicate(op, Expressions.ref("f"), "12.40") + + try: + unbound.bind(struct) + except ValidationException as e: + assert e.args[0].startswith("Invalid value for comparison inclusive type float: 12.40") + + +def test_long_to_integer_conversion(assert_and_unwrap): + struct = StructType.of([NestedField.required(17, "i", IntegerType.get())]) + + lt = UnboundPredicate(Operation.LT, Expressions.ref("i"), Literal.JAVA_MAX_INT + 1) + assert lt.bind(struct) == Expressions.always_true() + + lt_eq = UnboundPredicate(Operation.LT_EQ, Expressions.ref("i"), Literal.JAVA_MAX_INT + 1) + assert lt_eq.bind(struct) == Expressions.always_true() + + gt = UnboundPredicate(Operation.GT, Expressions.ref("i"), Literal.JAVA_MIN_INT - 1) + assert gt.bind(struct) == Expressions.always_true() + + gt_eq = UnboundPredicate(Operation.GT_EQ, Expressions.ref("i"), Literal.JAVA_MIN_INT - 1) + assert gt_eq.bind(struct) == Expressions.always_true() + + gt_max = UnboundPredicate(Operation.GT, Expressions.ref("i"), Literal.JAVA_MAX_INT + 1) + assert gt_max.bind(struct) == Expressions.always_false() + + gt_eq_max = UnboundPredicate(Operation.GT_EQ, Expressions.ref("i"), Literal.JAVA_MAX_INT + 1) + assert gt_eq_max.bind(struct) == Expressions.always_false() + + lt_min = UnboundPredicate(Operation.LT, Expressions.ref("i"), Literal.JAVA_MIN_INT - 1) + assert lt_min.bind(struct) == Expressions.always_false() + + lt_eq_min = UnboundPredicate(Operation.LT_EQ, Expressions.ref("i"), Literal.JAVA_MIN_INT - 1) + assert lt_eq_min.bind(struct) == Expressions.always_false() + + lt_expr = UnboundPredicate(Operation.LT, Expressions.ref("i"), Literal.JAVA_MAX_INT).bind(struct) + lt_max = assert_and_unwrap(lt_expr) + assert lt_max.lit.value == Literal.JAVA_MAX_INT + + lt_eq_expr = UnboundPredicate(Operation.LT_EQ, Expressions.ref("i"), Literal.JAVA_MAX_INT).bind(struct) + lt_eq_max = assert_and_unwrap(lt_eq_expr) + assert lt_eq_max.lit.value == Literal.JAVA_MAX_INT + + gt_expr = UnboundPredicate(Operation.GT, Expressions.ref("i"), Literal.JAVA_MIN_INT).bind(struct) + gt_min = assert_and_unwrap(gt_expr) + assert gt_min.lit.value == Literal.JAVA_MIN_INT + + gt_eq_expr = UnboundPredicate(Operation.GT_EQ, Expressions.ref("i"), Literal.JAVA_MIN_INT).bind(struct) + gt_eq_min = assert_and_unwrap(gt_eq_expr) + assert gt_eq_min.lit.value == Literal.JAVA_MIN_INT + + +def test_double_to_float_conversion(assert_and_unwrap): + struct = StructType.of([NestedField.required(18, "f", FloatType.get())]) + + lt = UnboundPredicate(Operation.LT, Expressions.ref("f"), Literal.JAVA_MAX_FLOAT * 2) + assert lt.bind(struct) == Expressions.always_true() + + lt_eq = UnboundPredicate(Operation.LT_EQ, Expressions.ref("f"), Literal.JAVA_MAX_FLOAT * 2) + assert lt_eq.bind(struct) == Expressions.always_true() + + gt = UnboundPredicate(Operation.GT, Expressions.ref("f"), Literal.JAVA_MAX_FLOAT * -2) + assert gt.bind(struct) == Expressions.always_true() + + gt_eq = UnboundPredicate(Operation.GT_EQ, Expressions.ref("f"), Literal.JAVA_MAX_FLOAT * -2) + assert gt_eq.bind(struct) == Expressions.always_true() + + gt_max = UnboundPredicate(Operation.GT, Expressions.ref("f"), Literal.JAVA_MAX_FLOAT * 2) + assert gt_max.bind(struct) == Expressions.always_false() + + gt_eq_max = UnboundPredicate(Operation.GT_EQ, Expressions.ref("f"), Literal.JAVA_MAX_FLOAT * 2) + assert gt_eq_max.bind(struct) == Expressions.always_false() + + lt_min = UnboundPredicate(Operation.LT, Expressions.ref("f"), Literal.JAVA_MAX_FLOAT * -2) + assert lt_min.bind(struct) == Expressions.always_false() + + lt_eq_min = UnboundPredicate(Operation.LT_EQ, Expressions.ref("f"), Literal.JAVA_MAX_FLOAT * -2) + assert lt_eq_min.bind(struct) == Expressions.always_false() + + lt_expr = UnboundPredicate(Operation.LT, Expressions.ref("f"), Literal.JAVA_MAX_FLOAT).bind(struct) + lt_max = assert_and_unwrap(lt_expr) + assert lt_max.lit.value == Literal.JAVA_MAX_FLOAT + + lt_eq_expr = UnboundPredicate(Operation.LT_EQ, Expressions.ref("f"), Literal.JAVA_MAX_FLOAT).bind(struct) + lt_eq_max = assert_and_unwrap(lt_eq_expr) + assert lt_eq_max.lit.value == Literal.JAVA_MAX_FLOAT + + gt_expr = UnboundPredicate(Operation.GT, Expressions.ref("f"), Literal.JAVA_MIN_INT).bind(struct) + gt_min = assert_and_unwrap(gt_expr) + assert gt_min.lit.value == Literal.JAVA_MIN_INT + + gt_eq_expr = UnboundPredicate(Operation.GT_EQ, Expressions.ref("f"), Literal.JAVA_MIN_INT).bind(struct) + gt_eq_min = assert_and_unwrap(gt_eq_expr) + assert gt_eq_min.lit.value == Literal.JAVA_MIN_INT + + +def test_is_null(assert_and_unwrap): + optional = StructType.of([NestedField.optional(19, "s", StringType.get())]) + unbound = UnboundPredicate(Operation.IS_NULL, Expressions.ref("s")) + expr = unbound.bind(optional) + bound = assert_and_unwrap(expr) + + assert Operation.IS_NULL == bound.op + assert 19 == bound.ref.field_id + assert bound.lit is None + + required = StructType.of([NestedField.required(20, "s", StringType.get())]) + assert Expressions.always_false() == unbound.bind(required) + + +def test_not_null(assert_and_unwrap): + optional = StructType.of([NestedField.optional(21, "s", StringType.get())]) + unbound = UnboundPredicate(Operation.NOT_NULL, Expressions.ref("s")) + expr = unbound.bind(optional) + bound = assert_and_unwrap(expr) + assert Operation.NOT_NULL == bound.op + assert 21 == bound.ref.field_id + assert bound.lit is None + + required = StructType.of([NestedField.required(22, "s", StringType.get())]) + assert Expressions.always_true() == unbound.bind(required) diff --git a/tests/api/expressions/test_strict_metrics_evaluator.py b/tests/api/expressions/test_strict_metrics_evaluator.py new file mode 100644 index 0000000000..36ab99db74 --- /dev/null +++ b/tests/api/expressions/test_strict_metrics_evaluator.py @@ -0,0 +1,159 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +from iceberg.api.expressions import (Expressions, + StrictMetricsEvaluator) +from iceberg.exceptions import ValidationException +from pytest import raises + + +def test_all_nulls(strict_schema, strict_file): + assert not StrictMetricsEvaluator(strict_schema, Expressions.not_null("all_nulls")).eval(strict_file) + assert not StrictMetricsEvaluator(strict_schema, Expressions.not_null("some_nulls")).eval(strict_file) + assert StrictMetricsEvaluator(strict_schema, Expressions.not_null("no_nulls")).eval(strict_file) + + +def test_no_nulls(strict_schema, strict_file): + assert StrictMetricsEvaluator(strict_schema, Expressions.is_null("all_nulls")).eval(strict_file) + assert not StrictMetricsEvaluator(strict_schema, Expressions.is_null("some_nulls")).eval(strict_file) + assert not StrictMetricsEvaluator(strict_schema, Expressions.is_null("no_nulls")).eval(strict_file) + + +def test_required_columns(strict_schema, strict_file): + assert StrictMetricsEvaluator(strict_schema, Expressions.not_null("required")).eval(strict_file) + assert not StrictMetricsEvaluator(strict_schema, Expressions.is_null("required")).eval(strict_file) + + +def test_missing_column(strict_schema, strict_file): + with raises(ValidationException): + StrictMetricsEvaluator(strict_schema, Expressions.less_than("missing", 5)).eval(strict_file) + + +def test_missing_stats(strict_schema, missing_stats): + exprs = [Expressions.less_than("no_stats", 5), + Expressions.less_than_or_equal("no_stats", 30), + Expressions.equal("no_stats", 70), + Expressions.greater_than("no_stats", 78), + Expressions.greater_than_or_equal("no_stats", 90), + Expressions.not_equal("no_stats", 101), + Expressions.is_null("no_stats"), + Expressions.not_null("no_stats")] + + for expr in exprs: + assert not StrictMetricsEvaluator(strict_schema, expr).eval(missing_stats) + + +def test_zero_record_file(strict_schema, empty): + + exprs = [Expressions.less_than("no_stats", 5), + Expressions.less_than_or_equal("no_stats", 30), + Expressions.equal("no_stats", 70), + Expressions.greater_than("no_stats", 78), + Expressions.greater_than_or_equal("no_stats", 90), + Expressions.not_equal("no_stats", 101), + Expressions.is_null("no_stats"), + Expressions.not_null("no_stats")] + for expr in exprs: + assert StrictMetricsEvaluator(strict_schema, expr).eval(empty) + + +def test_not(strict_schema, strict_file): + assert StrictMetricsEvaluator(strict_schema, + Expressions.not_(Expressions.less_than("id", 5))).eval(strict_file) + assert not StrictMetricsEvaluator(strict_schema, + Expressions.not_(Expressions.greater_than("id", 5))).eval(strict_file) + + +def test_and(strict_schema, strict_file): + assert not StrictMetricsEvaluator(strict_schema, + Expressions.and_(Expressions.greater_than("id", 5), + Expressions.less_than_or_equal("id", 30))).eval(strict_file) + assert not StrictMetricsEvaluator(strict_schema, + Expressions.and_(Expressions.less_than("id", 5), + Expressions.greater_than_or_equal("id", 0))).eval(strict_file) + assert StrictMetricsEvaluator(strict_schema, + Expressions.and_(Expressions.less_than("id", 85), + Expressions.greater_than_or_equal("id", 0))).eval(strict_file) + + +def test_or(strict_schema, strict_file): + assert not StrictMetricsEvaluator(strict_schema, + Expressions.or_(Expressions.less_than("id", 5), + Expressions.greater_than_or_equal("id", 80))).eval(strict_file) + assert not StrictMetricsEvaluator(strict_schema, + Expressions.or_(Expressions.less_than("id", 5), + Expressions.greater_than_or_equal("id", 60))).eval(strict_file) + assert StrictMetricsEvaluator(strict_schema, + Expressions.or_(Expressions.less_than("id", 5), + Expressions.greater_than_or_equal("id", 30))).eval(strict_file) + + +def test_integer_lt(strict_schema, strict_file): + assert not StrictMetricsEvaluator(strict_schema, Expressions.less_than("id", 5)).eval(strict_file) + assert not StrictMetricsEvaluator(strict_schema, Expressions.less_than("id", 31)).eval(strict_file) + assert not StrictMetricsEvaluator(strict_schema, Expressions.less_than("id", 79)).eval(strict_file) + assert StrictMetricsEvaluator(strict_schema, Expressions.less_than("id", 80)).eval(strict_file) + + +def test_integer_lt_eq(strict_schema, strict_file): + assert not StrictMetricsEvaluator(strict_schema, Expressions.less_than_or_equal("id", 29)).eval(strict_file) + assert not StrictMetricsEvaluator(strict_schema, Expressions.less_than_or_equal("id", 30)).eval(strict_file) + assert StrictMetricsEvaluator(strict_schema, Expressions.less_than_or_equal("id", 79)).eval(strict_file) + assert StrictMetricsEvaluator(strict_schema, Expressions.less_than_or_equal("id", 80)).eval(strict_file) + + +def test_integer_gt(strict_schema, strict_file): + assert not StrictMetricsEvaluator(strict_schema, Expressions.greater_than("id", 79)).eval(strict_file) + assert not StrictMetricsEvaluator(strict_schema, Expressions.greater_than("id", 78)).eval(strict_file) + assert not StrictMetricsEvaluator(strict_schema, Expressions.greater_than("id", 30)).eval(strict_file) + assert StrictMetricsEvaluator(strict_schema, Expressions.greater_than("id", 29)).eval(strict_file) + + +def test_integer_gt_eq(strict_schema, strict_file): + assert not StrictMetricsEvaluator(strict_schema, Expressions.greater_than_or_equal("id", 80)).eval(strict_file) + assert not StrictMetricsEvaluator(strict_schema, Expressions.greater_than_or_equal("id", 79)).eval(strict_file) + assert not StrictMetricsEvaluator(strict_schema, Expressions.greater_than_or_equal("id", 31)).eval(strict_file) + assert StrictMetricsEvaluator(strict_schema, Expressions.greater_than_or_equal("id", 30)).eval(strict_file) + + +def test_integer_eq(strict_schema, strict_file): + assert not StrictMetricsEvaluator(strict_schema, Expressions.equal("id", 5)).eval(strict_file) + assert not StrictMetricsEvaluator(strict_schema, Expressions.equal("id", 30)).eval(strict_file) + assert not StrictMetricsEvaluator(strict_schema, Expressions.equal("id", 75)).eval(strict_file) + assert not StrictMetricsEvaluator(strict_schema, Expressions.equal("id", 79)).eval(strict_file) + assert not StrictMetricsEvaluator(strict_schema, Expressions.equal("id", 80)).eval(strict_file) + assert StrictMetricsEvaluator(strict_schema, Expressions.equal("always_5", 5)).eval(strict_file) + + +def test_integer_not_eq(strict_schema, strict_file): + assert StrictMetricsEvaluator(strict_schema, Expressions.not_equal("id", 5)).eval(strict_file) + assert StrictMetricsEvaluator(strict_schema, Expressions.not_equal("id", 29)).eval(strict_file) + assert not StrictMetricsEvaluator(strict_schema, Expressions.not_equal("id", 30)).eval(strict_file) + assert not StrictMetricsEvaluator(strict_schema, Expressions.not_equal("id", 75)).eval(strict_file) + assert not StrictMetricsEvaluator(strict_schema, Expressions.not_equal("id", 79)).eval(strict_file) + assert StrictMetricsEvaluator(strict_schema, Expressions.not_equal("id", 80)).eval(strict_file) + assert StrictMetricsEvaluator(strict_schema, Expressions.not_equal("id", 85)).eval(strict_file) + + +def test_not_eq_rewritten(strict_schema, strict_file): + assert StrictMetricsEvaluator(strict_schema, Expressions.not_(Expressions.equal("id", 5))).eval(strict_file) + assert StrictMetricsEvaluator(strict_schema, Expressions.not_(Expressions.equal("id", 29))).eval(strict_file) + assert not StrictMetricsEvaluator(strict_schema, Expressions.not_(Expressions.equal("id", 30))).eval(strict_file) + assert not StrictMetricsEvaluator(strict_schema, Expressions.not_(Expressions.equal("id", 75))).eval(strict_file) + assert not StrictMetricsEvaluator(strict_schema, Expressions.not_(Expressions.equal("id", 79))).eval(strict_file) + assert StrictMetricsEvaluator(strict_schema, Expressions.not_(Expressions.equal("id", 80))).eval(strict_file) + assert StrictMetricsEvaluator(strict_schema, Expressions.not_(Expressions.equal("id", 85))).eval(strict_file) diff --git a/tests/api/expressions/test_string_literal_conversions.py b/tests/api/expressions/test_string_literal_conversions.py new file mode 100644 index 0000000000..57c2140af9 --- /dev/null +++ b/tests/api/expressions/test_string_literal_conversions.py @@ -0,0 +1,102 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +from datetime import datetime +from decimal import Decimal +import uuid + +import dateutil.parser +from fastavro.write import LOGICAL_WRITERS as avro_conversion +from iceberg.api.expressions import Literal +from iceberg.api.types import (DateType, + DecimalType, + StringType, + TimestampType, + TimeType, + UUIDType) +from pytest import raises + + +def test_string_to_string_literal(): + assert Literal.of("abc") == Literal.of("abc").to(StringType.get()) + + +def test_string_to_date_literal(): + date_str = Literal.of("2017-08-18") + date = date_str.to(DateType.get()) + + avro_val = avro_conversion["int-date"](datetime.strptime("2017-08-18", "%Y-%m-%d"), None) + assert avro_val == date.value + + +def test_string_to_time_literal(): + time_str = Literal.of("14:21:01.919") + time_lit = time_str.to(TimeType.get()) + + avro_val = avro_conversion["long-time-micros"](datetime.strptime("14:21:01.919", "%H:%M:%S.%f").time(), None) + + assert avro_val == time_lit.value + + +def test_string_to_timestamp_literal(): + timestamp_str = Literal.of("2017-08-18T14:21:01.919+00:00") + timestamp = timestamp_str.to(TimestampType.with_timezone()) + + avro_val = avro_conversion["long-timestamp-micros"](dateutil.parser.parse("2017-08-18T14:21:01.919+00:00"), + None) + assert avro_val == timestamp.value + + timestamp_str = Literal.of("2017-08-18T14:21:01.919") + timestamp = timestamp_str.to(TimestampType.without_timezone()) + assert avro_val == timestamp.value + + timestamp_str = Literal.of("2017-08-18T14:21:01.919-07:00") + timestamp = timestamp_str.to(TimestampType.with_timezone()) + avro_val = avro_conversion["long-timestamp-micros"](dateutil.parser.parse("2017-08-18T21:21:01.919+00:00"), + None) + assert avro_val == timestamp.value + + +def test_timestamp_with_zone_without_zone_in_literal(): + with raises(RuntimeError): + timestamp_str = Literal.of("2017-08-18T14:21:01.919") + timestamp_str.to(TimestampType.with_timezone()) + + +def test_timestamp_without_zone_with_zone_in_literal(): + with raises(RuntimeError): + timestamp_str = Literal.of("2017-08-18T14:21:01.919+07:00") + timestamp_str.to(TimestampType.without_timezone()) + + +def test_string_to_uuid_literal(): + expected = uuid.uuid4() + uuid_str = Literal.of(str(expected)) + uuid_lit = uuid_str.to(UUIDType.get()) + + assert expected == uuid_lit.value + + +def test_string_to_decimal_literal(): + decimal_str = Literal.of("34.560") + decimal_lit = decimal_str.to(DecimalType.of(9, 3)) + + assert 3 == abs(decimal_lit.value.as_tuple().exponent) + assert Decimal("34.560").as_tuple() == decimal_lit.value.as_tuple() + + assert decimal_str.to(DecimalType.of(9, 2)) is None + assert decimal_str.to(DecimalType.of(9, 4)) is None diff --git a/tests/api/test_conversions.py b/tests/api/test_conversions.py new file mode 100644 index 0000000000..be8587ab66 --- /dev/null +++ b/tests/api/test_conversions.py @@ -0,0 +1,34 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +import unittest + +from iceberg.api.types import (DoubleType, + IntegerType, + LongType) +from iceberg.api.types.conversions import Conversions + + +class TestConversions(unittest.TestCase): + + def test_from_bytes(self): + self.assertEqual(1234, Conversions.from_byte_buffer(IntegerType.get(), + b'\xd2\x04\x00\x00')) + self.assertEqual(1234, Conversions.from_byte_buffer(LongType.get(), + b'\xd2\x04\x00\x00\x00\x00\x00\x00')) + self.assertAlmostEqual(1.2345, Conversions.from_byte_buffer(DoubleType.get(), + b'\x8d\x97\x6e\x12\x83\xc0\xf3\x3f')) diff --git a/tests/api/test_helpers.py b/tests/api/test_helpers.py new file mode 100644 index 0000000000..baaabbe3b8 --- /dev/null +++ b/tests/api/test_helpers.py @@ -0,0 +1,79 @@ +import io +import pickle + +from iceberg.api.data_file import DataFile +from iceberg.api.expressions.expressions import ExpressionVisitors +from iceberg.api.expressions.predicate import (BoundPredicate, + UnboundPredicate) +from iceberg.api.struct_like import StructLike +from nose.tools import assert_true + + +class TestHelpers(object): + + @staticmethod + def assert_all_references_bound(message, expr): + ExpressionVisitors.visit(expr, TestHelpers.CheckReferencesBound(message)) + + @staticmethod + def assert_and_unwrap(expr, expected=None): + if expected is not None: + assert_true(isinstance(expr, expected)) + else: + assert_true(isinstance(expr, BoundPredicate)) + + return expr + + @staticmethod + def round_trip_serialize(type_var): + stream = io.BytesIO() + pickle.dump(type_var, stream, pickle.HIGHEST_PROTOCOL) + stream.seek(0) + + return pickle.load(stream) + + class Row(StructLike): + + @staticmethod + def of(values=None): + return TestHelpers.Row(values) + + def __init__(self, values): + self.values = values + + def get(self, pos): + return self.values[pos] + + def set(self, pos, value): + raise RuntimeError("Setting values is not supported") + + class CheckReferencesBound(ExpressionVisitors.ExpressionVisitor): + + def __init__(self, message): + self.message = message + + def predicate(self, pred): + if isinstance(pred, UnboundPredicate): + assert_true(False) + + return None + + +class TestDataFile(DataFile): + + def __init__(self, path, partition, record_count, value_counts=None, null_value_counts=None, + lower_bounds=None, upper_bounds=None): + self.path = path + self.partition = partition + self.record_count = record_count + self.value_counts = value_counts + self.null_value_counts = null_value_counts + self.lower_bounds = lower_bounds + self.upper_bounds = upper_bounds + self.file_size_in_bytes = 0 + self.block_size_in_bytes = 0 + self.file_ordinal = None + self.column_size = None + + def copy(self): + return self diff --git a/tests/api/transforms/__init__.py b/tests/api/transforms/__init__.py new file mode 100644 index 0000000000..245692337b --- /dev/null +++ b/tests/api/transforms/__init__.py @@ -0,0 +1,17 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + diff --git a/tests/api/transforms/test_bucketing.py b/tests/api/transforms/test_bucketing.py new file mode 100644 index 0000000000..a0eed1c20e --- /dev/null +++ b/tests/api/transforms/test_bucketing.py @@ -0,0 +1,64 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +from decimal import Decimal, getcontext + +from iceberg.api.expressions import Literal +from iceberg.api.transforms import (Bucket, + BucketDouble, + BucketFloat) +from iceberg.api.types import (DateType, + DecimalType, + IntegerType, + LongType, + TimestampType, + TimeType) +import pytest + + +@pytest.mark.parametrize("test_input,test_type,expected", [ + (1, IntegerType, 1392991556), + (34, IntegerType, 2017239379), + (34, LongType, 2017239379)]) +def test_spec_values_int(test_input, test_type, expected): + assert Bucket.get(test_type.get(), 100).hash(test_input) == expected + + +@pytest.mark.parametrize("test_input,test_type,expected", [ + (1, BucketFloat(100), -142385009), + (1, BucketDouble(100), -142385009)]) +def test_spec_values_dbl(test_input, test_type, expected): + assert test_type.hash(test_input) == expected + + +@pytest.mark.parametrize("test_input,test_type,scale_factor,expected", [ + (Decimal("14.20"), DecimalType.of(9, 2), Decimal(10) ** -2, -500754589), + (Decimal("137302769811943318102518958871258.37580"), DecimalType.of(38, 5), Decimal(10) ** -5, -32334285)]) +def test_spec_values_dec(test_input, test_type, scale_factor, expected): + getcontext().prec = 38 + assert Bucket.get(test_type, 100).hash(test_input.quantize(scale_factor)) == expected + + +@pytest.mark.parametrize("test_input,test_type,expected", [ + (Literal.of("2017-11-16").to(DateType.get()), DateType.get(), -653330422), + (Literal.of("22:31:08").to(TimeType.get()), TimeType.get(), -662762989), + (Literal.of("2017-11-16T22:31:08").to(TimestampType.without_timezone()), + TimestampType.without_timezone(), -2047944441), + (Literal.of("2017-11-16T14:31:08-08:00").to(TimestampType.with_timezone()), + TimestampType.with_timezone(), -2047944441)]) +def test_spec_values_datetime_uuid(test_input, test_type, expected): + assert Bucket.get(test_type, 100).hash(test_input.value) == expected diff --git a/tests/api/transforms/test_dates.py b/tests/api/transforms/test_dates.py new file mode 100644 index 0000000000..9f782d6712 --- /dev/null +++ b/tests/api/transforms/test_dates.py @@ -0,0 +1,44 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + + +from iceberg.api.expressions import Literal +from iceberg.api.transforms import Transforms +from iceberg.api.types import DateType +import pytest + + +@pytest.mark.parametrize("transform_gran,expected", [ + (Transforms.year, "2017"), + (Transforms.month, "2017-12"), + (Transforms.day, "2017-12-01")]) +def test_date_to_human_string(transform_gran, expected): + type_var = DateType.get() + date = Literal.of("2017-12-01").to(type_var) + + assert (transform_gran(DateType.get()) + .to_human_string(transform_gran(DateType.get()) + .apply(date.value))) == expected + + +@pytest.mark.parametrize("transform_gran", [ + Transforms.year, + Transforms.month, + Transforms.day]) +def test_null_human_string(transform_gran): + type_var = DateType.get() + assert transform_gran(type_var).to_human_string(None) == "null" diff --git a/tests/api/transforms/test_identity.py b/tests/api/transforms/test_identity.py new file mode 100644 index 0000000000..d23f2026ed --- /dev/null +++ b/tests/api/transforms/test_identity.py @@ -0,0 +1,93 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +from decimal import Decimal + +from iceberg.api.expressions import Literal +from iceberg.api.transforms import (Identity, + Transforms) +from iceberg.api.types import (DateType, + DecimalType, + LongType, + StringType, + TimestampType, + TimeType) + + +def test_null_human_string(): + long_type = LongType.get() + identity = Identity(long_type) + assert identity.to_human_string(None) == "null" + + +def test_date_human_string(): + date = DateType.get() + + identity = Transforms.identity(date) + date_str = "2017-12-01" + d = Literal.of(date_str).to(date) + assert identity.to_human_string(d.value) == date_str + + +def test_time_human_string(): + time = TimeType.get() + + identity = Transforms.identity(time) + time_str = "10:12:55.038194" + d = Literal.of(time_str).to(time) + assert identity.to_human_string(d.value) == time_str + + +def test_timestamp_with_zone_human_string(): + ts_tz = TimestampType.with_timezone() + identity = Transforms.identity(ts_tz) + ts = Literal.of("2017-12-01T10:12:55.038194-08:00").to(ts_tz) + + assert identity.to_human_string(ts.value) == "2017-12-01T18:12:55.038194Z" + + +def test_timestamp_without_zone_human_string(): + ts_tz = TimestampType.without_timezone() + identity = Transforms.identity(ts_tz) + ts_str = "2017-12-01T10:12:55.038194" + ts = Literal.of(ts_str).to(ts_tz) + + assert identity.to_human_string(ts.value) == ts_str + + +def test_long_to_human_string(): + long_type = LongType.get() + identity = Transforms.identity(long_type) + assert identity.to_human_string(-1234567890000) == "-1234567890000" + + +def test_string_to_human_string(): + str_type = StringType + identity = Transforms.identity(str_type) + + with_slash = "a/b/c=d" + assert identity.to_human_string(with_slash) == with_slash + + +def test_big_decimal_to_human_string(): + big_dec = DecimalType.of(9, 2) + identity = Transforms.identity(big_dec) + + dec_str = "-1.50" + dec_var = Decimal(dec_str) + + assert identity.to_human_string(dec_var) == dec_str diff --git a/tests/api/transforms/test_timestamps.py b/tests/api/transforms/test_timestamps.py new file mode 100644 index 0000000000..98784571df --- /dev/null +++ b/tests/api/transforms/test_timestamps.py @@ -0,0 +1,46 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +from iceberg.api.expressions import Literal +from iceberg.api.transforms import Transforms +from iceberg.api.types import TimestampType +import pytest + + +@pytest.mark.parametrize("lit,type_var", [ + (Literal.of("2017-12-01T10:12:55.038194-08:00"), TimestampType.with_timezone()), + (Literal.of("2017-12-01T18:12:55.038194"), TimestampType.without_timezone())]) +@pytest.mark.parametrize("transform_gran,expected", [ + (Transforms.year, "2017"), + (Transforms.month, "2017-12"), + (Transforms.day, "2017-12-01"), + (Transforms.hour, "2017-12-01-18")]) +def test_ts_to_human_string(lit, type_var, transform_gran, expected): + date_var = lit.to(type_var) + assert (transform_gran(type_var) + .to_human_string(transform_gran(type_var) + .apply(date_var.value))) == expected + + +@pytest.mark.parametrize("transform_gran", [ + Transforms.year, + Transforms.month, + Transforms.day, + Transforms.hour]) +def test_null_human_string(transform_gran): + type_var = TimestampType.with_timezone() + assert "null" == transform_gran(type_var).to_human_string(None) diff --git a/tests/api/transforms/test_truncate.py b/tests/api/transforms/test_truncate.py new file mode 100644 index 0000000000..128d145726 --- /dev/null +++ b/tests/api/transforms/test_truncate.py @@ -0,0 +1,59 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +from decimal import Decimal + +from iceberg.api.transforms import Truncate +from iceberg.api.types import (DecimalType, + IntegerType, + LongType, + StringType) +import pytest + + +@pytest.mark.parametrize("type_var", [IntegerType.get(), LongType.get()]) +@pytest.mark.parametrize("input_var,expected", [ + (1, 0), + (5, 0), + (9, 0), + (10, 10), + (11, 10), + (-1, -10), + (-10, -10), + (-12, -20)]) +def test_truncate_integer(type_var, input_var, expected): + trunc = Truncate.get(type_var, 10) + assert trunc.apply(input_var) == expected + + +@pytest.mark.parametrize("input_var,expected", [ + (Decimal(12.34).quantize(Decimal(".01")), Decimal("12.30")), + (Decimal(12.30).quantize(Decimal(".01")), Decimal("12.30")), + (Decimal(12.20).quantize(Decimal(".01")), Decimal("12.20")), + (Decimal(0.05).quantize(Decimal(".01")), Decimal("0.00")), + (Decimal(-0.05).quantize(Decimal(".01")), Decimal("-0.10"))]) +def test_truncate_decimal(input_var, expected): + trunc = Truncate.get(DecimalType.of(9, 2), 10) + assert trunc.apply(input_var) == expected + + +@pytest.mark.parametrize("input_var,expected", [ + ("abcdefg", "abcde"), + ("abc", "abc")]) +def test_truncate_string(input_var, expected): + trunc = Truncate.get(StringType.get(), 5) + assert trunc.apply(input_var) == expected diff --git a/tests/api/types/__init__.py b/tests/api/types/__init__.py new file mode 100644 index 0000000000..245692337b --- /dev/null +++ b/tests/api/types/__init__.py @@ -0,0 +1,17 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + diff --git a/tests/api/types/test_binary_comparator.py b/tests/api/types/test_binary_comparator.py new file mode 100644 index 0000000000..ca0cf39fbe --- /dev/null +++ b/tests/api/types/test_binary_comparator.py @@ -0,0 +1,41 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +from iceberg.api.expressions import Literal +from iceberg.api.types import FixedType + + +def test_binary_unsigned_comparator(): + b1 = bytearray([0x01, 0x01, 0x02]) + b2 = bytearray([0x01, 0xFF, 0x01]) + + assert Literal.of(b2) > Literal.of(b1) + + +def test_fixed_unsigned_comparator(): + b1 = bytearray([0x01, 0x01, 0x02]) + b2 = bytearray([0x01, 0xFF, 0x01]) + + assert Literal.of(b2) > Literal.of(b1).to(FixedType.of_length(3)) + + +def test_null_handling(): + b1 = bytearray([0x01]) + + assert None < Literal.of(b1) + assert Literal.of(b1) > None + assert Literal.of(b1).to(FixedType.of_length(3)) == Literal.of(b1).to(FixedType.of_length(4)) diff --git a/tests/api/types/test_char_seq_comparator.py b/tests/api/types/test_char_seq_comparator.py new file mode 100644 index 0000000000..1f669fb4c3 --- /dev/null +++ b/tests/api/types/test_char_seq_comparator.py @@ -0,0 +1,43 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +from iceberg.api.expressions import (Literal, + StringLiteral) +import pytest + + +@pytest.mark.parametrize("input_vals", [ + (Literal.of("abc"), Literal.of(u'abc')), # unicode and str are same + (StringLiteral(None), StringLiteral(None)) # None literals are equal +]) +def test_special_equality(input_vals): + assert input_vals[0] == input_vals[1] + + +@pytest.mark.parametrize("input_vals", [ + (Literal.of("abc"), Literal.of('abcd')), # test_seq_length, longer is greater + (Literal.of('abcd'), Literal.of("adc")), # test_char_order, first difference takes precedence over length + (None, Literal.of('abc')) # test_null_handling, null comes before non-null +]) +@pytest.mark.parametrize("eval_func", [ + lambda x, y: y > x, + lambda x, y: x < y]) +def test_seq_length(input_vals, eval_func): + if input_vals[0] is not None: + assert eval_func(input_vals[0].value, input_vals[1].value) + + assert eval_func(input_vals[0], input_vals[1]) diff --git a/tests/api/types/test_comparable_comparator.py b/tests/api/types/test_comparable_comparator.py new file mode 100644 index 0000000000..4d85a28f05 --- /dev/null +++ b/tests/api/types/test_comparable_comparator.py @@ -0,0 +1,51 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +from iceberg.api.expressions import (IntegerLiteral, + Literal) +import pytest + + +@pytest.mark.parametrize("larger,smaller", [ + (34, 33), + (-1, -2)]) +@pytest.mark.parametrize("op", [ + lambda x, y: x > y, + lambda y, x: x < y]) +def test_natural_order(larger, smaller, op): + assert op(Literal.of(larger), Literal.of(smaller)) + + +@pytest.mark.parametrize("input_val", [ + 1, + 0, + -1]) +def test_natural_order_eq(input_val): + assert Literal.of(input_val) == Literal.of(input_val) + + +@pytest.mark.parametrize("larger,smaller", [ + (34, None)]) +@pytest.mark.parametrize("op", [ + lambda x, y: x > y, + lambda y, x: x < y]) +def test_null_handling(larger, smaller, op): + assert op(IntegerLiteral(larger), IntegerLiteral(smaller)) + + +def test_null_handling_eq(): + assert IntegerLiteral(None) == IntegerLiteral(None) diff --git a/tests/api/types/test_readabilty_checks.py b/tests/api/types/test_readabilty_checks.py new file mode 100644 index 0000000000..2acd62276c --- /dev/null +++ b/tests/api/types/test_readabilty_checks.py @@ -0,0 +1,57 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +import unittest + +from iceberg.api.types import (BinaryType, + BooleanType, + DateType, + DecimalType, + DoubleType, + FixedType, + FloatType, + IntegerType, + LongType, + StringType, + TimestampType, + TimeType, + UUIDType) + +PRIMITIVES = [BinaryType.get(), + BooleanType.get(), + DateType.get(), + DecimalType.of(9, 2), + DecimalType.of(11, 2), + DecimalType.of(9, 3), + DoubleType.get(), + FixedType.of_length(3), + FixedType.of_length(4), + FloatType.get(), + IntegerType.get(), + LongType.get(), + StringType.get(), + TimestampType.with_timezone(), + TimestampType.without_timezone(), + TimeType.get(), + UUIDType.get()] + + +class TestReadabilityChecks(unittest.TestCase): + + def test_primitive_types(self): + # TO-DO: Need to implement CheckCompatibility in type_util + pass diff --git a/tests/core/__init__.py b/tests/core/__init__.py new file mode 100644 index 0000000000..245692337b --- /dev/null +++ b/tests/core/__init__.py @@ -0,0 +1,17 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + diff --git a/tests/core/avro/__init__.py b/tests/core/avro/__init__.py new file mode 100644 index 0000000000..245692337b --- /dev/null +++ b/tests/core/avro/__init__.py @@ -0,0 +1,17 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + diff --git a/tests/core/avro/test_avro.py b/tests/core/avro/test_avro.py new file mode 100644 index 0000000000..53227a37c4 --- /dev/null +++ b/tests/core/avro/test_avro.py @@ -0,0 +1,7 @@ +import unittest + + +class TestAvro(unittest.TestCase): + + def test_read_avro(self): + pass diff --git a/tests/core/conftest.py b/tests/core/conftest.py new file mode 100644 index 0000000000..62e7296d44 --- /dev/null +++ b/tests/core/conftest.py @@ -0,0 +1,259 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +import os +import random +import tempfile +import time + +from iceberg.api import Files, PartitionSpec, Schema +from iceberg.api.types import BooleanType, LongType, NestedField +from iceberg.core import (BaseSnapshot, + BaseTable, + ConfigProperties, + GenericManifestFile, + SnapshotLogEntry, + TableMetadata, + TableMetadataParser, + TableOperations) +from iceberg.exceptions import AlreadyExistsException, CommitFailedException +import pytest + +SCHEMA = Schema([NestedField.optional(1, "b", BooleanType.get())]) +METADATA = dict() +VERSIONS = dict() + + +class LocalTableOperations(TableOperations): + + def current(self): + raise RuntimeError("Not implemented for tests") + + def refresh(self): + raise RuntimeError("Not implemented for tests") + + def commit(self, base, metadata): + raise RuntimeError("Not implemented for tests") + + def new_input_file(self, path): + return Files.local_input(path) + + def new_metadata_file(self, filename): + return Files.local_output(tempfile.mkstemp(prefix=filename)) + + def delete_file(self, path): + if os.path.exists(path): + os.remove(path) + + def new_snapshot_id(self): + raise RuntimeError("Not implemented for tests") + + +def create(temp, name, schema, spec): + ops = TestTableOperations(name, temp) + if ops.current() is not None: + raise AlreadyExistsException("Table %s already exists at location: %s" % (name, temp)) + ops.commit(None, TableMetadata.new_table_metadata(ops, schema, spec, str(temp))) + return TestTable(ops, name) + + +def begin_create(temp, name, schema, spec): + raise RuntimeError("Not yet implemented") + # ops = TestTableOperations(name, temp) + # if ops.current() is None: + # raise AlreadyExistsException("Table %s already exists at location: %s" % (name, temp)) + # + # metadata = TableMetadata.new_table_metadata(ops, schema, spec, str(temp)) + # return BaseTransaction.create_table_transaction(ops, metadata) + + +class TestTable(BaseTable): + def __init__(self, ops, name): + super(TestTable, self).__init__(ops, name) + self.ops = ops + + +class TestTableOperations(TableOperations): + + def __init__(self, table_name, location): + self.last_snapshot_id = 0 + self._fail_commits = 0 + self.table_name = table_name + self.metadata = os.path.join(location, "metadata") + os.makedirs(self.metadata) + self._current = None + self.refresh() + if self._current is not None: + for snap in self.current.snapshots: + self.last_snapshot_id = max(self.last_snapshot_id, snap.snapshot_id) + + def current(self): + return self._current + + def refresh(self): + self._current = METADATA.get(self.table_name) + return self._current + + def commit(self, base, metadata): + if base != self.current(): + raise RuntimeError("Cannot commit changes based on stale metadata") + + self.refresh() + if base == self.current: + if self._fail_commits > 0: + self._fail_commits - 1 + raise RuntimeError("Injected failure") + version = VERSIONS.get(self.table_name) + VERSIONS[self.table_name] = 0 if version is None else version + 1 + METADATA[self.table_name] = metadata + self._current = metadata + else: + raise CommitFailedException("Commit failed: table was updated at %s", self.current.last_updated_millis) + + def new_input_file(self, path): + return Files.local_input(path) + + def new_metadata_file(self, filename): + return Files.local_output(os.path.join(self.metadata, filename)) + + def delete_file(self, path): + if not os.remove(path): + raise RuntimeError("Failed to delete file: %s" % path) + + def new_snapshot_id(self): + next_snapshot_id = self.last_snapshot_id + 1 + self.last_snapshot_id = next_snapshot_id + return next_snapshot_id + + +@pytest.fixture(scope="session") +def expected(): + return TableMetadata.new_table_metadata(None, SCHEMA, PartitionSpec.unpartitioned(), "file://tmp/db/table") + + +@pytest.fixture(scope="session", + params=[True, False]) +def prop(request): + config = {ConfigProperties.COMPRESS_METADATA: request.param} + yield request.param + + if os.path.exists(TableMetadataParser.get_file_extension(config)): + os.remove(TableMetadataParser.get_file_extension(config)) + + +@pytest.fixture(scope="session") +def ops(): + return LocalTableOperations() + + +@pytest.fixture(scope="session") +def expected_metadata(): + spec_schema = Schema(NestedField.required(1, "x", LongType.get()), + NestedField.required(2, "y", LongType.get()), + NestedField.required(3, "z", LongType.get())) + spec = PartitionSpec \ + .builder_for(spec_schema) \ + .with_spec_id(5) \ + .build() + + random.seed(1234) + previous_snapshot_id = int(time.time()) - random.randint(0, 3600) + + previous_snapshot = BaseSnapshot(None, previous_snapshot_id, None, + timestamp_millis=previous_snapshot_id, + manifests=[GenericManifestFile(file=Files.local_input("file:/tmp/manfiest.1.avro"), + spec_id=spec.spec_id)]) + + current_snapshot_id = int(time.time()) + current_snapshot = BaseSnapshot(None, current_snapshot_id, previous_snapshot_id, + timestamp_millis=current_snapshot_id, + manifests=[GenericManifestFile(file=Files.local_input("file:/tmp/manfiest.2.avro"), + spec_id=spec.spec_id)]) + + snapshot_log = [SnapshotLogEntry(previous_snapshot.timestamp_millis, previous_snapshot.snapshot_id), + SnapshotLogEntry(current_snapshot.timestamp_millis, current_snapshot.snapshot_id)] + + return TableMetadata(ops, None, "s3://bucket/test/location", + int(time.time()), 3, spec_schema, 5, [spec], {"property": "value"}, current_snapshot_id, + [previous_snapshot, current_snapshot], snapshot_log) + + +@pytest.fixture(scope="session") +def expected_metadata_sorting(): + spec_schema = Schema(NestedField.required(1, "x", LongType.get()), + NestedField.required(2, "y", LongType.get()), + NestedField.required(3, "z", LongType.get())) + + spec = PartitionSpec \ + .builder_for(spec_schema) \ + .with_spec_id(5) \ + .build() + + random.seed(1234) + previous_snapshot_id = int(time.time()) - random.randint(0, 3600) + + previous_snapshot = BaseSnapshot(ops, previous_snapshot_id, None, + timestamp_millis=previous_snapshot_id, + manifests=[GenericManifestFile(file=Files.local_input("file:/tmp/manfiest.1.avro"), + spec_id=spec.spec_id)]) + + current_snapshot_id = int(time.time()) + current_snapshot = BaseSnapshot(ops, current_snapshot_id, previous_snapshot_id, + timestamp_millis=current_snapshot_id, + manifests=[GenericManifestFile(file=Files.local_input("file:/tmp/manfiest.2.avro"), + spec_id=spec.spec_id)]) + + reversed_snapshot_log = list() + metadata = TableMetadata(ops, None, "s3://bucket/test/location", + int(time.time()), 3, spec_schema, 5, [spec], {"property": "value"}, current_snapshot_id, + [previous_snapshot, current_snapshot], reversed_snapshot_log) + + reversed_snapshot_log.append(SnapshotLogEntry(current_snapshot.timestamp_millis, current_snapshot.snapshot_id)) + reversed_snapshot_log.append(SnapshotLogEntry(previous_snapshot.timestamp_millis, previous_snapshot.snapshot_id)) + + return metadata + + +@pytest.fixture(scope="session") +def missing_spec_list(): + schema = Schema(NestedField.required(1, "x", LongType.get()), + NestedField.required(2, "y", LongType.get()), + NestedField.required(3, "z", LongType.get())) + + spec = PartitionSpec.builder_for(schema).identity("x").with_spec_id(6).build() + random.seed(1234) + previous_snapshot_id = int(time.time()) - random.randint(0, 3600) + + previous_snapshot = BaseSnapshot(ops, previous_snapshot_id, None, + timestamp_millis=previous_snapshot_id, + manifests=[GenericManifestFile(file=Files.local_input("file:/tmp/manfiest.1.avro"), + spec_id=spec.spec_id)]) + + current_snapshot_id = int(time.time()) + current_snapshot = BaseSnapshot(ops, current_snapshot_id, previous_snapshot_id, + timestamp_millis=current_snapshot_id, + manifests=[GenericManifestFile(file=Files.local_input("file:/tmp/manfiest.2.avro"), + spec_id=spec.spec_id)]) + return TableMetadata(ops, None, "s3://bucket/test/location", int(time.time()), 3, schema, 6, + (spec,), {"property": "value"}, current_snapshot_id, [previous_snapshot, current_snapshot], + []) + + +@pytest.fixture(scope="session") +def expected_base_snapshot(): + return BaseSnapshot(LocalTableOperations(), int(time.time()), manifests=["file:/tmp/manfiest.1.avro", + "file:/tmp/manfiest.2.avro"]) diff --git a/tests/core/test_snapshot_json.py b/tests/core/test_snapshot_json.py new file mode 100644 index 0000000000..204f4d0fd0 --- /dev/null +++ b/tests/core/test_snapshot_json.py @@ -0,0 +1,26 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +from iceberg.core import SnapshotParser + + +def test_json_conversion(ops, expected_base_snapshot): + as_json = SnapshotParser.to_json(expected_base_snapshot) + snapshot = SnapshotParser.from_json(ops, as_json) + + assert expected_base_snapshot.snapshot_id == snapshot.snapshot_id + assert expected_base_snapshot.manifests == snapshot.manifests diff --git a/tests/core/test_table_metadata_json.py b/tests/core/test_table_metadata_json.py new file mode 100644 index 0000000000..c1c0d7cdff --- /dev/null +++ b/tests/core/test_table_metadata_json.py @@ -0,0 +1,111 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +import json + +from iceberg.core import (PartitionSpecParser, + SchemaParser, + SnapshotLogEntry, + SnapshotParser, + TableMetadata, + TableMetadataParser) + + +def test_json_conversion(ops, expected_metadata): + current_snapshot_id = expected_metadata.current_snapshot_id + previous_snapshot_id = expected_metadata.current_snapshot().parent_id + previous_snapshot = expected_metadata.snapshot(previous_snapshot_id) + + as_json = TableMetadataParser.to_json(expected_metadata) + metadata = TableMetadataParser.from_json(ops, None, as_json) + + assert metadata.location == expected_metadata.location + assert metadata.last_column_id == expected_metadata.last_column_id + assert metadata.schema.as_struct() == expected_metadata.schema.as_struct() + assert str(metadata.spec) == str(expected_metadata.spec) + assert metadata.default_spec_id == expected_metadata.default_spec_id + assert metadata.specs == expected_metadata.specs + assert metadata.properties == expected_metadata.properties + assert metadata.snapshot_log == expected_metadata.snapshot_log + assert metadata.current_snapshot().snapshot_id == current_snapshot_id + assert metadata.current_snapshot().parent_id == previous_snapshot_id + assert metadata.current_snapshot().manifests == expected_metadata.current_snapshot().manifests + assert metadata.snapshot(previous_snapshot_id).snapshot_id == previous_snapshot_id + assert metadata.snapshot(previous_snapshot_id).manifests == previous_snapshot.manifests + + +def test_from_json_sorts_snapshot_log(ops, expected_metadata_sorting): + current_snapshot = expected_metadata_sorting.current_snapshot() + previous_snapshot_id = expected_metadata_sorting.current_snapshot().parent_id + previous_snapshot = expected_metadata_sorting.snapshot(previous_snapshot_id) + + expected_log = [SnapshotLogEntry(previous_snapshot.timestamp_millis, previous_snapshot.snapshot_id), + SnapshotLogEntry(current_snapshot.timestamp_millis, current_snapshot.snapshot_id)] + + as_json = TableMetadataParser.to_json(expected_metadata_sorting) + metadata = TableMetadataParser.from_json(ops, None, as_json) + + assert expected_log == metadata.snapshot_log + + +def test_backward_compatibility_missing_part_spec_list(ops, missing_spec_list): + as_json = to_json_without_spec_list(missing_spec_list) + metadata = TableMetadataParser.from_json(ops, None, as_json) + assert metadata.location == missing_spec_list.location + assert metadata.last_column_id == missing_spec_list.last_column_id + assert metadata.schema.as_struct() == missing_spec_list.schema.as_struct() + assert str(metadata.spec) == str(missing_spec_list.spec) + """ + Assert.assertEquals("Partition spec should be the default", + expected.spec().toString(), metadata.spec().toString()); + Assert.assertEquals("Default spec ID should default to TableMetadata.INITIAL_SPEC_ID", + TableMetadata.INITIAL_SPEC_ID, metadata.defaultSpecId()); + Assert.assertEquals("PartitionSpec should contain the spec", + 1, metadata.specs().size()); + Assert.assertTrue("PartitionSpec should contain the spec", + metadata.specs().get(0).compatibleWith(spec)); + Assert.assertEquals("PartitionSpec should have ID TableMetadata.INITIAL_SPEC_ID", + TableMetadata.INITIAL_SPEC_ID, metadata.specs().get(0).specId()); + Assert.assertEquals("Properties should match", + expected.properties(), metadata.properties()); + Assert.assertEquals("Snapshot logs should match", + expected.snapshotLog(), metadata.snapshotLog()); + Assert.assertEquals("Current snapshot ID should match", + currentSnapshotId, metadata.currentSnapshot().snapshotId()); + Assert.assertEquals("Parent snapshot ID should match", + (Long) previousSnapshotId, metadata.currentSnapshot().parentId()); + Assert.assertEquals("Current snapshot files should match", + currentSnapshot.manifests(), metadata.currentSnapshot().manifests()); + Assert.assertEquals("Previous snapshot ID should match", + previousSnapshotId, metadata.snapshot(previousSnapshotId).snapshotId()); + Assert.assertEquals("Previous snapshot files should match", + previousSnapshot.manifests(), + metadata.snapshot(previousSnapshotId).manifests());""" + + +def to_json_without_spec_list(metadata): + return json.dumps({TableMetadataParser.FORMAT_VERSION: TableMetadata.TABLE_FORMAT_VERSION, + TableMetadataParser.LOCATION: metadata.location, + TableMetadataParser.LAST_UPDATED_MILLIS: metadata.last_updated_millis, + TableMetadataParser.LAST_COLUMN_ID: metadata.last_column_id, + TableMetadataParser.SCHEMA: SchemaParser.to_dict(metadata.schema), + TableMetadataParser.PARTITION_SPEC: PartitionSpecParser.to_json_fields(metadata.spec), + TableMetadataParser.PROPERTIES: metadata.properties, + TableMetadataParser.CURRENT_SNAPSHOT_ID: (-1 if metadata.current_snapshot() is None + else metadata.current_snapshot().snapshot_id), + TableMetadataParser.SNAPSHOTS: [SnapshotParser.to_dict(snapshot) + for snapshot in metadata.snapshots]}) diff --git a/tests/core/test_table_metadata_parser.py b/tests/core/test_table_metadata_parser.py new file mode 100644 index 0000000000..d9d3b28d10 --- /dev/null +++ b/tests/core/test_table_metadata_parser.py @@ -0,0 +1,44 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +import binascii + +from iceberg.api import Files +from iceberg.core import ConfigProperties, TableMetadataParser +from iceberg.core.hadoop import HadoopInputFile + + +def test_compression_property(expected, prop): + config = {ConfigProperties.COMPRESS_METADATA: prop} + output_file = Files.local_output(TableMetadataParser.get_file_extension(config)) + TableMetadataParser.write(expected, output_file) + assert prop == is_compressed(TableMetadataParser.get_file_extension(config)) + read = TableMetadataParser.read(None, + HadoopInputFile.from_location(TableMetadataParser.get_file_extension(config), None)) + verify_metadata(read, expected) + + +def verify_metadata(read, expected): + assert expected.schema.as_struct() == read.schema.as_struct() + assert expected.location == read.location + assert expected.last_column_id == read.last_column_id + assert expected.properties == read.properties + + +def is_compressed(file): + with open(file, 'rb') as test_f: + return binascii.hexlify(test_f.read(2)) == b'1f8b' diff --git a/tox.ini b/tox.ini new file mode 100644 index 0000000000..aee9cc7640 --- /dev/null +++ b/tox.ini @@ -0,0 +1,83 @@ +[tox] +envlist = py36,linters + +[testenv] +deps = + coverage + nose + pytest +setenv = + COVERAGE_FILE = test-reports/{envname}/.coverage + PYTEST_ADDOPTS = --junitxml=test-reports/{envname}/junit.xml -vv +commands = + coverage run --source iceberg --parallel-mode -m pytest {posargs} + coverage combine + coverage report -m + coverage html -d test-reports/{envname}/coverage-html + coverage xml -o test-reports/{envname}/coverage.xml + +[testenv:linters] +basepython = python3 +skip_install = true +deps = + . + {[testenv:flake8]deps} + {[testenv:bandit]deps} +commands = + {[testenv:flake8]commands} + {[testenv:bandit]commands} + +[testenv:flake8] +basepython = python3 +skip_install = true +deps = + flake8 + flake8-import-order>=0.9 +commands = + flake8 iceberg setup.py tests + +[testenv:bandit] +basepython = python3 +skip_install = true +deps = + bandit +commands = + bandit --ini tox.ini -r iceberg_python + +# [testenv:docs] +# basepython = python3 +# deps = +# -r docs/source/requirements.txt +# commands = +# sphinx-build -E -W -c docs/source/ -b html docs/source/ docs/build/html +# sphinx-build -E -W -c docs/source/ -b man docs/source/ docs/build/man + +# [testenv:serve-docs] +# basepython = python3 +# skip_install = true +# changedir = docs/build/html +# deps = +# commands = +# python -m http.server {posargs} + +[bandit] +skips = B104 + +[flake8] +ignore = E501 +exclude = + *.egg-info, + *.pyc, + .cache, + .coverage.*, + .gradle, + .tox, + build, + dist, + htmlcov.* +max-complexity = 10 +import-order-style = google +application-import-names = flake8 + +[pytest] +norecursedirs=.* \ No newline at end of file From e5f22c5a29d6b1c28ae1c52945dd123b732188b9 Mon Sep 17 00:00:00 2001 From: Tao Feng Date: Tue, 19 Mar 2019 10:50:42 -0700 Subject: [PATCH 002/642] Python: Remove unused conversions file (#129) --- iceberg/api/types/#conversions.py# | 87 ------------------------------ 1 file changed, 87 deletions(-) delete mode 100644 iceberg/api/types/#conversions.py# diff --git a/iceberg/api/types/#conversions.py# b/iceberg/api/types/#conversions.py# deleted file mode 100644 index 3e8031da7d..0000000000 --- a/iceberg/api/types/#conversions.py# +++ /dev/null @@ -1,87 +0,0 @@ -1from decimal import Decimal -import struct -import sys -import uuid - -from .type import TypeID - - -class Conversions(object): - HIVE_NULL = "__HIVE_DEFAULT_PARTITION__" - value_mapping = {TypeID.BOOLEAN: lambda as_str: as_str.lower() == "true" if as_str is not None else False, - TypeID.INTEGER: lambda as_str: int(float(as_str)), - TypeID.LONG: lambda as_str: int(float(as_str)), - TypeID.FLOAT: lambda as_str: float(as_str), - TypeID.DOUBLE: lambda as_str: float(as_str), - TypeID.STRING: lambda as_str: as_str, - TypeID.UUID: lambda as_str: uuid.UUID(as_str), - TypeID.FIXED: lambda as_str: bytearray(bytes(as_str, "UTF-8") - if sys.version_info >= (3, 0) - else bytes(as_str)), - TypeID.BINARY: lambda as_str: bytes(as_str, "UTF-8") if sys.version_info >= (3, 0) else bytes(as_str), - TypeID.DECIMAL: lambda as_str: Decimal(as_str), - } - - to_byte_buff_mapping = {TypeID.BOOLEAN: lambda type_var, value: struct.pack("QQ', (value.int >> 64) & 0xFFFFFFFFFFFFFFFF, - value.int & 0xFFFFFFFFFFFFFFFF), - # TypeId.FIXED: lambda as_str: None, - # TypeId.BINARY: lambda as_str: None, - # TypeId.DECIMAL: lambda type_var, value: struct.pack(value.quantize( - # Decimal('.' + "".join(['0' for x in range(0, type_var.scale)]) + '1')) - } - - from_byte_buff_mapping = {TypeID.BOOLEAN: lambda type_var, value: struct.unpack('QQ', -value)[0] << 64 | - struct.unpack('>QQ', value)[1]), - TypeID.FIXED: lambda type_var, value: value, - TypeID.BINARY: lambda type_var, value: value} - - @staticmethod - def from_partition_string(type_var, as_string): - if as_string is None or Conversions.HIVE_NULL == as_string: - return None - part_func = Conversions.value_mapping.get(type_var) - if part_func is None: - raise RuntimeError("Unsupported type for fromPartitionString: %s" % type_var) - - return part_func(as_string) - - @staticmethod - def to_byte_buffer(type_var, value): - byte_buf_func = Conversions.to_byte_buff_mapping.get(type_var) - if byte_buf_func is None: - raise RuntimeError("Cannot Serialize Type: %s" % type_var) - - return byte_buf_func(type_var, value) - - @staticmethod - def from_byte_buffer(type_var, buffer_var): - return Conversions.internal_from_byte_buffer(type_var, buffer_var) - - @staticmethod - def internal_from_byte_buffer(type_var, buffer_var): - tmp = bytearray(len(buffer_var)) - tmp[:] = buffer_var - byte_buf_func = Conversions.from_byte_buff_mapping.get(type_var) - if byte_buf_func is None: - raise RuntimeError("Cannot Serialize Type: %s" % type_var) - - return byte_buf_func(type_var, tmp) From c7adc8fb064e26ae488c1568bc5c7b2650235ced Mon Sep 17 00:00:00 2001 From: TGooch44 Date: Fri, 31 May 2019 12:27:18 -0700 Subject: [PATCH 003/642] Backporting case-sensitivity flag to expression module (#196) * [WIP] Initial python implementation commit * Updating PR per comments from @xhochy * Backporting case-insensitve expressions --- iceberg/api/exceptions/__init__.py | 23 ++- iceberg/api/exceptions/already_exists.py | 19 --- .../api/exceptions/validation_exception.py | 26 ---- iceberg/api/expressions/__init__.py | 8 + iceberg/api/expressions/binder.py | 13 +- iceberg/api/expressions/evaluator.py | 4 +- .../inclusive_metrics_evaluator.py | 15 +- iceberg/api/expressions/predicate.py | 8 +- iceberg/api/expressions/projections.py | 114 ++++++++++++++ iceberg/api/expressions/reference.py | 4 + .../expressions/strict_metrics_evaluator.py | 10 +- iceberg/api/schema.py | 61 +++++--- iceberg/api/table_scan.py | 17 ++- iceberg/api/types/check_compatibility.py | 0 iceberg/api/types/types.py | 35 +++-- tests/api/expressions/__init__.py | 1 - tests/api/expressions/conftest.py | 140 ++++++++++++++++-- tests/api/expressions/test_evaluator.py | 18 +++ .../expressions/test_expression_binding.py | 12 ++ .../test_inclusive_metrics_evaluator.py | 12 ++ 20 files changed, 415 insertions(+), 125 deletions(-) delete mode 100644 iceberg/api/exceptions/already_exists.py delete mode 100644 iceberg/api/exceptions/validation_exception.py create mode 100644 iceberg/api/expressions/projections.py delete mode 100644 iceberg/api/types/check_compatibility.py diff --git a/iceberg/api/exceptions/__init__.py b/iceberg/api/exceptions/__init__.py index d0d7056ef9..e6810ef4e9 100644 --- a/iceberg/api/exceptions/__init__.py +++ b/iceberg/api/exceptions/__init__.py @@ -6,14 +6,21 @@ # "License"); you may not use this file except in compliance # with the License. You may obtain a copy of the License at # -# http://www.apache.org/licenses/LICENSE-2.0 +# http://www.apache.org/licenses/LICENSE-2.0 # -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +__all__ = ["AlreadyExistsException", + "CommitFailedException", + "NoSuchTableException", + "ValidationException"] -from .already_exists import AlreadyExists # noqa -from .validation_exception import ValidationException # noqa \ No newline at end of file +from .exceptions import (AlreadyExistsException, + CommitFailedException, + NoSuchTableException, + ValidationException) diff --git a/iceberg/api/exceptions/already_exists.py b/iceberg/api/exceptions/already_exists.py deleted file mode 100644 index e92de04b35..0000000000 --- a/iceberg/api/exceptions/already_exists.py +++ /dev/null @@ -1,19 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - - -class AlreadyExists(Exception): - pass diff --git a/iceberg/api/exceptions/validation_exception.py b/iceberg/api/exceptions/validation_exception.py deleted file mode 100644 index 7261e6bd8c..0000000000 --- a/iceberg/api/exceptions/validation_exception.py +++ /dev/null @@ -1,26 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - - -class ValidationException(RuntimeError): - - @staticmethod - def check(test, message, args): - if not test: - raise ValidationException(message, args) - - def __init__(self, message, args): - super(ValidationException, self).__init__(message % args) diff --git a/iceberg/api/expressions/__init__.py b/iceberg/api/expressions/__init__.py index 126bdd703a..fdc5448c6c 100644 --- a/iceberg/api/expressions/__init__.py +++ b/iceberg/api/expressions/__init__.py @@ -36,7 +36,9 @@ "FalseExp", "FixedLiteral", "FloatLiteral", + "inclusive", "InclusiveMetricsEvaluator", + "InclusiveProjection", "IntegerLiteral", "JAVA_MAX_FLOAT", "JAVA_MAX_INT", @@ -50,7 +52,9 @@ "Or", "Predicate", "Reference", + "strict", "StrictMetricsEvaluator", + "StrictProjection", "StringLiteral", "TRUE", "TrueExp", @@ -91,6 +95,10 @@ from .predicate import (BoundPredicate, Predicate, UnboundPredicate) +from .projections import (inclusive, + InclusiveProjection, + strict, + StrictProjection) from .reference import (BoundReference, NamedReference, Reference) diff --git a/iceberg/api/expressions/binder.py b/iceberg/api/expressions/binder.py index ad5921ca9a..3be2d46e03 100644 --- a/iceberg/api/expressions/binder.py +++ b/iceberg/api/expressions/binder.py @@ -22,16 +22,16 @@ class Binder(object): @staticmethod - def bind(struct, expr): - return ExpressionVisitors.visit(expr, Binder.BindVisitor(struct)) + def bind(struct, expr, case_sensitive=True): + return ExpressionVisitors.visit(expr, Binder.BindVisitor(struct, case_sensitive)) @staticmethod - def bound_references(struct, exprs): + def bound_references(struct, exprs, case_sensitive=True): if exprs is None: return set() visitor = Binder.ReferenceVisitor() for expr in exprs: - ExpressionVisitors.visit(Binder.bind(struct, expr), visitor) + ExpressionVisitors.visit(Binder.bind(struct, expr, case_sensitive), visitor) return visitor.references @@ -40,8 +40,9 @@ def __init__(self): class BindVisitor(ExpressionVisitors.ExpressionVisitor): - def __init__(self, struct): + def __init__(self, struct, case_sensitive=True): self.struct = struct + self.case_sensitive = case_sensitive def always_true(self): return Expressions.always_true() @@ -62,7 +63,7 @@ def predicate(self, pred): if isinstance(pred, BoundPredicate): raise RuntimeError("Found already bound predicate: {}".format(pred)) - return pred.bind(self.struct) + return pred.bind(self.struct, self.case_sensitive) class ReferenceVisitor(ExpressionVisitors.ExpressionVisitor): diff --git a/iceberg/api/expressions/evaluator.py b/iceberg/api/expressions/evaluator.py index d4bf26933a..340f35215f 100644 --- a/iceberg/api/expressions/evaluator.py +++ b/iceberg/api/expressions/evaluator.py @@ -21,8 +21,8 @@ class Evaluator(object): - def __init__(self, struct, unbound): - self.expr = Binder.bind(struct, unbound) + def __init__(self, struct, unbound, case_sensitive=True): + self.expr = Binder.bind(struct, unbound, case_sensitive) self.visitors = None def visitor(self): diff --git a/iceberg/api/expressions/inclusive_metrics_evaluator.py b/iceberg/api/expressions/inclusive_metrics_evaluator.py index 5b943e2ebf..3134831ecb 100644 --- a/iceberg/api/expressions/inclusive_metrics_evaluator.py +++ b/iceberg/api/expressions/inclusive_metrics_evaluator.py @@ -22,10 +22,11 @@ class InclusiveMetricsEvaluator(object): - def __init__(self, schema, unbound): + def __init__(self, schema, unbound, case_sensitive=True): self.schema = schema self.struct = schema.as_struct() - self.expr = Binder.bind(self.struct, Expressions.rewrite_not(unbound)) + self.case_sensitive = case_sensitive + self.expr = Binder.bind(self.struct, Expressions.rewrite_not(unbound), case_sensitive) self._visitors = None def _visitor(self): @@ -52,13 +53,13 @@ def __init__(self, expr, schema, struct): self.struct = struct def eval(self, file): - if file.record_count <= 0: + if file.record_count() <= 0: return MetricsEvalVisitor.ROWS_CANNOT_MATCH - self.value_counts = file.value_counts - self.null_counts = file.null_value_counts - self.lower_bounds = file.lower_bounds - self.upper_bounds = file.upper_bounds + self.value_counts = file.value_counts() + self.null_counts = file.null_value_counts() + self.lower_bounds = file.lower_bounds() + self.upper_bounds = file.upper_bounds() return ExpressionVisitors.visit(self.expr, self) diff --git a/iceberg/api/expressions/predicate.py b/iceberg/api/expressions/predicate.py index 55625dcc7e..4558dc95de 100644 --- a/iceberg/api/expressions/predicate.py +++ b/iceberg/api/expressions/predicate.py @@ -93,8 +93,12 @@ def __init__(self, op, named_ref, value=None, lit=None): def negate(self): return UnboundPredicate(self.op.negate(), self.ref, self.lit) - def bind(self, struct): # noqa: C901 - field = struct.field(self.ref.name) + def bind(self, struct, case_sensitive=True): # noqa: C901 + if case_sensitive: + field = struct.field(self.ref.name) + else: + field = struct.case_insensitive_field(self.ref.name.lower()) + ValidationException.check(field is not None, "Cannot find field '%s' in struct %s", (self.ref.name, struct)) diff --git a/iceberg/api/expressions/projections.py b/iceberg/api/expressions/projections.py new file mode 100644 index 0000000000..1a4a6bd98f --- /dev/null +++ b/iceberg/api/expressions/projections.py @@ -0,0 +1,114 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + + +from .expressions import Expressions, ExpressionVisitors, RewriteNot +from .predicate import BoundPredicate, UnboundPredicate + + +def inclusive(spec, case_sensitive=True): + return InclusiveProjection(spec, case_sensitive) + + +def strict(spec): + return StrictProjection(spec) + + +class ProjectionEvaluator(ExpressionVisitors.ExpressionVisitor): + + def project(self, expr): + raise NotImplementedError() + + +class BaseProjectionEvaluator(ProjectionEvaluator): + + def __init__(self, spec, case_sensitive=True): + self.spec = spec + self.case_sensitive = case_sensitive + + def project(self, expr): + # projections assume that there are no NOT nodes in the expression tree. to ensure that this + # is the case, the expression is rewritten to push all NOT nodes down to the expression + # leaf nodes. + # this is necessary to ensure that the default expression returned when a predicate can't be + # projected is correct. + # + return ExpressionVisitors.visit(ExpressionVisitors.visit(expr, RewriteNot.get()), self) + + def always_true(self): + return Expressions.always_true() + + def always_false(self): + return Expressions.always_false() + + def not_(self, result): + raise RuntimeError("[BUG] project called on expression with a not") + + def and_(self, left_result, right_result): + return Expressions.and_(left_result, right_result) + + def or_(self, left_result, right_result): + return Expressions.or_(left_result, right_result) + + def predicate(self, pred): + bound = pred.bind(self.spec.schema.as_struct(), case_sensitive=self.case_sensitive) + + if isinstance(bound, BoundPredicate): + return self.predicate(bound) + + return bound + + +class InclusiveProjection(BaseProjectionEvaluator): + + def __init__(self, spec, case_sensitive=True): + super(InclusiveProjection, self).__init__(spec, + case_sensitive=case_sensitive) + + def predicate(self, pred): + if isinstance(pred, UnboundPredicate): + return super(InclusiveProjection, self).predicate(pred) + + part = self.spec.get_field_by_source_id(pred.ref.field_id) + + if part is None: + return self.always_true() + + result = part.transform.project(part.name, pred) + if result is not None: + return result + + return self.always_true() + + +class StrictProjection(BaseProjectionEvaluator): + + def __init__(self, spec): + super(StrictProjection, self).__init__(spec) + + def predicate(self, pred): + part = self.spec.get_field_by_source_id(pred.ref.field_id) + + if part is None: + return self.always_false() + + result = part.transform.project_strict(part.name, pred) + + if result is not None: + return result + + return self.always_false() diff --git a/iceberg/api/expressions/reference.py b/iceberg/api/expressions/reference.py index 1ceeb3a150..d634e34fd5 100644 --- a/iceberg/api/expressions/reference.py +++ b/iceberg/api/expressions/reference.py @@ -29,6 +29,10 @@ def __init__(self, struct, field_id): self.pos = self.find(field_id, struct) self._type = struct.fields[self.pos].type + @property + def type(self): + return self._type + def __eq__(self, other): if id(self) == id(other): return True diff --git a/iceberg/api/expressions/strict_metrics_evaluator.py b/iceberg/api/expressions/strict_metrics_evaluator.py index 1997bdd321..58b7230db7 100644 --- a/iceberg/api/expressions/strict_metrics_evaluator.py +++ b/iceberg/api/expressions/strict_metrics_evaluator.py @@ -51,13 +51,13 @@ def __init__(self, expr, schema, struct): self.upper_bounds = None def eval(self, file): - if file.record_count <= 0: + if file.record_count() <= 0: return StrictMetricsEvaluator.MetricsEvalVisitor.ROWS_MUST_MATCH - self.value_counts = file.value_counts - self.null_counts = file.null_value_counts - self.lower_bounds = file.lower_bounds - self.upper_bounds = file.upper_bounds + self.value_counts = file.value_counts() + self.null_counts = file.null_value_counts() + self.lower_bounds = file.lower_bounds() + self.upper_bounds = file.upper_bounds() return ExpressionVisitors.visit(self.expr, self) diff --git a/iceberg/api/schema.py b/iceberg/api/schema.py index 19a154d96b..76b5339da0 100644 --- a/iceberg/api/schema.py +++ b/iceberg/api/schema.py @@ -28,7 +28,7 @@ class Schema(object): ALL_COLUMNS = "*" def __init__(self, *argv): - aliases = dict() + aliases = None if len(argv) == 1 and isinstance(argv[0], (list, tuple)): columns = argv[0] elif len(argv) == 2 and isinstance(argv[0], list) and isinstance(argv[1], dict): @@ -46,6 +46,7 @@ def __init__(self, *argv): self._id_to_field = None self._name_to_id = None + self._lowercase_name_to_id = None self._id_to_name = None def as_struct(self): @@ -66,25 +67,33 @@ def lazy_name_to_id(self): if self._name_to_id is None: self._name_to_id = index_by_name(self.struct) self._id_to_name = {v: k for k, v in self._name_to_id.items()} + self._lowercase_name_to_id = {k.lower(): v for k, v in self._name_to_id.items()} + return self._name_to_id + def lazy_lowercase_name_to_id(self): + from .types import index_by_name + if self._lowercase_name_to_id is None: + self._name_to_id = index_by_name(self.struct) + self._id_to_name = {v: k for k, v in self._name_to_id.items()} + self._lowercase_name_to_id = {k.lower(): v for k, v in self._name_to_id.items()} + + return self._lowercase_name_to_id + def columns(self): return self.struct.fields def find_type(self, name): - if not name: - raise RuntimeError("Invalid Column Name (empty)") - if isinstance(name, int): field = self.lazy_id_to_field().get(name) if field: return field.type - - id = self.lazy_name_to_id().get(name) - if id: - return self.find_type(id) - - raise RuntimeError("Invalid Column (could not find): %s" % name) + elif isinstance(name, str): + id = self.lazy_name_to_id().get(name) + if id: + return self.find_type(id) + else: + raise RuntimeError("Invalid Column (could not find): %s" % name) def find_field(self, id): if isinstance(id, int): @@ -109,25 +118,29 @@ def id_to_alias(self, field_id): if self._id_to_alias: return self._id_to_alias.get(field_id) - def select(self, *argv): + def select(self, cols): + return self._internal_select(True, cols) + + def case_insensitive_select(self, cols): + return self._internal_select(False, cols) + + def _internal_select(self, case_sensitive, cols): from .types import select - if not(len(argv) == 1 and isinstance(argv[0], list)): - return self.select(argv) - if len(argv) == 1: - names = argv[0] - if Schema.ALL_COLUMNS in names: - return self + if Schema.ALL_COLUMNS in cols: + return self + + selected = set() + for name in cols: + if case_sensitive: + field_id = self.lazy_name_to_id().get(name) else: - selected = list() - for name in names: - id = self.lazy_name_to_id().get(name) - if id: - selected.append(id) + field_id = self.lazy_lowercase_name_to_id().get(name.lower()) - return select(self, selected) # noqa + if field_id is not None: + selected.add(field_id) - raise RuntimeError("Illegal argument for select %s", argv) + return select(self, selected) def __repr__(self): return "Schema(%s)" % self.struct.fields diff --git a/iceberg/api/table_scan.py b/iceberg/api/table_scan.py index e457b9a73f..aa679430fc 100644 --- a/iceberg/api/table_scan.py +++ b/iceberg/api/table_scan.py @@ -21,20 +21,33 @@ class TableScan(object): def __init__(self): raise NotImplementedError() + @property + def row_filter(self): + raise NotImplementedError() + def use_snapshot(self, snapshot_id): raise NotImplementedError() def as_of_time(self, timestamp_millis): raise NotImplementedError() + def project(self, schema): + raise NotImplementedError() + def select(self, columns): raise NotImplementedError() - def filter(self, expr=None): + def filter(self, expr): raise NotImplementedError() - def plan_file(self): + def plan_files(self): raise NotImplementedError() def plan_tasks(self): raise NotImplementedError() + + def is_case_sensitive(self): + raise NotImplementedError() + + def options(self): + raise NotImplementedError() diff --git a/iceberg/api/types/check_compatibility.py b/iceberg/api/types/check_compatibility.py deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/iceberg/api/types/types.py b/iceberg/api/types/types.py index 40e1d88ca8..225bdeee44 100644 --- a/iceberg/api/types/types.py +++ b/iceberg/api/types/types.py @@ -337,7 +337,7 @@ def __ne__(self, other): return not self.__eq__(other) def __repr__(self): - return "fixed[%s]" % (self.length()) + return "fixed[%s]" % (self.length) def __str__(self): return self.__repr__() @@ -475,6 +475,7 @@ def __init__(self, fields): self._fieldList = None self._fieldsByName = None + self._fieldsByLowercaseName = None self._fieldsById = None def __eq__(self, other): @@ -490,16 +491,19 @@ def __ne__(self, other): @property def fields(self): - return self._lazyFieldList() + return self._lazy_field_list() def field(self, name=None, id=None): if name: - return self._lazyFieldsByName().get(name) + return self._lazy_fields_by_name().get(name) elif id: - return self._lazyFieldsById()[id] + return self._lazy_fields_by_id()[id] raise RuntimeError("No valid field info passed in ") + def case_insensitive_field(self, name): + return self._lazy_fields_by_lowercase_name().get(name) + @property def type_id(self): return TypeID.STRUCT @@ -519,27 +523,34 @@ def __hash__(self): def __key(self): return StructType.__class__, self.fields - def _lazyFieldList(self): + def _lazy_field_list(self): if self._fieldList is None: self._fieldList = tuple(self._fields) return self._fieldList - def _lazyFieldsByName(self): + def _lazy_fields_by_name(self): + if self._fieldsByName is None: + self.index_fields() + return self._fieldsByName + + def _lazy_fields_by_lowercase_name(self): if self._fieldsByName is None: - self.indexFields() + self.index_fields() return self._fieldsByName - def _lazyFieldsById(self): + def _lazy_fields_by_id(self): if self._fieldsById is None: - self.indexFields() + self.index_fields() return self._fieldsById - def indexFields(self): + def index_fields(self): self._fieldsByName = dict() + self._fieldsByLowercaseName = dict() self._fieldsById = dict() for field in self.fields: self._fieldsByName[field.name] = field + self._fieldsByLowercaseName[field.name.lower()] = field self._fieldsById[field.id] = field @@ -668,7 +679,7 @@ def field(self, id): return self.value_field def fields(self): - return self._lazyFieldList() + return self._lazy_field_list() def key_id(self): return self.key_field.field_id @@ -702,5 +713,5 @@ def __hash__(self): def __key(self): return MapType.__class__, self.key_field, self.value_field - def _lazyFieldList(self): + def _lazy_field_list(self): return tuple(self.key_field, self.value_field) diff --git a/tests/api/expressions/__init__.py b/tests/api/expressions/__init__.py index 245692337b..13a83393a9 100644 --- a/tests/api/expressions/__init__.py +++ b/tests/api/expressions/__init__.py @@ -14,4 +14,3 @@ # KIND, either express or implied. See the License for the # specific language governing permissions and limitations # under the License. - diff --git a/tests/api/expressions/conftest.py b/tests/api/expressions/conftest.py index 54039b209f..fe1d98205a 100644 --- a/tests/api/expressions/conftest.py +++ b/tests/api/expressions/conftest.py @@ -18,9 +18,10 @@ from decimal import Decimal import io import pickle +import time import uuid -from iceberg.api import DataFile +from iceberg.api import DataFile, ManifestFile, PartitionFieldSummary, PartitionSpec from iceberg.api.expressions import (BoundPredicate, Expressions, ExpressionVisitors, @@ -96,17 +97,83 @@ class TestDataFile(DataFile): def __init__(self, path, partition, record_count, value_counts=None, null_value_counts=None, lower_bounds=None, upper_bounds=None): + self._path = path + self._partition = partition + self._record_count = record_count + self._value_counts = value_counts + self._null_value_counts = null_value_counts + self._lower_bounds = lower_bounds + self._upper_bounds = upper_bounds + self._file_size_in_bytes = 0 + self._block_size_in_bytes = 0 + self._file_ordinal = None + self._column_sizes = None + + def path(self): + return self._path + + def partition(self): + return self._partition + + def record_count(self): + return self._record_count + + def value_counts(self): + return self._value_counts + + def null_value_counts(self): + return self._null_value_counts + + def lower_bounds(self): + return self._lower_bounds + + def upper_bounds(self): + return self._upper_bounds + + def file_size_in_bytes(self): + return self._file_size_in_bytes + + def file_ordinal(self): + return self._file_ordinal + + def column_sizes(self): + return self._column_sizes + + def copy(self): + return self + + +class TestManifestFile(ManifestFile): + + def __init__(self, path, length, spec_id, snapshot_id, added_files, existing_files, deleted_files, partitions): self.path = path - self.partition = partition - self.record_count = record_count - self.value_counts = value_counts - self.null_value_counts = null_value_counts - self.lower_bounds = lower_bounds - self.upper_bounds = upper_bounds - self.file_size_in_bytes = 0 - self.block_size_in_bytes = 0 - self.file_ordinal = None - self.column_size = None + self.length = length + self.spec_id = spec_id + self.snapshot_id = snapshot_id + self.added_files = added_files + self.existing_files = existing_files + self.deleted_files = deleted_files + self.partitions = partitions + + def copy(self): + return self + + +class TestFieldSummary(PartitionFieldSummary): + + def __init__(self, contains_null, lower_bound, upper_bound): + self._contains_null = contains_null + self._lower_bound = lower_bound + self._upper_bound = upper_bound + + def contains_null(self): + return self._contains_null + + def lower_bound(self): + return self._lower_bound + + def upper_bound(self): + return self._upper_bound def copy(self): return self @@ -272,6 +339,18 @@ def not_eq_rewrite(request): yield request.param +@pytest.fixture(scope="session", + params=[Expressions.equal("ID", 5), + Expressions.equal("ID", 29), + Expressions.equal("ID", 30), + Expressions.equal("ID", 75), + Expressions.equal("ID", 79), + Expressions.equal("ID", 80), + Expressions.equal("ID", 85)]) +def not_eq_uc(request): + yield request.param + + @pytest.fixture(scope="session", params=[Literal.of(False), Literal.of(34), @@ -305,3 +384,42 @@ def type_val_tuples(request): (DecimalType.of(9, 4), "34.5600")]) def float_type_val_tuples(request): yield request.param + + +@pytest.fixture(scope="session") +def inc_man_spec(): + inc_schema = Schema(NestedField.required(1, "id", IntegerType.get()), + NestedField.optional(4, "all_nulls", StringType.get()), + NestedField.optional(5, "some_nulls", StringType.get()), + NestedField.optional(6, "no_nulls", StringType.get())) + return (PartitionSpec.builder_for(inc_schema) + .with_spec_id(0) + .identity("id") + .identity("all_nulls") + .identity("some_nulls") + .identity("no_nulls") + .build() + ) + + +@pytest.fixture(scope="session") +def inc_man_file(): + return TestManifestFile("manifest-list.avro", 1024, 0, int(time.time() * 1000), 5, 10, 0, + (TestFieldSummary(False, + Conversions.to_byte_buffer(IntegerType.get(), 30), + Conversions.to_byte_buffer(IntegerType.get(), 79)), + TestFieldSummary(True, + None, + None), + TestFieldSummary(True, + Conversions.to_byte_buffer(StringType.get(), 'a'), + Conversions.to_byte_buffer(StringType.get(), 'z')), + TestFieldSummary(False, + Conversions.to_byte_buffer(StringType.get(), 'a'), + Conversions.to_byte_buffer(StringType.get(), 'z')) + )) + + +@pytest.fixture(scope="session") +def inc_man_file_ns(): + return TestManifestFile("manifest-list.avro", 1024, 0, int(time.time() * 1000), None, None, None, None) diff --git a/tests/api/expressions/test_evaluator.py b/tests/api/expressions/test_evaluator.py index ec3d08f672..b5820f4dc5 100644 --- a/tests/api/expressions/test_evaluator.py +++ b/tests/api/expressions/test_evaluator.py @@ -15,11 +15,14 @@ # specific language governing permissions and limitations # under the License. + import iceberg.api.expressions as exp from iceberg.api.types import (IntegerType, NestedField, StringType, StructType) +from iceberg.exceptions import ValidationException +from pytest import raises STRUCT = StructType.of([NestedField.required(13, "x", IntegerType.get()), NestedField.required(14, "y", IntegerType.get()), @@ -123,6 +126,21 @@ def test_not(row_of): assert evaluator.eval(row_of((8,))) +def test_case_insensitive_not(row_of): + evaluator = exp.evaluator.Evaluator(STRUCT, + exp.expressions.Expressions.not_(exp.expressions.Expressions.equal("X", 7)), + case_sensitive=False) + assert not evaluator.eval(row_of((7,))) + assert evaluator.eval(row_of((8,))) + + +def test_case_sensitive_not(): + with raises(ValidationException): + exp.evaluator.Evaluator(STRUCT, + exp.expressions.Expressions.not_(exp.expressions.Expressions.equal("X", 7)), + case_sensitive=True) + + def test_char_seq_value(row_of): struct = StructType.of([NestedField.required(34, "s", StringType.get())]) evaluator = exp.evaluator.Evaluator(struct, exp.expressions.Expressions.equal("s", "abc")) diff --git a/tests/api/expressions/test_expression_binding.py b/tests/api/expressions/test_expression_binding.py index 47f6fead84..f6a4d983fb 100644 --- a/tests/api/expressions/test_expression_binding.py +++ b/tests/api/expressions/test_expression_binding.py @@ -53,6 +53,18 @@ def test_single_reference(assert_all_bound): Binder.bind(STRUCT, expr)) +def test_case_insensitive_reference(assert_all_bound): + expr = Expressions.not_(Expressions.equal("X", 7)) + assert_all_bound("Single reference", + Binder.bind(STRUCT, expr, case_sensitive=False)) + + +def test_case_sensitive_reference(): + with raises(ice_ex.ValidationException): + expr = Expressions.not_(Expressions.equal("X", 7)) + Binder.bind(STRUCT, expr, case_sensitive=True) + + def test_multiple_references(assert_all_bound): expr = Expressions.or_(Expressions.and_(Expressions.equal("x", 7), Expressions.less_than("y", 100)), diff --git a/tests/api/expressions/test_inclusive_metrics_evaluator.py b/tests/api/expressions/test_inclusive_metrics_evaluator.py index fcbef1e213..e9f0473d1a 100644 --- a/tests/api/expressions/test_inclusive_metrics_evaluator.py +++ b/tests/api/expressions/test_inclusive_metrics_evaluator.py @@ -17,6 +17,7 @@ from iceberg.api.expressions import (Expressions, InclusiveMetricsEvaluator) +from iceberg.exceptions import ValidationException from pytest import raises @@ -116,3 +117,14 @@ def test_integer_not_eq(schema, file, not_eq): def test_not_eq_rewritten(schema, file, not_eq_rewrite): assert InclusiveMetricsEvaluator(schema, Expressions.not_(not_eq_rewrite)).eval(file) + + +def test_case_insensitive_int_not_eq_rewritten(schema, file, not_eq_uc): + assert InclusiveMetricsEvaluator(schema, Expressions.not_(not_eq_uc), + case_sensitive=False).eval(file) + + +def test_case_sensitive_int_not_eq_rewritten(schema, file, not_eq_uc): + with raises(ValidationException): + assert InclusiveMetricsEvaluator(schema, Expressions.not_(not_eq_uc), + case_sensitive=True).eval(file) From e64d4474960a07f17b6d184518ed32f54925cddf Mon Sep 17 00:00:00 2001 From: TGooch44 Date: Wed, 5 Jun 2019 16:30:37 -0700 Subject: [PATCH 004/642] Adding InclusiveManifestEvaluator and ResidualEvaluator (#205) --- iceberg/api/expressions/__init__.py | 4 + iceberg/api/expressions/evaluator.py | 15 +- .../inclusive_manifest_evaluator.py | 161 +++++++++++++++++ .../inclusive_metrics_evaluator.py | 18 +- iceberg/api/expressions/residual_evaluator.py | 119 +++++++++++++ .../expressions/strict_metrics_evaluator.py | 20 ++- .../test_inclusive_manifest_evaluator.py | 166 ++++++++++++++++++ 7 files changed, 481 insertions(+), 22 deletions(-) create mode 100644 iceberg/api/expressions/inclusive_manifest_evaluator.py create mode 100644 iceberg/api/expressions/residual_evaluator.py create mode 100644 tests/api/expressions/test_inclusive_manifest_evaluator.py diff --git a/iceberg/api/expressions/__init__.py b/iceberg/api/expressions/__init__.py index fdc5448c6c..4c2189fb20 100644 --- a/iceberg/api/expressions/__init__.py +++ b/iceberg/api/expressions/__init__.py @@ -37,6 +37,7 @@ "FixedLiteral", "FloatLiteral", "inclusive", + "InclusiveManifestEvaluator", "InclusiveMetricsEvaluator", "InclusiveProjection", "IntegerLiteral", @@ -52,6 +53,7 @@ "Or", "Predicate", "Reference", + "ResidualEvaluator", "strict", "StrictMetricsEvaluator", "StrictProjection", @@ -73,6 +75,7 @@ TRUE, TrueExp) from .expressions import Expressions, ExpressionVisitors +from .inclusive_manifest_evaluator import InclusiveManifestEvaluator from .inclusive_metrics_evaluator import InclusiveMetricsEvaluator from .java_variables import (JAVA_MAX_FLOAT, JAVA_MAX_INT, @@ -102,4 +105,5 @@ from .reference import (BoundReference, NamedReference, Reference) +from .residual_evaluator import ResidualEvaluator from .strict_metrics_evaluator import StrictMetricsEvaluator diff --git a/iceberg/api/expressions/evaluator.py b/iceberg/api/expressions/evaluator.py index 340f35215f..3b0bbc6cf5 100644 --- a/iceberg/api/expressions/evaluator.py +++ b/iceberg/api/expressions/evaluator.py @@ -15,22 +15,25 @@ # specific language governing permissions and limitations # under the License. +import threading + from .binder import Binder from .expressions import ExpressionVisitors class Evaluator(object): + THREAD_LOCAL_DATA = threading.local() + + def visitor(self): + if not hasattr(Evaluator.THREAD_LOCAL_DATA, "visitors") : + Evaluator.THREAD_LOCAL_DATA.visitors = Evaluator.EvalVisitor() + + return Evaluator.THREAD_LOCAL_DATA.visitors def __init__(self, struct, unbound, case_sensitive=True): self.expr = Binder.bind(struct, unbound, case_sensitive) self.visitors = None - def visitor(self): - if self.visitors is None: - self.visitors = Evaluator.EvalVisitor() - - return self.visitors - def eval(self, data): return self.visitor().eval(data, self.expr) diff --git a/iceberg/api/expressions/inclusive_manifest_evaluator.py b/iceberg/api/expressions/inclusive_manifest_evaluator.py new file mode 100644 index 0000000000..e1c55fb6ad --- /dev/null +++ b/iceberg/api/expressions/inclusive_manifest_evaluator.py @@ -0,0 +1,161 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +import threading + +from .binder import Binder +from .expressions import Expressions, ExpressionVisitors +from .projections import inclusive +from ..types import Conversions + +ROWS_MIGHT_MATCH = True +ROWS_CANNOT_MATCH = False + + +class InclusiveManifestEvaluator(object): + + def visitor(self): + if not hasattr(self.thread_local_data, "visitors"): + self.thread_local_data.visitors = ManifestEvalVistor(self.expr) + + return self.thread_local_data.visitors + + def __init__(self, spec, row_filter, case_sensitive=True): + self.struct = spec.partition_type() + self.expr = Binder.bind(self.struct, + Expressions.rewrite_not(inclusive(spec, case_sensitive=case_sensitive) + .project(row_filter)), + case_sensitive=case_sensitive) + self.thread_local_data = threading.local() + + def eval(self, manifest): + return self.visitor().eval(manifest) + + +class ManifestEvalVistor(ExpressionVisitors.BoundExpressionVisitor): + + def __init__(self, expr): + self.expr = expr + self.stats = None + + def eval(self, manifest): + self.stats = manifest.partitions + if self.stats is None: + return ROWS_MIGHT_MATCH + + return ExpressionVisitors.visit(self.expr, self) + + def always_true(self): + return ROWS_MIGHT_MATCH + + def always_false(self): + return ROWS_CANNOT_MATCH + + def not_(self, result): + return not result + + def and_(self, left_result, right_result): + return left_result and right_result + + def or_(self, left_result, right_result): + return left_result or right_result + + def is_null(self, ref): + if not self.stats[ref.pos].contains_null(): + return ROWS_CANNOT_MATCH + + return ROWS_MIGHT_MATCH + + def not_null(self, ref): + lower_bound = self.stats[ref.pos].lower_bound() + if lower_bound is None: + return ROWS_CANNOT_MATCH + + return ROWS_MIGHT_MATCH + + def lt(self, ref, lit): + lower_bound = self.stats[ref.pos].lower_bound() + if lower_bound is None: + return ROWS_CANNOT_MATCH + + lower = Conversions.from_byte_buffer(ref.type, lower_bound) + + if lower >= lit.value: + return ROWS_CANNOT_MATCH + + return ROWS_MIGHT_MATCH + + def lt_eq(self, ref, lit): + lower_bound = self.stats[ref.pos].lower_bound() + if lower_bound is None: + return ROWS_CANNOT_MATCH + + lower = Conversions.from_byte_buffer(ref.type, lower_bound) + + if lower > lit.value: + return ROWS_CANNOT_MATCH + + return ROWS_MIGHT_MATCH + + def gt(self, ref, lit): + upper_bound = self.stats[ref.pos].upper_bound() + if upper_bound is None: + return ROWS_CANNOT_MATCH + + upper = Conversions.from_byte_buffer(ref.type, upper_bound) + + if upper <= lit.value: + return ROWS_CANNOT_MATCH + + return ROWS_MIGHT_MATCH + + def gt_eq(self, ref, lit): + upper_bound = self.stats[ref.pos].upper_bound() + if upper_bound is None: + return ROWS_CANNOT_MATCH + + upper = Conversions.from_byte_buffer(ref.type, upper_bound) + + if upper < lit.value: + return ROWS_CANNOT_MATCH + + return ROWS_MIGHT_MATCH + + def eq(self, ref, lit): + field_stats = self.stats[ref.pos] + if field_stats.lower_bound() is None: + return ROWS_CANNOT_MATCH + + lower = Conversions.from_byte_buffer(ref.type, field_stats.lower_bound()) + if lower > lit.value: + return ROWS_CANNOT_MATCH + + upper = Conversions.from_byte_buffer(ref.type, field_stats.upper_bound()) + + if upper < lit.value: + return ROWS_CANNOT_MATCH + + return ROWS_MIGHT_MATCH + + def not_eq(self, ref, lit): + return ROWS_MIGHT_MATCH + + def in_(self, ref, lit): + return ROWS_MIGHT_MATCH + + def not_in(self, ref, lit): + return ROWS_MIGHT_MATCH diff --git a/iceberg/api/expressions/inclusive_metrics_evaluator.py b/iceberg/api/expressions/inclusive_metrics_evaluator.py index 3134831ecb..ac549355d0 100644 --- a/iceberg/api/expressions/inclusive_metrics_evaluator.py +++ b/iceberg/api/expressions/inclusive_metrics_evaluator.py @@ -15,6 +15,8 @@ # specific language governing permissions and limitations # under the License. +import threading + from .expressions import Expressions, ExpressionVisitors from ..expressions.binder import Binder from ..types import Conversions @@ -22,21 +24,21 @@ class InclusiveMetricsEvaluator(object): + def visitor(self): + if not hasattr(self.thread_local_data, "visitors"): + self.thread_local_data.visitors = MetricsEvalVisitor(self.expr, self.schema, self.struct) + + return self.thread_local_data.visitors + def __init__(self, schema, unbound, case_sensitive=True): self.schema = schema self.struct = schema.as_struct() self.case_sensitive = case_sensitive self.expr = Binder.bind(self.struct, Expressions.rewrite_not(unbound), case_sensitive) - self._visitors = None - - def _visitor(self): - if self._visitors is None: - self._visitors = MetricsEvalVisitor(self.expr, self.schema, self.struct) - - return self._visitors + self.thread_local_data = threading.local() def eval(self, file): - return self._visitor().eval(file) + return self.visitor().eval(file) class MetricsEvalVisitor(ExpressionVisitors.BoundExpressionVisitor): diff --git a/iceberg/api/expressions/residual_evaluator.py b/iceberg/api/expressions/residual_evaluator.py new file mode 100644 index 0000000000..917c49dffe --- /dev/null +++ b/iceberg/api/expressions/residual_evaluator.py @@ -0,0 +1,119 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +import threading + +from .expressions import Expressions, ExpressionVisitors +from .predicate import BoundPredicate, Predicate, UnboundPredicate + + +class ResidualEvaluator(object): + + def visitor(self): + if not hasattr(self.thread_local_data, "visitors"): + self.thread_local_data.visitors = ResidualVisitor() + + return self.thread_local_data.visitors + + def __init__(self, spec, expr): + self._spec = spec + self._expr = expr + self.thread_local_data = threading.local() + + def residual_for(self, partition_data): + return self.visitor().eval(partition_data) + + +class ResidualVisitor(ExpressionVisitors.BoundExpressionVisitor): + + def __init__(self): + self.struct = None + + def eval(self, struct): + self.struct = struct + + def always_true(self): + return Expressions.always_true() + + def always_false(self): + return Expressions.always_false() + + def is_null(self, ref): + return self.always_true() if ref.get(self.struct) is None else self.always_false() + + def not_null(self, ref): + return self.always_true() if ref.get(self.struct) is not None else self.always_false() + + def lt(self, ref, lit): + return self.always_true() if ref.get(self.struct) < lit.value else self.always_false() + + def lt_eq(self, ref, lit): + return self.always_true() if ref.get(self.struct) <= lit.value else self.always_false() + + def gt(self, ref, lit): + return self.always_true() if ref.get(self.struct) > lit.value else self.always_false() + + def gt_eq(self, ref, lit): + return self.always_true() if ref.get(self.struct) >= lit.value else self.always_false() + + def eq(self, ref, lit): + return self.always_true() if ref.get(self.struct) == lit.value else self.always_false() + + def not_eq(self, ref, lit): + return self.always_true() if ref.get(self.struct) != lit.value else self.always_false() + + def not_(self, result): + return Expressions.not_(result) + + def and_(self, left_result, right_result): + return Expressions.and_(left_result, right_result) + + def or_(self, left_result, right_result): + return Expressions.or_(left_result, right_result) + + def predicate(self, pred): + if isinstance(pred, BoundPredicate): + return self.bound_predicate(pred) + elif isinstance(pred, UnboundPredicate): + return self.unbound_predicate(pred) + + raise RuntimeError("Invalid predicate argument %s" % pred) + + def bound_predicate(self, pred): + part = self.spec.get_field_by_source_id(pred.ref.field_id) + if part is None: + return pred + + strict_projection = part.transform.project_strict(part.name, pred) + if strict_projection is None: + bound = strict_projection.bind(self.spec.partition_type()) + if isinstance(bound, BoundPredicate): + return super(ResidualVisitor, self).predicate(bound) + return bound + + return pred + + def unbound_predicate(self, pred): + bound = pred.bind(self.spec.schema.as_struct()) + + if isinstance(bound, BoundPredicate): + bound_residual = self.predicate(bound) + if isinstance(bound_residual, Predicate): + return pred + return bound_residual + + return bound diff --git a/iceberg/api/expressions/strict_metrics_evaluator.py b/iceberg/api/expressions/strict_metrics_evaluator.py index 58b7230db7..c8c8a78c3d 100644 --- a/iceberg/api/expressions/strict_metrics_evaluator.py +++ b/iceberg/api/expressions/strict_metrics_evaluator.py @@ -15,6 +15,8 @@ # specific language governing permissions and limitations # under the License. +import threading + from .expressions import Expressions, ExpressionVisitors from ..expressions.binder import Binder from ..types import Conversions @@ -22,20 +24,22 @@ class StrictMetricsEvaluator(object): + def visitor(self): + if not hasattr(self.thread_local_data, "visitors"): + self.thread_local_data.visitors = StrictMetricsEvaluator.MetricsEvalVisitor(self.expr, + self.schema, + self.struct) + + return self.thread_local_data.visitors + def __init__(self, schema, unbound): self.schema = schema self.struct = schema.as_struct() self.expr = Binder.bind(self.struct, Expressions.rewrite_not(unbound)) - self._visitors = None - - def _visitor(self): - if self._visitors is None: - self._visitors = StrictMetricsEvaluator.MetricsEvalVisitor(self.expr, self.schema, self.struct) - - return self._visitors + self.thread_local_data = threading.local() def eval(self, file): - return self._visitor().eval(file) + return self.visitor().eval(file) class MetricsEvalVisitor(ExpressionVisitors.BoundExpressionVisitor): ROWS_MUST_MATCH = True diff --git a/tests/api/expressions/test_inclusive_manifest_evaluator.py b/tests/api/expressions/test_inclusive_manifest_evaluator.py new file mode 100644 index 0000000000..165ed2a75c --- /dev/null +++ b/tests/api/expressions/test_inclusive_manifest_evaluator.py @@ -0,0 +1,166 @@ + +from iceberg.api.expressions import Expressions, InclusiveManifestEvaluator +from iceberg.exceptions import ValidationException +import pytest + + +@pytest.mark.parametrize("expression,expected", [ + (Expressions.not_null("all_nulls"), False), + (Expressions.not_null("some_nulls"), True), + (Expressions.not_null("no_nulls"), True)]) +def test_all_nulls(inc_man_spec, inc_man_file, expression, expected): + assert InclusiveManifestEvaluator(inc_man_spec, expression).eval(inc_man_file) == expected + + +@pytest.mark.parametrize("expression,expected", [ + (Expressions.is_null("all_nulls"), True), + (Expressions.is_null("some_nulls"), True), + (Expressions.is_null("no_nulls"), False)]) +def test_no_nulls(inc_man_spec, inc_man_file, expression, expected): + assert InclusiveManifestEvaluator(inc_man_spec, expression).eval(inc_man_file) == expected + + +def test_missing_column(inc_man_spec, inc_man_file): + with pytest.raises(ValidationException): + InclusiveManifestEvaluator(inc_man_spec, Expressions.less_than("missing", 5)).eval(inc_man_file) + + +@pytest.mark.parametrize("expression", [ + Expressions.less_than("id", 5), + Expressions.less_than_or_equal("id", 30), + Expressions.equal("id", 70), + Expressions.greater_than("id", 78), + Expressions.greater_than_or_equal("id", 90), + Expressions.not_equal("id", 101), + Expressions.less_than_or_equal("id", 30), + Expressions.is_null("id"), + Expressions.not_null("id")]) +def test_missing_stats(inc_man_spec, inc_man_file_ns, expression): + assert InclusiveManifestEvaluator(inc_man_spec, expression).eval(inc_man_file_ns) + + +@pytest.mark.parametrize("expression, expected", [ + (Expressions.less_than("id", 5), True), + (Expressions.greater_than("id", 5), False)]) +def test_not(inc_man_spec, inc_man_file, expression, expected): + assert InclusiveManifestEvaluator(inc_man_spec, Expressions.not_(expression)).eval(inc_man_file) == expected + + +@pytest.mark.parametrize("expr1, expr2, expected", [ + (Expressions.less_than("id", 5), Expressions.greater_than_or_equal("id", 0), False), + (Expressions.greater_than("id", 5), Expressions.less_than_or_equal("id", 30), True)]) +def test_and(inc_man_spec, inc_man_file, expr1, expr2, expected): + assert InclusiveManifestEvaluator(inc_man_spec, Expressions.and_(expr1, expr2)).eval(inc_man_file) == expected + + +@pytest.mark.parametrize("expr1, expr2, expected", [ + (Expressions.less_than("id", 5), Expressions.greater_than_or_equal("id", 80), False), + (Expressions.less_than("id", 5), Expressions.greater_than_or_equal("id", 60), True)]) +def test_or(inc_man_spec, inc_man_file, expr1, expr2, expected): + assert InclusiveManifestEvaluator(inc_man_spec, Expressions.or_(expr1, expr2)).eval(inc_man_file) == expected + + +@pytest.mark.parametrize("val, expected", [ + (5, False), + (30, False), + (31, True), + (79, True)]) +def test_int_lt(inc_man_spec, inc_man_file, val, expected): + assert InclusiveManifestEvaluator(inc_man_spec, Expressions.less_than("id", val)).eval(inc_man_file) == expected + + +@pytest.mark.parametrize("val, expected", [ + (5, False), + (29, False), + (30, True), + (79, True)]) +def test_int_lt_eq(inc_man_spec, inc_man_file, val, expected): + assert InclusiveManifestEvaluator(inc_man_spec, + Expressions.less_than_or_equal("id", val)).eval(inc_man_file) == expected + + +@pytest.mark.parametrize("val, expected", [ + (85, False), + (79, False), + (78, True), + (75, True)]) +def test_int_gt(inc_man_spec, inc_man_file, val, expected): + assert InclusiveManifestEvaluator(inc_man_spec, Expressions.greater_than("id", val)).eval(inc_man_file) == expected + + +@pytest.mark.parametrize("val, expected", [ + (85, False), + (80, False), + (79, True), + (75, True)]) +def test_int_gt_eq(inc_man_spec, inc_man_file, val, expected): + assert InclusiveManifestEvaluator(inc_man_spec, + Expressions.greater_than_or_equal("id", val)).eval(inc_man_file) == expected + + +@pytest.mark.parametrize("val, expected", [ + (5, False), + (29, False), + (30, True), + (75, True), + (79, True), + (80, False), + (85, False)]) +def test_int_eq(inc_man_spec, inc_man_file, val, expected): + assert InclusiveManifestEvaluator(inc_man_spec, + Expressions.equal("id", val)).eval(inc_man_file) == expected + + +@pytest.mark.parametrize("val, expected", [ + (5, True), + (29, True), + (30, True), + (75, True), + (79, True), + (80, True), + (85, True)]) +def test_int_not_eq(inc_man_spec, inc_man_file, val, expected): + assert InclusiveManifestEvaluator(inc_man_spec, + Expressions.not_equal("id", val)).eval(inc_man_file) == expected + + +@pytest.mark.parametrize("val, expected", [ + (5, True), + (29, True), + (30, True), + (75, True), + (79, True), + (80, True), + (85, True)]) +def test_int_not_eq_rewritten(inc_man_spec, inc_man_file, val, expected): + assert InclusiveManifestEvaluator(inc_man_spec, + Expressions.not_(Expressions.equal("id", val))).eval(inc_man_file) == expected + + +@pytest.mark.parametrize("val, expected", [ + (5, True), + (29, True), + (30, True), + (75, True), + (79, True), + (80, True), + (85, True)]) +def test_case_insensitive_int_not_eq_rewritten(inc_man_spec, inc_man_file, val, expected): + assert InclusiveManifestEvaluator(inc_man_spec, + Expressions.not_(Expressions.equal("ID", val)), + case_sensitive=False).eval(inc_man_file) == expected + + +@pytest.mark.parametrize("val, expected", [ + (5, True), + (29, True), + (30, True), + (75, True), + (79, True), + (80, True), + (85, True)]) +def test_case_sensitive_int_not_eq_rewritten(inc_man_spec, inc_man_file, val, expected): + with pytest.raises(ValidationException): + InclusiveManifestEvaluator(inc_man_spec, + Expressions.not_(Expressions.equal("ID", val)), + case_sensitive=True).eval(inc_man_file) == expected From 18998acabcdbfd19f6cd609d764454e3af213426 Mon Sep 17 00:00:00 2001 From: TGooch44 Date: Thu, 6 Jun 2019 13:39:05 -0700 Subject: [PATCH 005/642] Adding Bin Packing util for Combining scan tasks (#208) --- iceberg/core/util/bin_packing.py | 77 ++++++++++++++++++++++++++++ tests/core/utils/__init__.py | 0 tests/core/utils/test_bin_packing.py | 36 +++++++++++++ 3 files changed, 113 insertions(+) create mode 100644 iceberg/core/util/bin_packing.py create mode 100644 tests/core/utils/__init__.py create mode 100644 tests/core/utils/test_bin_packing.py diff --git a/iceberg/core/util/bin_packing.py b/iceberg/core/util/bin_packing.py new file mode 100644 index 0000000000..d7a3105351 --- /dev/null +++ b/iceberg/core/util/bin_packing.py @@ -0,0 +1,77 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +from copy import deepcopy + +class PackingIterator(object): + + def __init__(self, items, target_weight, lookback, weight_func): + self.items = deepcopy(items) + self.target_weight = target_weight + self.lookback = lookback + self.weight_func = weight_func + self.bins = list() + + def __iter__(self): + return self + + def __next__(self): + item_copy = list(self.items) + i = 0 + for item in item_copy: + weight = self.weight_func(item) + curr_bin = PackingIterator.find(self.bins, weight) + if curr_bin is not None: + curr_bin.add(item, weight) + self.items.pop(i) + i -= 1 + else: + curr_bin = Bin(self.target_weight) + self.items.pop(i) + i -= 1 + curr_bin.add(item, weight) + self.bins.append(curr_bin) + + if len(self.bins) > self.lookback: + return list(self.bins.pop(0).items) + i += 1 + + if len(self.bins) == 0: + raise StopIteration() + + return list(self.bins.pop(0).items) + + @staticmethod + def find(bins, weight): + for curr_bin in bins: + if curr_bin.can_add(weight): + return curr_bin + + +class Bin(object): + + def __init__(self, target_weight): + self.bin_weight = 0 + self.target_weight = target_weight + self.items = list() + + def can_add(self, weight): + return self.bin_weight + weight <= self.target_weight + + def add(self, item, weight): + self.bin_weight += weight + self.items.append(item) diff --git a/tests/core/utils/__init__.py b/tests/core/utils/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/tests/core/utils/test_bin_packing.py b/tests/core/utils/test_bin_packing.py new file mode 100644 index 0000000000..a35386eba2 --- /dev/null +++ b/tests/core/utils/test_bin_packing.py @@ -0,0 +1,36 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +import random + +from iceberg.core.util import PackingIterator +import pytest + + +@pytest.mark.parametrize("splits,lookback,split_size, open_cost", [ + ([random.randint(0, 64) for x in range(200)], 20, 128, 4), # random splits + ([], 20, 128, 4), # no splits + ([0]*100 + [random.randint(0, 64) for x in range(10)] + [0]*100, 20, 128, 4) # sparse +]) +def test_bin_packing(splits, lookback, split_size, open_cost): + + def weight_func(x): + return max(x, open_cost) + + item_list_sums = [sum(item) + for item in PackingIterator(splits, split_size, lookback, weight_func)] + assert all([split_size >= item_sum >= 0 for item_sum in item_list_sums]) From ecf02a01aa77311d2d29ac2c5411b1ca9f63d897 Mon Sep 17 00:00:00 2001 From: chyzzqo2 <52714809+chyzzqo2@users.noreply.github.com> Date: Tue, 6 Aug 2019 12:41:53 -0700 Subject: [PATCH 006/642] Add Travis CI config for Python (#354) --- iceberg/__init__.py | 1 - iceberg/api/expressions/evaluator.py | 2 +- iceberg/api/expressions/strict_metrics_evaluator.py | 8 +++++--- iceberg/core/util/bin_packing.py | 1 + tests/__init__.py | 1 - tests/api/__init__.py | 1 - tests/api/test_helpers.py | 12 ++++++------ tests/api/transforms/__init__.py | 1 - tests/api/types/__init__.py | 1 - tests/core/__init__.py | 1 - tests/core/avro/__init__.py | 1 - tests/core/utils/test_bin_packing.py | 4 ++-- 12 files changed, 15 insertions(+), 19 deletions(-) diff --git a/iceberg/__init__.py b/iceberg/__init__.py index 245692337b..13a83393a9 100644 --- a/iceberg/__init__.py +++ b/iceberg/__init__.py @@ -14,4 +14,3 @@ # KIND, either express or implied. See the License for the # specific language governing permissions and limitations # under the License. - diff --git a/iceberg/api/expressions/evaluator.py b/iceberg/api/expressions/evaluator.py index 3b0bbc6cf5..3f2a29fd69 100644 --- a/iceberg/api/expressions/evaluator.py +++ b/iceberg/api/expressions/evaluator.py @@ -25,7 +25,7 @@ class Evaluator(object): THREAD_LOCAL_DATA = threading.local() def visitor(self): - if not hasattr(Evaluator.THREAD_LOCAL_DATA, "visitors") : + if not hasattr(Evaluator.THREAD_LOCAL_DATA, "visitors"): Evaluator.THREAD_LOCAL_DATA.visitors = Evaluator.EvalVisitor() return Evaluator.THREAD_LOCAL_DATA.visitors diff --git a/iceberg/api/expressions/strict_metrics_evaluator.py b/iceberg/api/expressions/strict_metrics_evaluator.py index c8c8a78c3d..a1752f992c 100644 --- a/iceberg/api/expressions/strict_metrics_evaluator.py +++ b/iceberg/api/expressions/strict_metrics_evaluator.py @@ -26,9 +26,11 @@ class StrictMetricsEvaluator(object): def visitor(self): if not hasattr(self.thread_local_data, "visitors"): - self.thread_local_data.visitors = StrictMetricsEvaluator.MetricsEvalVisitor(self.expr, - self.schema, - self.struct) + self.thread_local_data.visitors = StrictMetricsEvaluator.MetricsEvalVisitor( + self.expr, + self.schema, + self.struct + ) return self.thread_local_data.visitors diff --git a/iceberg/core/util/bin_packing.py b/iceberg/core/util/bin_packing.py index d7a3105351..1ccca2ec74 100644 --- a/iceberg/core/util/bin_packing.py +++ b/iceberg/core/util/bin_packing.py @@ -17,6 +17,7 @@ from copy import deepcopy + class PackingIterator(object): def __init__(self, items, target_weight, lookback, weight_func): diff --git a/tests/__init__.py b/tests/__init__.py index 245692337b..13a83393a9 100644 --- a/tests/__init__.py +++ b/tests/__init__.py @@ -14,4 +14,3 @@ # KIND, either express or implied. See the License for the # specific language governing permissions and limitations # under the License. - diff --git a/tests/api/__init__.py b/tests/api/__init__.py index 245692337b..13a83393a9 100644 --- a/tests/api/__init__.py +++ b/tests/api/__init__.py @@ -14,4 +14,3 @@ # KIND, either express or implied. See the License for the # specific language governing permissions and limitations # under the License. - diff --git a/tests/api/test_helpers.py b/tests/api/test_helpers.py index baaabbe3b8..c526e5f55b 100644 --- a/tests/api/test_helpers.py +++ b/tests/api/test_helpers.py @@ -49,14 +49,14 @@ def set(self, pos, value): class CheckReferencesBound(ExpressionVisitors.ExpressionVisitor): - def __init__(self, message): - self.message = message + def __init__(self, message): + self.message = message - def predicate(self, pred): - if isinstance(pred, UnboundPredicate): - assert_true(False) + def predicate(self, pred): + if isinstance(pred, UnboundPredicate): + assert_true(False) - return None + return None class TestDataFile(DataFile): diff --git a/tests/api/transforms/__init__.py b/tests/api/transforms/__init__.py index 245692337b..13a83393a9 100644 --- a/tests/api/transforms/__init__.py +++ b/tests/api/transforms/__init__.py @@ -14,4 +14,3 @@ # KIND, either express or implied. See the License for the # specific language governing permissions and limitations # under the License. - diff --git a/tests/api/types/__init__.py b/tests/api/types/__init__.py index 245692337b..13a83393a9 100644 --- a/tests/api/types/__init__.py +++ b/tests/api/types/__init__.py @@ -14,4 +14,3 @@ # KIND, either express or implied. See the License for the # specific language governing permissions and limitations # under the License. - diff --git a/tests/core/__init__.py b/tests/core/__init__.py index 245692337b..13a83393a9 100644 --- a/tests/core/__init__.py +++ b/tests/core/__init__.py @@ -14,4 +14,3 @@ # KIND, either express or implied. See the License for the # specific language governing permissions and limitations # under the License. - diff --git a/tests/core/avro/__init__.py b/tests/core/avro/__init__.py index 245692337b..13a83393a9 100644 --- a/tests/core/avro/__init__.py +++ b/tests/core/avro/__init__.py @@ -14,4 +14,3 @@ # KIND, either express or implied. See the License for the # specific language governing permissions and limitations # under the License. - diff --git a/tests/core/utils/test_bin_packing.py b/tests/core/utils/test_bin_packing.py index a35386eba2..8e5ccc0c6a 100644 --- a/tests/core/utils/test_bin_packing.py +++ b/tests/core/utils/test_bin_packing.py @@ -17,14 +17,14 @@ import random -from iceberg.core.util import PackingIterator +from iceberg.core.util.bin_packing import PackingIterator import pytest @pytest.mark.parametrize("splits,lookback,split_size, open_cost", [ ([random.randint(0, 64) for x in range(200)], 20, 128, 4), # random splits ([], 20, 128, 4), # no splits - ([0]*100 + [random.randint(0, 64) for x in range(10)] + [0]*100, 20, 128, 4) # sparse + ([0] * 100 + [random.randint(0, 64) for x in range(10)] + [0] * 100, 20, 128, 4) # sparse ]) def test_bin_packing(splits, lookback, split_size, open_cost): From 32e52983d31b25a4414fcc1697dcb9655ed100f1 Mon Sep 17 00:00:00 2001 From: chyzzqo2 <52714809+chyzzqo2@users.noreply.github.com> Date: Wed, 7 Aug 2019 10:56:16 -0700 Subject: [PATCH 007/642] Python: Remove unused dependencies (#355) --- setup.py | 4 +--- tox.ini | 4 ++-- 2 files changed, 3 insertions(+), 5 deletions(-) diff --git a/setup.py b/setup.py index af55914a2d..5a5710167e 100755 --- a/setup.py +++ b/setup.py @@ -29,11 +29,9 @@ install_requires=['boto3', 'fastavro', 'mmh3', - "pytest", 'python-dateutil', 'pytz', 'retrying', - 's3fs', - 'six'], + 's3fs'], setup_requires=['setupmeta'] ) diff --git a/tox.ini b/tox.ini index aee9cc7640..7a600ab8c6 100644 --- a/tox.ini +++ b/tox.ini @@ -42,7 +42,7 @@ skip_install = true deps = bandit commands = - bandit --ini tox.ini -r iceberg_python + bandit --ini tox.ini -r iceberg # [testenv:docs] # basepython = python3 @@ -80,4 +80,4 @@ import-order-style = google application-import-names = flake8 [pytest] -norecursedirs=.* \ No newline at end of file +norecursedirs=.* From 055e7290dc9e3552af3eb12624090450d254a39f Mon Sep 17 00:00:00 2001 From: TGooch44 Date: Fri, 9 Aug 2019 12:18:43 -0700 Subject: [PATCH 008/642] Moving/Renameing hadoop module to filesystem (#277) --- iceberg/core/avro/avro_schema_util.py | 14 +- .../core/base_metastore_table_operations.py | 43 ++- iceberg/core/data_files.py | 8 +- .../core/{hadoop => filesystem}/__init__.py | 10 +- .../{hadoop => filesystem}/file_status.py | 0 .../file_system.py} | 69 ++++- .../filesystem/filesystem_table_operations.py | 140 ++++++++++ iceberg/core/filesystem/filesystem_tables.py | 53 ++++ .../local_filesystem.py | 65 +++-- iceberg/core/filesystem/s3_filesystem.py | 257 ++++++++++++++++++ iceberg/core/{hadoop => filesystem}/util.py | 28 +- iceberg/core/hadoop/file_system.py | 25 -- iceberg/core/hadoop/hadoop_output_file.py | 49 ---- .../core/hadoop/hadoop_table_operations.py | 49 ---- iceberg/core/hadoop/s3_filesystem_wrapper.py | 54 ---- iceberg/core/table_properties.py | 77 ++++++ iceberg/core/util/__init__.py | 7 +- iceberg/core/util/bin_packing.py | 1 + iceberg/spark/__init__.py | 19 -- iceberg/spark/source/__init__.py | 19 -- iceberg/spark/source/spark_catalog.py | 53 ---- iceberg/spark/table_identifier.py | 21 -- setup.py | 7 +- tests/core/test_table_metadata_parser.py | 7 +- tox.ini | 2 +- 25 files changed, 696 insertions(+), 381 deletions(-) rename iceberg/core/{hadoop => filesystem}/__init__.py (63%) rename iceberg/core/{hadoop => filesystem}/file_status.py (100%) rename iceberg/core/{hadoop/hadoop_input_file.py => filesystem/file_system.py} (50%) create mode 100644 iceberg/core/filesystem/filesystem_table_operations.py create mode 100644 iceberg/core/filesystem/filesystem_tables.py rename iceberg/core/{hadoop => filesystem}/local_filesystem.py (63%) create mode 100644 iceberg/core/filesystem/s3_filesystem.py rename iceberg/core/{hadoop => filesystem}/util.py (60%) delete mode 100644 iceberg/core/hadoop/file_system.py delete mode 100644 iceberg/core/hadoop/hadoop_output_file.py delete mode 100644 iceberg/core/hadoop/hadoop_table_operations.py delete mode 100644 iceberg/core/hadoop/s3_filesystem_wrapper.py create mode 100644 iceberg/core/table_properties.py delete mode 100644 iceberg/spark/__init__.py delete mode 100644 iceberg/spark/source/__init__.py delete mode 100644 iceberg/spark/source/spark_catalog.py delete mode 100644 iceberg/spark/table_identifier.py diff --git a/iceberg/core/avro/avro_schema_util.py b/iceberg/core/avro/avro_schema_util.py index efe729987b..5d478908bf 100644 --- a/iceberg/core/avro/avro_schema_util.py +++ b/iceberg/core/avro/avro_schema_util.py @@ -6,14 +6,14 @@ # "License"); you may not use this file except in compliance # with the License. You may obtain a copy of the License at # -# http://www.apache.org/licenses/LICENSE-2.0 +# http://www.apache.org/licenses/LICENSE-2.0 # -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. class AvroSchemaUtil(object): FIELD_ID_PROP = "field-id" diff --git a/iceberg/core/base_metastore_table_operations.py b/iceberg/core/base_metastore_table_operations.py index b536110e78..a50f34283b 100644 --- a/iceberg/core/base_metastore_table_operations.py +++ b/iceberg/core/base_metastore_table_operations.py @@ -18,13 +18,11 @@ import logging import uuid -from iceberg.core.hadoop import (get_fs, - HadoopInputFile, - HadoopOutputFile) from retrying import retry from .table_metadata_parser import TableMetadataParser from .table_operations import TableOperations +from .table_properties import TableProperties _logger = logging.getLogger(__name__) @@ -46,6 +44,7 @@ def __init__(self, conf): self.current_metadata = None self.current_metadata_location = None self.base_location = None + self.should_refresh = True self.version = -1 def current(self): @@ -59,13 +58,18 @@ def data_location(self): return "{base_location}/{data}".format(base_location=self.base_location, data=BaseMetastoreTableOperations.DATA_FOLDER_NAME) + def request_refresh(self): + self.should_refresh = True + def write_new_metadata(self, metadata, version): + from .filesystem import FileSystemOutputFile + if self.base_location is None: self.base_location = metadata.location new_filename = BaseMetastoreTableOperations.new_table_metadata_filename(self.base_location, version) - new_metadata_location = HadoopOutputFile.from_path(new_filename, self.conf) + new_metadata_location = FileSystemOutputFile.from_path(new_filename, self.conf) TableMetadataParser.write(metadata, new_metadata_location) return new_filename @@ -73,24 +77,43 @@ def write_new_metadata(self, metadata, version): def refresh_from_metadata_location(self, new_location, num_retries=20): if not self.current_metadata_location == new_location: _logger.info("Refreshing table metadata from new version: %s" % new_location) + self.retryable_refresh(new_location) - self.retryable_refresh(new_location) + self.should_refresh = False def new_input_file(self, path): - return HadoopInputFile.from_location(path, self.conf) + from .filesystem import FileSystemInputFile + + return FileSystemInputFile.from_location(path, self.conf) def new_metadata_file(self, filename): - return HadoopOutputFile.from_path(BaseMetastoreTableOperations.new_metadata_location(self.base_location, - filename), - self.conf) + from .filesystem import FileSystemOutputFile + + return FileSystemOutputFile.from_path(BaseMetastoreTableOperations.new_metadata_location(self.base_location, + filename), + self.conf) + + def metadata_file_location(self, file_name, metadata=None): + if metadata is None: + return self.metadata_file_location(file_name, metadata=self.current()) + + metadata_location = metadata.properties.get(TableProperties.WRITE_METADATA_LOCATION) + + if metadata_location is not None: + return "{}/{}".format(metadata_location, file_name) + else: + return "{}/{}/{}".format(metadata.location, BaseMetastoreTableOperations.METADATA_FOLDER_NAME, file_name) def delete_file(self, path): + from .filesystem import get_fs get_fs(path, self.conf).delete(path, False) @retry(wait_incrementing_start=100, wait_exponential_multiplier=4, wait_exponential_max=5000, stop_max_delay=600000, stop_max_attempt_number=2) def retryable_refresh(self, location): - self.current_metadata = TableMetadataParser.read(self, HadoopInputFile.from_location(location, self.conf)) + from .filesystem import FileSystemInputFile + + self.current_metadata = TableMetadataParser.read(self, FileSystemInputFile.from_location(location, self.conf)) self.current_metadata_location = location self.base_location = self.current_metadata.location self.version = BaseMetastoreTableOperations.parse_version(location) diff --git a/iceberg/core/data_files.py b/iceberg/core/data_files.py index 34d4422ac0..0a50e5babd 100644 --- a/iceberg/core/data_files.py +++ b/iceberg/core/data_files.py @@ -20,8 +20,8 @@ Metrics) from iceberg.api.types import Conversions +from .filesystem import FileSystemInputFile from .generic_data_file import GenericDataFile -from .hadoop import HadoopInputFile from .partition_data import PartitionData @@ -78,7 +78,7 @@ def copy(spec, partition): @staticmethod def from_input_file(input_file, row_count, partition_data=None, metrics=None): - if isinstance(input_file, HadoopInputFile): + if isinstance(input_file, FileSystemInputFile): return DataFiles.from_stat(input_file.get_stat(), row_count, partition_data=partition_data, metrics=metrics) @@ -124,6 +124,7 @@ def clear(self): self.null_value_counts = None self.lower_bounds = None self.upper_bounds = None + return self def copy(self, to_copy): if self.is_partitioned: @@ -148,7 +149,7 @@ def with_status(self, stat): return self def with_input_file(self, input_file): - if isinstance(input_file, HadoopInputFile): + if isinstance(input_file, FileSystemInputFile): self.with_status(input_file.get_stat()) self.file_path = self.location() @@ -158,6 +159,7 @@ def with_input_file(self, input_file): def with_path(self, path): self.file_path = path + return self def with_format(self, fmt): if isinstance(fmt, FileFormat): diff --git a/iceberg/core/hadoop/__init__.py b/iceberg/core/filesystem/__init__.py similarity index 63% rename from iceberg/core/hadoop/__init__.py rename to iceberg/core/filesystem/__init__.py index 2ac455f573..b836e2fc3b 100644 --- a/iceberg/core/hadoop/__init__.py +++ b/iceberg/core/filesystem/__init__.py @@ -15,8 +15,12 @@ # specific language governing permissions and limitations # under the License. -__all__ = ["get_fs", "HadoopInputFile", "HadoopOutputFile"] +__all__ = ["get_fs", "FileStatus", "FileSystem", "FileSystemInputFile", "FileSystemOutputFile", + "FilesystemTableOperations", "FilesystemTables", "S3File", "S3FileSystem"] -from .hadoop_input_file import HadoopInputFile -from .hadoop_output_file import HadoopOutputFile +from .file_status import FileStatus +from .file_system import FileSystem, FileSystemInputFile, FileSystemOutputFile +from .filesystem_table_operations import FilesystemTableOperations +from .filesystem_tables import FilesystemTables +from .s3_filesystem import S3File, S3FileSystem from .util import get_fs diff --git a/iceberg/core/hadoop/file_status.py b/iceberg/core/filesystem/file_status.py similarity index 100% rename from iceberg/core/hadoop/file_status.py rename to iceberg/core/filesystem/file_status.py diff --git a/iceberg/core/hadoop/hadoop_input_file.py b/iceberg/core/filesystem/file_system.py similarity index 50% rename from iceberg/core/hadoop/hadoop_input_file.py rename to iceberg/core/filesystem/file_system.py index 66049dd11b..1ed2029537 100644 --- a/iceberg/core/hadoop/hadoop_input_file.py +++ b/iceberg/core/filesystem/file_system.py @@ -17,12 +17,33 @@ import gzip -from iceberg.api.io import InputFile +from iceberg.api.io import InputFile, OutputFile from .util import get_fs -class HadoopInputFile(InputFile): +class FileSystem(object): + + def open(self, path, mode='rb'): + raise NotImplementedError() + + def create(self, path, overwrite=False): + raise NotImplementedError() + + def exists(self, path): + raise NotImplementedError() + + def delete(self, path): + raise NotImplementedError() + + def stat(self, path): + raise NotImplementedError() + + def rename(self, src, dest): + raise NotImplementedError() + + +class FileSystemInputFile(InputFile): def __init__(self, fs, path, conf, length=None, stat=None): self.fs = fs @@ -34,7 +55,7 @@ def __init__(self, fs, path, conf, length=None, stat=None): @staticmethod def from_location(location, conf): fs = get_fs(location, conf) - return HadoopInputFile(fs, location, conf) + return FileSystemInputFile(fs, location, conf) def location(self): return self.path @@ -57,5 +78,43 @@ def new_stream(self, gzipped=False): for line in fo: yield line - def new_fo(self): - return self.fs.open(self.location()) + def new_fo(self, mode="rb"): + return self.fs.open(self.location(), mode=mode) + + def __repr__(self): + return "FileSystemInputFile({})".format(self.path) + + def __str__(self): + return self.__repr__() + + +class FileSystemOutputFile(OutputFile): + + @staticmethod + def from_path(path, conf): + return FileSystemOutputFile(path, conf) + + def __init__(self, path, conf): + self.path = path + self.conf = conf + + def create(self, mode="w"): + fs = get_fs(self.path, self.conf) + if fs.exists(self.path): + raise RuntimeError("File %s already exists" % self.path) + + return fs.open(self.path, mode=mode) + + def create_or_overwrite(self): + fs = get_fs(self.path, self.conf) + + return fs.open(self.path, "wb") + + def location(self): + return str(self.path) + + def __repr__(self): + return "FileSystemOutputFile({})".format(self.path) + + def __str__(self): + return self.__repr__() diff --git a/iceberg/core/filesystem/filesystem_table_operations.py b/iceberg/core/filesystem/filesystem_table_operations.py new file mode 100644 index 0000000000..121565433e --- /dev/null +++ b/iceberg/core/filesystem/filesystem_table_operations.py @@ -0,0 +1,140 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +import logging +from pathlib import Path +import uuid + +from iceberg.exceptions import CommitFailedException, ValidationException + +from .file_system import FileSystemInputFile, FileSystemOutputFile +from .util import get_fs +from ..table_metadata_parser import TableMetadataParser +from ..table_operations import TableOperations +from ..table_properties import TableProperties + +_logger = logging.getLogger(__name__) + + +class FilesystemTableOperations(TableOperations): + + def __init__(self, location, conf): + self.conf = conf + self.location = Path(location) + self.should_refresh = True + self.version = None + self.current_metadata = None + + def current(self): + if self.should_refresh: + return self.refresh() + + return self.current_metadata + + def refresh(self): + ver = self.version if self.version is not None else self.read_version_hint() + metadata_file = self.metadata_file(ver) + fs = get_fs(str(metadata_file), self.conf) + + if ver is not None and not fs.exists(metadata_file): + if ver == 0: + return None + raise ValidationException("Metadata file is missing: %s" % metadata_file) + + while fs.exists(str(self.metadata_file(ver + 1))): + ver += 1 + metadata_file = self.metadata_file(ver) + + self.version = ver + self.current_metadata = TableMetadataParser.read(self, FileSystemInputFile.from_location(str(metadata_file), + self.conf)) + self.should_refresh = False + return self.current_metadata + + def commit(self, base, metadata): + if base != self.current(): + raise CommitFailedException("Cannot commit changes based on stale table metadata") + + if not (base is None or base.location() == metadata.location()): + raise RuntimeError("Hadoop path-based tables cannot be relocated") + if TableProperties.WRITE_METADATA_LOCATION in metadata.properties: + raise RuntimeError("Hadoop path-based tables cannot be relocated") + + temp_metadata_file = self.metadata_path("{}{}".format(uuid.uuid4(), + TableMetadataParser.get_file_extension(self.conf))) + TableMetadataParser.write(metadata, FileSystemOutputFile.from_path(str(temp_metadata_file), self.conf)) + + next_version = (self.version if self.version is not None else 0) + 1 + final_metadata_file = self.metadata_file(next_version) + fs = get_fs(str(final_metadata_file), self.conf) + + if fs.exists(final_metadata_file): + raise CommitFailedException("Version %s already exists: %s" % (next_version, final_metadata_file)) + + if not fs.rename(temp_metadata_file, final_metadata_file): + raise CommitFailedException("Failed to commit changes using rename: %s" % final_metadata_file) + + self.write_version_hint(next_version) + self.should_refresh = True + + def new_input_file(self, path): + return FileSystemInputFile.from_location(path, self.conf) + + def new_output_file(self, path): + return FileSystemOutputFile.from_path(path, self.conf) + + def new_metadata_file(self, filename): + raise RuntimeError("Not yet implemented") + + def delete_file(self, path): + get_fs(path, self.conf).delete(path) + + def new_snapshot_id(self): + raise RuntimeError("Not yet implemented") + + def metadata_file_location(self, file): + return str(self.metadata_path(file)) + + def metadata_file(self, version): + return self.metadata_path("v{}{}".format(version, TableMetadataParser.get_file_extension(self.conf))) + + def metadata_path(self, filename): + return self.location / Path("metadata") / Path(filename) + + def version_hint_file(self): + return self.metadata_path("version-hint.text") + + def read_version_hint(self): + version_hint_file = str(self.version_hint_file()) + fs = get_fs(version_hint_file, self.conf) + + if not fs.exists(version_hint_file): + return 0 + else: + with fs.open(version_hint_file, "r") as fo: + return int(fo.readline().replace("\n", "")) + + def write_version_hint(self, version): + version_hint_file = str(self.version_hint_file()) + fs = get_fs(version_hint_file, self.conf) + try: + + with fs.create(version_hint_file, True) as fo: + fo.write("{}".format(version)) + + except RuntimeError as e: + _logger.warning("Unable to update version hint: %s" % e) diff --git a/iceberg/core/filesystem/filesystem_tables.py b/iceberg/core/filesystem/filesystem_tables.py new file mode 100644 index 0000000000..77bd75140d --- /dev/null +++ b/iceberg/core/filesystem/filesystem_tables.py @@ -0,0 +1,53 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + + +from iceberg.api import Tables +from iceberg.exceptions import NoSuchTableException + +from .filesystem_table_operations import FilesystemTableOperations +from ..table_metadata import TableMetadata + + +class FilesystemTables(Tables): + + def __init__(self, conf=None): + self.conf = conf if conf is not None else dict() + + def load(self, location): + from ..base_table import BaseTable + ops = self.new_table_ops(location) + if ops.current() is None: + raise NoSuchTableException("Table does not exist at location: %s" % location) + + return BaseTable(ops, location) + + def create(self, schema, table_identifier=None, spec=None, properties=None, location=None): + from ..base_table import BaseTable + spec, properties = super(FilesystemTables, self).default_args(spec, properties) + ops = self.new_table_ops(location) + + metadata = TableMetadata.new_table_metadata(ops, schema, spec, location, properties) + ops.commit(None, metadata) + + return BaseTable(ops, location) + + def new_table_ops(self, location): + if location is None: + raise RuntimeError("location cannot be None") + + return FilesystemTableOperations(location, self.conf) diff --git a/iceberg/core/hadoop/local_filesystem.py b/iceberg/core/filesystem/local_filesystem.py similarity index 63% rename from iceberg/core/hadoop/local_filesystem.py rename to iceberg/core/filesystem/local_filesystem.py index aa55de9cec..826700a774 100644 --- a/iceberg/core/hadoop/local_filesystem.py +++ b/iceberg/core/filesystem/local_filesystem.py @@ -15,8 +15,9 @@ # specific language governing permissions and limitations # under the License. +import errno import os -import stat +from pathlib import Path from .file_status import FileStatus from .file_system import FileSystem @@ -36,12 +37,23 @@ def __init__(self): LocalFileSystem.fs_inst = self def open(self, path, mode='rb'): + open_path = Path(LocalFileSystem.fix_path(path)) - return open(LocalFileSystem.fix_path(path), mode=mode) + if "w" in mode and not open_path.parents[0].exists(): + try: + open_path.parents[0].mkdir(parents=True) + except OSError as exc: + if exc.errno != errno.EEXIST: + raise + + return open(open_path, mode=mode) + + def delete(self, path): + raise NotImplementedError() def stat(self, path): st = os.stat(LocalFileSystem.fix_path(path)) - return FileStatus(path=path, length=st.st_size, is_dir=stat.S_ISDIR(st.st_mode), + return FileStatus(path=path, length=st.st_size, is_dir=os.stat.S_ISDIR(st.st_mode), blocksize=st.st_blksize, modification_time=st.st_mtime, access_time=st.st_atime, permission=st.st_mode, owner=st.st_uid, group=st.st_gid) @@ -51,34 +63,19 @@ def fix_path(path): path = str(path[7:]) return path - # def ls(self, path): - # return os.listdir(path) - # - # def delete(self, path, recursive=False): - # return os.remove(path) - # - - # - # def rename(self, path, new_path): - # return os.rename(path, new_path) - # - # def mkdir(self, path, create_parents=True): - # if create_parents: - # return os.makedirs(path) - # - # return os.mkdir(path) - # - # def exists(self, path): - # return os.path.isfile(path) - # - # def isdir(self, path): - # return os.path.isdir(path) - # - # def isfile(self, path): - # return os.path.isfile(path) - # - # def _isfilestore(self): - # return True - # - # def open(self, path, mode='rb'): - # return open(path, mode=mode) + def create(self, path, overwrite=False): + if os.path.exists(path) and not overwrite: + raise RuntimeError("Path %s already exists" % path) + + return open(path, "w") + + def rename(self, src, dest): + try: + os.rename(src, dest) + except OSError: + return False + + return True + + def exists(self, path): + return os.path.exists(path) diff --git a/iceberg/core/filesystem/s3_filesystem.py b/iceberg/core/filesystem/s3_filesystem.py new file mode 100644 index 0000000000..9423ee0d1c --- /dev/null +++ b/iceberg/core/filesystem/s3_filesystem.py @@ -0,0 +1,257 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +import io +import logging +import re +import time +from urllib.parse import urlparse + +import boto3 +from botocore.credentials import RefreshableCredentials +from botocore.exceptions import ClientError +from botocore.session import get_session +from retrying import retry + +from .file_status import FileStatus +from .file_system import FileSystem + +_logger = logging.getLogger(__name__) + + +S3_CLIENT = dict() +BOTO_STS_CLIENT = boto3.client('sts') +CONF = None +ROLE_ARN = "default" +AUTOREFRESH_SESSION = None + + + +@retry(wait_incrementing_start=100, wait_exponential_multiplier=4, + wait_exponential_max=5000, stop_max_delay=600000, stop_max_attempt_number=7) +def get_s3(obj="resource"): + global AUTOREFRESH_SESSION + global ROLE_ARN + if ROLE_ARN not in S3_CLIENT: + S3_CLIENT[ROLE_ARN] = dict() + + if ROLE_ARN == "default": + if AUTOREFRESH_SESSION is None: + AUTOREFRESH_SESSION = boto3.Session() + S3_CLIENT["default"]["resource"] = AUTOREFRESH_SESSION.resource('s3') + S3_CLIENT["default"]["client"] = AUTOREFRESH_SESSION.client('s3') + else: + if AUTOREFRESH_SESSION is None: + sess = get_session() + sess._credentials = RefreshableCredentials.create_from_metadata(metadata=refresh_sts_session_keys(), + refresh_using=refresh_sts_session_keys, + method="sts-assume-role") + AUTOREFRESH_SESSION = boto3.Session(botocore_session=sess) + + S3_CLIENT[ROLE_ARN]["resource"] = AUTOREFRESH_SESSION.resource("s3") + S3_CLIENT[ROLE_ARN]["client"] = AUTOREFRESH_SESSION.client("s3") + + return S3_CLIENT.get(ROLE_ARN, dict()).get(obj) + + +def refresh_sts_session_keys(): + params = {"RoleArn": ROLE_ARN, + "RoleSessionName": "iceberg_python_client_{}".format(int(time.time() * 1000.00))} + + sts_creds = BOTO_STS_CLIENT.assume_role(**params).get("Credentials") + credentials = {"access_key": sts_creds.get("AccessKeyId"), + "secret_key": sts_creds.get("SecretAccessKey"), + "token": sts_creds.get("SessionToken"), + "expiry_time": sts_creds.get("Expiration").isoformat()} + return credentials + + +def url_to_bucket_key_name_tuple(url): + parsed_url = urlparse(url) + return parsed_url.netloc, parsed_url.path[1:], parsed_url.path.split("/")[-1] + + +class S3FileSystem(FileSystem): + fs_inst = None + + @staticmethod + def get_instance(): + if S3FileSystem.fs_inst is None: + S3FileSystem() + return S3FileSystem.fs_inst + + def __init__(self): + if S3FileSystem.fs_inst is None: + S3FileSystem.fs_inst = self + + def set_conf(self, conf): + global CONF + + CONF = conf + self.set_role(CONF.get("hive.aws_iam_role", "default")) + + def set_role(self, role): + global ROLE_ARN + + if role is not None: + ROLE_ARN = role + + def exists(self, path): + try: + self.info(path) + except ClientError as ce: + if ce.response['Error']['Code'] == "404": + return False + else: + raise + + return True + + def open(self, path, mode='rb'): + return S3File(path, mode=mode) + + def delete(self, path): + bucket, key, _ = url_to_bucket_key_name_tuple(S3FileSystem.normalize_s3_path(path)) + get_s3().Object(bucket_name=bucket, + key=key).delete() + + def stat(self, path): + st = self.info(S3FileSystem.normalize_s3_path(path)) + + return FileStatus(path=path, length=st.get("ContentLength"), is_dir=False, + blocksize=None, modification_time=st.get("LastModified"), access_time=None, + permission=None, owner=None, group=None) + + @staticmethod + def info(url): + bucket, key, _ = url_to_bucket_key_name_tuple(url) + return get_s3("client").head_object(Bucket=bucket, + Key=key) + + @staticmethod + def normalize_s3_path(path): + return re.sub(r'^s3n://|s3a://', 's3://', path) + + +class S3File(object): + MAX_CHUNK_SIZE = 4 * 1048576 + MIN_CHUNK_SIZE = 2 * 65536 + + def __init__(self, path, mode="rb"): + self.path = path + bucket, key, name = url_to_bucket_key_name_tuple(S3FileSystem.normalize_s3_path(path)) + self.curr_pos = 0 + self.obj = (get_s3() + .Object(bucket_name=bucket, + key=key)) + self.name = name + if mode.startswith("r"): + self.size = self.obj.content_length + + self.isatty = False + self.closed = False + + self.mode = mode + + self.buffer_remote_reads = True + + self.curr_buffer = None + self.curr_buffer_start = -1 + self.curr_buffer_end = -1 + self.buffer_reads = 0 + self.buffer_hits = 0 + + self.chunk_size = self.MAX_CHUNK_SIZE + + def close(self): + self.closed = True + + def flush(self): + pass + + def next(self): + return next(self.readline()) + + def read(self, n=0): + if self.curr_pos >= self.size: + return None + if self.buffer_remote_reads: + stream = self._read_from_buffer(n) + else: + if n <= 0: + n = self.size + stream = self.obj.get(Range='bytes={}-{}'.format(self.curr_pos, + self.size - self.curr_pos))['Body'].read() + else: + stream = self.obj.get(Range='bytes={}-{}'.format(self.curr_pos, self.curr_pos + n - 1))['Body'].read() + + self.curr_pos = min(self.curr_pos + n, self.size) + return stream + + def _read_from_buffer(self, n): + self.buffer_reads += 1 + # if the buffer is none or if the entire read is not contained + # in our current buffer fill the buffer + if self.curr_buffer is None or not (self.curr_buffer_start <= self.curr_pos + and self.curr_pos + n < self.curr_buffer_end): + self.curr_buffer_start = self.curr_pos + self.curr_buffer_end = self.curr_pos + max(self.chunk_size, n) + byte_range = 'bytes={}-{}'.format(self.curr_buffer_start, + self.curr_buffer_end) + self.curr_buffer = io.BytesIO(self.obj.get(Range=byte_range)['Body'].read()) + else: + self.buffer_hits += 1 + + # seek to the right position if we aren't at the start of the buffer + if self.curr_buffer_start != self.curr_pos: + self.curr_buffer.seek(self.curr_pos - self.curr_buffer_start) + + return self.curr_buffer.read(n) + + def readline(self, n=0): + if self.curr_buffer is None: + self.curr_buffer = io.BytesIO(self.obj.get()['Body'].read()) + for line in self.curr_buffer: + yield line + + def seek(self, offset, whence=0): + if whence == 0: + self.curr_pos = offset + elif whence == 1: + self.curr_pos += offset + elif whence == 2: + self.curr_pos = self.size + offset + + def tell(self): + return self.curr_pos + + def write(self, string): + resp = self.obj.put(Body=string) + if not resp.get("ResponseMetadata", dict()).get("HTTPStatusCode") == 200: + raise RuntimeError("Unable to write to {}".format(self.path)) + + def __enter__(self): + return self + + def __exit__(self, exc_type, exc_val, exc_tb): + self.close() + + def __iter__(self): + return self + + def __next__(self): + return self.next() diff --git a/iceberg/core/hadoop/util.py b/iceberg/core/filesystem/util.py similarity index 60% rename from iceberg/core/hadoop/util.py rename to iceberg/core/filesystem/util.py index 8516e1eb37..d73dc11425 100644 --- a/iceberg/core/hadoop/util.py +++ b/iceberg/core/filesystem/util.py @@ -15,16 +15,13 @@ # specific language governing permissions and limitations # under the License. -import time from urllib.parse import urlparse -import boto3 - -from .local_filesystem import LocalFileSystem -from .s3_filesystem_wrapper import S3FileSystemWrapper - def get_fs(path, conf, local_only=False): + from .local_filesystem import LocalFileSystem + from .s3_filesystem import S3FileSystem + if local_only: return LocalFileSystem.get_instance() else: @@ -32,22 +29,11 @@ def get_fs(path, conf, local_only=False): if parsed_path.scheme in ["", "file"]: return LocalFileSystem.get_instance() - elif parsed_path.scheme in ["s3", "s3n"]: - if conf.get("hive.aws_iam_role") is not None: - key, secret, token = get_sts_session_keys(conf.get("hive.aws_iam_role")) - return S3FileSystemWrapper(key=key, secret=secret, token=token) - return S3FileSystemWrapper() + elif parsed_path.scheme in ["s3", "s3n", "s3a"]: + fs = S3FileSystem.get_instance() + fs.set_conf(conf) + return fs elif parsed_path.scheme in ["hdfs"]: raise RuntimeError("Hadoop FS not implemented") raise RuntimeError("No filesystem found for this location: %s" % path) - - -def get_sts_session_keys(role_arn): - client = boto3.client('sts') - response = client.assume_role( - RoleArn=role_arn, - RoleSessionName='iceberg_python_client_{}'.format(int(time.time()))) - return (response["Credentials"]["AccessKeyId"], - response["Credentials"]["SecretAccessKey"], - response["Credentials"]["SessionToken"]) diff --git a/iceberg/core/hadoop/file_system.py b/iceberg/core/hadoop/file_system.py deleted file mode 100644 index 2630227688..0000000000 --- a/iceberg/core/hadoop/file_system.py +++ /dev/null @@ -1,25 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - - -class FileSystem(object): - - def open(self, path, mode='rb'): - raise NotImplementedError() - - def stat(self, path): - raise NotImplementedError() diff --git a/iceberg/core/hadoop/hadoop_output_file.py b/iceberg/core/hadoop/hadoop_output_file.py deleted file mode 100644 index 4c0e2436d5..0000000000 --- a/iceberg/core/hadoop/hadoop_output_file.py +++ /dev/null @@ -1,49 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -from iceberg.api.io import OutputFile - -from .util import get_fs - - -class HadoopOutputFile(OutputFile): - - @staticmethod - def from_path(path, conf): - return HadoopOutputFile(path, conf) - - def __init__(self, path, conf): - self.path = path - self.conf = conf - - def create(self): - fs = get_fs(self.path, self.conf) - if fs.exists(): - raise RuntimeError("File %s already exists" % self.path) - - return fs.open(self.path) - - def create_or_overwrite(self): - fs = get_fs(self.path, self.conf) - - return fs.open(self.path, "wb") - - def location(self): - return str(self.path) - - def __str__(self): - return self.location() diff --git a/iceberg/core/hadoop/hadoop_table_operations.py b/iceberg/core/hadoop/hadoop_table_operations.py deleted file mode 100644 index e771084c4b..0000000000 --- a/iceberg/core/hadoop/hadoop_table_operations.py +++ /dev/null @@ -1,49 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -import iceberg.core as core - - -class HadoopTableOperations(core.TableOperations): - - def __init__(self, location, conf): - self.conf = conf - self.location = location - self.should_refresh = True - self.verison = None - self.current_metadata = None - - def current(self): - pass - - def refresh(self): - raise RuntimeError("Not yet implemented") - - def commit(self, base, metadata): - raise RuntimeError("Not yet implemented") - - def new_input_file(self, path): - raise RuntimeError("Not yet implemented") - - def new_metadata_file(self, filename): - raise RuntimeError("Not yet implemented") - - def delete_file(self, path): - raise RuntimeError("Not yet implemented") - - def new_snapshot_id(self): - raise RuntimeError("Not yet implemented") diff --git a/iceberg/core/hadoop/s3_filesystem_wrapper.py b/iceberg/core/hadoop/s3_filesystem_wrapper.py deleted file mode 100644 index f1fd713b95..0000000000 --- a/iceberg/core/hadoop/s3_filesystem_wrapper.py +++ /dev/null @@ -1,54 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -import s3fs - -from .file_status import FileStatus -from .file_system import FileSystem - - -class S3FileSystemWrapper(FileSystem): - - def __init__(self, anon=False, key=None, secret=None, token=None): - self._fs = s3fs.S3FileSystem(anon=anon, key=key, secret=secret, token=token) - - def open(self, path, mode='rb'): - return self._fs.open(S3FileSystemWrapper.fix_s3_path(path), mode=mode) - - def stat(self, path): - is_dir = False - st = dict() - try: - st = self._fs.info(S3FileSystemWrapper.fix_s3_path(path), refresh=True) - except RuntimeError: - # must be a directory or subsequent du will fail - du = self._fs.du(S3FileSystemWrapper.fix_s3_path(path), refresh=True) - if len(du) > 0: - is_dir = True - length = 0 - for key, val in du.items(): - length += val - - return FileStatus(path=path, length=st.get("Size", length), is_dir=is_dir, - blocksize=None, modification_time=st.get("LastModified"), access_time=None, - permission=None, owner=None, group=None) - - @staticmethod - def fix_s3_path(path): - if path.startswith("s3n"): - path = "s3" + str(path[3:]) - return path diff --git a/iceberg/core/table_properties.py b/iceberg/core/table_properties.py new file mode 100644 index 0000000000..0511142529 --- /dev/null +++ b/iceberg/core/table_properties.py @@ -0,0 +1,77 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + + + +class TableProperties(object): + COMMIT_NUM_RETRIES = "commit.retry.num-retries" + COMMIT_NUM_RETRIES_DEFAULT = 4 + + COMMIT_MIN_RETRY_WAIT_MS = "commit.retry.min-wait-ms" + COMMIT_MIN_RETRY_WAIT_MS_DEFAULT = 100 + + COMMIT_MAX_RETRY_WAIT_MS = "commit.retry.max-wait-ms" + COMMIT_MAX_RETRY_WAIT_MS_DEFAULT = 60000 + + COMMIT_TOTAL_RETRY_TIME_MS = "commit.retry.total-timeout-ms" + COMMIT_TOTAL_RETRY_TIME_MS_DEFAULT = 1800000 + + MANIFEST_TARGET_SIZE_BYTES = "commit.manifest.target-size-bytes" + MANIFEST_TARGET_SIZE_BYTES_DEFAULT = 8388608 + + MANIFEST_MIN_MERGE_COUNT = "commit.manifest.min-count-to-merge" + MANIFEST_MIN_MERGE_COUNT_DEFAULT = 100 + + DEFAULT_FILE_FORMAT = "write.format.default" + DEFAULT_FILE_FORMAT_DEFAULT = "parquet" + + PARQUET_ROW_GROUP_SIZE_BYTES = "write.parquet.row-group-size-bytes" + PARQUET_ROW_GROUP_SIZE_BYTES_DEFAULT = "134217728" + + PARQUET_PAGE_SIZE_BYTES = "write.parquet.page-size-bytes" + PARQUET_PAGE_SIZE_BYTES_DEFAULT = "1048576" + + PARQUET_DICT_SIZE_BYTES = "write.parquet.dict-size-bytes" + PARQUET_DICT_SIZE_BYTES_DEFAULT = "2097152" + + PARQUET_COMPRESSION = "write.parquet.compression-codec" + PARQUET_COMPRESSION_DEFAULT = "gzip" + + AVRO_COMPRESSION = "write.avro.compression-codec" + AVRO_COMPRESSION_DEFAULT = "gzip" + + SPLIT_SIZE = "read.split.target-size" + SPLIT_SIZE_DEFAULT = 134217728 + + SPLIT_LOOKBACK = "read.split.planning-lookback" + SPLIT_LOOKBACK_DEFAULT = 10 + + SPLIT_OPEN_FILE_COST = "read.split.open-file-cost" + SPLIT_OPEN_FILE_COST_DEFAULT = 4 * 1024 * 1024 + + OBJECT_STORE_ENABLED = "write.object-storage.enabled" + OBJECT_STORE_ENABLED_DEFAULT = False + + OBJECT_STORE_PATH = "write.object-storage.path" + + WRITE_NEW_DATA_LOCATION = "write.folder-storage.path" + + WRITE_METADATA_LOCATION = "write.metadata.path" + + MANIFEST_LISTS_ENABLED = "write.manifest-lists.enabled" + + MANIFEST_LISTS_ENABLED_DEFAULT = True diff --git a/iceberg/core/util/__init__.py b/iceberg/core/util/__init__.py index 5df92bc141..ba17648735 100644 --- a/iceberg/core/util/__init__.py +++ b/iceberg/core/util/__init__.py @@ -15,6 +15,11 @@ # specific language governing permissions and limitations # under the License. -__all__ = ["AtomicInteger"] +__all__ = ["AtomicInteger", "PackingIterator", "str_as_bool"] from .atomic_integer import AtomicInteger +from .bin_packing import PackingIterator + + +def str_as_bool(str_var): + return str_var is not None and str_var.lower() == "true" diff --git a/iceberg/core/util/bin_packing.py b/iceberg/core/util/bin_packing.py index 1ccca2ec74..2b7f0e19d7 100644 --- a/iceberg/core/util/bin_packing.py +++ b/iceberg/core/util/bin_packing.py @@ -15,6 +15,7 @@ # specific language governing permissions and limitations # under the License. + from copy import deepcopy diff --git a/iceberg/spark/__init__.py b/iceberg/spark/__init__.py deleted file mode 100644 index ede72c06ae..0000000000 --- a/iceberg/spark/__init__.py +++ /dev/null @@ -1,19 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -# flake8: noqa - -from .table_identifier import TableIdentifier diff --git a/iceberg/spark/source/__init__.py b/iceberg/spark/source/__init__.py deleted file mode 100644 index cc91ef472b..0000000000 --- a/iceberg/spark/source/__init__.py +++ /dev/null @@ -1,19 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -# flake8: noqa - -from .spark_catalog import SparkCatalog diff --git a/iceberg/spark/source/spark_catalog.py b/iceberg/spark/source/spark_catalog.py deleted file mode 100644 index 858be62ce5..0000000000 --- a/iceberg/spark/source/spark_catalog.py +++ /dev/null @@ -1,53 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - - -class SparkCatalog(object): - - def create(self, ident, schema, spec, properties): - pass - - def load(self, ident): - pass - - def drop(self, ident): - pass - - def load_table(self, ident): - pass - - def create_table(self, ident, table_type, partitions, properties): - pass - - def apply_property_changes(self, table, changes): - pass - - def apply_schema_changes(self, table, changes): - pass - - def drop_table(self, ident): - pass - - def initialize(self, name, options): - self.name = name - self.options = options - - def convert(self, schema, partitioning): - pass - - def __init__(self): - self.name = "" - self.options = None diff --git a/iceberg/spark/table_identifier.py b/iceberg/spark/table_identifier.py deleted file mode 100644 index 84d04f1309..0000000000 --- a/iceberg/spark/table_identifier.py +++ /dev/null @@ -1,21 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - - -class TableIdentifier(object): - - def __init__(self): - pass diff --git a/setup.py b/setup.py index 5a5710167e..20f01f28bb 100755 --- a/setup.py +++ b/setup.py @@ -26,12 +26,13 @@ keywords='iceberg', url='https://github.com/apache/incubator-iceberg/blob/master/README.md', python_requires='>=3.4', - install_requires=['boto3', + install_requires=['botocore', + 'boto3', 'fastavro', 'mmh3', 'python-dateutil', 'pytz', - 'retrying', - 's3fs'], + 'requests', + 'retrying'], setup_requires=['setupmeta'] ) diff --git a/tests/core/test_table_metadata_parser.py b/tests/core/test_table_metadata_parser.py index d9d3b28d10..9c8ad2c70a 100644 --- a/tests/core/test_table_metadata_parser.py +++ b/tests/core/test_table_metadata_parser.py @@ -17,18 +17,17 @@ import binascii -from iceberg.api import Files from iceberg.core import ConfigProperties, TableMetadataParser -from iceberg.core.hadoop import HadoopInputFile +from iceberg.core.filesystem import FileSystemInputFile, FileSystemOutputFile def test_compression_property(expected, prop): config = {ConfigProperties.COMPRESS_METADATA: prop} - output_file = Files.local_output(TableMetadataParser.get_file_extension(config)) + output_file = FileSystemOutputFile.from_path(TableMetadataParser.get_file_extension(config), dict) TableMetadataParser.write(expected, output_file) assert prop == is_compressed(TableMetadataParser.get_file_extension(config)) read = TableMetadataParser.read(None, - HadoopInputFile.from_location(TableMetadataParser.get_file_extension(config), None)) + FileSystemInputFile.from_location(TableMetadataParser.get_file_extension(config), None)) verify_metadata(read, expected) diff --git a/tox.ini b/tox.ini index 7a600ab8c6..78ed9499e1 100644 --- a/tox.ini +++ b/tox.ini @@ -64,7 +64,7 @@ commands = skips = B104 [flake8] -ignore = E501 +ignore = E501,W503 exclude = *.egg-info, *.pyc, From a962a46521cf09aeaaef76298cd32e5bc81f9ad9 Mon Sep 17 00:00:00 2001 From: Frank Cash Date: Wed, 14 Aug 2019 16:40:37 -0700 Subject: [PATCH 009/642] Update formatting in python/README.md (#392) --- README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 2b30b5c00f..c4d5cca2d5 100644 --- a/README.md +++ b/README.md @@ -1,3 +1,3 @@ -#Iceberg +# Iceberg A python implementation of the Iceberg table format. -See the project level README for more details: https://github.com/apache/incubator-iceberg \ No newline at end of file +See the project level README for more details: https://github.com/apache/incubator-iceberg From c8ae3919e589fb6ba5f1b187a1e17be9dd828130 Mon Sep 17 00:00:00 2001 From: TGooch44 Date: Mon, 19 Aug 2019 16:43:51 -0700 Subject: [PATCH 010/642] Bringing type module into sync (#382) --- iceberg/api/types/conversions.py | 14 ++++---- iceberg/api/types/type.py | 7 +--- iceberg/api/types/type_util.py | 62 ++++++++++++++++++-------------- iceberg/api/types/types.py | 28 +++++++++------ 4 files changed, 59 insertions(+), 52 deletions(-) diff --git a/iceberg/api/types/conversions.py b/iceberg/api/types/conversions.py index a8bc3f5f25..6768b6170a 100644 --- a/iceberg/api/types/conversions.py +++ b/iceberg/api/types/conversions.py @@ -82,20 +82,18 @@ def from_partition_string(type_var, as_string): @staticmethod def to_byte_buffer(type_var, value): - byte_buf_func = Conversions.to_byte_buff_mapping.get(type_var.type_id) - if byte_buf_func is None: + try: + return Conversions.to_byte_buff_mapping.get(type_var.type_id)(type_var, value) + except KeyError: raise RuntimeError("Cannot Serialize Type: %s" % type_var) - return byte_buf_func(type_var, value) - @staticmethod def from_byte_buffer(type_var, buffer_var): return Conversions.internal_from_byte_buffer(type_var, buffer_var) @staticmethod def internal_from_byte_buffer(type_var, buffer_var): - byte_buf_func = Conversions.from_byte_buff_mapping.get(type_var.type_id) - if byte_buf_func is None: + try: + return Conversions.from_byte_buff_mapping[type_var.type_id](type_var.type_id, buffer_var) + except KeyError: raise RuntimeError("Cannot Serialize Type: %s" % type_var) - - return byte_buf_func(type_var.type_id, buffer_var) diff --git a/iceberg/api/types/type.py b/iceberg/api/types/type.py index c855637820..367e366b99 100644 --- a/iceberg/api/types/type.py +++ b/iceberg/api/types/type.py @@ -81,12 +81,7 @@ def as_nested_type(self): class PrimitiveType(Type): def __eq__(self, other): - if id(self) == id(other): - return True - elif other is None or not isinstance(other, PrimitiveType): - return False - - return True + return type(self) == type(other) def __ne__(self, other): return not self.__eq__(other) diff --git a/iceberg/api/types/type_util.py b/iceberg/api/types/type_util.py index 7898262a06..7bc8dd74cb 100644 --- a/iceberg/api/types/type_util.py +++ b/iceberg/api/types/type_util.py @@ -52,9 +52,9 @@ def select(schema, field_ids): return schema elif result is not None: if schema.get_aliases() is not None: - return iceberg.api.schema.Schema(result.as_nested_type(), schema.get_aliases()) + return iceberg.api.schema.Schema(result.as_nested_type().fields, schema.get_aliases()) else: - return iceberg.api.schema.Schema(result.as_nested_type()) + return iceberg.api.schema.Schema(result.as_nested_type().fields) return iceberg.api.schema.Schema(list(), schema.get_aliases()) @@ -161,6 +161,11 @@ def visit_custom_order(arg, visitor): results.append(VisitFieldFuture(field, visitor)) struct = visitor.struct(struct, [x.get() for x in results]) return struct + elif type_var.type_id == TypeID.LIST: + list_var = type_var.as_nested_type().as_list_type() + return visitor.list(list_var, VisitFuture(list_var.element_type, visitor)) + elif type_var.type_id == TypeID.MAP: + raise NotImplementedError() return visitor.primitive(type_var.as_primitive_type()) @@ -172,22 +177,22 @@ def __init__(self): self.field_ids = list() def schema(self, schema, struct_result): - return NotImplementedError() + return None def struct(self, struct, field_results): - return NotImplementedError() + return None def field(self, field, field_result): - return NotImplementedError() + return None def list(self, list_var, element_result): - return NotImplementedError() + return None def map(self, map_var, key_result, value_result): - return NotImplementedError() + return None def primitive(self, primitive_var): - return NotImplementedError() + return None class CustomOrderSchemaVisitor(object): @@ -195,22 +200,22 @@ def __init__(self): super(CustomOrderSchemaVisitor, self).__init__() def schema(self, schema, struct_result): - return NotImplementedError() + return None def struct(self, struct, field_results): - return NotImplementedError() + return None def field(self, field, field_result): - return NotImplementedError() + return None def list(self, list_var, element_result): - return NotImplementedError() + return None def map(self, map_var, key_result, value_result): - return NotImplementedError() + return None def primitive(self, primitive_var): - return NotImplementedError() + return None class VisitFuture(object): @@ -245,7 +250,7 @@ class GetProjectedIds(SchemaVisitor): def __init__(self): super(GetProjectedIds, self).__init__() - self.field_ids = set() + self.field_ids = list() def schema(self, schema, struct_result): return self.field_ids @@ -254,22 +259,22 @@ def struct(self, struct, field_results): return self.field_ids def field(self, field, field_result): - if field_result is not None: - self.field_ids.add(field.field_id) + if field_result is None: + self.field_ids.append(field.field_id) return self.field_ids def list(self, list_var, element_result): if element_result is None: - for field in list_var.fields: - self.field_ids.add(field.field_id) + for field in list_var.fields(): + self.field_ids.append(field.field_id) return self.field_ids def map(self, map_var, key_result, value_result): if value_result is None: - for field in map_var.fields: - self.field_ids.add(field.field_id) + for field in map_var.fields(): + self.field_ids.append(field.field_id) return self.field_ids @@ -278,7 +283,7 @@ class PruneColumns(SchemaVisitor): def __init__(self, selected): super(PruneColumns, self).__init__() - self.selected = selected + self.selected = list(selected) def schema(self, schema, struct_result): return struct_result @@ -288,10 +293,10 @@ def struct(self, struct, field_results): selected_fields = list() same_types = True - for i, field in enumerate(field_results): - projected_type = field_results[i] + for i, projected_type in enumerate(field_results): + field = fields[i] if projected_type is not None: - if field.type_id == projected_type.type_id: + if field.type == projected_type: selected_fields.append(field) elif projected_type is not None: same_types = False @@ -316,6 +321,9 @@ def field(self, field, field_result): elif field_result is not None: return field_result + def primitive(self, primitive_var): + return None + class IndexByName(SchemaVisitor): @@ -416,9 +424,9 @@ def field(self, field, field_result): def list(self, list_var, element_result): new_id = self.next_id() if list_var.is_element_optional(): - return ListType.of_optional(new_id, element_result()) + return ListType.of_optional(new_id, element_result.get()) else: - return ListType.of_required(new_id, element_result()) + return ListType.of_required(new_id, element_result.get()) def map(self, map_var, key_result, value_result): new_key_id = self.next_id() diff --git a/iceberg/api/types/types.py b/iceberg/api/types/types.py index 225bdeee44..941c9bc1b3 100644 --- a/iceberg/api/types/types.py +++ b/iceberg/api/types/types.py @@ -375,10 +375,10 @@ def of(precison, scale): return DecimalType(precison, scale) def __init__(self, precision, scale): - if precision > 38: + if int(precision) > 38: raise RuntimeError("Decimals with precision larger than 38 are not supported: %s", precision) - self.precision = precision - self.scale = scale + self.precision = int(precision) + self.scale = int(scale) @property def type_id(self): @@ -410,18 +410,19 @@ def __key(self): class NestedField(): @staticmethod - def optional(id, name, type_var): - return NestedField(True, id, name, type_var) + def optional(id, name, type_var, doc=None): + return NestedField(True, id, name, type_var, doc=doc) @staticmethod - def required(id, name, type): - return NestedField(False, id, name, type) + def required(id, name, type, doc=None): + return NestedField(False, id, name, type, doc=doc) - def __init__(self, is_optional, id, name, type): + def __init__(self, is_optional, id, name, type, doc=None): self.is_optional = is_optional self.id = id self.name = name self.type = type + self.doc = doc @property def is_required(self): @@ -432,7 +433,11 @@ def field_id(self): return self.id def __repr__(self): - return "%s: %s: %s %s" % (self.id, self.name, "optional" if self.is_optional else "required", self.type) + return "%s: %s: %s %s(%s)" % (self.id, + self.name, + "optional" if self.is_optional else "required", + self.type, + self.doc) def __str__(self): return self.__repr__() @@ -445,7 +450,8 @@ def __eq__(self, other): return self.is_optional == other.is_optional \ and self.id == other.id \ - and self.name == other.name and self.type == other.type + and self.name == other.name and self.type == other.type \ + and self.doc == other.doc def __ne__(self, other): return not self.__eq__(other) @@ -455,7 +461,7 @@ def __hash__(self): def __key(self): type_name = self.type.type_id.name - return NestedField.__class__, self.is_optional, self.id, self.name, type_name + return NestedField.__class__, self.is_optional, self.id, self.name, self.doc, type_name class StructType(NestedType): From 4a5b080ad6e70d93543f4de2482ecf13b6faefa3 Mon Sep 17 00:00:00 2001 From: TGooch44 Date: Mon, 19 Aug 2019 16:52:36 -0700 Subject: [PATCH 011/642] Bringing expression implementations into a consistent state (#381) --- iceberg/api/expressions/evaluator.py | 17 ++++++++--------- .../expressions/inclusive_manifest_evaluator.py | 16 ++++++++-------- .../expressions/inclusive_metrics_evaluator.py | 14 +++++++------- iceberg/api/expressions/literals.py | 10 ++-------- iceberg/api/expressions/residual_evaluator.py | 16 ++++++++-------- .../api/expressions/strict_metrics_evaluator.py | 16 ++++++++-------- 6 files changed, 41 insertions(+), 48 deletions(-) diff --git a/iceberg/api/expressions/evaluator.py b/iceberg/api/expressions/evaluator.py index 3f2a29fd69..2371624e15 100644 --- a/iceberg/api/expressions/evaluator.py +++ b/iceberg/api/expressions/evaluator.py @@ -22,20 +22,19 @@ class Evaluator(object): - THREAD_LOCAL_DATA = threading.local() - - def visitor(self): - if not hasattr(Evaluator.THREAD_LOCAL_DATA, "visitors"): - Evaluator.THREAD_LOCAL_DATA.visitors = Evaluator.EvalVisitor() - - return Evaluator.THREAD_LOCAL_DATA.visitors def __init__(self, struct, unbound, case_sensitive=True): self.expr = Binder.bind(struct, unbound, case_sensitive) - self.visitors = None + self.thread_local_data = threading.local() + + def _visitor(self): + if not hasattr(self.thread_local_data, "visitors"): + self.thread_local_data.visitors = Evaluator.EvalVisitor() + + return self.thread_local_data.visitors def eval(self, data): - return self.visitor().eval(data, self.expr) + return self._visitor().eval(data, self.expr) class EvalVisitor(ExpressionVisitors.BoundExpressionVisitor): diff --git a/iceberg/api/expressions/inclusive_manifest_evaluator.py b/iceberg/api/expressions/inclusive_manifest_evaluator.py index e1c55fb6ad..953ee99d08 100644 --- a/iceberg/api/expressions/inclusive_manifest_evaluator.py +++ b/iceberg/api/expressions/inclusive_manifest_evaluator.py @@ -28,12 +28,6 @@ class InclusiveManifestEvaluator(object): - def visitor(self): - if not hasattr(self.thread_local_data, "visitors"): - self.thread_local_data.visitors = ManifestEvalVistor(self.expr) - - return self.thread_local_data.visitors - def __init__(self, spec, row_filter, case_sensitive=True): self.struct = spec.partition_type() self.expr = Binder.bind(self.struct, @@ -42,11 +36,17 @@ def __init__(self, spec, row_filter, case_sensitive=True): case_sensitive=case_sensitive) self.thread_local_data = threading.local() + def _visitor(self): + if not hasattr(self.thread_local_data, "visitors"): + self.thread_local_data.visitors = ManifestEvalVisitor(self.expr) + + return self.thread_local_data.visitors + def eval(self, manifest): - return self.visitor().eval(manifest) + return self._visitor().eval(manifest) -class ManifestEvalVistor(ExpressionVisitors.BoundExpressionVisitor): +class ManifestEvalVisitor(ExpressionVisitors.BoundExpressionVisitor): def __init__(self, expr): self.expr = expr diff --git a/iceberg/api/expressions/inclusive_metrics_evaluator.py b/iceberg/api/expressions/inclusive_metrics_evaluator.py index ac549355d0..e9ab1a0bd7 100644 --- a/iceberg/api/expressions/inclusive_metrics_evaluator.py +++ b/iceberg/api/expressions/inclusive_metrics_evaluator.py @@ -24,12 +24,6 @@ class InclusiveMetricsEvaluator(object): - def visitor(self): - if not hasattr(self.thread_local_data, "visitors"): - self.thread_local_data.visitors = MetricsEvalVisitor(self.expr, self.schema, self.struct) - - return self.thread_local_data.visitors - def __init__(self, schema, unbound, case_sensitive=True): self.schema = schema self.struct = schema.as_struct() @@ -37,8 +31,14 @@ def __init__(self, schema, unbound, case_sensitive=True): self.expr = Binder.bind(self.struct, Expressions.rewrite_not(unbound), case_sensitive) self.thread_local_data = threading.local() + def _visitor(self): + if not hasattr(self.thread_local_data, "visitors"): + self.thread_local_data.visitors = MetricsEvalVisitor(self.expr, self.schema, self.struct) + + return self.thread_local_data.visitors + def eval(self, file): - return self.visitor().eval(file) + return self._visitor().eval(file) class MetricsEvalVisitor(ExpressionVisitors.BoundExpressionVisitor): diff --git a/iceberg/api/expressions/literals.py b/iceberg/api/expressions/literals.py index d340172e23..e8a9fa5915 100644 --- a/iceberg/api/expressions/literals.py +++ b/iceberg/api/expressions/literals.py @@ -18,7 +18,6 @@ import datetime from decimal import (Decimal, ROUND_HALF_UP) -import sys import uuid import pytz @@ -30,11 +29,6 @@ JAVA_MIN_FLOAT) from ..types.type import TypeID -# to-do un-wind python 2 switches -if sys.version_info >= (3, 0): - unicode = str - long = int - class Literals(object): @@ -55,7 +49,7 @@ def from_(value): if Literal.JAVA_MIN_FLOAT < value < Literal.JAVA_MAX_FLOAT: return FloatLiteral(value) return DoubleLiteral(value) - elif isinstance(value, (str, unicode)): + elif isinstance(value, str): return StringLiteral(value) elif isinstance(value, uuid.UUID): return UUIDLiteral(value) @@ -96,7 +90,7 @@ def of(value): if value < Literal.JAVA_MIN_FLOAT or value > Literal.JAVA_MAX_FLOAT: return DoubleLiteral(value) return FloatLiteral(value) - elif isinstance(value, (str, unicode)): + elif isinstance(value, str): return StringLiteral(value) elif isinstance(value, uuid.UUID): return UUIDLiteral(value) diff --git a/iceberg/api/expressions/residual_evaluator.py b/iceberg/api/expressions/residual_evaluator.py index 917c49dffe..d8d506f5da 100644 --- a/iceberg/api/expressions/residual_evaluator.py +++ b/iceberg/api/expressions/residual_evaluator.py @@ -23,19 +23,19 @@ class ResidualEvaluator(object): - def visitor(self): - if not hasattr(self.thread_local_data, "visitors"): - self.thread_local_data.visitors = ResidualVisitor() - - return self.thread_local_data.visitors - def __init__(self, spec, expr): self._spec = spec self._expr = expr - self.thread_local_data = threading.local() + self.__visitor = None + + def _visitor(self): + if self.__visitor is None: + self.__visitor = ResidualVisitor() + + return self.__visitor def residual_for(self, partition_data): - return self.visitor().eval(partition_data) + return self._visitor().eval(partition_data) class ResidualVisitor(ExpressionVisitors.BoundExpressionVisitor): diff --git a/iceberg/api/expressions/strict_metrics_evaluator.py b/iceberg/api/expressions/strict_metrics_evaluator.py index a1752f992c..6d44df3408 100644 --- a/iceberg/api/expressions/strict_metrics_evaluator.py +++ b/iceberg/api/expressions/strict_metrics_evaluator.py @@ -24,7 +24,13 @@ class StrictMetricsEvaluator(object): - def visitor(self): + def __init__(self, schema, unbound): + self.schema = schema + self.struct = schema.as_struct() + self.expr = Binder.bind(self.struct, Expressions.rewrite_not(unbound)) + self.thread_local_data = threading.local() + + def _visitor(self): if not hasattr(self.thread_local_data, "visitors"): self.thread_local_data.visitors = StrictMetricsEvaluator.MetricsEvalVisitor( self.expr, @@ -34,14 +40,8 @@ def visitor(self): return self.thread_local_data.visitors - def __init__(self, schema, unbound): - self.schema = schema - self.struct = schema.as_struct() - self.expr = Binder.bind(self.struct, Expressions.rewrite_not(unbound)) - self.thread_local_data = threading.local() - def eval(self, file): - return self.visitor().eval(file) + return self._visitor().eval(file) class MetricsEvalVisitor(ExpressionVisitors.BoundExpressionVisitor): ROWS_MUST_MATCH = True From 37081c42e3ba3cc01d5c5690b42762d9270dc99d Mon Sep 17 00:00:00 2001 From: TGooch44 Date: Tue, 3 Sep 2019 15:11:21 -0700 Subject: [PATCH 012/642] Python: Fix test failing CI builds (#445) --- iceberg/api/expressions/residual_evaluator.py | 2 -- iceberg/api/types/types.py | 8 ++++---- iceberg/core/avro/avro_schema_util.py | 1 + iceberg/core/filesystem/s3_filesystem.py | 1 - iceberg/core/table_properties.py | 1 - 5 files changed, 5 insertions(+), 8 deletions(-) diff --git a/iceberg/api/expressions/residual_evaluator.py b/iceberg/api/expressions/residual_evaluator.py index d8d506f5da..285abae051 100644 --- a/iceberg/api/expressions/residual_evaluator.py +++ b/iceberg/api/expressions/residual_evaluator.py @@ -15,8 +15,6 @@ # specific language governing permissions and limitations # under the License. -import threading - from .expressions import Expressions, ExpressionVisitors from .predicate import BoundPredicate, Predicate, UnboundPredicate diff --git a/iceberg/api/types/types.py b/iceberg/api/types/types.py index 941c9bc1b3..373463061f 100644 --- a/iceberg/api/types/types.py +++ b/iceberg/api/types/types.py @@ -371,8 +371,8 @@ def __str__(self): class DecimalType(PrimitiveType): @staticmethod - def of(precison, scale): - return DecimalType(precison, scale) + def of(precision, scale): + return DecimalType(precision, scale) def __init__(self, precision, scale): if int(precision) > 38: @@ -385,7 +385,7 @@ def type_id(self): return TypeID.DECIMAL def __repr__(self): - return "decimal(%s,%s)" % (self.precision, self.scale) + return "decimal(%s, %s)" % (self.precision, self.scale) def __str__(self): return self.__repr__() @@ -396,7 +396,7 @@ def __eq__(self, other): elif other is None or not isinstance(other, DecimalType): return False - return self.precision == other.precison and self.scale == other.scale + return self.precision == other.precision and self.scale == other.scale def __ne__(self, other): return not self.__eq__(other) diff --git a/iceberg/core/avro/avro_schema_util.py b/iceberg/core/avro/avro_schema_util.py index 5d478908bf..f513a9cf8c 100644 --- a/iceberg/core/avro/avro_schema_util.py +++ b/iceberg/core/avro/avro_schema_util.py @@ -15,6 +15,7 @@ # specific language governing permissions and limitations # under the License. + class AvroSchemaUtil(object): FIELD_ID_PROP = "field-id" KEY_ID_PROP = "key-id" diff --git a/iceberg/core/filesystem/s3_filesystem.py b/iceberg/core/filesystem/s3_filesystem.py index 9423ee0d1c..6843df6fa2 100644 --- a/iceberg/core/filesystem/s3_filesystem.py +++ b/iceberg/core/filesystem/s3_filesystem.py @@ -40,7 +40,6 @@ AUTOREFRESH_SESSION = None - @retry(wait_incrementing_start=100, wait_exponential_multiplier=4, wait_exponential_max=5000, stop_max_delay=600000, stop_max_attempt_number=7) def get_s3(obj="resource"): diff --git a/iceberg/core/table_properties.py b/iceberg/core/table_properties.py index 0511142529..a50546c78b 100644 --- a/iceberg/core/table_properties.py +++ b/iceberg/core/table_properties.py @@ -16,7 +16,6 @@ # under the License. - class TableProperties(object): COMMIT_NUM_RETRIES = "commit.retry.num-retries" COMMIT_NUM_RETRIES_DEFAULT = 4 From 2edebf5cb3db1559a9c96d765a163f60636d9a6c Mon Sep 17 00:00:00 2001 From: TGooch44 Date: Tue, 10 Sep 2019 15:57:46 -0700 Subject: [PATCH 013/642] [python] Adding a string to iceberg expression converter an tests (#426) --- iceberg/api/expressions/expressions.py | 88 +++++++++++++++ setup.py | 2 + tests/api/expressions/test_str_to_expr.py | 128 ++++++++++++++++++++++ 3 files changed, 218 insertions(+) create mode 100644 tests/api/expressions/test_str_to_expr.py diff --git a/iceberg/api/expressions/expressions.py b/iceberg/api/expressions/expressions.py index 72bb5621f2..190382d34d 100644 --- a/iceberg/api/expressions/expressions.py +++ b/iceberg/api/expressions/expressions.py @@ -15,6 +15,8 @@ # specific language governing permissions and limitations # under the License. +import logging + from .expression import (And, FALSE, Not, @@ -25,6 +27,8 @@ UnboundPredicate) from .reference import NamedReference +_logger = logging.getLogger(__name__) + class Expressions(object): @@ -122,6 +126,90 @@ def rewrite_not(expr): def ref(name): return NamedReference(name) + @staticmethod + def convert_string_to_expr(predicate_string): + from moz_sql_parser import parse + from pyparsing import ParseException + + expr_map = {"and": (Expressions.and_,), + "eq": (Expressions.equal,), + "exists": (Expressions.not_null,), + "gt": (Expressions.greater_than,), + "gte": (Expressions.greater_than_or_equal,), + "lt": (Expressions.less_than,), + "lte": (Expressions.less_than_or_equal,), + "missing": (Expressions.is_null,), + "neq": (Expressions.not_equal,), + "not": (Expressions.not_,), + "or": (Expressions.or_,)} + + dummy_query = "SELECT * FROM tbl WHERE {}".format(predicate_string) # nosec + try: + expr = (Expressions. + _transform_to_binary_tuples(Expressions. + _transform_between_op(parse(dummy_query)["where"]))) + return Expressions._get_expr(expr, expr_map) + except ParseException as pe: + _logger.error("Error parsing string expression into iceberg expression: %s" % str(pe)) + raise + + @staticmethod + def _get_expr(node, expr_map): + if isinstance(node, dict): + for i in node.keys(): + op = i + if op == "literal": + return node["literal"] + mapped_op = expr_map.get(op, expr_map) + if len(mapped_op) == 1: + mapped_op = mapped_op[0] + if mapped_op is None: + raise RuntimeError("no mapping for op: %s" % op) + if mapped_op in (Expressions.not_, Expressions.not_null, Expressions.is_null): + return mapped_op(Expressions._get_expr(node[op], expr_map)) + + return mapped_op(*Expressions._get_expr(node[op], expr_map)) + elif isinstance(node, (list, tuple)): + return (Expressions._get_expr(item, expr_map) for item in node) + elif isinstance(node, (str, int, float)): + return node + else: + raise RuntimeError("unknown node type" % node) + + @staticmethod + def _transform_to_binary_tuples(expr): + if not isinstance(expr, dict): + return expr + for op in expr.keys(): + if op in ("exists", "literal", "missing", "not"): + return expr + new_expr = [Expressions._transform_to_binary_tuples(child) + for child in expr[op]] + while len(new_expr) > 2: + new_and = {op: [new_expr[-2], new_expr[-1]]} + new_expr[-2] = new_and + del new_expr[-1] + expr[op] = new_expr + + return expr + + @staticmethod + def _transform_between_op(expr): + if isinstance(expr, (bool, float, int, str)): + return expr + for op, children in expr.items(): + if op in ("exists", "literal", "missing", "not"): + return expr + new_children = [] + for child in children: + new_children.append(Expressions._transform_between_op(child)) + expr[op] = new_children + if op == "between": + return {"and": [{"gte": [expr[op][0], expr[op][1]]}, + {"lte": [expr[op][0], expr[op][2]]}]} + else: + return expr + class ExpressionVisitors(object): diff --git a/setup.py b/setup.py index 20f01f28bb..4ee888b9fe 100755 --- a/setup.py +++ b/setup.py @@ -30,6 +30,8 @@ 'boto3', 'fastavro', 'mmh3', + 'moz_sql_parser', + 'pyparsing', 'python-dateutil', 'pytz', 'requests', diff --git a/tests/api/expressions/test_str_to_expr.py b/tests/api/expressions/test_str_to_expr.py new file mode 100644 index 0000000000..d44d52e4b8 --- /dev/null +++ b/tests/api/expressions/test_str_to_expr.py @@ -0,0 +1,128 @@ +from iceberg.api.expressions import Expressions + + +def test_equal(): + expected_expr = Expressions.equal("col_a", 1) + conv_expr = Expressions.convert_string_to_expr("col_a=1") + assert expected_expr == conv_expr + + +def test_equal_alt_syntax(): + expected_expr = Expressions.equal("col_a", 1) + conv_expr = Expressions.convert_string_to_expr("col_a==1") + assert expected_expr == conv_expr + + +def test_gt(): + expected_expr = Expressions.greater_than("col_a", 1) + conv_expr = Expressions.convert_string_to_expr("col_a > 1") + assert expected_expr == conv_expr + + +def test_gte(): + expected_expr = Expressions.greater_than_or_equal("col_a", 1) + conv_expr = Expressions.convert_string_to_expr("col_a >= 1") + assert expected_expr == conv_expr + + +def test_lt(): + expected_expr = Expressions.less_than("col_a", 1) + conv_expr = Expressions.convert_string_to_expr("col_a < 1") + assert expected_expr == conv_expr + + +def test_lte(): + expected_expr = Expressions.less_than_or_equal("col_a", 1) + conv_expr = Expressions.convert_string_to_expr("col_a <= 1") + assert expected_expr == conv_expr\ + + +def test_and(): + expected_expr = Expressions.and_(Expressions.equal("col_a", 1), Expressions.equal("col_b", 2)) + conv_expr = Expressions.convert_string_to_expr("col_a=1 and col_b=2") + assert expected_expr == conv_expr + + +def test_or(): + expected_expr = Expressions.or_(Expressions.equal("col_a", 1), Expressions.equal("col_b", 2)) + conv_expr = Expressions.convert_string_to_expr("col_a=1 or col_b=2") + assert expected_expr == conv_expr + + +def test_between(): + expected_expr = Expressions.and_(Expressions.greater_than_or_equal("col_a", 1), + Expressions.less_than_or_equal("col_a", 2)) + conv_expr = Expressions.convert_string_to_expr("col_a between 1 and 2") + assert expected_expr == conv_expr + + +def test_is_null(): + expected_expr = Expressions.is_null("col_a") + conv_expr = Expressions.convert_string_to_expr("col_a is null") + assert expected_expr == conv_expr + + +def test_not_null(): + expected_expr = Expressions.not_null("col_a") + conv_expr = Expressions.convert_string_to_expr("col_a is not null") + assert expected_expr == conv_expr + + +def test_not(): + expected_expr = Expressions.not_("col_a") + conv_expr = Expressions.convert_string_to_expr("not col_a") + assert expected_expr == conv_expr + + +def test_not_equal(): + expected_expr = Expressions.not_equal("col_a", 7) + conv_expr = Expressions.convert_string_to_expr("col_a <> 7") + assert expected_expr == conv_expr + + +def test_not_equal_alt_syntax(): + expected_expr = Expressions.not_equal("col_a", 7) + conv_expr = Expressions.convert_string_to_expr("col_a != 7") + assert expected_expr == conv_expr + + +def test_compound_not_equal(): + expected_expr = Expressions.not_(Expressions.equal("col_a", 7)) + conv_expr = Expressions.convert_string_to_expr("not (col_a = 7)") + assert expected_expr == conv_expr + + +def test_ternary_condition(): + expected_expr = Expressions.and_(Expressions.equal("col_a", 1), + Expressions.and_(Expressions.equal("col_b", 2), + Expressions.equal("col_c", 3))) + + conv_expr = Expressions.convert_string_to_expr("col_a=1 and col_b=2 and col_c=3") + assert expected_expr == conv_expr + + +def test_precedence(): + expected_expr = Expressions.and_(Expressions.or_(Expressions.equal("col_a", 1), + Expressions.equal("col_b", 2)), + Expressions.equal("col_c", 3)) + + conv_expr = Expressions.convert_string_to_expr("col_a=1 or col_b=2 and col_c=3") + assert expected_expr == conv_expr + + +def test_precedence_with_between(): + expected_expr = Expressions.or_(Expressions.and_(Expressions.greater_than_or_equal("col_a", 1), + Expressions.less_than_or_equal("col_a", 2)), + Expressions.equal("col_c", 3)) + + conv_expr = Expressions.convert_string_to_expr("col_a between 1 and 2 or col_c=3") + assert expected_expr == conv_expr + + +def test_complex_expansion(): + expected_expr = Expressions.or_(Expressions.and_(Expressions.equal("a", 1), + Expressions.and_(Expressions.equal("b", 2), + Expressions.not_equal("c", 3))), + Expressions.is_null("d")) + conv_expr = Expressions.convert_string_to_expr("(a=1 and b=2 and c<>3) or d is null") + assert expected_expr == conv_expr From b1f92760a68aac8864bf52dca6d4f8f43beaeb62 Mon Sep 17 00:00:00 2001 From: TGooch44 Date: Thu, 3 Oct 2019 12:19:22 -0700 Subject: [PATCH 014/642] [python] Part 1 of adding parquet read path (#407) +1 LGTM. Thanks Ted --- iceberg/api/__init__.py | 7 +- iceberg/api/data_file.py | 2 +- iceberg/api/data_operations.py | 23 + iceberg/api/file_format.py | 16 +- iceberg/api/file_scan_task.py | 5 + iceberg/api/files.py | 2 +- iceberg/api/manifest_file.py | 21 + iceberg/api/partition_spec.py | 15 +- iceberg/api/schema.py | 9 +- iceberg/api/snapshot.py | 12 + iceberg/api/table_scan.py | 9 + iceberg/api/tables.py | 11 +- iceberg/core/__init__.py | 4 +- iceberg/core/avro/avro_to_iceberg.py | 69 +-- iceberg/core/base_combined_scan_task.py | 37 ++ iceberg/core/base_file_scan_task.py | 126 ++++++ .../core/base_metastore_table_operations.py | 1 + iceberg/core/base_snapshot.py | 24 +- iceberg/core/base_table.py | 3 +- iceberg/core/base_table_scan.py | 187 +++++++- iceberg/core/data_files.py | 3 +- iceberg/core/data_table_scan.py | 98 +++++ iceberg/core/filtered_manifest.py | 118 +++++ iceberg/core/generic_data_file.py | 117 ++++- iceberg/core/generic_manifest_file.py | 41 +- .../core/generic_partition_field_summary.py | 26 +- iceberg/core/manifest_entry.py | 8 +- iceberg/core/manifest_reader.py | 132 ++++-- iceberg/core/partition_data.py | 11 +- iceberg/core/partition_summary.py | 39 +- iceberg/core/scan_summary.py | 409 ++++++++++++++++++ iceberg/core/schema_parser.py | 18 +- iceberg/core/snapshot_parser.py | 9 +- iceberg/core/table_metadata.py | 27 +- iceberg/core/table_metadata_parser.py | 10 +- iceberg/core/util/__init__.py | 17 +- setup.py | 6 +- .../test_inclusive_manifest_evaluator.py | 16 + tests/api/test_file_format.py | 29 ++ tests/core/avro/conftest.py | 49 +++ tests/core/avro/test_read_projection.py | 12 + tests/core/conftest.py | 52 ++- tests/core/test_base_table_scan.py | 38 ++ tests/core/test_snapshot_json.py | 34 +- tests/core/utils/test_bin_packing.py | 2 +- 45 files changed, 1717 insertions(+), 187 deletions(-) create mode 100644 iceberg/api/data_operations.py create mode 100644 iceberg/core/base_combined_scan_task.py create mode 100644 iceberg/core/base_file_scan_task.py create mode 100644 iceberg/core/data_table_scan.py create mode 100644 iceberg/core/filtered_manifest.py create mode 100644 iceberg/core/scan_summary.py create mode 100644 tests/api/test_file_format.py create mode 100644 tests/core/avro/conftest.py create mode 100644 tests/core/avro/test_read_projection.py create mode 100644 tests/core/test_base_table_scan.py diff --git a/iceberg/api/__init__.py b/iceberg/api/__init__.py index 9586e3603f..b6d22bbffe 100644 --- a/iceberg/api/__init__.py +++ b/iceberg/api/__init__.py @@ -15,14 +15,17 @@ # specific language governing permissions and limitations # under the License. -__all__ = ["DataFile", "FileFormat", "Files", "Filterable", - "FilteredSnapshot", "ManifestFile", "PartitionFieldSummary", +__all__ = ["CombinedScanTask", "DataFile", "DataOperations", "FileFormat", "FileScanTask", + "Files", "Filterable", "FilteredSnapshot", "ManifestFile", "PartitionFieldSummary", "Metrics", "PartitionSpec", "PartitionSpecBuilder", "Schema", "Snapshot", "SnapshotIterable", "StructLike", "Table", "Tables", "TableScan", "Transaction", "UpdateSchema"] +from .combined_scan_task import CombinedScanTask from .data_file import DataFile +from .data_operations import DataOperations from .file_format import FileFormat +from .file_scan_task import FileScanTask from .files import Files from .filterable import Filterable from .filtered_snapshot import FilteredSnapshot diff --git a/iceberg/api/data_file.py b/iceberg/api/data_file.py index 7f99ffa563..6e85d6e84f 100644 --- a/iceberg/api/data_file.py +++ b/iceberg/api/data_file.py @@ -79,7 +79,7 @@ def file_ordinal(self): def sort_columns(self): raise NotImplementedError() - def column_size(self): + def column_sizes(self): raise NotImplementedError() def value_counts(self): diff --git a/iceberg/api/data_operations.py b/iceberg/api/data_operations.py new file mode 100644 index 0000000000..7740a2bce1 --- /dev/null +++ b/iceberg/api/data_operations.py @@ -0,0 +1,23 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + + +class DataOperations(object): + APPEND = "append" + REPLACE = "replace" + OVERWRITE = "overwrite" + DELETE = "delete" diff --git a/iceberg/api/file_format.py b/iceberg/api/file_format.py index 0665e938b0..a1e8998b66 100644 --- a/iceberg/api/file_format.py +++ b/iceberg/api/file_format.py @@ -20,16 +20,18 @@ @unique class FileFormat(Enum): - - ORC = "orc" - PARQUET = "parquet" - AVRO = "avro" + ORC = {"extension": "orc", "splittable": True} + PARQUET = {"extension": "parquet", "splittable": True} + AVRO = {"extension": "avro", "splittable": True} def add_extension(self, filename): - if filename.endswith(self.value): + if filename.endswith(self.value["extension"]): return filename else: - return filename + "." + self.value + return filename + "." + self.value["extension"] + + def is_splittable(self): + return self.value["splittable"] @staticmethod def from_file_name(filename): @@ -38,5 +40,5 @@ def from_file_name(filename): return None ext = filename[last_index_of + 1:] for fmt in FileFormat: - if ext == fmt.value: + if ext == fmt.value["extension"]: return fmt diff --git a/iceberg/api/file_scan_task.py b/iceberg/api/file_scan_task.py index 20865f0c74..f51103b38e 100644 --- a/iceberg/api/file_scan_task.py +++ b/iceberg/api/file_scan_task.py @@ -20,18 +20,23 @@ class FileScanTask(ScanTask): + @property def file(self): raise NotImplementedError() + @property def spec(self): raise NotImplementedError() + @property def start(self): raise NotImplementedError() + @property def length(self): raise NotImplementedError() + @property def residual(self): raise NotImplementedError() diff --git a/iceberg/api/files.py b/iceberg/api/files.py index 6ec90bed3c..7c58032e8e 100644 --- a/iceberg/api/files.py +++ b/iceberg/api/files.py @@ -47,7 +47,7 @@ def create(self): if os.path.exists(self.file): raise AlreadyExistsException("File already exists: %s" % self.file) - return PositionOutputStream(open(self.file, "rw")) + return PositionOutputStream(open(self.file, "w")) def create_or_overwrite(self): if os.path.exists(self.file): diff --git a/iceberg/api/manifest_file.py b/iceberg/api/manifest_file.py index e8fa5c9900..a4a0274da1 100644 --- a/iceberg/api/manifest_file.py +++ b/iceberg/api/manifest_file.py @@ -50,9 +50,30 @@ class ManifestFile(object): def schema(): return ManifestFile.SCHEMA + @property + def added_files_count(self): + raise NotImplementedError() + + @property + def existing_files_count(self): + raise NotImplementedError() + + @property + def deleted_files_count(self): + raise NotImplementedError() + def copy(self): raise NotImplementedError() + def has_added_files(self): + return self.added_files_count is None or self.added_files_count > 0 + + def has_existing_files(self): + return self.existing_files_count is None or self.existing_files_count > 0 + + def has_deleted_files(self): + return self.deleted_files_count is None or self.deleted_files_count > 0 + class PartitionFieldSummary(object): TYPE = ManifestFile.schema().find_type("partitions").as_list_type().element_type.as_struct_type() diff --git a/iceberg/api/partition_spec.py b/iceberg/api/partition_spec.py index 272f49536e..e830b28b69 100644 --- a/iceberg/api/partition_spec.py +++ b/iceberg/api/partition_spec.py @@ -25,11 +25,6 @@ from .types import (NestedField, StructType) -""" -TO-DO: Needs some work, please review - -""" - class PartitionSpec(object): @@ -166,7 +161,7 @@ def __hash__(self): return hash(self.__key()) def __key(self): - return PartitionSpec.__class__, self.fields + return PartitionSpec.__class__, tuple(self.fields) def __str__(self): return self.__repr__() @@ -194,9 +189,9 @@ def check_compatibility(spec, schema): for field in spec.fields: src_type = schema.find_type(field.source_id) if not src_type.is_primitive_type(): - raise ValidationException("Cannot partition by non-primitive source field: %s" % src_type) + raise ValidationException("Cannot partition by non-primitive source field: %s", src_type) if not field.transform.can_transform(src_type): - ValidationException("Invalid source type %s for transform: %s" % (src_type, field.transform)) + raise ValidationException("Invalid source type %s for transform: %s", (src_type, field.transform)) class PartitionSpecBuilder(object): @@ -268,7 +263,7 @@ def hour(self, source_name): source_column = self.find_source_column(source_name) self.fields.append(PartitionField(source_column.field_id, source_name, - Transforms.hour(source_column.types))) + Transforms.hour(source_column.type))) return self def bucket(self, source_name, num_buckets): @@ -277,7 +272,7 @@ def bucket(self, source_name, num_buckets): source_column = self.find_source_column(source_name) self.fields.append(PartitionField(source_column.field_id, source_name, - Transforms.bucket(source_column.types, num_buckets))) + Transforms.bucket(source_column.type, num_buckets))) return self def truncate(self, source_name, width): diff --git a/iceberg/api/schema.py b/iceberg/api/schema.py index 76b5339da0..39ae9adf9a 100644 --- a/iceberg/api/schema.py +++ b/iceberg/api/schema.py @@ -108,7 +108,9 @@ def find_field(self, id): def find_column_name(self, id): if isinstance(id, int): - return self._id_to_name.get(id) + field = self.lazy_id_to_field().get(id) + + return None if field is None else field.name def alias_to_id(self, alias): if self._alias_to_id: @@ -142,8 +144,11 @@ def _internal_select(self, case_sensitive, cols): return select(self, selected) + def __len__(self): + return len(self.struct.fields) + def __repr__(self): - return "Schema(%s)" % self.struct.fields + return "Schema(%s)" % ",".join([str(field) for field in self.struct.fields]) def __str__(self): return "table {\n%s\n}" % Schema.NEWLINE.join([" " + str(field) for field in self.struct.fields]) diff --git a/iceberg/api/snapshot.py b/iceberg/api/snapshot.py index a112b0e5b7..d8164a3142 100644 --- a/iceberg/api/snapshot.py +++ b/iceberg/api/snapshot.py @@ -22,6 +22,10 @@ class Snapshot(object): def snapshot_id(self): raise NotImplementedError() + @property + def parent_id(self): + raise NotImplementedError() + @property def timestamp_millis(self): raise NotImplementedError() @@ -30,6 +34,14 @@ def timestamp_millis(self): def manifests(self): raise NotImplementedError() + @property + def summary(self): + raise NotImplementedError() + + @property + def operation(self): + raise NotImplementedError() + def added_files(self): raise NotImplementedError() diff --git a/iceberg/api/table_scan.py b/iceberg/api/table_scan.py index aa679430fc..d93d857a02 100644 --- a/iceberg/api/table_scan.py +++ b/iceberg/api/table_scan.py @@ -37,6 +37,9 @@ def project(self, schema): def select(self, columns): raise NotImplementedError() + def select_except(self, columns): + raise NotImplementedError() + def filter(self, expr): raise NotImplementedError() @@ -51,3 +54,9 @@ def is_case_sensitive(self): def options(self): raise NotImplementedError() + + def to_arrow_table(self): + raise NotImplementedError() + + def to_pandas(self): + raise NotImplementedError() diff --git a/iceberg/api/tables.py b/iceberg/api/tables.py index 7cd2e0501e..1a0b22e448 100644 --- a/iceberg/api/tables.py +++ b/iceberg/api/tables.py @@ -15,11 +15,20 @@ # specific language governing permissions and limitations # under the License. +from .partition_spec import PartitionSpec + class Tables(object): - def create(self, schema, spec, table_identifier): + def create(self, schema, table_identifier=None, spec=None, properties=None): raise NotImplementedError() def load(self, table_identifier): raise NotImplementedError() + + @staticmethod + def default_args(spec=None, properties=None): + spec = spec if spec is not None else PartitionSpec.unpartitioned() + properties = properties if properties is not None else dict() + + return spec, properties diff --git a/iceberg/core/__init__.py b/iceberg/core/__init__.py index 83fb8fc5b7..1d38c1a404 100644 --- a/iceberg/core/__init__.py +++ b/iceberg/core/__init__.py @@ -35,7 +35,8 @@ "SnapshotLogEntry", "TableMetadata", "TableMetadataParser", - "TableOperations"] + "TableOperations", + "TableProperties"] from .base_metastore_table_operations import BaseMetastoreTableOperations from .base_metastore_tables import BaseMetastoreTables @@ -57,3 +58,4 @@ TableMetadata) from .table_metadata_parser import TableMetadataParser from .table_operations import TableOperations +from .table_properties import TableProperties diff --git a/iceberg/core/avro/avro_to_iceberg.py b/iceberg/core/avro/avro_to_iceberg.py index 8e08f1ff05..3aa7f6d10a 100644 --- a/iceberg/core/avro/avro_to_iceberg.py +++ b/iceberg/core/avro/avro_to_iceberg.py @@ -15,6 +15,7 @@ # specific language governing permissions and limitations # under the License. +import fastavro from iceberg.api import Schema from iceberg.api.types import (BinaryType, BooleanType, @@ -62,8 +63,11 @@ class AvroToIceberg(object): "long": LongType.get(), "string": StringType.get(), "time-millis": TimeType.get(), - "timestamp-millis": TimestampType.without_timezone() - } + "timestamp-millis": TimestampType.without_timezone()} + + PROCESS_FUNCS = {TypeID.STRUCT: lambda avro_row, field: AvroToIceberg.get_field_from_struct(avro_row, field), + TypeID.LIST: lambda avro_row, field: AvroToIceberg.get_field_from_list(avro_row, field), + TypeID.MAP: lambda avro_row, field: AvroToIceberg.get_field_from_map(avro_row, field)} @staticmethod def convert_avro_schema_to_iceberg(avro_schema): @@ -228,37 +232,42 @@ def is_option_schema(field_type): return False @staticmethod - def read_avro_row(iceberg_schema, avro_reader): - try: - avro_row = avro_reader.__next__() + def read_avro_file(iceberg_schema, data_file): + fo = data_file.new_fo() + avro_reader = fastavro.reader(fo) + for avro_row in avro_reader: iceberg_row = dict() for field in iceberg_schema.as_struct().fields: iceberg_row[field.name] = AvroToIceberg.get_field_from_avro(avro_row, field) yield iceberg_row + fo.close() + + @staticmethod + def read_avro_row(iceberg_schema, avro_reader): + try: + for avro_row in avro_reader: + iceberg_row = dict() + for field in iceberg_schema.as_struct().fields: + iceberg_row[field.name] = AvroToIceberg.get_field_from_avro(avro_row, field) + yield iceberg_row except StopIteration: return @staticmethod def get_field_from_avro(avro_row, field): - process_funcs = {TypeID.STRUCT: lambda avro_row, field: AvroToIceberg.get_field_from_struct(avro_row, field), - TypeID.LIST: lambda avro_row, field: AvroToIceberg.get_field_from_list(avro_row, field), - TypeID.MAP: lambda avro_row, field: AvroToIceberg.get_field_from_map(avro_row, field)} - if field.type.is_primitive_type(): - processing_func = AvroToIceberg.get_field_from_primitive - else: - processing_func = process_funcs.get(field.type.type_id) - - if processing_func is None: + try: + return AvroToIceberg.PROCESS_FUNCS.get(field.type.type_id, + AvroToIceberg.get_field_from_primitive)(avro_row, field) + except KeyError: raise RuntimeError("Don't know how to get field of type: %s" % field.type.type_id) - return processing_func(avro_row, field) - @staticmethod def get_field_from_primitive(avro_row, field): - avro_value = avro_row.get(field.name) - if avro_row is None and field.is_required: - raise RuntimeError("Field is required but missing in source %s\n%s:" % (field, avro_row)) - return avro_value + try: + return avro_row[field.name] + except KeyError: + if field.is_required: + raise RuntimeError("Field is required but missing in source %s\n%s:" % (field, avro_row)) @staticmethod def get_field_from_struct(avro_row, field): @@ -269,21 +278,25 @@ def get_field_from_struct(avro_row, field): @staticmethod def get_field_from_list(avro_row, field): - avro_value = avro_row.get(field.name) - if avro_value is None: + try: + return avro_row[field.name] + except KeyError: if field.is_required: raise RuntimeError("Field is required but missing in source %s\n%s:" % (field, avro_row)) - return None - - return avro_value @staticmethod def get_field_from_map(avro_row, field): val_map = dict() - avro_value = avro_row.get(field.name) - if avro_value is None and field.is_required: - raise RuntimeError("Field is required but missing in source %s\n%s:" % (field, avro_row)) + + try: + avro_value = avro_row[field.name] + except KeyError: + if field.is_required: + raise RuntimeError("Field is required but missing in source %s\n%s:" % (field, avro_row)) + else: + return None for val in avro_value: val_map[val['key']] = val['value'] + return val_map diff --git a/iceberg/core/base_combined_scan_task.py b/iceberg/core/base_combined_scan_task.py new file mode 100644 index 0000000000..8cb209e2f9 --- /dev/null +++ b/iceberg/core/base_combined_scan_task.py @@ -0,0 +1,37 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +from copy import deepcopy + +from iceberg.api import CombinedScanTask + + +class BaseCombinedScanTask(CombinedScanTask): + + def __init__(self, tasks): + self.tasks = deepcopy(tasks) + + @property + def files(self): + return self.tasks + + def __repr__(self): + return "BaseCombinedScanTask([{}])".format(self.tasks) + + def __str__(self): + total_size = sum([task.length for task in self.tasks]) + return "BaseCombinedScanTask(num_tasks={}, total_size={})".format(len(self.tasks), total_size) diff --git a/iceberg/core/base_file_scan_task.py b/iceberg/core/base_file_scan_task.py new file mode 100644 index 0000000000..f36fcf86f7 --- /dev/null +++ b/iceberg/core/base_file_scan_task.py @@ -0,0 +1,126 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +from iceberg.api import FileScanTask + +from .partition_spec_parser import PartitionSpecParser +from .schema_parser import SchemaParser + + +class BaseFileScanTask(FileScanTask): + + def __init__(self, file, schema_str, spec_str, residuals): + self._file = file + self._schema_str = schema_str + self._spec_str = spec_str + self._spec = None + self._residuals = residuals + + @property + def file(self): + return self._file + + @property + def spec(self): + if self._spec is None: + self._spec = PartitionSpecParser.from_json(SchemaParser.from_json(self._schema_str), self._spec_str) + + return self._spec + + @property + def start(self): + return 0 + + @property + def length(self): + return self._file.file_size_in_bytes() + + @property + def residual(self): + return self._residuals.residual_for(self._file.partition()) + + def split(self, split_size): + if self.file.format().is_splittable(): + return [task for task in SplitScanTaskIterator(split_size, self)] + else: + return self + + def __repr__(self): + fields = ["file: {}".format(self._file.path()), + "partition_data: {}".format(self._file.partition()), + "residual: {}".format(self.residual)] + + return "BaseFileScanTask({})".format(", ".join(fields)) + + def __str__(self): + return self.__repr__() + + +class SplitScanTaskIterator(object): + + def __init__(self, split_size, file_scan_task): + self._offset = 0 + self._remaining_len = file_scan_task.length + self._split_size = split_size + self._file_scan_task = file_scan_task + + def has_next(self): + return self._remaining_len > 0 + + def __iter__(self): + return self + + def __next__(self): + if self.has_next(): + len = min(self._split_size, self._remaining_len) + split_task = SplitScanTask(self._offset, len, self._file_scan_task) + self._offset += len + self._remaining_len -= len + return split_task + + raise StopIteration + + +class SplitScanTask(FileScanTask): + + def __init__(self, offset, len, file_scan_task): + self._offset = offset + self._len = len + self._file_scan_task = file_scan_task + + @property + def file(self): + return self._file_scan_task.file + + @property + def spec(self): + return self._file_scan_task.spec + + @property + def start(self): + return self._offset + + @property + def length(self): + return self._len + + @property + def residual(self): + return self._file_scan_task.residual() + + def split(self): + raise RuntimeError("Cannot split a task which is already split") diff --git a/iceberg/core/base_metastore_table_operations.py b/iceberg/core/base_metastore_table_operations.py index a50f34283b..dc4594a182 100644 --- a/iceberg/core/base_metastore_table_operations.py +++ b/iceberg/core/base_metastore_table_operations.py @@ -32,6 +32,7 @@ class BaseMetastoreTableOperations(TableOperations): TABLE_TYPE_PROP = "table_type" ICEBERG_TABLE_TYPE_VALUE = "iceberg" METADATA_LOCATION_PROP = "metadata_location" + PARTITION_SPEC_PROP = "partition_spec" PREVIOUS_METADATA_LOCATION_PROP = "previous_metadata_location" METADATA_FOLDER_NAME = "metadata" diff --git a/iceberg/core/base_snapshot.py b/iceberg/core/base_snapshot.py index b1c222c55a..bf90ec4d16 100644 --- a/iceberg/core/base_snapshot.py +++ b/iceberg/core/base_snapshot.py @@ -17,7 +17,6 @@ import time -import fastavro from iceberg.api import (Filterable, FilteredSnapshot, ManifestFile, @@ -56,8 +55,8 @@ def __init__(self, ops, snapshot_id, parent_id=None, manifests=None, manifest_li else: self._manifests = None self._manifest_list = manifest_list - self.operation = operation - self.summary = summary + self._operation = operation + self._summary = summary self._adds = None self._deletes = None @@ -78,14 +77,23 @@ def parent_id(self): def manifests(self): if self._manifests is None: # if manifest isn't set then the snapshot_file is set and should be read to get the list - with self._manifest_list.new_fo() as fo: - avro_reader = fastavro.reader(fo) - - self._manifests = [GenericManifestFile.from_avro_record_json(manifest) - for manifest in AvroToIceberg.read_avro_row(ManifestFile.schema(), avro_reader)] + return (GenericManifestFile.from_avro_record_json(manifest) + for manifest in AvroToIceberg.read_avro_file(ManifestFile.schema(), self._manifest_list)) return self._manifests + @property + def manifest_location(self): + return self._manifest_list.location if self._manifest_list is not None else None + + @property + def summary(self): + return self._summary + + @property + def operation(self): + return self._operation + def select(self, columns): return FilteredSnapshot(self, Expressions.always_true(), Expressions.always_true(), columns) diff --git a/iceberg/core/base_table.py b/iceberg/core/base_table.py index 6c1d3b5971..a86d9960b3 100644 --- a/iceberg/core/base_table.py +++ b/iceberg/core/base_table.py @@ -17,6 +17,7 @@ from iceberg.api import Table +from .data_table_scan import DataTableScan from .schema_update import SchemaUpdate @@ -30,7 +31,7 @@ def refresh(self): self.ops.refresh() def new_scan(self): - raise NotImplementedError() + return DataTableScan(self.ops, self) def schema(self): return self.ops.current().schema diff --git a/iceberg/core/base_table_scan.py b/iceberg/core/base_table_scan.py index 254b95f7f6..d205364aad 100644 --- a/iceberg/core/base_table_scan.py +++ b/iceberg/core/base_table_scan.py @@ -15,22 +15,199 @@ # specific language governing permissions and limitations # under the License. +from datetime import datetime +import logging from iceberg.api import Filterable from iceberg.api import TableScan -from iceberg.api.expressions import Expressions +from iceberg.api.expressions import (Binder, + Expressions) from iceberg.api.io import CloseableGroup +from iceberg.api.types import get_projected_ids, select + +from .base_combined_scan_task import BaseCombinedScanTask +from .table_properties import TableProperties +from .util import PackingIterator + +_logger = logging.getLogger(__name__) class BaseTableScan(CloseableGroup, TableScan): + DATE_FORMAT = "%Y-%m-%d %H:%M:%S.%f" + SNAPSHOT_COLUMNS = ["snapshot_id", "file_path", "file_ordinal", "file_format", "block_size_in_bytes", + "file_size_in_bytes", "record_count", "partition", "value_counts", "null_value_counts", + "lower_bounds", "upper_bounds"] + + def new_refined_scan(self, ops, table, schema, snapshot_id, row_filter, + case_sensitive, selected_columns, options, minused_cols): + raise NotImplementedError() - def __init__(self, ops, table, snapshot_id=None, columns=None, row_filter=None): + def target_split_size(self, ops): + raise NotImplementedError() + + def __init__(self, ops, table, schema, snapshot_id=None, columns=None, + row_filter=None, case_sensitive=True, selected_columns=None, options=None, + minused_cols=None): self.ops = ops self.table = table + self._schema = schema self.snapshot_id = snapshot_id self.columns = columns - self.row_filter = row_filter + self._row_filter = row_filter + self._case_sensitive = case_sensitive + self.selected_columns = selected_columns + self.minused_cols = minused_cols or list() + self.options = options if options is not None else dict() - if self.columns is None and self.row_filter is None: + if self.columns is None and self._row_filter is None: self.columns = Filterable.ALL_COLUMNS - self.row_filter = Expressions.always_true() + self._row_filter = Expressions.always_true() + + self._stats = dict() + + def is_case_sensitive(self): + return self.case_sensitive + + def use_snapshot(self, snapshot_id): + if self.snapshot_id is not None: + raise RuntimeError("Cannot override snapshot, already set to id=%s" % self.snapshot_id) + if self.ops.current().snapshot(snapshot_id) is None: + raise RuntimeError("Cannot find snapshot with ID %s" % self.snapshot_id) + + return self.new_refined_scan(self.ops, self.table, self._schema, snapshot_id=snapshot_id, + row_filter=self._row_filter, case_sensitive=self._case_sensitive, + selected_columns=self.selected_columns, options=self.options, + minused_cols=self.minused_cols) + + def as_of_time(self, timestamp_millis): + raise NotImplementedError() + + def project(self, schema): + return self.new_refined_scan(self.ops, self.table, schema, snapshot_id=self.snapshot_id, + row_filter=self._row_filter, case_sensitive=self._case_sensitive, + selected_columns=self.selected_columns, options=self.options, + minused_cols=self.minused_cols) + + def case_sensitive(self, case_sensitive): + return self.new_refined_scan(self.ops, self.table, self._schema, snapshot_id=self.snapshot_id, + row_filter=self._row_filter, case_sensitive=case_sensitive, + selected_columns=self.selected_columns, options=self.options, + minused_cols=self.minused_cols) + + def select(self, columns): + return self.new_refined_scan(self.ops, self.table, self._schema, snapshot_id=self.snapshot_id, + row_filter=self._row_filter, case_sensitive=self._case_sensitive, + selected_columns=columns, options=self.options, + minused_cols=self.minused_cols) + + def select_except(self, columns): + return self.new_refined_scan(self.ops, self.table, self._schema, snapshot_id=self.snapshot_id, + row_filter=self._row_filter, case_sensitive=self._case_sensitive, + selected_columns=self.selected_columns, options=self.options, + minused_cols=columns) + + @property + def row_filter(self): + return self._row_filter + + def filter(self, expr): + return self.new_refined_scan(self.ops, self.table, self._schema, snapshot_id=self.snapshot_id, + row_filter=Expressions.and_(self._row_filter, expr), + case_sensitive=self._case_sensitive, selected_columns=self.selected_columns, + options=self.options, minused_cols=self.minused_cols) + + def option(self, property, value): + builder = dict() + builder.update(self.options) + builder[property] = value + return self.new_refined_scan(self.ops, self.table, self._schema, snapshot_id=self.snapshot_id, + row_filter=self._row_filter, case_sensitive=self._case_sensitive, + selected_columns=self.selected_columns, options=builder, + minused_cols=self.minused_cols) + + def plan_files(self, ops=None, snapshot=None, row_filter=None): + + if not all(i is None for i in [ops, snapshot, row_filter]): + raise NotImplementedError() + + snapshot = self.ops.current().snapshot(self.snapshot_id) \ + if self.snapshot_id is not None else self.ops.current().current_snapshot() + + if snapshot is not None: + _logger.info("Scanning table {} snapshot {} created at {} with filter {}" + .format(self.table, + snapshot.snapshot_id, + datetime.fromtimestamp(snapshot.timestamp_millis / 1000.0) + .strftime(BaseTableScan.DATE_FORMAT), + self._row_filter)) + + return self.plan_files(ops, snapshot, row_filter) + else: + _logger.info("Scanning empty table {}" % self.table) + + def plan_tasks(self): + split_size = self.target_split_size(self.ops) + lookback = int(self.ops.current().properties.get(TableProperties.SPLIT_LOOKBACK, + TableProperties.SPLIT_LOOKBACK_DEFAULT)) + open_file_cost = int(self.ops.current().properties.get(TableProperties.SPLIT_OPEN_FILE_COST, + TableProperties.SPLIT_OPEN_FILE_COST_DEFAULT)) + + if not self.ops.conf.get("iceberg.scan.split-file-tasks", True): + split_files = list(self.plan_files()) + else: + split_files = self.split_files(split_size) + + def weight_func(file): + return max(file.length, open_file_cost) + + return (BaseCombinedScanTask(scan_tasks) + for scan_tasks in PackingIterator(split_files, split_size, lookback, weight_func)) + + def split_files(self, split_size): + file_scan_tasks = list(self.plan_files()) + split_tasks = [task for split_tasks in [scan_task.split(split_size) for scan_task in file_scan_tasks] + for task in split_tasks] + + return split_tasks + + @property + def schema(self): + return self._lazy_column_projection() + + def to_arrow_table(self): + raise NotImplementedError() + + def to_pandas(self): + raise NotImplementedError() + + def _lazy_column_projection(self): + if "*" in self.selected_columns: + if len(self.minused_cols) == 0: + return self._schema + self.selected_columns = [field.name for field in self._schema.as_struct().fields] + final_selected_cols = [column for column in self.selected_columns if column not in self.minused_cols] + else: + final_selected_cols = self.selected_columns + + required_field_ids = set() + required_field_ids.update(Binder.bound_references(self.table.schema().as_struct(), + [self._row_filter], + self._case_sensitive)) + + if self._case_sensitive: + selected_ids = get_projected_ids(self.table.schema().select(final_selected_cols)) + else: + selected_ids = get_projected_ids(self.table.schema().case_insensitive_select(final_selected_cols)) + + required_field_ids.update(selected_ids) + + return select(self.table.schema(), required_field_ids) + + def __repr__(self): + return "BaseTableScan(table={}, projection={}, filter={}, case_sensitive={}".format(self.table, + self._schema.as_struct(), + self._row_filter, + self._case_sensitive) + + def __str__(self): + return self.__repr__() diff --git a/iceberg/core/data_files.py b/iceberg/core/data_files.py index 0a50e5babd..39503f6529 100644 --- a/iceberg/core/data_files.py +++ b/iceberg/core/data_files.py @@ -20,7 +20,6 @@ Metrics) from iceberg.api.types import Conversions -from .filesystem import FileSystemInputFile from .generic_data_file import GenericDataFile from .partition_data import PartitionData @@ -78,6 +77,7 @@ def copy(spec, partition): @staticmethod def from_input_file(input_file, row_count, partition_data=None, metrics=None): + from .filesystem import FileSystemInputFile if isinstance(input_file, FileSystemInputFile): return DataFiles.from_stat(input_file.get_stat(), row_count, partition_data=partition_data, metrics=metrics) @@ -149,6 +149,7 @@ def with_status(self, stat): return self def with_input_file(self, input_file): + from .filesystem import FileSystemInputFile if isinstance(input_file, FileSystemInputFile): self.with_status(input_file.get_stat()) diff --git a/iceberg/core/data_table_scan.py b/iceberg/core/data_table_scan.py new file mode 100644 index 0000000000..6d8c7b001c --- /dev/null +++ b/iceberg/core/data_table_scan.py @@ -0,0 +1,98 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +import itertools +import logging +from multiprocessing import cpu_count +from multiprocessing.dummy import Pool + +from iceberg.api.expressions import (InclusiveManifestEvaluator, + ResidualEvaluator) + +from .base_file_scan_task import BaseFileScanTask +from .base_table_scan import BaseTableScan +from .manifest_reader import ManifestReader +from .partition_spec_parser import PartitionSpecParser +from .schema_parser import SchemaParser +from .table_properties import TableProperties +from .util import SCAN_THREAD_POOL_ENABLED, WORKER_THREAD_POOL_SIZE_PROP + + +_logger = logging.getLogger(__name__) + + +class DataTableScan(BaseTableScan): + + SNAPSHOT_COLUMNS = ("snapshot_id", "file_path", "file_ordinal", "file_format", "block_size_in_bytes", + "file_size_in_bytes", "record_count", "partition", "value_counts", "null_value_counts", + "lower_bounds", "upper_bounds") + + def __init__(self, ops, table, schema=None, snapshot_id=None, row_filter=None, + case_sensitive=True, selected_columns=None, options=None, minused_cols=None): + super(DataTableScan, self).__init__(ops, table, schema if schema is not None else table.schema(), + snapshot_id=snapshot_id, row_filter=row_filter, + case_sensitive=case_sensitive, selected_columns=selected_columns, + options=options, minused_cols=minused_cols) + self._cached_evaluators = dict() + + def new_refined_scan(self, ops, table, schema, snapshot_id=None, row_filter=None, case_sensitive=None, + selected_columns=None, options=None, minused_cols=None): + return DataTableScan(ops, table, schema, + snapshot_id=snapshot_id, row_filter=row_filter, case_sensitive=case_sensitive, + selected_columns=selected_columns, options=options, minused_cols=minused_cols) + + def plan_files(self, ops=None, snapshot=None, row_filter=None): + if all(i is None for i in [ops, snapshot, row_filter]): + return super(DataTableScan, self).plan_files() + + matching_manifests = [manifest for manifest in snapshot.manifests + if self.cache_loader(manifest.spec_id).eval(manifest)] + + if self.ops.conf.get(SCAN_THREAD_POOL_ENABLED): + with Pool(self.ops.conf.get(WORKER_THREAD_POOL_SIZE_PROP, + cpu_count())) as reader_scan_pool: + return itertools.chain.from_iterable([scan for scan + in reader_scan_pool.map(self.get_scans_for_manifest, + matching_manifests)]) + else: + return itertools.chain.from_iterable([self.get_scans_for_manifest(manifest) + for manifest in matching_manifests]) + + def cache_loader(self, spec_id): + spec = self.ops.current().spec_id(spec_id) + return InclusiveManifestEvaluator(spec, self.row_filter) + + def get_scans_for_manifest(self, manifest): + from .filesystem import FileSystemInputFile + input_file = FileSystemInputFile.from_location(manifest.manifest_path, self.ops.conf) + reader = ManifestReader.read(input_file) + schema_str = SchemaParser.to_json(reader.spec.schema) + spec_str = PartitionSpecParser.to_json(reader.spec) + residuals = ResidualEvaluator(reader.spec, self.row_filter) + return [BaseFileScanTask(file, schema_str, spec_str, residuals) + for file in reader.filter_rows(self.row_filter).select(BaseTableScan.SNAPSHOT_COLUMNS).iterator()] + + def target_split_size(self, ops): + scan_split_size_str = self.options.get(TableProperties.SPLIT_SIZE) + + if scan_split_size_str is not None: + try: + return int(scan_split_size_str) + except ValueError: + _logger.warning("Invalid %s option: %s" % (TableProperties.SPLIT_SIZE, scan_split_size_str)) + + return int(self.ops.current().properties.get(TableProperties.SPLIT_SIZE, TableProperties.SPLIT_SIZE_DEFAULT)) diff --git a/iceberg/core/filtered_manifest.py b/iceberg/core/filtered_manifest.py new file mode 100644 index 0000000000..d5dfc2e307 --- /dev/null +++ b/iceberg/core/filtered_manifest.py @@ -0,0 +1,118 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +from iceberg.api.expressions import Evaluator, Expressions, inclusive, InclusiveMetricsEvaluator + +from .manifest_entry import Status + + +class FilteredManifest(object): + + def __init__(self, reader, part_filter, row_filter, columns, case_sensitive=True): + if reader is None: + raise RuntimeError("ManifestReader cannot be null") + + self.reader = reader + self.part_filter = part_filter + self.row_filter = row_filter + self.columns = columns + self.case_sensitive = case_sensitive + + self.lazy_evaluator = None + self.lazy_metrics_evaluator = None + + def select(self, columns): + return FilteredManifest(self.reader, self.part_filter, self.row_filter, columns, self.case_sensitive) + + def filter_partitions(self, expr): + return FilteredManifest(self.reader, + Expressions.and_(self.part_filter, expr), + self.row_filter, + self.columns, + self.case_sensitive) + + def filter_rows(self, expr): + projected = inclusive(self.reader.spec).project(expr) + return FilteredManifest(self.reader, + Expressions.and_(self.part_filter, projected), + Expressions.and_(self.row_filter, expr), + self.columns, self.case_sensitive) + + def all_entries(self): + if self.row_filter is not None and self.row_filter != Expressions.always_true() \ + or self.part_filter is not None and self. part_filter != Expressions.always_true(): + evaluator = self.evaluator() + metrics_evaluator = self.metrics_evaluator() + return [entry for entry in self.reader.entries(self.columns) + if entry is not None + and evaluator.eval(entry.file.partition()) + and metrics_evaluator.eval(entry.file)] + else: + return self.reader.entries(self.columns) + + def live_entries(self): + if self.row_filter is not None and self.row_filter != Expressions.always_true() \ + or self.part_filter is not None and self. part_filter != Expressions.always_true(): + evaluator = self.evaluator() + metrics_evaluator = self.metrics_evaluator() + return [entry for entry in self.reader.entries(self.columns) + if entry is not None + and entry.status != Status.DELETED + and evaluator.eval(entry.file.partition()) + and metrics_evaluator.eval(entry.file)] + else: + + return [entry for entry in self.reader.entries(self.columns) + if entry is not None and entry.status != Status.DELETED] + + def iterator(self): + if self.row_filter is not None and self.row_filter != Expressions.always_true() \ + or self.part_filter is not None and self.part_filter != Expressions.always_true(): + evaluator = self.evaluator() + metrics_evaluator = self.metrics_evaluator() + + return (input.copy() for input in self.reader.iterator(self.part_filter, self.columns) + if input is not None + and evaluator.eval(input.partition()) + and metrics_evaluator.eval(input)) + else: + return (entry.copy() for entry in self.reader.iterator(self.part_filter, self.columns)) + + def evaluator(self): + if self.lazy_evaluator is None: + if self.part_filter is not None: + self.lazy_evaluator = Evaluator(self.reader.spec.partition_type(), + self.part_filter, + self.case_sensitive) + else: + self.lazy_evaluator = Evaluator(self.reader.spec.partition_type(), + Expressions.always_true(), + self.case_sensitive) + + return self.lazy_evaluator + + def metrics_evaluator(self): + if self.lazy_metrics_evaluator is None: + if self.row_filter is not None: + self.lazy_metrics_evaluator = InclusiveMetricsEvaluator(self.reader.spec.schema, + self.row_filter, self.case_sensitive) + else: + self.lazy_metrics_evaluator = InclusiveMetricsEvaluator(self.reader.spec.schema, + Expressions.always_true(), + self.case_sensitive) + + return self.lazy_metrics_evaluator diff --git a/iceberg/core/generic_data_file.py b/iceberg/core/generic_data_file.py index 1f5e84f7a5..bf86e4c9cd 100644 --- a/iceberg/core/generic_data_file.py +++ b/iceberg/core/generic_data_file.py @@ -15,6 +15,7 @@ # specific language governing permissions and limitations # under the License. +import copy from iceberg.api import (DataFile, StructLike) @@ -31,38 +32,106 @@ class GenericDataFile(DataFile, StructLike): def __init__(self, file_path, format, file_size_in_bytes, block_size_in_bytes, row_count=None, partition=None, metrics=None): - self.file_path = file_path - self.format = format - self.row_count = row_count - self.file_size_in_bytes = file_size_in_bytes - self.block_size_in_bytes = block_size_in_bytes - self.file_ordinal = None - self.sort_columns = None + + self._file_path = file_path + self._format = format + self._row_count = row_count + self._file_size_in_bytes = file_size_in_bytes + self._block_size_in_bytes = block_size_in_bytes + self._file_ordinal = None + self._sort_columns = None if partition is None: - self.partition_data = GenericDataFile.EMPTY_PARTITION_DATA - self.partition_type = GenericDataFile.EMPTY_PARTITION_DATA.partition_type + self._partition_data = GenericDataFile.EMPTY_PARTITION_DATA + self._partition_type = GenericDataFile.EMPTY_PARTITION_DATA.partition_type else: - self.partition_data = partition - self.partition_type = partition.get_partition_type() + self._partition_data = partition + self._partition_type = partition.get_partition_type() if metrics is None: - self.row_count = row_count - self.column_sizes = None - self.value_counts = None - self.null_value_counts = None - self.lower_bounds = None - self.upper_bounds = None + self._row_count = row_count + self._column_sizes = None + self._value_counts = None + self._null_value_counts = None + self._lower_bounds = None + self._upper_bounds = None else: - self.row_count = metrics.row_count - self.column_sizes = metrics.column_sizes - self.value_counts = metrics.value_counts - self.null_value_counts = metrics.null_value_counts - self.lower_bounds = metrics.lower_bounds - self.upper_bounds = metrics.upper_bounds + self._row_count = metrics.row_count + self._column_sizes = metrics.column_sizes + self._value_counts = metrics.value_counts + self._null_value_counts = metrics.null_value_counts + self._lower_bounds = metrics.lower_bounds + self._upper_bounds = metrics.upper_bounds def partition(self): - return self.partition_data + return self._partition_data + + def path(self): + return self._file_path + + def format(self): + return self._format + + def record_count(self): + return self._row_count + + def file_size_in_bytes(self): + return self._file_size_in_bytes + + def block_size_in_bytes(self): + return self._block_size_in_bytes + + def file_ordinal(self): + return self._file_ordinal + + def sort_columns(self): + return self._sort_columns + + def column_sizes(self): + return self._column_sizes + + def value_counts(self): + return self._value_counts + + def null_value_counts(self): + return self._null_value_counts + + def lower_bounds(self): + return self._lower_bounds + + def upper_bounds(self): + return self._upper_bounds + + def copy(self): + return copy.deepcopy(self) @staticmethod def get_avro_schema(partition_type): return IcebergToAvro.type_to_schema(DataFile.get_type(partition_type), DataFile.__class__.__name__) + + def __repr__(self): + fields = ["file_path: {}".format(self._file_path), + "file_format: {}".format(self._format), + "partition: {}".format(self._partition_data), + "record_count: {}".format(self._row_count), + "file_size_in_bytes: {}".format(self._file_size_in_bytes), + "block_size_in_bytes: {}".format(self._block_size_in_bytes), + "column_sizes: {}".format(self._column_sizes), + "value_counts: {}".format(self._value_counts), + "null_value_counts: {}".format(self._null_value_counts), + "lower_bounds: {}".format(self._lower_bounds), + "upper_bounds: {}".format(self._upper_bounds), + ] + return "GenericDataFile({})".format("\n,".join(fields)) + + def __str__(self): + return self.__repr__() + + def __deepcopy__(self, memodict): + cls = self.__class__ + result = cls.__new__(cls) + memodict[id(self)] = result + + for k, v in self.__dict__.items(): + setattr(result, k, copy.deepcopy(v, memodict)) + + return result diff --git a/iceberg/core/generic_manifest_file.py b/iceberg/core/generic_manifest_file.py index 73e669b3d0..cdeeceadb3 100644 --- a/iceberg/core/generic_manifest_file.py +++ b/iceberg/core/generic_manifest_file.py @@ -61,16 +61,28 @@ def __init__(self, path=None, file=None, spec_id=None, length=None, snapshot_id= self._length = length self.spec_id = spec_id self.snapshot_id = snapshot_id - self.added_files_count = added_files_count - self.existing_files_count = existing_files_count - self.deleted_files_count = deleted_files_count - self.partitions = partitions if partitions is not None else list() + self._added_files_count = added_files_count + self._existing_files_count = existing_files_count + self._deleted_files_count = deleted_files_count + self.partitions = partitions self.from_projection_pos = None @property def length(self): return self.lazy_length() + @property + def added_files_count(self): + return self._added_files_count + + @property + def existing_files_count(self): + return self._existing_files_count + + @property + def deleted_files_count(self): + return self._deleted_files_count + def lazy_length(self): if self._length is None: if self.file is not None: @@ -103,17 +115,17 @@ def set(self, pos, value): if pos == 0: self.manifest_path = str(value) elif pos == 1: - self.length = int(value) + self._length = int(value) elif pos == 2: self.spec_id = int(value) elif pos == 3: self.snapshot_id = int(value) elif pos == 4: - self.added_files_count = int(value) + self._added_files_count = int(value) elif pos == 5: - self.existing_files_count = int(value) + self._existing_files_count = int(value) elif pos == 6: - self.deleted_files_count = int(value) + self._deleted_files_count = int(value) elif pos == 7: self.partitions = value @@ -122,7 +134,7 @@ def copy(self): snapshot_id=self.snapshot_id, added_files_count=self.added_files_count, existing_files_count=self.existing_files_count, deleted_files_count=self.deleted_files_count, - partitions=list(self.partitions), avro_schema=self.avro_schema) + partitions=list(self.partitions)) @staticmethod def get_schema(): @@ -145,16 +157,19 @@ def to_avro_record_dict(manifest): @staticmethod def from_avro_record_json(row): + partitions = row.get("partitions") + if partitions is not None: + partitions = [GenericPartitionFieldSummary(contains_null=partition["contains_null"], + lower_bound=partition["lower_bound"], + upper_bound=partition["upper_bound"]) + for partition in row.get("partitions")] return GenericManifestFile(path=row.get("manifest_path"), length=row.get("manifest_length"), spec_id=row.get("partition_spec_id"), added_files_count=row.get("added_data_files_count"), existing_files_count=row.get("existing_data_files_count"), deleted_files_count=row.get("existing_data_files_count"), - partitions=[GenericPartitionFieldSummary(contains_null=partition["contains_null"], - lower_bound=partition["lower_bound"], - upper_bound=partition["upper_bound"]) - for partition in row.get("partitions")]) + partitions=partitions) def __eq__(self, other): if id(self) == id(other): diff --git a/iceberg/core/generic_partition_field_summary.py b/iceberg/core/generic_partition_field_summary.py index 8fa054bd21..77ade67df6 100644 --- a/iceberg/core/generic_partition_field_summary.py +++ b/iceberg/core/generic_partition_field_summary.py @@ -23,21 +23,27 @@ class GenericPartitionFieldSummary(PartitionFieldSummary, StructLike): AVRO_SCHEMA = None # IcebergToAvro.type_to_schema(PartitionFieldSummary.get_type()) - def __init__(self, avro_schema=None, contains_null=False, lower_bound=None, upper_bound=None): + def __init__(self, avro_schema=None, contains_null=False, lower_bound=None, upper_bound=None, copy=None): + if copy is not None: + avro_schema = copy.avro_schema + contains_null = copy.contains_null() + lower_bound = copy.lower_bound() + upper_bound = copy.upper_bound() + if avro_schema is None: avro_schema = GenericPartitionFieldSummary.AVRO_SCHEMA self.avro_schema = avro_schema - self.contains_null = contains_null - self.lower_bound = lower_bound - self.upper_bound = upper_bound + self._contains_null = contains_null + self._lower_bound = lower_bound + self._upper_bound = upper_bound def __str__(self): - return ("GenericPartitionFieldSummary(contains_null={},lower_bound={}, upper_bound={}" - .format(self.contains_null, self.lower_bound, self.upper_bound)) + return ("GenericPartitionFieldSummary(contains_null={},lower_bound={}, upper_bound={})" + .format(self.contains_null(), self.lower_bound(), self.upper_bound())) def contains_null(self): - raise NotImplementedError() + return self._contains_null def get(self, pos): raise NotImplementedError() @@ -46,10 +52,10 @@ def set(self, pos, value): raise NotImplementedError() def lower_bound(self): - raise NotImplementedError() + return self._lower_bound def upper_bound(self): - raise NotImplementedError() + return self._upper_bound def copy(self): - raise NotImplementedError() + return GenericPartitionFieldSummary(copy=self) diff --git a/iceberg/core/manifest_entry.py b/iceberg/core/manifest_entry.py index b0ac5e9af9..44aba5d113 100644 --- a/iceberg/core/manifest_entry.py +++ b/iceberg/core/manifest_entry.py @@ -18,6 +18,7 @@ from enum import Enum from iceberg.api import (DataFile, + FileFormat, Metrics, Schema) from iceberg.api.types import (IntegerType, @@ -93,7 +94,7 @@ def put(self, i, v): .field(name="partition").type, v.get("partition")) v = GenericDataFile(v.get("file_path"), - v.get("file_format"), + FileFormat[v.get("file_format")], v.get("file_size_in_bytes"), v.get("block_size_in_byte"), row_count=v.get("record_count"), @@ -110,9 +111,12 @@ def get(self, i): elif i == 2: return self.file - def __str__(self): + def __repr__(self): return "ManifestEntry(status=%s, snapshot_id=%s, file=%s" % (self.status, self.snapshot_id, self.file) + def __str__(self): + return self.__repr__() + @staticmethod def project_schema(part_type, columns): return ManifestEntry.wrap_file_schema(Schema(DataFile.get_type(part_type).fields) diff --git a/iceberg/core/manifest_reader.py b/iceberg/core/manifest_reader.py index 812681e1f8..6a4581293a 100644 --- a/iceberg/core/manifest_reader.py +++ b/iceberg/core/manifest_reader.py @@ -15,92 +15,148 @@ # specific language governing permissions and limitations # under the License. +import logging + import fastavro from iceberg.api import FileFormat, Filterable -from iceberg.api.expressions import Expressions +from iceberg.api.expressions import Expressions, inclusive from iceberg.api.io import CloseableGroup from .avro import AvroToIceberg +from .filtered_manifest import FilteredManifest from .manifest_entry import ManifestEntry, Status from .partition_spec_parser import PartitionSpecParser from .schema_parser import SchemaParser from .table_metadata import TableMetadata +_logger = logging.getLogger(__name__) + class ManifestReader(CloseableGroup, Filterable): ALL_COLUMNS = ["*"] CHANGE_COLUMNS = ["file_path", "file_format", "partition", "record_count", "file_size_in_bytes"] @staticmethod - def read(file): - return ManifestReader(file=file) + def read(file, spec_lookup=None): + return ManifestReader(file=file, spec_lookup=spec_lookup) def select(self, columns): - raise NotImplementedError() + return FilteredManifest(self, + Expressions.always_true(), + Expressions.always_true(), + list(columns), + self.case_sensitive) def filter_partitions(self, expr): - raise NotImplementedError() + return FilteredManifest(self, + expr, + Expressions.always_true(), + ManifestReader.ALL_COLUMNS, + self.case_sensitive) def filter_rows(self, expr): - raise NotImplementedError() + return FilteredManifest(self, + inclusive(self.spec).project(expr), + expr, + ManifestReader.ALL_COLUMNS, + self.case_sensitive) @staticmethod def in_memory(spec, entries): return ManifestReader(spec=spec, entries=entries) - def __init__(self, file=None, spec=None, entries=None): + def __init__(self, file=None, spec=None, metadata=None, schema=None, case_sensitive=True, spec_lookup=None): self.file = file - self.schema = None - self.metadata = None + self.schema = schema + self.metadata = metadata self.spec = spec - self._entries = entries - if self.file is not None: - self.__init_from_file() - else: - self.__init_from_spec() - - def __init_from_file(self): - try: - with self.file.new_fo() as fo: - avro_reader = fastavro.reader(fo) - self.metadata = avro_reader.metadata - except Exception as e: - raise e - - self.schema = SchemaParser.from_json(self.metadata.get("schema")) + self._case_sensitive = case_sensitive + + self._entries = None + self._avro_rows = list() + self._fo = None + self._avro_reader = None + self._avro_rows = None + + if not all([item is not None for item in [self.file, self.metadata, self.spec, self.schema]]): + if self.spec is not None: + self.__init_from_spec() + else: + self.__init_from_file(spec_lookup) + + self._adds = None + self._deletes = None + + def __init_from_file(self, spec_lookup): + self._fo = self.file.new_fo() + self._avro_reader = fastavro.reader(self._fo) + self.metadata = self._avro_reader.metadata spec_id = int(self.metadata.get("partition-spec-id", TableMetadata.INITIAL_SPEC_ID)) - self.spec = PartitionSpecParser.from_json_fields(self.schema, spec_id, self.metadata.get("partition-spec")) + + if spec_lookup is not None: + self.spec = spec_lookup(spec_id) + self.schema = self.spec.schema + else: + self.schema = SchemaParser.from_json(self.metadata.get("schema")) + self.spec = PartitionSpecParser.from_json_fields(self.schema, spec_id, self.metadata.get("partition-spec")) def __init_from_spec(self): self.metadata = dict() self.schema = self.spec.schema + def case_sensitive(self, case_sensitive): + return ManifestReader(file=self.file, metadata=self.metadata, spec=self.spec, + schema=self.schema, case_sensitive=case_sensitive) + + def cache_changes(self): + adds = list() + deletes = list() + for entry in self.entries(ManifestReader.CHANGE_COLUMNS): + if entry.status == "ADDED": + adds.append(entry.copy()) + elif entry.status == "DELETED": + deletes.append(entry.copy()) + + self._adds = adds + self._deletes = deletes + + def added_files(self): + if self._adds is None: + self.cache_changes() + + return self._adds + + def deleted_files(self): + if self._adds is None: + self.cache_changes() + + return self._deletes + def entries(self, columns=None): if columns is None: columns = ManifestReader.ALL_COLUMNS - format = FileFormat.from_file_name(self.file.location()) - if format is None: - raise RuntimeError("Unable to determine format of manifest: " + self.file) + file_format = FileFormat.from_file_name(self.file.location()) + if file_format is None: + raise RuntimeError("Unable to determine format of manifest: %s" % self.file) proj_schema = ManifestEntry.project_schema(self.spec.partition_type(), columns) - read_entries = list() - if format == FileFormat.AVRO: - with self.file.new_fo() as fo: - avro_reader = fastavro.reader(fo) - for read_entry in AvroToIceberg.read_avro_row(proj_schema, avro_reader): + if self._entries is None: + if file_format is FileFormat.AVRO: + self._entries = list() + for read_entry in AvroToIceberg.read_avro_row(proj_schema, self._avro_reader): entry = ManifestEntry(schema=proj_schema, partition_type=self.spec.partition_type()) for i, key in enumerate(read_entry.keys()): entry.put(i, read_entry[key]) - read_entries.append(entry) - else: - raise RuntimeError("Invalid format for manifest file: " + format) + self._entries.append(entry) + self._fo.close() + self._avro_reader = None - return read_entries + return self._entries def iterator(self, part_filter=None, columns=None): if part_filter is None and columns is None: return self.iterator(Expressions.always_true(), Filterable.ALL_COLUMNS) - return (entry.file for entry in self.entries if entry.status != Status.DELETED) + return [entry.file for entry in self.entries() if entry.status != Status.DELETED] diff --git a/iceberg/core/partition_data.py b/iceberg/core/partition_data.py index 87f98895b7..d74f34823d 100644 --- a/iceberg/core/partition_data.py +++ b/iceberg/core/partition_data.py @@ -32,7 +32,7 @@ def __init__(self, schema=None, partition_type=None): raise RuntimeError("Partitions cannot contain nested types: %s" % field.type) self.partition_type = partition_type # schema = PartitionData.get_schema(self.partition_type) - self.size = len(self.partition_type.fields) + self._size = len(self.partition_type.fields) self.data = list() self.schema = schema self.string_schema = str(schema) @@ -68,15 +68,18 @@ def __str__(self): return "PartitionData{%s}" % (",".join(["{}={}".format(self.partition_type.fields[i], datum) for i, datum in enumerate(self.data)])) + def __len__(self): + return self._size + def put(self, i, v): self.data.insert(i, v) def get(self, pos): - if pos > len(self.data): + try: + return self.data[pos][1] + except IndexError: return None - return self.data[pos][1] - @staticmethod def from_json(schema, json_obj): if isinstance(json_obj, str): diff --git a/iceberg/core/partition_summary.py b/iceberg/core/partition_summary.py index d7fbdb6324..8b7ec3bdd4 100644 --- a/iceberg/core/partition_summary.py +++ b/iceberg/core/partition_summary.py @@ -15,11 +15,29 @@ # specific language governing permissions and limitations # under the License. +from iceberg.api.types.conversions import Conversions + +from .generic_partition_field_summary import GenericPartitionFieldSummary + class PartitionSummary(object): def __init__(self, spec): - raise NotImplementedError() + self.java_classes = spec.java_classes + self.fields = [PartitionFieldStats(spec.partition_type.fields[i].type) + for i in range(len(self.java_classes))] + + def summaries(self): + return [field.to_summary() for field in self.fields] + + def update(self, partition_key): + self.update_fields(partition_key) + + def update_fields(self, key): + for i in range(self.java_classes): + stats = self.fields[i] + java_class = self.java_classes[i] + stats.update(key.get(i, java_class)) class PartitionFieldStats(object): @@ -29,3 +47,22 @@ def __init__(self, type): self.type = type self.min = None self.max = None + + def to_summary(self): + lower_bound = None if self.min is None else Conversions.to_byte_buffer(self.type, self.min) + upper_bound = None if self.max is None else Conversions.to_byte_buffer(self.type, self.max) + return GenericPartitionFieldSummary(contains_null=self.contains_null, + lower_bound=lower_bound, + uppwer_bound=upper_bound) + + def update(self, value): + if value is None: + self.contains_null = True + elif self.min is None: + self.min = value + self.max = value + else: + if value < self.min: + self.min = value + if value > self.max: + self.max = value diff --git a/iceberg/core/scan_summary.py b/iceberg/core/scan_summary.py new file mode 100644 index 0000000000..f7e5e15bb2 --- /dev/null +++ b/iceberg/core/scan_summary.py @@ -0,0 +1,409 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +from copy import deepcopy +import functools + +from iceberg.api import DataOperations +from iceberg.api.expressions import Expressions, Literal, Operation, UnboundPredicate +from iceberg.api.types import TimestampType + +from .manifest_group import ManifestGroup +from .util import str_as_bool + +TIMESTAMP_RANGE_MAP = {Operation.LT: lambda min_val, max_val, val: (min_val, val - 1 if val - 1 < max_val else max_val), + Operation.LT_EQ: lambda min_val, max_val, val: (min_val, val if val < max_val else max_val), + Operation.GT: lambda min_val, max_val, val: (val - 1 if val - 1 > min_val else min_val, max_val), + Operation.GT_EQ: lambda min_val, max_val, val: (val if val > min_val else min_val, max_val), + Operation.EQ: lambda min_val, max_val, val: (val if val > min_val else min_val, + val if val < max_val else max_val)} + + +class ScanSummary(object): + + IGNORED_OPERATIONS = {DataOperations.DELETE, DataOperations.REPLACE} + SCAN_SUMMARY_COLUMNS = ["partition", "record_count", "file_size_in_bytes"] + + +class ScanSummaryBuilder(object): + + TIMESTAMP_NAMES = {"dateCreated", "lastUpdated"} + + def __init__(self, scan): + self.scan = scan + self.table = scan.table + self.ops = self.table.ops + self.snapshot_timestamps = {snap.snapshot_id: snap.timestamp_millis + for snap in self.table.snapshots()} + self._throw_if_limited = False + self.force_use_manifests = False + self._limit = 2**31 - 1 + + self.time_filters = list() + + def add_timestamp_filter(self, filter): + self.throw_if_limited() + self.time_filters.append(filter) + return self + + def after(self, timestamp): + self.add_timestamp_expression(timestamp, Expressions.greater_than_or_equal) + return self + + def before(self, timestamp): + self.add_timestamp_expression(timestamp, Expressions.less_than_or_equal) + return self + + def add_timestamp_expression(self, timestamp, expr_func): + if isinstance(timestamp, str): + timestamp = Literal.of(timestamp).to(TimestampType.without_timezone()).value / 1000 + + self.add_timestamp_filter(expr_func("timestamp_ms", timestamp)) + return self + + def throw_if_limited(self): + self._throw_if_limited = True + return self + + def limit(self, num_partitions): + self._limit = num_partitions + return self + + def use_manifests(self): + self.force_use_manifests = True + return self + + def remove_time_filters(self, expressions, expression): + if expression.op == Operation.AND: + self.remove_time_filters(expressions, expression.left) + self.remove_time_filters(expressions, expression.right) + return + elif isinstance(expression, UnboundPredicate): + pred = expression + ref = pred.ref + lit = pred.lit + + if ref.name in ScanSummaryBuilder.TIMESTAMP_NAMES: + ts_literal = lit.to(TimestampType.without_timezone()) + millis = ScanSummaryBuilder.to_millis(ts_literal.value) + self.add_timestamp_filter(Expressions.predicate(pred.op, "timestamp_ms", millis)) + return + + expressions.append(expression) + + def build(self): + if self.table.current_snapshot() is None: + return dict() + + filters = list() + self.remove_time_filters(filters, Expressions.rewrite_not(self.scan.row_filter)) + row_filter = self.join_filters(filters) + + if len(self.time_filters) == 0: + return self.from_manifest_scan(self.table.current_snapshot().manifests, row_filter) + + min_timestamp, max_timestamp = self.timestamp_range(self.time_filters) + oldest_snapshot = self.table.current_snapshot() + for key, val in self.snapshot_timestamps.items(): + if val < oldest_snapshot.timestamp_millis: + oldest_snapshot = self.ops.current().snapshot(key) + + # if oldest known snapshot is in the range, then there may be an expired snapshot that has + # been removed that matched the range. because the timestamp of that snapshot is unknown, + # it can't be included in the results and the results are not reliable.""" + if oldest_snapshot.timestamp_millis >= min_timestamp and oldest_snapshot <= max_timestamp: + raise RuntimeError("Cannot satisfy time filters: time range may include expired snapshots") + + snapshots = [snapshot for snapshot in ScanSummaryBuilder.snapshots_in_time_range(self.ops.current(), + min_timestamp, + max_timestamp) + if snapshot.operation not in ScanSummary.IGNORED_OPERATIONS] + + result = self.from_partition_summaries(snapshots) + if result is not None and not self.force_use_manifests: + return result + + # filter down to the the set of manifest files that were created in the time range, ignoring + # the snapshots created by delete or replace operations. this is complete because it finds + # files in the snapshot where they were added to the dataset in either an append or an + # overwrite. if those files are later compacted with a replace or deleted, those changes are + # ignored. + manifests_to_scan = list() + snapshot_ids = set() + + for snap in snapshots: + snapshot_ids.add(snap) + for manifest in snap.manifests: + if manifest.snapshot_id is not None or manifest.snapshot_id == snap.snapshot_id: + manifests_to_scan.append(manifest) + + return self.from_manifest_scan(manifests_to_scan, row_filter, True) + + def from_manifest_scan(self, manifests, row_filter, ignore_existing=False): + top_n = TopN(self._limit, self._throw_if_limited, lambda x, y: 0 if x == y else -1 if x < y else 1) + + entries = (ManifestGroup(self.ops, manifests) + .filter_data(row_filter) + .ignore_deleted() + .ignore_existing(ignore_existing) + .select(ScanSummary.SCAN_SUMMARY_COLUMNS) + .entries()) + + spec = self.table.spec() + for entry in entries: + timestamp = self.snapshot_timestamps.get(entry.snapshot_id) + partition = spec.partition_to_path(entry.file.partition()) + top_n.update(partition, + lambda metrics: ((metrics if metrics is not None else PartitionMetrics()) + .update_from_file(entry.file, timestamp))) + + return top_n.get() + + def from_partition_summaries(self, snapshots): + # try to build the result from snapshot metadata, but fall back if: + # any snapshot has no summary + # any snapshot has + top_n = TopN(self._limit, self._throw_if_limited, lambda x, y: 0 if x == y else -1 if x < y else 1) + + for snap in snapshots: + if snap.operation is None or snap.summary is None \ + or str_as_bool(snap.summary.get(SnapshotSummary.PARTITION_SUMMARY_PROP, "false")): + return None + + for key, val in snap.summary.items(): + if key.startswith(SnapshotSummary.CHANGED_PARTITION_PREFIX): + part_key = key[len(SnapshotSummary.CHANGED_PARTITION_PREFIX):] + # part = dict(entry.split("=") for entry in val.split(",")) + # UPDATE THIS BEFORE FINISHING + added_files = 0 + added_records = 0 + added_size = 0 + top_n.update(part_key, + lambda metrics: ((PartitionMetrics() if metrics is None else metrics) + .update_from_counts(added_files, + added_records, + added_size, + snap.timestamp_millis))) + + return top_n.get() + + @staticmethod + def snapshots_in_time_range(meta, min_ts, max_ts): + snapshots = [] + current = meta.current_snapshot() + while current is not None and current.timestamp_millis >= min_ts: + current = meta.snapshot(current.parent_id) + + if current.timestamp_millis <= max_ts: + snapshots.add(current) + + snapshots.reverse() + return snapshots + + @staticmethod + def timestamp_range(time_filters): + min_timestamp = float('-inf') + max_timestamp = float('inf') + + for pred in time_filters: + value = pred.lit.val + try: + min_timestamp, max_timestamp = TIMESTAMP_RANGE_MAP[pred.op](min_timestamp, max_timestamp, value) + except KeyError: + raise RuntimeError("Cannot filter timestamps using predicate: %s" % pred) + + if max_timestamp < min_timestamp: + raise RuntimeError("No timestamps can match filters: %s" % ", ".join([str(pred) + for pred in time_filters])) + + return min_timestamp, max_timestamp + + @staticmethod + def join_filters(expressions): + result = Expressions.always_true() + for expression in expressions: + result = Expressions.and_(result, expression) + + return result + + @staticmethod + def to_millis(timestamp): + if timestamp < 10000000000: + # in seconds + return timestamp * 1000 + elif timestamp < 10000000000000: + # in millis + return timestamp + + # in micros + return timestamp / 1000 + + +class TopN(object): + + def __init__(self, N, throw_if_limited, key_comparator): + self.max_size = N + self.throw_if_limited = throw_if_limited + self.map = dict() + self.key_comparator = key_comparator + self.cut = None + + def update(self, key, update_func): + if self.cut is not None and self.key_comparator(self.cut, key) <= 0: + return + + self.map[key] = update_func(self.map.get(key)) + + while len(map.keys()) > self.max_size: + if self.throw_if_limited: + raise RuntimeError("Too many matching keys: more than %s" % self.max_size) + + self.cut = sorted(self.map, key=functools.cmp_to_key(self.key_comparator))[-1] + del self.map[self.cut] + + def get(self): + return deepcopy(self.map) + + +class PartitionMetrics(object): + + def __init__(self): + self.file_count = 0 + self.record_count = 0 + self.total_size = 0 + self.data_timestamp_millis = None + + def update_from_counts(self, file_count, record_count, files_size, timestamp_millis): + self.file_count += file_count + self.record_count += record_count + self.total_size += files_size + + if self.data_timestamp_millis is None or self.data_timestamp_millis < timestamp_millis: + self.data_timestamp_millis = timestamp_millis + + return self + + def update_from_file(self, file, timestamp_millis): + self.file_count += 1 + self.record_count += file.record_count() + self.total_size += file.files_size_in_bytes() + + if self.data_timestamp_millis is None or self.data_timestamp_millis < timestamp_millis: + self.data_timestamp_millis = timestamp_millis + + return self + + def __repr__(self): + items = ("%s=%r" % (k, v) for k, v in self.__dict__.items()) + return "%s(%s)" % (self.__class__.__name__, ','.join(items)) + + def __str__(self): + return self.__repr__() + + +class SnapshotSummary(object): + GENIE_ID_PROP = "genie-id" + ADDED_FILES_PROP = "added-data-files" + DELETED_FILES_PROP = "deleted-data-files" + TOTAL_FILES_PROP = "total-data-files" + ADDED_RECORDS_PROP = "added-records" + DELETED_RECORDS_PROP = "deleted-records" + TOTAL_RECORDS_PROP = "total-records" + ADDED_FILE_SIZE_PROP = "added-files-size" + DELETED_DUPLICATE_FILES = "deleted-duplicate-files" + CHANGED_PARTITION_COUNT_PROP = "changed-partition-count" + CHANGED_PARTITION_PREFIX = "partitions." + PARTITION_SUMMARY_PROP = "partition-summaries-included" + + def __init__(self): + pass + + +class SnapshotSummaryBuilder(object): + + def __init__(self): + self.changed_partitions = dict() + self.added_files = 0 + self.deleted_files = 0 + self.deleted_dupicate_files = 0 + self.added_records = 0 + self.deleted_records = 0 + self.properties = dict() + + def clear(self): + self.changed_partitions = dict() + self.added_files = 0 + self.deleted_files = 0 + self.deleted_dupicate_files = 0 + self.added_records = 0 + self.deleted_records = 0 + + def increment_duplicate_deletes(self): + self.deleted_dupicate_files += 1 + + def deleted_file(self, spec, data_file): + self.update_partitiomns(spec, data_file, False) + self.deleted_files += 1 + self.deleted_records += data_file.record_count + + def added_file(self, spec, data_file): + self.update_partitiomns(spec, data_file, True) + self.added_files += 1 + self.added_records += data_file.record_count + + def update_partitions(self, spec, file, is_addition): + key = spec.partition_to_path(file.partition()) + metrics = self.changed_partitions.get(key, PartitionMetrics()) + + if is_addition: + self.changed_partitions[key] = metrics.update_from_file(file, None) + + def build(self): + builder = dict() + builder.update(self.properties) + + SnapshotSummaryBuilder.set_if(self.added_files > 0, builder, + SnapshotSummary.ADDED_FILES_PROP, self.added_files) + SnapshotSummaryBuilder.set_if(self.deleted_files > 0, + builder, SnapshotSummary.DELETED_FILES_PROP, self.deleted_files) + SnapshotSummaryBuilder.set_if(self.deleted_dupicate_files > 0, + builder, SnapshotSummary.DELETED_DUPLICATE_FILES, self.deleted_dupicate_files) + SnapshotSummaryBuilder.set_if(self.added_records > 0, + builder, SnapshotSummary.ADDED_RECORDS_PROP, self.added_records) + SnapshotSummaryBuilder.set_if(self.deleted_records > 0, + builder, SnapshotSummary.DELETED_RECORDS_PROP, self.deleted_records) + builder[SnapshotSummary.CHANGED_PARTITION_COUNT_PROP] = len(self.changed_partitions.items()) + + if len(self.changed_partitions.items()) < 100: + builder[SnapshotSummary.PARTITION_SUMMARY_PROP] = "true" + for key, metrics in self.changed_partitions: + metric_dict = {SnapshotSummary.ADDED_FILES_PROP: metrics.file_count, + SnapshotSummary.ADDED_RECORDS_PROP: metrics.record_count, + SnapshotSummary.ADDED_FILE_SIZE_PROP: metrics.total_size} + builder[SnapshotSummary.CHANGED_PARTITION_PREFIX + key] = ",".join(["{}={}".format(key, val) + for inner_key, val + in metric_dict.items()]) + + return builder + + def set(self, prop, value): + self.properties[prop] = value + + @staticmethod + def set_if(expression, builder, prop, value): + if expression: + builder[prop] = value diff --git a/iceberg/core/schema_parser.py b/iceberg/core/schema_parser.py index 0ab4fe8f25..3227b49a8a 100644 --- a/iceberg/core/schema_parser.py +++ b/iceberg/core/schema_parser.py @@ -37,6 +37,7 @@ class SchemaParser(object): ELEMENT = "element" KEY = "key" VALUE = "value" + DOC = "doc" NAME = "name" ID = "id" ELEMENT_ID = "element-id" @@ -85,10 +86,14 @@ def _struct_to_dict(struct): @staticmethod def _struct_field_to_dict(field): - return {SchemaParser.ID: field.field_id, - SchemaParser.NAME: field.name, - SchemaParser.REQUIRED: field.is_required, - SchemaParser.TYPE: SchemaParser._type_to_dict(field.type)} + schema = {SchemaParser.ID: field.field_id, + SchemaParser.NAME: field.name, + SchemaParser.REQUIRED: field.is_required, + SchemaParser.TYPE: SchemaParser._type_to_dict(field.type)} + if field.doc is not None: + schema[SchemaParser.DOC] = field.doc + + return schema @staticmethod def _list_to_dict(list_type): @@ -129,11 +134,12 @@ def struct_from_dict(dict_obj): field_id = field.get(SchemaParser.ID) field_name = field.get(SchemaParser.NAME) field_type = SchemaParser.type_from_dict(field.get(SchemaParser.TYPE)) + field_doc = field.get(SchemaParser.DOC) if field.get(SchemaParser.REQUIRED): - struct_fields.append(NestedField.required(field_id, field_name, field_type)) + struct_fields.append(NestedField.required(field_id, field_name, field_type, field_doc)) else: - struct_fields.append(NestedField.optional(field_id, field_name, field_type)) + struct_fields.append(NestedField.optional(field_id, field_name, field_type, field_doc)) return StructType.of(struct_fields) diff --git a/iceberg/core/snapshot_parser.py b/iceberg/core/snapshot_parser.py index a1b7c96f11..3e94af5c5d 100644 --- a/iceberg/core/snapshot_parser.py +++ b/iceberg/core/snapshot_parser.py @@ -18,7 +18,6 @@ import json from .base_snapshot import BaseSnapshot -from .generic_manifest_file import GenericManifestFile class SnapshotParser(object): @@ -39,10 +38,12 @@ def to_dict(snapshot): return {SnapshotParser.SNAPSHOT_ID: snapshot.snapshot_id, SnapshotParser.TIMESTAMP_MS: snapshot.timestamp_millis, SnapshotParser.PARENT_SNAPSHOT_ID: snapshot.parent_id, + SnapshotParser.OPERATION: snapshot.operation, + SnapshotParser.SUMMARY: snapshot.summary, SnapshotParser.MANIFESTS: [manifest.manifest_path for manifest in snapshot.manifests]} @staticmethod - def from_json(ops, json_obj): + def from_json(ops, json_obj, spec=None): if isinstance(json_obj, str): json_obj = json.loads(json_obj) @@ -63,8 +64,8 @@ def from_json(ops, json_obj): operation=operation, summary=summary) else: - manifests = [GenericManifestFile.generic_manifest_from_file(ops.new_input_file(location), 0) # noqa: F841 - for location in json_obj.get(SnapshotParser.MANIFESTS, list())] + manifests = json_obj.get(SnapshotParser.MANIFESTS, list()) + return BaseSnapshot(ops, version_id, parent_id, manifests=manifests, timestamp_millis=timestamp_millis, diff --git a/iceberg/core/table_metadata.py b/iceberg/core/table_metadata.py index b816f90a50..b289e440a8 100644 --- a/iceberg/core/table_metadata.py +++ b/iceberg/core/table_metadata.py @@ -28,22 +28,24 @@ class TableMetadata(object): TABLE_FORMAT_VERSION = 1 @staticmethod - def new_table_metadata(ops, schema, spec, location): + def new_table_metadata(ops, schema, spec, location, properties=None): last_column_id = AtomicInteger(0) fresh_schema = assign_fresh_ids(schema, last_column_id.increment_and_get) spec_builder = PartitionSpec.builder_for(fresh_schema) for field in spec.fields: src_name = schema.find_column_name(field.source_id) - spec_builder.add(fresh_schema.find_field(src_name), - field, - str(field.fransform())) + spec_builder.add(fresh_schema.find_field(src_name).field_id, + field.name, + str(field.transform)) fresh_spec = spec_builder.build() + properties = properties if properties is not None else dict() + return TableMetadata(ops, None, location, int(time.time() * 1000), last_column_id.get(), fresh_schema, TableMetadata.INITIAL_SPEC_ID, [fresh_spec], - dict(), -1, list(), list()) + properties, -1, list(), list()) def __init__(self, ops, file, location, last_updated_millis, last_column_id, schema, default_spec_id, specs, properties, @@ -57,6 +59,7 @@ def __init__(self, ops, file, location, last_updated_millis, self.default_spec_id = default_spec_id self.specs = specs self.properties = properties + self.properties["provider"] = "ICEBERG" self.current_snapshot_id = current_snapshot_id self.snapshots = snapshots self.snapshot_log = snapshot_log @@ -112,6 +115,20 @@ def add_snapshot(self, snapshot): int(time.time() * 1000), self.last_column_id, self.schema, self.spec, self.properties, self.current_snapshot_id, new_snapshots, new_snapshot_log) + def add_staged_snapshot(self, snapshot): + self.snapshots.append(snapshot) + return TableMetadata(self.ops, None, self.location, snapshot.timestamp_millis, + self.last_column_id, self.schema, self.default_spec_id, self.specs, + self.current_snapshot_id, self.snapshots, self.snapshot_log) + + def replace_current_snapshot(self, snapshot): + self.snapshots.append(snapshot) + self.snapshot_log.append(SnapshotLogEntry(snapshot.timestamp_millis, snapshot.snapshot_id)) + + return TableMetadata(self.ops, None, self.location, snapshot.timestamp_millis, + self.last_column_id, self.schema, self.default_spec_id, self.specs, + self.current_snapshot_id, self.snapshots, self.snapshot_log) + def remove_snapshots_if(self, remove_if): filtered = list() diff --git a/iceberg/core/table_metadata_parser.py b/iceberg/core/table_metadata_parser.py index 1f364c5d6a..d7def842a4 100644 --- a/iceberg/core/table_metadata_parser.py +++ b/iceberg/core/table_metadata_parser.py @@ -43,7 +43,7 @@ class TableMetadataParser(object): LOG = "snapshot-log" @staticmethod - def to_json(metadata): + def to_json(metadata, indent=4): return json.dumps({TableMetadataParser.FORMAT_VERSION: TableMetadata.TABLE_FORMAT_VERSION, TableMetadataParser.LOCATION: metadata.location, TableMetadataParser.LAST_UPDATED_MILLIS: metadata.last_updated_millis, @@ -61,17 +61,17 @@ def to_json(metadata): for snapshot in metadata.snapshots], TableMetadataParser.LOG: [{TableMetadataParser.TIMESTAMP_MS: log_entry.timestamp_millis, TableMetadataParser.SNAPSHOT_ID: log_entry.snapshot_id} - for log_entry in metadata.snapshot_log]}) + for log_entry in metadata.snapshot_log]}, indent=indent) @staticmethod def write(metadata, metadata_location): if metadata_location.location().endswith(".gz"): - output_file = gzip.open(metadata_location.location(), "wt") + output_file = gzip.open(metadata_location.create("wb"), "wb") else: - output_file = open(metadata_location.location(), "w") + output_file = metadata_location.create("wb") json_str = TableMetadataParser.to_json(metadata) - output_file.write(json_str) + output_file.write(json_str.encode("utf-8")) output_file.close() @staticmethod diff --git a/iceberg/core/util/__init__.py b/iceberg/core/util/__init__.py index ba17648735..1c8b486506 100644 --- a/iceberg/core/util/__init__.py +++ b/iceberg/core/util/__init__.py @@ -15,11 +15,26 @@ # specific language governing permissions and limitations # under the License. -__all__ = ["AtomicInteger", "PackingIterator", "str_as_bool"] + +__all__ = ["AtomicInteger", + "METAFLOW_ENABLED", + "METAFLOW_ROOTDIR", + "PackingIterator", + "PLANNER_THREAD_POOL_SIZE_PROP", + "SCAN_THREAD_POOL_ENABLED", + "str_as_bool", + "WORKER_THREAD_POOL_SIZE_PROP", + ] from .atomic_integer import AtomicInteger from .bin_packing import PackingIterator +PLANNER_THREAD_POOL_SIZE_PROP = "iceberg.planner.num-threads" +WORKER_THREAD_POOL_SIZE_PROP = "iceberg.worker.num-threads" +SCAN_THREAD_POOL_ENABLED = "iceberg.scan.plan-in-worker-pool" +METAFLOW_ENABLED = "iceberg.s3.use-metaflow" +METAFLOW_ROOTDIR = "iceberg.s3.metaflow-root-dir" + def str_as_bool(str_var): return str_var is not None and str_var.lower() == "true" diff --git a/setup.py b/setup.py index 4ee888b9fe..482740581d 100755 --- a/setup.py +++ b/setup.py @@ -29,12 +29,16 @@ install_requires=['botocore', 'boto3', 'fastavro', + 'fastparquet>=0.3.1', 'mmh3', 'moz_sql_parser', 'pyparsing', 'python-dateutil', 'pytz', 'requests', - 'retrying'], + 'retrying', + 'pandas', + 'pyarrow' + ], setup_requires=['setupmeta'] ) diff --git a/tests/api/expressions/test_inclusive_manifest_evaluator.py b/tests/api/expressions/test_inclusive_manifest_evaluator.py index 165ed2a75c..eaaa93fc16 100644 --- a/tests/api/expressions/test_inclusive_manifest_evaluator.py +++ b/tests/api/expressions/test_inclusive_manifest_evaluator.py @@ -1,3 +1,19 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. from iceberg.api.expressions import Expressions, InclusiveManifestEvaluator from iceberg.exceptions import ValidationException diff --git a/tests/api/test_file_format.py b/tests/api/test_file_format.py new file mode 100644 index 0000000000..b83d7b1205 --- /dev/null +++ b/tests/api/test_file_format.py @@ -0,0 +1,29 @@ + +from iceberg.api import FileFormat + + +def test_parquet(): + file_fmt = FileFormat.PARQUET + file_name = "test_file.parquet" + add_extension_file = "test_file" + assert file_fmt.is_splittable() + assert FileFormat.from_file_name(file_name) == FileFormat.PARQUET + assert file_name == FileFormat.PARQUET.add_extension(add_extension_file) + + +def test_avro(): + file_fmt = FileFormat.AVRO + file_name = "test_file.avro" + add_extension_file = "test_file" + assert file_fmt.is_splittable() + assert FileFormat.from_file_name(file_name) == FileFormat.AVRO + assert file_name == FileFormat.AVRO.add_extension(add_extension_file) + + +def test_orc(): + file_fmt = FileFormat.ORC + file_name = "test_file.orc" + add_extension_file = "test_file" + assert file_fmt.is_splittable() + assert FileFormat.from_file_name(file_name) == FileFormat.ORC + assert file_name == FileFormat.ORC.add_extension(add_extension_file) diff --git a/tests/core/avro/conftest.py b/tests/core/avro/conftest.py new file mode 100644 index 0000000000..0e64646938 --- /dev/null +++ b/tests/core/avro/conftest.py @@ -0,0 +1,49 @@ +from tempfile import NamedTemporaryFile + +from iceberg.api import Schema +from iceberg.api.types import (BinaryType, + BooleanType, + DateType, + DecimalType, + DoubleType, + FixedType, + FloatType, + IntegerType, + LongType, + NestedField, + StringType, + StructType, + TimestampType, + UUIDType) +import pytest + + +@pytest.fixture(scope="session") +def supported_primitives(): + return StructType.of([NestedField.required(100, "id", LongType.get()), + NestedField.optional(101, "data", StringType.get()), + NestedField.required(102, "b", BooleanType.get()), + NestedField.optional(103, "i", IntegerType.get()), + NestedField.required(104, "l", LongType.get()), + NestedField.optional(105, "f", FloatType.get()), + NestedField.required(106, "d", DoubleType.get()), + NestedField.optional(107, "date", DateType.get()), + NestedField.required(108, "ts", TimestampType.with_timezone()), + NestedField.required(110, "s", StringType.get()), + NestedField.required(111, "uuid", UUIDType.get()), + NestedField.required(112, "fixed", FixedType.of_length(7)), + NestedField.optional(113, "bytes", BinaryType.get()), + NestedField.required(114, "dec_9_0", DecimalType.of(9, 0)), + NestedField.required(114, "dec_11_2", DecimalType.of(11, 2)), + NestedField.required(114, "dec_38_10", DecimalType.of(38, 10))]) + + +@pytest.fixture(scope="session") +def iceberg_full_read_projection_schema(): + return Schema([NestedField.required(0, "id", LongType.get()), + NestedField.optional(1, "data", StringType.get())]) + + +def write_and_read(desc, write_schema, read_schema, record): + with NamedTemporaryFile(delete=True, mode='wb') as temp_file: + return temp_file diff --git a/tests/core/avro/test_read_projection.py b/tests/core/avro/test_read_projection.py new file mode 100644 index 0000000000..f6aff79c46 --- /dev/null +++ b/tests/core/avro/test_read_projection.py @@ -0,0 +1,12 @@ +# from .conftest import write_and_read +from iceberg.core import ManifestEntry, PartitionSpecParser, SchemaParser +from iceberg.core.avro import IcebergToAvro + + +def test_full_projection(iceberg_full_read_projection_schema): + schema = SchemaParser.from_json({'type': 'struct', 'fields': [{'id': 1, 'name': 'account_id', 'required': False, 'type': 'long', 'doc': 'Lookup table: account_d'}, {'id': 2, 'name': 'subscrn_id', 'required': False, 'type': 'long'}, {'id': 3, 'name': 'is_in_free_trial', 'required': False, 'type': 'int'}, {'id': 4, 'name': 'subscrn_service_days', 'required': False, 'type': 'int'}, {'id': 5, 'name': 'subscrn_period_nbr', 'required': False, 'type': 'int'}, {'id': 6, 'name': 'account_service_days', 'required': False, 'type': 'int', 'doc': 'number of days subscriber has had service over all subscriptions'}, {'id': 7, 'name': 'account_period_nbr', 'required': False, 'type': 'int'}, {'id': 8, 'name': 'current_plan_id', 'required': False, 'type': 'int', 'doc': 'Lookup table: plan_d'}, {'id': 10, 'name': 'country_iso_code', 'required': False, 'type': 'string', 'doc': 'Registration country id. Lookup table: geo_country_d'}, {'id': 11, 'name': 'is_onhold', 'required': False, 'type': 'int', 'doc': 'Subscriptions can be put on hold due to suspected fraud or failure to pay. subscriber is no longer able to view and is no longer considered a member while on hold'}, {'id': 12, 'name': 'same_day_hold', 'required': False, 'type': 'int', 'doc': 'When a member goes on hold and off hold in the same day'}, {'id': 13, 'name': 'dateint', 'required': False, 'type': 'int', 'doc': 'Will be deprecated'}, {'id': 14, 'name': 'plan_rollup_id', 'required': False, 'type': 'int', 'doc': 'Lookup table: plan_rollup_d'}, {'id': 15, 'name': 'price_tier_code', 'required': False, 'type': 'string', 'doc': 'Lookup table: price_tier_d. You can also use plan_price_d to lookup the combination of (plan_rollup_id, price_tier_code, country_iso_code)'}, {'id': 16, 'name': 'latest_plan_change_date', 'required': False, 'type': 'int', 'doc': 'Members can upgrade or downgrade plans'}, {'id': 17, 'name': 'is_in_product_grace_period', 'required': False, 'type': 'int', 'doc': 'Members on grace period can continue to view, but they are not considered active members since they are not in good payment standing.'}, {'id': 18, 'name': 'is_in_member_cnt', 'required': False, 'type': 'int', 'doc': 'Total # of entitlements. An entitlement is the right to stream Netflix and excludes those on hold or in grace periods and can include both paid and free subs.'}, {'id': 19, 'name': 'is_grace_period_to_member_cnt', 'required': False, 'type': 'int', 'doc': 'Members that were on grace period before, but cleared out and become in good standing'}, {'id': 20, 'name': 'is_grace_period_to_on_hold', 'required': False, 'type': 'int', 'doc': 'Members that were on grace period (usually for 7 days), but they failed to provide a valid payment method so they placed on hold'}, {'id': 21, 'name': 'is_from_gp_start', 'required': False, 'type': 'int', 'doc': 'Grace period start date'}, {'id': 22, 'name': 'is_onhold_without_gp_end', 'required': False, 'type': 'int'}, {'id': 23, 'name': 'subscription_type', 'required': False, 'type': 'string', 'doc': 'Added with NIO project. Values: P (paying only), S (streaming only), PS (paying and streaming)'}, {'id': 24, 'name': 'connected_to_account_id', 'required': False, 'type': 'long', 'doc': 'Added with NIO project. This will only be populated when the paying accounts have logins. Possible values: subscription_type = P, connected_to_account_id = value or null, subscription_type = S, connected_to_account_id = null, subscription_type = PS, connected_to_account_id = null'}, {'id': 25, 'name': 'can_stream', 'required': False, 'type': 'int', 'doc': 'Added with NIO project. Ability to stream whether a member is in good standing or on grace period. Possible values: subscription_type = P, can_stream = 0, subscription_type = S, can_stream = 1, subscription_type = PS, can_stream = 0 or 1 based hold and grace period'}, {'id': 26, 'name': 'is_untethered_account', 'required': False, 'type': 'int', 'doc': 'Added with NIO project. This column is used to identify paying accounts that are not associated with logins. Logic used to populate it: (subscription_type = P, can_stream=0, is_in_member_cnt =1, connected_to_account_id = null) Possible values: subscription_type = P, is_untethered_account = 0 or 1, subscription_type = S, is_untethered_account = 0, subscription_type = PS, is_untethered_account = 0'}, {'id': 27, 'name': 'is_billing_paused', 'required': False, 'type': 'int', 'doc': 'Added with NIO project. Possible values: subscription_type = P, is_billing_paused = 0, subscription_type = S, is_billing_paused = 0 or 1, subscription_type = PS, is_billing_paused = 0.'}, {'id': 28, 'name': 'is_in_customer_count', 'required': False, 'type': 'int'}, {'id': 29, 'name': 'paid_category', 'required': False, 'type': 'string'}, {'id': 30, 'name': 'is_in_paid_member_cnt', 'required': False, 'type': 'int'}, {'id': 31, 'name': 'scale', 'required': False, 'type': 'string'}, {'id': 32, 'name': 'test_new_col', 'required': False, 'type': 'string', 'doc': 'new_column docs'}, {'id': 33, 'name': 'test_new_col_2', 'required': False, 'type': 'string', 'doc': 'new_column docs 2'}]}) + spec = PartitionSpecParser.from_json_fields(schema, 0, [{"name": "scale", + "transform": "identity", + "source-id": 31}]) + proj_schema = ManifestEntry.project_schema(spec.partition_type(), ["*"]) + print(IcebergToAvro.type_to_schema(proj_schema.as_struct(), "manifest_entry")) diff --git a/tests/core/conftest.py b/tests/core/conftest.py index 62e7296d44..e446c63ba2 100644 --- a/tests/core/conftest.py +++ b/tests/core/conftest.py @@ -21,7 +21,7 @@ import time from iceberg.api import Files, PartitionSpec, Schema -from iceberg.api.types import BooleanType, LongType, NestedField +from iceberg.api.types import BooleanType, IntegerType, LongType, NestedField, StringType from iceberg.core import (BaseSnapshot, BaseTable, ConfigProperties, @@ -29,7 +29,8 @@ SnapshotLogEntry, TableMetadata, TableMetadataParser, - TableOperations) + TableOperations, + TableProperties) from iceberg.exceptions import AlreadyExistsException, CommitFailedException import pytest @@ -113,7 +114,7 @@ def commit(self, base, metadata): raise RuntimeError("Cannot commit changes based on stale metadata") self.refresh() - if base == self.current: + if base == self.current(): if self._fail_commits > 0: self._fail_commits - 1 raise RuntimeError("Injected failure") @@ -140,6 +141,18 @@ def new_snapshot_id(self): return next_snapshot_id +class TestTables(object): + @staticmethod + def create(temp, name, schema, spec): + ops = TestTableOperations(name, temp) + + if ops.current() is not None: + raise RuntimeError("Table %s already exists at location: %s" % (name, temp)) + + ops.commit(None, TableMetadata.new_table_metadata(ops, schema, spec, str(temp))) + return TestTable(ops, name) + + @pytest.fixture(scope="session") def expected(): return TableMetadata.new_table_metadata(None, SCHEMA, PartitionSpec.unpartitioned(), "file://tmp/db/table") @@ -253,7 +266,40 @@ def missing_spec_list(): []) +@pytest.fixture(scope="session") +def snapshot_manifests(): + return (GenericManifestFile(file=Files.local_input("file:/tmp/manifest1.avro"), spec_id=0), + GenericManifestFile(file=Files.local_input("file:/tmp/manifest2.avro"), spec_id=0)) + + @pytest.fixture(scope="session") def expected_base_snapshot(): return BaseSnapshot(LocalTableOperations(), int(time.time()), manifests=["file:/tmp/manfiest.1.avro", "file:/tmp/manfiest.2.avro"]) + + +@pytest.fixture(scope="session") +def base_scan_schema(): + return Schema([NestedField.required(1, "id", IntegerType.get()), + NestedField.required(2, "data", StringType.get())]) + + +@pytest.fixture(scope="session") +def ts_table(base_scan_schema): + with tempfile.TemporaryDirectory() as td: + spec = PartitionSpec.unpartitioned() + return TestTables.create(td, "test", base_scan_schema, spec) + + +@pytest.fixture(scope="session") +def split_planning_table(base_scan_schema): + from iceberg.core.filesystem import FilesystemTables + tables = FilesystemTables() + + with tempfile.TemporaryDirectory() as td: + table = tables.create(base_scan_schema, location=td) + table.properties().update({TableProperties.SPLIT_SIZE: "{}".format(128 * 1024 * 1024), + TableProperties.SPLIT_OPEN_FILE_COST: "{}".format(4 * 1024 * 1024), + TableProperties.SPLIT_LOOKBACK: "{}".format(2 ** 31 - 1)}) + + return table diff --git a/tests/core/test_base_table_scan.py b/tests/core/test_base_table_scan.py new file mode 100644 index 0000000000..fb8d2479fc --- /dev/null +++ b/tests/core/test_base_table_scan.py @@ -0,0 +1,38 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +from iceberg.api import Schema +from iceberg.api.types import IntegerType, NestedField + + +def test_table_scan_honors_select(ts_table): + scan = ts_table.new_scan().select(["id"]) + + expected_schema = Schema([NestedField.required(1, "id", IntegerType.get())]) + + assert scan.schema.as_struct() == expected_schema.as_struct() + + +def test_table_scan_honors_select_without_case_sensitivity(ts_table): + scan1 = ts_table.new_scan().case_sensitive(False).select(["ID"]) + # order of refinements shouldn't matter + scan2 = ts_table.new_scan().select(["ID"]).case_sensitive(False) + + expected_schema = Schema([NestedField.required(1, "id", IntegerType.get())]) + + assert scan1.schema.as_struct() == expected_schema.as_struct() + assert scan2.schema.as_struct() == expected_schema.as_struct() diff --git a/tests/core/test_snapshot_json.py b/tests/core/test_snapshot_json.py index 204f4d0fd0..733f77f693 100644 --- a/tests/core/test_snapshot_json.py +++ b/tests/core/test_snapshot_json.py @@ -15,7 +15,8 @@ # specific language governing permissions and limitations # under the License. -from iceberg.core import SnapshotParser +from iceberg.api import DataOperations +from iceberg.core import BaseSnapshot, SnapshotParser def test_json_conversion(ops, expected_base_snapshot): @@ -24,3 +25,34 @@ def test_json_conversion(ops, expected_base_snapshot): assert expected_base_snapshot.snapshot_id == snapshot.snapshot_id assert expected_base_snapshot.manifests == snapshot.manifests + + +def test_json_conversion_with_operation(ops, snapshot_manifests): + parent_id = 1 + id = 2 + expected = BaseSnapshot(ops=ops, snapshot_id=id, parent_id=parent_id, + manifests=snapshot_manifests, operation=DataOperations.REPLACE, + summary={"files-added": 4, + "files-deleted": "100"}) + + json_obj = SnapshotParser.to_json(expected) + snapshot = SnapshotParser.from_json(ops, json_obj) + + assert expected.snapshot_id == snapshot.snapshot_id + assert expected.timestamp_millis == snapshot.timestamp_millis + assert expected.parent_id == snapshot.parent_id + assert expected.manifest_location == snapshot.manifest_location + assert expected.manifests == snapshot.manifests + assert expected.operation == snapshot.operation + assert expected.summary == snapshot.summary + +# def test_conversion_with_manifest_list(snapshot_manifests): +# parent_id = 1 +# id = 2 +# +# with NamedTemporaryFile() as manifest_list: +# with ManifestListWriter(Files.local_output(manifest_list), id, parent_id) as writer: +# writer.add_all(snapshot_manifests) +# +# with open(manifest_list) as fo: +# print(fo.read()) diff --git a/tests/core/utils/test_bin_packing.py b/tests/core/utils/test_bin_packing.py index 8e5ccc0c6a..81fe53101f 100644 --- a/tests/core/utils/test_bin_packing.py +++ b/tests/core/utils/test_bin_packing.py @@ -17,7 +17,7 @@ import random -from iceberg.core.util.bin_packing import PackingIterator +from iceberg.core.util import PackingIterator import pytest From 3d5828329dadae54abde98aa6a00546cba45ff35 Mon Sep 17 00:00:00 2001 From: Ryan Blue Date: Thu, 10 Oct 2019 10:27:36 -0700 Subject: [PATCH 015/642] Update build for Apache releases (#531) * Add Apache publication * Add source, javadoc, and test artifacts with signatures to publication * Add Apache snapshot and release repositories * Add metadata to publication POM files * Add missing licenses and update RAT excludes * Add DISCLAIMER * Add disclaimer page to ASF site * Add release scripts --- CHANGELOG.md | 17 +++++++++++++++++ README.md | 17 +++++++++++++++++ tests/api/expressions/test_str_to_expr.py | 15 +++++++++++++++ tests/api/test_file_format.py | 14 ++++++++++++++ tests/api/test_helpers.py | 15 +++++++++++++++ tests/core/avro/conftest.py | 15 +++++++++++++++ tests/core/avro/test_avro.py | 15 +++++++++++++++ tests/core/avro/test_read_projection.py | 15 +++++++++++++++ tests/core/utils/__init__.py | 15 +++++++++++++++ tox.ini | 15 +++++++++++++++ 10 files changed, 153 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 2ac6aeaac2..c19b3abd76 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,20 @@ + + # iceberg Changelog ## iceberg 0.0.1 (2019-02-08) diff --git a/README.md b/README.md index c4d5cca2d5..ead3fbef75 100644 --- a/README.md +++ b/README.md @@ -1,3 +1,20 @@ + + # Iceberg A python implementation of the Iceberg table format. See the project level README for more details: https://github.com/apache/incubator-iceberg diff --git a/tests/api/expressions/test_str_to_expr.py b/tests/api/expressions/test_str_to_expr.py index d44d52e4b8..dd2bb1cc3f 100644 --- a/tests/api/expressions/test_str_to_expr.py +++ b/tests/api/expressions/test_str_to_expr.py @@ -1,3 +1,18 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + from iceberg.api.expressions import Expressions diff --git a/tests/api/test_file_format.py b/tests/api/test_file_format.py index b83d7b1205..773e57fe46 100644 --- a/tests/api/test_file_format.py +++ b/tests/api/test_file_format.py @@ -1,3 +1,17 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. from iceberg.api import FileFormat diff --git a/tests/api/test_helpers.py b/tests/api/test_helpers.py index c526e5f55b..7d51e5f02a 100644 --- a/tests/api/test_helpers.py +++ b/tests/api/test_helpers.py @@ -1,3 +1,18 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + import io import pickle diff --git a/tests/core/avro/conftest.py b/tests/core/avro/conftest.py index 0e64646938..de595babbb 100644 --- a/tests/core/avro/conftest.py +++ b/tests/core/avro/conftest.py @@ -1,3 +1,18 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + from tempfile import NamedTemporaryFile from iceberg.api import Schema diff --git a/tests/core/avro/test_avro.py b/tests/core/avro/test_avro.py index 53227a37c4..5415e9fcdd 100644 --- a/tests/core/avro/test_avro.py +++ b/tests/core/avro/test_avro.py @@ -1,3 +1,18 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + import unittest diff --git a/tests/core/avro/test_read_projection.py b/tests/core/avro/test_read_projection.py index f6aff79c46..9cc35acd28 100644 --- a/tests/core/avro/test_read_projection.py +++ b/tests/core/avro/test_read_projection.py @@ -1,3 +1,18 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + # from .conftest import write_and_read from iceberg.core import ManifestEntry, PartitionSpecParser, SchemaParser from iceberg.core.avro import IcebergToAvro diff --git a/tests/core/utils/__init__.py b/tests/core/utils/__init__.py index e69de29bb2..8afd240a72 100644 --- a/tests/core/utils/__init__.py +++ b/tests/core/utils/__init__.py @@ -0,0 +1,15 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + diff --git a/tox.ini b/tox.ini index 78ed9499e1..f55ca63d5a 100644 --- a/tox.ini +++ b/tox.ini @@ -1,3 +1,18 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + [tox] envlist = py36,linters From 911808d735b95aff49e14cc98eb1bed6b3170af6 Mon Sep 17 00:00:00 2001 From: Ted Gooch Date: Thu, 17 Oct 2019 17:22:46 -0700 Subject: [PATCH 016/642] Site: Add Python API and quickstart docs (#551) --- README.md | 33 ++++++++++++++++++++++++++++++--- 1 file changed, 30 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index ead3fbef75..be72674872 100644 --- a/README.md +++ b/README.md @@ -15,6 +15,33 @@ - limitations under the License. --> -# Iceberg -A python implementation of the Iceberg table format. -See the project level README for more details: https://github.com/apache/incubator-iceberg +# Iceberg Python + +Iceberg is a python library for programatic access to iceberg table metadata as well as data access. The intention is to provide a functional subset of the java library. + +## Getting Started + +Iceberg python is currently in development, for development and testing purposes the best way to install the library is to perform the following steps: + +``` +git clone https://github.com/apache/incubator-iceberg.git +cd incubator-iceberg/python +pip install -e . +``` + +## Testing +Testing is done using tox. The config can be found in `tox.ini` within the python directory of the iceberg project. + +``` +# simply run tox from within the python dir +tox +``` + +## Get in Touch + +- Email: + * [dev@iceberg.apache.org](mailto:dev@iceberg.apache.org) + +- Issues + * [File a github incident](https://github.com/apache/incubator-iceberg/issues) + From 8620d8c1091971243e30142360e48b9b75597bbd Mon Sep 17 00:00:00 2001 From: Ted Gooch Date: Fri, 18 Oct 2019 11:28:12 -0700 Subject: [PATCH 017/642] Python: Support reading Iceberg tables in a Hive metastore (#530) This implements the same conventions for storing Iceberg metadata in the Hive Metastore that are used by the Java implementation. --- iceberg/hive/__init__.py | 25 +++++ iceberg/hive/hive_table_operations.py | 59 +++++++++++ iceberg/hive/hive_tables.py | 57 ++++++++++ setup.py | 1 + tests/api/expressions/test_str_to_expr.py | 2 +- tests/hive/__init__.py | 18 ++++ tests/hive/conftest.py | 18 ++++ tests/hive/test_hive_tables.py | 120 ++++++++++++++++++++++ tox.ini | 1 + 9 files changed, 300 insertions(+), 1 deletion(-) create mode 100644 iceberg/hive/__init__.py create mode 100644 iceberg/hive/hive_table_operations.py create mode 100644 iceberg/hive/hive_tables.py create mode 100644 tests/hive/__init__.py create mode 100644 tests/hive/conftest.py create mode 100644 tests/hive/test_hive_tables.py diff --git a/iceberg/hive/__init__.py b/iceberg/hive/__init__.py new file mode 100644 index 0000000000..37b9a13613 --- /dev/null +++ b/iceberg/hive/__init__.py @@ -0,0 +1,25 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# + + +__all__ = ["HiveTableOperations", "HiveTables"] + + +from .hive_table_operations import HiveTableOperations +from .hive_tables import HiveTables diff --git a/iceberg/hive/hive_table_operations.py b/iceberg/hive/hive_table_operations.py new file mode 100644 index 0000000000..ed1b867210 --- /dev/null +++ b/iceberg/hive/hive_table_operations.py @@ -0,0 +1,59 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# + + +from iceberg.core import BaseMetastoreTableOperations + + +class HiveTableOperations(BaseMetastoreTableOperations): + + def __init__(self, conf, client, database, table): + super(HiveTableOperations, self).__init__(conf) + self._client = client + self.database = database + self.table = table + self.refresh() + + def refresh(self): + with self._client as open_client: + tbl_info = open_client.get_table(self.database, self.table) + + table_type = tbl_info.parameters.get(BaseMetastoreTableOperations.TABLE_TYPE_PROP) + + if table_type is None or table_type.lower() != BaseMetastoreTableOperations.ICEBERG_TABLE_TYPE_VALUE: + raise RuntimeError("Invalid table, not Iceberg: %s.%s" % (self.database, + self.table)) + + metadata_location = tbl_info.parameters.get(BaseMetastoreTableOperations.METADATA_LOCATION_PROP) + if metadata_location is None: + raise RuntimeError("Invalid table, missing metadata_location: %s.%s" % (self.database, + self.table)) + + self.refresh_from_metadata_location(metadata_location) + + return self.current() + + def commit(self, base, metadata): + raise NotImplementedError() + + def io(self): + raise NotImplementedError() + + def close(self): + self._client.close() diff --git a/iceberg/hive/hive_tables.py b/iceberg/hive/hive_tables.py new file mode 100644 index 0000000000..42d734c468 --- /dev/null +++ b/iceberg/hive/hive_tables.py @@ -0,0 +1,57 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# + + +from hmsclient import hmsclient +from iceberg.core import BaseMetastoreTables + +from .hive_table_operations import HiveTableOperations + + +class HiveTables(BaseMetastoreTables): + DOT = "." + THRIFT_URIS = "hive.metastore.uris" + + def __init__(self, conf): + super(HiveTables, self).__init__(conf) + + def create(self, schema, table_identifier=None, spec=None, properties=None, database=None, table=None): + raise NotImplementedError() + + def load(self, table_identifier): + parts = table_identifier.split(HiveTables.DOT) + if len(parts) == 2: + return super(HiveTables, self).load(database=parts[0], table=parts[1]) + elif len(parts) == 1: + return super(HiveTables, self).load("default", table=parts[1]) + else: + raise RuntimeError("Could not parse table identifier: %s" % table_identifier) + + def new_table_ops(self, conf, database, table): + return HiveTableOperations(conf, self.get_client(), database, table) + + def drop(self, database, table): + raise RuntimeError("Not yet implemented") + + def get_client(self): + from urllib.parse import urlparse + metastore_uri = urlparse(self.conf[HiveTables.THRIFT_URIS]) + + client = hmsclient.HMSClient(host=metastore_uri.hostname, port=metastore_uri.port) + return client diff --git a/setup.py b/setup.py index 482740581d..998fd14f78 100755 --- a/setup.py +++ b/setup.py @@ -30,6 +30,7 @@ 'boto3', 'fastavro', 'fastparquet>=0.3.1', + 'hmsclient', 'mmh3', 'moz_sql_parser', 'pyparsing', diff --git a/tests/api/expressions/test_str_to_expr.py b/tests/api/expressions/test_str_to_expr.py index dd2bb1cc3f..8e8e0109e6 100644 --- a/tests/api/expressions/test_str_to_expr.py +++ b/tests/api/expressions/test_str_to_expr.py @@ -49,7 +49,7 @@ def test_lt(): def test_lte(): expected_expr = Expressions.less_than_or_equal("col_a", 1) conv_expr = Expressions.convert_string_to_expr("col_a <= 1") - assert expected_expr == conv_expr\ + assert expected_expr == conv_expr def test_and(): diff --git a/tests/hive/__init__.py b/tests/hive/__init__.py new file mode 100644 index 0000000000..fe95886d5c --- /dev/null +++ b/tests/hive/__init__.py @@ -0,0 +1,18 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# diff --git a/tests/hive/conftest.py b/tests/hive/conftest.py new file mode 100644 index 0000000000..fe95886d5c --- /dev/null +++ b/tests/hive/conftest.py @@ -0,0 +1,18 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# diff --git a/tests/hive/test_hive_tables.py b/tests/hive/test_hive_tables.py new file mode 100644 index 0000000000..cba43378b3 --- /dev/null +++ b/tests/hive/test_hive_tables.py @@ -0,0 +1,120 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# + +from iceberg.hive import HiveTables +import mock +from pytest import raises + + +class TestHMSTable(object): + def __init__(self, params): + self.parameters = params + + +@mock.patch("iceberg.hive.HiveTableOperations.refresh_from_metadata_location") +@mock.patch("iceberg.hive.HiveTableOperations.current") +@mock.patch("iceberg.hive.HiveTables.get_client") +def test_load_tables_check_valid_props(client, current_call, refresh_call): + parameters = {"table_type": "ICEBERG", + "partition_spec": [], + "metadata_location": "s3://path/to/iceberg.metadata.json"} + + client.return_value.__enter__.return_value.get_table.return_value = TestHMSTable(parameters) + + conf = {"hive.metastore.uris": 'thrift://hms:port'} + tables = HiveTables(conf) + tables.load("test.test_123") + + +@mock.patch("iceberg.hive.HiveTableOperations.refresh_from_metadata_location") +@mock.patch("iceberg.hive.HiveTableOperations.current") +@mock.patch("iceberg.hive.HiveTables.get_client") +def test_load_tables_check_missing_iceberg_type(client, current_call, refresh_call): + parameters = {"partition_spec": [], + "metadata_location": "s3://path/to/iceberg.metdata.json"} + + client.return_value.__enter__.return_value.get_table.return_value = TestHMSTable(parameters) + + conf = {"hive.metastore.uris": 'thrift://hms:port'} + tables = HiveTables(conf) + with raises(RuntimeError): + tables.load("test.test_123") + + +@mock.patch("iceberg.hive.HiveTableOperations.refresh_from_metadata_location") +@mock.patch("iceberg.hive.HiveTableOperations.current") +@mock.patch("iceberg.hive.HiveTables.get_client") +def test_load_tables_check_non_iceberg_type(client, current_call, refresh_call): + parameters = {"table_type": "HIVE", + "partition_spec": [], + "metadata_location": "s3://path/to/iceberg.metdata.json"} + + client.return_value.__enter__.return_value.get_table.return_value = TestHMSTable(parameters) + + conf = {"hive.metastore.uris": 'thrift://hms:port'} + tables = HiveTables(conf) + with raises(RuntimeError): + tables.load("test.test_123") + + +@mock.patch("iceberg.hive.HiveTableOperations.refresh_from_metadata_location") +@mock.patch("iceberg.hive.HiveTableOperations.current") +@mock.patch("iceberg.hive.HiveTables.get_client") +def test_load_tables_check_none_table_type(client, current_call, refresh_call): + parameters = {"table_type": None, + "partition_spec": [], + "metadata_location": "s3://path/to/iceberg.metdata.json"} + + client.return_value.__enter__.return_value.get_table.return_value = TestHMSTable(parameters) + + conf = {"hive.metastore.uris": 'thrift://hms:port'} + tables = HiveTables(conf) + with raises(RuntimeError): + tables.load("test.test_123") + + +@mock.patch("iceberg.hive.HiveTableOperations.refresh_from_metadata_location") +@mock.patch("iceberg.hive.HiveTableOperations.current") +@mock.patch("iceberg.hive.HiveTables.get_client") +def test_load_tables_check_no_location(client, current_call, refresh_call): + parameters = {"table_type": "ICEBERG", + "partition_spec": []} + + client.return_value.__enter__.return_value.get_table.return_value = TestHMSTable(parameters) + + conf = {"hive.metastore.uris": 'thrift://hms:port'} + tables = HiveTables(conf) + with raises(RuntimeError): + tables.load("test.test_123") + + +@mock.patch("iceberg.hive.HiveTableOperations.refresh_from_metadata_location") +@mock.patch("iceberg.hive.HiveTableOperations.current") +@mock.patch("iceberg.hive.HiveTables.get_client") +def test_load_tables_check_none_location(client, current_call, refresh_call): + parameters = {"table_type": "ICEBERG", + "partition_spec": [], + "metadata_location": None} + + client.return_value.__enter__.return_value.get_table.return_value = TestHMSTable(parameters) + + conf = {"hive.metastore.uris": 'thrift://hms:port'} + tables = HiveTables(conf) + with raises(RuntimeError): + tables.load("test.test_123") diff --git a/tox.ini b/tox.ini index f55ca63d5a..7b0fe41b5e 100644 --- a/tox.ini +++ b/tox.ini @@ -19,6 +19,7 @@ envlist = py36,linters [testenv] deps = coverage + mock nose pytest setenv = From cb317797e9ac55e5cb7b0e630ab7adb76a0be67a Mon Sep 17 00:00:00 2001 From: Ted Gooch Date: Fri, 18 Oct 2019 13:20:07 -0700 Subject: [PATCH 018/642] Python: Fix status in ManifestEntry, remove unused configs (#558) --- iceberg/core/manifest_entry.py | 2 +- iceberg/core/util/__init__.py | 4 ---- 2 files changed, 1 insertion(+), 5 deletions(-) diff --git a/iceberg/core/manifest_entry.py b/iceberg/core/manifest_entry.py index 44aba5d113..fde4e36fa7 100644 --- a/iceberg/core/manifest_entry.py +++ b/iceberg/core/manifest_entry.py @@ -76,7 +76,7 @@ def copy(self): def put(self, i, v): if i == 0: - self.status = Status.from_id(i) + self.status = Status.from_id(v) elif i == 1: self.snapshot_id = v elif i == 2: diff --git a/iceberg/core/util/__init__.py b/iceberg/core/util/__init__.py index 1c8b486506..3d4dcc0b02 100644 --- a/iceberg/core/util/__init__.py +++ b/iceberg/core/util/__init__.py @@ -17,8 +17,6 @@ __all__ = ["AtomicInteger", - "METAFLOW_ENABLED", - "METAFLOW_ROOTDIR", "PackingIterator", "PLANNER_THREAD_POOL_SIZE_PROP", "SCAN_THREAD_POOL_ENABLED", @@ -32,8 +30,6 @@ PLANNER_THREAD_POOL_SIZE_PROP = "iceberg.planner.num-threads" WORKER_THREAD_POOL_SIZE_PROP = "iceberg.worker.num-threads" SCAN_THREAD_POOL_ENABLED = "iceberg.scan.plan-in-worker-pool" -METAFLOW_ENABLED = "iceberg.s3.use-metaflow" -METAFLOW_ROOTDIR = "iceberg.s3.metaflow-root-dir" def str_as_bool(str_var): From 0191759c824bf4260dce2edde15d885c20f8d598 Mon Sep 17 00:00:00 2001 From: Ted Gooch Date: Tue, 12 Nov 2019 11:31:21 -0800 Subject: [PATCH 019/642] Python: Pin mo_sql_parser to unbroken version (#630) --- setup.py | 3 ++- tests/core/utils/__init__.py | 1 - 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/setup.py b/setup.py index 998fd14f78..51a080450f 100755 --- a/setup.py +++ b/setup.py @@ -32,7 +32,8 @@ 'fastparquet>=0.3.1', 'hmsclient', 'mmh3', - 'moz_sql_parser', + 'mo_future<2.50.19316', + 'moz_sql_parser==2.44.19084', 'pyparsing', 'python-dateutil', 'pytz', diff --git a/tests/core/utils/__init__.py b/tests/core/utils/__init__.py index 8afd240a72..6acb5d125a 100644 --- a/tests/core/utils/__init__.py +++ b/tests/core/utils/__init__.py @@ -12,4 +12,3 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. - From 7be44951dd11242db47289704fc8d97e9f7335c3 Mon Sep 17 00:00:00 2001 From: Ted Gooch Date: Fri, 15 Nov 2019 11:46:36 -0800 Subject: [PATCH 020/642] Python: Remove unused iceberg.api.exception module causing test failures (#649) --- iceberg/api/exceptions/__init__.py | 26 -------------------------- 1 file changed, 26 deletions(-) delete mode 100644 iceberg/api/exceptions/__init__.py diff --git a/iceberg/api/exceptions/__init__.py b/iceberg/api/exceptions/__init__.py deleted file mode 100644 index e6810ef4e9..0000000000 --- a/iceberg/api/exceptions/__init__.py +++ /dev/null @@ -1,26 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -__all__ = ["AlreadyExistsException", - "CommitFailedException", - "NoSuchTableException", - "ValidationException"] - -from .exceptions import (AlreadyExistsException, - CommitFailedException, - NoSuchTableException, - ValidationException) From a36d42fd2a7d83015726a3ae43ea07b739a1e94f Mon Sep 17 00:00:00 2001 From: Fokko Driesprong Date: Thu, 13 Feb 2020 22:01:37 +0100 Subject: [PATCH 021/642] Python: Support 3.6+ (#795) --- setup.py | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/setup.py b/setup.py index 51a080450f..aace63f20d 100755 --- a/setup.py +++ b/setup.py @@ -25,7 +25,7 @@ description='Iceberg is a new table format for storing large, slow-moving tabular data', keywords='iceberg', url='https://github.com/apache/incubator-iceberg/blob/master/README.md', - python_requires='>=3.4', + python_requires='>=3.6', install_requires=['botocore', 'boto3', 'fastavro', @@ -42,5 +42,12 @@ 'pandas', 'pyarrow' ], - setup_requires=['setupmeta'] + setup_requires=['setupmeta'], + license="Apache License 2.0", + classifiers=[ + "License :: OSI Approved :: Apache Software License", + "Programming Language :: Python :: 3.6", + "Programming Language :: Python :: 3.7", + "Programming Language :: Python :: 3.8", + ], ) From cb09579e90aa9d3b8c04fd37257f47e2c1223a71 Mon Sep 17 00:00:00 2001 From: Ted Gooch Date: Wed, 6 May 2020 11:40:48 -0700 Subject: [PATCH 022/642] Python: Removing dependency on moz_sql_parser (#993) Co-authored-by: Ted Gooch --- iceberg/api/expressions/expression_parser.py | 161 +++++++++++++++++++ iceberg/api/expressions/expressions.py | 71 +------- setup.py | 2 - tests/api/expressions/test_str_to_expr.py | 24 ++- 4 files changed, 184 insertions(+), 74 deletions(-) create mode 100644 iceberg/api/expressions/expression_parser.py diff --git a/iceberg/api/expressions/expression_parser.py b/iceberg/api/expressions/expression_parser.py new file mode 100644 index 0000000000..0ffde4e23e --- /dev/null +++ b/iceberg/api/expressions/expression_parser.py @@ -0,0 +1,161 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Derived from the SimpleSQL Parser example in pyparsing, retrofitted to just handle the +# where clause predicates +# https://github.com/pyparsing/pyparsing/blob/master/examples/simpleSQL.py + +import logging + +from pyparsing import ( + alphanums, + alphas, + CaselessKeyword, + delimitedList, + Group, + infixNotation, + oneOf, + opAssoc, + pyparsing_common as ppc, + quotedString, + Word +) + + +_logger = logging.getLogger(__name__) + +AND, OR, IN, IS, NOT, NULL, BETWEEN = map( + CaselessKeyword, "and or in is not null between".split() +) +NOT_NULL = NOT + NULL + +ident = Word(alphas, alphanums + "_$").setName("identifier") +columnName = delimitedList(ident, ".", combine=True).setName("column name") + +binop = oneOf("= == != < > >= <= eq ne lt le gt ge <>", caseless=False) +realNum = ppc.real() +intNum = ppc.signed_integer() + +columnRval = (realNum + | intNum + | quotedString + | columnName) # need to add support for alg expressions +whereCondition = Group( + (columnName + binop + columnRval) + | (columnName + IN + Group("(" + delimitedList(columnRval) + ")")) + | (columnName + IS + (NULL | NOT_NULL)) + | (columnName + BETWEEN + columnRval + AND + columnRval) + +) + +whereExpression = infixNotation( + Group(whereCondition + | NOT + whereCondition + | NOT + Group('(' + whereCondition + ')') + | NOT + columnName), + [(NOT, 1, opAssoc.LEFT), (AND, 2, opAssoc.LEFT), (OR, 2, opAssoc.LEFT), (IS, 2, opAssoc.LEFT)], +) + +op_map = {"=": "eq", + "==": "eq", + "eq": "eq", + ">": "gt", + "gt": "gt", + ">=": "gte", + "gte": "gte", + "<": "lt", + "lt": "lt", + "<=": "lte", + "lte": "lte", + "!": "not", + "not": "not", + "!=": "neq", + "<>": "neq", + "neq": "neq", + "||": "or", + "or": "or", + "&&": "and", + "and": "and", + "in": "in", + "between": "between", + "is": "is"} + + +def get_expr_tree(tokens): + if isinstance(tokens, (str, int)): + return tokens + if len(tokens) > 1: + if (tokens[0] == "not"): + return {"not": get_expr_tree(tokens[1])} + if (tokens[0] == "(" and tokens[-1] == ")"): + return get_expr_tree(tokens[1:-1]) + else: + return get_expr_tree(tokens[0]) + + op = op_map[tokens[1]] + + if op == "in": + return {'in': [get_expr_tree(tokens[0]), [token for token in tokens[2][1:-1]]]} + elif op == "between": + return {'and': [{"gte": [get_expr_tree(tokens[0]), tokens[2]]}, + {"lte": [get_expr_tree(tokens[0]), tokens[4]]}]} + elif op == "is": + + if tokens[2] == 'null': + return {"missing": tokens[0]} + else: + return {"exists": tokens[0]} + if len(tokens) > 3: + binary_tuples = get_expr_tree(tokens[2:]) + else: + binary_tuples = get_expr_tree(tokens[2]) + + return {op: [get_expr_tree(tokens[0]), + binary_tuples]} + + +def get_expr(node, expr_map): + if isinstance(node, dict): + for i in node.keys(): + op = i + if op == "literal": + return node["literal"] + mapped_op = expr_map.get(op, expr_map) + if len(mapped_op) == 1: + mapped_op = mapped_op[0] + if mapped_op is None: + raise RuntimeError("no mapping for op: %s" % op) + if op in ("not", "exists", "missing"): + return mapped_op(get_expr(node[op], expr_map)) + + return mapped_op(*get_expr(node[op], expr_map)) + elif isinstance(node, (list, tuple)): + return (get_expr(item, expr_map) for item in node) + elif isinstance(node, (str, int, float)): + return node + else: + raise RuntimeError("unknown node type" % node) + + +def parse_expr_string(predicate_string, expr_map): + from pyparsing import ParseException + + try: + expr = whereExpression.parseString(predicate_string, parseAll=True) + expr = get_expr_tree(expr) + return get_expr(expr, expr_map) + except ParseException as pe: + _logger.error("Error parsing string expression into iceberg expression: %s" % str(pe)) + raise diff --git a/iceberg/api/expressions/expressions.py b/iceberg/api/expressions/expressions.py index 190382d34d..4b6f4e5f78 100644 --- a/iceberg/api/expressions/expressions.py +++ b/iceberg/api/expressions/expressions.py @@ -23,6 +23,7 @@ Operation, Or, TRUE) +from .expression_parser import parse_expr_string from .predicate import (Predicate, UnboundPredicate) from .reference import NamedReference @@ -128,9 +129,6 @@ def ref(name): @staticmethod def convert_string_to_expr(predicate_string): - from moz_sql_parser import parse - from pyparsing import ParseException - expr_map = {"and": (Expressions.and_,), "eq": (Expressions.equal,), "exists": (Expressions.not_null,), @@ -143,72 +141,7 @@ def convert_string_to_expr(predicate_string): "not": (Expressions.not_,), "or": (Expressions.or_,)} - dummy_query = "SELECT * FROM tbl WHERE {}".format(predicate_string) # nosec - try: - expr = (Expressions. - _transform_to_binary_tuples(Expressions. - _transform_between_op(parse(dummy_query)["where"]))) - return Expressions._get_expr(expr, expr_map) - except ParseException as pe: - _logger.error("Error parsing string expression into iceberg expression: %s" % str(pe)) - raise - - @staticmethod - def _get_expr(node, expr_map): - if isinstance(node, dict): - for i in node.keys(): - op = i - if op == "literal": - return node["literal"] - mapped_op = expr_map.get(op, expr_map) - if len(mapped_op) == 1: - mapped_op = mapped_op[0] - if mapped_op is None: - raise RuntimeError("no mapping for op: %s" % op) - if mapped_op in (Expressions.not_, Expressions.not_null, Expressions.is_null): - return mapped_op(Expressions._get_expr(node[op], expr_map)) - - return mapped_op(*Expressions._get_expr(node[op], expr_map)) - elif isinstance(node, (list, tuple)): - return (Expressions._get_expr(item, expr_map) for item in node) - elif isinstance(node, (str, int, float)): - return node - else: - raise RuntimeError("unknown node type" % node) - - @staticmethod - def _transform_to_binary_tuples(expr): - if not isinstance(expr, dict): - return expr - for op in expr.keys(): - if op in ("exists", "literal", "missing", "not"): - return expr - new_expr = [Expressions._transform_to_binary_tuples(child) - for child in expr[op]] - while len(new_expr) > 2: - new_and = {op: [new_expr[-2], new_expr[-1]]} - new_expr[-2] = new_and - del new_expr[-1] - expr[op] = new_expr - - return expr - - @staticmethod - def _transform_between_op(expr): - if isinstance(expr, (bool, float, int, str)): - return expr - for op, children in expr.items(): - if op in ("exists", "literal", "missing", "not"): - return expr - new_children = [] - for child in children: - new_children.append(Expressions._transform_between_op(child)) - expr[op] = new_children - if op == "between": - return {"and": [{"gte": [expr[op][0], expr[op][1]]}, - {"lte": [expr[op][0], expr[op][2]]}]} - else: - return expr + return parse_expr_string(predicate_string, expr_map) class ExpressionVisitors(object): diff --git a/setup.py b/setup.py index aace63f20d..6a75f3ccf8 100755 --- a/setup.py +++ b/setup.py @@ -32,8 +32,6 @@ 'fastparquet>=0.3.1', 'hmsclient', 'mmh3', - 'mo_future<2.50.19316', - 'moz_sql_parser==2.44.19084', 'pyparsing', 'python-dateutil', 'pytz', diff --git a/tests/api/expressions/test_str_to_expr.py b/tests/api/expressions/test_str_to_expr.py index 8e8e0109e6..34b3abdf77 100644 --- a/tests/api/expressions/test_str_to_expr.py +++ b/tests/api/expressions/test_str_to_expr.py @@ -117,14 +117,32 @@ def test_ternary_condition(): def test_precedence(): - expected_expr = Expressions.and_(Expressions.or_(Expressions.equal("col_a", 1), - Expressions.equal("col_b", 2)), - Expressions.equal("col_c", 3)) + expected_expr = Expressions.or_(Expressions.equal("col_a", 1), + Expressions.and_(Expressions.equal("col_b", 2), + Expressions.equal("col_c", 3))) conv_expr = Expressions.convert_string_to_expr("col_a=1 or col_b=2 and col_c=3") assert expected_expr == conv_expr +def test_precedence_opposite_order(): + expected_expr = Expressions.or_(Expressions.and_(Expressions.equal("col_a", 1), + Expressions.equal("col_b", 2)), + Expressions.equal("col_c", 3)) + + conv_expr = Expressions.convert_string_to_expr("col_a=1 and col_b=2 or col_c=3") + assert expected_expr == conv_expr + + +def test_precedence_explicit(): + expected_expr = Expressions.and_(Expressions.equal("col_a", 1), + Expressions.or_(Expressions.equal("col_b", 2), + Expressions.equal("col_c", 3))) + + conv_expr = Expressions.convert_string_to_expr("col_a=1 and (col_b=2 or col_c=3)") + assert expected_expr == conv_expr + + def test_precedence_with_between(): expected_expr = Expressions.or_(Expressions.and_(Expressions.greater_than_or_equal("col_a", 1), Expressions.less_than_or_equal("col_a", 2)), From f6d84638cec9729334069fb5a9025dc11143bef6 Mon Sep 17 00:00:00 2001 From: jun-he Date: Thu, 7 May 2020 09:37:18 -0700 Subject: [PATCH 023/642] Python: Fix partition spec names (#996) --- iceberg/api/partition_spec.py | 24 ++++++++-------- tests/core/test_partition_spec_parser.py | 36 ++++++++++++++++++++++++ 2 files changed, 48 insertions(+), 12 deletions(-) create mode 100644 tests/core/test_partition_spec_parser.py diff --git a/iceberg/api/partition_spec.py b/iceberg/api/partition_spec.py index e830b28b69..9f4d37e899 100644 --- a/iceberg/api/partition_spec.py +++ b/iceberg/api/partition_spec.py @@ -231,56 +231,56 @@ def identity(self, source_name): return self def year(self, source_name): - name = "%s_year".format(source_name) + name = "{}_year".format(source_name) self.check_and_add_partition_name(name) source_column = self.find_source_column(source_name) self.fields.append(PartitionField(source_column.field_id, - source_name, + name, Transforms.year(source_column.types))) return self def month(self, source_name): - name = "%s_month".format(source_name) + name = "{}_month".format(source_name) self.check_and_add_partition_name(name) source_column = self.find_source_column(source_name) self.fields.append(PartitionField(source_column.field_id, - source_name, + name, Transforms.month(source_column.types))) return self def day(self, source_name): - name = "%s_day".format(source_name) + name = "{}_day".format(source_name) self.check_and_add_partition_name(name) source_column = self.find_source_column(source_name) self.fields.append(PartitionField(source_column.field_id, - source_name, + name, Transforms.day(source_column.types))) return self def hour(self, source_name): - name = "%s_hour".format(source_name) + name = "{}_hour".format(source_name) self.check_and_add_partition_name(name) source_column = self.find_source_column(source_name) self.fields.append(PartitionField(source_column.field_id, - source_name, + name, Transforms.hour(source_column.type))) return self def bucket(self, source_name, num_buckets): - name = "%s_bucket".format(source_name) + name = "{}_bucket".format(source_name) self.check_and_add_partition_name(name) source_column = self.find_source_column(source_name) self.fields.append(PartitionField(source_column.field_id, - source_name, + name, Transforms.bucket(source_column.type, num_buckets))) return self def truncate(self, source_name, width): - name = "%s_truncate".format(source_name) + name = "{}_truncate".format(source_name) self.check_and_add_partition_name(name) source_column = self.find_source_column(source_name) self.fields.append(PartitionField(source_column.field_id, - source_name, + name, Transforms.truncate(source_column.types, width))) return self diff --git a/tests/core/test_partition_spec_parser.py b/tests/core/test_partition_spec_parser.py new file mode 100644 index 0000000000..53285362f6 --- /dev/null +++ b/tests/core/test_partition_spec_parser.py @@ -0,0 +1,36 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +from iceberg.api import PartitionSpec, Schema +from iceberg.api.types import IntegerType, NestedField, StringType +from iceberg.core import PartitionSpecParser + + +def test_to_json_conversion(): + spec_schema = Schema(NestedField.required(1, "id", IntegerType.get()), + NestedField.required(2, "data", StringType.get())) + + spec = PartitionSpec\ + .builder_for(spec_schema) \ + .identity("id")\ + .bucket("data", 16)\ + .build() + + expected = '{"spec-id": 0, "fields": [' \ + '{"name": "id", "transform": "identity", "source-id": 1}, ' \ + '{"name": "data_bucket", "transform": "bucket[16]", "source-id": 2}]}' + assert expected == PartitionSpecParser.to_json(spec) From 8a12ca54442338826bc61d2ea2e0afa2270af797 Mon Sep 17 00:00:00 2001 From: Fokko Driesprong Date: Wed, 13 May 2020 18:19:24 +0200 Subject: [PATCH 024/642] Python: Add style checking with Flake8 to CI (#1034) --- README.md | 1 + iceberg/api/io/closeable_iterable.py | 5 +---- iceberg/api/partition_spec.py | 2 +- iceberg/api/types/type_util.py | 2 +- iceberg/core/avro/avro_to_iceberg.py | 3 +-- iceberg/core/base_metastore_tables.py | 2 +- iceberg/core/data_files.py | 2 +- iceberg/core/filesystem/s3_filesystem.py | 5 +---- iceberg/core/scan_summary.py | 2 +- setup.py | 6 ++++++ tests/api/expressions/conftest.py | 2 +- tox.ini | 1 + 12 files changed, 17 insertions(+), 16 deletions(-) diff --git a/README.md b/README.md index be72674872..c2e612f1ec 100644 --- a/README.md +++ b/README.md @@ -30,6 +30,7 @@ pip install -e . ``` ## Testing + Testing is done using tox. The config can be found in `tox.ini` within the python directory of the iceberg project. ``` diff --git a/iceberg/api/io/closeable_iterable.py b/iceberg/api/io/closeable_iterable.py index 1d62e2db41..2ea80effa1 100644 --- a/iceberg/api/io/closeable_iterable.py +++ b/iceberg/api/io/closeable_iterable.py @@ -20,11 +20,8 @@ class CloseableIterable(collections.Iterator): - def next(self): - raise NotImplementedError() - def __next__(self): - return self.next() + raise NotImplementedError() def close(self): raise NotImplementedError() diff --git a/iceberg/api/partition_spec.py b/iceberg/api/partition_spec.py index 9f4d37e899..95cce0e6a6 100644 --- a/iceberg/api/partition_spec.py +++ b/iceberg/api/partition_spec.py @@ -58,7 +58,7 @@ def fields(self): def java_classes(self): if self.__java_classes is None: self.__java_classes - for i, field in enumerate(self.__fields): + for field in self.__fields: source_type = self.schema.find_type(field.source_id) result = field.transform().get_result_by_type(source_type) self.__java_classes.append(result.type_id.java_class()) diff --git a/iceberg/api/types/type_util.py b/iceberg/api/types/type_util.py index 7bc8dd74cb..ca52a7db9c 100644 --- a/iceberg/api/types/type_util.py +++ b/iceberg/api/types/type_util.py @@ -403,7 +403,7 @@ def struct(self, struct, field_results): length = len(struct.fields) new_ids = list() - for i in range(length): + for _ in range(length): new_ids.append(self.next_id()) new_fields = list() diff --git a/iceberg/core/avro/avro_to_iceberg.py b/iceberg/core/avro/avro_to_iceberg.py index 3aa7f6d10a..14dc9ce9d4 100644 --- a/iceberg/core/avro/avro_to_iceberg.py +++ b/iceberg/core/avro/avro_to_iceberg.py @@ -203,8 +203,7 @@ def convert_map_type(avro_field, next_id=None): avro_field_type = avro_field.get(AvroToIceberg.FIELD_TYPE_PROP) avro_logical_type = avro_field.get(AvroToIceberg.FIELD_LOGICAL_TYPE_PROP) if avro_field_type != "array" or avro_logical_type != "map": - raise RuntimeError("Avro type must be array and logical type must be map: %s" % (avro_field_type, - avro_logical_type)) + raise RuntimeError("Avro type must be array and logical type must be map: %s" % avro_logical_type) is_optional = False items = avro_field.get(AvroToIceberg.FIELD_ITEMS_PROP) for field in items.get(AvroToIceberg.FIELD_FIELDS_PROP, list()): diff --git a/iceberg/core/base_metastore_tables.py b/iceberg/core/base_metastore_tables.py index bdb91de5ce..17bdde9bb6 100644 --- a/iceberg/core/base_metastore_tables.py +++ b/iceberg/core/base_metastore_tables.py @@ -50,4 +50,4 @@ def default_warehouse_location(self, conf, database, table): if warehouse_location is None: raise RuntimeError("Warehouse location is not set: hive.metastore.warehouse.dir=null") - return "%s/%s.db/%s".format(warehouse_location, database, table) + return f"{warehouse_location}/{database}.db/{table}" diff --git a/iceberg/core/data_files.py b/iceberg/core/data_files.py index 39503f6529..8b73ebcd12 100644 --- a/iceberg/core/data_files.py +++ b/iceberg/core/data_files.py @@ -37,7 +37,7 @@ def copy_partition_data(spec, partition_data, reuse): if data is None: data = DataFiles.new_partition_data(spec) - for i, field in enumerate(spec.fields): + for i, _ in enumerate(spec.fields): data.set(i, partition_data.get(i)) return data diff --git a/iceberg/core/filesystem/s3_filesystem.py b/iceberg/core/filesystem/s3_filesystem.py index 6843df6fa2..a8ac9d4cca 100644 --- a/iceberg/core/filesystem/s3_filesystem.py +++ b/iceberg/core/filesystem/s3_filesystem.py @@ -182,7 +182,7 @@ def close(self): def flush(self): pass - def next(self): + def __next__(self): return next(self.readline()) def read(self, n=0): @@ -251,6 +251,3 @@ def __exit__(self, exc_type, exc_val, exc_tb): def __iter__(self): return self - - def __next__(self): - return self.next() diff --git a/iceberg/core/scan_summary.py b/iceberg/core/scan_summary.py index f7e5e15bb2..cef872f7ea 100644 --- a/iceberg/core/scan_summary.py +++ b/iceberg/core/scan_summary.py @@ -184,7 +184,7 @@ def from_partition_summaries(self, snapshots): or str_as_bool(snap.summary.get(SnapshotSummary.PARTITION_SUMMARY_PROP, "false")): return None - for key, val in snap.summary.items(): + for key, _ in snap.summary.items(): if key.startswith(SnapshotSummary.CHANGED_PARTITION_PREFIX): part_key = key[len(SnapshotSummary.CHANGED_PARTITION_PREFIX):] # part = dict(entry.split("=") for entry in val.split(",")) diff --git a/setup.py b/setup.py index 6a75f3ccf8..b1d5c8c8c3 100755 --- a/setup.py +++ b/setup.py @@ -40,6 +40,12 @@ 'pandas', 'pyarrow' ], + extras_require={ + "dev": [ + "tox-travis==0.12", + "virtualenv<20.0.0", + ], + }, setup_requires=['setupmeta'], license="Apache License 2.0", classifiers=[ diff --git a/tests/api/expressions/conftest.py b/tests/api/expressions/conftest.py index fe1d98205a..3ee4b4cd5e 100644 --- a/tests/api/expressions/conftest.py +++ b/tests/api/expressions/conftest.py @@ -90,7 +90,7 @@ def __init__(self, message): def predicate(self, pred): if isinstance(pred, UnboundPredicate): - assert False + raise AssertionError("Predicate should be a BoundPredicate") class TestDataFile(DataFile): diff --git a/tox.ini b/tox.ini index 7b0fe41b5e..9c49b0bd33 100644 --- a/tox.ini +++ b/tox.ini @@ -49,6 +49,7 @@ skip_install = true deps = flake8 flake8-import-order>=0.9 + flake8-bugbear commands = flake8 iceberg setup.py tests From a12dd8ef1c266575167a44db319863576b940dd6 Mon Sep 17 00:00:00 2001 From: jun-he Date: Mon, 18 May 2020 13:38:22 -0700 Subject: [PATCH 025/642] Python: Fix partition spec and transforms (#1045) --- iceberg/api/partition_spec.py | 8 +- iceberg/api/transforms/bucket.py | 12 +-- iceberg/api/transforms/dates.py | 4 +- iceberg/api/transforms/timestamps.py | 2 +- iceberg/api/transforms/transforms.py | 14 ++-- iceberg/api/transforms/truncate.py | 2 +- tests/core/test_partition_spec.py | 109 +++++++++++++++++++++++++++ 7 files changed, 130 insertions(+), 21 deletions(-) create mode 100644 tests/core/test_partition_spec.py diff --git a/iceberg/api/partition_spec.py b/iceberg/api/partition_spec.py index 95cce0e6a6..bcea93f2af 100644 --- a/iceberg/api/partition_spec.py +++ b/iceberg/api/partition_spec.py @@ -236,7 +236,7 @@ def year(self, source_name): source_column = self.find_source_column(source_name) self.fields.append(PartitionField(source_column.field_id, name, - Transforms.year(source_column.types))) + Transforms.year(source_column.type))) return self def month(self, source_name): @@ -245,7 +245,7 @@ def month(self, source_name): source_column = self.find_source_column(source_name) self.fields.append(PartitionField(source_column.field_id, name, - Transforms.month(source_column.types))) + Transforms.month(source_column.type))) return self def day(self, source_name): @@ -254,7 +254,7 @@ def day(self, source_name): source_column = self.find_source_column(source_name) self.fields.append(PartitionField(source_column.field_id, name, - Transforms.day(source_column.types))) + Transforms.day(source_column.type))) return self def hour(self, source_name): @@ -281,7 +281,7 @@ def truncate(self, source_name, width): source_column = self.find_source_column(source_name) self.fields.append(PartitionField(source_column.field_id, name, - Transforms.truncate(source_column.types, width))) + Transforms.truncate(source_column.type, width))) return self def add(self, source_id, name, transform): diff --git a/iceberg/api/transforms/bucket.py b/iceberg/api/transforms/bucket.py index c1672da4a4..64db712208 100644 --- a/iceberg/api/transforms/bucket.py +++ b/iceberg/api/transforms/bucket.py @@ -101,7 +101,7 @@ def hash(self, value): return Bucket.MURMUR3.hash(struct.pack("q", value)) def can_transform(self, type_var): - return type_var.type_id() in [TypeID.INTEGER, TypeID.DATE] + return type_var.type_id in [TypeID.INTEGER, TypeID.DATE] class BucketLong(Bucket): @@ -112,9 +112,9 @@ def hash(self, value): return Bucket.MURMUR3.hash(struct.pack("q", value)) def can_transform(self, type_var): - return type_var.type_id() in [TypeID.LONG, - TypeID.TIME, - TypeID.TIMESTAMP] + return type_var.type_id in [TypeID.LONG, + TypeID.TIME, + TypeID.TIMESTAMP] class BucketFloat(Bucket): @@ -125,7 +125,7 @@ def hash(self, value): return Bucket.MURMUR3.hash(struct.pack("d", value)) def can_transform(self, type_var): - return type_var.type_id() == TypeID.FLOAT + return type_var.type_id == TypeID.FLOAT class BucketDouble(Bucket): @@ -136,7 +136,7 @@ def hash(self, value): return Bucket.MURMUR3.hash(struct.pack("d", value)) def can_transform(self, type_var): - return type_var.type_id() == TypeID.DOUBLE + return type_var.type_id == TypeID.DOUBLE class BucketDecimal(Bucket): diff --git a/iceberg/api/transforms/dates.py b/iceberg/api/transforms/dates.py index 628281b20b..474b986f69 100644 --- a/iceberg/api/transforms/dates.py +++ b/iceberg/api/transforms/dates.py @@ -52,7 +52,7 @@ def apply(self, days): return apply_func(datetime.datetime.utcfromtimestamp(days * Dates.SECONDS_IN_DAY), Dates.EPOCH) def can_transform(self, type): - return type.type_id() == TypeID.DATE + return type.type_id == TypeID.DATE def get_result_type(self, source_type): return IntegerType.get() @@ -73,4 +73,4 @@ def to_human_string(self, value): return Dates.HUMAN_FUNCS[self.granularity](value) def __str__(self): - return "%s" % self + return self.name diff --git a/iceberg/api/transforms/timestamps.py b/iceberg/api/transforms/timestamps.py index 697cec67e5..25c4439bc1 100644 --- a/iceberg/api/transforms/timestamps.py +++ b/iceberg/api/transforms/timestamps.py @@ -50,7 +50,7 @@ def apply(self, value): return apply_func(datetime.datetime.utcfromtimestamp(value / 1000000), Timestamps.EPOCH) def can_transform(self, type_var): - return type_var == TypeID.TIMESTAMP + return type_var.type_id == TypeID.TIMESTAMP def get_result_type(self, source_type): return IntegerType.get() diff --git a/iceberg/api/transforms/transforms.py b/iceberg/api/transforms/transforms.py index c14d84930f..33877f1fd8 100644 --- a/iceberg/api/transforms/transforms.py +++ b/iceberg/api/transforms/transforms.py @@ -42,22 +42,22 @@ def __init__(self): pass @staticmethod - def from_string(type, transform): + def from_string(type_var, transform): match = Transforms.HAS_WIDTH.match(transform) if match is not None: name = match.group(1) w = match.group(2) if name.lower() == "truncate": - return Truncate.get(type, w) + return Truncate.get(type_var, w) elif name.lower() == "bucket": - return Bucket.get(type, w) + return Bucket.get(type_var, w) if transform.lower() == "identity": - return Identity.get(type) - elif type.type_id() == TypeID.TIMESTAMP: + return Identity.get(type_var) + elif type_var.type_id == TypeID.TIMESTAMP: return Timestamps(transform.lower(), transform.lower()) - elif type.type_id() == TypeID.DATE: + elif type_var.type_id == TypeID.DATE: return Dates(transform.lower(), transform.lower()) raise RuntimeError("Unknown transform: %s" % transform) @@ -108,4 +108,4 @@ def bucket(type_var, num_buckets): @staticmethod def truncate(type_var, width): - return Truncate.get(type, width) + return Truncate.get(type_var, width) diff --git a/iceberg/api/transforms/truncate.py b/iceberg/api/transforms/truncate.py index cd001ad44e..b37e6f0b06 100644 --- a/iceberg/api/transforms/truncate.py +++ b/iceberg/api/transforms/truncate.py @@ -32,7 +32,7 @@ def get(type_var, width): if type_var.type_id == TypeID.INTEGER: return TruncateInteger(width) elif type_var.type_id == TypeID.LONG: - return TruncateInteger(width) + return TruncateLong(width) elif type_var.type_id == TypeID.DECIMAL: return TruncateDecimal(width) elif type_var.type_id == TypeID.STRING: diff --git a/tests/core/test_partition_spec.py b/tests/core/test_partition_spec.py new file mode 100644 index 0000000000..09d108cf3c --- /dev/null +++ b/tests/core/test_partition_spec.py @@ -0,0 +1,109 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +from iceberg.api import PartitionSpec, Schema +from iceberg.api.types import (BinaryType, + DateType, + DecimalType, + FixedType, + IntegerType, + LongType, + NestedField, + StringType, + TimestampType, + TimeType, + UUIDType) + + +def test_to_json_conversion(): + spec_schema = Schema(NestedField.required(1, "i", IntegerType.get()), + NestedField.required(2, "l", LongType.get()), + NestedField.required(3, "d", DateType.get()), + NestedField.required(4, "t", TimeType.get()), + NestedField.required(5, "ts", TimestampType.without_timezone()), + NestedField.required(6, "dec", DecimalType.of(9, 2)), + NestedField.required(7, "s", StringType.get()), + NestedField.required(8, "u", UUIDType.get()), + NestedField.required(9, "f", FixedType.of_length(3)), + NestedField.required(10, "b", BinaryType.get())) + + specs = [ + PartitionSpec.builder_for(spec_schema).identity("i").build(), + PartitionSpec.builder_for(spec_schema).identity("l").build(), + PartitionSpec.builder_for(spec_schema).identity("d").build(), + PartitionSpec.builder_for(spec_schema).identity("t").build(), + PartitionSpec.builder_for(spec_schema).identity("ts").build(), + PartitionSpec.builder_for(spec_schema).identity("dec").build(), + PartitionSpec.builder_for(spec_schema).identity("s").build(), + PartitionSpec.builder_for(spec_schema).identity("u").build(), + PartitionSpec.builder_for(spec_schema).identity("f").build(), + PartitionSpec.builder_for(spec_schema).identity("b").build(), + PartitionSpec.builder_for(spec_schema).bucket("i", 128).build(), + PartitionSpec.builder_for(spec_schema).bucket("l", 128).build(), + PartitionSpec.builder_for(spec_schema).bucket("d", 128).build(), + PartitionSpec.builder_for(spec_schema).bucket("t", 128).build(), + PartitionSpec.builder_for(spec_schema).bucket("ts", 128).build(), + PartitionSpec.builder_for(spec_schema).bucket("dec", 128).build(), + PartitionSpec.builder_for(spec_schema).bucket("s", 128).build(), + PartitionSpec.builder_for(spec_schema).year("d").build(), + PartitionSpec.builder_for(spec_schema).month("d").build(), + PartitionSpec.builder_for(spec_schema).day("d").build(), + PartitionSpec.builder_for(spec_schema).year("ts").build(), + PartitionSpec.builder_for(spec_schema).month("ts").build(), + PartitionSpec.builder_for(spec_schema).day("ts").build(), + PartitionSpec.builder_for(spec_schema).hour("ts").build(), + PartitionSpec.builder_for(spec_schema).truncate("i", 10).build(), + PartitionSpec.builder_for(spec_schema).truncate("l", 10).build(), + PartitionSpec.builder_for(spec_schema).truncate("dec", 10).build(), + PartitionSpec.builder_for(spec_schema).truncate("s", 10).build(), + PartitionSpec.builder_for(spec_schema).add(6, "dec_bucket", "bucket[16]").build() + ] + + expected_spec_strs = [ + "[\n i: identity(1)\n]", + "[\n l: identity(2)\n]", + "[\n d: identity(3)\n]", + "[\n t: identity(4)\n]", + "[\n ts: identity(5)\n]", + "[\n dec: identity(6)\n]", + "[\n s: identity(7)\n]", + "[\n u: identity(8)\n]", + "[\n f: identity(9)\n]", + "[\n b: identity(10)\n]", + "[\n i_bucket: bucket[128](1)\n]", + "[\n l_bucket: bucket[128](2)\n]", + "[\n d_bucket: bucket[128](3)\n]", + "[\n t_bucket: bucket[128](4)\n]", + "[\n ts_bucket: bucket[128](5)\n]", + "[\n dec_bucket: bucket[128](6)\n]", + "[\n s_bucket: bucket[128](7)\n]", + "[\n d_year: year(3)\n]", + "[\n d_month: month(3)\n]", + "[\n d_day: day(3)\n]", + "[\n ts_year: year(5)\n]", + "[\n ts_month: month(5)\n]", + "[\n ts_day: day(5)\n]", + "[\n ts_hour: hour(5)\n]", + "[\n i_truncate: truncate[10](1)\n]", + "[\n l_truncate: truncate[10](2)\n]", + "[\n dec_truncate: truncate[10](6)\n]", + "[\n s_truncate: truncate[10](7)\n]", + "[\n dec_bucket: bucket[16](6)\n]", + ] + + for (spec, expected_spec_str) in zip(specs, expected_spec_strs): + assert str(spec) == expected_spec_str From 5dcdbd915449989f15c4525d5d3fd421b356166b Mon Sep 17 00:00:00 2001 From: jun-he Date: Mon, 18 May 2020 13:56:36 -0700 Subject: [PATCH 026/642] Python: Add persistent IDs to partition fields (#1047) Python port of #845. --- iceberg/api/partition_field.py | 8 ++- iceberg/api/partition_spec.py | 37 +++++++--- iceberg/api/transforms/transforms.py | 2 +- iceberg/core/partition_spec_parser.py | 49 +++++++------ tests/core/test_partition_spec_parser.py | 88 ++++++++++++++++++++++-- 5 files changed, 144 insertions(+), 40 deletions(-) diff --git a/iceberg/api/partition_field.py b/iceberg/api/partition_field.py index adaca28943..8e789a1089 100644 --- a/iceberg/api/partition_field.py +++ b/iceberg/api/partition_field.py @@ -18,8 +18,9 @@ class PartitionField(object): - def __init__(self, source_id, name, transform): + def __init__(self, source_id, field_id, name, transform): self.source_id = source_id + self.field_id = field_id self.name = name self.transform = transform @@ -29,7 +30,8 @@ def __eq__(self, other): elif other is None or not isinstance(other, PartitionField): return False - return self.source_id == other.source_id and self.name == other.name and self.transform == other.transform + return self.source_id == other.source_id and self.field_id == other.field_id and \ + self.name == other.name and self.transform == other.transform def __ne__(self, other): return not self.__eq__(other) @@ -38,4 +40,4 @@ def __hash__(self): return hash(self.__key()) def __key(self): - return PartitionField.__class__, self.source_id, self.name, self.transform + return PartitionField.__class__, self.source_id, self.field_id, self.name, self.transform diff --git a/iceberg/api/partition_spec.py b/iceberg/api/partition_spec.py index bcea93f2af..1451fbfe61 100644 --- a/iceberg/api/partition_spec.py +++ b/iceberg/api/partition_spec.py @@ -32,13 +32,13 @@ class PartitionSpec(object): @staticmethod def UNPARTITIONED_SPEC(): - return PartitionSpec(Schema(), 0, []) + return PartitionSpec(Schema(), 0, [], PartitionSpec.PARTITION_DATA_ID_START - 1) @staticmethod def unpartitioned(): return PartitionSpec.UNPARTITIONED_SPEC() - def __init__(self, schema, spec_id, fields): + def __init__(self, schema, spec_id, fields, last_assigned_field_id): self.fields_by_source_id = None self.fields_by_name = None self.__java_classes = None @@ -49,6 +49,7 @@ def __init__(self, schema, spec_id, fields): self.__fields = list() for field in fields: self.__fields.append(field) + self.last_assigned_field_id = last_assigned_field_id @property def fields(self): @@ -70,10 +71,10 @@ def get_field_by_source_id(self, field_id): def partition_type(self): struct_fields = list() - for i, field in enumerate(self.__fields): + for _i, field in enumerate(self.__fields): source_type = self.schema.find_type(field.source_id) result_type = field.transform.get_result_type(source_type) - struct_fields.append(NestedField.optional(PartitionSpec.PARTITION_DATA_ID_START + i, + struct_fields.append(NestedField.optional(field.field_id, field.name, result_type)) @@ -170,9 +171,10 @@ def __repr__(self): sb = ["["] for field in self.__fields: - sb.append("\n {name}: {transform}({source_id})".format(name=field.name, - transform=str(field.transform), - source_id=field.source_id)) + sb.append("\n {field_id}: {name}: {transform}({source_id})".format(field_id=field.field_id, + name=field.name, + transform=str(field.transform), + source_id=field.source_id)) if len(self.__fields) > 0: sb.append("\n") @@ -201,6 +203,11 @@ def __init__(self, schema): self.fields = list() self.partition_names = set() self.spec_id = 0 + self.last_assigned_field_id = PartitionSpec.PARTITION_DATA_ID_START - 1 + + def __next_field_id(self): + self.last_assigned_field_id = self.last_assigned_field_id + 1 + return self.last_assigned_field_id def with_spec_id(self, spec_id): self.spec_id = spec_id @@ -226,6 +233,7 @@ def identity(self, source_name): self.check_and_add_partition_name(source_name) source_column = self.find_source_column(source_name) self.fields.append(PartitionField(source_column.field_id, + self.__next_field_id(), source_name, Transforms.identity(source_column.type))) return self @@ -235,6 +243,7 @@ def year(self, source_name): self.check_and_add_partition_name(name) source_column = self.find_source_column(source_name) self.fields.append(PartitionField(source_column.field_id, + self.__next_field_id(), name, Transforms.year(source_column.type))) return self @@ -244,6 +253,7 @@ def month(self, source_name): self.check_and_add_partition_name(name) source_column = self.find_source_column(source_name) self.fields.append(PartitionField(source_column.field_id, + self.__next_field_id(), name, Transforms.month(source_column.type))) return self @@ -253,6 +263,7 @@ def day(self, source_name): self.check_and_add_partition_name(name) source_column = self.find_source_column(source_name) self.fields.append(PartitionField(source_column.field_id, + self.__next_field_id(), name, Transforms.day(source_column.type))) return self @@ -262,6 +273,7 @@ def hour(self, source_name): self.check_and_add_partition_name(name) source_column = self.find_source_column(source_name) self.fields.append(PartitionField(source_column.field_id, + self.__next_field_id(), name, Transforms.hour(source_column.type))) return self @@ -271,6 +283,7 @@ def bucket(self, source_name, num_buckets): self.check_and_add_partition_name(name) source_column = self.find_source_column(source_name) self.fields.append(PartitionField(source_column.field_id, + self.__next_field_id(), name, Transforms.bucket(source_column.type, num_buckets))) return self @@ -280,11 +293,15 @@ def truncate(self, source_name, width): self.check_and_add_partition_name(name) source_column = self.find_source_column(source_name) self.fields.append(PartitionField(source_column.field_id, + self.__next_field_id(), name, Transforms.truncate(source_column.type, width))) return self - def add(self, source_id, name, transform): + def add_without_field_id(self, source_id, name, transform): + return self.add(source_id, self.__next_field_id(), name, transform) + + def add(self, source_id, field_id, name, transform): self.check_and_add_partition_name(name) column = self.schema.find_field(source_id) if column is None: @@ -292,12 +309,14 @@ def add(self, source_id, name, transform): transform_obj = Transforms.from_string(column.type, transform) field = PartitionField(source_id, + field_id, name, transform_obj) self.fields.append(field) + self.last_assigned_field_id = max(self.last_assigned_field_id, field_id) return self def build(self): - spec = PartitionSpec(self.schema, self.spec_id, self.fields) + spec = PartitionSpec(self.schema, self.spec_id, self.fields, self.last_assigned_field_id) PartitionSpec.check_compatibility(spec, self.schema) return spec diff --git a/iceberg/api/transforms/transforms.py b/iceberg/api/transforms/transforms.py index 33877f1fd8..0cf243e0f9 100644 --- a/iceberg/api/transforms/transforms.py +++ b/iceberg/api/transforms/transforms.py @@ -47,7 +47,7 @@ def from_string(type_var, transform): if match is not None: name = match.group(1) - w = match.group(2) + w = int(match.group(2)) if name.lower() == "truncate": return Truncate.get(type_var, w) elif name.lower() == "bucket": diff --git a/iceberg/core/partition_spec_parser.py b/iceberg/core/partition_spec_parser.py index 600904d53a..ab57079c55 100644 --- a/iceberg/core/partition_spec_parser.py +++ b/iceberg/core/partition_spec_parser.py @@ -25,6 +25,7 @@ class PartitionSpecParser(object): SPEC_ID = "spec-id" FIELDS = "fields" SOURCE_ID = "source-id" + FIELD_ID = "field-id" TRANSFORM = "transform" NAME = "name" @@ -41,7 +42,8 @@ def to_dict(spec): def to_json_fields(spec): return [{PartitionSpecParser.NAME: field.name, PartitionSpecParser.TRANSFORM: str(field.transform), - PartitionSpecParser.SOURCE_ID: field.source_id} + PartitionSpecParser.SOURCE_ID: field.source_id, + PartitionSpecParser.FIELD_ID: field.field_id} for field in spec.fields] @staticmethod @@ -56,18 +58,8 @@ def from_json(schema, json_obj): builder = PartitionSpec.builder_for(schema).with_spec_id(spec_id) fields = json_obj.get(PartitionSpecParser.FIELDS) - if not isinstance(fields, (list, tuple)): - raise RuntimeError("Cannot parse partition spec fields, not an array: %s" % fields) - for element in fields: - if not isinstance(element, dict): - raise RuntimeError("Cannot parse partition field, not an object: %s" % element) - - builder.add(element.get(PartitionSpecParser.SOURCE_ID), - element.get(PartitionSpecParser.NAME), - element.get(PartitionSpecParser.TRANSFORM)) - - return builder.build() + return PartitionSpecParser.__build_from_json_fields(builder, fields) @staticmethod def from_json_fields(schema, spec_id, json_obj): @@ -76,14 +68,31 @@ def from_json_fields(schema, spec_id, json_obj): if isinstance(json_obj, str): json_obj = json.loads(json_obj) - if not isinstance(json_obj, list): - raise RuntimeError("Cannot parse partition spec fields, not an array: %s" % json_obj) + return PartitionSpecParser.__build_from_json_fields(builder, json_obj) + + @staticmethod + def __build_from_json_fields(builder, json_fields): + if not isinstance(json_fields, (list, tuple)): + raise RuntimeError("Cannot parse partition spec fields, not an array: %s" % json_fields) + + field_id_count = 0 + for element in json_fields: + if not isinstance(element, dict): + raise RuntimeError("Cannot parse partition field, not an object: %s" % element) - for item in json_obj: - if not isinstance(item, dict): - raise RuntimeError("Cannot parse partition field, not an object: %s" % json_obj) - builder.add(item.get(PartitionSpecParser.SOURCE_ID), - item.get(PartitionSpecParser.NAME), - item.get(PartitionSpecParser.TRANSFORM)) + if element.get(PartitionSpecParser.FIELD_ID) is not None: + builder.add(element.get(PartitionSpecParser.SOURCE_ID), + element.get(PartitionSpecParser.FIELD_ID), + element.get(PartitionSpecParser.NAME), + element.get(PartitionSpecParser.TRANSFORM)) + field_id_count = field_id_count + 1 + else: + builder.add_without_field_id(element.get(PartitionSpecParser.SOURCE_ID), + element.get(PartitionSpecParser.NAME), + element.get(PartitionSpecParser.TRANSFORM)) + + if field_id_count > 0 and field_id_count != len(json_fields): + raise RuntimeError("Cannot parse spec with missing field IDs: %s missing of %s fields." % + (len(json_fields) - field_id_count, len(json_fields))) return builder.build() diff --git a/tests/core/test_partition_spec_parser.py b/tests/core/test_partition_spec_parser.py index 53285362f6..256186cb84 100644 --- a/tests/core/test_partition_spec_parser.py +++ b/tests/core/test_partition_spec_parser.py @@ -16,21 +16,95 @@ # under the License. from iceberg.api import PartitionSpec, Schema -from iceberg.api.types import IntegerType, NestedField, StringType +from iceberg.api.types import DecimalType, IntegerType, NestedField, StringType from iceberg.core import PartitionSpecParser +import pytest def test_to_json_conversion(): spec_schema = Schema(NestedField.required(1, "id", IntegerType.get()), - NestedField.required(2, "data", StringType.get())) + NestedField.required(2, "data", StringType.get()), + NestedField.required(3, "num", DecimalType.of(9, 2))) - spec = PartitionSpec\ + spec = PartitionSpec \ .builder_for(spec_schema) \ - .identity("id")\ - .bucket("data", 16)\ + .identity("id") \ + .bucket("data", 16) \ + .add_without_field_id(2, "data1", "bucket[16]") \ + .add(2, 1010, "data2", "bucket[8]") \ + .bucket("num", 8) \ .build() expected = '{"spec-id": 0, "fields": [' \ - '{"name": "id", "transform": "identity", "source-id": 1}, ' \ - '{"name": "data_bucket", "transform": "bucket[16]", "source-id": 2}]}' + '{"name": "id", "transform": "identity", "source-id": 1, "field-id": 1000}, ' \ + '{"name": "data_bucket", "transform": "bucket[16]", "source-id": 2, "field-id": 1001}, ' \ + '{"name": "data1", "transform": "bucket[16]", "source-id": 2, "field-id": 1002}, ' \ + '{"name": "data2", "transform": "bucket[8]", "source-id": 2, "field-id": 1010}, ' \ + '{"name": "num_bucket", "transform": "bucket[8]", "source-id": 3, "field-id": 1011}]}' assert expected == PartitionSpecParser.to_json(spec) + + +def test_from_json_conversion_with_field_ids(): + spec_schema = Schema(NestedField.required(1, "id", IntegerType.get()), + NestedField.required(2, "data", StringType.get()), + NestedField.required(3, "num", DecimalType.of(9, 2))) + + spec_string = '{"spec-id": 0, "fields": [' \ + '{"name": "id", "transform": "identity", "source-id": 1, "field-id": 1000}, ' \ + '{"name": "data_bucket", "transform": "bucket[16]", "source-id": 2, "field-id": 1001}, ' \ + '{"name": "data1", "transform": "bucket[16]", "source-id": 2, "field-id": 1002}, ' \ + '{"name": "data2", "transform": "bucket[8]", "source-id": 2, "field-id": 1010}, ' \ + '{"name": "num_bucket", "transform": "bucket[8]", "source-id": 3, "field-id": 1011}]}' + + spec = PartitionSpecParser.from_json(spec_schema, spec_string) + + expected_spec = PartitionSpec \ + .builder_for(spec_schema) \ + .identity("id") \ + .bucket("data", 16) \ + .add_without_field_id(2, "data1", "bucket[16]") \ + .add(2, 1010, "data2", "bucket[8]") \ + .bucket("num", 8) \ + .build() + assert expected_spec == spec + + +def test_from_json_conversion_without_field_ids(): + spec_schema = Schema(NestedField.required(1, "id", IntegerType.get()), + NestedField.required(2, "data", StringType.get()), + NestedField.required(3, "num", DecimalType.of(9, 2))) + + spec_string = '{"spec-id": 0, "fields": [' \ + '{"name": "id", "transform": "identity", "source-id": 1}, ' \ + '{"name": "data_bucket", "transform": "bucket[16]", "source-id": 2}, ' \ + '{"name": "data1", "transform": "bucket[16]", "source-id": 2}, ' \ + '{"name": "data2", "transform": "bucket[8]", "source-id": 2}, ' \ + '{"name": "num_bucket", "transform": "bucket[8]", "source-id": 3}]}' + + spec = PartitionSpecParser.from_json(spec_schema, spec_string) + + expected_spec = PartitionSpec \ + .builder_for(spec_schema) \ + .identity("id") \ + .bucket("data", 16) \ + .add_without_field_id(2, "data1", "bucket[16]") \ + .add(2, 1003, "data2", "bucket[8]") \ + .bucket("num", 8) \ + .build() + assert expected_spec == spec + + +def test_raise_exception_with_invalid_json(): + spec_schema = Schema(NestedField.required(1, "id", IntegerType.get()), + NestedField.required(2, "data", StringType.get()), + NestedField.required(3, "num", DecimalType.of(9, 2))) + + spec_string = '{"spec-id": 0, "fields": [' \ + '{"name": "id", "transform": "identity", "source-id": 1, "field-id": 1000}, ' \ + '{"name": "data_bucket", "transform": "bucket[16]", "source-id": 2, "field-id": 1001}, ' \ + '{"name": "data1", "transform": "bucket[16]", "source-id": 2}, ' \ + '{"name": "data2", "transform": "bucket[8]", "source-id": 2}, ' \ + '{"name": "num_bucket", "transform": "bucket[8]", "source-id": 3}]}' + + with pytest.raises(RuntimeError): + PartitionSpecParser.from_json(spec_schema, spec_string) From 1dc08b4b42c1f3fba982ac58006edd794fc17f4e Mon Sep 17 00:00:00 2001 From: jun-he Date: Tue, 19 May 2020 21:48:14 -0700 Subject: [PATCH 027/642] Python: Update tests for partition field ID (#1049) --- tests/core/test_partition_spec.py | 62 ++++++++++++++++--------------- 1 file changed, 32 insertions(+), 30 deletions(-) diff --git a/tests/core/test_partition_spec.py b/tests/core/test_partition_spec.py index 09d108cf3c..f0f3b9b01a 100644 --- a/tests/core/test_partition_spec.py +++ b/tests/core/test_partition_spec.py @@ -70,39 +70,41 @@ def test_to_json_conversion(): PartitionSpec.builder_for(spec_schema).truncate("l", 10).build(), PartitionSpec.builder_for(spec_schema).truncate("dec", 10).build(), PartitionSpec.builder_for(spec_schema).truncate("s", 10).build(), - PartitionSpec.builder_for(spec_schema).add(6, "dec_bucket", "bucket[16]").build() + PartitionSpec.builder_for(spec_schema).add_without_field_id(6, "dec_bucket", "bucket[16]").build(), + PartitionSpec.builder_for(spec_schema).add(6, 1011, "dec_bucket", "bucket[16]").build(), ] expected_spec_strs = [ - "[\n i: identity(1)\n]", - "[\n l: identity(2)\n]", - "[\n d: identity(3)\n]", - "[\n t: identity(4)\n]", - "[\n ts: identity(5)\n]", - "[\n dec: identity(6)\n]", - "[\n s: identity(7)\n]", - "[\n u: identity(8)\n]", - "[\n f: identity(9)\n]", - "[\n b: identity(10)\n]", - "[\n i_bucket: bucket[128](1)\n]", - "[\n l_bucket: bucket[128](2)\n]", - "[\n d_bucket: bucket[128](3)\n]", - "[\n t_bucket: bucket[128](4)\n]", - "[\n ts_bucket: bucket[128](5)\n]", - "[\n dec_bucket: bucket[128](6)\n]", - "[\n s_bucket: bucket[128](7)\n]", - "[\n d_year: year(3)\n]", - "[\n d_month: month(3)\n]", - "[\n d_day: day(3)\n]", - "[\n ts_year: year(5)\n]", - "[\n ts_month: month(5)\n]", - "[\n ts_day: day(5)\n]", - "[\n ts_hour: hour(5)\n]", - "[\n i_truncate: truncate[10](1)\n]", - "[\n l_truncate: truncate[10](2)\n]", - "[\n dec_truncate: truncate[10](6)\n]", - "[\n s_truncate: truncate[10](7)\n]", - "[\n dec_bucket: bucket[16](6)\n]", + "[\n 1000: i: identity(1)\n]", + "[\n 1000: l: identity(2)\n]", + "[\n 1000: d: identity(3)\n]", + "[\n 1000: t: identity(4)\n]", + "[\n 1000: ts: identity(5)\n]", + "[\n 1000: dec: identity(6)\n]", + "[\n 1000: s: identity(7)\n]", + "[\n 1000: u: identity(8)\n]", + "[\n 1000: f: identity(9)\n]", + "[\n 1000: b: identity(10)\n]", + "[\n 1000: i_bucket: bucket[128](1)\n]", + "[\n 1000: l_bucket: bucket[128](2)\n]", + "[\n 1000: d_bucket: bucket[128](3)\n]", + "[\n 1000: t_bucket: bucket[128](4)\n]", + "[\n 1000: ts_bucket: bucket[128](5)\n]", + "[\n 1000: dec_bucket: bucket[128](6)\n]", + "[\n 1000: s_bucket: bucket[128](7)\n]", + "[\n 1000: d_year: year(3)\n]", + "[\n 1000: d_month: month(3)\n]", + "[\n 1000: d_day: day(3)\n]", + "[\n 1000: ts_year: year(5)\n]", + "[\n 1000: ts_month: month(5)\n]", + "[\n 1000: ts_day: day(5)\n]", + "[\n 1000: ts_hour: hour(5)\n]", + "[\n 1000: i_truncate: truncate[10](1)\n]", + "[\n 1000: l_truncate: truncate[10](2)\n]", + "[\n 1000: dec_truncate: truncate[10](6)\n]", + "[\n 1000: s_truncate: truncate[10](7)\n]", + "[\n 1000: dec_bucket: bucket[16](6)\n]", + "[\n 1011: dec_bucket: bucket[16](6)\n]", ] for (spec, expected_spec_str) in zip(specs, expected_spec_strs): From 5ce2edf428d08c1b9168e5283a0783d18a0e256f Mon Sep 17 00:00:00 2001 From: Fokko Driesprong Date: Thu, 21 May 2020 19:25:39 +0200 Subject: [PATCH 028/642] Python: Enable mypy type validation (#1041) --- iceberg/api/types/type_util.py | 10 +++++----- iceberg/core/base_table_scan.py | 4 ++-- iceberg/core/manifest_reader.py | 4 ++-- tox.ini | 13 ++++++++++--- 4 files changed, 19 insertions(+), 12 deletions(-) diff --git a/iceberg/api/types/type_util.py b/iceberg/api/types/type_util.py index ca52a7db9c..68f0f595ac 100644 --- a/iceberg/api/types/type_util.py +++ b/iceberg/api/types/type_util.py @@ -16,6 +16,7 @@ # under the License. import math +from typing import List from .type import (Type, TypeID) @@ -238,7 +239,6 @@ def get(self): return self.visitor.field(self.field, VisitFuture(self.field.type, self.visitor).get) -@staticmethod def decimal_required_bytes(precision): if precision < 0 or precision > 40: raise RuntimeError("Unsupported decimal precision: %s" % precision) @@ -451,11 +451,11 @@ def write_compatibility_errors(read_schema, write_schema): def read_compatibility_errors(read_schema, write_schema): visit(write_schema, CheckCompatibility(read_schema, False)) - NO_ERRORS = [] + NO_ERRORS: List[str] = [] def __init__(self, schema, check_ordering): self.schema = schema - self.check_ordering + self.check_ordering = check_ordering self.current_type = None def schema(self, schema, struct_result): @@ -498,10 +498,10 @@ def struct(self, struct, field_results): return errors - def field(self, field, field_result): + def field(self, field, field_result) -> List[str]: struct = self.current_type.as_struct_type() curr_field = struct.field(field.field_id) - errors = list() + errors = [] if curr_field is None: if not field.is_optional: diff --git a/iceberg/core/base_table_scan.py b/iceberg/core/base_table_scan.py index d205364aad..e64038d313 100644 --- a/iceberg/core/base_table_scan.py +++ b/iceberg/core/base_table_scan.py @@ -34,9 +34,9 @@ class BaseTableScan(CloseableGroup, TableScan): DATE_FORMAT = "%Y-%m-%d %H:%M:%S.%f" - SNAPSHOT_COLUMNS = ["snapshot_id", "file_path", "file_ordinal", "file_format", "block_size_in_bytes", + SNAPSHOT_COLUMNS = ("snapshot_id", "file_path", "file_ordinal", "file_format", "block_size_in_bytes", "file_size_in_bytes", "record_count", "partition", "value_counts", "null_value_counts", - "lower_bounds", "upper_bounds"] + "lower_bounds", "upper_bounds") def new_refined_scan(self, ops, table, schema, snapshot_id, row_filter, case_sensitive, selected_columns, options, minused_cols): diff --git a/iceberg/core/manifest_reader.py b/iceberg/core/manifest_reader.py index 6a4581293a..e648684f29 100644 --- a/iceberg/core/manifest_reader.py +++ b/iceberg/core/manifest_reader.py @@ -33,8 +33,8 @@ class ManifestReader(CloseableGroup, Filterable): - ALL_COLUMNS = ["*"] - CHANGE_COLUMNS = ["file_path", "file_format", "partition", "record_count", "file_size_in_bytes"] + ALL_COLUMNS = ("*",) + CHANGE_COLUMNS = ("file_path", "file_format", "partition", "record_count", "file_size_in_bytes") @staticmethod def read(file, spec_lookup=None): diff --git a/tox.ini b/tox.ini index 9c49b0bd33..b011511130 100644 --- a/tox.ini +++ b/tox.ini @@ -39,9 +39,11 @@ deps = . {[testenv:flake8]deps} {[testenv:bandit]deps} + {[testenv:mypy]deps} commands = {[testenv:flake8]commands} {[testenv:bandit]commands} + {[testenv:mypy]commands} [testenv:flake8] basepython = python3 @@ -53,6 +55,14 @@ deps = commands = flake8 iceberg setup.py tests +[testenv:mypy] +basepython = python3 +skip_install = true +deps = + mypy +commands = + mypy --ignore-missing-imports iceberg/ + [testenv:bandit] basepython = python3 skip_install = true @@ -77,9 +87,6 @@ commands = # commands = # python -m http.server {posargs} -[bandit] -skips = B104 - [flake8] ignore = E501,W503 exclude = From 74df3fc8d2db2101ed58b7bea78107f3854aeb07 Mon Sep 17 00:00:00 2001 From: Ryan Blue Date: Thu, 28 May 2020 08:39:25 -0700 Subject: [PATCH 029/642] Replace references for incubator-iceberg with iceberg repository (#1062) --- README.md | 6 +++--- setup.py | 4 ++-- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/README.md b/README.md index c2e612f1ec..0d4a3475cd 100644 --- a/README.md +++ b/README.md @@ -24,8 +24,8 @@ Iceberg is a python library for programatic access to iceberg table metadata as Iceberg python is currently in development, for development and testing purposes the best way to install the library is to perform the following steps: ``` -git clone https://github.com/apache/incubator-iceberg.git -cd incubator-iceberg/python +git clone https://github.com/apache/iceberg.git +cd iceberg/python pip install -e . ``` @@ -44,5 +44,5 @@ tox * [dev@iceberg.apache.org](mailto:dev@iceberg.apache.org) - Issues - * [File a github incident](https://github.com/apache/incubator-iceberg/issues) + * [File a github incident](https://github.com/apache/iceberg/issues) diff --git a/setup.py b/setup.py index b1d5c8c8c3..363d61a386 100755 --- a/setup.py +++ b/setup.py @@ -20,11 +20,11 @@ setup( name='iceberg', - maintainer='Apache Incubator Iceberg Devs', + maintainer='Apache Iceberg Devs', author_email='dev@iceberg.apache.org', description='Iceberg is a new table format for storing large, slow-moving tabular data', keywords='iceberg', - url='https://github.com/apache/incubator-iceberg/blob/master/README.md', + url='https://github.com/apache/iceberg/blob/master/README.md', python_requires='>=3.6', install_requires=['botocore', 'boto3', From 039355d413da4b857d815776c62c89a67392cfb3 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Tue, 21 Jul 2020 17:12:43 +0100 Subject: [PATCH 030/642] Python: Minor fixes for tests (#1214) In the course of implementing create table I came across minor issues: * rename test fixtures to suppress warnings and prevent pytest from treating them as tests * test partition code path & fix incorrect signature when creating metadata * handle `file:` as well as `file://` as it seems both schemas are used * add mypy annotations in areas where I fixed code * fix tox/mypy --- iceberg/api/partition_spec.py | 4 ++-- iceberg/api/types/type_util.py | 5 +++-- iceberg/core/filesystem/local_filesystem.py | 7 +++---- iceberg/core/table_metadata.py | 9 ++++++--- tests/api/expressions/conftest.py | 10 +++++----- tests/api/test_helpers.py | 2 +- tests/core/conftest.py | 15 +++++++++------ tests/hive/test_hive_tables.py | 14 +++++++------- 8 files changed, 36 insertions(+), 30 deletions(-) diff --git a/iceberg/api/partition_spec.py b/iceberg/api/partition_spec.py index 1451fbfe61..395001b601 100644 --- a/iceberg/api/partition_spec.py +++ b/iceberg/api/partition_spec.py @@ -183,7 +183,7 @@ def __repr__(self): return "".join(sb) @staticmethod - def builder_for(schema): + def builder_for(schema: Schema) -> "PartitionSpecBuilder": return PartitionSpecBuilder(schema) @staticmethod @@ -301,7 +301,7 @@ def truncate(self, source_name, width): def add_without_field_id(self, source_id, name, transform): return self.add(source_id, self.__next_field_id(), name, transform) - def add(self, source_id, field_id, name, transform): + def add(self, source_id: int, field_id: int, name: str, transform: str) -> "PartitionSpecBuilder": self.check_and_add_partition_name(name) column = self.schema.find_field(source_id) if column is None: diff --git a/iceberg/api/types/type_util.py b/iceberg/api/types/type_util.py index 68f0f595ac..f7f8b138f6 100644 --- a/iceberg/api/types/type_util.py +++ b/iceberg/api/types/type_util.py @@ -505,14 +505,14 @@ def field(self, field, field_result) -> List[str]: if curr_field is None: if not field.is_optional: - errors.add("{} is required, but is missing".format(field.name)) + errors.append("{} is required, but is missing".format(field.name)) return self.NO_ERRORS self.current_type = curr_field.type try: if not field.is_optional and curr_field.is_optional: - errors.add(field.name + " should be required, but is optional") + errors.append(field.name + " should be required, but is optional") for error in field_result: if error.startswith(":"): @@ -525,6 +525,7 @@ def field(self, field, field_result) -> List[str]: pass finally: self.current_type = struct + return errors def list(self, list_var, element_result): raise NotImplementedError() diff --git a/iceberg/core/filesystem/local_filesystem.py b/iceberg/core/filesystem/local_filesystem.py index 826700a774..2b74309f48 100644 --- a/iceberg/core/filesystem/local_filesystem.py +++ b/iceberg/core/filesystem/local_filesystem.py @@ -18,6 +18,7 @@ import errno import os from pathlib import Path +from urllib.parse import urlparse from .file_status import FileStatus from .file_system import FileSystem @@ -58,10 +59,8 @@ def stat(self, path): permission=st.st_mode, owner=st.st_uid, group=st.st_gid) @staticmethod - def fix_path(path): - if path.startswith("file://"): - path = str(path[7:]) - return path + def fix_path(path: str) -> str: + return urlparse(path).path def create(self, path, overwrite=False): if os.path.exists(path) and not overwrite: diff --git a/iceberg/core/table_metadata.py b/iceberg/core/table_metadata.py index b289e440a8..8f921e64f9 100644 --- a/iceberg/core/table_metadata.py +++ b/iceberg/core/table_metadata.py @@ -17,8 +17,9 @@ import time -from iceberg.api import PartitionSpec +from iceberg.api import PartitionSpec, Schema from iceberg.api.types import assign_fresh_ids +from iceberg.core.table_operations import TableOperations from iceberg.core.util import AtomicInteger from iceberg.exceptions import ValidationException @@ -28,14 +29,16 @@ class TableMetadata(object): TABLE_FORMAT_VERSION = 1 @staticmethod - def new_table_metadata(ops, schema, spec, location, properties=None): + def new_table_metadata(ops: TableOperations, schema: Schema, spec: PartitionSpec, location: str, + properties: dict = None) -> "TableMetadata": last_column_id = AtomicInteger(0) fresh_schema = assign_fresh_ids(schema, last_column_id.increment_and_get) spec_builder = PartitionSpec.builder_for(fresh_schema) for field in spec.fields: src_name = schema.find_column_name(field.source_id) - spec_builder.add(fresh_schema.find_field(src_name).field_id, + spec_builder.add(field.source_id, + fresh_schema.find_field(src_name).field_id, field.name, str(field.transform)) diff --git a/tests/api/expressions/conftest.py b/tests/api/expressions/conftest.py index 3ee4b4cd5e..28cc43d239 100644 --- a/tests/api/expressions/conftest.py +++ b/tests/api/expressions/conftest.py @@ -93,7 +93,7 @@ def predicate(self, pred): raise AssertionError("Predicate should be a BoundPredicate") -class TestDataFile(DataFile): +class MockDataFile(DataFile): def __init__(self, path, partition, record_count, value_counts=None, null_value_counts=None, lower_bounds=None, upper_bounds=None): @@ -202,7 +202,7 @@ def strict_schema(): @pytest.fixture(scope="session") def file(): - return TestDataFile("file.avro", TestHelpers.Row.of(), 50, + return MockDataFile("file.avro", TestHelpers.Row.of(), 50, # value counts {4: 50, 5: 50, 6: 50}, # null value counts @@ -215,7 +215,7 @@ def file(): @pytest.fixture(scope="session") def strict_file(): - return TestDataFile("file.avro", + return MockDataFile("file.avro", TestHelpers.Row.of(), 50, {4: 50, 5: 50, 6: 50}, @@ -229,12 +229,12 @@ def strict_file(): @pytest.fixture(scope="session") def missing_stats(): - return TestDataFile("file.parquet", TestHelpers.Row.of(), 50) + return MockDataFile("file.parquet", TestHelpers.Row.of(), 50) @pytest.fixture(scope="session") def empty(): - return TestDataFile("file.parquet", TestHelpers.Row.of(), record_count=0) + return MockDataFile("file.parquet", TestHelpers.Row.of(), record_count=0) @pytest.fixture(scope="session") diff --git a/tests/api/test_helpers.py b/tests/api/test_helpers.py index 7d51e5f02a..2f75690efc 100644 --- a/tests/api/test_helpers.py +++ b/tests/api/test_helpers.py @@ -74,7 +74,7 @@ def predicate(self, pred): return None -class TestDataFile(DataFile): +class MockDataFile(DataFile): def __init__(self, path, partition, record_count, value_counts=None, null_value_counts=None, lower_bounds=None, upper_bounds=None): diff --git a/tests/core/conftest.py b/tests/core/conftest.py index e446c63ba2..8d414ac790 100644 --- a/tests/core/conftest.py +++ b/tests/core/conftest.py @@ -20,7 +20,7 @@ import tempfile import time -from iceberg.api import Files, PartitionSpec, Schema +from iceberg.api import Files, PartitionSpec, PartitionSpecBuilder, Schema from iceberg.api.types import BooleanType, IntegerType, LongType, NestedField, StringType from iceberg.core import (BaseSnapshot, BaseTable, @@ -99,7 +99,7 @@ def __init__(self, table_name, location): self._current = None self.refresh() if self._current is not None: - for snap in self.current.snapshots: + for snap in self.current().snapshots: self.last_snapshot_id = max(self.last_snapshot_id, snap.snapshot_id) def current(self): @@ -284,11 +284,14 @@ def base_scan_schema(): NestedField.required(2, "data", StringType.get())]) -@pytest.fixture(scope="session") -def ts_table(base_scan_schema): +@pytest.fixture(scope="session", params=["none", "one"]) +def ts_table(base_scan_schema, request): with tempfile.TemporaryDirectory() as td: - spec = PartitionSpec.unpartitioned() - return TestTables.create(td, "test", base_scan_schema, spec) + if request.param == "none": + spec = PartitionSpec.unpartitioned() + else: + spec = PartitionSpecBuilder(base_scan_schema).add(1, 1000, "id", "identity").build() + return TestTables.create(td, "test-" + request.param, base_scan_schema, spec) @pytest.fixture(scope="session") diff --git a/tests/hive/test_hive_tables.py b/tests/hive/test_hive_tables.py index cba43378b3..f979d098d6 100644 --- a/tests/hive/test_hive_tables.py +++ b/tests/hive/test_hive_tables.py @@ -22,7 +22,7 @@ from pytest import raises -class TestHMSTable(object): +class MockHMSTable(object): def __init__(self, params): self.parameters = params @@ -35,7 +35,7 @@ def test_load_tables_check_valid_props(client, current_call, refresh_call): "partition_spec": [], "metadata_location": "s3://path/to/iceberg.metadata.json"} - client.return_value.__enter__.return_value.get_table.return_value = TestHMSTable(parameters) + client.return_value.__enter__.return_value.get_table.return_value = MockHMSTable(parameters) conf = {"hive.metastore.uris": 'thrift://hms:port'} tables = HiveTables(conf) @@ -49,7 +49,7 @@ def test_load_tables_check_missing_iceberg_type(client, current_call, refresh_ca parameters = {"partition_spec": [], "metadata_location": "s3://path/to/iceberg.metdata.json"} - client.return_value.__enter__.return_value.get_table.return_value = TestHMSTable(parameters) + client.return_value.__enter__.return_value.get_table.return_value = MockHMSTable(parameters) conf = {"hive.metastore.uris": 'thrift://hms:port'} tables = HiveTables(conf) @@ -65,7 +65,7 @@ def test_load_tables_check_non_iceberg_type(client, current_call, refresh_call): "partition_spec": [], "metadata_location": "s3://path/to/iceberg.metdata.json"} - client.return_value.__enter__.return_value.get_table.return_value = TestHMSTable(parameters) + client.return_value.__enter__.return_value.get_table.return_value = MockHMSTable(parameters) conf = {"hive.metastore.uris": 'thrift://hms:port'} tables = HiveTables(conf) @@ -81,7 +81,7 @@ def test_load_tables_check_none_table_type(client, current_call, refresh_call): "partition_spec": [], "metadata_location": "s3://path/to/iceberg.metdata.json"} - client.return_value.__enter__.return_value.get_table.return_value = TestHMSTable(parameters) + client.return_value.__enter__.return_value.get_table.return_value = MockHMSTable(parameters) conf = {"hive.metastore.uris": 'thrift://hms:port'} tables = HiveTables(conf) @@ -96,7 +96,7 @@ def test_load_tables_check_no_location(client, current_call, refresh_call): parameters = {"table_type": "ICEBERG", "partition_spec": []} - client.return_value.__enter__.return_value.get_table.return_value = TestHMSTable(parameters) + client.return_value.__enter__.return_value.get_table.return_value = MockHMSTable(parameters) conf = {"hive.metastore.uris": 'thrift://hms:port'} tables = HiveTables(conf) @@ -112,7 +112,7 @@ def test_load_tables_check_none_location(client, current_call, refresh_call): "partition_spec": [], "metadata_location": None} - client.return_value.__enter__.return_value.get_table.return_value = TestHMSTable(parameters) + client.return_value.__enter__.return_value.get_table.return_value = MockHMSTable(parameters) conf = {"hive.metastore.uris": 'thrift://hms:port'} tables = HiveTables(conf) From f7ea008bc8098335f55c9a14b8f040c84c708c08 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Tue, 25 Aug 2020 01:20:33 +0100 Subject: [PATCH 031/642] Python: Add 3.7 and 3.8 to CI testing (#1369) --- tox.ini | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/tox.ini b/tox.ini index b011511130..29868f7a7d 100644 --- a/tox.ini +++ b/tox.ini @@ -14,7 +14,7 @@ # limitations under the License. [tox] -envlist = py36,linters +envlist = py36,py37,py38,linters [testenv] deps = @@ -105,3 +105,9 @@ application-import-names = flake8 [pytest] norecursedirs=.* + +[travis] +python = + 3.6: py36, linters + 3.7: py37 + 3.8: py38 \ No newline at end of file From a50b7e1806a6aa64175d5e0ae6aa36105e95e217 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Wed, 26 Aug 2020 16:53:53 +0100 Subject: [PATCH 032/642] Python: Support creating tables (#1216) --- iceberg/api/tables.py | 2 +- iceberg/core/__init__.py | 4 +- iceberg/core/base_metastore_tables.py | 66 +++++-- iceberg/core/base_transaction.py | 2 +- iceberg/core/filesystem/filesystem_tables.py | 48 ++++-- iceberg/core/filesystem/local_filesystem.py | 25 +-- iceberg/core/table_metadata.py | 13 +- iceberg/hive/hive_table_operations.py | 171 +++++++++++++++++-- iceberg/hive/hive_tables.py | 14 +- iceberg/hive/hive_types.py | 37 ++++ tests/core/conftest.py | 20 ++- tests/core/test_filesystem_tables.py | 32 ++++ tests/hive/conftest.py | 19 +++ tests/hive/test_hive_tables.py | 39 +++++ 14 files changed, 408 insertions(+), 84 deletions(-) create mode 100644 iceberg/hive/hive_types.py create mode 100644 tests/core/test_filesystem_tables.py diff --git a/iceberg/api/tables.py b/iceberg/api/tables.py index 1a0b22e448..0210437dfc 100644 --- a/iceberg/api/tables.py +++ b/iceberg/api/tables.py @@ -20,7 +20,7 @@ class Tables(object): - def create(self, schema, table_identifier=None, spec=None, properties=None): + def create(self, schema, table_identifier, spec=None, properties=None, location=None): raise NotImplementedError() def load(self, table_identifier): diff --git a/iceberg/core/__init__.py b/iceberg/core/__init__.py index 1d38c1a404..d6079a5816 100644 --- a/iceberg/core/__init__.py +++ b/iceberg/core/__init__.py @@ -14,6 +14,7 @@ # KIND, either express or implied. See the License for the # specific language governing permissions and limitations # under the License. +# flake8: noqa __all__ = ["BaseMetastoreTableOperations", "BaseMetastoreTables", @@ -31,13 +32,13 @@ "SchemaParser", "SchemaUpdate", "SnapshotParser", - "TableOperations", "SnapshotLogEntry", "TableMetadata", "TableMetadataParser", "TableOperations", "TableProperties"] +from .table_operations import TableOperations # out of order import to avoid circular deps from .base_metastore_table_operations import BaseMetastoreTableOperations from .base_metastore_tables import BaseMetastoreTables from .base_snapshot import BaseSnapshot @@ -57,5 +58,4 @@ from .table_metadata import (SnapshotLogEntry, TableMetadata) from .table_metadata_parser import TableMetadataParser -from .table_operations import TableOperations from .table_properties import TableProperties diff --git a/iceberg/core/base_metastore_tables.py b/iceberg/core/base_metastore_tables.py index 17bdde9bb6..a78041c650 100644 --- a/iceberg/core/base_metastore_tables.py +++ b/iceberg/core/base_metastore_tables.py @@ -14,40 +14,72 @@ # KIND, either express or implied. See the License for the # specific language governing permissions and limitations # under the License. +from typing import Tuple -from iceberg.api import Tables -from iceberg.exceptions import NoSuchTableException - +from . import TableOperations from .base_table import BaseTable +from .table_metadata import TableMetadata +from ..api import PartitionSpec, Schema, Table, Tables +from ..exceptions import AlreadyExistsException, CommitFailedException, NoSuchTableException class BaseMetastoreTables(Tables): - def __init__(self, conf): + def __init__(self: "BaseMetastoreTables", conf: dict) -> None: self.conf = conf - def new_table_ops(self, conf, database, table): + def new_table_ops(self: "BaseMetastoreTables", conf: dict, database: str, table: str) -> "TableOperations": raise RuntimeError("Abstract Implementation") - def load(self, database, table): + def load(self: "BaseMetastoreTables", table_identifier: str) -> Table: + database, table = _parse_table_identifier(table_identifier) ops = self.new_table_ops(self.conf, database, table) - if ops.current() is None: - raise NoSuchTableException("Table does not exist: {}.{}".format(database, table)) + if ops.current(): + return BaseTable(ops, "{}.{}".format(database, table)) + raise NoSuchTableException("Table does not exist: {}.{}".format(database, table)) - return BaseTable(ops, "{}.{}".format(database, table)) + def create(self: "BaseMetastoreTables", schema: Schema, table_identifier: str, spec: PartitionSpec = None, + properties: dict = None, location: str = None) -> Table: + database, table = _parse_table_identifier(table_identifier) + ops = self.new_table_ops(self.conf, database, table) + if ops.current(): # not None check here to ensure MagicMocks aren't treated as None + raise AlreadyExistsException("Table already exists: " + table_identifier) - def create(self, schema, spec, table_identifier=None, database=None, table=None): - raise RuntimeError("Not Yet Implemented") + base_location = location if location else self.default_warehouse_location(self.conf, database, table) + full_spec, properties = super(BaseMetastoreTables, self).default_args(spec, properties) + metadata = TableMetadata.new_table_metadata(ops, schema, full_spec, base_location, properties) + + try: + ops.commit(None, metadata) + except CommitFailedException: + raise AlreadyExistsException("Table was created concurrently: " + table_identifier) - def begin_create(self, schema, spec, database, table_name, properties=None): + return BaseTable(ops, "{}.{}".format(database, table)) + + def begin_create(self: "BaseMetastoreTables", schema: Schema, spec: PartitionSpec, database: str, table_name: str, + properties: dict = None): raise RuntimeError("Not Yet Implemented") - def begin_replace(self, schema, spec, database, table, properties=None): + def begin_replace(self: "BaseMetastoreTables", schema: Schema, spec: PartitionSpec, database: str, table: str, + properties: dict = None): raise RuntimeError("Not Yet Implemented") - def default_warehouse_location(self, conf, database, table): + def default_warehouse_location(self: "BaseMetastoreTables", conf: dict, database: str, table: str) -> str: warehouse_location = conf.get("hive.metastore.warehouse.dir") - if warehouse_location is None: - raise RuntimeError("Warehouse location is not set: hive.metastore.warehouse.dir=null") + if warehouse_location: + return f"{warehouse_location}/{database}.db/{table}" + raise RuntimeError("Warehouse location is not set: hive.metastore.warehouse.dir=null") + + +_DOT = '.' + - return f"{warehouse_location}/{database}.db/{table}" +def _parse_table_identifier(table_identifier: str) -> Tuple[str, str]: + parts = table_identifier.rsplit(_DOT, 1) + if len(parts) > 1: + database = parts[0] + table = parts[1] + else: + database = "default" + table = parts[0] + return database, table diff --git a/iceberg/core/base_transaction.py b/iceberg/core/base_transaction.py index 3d4f711b56..19616f3b7b 100644 --- a/iceberg/core/base_transaction.py +++ b/iceberg/core/base_transaction.py @@ -93,7 +93,7 @@ def refresh(self): return self._bt.current def commit(self, base, metadata): - if base != self._current: + if base != self.current(): raise CommitFailedException("Table metadata refresh is required") old_id = BaseTransaction.current_id(self._bt.current) diff --git a/iceberg/core/filesystem/filesystem_tables.py b/iceberg/core/filesystem/filesystem_tables.py index 77bd75140d..113495f764 100644 --- a/iceberg/core/filesystem/filesystem_tables.py +++ b/iceberg/core/filesystem/filesystem_tables.py @@ -15,39 +15,51 @@ # specific language governing permissions and limitations # under the License. - -from iceberg.api import Tables -from iceberg.exceptions import NoSuchTableException - from .filesystem_table_operations import FilesystemTableOperations +from .. import TableOperations from ..table_metadata import TableMetadata +from ...api import PartitionSpec, Schema, Table, Tables +from ...exceptions import NoSuchTableException class FilesystemTables(Tables): - def __init__(self, conf=None): + def __init__(self: "FilesystemTables", conf: dict = None) -> None: self.conf = conf if conf is not None else dict() - def load(self, location): + def load(self: "FilesystemTables", table_identifier: str) -> Table: from ..base_table import BaseTable - ops = self.new_table_ops(location) + ops = self.new_table_ops(table_identifier) if ops.current() is None: - raise NoSuchTableException("Table does not exist at location: %s" % location) + raise NoSuchTableException("Table does not exist at location: %s" % table_identifier) + + return BaseTable(ops, table_identifier) - return BaseTable(ops, location) + def create(self: "FilesystemTables", schema: Schema, table_identifier: str, spec: PartitionSpec = None, + properties: dict = None, location: str = None) -> Table: + """ + Create a new table on the filesystem. - def create(self, schema, table_identifier=None, spec=None, properties=None, location=None): + Note: it is expected that the filesystem has atomic operations to ensure consistency for metadata updates. + Filesystems that don't have this guarantee could lead to data loss. + + Location should always be None as the table location on disk is taken from `table_identifier` + """ from ..base_table import BaseTable - spec, properties = super(FilesystemTables, self).default_args(spec, properties) - ops = self.new_table_ops(location) + if location: + raise RuntimeError("""location has to be None. Both table_identifier and location have been declared. + table_identifier: {} and location: {}""".format(table_identifier, location)) + + full_spec, properties = super(FilesystemTables, self).default_args(spec, properties) + ops = self.new_table_ops(table_identifier) - metadata = TableMetadata.new_table_metadata(ops, schema, spec, location, properties) + metadata = TableMetadata.new_table_metadata(ops, schema, full_spec, table_identifier, properties) ops.commit(None, metadata) - return BaseTable(ops, location) + return BaseTable(ops, table_identifier) - def new_table_ops(self, location): - if location is None: - raise RuntimeError("location cannot be None") + def new_table_ops(self: "FilesystemTables", table_identifier: str) -> TableOperations: + if table_identifier is None: + raise RuntimeError("table_identifier cannot be None") - return FilesystemTableOperations(location, self.conf) + return FilesystemTableOperations(table_identifier, self.conf) diff --git a/iceberg/core/filesystem/local_filesystem.py b/iceberg/core/filesystem/local_filesystem.py index 2b74309f48..ac03b4c733 100644 --- a/iceberg/core/filesystem/local_filesystem.py +++ b/iceberg/core/filesystem/local_filesystem.py @@ -18,6 +18,7 @@ import errno import os from pathlib import Path +import stat from urllib.parse import urlparse from .file_status import FileStatus @@ -28,16 +29,16 @@ class LocalFileSystem(FileSystem): fs_inst = None @staticmethod - def get_instance(): - if LocalFileSystem.fs_inst is None: - LocalFileSystem() + def get_instance() -> "LocalFileSystem": + if not LocalFileSystem.fs_inst: + return LocalFileSystem() return LocalFileSystem.fs_inst - def __init__(self): + def __init__(self: "LocalFileSystem") -> None: if LocalFileSystem.fs_inst is None: LocalFileSystem.fs_inst = self - def open(self, path, mode='rb'): + def open(self: "LocalFileSystem", path: str, mode: str = 'rb') -> object: open_path = Path(LocalFileSystem.fix_path(path)) if "w" in mode and not open_path.parents[0].exists(): @@ -49,12 +50,12 @@ def open(self, path, mode='rb'): return open(open_path, mode=mode) - def delete(self, path): - raise NotImplementedError() + def delete(self: "LocalFileSystem", path: str) -> None: + os.remove(path) - def stat(self, path): + def stat(self: "LocalFileSystem", path: str) -> FileStatus: st = os.stat(LocalFileSystem.fix_path(path)) - return FileStatus(path=path, length=st.st_size, is_dir=os.stat.S_ISDIR(st.st_mode), + return FileStatus(path=path, length=st.st_size, is_dir=stat.S_ISDIR(st.st_mode), blocksize=st.st_blksize, modification_time=st.st_mtime, access_time=st.st_atime, permission=st.st_mode, owner=st.st_uid, group=st.st_gid) @@ -62,13 +63,13 @@ def stat(self, path): def fix_path(path: str) -> str: return urlparse(path).path - def create(self, path, overwrite=False): + def create(self: "LocalFileSystem", path: str, overwrite: bool = False) -> object: if os.path.exists(path) and not overwrite: raise RuntimeError("Path %s already exists" % path) return open(path, "w") - def rename(self, src, dest): + def rename(self: "LocalFileSystem", src: str, dest: str) -> bool: try: os.rename(src, dest) except OSError: @@ -76,5 +77,5 @@ def rename(self, src, dest): return True - def exists(self, path): + def exists(self: "LocalFileSystem", path: str) -> bool: return os.path.exists(path) diff --git a/iceberg/core/table_metadata.py b/iceberg/core/table_metadata.py index 8f921e64f9..92cb39e34e 100644 --- a/iceberg/core/table_metadata.py +++ b/iceberg/core/table_metadata.py @@ -16,6 +16,7 @@ # under the License. import time +from typing import Optional from iceberg.api import PartitionSpec, Schema from iceberg.api.types import assign_fresh_ids @@ -54,8 +55,8 @@ def __init__(self, ops, file, location, last_updated_millis, last_column_id, schema, default_spec_id, specs, properties, current_snapshot_id, snapshots, snapshot_log): self.ops = ops - self.file = file - self.location = location + self._file = file + self._location = location self.last_updated_millis = last_updated_millis self.last_column_id = last_column_id self.schema = schema @@ -80,6 +81,14 @@ def __init__(self, ops, file, location, last_updated_millis, if not (len(self.snapshot_by_id) == 0 or self.current_snapshot_id in self.snapshot_by_id): raise RuntimeError("Invalid table metadata: Cannot find current version") + @property + def location(self: "TableMetadata") -> str: + return self._location + + @property + def metadata_location(self: "TableMetadata") -> Optional[str]: + return self._file.location() if self._file else None + @property def spec(self): return self.specs_by_id[self.default_spec_id] diff --git a/iceberg/hive/hive_table_operations.py b/iceberg/hive/hive_table_operations.py index ed1b867210..d78e6aaa7e 100644 --- a/iceberg/hive/hive_table_operations.py +++ b/iceberg/hive/hive_table_operations.py @@ -16,32 +16,50 @@ # specific language governing permissions and limitations # under the License. # +import getpass +import logging +import socket +import time +from typing import List +from hmsclient import HMSClient +from hmsclient.genthrift.hive_metastore.ttypes import AlreadyExistsException, EnvironmentContext, FieldSchema, \ + LockComponent, LockLevel, LockRequest, LockResponse, LockState, LockType, NoSuchObjectException, \ + SerDeInfo, StorageDescriptor, Table, TException -from iceberg.core import BaseMetastoreTableOperations +from .hive_types import hive_types +from ..api import Schema +from ..api.types import Type +from ..core import BaseMetastoreTableOperations, TableMetadata +from ..core.filesystem import FileSystem, get_fs +from ..exceptions import AlreadyExistsException as IcebergAlreadyExistsException +from ..exceptions import CommitFailedException class HiveTableOperations(BaseMetastoreTableOperations): - def __init__(self, conf, client, database, table): + def __init__(self: "HiveTableOperations", conf: dict, client: HMSClient, database: str, table: str) -> None: super(HiveTableOperations, self).__init__(conf) self._client = client self.database = database self.table = table - self.refresh() + try: + self.refresh() + except NoSuchObjectException: + pass # Object hasn't been created yet, can't refresh. - def refresh(self): + def refresh(self: "HiveTableOperations") -> TableMetadata: with self._client as open_client: tbl_info = open_client.get_table(self.database, self.table) table_type = tbl_info.parameters.get(BaseMetastoreTableOperations.TABLE_TYPE_PROP) - if table_type is None or table_type.lower() != BaseMetastoreTableOperations.ICEBERG_TABLE_TYPE_VALUE: + if not table_type or table_type.lower() != BaseMetastoreTableOperations.ICEBERG_TABLE_TYPE_VALUE: raise RuntimeError("Invalid table, not Iceberg: %s.%s" % (self.database, self.table)) metadata_location = tbl_info.parameters.get(BaseMetastoreTableOperations.METADATA_LOCATION_PROP) - if metadata_location is None: + if not metadata_location: raise RuntimeError("Invalid table, missing metadata_location: %s.%s" % (self.database, self.table)) @@ -49,11 +67,142 @@ def refresh(self): return self.current() - def commit(self, base, metadata): - raise NotImplementedError() + def commit(self: "HiveTableOperations", base: TableMetadata, metadata: TableMetadata) -> None: + new_metadata_location = self.write_new_metadata(metadata, self.version + 1) - def io(self): - raise NotImplementedError() + threw = True + lock_id = None + try: + lock_id = self.acquire_lock() + if base: + with self._client as open_client: + tbl = open_client.get_table(self.database, self.table) + else: + current_time_millis = int(time.time()) + tbl = Table(self.table, + self.database, + getpass.getuser(), + current_time_millis // 1000, + current_time_millis // 1000, + sd=storage_descriptor(metadata), + tableType="EXTERNAL_TABLE", + parameters={'EXTERNAL': 'TRUE'}) - def close(self): + tbl.sd = storage_descriptor(metadata) + metadata_location = tbl.parameters.get(BaseMetastoreTableOperations.METADATA_LOCATION_PROP, None) + base_metadata_location = base.metadata_location if base else None + if base_metadata_location != metadata_location: + raise CommitFailedException( + "Base metadata location '%s' is not same as the current table metadata location '%s' for %s.%s", + base_metadata_location, metadata_location, self.database, self.table) + + self.set_parameters(new_metadata_location, tbl) + + if base: + with self._client as open_client: + env_context = EnvironmentContext( + {"DO_NOT_UPDATE_STATS": "true"} + ) + open_client.alter_table_with_environment_context(self.database, self.table, tbl, env_context) + + else: + with self._client as open_client: + open_client.create_table(tbl) + threw = False + except AlreadyExistsException: + raise IcebergAlreadyExistsException("Table already exists: {}.{}".format(self.database, self.table)) + except TException as e: + if e and "Table/View 'HIVE_LOCKS' does not exist" in str(e): + raise Exception("""Failed to acquire locks from metastore because 'HIVE_LOCKS' doesn't + exist, this probably happened when using embedded metastore or doesn't create a + transactional meta table. To fix this, use an alternative metastore""", e) + + raise Exception("Metastore operation failed for {}.{}".format(self.database, self.table), e) + finally: + if threw: + self.io().delete(new_metadata_location) + self.unlock(lock_id) + + def set_parameters(self: "HiveTableOperations", new_metadata_location: str, tbl: Table) -> None: + parameters = tbl.parameters + + if not parameters: + parameters = dict() + + parameters[BaseMetastoreTableOperations.TABLE_TYPE_PROP] = \ + BaseMetastoreTableOperations.ICEBERG_TABLE_TYPE_VALUE.upper() + parameters[BaseMetastoreTableOperations.METADATA_LOCATION_PROP] = new_metadata_location + + if self.current_metadata_location and len(self.current_metadata_location) > 0: + parameters[BaseMetastoreTableOperations.PREVIOUS_METADATA_LOCATION_PROP] = self.current_metadata_location + + tbl.parameters = parameters + + def unlock(self: "HiveTableOperations", lock_id: int = None) -> None: + if lock_id: + try: + with self._client as open_client: + open_client.unlock(LockResponse(lock_id)) + except Exception as e: + logging.warning("Failed to unlock {}.{}".format(self.database, self.table), e) + + def acquire_lock(self: "HiveTableOperations") -> int: + lock_component = LockComponent(LockType.EXCLUSIVE, LockLevel.TABLE, self.database, self.table) + + lock_request = LockRequest([lock_component], user=getpass.getuser(), hostname=socket.gethostname()) + with self._client as open_client: + lock_response = open_client.lock(lock_request) + + state = lock_response.state + lock_id = lock_response.lockid + start = int(time.time()) + duration = 0 + timed_out = False + while not timed_out and state == LockState.WAITING: + with self._client as open_client: + lock_response = open_client.check_lock(lock_response) + state = lock_response.state + + duration = int(time.time()) - start + if duration > 3 * 60 * 1000: + timed_out = True + else: + time.sleep(0.05) + + if timed_out and state != LockState.ACQUIRED: + raise CommitFailedException("Timed out after {} ms waiting for lock on {}.{}".format(duration, + self.database, + self.table)) + + if state != LockState.ACQUIRED: + raise CommitFailedException( + "Could not acquire the lock on {}.{}, lock request ended in state {}".format(self.database, self.table, + state)) + return lock_id + + def io(self: "HiveTableOperations") -> FileSystem: + return get_fs(self.base_location, self.conf) + + def close(self: "HiveTableOperations") -> None: self._client.close() + + +def storage_descriptor(metadata: TableMetadata) -> StorageDescriptor: + ser_de_info = SerDeInfo(serializationLib="org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe") + return StorageDescriptor(columns(metadata.schema), + metadata.location, + "org.apache.hadoop.mapred.FileInputFormat", + "org.apache.hadoop.mapred.FileOutputFormat", + serdeInfo=ser_de_info) + + +def columns(schema: Schema) -> List[FieldSchema]: + return [FieldSchema(col.name, convert_hive_type(col.type), "") for col in schema.columns()] + + +def convert_hive_type(col_type: Type) -> str: + type_id = col_type.type_id + hive_type_id = hive_types.get(type_id) # type: ignore + if hive_type_id: + return hive_type_id + raise NotImplementedError("Not yet implemented column type " + str(col_type)) diff --git a/iceberg/hive/hive_tables.py b/iceberg/hive/hive_tables.py index 42d734c468..f5a7beefce 100644 --- a/iceberg/hive/hive_tables.py +++ b/iceberg/hive/hive_tables.py @@ -25,24 +25,12 @@ class HiveTables(BaseMetastoreTables): - DOT = "." + _DOT = "." THRIFT_URIS = "hive.metastore.uris" def __init__(self, conf): super(HiveTables, self).__init__(conf) - def create(self, schema, table_identifier=None, spec=None, properties=None, database=None, table=None): - raise NotImplementedError() - - def load(self, table_identifier): - parts = table_identifier.split(HiveTables.DOT) - if len(parts) == 2: - return super(HiveTables, self).load(database=parts[0], table=parts[1]) - elif len(parts) == 1: - return super(HiveTables, self).load("default", table=parts[1]) - else: - raise RuntimeError("Could not parse table identifier: %s" % table_identifier) - def new_table_ops(self, conf, database, table): return HiveTableOperations(conf, self.get_client(), database, table) diff --git a/iceberg/hive/hive_types.py b/iceberg/hive/hive_types.py new file mode 100644 index 0000000000..39aade23f6 --- /dev/null +++ b/iceberg/hive/hive_types.py @@ -0,0 +1,37 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +from iceberg.api.types import TypeID + +hive_types = { + TypeID.BOOLEAN: 'boolean', + TypeID.INTEGER: 'int', + TypeID.LONG: 'bigint', + TypeID.FLOAT: 'float', + TypeID.DOUBLE: 'double', + TypeID.DATE: 'date', + TypeID.TIME: 'string', + TypeID.TIMESTAMP: 'timestamp', + TypeID.STRING: 'string', + TypeID.UUID: 'string', + TypeID.FIXED: 'binary', + TypeID.BINARY: "binary", + TypeID.DECIMAL: None, + TypeID.STRUCT: None, + TypeID.LIST: None, + TypeID.MAP: None +} diff --git a/tests/core/conftest.py b/tests/core/conftest.py index 8d414ac790..933743e764 100644 --- a/tests/core/conftest.py +++ b/tests/core/conftest.py @@ -285,13 +285,19 @@ def base_scan_schema(): @pytest.fixture(scope="session", params=["none", "one"]) -def ts_table(base_scan_schema, request): +def base_scan_partition(base_scan_schema, request): + if request.param == "none": + spec = PartitionSpec.unpartitioned() + else: + spec = PartitionSpecBuilder(base_scan_schema).add(1, 1000, "id", "identity").build() + return spec + + +@pytest.fixture(scope="session") +def ts_table(base_scan_schema, base_scan_partition): with tempfile.TemporaryDirectory() as td: - if request.param == "none": - spec = PartitionSpec.unpartitioned() - else: - spec = PartitionSpecBuilder(base_scan_schema).add(1, 1000, "id", "identity").build() - return TestTables.create(td, "test-" + request.param, base_scan_schema, spec) + return TestTables.create(td, "test-" + str(len(base_scan_partition.fields)), base_scan_schema, + base_scan_partition) @pytest.fixture(scope="session") @@ -300,7 +306,7 @@ def split_planning_table(base_scan_schema): tables = FilesystemTables() with tempfile.TemporaryDirectory() as td: - table = tables.create(base_scan_schema, location=td) + table = tables.create(base_scan_schema, table_identifier=td) table.properties().update({TableProperties.SPLIT_SIZE: "{}".format(128 * 1024 * 1024), TableProperties.SPLIT_OPEN_FILE_COST: "{}".format(4 * 1024 * 1024), TableProperties.SPLIT_LOOKBACK: "{}".format(2 ** 31 - 1)}) diff --git a/tests/core/test_filesystem_tables.py b/tests/core/test_filesystem_tables.py new file mode 100644 index 0000000000..5f4d6e386d --- /dev/null +++ b/tests/core/test_filesystem_tables.py @@ -0,0 +1,32 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# +import os + +from iceberg.core.filesystem import FilesystemTables + + +def test_create_tables(base_scan_schema, base_scan_partition, tmpdir): + + conf = {"hive.metastore.uris": 'thrift://hms:port', + "hive.metastore.warehouse.dir": tmpdir} + tables = FilesystemTables(conf) + table_location = os.path.join(str(tmpdir), "test", "test_123") + tables.create(base_scan_schema, table_location, base_scan_partition) + + tables.load(table_location) diff --git a/tests/hive/conftest.py b/tests/hive/conftest.py index fe95886d5c..44c60d74d4 100644 --- a/tests/hive/conftest.py +++ b/tests/hive/conftest.py @@ -16,3 +16,22 @@ # specific language governing permissions and limitations # under the License. # + +from iceberg.api import PartitionSpec, PartitionSpecBuilder, Schema +from iceberg.api.types import IntegerType, NestedField, StringType +import pytest + + +@pytest.fixture(scope="session") +def base_scan_schema(): + return Schema([NestedField.required(1, "id", IntegerType.get()), + NestedField.required(2, "data", StringType.get())]) + + +@pytest.fixture(scope="session", params=["none", "one"]) +def base_scan_partition(base_scan_schema, request): + if request.param == "none": + spec = PartitionSpec.unpartitioned() + else: + spec = PartitionSpecBuilder(base_scan_schema).add(1, 1000, "id", "identity").build() + return spec diff --git a/tests/hive/test_hive_tables.py b/tests/hive/test_hive_tables.py index f979d098d6..0729287f13 100644 --- a/tests/hive/test_hive_tables.py +++ b/tests/hive/test_hive_tables.py @@ -16,9 +16,13 @@ # specific language governing permissions and limitations # under the License. # +import os +from hmsclient.genthrift.hive_metastore.ttypes import LockResponse, LockState, NoSuchObjectException +from iceberg.exceptions import AlreadyExistsException from iceberg.hive import HiveTables import mock +import pytest from pytest import raises @@ -118,3 +122,38 @@ def test_load_tables_check_none_location(client, current_call, refresh_call): tables = HiveTables(conf) with raises(RuntimeError): tables.load("test.test_123") + + +@mock.patch("iceberg.hive.HiveTableOperations.refresh_from_metadata_location") +@mock.patch("iceberg.hive.HiveTableOperations.current") +@mock.patch("iceberg.hive.HiveTables.get_client") +def test_create_tables_failed(client, current_call, refresh_call, base_scan_schema, base_scan_partition, tmpdir): + client.return_value.__enter__.return_value.get_table.side_effect = NoSuchObjectException() + current_call.return_value = None + conf = {"hive.metastore.uris": 'thrift://hms:port', + "hive.metastore.warehouse.dir": tmpdir} + tables = HiveTables(conf) + with pytest.raises(AlreadyExistsException): + tables.create(base_scan_schema, "test.test_123", base_scan_partition) + assert len(os.listdir(os.path.join(tmpdir, "test.db", "test_123", "metadata"))) == 0 + + +@mock.patch("iceberg.hive.HiveTableOperations.current") +@mock.patch("iceberg.hive.HiveTables.get_client") +def test_create_tables(client, current_call, base_scan_schema, base_scan_partition, tmpdir): + + client.return_value.__enter__.return_value.get_table.side_effect = NoSuchObjectException() + current_call.return_value = None + client.return_value.__enter__.return_value.lock.return_value = LockResponse("x", LockState.WAITING) + client.return_value.__enter__.return_value.check_lock.return_value = LockResponse("x", LockState.ACQUIRED) + tbl = client.return_value.__enter__.return_value.create_table.call_args_list + conf = {"hive.metastore.uris": 'thrift://hms:port', + "hive.metastore.warehouse.dir": tmpdir} + tables = HiveTables(conf) + tables.create(base_scan_schema, "test.test_123", base_scan_partition) + + client.return_value.__enter__.return_value.get_table.return_value = MockHMSTable(tbl[0].args[0].parameters) + client.return_value.__enter__.return_value.get_table.side_effect = None + current_call.return_value = tbl[0].args[0].parameters['metadata_location'] + + tables.load("test.test_123") From d5ec964ee498c5f612ccbc0279681aa7fa0be741 Mon Sep 17 00:00:00 2001 From: mickjermsurawong-stripe <42550941+mickjermsurawong-stripe@users.noreply.github.com> Date: Mon, 5 Oct 2020 19:38:48 -0700 Subject: [PATCH 033/642] Core: Allow LocationProvider to be customized using reflection (#1531) --- iceberg/core/table_properties.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/iceberg/core/table_properties.py b/iceberg/core/table_properties.py index a50546c78b..da75eff960 100644 --- a/iceberg/core/table_properties.py +++ b/iceberg/core/table_properties.py @@ -67,6 +67,8 @@ class TableProperties(object): OBJECT_STORE_PATH = "write.object-storage.path" + WRITE_LOCATION_PROVIDER_IMPL = "write.location-provider.impl" + WRITE_NEW_DATA_LOCATION = "write.folder-storage.path" WRITE_METADATA_LOCATION = "write.metadata.path" From a9f9238445b3cc4e3fc2ed02c6e33e208b9ec853 Mon Sep 17 00:00:00 2001 From: Ryan Blue Date: Fri, 6 Nov 2020 09:55:35 -0800 Subject: [PATCH 034/642] Build: Add Python tox action (#1733) --- tox.ini | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tox.ini b/tox.ini index 29868f7a7d..5f9b3ce744 100644 --- a/tox.ini +++ b/tox.ini @@ -106,8 +106,8 @@ application-import-names = flake8 [pytest] norecursedirs=.* -[travis] +[gh-actions] python = 3.6: py36, linters 3.7: py37 - 3.8: py38 \ No newline at end of file + 3.8: py38 From c20e64641390d71b6795e64fe17e06f89bfca529 Mon Sep 17 00:00:00 2001 From: Fokko Driesprong Date: Wed, 25 Nov 2020 22:31:16 +0100 Subject: [PATCH 035/642] Python: Fix missing assert in test case (#1828) --- tests/api/expressions/test_inclusive_manifest_evaluator.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tests/api/expressions/test_inclusive_manifest_evaluator.py b/tests/api/expressions/test_inclusive_manifest_evaluator.py index eaaa93fc16..69e46355fe 100644 --- a/tests/api/expressions/test_inclusive_manifest_evaluator.py +++ b/tests/api/expressions/test_inclusive_manifest_evaluator.py @@ -177,6 +177,6 @@ def test_case_insensitive_int_not_eq_rewritten(inc_man_spec, inc_man_file, val, (85, True)]) def test_case_sensitive_int_not_eq_rewritten(inc_man_spec, inc_man_file, val, expected): with pytest.raises(ValidationException): - InclusiveManifestEvaluator(inc_man_spec, - Expressions.not_(Expressions.equal("ID", val)), - case_sensitive=True).eval(inc_man_file) == expected + assert InclusiveManifestEvaluator(inc_man_spec, + Expressions.not_(Expressions.equal("ID", val)), + case_sensitive=True).eval(inc_man_file) == expected From 3051fb63d3759c07f1a6c19101f667c3953fabb7 Mon Sep 17 00:00:00 2001 From: Ted Gooch Date: Fri, 4 Dec 2020 16:19:04 -0800 Subject: [PATCH 036/642] Python: Add Parquet reads and Arrow schema conversion (#1727) --- iceberg/api/expressions/expression.py | 12 + iceberg/api/io/input_file.py | 10 + iceberg/api/types/type.py | 4 + iceberg/api/types/types.py | 7 +- iceberg/core/filesystem/__init__.py | 3 +- iceberg/core/util/profile.py | 36 +++ iceberg/exceptions/__init__.py | 4 + iceberg/exceptions/exceptions.py | 8 + iceberg/parquet/__init__.py | 20 ++ iceberg/parquet/dataset_utils.py | 158 +++++++++++++ iceberg/parquet/parquet_reader.py | 224 ++++++++++++++++++ iceberg/parquet/parquet_schema_utils.py | 41 ++++ iceberg/parquet/parquet_to_iceberg.py | 152 ++++++++++++ setup.py | 3 +- tests/parquet/__init__.py | 16 ++ tests/parquet/conftest.py | 226 ++++++++++++++++++ tests/parquet/test_dataset_utils.py | 52 +++++ tests/parquet/test_parquet_reader.py | 285 +++++++++++++++++++++++ tests/parquet/test_parquet_to_iceberg.py | 76 ++++++ 19 files changed, 1332 insertions(+), 5 deletions(-) create mode 100644 iceberg/core/util/profile.py create mode 100644 iceberg/parquet/__init__.py create mode 100644 iceberg/parquet/dataset_utils.py create mode 100644 iceberg/parquet/parquet_reader.py create mode 100644 iceberg/parquet/parquet_schema_utils.py create mode 100644 iceberg/parquet/parquet_to_iceberg.py create mode 100644 tests/parquet/__init__.py create mode 100644 tests/parquet/conftest.py create mode 100644 tests/parquet/test_dataset_utils.py create mode 100644 tests/parquet/test_parquet_reader.py create mode 100644 tests/parquet/test_parquet_to_iceberg.py diff --git a/iceberg/api/expressions/expression.py b/iceberg/api/expressions/expression.py index 1aedf5ccb2..eefa89e89d 100644 --- a/iceberg/api/expressions/expression.py +++ b/iceberg/api/expressions/expression.py @@ -16,12 +16,24 @@ # under the License. from enum import Enum +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + from .predicate import Predicate class Expression(object): + + left: 'Predicate' + right: 'Predicate' + child: 'Predicate' + def __init__(self): pass + def op(self): + raise RuntimeError("No implementation for base class") + def negate(self): raise RuntimeError("%s cannot be negated" % self) diff --git a/iceberg/api/io/input_file.py b/iceberg/api/io/input_file.py index 67277f79f5..08b68ad0a2 100644 --- a/iceberg/api/io/input_file.py +++ b/iceberg/api/io/input_file.py @@ -15,8 +15,15 @@ # specific language governing permissions and limitations # under the License. +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + from iceberg.core.filesystem import FileSystem + class InputFile(object): + fs: 'FileSystem' + path: str def get_length(self): raise NotImplementedError() @@ -26,3 +33,6 @@ def new_stream(self): def location(self): raise NotImplementedError() + + def new_fo(self): + raise NotImplementedError() diff --git a/iceberg/api/types/type.py b/iceberg/api/types/type.py index 367e366b99..437b95bfd1 100644 --- a/iceberg/api/types/type.py +++ b/iceberg/api/types/type.py @@ -41,6 +41,10 @@ class TypeID(Enum): class Type(object): + length: int + scale: int + precision: int + def __init__(self): pass diff --git a/iceberg/api/types/types.py b/iceberg/api/types/types.py index 373463061f..b5957c2345 100644 --- a/iceberg/api/types/types.py +++ b/iceberg/api/types/types.py @@ -310,10 +310,11 @@ def of_length(length): return FixedType(length) def __init__(self, length): - self.length = length + self._length = length + @property def length(self): - return self.length + return self._length @property def type_id(self): @@ -409,6 +410,8 @@ def __key(self): class NestedField(): + length: int + @staticmethod def optional(id, name, type_var, doc=None): return NestedField(True, id, name, type_var, doc=doc) diff --git a/iceberg/core/filesystem/__init__.py b/iceberg/core/filesystem/__init__.py index b836e2fc3b..c686aa1262 100644 --- a/iceberg/core/filesystem/__init__.py +++ b/iceberg/core/filesystem/__init__.py @@ -16,11 +16,12 @@ # under the License. __all__ = ["get_fs", "FileStatus", "FileSystem", "FileSystemInputFile", "FileSystemOutputFile", - "FilesystemTableOperations", "FilesystemTables", "S3File", "S3FileSystem"] + "FilesystemTableOperations", "FilesystemTables", "LocalFileSystem", "S3File", "S3FileSystem"] from .file_status import FileStatus from .file_system import FileSystem, FileSystemInputFile, FileSystemOutputFile from .filesystem_table_operations import FilesystemTableOperations from .filesystem_tables import FilesystemTables +from .local_filesystem import LocalFileSystem from .s3_filesystem import S3File, S3FileSystem from .util import get_fs diff --git a/iceberg/core/util/profile.py b/iceberg/core/util/profile.py new file mode 100644 index 0000000000..4df7acd9aa --- /dev/null +++ b/iceberg/core/util/profile.py @@ -0,0 +1,36 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + + +from contextlib import contextmanager +import logging +import time + +_logger = logging.getLogger(__name__) + + +@contextmanager +def profile(label, stats_dict=None): + if stats_dict is None: + _logger.debug('PROFILE: %s starting' % label) + start = time.time() + yield + took = int((time.time() - start) * 1000) + if stats_dict is None: + _logger.debug('PROFILE: %s completed in %dms' % (label, took)) + else: + stats_dict[label] = stats_dict.get(label, 0) + took diff --git a/iceberg/exceptions/__init__.py b/iceberg/exceptions/__init__.py index e6810ef4e9..3250a59cac 100644 --- a/iceberg/exceptions/__init__.py +++ b/iceberg/exceptions/__init__.py @@ -17,10 +17,14 @@ __all__ = ["AlreadyExistsException", "CommitFailedException", + "FileSystemNotFound", + "InvalidCastException", "NoSuchTableException", "ValidationException"] from .exceptions import (AlreadyExistsException, CommitFailedException, + FileSystemNotFound, + InvalidCastException, NoSuchTableException, ValidationException) diff --git a/iceberg/exceptions/exceptions.py b/iceberg/exceptions/exceptions.py index 3a1072e7bb..73bf641b21 100644 --- a/iceberg/exceptions/exceptions.py +++ b/iceberg/exceptions/exceptions.py @@ -24,6 +24,14 @@ class CommitFailedException(RuntimeError): pass +class FileSystemNotFound(Exception): + pass + + +class InvalidCastException(ValueError): + pass + + class NoSuchTableException(RuntimeError): pass diff --git a/iceberg/parquet/__init__.py b/iceberg/parquet/__init__.py new file mode 100644 index 0000000000..af6dab4e05 --- /dev/null +++ b/iceberg/parquet/__init__.py @@ -0,0 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +__all__ = ["ParquetReader"] + +from .parquet_reader import ParquetReader diff --git a/iceberg/parquet/dataset_utils.py b/iceberg/parquet/dataset_utils.py new file mode 100644 index 0000000000..7472b2bddc --- /dev/null +++ b/iceberg/parquet/dataset_utils.py @@ -0,0 +1,158 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + + +from iceberg.api.expressions import Expression, Operation, Predicate +import pyarrow.dataset as ds + + +def get_dataset_filter(expr: Expression, expected_to_file_map: dict) -> ds.Expression: + """ + Given an Iceberg Expression and a mapping of names in the iceberg schema to the file schema, + convert to an equivalent dataset filter using the file column names. Recursively iterate through the + expressions to convert each portion one predicate at a time + + Parameters + ---------- + expr : iceberg.api.expressions.Expression + An Iceberg Expression to be converted + expected_to_file_map : dict + A dict that maps the iceberg schema names to the names from the file schema + Returns + ------- + pyarrow._dataset.Expression + An equivalent dataset expression + """ + if expr is None: + return None + + if isinstance(expr, Predicate): + return predicate(expr, expected_to_file_map) + + if expr.op() == Operation.TRUE: + return None + elif expr.op() == Operation.FALSE: + return False + elif expr.op() == Operation.NOT: + return not_(get_dataset_filter(expr.child, expected_to_file_map)) + elif expr.op() == Operation.AND: + return and_(get_dataset_filter(expr.left, expected_to_file_map), + get_dataset_filter(expr.right, expected_to_file_map)) + elif expr.op() == Operation.OR: + return or_(get_dataset_filter(expr.left, expected_to_file_map), + get_dataset_filter(expr.right, expected_to_file_map)) + else: + raise RuntimeError("Unknown operation: {}".format(expr.op())) + + +def predicate(pred: Predicate, field_map: dict) -> ds.Expression: # noqa: ignore=C901 + """ + Given an Iceberg Predicate and a mapping of names in the iceberg schema to the file schema, + convert to an equivalent dataset expression using the file column names. + + Parameters + ---------- + pred : iceberg.api.expressions.Predicate + An Iceberg Predicate to be converted + field_map : dict + A dict that maps the iceberg schema names to the names from the file schema + Returns + ------- + pyarrow._dataset.Expression + An equivalent dataset expression + """ + # get column name in the file schema so we can apply the predicate + col_name = field_map.get(pred.ref.name) + + if col_name is None: + if pred.op == Operation.IS_NULL: + return ds.scalar(True) == ds.scalar(True) + + return ds.scalar(True) == ds.scalar(False) + + if pred.op == Operation.IS_NULL: + return ~ds.field(col_name).is_valid() + elif pred.op == Operation.NOT_NULL: + return ds.field(col_name).is_valid() + elif pred.op == Operation.LT: + return ds.field(col_name) < pred.lit.value + elif pred.op == Operation.LT_EQ: + return ds.field(col_name) <= pred.lit.value + elif pred.op == Operation.GT: + return ds.field(col_name) > pred.lit.value + elif pred.op == Operation.GT_EQ: + return ds.field(col_name) >= pred.lit.value + elif pred.op == Operation.EQ: + return ds.field(col_name) == pred.lit.value + elif pred.op == Operation.NOT_EQ: + return ds.field(col_name) != pred.lit.value + elif pred.op == Operation.IN: + return ds.field(col_name).isin(pred.lit.value) + elif pred.op == Operation.NOT_IN: + return ds.field(col_name).isin(pred.lit.value) + + +def and_(left: ds.Expression, right: ds.Expression) -> ds.Expression: + """ + Given a left and right expression combined them using the `AND` logical operator + + Parameters + ---------- + left : pyarrow._dataset.Expression + A Dataset `Expression` to logically `AND` + right : pyarrow._dataset.Expression + A Dataset `Expression` to logically `AND` + Returns + ------- + pyarrow._dataset.Expression + The left and right `Expression` combined with `AND` + """ + return left & right + + +def or_(left: ds.Expression, right: ds.Expression) -> ds.Expression: + """ + Given a left and right expression combined them using the `OR` logical operator + + Parameters + ---------- + left : pyarrow._dataset.Expression + A Dataset `Expression` to logically `OR` + right : pyarrow._dataset.Expression + A Dataset `Expression` to logically `OR` + Returns + ------- + pyarrow._dataset.Expression + The left and right `Expression` combined with `OR` + """ + return left | right + + +def not_(child: ds.Expression) -> ds.Expression: + """ + Given a child expression create the logical negation + + Parameters + ---------- + child : pyarrow._dataset.Expression + A Dataset `Expression` to logically `OR` + Returns + ------- + pyarrow._dataset.Expression + The negation of the input `Expression` + """ + return ~child diff --git a/iceberg/parquet/parquet_reader.py b/iceberg/parquet/parquet_reader.py new file mode 100644 index 0000000000..0728546961 --- /dev/null +++ b/iceberg/parquet/parquet_reader.py @@ -0,0 +1,224 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + + +from datetime import datetime +import decimal +import logging +import typing + +from iceberg.api import Schema +from iceberg.api.expressions import Expression +from iceberg.api.io import InputFile +from iceberg.api.types import NestedField, Type, TypeID +from iceberg.core.filesystem import FileSystem, LocalFileSystem, S3FileSystem +from iceberg.core.util.profile import profile +from iceberg.exceptions import FileSystemNotFound, InvalidCastException +import numpy as np +import pyarrow as pa +from pyarrow import fs +import pyarrow.dataset as ds +import pyarrow.parquet as pq + +from .dataset_utils import get_dataset_filter +from .parquet_schema_utils import prune_columns +from .parquet_to_iceberg import convert_parquet_to_iceberg + +_logger = logging.getLogger(__name__) + +DTYPE_MAP: typing.Dict[TypeID, + typing.Callable[[NestedField], typing.Tuple[pa.Field, typing.Any]]] = \ + {TypeID.BINARY: lambda field: pa.binary(), + TypeID.BOOLEAN: lambda field: (pa.bool_(), False), + TypeID.DATE: lambda field: (pa.date32(), datetime.now()), + TypeID.DECIMAL: lambda field: (pa.decimal128(field.type.precision, field.type.scale), + decimal.Decimal()), + TypeID.DOUBLE: lambda field: (pa.float64(), np.nan), + TypeID.FIXED: lambda field: pa.binary(field.length), + TypeID.FLOAT: lambda field: (pa.float32(), np.nan), + TypeID.INTEGER: lambda field: (pa.int32(), np.nan), + TypeID.LIST: lambda field: (pa.list_(pa.field("element", + DTYPE_MAP[field.type.element_type.type_id](field.type)[0])), + None), + TypeID.LONG: lambda field: (pa.int64(), np.nan), + # To-Do: update to support reading map fields + # TypeID.MAP: lambda field: (,), + TypeID.STRING: lambda field: (pa.string(), ""), + TypeID.STRUCT: lambda field: (pa.struct([(nested_field.name, + DTYPE_MAP[nested_field.type.type_id](nested_field.type)[0]) + for nested_field in field.type.fields]), {}), + TypeID.TIMESTAMP: lambda field: (pa.timestamp("us"), datetime.now()), + # not used in SPARK, so not implementing for now + # TypeID.TIME: pa.time64(None) + } + +FS_MAP: typing.Dict[typing.Type[FileSystem], typing.Type[fs.FileSystem]] = {LocalFileSystem: fs.LocalFileSystem} + +try: + FS_MAP[S3FileSystem] = fs.S3FileSystem + +except ImportError: + _logger.warning("Mapped filesystem not available") + + +class ParquetReader(object): + + def __init__(self, input: InputFile, expected_schema: Schema, options, filter_expr: Expression, + case_sensitive: bool, start: int = None, end: int = None): + self._stats: typing.Dict[str, int] = dict() + + self._input = input + self._input_fo = input.new_fo() + + self._arrow_file = pq.ParquetFile(self._input_fo) + self._file_schema = convert_parquet_to_iceberg(self._arrow_file) + self._expected_schema = expected_schema + self._file_to_expected_name_map = ParquetReader.get_field_map(self._file_schema, + self._expected_schema) + self._options = options + self._filter = get_dataset_filter(filter_expr, ParquetReader.get_reverse_field_map(self._file_schema, + self._expected_schema)) + + self._case_sensitive = case_sensitive + if start is not None or end is not None: + raise NotImplementedError("Partial file reads are not yet supported") + # self.start = start + # self.end = end + + self.materialized_table = False + self._table = None + + _logger.debug("Reader initialized for %s" % self._input.path) + + @property + def stats(self) -> typing.Dict[str, int]: + return dict(self._stats) + + def read(self, force=False) -> pa.Table: + if not self.materialized_table or force: + self._read_data() + + return self._table + + def _read_data(self) -> None: + _logger.debug("Starting data read") + + # only scan the columns projected and in our file + cols_to_read = prune_columns(self._file_schema, self._expected_schema) + + with profile("read data", self._stats): + try: + read_fs = FS_MAP[type(self._input.fs)] + except KeyError: + raise FileSystemNotFound(f"No mapped filesystem found for {type(self._input.fs)}") + + arrow_dataset = ds.FileSystemDataset.from_paths([self._input.location()], + schema=self._arrow_file.schema_arrow, + format=ds.ParquetFileFormat(), + filesystem=read_fs()) + + arrow_table = arrow_dataset.to_table(columns=cols_to_read, filter=self._filter) + + # process schema evolution if needed + with profile("schema_evol_proc", self._stats): + processed_tbl = self.migrate_schema(arrow_table) + for i, field in self.get_missing_fields(): + dtype_func = DTYPE_MAP.get(field.type.type_id) + if dtype_func is None: + raise RuntimeError("Unable to create null column for type %s" % field.type.type_id) + + dtype = dtype_func(field) + processed_tbl = (processed_tbl.add_column(i, + pa.field(field.name, dtype[0], True, None), + ParquetReader.create_null_column(processed_tbl[0], + dtype))) + self._table = processed_tbl + self.materialized_table = True + + def __enter__(self): + return self + + def __exit__(self, exc_type, exc_val, exc_tb): + _logger.debug(self._stats) + self.close() + + def close(self): + self._input_fo.close() + + def get_missing_fields(self) -> typing.List[typing.Tuple[int, NestedField]]: + return [(i, field) for i, field in enumerate(self._expected_schema.as_struct().fields) + if self._file_schema.find_field(field.id) is None] + + @staticmethod + def get_field_map(file_schema, expected_schema) -> typing.Dict[str, str]: + return {file_schema.find_field(field.id).name: field.name + for field in expected_schema.as_struct().fields + if file_schema.find_field(field.id) is not None} + + @staticmethod + def get_reverse_field_map(file_schema, expected_schema) -> typing.Dict[str, str]: + return {expected_schema.find_field(field.id).name: field.name + for field in file_schema.as_struct().fields + if expected_schema.find_field(field.id) is not None} + + def migrate_schema(self, table: pa.Table) -> pa.Table: + data_arrays: typing.List[pa.ChunkedArray] = [] + schema: typing.List[pa.Field] = [] + for key, value in self._file_to_expected_name_map.items(): + column_idx: int = table.schema.get_field_index(key) + column_field: pa.Field = table.schema[column_idx] + column_arrow_type: pa.DataType = column_field.type + column_data: pa.ChunkedArray = table[column_idx] + + iceberg_field: NestedField = self._expected_schema.find_field(value) + converted_field: NestedField = self._file_schema.find_field(key) + + if iceberg_field.type != converted_field.type: + if not ParquetReader.is_supported_cast(converted_field.type, iceberg_field.type): + _logger.error(f"unsupported cast {converted_field.type} -> {iceberg_field.type}") + raise InvalidCastException("") + try: + column_arrow_type = DTYPE_MAP[iceberg_field.type.type_id](iceberg_field)[0] + column_data = column_data.cast(column_arrow_type) + except KeyError: + _logger.error(f"Unable to map {iceberg_field.type} to an arrow type") + raise + + data_arrays.append(column_data) + schema.append(pa.field(value, column_arrow_type, column_field.nullable, column_field.metadata)) + + return pa.table(data_arrays, schema=pa.schema(schema)) + + @staticmethod + def create_null_column(reference_column: pa.ChunkedArray, dtype_tuple: typing.Tuple[pa.DataType, typing.Any]) -> pa.ChunkedArray: + dtype, init_val = dtype_tuple + return pa.chunked_array([pa.array(np.full(len(c), init_val), + type=dtype, + mask=np.array([True] * len(reference_column.chunks[0]), dtype="bool")) + for c in reference_column.chunks], type=dtype) + + @staticmethod + def is_supported_cast(old_type: Type, new_type: Type) -> bool: + if old_type.type_id == TypeID.INTEGER and new_type.type_id == TypeID.LONG: + return True + elif old_type.type_id == TypeID.FLOAT and new_type.type_id == TypeID.DOUBLE: + return True + elif old_type.type_id == TypeID.DECIMAL and new_type.type_id == TypeID.DECIMAL \ + and old_type.precision < new_type.precision \ + and old_type.scale == new_type.scale: + return True + return False diff --git a/iceberg/parquet/parquet_schema_utils.py b/iceberg/parquet/parquet_schema_utils.py new file mode 100644 index 0000000000..cdccfd5e72 --- /dev/null +++ b/iceberg/parquet/parquet_schema_utils.py @@ -0,0 +1,41 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +from typing import List + +from iceberg.api import Schema +from iceberg.api.types import get_projected_ids + + +def prune_columns(file_schema: Schema, expected_schema: Schema) -> List[str]: + """ + Given two Iceberg schema's returns a list of column_names for all id's in the + file schema that are projected in the expected schema + + Parameters + ---------- + file_schema : iceberg.api.Schema + An Iceberg schema of the file being read + expected_schema : iceberg.api.Schema + An Iceberg schema of the final projection + Returns + ------- + list + The column names in the file that matched ids in the expected schema + """ + return [column.name for column in file_schema.as_struct().fields + if column.id in get_projected_ids(expected_schema)] diff --git a/iceberg/parquet/parquet_to_iceberg.py b/iceberg/parquet/parquet_to_iceberg.py new file mode 100644 index 0000000000..674d02db0c --- /dev/null +++ b/iceberg/parquet/parquet_to_iceberg.py @@ -0,0 +1,152 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + + +import logging + +from iceberg.api import Schema +from iceberg.api.types import (BinaryType, + BooleanType, + DateType, + DecimalType, + DoubleType, + FixedType, + FloatType, + IntegerType, + ListType, + LongType, + MapType, + NestedField, + StringType, + StructType, + TimestampType) +from iceberg.api.types import Type +import pyarrow as pa +from pyarrow.parquet import lib, ParquetFile + +_logger = logging.getLogger(__name__) + +arrow_type_map = {lib.Type_BOOL: lambda x=None: BooleanType.get(), + lib.Type_DATE32: lambda x=None: DateType.get(), + lib.Type_DECIMAL: lambda x=None: DecimalType.of(x.precision, x.scale), + lib.Type_DOUBLE: lambda x=None: DoubleType.get(), + lib.Type_FIXED_SIZE_BINARY: lambda x=None: FixedType.of_length(x.byte_width), + lib.Type_BINARY: lambda x=None: BinaryType.get(), + lib.Type_FLOAT: lambda x=None: FloatType.get(), + lib.Type_STRING: lambda x=None: StringType.get(), + lib.Type_INT32: lambda x=None: IntegerType.get(), + lib.Type_INT64: lambda x=None: LongType.get(), + lib.Type_TIMESTAMP: lambda x=None: (TimestampType.without_timezone() + if x.tz is None + else TimestampType.with_timezone()) + } + + +def get_nested_field(field_id: int, field_name: str, field_type: Type, nullable: bool) -> NestedField: + if nullable: + return NestedField.optional(field_id, field_name, field_type) + else: + return NestedField.required(field_id, field_name, field_type) + + +def get_list_field(col_type: pa.ListType) -> ListType: + if col_type.value_field.nullable: + return ListType.of_optional(int(col_type.value_field.metadata[b"PARQUET:field_id"].decode("utf-8")), + arrow_type_map[col_type.value_field.type.id](col_type.value_field.type)) + else: + return ListType.of_required(col_type.value_field.metadata[b"PARQUET:field_id"].decode("utf-8"), + arrow_type_map[col_type.value_field.type.id](col_type.value_field.type)) + + +def get_field(col: pa.Field) -> NestedField: + try: + return get_nested_field(int(col.metadata[b"PARQUET:field_id"].decode("utf-8")), + col.name, + arrow_type_map[col.type.id](col.type), + col.nullable) + except KeyError: + _logger.error(f"\t{int(col.metadata[b'PARQUET:field_id'].decode('utf-8'))}: {col.type}") + raise + + +def get_struct_field(col_type: pa.StructType) -> StructType: + if col_type.num_fields > 0: + if col_type[0].name == "map": + return get_inferred_map(col_type) + else: + return StructType.of([get_field(child) for child in col_type]) + else: + raise RuntimeError("Unable to convert type to iceberg %s" % col_type) + + +def get_inferred_map(col_type: pa.StructType): + base_field = col_type[0] + key_field = get_field(col_type[0].type.value_field.type[0]) + value_field = get_field(col_type[0].type.value_field.type[1]) + + if base_field.nullable: + return MapType.of_optional(key_field.field_id, value_field.field_id, + key_field.type, value_field.type) + else: + return MapType.of_required(key_field.field_id, value_field.field_id, + key_field.type, value_field.type) + + +def get_map_field(col): + # Spark Map type gets serialized to Struct>> + raise NotImplementedError("Arrow Map type not implemented yet") + + +# adding function mappings for nested types +arrow_type_map.update({lib.Type_LIST: get_list_field, + lib.Type_MAP: get_map_field, + lib.Type_STRUCT: get_struct_field}) + + +def convert_parquet_to_iceberg(parquet_file: ParquetFile) -> Schema: # noqa: ignore=C901 + """ + Given two Iceberg schema's returns a list of column_names for all id's in the + file schema that are projected in the expected schema + + Parameters + ---------- + parquet_file : pyarrow.parquet.ParquetFile + The Parquet File to use to extract the iceberg schema + + Returns + ------- + iceberg.api.Schema + returns an equivalent iceberg Schema based on the arrow schema read from the file + """ + return arrow_to_iceberg(parquet_file.schema_arrow) + + +def arrow_to_iceberg(arrow_schema: pa.Schema) -> Schema: + """ + Use an arrow schema, which contains the field_id metadata, to create an equivalent iceberg Schema + + Parameters + ---------- + arrow_schema : pyarrow.Schema + An Arrow schema with the parquet field_id metadata + + Returns + ------- + iceberg.api.Schema + returns an equivalent iceberg Schema based on the arrow schema read from the file + """ + return Schema([get_field(col) for col in arrow_schema]) diff --git a/setup.py b/setup.py index 363d61a386..3b33a546e1 100755 --- a/setup.py +++ b/setup.py @@ -29,7 +29,6 @@ install_requires=['botocore', 'boto3', 'fastavro', - 'fastparquet>=0.3.1', 'hmsclient', 'mmh3', 'pyparsing', @@ -38,7 +37,7 @@ 'requests', 'retrying', 'pandas', - 'pyarrow' + 'pyarrow>=2.0.0' ], extras_require={ "dev": [ diff --git a/tests/parquet/__init__.py b/tests/parquet/__init__.py new file mode 100644 index 0000000000..13a83393a9 --- /dev/null +++ b/tests/parquet/__init__.py @@ -0,0 +1,16 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. diff --git a/tests/parquet/conftest.py b/tests/parquet/conftest.py new file mode 100644 index 0000000000..5f615f6f7a --- /dev/null +++ b/tests/parquet/conftest.py @@ -0,0 +1,226 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +from collections import namedtuple +from datetime import datetime +from decimal import Decimal +from tempfile import NamedTemporaryFile + +from iceberg.api import Schema +from iceberg.api.types import (DateType, + DecimalType, + FloatType, + IntegerType, + LongType, + NestedField, + StringType, + TimestampType) +import pyarrow as pa +import pyarrow.parquet as pq +import pytest + + +TestRowGroupColumnStatistics = namedtuple("TestRowGroupColumnStatistics", ["min", "max", "null_count"]) +TestRowGroupColumn = namedtuple("TestRowGroupColumn", ["path_in_schema", + "file_offset", + "total_compressed_size", + "statistics"]) + + +class TestArrowParquetMetadata: + __test__ = False + + def __init__(self, col_metadata, num_rows=100): + self._col_metadata = col_metadata + self._num_rows = num_rows + + @property + def num_rows(self): + return self._num_rows + + @property + def num_columns(self): + return len(self._col_metadata) + + def column(self, i): + return self._col_metadata[i] + + def __getitem__(self, index): + return self._col_metadata[index] + + +@pytest.fixture(scope="session") +def pyarrow_array(): + return [pa.array([1, 2, 3, None, 5], type=pa.int32()), + pa.array(['us', 'can', 'us', 'us', 'can'], type=pa.string()), + pa.array([[0], [1, 2], [1], [1, 2, 3], None], type=pa.list_(pa.int64())), + pa.array([True, None, False, True, True], pa.bool_())] + + +@pytest.fixture(scope="session") +def pytable_colnames(): + return ['int_col', 'str_col', 'list_col', 'bool_col'] + + +@pytest.fixture(scope="session") +def rg_expected_schema(): + return Schema([NestedField.required(1, "string_col", StringType.get()), + NestedField.required(2, "long_col", LongType.get()), + NestedField.required(3, "int_col", IntegerType.get()), + NestedField.optional(4, "float_col", FloatType.get()), + NestedField.optional(5, "null_col", StringType.get()), + NestedField.optional(6, "missing_col", StringType.get()), + NestedField.optional(7, "no_stats_col", StringType.get()), + NestedField.optional(8, "ts_wtz_col", TimestampType.with_timezone()), + NestedField.optional(9, "ts_wotz_col", TimestampType.without_timezone()), + NestedField.optional(10, "big_decimal_type", DecimalType.of(38, 5)), + NestedField.optional(11, "small_decimal_type", DecimalType.of(10, 2)), + NestedField.optional(12, "date_type", DateType.get()), + ]) + + +@pytest.fixture(scope="session") +def rg_expected_schema_map(): + return {"string_col": "string_col", + "long_col": "long_col", + "int_col_renamed": "int_col", + "float_col": "float_col", + "null_col": "null_col", + "no_stats_col": "no_stats_col", + "ts_wtz_col": "ts_wtz_col", + "ts_wotz_col": "ts_wotz_col", + "big_decimal_type": "big_decimal_type", + "small_decimal_type": "small_decimal_type", + "date_type": "date_type" + } + + +@pytest.fixture(scope="session") +def rg_col_metadata(): + return [TestRowGroupColumn("string_col", 4, 12345, TestRowGroupColumnStatistics("b", "e", 0)), + TestRowGroupColumn("long_col", 12349, 12345, TestRowGroupColumnStatistics(0, 1234567890123, 0)), + TestRowGroupColumn("int_col_renamed", 24698, 12345, TestRowGroupColumnStatistics(0, 12345, 0)), + TestRowGroupColumn("float_col", 37043, 12345, TestRowGroupColumnStatistics(0.0, 123.45, 50)), + TestRowGroupColumn("null_col", 49388, 4, TestRowGroupColumnStatistics(None, None, 100)), + TestRowGroupColumn("no_stats_col", 61733, 4, None), + TestRowGroupColumn("ts_wtz_col", 74078, 4, + TestRowGroupColumnStatistics(datetime.strptime("2019-01-01 00:00:00-0000", + "%Y-%m-%d %H:%M:%S%z"), + datetime.strptime("2019-12-31 00:00:00-0000", + "%Y-%m-%d %H:%M:%S%z"), + 0)), + TestRowGroupColumn("ts_wotz_col", 86423, 4, + TestRowGroupColumnStatistics(datetime.strptime("2019-01-01 00:00:00-0000", + "%Y-%m-%d %H:%M:%S%z"), + datetime.strptime("2019-12-31 00:00:00-0000", + "%Y-%m-%d %H:%M:%S%z"), + 10)), + TestRowGroupColumn("big_decimal_type", 98768, 4, + # -123456789012345678.12345 to 123456789012345678.12345 + TestRowGroupColumnStatistics(b'\xff\xff\xff\xff\xff\xff\xfdb\xbdI\xb1\x89\x8e\xbe\xeb\x07', + b'\x00\x00\x00\x00\x00\x00\x02\x9dB\xb6NvqA\x14\xf9', + 10)), + TestRowGroupColumn("small_decimal_type", 111113, 4, + # 0 to 123.45 + TestRowGroupColumnStatistics(0, + 12345, + 10)), + TestRowGroupColumn("date_type", 123458, 4, + # 2020-01-01 to 2020-12-31 + TestRowGroupColumnStatistics(18262, 18262 + 365, 10)) + ] + + +@pytest.fixture(scope="session") +def pyarrow_schema(): + return pa.schema([pa.field("int_col", pa.int32(), False), + pa.field("bigint_col", pa.int64(), True), + pa.field("str_col", pa.string(), True), + pa.field("float_col", pa.float32(), True), + pa.field("dbl_col", pa.float64(), True), + pa.field("decimal_col", + pa.decimal128(9, 2), True), + pa.field("big_decimal_col", + pa.decimal128(19, 5), True), + pa.field("huge_decimal_col", + pa.decimal128(38, 9), True), + pa.field("date_col", pa.date32(), True), + pa.field("ts_col", pa.timestamp('us'), True), + pa.field("ts_wtz_col", + pa.timestamp('us', + 'America/New_York'), + True), + pa.field("bool_col", pa.bool_(), True)]) + + +@pytest.fixture(scope="session") +def pyarrow_primitive_array(): + return [pa.array([1, 2, 3, 4, 5], type=pa.int32()), + pa.array([1, 2, 3, None, 5], type=pa.int64()), + pa.array(['us', 'can', 'us', 'us', 'can'], type=pa.string()), + pa.array([1.0, 2.0, 3.0, 4.0, 5.0], type=pa.float32()), + pa.array([1.0, 2.0, 3.0, 4.0, 5.0], type=pa.float64()), + pa.array([Decimal("1.0"), Decimal("2.0"), Decimal("3.0"), + Decimal("4.0"), Decimal("5.0")], type=pa.decimal128(9, 2)), + pa.array([Decimal("1.0"), Decimal("2.0"), Decimal("3.0"), + Decimal("4.0"), Decimal("5.0")], type=pa.decimal128(19, 5)), + pa.array([Decimal("1.0"), Decimal("2.0"), Decimal("3.0"), + Decimal("4.0"), Decimal("5.0")], type=pa.decimal128(38, 9)), + pa.array([18506, 18507, 18508, 18508, 18510], type=pa.date32()), + pa.array([1598918400000000, 1599004800000000, + 1599091200000000, 1599177600000000, 1599264000000000], + type=pa.timestamp("us")), + pa.array([1598918400000000, 1599004800000000, + 1599091200000000, 1599177600000000, 1599264000000000], + type=pa.timestamp("us", 'America/New_York')), + pa.array([True, None, False, True, True], pa.bool_())] + + +@pytest.fixture(scope="session") +def primitive_type_test_file(pyarrow_primitive_array, pyarrow_schema): + with NamedTemporaryFile() as temp_file: + pq.write_table(pa.table(pyarrow_primitive_array, schema=pyarrow_schema), temp_file.name) + yield temp_file.name + + +@pytest.fixture(scope="session") +def primitive_type_test_parquet_file(primitive_type_test_file): + yield pq.ParquetFile(primitive_type_test_file) + + +@pytest.fixture(scope="session") +def unnested_complex_type_test_parquet_file(): + struct_fields = [("f1", pa.int32()), ("f2", pa.string())] + struct_type = pa.struct(struct_fields) + pyarrow_array = [pa.array([[1, 2, 3], [4, None, 6], None, [7, 8, 9]], type=pa.list_(pa.int32())), + pa.array([["a", "b", "c"], ["d", None, "e"], None, ["f", "g", "h"]], type=pa.list_(pa.string())), + pa.array([None, None, {"f1": 3, "f2": "c"}, {"f1": 4, "f2": "d"}], type=struct_type) + ] + with NamedTemporaryFile() as temp_file: + pq.write_table(pa.table(pyarrow_array, names=["list_int_col", 'list_str_col', 'struct_col']), + temp_file.name) + yield pq.ParquetFile(temp_file.name) + + +@pytest.fixture(scope="session") +def parquet_schema(type_test_parquet_file): + return type_test_parquet_file.schema + + +@pytest.fixture(scope="session") +def arrow_schema(type_test_parquet_file): + return type_test_parquet_file.schema_arrow diff --git a/tests/parquet/test_dataset_utils.py b/tests/parquet/test_dataset_utils.py new file mode 100644 index 0000000000..2b8084b6de --- /dev/null +++ b/tests/parquet/test_dataset_utils.py @@ -0,0 +1,52 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +from iceberg.api.expressions import Expressions +from iceberg.parquet.dataset_utils import get_dataset_filter +import pyarrow.dataset as ds +import pytest + + +@pytest.mark.parametrize("expr, dataset_filter, column_map", + [(Expressions.greater_than('a', 1), ds.field('a') > 1, {'a': 'a'}), + (Expressions.greater_than_or_equal('a', 1), ds.field('a') >= 1, {'a': 'a'}), + (Expressions.less_than('a', 1), ds.field('a') < 1, {'a': 'a'}), + (Expressions.less_than_or_equal('a', 1), ds.field('a') <= 1, {'a': 'a'}), + (Expressions.equal('a', 1), ds.field('a') == 1, {'a': 'a'}), + (Expressions.not_equal('a', 1), ds.field('a') != 1, {'a': 'a'}), + (Expressions.not_null('a'), ds.field('a').is_valid(), {'a': 'a'}), + (Expressions.is_null('a'), ~ds.field('a').is_valid(), {'a': 'a'}) + + ]) +def test_simple(expr, dataset_filter, column_map): + translated_dataset_filter = get_dataset_filter(expr, column_map) + assert dataset_filter.equals(translated_dataset_filter) + + +def test_not_conversion(): + expr = Expressions.not_(Expressions.greater_than('a', 1)) + translated_dataset_filter = get_dataset_filter(expr, {'a': 'a'}) + assert (~(ds.field("a") > 1)).equals(translated_dataset_filter) + + +def test_complex_expr(): + expr = Expressions.or_(Expressions.and_(Expressions.greater_than('a', 1), Expressions.equal("b", "US")), + Expressions.equal("c", True)) + + translated_dataset_filter = get_dataset_filter(expr, {'a': 'a', 'b': 'b', 'c': 'c'}) + dataset_filter = (((ds.field("a") > 1) & (ds.field("b") == "US")) | (ds.field("c") == True)) # noqa: E712 + assert dataset_filter.equals(translated_dataset_filter) diff --git a/tests/parquet/test_parquet_reader.py b/tests/parquet/test_parquet_reader.py new file mode 100644 index 0000000000..e170c3f6a0 --- /dev/null +++ b/tests/parquet/test_parquet_reader.py @@ -0,0 +1,285 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +from iceberg.api import Schema +from iceberg.api.expressions import Expressions +from iceberg.api.types import (BooleanType, + DateType, + DecimalType, + DoubleType, + FloatType, + IntegerType, + LongType, + NestedField, + StringType, + TimestampType) +from iceberg.core.filesystem import FileSystemInputFile, get_fs +from iceberg.parquet import ParquetReader +import pyarrow as pa + + +def test_basic_read(primitive_type_test_file, pyarrow_primitive_array, pyarrow_schema): + expected_schema = Schema([NestedField.required(1, "int_col", IntegerType.get()), + NestedField.optional(2, "bigint_col", LongType.get()), + NestedField.optional(3, "str_col", StringType.get()), + NestedField.optional(4, "float_col", FloatType.get()), + NestedField.optional(5, "dbl_col", DoubleType.get()), + NestedField.optional(6, "decimal_col", DecimalType.of(9, 2)), + NestedField.optional(7, "big_decimal_col", DecimalType.of(19, 5)), + NestedField.optional(8, "huge_decimal_col", DecimalType.of(38, 9)), + NestedField.optional(9, "date_col", DateType.get()), + NestedField.optional(10, "ts_col", TimestampType.without_timezone()), + NestedField.optional(11, "ts_wtz_col", TimestampType.with_timezone()), + NestedField.optional(12, "bool_col", BooleanType.get())]) + + input_file = FileSystemInputFile(get_fs(primitive_type_test_file, conf={}), primitive_type_test_file, {}) + reader = ParquetReader(input_file, expected_schema, {}, Expressions.always_true(), True) + + source_table = pa.table(pyarrow_primitive_array, schema=pyarrow_schema) + assert reader.read() == source_table + + +def test_projection(primitive_type_test_file, pyarrow_primitive_array, pyarrow_schema): + expected_schema = Schema([NestedField.required(1, "int_col", IntegerType.get()), + NestedField.optional(2, "bigint_col", LongType.get())]) + + input_file = FileSystemInputFile(get_fs(primitive_type_test_file, conf={}), primitive_type_test_file, {}) + reader = ParquetReader(input_file, expected_schema, {}, Expressions.always_true(), True) + + source_table = pa.table(pyarrow_primitive_array, schema=pyarrow_schema) + num_cols = source_table.num_columns + for i in range(1, num_cols - 1): + source_table = source_table.remove_column(num_cols - i) + + assert source_table == reader.read() + + +def test_column_rename(primitive_type_test_file): + expected_schema = Schema([NestedField.required(1, "int_col", IntegerType.get()), + NestedField.optional(2, "bigint_col", LongType.get()), + NestedField.optional(3, "string_col", StringType.get()), + NestedField.optional(4, "float_col", FloatType.get()), + NestedField.optional(5, "dbl_col", DoubleType.get())]) + + input_file = FileSystemInputFile(get_fs(primitive_type_test_file, conf={}), primitive_type_test_file, {}) + reader = ParquetReader(input_file, expected_schema, {}, Expressions.always_true(), True) + pyarrow_array = [pa.array([1, 2, 3, 4, 5], type=pa.int32()), + pa.array([1, 2, 3, None, 5], type=pa.int64()), + pa.array(['us', 'can', 'us', 'us', 'can'], type=pa.string()), + pa.array([1.0, 2.0, 3.0, 4.0, 5.0], type=pa.float32()), + pa.array([1.0, 2.0, 3.0, 4.0, 5.0], type=pa.float64())] + schema = pa.schema([pa.field("int_col", pa.int32(), False), + pa.field("bigint_col", pa.int64(), True), + pa.field("string_col", pa.string(), True), + pa.field("float_col", pa.float32(), True), + pa.field("dbl_col", pa.float64(), True)]) + + source_table = pa.table(pyarrow_array, schema=schema) + + target_table = reader.read() + assert source_table == target_table + + +def test_column_add(primitive_type_test_file): + expected_schema = Schema([NestedField.required(1, "int_col", IntegerType.get()), + NestedField.optional(2, "bigint_col", LongType.get()), + NestedField.optional(3, "string_col", StringType.get()), + NestedField.optional(4, "float_col", FloatType.get()), + NestedField.optional(5, "dbl_col", DoubleType.get()), + NestedField.optional(13, "int_col2", IntegerType.get())]) + + input_file = FileSystemInputFile(get_fs(primitive_type_test_file, conf={}), primitive_type_test_file, {}) + reader = ParquetReader(input_file, expected_schema, {}, Expressions.always_true(), True) + pyarrow_array = [pa.array([1, 2, 3, 4, 5], type=pa.int32()), + pa.array([1, 2, 3, None, 5], type=pa.int64()), + pa.array(['us', 'can', 'us', 'us', 'can'], type=pa.string()), + pa.array([1.0, 2.0, 3.0, 4.0, 5.0], type=pa.float32()), + pa.array([1.0, 2.0, 3.0, 4.0, 5.0], type=pa.float64()), + pa.array([None, None, None, None, None], type=pa.int32())] + source_table = pa.table(pyarrow_array, schema=pa.schema([pa.field("int_col", pa.int32(), nullable=False), + pa.field("bigint_col", pa.int64(), nullable=True), + pa.field("string_col", pa.string(), nullable=True), + pa.field("float_col", pa.float32(), nullable=True), + pa.field("dbl_col", pa.float64(), nullable=True), + pa.field("int_col2", pa.int32(), nullable=True), + ])) + + target_table = reader.read() + assert source_table == target_table + + +def test_decimal_column_add(primitive_type_test_file): + expected_schema = Schema([NestedField.required(1, "int_col", IntegerType.get()), + NestedField.optional(2, "bigint_col", LongType.get()), + NestedField.optional(4, "float_col", FloatType.get()), + NestedField.optional(5, "dbl_col", DoubleType.get()), + NestedField.optional(13, "new_dec_col", DecimalType.of(38, 9)) + ]) + + input_file = FileSystemInputFile(get_fs(primitive_type_test_file, conf={}), primitive_type_test_file, {}) + reader = ParquetReader(input_file, expected_schema, {}, Expressions.always_true(), True) + pyarrow_array = [pa.array([1, 2, 3, 4, 5], type=pa.int32()), + pa.array([1, 2, 3, None, 5], type=pa.int64()), + pa.array([1.0, 2.0, 3.0, 4.0, 5.0], type=pa.float32()), + pa.array([1.0, 2.0, 3.0, 4.0, 5.0], type=pa.float64()), + pa.array([None, None, None, None, None], type=pa.decimal128(38, 9))] + + source_table = pa.table(pyarrow_array, schema=pa.schema([pa.field("int_col", pa.int32(), nullable=False), + pa.field("bigint_col", pa.int64(), nullable=True), + pa.field("float_col", pa.float32(), nullable=True), + pa.field("dbl_col", pa.float64(), nullable=True), + pa.field("new_dec_col", pa.decimal128(38, 9), nullable=True) + ])) + + target_table = reader.read() + assert source_table == target_table + + +def test_column_reorder(primitive_type_test_file): + expected_schema = Schema([NestedField.required(1, "int_col", IntegerType.get()), + NestedField.optional(2, "bigint_col", LongType.get()), + NestedField.optional(4, "float_col", FloatType.get()), + NestedField.optional(5, "dbl_col", DoubleType.get()), + NestedField.optional(3, "string_col", StringType.get())]) + + input_file = FileSystemInputFile(get_fs(primitive_type_test_file, conf={}), primitive_type_test_file, {}) + reader = ParquetReader(input_file, expected_schema, {}, Expressions.always_true(), True) + pyarrow_array = [pa.array([1, 2, 3, 4, 5], type=pa.int32()), + pa.array([1, 2, 3, None, 5], type=pa.int64()), + pa.array([1.0, 2.0, 3.0, 4.0, 5.0], type=pa.float32()), + pa.array([1.0, 2.0, 3.0, 4.0, 5.0], type=pa.float64()), + pa.array(['us', 'can', 'us', 'us', 'can'], type=pa.string())] + source_table = pa.table(pyarrow_array, schema=pa.schema([pa.field("int_col", pa.int32(), nullable=False), + pa.field("bigint_col", pa.int64(), nullable=True), + pa.field("float_col", pa.float32(), nullable=True), + pa.field("dbl_col", pa.float64(), nullable=True), + pa.field("string_col", pa.string(), nullable=True) + ])) + + target_table = reader.read() + assert source_table == target_table + + +def test_column_upcast(primitive_type_test_file): + expected_schema = Schema([NestedField.required(1, "int_col", LongType.get())]) + + input_file = FileSystemInputFile(get_fs(primitive_type_test_file, conf={}), primitive_type_test_file, {}) + reader = ParquetReader(input_file, expected_schema, {}, Expressions.always_true(), True) + pyarrow_array = [pa.array([1, 2, 3, 4, 5], type=pa.int32())] + source_table = pa.table(pyarrow_array, schema=pa.schema([pa.field("int_col", pa.int64(), nullable=False)])) + + target_table = reader.read() + assert source_table == target_table + + +def test_filter(primitive_type_test_file): + expected_schema = Schema([NestedField.required(1, "int_col", IntegerType.get()), + NestedField.optional(2, "bigint_col", LongType.get()), + NestedField.optional(4, "float_col", FloatType.get()), + NestedField.optional(5, "dbl_col", DoubleType.get()), + NestedField.optional(3, "string_col", StringType.get())]) + + input_file = FileSystemInputFile(get_fs(primitive_type_test_file, conf={}), primitive_type_test_file, {}) + reader = ParquetReader(input_file, expected_schema, {}, Expressions.equal("string_col", "us"), True) + pyarrow_array = [pa.array([1, 3, 4], type=pa.int32()), + pa.array([1, 3, None], type=pa.int64()), + pa.array([1.0, 3.0, 4.0], type=pa.float32()), + pa.array([1.0, 3.0, 4.0], type=pa.float64()), + pa.array(['us', 'us', 'us'], type=pa.string())] + source_table = pa.table(pyarrow_array, schema=pa.schema([pa.field("int_col", pa.int32(), nullable=False), + pa.field("bigint_col", pa.int64(), nullable=True), + pa.field("float_col", pa.float32(), nullable=True), + pa.field("dbl_col", pa.float64(), nullable=True), + pa.field("string_col", pa.string(), nullable=True) + ])) + + target_table = reader.read() + assert source_table == target_table + + +def test_compound_filter(primitive_type_test_file): + expected_schema = Schema([NestedField.required(1, "int_col", IntegerType.get()), + NestedField.optional(2, "bigint_col", LongType.get()), + NestedField.optional(4, "float_col", FloatType.get()), + NestedField.optional(5, "dbl_col", DoubleType.get()), + NestedField.optional(3, "string_col", StringType.get())]) + + input_file = FileSystemInputFile(get_fs(primitive_type_test_file, conf={}), primitive_type_test_file, {}) + reader = ParquetReader(input_file, expected_schema, {}, Expressions.and_(Expressions.equal("string_col", "us"), + Expressions.equal("int_col", 1)), + True) + pyarrow_array = [pa.array([1], type=pa.int32()), + pa.array([1], type=pa.int64()), + pa.array([1.0], type=pa.float32()), + pa.array([1.0], type=pa.float64()), + pa.array(['us'], type=pa.string())] + + source_table = pa.table(pyarrow_array, schema=pa.schema([pa.field("int_col", pa.int32(), nullable=False), + pa.field("bigint_col", pa.int64(), nullable=True), + pa.field("float_col", pa.float32(), nullable=True), + pa.field("dbl_col", pa.float64(), nullable=True), + pa.field("string_col", pa.string(), nullable=True) + ])) + + target_table = reader.read() + assert source_table == target_table + + +def test_schema_evolution_filter(primitive_type_test_file): + expected_schema = Schema([NestedField.required(1, "int_col", IntegerType.get()), + NestedField.optional(2, "bigint_col", LongType.get()), + NestedField.optional(16, "other_new_col", LongType.get()), + NestedField.optional(4, "float_col", FloatType.get()), + NestedField.optional(5, "dbl_col", DoubleType.get()), + NestedField.optional(3, "string_col", StringType.get()), + NestedField.optional(15, "new_col", StringType.get())]) + + input_file = FileSystemInputFile(get_fs(primitive_type_test_file, conf={}), primitive_type_test_file, {}) + reader = ParquetReader(input_file, expected_schema, {}, Expressions.not_null("new_col"), True) + + schema = pa.schema([pa.field("int_col", pa.int32(), nullable=False), + pa.field("bigint_col", pa.int64(), nullable=True), + pa.field("other_new_col", pa.int64(), nullable=True), + pa.field("float_col", pa.float32(), nullable=True), + pa.field("dbl_col", pa.float64(), nullable=True), + pa.field("string_col", pa.string(), nullable=True), + pa.field("new_col", pa.string(), nullable=True)]) + + pyarrow_not_null_array = [pa.array([], type=pa.int32()), + pa.array([], type=pa.int64()), + pa.array([], type=pa.int32()), + pa.array([], type=pa.float32()), + pa.array([], type=pa.float64()), + pa.array([], type=pa.string()), + pa.array([], type=pa.string())] + + not_null_table = pa.table(pyarrow_not_null_array, schema=schema) + pyarrow_null_array = [pa.array([1, 2, 3, 4, 5], type=pa.int32()), + pa.array([1, 2, 3, None, 5], type=pa.int64()), + pa.array([None, None, None, None, None], type=pa.int64()), + pa.array([1.0, 2.0, 3.0, 4.0, 5.0], type=pa.float32()), + pa.array([1.0, 2.0, 3.0, 4.0, 5.0], type=pa.float64()), + pa.array(['us', 'can', 'us', 'us', 'can'], type=pa.string()), + pa.array([None, None, None, None, None], type=pa.string())] + null_table = pa.table(pyarrow_null_array, schema=schema) + + target_table = reader.read() + assert not_null_table == target_table + + reader = ParquetReader(input_file, expected_schema, {}, Expressions.is_null("new_col"), True) + target_table = reader.read() + assert null_table == target_table diff --git a/tests/parquet/test_parquet_to_iceberg.py b/tests/parquet/test_parquet_to_iceberg.py new file mode 100644 index 0000000000..478d016afa --- /dev/null +++ b/tests/parquet/test_parquet_to_iceberg.py @@ -0,0 +1,76 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +from iceberg.api import Schema +from iceberg.api.types import (BooleanType, + DateType, + DecimalType, + DoubleType, + FloatType, + IntegerType, + ListType, + LongType, + NestedField, + StringType, + StructType, + TimestampType) +from iceberg.parquet.parquet_to_iceberg import convert_parquet_to_iceberg + + +def compare_schema(expected_schema, converted_schema): + + assert len(expected_schema) == len(converted_schema) + for field in expected_schema.as_struct().fields: + converted_field = converted_schema.find_field(field.id) + assert converted_field is not None + assert field.name == converted_field.name + assert field.is_optional == converted_field.is_optional + assert field.type == converted_field.type + + +def test_primitive_types(primitive_type_test_parquet_file): + expected_schema = Schema([NestedField.required(1, "int_col", IntegerType.get()), + NestedField.optional(2, "bigint_col", LongType.get()), + NestedField.optional(3, "str_col", StringType.get()), + NestedField.optional(4, "float_col", FloatType.get()), + NestedField.optional(5, "dbl_col", DoubleType.get()), + NestedField.optional(6, "decimal_col", DecimalType.of(9, 2)), + NestedField.optional(7, "big_decimal_col", DecimalType.of(19, 5)), + NestedField.optional(8, "huge_decimal_col", DecimalType.of(38, 9)), + NestedField.optional(9, "date_col", DateType.get()), + NestedField.optional(10, "ts_col", TimestampType.without_timezone()), + NestedField.optional(11, "ts_wtz_col", TimestampType.with_timezone()), + NestedField.optional(12, "bool_col", BooleanType.get())]) + compare_schema(expected_schema, convert_parquet_to_iceberg(primitive_type_test_parquet_file)) + + +def test_unnested_complex_types(unnested_complex_type_test_parquet_file): + expected_schema = Schema([NestedField.optional(1, "list_int_col", ListType.of_optional(3, IntegerType.get())), + NestedField.optional(4, "list_str_col", ListType.of_optional(6, StringType.get())), + NestedField.optional(7, "struct_col", + StructType.of([NestedField.optional(8, "f1", IntegerType.get()), + NestedField.optional(9, "f2", StringType.get())])) + ]) + converted_schema = convert_parquet_to_iceberg(unnested_complex_type_test_parquet_file) + compare_schema(expected_schema, converted_schema) + + +def test_nested_complex_types(): + # This is a placeholder until pyarrow fully supports reading and writing nested types. + # Following a release containing this PR: + # https://github.com/apache/arrow/pull/8177 + pass From 1dd5994ad6ca10f5d1bf7c73cf2e93a8c69f3fdb Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Sat, 6 Mar 2021 02:23:33 +0100 Subject: [PATCH 037/642] Python: Fix build and pin dependencies (#2268) --- iceberg/api/expressions/literals.py | 8 ++++---- iceberg/parquet/parquet_to_iceberg.py | 2 +- setup.py | 13 ++++++------- tox.ini | 8 ++++---- 4 files changed, 15 insertions(+), 16 deletions(-) diff --git a/iceberg/api/expressions/literals.py b/iceberg/api/expressions/literals.py index e8a9fa5915..5090123a85 100644 --- a/iceberg/api/expressions/literals.py +++ b/iceberg/api/expressions/literals.py @@ -35,8 +35,8 @@ class Literals(object): EPOCH = datetime.datetime.utcfromtimestamp(0) EPOCH_DAY = EPOCH.date() - @staticmethod # noqa: C901 - def from_(value): + @staticmethod + def from_(value): # noqa: C901 if value is None: raise RuntimeError("Cannot create an expression literal from None") if isinstance(value, bool): @@ -77,8 +77,8 @@ class Literal(object): JAVA_MAX_FLOAT = 3.4028235E38 JAVA_MIN_FLOAT = -3.4028235E38 - @staticmethod # noqa: C901 - def of(value): + @staticmethod + def of(value): # noqa: C901 if isinstance(value, bool): return BooleanLiteral(value) diff --git a/iceberg/parquet/parquet_to_iceberg.py b/iceberg/parquet/parquet_to_iceberg.py index 674d02db0c..d2ea15c5af 100644 --- a/iceberg/parquet/parquet_to_iceberg.py +++ b/iceberg/parquet/parquet_to_iceberg.py @@ -42,7 +42,7 @@ arrow_type_map = {lib.Type_BOOL: lambda x=None: BooleanType.get(), lib.Type_DATE32: lambda x=None: DateType.get(), - lib.Type_DECIMAL: lambda x=None: DecimalType.of(x.precision, x.scale), + lib.Type_DECIMAL128: lambda x=None: DecimalType.of(x.precision, x.scale), lib.Type_DOUBLE: lambda x=None: DoubleType.get(), lib.Type_FIXED_SIZE_BINARY: lambda x=None: FixedType.of_length(x.byte_width), lib.Type_BINARY: lambda x=None: BinaryType.get(), diff --git a/setup.py b/setup.py index 3b33a546e1..062f2bcde1 100755 --- a/setup.py +++ b/setup.py @@ -17,7 +17,6 @@ from setuptools import setup - setup( name='iceberg', maintainer='Apache Iceberg Devs', @@ -25,19 +24,19 @@ description='Iceberg is a new table format for storing large, slow-moving tabular data', keywords='iceberg', url='https://github.com/apache/iceberg/blob/master/README.md', - python_requires='>=3.6', + python_requires='>=3.7', install_requires=['botocore', 'boto3', - 'fastavro', - 'hmsclient', + 'fastavro>=1.3.2<1.4.0', + 'hmsclient==0.1.1', 'mmh3', - 'pyparsing', + 'pyparsing>=2.4.7<2.5.0', 'python-dateutil', 'pytz', 'requests', 'retrying', 'pandas', - 'pyarrow>=2.0.0' + 'pyarrow>=3.0.0<4.0.0' ], extras_require={ "dev": [ @@ -49,8 +48,8 @@ license="Apache License 2.0", classifiers=[ "License :: OSI Approved :: Apache Software License", - "Programming Language :: Python :: 3.6", "Programming Language :: Python :: 3.7", "Programming Language :: Python :: 3.8", + "Programming Language :: Python :: 3.9", ], ) diff --git a/tox.ini b/tox.ini index 5f9b3ce744..dcc30c2d48 100644 --- a/tox.ini +++ b/tox.ini @@ -14,7 +14,7 @@ # limitations under the License. [tox] -envlist = py36,py37,py38,linters +envlist = py37,py38,py39,linters [testenv] deps = @@ -49,7 +49,7 @@ commands = basepython = python3 skip_install = true deps = - flake8 + flake8>=3.8.4 flake8-import-order>=0.9 flake8-bugbear commands = @@ -108,6 +108,6 @@ norecursedirs=.* [gh-actions] python = - 3.6: py36, linters 3.7: py37 - 3.8: py38 + 3.8: py38, linters + 3.9: py39 From 2af31a9eb2803e007152c52da8aedf8fa96de96c Mon Sep 17 00:00:00 2001 From: Szehon Ho Date: Thu, 1 Apr 2021 17:39:34 +0200 Subject: [PATCH 038/642] Python: Implement python drop_table (#2266) --- iceberg/api/snapshot.py | 4 + .../core/base_metastore_table_operations.py | 2 +- iceberg/core/base_snapshot.py | 7 +- iceberg/hive/hive_tables.py | 64 ++++++++++++-- tests/api/test_helpers.py | 62 +++++++++++++ tests/hive/test_hive_tables.py | 88 +++++++++++++++++-- 6 files changed, 211 insertions(+), 16 deletions(-) diff --git a/iceberg/api/snapshot.py b/iceberg/api/snapshot.py index d8164a3142..8a049e996e 100644 --- a/iceberg/api/snapshot.py +++ b/iceberg/api/snapshot.py @@ -34,6 +34,10 @@ def timestamp_millis(self): def manifests(self): raise NotImplementedError() + @property + def manifest_location(self): + raise NotImplementedError() + @property def summary(self): raise NotImplementedError() diff --git a/iceberg/core/base_metastore_table_operations.py b/iceberg/core/base_metastore_table_operations.py index dc4594a182..97bdae2e0c 100644 --- a/iceberg/core/base_metastore_table_operations.py +++ b/iceberg/core/base_metastore_table_operations.py @@ -107,7 +107,7 @@ def metadata_file_location(self, file_name, metadata=None): def delete_file(self, path): from .filesystem import get_fs - get_fs(path, self.conf).delete(path, False) + get_fs(path, self.conf).delete(path) @retry(wait_incrementing_start=100, wait_exponential_multiplier=4, wait_exponential_max=5000, stop_max_delay=600000, stop_max_attempt_number=2) diff --git a/iceberg/core/base_snapshot.py b/iceberg/core/base_snapshot.py index bf90ec4d16..c849825f6e 100644 --- a/iceberg/core/base_snapshot.py +++ b/iceberg/core/base_snapshot.py @@ -84,7 +84,7 @@ def manifests(self): @property def manifest_location(self): - return self._manifest_list.location if self._manifest_list is not None else None + return self._manifest_list.location() if self._manifest_list is not None else None @property def summary(self): @@ -106,7 +106,6 @@ def filter_rows(self, expr): def iterator(self, part_filter=None, row_filter=None, columns=None): if part_filter is None and row_filter is None and columns is None: return self.iterator(Expressions.always_true(), Expressions.always_true(), Filterable.ALL_COLUMNS) - return iter([self.get_filtered_manifest(path, part_filter, row_filter, columns) for path in self._manifest_files]) @@ -127,7 +126,7 @@ def __repr__(self): def __str__(self): return self.__repr__() - def get_filtered_manifest(self, path, part_filter, row_filter, columns): - reader = ManifestReader.read(self.ops.new_input_file(path)) + def get_filtered_manifest(self, path, part_filter=None, row_filter=None, columns=None): + reader = ManifestReader.read(self._ops.new_input_file(path)) self.add_closeable(reader) return reader diff --git a/iceberg/hive/hive_tables.py b/iceberg/hive/hive_tables.py index f5a7beefce..9260a91ea6 100644 --- a/iceberg/hive/hive_tables.py +++ b/iceberg/hive/hive_tables.py @@ -17,11 +17,42 @@ # under the License. # +import logging +from multiprocessing import cpu_count +from multiprocessing.dummy import Pool +import threading +from typing import Set -from hmsclient import hmsclient +from hmsclient import HMSClient, hmsclient from iceberg.core import BaseMetastoreTables +from iceberg.core.util import WORKER_THREAD_POOL_SIZE_PROP +from iceberg.hive import HiveTableOperations -from .hive_table_operations import HiveTableOperations +_logger = logging.getLogger(__name__) + + +# Handles physical deletion of files. +# Does not delete a file if it is referred more than once via Iceberg snapshot metadata pointers. +class DeleteFiles(object): + + def __init__(self, ops: HiveTableOperations): + self.ops = ops + self.seen: Set[str] = set() + self.set_lock = threading.Lock() + + def delete_file(self, path: str) -> None: + have_seen = True + with self.set_lock: + if path not in self.seen: + have_seen = False + self.seen.add(path) + + if not have_seen: + _logger.info("Deleting file: {path}".format(path=path)) + try: + self.ops.delete_file(path) + except OSError as e: + _logger.info("Error deleting file: {path}: {e}".format(path=path, e=e)) class HiveTables(BaseMetastoreTables): @@ -34,12 +65,33 @@ def __init__(self, conf): def new_table_ops(self, conf, database, table): return HiveTableOperations(conf, self.get_client(), database, table) - def drop(self, database, table): - raise RuntimeError("Not yet implemented") - - def get_client(self): + def get_client(self) -> HMSClient: from urllib.parse import urlparse metastore_uri = urlparse(self.conf[HiveTables.THRIFT_URIS]) client = hmsclient.HMSClient(host=metastore_uri.hostname, port=metastore_uri.port) return client + + def drop(self, database: str, table: str, purge: bool = False) -> None: + ops = self.new_table_ops(self.conf, database, table) + metadata = ops.current() + + # Drop from Hive Metastore + with self.get_client() as open_client: + _logger.info("Deleting {database}.{table} from Hive Metastore".format(database=database, table=table)) + open_client.drop_table(database, table, deleteData=False) + + if purge: + # Follow Iceberg metadata pointers to delete every file + if metadata is not None: + with Pool(self.conf.get(WORKER_THREAD_POOL_SIZE_PROP, + cpu_count())) as delete_pool: + deleter = DeleteFiles(ops) + for s in metadata.snapshots: + for m in s.manifests: + delete_pool.map(deleter.delete_file, + (i.path() for i in s.get_filtered_manifest(m.manifest_path).iterator())) + delete_pool.map(deleter.delete_file, (m.manifest_path for m in s.manifests)) + if s.manifest_location is not None: + delete_pool.map(deleter.delete_file, [s.manifest_location]) + delete_pool.map(deleter.delete_file, [ops.current_metadata_location]) diff --git a/tests/api/test_helpers.py b/tests/api/test_helpers.py index 2f75690efc..55d85ac2b0 100644 --- a/tests/api/test_helpers.py +++ b/tests/api/test_helpers.py @@ -92,3 +92,65 @@ def __init__(self, path, partition, record_count, value_counts=None, null_value_ def copy(self): return self + + +class MockHMSTable(object): + def __init__(self, params): + self.parameters = params + + +class MockManifestEntry(object): + def __init__(self, path): + self._path = path + + def path(self): + return self._path + + +class MockReader(object): + def __init__(self, entries): + self._entries = entries + + def iterator(self): + return iter(self._entries) + + +class MockTableOperations(object): + def __init__(self, metadata, location): + self.metadata = metadata + self.current_metadata_location = location + self.deleted = [] + + def current(self): + return self.metadata + + def delete_file(self, path): + self.deleted.append(path) + + +class MockManifest(object): + def __init__(self, manifest_path): + self.manifest_path = manifest_path + + +class MockSnapshot(object): + def __init__(self, location, manifests, manifest_to_entries): + self._location = location + self._manifests = manifests + self._manifest_to_entries = manifest_to_entries + + @property + def manifests(self): + return iter(self._manifests) + + @property + def manifest_location(self): + return self._location + + def get_filtered_manifest(self, path): + return self._manifest_to_entries[path] + + +class MockMetadata(object): + def __init__(self, snapshots): + self.snapshots = snapshots diff --git a/tests/hive/test_hive_tables.py b/tests/hive/test_hive_tables.py index 0729287f13..481674712c 100644 --- a/tests/hive/test_hive_tables.py +++ b/tests/hive/test_hive_tables.py @@ -24,11 +24,8 @@ import mock import pytest from pytest import raises - - -class MockHMSTable(object): - def __init__(self, params): - self.parameters = params +from tests.api.test_helpers import MockHMSTable, MockManifest, MockManifestEntry, MockMetadata, MockReader, \ + MockSnapshot, MockTableOperations @mock.patch("iceberg.hive.HiveTableOperations.refresh_from_metadata_location") @@ -157,3 +154,84 @@ def test_create_tables(client, current_call, base_scan_schema, base_scan_partiti current_call.return_value = tbl[0].args[0].parameters['metadata_location'] tables.load("test.test_123") + + +@mock.patch("iceberg.hive.HiveTableOperations.refresh_from_metadata_location") +@mock.patch("iceberg.hive.HiveTableOperations.current") +@mock.patch("iceberg.hive.HiveTables.get_client") +def test_drop_tables(client, metadata, refresh_call, tmpdir): + + parameters = {"table_type": "ICEBERG", + "partition_spec": [], + "metadata_location": "s3://path/to/iceberg.metadata.json"} + + client.return_value.__enter__.return_value.get_table.return_value = MockHMSTable(parameters) + conf = {"hive.metastore.uris": 'thrift://hms:port', + "hive.metastore.warehouse.dir": tmpdir} + tables = HiveTables(conf) + tables.drop("test", "test_123", purge=False) + client.return_value.__enter__.return_value.drop_table.assert_called_with("test", "test_123", deleteData=False) + + +@mock.patch("iceberg.hive.HiveTableOperations.refresh_from_metadata_location") +@mock.patch("iceberg.hive.HiveTables.new_table_ops") +@mock.patch("iceberg.hive.HiveTables.get_client") +def test_drop_tables_purge(client, current_ops, refresh_call, tmpdir): + mock_snapshots = [MockSnapshot(location="snap-a.avro", + manifests=[ + MockManifest("a-manifest.avro"), + MockManifest("b-manifest.avro")], + manifest_to_entries={ + "a-manifest.avro": MockReader( + [MockManifestEntry("a.parquet"), + MockManifestEntry("b.parquet")]), + "b-manifest.avro": MockReader( + [MockManifestEntry("c.parquet"), + MockManifestEntry("d.parquet")]) + }), + MockSnapshot(location="snap-b.avro", + manifests=[ + MockManifest("b-manifest.avro"), + MockManifest("c-manifest.avro"), + MockManifest("d-manifest.avro")], + manifest_to_entries={ + "b-manifest.avro": MockReader( + [MockManifestEntry("c.parquet"), + MockManifestEntry("d.parquet")]), + "c-manifest.avro": MockReader( + [MockManifestEntry("e.parquet"), + MockManifestEntry("f.parquet")]), + "d-manifest.avro": MockReader( + [MockManifestEntry("g.parquet"), + MockManifestEntry("h.parquet")]) + }) + ] + ops = MockTableOperations(MockMetadata(mock_snapshots), "a.json") + current_ops.return_value = ops + + parameters = {"table_type": "ICEBERG", + "partition_spec": [], + "metadata_location": "s3://path/to/iceberg.metadata.json"} + client.return_value.__enter__.return_value.get_table.return_value = MockHMSTable(parameters) + + conf = {"hive.metastore.uris": 'thrift://hms:port', + "hive.metastore.warehouse.dir": tmpdir} + tables = HiveTables(conf) + tables.drop("test", "test_123", purge=True) + + assert len(ops.deleted) == len(set(ops.deleted)), "Paths should only be deleted once" + assert "a.json" in ops.deleted + assert "snap-a.avro" in ops.deleted + assert "snap-b.avro" in ops.deleted + assert "a-manifest.avro" in ops.deleted + assert "b-manifest.avro" in ops.deleted + assert "c-manifest.avro" in ops.deleted + assert "d-manifest.avro" in ops.deleted + assert "a.parquet" in ops.deleted + assert "b.parquet" in ops.deleted + assert "c.parquet" in ops.deleted + assert "d.parquet" in ops.deleted + assert "e.parquet" in ops.deleted + assert "f.parquet" in ops.deleted + assert "g.parquet" in ops.deleted + assert "h.parquet" in ops.deleted From 64ea0f1d4fbf8a59c28accd3af543f6fba3f84b3 Mon Sep 17 00:00:00 2001 From: Ted Gooch Date: Fri, 9 Apr 2021 05:58:39 -0700 Subject: [PATCH 039/642] [python] Updating Expressions with the Term object used in Predicates (#2399) Co-authored-by: tgooch --- iceberg/api/expressions/__init__.py | 4 +- iceberg/api/expressions/expression.py | 6 + iceberg/api/expressions/expressions.py | 16 + .../inclusive_metrics_evaluator.py | 14 +- iceberg/api/expressions/predicate.py | 280 ++++++++++++++---- iceberg/api/expressions/projections.py | 4 +- iceberg/api/expressions/reference.py | 46 ++- .../expressions/strict_metrics_evaluator.py | 16 +- iceberg/api/expressions/term.py | 73 +++++ iceberg/api/expressions/transform.py | 75 +++++ iceberg/api/schema.py | 14 +- tests/api/expressions/test_evaluator.py | 15 +- .../expressions/test_expression_binding.py | 12 +- .../api/expressions/test_predicate_binding.py | 12 +- 14 files changed, 479 insertions(+), 108 deletions(-) create mode 100644 iceberg/api/expressions/term.py create mode 100644 iceberg/api/expressions/transform.py diff --git a/iceberg/api/expressions/__init__.py b/iceberg/api/expressions/__init__.py index 4c2189fb20..18450a2bd5 100644 --- a/iceberg/api/expressions/__init__.py +++ b/iceberg/api/expressions/__init__.py @@ -52,7 +52,6 @@ "Operation", "Or", "Predicate", - "Reference", "ResidualEvaluator", "strict", "StrictMetricsEvaluator", @@ -103,7 +102,6 @@ strict, StrictProjection) from .reference import (BoundReference, - NamedReference, - Reference) + NamedReference) from .residual_evaluator import ResidualEvaluator from .strict_metrics_evaluator import StrictMetricsEvaluator diff --git a/iceberg/api/expressions/expression.py b/iceberg/api/expressions/expression.py index eefa89e89d..3dc0f06a31 100644 --- a/iceberg/api/expressions/expression.py +++ b/iceberg/api/expressions/expression.py @@ -43,6 +43,8 @@ class Operation(Enum): FALSE = "FALSE" IS_NULL = "IS_NULL" NOT_NULL = "NOT_NULL" + IS_NAN = "IS_NAN" + NOT_NAN = "NOT_NAN" LT = "LT" LT_EQ = "LT_EQ" GT = "GT" @@ -60,6 +62,10 @@ def negate(self): # noqa return Operation.NOT_NULL elif self == Operation.NOT_NULL: return Operation.IS_NULL + elif self == Operation.IS_NAN: + return Operation.NOT_NAN + elif self == Operation.NOT_NAN: + return Operation.IS_NAN elif self == Operation.LT: return Operation.GT_EQ elif self == Operation.LT_EQ: diff --git a/iceberg/api/expressions/expressions.py b/iceberg/api/expressions/expressions.py index 4b6f4e5f78..4bcdc2cd12 100644 --- a/iceberg/api/expressions/expressions.py +++ b/iceberg/api/expressions/expressions.py @@ -74,6 +74,14 @@ def is_null(name): def not_null(name): return UnboundPredicate(Operation.NOT_NULL, Expressions.ref(name)) + @staticmethod + def is_nan(name): + return UnboundPredicate(Operation.IS_NAN, Expressions.ref(name)) + + @staticmethod + def not_nan(name): + return UnboundPredicate(Operation.NOT_NAN, Expressions.ref(name)) + @staticmethod def less_than(name, value): return UnboundPredicate(Operation.LT, Expressions.ref(name), value) @@ -197,6 +205,12 @@ def is_null(self, ref): def not_null(self, ref): return NotImplementedError() + def is_nan(self, ref): + return NotImplementedError() + + def not_nan(self, ref): + return NotImplementedError() + def lt(self, ref, lit): return NotImplementedError() @@ -230,6 +244,8 @@ def predicate(self, pred): # noqa return self.is_null(pred.ref) elif pred.op == Operation.NOT_NULL: return self.not_null(pred.ref) + elif pred.op in [Operation.IS_NAN, Operation.NOT_NAN]: + raise NotImplementedError("IS_NAN and NOT_NAN not fully implemented for expressions") elif pred.op == Operation.LT: return self.lt(pred.ref, pred.lit) elif pred.op == Operation.LT_EQ: diff --git a/iceberg/api/expressions/inclusive_metrics_evaluator.py b/iceberg/api/expressions/inclusive_metrics_evaluator.py index e9ab1a0bd7..73b636fff0 100644 --- a/iceberg/api/expressions/inclusive_metrics_evaluator.py +++ b/iceberg/api/expressions/inclusive_metrics_evaluator.py @@ -81,7 +81,7 @@ def or_(self, left_result, right_result): return left_result or right_result def is_null(self, ref): - id = ref.field_id + id = ref.field.field_id if self.struct.field(id=id) is None: raise RuntimeError("Cannot filter by nested column: %s" % self.schema.find_field(id)) @@ -92,7 +92,7 @@ def is_null(self, ref): return MetricsEvalVisitor.ROWS_MIGHT_MATCH def not_null(self, ref): - id = ref.field_id + id = ref.field.field_id if self.struct.field(id=id) is None: raise RuntimeError("Cannot filter by nested column: %s" % self.schema.find_field(id)) @@ -104,7 +104,7 @@ def not_null(self, ref): return MetricsEvalVisitor.ROWS_MIGHT_MATCH def lt(self, ref, lit): - id = ref.field_id + id = ref.field.field_id field = self.struct.field(id=id) if field is None: @@ -118,7 +118,7 @@ def lt(self, ref, lit): return MetricsEvalVisitor.ROWS_MIGHT_MATCH def lt_eq(self, ref, lit): - id = ref.field_id + id = ref.field.field_id field = self.struct.field(id=id) if field is None: @@ -132,7 +132,7 @@ def lt_eq(self, ref, lit): return MetricsEvalVisitor.ROWS_MIGHT_MATCH def gt(self, ref, lit): - id = ref.field_id + id = ref.field.field_id field = self.struct.field(id=id) if field is None: @@ -146,7 +146,7 @@ def gt(self, ref, lit): return MetricsEvalVisitor.ROWS_MIGHT_MATCH def gt_eq(self, ref, lit): - id = ref.field_id + id = ref.field.field_id field = self.struct.field(id=id) if field is None: @@ -160,7 +160,7 @@ def gt_eq(self, ref, lit): return MetricsEvalVisitor.ROWS_MIGHT_MATCH def eq(self, ref, lit): - id = ref.field_id + id = ref.field.field_id field = self.struct.field(id=id) if field is None: diff --git a/iceberg/api/expressions/predicate.py b/iceberg/api/expressions/predicate.py index 4558dc95de..e868a07243 100644 --- a/iceberg/api/expressions/predicate.py +++ b/iceberg/api/expressions/predicate.py @@ -15,23 +15,40 @@ # specific language governing permissions and limitations # under the License. +from __future__ import annotations + +from math import isnan +from typing import Any, List, Optional, TYPE_CHECKING, Union + from iceberg.exceptions import ValidationException from .expression import (Expression, - FALSE, - Operation, - TRUE) -from .literals import (Literal, + Operation) +from .literals import (BaseLiteral, Literals) -from .reference import BoundReference +from .term import BoundTerm, UnboundTerm +from ..types import TypeID + +if TYPE_CHECKING: + from iceberg.api import StructLike class Predicate(Expression): - def __init__(self, op, ref, lit): - self.op = op - self.ref = ref - self.lit = lit + def __init__(self, op: Operation, term: Union[BoundTerm, UnboundTerm]): + if term is None: + raise ValueError("Term cannot be None") + + self.op: Operation = op + self.term: Union[BoundTerm, UnboundTerm] = term + + @property + def ref(self): + return self.term.ref + + @property + def lit(self): + raise NotImplementedError("Not Implemented for base class") def __eq__(self, other): if id(self) == id(other): @@ -70,72 +87,209 @@ def __str__(self): class BoundPredicate(Predicate): - def __init__(self, op, ref, lit=None): - super(BoundPredicate, self).__init__(op, ref, lit) + def __init__(self, op: Operation, term: BoundTerm, lit: BaseLiteral = None, literals: List[BaseLiteral] = None, + is_unary_predicate: bool = False, is_literal_predicate: bool = False, + is_set_predicate: bool = False): + self.is_unary_predicate = is_unary_predicate + self.is_literal_predicate = is_literal_predicate + self.is_set_predicate = is_set_predicate - def negate(self): - return BoundPredicate(self.op.negate(), self.ref, self.lit) + super(BoundPredicate, self).__init__(op, term) + ValidationException.check(sum([is_unary_predicate, is_literal_predicate, is_set_predicate]) == 1, + "Only a single predicate type may be set: %s=%s, %s=%s, %s=%s", + ("is_unary_predicate", is_unary_predicate, + "is_literal_predicate", is_literal_predicate, + "is_set_predicate", is_set_predicate)) + + self._literals: Optional[List[BaseLiteral]] = None + if self.is_unary_predicate: + ValidationException.check(lit is None, "Unary Predicates may not have a literal", ()) + + elif self.is_literal_predicate: + ValidationException.check(lit is not None, "Literal Predicates must have a literal set", ()) + self._literals = [lit] # type: ignore + + elif self.is_set_predicate: + ValidationException.check(literals is not None, "Set Predicates must have literals set", ()) + self._literals = literals + else: + raise ValueError(f"Unable to instantiate {op} -> (lit={lit}, literal={literals}") + + @property + def lit(self) -> Optional[BaseLiteral]: + if self._literals is None or len(self._literals) == 0: + return None + return self._literals[0] + + def eval(self, struct: StructLike) -> bool: + ValidationException.check(isinstance(self.term, BoundTerm), "Term must be bound to eval: %s", (self.term)) + return self.test(self.term.eval(struct)) # type: ignore + + def test(self, struct: StructLike = None, value: Any = None) -> bool: + ValidationException.check(struct is None or value is None, "Either struct or value must be none", ()) + if struct is not None: + ValidationException.check(isinstance(self.term, BoundTerm), "Term must be bound to eval: %s", (self.term)) + return self.test(value=self.term.eval(struct)) # type: ignore + else: + if self.is_unary_predicate: + return self.test_unary_predicate(value) + elif self.is_literal_predicate: + return self.test_literal_predicate(value) + else: + return self.test_set_predicate(value) + + def test_unary_predicate(self, value: Any) -> bool: + + if self.op == Operation.IS_NULL: + return value is None + elif self.op == Operation.NOT_NULL: + return value is not None + elif self.op == Operation.IS_NAN: + return isnan(value) + elif self.op == Operation.NOT_NAN: + return not isnan(value) + else: + raise ValueError(f"{self.op} is not a valid unary predicate") + + def test_literal_predicate(self, value: Any) -> bool: + if self.lit is None: + raise ValidationException("Literal must not be none", ()) + + if self.op == Operation.LT: + return value < self.lit.value + elif self.op == Operation.LT_EQ: + return value <= self.lit.value + elif self.op == Operation.GT: + return value > self.lit.value + elif self.op == Operation.GT_EQ: + return value >= self.lit.value + elif self.op == Operation.EQ: + return value == self.lit.value + elif self.op == Operation.NOT_EQ: + return value != self.lit.value + else: + raise ValueError(f"{self.op} is not a valid literal predicate") + + def test_set_predicate(self, value: Any) -> bool: + if self._literals is None: + raise ValidationException("Literals must not be none", ()) + + if self.op == Operation.IN: + return value in self._literals + elif self.op == Operation.NOT_IN: + return value not in self._literals + else: + raise ValueError(f"{self.op} is not a valid set predicate") class UnboundPredicate(Predicate): - def __init__(self, op, named_ref, value=None, lit=None): - if value is None and lit is None: - super(UnboundPredicate, self).__init__(op, named_ref, None) - if isinstance(value, Literal): + def __init__(self, op, term, value=None, lit=None, values=None, literals=None): + self._literals = None + num_set_args = sum([1 for x in [value, lit, values, literals] if x is not None]) + + if num_set_args > 1: + raise ValueError(f"Only one of value={value}, lit={lit}, values={values}, literals={literals} may be set") + super(UnboundPredicate, self).__init__(op, term) + if isinstance(value, BaseLiteral): lit = value value = None if value is not None: - super(UnboundPredicate, self).__init__(op, named_ref, Literals.from_(value)) + self._literals = [Literals.from_(value)] elif lit is not None: - super(UnboundPredicate, self).__init__(op, named_ref, lit) + self._literals = [lit] + elif values is not None: + self._literals = map(Literals.from_, values) + elif literals is not None: + self._literals = literals + + @property + def literals(self): + return self._literals + + @property + def lit(self): + if self.op in [Operation.IN, Operation.NOT_IN]: + raise ValueError(f"{self.op} predicate cannot return a literal") + + return None if self.literals is None else self.literals[0] def negate(self): - return UnboundPredicate(self.op.negate(), self.ref, self.lit) + return UnboundPredicate(self.op.negate(), self.term, literals=self.literals) - def bind(self, struct, case_sensitive=True): # noqa: C901 - if case_sensitive: - field = struct.field(self.ref.name) - else: - field = struct.case_insensitive_field(self.ref.name.lower()) + def bind(self, struct, case_sensitive=True): + bound = self.term.bind(struct, case_sensitive=case_sensitive) - ValidationException.check(field is not None, - "Cannot find field '%s' in struct %s", (self.ref.name, struct)) + if self.literals is None: + return self.bind_unary_operation(bound) + elif self.op in [Operation.IN, Operation.NOT_IN]: + return self.bind_in_operation(bound) - if self.lit is None: - if self.op == Operation.IS_NULL: - if field.is_required: - return FALSE - return BoundPredicate(Operation.IS_NULL, BoundReference(struct, field.field_id)) - elif self.op == Operation.NOT_NULL: - if field.is_required: - return TRUE - return BoundPredicate(Operation.NOT_NULL, BoundReference(struct, field.field_id)) + return self.bind_literal_operation(bound) + + def bind_unary_operation(self, bound_term: BoundTerm) -> BoundPredicate: + from .expressions import Expressions + if self.op == Operation.IS_NULL: + if bound_term.ref.field.is_required: + return Expressions.always_false() + return BoundPredicate(Operation.IS_NULL, bound_term, is_unary_predicate=True) + elif self.op == Operation.NOT_NULL: + if bound_term.ref.field.is_required: + return Expressions.always_true() + return BoundPredicate(Operation.NOT_NULL, bound_term, is_unary_predicate=True) + elif self.op in [Operation.IS_NAN, Operation.NOT_NAN]: + if not self.floating_type(bound_term.ref.type.type_id): + raise ValidationException(f"{self.op} cannot be used with a non-floating column", ()) + return BoundPredicate(self.op, bound_term, is_unary_predicate=True) + + raise ValidationException(f"Operation must be in [IS_NULL, NOT_NULL, IS_NAN, NOT_NAN] was:{self.op}", ()) + + def bind_in_operation(self, bound_term): + from .expressions import Expressions + + def convert_literal(lit): + converted = lit.to(bound_term) + ValidationException.check(converted is not None, + "Invalid Value for conversion to type %s: %s (%s)", + (bound_term.type, lit, lit.__class__.__name__)) + return converted + + converted_literals = filter(lambda x: x != Literals.above_max() and x != Literals.below_min(), + [convert_literal(lit) for lit in self.literals]) + if len(converted_literals) == 0: + return Expressions.always_true() if Operation.NOT_IN else Expressions.always_false() + literal_set = set(converted_literals) + if len(literal_set) == 1: + if self.op == Operation.IN: + return BoundPredicate(Operation.EQ, bound_term, literal_set[0]) + elif self.op == Operation.NOT_IN: + return BoundPredicate(Operation.NOT_EQ, bound_term, literal_set[0]) else: - raise ValidationException("Operation must be IS_NULL or NOT_NULL", None) - - literal = self.lit.to(field.type) - if literal is None: - raise ValidationException("Invalid value for comparison inclusive type %s: %s (%s)", - (field.type, self.lit.value, type(self.lit.value))) - elif literal == Literals.above_max(): - if self.op in (Operation.LT, - Operation.LT_EQ, - Operation.NOT_EQ): - return TRUE - elif self.op in (Operation.GT, - Operation.GT_EQ, - Operation.EQ): - return FALSE - - elif literal == Literals.below_min(): - if self.op in (Operation.LT, - Operation.LT_EQ, - Operation.NOT_EQ): - return FALSE - elif self.op in (Operation.GT, - Operation.GT_EQ, - Operation.EQ): - return TRUE - - return BoundPredicate(self.op, BoundReference(struct, field.field_id), literal) + raise ValidationException("Operation must be in or not in", ()) + + return BoundPredicate(self.op, bound_term, literals=literal_set, is_set_predicate=True) + + def bind_literal_operation(self, bound_term): + from .expressions import Expressions + + lit = self.lit.to(bound_term.type) + ValidationException.check(lit is not None, + "Invalid Value for conversion to type %s: %s (%s)", + (bound_term.type, self.lit, self.lit.__class__.__name__)) + + if lit == Literals.above_max(): + if self.op in [Operation.LT, Operation.LT_EQ, Operation.NOT_EQ]: + return Expressions.always_true() + elif self.op in [Operation.GT, Operation.GT_EQ, Operation.EQ]: + return Expressions.always_false() + elif lit == Literals.below_min(): + if self.op in [Operation.LT, Operation.LT_EQ, Operation.NOT_EQ]: + return Expressions.always_false() + elif self.op in [Operation.GT, Operation.GT_EQ, Operation.EQ]: + return Expressions.always_true() + + return BoundPredicate(self.op, bound_term, lit=lit, is_literal_predicate=True) + + @staticmethod + def floating_type(type_id: TypeID) -> bool: + return type_id in [TypeID.FLOAT, TypeID.DOUBLE] diff --git a/iceberg/api/expressions/projections.py b/iceberg/api/expressions/projections.py index 1a4a6bd98f..7521b42b65 100644 --- a/iceberg/api/expressions/projections.py +++ b/iceberg/api/expressions/projections.py @@ -83,7 +83,7 @@ def predicate(self, pred): if isinstance(pred, UnboundPredicate): return super(InclusiveProjection, self).predicate(pred) - part = self.spec.get_field_by_source_id(pred.ref.field_id) + part = self.spec.get_field_by_source_id(pred.ref.field.field_id) if part is None: return self.always_true() @@ -101,7 +101,7 @@ def __init__(self, spec): super(StrictProjection, self).__init__(spec) def predicate(self, pred): - part = self.spec.get_field_by_source_id(pred.ref.field_id) + part = self.spec.get_field_by_source_id(pred.ref.field.field_id) if part is None: return self.always_false() diff --git a/iceberg/api/expressions/reference.py b/iceberg/api/expressions/reference.py index d634e34fd5..9637607e56 100644 --- a/iceberg/api/expressions/reference.py +++ b/iceberg/api/expressions/reference.py @@ -15,18 +15,23 @@ # specific language governing permissions and limitations # under the License. +from __future__ import annotations + +from typing import Any, TYPE_CHECKING + from iceberg.exceptions import ValidationException +from ..types import StructType -class Reference(object): - pass +if TYPE_CHECKING: + from iceberg.api import StructLike -class BoundReference(Reference): +class BoundReference: - def __init__(self, struct, field_id): - self.field_id = field_id - self.pos = self.find(field_id, struct) + def __init__(self, struct, field): + self.field = field + self.pos = self.find(field.field_id, struct) self._type = struct.fields[self.pos].type @property @@ -39,7 +44,7 @@ def __eq__(self, other): elif other is None or not isinstance(other, BoundReference): return False - return self.field_id == other.field_id and self.pos == other.pos and self._type == other._type + return self.field.field_id == other.field.field_id and self.pos == other.pos and self._type == other._type def __ne__(self, other): return not self.__eq__(other) @@ -47,7 +52,7 @@ def __ne__(self, other): def find(self, field_id, struct): fields = struct.fields for i, field in enumerate(fields): - if field.field_id == self.field_id: + if field.field_id == self.field.field_id: return i raise ValidationException("Cannot find top-level field id %d in struct: %s", (field_id, struct)) @@ -56,12 +61,19 @@ def get(self, struct): return struct.get(self.pos) def __str__(self): - return "ref(id={id}, pos={pos}, type={_type})".format(id=self.field_id, + return "ref(id={id}, pos={pos}, type={_type})".format(id=self.field.field_id, pos=self.pos, _type=self._type) + @property + def ref(self): + return self + + def eval(self, struct: StructLike) -> Any: + return self.get(struct) + -class NamedReference(Reference): +class NamedReference: def __init__(self, name): super(NamedReference, self).__init__() @@ -70,6 +82,20 @@ def __init__(self, name): self.name = name + @property + def ref(self): + return self + + def bind(self, struct: StructType, case_sensitive: bool = True) -> BoundReference: + from iceberg.api import Schema + schema = Schema(struct.fields) + field = schema.find_field(self.name) if case_sensitive else schema.case_insensitive_find_field(self.name) + + ValidationException.check(field is not None, "Cannot find field '%s' in struct: %s", (self.name, + schema.as_struct())) + + return BoundReference(struct, field) + def __eq__(self, other): if id(self) == id(other): return True diff --git a/iceberg/api/expressions/strict_metrics_evaluator.py b/iceberg/api/expressions/strict_metrics_evaluator.py index 6d44df3408..16b7728eed 100644 --- a/iceberg/api/expressions/strict_metrics_evaluator.py +++ b/iceberg/api/expressions/strict_metrics_evaluator.py @@ -83,7 +83,7 @@ def or_(self, left_result, right_result): return left_result or right_result def is_null(self, ref): - id = ref.field_id + id = ref.field.field_id if self.struct.field(id=id) is None: raise RuntimeError("Cannot filter by nested column: %s" % self.schema.find_field(id)) @@ -95,7 +95,7 @@ def is_null(self, ref): return StrictMetricsEvaluator.MetricsEvalVisitor.ROWS_MIGHT_NOT_MATCH def not_null(self, ref): - id = ref.field_id + id = ref.field.field_id if self.struct.field(id=id) is None: raise RuntimeError("Cannot filter by nested column: %s" % self.schema.find_field(id)) @@ -106,7 +106,7 @@ def not_null(self, ref): def lt(self, ref, lit): # Rows must match when: <----------Min----Max---X-------> - id = ref.field_id + id = ref.field.field_id field = self.struct.field(id=id) @@ -122,7 +122,7 @@ def lt(self, ref, lit): def lt_eq(self, ref, lit): # Rows must match when: <----------Min----Max---X-------> - id = ref.field_id + id = ref.field.field_id field = self.struct.field(id=id) @@ -138,7 +138,7 @@ def lt_eq(self, ref, lit): def gt(self, ref, lit): # Rows must match when: <-------X---Min----Max----------> - id = ref.field_id + id = ref.field.field_id field = self.struct.field(id=id) @@ -154,7 +154,7 @@ def gt(self, ref, lit): def gt_eq(self, ref, lit): # Rows must match when: <-------X---Min----Max----------> - id = ref.field_id + id = ref.field.field_id field = self.struct.field(id=id) @@ -170,7 +170,7 @@ def gt_eq(self, ref, lit): def eq(self, ref, lit): # Rows must match when Min == X == Max - id = ref.field_id + id = ref.field.field_id field = self.struct.field(id=id) @@ -194,7 +194,7 @@ def eq(self, ref, lit): def not_eq(self, ref, lit): # Rows must match when X < Min or Max < X because it is not in the range - id = ref.field_id + id = ref.field.field_id field = self.struct.field(id=id) diff --git a/iceberg/api/expressions/term.py b/iceberg/api/expressions/term.py new file mode 100644 index 0000000000..e135783a08 --- /dev/null +++ b/iceberg/api/expressions/term.py @@ -0,0 +1,73 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +from __future__ import annotations + +from typing import Any, TYPE_CHECKING + +from iceberg.api.types import StructType + +if TYPE_CHECKING: + from iceberg.api import StructLike + + +class Bound(object): + def eval(self, struct: StructLike) -> Any: + """Evaluate current object against struct""" + + +class Unbound(object): + @property + def ref(self): + """the underlying reference""" + + def bind(self, struct: StructType, case_sensitive: bool = True) -> Bound: + """bind the current object based on the given struct and return the Bound object""" + + +class Term(object): + @property + def ref(self): + """Expose the reference for this term""" + + @property + def type(self): + """accessor for term type """ + + +class BoundTerm(Bound, Term): + + @property + def ref(self): + raise NotImplementedError("Base class does not have implementation") + + @property + def type(self): + raise NotImplementedError("Base class does not have implementation") + + def eval(self, struct: StructLike): + raise NotImplementedError("Base class does not have implementation") + + +class UnboundTerm(Unbound, Term): + + @property + def ref(self): + raise NotImplementedError("Base class does not have implementation") + + def bind(self, struct: StructType, case_sensitive: bool = True): + raise NotImplementedError("Base class does not have implementation") diff --git a/iceberg/api/expressions/transform.py b/iceberg/api/expressions/transform.py new file mode 100644 index 0000000000..cf2e4563f0 --- /dev/null +++ b/iceberg/api/expressions/transform.py @@ -0,0 +1,75 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +from .reference import BoundReference, NamedReference +from .. import StructLike +from ..transforms import Transform, Transforms +from ..types import StructType +from ...exceptions import ValidationException + + +class BoundTransform: + + def __init__(self, ref: BoundReference, transform: Transform): + self._ref = ref + self._transform = transform + + @property + def ref(self): + return self._ref + + @property + def type(self): + return self.transform.get_result_type(self.ref.type) + + @property + def transform(self): + return self._transform + + def eval(self, struct: StructLike): + return self.transform.apply(self.ref.eval(struct)) + + def __str__(self): + return f"{self.transform}({self.ref})" + + +class UnboundTransform: + + def __init__(self, ref: NamedReference, transform: Transform): + self._ref = ref + self._transform = transform + + @property + def ref(self): + return self._ref + + @property + def transform(self): + return self._transform + + def bind(self, struct: StructType, case_sensitive: bool = True): + bound_ref = self.ref.bind(struct, case_sensitive=case_sensitive) + + type_transform = Transforms.from_string(bound_ref.type, str(self.transform)) + ValidationException.check(type_transform.can_transform(bound_ref.type), + "Cannot bind: %s cannot transform %s values from '%s'", (self.transform, + bound_ref.type, + self.ref.name)) + + return BoundTransform(bound_ref, type_transform) + + def __str__(self): + return f"{self.transform}({self.ref})" diff --git a/iceberg/api/schema.py b/iceberg/api/schema.py index 39ae9adf9a..0d70bf8a3e 100644 --- a/iceberg/api/schema.py +++ b/iceberg/api/schema.py @@ -100,12 +100,22 @@ def find_field(self, id): return self.lazy_id_to_field().get(id) if not id: - raise RuntimeError("Invalid Column Name (empty)") + raise ValueError("Invalid Column Name (empty)") id = self.lazy_name_to_id().get(id) - if id: + if id is not None: return self.lazy_id_to_field().get(id) + def case_insensitive_find_field(self, name): + if name is None: + raise ValueError("Invalid Column Name (empty)") + + id = self.lazy_lowercase_name_to_id().get(name.lower()) + if id is not None: + return self.lazy_id_to_field().get(id) + + return None + def find_column_name(self, id): if isinstance(id, int): field = self.lazy_id_to_field().get(id) diff --git a/tests/api/expressions/test_evaluator.py b/tests/api/expressions/test_evaluator.py index b5820f4dc5..da7b454d5b 100644 --- a/tests/api/expressions/test_evaluator.py +++ b/tests/api/expressions/test_evaluator.py @@ -17,7 +17,8 @@ import iceberg.api.expressions as exp -from iceberg.api.types import (IntegerType, +from iceberg.api.types import (FloatType, + IntegerType, NestedField, StringType, StructType) @@ -146,3 +147,15 @@ def test_char_seq_value(row_of): evaluator = exp.evaluator.Evaluator(struct, exp.expressions.Expressions.equal("s", "abc")) assert evaluator.eval(row_of(("abc",))) assert not evaluator.eval(row_of(("abcd",))) + + +def test_nan_errors(row_of): + # Placeholder until NaN support is fully implemented + struct = StructType.of([NestedField.required(34, "f", FloatType.get())]) + evaluator = exp.evaluator.Evaluator(struct, exp.expressions.Expressions.is_nan("f")) + with raises(NotImplementedError): + evaluator.eval(row_of((123.4,))) + + evaluator = exp.evaluator.Evaluator(struct, exp.expressions.Expressions.not_nan("f")) + with raises(NotImplementedError): + evaluator.eval(row_of((123.4,))) diff --git a/tests/api/expressions/test_expression_binding.py b/tests/api/expressions/test_expression_binding.py index f6a4d983fb..556a54ae30 100644 --- a/tests/api/expressions/test_expression_binding.py +++ b/tests/api/expressions/test_expression_binding.py @@ -83,10 +83,10 @@ def test_and(assert_all_bound, assert_and_unwrap): left = assert_and_unwrap(and_.left, None) # should bind x correctly - assert 0 == left.ref.field_id + assert 0 == left.ref.field.field_id right = assert_and_unwrap(and_.right, None) # should bind y correctly - assert 1 == right.ref.field_id + assert 1 == right.ref.field.field_id def test_or(assert_all_bound, assert_and_unwrap): @@ -99,10 +99,10 @@ def test_or(assert_all_bound, assert_and_unwrap): left = assert_and_unwrap(or_.left, None) # should bind z correctly - assert 2 == left.ref.field_id + assert 2 == left.ref.field.field_id right = assert_and_unwrap(or_.right, None) # should bind y correctly - assert 1 == right.ref.field_id + assert 1 == right.ref.field.field_id def test_not(assert_all_bound, assert_and_unwrap): @@ -114,7 +114,7 @@ def test_not(assert_all_bound, assert_and_unwrap): child = assert_and_unwrap(not_.child, None) # should bind x correctly - assert 0 == child.ref.field_id + assert 0 == child.ref.field.field_id def test_always_true(): @@ -140,4 +140,4 @@ def test_basic_simplification(assert_and_unwrap): bound = Binder.bind(STRUCT, Expressions.not_(Expressions.not_(Expressions.less_than("y", 100)))) pred = assert_and_unwrap(bound, None) - assert 1 == pred.ref.field_id + assert 1 == pred.ref.field.field_id diff --git a/tests/api/expressions/test_predicate_binding.py b/tests/api/expressions/test_predicate_binding.py index 44d45167a1..7b7e4beafb 100644 --- a/tests/api/expressions/test_predicate_binding.py +++ b/tests/api/expressions/test_predicate_binding.py @@ -39,7 +39,7 @@ def test_multiple_fields(assert_and_unwrap): expr = unbound.bind(struct) bound = assert_and_unwrap(expr) - assert 11 == bound.ref.field_id + assert 11 == bound.ref.field.field_id assert Operation.LT == bound.op assert 6 == bound.lit.value @@ -60,7 +60,7 @@ def test_comparison_predicate_binding(op, assert_and_unwrap): bound = assert_and_unwrap(unbound.bind(struct)) assert 5 == bound.lit.value - assert 14 == bound.ref.field_id + assert 14 == bound.ref.field.field_id assert op == bound.op @@ -70,7 +70,7 @@ def test_literal_converison(op, assert_and_unwrap): bound = assert_and_unwrap(unbound.bind(struct)) assert Decimal(12.40).quantize(Decimal(".01")).as_tuple() == bound.lit.value.as_tuple() - assert 15 == bound.ref.field_id + assert 15 == bound.ref.field.field_id assert op == bound.op @@ -81,7 +81,7 @@ def test_invalid_conversions(op): try: unbound.bind(struct) except ValidationException as e: - assert e.args[0].startswith("Invalid value for comparison inclusive type float: 12.40") + assert e.args[0].startswith('Invalid Value for conversion to type float: "12.40" (StringLiteral)') def test_long_to_integer_conversion(assert_and_unwrap): @@ -179,7 +179,7 @@ def test_is_null(assert_and_unwrap): bound = assert_and_unwrap(expr) assert Operation.IS_NULL == bound.op - assert 19 == bound.ref.field_id + assert 19 == bound.ref.field.field_id assert bound.lit is None required = StructType.of([NestedField.required(20, "s", StringType.get())]) @@ -192,7 +192,7 @@ def test_not_null(assert_and_unwrap): expr = unbound.bind(optional) bound = assert_and_unwrap(expr) assert Operation.NOT_NULL == bound.op - assert 21 == bound.ref.field_id + assert 21 == bound.ref.field.field_id assert bound.lit is None required = StructType.of([NestedField.required(22, "s", StringType.get())]) From d254594a72c5cc67a7554ac0c0177747103851d7 Mon Sep 17 00:00:00 2001 From: jun-he Date: Thu, 3 Jun 2021 00:29:05 -0700 Subject: [PATCH 040/642] [Python] add to_byte_buffer to literal classes (#2655) --- iceberg/api/expressions/literals.py | 50 ++++++++++++++++++----------- iceberg/api/types/conversions.py | 29 +++++++++-------- iceberg/core/partition_summary.py | 10 +++--- tests/api/expressions/conftest.py | 24 +++++++------- tests/api/test_conversions.py | 15 +++++++-- 5 files changed, 76 insertions(+), 52 deletions(-) diff --git a/iceberg/api/expressions/literals.py b/iceberg/api/expressions/literals.py index 5090123a85..41387ce2bc 100644 --- a/iceberg/api/expressions/literals.py +++ b/iceberg/api/expressions/literals.py @@ -27,6 +27,7 @@ TRUE) from .java_variables import (JAVA_MAX_FLOAT, JAVA_MIN_FLOAT) +from ..types.conversions import Conversions from ..types.type import TypeID @@ -60,7 +61,7 @@ def from_(value): # noqa: C901 elif isinstance(value, Decimal): return DecimalLiteral(value) else: - raise RuntimeError("Unimplemented Type Literal") + raise NotImplementedError("Unimplemented Type Literal for value: %s" % value) @staticmethod def above_max(): @@ -101,15 +102,20 @@ def of(value): # noqa: C901 elif isinstance(value, Decimal): return DecimalLiteral(value) - def to(self, type): + def to(self, type_var): + raise NotImplementedError() + + def to_byte_buffer(self): raise NotImplementedError() class BaseLiteral(Literal): - def __init__(self, value): + def __init__(self, value, type_id): self.value = value + self.byte_buffer = None + self.type_id = type_id - def to(self, type): + def to(self, type_var): raise NotImplementedError() def __eq__(self, other): @@ -129,11 +135,17 @@ def __repr__(self): def __str__(self): return str(self.value) + def to_byte_buffer(self): + if self.byte_buffer is None: + self.byte_buffer = Conversions.to_byte_buffer(self.type_id, self.value) + + return self.byte_buffer + class ComparableLiteral(BaseLiteral): - def __init__(self, value): - super(ComparableLiteral, self).__init__(value) + def __init__(self, value, type_id): + super(ComparableLiteral, self).__init__(value, type_id) def to(self, type): raise NotImplementedError() @@ -212,7 +224,7 @@ def __str__(self): class BooleanLiteral(ComparableLiteral): def __init__(self, value): - super(BooleanLiteral, self).__init__(value) + super(BooleanLiteral, self).__init__(value, TypeID.BOOLEAN) def to(self, type_var): if type_var.type_id == TypeID.BOOLEAN: @@ -222,7 +234,7 @@ def to(self, type_var): class IntegerLiteral(ComparableLiteral): def __init__(self, value): - super(IntegerLiteral, self).__init__(value) + super(IntegerLiteral, self).__init__(value, TypeID.INTEGER) def to(self, type_var): if type_var.type_id == TypeID.INTEGER: @@ -247,7 +259,7 @@ def to(self, type_var): class LongLiteral(ComparableLiteral): def __init__(self, value): - super(LongLiteral, self).__init__(value) + super(LongLiteral, self).__init__(value, TypeID.LONG) def to(self, type_var): # noqa: C901 if type_var.type_id == TypeID.INTEGER: @@ -279,7 +291,7 @@ def to(self, type_var): # noqa: C901 class FloatLiteral(ComparableLiteral): def __init__(self, value): - super(FloatLiteral, self).__init__(value) + super(FloatLiteral, self).__init__(value, TypeID.FLOAT) def to(self, type_var): if type_var.type_id == TypeID.FLOAT: @@ -300,7 +312,7 @@ def to(self, type_var): class DoubleLiteral(ComparableLiteral): def __init__(self, value): - super(DoubleLiteral, self).__init__(value) + super(DoubleLiteral, self).__init__(value, TypeID.DOUBLE) def to(self, type_var): if type_var.type_id == TypeID.FLOAT: @@ -326,7 +338,7 @@ def to(self, type_var): class DateLiteral(ComparableLiteral): def __init__(self, value): - super(DateLiteral, self).__init__(value) + super(DateLiteral, self).__init__(value, TypeID.DATE) def to(self, type_var): if type_var.type_id == TypeID.DATE: @@ -336,7 +348,7 @@ def to(self, type_var): class TimeLiteral(ComparableLiteral): def __init__(self, value): - super(TimeLiteral, self).__init__(value) + super(TimeLiteral, self).__init__(value, TypeID.TIME) def to(self, type_var): if type_var.type_id == TypeID.TIME: @@ -346,7 +358,7 @@ def to(self, type_var): class TimestampLiteral(ComparableLiteral): def __init__(self, value): - super(TimestampLiteral, self).__init__(value) + super(TimestampLiteral, self).__init__(value, TypeID.TIMESTAMP) def to(self, type_var): if type_var.type_id == TypeID.TIMESTAMP: @@ -358,7 +370,7 @@ def to(self, type_var): class DecimalLiteral(ComparableLiteral): def __init__(self, value): - super(DecimalLiteral, self).__init__(value) + super(DecimalLiteral, self).__init__(value, TypeID.DECIMAL) def to(self, type_var): if type_var.type_id == TypeID.DECIMAL and type_var.scale == abs(self.value.as_tuple().exponent): @@ -367,7 +379,7 @@ def to(self, type_var): class StringLiteral(BaseLiteral): def __init__(self, value): - super(StringLiteral, self).__init__(value) + super(StringLiteral, self).__init__(value, TypeID.STRING) def to(self, type_var): # noqa: C901 import dateutil.parser @@ -445,7 +457,7 @@ def __str__(self): class UUIDLiteral(ComparableLiteral): def __init__(self, value): - super(UUIDLiteral, self).__init__(value) + super(UUIDLiteral, self).__init__(value, TypeID.UUID) def to(self, type_var): if type_var.type_id == TypeID.UUID: @@ -454,7 +466,7 @@ def to(self, type_var): class FixedLiteral(BaseLiteral): def __init__(self, value): - super(FixedLiteral, self).__init__(value) + super(FixedLiteral, self).__init__(value, TypeID.FIXED) def to(self, type_var): if type_var.type_id == TypeID.FIXED: @@ -499,7 +511,7 @@ def __ge__(self, other): class BinaryLiteral(BaseLiteral): def __init__(self, value): - super(BinaryLiteral, self).__init__(value) + super(BinaryLiteral, self).__init__(value, TypeID.BINARY) def to(self, type_var): if type_var.type_id == TypeID.FIXED: diff --git a/iceberg/api/types/conversions.py b/iceberg/api/types/conversions.py index 6768b6170a..ae92a636ba 100644 --- a/iceberg/api/types/conversions.py +++ b/iceberg/api/types/conversions.py @@ -39,17 +39,18 @@ class Conversions(object): TypeID.DECIMAL: lambda as_str: Decimal(as_str), } - to_byte_buff_mapping = {TypeID.BOOLEAN: lambda type_var, value: struct.pack("QQ', (value.int >> 64) & 0xFFFFFFFFFFFFFFFF, - value.int & 0xFFFFFFFFFFFFFFFF), + to_byte_buff_mapping = {TypeID.BOOLEAN: lambda type_id, value: struct.pack("QQ', (value.int >> 64) + & 0xFFFFFFFFFFFFFFFF, value.int + & 0xFFFFFFFFFFFFFFFF), # TypeId.FIXED: lambda as_str: None, # TypeId.BINARY: lambda as_str: None, # TypeId.DECIMAL: lambda type_var, value: struct.pack(value.quantize( @@ -81,11 +82,11 @@ def from_partition_string(type_var, as_string): return part_func(as_string) @staticmethod - def to_byte_buffer(type_var, value): + def to_byte_buffer(type_id, value): try: - return Conversions.to_byte_buff_mapping.get(type_var.type_id)(type_var, value) + return Conversions.to_byte_buff_mapping.get(type_id)(type_id, value) except KeyError: - raise RuntimeError("Cannot Serialize Type: %s" % type_var) + raise TypeError("Cannot Serialize Type: %s" % type_id) @staticmethod def from_byte_buffer(type_var, buffer_var): diff --git a/iceberg/core/partition_summary.py b/iceberg/core/partition_summary.py index 8b7ec3bdd4..45239ae854 100644 --- a/iceberg/core/partition_summary.py +++ b/iceberg/core/partition_summary.py @@ -42,18 +42,18 @@ def update_fields(self, key): class PartitionFieldStats(object): - def __init__(self, type): + def __init__(self, type_var): self.contains_null = False - self.type = type + self.type = type_var self.min = None self.max = None def to_summary(self): - lower_bound = None if self.min is None else Conversions.to_byte_buffer(self.type, self.min) - upper_bound = None if self.max is None else Conversions.to_byte_buffer(self.type, self.max) + lower_bound = None if self.min is None else Conversions.to_byte_buffer(self.type.type_id, self.min) + upper_bound = None if self.max is None else Conversions.to_byte_buffer(self.type.type_id, self.max) return GenericPartitionFieldSummary(contains_null=self.contains_null, lower_bound=lower_bound, - uppwer_bound=upper_bound) + upper_bound=upper_bound) def update(self, value): if value is None: diff --git a/tests/api/expressions/conftest.py b/tests/api/expressions/conftest.py index 28cc43d239..a0e748f582 100644 --- a/tests/api/expressions/conftest.py +++ b/tests/api/expressions/conftest.py @@ -208,9 +208,9 @@ def file(): # null value counts {4: 50, 5: 10, 6: 0}, # lower bounds - {1: Conversions.to_byte_buffer(IntegerType.get(), 30)}, + {1: Conversions.to_byte_buffer(IntegerType.get().type_id, 30)}, # upper bounds - {1: Conversions.to_byte_buffer(IntegerType.get(), 79)}) + {1: Conversions.to_byte_buffer(IntegerType.get().type_id, 79)}) @pytest.fixture(scope="session") @@ -220,10 +220,10 @@ def strict_file(): 50, {4: 50, 5: 50, 6: 50}, {4: 50, 5: 10, 6: 0}, - {1: Conversions.to_byte_buffer(IntegerType.get(), 30), - 7: Conversions.to_byte_buffer(IntegerType.get(), 5)}, - {1: Conversions.to_byte_buffer(IntegerType.get(), 79), - 7: Conversions.to_byte_buffer(IntegerType.get(), 5)} + {1: Conversions.to_byte_buffer(IntegerType.get().type_id, 30), + 7: Conversions.to_byte_buffer(IntegerType.get().type_id, 5)}, + {1: Conversions.to_byte_buffer(IntegerType.get().type_id, 79), + 7: Conversions.to_byte_buffer(IntegerType.get().type_id, 5)} ) @@ -406,17 +406,17 @@ def inc_man_spec(): def inc_man_file(): return TestManifestFile("manifest-list.avro", 1024, 0, int(time.time() * 1000), 5, 10, 0, (TestFieldSummary(False, - Conversions.to_byte_buffer(IntegerType.get(), 30), - Conversions.to_byte_buffer(IntegerType.get(), 79)), + Conversions.to_byte_buffer(IntegerType.get().type_id, 30), + Conversions.to_byte_buffer(IntegerType.get().type_id, 79)), TestFieldSummary(True, None, None), TestFieldSummary(True, - Conversions.to_byte_buffer(StringType.get(), 'a'), - Conversions.to_byte_buffer(StringType.get(), 'z')), + Conversions.to_byte_buffer(StringType.get().type_id, 'a'), + Conversions.to_byte_buffer(StringType.get().type_id, 'z')), TestFieldSummary(False, - Conversions.to_byte_buffer(StringType.get(), 'a'), - Conversions.to_byte_buffer(StringType.get(), 'z')) + Conversions.to_byte_buffer(StringType.get().type_id, 'a'), + Conversions.to_byte_buffer(StringType.get().type_id, 'z')) )) diff --git a/tests/api/test_conversions.py b/tests/api/test_conversions.py index be8587ab66..7f6778c6d7 100644 --- a/tests/api/test_conversions.py +++ b/tests/api/test_conversions.py @@ -17,9 +17,12 @@ import unittest -from iceberg.api.types import (DoubleType, +from iceberg.api.expressions import Literal +from iceberg.api.types import (DateType, + DoubleType, IntegerType, - LongType) + LongType, + StringType) from iceberg.api.types.conversions import Conversions @@ -32,3 +35,11 @@ def test_from_bytes(self): b'\xd2\x04\x00\x00\x00\x00\x00\x00')) self.assertAlmostEqual(1.2345, Conversions.from_byte_buffer(DoubleType.get(), b'\x8d\x97\x6e\x12\x83\xc0\xf3\x3f')) + self.assertEqual("foo", Conversions.from_byte_buffer(StringType.get(), b'foo')) + + def test_to_bytes(self): + self.assertEqual(b'\x00\x00', Literal.of(False).to_byte_buffer()) + self.assertEqual(b'\x01\x00', Literal.of(True).to_byte_buffer()) + self.assertEqual(b'foo', Literal.of("foo").to_byte_buffer()) + self.assertEqual(b'\xd2\x04\x00\x00', Literal.of(1234).to_byte_buffer()) + self.assertEqual(b'\xe8\x03\x00\x00', Literal.of(1000).to(DateType.get()).to_byte_buffer()) From 943cc9355845bbcd73d52ed8eee1242cdf004ff7 Mon Sep 17 00:00:00 2001 From: jun-he Date: Mon, 7 Jun 2021 01:02:06 -0700 Subject: [PATCH 041/642] Update to_byte_buff_mapping and from_byte_buff_mapping to be consistent and add additional tests. (#2672) --- iceberg/api/types/conversions.py | 42 +++++++++++++++--------------- tests/api/test_conversions.py | 44 +++++++++++++++++++++++++++++--- 2 files changed, 61 insertions(+), 25 deletions(-) diff --git a/iceberg/api/types/conversions.py b/iceberg/api/types/conversions.py index ae92a636ba..c6dbbf6a15 100644 --- a/iceberg/api/types/conversions.py +++ b/iceberg/api/types/conversions.py @@ -42,34 +42,34 @@ class Conversions(object): to_byte_buff_mapping = {TypeID.BOOLEAN: lambda type_id, value: struct.pack("QQ', (value.int >> 64) & 0xFFFFFFFFFFFFFFFF, value.int & 0xFFFFFFFFFFFFFFFF), - # TypeId.FIXED: lambda as_str: None, - # TypeId.BINARY: lambda as_str: None, + TypeID.FIXED: lambda type_id, value: value, + TypeID.BINARY: lambda type_id, value: value, # TypeId.DECIMAL: lambda type_var, value: struct.pack(value.quantize( # Decimal('.' + "".join(['0' for x in range(0, type_var.scale)]) + '1')) } - from_byte_buff_mapping = {TypeID.BOOLEAN: lambda type_var, value: struct.unpack('QQ', value)[0] << 64 | struct.unpack('>QQ', value)[1]), - TypeID.FIXED: lambda type_var, value: value, - TypeID.BINARY: lambda type_var, value: value} + TypeID.FIXED: lambda type_id, value: value, + TypeID.BINARY: lambda type_id, value: value} @staticmethod def from_partition_string(type_var, as_string): @@ -90,11 +90,11 @@ def to_byte_buffer(type_id, value): @staticmethod def from_byte_buffer(type_var, buffer_var): - return Conversions.internal_from_byte_buffer(type_var, buffer_var) + return Conversions.internal_from_byte_buffer(type_var.type_id, buffer_var) @staticmethod - def internal_from_byte_buffer(type_var, buffer_var): + def internal_from_byte_buffer(type_id, buffer_var): try: - return Conversions.from_byte_buff_mapping[type_var.type_id](type_var.type_id, buffer_var) + return Conversions.from_byte_buff_mapping.get(type_id)(type_id, buffer_var) except KeyError: - raise RuntimeError("Cannot Serialize Type: %s" % type_var) + raise TypeError("Cannot deserialize Type: %s" % type_id) diff --git a/tests/api/test_conversions.py b/tests/api/test_conversions.py index 7f6778c6d7..d7397ceb37 100644 --- a/tests/api/test_conversions.py +++ b/tests/api/test_conversions.py @@ -16,30 +16,66 @@ # under the License. import unittest +import uuid from iceberg.api.expressions import Literal -from iceberg.api.types import (DateType, +from iceberg.api.types import (BinaryType, + BooleanType, + DateType, DoubleType, + FixedType, + FloatType, IntegerType, LongType, - StringType) + StringType, + TimestampType, + TimeType, + UUIDType) from iceberg.api.types.conversions import Conversions class TestConversions(unittest.TestCase): def test_from_bytes(self): + self.assertEqual(False, Conversions.from_byte_buffer(BooleanType.get(), b'\x00\x00')) + self.assertEqual(True, Conversions.from_byte_buffer(BooleanType.get(), b'\x01\x00')) self.assertEqual(1234, Conversions.from_byte_buffer(IntegerType.get(), b'\xd2\x04\x00\x00')) self.assertEqual(1234, Conversions.from_byte_buffer(LongType.get(), b'\xd2\x04\x00\x00\x00\x00\x00\x00')) + self.assertAlmostEqual(1.2345, Conversions.from_byte_buffer(FloatType.get(), + b'\x19\x04\x9e?'), places=5) self.assertAlmostEqual(1.2345, Conversions.from_byte_buffer(DoubleType.get(), b'\x8d\x97\x6e\x12\x83\xc0\xf3\x3f')) + self.assertEqual(1234, Conversions.from_byte_buffer(DateType.get(), + b'\xd2\x04\x00\x00')) + self.assertEqual(100000000000, Conversions.from_byte_buffer(TimeType.get(), + b'\x00\xe8vH\x17\x00\x00\x00')) + self.assertEqual(100000000000, Conversions.from_byte_buffer(TimestampType.with_timezone(), + b'\x00\xe8vH\x17\x00\x00\x00')) + self.assertEqual(100000000000, Conversions.from_byte_buffer(TimestampType.without_timezone(), + b'\x00\xe8vH\x17\x00\x00\x00')) self.assertEqual("foo", Conversions.from_byte_buffer(StringType.get(), b'foo')) + self.assertEqual(uuid.UUID("f79c3e09-677c-4bbd-a479-3f349cb785e7"), + Conversions.from_byte_buffer(UUIDType.get(), b'\xf7\x9c>\tg|K\xbd\xa4y?4\x9c\xb7\x85\xe7')) + self.assertEqual(b'foo', Conversions.from_byte_buffer(FixedType.of_length(3), b'foo')) + self.assertEqual(b'foo', Conversions.from_byte_buffer(BinaryType.get(), b'foo')) def test_to_bytes(self): self.assertEqual(b'\x00\x00', Literal.of(False).to_byte_buffer()) self.assertEqual(b'\x01\x00', Literal.of(True).to_byte_buffer()) - self.assertEqual(b'foo', Literal.of("foo").to_byte_buffer()) self.assertEqual(b'\xd2\x04\x00\x00', Literal.of(1234).to_byte_buffer()) - self.assertEqual(b'\xe8\x03\x00\x00', Literal.of(1000).to(DateType.get()).to_byte_buffer()) + self.assertEqual(b'\xd2\x04\x00\x00\x00\x00\x00\x00', Literal.of(1234).to(LongType.get()).to_byte_buffer()) + self.assertEqual(b'\x19\x04\x9e?', Literal.of(1.2345).to_byte_buffer()) + self.assertEqual(b'\x8d\x97\x6e\x12\x83\xc0\xf3\x3f', Literal.of(1.2345).to(DoubleType.get()).to_byte_buffer()) + self.assertEqual(b'\xd2\x04\x00\x00', Literal.of(1234).to(DateType.get()).to_byte_buffer()) + self.assertEqual(b'\x00\xe8vH\x17\x00\x00\x00', Literal.of(100000000000).to(TimeType.get()).to_byte_buffer()) + self.assertEqual(b'\x00\xe8vH\x17\x00\x00\x00', + Literal.of(100000000000).to(TimestampType.with_timezone()).to_byte_buffer()) + self.assertEqual(b'\x00\xe8vH\x17\x00\x00\x00', + Literal.of(100000000000).to(TimestampType.without_timezone()).to_byte_buffer()) + self.assertEqual(b'foo', Literal.of("foo").to_byte_buffer()) + self.assertEqual(b'\xf7\x9c>\tg|K\xbd\xa4y?4\x9c\xb7\x85\xe7', + Literal.of(uuid.UUID("f79c3e09-677c-4bbd-a479-3f349cb785e7")).to_byte_buffer()) + self.assertEqual(b'foo', Literal.of(bytes(b'foo')).to_byte_buffer()) + self.assertEqual(b'foo', Literal.of(bytearray(b'foo')).to_byte_buffer()) From 16fbc0cd65e2a1045e746d6581a877d25b909525 Mon Sep 17 00:00:00 2001 From: Ted Gooch Date: Thu, 17 Jun 2021 09:43:39 -0700 Subject: [PATCH 042/642] [python] Adding type ignores for dependencies without type support (#2698) --- tox.ini | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tox.ini b/tox.ini index dcc30c2d48..7de9703cb7 100644 --- a/tox.ini +++ b/tox.ini @@ -60,6 +60,8 @@ basepython = python3 skip_install = true deps = mypy + types-pytz + types-python-dateutil commands = mypy --ignore-missing-imports iceberg/ From b5d4a184268e82ca9da85e8e1982df01596a7650 Mon Sep 17 00:00:00 2001 From: jun-he Date: Thu, 17 Jun 2021 16:31:56 -0700 Subject: [PATCH 043/642] [Python] support custom target name in partition spec builder (#2689) * support custom target name in partition spec builder * address the comments. --- iceberg/api/partition_spec.py | 136 ++++++++++++++++----------- iceberg/api/transforms/dates.py | 11 +++ iceberg/api/transforms/timestamps.py | 11 +++ iceberg/api/transforms/transform.py | 3 + tests/api/test_partition_spec.py | 84 +++++++++++++++++ 5 files changed, 192 insertions(+), 53 deletions(-) create mode 100644 tests/api/test_partition_spec.py diff --git a/iceberg/api/partition_spec.py b/iceberg/api/partition_spec.py index 395001b601..e2f106d76b 100644 --- a/iceberg/api/partition_spec.py +++ b/iceberg/api/partition_spec.py @@ -21,7 +21,7 @@ from .partition_field import PartitionField from .schema import Schema -from .transforms import Transforms +from .transforms import Transform, Transforms from .types import (NestedField, StructType) @@ -202,6 +202,7 @@ def __init__(self, schema): self.schema = schema self.fields = list() self.partition_names = set() + self.dedup_fields = dict() self.spec_id = 0 self.last_assigned_field_id = PartitionSpec.PARTITION_DATA_ID_START - 1 @@ -213,15 +214,35 @@ def with_spec_id(self, spec_id): self.spec_id = spec_id return self - def check_and_add_partition_name(self, name): + def check_and_add_partition_name(self, name, source_column_id=None): + schema_field = self.schema.find_field(name) + if source_column_id is not None: + if schema_field is not None and schema_field.field_id != source_column_id: + raise ValueError("Cannot create identity partition sourced from different field in schema: %s" % name) + else: + if schema_field is not None: + raise ValueError("Cannot create partition from name that exists in schema: %s" % name) + if name is None or name == "": - raise RuntimeError("Cannot use empty or null partition name") + raise ValueError("Cannot use empty or null partition name: %s" % name) if name in self.partition_names: - raise RuntimeError("Cannot use partition names more than once: %s" % name) + raise ValueError("Cannot use partition names more than once: %s" % name) self.partition_names.add(name) return self + def check_redundant_and_add_field(self, field_id: int, name: str, transform: Transform) -> None: + field = PartitionField(field_id, + self.__next_field_id(), + name, + transform) + dedup_key = (field.source_id, field.transform.dedup_name()) + partition_field = self.dedup_fields.get(dedup_key) + if partition_field is not None: + raise ValueError("Cannot add redundant partition: %s conflicts with %s" % (partition_field, field)) + self.dedup_fields[dedup_key] = field + self.fields.append(field) + def find_source_column(self, source_name): source_column = self.schema.find_field(source_name) if source_column is None: @@ -229,72 +250,82 @@ def find_source_column(self, source_name): return source_column - def identity(self, source_name): - self.check_and_add_partition_name(source_name) + def identity(self, source_name, target_name=None): + if target_name is None: + target_name = source_name + source_column = self.find_source_column(source_name) - self.fields.append(PartitionField(source_column.field_id, - self.__next_field_id(), - source_name, - Transforms.identity(source_column.type))) + self.check_and_add_partition_name(target_name, source_column.field_id) + self.check_redundant_and_add_field(source_column.field_id, + target_name, + Transforms.identity(source_column.type)) return self - def year(self, source_name): - name = "{}_year".format(source_name) - self.check_and_add_partition_name(name) + def year(self, source_name, target_name=None): + if target_name is None: + target_name = "{}_year".format(source_name) + + self.check_and_add_partition_name(target_name) source_column = self.find_source_column(source_name) - self.fields.append(PartitionField(source_column.field_id, - self.__next_field_id(), - name, - Transforms.year(source_column.type))) + self.check_redundant_and_add_field(source_column.field_id, + target_name, + Transforms.year(source_column.type)) return self - def month(self, source_name): - name = "{}_month".format(source_name) - self.check_and_add_partition_name(name) + def month(self, source_name, target_name=None): + if target_name is None: + target_name = "{}_month".format(source_name) + + self.check_and_add_partition_name(target_name) source_column = self.find_source_column(source_name) - self.fields.append(PartitionField(source_column.field_id, - self.__next_field_id(), - name, - Transforms.month(source_column.type))) + self.check_redundant_and_add_field(source_column.field_id, + target_name, + Transforms.month(source_column.type)) return self - def day(self, source_name): - name = "{}_day".format(source_name) - self.check_and_add_partition_name(name) + def day(self, source_name, target_name=None): + if target_name is None: + target_name = "{}_day".format(source_name) + + self.check_and_add_partition_name(target_name) source_column = self.find_source_column(source_name) - self.fields.append(PartitionField(source_column.field_id, - self.__next_field_id(), - name, - Transforms.day(source_column.type))) + self.check_redundant_and_add_field(source_column.field_id, + target_name, + Transforms.day(source_column.type)) return self - def hour(self, source_name): - name = "{}_hour".format(source_name) - self.check_and_add_partition_name(name) + def hour(self, source_name, target_name=None): + if target_name is None: + target_name = "{}_hour".format(source_name) + + self.check_and_add_partition_name(target_name) source_column = self.find_source_column(source_name) - self.fields.append(PartitionField(source_column.field_id, - self.__next_field_id(), - name, - Transforms.hour(source_column.type))) + self.check_redundant_and_add_field(source_column.field_id, + target_name, + Transforms.hour(source_column.type)) return self - def bucket(self, source_name, num_buckets): - name = "{}_bucket".format(source_name) - self.check_and_add_partition_name(name) + def bucket(self, source_name, num_buckets, target_name=None): + if target_name is None: + target_name = "{}_bucket".format(source_name) + + self.check_and_add_partition_name(target_name) source_column = self.find_source_column(source_name) self.fields.append(PartitionField(source_column.field_id, self.__next_field_id(), - name, + target_name, Transforms.bucket(source_column.type, num_buckets))) return self - def truncate(self, source_name, width): - name = "{}_truncate".format(source_name) - self.check_and_add_partition_name(name) + def truncate(self, source_name, width, target_name=None): + if target_name is None: + target_name = "{}_truncate".format(source_name) + + self.check_and_add_partition_name(target_name) source_column = self.find_source_column(source_name) self.fields.append(PartitionField(source_column.field_id, self.__next_field_id(), - name, + target_name, Transforms.truncate(source_column.type, width))) return self @@ -302,17 +333,16 @@ def add_without_field_id(self, source_id, name, transform): return self.add(source_id, self.__next_field_id(), name, transform) def add(self, source_id: int, field_id: int, name: str, transform: str) -> "PartitionSpecBuilder": - self.check_and_add_partition_name(name) column = self.schema.find_field(source_id) if column is None: - raise RuntimeError("Cannot find source column: %s" % source_id) + raise ValueError("Cannot find source column: %s" % source_id) transform_obj = Transforms.from_string(column.type, transform) - field = PartitionField(source_id, - field_id, - name, - transform_obj) - self.fields.append(field) + self.check_and_add_partition_name(name, source_id) + self.fields.append(PartitionField(source_id, + field_id, + name, + transform_obj)) self.last_assigned_field_id = max(self.last_assigned_field_id, field_id) return self diff --git a/iceberg/api/transforms/dates.py b/iceberg/api/transforms/dates.py index 474b986f69..dfc6b9a2cd 100644 --- a/iceberg/api/transforms/dates.py +++ b/iceberg/api/transforms/dates.py @@ -74,3 +74,14 @@ def to_human_string(self, value): def __str__(self): return self.name + + def dedup_name(self): + return "time" + + def __eq__(self, other): + if id(self) == id(other): + return True + if other is None or not isinstance(other, Dates): + return False + + return self.granularity == other.granularity and self.name == other.name diff --git a/iceberg/api/transforms/timestamps.py b/iceberg/api/transforms/timestamps.py index 25c4439bc1..ca38a1c3be 100644 --- a/iceberg/api/transforms/timestamps.py +++ b/iceberg/api/transforms/timestamps.py @@ -70,3 +70,14 @@ def to_human_string(self, value): def __str__(self): return self.name + + def dedup_name(self): + return "time" + + def __eq__(self, other): + if id(self) == id(other): + return True + if other is None or not isinstance(other, Timestamps): + return False + + return self.granularity == other.granularity and self.name == other.name diff --git a/iceberg/api/transforms/transform.py b/iceberg/api/transforms/transform.py index 776b0a507b..0ccbf2e3c2 100644 --- a/iceberg/api/transforms/transform.py +++ b/iceberg/api/transforms/transform.py @@ -38,3 +38,6 @@ def project_strict(self, name, predicate): def to_human_string(self, value): return str(value) + + def dedup_name(self): + return self.__str__() diff --git a/tests/api/test_partition_spec.py b/tests/api/test_partition_spec.py new file mode 100644 index 0000000000..93cee14960 --- /dev/null +++ b/tests/api/test_partition_spec.py @@ -0,0 +1,84 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +import unittest + +from iceberg.api import PartitionSpec +from iceberg.api.schema import Schema +from iceberg.api.types import (BinaryType, + DateType, + DecimalType, + FixedType, + IntegerType, + LongType, + NestedField, + StringType, + TimestampType, + TimeType, + UUIDType) +from tests.api.test_helpers import TestHelpers + + +class TestConversions(unittest.TestCase): + + def test_transforms(self): + schema = Schema(NestedField.required(1, "i", IntegerType.get()), + NestedField.required(2, "l", LongType.get()), + NestedField.required(3, "d", DateType.get()), + NestedField.required(4, "t", TimeType.get()), + NestedField.required(5, "ts", TimestampType.without_timezone()), + NestedField.required(6, "dec", DecimalType.of(9, 2)), + NestedField.required(7, "s", StringType.get()), + NestedField.required(8, "u", UUIDType.get()), + NestedField.required(9, "f", FixedType.of_length(3)), + NestedField.required(10, "b", BinaryType.get())) + specs = [PartitionSpec.builder_for(schema).identity("i").build(), + PartitionSpec.builder_for(schema).identity("l").build(), + PartitionSpec.builder_for(schema).identity("d").build(), + PartitionSpec.builder_for(schema).identity("t").build(), + PartitionSpec.builder_for(schema).identity("ts").build(), + PartitionSpec.builder_for(schema).identity("dec").build(), + PartitionSpec.builder_for(schema).identity("s").build(), + PartitionSpec.builder_for(schema).identity("u").build(), + PartitionSpec.builder_for(schema).identity("f").build(), + PartitionSpec.builder_for(schema).identity("b").build(), + PartitionSpec.builder_for(schema).bucket("i", 128).build(), + PartitionSpec.builder_for(schema).bucket("l", 128).build(), + PartitionSpec.builder_for(schema).bucket("d", 128).build(), + PartitionSpec.builder_for(schema).bucket("t", 128).build(), + PartitionSpec.builder_for(schema).bucket("ts", 128).build(), + PartitionSpec.builder_for(schema).bucket("dec", 128).build(), + PartitionSpec.builder_for(schema).bucket("s", 128).build(), + # todo support them + # PartitionSpec.builder_for(schema).bucket("u", 128).build(), + # PartitionSpec.builder_for(schema).bucket("f", 128).build(), + # PartitionSpec.builder_for(schema).bucket("b", 128).build(), + PartitionSpec.builder_for(schema).year("d").build(), + PartitionSpec.builder_for(schema).month("d").build(), + PartitionSpec.builder_for(schema).day("d").build(), + PartitionSpec.builder_for(schema).year("ts").build(), + PartitionSpec.builder_for(schema).month("ts").build(), + PartitionSpec.builder_for(schema).day("ts").build(), + PartitionSpec.builder_for(schema).hour("ts").build(), + PartitionSpec.builder_for(schema).truncate("i", 10).build(), + PartitionSpec.builder_for(schema).truncate("l", 10).build(), + PartitionSpec.builder_for(schema).truncate("dec", 10).build(), + PartitionSpec.builder_for(schema).truncate("s", 10).build(), + # todo support them + # PartitionSpec.builder_for(schema).add_without_field_id(6, "dec_unsupported", "unsupported").build(), + # PartitionSpec.builder_for(schema).add(6, 1111, "dec_unsupported", "unsupported").build(), + ] + + for spec in specs: + self.assertEqual(spec, TestHelpers.round_trip_serialize(spec)) From f5f892a11e0d1c29c8228982233ae06373214378 Mon Sep 17 00:00:00 2001 From: Ted Gooch Date: Sat, 19 Jun 2021 16:51:18 -0700 Subject: [PATCH 044/642] [python] Adding Unknown and Void transforms (#2697) Co-authored-by: tgooch --- iceberg/api/transforms/transforms.py | 11 +++- iceberg/api/transforms/unknown_transform.py | 61 +++++++++++++++++++++ iceberg/api/transforms/void_transform.py | 52 ++++++++++++++++++ 3 files changed, 123 insertions(+), 1 deletion(-) create mode 100644 iceberg/api/transforms/unknown_transform.py create mode 100644 iceberg/api/transforms/void_transform.py diff --git a/iceberg/api/transforms/transforms.py b/iceberg/api/transforms/transforms.py index 0cf243e0f9..205a9de737 100644 --- a/iceberg/api/transforms/transforms.py +++ b/iceberg/api/transforms/transforms.py @@ -22,6 +22,8 @@ from .identity import Identity from .timestamps import Timestamps from .truncate import Truncate +from .unknown_transform import UnknownTransform +from .void_transform import VoidTransform from ..types import (TypeID) @@ -60,7 +62,10 @@ def from_string(type_var, transform): elif type_var.type_id == TypeID.DATE: return Dates(transform.lower(), transform.lower()) - raise RuntimeError("Unknown transform: %s" % transform) + if transform.lower() == "void": + return VoidTransform.get() + + return UnknownTransform(type_var, transform) @staticmethod def identity(type_var): @@ -109,3 +114,7 @@ def bucket(type_var, num_buckets): @staticmethod def truncate(type_var, width): return Truncate.get(type_var, width) + + @staticmethod + def always_null(): + return VoidTransform.get() diff --git a/iceberg/api/transforms/unknown_transform.py b/iceberg/api/transforms/unknown_transform.py new file mode 100644 index 0000000000..de326445de --- /dev/null +++ b/iceberg/api/transforms/unknown_transform.py @@ -0,0 +1,61 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +from typing import Union + +from iceberg.api.types import StringType, Type + +from .transform import Transform + + +class UnknownTransform(Transform): + + def __init__(self, source_type: Type, transform: str): + self.source_type = source_type + self.transform = transform + + def apply(self, value): + raise AttributeError(f"Cannot apply unsupported transform: {self.transform}") + + def can_transform(self, type_var) -> bool: + # assume the transform function can be applied for this type because unknown transform is only used when parsing + # a transform in an existing table. a different Iceberg version must have already validated it. + return self.source_type == type_var + + def get_result_type(self, source_type): + # the actual result type is not known + return StringType.get() + + def project(self, name, predicate): + return None + + def project_strict(self, name, predicate): + return None + + def __str__(self): + return self.transform + + def __eq__(self, other: Union['UnknownTransform', Transform, object]): + if id(self) == id(other): + return True + elif not isinstance(other, UnknownTransform): + return False + + return self.source_type == other.source_type and self.transform == other.transform + + def __hash__(self): + return hash((self.source_type, self.transform)) diff --git a/iceberg/api/transforms/void_transform.py b/iceberg/api/transforms/void_transform.py new file mode 100644 index 0000000000..ea859641ad --- /dev/null +++ b/iceberg/api/transforms/void_transform.py @@ -0,0 +1,52 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +from .transform import Transform + + +class VoidTransform(Transform): + _INSTANCE = None + + @staticmethod + def get(): + if VoidTransform._INSTANCE is None: + VoidTransform._INSTANCE = VoidTransform() + return VoidTransform._INSTANCE + + def __init__(self): + pass + + def apply(self, value): + return None + + def can_transform(self, type_var): + return True + + def get_result_type(self, source_type): + return source_type + + def project(self, name, predicate): + return None + + def project_strict(self, name, predicate): + return None + + def to_human_string(self, value): + return "null" + + def __str__(self): + return "void" From fb6468e67c0e0692314f0963b8a4e40deba13fa0 Mon Sep 17 00:00:00 2001 From: jun-he Date: Wed, 28 Jul 2021 12:34:22 -0700 Subject: [PATCH 045/642] [Python] support BucketByteBuffer and BucketUUID (#2836) * [Python] support BucketByteBuffer and BucketUUID * Add additional unit tests for bucket hash methods. --- iceberg/api/transforms/bucket.py | 21 ++++++++++--- tests/api/test_partition_spec.py | 16 +++++----- tests/api/transforms/test_bucket.py | 49 +++++++++++++++++++++++++++++ 3 files changed, 72 insertions(+), 14 deletions(-) create mode 100644 tests/api/transforms/test_bucket.py diff --git a/iceberg/api/transforms/bucket.py b/iceberg/api/transforms/bucket.py index 64db712208..2884b81e17 100644 --- a/iceberg/api/transforms/bucket.py +++ b/iceberg/api/transforms/bucket.py @@ -28,6 +28,7 @@ Operation) from ..types.types import (IntegerType, TypeID) +from ...api.types.conversions import Conversions class Bucket(Transform): @@ -69,7 +70,7 @@ def __key(self): return Bucket.__class__, self.n def __repr__(self): - return "Bucket(n)" % self.n + return "Bucket[%s]" % self.n def __str__(self): return "bucket[%s]" % self.n @@ -167,14 +168,24 @@ def can_transform(self, type_var): class BucketByteBuffer(Bucket): def __init__(self, n): - # super(BucketByteBuffer, self).__init__(n) - raise NotImplementedError() + super(BucketByteBuffer, self).__init__(n) + + def hash(self, value): + return Bucket.MURMUR3.hash(value) + + def can_transform(self, type_var): + return type_var.type_id in [TypeID.BINARY, TypeID.FIXED] class BucketUUID(Bucket): def __init__(self, n): - # super(BucketUUID, self).__init__(n) - raise NotImplementedError() + super(BucketUUID, self).__init__(n) + + def hash(self, value): + return Bucket.MURMUR3.hash(Conversions.to_byte_buffer(TypeID.UUID, value)) + + def can_transform(self, type_var): + return type_var.type_id == TypeID.UUID def to_bytes(n, length, byteorder='big'): diff --git a/tests/api/test_partition_spec.py b/tests/api/test_partition_spec.py index 93cee14960..70ab574349 100644 --- a/tests/api/test_partition_spec.py +++ b/tests/api/test_partition_spec.py @@ -30,9 +30,9 @@ from tests.api.test_helpers import TestHelpers -class TestConversions(unittest.TestCase): +class TestPartitionSpec(unittest.TestCase): - def test_transforms(self): + def test_partition_spec(self): schema = Schema(NestedField.required(1, "i", IntegerType.get()), NestedField.required(2, "l", LongType.get()), NestedField.required(3, "d", DateType.get()), @@ -60,10 +60,9 @@ def test_transforms(self): PartitionSpec.builder_for(schema).bucket("ts", 128).build(), PartitionSpec.builder_for(schema).bucket("dec", 128).build(), PartitionSpec.builder_for(schema).bucket("s", 128).build(), - # todo support them - # PartitionSpec.builder_for(schema).bucket("u", 128).build(), - # PartitionSpec.builder_for(schema).bucket("f", 128).build(), - # PartitionSpec.builder_for(schema).bucket("b", 128).build(), + PartitionSpec.builder_for(schema).bucket("u", 128).build(), + PartitionSpec.builder_for(schema).bucket("f", 128).build(), + PartitionSpec.builder_for(schema).bucket("b", 128).build(), PartitionSpec.builder_for(schema).year("d").build(), PartitionSpec.builder_for(schema).month("d").build(), PartitionSpec.builder_for(schema).day("d").build(), @@ -75,9 +74,8 @@ def test_transforms(self): PartitionSpec.builder_for(schema).truncate("l", 10).build(), PartitionSpec.builder_for(schema).truncate("dec", 10).build(), PartitionSpec.builder_for(schema).truncate("s", 10).build(), - # todo support them - # PartitionSpec.builder_for(schema).add_without_field_id(6, "dec_unsupported", "unsupported").build(), - # PartitionSpec.builder_for(schema).add(6, 1111, "dec_unsupported", "unsupported").build(), + PartitionSpec.builder_for(schema).add_without_field_id(6, "dec_unsupported", "unsupported").build(), + PartitionSpec.builder_for(schema).add(6, 1111, "dec_unsupported", "unsupported").build(), ] for spec in specs: diff --git a/tests/api/transforms/test_bucket.py b/tests/api/transforms/test_bucket.py new file mode 100644 index 0000000000..afda054017 --- /dev/null +++ b/tests/api/transforms/test_bucket.py @@ -0,0 +1,49 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +import decimal +import unittest +import uuid + +from iceberg.api.transforms import Transforms +from iceberg.api.types import (BinaryType, + DateType, + DecimalType, + FixedType, + IntegerType, + LongType, + StringType, + TimestampType, + TimeType, + UUIDType) + + +class TestBucket(unittest.TestCase): + + def test_bucket_hash(self): + buckets = [ + [Transforms.bucket(IntegerType.get(), 100), 34, 2017239379], + [Transforms.bucket(LongType.get(), 100), 34, 2017239379], + [Transforms.bucket(DateType.get(), 100), 17486, -653330422], + [Transforms.bucket(TimeType.get(), 100), 81068000000, -662762989], + [Transforms.bucket(TimestampType.without_timezone(), 100), 1510871468000000, -2047944441], + [Transforms.bucket(DecimalType.of(9, 2), 100), decimal.Decimal("14.20"), -500754589], + [Transforms.bucket(StringType.get(), 100), "iceberg", 1210000089], + [Transforms.bucket(UUIDType.get(), 100), uuid.UUID("f79c3e09-677c-4bbd-a479-3f349cb785e7"), 1488055340], + [Transforms.bucket(FixedType.of_length(3), 128), b'foo', -156908512], + [Transforms.bucket(BinaryType.get(), 128), b'\x00\x01\x02\x03', -188683207] + ] + + for bucket in buckets: + self.assertEqual(bucket[2], bucket[0].hash(bucket[1])) From 37332ab2e12920d249a00afde0b3ed7eea7bb92f Mon Sep 17 00:00:00 2001 From: Ted Gooch Date: Wed, 28 Jul 2021 16:50:51 -0700 Subject: [PATCH 046/642] [python] Updating pyarrow dependencies (#2888) Co-authored-by: tgooch --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 062f2bcde1..cbf120b8b7 100755 --- a/setup.py +++ b/setup.py @@ -36,7 +36,7 @@ 'requests', 'retrying', 'pandas', - 'pyarrow>=3.0.0<4.0.0' + 'pyarrow>=3.0.0,<=4.0.1' ], extras_require={ "dev": [ From 416c73c4de7d49e1f6dae06dbdf631df0101c864 Mon Sep 17 00:00:00 2001 From: jun-he Date: Fri, 27 Aug 2021 15:36:50 -0700 Subject: [PATCH 047/642] [Python] Support schema field ID validation in python (#2866) * Support schema field ID validation in python. Also fix bugs in type_util and add unit tests. * Add license to the test file * update unit tests * address the comments * only ignore NotImplementedError error during visit. --- iceberg/api/schema.py | 8 ++--- iceberg/api/types/type_util.py | 15 +++++++--- tests/api/types/test_type_util.py | 49 +++++++++++++++++++++++++++++++ 3 files changed, 64 insertions(+), 8 deletions(-) create mode 100644 tests/api/types/test_type_util.py diff --git a/iceberg/api/schema.py b/iceberg/api/schema.py index 0d70bf8a3e..50d99e73ab 100644 --- a/iceberg/api/schema.py +++ b/iceberg/api/schema.py @@ -16,7 +16,7 @@ # under the License. from .types import StructType - +from .types import type_util """ The schema of a data table. @@ -45,9 +45,9 @@ def __init__(self, *argv): self._id_to_alias = {v: k for k, v in self._alias_to_id.items()} self._id_to_field = None - self._name_to_id = None - self._lowercase_name_to_id = None - self._id_to_name = None + self._name_to_id = type_util.index_by_name(self.struct) + self._id_to_name = {v: k for k, v in self._name_to_id.items()} + self._lowercase_name_to_id = {k.lower(): v for k, v in self._name_to_id.items()} def as_struct(self): return self.struct diff --git a/iceberg/api/types/type_util.py b/iceberg/api/types/type_util.py index f7f8b138f6..5a465e50fe 100644 --- a/iceberg/api/types/type_util.py +++ b/iceberg/api/types/type_util.py @@ -24,6 +24,7 @@ MapType, NestedField, StructType) +from ...exceptions import ValidationException MAX_PRECISION = list() REQUIRED_LENGTH = [-1 for item in range(40)] @@ -121,7 +122,8 @@ def visit(arg, visitor): # noqa: ignore=C901 result = None try: result = visit(field.type, visitor) - except RuntimeError: + except NotImplementedError: + # will remove it after missing functions are implemented. pass finally: visitor.field_ids.pop() @@ -133,7 +135,8 @@ def visit(arg, visitor): # noqa: ignore=C901 visitor.field_ids.append(list_var.element_id) try: element_result = visit(list_var.element_type, visitor) - except RuntimeError: + except NotImplementedError: + # will remove it after missing functions are implemented. pass finally: visitor.field_ids.pop() @@ -352,8 +355,12 @@ def map(self, map_var, key_result, value_result): def add_field(self, name, field_id): full_name = name - if not self.field_names and len(self.field_names) > 0: - full_name = IndexByName.DOT.join([IndexByName.DOT.join(reversed(self.field_names)), name]) + if self.field_names is not None and len(self.field_names) > 0: + full_name = IndexByName.DOT.join([IndexByName.DOT.join(self.field_names), name]) + + existing_field_id = self.name_to_id.get(full_name) + ValidationException.check(existing_field_id is None, "Invalid schema: multiple fields for name %s: %s and %s", + (full_name, existing_field_id, field_id)) self.name_to_id[full_name] = field_id diff --git a/tests/api/types/test_type_util.py b/tests/api/types/test_type_util.py new file mode 100644 index 0000000000..58036036ad --- /dev/null +++ b/tests/api/types/test_type_util.py @@ -0,0 +1,49 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +from iceberg.api.types import (BooleanType, NestedField, StructType) +from iceberg.api.types import type_util +from iceberg.exceptions import ValidationException +import pytest + + +def test_invalid_schema_via_index_by_name(): + bool_type1 = NestedField.required(1, "a", BooleanType.get()) + bool_type2 = NestedField.required(2, "a", BooleanType.get()) + + with pytest.raises(ValidationException) as context: + type_util.index_by_name(StructType.of([bool_type1, bool_type2])) + assert str(context.value) == 'Invalid schema: multiple fields for name a: 1 and 2' + + +def test_valid_schema_via_index_by_name(): + bool_type1 = NestedField.required(1, "a", BooleanType.get()) + bool_type2 = NestedField.required(2, "b", BooleanType.get()) + + assert {'a': 1, 'b': 2} == type_util.index_by_name(StructType.of([bool_type1, bool_type2])) + + +def test_validate_schema_via_index_by_name_for_nested_type(): + nested_type = NestedField.required( + 1, "a", StructType.of( + [NestedField.required(2, "b", StructType.of( + [NestedField.required(3, "c", BooleanType.get())])), + NestedField.required(4, "b.c", BooleanType.get())])) + + with pytest.raises(ValidationException) as context: + type_util.index_by_name(StructType.of([nested_type])) + assert str(context.value) == 'Invalid schema: multiple fields for name a.b.c: 3 and 4' From af8bb6824c599fcf20e7cf4fec4f26b2fea5bcd9 Mon Sep 17 00:00:00 2001 From: jun-he Date: Fri, 17 Sep 2021 14:24:08 -0700 Subject: [PATCH 048/642] Rename the existing python module to python_legacy. (#3074) * Rename the existing python module to python_legacy. * update github script to use python_legacy path. --- CHANGELOG.md | 21 - README.md | 48 -- iceberg/__init__.py | 16 - iceberg/api/__init__.py | 44 -- iceberg/api/append_files.py | 24 - iceberg/api/combined_scan_task.py | 27 - iceberg/api/data_file.py | 98 --- iceberg/api/data_operations.py | 23 - iceberg/api/delete_files.py | 34 - iceberg/api/expire_snapshots.py | 30 - iceberg/api/expressions/__init__.py | 107 --- iceberg/api/expressions/binder.py | 91 --- iceberg/api/expressions/evaluator.py | 92 --- iceberg/api/expressions/expression.py | 252 ------ iceberg/api/expressions/expression_parser.py | 161 ---- iceberg/api/expressions/expressions.py | 299 -------- .../inclusive_manifest_evaluator.py | 161 ---- .../inclusive_metrics_evaluator.py | 188 ----- .../expressions/java_variables/__init__.py | 21 - iceberg/api/expressions/literals.py | 591 -------------- iceberg/api/expressions/predicate.py | 295 ------- iceberg/api/expressions/projections.py | 114 --- iceberg/api/expressions/reference.py | 114 --- iceberg/api/expressions/residual_evaluator.py | 117 --- .../expressions/strict_metrics_evaluator.py | 221 ------ iceberg/api/expressions/term.py | 73 -- iceberg/api/expressions/transform.py | 75 -- iceberg/api/file_format.py | 44 -- iceberg/api/file_scan_task.py | 47 -- iceberg/api/files.py | 126 --- iceberg/api/filterable.py | 32 - iceberg/api/filtered_snapshot.py | 40 - iceberg/api/io/__init__.py | 30 - iceberg/api/io/closeable_group.py | 31 - iceberg/api/io/closeable_iterable.py | 27 - iceberg/api/io/delegating_input_stream.py | 22 - iceberg/api/io/delegating_output_stream.py | 22 - iceberg/api/io/file_appender.py | 29 - iceberg/api/io/input_file.py | 38 - iceberg/api/io/output_file.py | 28 - iceberg/api/io/position_output_stream.py | 32 - iceberg/api/io/seekable_input_stream.py | 46 -- iceberg/api/manifest_file.py | 95 --- iceberg/api/metrics.py | 30 - iceberg/api/overwrite_files.py | 30 - iceberg/api/partition_field.py | 43 -- iceberg/api/partition_spec.py | 352 --------- iceberg/api/pending_update.py | 25 - iceberg/api/replace_partitions.py | 30 - iceberg/api/rewrite_files.py | 30 - iceberg/api/rollback.py | 33 - iceberg/api/scan_task.py | 28 - iceberg/api/schema.py | 164 ---- iceberg/api/snapshot.py | 53 -- iceberg/api/snapshot_iterable.py | 21 - iceberg/api/struct_like.py | 28 - iceberg/api/table.py | 76 -- iceberg/api/table_scan.py | 62 -- iceberg/api/tables.py | 34 - iceberg/api/transaction.py | 55 -- iceberg/api/transforms/__init__.py | 48 -- iceberg/api/transforms/bucket.py | 196 ----- iceberg/api/transforms/dates.py | 87 --- iceberg/api/transforms/identity.py | 91 --- iceberg/api/transforms/projection_util.py | 66 -- iceberg/api/transforms/timestamps.py | 83 -- iceberg/api/transforms/transform.py | 43 -- iceberg/api/transforms/transform_util.py | 84 -- iceberg/api/transforms/transforms.py | 120 --- iceberg/api/transforms/truncate.py | 231 ------ iceberg/api/transforms/unknown_transform.py | 61 -- iceberg/api/transforms/void_transform.py | 52 -- iceberg/api/types/__init__.py | 134 ---- iceberg/api/types/conversions.py | 100 --- iceberg/api/types/type.py | 118 --- iceberg/api/types/type_util.py | 544 ------------- iceberg/api/types/types.py | 726 ------------------ iceberg/api/update_properties.py | 30 - iceberg/api/update_schema.py | 33 - iceberg/core/__init__.py | 61 -- iceberg/core/avro/__init__.py | 21 - iceberg/core/avro/avro_schema_util.py | 35 - iceberg/core/avro/avro_to_iceberg.py | 301 -------- iceberg/core/avro/iceberg_to_avro.py | 90 --- iceberg/core/base_combined_scan_task.py | 37 - iceberg/core/base_file_scan_task.py | 126 --- .../core/base_metastore_table_operations.py | 137 ---- iceberg/core/base_metastore_tables.py | 85 -- iceberg/core/base_snapshot.py | 132 ---- iceberg/core/base_table.py | 110 --- iceberg/core/base_table_scan.py | 213 ----- iceberg/core/base_transaction.py | 176 ----- iceberg/core/config_properties.py | 26 - iceberg/core/data_files.py | 228 ------ iceberg/core/data_table_scan.py | 98 --- iceberg/core/filesystem/__init__.py | 27 - iceberg/core/filesystem/file_status.py | 34 - iceberg/core/filesystem/file_system.py | 120 --- .../filesystem/filesystem_table_operations.py | 140 ---- iceberg/core/filesystem/filesystem_tables.py | 65 -- iceberg/core/filesystem/local_filesystem.py | 81 -- iceberg/core/filesystem/s3_filesystem.py | 253 ------ iceberg/core/filesystem/util.py | 39 - iceberg/core/filtered_manifest.py | 118 --- iceberg/core/generic_data_file.py | 137 ---- iceberg/core/generic_manifest_file.py | 194 ----- .../core/generic_partition_field_summary.py | 61 -- iceberg/core/manifest_entry.py | 145 ---- iceberg/core/manifest_list_writer.py | 55 -- iceberg/core/manifest_reader.py | 162 ---- iceberg/core/partition_data.py | 94 --- iceberg/core/partition_spec_parser.py | 98 --- iceberg/core/partition_summary.py | 68 -- iceberg/core/scan_summary.py | 409 ---------- iceberg/core/schema_parser.py | 182 ----- iceberg/core/schema_update.py | 40 - iceberg/core/snapshot_parser.py | 73 -- iceberg/core/table_metadata.py | 218 ------ iceberg/core/table_metadata_parser.py | 124 --- iceberg/core/table_operations.py | 42 - iceberg/core/table_properties.py | 78 -- iceberg/core/util/__init__.py | 36 - iceberg/core/util/atomic_integer.py | 36 - iceberg/core/util/bin_packing.py | 79 -- iceberg/core/util/profile.py | 36 - iceberg/exceptions/__init__.py | 30 - iceberg/exceptions/exceptions.py | 47 -- iceberg/hive/__init__.py | 25 - iceberg/hive/hive_table_operations.py | 208 ----- iceberg/hive/hive_tables.py | 97 --- iceberg/hive/hive_types.py | 37 - iceberg/parquet/__init__.py | 20 - iceberg/parquet/dataset_utils.py | 158 ---- iceberg/parquet/parquet_reader.py | 224 ------ iceberg/parquet/parquet_schema_utils.py | 41 - iceberg/parquet/parquet_to_iceberg.py | 152 ---- setup.py | 55 -- tests/__init__.py | 16 - tests/api/__init__.py | 16 - tests/api/expressions/__init__.py | 16 - tests/api/expressions/conftest.py | 425 ---------- tests/api/expressions/test_evaluator.py | 161 ---- .../expressions/test_expression_binding.py | 143 ---- .../expressions/test_expression_helpers.py | 47 -- .../test_expression_serializations.py | 21 - .../test_inclusive_manifest_evaluator.py | 182 ----- .../test_inclusive_metrics_evaluator.py | 130 ---- .../expressions/test_literal_serialization.py | 20 - .../test_misc_literal_conversions.py | 263 ------- .../test_numeric_literal_conversions.py | 116 --- .../api/expressions/test_predicate_binding.py | 199 ----- tests/api/expressions/test_str_to_expr.py | 161 ---- .../test_strict_metrics_evaluator.py | 159 ---- .../test_string_literal_conversions.py | 102 --- tests/api/test_conversions.py | 81 -- tests/api/test_file_format.py | 43 -- tests/api/test_helpers.py | 156 ---- tests/api/test_partition_spec.py | 82 -- tests/api/transforms/__init__.py | 16 - tests/api/transforms/test_bucket.py | 49 -- tests/api/transforms/test_bucketing.py | 64 -- tests/api/transforms/test_dates.py | 44 -- tests/api/transforms/test_identity.py | 93 --- tests/api/transforms/test_timestamps.py | 46 -- tests/api/transforms/test_truncate.py | 59 -- tests/api/types/__init__.py | 16 - tests/api/types/test_binary_comparator.py | 41 - tests/api/types/test_char_seq_comparator.py | 43 -- tests/api/types/test_comparable_comparator.py | 51 -- tests/api/types/test_readabilty_checks.py | 57 -- tests/api/types/test_type_util.py | 49 -- tests/core/__init__.py | 16 - tests/core/avro/__init__.py | 16 - tests/core/avro/conftest.py | 64 -- tests/core/avro/test_avro.py | 22 - tests/core/avro/test_read_projection.py | 27 - tests/core/conftest.py | 314 -------- tests/core/test_base_table_scan.py | 38 - tests/core/test_filesystem_tables.py | 32 - tests/core/test_partition_spec.py | 111 --- tests/core/test_partition_spec_parser.py | 110 --- tests/core/test_snapshot_json.py | 58 -- tests/core/test_table_metadata_json.py | 111 --- tests/core/test_table_metadata_parser.py | 43 -- tests/core/utils/__init__.py | 14 - tests/core/utils/test_bin_packing.py | 36 - tests/hive/__init__.py | 18 - tests/hive/conftest.py | 37 - tests/hive/test_hive_tables.py | 237 ------ tests/parquet/__init__.py | 16 - tests/parquet/conftest.py | 226 ------ tests/parquet/test_dataset_utils.py | 52 -- tests/parquet/test_parquet_reader.py | 285 ------- tests/parquet/test_parquet_to_iceberg.py | 76 -- tox.ini | 115 --- 195 files changed, 19422 deletions(-) delete mode 100644 CHANGELOG.md delete mode 100644 README.md delete mode 100644 iceberg/__init__.py delete mode 100644 iceberg/api/__init__.py delete mode 100644 iceberg/api/append_files.py delete mode 100644 iceberg/api/combined_scan_task.py delete mode 100644 iceberg/api/data_file.py delete mode 100644 iceberg/api/data_operations.py delete mode 100644 iceberg/api/delete_files.py delete mode 100644 iceberg/api/expire_snapshots.py delete mode 100644 iceberg/api/expressions/__init__.py delete mode 100644 iceberg/api/expressions/binder.py delete mode 100644 iceberg/api/expressions/evaluator.py delete mode 100644 iceberg/api/expressions/expression.py delete mode 100644 iceberg/api/expressions/expression_parser.py delete mode 100644 iceberg/api/expressions/expressions.py delete mode 100644 iceberg/api/expressions/inclusive_manifest_evaluator.py delete mode 100644 iceberg/api/expressions/inclusive_metrics_evaluator.py delete mode 100644 iceberg/api/expressions/java_variables/__init__.py delete mode 100644 iceberg/api/expressions/literals.py delete mode 100644 iceberg/api/expressions/predicate.py delete mode 100644 iceberg/api/expressions/projections.py delete mode 100644 iceberg/api/expressions/reference.py delete mode 100644 iceberg/api/expressions/residual_evaluator.py delete mode 100644 iceberg/api/expressions/strict_metrics_evaluator.py delete mode 100644 iceberg/api/expressions/term.py delete mode 100644 iceberg/api/expressions/transform.py delete mode 100644 iceberg/api/file_format.py delete mode 100644 iceberg/api/file_scan_task.py delete mode 100644 iceberg/api/files.py delete mode 100644 iceberg/api/filterable.py delete mode 100644 iceberg/api/filtered_snapshot.py delete mode 100644 iceberg/api/io/__init__.py delete mode 100644 iceberg/api/io/closeable_group.py delete mode 100644 iceberg/api/io/closeable_iterable.py delete mode 100644 iceberg/api/io/delegating_input_stream.py delete mode 100644 iceberg/api/io/delegating_output_stream.py delete mode 100644 iceberg/api/io/file_appender.py delete mode 100644 iceberg/api/io/input_file.py delete mode 100644 iceberg/api/io/output_file.py delete mode 100644 iceberg/api/io/position_output_stream.py delete mode 100644 iceberg/api/io/seekable_input_stream.py delete mode 100644 iceberg/api/manifest_file.py delete mode 100644 iceberg/api/metrics.py delete mode 100644 iceberg/api/overwrite_files.py delete mode 100644 iceberg/api/partition_field.py delete mode 100644 iceberg/api/partition_spec.py delete mode 100644 iceberg/api/pending_update.py delete mode 100644 iceberg/api/replace_partitions.py delete mode 100644 iceberg/api/rewrite_files.py delete mode 100644 iceberg/api/rollback.py delete mode 100644 iceberg/api/scan_task.py delete mode 100644 iceberg/api/schema.py delete mode 100644 iceberg/api/snapshot.py delete mode 100644 iceberg/api/snapshot_iterable.py delete mode 100644 iceberg/api/struct_like.py delete mode 100644 iceberg/api/table.py delete mode 100644 iceberg/api/table_scan.py delete mode 100644 iceberg/api/tables.py delete mode 100644 iceberg/api/transaction.py delete mode 100644 iceberg/api/transforms/__init__.py delete mode 100644 iceberg/api/transforms/bucket.py delete mode 100644 iceberg/api/transforms/dates.py delete mode 100644 iceberg/api/transforms/identity.py delete mode 100644 iceberg/api/transforms/projection_util.py delete mode 100644 iceberg/api/transforms/timestamps.py delete mode 100644 iceberg/api/transforms/transform.py delete mode 100644 iceberg/api/transforms/transform_util.py delete mode 100644 iceberg/api/transforms/transforms.py delete mode 100644 iceberg/api/transforms/truncate.py delete mode 100644 iceberg/api/transforms/unknown_transform.py delete mode 100644 iceberg/api/transforms/void_transform.py delete mode 100644 iceberg/api/types/__init__.py delete mode 100644 iceberg/api/types/conversions.py delete mode 100644 iceberg/api/types/type.py delete mode 100644 iceberg/api/types/type_util.py delete mode 100644 iceberg/api/types/types.py delete mode 100644 iceberg/api/update_properties.py delete mode 100644 iceberg/api/update_schema.py delete mode 100644 iceberg/core/__init__.py delete mode 100644 iceberg/core/avro/__init__.py delete mode 100644 iceberg/core/avro/avro_schema_util.py delete mode 100644 iceberg/core/avro/avro_to_iceberg.py delete mode 100644 iceberg/core/avro/iceberg_to_avro.py delete mode 100644 iceberg/core/base_combined_scan_task.py delete mode 100644 iceberg/core/base_file_scan_task.py delete mode 100644 iceberg/core/base_metastore_table_operations.py delete mode 100644 iceberg/core/base_metastore_tables.py delete mode 100644 iceberg/core/base_snapshot.py delete mode 100644 iceberg/core/base_table.py delete mode 100644 iceberg/core/base_table_scan.py delete mode 100644 iceberg/core/base_transaction.py delete mode 100644 iceberg/core/config_properties.py delete mode 100644 iceberg/core/data_files.py delete mode 100644 iceberg/core/data_table_scan.py delete mode 100644 iceberg/core/filesystem/__init__.py delete mode 100644 iceberg/core/filesystem/file_status.py delete mode 100644 iceberg/core/filesystem/file_system.py delete mode 100644 iceberg/core/filesystem/filesystem_table_operations.py delete mode 100644 iceberg/core/filesystem/filesystem_tables.py delete mode 100644 iceberg/core/filesystem/local_filesystem.py delete mode 100644 iceberg/core/filesystem/s3_filesystem.py delete mode 100644 iceberg/core/filesystem/util.py delete mode 100644 iceberg/core/filtered_manifest.py delete mode 100644 iceberg/core/generic_data_file.py delete mode 100644 iceberg/core/generic_manifest_file.py delete mode 100644 iceberg/core/generic_partition_field_summary.py delete mode 100644 iceberg/core/manifest_entry.py delete mode 100644 iceberg/core/manifest_list_writer.py delete mode 100644 iceberg/core/manifest_reader.py delete mode 100644 iceberg/core/partition_data.py delete mode 100644 iceberg/core/partition_spec_parser.py delete mode 100644 iceberg/core/partition_summary.py delete mode 100644 iceberg/core/scan_summary.py delete mode 100644 iceberg/core/schema_parser.py delete mode 100644 iceberg/core/schema_update.py delete mode 100644 iceberg/core/snapshot_parser.py delete mode 100644 iceberg/core/table_metadata.py delete mode 100644 iceberg/core/table_metadata_parser.py delete mode 100644 iceberg/core/table_operations.py delete mode 100644 iceberg/core/table_properties.py delete mode 100644 iceberg/core/util/__init__.py delete mode 100644 iceberg/core/util/atomic_integer.py delete mode 100644 iceberg/core/util/bin_packing.py delete mode 100644 iceberg/core/util/profile.py delete mode 100644 iceberg/exceptions/__init__.py delete mode 100644 iceberg/exceptions/exceptions.py delete mode 100644 iceberg/hive/__init__.py delete mode 100644 iceberg/hive/hive_table_operations.py delete mode 100644 iceberg/hive/hive_tables.py delete mode 100644 iceberg/hive/hive_types.py delete mode 100644 iceberg/parquet/__init__.py delete mode 100644 iceberg/parquet/dataset_utils.py delete mode 100644 iceberg/parquet/parquet_reader.py delete mode 100644 iceberg/parquet/parquet_schema_utils.py delete mode 100644 iceberg/parquet/parquet_to_iceberg.py delete mode 100755 setup.py delete mode 100644 tests/__init__.py delete mode 100644 tests/api/__init__.py delete mode 100644 tests/api/expressions/__init__.py delete mode 100644 tests/api/expressions/conftest.py delete mode 100644 tests/api/expressions/test_evaluator.py delete mode 100644 tests/api/expressions/test_expression_binding.py delete mode 100644 tests/api/expressions/test_expression_helpers.py delete mode 100644 tests/api/expressions/test_expression_serializations.py delete mode 100644 tests/api/expressions/test_inclusive_manifest_evaluator.py delete mode 100644 tests/api/expressions/test_inclusive_metrics_evaluator.py delete mode 100644 tests/api/expressions/test_literal_serialization.py delete mode 100644 tests/api/expressions/test_misc_literal_conversions.py delete mode 100644 tests/api/expressions/test_numeric_literal_conversions.py delete mode 100644 tests/api/expressions/test_predicate_binding.py delete mode 100644 tests/api/expressions/test_str_to_expr.py delete mode 100644 tests/api/expressions/test_strict_metrics_evaluator.py delete mode 100644 tests/api/expressions/test_string_literal_conversions.py delete mode 100644 tests/api/test_conversions.py delete mode 100644 tests/api/test_file_format.py delete mode 100644 tests/api/test_helpers.py delete mode 100644 tests/api/test_partition_spec.py delete mode 100644 tests/api/transforms/__init__.py delete mode 100644 tests/api/transforms/test_bucket.py delete mode 100644 tests/api/transforms/test_bucketing.py delete mode 100644 tests/api/transforms/test_dates.py delete mode 100644 tests/api/transforms/test_identity.py delete mode 100644 tests/api/transforms/test_timestamps.py delete mode 100644 tests/api/transforms/test_truncate.py delete mode 100644 tests/api/types/__init__.py delete mode 100644 tests/api/types/test_binary_comparator.py delete mode 100644 tests/api/types/test_char_seq_comparator.py delete mode 100644 tests/api/types/test_comparable_comparator.py delete mode 100644 tests/api/types/test_readabilty_checks.py delete mode 100644 tests/api/types/test_type_util.py delete mode 100644 tests/core/__init__.py delete mode 100644 tests/core/avro/__init__.py delete mode 100644 tests/core/avro/conftest.py delete mode 100644 tests/core/avro/test_avro.py delete mode 100644 tests/core/avro/test_read_projection.py delete mode 100644 tests/core/conftest.py delete mode 100644 tests/core/test_base_table_scan.py delete mode 100644 tests/core/test_filesystem_tables.py delete mode 100644 tests/core/test_partition_spec.py delete mode 100644 tests/core/test_partition_spec_parser.py delete mode 100644 tests/core/test_snapshot_json.py delete mode 100644 tests/core/test_table_metadata_json.py delete mode 100644 tests/core/test_table_metadata_parser.py delete mode 100644 tests/core/utils/__init__.py delete mode 100644 tests/core/utils/test_bin_packing.py delete mode 100644 tests/hive/__init__.py delete mode 100644 tests/hive/conftest.py delete mode 100644 tests/hive/test_hive_tables.py delete mode 100644 tests/parquet/__init__.py delete mode 100644 tests/parquet/conftest.py delete mode 100644 tests/parquet/test_dataset_utils.py delete mode 100644 tests/parquet/test_parquet_reader.py delete mode 100644 tests/parquet/test_parquet_to_iceberg.py delete mode 100644 tox.ini diff --git a/CHANGELOG.md b/CHANGELOG.md deleted file mode 100644 index c19b3abd76..0000000000 --- a/CHANGELOG.md +++ /dev/null @@ -1,21 +0,0 @@ - - -# iceberg Changelog - -## iceberg 0.0.1 (2019-02-08) -* Library creation. - [Ted Gooch] diff --git a/README.md b/README.md deleted file mode 100644 index 0d4a3475cd..0000000000 --- a/README.md +++ /dev/null @@ -1,48 +0,0 @@ - - -# Iceberg Python - -Iceberg is a python library for programatic access to iceberg table metadata as well as data access. The intention is to provide a functional subset of the java library. - -## Getting Started - -Iceberg python is currently in development, for development and testing purposes the best way to install the library is to perform the following steps: - -``` -git clone https://github.com/apache/iceberg.git -cd iceberg/python -pip install -e . -``` - -## Testing - -Testing is done using tox. The config can be found in `tox.ini` within the python directory of the iceberg project. - -``` -# simply run tox from within the python dir -tox -``` - -## Get in Touch - -- Email: - * [dev@iceberg.apache.org](mailto:dev@iceberg.apache.org) - -- Issues - * [File a github incident](https://github.com/apache/iceberg/issues) - diff --git a/iceberg/__init__.py b/iceberg/__init__.py deleted file mode 100644 index 13a83393a9..0000000000 --- a/iceberg/__init__.py +++ /dev/null @@ -1,16 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. diff --git a/iceberg/api/__init__.py b/iceberg/api/__init__.py deleted file mode 100644 index b6d22bbffe..0000000000 --- a/iceberg/api/__init__.py +++ /dev/null @@ -1,44 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -__all__ = ["CombinedScanTask", "DataFile", "DataOperations", "FileFormat", "FileScanTask", - "Files", "Filterable", "FilteredSnapshot", "ManifestFile", "PartitionFieldSummary", - "Metrics", "PartitionSpec", "PartitionSpecBuilder", - "Schema", "Snapshot", "SnapshotIterable", "StructLike", - "Table", "Tables", "TableScan", "Transaction", "UpdateSchema"] - -from .combined_scan_task import CombinedScanTask -from .data_file import DataFile -from .data_operations import DataOperations -from .file_format import FileFormat -from .file_scan_task import FileScanTask -from .files import Files -from .filterable import Filterable -from .filtered_snapshot import FilteredSnapshot -from .manifest_file import ManifestFile, PartitionFieldSummary -from .metrics import Metrics -from .partition_spec import (PartitionSpec, - PartitionSpecBuilder) -from .schema import Schema -from .snapshot import Snapshot -from .snapshot_iterable import SnapshotIterable -from .struct_like import StructLike -from .table import Table -from .table_scan import TableScan -from .tables import Tables -from .transaction import Transaction -from .update_schema import UpdateSchema diff --git a/iceberg/api/append_files.py b/iceberg/api/append_files.py deleted file mode 100644 index 435f2f825b..0000000000 --- a/iceberg/api/append_files.py +++ /dev/null @@ -1,24 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -from .pending_update import PendingUpdate - - -class AppendFiles(PendingUpdate): - - def append_file(self, file): - raise NotImplementedError() diff --git a/iceberg/api/combined_scan_task.py b/iceberg/api/combined_scan_task.py deleted file mode 100644 index e385ba524e..0000000000 --- a/iceberg/api/combined_scan_task.py +++ /dev/null @@ -1,27 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -from .scan_task import ScanTask - - -class CombinedScanTask(ScanTask): - - def files(self): - raise NotImplementedError() - - def as_combined_scan_task(self): - return self diff --git a/iceberg/api/data_file.py b/iceberg/api/data_file.py deleted file mode 100644 index 6e85d6e84f..0000000000 --- a/iceberg/api/data_file.py +++ /dev/null @@ -1,98 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -from iceberg.api.types import (BinaryType, - IntegerType, - ListType, - LongType, - MapType, - NestedField, - StringType, - StructType) - - -class DataFile(object): - - @staticmethod - def get_type(partition_type): - return StructType.of([NestedField.required(100, "file_path", StringType.get()), - NestedField.required(101, "file_format", StringType.get()), - NestedField.required(102, "partition", partition_type), - NestedField.required(103, "record_count", LongType.get()), - NestedField.required(104, "file_size_in_bytes", LongType.get()), - NestedField.required(105, "block_size_in_bytes", LongType.get()), - NestedField.optional(106, "file_ordinal", IntegerType.get()), - NestedField.optional(107, "sort_columns", ListType.of_required(112, IntegerType.get())), - NestedField.optional(108, "column_sizes", MapType.of_required(117, 118, - IntegerType.get(), - LongType.get())), - NestedField.optional(109, "value_counts", MapType.of_required(119, 120, - IntegerType.get(), - LongType.get())), - NestedField.optional(110, "null_value_counts", MapType.of_required(121, 122, - IntegerType.get(), - LongType.get())), - NestedField.optional(125, "lower_bounds", MapType.of_required(126, 127, - IntegerType.get(), - BinaryType.get())), - NestedField.optional(128, "upper_bounds", MapType.of_required(129, 130, - IntegerType.get(), - BinaryType.get()))] - # NEXT ID TO ASSIGN: 131 - ) - - def path(self): - raise NotImplementedError() - - def format(self): - raise NotImplementedError() - - def partition(self): - raise NotImplementedError() - - def record_count(self): - raise NotImplementedError() - - def file_size_in_bytes(self): - raise NotImplementedError() - - def block_size_in_bytes(self): - raise NotImplementedError() - - def file_ordinal(self): - raise NotImplementedError() - - def sort_columns(self): - raise NotImplementedError() - - def column_sizes(self): - raise NotImplementedError() - - def value_counts(self): - raise NotImplementedError() - - def null_value_counts(self): - raise NotImplementedError() - - def lower_bounds(self): - raise NotImplementedError() - - def upper_bounds(self): - raise NotImplementedError() - - def copy(self): - raise NotImplementedError() diff --git a/iceberg/api/data_operations.py b/iceberg/api/data_operations.py deleted file mode 100644 index 7740a2bce1..0000000000 --- a/iceberg/api/data_operations.py +++ /dev/null @@ -1,23 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - - -class DataOperations(object): - APPEND = "append" - REPLACE = "replace" - OVERWRITE = "overwrite" - DELETE = "delete" diff --git a/iceberg/api/delete_files.py b/iceberg/api/delete_files.py deleted file mode 100644 index 9b6c01846b..0000000000 --- a/iceberg/api/delete_files.py +++ /dev/null @@ -1,34 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -from .pending_update import PendingUpdate - - -class DeleteFiles(PendingUpdate): - - def delete_files(self, path=None, datafile=None): - if datafile is not None: - path = datafile.path - - self._delete_from_path(path) - return self - - def _delete_from_path(self): - raise NotImplementedError() - - def delete_from_row_filter(self, expr): - raise NotImplementedError() diff --git a/iceberg/api/expire_snapshots.py b/iceberg/api/expire_snapshots.py deleted file mode 100644 index 0406844097..0000000000 --- a/iceberg/api/expire_snapshots.py +++ /dev/null @@ -1,30 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -from .pending_update import PendingUpdate - - -class ExpireSnapshots(PendingUpdate): - - def expire_snapshot_id(self, snapshot_id): - raise NotImplementedError() - - def expire_older_than(self, timestamp_millis): - raise NotImplementedError() - - def delete_with(self, delete_funct): - raise NotImplementedError() diff --git a/iceberg/api/expressions/__init__.py b/iceberg/api/expressions/__init__.py deleted file mode 100644 index 18450a2bd5..0000000000 --- a/iceberg/api/expressions/__init__.py +++ /dev/null @@ -1,107 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -from __future__ import absolute_import - -__all__ = ["ABOVE_MAX", - "And", - "BELOW_MIN", - "BinaryLiteral", - "Binder", - "BooleanLiteral", - "BoundPredicate", - "BoundReference", - "Evaluator", - "Expression", - "ExpressionVisitors", - "Expressions", - "DateLiteral", - "DecimalLiteral", - "DoubleLiteral", - "FALSE", - "FalseExp", - "FixedLiteral", - "FloatLiteral", - "inclusive", - "InclusiveManifestEvaluator", - "InclusiveMetricsEvaluator", - "InclusiveProjection", - "IntegerLiteral", - "JAVA_MAX_FLOAT", - "JAVA_MAX_INT", - "JAVA_MIN_FLOAT", - "JAVA_MIN_INT", - "Literal", - "Literals", - "NamedReference", - "Not", - "Operation", - "Or", - "Predicate", - "ResidualEvaluator", - "strict", - "StrictMetricsEvaluator", - "StrictProjection", - "StringLiteral", - "TRUE", - "TrueExp", - "UUIDLiteral", - "UnboundPredicate"] - -from .evaluator import (Binder, - Evaluator) -from .expression import (And, - Expression, - FALSE, - FalseExp, - Not, - Operation, - Or, - TRUE, - TrueExp) -from .expressions import Expressions, ExpressionVisitors -from .inclusive_manifest_evaluator import InclusiveManifestEvaluator -from .inclusive_metrics_evaluator import InclusiveMetricsEvaluator -from .java_variables import (JAVA_MAX_FLOAT, - JAVA_MAX_INT, - JAVA_MIN_FLOAT, - JAVA_MIN_INT) -from .literals import (ABOVE_MAX, - BELOW_MIN, - BinaryLiteral, - BooleanLiteral, - DateLiteral, - DecimalLiteral, - DoubleLiteral, - FixedLiteral, - FloatLiteral, - IntegerLiteral, - Literal, - Literals, - StringLiteral, - UUIDLiteral) -from .predicate import (BoundPredicate, - Predicate, - UnboundPredicate) -from .projections import (inclusive, - InclusiveProjection, - strict, - StrictProjection) -from .reference import (BoundReference, - NamedReference) -from .residual_evaluator import ResidualEvaluator -from .strict_metrics_evaluator import StrictMetricsEvaluator diff --git a/iceberg/api/expressions/binder.py b/iceberg/api/expressions/binder.py deleted file mode 100644 index 3be2d46e03..0000000000 --- a/iceberg/api/expressions/binder.py +++ /dev/null @@ -1,91 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -from .expressions import Expressions, ExpressionVisitors -from .predicate import BoundPredicate - - -class Binder(object): - - @staticmethod - def bind(struct, expr, case_sensitive=True): - return ExpressionVisitors.visit(expr, Binder.BindVisitor(struct, case_sensitive)) - - @staticmethod - def bound_references(struct, exprs, case_sensitive=True): - if exprs is None: - return set() - visitor = Binder.ReferenceVisitor() - for expr in exprs: - ExpressionVisitors.visit(Binder.bind(struct, expr, case_sensitive), visitor) - - return visitor.references - - def __init__(self): - pass - - class BindVisitor(ExpressionVisitors.ExpressionVisitor): - - def __init__(self, struct, case_sensitive=True): - self.struct = struct - self.case_sensitive = case_sensitive - - def always_true(self): - return Expressions.always_true() - - def always_false(self): - return Expressions.always_false() - - def not_(self, result): - return Expressions.not_(result) - - def and_(self, left_result, right_result): - return Expressions.and_(left_result, right_result) - - def or_(self, left_result, right_result): - return Expressions.or_(left_result, right_result) - - def predicate(self, pred): - if isinstance(pred, BoundPredicate): - raise RuntimeError("Found already bound predicate: {}".format(pred)) - - return pred.bind(self.struct, self.case_sensitive) - - class ReferenceVisitor(ExpressionVisitors.ExpressionVisitor): - - def __init__(self): - self.references = set() - - def always_true(self): - return self.references - - def always_false(self): - return self.references - - def not_(self, result): - return self.references - - def and_(self, left_result, right_result): - return self.references - - def or_(self, left_result, right_result): - return self.references - - def predicate(self, pred): - if isinstance(pred, BoundPredicate): - self.references.add(pred.ref.field_id) - return self.references diff --git a/iceberg/api/expressions/evaluator.py b/iceberg/api/expressions/evaluator.py deleted file mode 100644 index 2371624e15..0000000000 --- a/iceberg/api/expressions/evaluator.py +++ /dev/null @@ -1,92 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -import threading - -from .binder import Binder -from .expressions import ExpressionVisitors - - -class Evaluator(object): - - def __init__(self, struct, unbound, case_sensitive=True): - self.expr = Binder.bind(struct, unbound, case_sensitive) - self.thread_local_data = threading.local() - - def _visitor(self): - if not hasattr(self.thread_local_data, "visitors"): - self.thread_local_data.visitors = Evaluator.EvalVisitor() - - return self.thread_local_data.visitors - - def eval(self, data): - return self._visitor().eval(data, self.expr) - - class EvalVisitor(ExpressionVisitors.BoundExpressionVisitor): - - def __init__(self): - super(Evaluator.EvalVisitor, self).__init__() - self.struct = None - - def eval(self, row, expr): - self.struct = row - return ExpressionVisitors.visit(expr, self) - - def always_true(self): - return True - - def always_false(self): - return False - - def not_(self, result): - return not result - - def and_(self, left_result, right_result): - return left_result and right_result - - def or_(self, left_result, right_result): - return left_result or right_result - - def is_null(self, ref): - return ref.get(self.struct) is None - - def not_null(self, ref): - return not (ref.get(self.struct) is None) - - def lt(self, ref, lit): - return ref.get(self.struct) < lit.value - - def lt_eq(self, ref, lit): - return ref.get(self.struct) <= lit.value - - def gt(self, ref, lit): - return ref.get(self.struct) > lit.value - - def gt_eq(self, ref, lit): - return ref.get(self.struct) >= lit.value - - def eq(self, ref, lit): - return ref.get(self.struct) == lit.value - - def not_eq(self, ref, lit): - return ref.get(self.struct) != lit.value - - def in_(self, ref, lit): - raise NotImplementedError() - - def not_in(self, ref, lit): - return not self.in_(ref, lit.value) diff --git a/iceberg/api/expressions/expression.py b/iceberg/api/expressions/expression.py deleted file mode 100644 index 3dc0f06a31..0000000000 --- a/iceberg/api/expressions/expression.py +++ /dev/null @@ -1,252 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -from enum import Enum -from typing import TYPE_CHECKING - -if TYPE_CHECKING: - from .predicate import Predicate - - -class Expression(object): - - left: 'Predicate' - right: 'Predicate' - child: 'Predicate' - - def __init__(self): - pass - - def op(self): - raise RuntimeError("No implementation for base class") - - def negate(self): - raise RuntimeError("%s cannot be negated" % self) - - -class Operation(Enum): - TRUE = "TRUE" - FALSE = "FALSE" - IS_NULL = "IS_NULL" - NOT_NULL = "NOT_NULL" - IS_NAN = "IS_NAN" - NOT_NAN = "NOT_NAN" - LT = "LT" - LT_EQ = "LT_EQ" - GT = "GT" - GT_EQ = "GT_EQ" - EQ = "EQ" - NOT_EQ = "NOT_EQ" - IN = "IN" - NOT_IN = "NOT_IN" - NOT = "NOT" - AND = "AND" - OR = "OR" - - def negate(self): # noqa - if self == Operation.IS_NULL: - return Operation.NOT_NULL - elif self == Operation.NOT_NULL: - return Operation.IS_NULL - elif self == Operation.IS_NAN: - return Operation.NOT_NAN - elif self == Operation.NOT_NAN: - return Operation.IS_NAN - elif self == Operation.LT: - return Operation.GT_EQ - elif self == Operation.LT_EQ: - return Operation.GT - elif self == Operation.GT: - return Operation.LT_EQ - elif self == Operation.GT_EQ: - return Operation.LT - elif self == Operation.EQ: - return Operation.NOT_EQ - elif self == Operation.NOT_EQ: - return Operation.EQ - elif self == Operation.IN: - return Operation.NOT_IN - elif self == Operation.NOT_IN: - return Operation.IN - else: - raise RuntimeError("No negation for operation: %s" % self) - - def flipLR(self): - if self == Operation.LT: - return Operation.GT - elif self == Operation.LT_EQ: - return Operation.GT_EQ - elif self == Operation.GT: - return Operation.LT - elif self == Operation.GT_EQ: - return Operation.LT_EQ - elif self == Operation.EQ: - return Operation.EQ - elif self == Operation.NOT_EQ: - return Operation.NOT_EQ - elif self == Operation.AND: - return Operation.AND - elif self == Operation.OR: - return Operation.OR - else: - raise RuntimeError("No left-right flip for operation: %s" % self) - - def op(self): - pass - - -class And(Expression): - - def __init__(self, left, right): - self.left = left - self.right = right - - def __eq__(self, other): - if id(self) == id(other): - return True - elif other is None or not isinstance(other, And): - return False - - return self.left == other.left and self.right == other.right - - def __ne__(self, other): - return not self.__eq__(other) - - def op(self): - return Operation.AND - - def negate(self): - from .expressions import Expressions - return Expressions.or_(self.left.negate(), self.right.negate()) # noqa - - def __repr__(self): - return "And({},{})".format(self.left, self.right) - - def __str__(self): - return '({} and {})'.format(self.left, self.right) - - -class FalseExp(Expression): - - def op(self): - return Operation.FALSE - - def negate(self): - return TRUE - - def __repr__(self): - return "false" - - def __str__(self): - return self.__repr__() - - def __eq__(self, other): - if isinstance(other, FalseExp): - return True - - return False - - def __ne__(self, other): - return not self.__eq__(other) - - -class Or(Expression): - - def __init__(self, left, right): - self.left = left - self.right = right - - def __eq__(self, other): - if id(self) == id(other): - return True - elif other is None or not isinstance(other, Or): - return False - - return self.left == other.left and self.right == other.right - - def __ne__(self, other): - return not self.__eq__(other) - - def op(self): - return Operation.OR - - def negate(self): - from .expressions import Expressions - return Expressions.and_(self.left.negate(), self.right.negate()) # noqa - - def __repr__(self): - return "Or({},{})".format(self.left, self.right) - - def __str__(self): - return '({} or {})'.format(self.left, self.right) - - -class Not(Expression): - - def __init__(self, child): - self.child = child - - def __eq__(self, other): - if id(self) == id(other): - return True - elif other is None or not isinstance(other, Not): - return False - - return self.child == other.child - - def __ne__(self, other): - return not self.__eq__(other) - - def op(self): - return Operation.NOT - - def negate(self): - return self.child - - def __repr__(self): - return "Not({})".format(self.child) - - def __str__(self): - return 'not({})'.format(self.child) - - -class TrueExp(Expression): - - def op(self): - return Operation.TRUE - - def negate(self): - return False - - def __repr__(self): - return "true" - - def __str__(self): - return self.__repr__() - - def __eq__(self, other): - if isinstance(other, TrueExp): - return True - - return False - - def __ne__(self, other): - return not self.__eq__(other) - - -TRUE = TrueExp() -FALSE = FalseExp() diff --git a/iceberg/api/expressions/expression_parser.py b/iceberg/api/expressions/expression_parser.py deleted file mode 100644 index 0ffde4e23e..0000000000 --- a/iceberg/api/expressions/expression_parser.py +++ /dev/null @@ -1,161 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -# Derived from the SimpleSQL Parser example in pyparsing, retrofitted to just handle the -# where clause predicates -# https://github.com/pyparsing/pyparsing/blob/master/examples/simpleSQL.py - -import logging - -from pyparsing import ( - alphanums, - alphas, - CaselessKeyword, - delimitedList, - Group, - infixNotation, - oneOf, - opAssoc, - pyparsing_common as ppc, - quotedString, - Word -) - - -_logger = logging.getLogger(__name__) - -AND, OR, IN, IS, NOT, NULL, BETWEEN = map( - CaselessKeyword, "and or in is not null between".split() -) -NOT_NULL = NOT + NULL - -ident = Word(alphas, alphanums + "_$").setName("identifier") -columnName = delimitedList(ident, ".", combine=True).setName("column name") - -binop = oneOf("= == != < > >= <= eq ne lt le gt ge <>", caseless=False) -realNum = ppc.real() -intNum = ppc.signed_integer() - -columnRval = (realNum - | intNum - | quotedString - | columnName) # need to add support for alg expressions -whereCondition = Group( - (columnName + binop + columnRval) - | (columnName + IN + Group("(" + delimitedList(columnRval) + ")")) - | (columnName + IS + (NULL | NOT_NULL)) - | (columnName + BETWEEN + columnRval + AND + columnRval) - -) - -whereExpression = infixNotation( - Group(whereCondition - | NOT + whereCondition - | NOT + Group('(' + whereCondition + ')') - | NOT + columnName), - [(NOT, 1, opAssoc.LEFT), (AND, 2, opAssoc.LEFT), (OR, 2, opAssoc.LEFT), (IS, 2, opAssoc.LEFT)], -) - -op_map = {"=": "eq", - "==": "eq", - "eq": "eq", - ">": "gt", - "gt": "gt", - ">=": "gte", - "gte": "gte", - "<": "lt", - "lt": "lt", - "<=": "lte", - "lte": "lte", - "!": "not", - "not": "not", - "!=": "neq", - "<>": "neq", - "neq": "neq", - "||": "or", - "or": "or", - "&&": "and", - "and": "and", - "in": "in", - "between": "between", - "is": "is"} - - -def get_expr_tree(tokens): - if isinstance(tokens, (str, int)): - return tokens - if len(tokens) > 1: - if (tokens[0] == "not"): - return {"not": get_expr_tree(tokens[1])} - if (tokens[0] == "(" and tokens[-1] == ")"): - return get_expr_tree(tokens[1:-1]) - else: - return get_expr_tree(tokens[0]) - - op = op_map[tokens[1]] - - if op == "in": - return {'in': [get_expr_tree(tokens[0]), [token for token in tokens[2][1:-1]]]} - elif op == "between": - return {'and': [{"gte": [get_expr_tree(tokens[0]), tokens[2]]}, - {"lte": [get_expr_tree(tokens[0]), tokens[4]]}]} - elif op == "is": - - if tokens[2] == 'null': - return {"missing": tokens[0]} - else: - return {"exists": tokens[0]} - if len(tokens) > 3: - binary_tuples = get_expr_tree(tokens[2:]) - else: - binary_tuples = get_expr_tree(tokens[2]) - - return {op: [get_expr_tree(tokens[0]), - binary_tuples]} - - -def get_expr(node, expr_map): - if isinstance(node, dict): - for i in node.keys(): - op = i - if op == "literal": - return node["literal"] - mapped_op = expr_map.get(op, expr_map) - if len(mapped_op) == 1: - mapped_op = mapped_op[0] - if mapped_op is None: - raise RuntimeError("no mapping for op: %s" % op) - if op in ("not", "exists", "missing"): - return mapped_op(get_expr(node[op], expr_map)) - - return mapped_op(*get_expr(node[op], expr_map)) - elif isinstance(node, (list, tuple)): - return (get_expr(item, expr_map) for item in node) - elif isinstance(node, (str, int, float)): - return node - else: - raise RuntimeError("unknown node type" % node) - - -def parse_expr_string(predicate_string, expr_map): - from pyparsing import ParseException - - try: - expr = whereExpression.parseString(predicate_string, parseAll=True) - expr = get_expr_tree(expr) - return get_expr(expr, expr_map) - except ParseException as pe: - _logger.error("Error parsing string expression into iceberg expression: %s" % str(pe)) - raise diff --git a/iceberg/api/expressions/expressions.py b/iceberg/api/expressions/expressions.py deleted file mode 100644 index 4bcdc2cd12..0000000000 --- a/iceberg/api/expressions/expressions.py +++ /dev/null @@ -1,299 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -import logging - -from .expression import (And, - FALSE, - Not, - Operation, - Or, - TRUE) -from .expression_parser import parse_expr_string -from .predicate import (Predicate, - UnboundPredicate) -from .reference import NamedReference - -_logger = logging.getLogger(__name__) - - -class Expressions(object): - - @staticmethod - def and_(left, right): - if left == Expressions.always_false() or right == Expressions.always_false(): - return Expressions.always_false() - elif left == Expressions.always_true(): - return right - elif right == Expressions.always_true(): - return left - - return And(left, right) - - @staticmethod - def or_(left, right): - if left == Expressions.always_true() or right == Expressions.always_true(): - return Expressions.always_true() - elif left == Expressions.always_false(): - return right - elif right == Expressions.always_false(): - return left - - return Or(left, right) - - @staticmethod - def not_(child): - if child == Expressions.always_true(): - return Expressions.always_false() - elif child == Expressions.always_false(): - return Expressions.always_true() - elif isinstance(child, Not): - return child.child - - return Not(child) - - @staticmethod - def is_null(name): - return UnboundPredicate(Operation.IS_NULL, Expressions.ref(name)) - - @staticmethod - def not_null(name): - return UnboundPredicate(Operation.NOT_NULL, Expressions.ref(name)) - - @staticmethod - def is_nan(name): - return UnboundPredicate(Operation.IS_NAN, Expressions.ref(name)) - - @staticmethod - def not_nan(name): - return UnboundPredicate(Operation.NOT_NAN, Expressions.ref(name)) - - @staticmethod - def less_than(name, value): - return UnboundPredicate(Operation.LT, Expressions.ref(name), value) - - @staticmethod - def less_than_or_equal(name, value): - return UnboundPredicate(Operation.LT_EQ, Expressions.ref(name), value) - - @staticmethod - def greater_than(name, value): - return UnboundPredicate(Operation.GT, Expressions.ref(name), value) - - @staticmethod - def greater_than_or_equal(name, value): - return UnboundPredicate(Operation.GT_EQ, Expressions.ref(name), value) - - @staticmethod - def equal(name, value): - return UnboundPredicate(Operation.EQ, Expressions.ref(name), value) - - @staticmethod - def not_equal(name, value): - return UnboundPredicate(Operation.NOT_EQ, Expressions.ref(name), value) - - @staticmethod - def predicate(op, name, value=None, lit=None): - if value is not None and op not in (Operation.IS_NULL, Operation.NOT_NULL): - return UnboundPredicate(op, Expressions.ref(name), value) - elif lit is not None and op not in (Operation.IS_NULL, Operation.NOT_NULL): - return UnboundPredicate(op, Expressions.ref(name), value) - elif op in (Operation.IS_NULL, Operation.NOT_NULL): - if value is not None or lit is not None: - raise RuntimeError("Cannot create {} predicate inclusive a value".format(op)) - return UnboundPredicate(op, Expressions.ref(name)) - else: - raise RuntimeError("Cannot create {} predicate without a value".format(op)) - - @staticmethod - def always_true(): - return TRUE - - @staticmethod - def always_false(): - return FALSE - - @staticmethod - def rewrite_not(expr): - return ExpressionVisitors.visit(expr, RewriteNot.get()) # noqa - - @staticmethod - def ref(name): - return NamedReference(name) - - @staticmethod - def convert_string_to_expr(predicate_string): - expr_map = {"and": (Expressions.and_,), - "eq": (Expressions.equal,), - "exists": (Expressions.not_null,), - "gt": (Expressions.greater_than,), - "gte": (Expressions.greater_than_or_equal,), - "lt": (Expressions.less_than,), - "lte": (Expressions.less_than_or_equal,), - "missing": (Expressions.is_null,), - "neq": (Expressions.not_equal,), - "not": (Expressions.not_,), - "or": (Expressions.or_,)} - - return parse_expr_string(predicate_string, expr_map) - - -class ExpressionVisitors(object): - - @staticmethod - def visit(expr, visitor): - if isinstance(expr, Predicate): - return visitor.predicate(expr) - - if expr.op() == Operation.TRUE: - return visitor.always_true() - elif expr.op() == Operation.FALSE: - return visitor.always_false() - elif expr.op() == Operation.NOT: - return visitor.not_(ExpressionVisitors.visit(expr.child, visitor)) - elif expr.op() == Operation.AND: - return visitor.and_(ExpressionVisitors.visit(expr.left, visitor), - ExpressionVisitors.visit(expr.right, visitor)) - elif expr.op() == Operation.OR: - return visitor.or_(ExpressionVisitors.visit(expr.left, visitor), - ExpressionVisitors.visit(expr.right, visitor)) - else: - raise RuntimeError("Unknown operation: {}".format(expr.op())) - - class ExpressionVisitor(object): - - def always_true(self): - return NotImplementedError() - - def always_false(self): - return NotImplementedError() - - def not_(self, result): - return NotImplementedError() - - def and_(self, left_result, right_result): - return NotImplementedError() - - def or_(self, left_result, right_result): - return NotImplementedError() - - def predicate(self, pred): - return NotImplementedError() - - class BoundExpressionVisitor(ExpressionVisitor): - - def __init__(self): - super(ExpressionVisitors.BoundExpressionVisitor, self).__init__() - - def is_null(self, ref): - return NotImplementedError() - - def not_null(self, ref): - return NotImplementedError() - - def is_nan(self, ref): - return NotImplementedError() - - def not_nan(self, ref): - return NotImplementedError() - - def lt(self, ref, lit): - return NotImplementedError() - - def lt_eq(self, ref, lit): - return NotImplementedError() - - def gt(self, ref, lit): - return NotImplementedError() - - def gt_eq(self, ref, lit): - return None - - def eq(self, ref, lit): - return None - - def not_eq(self, ref, lit): - return None - - def in_(self, ref, lit): - return None - - def not_in(self, ref, lit): - return None - - def predicate(self, pred): # noqa - - if isinstance(pred, UnboundPredicate): - raise RuntimeError("Not a bound Predicate: {}".format(pred)) - - if pred.op == Operation.IS_NULL: - return self.is_null(pred.ref) - elif pred.op == Operation.NOT_NULL: - return self.not_null(pred.ref) - elif pred.op in [Operation.IS_NAN, Operation.NOT_NAN]: - raise NotImplementedError("IS_NAN and NOT_NAN not fully implemented for expressions") - elif pred.op == Operation.LT: - return self.lt(pred.ref, pred.lit) - elif pred.op == Operation.LT_EQ: - return self.lt_eq(pred.ref, pred.lit) - elif pred.op == Operation.GT: - return self.gt(pred.ref, pred.lit) - elif pred.op == Operation.GT_EQ: - return self.gt_eq(pred.ref, pred.lit) - elif pred.op == Operation.EQ: - return self.eq(pred.ref, pred.lit) - elif pred.op == Operation.NOT_EQ: - return self.not_eq(pred.ref, pred.lit) - elif pred.op == Operation.IN: - return self.in_(pred.ref, pred.lit) - elif pred.op == Operation.NOT_IN: - return self.not_in(pred.ref, pred.lit) - else: - raise RuntimeError("Unknown operation for Predicate: {}".format(pred.op)) - - -class RewriteNot(ExpressionVisitors.ExpressionVisitor): - __instance = None - - @staticmethod - def get(): - if RewriteNot.__instance is None: - RewriteNot() - return RewriteNot.__instance - - def __init__(self): - if RewriteNot.__instance is not None: - raise Exception("Multiple RewriteNot Types created") - RewriteNot.__instance = self - - def always_true(self): - return Expressions.always_true() - - def always_false(self): - return Expressions.always_false() - - def not_(self, result): - return result.negate() - - def and_(self, left_result, right_result): - return Expressions.and_(left_result, right_result) - - def or_(self, left_result, right_result): - return Expressions.or_(left_result, right_result) - - def predicate(self, pred): - return pred diff --git a/iceberg/api/expressions/inclusive_manifest_evaluator.py b/iceberg/api/expressions/inclusive_manifest_evaluator.py deleted file mode 100644 index 953ee99d08..0000000000 --- a/iceberg/api/expressions/inclusive_manifest_evaluator.py +++ /dev/null @@ -1,161 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -import threading - -from .binder import Binder -from .expressions import Expressions, ExpressionVisitors -from .projections import inclusive -from ..types import Conversions - -ROWS_MIGHT_MATCH = True -ROWS_CANNOT_MATCH = False - - -class InclusiveManifestEvaluator(object): - - def __init__(self, spec, row_filter, case_sensitive=True): - self.struct = spec.partition_type() - self.expr = Binder.bind(self.struct, - Expressions.rewrite_not(inclusive(spec, case_sensitive=case_sensitive) - .project(row_filter)), - case_sensitive=case_sensitive) - self.thread_local_data = threading.local() - - def _visitor(self): - if not hasattr(self.thread_local_data, "visitors"): - self.thread_local_data.visitors = ManifestEvalVisitor(self.expr) - - return self.thread_local_data.visitors - - def eval(self, manifest): - return self._visitor().eval(manifest) - - -class ManifestEvalVisitor(ExpressionVisitors.BoundExpressionVisitor): - - def __init__(self, expr): - self.expr = expr - self.stats = None - - def eval(self, manifest): - self.stats = manifest.partitions - if self.stats is None: - return ROWS_MIGHT_MATCH - - return ExpressionVisitors.visit(self.expr, self) - - def always_true(self): - return ROWS_MIGHT_MATCH - - def always_false(self): - return ROWS_CANNOT_MATCH - - def not_(self, result): - return not result - - def and_(self, left_result, right_result): - return left_result and right_result - - def or_(self, left_result, right_result): - return left_result or right_result - - def is_null(self, ref): - if not self.stats[ref.pos].contains_null(): - return ROWS_CANNOT_MATCH - - return ROWS_MIGHT_MATCH - - def not_null(self, ref): - lower_bound = self.stats[ref.pos].lower_bound() - if lower_bound is None: - return ROWS_CANNOT_MATCH - - return ROWS_MIGHT_MATCH - - def lt(self, ref, lit): - lower_bound = self.stats[ref.pos].lower_bound() - if lower_bound is None: - return ROWS_CANNOT_MATCH - - lower = Conversions.from_byte_buffer(ref.type, lower_bound) - - if lower >= lit.value: - return ROWS_CANNOT_MATCH - - return ROWS_MIGHT_MATCH - - def lt_eq(self, ref, lit): - lower_bound = self.stats[ref.pos].lower_bound() - if lower_bound is None: - return ROWS_CANNOT_MATCH - - lower = Conversions.from_byte_buffer(ref.type, lower_bound) - - if lower > lit.value: - return ROWS_CANNOT_MATCH - - return ROWS_MIGHT_MATCH - - def gt(self, ref, lit): - upper_bound = self.stats[ref.pos].upper_bound() - if upper_bound is None: - return ROWS_CANNOT_MATCH - - upper = Conversions.from_byte_buffer(ref.type, upper_bound) - - if upper <= lit.value: - return ROWS_CANNOT_MATCH - - return ROWS_MIGHT_MATCH - - def gt_eq(self, ref, lit): - upper_bound = self.stats[ref.pos].upper_bound() - if upper_bound is None: - return ROWS_CANNOT_MATCH - - upper = Conversions.from_byte_buffer(ref.type, upper_bound) - - if upper < lit.value: - return ROWS_CANNOT_MATCH - - return ROWS_MIGHT_MATCH - - def eq(self, ref, lit): - field_stats = self.stats[ref.pos] - if field_stats.lower_bound() is None: - return ROWS_CANNOT_MATCH - - lower = Conversions.from_byte_buffer(ref.type, field_stats.lower_bound()) - if lower > lit.value: - return ROWS_CANNOT_MATCH - - upper = Conversions.from_byte_buffer(ref.type, field_stats.upper_bound()) - - if upper < lit.value: - return ROWS_CANNOT_MATCH - - return ROWS_MIGHT_MATCH - - def not_eq(self, ref, lit): - return ROWS_MIGHT_MATCH - - def in_(self, ref, lit): - return ROWS_MIGHT_MATCH - - def not_in(self, ref, lit): - return ROWS_MIGHT_MATCH diff --git a/iceberg/api/expressions/inclusive_metrics_evaluator.py b/iceberg/api/expressions/inclusive_metrics_evaluator.py deleted file mode 100644 index 73b636fff0..0000000000 --- a/iceberg/api/expressions/inclusive_metrics_evaluator.py +++ /dev/null @@ -1,188 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -import threading - -from .expressions import Expressions, ExpressionVisitors -from ..expressions.binder import Binder -from ..types import Conversions - - -class InclusiveMetricsEvaluator(object): - - def __init__(self, schema, unbound, case_sensitive=True): - self.schema = schema - self.struct = schema.as_struct() - self.case_sensitive = case_sensitive - self.expr = Binder.bind(self.struct, Expressions.rewrite_not(unbound), case_sensitive) - self.thread_local_data = threading.local() - - def _visitor(self): - if not hasattr(self.thread_local_data, "visitors"): - self.thread_local_data.visitors = MetricsEvalVisitor(self.expr, self.schema, self.struct) - - return self.thread_local_data.visitors - - def eval(self, file): - return self._visitor().eval(file) - - -class MetricsEvalVisitor(ExpressionVisitors.BoundExpressionVisitor): - ROWS_MIGHT_MATCH = True - ROWS_CANNOT_MATCH = False - - def __init__(self, expr, schema, struct): - self.expr = expr - self.value_counts = None - self.null_counts = None - self.lower_bounds = None - self.upper_bounds = None - self.schema = schema - self.struct = struct - - def eval(self, file): - if file.record_count() <= 0: - return MetricsEvalVisitor.ROWS_CANNOT_MATCH - - self.value_counts = file.value_counts() - self.null_counts = file.null_value_counts() - self.lower_bounds = file.lower_bounds() - self.upper_bounds = file.upper_bounds() - - return ExpressionVisitors.visit(self.expr, self) - - def always_true(self): - return MetricsEvalVisitor.ROWS_MIGHT_MATCH - - def always_false(self): - return MetricsEvalVisitor.ROWS_CANNOT_MATCH - - def not_(self, result): - return not result - - def and_(self, left_result, right_result): - return left_result and right_result - - def or_(self, left_result, right_result): - return left_result or right_result - - def is_null(self, ref): - id = ref.field.field_id - - if self.struct.field(id=id) is None: - raise RuntimeError("Cannot filter by nested column: %s" % self.schema.find_field(id)) - - if self.null_counts is not None and self.null_counts.get(id, -1) == 0: - return MetricsEvalVisitor.ROWS_CANNOT_MATCH - - return MetricsEvalVisitor.ROWS_MIGHT_MATCH - - def not_null(self, ref): - id = ref.field.field_id - - if self.struct.field(id=id) is None: - raise RuntimeError("Cannot filter by nested column: %s" % self.schema.find_field(id)) - - if self.value_counts is not None and id in self.value_counts and id in self.null_counts \ - and self.value_counts.get(id) - self.null_counts.get(id) == 0: - return MetricsEvalVisitor.ROWS_CANNOT_MATCH - - return MetricsEvalVisitor.ROWS_MIGHT_MATCH - - def lt(self, ref, lit): - id = ref.field.field_id - field = self.struct.field(id=id) - - if field is None: - raise RuntimeError("Cannot filter by nested column: %s" % self.schema.find_field(id)) - - if self.lower_bounds is not None and id in self.lower_bounds: - lower = Conversions.from_byte_buffer(field.type, self.lower_bounds.get(id)) - if lower >= lit.value: - return MetricsEvalVisitor.ROWS_CANNOT_MATCH - - return MetricsEvalVisitor.ROWS_MIGHT_MATCH - - def lt_eq(self, ref, lit): - id = ref.field.field_id - field = self.struct.field(id=id) - - if field is None: - raise RuntimeError("Cannot filter by nested column: %s" % self.schema.find_field(id)) - - if self.lower_bounds is not None and id in self.lower_bounds: - lower = Conversions.from_byte_buffer(field.type, self.lower_bounds.get(id)) - if lower > lit.value: - return MetricsEvalVisitor.ROWS_CANNOT_MATCH - - return MetricsEvalVisitor.ROWS_MIGHT_MATCH - - def gt(self, ref, lit): - id = ref.field.field_id - field = self.struct.field(id=id) - - if field is None: - raise RuntimeError("Cannot filter by nested column: %s" % self.schema.find_field(id)) - - if self.upper_bounds is not None and id in self.upper_bounds: - upper = Conversions.from_byte_buffer(field.type, self.upper_bounds.get(id)) - if upper <= lit.value: - return MetricsEvalVisitor.ROWS_CANNOT_MATCH - - return MetricsEvalVisitor.ROWS_MIGHT_MATCH - - def gt_eq(self, ref, lit): - id = ref.field.field_id - field = self.struct.field(id=id) - - if field is None: - raise RuntimeError("Cannot filter by nested column: %s" % self.schema.find_field(id)) - - if self.upper_bounds is not None and id in self.upper_bounds: - upper = Conversions.from_byte_buffer(field.type, self.upper_bounds.get(id)) - if upper < lit.value: - return MetricsEvalVisitor.ROWS_CANNOT_MATCH - - return MetricsEvalVisitor.ROWS_MIGHT_MATCH - - def eq(self, ref, lit): - id = ref.field.field_id - field = self.struct.field(id=id) - - if field is None: - raise RuntimeError("Cannot filter by nested column: %s" % self.schema.find_field(id)) - - if self.lower_bounds is not None and id in self.lower_bounds: - lower = Conversions.from_byte_buffer(field.type, self.lower_bounds.get(id)) - if lower > lit.value: - return MetricsEvalVisitor.ROWS_CANNOT_MATCH - - if self.upper_bounds is not None and id in self.upper_bounds: - upper = Conversions.from_byte_buffer(field.type, self.upper_bounds.get(id)) - if upper < lit.value: - return MetricsEvalVisitor.ROWS_CANNOT_MATCH - - return MetricsEvalVisitor.ROWS_MIGHT_MATCH - - def not_eq(self, ref, lit): - return MetricsEvalVisitor.ROWS_MIGHT_MATCH - - def in_(self, ref, lit): - return MetricsEvalVisitor.ROWS_MIGHT_MATCH - - def not_in(self, ref, lit): - return MetricsEvalVisitor.ROWS_MIGHT_MATCH diff --git a/iceberg/api/expressions/java_variables/__init__.py b/iceberg/api/expressions/java_variables/__init__.py deleted file mode 100644 index 1ee068e2f7..0000000000 --- a/iceberg/api/expressions/java_variables/__init__.py +++ /dev/null @@ -1,21 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -JAVA_MAX_INT = 2147483647 -JAVA_MIN_INT = -2147483648 -JAVA_MAX_FLOAT = 3.4028235E38 -JAVA_MIN_FLOAT = -3.4028235E38 diff --git a/iceberg/api/expressions/literals.py b/iceberg/api/expressions/literals.py deleted file mode 100644 index 41387ce2bc..0000000000 --- a/iceberg/api/expressions/literals.py +++ /dev/null @@ -1,591 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -import datetime -from decimal import (Decimal, - ROUND_HALF_UP) -import uuid - -import pytz - - -from .expression import (FALSE, - TRUE) -from .java_variables import (JAVA_MAX_FLOAT, - JAVA_MIN_FLOAT) -from ..types.conversions import Conversions -from ..types.type import TypeID - - -class Literals(object): - - EPOCH = datetime.datetime.utcfromtimestamp(0) - EPOCH_DAY = EPOCH.date() - - @staticmethod - def from_(value): # noqa: C901 - if value is None: - raise RuntimeError("Cannot create an expression literal from None") - if isinstance(value, bool): - return BooleanLiteral(value) - elif isinstance(value, int): - if Literal.JAVA_MIN_INT < value < Literal.JAVA_MAX_INT: - return IntegerLiteral(value) - return LongLiteral(value) - elif isinstance(value, float): - if Literal.JAVA_MIN_FLOAT < value < Literal.JAVA_MAX_FLOAT: - return FloatLiteral(value) - return DoubleLiteral(value) - elif isinstance(value, str): - return StringLiteral(value) - elif isinstance(value, uuid.UUID): - return UUIDLiteral(value) - elif isinstance(value, bytearray): - return BinaryLiteral(value) - elif isinstance(value, bytes): - return FixedLiteral(value) - elif isinstance(value, Decimal): - return DecimalLiteral(value) - else: - raise NotImplementedError("Unimplemented Type Literal for value: %s" % value) - - @staticmethod - def above_max(): - return ABOVE_MAX - - @staticmethod - def below_min(): - return BELOW_MIN - - -class Literal(object): - JAVA_MAX_INT = 2147483647 - JAVA_MIN_INT = -2147483648 - JAVA_MAX_FLOAT = 3.4028235E38 - JAVA_MIN_FLOAT = -3.4028235E38 - - @staticmethod - def of(value): # noqa: C901 - - if isinstance(value, bool): - return BooleanLiteral(value) - elif isinstance(value, int): - if value < Literal.JAVA_MIN_INT or value > Literal.JAVA_MAX_INT: - return LongLiteral(value) - return IntegerLiteral(value) - elif isinstance(value, float): - if value < Literal.JAVA_MIN_FLOAT or value > Literal.JAVA_MAX_FLOAT: - return DoubleLiteral(value) - return FloatLiteral(value) - elif isinstance(value, str): - return StringLiteral(value) - elif isinstance(value, uuid.UUID): - return UUIDLiteral(value) - elif isinstance(value, bytes): - return FixedLiteral(value) - elif isinstance(value, bytearray): - return BinaryLiteral(value) - elif isinstance(value, Decimal): - return DecimalLiteral(value) - - def to(self, type_var): - raise NotImplementedError() - - def to_byte_buffer(self): - raise NotImplementedError() - - -class BaseLiteral(Literal): - def __init__(self, value, type_id): - self.value = value - self.byte_buffer = None - self.type_id = type_id - - def to(self, type_var): - raise NotImplementedError() - - def __eq__(self, other): - if id(self) == id(other): - return True - elif other is None or not isinstance(other, BaseLiteral): - return False - - return self.value == other.value - - def __ne__(self, other): - return not self.__eq__(other) - - def __repr__(self): - return "BaseLiteral(%s)" % str(self.value) - - def __str__(self): - return str(self.value) - - def to_byte_buffer(self): - if self.byte_buffer is None: - self.byte_buffer = Conversions.to_byte_buffer(self.type_id, self.value) - - return self.byte_buffer - - -class ComparableLiteral(BaseLiteral): - - def __init__(self, value, type_id): - super(ComparableLiteral, self).__init__(value, type_id) - - def to(self, type): - raise NotImplementedError() - - def __eq__(self, other): - return self.value == other.value - - def __ne__(self, other): - return not self.__eq__(other) - - def __lt__(self, other): - if self.value is None: - return True - - if other is None or other.value is None: - return False - - return self.value < other.value - - def __gt__(self, other): - if self.value is None: - return False - - if other is None or other.value is None: - return True - - return self.value > other.value - - def __le__(self, other): - if self.value is None: - return True - - if other is None or other.value is None: - return False - - return self.value <= other.value - - def __ge__(self, other): - if self.value is None: - return False - - if other is None or other.value is None: - return True - - return self.value >= other.value - - -class AboveMax(Literal): - def __init__(self): - super(AboveMax, self).__init__() - - def value(self): - raise RuntimeError("AboveMax has no value") - - def to(self, type): - raise RuntimeError("Cannot change the type of AboveMax") - - def __str__(self): - return "aboveMax" - - -class BelowMin(Literal): - def __init__(self): - super(BelowMin, self).__init__() - - def value(self): - raise RuntimeError("BelowMin has no value") - - def to(self, type): - raise RuntimeError("Cannot change the type of BelowMin") - - def __str__(self): - return "belowMin" - - -class BooleanLiteral(ComparableLiteral): - - def __init__(self, value): - super(BooleanLiteral, self).__init__(value, TypeID.BOOLEAN) - - def to(self, type_var): - if type_var.type_id == TypeID.BOOLEAN: - return self - - -class IntegerLiteral(ComparableLiteral): - - def __init__(self, value): - super(IntegerLiteral, self).__init__(value, TypeID.INTEGER) - - def to(self, type_var): - if type_var.type_id == TypeID.INTEGER: - return self - elif type_var.type_id == TypeID.LONG: - return LongLiteral(self.value) - elif type_var.type_id == TypeID.FLOAT: - return FloatLiteral(float(self.value)) - elif type_var.type_id == TypeID.DOUBLE: - return DoubleLiteral(float(self.value)) - elif type_var.type_id == TypeID.DATE: - return DateLiteral(self.value) - elif type_var.type_id == TypeID.DECIMAL: - if type_var.scale == 0: - return DecimalLiteral(Decimal(self.value)) - else: - return DecimalLiteral(Decimal(self.value) - .quantize(Decimal("." + "".join(["0" for i in range(1, type_var.scale)]) + "1"), - rounding=ROUND_HALF_UP)) - - -class LongLiteral(ComparableLiteral): - - def __init__(self, value): - super(LongLiteral, self).__init__(value, TypeID.LONG) - - def to(self, type_var): # noqa: C901 - if type_var.type_id == TypeID.INTEGER: - if Literal.JAVA_MAX_INT < self.value: - return ABOVE_MAX - elif Literal.JAVA_MIN_INT > self.value: - return BELOW_MIN - - return IntegerLiteral(self.value) - elif type_var.type_id == TypeID.LONG: - return self - elif type_var.type_id == TypeID.FLOAT: - return FloatLiteral(float(self.value)) - elif type_var.type_id == TypeID.DOUBLE: - return DoubleLiteral(float(self.value)) - elif type_var.type_id == TypeID.TIME: - return TimeLiteral(self.value) - elif type_var.type_id == TypeID.TIMESTAMP: - return TimestampLiteral(self.value) - elif type_var.type_id == TypeID.DECIMAL: - if type_var.scale == 0: - return DecimalLiteral(Decimal(self.value)) - else: - return DecimalLiteral(Decimal(self.value) - .quantize(Decimal("." + "".join(["0" for i in range(1, type_var.scale)]) + "1"), - rounding=ROUND_HALF_UP)) - - -class FloatLiteral(ComparableLiteral): - - def __init__(self, value): - super(FloatLiteral, self).__init__(value, TypeID.FLOAT) - - def to(self, type_var): - if type_var.type_id == TypeID.FLOAT: - return self - elif type_var.type_id == TypeID.DOUBLE: - return DoubleLiteral(self.value) - elif type_var.type_id == TypeID.DECIMAL: - if type_var.scale == 0: - return DecimalLiteral(Decimal(self.value) - .quantize(Decimal('1.'), - rounding=ROUND_HALF_UP)) - else: - return DecimalLiteral(Decimal(self.value) - .quantize(Decimal("." + "".join(["0" for i in range(1, type_var.scale)]) + "1"), - rounding=ROUND_HALF_UP)) - - -class DoubleLiteral(ComparableLiteral): - - def __init__(self, value): - super(DoubleLiteral, self).__init__(value, TypeID.DOUBLE) - - def to(self, type_var): - if type_var.type_id == TypeID.FLOAT: - if JAVA_MAX_FLOAT < self.value: - return ABOVE_MAX - elif JAVA_MIN_FLOAT > self.value: - return BELOW_MIN - - return FloatLiteral(self.value) - elif type_var.type_id == TypeID.DOUBLE: - return self - elif type_var.type_id == TypeID.DECIMAL: - if type_var.scale == 0: - return DecimalLiteral(Decimal(self.value) - .quantize(Decimal('1.'), - rounding=ROUND_HALF_UP)) - else: - return DecimalLiteral(Decimal(self.value) - .quantize(Decimal("." + "".join(["0" for i in range(1, type_var.scale)]) + "1"), - rounding=ROUND_HALF_UP)) - - -class DateLiteral(ComparableLiteral): - - def __init__(self, value): - super(DateLiteral, self).__init__(value, TypeID.DATE) - - def to(self, type_var): - if type_var.type_id == TypeID.DATE: - return self - - -class TimeLiteral(ComparableLiteral): - - def __init__(self, value): - super(TimeLiteral, self).__init__(value, TypeID.TIME) - - def to(self, type_var): - if type_var.type_id == TypeID.TIME: - return self - - -class TimestampLiteral(ComparableLiteral): - - def __init__(self, value): - super(TimestampLiteral, self).__init__(value, TypeID.TIMESTAMP) - - def to(self, type_var): - if type_var.type_id == TypeID.TIMESTAMP: - return self - elif type_var.type_id == TypeID.DATE: - return DateLiteral((datetime.datetime.fromtimestamp(self.value / 1000000) - Literals.EPOCH).days) - - -class DecimalLiteral(ComparableLiteral): - - def __init__(self, value): - super(DecimalLiteral, self).__init__(value, TypeID.DECIMAL) - - def to(self, type_var): - if type_var.type_id == TypeID.DECIMAL and type_var.scale == abs(self.value.as_tuple().exponent): - return self - - -class StringLiteral(BaseLiteral): - def __init__(self, value): - super(StringLiteral, self).__init__(value, TypeID.STRING) - - def to(self, type_var): # noqa: C901 - import dateutil.parser - if type_var.type_id == TypeID.DATE: - return DateLiteral((dateutil.parser.parse(self.value) - Literals.EPOCH).days) - elif type_var.type_id == TypeID.TIME: - return TimeLiteral( - int((dateutil.parser.parse(Literals.EPOCH.strftime("%Y-%m-%d ") + self.value) - Literals.EPOCH) - .total_seconds() * 1000000)) - elif type_var.type_id == TypeID.TIMESTAMP: - timestamp = dateutil.parser.parse(self.value) - EPOCH = Literals.EPOCH - if bool(timestamp.tzinfo) != bool(type_var.adjust_to_utc): - raise RuntimeError("Cannot convert to %s when string is: %s" % (type_var, self.value)) - - if timestamp.tzinfo is not None: - EPOCH = EPOCH.replace(tzinfo=pytz.UTC) - - return TimestampLiteral(int((timestamp - EPOCH).total_seconds() * 1000000)) - elif type_var.type_id == TypeID.STRING: - return self - elif type_var.type_id == TypeID.UUID: - return UUIDLiteral(uuid.UUID(self.value)) - elif type_var.type_id == TypeID.DECIMAL: - dec_val = Decimal(str(self.value)) - if abs(dec_val.as_tuple().exponent) == type_var.scale: - if type_var.scale == 0: - return DecimalLiteral(Decimal(str(self.value)) - .quantize(Decimal('1.'), - rounding=ROUND_HALF_UP)) - else: - return DecimalLiteral(Decimal(str(self.value)) - .quantize(Decimal("." + "".join(["0" for i in range(1, type_var.scale)]) + "1"), - rounding=ROUND_HALF_UP)) - - def __eq__(self, other): - if id(self) == id(other): - return True - - if other is None or not isinstance(other, StringLiteral): - return False - - return self.value == other.value - - def __ne__(self, other): - return not self.__eq__(other) - - def __lt__(self, other): - if other is None: - return False - - return self.value < other.value - - def __gt__(self, other): - if other is None: - return True - - return self.value > other.value - - def __le__(self, other): - if other is None: - return False - - return self.value <= other.value - - def __ge__(self, other): - if other is None: - return True - - return self.value >= other.value - - def __str__(self): - return '"' + self.value + '"' - - -class UUIDLiteral(ComparableLiteral): - def __init__(self, value): - super(UUIDLiteral, self).__init__(value, TypeID.UUID) - - def to(self, type_var): - if type_var.type_id == TypeID.UUID: - return self - - -class FixedLiteral(BaseLiteral): - def __init__(self, value): - super(FixedLiteral, self).__init__(value, TypeID.FIXED) - - def to(self, type_var): - if type_var.type_id == TypeID.FIXED: - if len(self.value) == type_var.length: - return self - elif type_var.type_id == TypeID.BINARY: - return BinaryLiteral(self.value) - - def write_replace(self): - return FixedLiteralProxy(self.value) - - def __eq__(self, other): - return self.value == other.value - - def __ne__(self, other): - return not self.__eq__(other) - - def __lt__(self, other): - if other is None: - return False - - return self.value < other.value - - def __gt__(self, other): - if other is None: - return True - - return self.value > other.value - - def __le__(self, other): - if other is None: - return False - - return self.value <= other.value - - def __ge__(self, other): - if other is None: - return True - - return self.value >= other.value - - -class BinaryLiteral(BaseLiteral): - def __init__(self, value): - super(BinaryLiteral, self).__init__(value, TypeID.BINARY) - - def to(self, type_var): - if type_var.type_id == TypeID.FIXED: - if type_var.length == len(self.value): - return FixedLiteral(self.value) - return None - elif type_var.type_id == TypeID.BINARY: - return self - - def write_replace(self): - return BinaryLiteralProxy(self.value) - - def __eq__(self, other): - return self.value == other.value - - def __ne__(self, other): - return not self.__eq__(other) - - def __lt__(self, other): - if other is None: - return False - - return self.value < other.value - - def __gt__(self, other): - if other is None: - return True - - return self.value > other.value - - def __le__(self, other): - if other is None: - return False - - return self.value <= other.value - - def __ge__(self, other): - if other is None: - return True - - return self.value >= other.value - - -class FixedLiteralProxy(object): - - def __init__(self, buffer=None): - if buffer is not None: - self.bytes = list(buffer) - - def read_resolve(self): - return FixedLiteral(self.bytes) - - -class ConstantExpressionProxy(object): - - def __init__(self, true_or_false=None): - if true_or_false is not None: - self.true_or_false = true_or_false - - def read_resolve(self): - if self.true_or_false: - return TRUE - else: - return FALSE - - -class BinaryLiteralProxy(FixedLiteralProxy): - - def __init__(self, buffer=None): - super(BinaryLiteralProxy, self).__init__(buffer) - - def read_resolve(self): - return BinaryLiteral(self.bytes) - - -ABOVE_MAX = AboveMax() -BELOW_MIN = BelowMin() diff --git a/iceberg/api/expressions/predicate.py b/iceberg/api/expressions/predicate.py deleted file mode 100644 index e868a07243..0000000000 --- a/iceberg/api/expressions/predicate.py +++ /dev/null @@ -1,295 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -from __future__ import annotations - -from math import isnan -from typing import Any, List, Optional, TYPE_CHECKING, Union - -from iceberg.exceptions import ValidationException - -from .expression import (Expression, - Operation) -from .literals import (BaseLiteral, - Literals) -from .term import BoundTerm, UnboundTerm -from ..types import TypeID - -if TYPE_CHECKING: - from iceberg.api import StructLike - - -class Predicate(Expression): - - def __init__(self, op: Operation, term: Union[BoundTerm, UnboundTerm]): - if term is None: - raise ValueError("Term cannot be None") - - self.op: Operation = op - self.term: Union[BoundTerm, UnboundTerm] = term - - @property - def ref(self): - return self.term.ref - - @property - def lit(self): - raise NotImplementedError("Not Implemented for base class") - - def __eq__(self, other): - if id(self) == id(other): - return True - elif other is None or not isinstance(other, Predicate): - return False - - return self.op == other.op and self.ref == other.ref and self.lit == other.lit - - def __ne__(self, other): - return not self.__eq__(other) - - def __repr__(self): - return "Predicate({},{},{})".format(self.op, self.ref, self.lit) - - def __str__(self): - if self.op == Operation.IS_NULL: - return "is_null({})".format(self.ref) - elif self.op == Operation.NOT_NULL: - return "not_null({})".format(self.ref) - elif self.op == Operation.LT: - return "less_than({})".format(self.ref) - elif self.op == Operation.LT_EQ: - return "less_than_equal({})".format(self.ref) - elif self.op == Operation.GT: - return "greater_than({})".format(self.ref) - elif self.op == Operation.GT_EQ: - return "greater_than_equal({})".format(self.ref) - elif self.op == Operation.EQ: - return "equal({})".format(self.ref) - elif self.op == Operation.NOT_EQ: - return "not_equal({})".format(self.ref) - else: - return "invalid predicate: operation = {}".format(self.op) - - -class BoundPredicate(Predicate): - - def __init__(self, op: Operation, term: BoundTerm, lit: BaseLiteral = None, literals: List[BaseLiteral] = None, - is_unary_predicate: bool = False, is_literal_predicate: bool = False, - is_set_predicate: bool = False): - self.is_unary_predicate = is_unary_predicate - self.is_literal_predicate = is_literal_predicate - self.is_set_predicate = is_set_predicate - - super(BoundPredicate, self).__init__(op, term) - ValidationException.check(sum([is_unary_predicate, is_literal_predicate, is_set_predicate]) == 1, - "Only a single predicate type may be set: %s=%s, %s=%s, %s=%s", - ("is_unary_predicate", is_unary_predicate, - "is_literal_predicate", is_literal_predicate, - "is_set_predicate", is_set_predicate)) - - self._literals: Optional[List[BaseLiteral]] = None - if self.is_unary_predicate: - ValidationException.check(lit is None, "Unary Predicates may not have a literal", ()) - - elif self.is_literal_predicate: - ValidationException.check(lit is not None, "Literal Predicates must have a literal set", ()) - self._literals = [lit] # type: ignore - - elif self.is_set_predicate: - ValidationException.check(literals is not None, "Set Predicates must have literals set", ()) - self._literals = literals - else: - raise ValueError(f"Unable to instantiate {op} -> (lit={lit}, literal={literals}") - - @property - def lit(self) -> Optional[BaseLiteral]: - if self._literals is None or len(self._literals) == 0: - return None - return self._literals[0] - - def eval(self, struct: StructLike) -> bool: - ValidationException.check(isinstance(self.term, BoundTerm), "Term must be bound to eval: %s", (self.term)) - return self.test(self.term.eval(struct)) # type: ignore - - def test(self, struct: StructLike = None, value: Any = None) -> bool: - ValidationException.check(struct is None or value is None, "Either struct or value must be none", ()) - if struct is not None: - ValidationException.check(isinstance(self.term, BoundTerm), "Term must be bound to eval: %s", (self.term)) - return self.test(value=self.term.eval(struct)) # type: ignore - else: - if self.is_unary_predicate: - return self.test_unary_predicate(value) - elif self.is_literal_predicate: - return self.test_literal_predicate(value) - else: - return self.test_set_predicate(value) - - def test_unary_predicate(self, value: Any) -> bool: - - if self.op == Operation.IS_NULL: - return value is None - elif self.op == Operation.NOT_NULL: - return value is not None - elif self.op == Operation.IS_NAN: - return isnan(value) - elif self.op == Operation.NOT_NAN: - return not isnan(value) - else: - raise ValueError(f"{self.op} is not a valid unary predicate") - - def test_literal_predicate(self, value: Any) -> bool: - if self.lit is None: - raise ValidationException("Literal must not be none", ()) - - if self.op == Operation.LT: - return value < self.lit.value - elif self.op == Operation.LT_EQ: - return value <= self.lit.value - elif self.op == Operation.GT: - return value > self.lit.value - elif self.op == Operation.GT_EQ: - return value >= self.lit.value - elif self.op == Operation.EQ: - return value == self.lit.value - elif self.op == Operation.NOT_EQ: - return value != self.lit.value - else: - raise ValueError(f"{self.op} is not a valid literal predicate") - - def test_set_predicate(self, value: Any) -> bool: - if self._literals is None: - raise ValidationException("Literals must not be none", ()) - - if self.op == Operation.IN: - return value in self._literals - elif self.op == Operation.NOT_IN: - return value not in self._literals - else: - raise ValueError(f"{self.op} is not a valid set predicate") - - -class UnboundPredicate(Predicate): - - def __init__(self, op, term, value=None, lit=None, values=None, literals=None): - self._literals = None - num_set_args = sum([1 for x in [value, lit, values, literals] if x is not None]) - - if num_set_args > 1: - raise ValueError(f"Only one of value={value}, lit={lit}, values={values}, literals={literals} may be set") - super(UnboundPredicate, self).__init__(op, term) - if isinstance(value, BaseLiteral): - lit = value - value = None - if value is not None: - self._literals = [Literals.from_(value)] - elif lit is not None: - self._literals = [lit] - elif values is not None: - self._literals = map(Literals.from_, values) - elif literals is not None: - self._literals = literals - - @property - def literals(self): - return self._literals - - @property - def lit(self): - if self.op in [Operation.IN, Operation.NOT_IN]: - raise ValueError(f"{self.op} predicate cannot return a literal") - - return None if self.literals is None else self.literals[0] - - def negate(self): - return UnboundPredicate(self.op.negate(), self.term, literals=self.literals) - - def bind(self, struct, case_sensitive=True): - bound = self.term.bind(struct, case_sensitive=case_sensitive) - - if self.literals is None: - return self.bind_unary_operation(bound) - elif self.op in [Operation.IN, Operation.NOT_IN]: - return self.bind_in_operation(bound) - - return self.bind_literal_operation(bound) - - def bind_unary_operation(self, bound_term: BoundTerm) -> BoundPredicate: - from .expressions import Expressions - if self.op == Operation.IS_NULL: - if bound_term.ref.field.is_required: - return Expressions.always_false() - return BoundPredicate(Operation.IS_NULL, bound_term, is_unary_predicate=True) - elif self.op == Operation.NOT_NULL: - if bound_term.ref.field.is_required: - return Expressions.always_true() - return BoundPredicate(Operation.NOT_NULL, bound_term, is_unary_predicate=True) - elif self.op in [Operation.IS_NAN, Operation.NOT_NAN]: - if not self.floating_type(bound_term.ref.type.type_id): - raise ValidationException(f"{self.op} cannot be used with a non-floating column", ()) - return BoundPredicate(self.op, bound_term, is_unary_predicate=True) - - raise ValidationException(f"Operation must be in [IS_NULL, NOT_NULL, IS_NAN, NOT_NAN] was:{self.op}", ()) - - def bind_in_operation(self, bound_term): - from .expressions import Expressions - - def convert_literal(lit): - converted = lit.to(bound_term) - ValidationException.check(converted is not None, - "Invalid Value for conversion to type %s: %s (%s)", - (bound_term.type, lit, lit.__class__.__name__)) - return converted - - converted_literals = filter(lambda x: x != Literals.above_max() and x != Literals.below_min(), - [convert_literal(lit) for lit in self.literals]) - if len(converted_literals) == 0: - return Expressions.always_true() if Operation.NOT_IN else Expressions.always_false() - literal_set = set(converted_literals) - if len(literal_set) == 1: - if self.op == Operation.IN: - return BoundPredicate(Operation.EQ, bound_term, literal_set[0]) - elif self.op == Operation.NOT_IN: - return BoundPredicate(Operation.NOT_EQ, bound_term, literal_set[0]) - else: - raise ValidationException("Operation must be in or not in", ()) - - return BoundPredicate(self.op, bound_term, literals=literal_set, is_set_predicate=True) - - def bind_literal_operation(self, bound_term): - from .expressions import Expressions - - lit = self.lit.to(bound_term.type) - ValidationException.check(lit is not None, - "Invalid Value for conversion to type %s: %s (%s)", - (bound_term.type, self.lit, self.lit.__class__.__name__)) - - if lit == Literals.above_max(): - if self.op in [Operation.LT, Operation.LT_EQ, Operation.NOT_EQ]: - return Expressions.always_true() - elif self.op in [Operation.GT, Operation.GT_EQ, Operation.EQ]: - return Expressions.always_false() - elif lit == Literals.below_min(): - if self.op in [Operation.LT, Operation.LT_EQ, Operation.NOT_EQ]: - return Expressions.always_false() - elif self.op in [Operation.GT, Operation.GT_EQ, Operation.EQ]: - return Expressions.always_true() - - return BoundPredicate(self.op, bound_term, lit=lit, is_literal_predicate=True) - - @staticmethod - def floating_type(type_id: TypeID) -> bool: - return type_id in [TypeID.FLOAT, TypeID.DOUBLE] diff --git a/iceberg/api/expressions/projections.py b/iceberg/api/expressions/projections.py deleted file mode 100644 index 7521b42b65..0000000000 --- a/iceberg/api/expressions/projections.py +++ /dev/null @@ -1,114 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - - -from .expressions import Expressions, ExpressionVisitors, RewriteNot -from .predicate import BoundPredicate, UnboundPredicate - - -def inclusive(spec, case_sensitive=True): - return InclusiveProjection(spec, case_sensitive) - - -def strict(spec): - return StrictProjection(spec) - - -class ProjectionEvaluator(ExpressionVisitors.ExpressionVisitor): - - def project(self, expr): - raise NotImplementedError() - - -class BaseProjectionEvaluator(ProjectionEvaluator): - - def __init__(self, spec, case_sensitive=True): - self.spec = spec - self.case_sensitive = case_sensitive - - def project(self, expr): - # projections assume that there are no NOT nodes in the expression tree. to ensure that this - # is the case, the expression is rewritten to push all NOT nodes down to the expression - # leaf nodes. - # this is necessary to ensure that the default expression returned when a predicate can't be - # projected is correct. - # - return ExpressionVisitors.visit(ExpressionVisitors.visit(expr, RewriteNot.get()), self) - - def always_true(self): - return Expressions.always_true() - - def always_false(self): - return Expressions.always_false() - - def not_(self, result): - raise RuntimeError("[BUG] project called on expression with a not") - - def and_(self, left_result, right_result): - return Expressions.and_(left_result, right_result) - - def or_(self, left_result, right_result): - return Expressions.or_(left_result, right_result) - - def predicate(self, pred): - bound = pred.bind(self.spec.schema.as_struct(), case_sensitive=self.case_sensitive) - - if isinstance(bound, BoundPredicate): - return self.predicate(bound) - - return bound - - -class InclusiveProjection(BaseProjectionEvaluator): - - def __init__(self, spec, case_sensitive=True): - super(InclusiveProjection, self).__init__(spec, - case_sensitive=case_sensitive) - - def predicate(self, pred): - if isinstance(pred, UnboundPredicate): - return super(InclusiveProjection, self).predicate(pred) - - part = self.spec.get_field_by_source_id(pred.ref.field.field_id) - - if part is None: - return self.always_true() - - result = part.transform.project(part.name, pred) - if result is not None: - return result - - return self.always_true() - - -class StrictProjection(BaseProjectionEvaluator): - - def __init__(self, spec): - super(StrictProjection, self).__init__(spec) - - def predicate(self, pred): - part = self.spec.get_field_by_source_id(pred.ref.field.field_id) - - if part is None: - return self.always_false() - - result = part.transform.project_strict(part.name, pred) - - if result is not None: - return result - - return self.always_false() diff --git a/iceberg/api/expressions/reference.py b/iceberg/api/expressions/reference.py deleted file mode 100644 index 9637607e56..0000000000 --- a/iceberg/api/expressions/reference.py +++ /dev/null @@ -1,114 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -from __future__ import annotations - -from typing import Any, TYPE_CHECKING - -from iceberg.exceptions import ValidationException - -from ..types import StructType - -if TYPE_CHECKING: - from iceberg.api import StructLike - - -class BoundReference: - - def __init__(self, struct, field): - self.field = field - self.pos = self.find(field.field_id, struct) - self._type = struct.fields[self.pos].type - - @property - def type(self): - return self._type - - def __eq__(self, other): - if id(self) == id(other): - return True - elif other is None or not isinstance(other, BoundReference): - return False - - return self.field.field_id == other.field.field_id and self.pos == other.pos and self._type == other._type - - def __ne__(self, other): - return not self.__eq__(other) - - def find(self, field_id, struct): - fields = struct.fields - for i, field in enumerate(fields): - if field.field_id == self.field.field_id: - return i - - raise ValidationException("Cannot find top-level field id %d in struct: %s", (field_id, struct)) - - def get(self, struct): - return struct.get(self.pos) - - def __str__(self): - return "ref(id={id}, pos={pos}, type={_type})".format(id=self.field.field_id, - pos=self.pos, - _type=self._type) - - @property - def ref(self): - return self - - def eval(self, struct: StructLike) -> Any: - return self.get(struct) - - -class NamedReference: - - def __init__(self, name): - super(NamedReference, self).__init__() - if name is None: - raise RuntimeError("Name cannot be null") - - self.name = name - - @property - def ref(self): - return self - - def bind(self, struct: StructType, case_sensitive: bool = True) -> BoundReference: - from iceberg.api import Schema - schema = Schema(struct.fields) - field = schema.find_field(self.name) if case_sensitive else schema.case_insensitive_find_field(self.name) - - ValidationException.check(field is not None, "Cannot find field '%s' in struct: %s", (self.name, - schema.as_struct())) - - return BoundReference(struct, field) - - def __eq__(self, other): - if id(self) == id(other): - return True - elif other is None or not isinstance(other, NamedReference): - return False - - return self.name == other.name - - def __ne__(self, other): - return not self.__eq__(other) - - def __repr__(self): - return "NamedReference({})".format(self.name) - - def __str__(self): - return 'ref(name="{}")'.format(self.name) diff --git a/iceberg/api/expressions/residual_evaluator.py b/iceberg/api/expressions/residual_evaluator.py deleted file mode 100644 index 285abae051..0000000000 --- a/iceberg/api/expressions/residual_evaluator.py +++ /dev/null @@ -1,117 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -from .expressions import Expressions, ExpressionVisitors -from .predicate import BoundPredicate, Predicate, UnboundPredicate - - -class ResidualEvaluator(object): - - def __init__(self, spec, expr): - self._spec = spec - self._expr = expr - self.__visitor = None - - def _visitor(self): - if self.__visitor is None: - self.__visitor = ResidualVisitor() - - return self.__visitor - - def residual_for(self, partition_data): - return self._visitor().eval(partition_data) - - -class ResidualVisitor(ExpressionVisitors.BoundExpressionVisitor): - - def __init__(self): - self.struct = None - - def eval(self, struct): - self.struct = struct - - def always_true(self): - return Expressions.always_true() - - def always_false(self): - return Expressions.always_false() - - def is_null(self, ref): - return self.always_true() if ref.get(self.struct) is None else self.always_false() - - def not_null(self, ref): - return self.always_true() if ref.get(self.struct) is not None else self.always_false() - - def lt(self, ref, lit): - return self.always_true() if ref.get(self.struct) < lit.value else self.always_false() - - def lt_eq(self, ref, lit): - return self.always_true() if ref.get(self.struct) <= lit.value else self.always_false() - - def gt(self, ref, lit): - return self.always_true() if ref.get(self.struct) > lit.value else self.always_false() - - def gt_eq(self, ref, lit): - return self.always_true() if ref.get(self.struct) >= lit.value else self.always_false() - - def eq(self, ref, lit): - return self.always_true() if ref.get(self.struct) == lit.value else self.always_false() - - def not_eq(self, ref, lit): - return self.always_true() if ref.get(self.struct) != lit.value else self.always_false() - - def not_(self, result): - return Expressions.not_(result) - - def and_(self, left_result, right_result): - return Expressions.and_(left_result, right_result) - - def or_(self, left_result, right_result): - return Expressions.or_(left_result, right_result) - - def predicate(self, pred): - if isinstance(pred, BoundPredicate): - return self.bound_predicate(pred) - elif isinstance(pred, UnboundPredicate): - return self.unbound_predicate(pred) - - raise RuntimeError("Invalid predicate argument %s" % pred) - - def bound_predicate(self, pred): - part = self.spec.get_field_by_source_id(pred.ref.field_id) - if part is None: - return pred - - strict_projection = part.transform.project_strict(part.name, pred) - if strict_projection is None: - bound = strict_projection.bind(self.spec.partition_type()) - if isinstance(bound, BoundPredicate): - return super(ResidualVisitor, self).predicate(bound) - return bound - - return pred - - def unbound_predicate(self, pred): - bound = pred.bind(self.spec.schema.as_struct()) - - if isinstance(bound, BoundPredicate): - bound_residual = self.predicate(bound) - if isinstance(bound_residual, Predicate): - return pred - return bound_residual - - return bound diff --git a/iceberg/api/expressions/strict_metrics_evaluator.py b/iceberg/api/expressions/strict_metrics_evaluator.py deleted file mode 100644 index 16b7728eed..0000000000 --- a/iceberg/api/expressions/strict_metrics_evaluator.py +++ /dev/null @@ -1,221 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -import threading - -from .expressions import Expressions, ExpressionVisitors -from ..expressions.binder import Binder -from ..types import Conversions - - -class StrictMetricsEvaluator(object): - - def __init__(self, schema, unbound): - self.schema = schema - self.struct = schema.as_struct() - self.expr = Binder.bind(self.struct, Expressions.rewrite_not(unbound)) - self.thread_local_data = threading.local() - - def _visitor(self): - if not hasattr(self.thread_local_data, "visitors"): - self.thread_local_data.visitors = StrictMetricsEvaluator.MetricsEvalVisitor( - self.expr, - self.schema, - self.struct - ) - - return self.thread_local_data.visitors - - def eval(self, file): - return self._visitor().eval(file) - - class MetricsEvalVisitor(ExpressionVisitors.BoundExpressionVisitor): - ROWS_MUST_MATCH = True - ROWS_MIGHT_NOT_MATCH = False - - def __init__(self, expr, schema, struct): - self.expr = expr - self.schema = schema - self.struct = struct - self.value_counts = None - self.null_counts = None - self.lower_bounds = None - self.upper_bounds = None - - def eval(self, file): - if file.record_count() <= 0: - return StrictMetricsEvaluator.MetricsEvalVisitor.ROWS_MUST_MATCH - - self.value_counts = file.value_counts() - self.null_counts = file.null_value_counts() - self.lower_bounds = file.lower_bounds() - self.upper_bounds = file.upper_bounds() - - return ExpressionVisitors.visit(self.expr, self) - - def always_true(self): - return StrictMetricsEvaluator.MetricsEvalVisitor.ROWS_MUST_MATCH - - def always_false(self): - return StrictMetricsEvaluator.MetricsEvalVisitor.ROWS_MIGHT_NOT_MATCH - - def not_(self, result): - return not result - - def and_(self, left_result, right_result): - return left_result and right_result - - def or_(self, left_result, right_result): - return left_result or right_result - - def is_null(self, ref): - id = ref.field.field_id - if self.struct.field(id=id) is None: - raise RuntimeError("Cannot filter by nested column: %s" % self.schema.find_field(id)) - - if self.value_counts is not None and self.value_counts.get(id) is not None \ - and self.null_counts is not None and self.null_counts.get(id) is not None \ - and self.value_counts.get(id) - self.null_counts.get(id) == 0: - return StrictMetricsEvaluator.MetricsEvalVisitor.ROWS_MUST_MATCH - - return StrictMetricsEvaluator.MetricsEvalVisitor.ROWS_MIGHT_NOT_MATCH - - def not_null(self, ref): - id = ref.field.field_id - if self.struct.field(id=id) is None: - raise RuntimeError("Cannot filter by nested column: %s" % self.schema.find_field(id)) - - if self.null_counts is not None and self.null_counts.get(id, -1) == 0: - return StrictMetricsEvaluator.MetricsEvalVisitor.ROWS_MUST_MATCH - - return StrictMetricsEvaluator.MetricsEvalVisitor.ROWS_MIGHT_NOT_MATCH - - def lt(self, ref, lit): - # Rows must match when: <----------Min----Max---X-------> - id = ref.field.field_id - - field = self.struct.field(id=id) - - if field is None: - raise RuntimeError("Cannot filter by nested column: %s" % self.schema.find_field(id)) - - if self.upper_bounds is not None and id in self.upper_bounds: - upper = Conversions.from_byte_buffer(field.type, self.upper_bounds.get(id)) - if upper < lit.value: - return StrictMetricsEvaluator.MetricsEvalVisitor.ROWS_MUST_MATCH - - return StrictMetricsEvaluator.MetricsEvalVisitor.ROWS_MIGHT_NOT_MATCH - - def lt_eq(self, ref, lit): - # Rows must match when: <----------Min----Max---X-------> - id = ref.field.field_id - - field = self.struct.field(id=id) - - if field is None: - raise RuntimeError("Cannot filter by nested column: %s" % self.schema.find_field(id)) - - if self.upper_bounds is not None and id in self.upper_bounds: - upper = Conversions.from_byte_buffer(field.type, self.upper_bounds.get(id)) - if upper <= lit.value: - return StrictMetricsEvaluator.MetricsEvalVisitor.ROWS_MUST_MATCH - - return StrictMetricsEvaluator.MetricsEvalVisitor.ROWS_MIGHT_NOT_MATCH - - def gt(self, ref, lit): - # Rows must match when: <-------X---Min----Max----------> - id = ref.field.field_id - - field = self.struct.field(id=id) - - if field is None: - raise RuntimeError("Cannot filter by nested column: %s" % self.schema.find_field(id)) - - if self.lower_bounds is not None and id in self.lower_bounds: - lower = Conversions.from_byte_buffer(field.type, self.lower_bounds.get(id)) - if lower > lit.value: - return StrictMetricsEvaluator.MetricsEvalVisitor.ROWS_MUST_MATCH - - return StrictMetricsEvaluator.MetricsEvalVisitor.ROWS_MIGHT_NOT_MATCH - - def gt_eq(self, ref, lit): - # Rows must match when: <-------X---Min----Max----------> - id = ref.field.field_id - - field = self.struct.field(id=id) - - if field is None: - raise RuntimeError("Cannot filter by nested column: %s" % self.schema.find_field(id)) - - if self.lower_bounds is not None and id in self.lower_bounds: - lower = Conversions.from_byte_buffer(field.type, self.lower_bounds.get(id)) - if lower >= lit.value: - return StrictMetricsEvaluator.MetricsEvalVisitor.ROWS_MUST_MATCH - - return StrictMetricsEvaluator.MetricsEvalVisitor.ROWS_MIGHT_NOT_MATCH - - def eq(self, ref, lit): - # Rows must match when Min == X == Max - id = ref.field.field_id - - field = self.struct.field(id=id) - - if field is None: - raise RuntimeError("Cannot filter by nested column: %s" % self.schema.find_field(id)) - - if self.lower_bounds is not None and id in self.lower_bounds \ - and self.upper_bounds is not None and id in self.upper_bounds: - lower = Conversions.from_byte_buffer(field.type, self.lower_bounds.get(id)) - if lower != lit.value: - return StrictMetricsEvaluator.MetricsEvalVisitor.ROWS_MIGHT_NOT_MATCH - - upper = Conversions.from_byte_buffer(field.type, self.upper_bounds.get(id)) - - if upper != lit.value: - return StrictMetricsEvaluator.MetricsEvalVisitor.ROWS_MIGHT_NOT_MATCH - - return StrictMetricsEvaluator.MetricsEvalVisitor.ROWS_MUST_MATCH - - return StrictMetricsEvaluator.MetricsEvalVisitor.ROWS_MIGHT_NOT_MATCH - - def not_eq(self, ref, lit): - # Rows must match when X < Min or Max < X because it is not in the range - id = ref.field.field_id - - field = self.struct.field(id=id) - - if field is None: - raise RuntimeError("Cannot filter by nested column: %s" % self.schema.find_field(id)) - - if self.lower_bounds is not None and id in self.lower_bounds: - lower = Conversions.from_byte_buffer(field.type, self.lower_bounds.get(id)) - if lower > lit.value: - return StrictMetricsEvaluator.MetricsEvalVisitor.ROWS_MUST_MATCH - - if self.upper_bounds is not None and id in self.upper_bounds: - upper = Conversions.from_byte_buffer(field.type, self.upper_bounds.get(id)) - - if upper < lit.value: - return StrictMetricsEvaluator.MetricsEvalVisitor.ROWS_MUST_MATCH - - return StrictMetricsEvaluator.MetricsEvalVisitor.ROWS_MIGHT_NOT_MATCH - - def in_(self, ref, lit): - return StrictMetricsEvaluator.MetricsEvalVisitor.ROWS_MIGHT_NOT_MATCH - - def not_in(self, ref, lit): - return StrictMetricsEvaluator.MetricsEvalVisitor.ROWS_MIGHT_NOT_MATCH diff --git a/iceberg/api/expressions/term.py b/iceberg/api/expressions/term.py deleted file mode 100644 index e135783a08..0000000000 --- a/iceberg/api/expressions/term.py +++ /dev/null @@ -1,73 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -from __future__ import annotations - -from typing import Any, TYPE_CHECKING - -from iceberg.api.types import StructType - -if TYPE_CHECKING: - from iceberg.api import StructLike - - -class Bound(object): - def eval(self, struct: StructLike) -> Any: - """Evaluate current object against struct""" - - -class Unbound(object): - @property - def ref(self): - """the underlying reference""" - - def bind(self, struct: StructType, case_sensitive: bool = True) -> Bound: - """bind the current object based on the given struct and return the Bound object""" - - -class Term(object): - @property - def ref(self): - """Expose the reference for this term""" - - @property - def type(self): - """accessor for term type """ - - -class BoundTerm(Bound, Term): - - @property - def ref(self): - raise NotImplementedError("Base class does not have implementation") - - @property - def type(self): - raise NotImplementedError("Base class does not have implementation") - - def eval(self, struct: StructLike): - raise NotImplementedError("Base class does not have implementation") - - -class UnboundTerm(Unbound, Term): - - @property - def ref(self): - raise NotImplementedError("Base class does not have implementation") - - def bind(self, struct: StructType, case_sensitive: bool = True): - raise NotImplementedError("Base class does not have implementation") diff --git a/iceberg/api/expressions/transform.py b/iceberg/api/expressions/transform.py deleted file mode 100644 index cf2e4563f0..0000000000 --- a/iceberg/api/expressions/transform.py +++ /dev/null @@ -1,75 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. -from .reference import BoundReference, NamedReference -from .. import StructLike -from ..transforms import Transform, Transforms -from ..types import StructType -from ...exceptions import ValidationException - - -class BoundTransform: - - def __init__(self, ref: BoundReference, transform: Transform): - self._ref = ref - self._transform = transform - - @property - def ref(self): - return self._ref - - @property - def type(self): - return self.transform.get_result_type(self.ref.type) - - @property - def transform(self): - return self._transform - - def eval(self, struct: StructLike): - return self.transform.apply(self.ref.eval(struct)) - - def __str__(self): - return f"{self.transform}({self.ref})" - - -class UnboundTransform: - - def __init__(self, ref: NamedReference, transform: Transform): - self._ref = ref - self._transform = transform - - @property - def ref(self): - return self._ref - - @property - def transform(self): - return self._transform - - def bind(self, struct: StructType, case_sensitive: bool = True): - bound_ref = self.ref.bind(struct, case_sensitive=case_sensitive) - - type_transform = Transforms.from_string(bound_ref.type, str(self.transform)) - ValidationException.check(type_transform.can_transform(bound_ref.type), - "Cannot bind: %s cannot transform %s values from '%s'", (self.transform, - bound_ref.type, - self.ref.name)) - - return BoundTransform(bound_ref, type_transform) - - def __str__(self): - return f"{self.transform}({self.ref})" diff --git a/iceberg/api/file_format.py b/iceberg/api/file_format.py deleted file mode 100644 index a1e8998b66..0000000000 --- a/iceberg/api/file_format.py +++ /dev/null @@ -1,44 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -from enum import Enum, unique - - -@unique -class FileFormat(Enum): - ORC = {"extension": "orc", "splittable": True} - PARQUET = {"extension": "parquet", "splittable": True} - AVRO = {"extension": "avro", "splittable": True} - - def add_extension(self, filename): - if filename.endswith(self.value["extension"]): - return filename - else: - return filename + "." + self.value["extension"] - - def is_splittable(self): - return self.value["splittable"] - - @staticmethod - def from_file_name(filename): - last_index_of = filename.rfind('.') - if last_index_of < 0: - return None - ext = filename[last_index_of + 1:] - for fmt in FileFormat: - if ext == fmt.value["extension"]: - return fmt diff --git a/iceberg/api/file_scan_task.py b/iceberg/api/file_scan_task.py deleted file mode 100644 index f51103b38e..0000000000 --- a/iceberg/api/file_scan_task.py +++ /dev/null @@ -1,47 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -from .scan_task import ScanTask - - -class FileScanTask(ScanTask): - - @property - def file(self): - raise NotImplementedError() - - @property - def spec(self): - raise NotImplementedError() - - @property - def start(self): - raise NotImplementedError() - - @property - def length(self): - raise NotImplementedError() - - @property - def residual(self): - raise NotImplementedError() - - def is_file_scan_task(self): - return True - - def as_file_scan_task(self): - return self diff --git a/iceberg/api/files.py b/iceberg/api/files.py deleted file mode 100644 index 7c58032e8e..0000000000 --- a/iceberg/api/files.py +++ /dev/null @@ -1,126 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -import gzip -import os - -from iceberg.exceptions import AlreadyExistsException - -from .expressions import JAVA_MAX_INT -from .io import (InputFile, - OutputFile, - PositionOutputStream, - SeekableInputStream) - - -class Files(object): - - @staticmethod - def local_output(file): - return LocalOutputFile(file) - - @staticmethod - def local_input(file): - return LocalInputFile(file) - - -class LocalOutputFile(OutputFile): - - def __init__(self, file): - self.file = file - - def create(self): - if os.path.exists(self.file): - raise AlreadyExistsException("File already exists: %s" % self.file) - - return PositionOutputStream(open(self.file, "w")) - - def create_or_overwrite(self): - if os.path.exists(self.file): - if not os.remove(self.file.name): - raise RuntimeError("File deleting file: %s" % self.file) - - return self.create() - - def location(self): - return self.file - - def __str__(self): - return self.location() - - -class LocalInputFile(InputFile): - - def __init__(self, file): - self.file = file - - def get_length(self): - return os.path.getsize(self.file) - - def new_stream(self, gzipped=False): - with open(self.file, "rb") as fo: - if gzipped: - fo = gzip.GzipFile(fileobj=fo) - return fo - - def location(self): - return self.file - - def __str__(self): - return self.location() - - -class SeekableFileInputStream(SeekableInputStream): - - def __init__(self, stream): - self.stream = stream - - def get_pos(self): - return self.stream.tell() - - def seek(self, new_pos): - return self.stream.seek(new_pos) - - def read(self, b=None, off=None, read_len=None): - if b is None and off is None and read_len is None: - return None, self.stream.read() - if b is None: - raise RuntimeError("supplied byte field is None") - - if off is None and read_len is None: - new_b = self.stream.read(b.length) - return len(new_b), new_b - - if off is not None and read_len is None or off is None and read_len is not None: - raise RuntimeError("Invalid args: read_len(%s), off(%s)" % (read_len, off)) - - if read_len < 0 or off < 0 or (len(b) - off < read_len): - raise RuntimeError("Invalid args: read_len(%s), off(%s), len_b_offset(%s)" % (read_len, off, len(b) - off)) - - new_b = bytes(self.stream.read(read_len), "utf8") - - if off > 0: - new_b = b[0:off] + new_b - if off + read_len < len(b): - new_b = new_b + b[off + read_len:] - return read_len, new_b - - def skip(self, n): - if n > JAVA_MAX_INT: - return self.stream.seek(JAVA_MAX_INT) - else: - return self.stream.seek(n) diff --git a/iceberg/api/filterable.py b/iceberg/api/filterable.py deleted file mode 100644 index beb8d48767..0000000000 --- a/iceberg/api/filterable.py +++ /dev/null @@ -1,32 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - - -class Filterable(object): - ALL_COLUMNS = ("*",) - - def select(self, columns): - raise NotImplementedError() - - def filter_partitions(self, expr): - raise NotImplementedError() - - def filter_rows(self, expr): - raise NotImplementedError() - - def iterator(self): - raise NotImplementedError() diff --git a/iceberg/api/filtered_snapshot.py b/iceberg/api/filtered_snapshot.py deleted file mode 100644 index 055480a283..0000000000 --- a/iceberg/api/filtered_snapshot.py +++ /dev/null @@ -1,40 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -from .expressions import Expressions -from .filterable import Filterable - - -class FilteredSnapshot(Filterable): - - def __init__(self, snapshot, part_filter, row_filter, columns): - self.snapshot = snapshot - self.part_filter = part_filter - self.row_filter = row_filter - self.columns = columns - - def select(self, columns): - return FilteredSnapshot(self.snapshot, self.part_filter, self.row_filter, columns) - - def filter_partitions(self, expr): - return FilteredSnapshot(self.snapshot, Expressions.and_(self.part_filter, expr), self.row_filter, self.columns) - - def filter_rows(self, expr): - return FilteredSnapshot(self.snapshot, self.part_filter, Expressions.and_(self.row_filter, expr), self.columns) - - def iterator(self): - return self.snapshot.iterator(self.part_filter, self.row_filter, self.columns) diff --git a/iceberg/api/io/__init__.py b/iceberg/api/io/__init__.py deleted file mode 100644 index 0fdac354b9..0000000000 --- a/iceberg/api/io/__init__.py +++ /dev/null @@ -1,30 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -__all__ = ["CloseableGroup", - "FileAppender", - "InputFile", - "OutputFile", - "PositionOutputStream", - "SeekableInputStream"] - -from .closeable_group import CloseableGroup -from .file_appender import FileAppender -from .input_file import InputFile -from .output_file import OutputFile -from .position_output_stream import PositionOutputStream -from .seekable_input_stream import SeekableInputStream diff --git a/iceberg/api/io/closeable_group.py b/iceberg/api/io/closeable_group.py deleted file mode 100644 index 4abe1b2fd1..0000000000 --- a/iceberg/api/io/closeable_group.py +++ /dev/null @@ -1,31 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - - -class CloseableGroup(object): - - def __init__(self): - self.closeables = list() - - def add_closeable(self, closeable): - self.closeables.append(closeable) - - def close(self): - while self.closeables: - to_close = self.closeables.pop(0) - if to_close is not None: - to_close.close() diff --git a/iceberg/api/io/closeable_iterable.py b/iceberg/api/io/closeable_iterable.py deleted file mode 100644 index 2ea80effa1..0000000000 --- a/iceberg/api/io/closeable_iterable.py +++ /dev/null @@ -1,27 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -import collections - - -class CloseableIterable(collections.Iterator): - - def __next__(self): - raise NotImplementedError() - - def close(self): - raise NotImplementedError() diff --git a/iceberg/api/io/delegating_input_stream.py b/iceberg/api/io/delegating_input_stream.py deleted file mode 100644 index 40ebe34dc2..0000000000 --- a/iceberg/api/io/delegating_input_stream.py +++ /dev/null @@ -1,22 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - - -class DelegatingInputStream(object): - - def get_delegate(self): - raise NotImplementedError() diff --git a/iceberg/api/io/delegating_output_stream.py b/iceberg/api/io/delegating_output_stream.py deleted file mode 100644 index 84064d0f3b..0000000000 --- a/iceberg/api/io/delegating_output_stream.py +++ /dev/null @@ -1,22 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - - -class DelegatingOutputStream(object): - - def get_delegate(self): - raise NotImplementedError() diff --git a/iceberg/api/io/file_appender.py b/iceberg/api/io/file_appender.py deleted file mode 100644 index fabb05d068..0000000000 --- a/iceberg/api/io/file_appender.py +++ /dev/null @@ -1,29 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - - -class FileAppender(object): - - def add(self, d): - raise NotImplementedError() - - def add_all(self, values): - for value in values: - self.add(value) - - def metrics(self): - raise NotImplementedError() diff --git a/iceberg/api/io/input_file.py b/iceberg/api/io/input_file.py deleted file mode 100644 index 08b68ad0a2..0000000000 --- a/iceberg/api/io/input_file.py +++ /dev/null @@ -1,38 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -from typing import TYPE_CHECKING - -if TYPE_CHECKING: - from iceberg.core.filesystem import FileSystem - - -class InputFile(object): - fs: 'FileSystem' - path: str - - def get_length(self): - raise NotImplementedError() - - def new_stream(self): - raise NotImplementedError() - - def location(self): - raise NotImplementedError() - - def new_fo(self): - raise NotImplementedError() diff --git a/iceberg/api/io/output_file.py b/iceberg/api/io/output_file.py deleted file mode 100644 index f091459b82..0000000000 --- a/iceberg/api/io/output_file.py +++ /dev/null @@ -1,28 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - - -class OutputFile(object): - - def create(self): - raise NotImplementedError() - - def create_or_overwrite(self): - raise NotImplementedError() - - def location(self): - raise NotImplementedError() diff --git a/iceberg/api/io/position_output_stream.py b/iceberg/api/io/position_output_stream.py deleted file mode 100644 index fb1e758043..0000000000 --- a/iceberg/api/io/position_output_stream.py +++ /dev/null @@ -1,32 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - - -class PositionOutputStream(object): - - # OutputStream methods - def close(self): - raise NotImplementedError() - - def flush(self): - raise NotImplementedError() - - def write(self, b, off=None, len=None): - raise NotImplementedError() - - def get_pos(self): - raise NotImplementedError() diff --git a/iceberg/api/io/seekable_input_stream.py b/iceberg/api/io/seekable_input_stream.py deleted file mode 100644 index 60808a8e38..0000000000 --- a/iceberg/api/io/seekable_input_stream.py +++ /dev/null @@ -1,46 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - - -class SeekableInputStream(object): - - def available(self): - raise NotImplementedError() - - def close(self): - raise NotImplementedError() - - def mark(self, read_limit): - raise NotImplementedError() - - def mark_supported(self): - raise NotImplementedError() - - def read(self, b=None, off=None, len=None): - raise NotImplementedError() - - def reset(self): - raise NotImplementedError() - - def skip(self, n): - raise NotImplementedError() - - def get_pos(self): - raise NotImplementedError() - - def seek(self, new_pos): - raise NotImplementedError() diff --git a/iceberg/api/manifest_file.py b/iceberg/api/manifest_file.py deleted file mode 100644 index a4a0274da1..0000000000 --- a/iceberg/api/manifest_file.py +++ /dev/null @@ -1,95 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -from .schema import Schema -from .types import (BinaryType, - BooleanType, - IntegerType, - ListType, - LongType, - NestedField, - StringType, - StructType) - - -class ManifestFile(object): - SCHEMA = Schema(NestedField.required(500, "manifest_path", StringType.get()), - NestedField.required(501, "manifest_length", LongType.get()), - NestedField.required(502, "partition_spec_id", IntegerType.get()), - NestedField.optional(503, "added_snapshot_id", LongType.get()), - NestedField.optional(504, "added_data_files_count", IntegerType.get()), - NestedField.optional(505, "existing_data_files_count", IntegerType.get()), - NestedField.optional(506, "deleted_data_files_count", IntegerType.get()), - NestedField - .optional(507, "partitions", - ListType.of_required(508, StructType.of([NestedField.required(509, - "contains_null", - BooleanType.get()), - NestedField.optional(510, - "lower_bound", - BinaryType.get()), - NestedField.optional(511, - "upper_bound", - BinaryType.get())])))) - - @staticmethod - def schema(): - return ManifestFile.SCHEMA - - @property - def added_files_count(self): - raise NotImplementedError() - - @property - def existing_files_count(self): - raise NotImplementedError() - - @property - def deleted_files_count(self): - raise NotImplementedError() - - def copy(self): - raise NotImplementedError() - - def has_added_files(self): - return self.added_files_count is None or self.added_files_count > 0 - - def has_existing_files(self): - return self.existing_files_count is None or self.existing_files_count > 0 - - def has_deleted_files(self): - return self.deleted_files_count is None or self.deleted_files_count > 0 - - -class PartitionFieldSummary(object): - TYPE = ManifestFile.schema().find_type("partitions").as_list_type().element_type.as_struct_type() - - @staticmethod - def get_type(): - return PartitionFieldSummary.TYPE - - def contains_null(self): - raise NotImplementedError() - - def lower_bound(self): - raise NotImplementedError() - - def upper_bound(self): - raise NotImplementedError() - - def copy(self): - raise NotImplementedError() diff --git a/iceberg/api/metrics.py b/iceberg/api/metrics.py deleted file mode 100644 index 231388a256..0000000000 --- a/iceberg/api/metrics.py +++ /dev/null @@ -1,30 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - - -class Metrics(object): - - def __init__(self, row_count=None, - column_sizes=None, - value_counts=None, null_value_counts=None, - lower_bounds=None, upper_bounds=None): - self.row_count = row_count - self.column_sizes = column_sizes - self.value_counts = value_counts - self.null_value_counts = null_value_counts - self.lower_bounds = lower_bounds - self.upper_bounds = upper_bounds diff --git a/iceberg/api/overwrite_files.py b/iceberg/api/overwrite_files.py deleted file mode 100644 index 3bcf0cec88..0000000000 --- a/iceberg/api/overwrite_files.py +++ /dev/null @@ -1,30 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -from .pending_update import PendingUpdate - - -class OverwriteFiles(PendingUpdate): - - def overwrite_by_row_filter(self, expr): - raise NotImplementedError() - - def add_file(self, file): - raise NotImplementedError() - - def validate_added_files(self): - raise NotImplementedError() diff --git a/iceberg/api/partition_field.py b/iceberg/api/partition_field.py deleted file mode 100644 index 8e789a1089..0000000000 --- a/iceberg/api/partition_field.py +++ /dev/null @@ -1,43 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - - -class PartitionField(object): - - def __init__(self, source_id, field_id, name, transform): - self.source_id = source_id - self.field_id = field_id - self.name = name - self.transform = transform - - def __eq__(self, other): - if id(self) == id(other): - return True - elif other is None or not isinstance(other, PartitionField): - return False - - return self.source_id == other.source_id and self.field_id == other.field_id and \ - self.name == other.name and self.transform == other.transform - - def __ne__(self, other): - return not self.__eq__(other) - - def __hash__(self): - return hash(self.__key()) - - def __key(self): - return PartitionField.__class__, self.source_id, self.field_id, self.name, self.transform diff --git a/iceberg/api/partition_spec.py b/iceberg/api/partition_spec.py deleted file mode 100644 index e2f106d76b..0000000000 --- a/iceberg/api/partition_spec.py +++ /dev/null @@ -1,352 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -from urllib.parse import quote_plus - -from iceberg.exceptions import ValidationException - -from .partition_field import PartitionField -from .schema import Schema -from .transforms import Transform, Transforms -from .types import (NestedField, - StructType) - - -class PartitionSpec(object): - - PARTITION_DATA_ID_START = 1000 - - @staticmethod - def UNPARTITIONED_SPEC(): - return PartitionSpec(Schema(), 0, [], PartitionSpec.PARTITION_DATA_ID_START - 1) - - @staticmethod - def unpartitioned(): - return PartitionSpec.UNPARTITIONED_SPEC() - - def __init__(self, schema, spec_id, fields, last_assigned_field_id): - self.fields_by_source_id = None - self.fields_by_name = None - self.__java_classes = None - self.field_list = None - - self.schema = schema - self.spec_id = spec_id - self.__fields = list() - for field in fields: - self.__fields.append(field) - self.last_assigned_field_id = last_assigned_field_id - - @property - def fields(self): - return self.lazy_field_list() - - @property - def java_classes(self): - if self.__java_classes is None: - self.__java_classes - for field in self.__fields: - source_type = self.schema.find_type(field.source_id) - result = field.transform().get_result_by_type(source_type) - self.__java_classes.append(result.type_id.java_class()) - - return self.__java_classes - - def get_field_by_source_id(self, field_id): - return self.lazy_fields_by_source_id().get(field_id) - - def partition_type(self): - struct_fields = list() - for _i, field in enumerate(self.__fields): - source_type = self.schema.find_type(field.source_id) - result_type = field.transform.get_result_type(source_type) - struct_fields.append(NestedField.optional(field.field_id, - field.name, - result_type)) - - return StructType.of(struct_fields) - - def get(self, data, pos, java_class): - data.get(pos, java_class) - - def escape(self, string): - return quote_plus(string, encoding="UTF-8") - - def partition_to_path(self, data): - sb = list() - java_classes = self.java_classes - for i, jclass in enumerate(java_classes): - field = self.__fields[i] - value_string = field.transform().to_human_string(self.get(data, i, jclass)) - - if i > 0: - sb.append("/") - sb.append(field.name) - sb.append("=") - sb.append(self.escape(value_string)) - - return "".join(sb) - - def compatible_with(self, other): - if self.__eq__(other): - return True - - if len(self.__fields) != len(other.__fields): - return False - - for i, field in enumerate(self.__fields): - that_field = other.__fields[i] - if field.source_id != that_field.source_id or str(field.transform) != str(that_field.transform): - return False - - return True - - def lazy_fields_by_source_id(self): - if self.fields_by_source_id is None: - self.fields_by_source_id = dict() - for field in self.fields: - self.fields_by_source_id[field.source_id] = field - - return self.fields_by_source_id - - def identity_source_ids(self): - source_ids = set() - fields = self.fields - for field in fields: - if "identity" == str(field.transform()): - source_ids.add(field) - - return source_ids - - def lazy_field_list(self): - if self.field_list is None: - self.field_list = list(self.__fields) - - return self.field_list - - def lazy_fields_by_source_name(self): - if self.fields_by_name is None: - self.fields_by_name = dict() - for field in self.__fields: - self.fields_by_name[field.name] = field - - return self.fields_by_name - - def __eq__(self, other): - if id(self) == id(other): - return True - - if other is None or not isinstance(other, PartitionSpec): - return False - - return self.__fields == other.__fields - - def __ne__(self, other): - return not self.__eq__(other) - - def __hash__(self): - return hash(self.__key()) - - def __key(self): - return PartitionSpec.__class__, tuple(self.fields) - - def __str__(self): - return self.__repr__() - - def __repr__(self): - sb = ["["] - - for field in self.__fields: - sb.append("\n {field_id}: {name}: {transform}({source_id})".format(field_id=field.field_id, - name=field.name, - transform=str(field.transform), - source_id=field.source_id)) - - if len(self.__fields) > 0: - sb.append("\n") - sb.append("]") - - return "".join(sb) - - @staticmethod - def builder_for(schema: Schema) -> "PartitionSpecBuilder": - return PartitionSpecBuilder(schema) - - @staticmethod - def check_compatibility(spec, schema): - for field in spec.fields: - src_type = schema.find_type(field.source_id) - if not src_type.is_primitive_type(): - raise ValidationException("Cannot partition by non-primitive source field: %s", src_type) - if not field.transform.can_transform(src_type): - raise ValidationException("Invalid source type %s for transform: %s", (src_type, field.transform)) - - -class PartitionSpecBuilder(object): - - def __init__(self, schema): - self.schema = schema - self.fields = list() - self.partition_names = set() - self.dedup_fields = dict() - self.spec_id = 0 - self.last_assigned_field_id = PartitionSpec.PARTITION_DATA_ID_START - 1 - - def __next_field_id(self): - self.last_assigned_field_id = self.last_assigned_field_id + 1 - return self.last_assigned_field_id - - def with_spec_id(self, spec_id): - self.spec_id = spec_id - return self - - def check_and_add_partition_name(self, name, source_column_id=None): - schema_field = self.schema.find_field(name) - if source_column_id is not None: - if schema_field is not None and schema_field.field_id != source_column_id: - raise ValueError("Cannot create identity partition sourced from different field in schema: %s" % name) - else: - if schema_field is not None: - raise ValueError("Cannot create partition from name that exists in schema: %s" % name) - - if name is None or name == "": - raise ValueError("Cannot use empty or null partition name: %s" % name) - if name in self.partition_names: - raise ValueError("Cannot use partition names more than once: %s" % name) - - self.partition_names.add(name) - return self - - def check_redundant_and_add_field(self, field_id: int, name: str, transform: Transform) -> None: - field = PartitionField(field_id, - self.__next_field_id(), - name, - transform) - dedup_key = (field.source_id, field.transform.dedup_name()) - partition_field = self.dedup_fields.get(dedup_key) - if partition_field is not None: - raise ValueError("Cannot add redundant partition: %s conflicts with %s" % (partition_field, field)) - self.dedup_fields[dedup_key] = field - self.fields.append(field) - - def find_source_column(self, source_name): - source_column = self.schema.find_field(source_name) - if source_column is None: - raise RuntimeError("Cannot find source column: %s" % source_name) - - return source_column - - def identity(self, source_name, target_name=None): - if target_name is None: - target_name = source_name - - source_column = self.find_source_column(source_name) - self.check_and_add_partition_name(target_name, source_column.field_id) - self.check_redundant_and_add_field(source_column.field_id, - target_name, - Transforms.identity(source_column.type)) - return self - - def year(self, source_name, target_name=None): - if target_name is None: - target_name = "{}_year".format(source_name) - - self.check_and_add_partition_name(target_name) - source_column = self.find_source_column(source_name) - self.check_redundant_and_add_field(source_column.field_id, - target_name, - Transforms.year(source_column.type)) - return self - - def month(self, source_name, target_name=None): - if target_name is None: - target_name = "{}_month".format(source_name) - - self.check_and_add_partition_name(target_name) - source_column = self.find_source_column(source_name) - self.check_redundant_and_add_field(source_column.field_id, - target_name, - Transforms.month(source_column.type)) - return self - - def day(self, source_name, target_name=None): - if target_name is None: - target_name = "{}_day".format(source_name) - - self.check_and_add_partition_name(target_name) - source_column = self.find_source_column(source_name) - self.check_redundant_and_add_field(source_column.field_id, - target_name, - Transforms.day(source_column.type)) - return self - - def hour(self, source_name, target_name=None): - if target_name is None: - target_name = "{}_hour".format(source_name) - - self.check_and_add_partition_name(target_name) - source_column = self.find_source_column(source_name) - self.check_redundant_and_add_field(source_column.field_id, - target_name, - Transforms.hour(source_column.type)) - return self - - def bucket(self, source_name, num_buckets, target_name=None): - if target_name is None: - target_name = "{}_bucket".format(source_name) - - self.check_and_add_partition_name(target_name) - source_column = self.find_source_column(source_name) - self.fields.append(PartitionField(source_column.field_id, - self.__next_field_id(), - target_name, - Transforms.bucket(source_column.type, num_buckets))) - return self - - def truncate(self, source_name, width, target_name=None): - if target_name is None: - target_name = "{}_truncate".format(source_name) - - self.check_and_add_partition_name(target_name) - source_column = self.find_source_column(source_name) - self.fields.append(PartitionField(source_column.field_id, - self.__next_field_id(), - target_name, - Transforms.truncate(source_column.type, width))) - return self - - def add_without_field_id(self, source_id, name, transform): - return self.add(source_id, self.__next_field_id(), name, transform) - - def add(self, source_id: int, field_id: int, name: str, transform: str) -> "PartitionSpecBuilder": - column = self.schema.find_field(source_id) - if column is None: - raise ValueError("Cannot find source column: %s" % source_id) - - transform_obj = Transforms.from_string(column.type, transform) - self.check_and_add_partition_name(name, source_id) - self.fields.append(PartitionField(source_id, - field_id, - name, - transform_obj)) - self.last_assigned_field_id = max(self.last_assigned_field_id, field_id) - return self - - def build(self): - spec = PartitionSpec(self.schema, self.spec_id, self.fields, self.last_assigned_field_id) - PartitionSpec.check_compatibility(spec, self.schema) - return spec diff --git a/iceberg/api/pending_update.py b/iceberg/api/pending_update.py deleted file mode 100644 index 6569895e90..0000000000 --- a/iceberg/api/pending_update.py +++ /dev/null @@ -1,25 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - - -class PendingUpdate(object): - - def apply(self): - raise NotImplementedError() - - def commit(self): - raise NotImplementedError() diff --git a/iceberg/api/replace_partitions.py b/iceberg/api/replace_partitions.py deleted file mode 100644 index ca7199daba..0000000000 --- a/iceberg/api/replace_partitions.py +++ /dev/null @@ -1,30 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -from .pending_update import PendingUpdate - - -class ReplacePartitions(PendingUpdate): - - def __init__(self): - raise NotImplementedError() - - def apply(self): - raise NotImplementedError() - - def commit(self): - raise NotImplementedError() diff --git a/iceberg/api/rewrite_files.py b/iceberg/api/rewrite_files.py deleted file mode 100644 index fd120985bc..0000000000 --- a/iceberg/api/rewrite_files.py +++ /dev/null @@ -1,30 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -from .pending_update import PendingUpdate - - -class RewriteFiles(PendingUpdate): - - def rewrite_files(self, files_to_delete=None, files_to_add=None): - raise NotImplementedError() - - def apply(self): - raise NotImplementedError() - - def commit(self): - raise NotImplementedError() diff --git a/iceberg/api/rollback.py b/iceberg/api/rollback.py deleted file mode 100644 index 5b19261238..0000000000 --- a/iceberg/api/rollback.py +++ /dev/null @@ -1,33 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -from .pending_update import PendingUpdate - - -class Rollback(PendingUpdate): - - def to_snapshot_id(self, snapshot_id): - raise NotImplementedError() - - def to_snapshot_at_time(self, timestamp_millis): - raise NotImplementedError() - - def apply(self): - raise NotImplementedError() - - def commit(self): - raise NotImplementedError() diff --git a/iceberg/api/scan_task.py b/iceberg/api/scan_task.py deleted file mode 100644 index 7b13560def..0000000000 --- a/iceberg/api/scan_task.py +++ /dev/null @@ -1,28 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - - -class ScanTask(object): - - def is_file_scan_task(self): - return False - - def as_file_scan_task(self): - raise RuntimeError("Not a FileScanTask: %s" % self) - - def as_combined_scan_task(self): - RuntimeError("Not a CombinedScanTask: %s" % self) diff --git a/iceberg/api/schema.py b/iceberg/api/schema.py deleted file mode 100644 index 50d99e73ab..0000000000 --- a/iceberg/api/schema.py +++ /dev/null @@ -1,164 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -from .types import StructType -from .types import type_util -""" - The schema of a data table. - -""" - - -class Schema(object): - NEWLINE = '\n' - ALL_COLUMNS = "*" - - def __init__(self, *argv): - aliases = None - if len(argv) == 1 and isinstance(argv[0], (list, tuple)): - columns = argv[0] - elif len(argv) == 2 and isinstance(argv[0], list) and isinstance(argv[1], dict): - columns = argv[0] - aliases = argv[1] - else: - columns = argv - - self.struct = StructType.of(columns) - self._alias_to_id = None - self._id_to_alias = None - if aliases is not None: - self._alias_to_id = dict(aliases) - self._id_to_alias = {v: k for k, v in self._alias_to_id.items()} - - self._id_to_field = None - self._name_to_id = type_util.index_by_name(self.struct) - self._id_to_name = {v: k for k, v in self._name_to_id.items()} - self._lowercase_name_to_id = {k.lower(): v for k, v in self._name_to_id.items()} - - def as_struct(self): - return self.struct - - def get_aliases(self): - return self._alias_to_id - - def lazy_id_to_field(self): - from .types import index_by_id - if self._id_to_field is None: - self._id_to_field = index_by_id(self.struct) # noqa - - return self._id_to_field - - def lazy_name_to_id(self): - from .types import index_by_name - if self._name_to_id is None: - self._name_to_id = index_by_name(self.struct) - self._id_to_name = {v: k for k, v in self._name_to_id.items()} - self._lowercase_name_to_id = {k.lower(): v for k, v in self._name_to_id.items()} - - return self._name_to_id - - def lazy_lowercase_name_to_id(self): - from .types import index_by_name - if self._lowercase_name_to_id is None: - self._name_to_id = index_by_name(self.struct) - self._id_to_name = {v: k for k, v in self._name_to_id.items()} - self._lowercase_name_to_id = {k.lower(): v for k, v in self._name_to_id.items()} - - return self._lowercase_name_to_id - - def columns(self): - return self.struct.fields - - def find_type(self, name): - if isinstance(name, int): - field = self.lazy_id_to_field().get(name) - if field: - return field.type - elif isinstance(name, str): - id = self.lazy_name_to_id().get(name) - if id: - return self.find_type(id) - else: - raise RuntimeError("Invalid Column (could not find): %s" % name) - - def find_field(self, id): - if isinstance(id, int): - return self.lazy_id_to_field().get(id) - - if not id: - raise ValueError("Invalid Column Name (empty)") - - id = self.lazy_name_to_id().get(id) - if id is not None: - return self.lazy_id_to_field().get(id) - - def case_insensitive_find_field(self, name): - if name is None: - raise ValueError("Invalid Column Name (empty)") - - id = self.lazy_lowercase_name_to_id().get(name.lower()) - if id is not None: - return self.lazy_id_to_field().get(id) - - return None - - def find_column_name(self, id): - if isinstance(id, int): - field = self.lazy_id_to_field().get(id) - - return None if field is None else field.name - - def alias_to_id(self, alias): - if self._alias_to_id: - return self._alias_to_id.get(alias) - - def id_to_alias(self, field_id): - if self._id_to_alias: - return self._id_to_alias.get(field_id) - - def select(self, cols): - return self._internal_select(True, cols) - - def case_insensitive_select(self, cols): - return self._internal_select(False, cols) - - def _internal_select(self, case_sensitive, cols): - from .types import select - - if Schema.ALL_COLUMNS in cols: - return self - - selected = set() - for name in cols: - if case_sensitive: - field_id = self.lazy_name_to_id().get(name) - else: - field_id = self.lazy_lowercase_name_to_id().get(name.lower()) - - if field_id is not None: - selected.add(field_id) - - return select(self, selected) - - def __len__(self): - return len(self.struct.fields) - - def __repr__(self): - return "Schema(%s)" % ",".join([str(field) for field in self.struct.fields]) - - def __str__(self): - return "table {\n%s\n}" % Schema.NEWLINE.join([" " + str(field) for field in self.struct.fields]) diff --git a/iceberg/api/snapshot.py b/iceberg/api/snapshot.py deleted file mode 100644 index 8a049e996e..0000000000 --- a/iceberg/api/snapshot.py +++ /dev/null @@ -1,53 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - - -class Snapshot(object): - - @property - def snapshot_id(self): - raise NotImplementedError() - - @property - def parent_id(self): - raise NotImplementedError() - - @property - def timestamp_millis(self): - raise NotImplementedError() - - @property - def manifests(self): - raise NotImplementedError() - - @property - def manifest_location(self): - raise NotImplementedError() - - @property - def summary(self): - raise NotImplementedError() - - @property - def operation(self): - raise NotImplementedError() - - def added_files(self): - raise NotImplementedError() - - def deleted_files(self): - raise NotImplementedError() diff --git a/iceberg/api/snapshot_iterable.py b/iceberg/api/snapshot_iterable.py deleted file mode 100644 index ef8a0fb0c7..0000000000 --- a/iceberg/api/snapshot_iterable.py +++ /dev/null @@ -1,21 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - - -class SnapshotIterable(object): - def iterator(self, part_filter, row_filter, columns): - raise RuntimeError("Interface Implementation") diff --git a/iceberg/api/struct_like.py b/iceberg/api/struct_like.py deleted file mode 100644 index e421c2eb9f..0000000000 --- a/iceberg/api/struct_like.py +++ /dev/null @@ -1,28 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - - -class StructLike(object): - - def __init__(self): - raise NotImplementedError() - - def get(self, pos): - raise NotImplementedError() - - def set(self, pos, value): - raise NotImplementedError() diff --git a/iceberg/api/table.py b/iceberg/api/table.py deleted file mode 100644 index 2152121a50..0000000000 --- a/iceberg/api/table.py +++ /dev/null @@ -1,76 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - - -class Table(object): - - def __init__(self): - raise NotImplementedError() - - def refresh(self): - raise NotImplementedError() - - def new_scan(self): - raise NotImplementedError() - - def schema(self): - raise NotImplementedError() - - def spec(self): - raise NotImplementedError() - - def properties(self): - raise NotImplementedError() - - def location(self): - raise NotImplementedError() - - def snapshots(self): - raise NotImplementedError() - - def update_schema(self): - raise NotImplementedError() - - def update_properties(self): - raise NotImplementedError() - - def new_append(self): - raise NotImplementedError() - - def new_fast_append(self): - return self.new_append() - - def new_rewrite(self): - raise NotImplementedError() - - def new_overwrite(self): - raise NotImplementedError() - - def new_replace_partitions(self): - raise NotImplementedError() - - def new_delete(self): - raise NotImplementedError() - - def expire_snapshots(self): - raise NotImplementedError() - - def rollback(self): - raise NotImplementedError() - - def new_transaction(self): - raise NotImplementedError() diff --git a/iceberg/api/table_scan.py b/iceberg/api/table_scan.py deleted file mode 100644 index d93d857a02..0000000000 --- a/iceberg/api/table_scan.py +++ /dev/null @@ -1,62 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - - -class TableScan(object): - - def __init__(self): - raise NotImplementedError() - - @property - def row_filter(self): - raise NotImplementedError() - - def use_snapshot(self, snapshot_id): - raise NotImplementedError() - - def as_of_time(self, timestamp_millis): - raise NotImplementedError() - - def project(self, schema): - raise NotImplementedError() - - def select(self, columns): - raise NotImplementedError() - - def select_except(self, columns): - raise NotImplementedError() - - def filter(self, expr): - raise NotImplementedError() - - def plan_files(self): - raise NotImplementedError() - - def plan_tasks(self): - raise NotImplementedError() - - def is_case_sensitive(self): - raise NotImplementedError() - - def options(self): - raise NotImplementedError() - - def to_arrow_table(self): - raise NotImplementedError() - - def to_pandas(self): - raise NotImplementedError() diff --git a/iceberg/api/tables.py b/iceberg/api/tables.py deleted file mode 100644 index 0210437dfc..0000000000 --- a/iceberg/api/tables.py +++ /dev/null @@ -1,34 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -from .partition_spec import PartitionSpec - - -class Tables(object): - - def create(self, schema, table_identifier, spec=None, properties=None, location=None): - raise NotImplementedError() - - def load(self, table_identifier): - raise NotImplementedError() - - @staticmethod - def default_args(spec=None, properties=None): - spec = spec if spec is not None else PartitionSpec.unpartitioned() - properties = properties if properties is not None else dict() - - return spec, properties diff --git a/iceberg/api/transaction.py b/iceberg/api/transaction.py deleted file mode 100644 index 1860c8af88..0000000000 --- a/iceberg/api/transaction.py +++ /dev/null @@ -1,55 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - - -class Transaction(object): - - def table(self): - raise NotImplementedError() - - def update_schema(self): - raise NotImplementedError() - - def update_properties(self): - raise NotImplementedError() - - def update_location(self): - raise NotImplementedError() - - def new_append(self): - raise NotImplementedError() - - def new_fast_append(self): - raise NotImplementedError() - - def new_rewrite(self): - raise NotImplementedError() - - def new_overwrite(self): - raise NotImplementedError() - - def new_replace_partitions(self): - raise NotImplementedError() - - def new_delete(self): - raise RuntimeError("Interface implementation") - - def expire_snapshots(self): - raise RuntimeError("Interface implementation") - - def commit_transaction(self): - raise RuntimeError("Interface implementation") diff --git a/iceberg/api/transforms/__init__.py b/iceberg/api/transforms/__init__.py deleted file mode 100644 index c0f29d778e..0000000000 --- a/iceberg/api/transforms/__init__.py +++ /dev/null @@ -1,48 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -__all__ = ["Bucket", - "BucketByteBuffer", - "BucketDecimal", - "BucketDouble", - "BucketFloat", - "BucketInteger", - "BucketLong", - "BucketString", - "BucketUUID", - "Dates", - "Identity", - "Timestamps", - "Transform", - "Transforms", - "Truncate"] - -from .bucket import (Bucket, - BucketByteBuffer, - BucketDecimal, - BucketDouble, - BucketFloat, - BucketInteger, - BucketLong, - BucketString, - BucketUUID) -from .dates import Dates -from .identity import Identity -from .timestamps import Timestamps -from .transform import Transform -from .transforms import Transforms -from .truncate import Truncate diff --git a/iceberg/api/transforms/bucket.py b/iceberg/api/transforms/bucket.py deleted file mode 100644 index 2884b81e17..0000000000 --- a/iceberg/api/transforms/bucket.py +++ /dev/null @@ -1,196 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -import math -import struct -import sys - -import mmh3 - -from .transform import Transform -from .transform_util import TransformUtil -from ..expressions import (Expressions, - JAVA_MAX_INT, - Operation) -from ..types.types import (IntegerType, - TypeID) -from ...api.types.conversions import Conversions - - -class Bucket(Transform): - MURMUR3 = mmh3 - - BUCKET_TYPE = {TypeID.DATE: lambda n: BucketInteger(n), - TypeID.INTEGER: lambda n: BucketInteger(n), - TypeID.TIME: lambda n: BucketLong(n), - TypeID.TIMESTAMP: lambda n: BucketLong(n), - TypeID.LONG: lambda n: BucketLong(n), - TypeID.DECIMAL: lambda n: BucketDecimal(n), - TypeID.STRING: lambda n: BucketString(n), - TypeID.FIXED: lambda n: BucketByteBuffer(n), - TypeID.BINARY: lambda n: BucketByteBuffer(n), - TypeID.UUID: lambda n: BucketUUID(n)} - - @staticmethod - def get(type_var, n): - bucket_type_func = Bucket.BUCKET_TYPE.get(type_var.type_id) - if not bucket_type_func: - raise RuntimeError("Cannot bucket by type: %s" % type_var) - return bucket_type_func(n) - - def __init__(self, n): - self.n = n - - def __eq__(self, other): - if id(self) == id(other): - return True - if other is None or not isinstance(other, Bucket): - return False - - return self.n == other.n - - def __hash__(self): - return hash(self.__key()) - - def __key(self): - return Bucket.__class__, self.n - - def __repr__(self): - return "Bucket[%s]" % self.n - - def __str__(self): - return "bucket[%s]" % self.n - - def apply(self, value): - return (self.hash(value) & JAVA_MAX_INT) % self.n - - def hash(self): - raise NotImplementedError() - - def project(self, name, predicate): - if predicate.op == Operation.EQ: - return Expressions.predicate(predicate.op, name, self.apply(predicate.lit.value)) - - def project_strict(self, name, predicate): - if predicate.op == Operation.NOT_EQ: - return Expressions.predicate(predicate.op, name, self.apply(predicate.lit.value)) - - def get_result_type(self, source_type): - return IntegerType.get() - - -class BucketInteger(Bucket): - - def __init__(self, n): - super(BucketInteger, self).__init__(n) - - def hash(self, value): - return Bucket.MURMUR3.hash(struct.pack("q", value)) - - def can_transform(self, type_var): - return type_var.type_id in [TypeID.INTEGER, TypeID.DATE] - - -class BucketLong(Bucket): - def __init__(self, n): - super(BucketLong, self).__init__(n) - - def hash(self, value): - return Bucket.MURMUR3.hash(struct.pack("q", value)) - - def can_transform(self, type_var): - return type_var.type_id in [TypeID.LONG, - TypeID.TIME, - TypeID.TIMESTAMP] - - -class BucketFloat(Bucket): - def __init__(self, n): - super(BucketFloat, self).__init__(n) - - def hash(self, value): - return Bucket.MURMUR3.hash(struct.pack("d", value)) - - def can_transform(self, type_var): - return type_var.type_id == TypeID.FLOAT - - -class BucketDouble(Bucket): - def __init__(self, n): - super(BucketDouble, self).__init__(n) - - def hash(self, value): - return Bucket.MURMUR3.hash(struct.pack("d", value)) - - def can_transform(self, type_var): - return type_var.type_id == TypeID.DOUBLE - - -class BucketDecimal(Bucket): - - def __init__(self, n): - super(BucketDecimal, self).__init__(n) - - def hash(self, value): - # to-do: unwrap to_bytes func since python2 support is being removed - unscaled_value = TransformUtil.unscale_decimal(value) - number_of_bytes = int(math.ceil(unscaled_value.bit_length() / 8)) - return Bucket.MURMUR3.hash(to_bytes(unscaled_value, number_of_bytes, byteorder='big')) - - def can_transform(self, type_var): - return type_var.type_id == TypeID.DECIMAL - - -class BucketString(Bucket): - def __init__(self, n): - super(BucketString, self).__init__(n) - - def hash(self, value): - return Bucket.MURMUR3.hash(value) - - def can_transform(self, type_var): - return type_var.type_id == TypeID.STRING - - -class BucketByteBuffer(Bucket): - def __init__(self, n): - super(BucketByteBuffer, self).__init__(n) - - def hash(self, value): - return Bucket.MURMUR3.hash(value) - - def can_transform(self, type_var): - return type_var.type_id in [TypeID.BINARY, TypeID.FIXED] - - -class BucketUUID(Bucket): - def __init__(self, n): - super(BucketUUID, self).__init__(n) - - def hash(self, value): - return Bucket.MURMUR3.hash(Conversions.to_byte_buffer(TypeID.UUID, value)) - - def can_transform(self, type_var): - return type_var.type_id == TypeID.UUID - - -def to_bytes(n, length, byteorder='big'): - if sys.version_info >= (3, 0): - return n.to_bytes(length, byteorder=byteorder) - h = '%x' % n - s = ('0' * (len(h) % 2) + h).zfill(length * 2).decode('hex') - return s if byteorder == 'big' else s[::-1] diff --git a/iceberg/api/transforms/dates.py b/iceberg/api/transforms/dates.py deleted file mode 100644 index dfc6b9a2cd..0000000000 --- a/iceberg/api/transforms/dates.py +++ /dev/null @@ -1,87 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -import datetime - -from .projection_util import ProjectionUtil -from .transform import Transform -from .transform_util import TransformUtil -from ..expressions import (Expressions, - Operation) -from ..types.types import (IntegerType, - TypeID) - - -class Dates(Transform): - YEAR = "year" - MONTH = "month" - DAY = "day" - - EPOCH = datetime.datetime.utcfromtimestamp(0) - SECONDS_IN_DAY = 86400 - - HUMAN_FUNCS = {"year": lambda x: TransformUtil.human_year(x), - "month": lambda x: TransformUtil.human_month(x), - "day": lambda x: TransformUtil.human_day(x)} - - def __init__(self, granularity, name): - if granularity not in (Dates.YEAR, Dates.MONTH, Dates.DAY): - raise RuntimeError("Invalid Granularity: %s" % granularity) - self.granularity = granularity - self.name = name - - def apply(self, days): - if self.granularity == Dates.DAY: - return days - else: - apply_func = getattr(TransformUtil, "diff_{}".format(self.granularity)) - return apply_func(datetime.datetime.utcfromtimestamp(days * Dates.SECONDS_IN_DAY), Dates.EPOCH) - - def can_transform(self, type): - return type.type_id == TypeID.DATE - - def get_result_type(self, source_type): - return IntegerType.get() - - def project(self, name, predicate): - if predicate.op == Operation.NOT_NULL or predicate.op == Operation.IS_NULL: - return Expressions.predicate(predicate.op, name) - - return ProjectionUtil.truncate_integer(name, predicate, self) - - def project_strict(self, name, predicate): - return None - - def to_human_string(self, value): - if value is None: - return "null" - - return Dates.HUMAN_FUNCS[self.granularity](value) - - def __str__(self): - return self.name - - def dedup_name(self): - return "time" - - def __eq__(self, other): - if id(self) == id(other): - return True - if other is None or not isinstance(other, Dates): - return False - - return self.granularity == other.granularity and self.name == other.name diff --git a/iceberg/api/transforms/identity.py b/iceberg/api/transforms/identity.py deleted file mode 100644 index f57524eb8a..0000000000 --- a/iceberg/api/transforms/identity.py +++ /dev/null @@ -1,91 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -from .transform import Transform -from .transform_util import TransformUtil -from ..expressions import Expressions -from ..types import TypeID - - -class Identity(Transform): - - @staticmethod - def get(type_var): - return Identity(type_var) - - def __init__(self, type_var): - self.type_var = type_var - - def apply(self, value): - return value - - def can_transform(self, type_var): - return type_var.is_primitive_type() - - def get_result_type(self, source_type): - return source_type - - def project(self, name, predicate): - return self.project_strict(name, predicate) - - def project_strict(self, name, predicate): - if predicate.lit is not None: - return Expressions.predicate(predicate.op, name, predicate.lit.value) - else: - return Expressions.predicate(predicate.op, name) - - def to_human_string(self, value): - if value is None: - return "null" - - if self.type_var.type_id == TypeID.DATE: - return TransformUtil.human_day(value) - elif self.type_var.type_id == TypeID.TIME: - return TransformUtil.human_time(value) - elif self.type_var.type_id == TypeID.TIMESTAMP: - if self.type_var.adjust_to_utc: - return TransformUtil.human_timestamp_with_timezone(value) - else: - return TransformUtil.human_timestamp_without_timezone(value) - elif self.type_var.type_id in (TypeID.BINARY, TypeID.FIXED): - raise NotImplementedError() - # if isinstance(value, bytearray): - # return base64.b64encode(value) - # elif isinstance(value, bytes): - # return base64.b64encode(bytes(value)) - # else: - # raise RuntimeError("Unsupported binary type: %s" % value.__class__.__name__) - else: - return str(value) - - def __str__(self): - return "identity" - - def __eq__(self, other): - if id(self) == id(other): - return True - - if other is None or not isinstance(other, Identity): - return False - - return self.type_var == other.type_var - - def __hash__(self): - return hash(self.__key()) - - def __key(self): - return Identity.__class__, self.type_var diff --git a/iceberg/api/transforms/projection_util.py b/iceberg/api/transforms/projection_util.py deleted file mode 100644 index f9b0bb547e..0000000000 --- a/iceberg/api/transforms/projection_util.py +++ /dev/null @@ -1,66 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - - -import decimal - -from iceberg.api.expressions import Expressions, Operation - - -class ProjectionUtil(object): - @staticmethod - def truncate_integer(name, pred, transform): - boundary = pred.lit.value - if pred.op == Operation.LT: - return Expressions.predicate(Operation.LT_EQ, name, transform.apply(boundary - 1)) - elif pred.op == Operation.LT_EQ: - return Expressions.predicate(Operation.LT_EQ, name, transform.apply(boundary)) - elif pred.op == Operation.GT: - return Expressions.predicate(Operation.GT_EQ, name, transform.apply(boundary + 1)) - elif pred.op == Operation.GT_EQ: - return Expressions.predicate(Operation.GT_EQ, name, transform.apply(boundary)) - elif pred.op == Operation.EQ: - return Expressions.predicate(pred.op, name, transform.apply(boundary)) - - def truncate_long(name, pred, transform): - return ProjectionUtil.truncate_integer(name, pred, transform) - - def truncate_decimal(name, pred, transform): - boundary = pred.lit.value - - if pred.op == Operation.LT: - minus_one = boundary - decimal.Decimal(1) - return Expressions.predicate(Operation.LT_EQ, name, transform.apply(minus_one)) - elif pred.op == Operation.LT_EQ: - return Expressions.predicate(Operation.LT_EQ, name, transform.apply(boundary)) - elif pred.op == Operation.GT: - plus_one = boundary + decimal.Decimal(1) - return Expressions.predicate(Operation.GT_EQ, name, transform.apply(plus_one)) - elif pred.op == Operation.GT_EQ: - return Expressions.predicate(Operation.GT_EQ, name, transform.apply(boundary)) - elif pred.op == Operation.EQ: - return Expressions.predicate(pred.op, name, transform.apply(boundary)) - - def truncate_array(name, pred, transform): - boundary = pred.lit.value - - if pred.op == Operation.LT or pred.op == Operation.LT_EQ: - return Expressions.predicate(Operation.LT_EQ, name, transform.apply(boundary)) - elif pred.op == Operation.GT or pred.op == Operation.GT_EQ: - return Expressions.predicate(Operation.GT_EQ, name, transform.apply(boundary)) - elif pred.op == Operation.EQ: - return Expressions.predicate(pred.op, name, transform.apply(boundary)) diff --git a/iceberg/api/transforms/timestamps.py b/iceberg/api/transforms/timestamps.py deleted file mode 100644 index ca38a1c3be..0000000000 --- a/iceberg/api/transforms/timestamps.py +++ /dev/null @@ -1,83 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - - -import datetime - -from .transform import Transform -from .transform_util import TransformUtil -from ..expressions import (Expressions, - Operation) -from ..types.types import (IntegerType, - TypeID) - - -class Timestamps(Transform): - YEAR = "year" - MONTH = "month" - DAY = "day" - HOUR = "hour" - - EPOCH = datetime.datetime.utcfromtimestamp(0) - HUMAN_FUNCS = {"year": lambda x: TransformUtil.human_year(x), - "month": lambda x: TransformUtil.human_month(x), - "day": lambda x: TransformUtil.human_day(x), - "hour": lambda x: TransformUtil.human_hour(x)} - - def __init__(self, granularity, name): - if granularity not in (Timestamps.YEAR, Timestamps.MONTH, Timestamps.DAY, Timestamps.HOUR): - raise RuntimeError("Invalid Granularity: %s" % granularity) - - self.granularity = granularity - self.name = name - - def apply(self, value): - apply_func = getattr(TransformUtil, "diff_{}".format(self.granularity)) - return apply_func(datetime.datetime.utcfromtimestamp(value / 1000000), Timestamps.EPOCH) - - def can_transform(self, type_var): - return type_var.type_id == TypeID.TIMESTAMP - - def get_result_type(self, source_type): - return IntegerType.get() - - def project(self, name, predicate): - if predicate.op == Operation.NOT_NULL or predicate.op == Operation.IS_NULL: - return Expressions.predicate(predicate.op, self.name) - - def project_strict(self, name, predicate): - return None - - def to_human_string(self, value): - if value is None: - return "null" - - return Timestamps.HUMAN_FUNCS[self.granularity](value) - - def __str__(self): - return self.name - - def dedup_name(self): - return "time" - - def __eq__(self, other): - if id(self) == id(other): - return True - if other is None or not isinstance(other, Timestamps): - return False - - return self.granularity == other.granularity and self.name == other.name diff --git a/iceberg/api/transforms/transform.py b/iceberg/api/transforms/transform.py deleted file mode 100644 index 0ccbf2e3c2..0000000000 --- a/iceberg/api/transforms/transform.py +++ /dev/null @@ -1,43 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - - -class Transform(object): - - def __init__(self): - raise NotImplementedError() - - def apply(self, value): - raise NotImplementedError() - - def can_transform(self, type_var): - raise NotImplementedError() - - def get_result_type(self, source_type): - raise NotImplementedError() - - def project(self, name, predicate): - raise NotImplementedError() - - def project_strict(self, name, predicate): - raise NotImplementedError() - - def to_human_string(self, value): - return str(value) - - def dedup_name(self): - return self.__str__() diff --git a/iceberg/api/transforms/transform_util.py b/iceberg/api/transforms/transform_util.py deleted file mode 100644 index 9042fcc40f..0000000000 --- a/iceberg/api/transforms/transform_util.py +++ /dev/null @@ -1,84 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -from datetime import datetime, timedelta - -import pytz - - -class TransformUtil(object): - EPOCH = datetime.utcfromtimestamp(0) - EPOCH_YEAR = datetime.utcfromtimestamp(0).year - - @staticmethod - def human_year(year_ordinal): - return "{0:0=4d}".format(TransformUtil.EPOCH_YEAR + year_ordinal) - - @staticmethod - def human_month(month_ordinal): - return "{0:0=4d}-{1:0=2d}".format(TransformUtil.EPOCH_YEAR + int(month_ordinal / 12), 1 + int(month_ordinal % 12)) - - @staticmethod - def human_day(day_ordinal): - day = TransformUtil.EPOCH + timedelta(days=day_ordinal) - return "{0:0=4d}-{1:0=2d}-{2:0=2d}".format(day.year, day.month, day.day) - - @staticmethod - def human_time(micros_from_midnight): - day = TransformUtil.EPOCH + timedelta(microseconds=micros_from_midnight) - return "{}".format(day.time()) - - @staticmethod - def human_timestamp_with_timezone(timestamp_micros): - day = TransformUtil.EPOCH + timedelta(microseconds=timestamp_micros) - return pytz.timezone("UTC").localize(day).strftime("%Y-%m-%dT%H:%M:%S.%fZ") - - @staticmethod - def human_timestamp_without_timezone(timestamp_micros): - day = TransformUtil.EPOCH + timedelta(microseconds=timestamp_micros) - return day.isoformat() - - @staticmethod - def human_hour(hour_ordinal): - time = TransformUtil.EPOCH + timedelta(hours=hour_ordinal) - return "{0:0=4d}-{1:0=2d}-{2:0=2d}-{3:0=2d}".format(time.year, time.month, time.day, time.hour) - - @staticmethod - def base_64_encode(buffer): - raise NotImplementedError() - - @staticmethod - def diff_hour(date1, date2): - return int((date1 - date2).total_seconds() / 3600) - - @staticmethod - def diff_day(date1, date2): - return (date1 - date2).days - - @staticmethod - def diff_month(date1, date2): - return (date1.year - date2.year) * 12 + (date1.month - date2.month) - (1 if date1.day < date2.day else 0) - - @staticmethod - def diff_year(date1, date2): - return (date1.year - date2.year) - \ - (1 if date1.month < date2.month or (date1.month == date2.month and date1.day < date2.day) else 0) - - @staticmethod - def unscale_decimal(decimal_value): - value_tuple = decimal_value.as_tuple() - return int(("-" if value_tuple.sign else "") + "".join([str(d) for d in value_tuple.digits])) diff --git a/iceberg/api/transforms/transforms.py b/iceberg/api/transforms/transforms.py deleted file mode 100644 index 205a9de737..0000000000 --- a/iceberg/api/transforms/transforms.py +++ /dev/null @@ -1,120 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -import re - -from .bucket import Bucket -from .dates import Dates -from .identity import Identity -from .timestamps import Timestamps -from .truncate import Truncate -from .unknown_transform import UnknownTransform -from .void_transform import VoidTransform -from ..types import (TypeID) - - -""" - Factory methods for transforms. -

- Most users should create transforms using a - {@link PartitionSpec.Builder#builderFor(Schema)} partition spec builder}. - - @see PartitionSpec#builderFor(Schema) The partition spec builder. -""" - - -class Transforms(object): - HAS_WIDTH = re.compile("(\\w+)\\[(\\d+)\\]") - - def __init__(self): - pass - - @staticmethod - def from_string(type_var, transform): - match = Transforms.HAS_WIDTH.match(transform) - - if match is not None: - name = match.group(1) - w = int(match.group(2)) - if name.lower() == "truncate": - return Truncate.get(type_var, w) - elif name.lower() == "bucket": - return Bucket.get(type_var, w) - - if transform.lower() == "identity": - return Identity.get(type_var) - elif type_var.type_id == TypeID.TIMESTAMP: - return Timestamps(transform.lower(), transform.lower()) - elif type_var.type_id == TypeID.DATE: - return Dates(transform.lower(), transform.lower()) - - if transform.lower() == "void": - return VoidTransform.get() - - return UnknownTransform(type_var, transform) - - @staticmethod - def identity(type_var): - return Identity.get(type_var) - - @staticmethod - def year(type_var): - if type_var.type_id == TypeID.DATE: - return Dates("year", "year") - elif type_var.type_id == TypeID.TIMESTAMP: - return Timestamps("year", "year") - else: - raise RuntimeError("Cannot partition type %s by year" % type_var) - - @staticmethod - def month(type_var): - if type_var.type_id == TypeID.DATE: - return Dates("month", "month") - elif type_var.type_id == TypeID.TIMESTAMP: - return Timestamps("month", "month") - else: - raise RuntimeError("Cannot partition type %s by month" % type_var) - - @staticmethod - def day(type_var): - if type_var.type_id == TypeID.DATE: - return Dates("day", "day") - elif type_var.type_id == TypeID.TIMESTAMP: - return Timestamps("day", "day") - else: - raise RuntimeError("Cannot partition type %s by day" % type_var) - - @staticmethod - def hour(type_var): - if type_var.type_id == TypeID.DATE: - return Dates("hour", "hour") - elif type_var.type_id == TypeID.TIMESTAMP: - return Timestamps("hour", "hour") - else: - raise RuntimeError("Cannot partition type %s by hour" % type_var) - - @staticmethod - def bucket(type_var, num_buckets): - return Bucket.get(type_var, num_buckets) - - @staticmethod - def truncate(type_var, width): - return Truncate.get(type_var, width) - - @staticmethod - def always_null(): - return VoidTransform.get() diff --git a/iceberg/api/transforms/truncate.py b/iceberg/api/transforms/truncate.py deleted file mode 100644 index b37e6f0b06..0000000000 --- a/iceberg/api/transforms/truncate.py +++ /dev/null @@ -1,231 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -from decimal import Decimal - -from .projection_util import ProjectionUtil -from .transform import Transform -from .transform_util import TransformUtil -from ..expressions import (Expressions, - Operation) -from ..types import TypeID - - -class Truncate(Transform): - - @staticmethod - def get(type_var, width): - if type_var.type_id == TypeID.INTEGER: - return TruncateInteger(width) - elif type_var.type_id == TypeID.LONG: - return TruncateLong(width) - elif type_var.type_id == TypeID.DECIMAL: - return TruncateDecimal(width) - elif type_var.type_id == TypeID.STRING: - return TruncateString(width) - - def __init__(self): - raise NotImplementedError() - - def apply(self, value): - raise NotImplementedError() - - def can_transform(self, type_var): - raise NotImplementedError() - - def get_result_type(self, source_type): - return type(source_type) - - def project(self, name, predicate): - raise NotImplementedError() - - def project_strict(self, name, predicate): - raise NotImplementedError() - - -class TruncateInteger(Truncate): - - def __init__(self, width): - self.W = width - - def apply(self, value): - return value - (((value % self.W) + self.W) % self.W) - - def can_transform(self, type_var): - return type_var.type_id == TypeID.INTEGER - - def project(self, name, predicate): - if predicate.op == Operation.NOT_NULL or predicate.op == Operation.IS_NULL: - return Expressions.predicate(predicate.op, name) - - return ProjectionUtil.truncate_integer(name, predicate, self) - - def project_strict(self, name, predicate): - if predicate.op == Operation.LT: - _in = predicate.lit.value - 1 - _out = predicate.lit.value - in_image = self.apply(_in) - out_image = self.apply(_out) - if in_image != out_image: - return Expressions.predicate(Operation.LT_EQ, name, in_image) - else: - return Expressions.predicate(Operation.LT, name, in_image) - elif predicate.op == Operation.LT_EQ: - _in = predicate.lit.value - _out = predicate.lit.value + 1 - in_image = self.apply(_in) - out_image = self.apply(_out) - if in_image != out_image: - return Expressions.predicate(Operation.LT_EQ, name, in_image) - else: - return Expressions.predicate(Operation.LT, name, in_image) - - def __eq__(self, other): - if id(self) == id(other): - return True - - if other is None or not isinstance(other, TruncateInteger): - return False - - return self.W == other.W - - def __hash__(self): - return hash(self.__key()) - - def __key(self): - return TruncateInteger.__class__, self.W - - def __str__(self): - return "truncate[%s]" % self.W - - -class TruncateLong(Truncate): - - def __init__(self, width): - self.W = width - - def apply(self, value): - return value - (((value % self.W) + self.W) % self.W) - - def can_transform(self, type_var): - return type_var.type_id == TypeID.LONG - - def project(self, name, predicate): - if predicate.op == Operation.NOT_NULL or predicate.op == Operation.IS_NULL: - return Expressions.predicate(predicate.op, name) - - return ProjectionUtil.truncate_long(name, predicate, self) - - def project_strict(self, name, predicate): - return None - - def __eq__(self, other): - if id(self) == id(other): - return True - - if other is None or not isinstance(other, TruncateLong): - return False - - return self.W == other.W - - def __hash__(self): - return hash(self.__key()) - - def __key(self): - return TruncateLong.__class__, self.W - - def __str__(self): - return "truncate[%s]" % self.W - - -class TruncateDecimal(Truncate): - - def __init__(self, unscaled_width): - self.unscaled_width = unscaled_width - - def apply(self, value): - unscaled_value = TransformUtil.unscale_decimal(value) - applied_value = unscaled_value - (((unscaled_value % self.unscaled_width) + self.unscaled_width) % self.unscaled_width) - return Decimal("{}e{}".format(applied_value, value.as_tuple().exponent)) - - def can_transform(self, type_var): - return type_var.type_id == TypeID.DECIMAL - - def project(self, name, predicate): - if predicate.op == Operation.NOT_NULL or predicate.op == Operation.IS_NULL: - return Expressions.predicate(predicate.op, name) - - return ProjectionUtil.truncate_decimal(name, predicate, self) - - def project_strict(self, name, predicate): - return None - - def __eq__(self, other): - if id(self) == id(other): - return True - - if other is None or not isinstance(other, TruncateDecimal): - return False - - return self.unscaled_width == other.unscaled_width - - def __hash__(self): - return hash(self.__key()) - - def __key(self): - return TruncateDecimal.__class__, self.unscaled_width - - def __str__(self): - return "truncate[%s]" % self.unscaled_width - - -class TruncateString(Truncate): - def __init__(self, length): - self.L = length - - def apply(self, value): - return value[0:min(self.L, len(value))] - - def can_transform(self, type_var): - return type_var.type_id == TypeID.STRING - - def project(self, name, predicate): - if predicate.op == Operation.NOT_NULL or predicate.op == Operation.IS_NULL: - return Expressions.predicate(predicate.op, name) - - return ProjectionUtil.truncate_array(name, predicate, self) - - def project_strict(self, name, predicate): - return None - - def __eq__(self, other): - if id(self) == id(other): - return True - - if other is None or not isinstance(other, TruncateString): - return False - - return self.L == other.L - - def __hash__(self): - return hash(self.__key()) - - def __key(self): - return TruncateString.__class__, self.L - - def __str__(self): - return "truncate[%s]" % self.L diff --git a/iceberg/api/transforms/unknown_transform.py b/iceberg/api/transforms/unknown_transform.py deleted file mode 100644 index de326445de..0000000000 --- a/iceberg/api/transforms/unknown_transform.py +++ /dev/null @@ -1,61 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -from typing import Union - -from iceberg.api.types import StringType, Type - -from .transform import Transform - - -class UnknownTransform(Transform): - - def __init__(self, source_type: Type, transform: str): - self.source_type = source_type - self.transform = transform - - def apply(self, value): - raise AttributeError(f"Cannot apply unsupported transform: {self.transform}") - - def can_transform(self, type_var) -> bool: - # assume the transform function can be applied for this type because unknown transform is only used when parsing - # a transform in an existing table. a different Iceberg version must have already validated it. - return self.source_type == type_var - - def get_result_type(self, source_type): - # the actual result type is not known - return StringType.get() - - def project(self, name, predicate): - return None - - def project_strict(self, name, predicate): - return None - - def __str__(self): - return self.transform - - def __eq__(self, other: Union['UnknownTransform', Transform, object]): - if id(self) == id(other): - return True - elif not isinstance(other, UnknownTransform): - return False - - return self.source_type == other.source_type and self.transform == other.transform - - def __hash__(self): - return hash((self.source_type, self.transform)) diff --git a/iceberg/api/transforms/void_transform.py b/iceberg/api/transforms/void_transform.py deleted file mode 100644 index ea859641ad..0000000000 --- a/iceberg/api/transforms/void_transform.py +++ /dev/null @@ -1,52 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -from .transform import Transform - - -class VoidTransform(Transform): - _INSTANCE = None - - @staticmethod - def get(): - if VoidTransform._INSTANCE is None: - VoidTransform._INSTANCE = VoidTransform() - return VoidTransform._INSTANCE - - def __init__(self): - pass - - def apply(self, value): - return None - - def can_transform(self, type_var): - return True - - def get_result_type(self, source_type): - return source_type - - def project(self, name, predicate): - return None - - def project_strict(self, name, predicate): - return None - - def to_human_string(self, value): - return "null" - - def __str__(self): - return "void" diff --git a/iceberg/api/types/__init__.py b/iceberg/api/types/__init__.py deleted file mode 100644 index cf3fe0167b..0000000000 --- a/iceberg/api/types/__init__.py +++ /dev/null @@ -1,134 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -__all__ = ["AssignFreshIds", - "assign_fresh_ids", - "from_primitive_string", - "get_projected_ids", - "Conversions", - "CustomOrderSchemaVisitor", - "NestedType", - "PrimitiveType", - "Type", - "TypeID", - "BinaryType", - "BooleanType", - "DateType", - "DecimalType", - "DoubleType", - "FixedType", - "FloatType", - "GetProjectedIds", - "IntegerType", - "IndexById", - "IndexByName", - "index_by_id", - "index_by_name", - "join", - "ListType", - "LongType", - "MapType", - "NestedField", - "PruneColumns", - "SchemaVisitor", - "select", - "select_not", - "StringType", - "StructType", - "TimeType", - "TimestampType", - "UUIDType", - "visit", - "visit_custom_order", - "VisitFieldFuture", - "VisitFuture" - ] - -import re - -from .conversions import Conversions -from .type import (NestedType, - PrimitiveType, - Type, - TypeID) -from .type_util import (assign_fresh_ids, - AssignFreshIds, - CustomOrderSchemaVisitor, - get_projected_ids, - GetProjectedIds, - index_by_id, - index_by_name, - IndexById, - IndexByName, - join, - PruneColumns, - SchemaVisitor, - select, - select_not, - visit, - visit_custom_order, - VisitFieldFuture, - VisitFuture) -from .types import (BinaryType, - BooleanType, - DateType, - DecimalType, - DoubleType, - FixedType, - FloatType, - IntegerType, - ListType, - LongType, - MapType, - NestedField, - StringType, - StructType, - TimestampType, - TimeType, - UUIDType) - -TYPES = {str(BooleanType.get()): BooleanType.get(), - str(IntegerType.get()): IntegerType.get(), - str(LongType.get()): LongType.get(), - str(FloatType.get()): FloatType.get(), - str(DoubleType.get()): DoubleType.get(), - str(DateType.get()): DateType.get(), - str(TimeType.get()): TimeType.get(), - str(TimestampType.with_timezone()): TimestampType.with_timezone(), - str(TimestampType.without_timezone()): TimestampType.without_timezone(), - str(StringType.get()): StringType.get(), - str(UUIDType.get()): UUIDType.get(), - str(BinaryType.get()): BinaryType.get()} - -FIXED = re.compile("fixed\\[(\\d+)\\]") -DECIMAL = re.compile("decimal\\((\\d+),\\s+(\\d+)\\)") - - -def from_primitive_string(type_string): - lower_type_string = type_string.lower() - if lower_type_string in TYPES.keys(): - return TYPES[lower_type_string] - - matches = FIXED.match(type_string) - if matches: - return FixedType.of_length(matches.group(1)) - - matches = DECIMAL.match(type_string) - if matches: - return DecimalType.of(matches.group(1), matches.group(2)) - - raise RuntimeError("Cannot parse type string to primitive: %s", type_string) diff --git a/iceberg/api/types/conversions.py b/iceberg/api/types/conversions.py deleted file mode 100644 index c6dbbf6a15..0000000000 --- a/iceberg/api/types/conversions.py +++ /dev/null @@ -1,100 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -from decimal import Decimal -import struct -import sys -import uuid - -from .type import TypeID - - -class Conversions(object): - HIVE_NULL = "__HIVE_DEFAULT_PARTITION__" - value_mapping = {TypeID.BOOLEAN: lambda as_str: as_str.lower() == "true" if as_str is not None else False, - TypeID.INTEGER: lambda as_str: int(float(as_str)), - TypeID.LONG: lambda as_str: int(float(as_str)), - TypeID.FLOAT: lambda as_str: float(as_str), - TypeID.DOUBLE: lambda as_str: float(as_str), - TypeID.STRING: lambda as_str: as_str, - TypeID.UUID: lambda as_str: uuid.UUID(as_str), - TypeID.FIXED: lambda as_str: bytearray(bytes(as_str, "UTF-8") - if sys.version_info >= (3, 0) - else bytes(as_str)), - TypeID.BINARY: lambda as_str: bytes(as_str, "UTF-8") if sys.version_info >= (3, 0) else bytes(as_str), - TypeID.DECIMAL: lambda as_str: Decimal(as_str), - } - - to_byte_buff_mapping = {TypeID.BOOLEAN: lambda type_id, value: struct.pack("QQ', (value.int >> 64) - & 0xFFFFFFFFFFFFFFFF, value.int - & 0xFFFFFFFFFFFFFFFF), - TypeID.FIXED: lambda type_id, value: value, - TypeID.BINARY: lambda type_id, value: value, - # TypeId.DECIMAL: lambda type_var, value: struct.pack(value.quantize( - # Decimal('.' + "".join(['0' for x in range(0, type_var.scale)]) + '1')) - } - - from_byte_buff_mapping = {TypeID.BOOLEAN: lambda type_id, value: struct.unpack('QQ', value)[0] << 64 | struct.unpack('>QQ', value)[1]), - TypeID.FIXED: lambda type_id, value: value, - TypeID.BINARY: lambda type_id, value: value} - - @staticmethod - def from_partition_string(type_var, as_string): - if as_string is None or Conversions.HIVE_NULL == as_string: - return None - part_func = Conversions.value_mapping.get(type_var.type_id) - if part_func is None: - raise RuntimeError("Unsupported type for fromPartitionString: %s" % type_var) - - return part_func(as_string) - - @staticmethod - def to_byte_buffer(type_id, value): - try: - return Conversions.to_byte_buff_mapping.get(type_id)(type_id, value) - except KeyError: - raise TypeError("Cannot Serialize Type: %s" % type_id) - - @staticmethod - def from_byte_buffer(type_var, buffer_var): - return Conversions.internal_from_byte_buffer(type_var.type_id, buffer_var) - - @staticmethod - def internal_from_byte_buffer(type_id, buffer_var): - try: - return Conversions.from_byte_buff_mapping.get(type_id)(type_id, buffer_var) - except KeyError: - raise TypeError("Cannot deserialize Type: %s" % type_id) diff --git a/iceberg/api/types/type.py b/iceberg/api/types/type.py deleted file mode 100644 index 437b95bfd1..0000000000 --- a/iceberg/api/types/type.py +++ /dev/null @@ -1,118 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -from decimal import Decimal -from enum import Enum, unique -import uuid - - -@unique -class TypeID(Enum): - BOOLEAN = {"java_class": "Boolean.class", "python_class": bool, "id": 1} - INTEGER = {"java_class": "Integer.class", "python_class": int, "id": 2} - LONG = {"java_class": "Long.class", "python_class": int, "id": 3} - FLOAT = {"java_class": "Float.class", "python_class": float, "id": 4} - DOUBLE = {"java_class": "Double.class", "python_class": float, "id": 5} - DATE = {"java_class": "Integer.class", "python_class": int, "id": 6} - TIME = {"java_class": "Long.class", "python_class": int, "id": 7} - TIMESTAMP = {"java_class": "Long.class", "python_class": int, "id": 8} - STRING = {"java_class": "CharSequence.class", "python_class": str, "id": 9} - UUID = {"java_class": "java.util.UUID.class", "python_class": uuid.UUID, "id": 10} - FIXED = {"java_class": "ByteBuffer.class", "python_class": bytes, "id": 11} - BINARY = {"java_class": "ByteBuffer.class", "python_class": bytearray, "id": 12} - DECIMAL = {"java_class": "BigDecimal.class", "python_class": Decimal, "id": 13} - STRUCT = {"java_class": "Void.class", "python_class": None, "id": 14} - LIST = {"java_class": "Void.class", "python_class": None, "id": 15} - MAP = {"java_class": "Void.class", "python_class": None, "id": 16} - - -class Type(object): - length: int - scale: int - precision: int - - def __init__(self): - pass - - def type_id(self): - pass - - def is_primitive_type(self): - return False - - def as_primitive_type(self): - raise ValueError("Not a primitive type: " + self) - - def as_struct_type(self): - raise ValueError("Not a struct type: " + self) - - def as_list_type(self): - raise ValueError("Not a list type: " + self) - - def asMapType(self): - raise ValueError("Not a map type: " + self) - - def is_nested_type(self): - return False - - def is_struct_type(self): - return False - - def is_list_type(self): - return False - - def is_map_type(self): - return False - - def as_nested_type(self): - raise ValueError("Not a nested type: " + self) - - -class PrimitiveType(Type): - - def __eq__(self, other): - return type(self) == type(other) - - def __ne__(self, other): - return not self.__eq__(other) - - def is_primitive_type(self): - return True - - def as_primitive_type(self): - return self - - -class NestedType(Type): - - def __init__(self): - super(NestedType, self).__init__() - - def is_nested_type(self): - return True - - def as_nested_type(self): - return self - - def fields(self): - pass - - def field_type(self, name): - pass - - def field(self, id): - pass diff --git a/iceberg/api/types/type_util.py b/iceberg/api/types/type_util.py deleted file mode 100644 index 5a465e50fe..0000000000 --- a/iceberg/api/types/type_util.py +++ /dev/null @@ -1,544 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -import math -from typing import List - -from .type import (Type, - TypeID) -from .types import (ListType, - MapType, - NestedField, - StructType) -from ...exceptions import ValidationException - -MAX_PRECISION = list() -REQUIRED_LENGTH = [-1 for item in range(40)] - -MAX_PRECISION.append(0) -for i in range(1, 24): - MAX_PRECISION.append(int(math.floor(math.log10(math.pow(2, 8 * i - 1) - 1)))) - -for i in range(len(REQUIRED_LENGTH)): - for j in range(len(MAX_PRECISION)): - if i <= MAX_PRECISION[j]: - REQUIRED_LENGTH[i] = j - break - if REQUIRED_LENGTH[i] < 0: - raise RuntimeError("Could not find required length for precision %s" % i) - - -def select(schema, field_ids): - import iceberg.api.schema - if schema is None: - raise RuntimeError("Schema cannot be None") - if field_ids is None: - raise RuntimeError("Field ids cannot be None") - - result = visit(schema, PruneColumns(field_ids)) - if schema.as_struct() == result: - return schema - elif result is not None: - if schema.get_aliases() is not None: - return iceberg.api.schema.Schema(result.as_nested_type().fields, schema.get_aliases()) - else: - return iceberg.api.schema.Schema(result.as_nested_type().fields) - - return iceberg.api.schema.Schema(list(), schema.get_aliases()) - - -def get_projected_ids(schema): - import iceberg.api.schema - if isinstance(schema, iceberg.api.schema.Schema): - return visit(schema, GetProjectedIds()) - elif isinstance(schema, Type): - if schema.is_primitive_type(): - return set() - - return set(visit(schema, GetProjectedIds)) - - else: - raise RuntimeError("Argument %s must be Schema or a Type" % schema) - - -def select_not(schema, field_ids): - projected_ids = get_projected_ids(schema) - projected_ids.difference(field_ids) - - return select(schema, projected_ids) - - -def join(left, right): - import iceberg.api.schema - return iceberg.api.schema.Schema(left + right) - - -def index_by_name(struct): - return visit(struct, IndexByName()) - - -def index_by_id(struct): - return visit(struct, IndexById()) - - -def assign_fresh_ids(type_var, next_id): - from ..schema import Schema - if isinstance(type_var, Type): - return visit(type_var, AssignFreshIds(next_id)) - elif isinstance(type_var, Schema): - schema = type_var - return Schema(list(visit(schema.as_struct(), AssignFreshIds(next_id)) - .as_nested_type().fields)) - - -def visit(arg, visitor): # noqa: ignore=C901 - from ..schema import Schema - if isinstance(visitor, CustomOrderSchemaVisitor): - return visit_custom_order(arg, visitor) - elif isinstance(arg, Schema): - return visitor.schema(arg, visit(arg.as_struct(), visitor)) - elif isinstance(arg, Type): - type_var = arg - if type_var.type_id == TypeID.STRUCT: - struct = type_var.as_nested_type().as_struct_type() - results = list() - for field in struct.fields: - visitor.field_ids.append(field.field_id) - visitor.field_names.append(field.name) - result = None - try: - result = visit(field.type, visitor) - except NotImplementedError: - # will remove it after missing functions are implemented. - pass - finally: - visitor.field_ids.pop() - visitor.field_names.pop() - results.append(visitor.field(field, result)) - return visitor.struct(struct, results) - elif type_var.type_id == TypeID.LIST: - list_var = type_var.as_nested_type().as_list_type() - visitor.field_ids.append(list_var.element_id) - try: - element_result = visit(list_var.element_type, visitor) - except NotImplementedError: - # will remove it after missing functions are implemented. - pass - finally: - visitor.field_ids.pop() - - return visitor.list(list_var, element_result) - elif type_var.type_id == TypeID.MAP: - raise NotImplementedError() - else: - return visitor.primitive(arg.as_primitive_type()) - else: - raise RuntimeError("Invalid type for arg: %s" % arg) - - -def visit_custom_order(arg, visitor): - from ..schema import Schema - if isinstance(arg, Schema): - schema = arg - return visitor.schema(arg, VisitFuture(schema.as_struct(), visitor)) - elif isinstance(arg, Type): - type_var = arg - if type_var.type_id == TypeID.STRUCT: - struct = type_var.as_nested_type().as_struct_type() - results = list() - fields = struct.fields - for field in fields: - results.append(VisitFieldFuture(field, visitor)) - struct = visitor.struct(struct, [x.get() for x in results]) - return struct - elif type_var.type_id == TypeID.LIST: - list_var = type_var.as_nested_type().as_list_type() - return visitor.list(list_var, VisitFuture(list_var.element_type, visitor)) - elif type_var.type_id == TypeID.MAP: - raise NotImplementedError() - - return visitor.primitive(type_var.as_primitive_type()) - - -class SchemaVisitor(object): - - def __init__(self): - self.field_names = list() - self.field_ids = list() - - def schema(self, schema, struct_result): - return None - - def struct(self, struct, field_results): - return None - - def field(self, field, field_result): - return None - - def list(self, list_var, element_result): - return None - - def map(self, map_var, key_result, value_result): - return None - - def primitive(self, primitive_var): - return None - - -class CustomOrderSchemaVisitor(object): - def __init__(self): - super(CustomOrderSchemaVisitor, self).__init__() - - def schema(self, schema, struct_result): - return None - - def struct(self, struct, field_results): - return None - - def field(self, field, field_result): - return None - - def list(self, list_var, element_result): - return None - - def map(self, map_var, key_result, value_result): - return None - - def primitive(self, primitive_var): - return None - - -class VisitFuture(object): - - def __init__(self, type, visitor): - self.type = type - self.visitor = visitor - - def get(self): - return visit(self.type, self.visitor) - - -class VisitFieldFuture(object): - - def __init__(self, field, visitor): - self.field = field - self.visitor = visitor - - def get(self): - return self.visitor.field(self.field, VisitFuture(self.field.type, self.visitor).get) - - -def decimal_required_bytes(precision): - if precision < 0 or precision > 40: - raise RuntimeError("Unsupported decimal precision: %s" % precision) - - return REQUIRED_LENGTH[precision] - - -class GetProjectedIds(SchemaVisitor): - - def __init__(self): - super(GetProjectedIds, self).__init__() - self.field_ids = list() - - def schema(self, schema, struct_result): - return self.field_ids - - def struct(self, struct, field_results): - return self.field_ids - - def field(self, field, field_result): - if field_result is None: - self.field_ids.append(field.field_id) - - return self.field_ids - - def list(self, list_var, element_result): - if element_result is None: - for field in list_var.fields(): - self.field_ids.append(field.field_id) - - return self.field_ids - - def map(self, map_var, key_result, value_result): - if value_result is None: - for field in map_var.fields(): - self.field_ids.append(field.field_id) - - return self.field_ids - - -class PruneColumns(SchemaVisitor): - - def __init__(self, selected): - super(PruneColumns, self).__init__() - self.selected = list(selected) - - def schema(self, schema, struct_result): - return struct_result - - def struct(self, struct, field_results): - fields = struct.fields - selected_fields = list() - same_types = True - - for i, projected_type in enumerate(field_results): - field = fields[i] - if projected_type is not None: - if field.type == projected_type: - selected_fields.append(field) - elif projected_type is not None: - same_types = False - if field.is_optional: - selected_fields.append(NestedField.optional(field.field_id, - field.name, - projected_type)) - else: - selected_fields.append(NestedField.required(field.field_id, - field.name, - projected_type)) - - if len(selected_fields) != 0: - if len(selected_fields) == len(fields) and same_types: - return struct - else: - return StructType.of(selected_fields) - - def field(self, field, field_result): - if field.field_id in self.selected: - return field.type - elif field_result is not None: - return field_result - - def primitive(self, primitive_var): - return None - - -class IndexByName(SchemaVisitor): - - DOT = "." - - def __init__(self): - super(IndexByName, self).__init__() - self.name_to_id = dict() - - def schema(self, schema, struct_result): - return self.name_to_id - - def struct(self, struct, field_results): - return self.name_to_id - - def field(self, field, field_result): - self.add_field(field.name, field.field_id) - - def list(self, list_var, element_result): - for field in list_var.fields(): - self.add_field(field.name, field.field_id) - - def map(self, map_var, key_result, value_result): - for field in map_var.fields(): - self.add_field(field.name, field.field_id) - - def add_field(self, name, field_id): - full_name = name - if self.field_names is not None and len(self.field_names) > 0: - full_name = IndexByName.DOT.join([IndexByName.DOT.join(self.field_names), name]) - - existing_field_id = self.name_to_id.get(full_name) - ValidationException.check(existing_field_id is None, "Invalid schema: multiple fields for name %s: %s and %s", - (full_name, existing_field_id, field_id)) - - self.name_to_id[full_name] = field_id - - -class IndexById(SchemaVisitor): - - def __init__(self): - super(IndexById, self).__init__() - self.index = dict() - - def schema(self, schema, struct_result): - return self.index - - def struct(self, struct, field_results): - return self.index - - def field(self, field, field_result): - self.index[field.field_id] = field - - def list(self, list_var, element_result): - for field in list_var.fields(): - self.index[field.field_id] = field - - def map(self, map_var, key_result, value_result): - for field in map_var.fields: - self.index[field.field_id] = field - - -class NextID(object): - def __init__(self): - raise NotImplementedError() - - def get(self): - raise NotImplementedError() - - -class AssignFreshIds(CustomOrderSchemaVisitor): - def __init__(self, next_id): - super(AssignFreshIds, self).__init__() - self.next_id = next_id - - def schema(self, schema, struct_result): - return self.next_id() - - def struct(self, struct, field_results): - fields = struct.fields - length = len(struct.fields) - new_ids = list() - - for _ in range(length): - new_ids.append(self.next_id()) - - new_fields = list() - types = iter(field_results) - for i in range(length): - field = fields[i] - type = next(types) - if field.is_optional: - new_fields.append(NestedField.optional(new_ids[i], field.name, type)) - else: - new_fields.append(NestedField.required(new_ids[i], field.name, type)) - - return StructType.of(new_fields) - - def field(self, field, field_result): - return field_result() - - def list(self, list_var, element_result): - new_id = self.next_id() - if list_var.is_element_optional(): - return ListType.of_optional(new_id, element_result.get()) - else: - return ListType.of_required(new_id, element_result.get()) - - def map(self, map_var, key_result, value_result): - new_key_id = self.next_id() - new_value_id = self.next_id() - - if map_var.is_value_optional(): - return MapType.of_optional(new_key_id, new_value_id, key_result(), value_result()) - else: - return MapType.of_required(new_key_id, new_value_id, key_result(), value_result()) - - def primitive(self, primitive_var): - return primitive_var - - -class CheckCompatibility(CustomOrderSchemaVisitor): - - @staticmethod - def write_compatibility_errors(read_schema, write_schema): - visit(read_schema, CheckCompatibility(write_schema, True)) - - @staticmethod - def read_compatibility_errors(read_schema, write_schema): - visit(write_schema, CheckCompatibility(read_schema, False)) - - NO_ERRORS: List[str] = [] - - def __init__(self, schema, check_ordering): - self.schema = schema - self.check_ordering = check_ordering - self.current_type = None - - def schema(self, schema, struct_result): - self.current_type = self.schema.as_struct() - try: - struct_result.get() - finally: - self.current_type = None - - def struct(self, struct, field_results): - if struct is None: - raise RuntimeError("Evaluation must start with a schema.") - - if not self.current_type.is_struct_type(): - return [": %s cannot be read as a struct" % self.current_type] - - errors = [] - - for field_errors in field_results: - errors = errors + field_errors - - if self.check_ordering: - new_struct = self.current_type.as_struct_type() - id_to_ord = {} - for i, val in enumerate(new_struct.fields): - id_to_ord[val.field_id] = i - - last_ordinal = -1 - - for read_field in self.struct.fields: - id_var = read_field.field_id - - field = struct.field(id=id_var) - if field is not None: - ordinal = id_to_ord[id] - if last_ordinal >= ordinal: - errors.append("%s is out of order before %s" % (read_field.name, - new_struct.fields[last_ordinal].name)) - last_ordinal = ordinal - - return errors - - def field(self, field, field_result) -> List[str]: - struct = self.current_type.as_struct_type() - curr_field = struct.field(field.field_id) - errors = [] - - if curr_field is None: - if not field.is_optional: - errors.append("{} is required, but is missing".format(field.name)) - return self.NO_ERRORS - - self.current_type = curr_field.type - - try: - if not field.is_optional and curr_field.is_optional: - errors.append(field.name + " should be required, but is optional") - - for error in field_result: - if error.startswith(":"): - errors.append("{}{}".format(field.field_name, error)) - else: - errors.append("{}.{}".format(field.field_name, error)) - - return errors - except RuntimeError: - pass - finally: - self.current_type = struct - return errors - - def list(self, list_var, element_result): - raise NotImplementedError() - - def map(self, map_var, key_result, value_result): - raise NotImplementedError() - - def primitive(self, primitive_var): - raise NotImplementedError() diff --git a/iceberg/api/types/types.py b/iceberg/api/types/types.py deleted file mode 100644 index b5957c2345..0000000000 --- a/iceberg/api/types/types.py +++ /dev/null @@ -1,726 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -from .type import (NestedType, - PrimitiveType, - TypeID) - - -class BooleanType(PrimitiveType): - __instance = None - - @staticmethod - def get(): - if BooleanType.__instance is None: - BooleanType() - return BooleanType.__instance - - def __init__(self): - if BooleanType.__instance is not None: - raise Exception("Multiple Boolean Types created") - BooleanType.__instance = self - - @property - def type_id(self): - return TypeID.BOOLEAN - - def __repr__(self): - return "boolean" - - def __str__(self): - return "boolean" - - -class IntegerType(PrimitiveType): - __instance = None - - @staticmethod - def get(): - if IntegerType.__instance is None: - IntegerType() - return IntegerType.__instance - - def __init__(self): - if IntegerType.__instance is not None: - raise Exception("Multiple Integer Types created") - IntegerType.__instance = self - - @property - def type_id(self): - return TypeID.INTEGER - - def __repr__(self): - return "int" - - def __str__(self): - return "int" - - -class LongType(PrimitiveType): - __instance = None - - @staticmethod - def get(): - if LongType.__instance is None: - LongType() - return LongType.__instance - - def __init__(self): - if LongType.__instance is not None: - raise Exception("Multiple Long Types created") - LongType.__instance = self - - @property - def type_id(self): - return TypeID.LONG - - def __repr__(self): - return "long" - - def __str__(self): - return "long" - - -class FloatType(PrimitiveType): - __instance = None - - @staticmethod - def get(): - if FloatType.__instance is None: - FloatType() - return FloatType.__instance - - def __init__(self): - if FloatType.__instance is not None: - raise Exception("Multiple Float Types created") - FloatType.__instance = self - - @property - def type_id(self): - return TypeID.FLOAT - - def __repr__(self): - return "float" - - def __str__(self): - return "float" - - -class DoubleType(PrimitiveType): - __instance = None - - @staticmethod - def get(): - if DoubleType.__instance is None: - DoubleType() - return DoubleType.__instance - - def __init__(self): - if DoubleType.__instance is not None: - raise Exception("Multiple Double Types created") - DoubleType.__instance = self - - @property - def type_id(self): - return TypeID.DOUBLE - - def __repr__(self): - return "double" - - def __str__(self): - return "double" - - -class DateType(PrimitiveType): - __instance = None - - @staticmethod - def get(): - if DateType.__instance is None: - DateType() - return DateType.__instance - - def __init__(self): - if DateType.__instance is not None: - raise Exception("Multiple Date Types created") - DateType.__instance = self - - @property - def type_id(self): - return TypeID.DATE - - def __repr__(self): - return "date" - - def __str__(self): - return "date" - - -class TimeType(PrimitiveType): - __instance = None - - @staticmethod - def get(): - if TimeType.__instance is None: - TimeType() - return TimeType.__instance - - def __init__(self): - if TimeType.__instance is not None: - raise Exception("Multiple Time Types created") - TimeType.__instance = self - - @property - def type_id(self): - return TypeID.TIME - - def __repr__(self): - return "time" - - def __str__(self): - return "time" - - -class TimestampType(PrimitiveType): - __instance_with_tz = None - __instance_without_tz = None - - @staticmethod - def with_timezone(): - if not TimestampType.__instance_with_tz: - TimestampType() - return TimestampType.__instance_with_tz - - @staticmethod - def without_timezone(): - if not TimestampType.__instance_without_tz: - TimestampType(False) - return TimestampType.__instance_without_tz - - def __init__(self, with_timezone=True): - self.adjust_to_utc = with_timezone - if (with_timezone and TimestampType.__instance_with_tz is not None)\ - or (not with_timezone and TimestampType.__instance_without_tz is not None): - raise Exception("Multiple Timestamp Types created") - - if with_timezone: - TimestampType.__instance_with_tz = self - else: - TimestampType.__instance_without_tz = self - - @property - def type_id(self): - return TypeID.TIMESTAMP - - def __eq__(self, other): - if id(self) == id(other): - return True - elif other is None or not isinstance(other, TimestampType): - return False - - return self.adjust_to_utc == other.adjust_to_utc - - def __hash__(self): - return hash(self.__key()) - - def __key(self): - return TimestampType.__class__, self.adjust_to_utc - - def __ne__(self, other): - return not self.__eq__(other) - - def __repr__(self): - if self.adjust_to_utc: - return "timestamptz" - else: - return "timestamp" - - def __str__(self): - return self.__repr__() - - -class StringType(PrimitiveType): - __instance = None - - @staticmethod - def get(): - if StringType.__instance is None: - StringType() - return StringType.__instance - - def __init__(self): - if StringType.__instance is not None: - raise Exception("Multiple String Types created") - StringType.__instance = self - - @property - def type_id(self): - return TypeID.STRING - - def __repr__(self): - return "string" - - def __str__(self): - return "string" - - -class UUIDType(PrimitiveType): - __instance = None - - @staticmethod - def get(): - if UUIDType.__instance is None: - UUIDType() - return UUIDType.__instance - - def __init__(self): - if UUIDType.__instance is not None: - raise Exception("Multiple UUID Types created") - UUIDType.__instance = self - - @property - def type_id(self): - return TypeID.UUID - - def __repr__(self): - return "uuid" - - def __str__(self): - return "uuid" - - -class FixedType(PrimitiveType): - - @staticmethod - def of_length(length): - return FixedType(length) - - def __init__(self, length): - self._length = length - - @property - def length(self): - return self._length - - @property - def type_id(self): - return TypeID.FIXED - - def __hash__(self): - return hash(self.__key()) - - def __eq__(self, other): - if id(self) == id(other): - return True - elif other is None or not isinstance(other, FixedType): - return False - - return self.length == other.length - - def __key(self): - return FixedType.__class__, self.length - - def __ne__(self, other): - return not self.__eq__(other) - - def __repr__(self): - return "fixed[%s]" % (self.length) - - def __str__(self): - return self.__repr__() - - -class BinaryType(PrimitiveType): - __instance = None - - @staticmethod - def get(): - if BinaryType.__instance is None: - BinaryType() - return BinaryType.__instance - - def __init__(self): - if BinaryType.__instance is not None: - raise Exception("Multiple Binary Types created") - BinaryType.__instance = self - - @property - def type_id(self): - return TypeID.BINARY - - def __repr__(self): - return "binary" - - def __str__(self): - return "binary" - - -class DecimalType(PrimitiveType): - - @staticmethod - def of(precision, scale): - return DecimalType(precision, scale) - - def __init__(self, precision, scale): - if int(precision) > 38: - raise RuntimeError("Decimals with precision larger than 38 are not supported: %s", precision) - self.precision = int(precision) - self.scale = int(scale) - - @property - def type_id(self): - return TypeID.DECIMAL - - def __repr__(self): - return "decimal(%s, %s)" % (self.precision, self.scale) - - def __str__(self): - return self.__repr__() - - def __eq__(self, other): - if id(self) == id(other): - return True - elif other is None or not isinstance(other, DecimalType): - return False - - return self.precision == other.precision and self.scale == other.scale - - def __ne__(self, other): - return not self.__eq__(other) - - def __hash__(self): - return hash(self.__key()) - - def __key(self): - return DecimalType.__class__, self.precision, self.scale - - -class NestedField(): - length: int - - @staticmethod - def optional(id, name, type_var, doc=None): - return NestedField(True, id, name, type_var, doc=doc) - - @staticmethod - def required(id, name, type, doc=None): - return NestedField(False, id, name, type, doc=doc) - - def __init__(self, is_optional, id, name, type, doc=None): - self.is_optional = is_optional - self.id = id - self.name = name - self.type = type - self.doc = doc - - @property - def is_required(self): - return not self.is_optional - - @property - def field_id(self): - return self.id - - def __repr__(self): - return "%s: %s: %s %s(%s)" % (self.id, - self.name, - "optional" if self.is_optional else "required", - self.type, - self.doc) - - def __str__(self): - return self.__repr__() - - def __eq__(self, other): - if id(self) == id(other): - return True - elif other is None or not isinstance(other, NestedField): - return False - - return self.is_optional == other.is_optional \ - and self.id == other.id \ - and self.name == other.name and self.type == other.type \ - and self.doc == other.doc - - def __ne__(self, other): - return not self.__eq__(other) - - def __hash__(self): - return hash(self.__key()) - - def __key(self): - type_name = self.type.type_id.name - return NestedField.__class__, self.is_optional, self.id, self.name, self.doc, type_name - - -class StructType(NestedType): - FIELD_SEP = ", " - - @staticmethod - def of(fields): - return StructType(fields) - - def __init__(self, fields): - if fields is None: - raise RuntimeError("Field list cannot be None") - - self._fields = list() - for i in range(0, len(fields)): - self._fields.append(fields[i]) - - self._fieldList = None - self._fieldsByName = None - self._fieldsByLowercaseName = None - self._fieldsById = None - - def __eq__(self, other): - if id(self) == id(other): - return True - elif other is None or not isinstance(other, StructType): - return False - - return self._fields == other._fields - - def __ne__(self, other): - return not self.__eq__(other) - - @property - def fields(self): - return self._lazy_field_list() - - def field(self, name=None, id=None): - if name: - return self._lazy_fields_by_name().get(name) - elif id: - return self._lazy_fields_by_id()[id] - - raise RuntimeError("No valid field info passed in ") - - def case_insensitive_field(self, name): - return self._lazy_fields_by_lowercase_name().get(name) - - @property - def type_id(self): - return TypeID.STRUCT - - def is_struct_type(self): - return True - - def as_struct_type(self): - return self - - def __str__(self): - return "struct<{}>".format(StructType.FIELD_SEP.join(str(x) for x in self.fields)) - - def __hash__(self): - return hash(self.__key()) - - def __key(self): - return StructType.__class__, self.fields - - def _lazy_field_list(self): - if self._fieldList is None: - self._fieldList = tuple(self._fields) - return self._fieldList - - def _lazy_fields_by_name(self): - if self._fieldsByName is None: - self.index_fields() - return self._fieldsByName - - def _lazy_fields_by_lowercase_name(self): - if self._fieldsByName is None: - self.index_fields() - return self._fieldsByName - - def _lazy_fields_by_id(self): - if self._fieldsById is None: - self.index_fields() - return self._fieldsById - - def index_fields(self): - self._fieldsByName = dict() - self._fieldsByLowercaseName = dict() - self._fieldsById = dict() - - for field in self.fields: - self._fieldsByName[field.name] = field - self._fieldsByLowercaseName[field.name.lower()] = field - self._fieldsById[field.id] = field - - -class ListType(NestedType): - @staticmethod - def of_optional(element_id, element_type): - if element_type is None: - raise RuntimeError("Element type cannot be null") - return ListType(NestedField.optional(element_id, "element", element_type)) - - @staticmethod - def of_required(element_id, element_type): - if element_type is None: - raise RuntimeError("Element type cannot be null") - return ListType(NestedField.required(element_id, "element", element_type)) - - def __init__(self, element_field): - self.element_field = element_field - self._fields = None - - @property - def type_id(self): - return TypeID.LIST - - @property - def element_type(self): - return self.element_field.type - - def field_type(self, name): - if "element" == name: - return self.element_type - - def field(self, id): - if self.element_field.id == id: - return self.element_field - - def fields(self): - return self._lazyFieldsList() - - @property - def element_id(self): - return self.element_field.id - - def is_element_required(self): - return not self.element_field.is_optional - - def is_element_optional(self): - return self.element_field.is_optional - - def is_list_type(self): - return True - - def as_list_type(self): - return self - - def __str__(self): - return "list<%s>" % self.element_field.type - - def __eq__(self, other): - if id(self) == id(other): - return True - elif other is None or not isinstance(other, ListType): - return False - - return self.element_field == other.element_field - - def __ne__(self, other): - return self.__eq__(other) - - def __hash__(self): - return hash(self.__key()) - - def __key(self): - return StructType.__class__, self.element_field - - def _lazyFieldsList(self): - if self._fields is None: - self._fields = [self.element_field] - - return self._fields - - -class MapType(NestedType): - - @staticmethod - def of_optional(key_id, value_id, key_type, value_type): - if value_type is None: - raise RuntimeError("Value type cannot be null") - - return MapType(NestedField.required(key_id, 'key', key_type), - NestedField.optional(value_id, 'value', value_type)) - - @staticmethod - def of_required(key_id, value_id, key_type, value_type): - if value_type is None: - raise RuntimeError("Value type cannot be null") - - return MapType(NestedField.required(key_id, 'key', key_type), - NestedField.required(value_id, 'value', value_type)) - - def __init__(self, key_field, value_field): - self.key_field = key_field - self.value_field = value_field - self._fields = None - - @property - def type_id(self): - return TypeID.MAP - - def key_type(self): - return self.key_field.type - - def value_type(self): - return self.value_field.type - - def field_type(self, name): - if "key" == name: - return self.key_field.type - elif "value" == name: - return self.value_field.type - - def field(self, id): - if self.key_field.id == id: - return self.key_field - elif self.value_field.id == id: - return self.value_field - - def fields(self): - return self._lazy_field_list() - - def key_id(self): - return self.key_field.field_id - - def value_id(self): - return self.value_field.field_id - - def is_value_optional(self): - return self.value_field.is_optional - - def is_value_required(self): - return not self.is_value_optional() - - def __str__(self): - return "map<%s, %s>" % (self.key_field.type, self.value_field.type) - - def __eq__(self, other): - if id(self) == id(other): - return True - elif other is None or not isinstance(other, MapType): - return False - - return self.key_field == other.key_field and self.value_field == other.value_field - - def __ne__(self, other): - return not self.__eq__(other) - - def __hash__(self): - return hash(self.__key()) - - def __key(self): - return MapType.__class__, self.key_field, self.value_field - - def _lazy_field_list(self): - return tuple(self.key_field, self.value_field) diff --git a/iceberg/api/update_properties.py b/iceberg/api/update_properties.py deleted file mode 100644 index 763bb06bcd..0000000000 --- a/iceberg/api/update_properties.py +++ /dev/null @@ -1,30 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -from .pending_update import PendingUpdate - - -class UpdateProperties(PendingUpdate): - - def set(self, key, value): - raise NotImplementedError() - - def remove(self, key): - raise NotImplementedError() - - def default_format(self, file_format): - raise NotImplementedError() diff --git a/iceberg/api/update_schema.py b/iceberg/api/update_schema.py deleted file mode 100644 index 22ef05e64a..0000000000 --- a/iceberg/api/update_schema.py +++ /dev/null @@ -1,33 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -from .pending_update import PendingUpdate - - -class UpdateSchema(PendingUpdate): - - def add_column(self, name, type, parent=None): - raise NotImplementedError() - - def rename_column(self, name, new_name): - raise NotImplementedError() - - def update_column(self, name, new_type): - raise NotImplementedError() - - def delete_column(self, name): - raise NotImplementedError() diff --git a/iceberg/core/__init__.py b/iceberg/core/__init__.py deleted file mode 100644 index d6079a5816..0000000000 --- a/iceberg/core/__init__.py +++ /dev/null @@ -1,61 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. -# flake8: noqa - -__all__ = ["BaseMetastoreTableOperations", - "BaseMetastoreTables", - "BaseSnapshot", - "BaseTable", - "ConfigProperties", - "DataFiles", - "GenericDataFile", - "GenericManifestFile", - "ManifestEntry", - "ManifestListWriter", - "ManifestReader", - "PartitionSpecParser", - "PartitionData", - "SchemaParser", - "SchemaUpdate", - "SnapshotParser", - "SnapshotLogEntry", - "TableMetadata", - "TableMetadataParser", - "TableOperations", - "TableProperties"] - -from .table_operations import TableOperations # out of order import to avoid circular deps -from .base_metastore_table_operations import BaseMetastoreTableOperations -from .base_metastore_tables import BaseMetastoreTables -from .base_snapshot import BaseSnapshot -from .base_table import BaseTable -from .config_properties import ConfigProperties -from .data_files import DataFiles -from .generic_data_file import GenericDataFile -from .generic_manifest_file import GenericManifestFile -from .manifest_entry import ManifestEntry -from .manifest_list_writer import ManifestListWriter -from .manifest_reader import ManifestReader -from .partition_data import PartitionData -from .partition_spec_parser import PartitionSpecParser -from .schema_parser import SchemaParser -from .schema_update import SchemaUpdate -from .snapshot_parser import SnapshotParser -from .table_metadata import (SnapshotLogEntry, - TableMetadata) -from .table_metadata_parser import TableMetadataParser -from .table_properties import TableProperties diff --git a/iceberg/core/avro/__init__.py b/iceberg/core/avro/__init__.py deleted file mode 100644 index 3d07a5b387..0000000000 --- a/iceberg/core/avro/__init__.py +++ /dev/null @@ -1,21 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -__all__ = ["AvroToIceberg", "IcebergToAvro"] - -from .avro_to_iceberg import AvroToIceberg -from .iceberg_to_avro import IcebergToAvro diff --git a/iceberg/core/avro/avro_schema_util.py b/iceberg/core/avro/avro_schema_util.py deleted file mode 100644 index f513a9cf8c..0000000000 --- a/iceberg/core/avro/avro_schema_util.py +++ /dev/null @@ -1,35 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - - -class AvroSchemaUtil(object): - FIELD_ID_PROP = "field-id" - KEY_ID_PROP = "key-id" - VALUE_ID_PROP = "value-id" - ELEMENT_ID_PROP = "element-id" - ADJUST_TO_UTC_PROP = "adjust-to_utc" - - # NULL = PrimitiveSchema(NULL) - - @staticmethod - def convert(iceberg_schema=None, avro_schema=None, table_name=None, names=None, - type_var=None, name=None): - if iceberg_schema is not None and table_name is not None: - return AvroSchemaUtil.convert(iceberg_schema=iceberg_schema, - names={iceberg_schema.as_struct(): table_name}) - elif iceberg_schema is not None and names is not None: - raise RuntimeError("Not yet implemented") diff --git a/iceberg/core/avro/avro_to_iceberg.py b/iceberg/core/avro/avro_to_iceberg.py deleted file mode 100644 index 14dc9ce9d4..0000000000 --- a/iceberg/core/avro/avro_to_iceberg.py +++ /dev/null @@ -1,301 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -import fastavro -from iceberg.api import Schema -from iceberg.api.types import (BinaryType, - BooleanType, - DateType, - DoubleType, - FloatType, - IntegerType, - ListType, - LongType, - MapType, - NestedField, - StringType, - StructType, - TimestampType, - TimeType, - TypeID) - - -class AvroToIceberg(object): - FIELD_ID_PROP = "field-id" - FIELD_TYPE_PROP = "type" - FIELD_NAME_PROP = "name" - FIELD_LOGICAL_TYPE_PROP = "logicalType" - FIELD_FIELDS_PROP = "fields" - FIELD_ITEMS_PROP = "items" - FIELD_ELEMENT_ID_PROP = "element-id" - - AVRO_JSON_PRIMITIVE_TYPES = ("boolean", "int", "long", "float", "double", "bytes", "string") - AVRO_JSON_COMPLEX_TYPES = ("record", "array", "enum", "fixed") - - TYPE_PROCESSING_MAP = {str: lambda x, y: AvroToIceberg.convert_str_type(x, y), - dict: lambda x, y: AvroToIceberg.convert_complex_type(x, y), - list: lambda x, y: AvroToIceberg.convert_union_type(x, y)} - - COMPLEX_TYPE_PROCESSING_MAP = {"record": lambda x, y: AvroToIceberg.convert_record_type(x, y), - "array": lambda x, y: AvroToIceberg.convert_array_type(x, y), - "map": lambda x, y: AvroToIceberg.convert_map_type(x, y)} - - PRIMITIVE_FIELD_TYPE_MAP = {"boolean": BooleanType.get(), - "bytes": BinaryType.get(), - "date": DateType.get(), - "double": DoubleType.get(), - "float": FloatType.get(), - "int": IntegerType.get(), - "long": LongType.get(), - "string": StringType.get(), - "time-millis": TimeType.get(), - "timestamp-millis": TimestampType.without_timezone()} - - PROCESS_FUNCS = {TypeID.STRUCT: lambda avro_row, field: AvroToIceberg.get_field_from_struct(avro_row, field), - TypeID.LIST: lambda avro_row, field: AvroToIceberg.get_field_from_list(avro_row, field), - TypeID.MAP: lambda avro_row, field: AvroToIceberg.get_field_from_map(avro_row, field)} - - @staticmethod - def convert_avro_schema_to_iceberg(avro_schema): - if avro_schema.get(AvroToIceberg.FIELD_TYPE_PROP) != "record": - raise RuntimeError("Cannot convert avro schema to iceberg %s" % avro_schema) - - struct = AvroToIceberg.convert_type(avro_schema, None) - - return Schema(struct[0].fields) - - @staticmethod - def convert_record_type(avro_field, next_id=None): - avro_field_type = avro_field.get(AvroToIceberg.FIELD_TYPE_PROP) - - if avro_field_type != "record": - raise RuntimeError("Field type muse be 'record': %s" % avro_field_type) - - fields = avro_field.get(AvroToIceberg.FIELD_FIELDS_PROP) - - iceberg_fields = [] - if next_id is None: - next_id = len(fields) - for field in fields: - iceberg_field, next_id = AvroToIceberg.convert_avro_field_to_iceberg(field, next_id=next_id) - iceberg_fields.append(iceberg_field) - - return StructType.of(iceberg_fields), next_id - - @staticmethod - def convert_avro_field_to_iceberg(field, next_id): - field_type, is_optional, next_id = AvroToIceberg.convert_type(field, next_id) - - if field.get(AvroToIceberg.FIELD_ID_PROP) is None: - return field_type, next_id - - if is_optional: - return NestedField.optional(field.get(AvroToIceberg.FIELD_ID_PROP), - field.get(AvroToIceberg.FIELD_NAME_PROP), - field_type), next_id - else: - return NestedField.required(field.get(AvroToIceberg.FIELD_ID_PROP), - field.get(AvroToIceberg.FIELD_NAME_PROP), - field_type), next_id - - @staticmethod - def convert_type(field, next_id=None): - avro_field_type = field.get(AvroToIceberg.FIELD_TYPE_PROP) - - optional = AvroToIceberg.is_option_schema(avro_field_type) - - processing_func = AvroToIceberg.TYPE_PROCESSING_MAP.get(type(avro_field_type)) - if processing_func is None: - raise RuntimeError("No function found to process %s" % avro_field_type) - - iceberg_type, next_id = processing_func(field, next_id) - - return iceberg_type, optional, next_id - - @staticmethod - def convert_str_type(avro_field, next_id=None): - avro_field_type = avro_field.get(AvroToIceberg.FIELD_TYPE_PROP) - logical_type = avro_field.get(AvroToIceberg.FIELD_LOGICAL_TYPE_PROP) - if not isinstance(avro_field_type, str): - raise RuntimeError("Field type must be of type str: %s" % avro_field_type) - - if avro_field_type in AvroToIceberg.AVRO_JSON_PRIMITIVE_TYPES: - if logical_type is not None: - return AvroToIceberg.PRIMITIVE_FIELD_TYPE_MAP.get(logical_type), next_id - else: - return AvroToIceberg.PRIMITIVE_FIELD_TYPE_MAP.get(avro_field_type), next_id - - elif avro_field_type in AvroToIceberg.AVRO_JSON_COMPLEX_TYPES: - if logical_type is not None: - processing_func = AvroToIceberg.COMPLEX_TYPE_PROCESSING_MAP.get(logical_type) - else: - processing_func = AvroToIceberg.COMPLEX_TYPE_PROCESSING_MAP.get(avro_field_type) - - if processing_func is None: - raise RuntimeError("No function found to process %s" % avro_field_type) - - return processing_func(avro_field, next_id) - else: - raise RuntimeError("Unknown type %s" % avro_field_type) - - @staticmethod - def convert_complex_type(avro_field, next_id=None): - avro_field_type = avro_field.get(AvroToIceberg.FIELD_TYPE_PROP) - if not isinstance(avro_field_type, dict): - raise RuntimeError("Complex field type must be of type dict: %s" % avro_field_type) - - return AvroToIceberg.convert_avro_field_to_iceberg(avro_field_type, next_id) - - @staticmethod - def convert_union_type(avro_field, next_id=None): - avro_field_type = avro_field.get(AvroToIceberg.FIELD_TYPE_PROP) - if not isinstance(avro_field_type, list): - raise RuntimeError("Union field type must be of type list: %s" % avro_field_type) - - if len(avro_field_type) > 2: - raise RuntimeError("Cannot process unions larger than 2 items: %s" % avro_field_type) - for item in avro_field_type: - if isinstance(item, str) and item == "null": - continue - avro_field_type = item - avro_field[AvroToIceberg.FIELD_TYPE_PROP] = avro_field_type - items = AvroToIceberg.convert_type(avro_field, next_id) - return items[0], items[2] - - @staticmethod - def convert_array_type(avro_field, next_id=None): - avro_field_type = avro_field.get(AvroToIceberg.FIELD_TYPE_PROP) - if avro_field_type != "array": - raise RuntimeError("Avro type must be array: %s" % avro_field_type) - element_id = avro_field.get(AvroToIceberg.FIELD_ELEMENT_ID_PROP) - items = avro_field.get(AvroToIceberg.FIELD_ITEMS_PROP) - - is_optional = AvroToIceberg.is_option_schema(items) - - if isinstance(items, str) and items in AvroToIceberg.PRIMITIVE_FIELD_TYPE_MAP: - item_type = AvroToIceberg.PRIMITIVE_FIELD_TYPE_MAP.get(items) - if item_type is None: - raise RuntimeError("No mapping found for type %s" % items) - else: - raise RuntimeError("Complex list types not yet implemented") - - if is_optional: - return ListType.of_optional(element_id, item_type), next_id - else: - return ListType.of_required(element_id, item_type), next_id - - @staticmethod - def convert_map_type(avro_field, next_id=None): - avro_field_type = avro_field.get(AvroToIceberg.FIELD_TYPE_PROP) - avro_logical_type = avro_field.get(AvroToIceberg.FIELD_LOGICAL_TYPE_PROP) - if avro_field_type != "array" or avro_logical_type != "map": - raise RuntimeError("Avro type must be array and logical type must be map: %s" % avro_logical_type) - is_optional = False - items = avro_field.get(AvroToIceberg.FIELD_ITEMS_PROP) - for field in items.get(AvroToIceberg.FIELD_FIELDS_PROP, list()): - if field.get(AvroToIceberg.FIELD_NAME_PROP) == "key": - key_id = field.get(AvroToIceberg.FIELD_ID_PROP) - if not isinstance(field.get(AvroToIceberg.FIELD_TYPE_PROP), str): - raise RuntimeError("Support for complex map keys not yet implemented") - key_type = AvroToIceberg.PRIMITIVE_FIELD_TYPE_MAP.get(field.get(AvroToIceberg.FIELD_TYPE_PROP)) - elif field.get(AvroToIceberg.FIELD_NAME_PROP) == "value": - value_id = field.get(AvroToIceberg.FIELD_ID_PROP) - if not isinstance(field.get(AvroToIceberg.FIELD_TYPE_PROP), str): - raise RuntimeError("Support for complex map values not yet imeplemented") - value_type = AvroToIceberg.PRIMITIVE_FIELD_TYPE_MAP.get(field.get(AvroToIceberg.FIELD_TYPE_PROP)) - - if is_optional: - return MapType.of_optional(key_id, value_id, key_type, value_type), next_id - else: - return MapType.of_required(key_id, value_id, key_type, value_type), next_id - - @staticmethod - def is_option_schema(field_type): - if isinstance(field_type, list) and len(field_type) == 2 and "null" in field_type: - return True - - return False - - @staticmethod - def read_avro_file(iceberg_schema, data_file): - fo = data_file.new_fo() - avro_reader = fastavro.reader(fo) - for avro_row in avro_reader: - iceberg_row = dict() - for field in iceberg_schema.as_struct().fields: - iceberg_row[field.name] = AvroToIceberg.get_field_from_avro(avro_row, field) - yield iceberg_row - fo.close() - - @staticmethod - def read_avro_row(iceberg_schema, avro_reader): - try: - for avro_row in avro_reader: - iceberg_row = dict() - for field in iceberg_schema.as_struct().fields: - iceberg_row[field.name] = AvroToIceberg.get_field_from_avro(avro_row, field) - yield iceberg_row - except StopIteration: - return - - @staticmethod - def get_field_from_avro(avro_row, field): - try: - return AvroToIceberg.PROCESS_FUNCS.get(field.type.type_id, - AvroToIceberg.get_field_from_primitive)(avro_row, field) - except KeyError: - raise RuntimeError("Don't know how to get field of type: %s" % field.type.type_id) - - @staticmethod - def get_field_from_primitive(avro_row, field): - try: - return avro_row[field.name] - except KeyError: - if field.is_required: - raise RuntimeError("Field is required but missing in source %s\n%s:" % (field, avro_row)) - - @staticmethod - def get_field_from_struct(avro_row, field): - field_obj = {} - for nested_field in field.type.fields: - field_obj[nested_field.name] = AvroToIceberg.get_field_from_avro(avro_row[field.name], nested_field) - return field_obj - - @staticmethod - def get_field_from_list(avro_row, field): - try: - return avro_row[field.name] - except KeyError: - if field.is_required: - raise RuntimeError("Field is required but missing in source %s\n%s:" % (field, avro_row)) - - @staticmethod - def get_field_from_map(avro_row, field): - val_map = dict() - - try: - avro_value = avro_row[field.name] - except KeyError: - if field.is_required: - raise RuntimeError("Field is required but missing in source %s\n%s:" % (field, avro_row)) - else: - return None - - for val in avro_value: - val_map[val['key']] = val['value'] - - return val_map diff --git a/iceberg/core/avro/iceberg_to_avro.py b/iceberg/core/avro/iceberg_to_avro.py deleted file mode 100644 index 29e4330b1b..0000000000 --- a/iceberg/core/avro/iceberg_to_avro.py +++ /dev/null @@ -1,90 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -from iceberg.api.types import BinaryType, TypeID - - -class IcebergToAvro(object): - - @staticmethod - def type_to_schema(struct_type, name): - struct_fields = list() - for field in struct_type.fields: - struct_fields.append({"field-id": field.id, - "name": field.name, - "type": IcebergToAvro.get_field(field)}) - return {"type": "record", - "name": name, - "fields": struct_fields} - - @staticmethod - def get_field(field): - if field.type.is_primitive_type(): - return IcebergToAvro.to_option(field) - - elif field.type.type_id == TypeID.STRUCT: - struct_fields = list() - for struct_field in field.type.fields: - field_dict = {"field-id": struct_field.id, - "name": struct_field.name, - "type": IcebergToAvro.get_field(struct_field)} - if struct_field.is_optional: - field_dict["default"] = None - - struct_fields.append(field_dict) - - return {"fields": struct_fields, - "name": field.name, - "type": "record"} - - elif field.type.type_id == TypeID.LIST: - array_obj = {'element-id': field.type.element_id, - "items": IcebergToAvro.get_field(field.type.element_field), - "type": 'array'} - if field.is_optional: - return ['null', array_obj] - return array_obj - - elif field.type.type_id == TypeID.MAP: - key = field.type.key_field - value = field.type.value_field - array_obj = {"items": {"fields": [{"field-id": key.field_id, - "name": key.name, - "type": IcebergToAvro.get_field(key)}, - {"field-id": value.field_id, - "name": value.name, - "type": IcebergToAvro.get_field(value)}], - "name": "k{}_v{}".format(key.field_id, value.field_id), - "type": "record"}, - "logicalType": "map", - "type": "array"} - if field.is_optional: - return ["null", array_obj] - - return array_obj - - @staticmethod - def to_option(field): - if field.type == BinaryType.get(): - type_name = "bytes" - else: - type_name = str(field.type) - - if field.is_optional: - return ["null", type_name] - else: - return type_name diff --git a/iceberg/core/base_combined_scan_task.py b/iceberg/core/base_combined_scan_task.py deleted file mode 100644 index 8cb209e2f9..0000000000 --- a/iceberg/core/base_combined_scan_task.py +++ /dev/null @@ -1,37 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -from copy import deepcopy - -from iceberg.api import CombinedScanTask - - -class BaseCombinedScanTask(CombinedScanTask): - - def __init__(self, tasks): - self.tasks = deepcopy(tasks) - - @property - def files(self): - return self.tasks - - def __repr__(self): - return "BaseCombinedScanTask([{}])".format(self.tasks) - - def __str__(self): - total_size = sum([task.length for task in self.tasks]) - return "BaseCombinedScanTask(num_tasks={}, total_size={})".format(len(self.tasks), total_size) diff --git a/iceberg/core/base_file_scan_task.py b/iceberg/core/base_file_scan_task.py deleted file mode 100644 index f36fcf86f7..0000000000 --- a/iceberg/core/base_file_scan_task.py +++ /dev/null @@ -1,126 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -from iceberg.api import FileScanTask - -from .partition_spec_parser import PartitionSpecParser -from .schema_parser import SchemaParser - - -class BaseFileScanTask(FileScanTask): - - def __init__(self, file, schema_str, spec_str, residuals): - self._file = file - self._schema_str = schema_str - self._spec_str = spec_str - self._spec = None - self._residuals = residuals - - @property - def file(self): - return self._file - - @property - def spec(self): - if self._spec is None: - self._spec = PartitionSpecParser.from_json(SchemaParser.from_json(self._schema_str), self._spec_str) - - return self._spec - - @property - def start(self): - return 0 - - @property - def length(self): - return self._file.file_size_in_bytes() - - @property - def residual(self): - return self._residuals.residual_for(self._file.partition()) - - def split(self, split_size): - if self.file.format().is_splittable(): - return [task for task in SplitScanTaskIterator(split_size, self)] - else: - return self - - def __repr__(self): - fields = ["file: {}".format(self._file.path()), - "partition_data: {}".format(self._file.partition()), - "residual: {}".format(self.residual)] - - return "BaseFileScanTask({})".format(", ".join(fields)) - - def __str__(self): - return self.__repr__() - - -class SplitScanTaskIterator(object): - - def __init__(self, split_size, file_scan_task): - self._offset = 0 - self._remaining_len = file_scan_task.length - self._split_size = split_size - self._file_scan_task = file_scan_task - - def has_next(self): - return self._remaining_len > 0 - - def __iter__(self): - return self - - def __next__(self): - if self.has_next(): - len = min(self._split_size, self._remaining_len) - split_task = SplitScanTask(self._offset, len, self._file_scan_task) - self._offset += len - self._remaining_len -= len - return split_task - - raise StopIteration - - -class SplitScanTask(FileScanTask): - - def __init__(self, offset, len, file_scan_task): - self._offset = offset - self._len = len - self._file_scan_task = file_scan_task - - @property - def file(self): - return self._file_scan_task.file - - @property - def spec(self): - return self._file_scan_task.spec - - @property - def start(self): - return self._offset - - @property - def length(self): - return self._len - - @property - def residual(self): - return self._file_scan_task.residual() - - def split(self): - raise RuntimeError("Cannot split a task which is already split") diff --git a/iceberg/core/base_metastore_table_operations.py b/iceberg/core/base_metastore_table_operations.py deleted file mode 100644 index 97bdae2e0c..0000000000 --- a/iceberg/core/base_metastore_table_operations.py +++ /dev/null @@ -1,137 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -import logging -import uuid - -from retrying import retry - -from .table_metadata_parser import TableMetadataParser -from .table_operations import TableOperations -from .table_properties import TableProperties - -_logger = logging.getLogger(__name__) - - -class BaseMetastoreTableOperations(TableOperations): - - TABLE_TYPE_PROP = "table_type" - ICEBERG_TABLE_TYPE_VALUE = "iceberg" - METADATA_LOCATION_PROP = "metadata_location" - PARTITION_SPEC_PROP = "partition_spec" - PREVIOUS_METADATA_LOCATION_PROP = "previous_metadata_location" - - METADATA_FOLDER_NAME = "metadata" - DATA_FOLDER_NAME = "data" - HIVE_LOCATION_FOLDER_NAME = "empty" - - def __init__(self, conf): - self.conf = conf - - self.current_metadata = None - self.current_metadata_location = None - self.base_location = None - self.should_refresh = True - self.version = -1 - - def current(self): - return self.current_metadata - - def hive_table_location(self): - return "{base_location}/{hive}".format(base_location=self.base_location, - hive=BaseMetastoreTableOperations.HIVE_LOCATION_FOLDER_NAME) - - def data_location(self): - return "{base_location}/{data}".format(base_location=self.base_location, - data=BaseMetastoreTableOperations.DATA_FOLDER_NAME) - - def request_refresh(self): - self.should_refresh = True - - def write_new_metadata(self, metadata, version): - from .filesystem import FileSystemOutputFile - - if self.base_location is None: - self.base_location = metadata.location - - new_filename = BaseMetastoreTableOperations.new_table_metadata_filename(self.base_location, - version) - new_metadata_location = FileSystemOutputFile.from_path(new_filename, self.conf) - - TableMetadataParser.write(metadata, new_metadata_location) - return new_filename - - def refresh_from_metadata_location(self, new_location, num_retries=20): - if not self.current_metadata_location == new_location: - _logger.info("Refreshing table metadata from new version: %s" % new_location) - self.retryable_refresh(new_location) - - self.should_refresh = False - - def new_input_file(self, path): - from .filesystem import FileSystemInputFile - - return FileSystemInputFile.from_location(path, self.conf) - - def new_metadata_file(self, filename): - from .filesystem import FileSystemOutputFile - - return FileSystemOutputFile.from_path(BaseMetastoreTableOperations.new_metadata_location(self.base_location, - filename), - self.conf) - - def metadata_file_location(self, file_name, metadata=None): - if metadata is None: - return self.metadata_file_location(file_name, metadata=self.current()) - - metadata_location = metadata.properties.get(TableProperties.WRITE_METADATA_LOCATION) - - if metadata_location is not None: - return "{}/{}".format(metadata_location, file_name) - else: - return "{}/{}/{}".format(metadata.location, BaseMetastoreTableOperations.METADATA_FOLDER_NAME, file_name) - - def delete_file(self, path): - from .filesystem import get_fs - get_fs(path, self.conf).delete(path) - - @retry(wait_incrementing_start=100, wait_exponential_multiplier=4, - wait_exponential_max=5000, stop_max_delay=600000, stop_max_attempt_number=2) - def retryable_refresh(self, location): - from .filesystem import FileSystemInputFile - - self.current_metadata = TableMetadataParser.read(self, FileSystemInputFile.from_location(location, self.conf)) - self.current_metadata_location = location - self.base_location = self.current_metadata.location - self.version = BaseMetastoreTableOperations.parse_version(location) - - @staticmethod - def parse_version(metadata_location): - version_start = metadata_location.rfind("/") + 1 - version_end = version_start + metadata_location[version_start:].find("-") - return int(metadata_location[version_start:version_end]) - - @staticmethod - def new_metadata_location(base_location, filename): - return "{}/{}/{}".format(base_location, BaseMetastoreTableOperations.METADATA_FOLDER_NAME, filename) - - @staticmethod - def new_table_metadata_filename(base_location, new_version): - return "{}/{}/{}-{}.metadata.json".format(base_location, - BaseMetastoreTableOperations.METADATA_FOLDER_NAME, - '%05d' % new_version, - uuid.uuid4()) diff --git a/iceberg/core/base_metastore_tables.py b/iceberg/core/base_metastore_tables.py deleted file mode 100644 index a78041c650..0000000000 --- a/iceberg/core/base_metastore_tables.py +++ /dev/null @@ -1,85 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. -from typing import Tuple - -from . import TableOperations -from .base_table import BaseTable -from .table_metadata import TableMetadata -from ..api import PartitionSpec, Schema, Table, Tables -from ..exceptions import AlreadyExistsException, CommitFailedException, NoSuchTableException - - -class BaseMetastoreTables(Tables): - - def __init__(self: "BaseMetastoreTables", conf: dict) -> None: - self.conf = conf - - def new_table_ops(self: "BaseMetastoreTables", conf: dict, database: str, table: str) -> "TableOperations": - raise RuntimeError("Abstract Implementation") - - def load(self: "BaseMetastoreTables", table_identifier: str) -> Table: - database, table = _parse_table_identifier(table_identifier) - ops = self.new_table_ops(self.conf, database, table) - if ops.current(): - return BaseTable(ops, "{}.{}".format(database, table)) - raise NoSuchTableException("Table does not exist: {}.{}".format(database, table)) - - def create(self: "BaseMetastoreTables", schema: Schema, table_identifier: str, spec: PartitionSpec = None, - properties: dict = None, location: str = None) -> Table: - database, table = _parse_table_identifier(table_identifier) - ops = self.new_table_ops(self.conf, database, table) - if ops.current(): # not None check here to ensure MagicMocks aren't treated as None - raise AlreadyExistsException("Table already exists: " + table_identifier) - - base_location = location if location else self.default_warehouse_location(self.conf, database, table) - full_spec, properties = super(BaseMetastoreTables, self).default_args(spec, properties) - metadata = TableMetadata.new_table_metadata(ops, schema, full_spec, base_location, properties) - - try: - ops.commit(None, metadata) - except CommitFailedException: - raise AlreadyExistsException("Table was created concurrently: " + table_identifier) - - return BaseTable(ops, "{}.{}".format(database, table)) - - def begin_create(self: "BaseMetastoreTables", schema: Schema, spec: PartitionSpec, database: str, table_name: str, - properties: dict = None): - raise RuntimeError("Not Yet Implemented") - - def begin_replace(self: "BaseMetastoreTables", schema: Schema, spec: PartitionSpec, database: str, table: str, - properties: dict = None): - raise RuntimeError("Not Yet Implemented") - - def default_warehouse_location(self: "BaseMetastoreTables", conf: dict, database: str, table: str) -> str: - warehouse_location = conf.get("hive.metastore.warehouse.dir") - if warehouse_location: - return f"{warehouse_location}/{database}.db/{table}" - raise RuntimeError("Warehouse location is not set: hive.metastore.warehouse.dir=null") - - -_DOT = '.' - - -def _parse_table_identifier(table_identifier: str) -> Tuple[str, str]: - parts = table_identifier.rsplit(_DOT, 1) - if len(parts) > 1: - database = parts[0] - table = parts[1] - else: - database = "default" - table = parts[0] - return database, table diff --git a/iceberg/core/base_snapshot.py b/iceberg/core/base_snapshot.py deleted file mode 100644 index c849825f6e..0000000000 --- a/iceberg/core/base_snapshot.py +++ /dev/null @@ -1,132 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -import time - -from iceberg.api import (Filterable, - FilteredSnapshot, - ManifestFile, - Snapshot, - SnapshotIterable) -from iceberg.api.expressions import Expressions -from iceberg.api.io import CloseableGroup -from iceberg.core.avro import AvroToIceberg - -from .generic_manifest_file import GenericManifestFile -from .manifest_reader import ManifestReader - - -class BaseSnapshot(Snapshot, SnapshotIterable, CloseableGroup): - - @staticmethod - def snapshot_from_files(ops, snapshot_id, files): - return BaseSnapshot(ops, snapshot_id, None, - manifests=[GenericManifestFile(file=ops.new_input_file(path), spec_id=0) - for path in files]) - - def __init__(self, ops, snapshot_id, parent_id=None, manifests=None, manifest_list=None, timestamp_millis=None, - operation=None, summary=None): - super(BaseSnapshot, self).__init__() - if timestamp_millis is None: - timestamp_millis = int(time.time() * 1000) - - self._ops = ops - self._snapshot_id = snapshot_id - self._parent_id = parent_id - self._timestamp_millis = timestamp_millis - if manifests is not None: - self._manifests = [manifest if isinstance(manifest, GenericManifestFile) - else GenericManifestFile(file=ops.new_input_file(manifest), spec_id=0) - for manifest in manifests] - else: - self._manifests = None - self._manifest_list = manifest_list - self._operation = operation - self._summary = summary - - self._adds = None - self._deletes = None - - @property - def snapshot_id(self): - return self._snapshot_id - - @property - def timestamp_millis(self): - return self._timestamp_millis - - @property - def parent_id(self): - return self._parent_id - - @property - def manifests(self): - if self._manifests is None: - # if manifest isn't set then the snapshot_file is set and should be read to get the list - return (GenericManifestFile.from_avro_record_json(manifest) - for manifest in AvroToIceberg.read_avro_file(ManifestFile.schema(), self._manifest_list)) - - return self._manifests - - @property - def manifest_location(self): - return self._manifest_list.location() if self._manifest_list is not None else None - - @property - def summary(self): - return self._summary - - @property - def operation(self): - return self._operation - - def select(self, columns): - return FilteredSnapshot(self, Expressions.always_true(), Expressions.always_true(), columns) - - def filter_partitions(self, expr): - return FilteredSnapshot(self, expr, Expressions.always_true(), Filterable.ALL_COLUMNS) - - def filter_rows(self, expr): - return FilteredSnapshot(self, Expressions.always_true(), expr, Filterable.ALL_COLUMNS) - - def iterator(self, part_filter=None, row_filter=None, columns=None): - if part_filter is None and row_filter is None and columns is None: - return self.iterator(Expressions.always_true(), Expressions.always_true(), Filterable.ALL_COLUMNS) - return iter([self.get_filtered_manifest(path, part_filter, row_filter, columns) - for path in self._manifest_files]) - - def added_files(self): - raise NotImplementedError() - - def deleted_files(self): - raise NotImplementedError() - - def cache_changes(self): - raise NotImplementedError - - def __repr__(self): - return "BaseSnapshot(id={id},timestamp_ms={ts_ms},manifests={manifests}".format(id=self._snapshot_id, - ts_ms=self._timestamp_millis, - manifests=self._manifests) - - def __str__(self): - return self.__repr__() - - def get_filtered_manifest(self, path, part_filter=None, row_filter=None, columns=None): - reader = ManifestReader.read(self._ops.new_input_file(path)) - self.add_closeable(reader) - return reader diff --git a/iceberg/core/base_table.py b/iceberg/core/base_table.py deleted file mode 100644 index a86d9960b3..0000000000 --- a/iceberg/core/base_table.py +++ /dev/null @@ -1,110 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -from iceberg.api import Table - -from .data_table_scan import DataTableScan -from .schema_update import SchemaUpdate - - -class BaseTable(Table): - - def __init__(self, ops, name): - self.ops = ops - self.name = name - - def refresh(self): - self.ops.refresh() - - def new_scan(self): - return DataTableScan(self.ops, self) - - def schema(self): - return self.ops.current().schema - - def spec(self): - return self.ops.current().spec - - def properties(self): - return self.ops.current().properties - - def location(self): - return self.ops.current().location - - def current_snapshot(self): - return self.ops.current().current_snapshot() - - def snapshots(self): - return self.ops.current().snapshots - - def snapshots_with_summary_property(self, prop_key, prop_val): - if prop_key is None: - raise RuntimeError("Property Key cannot be None: (%s, %s)" % (prop_key, prop_val)) - - for snapshot in self.ops.current().snapshots: - if prop_key in snapshot.summary.keys() and snapshot.summary.get(prop_key) == prop_val: - yield snapshot - - def update_schema(self): - return SchemaUpdate(self.ops) - - def update_properties(self): - # PropertiesUpdate(self.ops) - raise NotImplementedError() - - def update_location(self): - # SetLocation(self.ops) - raise NotImplementedError() - - def new_append(self): - # MergeAppend(ops) - raise NotImplementedError() - - def new_fast_append(self): - # FastAppend(ops) - raise NotImplementedError() - - def new_rewrite(self): - # ReplaceFiles(ops) - raise NotImplementedError() - - def new_overwrite(self): - # OverwriteData(ops) - raise NotImplementedError() - - def new_replace_partitions(self): - # ReplacePartitionsOperation(ops) - raise NotImplementedError() - - def new_delete(self): - # StreamingDelete(ops) - raise NotImplementedError() - - def expire_snapshots(self): - # RemoveSnapshots(ops) - raise NotImplementedError() - - def rollback(self): - # RollbackToSnapshot(ops) - raise NotImplementedError() - - def new_transaction(self): - # BaseTransaction.newTransaction(ops) - raise NotImplementedError() - - def __str__(self): - return self.name diff --git a/iceberg/core/base_table_scan.py b/iceberg/core/base_table_scan.py deleted file mode 100644 index e64038d313..0000000000 --- a/iceberg/core/base_table_scan.py +++ /dev/null @@ -1,213 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -from datetime import datetime -import logging - -from iceberg.api import Filterable -from iceberg.api import TableScan -from iceberg.api.expressions import (Binder, - Expressions) -from iceberg.api.io import CloseableGroup -from iceberg.api.types import get_projected_ids, select - -from .base_combined_scan_task import BaseCombinedScanTask -from .table_properties import TableProperties -from .util import PackingIterator - -_logger = logging.getLogger(__name__) - - -class BaseTableScan(CloseableGroup, TableScan): - DATE_FORMAT = "%Y-%m-%d %H:%M:%S.%f" - SNAPSHOT_COLUMNS = ("snapshot_id", "file_path", "file_ordinal", "file_format", "block_size_in_bytes", - "file_size_in_bytes", "record_count", "partition", "value_counts", "null_value_counts", - "lower_bounds", "upper_bounds") - - def new_refined_scan(self, ops, table, schema, snapshot_id, row_filter, - case_sensitive, selected_columns, options, minused_cols): - raise NotImplementedError() - - def target_split_size(self, ops): - raise NotImplementedError() - - def __init__(self, ops, table, schema, snapshot_id=None, columns=None, - row_filter=None, case_sensitive=True, selected_columns=None, options=None, - minused_cols=None): - self.ops = ops - self.table = table - self._schema = schema - self.snapshot_id = snapshot_id - self.columns = columns - self._row_filter = row_filter - self._case_sensitive = case_sensitive - self.selected_columns = selected_columns - self.minused_cols = minused_cols or list() - self.options = options if options is not None else dict() - - if self.columns is None and self._row_filter is None: - self.columns = Filterable.ALL_COLUMNS - self._row_filter = Expressions.always_true() - - self._stats = dict() - - def is_case_sensitive(self): - return self.case_sensitive - - def use_snapshot(self, snapshot_id): - if self.snapshot_id is not None: - raise RuntimeError("Cannot override snapshot, already set to id=%s" % self.snapshot_id) - if self.ops.current().snapshot(snapshot_id) is None: - raise RuntimeError("Cannot find snapshot with ID %s" % self.snapshot_id) - - return self.new_refined_scan(self.ops, self.table, self._schema, snapshot_id=snapshot_id, - row_filter=self._row_filter, case_sensitive=self._case_sensitive, - selected_columns=self.selected_columns, options=self.options, - minused_cols=self.minused_cols) - - def as_of_time(self, timestamp_millis): - raise NotImplementedError() - - def project(self, schema): - return self.new_refined_scan(self.ops, self.table, schema, snapshot_id=self.snapshot_id, - row_filter=self._row_filter, case_sensitive=self._case_sensitive, - selected_columns=self.selected_columns, options=self.options, - minused_cols=self.minused_cols) - - def case_sensitive(self, case_sensitive): - return self.new_refined_scan(self.ops, self.table, self._schema, snapshot_id=self.snapshot_id, - row_filter=self._row_filter, case_sensitive=case_sensitive, - selected_columns=self.selected_columns, options=self.options, - minused_cols=self.minused_cols) - - def select(self, columns): - return self.new_refined_scan(self.ops, self.table, self._schema, snapshot_id=self.snapshot_id, - row_filter=self._row_filter, case_sensitive=self._case_sensitive, - selected_columns=columns, options=self.options, - minused_cols=self.minused_cols) - - def select_except(self, columns): - return self.new_refined_scan(self.ops, self.table, self._schema, snapshot_id=self.snapshot_id, - row_filter=self._row_filter, case_sensitive=self._case_sensitive, - selected_columns=self.selected_columns, options=self.options, - minused_cols=columns) - - @property - def row_filter(self): - return self._row_filter - - def filter(self, expr): - return self.new_refined_scan(self.ops, self.table, self._schema, snapshot_id=self.snapshot_id, - row_filter=Expressions.and_(self._row_filter, expr), - case_sensitive=self._case_sensitive, selected_columns=self.selected_columns, - options=self.options, minused_cols=self.minused_cols) - - def option(self, property, value): - builder = dict() - builder.update(self.options) - builder[property] = value - return self.new_refined_scan(self.ops, self.table, self._schema, snapshot_id=self.snapshot_id, - row_filter=self._row_filter, case_sensitive=self._case_sensitive, - selected_columns=self.selected_columns, options=builder, - minused_cols=self.minused_cols) - - def plan_files(self, ops=None, snapshot=None, row_filter=None): - - if not all(i is None for i in [ops, snapshot, row_filter]): - raise NotImplementedError() - - snapshot = self.ops.current().snapshot(self.snapshot_id) \ - if self.snapshot_id is not None else self.ops.current().current_snapshot() - - if snapshot is not None: - _logger.info("Scanning table {} snapshot {} created at {} with filter {}" - .format(self.table, - snapshot.snapshot_id, - datetime.fromtimestamp(snapshot.timestamp_millis / 1000.0) - .strftime(BaseTableScan.DATE_FORMAT), - self._row_filter)) - - return self.plan_files(ops, snapshot, row_filter) - else: - _logger.info("Scanning empty table {}" % self.table) - - def plan_tasks(self): - split_size = self.target_split_size(self.ops) - lookback = int(self.ops.current().properties.get(TableProperties.SPLIT_LOOKBACK, - TableProperties.SPLIT_LOOKBACK_DEFAULT)) - open_file_cost = int(self.ops.current().properties.get(TableProperties.SPLIT_OPEN_FILE_COST, - TableProperties.SPLIT_OPEN_FILE_COST_DEFAULT)) - - if not self.ops.conf.get("iceberg.scan.split-file-tasks", True): - split_files = list(self.plan_files()) - else: - split_files = self.split_files(split_size) - - def weight_func(file): - return max(file.length, open_file_cost) - - return (BaseCombinedScanTask(scan_tasks) - for scan_tasks in PackingIterator(split_files, split_size, lookback, weight_func)) - - def split_files(self, split_size): - file_scan_tasks = list(self.plan_files()) - split_tasks = [task for split_tasks in [scan_task.split(split_size) for scan_task in file_scan_tasks] - for task in split_tasks] - - return split_tasks - - @property - def schema(self): - return self._lazy_column_projection() - - def to_arrow_table(self): - raise NotImplementedError() - - def to_pandas(self): - raise NotImplementedError() - - def _lazy_column_projection(self): - if "*" in self.selected_columns: - if len(self.minused_cols) == 0: - return self._schema - self.selected_columns = [field.name for field in self._schema.as_struct().fields] - final_selected_cols = [column for column in self.selected_columns if column not in self.minused_cols] - else: - final_selected_cols = self.selected_columns - - required_field_ids = set() - required_field_ids.update(Binder.bound_references(self.table.schema().as_struct(), - [self._row_filter], - self._case_sensitive)) - - if self._case_sensitive: - selected_ids = get_projected_ids(self.table.schema().select(final_selected_cols)) - else: - selected_ids = get_projected_ids(self.table.schema().case_insensitive_select(final_selected_cols)) - - required_field_ids.update(selected_ids) - - return select(self.table.schema(), required_field_ids) - - def __repr__(self): - return "BaseTableScan(table={}, projection={}, filter={}, case_sensitive={}".format(self.table, - self._schema.as_struct(), - self._row_filter, - self._case_sensitive) - - def __str__(self): - return self.__repr__() diff --git a/iceberg/core/base_transaction.py b/iceberg/core/base_transaction.py deleted file mode 100644 index 19616f3b7b..0000000000 --- a/iceberg/core/base_transaction.py +++ /dev/null @@ -1,176 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - - -import enum - - -from iceberg.api import (Table, - Transaction) -from iceberg.core import TableOperations -from iceberg.exceptions import CommitFailedException - - -class BaseTransaction(Transaction): - - @staticmethod - def replace_table_transaction(ops, start): - return BaseTransaction(ops, start) - - @staticmethod - def create_table_transaction(ops, start): - if ops.current() is not None: - raise RuntimeError("Cannot start create table transaction: table already exists") - - @staticmethod - def new_transaction(ops): - return BaseTransaction(ops, ops.refesh()) - - def __init__(self, ops, start): - self.ops = ops - self.updates = list() - self.intermediate_snapshot_ids = set() - self.base = ops.current - if self.base is None and start is None: - self.type = TransactionType.CREATE_TABLE - elif self.base is not None and start != self.base: - self.type = TransactionType.REPLACE_TABLE - else: - self.type = TransactionType.SIMPLE - - self.last_base = None - self.current = start - self.transaction_table = TransactionTable(self, self.current) - self.transaction_ops = TransactionTableOperations - - def table(self): - return self.transaction_table - - def check_last_operation_commited(self, operation): - if self.last_base == self.current: - raise RuntimeError("Cannot create new %s: last operation has not committed" % operation) - self.last_base = self.current - - def update_schema(self): - self.check_last_operation_commited("UpdateSchema") - - @staticmethod - def current_id(meta): - if meta is not None and meta.current_snapshot() is not None: - return meta.current_snapshot().snapshot_id - - -class TransactionType(enum.Enum): - - CREATE_TABLE = 0 - REPLACE_TABLE = 1 - SIMPLE = 1 - - -class TransactionTableOperations(TableOperations): - - def __init__(self, bt): - self._bt = bt - - def current(self): - return self._bt.current - - def refresh(self): - return self._bt.current - - def commit(self, base, metadata): - if base != self.current(): - raise CommitFailedException("Table metadata refresh is required") - - old_id = BaseTransaction.current_id(self._bt.current) - if old_id is not None and old_id not in (BaseTransaction.current_id(metadata), - BaseTransaction.current_id(base)): - self._bt.intermediate_snapshot_ids.add(old_id) - - self._bt.current = metadata - - def io(self): - return self._bt.ops.io() - - def metadata_file_location(self, file): - return self._bt.ops.metadata_file_location(file) - - def new_snapshot_id(self): - return self._bt.ops.new_snapshot_id() - - -class TransactionTable(Table): - def __init__(self, bt, current): - self.bt = bt - self.current = current - - def refresh(self): - pass - - def new_scan(self): - raise RuntimeError("Transaction tables do not support scans") - - def schema(self): - return self.current.schema - - def spec(self): - return self.current.spec - - def properties(self): - return self.current.properties - - def location(self): - return self.current.location - - def current_snapshot(self): - return self.current.current_snapshot() - - def snapshots(self): - return self.current.snapshots - - def update_schema(self): - return self.bt.update_schema() - - def update_properties(self): - return self.bt.update_properties() - - def update_location(self): - return self.bt.update_location() - - def new_append(self): - return self.bt.new_append() - - def new_rewrite(self): - return self.bt.new_rewrite() - - def new_overwrite(self): - return self.bt.new_overwrite() - - def new_replace_partitions(self): - return self.bt.new_replace_partitions() - - def new_delete(self): - return self.bt.new_delete() - - def expire_snapshots(self): - return self.bt.expire_snapshots() - - def rollback(self): - raise RuntimeError("Transaction tables do not support rollback") - - def new_transaction(self): - raise RuntimeError("Cannot create a transaction within a transaction") diff --git a/iceberg/core/config_properties.py b/iceberg/core/config_properties.py deleted file mode 100644 index 1cb2ecfb67..0000000000 --- a/iceberg/core/config_properties.py +++ /dev/null @@ -1,26 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - - -class ConfigProperties(object): - - COMPRESS_METADATA = "iceberg.compress.metadata" - COMPRESS_METADATA_DEFAULT = False - - @staticmethod - def should_compress(config): - return config.get(ConfigProperties.COMPRESS_METADATA, ConfigProperties.COMPRESS_METADATA_DEFAULT) diff --git a/iceberg/core/data_files.py b/iceberg/core/data_files.py deleted file mode 100644 index 8b73ebcd12..0000000000 --- a/iceberg/core/data_files.py +++ /dev/null @@ -1,228 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - - -from iceberg.api import (FileFormat, - Metrics) -from iceberg.api.types import Conversions - -from .generic_data_file import GenericDataFile -from .partition_data import PartitionData - - -class DataFiles(object): - DEFAULT_BLOCK_SIZE = 64 * 1024 * 1024 - - @staticmethod - def new_partition_data(spec): - return PartitionData(spec.partition_type()) - - @staticmethod - def copy_partition_data(spec, partition_data, reuse): - data = reuse - if data is None: - data = DataFiles.new_partition_data(spec) - - for i, _ in enumerate(spec.fields): - data.set(i, partition_data.get(i)) - - return data - - @staticmethod - def fill_from_path(spec, partition_path, reuse): - data = reuse - if data is None: - data = DataFiles.new_partition_data(spec) - - partitions = partition_path.split("/") - if len(partitions) > len(spec.fields): - raise RuntimeError("Invalid partition data, too many fields (expecting %s): %s" % (len(spec.fields), - partition_path)) - - if len(partitions) < len(spec.fields): - raise RuntimeError("Invalid partition data, not enough fields(expecting %s): %s" % (len(spec.fields), - partition_path)) - - for i, part in enumerate(partitions): - field = spec.fields[i] - parts = part.split("=") - if len(parts) != 2 or parts[0] is None or parts[0] != field.name: - raise RuntimeError("Invalid partition: %s" % part) - - data.set(i, Conversions.from_partition_string(data.get_type(i), parts[1])) - - return data - - @staticmethod - def data(spec, partition_path): - return DataFiles.fill_from_path(spec, partition_path, None) - - @staticmethod - def copy(spec, partition): - return DataFiles.copy_partition_data(spec, partition, None) - - @staticmethod - def from_input_file(input_file, row_count, partition_data=None, metrics=None): - from .filesystem import FileSystemInputFile - if isinstance(input_file, FileSystemInputFile): - return DataFiles.from_stat(input_file.get_stat(), row_count, - partition_data=partition_data, metrics=metrics) - - @staticmethod - def from_stat(stat, row_count, partition_data=None, metrics=None): - location = stat.path - format = FileFormat.from_file_name(location) - return GenericDataFile(location, format, stat.length, stat.block_size, - row_count=row_count, partition=partition_data, metrics=metrics) - - @staticmethod - def builder(spec=None): - return DataFileBuilder(spec) - - -class DataFileBuilder(object): - - def __init__(self, spec=None): - self.spec = spec - self.partition_data = DataFiles.new_partition_data(spec) if spec is not None else None - self.is_partitioned = spec is not None - self.file_path = None - self.format = None - self.record_count = -1 - self.file_size_in_bytes = -1 - self.block_size_in_bytes = -1 - self.column_sizes = None - self.value_counts = None - self.null_value_counts = None - self.lower_bounds = None - self.upper_bounds = None - - def clear(self): - if self.is_partitioned: - self.partition_data.clear() - self.file_path = None - self.format = None - self.record_count = -1 - self.file_size_in_bytes = -1 - self.block_size_in_bytes = -1 - self.column_sizes = None - self.value_counts = None - self.null_value_counts = None - self.lower_bounds = None - self.upper_bounds = None - return self - - def copy(self, to_copy): - if self.is_partitioned: - self.partition_data = DataFiles.copy_partition_data(self.spec, to_copy.partition, self.partition_data) - - self.file_path = str(to_copy.path()) - self.format = to_copy.format - self.record_count = to_copy.record_count - self.file_size_in_bytes = to_copy.file_size_in_bytes - self.block_size_in_bytes = to_copy.block_size_in_bytes - self.column_sizes = to_copy.column_sizes - self.value_counts = to_copy.value_counts - self.null_value_counts = to_copy.null_value_counts - self.lower_bounds = to_copy.lower_bounds - self.upper_bounds = to_copy.upper_bounds - return self - - def with_status(self, stat): - self.file_path = stat.path - self.file_size_in_bytes = stat.length - self.block_size_in_bytes = stat.blocksize - return self - - def with_input_file(self, input_file): - from .filesystem import FileSystemInputFile - if isinstance(input_file, FileSystemInputFile): - self.with_status(input_file.get_stat()) - - self.file_path = self.location() - self.file_size_in_bytes = self.get_length() - - return self - - def with_path(self, path): - self.file_path = path - return self - - def with_format(self, fmt): - if isinstance(fmt, FileFormat): - self.format = fmt - else: - self.format = FileFormat[str(fmt).upper()] - - return self - - def with_partition(self, partition): - self.partition_data = DataFiles.copy_partition_data(self.spec, partition, self.partition_data) - return self - - def with_record_count(self, record_count): - self.record_count = record_count - return self - - def with_file_size_in_bytes(self, file_size_in_bytes): - self.file_size_in_bytes = file_size_in_bytes - return self - - def with_block_size_in_bytes(self, block_size_in_bytes): - self.block_size_in_bytes = block_size_in_bytes - return self - - def with_partition_path(self, partition_path): - if not self.is_partitioned: - raise RuntimeError("Cannot add partition data for an unpartitioned table") - - self.partition_data = DataFiles.fill_from_path(self.spec, partition_path, self.partition_data) - return self - - def with_metrics(self, metrics): - self.record_count = metrics.row_count if metrics.row_count is not None else -1 - self.column_sizes = metrics.column_sizes - self.value_counts = metrics.value_counts - self.null_value_counts = metrics.null_value_counts - self.lower_bounds = metrics.lower_bounds - self.upper_bounds = metrics.upper_bounds - return self - - def build(self): - if self.file_path is None: - raise RuntimeError("File path is required") - if self.format is None: - self.format = FileFormat.from_file_name(self.file_path) - - if self.format is None: - raise RuntimeError("File format is required") - - if self.file_size_in_bytes < 0: - raise RuntimeError("File size is required") - - if self.record_count < 0: - raise RuntimeError("Record count is required") - - if self.block_size_in_bytes is None: - self.block_size_in_bytes = DataFiles.DEFAULT_BLOCK_SIZE - - return GenericDataFile(self.file_path, self.format, self.file_size_in_bytes, self.block_size_in_bytes, - partition=self.partition_data.copy() if self.is_partitioned else None, - metrics=Metrics(row_count=self.record_count, column_sizes=self.column_sizes, - value_counts=self.value_counts, - null_value_counts=self.null_value_counts, - lower_bounds=self.lower_bounds, upper_bounds=self.upper_bounds)) diff --git a/iceberg/core/data_table_scan.py b/iceberg/core/data_table_scan.py deleted file mode 100644 index 6d8c7b001c..0000000000 --- a/iceberg/core/data_table_scan.py +++ /dev/null @@ -1,98 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -import itertools -import logging -from multiprocessing import cpu_count -from multiprocessing.dummy import Pool - -from iceberg.api.expressions import (InclusiveManifestEvaluator, - ResidualEvaluator) - -from .base_file_scan_task import BaseFileScanTask -from .base_table_scan import BaseTableScan -from .manifest_reader import ManifestReader -from .partition_spec_parser import PartitionSpecParser -from .schema_parser import SchemaParser -from .table_properties import TableProperties -from .util import SCAN_THREAD_POOL_ENABLED, WORKER_THREAD_POOL_SIZE_PROP - - -_logger = logging.getLogger(__name__) - - -class DataTableScan(BaseTableScan): - - SNAPSHOT_COLUMNS = ("snapshot_id", "file_path", "file_ordinal", "file_format", "block_size_in_bytes", - "file_size_in_bytes", "record_count", "partition", "value_counts", "null_value_counts", - "lower_bounds", "upper_bounds") - - def __init__(self, ops, table, schema=None, snapshot_id=None, row_filter=None, - case_sensitive=True, selected_columns=None, options=None, minused_cols=None): - super(DataTableScan, self).__init__(ops, table, schema if schema is not None else table.schema(), - snapshot_id=snapshot_id, row_filter=row_filter, - case_sensitive=case_sensitive, selected_columns=selected_columns, - options=options, minused_cols=minused_cols) - self._cached_evaluators = dict() - - def new_refined_scan(self, ops, table, schema, snapshot_id=None, row_filter=None, case_sensitive=None, - selected_columns=None, options=None, minused_cols=None): - return DataTableScan(ops, table, schema, - snapshot_id=snapshot_id, row_filter=row_filter, case_sensitive=case_sensitive, - selected_columns=selected_columns, options=options, minused_cols=minused_cols) - - def plan_files(self, ops=None, snapshot=None, row_filter=None): - if all(i is None for i in [ops, snapshot, row_filter]): - return super(DataTableScan, self).plan_files() - - matching_manifests = [manifest for manifest in snapshot.manifests - if self.cache_loader(manifest.spec_id).eval(manifest)] - - if self.ops.conf.get(SCAN_THREAD_POOL_ENABLED): - with Pool(self.ops.conf.get(WORKER_THREAD_POOL_SIZE_PROP, - cpu_count())) as reader_scan_pool: - return itertools.chain.from_iterable([scan for scan - in reader_scan_pool.map(self.get_scans_for_manifest, - matching_manifests)]) - else: - return itertools.chain.from_iterable([self.get_scans_for_manifest(manifest) - for manifest in matching_manifests]) - - def cache_loader(self, spec_id): - spec = self.ops.current().spec_id(spec_id) - return InclusiveManifestEvaluator(spec, self.row_filter) - - def get_scans_for_manifest(self, manifest): - from .filesystem import FileSystemInputFile - input_file = FileSystemInputFile.from_location(manifest.manifest_path, self.ops.conf) - reader = ManifestReader.read(input_file) - schema_str = SchemaParser.to_json(reader.spec.schema) - spec_str = PartitionSpecParser.to_json(reader.spec) - residuals = ResidualEvaluator(reader.spec, self.row_filter) - return [BaseFileScanTask(file, schema_str, spec_str, residuals) - for file in reader.filter_rows(self.row_filter).select(BaseTableScan.SNAPSHOT_COLUMNS).iterator()] - - def target_split_size(self, ops): - scan_split_size_str = self.options.get(TableProperties.SPLIT_SIZE) - - if scan_split_size_str is not None: - try: - return int(scan_split_size_str) - except ValueError: - _logger.warning("Invalid %s option: %s" % (TableProperties.SPLIT_SIZE, scan_split_size_str)) - - return int(self.ops.current().properties.get(TableProperties.SPLIT_SIZE, TableProperties.SPLIT_SIZE_DEFAULT)) diff --git a/iceberg/core/filesystem/__init__.py b/iceberg/core/filesystem/__init__.py deleted file mode 100644 index c686aa1262..0000000000 --- a/iceberg/core/filesystem/__init__.py +++ /dev/null @@ -1,27 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -__all__ = ["get_fs", "FileStatus", "FileSystem", "FileSystemInputFile", "FileSystemOutputFile", - "FilesystemTableOperations", "FilesystemTables", "LocalFileSystem", "S3File", "S3FileSystem"] - -from .file_status import FileStatus -from .file_system import FileSystem, FileSystemInputFile, FileSystemOutputFile -from .filesystem_table_operations import FilesystemTableOperations -from .filesystem_tables import FilesystemTables -from .local_filesystem import LocalFileSystem -from .s3_filesystem import S3File, S3FileSystem -from .util import get_fs diff --git a/iceberg/core/filesystem/file_status.py b/iceberg/core/filesystem/file_status.py deleted file mode 100644 index bea4e1b0b3..0000000000 --- a/iceberg/core/filesystem/file_status.py +++ /dev/null @@ -1,34 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - - -class FileStatus(): - - def __init__(self, path=None, length=None, is_dir=None, block_replication=None, - blocksize=None, modification_time=None, access_time=None, permission=None, - owner=None, group=None, symlink=None): - self.path = path - self.length = length - self.is_dir = is_dir - self.block_replication = block_replication - self.block_size = blocksize - self.modification_time = modification_time - self.access_time = access_time - self.permission = permission - self.owner = owner - self.group = group - self.symlink = symlink diff --git a/iceberg/core/filesystem/file_system.py b/iceberg/core/filesystem/file_system.py deleted file mode 100644 index 1ed2029537..0000000000 --- a/iceberg/core/filesystem/file_system.py +++ /dev/null @@ -1,120 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -import gzip - -from iceberg.api.io import InputFile, OutputFile - -from .util import get_fs - - -class FileSystem(object): - - def open(self, path, mode='rb'): - raise NotImplementedError() - - def create(self, path, overwrite=False): - raise NotImplementedError() - - def exists(self, path): - raise NotImplementedError() - - def delete(self, path): - raise NotImplementedError() - - def stat(self, path): - raise NotImplementedError() - - def rename(self, src, dest): - raise NotImplementedError() - - -class FileSystemInputFile(InputFile): - - def __init__(self, fs, path, conf, length=None, stat=None): - self.fs = fs - self.path = path - self.conf = conf - self.length = length - self.stat = stat - - @staticmethod - def from_location(location, conf): - fs = get_fs(location, conf) - return FileSystemInputFile(fs, location, conf) - - def location(self): - return self.path - - def get_length(self): - return self.get_stat().length - - def get_stat(self): - return self.lazy_stat() - - def lazy_stat(self): - if self.stat is None: - self.stat = self.fs.stat(self.path) - return self.stat - - def new_stream(self, gzipped=False): - with self.fs.open(self.location()) as fo: - if gzipped: - fo = gzip.GzipFile(fileobj=fo) - for line in fo: - yield line - - def new_fo(self, mode="rb"): - return self.fs.open(self.location(), mode=mode) - - def __repr__(self): - return "FileSystemInputFile({})".format(self.path) - - def __str__(self): - return self.__repr__() - - -class FileSystemOutputFile(OutputFile): - - @staticmethod - def from_path(path, conf): - return FileSystemOutputFile(path, conf) - - def __init__(self, path, conf): - self.path = path - self.conf = conf - - def create(self, mode="w"): - fs = get_fs(self.path, self.conf) - if fs.exists(self.path): - raise RuntimeError("File %s already exists" % self.path) - - return fs.open(self.path, mode=mode) - - def create_or_overwrite(self): - fs = get_fs(self.path, self.conf) - - return fs.open(self.path, "wb") - - def location(self): - return str(self.path) - - def __repr__(self): - return "FileSystemOutputFile({})".format(self.path) - - def __str__(self): - return self.__repr__() diff --git a/iceberg/core/filesystem/filesystem_table_operations.py b/iceberg/core/filesystem/filesystem_table_operations.py deleted file mode 100644 index 121565433e..0000000000 --- a/iceberg/core/filesystem/filesystem_table_operations.py +++ /dev/null @@ -1,140 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -import logging -from pathlib import Path -import uuid - -from iceberg.exceptions import CommitFailedException, ValidationException - -from .file_system import FileSystemInputFile, FileSystemOutputFile -from .util import get_fs -from ..table_metadata_parser import TableMetadataParser -from ..table_operations import TableOperations -from ..table_properties import TableProperties - -_logger = logging.getLogger(__name__) - - -class FilesystemTableOperations(TableOperations): - - def __init__(self, location, conf): - self.conf = conf - self.location = Path(location) - self.should_refresh = True - self.version = None - self.current_metadata = None - - def current(self): - if self.should_refresh: - return self.refresh() - - return self.current_metadata - - def refresh(self): - ver = self.version if self.version is not None else self.read_version_hint() - metadata_file = self.metadata_file(ver) - fs = get_fs(str(metadata_file), self.conf) - - if ver is not None and not fs.exists(metadata_file): - if ver == 0: - return None - raise ValidationException("Metadata file is missing: %s" % metadata_file) - - while fs.exists(str(self.metadata_file(ver + 1))): - ver += 1 - metadata_file = self.metadata_file(ver) - - self.version = ver - self.current_metadata = TableMetadataParser.read(self, FileSystemInputFile.from_location(str(metadata_file), - self.conf)) - self.should_refresh = False - return self.current_metadata - - def commit(self, base, metadata): - if base != self.current(): - raise CommitFailedException("Cannot commit changes based on stale table metadata") - - if not (base is None or base.location() == metadata.location()): - raise RuntimeError("Hadoop path-based tables cannot be relocated") - if TableProperties.WRITE_METADATA_LOCATION in metadata.properties: - raise RuntimeError("Hadoop path-based tables cannot be relocated") - - temp_metadata_file = self.metadata_path("{}{}".format(uuid.uuid4(), - TableMetadataParser.get_file_extension(self.conf))) - TableMetadataParser.write(metadata, FileSystemOutputFile.from_path(str(temp_metadata_file), self.conf)) - - next_version = (self.version if self.version is not None else 0) + 1 - final_metadata_file = self.metadata_file(next_version) - fs = get_fs(str(final_metadata_file), self.conf) - - if fs.exists(final_metadata_file): - raise CommitFailedException("Version %s already exists: %s" % (next_version, final_metadata_file)) - - if not fs.rename(temp_metadata_file, final_metadata_file): - raise CommitFailedException("Failed to commit changes using rename: %s" % final_metadata_file) - - self.write_version_hint(next_version) - self.should_refresh = True - - def new_input_file(self, path): - return FileSystemInputFile.from_location(path, self.conf) - - def new_output_file(self, path): - return FileSystemOutputFile.from_path(path, self.conf) - - def new_metadata_file(self, filename): - raise RuntimeError("Not yet implemented") - - def delete_file(self, path): - get_fs(path, self.conf).delete(path) - - def new_snapshot_id(self): - raise RuntimeError("Not yet implemented") - - def metadata_file_location(self, file): - return str(self.metadata_path(file)) - - def metadata_file(self, version): - return self.metadata_path("v{}{}".format(version, TableMetadataParser.get_file_extension(self.conf))) - - def metadata_path(self, filename): - return self.location / Path("metadata") / Path(filename) - - def version_hint_file(self): - return self.metadata_path("version-hint.text") - - def read_version_hint(self): - version_hint_file = str(self.version_hint_file()) - fs = get_fs(version_hint_file, self.conf) - - if not fs.exists(version_hint_file): - return 0 - else: - with fs.open(version_hint_file, "r") as fo: - return int(fo.readline().replace("\n", "")) - - def write_version_hint(self, version): - version_hint_file = str(self.version_hint_file()) - fs = get_fs(version_hint_file, self.conf) - try: - - with fs.create(version_hint_file, True) as fo: - fo.write("{}".format(version)) - - except RuntimeError as e: - _logger.warning("Unable to update version hint: %s" % e) diff --git a/iceberg/core/filesystem/filesystem_tables.py b/iceberg/core/filesystem/filesystem_tables.py deleted file mode 100644 index 113495f764..0000000000 --- a/iceberg/core/filesystem/filesystem_tables.py +++ /dev/null @@ -1,65 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -from .filesystem_table_operations import FilesystemTableOperations -from .. import TableOperations -from ..table_metadata import TableMetadata -from ...api import PartitionSpec, Schema, Table, Tables -from ...exceptions import NoSuchTableException - - -class FilesystemTables(Tables): - - def __init__(self: "FilesystemTables", conf: dict = None) -> None: - self.conf = conf if conf is not None else dict() - - def load(self: "FilesystemTables", table_identifier: str) -> Table: - from ..base_table import BaseTable - ops = self.new_table_ops(table_identifier) - if ops.current() is None: - raise NoSuchTableException("Table does not exist at location: %s" % table_identifier) - - return BaseTable(ops, table_identifier) - - def create(self: "FilesystemTables", schema: Schema, table_identifier: str, spec: PartitionSpec = None, - properties: dict = None, location: str = None) -> Table: - """ - Create a new table on the filesystem. - - Note: it is expected that the filesystem has atomic operations to ensure consistency for metadata updates. - Filesystems that don't have this guarantee could lead to data loss. - - Location should always be None as the table location on disk is taken from `table_identifier` - """ - from ..base_table import BaseTable - if location: - raise RuntimeError("""location has to be None. Both table_identifier and location have been declared. - table_identifier: {} and location: {}""".format(table_identifier, location)) - - full_spec, properties = super(FilesystemTables, self).default_args(spec, properties) - ops = self.new_table_ops(table_identifier) - - metadata = TableMetadata.new_table_metadata(ops, schema, full_spec, table_identifier, properties) - ops.commit(None, metadata) - - return BaseTable(ops, table_identifier) - - def new_table_ops(self: "FilesystemTables", table_identifier: str) -> TableOperations: - if table_identifier is None: - raise RuntimeError("table_identifier cannot be None") - - return FilesystemTableOperations(table_identifier, self.conf) diff --git a/iceberg/core/filesystem/local_filesystem.py b/iceberg/core/filesystem/local_filesystem.py deleted file mode 100644 index ac03b4c733..0000000000 --- a/iceberg/core/filesystem/local_filesystem.py +++ /dev/null @@ -1,81 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -import errno -import os -from pathlib import Path -import stat -from urllib.parse import urlparse - -from .file_status import FileStatus -from .file_system import FileSystem - - -class LocalFileSystem(FileSystem): - fs_inst = None - - @staticmethod - def get_instance() -> "LocalFileSystem": - if not LocalFileSystem.fs_inst: - return LocalFileSystem() - return LocalFileSystem.fs_inst - - def __init__(self: "LocalFileSystem") -> None: - if LocalFileSystem.fs_inst is None: - LocalFileSystem.fs_inst = self - - def open(self: "LocalFileSystem", path: str, mode: str = 'rb') -> object: - open_path = Path(LocalFileSystem.fix_path(path)) - - if "w" in mode and not open_path.parents[0].exists(): - try: - open_path.parents[0].mkdir(parents=True) - except OSError as exc: - if exc.errno != errno.EEXIST: - raise - - return open(open_path, mode=mode) - - def delete(self: "LocalFileSystem", path: str) -> None: - os.remove(path) - - def stat(self: "LocalFileSystem", path: str) -> FileStatus: - st = os.stat(LocalFileSystem.fix_path(path)) - return FileStatus(path=path, length=st.st_size, is_dir=stat.S_ISDIR(st.st_mode), - blocksize=st.st_blksize, modification_time=st.st_mtime, access_time=st.st_atime, - permission=st.st_mode, owner=st.st_uid, group=st.st_gid) - - @staticmethod - def fix_path(path: str) -> str: - return urlparse(path).path - - def create(self: "LocalFileSystem", path: str, overwrite: bool = False) -> object: - if os.path.exists(path) and not overwrite: - raise RuntimeError("Path %s already exists" % path) - - return open(path, "w") - - def rename(self: "LocalFileSystem", src: str, dest: str) -> bool: - try: - os.rename(src, dest) - except OSError: - return False - - return True - - def exists(self: "LocalFileSystem", path: str) -> bool: - return os.path.exists(path) diff --git a/iceberg/core/filesystem/s3_filesystem.py b/iceberg/core/filesystem/s3_filesystem.py deleted file mode 100644 index a8ac9d4cca..0000000000 --- a/iceberg/core/filesystem/s3_filesystem.py +++ /dev/null @@ -1,253 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -import io -import logging -import re -import time -from urllib.parse import urlparse - -import boto3 -from botocore.credentials import RefreshableCredentials -from botocore.exceptions import ClientError -from botocore.session import get_session -from retrying import retry - -from .file_status import FileStatus -from .file_system import FileSystem - -_logger = logging.getLogger(__name__) - - -S3_CLIENT = dict() -BOTO_STS_CLIENT = boto3.client('sts') -CONF = None -ROLE_ARN = "default" -AUTOREFRESH_SESSION = None - - -@retry(wait_incrementing_start=100, wait_exponential_multiplier=4, - wait_exponential_max=5000, stop_max_delay=600000, stop_max_attempt_number=7) -def get_s3(obj="resource"): - global AUTOREFRESH_SESSION - global ROLE_ARN - if ROLE_ARN not in S3_CLIENT: - S3_CLIENT[ROLE_ARN] = dict() - - if ROLE_ARN == "default": - if AUTOREFRESH_SESSION is None: - AUTOREFRESH_SESSION = boto3.Session() - S3_CLIENT["default"]["resource"] = AUTOREFRESH_SESSION.resource('s3') - S3_CLIENT["default"]["client"] = AUTOREFRESH_SESSION.client('s3') - else: - if AUTOREFRESH_SESSION is None: - sess = get_session() - sess._credentials = RefreshableCredentials.create_from_metadata(metadata=refresh_sts_session_keys(), - refresh_using=refresh_sts_session_keys, - method="sts-assume-role") - AUTOREFRESH_SESSION = boto3.Session(botocore_session=sess) - - S3_CLIENT[ROLE_ARN]["resource"] = AUTOREFRESH_SESSION.resource("s3") - S3_CLIENT[ROLE_ARN]["client"] = AUTOREFRESH_SESSION.client("s3") - - return S3_CLIENT.get(ROLE_ARN, dict()).get(obj) - - -def refresh_sts_session_keys(): - params = {"RoleArn": ROLE_ARN, - "RoleSessionName": "iceberg_python_client_{}".format(int(time.time() * 1000.00))} - - sts_creds = BOTO_STS_CLIENT.assume_role(**params).get("Credentials") - credentials = {"access_key": sts_creds.get("AccessKeyId"), - "secret_key": sts_creds.get("SecretAccessKey"), - "token": sts_creds.get("SessionToken"), - "expiry_time": sts_creds.get("Expiration").isoformat()} - return credentials - - -def url_to_bucket_key_name_tuple(url): - parsed_url = urlparse(url) - return parsed_url.netloc, parsed_url.path[1:], parsed_url.path.split("/")[-1] - - -class S3FileSystem(FileSystem): - fs_inst = None - - @staticmethod - def get_instance(): - if S3FileSystem.fs_inst is None: - S3FileSystem() - return S3FileSystem.fs_inst - - def __init__(self): - if S3FileSystem.fs_inst is None: - S3FileSystem.fs_inst = self - - def set_conf(self, conf): - global CONF - - CONF = conf - self.set_role(CONF.get("hive.aws_iam_role", "default")) - - def set_role(self, role): - global ROLE_ARN - - if role is not None: - ROLE_ARN = role - - def exists(self, path): - try: - self.info(path) - except ClientError as ce: - if ce.response['Error']['Code'] == "404": - return False - else: - raise - - return True - - def open(self, path, mode='rb'): - return S3File(path, mode=mode) - - def delete(self, path): - bucket, key, _ = url_to_bucket_key_name_tuple(S3FileSystem.normalize_s3_path(path)) - get_s3().Object(bucket_name=bucket, - key=key).delete() - - def stat(self, path): - st = self.info(S3FileSystem.normalize_s3_path(path)) - - return FileStatus(path=path, length=st.get("ContentLength"), is_dir=False, - blocksize=None, modification_time=st.get("LastModified"), access_time=None, - permission=None, owner=None, group=None) - - @staticmethod - def info(url): - bucket, key, _ = url_to_bucket_key_name_tuple(url) - return get_s3("client").head_object(Bucket=bucket, - Key=key) - - @staticmethod - def normalize_s3_path(path): - return re.sub(r'^s3n://|s3a://', 's3://', path) - - -class S3File(object): - MAX_CHUNK_SIZE = 4 * 1048576 - MIN_CHUNK_SIZE = 2 * 65536 - - def __init__(self, path, mode="rb"): - self.path = path - bucket, key, name = url_to_bucket_key_name_tuple(S3FileSystem.normalize_s3_path(path)) - self.curr_pos = 0 - self.obj = (get_s3() - .Object(bucket_name=bucket, - key=key)) - self.name = name - if mode.startswith("r"): - self.size = self.obj.content_length - - self.isatty = False - self.closed = False - - self.mode = mode - - self.buffer_remote_reads = True - - self.curr_buffer = None - self.curr_buffer_start = -1 - self.curr_buffer_end = -1 - self.buffer_reads = 0 - self.buffer_hits = 0 - - self.chunk_size = self.MAX_CHUNK_SIZE - - def close(self): - self.closed = True - - def flush(self): - pass - - def __next__(self): - return next(self.readline()) - - def read(self, n=0): - if self.curr_pos >= self.size: - return None - if self.buffer_remote_reads: - stream = self._read_from_buffer(n) - else: - if n <= 0: - n = self.size - stream = self.obj.get(Range='bytes={}-{}'.format(self.curr_pos, - self.size - self.curr_pos))['Body'].read() - else: - stream = self.obj.get(Range='bytes={}-{}'.format(self.curr_pos, self.curr_pos + n - 1))['Body'].read() - - self.curr_pos = min(self.curr_pos + n, self.size) - return stream - - def _read_from_buffer(self, n): - self.buffer_reads += 1 - # if the buffer is none or if the entire read is not contained - # in our current buffer fill the buffer - if self.curr_buffer is None or not (self.curr_buffer_start <= self.curr_pos - and self.curr_pos + n < self.curr_buffer_end): - self.curr_buffer_start = self.curr_pos - self.curr_buffer_end = self.curr_pos + max(self.chunk_size, n) - byte_range = 'bytes={}-{}'.format(self.curr_buffer_start, - self.curr_buffer_end) - self.curr_buffer = io.BytesIO(self.obj.get(Range=byte_range)['Body'].read()) - else: - self.buffer_hits += 1 - - # seek to the right position if we aren't at the start of the buffer - if self.curr_buffer_start != self.curr_pos: - self.curr_buffer.seek(self.curr_pos - self.curr_buffer_start) - - return self.curr_buffer.read(n) - - def readline(self, n=0): - if self.curr_buffer is None: - self.curr_buffer = io.BytesIO(self.obj.get()['Body'].read()) - for line in self.curr_buffer: - yield line - - def seek(self, offset, whence=0): - if whence == 0: - self.curr_pos = offset - elif whence == 1: - self.curr_pos += offset - elif whence == 2: - self.curr_pos = self.size + offset - - def tell(self): - return self.curr_pos - - def write(self, string): - resp = self.obj.put(Body=string) - if not resp.get("ResponseMetadata", dict()).get("HTTPStatusCode") == 200: - raise RuntimeError("Unable to write to {}".format(self.path)) - - def __enter__(self): - return self - - def __exit__(self, exc_type, exc_val, exc_tb): - self.close() - - def __iter__(self): - return self diff --git a/iceberg/core/filesystem/util.py b/iceberg/core/filesystem/util.py deleted file mode 100644 index d73dc11425..0000000000 --- a/iceberg/core/filesystem/util.py +++ /dev/null @@ -1,39 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -from urllib.parse import urlparse - - -def get_fs(path, conf, local_only=False): - from .local_filesystem import LocalFileSystem - from .s3_filesystem import S3FileSystem - - if local_only: - return LocalFileSystem.get_instance() - else: - parsed_path = urlparse(path) - - if parsed_path.scheme in ["", "file"]: - return LocalFileSystem.get_instance() - elif parsed_path.scheme in ["s3", "s3n", "s3a"]: - fs = S3FileSystem.get_instance() - fs.set_conf(conf) - return fs - elif parsed_path.scheme in ["hdfs"]: - raise RuntimeError("Hadoop FS not implemented") - - raise RuntimeError("No filesystem found for this location: %s" % path) diff --git a/iceberg/core/filtered_manifest.py b/iceberg/core/filtered_manifest.py deleted file mode 100644 index d5dfc2e307..0000000000 --- a/iceberg/core/filtered_manifest.py +++ /dev/null @@ -1,118 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -from iceberg.api.expressions import Evaluator, Expressions, inclusive, InclusiveMetricsEvaluator - -from .manifest_entry import Status - - -class FilteredManifest(object): - - def __init__(self, reader, part_filter, row_filter, columns, case_sensitive=True): - if reader is None: - raise RuntimeError("ManifestReader cannot be null") - - self.reader = reader - self.part_filter = part_filter - self.row_filter = row_filter - self.columns = columns - self.case_sensitive = case_sensitive - - self.lazy_evaluator = None - self.lazy_metrics_evaluator = None - - def select(self, columns): - return FilteredManifest(self.reader, self.part_filter, self.row_filter, columns, self.case_sensitive) - - def filter_partitions(self, expr): - return FilteredManifest(self.reader, - Expressions.and_(self.part_filter, expr), - self.row_filter, - self.columns, - self.case_sensitive) - - def filter_rows(self, expr): - projected = inclusive(self.reader.spec).project(expr) - return FilteredManifest(self.reader, - Expressions.and_(self.part_filter, projected), - Expressions.and_(self.row_filter, expr), - self.columns, self.case_sensitive) - - def all_entries(self): - if self.row_filter is not None and self.row_filter != Expressions.always_true() \ - or self.part_filter is not None and self. part_filter != Expressions.always_true(): - evaluator = self.evaluator() - metrics_evaluator = self.metrics_evaluator() - return [entry for entry in self.reader.entries(self.columns) - if entry is not None - and evaluator.eval(entry.file.partition()) - and metrics_evaluator.eval(entry.file)] - else: - return self.reader.entries(self.columns) - - def live_entries(self): - if self.row_filter is not None and self.row_filter != Expressions.always_true() \ - or self.part_filter is not None and self. part_filter != Expressions.always_true(): - evaluator = self.evaluator() - metrics_evaluator = self.metrics_evaluator() - return [entry for entry in self.reader.entries(self.columns) - if entry is not None - and entry.status != Status.DELETED - and evaluator.eval(entry.file.partition()) - and metrics_evaluator.eval(entry.file)] - else: - - return [entry for entry in self.reader.entries(self.columns) - if entry is not None and entry.status != Status.DELETED] - - def iterator(self): - if self.row_filter is not None and self.row_filter != Expressions.always_true() \ - or self.part_filter is not None and self.part_filter != Expressions.always_true(): - evaluator = self.evaluator() - metrics_evaluator = self.metrics_evaluator() - - return (input.copy() for input in self.reader.iterator(self.part_filter, self.columns) - if input is not None - and evaluator.eval(input.partition()) - and metrics_evaluator.eval(input)) - else: - return (entry.copy() for entry in self.reader.iterator(self.part_filter, self.columns)) - - def evaluator(self): - if self.lazy_evaluator is None: - if self.part_filter is not None: - self.lazy_evaluator = Evaluator(self.reader.spec.partition_type(), - self.part_filter, - self.case_sensitive) - else: - self.lazy_evaluator = Evaluator(self.reader.spec.partition_type(), - Expressions.always_true(), - self.case_sensitive) - - return self.lazy_evaluator - - def metrics_evaluator(self): - if self.lazy_metrics_evaluator is None: - if self.row_filter is not None: - self.lazy_metrics_evaluator = InclusiveMetricsEvaluator(self.reader.spec.schema, - self.row_filter, self.case_sensitive) - else: - self.lazy_metrics_evaluator = InclusiveMetricsEvaluator(self.reader.spec.schema, - Expressions.always_true(), - self.case_sensitive) - - return self.lazy_metrics_evaluator diff --git a/iceberg/core/generic_data_file.py b/iceberg/core/generic_data_file.py deleted file mode 100644 index bf86e4c9cd..0000000000 --- a/iceberg/core/generic_data_file.py +++ /dev/null @@ -1,137 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -import copy - -from iceberg.api import (DataFile, - StructLike) -from iceberg.api.types import StructType - -from .avro.iceberg_to_avro import IcebergToAvro -from .partition_data import PartitionData - - -class GenericDataFile(DataFile, StructLike): - - EMPTY_STRUCT_TYPE = StructType.of([]) - EMPTY_PARTITION_DATA = PartitionData(EMPTY_STRUCT_TYPE) - - def __init__(self, file_path, format, file_size_in_bytes, block_size_in_bytes, - row_count=None, partition=None, metrics=None): - - self._file_path = file_path - self._format = format - self._row_count = row_count - self._file_size_in_bytes = file_size_in_bytes - self._block_size_in_bytes = block_size_in_bytes - self._file_ordinal = None - self._sort_columns = None - - if partition is None: - self._partition_data = GenericDataFile.EMPTY_PARTITION_DATA - self._partition_type = GenericDataFile.EMPTY_PARTITION_DATA.partition_type - else: - self._partition_data = partition - self._partition_type = partition.get_partition_type() - if metrics is None: - self._row_count = row_count - self._column_sizes = None - self._value_counts = None - self._null_value_counts = None - self._lower_bounds = None - self._upper_bounds = None - else: - self._row_count = metrics.row_count - self._column_sizes = metrics.column_sizes - self._value_counts = metrics.value_counts - self._null_value_counts = metrics.null_value_counts - self._lower_bounds = metrics.lower_bounds - self._upper_bounds = metrics.upper_bounds - - def partition(self): - return self._partition_data - - def path(self): - return self._file_path - - def format(self): - return self._format - - def record_count(self): - return self._row_count - - def file_size_in_bytes(self): - return self._file_size_in_bytes - - def block_size_in_bytes(self): - return self._block_size_in_bytes - - def file_ordinal(self): - return self._file_ordinal - - def sort_columns(self): - return self._sort_columns - - def column_sizes(self): - return self._column_sizes - - def value_counts(self): - return self._value_counts - - def null_value_counts(self): - return self._null_value_counts - - def lower_bounds(self): - return self._lower_bounds - - def upper_bounds(self): - return self._upper_bounds - - def copy(self): - return copy.deepcopy(self) - - @staticmethod - def get_avro_schema(partition_type): - return IcebergToAvro.type_to_schema(DataFile.get_type(partition_type), DataFile.__class__.__name__) - - def __repr__(self): - fields = ["file_path: {}".format(self._file_path), - "file_format: {}".format(self._format), - "partition: {}".format(self._partition_data), - "record_count: {}".format(self._row_count), - "file_size_in_bytes: {}".format(self._file_size_in_bytes), - "block_size_in_bytes: {}".format(self._block_size_in_bytes), - "column_sizes: {}".format(self._column_sizes), - "value_counts: {}".format(self._value_counts), - "null_value_counts: {}".format(self._null_value_counts), - "lower_bounds: {}".format(self._lower_bounds), - "upper_bounds: {}".format(self._upper_bounds), - ] - return "GenericDataFile({})".format("\n,".join(fields)) - - def __str__(self): - return self.__repr__() - - def __deepcopy__(self, memodict): - cls = self.__class__ - result = cls.__new__(cls) - memodict[id(self)] = result - - for k, v in self.__dict__.items(): - setattr(result, k, copy.deepcopy(v, memodict)) - - return result diff --git a/iceberg/core/generic_manifest_file.py b/iceberg/core/generic_manifest_file.py deleted file mode 100644 index cdeeceadb3..0000000000 --- a/iceberg/core/generic_manifest_file.py +++ /dev/null @@ -1,194 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - - -import json - -from iceberg.api import (ManifestFile, - StructLike) -from iceberg.core.avro import IcebergToAvro - -from .generic_partition_field_summary import GenericPartitionFieldSummary - - -class GenericManifestFile(ManifestFile, StructLike): - AVRO_SCHEMA = IcebergToAvro.type_to_schema(ManifestFile.schema().as_struct(), "manifest_file") - - GET_POS_MAP = {0: lambda curr_manifest: curr_manifest.manifest_path, - 1: lambda curr_manifest: curr_manifest.lazy_length(), - 2: lambda curr_manifest: curr_manifest.spec_id, - 3: lambda curr_manifest: curr_manifest.snapshot_id, - 4: lambda curr_manifest: curr_manifest.added_files_count, - 5: lambda curr_manifest: curr_manifest.existing_files_count, - 6: lambda curr_manifest: curr_manifest.deleted_files_count, - 7: lambda curr_manifest: curr_manifest.partitions} - - @staticmethod - def generic_manifest_from_file(file, spec_id): - return GenericManifestFile(file=file, spec_id=spec_id) - - @staticmethod - def generic_manifest_from_path(path, length, spec_id, snapshot_id, - added_files_count, existing_files_count, deleted_files_count, - partitions): - return GenericManifestFile(path, length, spec_id, snapshot_id, - added_files_count, existing_files_count, deleted_files_count, - partitions) - - def __init__(self, path=None, file=None, spec_id=None, length=None, snapshot_id=None, - added_files_count=None, existing_files_count=None, deleted_files_count=None, - partitions=None): - if file is not None: - self.file = file - self.manifest_path = file.location() - else: - self.manifest_path = path - - self._length = length - self.spec_id = spec_id - self.snapshot_id = snapshot_id - self._added_files_count = added_files_count - self._existing_files_count = existing_files_count - self._deleted_files_count = deleted_files_count - self.partitions = partitions - self.from_projection_pos = None - - @property - def length(self): - return self.lazy_length() - - @property - def added_files_count(self): - return self._added_files_count - - @property - def existing_files_count(self): - return self._existing_files_count - - @property - def deleted_files_count(self): - return self._deleted_files_count - - def lazy_length(self): - if self._length is None: - if self.file is not None: - self._length = self.file.get_length() - else: - return None - else: - return self._length - - def size(self): - return len(ManifestFile.schema().columns()) - - def get(self, pos, cast_type=None): - if self.from_projection_pos: - pos = self.from_projection_pos[pos] - - get_func = GenericManifestFile.GET_POS_MAP.get(pos) - if get_func is None: - raise RuntimeError("Unknown field ordinal: %s" % pos) - - if cast_type is not None: - return cast_type(get_func(self)) - - return get_func(self) - - def set(self, pos, value): - if self.from_projection_pos: - pos = self.from_projection_pos[pos] - - if pos == 0: - self.manifest_path = str(value) - elif pos == 1: - self._length = int(value) - elif pos == 2: - self.spec_id = int(value) - elif pos == 3: - self.snapshot_id = int(value) - elif pos == 4: - self._added_files_count = int(value) - elif pos == 5: - self._existing_files_count = int(value) - elif pos == 6: - self._deleted_files_count = int(value) - elif pos == 7: - self.partitions = value - - def copy(self): - return GenericManifestFile(path=self.manifest_path, spec_id=self.spec_id, length=self.length, - snapshot_id=self.snapshot_id, added_files_count=self.added_files_count, - existing_files_count=self.existing_files_count, - deleted_files_count=self.deleted_files_count, - partitions=list(self.partitions)) - - @staticmethod - def get_schema(): - return GenericManifestFile.AVRO_SCHEMA - - @staticmethod - def to_avro_record_json(manifest): - return json.dumps(GenericManifestFile.to_avro_record_dict(manifest)) - - @staticmethod - def to_avro_record_dict(manifest): - return {"manifest_path": manifest.manifest_path, - "manifest_length": manifest._length, - "partition_spec_id": manifest.spec_id, - "added_snapshot_id": manifest.snapshot_id, - "added_data_files_count": manifest.added_files_count, - "existing_data_files_count": manifest.existing_files_count, - "deleted_data_files_count": manifest.deleted_files_count, - "partitions": manifest.partitions} - - @staticmethod - def from_avro_record_json(row): - partitions = row.get("partitions") - if partitions is not None: - partitions = [GenericPartitionFieldSummary(contains_null=partition["contains_null"], - lower_bound=partition["lower_bound"], - upper_bound=partition["upper_bound"]) - for partition in row.get("partitions")] - return GenericManifestFile(path=row.get("manifest_path"), - length=row.get("manifest_length"), - spec_id=row.get("partition_spec_id"), - added_files_count=row.get("added_data_files_count"), - existing_files_count=row.get("existing_data_files_count"), - deleted_files_count=row.get("existing_data_files_count"), - partitions=partitions) - - def __eq__(self, other): - if id(self) == id(other): - return True - - if other is None or not isinstance(other, GenericManifestFile): - return False - - return self.__key() == other.__key() - - def __hash__(self): - return hash(self.__key()) - - def __str__(self): - return self.__repr__() - - def __repr__(self): - return "GenericManifestFile({})".format(self.manifest_path) - - def __key(self): - return (GenericManifestFile.__class__, - self.manifest_path) diff --git a/iceberg/core/generic_partition_field_summary.py b/iceberg/core/generic_partition_field_summary.py deleted file mode 100644 index 77ade67df6..0000000000 --- a/iceberg/core/generic_partition_field_summary.py +++ /dev/null @@ -1,61 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - - -from iceberg.api import PartitionFieldSummary, StructLike - - -class GenericPartitionFieldSummary(PartitionFieldSummary, StructLike): - - AVRO_SCHEMA = None # IcebergToAvro.type_to_schema(PartitionFieldSummary.get_type()) - - def __init__(self, avro_schema=None, contains_null=False, lower_bound=None, upper_bound=None, copy=None): - if copy is not None: - avro_schema = copy.avro_schema - contains_null = copy.contains_null() - lower_bound = copy.lower_bound() - upper_bound = copy.upper_bound() - - if avro_schema is None: - avro_schema = GenericPartitionFieldSummary.AVRO_SCHEMA - - self.avro_schema = avro_schema - self._contains_null = contains_null - self._lower_bound = lower_bound - self._upper_bound = upper_bound - - def __str__(self): - return ("GenericPartitionFieldSummary(contains_null={},lower_bound={}, upper_bound={})" - .format(self.contains_null(), self.lower_bound(), self.upper_bound())) - - def contains_null(self): - return self._contains_null - - def get(self, pos): - raise NotImplementedError() - - def set(self, pos, value): - raise NotImplementedError() - - def lower_bound(self): - return self._lower_bound - - def upper_bound(self): - return self._upper_bound - - def copy(self): - return GenericPartitionFieldSummary(copy=self) diff --git a/iceberg/core/manifest_entry.py b/iceberg/core/manifest_entry.py deleted file mode 100644 index fde4e36fa7..0000000000 --- a/iceberg/core/manifest_entry.py +++ /dev/null @@ -1,145 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -from enum import Enum - -from iceberg.api import (DataFile, - FileFormat, - Metrics, - Schema) -from iceberg.api.types import (IntegerType, - LongType, - NestedField) -from iceberg.core.avro import IcebergToAvro - -from .generic_data_file import GenericDataFile -from .partition_data import PartitionData - - -class ManifestEntry(): - AVRO_NAME = "manifest_entry" - - def __init__(self, schema=None, partition_type=None, to_copy=None): - self.schema = schema - self.snapshot_id = 0 - self.file = None - self.status = Status.EXISTING - - if self.schema is None and partition_type is not None: - self.schema = IcebergToAvro.type_to_schema(ManifestEntry.get_schema(partition_type)) - - elif self.schema is None and partition_type is None and to_copy is not None: - self.schema = to_copy.schema - self.status = to_copy.status - self.snapshot_id = to_copy.snapshot_id - self.file = to_copy.file.copy() - - if self.schema is None: - raise RuntimeError("Invalid arguments schema=%s, partition_type=%s, to_copy=%s" % (self.schema, - partition_type, - to_copy)) - - def wrap_existing(self, snapshot_id, file): - self.status = Status.EXISTING - self.snapshot_id = snapshot_id - self.file = file - return self - - def wrap_append(self, snapshot_id, file): - self.status = Status.ADDED - self.snapshot_id = snapshot_id - self.file = file - return self - - def wrap_delete(self, snapshot_id, file): - self.status = Status.DELETED - self.snapshot_id = snapshot_id - self.file = file - return self - - def copy(self): - return ManifestEntry(to_copy=self) - - def put(self, i, v): - if i == 0: - self.status = Status.from_id(v) - elif i == 1: - self.snapshot_id = v - elif i == 2: - if isinstance(v, dict): - metrics = Metrics(row_count=v.get("record_count"), - column_sizes=v.get("column_sizes"), - value_counts=v.get("value_counts"), - null_value_counts=v.get("null_value_counts"), - lower_bounds=v.get("lower_bounds"), - upper_bounds=v.get("upper_bounds")) - - data_file_schema = self.schema.as_struct().field(name="data_file") - part_data = PartitionData.from_json(data_file_schema - .type - .field(name="partition").type, v.get("partition")) - - v = GenericDataFile(v.get("file_path"), - FileFormat[v.get("file_format")], - v.get("file_size_in_bytes"), - v.get("block_size_in_byte"), - row_count=v.get("record_count"), - partition=part_data, - metrics=metrics - ) - self.file = v - - def get(self, i): - if i == 0: - return self.status.value - elif i == 1: - return self.snapshot_id - elif i == 2: - return self.file - - def __repr__(self): - return "ManifestEntry(status=%s, snapshot_id=%s, file=%s" % (self.status, self.snapshot_id, self.file) - - def __str__(self): - return self.__repr__() - - @staticmethod - def project_schema(part_type, columns): - return ManifestEntry.wrap_file_schema(Schema(DataFile.get_type(part_type).fields) - .select(columns) - .as_struct()) - - @staticmethod - def get_schema(partition_type): - return ManifestEntry.wrap_file_schema(DataFile.get_type(partition_type)) - - @staticmethod - def wrap_file_schema(file_struct): - return Schema(NestedField.required(0, "status", IntegerType.get()), - NestedField.required(1, "snapshot_id", LongType.get()), - NestedField.required(2, "data_file", file_struct)) - - -class Status(Enum): - - EXISTING = 0 - ADDED = 1 - DELETED = 2 - - @staticmethod - def from_id(id): - return Status(id) diff --git a/iceberg/core/manifest_list_writer.py b/iceberg/core/manifest_list_writer.py deleted file mode 100644 index b3078edf53..0000000000 --- a/iceberg/core/manifest_list_writer.py +++ /dev/null @@ -1,55 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -import json - -from fastavro import parse_schema, writer -from iceberg.api import ManifestFile -from iceberg.api.io import FileAppender -from iceberg.core import GenericManifestFile -from iceberg.core.avro import IcebergToAvro - - -class ManifestListWriter(FileAppender): - - def __init__(self, snapshot_file, snapshot_id, parent_snapshot_id): - self.file = snapshot_file - self.meta = {"snapshot-id": str(snapshot_id), - "parent-snapshot-id": str(parent_snapshot_id)} - tmp_schema = IcebergToAvro.type_to_schema(ManifestFile.SCHEMA.as_struct(), - "manifest_file") - - self.schema = parse_schema(json.dumps(tmp_schema)) - - def add(self, d): - writer(self.file, - self.schema, - d, - metadata=self.meta) - - def add_all(self, values): - manifest_records = [GenericManifestFile.to_avro_record_dict(value) - for value in values if not isinstance(value, str)] - writer(self.file, - self.schema, - manifest_records) - - def close(self): - self.writer.flush() - - def metrics(self): - raise RuntimeError("Metrics not available") diff --git a/iceberg/core/manifest_reader.py b/iceberg/core/manifest_reader.py deleted file mode 100644 index e648684f29..0000000000 --- a/iceberg/core/manifest_reader.py +++ /dev/null @@ -1,162 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -import logging - -import fastavro -from iceberg.api import FileFormat, Filterable -from iceberg.api.expressions import Expressions, inclusive -from iceberg.api.io import CloseableGroup - -from .avro import AvroToIceberg -from .filtered_manifest import FilteredManifest -from .manifest_entry import ManifestEntry, Status -from .partition_spec_parser import PartitionSpecParser -from .schema_parser import SchemaParser -from .table_metadata import TableMetadata - -_logger = logging.getLogger(__name__) - - -class ManifestReader(CloseableGroup, Filterable): - ALL_COLUMNS = ("*",) - CHANGE_COLUMNS = ("file_path", "file_format", "partition", "record_count", "file_size_in_bytes") - - @staticmethod - def read(file, spec_lookup=None): - return ManifestReader(file=file, spec_lookup=spec_lookup) - - def select(self, columns): - return FilteredManifest(self, - Expressions.always_true(), - Expressions.always_true(), - list(columns), - self.case_sensitive) - - def filter_partitions(self, expr): - return FilteredManifest(self, - expr, - Expressions.always_true(), - ManifestReader.ALL_COLUMNS, - self.case_sensitive) - - def filter_rows(self, expr): - return FilteredManifest(self, - inclusive(self.spec).project(expr), - expr, - ManifestReader.ALL_COLUMNS, - self.case_sensitive) - - @staticmethod - def in_memory(spec, entries): - return ManifestReader(spec=spec, entries=entries) - - def __init__(self, file=None, spec=None, metadata=None, schema=None, case_sensitive=True, spec_lookup=None): - self.file = file - self.schema = schema - self.metadata = metadata - self.spec = spec - self._case_sensitive = case_sensitive - - self._entries = None - self._avro_rows = list() - self._fo = None - self._avro_reader = None - self._avro_rows = None - - if not all([item is not None for item in [self.file, self.metadata, self.spec, self.schema]]): - if self.spec is not None: - self.__init_from_spec() - else: - self.__init_from_file(spec_lookup) - - self._adds = None - self._deletes = None - - def __init_from_file(self, spec_lookup): - self._fo = self.file.new_fo() - self._avro_reader = fastavro.reader(self._fo) - self.metadata = self._avro_reader.metadata - spec_id = int(self.metadata.get("partition-spec-id", TableMetadata.INITIAL_SPEC_ID)) - - if spec_lookup is not None: - self.spec = spec_lookup(spec_id) - self.schema = self.spec.schema - else: - self.schema = SchemaParser.from_json(self.metadata.get("schema")) - self.spec = PartitionSpecParser.from_json_fields(self.schema, spec_id, self.metadata.get("partition-spec")) - - def __init_from_spec(self): - self.metadata = dict() - self.schema = self.spec.schema - - def case_sensitive(self, case_sensitive): - return ManifestReader(file=self.file, metadata=self.metadata, spec=self.spec, - schema=self.schema, case_sensitive=case_sensitive) - - def cache_changes(self): - adds = list() - deletes = list() - for entry in self.entries(ManifestReader.CHANGE_COLUMNS): - if entry.status == "ADDED": - adds.append(entry.copy()) - elif entry.status == "DELETED": - deletes.append(entry.copy()) - - self._adds = adds - self._deletes = deletes - - def added_files(self): - if self._adds is None: - self.cache_changes() - - return self._adds - - def deleted_files(self): - if self._adds is None: - self.cache_changes() - - return self._deletes - - def entries(self, columns=None): - if columns is None: - columns = ManifestReader.ALL_COLUMNS - - file_format = FileFormat.from_file_name(self.file.location()) - if file_format is None: - raise RuntimeError("Unable to determine format of manifest: %s" % self.file) - - proj_schema = ManifestEntry.project_schema(self.spec.partition_type(), columns) - - if self._entries is None: - if file_format is FileFormat.AVRO: - self._entries = list() - for read_entry in AvroToIceberg.read_avro_row(proj_schema, self._avro_reader): - entry = ManifestEntry(schema=proj_schema, partition_type=self.spec.partition_type()) - for i, key in enumerate(read_entry.keys()): - entry.put(i, read_entry[key]) - self._entries.append(entry) - self._fo.close() - self._avro_reader = None - - return self._entries - - def iterator(self, part_filter=None, columns=None): - if part_filter is None and columns is None: - return self.iterator(Expressions.always_true(), Filterable.ALL_COLUMNS) - - return [entry.file for entry in self.entries() if entry.status != Status.DELETED] diff --git a/iceberg/core/partition_data.py b/iceberg/core/partition_data.py deleted file mode 100644 index d74f34823d..0000000000 --- a/iceberg/core/partition_data.py +++ /dev/null @@ -1,94 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -import copy -import json - -from iceberg.api.struct_like import StructLike - - -class PartitionData(StructLike): - - def __init__(self, schema=None, partition_type=None): - if partition_type is None: - self.partition_type = schema.as_nested_type().as_struct_type() - else: - for field in partition_type.fields: - if not field.type.is_primitive_type(): - raise RuntimeError("Partitions cannot contain nested types: %s" % field.type) - self.partition_type = partition_type - # schema = PartitionData.get_schema(self.partition_type) - self._size = len(self.partition_type.fields) - self.data = list() - self.schema = schema - self.string_schema = str(schema) - - def clear(self): - self.data = list() - - def copy(self): - return copy.deepcopy(self) - - def get_partition_type(self): - return self.partition_type - - def __eq__(self, other): - if id(self) == id(other): - return True - - if other is None or not isinstance(other, PartitionData): - return False - - return self.partition_type == other.partition_type and self.data == other.data - - def __hash__(self): - return hash(self.__key()) - - def __key(self): - return PartitionData.__class__, self.partition_type, self.data - - def __ne__(self, other): - return not self.__eq__(other) - - def __str__(self): - return "PartitionData{%s}" % (",".join(["{}={}".format(self.partition_type.fields[i], - datum) for i, datum in enumerate(self.data)])) - - def __len__(self): - return self._size - - def put(self, i, v): - self.data.insert(i, v) - - def get(self, pos): - try: - return self.data[pos][1] - except IndexError: - return None - - @staticmethod - def from_json(schema, json_obj): - if isinstance(json_obj, str): - json_obj = json.loads(json_obj) - - data = PartitionData(partition_type=schema) - for i, kv_tuple in enumerate(json_obj.items()): - data.put(i, kv_tuple) - return data - - def set(self, pos, value): - raise NotImplementedError() diff --git a/iceberg/core/partition_spec_parser.py b/iceberg/core/partition_spec_parser.py deleted file mode 100644 index ab57079c55..0000000000 --- a/iceberg/core/partition_spec_parser.py +++ /dev/null @@ -1,98 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -import json - -from iceberg.api import PartitionSpec - - -class PartitionSpecParser(object): - - SPEC_ID = "spec-id" - FIELDS = "fields" - SOURCE_ID = "source-id" - FIELD_ID = "field-id" - TRANSFORM = "transform" - NAME = "name" - - @staticmethod - def to_json(spec, indent=None): - return json.dumps(PartitionSpecParser.to_dict(spec), indent=indent) - - @staticmethod - def to_dict(spec): - return {PartitionSpecParser.SPEC_ID: spec.spec_id, - PartitionSpecParser.FIELDS: PartitionSpecParser.to_json_fields(spec)} - - @staticmethod - def to_json_fields(spec): - return [{PartitionSpecParser.NAME: field.name, - PartitionSpecParser.TRANSFORM: str(field.transform), - PartitionSpecParser.SOURCE_ID: field.source_id, - PartitionSpecParser.FIELD_ID: field.field_id} - for field in spec.fields] - - @staticmethod - def from_json(schema, json_obj): - if isinstance(json_obj, str): - json_obj = json.loads(json_obj) - - if not isinstance(json_obj, dict): - raise RuntimeError("Cannot parse partition spec, not an object: %s" % json_obj) - - spec_id = json_obj.get(PartitionSpecParser.SPEC_ID) - - builder = PartitionSpec.builder_for(schema).with_spec_id(spec_id) - fields = json_obj.get(PartitionSpecParser.FIELDS) - - return PartitionSpecParser.__build_from_json_fields(builder, fields) - - @staticmethod - def from_json_fields(schema, spec_id, json_obj): - builder = PartitionSpec.builder_for(schema).with_spec_id(spec_id) - - if isinstance(json_obj, str): - json_obj = json.loads(json_obj) - - return PartitionSpecParser.__build_from_json_fields(builder, json_obj) - - @staticmethod - def __build_from_json_fields(builder, json_fields): - if not isinstance(json_fields, (list, tuple)): - raise RuntimeError("Cannot parse partition spec fields, not an array: %s" % json_fields) - - field_id_count = 0 - for element in json_fields: - if not isinstance(element, dict): - raise RuntimeError("Cannot parse partition field, not an object: %s" % element) - - if element.get(PartitionSpecParser.FIELD_ID) is not None: - builder.add(element.get(PartitionSpecParser.SOURCE_ID), - element.get(PartitionSpecParser.FIELD_ID), - element.get(PartitionSpecParser.NAME), - element.get(PartitionSpecParser.TRANSFORM)) - field_id_count = field_id_count + 1 - else: - builder.add_without_field_id(element.get(PartitionSpecParser.SOURCE_ID), - element.get(PartitionSpecParser.NAME), - element.get(PartitionSpecParser.TRANSFORM)) - - if field_id_count > 0 and field_id_count != len(json_fields): - raise RuntimeError("Cannot parse spec with missing field IDs: %s missing of %s fields." % - (len(json_fields) - field_id_count, len(json_fields))) - - return builder.build() diff --git a/iceberg/core/partition_summary.py b/iceberg/core/partition_summary.py deleted file mode 100644 index 45239ae854..0000000000 --- a/iceberg/core/partition_summary.py +++ /dev/null @@ -1,68 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -from iceberg.api.types.conversions import Conversions - -from .generic_partition_field_summary import GenericPartitionFieldSummary - - -class PartitionSummary(object): - - def __init__(self, spec): - self.java_classes = spec.java_classes - self.fields = [PartitionFieldStats(spec.partition_type.fields[i].type) - for i in range(len(self.java_classes))] - - def summaries(self): - return [field.to_summary() for field in self.fields] - - def update(self, partition_key): - self.update_fields(partition_key) - - def update_fields(self, key): - for i in range(self.java_classes): - stats = self.fields[i] - java_class = self.java_classes[i] - stats.update(key.get(i, java_class)) - - -class PartitionFieldStats(object): - - def __init__(self, type_var): - self.contains_null = False - self.type = type_var - self.min = None - self.max = None - - def to_summary(self): - lower_bound = None if self.min is None else Conversions.to_byte_buffer(self.type.type_id, self.min) - upper_bound = None if self.max is None else Conversions.to_byte_buffer(self.type.type_id, self.max) - return GenericPartitionFieldSummary(contains_null=self.contains_null, - lower_bound=lower_bound, - upper_bound=upper_bound) - - def update(self, value): - if value is None: - self.contains_null = True - elif self.min is None: - self.min = value - self.max = value - else: - if value < self.min: - self.min = value - if value > self.max: - self.max = value diff --git a/iceberg/core/scan_summary.py b/iceberg/core/scan_summary.py deleted file mode 100644 index cef872f7ea..0000000000 --- a/iceberg/core/scan_summary.py +++ /dev/null @@ -1,409 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -from copy import deepcopy -import functools - -from iceberg.api import DataOperations -from iceberg.api.expressions import Expressions, Literal, Operation, UnboundPredicate -from iceberg.api.types import TimestampType - -from .manifest_group import ManifestGroup -from .util import str_as_bool - -TIMESTAMP_RANGE_MAP = {Operation.LT: lambda min_val, max_val, val: (min_val, val - 1 if val - 1 < max_val else max_val), - Operation.LT_EQ: lambda min_val, max_val, val: (min_val, val if val < max_val else max_val), - Operation.GT: lambda min_val, max_val, val: (val - 1 if val - 1 > min_val else min_val, max_val), - Operation.GT_EQ: lambda min_val, max_val, val: (val if val > min_val else min_val, max_val), - Operation.EQ: lambda min_val, max_val, val: (val if val > min_val else min_val, - val if val < max_val else max_val)} - - -class ScanSummary(object): - - IGNORED_OPERATIONS = {DataOperations.DELETE, DataOperations.REPLACE} - SCAN_SUMMARY_COLUMNS = ["partition", "record_count", "file_size_in_bytes"] - - -class ScanSummaryBuilder(object): - - TIMESTAMP_NAMES = {"dateCreated", "lastUpdated"} - - def __init__(self, scan): - self.scan = scan - self.table = scan.table - self.ops = self.table.ops - self.snapshot_timestamps = {snap.snapshot_id: snap.timestamp_millis - for snap in self.table.snapshots()} - self._throw_if_limited = False - self.force_use_manifests = False - self._limit = 2**31 - 1 - - self.time_filters = list() - - def add_timestamp_filter(self, filter): - self.throw_if_limited() - self.time_filters.append(filter) - return self - - def after(self, timestamp): - self.add_timestamp_expression(timestamp, Expressions.greater_than_or_equal) - return self - - def before(self, timestamp): - self.add_timestamp_expression(timestamp, Expressions.less_than_or_equal) - return self - - def add_timestamp_expression(self, timestamp, expr_func): - if isinstance(timestamp, str): - timestamp = Literal.of(timestamp).to(TimestampType.without_timezone()).value / 1000 - - self.add_timestamp_filter(expr_func("timestamp_ms", timestamp)) - return self - - def throw_if_limited(self): - self._throw_if_limited = True - return self - - def limit(self, num_partitions): - self._limit = num_partitions - return self - - def use_manifests(self): - self.force_use_manifests = True - return self - - def remove_time_filters(self, expressions, expression): - if expression.op == Operation.AND: - self.remove_time_filters(expressions, expression.left) - self.remove_time_filters(expressions, expression.right) - return - elif isinstance(expression, UnboundPredicate): - pred = expression - ref = pred.ref - lit = pred.lit - - if ref.name in ScanSummaryBuilder.TIMESTAMP_NAMES: - ts_literal = lit.to(TimestampType.without_timezone()) - millis = ScanSummaryBuilder.to_millis(ts_literal.value) - self.add_timestamp_filter(Expressions.predicate(pred.op, "timestamp_ms", millis)) - return - - expressions.append(expression) - - def build(self): - if self.table.current_snapshot() is None: - return dict() - - filters = list() - self.remove_time_filters(filters, Expressions.rewrite_not(self.scan.row_filter)) - row_filter = self.join_filters(filters) - - if len(self.time_filters) == 0: - return self.from_manifest_scan(self.table.current_snapshot().manifests, row_filter) - - min_timestamp, max_timestamp = self.timestamp_range(self.time_filters) - oldest_snapshot = self.table.current_snapshot() - for key, val in self.snapshot_timestamps.items(): - if val < oldest_snapshot.timestamp_millis: - oldest_snapshot = self.ops.current().snapshot(key) - - # if oldest known snapshot is in the range, then there may be an expired snapshot that has - # been removed that matched the range. because the timestamp of that snapshot is unknown, - # it can't be included in the results and the results are not reliable.""" - if oldest_snapshot.timestamp_millis >= min_timestamp and oldest_snapshot <= max_timestamp: - raise RuntimeError("Cannot satisfy time filters: time range may include expired snapshots") - - snapshots = [snapshot for snapshot in ScanSummaryBuilder.snapshots_in_time_range(self.ops.current(), - min_timestamp, - max_timestamp) - if snapshot.operation not in ScanSummary.IGNORED_OPERATIONS] - - result = self.from_partition_summaries(snapshots) - if result is not None and not self.force_use_manifests: - return result - - # filter down to the the set of manifest files that were created in the time range, ignoring - # the snapshots created by delete or replace operations. this is complete because it finds - # files in the snapshot where they were added to the dataset in either an append or an - # overwrite. if those files are later compacted with a replace or deleted, those changes are - # ignored. - manifests_to_scan = list() - snapshot_ids = set() - - for snap in snapshots: - snapshot_ids.add(snap) - for manifest in snap.manifests: - if manifest.snapshot_id is not None or manifest.snapshot_id == snap.snapshot_id: - manifests_to_scan.append(manifest) - - return self.from_manifest_scan(manifests_to_scan, row_filter, True) - - def from_manifest_scan(self, manifests, row_filter, ignore_existing=False): - top_n = TopN(self._limit, self._throw_if_limited, lambda x, y: 0 if x == y else -1 if x < y else 1) - - entries = (ManifestGroup(self.ops, manifests) - .filter_data(row_filter) - .ignore_deleted() - .ignore_existing(ignore_existing) - .select(ScanSummary.SCAN_SUMMARY_COLUMNS) - .entries()) - - spec = self.table.spec() - for entry in entries: - timestamp = self.snapshot_timestamps.get(entry.snapshot_id) - partition = spec.partition_to_path(entry.file.partition()) - top_n.update(partition, - lambda metrics: ((metrics if metrics is not None else PartitionMetrics()) - .update_from_file(entry.file, timestamp))) - - return top_n.get() - - def from_partition_summaries(self, snapshots): - # try to build the result from snapshot metadata, but fall back if: - # any snapshot has no summary - # any snapshot has - top_n = TopN(self._limit, self._throw_if_limited, lambda x, y: 0 if x == y else -1 if x < y else 1) - - for snap in snapshots: - if snap.operation is None or snap.summary is None \ - or str_as_bool(snap.summary.get(SnapshotSummary.PARTITION_SUMMARY_PROP, "false")): - return None - - for key, _ in snap.summary.items(): - if key.startswith(SnapshotSummary.CHANGED_PARTITION_PREFIX): - part_key = key[len(SnapshotSummary.CHANGED_PARTITION_PREFIX):] - # part = dict(entry.split("=") for entry in val.split(",")) - # UPDATE THIS BEFORE FINISHING - added_files = 0 - added_records = 0 - added_size = 0 - top_n.update(part_key, - lambda metrics: ((PartitionMetrics() if metrics is None else metrics) - .update_from_counts(added_files, - added_records, - added_size, - snap.timestamp_millis))) - - return top_n.get() - - @staticmethod - def snapshots_in_time_range(meta, min_ts, max_ts): - snapshots = [] - current = meta.current_snapshot() - while current is not None and current.timestamp_millis >= min_ts: - current = meta.snapshot(current.parent_id) - - if current.timestamp_millis <= max_ts: - snapshots.add(current) - - snapshots.reverse() - return snapshots - - @staticmethod - def timestamp_range(time_filters): - min_timestamp = float('-inf') - max_timestamp = float('inf') - - for pred in time_filters: - value = pred.lit.val - try: - min_timestamp, max_timestamp = TIMESTAMP_RANGE_MAP[pred.op](min_timestamp, max_timestamp, value) - except KeyError: - raise RuntimeError("Cannot filter timestamps using predicate: %s" % pred) - - if max_timestamp < min_timestamp: - raise RuntimeError("No timestamps can match filters: %s" % ", ".join([str(pred) - for pred in time_filters])) - - return min_timestamp, max_timestamp - - @staticmethod - def join_filters(expressions): - result = Expressions.always_true() - for expression in expressions: - result = Expressions.and_(result, expression) - - return result - - @staticmethod - def to_millis(timestamp): - if timestamp < 10000000000: - # in seconds - return timestamp * 1000 - elif timestamp < 10000000000000: - # in millis - return timestamp - - # in micros - return timestamp / 1000 - - -class TopN(object): - - def __init__(self, N, throw_if_limited, key_comparator): - self.max_size = N - self.throw_if_limited = throw_if_limited - self.map = dict() - self.key_comparator = key_comparator - self.cut = None - - def update(self, key, update_func): - if self.cut is not None and self.key_comparator(self.cut, key) <= 0: - return - - self.map[key] = update_func(self.map.get(key)) - - while len(map.keys()) > self.max_size: - if self.throw_if_limited: - raise RuntimeError("Too many matching keys: more than %s" % self.max_size) - - self.cut = sorted(self.map, key=functools.cmp_to_key(self.key_comparator))[-1] - del self.map[self.cut] - - def get(self): - return deepcopy(self.map) - - -class PartitionMetrics(object): - - def __init__(self): - self.file_count = 0 - self.record_count = 0 - self.total_size = 0 - self.data_timestamp_millis = None - - def update_from_counts(self, file_count, record_count, files_size, timestamp_millis): - self.file_count += file_count - self.record_count += record_count - self.total_size += files_size - - if self.data_timestamp_millis is None or self.data_timestamp_millis < timestamp_millis: - self.data_timestamp_millis = timestamp_millis - - return self - - def update_from_file(self, file, timestamp_millis): - self.file_count += 1 - self.record_count += file.record_count() - self.total_size += file.files_size_in_bytes() - - if self.data_timestamp_millis is None or self.data_timestamp_millis < timestamp_millis: - self.data_timestamp_millis = timestamp_millis - - return self - - def __repr__(self): - items = ("%s=%r" % (k, v) for k, v in self.__dict__.items()) - return "%s(%s)" % (self.__class__.__name__, ','.join(items)) - - def __str__(self): - return self.__repr__() - - -class SnapshotSummary(object): - GENIE_ID_PROP = "genie-id" - ADDED_FILES_PROP = "added-data-files" - DELETED_FILES_PROP = "deleted-data-files" - TOTAL_FILES_PROP = "total-data-files" - ADDED_RECORDS_PROP = "added-records" - DELETED_RECORDS_PROP = "deleted-records" - TOTAL_RECORDS_PROP = "total-records" - ADDED_FILE_SIZE_PROP = "added-files-size" - DELETED_DUPLICATE_FILES = "deleted-duplicate-files" - CHANGED_PARTITION_COUNT_PROP = "changed-partition-count" - CHANGED_PARTITION_PREFIX = "partitions." - PARTITION_SUMMARY_PROP = "partition-summaries-included" - - def __init__(self): - pass - - -class SnapshotSummaryBuilder(object): - - def __init__(self): - self.changed_partitions = dict() - self.added_files = 0 - self.deleted_files = 0 - self.deleted_dupicate_files = 0 - self.added_records = 0 - self.deleted_records = 0 - self.properties = dict() - - def clear(self): - self.changed_partitions = dict() - self.added_files = 0 - self.deleted_files = 0 - self.deleted_dupicate_files = 0 - self.added_records = 0 - self.deleted_records = 0 - - def increment_duplicate_deletes(self): - self.deleted_dupicate_files += 1 - - def deleted_file(self, spec, data_file): - self.update_partitiomns(spec, data_file, False) - self.deleted_files += 1 - self.deleted_records += data_file.record_count - - def added_file(self, spec, data_file): - self.update_partitiomns(spec, data_file, True) - self.added_files += 1 - self.added_records += data_file.record_count - - def update_partitions(self, spec, file, is_addition): - key = spec.partition_to_path(file.partition()) - metrics = self.changed_partitions.get(key, PartitionMetrics()) - - if is_addition: - self.changed_partitions[key] = metrics.update_from_file(file, None) - - def build(self): - builder = dict() - builder.update(self.properties) - - SnapshotSummaryBuilder.set_if(self.added_files > 0, builder, - SnapshotSummary.ADDED_FILES_PROP, self.added_files) - SnapshotSummaryBuilder.set_if(self.deleted_files > 0, - builder, SnapshotSummary.DELETED_FILES_PROP, self.deleted_files) - SnapshotSummaryBuilder.set_if(self.deleted_dupicate_files > 0, - builder, SnapshotSummary.DELETED_DUPLICATE_FILES, self.deleted_dupicate_files) - SnapshotSummaryBuilder.set_if(self.added_records > 0, - builder, SnapshotSummary.ADDED_RECORDS_PROP, self.added_records) - SnapshotSummaryBuilder.set_if(self.deleted_records > 0, - builder, SnapshotSummary.DELETED_RECORDS_PROP, self.deleted_records) - builder[SnapshotSummary.CHANGED_PARTITION_COUNT_PROP] = len(self.changed_partitions.items()) - - if len(self.changed_partitions.items()) < 100: - builder[SnapshotSummary.PARTITION_SUMMARY_PROP] = "true" - for key, metrics in self.changed_partitions: - metric_dict = {SnapshotSummary.ADDED_FILES_PROP: metrics.file_count, - SnapshotSummary.ADDED_RECORDS_PROP: metrics.record_count, - SnapshotSummary.ADDED_FILE_SIZE_PROP: metrics.total_size} - builder[SnapshotSummary.CHANGED_PARTITION_PREFIX + key] = ",".join(["{}={}".format(key, val) - for inner_key, val - in metric_dict.items()]) - - return builder - - def set(self, prop, value): - self.properties[prop] = value - - @staticmethod - def set_if(expression, builder, prop, value): - if expression: - builder[prop] = value diff --git a/iceberg/core/schema_parser.py b/iceberg/core/schema_parser.py deleted file mode 100644 index 3227b49a8a..0000000000 --- a/iceberg/core/schema_parser.py +++ /dev/null @@ -1,182 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -import json - -from iceberg.api import Schema -from iceberg.api.types import (from_primitive_string, - ListType, - MapType, - NestedField, - StructType, - Type, - TypeID) - - -class SchemaParser(object): - - TYPE = "type" - STRUCT = "struct" - LIST = "list" - MAP = "map" - FIELDS = "fields" - ELEMENT = "element" - KEY = "key" - VALUE = "value" - DOC = "doc" - NAME = "name" - ID = "id" - ELEMENT_ID = "element-id" - KEY_ID = "key-id" - VALUE_ID = "value-id" - REQUIRED = "required" - ELEMENT_REQUIRED = "element-required" - VALUE_REQUIRED = "value-required" - - @staticmethod - def to_json(type_var, indent=None): - return json.dumps(SchemaParser.to_dict(type_var), indent=indent) - - @staticmethod - def to_dict(type_var): - if isinstance(type_var, Schema): - struct = type_var.as_struct() - elif isinstance(type_var, Type): - struct = type_var - - return SchemaParser._type_to_dict(struct) - - @staticmethod - def _type_to_dict(type_var): - if type_var.is_primitive_type(): - return SchemaParser._primitive_to_dict(type_var) - else: - nested = type_var.as_nested_type() - if type_var.type_id == TypeID.STRUCT: - return SchemaParser._struct_to_dict(nested) - elif type_var.type_id == TypeID.LIST: - return SchemaParser._list_to_dict(nested) - elif type_var.type_id == TypeID.MAP: - return SchemaParser._map_to_dict(nested) - else: - raise RuntimeError("Cannot write unknown type: %s" % type) - - @staticmethod - def _primitive_to_dict(type_var): - return str(type_var) - - @staticmethod - def _struct_to_dict(struct): - return {SchemaParser.TYPE: SchemaParser.STRUCT, - SchemaParser.FIELDS: [SchemaParser._struct_field_to_dict(field) for field in struct.fields]} - - @staticmethod - def _struct_field_to_dict(field): - schema = {SchemaParser.ID: field.field_id, - SchemaParser.NAME: field.name, - SchemaParser.REQUIRED: field.is_required, - SchemaParser.TYPE: SchemaParser._type_to_dict(field.type)} - if field.doc is not None: - schema[SchemaParser.DOC] = field.doc - - return schema - - @staticmethod - def _list_to_dict(list_type): - return {SchemaParser.TYPE: SchemaParser.LIST, - SchemaParser.ELEMENT_ID: list_type.element_id, - SchemaParser.ELEMENT: SchemaParser._type_to_dict(list_type.element_type), - SchemaParser.ELEMENT_REQUIRED: list_type.is_element_required()} - - @staticmethod - def _map_to_dict(map_type): - return {SchemaParser.TYPE: SchemaParser.MAP, - SchemaParser.KEY_ID: map_type.key_id(), - SchemaParser.KEY: SchemaParser._type_to_dict(map_type.key_type()), - SchemaParser.VALUE_ID: map_type.value_id(), - SchemaParser.VALUE: SchemaParser._type_to_dict(map_type.value_type()), - SchemaParser.VALUE_REQUIRED: map_type.is_value_required()} - - @staticmethod - def type_from_dict(dict_obj): - if isinstance(dict_obj, (str, bool, int, float)): - return from_primitive_string(dict_obj) - else: - type_str = dict_obj.get(SchemaParser.TYPE) - if type_str.upper() == "STRUCT": - return SchemaParser.struct_from_dict(dict_obj) - elif type_str.upper() == "LIST": - return SchemaParser.list_from_dict(dict_obj) - elif type_str.upper() == "MAP": - return SchemaParser.map_from_dict(dict_obj) - else: - raise RuntimeError("Cannot parse type from dict: %s" % dict_obj) - - @staticmethod - def struct_from_dict(dict_obj): - struct_fields = list() - fields = dict_obj.get(SchemaParser.FIELDS) - for field in fields: - field_id = field.get(SchemaParser.ID) - field_name = field.get(SchemaParser.NAME) - field_type = SchemaParser.type_from_dict(field.get(SchemaParser.TYPE)) - field_doc = field.get(SchemaParser.DOC) - - if field.get(SchemaParser.REQUIRED): - struct_fields.append(NestedField.required(field_id, field_name, field_type, field_doc)) - else: - struct_fields.append(NestedField.optional(field_id, field_name, field_type, field_doc)) - - return StructType.of(struct_fields) - - @staticmethod - def list_from_dict(dict_obj): - elem_id = dict_obj.get(SchemaParser.ELEMENT_ID) - elem_type = SchemaParser.type_from_dict(dict_obj.get(SchemaParser.ELEMENT)) - is_required = dict_obj.get(SchemaParser.REQUIRED) - - if is_required: - return ListType.of_required(elem_id, elem_type) - else: - return ListType.of_optional(elem_id, elem_type) - - @staticmethod - def map_from_dict(dict_obj): - key_id = dict_obj.get(SchemaParser.KEY_ID) - key_type = SchemaParser.type_from_dict(dict_obj.get(SchemaParser.KEY)) - - value_id = dict_obj.get(SchemaParser.VALUE_ID) - value_type = SchemaParser.type_from_dict(dict_obj.get(SchemaParser.VALUE)) - - is_required = dict_obj.get(SchemaParser.VALUE_REQUIRED) - - if is_required: - return MapType.of_required(key_id, value_id, key_type, value_type) - else: - return MapType.of_optional(key_id, value_id, key_type, value_type) - - @staticmethod - def from_json(json_obj): - if isinstance(json_obj, str): - type_var = SchemaParser.type_from_dict(json.loads(json_obj)) - else: - type_var = SchemaParser.type_from_dict(json_obj) - - if type_var is not None and type_var.is_nested_type() and type_var.as_nested_type().is_struct_type(): - return Schema(type_var.as_nested_type().as_struct_type().fields) - else: - raise RuntimeError("Cannot create schema, not a struct type: %s" % type_var) diff --git a/iceberg/core/schema_update.py b/iceberg/core/schema_update.py deleted file mode 100644 index 8363801291..0000000000 --- a/iceberg/core/schema_update.py +++ /dev/null @@ -1,40 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -from iceberg.api import UpdateSchema - - -class SchemaUpdate(UpdateSchema): - TABLE_ROOT_ID = -1 - - def add_column(self, name, type, parent=None): - raise NotImplementedError() - - def rename_column(self, name, new_name): - raise NotImplementedError() - - def update_column(self, name, new_type): - raise NotImplementedError() - - def delete_column(self, name): - raise NotImplementedError() - - def apply(self): - raise NotImplementedError() - - def commit(self): - raise NotImplementedError() diff --git a/iceberg/core/snapshot_parser.py b/iceberg/core/snapshot_parser.py deleted file mode 100644 index 3e94af5c5d..0000000000 --- a/iceberg/core/snapshot_parser.py +++ /dev/null @@ -1,73 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -import json - -from .base_snapshot import BaseSnapshot - - -class SnapshotParser(object): - SNAPSHOT_ID = "snapshot-id" - PARENT_SNAPSHOT_ID = "parent-snapshot-id" - TIMESTAMP_MS = "timestamp-ms" - SUMMARY = "summary" - OPERATION = "operation" - MANIFESTS = "manifests" - MANIFEST_LIST = "manifest-list" - - @staticmethod - def to_json(snapshot): - return json.dumps(SnapshotParser.to_dict(snapshot)) - - @staticmethod - def to_dict(snapshot): - return {SnapshotParser.SNAPSHOT_ID: snapshot.snapshot_id, - SnapshotParser.TIMESTAMP_MS: snapshot.timestamp_millis, - SnapshotParser.PARENT_SNAPSHOT_ID: snapshot.parent_id, - SnapshotParser.OPERATION: snapshot.operation, - SnapshotParser.SUMMARY: snapshot.summary, - SnapshotParser.MANIFESTS: [manifest.manifest_path for manifest in snapshot.manifests]} - - @staticmethod - def from_json(ops, json_obj, spec=None): - if isinstance(json_obj, str): - json_obj = json.loads(json_obj) - - if not isinstance(json_obj, dict): - raise RuntimeError("Cannot parse table version from a non-object: %s" % json_obj) - - version_id = json_obj.get(SnapshotParser.SNAPSHOT_ID) - parent_id = json_obj.get(SnapshotParser.PARENT_SNAPSHOT_ID) - timestamp_millis = json_obj.get(SnapshotParser.TIMESTAMP_MS) - operation = json_obj.get(SnapshotParser.OPERATION) - summary = json_obj.get(SnapshotParser.SUMMARY) - - if SnapshotParser.MANIFEST_LIST in json_obj: - # the manifest list is stored in a manifest list file - return BaseSnapshot(ops, version_id, parent_id, - manifest_list=ops.new_input_file(json_obj.get(SnapshotParser.MANIFEST_LIST)), - timestamp_millis=timestamp_millis, - operation=operation, - summary=summary) - else: - manifests = json_obj.get(SnapshotParser.MANIFESTS, list()) - - return BaseSnapshot(ops, version_id, parent_id, - manifests=manifests, - timestamp_millis=timestamp_millis, - operation=operation, - summary=summary) diff --git a/iceberg/core/table_metadata.py b/iceberg/core/table_metadata.py deleted file mode 100644 index 92cb39e34e..0000000000 --- a/iceberg/core/table_metadata.py +++ /dev/null @@ -1,218 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -import time -from typing import Optional - -from iceberg.api import PartitionSpec, Schema -from iceberg.api.types import assign_fresh_ids -from iceberg.core.table_operations import TableOperations -from iceberg.core.util import AtomicInteger -from iceberg.exceptions import ValidationException - - -class TableMetadata(object): - INITIAL_SPEC_ID = 0 - TABLE_FORMAT_VERSION = 1 - - @staticmethod - def new_table_metadata(ops: TableOperations, schema: Schema, spec: PartitionSpec, location: str, - properties: dict = None) -> "TableMetadata": - last_column_id = AtomicInteger(0) - fresh_schema = assign_fresh_ids(schema, last_column_id.increment_and_get) - - spec_builder = PartitionSpec.builder_for(fresh_schema) - for field in spec.fields: - src_name = schema.find_column_name(field.source_id) - spec_builder.add(field.source_id, - fresh_schema.find_field(src_name).field_id, - field.name, - str(field.transform)) - - fresh_spec = spec_builder.build() - properties = properties if properties is not None else dict() - - return TableMetadata(ops, None, location, - int(time.time() * 1000), - last_column_id.get(), fresh_schema, TableMetadata.INITIAL_SPEC_ID, [fresh_spec], - properties, -1, list(), list()) - - def __init__(self, ops, file, location, last_updated_millis, - last_column_id, schema, default_spec_id, specs, properties, - current_snapshot_id, snapshots, snapshot_log): - self.ops = ops - self._file = file - self._location = location - self.last_updated_millis = last_updated_millis - self.last_column_id = last_column_id - self.schema = schema - self.default_spec_id = default_spec_id - self.specs = specs - self.properties = properties - self.properties["provider"] = "ICEBERG" - self.current_snapshot_id = current_snapshot_id - self.snapshots = snapshots - self.snapshot_log = snapshot_log - - self.snapshot_by_id = {version.snapshot_id: version for version in self.snapshots} - self.specs_by_id = {spec.spec_id: spec for spec in self.specs} - - last = None - for log_entry in snapshot_log: - if last is not None: - if not (log_entry.timestamp_millis - last.timestamp_millis > 0): - raise RuntimeError("[BUG] Expected sorted snapshot log entries.") - last = log_entry - - if not (len(self.snapshot_by_id) == 0 or self.current_snapshot_id in self.snapshot_by_id): - raise RuntimeError("Invalid table metadata: Cannot find current version") - - @property - def location(self: "TableMetadata") -> str: - return self._location - - @property - def metadata_location(self: "TableMetadata") -> Optional[str]: - return self._file.location() if self._file else None - - @property - def spec(self): - return self.specs_by_id[self.default_spec_id] - - def spec_id(self, spec_id): - if spec_id is None: - spec_id = self.default_spec_id - - return self.specs_by_id[spec_id] - - def property_as_int(self, property_name, default_value): - return int(self.properties.get(property_name, default_value)) - - def current_snapshot(self): - return self.snapshot_by_id[self.current_snapshot_id] - - def snapshot(self, snapshot_id): - return self.snapshot_by_id[snapshot_id] - - def update_metadata_location(self, new_location): - return TableMetadata(self.ops, None, new_location, - int(time.time() * 1000), self.last_column_id, self.schema, self.spec, self.properties, - self.current_snapshot_id, self.snapshots, self.snapshot_log) - - def update_schema(self, schema, last_column_id): - PartitionSpec.check_compatibility(self.spec, schema) - return TableMetadata(self.ops, None, self.location, - int(time.time() * 1000), last_column_id, schema, self.spec, self.properties, - self.current_snapshot_id, self.snapshots, self.snapshot_log) - - def add_snapshot(self, snapshot): - new_snapshots = self.snapshots + snapshot - new_snapshot_log = self.snapshot_log + [SnapshotLogEntry(snapshot.timestamp_millis, snapshot.snapshot_id)] - - return TableMetadata(self.ops, None, self.location, - int(time.time() * 1000), self.last_column_id, self.schema, self.spec, self.properties, - self.current_snapshot_id, new_snapshots, new_snapshot_log) - - def add_staged_snapshot(self, snapshot): - self.snapshots.append(snapshot) - return TableMetadata(self.ops, None, self.location, snapshot.timestamp_millis, - self.last_column_id, self.schema, self.default_spec_id, self.specs, - self.current_snapshot_id, self.snapshots, self.snapshot_log) - - def replace_current_snapshot(self, snapshot): - self.snapshots.append(snapshot) - self.snapshot_log.append(SnapshotLogEntry(snapshot.timestamp_millis, snapshot.snapshot_id)) - - return TableMetadata(self.ops, None, self.location, snapshot.timestamp_millis, - self.last_column_id, self.schema, self.default_spec_id, self.specs, - self.current_snapshot_id, self.snapshots, self.snapshot_log) - - def remove_snapshots_if(self, remove_if): - filtered = list() - - for snapshot in self.snapshots: - if snapshot.snapshot_id == self.current_snapshot_id or not remove_if(snapshot): - filtered.append(snapshot) - - valid_ids = [snapshot.snapshot_id for snapshot in filtered] - new_snapshot_log = list() - for log_entry in self.snapshot_log: - if log_entry.snapshot_id in valid_ids: - new_snapshot_log.append(log_entry) - else: - new_snapshot_log.clear() - - return TableMetadata(self.ops, None, self.location, - int(time.time() * 1000), self.last_column_id, self.schema, self.spec, self.properties, - self.current_snapshot_id, filtered, new_snapshot_log) - - def rollback_to(self, snapshot): - ValidationException.check(snapshot.snapshot_id not in self.snapshot_by_id, - "Cannot set current snapshot to unknown: %s", (snapshot.snapshot_id,)) - - now_millis = int(time.time() * 1000) - new_snapshot_log = self.snapshot_log + [SnapshotLogEntry(now_millis, snapshot.snapshot_id)] - - return TableMetadata(self.ops, None, self.location, - now_millis, self.last_column_id, self.schema, self.spec, self.properties, - snapshot.snapshot_id, self.snapshots, new_snapshot_log) - - def replace_properties(self, new_properties): - ValidationException.check(new_properties is not None, "Cannot set properties to null") - - return TableMetadata(self.ops, None, self.location, - int(time.time() * 1000), self.last_column_id, self.schema, self.spec, new_properties, - self.current_snapshot_id, self.snapshots, self.snapshot_log) - - def remove_snapshot_log_entries(self, snapshot_ids): - new_snapshot_log = list() - - for entry in self.snapshot_log: - if entry.snapshot_id not in snapshot_ids: - new_snapshot_log.append(entry) - - check_snapshot = self.current_snapshot_id < 0 or new_snapshot_log[-1].snapshot_id == self.current_snapshot_id - ValidationException.check(check_snapshot, - "Cannot set invalid snapshot log: latest entry is not the current snapshot") - - return TableMetadata(self.ops, None, self.location, - int(time.time() * 1000), self.last_column_id, self.schema, self.spec, self.properties, - self.current_snapshot_id, self.snapshots, new_snapshot_log) - - -class SnapshotLogEntry(object): - - def __init__(self, timestamp_millis, snapshot_id): - self.timestamp_millis = timestamp_millis - self.snapshot_id = snapshot_id - - def __hash__(self): - return hash(self.__key()) - - def __key(self): - return SnapshotLogEntry.__class__, self.snapshot_id, self.timestamp_millis - - def __eq__(self, other): - if id(self) == id(other): - return True - if other is None or not isinstance(other, SnapshotLogEntry): - return False - - return self.snapshot_id == other.snapshot_id and self.timestamp_millis == other.timestamp_millis - - def __ne__(self, other): - return not self.__eq__() diff --git a/iceberg/core/table_metadata_parser.py b/iceberg/core/table_metadata_parser.py deleted file mode 100644 index d7def842a4..0000000000 --- a/iceberg/core/table_metadata_parser.py +++ /dev/null @@ -1,124 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -import gzip -import json - -from .config_properties import ConfigProperties -from .partition_spec_parser import PartitionSpecParser -from .schema_parser import SchemaParser -from .snapshot_parser import SnapshotParser -from .table_metadata import (SnapshotLogEntry, - TableMetadata) - - -class TableMetadataParser(object): - FORMAT_VERSION = "format-version" - LOCATION = "location" - LAST_UPDATED_MILLIS = "last-updated-ms" - LAST_COLUMN_ID = "last-column-id" - SCHEMA = "schema" - PARTITION_SPEC = "partition-spec" - PARTITION_SPECS = "partition-specs" - DEFAULT_SPEC_ID = "default-spec-id" - PROPERTIES = "properties" - CURRENT_SNAPSHOT_ID = "current-snapshot-id" - SNAPSHOTS = "snapshots" - SNAPSHOT_ID = "snapshot-id" - TIMESTAMP_MS = "timestamp-ms" - LOG = "snapshot-log" - - @staticmethod - def to_json(metadata, indent=4): - return json.dumps({TableMetadataParser.FORMAT_VERSION: TableMetadata.TABLE_FORMAT_VERSION, - TableMetadataParser.LOCATION: metadata.location, - TableMetadataParser.LAST_UPDATED_MILLIS: metadata.last_updated_millis, - TableMetadataParser.LAST_COLUMN_ID: metadata.last_column_id, - TableMetadataParser.SCHEMA: SchemaParser.to_dict(metadata.schema), - TableMetadataParser.PARTITION_SPEC: PartitionSpecParser.to_json_fields(metadata.spec), - TableMetadataParser.DEFAULT_SPEC_ID: int(metadata.default_spec_id), - TableMetadataParser.PARTITION_SPECS: [PartitionSpecParser.to_dict(spec) - for spec in metadata.specs], - TableMetadataParser.PROPERTIES: metadata.properties, - TableMetadataParser.CURRENT_SNAPSHOT_ID: (metadata.current_snapshot_id - if metadata.current_snapshot_id is not None - else -1), - TableMetadataParser.SNAPSHOTS: [SnapshotParser.to_dict(snapshot) - for snapshot in metadata.snapshots], - TableMetadataParser.LOG: [{TableMetadataParser.TIMESTAMP_MS: log_entry.timestamp_millis, - TableMetadataParser.SNAPSHOT_ID: log_entry.snapshot_id} - for log_entry in metadata.snapshot_log]}, indent=indent) - - @staticmethod - def write(metadata, metadata_location): - if metadata_location.location().endswith(".gz"): - output_file = gzip.open(metadata_location.create("wb"), "wb") - else: - output_file = metadata_location.create("wb") - - json_str = TableMetadataParser.to_json(metadata) - output_file.write(json_str.encode("utf-8")) - output_file.close() - - @staticmethod - def get_file_extension(config): - return ".metadata.json.gz" if ConfigProperties.should_compress(config) else ".metadata.json" - - @staticmethod - def read(ops, file): - metadata = "".join([line.decode("utf-8") for line in file.new_stream(gzipped=file.location().endswith("gz"))]) - return TableMetadataParser.from_json(ops, file.location(), metadata) - - @staticmethod - def from_json(ops, file, json_obj): - if isinstance(json_obj, str): - json_obj = json.loads(json_obj) - - if not isinstance(json_obj, dict): - raise RuntimeError("Cannot parse metadata from non-object: %s" % json_obj) - - format_version = json_obj.get(TableMetadataParser.FORMAT_VERSION) - if format_version != TableMetadata.TABLE_FORMAT_VERSION: - raise RuntimeError("Cannot read unsupported version: %s" % format_version) - - location = json_obj.get(TableMetadataParser.LOCATION) - last_assigned_column = json_obj.get(TableMetadataParser.LAST_COLUMN_ID) - schema = SchemaParser.from_json(json_obj.get(TableMetadataParser.SCHEMA)) - - spec_array = json_obj.get(TableMetadataParser.PARTITION_SPECS) - if spec_array is not None: - default_spec_id = json_obj.get(TableMetadataParser.DEFAULT_SPEC_ID) - specs = [PartitionSpecParser.from_json(schema, spec) - for spec in spec_array] - else: - default_spec_id = TableMetadata.INITIAL_SPEC_ID - specs = (PartitionSpecParser.from_json_fields(schema, - default_spec_id, - json_obj.get(TableMetadataParser.PARTITION_SPEC)),) - - props = json_obj.get(TableMetadataParser.PROPERTIES) - current_version_id = json_obj.get(TableMetadataParser.CURRENT_SNAPSHOT_ID) - last_updated_millis = json_obj.get(TableMetadataParser.LAST_UPDATED_MILLIS) - snapshots = [SnapshotParser.from_json(ops, snapshot) for snapshot in json_obj.get(TableMetadataParser.SNAPSHOTS)] - entries = [SnapshotLogEntry(log_entry.get(TableMetadataParser.TIMESTAMP_MS), - log_entry.get(TableMetadataParser.SNAPSHOT_ID)) - for log_entry in sorted(json_obj.get(TableMetadataParser.LOG, []), - key=lambda x: x.get(TableMetadataParser.TIMESTAMP_MS))] - - return TableMetadata(ops, file, location, - last_updated_millis, last_assigned_column, schema, default_spec_id, specs, props, current_version_id, - snapshots, entries) diff --git a/iceberg/core/table_operations.py b/iceberg/core/table_operations.py deleted file mode 100644 index 659b9b8b05..0000000000 --- a/iceberg/core/table_operations.py +++ /dev/null @@ -1,42 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -import struct -import uuid - - -class TableOperations(object): - - def current(self): - raise NotImplementedError() - - def refresh(self): - raise NotImplementedError() - - def commit(self, base, metadata): - raise NotImplementedError() - - def io(self): - raise NotImplementedError() - - def metadata_file_location(self, file): - raise NotImplementedError() - - def new_snapshot_id(self): - new_uuid = uuid.uuid4() - msb, lsb = struct.unpack(">qq", new_uuid.bytes) - return abs(msb ^ lsb) diff --git a/iceberg/core/table_properties.py b/iceberg/core/table_properties.py deleted file mode 100644 index da75eff960..0000000000 --- a/iceberg/core/table_properties.py +++ /dev/null @@ -1,78 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - - -class TableProperties(object): - COMMIT_NUM_RETRIES = "commit.retry.num-retries" - COMMIT_NUM_RETRIES_DEFAULT = 4 - - COMMIT_MIN_RETRY_WAIT_MS = "commit.retry.min-wait-ms" - COMMIT_MIN_RETRY_WAIT_MS_DEFAULT = 100 - - COMMIT_MAX_RETRY_WAIT_MS = "commit.retry.max-wait-ms" - COMMIT_MAX_RETRY_WAIT_MS_DEFAULT = 60000 - - COMMIT_TOTAL_RETRY_TIME_MS = "commit.retry.total-timeout-ms" - COMMIT_TOTAL_RETRY_TIME_MS_DEFAULT = 1800000 - - MANIFEST_TARGET_SIZE_BYTES = "commit.manifest.target-size-bytes" - MANIFEST_TARGET_SIZE_BYTES_DEFAULT = 8388608 - - MANIFEST_MIN_MERGE_COUNT = "commit.manifest.min-count-to-merge" - MANIFEST_MIN_MERGE_COUNT_DEFAULT = 100 - - DEFAULT_FILE_FORMAT = "write.format.default" - DEFAULT_FILE_FORMAT_DEFAULT = "parquet" - - PARQUET_ROW_GROUP_SIZE_BYTES = "write.parquet.row-group-size-bytes" - PARQUET_ROW_GROUP_SIZE_BYTES_DEFAULT = "134217728" - - PARQUET_PAGE_SIZE_BYTES = "write.parquet.page-size-bytes" - PARQUET_PAGE_SIZE_BYTES_DEFAULT = "1048576" - - PARQUET_DICT_SIZE_BYTES = "write.parquet.dict-size-bytes" - PARQUET_DICT_SIZE_BYTES_DEFAULT = "2097152" - - PARQUET_COMPRESSION = "write.parquet.compression-codec" - PARQUET_COMPRESSION_DEFAULT = "gzip" - - AVRO_COMPRESSION = "write.avro.compression-codec" - AVRO_COMPRESSION_DEFAULT = "gzip" - - SPLIT_SIZE = "read.split.target-size" - SPLIT_SIZE_DEFAULT = 134217728 - - SPLIT_LOOKBACK = "read.split.planning-lookback" - SPLIT_LOOKBACK_DEFAULT = 10 - - SPLIT_OPEN_FILE_COST = "read.split.open-file-cost" - SPLIT_OPEN_FILE_COST_DEFAULT = 4 * 1024 * 1024 - - OBJECT_STORE_ENABLED = "write.object-storage.enabled" - OBJECT_STORE_ENABLED_DEFAULT = False - - OBJECT_STORE_PATH = "write.object-storage.path" - - WRITE_LOCATION_PROVIDER_IMPL = "write.location-provider.impl" - - WRITE_NEW_DATA_LOCATION = "write.folder-storage.path" - - WRITE_METADATA_LOCATION = "write.metadata.path" - - MANIFEST_LISTS_ENABLED = "write.manifest-lists.enabled" - - MANIFEST_LISTS_ENABLED_DEFAULT = True diff --git a/iceberg/core/util/__init__.py b/iceberg/core/util/__init__.py deleted file mode 100644 index 3d4dcc0b02..0000000000 --- a/iceberg/core/util/__init__.py +++ /dev/null @@ -1,36 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - - -__all__ = ["AtomicInteger", - "PackingIterator", - "PLANNER_THREAD_POOL_SIZE_PROP", - "SCAN_THREAD_POOL_ENABLED", - "str_as_bool", - "WORKER_THREAD_POOL_SIZE_PROP", - ] - -from .atomic_integer import AtomicInteger -from .bin_packing import PackingIterator - -PLANNER_THREAD_POOL_SIZE_PROP = "iceberg.planner.num-threads" -WORKER_THREAD_POOL_SIZE_PROP = "iceberg.worker.num-threads" -SCAN_THREAD_POOL_ENABLED = "iceberg.scan.plan-in-worker-pool" - - -def str_as_bool(str_var): - return str_var is not None and str_var.lower() == "true" diff --git a/iceberg/core/util/atomic_integer.py b/iceberg/core/util/atomic_integer.py deleted file mode 100644 index 87d70236a6..0000000000 --- a/iceberg/core/util/atomic_integer.py +++ /dev/null @@ -1,36 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -from threading import RLock - - -class AtomicInteger(object): - - def __init__(self, value): - if not isinstance(value, int): - raise RuntimeError("Value %s is not an int" % value) - self.value = value - self._lock = RLock() - - def increment_and_get(self): - with self._lock: - self.value += 1 - return self.value - - def get(self): - with self._lock: - return self.value diff --git a/iceberg/core/util/bin_packing.py b/iceberg/core/util/bin_packing.py deleted file mode 100644 index 2b7f0e19d7..0000000000 --- a/iceberg/core/util/bin_packing.py +++ /dev/null @@ -1,79 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - - -from copy import deepcopy - - -class PackingIterator(object): - - def __init__(self, items, target_weight, lookback, weight_func): - self.items = deepcopy(items) - self.target_weight = target_weight - self.lookback = lookback - self.weight_func = weight_func - self.bins = list() - - def __iter__(self): - return self - - def __next__(self): - item_copy = list(self.items) - i = 0 - for item in item_copy: - weight = self.weight_func(item) - curr_bin = PackingIterator.find(self.bins, weight) - if curr_bin is not None: - curr_bin.add(item, weight) - self.items.pop(i) - i -= 1 - else: - curr_bin = Bin(self.target_weight) - self.items.pop(i) - i -= 1 - curr_bin.add(item, weight) - self.bins.append(curr_bin) - - if len(self.bins) > self.lookback: - return list(self.bins.pop(0).items) - i += 1 - - if len(self.bins) == 0: - raise StopIteration() - - return list(self.bins.pop(0).items) - - @staticmethod - def find(bins, weight): - for curr_bin in bins: - if curr_bin.can_add(weight): - return curr_bin - - -class Bin(object): - - def __init__(self, target_weight): - self.bin_weight = 0 - self.target_weight = target_weight - self.items = list() - - def can_add(self, weight): - return self.bin_weight + weight <= self.target_weight - - def add(self, item, weight): - self.bin_weight += weight - self.items.append(item) diff --git a/iceberg/core/util/profile.py b/iceberg/core/util/profile.py deleted file mode 100644 index 4df7acd9aa..0000000000 --- a/iceberg/core/util/profile.py +++ /dev/null @@ -1,36 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - - -from contextlib import contextmanager -import logging -import time - -_logger = logging.getLogger(__name__) - - -@contextmanager -def profile(label, stats_dict=None): - if stats_dict is None: - _logger.debug('PROFILE: %s starting' % label) - start = time.time() - yield - took = int((time.time() - start) * 1000) - if stats_dict is None: - _logger.debug('PROFILE: %s completed in %dms' % (label, took)) - else: - stats_dict[label] = stats_dict.get(label, 0) + took diff --git a/iceberg/exceptions/__init__.py b/iceberg/exceptions/__init__.py deleted file mode 100644 index 3250a59cac..0000000000 --- a/iceberg/exceptions/__init__.py +++ /dev/null @@ -1,30 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -__all__ = ["AlreadyExistsException", - "CommitFailedException", - "FileSystemNotFound", - "InvalidCastException", - "NoSuchTableException", - "ValidationException"] - -from .exceptions import (AlreadyExistsException, - CommitFailedException, - FileSystemNotFound, - InvalidCastException, - NoSuchTableException, - ValidationException) diff --git a/iceberg/exceptions/exceptions.py b/iceberg/exceptions/exceptions.py deleted file mode 100644 index 73bf641b21..0000000000 --- a/iceberg/exceptions/exceptions.py +++ /dev/null @@ -1,47 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - - -class AlreadyExistsException(RuntimeError): - pass - - -class CommitFailedException(RuntimeError): - pass - - -class FileSystemNotFound(Exception): - pass - - -class InvalidCastException(ValueError): - pass - - -class NoSuchTableException(RuntimeError): - pass - - -class ValidationException(RuntimeError): - - @staticmethod - def check(test, message, args): - if not test: - raise ValidationException(message, args) - - def __init__(self, message, args): - super(ValidationException, self).__init__(message % args) diff --git a/iceberg/hive/__init__.py b/iceberg/hive/__init__.py deleted file mode 100644 index 37b9a13613..0000000000 --- a/iceberg/hive/__init__.py +++ /dev/null @@ -1,25 +0,0 @@ -# -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. -# - - -__all__ = ["HiveTableOperations", "HiveTables"] - - -from .hive_table_operations import HiveTableOperations -from .hive_tables import HiveTables diff --git a/iceberg/hive/hive_table_operations.py b/iceberg/hive/hive_table_operations.py deleted file mode 100644 index d78e6aaa7e..0000000000 --- a/iceberg/hive/hive_table_operations.py +++ /dev/null @@ -1,208 +0,0 @@ -# -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. -# -import getpass -import logging -import socket -import time -from typing import List - -from hmsclient import HMSClient -from hmsclient.genthrift.hive_metastore.ttypes import AlreadyExistsException, EnvironmentContext, FieldSchema, \ - LockComponent, LockLevel, LockRequest, LockResponse, LockState, LockType, NoSuchObjectException, \ - SerDeInfo, StorageDescriptor, Table, TException - -from .hive_types import hive_types -from ..api import Schema -from ..api.types import Type -from ..core import BaseMetastoreTableOperations, TableMetadata -from ..core.filesystem import FileSystem, get_fs -from ..exceptions import AlreadyExistsException as IcebergAlreadyExistsException -from ..exceptions import CommitFailedException - - -class HiveTableOperations(BaseMetastoreTableOperations): - - def __init__(self: "HiveTableOperations", conf: dict, client: HMSClient, database: str, table: str) -> None: - super(HiveTableOperations, self).__init__(conf) - self._client = client - self.database = database - self.table = table - try: - self.refresh() - except NoSuchObjectException: - pass # Object hasn't been created yet, can't refresh. - - def refresh(self: "HiveTableOperations") -> TableMetadata: - with self._client as open_client: - tbl_info = open_client.get_table(self.database, self.table) - - table_type = tbl_info.parameters.get(BaseMetastoreTableOperations.TABLE_TYPE_PROP) - - if not table_type or table_type.lower() != BaseMetastoreTableOperations.ICEBERG_TABLE_TYPE_VALUE: - raise RuntimeError("Invalid table, not Iceberg: %s.%s" % (self.database, - self.table)) - - metadata_location = tbl_info.parameters.get(BaseMetastoreTableOperations.METADATA_LOCATION_PROP) - if not metadata_location: - raise RuntimeError("Invalid table, missing metadata_location: %s.%s" % (self.database, - self.table)) - - self.refresh_from_metadata_location(metadata_location) - - return self.current() - - def commit(self: "HiveTableOperations", base: TableMetadata, metadata: TableMetadata) -> None: - new_metadata_location = self.write_new_metadata(metadata, self.version + 1) - - threw = True - lock_id = None - try: - lock_id = self.acquire_lock() - if base: - with self._client as open_client: - tbl = open_client.get_table(self.database, self.table) - else: - current_time_millis = int(time.time()) - tbl = Table(self.table, - self.database, - getpass.getuser(), - current_time_millis // 1000, - current_time_millis // 1000, - sd=storage_descriptor(metadata), - tableType="EXTERNAL_TABLE", - parameters={'EXTERNAL': 'TRUE'}) - - tbl.sd = storage_descriptor(metadata) - metadata_location = tbl.parameters.get(BaseMetastoreTableOperations.METADATA_LOCATION_PROP, None) - base_metadata_location = base.metadata_location if base else None - if base_metadata_location != metadata_location: - raise CommitFailedException( - "Base metadata location '%s' is not same as the current table metadata location '%s' for %s.%s", - base_metadata_location, metadata_location, self.database, self.table) - - self.set_parameters(new_metadata_location, tbl) - - if base: - with self._client as open_client: - env_context = EnvironmentContext( - {"DO_NOT_UPDATE_STATS": "true"} - ) - open_client.alter_table_with_environment_context(self.database, self.table, tbl, env_context) - - else: - with self._client as open_client: - open_client.create_table(tbl) - threw = False - except AlreadyExistsException: - raise IcebergAlreadyExistsException("Table already exists: {}.{}".format(self.database, self.table)) - except TException as e: - if e and "Table/View 'HIVE_LOCKS' does not exist" in str(e): - raise Exception("""Failed to acquire locks from metastore because 'HIVE_LOCKS' doesn't - exist, this probably happened when using embedded metastore or doesn't create a - transactional meta table. To fix this, use an alternative metastore""", e) - - raise Exception("Metastore operation failed for {}.{}".format(self.database, self.table), e) - finally: - if threw: - self.io().delete(new_metadata_location) - self.unlock(lock_id) - - def set_parameters(self: "HiveTableOperations", new_metadata_location: str, tbl: Table) -> None: - parameters = tbl.parameters - - if not parameters: - parameters = dict() - - parameters[BaseMetastoreTableOperations.TABLE_TYPE_PROP] = \ - BaseMetastoreTableOperations.ICEBERG_TABLE_TYPE_VALUE.upper() - parameters[BaseMetastoreTableOperations.METADATA_LOCATION_PROP] = new_metadata_location - - if self.current_metadata_location and len(self.current_metadata_location) > 0: - parameters[BaseMetastoreTableOperations.PREVIOUS_METADATA_LOCATION_PROP] = self.current_metadata_location - - tbl.parameters = parameters - - def unlock(self: "HiveTableOperations", lock_id: int = None) -> None: - if lock_id: - try: - with self._client as open_client: - open_client.unlock(LockResponse(lock_id)) - except Exception as e: - logging.warning("Failed to unlock {}.{}".format(self.database, self.table), e) - - def acquire_lock(self: "HiveTableOperations") -> int: - lock_component = LockComponent(LockType.EXCLUSIVE, LockLevel.TABLE, self.database, self.table) - - lock_request = LockRequest([lock_component], user=getpass.getuser(), hostname=socket.gethostname()) - with self._client as open_client: - lock_response = open_client.lock(lock_request) - - state = lock_response.state - lock_id = lock_response.lockid - start = int(time.time()) - duration = 0 - timed_out = False - while not timed_out and state == LockState.WAITING: - with self._client as open_client: - lock_response = open_client.check_lock(lock_response) - state = lock_response.state - - duration = int(time.time()) - start - if duration > 3 * 60 * 1000: - timed_out = True - else: - time.sleep(0.05) - - if timed_out and state != LockState.ACQUIRED: - raise CommitFailedException("Timed out after {} ms waiting for lock on {}.{}".format(duration, - self.database, - self.table)) - - if state != LockState.ACQUIRED: - raise CommitFailedException( - "Could not acquire the lock on {}.{}, lock request ended in state {}".format(self.database, self.table, - state)) - return lock_id - - def io(self: "HiveTableOperations") -> FileSystem: - return get_fs(self.base_location, self.conf) - - def close(self: "HiveTableOperations") -> None: - self._client.close() - - -def storage_descriptor(metadata: TableMetadata) -> StorageDescriptor: - ser_de_info = SerDeInfo(serializationLib="org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe") - return StorageDescriptor(columns(metadata.schema), - metadata.location, - "org.apache.hadoop.mapred.FileInputFormat", - "org.apache.hadoop.mapred.FileOutputFormat", - serdeInfo=ser_de_info) - - -def columns(schema: Schema) -> List[FieldSchema]: - return [FieldSchema(col.name, convert_hive_type(col.type), "") for col in schema.columns()] - - -def convert_hive_type(col_type: Type) -> str: - type_id = col_type.type_id - hive_type_id = hive_types.get(type_id) # type: ignore - if hive_type_id: - return hive_type_id - raise NotImplementedError("Not yet implemented column type " + str(col_type)) diff --git a/iceberg/hive/hive_tables.py b/iceberg/hive/hive_tables.py deleted file mode 100644 index 9260a91ea6..0000000000 --- a/iceberg/hive/hive_tables.py +++ /dev/null @@ -1,97 +0,0 @@ -# -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. -# - -import logging -from multiprocessing import cpu_count -from multiprocessing.dummy import Pool -import threading -from typing import Set - -from hmsclient import HMSClient, hmsclient -from iceberg.core import BaseMetastoreTables -from iceberg.core.util import WORKER_THREAD_POOL_SIZE_PROP -from iceberg.hive import HiveTableOperations - -_logger = logging.getLogger(__name__) - - -# Handles physical deletion of files. -# Does not delete a file if it is referred more than once via Iceberg snapshot metadata pointers. -class DeleteFiles(object): - - def __init__(self, ops: HiveTableOperations): - self.ops = ops - self.seen: Set[str] = set() - self.set_lock = threading.Lock() - - def delete_file(self, path: str) -> None: - have_seen = True - with self.set_lock: - if path not in self.seen: - have_seen = False - self.seen.add(path) - - if not have_seen: - _logger.info("Deleting file: {path}".format(path=path)) - try: - self.ops.delete_file(path) - except OSError as e: - _logger.info("Error deleting file: {path}: {e}".format(path=path, e=e)) - - -class HiveTables(BaseMetastoreTables): - _DOT = "." - THRIFT_URIS = "hive.metastore.uris" - - def __init__(self, conf): - super(HiveTables, self).__init__(conf) - - def new_table_ops(self, conf, database, table): - return HiveTableOperations(conf, self.get_client(), database, table) - - def get_client(self) -> HMSClient: - from urllib.parse import urlparse - metastore_uri = urlparse(self.conf[HiveTables.THRIFT_URIS]) - - client = hmsclient.HMSClient(host=metastore_uri.hostname, port=metastore_uri.port) - return client - - def drop(self, database: str, table: str, purge: bool = False) -> None: - ops = self.new_table_ops(self.conf, database, table) - metadata = ops.current() - - # Drop from Hive Metastore - with self.get_client() as open_client: - _logger.info("Deleting {database}.{table} from Hive Metastore".format(database=database, table=table)) - open_client.drop_table(database, table, deleteData=False) - - if purge: - # Follow Iceberg metadata pointers to delete every file - if metadata is not None: - with Pool(self.conf.get(WORKER_THREAD_POOL_SIZE_PROP, - cpu_count())) as delete_pool: - deleter = DeleteFiles(ops) - for s in metadata.snapshots: - for m in s.manifests: - delete_pool.map(deleter.delete_file, - (i.path() for i in s.get_filtered_manifest(m.manifest_path).iterator())) - delete_pool.map(deleter.delete_file, (m.manifest_path for m in s.manifests)) - if s.manifest_location is not None: - delete_pool.map(deleter.delete_file, [s.manifest_location]) - delete_pool.map(deleter.delete_file, [ops.current_metadata_location]) diff --git a/iceberg/hive/hive_types.py b/iceberg/hive/hive_types.py deleted file mode 100644 index 39aade23f6..0000000000 --- a/iceberg/hive/hive_types.py +++ /dev/null @@ -1,37 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -from iceberg.api.types import TypeID - -hive_types = { - TypeID.BOOLEAN: 'boolean', - TypeID.INTEGER: 'int', - TypeID.LONG: 'bigint', - TypeID.FLOAT: 'float', - TypeID.DOUBLE: 'double', - TypeID.DATE: 'date', - TypeID.TIME: 'string', - TypeID.TIMESTAMP: 'timestamp', - TypeID.STRING: 'string', - TypeID.UUID: 'string', - TypeID.FIXED: 'binary', - TypeID.BINARY: "binary", - TypeID.DECIMAL: None, - TypeID.STRUCT: None, - TypeID.LIST: None, - TypeID.MAP: None -} diff --git a/iceberg/parquet/__init__.py b/iceberg/parquet/__init__.py deleted file mode 100644 index af6dab4e05..0000000000 --- a/iceberg/parquet/__init__.py +++ /dev/null @@ -1,20 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -__all__ = ["ParquetReader"] - -from .parquet_reader import ParquetReader diff --git a/iceberg/parquet/dataset_utils.py b/iceberg/parquet/dataset_utils.py deleted file mode 100644 index 7472b2bddc..0000000000 --- a/iceberg/parquet/dataset_utils.py +++ /dev/null @@ -1,158 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - - -from iceberg.api.expressions import Expression, Operation, Predicate -import pyarrow.dataset as ds - - -def get_dataset_filter(expr: Expression, expected_to_file_map: dict) -> ds.Expression: - """ - Given an Iceberg Expression and a mapping of names in the iceberg schema to the file schema, - convert to an equivalent dataset filter using the file column names. Recursively iterate through the - expressions to convert each portion one predicate at a time - - Parameters - ---------- - expr : iceberg.api.expressions.Expression - An Iceberg Expression to be converted - expected_to_file_map : dict - A dict that maps the iceberg schema names to the names from the file schema - Returns - ------- - pyarrow._dataset.Expression - An equivalent dataset expression - """ - if expr is None: - return None - - if isinstance(expr, Predicate): - return predicate(expr, expected_to_file_map) - - if expr.op() == Operation.TRUE: - return None - elif expr.op() == Operation.FALSE: - return False - elif expr.op() == Operation.NOT: - return not_(get_dataset_filter(expr.child, expected_to_file_map)) - elif expr.op() == Operation.AND: - return and_(get_dataset_filter(expr.left, expected_to_file_map), - get_dataset_filter(expr.right, expected_to_file_map)) - elif expr.op() == Operation.OR: - return or_(get_dataset_filter(expr.left, expected_to_file_map), - get_dataset_filter(expr.right, expected_to_file_map)) - else: - raise RuntimeError("Unknown operation: {}".format(expr.op())) - - -def predicate(pred: Predicate, field_map: dict) -> ds.Expression: # noqa: ignore=C901 - """ - Given an Iceberg Predicate and a mapping of names in the iceberg schema to the file schema, - convert to an equivalent dataset expression using the file column names. - - Parameters - ---------- - pred : iceberg.api.expressions.Predicate - An Iceberg Predicate to be converted - field_map : dict - A dict that maps the iceberg schema names to the names from the file schema - Returns - ------- - pyarrow._dataset.Expression - An equivalent dataset expression - """ - # get column name in the file schema so we can apply the predicate - col_name = field_map.get(pred.ref.name) - - if col_name is None: - if pred.op == Operation.IS_NULL: - return ds.scalar(True) == ds.scalar(True) - - return ds.scalar(True) == ds.scalar(False) - - if pred.op == Operation.IS_NULL: - return ~ds.field(col_name).is_valid() - elif pred.op == Operation.NOT_NULL: - return ds.field(col_name).is_valid() - elif pred.op == Operation.LT: - return ds.field(col_name) < pred.lit.value - elif pred.op == Operation.LT_EQ: - return ds.field(col_name) <= pred.lit.value - elif pred.op == Operation.GT: - return ds.field(col_name) > pred.lit.value - elif pred.op == Operation.GT_EQ: - return ds.field(col_name) >= pred.lit.value - elif pred.op == Operation.EQ: - return ds.field(col_name) == pred.lit.value - elif pred.op == Operation.NOT_EQ: - return ds.field(col_name) != pred.lit.value - elif pred.op == Operation.IN: - return ds.field(col_name).isin(pred.lit.value) - elif pred.op == Operation.NOT_IN: - return ds.field(col_name).isin(pred.lit.value) - - -def and_(left: ds.Expression, right: ds.Expression) -> ds.Expression: - """ - Given a left and right expression combined them using the `AND` logical operator - - Parameters - ---------- - left : pyarrow._dataset.Expression - A Dataset `Expression` to logically `AND` - right : pyarrow._dataset.Expression - A Dataset `Expression` to logically `AND` - Returns - ------- - pyarrow._dataset.Expression - The left and right `Expression` combined with `AND` - """ - return left & right - - -def or_(left: ds.Expression, right: ds.Expression) -> ds.Expression: - """ - Given a left and right expression combined them using the `OR` logical operator - - Parameters - ---------- - left : pyarrow._dataset.Expression - A Dataset `Expression` to logically `OR` - right : pyarrow._dataset.Expression - A Dataset `Expression` to logically `OR` - Returns - ------- - pyarrow._dataset.Expression - The left and right `Expression` combined with `OR` - """ - return left | right - - -def not_(child: ds.Expression) -> ds.Expression: - """ - Given a child expression create the logical negation - - Parameters - ---------- - child : pyarrow._dataset.Expression - A Dataset `Expression` to logically `OR` - Returns - ------- - pyarrow._dataset.Expression - The negation of the input `Expression` - """ - return ~child diff --git a/iceberg/parquet/parquet_reader.py b/iceberg/parquet/parquet_reader.py deleted file mode 100644 index 0728546961..0000000000 --- a/iceberg/parquet/parquet_reader.py +++ /dev/null @@ -1,224 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - - -from datetime import datetime -import decimal -import logging -import typing - -from iceberg.api import Schema -from iceberg.api.expressions import Expression -from iceberg.api.io import InputFile -from iceberg.api.types import NestedField, Type, TypeID -from iceberg.core.filesystem import FileSystem, LocalFileSystem, S3FileSystem -from iceberg.core.util.profile import profile -from iceberg.exceptions import FileSystemNotFound, InvalidCastException -import numpy as np -import pyarrow as pa -from pyarrow import fs -import pyarrow.dataset as ds -import pyarrow.parquet as pq - -from .dataset_utils import get_dataset_filter -from .parquet_schema_utils import prune_columns -from .parquet_to_iceberg import convert_parquet_to_iceberg - -_logger = logging.getLogger(__name__) - -DTYPE_MAP: typing.Dict[TypeID, - typing.Callable[[NestedField], typing.Tuple[pa.Field, typing.Any]]] = \ - {TypeID.BINARY: lambda field: pa.binary(), - TypeID.BOOLEAN: lambda field: (pa.bool_(), False), - TypeID.DATE: lambda field: (pa.date32(), datetime.now()), - TypeID.DECIMAL: lambda field: (pa.decimal128(field.type.precision, field.type.scale), - decimal.Decimal()), - TypeID.DOUBLE: lambda field: (pa.float64(), np.nan), - TypeID.FIXED: lambda field: pa.binary(field.length), - TypeID.FLOAT: lambda field: (pa.float32(), np.nan), - TypeID.INTEGER: lambda field: (pa.int32(), np.nan), - TypeID.LIST: lambda field: (pa.list_(pa.field("element", - DTYPE_MAP[field.type.element_type.type_id](field.type)[0])), - None), - TypeID.LONG: lambda field: (pa.int64(), np.nan), - # To-Do: update to support reading map fields - # TypeID.MAP: lambda field: (,), - TypeID.STRING: lambda field: (pa.string(), ""), - TypeID.STRUCT: lambda field: (pa.struct([(nested_field.name, - DTYPE_MAP[nested_field.type.type_id](nested_field.type)[0]) - for nested_field in field.type.fields]), {}), - TypeID.TIMESTAMP: lambda field: (pa.timestamp("us"), datetime.now()), - # not used in SPARK, so not implementing for now - # TypeID.TIME: pa.time64(None) - } - -FS_MAP: typing.Dict[typing.Type[FileSystem], typing.Type[fs.FileSystem]] = {LocalFileSystem: fs.LocalFileSystem} - -try: - FS_MAP[S3FileSystem] = fs.S3FileSystem - -except ImportError: - _logger.warning("Mapped filesystem not available") - - -class ParquetReader(object): - - def __init__(self, input: InputFile, expected_schema: Schema, options, filter_expr: Expression, - case_sensitive: bool, start: int = None, end: int = None): - self._stats: typing.Dict[str, int] = dict() - - self._input = input - self._input_fo = input.new_fo() - - self._arrow_file = pq.ParquetFile(self._input_fo) - self._file_schema = convert_parquet_to_iceberg(self._arrow_file) - self._expected_schema = expected_schema - self._file_to_expected_name_map = ParquetReader.get_field_map(self._file_schema, - self._expected_schema) - self._options = options - self._filter = get_dataset_filter(filter_expr, ParquetReader.get_reverse_field_map(self._file_schema, - self._expected_schema)) - - self._case_sensitive = case_sensitive - if start is not None or end is not None: - raise NotImplementedError("Partial file reads are not yet supported") - # self.start = start - # self.end = end - - self.materialized_table = False - self._table = None - - _logger.debug("Reader initialized for %s" % self._input.path) - - @property - def stats(self) -> typing.Dict[str, int]: - return dict(self._stats) - - def read(self, force=False) -> pa.Table: - if not self.materialized_table or force: - self._read_data() - - return self._table - - def _read_data(self) -> None: - _logger.debug("Starting data read") - - # only scan the columns projected and in our file - cols_to_read = prune_columns(self._file_schema, self._expected_schema) - - with profile("read data", self._stats): - try: - read_fs = FS_MAP[type(self._input.fs)] - except KeyError: - raise FileSystemNotFound(f"No mapped filesystem found for {type(self._input.fs)}") - - arrow_dataset = ds.FileSystemDataset.from_paths([self._input.location()], - schema=self._arrow_file.schema_arrow, - format=ds.ParquetFileFormat(), - filesystem=read_fs()) - - arrow_table = arrow_dataset.to_table(columns=cols_to_read, filter=self._filter) - - # process schema evolution if needed - with profile("schema_evol_proc", self._stats): - processed_tbl = self.migrate_schema(arrow_table) - for i, field in self.get_missing_fields(): - dtype_func = DTYPE_MAP.get(field.type.type_id) - if dtype_func is None: - raise RuntimeError("Unable to create null column for type %s" % field.type.type_id) - - dtype = dtype_func(field) - processed_tbl = (processed_tbl.add_column(i, - pa.field(field.name, dtype[0], True, None), - ParquetReader.create_null_column(processed_tbl[0], - dtype))) - self._table = processed_tbl - self.materialized_table = True - - def __enter__(self): - return self - - def __exit__(self, exc_type, exc_val, exc_tb): - _logger.debug(self._stats) - self.close() - - def close(self): - self._input_fo.close() - - def get_missing_fields(self) -> typing.List[typing.Tuple[int, NestedField]]: - return [(i, field) for i, field in enumerate(self._expected_schema.as_struct().fields) - if self._file_schema.find_field(field.id) is None] - - @staticmethod - def get_field_map(file_schema, expected_schema) -> typing.Dict[str, str]: - return {file_schema.find_field(field.id).name: field.name - for field in expected_schema.as_struct().fields - if file_schema.find_field(field.id) is not None} - - @staticmethod - def get_reverse_field_map(file_schema, expected_schema) -> typing.Dict[str, str]: - return {expected_schema.find_field(field.id).name: field.name - for field in file_schema.as_struct().fields - if expected_schema.find_field(field.id) is not None} - - def migrate_schema(self, table: pa.Table) -> pa.Table: - data_arrays: typing.List[pa.ChunkedArray] = [] - schema: typing.List[pa.Field] = [] - for key, value in self._file_to_expected_name_map.items(): - column_idx: int = table.schema.get_field_index(key) - column_field: pa.Field = table.schema[column_idx] - column_arrow_type: pa.DataType = column_field.type - column_data: pa.ChunkedArray = table[column_idx] - - iceberg_field: NestedField = self._expected_schema.find_field(value) - converted_field: NestedField = self._file_schema.find_field(key) - - if iceberg_field.type != converted_field.type: - if not ParquetReader.is_supported_cast(converted_field.type, iceberg_field.type): - _logger.error(f"unsupported cast {converted_field.type} -> {iceberg_field.type}") - raise InvalidCastException("") - try: - column_arrow_type = DTYPE_MAP[iceberg_field.type.type_id](iceberg_field)[0] - column_data = column_data.cast(column_arrow_type) - except KeyError: - _logger.error(f"Unable to map {iceberg_field.type} to an arrow type") - raise - - data_arrays.append(column_data) - schema.append(pa.field(value, column_arrow_type, column_field.nullable, column_field.metadata)) - - return pa.table(data_arrays, schema=pa.schema(schema)) - - @staticmethod - def create_null_column(reference_column: pa.ChunkedArray, dtype_tuple: typing.Tuple[pa.DataType, typing.Any]) -> pa.ChunkedArray: - dtype, init_val = dtype_tuple - return pa.chunked_array([pa.array(np.full(len(c), init_val), - type=dtype, - mask=np.array([True] * len(reference_column.chunks[0]), dtype="bool")) - for c in reference_column.chunks], type=dtype) - - @staticmethod - def is_supported_cast(old_type: Type, new_type: Type) -> bool: - if old_type.type_id == TypeID.INTEGER and new_type.type_id == TypeID.LONG: - return True - elif old_type.type_id == TypeID.FLOAT and new_type.type_id == TypeID.DOUBLE: - return True - elif old_type.type_id == TypeID.DECIMAL and new_type.type_id == TypeID.DECIMAL \ - and old_type.precision < new_type.precision \ - and old_type.scale == new_type.scale: - return True - return False diff --git a/iceberg/parquet/parquet_schema_utils.py b/iceberg/parquet/parquet_schema_utils.py deleted file mode 100644 index cdccfd5e72..0000000000 --- a/iceberg/parquet/parquet_schema_utils.py +++ /dev/null @@ -1,41 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -from typing import List - -from iceberg.api import Schema -from iceberg.api.types import get_projected_ids - - -def prune_columns(file_schema: Schema, expected_schema: Schema) -> List[str]: - """ - Given two Iceberg schema's returns a list of column_names for all id's in the - file schema that are projected in the expected schema - - Parameters - ---------- - file_schema : iceberg.api.Schema - An Iceberg schema of the file being read - expected_schema : iceberg.api.Schema - An Iceberg schema of the final projection - Returns - ------- - list - The column names in the file that matched ids in the expected schema - """ - return [column.name for column in file_schema.as_struct().fields - if column.id in get_projected_ids(expected_schema)] diff --git a/iceberg/parquet/parquet_to_iceberg.py b/iceberg/parquet/parquet_to_iceberg.py deleted file mode 100644 index d2ea15c5af..0000000000 --- a/iceberg/parquet/parquet_to_iceberg.py +++ /dev/null @@ -1,152 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - - -import logging - -from iceberg.api import Schema -from iceberg.api.types import (BinaryType, - BooleanType, - DateType, - DecimalType, - DoubleType, - FixedType, - FloatType, - IntegerType, - ListType, - LongType, - MapType, - NestedField, - StringType, - StructType, - TimestampType) -from iceberg.api.types import Type -import pyarrow as pa -from pyarrow.parquet import lib, ParquetFile - -_logger = logging.getLogger(__name__) - -arrow_type_map = {lib.Type_BOOL: lambda x=None: BooleanType.get(), - lib.Type_DATE32: lambda x=None: DateType.get(), - lib.Type_DECIMAL128: lambda x=None: DecimalType.of(x.precision, x.scale), - lib.Type_DOUBLE: lambda x=None: DoubleType.get(), - lib.Type_FIXED_SIZE_BINARY: lambda x=None: FixedType.of_length(x.byte_width), - lib.Type_BINARY: lambda x=None: BinaryType.get(), - lib.Type_FLOAT: lambda x=None: FloatType.get(), - lib.Type_STRING: lambda x=None: StringType.get(), - lib.Type_INT32: lambda x=None: IntegerType.get(), - lib.Type_INT64: lambda x=None: LongType.get(), - lib.Type_TIMESTAMP: lambda x=None: (TimestampType.without_timezone() - if x.tz is None - else TimestampType.with_timezone()) - } - - -def get_nested_field(field_id: int, field_name: str, field_type: Type, nullable: bool) -> NestedField: - if nullable: - return NestedField.optional(field_id, field_name, field_type) - else: - return NestedField.required(field_id, field_name, field_type) - - -def get_list_field(col_type: pa.ListType) -> ListType: - if col_type.value_field.nullable: - return ListType.of_optional(int(col_type.value_field.metadata[b"PARQUET:field_id"].decode("utf-8")), - arrow_type_map[col_type.value_field.type.id](col_type.value_field.type)) - else: - return ListType.of_required(col_type.value_field.metadata[b"PARQUET:field_id"].decode("utf-8"), - arrow_type_map[col_type.value_field.type.id](col_type.value_field.type)) - - -def get_field(col: pa.Field) -> NestedField: - try: - return get_nested_field(int(col.metadata[b"PARQUET:field_id"].decode("utf-8")), - col.name, - arrow_type_map[col.type.id](col.type), - col.nullable) - except KeyError: - _logger.error(f"\t{int(col.metadata[b'PARQUET:field_id'].decode('utf-8'))}: {col.type}") - raise - - -def get_struct_field(col_type: pa.StructType) -> StructType: - if col_type.num_fields > 0: - if col_type[0].name == "map": - return get_inferred_map(col_type) - else: - return StructType.of([get_field(child) for child in col_type]) - else: - raise RuntimeError("Unable to convert type to iceberg %s" % col_type) - - -def get_inferred_map(col_type: pa.StructType): - base_field = col_type[0] - key_field = get_field(col_type[0].type.value_field.type[0]) - value_field = get_field(col_type[0].type.value_field.type[1]) - - if base_field.nullable: - return MapType.of_optional(key_field.field_id, value_field.field_id, - key_field.type, value_field.type) - else: - return MapType.of_required(key_field.field_id, value_field.field_id, - key_field.type, value_field.type) - - -def get_map_field(col): - # Spark Map type gets serialized to Struct>> - raise NotImplementedError("Arrow Map type not implemented yet") - - -# adding function mappings for nested types -arrow_type_map.update({lib.Type_LIST: get_list_field, - lib.Type_MAP: get_map_field, - lib.Type_STRUCT: get_struct_field}) - - -def convert_parquet_to_iceberg(parquet_file: ParquetFile) -> Schema: # noqa: ignore=C901 - """ - Given two Iceberg schema's returns a list of column_names for all id's in the - file schema that are projected in the expected schema - - Parameters - ---------- - parquet_file : pyarrow.parquet.ParquetFile - The Parquet File to use to extract the iceberg schema - - Returns - ------- - iceberg.api.Schema - returns an equivalent iceberg Schema based on the arrow schema read from the file - """ - return arrow_to_iceberg(parquet_file.schema_arrow) - - -def arrow_to_iceberg(arrow_schema: pa.Schema) -> Schema: - """ - Use an arrow schema, which contains the field_id metadata, to create an equivalent iceberg Schema - - Parameters - ---------- - arrow_schema : pyarrow.Schema - An Arrow schema with the parquet field_id metadata - - Returns - ------- - iceberg.api.Schema - returns an equivalent iceberg Schema based on the arrow schema read from the file - """ - return Schema([get_field(col) for col in arrow_schema]) diff --git a/setup.py b/setup.py deleted file mode 100755 index cbf120b8b7..0000000000 --- a/setup.py +++ /dev/null @@ -1,55 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -from setuptools import setup - -setup( - name='iceberg', - maintainer='Apache Iceberg Devs', - author_email='dev@iceberg.apache.org', - description='Iceberg is a new table format for storing large, slow-moving tabular data', - keywords='iceberg', - url='https://github.com/apache/iceberg/blob/master/README.md', - python_requires='>=3.7', - install_requires=['botocore', - 'boto3', - 'fastavro>=1.3.2<1.4.0', - 'hmsclient==0.1.1', - 'mmh3', - 'pyparsing>=2.4.7<2.5.0', - 'python-dateutil', - 'pytz', - 'requests', - 'retrying', - 'pandas', - 'pyarrow>=3.0.0,<=4.0.1' - ], - extras_require={ - "dev": [ - "tox-travis==0.12", - "virtualenv<20.0.0", - ], - }, - setup_requires=['setupmeta'], - license="Apache License 2.0", - classifiers=[ - "License :: OSI Approved :: Apache Software License", - "Programming Language :: Python :: 3.7", - "Programming Language :: Python :: 3.8", - "Programming Language :: Python :: 3.9", - ], -) diff --git a/tests/__init__.py b/tests/__init__.py deleted file mode 100644 index 13a83393a9..0000000000 --- a/tests/__init__.py +++ /dev/null @@ -1,16 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. diff --git a/tests/api/__init__.py b/tests/api/__init__.py deleted file mode 100644 index 13a83393a9..0000000000 --- a/tests/api/__init__.py +++ /dev/null @@ -1,16 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. diff --git a/tests/api/expressions/__init__.py b/tests/api/expressions/__init__.py deleted file mode 100644 index 13a83393a9..0000000000 --- a/tests/api/expressions/__init__.py +++ /dev/null @@ -1,16 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. diff --git a/tests/api/expressions/conftest.py b/tests/api/expressions/conftest.py deleted file mode 100644 index a0e748f582..0000000000 --- a/tests/api/expressions/conftest.py +++ /dev/null @@ -1,425 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -from decimal import Decimal -import io -import pickle -import time -import uuid - -from iceberg.api import DataFile, ManifestFile, PartitionFieldSummary, PartitionSpec -from iceberg.api.expressions import (BoundPredicate, - Expressions, - ExpressionVisitors, - Literal, - Operation, - UnboundPredicate) -from iceberg.api.schema import Schema -from iceberg.api.struct_like import StructLike -from iceberg.api.types import (BinaryType, - Conversions, - DateType, - DecimalType, - FixedType, - IntegerType, - NestedField, - StringType, - TimestampType, - TimeType) -import pytest - -exp_schema = Schema(NestedField.optional(34, "a", IntegerType.get())) - - -class TestHelpers(object): - - @staticmethod - def assert_all_references_bound(message, expr): - ExpressionVisitors.visit(expr, TestHelpers.CheckReferencesBound(message)) - - @staticmethod - def assert_and_unwrap(expr, expected=None): - if expected is not None: - assert isinstance(expr, expected) - else: - assert isinstance(expr, BoundPredicate) - - return expr - - @staticmethod - def round_trip_serialize(type_var): - stream = io.BytesIO() - pickle.dump(type_var, stream, pickle.HIGHEST_PROTOCOL) - stream.seek(0) - - return pickle.load(stream) - - class Row(StructLike): - - @staticmethod - def of(values=None): - return TestHelpers.Row(values) - - def __init__(self, values): - self.values = values - - def get(self, pos): - return self.values[pos] - - def set(self, pos, value): - raise RuntimeError("Setting values is not supported") - - class CheckReferencesBound(ExpressionVisitors.ExpressionVisitor): - - def __init__(self, message): - self.message = message - - def predicate(self, pred): - if isinstance(pred, UnboundPredicate): - raise AssertionError("Predicate should be a BoundPredicate") - - -class MockDataFile(DataFile): - - def __init__(self, path, partition, record_count, value_counts=None, null_value_counts=None, - lower_bounds=None, upper_bounds=None): - self._path = path - self._partition = partition - self._record_count = record_count - self._value_counts = value_counts - self._null_value_counts = null_value_counts - self._lower_bounds = lower_bounds - self._upper_bounds = upper_bounds - self._file_size_in_bytes = 0 - self._block_size_in_bytes = 0 - self._file_ordinal = None - self._column_sizes = None - - def path(self): - return self._path - - def partition(self): - return self._partition - - def record_count(self): - return self._record_count - - def value_counts(self): - return self._value_counts - - def null_value_counts(self): - return self._null_value_counts - - def lower_bounds(self): - return self._lower_bounds - - def upper_bounds(self): - return self._upper_bounds - - def file_size_in_bytes(self): - return self._file_size_in_bytes - - def file_ordinal(self): - return self._file_ordinal - - def column_sizes(self): - return self._column_sizes - - def copy(self): - return self - - -class TestManifestFile(ManifestFile): - - def __init__(self, path, length, spec_id, snapshot_id, added_files, existing_files, deleted_files, partitions): - self.path = path - self.length = length - self.spec_id = spec_id - self.snapshot_id = snapshot_id - self.added_files = added_files - self.existing_files = existing_files - self.deleted_files = deleted_files - self.partitions = partitions - - def copy(self): - return self - - -class TestFieldSummary(PartitionFieldSummary): - - def __init__(self, contains_null, lower_bound, upper_bound): - self._contains_null = contains_null - self._lower_bound = lower_bound - self._upper_bound = upper_bound - - def contains_null(self): - return self._contains_null - - def lower_bound(self): - return self._lower_bound - - def upper_bound(self): - return self._upper_bound - - def copy(self): - return self - - -@pytest.fixture(scope="session") -def schema(): - return Schema(NestedField.required(1, "id", IntegerType.get()), - NestedField.optional(2, "no_stats", IntegerType.get()), - NestedField.required(3, "required", StringType.get()), - NestedField.optional(4, "all_nulls", StringType.get()), - NestedField.optional(5, "some_nulls", StringType.get()), - NestedField.optional(6, "no_nulls", StringType.get())) - - -@pytest.fixture(scope="session") -def strict_schema(): - return Schema(NestedField.required(1, "id", IntegerType.get()), - NestedField.optional(2, "no_stats", IntegerType.get()), - NestedField.required(3, "required", StringType.get()), - NestedField.optional(4, "all_nulls", StringType.get()), - NestedField.optional(5, "some_nulls", StringType.get()), - NestedField.optional(6, "no_nulls", StringType.get()), - NestedField.required(7, "always_5", IntegerType.get())) - - -@pytest.fixture(scope="session") -def file(): - return MockDataFile("file.avro", TestHelpers.Row.of(), 50, - # value counts - {4: 50, 5: 50, 6: 50}, - # null value counts - {4: 50, 5: 10, 6: 0}, - # lower bounds - {1: Conversions.to_byte_buffer(IntegerType.get().type_id, 30)}, - # upper bounds - {1: Conversions.to_byte_buffer(IntegerType.get().type_id, 79)}) - - -@pytest.fixture(scope="session") -def strict_file(): - return MockDataFile("file.avro", - TestHelpers.Row.of(), - 50, - {4: 50, 5: 50, 6: 50}, - {4: 50, 5: 10, 6: 0}, - {1: Conversions.to_byte_buffer(IntegerType.get().type_id, 30), - 7: Conversions.to_byte_buffer(IntegerType.get().type_id, 5)}, - {1: Conversions.to_byte_buffer(IntegerType.get().type_id, 79), - 7: Conversions.to_byte_buffer(IntegerType.get().type_id, 5)} - ) - - -@pytest.fixture(scope="session") -def missing_stats(): - return MockDataFile("file.parquet", TestHelpers.Row.of(), 50) - - -@pytest.fixture(scope="session") -def empty(): - return MockDataFile("file.parquet", TestHelpers.Row.of(), record_count=0) - - -@pytest.fixture(scope="session") -def assert_and_unwrap(): - return lambda x, y=None: TestHelpers.assert_and_unwrap(x, y) - - -@pytest.fixture(scope="session") -def assert_all_bound(): - return lambda msg, expr: TestHelpers.assert_all_references_bound(msg, expr) - - -@pytest.fixture(scope="session") -def round_trip_serialize(): - return lambda x: TestHelpers.round_trip_serialize(x) - - -@pytest.fixture(scope="session") -def row_of(): - return lambda x: TestHelpers.Row.of(x) - - -@pytest.fixture(scope="session", - params=[Operation.LT, - Operation.LT_EQ, - Operation.GT, - Operation.GT_EQ, - Operation.EQ, - Operation.NOT_EQ]) -def op(request): - yield request.param - - -@pytest.fixture(scope="session", - params=[Expressions.always_false(), - Expressions.always_true(), - Expressions.less_than("x", 5), - Expressions.less_than_or_equal("y", -3), - Expressions.greater_than("z", 0), - Expressions.greater_than_or_equal("t", 129), - Expressions.equal("col", "data"), - Expressions.not_equal("col", "abc"), - Expressions.not_null("maybeNull"), - Expressions.is_null("maybeNull2"), - Expressions.not_(Expressions.greater_than("a", 10)), - Expressions.and_(Expressions.greater_than_or_equal("a", 0), - Expressions.less_than("a", 3)), - Expressions.or_(Expressions.less_than("a", 0), - Expressions.greater_than("a", 10)), - Expressions.equal("a", 5).bind(exp_schema.as_struct())]) -def expression(request): - yield request.param - - -@pytest.fixture(scope="session", - params=[Expressions.less_than("no_stats", 5), - Expressions.less_than_or_equal("no_stats", 30), - Expressions.equal("no_stats", 70), - Expressions.greater_than("no_stats", 78), - Expressions.greater_than_or_equal("no_stats", 90), - Expressions.not_equal("no_stats", 101), - Expressions.is_null("no_stats"), - Expressions.not_null("no_stats")]) -def missing_stats_exprs(request): - yield request.param - - -@pytest.fixture(scope="session", - params=[Expressions.less_than("id", 5), - Expressions.less_than_or_equal("id", 30), - Expressions.equal("id", 70), - Expressions.greater_than("id", 78), - Expressions.greater_than_or_equal("id", 90), - Expressions.not_equal("id", 101), - Expressions.is_null("some_nulls"), - Expressions.not_null("some_nulls")]) -def zero_rows_exprs(request): - yield request.param - - -@pytest.fixture(scope="session", - params=[Expressions.not_equal("id", 5), - Expressions.not_equal("id", 29), - Expressions.not_equal("id", 30), - Expressions.not_equal("id", 75), - Expressions.not_equal("id", 79), - Expressions.not_equal("id", 80), - Expressions.not_equal("id", 85)]) -def not_eq(request): - yield request.param - - -@pytest.fixture(scope="session", - params=[Expressions.equal("id", 5), - Expressions.equal("id", 29), - Expressions.equal("id", 30), - Expressions.equal("id", 75), - Expressions.equal("id", 79), - Expressions.equal("id", 80), - Expressions.equal("id", 85)]) -def not_eq_rewrite(request): - yield request.param - - -@pytest.fixture(scope="session", - params=[Expressions.equal("ID", 5), - Expressions.equal("ID", 29), - Expressions.equal("ID", 30), - Expressions.equal("ID", 75), - Expressions.equal("ID", 79), - Expressions.equal("ID", 80), - Expressions.equal("ID", 85)]) -def not_eq_uc(request): - yield request.param - - -@pytest.fixture(scope="session", - params=[Literal.of(False), - Literal.of(34), - Literal.of(35), - Literal.of(36.75), - Literal.of(8.75), - Literal.of("2017-11-29").to(DateType.get()), - Literal.of("11:30:0").to(TimeType.get()), - Literal.of("2017-11-29T11:30:07.123").to(TimestampType.without_timezone()), - Literal.of("2017-11-29T11:30:07.123+01:00").to(TimestampType.with_timezone()), - Literal.of("abc"), - Literal.of(uuid.uuid4()), - Literal.of(bytes([0x01, 0x02, 0x03])).to(FixedType.of_length(3)), - Literal.of(bytes([0x03, 0x04, 0x05, 0x06])).to(BinaryType.get()), - Literal.of(Decimal(122.50).quantize(Decimal(".01")))]) -def literal(request): - yield request.param - - -@pytest.fixture(scope="session", - params=[(DecimalType.of(9, 0), "34"), - (DecimalType.of(9, 2), "34.00"), - (DecimalType.of(9, 4), "34.0000")]) -def type_val_tuples(request): - yield request.param - - -@pytest.fixture(scope="session", - params=[(DecimalType.of(9, 1), "34.6"), - (DecimalType.of(9, 2), "34.56"), - (DecimalType.of(9, 4), "34.5600")]) -def float_type_val_tuples(request): - yield request.param - - -@pytest.fixture(scope="session") -def inc_man_spec(): - inc_schema = Schema(NestedField.required(1, "id", IntegerType.get()), - NestedField.optional(4, "all_nulls", StringType.get()), - NestedField.optional(5, "some_nulls", StringType.get()), - NestedField.optional(6, "no_nulls", StringType.get())) - return (PartitionSpec.builder_for(inc_schema) - .with_spec_id(0) - .identity("id") - .identity("all_nulls") - .identity("some_nulls") - .identity("no_nulls") - .build() - ) - - -@pytest.fixture(scope="session") -def inc_man_file(): - return TestManifestFile("manifest-list.avro", 1024, 0, int(time.time() * 1000), 5, 10, 0, - (TestFieldSummary(False, - Conversions.to_byte_buffer(IntegerType.get().type_id, 30), - Conversions.to_byte_buffer(IntegerType.get().type_id, 79)), - TestFieldSummary(True, - None, - None), - TestFieldSummary(True, - Conversions.to_byte_buffer(StringType.get().type_id, 'a'), - Conversions.to_byte_buffer(StringType.get().type_id, 'z')), - TestFieldSummary(False, - Conversions.to_byte_buffer(StringType.get().type_id, 'a'), - Conversions.to_byte_buffer(StringType.get().type_id, 'z')) - )) - - -@pytest.fixture(scope="session") -def inc_man_file_ns(): - return TestManifestFile("manifest-list.avro", 1024, 0, int(time.time() * 1000), None, None, None, None) diff --git a/tests/api/expressions/test_evaluator.py b/tests/api/expressions/test_evaluator.py deleted file mode 100644 index da7b454d5b..0000000000 --- a/tests/api/expressions/test_evaluator.py +++ /dev/null @@ -1,161 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - - -import iceberg.api.expressions as exp -from iceberg.api.types import (FloatType, - IntegerType, - NestedField, - StringType, - StructType) -from iceberg.exceptions import ValidationException -from pytest import raises - -STRUCT = StructType.of([NestedField.required(13, "x", IntegerType.get()), - NestedField.required(14, "y", IntegerType.get()), - NestedField.optional(15, "z", IntegerType.get())]) - - -def test_less_than(row_of): - evaluator = exp.evaluator.Evaluator(STRUCT, - exp.expressions.Expressions.less_than("x", 7)) - assert not evaluator.eval(row_of((7, 8, None))) - assert evaluator.eval(row_of((6, 8, None))) - - -def test_less_than_or_equal(row_of): - evaluator = exp.evaluator.Evaluator(STRUCT, - exp.expressions.Expressions.less_than_or_equal("x", 7)) - assert evaluator.eval(row_of((7, 8, None))) - assert evaluator.eval(row_of((6, 8, None))) - assert not evaluator.eval(row_of((8, 8, None))) - - -def test_greater_than(row_of): - evaluator = exp.evaluator.Evaluator(STRUCT, - exp.expressions.Expressions.greater_than("x", 7)) - assert not evaluator.eval(row_of((7, 8, None))) - assert not evaluator.eval(row_of((6, 8, None))) - assert evaluator.eval(row_of((8, 8, None))) - - -def test_greater_than_or_equal(row_of): - evaluator = exp.evaluator.Evaluator(STRUCT, - exp.expressions.Expressions.greater_than_or_equal("x", 7)) - assert evaluator.eval(row_of((7, 8, None))) - assert not evaluator.eval(row_of((6, 8, None))) - assert evaluator.eval(row_of((8, 8, None))) - - -def test_equal(row_of): - evaluator = exp.evaluator.Evaluator(STRUCT, - exp.expressions.Expressions.equal("x", 7)) - assert evaluator.eval(row_of((7, 8, None))) - - -def test_not_equal(row_of): - evaluator = exp.evaluator.Evaluator(STRUCT, - exp.expressions.Expressions.not_equal("x", 7)) - assert not evaluator.eval(row_of((7, 8, None))) - assert evaluator.eval(row_of((6, 8, None))) - - -def test_always_true(row_of): - evaluator = exp.evaluator.Evaluator(STRUCT, - exp.expressions.Expressions.always_true()) - assert evaluator.eval(row_of(())) - - -def test_always_false(row_of): - evaluator = exp.evaluator.Evaluator(STRUCT, - exp.expressions.Expressions.always_false()) - assert not evaluator.eval(row_of(())) - - -def test_is_null(row_of): - evaluator = exp.evaluator.Evaluator(STRUCT, - exp.expressions.Expressions.is_null("z")) - assert evaluator.eval(row_of((1, 2, None))) - assert not evaluator.eval(row_of((1, 2, 3))) - - -def test_is_not_null(row_of): - evaluator = exp.evaluator.Evaluator(STRUCT, - exp.expressions.Expressions.not_null("z")) - assert not evaluator.eval(row_of((1, 2, None))) - assert evaluator.eval(row_of((1, 2, 3))) - - -def test_and(row_of): - evaluator = exp.evaluator.Evaluator(STRUCT, - exp.expressions.Expressions.and_(exp.expressions.Expressions.equal("x", 7), - exp.expressions.Expressions.not_null("z"))) - assert evaluator.eval(row_of((7, 0, 3))) - assert not evaluator.eval(row_of((8, 0, 3))) - assert not evaluator.eval(row_of((7, 0, None))) - assert not evaluator.eval(row_of((8, 0, None))) - - -def test_or(row_of): - evaluator = exp.evaluator.Evaluator(STRUCT, - exp.expressions.Expressions.or_(exp.expressions.Expressions.equal("x", 7), - exp.expressions.Expressions.not_null("z"))) - assert evaluator.eval(row_of((7, 0, 3))) - assert evaluator.eval(row_of((8, 0, 3))) - assert evaluator.eval(row_of((7, 0, None))) - assert not evaluator.eval(row_of((8, 0, None))) - - -def test_not(row_of): - evaluator = exp.evaluator.Evaluator(STRUCT, - exp.expressions.Expressions.not_(exp.expressions.Expressions.equal("x", 7))) - assert not evaluator.eval(row_of((7,))) - assert evaluator.eval(row_of((8,))) - - -def test_case_insensitive_not(row_of): - evaluator = exp.evaluator.Evaluator(STRUCT, - exp.expressions.Expressions.not_(exp.expressions.Expressions.equal("X", 7)), - case_sensitive=False) - assert not evaluator.eval(row_of((7,))) - assert evaluator.eval(row_of((8,))) - - -def test_case_sensitive_not(): - with raises(ValidationException): - exp.evaluator.Evaluator(STRUCT, - exp.expressions.Expressions.not_(exp.expressions.Expressions.equal("X", 7)), - case_sensitive=True) - - -def test_char_seq_value(row_of): - struct = StructType.of([NestedField.required(34, "s", StringType.get())]) - evaluator = exp.evaluator.Evaluator(struct, exp.expressions.Expressions.equal("s", "abc")) - assert evaluator.eval(row_of(("abc",))) - assert not evaluator.eval(row_of(("abcd",))) - - -def test_nan_errors(row_of): - # Placeholder until NaN support is fully implemented - struct = StructType.of([NestedField.required(34, "f", FloatType.get())]) - evaluator = exp.evaluator.Evaluator(struct, exp.expressions.Expressions.is_nan("f")) - with raises(NotImplementedError): - evaluator.eval(row_of((123.4,))) - - evaluator = exp.evaluator.Evaluator(struct, exp.expressions.Expressions.not_nan("f")) - with raises(NotImplementedError): - evaluator.eval(row_of((123.4,))) diff --git a/tests/api/expressions/test_expression_binding.py b/tests/api/expressions/test_expression_binding.py deleted file mode 100644 index 556a54ae30..0000000000 --- a/tests/api/expressions/test_expression_binding.py +++ /dev/null @@ -1,143 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -from iceberg.api.expressions import (And, - Binder, - Expressions, - Not, - Or) -from iceberg.api.types import (IntegerType, - NestedField, - StructType) -import iceberg.exceptions as ice_ex -from pytest import raises - -STRUCT = StructType.of([NestedField.required(0, "x", IntegerType.get()), - NestedField.required(1, "y", IntegerType.get()), - NestedField.required(2, "z", IntegerType.get())]) - - -def test_missing_reference(): - expr = Expressions.and_(Expressions.equal("t", 5), - Expressions.equal("x", 7)) - try: - Binder.bind(STRUCT, expr) - except ice_ex.ValidationException as e: - assert "Cannot find field 't' in struct" in "{}".format(e) - - -def test_bound_expression_fails(): - with raises(RuntimeError): - expr = Expressions.not_(Expressions.equal("x", 7)) - Binder.bind(STRUCT, - Binder.bind(STRUCT, expr)) - - -def test_single_reference(assert_all_bound): - expr = Expressions.not_(Expressions.equal("x", 7)) - assert_all_bound("Single reference", - Binder.bind(STRUCT, expr)) - - -def test_case_insensitive_reference(assert_all_bound): - expr = Expressions.not_(Expressions.equal("X", 7)) - assert_all_bound("Single reference", - Binder.bind(STRUCT, expr, case_sensitive=False)) - - -def test_case_sensitive_reference(): - with raises(ice_ex.ValidationException): - expr = Expressions.not_(Expressions.equal("X", 7)) - Binder.bind(STRUCT, expr, case_sensitive=True) - - -def test_multiple_references(assert_all_bound): - expr = Expressions.or_(Expressions.and_(Expressions.equal("x", 7), - Expressions.less_than("y", 100)), - Expressions.greater_than("z", -100)) - - assert_all_bound("Multiple references", Binder.bind(STRUCT, expr)) - - -def test_and(assert_all_bound, assert_and_unwrap): - expr = Expressions.and_(Expressions.equal("x", 7), - Expressions.less_than("y", 100)) - bound_expr = Binder.bind(STRUCT, expr) - assert_all_bound("And", bound_expr) - - and_ = assert_and_unwrap(bound_expr, And) - - left = assert_and_unwrap(and_.left, None) - # should bind x correctly - assert 0 == left.ref.field.field_id - right = assert_and_unwrap(and_.right, None) - # should bind y correctly - assert 1 == right.ref.field.field_id - - -def test_or(assert_all_bound, assert_and_unwrap): - expr = Expressions.or_(Expressions.greater_than("z", -100), - Expressions.less_than("y", 100)) - bound_expr = Binder.bind(STRUCT, expr) - assert_all_bound("Or", bound_expr) - - or_ = assert_and_unwrap(bound_expr, Or) - - left = assert_and_unwrap(or_.left, None) - # should bind z correctly - assert 2 == left.ref.field.field_id - right = assert_and_unwrap(or_.right, None) - # should bind y correctly - assert 1 == right.ref.field.field_id - - -def test_not(assert_all_bound, assert_and_unwrap): - expr = Expressions.not_(Expressions.equal("x", 7)) - bound_expr = Binder.bind(STRUCT, expr) - assert_all_bound("Not", bound_expr) - - not_ = assert_and_unwrap(bound_expr, Not) - - child = assert_and_unwrap(not_.child, None) - # should bind x correctly - assert 0 == child.ref.field.field_id - - -def test_always_true(): - assert Expressions.always_true() == Binder.bind(STRUCT, - Expressions.always_true()) - - -def test_always_false(): - assert Expressions.always_false() == Binder.bind(STRUCT, - Expressions.always_false()) - - -def test_basic_simplification(assert_and_unwrap): - # Should simplify or expression to alwaysTrue - assert Expressions.always_true() == Binder.bind(STRUCT, - Expressions.or_(Expressions.less_than("y", 100), - Expressions.greater_than("z", -9999999999))) - # Should simplify or expression to alwaysfalse - assert Expressions.always_false() == Binder.bind(STRUCT, - Expressions.and_(Expressions.less_than("y", 100), - Expressions.less_than("z", -9999999999))) - - bound = Binder.bind(STRUCT, - Expressions.not_(Expressions.not_(Expressions.less_than("y", 100)))) - pred = assert_and_unwrap(bound, None) - assert 1 == pred.ref.field.field_id diff --git a/tests/api/expressions/test_expression_helpers.py b/tests/api/expressions/test_expression_helpers.py deleted file mode 100644 index 02f0b7db39..0000000000 --- a/tests/api/expressions/test_expression_helpers.py +++ /dev/null @@ -1,47 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -from iceberg.api.expressions.expressions import Expressions -from pytest import raises - -pred = Expressions.less_than("x", 7) - - -def test_simplify_or(): - assert Expressions.always_true() == Expressions.or_(Expressions.always_true(), pred) - assert Expressions.always_true() == Expressions.or_(pred, Expressions.always_true()) - assert pred == Expressions.or_(Expressions.always_false(), pred) - assert pred == Expressions.or_(pred, Expressions.always_false()) - - -def test_simplify_and(): - assert pred == Expressions.and_(Expressions.always_true(), pred) - assert pred == Expressions.and_(pred, Expressions.always_true()) - - assert Expressions.always_false() == Expressions.and_(Expressions.always_false(), pred) - assert Expressions.always_false() == Expressions.and_(pred, Expressions.always_false()) - - -def test_simplify_not(): - assert Expressions.always_false() == Expressions.not_(Expressions.always_true()) - assert Expressions.always_true() == Expressions.not_(Expressions.always_false()) - assert pred == Expressions.not_(Expressions.not_(pred)) - - -def test_null_name(): - with raises(RuntimeError): - Expressions.equal(None, 5) diff --git a/tests/api/expressions/test_expression_serializations.py b/tests/api/expressions/test_expression_serializations.py deleted file mode 100644 index 1ada7e9447..0000000000 --- a/tests/api/expressions/test_expression_serializations.py +++ /dev/null @@ -1,21 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - - -def test_expressions(expression, round_trip_serialize): - copy = round_trip_serialize(expression) - assert copy == expression diff --git a/tests/api/expressions/test_inclusive_manifest_evaluator.py b/tests/api/expressions/test_inclusive_manifest_evaluator.py deleted file mode 100644 index 69e46355fe..0000000000 --- a/tests/api/expressions/test_inclusive_manifest_evaluator.py +++ /dev/null @@ -1,182 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -from iceberg.api.expressions import Expressions, InclusiveManifestEvaluator -from iceberg.exceptions import ValidationException -import pytest - - -@pytest.mark.parametrize("expression,expected", [ - (Expressions.not_null("all_nulls"), False), - (Expressions.not_null("some_nulls"), True), - (Expressions.not_null("no_nulls"), True)]) -def test_all_nulls(inc_man_spec, inc_man_file, expression, expected): - assert InclusiveManifestEvaluator(inc_man_spec, expression).eval(inc_man_file) == expected - - -@pytest.mark.parametrize("expression,expected", [ - (Expressions.is_null("all_nulls"), True), - (Expressions.is_null("some_nulls"), True), - (Expressions.is_null("no_nulls"), False)]) -def test_no_nulls(inc_man_spec, inc_man_file, expression, expected): - assert InclusiveManifestEvaluator(inc_man_spec, expression).eval(inc_man_file) == expected - - -def test_missing_column(inc_man_spec, inc_man_file): - with pytest.raises(ValidationException): - InclusiveManifestEvaluator(inc_man_spec, Expressions.less_than("missing", 5)).eval(inc_man_file) - - -@pytest.mark.parametrize("expression", [ - Expressions.less_than("id", 5), - Expressions.less_than_or_equal("id", 30), - Expressions.equal("id", 70), - Expressions.greater_than("id", 78), - Expressions.greater_than_or_equal("id", 90), - Expressions.not_equal("id", 101), - Expressions.less_than_or_equal("id", 30), - Expressions.is_null("id"), - Expressions.not_null("id")]) -def test_missing_stats(inc_man_spec, inc_man_file_ns, expression): - assert InclusiveManifestEvaluator(inc_man_spec, expression).eval(inc_man_file_ns) - - -@pytest.mark.parametrize("expression, expected", [ - (Expressions.less_than("id", 5), True), - (Expressions.greater_than("id", 5), False)]) -def test_not(inc_man_spec, inc_man_file, expression, expected): - assert InclusiveManifestEvaluator(inc_man_spec, Expressions.not_(expression)).eval(inc_man_file) == expected - - -@pytest.mark.parametrize("expr1, expr2, expected", [ - (Expressions.less_than("id", 5), Expressions.greater_than_or_equal("id", 0), False), - (Expressions.greater_than("id", 5), Expressions.less_than_or_equal("id", 30), True)]) -def test_and(inc_man_spec, inc_man_file, expr1, expr2, expected): - assert InclusiveManifestEvaluator(inc_man_spec, Expressions.and_(expr1, expr2)).eval(inc_man_file) == expected - - -@pytest.mark.parametrize("expr1, expr2, expected", [ - (Expressions.less_than("id", 5), Expressions.greater_than_or_equal("id", 80), False), - (Expressions.less_than("id", 5), Expressions.greater_than_or_equal("id", 60), True)]) -def test_or(inc_man_spec, inc_man_file, expr1, expr2, expected): - assert InclusiveManifestEvaluator(inc_man_spec, Expressions.or_(expr1, expr2)).eval(inc_man_file) == expected - - -@pytest.mark.parametrize("val, expected", [ - (5, False), - (30, False), - (31, True), - (79, True)]) -def test_int_lt(inc_man_spec, inc_man_file, val, expected): - assert InclusiveManifestEvaluator(inc_man_spec, Expressions.less_than("id", val)).eval(inc_man_file) == expected - - -@pytest.mark.parametrize("val, expected", [ - (5, False), - (29, False), - (30, True), - (79, True)]) -def test_int_lt_eq(inc_man_spec, inc_man_file, val, expected): - assert InclusiveManifestEvaluator(inc_man_spec, - Expressions.less_than_or_equal("id", val)).eval(inc_man_file) == expected - - -@pytest.mark.parametrize("val, expected", [ - (85, False), - (79, False), - (78, True), - (75, True)]) -def test_int_gt(inc_man_spec, inc_man_file, val, expected): - assert InclusiveManifestEvaluator(inc_man_spec, Expressions.greater_than("id", val)).eval(inc_man_file) == expected - - -@pytest.mark.parametrize("val, expected", [ - (85, False), - (80, False), - (79, True), - (75, True)]) -def test_int_gt_eq(inc_man_spec, inc_man_file, val, expected): - assert InclusiveManifestEvaluator(inc_man_spec, - Expressions.greater_than_or_equal("id", val)).eval(inc_man_file) == expected - - -@pytest.mark.parametrize("val, expected", [ - (5, False), - (29, False), - (30, True), - (75, True), - (79, True), - (80, False), - (85, False)]) -def test_int_eq(inc_man_spec, inc_man_file, val, expected): - assert InclusiveManifestEvaluator(inc_man_spec, - Expressions.equal("id", val)).eval(inc_man_file) == expected - - -@pytest.mark.parametrize("val, expected", [ - (5, True), - (29, True), - (30, True), - (75, True), - (79, True), - (80, True), - (85, True)]) -def test_int_not_eq(inc_man_spec, inc_man_file, val, expected): - assert InclusiveManifestEvaluator(inc_man_spec, - Expressions.not_equal("id", val)).eval(inc_man_file) == expected - - -@pytest.mark.parametrize("val, expected", [ - (5, True), - (29, True), - (30, True), - (75, True), - (79, True), - (80, True), - (85, True)]) -def test_int_not_eq_rewritten(inc_man_spec, inc_man_file, val, expected): - assert InclusiveManifestEvaluator(inc_man_spec, - Expressions.not_(Expressions.equal("id", val))).eval(inc_man_file) == expected - - -@pytest.mark.parametrize("val, expected", [ - (5, True), - (29, True), - (30, True), - (75, True), - (79, True), - (80, True), - (85, True)]) -def test_case_insensitive_int_not_eq_rewritten(inc_man_spec, inc_man_file, val, expected): - assert InclusiveManifestEvaluator(inc_man_spec, - Expressions.not_(Expressions.equal("ID", val)), - case_sensitive=False).eval(inc_man_file) == expected - - -@pytest.mark.parametrize("val, expected", [ - (5, True), - (29, True), - (30, True), - (75, True), - (79, True), - (80, True), - (85, True)]) -def test_case_sensitive_int_not_eq_rewritten(inc_man_spec, inc_man_file, val, expected): - with pytest.raises(ValidationException): - assert InclusiveManifestEvaluator(inc_man_spec, - Expressions.not_(Expressions.equal("ID", val)), - case_sensitive=True).eval(inc_man_file) == expected diff --git a/tests/api/expressions/test_inclusive_metrics_evaluator.py b/tests/api/expressions/test_inclusive_metrics_evaluator.py deleted file mode 100644 index e9f0473d1a..0000000000 --- a/tests/api/expressions/test_inclusive_metrics_evaluator.py +++ /dev/null @@ -1,130 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -from iceberg.api.expressions import (Expressions, - InclusiveMetricsEvaluator) -from iceberg.exceptions import ValidationException -from pytest import raises - - -def test_all_nulls(schema, file): - # Should skip: no non-null value in all null column - assert not InclusiveMetricsEvaluator(schema, Expressions.not_null("all_nulls")).eval(file) - # Should read: column with some nulls contains a non-null value - assert InclusiveMetricsEvaluator(schema, Expressions.not_null("some_nulls")).eval(file) - # Should read: non-null column contains a non-null value - assert InclusiveMetricsEvaluator(schema, Expressions.not_null("no_nulls")).eval(file) - - -def test_no_nulls(schema, file): - # Should read: at least one null value in all null column - assert InclusiveMetricsEvaluator(schema, Expressions.is_null("all_nulls")).eval(file) - # Should read: column with some nulls contains a null value - assert InclusiveMetricsEvaluator(schema, Expressions.is_null("some_nulls")).eval(file) - # Should skip: non-null column contains no null values - assert not InclusiveMetricsEvaluator(schema, Expressions.is_null("no_nulls")).eval(file) - - -def test_required_column(schema, file): - assert InclusiveMetricsEvaluator(schema, Expressions.not_null("required")).eval(file) - assert not InclusiveMetricsEvaluator(schema, Expressions.is_null("required")).eval(file) - - -def test_missing_column(schema, file): - with raises(RuntimeError): - InclusiveMetricsEvaluator(schema, Expressions.less_than("missing", 5)).eval(file) - - -def test_missing_stats(schema, missing_stats, missing_stats_exprs): - assert InclusiveMetricsEvaluator(schema, missing_stats_exprs).eval(missing_stats) - - -def test_zero_record_file(schema, empty, zero_rows_exprs): - assert not InclusiveMetricsEvaluator(schema, zero_rows_exprs).eval(empty) - - -def test_not(schema, file): - assert InclusiveMetricsEvaluator(schema, Expressions.not_(Expressions.less_than("id", 5))).eval(file) - assert not InclusiveMetricsEvaluator(schema, - Expressions.not_(Expressions.greater_than("id", 5))).eval(file) - - -def test_and(schema, file): - assert not InclusiveMetricsEvaluator(schema, - Expressions.and_(Expressions.less_than("id", 5), - Expressions.greater_than_or_equal("id", 0))).eval(file) - assert InclusiveMetricsEvaluator(schema, Expressions.and_(Expressions.greater_than("id", 5), - Expressions.less_than_or_equal("id", 30))).eval(file) - - -def test_or(schema, file): - assert not InclusiveMetricsEvaluator(schema, - Expressions.or_(Expressions.less_than("id", 5), - Expressions.greater_than_or_equal("id", 80))).eval(file) - assert InclusiveMetricsEvaluator(schema, - Expressions.or_(Expressions.less_than("id", 5), - Expressions.greater_than_or_equal("id", 60))).eval(file) - - -def test_integer_lt(schema, file): - assert not InclusiveMetricsEvaluator(schema, Expressions.less_than("id", 5)).eval(file) - assert not InclusiveMetricsEvaluator(schema, Expressions.less_than("id", 30)).eval(file) - assert InclusiveMetricsEvaluator(schema, Expressions.less_than("id", 31)).eval(file) - assert InclusiveMetricsEvaluator(schema, Expressions.less_than("id", 79)).eval(file) - - -def test_integer_gt(schema, file): - assert not InclusiveMetricsEvaluator(schema, Expressions.greater_than("id", 85)).eval(file) - assert not InclusiveMetricsEvaluator(schema, Expressions.greater_than("id", 79)).eval(file) - assert InclusiveMetricsEvaluator(schema, Expressions.greater_than("id", 78)).eval(file) - assert InclusiveMetricsEvaluator(schema, Expressions.greater_than("id", 75)).eval(file) - - -def test_integer_gt_eq(schema, file): - assert not InclusiveMetricsEvaluator(schema, Expressions.greater_than_or_equal("id", 85)).eval(file) - assert not InclusiveMetricsEvaluator(schema, Expressions.greater_than_or_equal("id", 80)).eval(file) - assert InclusiveMetricsEvaluator(schema, Expressions.greater_than_or_equal("id", 79)).eval(file) - assert InclusiveMetricsEvaluator(schema, Expressions.greater_than_or_equal("id", 75)).eval(file) - - -def test_integer_eq(schema, file): - assert not InclusiveMetricsEvaluator(schema, Expressions.equal("id", 5)).eval(file) - assert not InclusiveMetricsEvaluator(schema, Expressions.equal("id", 29)).eval(file) - assert not InclusiveMetricsEvaluator(schema, Expressions.equal("id", 80)).eval(file) - assert not InclusiveMetricsEvaluator(schema, Expressions.equal("id", 85)).eval(file) - assert InclusiveMetricsEvaluator(schema, Expressions.equal("id", 30)).eval(file) - assert InclusiveMetricsEvaluator(schema, Expressions.equal("id", 75)).eval(file) - assert InclusiveMetricsEvaluator(schema, Expressions.equal("id", 79)).eval(file) - - -def test_integer_not_eq(schema, file, not_eq): - assert InclusiveMetricsEvaluator(schema, not_eq).eval(file) - - -def test_not_eq_rewritten(schema, file, not_eq_rewrite): - assert InclusiveMetricsEvaluator(schema, Expressions.not_(not_eq_rewrite)).eval(file) - - -def test_case_insensitive_int_not_eq_rewritten(schema, file, not_eq_uc): - assert InclusiveMetricsEvaluator(schema, Expressions.not_(not_eq_uc), - case_sensitive=False).eval(file) - - -def test_case_sensitive_int_not_eq_rewritten(schema, file, not_eq_uc): - with raises(ValidationException): - assert InclusiveMetricsEvaluator(schema, Expressions.not_(not_eq_uc), - case_sensitive=True).eval(file) diff --git a/tests/api/expressions/test_literal_serialization.py b/tests/api/expressions/test_literal_serialization.py deleted file mode 100644 index 3e0a2f8aa3..0000000000 --- a/tests/api/expressions/test_literal_serialization.py +++ /dev/null @@ -1,20 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - - -def test_literals(round_trip_serialize, literal): - assert round_trip_serialize(literal) == literal diff --git a/tests/api/expressions/test_misc_literal_conversions.py b/tests/api/expressions/test_misc_literal_conversions.py deleted file mode 100644 index d87ca8a8d3..0000000000 --- a/tests/api/expressions/test_misc_literal_conversions.py +++ /dev/null @@ -1,263 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -from decimal import Decimal -import sys -import uuid - -from iceberg.api.expressions import Literal -from iceberg.api.types import (BinaryType, - BooleanType, - DateType, - DecimalType, - DoubleType, - FixedType, - FloatType, - IntegerType, - LongType, - StringType, - TimestampType, - TimeType, - UUIDType) -import pytest - - -def test_identity_conversions(): - pairs = [(Literal.of(True), BooleanType.get()), - (Literal.of(34), IntegerType.get()), - (Literal.of(3400000000), LongType.get()), - (Literal.of(34.11), FloatType.get()), - (Literal.of(34.11), DoubleType.get()), - (Literal.of(Decimal(34.55).quantize(Decimal("0.01"))), DecimalType.of(9, 2)), - (Literal.of("2017-08-18"), DateType.get()), - (Literal.of("14:21:01.919"), TimeType.get()), - (Literal.of("2017-08-18T14:21:01.919"), TimestampType.without_timezone()), - (Literal.of("abc"), StringType.get()), - (Literal.of(uuid.uuid4()), UUIDType.get()) - ] - - if sys.version_info[0] >= 3: - pairs = pairs + [(Literal.of(bytes([0x01, 0x02, 0x03])), FixedType.of_length(3)), - (Literal.of(bytearray([0x03, 0x04, 0x05, 0x06])), BinaryType.get())] - - for pair in pairs: - expected = pair[0].to(pair[1]) - assert expected is expected.to(pair[1]) - - -def test_binary_to_fixed(): - if sys.version_info[0] >= 3: - lit = Literal.of(bytearray([0x00, 0x01, 0x02])) - fixed_lit = lit.to(FixedType.of_length(3)) - assert fixed_lit is not None - assert lit.value == fixed_lit.value - assert lit.to(FixedType.of_length(4)) is None - assert lit.to(FixedType.of_length(2)) is None - - -def test_fixed_to_binary(): - if sys.version_info[0] >= 3: - lit = Literal.of(bytes([0x00, 0x01, 0x02])) - binary_lit = lit.to(BinaryType.get()) - assert binary_lit is not None - assert lit.value == binary_lit.value - - -def test_invalid_boolean_conversions(): - assert_invalid_conversions(Literal.of(True), [IntegerType.get(), - LongType.get(), - FloatType.get(), - DoubleType.get(), - DateType.get(), - TimeType.get(), - TimestampType.with_timezone(), - TimestampType.without_timezone(), - DecimalType.of(9, 2), - StringType.get(), - UUIDType.get(), - FixedType.of_length(1), - BinaryType.get()]) - - -def test_invalid_integer_conversions(): - assert_invalid_conversions(Literal.of(34), [BooleanType.get(), - TimeType.get(), - TimestampType.with_timezone(), - TimestampType.without_timezone(), - StringType.get(), - UUIDType.get(), - FixedType.of_length(1), - BinaryType.get()]) - - -def test_invalid_long_conversions(): - assert_invalid_conversions(Literal.of(34).to(LongType.get()), [BooleanType.get(), - DateType.get(), - StringType.get(), - UUIDType.get(), - FixedType.of_length(1), - BinaryType.get()]) - - -@pytest.mark.parametrize("lit", [Literal.of(34.11), - # double - Literal.of(34.11).to(DoubleType.get())]) -@pytest.mark.parametrize("test_type", [BooleanType.get(), - IntegerType.get(), - LongType.get(), - DateType.get(), - TimeType.get(), - TimestampType.with_timezone(), - TimestampType.without_timezone(), - StringType.get(), - UUIDType.get(), - FixedType.of_length(1), - BinaryType.get()]) -def test_invalid_float_conversions(lit, test_type): - assert lit.to(test_type) is None - - -@pytest.mark.parametrize("lit", [Literal.of("2017-08-18").to(DateType.get())]) -@pytest.mark.parametrize("test_type", [BooleanType.get(), - IntegerType.get(), - LongType.get(), - FloatType.get(), - DoubleType.get(), - TimeType.get(), - TimestampType.with_timezone(), - TimestampType.without_timezone(), - DecimalType.of(9, 2), - StringType.get(), - UUIDType.get(), - FixedType.of_length(1), - BinaryType.get()]) -def test_invalid_datetime_conversions(lit, test_type): - assert_invalid_conversions(lit, (test_type,)) - - -def test_invalid_time_conversions(): - assert_invalid_conversions(Literal.of("14:21:01.919") - .to(TimeType.get()), [BooleanType.get(), - IntegerType.get(), - LongType.get(), - FloatType.get(), - DoubleType.get(), - DateType.get(), - TimestampType.with_timezone(), - TimestampType.without_timezone(), - DecimalType.of(9, 2), - StringType.get(), - UUIDType.get(), - FixedType.of_length(1), - BinaryType.get()]) - - -def test_invalid_timestamp_conversions(): - assert_invalid_conversions(Literal.of("2017-08-18T14:21:01.919") - .to(TimestampType.without_timezone()), [BooleanType.get(), - IntegerType.get(), - LongType.get(), - FloatType.get(), - DoubleType.get(), - TimeType.get(), - DecimalType.of(9, 2), - StringType.get(), - UUIDType.get(), - FixedType.of_length(1), - BinaryType.get()]) - - -def test_invalid_decimal_conversions(): - assert_invalid_conversions(Literal.of(Decimal("34.11")), [BooleanType.get(), - IntegerType.get(), - LongType.get(), - FloatType.get(), - DoubleType.get(), - DateType.get(), - TimeType.get(), - TimestampType.with_timezone(), - TimestampType.without_timezone(), - DecimalType.of(9, 4), - StringType.get(), - UUIDType.get(), - FixedType.of_length(1), - BinaryType.get()]) - - -def test_invalid_string_conversions(): - assert_invalid_conversions(Literal.of("abc"), [BooleanType.get(), - IntegerType.get(), - LongType.get(), - FloatType.get(), - DoubleType.get(), - FixedType.of_length(1), - BinaryType.get()]) - - -def test_invalid_uuid_conversions(): - assert_invalid_conversions(Literal.of(uuid.uuid4()), [BooleanType.get(), - IntegerType.get(), - LongType.get(), - FloatType.get(), - DoubleType.get(), - DateType.get(), - TimeType.get(), - TimestampType.with_timezone(), - TimestampType.without_timezone(), - DecimalType.of(9, 2), - StringType.get(), - FixedType.of_length(1), - BinaryType.get()]) - - -def test_invalid_fixed_conversions(): - if sys.version_info[0] >= 3: - assert_invalid_conversions(Literal.of(bytes([0x00, 0x01, 0x02])), [BooleanType.get(), - IntegerType.get(), - LongType.get(), - FloatType.get(), - DoubleType.get(), - DateType.get(), - TimeType.get(), - TimestampType.with_timezone(), - TimestampType.without_timezone(), - DecimalType.of(9, 2), - StringType.get(), - UUIDType.get(), - FixedType.of_length(1)]) - - -def test_invalid_binary_conversions(): - if sys.version_info[0] >= 3: - assert_invalid_conversions(Literal.of(bytearray([0x00, 0x01, 0x02])), [BooleanType.get(), - IntegerType.get(), - LongType.get(), - FloatType.get(), - DoubleType.get(), - DateType.get(), - TimeType.get(), - TimestampType.with_timezone(), - TimestampType.without_timezone(), - DecimalType.of(9, 2), - StringType.get(), - UUIDType.get(), - FixedType.of_length(1)]) - - -def assert_invalid_conversions(lit, types=None): - for type_var in types: - assert lit.to(type_var) is None diff --git a/tests/api/expressions/test_numeric_literal_conversions.py b/tests/api/expressions/test_numeric_literal_conversions.py deleted file mode 100644 index 7c1dd2a9fd..0000000000 --- a/tests/api/expressions/test_numeric_literal_conversions.py +++ /dev/null @@ -1,116 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -from decimal import Decimal -import math - -from iceberg.api.expressions import Literal -from iceberg.api.types import (DecimalType, - DoubleType, - FloatType, - IntegerType, - LongType) - - -def test_integer_to_long_conversion(): - lit = Literal.of(34) - long_lit = lit.to(LongType.get()) - - assert lit.value == long_lit.value - - -def test_integer_to_float_conversion(): - lit = Literal.of(34) - float_lit = lit.to(FloatType.get()) - - assert math.isclose(lit.value, float_lit.value) - - -def test_integer_to_double_conversion(): - lit = Literal.of(34) - dbl_lit = lit.to(DoubleType.get()) - - assert math.isclose(lit.value, dbl_lit.value) - - -def test_integer_to_decimal_conversion(type_val_tuples): - lit = Literal.of(34) - - assert lit.to(type_val_tuples[0]).value.as_tuple() == Decimal(type_val_tuples[1]).as_tuple() - - -def test_long_to_integer(): - lit = Literal.of(34).to(LongType.get()) - int_lit = lit.to(IntegerType.get()) - - assert lit.value == int_lit.value - - -def test_long_to_float_conversion(): - lit = Literal.of(34).to(LongType.get()) - float_lit = lit.to(FloatType.get()) - - assert math.isclose(lit.value, float_lit.value) - - -def test_long_to_double_conversion(): - lit = Literal.of(34).to(LongType.get()) - dbl_lit = lit.to(DoubleType.get()) - - assert math.isclose(lit.value, dbl_lit.value) - - -def test_long_to_decimal_conversion(type_val_tuples): - lit = Literal.of(34).to(LongType.get()) - - assert lit.to(type_val_tuples[0]).value.as_tuple() == Decimal(type_val_tuples[1]).as_tuple() - - -def test_float_to_double(): - lit = Literal.of(34.56) - dbl_lit = lit.to(DoubleType.get()) - - assert math.isclose(lit.value, dbl_lit.value) - - -def test_float_to_decimal_conversion(float_type_val_tuples): - lit = Literal.of(34.56) - - assert lit.to(float_type_val_tuples[0]).value.as_tuple() == Decimal(float_type_val_tuples[1]).as_tuple() - - -def test_double_to_float(): - lit = Literal.of(34.56).to(DoubleType.get()) - float_lit = lit.to(FloatType.get()) - - assert math.isclose(lit.value, float_lit.value) - - -def test_double_to_decimal_conversion(float_type_val_tuples): - lit = Literal.of(34.56).to(DoubleType.get()) - - assert lit.to(float_type_val_tuples[0]).value.as_tuple() == Decimal(float_type_val_tuples[1]).as_tuple() - - -def test_decimal_to_decimal_conversion(): - lit = Literal.of(Decimal("34.11").quantize(Decimal(".01"))) - - assert lit.value.as_tuple() == lit.to(DecimalType.of(9, 2)).value.as_tuple() - assert lit.value.as_tuple() == lit.to(DecimalType.of(11, 2)).value.as_tuple() - assert lit.to(DecimalType.of(9, 0)) is None - assert lit.to(DecimalType.of(9, 1)) is None - assert lit.to(DecimalType.of(9, 3)) is None diff --git a/tests/api/expressions/test_predicate_binding.py b/tests/api/expressions/test_predicate_binding.py deleted file mode 100644 index 7b7e4beafb..0000000000 --- a/tests/api/expressions/test_predicate_binding.py +++ /dev/null @@ -1,199 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -from decimal import Decimal - -from iceberg.api.expressions import (Expressions, - Literal, - Operation, - UnboundPredicate) -from iceberg.api.types import (DecimalType, - FloatType, - IntegerType, - NestedField, - StringType, - StructType) -from iceberg.exceptions import ValidationException - - -def test_multiple_fields(assert_and_unwrap): - struct = StructType.of([NestedField.required(10, 'x', IntegerType.get()), - NestedField.required(11, 'y', IntegerType.get()), - NestedField.required(12, 'z', IntegerType.get())]) - - unbound = UnboundPredicate(Operation.LT, Expressions.ref("y"), 6) - expr = unbound.bind(struct) - - bound = assert_and_unwrap(expr) - assert 11 == bound.ref.field.field_id - assert Operation.LT == bound.op - assert 6 == bound.lit.value - - -def test_missing_field(): - struct = StructType.of([NestedField.required(13, "x", IntegerType.get())]) - - unbound = UnboundPredicate(Operation.LT, Expressions.ref("missing"), 6) - try: - unbound.bind(struct) - except ValidationException as e: - assert e.args[0].startswith("Cannot find field 'missing' in struct") - - -def test_comparison_predicate_binding(op, assert_and_unwrap): - struct = StructType.of([NestedField.required(14, "x", IntegerType.get())]) - unbound = UnboundPredicate(op, Expressions.ref("x"), 5) - bound = assert_and_unwrap(unbound.bind(struct)) - - assert 5 == bound.lit.value - assert 14 == bound.ref.field.field_id - assert op == bound.op - - -def test_literal_converison(op, assert_and_unwrap): - struct = StructType.of([NestedField.required(15, "d", DecimalType.of(9, 2))]) - unbound = UnboundPredicate(op, Expressions.ref("d"), "12.40") - bound = assert_and_unwrap(unbound.bind(struct)) - - assert Decimal(12.40).quantize(Decimal(".01")).as_tuple() == bound.lit.value.as_tuple() - assert 15 == bound.ref.field.field_id - assert op == bound.op - - -def test_invalid_conversions(op): - struct = StructType.of([NestedField.required(16, "f", FloatType.get())]) - unbound = UnboundPredicate(op, Expressions.ref("f"), "12.40") - - try: - unbound.bind(struct) - except ValidationException as e: - assert e.args[0].startswith('Invalid Value for conversion to type float: "12.40" (StringLiteral)') - - -def test_long_to_integer_conversion(assert_and_unwrap): - struct = StructType.of([NestedField.required(17, "i", IntegerType.get())]) - - lt = UnboundPredicate(Operation.LT, Expressions.ref("i"), Literal.JAVA_MAX_INT + 1) - assert lt.bind(struct) == Expressions.always_true() - - lt_eq = UnboundPredicate(Operation.LT_EQ, Expressions.ref("i"), Literal.JAVA_MAX_INT + 1) - assert lt_eq.bind(struct) == Expressions.always_true() - - gt = UnboundPredicate(Operation.GT, Expressions.ref("i"), Literal.JAVA_MIN_INT - 1) - assert gt.bind(struct) == Expressions.always_true() - - gt_eq = UnboundPredicate(Operation.GT_EQ, Expressions.ref("i"), Literal.JAVA_MIN_INT - 1) - assert gt_eq.bind(struct) == Expressions.always_true() - - gt_max = UnboundPredicate(Operation.GT, Expressions.ref("i"), Literal.JAVA_MAX_INT + 1) - assert gt_max.bind(struct) == Expressions.always_false() - - gt_eq_max = UnboundPredicate(Operation.GT_EQ, Expressions.ref("i"), Literal.JAVA_MAX_INT + 1) - assert gt_eq_max.bind(struct) == Expressions.always_false() - - lt_min = UnboundPredicate(Operation.LT, Expressions.ref("i"), Literal.JAVA_MIN_INT - 1) - assert lt_min.bind(struct) == Expressions.always_false() - - lt_eq_min = UnboundPredicate(Operation.LT_EQ, Expressions.ref("i"), Literal.JAVA_MIN_INT - 1) - assert lt_eq_min.bind(struct) == Expressions.always_false() - - lt_expr = UnboundPredicate(Operation.LT, Expressions.ref("i"), Literal.JAVA_MAX_INT).bind(struct) - lt_max = assert_and_unwrap(lt_expr) - assert lt_max.lit.value == Literal.JAVA_MAX_INT - - lt_eq_expr = UnboundPredicate(Operation.LT_EQ, Expressions.ref("i"), Literal.JAVA_MAX_INT).bind(struct) - lt_eq_max = assert_and_unwrap(lt_eq_expr) - assert lt_eq_max.lit.value == Literal.JAVA_MAX_INT - - gt_expr = UnboundPredicate(Operation.GT, Expressions.ref("i"), Literal.JAVA_MIN_INT).bind(struct) - gt_min = assert_and_unwrap(gt_expr) - assert gt_min.lit.value == Literal.JAVA_MIN_INT - - gt_eq_expr = UnboundPredicate(Operation.GT_EQ, Expressions.ref("i"), Literal.JAVA_MIN_INT).bind(struct) - gt_eq_min = assert_and_unwrap(gt_eq_expr) - assert gt_eq_min.lit.value == Literal.JAVA_MIN_INT - - -def test_double_to_float_conversion(assert_and_unwrap): - struct = StructType.of([NestedField.required(18, "f", FloatType.get())]) - - lt = UnboundPredicate(Operation.LT, Expressions.ref("f"), Literal.JAVA_MAX_FLOAT * 2) - assert lt.bind(struct) == Expressions.always_true() - - lt_eq = UnboundPredicate(Operation.LT_EQ, Expressions.ref("f"), Literal.JAVA_MAX_FLOAT * 2) - assert lt_eq.bind(struct) == Expressions.always_true() - - gt = UnboundPredicate(Operation.GT, Expressions.ref("f"), Literal.JAVA_MAX_FLOAT * -2) - assert gt.bind(struct) == Expressions.always_true() - - gt_eq = UnboundPredicate(Operation.GT_EQ, Expressions.ref("f"), Literal.JAVA_MAX_FLOAT * -2) - assert gt_eq.bind(struct) == Expressions.always_true() - - gt_max = UnboundPredicate(Operation.GT, Expressions.ref("f"), Literal.JAVA_MAX_FLOAT * 2) - assert gt_max.bind(struct) == Expressions.always_false() - - gt_eq_max = UnboundPredicate(Operation.GT_EQ, Expressions.ref("f"), Literal.JAVA_MAX_FLOAT * 2) - assert gt_eq_max.bind(struct) == Expressions.always_false() - - lt_min = UnboundPredicate(Operation.LT, Expressions.ref("f"), Literal.JAVA_MAX_FLOAT * -2) - assert lt_min.bind(struct) == Expressions.always_false() - - lt_eq_min = UnboundPredicate(Operation.LT_EQ, Expressions.ref("f"), Literal.JAVA_MAX_FLOAT * -2) - assert lt_eq_min.bind(struct) == Expressions.always_false() - - lt_expr = UnboundPredicate(Operation.LT, Expressions.ref("f"), Literal.JAVA_MAX_FLOAT).bind(struct) - lt_max = assert_and_unwrap(lt_expr) - assert lt_max.lit.value == Literal.JAVA_MAX_FLOAT - - lt_eq_expr = UnboundPredicate(Operation.LT_EQ, Expressions.ref("f"), Literal.JAVA_MAX_FLOAT).bind(struct) - lt_eq_max = assert_and_unwrap(lt_eq_expr) - assert lt_eq_max.lit.value == Literal.JAVA_MAX_FLOAT - - gt_expr = UnboundPredicate(Operation.GT, Expressions.ref("f"), Literal.JAVA_MIN_INT).bind(struct) - gt_min = assert_and_unwrap(gt_expr) - assert gt_min.lit.value == Literal.JAVA_MIN_INT - - gt_eq_expr = UnboundPredicate(Operation.GT_EQ, Expressions.ref("f"), Literal.JAVA_MIN_INT).bind(struct) - gt_eq_min = assert_and_unwrap(gt_eq_expr) - assert gt_eq_min.lit.value == Literal.JAVA_MIN_INT - - -def test_is_null(assert_and_unwrap): - optional = StructType.of([NestedField.optional(19, "s", StringType.get())]) - unbound = UnboundPredicate(Operation.IS_NULL, Expressions.ref("s")) - expr = unbound.bind(optional) - bound = assert_and_unwrap(expr) - - assert Operation.IS_NULL == bound.op - assert 19 == bound.ref.field.field_id - assert bound.lit is None - - required = StructType.of([NestedField.required(20, "s", StringType.get())]) - assert Expressions.always_false() == unbound.bind(required) - - -def test_not_null(assert_and_unwrap): - optional = StructType.of([NestedField.optional(21, "s", StringType.get())]) - unbound = UnboundPredicate(Operation.NOT_NULL, Expressions.ref("s")) - expr = unbound.bind(optional) - bound = assert_and_unwrap(expr) - assert Operation.NOT_NULL == bound.op - assert 21 == bound.ref.field.field_id - assert bound.lit is None - - required = StructType.of([NestedField.required(22, "s", StringType.get())]) - assert Expressions.always_true() == unbound.bind(required) diff --git a/tests/api/expressions/test_str_to_expr.py b/tests/api/expressions/test_str_to_expr.py deleted file mode 100644 index 34b3abdf77..0000000000 --- a/tests/api/expressions/test_str_to_expr.py +++ /dev/null @@ -1,161 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -from iceberg.api.expressions import Expressions - - -def test_equal(): - expected_expr = Expressions.equal("col_a", 1) - conv_expr = Expressions.convert_string_to_expr("col_a=1") - assert expected_expr == conv_expr - - -def test_equal_alt_syntax(): - expected_expr = Expressions.equal("col_a", 1) - conv_expr = Expressions.convert_string_to_expr("col_a==1") - assert expected_expr == conv_expr - - -def test_gt(): - expected_expr = Expressions.greater_than("col_a", 1) - conv_expr = Expressions.convert_string_to_expr("col_a > 1") - assert expected_expr == conv_expr - - -def test_gte(): - expected_expr = Expressions.greater_than_or_equal("col_a", 1) - conv_expr = Expressions.convert_string_to_expr("col_a >= 1") - assert expected_expr == conv_expr - - -def test_lt(): - expected_expr = Expressions.less_than("col_a", 1) - conv_expr = Expressions.convert_string_to_expr("col_a < 1") - assert expected_expr == conv_expr - - -def test_lte(): - expected_expr = Expressions.less_than_or_equal("col_a", 1) - conv_expr = Expressions.convert_string_to_expr("col_a <= 1") - assert expected_expr == conv_expr - - -def test_and(): - expected_expr = Expressions.and_(Expressions.equal("col_a", 1), Expressions.equal("col_b", 2)) - conv_expr = Expressions.convert_string_to_expr("col_a=1 and col_b=2") - assert expected_expr == conv_expr - - -def test_or(): - expected_expr = Expressions.or_(Expressions.equal("col_a", 1), Expressions.equal("col_b", 2)) - conv_expr = Expressions.convert_string_to_expr("col_a=1 or col_b=2") - assert expected_expr == conv_expr - - -def test_between(): - expected_expr = Expressions.and_(Expressions.greater_than_or_equal("col_a", 1), - Expressions.less_than_or_equal("col_a", 2)) - conv_expr = Expressions.convert_string_to_expr("col_a between 1 and 2") - assert expected_expr == conv_expr - - -def test_is_null(): - expected_expr = Expressions.is_null("col_a") - conv_expr = Expressions.convert_string_to_expr("col_a is null") - assert expected_expr == conv_expr - - -def test_not_null(): - expected_expr = Expressions.not_null("col_a") - conv_expr = Expressions.convert_string_to_expr("col_a is not null") - assert expected_expr == conv_expr - - -def test_not(): - expected_expr = Expressions.not_("col_a") - conv_expr = Expressions.convert_string_to_expr("not col_a") - assert expected_expr == conv_expr - - -def test_not_equal(): - expected_expr = Expressions.not_equal("col_a", 7) - conv_expr = Expressions.convert_string_to_expr("col_a <> 7") - assert expected_expr == conv_expr - - -def test_not_equal_alt_syntax(): - expected_expr = Expressions.not_equal("col_a", 7) - conv_expr = Expressions.convert_string_to_expr("col_a != 7") - assert expected_expr == conv_expr - - -def test_compound_not_equal(): - expected_expr = Expressions.not_(Expressions.equal("col_a", 7)) - conv_expr = Expressions.convert_string_to_expr("not (col_a = 7)") - assert expected_expr == conv_expr - - -def test_ternary_condition(): - expected_expr = Expressions.and_(Expressions.equal("col_a", 1), - Expressions.and_(Expressions.equal("col_b", 2), - Expressions.equal("col_c", 3))) - - conv_expr = Expressions.convert_string_to_expr("col_a=1 and col_b=2 and col_c=3") - assert expected_expr == conv_expr - - -def test_precedence(): - expected_expr = Expressions.or_(Expressions.equal("col_a", 1), - Expressions.and_(Expressions.equal("col_b", 2), - Expressions.equal("col_c", 3))) - - conv_expr = Expressions.convert_string_to_expr("col_a=1 or col_b=2 and col_c=3") - assert expected_expr == conv_expr - - -def test_precedence_opposite_order(): - expected_expr = Expressions.or_(Expressions.and_(Expressions.equal("col_a", 1), - Expressions.equal("col_b", 2)), - Expressions.equal("col_c", 3)) - - conv_expr = Expressions.convert_string_to_expr("col_a=1 and col_b=2 or col_c=3") - assert expected_expr == conv_expr - - -def test_precedence_explicit(): - expected_expr = Expressions.and_(Expressions.equal("col_a", 1), - Expressions.or_(Expressions.equal("col_b", 2), - Expressions.equal("col_c", 3))) - - conv_expr = Expressions.convert_string_to_expr("col_a=1 and (col_b=2 or col_c=3)") - assert expected_expr == conv_expr - - -def test_precedence_with_between(): - expected_expr = Expressions.or_(Expressions.and_(Expressions.greater_than_or_equal("col_a", 1), - Expressions.less_than_or_equal("col_a", 2)), - Expressions.equal("col_c", 3)) - - conv_expr = Expressions.convert_string_to_expr("col_a between 1 and 2 or col_c=3") - assert expected_expr == conv_expr - - -def test_complex_expansion(): - expected_expr = Expressions.or_(Expressions.and_(Expressions.equal("a", 1), - Expressions.and_(Expressions.equal("b", 2), - Expressions.not_equal("c", 3))), - Expressions.is_null("d")) - conv_expr = Expressions.convert_string_to_expr("(a=1 and b=2 and c<>3) or d is null") - assert expected_expr == conv_expr diff --git a/tests/api/expressions/test_strict_metrics_evaluator.py b/tests/api/expressions/test_strict_metrics_evaluator.py deleted file mode 100644 index 36ab99db74..0000000000 --- a/tests/api/expressions/test_strict_metrics_evaluator.py +++ /dev/null @@ -1,159 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -from iceberg.api.expressions import (Expressions, - StrictMetricsEvaluator) -from iceberg.exceptions import ValidationException -from pytest import raises - - -def test_all_nulls(strict_schema, strict_file): - assert not StrictMetricsEvaluator(strict_schema, Expressions.not_null("all_nulls")).eval(strict_file) - assert not StrictMetricsEvaluator(strict_schema, Expressions.not_null("some_nulls")).eval(strict_file) - assert StrictMetricsEvaluator(strict_schema, Expressions.not_null("no_nulls")).eval(strict_file) - - -def test_no_nulls(strict_schema, strict_file): - assert StrictMetricsEvaluator(strict_schema, Expressions.is_null("all_nulls")).eval(strict_file) - assert not StrictMetricsEvaluator(strict_schema, Expressions.is_null("some_nulls")).eval(strict_file) - assert not StrictMetricsEvaluator(strict_schema, Expressions.is_null("no_nulls")).eval(strict_file) - - -def test_required_columns(strict_schema, strict_file): - assert StrictMetricsEvaluator(strict_schema, Expressions.not_null("required")).eval(strict_file) - assert not StrictMetricsEvaluator(strict_schema, Expressions.is_null("required")).eval(strict_file) - - -def test_missing_column(strict_schema, strict_file): - with raises(ValidationException): - StrictMetricsEvaluator(strict_schema, Expressions.less_than("missing", 5)).eval(strict_file) - - -def test_missing_stats(strict_schema, missing_stats): - exprs = [Expressions.less_than("no_stats", 5), - Expressions.less_than_or_equal("no_stats", 30), - Expressions.equal("no_stats", 70), - Expressions.greater_than("no_stats", 78), - Expressions.greater_than_or_equal("no_stats", 90), - Expressions.not_equal("no_stats", 101), - Expressions.is_null("no_stats"), - Expressions.not_null("no_stats")] - - for expr in exprs: - assert not StrictMetricsEvaluator(strict_schema, expr).eval(missing_stats) - - -def test_zero_record_file(strict_schema, empty): - - exprs = [Expressions.less_than("no_stats", 5), - Expressions.less_than_or_equal("no_stats", 30), - Expressions.equal("no_stats", 70), - Expressions.greater_than("no_stats", 78), - Expressions.greater_than_or_equal("no_stats", 90), - Expressions.not_equal("no_stats", 101), - Expressions.is_null("no_stats"), - Expressions.not_null("no_stats")] - for expr in exprs: - assert StrictMetricsEvaluator(strict_schema, expr).eval(empty) - - -def test_not(strict_schema, strict_file): - assert StrictMetricsEvaluator(strict_schema, - Expressions.not_(Expressions.less_than("id", 5))).eval(strict_file) - assert not StrictMetricsEvaluator(strict_schema, - Expressions.not_(Expressions.greater_than("id", 5))).eval(strict_file) - - -def test_and(strict_schema, strict_file): - assert not StrictMetricsEvaluator(strict_schema, - Expressions.and_(Expressions.greater_than("id", 5), - Expressions.less_than_or_equal("id", 30))).eval(strict_file) - assert not StrictMetricsEvaluator(strict_schema, - Expressions.and_(Expressions.less_than("id", 5), - Expressions.greater_than_or_equal("id", 0))).eval(strict_file) - assert StrictMetricsEvaluator(strict_schema, - Expressions.and_(Expressions.less_than("id", 85), - Expressions.greater_than_or_equal("id", 0))).eval(strict_file) - - -def test_or(strict_schema, strict_file): - assert not StrictMetricsEvaluator(strict_schema, - Expressions.or_(Expressions.less_than("id", 5), - Expressions.greater_than_or_equal("id", 80))).eval(strict_file) - assert not StrictMetricsEvaluator(strict_schema, - Expressions.or_(Expressions.less_than("id", 5), - Expressions.greater_than_or_equal("id", 60))).eval(strict_file) - assert StrictMetricsEvaluator(strict_schema, - Expressions.or_(Expressions.less_than("id", 5), - Expressions.greater_than_or_equal("id", 30))).eval(strict_file) - - -def test_integer_lt(strict_schema, strict_file): - assert not StrictMetricsEvaluator(strict_schema, Expressions.less_than("id", 5)).eval(strict_file) - assert not StrictMetricsEvaluator(strict_schema, Expressions.less_than("id", 31)).eval(strict_file) - assert not StrictMetricsEvaluator(strict_schema, Expressions.less_than("id", 79)).eval(strict_file) - assert StrictMetricsEvaluator(strict_schema, Expressions.less_than("id", 80)).eval(strict_file) - - -def test_integer_lt_eq(strict_schema, strict_file): - assert not StrictMetricsEvaluator(strict_schema, Expressions.less_than_or_equal("id", 29)).eval(strict_file) - assert not StrictMetricsEvaluator(strict_schema, Expressions.less_than_or_equal("id", 30)).eval(strict_file) - assert StrictMetricsEvaluator(strict_schema, Expressions.less_than_or_equal("id", 79)).eval(strict_file) - assert StrictMetricsEvaluator(strict_schema, Expressions.less_than_or_equal("id", 80)).eval(strict_file) - - -def test_integer_gt(strict_schema, strict_file): - assert not StrictMetricsEvaluator(strict_schema, Expressions.greater_than("id", 79)).eval(strict_file) - assert not StrictMetricsEvaluator(strict_schema, Expressions.greater_than("id", 78)).eval(strict_file) - assert not StrictMetricsEvaluator(strict_schema, Expressions.greater_than("id", 30)).eval(strict_file) - assert StrictMetricsEvaluator(strict_schema, Expressions.greater_than("id", 29)).eval(strict_file) - - -def test_integer_gt_eq(strict_schema, strict_file): - assert not StrictMetricsEvaluator(strict_schema, Expressions.greater_than_or_equal("id", 80)).eval(strict_file) - assert not StrictMetricsEvaluator(strict_schema, Expressions.greater_than_or_equal("id", 79)).eval(strict_file) - assert not StrictMetricsEvaluator(strict_schema, Expressions.greater_than_or_equal("id", 31)).eval(strict_file) - assert StrictMetricsEvaluator(strict_schema, Expressions.greater_than_or_equal("id", 30)).eval(strict_file) - - -def test_integer_eq(strict_schema, strict_file): - assert not StrictMetricsEvaluator(strict_schema, Expressions.equal("id", 5)).eval(strict_file) - assert not StrictMetricsEvaluator(strict_schema, Expressions.equal("id", 30)).eval(strict_file) - assert not StrictMetricsEvaluator(strict_schema, Expressions.equal("id", 75)).eval(strict_file) - assert not StrictMetricsEvaluator(strict_schema, Expressions.equal("id", 79)).eval(strict_file) - assert not StrictMetricsEvaluator(strict_schema, Expressions.equal("id", 80)).eval(strict_file) - assert StrictMetricsEvaluator(strict_schema, Expressions.equal("always_5", 5)).eval(strict_file) - - -def test_integer_not_eq(strict_schema, strict_file): - assert StrictMetricsEvaluator(strict_schema, Expressions.not_equal("id", 5)).eval(strict_file) - assert StrictMetricsEvaluator(strict_schema, Expressions.not_equal("id", 29)).eval(strict_file) - assert not StrictMetricsEvaluator(strict_schema, Expressions.not_equal("id", 30)).eval(strict_file) - assert not StrictMetricsEvaluator(strict_schema, Expressions.not_equal("id", 75)).eval(strict_file) - assert not StrictMetricsEvaluator(strict_schema, Expressions.not_equal("id", 79)).eval(strict_file) - assert StrictMetricsEvaluator(strict_schema, Expressions.not_equal("id", 80)).eval(strict_file) - assert StrictMetricsEvaluator(strict_schema, Expressions.not_equal("id", 85)).eval(strict_file) - - -def test_not_eq_rewritten(strict_schema, strict_file): - assert StrictMetricsEvaluator(strict_schema, Expressions.not_(Expressions.equal("id", 5))).eval(strict_file) - assert StrictMetricsEvaluator(strict_schema, Expressions.not_(Expressions.equal("id", 29))).eval(strict_file) - assert not StrictMetricsEvaluator(strict_schema, Expressions.not_(Expressions.equal("id", 30))).eval(strict_file) - assert not StrictMetricsEvaluator(strict_schema, Expressions.not_(Expressions.equal("id", 75))).eval(strict_file) - assert not StrictMetricsEvaluator(strict_schema, Expressions.not_(Expressions.equal("id", 79))).eval(strict_file) - assert StrictMetricsEvaluator(strict_schema, Expressions.not_(Expressions.equal("id", 80))).eval(strict_file) - assert StrictMetricsEvaluator(strict_schema, Expressions.not_(Expressions.equal("id", 85))).eval(strict_file) diff --git a/tests/api/expressions/test_string_literal_conversions.py b/tests/api/expressions/test_string_literal_conversions.py deleted file mode 100644 index 57c2140af9..0000000000 --- a/tests/api/expressions/test_string_literal_conversions.py +++ /dev/null @@ -1,102 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -from datetime import datetime -from decimal import Decimal -import uuid - -import dateutil.parser -from fastavro.write import LOGICAL_WRITERS as avro_conversion -from iceberg.api.expressions import Literal -from iceberg.api.types import (DateType, - DecimalType, - StringType, - TimestampType, - TimeType, - UUIDType) -from pytest import raises - - -def test_string_to_string_literal(): - assert Literal.of("abc") == Literal.of("abc").to(StringType.get()) - - -def test_string_to_date_literal(): - date_str = Literal.of("2017-08-18") - date = date_str.to(DateType.get()) - - avro_val = avro_conversion["int-date"](datetime.strptime("2017-08-18", "%Y-%m-%d"), None) - assert avro_val == date.value - - -def test_string_to_time_literal(): - time_str = Literal.of("14:21:01.919") - time_lit = time_str.to(TimeType.get()) - - avro_val = avro_conversion["long-time-micros"](datetime.strptime("14:21:01.919", "%H:%M:%S.%f").time(), None) - - assert avro_val == time_lit.value - - -def test_string_to_timestamp_literal(): - timestamp_str = Literal.of("2017-08-18T14:21:01.919+00:00") - timestamp = timestamp_str.to(TimestampType.with_timezone()) - - avro_val = avro_conversion["long-timestamp-micros"](dateutil.parser.parse("2017-08-18T14:21:01.919+00:00"), - None) - assert avro_val == timestamp.value - - timestamp_str = Literal.of("2017-08-18T14:21:01.919") - timestamp = timestamp_str.to(TimestampType.without_timezone()) - assert avro_val == timestamp.value - - timestamp_str = Literal.of("2017-08-18T14:21:01.919-07:00") - timestamp = timestamp_str.to(TimestampType.with_timezone()) - avro_val = avro_conversion["long-timestamp-micros"](dateutil.parser.parse("2017-08-18T21:21:01.919+00:00"), - None) - assert avro_val == timestamp.value - - -def test_timestamp_with_zone_without_zone_in_literal(): - with raises(RuntimeError): - timestamp_str = Literal.of("2017-08-18T14:21:01.919") - timestamp_str.to(TimestampType.with_timezone()) - - -def test_timestamp_without_zone_with_zone_in_literal(): - with raises(RuntimeError): - timestamp_str = Literal.of("2017-08-18T14:21:01.919+07:00") - timestamp_str.to(TimestampType.without_timezone()) - - -def test_string_to_uuid_literal(): - expected = uuid.uuid4() - uuid_str = Literal.of(str(expected)) - uuid_lit = uuid_str.to(UUIDType.get()) - - assert expected == uuid_lit.value - - -def test_string_to_decimal_literal(): - decimal_str = Literal.of("34.560") - decimal_lit = decimal_str.to(DecimalType.of(9, 3)) - - assert 3 == abs(decimal_lit.value.as_tuple().exponent) - assert Decimal("34.560").as_tuple() == decimal_lit.value.as_tuple() - - assert decimal_str.to(DecimalType.of(9, 2)) is None - assert decimal_str.to(DecimalType.of(9, 4)) is None diff --git a/tests/api/test_conversions.py b/tests/api/test_conversions.py deleted file mode 100644 index d7397ceb37..0000000000 --- a/tests/api/test_conversions.py +++ /dev/null @@ -1,81 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -import unittest -import uuid - -from iceberg.api.expressions import Literal -from iceberg.api.types import (BinaryType, - BooleanType, - DateType, - DoubleType, - FixedType, - FloatType, - IntegerType, - LongType, - StringType, - TimestampType, - TimeType, - UUIDType) -from iceberg.api.types.conversions import Conversions - - -class TestConversions(unittest.TestCase): - - def test_from_bytes(self): - self.assertEqual(False, Conversions.from_byte_buffer(BooleanType.get(), b'\x00\x00')) - self.assertEqual(True, Conversions.from_byte_buffer(BooleanType.get(), b'\x01\x00')) - self.assertEqual(1234, Conversions.from_byte_buffer(IntegerType.get(), - b'\xd2\x04\x00\x00')) - self.assertEqual(1234, Conversions.from_byte_buffer(LongType.get(), - b'\xd2\x04\x00\x00\x00\x00\x00\x00')) - self.assertAlmostEqual(1.2345, Conversions.from_byte_buffer(FloatType.get(), - b'\x19\x04\x9e?'), places=5) - self.assertAlmostEqual(1.2345, Conversions.from_byte_buffer(DoubleType.get(), - b'\x8d\x97\x6e\x12\x83\xc0\xf3\x3f')) - self.assertEqual(1234, Conversions.from_byte_buffer(DateType.get(), - b'\xd2\x04\x00\x00')) - self.assertEqual(100000000000, Conversions.from_byte_buffer(TimeType.get(), - b'\x00\xe8vH\x17\x00\x00\x00')) - self.assertEqual(100000000000, Conversions.from_byte_buffer(TimestampType.with_timezone(), - b'\x00\xe8vH\x17\x00\x00\x00')) - self.assertEqual(100000000000, Conversions.from_byte_buffer(TimestampType.without_timezone(), - b'\x00\xe8vH\x17\x00\x00\x00')) - self.assertEqual("foo", Conversions.from_byte_buffer(StringType.get(), b'foo')) - self.assertEqual(uuid.UUID("f79c3e09-677c-4bbd-a479-3f349cb785e7"), - Conversions.from_byte_buffer(UUIDType.get(), b'\xf7\x9c>\tg|K\xbd\xa4y?4\x9c\xb7\x85\xe7')) - self.assertEqual(b'foo', Conversions.from_byte_buffer(FixedType.of_length(3), b'foo')) - self.assertEqual(b'foo', Conversions.from_byte_buffer(BinaryType.get(), b'foo')) - - def test_to_bytes(self): - self.assertEqual(b'\x00\x00', Literal.of(False).to_byte_buffer()) - self.assertEqual(b'\x01\x00', Literal.of(True).to_byte_buffer()) - self.assertEqual(b'\xd2\x04\x00\x00', Literal.of(1234).to_byte_buffer()) - self.assertEqual(b'\xd2\x04\x00\x00\x00\x00\x00\x00', Literal.of(1234).to(LongType.get()).to_byte_buffer()) - self.assertEqual(b'\x19\x04\x9e?', Literal.of(1.2345).to_byte_buffer()) - self.assertEqual(b'\x8d\x97\x6e\x12\x83\xc0\xf3\x3f', Literal.of(1.2345).to(DoubleType.get()).to_byte_buffer()) - self.assertEqual(b'\xd2\x04\x00\x00', Literal.of(1234).to(DateType.get()).to_byte_buffer()) - self.assertEqual(b'\x00\xe8vH\x17\x00\x00\x00', Literal.of(100000000000).to(TimeType.get()).to_byte_buffer()) - self.assertEqual(b'\x00\xe8vH\x17\x00\x00\x00', - Literal.of(100000000000).to(TimestampType.with_timezone()).to_byte_buffer()) - self.assertEqual(b'\x00\xe8vH\x17\x00\x00\x00', - Literal.of(100000000000).to(TimestampType.without_timezone()).to_byte_buffer()) - self.assertEqual(b'foo', Literal.of("foo").to_byte_buffer()) - self.assertEqual(b'\xf7\x9c>\tg|K\xbd\xa4y?4\x9c\xb7\x85\xe7', - Literal.of(uuid.UUID("f79c3e09-677c-4bbd-a479-3f349cb785e7")).to_byte_buffer()) - self.assertEqual(b'foo', Literal.of(bytes(b'foo')).to_byte_buffer()) - self.assertEqual(b'foo', Literal.of(bytearray(b'foo')).to_byte_buffer()) diff --git a/tests/api/test_file_format.py b/tests/api/test_file_format.py deleted file mode 100644 index 773e57fe46..0000000000 --- a/tests/api/test_file_format.py +++ /dev/null @@ -1,43 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -from iceberg.api import FileFormat - - -def test_parquet(): - file_fmt = FileFormat.PARQUET - file_name = "test_file.parquet" - add_extension_file = "test_file" - assert file_fmt.is_splittable() - assert FileFormat.from_file_name(file_name) == FileFormat.PARQUET - assert file_name == FileFormat.PARQUET.add_extension(add_extension_file) - - -def test_avro(): - file_fmt = FileFormat.AVRO - file_name = "test_file.avro" - add_extension_file = "test_file" - assert file_fmt.is_splittable() - assert FileFormat.from_file_name(file_name) == FileFormat.AVRO - assert file_name == FileFormat.AVRO.add_extension(add_extension_file) - - -def test_orc(): - file_fmt = FileFormat.ORC - file_name = "test_file.orc" - add_extension_file = "test_file" - assert file_fmt.is_splittable() - assert FileFormat.from_file_name(file_name) == FileFormat.ORC - assert file_name == FileFormat.ORC.add_extension(add_extension_file) diff --git a/tests/api/test_helpers.py b/tests/api/test_helpers.py deleted file mode 100644 index 55d85ac2b0..0000000000 --- a/tests/api/test_helpers.py +++ /dev/null @@ -1,156 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import io -import pickle - -from iceberg.api.data_file import DataFile -from iceberg.api.expressions.expressions import ExpressionVisitors -from iceberg.api.expressions.predicate import (BoundPredicate, - UnboundPredicate) -from iceberg.api.struct_like import StructLike -from nose.tools import assert_true - - -class TestHelpers(object): - - @staticmethod - def assert_all_references_bound(message, expr): - ExpressionVisitors.visit(expr, TestHelpers.CheckReferencesBound(message)) - - @staticmethod - def assert_and_unwrap(expr, expected=None): - if expected is not None: - assert_true(isinstance(expr, expected)) - else: - assert_true(isinstance(expr, BoundPredicate)) - - return expr - - @staticmethod - def round_trip_serialize(type_var): - stream = io.BytesIO() - pickle.dump(type_var, stream, pickle.HIGHEST_PROTOCOL) - stream.seek(0) - - return pickle.load(stream) - - class Row(StructLike): - - @staticmethod - def of(values=None): - return TestHelpers.Row(values) - - def __init__(self, values): - self.values = values - - def get(self, pos): - return self.values[pos] - - def set(self, pos, value): - raise RuntimeError("Setting values is not supported") - - class CheckReferencesBound(ExpressionVisitors.ExpressionVisitor): - - def __init__(self, message): - self.message = message - - def predicate(self, pred): - if isinstance(pred, UnboundPredicate): - assert_true(False) - - return None - - -class MockDataFile(DataFile): - - def __init__(self, path, partition, record_count, value_counts=None, null_value_counts=None, - lower_bounds=None, upper_bounds=None): - self.path = path - self.partition = partition - self.record_count = record_count - self.value_counts = value_counts - self.null_value_counts = null_value_counts - self.lower_bounds = lower_bounds - self.upper_bounds = upper_bounds - self.file_size_in_bytes = 0 - self.block_size_in_bytes = 0 - self.file_ordinal = None - self.column_size = None - - def copy(self): - return self - - -class MockHMSTable(object): - def __init__(self, params): - self.parameters = params - - -class MockManifestEntry(object): - def __init__(self, path): - self._path = path - - def path(self): - return self._path - - -class MockReader(object): - def __init__(self, entries): - self._entries = entries - - def iterator(self): - return iter(self._entries) - - -class MockTableOperations(object): - def __init__(self, metadata, location): - self.metadata = metadata - self.current_metadata_location = location - self.deleted = [] - - def current(self): - return self.metadata - - def delete_file(self, path): - self.deleted.append(path) - - -class MockManifest(object): - def __init__(self, manifest_path): - self.manifest_path = manifest_path - - -class MockSnapshot(object): - def __init__(self, location, manifests, manifest_to_entries): - self._location = location - self._manifests = manifests - self._manifest_to_entries = manifest_to_entries - - @property - def manifests(self): - return iter(self._manifests) - - @property - def manifest_location(self): - return self._location - - def get_filtered_manifest(self, path): - return self._manifest_to_entries[path] - - -class MockMetadata(object): - def __init__(self, snapshots): - self.snapshots = snapshots diff --git a/tests/api/test_partition_spec.py b/tests/api/test_partition_spec.py deleted file mode 100644 index 70ab574349..0000000000 --- a/tests/api/test_partition_spec.py +++ /dev/null @@ -1,82 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -import unittest - -from iceberg.api import PartitionSpec -from iceberg.api.schema import Schema -from iceberg.api.types import (BinaryType, - DateType, - DecimalType, - FixedType, - IntegerType, - LongType, - NestedField, - StringType, - TimestampType, - TimeType, - UUIDType) -from tests.api.test_helpers import TestHelpers - - -class TestPartitionSpec(unittest.TestCase): - - def test_partition_spec(self): - schema = Schema(NestedField.required(1, "i", IntegerType.get()), - NestedField.required(2, "l", LongType.get()), - NestedField.required(3, "d", DateType.get()), - NestedField.required(4, "t", TimeType.get()), - NestedField.required(5, "ts", TimestampType.without_timezone()), - NestedField.required(6, "dec", DecimalType.of(9, 2)), - NestedField.required(7, "s", StringType.get()), - NestedField.required(8, "u", UUIDType.get()), - NestedField.required(9, "f", FixedType.of_length(3)), - NestedField.required(10, "b", BinaryType.get())) - specs = [PartitionSpec.builder_for(schema).identity("i").build(), - PartitionSpec.builder_for(schema).identity("l").build(), - PartitionSpec.builder_for(schema).identity("d").build(), - PartitionSpec.builder_for(schema).identity("t").build(), - PartitionSpec.builder_for(schema).identity("ts").build(), - PartitionSpec.builder_for(schema).identity("dec").build(), - PartitionSpec.builder_for(schema).identity("s").build(), - PartitionSpec.builder_for(schema).identity("u").build(), - PartitionSpec.builder_for(schema).identity("f").build(), - PartitionSpec.builder_for(schema).identity("b").build(), - PartitionSpec.builder_for(schema).bucket("i", 128).build(), - PartitionSpec.builder_for(schema).bucket("l", 128).build(), - PartitionSpec.builder_for(schema).bucket("d", 128).build(), - PartitionSpec.builder_for(schema).bucket("t", 128).build(), - PartitionSpec.builder_for(schema).bucket("ts", 128).build(), - PartitionSpec.builder_for(schema).bucket("dec", 128).build(), - PartitionSpec.builder_for(schema).bucket("s", 128).build(), - PartitionSpec.builder_for(schema).bucket("u", 128).build(), - PartitionSpec.builder_for(schema).bucket("f", 128).build(), - PartitionSpec.builder_for(schema).bucket("b", 128).build(), - PartitionSpec.builder_for(schema).year("d").build(), - PartitionSpec.builder_for(schema).month("d").build(), - PartitionSpec.builder_for(schema).day("d").build(), - PartitionSpec.builder_for(schema).year("ts").build(), - PartitionSpec.builder_for(schema).month("ts").build(), - PartitionSpec.builder_for(schema).day("ts").build(), - PartitionSpec.builder_for(schema).hour("ts").build(), - PartitionSpec.builder_for(schema).truncate("i", 10).build(), - PartitionSpec.builder_for(schema).truncate("l", 10).build(), - PartitionSpec.builder_for(schema).truncate("dec", 10).build(), - PartitionSpec.builder_for(schema).truncate("s", 10).build(), - PartitionSpec.builder_for(schema).add_without_field_id(6, "dec_unsupported", "unsupported").build(), - PartitionSpec.builder_for(schema).add(6, 1111, "dec_unsupported", "unsupported").build(), - ] - - for spec in specs: - self.assertEqual(spec, TestHelpers.round_trip_serialize(spec)) diff --git a/tests/api/transforms/__init__.py b/tests/api/transforms/__init__.py deleted file mode 100644 index 13a83393a9..0000000000 --- a/tests/api/transforms/__init__.py +++ /dev/null @@ -1,16 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. diff --git a/tests/api/transforms/test_bucket.py b/tests/api/transforms/test_bucket.py deleted file mode 100644 index afda054017..0000000000 --- a/tests/api/transforms/test_bucket.py +++ /dev/null @@ -1,49 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -import decimal -import unittest -import uuid - -from iceberg.api.transforms import Transforms -from iceberg.api.types import (BinaryType, - DateType, - DecimalType, - FixedType, - IntegerType, - LongType, - StringType, - TimestampType, - TimeType, - UUIDType) - - -class TestBucket(unittest.TestCase): - - def test_bucket_hash(self): - buckets = [ - [Transforms.bucket(IntegerType.get(), 100), 34, 2017239379], - [Transforms.bucket(LongType.get(), 100), 34, 2017239379], - [Transforms.bucket(DateType.get(), 100), 17486, -653330422], - [Transforms.bucket(TimeType.get(), 100), 81068000000, -662762989], - [Transforms.bucket(TimestampType.without_timezone(), 100), 1510871468000000, -2047944441], - [Transforms.bucket(DecimalType.of(9, 2), 100), decimal.Decimal("14.20"), -500754589], - [Transforms.bucket(StringType.get(), 100), "iceberg", 1210000089], - [Transforms.bucket(UUIDType.get(), 100), uuid.UUID("f79c3e09-677c-4bbd-a479-3f349cb785e7"), 1488055340], - [Transforms.bucket(FixedType.of_length(3), 128), b'foo', -156908512], - [Transforms.bucket(BinaryType.get(), 128), b'\x00\x01\x02\x03', -188683207] - ] - - for bucket in buckets: - self.assertEqual(bucket[2], bucket[0].hash(bucket[1])) diff --git a/tests/api/transforms/test_bucketing.py b/tests/api/transforms/test_bucketing.py deleted file mode 100644 index a0eed1c20e..0000000000 --- a/tests/api/transforms/test_bucketing.py +++ /dev/null @@ -1,64 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -from decimal import Decimal, getcontext - -from iceberg.api.expressions import Literal -from iceberg.api.transforms import (Bucket, - BucketDouble, - BucketFloat) -from iceberg.api.types import (DateType, - DecimalType, - IntegerType, - LongType, - TimestampType, - TimeType) -import pytest - - -@pytest.mark.parametrize("test_input,test_type,expected", [ - (1, IntegerType, 1392991556), - (34, IntegerType, 2017239379), - (34, LongType, 2017239379)]) -def test_spec_values_int(test_input, test_type, expected): - assert Bucket.get(test_type.get(), 100).hash(test_input) == expected - - -@pytest.mark.parametrize("test_input,test_type,expected", [ - (1, BucketFloat(100), -142385009), - (1, BucketDouble(100), -142385009)]) -def test_spec_values_dbl(test_input, test_type, expected): - assert test_type.hash(test_input) == expected - - -@pytest.mark.parametrize("test_input,test_type,scale_factor,expected", [ - (Decimal("14.20"), DecimalType.of(9, 2), Decimal(10) ** -2, -500754589), - (Decimal("137302769811943318102518958871258.37580"), DecimalType.of(38, 5), Decimal(10) ** -5, -32334285)]) -def test_spec_values_dec(test_input, test_type, scale_factor, expected): - getcontext().prec = 38 - assert Bucket.get(test_type, 100).hash(test_input.quantize(scale_factor)) == expected - - -@pytest.mark.parametrize("test_input,test_type,expected", [ - (Literal.of("2017-11-16").to(DateType.get()), DateType.get(), -653330422), - (Literal.of("22:31:08").to(TimeType.get()), TimeType.get(), -662762989), - (Literal.of("2017-11-16T22:31:08").to(TimestampType.without_timezone()), - TimestampType.without_timezone(), -2047944441), - (Literal.of("2017-11-16T14:31:08-08:00").to(TimestampType.with_timezone()), - TimestampType.with_timezone(), -2047944441)]) -def test_spec_values_datetime_uuid(test_input, test_type, expected): - assert Bucket.get(test_type, 100).hash(test_input.value) == expected diff --git a/tests/api/transforms/test_dates.py b/tests/api/transforms/test_dates.py deleted file mode 100644 index 9f782d6712..0000000000 --- a/tests/api/transforms/test_dates.py +++ /dev/null @@ -1,44 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - - -from iceberg.api.expressions import Literal -from iceberg.api.transforms import Transforms -from iceberg.api.types import DateType -import pytest - - -@pytest.mark.parametrize("transform_gran,expected", [ - (Transforms.year, "2017"), - (Transforms.month, "2017-12"), - (Transforms.day, "2017-12-01")]) -def test_date_to_human_string(transform_gran, expected): - type_var = DateType.get() - date = Literal.of("2017-12-01").to(type_var) - - assert (transform_gran(DateType.get()) - .to_human_string(transform_gran(DateType.get()) - .apply(date.value))) == expected - - -@pytest.mark.parametrize("transform_gran", [ - Transforms.year, - Transforms.month, - Transforms.day]) -def test_null_human_string(transform_gran): - type_var = DateType.get() - assert transform_gran(type_var).to_human_string(None) == "null" diff --git a/tests/api/transforms/test_identity.py b/tests/api/transforms/test_identity.py deleted file mode 100644 index d23f2026ed..0000000000 --- a/tests/api/transforms/test_identity.py +++ /dev/null @@ -1,93 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -from decimal import Decimal - -from iceberg.api.expressions import Literal -from iceberg.api.transforms import (Identity, - Transforms) -from iceberg.api.types import (DateType, - DecimalType, - LongType, - StringType, - TimestampType, - TimeType) - - -def test_null_human_string(): - long_type = LongType.get() - identity = Identity(long_type) - assert identity.to_human_string(None) == "null" - - -def test_date_human_string(): - date = DateType.get() - - identity = Transforms.identity(date) - date_str = "2017-12-01" - d = Literal.of(date_str).to(date) - assert identity.to_human_string(d.value) == date_str - - -def test_time_human_string(): - time = TimeType.get() - - identity = Transforms.identity(time) - time_str = "10:12:55.038194" - d = Literal.of(time_str).to(time) - assert identity.to_human_string(d.value) == time_str - - -def test_timestamp_with_zone_human_string(): - ts_tz = TimestampType.with_timezone() - identity = Transforms.identity(ts_tz) - ts = Literal.of("2017-12-01T10:12:55.038194-08:00").to(ts_tz) - - assert identity.to_human_string(ts.value) == "2017-12-01T18:12:55.038194Z" - - -def test_timestamp_without_zone_human_string(): - ts_tz = TimestampType.without_timezone() - identity = Transforms.identity(ts_tz) - ts_str = "2017-12-01T10:12:55.038194" - ts = Literal.of(ts_str).to(ts_tz) - - assert identity.to_human_string(ts.value) == ts_str - - -def test_long_to_human_string(): - long_type = LongType.get() - identity = Transforms.identity(long_type) - assert identity.to_human_string(-1234567890000) == "-1234567890000" - - -def test_string_to_human_string(): - str_type = StringType - identity = Transforms.identity(str_type) - - with_slash = "a/b/c=d" - assert identity.to_human_string(with_slash) == with_slash - - -def test_big_decimal_to_human_string(): - big_dec = DecimalType.of(9, 2) - identity = Transforms.identity(big_dec) - - dec_str = "-1.50" - dec_var = Decimal(dec_str) - - assert identity.to_human_string(dec_var) == dec_str diff --git a/tests/api/transforms/test_timestamps.py b/tests/api/transforms/test_timestamps.py deleted file mode 100644 index 98784571df..0000000000 --- a/tests/api/transforms/test_timestamps.py +++ /dev/null @@ -1,46 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -from iceberg.api.expressions import Literal -from iceberg.api.transforms import Transforms -from iceberg.api.types import TimestampType -import pytest - - -@pytest.mark.parametrize("lit,type_var", [ - (Literal.of("2017-12-01T10:12:55.038194-08:00"), TimestampType.with_timezone()), - (Literal.of("2017-12-01T18:12:55.038194"), TimestampType.without_timezone())]) -@pytest.mark.parametrize("transform_gran,expected", [ - (Transforms.year, "2017"), - (Transforms.month, "2017-12"), - (Transforms.day, "2017-12-01"), - (Transforms.hour, "2017-12-01-18")]) -def test_ts_to_human_string(lit, type_var, transform_gran, expected): - date_var = lit.to(type_var) - assert (transform_gran(type_var) - .to_human_string(transform_gran(type_var) - .apply(date_var.value))) == expected - - -@pytest.mark.parametrize("transform_gran", [ - Transforms.year, - Transforms.month, - Transforms.day, - Transforms.hour]) -def test_null_human_string(transform_gran): - type_var = TimestampType.with_timezone() - assert "null" == transform_gran(type_var).to_human_string(None) diff --git a/tests/api/transforms/test_truncate.py b/tests/api/transforms/test_truncate.py deleted file mode 100644 index 128d145726..0000000000 --- a/tests/api/transforms/test_truncate.py +++ /dev/null @@ -1,59 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -from decimal import Decimal - -from iceberg.api.transforms import Truncate -from iceberg.api.types import (DecimalType, - IntegerType, - LongType, - StringType) -import pytest - - -@pytest.mark.parametrize("type_var", [IntegerType.get(), LongType.get()]) -@pytest.mark.parametrize("input_var,expected", [ - (1, 0), - (5, 0), - (9, 0), - (10, 10), - (11, 10), - (-1, -10), - (-10, -10), - (-12, -20)]) -def test_truncate_integer(type_var, input_var, expected): - trunc = Truncate.get(type_var, 10) - assert trunc.apply(input_var) == expected - - -@pytest.mark.parametrize("input_var,expected", [ - (Decimal(12.34).quantize(Decimal(".01")), Decimal("12.30")), - (Decimal(12.30).quantize(Decimal(".01")), Decimal("12.30")), - (Decimal(12.20).quantize(Decimal(".01")), Decimal("12.20")), - (Decimal(0.05).quantize(Decimal(".01")), Decimal("0.00")), - (Decimal(-0.05).quantize(Decimal(".01")), Decimal("-0.10"))]) -def test_truncate_decimal(input_var, expected): - trunc = Truncate.get(DecimalType.of(9, 2), 10) - assert trunc.apply(input_var) == expected - - -@pytest.mark.parametrize("input_var,expected", [ - ("abcdefg", "abcde"), - ("abc", "abc")]) -def test_truncate_string(input_var, expected): - trunc = Truncate.get(StringType.get(), 5) - assert trunc.apply(input_var) == expected diff --git a/tests/api/types/__init__.py b/tests/api/types/__init__.py deleted file mode 100644 index 13a83393a9..0000000000 --- a/tests/api/types/__init__.py +++ /dev/null @@ -1,16 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. diff --git a/tests/api/types/test_binary_comparator.py b/tests/api/types/test_binary_comparator.py deleted file mode 100644 index ca0cf39fbe..0000000000 --- a/tests/api/types/test_binary_comparator.py +++ /dev/null @@ -1,41 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -from iceberg.api.expressions import Literal -from iceberg.api.types import FixedType - - -def test_binary_unsigned_comparator(): - b1 = bytearray([0x01, 0x01, 0x02]) - b2 = bytearray([0x01, 0xFF, 0x01]) - - assert Literal.of(b2) > Literal.of(b1) - - -def test_fixed_unsigned_comparator(): - b1 = bytearray([0x01, 0x01, 0x02]) - b2 = bytearray([0x01, 0xFF, 0x01]) - - assert Literal.of(b2) > Literal.of(b1).to(FixedType.of_length(3)) - - -def test_null_handling(): - b1 = bytearray([0x01]) - - assert None < Literal.of(b1) - assert Literal.of(b1) > None - assert Literal.of(b1).to(FixedType.of_length(3)) == Literal.of(b1).to(FixedType.of_length(4)) diff --git a/tests/api/types/test_char_seq_comparator.py b/tests/api/types/test_char_seq_comparator.py deleted file mode 100644 index 1f669fb4c3..0000000000 --- a/tests/api/types/test_char_seq_comparator.py +++ /dev/null @@ -1,43 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -from iceberg.api.expressions import (Literal, - StringLiteral) -import pytest - - -@pytest.mark.parametrize("input_vals", [ - (Literal.of("abc"), Literal.of(u'abc')), # unicode and str are same - (StringLiteral(None), StringLiteral(None)) # None literals are equal -]) -def test_special_equality(input_vals): - assert input_vals[0] == input_vals[1] - - -@pytest.mark.parametrize("input_vals", [ - (Literal.of("abc"), Literal.of('abcd')), # test_seq_length, longer is greater - (Literal.of('abcd'), Literal.of("adc")), # test_char_order, first difference takes precedence over length - (None, Literal.of('abc')) # test_null_handling, null comes before non-null -]) -@pytest.mark.parametrize("eval_func", [ - lambda x, y: y > x, - lambda x, y: x < y]) -def test_seq_length(input_vals, eval_func): - if input_vals[0] is not None: - assert eval_func(input_vals[0].value, input_vals[1].value) - - assert eval_func(input_vals[0], input_vals[1]) diff --git a/tests/api/types/test_comparable_comparator.py b/tests/api/types/test_comparable_comparator.py deleted file mode 100644 index 4d85a28f05..0000000000 --- a/tests/api/types/test_comparable_comparator.py +++ /dev/null @@ -1,51 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -from iceberg.api.expressions import (IntegerLiteral, - Literal) -import pytest - - -@pytest.mark.parametrize("larger,smaller", [ - (34, 33), - (-1, -2)]) -@pytest.mark.parametrize("op", [ - lambda x, y: x > y, - lambda y, x: x < y]) -def test_natural_order(larger, smaller, op): - assert op(Literal.of(larger), Literal.of(smaller)) - - -@pytest.mark.parametrize("input_val", [ - 1, - 0, - -1]) -def test_natural_order_eq(input_val): - assert Literal.of(input_val) == Literal.of(input_val) - - -@pytest.mark.parametrize("larger,smaller", [ - (34, None)]) -@pytest.mark.parametrize("op", [ - lambda x, y: x > y, - lambda y, x: x < y]) -def test_null_handling(larger, smaller, op): - assert op(IntegerLiteral(larger), IntegerLiteral(smaller)) - - -def test_null_handling_eq(): - assert IntegerLiteral(None) == IntegerLiteral(None) diff --git a/tests/api/types/test_readabilty_checks.py b/tests/api/types/test_readabilty_checks.py deleted file mode 100644 index 2acd62276c..0000000000 --- a/tests/api/types/test_readabilty_checks.py +++ /dev/null @@ -1,57 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -import unittest - -from iceberg.api.types import (BinaryType, - BooleanType, - DateType, - DecimalType, - DoubleType, - FixedType, - FloatType, - IntegerType, - LongType, - StringType, - TimestampType, - TimeType, - UUIDType) - -PRIMITIVES = [BinaryType.get(), - BooleanType.get(), - DateType.get(), - DecimalType.of(9, 2), - DecimalType.of(11, 2), - DecimalType.of(9, 3), - DoubleType.get(), - FixedType.of_length(3), - FixedType.of_length(4), - FloatType.get(), - IntegerType.get(), - LongType.get(), - StringType.get(), - TimestampType.with_timezone(), - TimestampType.without_timezone(), - TimeType.get(), - UUIDType.get()] - - -class TestReadabilityChecks(unittest.TestCase): - - def test_primitive_types(self): - # TO-DO: Need to implement CheckCompatibility in type_util - pass diff --git a/tests/api/types/test_type_util.py b/tests/api/types/test_type_util.py deleted file mode 100644 index 58036036ad..0000000000 --- a/tests/api/types/test_type_util.py +++ /dev/null @@ -1,49 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -from iceberg.api.types import (BooleanType, NestedField, StructType) -from iceberg.api.types import type_util -from iceberg.exceptions import ValidationException -import pytest - - -def test_invalid_schema_via_index_by_name(): - bool_type1 = NestedField.required(1, "a", BooleanType.get()) - bool_type2 = NestedField.required(2, "a", BooleanType.get()) - - with pytest.raises(ValidationException) as context: - type_util.index_by_name(StructType.of([bool_type1, bool_type2])) - assert str(context.value) == 'Invalid schema: multiple fields for name a: 1 and 2' - - -def test_valid_schema_via_index_by_name(): - bool_type1 = NestedField.required(1, "a", BooleanType.get()) - bool_type2 = NestedField.required(2, "b", BooleanType.get()) - - assert {'a': 1, 'b': 2} == type_util.index_by_name(StructType.of([bool_type1, bool_type2])) - - -def test_validate_schema_via_index_by_name_for_nested_type(): - nested_type = NestedField.required( - 1, "a", StructType.of( - [NestedField.required(2, "b", StructType.of( - [NestedField.required(3, "c", BooleanType.get())])), - NestedField.required(4, "b.c", BooleanType.get())])) - - with pytest.raises(ValidationException) as context: - type_util.index_by_name(StructType.of([nested_type])) - assert str(context.value) == 'Invalid schema: multiple fields for name a.b.c: 3 and 4' diff --git a/tests/core/__init__.py b/tests/core/__init__.py deleted file mode 100644 index 13a83393a9..0000000000 --- a/tests/core/__init__.py +++ /dev/null @@ -1,16 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. diff --git a/tests/core/avro/__init__.py b/tests/core/avro/__init__.py deleted file mode 100644 index 13a83393a9..0000000000 --- a/tests/core/avro/__init__.py +++ /dev/null @@ -1,16 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. diff --git a/tests/core/avro/conftest.py b/tests/core/avro/conftest.py deleted file mode 100644 index de595babbb..0000000000 --- a/tests/core/avro/conftest.py +++ /dev/null @@ -1,64 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -from tempfile import NamedTemporaryFile - -from iceberg.api import Schema -from iceberg.api.types import (BinaryType, - BooleanType, - DateType, - DecimalType, - DoubleType, - FixedType, - FloatType, - IntegerType, - LongType, - NestedField, - StringType, - StructType, - TimestampType, - UUIDType) -import pytest - - -@pytest.fixture(scope="session") -def supported_primitives(): - return StructType.of([NestedField.required(100, "id", LongType.get()), - NestedField.optional(101, "data", StringType.get()), - NestedField.required(102, "b", BooleanType.get()), - NestedField.optional(103, "i", IntegerType.get()), - NestedField.required(104, "l", LongType.get()), - NestedField.optional(105, "f", FloatType.get()), - NestedField.required(106, "d", DoubleType.get()), - NestedField.optional(107, "date", DateType.get()), - NestedField.required(108, "ts", TimestampType.with_timezone()), - NestedField.required(110, "s", StringType.get()), - NestedField.required(111, "uuid", UUIDType.get()), - NestedField.required(112, "fixed", FixedType.of_length(7)), - NestedField.optional(113, "bytes", BinaryType.get()), - NestedField.required(114, "dec_9_0", DecimalType.of(9, 0)), - NestedField.required(114, "dec_11_2", DecimalType.of(11, 2)), - NestedField.required(114, "dec_38_10", DecimalType.of(38, 10))]) - - -@pytest.fixture(scope="session") -def iceberg_full_read_projection_schema(): - return Schema([NestedField.required(0, "id", LongType.get()), - NestedField.optional(1, "data", StringType.get())]) - - -def write_and_read(desc, write_schema, read_schema, record): - with NamedTemporaryFile(delete=True, mode='wb') as temp_file: - return temp_file diff --git a/tests/core/avro/test_avro.py b/tests/core/avro/test_avro.py deleted file mode 100644 index 5415e9fcdd..0000000000 --- a/tests/core/avro/test_avro.py +++ /dev/null @@ -1,22 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import unittest - - -class TestAvro(unittest.TestCase): - - def test_read_avro(self): - pass diff --git a/tests/core/avro/test_read_projection.py b/tests/core/avro/test_read_projection.py deleted file mode 100644 index 9cc35acd28..0000000000 --- a/tests/core/avro/test_read_projection.py +++ /dev/null @@ -1,27 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -# from .conftest import write_and_read -from iceberg.core import ManifestEntry, PartitionSpecParser, SchemaParser -from iceberg.core.avro import IcebergToAvro - - -def test_full_projection(iceberg_full_read_projection_schema): - schema = SchemaParser.from_json({'type': 'struct', 'fields': [{'id': 1, 'name': 'account_id', 'required': False, 'type': 'long', 'doc': 'Lookup table: account_d'}, {'id': 2, 'name': 'subscrn_id', 'required': False, 'type': 'long'}, {'id': 3, 'name': 'is_in_free_trial', 'required': False, 'type': 'int'}, {'id': 4, 'name': 'subscrn_service_days', 'required': False, 'type': 'int'}, {'id': 5, 'name': 'subscrn_period_nbr', 'required': False, 'type': 'int'}, {'id': 6, 'name': 'account_service_days', 'required': False, 'type': 'int', 'doc': 'number of days subscriber has had service over all subscriptions'}, {'id': 7, 'name': 'account_period_nbr', 'required': False, 'type': 'int'}, {'id': 8, 'name': 'current_plan_id', 'required': False, 'type': 'int', 'doc': 'Lookup table: plan_d'}, {'id': 10, 'name': 'country_iso_code', 'required': False, 'type': 'string', 'doc': 'Registration country id. Lookup table: geo_country_d'}, {'id': 11, 'name': 'is_onhold', 'required': False, 'type': 'int', 'doc': 'Subscriptions can be put on hold due to suspected fraud or failure to pay. subscriber is no longer able to view and is no longer considered a member while on hold'}, {'id': 12, 'name': 'same_day_hold', 'required': False, 'type': 'int', 'doc': 'When a member goes on hold and off hold in the same day'}, {'id': 13, 'name': 'dateint', 'required': False, 'type': 'int', 'doc': 'Will be deprecated'}, {'id': 14, 'name': 'plan_rollup_id', 'required': False, 'type': 'int', 'doc': 'Lookup table: plan_rollup_d'}, {'id': 15, 'name': 'price_tier_code', 'required': False, 'type': 'string', 'doc': 'Lookup table: price_tier_d. You can also use plan_price_d to lookup the combination of (plan_rollup_id, price_tier_code, country_iso_code)'}, {'id': 16, 'name': 'latest_plan_change_date', 'required': False, 'type': 'int', 'doc': 'Members can upgrade or downgrade plans'}, {'id': 17, 'name': 'is_in_product_grace_period', 'required': False, 'type': 'int', 'doc': 'Members on grace period can continue to view, but they are not considered active members since they are not in good payment standing.'}, {'id': 18, 'name': 'is_in_member_cnt', 'required': False, 'type': 'int', 'doc': 'Total # of entitlements. An entitlement is the right to stream Netflix and excludes those on hold or in grace periods and can include both paid and free subs.'}, {'id': 19, 'name': 'is_grace_period_to_member_cnt', 'required': False, 'type': 'int', 'doc': 'Members that were on grace period before, but cleared out and become in good standing'}, {'id': 20, 'name': 'is_grace_period_to_on_hold', 'required': False, 'type': 'int', 'doc': 'Members that were on grace period (usually for 7 days), but they failed to provide a valid payment method so they placed on hold'}, {'id': 21, 'name': 'is_from_gp_start', 'required': False, 'type': 'int', 'doc': 'Grace period start date'}, {'id': 22, 'name': 'is_onhold_without_gp_end', 'required': False, 'type': 'int'}, {'id': 23, 'name': 'subscription_type', 'required': False, 'type': 'string', 'doc': 'Added with NIO project. Values: P (paying only), S (streaming only), PS (paying and streaming)'}, {'id': 24, 'name': 'connected_to_account_id', 'required': False, 'type': 'long', 'doc': 'Added with NIO project. This will only be populated when the paying accounts have logins. Possible values: subscription_type = P, connected_to_account_id = value or null, subscription_type = S, connected_to_account_id = null, subscription_type = PS, connected_to_account_id = null'}, {'id': 25, 'name': 'can_stream', 'required': False, 'type': 'int', 'doc': 'Added with NIO project. Ability to stream whether a member is in good standing or on grace period. Possible values: subscription_type = P, can_stream = 0, subscription_type = S, can_stream = 1, subscription_type = PS, can_stream = 0 or 1 based hold and grace period'}, {'id': 26, 'name': 'is_untethered_account', 'required': False, 'type': 'int', 'doc': 'Added with NIO project. This column is used to identify paying accounts that are not associated with logins. Logic used to populate it: (subscription_type = P, can_stream=0, is_in_member_cnt =1, connected_to_account_id = null) Possible values: subscription_type = P, is_untethered_account = 0 or 1, subscription_type = S, is_untethered_account = 0, subscription_type = PS, is_untethered_account = 0'}, {'id': 27, 'name': 'is_billing_paused', 'required': False, 'type': 'int', 'doc': 'Added with NIO project. Possible values: subscription_type = P, is_billing_paused = 0, subscription_type = S, is_billing_paused = 0 or 1, subscription_type = PS, is_billing_paused = 0.'}, {'id': 28, 'name': 'is_in_customer_count', 'required': False, 'type': 'int'}, {'id': 29, 'name': 'paid_category', 'required': False, 'type': 'string'}, {'id': 30, 'name': 'is_in_paid_member_cnt', 'required': False, 'type': 'int'}, {'id': 31, 'name': 'scale', 'required': False, 'type': 'string'}, {'id': 32, 'name': 'test_new_col', 'required': False, 'type': 'string', 'doc': 'new_column docs'}, {'id': 33, 'name': 'test_new_col_2', 'required': False, 'type': 'string', 'doc': 'new_column docs 2'}]}) - spec = PartitionSpecParser.from_json_fields(schema, 0, [{"name": "scale", - "transform": "identity", - "source-id": 31}]) - proj_schema = ManifestEntry.project_schema(spec.partition_type(), ["*"]) - print(IcebergToAvro.type_to_schema(proj_schema.as_struct(), "manifest_entry")) diff --git a/tests/core/conftest.py b/tests/core/conftest.py deleted file mode 100644 index 933743e764..0000000000 --- a/tests/core/conftest.py +++ /dev/null @@ -1,314 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -import os -import random -import tempfile -import time - -from iceberg.api import Files, PartitionSpec, PartitionSpecBuilder, Schema -from iceberg.api.types import BooleanType, IntegerType, LongType, NestedField, StringType -from iceberg.core import (BaseSnapshot, - BaseTable, - ConfigProperties, - GenericManifestFile, - SnapshotLogEntry, - TableMetadata, - TableMetadataParser, - TableOperations, - TableProperties) -from iceberg.exceptions import AlreadyExistsException, CommitFailedException -import pytest - -SCHEMA = Schema([NestedField.optional(1, "b", BooleanType.get())]) -METADATA = dict() -VERSIONS = dict() - - -class LocalTableOperations(TableOperations): - - def current(self): - raise RuntimeError("Not implemented for tests") - - def refresh(self): - raise RuntimeError("Not implemented for tests") - - def commit(self, base, metadata): - raise RuntimeError("Not implemented for tests") - - def new_input_file(self, path): - return Files.local_input(path) - - def new_metadata_file(self, filename): - return Files.local_output(tempfile.mkstemp(prefix=filename)) - - def delete_file(self, path): - if os.path.exists(path): - os.remove(path) - - def new_snapshot_id(self): - raise RuntimeError("Not implemented for tests") - - -def create(temp, name, schema, spec): - ops = TestTableOperations(name, temp) - if ops.current() is not None: - raise AlreadyExistsException("Table %s already exists at location: %s" % (name, temp)) - ops.commit(None, TableMetadata.new_table_metadata(ops, schema, spec, str(temp))) - return TestTable(ops, name) - - -def begin_create(temp, name, schema, spec): - raise RuntimeError("Not yet implemented") - # ops = TestTableOperations(name, temp) - # if ops.current() is None: - # raise AlreadyExistsException("Table %s already exists at location: %s" % (name, temp)) - # - # metadata = TableMetadata.new_table_metadata(ops, schema, spec, str(temp)) - # return BaseTransaction.create_table_transaction(ops, metadata) - - -class TestTable(BaseTable): - def __init__(self, ops, name): - super(TestTable, self).__init__(ops, name) - self.ops = ops - - -class TestTableOperations(TableOperations): - - def __init__(self, table_name, location): - self.last_snapshot_id = 0 - self._fail_commits = 0 - self.table_name = table_name - self.metadata = os.path.join(location, "metadata") - os.makedirs(self.metadata) - self._current = None - self.refresh() - if self._current is not None: - for snap in self.current().snapshots: - self.last_snapshot_id = max(self.last_snapshot_id, snap.snapshot_id) - - def current(self): - return self._current - - def refresh(self): - self._current = METADATA.get(self.table_name) - return self._current - - def commit(self, base, metadata): - if base != self.current(): - raise RuntimeError("Cannot commit changes based on stale metadata") - - self.refresh() - if base == self.current(): - if self._fail_commits > 0: - self._fail_commits - 1 - raise RuntimeError("Injected failure") - version = VERSIONS.get(self.table_name) - VERSIONS[self.table_name] = 0 if version is None else version + 1 - METADATA[self.table_name] = metadata - self._current = metadata - else: - raise CommitFailedException("Commit failed: table was updated at %s", self.current.last_updated_millis) - - def new_input_file(self, path): - return Files.local_input(path) - - def new_metadata_file(self, filename): - return Files.local_output(os.path.join(self.metadata, filename)) - - def delete_file(self, path): - if not os.remove(path): - raise RuntimeError("Failed to delete file: %s" % path) - - def new_snapshot_id(self): - next_snapshot_id = self.last_snapshot_id + 1 - self.last_snapshot_id = next_snapshot_id - return next_snapshot_id - - -class TestTables(object): - @staticmethod - def create(temp, name, schema, spec): - ops = TestTableOperations(name, temp) - - if ops.current() is not None: - raise RuntimeError("Table %s already exists at location: %s" % (name, temp)) - - ops.commit(None, TableMetadata.new_table_metadata(ops, schema, spec, str(temp))) - return TestTable(ops, name) - - -@pytest.fixture(scope="session") -def expected(): - return TableMetadata.new_table_metadata(None, SCHEMA, PartitionSpec.unpartitioned(), "file://tmp/db/table") - - -@pytest.fixture(scope="session", - params=[True, False]) -def prop(request): - config = {ConfigProperties.COMPRESS_METADATA: request.param} - yield request.param - - if os.path.exists(TableMetadataParser.get_file_extension(config)): - os.remove(TableMetadataParser.get_file_extension(config)) - - -@pytest.fixture(scope="session") -def ops(): - return LocalTableOperations() - - -@pytest.fixture(scope="session") -def expected_metadata(): - spec_schema = Schema(NestedField.required(1, "x", LongType.get()), - NestedField.required(2, "y", LongType.get()), - NestedField.required(3, "z", LongType.get())) - spec = PartitionSpec \ - .builder_for(spec_schema) \ - .with_spec_id(5) \ - .build() - - random.seed(1234) - previous_snapshot_id = int(time.time()) - random.randint(0, 3600) - - previous_snapshot = BaseSnapshot(None, previous_snapshot_id, None, - timestamp_millis=previous_snapshot_id, - manifests=[GenericManifestFile(file=Files.local_input("file:/tmp/manfiest.1.avro"), - spec_id=spec.spec_id)]) - - current_snapshot_id = int(time.time()) - current_snapshot = BaseSnapshot(None, current_snapshot_id, previous_snapshot_id, - timestamp_millis=current_snapshot_id, - manifests=[GenericManifestFile(file=Files.local_input("file:/tmp/manfiest.2.avro"), - spec_id=spec.spec_id)]) - - snapshot_log = [SnapshotLogEntry(previous_snapshot.timestamp_millis, previous_snapshot.snapshot_id), - SnapshotLogEntry(current_snapshot.timestamp_millis, current_snapshot.snapshot_id)] - - return TableMetadata(ops, None, "s3://bucket/test/location", - int(time.time()), 3, spec_schema, 5, [spec], {"property": "value"}, current_snapshot_id, - [previous_snapshot, current_snapshot], snapshot_log) - - -@pytest.fixture(scope="session") -def expected_metadata_sorting(): - spec_schema = Schema(NestedField.required(1, "x", LongType.get()), - NestedField.required(2, "y", LongType.get()), - NestedField.required(3, "z", LongType.get())) - - spec = PartitionSpec \ - .builder_for(spec_schema) \ - .with_spec_id(5) \ - .build() - - random.seed(1234) - previous_snapshot_id = int(time.time()) - random.randint(0, 3600) - - previous_snapshot = BaseSnapshot(ops, previous_snapshot_id, None, - timestamp_millis=previous_snapshot_id, - manifests=[GenericManifestFile(file=Files.local_input("file:/tmp/manfiest.1.avro"), - spec_id=spec.spec_id)]) - - current_snapshot_id = int(time.time()) - current_snapshot = BaseSnapshot(ops, current_snapshot_id, previous_snapshot_id, - timestamp_millis=current_snapshot_id, - manifests=[GenericManifestFile(file=Files.local_input("file:/tmp/manfiest.2.avro"), - spec_id=spec.spec_id)]) - - reversed_snapshot_log = list() - metadata = TableMetadata(ops, None, "s3://bucket/test/location", - int(time.time()), 3, spec_schema, 5, [spec], {"property": "value"}, current_snapshot_id, - [previous_snapshot, current_snapshot], reversed_snapshot_log) - - reversed_snapshot_log.append(SnapshotLogEntry(current_snapshot.timestamp_millis, current_snapshot.snapshot_id)) - reversed_snapshot_log.append(SnapshotLogEntry(previous_snapshot.timestamp_millis, previous_snapshot.snapshot_id)) - - return metadata - - -@pytest.fixture(scope="session") -def missing_spec_list(): - schema = Schema(NestedField.required(1, "x", LongType.get()), - NestedField.required(2, "y", LongType.get()), - NestedField.required(3, "z", LongType.get())) - - spec = PartitionSpec.builder_for(schema).identity("x").with_spec_id(6).build() - random.seed(1234) - previous_snapshot_id = int(time.time()) - random.randint(0, 3600) - - previous_snapshot = BaseSnapshot(ops, previous_snapshot_id, None, - timestamp_millis=previous_snapshot_id, - manifests=[GenericManifestFile(file=Files.local_input("file:/tmp/manfiest.1.avro"), - spec_id=spec.spec_id)]) - - current_snapshot_id = int(time.time()) - current_snapshot = BaseSnapshot(ops, current_snapshot_id, previous_snapshot_id, - timestamp_millis=current_snapshot_id, - manifests=[GenericManifestFile(file=Files.local_input("file:/tmp/manfiest.2.avro"), - spec_id=spec.spec_id)]) - return TableMetadata(ops, None, "s3://bucket/test/location", int(time.time()), 3, schema, 6, - (spec,), {"property": "value"}, current_snapshot_id, [previous_snapshot, current_snapshot], - []) - - -@pytest.fixture(scope="session") -def snapshot_manifests(): - return (GenericManifestFile(file=Files.local_input("file:/tmp/manifest1.avro"), spec_id=0), - GenericManifestFile(file=Files.local_input("file:/tmp/manifest2.avro"), spec_id=0)) - - -@pytest.fixture(scope="session") -def expected_base_snapshot(): - return BaseSnapshot(LocalTableOperations(), int(time.time()), manifests=["file:/tmp/manfiest.1.avro", - "file:/tmp/manfiest.2.avro"]) - - -@pytest.fixture(scope="session") -def base_scan_schema(): - return Schema([NestedField.required(1, "id", IntegerType.get()), - NestedField.required(2, "data", StringType.get())]) - - -@pytest.fixture(scope="session", params=["none", "one"]) -def base_scan_partition(base_scan_schema, request): - if request.param == "none": - spec = PartitionSpec.unpartitioned() - else: - spec = PartitionSpecBuilder(base_scan_schema).add(1, 1000, "id", "identity").build() - return spec - - -@pytest.fixture(scope="session") -def ts_table(base_scan_schema, base_scan_partition): - with tempfile.TemporaryDirectory() as td: - return TestTables.create(td, "test-" + str(len(base_scan_partition.fields)), base_scan_schema, - base_scan_partition) - - -@pytest.fixture(scope="session") -def split_planning_table(base_scan_schema): - from iceberg.core.filesystem import FilesystemTables - tables = FilesystemTables() - - with tempfile.TemporaryDirectory() as td: - table = tables.create(base_scan_schema, table_identifier=td) - table.properties().update({TableProperties.SPLIT_SIZE: "{}".format(128 * 1024 * 1024), - TableProperties.SPLIT_OPEN_FILE_COST: "{}".format(4 * 1024 * 1024), - TableProperties.SPLIT_LOOKBACK: "{}".format(2 ** 31 - 1)}) - - return table diff --git a/tests/core/test_base_table_scan.py b/tests/core/test_base_table_scan.py deleted file mode 100644 index fb8d2479fc..0000000000 --- a/tests/core/test_base_table_scan.py +++ /dev/null @@ -1,38 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -from iceberg.api import Schema -from iceberg.api.types import IntegerType, NestedField - - -def test_table_scan_honors_select(ts_table): - scan = ts_table.new_scan().select(["id"]) - - expected_schema = Schema([NestedField.required(1, "id", IntegerType.get())]) - - assert scan.schema.as_struct() == expected_schema.as_struct() - - -def test_table_scan_honors_select_without_case_sensitivity(ts_table): - scan1 = ts_table.new_scan().case_sensitive(False).select(["ID"]) - # order of refinements shouldn't matter - scan2 = ts_table.new_scan().select(["ID"]).case_sensitive(False) - - expected_schema = Schema([NestedField.required(1, "id", IntegerType.get())]) - - assert scan1.schema.as_struct() == expected_schema.as_struct() - assert scan2.schema.as_struct() == expected_schema.as_struct() diff --git a/tests/core/test_filesystem_tables.py b/tests/core/test_filesystem_tables.py deleted file mode 100644 index 5f4d6e386d..0000000000 --- a/tests/core/test_filesystem_tables.py +++ /dev/null @@ -1,32 +0,0 @@ -# -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. -# -import os - -from iceberg.core.filesystem import FilesystemTables - - -def test_create_tables(base_scan_schema, base_scan_partition, tmpdir): - - conf = {"hive.metastore.uris": 'thrift://hms:port', - "hive.metastore.warehouse.dir": tmpdir} - tables = FilesystemTables(conf) - table_location = os.path.join(str(tmpdir), "test", "test_123") - tables.create(base_scan_schema, table_location, base_scan_partition) - - tables.load(table_location) diff --git a/tests/core/test_partition_spec.py b/tests/core/test_partition_spec.py deleted file mode 100644 index f0f3b9b01a..0000000000 --- a/tests/core/test_partition_spec.py +++ /dev/null @@ -1,111 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -from iceberg.api import PartitionSpec, Schema -from iceberg.api.types import (BinaryType, - DateType, - DecimalType, - FixedType, - IntegerType, - LongType, - NestedField, - StringType, - TimestampType, - TimeType, - UUIDType) - - -def test_to_json_conversion(): - spec_schema = Schema(NestedField.required(1, "i", IntegerType.get()), - NestedField.required(2, "l", LongType.get()), - NestedField.required(3, "d", DateType.get()), - NestedField.required(4, "t", TimeType.get()), - NestedField.required(5, "ts", TimestampType.without_timezone()), - NestedField.required(6, "dec", DecimalType.of(9, 2)), - NestedField.required(7, "s", StringType.get()), - NestedField.required(8, "u", UUIDType.get()), - NestedField.required(9, "f", FixedType.of_length(3)), - NestedField.required(10, "b", BinaryType.get())) - - specs = [ - PartitionSpec.builder_for(spec_schema).identity("i").build(), - PartitionSpec.builder_for(spec_schema).identity("l").build(), - PartitionSpec.builder_for(spec_schema).identity("d").build(), - PartitionSpec.builder_for(spec_schema).identity("t").build(), - PartitionSpec.builder_for(spec_schema).identity("ts").build(), - PartitionSpec.builder_for(spec_schema).identity("dec").build(), - PartitionSpec.builder_for(spec_schema).identity("s").build(), - PartitionSpec.builder_for(spec_schema).identity("u").build(), - PartitionSpec.builder_for(spec_schema).identity("f").build(), - PartitionSpec.builder_for(spec_schema).identity("b").build(), - PartitionSpec.builder_for(spec_schema).bucket("i", 128).build(), - PartitionSpec.builder_for(spec_schema).bucket("l", 128).build(), - PartitionSpec.builder_for(spec_schema).bucket("d", 128).build(), - PartitionSpec.builder_for(spec_schema).bucket("t", 128).build(), - PartitionSpec.builder_for(spec_schema).bucket("ts", 128).build(), - PartitionSpec.builder_for(spec_schema).bucket("dec", 128).build(), - PartitionSpec.builder_for(spec_schema).bucket("s", 128).build(), - PartitionSpec.builder_for(spec_schema).year("d").build(), - PartitionSpec.builder_for(spec_schema).month("d").build(), - PartitionSpec.builder_for(spec_schema).day("d").build(), - PartitionSpec.builder_for(spec_schema).year("ts").build(), - PartitionSpec.builder_for(spec_schema).month("ts").build(), - PartitionSpec.builder_for(spec_schema).day("ts").build(), - PartitionSpec.builder_for(spec_schema).hour("ts").build(), - PartitionSpec.builder_for(spec_schema).truncate("i", 10).build(), - PartitionSpec.builder_for(spec_schema).truncate("l", 10).build(), - PartitionSpec.builder_for(spec_schema).truncate("dec", 10).build(), - PartitionSpec.builder_for(spec_schema).truncate("s", 10).build(), - PartitionSpec.builder_for(spec_schema).add_without_field_id(6, "dec_bucket", "bucket[16]").build(), - PartitionSpec.builder_for(spec_schema).add(6, 1011, "dec_bucket", "bucket[16]").build(), - ] - - expected_spec_strs = [ - "[\n 1000: i: identity(1)\n]", - "[\n 1000: l: identity(2)\n]", - "[\n 1000: d: identity(3)\n]", - "[\n 1000: t: identity(4)\n]", - "[\n 1000: ts: identity(5)\n]", - "[\n 1000: dec: identity(6)\n]", - "[\n 1000: s: identity(7)\n]", - "[\n 1000: u: identity(8)\n]", - "[\n 1000: f: identity(9)\n]", - "[\n 1000: b: identity(10)\n]", - "[\n 1000: i_bucket: bucket[128](1)\n]", - "[\n 1000: l_bucket: bucket[128](2)\n]", - "[\n 1000: d_bucket: bucket[128](3)\n]", - "[\n 1000: t_bucket: bucket[128](4)\n]", - "[\n 1000: ts_bucket: bucket[128](5)\n]", - "[\n 1000: dec_bucket: bucket[128](6)\n]", - "[\n 1000: s_bucket: bucket[128](7)\n]", - "[\n 1000: d_year: year(3)\n]", - "[\n 1000: d_month: month(3)\n]", - "[\n 1000: d_day: day(3)\n]", - "[\n 1000: ts_year: year(5)\n]", - "[\n 1000: ts_month: month(5)\n]", - "[\n 1000: ts_day: day(5)\n]", - "[\n 1000: ts_hour: hour(5)\n]", - "[\n 1000: i_truncate: truncate[10](1)\n]", - "[\n 1000: l_truncate: truncate[10](2)\n]", - "[\n 1000: dec_truncate: truncate[10](6)\n]", - "[\n 1000: s_truncate: truncate[10](7)\n]", - "[\n 1000: dec_bucket: bucket[16](6)\n]", - "[\n 1011: dec_bucket: bucket[16](6)\n]", - ] - - for (spec, expected_spec_str) in zip(specs, expected_spec_strs): - assert str(spec) == expected_spec_str diff --git a/tests/core/test_partition_spec_parser.py b/tests/core/test_partition_spec_parser.py deleted file mode 100644 index 256186cb84..0000000000 --- a/tests/core/test_partition_spec_parser.py +++ /dev/null @@ -1,110 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -from iceberg.api import PartitionSpec, Schema -from iceberg.api.types import DecimalType, IntegerType, NestedField, StringType -from iceberg.core import PartitionSpecParser -import pytest - - -def test_to_json_conversion(): - spec_schema = Schema(NestedField.required(1, "id", IntegerType.get()), - NestedField.required(2, "data", StringType.get()), - NestedField.required(3, "num", DecimalType.of(9, 2))) - - spec = PartitionSpec \ - .builder_for(spec_schema) \ - .identity("id") \ - .bucket("data", 16) \ - .add_without_field_id(2, "data1", "bucket[16]") \ - .add(2, 1010, "data2", "bucket[8]") \ - .bucket("num", 8) \ - .build() - - expected = '{"spec-id": 0, "fields": [' \ - '{"name": "id", "transform": "identity", "source-id": 1, "field-id": 1000}, ' \ - '{"name": "data_bucket", "transform": "bucket[16]", "source-id": 2, "field-id": 1001}, ' \ - '{"name": "data1", "transform": "bucket[16]", "source-id": 2, "field-id": 1002}, ' \ - '{"name": "data2", "transform": "bucket[8]", "source-id": 2, "field-id": 1010}, ' \ - '{"name": "num_bucket", "transform": "bucket[8]", "source-id": 3, "field-id": 1011}]}' - assert expected == PartitionSpecParser.to_json(spec) - - -def test_from_json_conversion_with_field_ids(): - spec_schema = Schema(NestedField.required(1, "id", IntegerType.get()), - NestedField.required(2, "data", StringType.get()), - NestedField.required(3, "num", DecimalType.of(9, 2))) - - spec_string = '{"spec-id": 0, "fields": [' \ - '{"name": "id", "transform": "identity", "source-id": 1, "field-id": 1000}, ' \ - '{"name": "data_bucket", "transform": "bucket[16]", "source-id": 2, "field-id": 1001}, ' \ - '{"name": "data1", "transform": "bucket[16]", "source-id": 2, "field-id": 1002}, ' \ - '{"name": "data2", "transform": "bucket[8]", "source-id": 2, "field-id": 1010}, ' \ - '{"name": "num_bucket", "transform": "bucket[8]", "source-id": 3, "field-id": 1011}]}' - - spec = PartitionSpecParser.from_json(spec_schema, spec_string) - - expected_spec = PartitionSpec \ - .builder_for(spec_schema) \ - .identity("id") \ - .bucket("data", 16) \ - .add_without_field_id(2, "data1", "bucket[16]") \ - .add(2, 1010, "data2", "bucket[8]") \ - .bucket("num", 8) \ - .build() - assert expected_spec == spec - - -def test_from_json_conversion_without_field_ids(): - spec_schema = Schema(NestedField.required(1, "id", IntegerType.get()), - NestedField.required(2, "data", StringType.get()), - NestedField.required(3, "num", DecimalType.of(9, 2))) - - spec_string = '{"spec-id": 0, "fields": [' \ - '{"name": "id", "transform": "identity", "source-id": 1}, ' \ - '{"name": "data_bucket", "transform": "bucket[16]", "source-id": 2}, ' \ - '{"name": "data1", "transform": "bucket[16]", "source-id": 2}, ' \ - '{"name": "data2", "transform": "bucket[8]", "source-id": 2}, ' \ - '{"name": "num_bucket", "transform": "bucket[8]", "source-id": 3}]}' - - spec = PartitionSpecParser.from_json(spec_schema, spec_string) - - expected_spec = PartitionSpec \ - .builder_for(spec_schema) \ - .identity("id") \ - .bucket("data", 16) \ - .add_without_field_id(2, "data1", "bucket[16]") \ - .add(2, 1003, "data2", "bucket[8]") \ - .bucket("num", 8) \ - .build() - assert expected_spec == spec - - -def test_raise_exception_with_invalid_json(): - spec_schema = Schema(NestedField.required(1, "id", IntegerType.get()), - NestedField.required(2, "data", StringType.get()), - NestedField.required(3, "num", DecimalType.of(9, 2))) - - spec_string = '{"spec-id": 0, "fields": [' \ - '{"name": "id", "transform": "identity", "source-id": 1, "field-id": 1000}, ' \ - '{"name": "data_bucket", "transform": "bucket[16]", "source-id": 2, "field-id": 1001}, ' \ - '{"name": "data1", "transform": "bucket[16]", "source-id": 2}, ' \ - '{"name": "data2", "transform": "bucket[8]", "source-id": 2}, ' \ - '{"name": "num_bucket", "transform": "bucket[8]", "source-id": 3}]}' - - with pytest.raises(RuntimeError): - PartitionSpecParser.from_json(spec_schema, spec_string) diff --git a/tests/core/test_snapshot_json.py b/tests/core/test_snapshot_json.py deleted file mode 100644 index 733f77f693..0000000000 --- a/tests/core/test_snapshot_json.py +++ /dev/null @@ -1,58 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -from iceberg.api import DataOperations -from iceberg.core import BaseSnapshot, SnapshotParser - - -def test_json_conversion(ops, expected_base_snapshot): - as_json = SnapshotParser.to_json(expected_base_snapshot) - snapshot = SnapshotParser.from_json(ops, as_json) - - assert expected_base_snapshot.snapshot_id == snapshot.snapshot_id - assert expected_base_snapshot.manifests == snapshot.manifests - - -def test_json_conversion_with_operation(ops, snapshot_manifests): - parent_id = 1 - id = 2 - expected = BaseSnapshot(ops=ops, snapshot_id=id, parent_id=parent_id, - manifests=snapshot_manifests, operation=DataOperations.REPLACE, - summary={"files-added": 4, - "files-deleted": "100"}) - - json_obj = SnapshotParser.to_json(expected) - snapshot = SnapshotParser.from_json(ops, json_obj) - - assert expected.snapshot_id == snapshot.snapshot_id - assert expected.timestamp_millis == snapshot.timestamp_millis - assert expected.parent_id == snapshot.parent_id - assert expected.manifest_location == snapshot.manifest_location - assert expected.manifests == snapshot.manifests - assert expected.operation == snapshot.operation - assert expected.summary == snapshot.summary - -# def test_conversion_with_manifest_list(snapshot_manifests): -# parent_id = 1 -# id = 2 -# -# with NamedTemporaryFile() as manifest_list: -# with ManifestListWriter(Files.local_output(manifest_list), id, parent_id) as writer: -# writer.add_all(snapshot_manifests) -# -# with open(manifest_list) as fo: -# print(fo.read()) diff --git a/tests/core/test_table_metadata_json.py b/tests/core/test_table_metadata_json.py deleted file mode 100644 index c1c0d7cdff..0000000000 --- a/tests/core/test_table_metadata_json.py +++ /dev/null @@ -1,111 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -import json - -from iceberg.core import (PartitionSpecParser, - SchemaParser, - SnapshotLogEntry, - SnapshotParser, - TableMetadata, - TableMetadataParser) - - -def test_json_conversion(ops, expected_metadata): - current_snapshot_id = expected_metadata.current_snapshot_id - previous_snapshot_id = expected_metadata.current_snapshot().parent_id - previous_snapshot = expected_metadata.snapshot(previous_snapshot_id) - - as_json = TableMetadataParser.to_json(expected_metadata) - metadata = TableMetadataParser.from_json(ops, None, as_json) - - assert metadata.location == expected_metadata.location - assert metadata.last_column_id == expected_metadata.last_column_id - assert metadata.schema.as_struct() == expected_metadata.schema.as_struct() - assert str(metadata.spec) == str(expected_metadata.spec) - assert metadata.default_spec_id == expected_metadata.default_spec_id - assert metadata.specs == expected_metadata.specs - assert metadata.properties == expected_metadata.properties - assert metadata.snapshot_log == expected_metadata.snapshot_log - assert metadata.current_snapshot().snapshot_id == current_snapshot_id - assert metadata.current_snapshot().parent_id == previous_snapshot_id - assert metadata.current_snapshot().manifests == expected_metadata.current_snapshot().manifests - assert metadata.snapshot(previous_snapshot_id).snapshot_id == previous_snapshot_id - assert metadata.snapshot(previous_snapshot_id).manifests == previous_snapshot.manifests - - -def test_from_json_sorts_snapshot_log(ops, expected_metadata_sorting): - current_snapshot = expected_metadata_sorting.current_snapshot() - previous_snapshot_id = expected_metadata_sorting.current_snapshot().parent_id - previous_snapshot = expected_metadata_sorting.snapshot(previous_snapshot_id) - - expected_log = [SnapshotLogEntry(previous_snapshot.timestamp_millis, previous_snapshot.snapshot_id), - SnapshotLogEntry(current_snapshot.timestamp_millis, current_snapshot.snapshot_id)] - - as_json = TableMetadataParser.to_json(expected_metadata_sorting) - metadata = TableMetadataParser.from_json(ops, None, as_json) - - assert expected_log == metadata.snapshot_log - - -def test_backward_compatibility_missing_part_spec_list(ops, missing_spec_list): - as_json = to_json_without_spec_list(missing_spec_list) - metadata = TableMetadataParser.from_json(ops, None, as_json) - assert metadata.location == missing_spec_list.location - assert metadata.last_column_id == missing_spec_list.last_column_id - assert metadata.schema.as_struct() == missing_spec_list.schema.as_struct() - assert str(metadata.spec) == str(missing_spec_list.spec) - """ - Assert.assertEquals("Partition spec should be the default", - expected.spec().toString(), metadata.spec().toString()); - Assert.assertEquals("Default spec ID should default to TableMetadata.INITIAL_SPEC_ID", - TableMetadata.INITIAL_SPEC_ID, metadata.defaultSpecId()); - Assert.assertEquals("PartitionSpec should contain the spec", - 1, metadata.specs().size()); - Assert.assertTrue("PartitionSpec should contain the spec", - metadata.specs().get(0).compatibleWith(spec)); - Assert.assertEquals("PartitionSpec should have ID TableMetadata.INITIAL_SPEC_ID", - TableMetadata.INITIAL_SPEC_ID, metadata.specs().get(0).specId()); - Assert.assertEquals("Properties should match", - expected.properties(), metadata.properties()); - Assert.assertEquals("Snapshot logs should match", - expected.snapshotLog(), metadata.snapshotLog()); - Assert.assertEquals("Current snapshot ID should match", - currentSnapshotId, metadata.currentSnapshot().snapshotId()); - Assert.assertEquals("Parent snapshot ID should match", - (Long) previousSnapshotId, metadata.currentSnapshot().parentId()); - Assert.assertEquals("Current snapshot files should match", - currentSnapshot.manifests(), metadata.currentSnapshot().manifests()); - Assert.assertEquals("Previous snapshot ID should match", - previousSnapshotId, metadata.snapshot(previousSnapshotId).snapshotId()); - Assert.assertEquals("Previous snapshot files should match", - previousSnapshot.manifests(), - metadata.snapshot(previousSnapshotId).manifests());""" - - -def to_json_without_spec_list(metadata): - return json.dumps({TableMetadataParser.FORMAT_VERSION: TableMetadata.TABLE_FORMAT_VERSION, - TableMetadataParser.LOCATION: metadata.location, - TableMetadataParser.LAST_UPDATED_MILLIS: metadata.last_updated_millis, - TableMetadataParser.LAST_COLUMN_ID: metadata.last_column_id, - TableMetadataParser.SCHEMA: SchemaParser.to_dict(metadata.schema), - TableMetadataParser.PARTITION_SPEC: PartitionSpecParser.to_json_fields(metadata.spec), - TableMetadataParser.PROPERTIES: metadata.properties, - TableMetadataParser.CURRENT_SNAPSHOT_ID: (-1 if metadata.current_snapshot() is None - else metadata.current_snapshot().snapshot_id), - TableMetadataParser.SNAPSHOTS: [SnapshotParser.to_dict(snapshot) - for snapshot in metadata.snapshots]}) diff --git a/tests/core/test_table_metadata_parser.py b/tests/core/test_table_metadata_parser.py deleted file mode 100644 index 9c8ad2c70a..0000000000 --- a/tests/core/test_table_metadata_parser.py +++ /dev/null @@ -1,43 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -import binascii - -from iceberg.core import ConfigProperties, TableMetadataParser -from iceberg.core.filesystem import FileSystemInputFile, FileSystemOutputFile - - -def test_compression_property(expected, prop): - config = {ConfigProperties.COMPRESS_METADATA: prop} - output_file = FileSystemOutputFile.from_path(TableMetadataParser.get_file_extension(config), dict) - TableMetadataParser.write(expected, output_file) - assert prop == is_compressed(TableMetadataParser.get_file_extension(config)) - read = TableMetadataParser.read(None, - FileSystemInputFile.from_location(TableMetadataParser.get_file_extension(config), None)) - verify_metadata(read, expected) - - -def verify_metadata(read, expected): - assert expected.schema.as_struct() == read.schema.as_struct() - assert expected.location == read.location - assert expected.last_column_id == read.last_column_id - assert expected.properties == read.properties - - -def is_compressed(file): - with open(file, 'rb') as test_f: - return binascii.hexlify(test_f.read(2)) == b'1f8b' diff --git a/tests/core/utils/__init__.py b/tests/core/utils/__init__.py deleted file mode 100644 index 6acb5d125a..0000000000 --- a/tests/core/utils/__init__.py +++ /dev/null @@ -1,14 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. diff --git a/tests/core/utils/test_bin_packing.py b/tests/core/utils/test_bin_packing.py deleted file mode 100644 index 81fe53101f..0000000000 --- a/tests/core/utils/test_bin_packing.py +++ /dev/null @@ -1,36 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -import random - -from iceberg.core.util import PackingIterator -import pytest - - -@pytest.mark.parametrize("splits,lookback,split_size, open_cost", [ - ([random.randint(0, 64) for x in range(200)], 20, 128, 4), # random splits - ([], 20, 128, 4), # no splits - ([0] * 100 + [random.randint(0, 64) for x in range(10)] + [0] * 100, 20, 128, 4) # sparse -]) -def test_bin_packing(splits, lookback, split_size, open_cost): - - def weight_func(x): - return max(x, open_cost) - - item_list_sums = [sum(item) - for item in PackingIterator(splits, split_size, lookback, weight_func)] - assert all([split_size >= item_sum >= 0 for item_sum in item_list_sums]) diff --git a/tests/hive/__init__.py b/tests/hive/__init__.py deleted file mode 100644 index fe95886d5c..0000000000 --- a/tests/hive/__init__.py +++ /dev/null @@ -1,18 +0,0 @@ -# -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. -# diff --git a/tests/hive/conftest.py b/tests/hive/conftest.py deleted file mode 100644 index 44c60d74d4..0000000000 --- a/tests/hive/conftest.py +++ /dev/null @@ -1,37 +0,0 @@ -# -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. -# - -from iceberg.api import PartitionSpec, PartitionSpecBuilder, Schema -from iceberg.api.types import IntegerType, NestedField, StringType -import pytest - - -@pytest.fixture(scope="session") -def base_scan_schema(): - return Schema([NestedField.required(1, "id", IntegerType.get()), - NestedField.required(2, "data", StringType.get())]) - - -@pytest.fixture(scope="session", params=["none", "one"]) -def base_scan_partition(base_scan_schema, request): - if request.param == "none": - spec = PartitionSpec.unpartitioned() - else: - spec = PartitionSpecBuilder(base_scan_schema).add(1, 1000, "id", "identity").build() - return spec diff --git a/tests/hive/test_hive_tables.py b/tests/hive/test_hive_tables.py deleted file mode 100644 index 481674712c..0000000000 --- a/tests/hive/test_hive_tables.py +++ /dev/null @@ -1,237 +0,0 @@ -# -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. -# -import os - -from hmsclient.genthrift.hive_metastore.ttypes import LockResponse, LockState, NoSuchObjectException -from iceberg.exceptions import AlreadyExistsException -from iceberg.hive import HiveTables -import mock -import pytest -from pytest import raises -from tests.api.test_helpers import MockHMSTable, MockManifest, MockManifestEntry, MockMetadata, MockReader, \ - MockSnapshot, MockTableOperations - - -@mock.patch("iceberg.hive.HiveTableOperations.refresh_from_metadata_location") -@mock.patch("iceberg.hive.HiveTableOperations.current") -@mock.patch("iceberg.hive.HiveTables.get_client") -def test_load_tables_check_valid_props(client, current_call, refresh_call): - parameters = {"table_type": "ICEBERG", - "partition_spec": [], - "metadata_location": "s3://path/to/iceberg.metadata.json"} - - client.return_value.__enter__.return_value.get_table.return_value = MockHMSTable(parameters) - - conf = {"hive.metastore.uris": 'thrift://hms:port'} - tables = HiveTables(conf) - tables.load("test.test_123") - - -@mock.patch("iceberg.hive.HiveTableOperations.refresh_from_metadata_location") -@mock.patch("iceberg.hive.HiveTableOperations.current") -@mock.patch("iceberg.hive.HiveTables.get_client") -def test_load_tables_check_missing_iceberg_type(client, current_call, refresh_call): - parameters = {"partition_spec": [], - "metadata_location": "s3://path/to/iceberg.metdata.json"} - - client.return_value.__enter__.return_value.get_table.return_value = MockHMSTable(parameters) - - conf = {"hive.metastore.uris": 'thrift://hms:port'} - tables = HiveTables(conf) - with raises(RuntimeError): - tables.load("test.test_123") - - -@mock.patch("iceberg.hive.HiveTableOperations.refresh_from_metadata_location") -@mock.patch("iceberg.hive.HiveTableOperations.current") -@mock.patch("iceberg.hive.HiveTables.get_client") -def test_load_tables_check_non_iceberg_type(client, current_call, refresh_call): - parameters = {"table_type": "HIVE", - "partition_spec": [], - "metadata_location": "s3://path/to/iceberg.metdata.json"} - - client.return_value.__enter__.return_value.get_table.return_value = MockHMSTable(parameters) - - conf = {"hive.metastore.uris": 'thrift://hms:port'} - tables = HiveTables(conf) - with raises(RuntimeError): - tables.load("test.test_123") - - -@mock.patch("iceberg.hive.HiveTableOperations.refresh_from_metadata_location") -@mock.patch("iceberg.hive.HiveTableOperations.current") -@mock.patch("iceberg.hive.HiveTables.get_client") -def test_load_tables_check_none_table_type(client, current_call, refresh_call): - parameters = {"table_type": None, - "partition_spec": [], - "metadata_location": "s3://path/to/iceberg.metdata.json"} - - client.return_value.__enter__.return_value.get_table.return_value = MockHMSTable(parameters) - - conf = {"hive.metastore.uris": 'thrift://hms:port'} - tables = HiveTables(conf) - with raises(RuntimeError): - tables.load("test.test_123") - - -@mock.patch("iceberg.hive.HiveTableOperations.refresh_from_metadata_location") -@mock.patch("iceberg.hive.HiveTableOperations.current") -@mock.patch("iceberg.hive.HiveTables.get_client") -def test_load_tables_check_no_location(client, current_call, refresh_call): - parameters = {"table_type": "ICEBERG", - "partition_spec": []} - - client.return_value.__enter__.return_value.get_table.return_value = MockHMSTable(parameters) - - conf = {"hive.metastore.uris": 'thrift://hms:port'} - tables = HiveTables(conf) - with raises(RuntimeError): - tables.load("test.test_123") - - -@mock.patch("iceberg.hive.HiveTableOperations.refresh_from_metadata_location") -@mock.patch("iceberg.hive.HiveTableOperations.current") -@mock.patch("iceberg.hive.HiveTables.get_client") -def test_load_tables_check_none_location(client, current_call, refresh_call): - parameters = {"table_type": "ICEBERG", - "partition_spec": [], - "metadata_location": None} - - client.return_value.__enter__.return_value.get_table.return_value = MockHMSTable(parameters) - - conf = {"hive.metastore.uris": 'thrift://hms:port'} - tables = HiveTables(conf) - with raises(RuntimeError): - tables.load("test.test_123") - - -@mock.patch("iceberg.hive.HiveTableOperations.refresh_from_metadata_location") -@mock.patch("iceberg.hive.HiveTableOperations.current") -@mock.patch("iceberg.hive.HiveTables.get_client") -def test_create_tables_failed(client, current_call, refresh_call, base_scan_schema, base_scan_partition, tmpdir): - client.return_value.__enter__.return_value.get_table.side_effect = NoSuchObjectException() - current_call.return_value = None - conf = {"hive.metastore.uris": 'thrift://hms:port', - "hive.metastore.warehouse.dir": tmpdir} - tables = HiveTables(conf) - with pytest.raises(AlreadyExistsException): - tables.create(base_scan_schema, "test.test_123", base_scan_partition) - assert len(os.listdir(os.path.join(tmpdir, "test.db", "test_123", "metadata"))) == 0 - - -@mock.patch("iceberg.hive.HiveTableOperations.current") -@mock.patch("iceberg.hive.HiveTables.get_client") -def test_create_tables(client, current_call, base_scan_schema, base_scan_partition, tmpdir): - - client.return_value.__enter__.return_value.get_table.side_effect = NoSuchObjectException() - current_call.return_value = None - client.return_value.__enter__.return_value.lock.return_value = LockResponse("x", LockState.WAITING) - client.return_value.__enter__.return_value.check_lock.return_value = LockResponse("x", LockState.ACQUIRED) - tbl = client.return_value.__enter__.return_value.create_table.call_args_list - conf = {"hive.metastore.uris": 'thrift://hms:port', - "hive.metastore.warehouse.dir": tmpdir} - tables = HiveTables(conf) - tables.create(base_scan_schema, "test.test_123", base_scan_partition) - - client.return_value.__enter__.return_value.get_table.return_value = MockHMSTable(tbl[0].args[0].parameters) - client.return_value.__enter__.return_value.get_table.side_effect = None - current_call.return_value = tbl[0].args[0].parameters['metadata_location'] - - tables.load("test.test_123") - - -@mock.patch("iceberg.hive.HiveTableOperations.refresh_from_metadata_location") -@mock.patch("iceberg.hive.HiveTableOperations.current") -@mock.patch("iceberg.hive.HiveTables.get_client") -def test_drop_tables(client, metadata, refresh_call, tmpdir): - - parameters = {"table_type": "ICEBERG", - "partition_spec": [], - "metadata_location": "s3://path/to/iceberg.metadata.json"} - - client.return_value.__enter__.return_value.get_table.return_value = MockHMSTable(parameters) - conf = {"hive.metastore.uris": 'thrift://hms:port', - "hive.metastore.warehouse.dir": tmpdir} - tables = HiveTables(conf) - tables.drop("test", "test_123", purge=False) - client.return_value.__enter__.return_value.drop_table.assert_called_with("test", "test_123", deleteData=False) - - -@mock.patch("iceberg.hive.HiveTableOperations.refresh_from_metadata_location") -@mock.patch("iceberg.hive.HiveTables.new_table_ops") -@mock.patch("iceberg.hive.HiveTables.get_client") -def test_drop_tables_purge(client, current_ops, refresh_call, tmpdir): - mock_snapshots = [MockSnapshot(location="snap-a.avro", - manifests=[ - MockManifest("a-manifest.avro"), - MockManifest("b-manifest.avro")], - manifest_to_entries={ - "a-manifest.avro": MockReader( - [MockManifestEntry("a.parquet"), - MockManifestEntry("b.parquet")]), - "b-manifest.avro": MockReader( - [MockManifestEntry("c.parquet"), - MockManifestEntry("d.parquet")]) - }), - MockSnapshot(location="snap-b.avro", - manifests=[ - MockManifest("b-manifest.avro"), - MockManifest("c-manifest.avro"), - MockManifest("d-manifest.avro")], - manifest_to_entries={ - "b-manifest.avro": MockReader( - [MockManifestEntry("c.parquet"), - MockManifestEntry("d.parquet")]), - "c-manifest.avro": MockReader( - [MockManifestEntry("e.parquet"), - MockManifestEntry("f.parquet")]), - "d-manifest.avro": MockReader( - [MockManifestEntry("g.parquet"), - MockManifestEntry("h.parquet")]) - }) - ] - ops = MockTableOperations(MockMetadata(mock_snapshots), "a.json") - current_ops.return_value = ops - - parameters = {"table_type": "ICEBERG", - "partition_spec": [], - "metadata_location": "s3://path/to/iceberg.metadata.json"} - client.return_value.__enter__.return_value.get_table.return_value = MockHMSTable(parameters) - - conf = {"hive.metastore.uris": 'thrift://hms:port', - "hive.metastore.warehouse.dir": tmpdir} - tables = HiveTables(conf) - tables.drop("test", "test_123", purge=True) - - assert len(ops.deleted) == len(set(ops.deleted)), "Paths should only be deleted once" - assert "a.json" in ops.deleted - assert "snap-a.avro" in ops.deleted - assert "snap-b.avro" in ops.deleted - assert "a-manifest.avro" in ops.deleted - assert "b-manifest.avro" in ops.deleted - assert "c-manifest.avro" in ops.deleted - assert "d-manifest.avro" in ops.deleted - assert "a.parquet" in ops.deleted - assert "b.parquet" in ops.deleted - assert "c.parquet" in ops.deleted - assert "d.parquet" in ops.deleted - assert "e.parquet" in ops.deleted - assert "f.parquet" in ops.deleted - assert "g.parquet" in ops.deleted - assert "h.parquet" in ops.deleted diff --git a/tests/parquet/__init__.py b/tests/parquet/__init__.py deleted file mode 100644 index 13a83393a9..0000000000 --- a/tests/parquet/__init__.py +++ /dev/null @@ -1,16 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. diff --git a/tests/parquet/conftest.py b/tests/parquet/conftest.py deleted file mode 100644 index 5f615f6f7a..0000000000 --- a/tests/parquet/conftest.py +++ /dev/null @@ -1,226 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -from collections import namedtuple -from datetime import datetime -from decimal import Decimal -from tempfile import NamedTemporaryFile - -from iceberg.api import Schema -from iceberg.api.types import (DateType, - DecimalType, - FloatType, - IntegerType, - LongType, - NestedField, - StringType, - TimestampType) -import pyarrow as pa -import pyarrow.parquet as pq -import pytest - - -TestRowGroupColumnStatistics = namedtuple("TestRowGroupColumnStatistics", ["min", "max", "null_count"]) -TestRowGroupColumn = namedtuple("TestRowGroupColumn", ["path_in_schema", - "file_offset", - "total_compressed_size", - "statistics"]) - - -class TestArrowParquetMetadata: - __test__ = False - - def __init__(self, col_metadata, num_rows=100): - self._col_metadata = col_metadata - self._num_rows = num_rows - - @property - def num_rows(self): - return self._num_rows - - @property - def num_columns(self): - return len(self._col_metadata) - - def column(self, i): - return self._col_metadata[i] - - def __getitem__(self, index): - return self._col_metadata[index] - - -@pytest.fixture(scope="session") -def pyarrow_array(): - return [pa.array([1, 2, 3, None, 5], type=pa.int32()), - pa.array(['us', 'can', 'us', 'us', 'can'], type=pa.string()), - pa.array([[0], [1, 2], [1], [1, 2, 3], None], type=pa.list_(pa.int64())), - pa.array([True, None, False, True, True], pa.bool_())] - - -@pytest.fixture(scope="session") -def pytable_colnames(): - return ['int_col', 'str_col', 'list_col', 'bool_col'] - - -@pytest.fixture(scope="session") -def rg_expected_schema(): - return Schema([NestedField.required(1, "string_col", StringType.get()), - NestedField.required(2, "long_col", LongType.get()), - NestedField.required(3, "int_col", IntegerType.get()), - NestedField.optional(4, "float_col", FloatType.get()), - NestedField.optional(5, "null_col", StringType.get()), - NestedField.optional(6, "missing_col", StringType.get()), - NestedField.optional(7, "no_stats_col", StringType.get()), - NestedField.optional(8, "ts_wtz_col", TimestampType.with_timezone()), - NestedField.optional(9, "ts_wotz_col", TimestampType.without_timezone()), - NestedField.optional(10, "big_decimal_type", DecimalType.of(38, 5)), - NestedField.optional(11, "small_decimal_type", DecimalType.of(10, 2)), - NestedField.optional(12, "date_type", DateType.get()), - ]) - - -@pytest.fixture(scope="session") -def rg_expected_schema_map(): - return {"string_col": "string_col", - "long_col": "long_col", - "int_col_renamed": "int_col", - "float_col": "float_col", - "null_col": "null_col", - "no_stats_col": "no_stats_col", - "ts_wtz_col": "ts_wtz_col", - "ts_wotz_col": "ts_wotz_col", - "big_decimal_type": "big_decimal_type", - "small_decimal_type": "small_decimal_type", - "date_type": "date_type" - } - - -@pytest.fixture(scope="session") -def rg_col_metadata(): - return [TestRowGroupColumn("string_col", 4, 12345, TestRowGroupColumnStatistics("b", "e", 0)), - TestRowGroupColumn("long_col", 12349, 12345, TestRowGroupColumnStatistics(0, 1234567890123, 0)), - TestRowGroupColumn("int_col_renamed", 24698, 12345, TestRowGroupColumnStatistics(0, 12345, 0)), - TestRowGroupColumn("float_col", 37043, 12345, TestRowGroupColumnStatistics(0.0, 123.45, 50)), - TestRowGroupColumn("null_col", 49388, 4, TestRowGroupColumnStatistics(None, None, 100)), - TestRowGroupColumn("no_stats_col", 61733, 4, None), - TestRowGroupColumn("ts_wtz_col", 74078, 4, - TestRowGroupColumnStatistics(datetime.strptime("2019-01-01 00:00:00-0000", - "%Y-%m-%d %H:%M:%S%z"), - datetime.strptime("2019-12-31 00:00:00-0000", - "%Y-%m-%d %H:%M:%S%z"), - 0)), - TestRowGroupColumn("ts_wotz_col", 86423, 4, - TestRowGroupColumnStatistics(datetime.strptime("2019-01-01 00:00:00-0000", - "%Y-%m-%d %H:%M:%S%z"), - datetime.strptime("2019-12-31 00:00:00-0000", - "%Y-%m-%d %H:%M:%S%z"), - 10)), - TestRowGroupColumn("big_decimal_type", 98768, 4, - # -123456789012345678.12345 to 123456789012345678.12345 - TestRowGroupColumnStatistics(b'\xff\xff\xff\xff\xff\xff\xfdb\xbdI\xb1\x89\x8e\xbe\xeb\x07', - b'\x00\x00\x00\x00\x00\x00\x02\x9dB\xb6NvqA\x14\xf9', - 10)), - TestRowGroupColumn("small_decimal_type", 111113, 4, - # 0 to 123.45 - TestRowGroupColumnStatistics(0, - 12345, - 10)), - TestRowGroupColumn("date_type", 123458, 4, - # 2020-01-01 to 2020-12-31 - TestRowGroupColumnStatistics(18262, 18262 + 365, 10)) - ] - - -@pytest.fixture(scope="session") -def pyarrow_schema(): - return pa.schema([pa.field("int_col", pa.int32(), False), - pa.field("bigint_col", pa.int64(), True), - pa.field("str_col", pa.string(), True), - pa.field("float_col", pa.float32(), True), - pa.field("dbl_col", pa.float64(), True), - pa.field("decimal_col", - pa.decimal128(9, 2), True), - pa.field("big_decimal_col", - pa.decimal128(19, 5), True), - pa.field("huge_decimal_col", - pa.decimal128(38, 9), True), - pa.field("date_col", pa.date32(), True), - pa.field("ts_col", pa.timestamp('us'), True), - pa.field("ts_wtz_col", - pa.timestamp('us', - 'America/New_York'), - True), - pa.field("bool_col", pa.bool_(), True)]) - - -@pytest.fixture(scope="session") -def pyarrow_primitive_array(): - return [pa.array([1, 2, 3, 4, 5], type=pa.int32()), - pa.array([1, 2, 3, None, 5], type=pa.int64()), - pa.array(['us', 'can', 'us', 'us', 'can'], type=pa.string()), - pa.array([1.0, 2.0, 3.0, 4.0, 5.0], type=pa.float32()), - pa.array([1.0, 2.0, 3.0, 4.0, 5.0], type=pa.float64()), - pa.array([Decimal("1.0"), Decimal("2.0"), Decimal("3.0"), - Decimal("4.0"), Decimal("5.0")], type=pa.decimal128(9, 2)), - pa.array([Decimal("1.0"), Decimal("2.0"), Decimal("3.0"), - Decimal("4.0"), Decimal("5.0")], type=pa.decimal128(19, 5)), - pa.array([Decimal("1.0"), Decimal("2.0"), Decimal("3.0"), - Decimal("4.0"), Decimal("5.0")], type=pa.decimal128(38, 9)), - pa.array([18506, 18507, 18508, 18508, 18510], type=pa.date32()), - pa.array([1598918400000000, 1599004800000000, - 1599091200000000, 1599177600000000, 1599264000000000], - type=pa.timestamp("us")), - pa.array([1598918400000000, 1599004800000000, - 1599091200000000, 1599177600000000, 1599264000000000], - type=pa.timestamp("us", 'America/New_York')), - pa.array([True, None, False, True, True], pa.bool_())] - - -@pytest.fixture(scope="session") -def primitive_type_test_file(pyarrow_primitive_array, pyarrow_schema): - with NamedTemporaryFile() as temp_file: - pq.write_table(pa.table(pyarrow_primitive_array, schema=pyarrow_schema), temp_file.name) - yield temp_file.name - - -@pytest.fixture(scope="session") -def primitive_type_test_parquet_file(primitive_type_test_file): - yield pq.ParquetFile(primitive_type_test_file) - - -@pytest.fixture(scope="session") -def unnested_complex_type_test_parquet_file(): - struct_fields = [("f1", pa.int32()), ("f2", pa.string())] - struct_type = pa.struct(struct_fields) - pyarrow_array = [pa.array([[1, 2, 3], [4, None, 6], None, [7, 8, 9]], type=pa.list_(pa.int32())), - pa.array([["a", "b", "c"], ["d", None, "e"], None, ["f", "g", "h"]], type=pa.list_(pa.string())), - pa.array([None, None, {"f1": 3, "f2": "c"}, {"f1": 4, "f2": "d"}], type=struct_type) - ] - with NamedTemporaryFile() as temp_file: - pq.write_table(pa.table(pyarrow_array, names=["list_int_col", 'list_str_col', 'struct_col']), - temp_file.name) - yield pq.ParquetFile(temp_file.name) - - -@pytest.fixture(scope="session") -def parquet_schema(type_test_parquet_file): - return type_test_parquet_file.schema - - -@pytest.fixture(scope="session") -def arrow_schema(type_test_parquet_file): - return type_test_parquet_file.schema_arrow diff --git a/tests/parquet/test_dataset_utils.py b/tests/parquet/test_dataset_utils.py deleted file mode 100644 index 2b8084b6de..0000000000 --- a/tests/parquet/test_dataset_utils.py +++ /dev/null @@ -1,52 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -from iceberg.api.expressions import Expressions -from iceberg.parquet.dataset_utils import get_dataset_filter -import pyarrow.dataset as ds -import pytest - - -@pytest.mark.parametrize("expr, dataset_filter, column_map", - [(Expressions.greater_than('a', 1), ds.field('a') > 1, {'a': 'a'}), - (Expressions.greater_than_or_equal('a', 1), ds.field('a') >= 1, {'a': 'a'}), - (Expressions.less_than('a', 1), ds.field('a') < 1, {'a': 'a'}), - (Expressions.less_than_or_equal('a', 1), ds.field('a') <= 1, {'a': 'a'}), - (Expressions.equal('a', 1), ds.field('a') == 1, {'a': 'a'}), - (Expressions.not_equal('a', 1), ds.field('a') != 1, {'a': 'a'}), - (Expressions.not_null('a'), ds.field('a').is_valid(), {'a': 'a'}), - (Expressions.is_null('a'), ~ds.field('a').is_valid(), {'a': 'a'}) - - ]) -def test_simple(expr, dataset_filter, column_map): - translated_dataset_filter = get_dataset_filter(expr, column_map) - assert dataset_filter.equals(translated_dataset_filter) - - -def test_not_conversion(): - expr = Expressions.not_(Expressions.greater_than('a', 1)) - translated_dataset_filter = get_dataset_filter(expr, {'a': 'a'}) - assert (~(ds.field("a") > 1)).equals(translated_dataset_filter) - - -def test_complex_expr(): - expr = Expressions.or_(Expressions.and_(Expressions.greater_than('a', 1), Expressions.equal("b", "US")), - Expressions.equal("c", True)) - - translated_dataset_filter = get_dataset_filter(expr, {'a': 'a', 'b': 'b', 'c': 'c'}) - dataset_filter = (((ds.field("a") > 1) & (ds.field("b") == "US")) | (ds.field("c") == True)) # noqa: E712 - assert dataset_filter.equals(translated_dataset_filter) diff --git a/tests/parquet/test_parquet_reader.py b/tests/parquet/test_parquet_reader.py deleted file mode 100644 index e170c3f6a0..0000000000 --- a/tests/parquet/test_parquet_reader.py +++ /dev/null @@ -1,285 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -from iceberg.api import Schema -from iceberg.api.expressions import Expressions -from iceberg.api.types import (BooleanType, - DateType, - DecimalType, - DoubleType, - FloatType, - IntegerType, - LongType, - NestedField, - StringType, - TimestampType) -from iceberg.core.filesystem import FileSystemInputFile, get_fs -from iceberg.parquet import ParquetReader -import pyarrow as pa - - -def test_basic_read(primitive_type_test_file, pyarrow_primitive_array, pyarrow_schema): - expected_schema = Schema([NestedField.required(1, "int_col", IntegerType.get()), - NestedField.optional(2, "bigint_col", LongType.get()), - NestedField.optional(3, "str_col", StringType.get()), - NestedField.optional(4, "float_col", FloatType.get()), - NestedField.optional(5, "dbl_col", DoubleType.get()), - NestedField.optional(6, "decimal_col", DecimalType.of(9, 2)), - NestedField.optional(7, "big_decimal_col", DecimalType.of(19, 5)), - NestedField.optional(8, "huge_decimal_col", DecimalType.of(38, 9)), - NestedField.optional(9, "date_col", DateType.get()), - NestedField.optional(10, "ts_col", TimestampType.without_timezone()), - NestedField.optional(11, "ts_wtz_col", TimestampType.with_timezone()), - NestedField.optional(12, "bool_col", BooleanType.get())]) - - input_file = FileSystemInputFile(get_fs(primitive_type_test_file, conf={}), primitive_type_test_file, {}) - reader = ParquetReader(input_file, expected_schema, {}, Expressions.always_true(), True) - - source_table = pa.table(pyarrow_primitive_array, schema=pyarrow_schema) - assert reader.read() == source_table - - -def test_projection(primitive_type_test_file, pyarrow_primitive_array, pyarrow_schema): - expected_schema = Schema([NestedField.required(1, "int_col", IntegerType.get()), - NestedField.optional(2, "bigint_col", LongType.get())]) - - input_file = FileSystemInputFile(get_fs(primitive_type_test_file, conf={}), primitive_type_test_file, {}) - reader = ParquetReader(input_file, expected_schema, {}, Expressions.always_true(), True) - - source_table = pa.table(pyarrow_primitive_array, schema=pyarrow_schema) - num_cols = source_table.num_columns - for i in range(1, num_cols - 1): - source_table = source_table.remove_column(num_cols - i) - - assert source_table == reader.read() - - -def test_column_rename(primitive_type_test_file): - expected_schema = Schema([NestedField.required(1, "int_col", IntegerType.get()), - NestedField.optional(2, "bigint_col", LongType.get()), - NestedField.optional(3, "string_col", StringType.get()), - NestedField.optional(4, "float_col", FloatType.get()), - NestedField.optional(5, "dbl_col", DoubleType.get())]) - - input_file = FileSystemInputFile(get_fs(primitive_type_test_file, conf={}), primitive_type_test_file, {}) - reader = ParquetReader(input_file, expected_schema, {}, Expressions.always_true(), True) - pyarrow_array = [pa.array([1, 2, 3, 4, 5], type=pa.int32()), - pa.array([1, 2, 3, None, 5], type=pa.int64()), - pa.array(['us', 'can', 'us', 'us', 'can'], type=pa.string()), - pa.array([1.0, 2.0, 3.0, 4.0, 5.0], type=pa.float32()), - pa.array([1.0, 2.0, 3.0, 4.0, 5.0], type=pa.float64())] - schema = pa.schema([pa.field("int_col", pa.int32(), False), - pa.field("bigint_col", pa.int64(), True), - pa.field("string_col", pa.string(), True), - pa.field("float_col", pa.float32(), True), - pa.field("dbl_col", pa.float64(), True)]) - - source_table = pa.table(pyarrow_array, schema=schema) - - target_table = reader.read() - assert source_table == target_table - - -def test_column_add(primitive_type_test_file): - expected_schema = Schema([NestedField.required(1, "int_col", IntegerType.get()), - NestedField.optional(2, "bigint_col", LongType.get()), - NestedField.optional(3, "string_col", StringType.get()), - NestedField.optional(4, "float_col", FloatType.get()), - NestedField.optional(5, "dbl_col", DoubleType.get()), - NestedField.optional(13, "int_col2", IntegerType.get())]) - - input_file = FileSystemInputFile(get_fs(primitive_type_test_file, conf={}), primitive_type_test_file, {}) - reader = ParquetReader(input_file, expected_schema, {}, Expressions.always_true(), True) - pyarrow_array = [pa.array([1, 2, 3, 4, 5], type=pa.int32()), - pa.array([1, 2, 3, None, 5], type=pa.int64()), - pa.array(['us', 'can', 'us', 'us', 'can'], type=pa.string()), - pa.array([1.0, 2.0, 3.0, 4.0, 5.0], type=pa.float32()), - pa.array([1.0, 2.0, 3.0, 4.0, 5.0], type=pa.float64()), - pa.array([None, None, None, None, None], type=pa.int32())] - source_table = pa.table(pyarrow_array, schema=pa.schema([pa.field("int_col", pa.int32(), nullable=False), - pa.field("bigint_col", pa.int64(), nullable=True), - pa.field("string_col", pa.string(), nullable=True), - pa.field("float_col", pa.float32(), nullable=True), - pa.field("dbl_col", pa.float64(), nullable=True), - pa.field("int_col2", pa.int32(), nullable=True), - ])) - - target_table = reader.read() - assert source_table == target_table - - -def test_decimal_column_add(primitive_type_test_file): - expected_schema = Schema([NestedField.required(1, "int_col", IntegerType.get()), - NestedField.optional(2, "bigint_col", LongType.get()), - NestedField.optional(4, "float_col", FloatType.get()), - NestedField.optional(5, "dbl_col", DoubleType.get()), - NestedField.optional(13, "new_dec_col", DecimalType.of(38, 9)) - ]) - - input_file = FileSystemInputFile(get_fs(primitive_type_test_file, conf={}), primitive_type_test_file, {}) - reader = ParquetReader(input_file, expected_schema, {}, Expressions.always_true(), True) - pyarrow_array = [pa.array([1, 2, 3, 4, 5], type=pa.int32()), - pa.array([1, 2, 3, None, 5], type=pa.int64()), - pa.array([1.0, 2.0, 3.0, 4.0, 5.0], type=pa.float32()), - pa.array([1.0, 2.0, 3.0, 4.0, 5.0], type=pa.float64()), - pa.array([None, None, None, None, None], type=pa.decimal128(38, 9))] - - source_table = pa.table(pyarrow_array, schema=pa.schema([pa.field("int_col", pa.int32(), nullable=False), - pa.field("bigint_col", pa.int64(), nullable=True), - pa.field("float_col", pa.float32(), nullable=True), - pa.field("dbl_col", pa.float64(), nullable=True), - pa.field("new_dec_col", pa.decimal128(38, 9), nullable=True) - ])) - - target_table = reader.read() - assert source_table == target_table - - -def test_column_reorder(primitive_type_test_file): - expected_schema = Schema([NestedField.required(1, "int_col", IntegerType.get()), - NestedField.optional(2, "bigint_col", LongType.get()), - NestedField.optional(4, "float_col", FloatType.get()), - NestedField.optional(5, "dbl_col", DoubleType.get()), - NestedField.optional(3, "string_col", StringType.get())]) - - input_file = FileSystemInputFile(get_fs(primitive_type_test_file, conf={}), primitive_type_test_file, {}) - reader = ParquetReader(input_file, expected_schema, {}, Expressions.always_true(), True) - pyarrow_array = [pa.array([1, 2, 3, 4, 5], type=pa.int32()), - pa.array([1, 2, 3, None, 5], type=pa.int64()), - pa.array([1.0, 2.0, 3.0, 4.0, 5.0], type=pa.float32()), - pa.array([1.0, 2.0, 3.0, 4.0, 5.0], type=pa.float64()), - pa.array(['us', 'can', 'us', 'us', 'can'], type=pa.string())] - source_table = pa.table(pyarrow_array, schema=pa.schema([pa.field("int_col", pa.int32(), nullable=False), - pa.field("bigint_col", pa.int64(), nullable=True), - pa.field("float_col", pa.float32(), nullable=True), - pa.field("dbl_col", pa.float64(), nullable=True), - pa.field("string_col", pa.string(), nullable=True) - ])) - - target_table = reader.read() - assert source_table == target_table - - -def test_column_upcast(primitive_type_test_file): - expected_schema = Schema([NestedField.required(1, "int_col", LongType.get())]) - - input_file = FileSystemInputFile(get_fs(primitive_type_test_file, conf={}), primitive_type_test_file, {}) - reader = ParquetReader(input_file, expected_schema, {}, Expressions.always_true(), True) - pyarrow_array = [pa.array([1, 2, 3, 4, 5], type=pa.int32())] - source_table = pa.table(pyarrow_array, schema=pa.schema([pa.field("int_col", pa.int64(), nullable=False)])) - - target_table = reader.read() - assert source_table == target_table - - -def test_filter(primitive_type_test_file): - expected_schema = Schema([NestedField.required(1, "int_col", IntegerType.get()), - NestedField.optional(2, "bigint_col", LongType.get()), - NestedField.optional(4, "float_col", FloatType.get()), - NestedField.optional(5, "dbl_col", DoubleType.get()), - NestedField.optional(3, "string_col", StringType.get())]) - - input_file = FileSystemInputFile(get_fs(primitive_type_test_file, conf={}), primitive_type_test_file, {}) - reader = ParquetReader(input_file, expected_schema, {}, Expressions.equal("string_col", "us"), True) - pyarrow_array = [pa.array([1, 3, 4], type=pa.int32()), - pa.array([1, 3, None], type=pa.int64()), - pa.array([1.0, 3.0, 4.0], type=pa.float32()), - pa.array([1.0, 3.0, 4.0], type=pa.float64()), - pa.array(['us', 'us', 'us'], type=pa.string())] - source_table = pa.table(pyarrow_array, schema=pa.schema([pa.field("int_col", pa.int32(), nullable=False), - pa.field("bigint_col", pa.int64(), nullable=True), - pa.field("float_col", pa.float32(), nullable=True), - pa.field("dbl_col", pa.float64(), nullable=True), - pa.field("string_col", pa.string(), nullable=True) - ])) - - target_table = reader.read() - assert source_table == target_table - - -def test_compound_filter(primitive_type_test_file): - expected_schema = Schema([NestedField.required(1, "int_col", IntegerType.get()), - NestedField.optional(2, "bigint_col", LongType.get()), - NestedField.optional(4, "float_col", FloatType.get()), - NestedField.optional(5, "dbl_col", DoubleType.get()), - NestedField.optional(3, "string_col", StringType.get())]) - - input_file = FileSystemInputFile(get_fs(primitive_type_test_file, conf={}), primitive_type_test_file, {}) - reader = ParquetReader(input_file, expected_schema, {}, Expressions.and_(Expressions.equal("string_col", "us"), - Expressions.equal("int_col", 1)), - True) - pyarrow_array = [pa.array([1], type=pa.int32()), - pa.array([1], type=pa.int64()), - pa.array([1.0], type=pa.float32()), - pa.array([1.0], type=pa.float64()), - pa.array(['us'], type=pa.string())] - - source_table = pa.table(pyarrow_array, schema=pa.schema([pa.field("int_col", pa.int32(), nullable=False), - pa.field("bigint_col", pa.int64(), nullable=True), - pa.field("float_col", pa.float32(), nullable=True), - pa.field("dbl_col", pa.float64(), nullable=True), - pa.field("string_col", pa.string(), nullable=True) - ])) - - target_table = reader.read() - assert source_table == target_table - - -def test_schema_evolution_filter(primitive_type_test_file): - expected_schema = Schema([NestedField.required(1, "int_col", IntegerType.get()), - NestedField.optional(2, "bigint_col", LongType.get()), - NestedField.optional(16, "other_new_col", LongType.get()), - NestedField.optional(4, "float_col", FloatType.get()), - NestedField.optional(5, "dbl_col", DoubleType.get()), - NestedField.optional(3, "string_col", StringType.get()), - NestedField.optional(15, "new_col", StringType.get())]) - - input_file = FileSystemInputFile(get_fs(primitive_type_test_file, conf={}), primitive_type_test_file, {}) - reader = ParquetReader(input_file, expected_schema, {}, Expressions.not_null("new_col"), True) - - schema = pa.schema([pa.field("int_col", pa.int32(), nullable=False), - pa.field("bigint_col", pa.int64(), nullable=True), - pa.field("other_new_col", pa.int64(), nullable=True), - pa.field("float_col", pa.float32(), nullable=True), - pa.field("dbl_col", pa.float64(), nullable=True), - pa.field("string_col", pa.string(), nullable=True), - pa.field("new_col", pa.string(), nullable=True)]) - - pyarrow_not_null_array = [pa.array([], type=pa.int32()), - pa.array([], type=pa.int64()), - pa.array([], type=pa.int32()), - pa.array([], type=pa.float32()), - pa.array([], type=pa.float64()), - pa.array([], type=pa.string()), - pa.array([], type=pa.string())] - - not_null_table = pa.table(pyarrow_not_null_array, schema=schema) - pyarrow_null_array = [pa.array([1, 2, 3, 4, 5], type=pa.int32()), - pa.array([1, 2, 3, None, 5], type=pa.int64()), - pa.array([None, None, None, None, None], type=pa.int64()), - pa.array([1.0, 2.0, 3.0, 4.0, 5.0], type=pa.float32()), - pa.array([1.0, 2.0, 3.0, 4.0, 5.0], type=pa.float64()), - pa.array(['us', 'can', 'us', 'us', 'can'], type=pa.string()), - pa.array([None, None, None, None, None], type=pa.string())] - null_table = pa.table(pyarrow_null_array, schema=schema) - - target_table = reader.read() - assert not_null_table == target_table - - reader = ParquetReader(input_file, expected_schema, {}, Expressions.is_null("new_col"), True) - target_table = reader.read() - assert null_table == target_table diff --git a/tests/parquet/test_parquet_to_iceberg.py b/tests/parquet/test_parquet_to_iceberg.py deleted file mode 100644 index 478d016afa..0000000000 --- a/tests/parquet/test_parquet_to_iceberg.py +++ /dev/null @@ -1,76 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -from iceberg.api import Schema -from iceberg.api.types import (BooleanType, - DateType, - DecimalType, - DoubleType, - FloatType, - IntegerType, - ListType, - LongType, - NestedField, - StringType, - StructType, - TimestampType) -from iceberg.parquet.parquet_to_iceberg import convert_parquet_to_iceberg - - -def compare_schema(expected_schema, converted_schema): - - assert len(expected_schema) == len(converted_schema) - for field in expected_schema.as_struct().fields: - converted_field = converted_schema.find_field(field.id) - assert converted_field is not None - assert field.name == converted_field.name - assert field.is_optional == converted_field.is_optional - assert field.type == converted_field.type - - -def test_primitive_types(primitive_type_test_parquet_file): - expected_schema = Schema([NestedField.required(1, "int_col", IntegerType.get()), - NestedField.optional(2, "bigint_col", LongType.get()), - NestedField.optional(3, "str_col", StringType.get()), - NestedField.optional(4, "float_col", FloatType.get()), - NestedField.optional(5, "dbl_col", DoubleType.get()), - NestedField.optional(6, "decimal_col", DecimalType.of(9, 2)), - NestedField.optional(7, "big_decimal_col", DecimalType.of(19, 5)), - NestedField.optional(8, "huge_decimal_col", DecimalType.of(38, 9)), - NestedField.optional(9, "date_col", DateType.get()), - NestedField.optional(10, "ts_col", TimestampType.without_timezone()), - NestedField.optional(11, "ts_wtz_col", TimestampType.with_timezone()), - NestedField.optional(12, "bool_col", BooleanType.get())]) - compare_schema(expected_schema, convert_parquet_to_iceberg(primitive_type_test_parquet_file)) - - -def test_unnested_complex_types(unnested_complex_type_test_parquet_file): - expected_schema = Schema([NestedField.optional(1, "list_int_col", ListType.of_optional(3, IntegerType.get())), - NestedField.optional(4, "list_str_col", ListType.of_optional(6, StringType.get())), - NestedField.optional(7, "struct_col", - StructType.of([NestedField.optional(8, "f1", IntegerType.get()), - NestedField.optional(9, "f2", StringType.get())])) - ]) - converted_schema = convert_parquet_to_iceberg(unnested_complex_type_test_parquet_file) - compare_schema(expected_schema, converted_schema) - - -def test_nested_complex_types(): - # This is a placeholder until pyarrow fully supports reading and writing nested types. - # Following a release containing this PR: - # https://github.com/apache/arrow/pull/8177 - pass diff --git a/tox.ini b/tox.ini deleted file mode 100644 index 7de9703cb7..0000000000 --- a/tox.ini +++ /dev/null @@ -1,115 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -[tox] -envlist = py37,py38,py39,linters - -[testenv] -deps = - coverage - mock - nose - pytest -setenv = - COVERAGE_FILE = test-reports/{envname}/.coverage - PYTEST_ADDOPTS = --junitxml=test-reports/{envname}/junit.xml -vv -commands = - coverage run --source iceberg --parallel-mode -m pytest {posargs} - coverage combine - coverage report -m - coverage html -d test-reports/{envname}/coverage-html - coverage xml -o test-reports/{envname}/coverage.xml - -[testenv:linters] -basepython = python3 -skip_install = true -deps = - . - {[testenv:flake8]deps} - {[testenv:bandit]deps} - {[testenv:mypy]deps} -commands = - {[testenv:flake8]commands} - {[testenv:bandit]commands} - {[testenv:mypy]commands} - -[testenv:flake8] -basepython = python3 -skip_install = true -deps = - flake8>=3.8.4 - flake8-import-order>=0.9 - flake8-bugbear -commands = - flake8 iceberg setup.py tests - -[testenv:mypy] -basepython = python3 -skip_install = true -deps = - mypy - types-pytz - types-python-dateutil -commands = - mypy --ignore-missing-imports iceberg/ - -[testenv:bandit] -basepython = python3 -skip_install = true -deps = - bandit -commands = - bandit --ini tox.ini -r iceberg - -# [testenv:docs] -# basepython = python3 -# deps = -# -r docs/source/requirements.txt -# commands = -# sphinx-build -E -W -c docs/source/ -b html docs/source/ docs/build/html -# sphinx-build -E -W -c docs/source/ -b man docs/source/ docs/build/man - -# [testenv:serve-docs] -# basepython = python3 -# skip_install = true -# changedir = docs/build/html -# deps = -# commands = -# python -m http.server {posargs} - -[flake8] -ignore = E501,W503 -exclude = - *.egg-info, - *.pyc, - .cache, - .coverage.*, - .gradle, - .tox, - build, - dist, - htmlcov.* -max-complexity = 10 -import-order-style = google -application-import-names = flake8 - -[pytest] -norecursedirs=.* - -[gh-actions] -python = - 3.7: py37 - 3.8: py38, linters - 3.9: py39 From f5286dafd69d6162edba3eac916ce2116550a660 Mon Sep 17 00:00:00 2001 From: jun-he Date: Sun, 3 Oct 2021 10:17:40 -0700 Subject: [PATCH 049/642] Python: Setup new python library module structure (#3184) --- LICENSE | 204 +++++++++++++++++++++++++++++++ MANIFEST.in | 18 +++ NOTICE | 8 ++ README.md | 43 +++++++ pyproject.toml | 23 ++++ setup.cfg | 47 +++++++ setup.py | 29 +++++ src/iceberg/__init__.py | 16 +++ src/iceberg/catalog/__init__.py | 16 +++ src/iceberg/io/__init__.py | 16 +++ src/iceberg/table/__init__.py | 16 +++ src/iceberg/utils/__init__.py | 16 +++ src/iceberg/utils/bin_packing.py | 81 ++++++++++++ tests/__init__.py | 16 +++ tests/utils/__init__.py | 16 +++ tests/utils/test_bin_packing.py | 50 ++++++++ tox.ini | 118 ++++++++++++++++++ 17 files changed, 733 insertions(+) create mode 100644 LICENSE create mode 100644 MANIFEST.in create mode 100644 NOTICE create mode 100644 README.md create mode 100644 pyproject.toml create mode 100644 setup.cfg create mode 100644 setup.py create mode 100644 src/iceberg/__init__.py create mode 100644 src/iceberg/catalog/__init__.py create mode 100644 src/iceberg/io/__init__.py create mode 100644 src/iceberg/table/__init__.py create mode 100644 src/iceberg/utils/__init__.py create mode 100644 src/iceberg/utils/bin_packing.py create mode 100644 tests/__init__.py create mode 100644 tests/utils/__init__.py create mode 100644 tests/utils/test_bin_packing.py create mode 100644 tox.ini diff --git a/LICENSE b/LICENSE new file mode 100644 index 0000000000..7868b9b440 --- /dev/null +++ b/LICENSE @@ -0,0 +1,204 @@ + + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + +-------------------------------------------------------------------------------- diff --git a/MANIFEST.in b/MANIFEST.in new file mode 100644 index 0000000000..a31dc633ad --- /dev/null +++ b/MANIFEST.in @@ -0,0 +1,18 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +graft src diff --git a/NOTICE b/NOTICE new file mode 100644 index 0000000000..906eca3c04 --- /dev/null +++ b/NOTICE @@ -0,0 +1,8 @@ + +Apache Iceberg +Copyright 2017-2021 The Apache Software Foundation + +This product includes software developed at +The Apache Software Foundation (http://www.apache.org/). + +-------------------------------------------------------------------------------- diff --git a/README.md b/README.md new file mode 100644 index 0000000000..35fe612026 --- /dev/null +++ b/README.md @@ -0,0 +1,43 @@ + + +# Iceberg Python + +py-iceberg is a python library for programmatic access to iceberg table metadata as well as to table data in iceberg format. +It is an implementation of [iceberg table spec](https://iceberg.apache.org/spec/) in Python. + +## Getting Started + +py-iceberg is currently in development, for development and testing purposes the best way to install the library is to perform the following steps: + +``` +git clone https://github.com/apache/iceberg.git +cd iceberg/python +pip install -e . +``` + +## Testing + +Testing is done using tox. The config can be found in `tox.ini` within the python directory of the iceberg project. + +``` +# simply run tox within the python dir +tox +``` + +## Get in Touch +- [Iceberg community](https://iceberg.apache.org/community/) diff --git a/pyproject.toml b/pyproject.toml new file mode 100644 index 0000000000..7389f493b7 --- /dev/null +++ b/pyproject.toml @@ -0,0 +1,23 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +[build-system] +requires = [ + "setuptools>=42", + "wheel" +] +build-backend = "setuptools.build_meta" \ No newline at end of file diff --git a/setup.cfg b/setup.cfg new file mode 100644 index 0000000000..5c58314131 --- /dev/null +++ b/setup.cfg @@ -0,0 +1,47 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +[metadata] +name = py-iceberg +version = 0.1.0a1 +author = Apache Software Foundation +author_email = dev@iceberg.apache.org +description = Apache Iceberg is an open table format for huge analytic datasets. +long_description = file: README.md +long_description_content_type = text/markdown +url = https://iceberg.apache.org/ +project_urls = + Source Code = https://github.com/apache/iceberg/ + Issue Tracker = https://github.com/apache/iceberg/issues +license="Apache License 2.0", +license_files = + LICENSE +classifiers = + License :: OSI Approved :: Apache Software License + Operating System :: OS Independent + Programming Language :: Python :: 3.7 + Programming Language :: Python :: 3.8 + Programming Language :: Python :: 3.9 + +[options] +package_dir = + = src +packages = find: +python_requires = >=3.7 + +[options.packages.find] +where = src diff --git a/setup.py b/setup.py new file mode 100644 index 0000000000..9acba2cc71 --- /dev/null +++ b/setup.py @@ -0,0 +1,29 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +from setuptools import setup + +setup( + name='py-iceberg', + install_requires=[], + extras_require={ + "dev": [ + "tox-travis==0.12", + "pytest", + ], + } +) diff --git a/src/iceberg/__init__.py b/src/iceberg/__init__.py new file mode 100644 index 0000000000..13a83393a9 --- /dev/null +++ b/src/iceberg/__init__.py @@ -0,0 +1,16 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. diff --git a/src/iceberg/catalog/__init__.py b/src/iceberg/catalog/__init__.py new file mode 100644 index 0000000000..13a83393a9 --- /dev/null +++ b/src/iceberg/catalog/__init__.py @@ -0,0 +1,16 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. diff --git a/src/iceberg/io/__init__.py b/src/iceberg/io/__init__.py new file mode 100644 index 0000000000..13a83393a9 --- /dev/null +++ b/src/iceberg/io/__init__.py @@ -0,0 +1,16 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. diff --git a/src/iceberg/table/__init__.py b/src/iceberg/table/__init__.py new file mode 100644 index 0000000000..13a83393a9 --- /dev/null +++ b/src/iceberg/table/__init__.py @@ -0,0 +1,16 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. diff --git a/src/iceberg/utils/__init__.py b/src/iceberg/utils/__init__.py new file mode 100644 index 0000000000..13a83393a9 --- /dev/null +++ b/src/iceberg/utils/__init__.py @@ -0,0 +1,16 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. diff --git a/src/iceberg/utils/bin_packing.py b/src/iceberg/utils/bin_packing.py new file mode 100644 index 0000000000..17d4faad26 --- /dev/null +++ b/src/iceberg/utils/bin_packing.py @@ -0,0 +1,81 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +class PackingIterator: + def __init__(self, items, target_weight, lookback, weight_func, largest_bin_first=False): + self.items = iter(items) + self.target_weight = target_weight + self.lookback = lookback + self.weight_func = weight_func + self.largest_bin_first = largest_bin_first + self.bins = list() + + def __iter__(self): + return self + + def __next__(self): + while True: + try: + item = next(self.items) + weight = self.weight_func(item) + bin_ = self.find_bin(weight) + if bin_ is not None: + bin_.add(item, weight) + else: + bin_ = self.Bin(self.target_weight) + bin_.add(item, weight) + self.bins.append(bin_) + + if len(self.bins) > self.lookback: + return list(self.remove_bin().items) + except StopIteration: + break + + if len(self.bins) == 0: + raise StopIteration() + + return list(self.remove_bin().items) + + def find_bin(self, weight): + for bin_ in self.bins: + if bin_.can_add(weight): + return bin_ + return None + + def remove_bin(self): + if self.largest_bin_first: + bin_ = max(self.bins, key=lambda b: b.weight()) + self.bins.remove(bin_) + return bin_ + else: + return self.bins.pop(0) + + class Bin: + def __init__(self, target_weight: int): + self.bin_weight = 0 + self.target_weight = target_weight + self.items: list = list() + + def weight(self) -> int: + return self.bin_weight + + def can_add(self, weight) -> bool: + return self.bin_weight + weight <= self.target_weight + + def add(self, item, weight): + self.bin_weight += weight + self.items.append(item) diff --git a/tests/__init__.py b/tests/__init__.py new file mode 100644 index 0000000000..13a83393a9 --- /dev/null +++ b/tests/__init__.py @@ -0,0 +1,16 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. diff --git a/tests/utils/__init__.py b/tests/utils/__init__.py new file mode 100644 index 0000000000..13a83393a9 --- /dev/null +++ b/tests/utils/__init__.py @@ -0,0 +1,16 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. diff --git a/tests/utils/test_bin_packing.py b/tests/utils/test_bin_packing.py new file mode 100644 index 0000000000..0cf44f132a --- /dev/null +++ b/tests/utils/test_bin_packing.py @@ -0,0 +1,50 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +import random + +from iceberg.utils.bin_packing import PackingIterator +import pytest + + +@pytest.mark.parametrize("splits, lookback, split_size, open_cost", [ + ([random.randint(0, 64) for x in range(200)], 20, 128, 4), # random splits + ([], 20, 128, 4), # no splits + ([0] * 100 + [random.randint(0, 64) in range(10)] + [0] * 100, 20, 128, 4) # sparse +]) +def test_bin_packing(splits, lookback, split_size, open_cost): + + def weight_func(x): + return max(x, open_cost) + + item_list_sums = [sum(item) + for item in PackingIterator(splits, split_size, lookback, weight_func)] + assert all([split_size >= item_sum >= 0 for item_sum in item_list_sums]) + + +@pytest.mark.parametrize("splits, target_weight, lookback, largest_bin_first, expected_lists", [ + ([36, 36, 36, 36, 73, 110, 128], 128, 2, True, [[110], [128], [36, 73], [36, 36, 36]]), + ([36, 36, 36, 36, 73, 110, 128], 128, 2, False, [[36, 36, 36], [36, 73], [110], [128]]), + ([64, 64, 128, 32, 32, 32, 32], 128, 1, True, [[64, 64], [128], [32, 32, 32, 32]]), + ([64, 64, 128, 32, 32, 32, 32], 128, 1, False, [[64, 64], [128], [32, 32, 32, 32]]), +]) +def test_bin_packing_lookback(splits, target_weight, lookback, largest_bin_first, expected_lists): + def weight_func(x): + return x + + assert [item for item in PackingIterator( + splits, target_weight, lookback, weight_func, largest_bin_first)] == expected_lists diff --git a/tox.ini b/tox.ini new file mode 100644 index 0000000000..d1596ea119 --- /dev/null +++ b/tox.ini @@ -0,0 +1,118 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +[tox] +envlist = py37,py38,py39,linters + +[testenv] +usedevelop = true +deps = + coverage + mock + nose + pytest +setenv = + COVERAGE_FILE = test-reports/{envname}/.coverage + PYTEST_ADDOPTS = --junitxml=test-reports/{envname}/junit.xml -vv +commands = + coverage run --source src --parallel-mode -m pytest {posargs} + coverage combine + coverage report -m --fail-under=90 + coverage html -d test-reports/{envname}/coverage-html + coverage xml -o test-reports/{envname}/coverage.xml + +[testenv:linters] +basepython = python3 +skip_install = true +deps = + . + {[testenv:flake8]deps} + {[testenv:bandit]deps} + {[testenv:mypy]deps} +commands = + {[testenv:flake8]commands} + {[testenv:bandit]commands} + {[testenv:mypy]commands} + +[testenv:flake8] +basepython = python3 +skip_install = true +deps = + flake8>=3.8.4 + flake8-import-order>=0.9 + flake8-bugbear +commands = + flake8 src setup.py tests + +[testenv:mypy] +basepython = python3 +skip_install = true +deps = + mypy + types-pytz + types-python-dateutil +commands = + mypy --ignore-missing-imports src/ + +[testenv:bandit] +basepython = python3 +skip_install = true +deps = + bandit +commands = + bandit --ini tox.ini -r src + +[testenv:docs] +basepython = python3 +deps = + -r docs/source/requirements.txt +commands = + sphinx-build -E -W -c docs/source/ -b html docs/source/ docs/build/html + sphinx-build -E -W -c docs/source/ -b man docs/source/ docs/build/man + +[testenv:serve-docs] +basepython = python3 +skip_install = true +changedir = docs/build/html +deps = +commands = + python -m http.server {posargs} + +[flake8] +ignore = E501,W503 +exclude = + *.egg-info, + *.pyc, + .cache, + .coverage.*, + .gradle, + .tox, + build, + dist, + htmlcov.* +max-complexity = 10 +import-order-style = google +application-import-names = flake8 + +[pytest] +norecursedirs=.* + +[gh-actions] +python = + 3.7: py37 + 3.8: py38, linters + 3.9: py39 From 89a613d4601a9331c307f185cd698da8eae0b472 Mon Sep 17 00:00:00 2001 From: jun-he Date: Thu, 21 Oct 2021 15:38:11 -0700 Subject: [PATCH 050/642] Python: Add all Iceberg types (#3234) --- src/iceberg/types.py | 144 +++++++++++++++++++++++++++++++++++++++++++ tests/test_types.py | 85 +++++++++++++++++++++++++ 2 files changed, 229 insertions(+) create mode 100644 src/iceberg/types.py create mode 100644 tests/test_types.py diff --git a/src/iceberg/types.py b/src/iceberg/types.py new file mode 100644 index 0000000000..e414b5cd59 --- /dev/null +++ b/src/iceberg/types.py @@ -0,0 +1,144 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +class Type(object): + def __init__(self, type_string: str, repr_string: str, is_primitive=False): + self._type_string = type_string + self._repr_string = repr_string + self._is_primitive = is_primitive + + def __repr__(self): + return self._repr_string + + def __str__(self): + return self._type_string + + @property + def is_primitive(self) -> bool: + return self._is_primitive + + +class FixedType(Type): + def __init__(self, length: int): + super().__init__(f"fixed[{length}]", f"FixedType({length})", is_primitive=True) + self._length = length + + @property + def length(self) -> int: + return self._length + + +class DecimalType(Type): + def __init__(self, precision: int, scale: int): + super().__init__(f"decimal({precision}, {scale})", f"DecimalType({precision}, {scale})", is_primitive=True) + self._precision = precision + self._scale = scale + + @property + def precision(self) -> int: + return self._precision + + @property + def scale(self) -> int: + return self._scale + + +class NestedField(object): + def __init__(self, is_optional: bool, field_id: int, name: str, field_type: Type, doc=None): + self._is_optional = is_optional + self._id = field_id + self._name = name + self._type = field_type + self._doc = doc + + @property + def is_optional(self) -> bool: + return self._is_optional + + @property + def is_required(self) -> bool: + return not self._is_optional + + @property + def field_id(self) -> int: + return self._id + + @property + def name(self) -> str: + return self._name + + @property + def type(self) -> Type: + return self._type + + def __repr__(self): + return (f"NestedField({self._is_optional}, {self._id}, " + f"{repr(self._name)}, {repr(self._type)}, {repr(self._doc)})") + + def __str__(self): + return (f"{self._id}: {self._name}: {'optional' if self._is_optional else 'required'} {self._type}" + "" if self._doc is None else f" ({self._doc})") + + +class StructType(Type): + def __init__(self, fields: list): + super().__init__(f"struct<{', '.join(map(str, fields))}>", f"StructType({repr(fields)})") + self._fields = fields + + @property + def fields(self) -> list: + return self._fields + + +class ListType(Type): + def __init__(self, element_field: NestedField): + super().__init__(f"list<{element_field.type}>", f"ListType({repr(element_field)})") + self._element_field = element_field + + @property + def element(self) -> NestedField: + return self._element_field + + +class MapType(Type): + def __init__(self, key_field: NestedField, value_field: NestedField): + super().__init__(f"map<{key_field.type}, {value_field.type}>", + f"MapType({repr(key_field)}, {repr(value_field)})") + self._key_field = key_field + self._value_field = value_field + + @property + def key(self) -> NestedField: + return self._key_field + + @property + def value(self) -> NestedField: + return self._value_field + + +BooleanType = Type("boolean", "BooleanType", is_primitive=True) +IntegerType = Type("int", "IntegerType", is_primitive=True) +LongType = Type("long", "LongType", is_primitive=True) +FloatType = Type("float", "FloatType", is_primitive=True) +DoubleType = Type("double", "DoubleType", is_primitive=True) +DateType = Type("date", "DateType", is_primitive=True) +TimeType = Type("time", "TimeType", is_primitive=True) +TimestampType = Type("timestamp", "TimestampType", is_primitive=True) +TimestamptzType = Type("timestamptz", "TimestamptzType", is_primitive=True) +StringType = Type("string", "StringType", is_primitive=True) +UUIDType = Type("uuid", "UUIDType", is_primitive=True) +BinaryType = Type("binary", "BinaryType", is_primitive=True) diff --git a/tests/test_types.py b/tests/test_types.py new file mode 100644 index 0000000000..026c9661e6 --- /dev/null +++ b/tests/test_types.py @@ -0,0 +1,85 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +from iceberg.types import (BinaryType, BooleanType, DateType, DecimalType, DoubleType, FixedType, + FloatType, IntegerType, ListType, LongType, MapType, NestedField, StringType, + StructType, TimestampType, TimestamptzType, TimeType, UUIDType) +import pytest + + +@pytest.mark.parametrize("input_type", + [BooleanType, IntegerType, LongType, FloatType, DoubleType, DateType, TimeType, + TimestampType, TimestamptzType, StringType, UUIDType, BinaryType]) +def test_repr_primitive_types(input_type): + assert input_type == eval(repr(input_type)) + + +def test_fixed_type(): + type_var = FixedType(5) + assert type_var.length == 5 + assert str(type_var) == str(eval(repr(type_var))) + + +def test_decimal_type(): + type_var = DecimalType(9, 2) + assert type_var.precision == 9 + assert type_var.scale == 2 + assert str(type_var) == str(eval(repr(type_var))) + + +def test_struct_type(): + type_var = StructType([NestedField(True, 1, "optional_field", IntegerType), + NestedField(False, 2, "required_field", FixedType(5)), + NestedField(False, 3, "required_field", StructType([ + NestedField(True, 4, "optional_field", DecimalType(8, 2)), + NestedField(False, 5, "required_field", LongType)]))]) + assert len(type_var.fields) == 3 + assert str(type_var) == str(eval(repr(type_var))) + + +def test_list_type(): + type_var = ListType(NestedField(False, 1, "required_field", StructType([ + NestedField(True, 2, "optional_field", DecimalType(8, 2)), + NestedField(False, 3, "required_field", LongType)]))) + assert isinstance(type_var.element.type, StructType) + assert len(type_var.element.type.fields) == 2 + assert type_var.element.field_id == 1 + assert str(type_var) == str(eval(repr(type_var))) + + +def test_map_type(): + type_var = MapType(NestedField(True, 1, "optional_field", DoubleType), + NestedField(False, 2, "required_field", UUIDType)) + assert type_var.key.type == DoubleType + assert type_var.key.field_id == 1 + assert type_var.value.type == UUIDType + assert type_var.value.field_id == 2 + assert str(type_var) == str(eval(repr(type_var))) + + +def test_nested_field(): + field_var = NestedField(True, 1, "optional_field1", StructType([ + NestedField(True, 2, "optional_field2", ListType( + NestedField(False, 3, "required_field3", DoubleType))), + NestedField(False, 4, "required_field4", MapType( + NestedField(True, 5, "optional_field5", TimeType), + NestedField(False, 6, "required_field6", UUIDType)))])) + assert field_var.is_optional + assert not field_var.is_required + assert field_var.field_id == 1 + assert isinstance(field_var.type, StructType) + assert str(field_var) == str(eval(repr(field_var))) From c0c0cf8ab244df82a1e12fb63517ad9714133a91 Mon Sep 17 00:00:00 2001 From: jun-he Date: Fri, 22 Oct 2021 08:20:30 -0700 Subject: [PATCH 051/642] Python: Minor changes to types classes, follow up to #3234 (#3350) --- src/iceberg/types.py | 29 +++++++++++++++-------------- tests/test_types.py | 8 ++++++-- 2 files changed, 21 insertions(+), 16 deletions(-) diff --git a/src/iceberg/types.py b/src/iceberg/types.py index e414b5cd59..2113787da4 100644 --- a/src/iceberg/types.py +++ b/src/iceberg/types.py @@ -34,7 +34,7 @@ def is_primitive(self) -> bool: class FixedType(Type): def __init__(self, length: int): - super().__init__(f"fixed[{length}]", f"FixedType({length})", is_primitive=True) + super().__init__(f"fixed[{length}]", f"FixedType(length={length})", is_primitive=True) self._length = length @property @@ -44,7 +44,8 @@ def length(self) -> int: class DecimalType(Type): def __init__(self, precision: int, scale: int): - super().__init__(f"decimal({precision}, {scale})", f"DecimalType({precision}, {scale})", is_primitive=True) + super().__init__(f"decimal({precision}, {scale})", + f"DecimalType(precision={precision}, scale={scale})", is_primitive=True) self._precision = precision self._scale = scale @@ -58,7 +59,7 @@ def scale(self) -> int: class NestedField(object): - def __init__(self, is_optional: bool, field_id: int, name: str, field_type: Type, doc=None): + def __init__(self, is_optional: bool, field_id: int, name: str, field_type: Type, doc: str = None): self._is_optional = is_optional self._id = field_id self._name = name @@ -86,8 +87,8 @@ def type(self) -> Type: return self._type def __repr__(self): - return (f"NestedField({self._is_optional}, {self._id}, " - f"{repr(self._name)}, {repr(self._type)}, {repr(self._doc)})") + return (f"NestedField(is_optional={self._is_optional}, field_id={self._id}, " + f"name={repr(self._name)}, field_type={repr(self._type)}, doc={repr(self._doc)})") def __str__(self): return (f"{self._id}: {self._name}: {'optional' if self._is_optional else 'required'} {self._type}" @@ -96,7 +97,7 @@ def __str__(self): class StructType(Type): def __init__(self, fields: list): - super().__init__(f"struct<{', '.join(map(str, fields))}>", f"StructType({repr(fields)})") + super().__init__(f"struct<{', '.join(map(str, fields))}>", f"StructType(fields={repr(fields)})") self._fields = fields @property @@ -105,9 +106,9 @@ def fields(self) -> list: class ListType(Type): - def __init__(self, element_field: NestedField): - super().__init__(f"list<{element_field.type}>", f"ListType({repr(element_field)})") - self._element_field = element_field + def __init__(self, element: NestedField): + super().__init__(f"list<{element.type}>", f"ListType(element={repr(element)})") + self._element_field = element @property def element(self) -> NestedField: @@ -115,11 +116,11 @@ def element(self) -> NestedField: class MapType(Type): - def __init__(self, key_field: NestedField, value_field: NestedField): - super().__init__(f"map<{key_field.type}, {value_field.type}>", - f"MapType({repr(key_field)}, {repr(value_field)})") - self._key_field = key_field - self._value_field = value_field + def __init__(self, key: NestedField, value: NestedField): + super().__init__(f"map<{key.type}, {value.type}>", + f"MapType(key={repr(key)}, value={repr(value)})") + self._key_field = key + self._value_field = value @property def key(self) -> NestedField: diff --git a/tests/test_types.py b/tests/test_types.py index 026c9661e6..b6d23dc555 100644 --- a/tests/test_types.py +++ b/tests/test_types.py @@ -29,15 +29,19 @@ def test_repr_primitive_types(input_type): def test_fixed_type(): - type_var = FixedType(5) + type_var = FixedType(length=5) assert type_var.length == 5 + assert str(type_var) == "fixed[5]" + assert repr(type_var) == "FixedType(length=5)" assert str(type_var) == str(eval(repr(type_var))) def test_decimal_type(): - type_var = DecimalType(9, 2) + type_var = DecimalType(precision=9, scale=2) assert type_var.precision == 9 assert type_var.scale == 2 + assert str(type_var) == 'decimal(9, 2)' + assert repr(type_var) == "DecimalType(precision=9, scale=2)" assert str(type_var) == str(eval(repr(type_var))) From 8edb07e393d089c3776d8af44d95abc9c647efee Mon Sep 17 00:00:00 2001 From: Samuel Redai <43911210+samredai@users.noreply.github.com> Date: Sun, 31 Oct 2021 12:00:00 -0700 Subject: [PATCH 052/642] Python: Add black, isort, and tox -e format command (#3383) --- setup.py | 4 +- src/iceberg/types.py | 46 +++++++++--- src/iceberg/utils/bin_packing.py | 5 +- tests/test_types.py | 120 +++++++++++++++++++++++++------ tests/utils/test_bin_packing.py | 78 +++++++++++++++----- tox.ini | 34 ++++++++- 6 files changed, 231 insertions(+), 56 deletions(-) diff --git a/setup.py b/setup.py index 9acba2cc71..5073c558a2 100644 --- a/setup.py +++ b/setup.py @@ -18,12 +18,12 @@ from setuptools import setup setup( - name='py-iceberg', + name="py-iceberg", install_requires=[], extras_require={ "dev": [ "tox-travis==0.12", "pytest", ], - } + }, ) diff --git a/src/iceberg/types.py b/src/iceberg/types.py index 2113787da4..57856f9252 100644 --- a/src/iceberg/types.py +++ b/src/iceberg/types.py @@ -15,6 +15,7 @@ # specific language governing permissions and limitations # under the License. + class Type(object): def __init__(self, type_string: str, repr_string: str, is_primitive=False): self._type_string = type_string @@ -34,7 +35,9 @@ def is_primitive(self) -> bool: class FixedType(Type): def __init__(self, length: int): - super().__init__(f"fixed[{length}]", f"FixedType(length={length})", is_primitive=True) + super().__init__( + f"fixed[{length}]", f"FixedType(length={length})", is_primitive=True + ) self._length = length @property @@ -44,8 +47,11 @@ def length(self) -> int: class DecimalType(Type): def __init__(self, precision: int, scale: int): - super().__init__(f"decimal({precision}, {scale})", - f"DecimalType(precision={precision}, scale={scale})", is_primitive=True) + super().__init__( + f"decimal({precision}, {scale})", + f"DecimalType(precision={precision}, scale={scale})", + is_primitive=True, + ) self._precision = precision self._scale = scale @@ -59,7 +65,14 @@ def scale(self) -> int: class NestedField(object): - def __init__(self, is_optional: bool, field_id: int, name: str, field_type: Type, doc: str = None): + def __init__( + self, + is_optional: bool, + field_id: int, + name: str, + field_type: Type, + doc: str = None, + ): self._is_optional = is_optional self._id = field_id self._name = name @@ -87,17 +100,26 @@ def type(self) -> Type: return self._type def __repr__(self): - return (f"NestedField(is_optional={self._is_optional}, field_id={self._id}, " - f"name={repr(self._name)}, field_type={repr(self._type)}, doc={repr(self._doc)})") + return ( + f"NestedField(is_optional={self._is_optional}, field_id={self._id}, " + f"name={repr(self._name)}, field_type={repr(self._type)}, doc={repr(self._doc)})" + ) def __str__(self): - return (f"{self._id}: {self._name}: {'optional' if self._is_optional else 'required'} {self._type}" - "" if self._doc is None else f" ({self._doc})") + return ( + f"{self._id}: {self._name}: {'optional' if self._is_optional else 'required'} {self._type}" + "" + if self._doc is None + else f" ({self._doc})" + ) class StructType(Type): def __init__(self, fields: list): - super().__init__(f"struct<{', '.join(map(str, fields))}>", f"StructType(fields={repr(fields)})") + super().__init__( + f"struct<{', '.join(map(str, fields))}>", + f"StructType(fields={repr(fields)})", + ) self._fields = fields @property @@ -117,8 +139,10 @@ def element(self) -> NestedField: class MapType(Type): def __init__(self, key: NestedField, value: NestedField): - super().__init__(f"map<{key.type}, {value.type}>", - f"MapType(key={repr(key)}, value={repr(value)})") + super().__init__( + f"map<{key.type}, {value.type}>", + f"MapType(key={repr(key)}, value={repr(value)})", + ) self._key_field = key self._value_field = value diff --git a/src/iceberg/utils/bin_packing.py b/src/iceberg/utils/bin_packing.py index 17d4faad26..8e32710028 100644 --- a/src/iceberg/utils/bin_packing.py +++ b/src/iceberg/utils/bin_packing.py @@ -15,8 +15,11 @@ # specific language governing permissions and limitations # under the License. + class PackingIterator: - def __init__(self, items, target_weight, lookback, weight_func, largest_bin_first=False): + def __init__( + self, items, target_weight, lookback, weight_func, largest_bin_first=False + ): self.items = iter(items) self.target_weight = target_weight self.lookback = lookback diff --git a/tests/test_types.py b/tests/test_types.py index b6d23dc555..3b1d834e05 100644 --- a/tests/test_types.py +++ b/tests/test_types.py @@ -15,15 +15,47 @@ # specific language governing permissions and limitations # under the License. -from iceberg.types import (BinaryType, BooleanType, DateType, DecimalType, DoubleType, FixedType, - FloatType, IntegerType, ListType, LongType, MapType, NestedField, StringType, - StructType, TimestampType, TimestamptzType, TimeType, UUIDType) import pytest +from iceberg.types import ( + BinaryType, + BooleanType, + DateType, + DecimalType, + DoubleType, + FixedType, + FloatType, + IntegerType, + ListType, + LongType, + MapType, + NestedField, + StringType, + StructType, + TimestampType, + TimestamptzType, + TimeType, + UUIDType, +) -@pytest.mark.parametrize("input_type", - [BooleanType, IntegerType, LongType, FloatType, DoubleType, DateType, TimeType, - TimestampType, TimestamptzType, StringType, UUIDType, BinaryType]) + +@pytest.mark.parametrize( + "input_type", + [ + BooleanType, + IntegerType, + LongType, + FloatType, + DoubleType, + DateType, + TimeType, + TimestampType, + TimestamptzType, + StringType, + UUIDType, + BinaryType, + ], +) def test_repr_primitive_types(input_type): assert input_type == eval(repr(input_type)) @@ -40,25 +72,47 @@ def test_decimal_type(): type_var = DecimalType(precision=9, scale=2) assert type_var.precision == 9 assert type_var.scale == 2 - assert str(type_var) == 'decimal(9, 2)' + assert str(type_var) == "decimal(9, 2)" assert repr(type_var) == "DecimalType(precision=9, scale=2)" assert str(type_var) == str(eval(repr(type_var))) def test_struct_type(): - type_var = StructType([NestedField(True, 1, "optional_field", IntegerType), - NestedField(False, 2, "required_field", FixedType(5)), - NestedField(False, 3, "required_field", StructType([ - NestedField(True, 4, "optional_field", DecimalType(8, 2)), - NestedField(False, 5, "required_field", LongType)]))]) + type_var = StructType( + [ + NestedField(True, 1, "optional_field", IntegerType), + NestedField(False, 2, "required_field", FixedType(5)), + NestedField( + False, + 3, + "required_field", + StructType( + [ + NestedField(True, 4, "optional_field", DecimalType(8, 2)), + NestedField(False, 5, "required_field", LongType), + ] + ), + ), + ] + ) assert len(type_var.fields) == 3 assert str(type_var) == str(eval(repr(type_var))) def test_list_type(): - type_var = ListType(NestedField(False, 1, "required_field", StructType([ - NestedField(True, 2, "optional_field", DecimalType(8, 2)), - NestedField(False, 3, "required_field", LongType)]))) + type_var = ListType( + NestedField( + False, + 1, + "required_field", + StructType( + [ + NestedField(True, 2, "optional_field", DecimalType(8, 2)), + NestedField(False, 3, "required_field", LongType), + ] + ), + ) + ) assert isinstance(type_var.element.type, StructType) assert len(type_var.element.type.fields) == 2 assert type_var.element.field_id == 1 @@ -66,8 +120,10 @@ def test_list_type(): def test_map_type(): - type_var = MapType(NestedField(True, 1, "optional_field", DoubleType), - NestedField(False, 2, "required_field", UUIDType)) + type_var = MapType( + NestedField(True, 1, "optional_field", DoubleType), + NestedField(False, 2, "required_field", UUIDType), + ) assert type_var.key.type == DoubleType assert type_var.key.field_id == 1 assert type_var.value.type == UUIDType @@ -76,12 +132,30 @@ def test_map_type(): def test_nested_field(): - field_var = NestedField(True, 1, "optional_field1", StructType([ - NestedField(True, 2, "optional_field2", ListType( - NestedField(False, 3, "required_field3", DoubleType))), - NestedField(False, 4, "required_field4", MapType( - NestedField(True, 5, "optional_field5", TimeType), - NestedField(False, 6, "required_field6", UUIDType)))])) + field_var = NestedField( + True, + 1, + "optional_field1", + StructType( + [ + NestedField( + True, + 2, + "optional_field2", + ListType(NestedField(False, 3, "required_field3", DoubleType)), + ), + NestedField( + False, + 4, + "required_field4", + MapType( + NestedField(True, 5, "optional_field5", TimeType), + NestedField(False, 6, "required_field6", UUIDType), + ), + ), + ] + ), + ) assert field_var.is_optional assert not field_var.is_required assert field_var.field_id == 1 diff --git a/tests/utils/test_bin_packing.py b/tests/utils/test_bin_packing.py index 0cf44f132a..7a5a0749fe 100644 --- a/tests/utils/test_bin_packing.py +++ b/tests/utils/test_bin_packing.py @@ -17,34 +17,76 @@ import random -from iceberg.utils.bin_packing import PackingIterator import pytest +from iceberg.utils.bin_packing import PackingIterator + -@pytest.mark.parametrize("splits, lookback, split_size, open_cost", [ - ([random.randint(0, 64) for x in range(200)], 20, 128, 4), # random splits - ([], 20, 128, 4), # no splits - ([0] * 100 + [random.randint(0, 64) in range(10)] + [0] * 100, 20, 128, 4) # sparse -]) +@pytest.mark.parametrize( + "splits, lookback, split_size, open_cost", + [ + ([random.randint(0, 64) for x in range(200)], 20, 128, 4), # random splits + ([], 20, 128, 4), # no splits + ( + [0] * 100 + [random.randint(0, 64) in range(10)] + [0] * 100, + 20, + 128, + 4, + ), # sparse + ], +) def test_bin_packing(splits, lookback, split_size, open_cost): - def weight_func(x): return max(x, open_cost) - item_list_sums = [sum(item) - for item in PackingIterator(splits, split_size, lookback, weight_func)] + item_list_sums = [ + sum(item) for item in PackingIterator(splits, split_size, lookback, weight_func) + ] assert all([split_size >= item_sum >= 0 for item_sum in item_list_sums]) -@pytest.mark.parametrize("splits, target_weight, lookback, largest_bin_first, expected_lists", [ - ([36, 36, 36, 36, 73, 110, 128], 128, 2, True, [[110], [128], [36, 73], [36, 36, 36]]), - ([36, 36, 36, 36, 73, 110, 128], 128, 2, False, [[36, 36, 36], [36, 73], [110], [128]]), - ([64, 64, 128, 32, 32, 32, 32], 128, 1, True, [[64, 64], [128], [32, 32, 32, 32]]), - ([64, 64, 128, 32, 32, 32, 32], 128, 1, False, [[64, 64], [128], [32, 32, 32, 32]]), -]) -def test_bin_packing_lookback(splits, target_weight, lookback, largest_bin_first, expected_lists): +@pytest.mark.parametrize( + "splits, target_weight, lookback, largest_bin_first, expected_lists", + [ + ( + [36, 36, 36, 36, 73, 110, 128], + 128, + 2, + True, + [[110], [128], [36, 73], [36, 36, 36]], + ), + ( + [36, 36, 36, 36, 73, 110, 128], + 128, + 2, + False, + [[36, 36, 36], [36, 73], [110], [128]], + ), + ( + [64, 64, 128, 32, 32, 32, 32], + 128, + 1, + True, + [[64, 64], [128], [32, 32, 32, 32]], + ), + ( + [64, 64, 128, 32, 32, 32, 32], + 128, + 1, + False, + [[64, 64], [128], [32, 32, 32, 32]], + ), + ], +) +def test_bin_packing_lookback( + splits, target_weight, lookback, largest_bin_first, expected_lists +): def weight_func(x): return x - assert [item for item in PackingIterator( - splits, target_weight, lookback, weight_func, largest_bin_first)] == expected_lists + assert [ + item + for item in PackingIterator( + splits, target_weight, lookback, weight_func, largest_bin_first + ) + ] == expected_lists diff --git a/tox.ini b/tox.ini index d1596ea119..b099a001e3 100644 --- a/tox.ini +++ b/tox.ini @@ -34,20 +34,52 @@ commands = coverage report -m --fail-under=90 coverage html -d test-reports/{envname}/coverage-html coverage xml -o test-reports/{envname}/coverage.xml +[testenv:format] +description = reformat all source code +basepython = python3 +deps = + black + isort + flake8 +skip_install = true +commands = + isort --project iceberg --profile black setup.py src tests + black setup.py src tests + flake8 setup.py src tests [testenv:linters] basepython = python3 skip_install = true deps = . + {[testenv:isort]deps} + {[testenv:black]deps} {[testenv:flake8]deps} {[testenv:bandit]deps} {[testenv:mypy]deps} commands = + {[testenv:isort]deps} + {[testenv:black]deps} {[testenv:flake8]commands} {[testenv:bandit]commands} {[testenv:mypy]commands} +[testenv:isort] +basepython = python3 +skip_install = true +deps = + isort +commands = + isort --recursive --project iceberg --profile black --check-only setup.py src tests + +[testenv:black] +basepython = python3 +skip_install = true +deps = + black +commands = + black --check --diff src setup.py tests + [testenv:flake8] basepython = python3 skip_install = true @@ -93,7 +125,7 @@ commands = python -m http.server {posargs} [flake8] -ignore = E501,W503 +ignore = E501,I100,I202,W503 exclude = *.egg-info, *.pyc, From 04fdcc6d7881c1c262264f5edf40f5233c39b7c1 Mon Sep 17 00:00:00 2001 From: Abhishek Date: Tue, 2 Nov 2021 08:30:24 -0700 Subject: [PATCH 053/642] Python: Add tox commands for code format and type checking (#3423) Closes #3282 --- src/iceberg/types.py | 4 +- tox.ini | 93 +++++++++----------------------------------- 2 files changed, 21 insertions(+), 76 deletions(-) diff --git a/src/iceberg/types.py b/src/iceberg/types.py index 57856f9252..c456c58e4c 100644 --- a/src/iceberg/types.py +++ b/src/iceberg/types.py @@ -15,6 +15,8 @@ # specific language governing permissions and limitations # under the License. +from typing import Optional + class Type(object): def __init__(self, type_string: str, repr_string: str, is_primitive=False): @@ -71,7 +73,7 @@ def __init__( field_id: int, name: str, field_type: Type, - doc: str = None, + doc: Optional[str] = None, ): self._is_optional = is_optional self._id = field_id diff --git a/tox.ini b/tox.ini index b099a001e3..ed9cb78c44 100644 --- a/tox.ini +++ b/tox.ini @@ -23,7 +23,6 @@ usedevelop = true deps = coverage mock - nose pytest setenv = COVERAGE_FILE = test-reports/{envname}/.coverage @@ -34,79 +33,39 @@ commands = coverage report -m --fail-under=90 coverage html -d test-reports/{envname}/coverage-html coverage xml -o test-reports/{envname}/coverage.xml -[testenv:format] -description = reformat all source code -basepython = python3 -deps = - black - isort - flake8 -skip_install = true -commands = - isort --project iceberg --profile black setup.py src tests - black setup.py src tests - flake8 setup.py src tests [testenv:linters] -basepython = python3 -skip_install = true -deps = - . - {[testenv:isort]deps} - {[testenv:black]deps} - {[testenv:flake8]deps} - {[testenv:bandit]deps} - {[testenv:mypy]deps} -commands = - {[testenv:isort]deps} - {[testenv:black]deps} - {[testenv:flake8]commands} - {[testenv:bandit]commands} - {[testenv:mypy]commands} - -[testenv:isort] -basepython = python3 -skip_install = true +usedevelop = true deps = - isort + {[testenv:format-check]deps} + {[testenv:type-check]deps} commands = - isort --recursive --project iceberg --profile black --check-only setup.py src tests + {[testenv:format-check]commands} + {[testenv:type-check]commands} -[testenv:black] -basepython = python3 -skip_install = true +[testenv:format-check] deps = black + isort + autoflake commands = - black --check --diff src setup.py tests + autoflake -r --check --ignore-init-module-imports --remove-all-unused-imports setup.py src tests + isort --profile black --check-only setup.py src tests + black --check setup.py src tests -[testenv:flake8] -basepython = python3 -skip_install = true +[testenv:format] deps = - flake8>=3.8.4 - flake8-import-order>=0.9 - flake8-bugbear + {[testenv:format-check]deps} commands = - flake8 src setup.py tests + autoflake -r --in-place --ignore-init-module-imports --remove-all-unused-imports setup.py src tests + isort --profile black setup.py src tests + black setup.py src tests -[testenv:mypy] -basepython = python3 -skip_install = true +[testenv:type-check] deps = mypy - types-pytz - types-python-dateutil -commands = - mypy --ignore-missing-imports src/ - -[testenv:bandit] -basepython = python3 -skip_install = true -deps = - bandit commands = - bandit --ini tox.ini -r src + mypy --no-implicit-optional src [testenv:docs] basepython = python3 @@ -124,22 +83,6 @@ deps = commands = python -m http.server {posargs} -[flake8] -ignore = E501,I100,I202,W503 -exclude = - *.egg-info, - *.pyc, - .cache, - .coverage.*, - .gradle, - .tox, - build, - dist, - htmlcov.* -max-complexity = 10 -import-order-style = google -application-import-names = flake8 - [pytest] norecursedirs=.* From 0934d78770b8662a2f4239ad24d6a555cf3600fd Mon Sep 17 00:00:00 2001 From: Samuel Redai <43911210+samredai@users.noreply.github.com> Date: Wed, 10 Nov 2021 11:31:59 -0500 Subject: [PATCH 054/642] Python: Add Operations for expressions (#3399) --- src/iceberg/expressions.py | 80 ++++++++++++++++++++++++++++++++++++++ tests/test_expressions.py | 58 +++++++++++++++++++++++++++ 2 files changed, 138 insertions(+) create mode 100644 src/iceberg/expressions.py create mode 100644 tests/test_expressions.py diff --git a/src/iceberg/expressions.py b/src/iceberg/expressions.py new file mode 100644 index 0000000000..48d1ae24f0 --- /dev/null +++ b/src/iceberg/expressions.py @@ -0,0 +1,80 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +from enum import Enum, auto + + +class Operation(Enum): + """Operations to be used as components in expressions + + Operations can be negated by calling the negate method. + >>> print(Operation.TRUE.negate()) + Operation.FALSE + >>> print(Operation.IS_NULL.negate()) + Operation.NOT_NULL + + The above example uses the OPERATION_NEGATIONS map which maps each enum + to it's opposite enum. + + Raises: + ValueError: This is raised when attempting to negate an operation + that cannot be negated. + """ + + TRUE = auto() + FALSE = auto() + IS_NULL = auto() + NOT_NULL = auto() + IS_NAN = auto() + NOT_NAN = auto() + LT = auto() + LT_EQ = auto() + GT = auto() + GT_EQ = auto() + EQ = auto() + NOT_EQ = auto() + IN = auto() + NOT_IN = auto() + NOT = auto() + AND = auto() + OR = auto() + + def negate(self): + """Returns the operation used when this is negated.""" + + try: + return OPERATION_NEGATIONS[self] + except KeyError: + raise ValueError(f"No negation defined for operation {self}") + + +OPERATION_NEGATIONS = { + Operation.TRUE: Operation.FALSE, + Operation.FALSE: Operation.TRUE, + Operation.IS_NULL: Operation.NOT_NULL, + Operation.NOT_NULL: Operation.IS_NULL, + Operation.IS_NAN: Operation.NOT_NAN, + Operation.NOT_NAN: Operation.IS_NAN, + Operation.LT: Operation.GT_EQ, + Operation.LT_EQ: Operation.GT, + Operation.GT: Operation.LT_EQ, + Operation.GT_EQ: Operation.LT, + Operation.EQ: Operation.NOT_EQ, + Operation.NOT_EQ: Operation.EQ, + Operation.IN: Operation.NOT_IN, + Operation.NOT_IN: Operation.IN, +} diff --git a/tests/test_expressions.py b/tests/test_expressions.py new file mode 100644 index 0000000000..00f1da1969 --- /dev/null +++ b/tests/test_expressions.py @@ -0,0 +1,58 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +import pytest + +from iceberg import expressions + + +@pytest.mark.parametrize( + "operation,opposite_operation", + [ + (expressions.Operation.TRUE, expressions.Operation.FALSE), + (expressions.Operation.FALSE, expressions.Operation.TRUE), + (expressions.Operation.IS_NULL, expressions.Operation.NOT_NULL), + (expressions.Operation.NOT_NULL, expressions.Operation.IS_NULL), + (expressions.Operation.IS_NAN, expressions.Operation.NOT_NAN), + (expressions.Operation.NOT_NAN, expressions.Operation.IS_NAN), + (expressions.Operation.LT, expressions.Operation.GT_EQ), + (expressions.Operation.LT_EQ, expressions.Operation.GT), + (expressions.Operation.GT, expressions.Operation.LT_EQ), + (expressions.Operation.GT_EQ, expressions.Operation.LT), + (expressions.Operation.EQ, expressions.Operation.NOT_EQ), + (expressions.Operation.NOT_EQ, expressions.Operation.EQ), + (expressions.Operation.IN, expressions.Operation.NOT_IN), + (expressions.Operation.NOT_IN, expressions.Operation.IN), + ], +) +def test_negation_of_operations(operation, opposite_operation): + assert operation.negate() == opposite_operation + + +@pytest.mark.parametrize( + "operation", + [ + expressions.Operation.NOT, + expressions.Operation.AND, + expressions.Operation.OR, + ], +) +def test_raise_on_no_negation_for_operation(operation): + with pytest.raises(ValueError) as exc_info: + operation.negate() + + assert str(exc_info.value) == f"No negation defined for operation {operation}" From b566d8cb9de3a2936b84d1c0a007e103fa5e40a1 Mon Sep 17 00:00:00 2001 From: Kyle Bendickson Date: Sun, 9 Jan 2022 14:35:43 -0800 Subject: [PATCH 055/642] Build: Update NOTICE to include copyright to 2022 (#3855) --- NOTICE | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/NOTICE b/NOTICE index 906eca3c04..d7a3c57526 100644 --- a/NOTICE +++ b/NOTICE @@ -1,6 +1,6 @@ Apache Iceberg -Copyright 2017-2021 The Apache Software Foundation +Copyright 2017-2022 The Apache Software Foundation This product includes software developed at The Apache Software Foundation (http://www.apache.org/). From 00acde052db1fcfd5d6d740d90ec2a42300aa2b9 Mon Sep 17 00:00:00 2001 From: Nick Ouellet <35903677+CircArgs@users.noreply.github.com> Date: Tue, 18 Jan 2022 19:03:34 -0500 Subject: [PATCH 056/642] Python: Expand primitive types to individual classes (#3839) Co-authored-by: Sam Redai --- src/iceberg/types.py | 217 ++++++++++++++++++++++++++++++++++++++++--- tests/test_types.py | 22 ++--- 2 files changed, 215 insertions(+), 24 deletions(-) diff --git a/src/iceberg/types.py b/src/iceberg/types.py index c456c58e4c..a32aa264ce 100644 --- a/src/iceberg/types.py +++ b/src/iceberg/types.py @@ -14,11 +14,27 @@ # KIND, either express or implied. See the License for the # specific language governing permissions and limitations # under the License. +"""Data types used in describing Iceberg schemas + +This module implements the data types described in the Iceberg specification for Iceberg schemas. To +describe an Iceberg table schema, these classes can be used in the construction of a StructType instance. + +Example: + >>> StructType( + [ + NestedField(True, 1, "required_field", StringType()), + NestedField(False, 2, "optional_field", IntegerType()), + ] + ) + +Notes: + - https://iceberg.apache.org/#spec/#primitive-types +""" from typing import Optional -class Type(object): +class Type: def __init__(self, type_string: str, repr_string: str, is_primitive=False): self._type_string = type_string self._repr_string = repr_string @@ -157,15 +173,190 @@ def value(self) -> NestedField: return self._value_field -BooleanType = Type("boolean", "BooleanType", is_primitive=True) -IntegerType = Type("int", "IntegerType", is_primitive=True) -LongType = Type("long", "LongType", is_primitive=True) -FloatType = Type("float", "FloatType", is_primitive=True) -DoubleType = Type("double", "DoubleType", is_primitive=True) -DateType = Type("date", "DateType", is_primitive=True) -TimeType = Type("time", "TimeType", is_primitive=True) -TimestampType = Type("timestamp", "TimestampType", is_primitive=True) -TimestamptzType = Type("timestamptz", "TimestamptzType", is_primitive=True) -StringType = Type("string", "StringType", is_primitive=True) -UUIDType = Type("uuid", "UUIDType", is_primitive=True) -BinaryType = Type("binary", "BinaryType", is_primitive=True) +class BooleanType(Type): + """A boolean data type in Iceberg can be represented using an instance of this class. + + Example: + >>> column_foo = BooleanType() + >>> isinstance(column_foo, BooleanType) + True + """ + + def __init__(self): + super().__init__("boolean", "BooleanType()", is_primitive=True) + + +class IntegerType(Type): + """An Integer data type in Iceberg can be represented using an instance of this class. Integers in Iceberg are + 32-bit signed and can be promoted to Longs. + + Example: + >>> column_foo = IntegerType() + >>> isinstance(column_foo, IntegerType) + True + + Attributes: + max (int): The maximum allowed value for Integers, inherited from the canonical Iceberg implementation + in Java (returns `2147483647`) + min (int): The minimum allowed value for Integers, inherited from the canonical Iceberg implementation + in Java (returns `-2147483648`) + """ + + max: int = 2147483647 + + min: int = -2147483648 + + def __init__(self): + super().__init__("int", "IntegerType()", is_primitive=True) + + +class LongType(Type): + """A Long data type in Iceberg can be represented using an instance of this class. Longs in Iceberg are + 64-bit signed integers. + + Example: + >>> column_foo = LongType() + >>> isinstance(column_foo, LongType) + True + + Attributes: + max (int): The maximum allowed value for Longs, inherited from the canonical Iceberg implementation + in Java. (returns `9223372036854775807`) + min (int): The minimum allowed value for Longs, inherited from the canonical Iceberg implementation + in Java (returns `-9223372036854775808`) + """ + + max: int = 9223372036854775807 + + min: int = -9223372036854775808 + + def __init__(self): + super().__init__("long", "LongType()", is_primitive=True) + + +class FloatType(Type): + """A Float data type in Iceberg can be represented using an instance of this class. Floats in Iceberg are + 32-bit IEEE 754 floating points and can be promoted to Doubles. + + Example: + >>> column_foo = FloatType() + >>> isinstance(column_foo, FloatType) + True + """ + + def __init__(self): + super().__init__("float", "FloatType()", is_primitive=True) + + +class DoubleType(Type): + """A Double data type in Iceberg can be represented using an instance of this class. Doubles in Iceberg are + 64-bit IEEE 754 floating points. + + Example: + >>> column_foo = DoubleType() + >>> isinstance(column_foo, DoubleType) + True + """ + + def __init__(self): + super().__init__("double", "DoubleType()", is_primitive=True) + + +class DateType(Type): + """A Date data type in Iceberg can be represented using an instance of this class. Dates in Iceberg are + calendar dates without a timezone or time. + + Example: + >>> column_foo = DateType() + >>> isinstance(column_foo, DateType) + True + """ + + def __init__(self): + super().__init__("date", "DateType()", is_primitive=True) + + +class TimeType(Type): + """A Time data type in Iceberg can be represented using an instance of this class. Times in Iceberg + have microsecond precision and are a time of day without a date or timezone. + + Example: + >>> column_foo = TimeType() + >>> isinstance(column_foo, TimeType) + True + + """ + + def __init__(self): + super().__init__("time", "TimeType()", is_primitive=True) + + +class TimestampType(Type): + """A Timestamp data type in Iceberg can be represented using an instance of this class. Timestamps in + Iceberg have microsecond precision and include a date and a time of day without a timezone. + + Example: + >>> column_foo = TimestampType() + >>> isinstance(column_foo, TimestampType) + True + + """ + + def __init__(self): + super().__init__("timestamp", "TimestampType()", is_primitive=True) + + +class TimestamptzType(Type): + """A Timestamptz data type in Iceberg can be represented using an instance of this class. Timestamptzs in + Iceberg are stored as UTC and include a date and a time of day with a timezone. + + Example: + >>> column_foo = TimestamptzType() + >>> isinstance(column_foo, TimestamptzType) + True + """ + + def __init__(self): + super().__init__("timestamptz", "TimestamptzType()", is_primitive=True) + + +class StringType(Type): + """A String data type in Iceberg can be represented using an instance of this class. Strings in + Iceberg are arbitrary-length character sequences and are encoded with UTF-8. + + Example: + >>> column_foo = StringType() + >>> isinstance(column_foo, StringType) + True + """ + + def __init__(self): + super().__init__("string", "StringType()", is_primitive=True) + + +class UUIDType(Type): + """A UUID data type in Iceberg can be represented using an instance of this class. UUIDs in + Iceberg are universally unique identifiers. + + Example: + >>> column_foo = UUIDType() + >>> isinstance(column_foo, UUIDType) + True + """ + + def __init__(self): + super().__init__("uuid", "UUIDType()", is_primitive=True) + + +class BinaryType(Type): + """A Binary data type in Iceberg can be represented using an instance of this class. Binarys in + Iceberg are arbitrary-length byte arrays. + + Example: + >>> column_foo = BinaryType() + >>> isinstance(column_foo, BinaryType) + True + """ + + def __init__(self): + super().__init__("binary", "BinaryType()", is_primitive=True) diff --git a/tests/test_types.py b/tests/test_types.py index 3b1d834e05..ea6bbcd045 100644 --- a/tests/test_types.py +++ b/tests/test_types.py @@ -57,7 +57,7 @@ ], ) def test_repr_primitive_types(input_type): - assert input_type == eval(repr(input_type)) + assert isinstance(eval(repr(input_type())), input_type) def test_fixed_type(): @@ -80,7 +80,7 @@ def test_decimal_type(): def test_struct_type(): type_var = StructType( [ - NestedField(True, 1, "optional_field", IntegerType), + NestedField(True, 1, "optional_field", IntegerType()), NestedField(False, 2, "required_field", FixedType(5)), NestedField( False, @@ -89,7 +89,7 @@ def test_struct_type(): StructType( [ NestedField(True, 4, "optional_field", DecimalType(8, 2)), - NestedField(False, 5, "required_field", LongType), + NestedField(False, 5, "required_field", LongType()), ] ), ), @@ -108,7 +108,7 @@ def test_list_type(): StructType( [ NestedField(True, 2, "optional_field", DecimalType(8, 2)), - NestedField(False, 3, "required_field", LongType), + NestedField(False, 3, "required_field", LongType()), ] ), ) @@ -121,12 +121,12 @@ def test_list_type(): def test_map_type(): type_var = MapType( - NestedField(True, 1, "optional_field", DoubleType), - NestedField(False, 2, "required_field", UUIDType), + NestedField(True, 1, "optional_field", DoubleType()), + NestedField(False, 2, "required_field", UUIDType()), ) - assert type_var.key.type == DoubleType + assert isinstance(type_var.key.type, DoubleType) assert type_var.key.field_id == 1 - assert type_var.value.type == UUIDType + assert isinstance(type_var.value.type, UUIDType) assert type_var.value.field_id == 2 assert str(type_var) == str(eval(repr(type_var))) @@ -142,15 +142,15 @@ def test_nested_field(): True, 2, "optional_field2", - ListType(NestedField(False, 3, "required_field3", DoubleType)), + ListType(NestedField(False, 3, "required_field3", DoubleType())), ), NestedField( False, 4, "required_field4", MapType( - NestedField(True, 5, "optional_field5", TimeType), - NestedField(False, 6, "required_field6", UUIDType), + NestedField(True, 5, "optional_field5", TimeType()), + NestedField(False, 6, "required_field6", UUIDType()), ), ), ] From 7d76962f1ac01af2f675a8dd360fb852bcc97113 Mon Sep 17 00:00:00 2001 From: Samuel Redai <43911210+samredai@users.noreply.github.com> Date: Sun, 23 Jan 2022 16:37:31 -0800 Subject: [PATCH 057/642] Python: Add FileIO, InputFile, and OutputFile abstract base classes (#3691) --- src/iceberg/io/base.py | 100 +++++++++++++ tests/io/test_base.py | 316 +++++++++++++++++++++++++++++++++++++++++ 2 files changed, 416 insertions(+) create mode 100644 src/iceberg/io/base.py create mode 100644 tests/io/test_base.py diff --git a/src/iceberg/io/base.py b/src/iceberg/io/base.py new file mode 100644 index 0000000000..705c8b2031 --- /dev/null +++ b/src/iceberg/io/base.py @@ -0,0 +1,100 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +"""Base FileIO classes for implementing reading and writing table files + +The FileIO abstraction includes a subset of full filesystem implementations. Specifically, +Iceberg needs to read or write a file at a given location (as a seekable stream), as well +as check if a file exists. An implementation of the FileIO abstract base class is responsible +for returning an InputFile instance, an OutputFile instance, and deleting a file given +its location. +""" + +from abc import ABC, abstractmethod +from typing import Union + + +class InputFile(ABC): + """A base class for InputFile implementations""" + + def __init__(self, location: str): + self._location = location + + @abstractmethod + def __len__(self) -> int: + """Returns the total length of the file, in bytes""" + + @property + def location(self) -> str: + """The fully-qualified location of the input file""" + return self._location + + @property + @abstractmethod + def exists(self) -> bool: + """Checks whether the file exists""" + + @abstractmethod + def open(self): + """This method should return an instance of an seekable input stream.""" + + +class OutputFile(ABC): + """A base class for OutputFile implementations""" + + def __init__(self, location: str): + self._location = location + + @abstractmethod + def __len__(self) -> int: + """Returns the total length of the file, in bytes""" + + @property + def location(self) -> str: + """The fully-qualified location of the output file""" + return self._location + + @property + @abstractmethod + def exists(self) -> bool: + """Checks whether the file exists""" + + @abstractmethod + def to_input_file(self) -> InputFile: + """Returns an InputFile for the location of this output file""" + + @abstractmethod + def create(self, overwrite: bool = False): + """This method should return a file-like object. + + Args: + overwrite(bool): If the file already exists at `self.location` + and `overwrite` is False a FileExistsError should be raised. + """ + + +class FileIO(ABC): + @abstractmethod + def new_input(self, location: str) -> InputFile: + """Get an InputFile instance to read bytes from the file at the given location""" + + @abstractmethod + def new_output(self, location: str) -> OutputFile: + """Get an OutputFile instance to write bytes to the file at the given location""" + + @abstractmethod + def delete(self, location: Union[str, InputFile, OutputFile]) -> None: + """Delete the file at the given path""" diff --git a/tests/io/test_base.py b/tests/io/test_base.py new file mode 100644 index 0000000000..da08d40761 --- /dev/null +++ b/tests/io/test_base.py @@ -0,0 +1,316 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +import os +import tempfile +from typing import Union +from urllib.parse import ParseResult, urlparse + +import pytest + +from iceberg.io.base import FileIO, InputFile, OutputFile + + +class LocalInputFile(InputFile): + """An InputFile implementation for local files (for test use only)""" + + def __init__(self, location: str): + + parsed_location = urlparse(location) # Create a ParseResult from the uri + if ( + parsed_location.scheme != "file" + ): # Validate that a uri is provided with a scheme of `file` + raise ValueError("LocalInputFile location must have a scheme of `file`") + elif parsed_location.netloc: + raise ValueError( + f"Network location is not allowed for LocalInputFile: {parsed_location.netloc}" + ) + + super().__init__(location=location) + self._parsed_location = parsed_location + + @property + def parsed_location(self) -> ParseResult: + return self._parsed_location + + def __len__(self): + return os.path.getsize(self.parsed_location.path) + + def exists(self): + return os.path.exists(self.parsed_location.path) + + def open(self): + return open(self.parsed_location.path, "rb") + + +class LocalOutputFile(OutputFile): + """An OutputFile implementation for local files (for test use only)""" + + def __init__(self, location: str): + + parsed_location = urlparse(location) # Create a ParseResult from the uri + if ( + parsed_location.scheme != "file" + ): # Validate that a uri is provided with a scheme of `file` + raise ValueError("LocalOutputFile location must have a scheme of `file`") + elif parsed_location.netloc: + raise ValueError( + f"Network location is not allowed for LocalOutputFile: {parsed_location.netloc}" + ) + + super().__init__(location=location) + self._parsed_location = parsed_location + + @property + def parsed_location(self) -> ParseResult: + return self._parsed_location + + def __len__(self): + return os.path.getsize(self.parsed_location.path) + + def exists(self): + return os.path.exists(self.parsed_location.path) + + def to_input_file(self): + return LocalInputFile(location=self.location) + + def create(self, overwrite: bool = False) -> None: + return open(self.parsed_location.path, "wb" if overwrite else "xb") + + +class LocalFileIO(FileIO): + """A FileIO implementation for local files (for test use only)""" + + def new_input(self, location: str): + return LocalInputFile(location=location) + + def new_output(self, location: str): + return LocalOutputFile(location=location) + + def delete(self, location: Union[str, LocalInputFile, LocalOutputFile]): + parsed_location = ( + location.parsed_location + if isinstance(location, (InputFile, OutputFile)) + else urlparse(location) + ) + os.remove(parsed_location.path) + + +@pytest.mark.parametrize("CustomInputFile", [LocalInputFile]) +def test_custom_local_input_file(CustomInputFile): + with tempfile.TemporaryDirectory() as tmpdirname: + file_location = os.path.join(tmpdirname, "foo.txt") + with open(file_location, "wb") as f: + f.write(b"foo") + + # Confirm that the file initially exists + assert os.path.exists(file_location) + + # Instantiate the input file + absolute_file_location = os.path.abspath(file_location) + input_file = CustomInputFile(location=f"file:{absolute_file_location}") + + # Test opening and reading the file + f = input_file.open() + data = f.read() + assert data == b"foo" + assert len(input_file) == 3 + + +@pytest.mark.parametrize("CustomOutputFile", [LocalOutputFile]) +def test_custom_local_output_file(CustomOutputFile): + with tempfile.TemporaryDirectory() as tmpdirname: + file_location = os.path.join(tmpdirname, "foo.txt") + + # Instantiate the output file + absolute_file_location = os.path.abspath(file_location) + output_file = CustomOutputFile(location=f"file:{absolute_file_location}") + + # Create the output file and write to it + f = output_file.create() + f.write(b"foo") + + # Confirm that bytes were written + with open(file_location, "rb") as f: + assert f.read() == b"foo" + + assert len(output_file) == 3 + + +@pytest.mark.parametrize("CustomOutputFile", [LocalOutputFile]) +def test_custom_local_output_file_with_overwrite(CustomOutputFile): + with tempfile.TemporaryDirectory() as tmpdirname: + output_file_location = os.path.join(tmpdirname, "foo.txt") + + # Create a file in the temporary directory + with open(output_file_location, "wb") as f: + f.write(b"foo") + + # Instantiate an output file + output_file = CustomOutputFile(location=f"file://{output_file_location}") + + # Confirm that a FileExistsError is raised when overwrite=False + with pytest.raises(FileExistsError): + f = output_file.create(overwrite=False) + f.write(b"foo") + + # Confirm that the file is overwritten with overwrite=True + f = output_file.create(overwrite=True) + f.write(b"bar") + with open(output_file_location, "rb") as f: + assert f.read() == b"bar" + + +@pytest.mark.parametrize("CustomOutputFile", [LocalOutputFile]) +def test_output_file_to_input_file(CustomOutputFile): + with tempfile.TemporaryDirectory() as tmpdirname: + output_file_location = os.path.join(tmpdirname, "foo.txt") + + # Create an output file instance + output_file = CustomOutputFile(location=f"file://{output_file_location}") + + # Create the output file and write to it + f = output_file.create() + f.write(b"foo") + + # Convert to an input file and confirm the contents + input_file = output_file.to_input_file() + with input_file.open() as f: + assert f.read() == b"foo" + + +@pytest.mark.parametrize( + "CustomFileIO,string_uri,scheme,netloc,path", + [ + (LocalFileIO, "file:///foo/bar.parquet", "file", "", "/foo/bar.parquet"), + (LocalFileIO, "file:/foo/bar/baz.parquet", "file", "", "/foo/bar/baz.parquet"), + ], +) +def test_custom_file_io_locations(CustomFileIO, string_uri, scheme, netloc, path): + # Instantiate the file-io and create a new input and output file + file_io = CustomFileIO() + input_file = file_io.new_input(location=string_uri) + assert input_file.location == string_uri + assert input_file.parsed_location.scheme == scheme + assert input_file.parsed_location.netloc == netloc + assert input_file.parsed_location.path == path + + output_file = file_io.new_output(location=string_uri) + assert output_file.location == string_uri + assert output_file.parsed_location.scheme == scheme + assert output_file.parsed_location.netloc == netloc + assert output_file.parsed_location.path == path + + +@pytest.mark.parametrize( + "string_uri_w_netloc", + ["file://localhost:80/foo/bar.parquet", "file://foo/bar.parquet"], +) +def test_raise_on_network_location_in_InputFile(string_uri_w_netloc): + + with pytest.raises(ValueError) as exc_info: + LocalInputFile(location=string_uri_w_netloc) + + assert ("Network location is not allowed for LocalInputFile") in str(exc_info.value) + + +@pytest.mark.parametrize( + "string_uri_w_netloc", + ["file://localhost:80/foo/bar.parquet", "file://foo/bar.parquet"], +) +def test_raise_on_network_location_in_OutputFile(string_uri_w_netloc): + + with pytest.raises(ValueError) as exc_info: + LocalInputFile(location=string_uri_w_netloc) + + assert ("Network location is not allowed for LocalInputFile") in str(exc_info.value) + + +@pytest.mark.parametrize("CustomFileIO", [LocalFileIO]) +def test_deleting_local_file_using_file_io(CustomFileIO): + + with tempfile.TemporaryDirectory() as tmpdirname: + # Write to the temporary file + output_file_location = os.path.join(tmpdirname, "foo.txt") + with open(output_file_location, "wb") as f: + f.write(b"foo") + + # Instantiate the file-io + file_io = CustomFileIO() + + # Confirm that the file initially exists + assert os.path.exists(output_file_location) + + # Delete the file using the file-io implementations delete method + file_io.delete(output_file_location) + + # Confirm that the file no longer exists + assert not os.path.exists(output_file_location) + + +@pytest.mark.parametrize( + "CustomFileIO, CustomInputFile", [(LocalFileIO, LocalInputFile)] +) +def test_deleting_local_file_using_file_io_InputFile(CustomFileIO, CustomInputFile): + + with tempfile.TemporaryDirectory() as tmpdirname: + # Write to the temporary file + file_location = os.path.join(tmpdirname, "foo.txt") + with open(file_location, "wb") as f: + f.write(b"foo") + + # Instantiate the file-io + file_io = CustomFileIO() + + # Confirm that the file initially exists + assert os.path.exists(file_location) + + # Instantiate the custom InputFile + input_file = CustomInputFile(location=f"file://{file_location}") + + # Delete the file using the file-io implementations delete method + file_io.delete(input_file) + + # Confirm that the file no longer exists + assert not os.path.exists(file_location) + + +@pytest.mark.parametrize( + "CustomFileIO, CustomOutputFile", [(LocalFileIO, LocalOutputFile)] +) +def test_deleting_local_file_using_file_io_OutputFile(CustomFileIO, CustomOutputFile): + + with tempfile.TemporaryDirectory() as tmpdirname: + # Write to the temporary file + file_location = os.path.join(tmpdirname, "foo.txt") + with open(file_location, "wb") as f: + f.write(b"foo") + + # Instantiate the file-io + file_io = CustomFileIO() + + # Confirm that the file initially exists + assert os.path.exists(file_location) + + # Instantiate the custom OutputFile + output_file = CustomOutputFile(location=f"file://{file_location}") + + # Delete the file using the file-io implementations delete method + file_io.delete(output_file) + + # Confirm that the file no longer exists + assert not os.path.exists(file_location) From d6bb5648736831e54ed94da3f14a370bae8a922f Mon Sep 17 00:00:00 2001 From: Samuel Redai <43911210+samredai@users.noreply.github.com> Date: Tue, 1 Feb 2022 12:27:43 -0800 Subject: [PATCH 058/642] Python: Add .python-version file for tox-pyenv support (#4022) --- .python-version | 3 +++ 1 file changed, 3 insertions(+) create mode 100644 .python-version diff --git a/.python-version b/.python-version new file mode 100644 index 0000000000..7ffc3f2f46 --- /dev/null +++ b/.python-version @@ -0,0 +1,3 @@ +3.7.12 +3.8.12 +3.9.10 From 4d22949314b837a1ac8494c41e1072eebb79ccde Mon Sep 17 00:00:00 2001 From: jun-he Date: Wed, 2 Feb 2022 14:13:48 -0800 Subject: [PATCH 059/642] Python: Implement __eq__ for iceberg type classes (#3965) --- src/iceberg/types.py | 75 ++++++++++++++++++++++++++++++++++-------- tests/test_types.py | 78 +++++++++++++++++++++++++++++++++----------- 2 files changed, 121 insertions(+), 32 deletions(-) diff --git a/src/iceberg/types.py b/src/iceberg/types.py index a32aa264ce..42abc4f449 100644 --- a/src/iceberg/types.py +++ b/src/iceberg/types.py @@ -34,6 +34,15 @@ from typing import Optional +class Singleton: + _instance = None + + def __new__(cls, *args, **kwargs): + if not isinstance(cls._instance, cls): + cls._instance = super(Singleton, cls).__new__(cls) + return cls._instance + + class Type: def __init__(self, type_string: str, repr_string: str, is_primitive=False): self._type_string = type_string @@ -62,6 +71,11 @@ def __init__(self, length: int): def length(self) -> int: return self._length + def __eq__(self, other): + if type(self) is type(other): + return self.length == other.length + return False + class DecimalType(Type): def __init__(self, precision: int, scale: int): @@ -81,8 +95,13 @@ def precision(self) -> int: def scale(self) -> int: return self._scale + def __eq__(self, other): + if type(self) is type(other): + return self.precision == other.precision and self.scale == other.scale + return False + -class NestedField(object): +class NestedField: def __init__( self, is_optional: bool, @@ -113,6 +132,10 @@ def field_id(self) -> int: def name(self) -> str: return self._name + @property + def doc(self) -> Optional[str]: + return self._doc + @property def type(self) -> Type: return self._type @@ -131,6 +154,17 @@ def __str__(self): else f" ({self._doc})" ) + def __eq__(self, other): + if type(self) is type(other): + return ( + self.is_optional == other.is_optional + and self.field_id == other.field_id + and self.name == other.name + and self.doc == other.doc + and self.type == other.type + ) + return False + class StructType(Type): def __init__(self, fields: list): @@ -144,6 +178,11 @@ def __init__(self, fields: list): def fields(self) -> list: return self._fields + def __eq__(self, other): + if type(self) is type(other): + return self.fields == other.fields + return False + class ListType(Type): def __init__(self, element: NestedField): @@ -154,6 +193,11 @@ def __init__(self, element: NestedField): def element(self) -> NestedField: return self._element_field + def __eq__(self, other): + if type(self) is type(other): + return self.element == other.element + return False + class MapType(Type): def __init__(self, key: NestedField, value: NestedField): @@ -172,8 +216,13 @@ def key(self) -> NestedField: def value(self) -> NestedField: return self._value_field + def __eq__(self, other): + if type(self) is type(other): + return self.key == other.key and self.value == other.value + return False + -class BooleanType(Type): +class BooleanType(Type, Singleton): """A boolean data type in Iceberg can be represented using an instance of this class. Example: @@ -186,7 +235,7 @@ def __init__(self): super().__init__("boolean", "BooleanType()", is_primitive=True) -class IntegerType(Type): +class IntegerType(Type, Singleton): """An Integer data type in Iceberg can be represented using an instance of this class. Integers in Iceberg are 32-bit signed and can be promoted to Longs. @@ -210,7 +259,7 @@ def __init__(self): super().__init__("int", "IntegerType()", is_primitive=True) -class LongType(Type): +class LongType(Type, Singleton): """A Long data type in Iceberg can be represented using an instance of this class. Longs in Iceberg are 64-bit signed integers. @@ -234,7 +283,7 @@ def __init__(self): super().__init__("long", "LongType()", is_primitive=True) -class FloatType(Type): +class FloatType(Type, Singleton): """A Float data type in Iceberg can be represented using an instance of this class. Floats in Iceberg are 32-bit IEEE 754 floating points and can be promoted to Doubles. @@ -248,7 +297,7 @@ def __init__(self): super().__init__("float", "FloatType()", is_primitive=True) -class DoubleType(Type): +class DoubleType(Type, Singleton): """A Double data type in Iceberg can be represented using an instance of this class. Doubles in Iceberg are 64-bit IEEE 754 floating points. @@ -262,7 +311,7 @@ def __init__(self): super().__init__("double", "DoubleType()", is_primitive=True) -class DateType(Type): +class DateType(Type, Singleton): """A Date data type in Iceberg can be represented using an instance of this class. Dates in Iceberg are calendar dates without a timezone or time. @@ -276,7 +325,7 @@ def __init__(self): super().__init__("date", "DateType()", is_primitive=True) -class TimeType(Type): +class TimeType(Type, Singleton): """A Time data type in Iceberg can be represented using an instance of this class. Times in Iceberg have microsecond precision and are a time of day without a date or timezone. @@ -291,7 +340,7 @@ def __init__(self): super().__init__("time", "TimeType()", is_primitive=True) -class TimestampType(Type): +class TimestampType(Type, Singleton): """A Timestamp data type in Iceberg can be represented using an instance of this class. Timestamps in Iceberg have microsecond precision and include a date and a time of day without a timezone. @@ -306,7 +355,7 @@ def __init__(self): super().__init__("timestamp", "TimestampType()", is_primitive=True) -class TimestamptzType(Type): +class TimestamptzType(Type, Singleton): """A Timestamptz data type in Iceberg can be represented using an instance of this class. Timestamptzs in Iceberg are stored as UTC and include a date and a time of day with a timezone. @@ -320,7 +369,7 @@ def __init__(self): super().__init__("timestamptz", "TimestamptzType()", is_primitive=True) -class StringType(Type): +class StringType(Type, Singleton): """A String data type in Iceberg can be represented using an instance of this class. Strings in Iceberg are arbitrary-length character sequences and are encoded with UTF-8. @@ -334,7 +383,7 @@ def __init__(self): super().__init__("string", "StringType()", is_primitive=True) -class UUIDType(Type): +class UUIDType(Type, Singleton): """A UUID data type in Iceberg can be represented using an instance of this class. UUIDs in Iceberg are universally unique identifiers. @@ -348,7 +397,7 @@ def __init__(self): super().__init__("uuid", "UUIDType()", is_primitive=True) -class BinaryType(Type): +class BinaryType(Type, Singleton): """A Binary data type in Iceberg can be represented using an instance of this class. Binarys in Iceberg are arbitrary-length byte arrays. diff --git a/tests/test_types.py b/tests/test_types.py index ea6bbcd045..af474cd86e 100644 --- a/tests/test_types.py +++ b/tests/test_types.py @@ -38,25 +38,24 @@ UUIDType, ) - -@pytest.mark.parametrize( - "input_type", - [ - BooleanType, - IntegerType, - LongType, - FloatType, - DoubleType, - DateType, - TimeType, - TimestampType, - TimestamptzType, - StringType, - UUIDType, - BinaryType, - ], -) -def test_repr_primitive_types(input_type): +non_parameterized_types = [ + (1, BooleanType), + (2, IntegerType), + (3, LongType), + (4, FloatType), + (5, DoubleType), + (6, DateType), + (7, TimeType), + (8, TimestampType), + (9, TimestamptzType), + (10, StringType), + (11, UUIDType), + (12, BinaryType), +] + + +@pytest.mark.parametrize("input_index, input_type", non_parameterized_types) +def test_repr_primitive_types(input_index, input_type): assert isinstance(eval(repr(input_type())), input_type) @@ -66,6 +65,8 @@ def test_fixed_type(): assert str(type_var) == "fixed[5]" assert repr(type_var) == "FixedType(length=5)" assert str(type_var) == str(eval(repr(type_var))) + assert type_var == FixedType(5) + assert type_var != FixedType(6) def test_decimal_type(): @@ -75,6 +76,8 @@ def test_decimal_type(): assert str(type_var) == "decimal(9, 2)" assert repr(type_var) == "DecimalType(precision=9, scale=2)" assert str(type_var) == str(eval(repr(type_var))) + assert type_var == DecimalType(9, 2) + assert type_var != DecimalType(9, 3) def test_struct_type(): @@ -97,6 +100,10 @@ def test_struct_type(): ) assert len(type_var.fields) == 3 assert str(type_var) == str(eval(repr(type_var))) + assert type_var == eval(repr(type_var)) + assert type_var != StructType( + [NestedField(True, 1, "optional_field", IntegerType())] + ) def test_list_type(): @@ -117,6 +124,19 @@ def test_list_type(): assert len(type_var.element.type.fields) == 2 assert type_var.element.field_id == 1 assert str(type_var) == str(eval(repr(type_var))) + assert type_var == eval(repr(type_var)) + assert type_var != ListType( + NestedField( + True, + 1, + "required_field", + StructType( + [ + NestedField(True, 2, "optional_field", DecimalType(8, 2)), + ] + ), + ) + ) def test_map_type(): @@ -129,6 +149,15 @@ def test_map_type(): assert isinstance(type_var.value.type, UUIDType) assert type_var.value.field_id == 2 assert str(type_var) == str(eval(repr(type_var))) + assert type_var == eval(repr(type_var)) + assert type_var != MapType( + NestedField(True, 1, "optional_field", LongType()), + NestedField(False, 2, "required_field", UUIDType()), + ) + assert type_var != MapType( + NestedField(True, 1, "optional_field", DoubleType()), + NestedField(False, 2, "required_field", StringType()), + ) def test_nested_field(): @@ -161,3 +190,14 @@ def test_nested_field(): assert field_var.field_id == 1 assert isinstance(field_var.type, StructType) assert str(field_var) == str(eval(repr(field_var))) + + +@pytest.mark.parametrize("input_index,input_type", non_parameterized_types) +@pytest.mark.parametrize("check_index,check_type", non_parameterized_types) +def test_non_parameterized_type_equality( + input_index, input_type, check_index, check_type +): + if input_index == check_index: + assert input_type() == check_type() + else: + assert input_type() != check_type() From f21e883765286a6f2a793ba1d3d4f0af5be311eb Mon Sep 17 00:00:00 2001 From: Samuel Redai <43911210+samredai@users.noreply.github.com> Date: Wed, 2 Feb 2022 14:25:04 -0800 Subject: [PATCH 060/642] Python: Set black line-length to 130 in tox settings (#4025) --- src/iceberg/types.py | 7 ++----- src/iceberg/utils/bin_packing.py | 4 +--- tests/io/test_base.py | 30 +++++++----------------------- tests/utils/test_bin_packing.py | 15 +++------------ tox.ini | 4 ++-- 5 files changed, 15 insertions(+), 45 deletions(-) diff --git a/src/iceberg/types.py b/src/iceberg/types.py index 42abc4f449..5a8ce2b911 100644 --- a/src/iceberg/types.py +++ b/src/iceberg/types.py @@ -62,9 +62,7 @@ def is_primitive(self) -> bool: class FixedType(Type): def __init__(self, length: int): - super().__init__( - f"fixed[{length}]", f"FixedType(length={length})", is_primitive=True - ) + super().__init__(f"fixed[{length}]", f"FixedType(length={length})", is_primitive=True) self._length = length @property @@ -148,8 +146,7 @@ def __repr__(self): def __str__(self): return ( - f"{self._id}: {self._name}: {'optional' if self._is_optional else 'required'} {self._type}" - "" + f"{self._id}: {self._name}: {'optional' if self._is_optional else 'required'} {self._type}" "" if self._doc is None else f" ({self._doc})" ) diff --git a/src/iceberg/utils/bin_packing.py b/src/iceberg/utils/bin_packing.py index 8e32710028..e3577f2198 100644 --- a/src/iceberg/utils/bin_packing.py +++ b/src/iceberg/utils/bin_packing.py @@ -17,9 +17,7 @@ class PackingIterator: - def __init__( - self, items, target_weight, lookback, weight_func, largest_bin_first=False - ): + def __init__(self, items, target_weight, lookback, weight_func, largest_bin_first=False): self.items = iter(items) self.target_weight = target_weight self.lookback = lookback diff --git a/tests/io/test_base.py b/tests/io/test_base.py index da08d40761..bf67bc62ee 100644 --- a/tests/io/test_base.py +++ b/tests/io/test_base.py @@ -31,14 +31,10 @@ class LocalInputFile(InputFile): def __init__(self, location: str): parsed_location = urlparse(location) # Create a ParseResult from the uri - if ( - parsed_location.scheme != "file" - ): # Validate that a uri is provided with a scheme of `file` + if parsed_location.scheme != "file": # Validate that a uri is provided with a scheme of `file` raise ValueError("LocalInputFile location must have a scheme of `file`") elif parsed_location.netloc: - raise ValueError( - f"Network location is not allowed for LocalInputFile: {parsed_location.netloc}" - ) + raise ValueError(f"Network location is not allowed for LocalInputFile: {parsed_location.netloc}") super().__init__(location=location) self._parsed_location = parsed_location @@ -63,14 +59,10 @@ class LocalOutputFile(OutputFile): def __init__(self, location: str): parsed_location = urlparse(location) # Create a ParseResult from the uri - if ( - parsed_location.scheme != "file" - ): # Validate that a uri is provided with a scheme of `file` + if parsed_location.scheme != "file": # Validate that a uri is provided with a scheme of `file` raise ValueError("LocalOutputFile location must have a scheme of `file`") elif parsed_location.netloc: - raise ValueError( - f"Network location is not allowed for LocalOutputFile: {parsed_location.netloc}" - ) + raise ValueError(f"Network location is not allowed for LocalOutputFile: {parsed_location.netloc}") super().__init__(location=location) self._parsed_location = parsed_location @@ -102,11 +94,7 @@ def new_output(self, location: str): return LocalOutputFile(location=location) def delete(self, location: Union[str, LocalInputFile, LocalOutputFile]): - parsed_location = ( - location.parsed_location - if isinstance(location, (InputFile, OutputFile)) - else urlparse(location) - ) + parsed_location = location.parsed_location if isinstance(location, (InputFile, OutputFile)) else urlparse(location) os.remove(parsed_location.path) @@ -262,9 +250,7 @@ def test_deleting_local_file_using_file_io(CustomFileIO): assert not os.path.exists(output_file_location) -@pytest.mark.parametrize( - "CustomFileIO, CustomInputFile", [(LocalFileIO, LocalInputFile)] -) +@pytest.mark.parametrize("CustomFileIO, CustomInputFile", [(LocalFileIO, LocalInputFile)]) def test_deleting_local_file_using_file_io_InputFile(CustomFileIO, CustomInputFile): with tempfile.TemporaryDirectory() as tmpdirname: @@ -289,9 +275,7 @@ def test_deleting_local_file_using_file_io_InputFile(CustomFileIO, CustomInputFi assert not os.path.exists(file_location) -@pytest.mark.parametrize( - "CustomFileIO, CustomOutputFile", [(LocalFileIO, LocalOutputFile)] -) +@pytest.mark.parametrize("CustomFileIO, CustomOutputFile", [(LocalFileIO, LocalOutputFile)]) def test_deleting_local_file_using_file_io_OutputFile(CustomFileIO, CustomOutputFile): with tempfile.TemporaryDirectory() as tmpdirname: diff --git a/tests/utils/test_bin_packing.py b/tests/utils/test_bin_packing.py index 7a5a0749fe..fba8d32983 100644 --- a/tests/utils/test_bin_packing.py +++ b/tests/utils/test_bin_packing.py @@ -39,9 +39,7 @@ def test_bin_packing(splits, lookback, split_size, open_cost): def weight_func(x): return max(x, open_cost) - item_list_sums = [ - sum(item) for item in PackingIterator(splits, split_size, lookback, weight_func) - ] + item_list_sums = [sum(item) for item in PackingIterator(splits, split_size, lookback, weight_func)] assert all([split_size >= item_sum >= 0 for item_sum in item_list_sums]) @@ -78,15 +76,8 @@ def weight_func(x): ), ], ) -def test_bin_packing_lookback( - splits, target_weight, lookback, largest_bin_first, expected_lists -): +def test_bin_packing_lookback(splits, target_weight, lookback, largest_bin_first, expected_lists): def weight_func(x): return x - assert [ - item - for item in PackingIterator( - splits, target_weight, lookback, weight_func, largest_bin_first - ) - ] == expected_lists + assert [item for item in PackingIterator(splits, target_weight, lookback, weight_func, largest_bin_first)] == expected_lists diff --git a/tox.ini b/tox.ini index ed9cb78c44..ad29bfaf15 100644 --- a/tox.ini +++ b/tox.ini @@ -51,7 +51,7 @@ deps = commands = autoflake -r --check --ignore-init-module-imports --remove-all-unused-imports setup.py src tests isort --profile black --check-only setup.py src tests - black --check setup.py src tests + black --line-length 130 --check setup.py src tests [testenv:format] deps = @@ -59,7 +59,7 @@ deps = commands = autoflake -r --in-place --ignore-init-module-imports --remove-all-unused-imports setup.py src tests isort --profile black setup.py src tests - black setup.py src tests + black --line-length 130 setup.py src tests [testenv:type-check] deps = From b3ac46b45fe7c2362d8729876be6fbd0ec399e7b Mon Sep 17 00:00:00 2001 From: Samuel Redai <43911210+samredai@users.noreply.github.com> Date: Thu, 3 Feb 2022 12:35:49 -0500 Subject: [PATCH 061/642] Python: Add --diff to show the line diff when black fails (#4034) --- tox.ini | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tox.ini b/tox.ini index ad29bfaf15..b6b894f70c 100644 --- a/tox.ini +++ b/tox.ini @@ -51,7 +51,7 @@ deps = commands = autoflake -r --check --ignore-init-module-imports --remove-all-unused-imports setup.py src tests isort --profile black --check-only setup.py src tests - black --line-length 130 --check setup.py src tests + black --line-length 130 --check --diff setup.py src tests [testenv:format] deps = From 4f80490672c3687dd1c6e1d4996d1b261d935e95 Mon Sep 17 00:00:00 2001 From: Samuel Redai <43911210+samredai@users.noreply.github.com> Date: Thu, 3 Feb 2022 12:39:22 -0500 Subject: [PATCH 062/642] Python: Autoformat test_types.py with new line-length (#4032) --- tests/test_types.py | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/tests/test_types.py b/tests/test_types.py index af474cd86e..5a6a411d8a 100644 --- a/tests/test_types.py +++ b/tests/test_types.py @@ -101,9 +101,7 @@ def test_struct_type(): assert len(type_var.fields) == 3 assert str(type_var) == str(eval(repr(type_var))) assert type_var == eval(repr(type_var)) - assert type_var != StructType( - [NestedField(True, 1, "optional_field", IntegerType())] - ) + assert type_var != StructType([NestedField(True, 1, "optional_field", IntegerType())]) def test_list_type(): @@ -194,9 +192,7 @@ def test_nested_field(): @pytest.mark.parametrize("input_index,input_type", non_parameterized_types) @pytest.mark.parametrize("check_index,check_type", non_parameterized_types) -def test_non_parameterized_type_equality( - input_index, input_type, check_index, check_type -): +def test_non_parameterized_type_equality(input_index, input_type, check_index, check_type): if input_index == check_index: assert input_type() == check_type() else: From 61811c8a53a17dd7a05a8afc9daa8db16d979c38 Mon Sep 17 00:00:00 2001 From: Samuel Redai <43911210+samredai@users.noreply.github.com> Date: Wed, 9 Feb 2022 19:54:45 -0500 Subject: [PATCH 063/642] Python: Add InputStream and OutputStream protocols for FileIO streams (#4021) --- src/iceberg/io/base.py | 61 +++++++++++++++++++++++++++++++++++++++--- tests/io/test_base.py | 16 +++++++---- 2 files changed, 68 insertions(+), 9 deletions(-) diff --git a/src/iceberg/io/base.py b/src/iceberg/io/base.py index 705c8b2031..8bb905a86d 100644 --- a/src/iceberg/io/base.py +++ b/src/iceberg/io/base.py @@ -26,6 +26,54 @@ from abc import ABC, abstractmethod from typing import Union +try: + from typing import Protocol, runtime_checkable +except ImportError: # pragma: no cover + from typing_extensions import Protocol # type: ignore + from typing_extensions import runtime_checkable + + +@runtime_checkable +class InputStream(Protocol): # pragma: no cover + """A protocol for the file-like object returned by InputFile.open(...) + + This outlines the minimally required methods for a seekable input stream returned from an InputFile + implementation's `open(...)` method. These methods are a subset of IOBase/RawIOBase. + """ + + def read(self, size: int) -> bytes: + ... + + def seek(self, offset: int, whence: int) -> None: + ... + + def tell(self) -> int: + ... + + def closed(self) -> bool: + ... + + def close(self) -> None: + ... + + +@runtime_checkable +class OutputStream(Protocol): # pragma: no cover + """A protocol for the file-like object returned by OutputFile.create(...) + + This outlines the minimally required methods for a writable output stream returned from an OutputFile + implementation's `create(...)` method. These methods are a subset of IOBase/RawIOBase. + """ + + def write(self, b: bytes) -> None: + ... + + def closed(self) -> bool: + ... + + def close(self) -> None: + ... + class InputFile(ABC): """A base class for InputFile implementations""" @@ -48,8 +96,11 @@ def exists(self) -> bool: """Checks whether the file exists""" @abstractmethod - def open(self): - """This method should return an instance of an seekable input stream.""" + def open(self) -> InputStream: + """This method should return an object that matches the InputStream protocol + + If a file does not exist at `self.location`, this should raise a FileNotFoundError. + """ class OutputFile(ABC): @@ -77,8 +128,8 @@ def to_input_file(self) -> InputFile: """Returns an InputFile for the location of this output file""" @abstractmethod - def create(self, overwrite: bool = False): - """This method should return a file-like object. + def create(self, overwrite: bool = False) -> OutputStream: + """This method should return an object that matches the OutputStream protocol. Args: overwrite(bool): If the file already exists at `self.location` @@ -87,6 +138,8 @@ def create(self, overwrite: bool = False): class FileIO(ABC): + """A base class for FileIO implementations""" + @abstractmethod def new_input(self, location: str) -> InputFile: """Get an InputFile instance to read bytes from the file at the given location""" diff --git a/tests/io/test_base.py b/tests/io/test_base.py index bf67bc62ee..6a7773cb67 100644 --- a/tests/io/test_base.py +++ b/tests/io/test_base.py @@ -22,7 +22,7 @@ import pytest -from iceberg.io.base import FileIO, InputFile, OutputFile +from iceberg.io.base import FileIO, InputFile, InputStream, OutputFile, OutputStream class LocalInputFile(InputFile): @@ -49,8 +49,11 @@ def __len__(self): def exists(self): return os.path.exists(self.parsed_location.path) - def open(self): - return open(self.parsed_location.path, "rb") + def open(self) -> InputStream: + input_file = open(self.parsed_location.path, "rb") + if not isinstance(input_file, InputStream): + raise TypeError("""Object returned from LocalInputFile.open does not match the OutputStream protocol.""") + return input_file class LocalOutputFile(OutputFile): @@ -80,8 +83,11 @@ def exists(self): def to_input_file(self): return LocalInputFile(location=self.location) - def create(self, overwrite: bool = False) -> None: - return open(self.parsed_location.path, "wb" if overwrite else "xb") + def create(self, overwrite: bool = False) -> OutputStream: + output_file = open(self.parsed_location.path, "wb" if overwrite else "xb") + if not isinstance(output_file, OutputStream): + raise TypeError("""Object returned from LocalOutputFile.create does not match the OutputStream protocol.""") + return output_file class LocalFileIO(FileIO): From c6c7d380d59fbd120fefc318ba4b58ee6d3a884b Mon Sep 17 00:00:00 2001 From: Nick Ouellet <35903677+CircArgs@users.noreply.github.com> Date: Tue, 22 Feb 2022 12:43:07 -0500 Subject: [PATCH 064/642] Python: Reuse types with __new__ (#4016) --- src/iceberg/types.py | 345 +++++++++++++++++++++++++++++-------------- tests/test_types.py | 136 +++++++++-------- 2 files changed, 302 insertions(+), 179 deletions(-) diff --git a/src/iceberg/types.py b/src/iceberg/types.py index 5a8ce2b911..af1f595663 100644 --- a/src/iceberg/types.py +++ b/src/iceberg/types.py @@ -21,17 +21,15 @@ Example: >>> StructType( - [ NestedField(True, 1, "required_field", StringType()), - NestedField(False, 2, "optional_field", IntegerType()), - ] + NestedField(False, 2, "optional_field", IntegerType()) ) Notes: - https://iceberg.apache.org/#spec/#primitive-types """ -from typing import Optional +from typing import Dict, Optional, Tuple class Singleton: @@ -43,11 +41,15 @@ def __new__(cls, *args, **kwargs): return cls._instance -class Type: - def __init__(self, type_string: str, repr_string: str, is_primitive=False): +class IcebergType: + """Base type for all Iceberg Types""" + + _initialized = False + + def __init__(self, type_string: str, repr_string: str): self._type_string = type_string self._repr_string = repr_string - self._is_primitive = is_primitive + self._initialized = True def __repr__(self): return self._repr_string @@ -57,33 +59,64 @@ def __str__(self): @property def is_primitive(self) -> bool: - return self._is_primitive + return isinstance(self, PrimitiveType) + + +class PrimitiveType(IcebergType): + """Base class for all Iceberg Primitive Types""" + + +class FixedType(PrimitiveType): + """A fixed data type in Iceberg. + + Example: + >>> FixedType(8) + FixedType(length=8) + >>> FixedType(8)==FixedType(8) + True + """ + _instances: Dict[int, "FixedType"] = {} + + def __new__(cls, length: int): + cls._instances[length] = cls._instances.get(length) or object.__new__(cls) + return cls._instances[length] -class FixedType(Type): def __init__(self, length: int): - super().__init__(f"fixed[{length}]", f"FixedType(length={length})", is_primitive=True) - self._length = length + if not self._initialized: + super().__init__(f"fixed[{length}]", f"FixedType(length={length})") + self._length = length @property def length(self) -> int: return self._length - def __eq__(self, other): - if type(self) is type(other): - return self.length == other.length - return False +class DecimalType(PrimitiveType): + """A fixed data type in Iceberg. + + Example: + >>> DecimalType(32, 3) + DecimalType(precision=32, scale=3) + >>> DecimalType(8, 3)==DecimalType(8, 3) + True + """ + + _instances: Dict[Tuple[int, int], "DecimalType"] = {} + + def __new__(cls, precision: int, scale: int): + key = (precision, scale) + cls._instances[key] = cls._instances.get(key) or object.__new__(cls) + return cls._instances[key] -class DecimalType(Type): def __init__(self, precision: int, scale: int): - super().__init__( - f"decimal({precision}, {scale})", - f"DecimalType(precision={precision}, scale={scale})", - is_primitive=True, - ) - self._precision = precision - self._scale = scale + if not self._initialized: + super().__init__( + f"decimal({precision}, {scale})", + f"DecimalType(precision={precision}, scale={scale})", + ) + self._precision = precision + self._scale = scale @property def precision(self) -> int: @@ -93,26 +126,51 @@ def precision(self) -> int: def scale(self) -> int: return self._scale - def __eq__(self, other): - if type(self) is type(other): - return self.precision == other.precision and self.scale == other.scale - return False +class NestedField(IcebergType): + """Represents a field of a struct, a map key, a map value, or a list element. + + This is where field IDs, names, docs, and nullability are tracked. + """ + + _instances: Dict[Tuple[bool, int, str, IcebergType, Optional[str]], "NestedField"] = {} + + def __new__( + cls, + field_id: int, + name: str, + field_type: IcebergType, + is_optional: bool = True, + doc: Optional[str] = None, + ): + key = (is_optional, field_id, name, field_type, doc) + cls._instances[key] = cls._instances.get(key) or object.__new__(cls) + return cls._instances[key] -class NestedField: def __init__( self, - is_optional: bool, field_id: int, name: str, - field_type: Type, + field_type: IcebergType, + is_optional: bool = True, doc: Optional[str] = None, ): - self._is_optional = is_optional - self._id = field_id - self._name = name - self._type = field_type - self._doc = doc + if not self._initialized: + docString = "" if doc is None else f", doc={repr(doc)}" + super().__init__( + ( + f"{field_id}: {name}: {'optional' if is_optional else 'required'} {field_type}" "" + if doc is None + else f" ({doc})" + ), + f"NestedField(field_id={field_id}, name={repr(name)}, field_type={repr(field_type)}, is_optional={is_optional}" + f"{docString})", + ) + self._is_optional = is_optional + self._id = field_id + self._name = name + self._type = field_type + self._doc = doc @property def is_optional(self) -> bool: @@ -135,75 +193,129 @@ def doc(self) -> Optional[str]: return self._doc @property - def type(self) -> Type: + def type(self) -> IcebergType: return self._type - def __repr__(self): - return ( - f"NestedField(is_optional={self._is_optional}, field_id={self._id}, " - f"name={repr(self._name)}, field_type={repr(self._type)}, doc={repr(self._doc)})" - ) - def __str__(self): - return ( - f"{self._id}: {self._name}: {'optional' if self._is_optional else 'required'} {self._type}" "" - if self._doc is None - else f" ({self._doc})" +class StructType(IcebergType): + """A struct type in Iceberg + + Example: + >>> StructType( + NestedField(True, 1, "required_field", StringType()), + NestedField(False, 2, "optional_field", IntegerType()) ) + """ - def __eq__(self, other): - if type(self) is type(other): - return ( - self.is_optional == other.is_optional - and self.field_id == other.field_id - and self.name == other.name - and self.doc == other.doc - and self.type == other.type - ) - return False + _instances: Dict[Tuple[NestedField, ...], "StructType"] = {} + def __new__(cls, *fields: NestedField): + cls._instances[fields] = cls._instances.get(fields) or object.__new__(cls) + return cls._instances[fields] -class StructType(Type): - def __init__(self, fields: list): - super().__init__( - f"struct<{', '.join(map(str, fields))}>", - f"StructType(fields={repr(fields)})", - ) - self._fields = fields + def __init__(self, *fields: NestedField): + if not self._initialized: + super().__init__( + f"struct<{', '.join(map(str, fields))}>", + f"StructType{repr(fields)}", + ) + self._fields = fields @property - def fields(self) -> list: + def fields(self) -> Tuple[NestedField, ...]: return self._fields - def __eq__(self, other): - if type(self) is type(other): - return self.fields == other.fields - return False +class ListType(IcebergType): + """A list type in Iceberg + + Example: + >>> ListType(element_id=3, element_type=StringType(), element_is_optional=True) + ListType(element=NestedField(is_optional=True, field_id=3, name='element', field_type=StringType(), doc=None)) + """ + + _instances: Dict[Tuple[bool, int, IcebergType], "ListType"] = {} -class ListType(Type): - def __init__(self, element: NestedField): - super().__init__(f"list<{element.type}>", f"ListType(element={repr(element)})") - self._element_field = element + def __new__( + cls, + element_id: int, + element_type: IcebergType, + element_is_optional: bool = True, + ): + key = (element_is_optional, element_id, element_type) + cls._instances[key] = cls._instances.get(key) or object.__new__(cls) + return cls._instances[key] + + def __init__( + self, + element_id: int, + element_type: IcebergType, + element_is_optional: bool = True, + ): + if not self._initialized: + super().__init__( + f"list<{element_type}>", + f"ListType(element_id={element_id}, element_type={repr(element_type)}, " + f"element_is_optional={element_is_optional})", + ) + self._element_field = NestedField( + name="element", + is_optional=element_is_optional, + field_id=element_id, + field_type=element_type, + ) @property def element(self) -> NestedField: return self._element_field - def __eq__(self, other): - if type(self) is type(other): - return self.element == other.element - return False +class MapType(IcebergType): + """A map type in Iceberg -class MapType(Type): - def __init__(self, key: NestedField, value: NestedField): - super().__init__( - f"map<{key.type}, {value.type}>", - f"MapType(key={repr(key)}, value={repr(value)})", + Example: + >>> MapType(key_id=1, key_type=StringType(), value_id=2, value_type=IntegerType(), value_is_optional=True) + MapType( + key=NestedField(is_optional=False, field_id=1, name='key', field_type=StringType(), doc=None), + value=NestedField(is_optional=True, field_id=2, name='value', field_type=IntegerType(), doc=None) ) - self._key_field = key - self._value_field = value + """ + + _instances: Dict[Tuple[int, IcebergType, int, IcebergType, bool], "MapType"] = {} + + def __new__( + cls, + key_id: int, + key_type: IcebergType, + value_id: int, + value_type: IcebergType, + value_is_optional: bool = True, + ): + impl_key = (key_id, key_type, value_id, value_type, value_is_optional) + cls._instances[impl_key] = cls._instances.get(impl_key) or object.__new__(cls) + return cls._instances[impl_key] + + def __init__( + self, + key_id: int, + key_type: IcebergType, + value_id: int, + value_type: IcebergType, + value_is_optional: bool = True, + ): + if not self._initialized: + super().__init__( + f"map<{key_type}, {value_type}>", + f"MapType(key_id={key_id}, key_type={repr(key_type)}, value_id={value_id}, value_type={repr(value_type)}, " + f"value_is_optional={value_is_optional})", + ) + self._key_field = NestedField(name="key", field_id=key_id, field_type=key_type, is_optional=False) + self._value_field = NestedField( + name="value", + field_id=value_id, + field_type=value_type, + is_optional=value_is_optional, + ) @property def key(self) -> NestedField: @@ -213,13 +325,8 @@ def key(self) -> NestedField: def value(self) -> NestedField: return self._value_field - def __eq__(self, other): - if type(self) is type(other): - return self.key == other.key and self.value == other.value - return False - -class BooleanType(Type, Singleton): +class BooleanType(PrimitiveType, Singleton): """A boolean data type in Iceberg can be represented using an instance of this class. Example: @@ -229,10 +336,11 @@ class BooleanType(Type, Singleton): """ def __init__(self): - super().__init__("boolean", "BooleanType()", is_primitive=True) + if not self._initialized: + super().__init__("boolean", "BooleanType()") -class IntegerType(Type, Singleton): +class IntegerType(PrimitiveType, Singleton): """An Integer data type in Iceberg can be represented using an instance of this class. Integers in Iceberg are 32-bit signed and can be promoted to Longs. @@ -253,10 +361,11 @@ class IntegerType(Type, Singleton): min: int = -2147483648 def __init__(self): - super().__init__("int", "IntegerType()", is_primitive=True) + if not self._initialized: + super().__init__("int", "IntegerType()") -class LongType(Type, Singleton): +class LongType(PrimitiveType, Singleton): """A Long data type in Iceberg can be represented using an instance of this class. Longs in Iceberg are 64-bit signed integers. @@ -277,10 +386,11 @@ class LongType(Type, Singleton): min: int = -9223372036854775808 def __init__(self): - super().__init__("long", "LongType()", is_primitive=True) + if not self._initialized: + super().__init__("long", "LongType()") -class FloatType(Type, Singleton): +class FloatType(PrimitiveType, Singleton): """A Float data type in Iceberg can be represented using an instance of this class. Floats in Iceberg are 32-bit IEEE 754 floating points and can be promoted to Doubles. @@ -291,10 +401,11 @@ class FloatType(Type, Singleton): """ def __init__(self): - super().__init__("float", "FloatType()", is_primitive=True) + if not self._initialized: + super().__init__("float", "FloatType()") -class DoubleType(Type, Singleton): +class DoubleType(PrimitiveType, Singleton): """A Double data type in Iceberg can be represented using an instance of this class. Doubles in Iceberg are 64-bit IEEE 754 floating points. @@ -305,10 +416,11 @@ class DoubleType(Type, Singleton): """ def __init__(self): - super().__init__("double", "DoubleType()", is_primitive=True) + if not self._initialized: + super().__init__("double", "DoubleType()") -class DateType(Type, Singleton): +class DateType(PrimitiveType, Singleton): """A Date data type in Iceberg can be represented using an instance of this class. Dates in Iceberg are calendar dates without a timezone or time. @@ -319,10 +431,11 @@ class DateType(Type, Singleton): """ def __init__(self): - super().__init__("date", "DateType()", is_primitive=True) + if not self._initialized: + super().__init__("date", "DateType()") -class TimeType(Type, Singleton): +class TimeType(PrimitiveType, Singleton): """A Time data type in Iceberg can be represented using an instance of this class. Times in Iceberg have microsecond precision and are a time of day without a date or timezone. @@ -330,14 +443,14 @@ class TimeType(Type, Singleton): >>> column_foo = TimeType() >>> isinstance(column_foo, TimeType) True - """ def __init__(self): - super().__init__("time", "TimeType()", is_primitive=True) + if not self._initialized: + super().__init__("time", "TimeType()") -class TimestampType(Type, Singleton): +class TimestampType(PrimitiveType, Singleton): """A Timestamp data type in Iceberg can be represented using an instance of this class. Timestamps in Iceberg have microsecond precision and include a date and a time of day without a timezone. @@ -345,14 +458,14 @@ class TimestampType(Type, Singleton): >>> column_foo = TimestampType() >>> isinstance(column_foo, TimestampType) True - """ def __init__(self): - super().__init__("timestamp", "TimestampType()", is_primitive=True) + if not self._initialized: + super().__init__("timestamp", "TimestampType()") -class TimestamptzType(Type, Singleton): +class TimestamptzType(PrimitiveType, Singleton): """A Timestamptz data type in Iceberg can be represented using an instance of this class. Timestamptzs in Iceberg are stored as UTC and include a date and a time of day with a timezone. @@ -363,10 +476,11 @@ class TimestamptzType(Type, Singleton): """ def __init__(self): - super().__init__("timestamptz", "TimestamptzType()", is_primitive=True) + if not self._initialized: + super().__init__("timestamptz", "TimestamptzType()") -class StringType(Type, Singleton): +class StringType(PrimitiveType, Singleton): """A String data type in Iceberg can be represented using an instance of this class. Strings in Iceberg are arbitrary-length character sequences and are encoded with UTF-8. @@ -377,10 +491,11 @@ class StringType(Type, Singleton): """ def __init__(self): - super().__init__("string", "StringType()", is_primitive=True) + if not self._initialized: + super().__init__("string", "StringType()") -class UUIDType(Type, Singleton): +class UUIDType(PrimitiveType, Singleton): """A UUID data type in Iceberg can be represented using an instance of this class. UUIDs in Iceberg are universally unique identifiers. @@ -391,10 +506,11 @@ class UUIDType(Type, Singleton): """ def __init__(self): - super().__init__("uuid", "UUIDType()", is_primitive=True) + if not self._initialized: + super().__init__("uuid", "UUIDType()") -class BinaryType(Type, Singleton): +class BinaryType(PrimitiveType, Singleton): """A Binary data type in Iceberg can be represented using an instance of this class. Binarys in Iceberg are arbitrary-length byte arrays. @@ -405,4 +521,5 @@ class BinaryType(Type, Singleton): """ def __init__(self): - super().__init__("binary", "BinaryType()", is_primitive=True) + if not self._initialized: + super().__init__("binary", "BinaryType()") diff --git a/tests/test_types.py b/tests/test_types.py index 5a6a411d8a..07d3bcd0cf 100644 --- a/tests/test_types.py +++ b/tests/test_types.py @@ -59,6 +59,42 @@ def test_repr_primitive_types(input_index, input_type): assert isinstance(eval(repr(input_type())), input_type) +@pytest.mark.parametrize( + "input_type, result", + [ + (BooleanType(), True), + (IntegerType(), True), + (LongType(), True), + (FloatType(), True), + (DoubleType(), True), + (DateType(), True), + (TimeType(), True), + (TimestampType(), True), + (TimestamptzType(), True), + (StringType(), True), + (UUIDType(), True), + (BinaryType(), True), + (DecimalType(32, 3), True), + (FixedType(8), True), + (ListType(1, StringType(), True), False), + ( + MapType(1, StringType(), 2, IntegerType(), False), + False, + ), + ( + StructType( + NestedField(1, "required_field", StringType(), is_optional=False), + NestedField(2, "optional_field", IntegerType(), is_optional=True), + ), + False, + ), + (NestedField(1, "required_field", StringType(), is_optional=False), False), + ], +) +def test_is_primitive(input_type, result): + assert input_type.is_primitive == result + + def test_fixed_type(): type_var = FixedType(length=5) assert type_var.length == 5 @@ -82,41 +118,32 @@ def test_decimal_type(): def test_struct_type(): type_var = StructType( - [ - NestedField(True, 1, "optional_field", IntegerType()), - NestedField(False, 2, "required_field", FixedType(5)), - NestedField( - False, - 3, - "required_field", - StructType( - [ - NestedField(True, 4, "optional_field", DecimalType(8, 2)), - NestedField(False, 5, "required_field", LongType()), - ] - ), + NestedField(1, "optional_field", IntegerType(), is_optional=True), + NestedField(2, "required_field", FixedType(5), is_optional=False), + NestedField( + 3, + "required_field", + StructType( + NestedField(4, "optional_field", DecimalType(8, 2), is_optional=True), + NestedField(5, "required_field", LongType(), is_optional=False), ), - ] + is_optional=False, + ), ) assert len(type_var.fields) == 3 assert str(type_var) == str(eval(repr(type_var))) assert type_var == eval(repr(type_var)) - assert type_var != StructType([NestedField(True, 1, "optional_field", IntegerType())]) + assert type_var != StructType(NestedField(1, "optional_field", IntegerType(), is_optional=True)) def test_list_type(): type_var = ListType( - NestedField( - False, - 1, - "required_field", - StructType( - [ - NestedField(True, 2, "optional_field", DecimalType(8, 2)), - NestedField(False, 3, "required_field", LongType()), - ] - ), - ) + 1, + StructType( + NestedField(2, "optional_field", DecimalType(8, 2), is_optional=True), + NestedField(3, "required_field", LongType(), is_optional=False), + ), + False, ) assert isinstance(type_var.element.type, StructType) assert len(type_var.element.type.fields) == 2 @@ -124,64 +151,43 @@ def test_list_type(): assert str(type_var) == str(eval(repr(type_var))) assert type_var == eval(repr(type_var)) assert type_var != ListType( - NestedField( - True, - 1, - "required_field", - StructType( - [ - NestedField(True, 2, "optional_field", DecimalType(8, 2)), - ] - ), - ) + 1, + StructType( + NestedField(2, "optional_field", DecimalType(8, 2), is_optional=True), + ), + True, ) def test_map_type(): - type_var = MapType( - NestedField(True, 1, "optional_field", DoubleType()), - NestedField(False, 2, "required_field", UUIDType()), - ) + type_var = MapType(1, DoubleType(), 2, UUIDType(), False) assert isinstance(type_var.key.type, DoubleType) assert type_var.key.field_id == 1 assert isinstance(type_var.value.type, UUIDType) assert type_var.value.field_id == 2 assert str(type_var) == str(eval(repr(type_var))) assert type_var == eval(repr(type_var)) - assert type_var != MapType( - NestedField(True, 1, "optional_field", LongType()), - NestedField(False, 2, "required_field", UUIDType()), - ) - assert type_var != MapType( - NestedField(True, 1, "optional_field", DoubleType()), - NestedField(False, 2, "required_field", StringType()), - ) + assert type_var != MapType(1, LongType(), 2, UUIDType(), False) + assert type_var != MapType(1, DoubleType(), 2, StringType(), True) def test_nested_field(): field_var = NestedField( - True, 1, "optional_field1", StructType( - [ - NestedField( - True, - 2, - "optional_field2", - ListType(NestedField(False, 3, "required_field3", DoubleType())), - ), - NestedField( - False, - 4, - "required_field4", - MapType( - NestedField(True, 5, "optional_field5", TimeType()), - NestedField(False, 6, "required_field6", UUIDType()), - ), + NestedField( + 2, + "optional_field2", + ListType( + 3, + DoubleType(), + element_is_optional=False, ), - ] + is_optional=True, + ), ), + is_optional=True, ) assert field_var.is_optional assert not field_var.is_required From 5e20bb9b14da4a2c15815134ff5d6a8b0f02639d Mon Sep 17 00:00:00 2001 From: Samuel Redai <43911210+samredai@users.noreply.github.com> Date: Wed, 23 Feb 2022 15:27:23 -0500 Subject: [PATCH 065/642] Python: Add PyArrowFileIO Implementation (#4081) --- src/iceberg/io/base.py | 75 ++++++++-- src/iceberg/io/pyarrow.py | 205 +++++++++++++++++++++++++++ tests/io/test_base.py | 137 +++++++++++++----- tests/io/test_pyarrow.py | 282 ++++++++++++++++++++++++++++++++++++++ tox.ini | 6 +- 5 files changed, 657 insertions(+), 48 deletions(-) create mode 100644 src/iceberg/io/pyarrow.py create mode 100644 tests/io/test_pyarrow.py diff --git a/src/iceberg/io/base.py b/src/iceberg/io/base.py index 8bb905a86d..6046770501 100644 --- a/src/iceberg/io/base.py +++ b/src/iceberg/io/base.py @@ -76,7 +76,15 @@ def close(self) -> None: class InputFile(ABC): - """A base class for InputFile implementations""" + """A base class for InputFile implementations + + Args: + location(str): A URI or a path to a local file + + Attributes: + location(str): The URI or path to a local file for an InputFile instance + exists(bool): Whether the file exists or not + """ def __init__(self, location: str): self._location = location @@ -90,21 +98,38 @@ def location(self) -> str: """The fully-qualified location of the input file""" return self._location - @property @abstractmethod def exists(self) -> bool: - """Checks whether the file exists""" + """Checks whether the location exists + + + Raises: + PermissionError: If the file at self.location cannot be accessed due to a permission error + """ @abstractmethod def open(self) -> InputStream: """This method should return an object that matches the InputStream protocol - If a file does not exist at `self.location`, this should raise a FileNotFoundError. + Returns: + InputStream: An object that matches the InputStream protocol + + Raises: + PermissionError: If the file at self.location cannot be accessed due to a permission error + FileNotFoundError: If the file at self.location does not exist """ class OutputFile(ABC): - """A base class for OutputFile implementations""" + """A base class for OutputFile implementations + + Args: + location(str): A URI or a path to a local file + + Attributes: + location(str): The URI or path to a local file for an OutputFile instance + exists(bool): Whether the file exists or not + """ def __init__(self, location: str): self._location = location @@ -118,10 +143,14 @@ def location(self) -> str: """The fully-qualified location of the output file""" return self._location - @property @abstractmethod def exists(self) -> bool: - """Checks whether the file exists""" + """Checks whether the location exists + + + Raises: + PermissionError: If the file at self.location cannot be accessed due to a permission error + """ @abstractmethod def to_input_file(self) -> InputFile: @@ -133,7 +162,14 @@ def create(self, overwrite: bool = False) -> OutputStream: Args: overwrite(bool): If the file already exists at `self.location` - and `overwrite` is False a FileExistsError should be raised. + and `overwrite` is False a FileExistsError should be raised + + Returns: + OutputStream: An object that matches the OutputStream protocol + + Raises: + PermissionError: If the file at self.location cannot be accessed due to a permission error + FileExistsError: If the file at self.location already exists and `overwrite=False` """ @@ -142,12 +178,29 @@ class FileIO(ABC): @abstractmethod def new_input(self, location: str) -> InputFile: - """Get an InputFile instance to read bytes from the file at the given location""" + """Get an InputFile instance to read bytes from the file at the given location + + Args: + location(str): A URI or a path to a local file + """ @abstractmethod def new_output(self, location: str) -> OutputFile: - """Get an OutputFile instance to write bytes to the file at the given location""" + """Get an OutputFile instance to write bytes to the file at the given location + + Args: + location(str): A URI or a path to a local file + """ @abstractmethod def delete(self, location: Union[str, InputFile, OutputFile]) -> None: - """Delete the file at the given path""" + """Delete the file at the given path + + Args: + location(str, InputFile, OutputFile): A URI or a path to a local file--if an InputFile instance or + an OutputFile instance is provided, the location attribute for that instance is used as the URI to delete + + Raises: + PermissionError: If the file at location cannot be accessed due to a permission error + FileNotFoundError: When the file at the provided location does not exist + """ diff --git a/src/iceberg/io/pyarrow.py b/src/iceberg/io/pyarrow.py new file mode 100644 index 0000000000..5eff2d2392 --- /dev/null +++ b/src/iceberg/io/pyarrow.py @@ -0,0 +1,205 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +"""FileIO implementation for reading and writing table files that uses pyarrow.fs + +This file contains a FileIO implementation that relies on the filesystem interface provided +by PyArrow. It relies on PyArrow's `from_uri` method that infers the correct filesystem +type to use. Theoretically, this allows the supported storage types to grow naturally +with the pyarrow library. +""" + +import os +from typing import Union +from urllib.parse import urlparse + +from pyarrow.fs import FileInfo, FileSystem, FileType + +from iceberg.io.base import FileIO, InputFile, InputStream, OutputFile, OutputStream + + +class PyArrowFile(InputFile, OutputFile): + """A combined InputFile and OutputFile implementation that uses a pyarrow filesystem to generate pyarrow.lib.NativeFile instances + + Args: + location(str): A URI or a path to a local file + + Attributes: + location(str): The URI or path to a local file for a PyArrowFile instance + + Examples: + >>> from iceberg.io.pyarrow import PyArrowFile + >>> input_file = PyArrowFile("s3://foo/bar.txt") + >>> file_content = input_file.open().read() # Read the contents of the PyArrowFile instance + >>> output_file = PyArrowFile("s3://baz/qux.txt") + >>> output_file.create().write(b'foobytes') # Write bytes to the PyArrowFile instance + """ + + def __init__(self, location: str): + parsed_location = urlparse(location) # Create a ParseResult from the uri + if not parsed_location.scheme: # If no scheme, assume the path is to a local file + self._filesystem, self._path = FileSystem.from_uri(os.path.abspath(location)) + else: + self._filesystem, self._path = FileSystem.from_uri(location) # Infer the proper filesystem + super().__init__(location=location) + + def _file_info(self) -> FileInfo: + """Retrieves a pyarrow.fs.FileInfo object for the location + + Raises: + PermissionError: If the file at self.location cannot be accessed due to a permission error such as + an AWS error code 15 + """ + try: + file_info = self._filesystem.get_file_info(self._path) + except OSError as e: + if e.errno == 13 or "AWS Error [code 15]" in str(e): + raise PermissionError(f"Cannot get file info, access denied: {self.location}") + raise # pragma: no cover - If some other kind of OSError, raise the raw error + + if file_info.type == FileType.NotFound: + raise FileNotFoundError(f"Cannot get file info, file not found: {self.location}") + return file_info + + def __len__(self) -> int: + """Returns the total length of the file, in bytes""" + file_info = self._file_info() + return file_info.size + + def exists(self) -> bool: + """Checks whether the location exists""" + try: + self._file_info() # raises FileNotFoundError if it does not exist + return True + except FileNotFoundError: + return False + + def open(self) -> InputStream: + """Opens the location using a PyArrow FileSystem inferred from the location + + Returns: + pyarrow.lib.NativeFile: A NativeFile instance for the file located at `self.location` + + Raises: + FileNotFoundError: If the file at self.location does not exist + PermissionError: If the file at self.location cannot be accessed due to a permission error such as + an AWS error code 15 + """ + try: + input_file = self._filesystem.open_input_file(self._path) + except FileNotFoundError: + raise + except PermissionError: + raise + except OSError as e: + if e.errno == 2 or "Path does not exist" in str(e): + raise FileNotFoundError(f"Cannot open file, does not exist: {self.location}") + elif e.errno == 13 or "AWS Error [code 15]" in str(e): + raise PermissionError(f"Cannot open file, access denied: {self.location}") + raise # pragma: no cover - If some other kind of OSError, raise the raw error + return input_file + + def create(self, overwrite: bool = False) -> OutputStream: + """Creates a writable pyarrow.lib.NativeFile for this PyArrowFile's location + + Args: + overwrite(bool): Whether to overwrite the file if it already exists + + Returns: + pyarrow.lib.NativeFile: A NativeFile instance for the file located at self.location + + Raises: + FileExistsError: If the file already exists at `self.location` and `overwrite` is False + + Note: + This retrieves a pyarrow NativeFile by opening an output stream. If overwrite is set to False, + a check is first performed to verify that the file does not exist. This is not thread-safe and + a possibility does exist that the file can be created by a concurrent process after the existence + check yet before the output stream is created. In such a case, the default pyarrow behavior will + truncate the contents of the existing file when opening the output stream. + """ + try: + if not overwrite and self.exists() is True: + raise FileExistsError(f"Cannot create file, already exists: {self.location}") + output_file = self._filesystem.open_output_stream(self._path) + except PermissionError: + raise + except OSError as e: + if e.errno == 13 or "AWS Error [code 15]" in str(e): + raise PermissionError(f"Cannot create file, access denied: {self.location}") + raise # pragma: no cover - If some other kind of OSError, raise the raw error + return output_file + + def to_input_file(self) -> "PyArrowFile": + """Returns a new PyArrowFile for the location of an existing PyArrowFile instance + + This method is included to abide by the OutputFile abstract base class. Since this implementation uses a single + PyArrowFile class (as opposed to separate InputFile and OutputFile implementations), this method effectively returns + a copy of the same instance. + """ + return self + + +class PyArrowFileIO(FileIO): + def new_input(self, location: str) -> PyArrowFile: + """Get a PyArrowFile instance to read bytes from the file at the given location + + Args: + location(str): A URI or a path to a local file + + Returns: + PyArrowFile: A PyArrowFile instance for the given location + """ + return PyArrowFile(location) + + def new_output(self, location: str) -> PyArrowFile: + """Get a PyArrowFile instance to write bytes to the file at the given location + + Args: + location(str): A URI or a path to a local file + + Returns: + PyArrowFile: A PyArrowFile instance for the given location + """ + return PyArrowFile(location) + + def delete(self, location: Union[str, InputFile, OutputFile]) -> None: + """Delete the file at the given location + + Args: + location(str, InputFile, OutputFile): The URI to the file--if an InputFile instance or an + OutputFile instance is provided, the location attribute for that instance is used as the location + to delete + + Raises: + FileNotFoundError: When the file at the provided location does not exist + PermissionError: If the file at the provided location cannot be accessed due to a permission error such as + an AWS error code 15 + """ + str_path = location.location if isinstance(location, (InputFile, OutputFile)) else location + filesystem, path = FileSystem.from_uri(str_path) # Infer the proper filesystem + try: + filesystem.delete_file(path) + except FileNotFoundError: + raise + except PermissionError: + raise + except OSError as e: + if e.errno == 2 or "Path does not exist" in str(e): + raise FileNotFoundError(f"Cannot delete file, does not exist: {location}") + elif e.errno == 13 or "AWS Error [code 15]" in str(e): + raise PermissionError(f"Cannot delete file, access denied: {location}") + raise # pragma: no cover - If some other kind of OSError, raise the raw error diff --git a/tests/io/test_base.py b/tests/io/test_base.py index 6a7773cb67..7bd61a0e0e 100644 --- a/tests/io/test_base.py +++ b/tests/io/test_base.py @@ -23,6 +23,7 @@ import pytest from iceberg.io.base import FileIO, InputFile, InputStream, OutputFile, OutputStream +from iceberg.io.pyarrow import PyArrowFile, PyArrowFileIO class LocalInputFile(InputFile): @@ -31,7 +32,7 @@ class LocalInputFile(InputFile): def __init__(self, location: str): parsed_location = urlparse(location) # Create a ParseResult from the uri - if parsed_location.scheme != "file": # Validate that a uri is provided with a scheme of `file` + if parsed_location.scheme and parsed_location.scheme != "file": # Validate that a uri is provided with a scheme of `file` raise ValueError("LocalInputFile location must have a scheme of `file`") elif parsed_location.netloc: raise ValueError(f"Network location is not allowed for LocalInputFile: {parsed_location.netloc}") @@ -41,6 +42,12 @@ def __init__(self, location: str): @property def parsed_location(self) -> ParseResult: + """The parsed location + + Returns: + ParseResult: The parsed results which has attributes `scheme`, `netloc`, `path`, + `params`, `query`, and `fragments`. + """ return self._parsed_location def __len__(self): @@ -52,7 +59,7 @@ def exists(self): def open(self) -> InputStream: input_file = open(self.parsed_location.path, "rb") if not isinstance(input_file, InputStream): - raise TypeError("""Object returned from LocalInputFile.open does not match the OutputStream protocol.""") + raise TypeError("Object returned from LocalInputFile.open() does not match the OutputStream protocol.") return input_file @@ -62,7 +69,7 @@ class LocalOutputFile(OutputFile): def __init__(self, location: str): parsed_location = urlparse(location) # Create a ParseResult from the uri - if parsed_location.scheme != "file": # Validate that a uri is provided with a scheme of `file` + if parsed_location.scheme and parsed_location.scheme != "file": # Validate that a uri is provided with a scheme of `file` raise ValueError("LocalOutputFile location must have a scheme of `file`") elif parsed_location.netloc: raise ValueError(f"Network location is not allowed for LocalOutputFile: {parsed_location.netloc}") @@ -72,6 +79,12 @@ def __init__(self, location: str): @property def parsed_location(self) -> ParseResult: + """The parsed location + + Returns: + ParseResult: The parsed results which has attributes `scheme`, `netloc`, `path`, + `params`, `query`, and `fragments`. + """ return self._parsed_location def __len__(self): @@ -86,7 +99,7 @@ def to_input_file(self): def create(self, overwrite: bool = False) -> OutputStream: output_file = open(self.parsed_location.path, "wb" if overwrite else "xb") if not isinstance(output_file, OutputStream): - raise TypeError("""Object returned from LocalOutputFile.create does not match the OutputStream protocol.""") + raise TypeError("Object returned from LocalOutputFile.create(...) does not match the OutputStream protocol.") return output_file @@ -101,11 +114,15 @@ def new_output(self, location: str): def delete(self, location: Union[str, LocalInputFile, LocalOutputFile]): parsed_location = location.parsed_location if isinstance(location, (InputFile, OutputFile)) else urlparse(location) - os.remove(parsed_location.path) + try: + os.remove(parsed_location.path) + except FileNotFoundError as e: + raise FileNotFoundError(f"Cannot delete file, does not exist: {parsed_location.path} - Caused by: " + str(e)) -@pytest.mark.parametrize("CustomInputFile", [LocalInputFile]) +@pytest.mark.parametrize("CustomInputFile", [LocalInputFile, PyArrowFile]) def test_custom_local_input_file(CustomInputFile): + """Test initializing an InputFile implementation to read a local file""" with tempfile.TemporaryDirectory() as tmpdirname: file_location = os.path.join(tmpdirname, "foo.txt") with open(file_location, "wb") as f: @@ -116,7 +133,7 @@ def test_custom_local_input_file(CustomInputFile): # Instantiate the input file absolute_file_location = os.path.abspath(file_location) - input_file = CustomInputFile(location=f"file:{absolute_file_location}") + input_file = CustomInputFile(location=f"{absolute_file_location}") # Test opening and reading the file f = input_file.open() @@ -125,14 +142,15 @@ def test_custom_local_input_file(CustomInputFile): assert len(input_file) == 3 -@pytest.mark.parametrize("CustomOutputFile", [LocalOutputFile]) +@pytest.mark.parametrize("CustomOutputFile", [LocalOutputFile, PyArrowFile]) def test_custom_local_output_file(CustomOutputFile): + """Test initializing an OutputFile implementation to write to a local file""" with tempfile.TemporaryDirectory() as tmpdirname: file_location = os.path.join(tmpdirname, "foo.txt") # Instantiate the output file absolute_file_location = os.path.abspath(file_location) - output_file = CustomOutputFile(location=f"file:{absolute_file_location}") + output_file = CustomOutputFile(location=f"{absolute_file_location}") # Create the output file and write to it f = output_file.create() @@ -145,8 +163,9 @@ def test_custom_local_output_file(CustomOutputFile): assert len(output_file) == 3 -@pytest.mark.parametrize("CustomOutputFile", [LocalOutputFile]) +@pytest.mark.parametrize("CustomOutputFile", [LocalOutputFile, PyArrowFile]) def test_custom_local_output_file_with_overwrite(CustomOutputFile): + """Test initializing an OutputFile implementation to overwrite a local file""" with tempfile.TemporaryDirectory() as tmpdirname: output_file_location = os.path.join(tmpdirname, "foo.txt") @@ -155,7 +174,7 @@ def test_custom_local_output_file_with_overwrite(CustomOutputFile): f.write(b"foo") # Instantiate an output file - output_file = CustomOutputFile(location=f"file://{output_file_location}") + output_file = CustomOutputFile(location=f"{output_file_location}") # Confirm that a FileExistsError is raised when overwrite=False with pytest.raises(FileExistsError): @@ -169,13 +188,40 @@ def test_custom_local_output_file_with_overwrite(CustomOutputFile): assert f.read() == b"bar" -@pytest.mark.parametrize("CustomOutputFile", [LocalOutputFile]) +@pytest.mark.parametrize("CustomFile", [LocalInputFile, LocalOutputFile, PyArrowFile, PyArrowFile]) +def test_custom_file_exists(CustomFile): + """Test that the exists property returns the proper value for existing and non-existing files""" + with tempfile.TemporaryDirectory() as tmpdirname: + file_location = os.path.join(tmpdirname, "foo.txt") + with open(file_location, "wb") as f: + f.write(b"foo") + + nonexistent_file_location = os.path.join(tmpdirname, "bar.txt") + + # Confirm that the file initially exists + assert os.path.exists(file_location) + + # Get an absolute path for an existing file and a nonexistent file + absolute_file_location = os.path.abspath(file_location) + non_existent_absolute_file_location = os.path.abspath(nonexistent_file_location) + + # Create File instances + file = CustomFile(location=f"{absolute_file_location}") + non_existent_file = CustomFile(location=f"{non_existent_absolute_file_location}") + + # Test opening and reading the file + assert file.exists() + assert not non_existent_file.exists() + + +@pytest.mark.parametrize("CustomOutputFile", [LocalOutputFile, PyArrowFile]) def test_output_file_to_input_file(CustomOutputFile): + """Test initializing an InputFile using the `to_input_file()` method on an OutputFile instance""" with tempfile.TemporaryDirectory() as tmpdirname: output_file_location = os.path.join(tmpdirname, "foo.txt") # Create an output file instance - output_file = CustomOutputFile(location=f"file://{output_file_location}") + output_file = CustomOutputFile(location=f"{output_file_location}") # Create the output file and write to it f = output_file.create() @@ -188,34 +234,33 @@ def test_output_file_to_input_file(CustomOutputFile): @pytest.mark.parametrize( - "CustomFileIO,string_uri,scheme,netloc,path", + "CustomFileIO,string_uri", [ - (LocalFileIO, "file:///foo/bar.parquet", "file", "", "/foo/bar.parquet"), - (LocalFileIO, "file:/foo/bar/baz.parquet", "file", "", "/foo/bar/baz.parquet"), + (LocalFileIO, "foo/bar.parquet"), + (LocalFileIO, "file:///foo/bar.parquet"), + (LocalFileIO, "file:/foo/bar/baz.parquet"), + (PyArrowFileIO, "foo/bar/baz.parquet"), + (PyArrowFileIO, "file:/foo/bar/baz.parquet"), + (PyArrowFileIO, "file:/foo/bar/baz.parquet"), ], ) -def test_custom_file_io_locations(CustomFileIO, string_uri, scheme, netloc, path): +def test_custom_file_io_locations(CustomFileIO, string_uri): + """Test that the location property is maintained as the value of the location argument""" # Instantiate the file-io and create a new input and output file file_io = CustomFileIO() input_file = file_io.new_input(location=string_uri) assert input_file.location == string_uri - assert input_file.parsed_location.scheme == scheme - assert input_file.parsed_location.netloc == netloc - assert input_file.parsed_location.path == path output_file = file_io.new_output(location=string_uri) assert output_file.location == string_uri - assert output_file.parsed_location.scheme == scheme - assert output_file.parsed_location.netloc == netloc - assert output_file.parsed_location.path == path @pytest.mark.parametrize( "string_uri_w_netloc", ["file://localhost:80/foo/bar.parquet", "file://foo/bar.parquet"], ) -def test_raise_on_network_location_in_InputFile(string_uri_w_netloc): - +def test_raise_on_network_location_in_input_file(string_uri_w_netloc): + """Test raising a ValueError when providing a network location to a LocalInputFile""" with pytest.raises(ValueError) as exc_info: LocalInputFile(location=string_uri_w_netloc) @@ -226,17 +271,17 @@ def test_raise_on_network_location_in_InputFile(string_uri_w_netloc): "string_uri_w_netloc", ["file://localhost:80/foo/bar.parquet", "file://foo/bar.parquet"], ) -def test_raise_on_network_location_in_OutputFile(string_uri_w_netloc): - +def test_raise_on_network_location_in_output_file(string_uri_w_netloc): + """Test raising a ValueError when providing a network location to a LocalOutputFile""" with pytest.raises(ValueError) as exc_info: LocalInputFile(location=string_uri_w_netloc) assert ("Network location is not allowed for LocalInputFile") in str(exc_info.value) -@pytest.mark.parametrize("CustomFileIO", [LocalFileIO]) +@pytest.mark.parametrize("CustomFileIO", [LocalFileIO, PyArrowFileIO]) def test_deleting_local_file_using_file_io(CustomFileIO): - + """Test deleting a local file using FileIO.delete(...)""" with tempfile.TemporaryDirectory() as tmpdirname: # Write to the temporary file output_file_location = os.path.join(tmpdirname, "foo.txt") @@ -256,9 +301,29 @@ def test_deleting_local_file_using_file_io(CustomFileIO): assert not os.path.exists(output_file_location) -@pytest.mark.parametrize("CustomFileIO, CustomInputFile", [(LocalFileIO, LocalInputFile)]) -def test_deleting_local_file_using_file_io_InputFile(CustomFileIO, CustomInputFile): +@pytest.mark.parametrize("CustomFileIO", [LocalFileIO, PyArrowFileIO]) +def test_raise_file_not_found_error_for_fileio_delete(CustomFileIO): + """Test raising a FileNotFound error when trying to delete a non-existent file""" + with tempfile.TemporaryDirectory() as tmpdirname: + # Write to the temporary file + output_file_location = os.path.join(tmpdirname, "foo.txt") + + # Instantiate the file-io + file_io = CustomFileIO() + + # Delete the non-existent file using the file-io implementations delete method + with pytest.raises(FileNotFoundError) as exc_info: + file_io.delete(output_file_location) + + assert (f"Cannot delete file") in str(exc_info.value) + + # Confirm that the file no longer exists + assert not os.path.exists(output_file_location) + +@pytest.mark.parametrize("CustomFileIO, CustomInputFile", [(LocalFileIO, LocalInputFile), (PyArrowFileIO, PyArrowFile)]) +def test_deleting_local_file_using_file_io_input_file(CustomFileIO, CustomInputFile): + """Test deleting a local file by passing an InputFile instance to FileIO.delete(...)""" with tempfile.TemporaryDirectory() as tmpdirname: # Write to the temporary file file_location = os.path.join(tmpdirname, "foo.txt") @@ -272,7 +337,7 @@ def test_deleting_local_file_using_file_io_InputFile(CustomFileIO, CustomInputFi assert os.path.exists(file_location) # Instantiate the custom InputFile - input_file = CustomInputFile(location=f"file://{file_location}") + input_file = CustomInputFile(location=f"{file_location}") # Delete the file using the file-io implementations delete method file_io.delete(input_file) @@ -281,9 +346,9 @@ def test_deleting_local_file_using_file_io_InputFile(CustomFileIO, CustomInputFi assert not os.path.exists(file_location) -@pytest.mark.parametrize("CustomFileIO, CustomOutputFile", [(LocalFileIO, LocalOutputFile)]) -def test_deleting_local_file_using_file_io_OutputFile(CustomFileIO, CustomOutputFile): - +@pytest.mark.parametrize("CustomFileIO, CustomOutputFile", [(LocalFileIO, LocalOutputFile), (PyArrowFileIO, PyArrowFile)]) +def test_deleting_local_file_using_file_io_output_file(CustomFileIO, CustomOutputFile): + """Test deleting a local file by passing an OutputFile instance to FileIO.delete(...)""" with tempfile.TemporaryDirectory() as tmpdirname: # Write to the temporary file file_location = os.path.join(tmpdirname, "foo.txt") @@ -297,7 +362,7 @@ def test_deleting_local_file_using_file_io_OutputFile(CustomFileIO, CustomOutput assert os.path.exists(file_location) # Instantiate the custom OutputFile - output_file = CustomOutputFile(location=f"file://{file_location}") + output_file = CustomOutputFile(location=f"{file_location}") # Delete the file using the file-io implementations delete method file_io.delete(output_file) diff --git a/tests/io/test_pyarrow.py b/tests/io/test_pyarrow.py new file mode 100644 index 0000000000..bbde45b39d --- /dev/null +++ b/tests/io/test_pyarrow.py @@ -0,0 +1,282 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +import os +import tempfile +from unittest.mock import MagicMock, patch + +import pytest +from pyarrow.fs import FileType + +from iceberg.io.base import InputStream, OutputStream +from iceberg.io.pyarrow import PyArrowFile, PyArrowFileIO + + +def test_pyarrow_input_file(): + """Test reading a file using PyArrowFile""" + + with tempfile.TemporaryDirectory() as tmpdirname: + file_location = os.path.join(tmpdirname, "foo.txt") + with open(file_location, "wb") as f: + f.write(b"foo") + + # Confirm that the file initially exists + assert os.path.exists(file_location) + + # Instantiate the input file + absolute_file_location = os.path.abspath(file_location) + input_file = PyArrowFile(location=f"{absolute_file_location}") + + # Test opening and reading the file + f = input_file.open() + assert isinstance(f, InputStream) # Test that the file object abides by the InputStream protocol + data = f.read() + assert data == b"foo" + assert len(input_file) == 3 + + +def test_pyarrow_output_file(): + """Test writing a file using PyArrowFile""" + + with tempfile.TemporaryDirectory() as tmpdirname: + file_location = os.path.join(tmpdirname, "foo.txt") + + # Instantiate the output file + absolute_file_location = os.path.abspath(file_location) + output_file = PyArrowFile(location=f"{absolute_file_location}") + + # Create the output file and write to it + f = output_file.create() + assert isinstance(f, OutputStream) # Test that the file object abides by the OutputStream protocol + f.write(b"foo") + + # Confirm that bytes were written + with open(file_location, "rb") as f: + assert f.read() == b"foo" + + assert len(output_file) == 3 + + +def test_pyarrow_invalid_scheme(): + """Test that a ValueError is raised if a location is provided with an invalid scheme""" + + with pytest.raises(ValueError) as exc_info: + PyArrowFile("foo://bar/baz.txt") + + assert ("Unrecognized filesystem type in URI") in str(exc_info.value) + + with pytest.raises(ValueError) as exc_info: + PyArrowFile("foo://bar/baz.txt") + + assert ("Unrecognized filesystem type in URI") in str(exc_info.value) + + +def test_pyarrow_violating_input_stream_protocol(): + """Test that a TypeError is raised if an input file is provided that violates the InputStream protocol""" + + # Missing seek, tell, closed, and close + input_file_mock = MagicMock(spec=["read"]) + + # Create a mocked filesystem that returns input_file_mock + filesystem_mock = MagicMock() + filesystem_mock.open_input_file.return_value = input_file_mock + + input_file = PyArrowFile("foo.txt") + input_file._filesystem = filesystem_mock + + f = input_file.open() + assert not isinstance(f, InputStream) + + +def test_pyarrow_violating_output_stream_protocol(): + """Test that a TypeError is raised if an output stream is provided that violates the OutputStream protocol""" + + # Missing closed, and close + output_file_mock = MagicMock(spec=["write", "exists"]) + output_file_mock.exists.return_value = False + + file_info_mock = MagicMock() + file_info_mock.type = FileType.NotFound + + # Create a mocked filesystem that returns output_file_mock + filesystem_mock = MagicMock() + filesystem_mock.open_output_stream.return_value = output_file_mock + filesystem_mock.get_file_info.return_value = file_info_mock + + output_file = PyArrowFile("foo.txt") + output_file._filesystem = filesystem_mock + + f = output_file.create() + + assert not isinstance(f, OutputStream) + + +def test_raise_on_opening_a_local_file_not_found(): + """Test that a PyArrowFile raises appropriately when a local file is not found""" + + with tempfile.TemporaryDirectory() as tmpdirname: + file_location = os.path.join(tmpdirname, "foo.txt") + f = PyArrowFile(file_location) + + with pytest.raises(FileNotFoundError) as exc_info: + f.open() + + assert "[Errno 2] Failed to open local file" in str(exc_info.value) + + +def test_raise_on_opening_a_local_file_no_permission(): + """Test that a PyArrowFile raises appropriately when opening a local file without permission""" + + with tempfile.TemporaryDirectory() as tmpdirname: + os.chmod(tmpdirname, 0o600) + file_location = os.path.join(tmpdirname, "foo.txt") + f = PyArrowFile(file_location) + + with pytest.raises(PermissionError) as exc_info: + f.open() + + assert "[Errno 13] Failed to open local file" in str(exc_info.value) + + +def test_raise_on_checking_if_local_file_exists_no_permission(): + """Test that a PyArrowFile raises when checking for existence on a file without permission""" + + with tempfile.TemporaryDirectory() as tmpdirname: + os.chmod(tmpdirname, 0o600) + file_location = os.path.join(tmpdirname, "foo.txt") + f = PyArrowFile(file_location) + + with pytest.raises(PermissionError) as exc_info: + f.create() + + assert "Cannot get file info, access denied:" in str(exc_info.value) + + +def test_raise_on_creating_a_local_file_no_permission(): + """Test that a PyArrowFile raises appropriately when creating a local file without permission""" + + with tempfile.TemporaryDirectory() as tmpdirname: + os.chmod(tmpdirname, 0o600) + file_location = os.path.join(tmpdirname, "foo.txt") + f = PyArrowFile(file_location) + + with pytest.raises(PermissionError) as exc_info: + f.create() + + assert "Cannot get file info, access denied:" in str(exc_info.value) + + +def test_raise_on_checking_if_local_file_exists_no_permission(): + """Test that a PyArrowFile raises when deleting a local file without permission""" + + with tempfile.TemporaryDirectory() as tmpdirname: + os.chmod(tmpdirname, 0o600) + file_location = os.path.join(tmpdirname, "foo.txt") + file_io = PyArrowFileIO() + + with pytest.raises(PermissionError) as exc_info: + file_io.delete(file_location) + + assert "Cannot delete file" in str(exc_info.value) + + +@patch("iceberg.io.pyarrow.PyArrowFile.exists", return_value=False) +@patch("iceberg.io.pyarrow.FileSystem") +def test_raise_on_opening_an_s3_file_no_permission(filesystem_mock, exists_mock): + """Test that opening a PyArrowFile raises a PermissionError when the pyarrow error includes 'AWS Error [code 15]'""" + + s3fs_mock = MagicMock() + s3fs_mock.open_input_file.side_effect = OSError("AWS Error [code 15]") + + filesystem_mock.from_uri.return_value = (s3fs_mock, "foo.txt") + + f = PyArrowFile("s3://foo/bar.txt") + + with pytest.raises(PermissionError) as exc_info: + f.open() + + assert "Cannot open file, access denied:" in str(exc_info.value) + + +@patch("iceberg.io.pyarrow.PyArrowFile.exists", return_value=False) +@patch("iceberg.io.pyarrow.FileSystem") +def test_raise_on_opening_an_s3_file_not_found(filesystem_mock, exists_mock): + """Test that a PyArrowFile raises a FileNotFoundError when the pyarrow error includes 'Path does not exist'""" + + s3fs_mock = MagicMock() + s3fs_mock.open_input_file.side_effect = OSError("Path does not exist") + + filesystem_mock.from_uri.return_value = (s3fs_mock, "foo.txt") + + f = PyArrowFile("s3://foo/bar.txt") + + with pytest.raises(FileNotFoundError) as exc_info: + f.open() + + assert "Cannot open file, does not exist:" in str(exc_info.value) + + +@patch("iceberg.io.pyarrow.PyArrowFile.exists", return_value=False) +@patch("iceberg.io.pyarrow.FileSystem") +def test_raise_on_creating_an_s3_file_no_permission(filesystem_mock, exists_mock): + """Test that creating a PyArrowFile raises a PermissionError when the pyarrow error includes 'AWS Error [code 15]'""" + + s3fs_mock = MagicMock() + s3fs_mock.open_output_stream.side_effect = OSError("AWS Error [code 15]") + + filesystem_mock.from_uri.return_value = (s3fs_mock, "foo.txt") + + f = PyArrowFile("s3://foo/bar.txt") + + with pytest.raises(PermissionError) as exc_info: + f.create() + + assert "Cannot create file, access denied:" in str(exc_info.value) + + +@patch("iceberg.io.pyarrow.FileSystem") +def test_deleting_s3_file_no_permission(filesystem_mock): + """Test that a PyArrowFile raises a PermissionError when the pyarrow OSError includes 'AWS Error [code 15]'""" + + s3fs_mock = MagicMock() + s3fs_mock.delete_file.side_effect = OSError("AWS Error [code 15]") + + filesystem_mock.from_uri.return_value = (s3fs_mock, "foo.txt") + + file_io = PyArrowFileIO() + + with pytest.raises(PermissionError) as exc_info: + file_io.delete("s3://foo/bar.txt") + + assert "Cannot delete file, access denied:" in str(exc_info.value) + + +@patch("iceberg.io.pyarrow.FileSystem") +def test_deleting_s3_file_not_found(filesystem_mock): + """Test that a PyArrowFile raises a PermissionError when the pyarrow error includes 'AWS Error [code 15]'""" + + s3fs_mock = MagicMock() + s3fs_mock.delete_file.side_effect = OSError("Path does not exist") + + filesystem_mock.from_uri.return_value = (s3fs_mock, "foo.txt") + + file_io = PyArrowFileIO() + + with pytest.raises(FileNotFoundError) as exc_info: + file_io.delete("s3://foo/bar.txt") + + assert "Cannot delete file, does not exist:" in str(exc_info.value) diff --git a/tox.ini b/tox.ini index b6b894f70c..214df438c9 100644 --- a/tox.ini +++ b/tox.ini @@ -24,6 +24,7 @@ deps = coverage mock pytest + pyarrow setenv = COVERAGE_FILE = test-reports/{envname}/.coverage PYTEST_ADDOPTS = --junitxml=test-reports/{envname}/junit.xml -vv @@ -65,7 +66,7 @@ commands = deps = mypy commands = - mypy --no-implicit-optional src + mypy --no-implicit-optional --config tox.ini src [testenv:docs] basepython = python3 @@ -91,3 +92,6 @@ python = 3.7: py37 3.8: py38, linters 3.9: py39 + +[mypy-pyarrow.*] +ignore_missing_imports = True \ No newline at end of file From 2f40c9c89e9376fa3ae7ce8eb3f89282eb290025 Mon Sep 17 00:00:00 2001 From: Fokko Driesprong Date: Tue, 22 Mar 2022 17:11:50 -0500 Subject: [PATCH 066/642] Python: Add doctest to tox (#4285) --- setup.cfg | 7 ++++++- setup.py | 29 ----------------------------- src/iceberg/expressions.py | 11 +++++------ src/iceberg/io/pyarrow.py | 9 +++++++-- src/iceberg/types.py | 27 +++++++++++++-------------- tox.ini | 16 +++++++++------- 6 files changed, 40 insertions(+), 59 deletions(-) delete mode 100644 setup.py diff --git a/setup.cfg b/setup.cfg index 5c58314131..a707fe123d 100644 --- a/setup.cfg +++ b/setup.cfg @@ -42,6 +42,11 @@ package_dir = = src packages = find: python_requires = >=3.7 - +[options.extras_require] +arrow = + pyarrow +dev= + tox-travis==0.12 + pytest [options.packages.find] where = src diff --git a/setup.py b/setup.py deleted file mode 100644 index 5073c558a2..0000000000 --- a/setup.py +++ /dev/null @@ -1,29 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -from setuptools import setup - -setup( - name="py-iceberg", - install_requires=[], - extras_require={ - "dev": [ - "tox-travis==0.12", - "pytest", - ], - }, -) diff --git a/src/iceberg/expressions.py b/src/iceberg/expressions.py index 48d1ae24f0..b34d082125 100644 --- a/src/iceberg/expressions.py +++ b/src/iceberg/expressions.py @@ -14,7 +14,6 @@ # KIND, either express or implied. See the License for the # specific language governing permissions and limitations # under the License. - from enum import Enum, auto @@ -22,10 +21,10 @@ class Operation(Enum): """Operations to be used as components in expressions Operations can be negated by calling the negate method. - >>> print(Operation.TRUE.negate()) - Operation.FALSE - >>> print(Operation.IS_NULL.negate()) - Operation.NOT_NULL + >>> Operation.TRUE.negate() + + >>> Operation.IS_NULL.negate() + The above example uses the OPERATION_NEGATIONS map which maps each enum to it's opposite enum. @@ -53,7 +52,7 @@ class Operation(Enum): AND = auto() OR = auto() - def negate(self): + def negate(self) -> "Operation": """Returns the operation used when this is negated.""" try: diff --git a/src/iceberg/io/pyarrow.py b/src/iceberg/io/pyarrow.py index 5eff2d2392..2cf5b9fc15 100644 --- a/src/iceberg/io/pyarrow.py +++ b/src/iceberg/io/pyarrow.py @@ -43,9 +43,14 @@ class PyArrowFile(InputFile, OutputFile): Examples: >>> from iceberg.io.pyarrow import PyArrowFile >>> input_file = PyArrowFile("s3://foo/bar.txt") - >>> file_content = input_file.open().read() # Read the contents of the PyArrowFile instance + >>> # Read the contents of the PyArrowFile instance + >>> # Make sure that you have permissions to read/write + >>> # file_content = input_file.open().read() + >>> output_file = PyArrowFile("s3://baz/qux.txt") - >>> output_file.create().write(b'foobytes') # Write bytes to the PyArrowFile instance + >>> # Write bytes to a file + >>> # Make sure that you have permissions to read/write + >>> # output_file.create().write(b'foobytes') """ def __init__(self, location: str): diff --git a/src/iceberg/types.py b/src/iceberg/types.py index af1f595663..2fd68691b1 100644 --- a/src/iceberg/types.py +++ b/src/iceberg/types.py @@ -20,10 +20,11 @@ describe an Iceberg table schema, these classes can be used in the construction of a StructType instance. Example: - >>> StructType( - NestedField(True, 1, "required_field", StringType()), - NestedField(False, 2, "optional_field", IntegerType()) - ) + >>> str(StructType( + ... NestedField(1, "required_field", StringType(), True), + ... NestedField(2, "optional_field", IntegerType()) + ... )) + 'struct<1: required_field: optional string, 2: optional_field: optional int>' Notes: - https://iceberg.apache.org/#spec/#primitive-types @@ -98,7 +99,7 @@ class DecimalType(PrimitiveType): Example: >>> DecimalType(32, 3) DecimalType(precision=32, scale=3) - >>> DecimalType(8, 3)==DecimalType(8, 3) + >>> DecimalType(8, 3) == DecimalType(8, 3) True """ @@ -201,10 +202,11 @@ class StructType(IcebergType): """A struct type in Iceberg Example: - >>> StructType( - NestedField(True, 1, "required_field", StringType()), - NestedField(False, 2, "optional_field", IntegerType()) - ) + >>> str(StructType( + ... NestedField(1, "required_field", StringType(), True), + ... NestedField(2, "optional_field", IntegerType()) + ... )) + 'struct<1: required_field: optional string, 2: optional_field: optional int>' """ _instances: Dict[Tuple[NestedField, ...], "StructType"] = {} @@ -231,7 +233,7 @@ class ListType(IcebergType): Example: >>> ListType(element_id=3, element_type=StringType(), element_is_optional=True) - ListType(element=NestedField(is_optional=True, field_id=3, name='element', field_type=StringType(), doc=None)) + ListType(element_id=3, element_type=StringType(), element_is_optional=True) """ _instances: Dict[Tuple[bool, int, IcebergType], "ListType"] = {} @@ -275,10 +277,7 @@ class MapType(IcebergType): Example: >>> MapType(key_id=1, key_type=StringType(), value_id=2, value_type=IntegerType(), value_is_optional=True) - MapType( - key=NestedField(is_optional=False, field_id=1, name='key', field_type=StringType(), doc=None), - value=NestedField(is_optional=True, field_id=2, name='value', field_type=IntegerType(), doc=None) - ) + MapType(key_id=1, key_type=StringType(), value_id=2, value_type=IntegerType(), value_is_optional=True) """ _instances: Dict[Tuple[int, IcebergType, int, IcebergType, bool], "MapType"] = {} diff --git a/tox.ini b/tox.ini index 214df438c9..dcd1ca7668 100644 --- a/tox.ini +++ b/tox.ini @@ -20,11 +20,12 @@ envlist = py37,py38,py39,linters [testenv] usedevelop = true +extras = arrow deps = coverage mock pytest - pyarrow + pytest-checkdocs setenv = COVERAGE_FILE = test-reports/{envname}/.coverage PYTEST_ADDOPTS = --junitxml=test-reports/{envname}/junit.xml -vv @@ -50,17 +51,17 @@ deps = isort autoflake commands = - autoflake -r --check --ignore-init-module-imports --remove-all-unused-imports setup.py src tests - isort --profile black --check-only setup.py src tests - black --line-length 130 --check --diff setup.py src tests + autoflake -r --check --ignore-init-module-imports --remove-all-unused-imports src tests + isort --profile black --check-only src tests + black --line-length 130 --check --diff src tests [testenv:format] deps = {[testenv:format-check]deps} commands = - autoflake -r --in-place --ignore-init-module-imports --remove-all-unused-imports setup.py src tests - isort --profile black setup.py src tests - black --line-length 130 setup.py src tests + autoflake -r --in-place --ignore-init-module-imports --remove-all-unused-imports src tests + isort --profile black src tests + black --line-length 130 src tests [testenv:type-check] deps = @@ -86,6 +87,7 @@ commands = [pytest] norecursedirs=.* +addopts = --doctest-modules [gh-actions] python = From d5731093f5178c6417402c85bb8f6d5929e8844b Mon Sep 17 00:00:00 2001 From: Hongyue/Steve Zhang Date: Wed, 30 Mar 2022 13:32:39 -0700 Subject: [PATCH 067/642] Python: Add literals (#4262) Co-authored-by: Steve Zhang Co-authored-by: samredai <43911210+samredai@users.noreply.github.com> --- setup.cfg | 2 + src/iceberg/expression/__init__.py | 16 + src/iceberg/expression/literals.py | 493 +++++++++++++++++++++ src/iceberg/types.py | 10 + tests/expression/__init__.py | 11 + tests/expression/test_literals.py | 667 +++++++++++++++++++++++++++++ tox.ini | 5 +- 7 files changed, 1202 insertions(+), 2 deletions(-) create mode 100644 src/iceberg/expression/__init__.py create mode 100644 src/iceberg/expression/literals.py create mode 100644 tests/expression/__init__.py create mode 100644 tests/expression/test_literals.py diff --git a/setup.cfg b/setup.cfg index a707fe123d..85b5909db9 100644 --- a/setup.cfg +++ b/setup.cfg @@ -42,6 +42,8 @@ package_dir = = src packages = find: python_requires = >=3.7 +install_requires = + singledispatch [options.extras_require] arrow = pyarrow diff --git a/src/iceberg/expression/__init__.py b/src/iceberg/expression/__init__.py new file mode 100644 index 0000000000..13a83393a9 --- /dev/null +++ b/src/iceberg/expression/__init__.py @@ -0,0 +1,16 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. diff --git a/src/iceberg/expression/literals.py b/src/iceberg/expression/literals.py new file mode 100644 index 0000000000..de0f7f7ad0 --- /dev/null +++ b/src/iceberg/expression/literals.py @@ -0,0 +1,493 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +import datetime +import struct +import sys +import uuid +from abc import ABC, abstractmethod +from decimal import ROUND_HALF_UP, Decimal +from functools import singledispatch +from typing import Generic, Optional, TypeVar, Union + +if sys.version_info >= (3, 8): + from functools import singledispatchmethod # pragma: no cover +else: + from singledispatch import singledispatchmethod # pragma: no cover + +from iceberg.types import ( + BinaryType, + BooleanType, + DateType, + DecimalType, + DoubleType, + FixedType, + FloatType, + IntegerType, + LongType, + Singleton, + StringType, + TimestampType, + TimestamptzType, + TimeType, + UUIDType, +) + +EPOCH = datetime.datetime.utcfromtimestamp(0) +T = TypeVar("T") + + +class Literal(Generic[T], ABC): + """Literal which has a value and can be converted between types""" + + def __init__(self, value: T): + if value is None: + raise TypeError(f"Invalid literal value: {value}") + self._value = value + + @property + def value(self) -> T: + return self._value + + @abstractmethod + def to(self, type_var): + ... # pragma: no cover + + def __repr__(self): + return f"{type(self).__name__}({self.value})" + + def __str__(self): + return str(self.value) + + def __eq__(self, other): + return self.value == other.value + + def __ne__(self, other): + return not self.__eq__(other) + + def __lt__(self, other): + return self.value < other.value + + def __gt__(self, other): + return self.value > other.value + + def __le__(self, other): + return self.value <= other.value + + def __ge__(self, other): + return self.value >= other.value + + +@singledispatch +def literal(value) -> Literal: + """ + A generic Literal factory to construct an iceberg Literal based on python primitive data type + using dynamic overloading + + Args: + value(python primitive type): the value to be associated with literal + + Example: + from iceberg.expressions.literals import literal + >>> literal(123) + LongLiteral(123) + """ + raise TypeError(f"Unimplemented Type Literal for value: {str(value)}") + + +@literal.register(bool) +def _(value: bool) -> Literal[bool]: + return BooleanLiteral(value) + + +@literal.register(int) +def _(value: int) -> "LongLiteral": + # expression binding can convert to IntegerLiteral if needed + return LongLiteral(value) + + +@literal.register(float) +def _(value: float) -> "DoubleLiteral": + # expression binding can convert to FloatLiteral if needed + return DoubleLiteral(value) + + +@literal.register(str) +def _(value: str) -> Literal[str]: + return StringLiteral(value) + + +@literal.register(uuid.UUID) +def _(value: uuid.UUID) -> Literal[uuid.UUID]: + return UUIDLiteral(value) + + +@literal.register(bytes) +def _(value: bytes) -> Literal[bytes]: + # expression binding can convert to FixedLiteral if needed + return BinaryLiteral(value) + + +@literal.register(bytearray) +def _(value: bytearray) -> Literal[bytes]: + return BinaryLiteral(value) + + +@literal.register(Decimal) +def _(value: Decimal) -> Literal[Decimal]: + return DecimalLiteral(value) + + +class AboveMax(Literal[None], Singleton): + def __init__(self): + pass + + def value(self): + raise ValueError("AboveMax has no value") + + def to(self, type_var): + raise TypeError("Cannot change the type of AboveMax") + + def __repr__(self): + return "AboveMax()" + + def __str__(self): + return "AboveMax" + + +class BelowMin(Literal[None], Singleton): + def __init__(self): + pass + + def value(self): + raise ValueError("BelowMin has no value") + + def to(self, type_var): + raise TypeError("Cannot change the type of BelowMin") + + def __repr__(self): + return "BelowMin()" + + def __str__(self): + return "BelowMin" + + +class BooleanLiteral(Literal[bool]): + @singledispatchmethod + def to(self, type_var): + return None + + @to.register(BooleanType) + def _(self, type_var): + return self + + +class IntegerLiteral(Literal[int]): + @singledispatchmethod + def to(self, type_var): + return None + + @to.register(IntegerType) + def _(self, type_var: IntegerType) -> "IntegerLiteral": + return self + + @to.register(LongType) + def _(self, type_var: LongType) -> "LongLiteral": + return LongLiteral(self.value) + + @to.register(FloatType) + def _(self, type_var: FloatType) -> "FloatLiteral": + return FloatLiteral(float(self.value)) + + @to.register(DoubleType) + def _(self, type_var: DoubleType) -> "DoubleLiteral": + return DoubleLiteral(self.value) + + @to.register(DateType) + def _(self, type_var: DateType) -> "DateLiteral": + return DateLiteral(self.value) + + @to.register(DecimalType) + def _(self, type_var: DecimalType) -> "DecimalLiteral": + if type_var.scale == 0: + return DecimalLiteral(Decimal(self.value)) + else: + return DecimalLiteral(Decimal(self.value).quantize(Decimal((0, (1,), -type_var.scale)), rounding=ROUND_HALF_UP)) + + +class LongLiteral(Literal[int]): + @singledispatchmethod + def to(self, type_var): + return None + + @to.register(LongType) + def _(self, type_var: LongType) -> "LongLiteral": + return self + + @to.register(IntegerType) + def _(self, type_var: IntegerType) -> Union[AboveMax, BelowMin, IntegerLiteral]: + if IntegerType.max < self.value: + return AboveMax() + elif IntegerType.min > self.value: + return BelowMin() + return IntegerLiteral(self.value) + + @to.register(FloatType) + def _(self, type_var: FloatType) -> "FloatLiteral": + return FloatLiteral(float(self.value)) + + @to.register(DoubleType) + def _(self, type_var: DoubleType) -> "DoubleLiteral": + return DoubleLiteral(self.value) + + @to.register(TimeType) + def _(self, type_var: TimeType) -> "TimeLiteral": + return TimeLiteral(self.value) + + @to.register(TimestampType) + def _(self, type_var: TimestampType) -> "TimestampLiteral": + return TimestampLiteral(self.value) + + @to.register(DecimalType) + def _(self, type_var: DecimalType) -> "DecimalLiteral": + if type_var.scale == 0: + return DecimalLiteral(Decimal(self.value)) + else: + return DecimalLiteral(Decimal(self.value).quantize(Decimal((0, (1,), -type_var.scale)), rounding=ROUND_HALF_UP)) + + +class FloatLiteral(Literal[float]): + def __init__(self, value: float): + super().__init__(value=value) + self._value32 = struct.unpack(" other + + def __le__(self, other): + return self._value32 <= other + + def __ge__(self, other): + return self._value32 >= other + + @singledispatchmethod + def to(self, type_var): + return None + + @to.register(FloatType) + def _(self, type_var: FloatType) -> "FloatLiteral": + return self + + @to.register(DoubleType) + def _(self, type_var: DoubleType) -> "DoubleLiteral": + return DoubleLiteral(self.value) + + @to.register(DecimalType) + def _(self, type_var: DecimalType) -> "DecimalLiteral": + if type_var.scale == 0: + return DecimalLiteral(Decimal(self.value).quantize(Decimal("1."), rounding=ROUND_HALF_UP)) + else: + return DecimalLiteral(Decimal(self.value).quantize(Decimal((0, (1,), -type_var.scale)), rounding=ROUND_HALF_UP)) + + +class DoubleLiteral(Literal[float]): + @singledispatchmethod + def to(self, type_var): + return None + + @to.register(DoubleType) + def _(self, type_var: DoubleType) -> "DoubleLiteral": + return self + + @to.register(FloatType) + def _(self, type_var: FloatType) -> Union[AboveMax, BelowMin, FloatLiteral]: + if FloatType.max < self.value: + return AboveMax() + elif FloatType.min > self.value: + return BelowMin() + return FloatLiteral(self.value) + + @to.register(DecimalType) + def _(self, type_var: DecimalType) -> "DecimalLiteral": + return DecimalLiteral(Decimal(self.value).quantize(Decimal((0, (1,), -type_var.scale)), rounding=ROUND_HALF_UP)) + + +class DateLiteral(Literal[int]): + @singledispatchmethod + def to(self, type_var): + return None + + @to.register(DateType) + def _(self, type_var: DateType) -> "DateLiteral": + return self + + +class TimeLiteral(Literal[int]): + @singledispatchmethod + def to(self, type_var): + return None + + @to.register(TimeType) + def _(self, type_var: TimeType) -> "TimeLiteral": + return self + + +class TimestampLiteral(Literal[int]): + @singledispatchmethod + def to(self, type_var): + return None + + @to.register(TimestampType) + def _(self, type_var: TimestampType) -> "TimestampLiteral": + return self + + @to.register(DateType) + def _(self, type_var: DateType) -> "DateLiteral": + return DateLiteral((datetime.datetime.fromtimestamp(self.value / 1_000_000) - EPOCH).days) + + +class DecimalLiteral(Literal[Decimal]): + @singledispatchmethod + def to(self, type_var): + return None + + @to.register(DecimalType) + def _(self, type_var: DecimalType) -> Optional["DecimalLiteral"]: + if type_var.scale == abs(self.value.as_tuple().exponent): + return self + return None + + +class StringLiteral(Literal[str]): + @singledispatchmethod + def to(self, type_var): + return None + + @to.register(StringType) + def _(self, type_var: StringType) -> "StringLiteral": + return self + + @to.register(DateType) + def _(self, type_var: DateType) -> "DateLiteral": + from datetime import date + + EPOCH_DATE = date.fromisoformat("1970-01-01") + return DateLiteral((date.fromisoformat(self.value) - EPOCH_DATE).days) + + @to.register(TimeType) + def _(self, type_var: TimeType) -> "TimeLiteral": + from datetime import time + + t = time.fromisoformat(self.value) + return TimeLiteral((((t.hour * 60 + t.minute) * 60) + t.second) * 1_000_000 + t.microsecond) + + @to.register(TimestampType) + def _(self, type_var: TimestampType) -> Optional[TimestampLiteral]: + import re + from datetime import datetime + + EPOCH_TIMESTAMP = datetime.fromisoformat("1970-01-01T00:00:00.000000") + ISO_TIMESTAMP = re.compile(r"\d\d\d\d-\d\d-\d\dT\d\d:\d\d:\d\d(.\d{1,6})?") + if ISO_TIMESTAMP.fullmatch(self.value): + try: + delta = datetime.fromisoformat(self.value) - EPOCH_TIMESTAMP + return TimestampLiteral((delta.days * 86400 + delta.seconds) * 1_000_000 + delta.microseconds) + except TypeError: + return None + return None + + @to.register(TimestamptzType) + def _(self, type_var: TimestamptzType) -> Optional[TimestampLiteral]: + import re + from datetime import datetime + + EPOCH_TIMESTAMPTZ = datetime.fromisoformat("1970-01-01T00:00:00.000000+00:00") + ISO_TIMESTAMPTZ = re.compile(r"\d\d\d\d-\d\d-\d\dT\d\d:\d\d:\d\d(.\d{1,6})?[-+]\d\d:\d\d") + ... + if ISO_TIMESTAMPTZ.fullmatch(self.value): + try: + delta = datetime.fromisoformat(self.value) - EPOCH_TIMESTAMPTZ + return TimestampLiteral((delta.days * 86400 + delta.seconds) * 1_000_000 + delta.microseconds) + except TypeError: + return None + return None + + @to.register(UUIDType) + def _(self, type_var: UUIDType) -> "UUIDLiteral": + return UUIDLiteral(uuid.UUID(self.value)) + + @to.register(DecimalType) + def _(self, type_var: DecimalType) -> DecimalLiteral: + dec_val = Decimal(self.value) + value_scale = abs(dec_val.as_tuple().exponent) + if value_scale == type_var.scale: + return DecimalLiteral(Decimal(self.value)) + raise ValueError(f"Cannot cast string to decimal, incorrect scale: got {value_scale} expected {type_var.scale}") + + +class UUIDLiteral(Literal[uuid.UUID]): + @singledispatchmethod + def to(self, type_var): + return None + + @to.register(UUIDType) + def _(self, type_var: UUIDType) -> "UUIDLiteral": + return self + + +class FixedLiteral(Literal[bytes]): + @singledispatchmethod + def to(self, type_var): + return None + + @to.register(FixedType) + def _(self, type_var: FixedType) -> Optional["FixedLiteral"]: + if len(self.value) == type_var.length: + return self + else: + return None + + @to.register(BinaryType) + def _(self, type_var: BinaryType) -> "BinaryLiteral": + return BinaryLiteral(self.value) + + +class BinaryLiteral(Literal[bytes]): + @singledispatchmethod + def to(self, type_var): + return None + + @to.register(BinaryType) + def _(self, type_var: BinaryType) -> "BinaryLiteral": + return self + + @to.register(FixedType) + def _(self, type_var: FixedType) -> Optional["FixedLiteral"]: + if type_var.length >= len(self.value): + return FixedLiteral(self.value) + else: + return None diff --git a/src/iceberg/types.py b/src/iceberg/types.py index 2fd68691b1..0b780119fa 100644 --- a/src/iceberg/types.py +++ b/src/iceberg/types.py @@ -397,8 +397,18 @@ class FloatType(PrimitiveType, Singleton): >>> column_foo = FloatType() >>> isinstance(column_foo, FloatType) True + + Attributes: + max (float): The maximum allowed value for Floats, inherited from the canonical Iceberg implementation + in Java. (returns `3.4028235e38`) + min (float): The minimum allowed value for Floats, inherited from the canonical Iceberg implementation + in Java (returns `-3.4028235e38`) """ + max: float = 3.4028235e38 + + min: float = -3.4028235e38 + def __init__(self): if not self._initialized: super().__init__("float", "FloatType()") diff --git a/tests/expression/__init__.py b/tests/expression/__init__.py new file mode 100644 index 0000000000..00eaa2ffe4 --- /dev/null +++ b/tests/expression/__init__.py @@ -0,0 +1,11 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/tests/expression/test_literals.py b/tests/expression/test_literals.py new file mode 100644 index 0000000000..d1d8387792 --- /dev/null +++ b/tests/expression/test_literals.py @@ -0,0 +1,667 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +import datetime +import uuid +from decimal import Decimal + +import pytest + +from iceberg.expression.literals import ( + EPOCH, + AboveMax, + BelowMin, + DateLiteral, + FixedLiteral, + IntegerLiteral, + StringLiteral, + TimeLiteral, + TimestampLiteral, + literal, +) +from iceberg.types import ( + BinaryType, + BooleanType, + DateType, + DecimalType, + DoubleType, + FixedType, + FloatType, + IntegerType, + LongType, + StringType, + TimestampType, + TimestamptzType, + TimeType, + UUIDType, +) + +# Base + + +def test_literal_from_none_error(): + with pytest.raises(TypeError) as e: + literal(None) + assert "Unimplemented Type Literal for value" in str(e.value) + + +def test_string_literal_with_none_value_error(): + with pytest.raises(TypeError) as e: + StringLiteral(None) + assert "Invalid literal value: None" in str(e.value) + + +# Numeric + + +def test_numeric_literal_comparison(): + small_lit = literal(10).to(IntegerType()) + big_lit = literal(1000).to(IntegerType()) + assert small_lit != big_lit + assert small_lit == literal(10) + assert small_lit < big_lit + assert small_lit <= big_lit + assert big_lit > small_lit + assert big_lit >= small_lit + + +def test_integer_to_long_conversion(): + lit = literal(34).to(IntegerType()) + long_lit = lit.to(LongType()) + + assert lit.value == long_lit.value + + +def test_integer_to_float_conversion(): + lit = literal(34).to(IntegerType()) + float_lit = lit.to(FloatType()) + + assert lit.value == float_lit.value + + +def test_integer_to_double_conversion(): + lit = literal(34).to(IntegerType()) + dbl_lit = lit.to(DoubleType()) + + assert lit.value == dbl_lit.value + + +@pytest.mark.parametrize( + "decimalType, decimalValue", [(DecimalType(9, 0), "34"), (DecimalType(9, 2), "34.00"), (DecimalType(9, 4), "34.0000")] +) +def test_integer_to_decimal_conversion(decimalType, decimalValue): + lit = literal(34).to(IntegerType()) + + assert lit.to(decimalType).value.as_tuple() == Decimal(decimalValue).as_tuple() + + +def test_integer_to_date_conversion(): + one_day = "2022-03-28" + date_delta = (datetime.date.fromisoformat(one_day) - datetime.date.fromisoformat("1970-01-01")).days + date_lit = IntegerLiteral(date_delta).to(DateType()) + + assert isinstance(date_lit, DateLiteral) + assert date_lit.value == date_delta + + +def test_long_to_integer_within_bound(): + lit = literal(34).to(LongType()) + int_lit = lit.to(IntegerType()) + + assert lit.value == int_lit.value + + +def test_long_to_integer_outside_bound(): + big_lit = literal(IntegerType.max + 1).to(LongType()) + above_max_lit = big_lit.to(IntegerType()) + assert above_max_lit == AboveMax() + + small_lit = literal(IntegerType.min - 1).to(LongType()) + below_min_lit = small_lit.to(IntegerType()) + assert below_min_lit == BelowMin() + + +def test_long_to_float_conversion(): + lit = literal(34).to(LongType()) + float_lit = lit.to(FloatType()) + + assert lit.value == float_lit.value + + +def test_long_to_double_conversion(): + lit = literal(34).to(LongType()) + dbl_lit = lit.to(DoubleType()) + + assert lit.value == dbl_lit.value + + +def test_long_to_time(): + long_lit = literal(51661919000).to(LongType()) + time_lit = long_lit.to(TimeType()) + + assert isinstance(time_lit, TimeLiteral) + assert time_lit.value == long_lit.value + + +def test_long_to_timestamp(): + long_lit = literal(1647305201).to(LongType()) + timestamp_lit = long_lit.to(TimestampType()) + + assert timestamp_lit.value == long_lit.value + + +@pytest.mark.parametrize( + "decimalType, decimalValue", [(DecimalType(9, 0), "34"), (DecimalType(9, 2), "34.00"), (DecimalType(9, 4), "34.0000")] +) +def test_long_to_decimal_conversion(decimalType, decimalValue): + lit = literal(34).to(LongType()) + + assert lit.to(decimalType).value.as_tuple() == Decimal(decimalValue).as_tuple() + + +def test_float_to_double(): + lit = literal(34.56).to(FloatType()) + dbl_lit = lit.to(DoubleType()) + + assert lit.value == dbl_lit.value + + +@pytest.mark.parametrize( + "decimalType, decimalValue", [(DecimalType(9, 1), "34.6"), (DecimalType(9, 2), "34.56"), (DecimalType(9, 4), "34.5600")] +) +def test_float_to_decimal_conversion(decimalType, decimalValue): + lit = literal(34.56).to(FloatType()) + + assert lit.to(decimalType).value.as_tuple() == Decimal(decimalValue).as_tuple() + + +def test_double_to_float_within_bound(): + lit = literal(34.56).to(DoubleType()) + float_lit = lit.to(FloatType()) + + assert lit.value == float_lit.value + + +def test_double_to_float_outside_bound(): + big_lit = literal(FloatType.max + 1.0e37).to(DoubleType()) + above_max_lit = big_lit.to(FloatType()) + assert above_max_lit == AboveMax() + + small_lit = literal(FloatType.min - 1.0e37).to(DoubleType()) + below_min_lit = small_lit.to(FloatType()) + assert below_min_lit == BelowMin() + + +@pytest.mark.parametrize( + "decimalType, decimalValue", [(DecimalType(9, 1), "34.6"), (DecimalType(9, 2), "34.56"), (DecimalType(9, 4), "34.5600")] +) +def test_double_to_decimal_conversion(decimalType, decimalValue): + lit = literal(34.56).to(DoubleType()) + + assert lit.to(decimalType).value.as_tuple() == Decimal(decimalValue).as_tuple() + + +def test_decimal_to_decimal_conversion(): + lit = literal(Decimal("34.11").quantize(Decimal(".01"))) + + assert lit.value.as_tuple() == lit.to(DecimalType(9, 2)).value.as_tuple() + assert lit.value.as_tuple() == lit.to(DecimalType(11, 2)).value.as_tuple() + assert lit.to(DecimalType(9, 0)) is None + assert lit.to(DecimalType(9, 1)) is None + assert lit.to(DecimalType(9, 3)) is None + + +def test_timestamp_to_date(): + epoch_lit = TimestampLiteral(EPOCH.timestamp() * 1000_000) + date_lit = epoch_lit.to(DateType()) + + assert date_lit.value == 0 + + +# STRING + + +def test_string_literal(): + sqrt2 = literal("1.414").to(StringType()) + pi = literal("3.141").to(StringType()) + pi_string_lit = StringLiteral("3.141") + pi_double_lit = literal(3.141).to(DoubleType()) + + assert sqrt2 != pi + assert pi != pi_double_lit + assert pi == pi_string_lit + assert pi == pi + assert sqrt2 < pi + assert sqrt2 <= pi + assert pi > sqrt2 + assert pi >= sqrt2 + assert str(pi) == "3.141" + + +def test_string_to_string_literal(): + assert literal("abc") == literal("abc").to(StringType()) + + +def test_string_to_date_literal(): + one_day = "2017-08-18" + date_lit = literal(one_day).to(DateType()) + + date_delta = (datetime.date.fromisoformat(one_day) - datetime.date.fromisoformat("1970-01-01")).days + assert date_delta == date_lit.value + + +def test_string_to_time_literal(): + time_str = literal("14:21:01.919") + time_lit = time_str.to(TimeType()) + + avro_val = 51661919000 + + assert isinstance(time_lit, TimeLiteral) + assert avro_val == time_lit.value + + +def test_string_to_timestamp_literal(): + timestamp_str = literal("2017-08-18T14:21:01.919+00:00") + timestamp = timestamp_str.to(TimestamptzType()) + + avro_val = 1503066061919000 + assert avro_val == timestamp.value + + timestamp_str = literal("2017-08-18T14:21:01.919") + timestamp = timestamp_str.to(TimestampType()) + assert avro_val == timestamp.value + + timestamp_str = literal("2017-08-18T14:21:01.919-07:00") + timestamp = timestamp_str.to(TimestamptzType()) + avro_val = 1503091261919000 + assert avro_val == timestamp.value + + +def test_timestamp_with_zone_without_zone_in_literal(): + timestamp_str = literal("2017-08-18T14:21:01.919") + assert timestamp_str.to(TimestamptzType()) is None + + +def test_timestamp_without_zone_with_zone_in_literal(): + timestamp_str = literal("2017-08-18T14:21:01.919+07:00") + assert timestamp_str.to(TimestampType()) is None + + +def test_string_to_uuid_literal(): + expected = uuid.uuid4() + uuid_str = literal(str(expected)) + uuid_lit = uuid_str.to(UUIDType()) + + assert expected == uuid_lit.value + + +def test_string_to_decimal_literal(): + decimal_str = literal("34.560") + decimal_lit = decimal_str.to(DecimalType(9, 3)) + + assert 3 == abs(decimal_lit.value.as_tuple().exponent) + assert Decimal("34.560").as_tuple() == decimal_lit.value.as_tuple() + + +# MISC + + +@pytest.mark.parametrize( + "lit, primitive_type", + [ + (literal(True), BooleanType()), + (literal(34), IntegerType()), + (literal(3400000000), LongType()), + (literal(34.11), FloatType()), + (literal(3.5028235e38), DoubleType()), + (literal(Decimal(34.55).quantize(Decimal("0.01"))), DecimalType(9, 2)), + (literal("2017-08-18"), DateType()), + (literal("14:21:01.919"), TimeType()), + (literal("2017-08-18T14:21:01.919"), TimestampType()), + (literal("abc"), StringType()), + (literal(uuid.uuid4()), UUIDType()), + (literal(bytes([0x01, 0x02, 0x03])), FixedType(3)), + (literal(bytearray([0x03, 0x04, 0x05, 0x06])), BinaryType()), + ], +) +def test_identity_conversions(lit, primitive_type): + expected = lit.to(primitive_type) + assert expected is expected.to(primitive_type) + + +def test_fixed_literal(): + fixed_lit012 = literal(bytes([0x00, 0x01, 0x02])) + fixed_lit013 = literal(bytes([0x00, 0x01, 0x03])) + assert fixed_lit012 == fixed_lit012 + assert fixed_lit012 != fixed_lit013 + assert fixed_lit012 < fixed_lit013 + assert fixed_lit012 <= fixed_lit013 + assert fixed_lit013 > fixed_lit012 + assert fixed_lit013 >= fixed_lit012 + + +def test_binary_literal(): + bin_lit012 = literal(bytearray([0x00, 0x01, 0x02])) + bin_lit013 = literal(bytearray([0x00, 0x01, 0x03])) + assert bin_lit012 == bin_lit012 + assert bin_lit012 != bin_lit013 + assert bin_lit012 < bin_lit013 + assert bin_lit012 <= bin_lit013 + assert bin_lit013 > bin_lit012 + assert bin_lit013 >= bin_lit012 + # None related + + +def test_raise_on_comparison_to_none(): + bin_lit012 = literal(bytearray([0x00, 0x01, 0x02])) + fixed_lit012 = literal(bytes([0x00, 0x01, 0x02])) + + with pytest.raises(AttributeError): + bin_lit012 < None + + with pytest.raises(AttributeError): + bin_lit012 <= None + + with pytest.raises(AttributeError): + bin_lit012 > None + + with pytest.raises(AttributeError): + bin_lit012 >= None + + with pytest.raises(AttributeError): + fixed_lit012 < None + + with pytest.raises(AttributeError): + fixed_lit012 <= None + + with pytest.raises(AttributeError): + fixed_lit012 > None + + with pytest.raises(AttributeError): + fixed_lit012 >= None + + +def test_binary_to_fixed(): + lit = literal(bytearray([0x00, 0x01, 0x02])) + fixed_lit = lit.to(FixedType(3)) + assert fixed_lit is not None + assert lit.value == fixed_lit.value + assert lit.to(FixedType(4)) == FixedLiteral(value=bytearray([0x00, 0x01, 0x02])) + + +def test_binary_to_smaller_fixed_none(): + lit = literal(bytearray([0x00, 0x01, 0x02])) + assert lit.to(FixedType(2)) is None + + +def test_fixed_to_binary(): + lit = literal(bytes([0x00, 0x01, 0x02])).to(FixedType(3)) + binary_lit = lit.to(BinaryType()) + assert binary_lit is not None + assert lit.value == binary_lit.value + + +def test_fixed_to_smaller_fixed_none(): + lit = literal(bytearray([0x00, 0x01, 0x02])).to(FixedType(3)) + assert lit.to(FixedType(2)) is None + + +def test_above_max(): + a = AboveMax() + # singleton + assert a == AboveMax() + assert str(a) == "AboveMax" + assert repr(a) == "AboveMax()" + with pytest.raises(ValueError) as e: + a.value() + assert "AboveMax has no value" in str(e.value) + with pytest.raises(TypeError) as e: + a.to(IntegerType()) + assert "Cannot change the type of AboveMax" in str(e.value) + + +def test_below_min(): + b = BelowMin() + # singleton + assert b == BelowMin() + assert str(b) == "BelowMin" + assert repr(b) == "BelowMin()" + with pytest.raises(ValueError) as e: + b.value() + assert "BelowMin has no value" in str(e.value) + with pytest.raises(TypeError) as e: + b.to(IntegerType()) + assert "Cannot change the type of BelowMin" in str(e.value) + + +def test_invalid_boolean_conversions(): + assert_invalid_conversions( + literal(True), + [ + IntegerType(), + LongType(), + FloatType(), + DoubleType(), + DateType(), + TimeType(), + TimestampType(), + TimestamptzType(), + DecimalType(9, 2), + StringType(), + UUIDType(), + BinaryType(), + ], + ) + + +def test_invalid_integer_conversions(): + assert_invalid_conversions( + literal(34).to(IntegerType()), + [BooleanType(), TimeType(), TimestampType(), TimestamptzType(), StringType(), UUIDType(), FixedType(1), BinaryType()], + ) + + +def test_invalid_long_conversions(): + assert_invalid_conversions( + literal(34).to(LongType()), + [BooleanType(), DateType(), StringType(), UUIDType(), FixedType(1), BinaryType()], + ) + + +@pytest.mark.parametrize( + "lit", + [ + literal(34.11).to(FloatType()), + # double + literal(34.11).to(DoubleType()), + ], +) +@pytest.mark.parametrize( + "test_type", + [ + BooleanType(), + IntegerType(), + LongType(), + DateType(), + TimeType(), + TimestampType(), + TimestamptzType(), + StringType(), + UUIDType(), + FixedType(1), + BinaryType(), + ], +) +def test_invalid_float_conversions(lit, test_type): + assert lit.to(test_type) is None + + +@pytest.mark.parametrize("lit", [literal("2017-08-18").to(DateType())]) +@pytest.mark.parametrize( + "test_type", + [ + BooleanType(), + IntegerType(), + LongType(), + FloatType(), + DoubleType(), + TimeType(), + TimestampType(), + TimestamptzType(), + DecimalType(9, 2), + StringType(), + UUIDType(), + FixedType(1), + BinaryType(), + ], +) +def test_invalid_datetime_conversions(lit, test_type): + assert_invalid_conversions(lit, (test_type,)) + + +def test_invalid_time_conversions(): + assert_invalid_conversions( + literal("14:21:01.919").to(TimeType()), + [ + BooleanType(), + IntegerType(), + LongType(), + FloatType(), + DoubleType(), + DateType(), + TimestampType(), + TimestamptzType(), + DecimalType(9, 2), + StringType(), + UUIDType(), + FixedType(1), + BinaryType(), + ], + ) + + +def test_invalid_timestamp_conversions(): + assert_invalid_conversions( + literal("2017-08-18T14:21:01.919").to(TimestampType()), + [ + BooleanType(), + IntegerType(), + LongType(), + FloatType(), + DoubleType(), + TimeType(), + DecimalType(9, 2), + StringType(), + UUIDType(), + FixedType(1), + BinaryType(), + ], + ) + + +def test_invalid_decimal_conversions(): + assert_invalid_conversions( + literal(Decimal("34.11")), + [ + BooleanType(), + IntegerType(), + LongType(), + FloatType(), + DoubleType(), + DateType(), + TimeType(), + TimestampType(), + TimestamptzType(), + DecimalType(9, 4), + StringType(), + UUIDType(), + FixedType(1), + BinaryType(), + ], + ) + + +def test_invalid_string_conversions(): + assert_invalid_conversions( + literal("abc"), + [BooleanType(), IntegerType(), LongType(), FloatType(), DoubleType(), FixedType(1), BinaryType()], + ) + + +def test_invalid_uuid_conversions(): + assert_invalid_conversions( + literal(uuid.uuid4()), + [ + BooleanType(), + IntegerType(), + LongType(), + FloatType(), + DoubleType(), + DateType(), + TimeType(), + TimestampType(), + TimestamptzType(), + DecimalType(9, 2), + StringType(), + FixedType(1), + BinaryType(), + ], + ) + + +def test_invalid_fixed_conversions(): + assert_invalid_conversions( + literal(bytes([0x00, 0x01, 0x02])).to(FixedType(3)), + [ + BooleanType(), + IntegerType(), + LongType(), + FloatType(), + DoubleType(), + DateType(), + TimeType(), + TimestampType(), + TimestamptzType(), + DecimalType(9, 2), + StringType(), + UUIDType(), + ], + ) + + +def test_invalid_binary_conversions(): + assert_invalid_conversions( + literal(bytearray([0x00, 0x01, 0x02])), + [ + BooleanType(), + IntegerType(), + LongType(), + FloatType(), + DoubleType(), + DateType(), + TimeType(), + TimestampType(), + TimestamptzType(), + DecimalType(9, 2), + StringType(), + UUIDType(), + ], + ) + + +def assert_invalid_conversions(lit, types=None): + for type_var in types: + assert lit.to(type_var) is None diff --git a/tox.ini b/tox.ini index dcd1ca7668..8e47a37204 100644 --- a/tox.ini +++ b/tox.ini @@ -26,6 +26,7 @@ deps = mock pytest pytest-checkdocs + pyarrow setenv = COVERAGE_FILE = test-reports/{envname}/.coverage PYTEST_ADDOPTS = --junitxml=test-reports/{envname}/junit.xml -vv @@ -67,7 +68,7 @@ commands = deps = mypy commands = - mypy --no-implicit-optional --config tox.ini src + mypy --no-implicit-optional --install-types --non-interactive --config tox.ini src [testenv:docs] basepython = python3 @@ -96,4 +97,4 @@ python = 3.9: py39 [mypy-pyarrow.*] -ignore_missing_imports = True \ No newline at end of file +ignore_missing_imports = True From 9de57d52d84ddc41e48d714bf6cd2dd8864a9067 Mon Sep 17 00:00:00 2001 From: Samuel Redai <43911210+samredai@users.noreply.github.com> Date: Wed, 30 Mar 2022 22:05:51 -0400 Subject: [PATCH 068/642] Python: Implement single-value serialization (#4263) --- src/iceberg/conversions.py | 319 +++++++++++++++++++++++ tests/test_conversions.py | 504 +++++++++++++++++++++++++++++++++++++ 2 files changed, 823 insertions(+) create mode 100644 src/iceberg/conversions.py create mode 100644 tests/test_conversions.py diff --git a/src/iceberg/conversions.py b/src/iceberg/conversions.py new file mode 100644 index 0000000000..6bf91e803c --- /dev/null +++ b/src/iceberg/conversions.py @@ -0,0 +1,319 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +"""Utility module for various conversions around PrimitiveType implementations + +This module enables: + - Converting partition strings to built-in python objects + - Converting a value to a byte buffer + - Converting a byte buffer to a value + +Note: + Conversion logic varies based on the PrimitiveType implementation. Therefore conversion functions + are defined here as generic functions using the @singledispatch decorator. For each PrimitiveType + implementation, a concrete function is registered for each generic conversion function. For PrimitiveType + implementations that share the same conversion logic, registrations can be stacked. +""" +import struct +import uuid +from decimal import Decimal +from functools import singledispatch +from typing import Union + +from iceberg.types import ( + BinaryType, + BooleanType, + DateType, + DecimalType, + DoubleType, + FixedType, + FloatType, + IntegerType, + LongType, + PrimitiveType, + StringType, + TimestampType, + TimestamptzType, + TimeType, + UUIDType, +) + + +def handle_none(func): + """A decorator function to handle cases where partition values are `None` or "__HIVE_DEFAULT_PARTITION__" + + Args: + func (Callable): A function registered to the singledispatch function `partition_to_py` + """ + + def wrapper(primitive_type, value_str): + if value_str is None: + return None + elif value_str == "__HIVE_DEFAULT_PARTITION__": + return None + return func(primitive_type, value_str) + + return wrapper + + +def decimal_to_unscaled(value: Decimal) -> int: + """Get an unscaled value given a Decimal value + + Args: + value (Decimal): A Decimal instance + + Returns: + int: The unscaled value + """ + sign, digits, _ = value.as_tuple() + return int(Decimal((sign, digits, 0)).to_integral_value()) + + +def unscaled_to_decimal(unscaled: int, scale: int) -> Decimal: + """Get a scaled Decimal value given an unscaled value and a scale + + Args: + unscaled (int): An unscaled value + scale (int): A scale to set for the returned Decimal instance + + Returns: + Decimal: A scaled Decimal instance + """ + sign, digits, _ = Decimal(unscaled).as_tuple() + return Decimal((sign, digits, -scale)) + + +@singledispatch +def partition_to_py(primitive_type, value_str: str): + """A generic function which converts a partition string to a python built-in + + Args: + primitive_type(PrimitiveType): An implementation of the PrimitiveType base class + value_str(str): A string representation of a partition value + """ + raise TypeError(f"Cannot convert '{value_str}' to unsupported type: {primitive_type}") + + +@partition_to_py.register(BooleanType) +@handle_none +def _(primitive_type, value_str: str) -> Union[int, float, str, uuid.UUID]: + return value_str.lower() == "true" + + +@partition_to_py.register(IntegerType) +@partition_to_py.register(LongType) +@partition_to_py.register(DateType) +@partition_to_py.register(TimeType) +@partition_to_py.register(TimestampType) +@partition_to_py.register(TimestamptzType) +@handle_none +def _(primitive_type, value_str: str) -> int: + """ + Raises: + ValueError: If the scale/exponent is not 0 + """ + _, digits, exponent = Decimal(value_str).as_tuple() + if exponent != 0: # Raise if there are digits to the right of the decimal + raise ValueError(f"Cannot convert partition value, value cannot have fractional digits for {primitive_type} partition") + return int(float(value_str)) + + +@partition_to_py.register(FloatType) +@partition_to_py.register(DoubleType) +@handle_none +def _(primitive_type, value_str: str) -> float: + return float(value_str) + + +@partition_to_py.register(StringType) +@handle_none +def _(primitive_type, value_str: str) -> str: + return value_str + + +@partition_to_py.register(UUIDType) +@handle_none +def _(primitive_type, value_str: str) -> uuid.UUID: + return uuid.UUID(value_str) + + +@partition_to_py.register(FixedType) +@partition_to_py.register(BinaryType) +@handle_none +def _(primitive_type, value_str: str) -> bytes: + return bytes(value_str, "UTF-8") + + +@partition_to_py.register(DecimalType) +@handle_none +def _(primitive_type, value_str: str) -> Decimal: + return Decimal(value_str) + + +@singledispatch +def to_bytes(primitive_type: PrimitiveType, value: Union[bool, bytes, Decimal, float, int, str, uuid.UUID]) -> bytes: + """A generic function which converts a built-in python value to bytes + + This conversion follows the serialization scheme for storing single values as individual binary values defined in the Iceberg specification that + can be found at https://iceberg.apache.org/spec/#appendix-d-single-value-serialization + + Args: + primitive_type(PrimitiveType): An implementation of the PrimitiveType base class + value: The value to convert to bytes (The type of this value depends on which dispatched function is + used--check dispatchable functions for typehints) + """ + raise TypeError(f"scale does not match {primitive_type}") + + +@to_bytes.register(BooleanType) +def _(primitive_type, value: bool) -> bytes: + return struct.pack(" bytes: + return struct.pack(" bytes: + return struct.pack(" bytes: + """ + Note: float in python is implemented using a double in C. Therefore this involves a conversion of a 32-bit (single precision) + float to a 64-bit (double precision) float which introduces some imprecision. + """ + return struct.pack(" bytes: + return struct.pack(" bytes: + return value.encode("UTF-8") + + +@to_bytes.register(UUIDType) +def _(primitive_type, value: uuid.UUID) -> bytes: + return struct.pack(">QQ", (value.int >> 64) & 0xFFFFFFFFFFFFFFFF, value.int & 0xFFFFFFFFFFFFFFFF) + + +@to_bytes.register(BinaryType) +@to_bytes.register(FixedType) +def _(primitive_type, value: bytes) -> bytes: + return value + + +@to_bytes.register(DecimalType) +def _(primitive_type, value: Decimal) -> bytes: + """Convert a Decimal value to bytes given a DecimalType instance with defined precision and scale + + Args: + primitive_type (Decimal): A DecimalType instance with precision and scale + value (Decimal): A Decimal instance + + Raises: + ValueError: If either the precision or scale of `value` does not match that defined in the DecimalType instance + + + Returns: + bytes: The byte representation of `value` + """ + sign, digits, exponent = value.as_tuple() + + if -exponent != primitive_type.scale: + raise ValueError(f"Cannot serialize value, scale of value does not match type {primitive_type}: {-exponent}") + elif len(digits) > primitive_type.precision: + raise ValueError( + f"Cannot serialize value, precision of value is greater than precision of type {primitive_type}: {len(digits)}" + ) + + unscaled_value = decimal_to_unscaled(value=Decimal((sign, digits, 0))) + min_num_bytes = ((unscaled_value).bit_length() + 7) // 8 + return unscaled_value.to_bytes(min_num_bytes, "big", signed=True) + + +@singledispatch +def from_bytes(primitive_type: PrimitiveType, b: bytes) -> Union[bool, bytes, Decimal, float, int, str, uuid.UUID]: + """A generic function which converts bytes to a built-in python value + + Args: + primitive_type(PrimitiveType): An implementation of the PrimitiveType base class + b(bytes): The bytes to convert + """ + raise TypeError(f"Cannot deserialize bytes, type {primitive_type} not supported: {str(b)}") + + +@from_bytes.register(BooleanType) +def _(primitive_type, b: bytes) -> bool: + return struct.unpack(" int: + return struct.unpack(" int: + return struct.unpack(" float: + return struct.unpack(" str: + return bytes(b).decode("utf-8") + + +@from_bytes.register(UUIDType) +def _(primitive_type, b: bytes) -> uuid.UUID: + unpacked_bytes = struct.unpack(">QQ", b) + return uuid.UUID(int=unpacked_bytes[0] << 64 | unpacked_bytes[1]) + + +@from_bytes.register(BinaryType) +@from_bytes.register(FixedType) +def _(primitive_type, b: bytes) -> bytes: + return b + + +@from_bytes.register(DecimalType) +def _(primitive_type, buf: bytes) -> Decimal: + unscaled = int.from_bytes(buf, "big", signed=True) + return unscaled_to_decimal(unscaled, primitive_type.scale) diff --git a/tests/test_conversions.py b/tests/test_conversions.py new file mode 100644 index 0000000000..c800523797 --- /dev/null +++ b/tests/test_conversions.py @@ -0,0 +1,504 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +"""This test module tests PrimitiveType based conversions of values to/from bytes + +Notes: + Boolean: + - Stored as 0x00 for False and non-zero byte for True + Integer: + - Stored as 4 bytes in little-endian order + - 84202 is 0...01|01001000|11101010 in binary + Long: + - Stored as 8 bytes in little-endian order + - 200L is 0...0|11001000 in binary + - 11001000 -> 200 (-56), 00000000 -> 0, ... , 00000000 -> 0 + Double: + - Stored as 8 bytes in little-endian order + - floating point numbers are represented as sign * 2ˆexponent * mantissa + - 6.0 is 1 * 2ˆ4 * 1.5 and encoded as 01000000|00011000|0...0 + - 00000000 -> 0, ... , 00011000 -> 24, 01000000 -> 64 + Date: + - Stored as days from 1970-01-01 in a 4-byte little-endian int + - 1000 is 0...0|00000011|11101000 in binary + - 11101000 -> 232 (-24), 00000011 -> 3, ... , 00000000 -> 0 + Time: + - Stored as microseconds from midnight in an 8-byte little-endian long + - 10000L is 0...0|00100111|00010000 in binary + - 00010000 -> 16, 00100111 -> 39, ... , 00000000 -> 0 + Timestamp: + - Stored as microseconds from 1970-01-01 00:00:00.000000 in an 8-byte little-endian long + - 400000L is 0...110|00011010|10000000 in binary + - 10000000 -> 128 (-128), 00011010 -> 26, 00000110 -> 6, ... , 00000000 -> 0 + String: + - Stored as UTF-8 bytes (without length) + - 'A' -> 65, 'B' -> 66, 'C' -> 67 + UUID: + - Stored as 16-byte big-endian values + - f79c3e09-677c-4bbd-a479-3f349cb785e7 is encoded as F7 9C 3E 09 67 7C 4B BD A4 79 3F 34 9C B7 85 E7 + - 0xF7 -> 11110111 -> 247 (-9), 0x9C -> 10011100 -> 156 (-100), 0x3E -> 00111110 -> 62, + - 0x09 -> 00001001 -> 9, 0x67 -> 01100111 -> 103, 0x7C -> 01111100 -> 124, + - 0x4B -> 01001011 -> 75, 0xBD -> 10111101 -> 189 (-67), 0xA4 -> 10100100 -> 164 (-92), + - 0x79 -> 01111001 -> 121, 0x3F -> 00111111 -> 63, 0x34 -> 00110100 -> 52, + - 0x9C -> 10011100 -> 156 (-100), 0xB7 -> 10110111 -> 183 (-73), 0x85 -> 10000101 -> 133 (-123), + - 0xE7 -> 11100111 -> 231 (-25) + Fixed: + - Stored directly + - 'a' -> 97, 'b' -> 98 + Binary: + - Stored directly + - 'Z' -> 90 + Decimal: + - Stored as unscaled values in the form of two's-complement big-endian binary using the minimum number of bytes for the values + - 345 is 0...1|01011001 in binary + - 00000001 -> 1, 01011001 -> 89 + Float: + - Stored as 4 bytes in little-endian order + - floating point numbers are represented as sign * 2ˆexponent * mantissa + - -4.5F is -1 * 2ˆ2 * 1.125 and encoded as 11000000|10010000|0...0 in binary + - 00000000 -> 0, 00000000 -> 0, 10010000 -> 144 (-112), 11000000 -> 192 (-64), +""" +import struct +import uuid +from decimal import Decimal + +import pytest + +from iceberg import conversions +from iceberg.types import ( + BinaryType, + BooleanType, + DateType, + DecimalType, + DoubleType, + FixedType, + FloatType, + IntegerType, + LongType, + StringType, + TimestampType, + TimestamptzType, + TimeType, + UUIDType, +) + + +@pytest.mark.parametrize( + "value, expected_result", + [ + (Decimal("1.2345"), 12345), + (Decimal("12.345"), 12345), + (Decimal("1234.5"), 12345), + (Decimal("9999999.9999"), 99999999999), + (Decimal("1.0"), 10), + (Decimal("1"), 1), + (Decimal("0.1"), 1), + (Decimal("0.12345"), 12345), + (Decimal("0.0000001"), 1), + ], +) +def test_decimal_to_unscaled(value, expected_result): + """Test converting a decimal to an unscaled value""" + assert conversions.decimal_to_unscaled(value=value) == expected_result + + +@pytest.mark.parametrize( + "unscaled, scale, expected_result", + [ + (12345, 4, Decimal("1.2345")), + (12345, 3, Decimal("12.345")), + (12345, 1, Decimal("1234.5")), + (99999999999, 4, Decimal("9999999.9999")), + (1, 1, Decimal("0.1")), + (1, 0, Decimal("1")), + (12345, 5, Decimal("0.12345")), + (1, 7, Decimal("0.0000001")), + ], +) +def test_unscaled_to_decimal(unscaled, scale, expected_result): + """Test converting an unscaled value to a decimal with a specified scale""" + assert conversions.unscaled_to_decimal(unscaled=unscaled, scale=scale) == expected_result + + +@pytest.mark.parametrize( + "primitive_type, value_str, expected_result", + [ + (BooleanType(), "true", True), + (BooleanType(), "false", False), + (BooleanType(), "TRUE", True), + (BooleanType(), "FALSE", False), + (IntegerType(), "1", 1), + (IntegerType(), "9999", 9999), + (LongType(), "123456789", 123456789), + (FloatType(), "1.1", 1.1), + (DoubleType(), "99999.9", 99999.9), + (DecimalType(5, 2), "123.45", Decimal("123.45")), + (StringType(), "foo", "foo"), + (UUIDType(), "f79c3e09-677c-4bbd-a479-3f349cb785e7", uuid.UUID("f79c3e09-677c-4bbd-a479-3f349cb785e7")), + (FixedType(3), "foo", b"foo"), + (BinaryType(), "foo", b"foo"), + ], +) +def test_partition_to_py(primitive_type, value_str, expected_result): + """Test converting a partition value to a python built-in""" + assert conversions.partition_to_py(primitive_type, value_str) == expected_result + + +@pytest.mark.parametrize( + "primitive_type", + [ + (BinaryType()), + (BooleanType()), + (DateType()), + (DecimalType(2, 1)), + (DoubleType()), + (FixedType(1)), + (FloatType()), + (IntegerType()), + (LongType()), + (StringType()), + (TimestampType()), + (TimestamptzType()), + (TimeType()), + (UUIDType()), + ], +) +def test_none_partition_values(primitive_type): + """Test converting a partition value to a python built-in""" + assert conversions.partition_to_py(primitive_type, None) == None + + +@pytest.mark.parametrize( + "primitive_type", + [ + (BinaryType()), + (BooleanType()), + (DateType()), + (DecimalType(2, 1)), + (DoubleType()), + (FixedType(1)), + (FloatType()), + (IntegerType()), + (LongType()), + (StringType()), + (TimestampType()), + (TimestamptzType()), + (TimeType()), + (UUIDType()), + ], +) +def test_hive_default_partition_values(primitive_type): + """Test converting a partition value to a python built-in""" + assert conversions.partition_to_py(primitive_type, "__HIVE_DEFAULT_PARTITION__") == None + + +@pytest.mark.parametrize( + "primitive_type, value, should_raise", + [ + (IntegerType(), "123.45", True), + (IntegerType(), "1234567.89", True), + (IntegerType(), "123.00", True), + (IntegerType(), "1234567.00", True), + (LongType(), "123.45", True), + (LongType(), "1234567.89", True), + (LongType(), "123.00", True), + (LongType(), "1234567.00", True), + (IntegerType(), "12345", False), + (IntegerType(), "123456789", False), + (IntegerType(), "12300", False), + (IntegerType(), "123456700", False), + (LongType(), "12345", False), + (LongType(), "123456789", False), + (LongType(), "12300", False), + (LongType(), "123456700", False), + ], +) +def test_partition_to_py_raise_on_incorrect_precision_or_scale(primitive_type, value, should_raise): + if should_raise: + with pytest.raises(ValueError) as exc_info: + conversions.partition_to_py(primitive_type, value) + + assert f"Cannot convert partition value, value cannot have fractional digits for {primitive_type} partition" in str( + exc_info.value + ) + else: + conversions.partition_to_py(primitive_type, value) + + +@pytest.mark.parametrize( + "primitive_type, b, result", + [ + (BooleanType(), b"\x00", False), + (BooleanType(), b"\x01", True), + (IntegerType(), b"\xd2\x04\x00\x00", 1234), + (LongType(), b"\xd2\x04\x00\x00\x00\x00\x00\x00", 1234), + (DoubleType(), b"\x8d\x97\x6e\x12\x83\xc0\xf3\x3f", 1.2345), + (DateType(), b"\xe8\x03\x00\x00", 1000), + (DateType(), b"\xd2\x04\x00\x00", 1234), + (TimeType(), b"\x10'\x00\x00\x00\x00\x00\x00", 10000), + (TimeType(), b"\x00\xe8vH\x17\x00\x00\x00", 100000000000), + (TimestamptzType(), b"\x80\x1a\x06\x00\x00\x00\x00\x00", 400000), + (TimestamptzType(), b"\x00\xe8vH\x17\x00\x00\x00", 100000000000), + (TimestampType(), b"\x80\x1a\x06\x00\x00\x00\x00\x00", 400000), + (TimestampType(), b"\x00\xe8vH\x17\x00\x00\x00", 100000000000), + (StringType(), b"ABC", "ABC"), + (StringType(), b"foo", "foo"), + ( + UUIDType(), + b"\xf7\x9c>\tg|K\xbd\xa4y?4\x9c\xb7\x85\xe7", + uuid.UUID("f79c3e09-677c-4bbd-a479-3f349cb785e7"), + ), + (UUIDType(), b"\xf7\x9c>\tg|K\xbd\xa4y?4\x9c\xb7\x85\xe7", uuid.UUID("f79c3e09-677c-4bbd-a479-3f349cb785e7")), + (FixedType(3), b"foo", b"foo"), + (BinaryType(), b"foo", b"foo"), + (DecimalType(5, 2), b"\x30\x39", Decimal("123.45")), + (DecimalType(7, 4), b"\x12\xd6\x87", Decimal("123.4567")), + (DecimalType(7, 4), b"\xff\xed\x29\x79", Decimal("-123.4567")), + ], +) +def test_from_bytes(primitive_type, b, result): + """Test converting from bytes""" + assert conversions.from_bytes(primitive_type, b) == result + + +@pytest.mark.parametrize( + "primitive_type, b, result", + [ + (BooleanType(), b"\x00", False), + (BooleanType(), b"\x01", True), + (IntegerType(), b"\xeaH\x01\x00", 84202), + (IntegerType(), b"\xd2\x04\x00\x00", 1234), + (LongType(), b"\xc8\x00\x00\x00\x00\x00\x00\x00", 200), + (LongType(), b"\xd2\x04\x00\x00\x00\x00\x00\x00", 1234), + (DoubleType(), b"\x00\x00\x00\x00\x00\x00\x18@", 6.0), + (DoubleType(), b"\x8d\x97n\x12\x83\xc0\xf3?", 1.2345), + (DateType(), b"\xe8\x03\x00\x00", 1000), + (DateType(), b"\xd2\x04\x00\x00", 1234), + (TimeType(), b"\x10'\x00\x00\x00\x00\x00\x00", 10000), + (TimeType(), b"\x00\xe8vH\x17\x00\x00\x00", 100000000000), + (TimestamptzType(), b"\x80\x1a\x06\x00\x00\x00\x00\x00", 400000), + (TimestamptzType(), b"\x00\xe8vH\x17\x00\x00\x00", 100000000000), + (TimestampType(), b"\x00\xe8vH\x17\x00\x00\x00", 100000000000), + (StringType(), b"ABC", "ABC"), + (StringType(), b"foo", "foo"), + ( + UUIDType(), + b"\xf7\x9c>\tg|K\xbd\xa4y?4\x9c\xb7\x85\xe7", + uuid.UUID("f79c3e09-677c-4bbd-a479-3f349cb785e7"), + ), + (UUIDType(), b"\xf7\x9c>\tg|K\xbd\xa4y?4\x9c\xb7\x85\xe7", uuid.UUID("f79c3e09-677c-4bbd-a479-3f349cb785e7")), + (FixedType(3), b"foo", b"foo"), + (BinaryType(), b"foo", b"foo"), + (DecimalType(5, 2), b"\x30\x39", Decimal("123.45")), + (DecimalType(3, 2), b"\x01Y", Decimal("3.45")), + # decimal on 3-bytes to test that we use the minimum number of bytes and not a power of 2 + # 1234567 is 00010010|11010110|10000111 in binary + # 00010010 -> 18, 11010110 -> 214, 10000111 -> 135 + (DecimalType(7, 4), b"\x12\xd6\x87", Decimal("123.4567")), + # negative decimal to test two's complement + # -1234567 is 11101101|00101001|01111001 in binary + # 11101101 -> 237, 00101001 -> 41, 01111001 -> 121 + (DecimalType(7, 4), b"\xed)y", Decimal("-123.4567")), + # test empty byte in decimal + # 11 is 00001011 in binary + # 00001011 -> 11 + (DecimalType(10, 3), b"\x0b", Decimal("0.011")), + (DecimalType(4, 2), b"\x04\xd2", Decimal("12.34")), + (FloatType(), b"\x00\x00\x90\xc0", struct.unpack("5\x93\xde\xb9.\xefS\xb3\xd8\x11\x80", + Decimal("12345.123456789123456789123456789123456"), + ), + ( + DecimalType(38, 34), + b"\tH\xd5\xd7\x90x\xdf\x08\x1a\xf6C\t\x06p\xaf\x07", + Decimal("1234.1234567891234567891234567891234567"), + ), + (DecimalType(38, 35), b"\tCE\x82\x85\xc7Vf$M\x16\x82@f\xd6N", Decimal("123.12345678912345678912345678912345678")), + (DecimalType(21, 16), b"\x06\xb1:\xe3\xc4N\x94\xaf\x07", Decimal("12345.1234567891234567")), + (DecimalType(22, 17), b"B\xecL\xe5\xab\x11\xce\xd6N", Decimal("12345.12345678912345678")), + (DecimalType(23, 18), b"\x02\x9d;\x00\xf8\xae\xb2\x14_\x15", Decimal("12345.123456789123456789")), + (DecimalType(24, 19), b"\x1a$N\t\xb6\xd2\xf4\xcb\xb6\xd3", Decimal("12345.1234567891234567891")), + (DecimalType(25, 20), b"\x01\x05k\x0ca$=\x8f\xf5$@", Decimal("12345.12345678912345678912")), + (DecimalType(26, 21), b"\n6.{\xcbjg\x9f\x93j\x83", Decimal("12345.123456789123456789123")), + (DecimalType(27, 22), b'f\x1d\xd0\xd5\xf2(\x0c;\xc2)"', Decimal("12345.1234567891234567891234")), + (DecimalType(28, 23), b"\x03\xfd*([u\x90zU\x95\x9bY", Decimal("12345.12345678912345678912345")), + (DecimalType(29, 24), b"'\xe3\xa5\x93\x92\x97\xa4\xc7W\xd8\x11\x80", Decimal("12345.123456789123456789123456")), + (DecimalType(30, 25), b"\x01\x8e\xe4w\xc3\xb9\xeco\xc9np\xaf\x07", Decimal("12345.1234567891234567891234567")), + (DecimalType(31, 26), b"\x0f\x94\xec\xad\xa5C<]\xdePf\xd6N", Decimal("12345.12345678912345678912345678")), + ], +) +def test_round_trip_conversion_large_decimals(primitive_type, b, result): + """Test round trip conversions of calling `conversions.from_bytes` and then `conversions.to_bytes` on the result""" + value_from_bytes = conversions.from_bytes(primitive_type, b) + assert value_from_bytes == result + + bytes_from_value = conversions.to_bytes(primitive_type, value_from_bytes) + assert bytes_from_value == b + + +@pytest.mark.parametrize( + "primitive_type, expected_max_value", + [ + (DecimalType(6, 2), Decimal("9999.99")), + (DecimalType(10, 10), Decimal(".9999999999")), + (DecimalType(2, 1), Decimal("9.9")), + (DecimalType(38, 37), Decimal("9.9999999999999999999999999999999999999")), + (DecimalType(20, 1), Decimal("9999999999999999999.9")), + ], +) +def test_max_value_round_trip_conversion(primitive_type, expected_max_value): + """Test round trip conversions of maximum DecimalType values""" + b = conversions.to_bytes(primitive_type, expected_max_value) + value_from_bytes = conversions.from_bytes(primitive_type, b) + + assert value_from_bytes == expected_max_value + + +@pytest.mark.parametrize( + "primitive_type, expected_min_value", + [ + (DecimalType(6, 2), Decimal("-9999.99")), + (DecimalType(10, 10), Decimal("-.9999999999")), + (DecimalType(2, 1), Decimal("-9.9")), + (DecimalType(38, 37), Decimal("-9.9999999999999999999999999999999999999")), + (DecimalType(20, 1), Decimal("-9999999999999999999.9")), + ], +) +def test_min_value_round_trip_conversion(primitive_type, expected_min_value): + """Test round trip conversions of minimum DecimalType values""" + b = conversions.to_bytes(primitive_type, expected_min_value) + value_from_bytes = conversions.from_bytes(primitive_type, b) + + assert value_from_bytes == expected_min_value + + +def test_raise_on_unregistered_type(): + """Test raising when a conversion is attempted for a type that has no registered method""" + + class FooUnknownType: + def __repr__(self): + return "FooUnknownType()" + + with pytest.raises(TypeError) as exc_info: + conversions.partition_to_py(FooUnknownType(), "foo") + assert (f"Cannot convert 'foo' to unsupported type: FooUnknownType()") in str(exc_info.value) + + with pytest.raises(TypeError) as exc_info: + conversions.to_bytes(FooUnknownType(), "foo") + assert ("scale does not match FooUnknownType()") in str(exc_info.value) + + with pytest.raises(TypeError) as exc_info: + conversions.from_bytes(FooUnknownType(), b"foo") + assert ("Cannot deserialize bytes, type FooUnknownType() not supported: b'foo'") in str(exc_info.value) + + +@pytest.mark.parametrize( + "primitive_type, value, expected_error_message", + [ + (DecimalType(7, 3), Decimal("123.4567"), "Cannot serialize value, scale of value does not match type decimal(7, 3): 4"), + ( + DecimalType(18, 8), + Decimal("123456789.123456789"), + "Cannot serialize value, scale of value does not match type decimal(18, 8): 9", + ), + ( + DecimalType(36, 34), + Decimal("1.23456789123456789123456789123456789"), + "Cannot serialize value, scale of value does not match type decimal(36, 34): 35", + ), + ( + DecimalType(7, 2), + Decimal("1234567.89"), + "Cannot serialize value, precision of value is greater than precision of type decimal(7, 2): 9", + ), + ( + DecimalType(17, 9), + Decimal("123456789.123456789"), + "Cannot serialize value, precision of value is greater than precision of type decimal(17, 9): 18", + ), + ( + DecimalType(35, 35), + Decimal("1.23456789123456789123456789123456789"), + "Cannot serialize value, precision of value is greater than precision of type decimal(35, 35): 36", + ), + ], +) +def test_raise_on_incorrect_precision_or_scale(primitive_type, value, expected_error_message): + with pytest.raises(ValueError) as exc_info: + conversions.to_bytes(primitive_type, value) + + assert expected_error_message in str(exc_info.value) From 1a8da7ad6b8cc4c286d0f8fbd6e677fe59caf2f4 Mon Sep 17 00:00:00 2001 From: Ryan Blue Date: Thu, 31 Mar 2022 08:43:36 -0700 Subject: [PATCH 069/642] Python: Updates to literals (#4436) --- src/iceberg/expression/literals.py | 214 ++++++++++++++--------------- tests/expression/test_literals.py | 59 ++++---- 2 files changed, 139 insertions(+), 134 deletions(-) diff --git a/src/iceberg/expression/literals.py b/src/iceberg/expression/literals.py index de0f7f7ad0..14708049d0 100644 --- a/src/iceberg/expression/literals.py +++ b/src/iceberg/expression/literals.py @@ -19,14 +19,15 @@ # specific language governing permissions and limitations # under the License. -import datetime +import re import struct import sys -import uuid from abc import ABC, abstractmethod +from datetime import date, datetime, time from decimal import ROUND_HALF_UP, Decimal from functools import singledispatch from typing import Generic, Optional, TypeVar, Union +from uuid import UUID if sys.version_info >= (3, 8): from functools import singledispatchmethod # pragma: no cover @@ -51,21 +52,25 @@ UUIDType, ) -EPOCH = datetime.datetime.utcfromtimestamp(0) +EPOCH_DATE = date.fromisoformat("1970-01-01") +EPOCH_TIMESTAMP = datetime.fromisoformat("1970-01-01T00:00:00.000000") +ISO_TIMESTAMP = re.compile(r"\d\d\d\d-\d\d-\d\dT\d\d:\d\d:\d\d(.\d{1,6})?") +EPOCH_TIMESTAMPTZ = datetime.fromisoformat("1970-01-01T00:00:00.000000+00:00") +ISO_TIMESTAMPTZ = re.compile(r"\d\d\d\d-\d\d-\d\dT\d\d:\d\d:\d\d(.\d{1,6})?[-+]\d\d:\d\d") T = TypeVar("T") class Literal(Generic[T], ABC): """Literal which has a value and can be converted between types""" - def __init__(self, value: T): - if value is None: - raise TypeError(f"Invalid literal value: {value}") + def __init__(self, value: T, value_type: type): + if value is None or not isinstance(value, value_type): + raise TypeError(f"Invalid literal value: {value} (not a {value_type})") self._value = value @property def value(self) -> T: - return self._value + return self._value # type: ignore @abstractmethod def to(self, type_var): @@ -110,7 +115,7 @@ def literal(value) -> Literal: >>> literal(123) LongLiteral(123) """ - raise TypeError(f"Unimplemented Type Literal for value: {str(value)}") + raise TypeError(f"Invalid literal value: {repr(value)}") @literal.register(bool) @@ -119,13 +124,12 @@ def _(value: bool) -> Literal[bool]: @literal.register(int) -def _(value: int) -> "LongLiteral": - # expression binding can convert to IntegerLiteral if needed +def _(value: int) -> Literal[int]: return LongLiteral(value) @literal.register(float) -def _(value: float) -> "DoubleLiteral": +def _(value: float) -> Literal[float]: # expression binding can convert to FloatLiteral if needed return DoubleLiteral(value) @@ -135,8 +139,8 @@ def _(value: str) -> Literal[str]: return StringLiteral(value) -@literal.register(uuid.UUID) -def _(value: uuid.UUID) -> Literal[uuid.UUID]: +@literal.register(UUID) +def _(value: UUID) -> Literal[UUID]: return UUIDLiteral(value) @@ -148,7 +152,7 @@ def _(value: bytes) -> Literal[bytes]: @literal.register(bytearray) def _(value: bytearray) -> Literal[bytes]: - return BinaryLiteral(value) + return BinaryLiteral(bytes(value)) @literal.register(Decimal) @@ -191,6 +195,9 @@ def __str__(self): class BooleanLiteral(Literal[bool]): + def __init__(self, value: bool): + super().__init__(value, bool) + @singledispatchmethod def to(self, type_var): return None @@ -200,83 +207,60 @@ def _(self, type_var): return self -class IntegerLiteral(Literal[int]): - @singledispatchmethod - def to(self, type_var): - return None - - @to.register(IntegerType) - def _(self, type_var: IntegerType) -> "IntegerLiteral": - return self - - @to.register(LongType) - def _(self, type_var: LongType) -> "LongLiteral": - return LongLiteral(self.value) - - @to.register(FloatType) - def _(self, type_var: FloatType) -> "FloatLiteral": - return FloatLiteral(float(self.value)) - - @to.register(DoubleType) - def _(self, type_var: DoubleType) -> "DoubleLiteral": - return DoubleLiteral(self.value) - - @to.register(DateType) - def _(self, type_var: DateType) -> "DateLiteral": - return DateLiteral(self.value) - - @to.register(DecimalType) - def _(self, type_var: DecimalType) -> "DecimalLiteral": - if type_var.scale == 0: - return DecimalLiteral(Decimal(self.value)) - else: - return DecimalLiteral(Decimal(self.value).quantize(Decimal((0, (1,), -type_var.scale)), rounding=ROUND_HALF_UP)) - - class LongLiteral(Literal[int]): + def __init__(self, value: int): + super().__init__(value, int) + @singledispatchmethod def to(self, type_var): return None @to.register(LongType) - def _(self, type_var: LongType) -> "LongLiteral": + def _(self, type_var: LongType) -> Literal[int]: return self @to.register(IntegerType) - def _(self, type_var: IntegerType) -> Union[AboveMax, BelowMin, IntegerLiteral]: + def _(self, type_var: IntegerType) -> Union[AboveMax, BelowMin, Literal[int]]: if IntegerType.max < self.value: return AboveMax() elif IntegerType.min > self.value: return BelowMin() - return IntegerLiteral(self.value) + return self @to.register(FloatType) - def _(self, type_var: FloatType) -> "FloatLiteral": + def _(self, type_var: FloatType) -> Literal[float]: return FloatLiteral(float(self.value)) @to.register(DoubleType) - def _(self, type_var: DoubleType) -> "DoubleLiteral": - return DoubleLiteral(self.value) + def _(self, type_var: DoubleType) -> Literal[float]: + return DoubleLiteral(float(self.value)) + + @to.register(DateType) + def _(self, type_var: DateType) -> Literal[int]: + return DateLiteral(self.value) @to.register(TimeType) - def _(self, type_var: TimeType) -> "TimeLiteral": + def _(self, type_var: TimeType) -> Literal[int]: return TimeLiteral(self.value) @to.register(TimestampType) - def _(self, type_var: TimestampType) -> "TimestampLiteral": + def _(self, type_var: TimestampType) -> Literal[int]: return TimestampLiteral(self.value) @to.register(DecimalType) - def _(self, type_var: DecimalType) -> "DecimalLiteral": + def _(self, type_var: DecimalType) -> Literal[Decimal]: + unscaled = Decimal(self.value) if type_var.scale == 0: - return DecimalLiteral(Decimal(self.value)) + return DecimalLiteral(unscaled) else: - return DecimalLiteral(Decimal(self.value).quantize(Decimal((0, (1,), -type_var.scale)), rounding=ROUND_HALF_UP)) + sign, digits, _ = unscaled.as_tuple() + zeros = (0,) * type_var.scale + return DecimalLiteral(Decimal((sign, digits + zeros, -type_var.scale))) class FloatLiteral(Literal[float]): def __init__(self, value: float): - super().__init__(value=value) + super().__init__(value, float) self._value32 = struct.unpack(" "FloatLiteral": + def _(self, type_var: FloatType) -> Literal[float]: return self @to.register(DoubleType) - def _(self, type_var: DoubleType) -> "DoubleLiteral": + def _(self, type_var: DoubleType) -> Literal[float]: return DoubleLiteral(self.value) @to.register(DecimalType) - def _(self, type_var: DecimalType) -> "DecimalLiteral": - if type_var.scale == 0: - return DecimalLiteral(Decimal(self.value).quantize(Decimal("1."), rounding=ROUND_HALF_UP)) - else: - return DecimalLiteral(Decimal(self.value).quantize(Decimal((0, (1,), -type_var.scale)), rounding=ROUND_HALF_UP)) + def _(self, type_var: DecimalType) -> Literal[Decimal]: + return DecimalLiteral(Decimal(self.value).quantize(Decimal((0, (1,), -type_var.scale)), rounding=ROUND_HALF_UP)) class DoubleLiteral(Literal[float]): + def __init__(self, value: float): + super().__init__(value, float) + @singledispatchmethod def to(self, type_var): return None @to.register(DoubleType) - def _(self, type_var: DoubleType) -> "DoubleLiteral": + def _(self, type_var: DoubleType) -> Literal[float]: return self @to.register(FloatType) - def _(self, type_var: FloatType) -> Union[AboveMax, BelowMin, FloatLiteral]: + def _(self, type_var: FloatType) -> Union[AboveMax, BelowMin, Literal[float]]: if FloatType.max < self.value: return AboveMax() elif FloatType.min > self.value: @@ -332,86 +316,91 @@ def _(self, type_var: FloatType) -> Union[AboveMax, BelowMin, FloatLiteral]: return FloatLiteral(self.value) @to.register(DecimalType) - def _(self, type_var: DecimalType) -> "DecimalLiteral": + def _(self, type_var: DecimalType) -> Literal[Decimal]: return DecimalLiteral(Decimal(self.value).quantize(Decimal((0, (1,), -type_var.scale)), rounding=ROUND_HALF_UP)) class DateLiteral(Literal[int]): + def __init__(self, value: int): + super().__init__(value, int) + @singledispatchmethod def to(self, type_var): return None @to.register(DateType) - def _(self, type_var: DateType) -> "DateLiteral": + def _(self, type_var: DateType) -> Literal[int]: return self class TimeLiteral(Literal[int]): + def __init__(self, value: int): + super().__init__(value, int) + @singledispatchmethod def to(self, type_var): return None @to.register(TimeType) - def _(self, type_var: TimeType) -> "TimeLiteral": + def _(self, type_var: TimeType) -> Literal[int]: return self class TimestampLiteral(Literal[int]): + def __init__(self, value: int): + super().__init__(value, int) + @singledispatchmethod def to(self, type_var): return None @to.register(TimestampType) - def _(self, type_var: TimestampType) -> "TimestampLiteral": + def _(self, type_var: TimestampType) -> Literal[int]: return self @to.register(DateType) - def _(self, type_var: DateType) -> "DateLiteral": - return DateLiteral((datetime.datetime.fromtimestamp(self.value / 1_000_000) - EPOCH).days) + def _(self, type_var: DateType) -> Literal[int]: + return DateLiteral((datetime.fromtimestamp(self.value / 1_000_000) - EPOCH_TIMESTAMP).days) class DecimalLiteral(Literal[Decimal]): + def __init__(self, value: Decimal): + super().__init__(value, Decimal) + @singledispatchmethod def to(self, type_var): return None @to.register(DecimalType) - def _(self, type_var: DecimalType) -> Optional["DecimalLiteral"]: + def _(self, type_var: DecimalType) -> Optional[Literal[Decimal]]: if type_var.scale == abs(self.value.as_tuple().exponent): return self return None class StringLiteral(Literal[str]): + def __init__(self, value: str): + super().__init__(value, str) + @singledispatchmethod def to(self, type_var): return None @to.register(StringType) - def _(self, type_var: StringType) -> "StringLiteral": + def _(self, type_var: StringType) -> Literal[str]: return self @to.register(DateType) - def _(self, type_var: DateType) -> "DateLiteral": - from datetime import date - - EPOCH_DATE = date.fromisoformat("1970-01-01") + def _(self, type_var: DateType) -> Literal[int]: return DateLiteral((date.fromisoformat(self.value) - EPOCH_DATE).days) @to.register(TimeType) - def _(self, type_var: TimeType) -> "TimeLiteral": - from datetime import time - + def _(self, type_var: TimeType) -> Literal[int]: t = time.fromisoformat(self.value) return TimeLiteral((((t.hour * 60 + t.minute) * 60) + t.second) * 1_000_000 + t.microsecond) @to.register(TimestampType) - def _(self, type_var: TimestampType) -> Optional[TimestampLiteral]: - import re - from datetime import datetime - - EPOCH_TIMESTAMP = datetime.fromisoformat("1970-01-01T00:00:00.000000") - ISO_TIMESTAMP = re.compile(r"\d\d\d\d-\d\d-\d\dT\d\d:\d\d:\d\d(.\d{1,6})?") + def _(self, type_var: TimestampType) -> Optional[Literal[int]]: if ISO_TIMESTAMP.fullmatch(self.value): try: delta = datetime.fromisoformat(self.value) - EPOCH_TIMESTAMP @@ -421,13 +410,7 @@ def _(self, type_var: TimestampType) -> Optional[TimestampLiteral]: return None @to.register(TimestamptzType) - def _(self, type_var: TimestamptzType) -> Optional[TimestampLiteral]: - import re - from datetime import datetime - - EPOCH_TIMESTAMPTZ = datetime.fromisoformat("1970-01-01T00:00:00.000000+00:00") - ISO_TIMESTAMPTZ = re.compile(r"\d\d\d\d-\d\d-\d\dT\d\d:\d\d:\d\d(.\d{1,6})?[-+]\d\d:\d\d") - ... + def _(self, type_var: TimestamptzType) -> Optional[Literal[int]]: if ISO_TIMESTAMPTZ.fullmatch(self.value): try: delta = datetime.fromisoformat(self.value) - EPOCH_TIMESTAMPTZ @@ -437,57 +420,66 @@ def _(self, type_var: TimestamptzType) -> Optional[TimestampLiteral]: return None @to.register(UUIDType) - def _(self, type_var: UUIDType) -> "UUIDLiteral": - return UUIDLiteral(uuid.UUID(self.value)) + def _(self, type_var: UUIDType) -> Literal[UUID]: + return UUIDLiteral(UUID(self.value)) @to.register(DecimalType) - def _(self, type_var: DecimalType) -> DecimalLiteral: - dec_val = Decimal(self.value) - value_scale = abs(dec_val.as_tuple().exponent) - if value_scale == type_var.scale: - return DecimalLiteral(Decimal(self.value)) - raise ValueError(f"Cannot cast string to decimal, incorrect scale: got {value_scale} expected {type_var.scale}") + def _(self, type_var: DecimalType) -> Optional[Literal[Decimal]]: + dec = Decimal(self.value) + if type_var.scale == abs(dec.as_tuple().exponent): + return DecimalLiteral(dec) + else: + return None -class UUIDLiteral(Literal[uuid.UUID]): +class UUIDLiteral(Literal[UUID]): + def __init__(self, value: UUID): + super().__init__(value, UUID) + @singledispatchmethod def to(self, type_var): return None @to.register(UUIDType) - def _(self, type_var: UUIDType) -> "UUIDLiteral": + def _(self, type_var: UUIDType) -> Literal[UUID]: return self class FixedLiteral(Literal[bytes]): + def __init__(self, value: bytes): + super().__init__(value, bytes) + @singledispatchmethod def to(self, type_var): return None @to.register(FixedType) - def _(self, type_var: FixedType) -> Optional["FixedLiteral"]: + def _(self, type_var: FixedType) -> Optional[Literal[bytes]]: if len(self.value) == type_var.length: return self else: return None @to.register(BinaryType) - def _(self, type_var: BinaryType) -> "BinaryLiteral": + def _(self, type_var: BinaryType) -> Literal[bytes]: return BinaryLiteral(self.value) class BinaryLiteral(Literal[bytes]): + def __init__(self, value: bytes): + super().__init__(value, bytes) + @singledispatchmethod def to(self, type_var): return None @to.register(BinaryType) - def _(self, type_var: BinaryType) -> "BinaryLiteral": + def _(self, type_var: BinaryType) -> Literal[bytes]: return self @to.register(FixedType) - def _(self, type_var: FixedType) -> Optional["FixedLiteral"]: - if type_var.length >= len(self.value): + def _(self, type_var: FixedType) -> Optional[Literal[bytes]]: + if type_var.length == len(self.value): return FixedLiteral(self.value) else: return None diff --git a/tests/expression/test_literals.py b/tests/expression/test_literals.py index d1d8387792..620b65b959 100644 --- a/tests/expression/test_literals.py +++ b/tests/expression/test_literals.py @@ -21,12 +21,16 @@ import pytest from iceberg.expression.literals import ( - EPOCH, AboveMax, BelowMin, + BinaryLiteral, + BooleanLiteral, DateLiteral, + DecimalLiteral, + DoubleLiteral, FixedLiteral, - IntegerLiteral, + FloatLiteral, + LongLiteral, StringLiteral, TimeLiteral, TimestampLiteral, @@ -55,12 +59,28 @@ def test_literal_from_none_error(): with pytest.raises(TypeError) as e: literal(None) - assert "Unimplemented Type Literal for value" in str(e.value) + assert "Invalid literal value: None" in str(e.value) -def test_string_literal_with_none_value_error(): +@pytest.mark.parametrize( + "literalClass", + [ + BooleanLiteral, + LongLiteral, + FloatLiteral, + DoubleLiteral, + DateLiteral, + TimeLiteral, + TimestampLiteral, + DecimalLiteral, + StringLiteral, + FixedLiteral, + BinaryLiteral, + ], +) +def test_string_literal_with_none_value_error(literalClass): with pytest.raises(TypeError) as e: - StringLiteral(None) + literalClass(None) assert "Invalid literal value: None" in str(e.value) @@ -111,7 +131,7 @@ def test_integer_to_decimal_conversion(decimalType, decimalValue): def test_integer_to_date_conversion(): one_day = "2022-03-28" date_delta = (datetime.date.fromisoformat(one_day) - datetime.date.fromisoformat("1970-01-01")).days - date_lit = IntegerLiteral(date_delta).to(DateType()) + date_lit = literal(date_delta).to(DateType()) assert isinstance(date_lit, DateLiteral) assert date_lit.value == date_delta @@ -225,7 +245,7 @@ def test_decimal_to_decimal_conversion(): def test_timestamp_to_date(): - epoch_lit = TimestampLiteral(EPOCH.timestamp() * 1000_000) + epoch_lit = TimestampLiteral(int(datetime.datetime.fromisoformat("1970-01-01T01:23:45.678").timestamp() * 1_000_000)) date_lit = epoch_lit.to(DateType()) assert date_lit.value == 0 @@ -274,29 +294,29 @@ def test_string_to_time_literal(): def test_string_to_timestamp_literal(): - timestamp_str = literal("2017-08-18T14:21:01.919+00:00") + timestamp_str = literal("2017-08-18T14:21:01.919234+00:00") timestamp = timestamp_str.to(TimestamptzType()) - avro_val = 1503066061919000 + avro_val = 1503066061919234 assert avro_val == timestamp.value - timestamp_str = literal("2017-08-18T14:21:01.919") + timestamp_str = literal("2017-08-18T14:21:01.919234") timestamp = timestamp_str.to(TimestampType()) assert avro_val == timestamp.value - timestamp_str = literal("2017-08-18T14:21:01.919-07:00") + timestamp_str = literal("2017-08-18T14:21:01.919234-07:00") timestamp = timestamp_str.to(TimestamptzType()) - avro_val = 1503091261919000 + avro_val = 1503091261919234 assert avro_val == timestamp.value def test_timestamp_with_zone_without_zone_in_literal(): - timestamp_str = literal("2017-08-18T14:21:01.919") + timestamp_str = literal("2017-08-18T14:21:01.919234") assert timestamp_str.to(TimestamptzType()) is None def test_timestamp_without_zone_with_zone_in_literal(): - timestamp_str = literal("2017-08-18T14:21:01.919+07:00") + timestamp_str = literal("2017-08-18T14:21:01.919234+07:00") assert timestamp_str.to(TimestampType()) is None @@ -399,7 +419,7 @@ def test_binary_to_fixed(): fixed_lit = lit.to(FixedType(3)) assert fixed_lit is not None assert lit.value == fixed_lit.value - assert lit.to(FixedType(4)) == FixedLiteral(value=bytearray([0x00, 0x01, 0x02])) + assert lit.to(FixedType(4)) is None def test_binary_to_smaller_fixed_none(): @@ -467,17 +487,10 @@ def test_invalid_boolean_conversions(): ) -def test_invalid_integer_conversions(): - assert_invalid_conversions( - literal(34).to(IntegerType()), - [BooleanType(), TimeType(), TimestampType(), TimestamptzType(), StringType(), UUIDType(), FixedType(1), BinaryType()], - ) - - def test_invalid_long_conversions(): assert_invalid_conversions( literal(34).to(LongType()), - [BooleanType(), DateType(), StringType(), UUIDType(), FixedType(1), BinaryType()], + [BooleanType(), StringType(), UUIDType(), FixedType(1), BinaryType()], ) From 3aafa1fb15d7163f303ce1f8f470e63ebb3d9ebb Mon Sep 17 00:00:00 2001 From: jun-he Date: Sun, 3 Apr 2022 10:42:56 -0700 Subject: [PATCH 070/642] Python: Add bucket transform (#4416) --- setup.cfg | 1 + src/iceberg/transforms.py | 240 ++++++++++++++++++++++++++++++++++++++ tests/test_transforms.py | 133 +++++++++++++++++++++ 3 files changed, 374 insertions(+) create mode 100644 src/iceberg/transforms.py create mode 100644 tests/test_transforms.py diff --git a/setup.cfg b/setup.cfg index 85b5909db9..559751011c 100644 --- a/setup.cfg +++ b/setup.cfg @@ -43,6 +43,7 @@ package_dir = packages = find: python_requires = >=3.7 install_requires = + mmh3 singledispatch [options.extras_require] arrow = diff --git a/src/iceberg/transforms.py b/src/iceberg/transforms.py new file mode 100644 index 0000000000..d24668cc60 --- /dev/null +++ b/src/iceberg/transforms.py @@ -0,0 +1,240 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +import math +import struct +from decimal import Decimal +from typing import Optional +from uuid import UUID + +import mmh3 # type: ignore + +from iceberg.types import ( + BinaryType, + DateType, + DecimalType, + FixedType, + IcebergType, + IntegerType, + LongType, + StringType, + TimestampType, + TimestamptzType, + TimeType, + UUIDType, +) + + +class Transform: + """Transform base class for concrete transforms. + + A base class to transform values and project predicates on partition values. + This class is not used directly. Instead, use one of module method to create the child classes. + + Args: + transform_string (str): name of the transform type + repr_string (str): string representation of a transform instance + """ + + def __init__(self, transform_string: str, repr_string: str): + self._transform_string = transform_string + self._repr_string = repr_string + + def __repr__(self): + return self._repr_string + + def __str__(self): + return self._transform_string + + def __call__(self, value): + return self.apply(value) + + def apply(self, value): + raise NotImplementedError() + + def can_transform(self, source: IcebergType) -> bool: + return False + + def result_type(self, source: IcebergType) -> IcebergType: + raise NotImplementedError() + + def preserves_order(self) -> bool: + return False + + def satisfies_order_of(self, other) -> bool: + return self == other + + def to_human_string(self, value) -> str: + if value is None: + return "null" + return str(value) + + def dedup_name(self) -> str: + return self._transform_string + + +class BaseBucketTransform(Transform): + """Base Transform class to transform a value into a bucket partition value + + Transforms are parameterized by a number of buckets. Bucket partition transforms use a 32-bit + hash of the source value to produce a positive value by mod the bucket number. + + Args: + source_type (Type): An Iceberg Type of IntegerType, LongType, DecimalType, DateType, TimeType, + TimestampType, TimestamptzType, StringType, BinaryType, FixedType, UUIDType. + num_buckets (int): The number of buckets. + """ + + def __init__(self, source_type: IcebergType, num_buckets: int): + super().__init__( + f"bucket[{num_buckets}]", + f"transforms.bucket(source_type={repr(source_type)}, num_buckets={num_buckets})", + ) + self._num_buckets = num_buckets + + @property + def num_buckets(self) -> int: + return self._num_buckets + + def hash(self, value) -> int: + raise NotImplementedError() + + def apply(self, value) -> Optional[int]: + if value is None: + return None + + return (self.hash(value) & IntegerType.max) % self._num_buckets + + def can_transform(self, source: IcebergType) -> bool: + raise NotImplementedError() + + def result_type(self, source: IcebergType) -> IcebergType: + return IntegerType() + + +class BucketNumberTransform(BaseBucketTransform): + """Transforms a value of IntegerType, LongType, DateType, TimeType, TimestampType, or TimestamptzType + into a bucket partition value + + Example: + >>> transform = BucketNumberTransform(LongType(), 100) + >>> transform.apply(81068000000) + 59 + """ + + def can_transform(self, source: IcebergType) -> bool: + return type(source) in {IntegerType, DateType, LongType, TimeType, TimestampType, TimestamptzType} + + def hash(self, value) -> int: + return mmh3.hash(struct.pack(">> transform = BucketDecimalTransform(DecimalType(9, 2), 100) + >>> transform.apply(Decimal("14.20")) + 59 + """ + + def can_transform(self, source: IcebergType) -> bool: + return isinstance(source, DecimalType) + + def hash(self, value: Decimal) -> int: + value_tuple = value.as_tuple() + unscaled_value = int(("-" if value_tuple.sign else "") + "".join([str(d) for d in value_tuple.digits])) + number_of_bytes = int(math.ceil(unscaled_value.bit_length() / 8)) + value_in_bytes = unscaled_value.to_bytes(length=number_of_bytes, byteorder="big") + return mmh3.hash(value_in_bytes) + + +class BucketStringTransform(BaseBucketTransform): + """Transforms a value of StringType into a bucket partition value. + + Example: + >>> transform = BucketStringTransform(100) + >>> transform.apply("iceberg") + 89 + """ + + def __init__(self, num_buckets: int): + super().__init__(StringType(), num_buckets) + + def can_transform(self, source: IcebergType) -> bool: + return isinstance(source, StringType) + + def hash(self, value: str) -> int: + return mmh3.hash(value) + + +class BucketBytesTransform(BaseBucketTransform): + """Transforms a value of FixedType or BinaryType into a bucket partition value. + + Example: + >>> transform = BucketBytesTransform(BinaryType(), 100) + >>> transform.apply(b"\\x00\\x01\\x02\\x03") + 41 + """ + + def can_transform(self, source: IcebergType) -> bool: + return type(source) in {FixedType, BinaryType} + + def hash(self, value: bytes) -> int: + return mmh3.hash(value) + + +class BucketUUIDTransform(BaseBucketTransform): + """Transforms a value of UUIDType into a bucket partition value. + + Example: + >>> transform = BucketUUIDTransform(100) + >>> transform.apply(UUID("f79c3e09-677c-4bbd-a479-3f349cb785e7")) + 40 + """ + + def __init__(self, num_buckets: int): + super().__init__(UUIDType(), num_buckets) + + def can_transform(self, source: IcebergType) -> bool: + return isinstance(source, UUIDType) + + def hash(self, value: UUID) -> int: + return mmh3.hash( + struct.pack( + ">QQ", + (value.int >> 64) & 0xFFFFFFFFFFFFFFFF, + value.int & 0xFFFFFFFFFFFFFFFF, + ) + ) + + +def bucket(source_type: IcebergType, num_buckets: int) -> BaseBucketTransform: + if type(source_type) in {IntegerType, LongType, DateType, TimeType, TimestampType, TimestamptzType}: + return BucketNumberTransform(source_type, num_buckets) + elif isinstance(source_type, DecimalType): + return BucketDecimalTransform(source_type, num_buckets) + elif isinstance(source_type, StringType): + return BucketStringTransform(num_buckets) + elif isinstance(source_type, BinaryType): + return BucketBytesTransform(source_type, num_buckets) + elif isinstance(source_type, FixedType): + return BucketBytesTransform(source_type, num_buckets) + elif isinstance(source_type, UUIDType): + return BucketUUIDTransform(num_buckets) + else: + raise ValueError(f"Cannot bucket by type: {source_type}") diff --git a/tests/test_transforms.py b/tests/test_transforms.py new file mode 100644 index 0000000000..96b68b8fb4 --- /dev/null +++ b/tests/test_transforms.py @@ -0,0 +1,133 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +from datetime import datetime +from decimal import Decimal, getcontext +from uuid import UUID + +import pytest + +from iceberg import transforms +from iceberg.types import ( + BinaryType, + DateType, + DecimalType, + FixedType, + IntegerType, + LongType, + StringType, + TimestampType, + TimestamptzType, + TimeType, + UUIDType, +) + + +@pytest.mark.parametrize( + "test_input,test_type,expected", + [ + (1, IntegerType(), 1392991556), + (34, IntegerType(), 2017239379), + (34, LongType(), 2017239379), + (17486, DateType(), -653330422), + (81068000000, TimeType(), -662762989), + ( + int(datetime.fromisoformat("2017-11-16T22:31:08+00:00").timestamp() * 1000000), + TimestampType(), + -2047944441, + ), + ( + int(datetime.fromisoformat("2017-11-16T14:31:08-08:00").timestamp() * 1000000), + TimestamptzType(), + -2047944441, + ), + (b"\x00\x01\x02\x03", BinaryType(), -188683207), + (b"\x00\x01\x02\x03", FixedType(4), -188683207), + ("iceberg", StringType(), 1210000089), + (UUID("f79c3e09-677c-4bbd-a479-3f349cb785e7"), UUIDType(), 1488055340), + ], +) +def test_bucket_hash_values(test_input, test_type, expected): + assert transforms.bucket(test_type, 8).hash(test_input) == expected + + +@pytest.mark.parametrize( + "test_input,test_type,scale_factor,expected_hash,expected", + [ + (Decimal("14.20"), DecimalType(9, 2), Decimal(10) ** -2, -500754589, 59), + ( + Decimal("137302769811943318102518958871258.37580"), + DecimalType(38, 5), + Decimal(10) ** -5, + -32334285, + 63, + ), + ], +) +def test_decimal_bucket(test_input, test_type, scale_factor, expected_hash, expected): + getcontext().prec = 38 + assert transforms.bucket(test_type, 100).hash(test_input.quantize(scale_factor)) == expected_hash + assert transforms.bucket(test_type, 100).apply(test_input.quantize(scale_factor)) == expected + + +@pytest.mark.parametrize( + "bucket,value,expected", + [ + (transforms.bucket(IntegerType(), 100), 34, 79), + (transforms.bucket(LongType(), 100), 34, 79), + (transforms.bucket(DateType(), 100), 17486, 26), + (transforms.bucket(TimeType(), 100), 81068000000, 59), + (transforms.bucket(TimestampType(), 100), 1510871468000000, 7), + (transforms.bucket(DecimalType(9, 2), 100), Decimal("14.20"), 59), + (transforms.bucket(StringType(), 100), "iceberg", 89), + ( + transforms.bucket(UUIDType(), 100), + UUID("f79c3e09-677c-4bbd-a479-3f349cb785e7"), + 40, + ), + (transforms.bucket(FixedType(3), 128), b"foo", 32), + (transforms.bucket(BinaryType(), 128), b"\x00\x01\x02\x03", 57), + ], +) +def test_buckets(bucket, value, expected): + assert bucket.apply(value) == expected + + +@pytest.mark.parametrize( + "type_var", + [ + BinaryType(), + DateType(), + DecimalType(8, 5), + FixedType(8), + IntegerType(), + LongType(), + StringType(), + TimestampType(), + TimestamptzType(), + TimeType(), + UUIDType(), + ], +) +def test_bucket_method(type_var): + bucket_transform = transforms.bucket(type_var, 8) + assert str(bucket_transform) == str(eval(repr(bucket_transform))) + assert bucket_transform.can_transform(type_var) + assert bucket_transform.result_type(type_var) == IntegerType() + assert bucket_transform.num_buckets == 8 + assert bucket_transform.apply(None) is None + assert bucket_transform.to_human_string("test") == "test" From ca7735fb06cc3883130561fdd435acb7192c67d6 Mon Sep 17 00:00:00 2001 From: Ryan Blue Date: Mon, 4 Apr 2022 13:39:20 -0700 Subject: [PATCH 071/642] Python: Refactor to use common decimal and datetime util (#4480) --- src/iceberg/conversions.py | 32 +------------ src/iceberg/expression/literals.py | 54 ++++++++++----------- src/iceberg/transforms.py | 35 +++++++------- src/iceberg/utils/datetime.py | 65 +++++++++++++++++++++++++ src/iceberg/utils/decimal.py | 77 ++++++++++++++++++++++++++++++ tests/test_conversions.py | 5 +- tests/test_transforms.py | 44 ++++++++--------- 7 files changed, 210 insertions(+), 102 deletions(-) create mode 100644 src/iceberg/utils/datetime.py create mode 100644 src/iceberg/utils/decimal.py diff --git a/src/iceberg/conversions.py b/src/iceberg/conversions.py index 6bf91e803c..0af73567e7 100644 --- a/src/iceberg/conversions.py +++ b/src/iceberg/conversions.py @@ -50,6 +50,7 @@ TimeType, UUIDType, ) +from iceberg.utils.decimal import decimal_to_bytes, unscaled_to_decimal def handle_none(func): @@ -69,33 +70,6 @@ def wrapper(primitive_type, value_str): return wrapper -def decimal_to_unscaled(value: Decimal) -> int: - """Get an unscaled value given a Decimal value - - Args: - value (Decimal): A Decimal instance - - Returns: - int: The unscaled value - """ - sign, digits, _ = value.as_tuple() - return int(Decimal((sign, digits, 0)).to_integral_value()) - - -def unscaled_to_decimal(unscaled: int, scale: int) -> Decimal: - """Get a scaled Decimal value given an unscaled value and a scale - - Args: - unscaled (int): An unscaled value - scale (int): A scale to set for the returned Decimal instance - - Returns: - Decimal: A scaled Decimal instance - """ - sign, digits, _ = Decimal(unscaled).as_tuple() - return Decimal((sign, digits, -scale)) - - @singledispatch def partition_to_py(primitive_type, value_str: str): """A generic function which converts a partition string to a python built-in @@ -251,9 +225,7 @@ def _(primitive_type, value: Decimal) -> bytes: f"Cannot serialize value, precision of value is greater than precision of type {primitive_type}: {len(digits)}" ) - unscaled_value = decimal_to_unscaled(value=Decimal((sign, digits, 0))) - min_num_bytes = ((unscaled_value).bit_length() + 7) // 8 - return unscaled_value.to_bytes(min_num_bytes, "big", signed=True) + return decimal_to_bytes(value) @singledispatch diff --git a/src/iceberg/expression/literals.py b/src/iceberg/expression/literals.py index 14708049d0..58dc66ef1b 100644 --- a/src/iceberg/expression/literals.py +++ b/src/iceberg/expression/literals.py @@ -19,16 +19,22 @@ # specific language governing permissions and limitations # under the License. -import re import struct import sys from abc import ABC, abstractmethod -from datetime import date, datetime, time from decimal import ROUND_HALF_UP, Decimal from functools import singledispatch from typing import Generic, Optional, TypeVar, Union from uuid import UUID +from iceberg.utils.datetime import ( + date_to_days, + micros_to_days, + time_to_micros, + timestamp_to_micros, + timestamptz_to_micros, +) + if sys.version_info >= (3, 8): from functools import singledispatchmethod # pragma: no cover else: @@ -52,11 +58,6 @@ UUIDType, ) -EPOCH_DATE = date.fromisoformat("1970-01-01") -EPOCH_TIMESTAMP = datetime.fromisoformat("1970-01-01T00:00:00.000000") -ISO_TIMESTAMP = re.compile(r"\d\d\d\d-\d\d-\d\dT\d\d:\d\d:\d\d(.\d{1,6})?") -EPOCH_TIMESTAMPTZ = datetime.fromisoformat("1970-01-01T00:00:00.000000+00:00") -ISO_TIMESTAMPTZ = re.compile(r"\d\d\d\d-\d\d-\d\dT\d\d:\d\d:\d\d(.\d{1,6})?[-+]\d\d:\d\d") T = TypeVar("T") @@ -360,7 +361,7 @@ def _(self, type_var: TimestampType) -> Literal[int]: @to.register(DateType) def _(self, type_var: DateType) -> Literal[int]: - return DateLiteral((datetime.fromtimestamp(self.value / 1_000_000) - EPOCH_TIMESTAMP).days) + return DateLiteral(micros_to_days(self.value)) class DecimalLiteral(Literal[Decimal]): @@ -391,33 +392,32 @@ def _(self, type_var: StringType) -> Literal[str]: return self @to.register(DateType) - def _(self, type_var: DateType) -> Literal[int]: - return DateLiteral((date.fromisoformat(self.value) - EPOCH_DATE).days) + def _(self, type_var: DateType) -> Optional[Literal[int]]: + try: + return DateLiteral(date_to_days(self.value)) + except (TypeError, ValueError): + return None @to.register(TimeType) - def _(self, type_var: TimeType) -> Literal[int]: - t = time.fromisoformat(self.value) - return TimeLiteral((((t.hour * 60 + t.minute) * 60) + t.second) * 1_000_000 + t.microsecond) + def _(self, type_var: TimeType) -> Optional[Literal[int]]: + try: + return TimeLiteral(time_to_micros(self.value)) + except (TypeError, ValueError): + return None @to.register(TimestampType) def _(self, type_var: TimestampType) -> Optional[Literal[int]]: - if ISO_TIMESTAMP.fullmatch(self.value): - try: - delta = datetime.fromisoformat(self.value) - EPOCH_TIMESTAMP - return TimestampLiteral((delta.days * 86400 + delta.seconds) * 1_000_000 + delta.microseconds) - except TypeError: - return None - return None + try: + return TimestampLiteral(timestamp_to_micros(self.value)) + except (TypeError, ValueError): + return None @to.register(TimestamptzType) def _(self, type_var: TimestamptzType) -> Optional[Literal[int]]: - if ISO_TIMESTAMPTZ.fullmatch(self.value): - try: - delta = datetime.fromisoformat(self.value) - EPOCH_TIMESTAMPTZ - return TimestampLiteral((delta.days * 86400 + delta.seconds) * 1_000_000 + delta.microseconds) - except TypeError: - return None - return None + try: + return TimestampLiteral(timestamptz_to_micros(self.value)) + except (TypeError, ValueError): + return None @to.register(UUIDType) def _(self, type_var: UUIDType) -> Literal[UUID]: diff --git a/src/iceberg/transforms.py b/src/iceberg/transforms.py index d24668cc60..cfa0b9ed73 100644 --- a/src/iceberg/transforms.py +++ b/src/iceberg/transforms.py @@ -15,10 +15,10 @@ # specific language governing permissions and limitations # under the License. -import math import struct +from abc import ABC from decimal import Decimal -from typing import Optional +from typing import Generic, Optional, TypeVar from uuid import UUID import mmh3 # type: ignore @@ -37,9 +37,13 @@ TimeType, UUIDType, ) +from iceberg.utils.decimal import decimal_to_bytes +S = TypeVar("S") +T = TypeVar("T") -class Transform: + +class Transform(ABC, Generic[S, T]): """Transform base class for concrete transforms. A base class to transform values and project predicates on partition values. @@ -60,18 +64,19 @@ def __repr__(self): def __str__(self): return self._transform_string - def __call__(self, value): + def __call__(self, value: S) -> Optional[T]: return self.apply(value) - def apply(self, value): - raise NotImplementedError() + def apply(self, value: S) -> Optional[T]: + ... def can_transform(self, source: IcebergType) -> bool: return False def result_type(self, source: IcebergType) -> IcebergType: - raise NotImplementedError() + ... + @property def preserves_order(self) -> bool: return False @@ -83,11 +88,12 @@ def to_human_string(self, value) -> str: return "null" return str(value) + @property def dedup_name(self) -> str: return self._transform_string -class BaseBucketTransform(Transform): +class BaseBucketTransform(Transform[S, int]): """Base Transform class to transform a value into a bucket partition value Transforms are parameterized by a number of buckets. Bucket partition transforms use a 32-bit @@ -110,18 +116,15 @@ def __init__(self, source_type: IcebergType, num_buckets: int): def num_buckets(self) -> int: return self._num_buckets - def hash(self, value) -> int: + def hash(self, value: S) -> int: raise NotImplementedError() - def apply(self, value) -> Optional[int]: + def apply(self, value: S) -> Optional[int]: if value is None: return None return (self.hash(value) & IntegerType.max) % self._num_buckets - def can_transform(self, source: IcebergType) -> bool: - raise NotImplementedError() - def result_type(self, source: IcebergType) -> IcebergType: return IntegerType() @@ -156,11 +159,7 @@ def can_transform(self, source: IcebergType) -> bool: return isinstance(source, DecimalType) def hash(self, value: Decimal) -> int: - value_tuple = value.as_tuple() - unscaled_value = int(("-" if value_tuple.sign else "") + "".join([str(d) for d in value_tuple.digits])) - number_of_bytes = int(math.ceil(unscaled_value.bit_length() / 8)) - value_in_bytes = unscaled_value.to_bytes(length=number_of_bytes, byteorder="big") - return mmh3.hash(value_in_bytes) + return mmh3.hash(decimal_to_bytes(value)) class BucketStringTransform(BaseBucketTransform): diff --git a/src/iceberg/utils/datetime.py b/src/iceberg/utils/datetime.py new file mode 100644 index 0000000000..c8c12393b6 --- /dev/null +++ b/src/iceberg/utils/datetime.py @@ -0,0 +1,65 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +"""Helper methods for working with date/time representations +""" +import re +from datetime import date, datetime, time + +EPOCH_DATE = date.fromisoformat("1970-01-01") +EPOCH_TIMESTAMP = datetime.fromisoformat("1970-01-01T00:00:00.000000") +ISO_TIMESTAMP = re.compile(r"\d\d\d\d-\d\d-\d\dT\d\d:\d\d:\d\d(.\d{1,6})?") +EPOCH_TIMESTAMPTZ = datetime.fromisoformat("1970-01-01T00:00:00.000000+00:00") +ISO_TIMESTAMPTZ = re.compile(r"\d\d\d\d-\d\d-\d\dT\d\d:\d\d:\d\d(.\d{1,6})?[-+]\d\d:\d\d") + + +def micros_to_days(timestamp: int) -> int: + """Converts a timestamp in microseconds to a date in days""" + return (datetime.fromtimestamp(timestamp / 1_000_000) - EPOCH_TIMESTAMP).days + + +def date_to_days(date_str: str) -> int: + """Converts an ISO-8601 formatted date to days from 1970-01-01""" + return (date.fromisoformat(date_str) - EPOCH_DATE).days + + +def time_to_micros(time_str: str) -> int: + """Converts an ISO-8601 formatted time to microseconds from midnight""" + t = time.fromisoformat(time_str) + return (((t.hour * 60 + t.minute) * 60) + t.second) * 1_000_000 + t.microsecond + + +def datetime_to_micros(dt: datetime) -> int: + """Converts a datetime to microseconds from 1970-01-01T00:00:00.000000""" + if dt.tzinfo: + delta = dt - EPOCH_TIMESTAMPTZ + else: + delta = dt - EPOCH_TIMESTAMP + return (delta.days * 86400 + delta.seconds) * 1_000_000 + delta.microseconds + + +def timestamp_to_micros(timestamp_str: str) -> int: + """Converts an ISO-9601 formatted timestamp without zone to microseconds from 1970-01-01T00:00:00.000000""" + if ISO_TIMESTAMP.fullmatch(timestamp_str): + return datetime_to_micros(datetime.fromisoformat(timestamp_str)) + raise ValueError(f"Invalid timestamp without zone: {timestamp_str} (must be ISO-8601)") + + +def timestamptz_to_micros(timestamptz_str: str) -> int: + """Converts an ISO-8601 formatted timestamp with zone to microseconds from 1970-01-01T00:00:00.000000+00:00""" + if ISO_TIMESTAMPTZ.fullmatch(timestamptz_str): + return datetime_to_micros(datetime.fromisoformat(timestamptz_str)) + raise ValueError(f"Invalid timestamp with zone: {timestamptz_str} (must be ISO-8601)") diff --git a/src/iceberg/utils/decimal.py b/src/iceberg/utils/decimal.py new file mode 100644 index 0000000000..1d4c2bddef --- /dev/null +++ b/src/iceberg/utils/decimal.py @@ -0,0 +1,77 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +"""Helper methods for working with Python Decimals +""" +from decimal import Decimal +from typing import Union + + +def decimal_to_unscaled(value: Decimal) -> int: + """Get an unscaled value given a Decimal value + + Args: + value (Decimal): A Decimal instance + + Returns: + int: The unscaled value + """ + sign, digits, _ = value.as_tuple() + return int(Decimal((sign, digits, 0)).to_integral_value()) + + +def unscaled_to_decimal(unscaled: int, scale: int) -> Decimal: + """Get a scaled Decimal value given an unscaled value and a scale + + Args: + unscaled (int): An unscaled value + scale (int): A scale to set for the returned Decimal instance + + Returns: + Decimal: A scaled Decimal instance + """ + sign, digits, _ = Decimal(unscaled).as_tuple() + return Decimal((sign, digits, -scale)) + + +def bytes_required(value: Union[int, Decimal]) -> int: + """Returns the minimum number of bytes needed to serialize a decimal or unscaled value + + Args: + value (int | Decimal): a Decimal value or unscaled int value + + Returns: + int: the minimum number of bytes needed to serialize the value + """ + if isinstance(value, int): + return (value.bit_length() + 7) // 8 + elif isinstance(value, Decimal): + return (decimal_to_unscaled(value).bit_length() + 7) // 8 + + raise ValueError(f"Unsupported value: {value}") + + +def decimal_to_bytes(value: Decimal) -> bytes: + """Returns a byte representation of a decimal + + Args: + value (Decimal): a decimal value + Returns: + bytes: the unscaled value of the Decimal as bytes + """ + unscaled_value = decimal_to_unscaled(value) + return unscaled_value.to_bytes(bytes_required(unscaled_value), byteorder="big", signed=True) diff --git a/tests/test_conversions.py b/tests/test_conversions.py index c800523797..b107023473 100644 --- a/tests/test_conversions.py +++ b/tests/test_conversions.py @@ -77,6 +77,7 @@ import pytest +import iceberg.utils.decimal as decimal_util from iceberg import conversions from iceberg.types import ( BinaryType, @@ -112,7 +113,7 @@ ) def test_decimal_to_unscaled(value, expected_result): """Test converting a decimal to an unscaled value""" - assert conversions.decimal_to_unscaled(value=value) == expected_result + assert decimal_util.decimal_to_unscaled(value=value) == expected_result @pytest.mark.parametrize( @@ -130,7 +131,7 @@ def test_decimal_to_unscaled(value, expected_result): ) def test_unscaled_to_decimal(unscaled, scale, expected_result): """Test converting an unscaled value to a decimal with a specified scale""" - assert conversions.unscaled_to_decimal(unscaled=unscaled, scale=scale) == expected_result + assert decimal_util.unscaled_to_decimal(unscaled=unscaled, scale=scale) == expected_result @pytest.mark.parametrize( diff --git a/tests/test_transforms.py b/tests/test_transforms.py index 96b68b8fb4..12e786f954 100644 --- a/tests/test_transforms.py +++ b/tests/test_transforms.py @@ -15,10 +15,10 @@ # specific language governing permissions and limitations # under the License. -from datetime import datetime -from decimal import Decimal, getcontext +from decimal import Decimal from uuid import UUID +import mmh3 as mmh3 import pytest from iceberg import transforms @@ -35,6 +35,12 @@ TimeType, UUIDType, ) +from iceberg.utils.datetime import ( + date_to_days, + time_to_micros, + timestamp_to_micros, + timestamptz_to_micros, +) @pytest.mark.parametrize( @@ -43,15 +49,15 @@ (1, IntegerType(), 1392991556), (34, IntegerType(), 2017239379), (34, LongType(), 2017239379), - (17486, DateType(), -653330422), - (81068000000, TimeType(), -662762989), + (date_to_days("2017-11-16"), DateType(), -653330422), + (time_to_micros("22:31:08"), TimeType(), -662762989), ( - int(datetime.fromisoformat("2017-11-16T22:31:08+00:00").timestamp() * 1000000), + timestamp_to_micros("2017-11-16T22:31:08"), TimestampType(), -2047944441, ), ( - int(datetime.fromisoformat("2017-11-16T14:31:08-08:00").timestamp() * 1000000), + timestamptz_to_micros("2017-11-16T14:31:08-08:00"), TimestamptzType(), -2047944441, ), @@ -65,25 +71,6 @@ def test_bucket_hash_values(test_input, test_type, expected): assert transforms.bucket(test_type, 8).hash(test_input) == expected -@pytest.mark.parametrize( - "test_input,test_type,scale_factor,expected_hash,expected", - [ - (Decimal("14.20"), DecimalType(9, 2), Decimal(10) ** -2, -500754589, 59), - ( - Decimal("137302769811943318102518958871258.37580"), - DecimalType(38, 5), - Decimal(10) ** -5, - -32334285, - 63, - ), - ], -) -def test_decimal_bucket(test_input, test_type, scale_factor, expected_hash, expected): - getcontext().prec = 38 - assert transforms.bucket(test_type, 100).hash(test_input.quantize(scale_factor)) == expected_hash - assert transforms.bucket(test_type, 100).apply(test_input.quantize(scale_factor)) == expected - - @pytest.mark.parametrize( "bucket,value,expected", [ @@ -131,3 +118,10 @@ def test_bucket_method(type_var): assert bucket_transform.num_buckets == 8 assert bucket_transform.apply(None) is None assert bucket_transform.to_human_string("test") == "test" + + +def test_string_with_surrogate_pair(): + string_with_surrogate_pair = "string with a surrogate pair: 💰" + as_bytes = bytes(string_with_surrogate_pair, "UTF-8") + bucket_transform = transforms.bucket(StringType(), 100) + assert bucket_transform.hash(string_with_surrogate_pair) == mmh3.hash(as_bytes) From aea90669b5c360115abf3d3767009f261bf14be9 Mon Sep 17 00:00:00 2001 From: Samuel Redai <43911210+samredai@users.noreply.github.com> Date: Mon, 4 Apr 2022 17:27:42 -0400 Subject: [PATCH 072/642] Python: Add Schema class (#4318) --- src/iceberg/schema.py | 447 ++++++++++++++++++++++++++++++++++++++++++ tests/conftest.py | 85 ++++++++ tests/test_schema.py | 330 +++++++++++++++++++++++++++++++ 3 files changed, 862 insertions(+) create mode 100644 src/iceberg/schema.py create mode 100644 tests/conftest.py create mode 100644 tests/test_schema.py diff --git a/src/iceberg/schema.py b/src/iceberg/schema.py new file mode 100644 index 0000000000..ad60d870bb --- /dev/null +++ b/src/iceberg/schema.py @@ -0,0 +1,447 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +from __future__ import annotations + +import sys +from abc import ABC, abstractmethod +from typing import Dict, Generic, Iterable, List, TypeVar + +if sys.version_info >= (3, 8): + from functools import singledispatch # pragma: no cover +else: + from singledispatch import singledispatch # pragma: no cover + +from iceberg.types import ( + IcebergType, + ListType, + MapType, + NestedField, + PrimitiveType, + StructType, +) + +T = TypeVar("T") + + +class Schema: + """A table Schema + + Example: + >>> from iceberg import schema + >>> from iceberg import types + """ + + def __init__(self, *columns: Iterable[NestedField], schema_id: int, identifier_field_ids: List[int] = []): + self._struct = StructType(*columns) # type: ignore + self._schema_id = schema_id + self._identifier_field_ids = identifier_field_ids + self._name_to_id: Dict[str, int] = index_by_name(self) + self._name_to_id_lower: Dict[str, int] = {} # Should be accessed through self._lazy_name_to_id_lower() + self._id_to_field: Dict[int, NestedField] = {} # Should be accessed through self._lazy_id_to_field() + + def __str__(self): + return "table {\n" + "\n".join([" " + str(field) for field in self.columns]) + "\n}" + + def __repr__(self): + return ( + f"Schema(fields={repr(self.columns)}, schema_id={self.schema_id}, identifier_field_ids={self.identifier_field_ids})" + ) + + @property + def columns(self) -> Iterable[NestedField]: + """A list of the top-level fields in the underlying struct""" + return self._struct.fields + + @property + def schema_id(self) -> int: + """The ID of this Schema""" + return self._schema_id + + @property + def identifier_field_ids(self) -> List[int]: + return self._identifier_field_ids + + def _lazy_id_to_field(self) -> Dict[int, NestedField]: + """Returns an index of field ID to NestedField instance + + This property is calculated once when called for the first time. Subsequent calls to this property will use a cached index. + """ + if not self._id_to_field: + self._id_to_field = index_by_id(self) + return self._id_to_field + + def _lazy_name_to_id_lower(self) -> Dict[str, int]: + """Returns an index of lower-case field names to field IDs + + This property is calculated once when called for the first time. Subsequent calls to this property will use a cached index. + """ + if not self._name_to_id_lower: + self._name_to_id_lower = {name.lower(): field_id for name, field_id in self._name_to_id.items()} + return self._name_to_id_lower + + def as_struct(self) -> StructType: + """Returns the underlying struct""" + return self._struct + + def find_field(self, name_or_id: str | int, case_sensitive: bool = True) -> NestedField: + """Find a field using a field name or field ID + + Args: + name_or_id (str | int): Either a field name or a field ID + case_sensitive (bool, optional): Whether to peform a case-sensitive lookup using a field name. Defaults to True. + + Returns: + NestedField: The matched NestedField + """ + if isinstance(name_or_id, int): + field = self._lazy_id_to_field().get(name_or_id) + return field # type: ignore + if case_sensitive: + field_id = self._name_to_id.get(name_or_id) + else: + field_id = self._lazy_name_to_id_lower().get(name_or_id.lower()) + return self._lazy_id_to_field().get(field_id) # type: ignore + + def find_type(self, name_or_id: str | int, case_sensitive: bool = True) -> IcebergType: + """Find a field type using a field name or field ID + + Args: + name_or_id (str | int): Either a field name or a field ID + case_sensitive (bool, optional): Whether to peform a case-sensitive lookup using a field name. Defaults to True. + + Returns: + NestedField: The type of the matched NestedField + """ + field = self.find_field(name_or_id=name_or_id, case_sensitive=case_sensitive) + return field.type # type: ignore + + def find_column_name(self, column_id: int): + """Find a column name given a column ID + + Args: + column_id (int): The ID of the column + + Raises: + ValueError: If no column name can be found for the given column ID + + Returns: + str: The column name + """ + raise NotImplementedError() + + def select(self, names: List[str], case_sensitive: bool = True) -> "Schema": + """Return a new schema instance pruned to a subset of columns + + Args: + names (List[str]): A list of column names + case_sensitive (bool, optional): Whether to peform a case-sensitive lookup for each column name. Defaults to True. + + Returns: + Schema: A new schema with pruned columns + """ + if case_sensitive: + return self._case_sensitive_select(schema=self, names=names) + return self._case_insensitive_select(schema=self, names=names) + + @classmethod + def _case_sensitive_select(cls, schema: "Schema", names: List[str]): + # TODO: Add a PruneColumns schema visitor and use it here + raise NotImplementedError() + + @classmethod + def _case_insensitive_select(cls, schema: "Schema", names: List[str]): + # TODO: Add a PruneColumns schema visitor and use it here + raise NotImplementedError() + + +class SchemaVisitor(Generic[T], ABC): + def before_field(self, field: NestedField) -> None: + """Override this method to perform an action immediately before visiting a field""" + + def after_field(self, field: NestedField) -> None: + """Override this method to perform an action immediately after visiting a field""" + + def before_list_element(self, element: NestedField) -> None: + """Override this method to perform an action immediately before visiting an element within a ListType""" + self.before_field(element) + + def after_list_element(self, element: NestedField) -> None: + """Override this method to perform an action immediately after visiting an element within a ListType""" + self.after_field(element) + + def before_map_key(self, key: NestedField) -> None: + """Override this method to perform an action immediately before visiting a key within a MapType""" + self.before_field(key) + + def after_map_key(self, key: NestedField) -> None: + """Override this method to perform an action immediately after visiting a key within a MapType""" + self.after_field(key) + + def before_map_value(self, value: NestedField) -> None: + """Override this method to perform an action immediately before visiting a value within a MapType""" + self.before_field(value) + + def after_map_value(self, value: NestedField) -> None: + """Override this method to perform an action immediately after visiting a value within a MapType""" + self.after_field(value) + + @abstractmethod + def schema(self, schema: Schema, struct_result: T) -> T: + """Visit a Schema""" + ... + + @abstractmethod + def struct(self, struct: StructType, field_results: List[T]) -> T: + """Visit a StructType""" + ... + + @abstractmethod + def field(self, field: NestedField, field_result: T) -> T: + """Visit a NestedField""" + ... + + @abstractmethod + def list(self, list_type: ListType, element_result: T) -> T: + """Visit a ListType""" + ... + + @abstractmethod + def map(self, map_type: MapType, key_result: T, value_result: T) -> T: + """Visit a MapType""" + ... + + @abstractmethod + def primitive(self, primitive: PrimitiveType) -> T: + """Visit a PrimitiveType""" + ... + + +@singledispatch +def visit(obj, visitor: SchemaVisitor[T]) -> T: + """A generic function for applying a schema visitor to any point within a schema + + Args: + obj(Schema | IcebergType): An instance of a Schema or an IcebergType + visitor (SchemaVisitor[T]): An instance of an implementation of the generic SchemaVisitor base class + + Raises: + NotImplementedError: If attempting to visit an unrecognized object type + """ + raise NotImplementedError("Cannot visit non-type: %s" % obj) + + +@visit.register(Schema) +def _(obj: Schema, visitor: SchemaVisitor[T]) -> T: + """Visit a Schema with a concrete SchemaVisitor""" + return visitor.schema(obj, visit(obj.as_struct(), visitor)) + + +@visit.register(StructType) +def _(obj: StructType, visitor: SchemaVisitor[T]) -> T: + """Visit a StructType with a concrete SchemaVisitor""" + results = [] + for field in obj.fields: + visitor.before_field(field) + try: + result = visit(field.type, visitor) + finally: + visitor.after_field(field) + + results.append(visitor.field(field, result)) + + return visitor.struct(obj, results) + + +@visit.register(ListType) +def _(obj: ListType, visitor: SchemaVisitor[T]) -> T: + """Visit a ListType with a concrete SchemaVisitor""" + visitor.before_list_element(obj.element) + try: + result = visit(obj.element.type, visitor) + finally: + visitor.after_list_element(obj.element) + + return visitor.list(obj, result) + + +@visit.register(MapType) +def _(obj: MapType, visitor: SchemaVisitor[T]) -> T: + """Visit a MapType with a concrete SchemaVisitor""" + visitor.before_map_key(obj.key) + try: + key_result = visit(obj.key.type, visitor) + finally: + visitor.after_map_key(obj.key) + + visitor.before_map_value(obj.value) + try: + value_result = visit(obj.value.type, visitor) + finally: + visitor.after_list_element(obj.value) + + return visitor.map(obj, key_result, value_result) + + +@visit.register(PrimitiveType) +def _(obj: PrimitiveType, visitor: SchemaVisitor[T]) -> T: + """Visit a PrimitiveType with a concrete SchemaVisitor""" + return visitor.primitive(obj) + + +class _IndexById(SchemaVisitor[Dict[int, NestedField]]): + """A schema visitor for generating a field ID to NestedField index""" + + def __init__(self) -> None: + self._index: Dict[int, NestedField] = {} + + def schema(self, schema, result): + return self._index + + def struct(self, struct, results): + return self._index + + def field(self, field, result): + """Add the field ID to the index""" + self._index[field.field_id] = field + return self._index + + def list(self, list_type, result): + """Add the list element ID to the index""" + self._index[list_type.element.field_id] = list_type.element + return self._index + + def map(self, map_type, key_result, value_result): + """Add the key ID and value ID as individual items in the index""" + self._index[map_type.key.field_id] = map_type.key + self._index[map_type.value.field_id] = map_type.value + return self._index + + def primitive(self, primitive): + return self._index + + +def index_by_id(schema_or_type) -> Dict[int, NestedField]: + """Generate an index of field IDs to NestedField instances + + Args: + schema_or_type (Schema | IcebergType): A schema or type to index + + Returns: + Dict[int, NestedField]: An index of field IDs to NestedField instances + """ + return visit(schema_or_type, _IndexById()) + + +class _IndexByName(SchemaVisitor[Dict[str, int]]): + """A schema visitor for generating a field name to field ID index""" + + def __init__(self) -> None: + self._index: Dict[str, int] = {} + self._short_name_to_id: Dict[str, int] = {} + self._combined_index: Dict[str, int] = {} + self._field_names: List[str] = [] + self._short_field_names: List[str] = [] + + def before_list_element(self, element: NestedField) -> None: + """Short field names omit element when the element is a StructType""" + if not isinstance(element.type, StructType): + self._short_field_names.append(element.name) + self._field_names.append(element.name) + + def after_list_element(self, element: NestedField) -> None: + if not isinstance(element.type, StructType): + self._short_field_names.pop() + self._field_names.pop() + + def before_field(self, field: NestedField) -> None: + """Store the field name""" + self._field_names.append(field.name) + self._short_field_names.append(field.name) + + def after_field(self, field: NestedField) -> None: + """Remove the last field name stored""" + self._field_names.pop() + self._short_field_names.pop() + + def schema(self, schema, struct_result): + return self._index + + def struct(self, struct, field_results): + return self._index + + def field(self, field, field_result): + """Add the field name to the index""" + self._add_field(field.name, field.field_id) + + def list(self, list_type, result): + """Add the list element name to the index""" + self._add_field(list_type.element.name, list_type.element.field_id) + + def map(self, map_type, key_result, value_result): + """Add the key name and value name as individual items in the index""" + self._add_field(map_type.key.name, map_type.key.field_id) + self._add_field(map_type.value.name, map_type.value.field_id) + + def _add_field(self, name: str, field_id: int): + """Add a field name to the index, mapping its full name to its field ID + + Args: + name (str): The field name + field_id (int): The field ID + + Raises: + ValueError: If the field name is already contained in the index + """ + full_name = name + + if self._field_names: + full_name = ".".join([".".join(self._field_names), name]) + + if full_name in self._index: + raise ValueError(f"Invalid schema, multiple fields for name {full_name}: {self._index[full_name]} and {field_id}") + self._index[full_name] = field_id + + if self._short_field_names: + short_name = ".".join([".".join(self._short_field_names), name]) + self._short_name_to_id[short_name] = field_id + + def primitive(self, primitive): + return self._index + + def by_name(self): + """Returns an index of combined full and short names + + Note: Only short names that do not conflict with full names are included. + """ + combined_index = self._short_name_to_id.copy() + combined_index.update(self._index) + return combined_index + + +def index_by_name(schema_or_type) -> Dict[str, int]: + """Generate an index of field names to field IDs + + Args: + schema_or_type (Schema | IcebergType): A schema or type to index + + Returns: + Dict[str, int]: An index of field names to field IDs + """ + indexer = _IndexByName() + visit(schema_or_type, indexer) + return indexer.by_name() diff --git a/tests/conftest.py b/tests/conftest.py new file mode 100644 index 0000000000..d8f5d35fba --- /dev/null +++ b/tests/conftest.py @@ -0,0 +1,85 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +import pytest + +from iceberg import schema +from iceberg.types import ( + BooleanType, + FloatType, + IntegerType, + ListType, + MapType, + NestedField, + StringType, + StructType, +) + + +@pytest.fixture(scope="session", autouse=True) +def table_schema_simple(): + return schema.Schema( + NestedField(field_id=1, name="foo", field_type=StringType(), is_optional=False), + NestedField(field_id=2, name="bar", field_type=IntegerType(), is_optional=True), + NestedField(field_id=3, name="baz", field_type=BooleanType(), is_optional=False), + schema_id=1, + identifier_field_ids=[1], + ) + + +@pytest.fixture(scope="session", autouse=True) +def table_schema_nested(): + return schema.Schema( + NestedField(field_id=1, name="foo", field_type=StringType(), is_optional=False), + NestedField(field_id=2, name="bar", field_type=IntegerType(), is_optional=True), + NestedField(field_id=3, name="baz", field_type=BooleanType(), is_optional=False), + NestedField( + field_id=4, + name="qux", + field_type=ListType(element_id=5, element_type=StringType(), element_is_optional=True), + is_optional=True, + ), + NestedField( + field_id=6, + name="quux", + field_type=MapType( + key_id=7, + key_type=StringType(), + value_id=8, + value_type=MapType( + key_id=9, key_type=StringType(), value_id=10, value_type=IntegerType(), value_is_optional=True + ), + value_is_optional=True, + ), + is_optional=True, + ), + NestedField( + field_id=11, + name="location", + field_type=ListType( + element_id=12, + element_type=StructType( + NestedField(field_id=13, name="latitude", field_type=FloatType(), is_optional=False), + NestedField(field_id=14, name="longitude", field_type=FloatType(), is_optional=False), + ), + element_is_optional=True, + ), + is_optional=True, + ), + schema_id=1, + identifier_field_ids=[1], + ) diff --git a/tests/test_schema.py b/tests/test_schema.py new file mode 100644 index 0000000000..321ee470b7 --- /dev/null +++ b/tests/test_schema.py @@ -0,0 +1,330 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +from textwrap import dedent + +import pytest + +from iceberg import schema +from iceberg.types import ( + BooleanType, + FloatType, + IntegerType, + ListType, + MapType, + NestedField, + StringType, + StructType, +) + + +def test_schema_str(table_schema_simple): + """Test casting a schema to a string""" + assert str(table_schema_simple) == dedent( + """\ + table { + 1: foo: required string + 2: bar: optional int + 3: baz: required boolean + }""" + ) + + +@pytest.mark.parametrize( + "schema, expected_repr", + [ + ( + schema.Schema(NestedField(1, "foo", StringType()), schema_id=1), + "Schema(fields=(NestedField(field_id=1, name='foo', field_type=StringType(), is_optional=True),), schema_id=1, identifier_field_ids=[])", + ), + ( + schema.Schema( + NestedField(1, "foo", StringType()), NestedField(2, "bar", IntegerType(), is_optional=False), schema_id=1 + ), + "Schema(fields=(NestedField(field_id=1, name='foo', field_type=StringType(), is_optional=True), NestedField(field_id=2, name='bar', field_type=IntegerType(), is_optional=False)), schema_id=1, identifier_field_ids=[])", + ), + ], +) +def test_schema_repr(schema, expected_repr): + """Test schema representation""" + assert repr(schema) == expected_repr + + +def test_schema_index_by_id_visitor(table_schema_nested): + """Test index_by_id visitor function""" + index = schema.index_by_id(table_schema_nested) + assert index == { + 1: NestedField(field_id=1, name="foo", field_type=StringType(), is_optional=False), + 2: NestedField(field_id=2, name="bar", field_type=IntegerType(), is_optional=True), + 3: NestedField(field_id=3, name="baz", field_type=BooleanType(), is_optional=False), + 4: NestedField( + field_id=4, + name="qux", + field_type=ListType(element_id=5, element_type=StringType(), element_is_optional=True), + is_optional=True, + ), + 5: NestedField(field_id=5, name="element", field_type=StringType(), is_optional=True), + 6: NestedField( + field_id=6, + name="quux", + field_type=MapType( + key_id=7, + key_type=StringType(), + value_id=8, + value_type=MapType( + key_id=9, key_type=StringType(), value_id=10, value_type=IntegerType(), value_is_optional=True + ), + value_is_optional=True, + ), + is_optional=True, + ), + 7: NestedField(field_id=7, name="key", field_type=StringType(), is_optional=False), + 9: NestedField(field_id=9, name="key", field_type=StringType(), is_optional=False), + 8: NestedField( + field_id=8, + name="value", + field_type=MapType(key_id=9, key_type=StringType(), value_id=10, value_type=IntegerType(), value_is_optional=True), + is_optional=True, + ), + 10: NestedField(field_id=10, name="value", field_type=IntegerType(), is_optional=True), + 11: NestedField( + field_id=11, + name="location", + field_type=ListType( + element_id=12, + element_type=StructType( + NestedField(field_id=13, name="latitude", field_type=FloatType(), is_optional=False), + NestedField(field_id=14, name="longitude", field_type=FloatType(), is_optional=False), + ), + element_is_optional=True, + ), + is_optional=True, + ), + 12: NestedField( + field_id=12, + name="element", + field_type=StructType( + NestedField(field_id=13, name="latitude", field_type=FloatType(), is_optional=False), + NestedField(field_id=14, name="longitude", field_type=FloatType(), is_optional=False), + ), + is_optional=True, + ), + 13: NestedField(field_id=13, name="latitude", field_type=FloatType(), is_optional=False), + 14: NestedField(field_id=14, name="longitude", field_type=FloatType(), is_optional=False), + } + + +def test_schema_index_by_name_visitor(table_schema_nested): + """Test index_by_name visitor function""" + index = schema.index_by_name(table_schema_nested) + assert index == { + "foo": 1, + "bar": 2, + "baz": 3, + "qux": 4, + "qux.element": 5, + "quux": 6, + "quux.key": 7, + "quux.value": 8, + "quux.value.key": 9, + "quux.value.value": 10, + "location": 11, + "location.element": 12, + "location.element.latitude": 13, + "location.element.longitude": 14, + "location.latitude": 13, + "location.longitude": 14, + } + + +# def test_schema_find_column_name(table_schema_nested): +# """Test finding a column name using its field ID""" +# assert table_schema_nested.find_column_name(1) == "foo" +# assert table_schema_nested.find_column_name(2) == "bar" +# assert table_schema_nested.find_column_name(3) == "baz" +# assert table_schema_nested.find_column_name(4) == "qux" +# assert table_schema_nested.find_column_name(5) == "qux.element" +# assert table_schema_nested.find_column_name(6) == "quux" +# assert table_schema_nested.find_column_name(7) == "quux.key" +# assert table_schema_nested.find_column_name(8) == "quux.value" +# assert table_schema_nested.find_column_name(9) == "quux.value.key" +# assert table_schema_nested.find_column_name(10) == "quux.value.value" + + +# def test_schema_find_column_name_on_id_not_found(table_schema_nested): +# """Test raising an error when a field ID cannot be found""" +# assert table_schema_nested.find_column_name(99) is None + + +# def test_schema_find_column_name(table_schema_simple): +# """Test finding a column name given its field ID""" +# assert table_schema_simple.find_column_name(1) == "foo" +# assert table_schema_simple.find_column_name(2) == "bar" +# assert table_schema_simple.find_column_name(3) == "baz" + + +def test_schema_find_field_by_id(table_schema_simple): + """Test finding a column using its field ID""" + index = schema.index_by_id(table_schema_simple) + + column1 = index[1] + assert isinstance(column1, NestedField) + assert column1.field_id == 1 + assert column1.type == StringType() + assert column1.is_optional == False + + column2 = index[2] + assert isinstance(column2, NestedField) + assert column2.field_id == 2 + assert column2.type == IntegerType() + assert column2.is_optional == True + + column3 = index[3] + assert isinstance(column3, NestedField) + assert column3.field_id == 3 + assert column3.type == BooleanType() + assert column3.is_optional == False + + +def test_schema_find_field_by_id_raise_on_unknown_field(table_schema_simple): + """Test raising when the field ID is not found among columns""" + index = schema.index_by_id(table_schema_simple) + with pytest.raises(Exception) as exc_info: + index[4] + assert str(exc_info.value) == "4" + + +def test_schema_find_field_type_by_id(table_schema_simple): + """Test retrieving a columns's type using its field ID""" + index = schema.index_by_id(table_schema_simple) + assert index[1] == NestedField(field_id=1, name="foo", field_type=StringType(), is_optional=False) + assert index[2] == NestedField(field_id=2, name="bar", field_type=IntegerType(), is_optional=True) + assert index[3] == NestedField(field_id=3, name="baz", field_type=BooleanType(), is_optional=False) + + +def test_index_by_id_schema_visitor(table_schema_nested): + """Test the index_by_id function that uses the IndexById schema visitor""" + assert schema.index_by_id(table_schema_nested) == { + 1: NestedField(field_id=1, name="foo", field_type=StringType(), is_optional=False), + 2: NestedField(field_id=2, name="bar", field_type=IntegerType(), is_optional=True), + 3: NestedField(field_id=3, name="baz", field_type=BooleanType(), is_optional=False), + 4: NestedField( + field_id=4, + name="qux", + field_type=ListType(element_id=5, element_type=StringType(), element_is_optional=True), + is_optional=True, + ), + 5: NestedField(field_id=5, name="element", field_type=StringType(), is_optional=True), + 6: NestedField( + field_id=6, + name="quux", + field_type=MapType( + key_id=7, + key_type=StringType(), + value_id=8, + value_type=MapType( + key_id=9, key_type=StringType(), value_id=10, value_type=IntegerType(), value_is_optional=True + ), + value_is_optional=True, + ), + is_optional=True, + ), + 7: NestedField(field_id=7, name="key", field_type=StringType(), is_optional=False), + 8: NestedField( + field_id=8, + name="value", + field_type=MapType(key_id=9, key_type=StringType(), value_id=10, value_type=IntegerType(), value_is_optional=True), + is_optional=True, + ), + 9: NestedField(field_id=9, name="key", field_type=StringType(), is_optional=False), + 10: NestedField(field_id=10, name="value", field_type=IntegerType(), is_optional=True), + 11: NestedField( + field_id=11, + name="location", + field_type=ListType( + element_id=12, + element_type=StructType( + NestedField(field_id=13, name="latitude", field_type=FloatType(), is_optional=False), + NestedField(field_id=14, name="longitude", field_type=FloatType(), is_optional=False), + ), + element_is_optional=True, + ), + is_optional=True, + ), + 12: NestedField( + field_id=12, + name="element", + field_type=StructType( + NestedField(field_id=13, name="latitude", field_type=FloatType(), is_optional=False), + NestedField(field_id=14, name="longitude", field_type=FloatType(), is_optional=False), + ), + is_optional=True, + ), + 13: NestedField(field_id=13, name="latitude", field_type=FloatType(), is_optional=False), + 14: NestedField(field_id=14, name="longitude", field_type=FloatType(), is_optional=False), + } + + +def test_index_by_id_schema_visitor_raise_on_unregistered_type(): + """Test raising a NotImplementedError when an invalid type is provided to the index_by_id function""" + with pytest.raises(NotImplementedError) as exc_info: + schema.index_by_id("foo") + assert "Cannot visit non-type: foo" in str(exc_info.value) + + +def test_schema_find_field(table_schema_simple): + """Test finding a field in a schema""" + assert ( + table_schema_simple.find_field(1) + == table_schema_simple.find_field("foo") + == table_schema_simple.find_field("FOO", case_sensitive=False) + == NestedField(field_id=1, name="foo", field_type=StringType(), is_optional=False) + ) + assert ( + table_schema_simple.find_field(2) + == table_schema_simple.find_field("bar") + == table_schema_simple.find_field("BAR", case_sensitive=False) + == NestedField(field_id=2, name="bar", field_type=IntegerType(), is_optional=True) + ) + assert ( + table_schema_simple.find_field(3) + == table_schema_simple.find_field("baz") + == table_schema_simple.find_field("BAZ", case_sensitive=False) + == NestedField(field_id=3, name="baz", field_type=BooleanType(), is_optional=False) + ) + + +def test_schema_find_type(table_schema_simple): + """Test finding the type of a column given its field ID""" + assert ( + table_schema_simple.find_type(1) + == table_schema_simple.find_type("foo") + == table_schema_simple.find_type("FOO", case_sensitive=False) + == StringType() + ) + assert ( + table_schema_simple.find_type(2) + == table_schema_simple.find_type("bar") + == table_schema_simple.find_type("BAR", case_sensitive=False) + == IntegerType() + ) + assert ( + table_schema_simple.find_type(3) + == table_schema_simple.find_type("baz") + == table_schema_simple.find_type("BAZ", case_sensitive=False) + == BooleanType() + ) From 86aa9223cd82b327b8bc72e62c4e6b77d7fdb388 Mon Sep 17 00:00:00 2001 From: Samuel Redai <43911210+samredai@users.noreply.github.com> Date: Tue, 5 Apr 2022 19:11:39 -0400 Subject: [PATCH 073/642] Python: Implement Schema.find_column_name (#4504) --- src/iceberg/io/pyarrow.py | 4 +-- src/iceberg/schema.py | 54 +++++++++++++++++++++++-------- tests/test_schema.py | 67 +++++++++++++++++++++++++-------------- 3 files changed, 85 insertions(+), 40 deletions(-) diff --git a/src/iceberg/io/pyarrow.py b/src/iceberg/io/pyarrow.py index 2cf5b9fc15..ced7001dfd 100644 --- a/src/iceberg/io/pyarrow.py +++ b/src/iceberg/io/pyarrow.py @@ -42,12 +42,12 @@ class PyArrowFile(InputFile, OutputFile): Examples: >>> from iceberg.io.pyarrow import PyArrowFile - >>> input_file = PyArrowFile("s3://foo/bar.txt") + >>> # input_file = PyArrowFile("s3://foo/bar.txt") >>> # Read the contents of the PyArrowFile instance >>> # Make sure that you have permissions to read/write >>> # file_content = input_file.open().read() - >>> output_file = PyArrowFile("s3://baz/qux.txt") + >>> # output_file = PyArrowFile("s3://baz/qux.txt") >>> # Write bytes to a file >>> # Make sure that you have permissions to read/write >>> # output_file.create().write(b'foobytes') diff --git a/src/iceberg/schema.py b/src/iceberg/schema.py index ad60d870bb..59aab2f3f9 100644 --- a/src/iceberg/schema.py +++ b/src/iceberg/schema.py @@ -53,6 +53,7 @@ def __init__(self, *columns: Iterable[NestedField], schema_id: int, identifier_f self._name_to_id: Dict[str, int] = index_by_name(self) self._name_to_id_lower: Dict[str, int] = {} # Should be accessed through self._lazy_name_to_id_lower() self._id_to_field: Dict[int, NestedField] = {} # Should be accessed through self._lazy_id_to_field() + self._id_to_name: Dict[int, str] = {} # Should be accessed through self._lazy_id_to_name() def __str__(self): return "table {\n" + "\n".join([" " + str(field) for field in self.columns]) + "\n}" @@ -79,7 +80,7 @@ def identifier_field_ids(self) -> List[int]: def _lazy_id_to_field(self) -> Dict[int, NestedField]: """Returns an index of field ID to NestedField instance - This property is calculated once when called for the first time. Subsequent calls to this property will use a cached index. + This is calculated once when called for the first time. Subsequent calls to this method will use a cached index. """ if not self._id_to_field: self._id_to_field = index_by_id(self) @@ -88,12 +89,21 @@ def _lazy_id_to_field(self) -> Dict[int, NestedField]: def _lazy_name_to_id_lower(self) -> Dict[str, int]: """Returns an index of lower-case field names to field IDs - This property is calculated once when called for the first time. Subsequent calls to this property will use a cached index. + This is calculated once when called for the first time. Subsequent calls to this method will use a cached index. """ if not self._name_to_id_lower: self._name_to_id_lower = {name.lower(): field_id for name, field_id in self._name_to_id.items()} return self._name_to_id_lower + def _lazy_id_to_name(self) -> Dict[int, str]: + """Returns an index of field ID to full name + + This is calculated once when called for the first time. Subsequent calls to this method will use a cached index. + """ + if not self._id_to_name: + self._id_to_name = index_name_by_id(self) + return self._id_to_name + def as_struct(self) -> StructType: """Returns the underlying struct""" return self._struct @@ -130,19 +140,16 @@ def find_type(self, name_or_id: str | int, case_sensitive: bool = True) -> Icebe field = self.find_field(name_or_id=name_or_id, case_sensitive=case_sensitive) return field.type # type: ignore - def find_column_name(self, column_id: int): + def find_column_name(self, column_id: int) -> str: """Find a column name given a column ID Args: column_id (int): The ID of the column - Raises: - ValueError: If no column name can be found for the given column ID - Returns: - str: The column name + str: The column name (or None if the column ID cannot be found) """ - raise NotImplementedError() + return self._lazy_id_to_name().get(column_id) # type: ignore def select(self, names: List[str], case_sensitive: bool = True) -> "Schema": """Return a new schema instance pruned to a subset of columns @@ -203,32 +210,32 @@ def after_map_value(self, value: NestedField) -> None: @abstractmethod def schema(self, schema: Schema, struct_result: T) -> T: """Visit a Schema""" - ... + ... # pragma: no cover @abstractmethod def struct(self, struct: StructType, field_results: List[T]) -> T: """Visit a StructType""" - ... + ... # pragma: no cover @abstractmethod def field(self, field: NestedField, field_result: T) -> T: """Visit a NestedField""" - ... + ... # pragma: no cover @abstractmethod def list(self, list_type: ListType, element_result: T) -> T: """Visit a ListType""" - ... + ... # pragma: no cover @abstractmethod def map(self, map_type: MapType, key_result: T, value_result: T) -> T: """Visit a MapType""" - ... + ... # pragma: no cover @abstractmethod def primitive(self, primitive: PrimitiveType) -> T: """Visit a PrimitiveType""" - ... + ... # pragma: no cover @singledispatch @@ -432,6 +439,11 @@ def by_name(self): combined_index.update(self._index) return combined_index + def by_id(self): + """Returns an index of ID to full names""" + id_to_full_name = dict([(value, key) for key, value in self._index.items()]) + return id_to_full_name + def index_by_name(schema_or_type) -> Dict[str, int]: """Generate an index of field names to field IDs @@ -445,3 +457,17 @@ def index_by_name(schema_or_type) -> Dict[str, int]: indexer = _IndexByName() visit(schema_or_type, indexer) return indexer.by_name() + + +def index_name_by_id(schema_or_type) -> Dict[int, str]: + """Generate an index of field IDs full field names + + Args: + schema_or_type (Schema | IcebergType): A schema or type to index + + Returns: + Dict[str, int]: An index of field IDs to full names + """ + indexer = _IndexByName() + visit(schema_or_type, indexer) + return indexer.by_id() diff --git a/tests/test_schema.py b/tests/test_schema.py index 321ee470b7..131e935309 100644 --- a/tests/test_schema.py +++ b/tests/test_schema.py @@ -64,6 +64,21 @@ def test_schema_repr(schema, expected_repr): assert repr(schema) == expected_repr +def test_schema_raise_on_duplicate_names(): + """Test schema representation""" + with pytest.raises(ValueError) as exc_info: + schema.Schema( + NestedField(field_id=1, name="foo", field_type=StringType(), is_optional=False), + NestedField(field_id=2, name="bar", field_type=IntegerType(), is_optional=True), + NestedField(field_id=3, name="baz", field_type=BooleanType(), is_optional=False), + NestedField(field_id=4, name="baz", field_type=BooleanType(), is_optional=False), + schema_id=1, + identifier_field_ids=[1], + ) + + assert "Invalid schema, multiple fields for name baz: 3 and 4" in str(exc_info.value) + + def test_schema_index_by_id_visitor(table_schema_nested): """Test index_by_id visitor function""" index = schema.index_by_id(table_schema_nested) @@ -151,30 +166,34 @@ def test_schema_index_by_name_visitor(table_schema_nested): } -# def test_schema_find_column_name(table_schema_nested): -# """Test finding a column name using its field ID""" -# assert table_schema_nested.find_column_name(1) == "foo" -# assert table_schema_nested.find_column_name(2) == "bar" -# assert table_schema_nested.find_column_name(3) == "baz" -# assert table_schema_nested.find_column_name(4) == "qux" -# assert table_schema_nested.find_column_name(5) == "qux.element" -# assert table_schema_nested.find_column_name(6) == "quux" -# assert table_schema_nested.find_column_name(7) == "quux.key" -# assert table_schema_nested.find_column_name(8) == "quux.value" -# assert table_schema_nested.find_column_name(9) == "quux.value.key" -# assert table_schema_nested.find_column_name(10) == "quux.value.value" - - -# def test_schema_find_column_name_on_id_not_found(table_schema_nested): -# """Test raising an error when a field ID cannot be found""" -# assert table_schema_nested.find_column_name(99) is None - - -# def test_schema_find_column_name(table_schema_simple): -# """Test finding a column name given its field ID""" -# assert table_schema_simple.find_column_name(1) == "foo" -# assert table_schema_simple.find_column_name(2) == "bar" -# assert table_schema_simple.find_column_name(3) == "baz" +def test_schema_find_column_name(table_schema_nested): + """Test finding a column name using its field ID""" + assert table_schema_nested.find_column_name(1) == "foo" + assert table_schema_nested.find_column_name(2) == "bar" + assert table_schema_nested.find_column_name(3) == "baz" + assert table_schema_nested.find_column_name(4) == "qux" + assert table_schema_nested.find_column_name(5) == "qux.element" + assert table_schema_nested.find_column_name(6) == "quux" + assert table_schema_nested.find_column_name(7) == "quux.key" + assert table_schema_nested.find_column_name(8) == "quux.value" + assert table_schema_nested.find_column_name(9) == "quux.value.key" + assert table_schema_nested.find_column_name(10) == "quux.value.value" + assert table_schema_nested.find_column_name(11) == "location" + assert table_schema_nested.find_column_name(12) == "location.element" + assert table_schema_nested.find_column_name(13) == "location.element.latitude" + assert table_schema_nested.find_column_name(14) == "location.element.longitude" + + +def test_schema_find_column_name_on_id_not_found(table_schema_nested): + """Test raising an error when a field ID cannot be found""" + assert table_schema_nested.find_column_name(99) is None + + +def test_schema_find_column_name(table_schema_simple): + """Test finding a column name given its field ID""" + assert table_schema_simple.find_column_name(1) == "foo" + assert table_schema_simple.find_column_name(2) == "bar" + assert table_schema_simple.find_column_name(3) == "baz" def test_schema_find_field_by_id(table_schema_simple): From cea944aae7d3f13c6b43e0edefcec03055492501 Mon Sep 17 00:00:00 2001 From: Hongyue/Steve Zhang Date: Sun, 10 Apr 2022 13:02:59 -0700 Subject: [PATCH 074/642] Python: Reorganize expressions module structure (#4468) Co-authored-by: Steve Zhang --- .../{expression => expressions}/__init__.py | 0 .../{expressions.py => expressions/base.py} | 45 ++++++++++++++++++ .../{expression => expressions}/literals.py | 47 +------------------ tests/{expression => expressions}/__init__.py | 0 .../test_expressions_base.py} | 36 +++++++------- .../test_literals.py | 2 +- tests/io/{test_base.py => test_io_base.py} | 0 7 files changed, 66 insertions(+), 64 deletions(-) rename src/iceberg/{expression => expressions}/__init__.py (100%) rename src/iceberg/{expressions.py => expressions/base.py} (68%) rename src/iceberg/{expression => expressions}/literals.py (91%) rename tests/{expression => expressions}/__init__.py (100%) rename tests/{test_expressions.py => expressions/test_expressions_base.py} (54%) rename tests/{expression => expressions}/test_literals.py (99%) rename tests/io/{test_base.py => test_io_base.py} (100%) diff --git a/src/iceberg/expression/__init__.py b/src/iceberg/expressions/__init__.py similarity index 100% rename from src/iceberg/expression/__init__.py rename to src/iceberg/expressions/__init__.py diff --git a/src/iceberg/expressions.py b/src/iceberg/expressions/base.py similarity index 68% rename from src/iceberg/expressions.py rename to src/iceberg/expressions/base.py index b34d082125..6651c3600f 100644 --- a/src/iceberg/expressions.py +++ b/src/iceberg/expressions/base.py @@ -14,7 +14,11 @@ # KIND, either express or implied. See the License for the # specific language governing permissions and limitations # under the License. +from abc import ABC, abstractmethod from enum import Enum, auto +from typing import Generic, TypeVar + +T = TypeVar("T") class Operation(Enum): @@ -77,3 +81,44 @@ def negate(self) -> "Operation": Operation.IN: Operation.NOT_IN, Operation.NOT_IN: Operation.IN, } + + +class Literal(Generic[T], ABC): + """Literal which has a value and can be converted between types""" + + def __init__(self, value: T, value_type: type): + if value is None or not isinstance(value, value_type): + raise TypeError(f"Invalid literal value: {value} (not a {value_type})") + self._value = value + + @property + def value(self) -> T: + return self._value # type: ignore + + @abstractmethod + def to(self, type_var): + ... # pragma: no cover + + def __repr__(self): + return f"{type(self).__name__}({self.value})" + + def __str__(self): + return str(self.value) + + def __eq__(self, other): + return self.value == other.value + + def __ne__(self, other): + return not self.__eq__(other) + + def __lt__(self, other): + return self.value < other.value + + def __gt__(self, other): + return self.value > other.value + + def __le__(self, other): + return self.value <= other.value + + def __ge__(self, other): + return self.value >= other.value diff --git a/src/iceberg/expression/literals.py b/src/iceberg/expressions/literals.py similarity index 91% rename from src/iceberg/expression/literals.py rename to src/iceberg/expressions/literals.py index 58dc66ef1b..489c8b29b3 100644 --- a/src/iceberg/expression/literals.py +++ b/src/iceberg/expressions/literals.py @@ -21,10 +21,9 @@ import struct import sys -from abc import ABC, abstractmethod from decimal import ROUND_HALF_UP, Decimal from functools import singledispatch -from typing import Generic, Optional, TypeVar, Union +from typing import Optional, Union from uuid import UUID from iceberg.utils.datetime import ( @@ -40,6 +39,7 @@ else: from singledispatch import singledispatchmethod # pragma: no cover +from iceberg.expressions.base import Literal from iceberg.types import ( BinaryType, BooleanType, @@ -58,49 +58,6 @@ UUIDType, ) -T = TypeVar("T") - - -class Literal(Generic[T], ABC): - """Literal which has a value and can be converted between types""" - - def __init__(self, value: T, value_type: type): - if value is None or not isinstance(value, value_type): - raise TypeError(f"Invalid literal value: {value} (not a {value_type})") - self._value = value - - @property - def value(self) -> T: - return self._value # type: ignore - - @abstractmethod - def to(self, type_var): - ... # pragma: no cover - - def __repr__(self): - return f"{type(self).__name__}({self.value})" - - def __str__(self): - return str(self.value) - - def __eq__(self, other): - return self.value == other.value - - def __ne__(self, other): - return not self.__eq__(other) - - def __lt__(self, other): - return self.value < other.value - - def __gt__(self, other): - return self.value > other.value - - def __le__(self, other): - return self.value <= other.value - - def __ge__(self, other): - return self.value >= other.value - @singledispatch def literal(value) -> Literal: diff --git a/tests/expression/__init__.py b/tests/expressions/__init__.py similarity index 100% rename from tests/expression/__init__.py rename to tests/expressions/__init__.py diff --git a/tests/test_expressions.py b/tests/expressions/test_expressions_base.py similarity index 54% rename from tests/test_expressions.py rename to tests/expressions/test_expressions_base.py index 00f1da1969..b9a797d5c4 100644 --- a/tests/test_expressions.py +++ b/tests/expressions/test_expressions_base.py @@ -17,26 +17,26 @@ import pytest -from iceberg import expressions +from iceberg.expressions import base @pytest.mark.parametrize( "operation,opposite_operation", [ - (expressions.Operation.TRUE, expressions.Operation.FALSE), - (expressions.Operation.FALSE, expressions.Operation.TRUE), - (expressions.Operation.IS_NULL, expressions.Operation.NOT_NULL), - (expressions.Operation.NOT_NULL, expressions.Operation.IS_NULL), - (expressions.Operation.IS_NAN, expressions.Operation.NOT_NAN), - (expressions.Operation.NOT_NAN, expressions.Operation.IS_NAN), - (expressions.Operation.LT, expressions.Operation.GT_EQ), - (expressions.Operation.LT_EQ, expressions.Operation.GT), - (expressions.Operation.GT, expressions.Operation.LT_EQ), - (expressions.Operation.GT_EQ, expressions.Operation.LT), - (expressions.Operation.EQ, expressions.Operation.NOT_EQ), - (expressions.Operation.NOT_EQ, expressions.Operation.EQ), - (expressions.Operation.IN, expressions.Operation.NOT_IN), - (expressions.Operation.NOT_IN, expressions.Operation.IN), + (base.Operation.TRUE, base.Operation.FALSE), + (base.Operation.FALSE, base.Operation.TRUE), + (base.Operation.IS_NULL, base.Operation.NOT_NULL), + (base.Operation.NOT_NULL, base.Operation.IS_NULL), + (base.Operation.IS_NAN, base.Operation.NOT_NAN), + (base.Operation.NOT_NAN, base.Operation.IS_NAN), + (base.Operation.LT, base.Operation.GT_EQ), + (base.Operation.LT_EQ, base.Operation.GT), + (base.Operation.GT, base.Operation.LT_EQ), + (base.Operation.GT_EQ, base.Operation.LT), + (base.Operation.EQ, base.Operation.NOT_EQ), + (base.Operation.NOT_EQ, base.Operation.EQ), + (base.Operation.IN, base.Operation.NOT_IN), + (base.Operation.NOT_IN, base.Operation.IN), ], ) def test_negation_of_operations(operation, opposite_operation): @@ -46,9 +46,9 @@ def test_negation_of_operations(operation, opposite_operation): @pytest.mark.parametrize( "operation", [ - expressions.Operation.NOT, - expressions.Operation.AND, - expressions.Operation.OR, + base.Operation.NOT, + base.Operation.AND, + base.Operation.OR, ], ) def test_raise_on_no_negation_for_operation(operation): diff --git a/tests/expression/test_literals.py b/tests/expressions/test_literals.py similarity index 99% rename from tests/expression/test_literals.py rename to tests/expressions/test_literals.py index 620b65b959..59b943797f 100644 --- a/tests/expression/test_literals.py +++ b/tests/expressions/test_literals.py @@ -20,7 +20,7 @@ import pytest -from iceberg.expression.literals import ( +from iceberg.expressions.literals import ( AboveMax, BelowMin, BinaryLiteral, diff --git a/tests/io/test_base.py b/tests/io/test_io_base.py similarity index 100% rename from tests/io/test_base.py rename to tests/io/test_io_base.py From d370c18d0b4fddfde7d3ad6729373fe726979a03 Mon Sep 17 00:00:00 2001 From: Samuel Redai <43911210+samredai@users.noreply.github.com> Date: Wed, 20 Apr 2022 13:23:40 -0400 Subject: [PATCH 075/642] Python: Add StructProtocol and file type enums (#4598) --- src/iceberg/files.py | 53 ++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 53 insertions(+) create mode 100644 src/iceberg/files.py diff --git a/src/iceberg/files.py b/src/iceberg/files.py new file mode 100644 index 0000000000..409aae5728 --- /dev/null +++ b/src/iceberg/files.py @@ -0,0 +1,53 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +from enum import Enum, auto +from typing import Any + +try: + from typing import Protocol, runtime_checkable +except ImportError: # pragma: no cover + from typing_extensions import Protocol # type: ignore + from typing_extensions import runtime_checkable + + +class FileContentType(Enum): + """An enum that includes all possible content types for an Iceberg data file""" + + DATA = auto() + POSITION_DELETES = auto() + EQUALITY_DELETES = auto() + + +class FileFormat(Enum): + """An enum that includes all possible formats for an Iceberg data file""" + + ORC = auto() + PARQUET = auto() + AVRO = auto() + METADATA = auto() + + +@runtime_checkable +class StructProtocol(Protocol): # pragma: no cover + """A generic protocol used by accessors to get and set at positions of an object""" + + def get(self, pos: int) -> Any: + ... + + def set(self, pos: int, value) -> None: + ... From 5a2814b8065778dd9a230cba7342c73592923d85 Mon Sep 17 00:00:00 2001 From: Samuel Redai <43911210+samredai@users.noreply.github.com> Date: Fri, 22 Apr 2022 15:17:35 -0400 Subject: [PATCH 076/642] Python: Add a generic Accessor class (#4609) --- src/iceberg/expressions/base.py | 27 ++++++++++++++++- tests/conftest.py | 20 +++++++++++++ tests/expressions/test_expressions_base.py | 35 ++++++++++++++++++++++ 3 files changed, 81 insertions(+), 1 deletion(-) diff --git a/src/iceberg/expressions/base.py b/src/iceberg/expressions/base.py index 6651c3600f..f48fb73dad 100644 --- a/src/iceberg/expressions/base.py +++ b/src/iceberg/expressions/base.py @@ -16,7 +16,9 @@ # under the License. from abc import ABC, abstractmethod from enum import Enum, auto -from typing import Generic, TypeVar +from typing import Any, Generic, TypeVar + +from iceberg.files import StructProtocol T = TypeVar("T") @@ -122,3 +124,26 @@ def __le__(self, other): def __ge__(self, other): return self.value >= other.value + + +class Accessor: + """An accessor for a specific position in a container that implements the StructProtocol""" + + def __init__(self, position: int): + self._position = position + + @property + def position(self): + """The position in the container to access""" + return self._position + + def get(self, container: StructProtocol) -> Any: + """Returns the value at self.position in `container` + + Args: + container(StructProtocol): A container to access at position `self.position` + + Returns: + Any: The value at position `self.position` in the container + """ + return container.get(self.position) diff --git a/tests/conftest.py b/tests/conftest.py index d8f5d35fba..a582bb3c0d 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -15,6 +15,8 @@ # specific language governing permissions and limitations # under the License. +from typing import Any + import pytest from iceberg import schema @@ -30,6 +32,19 @@ ) +class FooStruct: + """An example of an object that abides by StructProtocol""" + + def __init__(self): + self.content = {} + + def get(self, pos: int) -> Any: + return self.content[pos] + + def set(self, pos: int, value) -> None: + self.content[pos] = value + + @pytest.fixture(scope="session", autouse=True) def table_schema_simple(): return schema.Schema( @@ -83,3 +98,8 @@ def table_schema_nested(): schema_id=1, identifier_field_ids=[1], ) + + +@pytest.fixture(scope="session", autouse=True) +def foo_struct(): + return FooStruct() diff --git a/tests/expressions/test_expressions_base.py b/tests/expressions/test_expressions_base.py index b9a797d5c4..04db1a6362 100644 --- a/tests/expressions/test_expressions_base.py +++ b/tests/expressions/test_expressions_base.py @@ -15,6 +15,9 @@ # specific language governing permissions and limitations # under the License. +import uuid +from decimal import Decimal + import pytest from iceberg.expressions import base @@ -56,3 +59,35 @@ def test_raise_on_no_negation_for_operation(operation): operation.negate() assert str(exc_info.value) == f"No negation defined for operation {operation}" + + +def test_accessor_base_class(foo_struct): + """Test retrieving a value at a position of a container using an accessor""" + + uuid_value = uuid.uuid4() + + foo_struct.set(0, "foo") + foo_struct.set(1, "bar") + foo_struct.set(2, "baz") + foo_struct.set(3, 1) + foo_struct.set(4, 2) + foo_struct.set(5, 3) + foo_struct.set(6, 1.234) + foo_struct.set(7, Decimal("1.234")) + foo_struct.set(8, uuid_value) + foo_struct.set(9, True) + foo_struct.set(10, False) + foo_struct.set(11, b"\x19\x04\x9e?") + + assert base.Accessor(position=0).get(foo_struct) == "foo" + assert base.Accessor(position=1).get(foo_struct) == "bar" + assert base.Accessor(position=2).get(foo_struct) == "baz" + assert base.Accessor(position=3).get(foo_struct) == 1 + assert base.Accessor(position=4).get(foo_struct) == 2 + assert base.Accessor(position=5).get(foo_struct) == 3 + assert base.Accessor(position=6).get(foo_struct) == 1.234 + assert base.Accessor(position=7).get(foo_struct) == Decimal("1.234") + assert base.Accessor(position=8).get(foo_struct) == uuid_value + assert base.Accessor(position=9).get(foo_struct) == True + assert base.Accessor(position=10).get(foo_struct) == False + assert base.Accessor(position=11).get(foo_struct) == b"\x19\x04\x9e?" From 614c14421952678c641953a4d61b31f5879555eb Mon Sep 17 00:00:00 2001 From: Nick Ouellet <35903677+CircArgs@users.noreply.github.com> Date: Sun, 24 Apr 2022 13:55:51 -0400 Subject: [PATCH 077/642] Python: Add And, Or, Not, AlwaysTrue, AlwaysFalse expressions (#4466) --- src/iceberg/expressions/base.py | 140 +++++++++++++++++++++ tests/expressions/test_expressions_base.py | 126 +++++++++++++++++++ 2 files changed, 266 insertions(+) diff --git a/src/iceberg/expressions/base.py b/src/iceberg/expressions/base.py index f48fb73dad..acac98b91f 100644 --- a/src/iceberg/expressions/base.py +++ b/src/iceberg/expressions/base.py @@ -16,9 +16,11 @@ # under the License. from abc import ABC, abstractmethod from enum import Enum, auto +from functools import reduce from typing import Any, Generic, TypeVar from iceberg.files import StructProtocol +from iceberg.types import Singleton T = TypeVar("T") @@ -126,6 +128,144 @@ def __ge__(self, other): return self.value >= other.value +class BooleanExpression(ABC): + """base class for all boolean expressions""" + + @abstractmethod + def __invert__(self) -> "BooleanExpression": + ... + + +class And(BooleanExpression): + """AND operation expression - logical conjunction""" + + def __new__(cls, left: BooleanExpression, right: BooleanExpression, *rest: BooleanExpression): + if rest: + return reduce(And, (left, right, *rest)) + if left is AlwaysFalse() or right is AlwaysFalse(): + return AlwaysFalse() + elif left is AlwaysTrue(): + return right + elif right is AlwaysTrue(): + return left + self = super().__new__(cls) + self._left = left # type: ignore + self._right = right # type: ignore + return self + + @property + def left(self) -> BooleanExpression: + return self._left # type: ignore + + @property + def right(self) -> BooleanExpression: + return self._right # type: ignore + + def __eq__(self, other) -> bool: + return id(self) == id(other) or (isinstance(other, And) and self.left == other.left and self.right == other.right) + + def __invert__(self) -> "Or": + return Or(~self.left, ~self.right) + + def __repr__(self) -> str: + return f"And({repr(self.left)}, {repr(self.right)})" + + def __str__(self) -> str: + return f"({self.left} and {self.right})" + + +class Or(BooleanExpression): + """OR operation expression - logical disjunction""" + + def __new__(cls, left: BooleanExpression, right: BooleanExpression, *rest: BooleanExpression): + if rest: + return reduce(Or, (left, right, *rest)) + if left is AlwaysTrue() or right is AlwaysTrue(): + return AlwaysTrue() + elif left is AlwaysFalse(): + return right + elif right is AlwaysFalse(): + return left + self = super().__new__(cls) + self._left = left # type: ignore + self._right = right # type: ignore + return self + + @property + def left(self) -> BooleanExpression: + return self._left # type: ignore + + @property + def right(self) -> BooleanExpression: + return self._right # type: ignore + + def __eq__(self, other) -> bool: + return id(self) == id(other) or (isinstance(other, Or) and self.left == other.left and self.right == other.right) + + def __invert__(self) -> "And": + return And(~self.left, ~self.right) + + def __repr__(self) -> str: + return f"Or({repr(self.left)}, {repr(self.right)})" + + def __str__(self) -> str: + return f"({self.left} or {self.right})" + + +class Not(BooleanExpression): + """NOT operation expression - logical negation""" + + def __new__(cls, child: BooleanExpression): + if child is AlwaysTrue(): + return AlwaysFalse() + elif child is AlwaysFalse(): + return AlwaysTrue() + elif isinstance(child, Not): + return child.child + return super().__new__(cls) + + def __init__(self, child): + self.child = child + + def __eq__(self, other) -> bool: + return id(self) == id(other) or (isinstance(other, Not) and self.child == other.child) + + def __invert__(self) -> BooleanExpression: + return self.child + + def __repr__(self) -> str: + return f"Not({repr(self.child)})" + + def __str__(self) -> str: + return f"(not {self.child})" + + +class AlwaysTrue(BooleanExpression, Singleton): + """TRUE expression""" + + def __invert__(self) -> "AlwaysFalse": + return AlwaysFalse() + + def __repr__(self) -> str: + return "AlwaysTrue()" + + def __str__(self) -> str: + return "true" + + +class AlwaysFalse(BooleanExpression, Singleton): + """FALSE expression""" + + def __invert__(self) -> "AlwaysTrue": + return AlwaysTrue() + + def __repr__(self) -> str: + return "AlwaysTrue()" + + def __str__(self) -> str: + return "false" + + class Accessor: """An accessor for a specific position in a container that implements the StructProtocol""" diff --git a/tests/expressions/test_expressions_base.py b/tests/expressions/test_expressions_base.py index 04db1a6362..c976edc494 100644 --- a/tests/expressions/test_expressions_base.py +++ b/tests/expressions/test_expressions_base.py @@ -21,6 +21,7 @@ import pytest from iceberg.expressions import base +from iceberg.types import Singleton @pytest.mark.parametrize( @@ -61,6 +62,131 @@ def test_raise_on_no_negation_for_operation(operation): assert str(exc_info.value) == f"No negation defined for operation {operation}" +class TestExpressionA(base.BooleanExpression, Singleton): + def __invert__(self): + return TestExpressionB() + + def __repr__(self): + return "TestExpressionA()" + + def __str__(self): + return "testexpra" + + +class TestExpressionB(base.BooleanExpression, Singleton): + def __invert__(self): + return TestExpressionA() + + def __repr__(self): + return "TestExpressionB()" + + def __str__(self): + return "testexprb" + + +@pytest.mark.parametrize( + "op, rep", + [ + ( + base.And(TestExpressionA(), TestExpressionB()), + "And(TestExpressionA(), TestExpressionB())", + ), + ( + base.Or(TestExpressionA(), TestExpressionB()), + "Or(TestExpressionA(), TestExpressionB())", + ), + (base.Not(TestExpressionA()), "Not(TestExpressionA())"), + ], +) +def test_reprs(op, rep): + assert repr(op) == rep + + +@pytest.mark.parametrize( + "op, string", + [ + (base.And(TestExpressionA(), TestExpressionB()), "(testexpra and testexprb)"), + (base.Or(TestExpressionA(), TestExpressionB()), "(testexpra or testexprb)"), + (base.Not(TestExpressionA()), "(not testexpra)"), + ], +) +def test_strs(op, string): + assert str(op) == string + + +@pytest.mark.parametrize( + "input, testexpra, testexprb", + [ + ( + base.And(TestExpressionA(), TestExpressionB()), + base.And(TestExpressionA(), TestExpressionB()), + base.Or(TestExpressionA(), TestExpressionB()), + ), + ( + base.Or(TestExpressionA(), TestExpressionB()), + base.Or(TestExpressionA(), TestExpressionB()), + base.And(TestExpressionA(), TestExpressionB()), + ), + (base.Not(TestExpressionA()), base.Not(TestExpressionA()), TestExpressionB()), + (TestExpressionA(), TestExpressionA(), TestExpressionB()), + (TestExpressionB(), TestExpressionB(), TestExpressionA()), + ], +) +def test_eq(input, testexpra, testexprb): + assert input == testexpra and input != testexprb + + +@pytest.mark.parametrize( + "input, exp", + [ + ( + base.And(TestExpressionA(), TestExpressionB()), + base.Or(TestExpressionB(), TestExpressionA()), + ), + ( + base.Or(TestExpressionA(), TestExpressionB()), + base.And(TestExpressionB(), TestExpressionA()), + ), + (base.Not(TestExpressionA()), TestExpressionA()), + (TestExpressionA(), TestExpressionB()), + ], +) +def test_negate(input, exp): + assert ~input == exp + + +@pytest.mark.parametrize( + "input, exp", + [ + ( + base.And(TestExpressionA(), TestExpressionB(), TestExpressionA()), + base.And(base.And(TestExpressionA(), TestExpressionB()), TestExpressionA()), + ), + ( + base.Or(TestExpressionA(), TestExpressionB(), TestExpressionA()), + base.Or(base.Or(TestExpressionA(), TestExpressionB()), TestExpressionA()), + ), + (base.Not(base.Not(TestExpressionA())), TestExpressionA()), + ], +) +def test_reduce(input, exp): + assert input == exp + + +@pytest.mark.parametrize( + "input, exp", + [ + (base.And(base.AlwaysTrue(), TestExpressionB()), TestExpressionB()), + (base.And(base.AlwaysFalse(), TestExpressionB()), base.AlwaysFalse()), + (base.Or(base.AlwaysTrue(), TestExpressionB()), base.AlwaysTrue()), + (base.Or(base.AlwaysFalse(), TestExpressionB()), TestExpressionB()), + (base.Not(base.Not(TestExpressionA())), TestExpressionA()), + ], +) +def test_base_AlwaysTrue_base_AlwaysFalse(input, exp): + assert input == exp + + def test_accessor_base_class(foo_struct): """Test retrieving a value at a position of a container using an accessor""" From 00ea758fb6556b2a5a0b0db23ba85f66bf9ff256 Mon Sep 17 00:00:00 2001 From: Samuel Redai <43911210+samredai@users.noreply.github.com> Date: Sun, 24 Apr 2022 18:07:12 -0400 Subject: [PATCH 078/642] Python: Add BoundReference class (#4619) --- src/iceberg/expressions/base.py | 43 +++++++++++++++++++++- tests/expressions/test_expressions_base.py | 42 ++++++++++++++++++++- 2 files changed, 83 insertions(+), 2 deletions(-) diff --git a/src/iceberg/expressions/base.py b/src/iceberg/expressions/base.py index acac98b91f..806e6f748d 100644 --- a/src/iceberg/expressions/base.py +++ b/src/iceberg/expressions/base.py @@ -20,7 +20,7 @@ from typing import Any, Generic, TypeVar from iceberg.files import StructProtocol -from iceberg.types import Singleton +from iceberg.types import NestedField, Singleton T = TypeVar("T") @@ -272,6 +272,12 @@ class Accessor: def __init__(self, position: int): self._position = position + def __str__(self): + return f"Accessor(position={self._position})" + + def __repr__(self): + return f"Accessor(position={self._position})" + @property def position(self): """The position in the container to access""" @@ -287,3 +293,38 @@ def get(self, container: StructProtocol) -> Any: Any: The value at position `self.position` in the container """ return container.get(self.position) + + +class BoundReference: + """A reference bound to a field in a schema + + Args: + field (NestedField): A referenced field in an Iceberg schema + accessor (Accessor): An Accessor object to access the value at the field's position + """ + + def __init__(self, field: NestedField, accessor: Accessor): + self._field = field + self._accessor = accessor + + def __str__(self): + return f"BoundReference(field={repr(self.field)}, accessor={repr(self._accessor)})" + + def __repr__(self): + return f"BoundReference(field={repr(self.field)}, accessor={repr(self._accessor)})" + + @property + def field(self) -> NestedField: + """The referenced field""" + return self._field + + def eval(self, struct: StructProtocol) -> Any: + """Returns the value at the referenced field's position in an object that abides by the StructProtocol + + Args: + struct (StructProtocol): A row object that abides by the StructProtocol and returns values given a position + + Returns: + Any: The value at the referenced field's position in `struct` + """ + return self._accessor.get(struct) diff --git a/tests/expressions/test_expressions_base.py b/tests/expressions/test_expressions_base.py index c976edc494..61b801883a 100644 --- a/tests/expressions/test_expressions_base.py +++ b/tests/expressions/test_expressions_base.py @@ -21,7 +21,7 @@ import pytest from iceberg.expressions import base -from iceberg.types import Singleton +from iceberg.types import NestedField, Singleton, StringType @pytest.mark.parametrize( @@ -217,3 +217,43 @@ def test_accessor_base_class(foo_struct): assert base.Accessor(position=9).get(foo_struct) == True assert base.Accessor(position=10).get(foo_struct) == False assert base.Accessor(position=11).get(foo_struct) == b"\x19\x04\x9e?" + + +def test_bound_reference_str_and_repr(): + """Test str and repr of BoundReference""" + field = NestedField(field_id=1, name="foo", field_type=StringType(), is_optional=False) + position1_accessor = base.Accessor(position=1) + bound_ref = base.BoundReference(field=field, accessor=position1_accessor) + assert str(bound_ref) == f"BoundReference(field={repr(field)}, accessor={repr(position1_accessor)})" + assert repr(bound_ref) == f"BoundReference(field={repr(field)}, accessor={repr(position1_accessor)})" + + +def test_bound_reference_field_property(): + """Test str and repr of BoundReference""" + field = NestedField(field_id=1, name="foo", field_type=StringType(), is_optional=False) + position1_accessor = base.Accessor(position=1) + bound_ref = base.BoundReference(field=field, accessor=position1_accessor) + assert bound_ref.field == NestedField(field_id=1, name="foo", field_type=StringType(), is_optional=False) + + +def test_bound_reference(table_schema_simple, foo_struct): + """Test creating a BoundReference and evaluating it on a StructProtocol""" + foo_struct.set(pos=1, value="foovalue") + foo_struct.set(pos=2, value=123) + foo_struct.set(pos=3, value=True) + + position1_accessor = base.Accessor(position=1) + position2_accessor = base.Accessor(position=2) + position3_accessor = base.Accessor(position=3) + + field1 = table_schema_simple.find_field(1) + field2 = table_schema_simple.find_field(2) + field3 = table_schema_simple.find_field(3) + + bound_ref1 = base.BoundReference(field=field1, accessor=position1_accessor) + bound_ref2 = base.BoundReference(field=field2, accessor=position2_accessor) + bound_ref3 = base.BoundReference(field=field3, accessor=position3_accessor) + + assert bound_ref1.eval(foo_struct) == "foovalue" + assert bound_ref2.eval(foo_struct) == 123 + assert bound_ref3.eval(foo_struct) == True From 0945b2b6d89779ca4687c7045ed315076629c134 Mon Sep 17 00:00:00 2001 From: Hongyue/Steve Zhang Date: Mon, 25 Apr 2022 08:14:36 -0700 Subject: [PATCH 079/642] Python: Add PartitionField (#4590) Co-authored-by: Steve Zhang --- src/iceberg/table/partitioning.py | 66 +++++++++++++++++++++++++++++++ tests/table/__init__.py | 16 ++++++++ tests/table/test_partitioning.py | 36 +++++++++++++++++ 3 files changed, 118 insertions(+) create mode 100644 src/iceberg/table/partitioning.py create mode 100644 tests/table/__init__.py create mode 100644 tests/table/test_partitioning.py diff --git a/src/iceberg/table/partitioning.py b/src/iceberg/table/partitioning.py new file mode 100644 index 0000000000..4a9faf4a39 --- /dev/null +++ b/src/iceberg/table/partitioning.py @@ -0,0 +1,66 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +from iceberg.transforms import Transform + + +class PartitionField: + """ + PartitionField is a single element with name and unique id, + It represents how one partition value is derived from the source column via transformation + + Attributes: + source_id(int): The source column id of table's schema + field_id(int): The partition field id across all the table metadata's partition specs + transform(Transform): The transform used to produce partition values from source column + name(str): The name of this partition field + """ + + def __init__(self, source_id: int, field_id: int, transform: Transform, name: str): + self._source_id = source_id + self._field_id = field_id + self._transform = transform + self._name = name + + @property + def source_id(self) -> int: + return self._source_id + + @property + def field_id(self) -> int: + return self._field_id + + @property + def name(self) -> str: + return self._name + + @property + def transform(self) -> Transform: + return self._transform + + def __eq__(self, other): + return ( + self.field_id == other.field_id + and self.source_id == other.source_id + and self.name == other.name + and self.transform == other.transform + ) + + def __str__(self): + return f"{self.field_id}: {self.name}: {self.transform}({self.source_id})" + + def __repr__(self): + return f"PartitionField(field_id={self.field_id}, name={self.name}, transform={repr(self.transform)}, source_id={self.source_id})" diff --git a/tests/table/__init__.py b/tests/table/__init__.py new file mode 100644 index 0000000000..13a83393a9 --- /dev/null +++ b/tests/table/__init__.py @@ -0,0 +1,16 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. diff --git a/tests/table/test_partitioning.py b/tests/table/test_partitioning.py new file mode 100644 index 0000000000..baf2e5df7a --- /dev/null +++ b/tests/table/test_partitioning.py @@ -0,0 +1,36 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +from iceberg.table.partitioning import PartitionField +from iceberg.transforms import bucket +from iceberg.types import IntegerType + + +def test_partition_field_init(): + bucket_transform = bucket(IntegerType(), 100) + partition_field = PartitionField(3, 1000, bucket_transform, "id") + + assert partition_field.source_id == 3 + assert partition_field.field_id == 1000 + assert partition_field.transform == bucket_transform + assert partition_field.name == "id" + assert partition_field == partition_field + assert str(partition_field) == "1000: id: bucket[100](3)" + assert ( + repr(partition_field) + == "PartitionField(field_id=1000, name=id, transform=transforms.bucket(source_type=IntegerType(), num_buckets=100), source_id=3)" + ) From 24754afd084c4527268959ae2b0a81f4ab0a1eac Mon Sep 17 00:00:00 2001 From: Samuel Redai <43911210+samredai@users.noreply.github.com> Date: Wed, 27 Apr 2022 17:21:09 -0400 Subject: [PATCH 080/642] Python: Update docs to avoid InterpreterNotFound errors from tox (#4650) --- README.md | 15 ++++++++++++++- 1 file changed, 14 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index 35fe612026..e2c2e38a40 100644 --- a/README.md +++ b/README.md @@ -35,9 +35,22 @@ pip install -e . Testing is done using tox. The config can be found in `tox.ini` within the python directory of the iceberg project. ``` -# simply run tox within the python dir tox ``` +## Solution for `InterpreterNotFound` Errors + +Currently, tests run against python `3.7.12`, `3.8.12`, and `3.9.10`. It's recommended to install and manage multiple interpreters using [pyenv](https://github.com/pyenv/pyenv). +``` +pyenv install 3.7.12 +pyenv install 3.8.12 +pyenv install 3.9.10 +``` + +Once all three versions are installed, you can set an application-specific pyenv environment by running the following in the python directory. +``` +pyenv local 3.7.12 3.8.12 3.9.10 +``` + ## Get in Touch - [Iceberg community](https://iceberg.apache.org/community/) From 68ab9202e15ee13717800893e2a692c2793b9d72 Mon Sep 17 00:00:00 2001 From: Samuel Redai <43911210+samredai@users.noreply.github.com> Date: Mon, 2 May 2022 14:57:59 -0400 Subject: [PATCH 081/642] Python: Specify pip>=21.1 in tox dependencies (#4676) --- tox.ini | 1 + 1 file changed, 1 insertion(+) diff --git a/tox.ini b/tox.ini index 8e47a37204..92757798b6 100644 --- a/tox.ini +++ b/tox.ini @@ -22,6 +22,7 @@ envlist = py37,py38,py39,linters usedevelop = true extras = arrow deps = + pip>=21.1 coverage mock pytest From 9326d0628a995e49732f7dd0c2a81016fed5572a Mon Sep 17 00:00:00 2001 From: Samuel Redai <43911210+samredai@users.noreply.github.com> Date: Mon, 2 May 2022 19:17:49 -0400 Subject: [PATCH 082/642] Python: Add a skeleton for the BuildPositionAccessors visitor (#4678) Co-authored-by: Fokko Driesprong --- src/iceberg/schema.py | 65 ++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 64 insertions(+), 1 deletion(-) diff --git a/src/iceberg/schema.py b/src/iceberg/schema.py index 59aab2f3f9..36f7e1dcc3 100644 --- a/src/iceberg/schema.py +++ b/src/iceberg/schema.py @@ -19,7 +19,10 @@ import sys from abc import ABC, abstractmethod -from typing import Dict, Generic, Iterable, List, TypeVar +from typing import TYPE_CHECKING, Dict, Generic, Iterable, List, TypeVar + +if TYPE_CHECKING: + from iceberg.expressions.base import Accessor if sys.version_info >= (3, 8): from functools import singledispatch # pragma: no cover @@ -54,6 +57,7 @@ def __init__(self, *columns: Iterable[NestedField], schema_id: int, identifier_f self._name_to_id_lower: Dict[str, int] = {} # Should be accessed through self._lazy_name_to_id_lower() self._id_to_field: Dict[int, NestedField] = {} # Should be accessed through self._lazy_id_to_field() self._id_to_name: Dict[int, str] = {} # Should be accessed through self._lazy_id_to_name() + self._id_to_accessor: Dict[int, Accessor] = {} # Should be accessed through self._lazy_id_to_accessor() def __str__(self): return "table {\n" + "\n".join([" " + str(field) for field in self.columns]) + "\n}" @@ -104,6 +108,15 @@ def _lazy_id_to_name(self) -> Dict[int, str]: self._id_to_name = index_name_by_id(self) return self._id_to_name + def _lazy_id_to_accessor(self) -> Dict[int, Accessor]: + """Returns an index of field ID to accessor + + This is calculated once when called for the first time. Subsequent calls to this method will use a cached index. + """ + if not self._id_to_accessor: + self._id_to_accessor = build_position_accessors(self) + return self._id_to_accessor + def as_struct(self) -> StructType: """Returns the underlying struct""" return self._struct @@ -151,6 +164,17 @@ def find_column_name(self, column_id: int) -> str: """ return self._lazy_id_to_name().get(column_id) # type: ignore + def accessor_for_field(self, field_id: int) -> Accessor: + """Find a schema position accessor given a field ID + + Args: + field_id (int): The ID of the field + + Returns: + Accessor: An accessor for the given field ID + """ + return self._lazy_id_to_accessor().get(field_id) # type: ignore + def select(self, names: List[str], case_sensitive: bool = True) -> "Schema": """Return a new schema instance pruned to a subset of columns @@ -471,3 +495,42 @@ def index_name_by_id(schema_or_type) -> Dict[int, str]: indexer = _IndexByName() visit(schema_or_type, indexer) return indexer.by_id() + + +class _BuildPositionAccessors(SchemaVisitor[Dict[int, "Accessor"]]): + """A schema visitor for generating a field ID to accessor index""" + + def __init__(self) -> None: + self._index: Dict[int, Accessor] = {} + + def schema(self, schema, result: Dict[int, Accessor]) -> Dict[int, Accessor]: + return self._index + + def struct(self, struct, result: List[Dict[int, Accessor]]) -> Dict[int, Accessor]: + # TODO: Populate the `self._index` dictionary where the key is the field ID and the value is an accessor for that field. + # The equivalent java logic can be found here: https://github.com/apache/iceberg/blob/master/api/src/main/java/org/apache/iceberg/Accessors.java#L213-L230 + return self._index + + def field(self, field: NestedField, result: Dict[int, Accessor]) -> Dict[int, Accessor]: + return self._index + + def list(self, list_type: ListType, result: Dict[int, Accessor]) -> Dict[int, Accessor]: + return self._index + + def map(self, map_type: MapType, key_result: Dict[int, Accessor], value_result: Dict[int, Accessor]) -> Dict[int, Accessor]: + return self._index + + def primitive(self, primitive: PrimitiveType) -> Dict[int, Accessor]: + return self._index + + +def build_position_accessors(schema_or_type: Schema | IcebergType) -> Dict[int, Accessor]: + """Generate an index of field IDs to schema position accessors + + Args: + schema_or_type (Schema | IcebergType): A schema or type to index + + Returns: + Dict[int, Accessor]: An index of field IDs to accessors + """ + return visit(schema_or_type, _BuildPositionAccessors()) From 38313abc7bab398f5b7f0df9980572429ae8eb87 Mon Sep 17 00:00:00 2001 From: jun-he Date: Wed, 4 May 2022 08:43:27 -0700 Subject: [PATCH 083/642] Python: Add UnknownTransform (#4684) --- src/iceberg/transforms.py | 27 +++++++++++++++++++++++++++ tests/test_transforms.py | 11 +++++++++++ 2 files changed, 38 insertions(+) diff --git a/src/iceberg/transforms.py b/src/iceberg/transforms.py index cfa0b9ed73..7945210ec0 100644 --- a/src/iceberg/transforms.py +++ b/src/iceberg/transforms.py @@ -222,6 +222,33 @@ def hash(self, value: UUID) -> int: ) +class UnknownTransform(Transform): + """A transform that represents when an unknown transform is provided + Args: + source_type (Type): An Iceberg `Type` + transform (str): A string name of a transform + Raises: + AttributeError: If the apply method is called. + """ + + def __init__(self, source_type: IcebergType, transform: str): + super().__init__( + transform, + f"transforms.UnknownTransform(source_type={repr(source_type)}, transform={repr(transform)})", + ) + self._type = source_type + self._transform = transform + + def apply(self, value): + raise AttributeError(f"Cannot apply unsupported transform: {self}") + + def can_transform(self, target: IcebergType) -> bool: + return self._type == target + + def result_type(self, source: IcebergType) -> IcebergType: + return StringType() + + def bucket(source_type: IcebergType, num_buckets: int) -> BaseBucketTransform: if type(source_type) in {IntegerType, LongType, DateType, TimeType, TimestampType, TimestamptzType}: return BucketNumberTransform(source_type, num_buckets) diff --git a/tests/test_transforms.py b/tests/test_transforms.py index 12e786f954..e27350cf8a 100644 --- a/tests/test_transforms.py +++ b/tests/test_transforms.py @@ -24,6 +24,7 @@ from iceberg import transforms from iceberg.types import ( BinaryType, + BooleanType, DateType, DecimalType, FixedType, @@ -125,3 +126,13 @@ def test_string_with_surrogate_pair(): as_bytes = bytes(string_with_surrogate_pair, "UTF-8") bucket_transform = transforms.bucket(StringType(), 100) assert bucket_transform.hash(string_with_surrogate_pair) == mmh3.hash(as_bytes) + + +def test_unknown_transform(): + unknown_transform = transforms.UnknownTransform(FixedType(8), "unknown") + assert str(unknown_transform) == str(eval(repr(unknown_transform))) + with pytest.raises(AttributeError): + unknown_transform.apply("test") + assert unknown_transform.can_transform(FixedType(8)) + assert not unknown_transform.can_transform(FixedType(5)) + assert isinstance(unknown_transform.result_type(BooleanType()), StringType) From 422c69ee3e44f1528189be6a393113cde25382b1 Mon Sep 17 00:00:00 2001 From: Samuel Redai <43911210+samredai@users.noreply.github.com> Date: Wed, 4 May 2022 14:37:13 -0400 Subject: [PATCH 084/642] Python: Add UnboundReference class (#4679) --- src/iceberg/expressions/base.py | 47 +++++++++++++++++++++++++++++++++ 1 file changed, 47 insertions(+) diff --git a/src/iceberg/expressions/base.py b/src/iceberg/expressions/base.py index 806e6f748d..216ae4f802 100644 --- a/src/iceberg/expressions/base.py +++ b/src/iceberg/expressions/base.py @@ -20,6 +20,7 @@ from typing import Any, Generic, TypeVar from iceberg.files import StructProtocol +from iceberg.schema import Schema from iceberg.types import NestedField, Singleton T = TypeVar("T") @@ -328,3 +329,49 @@ def eval(self, struct: StructProtocol) -> Any: Any: The value at the referenced field's position in `struct` """ return self._accessor.get(struct) + + +class UnboundReference: + """A reference not yet bound to a field in a schema + + Args: + name (str): The name of the field + + Note: + An unbound reference is sometimes referred to as a "named" reference + """ + + def __init__(self, name: str): + if not name: + raise ValueError(f"Name cannot be null: {name}") + self._name = name + + def __str__(self) -> str: + return f"UnboundReference(name={repr(self.name)})" + + def __repr__(self) -> str: + return f"UnboundReference(name={repr(self.name)})" + + @property + def name(self) -> str: + return self._name + + def bind(self, schema: Schema, case_sensitive: bool) -> BoundReference: + """Bind the reference to an Iceberg schema + + Args: + schema (Schema): An Iceberg schema + case_sensitive (bool): Whether to consider case when binding the reference to the field + + Raises: + ValueError: If an empty name is provided + + Returns: + BoundReference: A reference bound to the specific field in the Iceberg schema + """ + field = schema.find_field(name_or_id=self.name, case_sensitive=case_sensitive) + + if not field: + raise ValueError(f"Cannot find field '{self.name}' in schema: {schema}") + + return BoundReference(field=field, accessor=schema.accessor_for_field(field.field_id)) From ef963ce7839ebf36ee2e74afba37775c75eacae0 Mon Sep 17 00:00:00 2001 From: Fokko Driesprong Date: Tue, 10 May 2022 04:06:08 +0200 Subject: [PATCH 085/642] Python: Skip missing tox interpreters (#4731) --- tox.ini | 1 + 1 file changed, 1 insertion(+) diff --git a/tox.ini b/tox.ini index 92757798b6..d3e039bb39 100644 --- a/tox.ini +++ b/tox.ini @@ -17,6 +17,7 @@ [tox] envlist = py37,py38,py39,linters +skip_missing_interpreters = true [testenv] usedevelop = true From a53d0537d10cd67701fd659cd25d4cc2c6572a3b Mon Sep 17 00:00:00 2001 From: Fokko Driesprong Date: Tue, 10 May 2022 04:09:06 +0200 Subject: [PATCH 086/642] Python: Remove mypy error in Transform (#4728) --- src/iceberg/transforms.py | 17 ++++++----------- tox.ini | 8 +++++++- 2 files changed, 13 insertions(+), 12 deletions(-) diff --git a/src/iceberg/transforms.py b/src/iceberg/transforms.py index 7945210ec0..3ebc19219b 100644 --- a/src/iceberg/transforms.py +++ b/src/iceberg/transforms.py @@ -64,10 +64,10 @@ def __repr__(self): def __str__(self): return self._transform_string - def __call__(self, value: S) -> Optional[T]: + def __call__(self, value: Optional[S]) -> Optional[T]: return self.apply(value) - def apply(self, value: S) -> Optional[T]: + def apply(self, value: Optional[S]) -> Optional[T]: ... def can_transform(self, source: IcebergType) -> bool: @@ -83,10 +83,8 @@ def preserves_order(self) -> bool: def satisfies_order_of(self, other) -> bool: return self == other - def to_human_string(self, value) -> str: - if value is None: - return "null" - return str(value) + def to_human_string(self, value: Optional[S]) -> str: + return str(value) if value is not None else "null" @property def dedup_name(self) -> str: @@ -119,11 +117,8 @@ def num_buckets(self) -> int: def hash(self, value: S) -> int: raise NotImplementedError() - def apply(self, value: S) -> Optional[int]: - if value is None: - return None - - return (self.hash(value) & IntegerType.max) % self._num_buckets + def apply(self, value: Optional[S]) -> Optional[int]: + return (self.hash(value) & IntegerType.max) % self._num_buckets if value else None def result_type(self, source: IcebergType) -> IcebergType: return IntegerType() diff --git a/tox.ini b/tox.ini index d3e039bb39..4cc905a82e 100644 --- a/tox.ini +++ b/tox.ini @@ -70,7 +70,7 @@ commands = deps = mypy commands = - mypy --no-implicit-optional --install-types --non-interactive --config tox.ini src + mypy --install-types --non-interactive --config tox.ini src [testenv:docs] basepython = python3 @@ -98,5 +98,11 @@ python = 3.8: py38, linters 3.9: py39 +[mypy] + +no_implicit_optional=True +warn_redundant_casts=True +warn_unreachable=True + [mypy-pyarrow.*] ignore_missing_imports = True From 7ad897dc7400c9ccbd883be9820ec0d072ccb895 Mon Sep 17 00:00:00 2001 From: Fokko Driesprong Date: Tue, 10 May 2022 04:34:09 +0200 Subject: [PATCH 087/642] Python: test_schema_find_column_name was defined twice (#4729) --- tests/test_schema.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_schema.py b/tests/test_schema.py index 131e935309..7fde7a27d8 100644 --- a/tests/test_schema.py +++ b/tests/test_schema.py @@ -189,7 +189,7 @@ def test_schema_find_column_name_on_id_not_found(table_schema_nested): assert table_schema_nested.find_column_name(99) is None -def test_schema_find_column_name(table_schema_simple): +def test_schema_find_column_name_by_id(table_schema_simple): """Test finding a column name given its field ID""" assert table_schema_simple.find_column_name(1) == "foo" assert table_schema_simple.find_column_name(2) == "bar" From 94fe8f22570197041ed5533bc97fee73dea6b033 Mon Sep 17 00:00:00 2001 From: jun-he Date: Fri, 13 May 2022 15:59:36 -0700 Subject: [PATCH 088/642] Python: Add VoidTransform (#4727) --- src/iceberg/transforms.py | 32 +++++++++++++++++++++++++++++++- tests/test_transforms.py | 14 ++++++++++++++ 2 files changed, 45 insertions(+), 1 deletion(-) diff --git a/src/iceberg/transforms.py b/src/iceberg/transforms.py index 3ebc19219b..8c54b214d8 100644 --- a/src/iceberg/transforms.py +++ b/src/iceberg/transforms.py @@ -234,7 +234,7 @@ def __init__(self, source_type: IcebergType, transform: str): self._type = source_type self._transform = transform - def apply(self, value): + def apply(self, value: Optional[S]): raise AttributeError(f"Cannot apply unsupported transform: {self}") def can_transform(self, target: IcebergType) -> bool: @@ -244,6 +244,32 @@ def result_type(self, source: IcebergType) -> IcebergType: return StringType() +class VoidTransform(Transform): + """A transform that always returns None""" + + _instance = None + + def __new__(cls): + if cls._instance is None: + cls._instance = super(VoidTransform, cls).__new__(cls) + return cls._instance + + def __init__(self): + super().__init__("void", "transforms.always_null()") + + def apply(self, value: Optional[S]) -> None: + return None + + def can_transform(self, target: IcebergType) -> bool: + return True + + def result_type(self, source: IcebergType) -> IcebergType: + return source + + def to_human_string(self, value: Optional[S]) -> str: + return "null" + + def bucket(source_type: IcebergType, num_buckets: int) -> BaseBucketTransform: if type(source_type) in {IntegerType, LongType, DateType, TimeType, TimestampType, TimestamptzType}: return BucketNumberTransform(source_type, num_buckets) @@ -259,3 +285,7 @@ def bucket(source_type: IcebergType, num_buckets: int) -> BaseBucketTransform: return BucketUUIDTransform(num_buckets) else: raise ValueError(f"Cannot bucket by type: {source_type}") + + +def always_null() -> Transform: + return VoidTransform() diff --git a/tests/test_transforms.py b/tests/test_transforms.py index e27350cf8a..a227124945 100644 --- a/tests/test_transforms.py +++ b/tests/test_transforms.py @@ -136,3 +136,17 @@ def test_unknown_transform(): assert unknown_transform.can_transform(FixedType(8)) assert not unknown_transform.can_transform(FixedType(5)) assert isinstance(unknown_transform.result_type(BooleanType()), StringType) + + +def test_void_transform(): + void_transform = transforms.always_null() + assert void_transform is transforms.always_null() + assert void_transform == eval(repr(void_transform)) + assert void_transform.apply("test") is None + assert void_transform.can_transform(BooleanType()) + assert isinstance(void_transform.result_type(BooleanType()), BooleanType) + assert not void_transform.preserves_order + assert void_transform.satisfies_order_of(transforms.always_null()) + assert not void_transform.satisfies_order_of(transforms.bucket(DateType(), 100)) + assert void_transform.to_human_string("test") == "null" + assert void_transform.dedup_name == "void" From 472fd28edeff2dcff660a05dce7edfdd2d2d027d Mon Sep 17 00:00:00 2001 From: Fokko Driesprong Date: Sat, 14 May 2022 01:09:03 +0200 Subject: [PATCH 089/642] Python: Add spellcheck to the CI (#4730) --- spellcheck-dictionary.txt | 45 +++++++++++++++++++++++++++++++ src/iceberg/conversions.py | 2 +- src/iceberg/io/pyarrow.py | 2 +- src/iceberg/schema.py | 6 ++--- src/iceberg/table/partitioning.py | 2 +- src/iceberg/types.py | 2 +- tests/test_schema.py | 2 +- tox.ini | 3 +++ 8 files changed, 56 insertions(+), 8 deletions(-) create mode 100644 spellcheck-dictionary.txt diff --git a/spellcheck-dictionary.txt b/spellcheck-dictionary.txt new file mode 100644 index 0000000000..2476d5afde --- /dev/null +++ b/spellcheck-dictionary.txt @@ -0,0 +1,45 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +accessor +accessors +Args +ASF +BD +bool +boolean +datetime +disjunction +dispatchable +endian +enum +FileInfo +filesystem +fs +func +io +NativeFile +nullability +pragma +pyarrow +repr +schemas +seekable +singledispatch +str +struct +StructProtocol +StructType +Timestamptz +Timestamptzs +unscaled +URI \ No newline at end of file diff --git a/src/iceberg/conversions.py b/src/iceberg/conversions.py index 0af73567e7..1df594985f 100644 --- a/src/iceberg/conversions.py +++ b/src/iceberg/conversions.py @@ -147,7 +147,7 @@ def to_bytes(primitive_type: PrimitiveType, value: Union[bool, bytes, Decimal, f Args: primitive_type(PrimitiveType): An implementation of the PrimitiveType base class value: The value to convert to bytes (The type of this value depends on which dispatched function is - used--check dispatchable functions for typehints) + used--check dispatchable functions for type hints) """ raise TypeError(f"scale does not match {primitive_type}") diff --git a/src/iceberg/io/pyarrow.py b/src/iceberg/io/pyarrow.py index ced7001dfd..6b138e7091 100644 --- a/src/iceberg/io/pyarrow.py +++ b/src/iceberg/io/pyarrow.py @@ -54,7 +54,7 @@ class PyArrowFile(InputFile, OutputFile): """ def __init__(self, location: str): - parsed_location = urlparse(location) # Create a ParseResult from the uri + parsed_location = urlparse(location) # Create a ParseResult from the URI if not parsed_location.scheme: # If no scheme, assume the path is to a local file self._filesystem, self._path = FileSystem.from_uri(os.path.abspath(location)) else: diff --git a/src/iceberg/schema.py b/src/iceberg/schema.py index 36f7e1dcc3..f3790036e2 100644 --- a/src/iceberg/schema.py +++ b/src/iceberg/schema.py @@ -126,7 +126,7 @@ def find_field(self, name_or_id: str | int, case_sensitive: bool = True) -> Nest Args: name_or_id (str | int): Either a field name or a field ID - case_sensitive (bool, optional): Whether to peform a case-sensitive lookup using a field name. Defaults to True. + case_sensitive (bool, optional): Whether to perform a case-sensitive lookup using a field name. Defaults to True. Returns: NestedField: The matched NestedField @@ -145,7 +145,7 @@ def find_type(self, name_or_id: str | int, case_sensitive: bool = True) -> Icebe Args: name_or_id (str | int): Either a field name or a field ID - case_sensitive (bool, optional): Whether to peform a case-sensitive lookup using a field name. Defaults to True. + case_sensitive (bool, optional): Whether to perform a case-sensitive lookup using a field name. Defaults to True. Returns: NestedField: The type of the matched NestedField @@ -180,7 +180,7 @@ def select(self, names: List[str], case_sensitive: bool = True) -> "Schema": Args: names (List[str]): A list of column names - case_sensitive (bool, optional): Whether to peform a case-sensitive lookup for each column name. Defaults to True. + case_sensitive (bool, optional): Whether to perform a case-sensitive lookup for each column name. Defaults to True. Returns: Schema: A new schema with pruned columns diff --git a/src/iceberg/table/partitioning.py b/src/iceberg/table/partitioning.py index 4a9faf4a39..d1e9debd49 100644 --- a/src/iceberg/table/partitioning.py +++ b/src/iceberg/table/partitioning.py @@ -24,7 +24,7 @@ class PartitionField: Attributes: source_id(int): The source column id of table's schema - field_id(int): The partition field id across all the table metadata's partition specs + field_id(int): The partition field id across all the table partition specs transform(Transform): The transform used to produce partition values from source column name(str): The name of this partition field """ diff --git a/src/iceberg/types.py b/src/iceberg/types.py index 0b780119fa..58b0d9de55 100644 --- a/src/iceberg/types.py +++ b/src/iceberg/types.py @@ -520,7 +520,7 @@ def __init__(self): class BinaryType(PrimitiveType, Singleton): - """A Binary data type in Iceberg can be represented using an instance of this class. Binarys in + """A Binary data type in Iceberg can be represented using an instance of this class. Binaries in Iceberg are arbitrary-length byte arrays. Example: diff --git a/tests/test_schema.py b/tests/test_schema.py index 7fde7a27d8..131974abec 100644 --- a/tests/test_schema.py +++ b/tests/test_schema.py @@ -228,7 +228,7 @@ def test_schema_find_field_by_id_raise_on_unknown_field(table_schema_simple): def test_schema_find_field_type_by_id(table_schema_simple): - """Test retrieving a columns's type using its field ID""" + """Test retrieving a columns' type using its field ID""" index = schema.index_by_id(table_schema_simple) assert index[1] == NestedField(field_id=1, name="foo", field_type=StringType(), is_optional=False) assert index[2] == NestedField(field_id=2, name="bar", field_type=IntegerType(), is_optional=True) diff --git a/tox.ini b/tox.ini index 4cc905a82e..5a1a0a7042 100644 --- a/tox.ini +++ b/tox.ini @@ -53,10 +53,13 @@ deps = black isort autoflake + pylint + pyenchant commands = autoflake -r --check --ignore-init-module-imports --remove-all-unused-imports src tests isort --profile black --check-only src tests black --line-length 130 --check --diff src tests + pylint --disable all --enable spelling --spelling-dict en_US --spelling-private-dict-file spellcheck-dictionary.txt src tests [testenv:format] deps = From f30e360fcd0d823a8974c866f89e970c7480128e Mon Sep 17 00:00:00 2001 From: Fokko Driesprong Date: Sat, 14 May 2022 01:23:53 +0200 Subject: [PATCH 090/642] Python: Add visitor to build Accessor for a Schema (#4685) --- src/iceberg/expressions/base.py | 31 +----- src/iceberg/files.py | 4 +- src/iceberg/schema.py | 184 ++++++++++++++++++++++---------- tests/conftest.py | 9 ++ tests/test_schema.py | 60 +++++++++++ 5 files changed, 201 insertions(+), 87 deletions(-) diff --git a/src/iceberg/expressions/base.py b/src/iceberg/expressions/base.py index 216ae4f802..71cf20a000 100644 --- a/src/iceberg/expressions/base.py +++ b/src/iceberg/expressions/base.py @@ -20,7 +20,7 @@ from typing import Any, Generic, TypeVar from iceberg.files import StructProtocol -from iceberg.schema import Schema +from iceberg.schema import Accessor, Schema from iceberg.types import NestedField, Singleton T = TypeVar("T") @@ -267,35 +267,6 @@ def __str__(self) -> str: return "false" -class Accessor: - """An accessor for a specific position in a container that implements the StructProtocol""" - - def __init__(self, position: int): - self._position = position - - def __str__(self): - return f"Accessor(position={self._position})" - - def __repr__(self): - return f"Accessor(position={self._position})" - - @property - def position(self): - """The position in the container to access""" - return self._position - - def get(self, container: StructProtocol) -> Any: - """Returns the value at self.position in `container` - - Args: - container(StructProtocol): A container to access at position `self.position` - - Returns: - Any: The value at position `self.position` in the container - """ - return container.get(self.position) - - class BoundReference: """A reference bound to a field in a schema diff --git a/src/iceberg/files.py b/src/iceberg/files.py index 409aae5728..c077574ce9 100644 --- a/src/iceberg/files.py +++ b/src/iceberg/files.py @@ -14,7 +14,7 @@ # KIND, either express or implied. See the License for the # specific language governing permissions and limitations # under the License. - +from abc import abstractmethod from enum import Enum, auto from typing import Any @@ -46,8 +46,10 @@ class FileFormat(Enum): class StructProtocol(Protocol): # pragma: no cover """A generic protocol used by accessors to get and set at positions of an object""" + @abstractmethod def get(self, pos: int) -> Any: ... + @abstractmethod def set(self, pos: int, value) -> None: ... diff --git a/src/iceberg/schema.py b/src/iceberg/schema.py index f3790036e2..3ae23feda7 100644 --- a/src/iceberg/schema.py +++ b/src/iceberg/schema.py @@ -19,10 +19,10 @@ import sys from abc import ABC, abstractmethod -from typing import TYPE_CHECKING, Dict, Generic, Iterable, List, TypeVar +from dataclasses import dataclass +from typing import Any, Dict, Generic, Iterable, List, Optional, TypeVar -if TYPE_CHECKING: - from iceberg.expressions.base import Accessor +from iceberg.files import StructProtocol if sys.version_info >= (3, 8): from functools import singledispatch # pragma: no cover @@ -49,10 +49,10 @@ class Schema: >>> from iceberg import types """ - def __init__(self, *columns: Iterable[NestedField], schema_id: int, identifier_field_ids: List[int] = []): + def __init__(self, *columns: Iterable[NestedField], schema_id: int, identifier_field_ids: Optional[List[int]] = None): self._struct = StructType(*columns) # type: ignore self._schema_id = schema_id - self._identifier_field_ids = identifier_field_ids + self._identifier_field_ids = identifier_field_ids or [] self._name_to_id: Dict[str, int] = index_by_name(self) self._name_to_id_lower: Dict[str, int] = {} # Should be accessed through self._lazy_name_to_id_lower() self._id_to_field: Dict[int, NestedField] = {} # Should be accessed through self._lazy_id_to_field() @@ -262,10 +262,44 @@ def primitive(self, primitive: PrimitiveType) -> T: ... # pragma: no cover +@dataclass(init=True, eq=True, frozen=True) +class Accessor: + """An accessor for a specific position in a container that implements the StructProtocol""" + + position: int + inner: Optional["Accessor"] = None + + def __str__(self): + return f"Accessor(position={self.position},inner={self.inner})" + + def __repr__(self): + return self.__str__() + + def get(self, container: StructProtocol) -> Any: + """Returns the value at self.position in `container` + + Args: + container(StructProtocol): A container to access at position `self.position` + + Returns: + Any: The value at position `self.position` in the container + """ + pos = self.position + val = container.get(pos) + inner = self + while inner.inner: + inner = inner.inner + val = val.get(inner.position) + + return val + + @singledispatch def visit(obj, visitor: SchemaVisitor[T]) -> T: """A generic function for applying a schema visitor to any point within a schema + The function traverses the schema in post-order fashion + Args: obj(Schema | IcebergType): An instance of a Schema or an IcebergType visitor (SchemaVisitor[T]): An instance of an implementation of the generic SchemaVisitor base class @@ -286,13 +320,11 @@ def _(obj: Schema, visitor: SchemaVisitor[T]) -> T: def _(obj: StructType, visitor: SchemaVisitor[T]) -> T: """Visit a StructType with a concrete SchemaVisitor""" results = [] + for field in obj.fields: visitor.before_field(field) - try: - result = visit(field.type, visitor) - finally: - visitor.after_field(field) - + result = visit(field.type, visitor) + visitor.after_field(field) results.append(visitor.field(field, result)) return visitor.struct(obj, results) @@ -301,11 +333,10 @@ def _(obj: StructType, visitor: SchemaVisitor[T]) -> T: @visit.register(ListType) def _(obj: ListType, visitor: SchemaVisitor[T]) -> T: """Visit a ListType with a concrete SchemaVisitor""" + visitor.before_list_element(obj.element) - try: - result = visit(obj.element.type, visitor) - finally: - visitor.after_list_element(obj.element) + result = visit(obj.element.type, visitor) + visitor.after_list_element(obj.element) return visitor.list(obj, result) @@ -314,16 +345,12 @@ def _(obj: ListType, visitor: SchemaVisitor[T]) -> T: def _(obj: MapType, visitor: SchemaVisitor[T]) -> T: """Visit a MapType with a concrete SchemaVisitor""" visitor.before_map_key(obj.key) - try: - key_result = visit(obj.key.type, visitor) - finally: - visitor.after_map_key(obj.key) + key_result = visit(obj.key.type, visitor) + visitor.after_map_key(obj.key) visitor.before_map_value(obj.value) - try: - value_result = visit(obj.value.type, visitor) - finally: - visitor.after_list_element(obj.value) + value_result = visit(obj.value.type, visitor) + visitor.after_list_element(obj.value) return visitor.map(obj, key_result, value_result) @@ -340,29 +367,29 @@ class _IndexById(SchemaVisitor[Dict[int, NestedField]]): def __init__(self) -> None: self._index: Dict[int, NestedField] = {} - def schema(self, schema, result): + def schema(self, schema: Schema, result) -> Dict[int, NestedField]: return self._index - def struct(self, struct, results): + def struct(self, struct: StructType, result) -> Dict[int, NestedField]: return self._index - def field(self, field, result): + def field(self, field: NestedField, result) -> Dict[int, NestedField]: """Add the field ID to the index""" self._index[field.field_id] = field return self._index - def list(self, list_type, result): + def list(self, list_type: ListType, result) -> Dict[int, NestedField]: """Add the list element ID to the index""" self._index[list_type.element.field_id] = list_type.element return self._index - def map(self, map_type, key_result, value_result): + def map(self, map_type: MapType, key_result, value_result) -> Dict[int, NestedField]: """Add the key ID and value ID as individual items in the index""" self._index[map_type.key.field_id] = map_type.key self._index[map_type.value.field_id] = map_type.value return self._index - def primitive(self, primitive): + def primitive(self, primitive) -> Dict[int, NestedField]: return self._index @@ -409,24 +436,27 @@ def after_field(self, field: NestedField) -> None: self._field_names.pop() self._short_field_names.pop() - def schema(self, schema, struct_result): + def schema(self, schema: Schema, struct_result: Dict[str, int]) -> Dict[str, int]: return self._index - def struct(self, struct, field_results): + def struct(self, struct: StructType, struct_result: List[Dict[str, int]]) -> Dict[str, int]: return self._index - def field(self, field, field_result): + def field(self, field: NestedField, struct_result: Dict[str, int]) -> Dict[str, int]: """Add the field name to the index""" self._add_field(field.name, field.field_id) + return self._index - def list(self, list_type, result): + def list(self, list_type: ListType, struct_result: Dict[str, int]) -> Dict[str, int]: """Add the list element name to the index""" self._add_field(list_type.element.name, list_type.element.field_id) + return self._index - def map(self, map_type, key_result, value_result): + def map(self, map_type: MapType, key_result: Dict[str, int], value_result: Dict[str, int]) -> Dict[str, int]: """Add the key name and value name as individual items in the index""" self._add_field(map_type.key.name, map_type.key.field_id) self._add_field(map_type.value.name, map_type.value.field_id) + return self._index def _add_field(self, name: str, field_id: int): """Add a field name to the index, mapping its full name to its field ID @@ -451,10 +481,10 @@ def _add_field(self, name: str, field_id: int): short_name = ".".join([".".join(self._short_field_names), name]) self._short_name_to_id[short_name] = field_id - def primitive(self, primitive): + def primitive(self, primitive) -> Dict[str, int]: return self._index - def by_name(self): + def by_name(self) -> Dict[str, int]: """Returns an index of combined full and short names Note: Only short names that do not conflict with full names are included. @@ -463,13 +493,13 @@ def by_name(self): combined_index.update(self._index) return combined_index - def by_id(self): + def by_id(self) -> Dict[int, str]: """Returns an index of ID to full names""" id_to_full_name = dict([(value, key) for key, value in self._index.items()]) return id_to_full_name -def index_by_name(schema_or_type) -> Dict[str, int]: +def index_by_name(schema_or_type: Schema | IcebergType) -> Dict[str, int]: """Generate an index of field names to field IDs Args: @@ -483,7 +513,7 @@ def index_by_name(schema_or_type) -> Dict[str, int]: return indexer.by_name() -def index_name_by_id(schema_or_type) -> Dict[int, str]: +def index_name_by_id(schema_or_type: Schema | IcebergType) -> Dict[int, str]: """Generate an index of field IDs full field names Args: @@ -497,31 +527,73 @@ def index_name_by_id(schema_or_type) -> Dict[int, str]: return indexer.by_id() -class _BuildPositionAccessors(SchemaVisitor[Dict[int, "Accessor"]]): - """A schema visitor for generating a field ID to accessor index""" +Position = int - def __init__(self) -> None: - self._index: Dict[int, Accessor] = {} - def schema(self, schema, result: Dict[int, Accessor]) -> Dict[int, Accessor]: - return self._index +class _BuildPositionAccessors(SchemaVisitor[Dict[Position, Accessor]]): + """A schema visitor for generating a field ID to accessor index - def struct(self, struct, result: List[Dict[int, Accessor]]) -> Dict[int, Accessor]: - # TODO: Populate the `self._index` dictionary where the key is the field ID and the value is an accessor for that field. - # The equivalent java logic can be found here: https://github.com/apache/iceberg/blob/master/api/src/main/java/org/apache/iceberg/Accessors.java#L213-L230 - return self._index + Example: + >>> from iceberg.schema import Schema + >>> from iceberg.types import * + >>> schema = Schema( + ... NestedField(field_id=2, name="id", field_type=IntegerType(), is_optional=False), + ... NestedField(field_id=1, name="data", field_type=StringType(), is_optional=True), + ... NestedField( + ... field_id=3, + ... name="location", + ... field_type=StructType( + ... NestedField(field_id=5, name="latitude", field_type=FloatType(), is_optional=False), + ... NestedField(field_id=6, name="longitude", field_type=FloatType(), is_optional=False), + ... ), + ... is_optional=True, + ... ), + ... schema_id=1, + ... identifier_field_ids=[1], + ... ) + >>> result = build_position_accessors(schema) + >>> expected = { + ... 2: Accessor(position=0, inner=None), + ... 1: Accessor(position=1, inner=None), + ... 5: Accessor(position=2, inner=Accessor(position=0, inner=None)), + ... 6: Accessor(position=2, inner=Accessor(position=1, inner=None)) + ... } + >>> result == expected + True + """ - def field(self, field: NestedField, result: Dict[int, Accessor]) -> Dict[int, Accessor]: - return self._index + @staticmethod + def _wrap_leaves(result: Dict[Position, Accessor], position: Position = 0) -> Dict[Position, Accessor]: + return {field_id: Accessor(position, inner=inner) for field_id, inner in result.items()} - def list(self, list_type: ListType, result: Dict[int, Accessor]) -> Dict[int, Accessor]: - return self._index + def schema(self, schema: Schema, result: Dict[Position, Accessor]) -> Dict[Position, Accessor]: + return result - def map(self, map_type: MapType, key_result: Dict[int, Accessor], value_result: Dict[int, Accessor]) -> Dict[int, Accessor]: - return self._index + def struct(self, struct: StructType, field_results: List[Dict[Position, Accessor]]) -> Dict[Position, Accessor]: + result = {} - def primitive(self, primitive: PrimitiveType) -> Dict[int, Accessor]: - return self._index + for position, field in enumerate(struct.fields): + if field_results[position]: + for inner_field_id, acc in field_results[position].items(): + result[inner_field_id] = Accessor(position, inner=acc) + else: + result[field.field_id] = Accessor(position) + + return result + + def field(self, field: NestedField, result: Dict[Position, Accessor]) -> Dict[Position, Accessor]: + return result + + def list(self, list_type: ListType, result: Dict[Position, Accessor]) -> Dict[Position, Accessor]: + return {} + + def map( + self, map_type: MapType, key_result: Dict[Position, Accessor], value_result: Dict[Position, Accessor] + ) -> Dict[Position, Accessor]: + return {} + + def primitive(self, primitive: PrimitiveType) -> Dict[Position, Accessor]: + return {} def build_position_accessors(schema_or_type: Schema | IcebergType) -> Dict[int, Accessor]: diff --git a/tests/conftest.py b/tests/conftest.py index a582bb3c0d..5f9a13a47b 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -95,6 +95,15 @@ def table_schema_nested(): ), is_optional=True, ), + NestedField( + field_id=15, + name="person", + field_type=StructType( + NestedField(field_id=16, name="name", field_type=StringType(), is_optional=False), + NestedField(field_id=17, name="age", field_type=IntegerType(), is_optional=True), + ), + is_optional=False, + ), schema_id=1, identifier_field_ids=[1], ) diff --git a/tests/test_schema.py b/tests/test_schema.py index 131974abec..eee9fbd676 100644 --- a/tests/test_schema.py +++ b/tests/test_schema.py @@ -16,10 +16,14 @@ # under the License. from textwrap import dedent +from typing import Any, Dict import pytest from iceberg import schema +from iceberg.expressions.base import Accessor +from iceberg.files import StructProtocol +from iceberg.schema import build_position_accessors from iceberg.types import ( BooleanType, FloatType, @@ -140,6 +144,17 @@ def test_schema_index_by_id_visitor(table_schema_nested): ), 13: NestedField(field_id=13, name="latitude", field_type=FloatType(), is_optional=False), 14: NestedField(field_id=14, name="longitude", field_type=FloatType(), is_optional=False), + 15: NestedField( + field_id=15, + name="person", + field_type=StructType( + NestedField(field_id=16, name="name", field_type=StringType(), is_optional=False), + NestedField(field_id=17, name="age", field_type=IntegerType(), is_optional=True), + ), + is_optional=False, + ), + 16: NestedField(field_id=16, name="name", field_type=StringType(), is_optional=False), + 17: NestedField(field_id=17, name="age", field_type=IntegerType(), is_optional=True), } @@ -163,6 +178,9 @@ def test_schema_index_by_name_visitor(table_schema_nested): "location.element.longitude": 14, "location.latitude": 13, "location.longitude": 14, + "person": 15, + "person.name": 16, + "person.age": 17, } @@ -295,6 +313,17 @@ def test_index_by_id_schema_visitor(table_schema_nested): ), 13: NestedField(field_id=13, name="latitude", field_type=FloatType(), is_optional=False), 14: NestedField(field_id=14, name="longitude", field_type=FloatType(), is_optional=False), + 15: NestedField( + field_id=15, + name="person", + field_type=StructType( + NestedField(field_id=16, name="name", field_type=StringType(), is_optional=False), + NestedField(field_id=17, name="age", field_type=IntegerType(), is_optional=True), + ), + is_optional=False, + ), + 16: NestedField(field_id=16, name="name", field_type=StringType(), is_optional=False), + 17: NestedField(field_id=17, name="age", field_type=IntegerType(), is_optional=True), } @@ -347,3 +376,34 @@ def test_schema_find_type(table_schema_simple): == table_schema_simple.find_type("BAZ", case_sensitive=False) == BooleanType() ) + + +def test_build_position_accessors(table_schema_nested): + accessors = build_position_accessors(table_schema_nested) + assert accessors == { + 1: Accessor(position=0, inner=None), + 2: Accessor(position=1, inner=None), + 3: Accessor(position=2, inner=None), + 4: Accessor(position=3, inner=None), + 6: Accessor(position=4, inner=None), + 11: Accessor(position=5, inner=None), + 16: Accessor(position=6, inner=Accessor(position=0, inner=None)), + 17: Accessor(position=6, inner=Accessor(position=1, inner=None)), + } + + +class TestStruct(StructProtocol): + def __init__(self, pos: Dict[int, Any] = None): + self._pos: Dict[int, Any] = pos or {} + + def set(self, pos: int, value) -> None: + pass + + def get(self, pos: int) -> Any: + return self._pos[pos] + + +def test_build_position_accessors_with_struct(table_schema_nested): + accessors = build_position_accessors(table_schema_nested) + container = TestStruct({6: TestStruct({0: "name"})}) + assert accessors.get(16).get(container) == "name" From 30834c0529e14c3d037d6d925869c1332830831e Mon Sep 17 00:00:00 2001 From: Fokko Driesprong Date: Tue, 17 May 2022 21:45:56 +0200 Subject: [PATCH 091/642] Python: Test for warning of PyLint (#4791) * Python: Test for warning of PyLint * Add ignore rule for overriding constructor --- pylintrc | 609 +++++++++++++++++++++ src/iceberg/conversions.py | 4 +- src/iceberg/expressions/base.py | 4 +- src/iceberg/expressions/literals.py | 9 +- src/iceberg/io/pyarrow.py | 12 +- src/iceberg/schema.py | 31 +- src/iceberg/transforms.py | 17 +- src/iceberg/types.py | 2 +- tests/expressions/test_expressions_base.py | 24 +- tests/expressions/test_literals.py | 16 +- tests/test_conversions.py | 6 +- tests/test_schema.py | 25 +- tests/test_transforms.py | 1 + tests/test_types.py | 1 + tox.ini | 2 +- 15 files changed, 687 insertions(+), 76 deletions(-) create mode 100644 pylintrc diff --git a/pylintrc b/pylintrc new file mode 100644 index 0000000000..b9aa99a6ff --- /dev/null +++ b/pylintrc @@ -0,0 +1,609 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +[MASTER] + +# A comma-separated list of package or module names from where C extensions may +# be loaded. Extensions are loading into the active Python interpreter and may +# run arbitrary code. +extension-pkg-allow-list= + +# A comma-separated list of package or module names from where C extensions may +# be loaded. Extensions are loading into the active Python interpreter and may +# run arbitrary code. (This is an alternative name to extension-pkg-allow-list +# for backward compatibility.) +extension-pkg-whitelist= + +# Return non-zero exit code if any of these messages/categories are detected, +# even if score is above --fail-under value. Syntax same as enable. Messages +# specified are enabled, while categories only check already-enabled messages. +fail-on= + +# Specify a score threshold to be exceeded before program exits with error. +fail-under=10.0 + +# Files or directories to be skipped. They should be base names, not paths. +ignore=CVS + +# Add files or directories matching the regex patterns to the ignore-list. The +# regex matches against paths and can be in Posix or Windows format. +ignore-paths= + +# Files or directories matching the regex patterns are skipped. The regex +# matches against base names, not paths. The default value ignores emacs file +# locks +ignore-patterns=^\.# + +# Python code to execute, usually for sys.path manipulation such as +# pygtk.require(). +#init-hook= + +# Use multiple processes to speed up Pylint. Specifying 0 will auto-detect the +# number of processors available to use. +jobs=1 + +# Control the amount of potential inferred values when inferring a single +# object. This can help the performance when dealing with large functions or +# complex, nested conditions. +limit-inference-results=100 + +# List of plugins (as comma separated values of python module names) to load, +# usually to register additional checkers. +load-plugins= + +# Pickle collected data for later comparisons. +persistent=yes + +# Minimum Python version to use for version dependent checks. Will default to +# the version used to run pylint. +py-version=3.9 + +# Discover python modules and packages in the file system subtree. +recursive=no + +# When enabled, pylint would attempt to guess common misconfiguration and emit +# user-friendly hints instead of false-positive error messages. +suggestion-mode=yes + +# Allow loading of arbitrary C extensions. Extensions are imported into the +# active Python interpreter and may run arbitrary code. +unsafe-load-any-extension=no + + +[MESSAGES CONTROL] + +# Only show warnings with the listed confidence levels. Leave empty to show +# all. Valid levels: HIGH, CONTROL_FLOW, INFERENCE, INFERENCE_FAILURE, +# UNDEFINED. +confidence= + +# Disable the message, report, category or checker with the given id(s). You +# can either give multiple identifiers separated by comma (,) or put this +# option multiple times (only on the command line, not in the configuration +# file where it should appear only once). You can also use "--disable=all" to +# disable everything first and then re-enable specific checks. For example, if +# you want to run only the similarities checker, you can use "--disable=all +# --enable=similarities". If you want to run only the classes checker, but have +# no Warning level messages displayed, use "--disable=all --enable=classes +# --disable=W". +disable=all + +# Enable the message, report, category or checker with the given id(s). You can +# either give multiple identifier separated by comma (,) or put this option +# multiple time (only on the command line, not in the configuration file where +# it should appear only once). See also the "--disable" option for examples. +enable=spelling,W + + +[REPORTS] + +# Python expression which should return a score less than or equal to 10. You +# have access to the variables 'fatal', 'error', 'warning', 'refactor', +# 'convention', and 'info' which contain the number of messages in each +# category, as well as 'statement' which is the total number of statements +# analyzed. This score is used by the global evaluation report (RP0004). +evaluation=max(0, 0 if fatal else 10.0 - ((float(5 * error + warning + refactor + convention) / statement) * 10)) + +# Template used to display messages. This is a python new-style format string +# used to format the message information. See doc for all details. +#msg-template= + +# Set the output format. Available formats are text, parseable, colorized, json +# and msvs (visual studio). You can also give a reporter class, e.g. +# mypackage.mymodule.MyReporterClass. +output-format=text + +# Tells whether to display a full report or only the messages. +reports=no + +# Activate the evaluation score. +score=yes + + +[REFACTORING] + +# Maximum number of nested blocks for function / method body +max-nested-blocks=5 + +# Complete name of functions that never returns. When checking for +# inconsistent-return-statements if a never returning function is called then +# it will be considered as an explicit return statement and no message will be +# printed. +never-returning-functions=sys.exit,argparse.parse_error + + +[LOGGING] + +# The type of string formatting that logging methods do. `old` means using % +# formatting, `new` is for `{}` formatting. +logging-format-style=old + +# Logging modules to check that the string format arguments are in logging +# function parameter format. +logging-modules=logging + + +[SPELLING] + +# Limits count of emitted suggestions for spelling mistakes. +max-spelling-suggestions=4 + +# Spelling dictionary name. Available dictionaries: af (aspell), am (aspell), +# ar (aspell), ast (aspell), az (aspell), be (aspell), be_BY (aspell), be_SU +# (aspell), bg (aspell), bn (aspell), br (aspell), ca (aspell), cs (aspell), +# csb (aspell), cy (aspell), da (aspell), de (aspell), de_AT (aspell), de_CH +# (aspell), de_DE (aspell), el (aspell), en (aspell), en_AU (aspell), en_CA +# (aspell), en_GB (aspell), en_US (aspell), eo (aspell), es (aspell), es_ES +# (AppleSpell), et (aspell), fa (aspell), fi (aspell), fo (aspell), fr +# (aspell), fr_CH (aspell), fr_FR (aspell), fy (aspell), ga (aspell), gd +# (aspell), gl (aspell), gr (aspell), grc (aspell), gu (aspell), gv (aspell), +# he (aspell), hi (aspell), hil (aspell), hr (aspell), hsb (aspell), hu +# (aspell), hu_HU (AppleSpell), hus (aspell), hy (aspell), ia (aspell), id +# (aspell), it (aspell), it_IT (AppleSpell), kn (aspell), ku (aspell), ky +# (aspell), la (aspell), lt (aspell), lv (aspell), mg (aspell), mi (aspell), mk +# (aspell), ml (aspell), mn (aspell), mr (aspell), ms (aspell), mt (aspell), +# nds (aspell), nl (aspell), nl_NL (AppleSpell), nn (aspell), ny (aspell), or +# (aspell), pa (aspell), pl (aspell), pt_BR (aspell), pt_PT (aspell), qu +# (aspell), ro (aspell), ru (aspell), rw (aspell), sc (aspell), sk (aspell), +# sk_SK (aspell), sl (aspell), sr (aspell), srd (aspell), sv (aspell), sv_SE +# (AppleSpell), sw (aspell), ta (aspell), te (aspell), tet (aspell), tk +# (aspell), tl (aspell), tn (aspell), tr (aspell), uk (aspell), uz (aspell), vi +# (aspell), wa (aspell), yi (aspell), zu (aspell). +spelling-dict=en_US + +# List of comma separated words that should be considered directives if they +# appear and the beginning of a comment and should not be checked. +spelling-ignore-comment-directives=fmt: on,fmt: off,noqa:,noqa,nosec,isort:skip,mypy: + +# List of comma separated words that should not be checked. +spelling-ignore-words= + +# A path to a file that contains the private dictionary; one word per line. +spelling-private-dict-file=spellcheck-dictionary.txt + +# Tells whether to store unknown words to the private dictionary (see the +# --spelling-private-dict-file option) instead of raising a message. +spelling-store-unknown-words=no + + +[MISCELLANEOUS] + +# List of note tags to take in consideration, separated by a comma. +notes=FIXME, + XXX, + TODO + +# Regular expression of note tags to take in consideration. +#notes-rgx= + + +[TYPECHECK] + +# List of decorators that produce context managers, such as +# contextlib.contextmanager. Add to this list to register other decorators that +# produce valid context managers. +contextmanager-decorators=contextlib.contextmanager + +# List of members which are set dynamically and missed by pylint inference +# system, and so shouldn't trigger E1101 when accessed. Python regular +# expressions are accepted. +generated-members= + +# Tells whether missing members accessed in mixin class should be ignored. A +# class is considered mixin if its name matches the mixin-class-rgx option. +ignore-mixin-members=yes + +# Tells whether to warn about missing members when the owner of the attribute +# is inferred to be None. +ignore-none=yes + +# This flag controls whether pylint should warn about no-member and similar +# checks whenever an opaque object is returned when inferring. The inference +# can return multiple potential results while evaluating a Python object, but +# some branches might not be evaluated, which results in partial inference. In +# that case, it might be useful to still emit no-member and other checks for +# the rest of the inferred objects. +ignore-on-opaque-inference=yes + +# List of class names for which member attributes should not be checked (useful +# for classes with dynamically set attributes). This supports the use of +# qualified names. +ignored-classes=optparse.Values,thread._local,_thread._local + +# List of module names for which member attributes should not be checked +# (useful for modules/projects where namespaces are manipulated during runtime +# and thus existing member attributes cannot be deduced by static analysis). It +# supports qualified module names, as well as Unix pattern matching. +ignored-modules= + +# Show a hint with possible names when a member name was not found. The aspect +# of finding the hint is based on edit distance. +missing-member-hint=yes + +# The minimum edit distance a name should have in order to be considered a +# similar match for a missing member name. +missing-member-hint-distance=1 + +# The total number of similar names that should be taken in consideration when +# showing a hint for a missing member. +missing-member-max-choices=1 + +# Regex pattern to define which classes are considered mixins ignore-mixin- +# members is set to 'yes' +mixin-class-rgx=.*[Mm]ixin + +# List of decorators that change the signature of a decorated function. +signature-mutators= + + +[VARIABLES] + +# List of additional names supposed to be defined in builtins. Remember that +# you should avoid defining new builtins when possible. +additional-builtins= + +# Tells whether unused global variables should be treated as a violation. +allow-global-unused-variables=yes + +# List of names allowed to shadow builtins +allowed-redefined-builtins= + +# List of strings which can identify a callback function by name. A callback +# name must start or end with one of those strings. +callbacks=cb_, + _cb + +# A regular expression matching the name of dummy variables (i.e. expected to +# not be used). +dummy-variables-rgx=_+$|(_[a-zA-Z0-9_]*[a-zA-Z0-9]+?$)|dummy|^ignored_|^unused_ + +# Argument names that match this expression will be ignored. Default to name +# with leading underscore. +ignored-argument-names=_.*|^ignored_|^unused_ + +# Tells whether we should check for unused import in __init__ files. +init-import=no + +# List of qualified module names which can have objects that can redefine +# builtins. +redefining-builtins-modules=six.moves,past.builtins,future.builtins,builtins,io + + +[FORMAT] + +# Expected format of line ending, e.g. empty (any line ending), LF or CRLF. +expected-line-ending-format= + +# Regexp for a line that is allowed to be longer than the limit. +ignore-long-lines=^\s*(# )??$ + +# Number of spaces of indent required inside a hanging or continued line. +indent-after-paren=4 + +# String used as indentation unit. This is usually " " (4 spaces) or "\t" (1 +# tab). +indent-string=' ' + +# Maximum number of characters on a single line. +max-line-length=130 + +# Maximum number of lines in a module. +max-module-lines=1000 + +# Allow the body of a class to be on the same line as the declaration if body +# contains single statement. +single-line-class-stmt=no + +# Allow the body of an if to be on the same line as the test if there is no +# else. +single-line-if-stmt=no + + +[SIMILARITIES] + +# Comments are removed from the similarity computation +ignore-comments=yes + +# Docstrings are removed from the similarity computation +ignore-docstrings=yes + +# Imports are removed from the similarity computation +ignore-imports=no + +# Signatures are removed from the similarity computation +ignore-signatures=no + +# Minimum lines number of a similarity. +min-similarity-lines=4 + + +[STRING] + +# This flag controls whether inconsistent-quotes generates a warning when the +# character used as a quote delimiter is used inconsistently within a module. +check-quote-consistency=no + +# This flag controls whether the implicit-str-concat should generate a warning +# on implicit string concatenation in sequences defined over several lines. +check-str-concat-over-line-jumps=no + + +[BASIC] + +# Naming style matching correct argument names. +argument-naming-style=snake_case + +# Regular expression matching correct argument names. Overrides argument- +# naming-style. If left empty, argument names will be checked with the set +# naming style. +#argument-rgx= + +# Naming style matching correct attribute names. +attr-naming-style=snake_case + +# Regular expression matching correct attribute names. Overrides attr-naming- +# style. If left empty, attribute names will be checked with the set naming +# style. +#attr-rgx= + +# Bad variable names which should always be refused, separated by a comma. +bad-names=foo, + bar, + baz, + toto, + tutu, + tata + +# Bad variable names regexes, separated by a comma. If names match any regex, +# they will always be refused +bad-names-rgxs= + +# Naming style matching correct class attribute names. +class-attribute-naming-style=any + +# Regular expression matching correct class attribute names. Overrides class- +# attribute-naming-style. If left empty, class attribute names will be checked +# with the set naming style. +#class-attribute-rgx= + +# Naming style matching correct class constant names. +class-const-naming-style=UPPER_CASE + +# Regular expression matching correct class constant names. Overrides class- +# const-naming-style. If left empty, class constant names will be checked with +# the set naming style. +#class-const-rgx= + +# Naming style matching correct class names. +class-naming-style=PascalCase + +# Regular expression matching correct class names. Overrides class-naming- +# style. If left empty, class names will be checked with the set naming style. +#class-rgx= + +# Naming style matching correct constant names. +const-naming-style=UPPER_CASE + +# Regular expression matching correct constant names. Overrides const-naming- +# style. If left empty, constant names will be checked with the set naming +# style. +#const-rgx= + +# Minimum line length for functions/classes that require docstrings, shorter +# ones are exempt. +docstring-min-length=-1 + +# Naming style matching correct function names. +function-naming-style=snake_case + +# Regular expression matching correct function names. Overrides function- +# naming-style. If left empty, function names will be checked with the set +# naming style. +#function-rgx= + +# Good variable names which should always be accepted, separated by a comma. +good-names=i, + j, + k, + ex, + Run, + _ + +# Good variable names regexes, separated by a comma. If names match any regex, +# they will always be accepted +good-names-rgxs= + +# Include a hint for the correct naming format with invalid-name. +include-naming-hint=no + +# Naming style matching correct inline iteration names. +inlinevar-naming-style=any + +# Regular expression matching correct inline iteration names. Overrides +# inlinevar-naming-style. If left empty, inline iteration names will be checked +# with the set naming style. +#inlinevar-rgx= + +# Naming style matching correct method names. +method-naming-style=snake_case + +# Regular expression matching correct method names. Overrides method-naming- +# style. If left empty, method names will be checked with the set naming style. +#method-rgx= + +# Naming style matching correct module names. +module-naming-style=snake_case + +# Regular expression matching correct module names. Overrides module-naming- +# style. If left empty, module names will be checked with the set naming style. +#module-rgx= + +# Colon-delimited sets of names that determine each other's naming style when +# the name regexes allow several styles. +name-group= + +# Regular expression which should only match function or class names that do +# not require a docstring. +no-docstring-rgx=^_ + +# List of decorators that produce properties, such as abc.abstractproperty. Add +# to this list to register other decorators that produce valid properties. +# These decorators are taken in consideration only for invalid-name. +property-classes=abc.abstractproperty + +# Regular expression matching correct type variable names. If left empty, type +# variable names will be checked with the set naming style. +#typevar-rgx= + +# Naming style matching correct variable names. +variable-naming-style=snake_case + +# Regular expression matching correct variable names. Overrides variable- +# naming-style. If left empty, variable names will be checked with the set +# naming style. +#variable-rgx= + + +[CLASSES] + +# Warn about protected attribute access inside special methods +check-protected-access-in-special-methods=no + +# List of method names used to declare (i.e. assign) instance attributes. +defining-attr-methods=__init__, + __new__, + setUp, + __post_init__ + +# List of member names, which should be excluded from the protected access +# warning. +exclude-protected=_asdict, + _fields, + _replace, + _source, + _make + +# List of valid names for the first argument in a class method. +valid-classmethod-first-arg=cls + +# List of valid names for the first argument in a metaclass class method. +valid-metaclass-classmethod-first-arg=cls + + +[IMPORTS] + +# List of modules that can be imported at any level, not just the top level +# one. +allow-any-import-level= + +# Allow wildcard imports from modules that define __all__. +allow-wildcard-with-all=no + +# Analyse import fallback blocks. This can be used to support both Python 2 and +# 3 compatible code, which means that the block might have code that exists +# only in one or another interpreter, leading to false positives when analysed. +analyse-fallback-blocks=no + +# Deprecated modules which should not be used, separated by a comma. +deprecated-modules= + +# Output a graph (.gv or any supported image format) of external dependencies +# to the given file (report RP0402 must not be disabled). +ext-import-graph= + +# Output a graph (.gv or any supported image format) of all (i.e. internal and +# external) dependencies to the given file (report RP0402 must not be +# disabled). +import-graph= + +# Output a graph (.gv or any supported image format) of internal dependencies +# to the given file (report RP0402 must not be disabled). +int-import-graph= + +# Force import order to recognize a module as part of the standard +# compatibility libraries. +known-standard-library= + +# Force import order to recognize a module as part of a third party library. +known-third-party=enchant + +# Couples of modules and preferred modules, separated by a comma. +preferred-modules= + + +[DESIGN] + +# List of regular expressions of class ancestor names to ignore when counting +# public methods (see R0903) +exclude-too-few-public-methods= + +# List of qualified class names to ignore when counting class parents (see +# R0901) +ignored-parents= + +# Maximum number of arguments for function / method. +max-args=5 + +# Maximum number of attributes for a class (see R0902). +max-attributes=7 + +# Maximum number of boolean expressions in an if statement (see R0916). +max-bool-expr=5 + +# Maximum number of branch for function / method body. +max-branches=12 + +# Maximum number of locals for function / method body. +max-locals=15 + +# Maximum number of parents for a class (see R0901). +max-parents=7 + +# Maximum number of public methods for a class (see R0904). +max-public-methods=20 + +# Maximum number of return / yield for function / method body. +max-returns=6 + +# Maximum number of statements in function / method body. +max-statements=50 + +# Minimum number of public methods for a class (see R0903). +min-public-methods=2 + + +[EXCEPTIONS] + +# Exceptions that will emit a warning when being caught. Defaults to +# "BaseException, Exception". +overgeneral-exceptions=BaseException, + Exception diff --git a/src/iceberg/conversions.py b/src/iceberg/conversions.py index 1df594985f..f5bf709d24 100644 --- a/src/iceberg/conversions.py +++ b/src/iceberg/conversions.py @@ -99,7 +99,7 @@ def _(primitive_type, value_str: str) -> int: Raises: ValueError: If the scale/exponent is not 0 """ - _, digits, exponent = Decimal(value_str).as_tuple() + _, _, exponent = Decimal(value_str).as_tuple() if exponent != 0: # Raise if there are digits to the right of the decimal raise ValueError(f"Cannot convert partition value, value cannot have fractional digits for {primitive_type} partition") return int(float(value_str)) @@ -216,7 +216,7 @@ def _(primitive_type, value: Decimal) -> bytes: Returns: bytes: The byte representation of `value` """ - sign, digits, exponent = value.as_tuple() + _, digits, exponent = value.as_tuple() if -exponent != primitive_type.scale: raise ValueError(f"Cannot serialize value, scale of value does not match type {primitive_type}: {-exponent}") diff --git a/src/iceberg/expressions/base.py b/src/iceberg/expressions/base.py index 71cf20a000..7f0b98b162 100644 --- a/src/iceberg/expressions/base.py +++ b/src/iceberg/expressions/base.py @@ -66,8 +66,8 @@ def negate(self) -> "Operation": try: return OPERATION_NEGATIONS[self] - except KeyError: - raise ValueError(f"No negation defined for operation {self}") + except KeyError as e: + raise ValueError(f"No negation defined for operation {self}") from e OPERATION_NEGATIONS = { diff --git a/src/iceberg/expressions/literals.py b/src/iceberg/expressions/literals.py index 489c8b29b3..1c21f00e6c 100644 --- a/src/iceberg/expressions/literals.py +++ b/src/iceberg/expressions/literals.py @@ -18,6 +18,7 @@ # KIND, either express or implied. See the License for the # specific language governing permissions and limitations # under the License. +# pylint: disable=W0613 import struct import sys @@ -118,10 +119,8 @@ def _(value: Decimal) -> Literal[Decimal]: return DecimalLiteral(value) -class AboveMax(Literal[None], Singleton): - def __init__(self): - pass - +class AboveMax(Singleton): + @property def value(self): raise ValueError("AboveMax has no value") @@ -135,7 +134,7 @@ def __str__(self): return "AboveMax" -class BelowMin(Literal[None], Singleton): +class BelowMin(Singleton): def __init__(self): pass diff --git a/src/iceberg/io/pyarrow.py b/src/iceberg/io/pyarrow.py index 6b138e7091..a67ff5d364 100644 --- a/src/iceberg/io/pyarrow.py +++ b/src/iceberg/io/pyarrow.py @@ -72,7 +72,7 @@ def _file_info(self) -> FileInfo: file_info = self._filesystem.get_file_info(self._path) except OSError as e: if e.errno == 13 or "AWS Error [code 15]" in str(e): - raise PermissionError(f"Cannot get file info, access denied: {self.location}") + raise PermissionError(f"Cannot get file info, access denied: {self.location}") from e raise # pragma: no cover - If some other kind of OSError, raise the raw error if file_info.type == FileType.NotFound: @@ -111,9 +111,9 @@ def open(self) -> InputStream: raise except OSError as e: if e.errno == 2 or "Path does not exist" in str(e): - raise FileNotFoundError(f"Cannot open file, does not exist: {self.location}") + raise FileNotFoundError(f"Cannot open file, does not exist: {self.location}") from e elif e.errno == 13 or "AWS Error [code 15]" in str(e): - raise PermissionError(f"Cannot open file, access denied: {self.location}") + raise PermissionError(f"Cannot open file, access denied: {self.location}") from e raise # pragma: no cover - If some other kind of OSError, raise the raw error return input_file @@ -144,7 +144,7 @@ def create(self, overwrite: bool = False) -> OutputStream: raise except OSError as e: if e.errno == 13 or "AWS Error [code 15]" in str(e): - raise PermissionError(f"Cannot create file, access denied: {self.location}") + raise PermissionError(f"Cannot create file, access denied: {self.location}") from e raise # pragma: no cover - If some other kind of OSError, raise the raw error return output_file @@ -204,7 +204,7 @@ def delete(self, location: Union[str, InputFile, OutputFile]) -> None: raise except OSError as e: if e.errno == 2 or "Path does not exist" in str(e): - raise FileNotFoundError(f"Cannot delete file, does not exist: {location}") + raise FileNotFoundError(f"Cannot delete file, does not exist: {location}") from e elif e.errno == 13 or "AWS Error [code 15]" in str(e): - raise PermissionError(f"Cannot delete file, access denied: {location}") + raise PermissionError(f"Cannot delete file, access denied: {location}") from e raise # pragma: no cover - If some other kind of OSError, raise the raw error diff --git a/src/iceberg/schema.py b/src/iceberg/schema.py index 3ae23feda7..9b96504480 100644 --- a/src/iceberg/schema.py +++ b/src/iceberg/schema.py @@ -14,6 +14,7 @@ # KIND, either express or implied. See the License for the # specific language governing permissions and limitations # under the License. +# pylint: disable=W0511 from __future__ import annotations @@ -234,32 +235,26 @@ def after_map_value(self, value: NestedField) -> None: @abstractmethod def schema(self, schema: Schema, struct_result: T) -> T: """Visit a Schema""" - ... # pragma: no cover @abstractmethod def struct(self, struct: StructType, field_results: List[T]) -> T: """Visit a StructType""" - ... # pragma: no cover @abstractmethod def field(self, field: NestedField, field_result: T) -> T: """Visit a NestedField""" - ... # pragma: no cover @abstractmethod def list(self, list_type: ListType, element_result: T) -> T: """Visit a ListType""" - ... # pragma: no cover @abstractmethod def map(self, map_type: MapType, key_result: T, value_result: T) -> T: """Visit a MapType""" - ... # pragma: no cover @abstractmethod def primitive(self, primitive: PrimitiveType) -> T: """Visit a PrimitiveType""" - ... # pragma: no cover @dataclass(init=True, eq=True, frozen=True) @@ -367,18 +362,18 @@ class _IndexById(SchemaVisitor[Dict[int, NestedField]]): def __init__(self) -> None: self._index: Dict[int, NestedField] = {} - def schema(self, schema: Schema, result) -> Dict[int, NestedField]: + def schema(self, schema: Schema, struct_result) -> Dict[int, NestedField]: return self._index - def struct(self, struct: StructType, result) -> Dict[int, NestedField]: + def struct(self, struct: StructType, field_results) -> Dict[int, NestedField]: return self._index - def field(self, field: NestedField, result) -> Dict[int, NestedField]: + def field(self, field: NestedField, field_result) -> Dict[int, NestedField]: """Add the field ID to the index""" self._index[field.field_id] = field return self._index - def list(self, list_type: ListType, result) -> Dict[int, NestedField]: + def list(self, list_type: ListType, element_result) -> Dict[int, NestedField]: """Add the list element ID to the index""" self._index[list_type.element.field_id] = list_type.element return self._index @@ -439,15 +434,15 @@ def after_field(self, field: NestedField) -> None: def schema(self, schema: Schema, struct_result: Dict[str, int]) -> Dict[str, int]: return self._index - def struct(self, struct: StructType, struct_result: List[Dict[str, int]]) -> Dict[str, int]: + def struct(self, struct: StructType, field_results: List[Dict[str, int]]) -> Dict[str, int]: return self._index - def field(self, field: NestedField, struct_result: Dict[str, int]) -> Dict[str, int]: + def field(self, field: NestedField, field_result: Dict[str, int]) -> Dict[str, int]: """Add the field name to the index""" self._add_field(field.name, field.field_id) return self._index - def list(self, list_type: ListType, struct_result: Dict[str, int]) -> Dict[str, int]: + def list(self, list_type: ListType, element_result: Dict[str, int]) -> Dict[str, int]: """Add the list element name to the index""" self._add_field(list_type.element.name, list_type.element.field_id) return self._index @@ -566,8 +561,8 @@ class _BuildPositionAccessors(SchemaVisitor[Dict[Position, Accessor]]): def _wrap_leaves(result: Dict[Position, Accessor], position: Position = 0) -> Dict[Position, Accessor]: return {field_id: Accessor(position, inner=inner) for field_id, inner in result.items()} - def schema(self, schema: Schema, result: Dict[Position, Accessor]) -> Dict[Position, Accessor]: - return result + def schema(self, schema: Schema, struct_result: Dict[Position, Accessor]) -> Dict[Position, Accessor]: + return struct_result def struct(self, struct: StructType, field_results: List[Dict[Position, Accessor]]) -> Dict[Position, Accessor]: result = {} @@ -581,10 +576,10 @@ def struct(self, struct: StructType, field_results: List[Dict[Position, Accessor return result - def field(self, field: NestedField, result: Dict[Position, Accessor]) -> Dict[Position, Accessor]: - return result + def field(self, field: NestedField, field_result: Dict[Position, Accessor]) -> Dict[Position, Accessor]: + return field_result - def list(self, list_type: ListType, result: Dict[Position, Accessor]) -> Dict[Position, Accessor]: + def list(self, list_type: ListType, element_result: Dict[Position, Accessor]) -> Dict[Position, Accessor]: return {} def map( diff --git a/src/iceberg/transforms.py b/src/iceberg/transforms.py index 8c54b214d8..6043d2aefa 100644 --- a/src/iceberg/transforms.py +++ b/src/iceberg/transforms.py @@ -16,7 +16,7 @@ # under the License. import struct -from abc import ABC +from abc import ABC, abstractmethod from decimal import Decimal from typing import Generic, Optional, TypeVar from uuid import UUID @@ -67,12 +67,15 @@ def __str__(self): def __call__(self, value: Optional[S]) -> Optional[T]: return self.apply(value) + @abstractmethod def apply(self, value: Optional[S]) -> Optional[T]: ... + @abstractmethod def can_transform(self, source: IcebergType) -> bool: return False + @abstractmethod def result_type(self, source: IcebergType) -> IcebergType: ... @@ -123,6 +126,10 @@ def apply(self, value: Optional[S]) -> Optional[int]: def result_type(self, source: IcebergType) -> IcebergType: return IntegerType() + @abstractmethod + def can_transform(self, source: IcebergType) -> bool: + pass + class BucketNumberTransform(BaseBucketTransform): """Transforms a value of IntegerType, LongType, DateType, TimeType, TimestampType, or TimestamptzType @@ -237,8 +244,8 @@ def __init__(self, source_type: IcebergType, transform: str): def apply(self, value: Optional[S]): raise AttributeError(f"Cannot apply unsupported transform: {self}") - def can_transform(self, target: IcebergType) -> bool: - return self._type == target + def can_transform(self, source: IcebergType) -> bool: + return self._type == source def result_type(self, source: IcebergType) -> IcebergType: return StringType() @@ -249,7 +256,7 @@ class VoidTransform(Transform): _instance = None - def __new__(cls): + def __new__(cls): # pylint: disable=W0221 if cls._instance is None: cls._instance = super(VoidTransform, cls).__new__(cls) return cls._instance @@ -260,7 +267,7 @@ def __init__(self): def apply(self, value: Optional[S]) -> None: return None - def can_transform(self, target: IcebergType) -> bool: + def can_transform(self, _: IcebergType) -> bool: return True def result_type(self, source: IcebergType) -> IcebergType: diff --git a/src/iceberg/types.py b/src/iceberg/types.py index 58b0d9de55..3173c4aca0 100644 --- a/src/iceberg/types.py +++ b/src/iceberg/types.py @@ -36,7 +36,7 @@ class Singleton: _instance = None - def __new__(cls, *args, **kwargs): + def __new__(cls): if not isinstance(cls._instance, cls): cls._instance = super(Singleton, cls).__new__(cls) return cls._instance diff --git a/tests/expressions/test_expressions_base.py b/tests/expressions/test_expressions_base.py index 61b801883a..f7fe4b5a45 100644 --- a/tests/expressions/test_expressions_base.py +++ b/tests/expressions/test_expressions_base.py @@ -115,7 +115,7 @@ def test_strs(op, string): @pytest.mark.parametrize( - "input, testexpra, testexprb", + "exp, testexpra, testexprb", [ ( base.And(TestExpressionA(), TestExpressionB()), @@ -132,12 +132,12 @@ def test_strs(op, string): (TestExpressionB(), TestExpressionB(), TestExpressionA()), ], ) -def test_eq(input, testexpra, testexprb): - assert input == testexpra and input != testexprb +def test_eq(exp, testexpra, testexprb): + assert exp == testexpra and exp != testexprb @pytest.mark.parametrize( - "input, exp", + "lhs, rhs", [ ( base.And(TestExpressionA(), TestExpressionB()), @@ -151,12 +151,12 @@ def test_eq(input, testexpra, testexprb): (TestExpressionA(), TestExpressionB()), ], ) -def test_negate(input, exp): - assert ~input == exp +def test_negate(lhs, rhs): + assert ~lhs == rhs @pytest.mark.parametrize( - "input, exp", + "lhs, rhs", [ ( base.And(TestExpressionA(), TestExpressionB(), TestExpressionA()), @@ -169,12 +169,12 @@ def test_negate(input, exp): (base.Not(base.Not(TestExpressionA())), TestExpressionA()), ], ) -def test_reduce(input, exp): - assert input == exp +def test_reduce(lhs, rhs): + assert lhs == rhs @pytest.mark.parametrize( - "input, exp", + "lhs, rhs", [ (base.And(base.AlwaysTrue(), TestExpressionB()), TestExpressionB()), (base.And(base.AlwaysFalse(), TestExpressionB()), base.AlwaysFalse()), @@ -183,8 +183,8 @@ def test_reduce(input, exp): (base.Not(base.Not(TestExpressionA())), TestExpressionA()), ], ) -def test_base_AlwaysTrue_base_AlwaysFalse(input, exp): - assert input == exp +def test_base_AlwaysTrue_base_AlwaysFalse(lhs, rhs): + assert lhs == rhs def test_accessor_base_class(foo_struct): diff --git a/tests/expressions/test_literals.py b/tests/expressions/test_literals.py index 59b943797f..c1eb48aa20 100644 --- a/tests/expressions/test_literals.py +++ b/tests/expressions/test_literals.py @@ -390,28 +390,28 @@ def test_raise_on_comparison_to_none(): fixed_lit012 = literal(bytes([0x00, 0x01, 0x02])) with pytest.raises(AttributeError): - bin_lit012 < None + _ = bin_lit012 < None with pytest.raises(AttributeError): - bin_lit012 <= None + _ = bin_lit012 <= None with pytest.raises(AttributeError): - bin_lit012 > None + _ = bin_lit012 > None with pytest.raises(AttributeError): - bin_lit012 >= None + _ = bin_lit012 >= None with pytest.raises(AttributeError): - fixed_lit012 < None + _ = fixed_lit012 < None with pytest.raises(AttributeError): - fixed_lit012 <= None + _ = fixed_lit012 <= None with pytest.raises(AttributeError): - fixed_lit012 > None + _ = fixed_lit012 > None with pytest.raises(AttributeError): - fixed_lit012 >= None + _ = fixed_lit012 >= None def test_binary_to_fixed(): diff --git a/tests/test_conversions.py b/tests/test_conversions.py index b107023473..76816a50c4 100644 --- a/tests/test_conversions.py +++ b/tests/test_conversions.py @@ -456,15 +456,15 @@ def __repr__(self): with pytest.raises(TypeError) as exc_info: conversions.partition_to_py(FooUnknownType(), "foo") - assert (f"Cannot convert 'foo' to unsupported type: FooUnknownType()") in str(exc_info.value) + assert "Cannot convert 'foo' to unsupported type: FooUnknownType()" in str(exc_info.value) with pytest.raises(TypeError) as exc_info: conversions.to_bytes(FooUnknownType(), "foo") - assert ("scale does not match FooUnknownType()") in str(exc_info.value) + assert "scale does not match FooUnknownType()" in str(exc_info.value) with pytest.raises(TypeError) as exc_info: conversions.from_bytes(FooUnknownType(), b"foo") - assert ("Cannot deserialize bytes, type FooUnknownType() not supported: b'foo'") in str(exc_info.value) + assert "Cannot deserialize bytes, type FooUnknownType() not supported: b'foo'" in str(exc_info.value) @pytest.mark.parametrize( diff --git a/tests/test_schema.py b/tests/test_schema.py index eee9fbd676..536277c913 100644 --- a/tests/test_schema.py +++ b/tests/test_schema.py @@ -49,7 +49,7 @@ def test_schema_str(table_schema_simple): @pytest.mark.parametrize( - "schema, expected_repr", + "schema_repr, expected_repr", [ ( schema.Schema(NestedField(1, "foo", StringType()), schema_id=1), @@ -63,9 +63,9 @@ def test_schema_str(table_schema_simple): ), ], ) -def test_schema_repr(schema, expected_repr): +def test_schema_repr(schema_repr, expected_repr): """Test schema representation""" - assert repr(schema) == expected_repr + assert repr(schema_repr) == expected_repr def test_schema_raise_on_duplicate_names(): @@ -241,7 +241,7 @@ def test_schema_find_field_by_id_raise_on_unknown_field(table_schema_simple): """Test raising when the field ID is not found among columns""" index = schema.index_by_id(table_schema_simple) with pytest.raises(Exception) as exc_info: - index[4] + _ = index[4] assert str(exc_info.value) == "4" @@ -392,18 +392,17 @@ def test_build_position_accessors(table_schema_nested): } -class TestStruct(StructProtocol): - def __init__(self, pos: Dict[int, Any] = None): - self._pos: Dict[int, Any] = pos or {} - - def set(self, pos: int, value) -> None: - pass +def test_build_position_accessors_with_struct(table_schema_nested): + class TestStruct(StructProtocol): + def __init__(self, pos: Dict[int, Any] = None): + self._pos: Dict[int, Any] = pos or {} - def get(self, pos: int) -> Any: - return self._pos[pos] + def set(self, pos: int, value) -> None: + pass + def get(self, pos: int) -> Any: + return self._pos[pos] -def test_build_position_accessors_with_struct(table_schema_nested): accessors = build_position_accessors(table_schema_nested) container = TestStruct({6: TestStruct({0: "name"})}) assert accessors.get(16).get(container) == "name" diff --git a/tests/test_transforms.py b/tests/test_transforms.py index a227124945..7855345f9d 100644 --- a/tests/test_transforms.py +++ b/tests/test_transforms.py @@ -14,6 +14,7 @@ # KIND, either express or implied. See the License for the # specific language governing permissions and limitations # under the License. +# pylint: disable=W0123 from decimal import Decimal from uuid import UUID diff --git a/tests/test_types.py b/tests/test_types.py index 07d3bcd0cf..cda56ab4e6 100644 --- a/tests/test_types.py +++ b/tests/test_types.py @@ -14,6 +14,7 @@ # KIND, either express or implied. See the License for the # specific language governing permissions and limitations # under the License. +# pylint: disable=W0123,W0613 import pytest diff --git a/tox.ini b/tox.ini index 5a1a0a7042..865a0a382a 100644 --- a/tox.ini +++ b/tox.ini @@ -59,7 +59,7 @@ commands = autoflake -r --check --ignore-init-module-imports --remove-all-unused-imports src tests isort --profile black --check-only src tests black --line-length 130 --check --diff src tests - pylint --disable all --enable spelling --spelling-dict en_US --spelling-private-dict-file spellcheck-dictionary.txt src tests + pylint src tests [testenv:format] deps = From 558402030045209ab78b9f298f93dd81c1368927 Mon Sep 17 00:00:00 2001 From: Fokko Driesprong Date: Tue, 17 May 2022 22:18:37 +0200 Subject: [PATCH 092/642] Change types into dataclasses (#4767) * Change types into dataclasses Proposal to change the types into dataclasses. This has several improvments: - We can use the dataclasss field(repr=True) to include the fields in the representation, instead of building our own strings - We can assign the types in the post_init when they are dynamic (List, Maps, Structs etc) , or just override them when they are static (Primitives) - We don't have to implement any eq methods because they come for free - The types are frozen, which is kind of nice since we re-use them - The code is much more consise - We can assign the min/max of the int/long/float as Final as of 3.8: https://peps.python.org/pep-0591/ My inspiration was the comment by Kyle: https://github.com/apache/iceberg/pull/4742#discussion_r869494393 This would entail implementing eq, but why not use the generated one since we're comparing all the attributes :) Would love to get you input * Remove explicit repr and eq * Use @cached_property to cache the string Add missing words to spelling * Add additional guard for initializing StructType using kwargs * Replace type with field_type --- setup.cfg | 1 + spellcheck-dictionary.txt | 6 +- src/iceberg/schema.py | 14 +- src/iceberg/types.py | 390 ++++++++++++++++++++------------------ tests/test_schema.py | 6 +- tests/test_types.py | 10 +- 6 files changed, 222 insertions(+), 205 deletions(-) diff --git a/setup.cfg b/setup.cfg index 559751011c..18f4d8245d 100644 --- a/setup.cfg +++ b/setup.cfg @@ -45,6 +45,7 @@ python_requires = >=3.7 install_requires = mmh3 singledispatch + cached-property; python_version <= '3.7' [options.extras_require] arrow = pyarrow diff --git a/spellcheck-dictionary.txt b/spellcheck-dictionary.txt index 2476d5afde..1100dd7036 100644 --- a/spellcheck-dictionary.txt +++ b/spellcheck-dictionary.txt @@ -26,10 +26,13 @@ FileInfo filesystem fs func +IcebergType io NativeFile +NestedField nullability pragma +PrimitiveType pyarrow repr schemas @@ -42,4 +45,5 @@ StructType Timestamptz Timestamptzs unscaled -URI \ No newline at end of file +URI + diff --git a/src/iceberg/schema.py b/src/iceberg/schema.py index 9b96504480..59b9cc8358 100644 --- a/src/iceberg/schema.py +++ b/src/iceberg/schema.py @@ -152,7 +152,7 @@ def find_type(self, name_or_id: str | int, case_sensitive: bool = True) -> Icebe NestedField: The type of the matched NestedField """ field = self.find_field(name_or_id=name_or_id, case_sensitive=case_sensitive) - return field.type # type: ignore + return field.field_type def find_column_name(self, column_id: int) -> str: """Find a column name given a column ID @@ -318,7 +318,7 @@ def _(obj: StructType, visitor: SchemaVisitor[T]) -> T: for field in obj.fields: visitor.before_field(field) - result = visit(field.type, visitor) + result = visit(field.field_type, visitor) visitor.after_field(field) results.append(visitor.field(field, result)) @@ -330,7 +330,7 @@ def _(obj: ListType, visitor: SchemaVisitor[T]) -> T: """Visit a ListType with a concrete SchemaVisitor""" visitor.before_list_element(obj.element) - result = visit(obj.element.type, visitor) + result = visit(obj.element.field_type, visitor) visitor.after_list_element(obj.element) return visitor.list(obj, result) @@ -340,11 +340,11 @@ def _(obj: ListType, visitor: SchemaVisitor[T]) -> T: def _(obj: MapType, visitor: SchemaVisitor[T]) -> T: """Visit a MapType with a concrete SchemaVisitor""" visitor.before_map_key(obj.key) - key_result = visit(obj.key.type, visitor) + key_result = visit(obj.key.field_type, visitor) visitor.after_map_key(obj.key) visitor.before_map_value(obj.value) - value_result = visit(obj.value.type, visitor) + value_result = visit(obj.value.field_type, visitor) visitor.after_list_element(obj.value) return visitor.map(obj, key_result, value_result) @@ -412,12 +412,12 @@ def __init__(self) -> None: def before_list_element(self, element: NestedField) -> None: """Short field names omit element when the element is a StructType""" - if not isinstance(element.type, StructType): + if not isinstance(element.field_type, StructType): self._short_field_names.append(element.name) self._field_names.append(element.name) def after_list_element(self, element: NestedField) -> None: - if not isinstance(element.type, StructType): + if not isinstance(element.field_type, StructType): self._short_field_names.pop() self._field_names.pop() diff --git a/src/iceberg/types.py b/src/iceberg/types.py index 3173c4aca0..5318e4f8a2 100644 --- a/src/iceberg/types.py +++ b/src/iceberg/types.py @@ -29,8 +29,15 @@ Notes: - https://iceberg.apache.org/#spec/#primitive-types """ +import sys +from dataclasses import dataclass, field +from typing import ClassVar, Dict, List, Optional, Tuple -from typing import Dict, Optional, Tuple +if sys.version_info >= (3, 8): + from functools import cached_property +else: + # In the case of <= Python 3.7 + from cached_property import cached_property class Singleton: @@ -42,57 +49,64 @@ def __new__(cls): return cls._instance +@dataclass(frozen=True) class IcebergType: - """Base type for all Iceberg Types""" + """Base type for all Iceberg Types - _initialized = False - - def __init__(self, type_string: str, repr_string: str): - self._type_string = type_string - self._repr_string = repr_string - self._initialized = True + Example: + >>> str(IcebergType()) + 'IcebergType()' + >>> repr(IcebergType()) + 'IcebergType()' + """ - def __repr__(self): - return self._repr_string + @property + def string_type(self) -> str: + return self.__repr__() - def __str__(self): - return self._type_string + def __str__(self) -> str: + return self.string_type @property def is_primitive(self) -> bool: return isinstance(self, PrimitiveType) +@dataclass(frozen=True, eq=True) class PrimitiveType(IcebergType): - """Base class for all Iceberg Primitive Types""" + """Base class for all Iceberg Primitive Types + + Example: + >>> str(PrimitiveType()) + 'PrimitiveType()' + """ +@dataclass(frozen=True) class FixedType(PrimitiveType): """A fixed data type in Iceberg. Example: >>> FixedType(8) FixedType(length=8) - >>> FixedType(8)==FixedType(8) + >>> FixedType(8) == FixedType(8) True """ - _instances: Dict[int, "FixedType"] = {} + length: int = field() + + _instances: ClassVar[Dict[int, "FixedType"]] = {} def __new__(cls, length: int): cls._instances[length] = cls._instances.get(length) or object.__new__(cls) return cls._instances[length] - def __init__(self, length: int): - if not self._initialized: - super().__init__(f"fixed[{length}]", f"FixedType(length={length})") - self._length = length - @property - def length(self) -> int: - return self._length + def string_type(self) -> str: + return f"fixed[{self.length}]" +@dataclass(frozen=True, eq=True) class DecimalType(PrimitiveType): """A fixed data type in Iceberg. @@ -103,38 +117,44 @@ class DecimalType(PrimitiveType): True """ - _instances: Dict[Tuple[int, int], "DecimalType"] = {} + precision: int = field() + scale: int = field() + + _instances: ClassVar[Dict[Tuple[int, int], "DecimalType"]] = {} def __new__(cls, precision: int, scale: int): key = (precision, scale) cls._instances[key] = cls._instances.get(key) or object.__new__(cls) return cls._instances[key] - def __init__(self, precision: int, scale: int): - if not self._initialized: - super().__init__( - f"decimal({precision}, {scale})", - f"DecimalType(precision={precision}, scale={scale})", - ) - self._precision = precision - self._scale = scale - - @property - def precision(self) -> int: - return self._precision - @property - def scale(self) -> int: - return self._scale + def string_type(self) -> str: + return f"decimal({self.precision}, {self.scale})" +@dataclass(frozen=True) class NestedField(IcebergType): """Represents a field of a struct, a map key, a map value, or a list element. This is where field IDs, names, docs, and nullability are tracked. + + Example: + >>> str(NestedField( + ... field_id=1, + ... name='foo', + ... field_type=FixedType(22), + ... is_optional=False, + ... )) + '1: foo: required fixed[22]' """ - _instances: Dict[Tuple[bool, int, str, IcebergType, Optional[str]], "NestedField"] = {} + field_id: int = field() + name: str = field() + field_type: IcebergType = field() + is_optional: bool = field(default=True) + doc: Optional[str] = field(default=None, repr=False) + + _instances: ClassVar[Dict[Tuple[bool, int, str, IcebergType, Optional[str]], "NestedField"]] = {} def __new__( cls, @@ -148,56 +168,20 @@ def __new__( cls._instances[key] = cls._instances.get(key) or object.__new__(cls) return cls._instances[key] - def __init__( - self, - field_id: int, - name: str, - field_type: IcebergType, - is_optional: bool = True, - doc: Optional[str] = None, - ): - if not self._initialized: - docString = "" if doc is None else f", doc={repr(doc)}" - super().__init__( - ( - f"{field_id}: {name}: {'optional' if is_optional else 'required'} {field_type}" "" - if doc is None - else f" ({doc})" - ), - f"NestedField(field_id={field_id}, name={repr(name)}, field_type={repr(field_type)}, is_optional={is_optional}" - f"{docString})", - ) - self._is_optional = is_optional - self._id = field_id - self._name = name - self._type = field_type - self._doc = doc - - @property - def is_optional(self) -> bool: - return self._is_optional - @property def is_required(self) -> bool: - return not self._is_optional - - @property - def field_id(self) -> int: - return self._id - - @property - def name(self) -> str: - return self._name - - @property - def doc(self) -> Optional[str]: - return self._doc + return not self.is_optional @property - def type(self) -> IcebergType: - return self._type + def string_type(self) -> str: + return ( + f"{self.field_id}: {self.name}: {'optional' if self.is_optional else 'required'} {self.field_type}" + if self.doc is None + else f" ({self.doc})" + ) +@dataclass(frozen=True, init=False) class StructType(IcebergType): """A struct type in Iceberg @@ -209,25 +193,27 @@ class StructType(IcebergType): 'struct<1: required_field: optional string, 2: optional_field: optional int>' """ - _instances: Dict[Tuple[NestedField, ...], "StructType"] = {} + fields: List[NestedField] = field() + + _instances: ClassVar[Dict[Tuple[NestedField, ...], "StructType"]] = {} - def __new__(cls, *fields: NestedField): + def __new__(cls, *fields: NestedField, **kwargs): + if not fields and "fields" in kwargs: + fields = kwargs["fields"] cls._instances[fields] = cls._instances.get(fields) or object.__new__(cls) return cls._instances[fields] - def __init__(self, *fields: NestedField): - if not self._initialized: - super().__init__( - f"struct<{', '.join(map(str, fields))}>", - f"StructType{repr(fields)}", - ) - self._fields = fields + def __init__(self, *fields: NestedField, **kwargs): + if not fields and "fields" in kwargs: + fields = kwargs["fields"] + object.__setattr__(self, "fields", fields) - @property - def fields(self) -> Tuple[NestedField, ...]: - return self._fields + @cached_property + def string_type(self) -> str: + return f"struct<{', '.join(map(str, self.fields))}>" +@dataclass(frozen=True) class ListType(IcebergType): """A list type in Iceberg @@ -236,7 +222,12 @@ class ListType(IcebergType): ListType(element_id=3, element_type=StringType(), element_is_optional=True) """ - _instances: Dict[Tuple[bool, int, IcebergType], "ListType"] = {} + element_id: int = field() + element_type: IcebergType = field() + element_is_optional: bool = field(default=True) + element: NestedField = field(init=False, repr=False) + + _instances: ClassVar[Dict[Tuple[bool, int, IcebergType], "ListType"]] = {} def __new__( cls, @@ -248,30 +239,24 @@ def __new__( cls._instances[key] = cls._instances.get(key) or object.__new__(cls) return cls._instances[key] - def __init__( - self, - element_id: int, - element_type: IcebergType, - element_is_optional: bool = True, - ): - if not self._initialized: - super().__init__( - f"list<{element_type}>", - f"ListType(element_id={element_id}, element_type={repr(element_type)}, " - f"element_is_optional={element_is_optional})", - ) - self._element_field = NestedField( + def __post_init__(self): + object.__setattr__( + self, + "element", + NestedField( name="element", - is_optional=element_is_optional, - field_id=element_id, - field_type=element_type, - ) + is_optional=self.element_is_optional, + field_id=self.element_id, + field_type=self.element_type, + ), + ) @property - def element(self) -> NestedField: - return self._element_field + def string_type(self) -> str: + return f"list<{self.element_type}>" +@dataclass(frozen=True) class MapType(IcebergType): """A map type in Iceberg @@ -280,7 +265,16 @@ class MapType(IcebergType): MapType(key_id=1, key_type=StringType(), value_id=2, value_type=IntegerType(), value_is_optional=True) """ - _instances: Dict[Tuple[int, IcebergType, int, IcebergType, bool], "MapType"] = {} + key_id: int = field() + key_type: IcebergType = field() + value_id: int = field() + value_type: IcebergType = field() + value_is_optional: bool = field(default=True) + key: NestedField = field(init=False, repr=False) + value: NestedField = field(init=False, repr=False) + + # _type_string_def = lambda self: f"map<{self.key_type}, {self.value_type}>" + _instances: ClassVar[Dict[Tuple[int, IcebergType, int, IcebergType, bool], "MapType"]] = {} def __new__( cls, @@ -294,37 +288,23 @@ def __new__( cls._instances[impl_key] = cls._instances.get(impl_key) or object.__new__(cls) return cls._instances[impl_key] - def __init__( - self, - key_id: int, - key_type: IcebergType, - value_id: int, - value_type: IcebergType, - value_is_optional: bool = True, - ): - if not self._initialized: - super().__init__( - f"map<{key_type}, {value_type}>", - f"MapType(key_id={key_id}, key_type={repr(key_type)}, value_id={value_id}, value_type={repr(value_type)}, " - f"value_is_optional={value_is_optional})", - ) - self._key_field = NestedField(name="key", field_id=key_id, field_type=key_type, is_optional=False) - self._value_field = NestedField( + def __post_init__(self): + object.__setattr__( + self, "key", NestedField(name="key", field_id=self.key_id, field_type=self.key_type, is_optional=False) + ) + object.__setattr__( + self, + "value", + NestedField( name="value", - field_id=value_id, - field_type=value_type, - is_optional=value_is_optional, - ) - - @property - def key(self) -> NestedField: - return self._key_field - - @property - def value(self) -> NestedField: - return self._value_field + field_id=self.value_id, + field_type=self.value_type, + is_optional=self.value_is_optional, + ), + ) +@dataclass(frozen=True) class BooleanType(PrimitiveType, Singleton): """A boolean data type in Iceberg can be represented using an instance of this class. @@ -332,13 +312,16 @@ class BooleanType(PrimitiveType, Singleton): >>> column_foo = BooleanType() >>> isinstance(column_foo, BooleanType) True + >>> column_foo + BooleanType() """ - def __init__(self): - if not self._initialized: - super().__init__("boolean", "BooleanType()") + @property + def string_type(self) -> str: + return "boolean" +@dataclass(frozen=True) class IntegerType(PrimitiveType, Singleton): """An Integer data type in Iceberg can be represented using an instance of this class. Integers in Iceberg are 32-bit signed and can be promoted to Longs. @@ -355,15 +338,15 @@ class IntegerType(PrimitiveType, Singleton): in Java (returns `-2147483648`) """ - max: int = 2147483647 + max: ClassVar[int] = 2147483647 + min: ClassVar[int] = -2147483648 - min: int = -2147483648 - - def __init__(self): - if not self._initialized: - super().__init__("int", "IntegerType()") + @property + def string_type(self) -> str: + return "int" +@dataclass(frozen=True) class LongType(PrimitiveType, Singleton): """A Long data type in Iceberg can be represented using an instance of this class. Longs in Iceberg are 64-bit signed integers. @@ -372,6 +355,10 @@ class LongType(PrimitiveType, Singleton): >>> column_foo = LongType() >>> isinstance(column_foo, LongType) True + >>> column_foo + LongType() + >>> str(column_foo) + 'long' Attributes: max (int): The maximum allowed value for Longs, inherited from the canonical Iceberg implementation @@ -380,15 +367,15 @@ class LongType(PrimitiveType, Singleton): in Java (returns `-9223372036854775808`) """ - max: int = 9223372036854775807 + max: ClassVar[int] = 9223372036854775807 + min: ClassVar[int] = -9223372036854775808 - min: int = -9223372036854775808 - - def __init__(self): - if not self._initialized: - super().__init__("long", "LongType()") + @property + def string_type(self) -> str: + return "long" +@dataclass(frozen=True) class FloatType(PrimitiveType, Singleton): """A Float data type in Iceberg can be represented using an instance of this class. Floats in Iceberg are 32-bit IEEE 754 floating points and can be promoted to Doubles. @@ -397,6 +384,8 @@ class FloatType(PrimitiveType, Singleton): >>> column_foo = FloatType() >>> isinstance(column_foo, FloatType) True + >>> column_foo + FloatType() Attributes: max (float): The maximum allowed value for Floats, inherited from the canonical Iceberg implementation @@ -405,15 +394,15 @@ class FloatType(PrimitiveType, Singleton): in Java (returns `-3.4028235e38`) """ - max: float = 3.4028235e38 + max: ClassVar[float] = 3.4028235e38 + min: ClassVar[float] = -3.4028235e38 - min: float = -3.4028235e38 - - def __init__(self): - if not self._initialized: - super().__init__("float", "FloatType()") + @property + def string_type(self) -> str: + return "float" +@dataclass(frozen=True) class DoubleType(PrimitiveType, Singleton): """A Double data type in Iceberg can be represented using an instance of this class. Doubles in Iceberg are 64-bit IEEE 754 floating points. @@ -422,13 +411,16 @@ class DoubleType(PrimitiveType, Singleton): >>> column_foo = DoubleType() >>> isinstance(column_foo, DoubleType) True + >>> column_foo + DoubleType() """ - def __init__(self): - if not self._initialized: - super().__init__("double", "DoubleType()") + @property + def string_type(self) -> str: + return "double" +@dataclass(frozen=True) class DateType(PrimitiveType, Singleton): """A Date data type in Iceberg can be represented using an instance of this class. Dates in Iceberg are calendar dates without a timezone or time. @@ -437,13 +429,16 @@ class DateType(PrimitiveType, Singleton): >>> column_foo = DateType() >>> isinstance(column_foo, DateType) True + >>> column_foo + DateType() """ - def __init__(self): - if not self._initialized: - super().__init__("date", "DateType()") + @property + def string_type(self) -> str: + return "date" +@dataclass(frozen=True) class TimeType(PrimitiveType, Singleton): """A Time data type in Iceberg can be represented using an instance of this class. Times in Iceberg have microsecond precision and are a time of day without a date or timezone. @@ -452,13 +447,16 @@ class TimeType(PrimitiveType, Singleton): >>> column_foo = TimeType() >>> isinstance(column_foo, TimeType) True + >>> column_foo + TimeType() """ - def __init__(self): - if not self._initialized: - super().__init__("time", "TimeType()") + @property + def string_type(self) -> str: + return "time" +@dataclass(frozen=True) class TimestampType(PrimitiveType, Singleton): """A Timestamp data type in Iceberg can be represented using an instance of this class. Timestamps in Iceberg have microsecond precision and include a date and a time of day without a timezone. @@ -467,13 +465,16 @@ class TimestampType(PrimitiveType, Singleton): >>> column_foo = TimestampType() >>> isinstance(column_foo, TimestampType) True + >>> column_foo + TimestampType() """ - def __init__(self): - if not self._initialized: - super().__init__("timestamp", "TimestampType()") + @property + def string_type(self) -> str: + return "timestamp" +@dataclass(frozen=True) class TimestamptzType(PrimitiveType, Singleton): """A Timestamptz data type in Iceberg can be represented using an instance of this class. Timestamptzs in Iceberg are stored as UTC and include a date and a time of day with a timezone. @@ -482,13 +483,16 @@ class TimestamptzType(PrimitiveType, Singleton): >>> column_foo = TimestamptzType() >>> isinstance(column_foo, TimestamptzType) True + >>> column_foo + TimestamptzType() """ - def __init__(self): - if not self._initialized: - super().__init__("timestamptz", "TimestamptzType()") + @property + def string_type(self) -> str: + return "timestamptz" +@dataclass(frozen=True) class StringType(PrimitiveType, Singleton): """A String data type in Iceberg can be represented using an instance of this class. Strings in Iceberg are arbitrary-length character sequences and are encoded with UTF-8. @@ -497,13 +501,16 @@ class StringType(PrimitiveType, Singleton): >>> column_foo = StringType() >>> isinstance(column_foo, StringType) True + >>> column_foo + StringType() """ - def __init__(self): - if not self._initialized: - super().__init__("string", "StringType()") + @property + def string_type(self) -> str: + return "string" +@dataclass(frozen=True) class UUIDType(PrimitiveType, Singleton): """A UUID data type in Iceberg can be represented using an instance of this class. UUIDs in Iceberg are universally unique identifiers. @@ -512,13 +519,16 @@ class UUIDType(PrimitiveType, Singleton): >>> column_foo = UUIDType() >>> isinstance(column_foo, UUIDType) True + >>> column_foo + UUIDType() """ - def __init__(self): - if not self._initialized: - super().__init__("uuid", "UUIDType()") + @property + def string_type(self) -> str: + return "uuid" +@dataclass(frozen=True) class BinaryType(PrimitiveType, Singleton): """A Binary data type in Iceberg can be represented using an instance of this class. Binaries in Iceberg are arbitrary-length byte arrays. @@ -527,8 +537,10 @@ class BinaryType(PrimitiveType, Singleton): >>> column_foo = BinaryType() >>> isinstance(column_foo, BinaryType) True + >>> column_foo + BinaryType() """ - def __init__(self): - if not self._initialized: - super().__init__("binary", "BinaryType()") + @property + def string_type(self) -> str: + return "binary" diff --git a/tests/test_schema.py b/tests/test_schema.py index 536277c913..d48198edd9 100644 --- a/tests/test_schema.py +++ b/tests/test_schema.py @@ -221,19 +221,19 @@ def test_schema_find_field_by_id(table_schema_simple): column1 = index[1] assert isinstance(column1, NestedField) assert column1.field_id == 1 - assert column1.type == StringType() + assert column1.field_type == StringType() assert column1.is_optional == False column2 = index[2] assert isinstance(column2, NestedField) assert column2.field_id == 2 - assert column2.type == IntegerType() + assert column2.field_type == IntegerType() assert column2.is_optional == True column3 = index[3] assert isinstance(column3, NestedField) assert column3.field_id == 3 - assert column3.type == BooleanType() + assert column3.field_type == BooleanType() assert column3.is_optional == False diff --git a/tests/test_types.py b/tests/test_types.py index cda56ab4e6..844e3ab7de 100644 --- a/tests/test_types.py +++ b/tests/test_types.py @@ -146,8 +146,8 @@ def test_list_type(): ), False, ) - assert isinstance(type_var.element.type, StructType) - assert len(type_var.element.type.fields) == 2 + assert isinstance(type_var.element.field_type, StructType) + assert len(type_var.element.field_type.fields) == 2 assert type_var.element.field_id == 1 assert str(type_var) == str(eval(repr(type_var))) assert type_var == eval(repr(type_var)) @@ -162,9 +162,9 @@ def test_list_type(): def test_map_type(): type_var = MapType(1, DoubleType(), 2, UUIDType(), False) - assert isinstance(type_var.key.type, DoubleType) + assert isinstance(type_var.key.field_type, DoubleType) assert type_var.key.field_id == 1 - assert isinstance(type_var.value.type, UUIDType) + assert isinstance(type_var.value.field_type, UUIDType) assert type_var.value.field_id == 2 assert str(type_var) == str(eval(repr(type_var))) assert type_var == eval(repr(type_var)) @@ -193,7 +193,7 @@ def test_nested_field(): assert field_var.is_optional assert not field_var.is_required assert field_var.field_id == 1 - assert isinstance(field_var.type, StructType) + assert isinstance(field_var.field_type, StructType) assert str(field_var) == str(eval(repr(field_var))) From b8266eff4db899aee57b55da9ac07f819206e080 Mon Sep 17 00:00:00 2001 From: Kyle Bendickson Date: Tue, 17 May 2022 17:53:14 -0700 Subject: [PATCH 093/642] Python - Add pyline disable super-init-not-called for one line (#4797) --- src/iceberg/types.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/iceberg/types.py b/src/iceberg/types.py index 5318e4f8a2..77b704a980 100644 --- a/src/iceberg/types.py +++ b/src/iceberg/types.py @@ -203,7 +203,7 @@ def __new__(cls, *fields: NestedField, **kwargs): cls._instances[fields] = cls._instances.get(fields) or object.__new__(cls) return cls._instances[fields] - def __init__(self, *fields: NestedField, **kwargs): + def __init__(self, *fields: NestedField, **kwargs): # pylint: disable=super-init-not-called if not fields and "fields" in kwargs: fields = kwargs["fields"] object.__setattr__(self, "fields", fields) From 79d4ff849b6209c823bfb3670055ddbf755a951c Mon Sep 17 00:00:00 2001 From: Fokko Driesprong Date: Wed, 18 May 2022 19:10:15 +0200 Subject: [PATCH 094/642] Python: Set Python 3.8 as minimum version (#4784) --- .python-version | 2 +- setup.cfg | 6 ++---- src/iceberg/expressions/literals.py | 23 ++++++++--------------- src/iceberg/schema.py | 8 +------- src/iceberg/types.py | 8 +------- tox.ini | 4 ++-- 6 files changed, 15 insertions(+), 36 deletions(-) diff --git a/.python-version b/.python-version index 7ffc3f2f46..879a094044 100644 --- a/.python-version +++ b/.python-version @@ -1,3 +1,3 @@ -3.7.12 3.8.12 3.9.10 +3.10.4 diff --git a/setup.cfg b/setup.cfg index 18f4d8245d..ab1aa16e35 100644 --- a/setup.cfg +++ b/setup.cfg @@ -33,19 +33,17 @@ license_files = classifiers = License :: OSI Approved :: Apache Software License Operating System :: OS Independent - Programming Language :: Python :: 3.7 Programming Language :: Python :: 3.8 Programming Language :: Python :: 3.9 + Programming Language :: Python :: 3.10 [options] package_dir = = src packages = find: -python_requires = >=3.7 +python_requires = >=3.8 install_requires = mmh3 - singledispatch - cached-property; python_version <= '3.7' [options.extras_require] arrow = pyarrow diff --git a/src/iceberg/expressions/literals.py b/src/iceberg/expressions/literals.py index 1c21f00e6c..a6128ee6c6 100644 --- a/src/iceberg/expressions/literals.py +++ b/src/iceberg/expressions/literals.py @@ -21,25 +21,11 @@ # pylint: disable=W0613 import struct -import sys from decimal import ROUND_HALF_UP, Decimal -from functools import singledispatch +from functools import singledispatch, singledispatchmethod from typing import Optional, Union from uuid import UUID -from iceberg.utils.datetime import ( - date_to_days, - micros_to_days, - time_to_micros, - timestamp_to_micros, - timestamptz_to_micros, -) - -if sys.version_info >= (3, 8): - from functools import singledispatchmethod # pragma: no cover -else: - from singledispatch import singledispatchmethod # pragma: no cover - from iceberg.expressions.base import Literal from iceberg.types import ( BinaryType, @@ -58,6 +44,13 @@ TimeType, UUIDType, ) +from iceberg.utils.datetime import ( + date_to_days, + micros_to_days, + time_to_micros, + timestamp_to_micros, + timestamptz_to_micros, +) @singledispatch diff --git a/src/iceberg/schema.py b/src/iceberg/schema.py index 59b9cc8358..15622fa7c7 100644 --- a/src/iceberg/schema.py +++ b/src/iceberg/schema.py @@ -18,18 +18,12 @@ from __future__ import annotations -import sys from abc import ABC, abstractmethod from dataclasses import dataclass +from functools import singledispatch from typing import Any, Dict, Generic, Iterable, List, Optional, TypeVar from iceberg.files import StructProtocol - -if sys.version_info >= (3, 8): - from functools import singledispatch # pragma: no cover -else: - from singledispatch import singledispatch # pragma: no cover - from iceberg.types import ( IcebergType, ListType, diff --git a/src/iceberg/types.py b/src/iceberg/types.py index 77b704a980..75bf1955a9 100644 --- a/src/iceberg/types.py +++ b/src/iceberg/types.py @@ -29,16 +29,10 @@ Notes: - https://iceberg.apache.org/#spec/#primitive-types """ -import sys from dataclasses import dataclass, field +from functools import cached_property from typing import ClassVar, Dict, List, Optional, Tuple -if sys.version_info >= (3, 8): - from functools import cached_property -else: - # In the case of <= Python 3.7 - from cached_property import cached_property - class Singleton: _instance = None diff --git a/tox.ini b/tox.ini index 865a0a382a..488957edf3 100644 --- a/tox.ini +++ b/tox.ini @@ -16,7 +16,7 @@ # under the License. [tox] -envlist = py37,py38,py39,linters +envlist = py38,py39,py310,linters skip_missing_interpreters = true [testenv] @@ -97,9 +97,9 @@ addopts = --doctest-modules [gh-actions] python = - 3.7: py37 3.8: py38, linters 3.9: py39 + 3.10: py310 [mypy] From 2b82b7d9eed6a3aeccef11e0a132305b633abec2 Mon Sep 17 00:00:00 2001 From: Fokko Driesprong Date: Fri, 20 May 2022 17:53:18 +0200 Subject: [PATCH 095/642] Python: Fix the type_string of the NestedField (#4814) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit If there is a doc, the rest gets ignored, which is kind of awkward. Before: ```python ➜ python git:(master) ✗ python3 Python 3.9.12 (main, Mar 26 2022, 15:44:31) [Clang 13.1.6 (clang-1316.0.21.2)] on darwin Type "help", "copyright", "credits" or "license" for more information. >>> from iceberg.types import NestedField, LongType >>> str(NestedField( ... field_id=2, ... name='bar', ... field_type=LongType(), ... is_optional=False, ... doc="Just a long" ... )) '2: bar: required long' >>> str(NestedField( ... field_id=2, ... name='bar', ... field_type=LongType(), ... is_optional=False, ... doc="Just a long" ... )) ' (Just a long)' ``` Now: ```python ➜ python git:(master) ✗ python3 Python 3.9.12 (main, Mar 26 2022, 15:44:31) [Clang 13.1.6 (clang-1316.0.21.2)] on darwin Type "help", "copyright", "credits" or "license" for more information. >>> str(NestedField( ... field_id=2, ... name='bar', ... field_type=LongType(), ... is_optional=False, ... doc="Just a long" ... )) '2: bar: required long (Just a long)' ``` --- src/iceberg/types.py | 16 +++++++++++----- 1 file changed, 11 insertions(+), 5 deletions(-) diff --git a/src/iceberg/types.py b/src/iceberg/types.py index 75bf1955a9..1b206a1159 100644 --- a/src/iceberg/types.py +++ b/src/iceberg/types.py @@ -140,6 +140,14 @@ class NestedField(IcebergType): ... is_optional=False, ... )) '1: foo: required fixed[22]' + >>> str(NestedField( + ... field_id=2, + ... name='bar', + ... field_type=LongType(), + ... is_optional=False, + ... doc="Just a long" + ... )) + '2: bar: required long (Just a long)' """ field_id: int = field() @@ -168,11 +176,9 @@ def is_required(self) -> bool: @property def string_type(self) -> str: - return ( - f"{self.field_id}: {self.name}: {'optional' if self.is_optional else 'required'} {self.field_type}" - if self.doc is None - else f" ({self.doc})" - ) + doc = "" if not self.doc else f" ({self.doc})" + req = "optional" if self.is_optional else "required" + return f"{self.field_id}: {self.name}: {req} {self.field_type}{doc}" @dataclass(frozen=True, init=False) From 5f0e214a949d7f8991eb535f4b7b0c037b471ddf Mon Sep 17 00:00:00 2001 From: Samuel Redai <43911210+samredai@users.noreply.github.com> Date: Fri, 27 May 2022 14:17:55 -0400 Subject: [PATCH 096/642] Python: Add BooleanExpressionVisitor and visit method (#4815) --- spellcheck-dictionary.txt | 5 +- src/iceberg/expressions/base.py | 113 ++++++++++++++++++++- tests/expressions/test_expressions_base.py | 96 +++++++++++++++++ 3 files changed, 211 insertions(+), 3 deletions(-) diff --git a/spellcheck-dictionary.txt b/spellcheck-dictionary.txt index 1100dd7036..299c83f2f1 100644 --- a/spellcheck-dictionary.txt +++ b/spellcheck-dictionary.txt @@ -46,4 +46,7 @@ Timestamptz Timestamptzs unscaled URI - +UnboundPredicate +BoundPredicate +BooleanExpression +BooleanExpressionVisitor \ No newline at end of file diff --git a/src/iceberg/expressions/base.py b/src/iceberg/expressions/base.py index 7f0b98b162..63a087ec12 100644 --- a/src/iceberg/expressions/base.py +++ b/src/iceberg/expressions/base.py @@ -16,7 +16,7 @@ # under the License. from abc import ABC, abstractmethod from enum import Enum, auto -from functools import reduce +from functools import reduce, singledispatch from typing import Any, Generic, TypeVar from iceberg.files import StructProtocol @@ -261,7 +261,7 @@ def __invert__(self) -> "AlwaysTrue": return AlwaysTrue() def __repr__(self) -> str: - return "AlwaysTrue()" + return "AlwaysFalse()" def __str__(self) -> str: return "false" @@ -346,3 +346,112 @@ def bind(self, schema: Schema, case_sensitive: bool) -> BoundReference: raise ValueError(f"Cannot find field '{self.name}' in schema: {schema}") return BoundReference(field=field, accessor=schema.accessor_for_field(field.field_id)) + + +class BooleanExpressionVisitor(Generic[T], ABC): + @abstractmethod + def visit_true(self) -> T: + """Visit method for an AlwaysTrue boolean expression + + Note: This visit method has no arguments since AlwaysTrue instances have no context. + """ + + @abstractmethod + def visit_false(self) -> T: + """Visit method for an AlwaysFalse boolean expression + + Note: This visit method has no arguments since AlwaysFalse instances have no context. + """ + + @abstractmethod + def visit_not(self, child_result: T) -> T: + """Visit method for a Not boolean expression + + Args: + result (T): The result of visiting the child of the Not boolean expression + """ + + @abstractmethod + def visit_and(self, left_result: T, right_result: T) -> T: + """Visit method for an And boolean expression + + Args: + left_result (T): The result of visiting the left side of the expression + right_result (T): The result of visiting the right side of the expression + """ + + @abstractmethod + def visit_or(self, left_result: T, right_result: T) -> T: + """Visit method for an Or boolean expression + + Args: + left_result (T): The result of visiting the left side of the expression + right_result (T): The result of visiting the right side of the expression + """ + + @abstractmethod + def visit_unbound_predicate(self, predicate) -> T: + """Visit method for an unbound predicate in an expression tree + + Args: + predicate (UnboundPredicate): An instance of an UnboundPredicate + """ + + @abstractmethod + def visit_bound_predicate(self, predicate) -> T: + """Visit method for a bound predicate in an expression tree + + Args: + predicate (BoundPredicate): An instance of a BoundPredicate + """ + + +@singledispatch +def visit(obj, visitor: BooleanExpressionVisitor[T]) -> T: + """A generic function for applying a boolean expression visitor to any point within an expression + + The function traverses the expression in post-order fashion + + Args: + obj(BooleanExpression): An instance of a BooleanExpression + visitor(BooleanExpressionVisitor[T]): An instance of an implementation of the generic BooleanExpressionVisitor base class + + Raises: + NotImplementedError: If attempting to visit an unsupported expression + """ + raise NotImplementedError(f"Cannot visit unsupported expression: {obj}") + + +@visit.register(AlwaysTrue) +def _(obj: AlwaysTrue, visitor: BooleanExpressionVisitor[T]) -> T: + """Visit an AlwaysTrue boolean expression with a concrete BooleanExpressionVisitor""" + return visitor.visit_true() + + +@visit.register(AlwaysFalse) +def _(obj: AlwaysFalse, visitor: BooleanExpressionVisitor[T]) -> T: + """Visit an AlwaysFalse boolean expression with a concrete BooleanExpressionVisitor""" + return visitor.visit_false() + + +@visit.register(Not) +def _(obj: Not, visitor: BooleanExpressionVisitor[T]) -> T: + """Visit a Not boolean expression with a concrete BooleanExpressionVisitor""" + child_result: T = visit(obj.child, visitor=visitor) + return visitor.visit_not(child_result=child_result) + + +@visit.register(And) +def _(obj: And, visitor: BooleanExpressionVisitor[T]) -> T: + """Visit an And boolean expression with a concrete BooleanExpressionVisitor""" + left_result: T = visit(obj.left, visitor=visitor) + right_result: T = visit(obj.right, visitor=visitor) + return visitor.visit_and(left_result=left_result, right_result=right_result) + + +@visit.register(Or) +def _(obj: Or, visitor: BooleanExpressionVisitor[T]) -> T: + """Visit an Or boolean expression with a concrete BooleanExpressionVisitor""" + left_result: T = visit(obj.left, visitor=visitor) + right_result: T = visit(obj.right, visitor=visitor) + return visitor.visit_or(left_result=left_result, right_result=right_result) diff --git a/tests/expressions/test_expressions_base.py b/tests/expressions/test_expressions_base.py index f7fe4b5a45..43fde0ad1c 100644 --- a/tests/expressions/test_expressions_base.py +++ b/tests/expressions/test_expressions_base.py @@ -17,6 +17,7 @@ import uuid from decimal import Decimal +from typing import List import pytest @@ -84,6 +85,65 @@ def __str__(self): return "testexprb" +class TestBooleanExpressionVisitor(base.BooleanExpressionVisitor[List]): + """A test implementation of a BooleanExpressionVisit + + As this visitor visits each node, it appends an element to a `visit_histor` list. This enables testing that a given expression is + visited in an expected order by the `visit` method. + """ + + def __init__(self): + self.visit_history: List = [] + + def visit_true(self) -> List: + self.visit_history.append("TRUE") + return self.visit_history + + def visit_false(self) -> List: + self.visit_history.append("FALSE") + return self.visit_history + + def visit_not(self, child_result: List) -> List: + self.visit_history.append("NOT") + return self.visit_history + + def visit_and(self, left_result: List, right_result: List) -> List: + self.visit_history.append("AND") + return self.visit_history + + def visit_or(self, left_result: List, right_result: List) -> List: + self.visit_history.append("OR") + return self.visit_history + + def visit_unbound_predicate(self, predicate) -> List: + self.visit_history.append("UNBOUND PREDICATE") + return self.visit_history + + def visit_bound_predicate(self, predicate) -> List: + self.visit_history.append("BOUND PREDICATE") + return self.visit_history + + def visit_test_expression_a(self) -> List: + self.visit_history.append("TestExpressionA") + return self.visit_history + + def visit_test_expression_b(self) -> List: + self.visit_history.append("TestExpressionB") + return self.visit_history + + +@base.visit.register(TestExpressionA) +def _(obj: TestExpressionA, visitor: TestBooleanExpressionVisitor) -> List: + """Visit a TestExpressionA with a TestBooleanExpressionVisitor""" + return visitor.visit_test_expression_a() + + +@base.visit.register(TestExpressionB) +def _(obj: TestExpressionB, visitor: TestBooleanExpressionVisitor) -> List: + """Visit a TestExpressionB with a TestBooleanExpressionVisitor""" + return visitor.visit_test_expression_b() + + @pytest.mark.parametrize( "op, rep", [ @@ -257,3 +317,39 @@ def test_bound_reference(table_schema_simple, foo_struct): assert bound_ref1.eval(foo_struct) == "foovalue" assert bound_ref2.eval(foo_struct) == 123 assert bound_ref3.eval(foo_struct) == True + + +def test_boolean_expression_visitor(): + """Test post-order traversal of boolean expression visit method""" + expr = base.And( + base.Or(base.Not(TestExpressionA()), base.Not(TestExpressionB()), TestExpressionA(), TestExpressionB()), + base.Not(TestExpressionA()), + TestExpressionB(), + ) + visitor = TestBooleanExpressionVisitor() + result = base.visit(expr, visitor=visitor) + assert result == [ + "TestExpressionA", + "NOT", + "TestExpressionB", + "NOT", + "OR", + "TestExpressionA", + "OR", + "TestExpressionB", + "OR", + "TestExpressionA", + "NOT", + "AND", + "TestExpressionB", + "AND", + ] + + +def test_boolean_expression_visit_raise_not_implemented_error(): + """Test raise NotImplementedError when visiting an unsupported object type""" + visitor = TestBooleanExpressionVisitor() + with pytest.raises(NotImplementedError) as exc_info: + base.visit("foo", visitor=visitor) + + assert str(exc_info.value) == "Cannot visit unsupported expression: foo" From 9dcbe58b07aff192be534cf9f85ecef947d7b118 Mon Sep 17 00:00:00 2001 From: Dhruv Pratap Date: Fri, 27 May 2022 15:23:07 -0400 Subject: [PATCH 097/642] Python: Add Catalog abstract base class (#3245) (#4706) --- spellcheck-dictionary.txt | 2 + src/iceberg/catalog/base.py | 247 ++++++++++++++++++++++ src/iceberg/exceptions.py | 32 +++ src/iceberg/table/base.py | 34 +++ tests/catalog/__init__.py | 16 ++ tests/catalog/test_base.py | 401 ++++++++++++++++++++++++++++++++++++ tests/conftest.py | 6 + 7 files changed, 738 insertions(+) create mode 100644 src/iceberg/catalog/base.py create mode 100644 src/iceberg/exceptions.py create mode 100644 src/iceberg/table/base.py create mode 100644 tests/catalog/__init__.py create mode 100644 tests/catalog/test_base.py diff --git a/spellcheck-dictionary.txt b/spellcheck-dictionary.txt index 299c83f2f1..5931d618c7 100644 --- a/spellcheck-dictionary.txt +++ b/spellcheck-dictionary.txt @@ -28,6 +28,8 @@ fs func IcebergType io +namespace +namespaces NativeFile NestedField nullability diff --git a/src/iceberg/catalog/base.py b/src/iceberg/catalog/base.py new file mode 100644 index 0000000000..df541fcc6d --- /dev/null +++ b/src/iceberg/catalog/base.py @@ -0,0 +1,247 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +from abc import ABC, abstractmethod +from typing import Dict, List, Optional, Set, Tuple, Union + +from iceberg.schema import Schema +from iceberg.table.base import PartitionSpec, Table + +Identifier = Tuple[str, ...] +Properties = Dict[str, str] + + +class Catalog(ABC): + """Base Catalog for table operations like - create, drop, load, list and others. + + The catalog table APIs accept a table identifier, which is fully classified table name. The identifier can be a string or + tuple of strings. If the identifier is a string, it is split into a tuple on '.'. If it is a tuple, it is used as-is. + + The catalog namespace APIs follow a similar convention wherein they also accept a namespace identifier that can be a string + or tuple of strings. + + Attributes: + name(str): Name of the catalog + properties(Properties): Catalog properties + """ + + def __init__(self, name: str, properties: Properties): + self._name = name + self._properties = properties + + @property + def name(self) -> str: + return self._name + + @property + def properties(self) -> Properties: + return self._properties + + @abstractmethod + def create_table( + self, + identifier: Union[str, Identifier], + schema: Schema, + location: Optional[str] = None, + partition_spec: Optional[PartitionSpec] = None, + properties: Optional[Properties] = None, + ) -> Table: + """Create a table + + Args: + identifier: Table identifier. + schema: Table's schema. + location: Location for the table. Optional Argument. + partition_spec: PartitionSpec for the table. Optional Argument. + properties: Table properties that can be a string based dictionary. Optional Argument. + + Returns: + Table: the created table instance + + Raises: + AlreadyExistsError: If a table with the name already exists + """ + + @abstractmethod + def load_table(self, identifier: Union[str, Identifier]) -> Table: + """Loads the table's metadata and returns the table instance. + + You can also use this method to check for table existence using 'try catalog.table() except TableNotFoundError' + Note: This method doesn't scan data stored in the table. + + Args: + identifier: Table identifier. + + Returns: + Table: the table instance with its metadata + + Raises: + TableNotFoundError: If a table with the name does not exist + """ + + @abstractmethod + def drop_table(self, identifier: Union[str, Identifier]) -> None: + """Drop a table. + + Args: + identifier: Table identifier. + + Raises: + TableNotFoundError: If a table with the name does not exist + """ + + @abstractmethod + def purge_table(self, identifier: Union[str, Identifier]) -> None: + """Drop a table and purge all data and metadata files. + + Args: + identifier: Table identifier. + + Raises: + TableNotFoundError: If a table with the name does not exist + """ + + @abstractmethod + def rename_table(self, from_identifier: Union[str, Identifier], to_identifier: Union[str, Identifier]) -> Table: + """Rename a fully classified table name + + Args: + from_identifier: Existing table identifier. + to_identifier: New table identifier. + + Returns: + Table: the updated table instance with its metadata + + Raises: + TableNotFoundError: If a table with the name does not exist + """ + + @abstractmethod + def create_namespace(self, namespace: Union[str, Identifier], properties: Optional[Properties] = None) -> None: + """Create a namespace in the catalog. + + Args: + namespace: Namespace identifier + properties: A string dictionary of properties for the given namespace + + Raises: + AlreadyExistsError: If a namespace with the given name already exists + """ + + @abstractmethod + def drop_namespace(self, namespace: Union[str, Identifier]) -> None: + """Drop a namespace. + + Args: + namespace: Namespace identifier + + Raises: + NamespaceNotFoundError: If a namespace with the given name does not exist + NamespaceNotEmptyError: If the namespace is not empty + """ + + @abstractmethod + def list_tables(self, namespace: Optional[Union[str, Identifier]] = None) -> List[Identifier]: + """List tables under the given namespace in the catalog. + + If namespace not provided, will list all tables in the catalog. + + Args: + namespace: Namespace identifier to search. + + Returns: + List[Identifier]: list of table identifiers. + + Raises: + NamespaceNotFoundError: If a namespace with the given name does not exist + """ + + @abstractmethod + def list_namespaces(self) -> List[Identifier]: + """List namespaces from the given namespace. If not given, list top-level namespaces from the catalog. + + Returns: + List[Identifier]: a List of namespace identifiers + """ + + @abstractmethod + def load_namespace_properties(self, namespace: Union[str, Identifier]) -> Properties: + """Get properties for a namespace. + + Args: + namespace: Namespace identifier + + Returns: + Properties: Properties for the given namespace + + Raises: + NamespaceNotFoundError: If a namespace with the given name does not exist + """ + + @abstractmethod + def update_namespace_properties( + self, namespace: Union[str, Identifier], removals: Optional[Set[str]] = None, updates: Optional[Properties] = None + ) -> None: + """Removes provided property keys and updates properties for a namespace. + + Args: + namespace: Namespace identifier + removals: Set of property keys that need to be removed. Optional Argument. + updates: Properties to be updated for the given namespace. Optional Argument. + + Raises: + NamespaceNotFoundError: If a namespace with the given name does not exist + ValueError: If removals and updates have overlapping keys. + """ + + @staticmethod + def identifier_to_tuple(identifier: Union[str, Identifier]) -> Identifier: + """Parses an identifier to a tuple. + + If the identifier is a string, it is split into a tuple on '.'. If it is a tuple, it is used as-is. + + Args: + identifier: an identifier, either a string or tuple of strings + + Returns: + Identifier: a tuple of strings + """ + return identifier if isinstance(identifier, tuple) else tuple(str.split(identifier, ".")) + + @staticmethod + def table_name_from(identifier: Union[str, Identifier]) -> str: + """Extracts table name from a table identifier + + Args: + identifier: a table identifier + + Returns: + str: Table name + """ + return Catalog.identifier_to_tuple(identifier)[-1] + + @staticmethod + def namespace_from(identifier: Union[str, Identifier]) -> Identifier: + """Extracts table namespace from a table identifier + + Args: + identifier: a table identifier + + Returns: + Identifier: Namespace identifier + """ + return Catalog.identifier_to_tuple(identifier)[:-1] diff --git a/src/iceberg/exceptions.py b/src/iceberg/exceptions.py new file mode 100644 index 0000000000..3dd68e55d4 --- /dev/null +++ b/src/iceberg/exceptions.py @@ -0,0 +1,32 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + + +class NoSuchTableError(Exception): + """Raised when a referenced table is not found""" + + +class NoSuchNamespaceError(Exception): + """Raised when a referenced name-space is not found""" + + +class NamespaceNotEmptyError(Exception): + """Raised when a name-space being dropped is not empty""" + + +class AlreadyExistsError(Exception): + """Raised when a table or name-space being created already exists in the catalog""" diff --git a/src/iceberg/table/base.py b/src/iceberg/table/base.py new file mode 100644 index 0000000000..023d1f3bbe --- /dev/null +++ b/src/iceberg/table/base.py @@ -0,0 +1,34 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +from __future__ import annotations + +from abc import ABC + + +class Table(ABC): + """Placeholder for Table managed by the Catalog that points to the current Table Metadata. + + To be implemented by https://github.com/apache/iceberg/issues/3227 + """ + + +class PartitionSpec: + """Placeholder for Partition Specification + + To be implemented by https://github.com/apache/iceberg/issues/4631 + """ diff --git a/tests/catalog/__init__.py b/tests/catalog/__init__.py new file mode 100644 index 0000000000..a67d5ea255 --- /dev/null +++ b/tests/catalog/__init__.py @@ -0,0 +1,16 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. diff --git a/tests/catalog/test_base.py b/tests/catalog/test_base.py new file mode 100644 index 0000000000..ecb34ca1c7 --- /dev/null +++ b/tests/catalog/test_base.py @@ -0,0 +1,401 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +from typing import Dict, List, Optional, Set, Union + +import pytest + +from iceberg.catalog.base import Catalog, Identifier, Properties +from iceberg.exceptions import ( + AlreadyExistsError, + NamespaceNotEmptyError, + NoSuchNamespaceError, + NoSuchTableError, +) +from iceberg.schema import Schema +from iceberg.table.base import PartitionSpec, Table + + +class InMemoryCatalog(Catalog): + """An in-memory catalog implementation for testing purposes.""" + + __tables: Dict[Identifier, Table] + __namespaces: Dict[Identifier, Properties] + + def __init__(self, name: str, properties: Properties): + super().__init__(name, properties) + self.__tables = {} + self.__namespaces = {} + + def create_table( + self, + identifier: Union[str, Identifier], + schema: Schema, + location: Optional[str] = None, + partition_spec: Optional[PartitionSpec] = None, + properties: Optional[Properties] = None, + ) -> Table: + + identifier = Catalog.identifier_to_tuple(identifier) + namespace = Catalog.namespace_from(identifier) + + if identifier in self.__tables: + raise AlreadyExistsError(f"Table already exists: {identifier}") + else: + if namespace not in self.__namespaces: + self.__namespaces[namespace] = {} + + table = Table() + table.identifier = identifier + self.__tables[identifier] = table + return table + + def load_table(self, identifier: Union[str, Identifier]) -> Table: + identifier = Catalog.identifier_to_tuple(identifier) + try: + return self.__tables[identifier] + except KeyError as error: + raise NoSuchTableError(f"Table does not exist: {identifier}") from error + + def drop_table(self, identifier: Union[str, Identifier]) -> None: + identifier = Catalog.identifier_to_tuple(identifier) + try: + self.__tables.pop(identifier) + except KeyError as error: + raise NoSuchTableError(f"Table does not exist: {identifier}") from error + + def purge_table(self, identifier: Union[str, Identifier]) -> None: + self.drop_table(identifier) + + def rename_table(self, from_identifier: Union[str, Identifier], to_identifier: Union[str, Identifier]) -> Table: + from_identifier = Catalog.identifier_to_tuple(from_identifier) + try: + table = self.__tables.pop(from_identifier) + except KeyError as error: + raise NoSuchTableError(f"Table does not exist: {from_identifier}") from error + + to_identifier = Catalog.identifier_to_tuple(to_identifier) + to_namespace = Catalog.namespace_from(to_identifier) + if to_namespace not in self.__namespaces: + self.__namespaces[to_namespace] = {} + + table.identifier = to_identifier + self.__tables[to_identifier] = table + return table + + def create_namespace(self, namespace: Union[str, Identifier], properties: Optional[Properties] = None) -> None: + namespace = Catalog.identifier_to_tuple(namespace) + if namespace in self.__namespaces: + raise AlreadyExistsError(f"Namespace already exists: {namespace}") + else: + self.__namespaces[namespace] = properties if properties else {} + + def drop_namespace(self, namespace: Union[str, Identifier]) -> None: + namespace = Catalog.identifier_to_tuple(namespace) + if [table_identifier for table_identifier in self.__tables.keys() if namespace == table_identifier[:-1]]: + raise NamespaceNotEmptyError(f"Namespace is not empty: {namespace}") + try: + self.__namespaces.pop(namespace) + except KeyError as error: + raise NoSuchNamespaceError(f"Namespace does not exist: {namespace}") from error + + def list_tables(self, namespace: Optional[Union[str, Identifier]] = None) -> List[Identifier]: + if namespace: + namespace = Catalog.identifier_to_tuple(namespace) + list_tables = [table_identifier for table_identifier in self.__tables.keys() if namespace == table_identifier[:-1]] + else: + list_tables = list(self.__tables.keys()) + + return list_tables + + def list_namespaces(self) -> List[Identifier]: + return list(self.__namespaces.keys()) + + def load_namespace_properties(self, namespace: Union[str, Identifier]) -> Properties: + namespace = Catalog.identifier_to_tuple(namespace) + try: + return self.__namespaces[namespace] + except KeyError as error: + raise NoSuchNamespaceError(f"Namespace does not exist: {namespace}") from error + + def update_namespace_properties( + self, namespace: Union[str, Identifier], removals: Optional[Set[str]] = None, updates: Optional[Properties] = None + ) -> None: + namespace = Catalog.identifier_to_tuple(namespace) + removals = {} if not removals else removals + updates = [] if not updates else updates + if namespace in self.__namespaces: + for key in removals: + if key in self.__namespaces[namespace]: + del self.__namespaces[namespace][key] + self.__namespaces[namespace].update(updates) + else: + raise NoSuchNamespaceError(f"Namespace does not exist: {namespace}") + + +TEST_TABLE_IDENTIFIER = ("com", "organization", "department", "my_table") +TEST_TABLE_NAMESPACE = ("com", "organization", "department") +TEST_TABLE_NAME = "my_table" +TEST_TABLE_SCHEMA = Schema(schema_id=1) +TEST_TABLE_LOCATION = "protocol://some/location" +TEST_TABLE_PARTITION_SPEC = PartitionSpec() +TEST_TABLE_PROPERTIES = {"key1": "value1", "key2": "value2"} +NO_SUCH_TABLE_ERROR = "Table does not exist: \\('com', 'organization', 'department', 'my_table'\\)" +TABLE_ALREADY_EXISTS_ERROR = "Table already exists: \\('com', 'organization', 'department', 'my_table'\\)" +NAMESPACE_ALREADY_EXISTS_ERROR = "Namespace already exists: \\('com', 'organization', 'department'\\)" +NO_SUCH_NAMESPACE_ERROR = "Namespace does not exist: \\('com', 'organization', 'department'\\)" +NAMESPACE_NOT_EMPTY_ERROR = "Namespace is not empty: \\('com', 'organization', 'department'\\)" + + +def given_catalog_has_a_table(catalog: InMemoryCatalog) -> Table: + return catalog.create_table( + identifier=TEST_TABLE_IDENTIFIER, + schema=TEST_TABLE_SCHEMA, + location=TEST_TABLE_LOCATION, + partition_spec=TEST_TABLE_PARTITION_SPEC, + properties=TEST_TABLE_PROPERTIES, + ) + + +def test_namespace_from_tuple(): + # Given + identifier = ("com", "organization", "department", "my_table") + # When + namespace_from = Catalog.namespace_from(identifier) + # Then + assert namespace_from == ("com", "organization", "department") + + +def test_namespace_from_str(): + # Given + identifier = "com.organization.department.my_table" + # When + namespace_from = Catalog.namespace_from(identifier) + # Then + assert namespace_from == ("com", "organization", "department") + + +def test_name_from_tuple(): + # Given + identifier = ("com", "organization", "department", "my_table") + # When + name_from = Catalog.table_name_from(identifier) + # Then + assert name_from == "my_table" + + +def test_name_from_str(): + # Given + identifier = "com.organization.department.my_table" + # When + name_from = Catalog.table_name_from(identifier) + # Then + assert name_from == "my_table" + + +def test_create_table(catalog: InMemoryCatalog): + table = catalog.create_table( + identifier=TEST_TABLE_IDENTIFIER, + schema=TEST_TABLE_SCHEMA, + location=TEST_TABLE_LOCATION, + partition_spec=TEST_TABLE_PARTITION_SPEC, + properties=TEST_TABLE_PROPERTIES, + ) + assert catalog.load_table(TEST_TABLE_IDENTIFIER) == table + + +def test_create_table_raises_error_when_table_already_exists(catalog: InMemoryCatalog): + # Given + given_catalog_has_a_table(catalog) + # When + with pytest.raises(AlreadyExistsError, match=TABLE_ALREADY_EXISTS_ERROR): + catalog.create_table( + identifier=TEST_TABLE_IDENTIFIER, + schema=TEST_TABLE_SCHEMA, + ) + + +def test_load_table(catalog: InMemoryCatalog): + # Given + given_table = given_catalog_has_a_table(catalog) + # When + table = catalog.load_table(TEST_TABLE_IDENTIFIER) + # Then + assert table == given_table + + +def test_table_raises_error_on_table_not_found(catalog: InMemoryCatalog): + with pytest.raises(NoSuchTableError, match=NO_SUCH_TABLE_ERROR): + catalog.load_table(TEST_TABLE_IDENTIFIER) + + +def test_drop_table(catalog: InMemoryCatalog): + # Given + given_catalog_has_a_table(catalog) + # When + catalog.drop_table(TEST_TABLE_IDENTIFIER) + # Then + with pytest.raises(NoSuchTableError, match=NO_SUCH_TABLE_ERROR): + catalog.load_table(TEST_TABLE_IDENTIFIER) + + +def test_drop_table_that_does_not_exist_raise_error(catalog: InMemoryCatalog): + with pytest.raises(NoSuchTableError, match=NO_SUCH_TABLE_ERROR): + catalog.load_table(TEST_TABLE_IDENTIFIER) + + +def test_purge_table(catalog: InMemoryCatalog): + # Given + given_catalog_has_a_table(catalog) + # When + catalog.purge_table(TEST_TABLE_IDENTIFIER) + # Then + with pytest.raises(NoSuchTableError, match=NO_SUCH_TABLE_ERROR): + catalog.load_table(TEST_TABLE_IDENTIFIER) + + +def test_rename_table(catalog: InMemoryCatalog): + # Given + given_catalog_has_a_table(catalog) + + # When + new_table = "new.namespace.new_table" + table = catalog.rename_table(TEST_TABLE_IDENTIFIER, new_table) + + # Then + assert table.identifier == Catalog.identifier_to_tuple(new_table) + + # And + table = catalog.load_table(new_table) + assert table.identifier == Catalog.identifier_to_tuple(new_table) + + # And + assert ("new", "namespace") in catalog.list_namespaces() + + # And + with pytest.raises(NoSuchTableError, match=NO_SUCH_TABLE_ERROR): + catalog.load_table(TEST_TABLE_IDENTIFIER) + + +def test_create_namespace(catalog: InMemoryCatalog): + # When + catalog.create_namespace(TEST_TABLE_NAMESPACE, TEST_TABLE_PROPERTIES) + + # Then + assert TEST_TABLE_NAMESPACE in catalog.list_namespaces() + assert TEST_TABLE_PROPERTIES == catalog.load_namespace_properties(TEST_TABLE_NAMESPACE) + + +def test_create_namespace_raises_error_on_existing_namespace(catalog: InMemoryCatalog): + # Given + catalog.create_namespace(TEST_TABLE_NAMESPACE, TEST_TABLE_PROPERTIES) + # When + with pytest.raises(AlreadyExistsError, match=NAMESPACE_ALREADY_EXISTS_ERROR): + catalog.create_namespace(TEST_TABLE_NAMESPACE, TEST_TABLE_PROPERTIES) + + +def test_get_namespace_metadata_raises_error_when_namespace_does_not_exist(catalog: InMemoryCatalog): + with pytest.raises(NoSuchNamespaceError, match=NO_SUCH_NAMESPACE_ERROR): + catalog.load_namespace_properties(TEST_TABLE_NAMESPACE) + + +def test_list_namespaces(catalog: InMemoryCatalog): + # Given + catalog.create_namespace(TEST_TABLE_NAMESPACE, TEST_TABLE_PROPERTIES) + # When + namespaces = catalog.list_namespaces() + # Then + assert TEST_TABLE_NAMESPACE in namespaces + + +def test_drop_namespace(catalog: InMemoryCatalog): + # Given + catalog.create_namespace(TEST_TABLE_NAMESPACE, TEST_TABLE_PROPERTIES) + # When + catalog.drop_namespace(TEST_TABLE_NAMESPACE) + # Then + assert TEST_TABLE_NAMESPACE not in catalog.list_namespaces() + + +def test_drop_namespace_raises_error_when_namespace_does_not_exist(catalog: InMemoryCatalog): + with pytest.raises(NoSuchNamespaceError, match=NO_SUCH_NAMESPACE_ERROR): + catalog.drop_namespace(TEST_TABLE_NAMESPACE) + + +def test_drop_namespace_raises_error_when_namespace_not_empty(catalog: InMemoryCatalog): + # Given + given_catalog_has_a_table(catalog) + # When + with pytest.raises(NamespaceNotEmptyError, match=NAMESPACE_NOT_EMPTY_ERROR): + catalog.drop_namespace(TEST_TABLE_NAMESPACE) + + +def test_list_tables(catalog: InMemoryCatalog): + # Given + given_catalog_has_a_table(catalog) + # When + tables = catalog.list_tables() + # Then + assert tables + assert TEST_TABLE_IDENTIFIER in tables + + +def test_list_tables_under_a_namespace(catalog: InMemoryCatalog): + # Given + given_catalog_has_a_table(catalog) + new_namespace = ("new", "namespace") + catalog.create_namespace(new_namespace) + # When + all_tables = catalog.list_tables() + new_namespace_tables = catalog.list_tables(new_namespace) + # Then + assert all_tables + assert TEST_TABLE_IDENTIFIER in all_tables + assert new_namespace_tables == [] + + +def test_update_namespace_metadata(catalog: InMemoryCatalog): + # Given + catalog.create_namespace(TEST_TABLE_NAMESPACE, TEST_TABLE_PROPERTIES) + + # When + new_metadata = {"key3": "value3", "key4": "value4"} + catalog.update_namespace_properties(TEST_TABLE_NAMESPACE, updates=new_metadata) + + # Then + assert TEST_TABLE_NAMESPACE in catalog.list_namespaces() + assert new_metadata.items() <= catalog.load_namespace_properties(TEST_TABLE_NAMESPACE).items() + + +def test_update_namespace_metadata_removals(catalog: InMemoryCatalog): + # Given + catalog.create_namespace(TEST_TABLE_NAMESPACE, TEST_TABLE_PROPERTIES) + + # When + new_metadata = {"key3": "value3", "key4": "value4"} + remove_metadata = {"key1"} + catalog.update_namespace_properties(TEST_TABLE_NAMESPACE, remove_metadata, new_metadata) + + # Then + assert TEST_TABLE_NAMESPACE in catalog.list_namespaces() + assert new_metadata.items() <= catalog.load_namespace_properties(TEST_TABLE_NAMESPACE).items() + assert remove_metadata.isdisjoint(catalog.load_namespace_properties(TEST_TABLE_NAMESPACE).keys()) + + +def test_update_namespace_metadata_raises_error_when_namespace_does_not_exist(catalog: InMemoryCatalog): + with pytest.raises(NoSuchNamespaceError, match=NO_SUCH_NAMESPACE_ERROR): + catalog.update_namespace_properties(TEST_TABLE_NAMESPACE, updates=TEST_TABLE_PROPERTIES) diff --git a/tests/conftest.py b/tests/conftest.py index 5f9a13a47b..2fd09a8cd3 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -30,6 +30,7 @@ StringType, StructType, ) +from tests.catalog.test_base import InMemoryCatalog class FooStruct: @@ -112,3 +113,8 @@ def table_schema_nested(): @pytest.fixture(scope="session", autouse=True) def foo_struct(): return FooStruct() + + +@pytest.fixture +def catalog() -> InMemoryCatalog: + return InMemoryCatalog("test.in.memory.catalog", {"test.key": "test.value"}) From 242139392728bee5508a3a92c582f4d308295b81 Mon Sep 17 00:00:00 2001 From: Fokko Driesprong Date: Sun, 29 May 2022 23:39:05 +0200 Subject: [PATCH 098/642] Python: Convert Avro to Iceberg schema (#4742) --- spellcheck-dictionary.txt | 4 + src/iceberg/schema.py | 23 +- src/iceberg/types.py | 4 +- src/iceberg/utils/schema_conversion.py | 454 +++++++++++++++++++++++++ tests/conftest.py | 187 +++++++++- tests/utils/test_schema_conversion.py | 231 +++++++++++++ 6 files changed, 896 insertions(+), 7 deletions(-) create mode 100644 src/iceberg/utils/schema_conversion.py create mode 100644 tests/utils/test_schema_conversion.py diff --git a/spellcheck-dictionary.txt b/spellcheck-dictionary.txt index 5931d618c7..3d7092938f 100644 --- a/spellcheck-dictionary.txt +++ b/spellcheck-dictionary.txt @@ -13,6 +13,8 @@ accessor accessors Args +Avro +Nestedfield ASF BD bool @@ -31,6 +33,8 @@ io namespace namespaces NativeFile +NaN +nan NestedField nullability pragma diff --git a/src/iceberg/schema.py b/src/iceberg/schema.py index 15622fa7c7..0ba7313ff0 100644 --- a/src/iceberg/schema.py +++ b/src/iceberg/schema.py @@ -21,7 +21,7 @@ from abc import ABC, abstractmethod from dataclasses import dataclass from functools import singledispatch -from typing import Any, Dict, Generic, Iterable, List, Optional, TypeVar +from typing import Any, Dict, Generic, List, Optional, Tuple, TypeVar from iceberg.files import StructProtocol from iceberg.types import ( @@ -44,8 +44,8 @@ class Schema: >>> from iceberg import types """ - def __init__(self, *columns: Iterable[NestedField], schema_id: int, identifier_field_ids: Optional[List[int]] = None): - self._struct = StructType(*columns) # type: ignore + def __init__(self, *columns: NestedField, schema_id: int, identifier_field_ids: Optional[List[int]] = None): + self._struct = StructType(*columns) self._schema_id = schema_id self._identifier_field_ids = identifier_field_ids or [] self._name_to_id: Dict[str, int] = index_by_name(self) @@ -62,8 +62,23 @@ def __repr__(self): f"Schema(fields={repr(self.columns)}, schema_id={self.schema_id}, identifier_field_ids={self.identifier_field_ids})" ) + def __eq__(self, other) -> bool: + if not other: + return False + + if not isinstance(other, Schema): + return False + + if len(self.columns) != len(other.columns): + return False + + identifier_field_ids_is_equal = self.identifier_field_ids == other.identifier_field_ids + schema_is_equal = all([lhs == rhs for lhs, rhs in zip(self.columns, other.columns)]) + + return identifier_field_ids_is_equal and schema_is_equal + @property - def columns(self) -> Iterable[NestedField]: + def columns(self) -> Tuple[NestedField, ...]: """A list of the top-level fields in the underlying struct""" return self._struct.fields diff --git a/src/iceberg/types.py b/src/iceberg/types.py index 1b206a1159..07e19e5f8c 100644 --- a/src/iceberg/types.py +++ b/src/iceberg/types.py @@ -31,7 +31,7 @@ """ from dataclasses import dataclass, field from functools import cached_property -from typing import ClassVar, Dict, List, Optional, Tuple +from typing import ClassVar, Dict, Optional, Tuple class Singleton: @@ -193,7 +193,7 @@ class StructType(IcebergType): 'struct<1: required_field: optional string, 2: optional_field: optional int>' """ - fields: List[NestedField] = field() + fields: Tuple[NestedField] = field() _instances: ClassVar[Dict[Tuple[NestedField, ...], "StructType"]] = {} diff --git a/src/iceberg/utils/schema_conversion.py b/src/iceberg/utils/schema_conversion.py new file mode 100644 index 0000000000..11571f27bd --- /dev/null +++ b/src/iceberg/utils/schema_conversion.py @@ -0,0 +1,454 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +"""Utility class for converting between Avro and Iceberg schemas + +""" +from __future__ import annotations + +import logging +from typing import Any, Dict, List, Tuple + +from iceberg.schema import Schema +from iceberg.types import ( + BinaryType, + BooleanType, + DateType, + DecimalType, + DoubleType, + FixedType, + FloatType, + IcebergType, + IntegerType, + ListType, + LongType, + MapType, + NestedField, + PrimitiveType, + StringType, + StructType, + TimestampType, + TimeType, + UUIDType, +) + +logger = logging.getLogger(__name__) + +PRIMITIVE_FIELD_TYPE_MAPPING: Dict[str, PrimitiveType] = { + "boolean": BooleanType(), + "bytes": BinaryType(), + "double": DoubleType(), + "float": FloatType(), + "int": IntegerType(), + "long": LongType(), + "string": StringType(), + "enum": StringType(), +} + +LOGICAL_FIELD_TYPE_MAPPING: Dict[Tuple[str, str], PrimitiveType] = { + ("date", "int"): DateType(), + ("time-millis", "int"): TimeType(), + ("timestamp-millis", "long"): TimestampType(), + ("time-micros", "int"): TimeType(), + ("timestamp-micros", "long"): TimestampType(), + ("uuid", "string"): UUIDType(), +} + + +class AvroSchemaConversion: + def avro_to_iceberg(self, avro_schema: Dict[str, Any]) -> Schema: + """Converts an Apache Avro into an Apache Iceberg schema equivalent + + This expects to have field id's to be encoded in the Avro schema:: + + { + "type": "record", + "name": "manifest_file", + "fields": [ + {"name": "manifest_path", "type": "string", "doc": "Location URI with FS scheme", "field-id": 500}, + {"name": "manifest_length", "type": "long", "doc": "Total file size in bytes", "field-id": 501} + ] + } + + Example: + This converts an Avro schema into an Iceberg schema: + + >>> avro_schema = AvroSchemaConversion().avro_to_iceberg({ + ... "type": "record", + ... "name": "manifest_file", + ... "fields": [ + ... {"name": "manifest_path", "type": "string", "doc": "Location URI with FS scheme", "field-id": 500}, + ... {"name": "manifest_length", "type": "long", "doc": "Total file size in bytes", "field-id": 501} + ... ] + ... }) + >>> iceberg_schema = Schema( + ... NestedField( + ... field_id=500, name="manifest_path", field_type=StringType(), is_optional=False, doc="Location URI with FS scheme" + ... ), + ... NestedField( + ... field_id=501, name="manifest_length", field_type=LongType(), is_optional=False, doc="Total file size in bytes" + ... ), + ... schema_id=1 + ... ) + >>> avro_schema == iceberg_schema + True + + Args: + avro_schema (Dict[str, Any]): The JSON decoded Avro schema + + Returns: + Equivalent Iceberg schema + """ + return Schema(*[self._convert_field(field) for field in avro_schema["fields"]], schema_id=1) + + def _resolve_union(self, type_union: Dict | List | str) -> Tuple[str | Dict[str, Any], bool]: + """ + Converts Unions into their type and resolves if the field is optional + + Examples: + >>> AvroSchemaConversion()._resolve_union('str') + ('str', False) + >>> AvroSchemaConversion()._resolve_union(['null', 'str']) + ('str', True) + >>> AvroSchemaConversion()._resolve_union([{'type': 'str'}]) + ({'type': 'str'}, False) + >>> AvroSchemaConversion()._resolve_union(['null', {'type': 'str'}]) + ({'type': 'str'}, True) + + Args: + type_union: The field, can be a string 'str', list ['null', 'str'], or dict {"type": 'str'} + + Returns: + A tuple containing the type and nullability + + Raises: + TypeError: In the case non-optional union types are encountered + """ + avro_types: Dict | List + if isinstance(type_union, str): + # It is a primitive and required + return type_union, False + elif isinstance(type_union, dict): + # It is a context and required + return type_union, False + else: + avro_types = type_union + + is_optional = "null" in avro_types + + if len(avro_types) > 2: + raise TypeError("Non-optional types aren't part of the Iceberg specification") + + # Filter the null value and return the type + return list(filter(lambda t: t != "null", avro_types))[0], is_optional + + def _convert_schema(self, avro_type: str | Dict[str, Any]) -> IcebergType: + """ + Resolves the Avro type + + Args: + avro_type: The Avro type, can be simple or complex + + Returns: + The equivalent IcebergType + + Raises: + ValueError: When there are unknown types + """ + if isinstance(avro_type, str): + return PRIMITIVE_FIELD_TYPE_MAPPING[avro_type] + elif isinstance(avro_type, dict): + if "logicalType" in avro_type: + return self._convert_logical_type(avro_type) + else: + # Resolve potential nested types + while "type" in avro_type and isinstance(avro_type["type"], dict): + avro_type = avro_type["type"] + type_identifier = avro_type["type"] + if type_identifier == "record": + return self._convert_record_type(avro_type) + elif type_identifier == "array": + return self._convert_array_type(avro_type) + elif type_identifier == "map": + return self._convert_map_type(avro_type) + elif type_identifier == "fixed": + return self._convert_fixed_type(avro_type) + elif isinstance(type_identifier, str): + return PRIMITIVE_FIELD_TYPE_MAPPING[type_identifier] + else: + raise ValueError(f"Unknown type: {avro_type}") + else: + raise ValueError(f"Unknown type: {avro_type}") + + def _convert_field(self, field: Dict[str, Any]) -> NestedField: + """ + Converts an Avro field into an Iceberg equivalent field + Args: + field: The Avro field + + Returns: + The Iceberg equivalent field + """ + if "field-id" not in field: + raise ValueError(f"Cannot convert field, missing field-id: {field}") + + plain_type, is_optional = self._resolve_union(field["type"]) + + return NestedField( + field_id=field["field-id"], + name=field["name"], + field_type=self._convert_schema(plain_type), + is_optional=is_optional, + doc=field.get("doc"), + ) + + def _convert_record_type(self, record_type: Dict[str, Any]) -> StructType: + """ + Converts the fields from a record into an Iceberg struct + + Examples: + >>> from iceberg.utils.schema_conversion import AvroSchemaConversion + >>> record_type = { + ... "type": "record", + ... "name": "r508", + ... "fields": [{ + ... "name": "contains_null", + ... "type": "boolean", + ... "doc": "True if any file has a null partition value", + ... "field-id": 509, + ... }, { + ... "name": "contains_nan", + ... "type": ["null", "boolean"], + ... "doc": "True if any file has a nan partition value", + ... "default": None, + ... "field-id": 518, + ... }], + ... } + >>> actual = AvroSchemaConversion()._convert_record_type(record_type) + >>> expected = StructType( + ... fields=( + ... NestedField( + ... field_id=509, + ... name="contains_null", + ... field_type=BooleanType(), + ... is_optional=False, + ... doc="True if any file has a null partition value", + ... ), + ... NestedField( + ... field_id=518, + ... name="contains_nan", + ... field_type=BooleanType(), + ... is_optional=True, + ... doc="True if any file has a nan partition value", + ... ), + ... ) + ... ) + >>> expected == actual + True + + Args: + record_type: The record type itself + + Returns: + """ + if record_type["type"] != "record": + raise ValueError(f"Expected type, got: {record_type}") + + return StructType(*[self._convert_field(field) for field in record_type["fields"]]) + + def _convert_array_type(self, array_type: Dict[str, Any]) -> ListType: + if "element-id" not in array_type: + raise ValueError(f"Cannot convert array-type, missing element-id: {array_type}") + + plain_type, element_is_optional = self._resolve_union(array_type["items"]) + + return ListType( + element_id=array_type["element-id"], + element_type=self._convert_schema(plain_type), + element_is_optional=element_is_optional, + ) + + def _convert_map_type(self, map_type: Dict[str, Any]) -> MapType: + """ + Args: + map_type: The dict that describes the Avro map type + + Examples: + >>> from iceberg.utils.schema_conversion import AvroSchemaConversion + >>> avro_field = { + ... "type": "map", + ... "values": ["long", "null"], + ... "key-id": 101, + ... "value-id": 102, + ... } + >>> actual = AvroSchemaConversion()._convert_map_type(avro_field) + >>> expected = MapType( + ... key_id=101, + ... key_type=StringType(), + ... value_id=102, + ... value_type=LongType(), + ... value_is_optional=True + ... ) + >>> actual == expected + True + + Returns: A MapType + """ + value_type, value_is_optional = self._resolve_union(map_type["values"]) + return MapType( + key_id=map_type["key-id"], + # Avro only supports string keys + key_type=StringType(), + value_id=map_type["value-id"], + value_type=self._convert_schema(value_type), + value_is_optional=value_is_optional, + ) + + def _convert_logical_type(self, avro_logical_type: Dict[str, Any]) -> IcebergType: + """ + Convert a schema with a logical type annotation. For the decimal and map + we need to fetch more keys from the dict, and for the simple ones we can just + look it up in the mapping. + + Examples: + >>> from iceberg.utils.schema_conversion import AvroSchemaConversion + >>> avro_logical_type = { + ... "type": "int", + ... "logicalType": "date" + ... } + >>> actual = AvroSchemaConversion()._convert_logical_type(avro_logical_type) + >>> actual == DateType() + True + + Args: + avro_logical_type: The logical type + + Returns: + The converted logical type + + Raises: + ValueError: When the logical type is unknown + """ + logical_type = avro_logical_type["logicalType"] + physical_type = avro_logical_type["type"] + if logical_type == "decimal": + return self._convert_logical_decimal_type(avro_logical_type) + elif logical_type == "map": + return self._convert_logical_map_type(avro_logical_type) + elif (logical_type, physical_type) in LOGICAL_FIELD_TYPE_MAPPING: + return LOGICAL_FIELD_TYPE_MAPPING[(logical_type, physical_type)] + else: + raise ValueError(f"Unknown logical/physical type combination: {avro_logical_type}") + + def _convert_logical_decimal_type(self, avro_type: Dict[str, Any]) -> DecimalType: + """ + Args: + avro_type: The Avro type + + Examples: + >>> from iceberg.utils.schema_conversion import AvroSchemaConversion + >>> avro_decimal_type = { + ... "type": "bytes", + ... "logicalType": "decimal", + ... "precision": 19, + ... "scale": 25 + ... } + >>> actual = AvroSchemaConversion()._convert_logical_decimal_type(avro_decimal_type) + >>> expected = DecimalType( + ... precision=19, + ... scale=25 + ... ) + >>> actual == expected + True + + Returns: + A Iceberg DecimalType + """ + return DecimalType(precision=avro_type["precision"], scale=avro_type["scale"]) + + def _convert_logical_map_type(self, avro_type: Dict[str, Any]) -> MapType: + """ + In the case where a map hasn't a key as a type you can use a logical map to + still encode this in Avro + + Args: + avro_type: The Avro Type + + Examples: + >>> from iceberg.utils.schema_conversion import AvroSchemaConversion + >>> avro_type = { + ... "type": "array", + ... "logicalType": "map", + ... "items": { + ... "type": "record", + ... "name": "k101_v102", + ... "fields": [ + ... {"name": "key", "type": "int", "field-id": 101}, + ... {"name": "value", "type": "string", "field-id": 102}, + ... ], + ... }, + ... } + >>> actual = AvroSchemaConversion()._convert_logical_map_type(avro_type) + >>> expected = MapType( + ... key_id=101, + ... key_type=IntegerType(), + ... value_id=102, + ... value_type=StringType(), + ... value_is_optional=False + ... ) + >>> actual == expected + True + + .. _Apache Iceberg specification: + https://iceberg.apache.org/spec/#appendix-a-format-specific-requirements + + Returns: + The logical map + """ + fields = avro_type["items"]["fields"] + if len(fields) != 2: + raise ValueError(f'Invalid key-value pair schema: {avro_type["items"]}') + key = self._convert_field(list(filter(lambda f: f["name"] == "key", fields))[0]) + value = self._convert_field(list(filter(lambda f: f["name"] == "value", fields))[0]) + return MapType( + key_id=key.field_id, + key_type=key.field_type, + value_id=value.field_id, + value_type=value.field_type, + value_is_optional=value.is_optional, + ) + + def _convert_fixed_type(self, avro_type: Dict[str, Any]) -> FixedType: + """ + https://avro.apache.org/docs/current/spec.html#Fixed + + Args: + avro_type: The Avro Type + + Examples: + >>> from iceberg.utils.schema_conversion import AvroSchemaConversion + >>> avro_fixed_type = { + ... "name": "md5", + ... "type": "fixed", + ... "size": 16 + ... } + >>> FixedType(length=16) == AvroSchemaConversion()._convert_fixed_type(avro_fixed_type) + True + + Returns: + An Iceberg equivalent fixed type + """ + return FixedType(length=avro_type["size"]) diff --git a/tests/conftest.py b/tests/conftest.py index 2fd09a8cd3..3457c4cb6f 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -15,7 +15,7 @@ # specific language governing permissions and limitations # under the License. -from typing import Any +from typing import Any, Dict import pytest @@ -115,6 +115,191 @@ def foo_struct(): return FooStruct() +@pytest.fixture(scope="session") +def manifest_schema() -> Dict[str, Any]: + return { + "type": "record", + "name": "manifest_file", + "fields": [ + {"name": "manifest_path", "type": "string", "doc": "Location URI with FS scheme", "field-id": 500}, + {"name": "manifest_length", "type": "long", "doc": "Total file size in bytes", "field-id": 501}, + {"name": "partition_spec_id", "type": "int", "doc": "Spec ID used to write", "field-id": 502}, + { + "name": "added_snapshot_id", + "type": ["null", "long"], + "doc": "Snapshot ID that added the manifest", + "default": None, + "field-id": 503, + }, + { + "name": "added_data_files_count", + "type": ["null", "int"], + "doc": "Added entry count", + "default": None, + "field-id": 504, + }, + { + "name": "existing_data_files_count", + "type": ["null", "int"], + "doc": "Existing entry count", + "default": None, + "field-id": 505, + }, + { + "name": "deleted_data_files_count", + "type": ["null", "int"], + "doc": "Deleted entry count", + "default": None, + "field-id": 506, + }, + { + "name": "partitions", + "type": [ + "null", + { + "type": "array", + "items": { + "type": "record", + "name": "r508", + "fields": [ + { + "name": "contains_null", + "type": "boolean", + "doc": "True if any file has a null partition value", + "field-id": 509, + }, + { + "name": "contains_nan", + "type": ["null", "boolean"], + "doc": "True if any file has a nan partition value", + "default": None, + "field-id": 518, + }, + { + "name": "lower_bound", + "type": ["null", "bytes"], + "doc": "Partition lower bound for all files", + "default": None, + "field-id": 510, + }, + { + "name": "upper_bound", + "type": ["null", "bytes"], + "doc": "Partition upper bound for all files", + "default": None, + "field-id": 511, + }, + ], + }, + "element-id": 508, + }, + ], + "doc": "Summary for each partition", + "default": None, + "field-id": 507, + }, + {"name": "added_rows_count", "type": ["null", "long"], "doc": "Added rows count", "default": None, "field-id": 512}, + { + "name": "existing_rows_count", + "type": ["null", "long"], + "doc": "Existing rows count", + "default": None, + "field-id": 513, + }, + { + "name": "deleted_rows_count", + "type": ["null", "long"], + "doc": "Deleted rows count", + "default": None, + "field-id": 514, + }, + ], + } + + +@pytest.fixture(scope="session") +def all_avro_types() -> Dict[str, Any]: + return { + "type": "record", + "name": "all_avro_types", + "fields": [ + {"name": "primitive_string", "type": "string", "field-id": 100}, + {"name": "primitive_int", "type": "int", "field-id": 200}, + {"name": "primitive_long", "type": "long", "field-id": 300}, + {"name": "primitive_float", "type": "float", "field-id": 400}, + {"name": "primitive_double", "type": "double", "field-id": 500}, + {"name": "primitive_bytes", "type": "bytes", "field-id": 600}, + { + "type": "record", + "name": "Person", + "fields": [ + {"name": "name", "type": "string", "field-id": 701}, + {"name": "age", "type": "long", "field-id": 702}, + {"name": "gender", "type": ["string", "null"], "field-id": 703}, + ], + "field-id": 700, + }, + { + "name": "array_with_string", + "type": { + "type": "array", + "items": "string", + "default": [], + "element-id": 801, + }, + "field-id": 800, + }, + { + "name": "array_with_optional_string", + "type": [ + "null", + { + "type": "array", + "items": ["string", "null"], + "default": [], + "element-id": 901, + }, + ], + "field-id": 900, + }, + { + "name": "array_with_optional_record", + "type": [ + "null", + { + "type": "array", + "items": [ + "null", + { + "type": "record", + "name": "person", + "fields": [ + {"name": "name", "type": "string", "field-id": 1002}, + {"name": "age", "type": "long", "field-id": 1003}, + {"name": "gender", "type": ["string", "null"], "field-id": 1004}, + ], + }, + ], + "element-id": 1001, + }, + ], + "field-id": 1000, + }, + { + "name": "map_with_longs", + "type": { + "type": "map", + "values": "long", + "default": {}, + "key-id": 1101, + "value-id": 1102, + }, + "field-id": 1000, + }, + ], + } + + @pytest.fixture def catalog() -> InMemoryCatalog: return InMemoryCatalog("test.in.memory.catalog", {"test.key": "test.value"}) diff --git a/tests/utils/test_schema_conversion.py b/tests/utils/test_schema_conversion.py new file mode 100644 index 0000000000..4dcc0ea0bb --- /dev/null +++ b/tests/utils/test_schema_conversion.py @@ -0,0 +1,231 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +from iceberg.schema import Schema +from iceberg.types import ( + BinaryType, + BooleanType, + IntegerType, + ListType, + LongType, + NestedField, + StringType, + StructType, +) +from iceberg.utils.schema_conversion import AvroSchemaConversion + + +def test_iceberg_to_avro(manifest_schema): + iceberg_schema = AvroSchemaConversion().avro_to_iceberg(manifest_schema) + expected_iceberg_schema = Schema( + NestedField( + field_id=500, name="manifest_path", field_type=StringType(), is_optional=False, doc="Location URI with FS scheme" + ), + NestedField( + field_id=501, name="manifest_length", field_type=LongType(), is_optional=False, doc="Total file size in bytes" + ), + NestedField( + field_id=502, name="partition_spec_id", field_type=IntegerType(), is_optional=False, doc="Spec ID used to write" + ), + NestedField( + field_id=503, + name="added_snapshot_id", + field_type=LongType(), + is_optional=True, + doc="Snapshot ID that added the manifest", + ), + NestedField( + field_id=504, name="added_data_files_count", field_type=IntegerType(), is_optional=True, doc="Added entry count" + ), + NestedField( + field_id=505, name="existing_data_files_count", field_type=IntegerType(), is_optional=True, doc="Existing entry count" + ), + NestedField( + field_id=506, name="deleted_data_files_count", field_type=IntegerType(), is_optional=True, doc="Deleted entry count" + ), + NestedField( + field_id=507, + name="partitions", + field_type=ListType( + element_id=508, + element_type=StructType( + fields=( + NestedField( + field_id=509, + name="contains_null", + field_type=BooleanType(), + is_optional=False, + doc="True if any file has a null partition value", + ), + NestedField( + field_id=518, + name="contains_nan", + field_type=BooleanType(), + is_optional=True, + doc="True if any file has a nan partition value", + ), + NestedField( + field_id=510, + name="lower_bound", + field_type=BinaryType(), + is_optional=True, + doc="Partition lower bound for all files", + ), + NestedField( + field_id=511, + name="upper_bound", + field_type=BinaryType(), + is_optional=True, + doc="Partition upper bound for all files", + ), + ) + ), + element_is_optional=False, + ), + is_optional=True, + doc="Summary for each partition", + ), + NestedField(field_id=512, name="added_rows_count", field_type=LongType(), is_optional=True, doc="Added rows count"), + NestedField(field_id=513, name="existing_rows_count", field_type=LongType(), is_optional=True, doc="Existing rows count"), + NestedField(field_id=514, name="deleted_rows_count", field_type=LongType(), is_optional=True, doc="Deleted rows count"), + schema_id=1, + identifier_field_ids=[], + ) + assert iceberg_schema == expected_iceberg_schema + + +def test_avro_list_required_primitive(): + avro_schema = { + "type": "record", + "name": "avro_schema", + "fields": [ + { + "name": "array_with_string", + "type": { + "type": "array", + "items": "string", + "default": [], + "element-id": 101, + }, + "field-id": 100, + }, + ], + } + + expected_iceberg_schema = Schema( + NestedField( + field_id=100, + name="array_with_string", + field_type=ListType(element_id=101, element_type=StringType(), element_is_optional=False), + is_optional=False, + ), + schema_id=1, + ) + + iceberg_schema = AvroSchemaConversion().avro_to_iceberg(avro_schema) + + assert expected_iceberg_schema == iceberg_schema + + +def test_avro_list_wrapped_primitive(): + avro_schema = { + "type": "record", + "name": "avro_schema", + "fields": [ + { + "name": "array_with_string", + "type": { + "type": "array", + "items": {"type": "string"}, + "default": [], + "element-id": 101, + }, + "field-id": 100, + }, + ], + } + + expected_iceberg_schema = Schema( + NestedField( + field_id=100, + name="array_with_string", + field_type=ListType(element_id=101, element_type=StringType(), element_is_optional=False), + is_optional=False, + ), + schema_id=1, + ) + + iceberg_schema = AvroSchemaConversion().avro_to_iceberg(avro_schema) + + assert expected_iceberg_schema == iceberg_schema + + +def test_avro_list_required_record(): + avro_schema = { + "type": "record", + "name": "avro_schema", + "fields": [ + { + "name": "array_with_record", + "type": { + "type": "array", + "items": { + "type": "record", + "name": "r101", + "fields": [ + { + "name": "contains_null", + "type": "boolean", + "field-id": 102, + }, + { + "name": "contains_nan", + "type": ["null", "boolean"], + "field-id": 103, + }, + ], + }, + "element-id": 101, + }, + "field-id": 100, + } + ], + } + + expected_iceberg_schema = Schema( + NestedField( + field_id=100, + name="array_with_record", + field_type=ListType( + element_id=101, + element_type=StructType( + fields=( + NestedField(field_id=102, name="contains_null", field_type=BooleanType(), is_optional=False), + NestedField(field_id=103, name="contains_nan", field_type=BooleanType(), is_optional=True), + ) + ), + element_is_optional=False, + ), + is_optional=False, + ), + schema_id=1, + identifier_field_ids=[], + ) + + iceberg_schema = AvroSchemaConversion().avro_to_iceberg(avro_schema) + + assert expected_iceberg_schema == iceberg_schema From 1766ccbc17d263b5bb1f63bd8297225bae7a0c32 Mon Sep 17 00:00:00 2001 From: Fokko Driesprong Date: Wed, 1 Jun 2022 00:03:02 +0200 Subject: [PATCH 099/642] Python: Replace tox with pre-commit (#4811) --- .pre-commit-config.yaml | 47 +++++++++++ CONTRIBUTING.md | 44 ++++++++++ Makefile | 28 +++++++ README.md | 2 +- pyproject.toml | 28 ++++++- setup.cfg | 11 ++- spellcheck-dictionary.txt | 2 +- src/iceberg/catalog/__init__.py | 4 + src/iceberg/catalog/base.py | 15 ++-- src/iceberg/io/pyarrow.py | 8 +- src/iceberg/schema.py | 10 ++- src/iceberg/table/base.py | 5 ++ src/iceberg/types.py | 7 +- src/iceberg/utils/schema_conversion.py | 7 +- tests/catalog/test_base.py | 23 +++-- tests/io/__init__.py | 11 +++ tests/io/test_io_base.py | 15 +++- tests/io/test_pyarrow.py | 2 +- tox.ini | 111 ------------------------- 19 files changed, 241 insertions(+), 139 deletions(-) create mode 100644 .pre-commit-config.yaml create mode 100644 CONTRIBUTING.md create mode 100644 Makefile create mode 100644 tests/io/__init__.py delete mode 100644 tox.ini diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml new file mode 100644 index 0000000000..6fea287cb8 --- /dev/null +++ b/.pre-commit-config.yaml @@ -0,0 +1,47 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +--- +files: ^python/ + +repos: + - repo: https://github.com/pre-commit/pre-commit-hooks + rev: v4.2.0 + hooks: + - id: trailing-whitespace + - id: end-of-file-fixer + - id: check-docstring-first + - id: debug-statements + - id: check-yaml + - id: check-ast + - repo: https://github.com/ambv/black + rev: 22.3.0 + hooks: + - id: black + - repo: https://github.com/pre-commit/mirrors-isort + rev: v5.10.1 + hooks: + - id: isort + args: [ --settings-path=python/pyproject.toml ] + - repo: https://github.com/pre-commit/mirrors-mypy + rev: v0.950 + hooks: + - id: mypy + - repo: https://github.com/hadialqattan/pycln + rev: v1.3.2 + hooks: + - id: pycln + args: [--config=python/pyproject.toml] diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md new file mode 100644 index 0000000000..13506a7b8e --- /dev/null +++ b/CONTRIBUTING.md @@ -0,0 +1,44 @@ + + +# Contributing to the Iceberg Python libary + +## Linting + +We rely on `pre-commit` to apply autoformatting and linting: + +```bash +make install +make lint +``` + +By default, it only runs on the files known by git. + +Pre-commit will automatically fix the violations such as import orders, formatting etc. Pylint errors you need to fix yourself. + +In contrast to the name, it doesn't run on the git pre-commit hook by default. If this is something that you like, you can set this up by running `pre-commit install`. + +## Testing + +For Python, we use pytest in combination with coverage to maintain 90% code coverage + +```bash +make install +make test +``` diff --git a/Makefile b/Makefile new file mode 100644 index 0000000000..c4ff2153d2 --- /dev/null +++ b/Makefile @@ -0,0 +1,28 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +install: + pip install -e ".[dev,arrow]" + +lint: + pre-commit run --all-files + +test: + coverage run --source=src/ -m pytest tests/ + coverage report -m --fail-under=90 + coverage html + coverage xml diff --git a/README.md b/README.md index e2c2e38a40..e06ab40b71 100644 --- a/README.md +++ b/README.md @@ -17,7 +17,7 @@ # Iceberg Python -py-iceberg is a python library for programmatic access to iceberg table metadata as well as to table data in iceberg format. +py-iceberg is a python library for programmatic access to iceberg table metadata as well as to table data in iceberg format. It is an implementation of [iceberg table spec](https://iceberg.apache.org/spec/) in Python. ## Getting Started diff --git a/pyproject.toml b/pyproject.toml index 7389f493b7..09b810c40f 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -20,4 +20,30 @@ requires = [ "setuptools>=42", "wheel" ] -build-backend = "setuptools.build_meta" \ No newline at end of file +build-backend = "setuptools.build_meta" + +[tool.black] +line-length = 130 +target-version = ['py38'] + +[tool.isort] +src_paths = ["src/", "tests/"] +multi_line_output = 3 +profile = 'black' +line_length = 130 +force_grid_wrap = 4 + +[tool.pycln] +all = true + +[tool.mypy] +no_implicit_optional = true +warn_redundant_casts = true +warn_unreachable = true + +[[tool.mypy.overrides]] +module = "mypy-pyarrow.*" +ignore_missing_imports = true + +[tool.coverage.run] +source = ['src/'] diff --git a/setup.cfg b/setup.cfg index ab1aa16e35..6a983275fd 100644 --- a/setup.cfg +++ b/setup.cfg @@ -46,9 +46,14 @@ install_requires = mmh3 [options.extras_require] arrow = - pyarrow + pyarrow==8.0.0 dev= - tox-travis==0.12 - pytest + pip>=21.1 + coverage[toml]==6.3.3 + mock==4.0.3 + pytest==7.1.2 + pytest-checkdocs==2.7.1 + pyenchant==3.2.2 + pre-commit==2.19.0 [options.packages.find] where = src diff --git a/spellcheck-dictionary.txt b/spellcheck-dictionary.txt index 3d7092938f..ef3ab6642b 100644 --- a/spellcheck-dictionary.txt +++ b/spellcheck-dictionary.txt @@ -55,4 +55,4 @@ URI UnboundPredicate BoundPredicate BooleanExpression -BooleanExpressionVisitor \ No newline at end of file +BooleanExpressionVisitor diff --git a/src/iceberg/catalog/__init__.py b/src/iceberg/catalog/__init__.py index 13a83393a9..d78cfc86a8 100644 --- a/src/iceberg/catalog/__init__.py +++ b/src/iceberg/catalog/__init__.py @@ -14,3 +14,7 @@ # KIND, either express or implied. See the License for the # specific language governing permissions and limitations # under the License. +from typing import Dict, Tuple + +Identifier = Tuple[str, ...] +Properties = Dict[str, str] diff --git a/src/iceberg/catalog/base.py b/src/iceberg/catalog/base.py index df541fcc6d..40f2747ab1 100644 --- a/src/iceberg/catalog/base.py +++ b/src/iceberg/catalog/base.py @@ -15,15 +15,20 @@ # specific language governing permissions and limitations # under the License. -from abc import ABC, abstractmethod -from typing import Dict, List, Optional, Set, Tuple, Union +from __future__ import annotations +from abc import ABC, abstractmethod +from typing import ( + List, + Optional, + Set, + Union, +) + +from iceberg.catalog import Identifier, Properties from iceberg.schema import Schema from iceberg.table.base import PartitionSpec, Table -Identifier = Tuple[str, ...] -Properties = Dict[str, str] - class Catalog(ABC): """Base Catalog for table operations like - create, drop, load, list and others. diff --git a/src/iceberg/io/pyarrow.py b/src/iceberg/io/pyarrow.py index a67ff5d364..008fd7449c 100644 --- a/src/iceberg/io/pyarrow.py +++ b/src/iceberg/io/pyarrow.py @@ -28,7 +28,13 @@ from pyarrow.fs import FileInfo, FileSystem, FileType -from iceberg.io.base import FileIO, InputFile, InputStream, OutputFile, OutputStream +from iceberg.io.base import ( + FileIO, + InputFile, + InputStream, + OutputFile, + OutputStream, +) class PyArrowFile(InputFile, OutputFile): diff --git a/src/iceberg/schema.py b/src/iceberg/schema.py index 0ba7313ff0..0b02955806 100644 --- a/src/iceberg/schema.py +++ b/src/iceberg/schema.py @@ -21,7 +21,15 @@ from abc import ABC, abstractmethod from dataclasses import dataclass from functools import singledispatch -from typing import Any, Dict, Generic, List, Optional, Tuple, TypeVar +from typing import ( + Any, + Dict, + Generic, + List, + Optional, + Tuple, + TypeVar, +) from iceberg.files import StructProtocol from iceberg.types import ( diff --git a/src/iceberg/table/base.py b/src/iceberg/table/base.py index 023d1f3bbe..f14ec64223 100644 --- a/src/iceberg/table/base.py +++ b/src/iceberg/table/base.py @@ -18,6 +18,9 @@ from __future__ import annotations from abc import ABC +from typing import Union + +from iceberg.catalog.base import Identifier class Table(ABC): @@ -26,6 +29,8 @@ class Table(ABC): To be implemented by https://github.com/apache/iceberg/issues/3227 """ + identifier: Union[str, Identifier] + class PartitionSpec: """Placeholder for Partition Specification diff --git a/src/iceberg/types.py b/src/iceberg/types.py index 07e19e5f8c..22aa6d8a63 100644 --- a/src/iceberg/types.py +++ b/src/iceberg/types.py @@ -31,7 +31,12 @@ """ from dataclasses import dataclass, field from functools import cached_property -from typing import ClassVar, Dict, Optional, Tuple +from typing import ( + ClassVar, + Dict, + Optional, + Tuple, +) class Singleton: diff --git a/src/iceberg/utils/schema_conversion.py b/src/iceberg/utils/schema_conversion.py index 11571f27bd..e6fba45a07 100644 --- a/src/iceberg/utils/schema_conversion.py +++ b/src/iceberg/utils/schema_conversion.py @@ -20,7 +20,12 @@ from __future__ import annotations import logging -from typing import Any, Dict, List, Tuple +from typing import ( + Any, + Dict, + List, + Tuple, +) from iceberg.schema import Schema from iceberg.types import ( diff --git a/tests/catalog/test_base.py b/tests/catalog/test_base.py index ecb34ca1c7..3e5606427c 100644 --- a/tests/catalog/test_base.py +++ b/tests/catalog/test_base.py @@ -15,11 +15,18 @@ # specific language governing permissions and limitations # under the License. -from typing import Dict, List, Optional, Set, Union +from typing import ( + Dict, + List, + Optional, + Set, + Union, +) import pytest -from iceberg.catalog.base import Catalog, Identifier, Properties +from iceberg.catalog import Identifier, Properties +from iceberg.catalog.base import Catalog from iceberg.exceptions import ( AlreadyExistsError, NamespaceNotEmptyError, @@ -136,13 +143,13 @@ def update_namespace_properties( self, namespace: Union[str, Identifier], removals: Optional[Set[str]] = None, updates: Optional[Properties] = None ) -> None: namespace = Catalog.identifier_to_tuple(namespace) - removals = {} if not removals else removals - updates = [] if not updates else updates if namespace in self.__namespaces: - for key in removals: - if key in self.__namespaces[namespace]: - del self.__namespaces[namespace][key] - self.__namespaces[namespace].update(updates) + if removals: + for key in removals: + if key in self.__namespaces[namespace]: + del self.__namespaces[namespace][key] + if updates: + self.__namespaces[namespace].update(updates) else: raise NoSuchNamespaceError(f"Namespace does not exist: {namespace}") diff --git a/tests/io/__init__.py b/tests/io/__init__.py new file mode 100644 index 0000000000..00eaa2ffe4 --- /dev/null +++ b/tests/io/__init__.py @@ -0,0 +1,11 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/tests/io/test_io_base.py b/tests/io/test_io_base.py index 7bd61a0e0e..3cccd90a05 100644 --- a/tests/io/test_io_base.py +++ b/tests/io/test_io_base.py @@ -22,7 +22,13 @@ import pytest -from iceberg.io.base import FileIO, InputFile, InputStream, OutputFile, OutputStream +from iceberg.io.base import ( + FileIO, + InputFile, + InputStream, + OutputFile, + OutputStream, +) from iceberg.io.pyarrow import PyArrowFile, PyArrowFileIO @@ -112,12 +118,13 @@ def new_input(self, location: str): def new_output(self, location: str): return LocalOutputFile(location=location) - def delete(self, location: Union[str, LocalInputFile, LocalOutputFile]): - parsed_location = location.parsed_location if isinstance(location, (InputFile, OutputFile)) else urlparse(location) + def delete(self, location: Union[str, InputFile, OutputFile]) -> None: + location = location.location if isinstance(location, (InputFile, OutputFile)) else location + parsed_location = urlparse(location) try: os.remove(parsed_location.path) except FileNotFoundError as e: - raise FileNotFoundError(f"Cannot delete file, does not exist: {parsed_location.path} - Caused by: " + str(e)) + raise FileNotFoundError(f"Cannot delete file, does not exist: {parsed_location.path} - Caused by: {e}") @pytest.mark.parametrize("CustomInputFile", [LocalInputFile, PyArrowFile]) diff --git a/tests/io/test_pyarrow.py b/tests/io/test_pyarrow.py index bbde45b39d..79e88f3e24 100644 --- a/tests/io/test_pyarrow.py +++ b/tests/io/test_pyarrow.py @@ -180,7 +180,7 @@ def test_raise_on_creating_a_local_file_no_permission(): assert "Cannot get file info, access denied:" in str(exc_info.value) -def test_raise_on_checking_if_local_file_exists_no_permission(): +def test_raise_on_delete_file_with_no_permission(): """Test that a PyArrowFile raises when deleting a local file without permission""" with tempfile.TemporaryDirectory() as tmpdirname: diff --git a/tox.ini b/tox.ini deleted file mode 100644 index 488957edf3..0000000000 --- a/tox.ini +++ /dev/null @@ -1,111 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -[tox] -envlist = py38,py39,py310,linters -skip_missing_interpreters = true - -[testenv] -usedevelop = true -extras = arrow -deps = - pip>=21.1 - coverage - mock - pytest - pytest-checkdocs - pyarrow -setenv = - COVERAGE_FILE = test-reports/{envname}/.coverage - PYTEST_ADDOPTS = --junitxml=test-reports/{envname}/junit.xml -vv -commands = - coverage run --source src --parallel-mode -m pytest {posargs} - coverage combine - coverage report -m --fail-under=90 - coverage html -d test-reports/{envname}/coverage-html - coverage xml -o test-reports/{envname}/coverage.xml - -[testenv:linters] -usedevelop = true -deps = - {[testenv:format-check]deps} - {[testenv:type-check]deps} -commands = - {[testenv:format-check]commands} - {[testenv:type-check]commands} - -[testenv:format-check] -deps = - black - isort - autoflake - pylint - pyenchant -commands = - autoflake -r --check --ignore-init-module-imports --remove-all-unused-imports src tests - isort --profile black --check-only src tests - black --line-length 130 --check --diff src tests - pylint src tests - -[testenv:format] -deps = - {[testenv:format-check]deps} -commands = - autoflake -r --in-place --ignore-init-module-imports --remove-all-unused-imports src tests - isort --profile black src tests - black --line-length 130 src tests - -[testenv:type-check] -deps = - mypy -commands = - mypy --install-types --non-interactive --config tox.ini src - -[testenv:docs] -basepython = python3 -deps = - -r docs/source/requirements.txt -commands = - sphinx-build -E -W -c docs/source/ -b html docs/source/ docs/build/html - sphinx-build -E -W -c docs/source/ -b man docs/source/ docs/build/man - -[testenv:serve-docs] -basepython = python3 -skip_install = true -changedir = docs/build/html -deps = -commands = - python -m http.server {posargs} - -[pytest] -norecursedirs=.* -addopts = --doctest-modules - -[gh-actions] -python = - 3.8: py38, linters - 3.9: py39 - 3.10: py310 - -[mypy] - -no_implicit_optional=True -warn_redundant_casts=True -warn_unreachable=True - -[mypy-pyarrow.*] -ignore_missing_imports = True From da29b1cd76f5a27ad26a4857efa15a5c62ccf8c7 Mon Sep 17 00:00:00 2001 From: Fokko Driesprong Date: Thu, 2 Jun 2022 17:57:10 +0200 Subject: [PATCH 100/642] Python: Remove typing extension imports (#4917) This is a Python 3.7 relic --- src/iceberg/files.py | 8 +------- src/iceberg/io/base.py | 8 +------- 2 files changed, 2 insertions(+), 14 deletions(-) diff --git a/src/iceberg/files.py b/src/iceberg/files.py index c077574ce9..586aab498c 100644 --- a/src/iceberg/files.py +++ b/src/iceberg/files.py @@ -16,13 +16,7 @@ # under the License. from abc import abstractmethod from enum import Enum, auto -from typing import Any - -try: - from typing import Protocol, runtime_checkable -except ImportError: # pragma: no cover - from typing_extensions import Protocol # type: ignore - from typing_extensions import runtime_checkable +from typing import Any, Protocol, runtime_checkable class FileContentType(Enum): diff --git a/src/iceberg/io/base.py b/src/iceberg/io/base.py index 6046770501..4e4ff30cdb 100644 --- a/src/iceberg/io/base.py +++ b/src/iceberg/io/base.py @@ -24,13 +24,7 @@ """ from abc import ABC, abstractmethod -from typing import Union - -try: - from typing import Protocol, runtime_checkable -except ImportError: # pragma: no cover - from typing_extensions import Protocol # type: ignore - from typing_extensions import runtime_checkable +from typing import Protocol, Union, runtime_checkable @runtime_checkable From 3f5f8204bf41ee01b52c20808ac303c8747e6cc1 Mon Sep 17 00:00:00 2001 From: Fokko Driesprong Date: Sat, 4 Jun 2022 00:21:33 +0200 Subject: [PATCH 101/642] Python: Bump pre-commit hooks (#4934) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit More as an example, since nothing really changed in the lint tools. Using pre-commit you can easily bump the integrations to the latest version using `pre-commit autoupdate`. ```bash ➜ python git:(master) pre-commit autoupdate Updating https://github.com/pre-commit/pre-commit-hooks ... already up to date. Updating https://github.com/ambv/black ... already up to date. Updating https://github.com/pre-commit/mirrors-isort ... already up to date. Updating https://github.com/pre-commit/mirrors-mypy ... updating v0.950 -> v0.960. Updating https://github.com/hadialqattan/pycln ... updating v1.3.2 -> v1.3.3. ``` --- .pre-commit-config.yaml | 4 ++-- CONTRIBUTING.md | 2 ++ 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 6fea287cb8..cc30a5aa54 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -37,11 +37,11 @@ repos: - id: isort args: [ --settings-path=python/pyproject.toml ] - repo: https://github.com/pre-commit/mirrors-mypy - rev: v0.950 + rev: v0.960 hooks: - id: mypy - repo: https://github.com/hadialqattan/pycln - rev: v1.3.2 + rev: v1.3.3 hooks: - id: pycln args: [--config=python/pyproject.toml] diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 13506a7b8e..dec8c246c7 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -34,6 +34,8 @@ Pre-commit will automatically fix the violations such as import orders, formatti In contrast to the name, it doesn't run on the git pre-commit hook by default. If this is something that you like, you can set this up by running `pre-commit install`. +You can bump the integrations to the latest version using `pre-commit autoupdate`. This will check if there is a newer version of `{black,mypy,isort,...}` and update the yaml. + ## Testing For Python, we use pytest in combination with coverage to maintain 90% code coverage From 8817745e01c651eea7cdb57a53bb578d4dc75542 Mon Sep 17 00:00:00 2001 From: chulucninh09 <43906170+chulucninh09@users.noreply.github.com> Date: Mon, 6 Jun 2022 01:15:15 +0700 Subject: [PATCH 102/642] Python: Use struct constant to improve pack/unpack performance (#4929) --- src/iceberg/conversions.py | 35 +++++++++++++++++++++-------------- 1 file changed, 21 insertions(+), 14 deletions(-) diff --git a/src/iceberg/conversions.py b/src/iceberg/conversions.py index f5bf709d24..d302f4e199 100644 --- a/src/iceberg/conversions.py +++ b/src/iceberg/conversions.py @@ -27,10 +27,10 @@ implementation, a concrete function is registered for each generic conversion function. For PrimitiveType implementations that share the same conversion logic, registrations can be stacked. """ -import struct import uuid from decimal import Decimal from functools import singledispatch +from struct import Struct from typing import Union from iceberg.types import ( @@ -52,6 +52,13 @@ ) from iceberg.utils.decimal import decimal_to_bytes, unscaled_to_decimal +_BOOL_STRUCT = Struct("QQ") + def handle_none(func): """A decorator function to handle cases where partition values are `None` or "__HIVE_DEFAULT_PARTITION__" @@ -154,13 +161,13 @@ def to_bytes(primitive_type: PrimitiveType, value: Union[bool, bytes, Decimal, f @to_bytes.register(BooleanType) def _(primitive_type, value: bool) -> bytes: - return struct.pack(" bytes: - return struct.pack(" bytes: @to_bytes.register(TimestampType) @to_bytes.register(TimestamptzType) def _(primitive_type, value: int) -> bytes: - return struct.pack(" bytes: Note: float in python is implemented using a double in C. Therefore this involves a conversion of a 32-bit (single precision) float to a 64-bit (double precision) float which introduces some imprecision. """ - return struct.pack(" bytes: - return struct.pack(" bytes: @to_bytes.register(UUIDType) def _(primitive_type, value: uuid.UUID) -> bytes: - return struct.pack(">QQ", (value.int >> 64) & 0xFFFFFFFFFFFFFFFF, value.int & 0xFFFFFFFFFFFFFFFF) + return _UUID_STRUCT.pack((value.int >> 64) & 0xFFFFFFFFFFFFFFFF, value.int & 0xFFFFFFFFFFFFFFFF) @to_bytes.register(BinaryType) @@ -241,13 +248,13 @@ def from_bytes(primitive_type: PrimitiveType, b: bytes) -> Union[bool, bytes, De @from_bytes.register(BooleanType) def _(primitive_type, b: bytes) -> bool: - return struct.unpack(" int: - return struct.unpack(" int: @from_bytes.register(TimestampType) @from_bytes.register(TimestamptzType) def _(primitive_type, b: bytes) -> int: - return struct.unpack(" float: + return _FLOAT_STRUCT.unpack(b)[0] @from_bytes.register(DoubleType) def _(primitive_type, b: bytes) -> float: - return struct.unpack(" str: @from_bytes.register(UUIDType) def _(primitive_type, b: bytes) -> uuid.UUID: - unpacked_bytes = struct.unpack(">QQ", b) + unpacked_bytes = _UUID_STRUCT.unpack(b) return uuid.UUID(int=unpacked_bytes[0] << 64 | unpacked_bytes[1]) From 749f8b2fd0d15d345e6a262d2c363f473fe2f44e Mon Sep 17 00:00:00 2001 From: Fokko Driesprong Date: Sun, 5 Jun 2022 20:17:16 +0200 Subject: [PATCH 103/642] Python: Add pyupgrade as a pre-commit hook (#4935) --- .pre-commit-config.yaml | 5 ++ src/iceberg/catalog/base.py | 40 ++++------ src/iceberg/schema.py | 103 ++++++++++++------------- src/iceberg/table/base.py | 3 +- src/iceberg/transforms.py | 2 +- src/iceberg/types.py | 2 +- src/iceberg/utils/schema_conversion.py | 35 ++++----- 7 files changed, 90 insertions(+), 100 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index cc30a5aa54..1d0db0ba78 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -45,3 +45,8 @@ repos: hooks: - id: pycln args: [--config=python/pyproject.toml] + - repo: https://github.com/asottile/pyupgrade + rev: v2.32.1 + hooks: + - id: pyupgrade + args: [--py38-plus] diff --git a/src/iceberg/catalog/base.py b/src/iceberg/catalog/base.py index 40f2747ab1..243f6a8f74 100644 --- a/src/iceberg/catalog/base.py +++ b/src/iceberg/catalog/base.py @@ -18,12 +18,6 @@ from __future__ import annotations from abc import ABC, abstractmethod -from typing import ( - List, - Optional, - Set, - Union, -) from iceberg.catalog import Identifier, Properties from iceberg.schema import Schema @@ -59,11 +53,11 @@ def properties(self) -> Properties: @abstractmethod def create_table( self, - identifier: Union[str, Identifier], + identifier: str | Identifier, schema: Schema, - location: Optional[str] = None, - partition_spec: Optional[PartitionSpec] = None, - properties: Optional[Properties] = None, + location: str | None = None, + partition_spec: PartitionSpec | None = None, + properties: Properties | None = None, ) -> Table: """Create a table @@ -82,7 +76,7 @@ def create_table( """ @abstractmethod - def load_table(self, identifier: Union[str, Identifier]) -> Table: + def load_table(self, identifier: str | Identifier) -> Table: """Loads the table's metadata and returns the table instance. You can also use this method to check for table existence using 'try catalog.table() except TableNotFoundError' @@ -99,7 +93,7 @@ def load_table(self, identifier: Union[str, Identifier]) -> Table: """ @abstractmethod - def drop_table(self, identifier: Union[str, Identifier]) -> None: + def drop_table(self, identifier: str | Identifier) -> None: """Drop a table. Args: @@ -110,7 +104,7 @@ def drop_table(self, identifier: Union[str, Identifier]) -> None: """ @abstractmethod - def purge_table(self, identifier: Union[str, Identifier]) -> None: + def purge_table(self, identifier: str | Identifier) -> None: """Drop a table and purge all data and metadata files. Args: @@ -121,7 +115,7 @@ def purge_table(self, identifier: Union[str, Identifier]) -> None: """ @abstractmethod - def rename_table(self, from_identifier: Union[str, Identifier], to_identifier: Union[str, Identifier]) -> Table: + def rename_table(self, from_identifier: str | Identifier, to_identifier: str | Identifier) -> Table: """Rename a fully classified table name Args: @@ -136,7 +130,7 @@ def rename_table(self, from_identifier: Union[str, Identifier], to_identifier: U """ @abstractmethod - def create_namespace(self, namespace: Union[str, Identifier], properties: Optional[Properties] = None) -> None: + def create_namespace(self, namespace: str | Identifier, properties: Properties | None = None) -> None: """Create a namespace in the catalog. Args: @@ -148,7 +142,7 @@ def create_namespace(self, namespace: Union[str, Identifier], properties: Option """ @abstractmethod - def drop_namespace(self, namespace: Union[str, Identifier]) -> None: + def drop_namespace(self, namespace: str | Identifier) -> None: """Drop a namespace. Args: @@ -160,7 +154,7 @@ def drop_namespace(self, namespace: Union[str, Identifier]) -> None: """ @abstractmethod - def list_tables(self, namespace: Optional[Union[str, Identifier]] = None) -> List[Identifier]: + def list_tables(self, namespace: str | Identifier | None = None) -> list[Identifier]: """List tables under the given namespace in the catalog. If namespace not provided, will list all tables in the catalog. @@ -176,7 +170,7 @@ def list_tables(self, namespace: Optional[Union[str, Identifier]] = None) -> Lis """ @abstractmethod - def list_namespaces(self) -> List[Identifier]: + def list_namespaces(self) -> list[Identifier]: """List namespaces from the given namespace. If not given, list top-level namespaces from the catalog. Returns: @@ -184,7 +178,7 @@ def list_namespaces(self) -> List[Identifier]: """ @abstractmethod - def load_namespace_properties(self, namespace: Union[str, Identifier]) -> Properties: + def load_namespace_properties(self, namespace: str | Identifier) -> Properties: """Get properties for a namespace. Args: @@ -199,7 +193,7 @@ def load_namespace_properties(self, namespace: Union[str, Identifier]) -> Proper @abstractmethod def update_namespace_properties( - self, namespace: Union[str, Identifier], removals: Optional[Set[str]] = None, updates: Optional[Properties] = None + self, namespace: str | Identifier, removals: set[str] | None = None, updates: Properties | None = None ) -> None: """Removes provided property keys and updates properties for a namespace. @@ -214,7 +208,7 @@ def update_namespace_properties( """ @staticmethod - def identifier_to_tuple(identifier: Union[str, Identifier]) -> Identifier: + def identifier_to_tuple(identifier: str | Identifier) -> Identifier: """Parses an identifier to a tuple. If the identifier is a string, it is split into a tuple on '.'. If it is a tuple, it is used as-is. @@ -228,7 +222,7 @@ def identifier_to_tuple(identifier: Union[str, Identifier]) -> Identifier: return identifier if isinstance(identifier, tuple) else tuple(str.split(identifier, ".")) @staticmethod - def table_name_from(identifier: Union[str, Identifier]) -> str: + def table_name_from(identifier: str | Identifier) -> str: """Extracts table name from a table identifier Args: @@ -240,7 +234,7 @@ def table_name_from(identifier: Union[str, Identifier]) -> str: return Catalog.identifier_to_tuple(identifier)[-1] @staticmethod - def namespace_from(identifier: Union[str, Identifier]) -> Identifier: + def namespace_from(identifier: str | Identifier) -> Identifier: """Extracts table namespace from a table identifier Args: diff --git a/src/iceberg/schema.py b/src/iceberg/schema.py index 0b02955806..b0fd8427c5 100644 --- a/src/iceberg/schema.py +++ b/src/iceberg/schema.py @@ -25,9 +25,6 @@ Any, Dict, Generic, - List, - Optional, - Tuple, TypeVar, ) @@ -52,15 +49,15 @@ class Schema: >>> from iceberg import types """ - def __init__(self, *columns: NestedField, schema_id: int, identifier_field_ids: Optional[List[int]] = None): + def __init__(self, *columns: NestedField, schema_id: int, identifier_field_ids: list[int] | None = None): self._struct = StructType(*columns) self._schema_id = schema_id self._identifier_field_ids = identifier_field_ids or [] - self._name_to_id: Dict[str, int] = index_by_name(self) - self._name_to_id_lower: Dict[str, int] = {} # Should be accessed through self._lazy_name_to_id_lower() - self._id_to_field: Dict[int, NestedField] = {} # Should be accessed through self._lazy_id_to_field() - self._id_to_name: Dict[int, str] = {} # Should be accessed through self._lazy_id_to_name() - self._id_to_accessor: Dict[int, Accessor] = {} # Should be accessed through self._lazy_id_to_accessor() + self._name_to_id: dict[str, int] = index_by_name(self) + self._name_to_id_lower: dict[str, int] = {} # Should be accessed through self._lazy_name_to_id_lower() + self._id_to_field: dict[int, NestedField] = {} # Should be accessed through self._lazy_id_to_field() + self._id_to_name: dict[int, str] = {} # Should be accessed through self._lazy_id_to_name() + self._id_to_accessor: dict[int, Accessor] = {} # Should be accessed through self._lazy_id_to_accessor() def __str__(self): return "table {\n" + "\n".join([" " + str(field) for field in self.columns]) + "\n}" @@ -86,7 +83,7 @@ def __eq__(self, other) -> bool: return identifier_field_ids_is_equal and schema_is_equal @property - def columns(self) -> Tuple[NestedField, ...]: + def columns(self) -> tuple[NestedField, ...]: """A list of the top-level fields in the underlying struct""" return self._struct.fields @@ -96,10 +93,10 @@ def schema_id(self) -> int: return self._schema_id @property - def identifier_field_ids(self) -> List[int]: + def identifier_field_ids(self) -> list[int]: return self._identifier_field_ids - def _lazy_id_to_field(self) -> Dict[int, NestedField]: + def _lazy_id_to_field(self) -> dict[int, NestedField]: """Returns an index of field ID to NestedField instance This is calculated once when called for the first time. Subsequent calls to this method will use a cached index. @@ -108,7 +105,7 @@ def _lazy_id_to_field(self) -> Dict[int, NestedField]: self._id_to_field = index_by_id(self) return self._id_to_field - def _lazy_name_to_id_lower(self) -> Dict[str, int]: + def _lazy_name_to_id_lower(self) -> dict[str, int]: """Returns an index of lower-case field names to field IDs This is calculated once when called for the first time. Subsequent calls to this method will use a cached index. @@ -117,7 +114,7 @@ def _lazy_name_to_id_lower(self) -> Dict[str, int]: self._name_to_id_lower = {name.lower(): field_id for name, field_id in self._name_to_id.items()} return self._name_to_id_lower - def _lazy_id_to_name(self) -> Dict[int, str]: + def _lazy_id_to_name(self) -> dict[int, str]: """Returns an index of field ID to full name This is calculated once when called for the first time. Subsequent calls to this method will use a cached index. @@ -126,7 +123,7 @@ def _lazy_id_to_name(self) -> Dict[int, str]: self._id_to_name = index_name_by_id(self) return self._id_to_name - def _lazy_id_to_accessor(self) -> Dict[int, Accessor]: + def _lazy_id_to_accessor(self) -> dict[int, Accessor]: """Returns an index of field ID to accessor This is calculated once when called for the first time. Subsequent calls to this method will use a cached index. @@ -193,7 +190,7 @@ def accessor_for_field(self, field_id: int) -> Accessor: """ return self._lazy_id_to_accessor().get(field_id) # type: ignore - def select(self, names: List[str], case_sensitive: bool = True) -> "Schema": + def select(self, names: list[str], case_sensitive: bool = True) -> Schema: """Return a new schema instance pruned to a subset of columns Args: @@ -208,12 +205,12 @@ def select(self, names: List[str], case_sensitive: bool = True) -> "Schema": return self._case_insensitive_select(schema=self, names=names) @classmethod - def _case_sensitive_select(cls, schema: "Schema", names: List[str]): + def _case_sensitive_select(cls, schema: Schema, names: list[str]): # TODO: Add a PruneColumns schema visitor and use it here raise NotImplementedError() @classmethod - def _case_insensitive_select(cls, schema: "Schema", names: List[str]): + def _case_insensitive_select(cls, schema: Schema, names: list[str]): # TODO: Add a PruneColumns schema visitor and use it here raise NotImplementedError() @@ -254,7 +251,7 @@ def schema(self, schema: Schema, struct_result: T) -> T: """Visit a Schema""" @abstractmethod - def struct(self, struct: StructType, field_results: List[T]) -> T: + def struct(self, struct: StructType, field_results: list[T]) -> T: """Visit a StructType""" @abstractmethod @@ -279,7 +276,7 @@ class Accessor: """An accessor for a specific position in a container that implements the StructProtocol""" position: int - inner: Optional["Accessor"] = None + inner: Accessor | None = None def __str__(self): return f"Accessor(position={self.position},inner={self.inner})" @@ -377,35 +374,35 @@ class _IndexById(SchemaVisitor[Dict[int, NestedField]]): """A schema visitor for generating a field ID to NestedField index""" def __init__(self) -> None: - self._index: Dict[int, NestedField] = {} + self._index: dict[int, NestedField] = {} - def schema(self, schema: Schema, struct_result) -> Dict[int, NestedField]: + def schema(self, schema: Schema, struct_result) -> dict[int, NestedField]: return self._index - def struct(self, struct: StructType, field_results) -> Dict[int, NestedField]: + def struct(self, struct: StructType, field_results) -> dict[int, NestedField]: return self._index - def field(self, field: NestedField, field_result) -> Dict[int, NestedField]: + def field(self, field: NestedField, field_result) -> dict[int, NestedField]: """Add the field ID to the index""" self._index[field.field_id] = field return self._index - def list(self, list_type: ListType, element_result) -> Dict[int, NestedField]: + def list(self, list_type: ListType, element_result) -> dict[int, NestedField]: """Add the list element ID to the index""" self._index[list_type.element.field_id] = list_type.element return self._index - def map(self, map_type: MapType, key_result, value_result) -> Dict[int, NestedField]: + def map(self, map_type: MapType, key_result, value_result) -> dict[int, NestedField]: """Add the key ID and value ID as individual items in the index""" self._index[map_type.key.field_id] = map_type.key self._index[map_type.value.field_id] = map_type.value return self._index - def primitive(self, primitive) -> Dict[int, NestedField]: + def primitive(self, primitive) -> dict[int, NestedField]: return self._index -def index_by_id(schema_or_type) -> Dict[int, NestedField]: +def index_by_id(schema_or_type) -> dict[int, NestedField]: """Generate an index of field IDs to NestedField instances Args: @@ -421,11 +418,11 @@ class _IndexByName(SchemaVisitor[Dict[str, int]]): """A schema visitor for generating a field name to field ID index""" def __init__(self) -> None: - self._index: Dict[str, int] = {} - self._short_name_to_id: Dict[str, int] = {} - self._combined_index: Dict[str, int] = {} - self._field_names: List[str] = [] - self._short_field_names: List[str] = [] + self._index: dict[str, int] = {} + self._short_name_to_id: dict[str, int] = {} + self._combined_index: dict[str, int] = {} + self._field_names: list[str] = [] + self._short_field_names: list[str] = [] def before_list_element(self, element: NestedField) -> None: """Short field names omit element when the element is a StructType""" @@ -448,23 +445,23 @@ def after_field(self, field: NestedField) -> None: self._field_names.pop() self._short_field_names.pop() - def schema(self, schema: Schema, struct_result: Dict[str, int]) -> Dict[str, int]: + def schema(self, schema: Schema, struct_result: dict[str, int]) -> dict[str, int]: return self._index - def struct(self, struct: StructType, field_results: List[Dict[str, int]]) -> Dict[str, int]: + def struct(self, struct: StructType, field_results: list[dict[str, int]]) -> dict[str, int]: return self._index - def field(self, field: NestedField, field_result: Dict[str, int]) -> Dict[str, int]: + def field(self, field: NestedField, field_result: dict[str, int]) -> dict[str, int]: """Add the field name to the index""" self._add_field(field.name, field.field_id) return self._index - def list(self, list_type: ListType, element_result: Dict[str, int]) -> Dict[str, int]: + def list(self, list_type: ListType, element_result: dict[str, int]) -> dict[str, int]: """Add the list element name to the index""" self._add_field(list_type.element.name, list_type.element.field_id) return self._index - def map(self, map_type: MapType, key_result: Dict[str, int], value_result: Dict[str, int]) -> Dict[str, int]: + def map(self, map_type: MapType, key_result: dict[str, int], value_result: dict[str, int]) -> dict[str, int]: """Add the key name and value name as individual items in the index""" self._add_field(map_type.key.name, map_type.key.field_id) self._add_field(map_type.value.name, map_type.value.field_id) @@ -493,10 +490,10 @@ def _add_field(self, name: str, field_id: int): short_name = ".".join([".".join(self._short_field_names), name]) self._short_name_to_id[short_name] = field_id - def primitive(self, primitive) -> Dict[str, int]: + def primitive(self, primitive) -> dict[str, int]: return self._index - def by_name(self) -> Dict[str, int]: + def by_name(self) -> dict[str, int]: """Returns an index of combined full and short names Note: Only short names that do not conflict with full names are included. @@ -505,13 +502,13 @@ def by_name(self) -> Dict[str, int]: combined_index.update(self._index) return combined_index - def by_id(self) -> Dict[int, str]: + def by_id(self) -> dict[int, str]: """Returns an index of ID to full names""" - id_to_full_name = dict([(value, key) for key, value in self._index.items()]) + id_to_full_name = {value: key for key, value in self._index.items()} return id_to_full_name -def index_by_name(schema_or_type: Schema | IcebergType) -> Dict[str, int]: +def index_by_name(schema_or_type: Schema | IcebergType) -> dict[str, int]: """Generate an index of field names to field IDs Args: @@ -525,7 +522,7 @@ def index_by_name(schema_or_type: Schema | IcebergType) -> Dict[str, int]: return indexer.by_name() -def index_name_by_id(schema_or_type: Schema | IcebergType) -> Dict[int, str]: +def index_name_by_id(schema_or_type: Schema | IcebergType) -> dict[int, str]: """Generate an index of field IDs full field names Args: @@ -575,13 +572,13 @@ class _BuildPositionAccessors(SchemaVisitor[Dict[Position, Accessor]]): """ @staticmethod - def _wrap_leaves(result: Dict[Position, Accessor], position: Position = 0) -> Dict[Position, Accessor]: + def _wrap_leaves(result: dict[Position, Accessor], position: Position = 0) -> dict[Position, Accessor]: return {field_id: Accessor(position, inner=inner) for field_id, inner in result.items()} - def schema(self, schema: Schema, struct_result: Dict[Position, Accessor]) -> Dict[Position, Accessor]: + def schema(self, schema: Schema, struct_result: dict[Position, Accessor]) -> dict[Position, Accessor]: return struct_result - def struct(self, struct: StructType, field_results: List[Dict[Position, Accessor]]) -> Dict[Position, Accessor]: + def struct(self, struct: StructType, field_results: list[dict[Position, Accessor]]) -> dict[Position, Accessor]: result = {} for position, field in enumerate(struct.fields): @@ -593,22 +590,22 @@ def struct(self, struct: StructType, field_results: List[Dict[Position, Accessor return result - def field(self, field: NestedField, field_result: Dict[Position, Accessor]) -> Dict[Position, Accessor]: + def field(self, field: NestedField, field_result: dict[Position, Accessor]) -> dict[Position, Accessor]: return field_result - def list(self, list_type: ListType, element_result: Dict[Position, Accessor]) -> Dict[Position, Accessor]: + def list(self, list_type: ListType, element_result: dict[Position, Accessor]) -> dict[Position, Accessor]: return {} def map( - self, map_type: MapType, key_result: Dict[Position, Accessor], value_result: Dict[Position, Accessor] - ) -> Dict[Position, Accessor]: + self, map_type: MapType, key_result: dict[Position, Accessor], value_result: dict[Position, Accessor] + ) -> dict[Position, Accessor]: return {} - def primitive(self, primitive: PrimitiveType) -> Dict[Position, Accessor]: + def primitive(self, primitive: PrimitiveType) -> dict[Position, Accessor]: return {} -def build_position_accessors(schema_or_type: Schema | IcebergType) -> Dict[int, Accessor]: +def build_position_accessors(schema_or_type: Schema | IcebergType) -> dict[int, Accessor]: """Generate an index of field IDs to schema position accessors Args: diff --git a/src/iceberg/table/base.py b/src/iceberg/table/base.py index f14ec64223..1cc8b39061 100644 --- a/src/iceberg/table/base.py +++ b/src/iceberg/table/base.py @@ -18,7 +18,6 @@ from __future__ import annotations from abc import ABC -from typing import Union from iceberg.catalog.base import Identifier @@ -29,7 +28,7 @@ class Table(ABC): To be implemented by https://github.com/apache/iceberg/issues/3227 """ - identifier: Union[str, Identifier] + identifier: str | Identifier class PartitionSpec: diff --git a/src/iceberg/transforms.py b/src/iceberg/transforms.py index 6043d2aefa..39ec5d538e 100644 --- a/src/iceberg/transforms.py +++ b/src/iceberg/transforms.py @@ -258,7 +258,7 @@ class VoidTransform(Transform): def __new__(cls): # pylint: disable=W0221 if cls._instance is None: - cls._instance = super(VoidTransform, cls).__new__(cls) + cls._instance = super().__new__(cls) return cls._instance def __init__(self): diff --git a/src/iceberg/types.py b/src/iceberg/types.py index 22aa6d8a63..6f1ad701c4 100644 --- a/src/iceberg/types.py +++ b/src/iceberg/types.py @@ -44,7 +44,7 @@ class Singleton: def __new__(cls): if not isinstance(cls._instance, cls): - cls._instance = super(Singleton, cls).__new__(cls) + cls._instance = super().__new__(cls) return cls._instance diff --git a/src/iceberg/utils/schema_conversion.py b/src/iceberg/utils/schema_conversion.py index e6fba45a07..25edc62ac6 100644 --- a/src/iceberg/utils/schema_conversion.py +++ b/src/iceberg/utils/schema_conversion.py @@ -20,12 +20,7 @@ from __future__ import annotations import logging -from typing import ( - Any, - Dict, - List, - Tuple, -) +from typing import Any from iceberg.schema import Schema from iceberg.types import ( @@ -52,7 +47,7 @@ logger = logging.getLogger(__name__) -PRIMITIVE_FIELD_TYPE_MAPPING: Dict[str, PrimitiveType] = { +PRIMITIVE_FIELD_TYPE_MAPPING: dict[str, PrimitiveType] = { "boolean": BooleanType(), "bytes": BinaryType(), "double": DoubleType(), @@ -63,7 +58,7 @@ "enum": StringType(), } -LOGICAL_FIELD_TYPE_MAPPING: Dict[Tuple[str, str], PrimitiveType] = { +LOGICAL_FIELD_TYPE_MAPPING: dict[tuple[str, str], PrimitiveType] = { ("date", "int"): DateType(), ("time-millis", "int"): TimeType(), ("timestamp-millis", "long"): TimestampType(), @@ -74,7 +69,7 @@ class AvroSchemaConversion: - def avro_to_iceberg(self, avro_schema: Dict[str, Any]) -> Schema: + def avro_to_iceberg(self, avro_schema: dict[str, Any]) -> Schema: """Converts an Apache Avro into an Apache Iceberg schema equivalent This expects to have field id's to be encoded in the Avro schema:: @@ -119,7 +114,7 @@ def avro_to_iceberg(self, avro_schema: Dict[str, Any]) -> Schema: """ return Schema(*[self._convert_field(field) for field in avro_schema["fields"]], schema_id=1) - def _resolve_union(self, type_union: Dict | List | str) -> Tuple[str | Dict[str, Any], bool]: + def _resolve_union(self, type_union: dict | list | str) -> tuple[str | dict[str, Any], bool]: """ Converts Unions into their type and resolves if the field is optional @@ -142,7 +137,7 @@ def _resolve_union(self, type_union: Dict | List | str) -> Tuple[str | Dict[str, Raises: TypeError: In the case non-optional union types are encountered """ - avro_types: Dict | List + avro_types: dict | list if isinstance(type_union, str): # It is a primitive and required return type_union, False @@ -160,7 +155,7 @@ def _resolve_union(self, type_union: Dict | List | str) -> Tuple[str | Dict[str, # Filter the null value and return the type return list(filter(lambda t: t != "null", avro_types))[0], is_optional - def _convert_schema(self, avro_type: str | Dict[str, Any]) -> IcebergType: + def _convert_schema(self, avro_type: str | dict[str, Any]) -> IcebergType: """ Resolves the Avro type @@ -198,7 +193,7 @@ def _convert_schema(self, avro_type: str | Dict[str, Any]) -> IcebergType: else: raise ValueError(f"Unknown type: {avro_type}") - def _convert_field(self, field: Dict[str, Any]) -> NestedField: + def _convert_field(self, field: dict[str, Any]) -> NestedField: """ Converts an Avro field into an Iceberg equivalent field Args: @@ -220,7 +215,7 @@ def _convert_field(self, field: Dict[str, Any]) -> NestedField: doc=field.get("doc"), ) - def _convert_record_type(self, record_type: Dict[str, Any]) -> StructType: + def _convert_record_type(self, record_type: dict[str, Any]) -> StructType: """ Converts the fields from a record into an Iceberg struct @@ -274,7 +269,7 @@ def _convert_record_type(self, record_type: Dict[str, Any]) -> StructType: return StructType(*[self._convert_field(field) for field in record_type["fields"]]) - def _convert_array_type(self, array_type: Dict[str, Any]) -> ListType: + def _convert_array_type(self, array_type: dict[str, Any]) -> ListType: if "element-id" not in array_type: raise ValueError(f"Cannot convert array-type, missing element-id: {array_type}") @@ -286,7 +281,7 @@ def _convert_array_type(self, array_type: Dict[str, Any]) -> ListType: element_is_optional=element_is_optional, ) - def _convert_map_type(self, map_type: Dict[str, Any]) -> MapType: + def _convert_map_type(self, map_type: dict[str, Any]) -> MapType: """ Args: map_type: The dict that describes the Avro map type @@ -322,7 +317,7 @@ def _convert_map_type(self, map_type: Dict[str, Any]) -> MapType: value_is_optional=value_is_optional, ) - def _convert_logical_type(self, avro_logical_type: Dict[str, Any]) -> IcebergType: + def _convert_logical_type(self, avro_logical_type: dict[str, Any]) -> IcebergType: """ Convert a schema with a logical type annotation. For the decimal and map we need to fetch more keys from the dict, and for the simple ones we can just @@ -358,7 +353,7 @@ def _convert_logical_type(self, avro_logical_type: Dict[str, Any]) -> IcebergTyp else: raise ValueError(f"Unknown logical/physical type combination: {avro_logical_type}") - def _convert_logical_decimal_type(self, avro_type: Dict[str, Any]) -> DecimalType: + def _convert_logical_decimal_type(self, avro_type: dict[str, Any]) -> DecimalType: """ Args: avro_type: The Avro type @@ -384,7 +379,7 @@ def _convert_logical_decimal_type(self, avro_type: Dict[str, Any]) -> DecimalTyp """ return DecimalType(precision=avro_type["precision"], scale=avro_type["scale"]) - def _convert_logical_map_type(self, avro_type: Dict[str, Any]) -> MapType: + def _convert_logical_map_type(self, avro_type: dict[str, Any]) -> MapType: """ In the case where a map hasn't a key as a type you can use a logical map to still encode this in Avro @@ -436,7 +431,7 @@ def _convert_logical_map_type(self, avro_type: Dict[str, Any]) -> MapType: value_is_optional=value.is_optional, ) - def _convert_fixed_type(self, avro_type: Dict[str, Any]) -> FixedType: + def _convert_fixed_type(self, avro_type: dict[str, Any]) -> FixedType: """ https://avro.apache.org/docs/current/spec.html#Fixed From 4c055e0ec74f488feb213592d34d74083a702e8e Mon Sep 17 00:00:00 2001 From: Fokko Driesprong Date: Mon, 6 Jun 2022 19:22:49 +0200 Subject: [PATCH 104/642] Python: Add Poetry build and release instructions (#4844) --- .pre-commit-config.yaml | 10 +- CONTRIBUTING.md | 25 +- Makefile | 13 +- README.md | 27 +- dev/RELEASE.md | 146 ++++++++++ poetry.lock | 602 ++++++++++++++++++++++++++++++++++++++++ pyproject.toml | 44 ++- setup.cfg | 59 ---- 8 files changed, 831 insertions(+), 95 deletions(-) create mode 100644 dev/RELEASE.md create mode 100644 poetry.lock delete mode 100644 setup.cfg diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 1d0db0ba78..b30c653478 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -40,11 +40,11 @@ repos: rev: v0.960 hooks: - id: mypy - - repo: https://github.com/hadialqattan/pycln - rev: v1.3.3 - hooks: - - id: pycln - args: [--config=python/pyproject.toml] + - repo: https://github.com/hadialqattan/pycln + rev: v1.3.3 + hooks: + - id: pycln + args: [--config=python/pyproject.toml] - repo: https://github.com/asottile/pyupgrade rev: v2.32.1 hooks: diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index dec8c246c7..1afa7bd948 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -17,30 +17,41 @@ - under the License. --> -# Contributing to the Iceberg Python libary +# Contributing to the Iceberg Python library + +For the development, we use poetry for packing and dependency management. You can install this using: + +```bash +pip install poetry +``` + +If you have an older version of pip and virtualenv you need to update these: +```bash +pip install --upgrade virtualenv pip +``` + +To get started, you can run `make install`, which will install poetry and it will install all the dependencies of the Iceberg library. This will also install the development dependencies. If you don't want to do this, you need to install using `poetry install --no-dev`. + +If you want to install the library on the host, you can simply run `pip3 install -e .`. If you wish to use a virtual environment, you can run `poetry shell`. Poetry will open up a virtual environment with all the dependencies set. ## Linting We rely on `pre-commit` to apply autoformatting and linting: ```bash -make install make lint ``` -By default, it only runs on the files known by git. - Pre-commit will automatically fix the violations such as import orders, formatting etc. Pylint errors you need to fix yourself. -In contrast to the name, it doesn't run on the git pre-commit hook by default. If this is something that you like, you can set this up by running `pre-commit install`. +In contrast to the name suggest, it doesn't run the checks on the commit. If this is something that you like, you can set this up by running `pre-commit install`. You can bump the integrations to the latest version using `pre-commit autoupdate`. This will check if there is a newer version of `{black,mypy,isort,...}` and update the yaml. ## Testing -For Python, we use pytest in combination with coverage to maintain 90% code coverage +For Python, we use pytest in combination with coverage to maintain 90% code coverage. ```bash -make install make test ``` diff --git a/Makefile b/Makefile index c4ff2153d2..356c94e642 100644 --- a/Makefile +++ b/Makefile @@ -16,13 +16,14 @@ # under the License. install: - pip install -e ".[dev,arrow]" + pip install poetry + poetry install -E pyarrow lint: - pre-commit run --all-files + poetry run pre-commit run --all-files test: - coverage run --source=src/ -m pytest tests/ - coverage report -m --fail-under=90 - coverage html - coverage xml + poetry run coverage run --source=src/ -m pytest tests/ + poetry run coverage report -m --fail-under=90 + poetry run coverage html + poetry run coverage xml diff --git a/README.md b/README.md index e06ab40b71..292fb7b9ee 100644 --- a/README.md +++ b/README.md @@ -30,26 +30,27 @@ cd iceberg/python pip install -e . ``` -## Testing +Development is made easy using [Poetry](https://python-poetry.org/docs/#installation). -Testing is done using tox. The config can be found in `tox.ini` within the python directory of the iceberg project. +## Development -``` -tox +Poetry provides virtual environments for development: + +```bash +poetry shell +poetry install -E pyarrow +pytest ``` -## Solution for `InterpreterNotFound` Errors +For more information, please refer to the [Manage environments](https://python-poetry.org/docs/managing-environments/) section of Poetry. -Currently, tests run against python `3.7.12`, `3.8.12`, and `3.9.10`. It's recommended to install and manage multiple interpreters using [pyenv](https://github.com/pyenv/pyenv). -``` -pyenv install 3.7.12 -pyenv install 3.8.12 -pyenv install 3.9.10 -``` +## Testing + +Testing is done using Poetry: -Once all three versions are installed, you can set an application-specific pyenv environment by running the following in the python directory. ``` -pyenv local 3.7.12 3.8.12 3.9.10 +poetry install -E pyarrow +poetry run pytest ``` ## Get in Touch diff --git a/dev/RELEASE.md b/dev/RELEASE.md new file mode 100644 index 0000000000..44305868bf --- /dev/null +++ b/dev/RELEASE.md @@ -0,0 +1,146 @@ + + +# How to release + +The guide to release the Python package. + +First we're going to release a release candidate (RC) and publish it to the public to test. Once the vote has passed on the RC, we can release the new version. + +## Running a release candidate + +Make sure that you're on the version that you want to release. + +```bash +export RC=rc1 +export VERSION=0.0.1${RC} +export VERSION_WITHOUT_RC=${VERSION/rc?/} +export VERSION_BRANCH=${VERSION_WITHOUT_RC//./-} + +git checkout -b apache-iceberg-python-${VERSION_BRANCH} + +git tag -s ${VERSION} -m "Apache Iceberg Python ${VERSION}" + +export GIT_TAG=$(git show-ref ${VERSION}) +export GIT_TAG_HASH=${GIT_TAG:0:40} +export LAST_COMMIT_ID=$(git rev-list ${VERSION} 2> /dev/null | head -n 1) +``` + +The `-s` option will sign the commit. If you don't have a key yet, you can find the instructions [here](http://www.apache.org/dev/openpgp.html#key-gen-generate-key). To install gpg on a M1 based Mac, a couple of additional steps are required: https://gist.github.com/phortuin/cf24b1cca3258720c71ad42977e1ba57 + +Next we'll create a source distribution (`sdist`) which will generate a `.tar.gz` with all the source files. + +```bash +# Update the version +poetry version ${VERSION} + +git diff src/iceberg/__init__.py +git add src/iceberg/__init__.py +git commit -s -m "Set to version ${VERSION}" +``` + +Now we can stage the version in pypi and upload the files to the Apache SVN. + +Next we're going to build the artifacts: + +```bash +rm -rf dist/ +poetry build +``` + +This will create two artifacts: + +``` +Building apache-iceberg (0.1.0) + - Building sdist + - Built apache-iceberg-0.1.0.tar.gz + - Building wheel + - Built apache_iceberg-0.1.0-py3-none-any.whl +``` + +The `sdist` contains the source which can be used for checking licenses, and the wheel is a compiled version for quick installation. + +Next, we can upload them to pypi. Please keep in mind that this **won't** bump the version for everyone that hasn't pinned their version, since it is RC [pre-release and those are ignored](https://packaging.python.org/en/latest/guides/distributing-packages-using-setuptools/#pre-release-versioning). + +``` +twine upload dist/* +``` + +Before committing the files to the Apache SVN artifact distribution SVN, we need to generate hashes, and we need t o sign them using gpg: + +```bash +for name in "apache_iceberg-${VERSION}-py3-none-any.whl" "apache-iceberg-${VERSION}.tar.gz" +do + gpg --yes --armor --local-user fokko@apache.org --output "dist/${name}.asc" --detach-sig "dist/${name}" + shasum -a 512 "dist/${name}" > "dist/${name}.sha512" +done +``` + +Next, we'll clone the Apache SVN, copy and commit the files: + +```bash +export SVN_TMP_DIR=/tmp/iceberg-${VERSION_BRANCH}/ +svn checkout https://dist.apache.org/repos/dist/dev/iceberg $SVN_TMP_DIR + +export SVN_TMP_DIR_VERSIONED=${SVN_TMP_DIR}apache-iceberg-$VERSION/ +mkdir -p $SVN_TMP_DIR_VERSIONED +cp dist/* $SVN_TMP_DIR_VERSIONED +svn add $SVN_TMP_DIR_VERSIONED +svn ci -m "Apache Iceberg ${VERSION}" ${SVN_TMP_DIR_VERSIONED} +``` + +Finally, we can generate the email what we'll send to the mail list: + +```bash +cat << EOF > release-announcement-email.txt +To: dev@iceberg.apache.org +Subject: [VOTE] Release Apache Iceberg Python Client $VERSION +Hi Everyone, + +I propose that we release the following RC as the official Apache Iceberg Python Client $VERSION release. + +The commit ID is $LAST_COMMIT_ID + +* This corresponds to the tag: $GIT_TAG_HASH +* https://github.com/apache/iceberg/commits/apache-iceberg-python-$VERSION_BRANCH +* https://github.com/apache/iceberg/tree/$GIT_TAG_HASH + +The release tarball, signature, and checksums are here: + +* https://dist.apache.org/repos/dist/dev/iceberg/apache-iceberg-python-$VERSION/ + +You can find the KEYS file here: + +* https://dist.apache.org/repos/dist/dev/iceberg/KEYS + +Convenience binary artifacts are staged on pypi: + +https://pypi.org/project/apache-iceberg/$VERSION/ + +And can be installed using: pip install apache-iceberg==$VERSION + +Please download, verify, and test. +Please vote in the next 72 hours. +[ ] +1 Release this as Apache Iceberg Python Client $VERSION +[ ] +0 +[ ] -1 Do not release this because... +EOF + +cat release-announcement-email.txt +``` diff --git a/poetry.lock b/poetry.lock new file mode 100644 index 0000000000..0feeb1f76f --- /dev/null +++ b/poetry.lock @@ -0,0 +1,602 @@ +[[package]] +name = "atomicwrites" +version = "1.4.0" +description = "Atomic file writes." +category = "dev" +optional = false +python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*" + +[[package]] +name = "attrs" +version = "21.4.0" +description = "Classes Without Boilerplate" +category = "dev" +optional = false +python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*" + +[package.extras] +dev = ["coverage[toml] (>=5.0.2)", "hypothesis", "pympler", "pytest (>=4.3.0)", "six", "mypy", "pytest-mypy-plugins", "zope.interface", "furo", "sphinx", "sphinx-notfound-page", "pre-commit", "cloudpickle"] +docs = ["furo", "sphinx", "zope.interface", "sphinx-notfound-page"] +tests = ["coverage[toml] (>=5.0.2)", "hypothesis", "pympler", "pytest (>=4.3.0)", "six", "mypy", "pytest-mypy-plugins", "zope.interface", "cloudpickle"] +tests_no_zope = ["coverage[toml] (>=5.0.2)", "hypothesis", "pympler", "pytest (>=4.3.0)", "six", "mypy", "pytest-mypy-plugins", "cloudpickle"] + +[[package]] +name = "cfgv" +version = "3.3.1" +description = "Validate configuration and produce human readable error messages." +category = "dev" +optional = false +python-versions = ">=3.6.1" + +[[package]] +name = "colorama" +version = "0.4.4" +description = "Cross-platform colored terminal text." +category = "dev" +optional = false +python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*" + +[[package]] +name = "coverage" +version = "6.4" +description = "Code coverage measurement for Python" +category = "dev" +optional = false +python-versions = ">=3.7" + +[package.dependencies] +tomli = {version = "*", optional = true, markers = "python_version < \"3.11\" and extra == \"toml\""} + +[package.extras] +toml = ["tomli"] + +[[package]] +name = "distlib" +version = "0.3.4" +description = "Distribution utilities" +category = "dev" +optional = false +python-versions = "*" + +[[package]] +name = "docutils" +version = "0.18.1" +description = "Docutils -- Python Documentation Utilities" +category = "dev" +optional = false +python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*" + +[[package]] +name = "filelock" +version = "3.7.1" +description = "A platform independent file lock." +category = "dev" +optional = false +python-versions = ">=3.7" + +[package.extras] +docs = ["furo (>=2021.8.17b43)", "sphinx (>=4.1)", "sphinx-autodoc-typehints (>=1.12)"] +testing = ["covdefaults (>=1.2.0)", "coverage (>=4)", "pytest (>=4)", "pytest-cov", "pytest-timeout (>=1.4.2)"] + +[[package]] +name = "identify" +version = "2.5.1" +description = "File identification library for Python" +category = "dev" +optional = false +python-versions = ">=3.7" + +[package.extras] +license = ["ukkonen"] + +[[package]] +name = "importlib-metadata" +version = "4.11.4" +description = "Read metadata from Python packages" +category = "dev" +optional = false +python-versions = ">=3.7" + +[package.dependencies] +zipp = ">=0.5" + +[package.extras] +docs = ["sphinx", "jaraco.packaging (>=9)", "rst.linker (>=1.9)"] +perf = ["ipython"] +testing = ["pytest (>=6)", "pytest-checkdocs (>=2.4)", "pytest-flake8", "pytest-cov", "pytest-enabler (>=1.0.1)", "packaging", "pyfakefs", "flufl.flake8", "pytest-perf (>=0.9.2)", "pytest-black (>=0.3.7)", "pytest-mypy (>=0.9.1)", "importlib-resources (>=1.3)"] + +[[package]] +name = "iniconfig" +version = "1.1.1" +description = "iniconfig: brain-dead simple config-ini parsing" +category = "dev" +optional = false +python-versions = "*" + +[[package]] +name = "mmh3" +version = "3.0.0" +description = "Python wrapper for MurmurHash (MurmurHash3), a set of fast and robust hash functions." +category = "main" +optional = false +python-versions = "*" + +[[package]] +name = "nodeenv" +version = "1.6.0" +description = "Node.js virtual environment builder" +category = "dev" +optional = false +python-versions = "*" + +[[package]] +name = "numpy" +version = "1.22.4" +description = "NumPy is the fundamental package for array computing with Python." +category = "main" +optional = true +python-versions = ">=3.8" + +[[package]] +name = "packaging" +version = "21.3" +description = "Core utilities for Python packages" +category = "dev" +optional = false +python-versions = ">=3.6" + +[package.dependencies] +pyparsing = ">=2.0.2,<3.0.5 || >3.0.5" + +[[package]] +name = "pep517" +version = "0.12.0" +description = "Wrappers to build Python packages using PEP 517 hooks" +category = "dev" +optional = false +python-versions = "*" + +[package.dependencies] +tomli = {version = ">=1.1.0", markers = "python_version >= \"3.6\""} + +[[package]] +name = "platformdirs" +version = "2.5.2" +description = "A small Python module for determining appropriate platform-specific dirs, e.g. a \"user data dir\"." +category = "dev" +optional = false +python-versions = ">=3.7" + +[package.extras] +docs = ["furo (>=2021.7.5b38)", "proselint (>=0.10.2)", "sphinx-autodoc-typehints (>=1.12)", "sphinx (>=4)"] +test = ["appdirs (==1.4.4)", "pytest-cov (>=2.7)", "pytest-mock (>=3.6)", "pytest (>=6)"] + +[[package]] +name = "pluggy" +version = "1.0.0" +description = "plugin and hook calling mechanisms for python" +category = "dev" +optional = false +python-versions = ">=3.6" + +[package.extras] +dev = ["pre-commit", "tox"] +testing = ["pytest", "pytest-benchmark"] + +[[package]] +name = "pre-commit" +version = "2.19.0" +description = "A framework for managing and maintaining multi-language pre-commit hooks." +category = "dev" +optional = false +python-versions = ">=3.7" + +[package.dependencies] +cfgv = ">=2.0.0" +identify = ">=1.0.0" +nodeenv = ">=0.11.1" +pyyaml = ">=5.1" +toml = "*" +virtualenv = ">=20.0.8" + +[[package]] +name = "py" +version = "1.11.0" +description = "library with cross-python path, ini-parsing, io, code, log facilities" +category = "dev" +optional = false +python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*" + +[[package]] +name = "pyarrow" +version = "8.0.0" +description = "Python library for Apache Arrow" +category = "main" +optional = true +python-versions = ">=3.7" + +[package.dependencies] +numpy = ">=1.16.6" + +[[package]] +name = "pyparsing" +version = "3.0.9" +description = "pyparsing module - Classes and methods to define and execute parsing grammars" +category = "dev" +optional = false +python-versions = ">=3.6.8" + +[package.extras] +diagrams = ["railroad-diagrams", "jinja2"] + +[[package]] +name = "pytest" +version = "7.1.2" +description = "pytest: simple powerful testing with Python" +category = "dev" +optional = false +python-versions = ">=3.7" + +[package.dependencies] +atomicwrites = {version = ">=1.0", markers = "sys_platform == \"win32\""} +attrs = ">=19.2.0" +colorama = {version = "*", markers = "sys_platform == \"win32\""} +iniconfig = "*" +packaging = "*" +pluggy = ">=0.12,<2.0" +py = ">=1.8.2" +tomli = ">=1.0.0" + +[package.extras] +testing = ["argcomplete", "hypothesis (>=3.56)", "mock", "nose", "pygments (>=2.7.2)", "requests", "xmlschema"] + +[[package]] +name = "pytest-checkdocs" +version = "2.7.1" +description = "check the README when running tests" +category = "dev" +optional = false +python-versions = ">=3.6" + +[package.dependencies] +docutils = ">=0.15" +importlib-metadata = ">=4" +pep517 = "*" + +[package.extras] +docs = ["sphinx", "jaraco.packaging (>=8.2)", "rst.linker (>=1.9)"] +testing = ["pytest (>=4.6)", "pytest-checkdocs (>=2.4)", "pytest-flake8", "pytest-cov", "pytest-enabler (>=1.0.1)", "types-docutils", "pytest-black (>=0.3.7)", "pytest-mypy"] + +[[package]] +name = "pyyaml" +version = "6.0" +description = "YAML parser and emitter for Python" +category = "dev" +optional = false +python-versions = ">=3.6" + +[[package]] +name = "six" +version = "1.16.0" +description = "Python 2 and 3 compatibility utilities" +category = "dev" +optional = false +python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*" + +[[package]] +name = "toml" +version = "0.10.2" +description = "Python Library for Tom's Obvious, Minimal Language" +category = "dev" +optional = false +python-versions = ">=2.6, !=3.0.*, !=3.1.*, !=3.2.*" + +[[package]] +name = "tomli" +version = "2.0.1" +description = "A lil' TOML parser" +category = "dev" +optional = false +python-versions = ">=3.7" + +[[package]] +name = "virtualenv" +version = "20.14.1" +description = "Virtual Python Environment builder" +category = "dev" +optional = false +python-versions = "!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,!=3.4.*,>=2.7" + +[package.dependencies] +distlib = ">=0.3.1,<1" +filelock = ">=3.2,<4" +platformdirs = ">=2,<3" +six = ">=1.9.0,<2" + +[package.extras] +docs = ["proselint (>=0.10.2)", "sphinx (>=3)", "sphinx-argparse (>=0.2.5)", "sphinx-rtd-theme (>=0.4.3)", "towncrier (>=21.3)"] +testing = ["coverage (>=4)", "coverage-enable-subprocess (>=1)", "flaky (>=3)", "pytest (>=4)", "pytest-env (>=0.6.2)", "pytest-freezegun (>=0.4.1)", "pytest-mock (>=2)", "pytest-randomly (>=1)", "pytest-timeout (>=1)", "packaging (>=20.0)"] + +[[package]] +name = "zipp" +version = "3.8.0" +description = "Backport of pathlib-compatible object wrapper for zip files" +category = "dev" +optional = false +python-versions = ">=3.7" + +[package.extras] +docs = ["sphinx", "jaraco.packaging (>=9)", "rst.linker (>=1.9)"] +testing = ["pytest (>=6)", "pytest-checkdocs (>=2.4)", "pytest-flake8", "pytest-cov", "pytest-enabler (>=1.0.1)", "jaraco.itertools", "func-timeout", "pytest-black (>=0.3.7)", "pytest-mypy (>=0.9.1)"] + +[extras] +pyarrow = ["pyarrow"] + +[metadata] +lock-version = "1.1" +python-versions = "^3.8" +content-hash = "3e07f5fb1c8f204b0a004541776b7d90cad2afac926c2c8554df599b977429e9" + +[metadata.files] +atomicwrites = [ + {file = "atomicwrites-1.4.0-py2.py3-none-any.whl", hash = "sha256:6d1784dea7c0c8d4a5172b6c620f40b6e4cbfdf96d783691f2e1302a7b88e197"}, + {file = "atomicwrites-1.4.0.tar.gz", hash = "sha256:ae70396ad1a434f9c7046fd2dd196fc04b12f9e91ffb859164193be8b6168a7a"}, +] +attrs = [ + {file = "attrs-21.4.0-py2.py3-none-any.whl", hash = "sha256:2d27e3784d7a565d36ab851fe94887c5eccd6a463168875832a1be79c82828b4"}, + {file = "attrs-21.4.0.tar.gz", hash = "sha256:626ba8234211db98e869df76230a137c4c40a12d72445c45d5f5b716f076e2fd"}, +] +cfgv = [ + {file = "cfgv-3.3.1-py2.py3-none-any.whl", hash = "sha256:c6a0883f3917a037485059700b9e75da2464e6c27051014ad85ba6aaa5884426"}, + {file = "cfgv-3.3.1.tar.gz", hash = "sha256:f5a830efb9ce7a445376bb66ec94c638a9787422f96264c98edc6bdeed8ab736"}, +] +colorama = [ + {file = "colorama-0.4.4-py2.py3-none-any.whl", hash = "sha256:9f47eda37229f68eee03b24b9748937c7dc3868f906e8ba69fbcbdd3bc5dc3e2"}, + {file = "colorama-0.4.4.tar.gz", hash = "sha256:5941b2b48a20143d2267e95b1c2a7603ce057ee39fd88e7329b0c292aa16869b"}, +] +coverage = [ + {file = "coverage-6.4-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:50ed480b798febce113709846b11f5d5ed1e529c88d8ae92f707806c50297abf"}, + {file = "coverage-6.4-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:26f8f92699756cb7af2b30720de0c5bb8d028e923a95b6d0c891088025a1ac8f"}, + {file = "coverage-6.4-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:60c2147921da7f4d2d04f570e1838db32b95c5509d248f3fe6417e91437eaf41"}, + {file = "coverage-6.4-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:750e13834b597eeb8ae6e72aa58d1d831b96beec5ad1d04479ae3772373a8088"}, + {file = "coverage-6.4-cp310-cp310-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:af5b9ee0fc146e907aa0f5fb858c3b3da9199d78b7bb2c9973d95550bd40f701"}, + {file = "coverage-6.4-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:a022394996419142b33a0cf7274cb444c01d2bb123727c4bb0b9acabcb515dea"}, + {file = "coverage-6.4-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:5a78cf2c43b13aa6b56003707c5203f28585944c277c1f3f109c7b041b16bd39"}, + {file = "coverage-6.4-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:9229d074e097f21dfe0643d9d0140ee7433814b3f0fc3706b4abffd1e3038632"}, + {file = "coverage-6.4-cp310-cp310-win32.whl", hash = "sha256:fb45fe08e1abc64eb836d187b20a59172053999823f7f6ef4f18a819c44ba16f"}, + {file = "coverage-6.4-cp310-cp310-win_amd64.whl", hash = "sha256:3cfd07c5889ddb96a401449109a8b97a165be9d67077df6802f59708bfb07720"}, + {file = "coverage-6.4-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:03014a74023abaf5a591eeeaf1ac66a73d54eba178ff4cb1fa0c0a44aae70383"}, + {file = "coverage-6.4-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:9c82f2cd69c71698152e943f4a5a6b83a3ab1db73b88f6e769fabc86074c3b08"}, + {file = "coverage-6.4-cp37-cp37m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:7b546cf2b1974ddc2cb222a109b37c6ed1778b9be7e6b0c0bc0cf0438d9e45a6"}, + {file = "coverage-6.4-cp37-cp37m-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:cc173f1ce9ffb16b299f51c9ce53f66a62f4d975abe5640e976904066f3c835d"}, + {file = "coverage-6.4-cp37-cp37m-musllinux_1_1_aarch64.whl", hash = "sha256:c53ad261dfc8695062fc8811ac7c162bd6096a05a19f26097f411bdf5747aee7"}, + {file = "coverage-6.4-cp37-cp37m-musllinux_1_1_i686.whl", hash = "sha256:eef5292b60b6de753d6e7f2d128d5841c7915fb1e3321c3a1fe6acfe76c38052"}, + {file = "coverage-6.4-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:543e172ce4c0de533fa892034cce260467b213c0ea8e39da2f65f9a477425211"}, + {file = "coverage-6.4-cp37-cp37m-win32.whl", hash = "sha256:00c8544510f3c98476bbd58201ac2b150ffbcce46a8c3e4fb89ebf01998f806a"}, + {file = "coverage-6.4-cp37-cp37m-win_amd64.whl", hash = "sha256:b84ab65444dcc68d761e95d4d70f3cfd347ceca5a029f2ffec37d4f124f61311"}, + {file = "coverage-6.4-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:d548edacbf16a8276af13063a2b0669d58bbcfca7c55a255f84aac2870786a61"}, + {file = "coverage-6.4-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:033ebec282793bd9eb988d0271c211e58442c31077976c19c442e24d827d356f"}, + {file = "coverage-6.4-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:742fb8b43835078dd7496c3c25a1ec8d15351df49fb0037bffb4754291ef30ce"}, + {file = "coverage-6.4-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:d55fae115ef9f67934e9f1103c9ba826b4c690e4c5bcf94482b8b2398311bf9c"}, + {file = "coverage-6.4-cp38-cp38-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:5cd698341626f3c77784858427bad0cdd54a713115b423d22ac83a28303d1d95"}, + {file = "coverage-6.4-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:62d382f7d77eeeaff14b30516b17bcbe80f645f5cf02bb755baac376591c653c"}, + {file = "coverage-6.4-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:016d7f5cf1c8c84f533a3c1f8f36126fbe00b2ec0ccca47cc5731c3723d327c6"}, + {file = "coverage-6.4-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:69432946f154c6add0e9ede03cc43b96e2ef2733110a77444823c053b1ff5166"}, + {file = "coverage-6.4-cp38-cp38-win32.whl", hash = "sha256:83bd142cdec5e4a5c4ca1d4ff6fa807d28460f9db919f9f6a31babaaa8b88426"}, + {file = "coverage-6.4-cp38-cp38-win_amd64.whl", hash = "sha256:4002f9e8c1f286e986fe96ec58742b93484195defc01d5cc7809b8f7acb5ece3"}, + {file = "coverage-6.4-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:e4f52c272fdc82e7c65ff3f17a7179bc5f710ebc8ce8a5cadac81215e8326740"}, + {file = "coverage-6.4-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:b5578efe4038be02d76c344007b13119b2b20acd009a88dde8adec2de4f630b5"}, + {file = "coverage-6.4-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:d8099ea680201c2221f8468c372198ceba9338a5fec0e940111962b03b3f716a"}, + {file = "coverage-6.4-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:a00441f5ea4504f5abbc047589d09e0dc33eb447dc45a1a527c8b74bfdd32c65"}, + {file = "coverage-6.4-cp39-cp39-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:2e76bd16f0e31bc2b07e0fb1379551fcd40daf8cdf7e24f31a29e442878a827c"}, + {file = "coverage-6.4-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:8d2e80dd3438e93b19e1223a9850fa65425e77f2607a364b6fd134fcd52dc9df"}, + {file = "coverage-6.4-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:341e9c2008c481c5c72d0e0dbf64980a4b2238631a7f9780b0fe2e95755fb018"}, + {file = "coverage-6.4-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:21e6686a95025927775ac501e74f5940cdf6fe052292f3a3f7349b0abae6d00f"}, + {file = "coverage-6.4-cp39-cp39-win32.whl", hash = "sha256:968ed5407f9460bd5a591cefd1388cc00a8f5099de9e76234655ae48cfdbe2c3"}, + {file = "coverage-6.4-cp39-cp39-win_amd64.whl", hash = "sha256:e35217031e4b534b09f9b9a5841b9344a30a6357627761d4218818b865d45055"}, + {file = "coverage-6.4-pp36.pp37.pp38-none-any.whl", hash = "sha256:e637ae0b7b481905358624ef2e81d7fb0b1af55f5ff99f9ba05442a444b11e45"}, + {file = "coverage-6.4.tar.gz", hash = "sha256:727dafd7f67a6e1cad808dc884bd9c5a2f6ef1f8f6d2f22b37b96cb0080d4f49"}, +] +distlib = [ + {file = "distlib-0.3.4-py2.py3-none-any.whl", hash = "sha256:6564fe0a8f51e734df6333d08b8b94d4ea8ee6b99b5ed50613f731fd4089f34b"}, + {file = "distlib-0.3.4.zip", hash = "sha256:e4b58818180336dc9c529bfb9a0b58728ffc09ad92027a3f30b7cd91e3458579"}, +] +docutils = [ + {file = "docutils-0.18.1-py2.py3-none-any.whl", hash = "sha256:23010f129180089fbcd3bc08cfefccb3b890b0050e1ca00c867036e9d161b98c"}, + {file = "docutils-0.18.1.tar.gz", hash = "sha256:679987caf361a7539d76e584cbeddc311e3aee937877c87346f31debc63e9d06"}, +] +filelock = [ + {file = "filelock-3.7.1-py3-none-any.whl", hash = "sha256:37def7b658813cda163b56fc564cdc75e86d338246458c4c28ae84cabefa2404"}, + {file = "filelock-3.7.1.tar.gz", hash = "sha256:3a0fd85166ad9dbab54c9aec96737b744106dc5f15c0b09a6744a445299fcf04"}, +] +identify = [ + {file = "identify-2.5.1-py2.py3-none-any.whl", hash = "sha256:0dca2ea3e4381c435ef9c33ba100a78a9b40c0bab11189c7cf121f75815efeaa"}, + {file = "identify-2.5.1.tar.gz", hash = "sha256:3d11b16f3fe19f52039fb7e39c9c884b21cb1b586988114fbe42671f03de3e82"}, +] +importlib-metadata = [ + {file = "importlib_metadata-4.11.4-py3-none-any.whl", hash = "sha256:c58c8eb8a762858f49e18436ff552e83914778e50e9d2f1660535ffb364552ec"}, + {file = "importlib_metadata-4.11.4.tar.gz", hash = "sha256:5d26852efe48c0a32b0509ffbc583fda1a2266545a78d104a6f4aff3db17d700"}, +] +iniconfig = [ + {file = "iniconfig-1.1.1-py2.py3-none-any.whl", hash = "sha256:011e24c64b7f47f6ebd835bb12a743f2fbe9a26d4cecaa7f53bc4f35ee9da8b3"}, + {file = "iniconfig-1.1.1.tar.gz", hash = "sha256:bc3af051d7d14b2ee5ef9969666def0cd1a000e121eaea580d4a313df4b37f32"}, +] +mmh3 = [ + {file = "mmh3-3.0.0-cp36-cp36m-macosx_10_9_x86_64.whl", hash = "sha256:23912dde2ad4f701926948dd8e79a0e42b000f73962806f153931f52985e1e07"}, + {file = "mmh3-3.0.0-cp36-cp36m-manylinux1_x86_64.whl", hash = "sha256:07f1308a410dc406d6a3c282a685728d00a87f3ed684f012671b96d6cc6a41c3"}, + {file = "mmh3-3.0.0-cp36-cp36m-manylinux2010_x86_64.whl", hash = "sha256:167cbc2b5ae27f3bccd797a2e8a9e7561791bee4cc2885f2c140eedc5df000ef"}, + {file = "mmh3-3.0.0-cp36-cp36m-manylinux2014_aarch64.whl", hash = "sha256:8fb833c2942917eff54f984b067d93e5a3c54dbb00720323460cdfed9292835f"}, + {file = "mmh3-3.0.0-cp36-cp36m-win32.whl", hash = "sha256:b7d26d0243ed9a5b8bf7aa8c53697cb79dff1e1d207f42396b7a7cb2a62298b7"}, + {file = "mmh3-3.0.0-cp36-cp36m-win_amd64.whl", hash = "sha256:2b6c79fc314b34b911245b460a79b601fff39bb807521fb7ed7c15cacf0394ac"}, + {file = "mmh3-3.0.0-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:6d0b3e9def1fdfe4eadd35ee26bf72bd715ba97711f7101302d54c9d2e70ba27"}, + {file = "mmh3-3.0.0-cp37-cp37m-manylinux1_x86_64.whl", hash = "sha256:8803d28c17cf898f5f00c0433e8b13d51fa3bb4ebecf59872ba1eaa20d94128a"}, + {file = "mmh3-3.0.0-cp37-cp37m-manylinux2010_x86_64.whl", hash = "sha256:01e456edf9cc381298a590923aadd1c0bf9934d93433099a5001d656112437c2"}, + {file = "mmh3-3.0.0-cp37-cp37m-manylinux2014_aarch64.whl", hash = "sha256:ff69ddc2d46e3e42720840b6b4f7bfb032fd1e677fac347fdfff6e4d9fd01212"}, + {file = "mmh3-3.0.0-cp37-cp37m-win32.whl", hash = "sha256:e08a5d81a2ff53625953290187bed4ae96a6972e2b5cd5984a6ebc5a9aab256c"}, + {file = "mmh3-3.0.0-cp37-cp37m-win_amd64.whl", hash = "sha256:12484ac80373db77d8a6beb7615e7dac8b6c3fb118905311a51450b4fc4a24d1"}, + {file = "mmh3-3.0.0-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:93c96e657e9bf9e9ef12ddaeae9f109c0b3134146e2eff2cbddde5a34190920e"}, + {file = "mmh3-3.0.0-cp38-cp38-manylinux1_x86_64.whl", hash = "sha256:9097be65aa95460bc68b6108601da8894757532450daf74034e4eaecd536acca"}, + {file = "mmh3-3.0.0-cp38-cp38-manylinux2010_x86_64.whl", hash = "sha256:19874e12acb4119ef1ef83062ef4ac953c3343dd07a67ede8fa096d0393f34be"}, + {file = "mmh3-3.0.0-cp38-cp38-manylinux2014_aarch64.whl", hash = "sha256:4589adcb609d1547aac7c1ac1064eb27cdd44b65b7e8a114e2971cd3b7110306"}, + {file = "mmh3-3.0.0-cp38-cp38-win32.whl", hash = "sha256:7a311efd4ecf122f21392ec6bf447c620cc783d20bdb9aec60bb469a54318419"}, + {file = "mmh3-3.0.0-cp38-cp38-win_amd64.whl", hash = "sha256:3566d1455fa4a09f8fb1aa5b37f68914949674f9aa2bd630e9fdf344207f55b5"}, + {file = "mmh3-3.0.0-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:92fdffd63edb67c30dbaba18a7448d762209c0e678b0c9d577d17b30362b59a3"}, + {file = "mmh3-3.0.0-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:3e52b869572c09db0c1a483f6e9cedbccfae8a282d95e552d3d4bd0712ab3196"}, + {file = "mmh3-3.0.0-cp39-cp39-manylinux1_x86_64.whl", hash = "sha256:f1cce018cc82a8a6287e6aeb139e441129837b810f2ddf372e3ff7f0fefb0947"}, + {file = "mmh3-3.0.0-cp39-cp39-manylinux2010_x86_64.whl", hash = "sha256:0fd09c4b61fcddbcf0a87d5463b4e6d2919896736a67efc5248d5c74c1c9c742"}, + {file = "mmh3-3.0.0-cp39-cp39-manylinux2014_aarch64.whl", hash = "sha256:c17fe2e276edd37ad8a6aff3b1663d3479c2c5c5993539c1050422a1dae33033"}, + {file = "mmh3-3.0.0-cp39-cp39-win32.whl", hash = "sha256:150439b906b4deaf6d796b2c2d11fb6159f08d02330d97723071ab3bf43b51df"}, + {file = "mmh3-3.0.0-cp39-cp39-win_amd64.whl", hash = "sha256:bd870aedd9189eff1cf4e1687869b56c7e9461ee869789139c3e704009e5c227"}, + {file = "mmh3-3.0.0.tar.gz", hash = "sha256:d1ec578c09a07d3518ec9be540b87546397fa3455de73c166fcce51eaa5c41c5"}, +] +nodeenv = [ + {file = "nodeenv-1.6.0-py2.py3-none-any.whl", hash = "sha256:621e6b7076565ddcacd2db0294c0381e01fd28945ab36bcf00f41c5daf63bef7"}, + {file = "nodeenv-1.6.0.tar.gz", hash = "sha256:3ef13ff90291ba2a4a7a4ff9a979b63ffdd00a464dbe04acf0ea6471517a4c2b"}, +] +numpy = [ + {file = "numpy-1.22.4-cp310-cp310-macosx_10_14_x86_64.whl", hash = "sha256:ba9ead61dfb5d971d77b6c131a9dbee62294a932bf6a356e48c75ae684e635b3"}, + {file = "numpy-1.22.4-cp310-cp310-macosx_10_15_x86_64.whl", hash = "sha256:1ce7ab2053e36c0a71e7a13a7475bd3b1f54750b4b433adc96313e127b870887"}, + {file = "numpy-1.22.4-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:7228ad13744f63575b3a972d7ee4fd61815b2879998e70930d4ccf9ec721dce0"}, + {file = "numpy-1.22.4-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:43a8ca7391b626b4c4fe20aefe79fec683279e31e7c79716863b4b25021e0e74"}, + {file = "numpy-1.22.4-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:a911e317e8c826ea632205e63ed8507e0dc877dcdc49744584dfc363df9ca08c"}, + {file = "numpy-1.22.4-cp310-cp310-win32.whl", hash = "sha256:9ce7df0abeabe7fbd8ccbf343dc0db72f68549856b863ae3dd580255d009648e"}, + {file = "numpy-1.22.4-cp310-cp310-win_amd64.whl", hash = "sha256:3e1ffa4748168e1cc8d3cde93f006fe92b5421396221a02f2274aab6ac83b077"}, + {file = "numpy-1.22.4-cp38-cp38-macosx_10_15_x86_64.whl", hash = "sha256:59d55e634968b8f77d3fd674a3cf0b96e85147cd6556ec64ade018f27e9479e1"}, + {file = "numpy-1.22.4-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:c1d937820db6e43bec43e8d016b9b3165dcb42892ea9f106c70fb13d430ffe72"}, + {file = "numpy-1.22.4-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:d4c5d5eb2ec8da0b4f50c9a843393971f31f1d60be87e0fb0917a49133d257d6"}, + {file = "numpy-1.22.4-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:64f56fc53a2d18b1924abd15745e30d82a5782b2cab3429aceecc6875bd5add0"}, + {file = "numpy-1.22.4-cp38-cp38-win32.whl", hash = "sha256:fb7a980c81dd932381f8228a426df8aeb70d59bbcda2af075b627bbc50207cba"}, + {file = "numpy-1.22.4-cp38-cp38-win_amd64.whl", hash = "sha256:e96d7f3096a36c8754207ab89d4b3282ba7b49ea140e4973591852c77d09eb76"}, + {file = "numpy-1.22.4-cp39-cp39-macosx_10_14_x86_64.whl", hash = "sha256:4c6036521f11a731ce0648f10c18ae66d7143865f19f7299943c985cdc95afb5"}, + {file = "numpy-1.22.4-cp39-cp39-macosx_10_15_x86_64.whl", hash = "sha256:b89bf9b94b3d624e7bb480344e91f68c1c6c75f026ed6755955117de00917a7c"}, + {file = "numpy-1.22.4-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:2d487e06ecbf1dc2f18e7efce82ded4f705f4bd0cd02677ffccfb39e5c284c7e"}, + {file = "numpy-1.22.4-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:f3eb268dbd5cfaffd9448113539e44e2dd1c5ca9ce25576f7c04a5453edc26fa"}, + {file = "numpy-1.22.4-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:37431a77ceb9307c28382c9773da9f306435135fae6b80b62a11c53cfedd8802"}, + {file = "numpy-1.22.4-cp39-cp39-win32.whl", hash = "sha256:cc7f00008eb7d3f2489fca6f334ec19ca63e31371be28fd5dad955b16ec285bd"}, + {file = "numpy-1.22.4-cp39-cp39-win_amd64.whl", hash = "sha256:f0725df166cf4785c0bc4cbfb320203182b1ecd30fee6e541c8752a92df6aa32"}, + {file = "numpy-1.22.4-pp38-pypy38_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:0791fbd1e43bf74b3502133207e378901272f3c156c4df4954cad833b1380207"}, + {file = "numpy-1.22.4.zip", hash = "sha256:425b390e4619f58d8526b3dcf656dde069133ae5c240229821f01b5f44ea07af"}, +] +packaging = [ + {file = "packaging-21.3-py3-none-any.whl", hash = "sha256:ef103e05f519cdc783ae24ea4e2e0f508a9c99b2d4969652eed6a2e1ea5bd522"}, + {file = "packaging-21.3.tar.gz", hash = "sha256:dd47c42927d89ab911e606518907cc2d3a1f38bbd026385970643f9c5b8ecfeb"}, +] +pep517 = [ + {file = "pep517-0.12.0-py2.py3-none-any.whl", hash = "sha256:dd884c326898e2c6e11f9e0b64940606a93eb10ea022a2e067959f3a110cf161"}, + {file = "pep517-0.12.0.tar.gz", hash = "sha256:931378d93d11b298cf511dd634cf5ea4cb249a28ef84160b3247ee9afb4e8ab0"}, +] +platformdirs = [ + {file = "platformdirs-2.5.2-py3-none-any.whl", hash = "sha256:027d8e83a2d7de06bbac4e5ef7e023c02b863d7ea5d079477e722bb41ab25788"}, + {file = "platformdirs-2.5.2.tar.gz", hash = "sha256:58c8abb07dcb441e6ee4b11d8df0ac856038f944ab98b7be6b27b2a3c7feef19"}, +] +pluggy = [ + {file = "pluggy-1.0.0-py2.py3-none-any.whl", hash = "sha256:74134bbf457f031a36d68416e1509f34bd5ccc019f0bcc952c7b909d06b37bd3"}, + {file = "pluggy-1.0.0.tar.gz", hash = "sha256:4224373bacce55f955a878bf9cfa763c1e360858e330072059e10bad68531159"}, +] +pre-commit = [ + {file = "pre_commit-2.19.0-py2.py3-none-any.whl", hash = "sha256:10c62741aa5704faea2ad69cb550ca78082efe5697d6f04e5710c3c229afdd10"}, + {file = "pre_commit-2.19.0.tar.gz", hash = "sha256:4233a1e38621c87d9dda9808c6606d7e7ba0e087cd56d3fe03202a01d2919615"}, +] +py = [ + {file = "py-1.11.0-py2.py3-none-any.whl", hash = "sha256:607c53218732647dff4acdfcd50cb62615cedf612e72d1724fb1a0cc6405b378"}, + {file = "py-1.11.0.tar.gz", hash = "sha256:51c75c4126074b472f746a24399ad32f6053d1b34b68d2fa41e558e6f4a98719"}, +] +pyarrow = [ + {file = "pyarrow-8.0.0-cp310-cp310-macosx_10_13_universal2.whl", hash = "sha256:d5ef4372559b191cafe7db8932801eee252bfc35e983304e7d60b6954576a071"}, + {file = "pyarrow-8.0.0-cp310-cp310-macosx_10_13_x86_64.whl", hash = "sha256:863be6bad6c53797129610930794a3e797cb7d41c0a30e6794a2ac0e42ce41b8"}, + {file = "pyarrow-8.0.0-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:69b043a3fce064ebd9fbae6abc30e885680296e5bd5e6f7353e6a87966cf2ad7"}, + {file = "pyarrow-8.0.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:51e58778fcb8829fca37fbfaea7f208d5ce7ea89ea133dd13d8ce745278ee6f0"}, + {file = "pyarrow-8.0.0-cp310-cp310-manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:15511ce2f50343f3fd5e9f7c30e4d004da9134e9597e93e9c96c3985928cbe82"}, + {file = "pyarrow-8.0.0-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:ea132067ec712d1b1116a841db1c95861508862b21eddbcafefbce8e4b96b867"}, + {file = "pyarrow-8.0.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:deb400df8f19a90b662babceb6dd12daddda6bb357c216e558b207c0770c7654"}, + {file = "pyarrow-8.0.0-cp310-cp310-win_amd64.whl", hash = "sha256:3bd201af6e01f475f02be88cf1f6ee9856ab98c11d8bbb6f58347c58cd07be00"}, + {file = "pyarrow-8.0.0-cp37-cp37m-macosx_10_13_x86_64.whl", hash = "sha256:78a6ac39cd793582998dac88ab5c1c1dd1e6503df6672f064f33a21937ec1d8d"}, + {file = "pyarrow-8.0.0-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:d6f1e1040413651819074ef5b500835c6c42e6c446532a1ddef8bc5054e8dba5"}, + {file = "pyarrow-8.0.0-cp37-cp37m-manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:98c13b2e28a91b0fbf24b483df54a8d7814c074c2623ecef40dce1fa52f6539b"}, + {file = "pyarrow-8.0.0-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:c9c97c8e288847e091dfbcdf8ce51160e638346f51919a9e74fe038b2e8aee62"}, + {file = "pyarrow-8.0.0-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:edad25522ad509e534400d6ab98cf1872d30c31bc5e947712bfd57def7af15bb"}, + {file = "pyarrow-8.0.0-cp37-cp37m-win_amd64.whl", hash = "sha256:ece333706a94c1221ced8b299042f85fd88b5db802d71be70024433ddf3aecab"}, + {file = "pyarrow-8.0.0-cp38-cp38-macosx_10_13_x86_64.whl", hash = "sha256:95c7822eb37663e073da9892f3499fe28e84f3464711a3e555e0c5463fd53a19"}, + {file = "pyarrow-8.0.0-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:25a5f7c7f36df520b0b7363ba9f51c3070799d4b05d587c60c0adaba57763479"}, + {file = "pyarrow-8.0.0-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:ce64bc1da3109ef5ab9e4c60316945a7239c798098a631358e9ab39f6e5529e9"}, + {file = "pyarrow-8.0.0-cp38-cp38-manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:541e7845ce5f27a861eb5b88ee165d931943347eec17b9ff1e308663531c9647"}, + {file = "pyarrow-8.0.0-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:8cd86e04a899bef43e25184f4b934584861d787cf7519851a8c031803d45c6d8"}, + {file = "pyarrow-8.0.0-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:ba2b7aa7efb59156b87987a06f5241932914e4d5bbb74a465306b00a6c808849"}, + {file = "pyarrow-8.0.0-cp38-cp38-win_amd64.whl", hash = "sha256:42b7982301a9ccd06e1dd4fabd2e8e5df74b93ce4c6b87b81eb9e2d86dc79871"}, + {file = "pyarrow-8.0.0-cp39-cp39-macosx_10_13_universal2.whl", hash = "sha256:1dd482ccb07c96188947ad94d7536ab696afde23ad172df8e18944ec79f55055"}, + {file = "pyarrow-8.0.0-cp39-cp39-macosx_10_13_x86_64.whl", hash = "sha256:81b87b782a1366279411f7b235deab07c8c016e13f9af9f7c7b0ee564fedcc8f"}, + {file = "pyarrow-8.0.0-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:03a10daad957970e914920b793f6a49416699e791f4c827927fd4e4d892a5d16"}, + {file = "pyarrow-8.0.0-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:65c7f4cc2be195e3db09296d31a654bb6d8786deebcab00f0e2455fd109d7456"}, + {file = "pyarrow-8.0.0-cp39-cp39-manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:3fee786259d986f8c046100ced54d63b0c8c9f7cdb7d1bbe07dc69e0f928141c"}, + {file = "pyarrow-8.0.0-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:6ea2c54e6b5ecd64e8299d2abb40770fe83a718f5ddc3825ddd5cd28e352cce1"}, + {file = "pyarrow-8.0.0-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:8392b9a1e837230090fe916415ed4c3433b2ddb1a798e3f6438303c70fbabcfc"}, + {file = "pyarrow-8.0.0-cp39-cp39-win_amd64.whl", hash = "sha256:cb06cacc19f3b426681f2f6803cc06ff481e7fe5b3a533b406bc5b2138843d4f"}, + {file = "pyarrow-8.0.0.tar.gz", hash = "sha256:4a18a211ed888f1ac0b0ebcb99e2d9a3e913a481120ee9b1fe33d3fedb945d4e"}, +] +pyparsing = [ + {file = "pyparsing-3.0.9-py3-none-any.whl", hash = "sha256:5026bae9a10eeaefb61dab2f09052b9f4307d44aee4eda64b309723d8d206bbc"}, + {file = "pyparsing-3.0.9.tar.gz", hash = "sha256:2b020ecf7d21b687f219b71ecad3631f644a47f01403fa1d1036b0c6416d70fb"}, +] +pytest = [ + {file = "pytest-7.1.2-py3-none-any.whl", hash = "sha256:13d0e3ccfc2b6e26be000cb6568c832ba67ba32e719443bfe725814d3c42433c"}, + {file = "pytest-7.1.2.tar.gz", hash = "sha256:a06a0425453864a270bc45e71f783330a7428defb4230fb5e6a731fde06ecd45"}, +] +pytest-checkdocs = [ + {file = "pytest-checkdocs-2.7.1.tar.gz", hash = "sha256:2b33b85eddfe5846a69bea4a759303e2d5a3be11d03bc7149f5ba1ef47e6c1ae"}, + {file = "pytest_checkdocs-2.7.1-py3-none-any.whl", hash = "sha256:294898c64c9ce1a178edc6660e48da23c7543bfd5a1cea7f0ca4c167745d8461"}, +] +pyyaml = [ + {file = "PyYAML-6.0-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:d4db7c7aef085872ef65a8fd7d6d09a14ae91f691dec3e87ee5ee0539d516f53"}, + {file = "PyYAML-6.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:9df7ed3b3d2e0ecfe09e14741b857df43adb5a3ddadc919a2d94fbdf78fea53c"}, + {file = "PyYAML-6.0-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:77f396e6ef4c73fdc33a9157446466f1cff553d979bd00ecb64385760c6babdc"}, + {file = "PyYAML-6.0-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:a80a78046a72361de73f8f395f1f1e49f956c6be882eed58505a15f3e430962b"}, + {file = "PyYAML-6.0-cp310-cp310-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:f84fbc98b019fef2ee9a1cb3ce93e3187a6df0b2538a651bfb890254ba9f90b5"}, + {file = "PyYAML-6.0-cp310-cp310-win32.whl", hash = "sha256:2cd5df3de48857ed0544b34e2d40e9fac445930039f3cfe4bcc592a1f836d513"}, + {file = "PyYAML-6.0-cp310-cp310-win_amd64.whl", hash = "sha256:daf496c58a8c52083df09b80c860005194014c3698698d1a57cbcfa182142a3a"}, + {file = "PyYAML-6.0-cp36-cp36m-macosx_10_9_x86_64.whl", hash = "sha256:897b80890765f037df3403d22bab41627ca8811ae55e9a722fd0392850ec4d86"}, + {file = "PyYAML-6.0-cp36-cp36m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:50602afada6d6cbfad699b0c7bb50d5ccffa7e46a3d738092afddc1f9758427f"}, + {file = "PyYAML-6.0-cp36-cp36m-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:48c346915c114f5fdb3ead70312bd042a953a8ce5c7106d5bfb1a5254e47da92"}, + {file = "PyYAML-6.0-cp36-cp36m-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:98c4d36e99714e55cfbaaee6dd5badbc9a1ec339ebfc3b1f52e293aee6bb71a4"}, + {file = "PyYAML-6.0-cp36-cp36m-win32.whl", hash = "sha256:0283c35a6a9fbf047493e3a0ce8d79ef5030852c51e9d911a27badfde0605293"}, + {file = "PyYAML-6.0-cp36-cp36m-win_amd64.whl", hash = "sha256:07751360502caac1c067a8132d150cf3d61339af5691fe9e87803040dbc5db57"}, + {file = "PyYAML-6.0-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:819b3830a1543db06c4d4b865e70ded25be52a2e0631ccd2f6a47a2822f2fd7c"}, + {file = "PyYAML-6.0-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:473f9edb243cb1935ab5a084eb238d842fb8f404ed2193a915d1784b5a6b5fc0"}, + {file = "PyYAML-6.0-cp37-cp37m-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:0ce82d761c532fe4ec3f87fc45688bdd3a4c1dc5e0b4a19814b9009a29baefd4"}, + {file = "PyYAML-6.0-cp37-cp37m-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:231710d57adfd809ef5d34183b8ed1eeae3f76459c18fb4a0b373ad56bedcdd9"}, + {file = "PyYAML-6.0-cp37-cp37m-win32.whl", hash = "sha256:c5687b8d43cf58545ade1fe3e055f70eac7a5a1a0bf42824308d868289a95737"}, + {file = "PyYAML-6.0-cp37-cp37m-win_amd64.whl", hash = "sha256:d15a181d1ecd0d4270dc32edb46f7cb7733c7c508857278d3d378d14d606db2d"}, + {file = "PyYAML-6.0-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:0b4624f379dab24d3725ffde76559cff63d9ec94e1736b556dacdfebe5ab6d4b"}, + {file = "PyYAML-6.0-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:213c60cd50106436cc818accf5baa1aba61c0189ff610f64f4a3e8c6726218ba"}, + {file = "PyYAML-6.0-cp38-cp38-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:9fa600030013c4de8165339db93d182b9431076eb98eb40ee068700c9c813e34"}, + {file = "PyYAML-6.0-cp38-cp38-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:277a0ef2981ca40581a47093e9e2d13b3f1fbbeffae064c1d21bfceba2030287"}, + {file = "PyYAML-6.0-cp38-cp38-win32.whl", hash = "sha256:d4eccecf9adf6fbcc6861a38015c2a64f38b9d94838ac1810a9023a0609e1b78"}, + {file = "PyYAML-6.0-cp38-cp38-win_amd64.whl", hash = "sha256:1e4747bc279b4f613a09eb64bba2ba602d8a6664c6ce6396a4d0cd413a50ce07"}, + {file = "PyYAML-6.0-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:055d937d65826939cb044fc8c9b08889e8c743fdc6a32b33e2390f66013e449b"}, + {file = "PyYAML-6.0-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:e61ceaab6f49fb8bdfaa0f92c4b57bcfbea54c09277b1b4f7ac376bfb7a7c174"}, + {file = "PyYAML-6.0-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:d67d839ede4ed1b28a4e8909735fc992a923cdb84e618544973d7dfc71540803"}, + {file = "PyYAML-6.0-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:cba8c411ef271aa037d7357a2bc8f9ee8b58b9965831d9e51baf703280dc73d3"}, + {file = "PyYAML-6.0-cp39-cp39-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:40527857252b61eacd1d9af500c3337ba8deb8fc298940291486c465c8b46ec0"}, + {file = "PyYAML-6.0-cp39-cp39-win32.whl", hash = "sha256:b5b9eccad747aabaaffbc6064800670f0c297e52c12754eb1d976c57e4f74dcb"}, + {file = "PyYAML-6.0-cp39-cp39-win_amd64.whl", hash = "sha256:b3d267842bf12586ba6c734f89d1f5b871df0273157918b0ccefa29deb05c21c"}, + {file = "PyYAML-6.0.tar.gz", hash = "sha256:68fb519c14306fec9720a2a5b45bc9f0c8d1b9c72adf45c37baedfcd949c35a2"}, +] +six = [ + {file = "six-1.16.0-py2.py3-none-any.whl", hash = "sha256:8abb2f1d86890a2dfb989f9a77cfcfd3e47c2a354b01111771326f8aa26e0254"}, + {file = "six-1.16.0.tar.gz", hash = "sha256:1e61c37477a1626458e36f7b1d82aa5c9b094fa4802892072e49de9c60c4c926"}, +] +toml = [ + {file = "toml-0.10.2-py2.py3-none-any.whl", hash = "sha256:806143ae5bfb6a3c6e736a764057db0e6a0e05e338b5630894a5f779cabb4f9b"}, + {file = "toml-0.10.2.tar.gz", hash = "sha256:b3bda1d108d5dd99f4a20d24d9c348e91c4db7ab1b749200bded2f839ccbe68f"}, +] +tomli = [ + {file = "tomli-2.0.1-py3-none-any.whl", hash = "sha256:939de3e7a6161af0c887ef91b7d41a53e7c5a1ca976325f429cb46ea9bc30ecc"}, + {file = "tomli-2.0.1.tar.gz", hash = "sha256:de526c12914f0c550d15924c62d72abc48d6fe7364aa87328337a31007fe8a4f"}, +] +virtualenv = [ + {file = "virtualenv-20.14.1-py2.py3-none-any.whl", hash = "sha256:e617f16e25b42eb4f6e74096b9c9e37713cf10bf30168fb4a739f3fa8f898a3a"}, + {file = "virtualenv-20.14.1.tar.gz", hash = "sha256:ef589a79795589aada0c1c5b319486797c03b67ac3984c48c669c0e4f50df3a5"}, +] +zipp = [ + {file = "zipp-3.8.0-py3-none-any.whl", hash = "sha256:c4f6e5bbf48e74f7a38e7cc5b0480ff42b0ae5178957d564d18932525d5cf099"}, + {file = "zipp-3.8.0.tar.gz", hash = "sha256:56bf8aadb83c24db6c4b577e13de374ccfb67da2078beba1d037c17980bf43ad"}, +] diff --git a/pyproject.toml b/pyproject.toml index 09b810c40f..b845f8b017 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -15,12 +15,46 @@ # specific language governing permissions and limitations # under the License. -[build-system] -requires = [ - "setuptools>=42", - "wheel" +[tool.poetry] +name = "apache-iceberg" +version = "0.0.1rc1" +readme = "README.md" +homepage = "https://iceberg.apache.org/" +repository = "https://github.com/apache/iceberg/" +description = "Apache Iceberg is an open table format for huge analytic datasets" +authors = ["Apache Software Foundation "] +license = "Apache License 2.0" + +classifiers = [ + "License :: OSI Approved :: Apache Software License", + "Operating System :: OS Independent", + "Programming Language :: Python :: 3.8", + "Programming Language :: Python :: 3.9", + "Programming Language :: Python :: 3.10" ] -build-backend = "setuptools.build_meta" + +packages = [ + { include = "iceberg", from = "src" }, +] + +[tool.poetry.dependencies] +python = "^3.8" +mmh3 = "^3.0.0" + +pyarrow = { version = "^8.0.0", optional = true } + +[tool.poetry.dev-dependencies] +pytest = "^7.0.0" +pytest-checkdocs = "^2.0.0" +pre-commit = "^2.0.0" +coverage = { version = "^6.0.0", extras = ["toml"] } + +[build-system] +requires = ["poetry-core>=1.0.0"] +build-backend = "poetry.core.masonry.api" + +[tool.poetry.extras] +pyarrow = ["pyarrow"] [tool.black] line-length = 130 diff --git a/setup.cfg b/setup.cfg deleted file mode 100644 index 6a983275fd..0000000000 --- a/setup.cfg +++ /dev/null @@ -1,59 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -[metadata] -name = py-iceberg -version = 0.1.0a1 -author = Apache Software Foundation -author_email = dev@iceberg.apache.org -description = Apache Iceberg is an open table format for huge analytic datasets. -long_description = file: README.md -long_description_content_type = text/markdown -url = https://iceberg.apache.org/ -project_urls = - Source Code = https://github.com/apache/iceberg/ - Issue Tracker = https://github.com/apache/iceberg/issues -license="Apache License 2.0", -license_files = - LICENSE -classifiers = - License :: OSI Approved :: Apache Software License - Operating System :: OS Independent - Programming Language :: Python :: 3.8 - Programming Language :: Python :: 3.9 - Programming Language :: Python :: 3.10 - -[options] -package_dir = - = src -packages = find: -python_requires = >=3.8 -install_requires = - mmh3 -[options.extras_require] -arrow = - pyarrow==8.0.0 -dev= - pip>=21.1 - coverage[toml]==6.3.3 - mock==4.0.3 - pytest==7.1.2 - pytest-checkdocs==2.7.1 - pyenchant==3.2.2 - pre-commit==2.19.0 -[options.packages.find] -where = src From 07df5bc774ab3640245a9736f8bf022cd2f27cb9 Mon Sep 17 00:00:00 2001 From: Fokko Driesprong Date: Mon, 6 Jun 2022 19:26:24 +0200 Subject: [PATCH 105/642] Python: Add more tests for schema_conversion (#4972) --- src/iceberg/utils/schema_conversion.py | 10 +- tests/utils/test_schema_conversion.py | 130 +++++++++++++++++++++++++ 2 files changed, 135 insertions(+), 5 deletions(-) diff --git a/src/iceberg/utils/schema_conversion.py b/src/iceberg/utils/schema_conversion.py index 25edc62ac6..6415fb836f 100644 --- a/src/iceberg/utils/schema_conversion.py +++ b/src/iceberg/utils/schema_conversion.py @@ -168,7 +168,7 @@ def _convert_schema(self, avro_type: str | dict[str, Any]) -> IcebergType: Raises: ValueError: When there are unknown types """ - if isinstance(avro_type, str): + if isinstance(avro_type, str) and avro_type in PRIMITIVE_FIELD_TYPE_MAPPING: return PRIMITIVE_FIELD_TYPE_MAPPING[avro_type] elif isinstance(avro_type, dict): if "logicalType" in avro_type: @@ -186,12 +186,12 @@ def _convert_schema(self, avro_type: str | dict[str, Any]) -> IcebergType: return self._convert_map_type(avro_type) elif type_identifier == "fixed": return self._convert_fixed_type(avro_type) - elif isinstance(type_identifier, str): + elif isinstance(type_identifier, str) and type_identifier in PRIMITIVE_FIELD_TYPE_MAPPING: return PRIMITIVE_FIELD_TYPE_MAPPING[type_identifier] else: - raise ValueError(f"Unknown type: {avro_type}") + raise TypeError(f"Unknown type: {avro_type}") else: - raise ValueError(f"Unknown type: {avro_type}") + raise TypeError(f"Unknown type: {avro_type}") def _convert_field(self, field: dict[str, Any]) -> NestedField: """ @@ -265,7 +265,7 @@ def _convert_record_type(self, record_type: dict[str, Any]) -> StructType: Returns: """ if record_type["type"] != "record": - raise ValueError(f"Expected type, got: {record_type}") + raise ValueError(f"Expected record type, got: {record_type}") return StructType(*[self._convert_field(field) for field in record_type["fields"]]) diff --git a/tests/utils/test_schema_conversion.py b/tests/utils/test_schema_conversion.py index 4dcc0ea0bb..0af7e3478d 100644 --- a/tests/utils/test_schema_conversion.py +++ b/tests/utils/test_schema_conversion.py @@ -14,14 +14,19 @@ # KIND, either express or implied. See the License for the # specific language governing permissions and limitations # under the License. +import pytest from iceberg.schema import Schema from iceberg.types import ( BinaryType, BooleanType, + DateType, + DecimalType, + FixedType, IntegerType, ListType, LongType, + MapType, NestedField, StringType, StructType, @@ -229,3 +234,128 @@ def test_avro_list_required_record(): iceberg_schema = AvroSchemaConversion().avro_to_iceberg(avro_schema) assert expected_iceberg_schema == iceberg_schema + + +def test_resolve_union(): + with pytest.raises(TypeError) as exc_info: + AvroSchemaConversion()._resolve_union(["null", "string", "long"]) + + assert "Non-optional types aren't part of the Iceberg specification" in str(exc_info.value) + + +def test_nested_type(): + # In the case a primitive field is nested + assert AvroSchemaConversion()._convert_schema({"type": {"type": "string"}}) == StringType() + + +def test_map_type(): + avro_type = { + "type": "map", + "values": ["long", "null"], + "key-id": 101, + "value-id": 102, + } + actual = AvroSchemaConversion()._convert_schema(avro_type) + expected = MapType(key_id=101, key_type=StringType(), value_id=102, value_type=LongType(), value_is_optional=True) + assert actual == expected + + +def test_fixed_type(): + avro_type = {"type": "fixed", "size": 22} + actual = AvroSchemaConversion()._convert_schema(avro_type) + expected = FixedType(22) + assert actual == expected + + +def test_unknown_primitive(): + with pytest.raises(TypeError) as exc_info: + avro_type = "UnknownType" + AvroSchemaConversion()._convert_schema(avro_type) + assert "Unknown type: UnknownType" in str(exc_info.value) + + +def test_unknown_complex_type(): + with pytest.raises(TypeError) as exc_info: + avro_type = { + "type": "UnknownType", + } + AvroSchemaConversion()._convert_schema(avro_type) + assert "Unknown type: {'type': 'UnknownType'}" in str(exc_info.value) + + +def test_convert_field_without_field_id(): + with pytest.raises(ValueError) as exc_info: + avro_field = { + "name": "contains_null", + "type": "boolean", + } + AvroSchemaConversion()._convert_field(avro_field) + assert "Cannot convert field, missing field-id" in str(exc_info.value) + + +def test_convert_record_type_without_record(): + with pytest.raises(ValueError) as exc_info: + avro_field = {"type": "non-record", "name": "avro_schema", "fields": []} + AvroSchemaConversion()._convert_record_type(avro_field) + assert "Expected record type, got" in str(exc_info.value) + + +def test_avro_list_missing_element_id(): + avro_type = { + "name": "array_with_string", + "type": { + "type": "array", + "items": "string", + "default": [], + # "element-id": 101, + }, + "field-id": 100, + } + + with pytest.raises(ValueError) as exc_info: + AvroSchemaConversion()._convert_array_type(avro_type) + + assert "Cannot convert array-type, missing element-id:" in str(exc_info.value) + + +def test_convert_decimal_type(): + avro_decimal_type = {"type": "bytes", "logicalType": "decimal", "precision": 19, "scale": 25} + actual = AvroSchemaConversion()._convert_logical_type(avro_decimal_type) + expected = DecimalType(precision=19, scale=25) + assert actual == expected + + +def test_convert_date_type(): + avro_logical_type = {"type": "int", "logicalType": "date"} + actual = AvroSchemaConversion()._convert_logical_type(avro_logical_type) + assert actual == DateType() + + +def test_unknown_logical_type(): + """Test raising a ValueError when converting an unknown logical type as part of an Avro schema conversion""" + avro_logical_type = {"type": "bytes", "logicalType": "date"} + with pytest.raises(ValueError) as exc_info: + AvroSchemaConversion()._convert_logical_type(avro_logical_type) + + assert "Unknown logical/physical type combination:" in str(exc_info.value) + + +def test_logical_map_with_invalid_fields(): + avro_type = { + "type": "array", + "logicalType": "map", + "items": { + "type": "record", + "name": "k101_v102", + "fields": [ + {"name": "key", "type": "int", "field-id": 101}, + {"name": "value", "type": "string", "field-id": 102}, + {"name": "other", "type": "bytes", "field-id": 103}, + ], + }, + } + + with pytest.raises(ValueError) as exc_info: + AvroSchemaConversion()._convert_logical_map_type(avro_type) + + assert "Invalid key-value pair schema:" in str(exc_info.value) From 7a7155ea30809b824629e628e418c56b0413dc95 Mon Sep 17 00:00:00 2001 From: Fokko Driesprong Date: Mon, 6 Jun 2022 19:26:47 +0200 Subject: [PATCH 106/642] Python: Decimal is actually DecimalType (#4967) --- src/iceberg/conversions.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/iceberg/conversions.py b/src/iceberg/conversions.py index d302f4e199..96724e5461 100644 --- a/src/iceberg/conversions.py +++ b/src/iceberg/conversions.py @@ -209,11 +209,11 @@ def _(primitive_type, value: bytes) -> bytes: @to_bytes.register(DecimalType) -def _(primitive_type, value: Decimal) -> bytes: +def _(primitive_type: DecimalType, value: Decimal) -> bytes: """Convert a Decimal value to bytes given a DecimalType instance with defined precision and scale Args: - primitive_type (Decimal): A DecimalType instance with precision and scale + primitive_type (DecimalType): A DecimalType instance with precision and scale value (Decimal): A Decimal instance Raises: @@ -293,6 +293,6 @@ def _(primitive_type, b: bytes) -> bytes: @from_bytes.register(DecimalType) -def _(primitive_type, buf: bytes) -> Decimal: +def _(primitive_type: DecimalType, buf: bytes) -> Decimal: unscaled = int.from_bytes(buf, "big", signed=True) return unscaled_to_decimal(unscaled, primitive_type.scale) From 8a8f1646319beacab18f3d9c9dbe2bdb393c5bc6 Mon Sep 17 00:00:00 2001 From: Fokko Driesprong Date: Mon, 6 Jun 2022 19:32:03 +0200 Subject: [PATCH 107/642] Python: Disable autouse of fixtures (#4964) --- tests/conftest.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tests/conftest.py b/tests/conftest.py index 3457c4cb6f..bf0161de03 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -46,7 +46,7 @@ def set(self, pos: int, value) -> None: self.content[pos] = value -@pytest.fixture(scope="session", autouse=True) +@pytest.fixture(scope="session") def table_schema_simple(): return schema.Schema( NestedField(field_id=1, name="foo", field_type=StringType(), is_optional=False), @@ -57,7 +57,7 @@ def table_schema_simple(): ) -@pytest.fixture(scope="session", autouse=True) +@pytest.fixture(scope="session") def table_schema_nested(): return schema.Schema( NestedField(field_id=1, name="foo", field_type=StringType(), is_optional=False), @@ -110,7 +110,7 @@ def table_schema_nested(): ) -@pytest.fixture(scope="session", autouse=True) +@pytest.fixture(scope="session") def foo_struct(): return FooStruct() From bb5c893c27843d20b45d2ec09be713280c63c184 Mon Sep 17 00:00:00 2001 From: Fokko Driesprong Date: Thu, 9 Jun 2022 17:52:57 +0200 Subject: [PATCH 108/642] Python: Add Poetry IDEA instructions (#4980) --- CONTRIBUTING.md | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 1afa7bd948..d760efc9f0 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -34,6 +34,22 @@ To get started, you can run `make install`, which will install poetry and it wil If you want to install the library on the host, you can simply run `pip3 install -e .`. If you wish to use a virtual environment, you can run `poetry shell`. Poetry will open up a virtual environment with all the dependencies set. +To set up IDEA with Poetry ([also on Loom](https://www.loom.com/share/6d36464d45f244729d91003e7f671fd2)): + +- Open up the Python project in IntelliJ +- Make sure that you're on a lastest master (that includes Poetry) +- Go to File -> Project Structure (⌘;) +- Go to Platform Settings -> SDKs +- Click the + sign -> Add Python SDK +- Select Poetry Environment from the left hand side bar and hit OK +- It can take some time to download all the dependencies based on your internet +- Go to Project Settings -> Project +- Select the Poetry SDK from the SDK dropdown, and click OK + +For IDEA ≤2021 you need to install the [Poetry integration as a plugin](https://plugins.jetbrains.com/plugin/14307-poetry/). + +Now you're set using Poetry, and all the tests will run in Poetry, and you'll have syntax highlighting in the pyproject.toml to indicate stale dependencies. + ## Linting We rely on `pre-commit` to apply autoformatting and linting: From ea425b0b81a7a4301432623d196a10a816808633 Mon Sep 17 00:00:00 2001 From: Fokko Driesprong Date: Thu, 9 Jun 2022 18:02:21 +0200 Subject: [PATCH 109/642] Python: Change Python API naming to the spec (#4992) See https://github.com/apache/iceberg/issues/4985 --- src/iceberg/schema.py | 10 +- src/iceberg/types.py | 44 +++--- src/iceberg/utils/schema_conversion.py | 20 +-- tests/conftest.py | 38 +++--- tests/expressions/test_expressions_base.py | 6 +- tests/test_schema.py | 150 ++++++++++----------- tests/test_types.py | 34 ++--- tests/utils/test_schema_conversion.py | 52 ++++--- 8 files changed, 171 insertions(+), 183 deletions(-) diff --git a/src/iceberg/schema.py b/src/iceberg/schema.py index b0fd8427c5..97b9801500 100644 --- a/src/iceberg/schema.py +++ b/src/iceberg/schema.py @@ -546,16 +546,16 @@ class _BuildPositionAccessors(SchemaVisitor[Dict[Position, Accessor]]): >>> from iceberg.schema import Schema >>> from iceberg.types import * >>> schema = Schema( - ... NestedField(field_id=2, name="id", field_type=IntegerType(), is_optional=False), - ... NestedField(field_id=1, name="data", field_type=StringType(), is_optional=True), + ... NestedField(field_id=2, name="id", field_type=IntegerType(), required=False), + ... NestedField(field_id=1, name="data", field_type=StringType(), required=True), ... NestedField( ... field_id=3, ... name="location", ... field_type=StructType( - ... NestedField(field_id=5, name="latitude", field_type=FloatType(), is_optional=False), - ... NestedField(field_id=6, name="longitude", field_type=FloatType(), is_optional=False), + ... NestedField(field_id=5, name="latitude", field_type=FloatType(), required=False), + ... NestedField(field_id=6, name="longitude", field_type=FloatType(), required=False), ... ), - ... is_optional=True, + ... required=True, ... ), ... schema_id=1, ... identifier_field_ids=[1], diff --git a/src/iceberg/types.py b/src/iceberg/types.py index 6f1ad701c4..46828e6d40 100644 --- a/src/iceberg/types.py +++ b/src/iceberg/types.py @@ -142,14 +142,14 @@ class NestedField(IcebergType): ... field_id=1, ... name='foo', ... field_type=FixedType(22), - ... is_optional=False, + ... required=False, ... )) '1: foo: required fixed[22]' >>> str(NestedField( ... field_id=2, ... name='bar', ... field_type=LongType(), - ... is_optional=False, + ... required=False, ... doc="Just a long" ... )) '2: bar: required long (Just a long)' @@ -158,7 +158,7 @@ class NestedField(IcebergType): field_id: int = field() name: str = field() field_type: IcebergType = field() - is_optional: bool = field(default=True) + required: bool = field(default=True) doc: Optional[str] = field(default=None, repr=False) _instances: ClassVar[Dict[Tuple[bool, int, str, IcebergType, Optional[str]], "NestedField"]] = {} @@ -168,21 +168,21 @@ def __new__( field_id: int, name: str, field_type: IcebergType, - is_optional: bool = True, + required: bool = True, doc: Optional[str] = None, ): - key = (is_optional, field_id, name, field_type, doc) + key = (required, field_id, name, field_type, doc) cls._instances[key] = cls._instances.get(key) or object.__new__(cls) return cls._instances[key] @property - def is_required(self) -> bool: - return not self.is_optional + def optional(self) -> bool: + return not self.required @property def string_type(self) -> str: doc = "" if not self.doc else f" ({self.doc})" - req = "optional" if self.is_optional else "required" + req = "optional" if self.required else "required" return f"{self.field_id}: {self.name}: {req} {self.field_type}{doc}" @@ -223,13 +223,13 @@ class ListType(IcebergType): """A list type in Iceberg Example: - >>> ListType(element_id=3, element_type=StringType(), element_is_optional=True) - ListType(element_id=3, element_type=StringType(), element_is_optional=True) + >>> ListType(element_id=3, element_type=StringType(), element_required=True) + ListType(element_id=3, element_type=StringType(), element_required=True) """ element_id: int = field() element_type: IcebergType = field() - element_is_optional: bool = field(default=True) + element_required: bool = field(default=True) element: NestedField = field(init=False, repr=False) _instances: ClassVar[Dict[Tuple[bool, int, IcebergType], "ListType"]] = {} @@ -238,9 +238,9 @@ def __new__( cls, element_id: int, element_type: IcebergType, - element_is_optional: bool = True, + element_required: bool = True, ): - key = (element_is_optional, element_id, element_type) + key = (element_required, element_id, element_type) cls._instances[key] = cls._instances.get(key) or object.__new__(cls) return cls._instances[key] @@ -250,7 +250,7 @@ def __post_init__(self): "element", NestedField( name="element", - is_optional=self.element_is_optional, + required=self.element_required, field_id=self.element_id, field_type=self.element_type, ), @@ -266,15 +266,15 @@ class MapType(IcebergType): """A map type in Iceberg Example: - >>> MapType(key_id=1, key_type=StringType(), value_id=2, value_type=IntegerType(), value_is_optional=True) - MapType(key_id=1, key_type=StringType(), value_id=2, value_type=IntegerType(), value_is_optional=True) + >>> MapType(key_id=1, key_type=StringType(), value_id=2, value_type=IntegerType(), value_required=True) + MapType(key_id=1, key_type=StringType(), value_id=2, value_type=IntegerType(), value_required=True) """ key_id: int = field() key_type: IcebergType = field() value_id: int = field() value_type: IcebergType = field() - value_is_optional: bool = field(default=True) + value_required: bool = field(default=True) key: NestedField = field(init=False, repr=False) value: NestedField = field(init=False, repr=False) @@ -287,16 +287,14 @@ def __new__( key_type: IcebergType, value_id: int, value_type: IcebergType, - value_is_optional: bool = True, + value_required: bool = True, ): - impl_key = (key_id, key_type, value_id, value_type, value_is_optional) + impl_key = (key_id, key_type, value_id, value_type, value_required) cls._instances[impl_key] = cls._instances.get(impl_key) or object.__new__(cls) return cls._instances[impl_key] def __post_init__(self): - object.__setattr__( - self, "key", NestedField(name="key", field_id=self.key_id, field_type=self.key_type, is_optional=False) - ) + object.__setattr__(self, "key", NestedField(name="key", field_id=self.key_id, field_type=self.key_type, required=False)) object.__setattr__( self, "value", @@ -304,7 +302,7 @@ def __post_init__(self): name="value", field_id=self.value_id, field_type=self.value_type, - is_optional=self.value_is_optional, + required=self.value_required, ), ) diff --git a/src/iceberg/utils/schema_conversion.py b/src/iceberg/utils/schema_conversion.py index 6415fb836f..4e57daa031 100644 --- a/src/iceberg/utils/schema_conversion.py +++ b/src/iceberg/utils/schema_conversion.py @@ -96,10 +96,10 @@ def avro_to_iceberg(self, avro_schema: dict[str, Any]) -> Schema: ... }) >>> iceberg_schema = Schema( ... NestedField( - ... field_id=500, name="manifest_path", field_type=StringType(), is_optional=False, doc="Location URI with FS scheme" + ... field_id=500, name="manifest_path", field_type=StringType(), required=False, doc="Location URI with FS scheme" ... ), ... NestedField( - ... field_id=501, name="manifest_length", field_type=LongType(), is_optional=False, doc="Total file size in bytes" + ... field_id=501, name="manifest_length", field_type=LongType(), required=False, doc="Total file size in bytes" ... ), ... schema_id=1 ... ) @@ -211,7 +211,7 @@ def _convert_field(self, field: dict[str, Any]) -> NestedField: field_id=field["field-id"], name=field["name"], field_type=self._convert_schema(plain_type), - is_optional=is_optional, + required=is_optional, doc=field.get("doc"), ) @@ -244,14 +244,14 @@ def _convert_record_type(self, record_type: dict[str, Any]) -> StructType: ... field_id=509, ... name="contains_null", ... field_type=BooleanType(), - ... is_optional=False, + ... required=False, ... doc="True if any file has a null partition value", ... ), ... NestedField( ... field_id=518, ... name="contains_nan", ... field_type=BooleanType(), - ... is_optional=True, + ... required=True, ... doc="True if any file has a nan partition value", ... ), ... ) @@ -278,7 +278,7 @@ def _convert_array_type(self, array_type: dict[str, Any]) -> ListType: return ListType( element_id=array_type["element-id"], element_type=self._convert_schema(plain_type), - element_is_optional=element_is_optional, + element_required=element_is_optional, ) def _convert_map_type(self, map_type: dict[str, Any]) -> MapType: @@ -300,7 +300,7 @@ def _convert_map_type(self, map_type: dict[str, Any]) -> MapType: ... key_type=StringType(), ... value_id=102, ... value_type=LongType(), - ... value_is_optional=True + ... value_required=True ... ) >>> actual == expected True @@ -314,7 +314,7 @@ def _convert_map_type(self, map_type: dict[str, Any]) -> MapType: key_type=StringType(), value_id=map_type["value-id"], value_type=self._convert_schema(value_type), - value_is_optional=value_is_optional, + value_required=value_is_optional, ) def _convert_logical_type(self, avro_logical_type: dict[str, Any]) -> IcebergType: @@ -407,7 +407,7 @@ def _convert_logical_map_type(self, avro_type: dict[str, Any]) -> MapType: ... key_type=IntegerType(), ... value_id=102, ... value_type=StringType(), - ... value_is_optional=False + ... value_required=False ... ) >>> actual == expected True @@ -428,7 +428,7 @@ def _convert_logical_map_type(self, avro_type: dict[str, Any]) -> MapType: key_type=key.field_type, value_id=value.field_id, value_type=value.field_type, - value_is_optional=value.is_optional, + value_required=value.required, ) def _convert_fixed_type(self, avro_type: dict[str, Any]) -> FixedType: diff --git a/tests/conftest.py b/tests/conftest.py index bf0161de03..b5dd18b8d8 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -49,9 +49,9 @@ def set(self, pos: int, value) -> None: @pytest.fixture(scope="session") def table_schema_simple(): return schema.Schema( - NestedField(field_id=1, name="foo", field_type=StringType(), is_optional=False), - NestedField(field_id=2, name="bar", field_type=IntegerType(), is_optional=True), - NestedField(field_id=3, name="baz", field_type=BooleanType(), is_optional=False), + NestedField(field_id=1, name="foo", field_type=StringType(), required=False), + NestedField(field_id=2, name="bar", field_type=IntegerType(), required=True), + NestedField(field_id=3, name="baz", field_type=BooleanType(), required=False), schema_id=1, identifier_field_ids=[1], ) @@ -60,14 +60,14 @@ def table_schema_simple(): @pytest.fixture(scope="session") def table_schema_nested(): return schema.Schema( - NestedField(field_id=1, name="foo", field_type=StringType(), is_optional=False), - NestedField(field_id=2, name="bar", field_type=IntegerType(), is_optional=True), - NestedField(field_id=3, name="baz", field_type=BooleanType(), is_optional=False), + NestedField(field_id=1, name="foo", field_type=StringType(), required=False), + NestedField(field_id=2, name="bar", field_type=IntegerType(), required=True), + NestedField(field_id=3, name="baz", field_type=BooleanType(), required=False), NestedField( field_id=4, name="qux", - field_type=ListType(element_id=5, element_type=StringType(), element_is_optional=True), - is_optional=True, + field_type=ListType(element_id=5, element_type=StringType(), element_required=True), + required=True, ), NestedField( field_id=6, @@ -76,12 +76,10 @@ def table_schema_nested(): key_id=7, key_type=StringType(), value_id=8, - value_type=MapType( - key_id=9, key_type=StringType(), value_id=10, value_type=IntegerType(), value_is_optional=True - ), - value_is_optional=True, + value_type=MapType(key_id=9, key_type=StringType(), value_id=10, value_type=IntegerType(), value_required=True), + value_required=True, ), - is_optional=True, + required=True, ), NestedField( field_id=11, @@ -89,21 +87,21 @@ def table_schema_nested(): field_type=ListType( element_id=12, element_type=StructType( - NestedField(field_id=13, name="latitude", field_type=FloatType(), is_optional=False), - NestedField(field_id=14, name="longitude", field_type=FloatType(), is_optional=False), + NestedField(field_id=13, name="latitude", field_type=FloatType(), required=False), + NestedField(field_id=14, name="longitude", field_type=FloatType(), required=False), ), - element_is_optional=True, + element_required=True, ), - is_optional=True, + required=True, ), NestedField( field_id=15, name="person", field_type=StructType( - NestedField(field_id=16, name="name", field_type=StringType(), is_optional=False), - NestedField(field_id=17, name="age", field_type=IntegerType(), is_optional=True), + NestedField(field_id=16, name="name", field_type=StringType(), required=False), + NestedField(field_id=17, name="age", field_type=IntegerType(), required=True), ), - is_optional=False, + required=False, ), schema_id=1, identifier_field_ids=[1], diff --git a/tests/expressions/test_expressions_base.py b/tests/expressions/test_expressions_base.py index 43fde0ad1c..7fe3e9817e 100644 --- a/tests/expressions/test_expressions_base.py +++ b/tests/expressions/test_expressions_base.py @@ -281,7 +281,7 @@ def test_accessor_base_class(foo_struct): def test_bound_reference_str_and_repr(): """Test str and repr of BoundReference""" - field = NestedField(field_id=1, name="foo", field_type=StringType(), is_optional=False) + field = NestedField(field_id=1, name="foo", field_type=StringType(), required=False) position1_accessor = base.Accessor(position=1) bound_ref = base.BoundReference(field=field, accessor=position1_accessor) assert str(bound_ref) == f"BoundReference(field={repr(field)}, accessor={repr(position1_accessor)})" @@ -290,10 +290,10 @@ def test_bound_reference_str_and_repr(): def test_bound_reference_field_property(): """Test str and repr of BoundReference""" - field = NestedField(field_id=1, name="foo", field_type=StringType(), is_optional=False) + field = NestedField(field_id=1, name="foo", field_type=StringType(), required=False) position1_accessor = base.Accessor(position=1) bound_ref = base.BoundReference(field=field, accessor=position1_accessor) - assert bound_ref.field == NestedField(field_id=1, name="foo", field_type=StringType(), is_optional=False) + assert bound_ref.field == NestedField(field_id=1, name="foo", field_type=StringType(), required=False) def test_bound_reference(table_schema_simple, foo_struct): diff --git a/tests/test_schema.py b/tests/test_schema.py index d48198edd9..e5afe5fb74 100644 --- a/tests/test_schema.py +++ b/tests/test_schema.py @@ -53,13 +53,11 @@ def test_schema_str(table_schema_simple): [ ( schema.Schema(NestedField(1, "foo", StringType()), schema_id=1), - "Schema(fields=(NestedField(field_id=1, name='foo', field_type=StringType(), is_optional=True),), schema_id=1, identifier_field_ids=[])", + "Schema(fields=(NestedField(field_id=1, name='foo', field_type=StringType(), required=True),), schema_id=1, identifier_field_ids=[])", ), ( - schema.Schema( - NestedField(1, "foo", StringType()), NestedField(2, "bar", IntegerType(), is_optional=False), schema_id=1 - ), - "Schema(fields=(NestedField(field_id=1, name='foo', field_type=StringType(), is_optional=True), NestedField(field_id=2, name='bar', field_type=IntegerType(), is_optional=False)), schema_id=1, identifier_field_ids=[])", + schema.Schema(NestedField(1, "foo", StringType()), NestedField(2, "bar", IntegerType(), required=False), schema_id=1), + "Schema(fields=(NestedField(field_id=1, name='foo', field_type=StringType(), required=True), NestedField(field_id=2, name='bar', field_type=IntegerType(), required=False)), schema_id=1, identifier_field_ids=[])", ), ], ) @@ -72,10 +70,10 @@ def test_schema_raise_on_duplicate_names(): """Test schema representation""" with pytest.raises(ValueError) as exc_info: schema.Schema( - NestedField(field_id=1, name="foo", field_type=StringType(), is_optional=False), - NestedField(field_id=2, name="bar", field_type=IntegerType(), is_optional=True), - NestedField(field_id=3, name="baz", field_type=BooleanType(), is_optional=False), - NestedField(field_id=4, name="baz", field_type=BooleanType(), is_optional=False), + NestedField(field_id=1, name="foo", field_type=StringType(), required=False), + NestedField(field_id=2, name="bar", field_type=IntegerType(), required=True), + NestedField(field_id=3, name="baz", field_type=BooleanType(), required=False), + NestedField(field_id=4, name="baz", field_type=BooleanType(), required=False), schema_id=1, identifier_field_ids=[1], ) @@ -87,16 +85,16 @@ def test_schema_index_by_id_visitor(table_schema_nested): """Test index_by_id visitor function""" index = schema.index_by_id(table_schema_nested) assert index == { - 1: NestedField(field_id=1, name="foo", field_type=StringType(), is_optional=False), - 2: NestedField(field_id=2, name="bar", field_type=IntegerType(), is_optional=True), - 3: NestedField(field_id=3, name="baz", field_type=BooleanType(), is_optional=False), + 1: NestedField(field_id=1, name="foo", field_type=StringType(), required=False), + 2: NestedField(field_id=2, name="bar", field_type=IntegerType(), required=True), + 3: NestedField(field_id=3, name="baz", field_type=BooleanType(), required=False), 4: NestedField( field_id=4, name="qux", - field_type=ListType(element_id=5, element_type=StringType(), element_is_optional=True), - is_optional=True, + field_type=ListType(element_id=5, element_type=StringType(), element_required=True), + required=True, ), - 5: NestedField(field_id=5, name="element", field_type=StringType(), is_optional=True), + 5: NestedField(field_id=5, name="element", field_type=StringType(), required=True), 6: NestedField( field_id=6, name="quux", @@ -104,57 +102,55 @@ def test_schema_index_by_id_visitor(table_schema_nested): key_id=7, key_type=StringType(), value_id=8, - value_type=MapType( - key_id=9, key_type=StringType(), value_id=10, value_type=IntegerType(), value_is_optional=True - ), - value_is_optional=True, + value_type=MapType(key_id=9, key_type=StringType(), value_id=10, value_type=IntegerType(), value_required=True), + value_required=True, ), - is_optional=True, + required=True, ), - 7: NestedField(field_id=7, name="key", field_type=StringType(), is_optional=False), - 9: NestedField(field_id=9, name="key", field_type=StringType(), is_optional=False), + 7: NestedField(field_id=7, name="key", field_type=StringType(), required=False), + 9: NestedField(field_id=9, name="key", field_type=StringType(), required=False), 8: NestedField( field_id=8, name="value", - field_type=MapType(key_id=9, key_type=StringType(), value_id=10, value_type=IntegerType(), value_is_optional=True), - is_optional=True, + field_type=MapType(key_id=9, key_type=StringType(), value_id=10, value_type=IntegerType(), value_required=True), + required=True, ), - 10: NestedField(field_id=10, name="value", field_type=IntegerType(), is_optional=True), + 10: NestedField(field_id=10, name="value", field_type=IntegerType(), required=True), 11: NestedField( field_id=11, name="location", field_type=ListType( element_id=12, element_type=StructType( - NestedField(field_id=13, name="latitude", field_type=FloatType(), is_optional=False), - NestedField(field_id=14, name="longitude", field_type=FloatType(), is_optional=False), + NestedField(field_id=13, name="latitude", field_type=FloatType(), required=False), + NestedField(field_id=14, name="longitude", field_type=FloatType(), required=False), ), - element_is_optional=True, + element_required=True, ), - is_optional=True, + required=True, ), 12: NestedField( field_id=12, name="element", field_type=StructType( - NestedField(field_id=13, name="latitude", field_type=FloatType(), is_optional=False), - NestedField(field_id=14, name="longitude", field_type=FloatType(), is_optional=False), + NestedField(field_id=13, name="latitude", field_type=FloatType(), required=False), + NestedField(field_id=14, name="longitude", field_type=FloatType(), required=False), ), - is_optional=True, + required=True, ), - 13: NestedField(field_id=13, name="latitude", field_type=FloatType(), is_optional=False), - 14: NestedField(field_id=14, name="longitude", field_type=FloatType(), is_optional=False), + 13: NestedField(field_id=13, name="latitude", field_type=FloatType(), required=False), + 14: NestedField(field_id=14, name="longitude", field_type=FloatType(), required=False), 15: NestedField( field_id=15, name="person", field_type=StructType( - NestedField(field_id=16, name="name", field_type=StringType(), is_optional=False), - NestedField(field_id=17, name="age", field_type=IntegerType(), is_optional=True), + NestedField(field_id=16, name="name", field_type=StringType(), required=False), + NestedField(field_id=17, name="age", field_type=IntegerType(), required=True), ), - is_optional=False, + required=False, ), - 16: NestedField(field_id=16, name="name", field_type=StringType(), is_optional=False), - 17: NestedField(field_id=17, name="age", field_type=IntegerType(), is_optional=True), + 16: NestedField(field_id=16, name="name", field_type=StringType(), required=False), + 17: NestedField(field_id=17, name="age", field_type=IntegerType(), required=True), } @@ -222,19 +218,19 @@ def test_schema_find_field_by_id(table_schema_simple): assert isinstance(column1, NestedField) assert column1.field_id == 1 assert column1.field_type == StringType() - assert column1.is_optional == False + assert column1.required == False column2 = index[2] assert isinstance(column2, NestedField) assert column2.field_id == 2 assert column2.field_type == IntegerType() - assert column2.is_optional == True + assert column2.required == True column3 = index[3] assert isinstance(column3, NestedField) assert column3.field_id == 3 assert column3.field_type == BooleanType() - assert column3.is_optional == False + assert column3.required == False def test_schema_find_field_by_id_raise_on_unknown_field(table_schema_simple): @@ -248,24 +244,24 @@ def test_schema_find_field_by_id_raise_on_unknown_field(table_schema_simple): def test_schema_find_field_type_by_id(table_schema_simple): """Test retrieving a columns' type using its field ID""" index = schema.index_by_id(table_schema_simple) - assert index[1] == NestedField(field_id=1, name="foo", field_type=StringType(), is_optional=False) - assert index[2] == NestedField(field_id=2, name="bar", field_type=IntegerType(), is_optional=True) - assert index[3] == NestedField(field_id=3, name="baz", field_type=BooleanType(), is_optional=False) + assert index[1] == NestedField(field_id=1, name="foo", field_type=StringType(), required=False) + assert index[2] == NestedField(field_id=2, name="bar", field_type=IntegerType(), required=True) + assert index[3] == NestedField(field_id=3, name="baz", field_type=BooleanType(), required=False) def test_index_by_id_schema_visitor(table_schema_nested): """Test the index_by_id function that uses the IndexById schema visitor""" assert schema.index_by_id(table_schema_nested) == { - 1: NestedField(field_id=1, name="foo", field_type=StringType(), is_optional=False), - 2: NestedField(field_id=2, name="bar", field_type=IntegerType(), is_optional=True), - 3: NestedField(field_id=3, name="baz", field_type=BooleanType(), is_optional=False), + 1: NestedField(field_id=1, name="foo", field_type=StringType(), required=False), + 2: NestedField(field_id=2, name="bar", field_type=IntegerType(), required=True), + 3: NestedField(field_id=3, name="baz", field_type=BooleanType(), required=False), 4: NestedField( field_id=4, name="qux", - field_type=ListType(element_id=5, element_type=StringType(), element_is_optional=True), - is_optional=True, + field_type=ListType(element_id=5, element_type=StringType(), element_required=True), + required=True, ), - 5: NestedField(field_id=5, name="element", field_type=StringType(), is_optional=True), + 5: NestedField(field_id=5, name="element", field_type=StringType(), required=True), 6: NestedField( field_id=6, name="quux", @@ -273,57 +269,55 @@ def test_index_by_id_schema_visitor(table_schema_nested): key_id=7, key_type=StringType(), value_id=8, - value_type=MapType( - key_id=9, key_type=StringType(), value_id=10, value_type=IntegerType(), value_is_optional=True - ), - value_is_optional=True, + value_type=MapType(key_id=9, key_type=StringType(), value_id=10, value_type=IntegerType(), value_required=True), + value_required=True, ), - is_optional=True, + required=True, ), - 7: NestedField(field_id=7, name="key", field_type=StringType(), is_optional=False), + 7: NestedField(field_id=7, name="key", field_type=StringType(), required=False), 8: NestedField( field_id=8, name="value", - field_type=MapType(key_id=9, key_type=StringType(), value_id=10, value_type=IntegerType(), value_is_optional=True), - is_optional=True, + field_type=MapType(key_id=9, key_type=StringType(), value_id=10, value_type=IntegerType(), value_required=True), + required=True, ), - 9: NestedField(field_id=9, name="key", field_type=StringType(), is_optional=False), - 10: NestedField(field_id=10, name="value", field_type=IntegerType(), is_optional=True), + 9: NestedField(field_id=9, name="key", field_type=StringType(), required=False), + 10: NestedField(field_id=10, name="value", field_type=IntegerType(), required=True), 11: NestedField( field_id=11, name="location", field_type=ListType( element_id=12, element_type=StructType( - NestedField(field_id=13, name="latitude", field_type=FloatType(), is_optional=False), - NestedField(field_id=14, name="longitude", field_type=FloatType(), is_optional=False), + NestedField(field_id=13, name="latitude", field_type=FloatType(), required=False), + NestedField(field_id=14, name="longitude", field_type=FloatType(), required=False), ), - element_is_optional=True, + element_required=True, ), - is_optional=True, + required=True, ), 12: NestedField( field_id=12, name="element", field_type=StructType( - NestedField(field_id=13, name="latitude", field_type=FloatType(), is_optional=False), - NestedField(field_id=14, name="longitude", field_type=FloatType(), is_optional=False), + NestedField(field_id=13, name="latitude", field_type=FloatType(), required=False), + NestedField(field_id=14, name="longitude", field_type=FloatType(), required=False), ), - is_optional=True, + required=True, ), - 13: NestedField(field_id=13, name="latitude", field_type=FloatType(), is_optional=False), - 14: NestedField(field_id=14, name="longitude", field_type=FloatType(), is_optional=False), + 13: NestedField(field_id=13, name="latitude", field_type=FloatType(), required=False), + 14: NestedField(field_id=14, name="longitude", field_type=FloatType(), required=False), 15: NestedField( field_id=15, name="person", field_type=StructType( - NestedField(field_id=16, name="name", field_type=StringType(), is_optional=False), - NestedField(field_id=17, name="age", field_type=IntegerType(), is_optional=True), + NestedField(field_id=16, name="name", field_type=StringType(), required=False), + NestedField(field_id=17, name="age", field_type=IntegerType(), required=True), ), - is_optional=False, + required=False, ), - 16: NestedField(field_id=16, name="name", field_type=StringType(), is_optional=False), - 17: NestedField(field_id=17, name="age", field_type=IntegerType(), is_optional=True), + 16: NestedField(field_id=16, name="name", field_type=StringType(), required=False), + 17: NestedField(field_id=17, name="age", field_type=IntegerType(), required=True), } @@ -340,19 +334,19 @@ def test_schema_find_field(table_schema_simple): table_schema_simple.find_field(1) == table_schema_simple.find_field("foo") == table_schema_simple.find_field("FOO", case_sensitive=False) - == NestedField(field_id=1, name="foo", field_type=StringType(), is_optional=False) + == NestedField(field_id=1, name="foo", field_type=StringType(), required=False) ) assert ( table_schema_simple.find_field(2) == table_schema_simple.find_field("bar") == table_schema_simple.find_field("BAR", case_sensitive=False) - == NestedField(field_id=2, name="bar", field_type=IntegerType(), is_optional=True) + == NestedField(field_id=2, name="bar", field_type=IntegerType(), required=True) ) assert ( table_schema_simple.find_field(3) == table_schema_simple.find_field("baz") == table_schema_simple.find_field("BAZ", case_sensitive=False) - == NestedField(field_id=3, name="baz", field_type=BooleanType(), is_optional=False) + == NestedField(field_id=3, name="baz", field_type=BooleanType(), required=False) ) diff --git a/tests/test_types.py b/tests/test_types.py index 844e3ab7de..26f6f421a7 100644 --- a/tests/test_types.py +++ b/tests/test_types.py @@ -84,12 +84,12 @@ def test_repr_primitive_types(input_index, input_type): ), ( StructType( - NestedField(1, "required_field", StringType(), is_optional=False), - NestedField(2, "optional_field", IntegerType(), is_optional=True), + NestedField(1, "required_field", StringType(), required=False), + NestedField(2, "optional_field", IntegerType(), required=True), ), False, ), - (NestedField(1, "required_field", StringType(), is_optional=False), False), + (NestedField(1, "required_field", StringType(), required=False), False), ], ) def test_is_primitive(input_type, result): @@ -119,30 +119,30 @@ def test_decimal_type(): def test_struct_type(): type_var = StructType( - NestedField(1, "optional_field", IntegerType(), is_optional=True), - NestedField(2, "required_field", FixedType(5), is_optional=False), + NestedField(1, "optional_field", IntegerType(), required=True), + NestedField(2, "required_field", FixedType(5), required=False), NestedField( 3, "required_field", StructType( - NestedField(4, "optional_field", DecimalType(8, 2), is_optional=True), - NestedField(5, "required_field", LongType(), is_optional=False), + NestedField(4, "optional_field", DecimalType(8, 2), required=True), + NestedField(5, "required_field", LongType(), required=False), ), - is_optional=False, + required=False, ), ) assert len(type_var.fields) == 3 assert str(type_var) == str(eval(repr(type_var))) assert type_var == eval(repr(type_var)) - assert type_var != StructType(NestedField(1, "optional_field", IntegerType(), is_optional=True)) + assert type_var != StructType(NestedField(1, "optional_field", IntegerType(), required=True)) def test_list_type(): type_var = ListType( 1, StructType( - NestedField(2, "optional_field", DecimalType(8, 2), is_optional=True), - NestedField(3, "required_field", LongType(), is_optional=False), + NestedField(2, "optional_field", DecimalType(8, 2), required=True), + NestedField(3, "required_field", LongType(), required=False), ), False, ) @@ -154,7 +154,7 @@ def test_list_type(): assert type_var != ListType( 1, StructType( - NestedField(2, "optional_field", DecimalType(8, 2), is_optional=True), + NestedField(2, "optional_field", DecimalType(8, 2), required=True), ), True, ) @@ -183,15 +183,15 @@ def test_nested_field(): ListType( 3, DoubleType(), - element_is_optional=False, + element_required=False, ), - is_optional=True, + required=True, ), ), - is_optional=True, + required=True, ) - assert field_var.is_optional - assert not field_var.is_required + assert field_var.required + assert not field_var.optional assert field_var.field_id == 1 assert isinstance(field_var.field_type, StructType) assert str(field_var) == str(eval(repr(field_var))) diff --git a/tests/utils/test_schema_conversion.py b/tests/utils/test_schema_conversion.py index 0af7e3478d..8776e1a14f 100644 --- a/tests/utils/test_schema_conversion.py +++ b/tests/utils/test_schema_conversion.py @@ -38,29 +38,27 @@ def test_iceberg_to_avro(manifest_schema): iceberg_schema = AvroSchemaConversion().avro_to_iceberg(manifest_schema) expected_iceberg_schema = Schema( NestedField( - field_id=500, name="manifest_path", field_type=StringType(), is_optional=False, doc="Location URI with FS scheme" + field_id=500, name="manifest_path", field_type=StringType(), required=False, doc="Location URI with FS scheme" ), + NestedField(field_id=501, name="manifest_length", field_type=LongType(), required=False, doc="Total file size in bytes"), NestedField( - field_id=501, name="manifest_length", field_type=LongType(), is_optional=False, doc="Total file size in bytes" - ), - NestedField( - field_id=502, name="partition_spec_id", field_type=IntegerType(), is_optional=False, doc="Spec ID used to write" + field_id=502, name="partition_spec_id", field_type=IntegerType(), required=False, doc="Spec ID used to write" ), NestedField( field_id=503, name="added_snapshot_id", field_type=LongType(), - is_optional=True, + required=True, doc="Snapshot ID that added the manifest", ), NestedField( - field_id=504, name="added_data_files_count", field_type=IntegerType(), is_optional=True, doc="Added entry count" + field_id=504, name="added_data_files_count", field_type=IntegerType(), required=True, doc="Added entry count" ), NestedField( - field_id=505, name="existing_data_files_count", field_type=IntegerType(), is_optional=True, doc="Existing entry count" + field_id=505, name="existing_data_files_count", field_type=IntegerType(), required=True, doc="Existing entry count" ), NestedField( - field_id=506, name="deleted_data_files_count", field_type=IntegerType(), is_optional=True, doc="Deleted entry count" + field_id=506, name="deleted_data_files_count", field_type=IntegerType(), required=True, doc="Deleted entry count" ), NestedField( field_id=507, @@ -73,40 +71,40 @@ def test_iceberg_to_avro(manifest_schema): field_id=509, name="contains_null", field_type=BooleanType(), - is_optional=False, + required=False, doc="True if any file has a null partition value", ), NestedField( field_id=518, name="contains_nan", field_type=BooleanType(), - is_optional=True, + required=True, doc="True if any file has a nan partition value", ), NestedField( field_id=510, name="lower_bound", field_type=BinaryType(), - is_optional=True, + required=True, doc="Partition lower bound for all files", ), NestedField( field_id=511, name="upper_bound", field_type=BinaryType(), - is_optional=True, + required=True, doc="Partition upper bound for all files", ), ) ), - element_is_optional=False, + element_required=False, ), - is_optional=True, + required=True, doc="Summary for each partition", ), - NestedField(field_id=512, name="added_rows_count", field_type=LongType(), is_optional=True, doc="Added rows count"), - NestedField(field_id=513, name="existing_rows_count", field_type=LongType(), is_optional=True, doc="Existing rows count"), - NestedField(field_id=514, name="deleted_rows_count", field_type=LongType(), is_optional=True, doc="Deleted rows count"), + NestedField(field_id=512, name="added_rows_count", field_type=LongType(), required=True, doc="Added rows count"), + NestedField(field_id=513, name="existing_rows_count", field_type=LongType(), required=True, doc="Existing rows count"), + NestedField(field_id=514, name="deleted_rows_count", field_type=LongType(), required=True, doc="Deleted rows count"), schema_id=1, identifier_field_ids=[], ) @@ -135,8 +133,8 @@ def test_avro_list_required_primitive(): NestedField( field_id=100, name="array_with_string", - field_type=ListType(element_id=101, element_type=StringType(), element_is_optional=False), - is_optional=False, + field_type=ListType(element_id=101, element_type=StringType(), element_required=False), + required=False, ), schema_id=1, ) @@ -168,8 +166,8 @@ def test_avro_list_wrapped_primitive(): NestedField( field_id=100, name="array_with_string", - field_type=ListType(element_id=101, element_type=StringType(), element_is_optional=False), - is_optional=False, + field_type=ListType(element_id=101, element_type=StringType(), element_required=False), + required=False, ), schema_id=1, ) @@ -219,13 +217,13 @@ def test_avro_list_required_record(): element_id=101, element_type=StructType( fields=( - NestedField(field_id=102, name="contains_null", field_type=BooleanType(), is_optional=False), - NestedField(field_id=103, name="contains_nan", field_type=BooleanType(), is_optional=True), + NestedField(field_id=102, name="contains_null", field_type=BooleanType(), required=False), + NestedField(field_id=103, name="contains_nan", field_type=BooleanType(), required=True), ) ), - element_is_optional=False, + element_required=False, ), - is_optional=False, + required=False, ), schema_id=1, identifier_field_ids=[], @@ -256,7 +254,7 @@ def test_map_type(): "value-id": 102, } actual = AvroSchemaConversion()._convert_schema(avro_type) - expected = MapType(key_id=101, key_type=StringType(), value_id=102, value_type=LongType(), value_is_optional=True) + expected = MapType(key_id=101, key_type=StringType(), value_id=102, value_type=LongType(), value_required=True) assert actual == expected From 22168889ed78828db7ae6318205de91c78739457 Mon Sep 17 00:00:00 2001 From: Fokko Driesprong Date: Sun, 12 Jun 2022 21:58:40 +0200 Subject: [PATCH 110/642] Python: Extend the use of the Singleton (#5008) --- src/iceberg/expressions/base.py | 15 +-- src/iceberg/expressions/literals.py | 6 +- src/iceberg/types.py | 104 +++------------------ src/iceberg/utils/singleton.py | 28 ++++++ tests/expressions/test_expressions_base.py | 7 +- tests/test_types.py | 7 ++ 6 files changed, 65 insertions(+), 102 deletions(-) create mode 100644 src/iceberg/utils/singleton.py diff --git a/src/iceberg/expressions/base.py b/src/iceberg/expressions/base.py index 63a087ec12..06cc4d2bdb 100644 --- a/src/iceberg/expressions/base.py +++ b/src/iceberg/expressions/base.py @@ -14,14 +14,15 @@ # KIND, either express or implied. See the License for the # specific language governing permissions and limitations # under the License. -from abc import ABC, abstractmethod +from abc import ABCMeta, abstractmethod from enum import Enum, auto from functools import reduce, singledispatch from typing import Any, Generic, TypeVar from iceberg.files import StructProtocol from iceberg.schema import Accessor, Schema -from iceberg.types import NestedField, Singleton +from iceberg.types import NestedField +from iceberg.utils.singleton import Singleton T = TypeVar("T") @@ -88,7 +89,7 @@ def negate(self) -> "Operation": } -class Literal(Generic[T], ABC): +class Literal(Generic[T], metaclass=ABCMeta): """Literal which has a value and can be converted between types""" def __init__(self, value: T, value_type: type): @@ -129,7 +130,7 @@ def __ge__(self, other): return self.value >= other.value -class BooleanExpression(ABC): +class BooleanExpression(metaclass=ABCMeta): """base class for all boolean expressions""" @abstractmethod @@ -241,7 +242,7 @@ def __str__(self) -> str: return f"(not {self.child})" -class AlwaysTrue(BooleanExpression, Singleton): +class AlwaysTrue(BooleanExpression, metaclass=Singleton): """TRUE expression""" def __invert__(self) -> "AlwaysFalse": @@ -254,7 +255,7 @@ def __str__(self) -> str: return "true" -class AlwaysFalse(BooleanExpression, Singleton): +class AlwaysFalse(BooleanExpression, metaclass=Singleton): """FALSE expression""" def __invert__(self) -> "AlwaysTrue": @@ -348,7 +349,7 @@ def bind(self, schema: Schema, case_sensitive: bool) -> BoundReference: return BoundReference(field=field, accessor=schema.accessor_for_field(field.field_id)) -class BooleanExpressionVisitor(Generic[T], ABC): +class BooleanExpressionVisitor(Generic[T], metaclass=ABCMeta): @abstractmethod def visit_true(self) -> T: """Visit method for an AlwaysTrue boolean expression diff --git a/src/iceberg/expressions/literals.py b/src/iceberg/expressions/literals.py index a6128ee6c6..c89894d8ad 100644 --- a/src/iceberg/expressions/literals.py +++ b/src/iceberg/expressions/literals.py @@ -37,7 +37,6 @@ FloatType, IntegerType, LongType, - Singleton, StringType, TimestampType, TimestamptzType, @@ -51,6 +50,7 @@ timestamp_to_micros, timestamptz_to_micros, ) +from iceberg.utils.singleton import Singleton @singledispatch @@ -112,7 +112,7 @@ def _(value: Decimal) -> Literal[Decimal]: return DecimalLiteral(value) -class AboveMax(Singleton): +class AboveMax(metaclass=Singleton): @property def value(self): raise ValueError("AboveMax has no value") @@ -127,7 +127,7 @@ def __str__(self): return "AboveMax" -class BelowMin(Singleton): +class BelowMin(metaclass=Singleton): def __init__(self): pass diff --git a/src/iceberg/types.py b/src/iceberg/types.py index 46828e6d40..3139d13f85 100644 --- a/src/iceberg/types.py +++ b/src/iceberg/types.py @@ -31,25 +31,13 @@ """ from dataclasses import dataclass, field from functools import cached_property -from typing import ( - ClassVar, - Dict, - Optional, - Tuple, -) +from typing import ClassVar, Optional, Tuple - -class Singleton: - _instance = None - - def __new__(cls): - if not isinstance(cls._instance, cls): - cls._instance = super().__new__(cls) - return cls._instance +from iceberg.utils.singleton import Singleton @dataclass(frozen=True) -class IcebergType: +class IcebergType(metaclass=Singleton): """Base type for all Iceberg Types Example: @@ -94,12 +82,6 @@ class FixedType(PrimitiveType): length: int = field() - _instances: ClassVar[Dict[int, "FixedType"]] = {} - - def __new__(cls, length: int): - cls._instances[length] = cls._instances.get(length) or object.__new__(cls) - return cls._instances[length] - @property def string_type(self) -> str: return f"fixed[{self.length}]" @@ -119,13 +101,6 @@ class DecimalType(PrimitiveType): precision: int = field() scale: int = field() - _instances: ClassVar[Dict[Tuple[int, int], "DecimalType"]] = {} - - def __new__(cls, precision: int, scale: int): - key = (precision, scale) - cls._instances[key] = cls._instances.get(key) or object.__new__(cls) - return cls._instances[key] - @property def string_type(self) -> str: return f"decimal({self.precision}, {self.scale})" @@ -161,20 +136,6 @@ class NestedField(IcebergType): required: bool = field(default=True) doc: Optional[str] = field(default=None, repr=False) - _instances: ClassVar[Dict[Tuple[bool, int, str, IcebergType, Optional[str]], "NestedField"]] = {} - - def __new__( - cls, - field_id: int, - name: str, - field_type: IcebergType, - required: bool = True, - doc: Optional[str] = None, - ): - key = (required, field_id, name, field_type, doc) - cls._instances[key] = cls._instances.get(key) or object.__new__(cls) - return cls._instances[key] - @property def optional(self) -> bool: return not self.required @@ -200,14 +161,6 @@ class StructType(IcebergType): fields: Tuple[NestedField] = field() - _instances: ClassVar[Dict[Tuple[NestedField, ...], "StructType"]] = {} - - def __new__(cls, *fields: NestedField, **kwargs): - if not fields and "fields" in kwargs: - fields = kwargs["fields"] - cls._instances[fields] = cls._instances.get(fields) or object.__new__(cls) - return cls._instances[fields] - def __init__(self, *fields: NestedField, **kwargs): # pylint: disable=super-init-not-called if not fields and "fields" in kwargs: fields = kwargs["fields"] @@ -232,18 +185,6 @@ class ListType(IcebergType): element_required: bool = field(default=True) element: NestedField = field(init=False, repr=False) - _instances: ClassVar[Dict[Tuple[bool, int, IcebergType], "ListType"]] = {} - - def __new__( - cls, - element_id: int, - element_type: IcebergType, - element_required: bool = True, - ): - key = (element_required, element_id, element_type) - cls._instances[key] = cls._instances.get(key) or object.__new__(cls) - return cls._instances[key] - def __post_init__(self): object.__setattr__( self, @@ -278,21 +219,6 @@ class MapType(IcebergType): key: NestedField = field(init=False, repr=False) value: NestedField = field(init=False, repr=False) - # _type_string_def = lambda self: f"map<{self.key_type}, {self.value_type}>" - _instances: ClassVar[Dict[Tuple[int, IcebergType, int, IcebergType, bool], "MapType"]] = {} - - def __new__( - cls, - key_id: int, - key_type: IcebergType, - value_id: int, - value_type: IcebergType, - value_required: bool = True, - ): - impl_key = (key_id, key_type, value_id, value_type, value_required) - cls._instances[impl_key] = cls._instances.get(impl_key) or object.__new__(cls) - return cls._instances[impl_key] - def __post_init__(self): object.__setattr__(self, "key", NestedField(name="key", field_id=self.key_id, field_type=self.key_type, required=False)) object.__setattr__( @@ -308,7 +234,7 @@ def __post_init__(self): @dataclass(frozen=True) -class BooleanType(PrimitiveType, Singleton): +class BooleanType(PrimitiveType): """A boolean data type in Iceberg can be represented using an instance of this class. Example: @@ -325,7 +251,7 @@ def string_type(self) -> str: @dataclass(frozen=True) -class IntegerType(PrimitiveType, Singleton): +class IntegerType(PrimitiveType): """An Integer data type in Iceberg can be represented using an instance of this class. Integers in Iceberg are 32-bit signed and can be promoted to Longs. @@ -350,7 +276,7 @@ def string_type(self) -> str: @dataclass(frozen=True) -class LongType(PrimitiveType, Singleton): +class LongType(PrimitiveType): """A Long data type in Iceberg can be represented using an instance of this class. Longs in Iceberg are 64-bit signed integers. @@ -379,7 +305,7 @@ def string_type(self) -> str: @dataclass(frozen=True) -class FloatType(PrimitiveType, Singleton): +class FloatType(PrimitiveType): """A Float data type in Iceberg can be represented using an instance of this class. Floats in Iceberg are 32-bit IEEE 754 floating points and can be promoted to Doubles. @@ -406,7 +332,7 @@ def string_type(self) -> str: @dataclass(frozen=True) -class DoubleType(PrimitiveType, Singleton): +class DoubleType(PrimitiveType): """A Double data type in Iceberg can be represented using an instance of this class. Doubles in Iceberg are 64-bit IEEE 754 floating points. @@ -424,7 +350,7 @@ def string_type(self) -> str: @dataclass(frozen=True) -class DateType(PrimitiveType, Singleton): +class DateType(PrimitiveType): """A Date data type in Iceberg can be represented using an instance of this class. Dates in Iceberg are calendar dates without a timezone or time. @@ -442,7 +368,7 @@ def string_type(self) -> str: @dataclass(frozen=True) -class TimeType(PrimitiveType, Singleton): +class TimeType(PrimitiveType): """A Time data type in Iceberg can be represented using an instance of this class. Times in Iceberg have microsecond precision and are a time of day without a date or timezone. @@ -460,7 +386,7 @@ def string_type(self) -> str: @dataclass(frozen=True) -class TimestampType(PrimitiveType, Singleton): +class TimestampType(PrimitiveType): """A Timestamp data type in Iceberg can be represented using an instance of this class. Timestamps in Iceberg have microsecond precision and include a date and a time of day without a timezone. @@ -478,7 +404,7 @@ def string_type(self) -> str: @dataclass(frozen=True) -class TimestamptzType(PrimitiveType, Singleton): +class TimestamptzType(PrimitiveType): """A Timestamptz data type in Iceberg can be represented using an instance of this class. Timestamptzs in Iceberg are stored as UTC and include a date and a time of day with a timezone. @@ -496,7 +422,7 @@ def string_type(self) -> str: @dataclass(frozen=True) -class StringType(PrimitiveType, Singleton): +class StringType(PrimitiveType): """A String data type in Iceberg can be represented using an instance of this class. Strings in Iceberg are arbitrary-length character sequences and are encoded with UTF-8. @@ -514,7 +440,7 @@ def string_type(self) -> str: @dataclass(frozen=True) -class UUIDType(PrimitiveType, Singleton): +class UUIDType(PrimitiveType): """A UUID data type in Iceberg can be represented using an instance of this class. UUIDs in Iceberg are universally unique identifiers. @@ -532,7 +458,7 @@ def string_type(self) -> str: @dataclass(frozen=True) -class BinaryType(PrimitiveType, Singleton): +class BinaryType(PrimitiveType): """A Binary data type in Iceberg can be represented using an instance of this class. Binaries in Iceberg are arbitrary-length byte arrays. diff --git a/src/iceberg/utils/singleton.py b/src/iceberg/utils/singleton.py new file mode 100644 index 0000000000..f6c6912fab --- /dev/null +++ b/src/iceberg/utils/singleton.py @@ -0,0 +1,28 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +from abc import ABCMeta +from typing import ClassVar, Dict + + +class Singleton(ABCMeta): + _instances: ClassVar[Dict] = {} + + def __call__(cls, *args, **kwargs): + key = (cls, args, tuple(sorted(kwargs.items()))) + if key not in cls._instances: + cls._instances[key] = super().__call__(*args, **kwargs) + return cls._instances[key] diff --git a/tests/expressions/test_expressions_base.py b/tests/expressions/test_expressions_base.py index 7fe3e9817e..3d9177bbfe 100644 --- a/tests/expressions/test_expressions_base.py +++ b/tests/expressions/test_expressions_base.py @@ -22,7 +22,8 @@ import pytest from iceberg.expressions import base -from iceberg.types import NestedField, Singleton, StringType +from iceberg.types import NestedField, StringType +from iceberg.utils.singleton import Singleton @pytest.mark.parametrize( @@ -63,7 +64,7 @@ def test_raise_on_no_negation_for_operation(operation): assert str(exc_info.value) == f"No negation defined for operation {operation}" -class TestExpressionA(base.BooleanExpression, Singleton): +class TestExpressionA(base.BooleanExpression, metaclass=Singleton): def __invert__(self): return TestExpressionB() @@ -74,7 +75,7 @@ def __str__(self): return "testexpra" -class TestExpressionB(base.BooleanExpression, Singleton): +class TestExpressionB(base.BooleanExpression, metaclass=Singleton): def __invert__(self): return TestExpressionA() diff --git a/tests/test_types.py b/tests/test_types.py index 26f6f421a7..3d75b88a58 100644 --- a/tests/test_types.py +++ b/tests/test_types.py @@ -204,3 +204,10 @@ def test_non_parameterized_type_equality(input_index, input_type, check_index, c assert input_type() == check_type() else: assert input_type() != check_type() + + +def test_types_singleton(): + """The types are immutable so we can return the same instance multiple times""" + assert id(BooleanType()) == id(BooleanType()) + assert id(FixedType(22)) == id(FixedType(22)) + assert id(FixedType(19)) != id(FixedType(25)) From b309e27f534d73d11f9cd8d3dd894eef393896d7 Mon Sep 17 00:00:00 2001 From: Hongyue/Steve Zhang Date: Sun, 12 Jun 2022 13:55:57 -0700 Subject: [PATCH 111/642] Python: Add PartitionSpec (#4717) Co-authored-by: Fokko Driesprong Co-authored-by: Steve Zhang --- spellcheck-dictionary.txt | 1 + src/iceberg/table/partitioning.py | 106 ++++++++++++++++++++++-------- tests/table/test_partitioning.py | 41 +++++++++++- 3 files changed, 120 insertions(+), 28 deletions(-) diff --git a/spellcheck-dictionary.txt b/spellcheck-dictionary.txt index ef3ab6642b..41da147758 100644 --- a/spellcheck-dictionary.txt +++ b/spellcheck-dictionary.txt @@ -37,6 +37,7 @@ NaN nan NestedField nullability +PartitionField pragma PrimitiveType pyarrow diff --git a/src/iceberg/table/partitioning.py b/src/iceberg/table/partitioning.py index d1e9debd49..36e59e8e1b 100644 --- a/src/iceberg/table/partitioning.py +++ b/src/iceberg/table/partitioning.py @@ -14,9 +14,16 @@ # KIND, either express or implied. See the License for the # specific language governing permissions and limitations # under the License. +from dataclasses import dataclass, field +from typing import Dict, List, Tuple + +from iceberg.schema import Schema from iceberg.transforms import Transform +_PARTITION_DATA_ID_START: int = 1000 + +@dataclass(frozen=True) class PartitionField: """ PartitionField is a single element with name and unique id, @@ -29,38 +36,85 @@ class PartitionField: name(str): The name of this partition field """ - def __init__(self, source_id: int, field_id: int, transform: Transform, name: str): - self._source_id = source_id - self._field_id = field_id - self._transform = transform - self._name = name + source_id: int + field_id: int + transform: Transform + name: str + + def __str__(self): + return f"{self.field_id}: {self.name}: {self.transform}({self.source_id})" + - @property - def source_id(self) -> int: - return self._source_id +@dataclass(eq=False, frozen=True) +class PartitionSpec: + """ + PartitionSpec captures the transformation from table data to partition values - @property - def field_id(self) -> int: - return self._field_id + Attributes: + schema(Schema): the schema of data table + spec_id(int): any change to PartitionSpec will produce a new specId + fields(List[PartitionField): list of partition fields to produce partition values + last_assigned_field_id(int): auto-increment partition field id starting from PARTITION_DATA_ID_START + """ - @property - def name(self) -> str: - return self._name + schema: Schema + spec_id: int + fields: Tuple[PartitionField, ...] + last_assigned_field_id: int + source_id_to_fields_map: Dict[int, List[PartitionField]] = field(init=False, repr=False) - @property - def transform(self) -> Transform: - return self._transform + def __post_init__(self): + source_id_to_fields_map = dict() + for partition_field in self.fields: + source_column = self.schema.find_column_name(partition_field.source_id) + if not source_column: + raise ValueError(f"Cannot find source column: {partition_field.source_id}") + existing = source_id_to_fields_map.get(partition_field.source_id, []) + existing.append(partition_field) + source_id_to_fields_map[partition_field.source_id] = existing + object.__setattr__(self, "source_id_to_fields_map", source_id_to_fields_map) def __eq__(self, other): - return ( - self.field_id == other.field_id - and self.source_id == other.source_id - and self.name == other.name - and self.transform == other.transform - ) + """ + Produce a boolean to return True if two objects are considered equal + + Note: + Equality of PartitionSpec is determined by spec_id and partition fields only + """ + if not isinstance(other, PartitionSpec): + return False + return self.spec_id == other.spec_id and self.fields == other.fields def __str__(self): - return f"{self.field_id}: {self.name}: {self.transform}({self.source_id})" + """ + Produce a human-readable string representation of PartitionSpec - def __repr__(self): - return f"PartitionField(field_id={self.field_id}, name={self.name}, transform={repr(self.transform)}, source_id={self.source_id})" + Note: + Only include list of partition fields in the PartitionSpec's string representation + """ + result_str = "[" + if self.fields: + result_str += "\n " + "\n ".join([str(field) for field in self.fields]) + "\n" + result_str += "]" + return result_str + + def is_unpartitioned(self) -> bool: + return not self.fields + + def fields_by_source_id(self, field_id: int) -> List[PartitionField]: + return self.source_id_to_fields_map[field_id] + + def compatible_with(self, other: "PartitionSpec") -> bool: + """ + Produce a boolean to return True if two PartitionSpec are considered compatible + """ + if self == other: + return True + if len(self.fields) != len(other.fields): + return False + return all( + this_field.source_id == that_field.source_id + and this_field.transform == that_field.transform + and this_field.name == that_field.name + for this_field, that_field in zip(self.fields, other.fields) + ) diff --git a/tests/table/test_partitioning.py b/tests/table/test_partitioning.py index baf2e5df7a..14107ffdd0 100644 --- a/tests/table/test_partitioning.py +++ b/tests/table/test_partitioning.py @@ -15,7 +15,8 @@ # specific language governing permissions and limitations # under the License. -from iceberg.table.partitioning import PartitionField +from iceberg.schema import Schema +from iceberg.table.partitioning import PartitionField, PartitionSpec from iceberg.transforms import bucket from iceberg.types import IntegerType @@ -32,5 +33,41 @@ def test_partition_field_init(): assert str(partition_field) == "1000: id: bucket[100](3)" assert ( repr(partition_field) - == "PartitionField(field_id=1000, name=id, transform=transforms.bucket(source_type=IntegerType(), num_buckets=100), source_id=3)" + == "PartitionField(source_id=3, field_id=1000, transform=transforms.bucket(source_type=IntegerType(), num_buckets=100), name='id')" ) + + +def test_partition_spec_init(table_schema_simple: Schema): + bucket_transform = bucket(IntegerType(), 4) + id_field1 = PartitionField(3, 1001, bucket_transform, "id") + partition_spec1 = PartitionSpec(table_schema_simple, 0, (id_field1,), 1001) + + assert partition_spec1.spec_id == 0 + assert partition_spec1.schema == table_schema_simple + assert partition_spec1 == partition_spec1 + assert partition_spec1 != id_field1 + assert str(partition_spec1) == f"[\n {str(id_field1)}\n]" + assert not partition_spec1.is_unpartitioned() + # only differ by PartitionField field_id + id_field2 = PartitionField(3, 1002, bucket_transform, "id") + partition_spec2 = PartitionSpec(table_schema_simple, 0, (id_field2,), 1001) + assert partition_spec1 != partition_spec2 + assert partition_spec1.compatible_with(partition_spec2) + assert partition_spec1.fields_by_source_id(3) == [id_field1] + + +def test_partition_compatible_with(table_schema_simple: Schema): + bucket_transform = bucket(IntegerType(), 4) + field1 = PartitionField(3, 100, bucket_transform, "id") + field2 = PartitionField(3, 102, bucket_transform, "id") + lhs = PartitionSpec(table_schema_simple, 0, (field1,), 1001) + rhs = PartitionSpec(table_schema_simple, 0, (field1, field2), 1001) + assert not lhs.compatible_with(rhs) + + +def test_unpartitioned(table_schema_simple: Schema): + unpartitioned = PartitionSpec(table_schema_simple, 1, tuple(), 1000) + + assert not unpartitioned.fields + assert unpartitioned.is_unpartitioned() + assert str(unpartitioned) == "[]" From 6379b5ba9e2a5de53cdd59f75c26726e595c7aad Mon Sep 17 00:00:00 2001 From: jun-he Date: Mon, 13 Jun 2022 13:15:29 -0700 Subject: [PATCH 112/642] Python: Add identity transform (#4908) --- src/iceberg/transforms.py | 81 +++++++++++++++++++++++++++++++++++ src/iceberg/utils/datetime.py | 34 ++++++++++++++- tests/test_transforms.py | 48 +++++++++++++++++++++ 3 files changed, 162 insertions(+), 1 deletion(-) diff --git a/src/iceberg/transforms.py b/src/iceberg/transforms.py index 39ec5d538e..fbbdd917aa 100644 --- a/src/iceberg/transforms.py +++ b/src/iceberg/transforms.py @@ -15,9 +15,11 @@ # specific language governing permissions and limitations # under the License. +import base64 import struct from abc import ABC, abstractmethod from decimal import Decimal +from functools import singledispatchmethod from typing import Generic, Optional, TypeVar from uuid import UUID @@ -37,6 +39,7 @@ TimeType, UUIDType, ) +from iceberg.utils import datetime from iceberg.utils.decimal import decimal_to_bytes S = TypeVar("S") @@ -224,6 +227,80 @@ def hash(self, value: UUID) -> int: ) +def _base64encode(buffer: bytes) -> str: + """Converts bytes to base64 string""" + return base64.b64encode(buffer).decode("ISO-8859-1") + + +class IdentityTransform(Transform[S, S]): + """Transforms a value into itself. + + Example: + >>> transform = IdentityTransform(StringType()) + >>> transform.apply('hello-world') + 'hello-world' + """ + + def __init__(self, source_type: IcebergType): + super().__init__( + "identity", + f"transforms.identity(source_type={repr(source_type)})", + ) + self._type = source_type + + def apply(self, value: Optional[S]) -> Optional[S]: + return value + + def can_transform(self, source: IcebergType) -> bool: + return source.is_primitive + + def result_type(self, source: IcebergType) -> IcebergType: + return source + + @property + def preserves_order(self) -> bool: + return True + + def satisfies_order_of(self, other: Transform) -> bool: + """ordering by value is the same as long as the other preserves order""" + return other.preserves_order + + def to_human_string(self, value: Optional[S]) -> str: + return self._human_string(value) + + @singledispatchmethod + def _human_string(self, value: Optional[S]) -> str: + return str(value) if value is not None else "null" + + @_human_string.register(bytes) + def _(self, value: bytes) -> str: + return _base64encode(value) + + @_human_string.register(int) + def _(self, value: int) -> str: + return self._int_to_human_string(self._type, value) + + @singledispatchmethod + def _int_to_human_string(self, value_type: IcebergType, value: int) -> str: + return str(value) + + @_int_to_human_string.register(DateType) + def _(self, value_type: IcebergType, value: int) -> str: + return datetime.to_human_day(value) + + @_int_to_human_string.register(TimeType) + def _(self, value_type: IcebergType, value: int) -> str: + return datetime.to_human_time(value) + + @_int_to_human_string.register(TimestampType) + def _(self, value_type: IcebergType, value: int) -> str: + return datetime.to_human_timestamp(value) + + @_int_to_human_string.register(TimestamptzType) + def _(self, value_type: IcebergType, value: int) -> str: + return datetime.to_human_timestamptz(value) + + class UnknownTransform(Transform): """A transform that represents when an unknown transform is provided Args: @@ -294,5 +371,9 @@ def bucket(source_type: IcebergType, num_buckets: int) -> BaseBucketTransform: raise ValueError(f"Cannot bucket by type: {source_type}") +def identity(source_type: IcebergType) -> IdentityTransform: + return IdentityTransform(source_type) + + def always_null() -> Transform: return VoidTransform() diff --git a/src/iceberg/utils/datetime.py b/src/iceberg/utils/datetime.py index c8c12393b6..c10dda4fb2 100644 --- a/src/iceberg/utils/datetime.py +++ b/src/iceberg/utils/datetime.py @@ -17,7 +17,12 @@ """Helper methods for working with date/time representations """ import re -from datetime import date, datetime, time +from datetime import ( + date, + datetime, + time, + timedelta, +) EPOCH_DATE = date.fromisoformat("1970-01-01") EPOCH_TIMESTAMP = datetime.fromisoformat("1970-01-01T00:00:00.000000") @@ -42,6 +47,13 @@ def time_to_micros(time_str: str) -> int: return (((t.hour * 60 + t.minute) * 60) + t.second) * 1_000_000 + t.microsecond +def time_from_micros(micros: int) -> time: + seconds = micros // 1_000_000 + minutes = seconds // 60 + hours = minutes // 60 + return time(hour=hours, minute=minutes % 60, second=seconds % 60, microsecond=micros % 1_000_000) + + def datetime_to_micros(dt: datetime) -> int: """Converts a datetime to microseconds from 1970-01-01T00:00:00.000000""" if dt.tzinfo: @@ -63,3 +75,23 @@ def timestamptz_to_micros(timestamptz_str: str) -> int: if ISO_TIMESTAMPTZ.fullmatch(timestamptz_str): return datetime_to_micros(datetime.fromisoformat(timestamptz_str)) raise ValueError(f"Invalid timestamp with zone: {timestamptz_str} (must be ISO-8601)") + + +def to_human_day(day_ordinal: int) -> str: + """Converts a DateType value to human string""" + return (EPOCH_DATE + timedelta(days=day_ordinal)).isoformat() + + +def to_human_time(micros_from_midnight: int) -> str: + """Converts a TimeType value to human string""" + return time_from_micros(micros_from_midnight).isoformat() + + +def to_human_timestamptz(timestamp_micros: int) -> str: + """Converts a TimestamptzType value to human string""" + return (EPOCH_TIMESTAMPTZ + timedelta(microseconds=timestamp_micros)).isoformat() + + +def to_human_timestamp(timestamp_micros: int) -> str: + """Converts a TimestampType value to human string""" + return (EPOCH_TIMESTAMP + timedelta(microseconds=timestamp_micros)).isoformat() diff --git a/tests/test_transforms.py b/tests/test_transforms.py index 7855345f9d..d7b8e968a9 100644 --- a/tests/test_transforms.py +++ b/tests/test_transforms.py @@ -28,7 +28,9 @@ BooleanType, DateType, DecimalType, + DoubleType, FixedType, + FloatType, IntegerType, LongType, StringType, @@ -129,6 +131,52 @@ def test_string_with_surrogate_pair(): assert bucket_transform.hash(string_with_surrogate_pair) == mmh3.hash(as_bytes) +@pytest.mark.parametrize( + "type_var,value,expected", + [ + (LongType(), None, "null"), + (DateType(), 17501, "2017-12-01"), + (TimeType(), 36775038194, "10:12:55.038194"), + (TimestamptzType(), 1512151975038194, "2017-12-01T18:12:55.038194+00:00"), + (TimestampType(), 1512151975038194, "2017-12-01T18:12:55.038194"), + (LongType(), -1234567890000, "-1234567890000"), + (StringType(), "a/b/c=d", "a/b/c=d"), + (DecimalType(9, 2), Decimal("-1.50"), "-1.50"), + (FixedType(100), b"foo", "Zm9v"), + ], +) +def test_identity_human_string(type_var, value, expected): + identity = transforms.identity(type_var) + assert identity.to_human_string(value) == expected + + +@pytest.mark.parametrize( + "type_var", + [ + BinaryType(), + BooleanType(), + DateType(), + DecimalType(8, 2), + DoubleType(), + FixedType(16), + FloatType(), + IntegerType(), + LongType(), + StringType(), + TimestampType(), + TimestamptzType(), + TimeType(), + UUIDType(), + ], +) +def test_identity_method(type_var): + identity_transform = transforms.identity(type_var) + assert str(identity_transform) == str(eval(repr(identity_transform))) + assert identity_transform.can_transform(type_var) + assert identity_transform.result_type(type_var) == type_var + assert identity_transform.apply("test") == "test" + + def test_unknown_transform(): unknown_transform = transforms.UnknownTransform(FixedType(8), "unknown") assert str(unknown_transform) == str(eval(repr(unknown_transform))) From ae6f7169fad1ba974fc7bb91a0f44939d97fc95b Mon Sep 17 00:00:00 2001 From: Fokko Driesprong Date: Wed, 15 Jun 2022 23:07:43 +0200 Subject: [PATCH 113/642] Python: Don't use a metaclass for the Singleton (#5055) Metaclasses in Python don't mix very well, and when they cause conflicts then it breaks at initialization. Example of metaclasses is the ABC, but also the Pydantic BaseModel. I tried to circumvent this by extending the Singleton from ABC but it is likely that we use different metaclasses in the future, and then it will bring us sadness and misery. --- src/iceberg/expressions/base.py | 12 ++++++------ src/iceberg/expressions/literals.py | 4 ++-- src/iceberg/types.py | 3 ++- src/iceberg/utils/singleton.py | 8 ++++---- tests/expressions/test_expressions_base.py | 4 ++-- 5 files changed, 16 insertions(+), 15 deletions(-) diff --git a/src/iceberg/expressions/base.py b/src/iceberg/expressions/base.py index 06cc4d2bdb..919cc09b86 100644 --- a/src/iceberg/expressions/base.py +++ b/src/iceberg/expressions/base.py @@ -14,7 +14,7 @@ # KIND, either express or implied. See the License for the # specific language governing permissions and limitations # under the License. -from abc import ABCMeta, abstractmethod +from abc import ABC, abstractmethod from enum import Enum, auto from functools import reduce, singledispatch from typing import Any, Generic, TypeVar @@ -89,7 +89,7 @@ def negate(self) -> "Operation": } -class Literal(Generic[T], metaclass=ABCMeta): +class Literal(Generic[T], ABC): """Literal which has a value and can be converted between types""" def __init__(self, value: T, value_type: type): @@ -130,7 +130,7 @@ def __ge__(self, other): return self.value >= other.value -class BooleanExpression(metaclass=ABCMeta): +class BooleanExpression(ABC): """base class for all boolean expressions""" @abstractmethod @@ -242,7 +242,7 @@ def __str__(self) -> str: return f"(not {self.child})" -class AlwaysTrue(BooleanExpression, metaclass=Singleton): +class AlwaysTrue(BooleanExpression, ABC, Singleton): """TRUE expression""" def __invert__(self) -> "AlwaysFalse": @@ -255,7 +255,7 @@ def __str__(self) -> str: return "true" -class AlwaysFalse(BooleanExpression, metaclass=Singleton): +class AlwaysFalse(BooleanExpression, ABC, Singleton): """FALSE expression""" def __invert__(self) -> "AlwaysTrue": @@ -349,7 +349,7 @@ def bind(self, schema: Schema, case_sensitive: bool) -> BoundReference: return BoundReference(field=field, accessor=schema.accessor_for_field(field.field_id)) -class BooleanExpressionVisitor(Generic[T], metaclass=ABCMeta): +class BooleanExpressionVisitor(Generic[T], ABC): @abstractmethod def visit_true(self) -> T: """Visit method for an AlwaysTrue boolean expression diff --git a/src/iceberg/expressions/literals.py b/src/iceberg/expressions/literals.py index c89894d8ad..2f0af3b6e0 100644 --- a/src/iceberg/expressions/literals.py +++ b/src/iceberg/expressions/literals.py @@ -112,7 +112,7 @@ def _(value: Decimal) -> Literal[Decimal]: return DecimalLiteral(value) -class AboveMax(metaclass=Singleton): +class AboveMax(Singleton): @property def value(self): raise ValueError("AboveMax has no value") @@ -127,7 +127,7 @@ def __str__(self): return "AboveMax" -class BelowMin(metaclass=Singleton): +class BelowMin(Singleton): def __init__(self): pass diff --git a/src/iceberg/types.py b/src/iceberg/types.py index 3139d13f85..d665dbcae6 100644 --- a/src/iceberg/types.py +++ b/src/iceberg/types.py @@ -29,6 +29,7 @@ Notes: - https://iceberg.apache.org/#spec/#primitive-types """ +from abc import ABC from dataclasses import dataclass, field from functools import cached_property from typing import ClassVar, Optional, Tuple @@ -37,7 +38,7 @@ @dataclass(frozen=True) -class IcebergType(metaclass=Singleton): +class IcebergType(ABC, Singleton): """Base type for all Iceberg Types Example: diff --git a/src/iceberg/utils/singleton.py b/src/iceberg/utils/singleton.py index f6c6912fab..c36155de14 100644 --- a/src/iceberg/utils/singleton.py +++ b/src/iceberg/utils/singleton.py @@ -14,15 +14,15 @@ # KIND, either express or implied. See the License for the # specific language governing permissions and limitations # under the License. -from abc import ABCMeta + from typing import ClassVar, Dict -class Singleton(ABCMeta): +class Singleton: _instances: ClassVar[Dict] = {} - def __call__(cls, *args, **kwargs): + def __new__(cls, *args, **kwargs): key = (cls, args, tuple(sorted(kwargs.items()))) if key not in cls._instances: - cls._instances[key] = super().__call__(*args, **kwargs) + cls._instances[key] = super().__new__(cls) return cls._instances[key] diff --git a/tests/expressions/test_expressions_base.py b/tests/expressions/test_expressions_base.py index 3d9177bbfe..773d7ecc53 100644 --- a/tests/expressions/test_expressions_base.py +++ b/tests/expressions/test_expressions_base.py @@ -64,7 +64,7 @@ def test_raise_on_no_negation_for_operation(operation): assert str(exc_info.value) == f"No negation defined for operation {operation}" -class TestExpressionA(base.BooleanExpression, metaclass=Singleton): +class TestExpressionA(base.BooleanExpression, Singleton): def __invert__(self): return TestExpressionB() @@ -75,7 +75,7 @@ def __str__(self): return "testexpra" -class TestExpressionB(base.BooleanExpression, metaclass=Singleton): +class TestExpressionB(base.BooleanExpression, Singleton): def __invert__(self): return TestExpressionA() From d843fa6e531ce51daa9e93f54e3441f3de15e0f1 Mon Sep 17 00:00:00 2001 From: Fokko Driesprong Date: Fri, 17 Jun 2022 17:33:56 +0200 Subject: [PATCH 114/642] Python: Add Flake8 for simple checks (#5073) --- .pre-commit-config.yaml | 5 +++++ tests/expressions/test_expressions_base.py | 6 +++--- tests/io/test_io_base.py | 2 +- tests/test_conversions.py | 4 ++-- tests/test_schema.py | 6 +++--- 5 files changed, 14 insertions(+), 9 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index b30c653478..8a8b31418e 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -50,3 +50,8 @@ repos: hooks: - id: pyupgrade args: [--py38-plus] + - repo: https://github.com/pycqa/flake8 + rev: '4.0.1' + hooks: + - id: flake8 + args: [ "--ignore=E501,W503" ] diff --git a/tests/expressions/test_expressions_base.py b/tests/expressions/test_expressions_base.py index 773d7ecc53..a6f8f07f76 100644 --- a/tests/expressions/test_expressions_base.py +++ b/tests/expressions/test_expressions_base.py @@ -275,8 +275,8 @@ def test_accessor_base_class(foo_struct): assert base.Accessor(position=6).get(foo_struct) == 1.234 assert base.Accessor(position=7).get(foo_struct) == Decimal("1.234") assert base.Accessor(position=8).get(foo_struct) == uuid_value - assert base.Accessor(position=9).get(foo_struct) == True - assert base.Accessor(position=10).get(foo_struct) == False + assert base.Accessor(position=9).get(foo_struct) is True + assert base.Accessor(position=10).get(foo_struct) is False assert base.Accessor(position=11).get(foo_struct) == b"\x19\x04\x9e?" @@ -317,7 +317,7 @@ def test_bound_reference(table_schema_simple, foo_struct): assert bound_ref1.eval(foo_struct) == "foovalue" assert bound_ref2.eval(foo_struct) == 123 - assert bound_ref3.eval(foo_struct) == True + assert bound_ref3.eval(foo_struct) is True def test_boolean_expression_visitor(): diff --git a/tests/io/test_io_base.py b/tests/io/test_io_base.py index 3cccd90a05..72495e19b6 100644 --- a/tests/io/test_io_base.py +++ b/tests/io/test_io_base.py @@ -322,7 +322,7 @@ def test_raise_file_not_found_error_for_fileio_delete(CustomFileIO): with pytest.raises(FileNotFoundError) as exc_info: file_io.delete(output_file_location) - assert (f"Cannot delete file") in str(exc_info.value) + assert "Cannot delete file" in str(exc_info.value) # Confirm that the file no longer exists assert not os.path.exists(output_file_location) diff --git a/tests/test_conversions.py b/tests/test_conversions.py index 76816a50c4..0f50868f7d 100644 --- a/tests/test_conversions.py +++ b/tests/test_conversions.py @@ -179,7 +179,7 @@ def test_partition_to_py(primitive_type, value_str, expected_result): ) def test_none_partition_values(primitive_type): """Test converting a partition value to a python built-in""" - assert conversions.partition_to_py(primitive_type, None) == None + assert conversions.partition_to_py(primitive_type, None) is None @pytest.mark.parametrize( @@ -203,7 +203,7 @@ def test_none_partition_values(primitive_type): ) def test_hive_default_partition_values(primitive_type): """Test converting a partition value to a python built-in""" - assert conversions.partition_to_py(primitive_type, "__HIVE_DEFAULT_PARTITION__") == None + assert conversions.partition_to_py(primitive_type, "__HIVE_DEFAULT_PARTITION__") is None @pytest.mark.parametrize( diff --git a/tests/test_schema.py b/tests/test_schema.py index e5afe5fb74..81ebcf3556 100644 --- a/tests/test_schema.py +++ b/tests/test_schema.py @@ -218,19 +218,19 @@ def test_schema_find_field_by_id(table_schema_simple): assert isinstance(column1, NestedField) assert column1.field_id == 1 assert column1.field_type == StringType() - assert column1.required == False + assert column1.required is False column2 = index[2] assert isinstance(column2, NestedField) assert column2.field_id == 2 assert column2.field_type == IntegerType() - assert column2.required == True + assert column2.required is True column3 = index[3] assert isinstance(column3, NestedField) assert column3.field_id == 3 assert column3.field_type == BooleanType() - assert column3.required == False + assert column3.required is False def test_schema_find_field_by_id_raise_on_unknown_field(table_schema_simple): From 517ab114a61fb3c50f86598c0d2969ca2f87c8b9 Mon Sep 17 00:00:00 2001 From: Fokko Driesprong Date: Mon, 20 Jun 2022 17:35:32 +0200 Subject: [PATCH 115/642] Python: Add py.typed file for annotations (#5091) https://blog.whtsky.me/tech/2021/dont-forget-py.typed-for-your-typed-python-package/ --- py.typed | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) create mode 100644 py.typed diff --git a/py.typed b/py.typed new file mode 100644 index 0000000000..fb618a1375 --- /dev/null +++ b/py.typed @@ -0,0 +1,18 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +# Marker file for PEP 561 From 64af49ebb51a16f1c131bf5d68864105c73811b5 Mon Sep 17 00:00:00 2001 From: Fokko Driesprong Date: Mon, 20 Jun 2022 17:47:41 +0200 Subject: [PATCH 116/642] Python: Add Avro read path (#4920) --- .pre-commit-config.yaml | 2 +- LICENSE | 11 + poetry.lock | 312 ++++++-- pyproject.toml | 15 + spellcheck-dictionary.txt | 6 + src/iceberg/avro/__init__.py | 16 + src/iceberg/avro/codecs/__init__.py | 40 + src/iceberg/avro/codecs/bzip2.py | 43 ++ src/iceberg/avro/codecs/codec.py | 33 + src/iceberg/avro/codecs/deflate.py | 36 + src/iceberg/avro/codecs/snappy_codec.py | 69 ++ src/iceberg/avro/codecs/zstandard_codec.py | 53 ++ src/iceberg/avro/decoder.py | 165 +++++ src/iceberg/avro/file.py | 181 +++++ src/iceberg/avro/reader.py | 316 ++++++++ src/iceberg/files.py | 2 +- src/iceberg/io/base.py | 15 +- src/iceberg/io/memory.py | 75 ++ src/iceberg/types.py | 2 +- src/iceberg/utils/datetime.py | 37 +- src/iceberg/utils/schema_conversion.py | 50 +- src/iceberg/utils/singleton.py | 13 + tests/avro/__init__.py | 16 + tests/avro/test_decoder.py | 140 ++++ tests/avro/test_file.py | 48 ++ tests/avro/test_reader.py | 455 ++++++++++++ tests/conftest.py | 825 ++++++++++++++++++--- tests/io/__init__.py | 23 +- tests/io/test_io_base.py | 2 +- tests/utils/test_schema_conversion.py | 110 ++- tests/utils/test_singleton.py | 24 + 31 files changed, 2905 insertions(+), 230 deletions(-) create mode 100644 src/iceberg/avro/__init__.py create mode 100644 src/iceberg/avro/codecs/__init__.py create mode 100644 src/iceberg/avro/codecs/bzip2.py create mode 100644 src/iceberg/avro/codecs/codec.py create mode 100644 src/iceberg/avro/codecs/deflate.py create mode 100644 src/iceberg/avro/codecs/snappy_codec.py create mode 100644 src/iceberg/avro/codecs/zstandard_codec.py create mode 100644 src/iceberg/avro/decoder.py create mode 100644 src/iceberg/avro/file.py create mode 100644 src/iceberg/avro/reader.py create mode 100644 src/iceberg/io/memory.py create mode 100644 tests/avro/__init__.py create mode 100644 tests/avro/test_decoder.py create mode 100644 tests/avro/test_file.py create mode 100644 tests/avro/test_reader.py create mode 100644 tests/utils/test_singleton.py diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 8a8b31418e..cf56bc3162 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -54,4 +54,4 @@ repos: rev: '4.0.1' hooks: - id: flake8 - args: [ "--ignore=E501,W503" ] + args: [ "--ignore=E501,W503,E203" ] diff --git a/LICENSE b/LICENSE index 7868b9b440..5823c32b65 100644 --- a/LICENSE +++ b/LICENSE @@ -202,3 +202,14 @@ limitations under the License. -------------------------------------------------------------------------------- + +This product includes code from Apache Avro. + +* Code for initializing the Avro (de)compression codecs +* The Binary decoder for reading in an Avro byte stream + +Copyright: 2014-2022 The Apache Software Foundation. +Home page: https://avro.apache.org/ +License: https://www.apache.org/licenses/LICENSE-2.0 + +-------------------------------------------------------------------------------- diff --git a/poetry.lock b/poetry.lock index 0feeb1f76f..91dfbc6914 100644 --- a/poetry.lock +++ b/poetry.lock @@ -20,6 +20,17 @@ docs = ["furo", "sphinx", "zope.interface", "sphinx-notfound-page"] tests = ["coverage[toml] (>=5.0.2)", "hypothesis", "pympler", "pytest (>=4.3.0)", "six", "mypy", "pytest-mypy-plugins", "zope.interface", "cloudpickle"] tests_no_zope = ["coverage[toml] (>=5.0.2)", "hypothesis", "pympler", "pytest (>=4.3.0)", "six", "mypy", "pytest-mypy-plugins", "cloudpickle"] +[[package]] +name = "cffi" +version = "1.15.0" +description = "Foreign Function Interface for Python calling C code." +category = "main" +optional = true +python-versions = "*" + +[package.dependencies] +pycparser = "*" + [[package]] name = "cfgv" version = "3.3.1" @@ -38,14 +49,14 @@ python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*" [[package]] name = "coverage" -version = "6.4" +version = "6.4.1" description = "Code coverage measurement for Python" category = "dev" optional = false python-versions = ">=3.7" [package.dependencies] -tomli = {version = "*", optional = true, markers = "python_version < \"3.11\" and extra == \"toml\""} +tomli = {version = "*", optional = true, markers = "python_full_version <= \"3.11.0a6\" and extra == \"toml\""} [package.extras] toml = ["tomli"] @@ -66,6 +77,20 @@ category = "dev" optional = false python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*" +[[package]] +name = "fastavro" +version = "1.5.1" +description = "Fast read/write of AVRO files" +category = "dev" +optional = false +python-versions = ">=3.7" + +[package.extras] +codecs = ["python-snappy", "zstandard", "lz4"] +lz4 = ["lz4"] +snappy = ["python-snappy"] +zstandard = ["zstandard"] + [[package]] name = "filelock" version = "3.7.1" @@ -218,6 +243,14 @@ python-versions = ">=3.7" [package.dependencies] numpy = ">=1.16.6" +[[package]] +name = "pycparser" +version = "2.21" +description = "C parser in Python" +category = "main" +optional = true +python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*" + [[package]] name = "pyparsing" version = "3.0.9" @@ -267,6 +300,14 @@ pep517 = "*" docs = ["sphinx", "jaraco.packaging (>=8.2)", "rst.linker (>=1.9)"] testing = ["pytest (>=4.6)", "pytest-checkdocs (>=2.4)", "pytest-flake8", "pytest-cov", "pytest-enabler (>=1.0.1)", "types-docutils", "pytest-black (>=0.3.7)", "pytest-mypy"] +[[package]] +name = "python-snappy" +version = "0.6.1" +description = "Python library for the snappy compression library from Google" +category = "main" +optional = true +python-versions = "*" + [[package]] name = "pyyaml" version = "6.0" @@ -329,13 +370,29 @@ python-versions = ">=3.7" docs = ["sphinx", "jaraco.packaging (>=9)", "rst.linker (>=1.9)"] testing = ["pytest (>=6)", "pytest-checkdocs (>=2.4)", "pytest-flake8", "pytest-cov", "pytest-enabler (>=1.0.1)", "jaraco.itertools", "func-timeout", "pytest-black (>=0.3.7)", "pytest-mypy (>=0.9.1)"] +[[package]] +name = "zstandard" +version = "0.17.0" +description = "Zstandard bindings for Python" +category = "main" +optional = true +python-versions = ">=3.6" + +[package.dependencies] +cffi = {version = ">=1.11", markers = "platform_python_implementation == \"PyPy\""} + +[package.extras] +cffi = ["cffi (>=1.11)"] + [extras] pyarrow = ["pyarrow"] +python-snappy = ["zstandard"] +snappy = ["python-snappy"] [metadata] lock-version = "1.1" python-versions = "^3.8" -content-hash = "3e07f5fb1c8f204b0a004541776b7d90cad2afac926c2c8554df599b977429e9" +content-hash = "439429e65911b8e768bd4617a4883e8a4ec2652df3a43b28755346da8ed17e19" [metadata.files] atomicwrites = [ @@ -346,6 +403,58 @@ attrs = [ {file = "attrs-21.4.0-py2.py3-none-any.whl", hash = "sha256:2d27e3784d7a565d36ab851fe94887c5eccd6a463168875832a1be79c82828b4"}, {file = "attrs-21.4.0.tar.gz", hash = "sha256:626ba8234211db98e869df76230a137c4c40a12d72445c45d5f5b716f076e2fd"}, ] +cffi = [ + {file = "cffi-1.15.0-cp27-cp27m-macosx_10_9_x86_64.whl", hash = "sha256:c2502a1a03b6312837279c8c1bd3ebedf6c12c4228ddbad40912d671ccc8a962"}, + {file = "cffi-1.15.0-cp27-cp27m-manylinux1_i686.whl", hash = "sha256:23cfe892bd5dd8941608f93348c0737e369e51c100d03718f108bf1add7bd6d0"}, + {file = "cffi-1.15.0-cp27-cp27m-manylinux1_x86_64.whl", hash = "sha256:41d45de54cd277a7878919867c0f08b0cf817605e4eb94093e7516505d3c8d14"}, + {file = "cffi-1.15.0-cp27-cp27m-win32.whl", hash = "sha256:4a306fa632e8f0928956a41fa8e1d6243c71e7eb59ffbd165fc0b41e316b2474"}, + {file = "cffi-1.15.0-cp27-cp27m-win_amd64.whl", hash = "sha256:e7022a66d9b55e93e1a845d8c9eba2a1bebd4966cd8bfc25d9cd07d515b33fa6"}, + {file = "cffi-1.15.0-cp27-cp27mu-manylinux1_i686.whl", hash = "sha256:14cd121ea63ecdae71efa69c15c5543a4b5fbcd0bbe2aad864baca0063cecf27"}, + {file = "cffi-1.15.0-cp27-cp27mu-manylinux1_x86_64.whl", hash = "sha256:d4d692a89c5cf08a8557fdeb329b82e7bf609aadfaed6c0d79f5a449a3c7c023"}, + {file = "cffi-1.15.0-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:0104fb5ae2391d46a4cb082abdd5c69ea4eab79d8d44eaaf79f1b1fd806ee4c2"}, + {file = "cffi-1.15.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:91ec59c33514b7c7559a6acda53bbfe1b283949c34fe7440bcf917f96ac0723e"}, + {file = "cffi-1.15.0-cp310-cp310-manylinux_2_12_i686.manylinux2010_i686.whl", hash = "sha256:f5c7150ad32ba43a07c4479f40241756145a1f03b43480e058cfd862bf5041c7"}, + {file = "cffi-1.15.0-cp310-cp310-manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:00c878c90cb53ccfaae6b8bc18ad05d2036553e6d9d1d9dbcf323bbe83854ca3"}, + {file = "cffi-1.15.0-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:abb9a20a72ac4e0fdb50dae135ba5e77880518e742077ced47eb1499e29a443c"}, + {file = "cffi-1.15.0-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:a5263e363c27b653a90078143adb3d076c1a748ec9ecc78ea2fb916f9b861962"}, + {file = "cffi-1.15.0-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:f54a64f8b0c8ff0b64d18aa76675262e1700f3995182267998c31ae974fbc382"}, + {file = "cffi-1.15.0-cp310-cp310-win32.whl", hash = "sha256:c21c9e3896c23007803a875460fb786118f0cdd4434359577ea25eb556e34c55"}, + {file = "cffi-1.15.0-cp310-cp310-win_amd64.whl", hash = "sha256:5e069f72d497312b24fcc02073d70cb989045d1c91cbd53979366077959933e0"}, + {file = "cffi-1.15.0-cp36-cp36m-macosx_10_9_x86_64.whl", hash = "sha256:64d4ec9f448dfe041705426000cc13e34e6e5bb13736e9fd62e34a0b0c41566e"}, + {file = "cffi-1.15.0-cp36-cp36m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:2756c88cbb94231c7a147402476be2c4df2f6078099a6f4a480d239a8817ae39"}, + {file = "cffi-1.15.0-cp36-cp36m-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:3b96a311ac60a3f6be21d2572e46ce67f09abcf4d09344c49274eb9e0bf345fc"}, + {file = "cffi-1.15.0-cp36-cp36m-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:75e4024375654472cc27e91cbe9eaa08567f7fbdf822638be2814ce059f58032"}, + {file = "cffi-1.15.0-cp36-cp36m-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:59888172256cac5629e60e72e86598027aca6bf01fa2465bdb676d37636573e8"}, + {file = "cffi-1.15.0-cp36-cp36m-manylinux_2_5_x86_64.manylinux1_x86_64.whl", hash = "sha256:27c219baf94952ae9d50ec19651a687b826792055353d07648a5695413e0c605"}, + {file = "cffi-1.15.0-cp36-cp36m-win32.whl", hash = "sha256:4958391dbd6249d7ad855b9ca88fae690783a6be9e86df65865058ed81fc860e"}, + {file = "cffi-1.15.0-cp36-cp36m-win_amd64.whl", hash = "sha256:f6f824dc3bce0edab5f427efcfb1d63ee75b6fcb7282900ccaf925be84efb0fc"}, + {file = "cffi-1.15.0-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:06c48159c1abed75c2e721b1715c379fa3200c7784271b3c46df01383b593636"}, + {file = "cffi-1.15.0-cp37-cp37m-manylinux_2_12_i686.manylinux2010_i686.whl", hash = "sha256:c2051981a968d7de9dd2d7b87bcb9c939c74a34626a6e2f8181455dd49ed69e4"}, + {file = "cffi-1.15.0-cp37-cp37m-manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:fd8a250edc26254fe5b33be00402e6d287f562b6a5b2152dec302fa15bb3e997"}, + {file = "cffi-1.15.0-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:91d77d2a782be4274da750752bb1650a97bfd8f291022b379bb8e01c66b4e96b"}, + {file = "cffi-1.15.0-cp37-cp37m-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:45db3a33139e9c8f7c09234b5784a5e33d31fd6907800b316decad50af323ff2"}, + {file = "cffi-1.15.0-cp37-cp37m-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:263cc3d821c4ab2213cbe8cd8b355a7f72a8324577dc865ef98487c1aeee2bc7"}, + {file = "cffi-1.15.0-cp37-cp37m-win32.whl", hash = "sha256:17771976e82e9f94976180f76468546834d22a7cc404b17c22df2a2c81db0c66"}, + {file = "cffi-1.15.0-cp37-cp37m-win_amd64.whl", hash = "sha256:3415c89f9204ee60cd09b235810be700e993e343a408693e80ce7f6a40108029"}, + {file = "cffi-1.15.0-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:4238e6dab5d6a8ba812de994bbb0a79bddbdf80994e4ce802b6f6f3142fcc880"}, + {file = "cffi-1.15.0-cp38-cp38-manylinux_2_12_i686.manylinux2010_i686.whl", hash = "sha256:0808014eb713677ec1292301ea4c81ad277b6cdf2fdd90fd540af98c0b101d20"}, + {file = "cffi-1.15.0-cp38-cp38-manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:57e9ac9ccc3101fac9d6014fba037473e4358ef4e89f8e181f8951a2c0162024"}, + {file = "cffi-1.15.0-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:8b6c2ea03845c9f501ed1313e78de148cd3f6cad741a75d43a29b43da27f2e1e"}, + {file = "cffi-1.15.0-cp38-cp38-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:10dffb601ccfb65262a27233ac273d552ddc4d8ae1bf93b21c94b8511bffe728"}, + {file = "cffi-1.15.0-cp38-cp38-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:786902fb9ba7433aae840e0ed609f45c7bcd4e225ebb9c753aa39725bb3e6ad6"}, + {file = "cffi-1.15.0-cp38-cp38-win32.whl", hash = "sha256:da5db4e883f1ce37f55c667e5c0de439df76ac4cb55964655906306918e7363c"}, + {file = "cffi-1.15.0-cp38-cp38-win_amd64.whl", hash = "sha256:181dee03b1170ff1969489acf1c26533710231c58f95534e3edac87fff06c443"}, + {file = "cffi-1.15.0-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:45e8636704eacc432a206ac7345a5d3d2c62d95a507ec70d62f23cd91770482a"}, + {file = "cffi-1.15.0-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:31fb708d9d7c3f49a60f04cf5b119aeefe5644daba1cd2a0fe389b674fd1de37"}, + {file = "cffi-1.15.0-cp39-cp39-manylinux_2_12_i686.manylinux2010_i686.whl", hash = "sha256:6dc2737a3674b3e344847c8686cf29e500584ccad76204efea14f451d4cc669a"}, + {file = "cffi-1.15.0-cp39-cp39-manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:74fdfdbfdc48d3f47148976f49fab3251e550a8720bebc99bf1483f5bfb5db3e"}, + {file = "cffi-1.15.0-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:ffaa5c925128e29efbde7301d8ecaf35c8c60ffbcd6a1ffd3a552177c8e5e796"}, + {file = "cffi-1.15.0-cp39-cp39-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:3f7d084648d77af029acb79a0ff49a0ad7e9d09057a9bf46596dac9514dc07df"}, + {file = "cffi-1.15.0-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:ef1f279350da2c586a69d32fc8733092fd32cc8ac95139a00377841f59a3f8d8"}, + {file = "cffi-1.15.0-cp39-cp39-win32.whl", hash = "sha256:2a23af14f408d53d5e6cd4e3d9a24ff9e05906ad574822a10563efcef137979a"}, + {file = "cffi-1.15.0-cp39-cp39-win_amd64.whl", hash = "sha256:3773c4d81e6e818df2efbc7dd77325ca0dcb688116050fb2b3011218eda36139"}, + {file = "cffi-1.15.0.tar.gz", hash = "sha256:920f0d66a896c2d99f0adbb391f990a84091179542c205fa53ce5787aff87954"}, +] cfgv = [ {file = "cfgv-3.3.1-py2.py3-none-any.whl", hash = "sha256:c6a0883f3917a037485059700b9e75da2464e6c27051014ad85ba6aaa5884426"}, {file = "cfgv-3.3.1.tar.gz", hash = "sha256:f5a830efb9ce7a445376bb66ec94c638a9787422f96264c98edc6bdeed8ab736"}, @@ -355,47 +464,47 @@ colorama = [ {file = "colorama-0.4.4.tar.gz", hash = "sha256:5941b2b48a20143d2267e95b1c2a7603ce057ee39fd88e7329b0c292aa16869b"}, ] coverage = [ - {file = "coverage-6.4-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:50ed480b798febce113709846b11f5d5ed1e529c88d8ae92f707806c50297abf"}, - {file = "coverage-6.4-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:26f8f92699756cb7af2b30720de0c5bb8d028e923a95b6d0c891088025a1ac8f"}, - {file = "coverage-6.4-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:60c2147921da7f4d2d04f570e1838db32b95c5509d248f3fe6417e91437eaf41"}, - {file = "coverage-6.4-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:750e13834b597eeb8ae6e72aa58d1d831b96beec5ad1d04479ae3772373a8088"}, - {file = "coverage-6.4-cp310-cp310-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:af5b9ee0fc146e907aa0f5fb858c3b3da9199d78b7bb2c9973d95550bd40f701"}, - {file = "coverage-6.4-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:a022394996419142b33a0cf7274cb444c01d2bb123727c4bb0b9acabcb515dea"}, - {file = "coverage-6.4-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:5a78cf2c43b13aa6b56003707c5203f28585944c277c1f3f109c7b041b16bd39"}, - {file = "coverage-6.4-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:9229d074e097f21dfe0643d9d0140ee7433814b3f0fc3706b4abffd1e3038632"}, - {file = "coverage-6.4-cp310-cp310-win32.whl", hash = "sha256:fb45fe08e1abc64eb836d187b20a59172053999823f7f6ef4f18a819c44ba16f"}, - {file = "coverage-6.4-cp310-cp310-win_amd64.whl", hash = "sha256:3cfd07c5889ddb96a401449109a8b97a165be9d67077df6802f59708bfb07720"}, - {file = "coverage-6.4-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:03014a74023abaf5a591eeeaf1ac66a73d54eba178ff4cb1fa0c0a44aae70383"}, - {file = "coverage-6.4-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:9c82f2cd69c71698152e943f4a5a6b83a3ab1db73b88f6e769fabc86074c3b08"}, - {file = "coverage-6.4-cp37-cp37m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:7b546cf2b1974ddc2cb222a109b37c6ed1778b9be7e6b0c0bc0cf0438d9e45a6"}, - {file = "coverage-6.4-cp37-cp37m-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:cc173f1ce9ffb16b299f51c9ce53f66a62f4d975abe5640e976904066f3c835d"}, - {file = "coverage-6.4-cp37-cp37m-musllinux_1_1_aarch64.whl", hash = "sha256:c53ad261dfc8695062fc8811ac7c162bd6096a05a19f26097f411bdf5747aee7"}, - {file = "coverage-6.4-cp37-cp37m-musllinux_1_1_i686.whl", hash = "sha256:eef5292b60b6de753d6e7f2d128d5841c7915fb1e3321c3a1fe6acfe76c38052"}, - {file = "coverage-6.4-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:543e172ce4c0de533fa892034cce260467b213c0ea8e39da2f65f9a477425211"}, - {file = "coverage-6.4-cp37-cp37m-win32.whl", hash = "sha256:00c8544510f3c98476bbd58201ac2b150ffbcce46a8c3e4fb89ebf01998f806a"}, - {file = "coverage-6.4-cp37-cp37m-win_amd64.whl", hash = "sha256:b84ab65444dcc68d761e95d4d70f3cfd347ceca5a029f2ffec37d4f124f61311"}, - {file = "coverage-6.4-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:d548edacbf16a8276af13063a2b0669d58bbcfca7c55a255f84aac2870786a61"}, - {file = "coverage-6.4-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:033ebec282793bd9eb988d0271c211e58442c31077976c19c442e24d827d356f"}, - {file = "coverage-6.4-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:742fb8b43835078dd7496c3c25a1ec8d15351df49fb0037bffb4754291ef30ce"}, - {file = "coverage-6.4-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:d55fae115ef9f67934e9f1103c9ba826b4c690e4c5bcf94482b8b2398311bf9c"}, - {file = "coverage-6.4-cp38-cp38-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:5cd698341626f3c77784858427bad0cdd54a713115b423d22ac83a28303d1d95"}, - {file = "coverage-6.4-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:62d382f7d77eeeaff14b30516b17bcbe80f645f5cf02bb755baac376591c653c"}, - {file = "coverage-6.4-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:016d7f5cf1c8c84f533a3c1f8f36126fbe00b2ec0ccca47cc5731c3723d327c6"}, - {file = "coverage-6.4-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:69432946f154c6add0e9ede03cc43b96e2ef2733110a77444823c053b1ff5166"}, - {file = "coverage-6.4-cp38-cp38-win32.whl", hash = "sha256:83bd142cdec5e4a5c4ca1d4ff6fa807d28460f9db919f9f6a31babaaa8b88426"}, - {file = "coverage-6.4-cp38-cp38-win_amd64.whl", hash = "sha256:4002f9e8c1f286e986fe96ec58742b93484195defc01d5cc7809b8f7acb5ece3"}, - {file = "coverage-6.4-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:e4f52c272fdc82e7c65ff3f17a7179bc5f710ebc8ce8a5cadac81215e8326740"}, - {file = "coverage-6.4-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:b5578efe4038be02d76c344007b13119b2b20acd009a88dde8adec2de4f630b5"}, - {file = "coverage-6.4-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:d8099ea680201c2221f8468c372198ceba9338a5fec0e940111962b03b3f716a"}, - {file = "coverage-6.4-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:a00441f5ea4504f5abbc047589d09e0dc33eb447dc45a1a527c8b74bfdd32c65"}, - {file = "coverage-6.4-cp39-cp39-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:2e76bd16f0e31bc2b07e0fb1379551fcd40daf8cdf7e24f31a29e442878a827c"}, - {file = "coverage-6.4-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:8d2e80dd3438e93b19e1223a9850fa65425e77f2607a364b6fd134fcd52dc9df"}, - {file = "coverage-6.4-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:341e9c2008c481c5c72d0e0dbf64980a4b2238631a7f9780b0fe2e95755fb018"}, - {file = "coverage-6.4-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:21e6686a95025927775ac501e74f5940cdf6fe052292f3a3f7349b0abae6d00f"}, - {file = "coverage-6.4-cp39-cp39-win32.whl", hash = "sha256:968ed5407f9460bd5a591cefd1388cc00a8f5099de9e76234655ae48cfdbe2c3"}, - {file = "coverage-6.4-cp39-cp39-win_amd64.whl", hash = "sha256:e35217031e4b534b09f9b9a5841b9344a30a6357627761d4218818b865d45055"}, - {file = "coverage-6.4-pp36.pp37.pp38-none-any.whl", hash = "sha256:e637ae0b7b481905358624ef2e81d7fb0b1af55f5ff99f9ba05442a444b11e45"}, - {file = "coverage-6.4.tar.gz", hash = "sha256:727dafd7f67a6e1cad808dc884bd9c5a2f6ef1f8f6d2f22b37b96cb0080d4f49"}, + {file = "coverage-6.4.1-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:f1d5aa2703e1dab4ae6cf416eb0095304f49d004c39e9db1d86f57924f43006b"}, + {file = "coverage-6.4.1-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:4ce1b258493cbf8aec43e9b50d89982346b98e9ffdfaae8ae5793bc112fb0068"}, + {file = "coverage-6.4.1-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:83c4e737f60c6936460c5be330d296dd5b48b3963f48634c53b3f7deb0f34ec4"}, + {file = "coverage-6.4.1-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:84e65ef149028516c6d64461b95a8dbcfce95cfd5b9eb634320596173332ea84"}, + {file = "coverage-6.4.1-cp310-cp310-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:f69718750eaae75efe506406c490d6fc5a6161d047206cc63ce25527e8a3adad"}, + {file = "coverage-6.4.1-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:e57816f8ffe46b1df8f12e1b348f06d164fd5219beba7d9433ba79608ef011cc"}, + {file = "coverage-6.4.1-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:01c5615d13f3dd3aa8543afc069e5319cfa0c7d712f6e04b920431e5c564a749"}, + {file = "coverage-6.4.1-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:75ab269400706fab15981fd4bd5080c56bd5cc07c3bccb86aab5e1d5a88dc8f4"}, + {file = "coverage-6.4.1-cp310-cp310-win32.whl", hash = "sha256:a7f3049243783df2e6cc6deafc49ea123522b59f464831476d3d1448e30d72df"}, + {file = "coverage-6.4.1-cp310-cp310-win_amd64.whl", hash = "sha256:ee2ddcac99b2d2aec413e36d7a429ae9ebcadf912946b13ffa88e7d4c9b712d6"}, + {file = "coverage-6.4.1-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:fb73e0011b8793c053bfa85e53129ba5f0250fdc0392c1591fd35d915ec75c46"}, + {file = "coverage-6.4.1-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:106c16dfe494de3193ec55cac9640dd039b66e196e4641fa8ac396181578b982"}, + {file = "coverage-6.4.1-cp37-cp37m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:87f4f3df85aa39da00fd3ec4b5abeb7407e82b68c7c5ad181308b0e2526da5d4"}, + {file = "coverage-6.4.1-cp37-cp37m-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:961e2fb0680b4f5ad63234e0bf55dfb90d302740ae9c7ed0120677a94a1590cb"}, + {file = "coverage-6.4.1-cp37-cp37m-musllinux_1_1_aarch64.whl", hash = "sha256:cec3a0f75c8f1031825e19cd86ee787e87cf03e4fd2865c79c057092e69e3a3b"}, + {file = "coverage-6.4.1-cp37-cp37m-musllinux_1_1_i686.whl", hash = "sha256:129cd05ba6f0d08a766d942a9ed4b29283aff7b2cccf5b7ce279d50796860bb3"}, + {file = "coverage-6.4.1-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:bf5601c33213d3cb19d17a796f8a14a9eaa5e87629a53979a5981e3e3ae166f6"}, + {file = "coverage-6.4.1-cp37-cp37m-win32.whl", hash = "sha256:269eaa2c20a13a5bf17558d4dc91a8d078c4fa1872f25303dddcbba3a813085e"}, + {file = "coverage-6.4.1-cp37-cp37m-win_amd64.whl", hash = "sha256:f02cbbf8119db68455b9d763f2f8737bb7db7e43720afa07d8eb1604e5c5ae28"}, + {file = "coverage-6.4.1-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:ffa9297c3a453fba4717d06df579af42ab9a28022444cae7fa605af4df612d54"}, + {file = "coverage-6.4.1-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:145f296d00441ca703a659e8f3eb48ae39fb083baba2d7ce4482fb2723e050d9"}, + {file = "coverage-6.4.1-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:d67d44996140af8b84284e5e7d398e589574b376fb4de8ccd28d82ad8e3bea13"}, + {file = "coverage-6.4.1-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:2bd9a6fc18aab8d2e18f89b7ff91c0f34ff4d5e0ba0b33e989b3cd4194c81fd9"}, + {file = "coverage-6.4.1-cp38-cp38-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:3384f2a3652cef289e38100f2d037956194a837221edd520a7ee5b42d00cc605"}, + {file = "coverage-6.4.1-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:9b3e07152b4563722be523e8cd0b209e0d1a373022cfbde395ebb6575bf6790d"}, + {file = "coverage-6.4.1-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:1480ff858b4113db2718848d7b2d1b75bc79895a9c22e76a221b9d8d62496428"}, + {file = "coverage-6.4.1-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:865d69ae811a392f4d06bde506d531f6a28a00af36f5c8649684a9e5e4a85c83"}, + {file = "coverage-6.4.1-cp38-cp38-win32.whl", hash = "sha256:664a47ce62fe4bef9e2d2c430306e1428ecea207ffd68649e3b942fa8ea83b0b"}, + {file = "coverage-6.4.1-cp38-cp38-win_amd64.whl", hash = "sha256:26dff09fb0d82693ba9e6231248641d60ba606150d02ed45110f9ec26404ed1c"}, + {file = "coverage-6.4.1-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:d9c80df769f5ec05ad21ea34be7458d1dc51ff1fb4b2219e77fe24edf462d6df"}, + {file = "coverage-6.4.1-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:39ee53946bf009788108b4dd2894bf1349b4e0ca18c2016ffa7d26ce46b8f10d"}, + {file = "coverage-6.4.1-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:f5b66caa62922531059bc5ac04f836860412f7f88d38a476eda0a6f11d4724f4"}, + {file = "coverage-6.4.1-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:fd180ed867e289964404051a958f7cccabdeed423f91a899829264bb7974d3d3"}, + {file = "coverage-6.4.1-cp39-cp39-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:84631e81dd053e8a0d4967cedab6db94345f1c36107c71698f746cb2636c63e3"}, + {file = "coverage-6.4.1-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:8c08da0bd238f2970230c2a0d28ff0e99961598cb2e810245d7fc5afcf1254e8"}, + {file = "coverage-6.4.1-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:d42c549a8f41dc103a8004b9f0c433e2086add8a719da00e246e17cbe4056f72"}, + {file = "coverage-6.4.1-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:309ce4a522ed5fca432af4ebe0f32b21d6d7ccbb0f5fcc99290e71feba67c264"}, + {file = "coverage-6.4.1-cp39-cp39-win32.whl", hash = "sha256:fdb6f7bd51c2d1714cea40718f6149ad9be6a2ee7d93b19e9f00934c0f2a74d9"}, + {file = "coverage-6.4.1-cp39-cp39-win_amd64.whl", hash = "sha256:342d4aefd1c3e7f620a13f4fe563154d808b69cccef415415aece4c786665397"}, + {file = "coverage-6.4.1-pp36.pp37.pp38-none-any.whl", hash = "sha256:4803e7ccf93230accb928f3a68f00ffa80a88213af98ed338a57ad021ef06815"}, + {file = "coverage-6.4.1.tar.gz", hash = "sha256:4321f075095a096e70aff1d002030ee612b65a205a0a0f5b815280d5dc58100c"}, ] distlib = [ {file = "distlib-0.3.4-py2.py3-none-any.whl", hash = "sha256:6564fe0a8f51e734df6333d08b8b94d4ea8ee6b99b5ed50613f731fd4089f34b"}, @@ -405,6 +514,21 @@ docutils = [ {file = "docutils-0.18.1-py2.py3-none-any.whl", hash = "sha256:23010f129180089fbcd3bc08cfefccb3b890b0050e1ca00c867036e9d161b98c"}, {file = "docutils-0.18.1.tar.gz", hash = "sha256:679987caf361a7539d76e584cbeddc311e3aee937877c87346f31debc63e9d06"}, ] +fastavro = [ + {file = "fastavro-1.5.1-cp310-cp310-macosx_10_14_x86_64.whl", hash = "sha256:920d170560198741fa196a62a97c220173339766e6c14369c5c68bfe8cdafa25"}, + {file = "fastavro-1.5.1-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:0b00b1711511981c4e2dd4a27ba5ae20897fe41ec7ab52eda868626d445081e5"}, + {file = "fastavro-1.5.1-cp310-cp310-win_amd64.whl", hash = "sha256:04438b592980633ccf5d1de7798480a634ca581ae7575ab7671ba16773b6b428"}, + {file = "fastavro-1.5.1-cp37-cp37m-macosx_10_15_x86_64.whl", hash = "sha256:0ab92ab744f9172da0791bfad0495d785c7c4f5a68924e3c6c6b39b78b044b11"}, + {file = "fastavro-1.5.1-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:84ca1a60cecd710ead076585b56b954ab3e6e001d8e7384cb4ed20019b29e7a9"}, + {file = "fastavro-1.5.1-cp37-cp37m-win_amd64.whl", hash = "sha256:b5ff657c0d48553492d8356a30b6112fcc6db69adce6bba31135272bc9d87d82"}, + {file = "fastavro-1.5.1-cp38-cp38-macosx_10_15_x86_64.whl", hash = "sha256:0f1ed38042a2a90a7a5da170006459e73134f4c14f4fda9ebba99017adb1b14c"}, + {file = "fastavro-1.5.1-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:df83ebdd7b67b52a37bc84c6e25f7056f756fb216c5c8e5c95ae1673fcbb6015"}, + {file = "fastavro-1.5.1-cp38-cp38-win_amd64.whl", hash = "sha256:0053347a92add6f448837ff00099b0a7200ec5dd58e173743d856d65d0574ddb"}, + {file = "fastavro-1.5.1-cp39-cp39-macosx_10_15_x86_64.whl", hash = "sha256:6b4f8551ccbe0c9b19867b8c93029e8cfe8fa3757245caae6228f35ef0656371"}, + {file = "fastavro-1.5.1-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:ff241b5ce36df7af7461d001ca98fec6eacd56c4754c8ac7718e2d4b7b690a82"}, + {file = "fastavro-1.5.1-cp39-cp39-win_amd64.whl", hash = "sha256:fb3491c88e7962a6b820548ddd12b9c0f6296ebd2385a3021296f14bfe35189a"}, + {file = "fastavro-1.5.1.tar.gz", hash = "sha256:0815da740ced2261f90b0ddbb5bbe645e9c893c8f00e5dc8d30b8ec20f3c7fa9"}, +] filelock = [ {file = "filelock-3.7.1-py3-none-any.whl", hash = "sha256:37def7b658813cda163b56fc564cdc75e86d338246458c4c28ae84cabefa2404"}, {file = "filelock-3.7.1.tar.gz", hash = "sha256:3a0fd85166ad9dbab54c9aec96737b744106dc5f15c0b09a6744a445299fcf04"}, @@ -533,6 +657,10 @@ pyarrow = [ {file = "pyarrow-8.0.0-cp39-cp39-win_amd64.whl", hash = "sha256:cb06cacc19f3b426681f2f6803cc06ff481e7fe5b3a533b406bc5b2138843d4f"}, {file = "pyarrow-8.0.0.tar.gz", hash = "sha256:4a18a211ed888f1ac0b0ebcb99e2d9a3e913a481120ee9b1fe33d3fedb945d4e"}, ] +pycparser = [ + {file = "pycparser-2.21-py2.py3-none-any.whl", hash = "sha256:8ee45429555515e1f6b185e78100aea234072576aa43ab53aefcae078162fca9"}, + {file = "pycparser-2.21.tar.gz", hash = "sha256:e644fdec12f7872f86c58ff790da456218b10f863970249516d60a5eaca77206"}, +] pyparsing = [ {file = "pyparsing-3.0.9-py3-none-any.whl", hash = "sha256:5026bae9a10eeaefb61dab2f09052b9f4307d44aee4eda64b309723d8d206bbc"}, {file = "pyparsing-3.0.9.tar.gz", hash = "sha256:2b020ecf7d21b687f219b71ecad3631f644a47f01403fa1d1036b0c6416d70fb"}, @@ -545,6 +673,56 @@ pytest-checkdocs = [ {file = "pytest-checkdocs-2.7.1.tar.gz", hash = "sha256:2b33b85eddfe5846a69bea4a759303e2d5a3be11d03bc7149f5ba1ef47e6c1ae"}, {file = "pytest_checkdocs-2.7.1-py3-none-any.whl", hash = "sha256:294898c64c9ce1a178edc6660e48da23c7543bfd5a1cea7f0ca4c167745d8461"}, ] +python-snappy = [ + {file = "python-snappy-0.6.1.tar.gz", hash = "sha256:b6a107ab06206acc5359d4c5632bd9b22d448702a79b3169b0c62e0fb808bb2a"}, + {file = "python_snappy-0.6.1-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:b7f920eaf46ebf41bd26f9df51c160d40f9e00b7b48471c3438cb8d027f7fb9b"}, + {file = "python_snappy-0.6.1-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:4ec533a8c1f8df797bded662ec3e494d225b37855bb63eb0d75464a07947477c"}, + {file = "python_snappy-0.6.1-cp310-cp310-manylinux_2_12_i686.manylinux2010_i686.whl", hash = "sha256:6f8bf4708a11b47517baf962f9a02196478bbb10fdb9582add4aa1459fa82380"}, + {file = "python_snappy-0.6.1-cp310-cp310-manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:8d0c019ee7dcf2c60e240877107cddbd95a5b1081787579bf179938392d66480"}, + {file = "python_snappy-0.6.1-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:cb18d9cd7b3f35a2f5af47bb8ed6a5bdbf4f3ddee37f3daade4ab7864c292f5b"}, + {file = "python_snappy-0.6.1-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:b265cde49774752aec9ca7f5d272e3f98718164afc85521622a8a5394158a2b5"}, + {file = "python_snappy-0.6.1-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:d017775851a778ec9cc32651c4464079d06d927303c2dde9ae9830ccf6fe94e1"}, + {file = "python_snappy-0.6.1-cp310-cp310-win32.whl", hash = "sha256:8277d1f6282463c40761f802b742f833f9f2449fcdbb20a96579aa05c8feb614"}, + {file = "python_snappy-0.6.1-cp310-cp310-win_amd64.whl", hash = "sha256:2aaaf618c68d8c9daebc23a20436bd01b09ee70d7fbf7072b7f38b06d2fab539"}, + {file = "python_snappy-0.6.1-cp36-cp36m-macosx_10_9_x86_64.whl", hash = "sha256:277757d5dad4e239dc1417438a0871b65b1b155beb108888e7438c27ffc6a8cc"}, + {file = "python_snappy-0.6.1-cp36-cp36m-manylinux_2_12_i686.manylinux2010_i686.whl", hash = "sha256:e066a0586833d610c4bbddba0be5ba0e3e4f8e0bc5bb6d82103d8f8fc47bb59a"}, + {file = "python_snappy-0.6.1-cp36-cp36m-manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:0d489b50f49433494160c45048fe806de6b3aeab0586e497ebd22a0bab56e427"}, + {file = "python_snappy-0.6.1-cp36-cp36m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:463fd340a499d47b26ca42d2f36a639188738f6e2098c6dbf80aef0e60f461e1"}, + {file = "python_snappy-0.6.1-cp36-cp36m-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:9837ac1650cc68d22a3cf5f15fb62c6964747d16cecc8b22431f113d6e39555d"}, + {file = "python_snappy-0.6.1-cp36-cp36m-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:5e973e637112391f05581f427659c05b30b6843bc522a65be35ac7b18ce3dedd"}, + {file = "python_snappy-0.6.1-cp36-cp36m-win32.whl", hash = "sha256:c20498bd712b6e31a4402e1d027a1cd64f6a4a0066a3fe3c7344475886d07fdf"}, + {file = "python_snappy-0.6.1-cp36-cp36m-win_amd64.whl", hash = "sha256:59e975be4206cc54d0a112ef72fa3970a57c2b1bcc2c97ed41d6df0ebe518228"}, + {file = "python_snappy-0.6.1-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:2a7e528ab6e09c0d67dcb61a1730a292683e5ff9bb088950638d3170cf2a0a54"}, + {file = "python_snappy-0.6.1-cp37-cp37m-manylinux_2_12_i686.manylinux2010_i686.whl", hash = "sha256:39692bedbe0b717001a99915ac0eb2d9d0bad546440d392a2042b96d813eede1"}, + {file = "python_snappy-0.6.1-cp37-cp37m-manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:6a7620404da966f637b9ce8d4d3d543d363223f7a12452a575189c5355fc2d25"}, + {file = "python_snappy-0.6.1-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:7778c224efc38a40d274da4eb82a04cac27aae20012372a7db3c4bbd8926c4d4"}, + {file = "python_snappy-0.6.1-cp37-cp37m-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:1d029f7051ec1bbeaa3e03030b6d8ed47ceb69cae9016f493c802a08af54e026"}, + {file = "python_snappy-0.6.1-cp37-cp37m-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:a0ad38bc98d0b0497a0b0dbc29409bcabfcecff4511ed7063403c86de16927bc"}, + {file = "python_snappy-0.6.1-cp37-cp37m-win32.whl", hash = "sha256:5a453c45178d7864c1bdd6bfe0ee3ed2883f63b9ba2c9bb967c6b586bf763f96"}, + {file = "python_snappy-0.6.1-cp37-cp37m-win_amd64.whl", hash = "sha256:9f0c0d88b84259f93c3aa46398680646f2c23e43394779758d9f739c34e15295"}, + {file = "python_snappy-0.6.1-cp38-cp38-macosx_10_9_universal2.whl", hash = "sha256:5bb05c28298803a74add08ba496879242ef159c75bc86a5406fac0ffc7dd021b"}, + {file = "python_snappy-0.6.1-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:9eac51307c6a1a38d5f86ebabc26a889fddf20cbba7a116ccb54ba1446601d5b"}, + {file = "python_snappy-0.6.1-cp38-cp38-manylinux_2_12_i686.manylinux2010_i686.whl", hash = "sha256:88b6ea78b83d2796f330b0af1b70cdd3965dbdab02d8ac293260ec2c8fe340ee"}, + {file = "python_snappy-0.6.1-cp38-cp38-manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:8c07220408d3268e8268c9351c5c08041bc6f8c6172e59d398b71020df108541"}, + {file = "python_snappy-0.6.1-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:4038019b1bcaadde726a57430718394076c5a21545ebc5badad2c045a09546cf"}, + {file = "python_snappy-0.6.1-cp38-cp38-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:dc96668d9c7cc656609764275c5f8da58ef56d89bdd6810f6923d36497468ff7"}, + {file = "python_snappy-0.6.1-cp38-cp38-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:cf5bb9254e1c38aacf253d510d3d9be631bba21f3d068b17672b38b5cbf2fff5"}, + {file = "python_snappy-0.6.1-cp38-cp38-win32.whl", hash = "sha256:eaf905a580f2747c4a474040a5063cd5e0cc3d1d2d6edb65f28196186493ad4a"}, + {file = "python_snappy-0.6.1-cp38-cp38-win_amd64.whl", hash = "sha256:546c1a7470ecbf6239101e9aff0f709b68ca0f0268b34d9023019a55baa1f7c6"}, + {file = "python_snappy-0.6.1-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:e3a013895c64352b49d0d8e107a84f99631b16dbab156ded33ebf0becf56c8b2"}, + {file = "python_snappy-0.6.1-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:3fb9a88a4dd6336488f3de67ce75816d0d796dce53c2c6e4d70e0b565633c7fd"}, + {file = "python_snappy-0.6.1-cp39-cp39-manylinux_2_12_i686.manylinux2010_i686.whl", hash = "sha256:735cd4528c55dbe4516d6d2b403331a99fc304f8feded8ae887cf97b67d589bb"}, + {file = "python_snappy-0.6.1-cp39-cp39-manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:90b0186516b7a101c14764b0c25931b741fb0102f21253eff67847b4742dfc72"}, + {file = "python_snappy-0.6.1-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:1a993dc8aadd901915a510fe6af5f20ae4256f527040066c22a154db8946751f"}, + {file = "python_snappy-0.6.1-cp39-cp39-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:530bfb9efebcc1aab8bb4ebcbd92b54477eed11f6cf499355e882970a6d3aa7d"}, + {file = "python_snappy-0.6.1-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:5843feb914796b1f0405ccf31ea0fb51034ceb65a7588edfd5a8250cb369e3b2"}, + {file = "python_snappy-0.6.1-cp39-cp39-win32.whl", hash = "sha256:66c80e9b366012dbee262bb1869e4fc5ba8786cda85928481528bc4a72ec2ee8"}, + {file = "python_snappy-0.6.1-cp39-cp39-win_amd64.whl", hash = "sha256:4d3cafdf454354a621c8ab7408e45aa4e9d5c0b943b61ff4815f71ca6bdf0130"}, + {file = "python_snappy-0.6.1-pp37-pypy37_pp73-manylinux_2_12_i686.manylinux2010_i686.whl", hash = "sha256:586724a0276d7a6083a17259d0b51622e492289a9998848a1b01b6441ca12b2f"}, + {file = "python_snappy-0.6.1-pp37-pypy37_pp73-manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:2be4f4550acd484912441f5f1209ba611ac399aac9355fee73611b9a0d4f949c"}, + {file = "python_snappy-0.6.1-pp37-pypy37_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:0bdb6942180660bda7f7d01f4c0def3cfc72b1c6d99aad964801775a3e379aba"}, + {file = "python_snappy-0.6.1-pp37-pypy37_pp73-win_amd64.whl", hash = "sha256:03bb511380fca2a13325b6f16fe8234c8e12da9660f0258cd45d9a02ffc916af"}, +] pyyaml = [ {file = "PyYAML-6.0-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:d4db7c7aef085872ef65a8fd7d6d09a14ae91f691dec3e87ee5ee0539d516f53"}, {file = "PyYAML-6.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:9df7ed3b3d2e0ecfe09e14741b857df43adb5a3ddadc919a2d94fbdf78fea53c"}, @@ -600,3 +778,49 @@ zipp = [ {file = "zipp-3.8.0-py3-none-any.whl", hash = "sha256:c4f6e5bbf48e74f7a38e7cc5b0480ff42b0ae5178957d564d18932525d5cf099"}, {file = "zipp-3.8.0.tar.gz", hash = "sha256:56bf8aadb83c24db6c4b577e13de374ccfb67da2078beba1d037c17980bf43ad"}, ] +zstandard = [ + {file = "zstandard-0.17.0-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:a1991cdf2e81e643b53fb8d272931d2bdf5f4e70d56a457e1ef95bde147ae627"}, + {file = "zstandard-0.17.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:4768449d8d1b0785309ace288e017cc5fa42e11a52bf08c90d9c3eb3a7a73cc6"}, + {file = "zstandard-0.17.0-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:b1ad6d2952b41d9a0ea702a474cc08c05210c6289e29dd496935c9ca3c7fb45c"}, + {file = "zstandard-0.17.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:90a9ba3a9c16b86afcb785b3c9418af39ccfb238fd5f6e429166e3ca8542b01f"}, + {file = "zstandard-0.17.0-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_12_i686.manylinux2010_i686.whl", hash = "sha256:9cf18c156b3a108197a8bf90b37d03c31c8ef35a7c18807b321d96b74e12c301"}, + {file = "zstandard-0.17.0-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:c81fd9386449df0ebf1ab3e01187bb30d61122c74df53ba4880a2454d866e55d"}, + {file = "zstandard-0.17.0-cp310-cp310-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:787efc741e61e00ffe5e65dac99b0dc5c88b9421012a207a91b869a8b1164921"}, + {file = "zstandard-0.17.0-cp310-cp310-win32.whl", hash = "sha256:49cd09ccbd1e3c0e2690dd62ebf95064d84aa42b9db381867e0b138631f969f2"}, + {file = "zstandard-0.17.0-cp310-cp310-win_amd64.whl", hash = "sha256:d78aac2ffc4e88ab1cbcad844669924c24e24c7c255de9628a18f14d832007c5"}, + {file = "zstandard-0.17.0-cp36-cp36m-macosx_10_9_x86_64.whl", hash = "sha256:c19d1e06569c277dcc872d80cbadf14a29e8199e013ff2a176d169f461439a40"}, + {file = "zstandard-0.17.0-cp36-cp36m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:d916018289d2f9a882e90d2e3bd41652861ce11b5ecd8515fa07ad31d97d56e5"}, + {file = "zstandard-0.17.0-cp36-cp36m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:f0c87f097d6867833a839b086eb8d03676bb87c2efa067a131099f04aa790683"}, + {file = "zstandard-0.17.0-cp36-cp36m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_12_i686.manylinux2010_i686.whl", hash = "sha256:60943f71e3117583655a1eb76188a7cc78a25267ef09cc74be4d25a0b0c8b947"}, + {file = "zstandard-0.17.0-cp36-cp36m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:208fa6bead577b2607205640078ee452e81fe20fe96321623c632bad9ebd7148"}, + {file = "zstandard-0.17.0-cp36-cp36m-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:42f3c02c7021073cafbc6cd152b288c56a25e585518861589bb08b063b6d2ad2"}, + {file = "zstandard-0.17.0-cp36-cp36m-win32.whl", hash = "sha256:2a2ac752162ba5cbc869c60c4a4e54e890b2ee2ffb57d3ff159feab1ae4518db"}, + {file = "zstandard-0.17.0-cp36-cp36m-win_amd64.whl", hash = "sha256:d1405caa964ba11b2396bd9fd19940440217345752e192c936d084ba5fe67dcb"}, + {file = "zstandard-0.17.0-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:ef62eb3bcfd6d786f439828bb544ebd3936432db669403e0b8f48e424f1d55f1"}, + {file = "zstandard-0.17.0-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:477f172807a9fa83467b30d7c58876af1410d20177c554c27525211edf535bae"}, + {file = "zstandard-0.17.0-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:de1aa618306a741e0497878b7f845fd6c397e52dd096fb76ed791e7268887176"}, + {file = "zstandard-0.17.0-cp37-cp37m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_12_i686.manylinux2010_i686.whl", hash = "sha256:a827b9c464ee966524f8e82ec1aabb4a77ff9514cae041667fa81ae2ec8bd3e9"}, + {file = "zstandard-0.17.0-cp37-cp37m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:3cf96ace804945e53bc3e5294097e5fa32a2d43bc52416c632b414b870ee0a21"}, + {file = "zstandard-0.17.0-cp37-cp37m-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:802109f67328c5b822d4fdac28e1cf65a24de2e2e99d76cdbeee9121cedb1b6c"}, + {file = "zstandard-0.17.0-cp37-cp37m-win32.whl", hash = "sha256:a628f20d019feb0f3a171c7a55cc4f75681f3b8c1bd7a5009165a487314887cd"}, + {file = "zstandard-0.17.0-cp37-cp37m-win_amd64.whl", hash = "sha256:7d2e7abac41d2b4b18f03575aca860d2cb647c343e13c23d6c769106a3db2f6f"}, + {file = "zstandard-0.17.0-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:f502fe79757434292174b04db114f9e25c767b2d5ca9e759d118b22a66f445f8"}, + {file = "zstandard-0.17.0-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:e37c4e21f696d6bcdbbc7caf98dffa505d04c0053909b9db0a6e8ca3b935eb07"}, + {file = "zstandard-0.17.0-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:8fd386d0ec1f9343f1776391d9e60d4eedced0a0b0e625bb89b91f6d05f70e83"}, + {file = "zstandard-0.17.0-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:91a228a077fc7cd8486c273788d4a006a37d060cb4293f471eb0325c3113af68"}, + {file = "zstandard-0.17.0-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_12_i686.manylinux2010_i686.whl", hash = "sha256:59eadb9f347d40e8f7ef77caffd0c04a31e82c1df82fe2d2a688032429d750ac"}, + {file = "zstandard-0.17.0-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:a71809ec062c5b7acf286ba6d4484e6fe8130fc2b93c25e596bb34e7810c79b2"}, + {file = "zstandard-0.17.0-cp38-cp38-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:8aedd38d357f6d5e2facd88ce62b4976afdc29db57216a23f14a0cd0ca05a8a3"}, + {file = "zstandard-0.17.0-cp38-cp38-win32.whl", hash = "sha256:bd842ae3dbb7cba88beb022161c819fa80ca7d0c5a4ddd209e7daae85d904e49"}, + {file = "zstandard-0.17.0-cp38-cp38-win_amd64.whl", hash = "sha256:d0e9fec68e304fb35c559c44530213adbc7d5918bdab906a45a0f40cd56c4de2"}, + {file = "zstandard-0.17.0-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:9ec62a4c2dbb0a86ee5138c16ef133e59a23ac108f8d7ac97aeb61d410ce6857"}, + {file = "zstandard-0.17.0-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:d5373a56b90052f171c8634fedc53a6ac371e6c742606e9825772a394bdbd4b0"}, + {file = "zstandard-0.17.0-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:f2e3ea5e4d5ecf3faefd4a5294acb6af1f0578b0cdd75d6b4529c45deaa54d6f"}, + {file = "zstandard-0.17.0-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:7a3a1aa9528087f6f4c47f4ece2d5e6a160527821263fb8174ff36429233e093"}, + {file = "zstandard-0.17.0-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_12_i686.manylinux2010_i686.whl", hash = "sha256:bdf691a205bc492956e6daef7a06fb38f8cbe8b2c1cb0386f35f4412c360c9e9"}, + {file = "zstandard-0.17.0-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:db993a56e21d903893933887984ca9b0d274f2b1db7b3cf21ba129783953864f"}, + {file = "zstandard-0.17.0-cp39-cp39-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:a7756a9446f83c81101f6c0a48c3bfd8d387a249933c57b0d095ca8b20541337"}, + {file = "zstandard-0.17.0-cp39-cp39-win32.whl", hash = "sha256:37e50501baaa935f13a1820ab2114f74313b5cb4cfff8146acb8c5b18cdced2a"}, + {file = "zstandard-0.17.0-cp39-cp39-win_amd64.whl", hash = "sha256:b4e671c4c0804cdf752be26f260058bb858fbdaaef1340af170635913ecca01e"}, + {file = "zstandard-0.17.0.tar.gz", hash = "sha256:fa9194cb91441df7242aa3ddc4cb184be38876cb10dd973674887f334bafbfb6"}, +] diff --git a/pyproject.toml b/pyproject.toml index b845f8b017..afeee351ce 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -43,10 +43,15 @@ mmh3 = "^3.0.0" pyarrow = { version = "^8.0.0", optional = true } +zstandard = { version = "^0.17.0", optional = true } + +python-snappy = { version = "^0.6.1", optional = true } + [tool.poetry.dev-dependencies] pytest = "^7.0.0" pytest-checkdocs = "^2.0.0" pre-commit = "^2.0.0" +fastavro = "^1.5.1" coverage = { version = "^6.0.0", extras = ["toml"] } [build-system] @@ -55,6 +60,8 @@ build-backend = "poetry.core.masonry.api" [tool.poetry.extras] pyarrow = ["pyarrow"] +snappy = ["python-snappy"] +python-snappy = ["zstandard"] [tool.black] line-length = 130 @@ -79,5 +86,13 @@ warn_unreachable = true module = "mypy-pyarrow.*" ignore_missing_imports = true +[[tool.mypy.overrides]] +module = "mypy-snappy.*" +ignore_missing_imports = true + +[[tool.mypy.overrides]] +module = "mypy-zstandard.*" +ignore_missing_imports = true + [tool.coverage.run] source = ['src/'] diff --git a/spellcheck-dictionary.txt b/spellcheck-dictionary.txt index 41da147758..8bb0b868c3 100644 --- a/spellcheck-dictionary.txt +++ b/spellcheck-dictionary.txt @@ -57,3 +57,9 @@ UnboundPredicate BoundPredicate BooleanExpression BooleanExpressionVisitor +zigzag +unix +zlib +Codecs +codecs +uri diff --git a/src/iceberg/avro/__init__.py b/src/iceberg/avro/__init__.py new file mode 100644 index 0000000000..13a83393a9 --- /dev/null +++ b/src/iceberg/avro/__init__.py @@ -0,0 +1,16 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. diff --git a/src/iceberg/avro/codecs/__init__.py b/src/iceberg/avro/codecs/__init__.py new file mode 100644 index 0000000000..28dd23f83f --- /dev/null +++ b/src/iceberg/avro/codecs/__init__.py @@ -0,0 +1,40 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +""" +Contains Codecs for Python Avro. + +Note that the word "codecs" means "compression/decompression algorithms" in the +Avro world (https://avro.apache.org/docs/current/spec.html#Object+Container+Files), +so don't confuse it with the Python's "codecs", which is a package mainly for +converting character sets (https://docs.python.org/3/library/codecs.html). +""" +from __future__ import annotations + +from iceberg.avro.codecs.bzip2 import BZip2Codec +from iceberg.avro.codecs.codec import Codec +from iceberg.avro.codecs.deflate import DeflateCodec +from iceberg.avro.codecs.snappy_codec import SnappyCodec +from iceberg.avro.codecs.zstandard_codec import ZStandardCodec + +KNOWN_CODECS: dict[str, type[Codec] | None] = { + "null": None, + "bzip2": BZip2Codec, + "snappy": SnappyCodec, + "zstandard": ZStandardCodec, + "deflate": DeflateCodec, +} diff --git a/src/iceberg/avro/codecs/bzip2.py b/src/iceberg/avro/codecs/bzip2.py new file mode 100644 index 0000000000..b92c248de2 --- /dev/null +++ b/src/iceberg/avro/codecs/bzip2.py @@ -0,0 +1,43 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +from __future__ import annotations + +from iceberg.avro.codecs.codec import Codec + +try: + import bz2 + + class BZip2Codec(Codec): + @staticmethod + def compress(data: bytes) -> tuple[bytes, int]: + compressed_data = bz2.compress(data) + return compressed_data, len(compressed_data) + + @staticmethod + def decompress(data: bytes) -> bytes: + return bz2.decompress(data) + +except ImportError: + + class BZip2Codec(Codec): # type: ignore + @staticmethod + def compress(data: bytes) -> tuple[bytes, int]: + raise ImportError("Python bzip2 support not installed, please install the extension") + + @staticmethod + def decompress(data: bytes) -> bytes: + raise ImportError("Python bzip2 support not installed, please install the extension") diff --git a/src/iceberg/avro/codecs/codec.py b/src/iceberg/avro/codecs/codec.py new file mode 100644 index 0000000000..1c04f0db3e --- /dev/null +++ b/src/iceberg/avro/codecs/codec.py @@ -0,0 +1,33 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +from __future__ import annotations + +from abc import ABC, abstractmethod + + +class Codec(ABC): + """Abstract base class for all Avro codec classes.""" + + @staticmethod + @abstractmethod + def compress(data: bytes) -> tuple[bytes, int]: + ... + + @staticmethod + @abstractmethod + def decompress(data: bytes) -> bytes: + ... diff --git a/src/iceberg/avro/codecs/deflate.py b/src/iceberg/avro/codecs/deflate.py new file mode 100644 index 0000000000..c1f8bf30b7 --- /dev/null +++ b/src/iceberg/avro/codecs/deflate.py @@ -0,0 +1,36 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +from __future__ import annotations + +import zlib + +from iceberg.avro.codecs.codec import Codec + + +class DeflateCodec(Codec): + @staticmethod + def compress(data: bytes) -> tuple[bytes, int]: + # The first two characters and last character are zlib + # wrappers around deflate data. + compressed_data = zlib.compress(data)[2:-1] + return compressed_data, len(compressed_data) + + @staticmethod + def decompress(data: bytes) -> bytes: + # -15 is the log of the window size; negative indicates + # "raw" (no zlib headers) decompression. See zlib.h. + return zlib.decompress(data, -15) diff --git a/src/iceberg/avro/codecs/snappy_codec.py b/src/iceberg/avro/codecs/snappy_codec.py new file mode 100644 index 0000000000..92b599cdfa --- /dev/null +++ b/src/iceberg/avro/codecs/snappy_codec.py @@ -0,0 +1,69 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +from __future__ import annotations + +import binascii +import struct + +from iceberg.avro.codecs.codec import Codec + +STRUCT_CRC32 = struct.Struct(">I") # big-endian unsigned int + +try: + import snappy + + class SnappyCodec(Codec): + @staticmethod + def _check_crc32(bytes_: bytes, checksum: bytes) -> None: + """Incrementally compute CRC-32 from bytes and compare to a checksum + + Args: + bytes_ (bytes): The bytes to check against `checksum` + checksum (bytes): Byte representation of a checksum + + Raises: + ValueError: If the computed CRC-32 does not match the checksum + """ + if binascii.crc32(bytes_) & 0xFFFFFFFF != STRUCT_CRC32.unpack(checksum)[0]: + raise ValueError("Checksum failure") + + @staticmethod + def compress(data: bytes) -> tuple[bytes, int]: + compressed_data = snappy.compress(data) + # A 4-byte, big-endian CRC32 checksum + compressed_data += STRUCT_CRC32.pack(binascii.crc32(data) & 0xFFFFFFFF) + return compressed_data, len(compressed_data) + + @staticmethod + def decompress(data: bytes) -> bytes: + # Compressed data includes a 4-byte CRC32 checksum + data = data[0:-4] + uncompressed = snappy.decompress(data) + checksum = data[-4:] + SnappyCodec._check_crc32(uncompressed, checksum) + return uncompressed + +except ImportError: + + class SnappyCodec(Codec): # type: ignore + @staticmethod + def compress(data: bytes) -> tuple[bytes, int]: + raise ImportError("Snappy support not installed, please install using `pip install pyiceberg[snappy]`") + + @staticmethod + def decompress(data: bytes) -> bytes: + raise ImportError("Snappy support not installed, please install using `pip install pyiceberg[snappy]`") diff --git a/src/iceberg/avro/codecs/zstandard_codec.py b/src/iceberg/avro/codecs/zstandard_codec.py new file mode 100644 index 0000000000..8144628b06 --- /dev/null +++ b/src/iceberg/avro/codecs/zstandard_codec.py @@ -0,0 +1,53 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +from __future__ import annotations + +from io import BytesIO + +from iceberg.avro.codecs.codec import Codec + +try: + from zstandard import ZstdCompressor, ZstdDecompressor + + class ZStandardCodec(Codec): + @staticmethod + def compress(data: bytes) -> tuple[bytes, int]: + compressed_data = ZstdCompressor().compress(data) + return compressed_data, len(compressed_data) + + @staticmethod + def decompress(data: bytes) -> bytes: + uncompressed = bytearray() + dctx = ZstdDecompressor() + with dctx.stream_reader(BytesIO(data)) as reader: + while True: + chunk = reader.read(16384) + if not chunk: + break + uncompressed.extend(chunk) + return uncompressed + +except ImportError: + + class ZStandardCodec(Codec): # type: ignore + @staticmethod + def compress(data: bytes) -> tuple[bytes, int]: + raise ImportError("Zstandard support not installed, please install using `pip install pyiceberg[zstandard]`") + + @staticmethod + def decompress(data: bytes) -> bytes: + raise ImportError("Zstandard support not installed, please install using `pip install pyiceberg[zstandard]`") diff --git a/src/iceberg/avro/decoder.py b/src/iceberg/avro/decoder.py new file mode 100644 index 0000000000..24312cdd25 --- /dev/null +++ b/src/iceberg/avro/decoder.py @@ -0,0 +1,165 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +import decimal +import struct +from datetime import date, datetime, time + +from iceberg.io.base import InputStream +from iceberg.utils.datetime import ( + days_to_date, + micros_to_time, + micros_to_timestamp, + micros_to_timestamptz, +) +from iceberg.utils.decimal import unscaled_to_decimal + +STRUCT_FLOAT = struct.Struct("h") # big-endian signed short +STRUCT_SIGNED_INT = struct.Struct(">i") # big-endian signed int +STRUCT_SIGNED_LONG = struct.Struct(">q") # big-endian signed long + + +class BinaryDecoder: + """Read leaf values.""" + + _input_stream: InputStream + + def __init__(self, input_stream: InputStream) -> None: + """ + reader is a Python object on which we can call read, seek, and tell. + """ + self._input_stream = input_stream + + def read(self, n: int) -> bytes: + """ + Read n bytes. + """ + if n < 0: + raise ValueError(f"Requested {n} bytes to read, expected positive integer.") + read_bytes = self._input_stream.read(n) + if len(read_bytes) != n: + raise ValueError(f"Read {len(read_bytes)} bytes, expected {n} bytes") + return read_bytes + + def read_boolean(self) -> bool: + """ + a boolean is written as a single byte + whose value is either 0 (false) or 1 (true). + """ + return ord(self.read(1)) == 1 + + def read_int(self) -> int: + """int values are written using variable-length, zigzag coding.""" + return self.read_long() + + def read_long(self) -> int: + """long values are written using variable-length, zigzag coding.""" + b = ord(self.read(1)) + n = b & 0x7F + shift = 7 + while (b & 0x80) != 0: + b = ord(self.read(1)) + n |= (b & 0x7F) << shift + shift += 7 + datum = (n >> 1) ^ -(n & 1) + return datum + + def read_float(self) -> float: + """ + A float is written as 4 bytes. + The float is converted into a 32-bit integer using a method equivalent to + Java's floatToIntBits and then encoded in little-endian format. + """ + return float(STRUCT_FLOAT.unpack(self.read(4))[0]) + + def read_double(self) -> float: + """ + A double is written as 8 bytes. + The double is converted into a 64-bit integer using a method equivalent to + Java's doubleToLongBits and then encoded in little-endian format. + """ + return float(STRUCT_DOUBLE.unpack(self.read(8))[0]) + + def read_decimal_from_bytes(self, precision: int, scale: int) -> decimal.Decimal: + """ + Decimal bytes are decoded as signed short, int or long depending on the + size of bytes. + """ + size = self.read_long() + return self.read_decimal_from_fixed(precision, scale, size) + + def read_decimal_from_fixed(self, precision: int, scale: int, size: int) -> decimal.Decimal: + """ + Decimal is encoded as fixed. Fixed instances are encoded using the + number of bytes declared in the schema. + """ + data = self.read(size) + unscaled_datum = int.from_bytes(data, byteorder="big", signed=True) + return unscaled_to_decimal(unscaled_datum, scale) + + def read_bytes(self) -> bytes: + """ + Bytes are encoded as a long followed by that many bytes of data. + """ + return self.read(self.read_long()) + + def read_utf8(self) -> str: + """ + A string is encoded as a long followed by + that many bytes of UTF-8 encoded character data. + """ + return self.read_bytes().decode("utf-8") + + def read_date_from_int(self) -> date: + """ + int is decoded as python date object. + int stores the number of days from + the unix epoch, 1 January 1970 (ISO calendar). + """ + return days_to_date(self.read_int()) + + def read_time_millis(self) -> time: + """ + int is decoded as python time object which represents + the number of milliseconds after midnight, 00:00:00.000. + """ + millis = self.read_int() + return micros_to_time(millis * 1000) + + def read_time_micros(self) -> time: + """ + long is decoded as python time object which represents + the number of microseconds after midnight, 00:00:00.000000. + """ + return micros_to_time(self.read_long()) + + def read_timestamp_micros(self) -> datetime: + """ + long is decoded as python datetime object which represents + the number of microseconds from the unix epoch, 1 January 1970. + """ + return micros_to_timestamp(self.read_long()) + + def read_timestamptz_micros(self): + """ + long is decoded as python datetime object which represents + the number of microseconds from the unix epoch, 1 January 1970. + + Adjusted to UTC + """ + return micros_to_timestamptz(self.read_long()) diff --git a/src/iceberg/avro/file.py b/src/iceberg/avro/file.py new file mode 100644 index 0000000000..0eec227e95 --- /dev/null +++ b/src/iceberg/avro/file.py @@ -0,0 +1,181 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# pylint: disable=W0621 +""" +Avro reader for reading Avro files +""" +from __future__ import annotations + +import json +from dataclasses import dataclass +from io import SEEK_SET + +from iceberg.avro.codecs import KNOWN_CODECS, Codec +from iceberg.avro.decoder import BinaryDecoder +from iceberg.avro.reader import AvroStruct, ConstructReader, StructReader +from iceberg.io.base import InputFile, InputStream +from iceberg.io.memory import MemoryInputStream +from iceberg.schema import Schema, visit +from iceberg.types import ( + FixedType, + MapType, + NestedField, + StringType, + StructType, +) +from iceberg.utils.schema_conversion import AvroSchemaConversion + +VERSION = 1 +MAGIC = bytes(b"Obj" + bytearray([VERSION])) +MAGIC_SIZE = len(MAGIC) +SYNC_SIZE = 16 +META_SCHEMA = StructType( + NestedField(name="magic", field_id=100, field_type=FixedType(length=MAGIC_SIZE), required=True), + NestedField( + field_id=200, + name="meta", + field_type=MapType(key_id=201, key_type=StringType(), value_id=202, value_type=StringType(), value_required=True), + required=True, + ), + NestedField(field_id=300, name="sync", field_type=FixedType(length=SYNC_SIZE), required=True), +) + +_CODEC_KEY = "avro.codec" +_SCHEMA_KEY = "avro.schema" + + +@dataclass(frozen=True) +class AvroFileHeader: + magic: bytes + meta: dict[str, str] + sync: bytes + + def compression_codec(self) -> type[Codec] | None: + """Get the file's compression codec algorithm from the file's metadata. + + In the case of a null codec, we return a None indicating that we + don't need to compress/decompress + """ + codec_name = self.meta.get(_CODEC_KEY, "null") + if codec_name not in KNOWN_CODECS: + raise ValueError(f"Unsupported codec: {codec_name}") + + return KNOWN_CODECS[codec_name] + + def get_schema(self) -> Schema: + if _SCHEMA_KEY in self.meta: + avro_schema_string = self.meta[_SCHEMA_KEY] + avro_schema = json.loads(avro_schema_string) + return AvroSchemaConversion().avro_to_iceberg(avro_schema) + else: + raise ValueError("No schema found in Avro file headers") + + +@dataclass +class Block: + reader: StructReader + block_records: int + block_decoder: BinaryDecoder + position: int = 0 + + def __iter__(self): + return self + + def has_next(self) -> bool: + return self.position < self.block_records + + def __next__(self) -> AvroStruct: + if self.has_next(): + self.position += 1 + return self.reader.read(self.block_decoder) + raise StopIteration + + +class AvroFile: + input_file: InputFile + input_stream: InputStream + header: AvroFileHeader + schema: Schema + file_length: int + reader: StructReader + + decoder: BinaryDecoder + block: Block | None = None + + def __init__(self, input_file: InputFile) -> None: + self.input_file = input_file + + def __enter__(self): + """ + Opens the file and reads the header and generates + a reader tree to start reading the payload + + Returns: + A generator returning the AvroStructs + """ + self.input_stream = self.input_file.open() + self.decoder = BinaryDecoder(self.input_stream) + self.header = self._read_header() + self.schema = self.header.get_schema() + self.file_length = len(self.input_file) + self.reader = visit(self.schema, ConstructReader()) + return self + + def __exit__(self, exc_type, exc_val, exc_tb): + self.input_stream.close() + + def __iter__(self) -> AvroFile: + return self + + def _read_block(self) -> int: + # If there is already a block, we'll have the sync bytes + if self.block: + sync_marker = self.decoder.read(SYNC_SIZE) + if sync_marker != self.header.sync: + raise ValueError(f"Expected sync bytes {self.header.sync!r}, but got {sync_marker!r}") + if self.is_EOF(): + raise StopIteration + block_records = self.decoder.read_long() + + block_bytes_len = self.decoder.read_long() + block_bytes = self.decoder.read(block_bytes_len) + if codec := self.header.compression_codec(): + block_bytes = codec.decompress(block_bytes) + + self.block = Block( + reader=self.reader, block_records=block_records, block_decoder=BinaryDecoder(MemoryInputStream(block_bytes)) + ) + return block_records + + def __next__(self) -> AvroStruct: + if self.block and self.block.has_next(): + return next(self.block) + + new_block = self._read_block() + + if new_block > 0: + return self.__next__() + raise StopIteration + + def _read_header(self) -> AvroFileHeader: + self.input_stream.seek(0, SEEK_SET) + reader = visit(META_SCHEMA, ConstructReader()) + _header = reader.read(self.decoder) + return AvroFileHeader(magic=_header.get(0), meta=_header.get(1), sync=_header.get(2)) + + def is_EOF(self) -> bool: + return self.input_stream.tell() == self.file_length diff --git a/src/iceberg/avro/reader.py b/src/iceberg/avro/reader.py new file mode 100644 index 0000000000..bd8f1b4e80 --- /dev/null +++ b/src/iceberg/avro/reader.py @@ -0,0 +1,316 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +""" +Classes for building the Reader tree + +Constructing a reader tree from the schema makes it easy +to decouple the reader implementation from the schema. + +The reader tree can be changed in such a way that the +read schema is different, while respecting the read schema +""" +from __future__ import annotations + +from abc import abstractmethod +from dataclasses import dataclass, field +from datetime import date, datetime, time +from decimal import Decimal +from functools import singledispatch +from typing import Any +from uuid import UUID + +from iceberg.avro.decoder import BinaryDecoder +from iceberg.files import StructProtocol +from iceberg.schema import Schema, SchemaVisitor +from iceberg.types import ( + BinaryType, + BooleanType, + DateType, + DecimalType, + DoubleType, + FixedType, + FloatType, + IntegerType, + ListType, + LongType, + MapType, + NestedField, + PrimitiveType, + StringType, + StructType, + TimestampType, + TimestamptzType, + TimeType, +) +from iceberg.utils.singleton import Singleton + + +@dataclass(frozen=True) +class AvroStruct(StructProtocol): + _data: list[Any | StructProtocol] = field() + + def set(self, pos: int, value: Any) -> None: + self._data[pos] = value + + def get(self, pos: int) -> Any: + return self._data[pos] + + +class Reader(Singleton): + @abstractmethod + def read(self, decoder: BinaryDecoder) -> Any: + ... + + +class NoneReader(Reader): + def read(self, _: BinaryDecoder) -> None: + return None + + +class BooleanReader(Reader): + def read(self, decoder: BinaryDecoder) -> bool: + return decoder.read_boolean() + + +class IntegerReader(Reader): + def read(self, decoder: BinaryDecoder) -> int: + return decoder.read_int() + + +class LongReader(Reader): + def read(self, decoder: BinaryDecoder) -> int: + return decoder.read_long() + + +class FloatReader(Reader): + def read(self, decoder: BinaryDecoder) -> float: + return decoder.read_float() + + +class DoubleReader(Reader): + def read(self, decoder: BinaryDecoder) -> float: + return decoder.read_double() + + +class DateReader(Reader): + def read(self, decoder: BinaryDecoder) -> date: + return decoder.read_date_from_int() + + +class TimeReader(Reader): + def read(self, decoder: BinaryDecoder) -> time: + return decoder.read_time_micros() + + +class TimestampReader(Reader): + def read(self, decoder: BinaryDecoder) -> datetime: + return decoder.read_timestamp_micros() + + +class TimestamptzReader(Reader): + def read(self, decoder: BinaryDecoder) -> datetime: + return decoder.read_timestamptz_micros() + + +class StringReader(Reader): + def read(self, decoder: BinaryDecoder) -> str: + return decoder.read_utf8() + + +class UUIDReader(Reader): + def read(self, decoder: BinaryDecoder) -> UUID: + return UUID(decoder.read_utf8()) + + +@dataclass(frozen=True) +class FixedReader(Reader): + length: int = field() + + def read(self, decoder: BinaryDecoder) -> bytes: + return decoder.read(self.length) + + +class BinaryReader(Reader): + def read(self, decoder: BinaryDecoder) -> bytes: + return decoder.read_bytes() + + +@dataclass(frozen=True) +class DecimalReader(Reader): + precision: int = field() + scale: int = field() + + def read(self, decoder: BinaryDecoder) -> Decimal: + return decoder.read_decimal_from_bytes(self.precision, self.scale) + + +@dataclass(frozen=True) +class OptionReader(Reader): + option: Reader = field() + + def read(self, decoder: BinaryDecoder) -> Any | None: + # For the Iceberg spec it is required to set the default value to null + # From https://iceberg.apache.org/spec/#avro + # Optional fields must always set the Avro field default value to null. + # + # This means that null has to come first: + # https://avro.apache.org/docs/current/spec.html + # type of the default value must match the first element of the union. + # This is enforced in the schema conversion, which happens prior + # to building the reader tree + if decoder.read_int() > 0: + return self.option.read(decoder) + return None + + +@dataclass(frozen=True) +class StructReader(Reader): + fields: tuple[Reader, ...] = field() + + def read(self, decoder: BinaryDecoder) -> AvroStruct: + return AvroStruct([field.read(decoder) for field in self.fields]) + + +@dataclass(frozen=True) +class ListReader(Reader): + element: Reader + + def read(self, decoder: BinaryDecoder) -> list: + read_items = [] + block_count = decoder.read_long() + while block_count != 0: + if block_count < 0: + block_count = -block_count + # We ignore the block size for now + _ = decoder.read_long() + for _ in range(block_count): + read_items.append(self.element.read(decoder)) + block_count = decoder.read_long() + return read_items + + +@dataclass(frozen=True) +class MapReader(Reader): + key: Reader + value: Reader + + def read(self, decoder: BinaryDecoder) -> dict: + read_items = {} + block_count = decoder.read_long() + while block_count != 0: + if block_count < 0: + block_count = -block_count + # We ignore the block size for now + _ = decoder.read_long() + for _ in range(block_count): + key = self.key.read(decoder) + read_items[key] = self.value.read(decoder) + block_count = decoder.read_long() + + return read_items + + +class ConstructReader(SchemaVisitor[Reader]): + def schema(self, schema: Schema, struct_result: Reader) -> Reader: + return struct_result + + def struct(self, struct: StructType, field_results: list[Reader]) -> Reader: + return StructReader(tuple(field_results)) + + def field(self, field: NestedField, field_result: Reader) -> Reader: + return field_result if field.required else OptionReader(field_result) + + def list(self, list_type: ListType, element_result: Reader) -> Reader: + element_reader = element_result if list_type.element_required else OptionReader(element_result) + return ListReader(element_reader) + + def map(self, map_type: MapType, key_result: Reader, value_result: Reader) -> Reader: + value_reader = value_result if map_type.value_required else OptionReader(value_result) + return MapReader(key_result, value_reader) + + def primitive(self, primitive: PrimitiveType) -> Reader: + return primitive_reader(primitive) + + +@singledispatch +def primitive_reader(primitive: PrimitiveType) -> Reader: + raise ValueError(f"Unknown type: {primitive}") + + +@primitive_reader.register(FixedType) +def _(primitive: FixedType) -> Reader: + return FixedReader(primitive.length) + + +@primitive_reader.register(DecimalType) +def _(primitive: DecimalType) -> Reader: + return DecimalReader(primitive.precision, primitive.scale) + + +@primitive_reader.register(BooleanType) +def _(_: BooleanType) -> Reader: + return BooleanReader() + + +@primitive_reader.register(IntegerType) +def _(_: IntegerType) -> Reader: + return IntegerReader() + + +@primitive_reader.register(LongType) +def _(_: LongType) -> Reader: + return LongReader() + + +@primitive_reader.register(FloatType) +def _(_: FloatType) -> Reader: + return FloatReader() + + +@primitive_reader.register(DoubleType) +def _(_: DoubleType) -> Reader: + return DoubleReader() + + +@primitive_reader.register(DateType) +def _(_: DateType) -> Reader: + return DateReader() + + +@primitive_reader.register(TimeType) +def _(_: TimeType) -> Reader: + return TimeReader() + + +@primitive_reader.register(TimestampType) +def _(_: TimestampType) -> Reader: + return TimestampReader() + + +@primitive_reader.register(TimestamptzType) +def _(_: TimestamptzType) -> Reader: + return TimestamptzReader() + + +@primitive_reader.register(StringType) +def _(_: StringType) -> Reader: + return StringReader() + + +@primitive_reader.register(BinaryType) +def _(_: StringType) -> Reader: + return BinaryReader() diff --git a/src/iceberg/files.py b/src/iceberg/files.py index 586aab498c..e560092357 100644 --- a/src/iceberg/files.py +++ b/src/iceberg/files.py @@ -45,5 +45,5 @@ def get(self, pos: int) -> Any: ... @abstractmethod - def set(self, pos: int, value) -> None: + def set(self, pos: int, value: Any) -> None: ... diff --git a/src/iceberg/io/base.py b/src/iceberg/io/base.py index 4e4ff30cdb..458a3d591e 100644 --- a/src/iceberg/io/base.py +++ b/src/iceberg/io/base.py @@ -24,29 +24,35 @@ """ from abc import ABC, abstractmethod +from io import SEEK_SET from typing import Protocol, Union, runtime_checkable @runtime_checkable -class InputStream(Protocol): # pragma: no cover +class InputStream(Protocol): """A protocol for the file-like object returned by InputFile.open(...) This outlines the minimally required methods for a seekable input stream returned from an InputFile implementation's `open(...)` method. These methods are a subset of IOBase/RawIOBase. """ - def read(self, size: int) -> bytes: + @abstractmethod + def read(self, size: int = 0) -> bytes: ... - def seek(self, offset: int, whence: int) -> None: + @abstractmethod + def seek(self, offset: int, whence: int = SEEK_SET) -> None: ... + @abstractmethod def tell(self) -> int: ... + @abstractmethod def closed(self) -> bool: ... + @abstractmethod def close(self) -> None: ... @@ -59,12 +65,15 @@ class OutputStream(Protocol): # pragma: no cover implementation's `create(...)` method. These methods are a subset of IOBase/RawIOBase. """ + @abstractmethod def write(self, b: bytes) -> None: ... + @abstractmethod def closed(self) -> bool: ... + @abstractmethod def close(self) -> None: ... diff --git a/src/iceberg/io/memory.py b/src/iceberg/io/memory.py new file mode 100644 index 0000000000..0e9dcb9c99 --- /dev/null +++ b/src/iceberg/io/memory.py @@ -0,0 +1,75 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +from io import SEEK_CUR, SEEK_END, SEEK_SET + +from iceberg.io.base import InputStream + + +class MemoryInputStream(InputStream): + """ + Simple in memory stream that we use to store decompressed blocks + + Examples: + >>> stream = MemoryInputStream(b'22memory1925') + >>> stream.tell() + 0 + >>> stream.read(2) + b'22' + >>> stream.tell() + 2 + >>> stream.seek(8) + >>> stream.read(4) + b'1925' + >>> stream.close() + >>> stream.closed() + True + """ + + buffer: bytes + len: int + pos: int + + def __init__(self, buffer: bytes): + self.buffer = buffer + self.len = len(buffer) + self.pos = 0 + + def read(self, size: int = 0) -> bytes: + b = self.buffer[self.pos : self.pos + size] + self.pos += size + return b + + def seek(self, offset: int, whence: int = SEEK_SET) -> None: + if whence == SEEK_SET: + self.pos = offset + elif whence == SEEK_CUR: + self.pos += offset + elif whence == SEEK_END: + self.pos = self.len + offset + else: + raise ValueError(f"Unknown whence {offset}") + + def tell(self) -> int: + return self.pos + + def closed(self) -> bool: + return not hasattr(self, "buffer") + + def close(self) -> None: + del self.buffer + self.pos = 0 diff --git a/src/iceberg/types.py b/src/iceberg/types.py index d665dbcae6..d3ae672373 100644 --- a/src/iceberg/types.py +++ b/src/iceberg/types.py @@ -165,7 +165,7 @@ class StructType(IcebergType): def __init__(self, *fields: NestedField, **kwargs): # pylint: disable=super-init-not-called if not fields and "fields" in kwargs: fields = kwargs["fields"] - object.__setattr__(self, "fields", fields) + object.__setattr__(self, "fields", tuple(fields)) @cached_property def string_type(self) -> str: diff --git a/src/iceberg/utils/datetime.py b/src/iceberg/utils/datetime.py index c10dda4fb2..9e491ce9ce 100644 --- a/src/iceberg/utils/datetime.py +++ b/src/iceberg/utils/datetime.py @@ -16,6 +16,8 @@ # under the License. """Helper methods for working with date/time representations """ +from __future__ import annotations + import re from datetime import ( date, @@ -36,24 +38,31 @@ def micros_to_days(timestamp: int) -> int: return (datetime.fromtimestamp(timestamp / 1_000_000) - EPOCH_TIMESTAMP).days +def micros_to_time(micros: int) -> time: + """Converts a timestamp in microseconds to a time""" + micros, microseconds = divmod(micros, 1000000) + micros, seconds = divmod(micros, 60) + micros, minutes = divmod(micros, 60) + hours = micros + return time(hour=hours, minute=minutes, second=seconds, microsecond=microseconds) + + def date_to_days(date_str: str) -> int: """Converts an ISO-8601 formatted date to days from 1970-01-01""" return (date.fromisoformat(date_str) - EPOCH_DATE).days +def days_to_date(days: int) -> date: + """Creates a date from the number of days from 1970-01-01""" + return EPOCH_DATE + timedelta(days) + + def time_to_micros(time_str: str) -> int: """Converts an ISO-8601 formatted time to microseconds from midnight""" t = time.fromisoformat(time_str) return (((t.hour * 60 + t.minute) * 60) + t.second) * 1_000_000 + t.microsecond -def time_from_micros(micros: int) -> time: - seconds = micros // 1_000_000 - minutes = seconds // 60 - hours = minutes // 60 - return time(hour=hours, minute=minutes % 60, second=seconds % 60, microsecond=micros % 1_000_000) - - def datetime_to_micros(dt: datetime) -> int: """Converts a datetime to microseconds from 1970-01-01T00:00:00.000000""" if dt.tzinfo: @@ -77,6 +86,18 @@ def timestamptz_to_micros(timestamptz_str: str) -> int: raise ValueError(f"Invalid timestamp with zone: {timestamptz_str} (must be ISO-8601)") +def micros_to_timestamp(micros: int): + """Converts microseconds from epoch to a timestamp""" + dt = timedelta(microseconds=micros) + return EPOCH_TIMESTAMP + dt + + +def micros_to_timestamptz(micros: int): + """Converts microseconds from epoch to an utc timestamp""" + dt = timedelta(microseconds=micros) + return EPOCH_TIMESTAMPTZ + dt + + def to_human_day(day_ordinal: int) -> str: """Converts a DateType value to human string""" return (EPOCH_DATE + timedelta(days=day_ordinal)).isoformat() @@ -84,7 +105,7 @@ def to_human_day(day_ordinal: int) -> str: def to_human_time(micros_from_midnight: int) -> str: """Converts a TimeType value to human string""" - return time_from_micros(micros_from_midnight).isoformat() + return micros_to_time(micros_from_midnight).isoformat() def to_human_timestamptz(timestamp_micros: int) -> str: diff --git a/src/iceberg/utils/schema_conversion.py b/src/iceberg/utils/schema_conversion.py index 4e57daa031..f5b70354b9 100644 --- a/src/iceberg/utils/schema_conversion.py +++ b/src/iceberg/utils/schema_conversion.py @@ -14,9 +14,7 @@ # KIND, either express or implied. See the License for the # specific language governing permissions and limitations # under the License. -"""Utility class for converting between Avro and Iceberg schemas - -""" +"""Utility class for converting between Avro and Iceberg schemas""" from __future__ import annotations import logging @@ -116,23 +114,23 @@ def avro_to_iceberg(self, avro_schema: dict[str, Any]) -> Schema: def _resolve_union(self, type_union: dict | list | str) -> tuple[str | dict[str, Any], bool]: """ - Converts Unions into their type and resolves if the field is optional + Converts Unions into their type and resolves if the field is required Examples: >>> AvroSchemaConversion()._resolve_union('str') - ('str', False) - >>> AvroSchemaConversion()._resolve_union(['null', 'str']) ('str', True) + >>> AvroSchemaConversion()._resolve_union(['null', 'str']) + ('str', False) >>> AvroSchemaConversion()._resolve_union([{'type': 'str'}]) - ({'type': 'str'}, False) - >>> AvroSchemaConversion()._resolve_union(['null', {'type': 'str'}]) ({'type': 'str'}, True) + >>> AvroSchemaConversion()._resolve_union(['null', {'type': 'str'}]) + ({'type': 'str'}, False) Args: type_union: The field, can be a string 'str', list ['null', 'str'], or dict {"type": 'str'} Returns: - A tuple containing the type and nullability + A tuple containing the type and if required Raises: TypeError: In the case non-optional union types are encountered @@ -140,20 +138,28 @@ def _resolve_union(self, type_union: dict | list | str) -> tuple[str | dict[str, avro_types: dict | list if isinstance(type_union, str): # It is a primitive and required - return type_union, False + return type_union, True elif isinstance(type_union, dict): # It is a context and required - return type_union, False + return type_union, True else: avro_types = type_union - is_optional = "null" in avro_types - if len(avro_types) > 2: - raise TypeError("Non-optional types aren't part of the Iceberg specification") + raise TypeError(f"Non-optional types aren't part of the Iceberg specification: {avro_types}") + + # For the Iceberg spec it is required to set the default value to null + # From https://iceberg.apache.org/spec/#avro + # Optional fields must always set the Avro field default value to null. + # + # This means that null has to come first: + # https://avro.apache.org/docs/current/spec.html + # type of the default value must match the first element of the union. + if "null" != avro_types[0]: + raise TypeError("Only null-unions are supported") # Filter the null value and return the type - return list(filter(lambda t: t != "null", avro_types))[0], is_optional + return list(filter(lambda t: t != "null", avro_types))[0], False def _convert_schema(self, avro_type: str | dict[str, Any]) -> IcebergType: """ @@ -205,13 +211,13 @@ def _convert_field(self, field: dict[str, Any]) -> NestedField: if "field-id" not in field: raise ValueError(f"Cannot convert field, missing field-id: {field}") - plain_type, is_optional = self._resolve_union(field["type"]) + plain_type, required = self._resolve_union(field["type"]) return NestedField( field_id=field["field-id"], name=field["name"], field_type=self._convert_schema(plain_type), - required=is_optional, + required=required, doc=field.get("doc"), ) @@ -273,12 +279,12 @@ def _convert_array_type(self, array_type: dict[str, Any]) -> ListType: if "element-id" not in array_type: raise ValueError(f"Cannot convert array-type, missing element-id: {array_type}") - plain_type, element_is_optional = self._resolve_union(array_type["items"]) + plain_type, element_required = self._resolve_union(array_type["items"]) return ListType( element_id=array_type["element-id"], element_type=self._convert_schema(plain_type), - element_required=element_is_optional, + element_required=element_required, ) def _convert_map_type(self, map_type: dict[str, Any]) -> MapType: @@ -290,7 +296,7 @@ def _convert_map_type(self, map_type: dict[str, Any]) -> MapType: >>> from iceberg.utils.schema_conversion import AvroSchemaConversion >>> avro_field = { ... "type": "map", - ... "values": ["long", "null"], + ... "values": ["null", "long"], ... "key-id": 101, ... "value-id": 102, ... } @@ -307,14 +313,14 @@ def _convert_map_type(self, map_type: dict[str, Any]) -> MapType: Returns: A MapType """ - value_type, value_is_optional = self._resolve_union(map_type["values"]) + value_type, value_required = self._resolve_union(map_type["values"]) return MapType( key_id=map_type["key-id"], # Avro only supports string keys key_type=StringType(), value_id=map_type["value-id"], value_type=self._convert_schema(value_type), - value_required=value_is_optional, + value_required=value_required, ) def _convert_logical_type(self, avro_logical_type: dict[str, Any]) -> IcebergType: diff --git a/src/iceberg/utils/singleton.py b/src/iceberg/utils/singleton.py index c36155de14..5643cdd172 100644 --- a/src/iceberg/utils/singleton.py +++ b/src/iceberg/utils/singleton.py @@ -14,7 +14,20 @@ # KIND, either express or implied. See the License for the # specific language governing permissions and limitations # under the License. +""" +This is a singleton metaclass that can be used to cache and re-use existing objects +In the Iceberg codebase we have a lot of objects that are stateless (for example Types such as StringType, +BooleanType etc). FixedTypes have arguments (eg. Fixed[22]) that we also make part of the key when caching +the newly created object. + +The Singleton uses a metaclass which essentially defines a new type. When the Type gets created, it will first +evaluate the `__call__` method with all the arguments. If we already initialized a class earlier, we'll just +return it. + +More information on metaclasses: https://docs.python.org/3/reference/datamodel.html#metaclasses + +""" from typing import ClassVar, Dict diff --git a/tests/avro/__init__.py b/tests/avro/__init__.py new file mode 100644 index 0000000000..a67d5ea255 --- /dev/null +++ b/tests/avro/__init__.py @@ -0,0 +1,16 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. diff --git a/tests/avro/test_decoder.py b/tests/avro/test_decoder.py new file mode 100644 index 0000000000..2957151151 --- /dev/null +++ b/tests/avro/test_decoder.py @@ -0,0 +1,140 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +from datetime import date, datetime, timezone +from decimal import Decimal +from io import SEEK_SET + +import pytest + +from iceberg.avro.decoder import BinaryDecoder +from iceberg.io.base import InputStream +from iceberg.io.memory import MemoryInputStream + + +def test_read_decimal_from_fixed(): + mis = MemoryInputStream(b"\x00\x00\x00\x05\x6A\x48\x1C\xFB\x2C\x7C\x50\x00") + decoder = BinaryDecoder(mis) + actual = decoder.read_decimal_from_fixed(28, 15, 12) + expected = Decimal("99892.123400000000000") + assert actual == expected + + +def test_read_long(): + mis = MemoryInputStream(b"\x18") + decoder = BinaryDecoder(mis) + assert decoder.read_long() == 12 + + +def test_read_decimal(): + mis = MemoryInputStream(b"\x18\x00\x00\x00\x05\x6A\x48\x1C\xFB\x2C\x7C\x50\x00") + decoder = BinaryDecoder(mis) + actual = decoder.read_decimal_from_bytes(28, 15) + expected = Decimal("99892.123400000000000") + assert actual == expected + + +def test_decimal_from_fixed_big(): + mis = MemoryInputStream(b"\x0E\xC2\x02\xE9\x06\x16\x33\x49\x77\x67\xA8\x00") + decoder = BinaryDecoder(mis) + actual = decoder.read_decimal_from_fixed(28, 15, 12) + expected = Decimal("4567335489766.998340000000000") + assert actual == expected + + +def test_read_negative_bytes(): + mis = MemoryInputStream(b"") + decoder = BinaryDecoder(mis) + + with pytest.raises(ValueError) as exc_info: + decoder.read(-1) + + assert "Requested -1 bytes to read, expected positive integer." in str(exc_info.value) + + +class OneByteAtATimeInputStream(InputStream): + """ + Fake input stream that just returns a single byte at the time + """ + + pos = 0 + + def read(self, size: int = 0) -> bytes: + self.pos += 1 + return int.to_bytes(1, self.pos, byteorder="little") + + def seek(self, offset: int, whence: int = SEEK_SET) -> None: + pass + + def tell(self) -> int: + pass + + def closed(self) -> bool: + pass + + def close(self) -> None: + pass + + +def test_read_single_byte_at_the_time(): + decoder = BinaryDecoder(OneByteAtATimeInputStream()) + + with pytest.raises(ValueError) as exc_info: + decoder.read(2) + + assert "Read 1 bytes, expected 2 bytes" in str(exc_info.value) + + +def test_read_float(): + mis = MemoryInputStream(b"\x00\x00\x9A\x41") + decoder = BinaryDecoder(mis) + assert decoder.read_float() == 19.25 + + +def test_read_double(): + mis = MemoryInputStream(b"\x00\x00\x00\x00\x00\x40\x33\x40") + decoder = BinaryDecoder(mis) + assert decoder.read_double() == 19.25 + + +def test_read_date(): + mis = MemoryInputStream(b"\xBC\x7D") + decoder = BinaryDecoder(mis) + assert decoder.read_date_from_int() == date(1991, 12, 27) + + +def test_read_time_millis(): + mis = MemoryInputStream(b"\xBC\x7D") + decoder = BinaryDecoder(mis) + assert decoder.read_time_millis().microsecond == 30000 + + +def test_read_time_micros(): + mis = MemoryInputStream(b"\xBC\x7D") + decoder = BinaryDecoder(mis) + assert decoder.read_time_micros().microsecond == 8030 + + +def test_read_timestamp_micros(): + mis = MemoryInputStream(b"\xBC\x7D") + decoder = BinaryDecoder(mis) + assert decoder.read_timestamp_micros() == datetime(1970, 1, 1, 0, 0, 0, 8030) + + +def test_read_timestamptz_micros(): + mis = MemoryInputStream(b"\xBC\x7D") + decoder = BinaryDecoder(mis) + assert decoder.read_timestamptz_micros() == datetime(1970, 1, 1, 0, 0, 0, 8030, tzinfo=timezone.utc) diff --git a/tests/avro/test_file.py b/tests/avro/test_file.py new file mode 100644 index 0000000000..d345865d11 --- /dev/null +++ b/tests/avro/test_file.py @@ -0,0 +1,48 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +import pytest + +from iceberg.avro.codecs import DeflateCodec +from iceberg.avro.file import AvroFileHeader + + +def get_deflate_compressor(): + header = AvroFileHeader(bytes(0), {"avro.codec": "deflate"}, bytes(16)) + assert header.compression_codec() == DeflateCodec + + +def get_null_compressor(): + header = AvroFileHeader(bytes(0), {"avro.codec": "null"}, bytes(16)) + assert header.compression_codec() is None + + +def test_unknown_codec(): + header = AvroFileHeader(bytes(0), {"avro.codec": "unknown"}, bytes(16)) + + with pytest.raises(ValueError) as exc_info: + header.compression_codec() + + assert "Unsupported codec: unknown" in str(exc_info.value) + + +def test_missing_schema(): + header = AvroFileHeader(bytes(0), {}, bytes(16)) + + with pytest.raises(ValueError) as exc_info: + header.get_schema() + + assert "No schema found in Avro file headers" in str(exc_info.value) diff --git a/tests/avro/test_reader.py b/tests/avro/test_reader.py new file mode 100644 index 0000000000..c310e69204 --- /dev/null +++ b/tests/avro/test_reader.py @@ -0,0 +1,455 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +import json + +import pytest + +from iceberg.avro.file import AvroFile +from iceberg.avro.reader import ( + AvroStruct, + BinaryReader, + BooleanReader, + DateReader, + DecimalReader, + DoubleReader, + FixedReader, + FloatReader, + IntegerReader, + LongReader, + StringReader, + TimeReader, + TimestampReader, + TimestamptzReader, + primitive_reader, +) +from iceberg.schema import Schema +from iceberg.types import ( + BinaryType, + BooleanType, + DateType, + DecimalType, + DoubleType, + FixedType, + FloatType, + IntegerType, + LongType, + PrimitiveType, + StringType, + TimestampType, + TimestamptzType, + TimeType, +) +from tests.io.test_io_base import LocalInputFile + + +def test_read_header(generated_manifest_entry_file: str, iceberg_manifest_entry_schema: Schema): + with AvroFile(LocalInputFile(generated_manifest_entry_file)) as reader: + header = reader._read_header() + + assert header.magic == b"Obj\x01" + assert json.loads(header.meta["avro.schema"]) == { + "type": "record", + "name": "manifest_entry", + "fields": [ + {"field-id": 0, "name": "status", "type": "int"}, + {"field-id": 1, "default": "null", "name": "snapshot_id", "type": ["null", "long"]}, + { + "field-id": 2, + "name": "data_file", + "type": { + "type": "record", + "name": "r2", + "fields": [ + {"field-id": 100, "doc": "Location URI with FS scheme", "name": "file_path", "type": "string"}, + { + "field-id": 101, + "doc": "File format name: avro, orc, or parquet", + "name": "file_format", + "type": "string", + }, + { + "field-id": 102, + "name": "partition", + "type": { + "type": "record", + "name": "r102", + "fields": [{"field-id": 1000, "default": "null", "name": "VendorID", "type": ["null", "int"]}], + }, + }, + {"field-id": 103, "doc": "Number of records in the file", "name": "record_count", "type": "long"}, + {"field-id": 104, "doc": "Total file size in bytes", "name": "file_size_in_bytes", "type": "long"}, + {"field-id": 105, "name": "block_size_in_bytes", "type": "long"}, + { + "field-id": 108, + "doc": "Map of column id to total size on disk", + "default": "null", + "name": "column_sizes", + "type": [ + "null", + { + "logicalType": "map", + "type": "array", + "items": { + "type": "record", + "name": "k117_v118", + "fields": [ + {"field-id": 117, "name": "key", "type": "int"}, + {"field-id": 118, "name": "value", "type": "long"}, + ], + }, + }, + ], + }, + { + "field-id": 109, + "doc": "Map of column id to total count, including null and NaN", + "default": "null", + "name": "value_counts", + "type": [ + "null", + { + "logicalType": "map", + "type": "array", + "items": { + "type": "record", + "name": "k119_v120", + "fields": [ + {"field-id": 119, "name": "key", "type": "int"}, + {"field-id": 120, "name": "value", "type": "long"}, + ], + }, + }, + ], + }, + { + "field-id": 110, + "doc": "Map of column id to null value count", + "default": "null", + "name": "null_value_counts", + "type": [ + "null", + { + "logicalType": "map", + "type": "array", + "items": { + "type": "record", + "name": "k121_v122", + "fields": [ + {"field-id": 121, "name": "key", "type": "int"}, + {"field-id": 122, "name": "value", "type": "long"}, + ], + }, + }, + ], + }, + { + "field-id": 137, + "doc": "Map of column id to number of NaN values in the column", + "default": "null", + "name": "nan_value_counts", + "type": [ + "null", + { + "logicalType": "map", + "type": "array", + "items": { + "type": "record", + "name": "k138_v139", + "fields": [ + {"field-id": 138, "name": "key", "type": "int"}, + {"field-id": 139, "name": "value", "type": "long"}, + ], + }, + }, + ], + }, + { + "field-id": 125, + "doc": "Map of column id to lower bound", + "default": "null", + "name": "lower_bounds", + "type": [ + "null", + { + "logicalType": "map", + "type": "array", + "items": { + "type": "record", + "name": "k126_v127", + "fields": [ + {"field-id": 126, "name": "key", "type": "int"}, + {"field-id": 127, "name": "value", "type": "bytes"}, + ], + }, + }, + ], + }, + { + "field-id": 128, + "doc": "Map of column id to upper bound", + "default": "null", + "name": "upper_bounds", + "type": [ + "null", + { + "logicalType": "map", + "type": "array", + "items": { + "type": "record", + "name": "k129_v130", + "fields": [ + {"field-id": 129, "name": "key", "type": "int"}, + {"field-id": 130, "name": "value", "type": "bytes"}, + ], + }, + }, + ], + }, + { + "field-id": 131, + "doc": "Encryption key metadata blob", + "default": "null", + "name": "key_metadata", + "type": ["null", "bytes"], + }, + { + "field-id": 132, + "doc": "Splittable offsets", + "default": "null", + "name": "split_offsets", + "type": ["null", {"element-id": 133, "type": "array", "items": "long"}], + }, + { + "field-id": 140, + "doc": "Sort order ID", + "default": "null", + "name": "sort_order_id", + "type": ["null", "int"], + }, + ], + }, + }, + ], + } + + assert header.get_schema() == iceberg_manifest_entry_schema + + +def test_read_manifest_entry_file(generated_manifest_entry_file: str): + with AvroFile(LocalInputFile(generated_manifest_entry_file)) as reader: + # Consume the generator + records = list(reader) + + assert len(records) == 2, f"Expected 2 records, got {len(records)}" + assert records[0] == AvroStruct( + _data=[ + 1, + 8744736658442914487, + AvroStruct( + _data=[ + "/home/iceberg/warehouse/nyc/taxis_partitioned/data/VendorID=null/00000-633-d8a4223e-dc97-45a1-86e1-adaba6e8abd7-00001.parquet", + "PARQUET", + AvroStruct(_data=[None]), + 19513, + 388872, + 67108864, + { + 1: 53, + 2: 98153, + 3: 98693, + 4: 53, + 5: 53, + 6: 53, + 7: 17425, + 8: 18528, + 9: 53, + 10: 44788, + 11: 35571, + 12: 53, + 13: 1243, + 14: 2355, + 15: 12750, + 16: 4029, + 17: 110, + 18: 47194, + 19: 2948, + }, + { + 1: 19513, + 2: 19513, + 3: 19513, + 4: 19513, + 5: 19513, + 6: 19513, + 7: 19513, + 8: 19513, + 9: 19513, + 10: 19513, + 11: 19513, + 12: 19513, + 13: 19513, + 14: 19513, + 15: 19513, + 16: 19513, + 17: 19513, + 18: 19513, + 19: 19513, + }, + { + 1: 19513, + 2: 0, + 3: 0, + 4: 19513, + 5: 19513, + 6: 19513, + 7: 0, + 8: 0, + 9: 19513, + 10: 0, + 11: 0, + 12: 19513, + 13: 0, + 14: 0, + 15: 0, + 16: 0, + 17: 0, + 18: 0, + 19: 0, + }, + {16: 0, 17: 0, 18: 0, 19: 0, 10: 0, 11: 0, 12: 0, 13: 0, 14: 0, 15: 0}, + { + 2: b"2020-04-01 00:00", + 3: b"2020-04-01 00:12", + 7: b"\x03\x00\x00\x00", + 8: b"\x01\x00\x00\x00", + 10: b"\xf6(\\\x8f\xc2\x05S\xc0", + 11: b"\x00\x00\x00\x00\x00\x00\x00\x00", + 13: b"\x00\x00\x00\x00\x00\x00\x00\x00", + 14: b"\x00\x00\x00\x00\x00\x00\xe0\xbf", + 15: b")\\\x8f\xc2\xf5(\x08\xc0", + 16: b"\x00\x00\x00\x00\x00\x00\x00\x00", + 17: b"\x00\x00\x00\x00\x00\x00\x00\x00", + 18: b"\xf6(\\\x8f\xc2\xc5S\xc0", + 19: b"\x00\x00\x00\x00\x00\x00\x04\xc0", + }, + { + 2: b"2020-04-30 23:5:", + 3: b"2020-05-01 00:41", + 7: b"\t\x01\x00\x00", + 8: b"\t\x01\x00\x00", + 10: b"\xcd\xcc\xcc\xcc\xcc,_@", + 11: b"\x1f\x85\xebQ\\\xe2\xfe@", + 13: b"\x00\x00\x00\x00\x00\x00\x12@", + 14: b"\x00\x00\x00\x00\x00\x00\xe0?", + 15: b"q=\n\xd7\xa3\xf01@", + 16: b"\x00\x00\x00\x00\x00`B@", + 17: b"333333\xd3?", + 18: b"\x00\x00\x00\x00\x00\x18b@", + 19: b"\x00\x00\x00\x00\x00\x00\x04@", + }, + None, + [4], + 0, + ] + ), + ] + ) + + +def test_read_manifest_file_file(generated_manifest_file_file: str): + with AvroFile(LocalInputFile(generated_manifest_file_file)) as reader: + # Consume the generator + records = list(reader) + + assert len(records) == 1, f"Expected 1 records, got {len(records)}" + assert records[0] == AvroStruct( + _data=[ + "/home/iceberg/warehouse/nyc/taxis_partitioned/metadata/0125c686-8aa6-4502-bdcc-b6d17ca41a3b-m0.avro", + 7989, + 0, + 9182715666859759686, + 3, + 0, + 0, + [AvroStruct(_data=[True, False, b"\x01\x00\x00\x00", b"\x02\x00\x00\x00"])], + 237993, + 0, + 0, + ] + ) + + +def test_fixed_reader(): + assert primitive_reader(FixedType(22)) == FixedReader(22) + + +def test_decimal_reader(): + assert primitive_reader(DecimalType(19, 25)) == DecimalReader(19, 25) + + +def test_boolean_reader(): + assert primitive_reader(BooleanType()) == BooleanReader() + + +def test_integer_reader(): + assert primitive_reader(IntegerType()) == IntegerReader() + + +def test_long_reader(): + assert primitive_reader(LongType()) == LongReader() + + +def test_float_reader(): + assert primitive_reader(FloatType()) == FloatReader() + + +def test_double_reader(): + assert primitive_reader(DoubleType()) == DoubleReader() + + +def test_date_reader(): + assert primitive_reader(DateType()) == DateReader() + + +def test_time_reader(): + assert primitive_reader(TimeType()) == TimeReader() + + +def test_timestamp_reader(): + assert primitive_reader(TimestampType()) == TimestampReader() + + +def test_timestamptz_reader(): + assert primitive_reader(TimestamptzType()) == TimestamptzReader() + + +def test_string_reader(): + assert primitive_reader(StringType()) == StringReader() + + +def test_binary_reader(): + assert primitive_reader(BinaryType()) == BinaryReader() + + +def test_unknown_type(): + class UnknownType(PrimitiveType): + ... + + with pytest.raises(ValueError) as exc_info: + primitive_reader(UnknownType()) + + assert "Unknown type:" in str(exc_info.value) diff --git a/tests/conftest.py b/tests/conftest.py index b5dd18b8d8..48f3bf0239 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -15,16 +15,20 @@ # specific language governing permissions and limitations # under the License. +from tempfile import TemporaryDirectory from typing import Any, Dict import pytest from iceberg import schema +from iceberg.schema import Schema from iceberg.types import ( + BinaryType, BooleanType, FloatType, IntegerType, ListType, + LongType, MapType, NestedField, StringType, @@ -114,7 +118,367 @@ def foo_struct(): @pytest.fixture(scope="session") -def manifest_schema() -> Dict[str, Any]: +def all_avro_types() -> Dict[str, Any]: + return { + "type": "record", + "name": "all_avro_types", + "fields": [ + {"name": "primitive_string", "type": "string", "field-id": 100}, + {"name": "primitive_int", "type": "int", "field-id": 200}, + {"name": "primitive_long", "type": "long", "field-id": 300}, + {"name": "primitive_float", "type": "float", "field-id": 400}, + {"name": "primitive_double", "type": "double", "field-id": 500}, + {"name": "primitive_bytes", "type": "bytes", "field-id": 600}, + { + "type": "record", + "name": "Person", + "fields": [ + {"name": "name", "type": "string", "field-id": 701}, + {"name": "age", "type": "long", "field-id": 702}, + {"name": "gender", "type": ["string", "null"], "field-id": 703}, + ], + "field-id": 700, + }, + { + "name": "array_with_string", + "type": { + "type": "array", + "items": "string", + "default": [], + "element-id": 801, + }, + "field-id": 800, + }, + { + "name": "array_with_optional_string", + "type": [ + "null", + { + "type": "array", + "items": ["string", "null"], + "default": [], + "element-id": 901, + }, + ], + "field-id": 900, + }, + { + "name": "array_with_optional_record", + "type": [ + "null", + { + "type": "array", + "items": [ + "null", + { + "type": "record", + "name": "person", + "fields": [ + {"name": "name", "type": "string", "field-id": 1002}, + {"name": "age", "type": "long", "field-id": 1003}, + {"name": "gender", "type": ["string", "null"], "field-id": 1004}, + ], + }, + ], + "element-id": 1001, + }, + ], + "field-id": 1000, + }, + { + "name": "map_with_longs", + "type": { + "type": "map", + "values": "long", + "default": {}, + "key-id": 1101, + "value-id": 1102, + }, + "field-id": 1000, + }, + ], + } + + +@pytest.fixture +def catalog() -> InMemoryCatalog: + return InMemoryCatalog("test.in.memory.catalog", {"test.key": "test.value"}) + + +manifest_entry_records = [ + { + "status": 1, + "snapshot_id": 8744736658442914487, + "data_file": { + "file_path": "/home/iceberg/warehouse/nyc/taxis_partitioned/data/VendorID=null/00000-633-d8a4223e-dc97-45a1-86e1-adaba6e8abd7-00001.parquet", + "file_format": "PARQUET", + "partition": {"VendorID": None}, + "record_count": 19513, + "file_size_in_bytes": 388872, + "block_size_in_bytes": 67108864, + "column_sizes": [ + {"key": 1, "value": 53}, + {"key": 2, "value": 98153}, + {"key": 3, "value": 98693}, + {"key": 4, "value": 53}, + {"key": 5, "value": 53}, + {"key": 6, "value": 53}, + {"key": 7, "value": 17425}, + {"key": 8, "value": 18528}, + {"key": 9, "value": 53}, + {"key": 10, "value": 44788}, + {"key": 11, "value": 35571}, + {"key": 12, "value": 53}, + {"key": 13, "value": 1243}, + {"key": 14, "value": 2355}, + {"key": 15, "value": 12750}, + {"key": 16, "value": 4029}, + {"key": 17, "value": 110}, + {"key": 18, "value": 47194}, + {"key": 19, "value": 2948}, + ], + "value_counts": [ + {"key": 1, "value": 19513}, + {"key": 2, "value": 19513}, + {"key": 3, "value": 19513}, + {"key": 4, "value": 19513}, + {"key": 5, "value": 19513}, + {"key": 6, "value": 19513}, + {"key": 7, "value": 19513}, + {"key": 8, "value": 19513}, + {"key": 9, "value": 19513}, + {"key": 10, "value": 19513}, + {"key": 11, "value": 19513}, + {"key": 12, "value": 19513}, + {"key": 13, "value": 19513}, + {"key": 14, "value": 19513}, + {"key": 15, "value": 19513}, + {"key": 16, "value": 19513}, + {"key": 17, "value": 19513}, + {"key": 18, "value": 19513}, + {"key": 19, "value": 19513}, + ], + "null_value_counts": [ + {"key": 1, "value": 19513}, + {"key": 2, "value": 0}, + {"key": 3, "value": 0}, + {"key": 4, "value": 19513}, + {"key": 5, "value": 19513}, + {"key": 6, "value": 19513}, + {"key": 7, "value": 0}, + {"key": 8, "value": 0}, + {"key": 9, "value": 19513}, + {"key": 10, "value": 0}, + {"key": 11, "value": 0}, + {"key": 12, "value": 19513}, + {"key": 13, "value": 0}, + {"key": 14, "value": 0}, + {"key": 15, "value": 0}, + {"key": 16, "value": 0}, + {"key": 17, "value": 0}, + {"key": 18, "value": 0}, + {"key": 19, "value": 0}, + ], + "nan_value_counts": [ + {"key": 16, "value": 0}, + {"key": 17, "value": 0}, + {"key": 18, "value": 0}, + {"key": 19, "value": 0}, + {"key": 10, "value": 0}, + {"key": 11, "value": 0}, + {"key": 12, "value": 0}, + {"key": 13, "value": 0}, + {"key": 14, "value": 0}, + {"key": 15, "value": 0}, + ], + "lower_bounds": [ + {"key": 2, "value": b"2020-04-01 00:00"}, + {"key": 3, "value": b"2020-04-01 00:12"}, + {"key": 7, "value": b"\x03\x00\x00\x00"}, + {"key": 8, "value": b"\x01\x00\x00\x00"}, + {"key": 10, "value": b"\xf6(\\\x8f\xc2\x05S\xc0"}, + {"key": 11, "value": b"\x00\x00\x00\x00\x00\x00\x00\x00"}, + {"key": 13, "value": b"\x00\x00\x00\x00\x00\x00\x00\x00"}, + {"key": 14, "value": b"\x00\x00\x00\x00\x00\x00\xe0\xbf"}, + {"key": 15, "value": b")\\\x8f\xc2\xf5(\x08\xc0"}, + {"key": 16, "value": b"\x00\x00\x00\x00\x00\x00\x00\x00"}, + {"key": 17, "value": b"\x00\x00\x00\x00\x00\x00\x00\x00"}, + {"key": 18, "value": b"\xf6(\\\x8f\xc2\xc5S\xc0"}, + {"key": 19, "value": b"\x00\x00\x00\x00\x00\x00\x04\xc0"}, + ], + "upper_bounds": [ + {"key": 2, "value": b"2020-04-30 23:5:"}, + {"key": 3, "value": b"2020-05-01 00:41"}, + {"key": 7, "value": b"\t\x01\x00\x00"}, + {"key": 8, "value": b"\t\x01\x00\x00"}, + {"key": 10, "value": b"\xcd\xcc\xcc\xcc\xcc,_@"}, + {"key": 11, "value": b"\x1f\x85\xebQ\\\xe2\xfe@"}, + {"key": 13, "value": b"\x00\x00\x00\x00\x00\x00\x12@"}, + {"key": 14, "value": b"\x00\x00\x00\x00\x00\x00\xe0?"}, + {"key": 15, "value": b"q=\n\xd7\xa3\xf01@"}, + {"key": 16, "value": b"\x00\x00\x00\x00\x00`B@"}, + {"key": 17, "value": b"333333\xd3?"}, + {"key": 18, "value": b"\x00\x00\x00\x00\x00\x18b@"}, + {"key": 19, "value": b"\x00\x00\x00\x00\x00\x00\x04@"}, + ], + "key_metadata": None, + "split_offsets": [4], + "sort_order_id": 0, + }, + }, + { + "status": 1, + "snapshot_id": 8744736658442914487, + "data_file": { + "file_path": "/home/iceberg/warehouse/nyc/taxis_partitioned/data/VendorID=1/00000-633-d8a4223e-dc97-45a1-86e1-adaba6e8abd7-00002.parquet", + "file_format": "PARQUET", + "partition": {"VendorID": 1}, + "record_count": 95050, + "file_size_in_bytes": 1265950, + "block_size_in_bytes": 67108864, + "column_sizes": [ + {"key": 1, "value": 318}, + {"key": 2, "value": 329806}, + {"key": 3, "value": 331632}, + {"key": 4, "value": 15343}, + {"key": 5, "value": 2351}, + {"key": 6, "value": 3389}, + {"key": 7, "value": 71269}, + {"key": 8, "value": 76429}, + {"key": 9, "value": 16383}, + {"key": 10, "value": 86992}, + {"key": 11, "value": 89608}, + {"key": 12, "value": 265}, + {"key": 13, "value": 19377}, + {"key": 14, "value": 1692}, + {"key": 15, "value": 76162}, + {"key": 16, "value": 4354}, + {"key": 17, "value": 759}, + {"key": 18, "value": 120650}, + {"key": 19, "value": 11804}, + ], + "value_counts": [ + {"key": 1, "value": 95050}, + {"key": 2, "value": 95050}, + {"key": 3, "value": 95050}, + {"key": 4, "value": 95050}, + {"key": 5, "value": 95050}, + {"key": 6, "value": 95050}, + {"key": 7, "value": 95050}, + {"key": 8, "value": 95050}, + {"key": 9, "value": 95050}, + {"key": 10, "value": 95050}, + {"key": 11, "value": 95050}, + {"key": 12, "value": 95050}, + {"key": 13, "value": 95050}, + {"key": 14, "value": 95050}, + {"key": 15, "value": 95050}, + {"key": 16, "value": 95050}, + {"key": 17, "value": 95050}, + {"key": 18, "value": 95050}, + {"key": 19, "value": 95050}, + ], + "null_value_counts": [ + {"key": 1, "value": 0}, + {"key": 2, "value": 0}, + {"key": 3, "value": 0}, + {"key": 4, "value": 0}, + {"key": 5, "value": 0}, + {"key": 6, "value": 0}, + {"key": 7, "value": 0}, + {"key": 8, "value": 0}, + {"key": 9, "value": 0}, + {"key": 10, "value": 0}, + {"key": 11, "value": 0}, + {"key": 12, "value": 95050}, + {"key": 13, "value": 0}, + {"key": 14, "value": 0}, + {"key": 15, "value": 0}, + {"key": 16, "value": 0}, + {"key": 17, "value": 0}, + {"key": 18, "value": 0}, + {"key": 19, "value": 0}, + ], + "nan_value_counts": [ + {"key": 16, "value": 0}, + {"key": 17, "value": 0}, + {"key": 18, "value": 0}, + {"key": 19, "value": 0}, + {"key": 10, "value": 0}, + {"key": 11, "value": 0}, + {"key": 12, "value": 0}, + {"key": 13, "value": 0}, + {"key": 14, "value": 0}, + {"key": 15, "value": 0}, + ], + "lower_bounds": [ + {"key": 1, "value": b"\x01\x00\x00\x00"}, + {"key": 2, "value": b"2020-04-01 00:00"}, + {"key": 3, "value": b"2020-04-01 00:03"}, + {"key": 4, "value": b"\x00\x00\x00\x00"}, + {"key": 5, "value": b"\x01\x00\x00\x00"}, + {"key": 6, "value": b"N"}, + {"key": 7, "value": b"\x01\x00\x00\x00"}, + {"key": 8, "value": b"\x01\x00\x00\x00"}, + {"key": 9, "value": b"\x01\x00\x00\x00"}, + {"key": 10, "value": b"\x00\x00\x00\x00\x00\x00\x00\x00"}, + {"key": 11, "value": b"\x00\x00\x00\x00\x00\x00\x00\x00"}, + {"key": 13, "value": b"\x00\x00\x00\x00\x00\x00\x00\x00"}, + {"key": 14, "value": b"\x00\x00\x00\x00\x00\x00\x00\x00"}, + {"key": 15, "value": b"\x00\x00\x00\x00\x00\x00\x00\x00"}, + {"key": 16, "value": b"\x00\x00\x00\x00\x00\x00\x00\x00"}, + {"key": 17, "value": b"\x00\x00\x00\x00\x00\x00\x00\x00"}, + {"key": 18, "value": b"\x00\x00\x00\x00\x00\x00\x00\x00"}, + {"key": 19, "value": b"\x00\x00\x00\x00\x00\x00\x00\x00"}, + ], + "upper_bounds": [ + {"key": 1, "value": b"\x01\x00\x00\x00"}, + {"key": 2, "value": b"2020-04-30 23:5:"}, + {"key": 3, "value": b"2020-05-01 00:1:"}, + {"key": 4, "value": b"\x06\x00\x00\x00"}, + {"key": 5, "value": b"c\x00\x00\x00"}, + {"key": 6, "value": b"Y"}, + {"key": 7, "value": b"\t\x01\x00\x00"}, + {"key": 8, "value": b"\t\x01\x00\x00"}, + {"key": 9, "value": b"\x04\x00\x00\x00"}, + {"key": 10, "value": b"\\\x8f\xc2\xf5(8\x8c@"}, + {"key": 11, "value": b"\xcd\xcc\xcc\xcc\xcc,f@"}, + {"key": 13, "value": b"\x00\x00\x00\x00\x00\x00\x1c@"}, + {"key": 14, "value": b"\x9a\x99\x99\x99\x99\x99\xf1?"}, + {"key": 15, "value": b"\x00\x00\x00\x00\x00\x00Y@"}, + {"key": 16, "value": b"\x00\x00\x00\x00\x00\xb0X@"}, + {"key": 17, "value": b"333333\xd3?"}, + {"key": 18, "value": b"\xc3\xf5(\\\x8f:\x8c@"}, + {"key": 19, "value": b"\x00\x00\x00\x00\x00\x00\x04@"}, + ], + "key_metadata": None, + "split_offsets": [4], + "sort_order_id": 0, + }, + }, +] + +manifest_file_records = [ + { + "manifest_path": "/home/iceberg/warehouse/nyc/taxis_partitioned/metadata/0125c686-8aa6-4502-bdcc-b6d17ca41a3b-m0.avro", + "manifest_length": 7989, + "partition_spec_id": 0, + "added_snapshot_id": 9182715666859759686, + "added_data_files_count": 3, + "existing_data_files_count": 0, + "deleted_data_files_count": 0, + "partitions": [ + {"contains_null": True, "contains_nan": False, "lower_bound": b"\x01\x00\x00\x00", "upper_bound": b"\x02\x00\x00\x00"} + ], + "added_rows_count": 237993, + "existing_rows_count": 0, + "deleted_rows_count": 0, + } +] + + +@pytest.fixture(scope="session") +def avro_schema_manifest_file() -> Dict[str, Any]: return { "type": "record", "name": "manifest_file", @@ -126,28 +490,28 @@ def manifest_schema() -> Dict[str, Any]: "name": "added_snapshot_id", "type": ["null", "long"], "doc": "Snapshot ID that added the manifest", - "default": None, + "default": "null", "field-id": 503, }, { "name": "added_data_files_count", "type": ["null", "int"], "doc": "Added entry count", - "default": None, + "default": "null", "field-id": 504, }, { "name": "existing_data_files_count", "type": ["null", "int"], "doc": "Existing entry count", - "default": None, + "default": "null", "field-id": 505, }, { "name": "deleted_data_files_count", "type": ["null", "int"], "doc": "Deleted entry count", - "default": None, + "default": "null", "field-id": 506, }, { @@ -170,21 +534,21 @@ def manifest_schema() -> Dict[str, Any]: "name": "contains_nan", "type": ["null", "boolean"], "doc": "True if any file has a nan partition value", - "default": None, + "default": "null", "field-id": 518, }, { "name": "lower_bound", "type": ["null", "bytes"], "doc": "Partition lower bound for all files", - "default": None, + "default": "null", "field-id": 510, }, { "name": "upper_bound", "type": ["null", "bytes"], "doc": "Partition upper bound for all files", - "default": None, + "default": "null", "field-id": 511, }, ], @@ -193,22 +557,22 @@ def manifest_schema() -> Dict[str, Any]: }, ], "doc": "Summary for each partition", - "default": None, + "default": "null", "field-id": 507, }, - {"name": "added_rows_count", "type": ["null", "long"], "doc": "Added rows count", "default": None, "field-id": 512}, + {"name": "added_rows_count", "type": ["null", "long"], "doc": "Added rows count", "default": "null", "field-id": 512}, { "name": "existing_rows_count", "type": ["null", "long"], "doc": "Existing rows count", - "default": None, + "default": "null", "field-id": 513, }, { "name": "deleted_rows_count", "type": ["null", "long"], "doc": "Deleted rows count", - "default": None, + "default": "null", "field-id": 514, }, ], @@ -216,88 +580,381 @@ def manifest_schema() -> Dict[str, Any]: @pytest.fixture(scope="session") -def all_avro_types() -> Dict[str, Any]: +def avro_schema_manifest_entry() -> Dict[str, Any]: return { "type": "record", - "name": "all_avro_types", + "name": "manifest_entry", "fields": [ - {"name": "primitive_string", "type": "string", "field-id": 100}, - {"name": "primitive_int", "type": "int", "field-id": 200}, - {"name": "primitive_long", "type": "long", "field-id": 300}, - {"name": "primitive_float", "type": "float", "field-id": 400}, - {"name": "primitive_double", "type": "double", "field-id": 500}, - {"name": "primitive_bytes", "type": "bytes", "field-id": 600}, - { - "type": "record", - "name": "Person", - "fields": [ - {"name": "name", "type": "string", "field-id": 701}, - {"name": "age", "type": "long", "field-id": 702}, - {"name": "gender", "type": ["string", "null"], "field-id": 703}, - ], - "field-id": 700, - }, + {"name": "status", "type": "int", "field-id": 0}, + {"name": "snapshot_id", "type": ["null", "long"], "default": "null", "field-id": 1}, { - "name": "array_with_string", + "name": "data_file", "type": { - "type": "array", - "items": "string", - "default": [], - "element-id": 801, - }, - "field-id": 800, - }, - { - "name": "array_with_optional_string", - "type": [ - "null", - { - "type": "array", - "items": ["string", "null"], - "default": [], - "element-id": 901, - }, - ], - "field-id": 900, - }, - { - "name": "array_with_optional_record", - "type": [ - "null", - { - "type": "array", - "items": [ - "null", - { + "type": "record", + "name": "r2", + "fields": [ + {"name": "file_path", "type": "string", "doc": "Location URI with FS scheme", "field-id": 100}, + { + "name": "file_format", + "type": "string", + "doc": "File format name: avro, orc, or parquet", + "field-id": 101, + }, + { + "name": "partition", + "type": { "type": "record", - "name": "person", - "fields": [ - {"name": "name", "type": "string", "field-id": 1002}, - {"name": "age", "type": "long", "field-id": 1003}, - {"name": "gender", "type": ["string", "null"], "field-id": 1004}, - ], + "name": "r102", + "fields": [{"name": "VendorID", "type": ["null", "int"], "default": "null", "field-id": 1000}], }, - ], - "element-id": 1001, - }, - ], - "field-id": 1000, - }, - { - "name": "map_with_longs", - "type": { - "type": "map", - "values": "long", - "default": {}, - "key-id": 1101, - "value-id": 1102, + "field-id": 102, + }, + {"name": "record_count", "type": "long", "doc": "Number of records in the file", "field-id": 103}, + {"name": "file_size_in_bytes", "type": "long", "doc": "Total file size in bytes", "field-id": 104}, + {"name": "block_size_in_bytes", "type": "long", "field-id": 105}, + { + "name": "column_sizes", + "type": [ + "null", + { + "type": "array", + "items": { + "type": "record", + "name": "k117_v118", + "fields": [ + {"name": "key", "type": "int", "field-id": 117}, + {"name": "value", "type": "long", "field-id": 118}, + ], + }, + "logicalType": "map", + }, + ], + "doc": "Map of column id to total size on disk", + "default": "null", + "field-id": 108, + }, + { + "name": "value_counts", + "type": [ + "null", + { + "type": "array", + "items": { + "type": "record", + "name": "k119_v120", + "fields": [ + {"name": "key", "type": "int", "field-id": 119}, + {"name": "value", "type": "long", "field-id": 120}, + ], + }, + "logicalType": "map", + }, + ], + "doc": "Map of column id to total count, including null and NaN", + "default": "null", + "field-id": 109, + }, + { + "name": "null_value_counts", + "type": [ + "null", + { + "type": "array", + "items": { + "type": "record", + "name": "k121_v122", + "fields": [ + {"name": "key", "type": "int", "field-id": 121}, + {"name": "value", "type": "long", "field-id": 122}, + ], + }, + "logicalType": "map", + }, + ], + "doc": "Map of column id to null value count", + "default": "null", + "field-id": 110, + }, + { + "name": "nan_value_counts", + "type": [ + "null", + { + "type": "array", + "items": { + "type": "record", + "name": "k138_v139", + "fields": [ + {"name": "key", "type": "int", "field-id": 138}, + {"name": "value", "type": "long", "field-id": 139}, + ], + }, + "logicalType": "map", + }, + ], + "doc": "Map of column id to number of NaN values in the column", + "default": "null", + "field-id": 137, + }, + { + "name": "lower_bounds", + "type": [ + "null", + { + "type": "array", + "items": { + "type": "record", + "name": "k126_v127", + "fields": [ + {"name": "key", "type": "int", "field-id": 126}, + {"name": "value", "type": "bytes", "field-id": 127}, + ], + }, + "logicalType": "map", + }, + ], + "doc": "Map of column id to lower bound", + "default": "null", + "field-id": 125, + }, + { + "name": "upper_bounds", + "type": [ + "null", + { + "type": "array", + "items": { + "type": "record", + "name": "k129_v130", + "fields": [ + {"name": "key", "type": "int", "field-id": 129}, + {"name": "value", "type": "bytes", "field-id": 130}, + ], + }, + "logicalType": "map", + }, + ], + "doc": "Map of column id to upper bound", + "default": "null", + "field-id": 128, + }, + { + "name": "key_metadata", + "type": ["null", "bytes"], + "doc": "Encryption key metadata blob", + "default": "null", + "field-id": 131, + }, + { + "name": "split_offsets", + "type": ["null", {"type": "array", "items": "long", "element-id": 133}], + "doc": "Splittable offsets", + "default": "null", + "field-id": 132, + }, + { + "name": "sort_order_id", + "type": ["null", "int"], + "doc": "Sort order ID", + "default": "null", + "field-id": 140, + }, + ], }, - "field-id": 1000, + "field-id": 2, }, ], } -@pytest.fixture -def catalog() -> InMemoryCatalog: - return InMemoryCatalog("test.in.memory.catalog", {"test.key": "test.value"}) +@pytest.fixture(scope="session") +def generated_manifest_entry_file(avro_schema_manifest_entry): + from fastavro import parse_schema, writer + + parsed_schema = parse_schema(avro_schema_manifest_entry) + + with TemporaryDirectory() as tmpdir: + tmp_avro_file = tmpdir + "/manifest.avro" + with open(tmp_avro_file, "wb") as out: + writer(out, parsed_schema, manifest_entry_records) + yield tmp_avro_file + + +@pytest.fixture(scope="session") +def generated_manifest_file_file(avro_schema_manifest_file): + from fastavro import parse_schema, writer + + parsed_schema = parse_schema(avro_schema_manifest_file) + + with TemporaryDirectory() as tmpdir: + tmp_avro_file = tmpdir + "/manifest.avro" + with open(tmp_avro_file, "wb") as out: + writer(out, parsed_schema, manifest_file_records) + yield tmp_avro_file + + +@pytest.fixture(scope="session") +def iceberg_manifest_entry_schema() -> Schema: + return Schema( + NestedField(field_id=0, name="status", field_type=IntegerType(), required=True), + NestedField(field_id=1, name="snapshot_id", field_type=LongType(), required=False), + NestedField( + field_id=2, + name="data_file", + field_type=StructType( + NestedField( + field_id=100, + name="file_path", + field_type=StringType(), + doc="Location URI with FS scheme", + required=True, + ), + NestedField( + field_id=101, + name="file_format", + field_type=StringType(), + doc="File format name: avro, orc, or parquet", + required=True, + ), + NestedField( + field_id=102, + name="partition", + field_type=StructType( + NestedField( + field_id=1000, + name="VendorID", + field_type=IntegerType(), + required=False, + ), + ), + required=True, + ), + NestedField( + field_id=103, + name="record_count", + field_type=LongType(), + doc="Number of records in the file", + required=True, + ), + NestedField( + field_id=104, + name="file_size_in_bytes", + field_type=LongType(), + doc="Total file size in bytes", + required=True, + ), + NestedField( + field_id=105, + name="block_size_in_bytes", + field_type=LongType(), + required=True, + ), + NestedField( + field_id=108, + name="column_sizes", + field_type=MapType( + key_id=117, + key_type=IntegerType(), + value_id=118, + value_type=LongType(), + value_required=True, + ), + doc="Map of column id to total size on disk", + required=False, + ), + NestedField( + field_id=109, + name="value_counts", + field_type=MapType( + key_id=119, + key_type=IntegerType(), + value_id=120, + value_type=LongType(), + value_required=True, + ), + doc="Map of column id to total count, including null and NaN", + required=False, + ), + NestedField( + field_id=110, + name="null_value_counts", + field_type=MapType( + key_id=121, + key_type=IntegerType(), + value_id=122, + value_type=LongType(), + value_required=True, + ), + doc="Map of column id to null value count", + required=False, + ), + NestedField( + field_id=137, + name="nan_value_counts", + field_type=MapType( + key_id=138, + key_type=IntegerType(), + value_id=139, + value_type=LongType(), + value_required=True, + ), + doc="Map of column id to number of NaN values in the column", + required=False, + ), + NestedField( + field_id=125, + name="lower_bounds", + field_type=MapType( + key_id=126, + key_type=IntegerType(), + value_id=127, + value_type=BinaryType(), + value_required=True, + ), + doc="Map of column id to lower bound", + required=False, + ), + NestedField( + field_id=128, + name="upper_bounds", + field_type=MapType( + key_id=129, + key_type=IntegerType(), + value_id=130, + value_type=BinaryType(), + value_required=True, + ), + doc="Map of column id to upper bound", + required=False, + ), + NestedField( + field_id=131, + name="key_metadata", + field_type=BinaryType(), + doc="Encryption key metadata blob", + required=False, + ), + NestedField( + field_id=132, + name="split_offsets", + field_type=ListType( + element_id=133, + element_type=LongType(), + element_required=True, + ), + doc="Splittable offsets", + required=False, + ), + NestedField( + field_id=140, + name="sort_order_id", + field_type=IntegerType(), + doc="Sort order ID", + required=False, + ), + ), + required=True, + ), + schema_id=1, + identifier_field_ids=[], + ) diff --git a/tests/io/__init__.py b/tests/io/__init__.py index 00eaa2ffe4..13a83393a9 100644 --- a/tests/io/__init__.py +++ b/tests/io/__init__.py @@ -1,11 +1,16 @@ -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at # -# http://www.apache.org/licenses/LICENSE-2.0 +# http://www.apache.org/licenses/LICENSE-2.0 # -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. diff --git a/tests/io/test_io_base.py b/tests/io/test_io_base.py index 72495e19b6..9008c453a2 100644 --- a/tests/io/test_io_base.py +++ b/tests/io/test_io_base.py @@ -124,7 +124,7 @@ def delete(self, location: Union[str, InputFile, OutputFile]) -> None: try: os.remove(parsed_location.path) except FileNotFoundError as e: - raise FileNotFoundError(f"Cannot delete file, does not exist: {parsed_location.path} - Caused by: {e}") + raise FileNotFoundError(f"Cannot delete file, does not exist: {parsed_location.path} - Caused by: {e}") from e @pytest.mark.parametrize("CustomInputFile", [LocalInputFile, PyArrowFile]) diff --git a/tests/utils/test_schema_conversion.py b/tests/utils/test_schema_conversion.py index 8776e1a14f..234ec1573f 100644 --- a/tests/utils/test_schema_conversion.py +++ b/tests/utils/test_schema_conversion.py @@ -14,6 +14,8 @@ # KIND, either express or implied. See the License for the # specific language governing permissions and limitations # under the License. +from typing import Any, Dict + import pytest from iceberg.schema import Schema @@ -34,31 +36,29 @@ from iceberg.utils.schema_conversion import AvroSchemaConversion -def test_iceberg_to_avro(manifest_schema): - iceberg_schema = AvroSchemaConversion().avro_to_iceberg(manifest_schema) +def test_iceberg_to_avro(avro_schema_manifest_file: Dict[str, Any]): + iceberg_schema = AvroSchemaConversion().avro_to_iceberg(avro_schema_manifest_file) expected_iceberg_schema = Schema( NestedField( - field_id=500, name="manifest_path", field_type=StringType(), required=False, doc="Location URI with FS scheme" - ), - NestedField(field_id=501, name="manifest_length", field_type=LongType(), required=False, doc="Total file size in bytes"), - NestedField( - field_id=502, name="partition_spec_id", field_type=IntegerType(), required=False, doc="Spec ID used to write" + field_id=500, name="manifest_path", field_type=StringType(), required=True, doc="Location URI with FS scheme" ), + NestedField(field_id=501, name="manifest_length", field_type=LongType(), required=True, doc="Total file size in bytes"), + NestedField(field_id=502, name="partition_spec_id", field_type=IntegerType(), required=True, doc="Spec ID used to write"), NestedField( field_id=503, name="added_snapshot_id", field_type=LongType(), - required=True, + required=False, doc="Snapshot ID that added the manifest", ), NestedField( - field_id=504, name="added_data_files_count", field_type=IntegerType(), required=True, doc="Added entry count" + field_id=504, name="added_data_files_count", field_type=IntegerType(), required=False, doc="Added entry count" ), NestedField( - field_id=505, name="existing_data_files_count", field_type=IntegerType(), required=True, doc="Existing entry count" + field_id=505, name="existing_data_files_count", field_type=IntegerType(), required=False, doc="Existing entry count" ), NestedField( - field_id=506, name="deleted_data_files_count", field_type=IntegerType(), required=True, doc="Deleted entry count" + field_id=506, name="deleted_data_files_count", field_type=IntegerType(), required=False, doc="Deleted entry count" ), NestedField( field_id=507, @@ -66,45 +66,43 @@ def test_iceberg_to_avro(manifest_schema): field_type=ListType( element_id=508, element_type=StructType( - fields=( - NestedField( - field_id=509, - name="contains_null", - field_type=BooleanType(), - required=False, - doc="True if any file has a null partition value", - ), - NestedField( - field_id=518, - name="contains_nan", - field_type=BooleanType(), - required=True, - doc="True if any file has a nan partition value", - ), - NestedField( - field_id=510, - name="lower_bound", - field_type=BinaryType(), - required=True, - doc="Partition lower bound for all files", - ), - NestedField( - field_id=511, - name="upper_bound", - field_type=BinaryType(), - required=True, - doc="Partition upper bound for all files", - ), - ) + NestedField( + field_id=509, + name="contains_null", + field_type=BooleanType(), + required=True, + doc="True if any file has a null partition value", + ), + NestedField( + field_id=518, + name="contains_nan", + field_type=BooleanType(), + required=False, + doc="True if any file has a nan partition value", + ), + NestedField( + field_id=510, + name="lower_bound", + field_type=BinaryType(), + required=False, + doc="Partition lower bound for all files", + ), + NestedField( + field_id=511, + name="upper_bound", + field_type=BinaryType(), + required=False, + doc="Partition upper bound for all files", + ), ), - element_required=False, + element_required=True, ), - required=True, + required=False, doc="Summary for each partition", ), - NestedField(field_id=512, name="added_rows_count", field_type=LongType(), required=True, doc="Added rows count"), - NestedField(field_id=513, name="existing_rows_count", field_type=LongType(), required=True, doc="Existing rows count"), - NestedField(field_id=514, name="deleted_rows_count", field_type=LongType(), required=True, doc="Deleted rows count"), + NestedField(field_id=512, name="added_rows_count", field_type=LongType(), required=False, doc="Added rows count"), + NestedField(field_id=513, name="existing_rows_count", field_type=LongType(), required=False, doc="Existing rows count"), + NestedField(field_id=514, name="deleted_rows_count", field_type=LongType(), required=False, doc="Deleted rows count"), schema_id=1, identifier_field_ids=[], ) @@ -133,8 +131,8 @@ def test_avro_list_required_primitive(): NestedField( field_id=100, name="array_with_string", - field_type=ListType(element_id=101, element_type=StringType(), element_required=False), - required=False, + field_type=ListType(element_id=101, element_type=StringType(), element_required=True), + required=True, ), schema_id=1, ) @@ -166,8 +164,8 @@ def test_avro_list_wrapped_primitive(): NestedField( field_id=100, name="array_with_string", - field_type=ListType(element_id=101, element_type=StringType(), element_required=False), - required=False, + field_type=ListType(element_id=101, element_type=StringType(), element_required=True), + required=True, ), schema_id=1, ) @@ -217,13 +215,13 @@ def test_avro_list_required_record(): element_id=101, element_type=StructType( fields=( - NestedField(field_id=102, name="contains_null", field_type=BooleanType(), required=False), - NestedField(field_id=103, name="contains_nan", field_type=BooleanType(), required=True), + NestedField(field_id=102, name="contains_null", field_type=BooleanType(), required=True), + NestedField(field_id=103, name="contains_nan", field_type=BooleanType(), required=False), ) ), - element_required=False, + element_required=True, ), - required=False, + required=True, ), schema_id=1, identifier_field_ids=[], @@ -249,12 +247,12 @@ def test_nested_type(): def test_map_type(): avro_type = { "type": "map", - "values": ["long", "null"], + "values": ["null", "long"], "key-id": 101, "value-id": 102, } actual = AvroSchemaConversion()._convert_schema(avro_type) - expected = MapType(key_id=101, key_type=StringType(), value_id=102, value_type=LongType(), value_required=True) + expected = MapType(key_id=101, key_type=StringType(), value_id=102, value_type=LongType(), value_required=False) assert actual == expected diff --git a/tests/utils/test_singleton.py b/tests/utils/test_singleton.py new file mode 100644 index 0000000000..92b923dd82 --- /dev/null +++ b/tests/utils/test_singleton.py @@ -0,0 +1,24 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +from iceberg.avro.reader import BooleanReader, FixedReader + + +def test_singleton(): + """We want to reuse the readers to avoid creating a gazillion of them""" + assert id(BooleanReader()) == id(BooleanReader()) + assert id(FixedReader(22)) == id(FixedReader(22)) + assert id(FixedReader(19)) != id(FixedReader(25)) From 1f3a65d938d8b8360492e40993018a8934018944 Mon Sep 17 00:00:00 2001 From: Fokko Driesprong Date: Tue, 21 Jun 2022 00:39:26 +0200 Subject: [PATCH 117/642] Python: Bump pre-commit versions (#5074) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ``` ➜ python git:(fd-bump-pre-commit) pre-commit autoupdate Updating https://github.com/pre-commit/pre-commit-hooks ... updating v4.2.0 -> v4.3.0. Updating https://github.com/ambv/black ... already up to date. Updating https://github.com/pre-commit/mirrors-isort ... already up to date. Updating https://github.com/pre-commit/mirrors-mypy ... updating v0.960 -> v0.961. Updating https://github.com/hadialqattan/pycln ... already up to date. Updating https://github.com/asottile/pyupgrade ... updating v2.32.1 -> v2.34.0. ➜ python git:(fd-bump-pre-commit) ✗ pre-commit run --all-files trim trailing whitespace.................................................Passed fix end of files.........................................................Passed check docstring is first.................................................Passed debug statements (python)................................................Passed check yaml...............................................................Passed check python ast.........................................................Passed black....................................................................Passed isort....................................................................Passed mypy.....................................................................Passed pycln....................................................................Passed pyupgrade................................................................Passed ``` --- .pre-commit-config.yaml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index cf56bc3162..e2e25c84bf 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -19,7 +19,7 @@ files: ^python/ repos: - repo: https://github.com/pre-commit/pre-commit-hooks - rev: v4.2.0 + rev: v4.3.0 hooks: - id: trailing-whitespace - id: end-of-file-fixer @@ -37,7 +37,7 @@ repos: - id: isort args: [ --settings-path=python/pyproject.toml ] - repo: https://github.com/pre-commit/mirrors-mypy - rev: v0.960 + rev: v0.961 hooks: - id: mypy - repo: https://github.com/hadialqattan/pycln @@ -46,7 +46,7 @@ repos: - id: pycln args: [--config=python/pyproject.toml] - repo: https://github.com/asottile/pyupgrade - rev: v2.32.1 + rev: v2.34.0 hooks: - id: pyupgrade args: [--py38-plus] From b264b74eb5ddf35b9c2b19ce22184cdd197a09a9 Mon Sep 17 00:00:00 2001 From: Fokko Driesprong Date: Tue, 21 Jun 2022 17:18:40 +0200 Subject: [PATCH 118/642] Python: Replace vars with cached_property decorator (#5068) --- src/iceberg/expressions/base.py | 7 ++++- src/iceberg/schema.py | 45 ++++++++++++++------------------- 2 files changed, 25 insertions(+), 27 deletions(-) diff --git a/src/iceberg/expressions/base.py b/src/iceberg/expressions/base.py index 919cc09b86..3e4910be8a 100644 --- a/src/iceberg/expressions/base.py +++ b/src/iceberg/expressions/base.py @@ -346,7 +346,12 @@ def bind(self, schema: Schema, case_sensitive: bool) -> BoundReference: if not field: raise ValueError(f"Cannot find field '{self.name}' in schema: {schema}") - return BoundReference(field=field, accessor=schema.accessor_for_field(field.field_id)) + accessor = schema.accessor_for_field(field.field_id) + + if not accessor: + raise ValueError(f"Cannot find accessor for field '{self.name}' in schema: {schema}") + + return BoundReference(field=field, accessor=accessor) class BooleanExpressionVisitor(Generic[T], ABC): diff --git a/src/iceberg/schema.py b/src/iceberg/schema.py index 97b9801500..99ef37c4b9 100644 --- a/src/iceberg/schema.py +++ b/src/iceberg/schema.py @@ -20,7 +20,7 @@ from abc import ABC, abstractmethod from dataclasses import dataclass -from functools import singledispatch +from functools import cached_property, singledispatch from typing import ( Any, Dict, @@ -54,10 +54,6 @@ def __init__(self, *columns: NestedField, schema_id: int, identifier_field_ids: self._schema_id = schema_id self._identifier_field_ids = identifier_field_ids or [] self._name_to_id: dict[str, int] = index_by_name(self) - self._name_to_id_lower: dict[str, int] = {} # Should be accessed through self._lazy_name_to_id_lower() - self._id_to_field: dict[int, NestedField] = {} # Should be accessed through self._lazy_id_to_field() - self._id_to_name: dict[int, str] = {} # Should be accessed through self._lazy_id_to_name() - self._id_to_accessor: dict[int, Accessor] = {} # Should be accessed through self._lazy_id_to_accessor() def __str__(self): return "table {\n" + "\n".join([" " + str(field) for field in self.columns]) + "\n}" @@ -96,47 +92,43 @@ def schema_id(self) -> int: def identifier_field_ids(self) -> list[int]: return self._identifier_field_ids + @cached_property def _lazy_id_to_field(self) -> dict[int, NestedField]: """Returns an index of field ID to NestedField instance This is calculated once when called for the first time. Subsequent calls to this method will use a cached index. """ - if not self._id_to_field: - self._id_to_field = index_by_id(self) - return self._id_to_field + return index_by_id(self) + @cached_property def _lazy_name_to_id_lower(self) -> dict[str, int]: """Returns an index of lower-case field names to field IDs This is calculated once when called for the first time. Subsequent calls to this method will use a cached index. """ - if not self._name_to_id_lower: - self._name_to_id_lower = {name.lower(): field_id for name, field_id in self._name_to_id.items()} - return self._name_to_id_lower + return {name.lower(): field_id for name, field_id in self._name_to_id.items()} + @cached_property def _lazy_id_to_name(self) -> dict[int, str]: """Returns an index of field ID to full name This is calculated once when called for the first time. Subsequent calls to this method will use a cached index. """ - if not self._id_to_name: - self._id_to_name = index_name_by_id(self) - return self._id_to_name + return index_name_by_id(self) + @cached_property def _lazy_id_to_accessor(self) -> dict[int, Accessor]: """Returns an index of field ID to accessor This is calculated once when called for the first time. Subsequent calls to this method will use a cached index. """ - if not self._id_to_accessor: - self._id_to_accessor = build_position_accessors(self) - return self._id_to_accessor + return build_position_accessors(self) def as_struct(self) -> StructType: """Returns the underlying struct""" return self._struct - def find_field(self, name_or_id: str | int, case_sensitive: bool = True) -> NestedField: + def find_field(self, name_or_id: str | int, case_sensitive: bool = True) -> NestedField | None: """Find a field using a field name or field ID Args: @@ -147,13 +139,12 @@ def find_field(self, name_or_id: str | int, case_sensitive: bool = True) -> Nest NestedField: The matched NestedField """ if isinstance(name_or_id, int): - field = self._lazy_id_to_field().get(name_or_id) - return field # type: ignore + return self._lazy_id_to_field.get(name_or_id) if case_sensitive: field_id = self._name_to_id.get(name_or_id) else: - field_id = self._lazy_name_to_id_lower().get(name_or_id.lower()) - return self._lazy_id_to_field().get(field_id) # type: ignore + field_id = self._lazy_name_to_id_lower.get(name_or_id.lower()) + return self._lazy_id_to_field.get(field_id) # type: ignore def find_type(self, name_or_id: str | int, case_sensitive: bool = True) -> IcebergType: """Find a field type using a field name or field ID @@ -166,9 +157,11 @@ def find_type(self, name_or_id: str | int, case_sensitive: bool = True) -> Icebe NestedField: The type of the matched NestedField """ field = self.find_field(name_or_id=name_or_id, case_sensitive=case_sensitive) + if not field: + raise ValueError(f"Could not find field with name or id {name_or_id}, case_sensitive={case_sensitive}") return field.field_type - def find_column_name(self, column_id: int) -> str: + def find_column_name(self, column_id: int) -> str | None: """Find a column name given a column ID Args: @@ -177,9 +170,9 @@ def find_column_name(self, column_id: int) -> str: Returns: str: The column name (or None if the column ID cannot be found) """ - return self._lazy_id_to_name().get(column_id) # type: ignore + return self._lazy_id_to_name.get(column_id) - def accessor_for_field(self, field_id: int) -> Accessor: + def accessor_for_field(self, field_id: int) -> Accessor | None: """Find a schema position accessor given a field ID Args: @@ -188,7 +181,7 @@ def accessor_for_field(self, field_id: int) -> Accessor: Returns: Accessor: An accessor for the given field ID """ - return self._lazy_id_to_accessor().get(field_id) # type: ignore + return self._lazy_id_to_accessor.get(field_id) def select(self, names: list[str], case_sensitive: bool = True) -> Schema: """Return a new schema instance pruned to a subset of columns From 9ecc030534f1586b4c29881400cbbdea106a64c7 Mon Sep 17 00:00:00 2001 From: Fokko Driesprong Date: Tue, 21 Jun 2022 19:55:07 +0200 Subject: [PATCH 119/642] Python: Renable pylint (#5066) * Python: Renable pylint For some reason pylint fell through the cracks when migrating from tox to pre-commit * Be more explicit in disabled checks --- .pre-commit-config.yaml | 5 +++ poetry.lock | 6 +-- pylintrc | 46 +------------------ pyproject.toml | 1 - spellcheck-dictionary.txt | 65 --------------------------- src/iceberg/avro/decoder.py | 2 +- src/iceberg/avro/reader.py | 15 ++++--- src/iceberg/transforms.py | 10 ++--- tests/avro/test_reader.py | 1 + tests/conftest.py | 1 + tests/io/test_io_base.py | 2 +- tests/io/test_pyarrow.py | 1 + tests/utils/test_schema_conversion.py | 1 + 13 files changed, 28 insertions(+), 128 deletions(-) delete mode 100644 spellcheck-dictionary.txt diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index e2e25c84bf..8160c428e1 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -50,6 +50,11 @@ repos: hooks: - id: pyupgrade args: [--py38-plus] + - repo: https://github.com/pycqa/pylint + rev: v2.14.2 + hooks: + - id: pylint + args: [ --rcfile=python/pylintrc ] - repo: https://github.com/pycqa/flake8 rev: '4.0.1' hooks: diff --git a/poetry.lock b/poetry.lock index 91dfbc6914..bc47c6e866 100644 --- a/poetry.lock +++ b/poetry.lock @@ -41,7 +41,7 @@ python-versions = ">=3.6.1" [[package]] name = "colorama" -version = "0.4.4" +version = "0.4.5" description = "Cross-platform colored terminal text." category = "dev" optional = false @@ -460,8 +460,8 @@ cfgv = [ {file = "cfgv-3.3.1.tar.gz", hash = "sha256:f5a830efb9ce7a445376bb66ec94c638a9787422f96264c98edc6bdeed8ab736"}, ] colorama = [ - {file = "colorama-0.4.4-py2.py3-none-any.whl", hash = "sha256:9f47eda37229f68eee03b24b9748937c7dc3868f906e8ba69fbcbdd3bc5dc3e2"}, - {file = "colorama-0.4.4.tar.gz", hash = "sha256:5941b2b48a20143d2267e95b1c2a7603ce057ee39fd88e7329b0c292aa16869b"}, + {file = "colorama-0.4.5-py2.py3-none-any.whl", hash = "sha256:854bf444933e37f5824ae7bfc1e98d5bce2ebe4160d46b5edf346a89358e99da"}, + {file = "colorama-0.4.5.tar.gz", hash = "sha256:e6c6b4334fc50988a639d9b98aa429a0b57da6e17b9a44f0451f930b6967b7a4"}, ] coverage = [ {file = "coverage-6.4.1-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:f1d5aa2703e1dab4ae6cf416eb0095304f49d004c39e9db1d86f57924f43006b"}, diff --git a/pylintrc b/pylintrc index b9aa99a6ff..9835535209 100644 --- a/pylintrc +++ b/pylintrc @@ -101,7 +101,7 @@ disable=all # either give multiple identifier separated by comma (,) or put this option # multiple time (only on the command line, not in the configuration file where # it should appear only once). See also the "--disable" option for examples. -enable=spelling,W +enable=W [REPORTS] @@ -151,50 +151,6 @@ logging-format-style=old # function parameter format. logging-modules=logging - -[SPELLING] - -# Limits count of emitted suggestions for spelling mistakes. -max-spelling-suggestions=4 - -# Spelling dictionary name. Available dictionaries: af (aspell), am (aspell), -# ar (aspell), ast (aspell), az (aspell), be (aspell), be_BY (aspell), be_SU -# (aspell), bg (aspell), bn (aspell), br (aspell), ca (aspell), cs (aspell), -# csb (aspell), cy (aspell), da (aspell), de (aspell), de_AT (aspell), de_CH -# (aspell), de_DE (aspell), el (aspell), en (aspell), en_AU (aspell), en_CA -# (aspell), en_GB (aspell), en_US (aspell), eo (aspell), es (aspell), es_ES -# (AppleSpell), et (aspell), fa (aspell), fi (aspell), fo (aspell), fr -# (aspell), fr_CH (aspell), fr_FR (aspell), fy (aspell), ga (aspell), gd -# (aspell), gl (aspell), gr (aspell), grc (aspell), gu (aspell), gv (aspell), -# he (aspell), hi (aspell), hil (aspell), hr (aspell), hsb (aspell), hu -# (aspell), hu_HU (AppleSpell), hus (aspell), hy (aspell), ia (aspell), id -# (aspell), it (aspell), it_IT (AppleSpell), kn (aspell), ku (aspell), ky -# (aspell), la (aspell), lt (aspell), lv (aspell), mg (aspell), mi (aspell), mk -# (aspell), ml (aspell), mn (aspell), mr (aspell), ms (aspell), mt (aspell), -# nds (aspell), nl (aspell), nl_NL (AppleSpell), nn (aspell), ny (aspell), or -# (aspell), pa (aspell), pl (aspell), pt_BR (aspell), pt_PT (aspell), qu -# (aspell), ro (aspell), ru (aspell), rw (aspell), sc (aspell), sk (aspell), -# sk_SK (aspell), sl (aspell), sr (aspell), srd (aspell), sv (aspell), sv_SE -# (AppleSpell), sw (aspell), ta (aspell), te (aspell), tet (aspell), tk -# (aspell), tl (aspell), tn (aspell), tr (aspell), uk (aspell), uz (aspell), vi -# (aspell), wa (aspell), yi (aspell), zu (aspell). -spelling-dict=en_US - -# List of comma separated words that should be considered directives if they -# appear and the beginning of a comment and should not be checked. -spelling-ignore-comment-directives=fmt: on,fmt: off,noqa:,noqa,nosec,isort:skip,mypy: - -# List of comma separated words that should not be checked. -spelling-ignore-words= - -# A path to a file that contains the private dictionary; one word per line. -spelling-private-dict-file=spellcheck-dictionary.txt - -# Tells whether to store unknown words to the private dictionary (see the -# --spelling-private-dict-file option) instead of raising a message. -spelling-store-unknown-words=no - - [MISCELLANEOUS] # List of note tags to take in consideration, separated by a comma. diff --git a/pyproject.toml b/pyproject.toml index afeee351ce..5d7b75f868 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -40,7 +40,6 @@ packages = [ [tool.poetry.dependencies] python = "^3.8" mmh3 = "^3.0.0" - pyarrow = { version = "^8.0.0", optional = true } zstandard = { version = "^0.17.0", optional = true } diff --git a/spellcheck-dictionary.txt b/spellcheck-dictionary.txt deleted file mode 100644 index 8bb0b868c3..0000000000 --- a/spellcheck-dictionary.txt +++ /dev/null @@ -1,65 +0,0 @@ -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -accessor -accessors -Args -Avro -Nestedfield -ASF -BD -bool -boolean -datetime -disjunction -dispatchable -endian -enum -FileInfo -filesystem -fs -func -IcebergType -io -namespace -namespaces -NativeFile -NaN -nan -NestedField -nullability -PartitionField -pragma -PrimitiveType -pyarrow -repr -schemas -seekable -singledispatch -str -struct -StructProtocol -StructType -Timestamptz -Timestamptzs -unscaled -URI -UnboundPredicate -BoundPredicate -BooleanExpression -BooleanExpressionVisitor -zigzag -unix -zlib -Codecs -codecs -uri diff --git a/src/iceberg/avro/decoder.py b/src/iceberg/avro/decoder.py index 24312cdd25..586aabf978 100644 --- a/src/iceberg/avro/decoder.py +++ b/src/iceberg/avro/decoder.py @@ -103,7 +103,7 @@ def read_decimal_from_bytes(self, precision: int, scale: int) -> decimal.Decimal size = self.read_long() return self.read_decimal_from_fixed(precision, scale, size) - def read_decimal_from_fixed(self, precision: int, scale: int, size: int) -> decimal.Decimal: + def read_decimal_from_fixed(self, _: int, scale: int, size: int) -> decimal.Decimal: """ Decimal is encoded as fixed. Fixed instances are encoded using the number of bytes declared in the schema. diff --git a/src/iceberg/avro/reader.py b/src/iceberg/avro/reader.py index bd8f1b4e80..012d611a1a 100644 --- a/src/iceberg/avro/reader.py +++ b/src/iceberg/avro/reader.py @@ -26,7 +26,8 @@ from __future__ import annotations from abc import abstractmethod -from dataclasses import dataclass, field +from dataclasses import dataclass +from dataclasses import field as dataclassfield from datetime import date, datetime, time from decimal import Decimal from functools import singledispatch @@ -61,7 +62,7 @@ @dataclass(frozen=True) class AvroStruct(StructProtocol): - _data: list[Any | StructProtocol] = field() + _data: list[Any | StructProtocol] = dataclassfield() def set(self, pos: int, value: Any) -> None: self._data[pos] = value @@ -138,7 +139,7 @@ def read(self, decoder: BinaryDecoder) -> UUID: @dataclass(frozen=True) class FixedReader(Reader): - length: int = field() + length: int = dataclassfield() def read(self, decoder: BinaryDecoder) -> bytes: return decoder.read(self.length) @@ -151,8 +152,8 @@ def read(self, decoder: BinaryDecoder) -> bytes: @dataclass(frozen=True) class DecimalReader(Reader): - precision: int = field() - scale: int = field() + precision: int = dataclassfield() + scale: int = dataclassfield() def read(self, decoder: BinaryDecoder) -> Decimal: return decoder.read_decimal_from_bytes(self.precision, self.scale) @@ -160,7 +161,7 @@ def read(self, decoder: BinaryDecoder) -> Decimal: @dataclass(frozen=True) class OptionReader(Reader): - option: Reader = field() + option: Reader = dataclassfield() def read(self, decoder: BinaryDecoder) -> Any | None: # For the Iceberg spec it is required to set the default value to null @@ -179,7 +180,7 @@ def read(self, decoder: BinaryDecoder) -> Any | None: @dataclass(frozen=True) class StructReader(Reader): - fields: tuple[Reader, ...] = field() + fields: tuple[Reader, ...] = dataclassfield() def read(self, decoder: BinaryDecoder) -> AvroStruct: return AvroStruct([field.read(decoder) for field in self.fields]) diff --git a/src/iceberg/transforms.py b/src/iceberg/transforms.py index fbbdd917aa..0947cf56b5 100644 --- a/src/iceberg/transforms.py +++ b/src/iceberg/transforms.py @@ -281,23 +281,23 @@ def _(self, value: int) -> str: return self._int_to_human_string(self._type, value) @singledispatchmethod - def _int_to_human_string(self, value_type: IcebergType, value: int) -> str: + def _int_to_human_string(self, _: IcebergType, value: int) -> str: return str(value) @_int_to_human_string.register(DateType) - def _(self, value_type: IcebergType, value: int) -> str: + def _(self, _: IcebergType, value: int) -> str: return datetime.to_human_day(value) @_int_to_human_string.register(TimeType) - def _(self, value_type: IcebergType, value: int) -> str: + def _(self, _: IcebergType, value: int) -> str: return datetime.to_human_time(value) @_int_to_human_string.register(TimestampType) - def _(self, value_type: IcebergType, value: int) -> str: + def _(self, _: IcebergType, value: int) -> str: return datetime.to_human_timestamp(value) @_int_to_human_string.register(TimestamptzType) - def _(self, value_type: IcebergType, value: int) -> str: + def _(self, _: IcebergType, value: int) -> str: return datetime.to_human_timestamptz(value) diff --git a/tests/avro/test_reader.py b/tests/avro/test_reader.py index c310e69204..e8c56f122e 100644 --- a/tests/avro/test_reader.py +++ b/tests/avro/test_reader.py @@ -14,6 +14,7 @@ # KIND, either express or implied. See the License for the # specific language governing permissions and limitations # under the License. +# pylint:disable=protected-access import json import pytest diff --git a/tests/conftest.py b/tests/conftest.py index 48f3bf0239..782316ec91 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -14,6 +14,7 @@ # KIND, either express or implied. See the License for the # specific language governing permissions and limitations # under the License. +# pylint:disable=redefined-outer-name from tempfile import TemporaryDirectory from typing import Any, Dict diff --git a/tests/io/test_io_base.py b/tests/io/test_io_base.py index 9008c453a2..a4ffe7dff9 100644 --- a/tests/io/test_io_base.py +++ b/tests/io/test_io_base.py @@ -124,7 +124,7 @@ def delete(self, location: Union[str, InputFile, OutputFile]) -> None: try: os.remove(parsed_location.path) except FileNotFoundError as e: - raise FileNotFoundError(f"Cannot delete file, does not exist: {parsed_location.path} - Caused by: {e}") from e + raise FileNotFoundError(f"Cannot delete file, does not exist: {parsed_location.path}") from e @pytest.mark.parametrize("CustomInputFile", [LocalInputFile, PyArrowFile]) diff --git a/tests/io/test_pyarrow.py b/tests/io/test_pyarrow.py index 79e88f3e24..f3a41bb54d 100644 --- a/tests/io/test_pyarrow.py +++ b/tests/io/test_pyarrow.py @@ -14,6 +14,7 @@ # KIND, either express or implied. See the License for the # specific language governing permissions and limitations # under the License. +# pylint: disable=protected-access,unused-argument import os import tempfile diff --git a/tests/utils/test_schema_conversion.py b/tests/utils/test_schema_conversion.py index 234ec1573f..e2eada5c40 100644 --- a/tests/utils/test_schema_conversion.py +++ b/tests/utils/test_schema_conversion.py @@ -14,6 +14,7 @@ # KIND, either express or implied. See the License for the # specific language governing permissions and limitations # under the License. +# pylint: disable=W0212 from typing import Any, Dict import pytest From 7cd798210072561991a4ee8d007c5ee23cb7c376 Mon Sep 17 00:00:00 2001 From: Fokko Driesprong Date: Thu, 23 Jun 2022 17:58:02 +0200 Subject: [PATCH 120/642] Python: Add flake8 bugbear (#5117) --- .pre-commit-config.yaml | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 8160c428e1..df929b62ce 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -41,7 +41,7 @@ repos: hooks: - id: mypy - repo: https://github.com/hadialqattan/pycln - rev: v1.3.3 + rev: v1.3.4 hooks: - id: pycln args: [--config=python/pyproject.toml] @@ -51,7 +51,7 @@ repos: - id: pyupgrade args: [--py38-plus] - repo: https://github.com/pycqa/pylint - rev: v2.14.2 + rev: v2.14.3 hooks: - id: pylint args: [ --rcfile=python/pylintrc ] @@ -60,3 +60,4 @@ repos: hooks: - id: flake8 args: [ "--ignore=E501,W503,E203" ] + additional_dependencies: [ flake8-bugbear==22.4.25 ] From 029d0ed8483679f2d8cf7ed453ab746359533d6f Mon Sep 17 00:00:00 2001 From: Fokko Driesprong Date: Sat, 25 Jun 2022 01:20:11 +0200 Subject: [PATCH 121/642] Python: Add flake8-comprehensions (#5130) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit It’s slower to call e.g. `dict()` than using the empty literal `{}`, because the name dict must be looked up in the global scope in case it has been rebound. --- .pre-commit-config.yaml | 2 +- src/iceberg/table/partitioning.py | 2 +- src/iceberg/utils/bin_packing.py | 4 ++-- tests/table/test_partitioning.py | 2 +- tests/utils/test_bin_packing.py | 2 +- 5 files changed, 6 insertions(+), 6 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index df929b62ce..235e44f2db 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -60,4 +60,4 @@ repos: hooks: - id: flake8 args: [ "--ignore=E501,W503,E203" ] - additional_dependencies: [ flake8-bugbear==22.4.25 ] + additional_dependencies: [ flake8-bugbear==22.4.25, flake8-comprehensions==3.10.0 ] diff --git a/src/iceberg/table/partitioning.py b/src/iceberg/table/partitioning.py index 36e59e8e1b..949045b986 100644 --- a/src/iceberg/table/partitioning.py +++ b/src/iceberg/table/partitioning.py @@ -64,7 +64,7 @@ class PartitionSpec: source_id_to_fields_map: Dict[int, List[PartitionField]] = field(init=False, repr=False) def __post_init__(self): - source_id_to_fields_map = dict() + source_id_to_fields_map = {} for partition_field in self.fields: source_column = self.schema.find_column_name(partition_field.source_id) if not source_column: diff --git a/src/iceberg/utils/bin_packing.py b/src/iceberg/utils/bin_packing.py index e3577f2198..ad27089686 100644 --- a/src/iceberg/utils/bin_packing.py +++ b/src/iceberg/utils/bin_packing.py @@ -23,7 +23,7 @@ def __init__(self, items, target_weight, lookback, weight_func, largest_bin_firs self.lookback = lookback self.weight_func = weight_func self.largest_bin_first = largest_bin_first - self.bins = list() + self.bins = [] def __iter__(self): return self @@ -69,7 +69,7 @@ class Bin: def __init__(self, target_weight: int): self.bin_weight = 0 self.target_weight = target_weight - self.items: list = list() + self.items: list = [] def weight(self) -> int: return self.bin_weight diff --git a/tests/table/test_partitioning.py b/tests/table/test_partitioning.py index 14107ffdd0..ff87f5b0d2 100644 --- a/tests/table/test_partitioning.py +++ b/tests/table/test_partitioning.py @@ -66,7 +66,7 @@ def test_partition_compatible_with(table_schema_simple: Schema): def test_unpartitioned(table_schema_simple: Schema): - unpartitioned = PartitionSpec(table_schema_simple, 1, tuple(), 1000) + unpartitioned = PartitionSpec(table_schema_simple, 1, (), 1000) assert not unpartitioned.fields assert unpartitioned.is_unpartitioned() diff --git a/tests/utils/test_bin_packing.py b/tests/utils/test_bin_packing.py index fba8d32983..cf96023a13 100644 --- a/tests/utils/test_bin_packing.py +++ b/tests/utils/test_bin_packing.py @@ -80,4 +80,4 @@ def test_bin_packing_lookback(splits, target_weight, lookback, largest_bin_first def weight_func(x): return x - assert [item for item in PackingIterator(splits, target_weight, lookback, weight_func, largest_bin_first)] == expected_lists + assert list(PackingIterator(splits, target_weight, lookback, weight_func, largest_bin_first)) == expected_lists From 9427dc39968c59a3ef75483f06cb282e51dea850 Mon Sep 17 00:00:00 2001 From: Fokko Driesprong Date: Mon, 27 Jun 2022 23:53:35 +0200 Subject: [PATCH 122/642] Python: Update dependencies (#5142) --- poetry.lock | 100 +++++++++++++++++++++++++++------------------------- 1 file changed, 52 insertions(+), 48 deletions(-) diff --git a/poetry.lock b/poetry.lock index bc47c6e866..572d84931d 100644 --- a/poetry.lock +++ b/poetry.lock @@ -79,7 +79,7 @@ python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*" [[package]] name = "fastavro" -version = "1.5.1" +version = "1.5.2" description = "Fast read/write of AVRO files" category = "dev" optional = false @@ -116,7 +116,7 @@ license = ["ukkonen"] [[package]] name = "importlib-metadata" -version = "4.11.4" +version = "4.12.0" description = "Read metadata from Python packages" category = "dev" optional = false @@ -128,7 +128,7 @@ zipp = ">=0.5" [package.extras] docs = ["sphinx", "jaraco.packaging (>=9)", "rst.linker (>=1.9)"] perf = ["ipython"] -testing = ["pytest (>=6)", "pytest-checkdocs (>=2.4)", "pytest-flake8", "pytest-cov", "pytest-enabler (>=1.0.1)", "packaging", "pyfakefs", "flufl.flake8", "pytest-perf (>=0.9.2)", "pytest-black (>=0.3.7)", "pytest-mypy (>=0.9.1)", "importlib-resources (>=1.3)"] +testing = ["pytest (>=6)", "pytest-checkdocs (>=2.4)", "pytest-flake8", "pytest-cov", "pytest-enabler (>=1.3)", "packaging", "pyfakefs", "flufl.flake8", "pytest-perf (>=0.9.2)", "pytest-black (>=0.3.7)", "pytest-mypy (>=0.9.1)", "importlib-resources (>=1.3)"] [[package]] name = "iniconfig" @@ -148,15 +148,15 @@ python-versions = "*" [[package]] name = "nodeenv" -version = "1.6.0" +version = "1.7.0" description = "Node.js virtual environment builder" category = "dev" optional = false -python-versions = "*" +python-versions = ">=2.7,!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,!=3.4.*,!=3.5.*,!=3.6.*" [[package]] name = "numpy" -version = "1.22.4" +version = "1.23.0" description = "NumPy is the fundamental package for array computing with Python." category = "main" optional = true @@ -342,7 +342,7 @@ python-versions = ">=3.7" [[package]] name = "virtualenv" -version = "20.14.1" +version = "20.15.0" description = "Virtual Python Environment builder" category = "dev" optional = false @@ -515,19 +515,23 @@ docutils = [ {file = "docutils-0.18.1.tar.gz", hash = "sha256:679987caf361a7539d76e584cbeddc311e3aee937877c87346f31debc63e9d06"}, ] fastavro = [ - {file = "fastavro-1.5.1-cp310-cp310-macosx_10_14_x86_64.whl", hash = "sha256:920d170560198741fa196a62a97c220173339766e6c14369c5c68bfe8cdafa25"}, - {file = "fastavro-1.5.1-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:0b00b1711511981c4e2dd4a27ba5ae20897fe41ec7ab52eda868626d445081e5"}, - {file = "fastavro-1.5.1-cp310-cp310-win_amd64.whl", hash = "sha256:04438b592980633ccf5d1de7798480a634ca581ae7575ab7671ba16773b6b428"}, - {file = "fastavro-1.5.1-cp37-cp37m-macosx_10_15_x86_64.whl", hash = "sha256:0ab92ab744f9172da0791bfad0495d785c7c4f5a68924e3c6c6b39b78b044b11"}, - {file = "fastavro-1.5.1-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:84ca1a60cecd710ead076585b56b954ab3e6e001d8e7384cb4ed20019b29e7a9"}, - {file = "fastavro-1.5.1-cp37-cp37m-win_amd64.whl", hash = "sha256:b5ff657c0d48553492d8356a30b6112fcc6db69adce6bba31135272bc9d87d82"}, - {file = "fastavro-1.5.1-cp38-cp38-macosx_10_15_x86_64.whl", hash = "sha256:0f1ed38042a2a90a7a5da170006459e73134f4c14f4fda9ebba99017adb1b14c"}, - {file = "fastavro-1.5.1-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:df83ebdd7b67b52a37bc84c6e25f7056f756fb216c5c8e5c95ae1673fcbb6015"}, - {file = "fastavro-1.5.1-cp38-cp38-win_amd64.whl", hash = "sha256:0053347a92add6f448837ff00099b0a7200ec5dd58e173743d856d65d0574ddb"}, - {file = "fastavro-1.5.1-cp39-cp39-macosx_10_15_x86_64.whl", hash = "sha256:6b4f8551ccbe0c9b19867b8c93029e8cfe8fa3757245caae6228f35ef0656371"}, - {file = "fastavro-1.5.1-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:ff241b5ce36df7af7461d001ca98fec6eacd56c4754c8ac7718e2d4b7b690a82"}, - {file = "fastavro-1.5.1-cp39-cp39-win_amd64.whl", hash = "sha256:fb3491c88e7962a6b820548ddd12b9c0f6296ebd2385a3021296f14bfe35189a"}, - {file = "fastavro-1.5.1.tar.gz", hash = "sha256:0815da740ced2261f90b0ddbb5bbe645e9c893c8f00e5dc8d30b8ec20f3c7fa9"}, + {file = "fastavro-1.5.2-cp310-cp310-macosx_10_14_x86_64.whl", hash = "sha256:4c447c81bf9002aa0258bb8e208698744b4e22362548d5423f718c92c22ce572"}, + {file = "fastavro-1.5.2-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:5de3473c759c884f88c0e4e078538b9199416183bd4f49a6baf39468d7c0900c"}, + {file = "fastavro-1.5.2-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:7e4ccdf1142a01b5a10b9451cc7f2e82b0bf4d2593921b9aa8d66a4758f42ee7"}, + {file = "fastavro-1.5.2-cp310-cp310-win_amd64.whl", hash = "sha256:cc162420a6619debb8aabffc961e8f9aa442dfb0246115c421a0fa108c5e0796"}, + {file = "fastavro-1.5.2-cp37-cp37m-macosx_10_15_x86_64.whl", hash = "sha256:8bfd03691fb57771b0f4d4a241bd5d5f2e622cbd55e88ab3718a58b7e8e00bdb"}, + {file = "fastavro-1.5.2-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:3600c5b8ccc0c5dfab5927486d71ba79ac155b0eeecce517d86e191a5186110c"}, + {file = "fastavro-1.5.2-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:2c4777ebd022aecb05829b52359351746fb8ac1f3958385e524addde22d212c4"}, + {file = "fastavro-1.5.2-cp37-cp37m-win_amd64.whl", hash = "sha256:faa8c9cd4920c2e4e4131dae2e5fb050cc64533c22384d5dbba55c6a6463771e"}, + {file = "fastavro-1.5.2-cp38-cp38-macosx_10_15_x86_64.whl", hash = "sha256:90517b77f9d8362efe8deaa3da06673f0ef961e973bd9daa49aba998d8ca849e"}, + {file = "fastavro-1.5.2-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:746d97e41d24c838c70a44cb4e6a7e4109f20923fda5a6de18482e31db06ccdd"}, + {file = "fastavro-1.5.2-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:49398a773fab076725abf8926525092c455f2cb2ae58127d69fcc132d970fd6e"}, + {file = "fastavro-1.5.2-cp38-cp38-win_amd64.whl", hash = "sha256:45b7bfabf42bb3882071599eef49228039febbb1b8801b3a65bf32d717eea075"}, + {file = "fastavro-1.5.2-cp39-cp39-macosx_10_15_x86_64.whl", hash = "sha256:e6f23f1f4e2726408a35d2f2b279c29da41a989351a0f8c28ec62348f01be977"}, + {file = "fastavro-1.5.2-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:7699d1283062f11a3bbfb705f1f2d3fae0f7a1238ae3ca6234c7c9208b0a11a6"}, + {file = "fastavro-1.5.2-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:778d134d144d025f72ed98f60b5a63d363f473c5b19a32f4173e2e11e7f30be9"}, + {file = "fastavro-1.5.2-cp39-cp39-win_amd64.whl", hash = "sha256:db4f1a65dfa6d26b4be74d9aacd13467a17928ada2c15096b401288671689e44"}, + {file = "fastavro-1.5.2.tar.gz", hash = "sha256:ee7ffc6882834499fd53877a08cdadbb107821a559bb78278dbbb3685ae92788"}, ] filelock = [ {file = "filelock-3.7.1-py3-none-any.whl", hash = "sha256:37def7b658813cda163b56fc564cdc75e86d338246458c4c28ae84cabefa2404"}, @@ -538,8 +542,8 @@ identify = [ {file = "identify-2.5.1.tar.gz", hash = "sha256:3d11b16f3fe19f52039fb7e39c9c884b21cb1b586988114fbe42671f03de3e82"}, ] importlib-metadata = [ - {file = "importlib_metadata-4.11.4-py3-none-any.whl", hash = "sha256:c58c8eb8a762858f49e18436ff552e83914778e50e9d2f1660535ffb364552ec"}, - {file = "importlib_metadata-4.11.4.tar.gz", hash = "sha256:5d26852efe48c0a32b0509ffbc583fda1a2266545a78d104a6f4aff3db17d700"}, + {file = "importlib_metadata-4.12.0-py3-none-any.whl", hash = "sha256:7401a975809ea1fdc658c3aa4f78cc2195a0e019c5cbc4c06122884e9ae80c23"}, + {file = "importlib_metadata-4.12.0.tar.gz", hash = "sha256:637245b8bab2b6502fcbc752cc4b7a6f6243bb02b31c5c26156ad103d3d45670"}, ] iniconfig = [ {file = "iniconfig-1.1.1-py2.py3-none-any.whl", hash = "sha256:011e24c64b7f47f6ebd835bb12a743f2fbe9a26d4cecaa7f53bc4f35ee9da8b3"}, @@ -574,32 +578,32 @@ mmh3 = [ {file = "mmh3-3.0.0.tar.gz", hash = "sha256:d1ec578c09a07d3518ec9be540b87546397fa3455de73c166fcce51eaa5c41c5"}, ] nodeenv = [ - {file = "nodeenv-1.6.0-py2.py3-none-any.whl", hash = "sha256:621e6b7076565ddcacd2db0294c0381e01fd28945ab36bcf00f41c5daf63bef7"}, - {file = "nodeenv-1.6.0.tar.gz", hash = "sha256:3ef13ff90291ba2a4a7a4ff9a979b63ffdd00a464dbe04acf0ea6471517a4c2b"}, + {file = "nodeenv-1.7.0-py2.py3-none-any.whl", hash = "sha256:27083a7b96a25f2f5e1d8cb4b6317ee8aeda3bdd121394e5ac54e498028a042e"}, + {file = "nodeenv-1.7.0.tar.gz", hash = "sha256:e0e7f7dfb85fc5394c6fe1e8fa98131a2473e04311a45afb6508f7cf1836fa2b"}, ] numpy = [ - {file = "numpy-1.22.4-cp310-cp310-macosx_10_14_x86_64.whl", hash = "sha256:ba9ead61dfb5d971d77b6c131a9dbee62294a932bf6a356e48c75ae684e635b3"}, - {file = "numpy-1.22.4-cp310-cp310-macosx_10_15_x86_64.whl", hash = "sha256:1ce7ab2053e36c0a71e7a13a7475bd3b1f54750b4b433adc96313e127b870887"}, - {file = "numpy-1.22.4-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:7228ad13744f63575b3a972d7ee4fd61815b2879998e70930d4ccf9ec721dce0"}, - {file = "numpy-1.22.4-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:43a8ca7391b626b4c4fe20aefe79fec683279e31e7c79716863b4b25021e0e74"}, - {file = "numpy-1.22.4-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:a911e317e8c826ea632205e63ed8507e0dc877dcdc49744584dfc363df9ca08c"}, - {file = "numpy-1.22.4-cp310-cp310-win32.whl", hash = "sha256:9ce7df0abeabe7fbd8ccbf343dc0db72f68549856b863ae3dd580255d009648e"}, - {file = "numpy-1.22.4-cp310-cp310-win_amd64.whl", hash = "sha256:3e1ffa4748168e1cc8d3cde93f006fe92b5421396221a02f2274aab6ac83b077"}, - {file = "numpy-1.22.4-cp38-cp38-macosx_10_15_x86_64.whl", hash = "sha256:59d55e634968b8f77d3fd674a3cf0b96e85147cd6556ec64ade018f27e9479e1"}, - {file = "numpy-1.22.4-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:c1d937820db6e43bec43e8d016b9b3165dcb42892ea9f106c70fb13d430ffe72"}, - {file = "numpy-1.22.4-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:d4c5d5eb2ec8da0b4f50c9a843393971f31f1d60be87e0fb0917a49133d257d6"}, - {file = "numpy-1.22.4-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:64f56fc53a2d18b1924abd15745e30d82a5782b2cab3429aceecc6875bd5add0"}, - {file = "numpy-1.22.4-cp38-cp38-win32.whl", hash = "sha256:fb7a980c81dd932381f8228a426df8aeb70d59bbcda2af075b627bbc50207cba"}, - {file = "numpy-1.22.4-cp38-cp38-win_amd64.whl", hash = "sha256:e96d7f3096a36c8754207ab89d4b3282ba7b49ea140e4973591852c77d09eb76"}, - {file = "numpy-1.22.4-cp39-cp39-macosx_10_14_x86_64.whl", hash = "sha256:4c6036521f11a731ce0648f10c18ae66d7143865f19f7299943c985cdc95afb5"}, - {file = "numpy-1.22.4-cp39-cp39-macosx_10_15_x86_64.whl", hash = "sha256:b89bf9b94b3d624e7bb480344e91f68c1c6c75f026ed6755955117de00917a7c"}, - {file = "numpy-1.22.4-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:2d487e06ecbf1dc2f18e7efce82ded4f705f4bd0cd02677ffccfb39e5c284c7e"}, - {file = "numpy-1.22.4-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:f3eb268dbd5cfaffd9448113539e44e2dd1c5ca9ce25576f7c04a5453edc26fa"}, - {file = "numpy-1.22.4-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:37431a77ceb9307c28382c9773da9f306435135fae6b80b62a11c53cfedd8802"}, - {file = "numpy-1.22.4-cp39-cp39-win32.whl", hash = "sha256:cc7f00008eb7d3f2489fca6f334ec19ca63e31371be28fd5dad955b16ec285bd"}, - {file = "numpy-1.22.4-cp39-cp39-win_amd64.whl", hash = "sha256:f0725df166cf4785c0bc4cbfb320203182b1ecd30fee6e541c8752a92df6aa32"}, - {file = "numpy-1.22.4-pp38-pypy38_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:0791fbd1e43bf74b3502133207e378901272f3c156c4df4954cad833b1380207"}, - {file = "numpy-1.22.4.zip", hash = "sha256:425b390e4619f58d8526b3dcf656dde069133ae5c240229821f01b5f44ea07af"}, + {file = "numpy-1.23.0-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:58bfd40eb478f54ff7a5710dd61c8097e169bc36cc68333d00a9bcd8def53b38"}, + {file = "numpy-1.23.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:196cd074c3f97c4121601790955f915187736f9cf458d3ee1f1b46aff2b1ade0"}, + {file = "numpy-1.23.0-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:f1d88ef79e0a7fa631bb2c3dda1ea46b32b1fe614e10fedd611d3d5398447f2f"}, + {file = "numpy-1.23.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:d54b3b828d618a19779a84c3ad952e96e2c2311b16384e973e671aa5be1f6187"}, + {file = "numpy-1.23.0-cp310-cp310-win32.whl", hash = "sha256:2b2da66582f3a69c8ce25ed7921dcd8010d05e59ac8d89d126a299be60421171"}, + {file = "numpy-1.23.0-cp310-cp310-win_amd64.whl", hash = "sha256:97a76604d9b0e79f59baeca16593c711fddb44936e40310f78bfef79ee9a835f"}, + {file = "numpy-1.23.0-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:d8cc87bed09de55477dba9da370c1679bd534df9baa171dd01accbb09687dac3"}, + {file = "numpy-1.23.0-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:f0f18804df7370571fb65db9b98bf1378172bd4e962482b857e612d1fec0f53e"}, + {file = "numpy-1.23.0-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:ac86f407873b952679f5f9e6c0612687e51547af0e14ddea1eedfcb22466babd"}, + {file = "numpy-1.23.0-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:ae8adff4172692ce56233db04b7ce5792186f179c415c37d539c25de7298d25d"}, + {file = "numpy-1.23.0-cp38-cp38-win32.whl", hash = "sha256:fe8b9683eb26d2c4d5db32cd29b38fdcf8381324ab48313b5b69088e0e355379"}, + {file = "numpy-1.23.0-cp38-cp38-win_amd64.whl", hash = "sha256:5043bcd71fcc458dfb8a0fc5509bbc979da0131b9d08e3d5f50fb0bbb36f169a"}, + {file = "numpy-1.23.0-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:1c29b44905af288b3919803aceb6ec7fec77406d8b08aaa2e8b9e63d0fe2f160"}, + {file = "numpy-1.23.0-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:98e8e0d8d69ff4d3fa63e6c61e8cfe2d03c29b16b58dbef1f9baa175bbed7860"}, + {file = "numpy-1.23.0-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:79a506cacf2be3a74ead5467aee97b81fca00c9c4c8b3ba16dbab488cd99ba10"}, + {file = "numpy-1.23.0-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:092f5e6025813e64ad6d1b52b519165d08c730d099c114a9247c9bb635a2a450"}, + {file = "numpy-1.23.0-cp39-cp39-win32.whl", hash = "sha256:d6ca8dabe696c2785d0c8c9b0d8a9b6e5fdbe4f922bde70d57fa1a2848134f95"}, + {file = "numpy-1.23.0-cp39-cp39-win_amd64.whl", hash = "sha256:fc431493df245f3c627c0c05c2bd134535e7929dbe2e602b80e42bf52ff760bc"}, + {file = "numpy-1.23.0-pp38-pypy38_pp73-macosx_10_9_x86_64.whl", hash = "sha256:f9c3fc2adf67762c9fe1849c859942d23f8d3e0bee7b5ed3d4a9c3eeb50a2f07"}, + {file = "numpy-1.23.0-pp38-pypy38_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:d0d2094e8f4d760500394d77b383a1b06d3663e8892cdf5df3c592f55f3bff66"}, + {file = "numpy-1.23.0-pp38-pypy38_pp73-win_amd64.whl", hash = "sha256:94b170b4fa0168cd6be4becf37cb5b127bd12a795123984385b8cd4aca9857e5"}, + {file = "numpy-1.23.0.tar.gz", hash = "sha256:bd3fa4fe2e38533d5336e1272fc4e765cabbbde144309ccee8675509d5cd7b05"}, ] packaging = [ {file = "packaging-21.3-py3-none-any.whl", hash = "sha256:ef103e05f519cdc783ae24ea4e2e0f508a9c99b2d4969652eed6a2e1ea5bd522"}, @@ -771,8 +775,8 @@ tomli = [ {file = "tomli-2.0.1.tar.gz", hash = "sha256:de526c12914f0c550d15924c62d72abc48d6fe7364aa87328337a31007fe8a4f"}, ] virtualenv = [ - {file = "virtualenv-20.14.1-py2.py3-none-any.whl", hash = "sha256:e617f16e25b42eb4f6e74096b9c9e37713cf10bf30168fb4a739f3fa8f898a3a"}, - {file = "virtualenv-20.14.1.tar.gz", hash = "sha256:ef589a79795589aada0c1c5b319486797c03b67ac3984c48c669c0e4f50df3a5"}, + {file = "virtualenv-20.15.0-py2.py3-none-any.whl", hash = "sha256:804cce4de5b8a322f099897e308eecc8f6e2951f1a8e7e2b3598dff865f01336"}, + {file = "virtualenv-20.15.0.tar.gz", hash = "sha256:4c44b1d77ca81f8368e2d7414f9b20c428ad16b343ac6d226206c5b84e2b4fcc"}, ] zipp = [ {file = "zipp-3.8.0-py3-none-any.whl", hash = "sha256:c4f6e5bbf48e74f7a38e7cc5b0480ff42b0ae5178957d564d18932525d5cf099"}, From 28f40d7df3ea0cd4eb699d48ec349dfde33da9db Mon Sep 17 00:00:00 2001 From: Fokko Driesprong Date: Tue, 28 Jun 2022 01:42:50 +0200 Subject: [PATCH 123/642] Python: Use Pydantic for serialization and deserialization (#5011) --- poetry.lock | 66 ++- pyproject.toml | 2 + src/iceberg/exceptions.py | 4 + src/iceberg/schema.py | 166 ++++---- src/iceberg/serializers.py | 75 ++++ src/iceberg/table/metadata.py | 356 ++++++++++++++++ src/iceberg/table/refs.py | 37 ++ src/iceberg/types.py | 370 ++++++++++------- src/iceberg/utils/iceberg_base_model.py | 52 +++ src/iceberg/utils/singleton.py | 13 +- tests/avro/test_reader.py | 2 +- tests/conftest.py | 86 +++- tests/table/test_metadata.py | 522 ++++++++++++++++++++++++ tests/test_schema.py | 40 +- tests/test_types.py | 400 +++++++++++++++++- tests/utils/test_bin_packing.py | 15 + 16 files changed, 1961 insertions(+), 245 deletions(-) create mode 100644 src/iceberg/serializers.py create mode 100644 src/iceberg/table/metadata.py create mode 100644 src/iceberg/table/refs.py create mode 100644 src/iceberg/utils/iceberg_base_model.py create mode 100644 tests/table/test_metadata.py diff --git a/poetry.lock b/poetry.lock index 572d84931d..f2fc0d9cff 100644 --- a/poetry.lock +++ b/poetry.lock @@ -251,6 +251,21 @@ category = "main" optional = true python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*" +[[package]] +name = "pydantic" +version = "1.9.1" +description = "Data validation and settings management using python type hints" +category = "main" +optional = false +python-versions = ">=3.6.1" + +[package.dependencies] +typing-extensions = ">=3.7.4.3" + +[package.extras] +dotenv = ["python-dotenv (>=0.10.4)"] +email = ["email-validator (>=1.0.3)"] + [[package]] name = "pyparsing" version = "3.0.9" @@ -340,6 +355,14 @@ category = "dev" optional = false python-versions = ">=3.7" +[[package]] +name = "typing-extensions" +version = "4.2.0" +description = "Backported and Experimental Type Hints for Python 3.7+" +category = "main" +optional = false +python-versions = ">=3.7" + [[package]] name = "virtualenv" version = "20.15.0" @@ -392,7 +415,7 @@ snappy = ["python-snappy"] [metadata] lock-version = "1.1" python-versions = "^3.8" -content-hash = "439429e65911b8e768bd4617a4883e8a4ec2652df3a43b28755346da8ed17e19" +content-hash = "c60cc02a89d6d22566af086e4f94dcf12662a59d0a9ceacbfbd90c65b5b34308" [metadata.files] atomicwrites = [ @@ -665,6 +688,43 @@ pycparser = [ {file = "pycparser-2.21-py2.py3-none-any.whl", hash = "sha256:8ee45429555515e1f6b185e78100aea234072576aa43ab53aefcae078162fca9"}, {file = "pycparser-2.21.tar.gz", hash = "sha256:e644fdec12f7872f86c58ff790da456218b10f863970249516d60a5eaca77206"}, ] +pydantic = [ + {file = "pydantic-1.9.1-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:c8098a724c2784bf03e8070993f6d46aa2eeca031f8d8a048dff277703e6e193"}, + {file = "pydantic-1.9.1-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:c320c64dd876e45254bdd350f0179da737463eea41c43bacbee9d8c9d1021f11"}, + {file = "pydantic-1.9.1-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:18f3e912f9ad1bdec27fb06b8198a2ccc32f201e24174cec1b3424dda605a310"}, + {file = "pydantic-1.9.1-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:c11951b404e08b01b151222a1cb1a9f0a860a8153ce8334149ab9199cd198131"}, + {file = "pydantic-1.9.1-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:8bc541a405423ce0e51c19f637050acdbdf8feca34150e0d17f675e72d119580"}, + {file = "pydantic-1.9.1-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:e565a785233c2d03724c4dc55464559639b1ba9ecf091288dd47ad9c629433bd"}, + {file = "pydantic-1.9.1-cp310-cp310-win_amd64.whl", hash = "sha256:a4a88dcd6ff8fd47c18b3a3709a89adb39a6373f4482e04c1b765045c7e282fd"}, + {file = "pydantic-1.9.1-cp36-cp36m-macosx_10_9_x86_64.whl", hash = "sha256:447d5521575f18e18240906beadc58551e97ec98142266e521c34968c76c8761"}, + {file = "pydantic-1.9.1-cp36-cp36m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:985ceb5d0a86fcaa61e45781e567a59baa0da292d5ed2e490d612d0de5796918"}, + {file = "pydantic-1.9.1-cp36-cp36m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:059b6c1795170809103a1538255883e1983e5b831faea6558ef873d4955b4a74"}, + {file = "pydantic-1.9.1-cp36-cp36m-musllinux_1_1_i686.whl", hash = "sha256:d12f96b5b64bec3f43c8e82b4aab7599d0157f11c798c9f9c528a72b9e0b339a"}, + {file = "pydantic-1.9.1-cp36-cp36m-musllinux_1_1_x86_64.whl", hash = "sha256:ae72f8098acb368d877b210ebe02ba12585e77bd0db78ac04a1ee9b9f5dd2166"}, + {file = "pydantic-1.9.1-cp36-cp36m-win_amd64.whl", hash = "sha256:79b485767c13788ee314669008d01f9ef3bc05db9ea3298f6a50d3ef596a154b"}, + {file = "pydantic-1.9.1-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:494f7c8537f0c02b740c229af4cb47c0d39840b829ecdcfc93d91dcbb0779892"}, + {file = "pydantic-1.9.1-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:f0f047e11febe5c3198ed346b507e1d010330d56ad615a7e0a89fae604065a0e"}, + {file = "pydantic-1.9.1-cp37-cp37m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:969dd06110cb780da01336b281f53e2e7eb3a482831df441fb65dd30403f4608"}, + {file = "pydantic-1.9.1-cp37-cp37m-musllinux_1_1_i686.whl", hash = "sha256:177071dfc0df6248fd22b43036f936cfe2508077a72af0933d0c1fa269b18537"}, + {file = "pydantic-1.9.1-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:9bcf8b6e011be08fb729d110f3e22e654a50f8a826b0575c7196616780683380"}, + {file = "pydantic-1.9.1-cp37-cp37m-win_amd64.whl", hash = "sha256:a955260d47f03df08acf45689bd163ed9df82c0e0124beb4251b1290fa7ae728"}, + {file = "pydantic-1.9.1-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:9ce157d979f742a915b75f792dbd6aa63b8eccaf46a1005ba03aa8a986bde34a"}, + {file = "pydantic-1.9.1-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:0bf07cab5b279859c253d26a9194a8906e6f4a210063b84b433cf90a569de0c1"}, + {file = "pydantic-1.9.1-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:5d93d4e95eacd313d2c765ebe40d49ca9dd2ed90e5b37d0d421c597af830c195"}, + {file = "pydantic-1.9.1-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:1542636a39c4892c4f4fa6270696902acb186a9aaeac6f6cf92ce6ae2e88564b"}, + {file = "pydantic-1.9.1-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:a9af62e9b5b9bc67b2a195ebc2c2662fdf498a822d62f902bf27cccb52dbbf49"}, + {file = "pydantic-1.9.1-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:fe4670cb32ea98ffbf5a1262f14c3e102cccd92b1869df3bb09538158ba90fe6"}, + {file = "pydantic-1.9.1-cp38-cp38-win_amd64.whl", hash = "sha256:9f659a5ee95c8baa2436d392267988fd0f43eb774e5eb8739252e5a7e9cf07e0"}, + {file = "pydantic-1.9.1-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:b83ba3825bc91dfa989d4eed76865e71aea3a6ca1388b59fc801ee04c4d8d0d6"}, + {file = "pydantic-1.9.1-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:1dd8fecbad028cd89d04a46688d2fcc14423e8a196d5b0a5c65105664901f810"}, + {file = "pydantic-1.9.1-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:02eefd7087268b711a3ff4db528e9916ac9aa18616da7bca69c1871d0b7a091f"}, + {file = "pydantic-1.9.1-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:7eb57ba90929bac0b6cc2af2373893d80ac559adda6933e562dcfb375029acee"}, + {file = "pydantic-1.9.1-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:4ce9ae9e91f46c344bec3b03d6ee9612802682c1551aaf627ad24045ce090761"}, + {file = "pydantic-1.9.1-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:72ccb318bf0c9ab97fc04c10c37683d9eea952ed526707fabf9ac5ae59b701fd"}, + {file = "pydantic-1.9.1-cp39-cp39-win_amd64.whl", hash = "sha256:61b6760b08b7c395975d893e0b814a11cf011ebb24f7d869e7118f5a339a82e1"}, + {file = "pydantic-1.9.1-py3-none-any.whl", hash = "sha256:4988c0f13c42bfa9ddd2fe2f569c9d54646ce84adc5de84228cfe83396f3bd58"}, + {file = "pydantic-1.9.1.tar.gz", hash = "sha256:1ed987c3ff29fff7fd8c3ea3a3ea877ad310aae2ef9889a119e22d3f2db0691a"}, +] pyparsing = [ {file = "pyparsing-3.0.9-py3-none-any.whl", hash = "sha256:5026bae9a10eeaefb61dab2f09052b9f4307d44aee4eda64b309723d8d206bbc"}, {file = "pyparsing-3.0.9.tar.gz", hash = "sha256:2b020ecf7d21b687f219b71ecad3631f644a47f01403fa1d1036b0c6416d70fb"}, @@ -774,6 +834,10 @@ tomli = [ {file = "tomli-2.0.1-py3-none-any.whl", hash = "sha256:939de3e7a6161af0c887ef91b7d41a53e7c5a1ca976325f429cb46ea9bc30ecc"}, {file = "tomli-2.0.1.tar.gz", hash = "sha256:de526c12914f0c550d15924c62d72abc48d6fe7364aa87328337a31007fe8a4f"}, ] +typing-extensions = [ + {file = "typing_extensions-4.2.0-py3-none-any.whl", hash = "sha256:6657594ee297170d19f67d55c05852a874e7eb634f4f753dbd667855e07c1708"}, + {file = "typing_extensions-4.2.0.tar.gz", hash = "sha256:f1c24655a0da0d1b67f07e17a5e6b2a105894e6824b92096378bb3668ef02376"}, +] virtualenv = [ {file = "virtualenv-20.15.0-py2.py3-none-any.whl", hash = "sha256:804cce4de5b8a322f099897e308eecc8f6e2951f1a8e7e2b3598dff865f01336"}, {file = "virtualenv-20.15.0.tar.gz", hash = "sha256:4c44b1d77ca81f8368e2d7414f9b20c428ad16b343ac6d226206c5b84e2b4fcc"}, diff --git a/pyproject.toml b/pyproject.toml index 5d7b75f868..07e5ff12ed 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -40,6 +40,8 @@ packages = [ [tool.poetry.dependencies] python = "^3.8" mmh3 = "^3.0.0" +pydantic = "^1.9.1" + pyarrow = { version = "^8.0.0", optional = true } zstandard = { version = "^0.17.0", optional = true } diff --git a/src/iceberg/exceptions.py b/src/iceberg/exceptions.py index 3dd68e55d4..b12e836e46 100644 --- a/src/iceberg/exceptions.py +++ b/src/iceberg/exceptions.py @@ -30,3 +30,7 @@ class NamespaceNotEmptyError(Exception): class AlreadyExistsError(Exception): """Raised when a table or name-space being created already exists in the catalog""" + + +class ValidationError(Exception): + ... diff --git a/src/iceberg/schema.py b/src/iceberg/schema.py index 99ef37c4b9..b57da36790 100644 --- a/src/iceberg/schema.py +++ b/src/iceberg/schema.py @@ -16,8 +16,6 @@ # under the License. # pylint: disable=W0511 -from __future__ import annotations - from abc import ABC, abstractmethod from dataclasses import dataclass from functools import cached_property, singledispatch @@ -25,9 +23,15 @@ Any, Dict, Generic, + List, + Optional, + Tuple, TypeVar, + Union, ) +from pydantic import Field, PrivateAttr + from iceberg.files import StructProtocol from iceberg.types import ( IcebergType, @@ -37,11 +41,12 @@ PrimitiveType, StructType, ) +from iceberg.utils.iceberg_base_model import IcebergBaseModel T = TypeVar("T") -class Schema: +class Schema(IcebergBaseModel): """A table Schema Example: @@ -49,11 +54,17 @@ class Schema: >>> from iceberg import types """ - def __init__(self, *columns: NestedField, schema_id: int, identifier_field_ids: list[int] | None = None): - self._struct = StructType(*columns) - self._schema_id = schema_id - self._identifier_field_ids = identifier_field_ids or [] - self._name_to_id: dict[str, int] = index_by_name(self) + fields: Tuple[NestedField, ...] = Field(default_factory=tuple) + schema_id: int = Field(alias="schema-id") + identifier_field_ids: List[int] = Field(alias="identifier-field-ids", default_factory=list) + + _name_to_id: Dict[str, int] = PrivateAttr() + + def __init__(self, *fields: NestedField, **data): + if fields: + data["fields"] = fields + super().__init__(**data) + self._name_to_id = index_by_name(self) def __str__(self): return "table {\n" + "\n".join([" " + str(field) for field in self.columns]) + "\n}" @@ -79,21 +90,12 @@ def __eq__(self, other) -> bool: return identifier_field_ids_is_equal and schema_is_equal @property - def columns(self) -> tuple[NestedField, ...]: - """A list of the top-level fields in the underlying struct""" - return self._struct.fields - - @property - def schema_id(self) -> int: - """The ID of this Schema""" - return self._schema_id - - @property - def identifier_field_ids(self) -> list[int]: - return self._identifier_field_ids + def columns(self) -> Tuple[NestedField, ...]: + """A tuple of the top-level fields""" + return self.fields @cached_property - def _lazy_id_to_field(self) -> dict[int, NestedField]: + def _lazy_id_to_field(self) -> Dict[int, NestedField]: """Returns an index of field ID to NestedField instance This is calculated once when called for the first time. Subsequent calls to this method will use a cached index. @@ -101,7 +103,7 @@ def _lazy_id_to_field(self) -> dict[int, NestedField]: return index_by_id(self) @cached_property - def _lazy_name_to_id_lower(self) -> dict[str, int]: + def _lazy_name_to_id_lower(self) -> Dict[str, int]: """Returns an index of lower-case field names to field IDs This is calculated once when called for the first time. Subsequent calls to this method will use a cached index. @@ -109,7 +111,7 @@ def _lazy_name_to_id_lower(self) -> dict[str, int]: return {name.lower(): field_id for name, field_id in self._name_to_id.items()} @cached_property - def _lazy_id_to_name(self) -> dict[int, str]: + def _lazy_id_to_name(self) -> Dict[int, str]: """Returns an index of field ID to full name This is calculated once when called for the first time. Subsequent calls to this method will use a cached index. @@ -117,7 +119,7 @@ def _lazy_id_to_name(self) -> dict[int, str]: return index_name_by_id(self) @cached_property - def _lazy_id_to_accessor(self) -> dict[int, Accessor]: + def _lazy_id_to_accessor(self) -> Dict[int, "Accessor"]: """Returns an index of field ID to accessor This is calculated once when called for the first time. Subsequent calls to this method will use a cached index. @@ -125,10 +127,10 @@ def _lazy_id_to_accessor(self) -> dict[int, Accessor]: return build_position_accessors(self) def as_struct(self) -> StructType: - """Returns the underlying struct""" - return self._struct + """Returns the schema as a struct""" + return StructType(*self.fields) - def find_field(self, name_or_id: str | int, case_sensitive: bool = True) -> NestedField | None: + def find_field(self, name_or_id: Union[str, int], case_sensitive: bool = True) -> Optional[NestedField]: """Find a field using a field name or field ID Args: @@ -146,7 +148,7 @@ def find_field(self, name_or_id: str | int, case_sensitive: bool = True) -> Nest field_id = self._lazy_name_to_id_lower.get(name_or_id.lower()) return self._lazy_id_to_field.get(field_id) # type: ignore - def find_type(self, name_or_id: str | int, case_sensitive: bool = True) -> IcebergType: + def find_type(self, name_or_id: Union[str, int], case_sensitive: bool = True) -> IcebergType: """Find a field type using a field name or field ID Args: @@ -161,7 +163,7 @@ def find_type(self, name_or_id: str | int, case_sensitive: bool = True) -> Icebe raise ValueError(f"Could not find field with name or id {name_or_id}, case_sensitive={case_sensitive}") return field.field_type - def find_column_name(self, column_id: int) -> str | None: + def find_column_name(self, column_id: int) -> Optional[str]: """Find a column name given a column ID Args: @@ -172,7 +174,7 @@ def find_column_name(self, column_id: int) -> str | None: """ return self._lazy_id_to_name.get(column_id) - def accessor_for_field(self, field_id: int) -> Accessor | None: + def accessor_for_field(self, field_id: int) -> Optional["Accessor"]: """Find a schema position accessor given a field ID Args: @@ -183,7 +185,7 @@ def accessor_for_field(self, field_id: int) -> Accessor | None: """ return self._lazy_id_to_accessor.get(field_id) - def select(self, names: list[str], case_sensitive: bool = True) -> Schema: + def select(self, names: List[str], case_sensitive: bool = True) -> "Schema": """Return a new schema instance pruned to a subset of columns Args: @@ -198,12 +200,12 @@ def select(self, names: list[str], case_sensitive: bool = True) -> Schema: return self._case_insensitive_select(schema=self, names=names) @classmethod - def _case_sensitive_select(cls, schema: Schema, names: list[str]): + def _case_sensitive_select(cls, schema: "Schema", names: List[str]): # TODO: Add a PruneColumns schema visitor and use it here raise NotImplementedError() @classmethod - def _case_insensitive_select(cls, schema: Schema, names: list[str]): + def _case_insensitive_select(cls, schema: "Schema", names: List[str]): # TODO: Add a PruneColumns schema visitor and use it here raise NotImplementedError() @@ -244,7 +246,7 @@ def schema(self, schema: Schema, struct_result: T) -> T: """Visit a Schema""" @abstractmethod - def struct(self, struct: StructType, field_results: list[T]) -> T: + def struct(self, struct: StructType, field_results: List[T]) -> T: """Visit a StructType""" @abstractmethod @@ -269,7 +271,7 @@ class Accessor: """An accessor for a specific position in a container that implements the StructProtocol""" position: int - inner: Accessor | None = None + inner: Optional["Accessor"] = None def __str__(self): return f"Accessor(position={self.position},inner={self.inner})" @@ -336,9 +338,9 @@ def _(obj: StructType, visitor: SchemaVisitor[T]) -> T: def _(obj: ListType, visitor: SchemaVisitor[T]) -> T: """Visit a ListType with a concrete SchemaVisitor""" - visitor.before_list_element(obj.element) - result = visit(obj.element.field_type, visitor) - visitor.after_list_element(obj.element) + visitor.before_list_element(obj.element_field) + result = visit(obj.element_type, visitor) + visitor.after_list_element(obj.element_field) return visitor.list(obj, result) @@ -346,13 +348,13 @@ def _(obj: ListType, visitor: SchemaVisitor[T]) -> T: @visit.register(MapType) def _(obj: MapType, visitor: SchemaVisitor[T]) -> T: """Visit a MapType with a concrete SchemaVisitor""" - visitor.before_map_key(obj.key) - key_result = visit(obj.key.field_type, visitor) - visitor.after_map_key(obj.key) + visitor.before_map_key(obj.key_field) + key_result = visit(obj.key_type, visitor) + visitor.after_map_key(obj.key_field) - visitor.before_map_value(obj.value) - value_result = visit(obj.value.field_type, visitor) - visitor.after_list_element(obj.value) + visitor.before_map_value(obj.value_field) + value_result = visit(obj.value_type, visitor) + visitor.after_list_element(obj.value_field) return visitor.map(obj, key_result, value_result) @@ -367,35 +369,35 @@ class _IndexById(SchemaVisitor[Dict[int, NestedField]]): """A schema visitor for generating a field ID to NestedField index""" def __init__(self) -> None: - self._index: dict[int, NestedField] = {} + self._index: Dict[int, NestedField] = {} - def schema(self, schema: Schema, struct_result) -> dict[int, NestedField]: + def schema(self, schema: Schema, struct_result) -> Dict[int, NestedField]: return self._index - def struct(self, struct: StructType, field_results) -> dict[int, NestedField]: + def struct(self, struct: StructType, field_results) -> Dict[int, NestedField]: return self._index - def field(self, field: NestedField, field_result) -> dict[int, NestedField]: + def field(self, field: NestedField, field_result) -> Dict[int, NestedField]: """Add the field ID to the index""" self._index[field.field_id] = field return self._index - def list(self, list_type: ListType, element_result) -> dict[int, NestedField]: + def list(self, list_type: ListType, element_result) -> Dict[int, NestedField]: """Add the list element ID to the index""" - self._index[list_type.element.field_id] = list_type.element + self._index[list_type.element_field.field_id] = list_type.element_field return self._index - def map(self, map_type: MapType, key_result, value_result) -> dict[int, NestedField]: + def map(self, map_type: MapType, key_result, value_result) -> Dict[int, NestedField]: """Add the key ID and value ID as individual items in the index""" - self._index[map_type.key.field_id] = map_type.key - self._index[map_type.value.field_id] = map_type.value + self._index[map_type.key_field.field_id] = map_type.key_field + self._index[map_type.value_field.field_id] = map_type.value_field return self._index - def primitive(self, primitive) -> dict[int, NestedField]: + def primitive(self, primitive) -> Dict[int, NestedField]: return self._index -def index_by_id(schema_or_type) -> dict[int, NestedField]: +def index_by_id(schema_or_type) -> Dict[int, NestedField]: """Generate an index of field IDs to NestedField instances Args: @@ -411,11 +413,11 @@ class _IndexByName(SchemaVisitor[Dict[str, int]]): """A schema visitor for generating a field name to field ID index""" def __init__(self) -> None: - self._index: dict[str, int] = {} - self._short_name_to_id: dict[str, int] = {} - self._combined_index: dict[str, int] = {} - self._field_names: list[str] = [] - self._short_field_names: list[str] = [] + self._index: Dict[str, int] = {} + self._short_name_to_id: Dict[str, int] = {} + self._combined_index: Dict[str, int] = {} + self._field_names: List[str] = [] + self._short_field_names: List[str] = [] def before_list_element(self, element: NestedField) -> None: """Short field names omit element when the element is a StructType""" @@ -438,26 +440,26 @@ def after_field(self, field: NestedField) -> None: self._field_names.pop() self._short_field_names.pop() - def schema(self, schema: Schema, struct_result: dict[str, int]) -> dict[str, int]: + def schema(self, schema: Schema, struct_result: Dict[str, int]) -> Dict[str, int]: return self._index - def struct(self, struct: StructType, field_results: list[dict[str, int]]) -> dict[str, int]: + def struct(self, struct: StructType, field_results: List[Dict[str, int]]) -> Dict[str, int]: return self._index - def field(self, field: NestedField, field_result: dict[str, int]) -> dict[str, int]: + def field(self, field: NestedField, field_result: Dict[str, int]) -> Dict[str, int]: """Add the field name to the index""" self._add_field(field.name, field.field_id) return self._index - def list(self, list_type: ListType, element_result: dict[str, int]) -> dict[str, int]: + def list(self, list_type: ListType, element_result: Dict[str, int]) -> Dict[str, int]: """Add the list element name to the index""" - self._add_field(list_type.element.name, list_type.element.field_id) + self._add_field(list_type.element_field.name, list_type.element_field.field_id) return self._index - def map(self, map_type: MapType, key_result: dict[str, int], value_result: dict[str, int]) -> dict[str, int]: + def map(self, map_type: MapType, key_result: Dict[str, int], value_result: Dict[str, int]) -> Dict[str, int]: """Add the key name and value name as individual items in the index""" - self._add_field(map_type.key.name, map_type.key.field_id) - self._add_field(map_type.value.name, map_type.value.field_id) + self._add_field(map_type.key_field.name, map_type.key_field.field_id) + self._add_field(map_type.value_field.name, map_type.value_field.field_id) return self._index def _add_field(self, name: str, field_id: int): @@ -483,10 +485,10 @@ def _add_field(self, name: str, field_id: int): short_name = ".".join([".".join(self._short_field_names), name]) self._short_name_to_id[short_name] = field_id - def primitive(self, primitive) -> dict[str, int]: + def primitive(self, primitive) -> Dict[str, int]: return self._index - def by_name(self) -> dict[str, int]: + def by_name(self) -> Dict[str, int]: """Returns an index of combined full and short names Note: Only short names that do not conflict with full names are included. @@ -495,13 +497,13 @@ def by_name(self) -> dict[str, int]: combined_index.update(self._index) return combined_index - def by_id(self) -> dict[int, str]: + def by_id(self) -> Dict[int, str]: """Returns an index of ID to full names""" id_to_full_name = {value: key for key, value in self._index.items()} return id_to_full_name -def index_by_name(schema_or_type: Schema | IcebergType) -> dict[str, int]: +def index_by_name(schema_or_type: Union[Schema, IcebergType]) -> Dict[str, int]: """Generate an index of field names to field IDs Args: @@ -515,7 +517,7 @@ def index_by_name(schema_or_type: Schema | IcebergType) -> dict[str, int]: return indexer.by_name() -def index_name_by_id(schema_or_type: Schema | IcebergType) -> dict[int, str]: +def index_name_by_id(schema_or_type: Union[Schema, IcebergType]) -> Dict[int, str]: """Generate an index of field IDs full field names Args: @@ -565,13 +567,13 @@ class _BuildPositionAccessors(SchemaVisitor[Dict[Position, Accessor]]): """ @staticmethod - def _wrap_leaves(result: dict[Position, Accessor], position: Position = 0) -> dict[Position, Accessor]: + def _wrap_leaves(result: Dict[Position, Accessor], position: Position = 0) -> Dict[Position, Accessor]: return {field_id: Accessor(position, inner=inner) for field_id, inner in result.items()} - def schema(self, schema: Schema, struct_result: dict[Position, Accessor]) -> dict[Position, Accessor]: + def schema(self, schema: Schema, struct_result: Dict[Position, Accessor]) -> Dict[Position, Accessor]: return struct_result - def struct(self, struct: StructType, field_results: list[dict[Position, Accessor]]) -> dict[Position, Accessor]: + def struct(self, struct: StructType, field_results: List[Dict[Position, Accessor]]) -> Dict[Position, Accessor]: result = {} for position, field in enumerate(struct.fields): @@ -583,22 +585,22 @@ def struct(self, struct: StructType, field_results: list[dict[Position, Accessor return result - def field(self, field: NestedField, field_result: dict[Position, Accessor]) -> dict[Position, Accessor]: + def field(self, field: NestedField, field_result: Dict[Position, Accessor]) -> Dict[Position, Accessor]: return field_result - def list(self, list_type: ListType, element_result: dict[Position, Accessor]) -> dict[Position, Accessor]: + def list(self, list_type: ListType, element_result: Dict[Position, Accessor]) -> Dict[Position, Accessor]: return {} def map( - self, map_type: MapType, key_result: dict[Position, Accessor], value_result: dict[Position, Accessor] - ) -> dict[Position, Accessor]: + self, map_type: MapType, key_result: Dict[Position, Accessor], value_result: Dict[Position, Accessor] + ) -> Dict[Position, Accessor]: return {} - def primitive(self, primitive: PrimitiveType) -> dict[Position, Accessor]: + def primitive(self, primitive: PrimitiveType) -> Dict[Position, Accessor]: return {} -def build_position_accessors(schema_or_type: Schema | IcebergType) -> dict[int, Accessor]: +def build_position_accessors(schema_or_type: Union[Schema, IcebergType]) -> Dict[int, Accessor]: """Generate an index of field IDs to schema position accessors Args: diff --git a/src/iceberg/serializers.py b/src/iceberg/serializers.py new file mode 100644 index 0000000000..98e279f624 --- /dev/null +++ b/src/iceberg/serializers.py @@ -0,0 +1,75 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +import codecs +import json +from typing import Union + +from iceberg.io.base import InputFile, InputStream, OutputFile +from iceberg.table.metadata import TableMetadata, TableMetadataV1, TableMetadataV2 + + +class FromByteStream: + """A collection of methods that deserialize dictionaries into Iceberg objects""" + + @staticmethod + def table_metadata(byte_stream: InputStream, encoding: str = "utf-8") -> TableMetadata: + """Instantiate a TableMetadata object from a byte stream + + Args: + byte_stream: A file-like byte stream object + encoding (default "utf-8"): The byte encoder to use for the reader + """ + reader = codecs.getreader(encoding) + metadata = json.load(reader(byte_stream)) # type: ignore + return TableMetadata.parse_obj(metadata) # type: ignore + + +class FromInputFile: + """A collection of methods that deserialize InputFiles into Iceberg objects""" + + @staticmethod + def table_metadata(input_file: InputFile, encoding: str = "utf-8") -> TableMetadata: + """Create a TableMetadata instance from an input file + + Args: + input_file (InputFile): A custom implementation of the iceberg.io.file.InputFile abstract base class + encoding (str): Encoding to use when loading bytestream + + Returns: + TableMetadata: A table metadata instance + + """ + return FromByteStream.table_metadata(byte_stream=input_file.open(), encoding=encoding) + + +class ToOutputFile: + """A collection of methods that serialize Iceberg objects into files given an OutputFile instance""" + + @staticmethod + def table_metadata( + metadata: Union[TableMetadataV1, TableMetadataV2], output_file: OutputFile, overwrite: bool = False + ) -> None: + """Write a TableMetadata instance to an output file + + Args: + output_file (OutputFile): A custom implementation of the iceberg.io.file.OutputFile abstract base class + overwrite (bool): Where to overwrite the file if it already exists. Defaults to `False`. + """ + f = output_file.create(overwrite=overwrite) + f.write(metadata.json().encode("utf-8")) + f.close() diff --git a/src/iceberg/table/metadata.py b/src/iceberg/table/metadata.py new file mode 100644 index 0000000000..55c43c1cf6 --- /dev/null +++ b/src/iceberg/table/metadata.py @@ -0,0 +1,356 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +from copy import copy +from typing import ( + Any, + Dict, + List, + Literal, + Optional, + Union, +) +from uuid import UUID, uuid4 + +from pydantic import Field, root_validator + +from iceberg.exceptions import ValidationError +from iceberg.schema import Schema +from iceberg.table.refs import MAIN_BRANCH, SnapshotRef, SnapshotRefType +from iceberg.utils.iceberg_base_model import IcebergBaseModel + +_INITIAL_SEQUENCE_NUMBER = 0 +INITIAL_SPEC_ID = 0 +DEFAULT_SCHEMA_ID = 0 +DEFAULT_SORT_ORDER_UNSORTED = 0 + + +def check_schemas(values: Dict[str, Any]) -> Dict[str, Any]: + """Validator to check if the current-schema-id is actually present in schemas""" + current_schema_id = values["current_schema_id"] + + for schema in values["schemas"]: + if schema.schema_id == current_schema_id: + return values + + raise ValidationError(f"current-schema-id {current_schema_id} can't be found in the schemas") + + +def check_partition_specs(values: Dict[str, Any]) -> Dict[str, Any]: + """Validator to check if the default-spec-id is present in partition-specs""" + default_spec_id = values["default_spec_id"] + + for spec in values["partition_specs"]: + if spec["spec-id"] == default_spec_id: + return values + + raise ValidationError(f"default-spec-id {default_spec_id} can't be found") + + +def check_sort_orders(values: Dict[str, Any]) -> Dict[str, Any]: + """Validator to check if the default_sort_order_id is present in sort-orders""" + default_sort_order_id = values["default_sort_order_id"] + + if default_sort_order_id != DEFAULT_SORT_ORDER_UNSORTED: + for sort in values["sort_orders"]: + if sort["order-id"] == default_sort_order_id: + return values + + raise ValidationError(f"default-sort-order-id {default_sort_order_id} can't be found") + return values + + +class TableMetadataCommonFields(IcebergBaseModel): + """Metadata for an Iceberg table as specified in the Apache Iceberg + spec (https://iceberg.apache.org/spec/#iceberg-table-spec)""" + + @root_validator(pre=True) + def cleanup_snapshot_id(cls, data: Dict[str, Any]): + if data.get("current-snapshot-id") == -1: + # We treat -1 and None the same, by cleaning this up + # in a pre-validator, we can simplify the logic later on + data["current-snapshot-id"] = None + return data + + @root_validator(skip_on_failure=True) + def construct_refs(cls, data: Dict[str, Any]): + # This is going to be much nicer as soon as refs is an actual pydantic object + if current_snapshot_id := data.get("current_snapshot_id"): + data["refs"] = {MAIN_BRANCH: SnapshotRef(snapshot_id=current_snapshot_id, snapshot_ref_type=SnapshotRefType.BRANCH)} + return data + + location: str = Field() + """The table’s base location. This is used by writers to determine where + to store data files, manifest files, and table metadata files.""" + + table_uuid: Optional[UUID] = Field(alias="table-uuid") + """A UUID that identifies the table, generated when the table is created. + Implementations must throw an exception if a table’s UUID does not match + the expected UUID after refreshing metadata.""" + + last_updated_ms: int = Field(alias="last-updated-ms") + """Timestamp in milliseconds from the unix epoch when the table + was last updated. Each table metadata file should update this + field just before writing.""" + + last_column_id: int = Field(alias="last-column-id") + """An integer; the highest assigned column ID for the table. + This is used to ensure fields are always assigned an unused ID + when evolving schemas.""" + + schemas: List[Schema] = Field(default_factory=list) + """A list of schemas, stored as objects with schema-id.""" + + current_schema_id: int = Field(alias="current-schema-id", default=DEFAULT_SCHEMA_ID) + """ID of the table’s current schema.""" + + partition_specs: list = Field(alias="partition-specs", default_factory=list) + """A list of partition specs, stored as full partition spec objects.""" + + default_spec_id: int = Field(alias="default-spec-id", default=INITIAL_SPEC_ID) + """ID of the “current” spec that writers should use by default.""" + + last_partition_id: int = Field(alias="last-partition-id") + """An integer; the highest assigned partition field ID across all + partition specs for the table. This is used to ensure partition fields + are always assigned an unused ID when evolving specs.""" + + properties: Dict[str, str] = Field(default_factory=dict) + """ A string to string map of table properties. This is used to + control settings that affect reading and writing and is not intended + to be used for arbitrary metadata. For example, commit.retry.num-retries + is used to control the number of commit retries.""" + + current_snapshot_id: Optional[int] = Field(alias="current-snapshot-id") + """ID of the current table snapshot.""" + + snapshots: list = Field(default_factory=list) + """A list of valid snapshots. Valid snapshots are snapshots for which + all data files exist in the file system. A data file must not be + deleted from the file system until the last snapshot in which it was + listed is garbage collected.""" + + snapshot_log: List[Dict[str, Any]] = Field(alias="snapshot-log", default_factory=list) + """A list (optional) of timestamp and snapshot ID pairs that encodes + changes to the current snapshot for the table. Each time the + current-snapshot-id is changed, a new entry should be added with the + last-updated-ms and the new current-snapshot-id. When snapshots are + expired from the list of valid snapshots, all entries before a snapshot + that has expired should be removed.""" + + metadata_log: List[Dict[str, Any]] = Field(alias="metadata-log", default_factory=list) + """A list (optional) of timestamp and metadata file location pairs that + encodes changes to the previous metadata files for the table. Each time + a new metadata file is created, a new entry of the previous metadata + file location should be added to the list. Tables can be configured to + remove oldest metadata log entries and keep a fixed-size log of the most + recent entries after a commit.""" + + sort_orders: List[Dict[str, Any]] = Field(alias="sort-orders", default_factory=list) + """A list of sort orders, stored as full sort order objects.""" + + default_sort_order_id: int = Field(alias="default-sort-order-id", default=DEFAULT_SORT_ORDER_UNSORTED) + """Default sort order id of the table. Note that this could be used by + writers, but is not used when reading because reads use the specs stored + in manifest files.""" + + refs: Dict[str, SnapshotRef] = Field(default_factory=dict) + """A map of snapshot references. + The map keys are the unique snapshot reference names in the table, + and the map values are snapshot reference objects. + There is always a main branch reference pointing to the + current-snapshot-id even if the refs map is null.""" + + +class TableMetadataV1(TableMetadataCommonFields, IcebergBaseModel): + """Represents version 1 of the Table Metadata + + More information about the specification: + https://iceberg.apache.org/spec/#version-1-analytic-data-tables + """ + + # When we read a V1 format-version, we'll make sure to populate the fields + # for V2 as well. This makes it easier downstream because we can just + # assume that everything is a TableMetadataV2. + # When writing, we should stick to the same version that it was, + # because bumping the version should be an explicit operation that is up + # to the owner of the table. + + @root_validator(pre=True) + def set_v2_compatible_defaults(cls, data: Dict[str, Any]) -> Dict[str, Any]: + """Sets default values to be compatible with the format v2 + + Set some sensible defaults for V1, so we comply with the schema + this is in pre=True, meaning that this will be done before validation. + We don't want to make the fields optional, since they are required for V2 + + Args: + data: The raw arguments when initializing a V1 TableMetadata + + Returns: + The TableMetadata with the defaults applied + """ + if "schema-id" not in data["schema"]: + data["schema"]["schema-id"] = DEFAULT_SCHEMA_ID + if "last-partition-id" not in data: + data["last-partition-id"] = max(spec["field-id"] for spec in data["partition-spec"]) + if "table-uuid" not in data: + data["table-uuid"] = uuid4() + return data + + @root_validator(skip_on_failure=True) + def construct_schemas(cls, data: Dict[str, Any]) -> Dict[str, Any]: + """Converts the schema into schemas + + For V1 schemas is optional, and if they aren't set, we'll set them + in this validator. This was we can always use the schemas when reading + table metadata, and we don't have to worry if it is a v1 or v2 format. + + Args: + data: The raw data after validation, meaning that the aliases are applied + + Returns: + The TableMetadata with the schemas set, if not provided + """ + if not data.get("schemas"): + schema = data["schema_"] + data["schemas"] = [schema] + else: + check_schemas(data["schemas"]) + return data + + @root_validator(skip_on_failure=True) + def construct_partition_specs(cls, data: Dict[str, Any]) -> Dict[str, Any]: + """Converts the partition_spec into partition_specs + + For V1 partition_specs is optional, and if they aren't set, we'll set them + in this validator. This was we can always use the partition_specs when reading + table metadata, and we don't have to worry if it is a v1 or v2 format. + + Args: + data: The raw data after validation, meaning that the aliases are applied + + Returns: + The TableMetadata with the partition_specs set, if not provided + """ + # This is going to be much nicer as soon as partition-spec is also migrated to pydantic + if not data.get("partition_specs"): + fields = data["partition_spec"] + data["partition_specs"] = [{"spec-id": INITIAL_SPEC_ID, "fields": fields}] + else: + check_partition_specs(data["partition_specs"]) + return data + + @root_validator(skip_on_failure=True) + def set_sort_orders(cls, data: Dict[str, Any]): + """Sets the sort_orders if not provided + + For V1 sort_orders is optional, and if they aren't set, we'll set them + in this validator. + + Args: + data: The raw data after validation, meaning that the aliases are applied + + Returns: + The TableMetadata with the sort_orders set, if not provided + """ + # This is going to be much nicer as soon as sort-order is an actual pydantic object + # Probably we'll just create a UNSORTED_ORDER constant then + if not data.get("sort_orders"): + data["sort_orders"] = [{"order_id": 0, "fields": []}] + else: + check_sort_orders(data["sort_orders"]) + return data + + def to_v2(self) -> "TableMetadataV2": + metadata = copy(self.dict()) + metadata["format_version"] = 2 + return TableMetadataV2(**metadata) + + format_version: Literal[1] = Field(alias="format-version") + """An integer version number for the format. Currently, this can be 1 or 2 + based on the spec. Implementations must throw an exception if a table’s + version is higher than the supported version.""" + + schema_: Schema = Field(alias="schema") + """The table’s current schema. (Deprecated: use schemas and + current-schema-id instead)""" + + partition_spec: List[Dict[str, Any]] = Field(alias="partition-spec") + """The table’s current partition spec, stored as only fields. + Note that this is used by writers to partition data, but is + not used when reading because reads use the specs stored in + manifest files. (Deprecated: use partition-specs and default-spec-id + instead)""" + + +class TableMetadataV2(TableMetadataCommonFields, IcebergBaseModel): + """Represents version 2 of the Table Metadata + + This extends Version 1 with row-level deletes, and adds some additional + information to the schema, such as all the historical schemas, partition-specs, + sort-orders. + + For more information: + https://iceberg.apache.org/spec/#version-2-row-level-deletes + """ + + @root_validator(skip_on_failure=True) + def check_schemas(cls, values: Dict[str, Any]): + return check_schemas(values) + + @root_validator + def check_partition_specs(cls, values: Dict[str, Any]): + return check_partition_specs(values) + + @root_validator(skip_on_failure=True) + def check_sort_orders(cls, values: Dict[str, Any]): + return check_sort_orders(values) + + format_version: Literal[2] = Field(alias="format-version") + """An integer version number for the format. Currently, this can be 1 or 2 + based on the spec. Implementations must throw an exception if a table’s + version is higher than the supported version.""" + + table_uuid: UUID = Field(alias="table-uuid") + """A UUID that identifies the table, generated when the table is created. + Implementations must throw an exception if a table’s UUID does not match + the expected UUID after refreshing metadata.""" + + last_sequence_number: int = Field(alias="last-sequence-number", default=_INITIAL_SEQUENCE_NUMBER) + """The table’s highest assigned sequence number, a monotonically + increasing long that tracks the order of snapshots in a table.""" + + +class TableMetadata: + """Helper class for parsing TableMetadata""" + + # Once this has been resolved, we can simplify this: https://github.com/samuelcolvin/pydantic/issues/3846 + # TableMetadata = Annotated[Union[TableMetadataV1, TableMetadataV2], Field(alias="format-version", discriminator="format-version")] + + @staticmethod + def parse_obj(data: dict) -> Union[TableMetadataV1, TableMetadataV2]: + if "format-version" not in data: + raise ValidationError(f"Missing format-version in TableMetadata: {data}") + + format_version = data["format-version"] + + if format_version == 1: + return TableMetadataV1(**data) + elif format_version == 2: + return TableMetadataV2(**data) + else: + raise ValidationError(f"Unknown format version: {format_version}") diff --git a/src/iceberg/table/refs.py b/src/iceberg/table/refs.py new file mode 100644 index 0000000000..285c5cfd93 --- /dev/null +++ b/src/iceberg/table/refs.py @@ -0,0 +1,37 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +from enum import Enum +from typing import Optional + +from pydantic import Field + +from iceberg.utils.iceberg_base_model import IcebergBaseModel + +MAIN_BRANCH = "main" + + +class SnapshotRefType(str, Enum): + BRANCH = "branch" + TAG = "tag" + + +class SnapshotRef(IcebergBaseModel): + snapshot_id: int = Field(alias="snapshot-id") + snapshot_ref_type: SnapshotRefType = Field(alias="type") + min_snapshots_to_keep: Optional[int] = Field(alias="min-snapshots-to-keep", default=None) + max_snapshot_age_ms: Optional[int] = Field(alias="max-snapshot-age-ms", default=None) + max_ref_age_ms: Optional[int] = Field(alias="max-ref-age-ms", default=None) diff --git a/src/iceberg/types.py b/src/iceberg/types.py index d3ae672373..ecfb65b10a 100644 --- a/src/iceberg/types.py +++ b/src/iceberg/types.py @@ -29,16 +29,25 @@ Notes: - https://iceberg.apache.org/#spec/#primitive-types """ -from abc import ABC -from dataclasses import dataclass, field -from functools import cached_property -from typing import ClassVar, Optional, Tuple - +import re +from typing import ( + ClassVar, + Dict, + Literal, + Optional, + Tuple, +) + +from pydantic import Field, PrivateAttr + +from iceberg.utils.iceberg_base_model import IcebergBaseModel from iceberg.utils.singleton import Singleton +DECIMAL_REGEX = re.compile(r"decimal\((\d+),\s*(\d+)\)") +FIXED_REGEX = re.compile(r"fixed\[(\d+)\]") + -@dataclass(frozen=True) -class IcebergType(ABC, Singleton): +class IcebergType(IcebergBaseModel, Singleton): """Base type for all Iceberg Types Example: @@ -48,50 +57,92 @@ class IcebergType(ABC, Singleton): 'IcebergType()' """ - @property - def string_type(self) -> str: - return self.__repr__() - - def __str__(self) -> str: - return self.string_type + @classmethod + def __get_validators__(cls): + # one or more validators may be yielded which will be called in the + # order to validate the input, each validator will receive as an input + # the value returned from the previous validator + yield cls.validate + + @classmethod + def validate(cls, v): + # When Pydantic is unable to determine the subtype + # In this case we'll help pydantic a bit by parsing the + # primitive type ourselves, or pointing it at the correct + # complex type by looking at the type field + + if isinstance(v, str): + if v.startswith("decimal"): + return DecimalType.parse(v) + elif v.startswith("fixed"): + return FixedType.parse(v) + else: + return PRIMITIVE_TYPES[v] + elif isinstance(v, dict): + if v.get("type") == "struct": + return StructType(**v) + elif v.get("type") == "list": + return ListType(**v) + elif v.get("type") == "map": + return MapType(**v) + else: + return NestedField(**v) + else: + return v @property def is_primitive(self) -> bool: return isinstance(self, PrimitiveType) -@dataclass(frozen=True, eq=True) class PrimitiveType(IcebergType): - """Base class for all Iceberg Primitive Types + """Base class for all Iceberg Primitive Types""" - Example: - >>> str(PrimitiveType()) - 'PrimitiveType()' - """ + __root__: str = Field() + + def __repr__(self) -> str: + return f"{type(self).__name__}()" + + def __str__(self) -> str: + return self.__root__ -@dataclass(frozen=True) class FixedType(PrimitiveType): """A fixed data type in Iceberg. - Example: >>> FixedType(8) FixedType(length=8) >>> FixedType(8) == FixedType(8) True + >>> FixedType(19) == FixedType(25) + False """ - length: int = field() + __root__: str = Field() + _length: int = PrivateAttr() + + @staticmethod + def parse(str_repr: str) -> "FixedType": + matches = FIXED_REGEX.search(str_repr) + if matches: + length = int(matches.group(1)) + return FixedType(length) + raise ValueError(f"Could not parse {str_repr} into a FixedType") + + def __init__(self, length: int): + super().__init__(__root__=f"fixed[{length}]") + self._length = length @property - def string_type(self) -> str: - return f"fixed[{self.length}]" + def length(self) -> int: + return self._length + + def __repr__(self) -> str: + return f"FixedType(length={self._length})" -@dataclass(frozen=True, eq=True) class DecimalType(PrimitiveType): """A fixed data type in Iceberg. - Example: >>> DecimalType(32, 3) DecimalType(precision=32, scale=3) @@ -99,15 +150,40 @@ class DecimalType(PrimitiveType): True """ - precision: int = field() - scale: int = field() + __root__: str = Field() + + _precision: int = PrivateAttr() + _scale: int = PrivateAttr() + + @staticmethod + def parse(str_repr: str) -> "DecimalType": + matches = DECIMAL_REGEX.search(str_repr) + if matches: + precision = int(matches.group(1)) + scale = int(matches.group(2)) + return DecimalType(precision, scale) + else: + raise ValueError(f"Could not parse {str_repr} into a DecimalType") + + def __init__(self, precision: int, scale: int): + super().__init__( + __root__=f"decimal({precision}, {scale})", + ) + self._precision = precision + self._scale = scale + + @property + def precision(self) -> int: + return self._precision @property - def string_type(self) -> str: - return f"decimal({self.precision}, {self.scale})" + def scale(self) -> int: + return self._scale + + def __repr__(self) -> str: + return f"DecimalType(precision={self._precision}, scale={self._scale})" -@dataclass(frozen=True) class NestedField(IcebergType): """Represents a field of a struct, a map key, a map value, or a list element. @@ -120,35 +196,51 @@ class NestedField(IcebergType): ... field_type=FixedType(22), ... required=False, ... )) - '1: foo: required fixed[22]' + '1: foo: optional fixed[22]' >>> str(NestedField( ... field_id=2, ... name='bar', ... field_type=LongType(), - ... required=False, + ... is_optional=False, ... doc="Just a long" ... )) '2: bar: required long (Just a long)' """ - field_id: int = field() - name: str = field() - field_type: IcebergType = field() - required: bool = field(default=True) - doc: Optional[str] = field(default=None, repr=False) + field_id: int = Field(alias="id") + name: str = Field() + field_type: IcebergType = Field(alias="type") + required: bool = Field(default=True) + doc: Optional[str] = Field(default=None, repr=False) + + def __init__( + self, + field_id: Optional[int] = None, + name: Optional[str] = None, + field_type: Optional[IcebergType] = None, + required: bool = True, + doc: Optional[str] = None, + **data, + ): + # We need an init when we want to use positional arguments, but + # need also to support the aliases. + data["field_id"] = data["id"] if "id" in data else field_id + data["name"] = name + data["field_type"] = data["type"] if "type" in data else field_type + data["required"] = required + data["doc"] = doc + super().__init__(**data) + + def __str__(self) -> str: + doc = "" if not self.doc else f" ({self.doc})" + req = "required" if self.required else "optional" + return f"{self.field_id}: {self.name}: {req} {self.field_type}{doc}" @property def optional(self) -> bool: return not self.required - @property - def string_type(self) -> str: - doc = "" if not self.doc else f" ({self.doc})" - req = "optional" if self.required else "required" - return f"{self.field_id}: {self.name}: {req} {self.field_type}{doc}" - -@dataclass(frozen=True, init=False) class StructType(IcebergType): """A struct type in Iceberg @@ -160,19 +252,22 @@ class StructType(IcebergType): 'struct<1: required_field: optional string, 2: optional_field: optional int>' """ - fields: Tuple[NestedField] = field() + type: Literal["struct"] = "struct" + fields: Tuple[NestedField, ...] = Field(default_factory=tuple) - def __init__(self, *fields: NestedField, **kwargs): # pylint: disable=super-init-not-called - if not fields and "fields" in kwargs: - fields = kwargs["fields"] - object.__setattr__(self, "fields", tuple(fields)) + def __init__(self, *fields: NestedField, **data): + # In case we use positional arguments, instead of keyword args + if fields: + data["fields"] = fields + super().__init__(**data) - @cached_property - def string_type(self) -> str: + def __str__(self) -> str: return f"struct<{', '.join(map(str, self.fields))}>" + def __repr__(self) -> str: + return f"StructType(fields=({', '.join(map(repr, self.fields))},))" + -@dataclass(frozen=True) class ListType(IcebergType): """A list type in Iceberg @@ -181,29 +276,33 @@ class ListType(IcebergType): ListType(element_id=3, element_type=StringType(), element_required=True) """ - element_id: int = field() - element_type: IcebergType = field() - element_required: bool = field(default=True) - element: NestedField = field(init=False, repr=False) - - def __post_init__(self): - object.__setattr__( - self, - "element", - NestedField( - name="element", - required=self.element_required, - field_id=self.element_id, - field_type=self.element_type, - ), + class Config: + fields = {"element_field": {"exclude": True}} + + type: Literal["list"] = "list" + element_id: int = Field(alias="element-id") + element_type: IcebergType = Field(alias="element") + element_required: bool = Field(alias="element-required", default=True) + element_field: NestedField = Field(init=False, repr=False) + + def __init__( + self, element_id: Optional[int] = None, element: Optional[IcebergType] = None, element_required: bool = True, **data + ): + data["element_id"] = data["element-id"] if "element-id" in data else element_id + data["element_type"] = element or data["element_type"] + data["element_required"] = data["element-required"] if "element-required" in data else element_required + data["element_field"] = NestedField( + name="element", + required=data["element_required"], + field_id=data["element_id"], + field_type=data["element_type"], ) + super().__init__(**data) - @property - def string_type(self) -> str: + def __str__(self) -> str: return f"list<{self.element_type}>" -@dataclass(frozen=True) class MapType(IcebergType): """A map type in Iceberg @@ -212,29 +311,43 @@ class MapType(IcebergType): MapType(key_id=1, key_type=StringType(), value_id=2, value_type=IntegerType(), value_required=True) """ - key_id: int = field() - key_type: IcebergType = field() - value_id: int = field() - value_type: IcebergType = field() - value_required: bool = field(default=True) - key: NestedField = field(init=False, repr=False) - value: NestedField = field(init=False, repr=False) - - def __post_init__(self): - object.__setattr__(self, "key", NestedField(name="key", field_id=self.key_id, field_type=self.key_type, required=False)) - object.__setattr__( - self, - "value", - NestedField( - name="value", - field_id=self.value_id, - field_type=self.value_type, - required=self.value_required, - ), + type: Literal["map"] = "map" + key_id: int = Field(alias="key-id") + key_type: IcebergType = Field(alias="key") + value_id: int = Field(alias="value-id") + value_type: IcebergType = Field(alias="value") + value_required: bool = Field(alias="value-required", default=True) + key_field: NestedField = Field(init=False, repr=False) + value_field: NestedField = Field(init=False, repr=False) + + class Config: + fields = {"key_field": {"exclude": True}, "value_field": {"exclude": True}} + + def __init__( + self, + key_id: Optional[int] = None, + key_type: Optional[IcebergType] = None, + value_id: Optional[int] = None, + value_type: Optional[IcebergType] = None, + value_required: bool = True, + **data, + ): + data["key_id"] = key_id or data["key-id"] + data["key_type"] = key_type or data["key"] + data["value_id"] = value_id or data["value-id"] + data["value_type"] = value_type or data["value"] + data["value_required"] = value_required if value_required is not None else data["value_required"] + + data["key_field"] = NestedField(name="key", field_id=data["key_id"], field_type=data["key_type"], required=True) + data["value_field"] = NestedField( + name="value", field_id=data["value_id"], field_type=data["value_type"], required=data["value_required"] ) + super().__init__(**data) + + def __str__(self) -> str: + return f"map<{self.key_type}, {self.value_type}>" -@dataclass(frozen=True) class BooleanType(PrimitiveType): """A boolean data type in Iceberg can be represented using an instance of this class. @@ -246,12 +359,9 @@ class BooleanType(PrimitiveType): BooleanType() """ - @property - def string_type(self) -> str: - return "boolean" + __root__ = "boolean" -@dataclass(frozen=True) class IntegerType(PrimitiveType): """An Integer data type in Iceberg can be represented using an instance of this class. Integers in Iceberg are 32-bit signed and can be promoted to Longs. @@ -271,12 +381,9 @@ class IntegerType(PrimitiveType): max: ClassVar[int] = 2147483647 min: ClassVar[int] = -2147483648 - @property - def string_type(self) -> str: - return "int" + __root__ = "int" -@dataclass(frozen=True) class LongType(PrimitiveType): """A Long data type in Iceberg can be represented using an instance of this class. Longs in Iceberg are 64-bit signed integers. @@ -300,12 +407,9 @@ class LongType(PrimitiveType): max: ClassVar[int] = 9223372036854775807 min: ClassVar[int] = -9223372036854775808 - @property - def string_type(self) -> str: - return "long" + __root__ = "long" -@dataclass(frozen=True) class FloatType(PrimitiveType): """A Float data type in Iceberg can be represented using an instance of this class. Floats in Iceberg are 32-bit IEEE 754 floating points and can be promoted to Doubles. @@ -327,12 +431,9 @@ class FloatType(PrimitiveType): max: ClassVar[float] = 3.4028235e38 min: ClassVar[float] = -3.4028235e38 - @property - def string_type(self) -> str: - return "float" + __root__ = "float" -@dataclass(frozen=True) class DoubleType(PrimitiveType): """A Double data type in Iceberg can be represented using an instance of this class. Doubles in Iceberg are 64-bit IEEE 754 floating points. @@ -345,12 +446,9 @@ class DoubleType(PrimitiveType): DoubleType() """ - @property - def string_type(self) -> str: - return "double" + __root__ = "double" -@dataclass(frozen=True) class DateType(PrimitiveType): """A Date data type in Iceberg can be represented using an instance of this class. Dates in Iceberg are calendar dates without a timezone or time. @@ -363,12 +461,9 @@ class DateType(PrimitiveType): DateType() """ - @property - def string_type(self) -> str: - return "date" + __root__ = "date" -@dataclass(frozen=True) class TimeType(PrimitiveType): """A Time data type in Iceberg can be represented using an instance of this class. Times in Iceberg have microsecond precision and are a time of day without a date or timezone. @@ -381,12 +476,9 @@ class TimeType(PrimitiveType): TimeType() """ - @property - def string_type(self) -> str: - return "time" + __root__ = "time" -@dataclass(frozen=True) class TimestampType(PrimitiveType): """A Timestamp data type in Iceberg can be represented using an instance of this class. Timestamps in Iceberg have microsecond precision and include a date and a time of day without a timezone. @@ -399,12 +491,9 @@ class TimestampType(PrimitiveType): TimestampType() """ - @property - def string_type(self) -> str: - return "timestamp" + __root__ = "timestamp" -@dataclass(frozen=True) class TimestamptzType(PrimitiveType): """A Timestamptz data type in Iceberg can be represented using an instance of this class. Timestamptzs in Iceberg are stored as UTC and include a date and a time of day with a timezone. @@ -417,12 +506,9 @@ class TimestamptzType(PrimitiveType): TimestamptzType() """ - @property - def string_type(self) -> str: - return "timestamptz" + __root__ = "timestamptz" -@dataclass(frozen=True) class StringType(PrimitiveType): """A String data type in Iceberg can be represented using an instance of this class. Strings in Iceberg are arbitrary-length character sequences and are encoded with UTF-8. @@ -435,12 +521,9 @@ class StringType(PrimitiveType): StringType() """ - @property - def string_type(self) -> str: - return "string" + __root__ = "string" -@dataclass(frozen=True) class UUIDType(PrimitiveType): """A UUID data type in Iceberg can be represented using an instance of this class. UUIDs in Iceberg are universally unique identifiers. @@ -453,12 +536,9 @@ class UUIDType(PrimitiveType): UUIDType() """ - @property - def string_type(self) -> str: - return "uuid" + __root__ = "uuid" -@dataclass(frozen=True) class BinaryType(PrimitiveType): """A Binary data type in Iceberg can be represented using an instance of this class. Binaries in Iceberg are arbitrary-length byte arrays. @@ -471,6 +551,20 @@ class BinaryType(PrimitiveType): BinaryType() """ - @property - def string_type(self) -> str: - return "binary" + __root__ = "binary" + + +PRIMITIVE_TYPES: Dict[str, PrimitiveType] = { + "boolean": BooleanType(), + "int": IntegerType(), + "long": LongType(), + "float": FloatType(), + "double": DoubleType(), + "date": DateType(), + "time": TimeType(), + "timestamp": TimestampType(), + "timestamptz": TimestamptzType(), + "string": StringType(), + "uuid": UUIDType(), + "binary": BinaryType(), +} diff --git a/src/iceberg/utils/iceberg_base_model.py b/src/iceberg/utils/iceberg_base_model.py new file mode 100644 index 0000000000..e218f9f89b --- /dev/null +++ b/src/iceberg/utils/iceberg_base_model.py @@ -0,0 +1,52 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +from functools import cached_property + +from pydantic import BaseModel + + +class IcebergBaseModel(BaseModel): + """ + This class extends the Pydantic BaseModel to set default values by overriding them. + + This is because we always want to set by_alias to True. In Python, the dash can't + be used in variable names, and this is used throughout the Iceberg spec. + + The same goes for exclude_none, if a field is None we want to omit it from + serialization, for example, the doc attribute on the NestedField object. + Default non-null values will be serialized. + + This is recommended by Pydantic: + https://pydantic-docs.helpmanual.io/usage/model_config/#change-behaviour-globally + """ + + class Config: + keep_untouched = (cached_property,) + allow_population_by_field_name = True + frozen = True + + def dict(self, exclude_none: bool = True, **kwargs): + return super().dict(exclude_none=exclude_none, **kwargs) + + def json(self, exclude_none: bool = True, by_alias: bool = True, **kwargs): + # A small trick to exclude private properties. Properties are serialized by pydantic, + # regardless if they start with an underscore. + # This will look at the dict, and find the fields and exclude them + exclude = set.union( + {field for field in self.__dict__ if field.startswith("_") and not field == "__root__"}, kwargs.get("exclude", set()) + ) + return super().json(exclude_none=exclude_none, exclude=exclude, by_alias=by_alias, **kwargs) diff --git a/src/iceberg/utils/singleton.py b/src/iceberg/utils/singleton.py index 5643cdd172..4dd48d5867 100644 --- a/src/iceberg/utils/singleton.py +++ b/src/iceberg/utils/singleton.py @@ -26,16 +26,23 @@ return it. More information on metaclasses: https://docs.python.org/3/reference/datamodel.html#metaclasses - """ -from typing import ClassVar, Dict +from typing import Any, ClassVar, Dict + + +def _convert_to_hashable_type(element: Any) -> Any: + if isinstance(element, dict): + return tuple((_convert_to_hashable_type(k), _convert_to_hashable_type(v)) for k, v in element.items()) + elif isinstance(element, list): + return tuple(map(_convert_to_hashable_type, element)) + return element class Singleton: _instances: ClassVar[Dict] = {} def __new__(cls, *args, **kwargs): - key = (cls, args, tuple(sorted(kwargs.items()))) + key = (cls, tuple(args), _convert_to_hashable_type(kwargs)) if key not in cls._instances: cls._instances[key] = super().__new__(cls) return cls._instances[key] diff --git a/tests/avro/test_reader.py b/tests/avro/test_reader.py index e8c56f122e..6ea8377e11 100644 --- a/tests/avro/test_reader.py +++ b/tests/avro/test_reader.py @@ -448,7 +448,7 @@ def test_binary_reader(): def test_unknown_type(): class UnknownType(PrimitiveType): - ... + __root__ = "UnknownType" with pytest.raises(ValueError) as exc_info: primitive_reader(UnknownType()) diff --git a/tests/conftest.py b/tests/conftest.py index 782316ec91..d211fd13dc 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -15,17 +15,34 @@ # specific language governing permissions and limitations # under the License. # pylint:disable=redefined-outer-name +"""This contains global pytest configurations. +Fixtures contained in this file will be automatically used if provided as an argument +to any pytest function. + +In the case where the fixture must be used in a pytest.mark.parametrize decorator, the string representation can be used +and the built-in pytest fixture request should be used as an additional argument in the function. The fixture can then be +retrieved using `request.getfixturevalue(fixture_name)`. +""" +import os from tempfile import TemporaryDirectory -from typing import Any, Dict +from typing import Any, Dict, Union +from urllib.parse import urlparse import pytest from iceberg import schema +from iceberg.io.base import ( + FileIO, + InputFile, + OutputFile, + OutputStream, +) from iceberg.schema import Schema from iceberg.types import ( BinaryType, BooleanType, + DoubleType, FloatType, IntegerType, ListType, @@ -36,6 +53,7 @@ StructType, ) from tests.catalog.test_base import InMemoryCatalog +from tests.io.test_io_base import LocalInputFile class FooStruct: @@ -768,6 +786,72 @@ def avro_schema_manifest_entry() -> Dict[str, Any]: } +@pytest.fixture(scope="session") +def simple_struct(): + return StructType( + NestedField(id=1, name="required_field", field_type=StringType(), required=True, doc="this is a doc"), + NestedField(id=2, name="optional_field", field_type=IntegerType()), + ) + + +@pytest.fixture(scope="session") +def simple_list(): + return ListType(element_id=22, element=StringType(), element_required=True) + + +@pytest.fixture(scope="session") +def simple_map(): + return MapType(key_id=19, key_type=StringType(), value_id=25, value_type=DoubleType(), value_required=False) + + +class LocalOutputFile(OutputFile): + """An OutputFile implementation for local files (for test use only)""" + + def __init__(self, location: str): + parsed_location = urlparse(location) # Create a ParseResult from the uri + if parsed_location.scheme and parsed_location.scheme != "file": # Validate that a uri is provided with a scheme of `file` + raise ValueError("LocalOutputFile location must have a scheme of `file`") + elif parsed_location.netloc: + raise ValueError(f"Network location is not allowed for LocalOutputFile: {parsed_location.netloc}") + + super().__init__(location=location) + self._path = parsed_location.path + + def __len__(self): + return os.path.getsize(self._path) + + def exists(self): + return os.path.exists(self._path) + + def to_input_file(self): + return LocalInputFile(location=self.location) + + def create(self, overwrite: bool = False) -> OutputStream: + output_file = open(self._path, "wb" if overwrite else "xb") + if not isinstance(output_file, OutputStream): + raise TypeError("Object returned from LocalOutputFile.create(...) does not match the OutputStream protocol.") + return output_file + + +class LocalFileIO(FileIO): + """A FileIO implementation for local files (for test use only)""" + + def new_input(self, location: str): + return LocalInputFile(location=location) + + def new_output(self, location: str): + return LocalOutputFile(location=location) + + def delete(self, location: Union[str, InputFile, OutputFile]): + location = location.location if isinstance(location, (InputFile, OutputFile)) else location + os.remove(location) + + +@pytest.fixture(scope="session", autouse=True) +def LocalFileIOFixture(): + return LocalFileIO + + @pytest.fixture(scope="session") def generated_manifest_entry_file(avro_schema_manifest_entry): from fastavro import parse_schema, writer diff --git a/tests/table/test_metadata.py b/tests/table/test_metadata.py new file mode 100644 index 0000000000..7e0613b3db --- /dev/null +++ b/tests/table/test_metadata.py @@ -0,0 +1,522 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +import io +import json +from uuid import UUID + +import pytest + +from iceberg.exceptions import ValidationError +from iceberg.schema import Schema +from iceberg.serializers import FromByteStream +from iceberg.table.metadata import TableMetadata, TableMetadataV1, TableMetadataV2 +from iceberg.table.refs import SnapshotRef, SnapshotRefType +from iceberg.types import LongType, NestedField + +EXAMPLE_TABLE_METADATA_V1 = { + "format-version": 1, + "table-uuid": "d20125c8-7284-442c-9aea-15fee620737c", + "location": "s3://bucket/test/location", + "last-updated-ms": 1602638573874, + "last-column-id": 3, + "schema": { + "type": "struct", + "fields": [ + {"id": 1, "name": "x", "required": True, "type": "long"}, + {"id": 2, "name": "y", "required": True, "type": "long", "doc": "comment"}, + {"id": 3, "name": "z", "required": True, "type": "long"}, + ], + }, + "partition-spec": [{"name": "x", "transform": "identity", "source-id": 1, "field-id": 1000}], + "properties": {}, + "current-snapshot-id": -1, + "snapshots": [{"snapshot-id": 1925, "timestamp-ms": 1602638573822}], +} +EXAMPLE_TABLE_METADATA_V2 = { + "format-version": 2, + "table-uuid": "9c12d441-03fe-4693-9a96-a0705ddf69c1", + "location": "s3://bucket/test/location", + "last-sequence-number": 34, + "last-updated-ms": 1602638573590, + "last-column-id": 3, + "current-schema-id": 1, + "schemas": [ + {"type": "struct", "schema-id": 0, "fields": [{"id": 1, "name": "x", "required": True, "type": "long"}]}, + { + "type": "struct", + "schema-id": 1, + "identifier-field-ids": [1, 2], + "fields": [ + {"id": 1, "name": "x", "required": True, "type": "long"}, + {"id": 2, "name": "y", "required": True, "type": "long", "doc": "comment"}, + {"id": 3, "name": "z", "required": True, "type": "long"}, + ], + }, + ], + "default-spec-id": 0, + "partition-specs": [{"spec-id": 0, "fields": [{"name": "x", "transform": "identity", "source-id": 1, "field-id": 1000}]}], + "last-partition-id": 1000, + "default-sort-order-id": 3, + "sort-orders": [ + { + "order-id": 3, + "fields": [ + {"transform": "identity", "source-id": 2, "direction": "asc", "null-order": "nulls-first"}, + {"transform": "bucket[4]", "source-id": 3, "direction": "desc", "null-order": "nulls-last"}, + ], + } + ], + "properties": {"read.split.target.size": 134217728}, + "current-snapshot-id": 3055729675574597004, + "snapshots": [ + { + "snapshot-id": 3051729675574597004, + "timestamp-ms": 1515100955770, + "sequence-number": 0, + "summary": {"operation": "append"}, + "manifest-list": "s3://a/b/1.avro", + }, + { + "snapshot-id": 3055729675574597004, + "parent-snapshot-id": 3051729675574597004, + "timestamp-ms": 1555100955770, + "sequence-number": 1, + "summary": {"operation": "append"}, + "manifest-list": "s3://a/b/2.avro", + "schema-id": 1, + }, + ], + "snapshot-log": [ + {"snapshot-id": 3051729675574597004, "timestamp-ms": 1515100955770}, + {"snapshot-id": 3055729675574597004, "timestamp-ms": 1555100955770}, + ], + "metadata-log": [], +} + + +@pytest.mark.parametrize( + "metadata", + [ + EXAMPLE_TABLE_METADATA_V1, + EXAMPLE_TABLE_METADATA_V2, + ], +) +def test_from_dict(metadata: dict): + """Test initialization of a TableMetadata instance from a dictionary""" + TableMetadata.parse_obj(metadata) + + +def test_from_byte_stream(): + """Test generating a TableMetadata instance from a file-like byte stream""" + data = bytes(json.dumps(EXAMPLE_TABLE_METADATA_V2), encoding="utf-8") + byte_stream = io.BytesIO(data) + FromByteStream.table_metadata(byte_stream=byte_stream) + + +def test_v2_metadata_parsing(): + """Test retrieving values from a TableMetadata instance of version 2""" + table_metadata = TableMetadata.parse_obj(EXAMPLE_TABLE_METADATA_V2) + + assert table_metadata.format_version == 2 + assert table_metadata.table_uuid == UUID("9c12d441-03fe-4693-9a96-a0705ddf69c1") + assert table_metadata.location == "s3://bucket/test/location" + assert table_metadata.last_sequence_number == 34 + assert table_metadata.last_updated_ms == 1602638573590 + assert table_metadata.last_column_id == 3 + assert table_metadata.schemas[0].schema_id == 0 + assert table_metadata.current_schema_id == 1 + assert table_metadata.partition_specs[0]["spec-id"] == 0 + assert table_metadata.default_spec_id == 0 + assert table_metadata.last_partition_id == 1000 + assert table_metadata.properties["read.split.target.size"] == "134217728" + assert table_metadata.current_snapshot_id == 3055729675574597004 + assert table_metadata.snapshots[0]["snapshot-id"] == 3051729675574597004 + assert table_metadata.snapshot_log[0]["timestamp-ms"] == 1515100955770 + assert table_metadata.sort_orders[0]["order-id"] == 3 + assert table_metadata.default_sort_order_id == 3 + + +def test_v1_metadata_parsing_directly(): + """Test retrieving values from a TableMetadata instance of version 1""" + table_metadata = TableMetadataV1(**EXAMPLE_TABLE_METADATA_V1) + + assert isinstance(table_metadata, TableMetadataV1) + + # The version 1 will automatically be bumped to version 2 + assert table_metadata.format_version == 1 + assert table_metadata.table_uuid == UUID("d20125c8-7284-442c-9aea-15fee620737c") + assert table_metadata.location == "s3://bucket/test/location" + assert table_metadata.last_updated_ms == 1602638573874 + assert table_metadata.last_column_id == 3 + assert table_metadata.schemas[0].schema_id == 0 + assert table_metadata.current_schema_id == 0 + assert table_metadata.default_spec_id == 0 + assert table_metadata.last_partition_id == 1000 + assert table_metadata.current_snapshot_id is None + assert table_metadata.default_sort_order_id == 0 + + +def test_parsing_correct_types(): + table_metadata = TableMetadataV2(**EXAMPLE_TABLE_METADATA_V2) + assert isinstance(table_metadata.schemas[0], Schema) + assert isinstance(table_metadata.schemas[0].fields[0], NestedField) + assert isinstance(table_metadata.schemas[0].fields[0].field_type, LongType) + + +def test_updating_metadata(): + """Test creating a new TableMetadata instance that's an updated version of + an existing TableMetadata instance""" + table_metadata = TableMetadataV2(**EXAMPLE_TABLE_METADATA_V2) + + new_schema = { + "type": "struct", + "schema-id": 1, + "fields": [ + {"id": 1, "name": "foo", "required": True, "type": "string"}, + {"id": 2, "name": "bar", "required": True, "type": "string"}, + {"id": 3, "name": "baz", "required": True, "type": "string"}, + ], + } + + mutable_table_metadata = table_metadata.dict() + mutable_table_metadata["schemas"].append(new_schema) + mutable_table_metadata["current-schema-id"] = 1 + + new_table_metadata = TableMetadataV2(**mutable_table_metadata) + + assert new_table_metadata.current_schema_id == 1 + assert new_table_metadata.schemas[-1] == Schema(**new_schema) + + +def test_serialize_v1(): + table_metadata = TableMetadataV1(**EXAMPLE_TABLE_METADATA_V1).json() + assert ( + table_metadata + == """{"location": "s3://bucket/test/location", "table-uuid": "d20125c8-7284-442c-9aea-15fee620737c", "last-updated-ms": 1602638573874, "last-column-id": 3, "schemas": [{"fields": [{"id": 1, "name": "x", "type": "long", "required": true}, {"id": 2, "name": "y", "type": "long", "required": true, "doc": "comment"}, {"id": 3, "name": "z", "type": "long", "required": true}], "schema-id": 0, "identifier-field-ids": []}], "current-schema-id": 0, "partition-specs": [{"spec-id": 0, "fields": [{"name": "x", "transform": "identity", "source-id": 1, "field-id": 1000}]}], "default-spec-id": 0, "last-partition-id": 1000, "properties": {}, "snapshots": [{"snapshot-id": 1925, "timestamp-ms": 1602638573822}], "snapshot-log": [], "metadata-log": [], "sort-orders": [{"order_id": 0, "fields": []}], "default-sort-order-id": 0, "refs": {}, "format-version": 1, "schema": {"fields": [{"id": 1, "name": "x", "type": "long", "required": true}, {"id": 2, "name": "y", "type": "long", "required": true, "doc": "comment"}, {"id": 3, "name": "z", "type": "long", "required": true}], "schema-id": 0, "identifier-field-ids": []}, "partition-spec": [{"name": "x", "transform": "identity", "source-id": 1, "field-id": 1000}]}""" + ) + + +def test_serialize_v2(): + table_metadata = TableMetadataV2(**EXAMPLE_TABLE_METADATA_V2).json() + assert ( + table_metadata + == """{"location": "s3://bucket/test/location", "table-uuid": "9c12d441-03fe-4693-9a96-a0705ddf69c1", "last-updated-ms": 1602638573590, "last-column-id": 3, "schemas": [{"fields": [{"id": 1, "name": "x", "type": "long", "required": true}], "schema-id": 0, "identifier-field-ids": []}, {"fields": [{"id": 1, "name": "x", "type": "long", "required": true}, {"id": 2, "name": "y", "type": "long", "required": true, "doc": "comment"}, {"id": 3, "name": "z", "type": "long", "required": true}], "schema-id": 1, "identifier-field-ids": [1, 2]}], "current-schema-id": 1, "partition-specs": [{"spec-id": 0, "fields": [{"name": "x", "transform": "identity", "source-id": 1, "field-id": 1000}]}], "default-spec-id": 0, "last-partition-id": 1000, "properties": {"read.split.target.size": "134217728"}, "current-snapshot-id": 3055729675574597004, "snapshots": [{"snapshot-id": 3051729675574597004, "timestamp-ms": 1515100955770, "sequence-number": 0, "summary": {"operation": "append"}, "manifest-list": "s3://a/b/1.avro"}, {"snapshot-id": 3055729675574597004, "parent-snapshot-id": 3051729675574597004, "timestamp-ms": 1555100955770, "sequence-number": 1, "summary": {"operation": "append"}, "manifest-list": "s3://a/b/2.avro", "schema-id": 1}], "snapshot-log": [{"snapshot-id": 3051729675574597004, "timestamp-ms": 1515100955770}, {"snapshot-id": 3055729675574597004, "timestamp-ms": 1555100955770}], "metadata-log": [], "sort-orders": [{"order-id": 3, "fields": [{"transform": "identity", "source-id": 2, "direction": "asc", "null-order": "nulls-first"}, {"transform": "bucket[4]", "source-id": 3, "direction": "desc", "null-order": "nulls-last"}]}], "default-sort-order-id": 3, "refs": {"main": {"snapshot-id": 3055729675574597004, "type": "branch"}}, "format-version": 2, "last-sequence-number": 34}""" + ) + + +def test_migrate_v1_schemas(): + table_metadata = TableMetadataV1(**EXAMPLE_TABLE_METADATA_V1) + + assert isinstance(table_metadata, TableMetadataV1) + assert len(table_metadata.schemas) == 1 + assert table_metadata.schemas[0] == table_metadata.schema_ + + +def test_migrate_v1_partition_specs(): + # Copy the example, and add a spec + table_metadata = TableMetadataV1(**EXAMPLE_TABLE_METADATA_V1) + assert isinstance(table_metadata, TableMetadataV1) + assert len(table_metadata.partition_specs) == 1 + # Spec ID gets added automatically + assert table_metadata.partition_specs == [ + {"spec-id": 0, "fields": [{"field-id": 1000, "name": "x", "source-id": 1, "transform": "identity"}]} + ] + + +def test_invalid_format_version(): + """Test the exception when trying to load an unknown version""" + table_metadata_invalid_format_version = { + "format-version": -1, + "table-uuid": "d20125c8-7284-442c-9aea-15fee620737c", + "location": "s3://bucket/test/location", + "last-updated-ms": 1602638573874, + "last-column-id": 3, + "schema": { + "type": "struct", + "fields": [ + {"id": 1, "name": "x", "required": True, "type": "long"}, + {"id": 2, "name": "y", "required": True, "type": "long", "doc": "comment"}, + {"id": 3, "name": "z", "required": True, "type": "long"}, + ], + }, + "partition-spec": [{"name": "x", "transform": "identity", "source-id": 1, "field-id": 1000}], + "properties": {}, + "current-snapshot-id": -1, + "snapshots": [], + } + + with pytest.raises(ValidationError) as exc_info: + TableMetadata.parse_obj(table_metadata_invalid_format_version) + + assert "Unknown format version: -1" in str(exc_info.value) + + +def test_current_schema_not_found(): + """Test that we raise an exception when the schema can't be found""" + + table_metadata_schema_not_found = { + "format-version": 2, + "table-uuid": "d20125c8-7284-442c-9aea-15fee620737c", + "location": "s3://bucket/test/location", + "last-updated-ms": 1602638573874, + "last-column-id": 3, + "schemas": [ + {"type": "struct", "schema-id": 0, "fields": [{"id": 1, "name": "x", "required": True, "type": "long"}]}, + { + "type": "struct", + "schema-id": 1, + "identifier-field-ids": [1, 2], + "fields": [ + {"id": 1, "name": "x", "required": True, "type": "long"}, + {"id": 2, "name": "y", "required": True, "type": "long", "doc": "comment"}, + {"id": 3, "name": "z", "required": True, "type": "long"}, + ], + }, + ], + "current-schema-id": 2, + "default-spec-id": 0, + "partition-specs": [{"spec-id": 0, "fields": [{"name": "x", "transform": "identity", "source-id": 1, "field-id": 1000}]}], + "last-partition-id": 1000, + "default-sort-order-id": 0, + "properties": {}, + "current-snapshot-id": -1, + "snapshots": [], + } + + with pytest.raises(ValidationError) as exc_info: + TableMetadata.parse_obj(table_metadata_schema_not_found) + + assert "current-schema-id 2 can't be found in the schemas" in str(exc_info.value) + + +def test_sort_order_not_found(): + """Test that we raise an exception when the schema can't be found""" + + table_metadata_schema_not_found = { + "format-version": 2, + "table-uuid": "d20125c8-7284-442c-9aea-15fee620737c", + "location": "s3://bucket/test/location", + "last-updated-ms": 1602638573874, + "last-column-id": 3, + "schemas": [ + { + "type": "struct", + "schema-id": 0, + "identifier-field-ids": [1, 2], + "fields": [ + {"id": 1, "name": "x", "required": True, "type": "long"}, + {"id": 2, "name": "y", "required": True, "type": "long", "doc": "comment"}, + {"id": 3, "name": "z", "required": True, "type": "long"}, + ], + }, + ], + "default-sort-order-id": 4, + "sort-orders": [ + { + "order-id": 3, + "fields": [ + {"transform": "identity", "source-id": 2, "direction": "asc", "null-order": "nulls-first"}, + {"transform": "bucket[4]", "source-id": 3, "direction": "desc", "null-order": "nulls-last"}, + ], + } + ], + "current-schema-id": 0, + "default-spec-id": 0, + "partition-specs": [{"spec-id": 0, "fields": [{"name": "x", "transform": "identity", "source-id": 1, "field-id": 1000}]}], + "last-partition-id": 1000, + "properties": {}, + "current-snapshot-id": -1, + "snapshots": [], + } + + with pytest.raises(ValidationError) as exc_info: + TableMetadata.parse_obj(table_metadata_schema_not_found) + + assert "default-sort-order-id 4 can't be found" in str(exc_info.value) + + +def test_sort_order_unsorted(): + """Test that we raise an exception when the schema can't be found""" + + table_metadata_schema_not_found = { + "format-version": 2, + "table-uuid": "d20125c8-7284-442c-9aea-15fee620737c", + "location": "s3://bucket/test/location", + "last-updated-ms": 1602638573874, + "last-column-id": 3, + "schemas": [ + { + "type": "struct", + "schema-id": 0, + "identifier-field-ids": [1, 2], + "fields": [ + {"id": 1, "name": "x", "required": True, "type": "long"}, + {"id": 2, "name": "y", "required": True, "type": "long", "doc": "comment"}, + {"id": 3, "name": "z", "required": True, "type": "long"}, + ], + }, + ], + "default-sort-order-id": 0, + "sort-orders": [], + "current-schema-id": 0, + "default-spec-id": 0, + "partition-specs": [{"spec-id": 0, "fields": [{"name": "x", "transform": "identity", "source-id": 1, "field-id": 1000}]}], + "last-partition-id": 1000, + "properties": {}, + "current-snapshot-id": -1, + "snapshots": [], + } + + table_metadata = TableMetadata.parse_obj(table_metadata_schema_not_found) + + # Most important here is that we correctly handle sort-order-id 0 + assert len(table_metadata.sort_orders) == 0 + + +def test_invalid_partition_spec(): + table_metadata_spec_not_found = { + "format-version": 2, + "table-uuid": "9c12d441-03fe-4693-9a96-a0705ddf69c1", + "location": "s3://bucket/test/location", + "last-sequence-number": 34, + "last-updated-ms": 1602638573590, + "last-column-id": 3, + "current-schema-id": 1, + "schemas": [ + {"type": "struct", "schema-id": 0, "fields": [{"id": 1, "name": "x", "required": True, "type": "long"}]}, + { + "type": "struct", + "schema-id": 1, + "identifier-field-ids": [1, 2], + "fields": [ + {"id": 1, "name": "x", "required": True, "type": "long"}, + {"id": 2, "name": "y", "required": True, "type": "long", "doc": "comment"}, + {"id": 3, "name": "z", "required": True, "type": "long"}, + ], + }, + ], + "sort-orders": [], + "default-sort-order-id": 0, + "default-spec-id": 1, + "partition-specs": [{"spec-id": 0, "fields": [{"name": "x", "transform": "identity", "source-id": 1, "field-id": 1000}]}], + "last-partition-id": 1000, + } + with pytest.raises(ValidationError) as exc_info: + TableMetadata.parse_obj(table_metadata_spec_not_found) + + assert "default-spec-id 1 can't be found" in str(exc_info.value) + + +def test_v1_writing_metadata(): + """ + https://iceberg.apache.org/spec/#version-2 + + Writing v1 metadata: + - Table metadata field last-sequence-number should not be written + """ + + table_metadata = TableMetadataV1(**EXAMPLE_TABLE_METADATA_V1) + metadata_v1_json = table_metadata.json() + metadata_v1 = json.loads(metadata_v1_json) + + assert "last-sequence-number" not in metadata_v1 + + +def test_v1_metadata_for_v2(): + """ + https://iceberg.apache.org/spec/#version-2 + + Reading v1 metadata for v2: + - Table metadata field last-sequence-number must default to 0 + """ + + table_metadata = TableMetadataV1(**EXAMPLE_TABLE_METADATA_V1).to_v2() + + assert table_metadata.last_sequence_number == 0 + + +def test_v1_write_metadata_for_v2(): + """ + https://iceberg.apache.org/spec/#version-2 + + Table metadata JSON: + - last-sequence-number was added and is required; default to 0 when reading v1 metadata + - table-uuid is now required + - current-schema-id is now required + - schemas is now required + - partition-specs is now required + - default-spec-id is now required + - last-partition-id is now required + - sort-orders is now required + - default-sort-order-id is now required + - schema is no longer required and should be omitted; use schemas and current-schema-id instead + - partition-spec is no longer required and should be omitted; use partition-specs and default-spec-id instead + """ + + minimal_example_v1 = { + "format-version": 1, + "location": "s3://bucket/test/location", + "last-updated-ms": 1602638573874, + "last-column-id": 3, + "schema": { + "type": "struct", + "fields": [ + {"id": 1, "name": "x", "required": True, "type": "long"}, + {"id": 2, "name": "y", "required": True, "type": "long", "doc": "comment"}, + {"id": 3, "name": "z", "required": True, "type": "long"}, + ], + }, + "partition-spec": [{"name": "x", "transform": "identity", "source-id": 1, "field-id": 1000}], + "properties": {}, + "current-snapshot-id": -1, + "snapshots": [{"snapshot-id": 1925, "timestamp-ms": 1602638573822}], + } + + table_metadata = TableMetadataV1(**minimal_example_v1).to_v2() + metadata_v2_json = table_metadata.json() + metadata_v2 = json.loads(metadata_v2_json) + + assert metadata_v2["last-sequence-number"] == 0 + assert UUID(metadata_v2["table-uuid"]) is not None + assert metadata_v2["current-schema-id"] == 0 + assert metadata_v2["schemas"] == [ + { + "fields": [ + {"id": 1, "name": "x", "required": True, "type": "long"}, + {"doc": "comment", "id": 2, "name": "y", "required": True, "type": "long"}, + {"id": 3, "name": "z", "required": True, "type": "long"}, + ], + "identifier-field-ids": [], + "schema-id": 0, + } + ] + assert metadata_v2["partition-specs"] == [ + {"spec-id": 0, "fields": [{"name": "x", "transform": "identity", "source-id": 1, "field-id": 1000}]} + ] + assert metadata_v2["default-spec-id"] == 0 + assert metadata_v2["last-partition-id"] == 1000 + assert metadata_v2["sort-orders"] == [{"fields": [], "order_id": 0}] + assert metadata_v2["default-sort-order-id"] == 0 + # Deprecated fields + assert "schema" not in metadata_v2 + assert "partition-spec" not in metadata_v2 + + +def test_v2_ref_creation(): + table_metadata = TableMetadataV2(**EXAMPLE_TABLE_METADATA_V2) + assert table_metadata.refs == {"main": SnapshotRef(snapshot_id=3055729675574597004, snapshot_ref_type=SnapshotRefType.BRANCH)} diff --git a/tests/test_schema.py b/tests/test_schema.py index 81ebcf3556..a4f93c1817 100644 --- a/tests/test_schema.py +++ b/tests/test_schema.py @@ -23,7 +23,7 @@ from iceberg import schema from iceberg.expressions.base import Accessor from iceberg.files import StructProtocol -from iceberg.schema import build_position_accessors +from iceberg.schema import Schema, build_position_accessors from iceberg.types import ( BooleanType, FloatType, @@ -36,14 +36,14 @@ ) -def test_schema_str(table_schema_simple): +def test_schema_str(table_schema_simple: Schema): """Test casting a schema to a string""" assert str(table_schema_simple) == dedent( """\ table { - 1: foo: required string - 2: bar: optional int - 3: baz: required boolean + 1: foo: optional string + 2: bar: required int + 3: baz: optional boolean }""" ) @@ -61,7 +61,7 @@ def test_schema_str(table_schema_simple): ), ], ) -def test_schema_repr(schema_repr, expected_repr): +def test_schema_repr(schema_repr: Schema, expected_repr: str): """Test schema representation""" assert repr(schema_repr) == expected_repr @@ -107,8 +107,8 @@ def test_schema_index_by_id_visitor(table_schema_nested): ), required=True, ), - 7: NestedField(field_id=7, name="key", field_type=StringType(), required=False), - 9: NestedField(field_id=9, name="key", field_type=StringType(), required=False), + 7: NestedField(field_id=7, name="key", field_type=StringType(), required=True), + 9: NestedField(field_id=9, name="key", field_type=StringType(), required=True), 8: NestedField( field_id=8, name="value", @@ -274,14 +274,14 @@ def test_index_by_id_schema_visitor(table_schema_nested): ), required=True, ), - 7: NestedField(field_id=7, name="key", field_type=StringType(), required=False), + 7: NestedField(field_id=7, name="key", field_type=StringType(), required=True), 8: NestedField( field_id=8, name="value", field_type=MapType(key_id=9, key_type=StringType(), value_id=10, value_type=IntegerType(), value_required=True), required=True, ), - 9: NestedField(field_id=9, name="key", field_type=StringType(), required=False), + 9: NestedField(field_id=9, name="key", field_type=StringType(), required=True), 10: NestedField(field_id=10, name="value", field_type=IntegerType(), required=True), 11: NestedField( field_id=11, @@ -386,7 +386,7 @@ def test_build_position_accessors(table_schema_nested): } -def test_build_position_accessors_with_struct(table_schema_nested): +def test_build_position_accessors_with_struct(table_schema_nested: Schema): class TestStruct(StructProtocol): def __init__(self, pos: Dict[int, Any] = None): self._pos: Dict[int, Any] = pos or {} @@ -399,4 +399,20 @@ def get(self, pos: int) -> Any: accessors = build_position_accessors(table_schema_nested) container = TestStruct({6: TestStruct({0: "name"})}) - assert accessors.get(16).get(container) == "name" + inner_accessor = accessors.get(16) + assert inner_accessor + assert inner_accessor.get(container) == "name" + + +def test_serialize_schema(table_schema_simple: Schema): + actual = table_schema_simple.json() + expected = """{"fields": [{"id": 1, "name": "foo", "type": "string", "required": false}, {"id": 2, "name": "bar", "type": "int", "required": true}, {"id": 3, "name": "baz", "type": "boolean", "required": false}], "schema-id": 1, "identifier-field-ids": [1]}""" + assert actual == expected + + +def test_deserialize_schema(table_schema_simple: Schema): + actual = Schema.parse_raw( + """{"fields": [{"id": 1, "name": "foo", "type": "string", "required": false}, {"id": 2, "name": "bar", "type": "int", "required": true}, {"id": 3, "name": "baz", "type": "boolean", "required": false}], "schema-id": 1, "identifier-field-ids": [1]}""" + ) + expected = table_schema_simple + assert actual == expected diff --git a/tests/test_types.py b/tests/test_types.py index 3d75b88a58..1695f26571 100644 --- a/tests/test_types.py +++ b/tests/test_types.py @@ -26,6 +26,7 @@ DoubleType, FixedType, FloatType, + IcebergType, IntegerType, ListType, LongType, @@ -38,6 +39,7 @@ TimeType, UUIDType, ) +from iceberg.utils.iceberg_base_model import IcebergBaseModel non_parameterized_types = [ (1, BooleanType), @@ -146,9 +148,9 @@ def test_list_type(): ), False, ) - assert isinstance(type_var.element.field_type, StructType) - assert len(type_var.element.field_type.fields) == 2 - assert type_var.element.field_id == 1 + assert isinstance(type_var.element_field.field_type, StructType) + assert len(type_var.element_field.field_type.fields) == 2 + assert type_var.element_field.field_id == 1 assert str(type_var) == str(eval(repr(type_var))) assert type_var == eval(repr(type_var)) assert type_var != ListType( @@ -162,10 +164,10 @@ def test_list_type(): def test_map_type(): type_var = MapType(1, DoubleType(), 2, UUIDType(), False) - assert isinstance(type_var.key.field_type, DoubleType) - assert type_var.key.field_id == 1 - assert isinstance(type_var.value.field_type, UUIDType) - assert type_var.value.field_id == 2 + assert isinstance(type_var.key_field.field_type, DoubleType) + assert type_var.key_field.field_id == 1 + assert isinstance(type_var.value_field.field_type, UUIDType) + assert type_var.value_field.field_id == 2 assert str(type_var) == str(eval(repr(type_var))) assert type_var == eval(repr(type_var)) assert type_var != MapType(1, LongType(), 2, UUIDType(), False) @@ -206,6 +208,390 @@ def test_non_parameterized_type_equality(input_index, input_type, check_index, c assert input_type() != check_type() +# Examples based on https://iceberg.apache.org/spec/#appendix-c-json-serialization + + +class TestType(IcebergBaseModel): + __root__: IcebergType + + +def test_serialization_boolean(): + assert BooleanType().json() == '"boolean"' + + +def test_deserialization_boolean(): + assert TestType.parse_raw('"boolean"') == BooleanType() + + +def test_str_boolean(): + assert str(BooleanType()) == "boolean" + + +def test_repr_boolean(): + assert repr(BooleanType()) == "BooleanType()" + + +def test_serialization_int(): + assert IntegerType().json() == '"int"' + + +def test_deserialization_int(): + assert TestType.parse_raw('"int"') == IntegerType() + + +def test_str_int(): + assert str(IntegerType()) == "int" + + +def test_repr_int(): + assert repr(IntegerType()) == "IntegerType()" + + +def test_serialization_long(): + assert LongType().json() == '"long"' + + +def test_deserialization_long(): + assert TestType.parse_raw('"long"') == LongType() + + +def test_str_long(): + assert str(LongType()) == "long" + + +def test_repr_long(): + assert repr(LongType()) == "LongType()" + + +def test_serialization_float(): + assert FloatType().json() == '"float"' + + +def test_deserialization_float(): + assert TestType.parse_raw('"float"') == FloatType() + + +def test_str_float(): + assert str(FloatType()) == "float" + + +def test_repr_float(): + assert repr(FloatType()) == "FloatType()" + + +def test_serialization_double(): + assert DoubleType().json() == '"double"' + + +def test_deserialization_double(): + assert TestType.parse_raw('"double"') == DoubleType() + + +def test_str_double(): + assert str(DoubleType()) == "double" + + +def test_repr_double(): + assert repr(DoubleType()) == "DoubleType()" + + +def test_serialization_date(): + assert DateType().json() == '"date"' + + +def test_deserialization_date(): + assert TestType.parse_raw('"date"') == DateType() + + +def test_str_date(): + assert str(DateType()) == "date" + + +def test_repr_date(): + assert repr(DateType()) == "DateType()" + + +def test_serialization_time(): + assert TimeType().json() == '"time"' + + +def test_deserialization_time(): + assert TestType.parse_raw('"time"') == TimeType() + + +def test_str_time(): + assert str(TimeType()) == "time" + + +def test_repr_time(): + assert repr(TimeType()) == "TimeType()" + + +def test_serialization_timestamp(): + assert TimestampType().json() == '"timestamp"' + + +def test_deserialization_timestamp(): + assert TestType.parse_raw('"timestamp"') == TimestampType() + + +def test_str_timestamp(): + assert str(TimestampType()) == "timestamp" + + +def test_repr_timestamp(): + assert repr(TimestampType()) == "TimestampType()" + + +def test_serialization_timestamptz(): + assert TimestamptzType().json() == '"timestamptz"' + + +def test_deserialization_timestamptz(): + assert TestType.parse_raw('"timestamptz"') == TimestamptzType() + + +def test_str_timestamptz(): + assert str(TimestamptzType()) == "timestamptz" + + +def test_repr_timestamptz(): + assert repr(TimestamptzType()) == "TimestamptzType()" + + +def test_serialization_string(): + assert StringType().json() == '"string"' + + +def test_deserialization_string(): + assert TestType.parse_raw('"string"') == StringType() + + +def test_str_string(): + assert str(StringType()) == "string" + + +def test_repr_string(): + assert repr(StringType()) == "StringType()" + + +def test_serialization_uuid(): + assert UUIDType().json() == '"uuid"' + + +def test_deserialization_uuid(): + assert TestType.parse_raw('"uuid"') == UUIDType() + + +def test_str_uuid(): + assert str(UUIDType()) == "uuid" + + +def test_repr_uuid(): + assert repr(UUIDType()) == "UUIDType()" + + +def test_serialization_fixed(): + assert FixedType(22).json() == '"fixed[22]"' + + +def test_deserialization_fixed(): + fixed = TestType.parse_raw('"fixed[22]"') + assert fixed == FixedType(22) + + inner = fixed.__root__ + assert isinstance(inner, FixedType) + assert inner.length == 22 + + +def test_str_fixed(): + assert str(FixedType(22)) == "fixed[22]" + + +def test_repr_fixed(): + assert repr(FixedType(22)) == "FixedType(length=22)" + + +def test_serialization_binary(): + assert BinaryType().json() == '"binary"' + + +def test_deserialization_binary(): + assert TestType.parse_raw('"binary"') == BinaryType() + + +def test_str_binary(): + assert str(BinaryType()) == "binary" + + +def test_repr_binary(): + assert repr(BinaryType()) == "BinaryType()" + + +def test_serialization_decimal(): + assert DecimalType(19, 25).json() == '"decimal(19, 25)"' + + +def test_deserialization_decimal(): + decimal = TestType.parse_raw('"decimal(19, 25)"') + assert decimal == DecimalType(19, 25) + + inner = decimal.__root__ + assert isinstance(inner, DecimalType) + assert inner.precision == 19 + assert inner.scale == 25 + + +def test_str_decimal(): + assert str(DecimalType(19, 25)) == "decimal(19, 25)" + + +def test_repr_decimal(): + assert repr(DecimalType(19, 25)) == "DecimalType(precision=19, scale=25)" + + +def test_serialization_nestedfield(): + expected = '{"id": 1, "name": "required_field", "type": "string", "required": true, "doc": "this is a doc"}' + actual = NestedField(1, "required_field", StringType(), True, "this is a doc").json() + assert expected == actual + + +def test_serialization_nestedfield_no_doc(): + expected = '{"id": 1, "name": "required_field", "type": "string", "required": true}' + actual = NestedField(1, "required_field", StringType(), True).json() + assert expected == actual + + +def test_str_nestedfield(): + assert str(NestedField(1, "required_field", StringType(), True)) == "1: required_field: required string" + + +def test_repr_nestedfield(): + assert ( + repr(NestedField(1, "required_field", StringType(), True)) + == "NestedField(field_id=1, name='required_field', field_type=StringType(), required=True)" + ) + + +def test_nestedfield_by_alias(): + # We should be able to initialize a NestedField by alias + expected = NestedField(1, "required_field", StringType(), True, "this is a doc") + actual = NestedField(**{"id": 1, "name": "required_field", "type": "string", "required": True, "doc": "this is a doc"}) + assert expected == actual + + +def test_deserialization_nestedfield(): + expected = NestedField(1, "required_field", StringType(), True, "this is a doc") + actual = NestedField.parse_raw( + '{"id": 1, "name": "required_field", "type": "string", "required": true, "doc": "this is a doc"}' + ) + assert expected == actual + + +def test_deserialization_nestedfield_inner(): + expected = NestedField(1, "required_field", StringType(), True, "this is a doc") + actual = TestType.parse_raw('{"id": 1, "name": "required_field", "type": "string", "required": true, "doc": "this is a doc"}') + assert expected == actual.__root__ + + +def test_serialization_struct(): + actual = StructType( + NestedField(1, "required_field", StringType(), True, "this is a doc"), NestedField(2, "optional_field", IntegerType()) + ).json() + expected = ( + '{"type": "struct", "fields": [' + '{"id": 1, "name": "required_field", "type": "string", "required": true, "doc": "this is a doc"}, ' + '{"id": 2, "name": "optional_field", "type": "int", "required": true}' + "]}" + ) + assert actual == expected + + +def test_deserialization_struct(): + actual = StructType.parse_raw( + """ + { + "type": "struct", + "fields": [{ + "id": 1, + "name": "required_field", + "type": "string", + "required": true, + "doc": "this is a doc" + }, + { + "id": 2, + "name": "optional_field", + "type": "int", + "required": true + } + ] + } + """ + ) + + expected = StructType( + NestedField(1, "required_field", StringType(), True, "this is a doc"), NestedField(2, "optional_field", IntegerType()) + ) + + assert actual == expected + + +def test_str_struct(simple_struct: StructType): + assert str(simple_struct) == "struct<1: required_field: required string (this is a doc), 2: optional_field: required int>" + + +def test_repr_struct(simple_struct: StructType): + assert ( + repr(simple_struct) + == "StructType(fields=(NestedField(field_id=1, name='required_field', field_type=StringType(), required=True), NestedField(field_id=2, name='optional_field', field_type=IntegerType(), required=True),))" + ) + + +def test_serialization_list(simple_list: ListType): + actual = simple_list.json() + expected = '{"type": "list", "element-id": 22, "element": "string", "element-required": true}' + assert actual == expected + + +def test_deserialization_list(simple_list: ListType): + actual = ListType.parse_raw('{"type": "list", "element-id": 22, "element": "string", "element-required": true}') + assert actual == simple_list + + +def test_str_list(simple_list: ListType): + assert str(simple_list) == "list" + + +def test_repr_list(simple_list: ListType): + assert repr(simple_list) == "ListType(type='list', element_id=22, element_type=StringType(), element_required=True)" + + +def test_serialization_map(simple_map: MapType): + actual = simple_map.json() + expected = """{"type": "map", "key-id": 19, "key": "string", "value-id": 25, "value": "double", "value-required": false}""" + + assert actual == expected + + +def test_deserialization_map(simple_map: MapType): + actual = MapType.parse_raw( + """{"type": "map", "key-id": 19, "key": "string", "value-id": 25, "value": "double", "value-required": false}""" + ) + assert actual == simple_map + + +def test_str_map(simple_map: MapType): + assert str(simple_map) == "map" + + +def test_repr_map(simple_map: MapType): + assert ( + repr(simple_map) + == "MapType(type='map', key_id=19, key_type=StringType(), value_id=25, value_type=DoubleType(), value_required=False)" + ) + + def test_types_singleton(): """The types are immutable so we can return the same instance multiple times""" assert id(BooleanType()) == id(BooleanType()) diff --git a/tests/utils/test_bin_packing.py b/tests/utils/test_bin_packing.py index cf96023a13..59e5f56fe2 100644 --- a/tests/utils/test_bin_packing.py +++ b/tests/utils/test_bin_packing.py @@ -19,6 +19,7 @@ import pytest +from iceberg.schema import Schema from iceberg.utils.bin_packing import PackingIterator @@ -81,3 +82,17 @@ def weight_func(x): return x assert list(PackingIterator(splits, target_weight, lookback, weight_func, largest_bin_first)) == expected_lists + + +def test_serialize_schema(table_schema_simple: Schema): + actual = table_schema_simple.json() + expected = """{"fields": [{"id": 1, "name": "foo", "type": "string", "required": false}, {"id": 2, "name": "bar", "type": "int", "required": true}, {"id": 3, "name": "baz", "type": "boolean", "required": false}], "schema-id": 1, "identifier-field-ids": [1]}""" + assert actual == expected + + +def test_deserialize_schema(table_schema_simple: Schema): + actual = Schema.parse_raw( + """{"fields": [{"id": 1, "name": "foo", "type": "string", "required": false}, {"id": 2, "name": "bar", "type": "int", "required": true}, {"id": 3, "name": "baz", "type": "boolean", "required": false}], "schema-id": 1, "identifier-field-ids": [1]}""" + ) + expected = table_schema_simple + assert actual == expected From f85e9569e6444beacc50803b3dec3b3423c0349d Mon Sep 17 00:00:00 2001 From: Fokko Driesprong Date: Wed, 29 Jun 2022 01:53:53 +0200 Subject: [PATCH 124/642] Python: Make the VoidTransform a singleton (#5149) --- src/iceberg/transforms.py | 10 ++-------- tests/utils/test_singleton.py | 6 ++++++ 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/src/iceberg/transforms.py b/src/iceberg/transforms.py index 0947cf56b5..a2fa8f67fd 100644 --- a/src/iceberg/transforms.py +++ b/src/iceberg/transforms.py @@ -41,6 +41,7 @@ ) from iceberg.utils import datetime from iceberg.utils.decimal import decimal_to_bytes +from src.iceberg.utils.singleton import Singleton S = TypeVar("S") T = TypeVar("T") @@ -328,16 +329,9 @@ def result_type(self, source: IcebergType) -> IcebergType: return StringType() -class VoidTransform(Transform): +class VoidTransform(Transform, Singleton): """A transform that always returns None""" - _instance = None - - def __new__(cls): # pylint: disable=W0221 - if cls._instance is None: - cls._instance = super().__new__(cls) - return cls._instance - def __init__(self): super().__init__("void", "transforms.always_null()") diff --git a/tests/utils/test_singleton.py b/tests/utils/test_singleton.py index 92b923dd82..78ac177a85 100644 --- a/tests/utils/test_singleton.py +++ b/tests/utils/test_singleton.py @@ -15,6 +15,7 @@ # specific language governing permissions and limitations # under the License. from iceberg.avro.reader import BooleanReader, FixedReader +from src.iceberg.transforms import VoidTransform def test_singleton(): @@ -22,3 +23,8 @@ def test_singleton(): assert id(BooleanReader()) == id(BooleanReader()) assert id(FixedReader(22)) == id(FixedReader(22)) assert id(FixedReader(19)) != id(FixedReader(25)) + + +def test_singleton_transform(): + """We want to reuse VoidTransform since it doesn't carry any state""" + assert id(VoidTransform()) == id(VoidTransform()) From c4490c5ec8cdba30549f14e1ee3ef6b9798a614b Mon Sep 17 00:00:00 2001 From: Fokko Driesprong Date: Wed, 29 Jun 2022 16:51:58 +0200 Subject: [PATCH 125/642] Python: Resolve write/read schemas (#5116) --- src/iceberg/avro/decoder.py | 40 +++++-- src/iceberg/avro/file.py | 15 ++- src/iceberg/avro/reader.py | 137 +++++++++++++++++++++--- src/iceberg/avro/resolver.py | 195 +++++++++++++++++++++++++++++++++++ src/iceberg/exceptions.py | 2 +- tests/avro/test_decoder.py | 79 +++++++++++++- tests/avro/test_reader.py | 3 +- tests/avro/test_resolver.py | 185 +++++++++++++++++++++++++++++++++ 8 files changed, 621 insertions(+), 35 deletions(-) create mode 100644 src/iceberg/avro/resolver.py create mode 100644 tests/avro/test_resolver.py diff --git a/src/iceberg/avro/decoder.py b/src/iceberg/avro/decoder.py index 586aabf978..702ad9d4da 100644 --- a/src/iceberg/avro/decoder.py +++ b/src/iceberg/avro/decoder.py @@ -17,6 +17,7 @@ import decimal import struct from datetime import date, datetime, time +from io import SEEK_CUR from iceberg.io.base import InputStream from iceberg.utils.datetime import ( @@ -56,6 +57,9 @@ def read(self, n: int) -> bytes: raise ValueError(f"Read {len(read_bytes)} bytes, expected {n} bytes") return read_bytes + def skip(self, n: int) -> None: + self._input_stream.seek(n, SEEK_CUR) + def read_boolean(self) -> bool: """ a boolean is written as a single byte @@ -64,11 +68,7 @@ def read_boolean(self) -> bool: return ord(self.read(1)) == 1 def read_int(self) -> int: - """int values are written using variable-length, zigzag coding.""" - return self.read_long() - - def read_long(self) -> int: - """long values are written using variable-length, zigzag coding.""" + """int/long values are written using variable-length, zigzag coding.""" b = ord(self.read(1)) n = b & 0x7F shift = 7 @@ -100,7 +100,7 @@ def read_decimal_from_bytes(self, precision: int, scale: int) -> decimal.Decimal Decimal bytes are decoded as signed short, int or long depending on the size of bytes. """ - size = self.read_long() + size = self.read_int() return self.read_decimal_from_fixed(precision, scale, size) def read_decimal_from_fixed(self, _: int, scale: int, size: int) -> decimal.Decimal: @@ -116,7 +116,7 @@ def read_bytes(self) -> bytes: """ Bytes are encoded as a long followed by that many bytes of data. """ - return self.read(self.read_long()) + return self.read(self.read_int()) def read_utf8(self) -> str: """ @@ -146,14 +146,14 @@ def read_time_micros(self) -> time: long is decoded as python time object which represents the number of microseconds after midnight, 00:00:00.000000. """ - return micros_to_time(self.read_long()) + return micros_to_time(self.read_int()) def read_timestamp_micros(self) -> datetime: """ long is decoded as python datetime object which represents the number of microseconds from the unix epoch, 1 January 1970. """ - return micros_to_timestamp(self.read_long()) + return micros_to_timestamp(self.read_int()) def read_timestamptz_micros(self): """ @@ -162,4 +162,24 @@ def read_timestamptz_micros(self): Adjusted to UTC """ - return micros_to_timestamptz(self.read_long()) + return micros_to_timestamptz(self.read_int()) + + def skip_boolean(self) -> None: + self.skip(1) + + def skip_int(self) -> None: + b = ord(self.read(1)) + while (b & 0x80) != 0: + b = ord(self.read(1)) + + def skip_float(self) -> None: + self.skip(4) + + def skip_double(self) -> None: + self.skip(8) + + def skip_bytes(self) -> None: + self.skip(self.read_int()) + + def skip_utf8(self) -> None: + self.skip_bytes() diff --git a/src/iceberg/avro/file.py b/src/iceberg/avro/file.py index 0eec227e95..585c74c07c 100644 --- a/src/iceberg/avro/file.py +++ b/src/iceberg/avro/file.py @@ -27,6 +27,7 @@ from iceberg.avro.codecs import KNOWN_CODECS, Codec from iceberg.avro.decoder import BinaryDecoder from iceberg.avro.reader import AvroStruct, ConstructReader, StructReader +from iceberg.avro.resolver import resolve from iceberg.io.base import InputFile, InputStream from iceberg.io.memory import MemoryInputStream from iceberg.schema import Schema, visit @@ -107,6 +108,7 @@ def __next__(self) -> AvroStruct: class AvroFile: input_file: InputFile + read_schema: Schema | None input_stream: InputStream header: AvroFileHeader schema: Schema @@ -116,8 +118,9 @@ class AvroFile: decoder: BinaryDecoder block: Block | None = None - def __init__(self, input_file: InputFile) -> None: + def __init__(self, input_file: InputFile, read_schema: Schema | None = None) -> None: self.input_file = input_file + self.read_schema = read_schema def __enter__(self): """ @@ -132,7 +135,11 @@ def __enter__(self): self.header = self._read_header() self.schema = self.header.get_schema() self.file_length = len(self.input_file) - self.reader = visit(self.schema, ConstructReader()) + if not self.read_schema: + self.reader = visit(self.schema, ConstructReader()) + else: + self.reader = resolve(self.schema, self.read_schema) + return self def __exit__(self, exc_type, exc_val, exc_tb): @@ -149,9 +156,9 @@ def _read_block(self) -> int: raise ValueError(f"Expected sync bytes {self.header.sync!r}, but got {sync_marker!r}") if self.is_EOF(): raise StopIteration - block_records = self.decoder.read_long() + block_records = self.decoder.read_int() - block_bytes_len = self.decoder.read_long() + block_bytes_len = self.decoder.read_int() block_bytes = self.decoder.read(block_bytes_len) if codec := self.header.compression_codec(): block_bytes = codec.decompress(block_bytes) diff --git a/src/iceberg/avro/reader.py b/src/iceberg/avro/reader.py index 012d611a1a..c122c8a780 100644 --- a/src/iceberg/avro/reader.py +++ b/src/iceberg/avro/reader.py @@ -31,7 +31,7 @@ from datetime import date, datetime, time from decimal import Decimal from functools import singledispatch -from typing import Any +from typing import Any, Callable from uuid import UUID from iceberg.avro.decoder import BinaryDecoder @@ -60,6 +60,42 @@ from iceberg.utils.singleton import Singleton +def _skip_map_array(decoder: BinaryDecoder, skip_entry: Callable) -> None: + """Skips over an array or map + + Both the array and map are encoded similar, and we can re-use + the logic of skipping in an efficient way. + + From the Avro spec: + + Maps (and arrays) are encoded as a series of blocks. + Each block consists of a long count value, followed by that many key/value pairs in the case of a map, + and followed by that many array items in the case of an array. A block with count zero indicates the + end of the map. Each item is encoded per the map's value schema. + + If a block's count is negative, its absolute value is used, and the count is followed immediately by a + long block size indicating the number of bytes in the block. This block size permits fast skipping + through data, e.g., when projecting a record to a subset of its fields. + + Args: + decoder: + The decoder that reads the types from the underlying data + skip_entry: + Function to skip over the underlying data, element in case of an array, and the + key/value in the case of a map + """ + block_count = decoder.read_int() + while block_count != 0: + if block_count < 0: + # The length in bytes in encoded, so we can skip over it right away + block_size = decoder.read_int() + decoder.skip(block_size) + else: + for _ in range(block_count): + skip_entry() + block_count = decoder.read_int() + + @dataclass(frozen=True) class AvroStruct(StructProtocol): _data: list[Any | StructProtocol] = dataclassfield() @@ -76,66 +112,100 @@ class Reader(Singleton): def read(self, decoder: BinaryDecoder) -> Any: ... + @abstractmethod + def skip(self, decoder: BinaryDecoder) -> None: + ... + class NoneReader(Reader): def read(self, _: BinaryDecoder) -> None: return None + def skip(self, decoder: BinaryDecoder) -> None: + return None + class BooleanReader(Reader): def read(self, decoder: BinaryDecoder) -> bool: return decoder.read_boolean() + def skip(self, decoder: BinaryDecoder) -> None: + decoder.skip_boolean() + class IntegerReader(Reader): + """Longs and ints are encoded the same way, and there is no long in Python""" + def read(self, decoder: BinaryDecoder) -> int: return decoder.read_int() - -class LongReader(Reader): - def read(self, decoder: BinaryDecoder) -> int: - return decoder.read_long() + def skip(self, decoder: BinaryDecoder) -> None: + decoder.skip_int() class FloatReader(Reader): def read(self, decoder: BinaryDecoder) -> float: return decoder.read_float() + def skip(self, decoder: BinaryDecoder) -> None: + decoder.skip_float() + class DoubleReader(Reader): def read(self, decoder: BinaryDecoder) -> float: return decoder.read_double() + def skip(self, decoder: BinaryDecoder) -> None: + decoder.skip_double() + class DateReader(Reader): def read(self, decoder: BinaryDecoder) -> date: return decoder.read_date_from_int() + def skip(self, decoder: BinaryDecoder) -> None: + decoder.skip_int() + class TimeReader(Reader): def read(self, decoder: BinaryDecoder) -> time: return decoder.read_time_micros() + def skip(self, decoder: BinaryDecoder) -> None: + decoder.skip_int() + class TimestampReader(Reader): def read(self, decoder: BinaryDecoder) -> datetime: return decoder.read_timestamp_micros() + def skip(self, decoder: BinaryDecoder) -> None: + decoder.skip_int() + class TimestamptzReader(Reader): def read(self, decoder: BinaryDecoder) -> datetime: return decoder.read_timestamptz_micros() + def skip(self, decoder: BinaryDecoder) -> None: + decoder.skip_int() + class StringReader(Reader): def read(self, decoder: BinaryDecoder) -> str: return decoder.read_utf8() + def skip(self, decoder: BinaryDecoder) -> None: + decoder.skip_utf8() + class UUIDReader(Reader): def read(self, decoder: BinaryDecoder) -> UUID: return UUID(decoder.read_utf8()) + def skip(self, decoder: BinaryDecoder) -> None: + decoder.skip_utf8() + @dataclass(frozen=True) class FixedReader(Reader): @@ -144,11 +214,17 @@ class FixedReader(Reader): def read(self, decoder: BinaryDecoder) -> bytes: return decoder.read(self.length) + def skip(self, decoder: BinaryDecoder) -> None: + decoder.skip(self.length) + class BinaryReader(Reader): def read(self, decoder: BinaryDecoder) -> bytes: return decoder.read_bytes() + def skip(self, decoder: BinaryDecoder) -> None: + decoder.skip_bytes() + @dataclass(frozen=True) class DecimalReader(Reader): @@ -158,6 +234,9 @@ class DecimalReader(Reader): def read(self, decoder: BinaryDecoder) -> Decimal: return decoder.read_decimal_from_bytes(self.precision, self.scale) + def skip(self, decoder: BinaryDecoder) -> None: + decoder.skip_bytes() + @dataclass(frozen=True) class OptionReader(Reader): @@ -177,13 +256,28 @@ def read(self, decoder: BinaryDecoder) -> Any | None: return self.option.read(decoder) return None + def skip(self, decoder: BinaryDecoder) -> None: + if decoder.read_int() > 0: + return self.option.skip(decoder) + @dataclass(frozen=True) class StructReader(Reader): - fields: tuple[Reader, ...] = dataclassfield() + fields: tuple[tuple[int | None, Reader], ...] = dataclassfield() def read(self, decoder: BinaryDecoder) -> AvroStruct: - return AvroStruct([field.read(decoder) for field in self.fields]) + result: list[Any | StructProtocol] = [None] * len(self.fields) + for (pos, field) in self.fields: + if pos is not None: + result[pos] = field.read(decoder) + else: + field.skip(decoder) + + return AvroStruct(result) + + def skip(self, decoder: BinaryDecoder) -> None: + for _, field in self.fields: + field.skip(decoder) @dataclass(frozen=True) @@ -192,17 +286,19 @@ class ListReader(Reader): def read(self, decoder: BinaryDecoder) -> list: read_items = [] - block_count = decoder.read_long() + block_count = decoder.read_int() while block_count != 0: if block_count < 0: block_count = -block_count - # We ignore the block size for now - _ = decoder.read_long() + _ = decoder.read_int() for _ in range(block_count): read_items.append(self.element.read(decoder)) - block_count = decoder.read_long() + block_count = decoder.read_int() return read_items + def skip(self, decoder: BinaryDecoder) -> None: + _skip_map_array(decoder, lambda: self.element.skip(decoder)) + @dataclass(frozen=True) class MapReader(Reader): @@ -211,26 +307,33 @@ class MapReader(Reader): def read(self, decoder: BinaryDecoder) -> dict: read_items = {} - block_count = decoder.read_long() + block_count = decoder.read_int() while block_count != 0: if block_count < 0: block_count = -block_count # We ignore the block size for now - _ = decoder.read_long() + _ = decoder.read_int() for _ in range(block_count): key = self.key.read(decoder) read_items[key] = self.value.read(decoder) - block_count = decoder.read_long() + block_count = decoder.read_int() return read_items + def skip(self, decoder: BinaryDecoder) -> None: + def skip(): + self.key.skip(decoder) + self.value.skip(decoder) + + _skip_map_array(decoder, skip) + class ConstructReader(SchemaVisitor[Reader]): def schema(self, schema: Schema, struct_result: Reader) -> Reader: return struct_result def struct(self, struct: StructType, field_results: list[Reader]) -> Reader: - return StructReader(tuple(field_results)) + return StructReader(tuple(enumerate(field_results))) def field(self, field: NestedField, field_result: Reader) -> Reader: return field_result if field.required else OptionReader(field_result) @@ -274,7 +377,9 @@ def _(_: IntegerType) -> Reader: @primitive_reader.register(LongType) def _(_: LongType) -> Reader: - return LongReader() + # Ints and longs are encoded the same way in Python and + # also binary compatible in Avro + return IntegerReader() @primitive_reader.register(FloatType) diff --git a/src/iceberg/avro/resolver.py b/src/iceberg/avro/resolver.py new file mode 100644 index 0000000000..11017742ab --- /dev/null +++ b/src/iceberg/avro/resolver.py @@ -0,0 +1,195 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +from functools import singledispatch +from typing import ( + List, + Optional, + Tuple, + Union, +) + +from iceberg.avro.reader import ( + ConstructReader, + ListReader, + MapReader, + NoneReader, + OptionReader, + Reader, + StructReader, + primitive_reader, +) +from iceberg.schema import Schema, visit +from iceberg.types import ( + BinaryType, + DecimalType, + DoubleType, + FloatType, + IcebergType, + IntegerType, + ListType, + LongType, + MapType, + PrimitiveType, + StringType, + StructType, +) + + +class ResolveException(Exception): + pass + + +@singledispatch +def resolve(file_schema: Union[Schema, IcebergType], read_schema: Union[Schema, IcebergType]) -> Reader: + """This resolves the file and read schema + + The function traverses the schema in post-order fashion + + Args: + file_schema (Schema | IcebergType): The schema of the Avro file + read_schema (Schema | IcebergType): The requested read schema which is equal, subset or superset of the file schema + + Raises: + NotImplementedError: If attempting to resolve an unrecognized object type + """ + raise NotImplementedError(f"Cannot resolve non-type: {file_schema}") + + +@resolve.register(Schema) +def _(file_schema: Schema, read_schema: Schema) -> Reader: + """Visit a Schema and starts resolving it by converting it to a struct""" + return resolve(file_schema.as_struct(), read_schema.as_struct()) + + +@resolve.register(StructType) +def _(file_struct: StructType, read_struct: IcebergType) -> Reader: + """Iterates over the file schema, and checks if the field is in the read schema""" + + if not isinstance(read_struct, StructType): + raise ResolveException(f"File/read schema are not aligned for {file_struct}, got {read_struct}") + + results: List[Tuple[Optional[int], Reader]] = [] + read_fields = {field.field_id: (pos, field) for pos, field in enumerate(read_struct.fields)} + + for file_field in file_struct.fields: + if file_field.field_id in read_fields: + read_pos, read_field = read_fields[file_field.field_id] + result_reader = resolve(file_field.field_type, read_field.field_type) + else: + read_pos = None + result_reader = visit(file_field.field_type, ConstructReader()) + result_reader = result_reader if file_field.required else OptionReader(result_reader) + results.append((read_pos, result_reader)) + + file_fields = {field.field_id: field for field in file_struct.fields} + for pos, read_field in enumerate(read_struct.fields): + if read_field.field_id not in file_fields: + if read_field.required: + raise ResolveException(f"{read_field} is non-optional, and not part of the file schema") + # Just set the new field to None + results.append((pos, NoneReader())) + + return StructReader(tuple(results)) + + +@resolve.register(ListType) +def _(file_list: ListType, read_list: IcebergType) -> Reader: + if not isinstance(read_list, ListType): + raise ResolveException(f"File/read schema are not aligned for {file_list}, got {read_list}") + element_reader = resolve(file_list.element_type, read_list.element_type) + return ListReader(element_reader) + + +@resolve.register(MapType) +def _(file_map: MapType, read_map: IcebergType) -> Reader: + if not isinstance(read_map, MapType): + raise ResolveException(f"File/read schema are not aligned for {file_map}, got {read_map}") + key_reader = resolve(file_map.key_type, read_map.key_type) + value_reader = resolve(file_map.value_type, read_map.value_type) + + return MapReader(key_reader, value_reader) + + +@resolve.register(PrimitiveType) +def _(file_type: PrimitiveType, read_type: IcebergType) -> Reader: + """Converting the primitive type into an actual reader that will decode the physical data""" + if not isinstance(read_type, PrimitiveType): + raise ResolveException(f"Cannot promote {file_type} to {read_type}") + + # In the case of a promotion, we want to check if it is valid + if file_type != read_type: + return promote(file_type, read_type) + return primitive_reader(read_type) + + +@singledispatch +def promote(file_type: IcebergType, read_type: IcebergType) -> Reader: + """Promotes reading a file type to a read type + + Args: + file_type (IcebergType): The type of the Avro file + read_type (IcebergType): The requested read type + + Raises: + ResolveException: If attempting to resolve an unrecognized object type + """ + raise ResolveException(f"Cannot promote {file_type} to {read_type}") + + +@promote.register(IntegerType) +def _(file_type: IntegerType, read_type: IcebergType) -> Reader: + if isinstance(read_type, LongType): + # Ints/Longs are binary compatible in Avro, so this is okay + return primitive_reader(read_type) + else: + raise ResolveException(f"Cannot promote an int to {read_type}") + + +@promote.register(FloatType) +def _(file_type: FloatType, read_type: IcebergType) -> Reader: + if isinstance(read_type, DoubleType): + # We should just read the float, and return it, since it both returns a float + return primitive_reader(file_type) + else: + raise ResolveException(f"Cannot promote an float to {read_type}") + + +@promote.register(StringType) +def _(file_type: StringType, read_type: IcebergType) -> Reader: + if isinstance(read_type, BinaryType): + return primitive_reader(read_type) + else: + raise ResolveException(f"Cannot promote an string to {read_type}") + + +@promote.register(BinaryType) +def _(file_type: BinaryType, read_type: IcebergType) -> Reader: + if isinstance(read_type, StringType): + return primitive_reader(read_type) + else: + raise ResolveException(f"Cannot promote an binary to {read_type}") + + +@promote.register(DecimalType) +def _(file_type: DecimalType, read_type: IcebergType) -> Reader: + if isinstance(read_type, DecimalType): + if file_type.precision <= read_type.precision and file_type.scale == file_type.scale: + return primitive_reader(read_type) + else: + raise ResolveException(f"Cannot reduce precision from {file_type} to {read_type}") + else: + raise ResolveException(f"Cannot promote an decimal to {read_type}") diff --git a/src/iceberg/exceptions.py b/src/iceberg/exceptions.py index b12e836e46..f9ac3333b1 100644 --- a/src/iceberg/exceptions.py +++ b/src/iceberg/exceptions.py @@ -33,4 +33,4 @@ class AlreadyExistsError(Exception): class ValidationError(Exception): - ... + """Raises when there is an issue with the schema""" diff --git a/tests/avro/test_decoder.py b/tests/avro/test_decoder.py index 2957151151..b988cb6c49 100644 --- a/tests/avro/test_decoder.py +++ b/tests/avro/test_decoder.py @@ -21,8 +21,10 @@ import pytest from iceberg.avro.decoder import BinaryDecoder +from iceberg.avro.resolver import promote from iceberg.io.base import InputStream from iceberg.io.memory import MemoryInputStream +from iceberg.types import DoubleType, FloatType def test_read_decimal_from_fixed(): @@ -33,10 +35,38 @@ def test_read_decimal_from_fixed(): assert actual == expected -def test_read_long(): +def test_read_boolean_true(): + mis = MemoryInputStream(b"\x01") + decoder = BinaryDecoder(mis) + assert decoder.read_boolean() is True + + +def test_read_boolean_false(): + mis = MemoryInputStream(b"\x00") + decoder = BinaryDecoder(mis) + assert decoder.read_boolean() is False + + +def test_skip_boolean(): + mis = MemoryInputStream(b"\x00") + decoder = BinaryDecoder(mis) + assert mis.tell() == 0 + decoder.skip_boolean() + assert mis.tell() == 1 + + +def test_read_int(): mis = MemoryInputStream(b"\x18") decoder = BinaryDecoder(mis) - assert decoder.read_long() == 12 + assert decoder.read_int() == 12 + + +def test_skip_int(): + mis = MemoryInputStream(b"\x18") + decoder = BinaryDecoder(mis) + assert mis.tell() == 0 + decoder.skip_int() + assert mis.tell() == 1 def test_read_decimal(): @@ -104,12 +134,28 @@ def test_read_float(): assert decoder.read_float() == 19.25 +def test_skip_float(): + mis = MemoryInputStream(b"\x00\x00\x9A\x41") + decoder = BinaryDecoder(mis) + assert mis.tell() == 0 + decoder.skip_float() + assert mis.tell() == 4 + + def test_read_double(): mis = MemoryInputStream(b"\x00\x00\x00\x00\x00\x40\x33\x40") decoder = BinaryDecoder(mis) assert decoder.read_double() == 19.25 +def test_skip_double(): + mis = MemoryInputStream(b"\x00\x00\x00\x00\x00\x40\x33\x40") + decoder = BinaryDecoder(mis) + assert mis.tell() == 0 + decoder.skip_double() + assert mis.tell() == 8 + + def test_read_date(): mis = MemoryInputStream(b"\xBC\x7D") decoder = BinaryDecoder(mis) @@ -138,3 +184,32 @@ def test_read_timestamptz_micros(): mis = MemoryInputStream(b"\xBC\x7D") decoder = BinaryDecoder(mis) assert decoder.read_timestamptz_micros() == datetime(1970, 1, 1, 0, 0, 0, 8030, tzinfo=timezone.utc) + + +def test_read_bytes(): + mis = MemoryInputStream(b"\x08\x01\x02\x03\x04") + decoder = BinaryDecoder(mis) + actual = decoder.read_bytes() + assert actual == b"\x01\x02\x03\x04" + + +def test_read_utf8(): + mis = MemoryInputStream(b"\x04\x76\x6F") + decoder = BinaryDecoder(mis) + assert decoder.read_utf8() == "vo" + + +def test_skip_utf8(): + mis = MemoryInputStream(b"\x04\x76\x6F") + decoder = BinaryDecoder(mis) + assert mis.tell() == 0 + decoder.skip_utf8() + assert mis.tell() == 3 + + +def test_read_int_as_float(): + mis = MemoryInputStream(b"\x00\x00\x9A\x41") + decoder = BinaryDecoder(mis) + reader = promote(FloatType(), DoubleType()) + + assert reader.read(decoder) == 19.25 diff --git a/tests/avro/test_reader.py b/tests/avro/test_reader.py index 6ea8377e11..96afa390f8 100644 --- a/tests/avro/test_reader.py +++ b/tests/avro/test_reader.py @@ -30,7 +30,6 @@ FixedReader, FloatReader, IntegerReader, - LongReader, StringReader, TimeReader, TimestampReader, @@ -411,7 +410,7 @@ def test_integer_reader(): def test_long_reader(): - assert primitive_reader(LongType()) == LongReader() + assert primitive_reader(LongType()) == IntegerReader() def test_float_reader(): diff --git a/tests/avro/test_resolver.py b/tests/avro/test_resolver.py new file mode 100644 index 0000000000..197dc95246 --- /dev/null +++ b/tests/avro/test_resolver.py @@ -0,0 +1,185 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +import pytest + +from iceberg.avro.reader import ( + DecimalReader, + DoubleReader, + FloatReader, + IntegerReader, + MapReader, + StringReader, + StructReader, +) +from iceberg.avro.resolver import ResolveException, promote, resolve +from iceberg.schema import Schema +from iceberg.types import ( + BinaryType, + DecimalType, + DoubleType, + FloatType, + IntegerType, + ListType, + LongType, + MapType, + NestedField, + StringType, + StructType, +) + + +def test_resolver(): + write_schema = Schema( + NestedField(1, "id", LongType()), + NestedField(2, "data", StringType()), + NestedField( + 3, + "location", + StructType( + NestedField(4, "lat", DoubleType()), + NestedField(5, "long", DoubleType()), + ), + ), + NestedField(6, "preferences", MapType(7, StringType(), 8, StringType())), + schema_id=1, + ) + read_schema = Schema( + NestedField( + 3, + "location", + StructType( + NestedField(4, "lat", DoubleType()), + NestedField(5, "long", DoubleType()), + ), + ), + NestedField(1, "id", LongType()), + NestedField(6, "preferences", MapType(7, StringType(), 8, StringType())), + schema_id=1, + ) + read_tree = resolve(write_schema, read_schema) + + assert read_tree == StructReader( + ( + (1, IntegerReader()), + (None, StringReader()), + ( + 0, + StructReader( + ( + (0, DoubleReader()), + (1, DoubleReader()), + ) + ), + ), + (2, MapReader(StringReader(), StringReader())), + ) + ) + + +def test_resolver_new_required_field(): + write_schema = Schema( + NestedField(1, "id", LongType()), + schema_id=1, + ) + read_schema = Schema( + NestedField(1, "id", LongType()), + NestedField(2, "data", StringType(), required=True), + schema_id=1, + ) + + with pytest.raises(ResolveException) as exc_info: + resolve(write_schema, read_schema) + + assert "2: data: required string is non-optional, and not part of the file schema" in str(exc_info.value) + + +def test_resolver_invalid_evolution(): + write_schema = Schema( + NestedField(1, "id", LongType()), + schema_id=1, + ) + read_schema = Schema( + NestedField(1, "id", DoubleType()), + schema_id=1, + ) + + with pytest.raises(ResolveException) as exc_info: + resolve(write_schema, read_schema) + + assert "Cannot promote long to double" in str(exc_info.value) + + +def test_resolver_promotion_string_to_binary(): + write_schema = Schema( + NestedField(1, "id", StringType()), + schema_id=1, + ) + read_schema = Schema( + NestedField(1, "id", BinaryType()), + schema_id=1, + ) + resolve(write_schema, read_schema) + + +def test_resolver_promotion_binary_to_string(): + write_schema = Schema( + NestedField(1, "id", BinaryType()), + schema_id=1, + ) + read_schema = Schema( + NestedField(1, "id", StringType()), + schema_id=1, + ) + resolve(write_schema, read_schema) + + +def test_resolver_change_type(): + write_schema = Schema( + NestedField(1, "properties", ListType(2, StringType())), + schema_id=1, + ) + read_schema = Schema( + NestedField(1, "properties", MapType(2, StringType(), 3, StringType())), + schema_id=1, + ) + + with pytest.raises(ResolveException) as exc_info: + resolve(write_schema, read_schema) + + assert "File/read schema are not aligned for list, got map" in str(exc_info.value) + + +def test_promote_int_to_long(): + assert promote(IntegerType(), LongType()) == IntegerReader() + + +def test_promote_float_to_double(): + # We should still read floats, because it is encoded in 4 bytes + assert promote(FloatType(), DoubleType()) == FloatReader() + + +def test_promote_decimal_to_decimal(): + # DecimalType(P, S) to DecimalType(P2, S) where P2 > P + assert promote(DecimalType(19, 25), DecimalType(22, 25)) == DecimalReader(22, 25) + + +def test_promote_decimal_to_decimal_reduce_precision(): + # DecimalType(P, S) to DecimalType(P2, S) where P2 > P + with pytest.raises(ResolveException) as exc_info: + _ = promote(DecimalType(19, 25), DecimalType(10, 25)) == DecimalReader(22, 25) + + assert "Cannot reduce precision from decimal(19, 25) to decimal(10, 25)" in str(exc_info.value) From c62612f161e6ed654614bec0d786371e2ba6be8b Mon Sep 17 00:00:00 2001 From: jun-he Date: Thu, 30 Jun 2022 09:27:58 -0700 Subject: [PATCH 126/642] Python: Add truncate transform (#5030) --- src/iceberg/transforms.py | 158 ++++++++++++++++++++++++++++------- src/iceberg/utils/decimal.py | 13 +++ tests/test_transforms.py | 57 +++++++++++++ 3 files changed, 200 insertions(+), 28 deletions(-) diff --git a/src/iceberg/transforms.py b/src/iceberg/transforms.py index a2fa8f67fd..aafda5df80 100644 --- a/src/iceberg/transforms.py +++ b/src/iceberg/transforms.py @@ -19,8 +19,13 @@ import struct from abc import ABC, abstractmethod from decimal import Decimal -from functools import singledispatchmethod -from typing import Generic, Optional, TypeVar +from functools import singledispatch +from typing import ( + Any, + Generic, + Optional, + TypeVar, +) from uuid import UUID import mmh3 # type: ignore @@ -40,7 +45,7 @@ UUIDType, ) from iceberg.utils import datetime -from iceberg.utils.decimal import decimal_to_bytes +from iceberg.utils.decimal import decimal_to_bytes, truncate_decimal from src.iceberg.utils.singleton import Singleton S = TypeVar("S") @@ -267,39 +272,132 @@ def satisfies_order_of(self, other: Transform) -> bool: return other.preserves_order def to_human_string(self, value: Optional[S]) -> str: - return self._human_string(value) + return _human_string(value, self._type) if value is not None else "null" - @singledispatchmethod - def _human_string(self, value: Optional[S]) -> str: - return str(value) if value is not None else "null" - @_human_string.register(bytes) - def _(self, value: bytes) -> str: - return _base64encode(value) +class TruncateTransform(Transform[S, S]): + """A transform for truncating a value to a specified width. + Args: + source_type (Type): An Iceberg Type of IntegerType, LongType, StringType, BinaryType or DecimalType + width (int): The truncate width + Raises: + ValueError: If a type is provided that is incompatible with a Truncate transform + """ + + def __init__(self, source_type: IcebergType, width: int): + assert width > 0, f"width ({width}) should be greater than 0" + super().__init__( + f"truncate[{width}]", + f"transforms.truncate(source_type={repr(source_type)}, width={width})", + ) + self._type = source_type + self._width = width - @_human_string.register(int) - def _(self, value: int) -> str: - return self._int_to_human_string(self._type, value) + @property + def width(self) -> int: + return self._width - @singledispatchmethod - def _int_to_human_string(self, _: IcebergType, value: int) -> str: - return str(value) + @property + def type(self) -> IcebergType: + return self._type - @_int_to_human_string.register(DateType) - def _(self, _: IcebergType, value: int) -> str: - return datetime.to_human_day(value) + def apply(self, value: Optional[S]) -> Optional[S]: + return _truncate_value(value, self._width) if value is not None else None - @_int_to_human_string.register(TimeType) - def _(self, _: IcebergType, value: int) -> str: - return datetime.to_human_time(value) + def can_transform(self, source: IcebergType) -> bool: + return self._type == source + + def result_type(self, source: IcebergType) -> IcebergType: + return source + + @property + def preserves_order(self) -> bool: + return True + + def satisfies_order_of(self, other: Transform) -> bool: + if self == other: + return True + elif isinstance(self._type, StringType) and isinstance(other, TruncateTransform) and isinstance(other.type, StringType): + return self._width >= other.width + + return False + + def to_human_string(self, value: Optional[S]) -> str: + if value is None: + return "null" + elif isinstance(value, bytes): + return _base64encode(value) + else: + return str(value) - @_int_to_human_string.register(TimestampType) - def _(self, _: IcebergType, value: int) -> str: - return datetime.to_human_timestamp(value) - @_int_to_human_string.register(TimestamptzType) - def _(self, _: IcebergType, value: int) -> str: - return datetime.to_human_timestamptz(value) +@singledispatch +def _human_string(value: Any, _type: IcebergType) -> str: + return str(value) + + +@_human_string.register(bytes) +def _(value: bytes, _type: IcebergType) -> str: + return _base64encode(value) + + +@_human_string.register(int) +def _(value: int, _type: IcebergType) -> str: + return _int_to_human_string(_type, value) + + +@singledispatch +def _int_to_human_string(_type: IcebergType, value: int) -> str: + return str(value) + + +@_int_to_human_string.register(DateType) +def _(_type: IcebergType, value: int) -> str: + return datetime.to_human_day(value) + + +@_int_to_human_string.register(TimeType) +def _(_type: IcebergType, value: int) -> str: + return datetime.to_human_time(value) + + +@_int_to_human_string.register(TimestampType) +def _(_type: IcebergType, value: int) -> str: + return datetime.to_human_timestamp(value) + + +@_int_to_human_string.register(TimestamptzType) +def _(_type: IcebergType, value: int) -> str: + return datetime.to_human_timestamptz(value) + + +@singledispatch +def _truncate_value(value: Any, _width: int) -> S: + raise ValueError(f"Cannot truncate value: {value}") + + +@_truncate_value.register(int) +def _(value: int, _width: int) -> int: + """Truncate a given int value into a given width if feasible.""" + return value - value % _width + + +@_truncate_value.register(str) +def _(value: str, _width: int) -> str: + """Truncate a given string to a given width.""" + return value[0 : min(_width, len(value))] + + +@_truncate_value.register(bytes) +def _(value: bytes, _width: int) -> bytes: + """Truncate a given binary bytes into a given width.""" + return value[0 : min(_width, len(value))] + + +@_truncate_value.register(Decimal) +def _(value: Decimal, _width: int) -> Decimal: + """Truncate a given decimal value into a given width.""" + return truncate_decimal(value, _width) class UnknownTransform(Transform): @@ -369,5 +467,9 @@ def identity(source_type: IcebergType) -> IdentityTransform: return IdentityTransform(source_type) +def truncate(source_type: IcebergType, width: int) -> TruncateTransform: + return TruncateTransform(source_type, width) + + def always_null() -> Transform: return VoidTransform() diff --git a/src/iceberg/utils/decimal.py b/src/iceberg/utils/decimal.py index 1d4c2bddef..40bc087390 100644 --- a/src/iceberg/utils/decimal.py +++ b/src/iceberg/utils/decimal.py @@ -75,3 +75,16 @@ def decimal_to_bytes(value: Decimal) -> bytes: """ unscaled_value = decimal_to_unscaled(value) return unscaled_value.to_bytes(bytes_required(unscaled_value), byteorder="big", signed=True) + + +def truncate_decimal(value: Decimal, width: int) -> Decimal: + """Get a truncated Decimal value given a decimal value and a width + Args: + value (Decimal): a decimal value + width (int): A width for the returned Decimal instance + Returns: + Decimal: A truncated Decimal instance + """ + unscaled_value = decimal_to_unscaled(value) + applied_value = unscaled_value - (((unscaled_value % width) + width) % width) + return unscaled_to_decimal(applied_value, -value.as_tuple().exponent) diff --git a/tests/test_transforms.py b/tests/test_transforms.py index d7b8e968a9..dc3ce4ec27 100644 --- a/tests/test_transforms.py +++ b/tests/test_transforms.py @@ -177,6 +177,63 @@ def test_identity_method(type_var): assert identity_transform.apply("test") == "test" +@pytest.mark.parametrize("type_var", [IntegerType(), LongType()]) +@pytest.mark.parametrize( + "input_var,expected", + [(1, 0), (5, 0), (9, 0), (10, 10), (11, 10), (-1, -10), (-10, -10), (-12, -20)], +) +def test_truncate_integer(type_var, input_var, expected): + trunc = transforms.truncate(type_var, 10) + assert trunc.apply(input_var) == expected + + +@pytest.mark.parametrize( + "input_var,expected", + [ + (Decimal("12.34"), Decimal("12.30")), + (Decimal("12.30"), Decimal("12.30")), + (Decimal("12.29"), Decimal("12.20")), + (Decimal("0.05"), Decimal("0.00")), + (Decimal("-0.05"), Decimal("-0.10")), + ], +) +def test_truncate_decimal(input_var, expected): + trunc = transforms.truncate(DecimalType(9, 2), 10) + assert trunc.apply(input_var) == expected + + +@pytest.mark.parametrize("input_var,expected", [("abcdefg", "abcde"), ("abc", "abc")]) +def test_truncate_string(input_var, expected): + trunc = transforms.truncate(StringType(), 5) + assert trunc.apply(input_var) == expected + + +@pytest.mark.parametrize( + "type_var,value,expected_human_str,expected", + [ + (BinaryType(), b"\x00\x01\x02\x03", "AAECAw==", b"\x00"), + (BinaryType(), bytes("\u2603de", "utf-8"), "4piDZGU=", b"\xe2"), + (DecimalType(8, 5), Decimal("14.21"), "14.21", Decimal("14.21")), + (IntegerType(), 123, "123", 123), + (LongType(), 123, "123", 123), + (StringType(), "foo", "foo", "f"), + (StringType(), "\u2603de", "\u2603de", "\u2603"), + ], +) +def test_truncate_method(type_var, value, expected_human_str, expected): + truncate_transform = transforms.truncate(type_var, 1) + assert str(truncate_transform) == str(eval(repr(truncate_transform))) + assert truncate_transform.can_transform(type_var) + assert truncate_transform.result_type(type_var) == type_var + assert truncate_transform.to_human_string(value) == expected_human_str + assert truncate_transform.apply(value) == expected + assert truncate_transform.to_human_string(None) == "null" + assert truncate_transform.width == 1 + assert truncate_transform.apply(None) is None + assert truncate_transform.preserves_order + assert truncate_transform.satisfies_order_of(truncate_transform) + + def test_unknown_transform(): unknown_transform = transforms.UnknownTransform(FixedType(8), "unknown") assert str(unknown_transform) == str(eval(repr(unknown_transform))) From 7befe1dc62c19c5a7cbdb005ec362a063cb75f43 Mon Sep 17 00:00:00 2001 From: Samuel Redai <43911210+samredai@users.noreply.github.com> Date: Thu, 30 Jun 2022 14:49:17 -0400 Subject: [PATCH 127/642] Python: Rename python top-level module pyiceberg, remove src (#5169) --- Makefile | 2 +- dev/RELEASE.md | 4 ++-- {src/iceberg => pyiceberg}/__init__.py | 0 {src/iceberg => pyiceberg}/avro/__init__.py | 0 .../avro/codecs/__init__.py | 10 +++++----- .../avro/codecs/bzip2.py | 2 +- .../avro/codecs/codec.py | 0 .../avro/codecs/deflate.py | 2 +- .../avro/codecs/snappy_codec.py | 2 +- .../avro/codecs/zstandard_codec.py | 2 +- {src/iceberg => pyiceberg}/avro/decoder.py | 6 +++--- {src/iceberg => pyiceberg}/avro/file.py | 18 ++++++++--------- {src/iceberg => pyiceberg}/avro/reader.py | 10 +++++----- {src/iceberg => pyiceberg}/avro/resolver.py | 6 +++--- .../iceberg => pyiceberg}/catalog/__init__.py | 0 {src/iceberg => pyiceberg}/catalog/base.py | 6 +++--- {src/iceberg => pyiceberg}/conversions.py | 4 ++-- {src/iceberg => pyiceberg}/exceptions.py | 0 .../expressions/__init__.py | 0 .../iceberg => pyiceberg}/expressions/base.py | 8 ++++---- .../expressions/literals.py | 10 +++++----- {src/iceberg => pyiceberg}/files.py | 0 {src/iceberg => pyiceberg}/io/__init__.py | 0 {src/iceberg => pyiceberg}/io/base.py | 0 {src/iceberg => pyiceberg}/io/memory.py | 2 +- {src/iceberg => pyiceberg}/io/pyarrow.py | 4 ++-- {src/iceberg => pyiceberg}/schema.py | 14 ++++++------- {src/iceberg => pyiceberg}/serializers.py | 4 ++-- {src/iceberg => pyiceberg}/table/__init__.py | 0 {src/iceberg => pyiceberg}/table/base.py | 2 +- {src/iceberg => pyiceberg}/table/metadata.py | 8 ++++---- .../table/partitioning.py | 4 ++-- {src/iceberg => pyiceberg}/table/refs.py | 2 +- {src/iceberg => pyiceberg}/transforms.py | 8 ++++---- {src/iceberg => pyiceberg}/types.py | 4 ++-- {src/iceberg => pyiceberg}/utils/__init__.py | 0 .../utils/bin_packing.py | 0 {src/iceberg => pyiceberg}/utils/datetime.py | 0 {src/iceberg => pyiceberg}/utils/decimal.py | 0 .../utils/iceberg_base_model.py | 0 .../utils/schema_conversion.py | 16 +++++++-------- {src/iceberg => pyiceberg}/utils/singleton.py | 0 pyproject.toml | 8 ++++---- tests/avro/test_decoder.py | 10 +++++----- tests/avro/test_file.py | 4 ++-- tests/avro/test_reader.py | 8 ++++---- tests/avro/test_resolver.py | 8 ++++---- tests/catalog/test_base.py | 10 +++++----- tests/conftest.py | 8 ++++---- tests/expressions/test_expressions_base.py | 6 +++--- tests/expressions/test_literals.py | 4 ++-- tests/io/test_io_base.py | 4 ++-- tests/io/test_pyarrow.py | 20 +++++++++---------- tests/table/test_metadata.py | 12 +++++------ tests/table/test_partitioning.py | 8 ++++---- tests/test_conversions.py | 6 +++--- tests/test_schema.py | 10 +++++----- tests/test_transforms.py | 6 +++--- tests/test_types.py | 4 ++-- tests/utils/test_bin_packing.py | 4 ++-- tests/utils/test_schema_conversion.py | 6 +++--- tests/utils/test_singleton.py | 4 ++-- 62 files changed, 155 insertions(+), 155 deletions(-) rename {src/iceberg => pyiceberg}/__init__.py (100%) rename {src/iceberg => pyiceberg}/avro/__init__.py (100%) rename {src/iceberg => pyiceberg}/avro/codecs/__init__.py (83%) rename {src/iceberg => pyiceberg}/avro/codecs/bzip2.py (97%) rename {src/iceberg => pyiceberg}/avro/codecs/codec.py (100%) rename {src/iceberg => pyiceberg}/avro/codecs/deflate.py (96%) rename {src/iceberg => pyiceberg}/avro/codecs/snappy_codec.py (98%) rename {src/iceberg => pyiceberg}/avro/codecs/zstandard_codec.py (97%) rename {src/iceberg => pyiceberg}/avro/decoder.py (97%) rename {src/iceberg => pyiceberg}/avro/file.py (92%) rename {src/iceberg => pyiceberg}/avro/reader.py (98%) rename {src/iceberg => pyiceberg}/avro/resolver.py (98%) rename {src/iceberg => pyiceberg}/catalog/__init__.py (100%) rename {src/iceberg => pyiceberg}/catalog/base.py (98%) rename {src/iceberg => pyiceberg}/conversions.py (98%) rename {src/iceberg => pyiceberg}/exceptions.py (100%) rename {src/iceberg => pyiceberg}/expressions/__init__.py (100%) rename {src/iceberg => pyiceberg}/expressions/base.py (98%) rename {src/iceberg => pyiceberg}/expressions/literals.py (98%) rename {src/iceberg => pyiceberg}/files.py (100%) rename {src/iceberg => pyiceberg}/io/__init__.py (100%) rename {src/iceberg => pyiceberg}/io/base.py (100%) rename {src/iceberg => pyiceberg}/io/memory.py (98%) rename {src/iceberg => pyiceberg}/io/pyarrow.py (99%) rename {src/iceberg => pyiceberg}/schema.py (98%) rename {src/iceberg => pyiceberg}/serializers.py (94%) rename {src/iceberg => pyiceberg}/table/__init__.py (100%) rename {src/iceberg => pyiceberg}/table/base.py (96%) rename {src/iceberg => pyiceberg}/table/metadata.py (98%) rename {src/iceberg => pyiceberg}/table/partitioning.py (98%) rename {src/iceberg => pyiceberg}/table/refs.py (95%) rename {src/iceberg => pyiceberg}/transforms.py (98%) rename {src/iceberg => pyiceberg}/types.py (99%) rename {src/iceberg => pyiceberg}/utils/__init__.py (100%) rename {src/iceberg => pyiceberg}/utils/bin_packing.py (100%) rename {src/iceberg => pyiceberg}/utils/datetime.py (100%) rename {src/iceberg => pyiceberg}/utils/decimal.py (100%) rename {src/iceberg => pyiceberg}/utils/iceberg_base_model.py (100%) rename {src/iceberg => pyiceberg}/utils/schema_conversion.py (96%) rename {src/iceberg => pyiceberg}/utils/singleton.py (100%) diff --git a/Makefile b/Makefile index 356c94e642..a73ab02278 100644 --- a/Makefile +++ b/Makefile @@ -23,7 +23,7 @@ lint: poetry run pre-commit run --all-files test: - poetry run coverage run --source=src/ -m pytest tests/ + poetry run coverage run --source=pyiceberg/ -m pytest tests/ poetry run coverage report -m --fail-under=90 poetry run coverage html poetry run coverage xml diff --git a/dev/RELEASE.md b/dev/RELEASE.md index 44305868bf..d54fa0132d 100644 --- a/dev/RELEASE.md +++ b/dev/RELEASE.md @@ -50,8 +50,8 @@ Next we'll create a source distribution (`sdist`) which will generate a `.tar.gz # Update the version poetry version ${VERSION} -git diff src/iceberg/__init__.py -git add src/iceberg/__init__.py +git diff pyiceberg/__init__.py +git add pyiceberg/__init__.py git commit -s -m "Set to version ${VERSION}" ``` diff --git a/src/iceberg/__init__.py b/pyiceberg/__init__.py similarity index 100% rename from src/iceberg/__init__.py rename to pyiceberg/__init__.py diff --git a/src/iceberg/avro/__init__.py b/pyiceberg/avro/__init__.py similarity index 100% rename from src/iceberg/avro/__init__.py rename to pyiceberg/avro/__init__.py diff --git a/src/iceberg/avro/codecs/__init__.py b/pyiceberg/avro/codecs/__init__.py similarity index 83% rename from src/iceberg/avro/codecs/__init__.py rename to pyiceberg/avro/codecs/__init__.py index 28dd23f83f..0512766280 100644 --- a/src/iceberg/avro/codecs/__init__.py +++ b/pyiceberg/avro/codecs/__init__.py @@ -25,11 +25,11 @@ """ from __future__ import annotations -from iceberg.avro.codecs.bzip2 import BZip2Codec -from iceberg.avro.codecs.codec import Codec -from iceberg.avro.codecs.deflate import DeflateCodec -from iceberg.avro.codecs.snappy_codec import SnappyCodec -from iceberg.avro.codecs.zstandard_codec import ZStandardCodec +from pyiceberg.avro.codecs.bzip2 import BZip2Codec +from pyiceberg.avro.codecs.codec import Codec +from pyiceberg.avro.codecs.deflate import DeflateCodec +from pyiceberg.avro.codecs.snappy_codec import SnappyCodec +from pyiceberg.avro.codecs.zstandard_codec import ZStandardCodec KNOWN_CODECS: dict[str, type[Codec] | None] = { "null": None, diff --git a/src/iceberg/avro/codecs/bzip2.py b/pyiceberg/avro/codecs/bzip2.py similarity index 97% rename from src/iceberg/avro/codecs/bzip2.py rename to pyiceberg/avro/codecs/bzip2.py index b92c248de2..24786681f2 100644 --- a/src/iceberg/avro/codecs/bzip2.py +++ b/pyiceberg/avro/codecs/bzip2.py @@ -16,7 +16,7 @@ # under the License. from __future__ import annotations -from iceberg.avro.codecs.codec import Codec +from pyiceberg.avro.codecs.codec import Codec try: import bz2 diff --git a/src/iceberg/avro/codecs/codec.py b/pyiceberg/avro/codecs/codec.py similarity index 100% rename from src/iceberg/avro/codecs/codec.py rename to pyiceberg/avro/codecs/codec.py diff --git a/src/iceberg/avro/codecs/deflate.py b/pyiceberg/avro/codecs/deflate.py similarity index 96% rename from src/iceberg/avro/codecs/deflate.py rename to pyiceberg/avro/codecs/deflate.py index c1f8bf30b7..33fc11cd43 100644 --- a/src/iceberg/avro/codecs/deflate.py +++ b/pyiceberg/avro/codecs/deflate.py @@ -18,7 +18,7 @@ import zlib -from iceberg.avro.codecs.codec import Codec +from pyiceberg.avro.codecs.codec import Codec class DeflateCodec(Codec): diff --git a/src/iceberg/avro/codecs/snappy_codec.py b/pyiceberg/avro/codecs/snappy_codec.py similarity index 98% rename from src/iceberg/avro/codecs/snappy_codec.py rename to pyiceberg/avro/codecs/snappy_codec.py index 92b599cdfa..22a72e1305 100644 --- a/src/iceberg/avro/codecs/snappy_codec.py +++ b/pyiceberg/avro/codecs/snappy_codec.py @@ -19,7 +19,7 @@ import binascii import struct -from iceberg.avro.codecs.codec import Codec +from pyiceberg.avro.codecs.codec import Codec STRUCT_CRC32 = struct.Struct(">I") # big-endian unsigned int diff --git a/src/iceberg/avro/codecs/zstandard_codec.py b/pyiceberg/avro/codecs/zstandard_codec.py similarity index 97% rename from src/iceberg/avro/codecs/zstandard_codec.py rename to pyiceberg/avro/codecs/zstandard_codec.py index 8144628b06..a048f68490 100644 --- a/src/iceberg/avro/codecs/zstandard_codec.py +++ b/pyiceberg/avro/codecs/zstandard_codec.py @@ -18,7 +18,7 @@ from io import BytesIO -from iceberg.avro.codecs.codec import Codec +from pyiceberg.avro.codecs.codec import Codec try: from zstandard import ZstdCompressor, ZstdDecompressor diff --git a/src/iceberg/avro/decoder.py b/pyiceberg/avro/decoder.py similarity index 97% rename from src/iceberg/avro/decoder.py rename to pyiceberg/avro/decoder.py index 702ad9d4da..3a05cfeec3 100644 --- a/src/iceberg/avro/decoder.py +++ b/pyiceberg/avro/decoder.py @@ -19,14 +19,14 @@ from datetime import date, datetime, time from io import SEEK_CUR -from iceberg.io.base import InputStream -from iceberg.utils.datetime import ( +from pyiceberg.io.base import InputStream +from pyiceberg.utils.datetime import ( days_to_date, micros_to_time, micros_to_timestamp, micros_to_timestamptz, ) -from iceberg.utils.decimal import unscaled_to_decimal +from pyiceberg.utils.decimal import unscaled_to_decimal STRUCT_FLOAT = struct.Struct(" None: diff --git a/src/iceberg/avro/resolver.py b/pyiceberg/avro/resolver.py similarity index 98% rename from src/iceberg/avro/resolver.py rename to pyiceberg/avro/resolver.py index 11017742ab..1b710ea191 100644 --- a/src/iceberg/avro/resolver.py +++ b/pyiceberg/avro/resolver.py @@ -22,7 +22,7 @@ Union, ) -from iceberg.avro.reader import ( +from pyiceberg.avro.reader import ( ConstructReader, ListReader, MapReader, @@ -32,8 +32,8 @@ StructReader, primitive_reader, ) -from iceberg.schema import Schema, visit -from iceberg.types import ( +from pyiceberg.schema import Schema, visit +from pyiceberg.types import ( BinaryType, DecimalType, DoubleType, diff --git a/src/iceberg/catalog/__init__.py b/pyiceberg/catalog/__init__.py similarity index 100% rename from src/iceberg/catalog/__init__.py rename to pyiceberg/catalog/__init__.py diff --git a/src/iceberg/catalog/base.py b/pyiceberg/catalog/base.py similarity index 98% rename from src/iceberg/catalog/base.py rename to pyiceberg/catalog/base.py index 243f6a8f74..42d07b088b 100644 --- a/src/iceberg/catalog/base.py +++ b/pyiceberg/catalog/base.py @@ -19,9 +19,9 @@ from abc import ABC, abstractmethod -from iceberg.catalog import Identifier, Properties -from iceberg.schema import Schema -from iceberg.table.base import PartitionSpec, Table +from pyiceberg.catalog import Identifier, Properties +from pyiceberg.schema import Schema +from pyiceberg.table.base import PartitionSpec, Table class Catalog(ABC): diff --git a/src/iceberg/conversions.py b/pyiceberg/conversions.py similarity index 98% rename from src/iceberg/conversions.py rename to pyiceberg/conversions.py index 96724e5461..e8c2a4e8cd 100644 --- a/src/iceberg/conversions.py +++ b/pyiceberg/conversions.py @@ -33,7 +33,7 @@ from struct import Struct from typing import Union -from iceberg.types import ( +from pyiceberg.types import ( BinaryType, BooleanType, DateType, @@ -50,7 +50,7 @@ TimeType, UUIDType, ) -from iceberg.utils.decimal import decimal_to_bytes, unscaled_to_decimal +from pyiceberg.utils.decimal import decimal_to_bytes, unscaled_to_decimal _BOOL_STRUCT = Struct(" Literal: value(python primitive type): the value to be associated with literal Example: - from iceberg.expressions.literals import literal + from pyiceberg.expressions.literals import literal >>> literal(123) LongLiteral(123) """ diff --git a/src/iceberg/files.py b/pyiceberg/files.py similarity index 100% rename from src/iceberg/files.py rename to pyiceberg/files.py diff --git a/src/iceberg/io/__init__.py b/pyiceberg/io/__init__.py similarity index 100% rename from src/iceberg/io/__init__.py rename to pyiceberg/io/__init__.py diff --git a/src/iceberg/io/base.py b/pyiceberg/io/base.py similarity index 100% rename from src/iceberg/io/base.py rename to pyiceberg/io/base.py diff --git a/src/iceberg/io/memory.py b/pyiceberg/io/memory.py similarity index 98% rename from src/iceberg/io/memory.py rename to pyiceberg/io/memory.py index 0e9dcb9c99..082a589f12 100644 --- a/src/iceberg/io/memory.py +++ b/pyiceberg/io/memory.py @@ -17,7 +17,7 @@ from io import SEEK_CUR, SEEK_END, SEEK_SET -from iceberg.io.base import InputStream +from pyiceberg.io.base import InputStream class MemoryInputStream(InputStream): diff --git a/src/iceberg/io/pyarrow.py b/pyiceberg/io/pyarrow.py similarity index 99% rename from src/iceberg/io/pyarrow.py rename to pyiceberg/io/pyarrow.py index 008fd7449c..8268cdc7f0 100644 --- a/src/iceberg/io/pyarrow.py +++ b/pyiceberg/io/pyarrow.py @@ -28,7 +28,7 @@ from pyarrow.fs import FileInfo, FileSystem, FileType -from iceberg.io.base import ( +from pyiceberg.io.base import ( FileIO, InputFile, InputStream, @@ -47,7 +47,7 @@ class PyArrowFile(InputFile, OutputFile): location(str): The URI or path to a local file for a PyArrowFile instance Examples: - >>> from iceberg.io.pyarrow import PyArrowFile + >>> from pyiceberg.io.pyarrow import PyArrowFile >>> # input_file = PyArrowFile("s3://foo/bar.txt") >>> # Read the contents of the PyArrowFile instance >>> # Make sure that you have permissions to read/write diff --git a/src/iceberg/schema.py b/pyiceberg/schema.py similarity index 98% rename from src/iceberg/schema.py rename to pyiceberg/schema.py index b57da36790..37372a0d5d 100644 --- a/src/iceberg/schema.py +++ b/pyiceberg/schema.py @@ -32,8 +32,8 @@ from pydantic import Field, PrivateAttr -from iceberg.files import StructProtocol -from iceberg.types import ( +from pyiceberg.files import StructProtocol +from pyiceberg.types import ( IcebergType, ListType, MapType, @@ -41,7 +41,7 @@ PrimitiveType, StructType, ) -from iceberg.utils.iceberg_base_model import IcebergBaseModel +from pyiceberg.utils.iceberg_base_model import IcebergBaseModel T = TypeVar("T") @@ -50,8 +50,8 @@ class Schema(IcebergBaseModel): """A table Schema Example: - >>> from iceberg import schema - >>> from iceberg import types + >>> from pyiceberg import schema + >>> from pyiceberg import types """ fields: Tuple[NestedField, ...] = Field(default_factory=tuple) @@ -538,8 +538,8 @@ class _BuildPositionAccessors(SchemaVisitor[Dict[Position, Accessor]]): """A schema visitor for generating a field ID to accessor index Example: - >>> from iceberg.schema import Schema - >>> from iceberg.types import * + >>> from pyiceberg.schema import Schema + >>> from pyiceberg.types import * >>> schema = Schema( ... NestedField(field_id=2, name="id", field_type=IntegerType(), required=False), ... NestedField(field_id=1, name="data", field_type=StringType(), required=True), diff --git a/src/iceberg/serializers.py b/pyiceberg/serializers.py similarity index 94% rename from src/iceberg/serializers.py rename to pyiceberg/serializers.py index 98e279f624..eee0c435ac 100644 --- a/src/iceberg/serializers.py +++ b/pyiceberg/serializers.py @@ -19,8 +19,8 @@ import json from typing import Union -from iceberg.io.base import InputFile, InputStream, OutputFile -from iceberg.table.metadata import TableMetadata, TableMetadataV1, TableMetadataV2 +from pyiceberg.io.base import InputFile, InputStream, OutputFile +from pyiceberg.table.metadata import TableMetadata, TableMetadataV1, TableMetadataV2 class FromByteStream: diff --git a/src/iceberg/table/__init__.py b/pyiceberg/table/__init__.py similarity index 100% rename from src/iceberg/table/__init__.py rename to pyiceberg/table/__init__.py diff --git a/src/iceberg/table/base.py b/pyiceberg/table/base.py similarity index 96% rename from src/iceberg/table/base.py rename to pyiceberg/table/base.py index 1cc8b39061..a1d7979063 100644 --- a/src/iceberg/table/base.py +++ b/pyiceberg/table/base.py @@ -19,7 +19,7 @@ from abc import ABC -from iceberg.catalog.base import Identifier +from pyiceberg.catalog.base import Identifier class Table(ABC): diff --git a/src/iceberg/table/metadata.py b/pyiceberg/table/metadata.py similarity index 98% rename from src/iceberg/table/metadata.py rename to pyiceberg/table/metadata.py index 55c43c1cf6..fef4110baa 100644 --- a/src/iceberg/table/metadata.py +++ b/pyiceberg/table/metadata.py @@ -27,10 +27,10 @@ from pydantic import Field, root_validator -from iceberg.exceptions import ValidationError -from iceberg.schema import Schema -from iceberg.table.refs import MAIN_BRANCH, SnapshotRef, SnapshotRefType -from iceberg.utils.iceberg_base_model import IcebergBaseModel +from pyiceberg.exceptions import ValidationError +from pyiceberg.schema import Schema +from pyiceberg.table.refs import MAIN_BRANCH, SnapshotRef, SnapshotRefType +from pyiceberg.utils.iceberg_base_model import IcebergBaseModel _INITIAL_SEQUENCE_NUMBER = 0 INITIAL_SPEC_ID = 0 diff --git a/src/iceberg/table/partitioning.py b/pyiceberg/table/partitioning.py similarity index 98% rename from src/iceberg/table/partitioning.py rename to pyiceberg/table/partitioning.py index 949045b986..b44c2a1ed9 100644 --- a/src/iceberg/table/partitioning.py +++ b/pyiceberg/table/partitioning.py @@ -17,8 +17,8 @@ from dataclasses import dataclass, field from typing import Dict, List, Tuple -from iceberg.schema import Schema -from iceberg.transforms import Transform +from pyiceberg.schema import Schema +from pyiceberg.transforms import Transform _PARTITION_DATA_ID_START: int = 1000 diff --git a/src/iceberg/table/refs.py b/pyiceberg/table/refs.py similarity index 95% rename from src/iceberg/table/refs.py rename to pyiceberg/table/refs.py index 285c5cfd93..ab8c03fc37 100644 --- a/src/iceberg/table/refs.py +++ b/pyiceberg/table/refs.py @@ -19,7 +19,7 @@ from pydantic import Field -from iceberg.utils.iceberg_base_model import IcebergBaseModel +from pyiceberg.utils.iceberg_base_model import IcebergBaseModel MAIN_BRANCH = "main" diff --git a/src/iceberg/transforms.py b/pyiceberg/transforms.py similarity index 98% rename from src/iceberg/transforms.py rename to pyiceberg/transforms.py index aafda5df80..d90736d613 100644 --- a/src/iceberg/transforms.py +++ b/pyiceberg/transforms.py @@ -30,7 +30,7 @@ import mmh3 # type: ignore -from iceberg.types import ( +from pyiceberg.types import ( BinaryType, DateType, DecimalType, @@ -44,9 +44,9 @@ TimeType, UUIDType, ) -from iceberg.utils import datetime -from iceberg.utils.decimal import decimal_to_bytes, truncate_decimal -from src.iceberg.utils.singleton import Singleton +from pyiceberg.utils import datetime +from pyiceberg.utils.decimal import decimal_to_bytes, truncate_decimal +from pyiceberg.utils.singleton import Singleton S = TypeVar("S") T = TypeVar("T") diff --git a/src/iceberg/types.py b/pyiceberg/types.py similarity index 99% rename from src/iceberg/types.py rename to pyiceberg/types.py index ecfb65b10a..b8068d6b78 100644 --- a/src/iceberg/types.py +++ b/pyiceberg/types.py @@ -40,8 +40,8 @@ from pydantic import Field, PrivateAttr -from iceberg.utils.iceberg_base_model import IcebergBaseModel -from iceberg.utils.singleton import Singleton +from pyiceberg.utils.iceberg_base_model import IcebergBaseModel +from pyiceberg.utils.singleton import Singleton DECIMAL_REGEX = re.compile(r"decimal\((\d+),\s*(\d+)\)") FIXED_REGEX = re.compile(r"fixed\[(\d+)\]") diff --git a/src/iceberg/utils/__init__.py b/pyiceberg/utils/__init__.py similarity index 100% rename from src/iceberg/utils/__init__.py rename to pyiceberg/utils/__init__.py diff --git a/src/iceberg/utils/bin_packing.py b/pyiceberg/utils/bin_packing.py similarity index 100% rename from src/iceberg/utils/bin_packing.py rename to pyiceberg/utils/bin_packing.py diff --git a/src/iceberg/utils/datetime.py b/pyiceberg/utils/datetime.py similarity index 100% rename from src/iceberg/utils/datetime.py rename to pyiceberg/utils/datetime.py diff --git a/src/iceberg/utils/decimal.py b/pyiceberg/utils/decimal.py similarity index 100% rename from src/iceberg/utils/decimal.py rename to pyiceberg/utils/decimal.py diff --git a/src/iceberg/utils/iceberg_base_model.py b/pyiceberg/utils/iceberg_base_model.py similarity index 100% rename from src/iceberg/utils/iceberg_base_model.py rename to pyiceberg/utils/iceberg_base_model.py diff --git a/src/iceberg/utils/schema_conversion.py b/pyiceberg/utils/schema_conversion.py similarity index 96% rename from src/iceberg/utils/schema_conversion.py rename to pyiceberg/utils/schema_conversion.py index f5b70354b9..d6c32e0121 100644 --- a/src/iceberg/utils/schema_conversion.py +++ b/pyiceberg/utils/schema_conversion.py @@ -20,8 +20,8 @@ import logging from typing import Any -from iceberg.schema import Schema -from iceberg.types import ( +from pyiceberg.schema import Schema +from pyiceberg.types import ( BinaryType, BooleanType, DateType, @@ -226,7 +226,7 @@ def _convert_record_type(self, record_type: dict[str, Any]) -> StructType: Converts the fields from a record into an Iceberg struct Examples: - >>> from iceberg.utils.schema_conversion import AvroSchemaConversion + >>> from pyiceberg.utils.schema_conversion import AvroSchemaConversion >>> record_type = { ... "type": "record", ... "name": "r508", @@ -293,7 +293,7 @@ def _convert_map_type(self, map_type: dict[str, Any]) -> MapType: map_type: The dict that describes the Avro map type Examples: - >>> from iceberg.utils.schema_conversion import AvroSchemaConversion + >>> from pyiceberg.utils.schema_conversion import AvroSchemaConversion >>> avro_field = { ... "type": "map", ... "values": ["null", "long"], @@ -330,7 +330,7 @@ def _convert_logical_type(self, avro_logical_type: dict[str, Any]) -> IcebergTyp look it up in the mapping. Examples: - >>> from iceberg.utils.schema_conversion import AvroSchemaConversion + >>> from pyiceberg.utils.schema_conversion import AvroSchemaConversion >>> avro_logical_type = { ... "type": "int", ... "logicalType": "date" @@ -365,7 +365,7 @@ def _convert_logical_decimal_type(self, avro_type: dict[str, Any]) -> DecimalTyp avro_type: The Avro type Examples: - >>> from iceberg.utils.schema_conversion import AvroSchemaConversion + >>> from pyiceberg.utils.schema_conversion import AvroSchemaConversion >>> avro_decimal_type = { ... "type": "bytes", ... "logicalType": "decimal", @@ -394,7 +394,7 @@ def _convert_logical_map_type(self, avro_type: dict[str, Any]) -> MapType: avro_type: The Avro Type Examples: - >>> from iceberg.utils.schema_conversion import AvroSchemaConversion + >>> from pyiceberg.utils.schema_conversion import AvroSchemaConversion >>> avro_type = { ... "type": "array", ... "logicalType": "map", @@ -445,7 +445,7 @@ def _convert_fixed_type(self, avro_type: dict[str, Any]) -> FixedType: avro_type: The Avro Type Examples: - >>> from iceberg.utils.schema_conversion import AvroSchemaConversion + >>> from pyiceberg.utils.schema_conversion import AvroSchemaConversion >>> avro_fixed_type = { ... "name": "md5", ... "type": "fixed", diff --git a/src/iceberg/utils/singleton.py b/pyiceberg/utils/singleton.py similarity index 100% rename from src/iceberg/utils/singleton.py rename to pyiceberg/utils/singleton.py diff --git a/pyproject.toml b/pyproject.toml index 07e5ff12ed..6500a5dbf4 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -16,7 +16,7 @@ # under the License. [tool.poetry] -name = "apache-iceberg" +name = "pyiceberg" version = "0.0.1rc1" readme = "README.md" homepage = "https://iceberg.apache.org/" @@ -34,7 +34,7 @@ classifiers = [ ] packages = [ - { include = "iceberg", from = "src" }, + { include = "pyiceberg" }, ] [tool.poetry.dependencies] @@ -69,7 +69,7 @@ line-length = 130 target-version = ['py38'] [tool.isort] -src_paths = ["src/", "tests/"] +src_paths = ["pyiceberg/", "tests/"] multi_line_output = 3 profile = 'black' line_length = 130 @@ -96,4 +96,4 @@ module = "mypy-zstandard.*" ignore_missing_imports = true [tool.coverage.run] -source = ['src/'] +source = ['pyiceberg/'] diff --git a/tests/avro/test_decoder.py b/tests/avro/test_decoder.py index b988cb6c49..dbf940eb56 100644 --- a/tests/avro/test_decoder.py +++ b/tests/avro/test_decoder.py @@ -20,11 +20,11 @@ import pytest -from iceberg.avro.decoder import BinaryDecoder -from iceberg.avro.resolver import promote -from iceberg.io.base import InputStream -from iceberg.io.memory import MemoryInputStream -from iceberg.types import DoubleType, FloatType +from pyiceberg.avro.decoder import BinaryDecoder +from pyiceberg.avro.resolver import promote +from pyiceberg.io.base import InputStream +from pyiceberg.io.memory import MemoryInputStream +from pyiceberg.types import DoubleType, FloatType def test_read_decimal_from_fixed(): diff --git a/tests/avro/test_file.py b/tests/avro/test_file.py index d345865d11..177f7eb98d 100644 --- a/tests/avro/test_file.py +++ b/tests/avro/test_file.py @@ -16,8 +16,8 @@ # under the License. import pytest -from iceberg.avro.codecs import DeflateCodec -from iceberg.avro.file import AvroFileHeader +from pyiceberg.avro.codecs import DeflateCodec +from pyiceberg.avro.file import AvroFileHeader def get_deflate_compressor(): diff --git a/tests/avro/test_reader.py b/tests/avro/test_reader.py index 96afa390f8..9790590fd4 100644 --- a/tests/avro/test_reader.py +++ b/tests/avro/test_reader.py @@ -19,8 +19,8 @@ import pytest -from iceberg.avro.file import AvroFile -from iceberg.avro.reader import ( +from pyiceberg.avro.file import AvroFile +from pyiceberg.avro.reader import ( AvroStruct, BinaryReader, BooleanReader, @@ -36,8 +36,8 @@ TimestamptzReader, primitive_reader, ) -from iceberg.schema import Schema -from iceberg.types import ( +from pyiceberg.schema import Schema +from pyiceberg.types import ( BinaryType, BooleanType, DateType, diff --git a/tests/avro/test_resolver.py b/tests/avro/test_resolver.py index 197dc95246..3b51a4ab48 100644 --- a/tests/avro/test_resolver.py +++ b/tests/avro/test_resolver.py @@ -16,7 +16,7 @@ # under the License. import pytest -from iceberg.avro.reader import ( +from pyiceberg.avro.reader import ( DecimalReader, DoubleReader, FloatReader, @@ -25,9 +25,9 @@ StringReader, StructReader, ) -from iceberg.avro.resolver import ResolveException, promote, resolve -from iceberg.schema import Schema -from iceberg.types import ( +from pyiceberg.avro.resolver import ResolveException, promote, resolve +from pyiceberg.schema import Schema +from pyiceberg.types import ( BinaryType, DecimalType, DoubleType, diff --git a/tests/catalog/test_base.py b/tests/catalog/test_base.py index 3e5606427c..3aa078e84c 100644 --- a/tests/catalog/test_base.py +++ b/tests/catalog/test_base.py @@ -25,16 +25,16 @@ import pytest -from iceberg.catalog import Identifier, Properties -from iceberg.catalog.base import Catalog -from iceberg.exceptions import ( +from pyiceberg.catalog import Identifier, Properties +from pyiceberg.catalog.base import Catalog +from pyiceberg.exceptions import ( AlreadyExistsError, NamespaceNotEmptyError, NoSuchNamespaceError, NoSuchTableError, ) -from iceberg.schema import Schema -from iceberg.table.base import PartitionSpec, Table +from pyiceberg.schema import Schema +from pyiceberg.table.base import PartitionSpec, Table class InMemoryCatalog(Catalog): diff --git a/tests/conftest.py b/tests/conftest.py index d211fd13dc..73ae489a3a 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -31,15 +31,15 @@ import pytest -from iceberg import schema -from iceberg.io.base import ( +from pyiceberg import schema +from pyiceberg.io.base import ( FileIO, InputFile, OutputFile, OutputStream, ) -from iceberg.schema import Schema -from iceberg.types import ( +from pyiceberg.schema import Schema +from pyiceberg.types import ( BinaryType, BooleanType, DoubleType, diff --git a/tests/expressions/test_expressions_base.py b/tests/expressions/test_expressions_base.py index a6f8f07f76..db67159571 100644 --- a/tests/expressions/test_expressions_base.py +++ b/tests/expressions/test_expressions_base.py @@ -21,9 +21,9 @@ import pytest -from iceberg.expressions import base -from iceberg.types import NestedField, StringType -from iceberg.utils.singleton import Singleton +from pyiceberg.expressions import base +from pyiceberg.types import NestedField, StringType +from pyiceberg.utils.singleton import Singleton @pytest.mark.parametrize( diff --git a/tests/expressions/test_literals.py b/tests/expressions/test_literals.py index c1eb48aa20..019c925bfc 100644 --- a/tests/expressions/test_literals.py +++ b/tests/expressions/test_literals.py @@ -20,7 +20,7 @@ import pytest -from iceberg.expressions.literals import ( +from pyiceberg.expressions.literals import ( AboveMax, BelowMin, BinaryLiteral, @@ -36,7 +36,7 @@ TimestampLiteral, literal, ) -from iceberg.types import ( +from pyiceberg.types import ( BinaryType, BooleanType, DateType, diff --git a/tests/io/test_io_base.py b/tests/io/test_io_base.py index a4ffe7dff9..a7a049ecdd 100644 --- a/tests/io/test_io_base.py +++ b/tests/io/test_io_base.py @@ -22,14 +22,14 @@ import pytest -from iceberg.io.base import ( +from pyiceberg.io.base import ( FileIO, InputFile, InputStream, OutputFile, OutputStream, ) -from iceberg.io.pyarrow import PyArrowFile, PyArrowFileIO +from pyiceberg.io.pyarrow import PyArrowFile, PyArrowFileIO class LocalInputFile(InputFile): diff --git a/tests/io/test_pyarrow.py b/tests/io/test_pyarrow.py index f3a41bb54d..67bdee4427 100644 --- a/tests/io/test_pyarrow.py +++ b/tests/io/test_pyarrow.py @@ -23,8 +23,8 @@ import pytest from pyarrow.fs import FileType -from iceberg.io.base import InputStream, OutputStream -from iceberg.io.pyarrow import PyArrowFile, PyArrowFileIO +from pyiceberg.io.base import InputStream, OutputStream +from pyiceberg.io.pyarrow import PyArrowFile, PyArrowFileIO def test_pyarrow_input_file(): @@ -195,8 +195,8 @@ def test_raise_on_delete_file_with_no_permission(): assert "Cannot delete file" in str(exc_info.value) -@patch("iceberg.io.pyarrow.PyArrowFile.exists", return_value=False) -@patch("iceberg.io.pyarrow.FileSystem") +@patch("pyiceberg.io.pyarrow.PyArrowFile.exists", return_value=False) +@patch("pyiceberg.io.pyarrow.FileSystem") def test_raise_on_opening_an_s3_file_no_permission(filesystem_mock, exists_mock): """Test that opening a PyArrowFile raises a PermissionError when the pyarrow error includes 'AWS Error [code 15]'""" @@ -213,8 +213,8 @@ def test_raise_on_opening_an_s3_file_no_permission(filesystem_mock, exists_mock) assert "Cannot open file, access denied:" in str(exc_info.value) -@patch("iceberg.io.pyarrow.PyArrowFile.exists", return_value=False) -@patch("iceberg.io.pyarrow.FileSystem") +@patch("pyiceberg.io.pyarrow.PyArrowFile.exists", return_value=False) +@patch("pyiceberg.io.pyarrow.FileSystem") def test_raise_on_opening_an_s3_file_not_found(filesystem_mock, exists_mock): """Test that a PyArrowFile raises a FileNotFoundError when the pyarrow error includes 'Path does not exist'""" @@ -231,8 +231,8 @@ def test_raise_on_opening_an_s3_file_not_found(filesystem_mock, exists_mock): assert "Cannot open file, does not exist:" in str(exc_info.value) -@patch("iceberg.io.pyarrow.PyArrowFile.exists", return_value=False) -@patch("iceberg.io.pyarrow.FileSystem") +@patch("pyiceberg.io.pyarrow.PyArrowFile.exists", return_value=False) +@patch("pyiceberg.io.pyarrow.FileSystem") def test_raise_on_creating_an_s3_file_no_permission(filesystem_mock, exists_mock): """Test that creating a PyArrowFile raises a PermissionError when the pyarrow error includes 'AWS Error [code 15]'""" @@ -249,7 +249,7 @@ def test_raise_on_creating_an_s3_file_no_permission(filesystem_mock, exists_mock assert "Cannot create file, access denied:" in str(exc_info.value) -@patch("iceberg.io.pyarrow.FileSystem") +@patch("pyiceberg.io.pyarrow.FileSystem") def test_deleting_s3_file_no_permission(filesystem_mock): """Test that a PyArrowFile raises a PermissionError when the pyarrow OSError includes 'AWS Error [code 15]'""" @@ -266,7 +266,7 @@ def test_deleting_s3_file_no_permission(filesystem_mock): assert "Cannot delete file, access denied:" in str(exc_info.value) -@patch("iceberg.io.pyarrow.FileSystem") +@patch("pyiceberg.io.pyarrow.FileSystem") def test_deleting_s3_file_not_found(filesystem_mock): """Test that a PyArrowFile raises a PermissionError when the pyarrow error includes 'AWS Error [code 15]'""" diff --git a/tests/table/test_metadata.py b/tests/table/test_metadata.py index 7e0613b3db..e7dac41919 100644 --- a/tests/table/test_metadata.py +++ b/tests/table/test_metadata.py @@ -21,12 +21,12 @@ import pytest -from iceberg.exceptions import ValidationError -from iceberg.schema import Schema -from iceberg.serializers import FromByteStream -from iceberg.table.metadata import TableMetadata, TableMetadataV1, TableMetadataV2 -from iceberg.table.refs import SnapshotRef, SnapshotRefType -from iceberg.types import LongType, NestedField +from pyiceberg.exceptions import ValidationError +from pyiceberg.schema import Schema +from pyiceberg.serializers import FromByteStream +from pyiceberg.table.metadata import TableMetadata, TableMetadataV1, TableMetadataV2 +from pyiceberg.table.refs import SnapshotRef, SnapshotRefType +from pyiceberg.types import LongType, NestedField EXAMPLE_TABLE_METADATA_V1 = { "format-version": 1, diff --git a/tests/table/test_partitioning.py b/tests/table/test_partitioning.py index ff87f5b0d2..e6afac293f 100644 --- a/tests/table/test_partitioning.py +++ b/tests/table/test_partitioning.py @@ -15,10 +15,10 @@ # specific language governing permissions and limitations # under the License. -from iceberg.schema import Schema -from iceberg.table.partitioning import PartitionField, PartitionSpec -from iceberg.transforms import bucket -from iceberg.types import IntegerType +from pyiceberg.schema import Schema +from pyiceberg.table.partitioning import PartitionField, PartitionSpec +from pyiceberg.transforms import bucket +from pyiceberg.types import IntegerType def test_partition_field_init(): diff --git a/tests/test_conversions.py b/tests/test_conversions.py index 0f50868f7d..86f015b454 100644 --- a/tests/test_conversions.py +++ b/tests/test_conversions.py @@ -77,9 +77,9 @@ import pytest -import iceberg.utils.decimal as decimal_util -from iceberg import conversions -from iceberg.types import ( +import pyiceberg.utils.decimal as decimal_util +from pyiceberg import conversions +from pyiceberg.types import ( BinaryType, BooleanType, DateType, diff --git a/tests/test_schema.py b/tests/test_schema.py index a4f93c1817..639fa0076e 100644 --- a/tests/test_schema.py +++ b/tests/test_schema.py @@ -20,11 +20,11 @@ import pytest -from iceberg import schema -from iceberg.expressions.base import Accessor -from iceberg.files import StructProtocol -from iceberg.schema import Schema, build_position_accessors -from iceberg.types import ( +from pyiceberg import schema +from pyiceberg.expressions.base import Accessor +from pyiceberg.files import StructProtocol +from pyiceberg.schema import Schema, build_position_accessors +from pyiceberg.types import ( BooleanType, FloatType, IntegerType, diff --git a/tests/test_transforms.py b/tests/test_transforms.py index dc3ce4ec27..f7a772212a 100644 --- a/tests/test_transforms.py +++ b/tests/test_transforms.py @@ -22,8 +22,8 @@ import mmh3 as mmh3 import pytest -from iceberg import transforms -from iceberg.types import ( +from pyiceberg import transforms +from pyiceberg.types import ( BinaryType, BooleanType, DateType, @@ -39,7 +39,7 @@ TimeType, UUIDType, ) -from iceberg.utils.datetime import ( +from pyiceberg.utils.datetime import ( date_to_days, time_to_micros, timestamp_to_micros, diff --git a/tests/test_types.py b/tests/test_types.py index 1695f26571..65f047acce 100644 --- a/tests/test_types.py +++ b/tests/test_types.py @@ -18,7 +18,7 @@ import pytest -from iceberg.types import ( +from pyiceberg.types import ( BinaryType, BooleanType, DateType, @@ -39,7 +39,7 @@ TimeType, UUIDType, ) -from iceberg.utils.iceberg_base_model import IcebergBaseModel +from pyiceberg.utils.iceberg_base_model import IcebergBaseModel non_parameterized_types = [ (1, BooleanType), diff --git a/tests/utils/test_bin_packing.py b/tests/utils/test_bin_packing.py index 59e5f56fe2..71fd53c027 100644 --- a/tests/utils/test_bin_packing.py +++ b/tests/utils/test_bin_packing.py @@ -19,8 +19,8 @@ import pytest -from iceberg.schema import Schema -from iceberg.utils.bin_packing import PackingIterator +from pyiceberg.schema import Schema +from pyiceberg.utils.bin_packing import PackingIterator @pytest.mark.parametrize( diff --git a/tests/utils/test_schema_conversion.py b/tests/utils/test_schema_conversion.py index e2eada5c40..3abf19c6da 100644 --- a/tests/utils/test_schema_conversion.py +++ b/tests/utils/test_schema_conversion.py @@ -19,8 +19,8 @@ import pytest -from iceberg.schema import Schema -from iceberg.types import ( +from pyiceberg.schema import Schema +from pyiceberg.types import ( BinaryType, BooleanType, DateType, @@ -34,7 +34,7 @@ StringType, StructType, ) -from iceberg.utils.schema_conversion import AvroSchemaConversion +from pyiceberg.utils.schema_conversion import AvroSchemaConversion def test_iceberg_to_avro(avro_schema_manifest_file: Dict[str, Any]): diff --git a/tests/utils/test_singleton.py b/tests/utils/test_singleton.py index 78ac177a85..b9cf33cc6a 100644 --- a/tests/utils/test_singleton.py +++ b/tests/utils/test_singleton.py @@ -14,8 +14,8 @@ # KIND, either express or implied. See the License for the # specific language governing permissions and limitations # under the License. -from iceberg.avro.reader import BooleanReader, FixedReader -from src.iceberg.transforms import VoidTransform +from pyiceberg.avro.reader import BooleanReader, FixedReader +from pyiceberg.transforms import VoidTransform def test_singleton(): From 822196cc9d2c84a6140d75172a64c400d195c218 Mon Sep 17 00:00:00 2001 From: Fokko Driesprong Date: Fri, 1 Jul 2022 17:21:15 +0200 Subject: [PATCH 128/642] Python: Move Transforms to Pydantic (#5170) --- pyiceberg/transforms.py | 145 +++++++++++++++++++++------------------ tests/test_transforms.py | 132 +++++++++++++++++++++++++++++++++++ 2 files changed, 210 insertions(+), 67 deletions(-) diff --git a/pyiceberg/transforms.py b/pyiceberg/transforms.py index d90736d613..ca337ed584 100644 --- a/pyiceberg/transforms.py +++ b/pyiceberg/transforms.py @@ -23,12 +23,14 @@ from typing import ( Any, Generic, + Literal, Optional, TypeVar, ) from uuid import UUID import mmh3 # type: ignore +from pydantic import Field, PositiveInt, PrivateAttr from pyiceberg.types import ( BinaryType, @@ -46,32 +48,21 @@ ) from pyiceberg.utils import datetime from pyiceberg.utils.decimal import decimal_to_bytes, truncate_decimal +from pyiceberg.utils.iceberg_base_model import IcebergBaseModel from pyiceberg.utils.singleton import Singleton S = TypeVar("S") T = TypeVar("T") -class Transform(ABC, Generic[S, T]): +class Transform(IcebergBaseModel, ABC, Generic[S, T]): """Transform base class for concrete transforms. A base class to transform values and project predicates on partition values. This class is not used directly. Instead, use one of module method to create the child classes. - - Args: - transform_string (str): name of the transform type - repr_string (str): string representation of a transform instance """ - def __init__(self, transform_string: str, repr_string: str): - self._transform_string = transform_string - self._repr_string = repr_string - - def __repr__(self): - return self._repr_string - - def __str__(self): - return self._transform_string + __root__: str = Field() def __call__(self, value: Optional[S]) -> Optional[T]: return self.apply(value) @@ -100,7 +91,10 @@ def to_human_string(self, value: Optional[S]) -> str: @property def dedup_name(self) -> str: - return self._transform_string + return self.__str__() + + def __str__(self) -> str: + return self.__root__ class BaseBucketTransform(Transform[S, int]): @@ -115,11 +109,12 @@ class BaseBucketTransform(Transform[S, int]): num_buckets (int): The number of buckets. """ - def __init__(self, source_type: IcebergType, num_buckets: int): - super().__init__( - f"bucket[{num_buckets}]", - f"transforms.bucket(source_type={repr(source_type)}, num_buckets={num_buckets})", - ) + _source_type: IcebergType = PrivateAttr() + _num_buckets: PositiveInt = PrivateAttr() + + def __init__(self, source_type: IcebergType, num_buckets: int, **data: Any): + super().__init__(__root__=f"bucket[{num_buckets}]", **data) + self._source_type = source_type self._num_buckets = num_buckets @property @@ -139,6 +134,9 @@ def result_type(self, source: IcebergType) -> IcebergType: def can_transform(self, source: IcebergType) -> bool: pass + def __repr__(self) -> str: + return f"transforms.bucket(source_type={repr(self._source_type)}, num_buckets={self._num_buckets})" + class BucketNumberTransform(BaseBucketTransform): """Transforms a value of IntegerType, LongType, DateType, TimeType, TimestampType, or TimestamptzType @@ -177,14 +175,11 @@ class BucketStringTransform(BaseBucketTransform): """Transforms a value of StringType into a bucket partition value. Example: - >>> transform = BucketStringTransform(100) + >>> transform = BucketStringTransform(StringType(), 100) >>> transform.apply("iceberg") 89 """ - def __init__(self, num_buckets: int): - super().__init__(StringType(), num_buckets) - def can_transform(self, source: IcebergType) -> bool: return isinstance(source, StringType) @@ -212,14 +207,11 @@ class BucketUUIDTransform(BaseBucketTransform): """Transforms a value of UUIDType into a bucket partition value. Example: - >>> transform = BucketUUIDTransform(100) + >>> transform = BucketUUIDTransform(UUIDType(), 100) >>> transform.apply(UUID("f79c3e09-677c-4bbd-a479-3f349cb785e7")) 40 """ - def __init__(self, num_buckets: int): - super().__init__(UUIDType(), num_buckets) - def can_transform(self, source: IcebergType) -> bool: return isinstance(source, UUIDType) @@ -247,12 +239,12 @@ class IdentityTransform(Transform[S, S]): 'hello-world' """ - def __init__(self, source_type: IcebergType): - super().__init__( - "identity", - f"transforms.identity(source_type={repr(source_type)})", - ) - self._type = source_type + __root__: Literal["identity"] = Field(default="identity") + _source_type: IcebergType = PrivateAttr() + + def __init__(self, source_type: IcebergType, **data: Any): + super().__init__(**data) + self._source_type = source_type def apply(self, value: Optional[S]) -> Optional[S]: return value @@ -272,40 +264,38 @@ def satisfies_order_of(self, other: Transform) -> bool: return other.preserves_order def to_human_string(self, value: Optional[S]) -> str: - return _human_string(value, self._type) if value is not None else "null" + return _human_string(value, self._source_type) if value is not None else "null" + + def __str__(self) -> str: + return "identity" + + def __repr__(self) -> str: + return f"transforms.identity(source_type={repr(self._source_type)})" class TruncateTransform(Transform[S, S]): """A transform for truncating a value to a specified width. Args: source_type (Type): An Iceberg Type of IntegerType, LongType, StringType, BinaryType or DecimalType - width (int): The truncate width + width (int): The truncate width, should be positive Raises: ValueError: If a type is provided that is incompatible with a Truncate transform """ - def __init__(self, source_type: IcebergType, width: int): - assert width > 0, f"width ({width}) should be greater than 0" - super().__init__( - f"truncate[{width}]", - f"transforms.truncate(source_type={repr(source_type)}, width={width})", - ) - self._type = source_type - self._width = width + __root__: str = Field() + _source_type: IcebergType = PrivateAttr() + _width: PositiveInt = PrivateAttr() - @property - def width(self) -> int: - return self._width - - @property - def type(self) -> IcebergType: - return self._type + def __init__(self, source_type: IcebergType, width: int, **data: Any): + super().__init__(__root__=f"truncate[{width}]", **data) + self._source_type = source_type + self._width = width def apply(self, value: Optional[S]) -> Optional[S]: return _truncate_value(value, self._width) if value is not None else None def can_transform(self, source: IcebergType) -> bool: - return self._type == source + return self._source_type == source def result_type(self, source: IcebergType) -> IcebergType: return source @@ -314,11 +304,23 @@ def result_type(self, source: IcebergType) -> IcebergType: def preserves_order(self) -> bool: return True + @property + def source_type(self) -> IcebergType: + return self._source_type + + @property + def width(self) -> int: + return self._width + def satisfies_order_of(self, other: Transform) -> bool: if self == other: return True - elif isinstance(self._type, StringType) and isinstance(other, TruncateTransform) and isinstance(other.type, StringType): - return self._width >= other.width + elif ( + isinstance(self.source_type, StringType) + and isinstance(other, TruncateTransform) + and isinstance(other.source_type, StringType) + ): + return self.width >= other.width return False @@ -330,6 +332,9 @@ def to_human_string(self, value: Optional[S]) -> str: else: return str(value) + def __repr__(self) -> str: + return f"transforms.truncate(source_type={repr(self._source_type)}, width={self._width})" + @singledispatch def _human_string(value: Any, _type: IcebergType) -> str: @@ -403,35 +408,38 @@ def _(value: Decimal, _width: int) -> Decimal: class UnknownTransform(Transform): """A transform that represents when an unknown transform is provided Args: - source_type (Type): An Iceberg `Type` + source_type (IcebergType): An Iceberg `Type` transform (str): A string name of a transform Raises: AttributeError: If the apply method is called. """ - def __init__(self, source_type: IcebergType, transform: str): - super().__init__( - transform, - f"transforms.UnknownTransform(source_type={repr(source_type)}, transform={repr(transform)})", - ) - self._type = source_type + __root__: Literal["unknown"] = Field(default="unknown") + _source_type: IcebergType = PrivateAttr() + _transform: str = PrivateAttr() + + def __init__(self, source_type: IcebergType, transform: str, **data: Any): + super().__init__(**data) + self._source_type = source_type self._transform = transform def apply(self, value: Optional[S]): raise AttributeError(f"Cannot apply unsupported transform: {self}") def can_transform(self, source: IcebergType) -> bool: - return self._type == source + return self._source_type == source def result_type(self, source: IcebergType) -> IcebergType: return StringType() + def __repr__(self) -> str: + return f"transforms.UnknownTransform(source_type={repr(self._source_type)}, transform={repr(self._transform)})" + class VoidTransform(Transform, Singleton): """A transform that always returns None""" - def __init__(self): - super().__init__("void", "transforms.always_null()") + __root__ = "void" def apply(self, value: Optional[S]) -> None: return None @@ -445,6 +453,9 @@ def result_type(self, source: IcebergType) -> IcebergType: def to_human_string(self, value: Optional[S]) -> str: return "null" + def __repr__(self) -> str: + return "transforms.always_null()" + def bucket(source_type: IcebergType, num_buckets: int) -> BaseBucketTransform: if type(source_type) in {IntegerType, LongType, DateType, TimeType, TimestampType, TimestamptzType}: @@ -452,13 +463,13 @@ def bucket(source_type: IcebergType, num_buckets: int) -> BaseBucketTransform: elif isinstance(source_type, DecimalType): return BucketDecimalTransform(source_type, num_buckets) elif isinstance(source_type, StringType): - return BucketStringTransform(num_buckets) + return BucketStringTransform(source_type, num_buckets) elif isinstance(source_type, BinaryType): return BucketBytesTransform(source_type, num_buckets) elif isinstance(source_type, FixedType): return BucketBytesTransform(source_type, num_buckets) elif isinstance(source_type, UUIDType): - return BucketUUIDTransform(num_buckets) + return BucketUUIDTransform(source_type, num_buckets) else: raise ValueError(f"Cannot bucket by type: {source_type}") @@ -471,5 +482,5 @@ def truncate(source_type: IcebergType, width: int) -> TruncateTransform: return TruncateTransform(source_type, width) -def always_null() -> Transform: +def always_null() -> VoidTransform: return VoidTransform() diff --git a/tests/test_transforms.py b/tests/test_transforms.py index f7a772212a..ca2f441bce 100644 --- a/tests/test_transforms.py +++ b/tests/test_transforms.py @@ -23,6 +23,17 @@ import pytest from pyiceberg import transforms +from pyiceberg.transforms import ( + BucketBytesTransform, + BucketDecimalTransform, + BucketNumberTransform, + BucketStringTransform, + BucketUUIDTransform, + IdentityTransform, + TruncateTransform, + UnknownTransform, + VoidTransform, +) from pyiceberg.types import ( BinaryType, BooleanType, @@ -256,3 +267,124 @@ def test_void_transform(): assert not void_transform.satisfies_order_of(transforms.bucket(DateType(), 100)) assert void_transform.to_human_string("test") == "null" assert void_transform.dedup_name == "void" + + +def test_bucket_number_transform_json(): + assert BucketNumberTransform(source_type=IntegerType(), num_buckets=22).json() == '"bucket[22]"' + + +def test_bucket_number_transform_str(): + assert str(BucketNumberTransform(source_type=IntegerType(), num_buckets=22)) == "bucket[22]" + + +def test_bucket_number_transform_repr(): + assert ( + repr(BucketNumberTransform(source_type=IntegerType(), num_buckets=22)) + == "transforms.bucket(source_type=IntegerType(), num_buckets=22)" + ) + + +def test_bucket_decimal_transform_json(): + assert BucketDecimalTransform(source_type=DecimalType(19, 25), num_buckets=22).json() == '"bucket[22]"' + + +def test_bucket_decimal_transform_str(): + assert str(BucketDecimalTransform(source_type=DecimalType(19, 25), num_buckets=22)) == "bucket[22]" + + +def test_bucket_decimal_transform_repr(): + assert ( + repr(BucketDecimalTransform(source_type=DecimalType(19, 25), num_buckets=22)) + == "transforms.bucket(source_type=DecimalType(precision=19, scale=25), num_buckets=22)" + ) + + +def test_bucket_string_transform_json(): + assert BucketStringTransform(StringType(), num_buckets=22).json() == '"bucket[22]"' + + +def test_bucket_string_transform_str(): + assert str(BucketStringTransform(StringType(), num_buckets=22)) == "bucket[22]" + + +def test_bucket_string_transform_repr(): + assert ( + repr(BucketStringTransform(StringType(), num_buckets=22)) == "transforms.bucket(source_type=StringType(), num_buckets=22)" + ) + + +def test_bucket_bytes_transform_json(): + assert BucketBytesTransform(BinaryType(), num_buckets=22).json() == '"bucket[22]"' + + +def test_bucket_bytes_transform_str(): + assert str(BucketBytesTransform(BinaryType(), num_buckets=22)) == "bucket[22]" + + +def test_bucket_bytes_transform_repr(): + assert ( + repr(BucketBytesTransform(BinaryType(), num_buckets=22)) == "transforms.bucket(source_type=BinaryType(), num_buckets=22)" + ) + + +def test_bucket_uuid_transform_json(): + assert BucketUUIDTransform(UUIDType(), num_buckets=22).json() == '"bucket[22]"' + + +def test_bucket_uuid_transform_str(): + assert str(BucketUUIDTransform(UUIDType(), num_buckets=22)) == "bucket[22]" + + +def test_bucket_uuid_transform_repr(): + assert repr(BucketUUIDTransform(UUIDType(), num_buckets=22)) == "transforms.bucket(source_type=UUIDType(), num_buckets=22)" + + +def test_identity_transform_json(): + assert IdentityTransform(StringType()).json() == '"identity"' + + +def test_identity_transform_str(): + assert str(IdentityTransform(StringType())) == "identity" + + +def test_identity_transform_repr(): + assert repr(IdentityTransform(StringType())) == "transforms.identity(source_type=StringType())" + + +def test_truncate_transform_json(): + assert TruncateTransform(StringType(), 22).json() == '"truncate[22]"' + + +def test_truncate_transform_str(): + assert str(TruncateTransform(StringType(), 22)) == "truncate[22]" + + +def test_truncate_transform_repr(): + assert repr(TruncateTransform(StringType(), 22)) == "transforms.truncate(source_type=StringType(), width=22)" + + +def test_unknown_transform_json(): + assert UnknownTransform(StringType(), "unknown").json() == '"unknown"' + + +def test_unknown_transform_str(): + assert str(UnknownTransform(StringType(), "unknown")) == "unknown" + + +def test_unknown_transform_repr(): + assert ( + repr(UnknownTransform(StringType(), "unknown")) + == "transforms.UnknownTransform(source_type=StringType(), transform='unknown')" + ) + + +def test_void_transform_json(): + assert VoidTransform().json() == '"void"' + + +def test_void_transform_str(): + assert str(VoidTransform()) == "void" + + +def test_void_transform_repr(): + assert repr(VoidTransform()) == "transforms.always_null()" From f67cc9d7a9d4e2f992ff276a0c3fc2c0d4342317 Mon Sep 17 00:00:00 2001 From: Fokko Driesprong Date: Mon, 11 Jul 2022 00:42:32 +0200 Subject: [PATCH 129/642] Python: Reenable mypy (#5171) --- .pre-commit-config.yaml | 1 + pyiceberg/io/base.py | 13 ++----------- pyiceberg/io/memory.py | 9 +++------ pyproject.toml | 22 +++++++++++++++++++--- tests/avro/test_decoder.py | 5 +++-- tests/conftest.py | 2 +- tests/io/test_io_base.py | 2 +- tests/test_schema.py | 4 ++-- 8 files changed, 32 insertions(+), 26 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 235e44f2db..0b01ae1afc 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -40,6 +40,7 @@ repos: rev: v0.961 hooks: - id: mypy + args: [--config=python/pyproject.toml] - repo: https://github.com/hadialqattan/pycln rev: v1.3.4 hooks: diff --git a/pyiceberg/io/base.py b/pyiceberg/io/base.py index 458a3d591e..3098102b99 100644 --- a/pyiceberg/io/base.py +++ b/pyiceberg/io/base.py @@ -22,7 +22,6 @@ for returning an InputFile instance, an OutputFile instance, and deleting a file given its location. """ - from abc import ABC, abstractmethod from io import SEEK_SET from typing import Protocol, Union, runtime_checkable @@ -41,17 +40,13 @@ def read(self, size: int = 0) -> bytes: ... @abstractmethod - def seek(self, offset: int, whence: int = SEEK_SET) -> None: + def seek(self, offset: int, whence: int = SEEK_SET) -> int: ... @abstractmethod def tell(self) -> int: ... - @abstractmethod - def closed(self) -> bool: - ... - @abstractmethod def close(self) -> None: ... @@ -66,11 +61,7 @@ class OutputStream(Protocol): # pragma: no cover """ @abstractmethod - def write(self, b: bytes) -> None: - ... - - @abstractmethod - def closed(self) -> bool: + def write(self, b: bytes) -> int: ... @abstractmethod diff --git a/pyiceberg/io/memory.py b/pyiceberg/io/memory.py index 082a589f12..73f0d5125d 100644 --- a/pyiceberg/io/memory.py +++ b/pyiceberg/io/memory.py @@ -36,8 +36,6 @@ class MemoryInputStream(InputStream): >>> stream.read(4) b'1925' >>> stream.close() - >>> stream.closed() - True """ buffer: bytes @@ -54,7 +52,7 @@ def read(self, size: int = 0) -> bytes: self.pos += size return b - def seek(self, offset: int, whence: int = SEEK_SET) -> None: + def seek(self, offset: int, whence: int = SEEK_SET) -> int: if whence == SEEK_SET: self.pos = offset elif whence == SEEK_CUR: @@ -64,11 +62,10 @@ def seek(self, offset: int, whence: int = SEEK_SET) -> None: else: raise ValueError(f"Unknown whence {offset}") - def tell(self) -> int: return self.pos - def closed(self) -> bool: - return not hasattr(self, "buffer") + def tell(self) -> int: + return self.pos def close(self) -> None: del self.buffer diff --git a/pyproject.toml b/pyproject.toml index 6500a5dbf4..a3e35bfc94 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -84,15 +84,31 @@ warn_redundant_casts = true warn_unreachable = true [[tool.mypy.overrides]] -module = "mypy-pyarrow.*" +module = "pyarrow.*" ignore_missing_imports = true [[tool.mypy.overrides]] -module = "mypy-snappy.*" +module = "snappy.*" ignore_missing_imports = true [[tool.mypy.overrides]] -module = "mypy-zstandard.*" +module = "zstandard.*" +ignore_missing_imports = true + +[[tool.mypy.overrides]] +module = "pydantic.*" +ignore_missing_imports = true + +[[tool.mypy.overrides]] +module = "pytest.*" +ignore_missing_imports = true + +[[tool.mypy.overrides]] +module = "fastavro.*" +ignore_missing_imports = true + +[[tool.mypy.overrides]] +module = "mmh3.*" ignore_missing_imports = true [tool.coverage.run] diff --git a/tests/avro/test_decoder.py b/tests/avro/test_decoder.py index dbf940eb56..0616d49314 100644 --- a/tests/avro/test_decoder.py +++ b/tests/avro/test_decoder.py @@ -106,14 +106,15 @@ def read(self, size: int = 0) -> bytes: self.pos += 1 return int.to_bytes(1, self.pos, byteorder="little") - def seek(self, offset: int, whence: int = SEEK_SET) -> None: + def seek(self, offset: int, whence: int = SEEK_SET) -> int: pass def tell(self) -> int: pass + @property def closed(self) -> bool: - pass + return False def close(self) -> None: pass diff --git a/tests/conftest.py b/tests/conftest.py index 73ae489a3a..1b004ec8f8 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -828,7 +828,7 @@ def to_input_file(self): def create(self, overwrite: bool = False) -> OutputStream: output_file = open(self._path, "wb" if overwrite else "xb") - if not isinstance(output_file, OutputStream): + if not issubclass(type(output_file), OutputStream): raise TypeError("Object returned from LocalOutputFile.create(...) does not match the OutputStream protocol.") return output_file diff --git a/tests/io/test_io_base.py b/tests/io/test_io_base.py index a7a049ecdd..3165c4099a 100644 --- a/tests/io/test_io_base.py +++ b/tests/io/test_io_base.py @@ -104,7 +104,7 @@ def to_input_file(self): def create(self, overwrite: bool = False) -> OutputStream: output_file = open(self.parsed_location.path, "wb" if overwrite else "xb") - if not isinstance(output_file, OutputStream): + if not issubclass(type(output_file), OutputStream): raise TypeError("Object returned from LocalOutputFile.create(...) does not match the OutputStream protocol.") return output_file diff --git a/tests/test_schema.py b/tests/test_schema.py index 639fa0076e..3d61323f02 100644 --- a/tests/test_schema.py +++ b/tests/test_schema.py @@ -16,7 +16,7 @@ # under the License. from textwrap import dedent -from typing import Any, Dict +from typing import Any, Dict, Optional import pytest @@ -388,7 +388,7 @@ def test_build_position_accessors(table_schema_nested): def test_build_position_accessors_with_struct(table_schema_nested: Schema): class TestStruct(StructProtocol): - def __init__(self, pos: Dict[int, Any] = None): + def __init__(self, pos: Optional[Dict[int, Any]] = None): self._pos: Dict[int, Any] = pos or {} def set(self, pos: int, value) -> None: From 5931e7d33e86735f97ae73780b6cc73f763ceeca Mon Sep 17 00:00:00 2001 From: Nick Ouellet <35903677+CircArgs@users.noreply.github.com> Date: Sun, 10 Jul 2022 19:10:45 -0400 Subject: [PATCH 130/642] Python: Remove operation enum, add In expression (#4816) Co-authored-by: Samuel Redai <43911210+samredai@users.noreply.github.com> --- pyiceberg/expressions/base.py | 274 ++++++++++----------- tests/expressions/test_expressions_base.py | 216 ++++++++-------- 2 files changed, 239 insertions(+), 251 deletions(-) diff --git a/pyiceberg/expressions/base.py b/pyiceberg/expressions/base.py index 7c5cd17bc0..b373049490 100644 --- a/pyiceberg/expressions/base.py +++ b/pyiceberg/expressions/base.py @@ -15,9 +15,14 @@ # specific language governing permissions and limitations # under the License. from abc import ABC, abstractmethod -from enum import Enum, auto +from dataclasses import dataclass from functools import reduce, singledispatch -from typing import Any, Generic, TypeVar +from typing import ( + Any, + Generic, + Tuple, + TypeVar, +) from pyiceberg.files import StructProtocol from pyiceberg.schema import Accessor, Schema @@ -25,68 +30,7 @@ from pyiceberg.utils.singleton import Singleton T = TypeVar("T") - - -class Operation(Enum): - """Operations to be used as components in expressions - - Operations can be negated by calling the negate method. - >>> Operation.TRUE.negate() - - >>> Operation.IS_NULL.negate() - - - The above example uses the OPERATION_NEGATIONS map which maps each enum - to it's opposite enum. - - Raises: - ValueError: This is raised when attempting to negate an operation - that cannot be negated. - """ - - TRUE = auto() - FALSE = auto() - IS_NULL = auto() - NOT_NULL = auto() - IS_NAN = auto() - NOT_NAN = auto() - LT = auto() - LT_EQ = auto() - GT = auto() - GT_EQ = auto() - EQ = auto() - NOT_EQ = auto() - IN = auto() - NOT_IN = auto() - NOT = auto() - AND = auto() - OR = auto() - - def negate(self) -> "Operation": - """Returns the operation used when this is negated.""" - - try: - return OPERATION_NEGATIONS[self] - except KeyError as e: - raise ValueError(f"No negation defined for operation {self}") from e - - -OPERATION_NEGATIONS = { - Operation.TRUE: Operation.FALSE, - Operation.FALSE: Operation.TRUE, - Operation.IS_NULL: Operation.NOT_NULL, - Operation.NOT_NULL: Operation.IS_NULL, - Operation.IS_NAN: Operation.NOT_NAN, - Operation.NOT_NAN: Operation.IS_NAN, - Operation.LT: Operation.GT_EQ, - Operation.LT_EQ: Operation.GT, - Operation.GT: Operation.LT_EQ, - Operation.GT_EQ: Operation.LT, - Operation.EQ: Operation.NOT_EQ, - Operation.NOT_EQ: Operation.EQ, - Operation.IN: Operation.NOT_IN, - Operation.NOT_IN: Operation.IN, -} +B = TypeVar("B") class Literal(Generic[T], ABC): @@ -102,7 +46,7 @@ def value(self) -> T: return self._value # type: ignore @abstractmethod - def to(self, type_var): + def to(self, type_var) -> "Literal": ... # pragma: no cover def __repr__(self): @@ -131,11 +75,115 @@ def __ge__(self, other): class BooleanExpression(ABC): - """base class for all boolean expressions""" + """Represents a boolean expression tree.""" @abstractmethod def __invert__(self) -> "BooleanExpression": - ... + """Transform the Expression into its negated version.""" + + +class Bound(Generic[T], ABC): + """Represents a bound value expression.""" + + def eval(self, struct: StructProtocol): # pylint: disable=W0613 + ... # pragma: no cover + + +class Unbound(Generic[T, B], ABC): + """Represents an unbound expression node.""" + + @abstractmethod + def bind(self, schema: Schema, case_sensitive: bool) -> B: + ... # pragma: no cover + + +class Term(ABC): + """An expression that evaluates to a value.""" + + +class BaseReference(Generic[T], Term, ABC): + """Represents a variable reference in an expression.""" + + +class BoundTerm(Bound[T], Term): + """Represents a bound term.""" + + +class UnboundTerm(Unbound[T, BoundTerm[T]], Term): + """Represents an unbound term.""" + + +@dataclass(frozen=True) +class BoundReference(BoundTerm[T], BaseReference[T]): + """A reference bound to a field in a schema + + Args: + field (NestedField): A referenced field in an Iceberg schema + accessor (Accessor): An Accessor object to access the value at the field's position + """ + + field: NestedField + accessor: Accessor + + def eval(self, struct: StructProtocol) -> Any: + """Returns the value at the referenced field's position in an object that abides by the StructProtocol + Args: + struct (StructProtocol): A row object that abides by the StructProtocol and returns values given a position + Returns: + Any: The value at the referenced field's position in `struct` + """ + return self.accessor.get(struct) + + +@dataclass(frozen=True) +class Reference(UnboundTerm[T], BaseReference[T]): + """A reference not yet bound to a field in a schema + + Args: + name (str): The name of the field + + Note: + An unbound reference is sometimes referred to as a "named" reference + """ + + name: str + + def bind(self, schema: Schema, case_sensitive: bool) -> BoundReference[T]: + """Bind the reference to an Iceberg schema + + Args: + schema (Schema): An Iceberg schema + case_sensitive (bool): Whether to consider case when binding the reference to the field + + Raises: + ValueError: If an empty name is provided + + Returns: + BoundReference: A reference bound to the specific field in the Iceberg schema + """ + field = schema.find_field(name_or_id=self.name, case_sensitive=case_sensitive) + + if not field: + raise ValueError(f"Cannot find field '{self.name}' in schema: {schema}") + + accessor = schema.accessor_for_field(field.field_id) + + if not accessor: + raise ValueError(f"Cannot find accessor for field '{self.name}' in schema: {schema}") + + return BoundReference(field=field, accessor=accessor) + + +@dataclass(frozen=True) # type: ignore[misc] +class BoundPredicate(Bound[T], BooleanExpression): + term: BoundReference[T] + literals: Tuple[Literal[T], ...] + + +@dataclass(frozen=True) # type: ignore[misc] +class UnboundPredicate(Unbound[T, BooleanExpression], BooleanExpression): + term: Reference[T] + literals: Tuple[Literal[T], ...] class And(BooleanExpression): @@ -268,90 +316,20 @@ def __str__(self) -> str: return "false" -class BoundReference: - """A reference bound to a field in a schema - - Args: - field (NestedField): A referenced field in an Iceberg schema - accessor (Accessor): An Accessor object to access the value at the field's position - """ - - def __init__(self, field: NestedField, accessor: Accessor): - self._field = field - self._accessor = accessor - - def __str__(self): - return f"BoundReference(field={repr(self.field)}, accessor={repr(self._accessor)})" - - def __repr__(self): - return f"BoundReference(field={repr(self.field)}, accessor={repr(self._accessor)})" - - @property - def field(self) -> NestedField: - """The referenced field""" - return self._field - - def eval(self, struct: StructProtocol) -> Any: - """Returns the value at the referenced field's position in an object that abides by the StructProtocol - - Args: - struct (StructProtocol): A row object that abides by the StructProtocol and returns values given a position +@dataclass(frozen=True) +class BoundIn(BoundPredicate[T]): + def __invert__(self): + raise TypeError("In expressions do not support negation.") - Returns: - Any: The value at the referenced field's position in `struct` - """ - return self._accessor.get(struct) +@dataclass(frozen=True) +class In(UnboundPredicate[T]): + def __invert__(self): + raise TypeError("In expressions do not support negation.") -class UnboundReference: - """A reference not yet bound to a field in a schema - - Args: - name (str): The name of the field - - Note: - An unbound reference is sometimes referred to as a "named" reference - """ - - def __init__(self, name: str): - if not name: - raise ValueError(f"Name cannot be null: {name}") - self._name = name - - def __str__(self) -> str: - return f"UnboundReference(name={repr(self.name)})" - - def __repr__(self) -> str: - return f"UnboundReference(name={repr(self.name)})" - - @property - def name(self) -> str: - return self._name - - def bind(self, schema: Schema, case_sensitive: bool) -> BoundReference: - """Bind the reference to an Iceberg schema - - Args: - schema (Schema): An Iceberg schema - case_sensitive (bool): Whether to consider case when binding the reference to the field - - Raises: - ValueError: If an empty name is provided - - Returns: - BoundReference: A reference bound to the specific field in the Iceberg schema - """ - field = schema.find_field(name_or_id=self.name, case_sensitive=case_sensitive) - - if not field: - raise ValueError(f"Cannot find field '{self.name}' in schema: {schema}") - - accessor = schema.accessor_for_field(field.field_id) - - if not accessor: - raise ValueError(f"Cannot find accessor for field '{self.name}' in schema: {schema}") - - return BoundReference(field=field, accessor=accessor) + def bind(self, schema: Schema, case_sensitive: bool) -> BoundIn: + bound_ref = self.term.bind(schema, case_sensitive) + return BoundIn(bound_ref, tuple(lit.to(bound_ref.field.field_type) for lit in self.literals)) # type: ignore class BooleanExpressionVisitor(Generic[T], ABC): diff --git a/tests/expressions/test_expressions_base.py b/tests/expressions/test_expressions_base.py index db67159571..ba5b04b741 100644 --- a/tests/expressions/test_expressions_base.py +++ b/tests/expressions/test_expressions_base.py @@ -22,71 +22,34 @@ import pytest from pyiceberg.expressions import base +from pyiceberg.expressions.literals import literal from pyiceberg.types import NestedField, StringType from pyiceberg.utils.singleton import Singleton -@pytest.mark.parametrize( - "operation,opposite_operation", - [ - (base.Operation.TRUE, base.Operation.FALSE), - (base.Operation.FALSE, base.Operation.TRUE), - (base.Operation.IS_NULL, base.Operation.NOT_NULL), - (base.Operation.NOT_NULL, base.Operation.IS_NULL), - (base.Operation.IS_NAN, base.Operation.NOT_NAN), - (base.Operation.NOT_NAN, base.Operation.IS_NAN), - (base.Operation.LT, base.Operation.GT_EQ), - (base.Operation.LT_EQ, base.Operation.GT), - (base.Operation.GT, base.Operation.LT_EQ), - (base.Operation.GT_EQ, base.Operation.LT), - (base.Operation.EQ, base.Operation.NOT_EQ), - (base.Operation.NOT_EQ, base.Operation.EQ), - (base.Operation.IN, base.Operation.NOT_IN), - (base.Operation.NOT_IN, base.Operation.IN), - ], -) -def test_negation_of_operations(operation, opposite_operation): - assert operation.negate() == opposite_operation - - -@pytest.mark.parametrize( - "operation", - [ - base.Operation.NOT, - base.Operation.AND, - base.Operation.OR, - ], -) -def test_raise_on_no_negation_for_operation(operation): - with pytest.raises(ValueError) as exc_info: - operation.negate() - - assert str(exc_info.value) == f"No negation defined for operation {operation}" - - -class TestExpressionA(base.BooleanExpression, Singleton): +class ExpressionA(base.BooleanExpression, Singleton): def __invert__(self): - return TestExpressionB() + return ExpressionB() def __repr__(self): - return "TestExpressionA()" + return "ExpressionA()" def __str__(self): return "testexpra" -class TestExpressionB(base.BooleanExpression, Singleton): +class ExpressionB(base.BooleanExpression, Singleton): def __invert__(self): - return TestExpressionA() + return ExpressionA() def __repr__(self): - return "TestExpressionB()" + return "ExpressionB()" def __str__(self): return "testexprb" -class TestBooleanExpressionVisitor(base.BooleanExpressionVisitor[List]): +class BooleanExpressionVisitor(base.BooleanExpressionVisitor[List]): """A test implementation of a BooleanExpressionVisit As this visitor visits each node, it appends an element to a `visit_histor` list. This enables testing that a given expression is @@ -125,23 +88,23 @@ def visit_bound_predicate(self, predicate) -> List: return self.visit_history def visit_test_expression_a(self) -> List: - self.visit_history.append("TestExpressionA") + self.visit_history.append("ExpressionA") return self.visit_history def visit_test_expression_b(self) -> List: - self.visit_history.append("TestExpressionB") + self.visit_history.append("ExpressionB") return self.visit_history -@base.visit.register(TestExpressionA) -def _(obj: TestExpressionA, visitor: TestBooleanExpressionVisitor) -> List: - """Visit a TestExpressionA with a TestBooleanExpressionVisitor""" +@base.visit.register(ExpressionA) +def _(obj: ExpressionA, visitor: BooleanExpressionVisitor) -> List: + """Visit a ExpressionA with a BooleanExpressionVisitor""" return visitor.visit_test_expression_a() -@base.visit.register(TestExpressionB) -def _(obj: TestExpressionB, visitor: TestBooleanExpressionVisitor) -> List: - """Visit a TestExpressionB with a TestBooleanExpressionVisitor""" +@base.visit.register(ExpressionB) +def _(obj: ExpressionB, visitor: BooleanExpressionVisitor) -> List: + """Visit a ExpressionB with a BooleanExpressionVisitor""" return visitor.visit_test_expression_b() @@ -149,14 +112,14 @@ def _(obj: TestExpressionB, visitor: TestBooleanExpressionVisitor) -> List: "op, rep", [ ( - base.And(TestExpressionA(), TestExpressionB()), - "And(TestExpressionA(), TestExpressionB())", + base.And(ExpressionA(), ExpressionB()), + "And(ExpressionA(), ExpressionB())", ), ( - base.Or(TestExpressionA(), TestExpressionB()), - "Or(TestExpressionA(), TestExpressionB())", + base.Or(ExpressionA(), ExpressionB()), + "Or(ExpressionA(), ExpressionB())", ), - (base.Not(TestExpressionA()), "Not(TestExpressionA())"), + (base.Not(ExpressionA()), "Not(ExpressionA())"), ], ) def test_reprs(op, rep): @@ -166,9 +129,9 @@ def test_reprs(op, rep): @pytest.mark.parametrize( "op, string", [ - (base.And(TestExpressionA(), TestExpressionB()), "(testexpra and testexprb)"), - (base.Or(TestExpressionA(), TestExpressionB()), "(testexpra or testexprb)"), - (base.Not(TestExpressionA()), "(not testexpra)"), + (base.And(ExpressionA(), ExpressionB()), "(testexpra and testexprb)"), + (base.Or(ExpressionA(), ExpressionB()), "(testexpra or testexprb)"), + (base.Not(ExpressionA()), "(not testexpra)"), ], ) def test_strs(op, string): @@ -176,58 +139,105 @@ def test_strs(op, string): @pytest.mark.parametrize( - "exp, testexpra, testexprb", + "a, schema, case_sensitive, success", [ ( - base.And(TestExpressionA(), TestExpressionB()), - base.And(TestExpressionA(), TestExpressionB()), - base.Or(TestExpressionA(), TestExpressionB()), + base.In(base.Reference("foo"), (literal("hello"), literal("world"))), + "table_schema_simple", + True, + True, + ), + ( + base.In(base.Reference("not_foo"), (literal("hello"), literal("world"))), + "table_schema_simple", + False, + False, + ), + ( + base.In(base.Reference("Bar"), (literal("hello"), literal("world"))), + "table_schema_simple", + False, + True, ), ( - base.Or(TestExpressionA(), TestExpressionB()), - base.Or(TestExpressionA(), TestExpressionB()), - base.And(TestExpressionA(), TestExpressionB()), + base.In(base.Reference("Bar"), (literal("hello"), literal("world"))), + "table_schema_simple", + True, + False, ), - (base.Not(TestExpressionA()), base.Not(TestExpressionA()), TestExpressionB()), - (TestExpressionA(), TestExpressionA(), TestExpressionB()), - (TestExpressionB(), TestExpressionB(), TestExpressionA()), ], ) -def test_eq(exp, testexpra, testexprb): - assert exp == testexpra and exp != testexprb +def test_bind(a, schema, case_sensitive, success, request): + schema = request.getfixturevalue(schema) + if success: + assert a.bind(schema, case_sensitive).term.field == schema.find_field(a.term.name, case_sensitive) + else: + with pytest.raises(ValueError): + a.bind(schema, case_sensitive) @pytest.mark.parametrize( - "lhs, rhs", + "exp, testexpra, testexprb", [ ( - base.And(TestExpressionA(), TestExpressionB()), - base.Or(TestExpressionB(), TestExpressionA()), + base.And(ExpressionA(), ExpressionB()), + base.And(ExpressionA(), ExpressionB()), + base.Or(ExpressionA(), ExpressionB()), + ), + ( + base.Or(ExpressionA(), ExpressionB()), + base.Or(ExpressionA(), ExpressionB()), + base.And(ExpressionA(), ExpressionB()), + ), + (base.Not(ExpressionA()), base.Not(ExpressionA()), ExpressionB()), + (ExpressionA(), ExpressionA(), ExpressionB()), + (ExpressionB(), ExpressionB(), ExpressionA()), + ( + base.In(base.Reference("foo"), (literal("hello"), literal("world"))), + base.In(base.Reference("foo"), (literal("hello"), literal("world"))), + base.In(base.Reference("not_foo"), (literal("hello"), literal("world"))), ), ( - base.Or(TestExpressionA(), TestExpressionB()), - base.And(TestExpressionB(), TestExpressionA()), + base.In(base.Reference("foo"), (literal("hello"), literal("world"))), + base.In(base.Reference("foo"), (literal("hello"), literal("world"))), + base.In(base.Reference("foo"), (literal("goodbye"), literal("world"))), ), - (base.Not(TestExpressionA()), TestExpressionA()), - (TestExpressionA(), TestExpressionB()), ], ) -def test_negate(lhs, rhs): - assert ~lhs == rhs +def test_eq(exp, testexpra, testexprb): + assert exp == testexpra and exp != testexprb + + +@pytest.mark.parametrize( + "lhs, rhs, raises", + [ + (base.And(ExpressionA(), ExpressionB()), base.Or(ExpressionB(), ExpressionA()), False), + (base.Or(ExpressionA(), ExpressionB()), base.And(ExpressionB(), ExpressionA()), False), + (base.Not(ExpressionA()), ExpressionA(), False), + (base.In(base.Reference("foo"), (literal("hello"), literal("world"))), None, True), + (ExpressionA(), ExpressionB(), False), + ], +) +def test_negate(lhs, rhs, raises): + if not raises: + assert ~lhs == rhs + else: + with pytest.raises(TypeError): + ~lhs # pylint: disable=W0104 @pytest.mark.parametrize( "lhs, rhs", [ ( - base.And(TestExpressionA(), TestExpressionB(), TestExpressionA()), - base.And(base.And(TestExpressionA(), TestExpressionB()), TestExpressionA()), + base.And(ExpressionA(), ExpressionB(), ExpressionA()), + base.And(base.And(ExpressionA(), ExpressionB()), ExpressionA()), ), ( - base.Or(TestExpressionA(), TestExpressionB(), TestExpressionA()), - base.Or(base.Or(TestExpressionA(), TestExpressionB()), TestExpressionA()), + base.Or(ExpressionA(), ExpressionB(), ExpressionA()), + base.Or(base.Or(ExpressionA(), ExpressionB()), ExpressionA()), ), - (base.Not(base.Not(TestExpressionA())), TestExpressionA()), + (base.Not(base.Not(ExpressionA())), ExpressionA()), ], ) def test_reduce(lhs, rhs): @@ -237,11 +247,11 @@ def test_reduce(lhs, rhs): @pytest.mark.parametrize( "lhs, rhs", [ - (base.And(base.AlwaysTrue(), TestExpressionB()), TestExpressionB()), - (base.And(base.AlwaysFalse(), TestExpressionB()), base.AlwaysFalse()), - (base.Or(base.AlwaysTrue(), TestExpressionB()), base.AlwaysTrue()), - (base.Or(base.AlwaysFalse(), TestExpressionB()), TestExpressionB()), - (base.Not(base.Not(TestExpressionA())), TestExpressionA()), + (base.And(base.AlwaysTrue(), ExpressionB()), ExpressionB()), + (base.And(base.AlwaysFalse(), ExpressionB()), base.AlwaysFalse()), + (base.Or(base.AlwaysTrue(), ExpressionB()), base.AlwaysTrue()), + (base.Or(base.AlwaysFalse(), ExpressionB()), ExpressionB()), + (base.Not(base.Not(ExpressionA())), ExpressionA()), ], ) def test_base_AlwaysTrue_base_AlwaysFalse(lhs, rhs): @@ -323,33 +333,33 @@ def test_bound_reference(table_schema_simple, foo_struct): def test_boolean_expression_visitor(): """Test post-order traversal of boolean expression visit method""" expr = base.And( - base.Or(base.Not(TestExpressionA()), base.Not(TestExpressionB()), TestExpressionA(), TestExpressionB()), - base.Not(TestExpressionA()), - TestExpressionB(), + base.Or(base.Not(ExpressionA()), base.Not(ExpressionB()), ExpressionA(), ExpressionB()), + base.Not(ExpressionA()), + ExpressionB(), ) - visitor = TestBooleanExpressionVisitor() + visitor = BooleanExpressionVisitor() result = base.visit(expr, visitor=visitor) assert result == [ - "TestExpressionA", + "ExpressionA", "NOT", - "TestExpressionB", + "ExpressionB", "NOT", "OR", - "TestExpressionA", + "ExpressionA", "OR", - "TestExpressionB", + "ExpressionB", "OR", - "TestExpressionA", + "ExpressionA", "NOT", "AND", - "TestExpressionB", + "ExpressionB", "AND", ] def test_boolean_expression_visit_raise_not_implemented_error(): """Test raise NotImplementedError when visiting an unsupported object type""" - visitor = TestBooleanExpressionVisitor() + visitor = BooleanExpressionVisitor() with pytest.raises(NotImplementedError) as exc_info: base.visit("foo", visitor=visitor) From e2b40ea9761d103e231ac3f13b9c658543221bef Mon Sep 17 00:00:00 2001 From: Samuel Redai <43911210+samredai@users.noreply.github.com> Date: Mon, 11 Jul 2022 14:26:20 -0400 Subject: [PATCH 131/642] Python: Add Makefile argument to pass args to pytest (#5246) --- CONTRIBUTING.md | 14 ++++++++++++++ Makefile | 2 +- 2 files changed, 15 insertions(+), 1 deletion(-) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index d760efc9f0..3a258189f7 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -71,3 +71,17 @@ For Python, we use pytest in combination with coverage to maintain 90% code cove ```bash make test ``` + +To pass additional arguments to pytest, you can use `PYTEST_ARGS`. + +*Run pytest in verbose mode* +```sh +make test PYTEST_ARGS="-v" +``` + +*Run pytest with pdb enabled* +```sh +make test PYTEST_ARGS="--pdb" +``` + +To see all available pytest arguments, run `make test PYTEST_ARGS="--help"`. diff --git a/Makefile b/Makefile index a73ab02278..66ce52cd8e 100644 --- a/Makefile +++ b/Makefile @@ -23,7 +23,7 @@ lint: poetry run pre-commit run --all-files test: - poetry run coverage run --source=pyiceberg/ -m pytest tests/ + poetry run coverage run --source=pyiceberg/ -m pytest tests/ ${PYTEST_ARGS} poetry run coverage report -m --fail-under=90 poetry run coverage html poetry run coverage xml From 160b91c3aeb91bd0378b0039f8f0bd0e7af9d0dc Mon Sep 17 00:00:00 2001 From: Fokko Driesprong Date: Tue, 12 Jul 2022 01:21:31 +0200 Subject: [PATCH 132/642] Python: Add sort order fields (#5124) --- pyiceberg/schema.py | 6 +- pyiceberg/table/metadata.py | 28 +-- pyiceberg/table/sorting.py | 111 +++++++++++ pyiceberg/transforms.py | 320 +++++++++++++------------------ pyiceberg/types.py | 13 +- pyiceberg/utils/parsing.py | 35 ++++ tests/table/test_metadata.py | 8 +- tests/table/test_partitioning.py | 11 +- tests/table/test_sorting.py | 70 +++++++ tests/test_transforms.py | 210 +++++++------------- 10 files changed, 453 insertions(+), 359 deletions(-) create mode 100644 pyiceberg/table/sorting.py create mode 100644 pyiceberg/utils/parsing.py create mode 100644 tests/table/test_sorting.py diff --git a/pyiceberg/schema.py b/pyiceberg/schema.py index 37372a0d5d..db2aaae35b 100644 --- a/pyiceberg/schema.py +++ b/pyiceberg/schema.py @@ -146,7 +146,11 @@ def find_field(self, name_or_id: Union[str, int], case_sensitive: bool = True) - field_id = self._name_to_id.get(name_or_id) else: field_id = self._lazy_name_to_id_lower.get(name_or_id.lower()) - return self._lazy_id_to_field.get(field_id) # type: ignore + + if not field_id: + raise ValueError(f"Could not find field with name or id {name_or_id}, case_sensitive={case_sensitive}") + + return self._lazy_id_to_field.get(field_id) def find_type(self, name_or_id: Union[str, int], case_sensitive: bool = True) -> IcebergType: """Find a field type using a field name or field ID diff --git a/pyiceberg/table/metadata.py b/pyiceberg/table/metadata.py index fef4110baa..cc6bf33e8e 100644 --- a/pyiceberg/table/metadata.py +++ b/pyiceberg/table/metadata.py @@ -30,12 +30,12 @@ from pyiceberg.exceptions import ValidationError from pyiceberg.schema import Schema from pyiceberg.table.refs import MAIN_BRANCH, SnapshotRef, SnapshotRefType +from pyiceberg.table.sorting import UNSORTED_SORT_ORDER, UNSORTED_SORT_ORDER_ID, SortOrder from pyiceberg.utils.iceberg_base_model import IcebergBaseModel _INITIAL_SEQUENCE_NUMBER = 0 INITIAL_SPEC_ID = 0 DEFAULT_SCHEMA_ID = 0 -DEFAULT_SORT_ORDER_UNSORTED = 0 def check_schemas(values: Dict[str, Any]) -> Dict[str, Any]: @@ -62,14 +62,15 @@ def check_partition_specs(values: Dict[str, Any]) -> Dict[str, Any]: def check_sort_orders(values: Dict[str, Any]) -> Dict[str, Any]: """Validator to check if the default_sort_order_id is present in sort-orders""" - default_sort_order_id = values["default_sort_order_id"] + default_sort_order_id: int = values["default_sort_order_id"] - if default_sort_order_id != DEFAULT_SORT_ORDER_UNSORTED: - for sort in values["sort_orders"]: - if sort["order-id"] == default_sort_order_id: + if default_sort_order_id != UNSORTED_SORT_ORDER_ID: + sort_orders: List[SortOrder] = values["sort_orders"] + for sort_order in sort_orders: + if sort_order.order_id == default_sort_order_id: return values - raise ValidationError(f"default-sort-order-id {default_sort_order_id} can't be found") + raise ValidationError(f"default-sort-order-id {default_sort_order_id} can't be found in {sort_orders}") return values @@ -77,6 +78,9 @@ class TableMetadataCommonFields(IcebergBaseModel): """Metadata for an Iceberg table as specified in the Apache Iceberg spec (https://iceberg.apache.org/spec/#iceberg-table-spec)""" + def current_schema(self) -> Schema: + return next(schema for schema in self.schemas if schema.schema_id == self.current_schema_id) + @root_validator(pre=True) def cleanup_snapshot_id(cls, data: Dict[str, Any]): if data.get("current-snapshot-id") == -1: @@ -159,10 +163,10 @@ def construct_refs(cls, data: Dict[str, Any]): remove oldest metadata log entries and keep a fixed-size log of the most recent entries after a commit.""" - sort_orders: List[Dict[str, Any]] = Field(alias="sort-orders", default_factory=list) + sort_orders: List[SortOrder] = Field(alias="sort-orders", default_factory=list) """A list of sort orders, stored as full sort order objects.""" - default_sort_order_id: int = Field(alias="default-sort-order-id", default=DEFAULT_SORT_ORDER_UNSORTED) + default_sort_order_id: int = Field(alias="default-sort-order-id", default=UNSORTED_SORT_ORDER_ID) """Default sort order id of the table. Note that this could be used by writers, but is not used when reading because reads use the specs stored in manifest files.""" @@ -267,12 +271,10 @@ def set_sort_orders(cls, data: Dict[str, Any]): Returns: The TableMetadata with the sort_orders set, if not provided """ - # This is going to be much nicer as soon as sort-order is an actual pydantic object - # Probably we'll just create a UNSORTED_ORDER constant then - if not data.get("sort_orders"): - data["sort_orders"] = [{"order_id": 0, "fields": []}] + if sort_orders := data.get("sort_orders"): + check_sort_orders(sort_orders) else: - check_sort_orders(data["sort_orders"]) + data["sort_orders"] = [UNSORTED_SORT_ORDER] return data def to_v2(self) -> "TableMetadataV2": diff --git a/pyiceberg/table/sorting.py b/pyiceberg/table/sorting.py new file mode 100644 index 0000000000..e2a72fd24c --- /dev/null +++ b/pyiceberg/table/sorting.py @@ -0,0 +1,111 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# pylint: disable=keyword-arg-before-vararg +from enum import Enum +from typing import ( + Any, + Callable, + Dict, + List, + Optional, + Union, +) + +from pydantic import Field, root_validator + +from pyiceberg.transforms import Transform +from pyiceberg.types import IcebergType +from pyiceberg.utils.iceberg_base_model import IcebergBaseModel + + +class SortDirection(Enum): + ASC = "asc" + DESC = "desc" + + +class NullOrder(Enum): + NULLS_FIRST = "nulls-first" + NULLS_LAST = "nulls-last" + + +class SortField(IcebergBaseModel): + """Sort order field + + Args: + source_id (int): Source column id from the table’s schema + transform (str): Transform that is used to produce values to be sorted on from the source column. + This is the same transform as described in partition transforms. + direction (SortDirection): Sort direction, that can only be either asc or desc + null_order (NullOrder): Null order that describes the order of null values when sorted. Can only be either nulls-first or nulls-last + """ + + def __init__( + self, + source_id: Optional[int] = None, + transform: Optional[Union[Transform, Callable[[IcebergType], Transform]]] = None, + direction: Optional[SortDirection] = None, + null_order: Optional[NullOrder] = None, + **data: Any, + ): + if source_id is not None: + data["source-id"] = source_id + if transform is not None: + data["transform"] = transform + if direction is not None: + data["direction"] = direction + if null_order is not None: + data["null-order"] = null_order + super().__init__(**data) + + @root_validator(pre=True) + def set_null_order(cls, values: Dict[str, Any]) -> Dict[str, Any]: + values["direction"] = values["direction"] if values.get("direction") else SortDirection.ASC + if not values.get("null-order"): + values["null-order"] = NullOrder.NULLS_FIRST if values["direction"] == SortDirection.ASC else NullOrder.NULLS_LAST + return values + + source_id: int = Field(alias="source-id") + transform: Transform = Field() + direction: SortDirection = Field() + null_order: NullOrder = Field(alias="null-order") + + +class SortOrder(IcebergBaseModel): + """Describes how the data is sorted within the table + + Users can sort their data within partitions by columns to gain performance. + + The order of the sort fields within the list defines the order in which the sort is applied to the data. + + Args: + order_id (int): The id of the sort-order. To keep track of historical sorting + fields (List[SortField]): The fields how the table is sorted + """ + + def __init__(self, order_id: Optional[int] = None, *fields: SortField, **data: Any): + if order_id is not None: + data["order-id"] = order_id + if fields: + data["fields"] = fields + super().__init__(**data) + + order_id: Optional[int] = Field(alias="order-id") + fields: List[SortField] = Field(default_factory=list) + + +UNSORTED_SORT_ORDER_ID = 0 +UNSORTED_SORT_ORDER = SortOrder(order_id=UNSORTED_SORT_ORDER_ID) diff --git a/pyiceberg/transforms.py b/pyiceberg/transforms.py index ca337ed584..824afc3419 100644 --- a/pyiceberg/transforms.py +++ b/pyiceberg/transforms.py @@ -18,18 +18,17 @@ import base64 import struct from abc import ABC, abstractmethod -from decimal import Decimal from functools import singledispatch from typing import ( Any, + Callable, Generic, Literal, Optional, TypeVar, ) -from uuid import UUID -import mmh3 # type: ignore +import mmh3 from pydantic import Field, PositiveInt, PrivateAttr from pyiceberg.types import ( @@ -49,11 +48,20 @@ from pyiceberg.utils import datetime from pyiceberg.utils.decimal import decimal_to_bytes, truncate_decimal from pyiceberg.utils.iceberg_base_model import IcebergBaseModel +from pyiceberg.utils.parsing import ParseNumberFromBrackets from pyiceberg.utils.singleton import Singleton S = TypeVar("S") T = TypeVar("T") +IDENTITY = "identity" +VOID = "void" +BUCKET = "bucket" +TRUNCATE = "truncate" + +BUCKET_PARSER = ParseNumberFromBrackets(BUCKET) +TRUNCATE_PARSER = ParseNumberFromBrackets(TRUNCATE) + class Transform(IcebergBaseModel, ABC, Generic[S, T]): """Transform base class for concrete transforms. @@ -64,11 +72,32 @@ class Transform(IcebergBaseModel, ABC, Generic[S, T]): __root__: str = Field() - def __call__(self, value: Optional[S]) -> Optional[T]: - return self.apply(value) + @classmethod + def __get_validators__(cls): + # one or more validators may be yielded which will be called in the + # order to validate the input, each validator will receive as an input + # the value returned from the previous validator + yield cls.validate + + @classmethod + def validate(cls, v: Any): + # When Pydantic is unable to determine the subtype + # In this case we'll help pydantic a bit by parsing the transform type ourselves + if isinstance(v, str): + if v == IDENTITY: + return IdentityTransform() + elif v == VOID: + return VoidTransform() + elif v.startswith(BUCKET): + return BucketTransform(num_buckets=BUCKET_PARSER.match(v)) + elif v.startswith(TRUNCATE): + return TruncateTransform(width=BUCKET_PARSER.match(v)) + else: + return UnknownTransform(transform=v) + return v @abstractmethod - def apply(self, value: Optional[S]) -> Optional[T]: + def transform(self, source: IcebergType) -> Callable[[Optional[S]], Optional[T]]: ... @abstractmethod @@ -86,7 +115,7 @@ def preserves_order(self) -> bool: def satisfies_order_of(self, other) -> bool: return self == other - def to_human_string(self, value: Optional[S]) -> str: + def to_human_string(self, _: IcebergType, value: Optional[S]) -> str: return str(value) if value is not None else "null" @property @@ -96,25 +125,27 @@ def dedup_name(self) -> str: def __str__(self) -> str: return self.__root__ + def __eq__(self, other: Any) -> bool: + if isinstance(other, Transform): + return self.__root__ == other.__root__ + return False -class BaseBucketTransform(Transform[S, int]): + +class BucketTransform(Transform[S, int]): """Base Transform class to transform a value into a bucket partition value Transforms are parameterized by a number of buckets. Bucket partition transforms use a 32-bit hash of the source value to produce a positive value by mod the bucket number. Args: - source_type (Type): An Iceberg Type of IntegerType, LongType, DecimalType, DateType, TimeType, - TimestampType, TimestamptzType, StringType, BinaryType, FixedType, UUIDType. num_buckets (int): The number of buckets. """ _source_type: IcebergType = PrivateAttr() _num_buckets: PositiveInt = PrivateAttr() - def __init__(self, source_type: IcebergType, num_buckets: int, **data: Any): + def __init__(self, num_buckets: int, **data: Any): super().__init__(__root__=f"bucket[{num_buckets}]", **data) - self._source_type = source_type self._num_buckets = num_buckets @property @@ -130,99 +161,58 @@ def apply(self, value: Optional[S]) -> Optional[int]: def result_type(self, source: IcebergType) -> IcebergType: return IntegerType() - @abstractmethod - def can_transform(self, source: IcebergType) -> bool: - pass - - def __repr__(self) -> str: - return f"transforms.bucket(source_type={repr(self._source_type)}, num_buckets={self._num_buckets})" - - -class BucketNumberTransform(BaseBucketTransform): - """Transforms a value of IntegerType, LongType, DateType, TimeType, TimestampType, or TimestamptzType - into a bucket partition value - - Example: - >>> transform = BucketNumberTransform(LongType(), 100) - >>> transform.apply(81068000000) - 59 - """ - def can_transform(self, source: IcebergType) -> bool: - return type(source) in {IntegerType, DateType, LongType, TimeType, TimestampType, TimestamptzType} - - def hash(self, value) -> int: - return mmh3.hash(struct.pack(">> transform = BucketDecimalTransform(DecimalType(9, 2), 100) - >>> transform.apply(Decimal("14.20")) - 59 - """ - - def can_transform(self, source: IcebergType) -> bool: - return isinstance(source, DecimalType) - - def hash(self, value: Decimal) -> int: - return mmh3.hash(decimal_to_bytes(value)) - - -class BucketStringTransform(BaseBucketTransform): - """Transforms a value of StringType into a bucket partition value. - - Example: - >>> transform = BucketStringTransform(StringType(), 100) - >>> transform.apply("iceberg") - 89 - """ - - def can_transform(self, source: IcebergType) -> bool: - return isinstance(source, StringType) - - def hash(self, value: str) -> int: - return mmh3.hash(value) - - -class BucketBytesTransform(BaseBucketTransform): - """Transforms a value of FixedType or BinaryType into a bucket partition value. - - Example: - >>> transform = BucketBytesTransform(BinaryType(), 100) - >>> transform.apply(b"\\x00\\x01\\x02\\x03") - 41 - """ - - def can_transform(self, source: IcebergType) -> bool: - return type(source) in {FixedType, BinaryType} - - def hash(self, value: bytes) -> int: - return mmh3.hash(value) + return type(source) in { + IntegerType, + DateType, + LongType, + TimeType, + TimestampType, + TimestamptzType, + DecimalType, + StringType, + FixedType, + BinaryType, + UUIDType, + } + + def transform(self, source: IcebergType, bucket: bool = True) -> Callable[[Optional[Any]], Optional[int]]: + source_type = type(source) + if source_type in {IntegerType, LongType, DateType, TimeType, TimestampType, TimestamptzType}: + + def hash_func(v): + return mmh3.hash(struct.pack("QQ", + (v.int >> 64) & 0xFFFFFFFFFFFFFFFF, + v.int & 0xFFFFFFFFFFFFFFFF, + ) + ) + else: + raise ValueError(f"Unknown type {source}") -class BucketUUIDTransform(BaseBucketTransform): - """Transforms a value of UUIDType into a bucket partition value. - - Example: - >>> transform = BucketUUIDTransform(UUIDType(), 100) - >>> transform.apply(UUID("f79c3e09-677c-4bbd-a479-3f349cb785e7")) - 40 - """ - - def can_transform(self, source: IcebergType) -> bool: - return isinstance(source, UUIDType) + if bucket: + return lambda v: (hash_func(v) & IntegerType.max) % self._num_buckets if v else None + return hash_func - def hash(self, value: UUID) -> int: - return mmh3.hash( - struct.pack( - ">QQ", - (value.int >> 64) & 0xFFFFFFFFFFFFFFFF, - value.int & 0xFFFFFFFFFFFFFFFF, - ) - ) + def __repr__(self) -> str: + return f"BucketTransform(num_buckets={self._num_buckets})" def _base64encode(buffer: bytes) -> str: @@ -234,20 +224,16 @@ class IdentityTransform(Transform[S, S]): """Transforms a value into itself. Example: - >>> transform = IdentityTransform(StringType()) - >>> transform.apply('hello-world') + >>> transform = IdentityTransform() + >>> transform.transform(StringType())('hello-world') 'hello-world' """ __root__: Literal["identity"] = Field(default="identity") _source_type: IcebergType = PrivateAttr() - def __init__(self, source_type: IcebergType, **data: Any): - super().__init__(**data) - self._source_type = source_type - - def apply(self, value: Optional[S]) -> Optional[S]: - return value + def transform(self, source: IcebergType) -> Callable[[Optional[S]], Optional[S]]: + return lambda v: v def can_transform(self, source: IcebergType) -> bool: return source.is_primitive @@ -263,20 +249,19 @@ def satisfies_order_of(self, other: Transform) -> bool: """ordering by value is the same as long as the other preserves order""" return other.preserves_order - def to_human_string(self, value: Optional[S]) -> str: - return _human_string(value, self._source_type) if value is not None else "null" + def to_human_string(self, source_type: IcebergType, value: Optional[S]) -> str: + return _human_string(value, source_type) if value is not None else "null" def __str__(self) -> str: return "identity" def __repr__(self) -> str: - return f"transforms.identity(source_type={repr(self._source_type)})" + return "IdentityTransform()" class TruncateTransform(Transform[S, S]): """A transform for truncating a value to a specified width. Args: - source_type (Type): An Iceberg Type of IntegerType, LongType, StringType, BinaryType or DecimalType width (int): The truncate width, should be positive Raises: ValueError: If a type is provided that is incompatible with a Truncate transform @@ -286,16 +271,12 @@ class TruncateTransform(Transform[S, S]): _source_type: IcebergType = PrivateAttr() _width: PositiveInt = PrivateAttr() - def __init__(self, source_type: IcebergType, width: int, **data: Any): + def __init__(self, width: int, **data: Any): super().__init__(__root__=f"truncate[{width}]", **data) - self._source_type = source_type self._width = width - def apply(self, value: Optional[S]) -> Optional[S]: - return _truncate_value(value, self._width) if value is not None else None - def can_transform(self, source: IcebergType) -> bool: - return self._source_type == source + return type(source) in {IntegerType, LongType, StringType, BinaryType, DecimalType} def result_type(self, source: IcebergType) -> IcebergType: return source @@ -312,6 +293,28 @@ def source_type(self) -> IcebergType: def width(self) -> int: return self._width + def transform(self, source: IcebergType) -> Callable[[Optional[S]], Optional[S]]: + source_type = type(source) + if source_type in {IntegerType, LongType}: + + def truncate_func(v): + return v - v % self._width + + elif source_type in {StringType, BinaryType}: + + def truncate_func(v): + return v[0 : min(self._width, len(v))] + + elif source_type == DecimalType: + + def truncate_func(v): + return truncate_decimal(v, self._width) + + else: + raise ValueError(f"Cannot truncate for type: {source}") + + return lambda v: truncate_func(v) if v else None + def satisfies_order_of(self, other: Transform) -> bool: if self == other: return True @@ -324,7 +327,7 @@ def satisfies_order_of(self, other: Transform) -> bool: return False - def to_human_string(self, value: Optional[S]) -> str: + def to_human_string(self, _: IcebergType, value: Optional[S]) -> str: if value is None: return "null" elif isinstance(value, bytes): @@ -333,7 +336,7 @@ def to_human_string(self, value: Optional[S]) -> str: return str(value) def __repr__(self) -> str: - return f"transforms.truncate(source_type={repr(self._source_type)}, width={self._width})" + return f"TruncateTransform(width={self._width})" @singledispatch @@ -376,35 +379,6 @@ def _(_type: IcebergType, value: int) -> str: return datetime.to_human_timestamptz(value) -@singledispatch -def _truncate_value(value: Any, _width: int) -> S: - raise ValueError(f"Cannot truncate value: {value}") - - -@_truncate_value.register(int) -def _(value: int, _width: int) -> int: - """Truncate a given int value into a given width if feasible.""" - return value - value % _width - - -@_truncate_value.register(str) -def _(value: str, _width: int) -> str: - """Truncate a given string to a given width.""" - return value[0 : min(_width, len(value))] - - -@_truncate_value.register(bytes) -def _(value: bytes, _width: int) -> bytes: - """Truncate a given binary bytes into a given width.""" - return value[0 : min(_width, len(value))] - - -@_truncate_value.register(Decimal) -def _(value: Decimal, _width: int) -> Decimal: - """Truncate a given decimal value into a given width.""" - return truncate_decimal(value, _width) - - class UnknownTransform(Transform): """A transform that represents when an unknown transform is provided Args: @@ -418,22 +392,21 @@ class UnknownTransform(Transform): _source_type: IcebergType = PrivateAttr() _transform: str = PrivateAttr() - def __init__(self, source_type: IcebergType, transform: str, **data: Any): + def __init__(self, transform: str, **data: Any): super().__init__(**data) - self._source_type = source_type self._transform = transform - def apply(self, value: Optional[S]): + def transform(self, source: IcebergType) -> Callable[[Optional[S]], Optional[T]]: raise AttributeError(f"Cannot apply unsupported transform: {self}") def can_transform(self, source: IcebergType) -> bool: - return self._source_type == source + return False def result_type(self, source: IcebergType) -> IcebergType: return StringType() def __repr__(self) -> str: - return f"transforms.UnknownTransform(source_type={repr(self._source_type)}, transform={repr(self._transform)})" + return f"UnknownTransform(transform={repr(self._transform)})" class VoidTransform(Transform, Singleton): @@ -441,8 +414,8 @@ class VoidTransform(Transform, Singleton): __root__ = "void" - def apply(self, value: Optional[S]) -> None: - return None + def transform(self, source: IcebergType) -> Callable[[Optional[S]], Optional[T]]: + return lambda v: None def can_transform(self, _: IcebergType) -> bool: return True @@ -450,37 +423,8 @@ def can_transform(self, _: IcebergType) -> bool: def result_type(self, source: IcebergType) -> IcebergType: return source - def to_human_string(self, value: Optional[S]) -> str: + def to_human_string(self, _: IcebergType, value: Optional[S]) -> str: return "null" def __repr__(self) -> str: - return "transforms.always_null()" - - -def bucket(source_type: IcebergType, num_buckets: int) -> BaseBucketTransform: - if type(source_type) in {IntegerType, LongType, DateType, TimeType, TimestampType, TimestamptzType}: - return BucketNumberTransform(source_type, num_buckets) - elif isinstance(source_type, DecimalType): - return BucketDecimalTransform(source_type, num_buckets) - elif isinstance(source_type, StringType): - return BucketStringTransform(source_type, num_buckets) - elif isinstance(source_type, BinaryType): - return BucketBytesTransform(source_type, num_buckets) - elif isinstance(source_type, FixedType): - return BucketBytesTransform(source_type, num_buckets) - elif isinstance(source_type, UUIDType): - return BucketUUIDTransform(source_type, num_buckets) - else: - raise ValueError(f"Cannot bucket by type: {source_type}") - - -def identity(source_type: IcebergType) -> IdentityTransform: - return IdentityTransform(source_type) - - -def truncate(source_type: IcebergType, width: int) -> TruncateTransform: - return TruncateTransform(source_type, width) - - -def always_null() -> VoidTransform: - return VoidTransform() + return "VoidTransform()" diff --git a/pyiceberg/types.py b/pyiceberg/types.py index b8068d6b78..94c7345726 100644 --- a/pyiceberg/types.py +++ b/pyiceberg/types.py @@ -31,6 +31,7 @@ """ import re from typing import ( + Any, ClassVar, Dict, Literal, @@ -41,10 +42,12 @@ from pydantic import Field, PrivateAttr from pyiceberg.utils.iceberg_base_model import IcebergBaseModel +from pyiceberg.utils.parsing import ParseNumberFromBrackets from pyiceberg.utils.singleton import Singleton DECIMAL_REGEX = re.compile(r"decimal\((\d+),\s*(\d+)\)") -FIXED_REGEX = re.compile(r"fixed\[(\d+)\]") +FIXED = "fixed" +FIXED_PARSER = ParseNumberFromBrackets(FIXED) class IcebergType(IcebergBaseModel, Singleton): @@ -65,7 +68,7 @@ def __get_validators__(cls): yield cls.validate @classmethod - def validate(cls, v): + def validate(cls, v: Any) -> "IcebergType": # When Pydantic is unable to determine the subtype # In this case we'll help pydantic a bit by parsing the # primitive type ourselves, or pointing it at the correct @@ -123,11 +126,7 @@ class FixedType(PrimitiveType): @staticmethod def parse(str_repr: str) -> "FixedType": - matches = FIXED_REGEX.search(str_repr) - if matches: - length = int(matches.group(1)) - return FixedType(length) - raise ValueError(f"Could not parse {str_repr} into a FixedType") + return FixedType(length=FIXED_PARSER.match(str_repr)) def __init__(self, length: int): super().__init__(__root__=f"fixed[{length}]") diff --git a/pyiceberg/utils/parsing.py b/pyiceberg/utils/parsing.py new file mode 100644 index 0000000000..0566ed6c28 --- /dev/null +++ b/pyiceberg/utils/parsing.py @@ -0,0 +1,35 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +import re +from re import Pattern + + +class ParseNumberFromBrackets: + """Extracts the size from a string in the form of prefix[22]""" + + regex: Pattern + prefix: str + + def __init__(self, prefix: str): + self.prefix = prefix + self.regex = re.compile(rf"{prefix}\[(\d+)\]") + + def match(self, str_repr: str) -> int: + matches = self.regex.search(str_repr) + if matches: + return int(matches.group(1)) + raise ValueError(f"Could not match {str_repr}, expected format {self.prefix}[22]") diff --git a/tests/table/test_metadata.py b/tests/table/test_metadata.py index e7dac41919..fa64b5b199 100644 --- a/tests/table/test_metadata.py +++ b/tests/table/test_metadata.py @@ -147,7 +147,7 @@ def test_v2_metadata_parsing(): assert table_metadata.current_snapshot_id == 3055729675574597004 assert table_metadata.snapshots[0]["snapshot-id"] == 3051729675574597004 assert table_metadata.snapshot_log[0]["timestamp-ms"] == 1515100955770 - assert table_metadata.sort_orders[0]["order-id"] == 3 + assert table_metadata.sort_orders[0].order_id == 3 assert table_metadata.default_sort_order_id == 3 @@ -207,7 +207,7 @@ def test_serialize_v1(): table_metadata = TableMetadataV1(**EXAMPLE_TABLE_METADATA_V1).json() assert ( table_metadata - == """{"location": "s3://bucket/test/location", "table-uuid": "d20125c8-7284-442c-9aea-15fee620737c", "last-updated-ms": 1602638573874, "last-column-id": 3, "schemas": [{"fields": [{"id": 1, "name": "x", "type": "long", "required": true}, {"id": 2, "name": "y", "type": "long", "required": true, "doc": "comment"}, {"id": 3, "name": "z", "type": "long", "required": true}], "schema-id": 0, "identifier-field-ids": []}], "current-schema-id": 0, "partition-specs": [{"spec-id": 0, "fields": [{"name": "x", "transform": "identity", "source-id": 1, "field-id": 1000}]}], "default-spec-id": 0, "last-partition-id": 1000, "properties": {}, "snapshots": [{"snapshot-id": 1925, "timestamp-ms": 1602638573822}], "snapshot-log": [], "metadata-log": [], "sort-orders": [{"order_id": 0, "fields": []}], "default-sort-order-id": 0, "refs": {}, "format-version": 1, "schema": {"fields": [{"id": 1, "name": "x", "type": "long", "required": true}, {"id": 2, "name": "y", "type": "long", "required": true, "doc": "comment"}, {"id": 3, "name": "z", "type": "long", "required": true}], "schema-id": 0, "identifier-field-ids": []}, "partition-spec": [{"name": "x", "transform": "identity", "source-id": 1, "field-id": 1000}]}""" + == """{"location": "s3://bucket/test/location", "table-uuid": "d20125c8-7284-442c-9aea-15fee620737c", "last-updated-ms": 1602638573874, "last-column-id": 3, "schemas": [{"fields": [{"id": 1, "name": "x", "type": "long", "required": true}, {"id": 2, "name": "y", "type": "long", "required": true, "doc": "comment"}, {"id": 3, "name": "z", "type": "long", "required": true}], "schema-id": 0, "identifier-field-ids": []}], "current-schema-id": 0, "partition-specs": [{"spec-id": 0, "fields": [{"name": "x", "transform": "identity", "source-id": 1, "field-id": 1000}]}], "default-spec-id": 0, "last-partition-id": 1000, "properties": {}, "snapshots": [{"snapshot-id": 1925, "timestamp-ms": 1602638573822}], "snapshot-log": [], "metadata-log": [], "sort-orders": [{"order-id": 0, "fields": []}], "default-sort-order-id": 0, "refs": {}, "format-version": 1, "schema": {"fields": [{"id": 1, "name": "x", "type": "long", "required": true}, {"id": 2, "name": "y", "type": "long", "required": true, "doc": "comment"}, {"id": 3, "name": "z", "type": "long", "required": true}], "schema-id": 0, "identifier-field-ids": []}, "partition-spec": [{"name": "x", "transform": "identity", "source-id": 1, "field-id": 1000}]}""" ) @@ -215,7 +215,7 @@ def test_serialize_v2(): table_metadata = TableMetadataV2(**EXAMPLE_TABLE_METADATA_V2).json() assert ( table_metadata - == """{"location": "s3://bucket/test/location", "table-uuid": "9c12d441-03fe-4693-9a96-a0705ddf69c1", "last-updated-ms": 1602638573590, "last-column-id": 3, "schemas": [{"fields": [{"id": 1, "name": "x", "type": "long", "required": true}], "schema-id": 0, "identifier-field-ids": []}, {"fields": [{"id": 1, "name": "x", "type": "long", "required": true}, {"id": 2, "name": "y", "type": "long", "required": true, "doc": "comment"}, {"id": 3, "name": "z", "type": "long", "required": true}], "schema-id": 1, "identifier-field-ids": [1, 2]}], "current-schema-id": 1, "partition-specs": [{"spec-id": 0, "fields": [{"name": "x", "transform": "identity", "source-id": 1, "field-id": 1000}]}], "default-spec-id": 0, "last-partition-id": 1000, "properties": {"read.split.target.size": "134217728"}, "current-snapshot-id": 3055729675574597004, "snapshots": [{"snapshot-id": 3051729675574597004, "timestamp-ms": 1515100955770, "sequence-number": 0, "summary": {"operation": "append"}, "manifest-list": "s3://a/b/1.avro"}, {"snapshot-id": 3055729675574597004, "parent-snapshot-id": 3051729675574597004, "timestamp-ms": 1555100955770, "sequence-number": 1, "summary": {"operation": "append"}, "manifest-list": "s3://a/b/2.avro", "schema-id": 1}], "snapshot-log": [{"snapshot-id": 3051729675574597004, "timestamp-ms": 1515100955770}, {"snapshot-id": 3055729675574597004, "timestamp-ms": 1555100955770}], "metadata-log": [], "sort-orders": [{"order-id": 3, "fields": [{"transform": "identity", "source-id": 2, "direction": "asc", "null-order": "nulls-first"}, {"transform": "bucket[4]", "source-id": 3, "direction": "desc", "null-order": "nulls-last"}]}], "default-sort-order-id": 3, "refs": {"main": {"snapshot-id": 3055729675574597004, "type": "branch"}}, "format-version": 2, "last-sequence-number": 34}""" + == """{"location": "s3://bucket/test/location", "table-uuid": "9c12d441-03fe-4693-9a96-a0705ddf69c1", "last-updated-ms": 1602638573590, "last-column-id": 3, "schemas": [{"fields": [{"id": 1, "name": "x", "type": "long", "required": true}], "schema-id": 0, "identifier-field-ids": []}, {"fields": [{"id": 1, "name": "x", "type": "long", "required": true}, {"id": 2, "name": "y", "type": "long", "required": true, "doc": "comment"}, {"id": 3, "name": "z", "type": "long", "required": true}], "schema-id": 1, "identifier-field-ids": [1, 2]}], "current-schema-id": 1, "partition-specs": [{"spec-id": 0, "fields": [{"name": "x", "transform": "identity", "source-id": 1, "field-id": 1000}]}], "default-spec-id": 0, "last-partition-id": 1000, "properties": {"read.split.target.size": "134217728"}, "current-snapshot-id": 3055729675574597004, "snapshots": [{"snapshot-id": 3051729675574597004, "timestamp-ms": 1515100955770, "sequence-number": 0, "summary": {"operation": "append"}, "manifest-list": "s3://a/b/1.avro"}, {"snapshot-id": 3055729675574597004, "parent-snapshot-id": 3051729675574597004, "timestamp-ms": 1555100955770, "sequence-number": 1, "summary": {"operation": "append"}, "manifest-list": "s3://a/b/2.avro", "schema-id": 1}], "snapshot-log": [{"snapshot-id": 3051729675574597004, "timestamp-ms": 1515100955770}, {"snapshot-id": 3055729675574597004, "timestamp-ms": 1555100955770}], "metadata-log": [], "sort-orders": [{"order-id": 3, "fields": [{"source-id": 2, "transform": "identity", "direction": "asc", "null-order": "nulls-first"}, {"source-id": 3, "transform": "bucket[4]", "direction": "desc", "null-order": "nulls-last"}]}], "default-sort-order-id": 3, "refs": {"main": {"snapshot-id": 3055729675574597004, "type": "branch"}}, "format-version": 2, "last-sequence-number": 34}""" ) @@ -510,7 +510,7 @@ def test_v1_write_metadata_for_v2(): ] assert metadata_v2["default-spec-id"] == 0 assert metadata_v2["last-partition-id"] == 1000 - assert metadata_v2["sort-orders"] == [{"fields": [], "order_id": 0}] + assert metadata_v2["sort-orders"] == [{"order-id": 0, "fields": []}] assert metadata_v2["default-sort-order-id"] == 0 # Deprecated fields assert "schema" not in metadata_v2 diff --git a/tests/table/test_partitioning.py b/tests/table/test_partitioning.py index e6afac293f..12c74625ee 100644 --- a/tests/table/test_partitioning.py +++ b/tests/table/test_partitioning.py @@ -17,12 +17,11 @@ from pyiceberg.schema import Schema from pyiceberg.table.partitioning import PartitionField, PartitionSpec -from pyiceberg.transforms import bucket -from pyiceberg.types import IntegerType +from pyiceberg.transforms import BucketTransform def test_partition_field_init(): - bucket_transform = bucket(IntegerType(), 100) + bucket_transform = BucketTransform(100) partition_field = PartitionField(3, 1000, bucket_transform, "id") assert partition_field.source_id == 3 @@ -33,12 +32,12 @@ def test_partition_field_init(): assert str(partition_field) == "1000: id: bucket[100](3)" assert ( repr(partition_field) - == "PartitionField(source_id=3, field_id=1000, transform=transforms.bucket(source_type=IntegerType(), num_buckets=100), name='id')" + == "PartitionField(source_id=3, field_id=1000, transform=BucketTransform(num_buckets=100), name='id')" ) def test_partition_spec_init(table_schema_simple: Schema): - bucket_transform = bucket(IntegerType(), 4) + bucket_transform: BucketTransform = BucketTransform(4) id_field1 = PartitionField(3, 1001, bucket_transform, "id") partition_spec1 = PartitionSpec(table_schema_simple, 0, (id_field1,), 1001) @@ -57,7 +56,7 @@ def test_partition_spec_init(table_schema_simple: Schema): def test_partition_compatible_with(table_schema_simple: Schema): - bucket_transform = bucket(IntegerType(), 4) + bucket_transform: BucketTransform = BucketTransform(4) field1 = PartitionField(3, 100, bucket_transform, "id") field2 = PartitionField(3, 102, bucket_transform, "id") lhs = PartitionSpec(table_schema_simple, 0, (field1,), 1001) diff --git a/tests/table/test_sorting.py b/tests/table/test_sorting.py new file mode 100644 index 0000000000..1cdf4d0c06 --- /dev/null +++ b/tests/table/test_sorting.py @@ -0,0 +1,70 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +from pyiceberg.table.metadata import TableMetadata +from pyiceberg.table.sorting import ( + UNSORTED_SORT_ORDER, + NullOrder, + SortDirection, + SortField, + SortOrder, +) +from pyiceberg.transforms import BucketTransform, IdentityTransform, VoidTransform +from tests.table.test_metadata import EXAMPLE_TABLE_METADATA_V2 + + +def test_serialize_sort_order_unsorted(): + assert UNSORTED_SORT_ORDER.json() == '{"order-id": 0, "fields": []}' + + +def test_serialize_sort_order(): + sort_order = SortOrder( + 22, + SortField(source_id=19, transform=IdentityTransform(), null_order=NullOrder.NULLS_FIRST), + SortField(source_id=25, transform=BucketTransform(4), direction=SortDirection.DESC), + SortField(source_id=22, transform=VoidTransform(), direction=SortDirection.ASC), + ) + expected = '{"order-id": 22, "fields": [{"source-id": 19, "transform": "identity", "direction": "asc", "null-order": "nulls-first"}, {"source-id": 25, "transform": "bucket[4]", "direction": "desc", "null-order": "nulls-last"}, {"source-id": 22, "transform": "void", "direction": "asc", "null-order": "nulls-first"}]}' + assert sort_order.json() == expected + + +def test_deserialize_sort_order(): + expected = SortOrder( + 22, + SortField(source_id=19, transform=IdentityTransform(), null_order=NullOrder.NULLS_FIRST), + SortField(source_id=25, transform=BucketTransform(4), direction=SortDirection.DESC), + SortField(source_id=22, transform=VoidTransform(), direction=SortDirection.ASC), + ) + payload = '{"order-id": 22, "fields": [{"source-id": 19, "transform": "identity", "direction": "asc", "null-order": "nulls-first"}, {"source-id": 25, "transform": "bucket[4]", "direction": "desc", "null-order": "nulls-last"}, {"source-id": 22, "transform": "void", "direction": "asc", "null-order": "nulls-first"}]}' + + assert SortOrder.parse_raw(payload) == expected + + +def test_sorting_schema(): + table_metadata = TableMetadata.parse_obj(EXAMPLE_TABLE_METADATA_V2) + + assert table_metadata.sort_orders == [ + SortOrder( + 3, + SortField(2, IdentityTransform(), SortDirection.ASC, null_order=NullOrder.NULLS_FIRST), + SortField( + 3, + BucketTransform(4), + direction=SortDirection.DESC, + null_order=NullOrder.NULLS_LAST, + ), + ) + ] diff --git a/tests/test_transforms.py b/tests/test_transforms.py index ca2f441bce..bdebae94e4 100644 --- a/tests/test_transforms.py +++ b/tests/test_transforms.py @@ -14,8 +14,7 @@ # KIND, either express or implied. See the License for the # specific language governing permissions and limitations # under the License. -# pylint: disable=W0123 - +# pylint: disable=eval-used,protected-access from decimal import Decimal from uuid import UUID @@ -24,12 +23,9 @@ from pyiceberg import transforms from pyiceberg.transforms import ( - BucketBytesTransform, - BucketDecimalTransform, - BucketNumberTransform, - BucketStringTransform, - BucketUUIDTransform, + BucketTransform, IdentityTransform, + Transform, TruncateTransform, UnknownTransform, VoidTransform, @@ -56,6 +52,7 @@ timestamp_to_micros, timestamptz_to_micros, ) +from pyiceberg.utils.iceberg_base_model import IcebergBaseModel @pytest.mark.parametrize( @@ -83,30 +80,30 @@ ], ) def test_bucket_hash_values(test_input, test_type, expected): - assert transforms.bucket(test_type, 8).hash(test_input) == expected + assert BucketTransform(num_buckets=8).transform(test_type, bucket=False)(test_input) == expected @pytest.mark.parametrize( - "bucket,value,expected", + "transform,value,expected", [ - (transforms.bucket(IntegerType(), 100), 34, 79), - (transforms.bucket(LongType(), 100), 34, 79), - (transforms.bucket(DateType(), 100), 17486, 26), - (transforms.bucket(TimeType(), 100), 81068000000, 59), - (transforms.bucket(TimestampType(), 100), 1510871468000000, 7), - (transforms.bucket(DecimalType(9, 2), 100), Decimal("14.20"), 59), - (transforms.bucket(StringType(), 100), "iceberg", 89), + (BucketTransform(100).transform(IntegerType()), 34, 79), + (BucketTransform(100).transform(LongType()), 34, 79), + (BucketTransform(100).transform(DateType()), 17486, 26), + (BucketTransform(100).transform(TimeType()), 81068000000, 59), + (BucketTransform(100).transform(TimestampType()), 1510871468000000, 7), + (BucketTransform(100).transform(DecimalType(9, 2)), Decimal("14.20"), 59), + (BucketTransform(100).transform(StringType()), "iceberg", 89), ( - transforms.bucket(UUIDType(), 100), + BucketTransform(100).transform(UUIDType()), UUID("f79c3e09-677c-4bbd-a479-3f349cb785e7"), 40, ), - (transforms.bucket(FixedType(3), 128), b"foo", 32), - (transforms.bucket(BinaryType(), 128), b"\x00\x01\x02\x03", 57), + (BucketTransform(128).transform(FixedType(3)), b"foo", 32), + (BucketTransform(128).transform(BinaryType()), b"\x00\x01\x02\x03", 57), ], ) -def test_buckets(bucket, value, expected): - assert bucket.apply(value) == expected +def test_buckets(transform, value, expected): + assert transform(value) == expected @pytest.mark.parametrize( @@ -126,20 +123,20 @@ def test_buckets(bucket, value, expected): ], ) def test_bucket_method(type_var): - bucket_transform = transforms.bucket(type_var, 8) + bucket_transform = BucketTransform(8) assert str(bucket_transform) == str(eval(repr(bucket_transform))) assert bucket_transform.can_transform(type_var) assert bucket_transform.result_type(type_var) == IntegerType() assert bucket_transform.num_buckets == 8 assert bucket_transform.apply(None) is None - assert bucket_transform.to_human_string("test") == "test" + assert bucket_transform.to_human_string(type_var, "test") == "test" def test_string_with_surrogate_pair(): string_with_surrogate_pair = "string with a surrogate pair: 💰" as_bytes = bytes(string_with_surrogate_pair, "UTF-8") - bucket_transform = transforms.bucket(StringType(), 100) - assert bucket_transform.hash(string_with_surrogate_pair) == mmh3.hash(as_bytes) + bucket_transform = BucketTransform(100).transform(StringType(), bucket=False) + assert bucket_transform(string_with_surrogate_pair) == mmh3.hash(as_bytes) @pytest.mark.parametrize( @@ -157,8 +154,8 @@ def test_string_with_surrogate_pair(): ], ) def test_identity_human_string(type_var, value, expected): - identity = transforms.identity(type_var) - assert identity.to_human_string(value) == expected + identity = IdentityTransform() + assert identity.to_human_string(type_var, value) == expected @pytest.mark.parametrize( @@ -181,11 +178,11 @@ def test_identity_human_string(type_var, value, expected): ], ) def test_identity_method(type_var): - identity_transform = transforms.identity(type_var) + identity_transform = IdentityTransform() assert str(identity_transform) == str(eval(repr(identity_transform))) assert identity_transform.can_transform(type_var) assert identity_transform.result_type(type_var) == type_var - assert identity_transform.apply("test") == "test" + assert identity_transform.transform(type_var)("test") == "test" @pytest.mark.parametrize("type_var", [IntegerType(), LongType()]) @@ -194,8 +191,8 @@ def test_identity_method(type_var): [(1, 0), (5, 0), (9, 0), (10, 10), (11, 10), (-1, -10), (-10, -10), (-12, -20)], ) def test_truncate_integer(type_var, input_var, expected): - trunc = transforms.truncate(type_var, 10) - assert trunc.apply(input_var) == expected + trunc = TruncateTransform(10) + assert trunc.transform(type_var)(input_var) == expected @pytest.mark.parametrize( @@ -209,14 +206,14 @@ def test_truncate_integer(type_var, input_var, expected): ], ) def test_truncate_decimal(input_var, expected): - trunc = transforms.truncate(DecimalType(9, 2), 10) - assert trunc.apply(input_var) == expected + trunc = TruncateTransform(10) + assert trunc.transform(DecimalType(9, 2))(input_var) == expected @pytest.mark.parametrize("input_var,expected", [("abcdefg", "abcde"), ("abc", "abc")]) def test_truncate_string(input_var, expected): - trunc = transforms.truncate(StringType(), 5) - assert trunc.apply(input_var) == expected + trunc = TruncateTransform(5) + assert trunc.transform(StringType())(input_var) == expected @pytest.mark.parametrize( @@ -232,159 +229,92 @@ def test_truncate_string(input_var, expected): ], ) def test_truncate_method(type_var, value, expected_human_str, expected): - truncate_transform = transforms.truncate(type_var, 1) + truncate_transform = TruncateTransform(1) assert str(truncate_transform) == str(eval(repr(truncate_transform))) assert truncate_transform.can_transform(type_var) assert truncate_transform.result_type(type_var) == type_var - assert truncate_transform.to_human_string(value) == expected_human_str - assert truncate_transform.apply(value) == expected - assert truncate_transform.to_human_string(None) == "null" + assert truncate_transform.to_human_string(type_var, value) == expected_human_str + assert truncate_transform.transform(type_var)(value) == expected + assert truncate_transform.to_human_string(type_var, None) == "null" assert truncate_transform.width == 1 - assert truncate_transform.apply(None) is None + assert truncate_transform.transform(type_var)(None) is None assert truncate_transform.preserves_order assert truncate_transform.satisfies_order_of(truncate_transform) def test_unknown_transform(): - unknown_transform = transforms.UnknownTransform(FixedType(8), "unknown") + unknown_transform = transforms.UnknownTransform("unknown") assert str(unknown_transform) == str(eval(repr(unknown_transform))) with pytest.raises(AttributeError): - unknown_transform.apply("test") - assert unknown_transform.can_transform(FixedType(8)) + unknown_transform.transform(StringType())("test") assert not unknown_transform.can_transform(FixedType(5)) assert isinstance(unknown_transform.result_type(BooleanType()), StringType) def test_void_transform(): - void_transform = transforms.always_null() - assert void_transform is transforms.always_null() + void_transform = VoidTransform() + assert void_transform is VoidTransform() assert void_transform == eval(repr(void_transform)) - assert void_transform.apply("test") is None + assert void_transform.transform(StringType())("test") is None assert void_transform.can_transform(BooleanType()) assert isinstance(void_transform.result_type(BooleanType()), BooleanType) assert not void_transform.preserves_order - assert void_transform.satisfies_order_of(transforms.always_null()) - assert not void_transform.satisfies_order_of(transforms.bucket(DateType(), 100)) - assert void_transform.to_human_string("test") == "null" + assert void_transform.satisfies_order_of(VoidTransform()) + assert not void_transform.satisfies_order_of(BucketTransform(100)) + assert void_transform.to_human_string(StringType(), "test") == "null" assert void_transform.dedup_name == "void" -def test_bucket_number_transform_json(): - assert BucketNumberTransform(source_type=IntegerType(), num_buckets=22).json() == '"bucket[22]"' - - -def test_bucket_number_transform_str(): - assert str(BucketNumberTransform(source_type=IntegerType(), num_buckets=22)) == "bucket[22]" - - -def test_bucket_number_transform_repr(): - assert ( - repr(BucketNumberTransform(source_type=IntegerType(), num_buckets=22)) - == "transforms.bucket(source_type=IntegerType(), num_buckets=22)" - ) - - -def test_bucket_decimal_transform_json(): - assert BucketDecimalTransform(source_type=DecimalType(19, 25), num_buckets=22).json() == '"bucket[22]"' - - -def test_bucket_decimal_transform_str(): - assert str(BucketDecimalTransform(source_type=DecimalType(19, 25), num_buckets=22)) == "bucket[22]" - - -def test_bucket_decimal_transform_repr(): - assert ( - repr(BucketDecimalTransform(source_type=DecimalType(19, 25), num_buckets=22)) - == "transforms.bucket(source_type=DecimalType(precision=19, scale=25), num_buckets=22)" - ) - - -def test_bucket_string_transform_json(): - assert BucketStringTransform(StringType(), num_buckets=22).json() == '"bucket[22]"' - - -def test_bucket_string_transform_str(): - assert str(BucketStringTransform(StringType(), num_buckets=22)) == "bucket[22]" - - -def test_bucket_string_transform_repr(): - assert ( - repr(BucketStringTransform(StringType(), num_buckets=22)) == "transforms.bucket(source_type=StringType(), num_buckets=22)" - ) +class TestType(IcebergBaseModel): + __root__: Transform -def test_bucket_bytes_transform_json(): - assert BucketBytesTransform(BinaryType(), num_buckets=22).json() == '"bucket[22]"' +def test_bucket_transform_serialize(): + assert BucketTransform(num_buckets=22).json() == '"bucket[22]"' -def test_bucket_bytes_transform_str(): - assert str(BucketBytesTransform(BinaryType(), num_buckets=22)) == "bucket[22]" +def test_bucket_transform_deserialize(): + transform = TestType.parse_raw('"bucket[22]"').__root__ + assert transform == BucketTransform(num_buckets=22) -def test_bucket_bytes_transform_repr(): - assert ( - repr(BucketBytesTransform(BinaryType(), num_buckets=22)) == "transforms.bucket(source_type=BinaryType(), num_buckets=22)" - ) +def test_bucket_transform_str(): + assert str(BucketTransform(num_buckets=22)) == "bucket[22]" -def test_bucket_uuid_transform_json(): - assert BucketUUIDTransform(UUIDType(), num_buckets=22).json() == '"bucket[22]"' +def test_bucket_transform_repr(): + assert repr(BucketTransform(num_buckets=22)) == "BucketTransform(num_buckets=22)" -def test_bucket_uuid_transform_str(): - assert str(BucketUUIDTransform(UUIDType(), num_buckets=22)) == "bucket[22]" +def test_truncate_transform_serialize(): + assert UnknownTransform("unknown").json() == '"unknown"' -def test_bucket_uuid_transform_repr(): - assert repr(BucketUUIDTransform(UUIDType(), num_buckets=22)) == "transforms.bucket(source_type=UUIDType(), num_buckets=22)" - - -def test_identity_transform_json(): - assert IdentityTransform(StringType()).json() == '"identity"' - - -def test_identity_transform_str(): - assert str(IdentityTransform(StringType())) == "identity" - - -def test_identity_transform_repr(): - assert repr(IdentityTransform(StringType())) == "transforms.identity(source_type=StringType())" - - -def test_truncate_transform_json(): - assert TruncateTransform(StringType(), 22).json() == '"truncate[22]"' - - -def test_truncate_transform_str(): - assert str(TruncateTransform(StringType(), 22)) == "truncate[22]" - - -def test_truncate_transform_repr(): - assert repr(TruncateTransform(StringType(), 22)) == "transforms.truncate(source_type=StringType(), width=22)" - - -def test_unknown_transform_json(): - assert UnknownTransform(StringType(), "unknown").json() == '"unknown"' +def test_unknown_transform_deserialize(): + transform = TestType.parse_raw('"unknown"').__root__ + assert transform == UnknownTransform("unknown") def test_unknown_transform_str(): - assert str(UnknownTransform(StringType(), "unknown")) == "unknown" + assert str(UnknownTransform("unknown")) == "unknown" def test_unknown_transform_repr(): - assert ( - repr(UnknownTransform(StringType(), "unknown")) - == "transforms.UnknownTransform(source_type=StringType(), transform='unknown')" - ) + assert repr(UnknownTransform("unknown")) == "UnknownTransform(transform='unknown')" -def test_void_transform_json(): +def test_void_transform_serialize(): assert VoidTransform().json() == '"void"' +def test_void_transform_deserialize(): + transform = TestType.parse_raw('"void"').__root__ + assert transform == VoidTransform() + + def test_void_transform_str(): assert str(VoidTransform()) == "void" def test_void_transform_repr(): - assert repr(VoidTransform()) == "transforms.always_null()" + assert repr(VoidTransform()) == "VoidTransform()" From d0ec2a7559df6bef42453c6f83d91d25f49c950d Mon Sep 17 00:00:00 2001 From: Samuel Redai <43911210+samredai@users.noreply.github.com> Date: Mon, 11 Jul 2022 19:28:27 -0400 Subject: [PATCH 133/642] Python: Add BindVisitor to bind a BooleanExpression to a Schema (#5239) --- pyiceberg/expressions/base.py | 51 +++- tests/expressions/test_expressions_base.py | 274 ++++++++++++++++++++- tests/test_types.py | 34 +-- 3 files changed, 333 insertions(+), 26 deletions(-) diff --git a/pyiceberg/expressions/base.py b/pyiceberg/expressions/base.py index b373049490..876965ae71 100644 --- a/pyiceberg/expressions/base.py +++ b/pyiceberg/expressions/base.py @@ -17,12 +17,7 @@ from abc import ABC, abstractmethod from dataclasses import dataclass from functools import reduce, singledispatch -from typing import ( - Any, - Generic, - Tuple, - TypeVar, -) +from typing import Generic, Tuple, TypeVar from pyiceberg.files import StructProtocol from pyiceberg.schema import Accessor, Schema @@ -125,7 +120,7 @@ class BoundReference(BoundTerm[T], BaseReference[T]): field: NestedField accessor: Accessor - def eval(self, struct: StructProtocol) -> Any: + def eval(self, struct: StructProtocol) -> T: """Returns the value at the referenced field's position in an object that abides by the StructProtocol Args: struct (StructProtocol): A row object that abides by the StructProtocol and returns values given a position @@ -327,7 +322,7 @@ class In(UnboundPredicate[T]): def __invert__(self): raise TypeError("In expressions do not support negation.") - def bind(self, schema: Schema, case_sensitive: bool) -> BoundIn: + def bind(self, schema: Schema, case_sensitive: bool) -> BoundIn[T]: bound_ref = self.term.bind(schema, case_sensitive) return BoundIn(bound_ref, tuple(lit.to(bound_ref.field.field_type) for lit in self.literals)) # type: ignore @@ -433,9 +428,49 @@ def _(obj: And, visitor: BooleanExpressionVisitor[T]) -> T: return visitor.visit_and(left_result=left_result, right_result=right_result) +@visit.register(In) +def _(obj: In, visitor: BooleanExpressionVisitor[T]) -> T: + """Visit an In boolean expression with a concrete BooleanExpressionVisitor""" + return visitor.visit_unbound_predicate(predicate=obj) + + @visit.register(Or) def _(obj: Or, visitor: BooleanExpressionVisitor[T]) -> T: """Visit an Or boolean expression with a concrete BooleanExpressionVisitor""" left_result: T = visit(obj.left, visitor=visitor) right_result: T = visit(obj.right, visitor=visitor) return visitor.visit_or(left_result=left_result, right_result=right_result) + + +class BindVisitor(BooleanExpressionVisitor[BooleanExpression]): + """Rewrites a boolean expression by replacing unbound references with references to fields in a struct schema + + Args: + schema (Schema): A schema to use when binding the expression + case_sensitive (bool): Whether to consider case when binding a reference to a field in a schema, defaults to True + """ + + def __init__(self, schema: Schema, case_sensitive: bool = True) -> None: + self._schema = schema + self._case_sensitive = case_sensitive + + def visit_true(self) -> BooleanExpression: + return AlwaysTrue() + + def visit_false(self) -> BooleanExpression: + return AlwaysFalse() + + def visit_not(self, child_result: BooleanExpression) -> BooleanExpression: + return Not(child=child_result) + + def visit_and(self, left_result: BooleanExpression, right_result: BooleanExpression) -> BooleanExpression: + return And(left=left_result, right=right_result) + + def visit_or(self, left_result: BooleanExpression, right_result: BooleanExpression) -> BooleanExpression: + return Or(left=left_result, right=right_result) + + def visit_unbound_predicate(self, predicate) -> BooleanExpression: + return predicate.bind(self._schema, case_sensitive=self._case_sensitive) + + def visit_bound_predicate(self, predicate) -> BooleanExpression: + raise TypeError(f"Found already bound predicate: {predicate}") diff --git a/tests/expressions/test_expressions_base.py b/tests/expressions/test_expressions_base.py index ba5b04b741..51630d71e8 100644 --- a/tests/expressions/test_expressions_base.py +++ b/tests/expressions/test_expressions_base.py @@ -22,8 +22,9 @@ import pytest from pyiceberg.expressions import base -from pyiceberg.expressions.literals import literal -from pyiceberg.types import NestedField, StringType +from pyiceberg.expressions.literals import LongLiteral, StringLiteral, literal +from pyiceberg.schema import Accessor +from pyiceberg.types import IntegerType, NestedField, StringType from pyiceberg.utils.singleton import Singleton @@ -364,3 +365,272 @@ def test_boolean_expression_visit_raise_not_implemented_error(): base.visit("foo", visitor=visitor) assert str(exc_info.value) == "Cannot visit unsupported expression: foo" + + +def test_always_true_expression_binding(table_schema_simple): + """Test that visiting an always-true expression returns always-true""" + unbound_expression = base.AlwaysTrue() + bound_expression = base.visit(unbound_expression, visitor=base.BindVisitor(schema=table_schema_simple)) + assert bound_expression == base.AlwaysTrue() + + +def test_always_false_expression_binding(table_schema_simple): + """Test that visiting an always-false expression returns always-false""" + unbound_expression = base.AlwaysFalse() + bound_expression = base.visit(unbound_expression, visitor=base.BindVisitor(schema=table_schema_simple)) + assert bound_expression == base.AlwaysFalse() + + +def test_always_false_and_always_true_expression_binding(table_schema_simple): + """Test that visiting both an always-true AND always-false expression returns always-false""" + unbound_expression = base.And(base.AlwaysTrue(), base.AlwaysFalse()) + bound_expression = base.visit(unbound_expression, visitor=base.BindVisitor(schema=table_schema_simple)) + assert bound_expression == base.AlwaysFalse() + + +def test_always_false_or_always_true_expression_binding(table_schema_simple): + """Test that visiting always-true OR always-false expression returns always-true""" + unbound_expression = base.Or(base.AlwaysTrue(), base.AlwaysFalse()) + bound_expression = base.visit(unbound_expression, visitor=base.BindVisitor(schema=table_schema_simple)) + assert bound_expression == base.AlwaysTrue() + + +@pytest.mark.parametrize( + "unbound_and_expression,expected_bound_expression", + [ + ( + base.And( + base.In(base.Reference("foo"), (literal("foo"), literal("bar"))), + base.In(base.Reference("bar"), (literal(1), literal(2), literal(3))), + ), + base.And( + base.BoundIn[str]( + term=base.BoundReference( + field=NestedField(field_id=1, name="foo", field_type=StringType(), required=False), + accessor=Accessor(position=0, inner=None), + ), + literals=(StringLiteral("foo"), StringLiteral("bar")), + ), + base.BoundIn[int]( + term=base.BoundReference( + field=NestedField(field_id=2, name="bar", field_type=IntegerType(), required=True), + accessor=Accessor(position=1, inner=None), + ), + literals=(LongLiteral(1), LongLiteral(2), LongLiteral(3)), + ), + ), + ), + ( + base.And( + base.In(base.Reference("foo"), (literal("bar"), literal("baz"))), + base.In(base.Reference("bar"), (literal(1),)), + base.In(base.Reference("foo"), (literal("baz"),)), + ), + base.And( + base.And( + base.BoundIn[str]( + term=base.BoundReference( + field=NestedField(field_id=1, name="foo", field_type=StringType(), required=False), + accessor=Accessor(position=0, inner=None), + ), + literals=(StringLiteral("bar"), StringLiteral("baz")), + ), + base.BoundIn[int]( + term=base.BoundReference( + field=NestedField(field_id=2, name="bar", field_type=IntegerType(), required=True), + accessor=Accessor(position=1, inner=None), + ), + literals=(LongLiteral(1),), + ), + ), + base.BoundIn[str]( + term=base.BoundReference( + field=NestedField(field_id=1, name="foo", field_type=StringType(), required=False), + accessor=Accessor(position=0, inner=None), + ), + literals=(StringLiteral("baz"),), + ), + ), + ), + ], +) +def test_and_expression_binding(unbound_and_expression, expected_bound_expression, table_schema_simple): + """Test that visiting an unbound AND expression with a bind-visitor returns the expected bound expression""" + bound_expression = base.visit(unbound_and_expression, visitor=base.BindVisitor(schema=table_schema_simple)) + assert bound_expression == expected_bound_expression + + +@pytest.mark.parametrize( + "unbound_or_expression,expected_bound_expression", + [ + ( + base.Or( + base.In(base.Reference("foo"), (literal("foo"), literal("bar"))), + base.In(base.Reference("bar"), (literal(1), literal(2), literal(3))), + ), + base.Or( + base.BoundIn[str]( + term=base.BoundReference( + field=NestedField(field_id=1, name="foo", field_type=StringType(), required=False), + accessor=Accessor(position=0, inner=None), + ), + literals=(StringLiteral("foo"), StringLiteral("bar")), + ), + base.BoundIn[int]( + term=base.BoundReference( + field=NestedField(field_id=2, name="bar", field_type=IntegerType(), required=True), + accessor=Accessor(position=1, inner=None), + ), + literals=(LongLiteral(1), LongLiteral(2), LongLiteral(3)), + ), + ), + ), + ( + base.Or( + base.In(base.Reference("foo"), (literal("bar"), literal("baz"))), + base.In(base.Reference("foo"), (literal("bar"),)), + base.In(base.Reference("foo"), (literal("baz"),)), + ), + base.Or( + base.Or( + base.BoundIn[str]( + term=base.BoundReference( + field=NestedField(field_id=1, name="foo", field_type=StringType(), required=False), + accessor=Accessor(position=0, inner=None), + ), + literals=(StringLiteral("bar"), StringLiteral("baz")), + ), + base.BoundIn[str]( + term=base.BoundReference( + field=NestedField(field_id=1, name="foo", field_type=StringType(), required=False), + accessor=Accessor(position=0, inner=None), + ), + literals=(StringLiteral("bar"),), + ), + ), + base.BoundIn[str]( + term=base.BoundReference( + field=NestedField(field_id=1, name="foo", field_type=StringType(), required=False), + accessor=Accessor(position=0, inner=None), + ), + literals=(StringLiteral("baz"),), + ), + ), + ), + ( + base.Or( + base.AlwaysTrue(), + base.AlwaysFalse(), + ), + base.AlwaysTrue(), + ), + ( + base.Or( + base.AlwaysTrue(), + base.AlwaysTrue(), + ), + base.AlwaysTrue(), + ), + ( + base.Or( + base.AlwaysFalse(), + base.AlwaysFalse(), + ), + base.AlwaysFalse(), + ), + ], +) +def test_or_expression_binding(unbound_or_expression, expected_bound_expression, table_schema_simple): + """Test that visiting an unbound OR expression with a bind-visitor returns the expected bound expression""" + bound_expression = base.visit(unbound_or_expression, visitor=base.BindVisitor(schema=table_schema_simple)) + assert bound_expression == expected_bound_expression + + +@pytest.mark.parametrize( + "unbound_in_expression,expected_bound_expression", + [ + ( + base.In(base.Reference("foo"), (literal("foo"), literal("bar"))), + base.BoundIn[str]( + term=base.BoundReference[str]( + field=NestedField(field_id=1, name="foo", field_type=StringType(), required=False), + accessor=Accessor(position=0, inner=None), + ), + literals=(StringLiteral("foo"), StringLiteral("bar")), + ), + ), + ( + base.In(base.Reference("foo"), (literal("bar"), literal("baz"))), + base.BoundIn[str]( + term=base.BoundReference[str]( + field=NestedField(field_id=1, name="foo", field_type=StringType(), required=False), + accessor=Accessor(position=0, inner=None), + ), + literals=(StringLiteral("bar"), StringLiteral("baz")), + ), + ), + ( + base.In(base.Reference("foo"), (literal("bar"),)), + base.BoundIn[str]( + term=base.BoundReference[str]( + field=NestedField(field_id=1, name="foo", field_type=StringType(), required=False), + accessor=Accessor(position=0, inner=None), + ), + literals=(StringLiteral("bar"),), + ), + ), + ], +) +def test_in_expression_binding(unbound_in_expression, expected_bound_expression, table_schema_simple): + """Test that visiting an unbound IN expression with a bind-visitor returns the expected bound expression""" + bound_expression = base.visit(unbound_in_expression, visitor=base.BindVisitor(schema=table_schema_simple)) + assert bound_expression == expected_bound_expression + + +@pytest.mark.parametrize( + "unbound_not_expression,expected_bound_expression", + [ + ( + base.Not(base.In(base.Reference("foo"), (literal("foo"), literal("bar")))), + base.Not( + base.BoundIn[str]( + term=base.BoundReference[str]( + field=NestedField(field_id=1, name="foo", field_type=StringType(), required=False), + accessor=Accessor(position=0, inner=None), + ), + literals=(StringLiteral("foo"), StringLiteral("bar")), + ) + ), + ), + ( + base.Not( + base.Or( + base.In(base.Reference("foo"), (literal("foo"), literal("bar"))), + base.In(base.Reference("foo"), (literal("foo"), literal("bar"), literal("baz"))), + ) + ), + base.Not( + base.Or( + base.BoundIn[str]( + term=base.BoundReference( + field=NestedField(field_id=1, name="foo", field_type=StringType(), required=False), + accessor=Accessor(position=0, inner=None), + ), + literals=(StringLiteral("foo"), StringLiteral("bar")), + ), + base.BoundIn[str]( + term=base.BoundReference( + field=NestedField(field_id=1, name="foo", field_type=StringType(), required=False), + accessor=Accessor(position=0, inner=None), + ), + literals=(StringLiteral("foo"), StringLiteral("bar"), StringLiteral("baz")), + ), + ), + ), + ), + ], +) +def test_not_expression_binding(unbound_not_expression, expected_bound_expression, table_schema_simple): + """Test that visiting an unbound NOT expression with a bind-visitor returns the expected bound expression""" + bound_expression = base.visit(unbound_not_expression, visitor=base.BindVisitor(schema=table_schema_simple)) + assert bound_expression == expected_bound_expression diff --git a/tests/test_types.py b/tests/test_types.py index 65f047acce..56ed291fad 100644 --- a/tests/test_types.py +++ b/tests/test_types.py @@ -211,7 +211,7 @@ def test_non_parameterized_type_equality(input_index, input_type, check_index, c # Examples based on https://iceberg.apache.org/spec/#appendix-c-json-serialization -class TestType(IcebergBaseModel): +class IcebergTestType(IcebergBaseModel): __root__: IcebergType @@ -220,7 +220,7 @@ def test_serialization_boolean(): def test_deserialization_boolean(): - assert TestType.parse_raw('"boolean"') == BooleanType() + assert IcebergTestType.parse_raw('"boolean"') == BooleanType() def test_str_boolean(): @@ -236,7 +236,7 @@ def test_serialization_int(): def test_deserialization_int(): - assert TestType.parse_raw('"int"') == IntegerType() + assert IcebergTestType.parse_raw('"int"') == IntegerType() def test_str_int(): @@ -252,7 +252,7 @@ def test_serialization_long(): def test_deserialization_long(): - assert TestType.parse_raw('"long"') == LongType() + assert IcebergTestType.parse_raw('"long"') == LongType() def test_str_long(): @@ -268,7 +268,7 @@ def test_serialization_float(): def test_deserialization_float(): - assert TestType.parse_raw('"float"') == FloatType() + assert IcebergTestType.parse_raw('"float"') == FloatType() def test_str_float(): @@ -284,7 +284,7 @@ def test_serialization_double(): def test_deserialization_double(): - assert TestType.parse_raw('"double"') == DoubleType() + assert IcebergTestType.parse_raw('"double"') == DoubleType() def test_str_double(): @@ -300,7 +300,7 @@ def test_serialization_date(): def test_deserialization_date(): - assert TestType.parse_raw('"date"') == DateType() + assert IcebergTestType.parse_raw('"date"') == DateType() def test_str_date(): @@ -316,7 +316,7 @@ def test_serialization_time(): def test_deserialization_time(): - assert TestType.parse_raw('"time"') == TimeType() + assert IcebergTestType.parse_raw('"time"') == TimeType() def test_str_time(): @@ -332,7 +332,7 @@ def test_serialization_timestamp(): def test_deserialization_timestamp(): - assert TestType.parse_raw('"timestamp"') == TimestampType() + assert IcebergTestType.parse_raw('"timestamp"') == TimestampType() def test_str_timestamp(): @@ -348,7 +348,7 @@ def test_serialization_timestamptz(): def test_deserialization_timestamptz(): - assert TestType.parse_raw('"timestamptz"') == TimestamptzType() + assert IcebergTestType.parse_raw('"timestamptz"') == TimestamptzType() def test_str_timestamptz(): @@ -364,7 +364,7 @@ def test_serialization_string(): def test_deserialization_string(): - assert TestType.parse_raw('"string"') == StringType() + assert IcebergTestType.parse_raw('"string"') == StringType() def test_str_string(): @@ -380,7 +380,7 @@ def test_serialization_uuid(): def test_deserialization_uuid(): - assert TestType.parse_raw('"uuid"') == UUIDType() + assert IcebergTestType.parse_raw('"uuid"') == UUIDType() def test_str_uuid(): @@ -396,7 +396,7 @@ def test_serialization_fixed(): def test_deserialization_fixed(): - fixed = TestType.parse_raw('"fixed[22]"') + fixed = IcebergTestType.parse_raw('"fixed[22]"') assert fixed == FixedType(22) inner = fixed.__root__ @@ -417,7 +417,7 @@ def test_serialization_binary(): def test_deserialization_binary(): - assert TestType.parse_raw('"binary"') == BinaryType() + assert IcebergTestType.parse_raw('"binary"') == BinaryType() def test_str_binary(): @@ -433,7 +433,7 @@ def test_serialization_decimal(): def test_deserialization_decimal(): - decimal = TestType.parse_raw('"decimal(19, 25)"') + decimal = IcebergTestType.parse_raw('"decimal(19, 25)"') assert decimal == DecimalType(19, 25) inner = decimal.__root__ @@ -490,7 +490,9 @@ def test_deserialization_nestedfield(): def test_deserialization_nestedfield_inner(): expected = NestedField(1, "required_field", StringType(), True, "this is a doc") - actual = TestType.parse_raw('{"id": 1, "name": "required_field", "type": "string", "required": true, "doc": "this is a doc"}') + actual = IcebergTestType.parse_raw( + '{"id": 1, "name": "required_field", "type": "string", "required": true, "doc": "this is a doc"}' + ) assert expected == actual.__root__ From 7951011db1d909a96f73e405fdb642052e10d7d2 Mon Sep 17 00:00:00 2001 From: Fokko Driesprong Date: Tue, 12 Jul 2022 19:54:22 +0200 Subject: [PATCH 134/642] Python: Move Snapshot to Pydantic (#5201) --- pyiceberg/table/metadata.py | 7 ++- pyiceberg/table/snapshots.py | 90 +++++++++++++++++++++++++++ tests/table/test_metadata.py | 14 ++--- tests/table/test_snapshots.py | 113 ++++++++++++++++++++++++++++++++++ 4 files changed, 212 insertions(+), 12 deletions(-) create mode 100644 pyiceberg/table/snapshots.py create mode 100644 tests/table/test_snapshots.py diff --git a/pyiceberg/table/metadata.py b/pyiceberg/table/metadata.py index cc6bf33e8e..9c506e515c 100644 --- a/pyiceberg/table/metadata.py +++ b/pyiceberg/table/metadata.py @@ -30,10 +30,11 @@ from pyiceberg.exceptions import ValidationError from pyiceberg.schema import Schema from pyiceberg.table.refs import MAIN_BRANCH, SnapshotRef, SnapshotRefType +from pyiceberg.table.snapshots import Snapshot from pyiceberg.table.sorting import UNSORTED_SORT_ORDER, UNSORTED_SORT_ORDER_ID, SortOrder from pyiceberg.utils.iceberg_base_model import IcebergBaseModel -_INITIAL_SEQUENCE_NUMBER = 0 +INITIAL_SEQUENCE_NUMBER = 0 INITIAL_SPEC_ID = 0 DEFAULT_SCHEMA_ID = 0 @@ -141,7 +142,7 @@ def construct_refs(cls, data: Dict[str, Any]): current_snapshot_id: Optional[int] = Field(alias="current-snapshot-id") """ID of the current table snapshot.""" - snapshots: list = Field(default_factory=list) + snapshots: List[Snapshot] = Field(default_factory=list) """A list of valid snapshots. Valid snapshots are snapshots for which all data files exist in the file system. A data file must not be deleted from the file system until the last snapshot in which it was @@ -332,7 +333,7 @@ def check_sort_orders(cls, values: Dict[str, Any]): Implementations must throw an exception if a table’s UUID does not match the expected UUID after refreshing metadata.""" - last_sequence_number: int = Field(alias="last-sequence-number", default=_INITIAL_SEQUENCE_NUMBER) + last_sequence_number: int = Field(alias="last-sequence-number", default=INITIAL_SEQUENCE_NUMBER) """The table’s highest assigned sequence number, a monotonically increasing long that tracks the order of snapshots in a table.""" diff --git a/pyiceberg/table/snapshots.py b/pyiceberg/table/snapshots.py new file mode 100644 index 0000000000..e6e613e179 --- /dev/null +++ b/pyiceberg/table/snapshots.py @@ -0,0 +1,90 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +from enum import Enum +from typing import Dict, Optional, Union + +from pydantic import Field, PrivateAttr, root_validator + +from pyiceberg.utils.iceberg_base_model import IcebergBaseModel + +OPERATION = "operation" + + +class Operation(Enum): + """Describes the operation + + Possible operation values are: + - append: Only data files were added and no files were removed. + - replace: Data and delete files were added and removed without changing table data; i.e., compaction, changing the data file format, or relocating data files. + - overwrite: Data and delete files were added and removed in a logical overwrite operation. + - delete: Data files were removed and their contents logically deleted and/or delete files were added to delete rows. + """ + + APPEND = "append" + REPLACE = "replace" + OVERWRITE = "overwrite" + DELETE = "delete" + + +class Summary(IcebergBaseModel): + """ + The snapshot summary’s operation field is used by some operations, + like snapshot expiration, to skip processing certain snapshots. + """ + + __root__: Dict[str, Union[str, Operation]] + _additional_properties: Dict[str, str] = PrivateAttr() + + @root_validator + def check_operation(cls, values: Dict[str, Dict[str, Union[str, Operation]]]) -> Dict[str, Dict[str, Union[str, Operation]]]: + if operation := values["__root__"].get(OPERATION): + if isinstance(operation, str): + values["__root__"][OPERATION] = Operation(operation.lower()) + else: + raise ValueError("Operation not set") + return values + + def __init__( + self, operation: Optional[Operation] = None, __root__: Optional[Dict[str, Union[str, Operation]]] = None, **data + ): + super().__init__(__root__={"operation": operation, **data} if not __root__ else __root__) + self._additional_properties = { + k: v for k, v in self.__root__.items() if k != OPERATION # type: ignore # We know that they are all string, and we don't want to check + } + + @property + def operation(self) -> Operation: + operation = self.__root__[OPERATION] + if isinstance(operation, Operation): + return operation + else: + # Should never happen + raise ValueError(f"Unknown type of operation: {operation}") + + @property + def additional_properties(self) -> Dict[str, str]: + return self._additional_properties + + +class Snapshot(IcebergBaseModel): + snapshot_id: int = Field(alias="snapshot-id") + parent_snapshot_id: Optional[int] = Field(alias="parent-snapshot-id") + sequence_number: Optional[int] = Field(alias="sequence-number", default=None) + timestamp_ms: int = Field(alias="timestamp-ms") + manifest_list: Optional[str] = Field(alias="manifest-list", description="Location of the snapshot's manifest list file") + summary: Optional[Summary] = Field() + schema_id: Optional[int] = Field(alias="schema-id", default=None) diff --git a/tests/table/test_metadata.py b/tests/table/test_metadata.py index fa64b5b199..1bd8d2572f 100644 --- a/tests/table/test_metadata.py +++ b/tests/table/test_metadata.py @@ -145,7 +145,7 @@ def test_v2_metadata_parsing(): assert table_metadata.last_partition_id == 1000 assert table_metadata.properties["read.split.target.size"] == "134217728" assert table_metadata.current_snapshot_id == 3055729675574597004 - assert table_metadata.snapshots[0]["snapshot-id"] == 3051729675574597004 + assert table_metadata.snapshots[0].snapshot_id == 3051729675574597004 assert table_metadata.snapshot_log[0]["timestamp-ms"] == 1515100955770 assert table_metadata.sort_orders[0].order_id == 3 assert table_metadata.default_sort_order_id == 3 @@ -205,18 +205,14 @@ def test_updating_metadata(): def test_serialize_v1(): table_metadata = TableMetadataV1(**EXAMPLE_TABLE_METADATA_V1).json() - assert ( - table_metadata - == """{"location": "s3://bucket/test/location", "table-uuid": "d20125c8-7284-442c-9aea-15fee620737c", "last-updated-ms": 1602638573874, "last-column-id": 3, "schemas": [{"fields": [{"id": 1, "name": "x", "type": "long", "required": true}, {"id": 2, "name": "y", "type": "long", "required": true, "doc": "comment"}, {"id": 3, "name": "z", "type": "long", "required": true}], "schema-id": 0, "identifier-field-ids": []}], "current-schema-id": 0, "partition-specs": [{"spec-id": 0, "fields": [{"name": "x", "transform": "identity", "source-id": 1, "field-id": 1000}]}], "default-spec-id": 0, "last-partition-id": 1000, "properties": {}, "snapshots": [{"snapshot-id": 1925, "timestamp-ms": 1602638573822}], "snapshot-log": [], "metadata-log": [], "sort-orders": [{"order-id": 0, "fields": []}], "default-sort-order-id": 0, "refs": {}, "format-version": 1, "schema": {"fields": [{"id": 1, "name": "x", "type": "long", "required": true}, {"id": 2, "name": "y", "type": "long", "required": true, "doc": "comment"}, {"id": 3, "name": "z", "type": "long", "required": true}], "schema-id": 0, "identifier-field-ids": []}, "partition-spec": [{"name": "x", "transform": "identity", "source-id": 1, "field-id": 1000}]}""" - ) + expected = """{"location": "s3://bucket/test/location", "table-uuid": "d20125c8-7284-442c-9aea-15fee620737c", "last-updated-ms": 1602638573874, "last-column-id": 3, "schemas": [{"fields": [{"id": 1, "name": "x", "type": "long", "required": true}, {"id": 2, "name": "y", "type": "long", "required": true, "doc": "comment"}, {"id": 3, "name": "z", "type": "long", "required": true}], "schema-id": 0, "identifier-field-ids": []}], "current-schema-id": 0, "partition-specs": [{"spec-id": 0, "fields": [{"name": "x", "transform": "identity", "source-id": 1, "field-id": 1000}]}], "default-spec-id": 0, "last-partition-id": 1000, "properties": {}, "snapshots": [{"snapshot-id": 1925, "timestamp-ms": 1602638573822}], "snapshot-log": [], "metadata-log": [], "sort-orders": [{"order-id": 0, "fields": []}], "default-sort-order-id": 0, "refs": {}, "format-version": 1, "schema": {"fields": [{"id": 1, "name": "x", "type": "long", "required": true}, {"id": 2, "name": "y", "type": "long", "required": true, "doc": "comment"}, {"id": 3, "name": "z", "type": "long", "required": true}], "schema-id": 0, "identifier-field-ids": []}, "partition-spec": [{"name": "x", "transform": "identity", "source-id": 1, "field-id": 1000}]}""" + assert table_metadata == expected def test_serialize_v2(): table_metadata = TableMetadataV2(**EXAMPLE_TABLE_METADATA_V2).json() - assert ( - table_metadata - == """{"location": "s3://bucket/test/location", "table-uuid": "9c12d441-03fe-4693-9a96-a0705ddf69c1", "last-updated-ms": 1602638573590, "last-column-id": 3, "schemas": [{"fields": [{"id": 1, "name": "x", "type": "long", "required": true}], "schema-id": 0, "identifier-field-ids": []}, {"fields": [{"id": 1, "name": "x", "type": "long", "required": true}, {"id": 2, "name": "y", "type": "long", "required": true, "doc": "comment"}, {"id": 3, "name": "z", "type": "long", "required": true}], "schema-id": 1, "identifier-field-ids": [1, 2]}], "current-schema-id": 1, "partition-specs": [{"spec-id": 0, "fields": [{"name": "x", "transform": "identity", "source-id": 1, "field-id": 1000}]}], "default-spec-id": 0, "last-partition-id": 1000, "properties": {"read.split.target.size": "134217728"}, "current-snapshot-id": 3055729675574597004, "snapshots": [{"snapshot-id": 3051729675574597004, "timestamp-ms": 1515100955770, "sequence-number": 0, "summary": {"operation": "append"}, "manifest-list": "s3://a/b/1.avro"}, {"snapshot-id": 3055729675574597004, "parent-snapshot-id": 3051729675574597004, "timestamp-ms": 1555100955770, "sequence-number": 1, "summary": {"operation": "append"}, "manifest-list": "s3://a/b/2.avro", "schema-id": 1}], "snapshot-log": [{"snapshot-id": 3051729675574597004, "timestamp-ms": 1515100955770}, {"snapshot-id": 3055729675574597004, "timestamp-ms": 1555100955770}], "metadata-log": [], "sort-orders": [{"order-id": 3, "fields": [{"source-id": 2, "transform": "identity", "direction": "asc", "null-order": "nulls-first"}, {"source-id": 3, "transform": "bucket[4]", "direction": "desc", "null-order": "nulls-last"}]}], "default-sort-order-id": 3, "refs": {"main": {"snapshot-id": 3055729675574597004, "type": "branch"}}, "format-version": 2, "last-sequence-number": 34}""" - ) + expected = """{"location": "s3://bucket/test/location", "table-uuid": "9c12d441-03fe-4693-9a96-a0705ddf69c1", "last-updated-ms": 1602638573590, "last-column-id": 3, "schemas": [{"fields": [{"id": 1, "name": "x", "type": "long", "required": true}], "schema-id": 0, "identifier-field-ids": []}, {"fields": [{"id": 1, "name": "x", "type": "long", "required": true}, {"id": 2, "name": "y", "type": "long", "required": true, "doc": "comment"}, {"id": 3, "name": "z", "type": "long", "required": true}], "schema-id": 1, "identifier-field-ids": [1, 2]}], "current-schema-id": 1, "partition-specs": [{"spec-id": 0, "fields": [{"name": "x", "transform": "identity", "source-id": 1, "field-id": 1000}]}], "default-spec-id": 0, "last-partition-id": 1000, "properties": {"read.split.target.size": "134217728"}, "current-snapshot-id": 3055729675574597004, "snapshots": [{"snapshot-id": 3051729675574597004, "sequence-number": 0, "timestamp-ms": 1515100955770, "manifest-list": "s3://a/b/1.avro", "summary": {"operation": "append"}}, {"snapshot-id": 3055729675574597004, "parent-snapshot-id": 3051729675574597004, "sequence-number": 1, "timestamp-ms": 1555100955770, "manifest-list": "s3://a/b/2.avro", "summary": {"operation": "append"}, "schema-id": 1}], "snapshot-log": [{"snapshot-id": 3051729675574597004, "timestamp-ms": 1515100955770}, {"snapshot-id": 3055729675574597004, "timestamp-ms": 1555100955770}], "metadata-log": [], "sort-orders": [{"order-id": 3, "fields": [{"source-id": 2, "transform": "identity", "direction": "asc", "null-order": "nulls-first"}, {"source-id": 3, "transform": "bucket[4]", "direction": "desc", "null-order": "nulls-last"}]}], "default-sort-order-id": 3, "refs": {"main": {"snapshot-id": 3055729675574597004, "type": "branch"}}, "format-version": 2, "last-sequence-number": 34}""" + assert table_metadata == expected def test_migrate_v1_schemas(): diff --git a/tests/table/test_snapshots.py b/tests/table/test_snapshots.py new file mode 100644 index 0000000000..afce7209de --- /dev/null +++ b/tests/table/test_snapshots.py @@ -0,0 +1,113 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +from pyiceberg.table.snapshots import Operation, Snapshot, Summary + + +def test_serialize_summary(): + assert Summary(Operation.APPEND).json() == """{"operation": "append"}""" + + +def test_serialize_summary_with_properties(): + assert Summary(Operation.APPEND, property="yes").json() == """{"operation": "append", "property": "yes"}""" + + +def test_serialize_snapshot(): + snapshot = Snapshot( + snapshot_id=25, + parent_snapshot_id=19, + sequence_number=200, + timestamp_ms=1602638573590, + manifest_list="s3:/a/b/c.avro", + summary=Summary(Operation.APPEND), + schema_id=3, + ) + + assert ( + snapshot.json() + == """{"snapshot-id": 25, "parent-snapshot-id": 19, "sequence-number": 200, "timestamp-ms": 1602638573590, "manifest-list": "s3:/a/b/c.avro", "summary": {"operation": "append"}, "schema-id": 3}""" + ) + + +def test_serialize_snapshot_without_sequence_number(): + snapshot = Snapshot( + snapshot_id=25, + parent_snapshot_id=19, + timestamp_ms=1602638573590, + manifest_list="s3:/a/b/c.avro", + summary=Summary(Operation.APPEND), + schema_id=3, + ) + + actual = snapshot.json() + expected = """{"snapshot-id": 25, "parent-snapshot-id": 19, "timestamp-ms": 1602638573590, "manifest-list": "s3:/a/b/c.avro", "summary": {"operation": "append"}, "schema-id": 3}""" + assert actual == expected + + +def test_serialize_snapshot_with_properties(): + snapshot = Snapshot( + snapshot_id=25, + parent_snapshot_id=19, + sequence_number=200, + timestamp_ms=1602638573590, + manifest_list="s3:/a/b/c.avro", + summary=Summary(Operation.APPEND, property="yes"), + schema_id=3, + ) + + assert ( + snapshot.json() + == """{"snapshot-id": 25, "parent-snapshot-id": 19, "sequence-number": 200, "timestamp-ms": 1602638573590, "manifest-list": "s3:/a/b/c.avro", "summary": {"operation": "append", "property": "yes"}, "schema-id": 3}""" + ) + + +def test_deserialize_summary(): + summary = Summary.parse_raw("""{"operation": "append"}""") + assert summary.operation == Operation.APPEND + + +def test_deserialize_summary_with_properties(): + summary = Summary.parse_raw("""{"operation": "append", "property": "yes"}""") + assert summary.operation == Operation.APPEND + assert summary.additional_properties == {"property": "yes"} + + +def test_deserialize_snapshot(): + payload = """{"snapshot-id": 25, "parent-snapshot-id": 19, "sequence-number": 200, "timestamp-ms": 1602638573590, "manifest-list": "s3:/a/b/c.avro", "summary": {"operation": "append"}, "schema-id": 3}""" + snapshot = Snapshot.parse_raw(payload) + assert snapshot == Snapshot( + snapshot_id=25, + parent_snapshot_id=19, + sequence_number=200, + timestamp_ms=1602638573590, + manifest_list="s3:/a/b/c.avro", + summary=Summary(Operation.APPEND), + schema_id=3, + ) + + +def test_deserialize_snapshot_with_properties(): + payload = """{"snapshot-id": 25, "parent-snapshot-id": 19, "sequence-number": 200, "timestamp-ms": 1602638573590, "manifest-list": "s3:/a/b/c.avro", "summary": {"operation": "append", "property": "yes"}, "schema-id": 3}""" + snapshot = Snapshot.parse_raw(payload) + assert snapshot == Snapshot( + snapshot_id=25, + parent_snapshot_id=19, + sequence_number=200, + timestamp_ms=1602638573590, + manifest_list="s3:/a/b/c.avro", + summary=Summary(Operation.APPEND, property="yes"), + schema_id=3, + ) From 12c18414d39b0c6e60ce6eae549f4e209941c429 Mon Sep 17 00:00:00 2001 From: Fokko Driesprong Date: Tue, 12 Jul 2022 22:23:26 +0200 Subject: [PATCH 135/642] Python: Bump pre-commit versions (#5203) --- .pre-commit-config.yaml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 0b01ae1afc..efbc6cc7f2 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -28,7 +28,7 @@ repos: - id: check-yaml - id: check-ast - repo: https://github.com/ambv/black - rev: 22.3.0 + rev: 22.6.0 hooks: - id: black - repo: https://github.com/pre-commit/mirrors-isort @@ -42,7 +42,7 @@ repos: - id: mypy args: [--config=python/pyproject.toml] - repo: https://github.com/hadialqattan/pycln - rev: v1.3.4 + rev: v2.0.1 hooks: - id: pycln args: [--config=python/pyproject.toml] @@ -52,7 +52,7 @@ repos: - id: pyupgrade args: [--py38-plus] - repo: https://github.com/pycqa/pylint - rev: v2.14.3 + rev: v2.14.4 hooks: - id: pylint args: [ --rcfile=python/pylintrc ] @@ -61,4 +61,4 @@ repos: hooks: - id: flake8 args: [ "--ignore=E501,W503,E203" ] - additional_dependencies: [ flake8-bugbear==22.4.25, flake8-comprehensions==3.10.0 ] + additional_dependencies: [ flake8-bugbear==22.7.1, flake8-comprehensions==3.10.0 ] From 35ed4543581e47e5724f12fc12436c27cb560adf Mon Sep 17 00:00:00 2001 From: Fokko Driesprong Date: Wed, 13 Jul 2022 21:09:30 +0200 Subject: [PATCH 136/642] Python: Move PartitionSpec to Pydantic (#5192) --- pyiceberg/catalog/base.py | 3 +- pyiceberg/table/base.py | 7 --- pyiceberg/table/metadata.py | 11 ++-- pyiceberg/table/partitioning.py | 96 ++++++++++++++++++++++---------- pyiceberg/transforms.py | 2 +- tests/catalog/test_base.py | 6 +- tests/table/test_metadata.py | 19 +++++-- tests/table/test_partitioning.py | 55 ++++++++++++++---- 8 files changed, 136 insertions(+), 63 deletions(-) diff --git a/pyiceberg/catalog/base.py b/pyiceberg/catalog/base.py index 42d07b088b..6a2dcf9e02 100644 --- a/pyiceberg/catalog/base.py +++ b/pyiceberg/catalog/base.py @@ -21,7 +21,8 @@ from pyiceberg.catalog import Identifier, Properties from pyiceberg.schema import Schema -from pyiceberg.table.base import PartitionSpec, Table +from pyiceberg.table.base import Table +from pyiceberg.table.partitioning import PartitionSpec class Catalog(ABC): diff --git a/pyiceberg/table/base.py b/pyiceberg/table/base.py index a1d7979063..77821547e7 100644 --- a/pyiceberg/table/base.py +++ b/pyiceberg/table/base.py @@ -29,10 +29,3 @@ class Table(ABC): """ identifier: str | Identifier - - -class PartitionSpec: - """Placeholder for Partition Specification - - To be implemented by https://github.com/apache/iceberg/issues/4631 - """ diff --git a/pyiceberg/table/metadata.py b/pyiceberg/table/metadata.py index 9c506e515c..6543d4ce30 100644 --- a/pyiceberg/table/metadata.py +++ b/pyiceberg/table/metadata.py @@ -29,6 +29,7 @@ from pyiceberg.exceptions import ValidationError from pyiceberg.schema import Schema +from pyiceberg.table.partitioning import PartitionSpec from pyiceberg.table.refs import MAIN_BRANCH, SnapshotRef, SnapshotRefType from pyiceberg.table.snapshots import Snapshot from pyiceberg.table.sorting import UNSORTED_SORT_ORDER, UNSORTED_SORT_ORDER_ID, SortOrder @@ -54,8 +55,9 @@ def check_partition_specs(values: Dict[str, Any]) -> Dict[str, Any]: """Validator to check if the default-spec-id is present in partition-specs""" default_spec_id = values["default_spec_id"] - for spec in values["partition_specs"]: - if spec["spec-id"] == default_spec_id: + partition_specs: List[PartitionSpec] = values["partition_specs"] + for spec in partition_specs: + if spec.spec_id == default_spec_id: return values raise ValidationError(f"default-spec-id {default_spec_id} can't be found") @@ -122,7 +124,7 @@ def construct_refs(cls, data: Dict[str, Any]): current_schema_id: int = Field(alias="current-schema-id", default=DEFAULT_SCHEMA_ID) """ID of the table’s current schema.""" - partition_specs: list = Field(alias="partition-specs", default_factory=list) + partition_specs: List[PartitionSpec] = Field(alias="partition-specs", default_factory=list) """A list of partition specs, stored as full partition spec objects.""" default_spec_id: int = Field(alias="default-spec-id", default=INITIAL_SPEC_ID) @@ -251,10 +253,9 @@ def construct_partition_specs(cls, data: Dict[str, Any]) -> Dict[str, Any]: Returns: The TableMetadata with the partition_specs set, if not provided """ - # This is going to be much nicer as soon as partition-spec is also migrated to pydantic if not data.get("partition_specs"): fields = data["partition_spec"] - data["partition_specs"] = [{"spec-id": INITIAL_SPEC_ID, "fields": fields}] + data["partition_specs"] = [PartitionSpec(spec_id=INITIAL_SPEC_ID, fields=fields)] else: check_partition_specs(data["partition_specs"]) return data diff --git a/pyiceberg/table/partitioning.py b/pyiceberg/table/partitioning.py index b44c2a1ed9..ef080bc307 100644 --- a/pyiceberg/table/partitioning.py +++ b/pyiceberg/table/partitioning.py @@ -14,17 +14,24 @@ # KIND, either express or implied. See the License for the # specific language governing permissions and limitations # under the License. -from dataclasses import dataclass, field -from typing import Dict, List, Tuple +from functools import cached_property +from typing import ( + Any, + Dict, + List, + Optional, + Tuple, +) + +from pydantic import Field -from pyiceberg.schema import Schema from pyiceberg.transforms import Transform +from pyiceberg.utils.iceberg_base_model import IcebergBaseModel _PARTITION_DATA_ID_START: int = 1000 -@dataclass(frozen=True) -class PartitionField: +class PartitionField(IcebergBaseModel): """ PartitionField is a single element with name and unique id, It represents how one partition value is derived from the source column via transformation @@ -36,17 +43,34 @@ class PartitionField: name(str): The name of this partition field """ - source_id: int - field_id: int - transform: Transform - name: str + source_id: int = Field(alias="source-id") + field_id: int = Field(alias="field-id") + transform: Transform = Field() + name: str = Field() + + def __init__( + self, + source_id: Optional[int] = None, + field_id: Optional[int] = None, + transform: Optional[Transform] = None, + name: Optional[str] = None, + **data: Any, + ): + if source_id is not None: + data["source-id"] = source_id + if field_id is not None: + data["field-id"] = field_id + if transform is not None: + data["transform"] = transform + if name is not None: + data["name"] = name + super().__init__(**data) def __str__(self): return f"{self.field_id}: {self.name}: {self.transform}({self.source_id})" -@dataclass(eq=False, frozen=True) -class PartitionSpec: +class PartitionSpec(IcebergBaseModel): """ PartitionSpec captures the transformation from table data to partition values @@ -54,27 +78,24 @@ class PartitionSpec: schema(Schema): the schema of data table spec_id(int): any change to PartitionSpec will produce a new specId fields(List[PartitionField): list of partition fields to produce partition values - last_assigned_field_id(int): auto-increment partition field id starting from PARTITION_DATA_ID_START """ - schema: Schema - spec_id: int - fields: Tuple[PartitionField, ...] - last_assigned_field_id: int - source_id_to_fields_map: Dict[int, List[PartitionField]] = field(init=False, repr=False) - - def __post_init__(self): - source_id_to_fields_map = {} - for partition_field in self.fields: - source_column = self.schema.find_column_name(partition_field.source_id) - if not source_column: - raise ValueError(f"Cannot find source column: {partition_field.source_id}") - existing = source_id_to_fields_map.get(partition_field.source_id, []) - existing.append(partition_field) - source_id_to_fields_map[partition_field.source_id] = existing - object.__setattr__(self, "source_id_to_fields_map", source_id_to_fields_map) - - def __eq__(self, other): + spec_id: int = Field(alias="spec-id") + fields: Tuple[PartitionField, ...] = Field() + + def __init__( + self, + spec_id: Optional[int] = None, + fields: Optional[Tuple[PartitionField, ...]] = None, + **data: Any, + ): + if spec_id is not None: + data["spec-id"] = spec_id + if fields is not None: + data["fields"] = fields + super().__init__(**data) + + def __eq__(self, other: Any) -> bool: """ Produce a boolean to return True if two objects are considered equal @@ -101,6 +122,21 @@ def __str__(self): def is_unpartitioned(self) -> bool: return not self.fields + @property + def last_assigned_field_id(self) -> int: + if self.fields: + return max(pf.field_id for pf in self.fields) + return _PARTITION_DATA_ID_START + + @cached_property + def source_id_to_fields_map(self) -> Dict[int, List[PartitionField]]: + source_id_to_fields_map: Dict[int, List[PartitionField]] = {} + for partition_field in self.fields: + existing = source_id_to_fields_map.get(partition_field.source_id, []) + existing.append(partition_field) + source_id_to_fields_map[partition_field.source_id] = existing + return source_id_to_fields_map + def fields_by_source_id(self, field_id: int) -> List[PartitionField]: return self.source_id_to_fields_map[field_id] diff --git a/pyiceberg/transforms.py b/pyiceberg/transforms.py index 824afc3419..fdb4978d60 100644 --- a/pyiceberg/transforms.py +++ b/pyiceberg/transforms.py @@ -91,7 +91,7 @@ def validate(cls, v: Any): elif v.startswith(BUCKET): return BucketTransform(num_buckets=BUCKET_PARSER.match(v)) elif v.startswith(TRUNCATE): - return TruncateTransform(width=BUCKET_PARSER.match(v)) + return TruncateTransform(width=TRUNCATE_PARSER.match(v)) else: return UnknownTransform(transform=v) return v diff --git a/tests/catalog/test_base.py b/tests/catalog/test_base.py index 3aa078e84c..e4dd4d3992 100644 --- a/tests/catalog/test_base.py +++ b/tests/catalog/test_base.py @@ -34,7 +34,9 @@ NoSuchTableError, ) from pyiceberg.schema import Schema -from pyiceberg.table.base import PartitionSpec, Table +from pyiceberg.table.base import Table +from pyiceberg.table.metadata import INITIAL_SPEC_ID +from pyiceberg.table.partitioning import PartitionSpec class InMemoryCatalog(Catalog): @@ -159,7 +161,7 @@ def update_namespace_properties( TEST_TABLE_NAME = "my_table" TEST_TABLE_SCHEMA = Schema(schema_id=1) TEST_TABLE_LOCATION = "protocol://some/location" -TEST_TABLE_PARTITION_SPEC = PartitionSpec() +TEST_TABLE_PARTITION_SPEC = PartitionSpec(spec_id=INITIAL_SPEC_ID, fields=()) TEST_TABLE_PROPERTIES = {"key1": "value1", "key2": "value2"} NO_SUCH_TABLE_ERROR = "Table does not exist: \\('com', 'organization', 'department', 'my_table'\\)" TABLE_ALREADY_EXISTS_ERROR = "Table already exists: \\('com', 'organization', 'department', 'my_table'\\)" diff --git a/tests/table/test_metadata.py b/tests/table/test_metadata.py index 1bd8d2572f..0a58f201de 100644 --- a/tests/table/test_metadata.py +++ b/tests/table/test_metadata.py @@ -25,7 +25,9 @@ from pyiceberg.schema import Schema from pyiceberg.serializers import FromByteStream from pyiceberg.table.metadata import TableMetadata, TableMetadataV1, TableMetadataV2 +from pyiceberg.table.partitioning import PartitionField, PartitionSpec from pyiceberg.table.refs import SnapshotRef, SnapshotRefType +from pyiceberg.transforms import IdentityTransform from pyiceberg.types import LongType, NestedField EXAMPLE_TABLE_METADATA_V1 = { @@ -140,7 +142,7 @@ def test_v2_metadata_parsing(): assert table_metadata.last_column_id == 3 assert table_metadata.schemas[0].schema_id == 0 assert table_metadata.current_schema_id == 1 - assert table_metadata.partition_specs[0]["spec-id"] == 0 + assert table_metadata.partition_specs[0].spec_id == 0 assert table_metadata.default_spec_id == 0 assert table_metadata.last_partition_id == 1000 assert table_metadata.properties["read.split.target.size"] == "134217728" @@ -205,13 +207,13 @@ def test_updating_metadata(): def test_serialize_v1(): table_metadata = TableMetadataV1(**EXAMPLE_TABLE_METADATA_V1).json() - expected = """{"location": "s3://bucket/test/location", "table-uuid": "d20125c8-7284-442c-9aea-15fee620737c", "last-updated-ms": 1602638573874, "last-column-id": 3, "schemas": [{"fields": [{"id": 1, "name": "x", "type": "long", "required": true}, {"id": 2, "name": "y", "type": "long", "required": true, "doc": "comment"}, {"id": 3, "name": "z", "type": "long", "required": true}], "schema-id": 0, "identifier-field-ids": []}], "current-schema-id": 0, "partition-specs": [{"spec-id": 0, "fields": [{"name": "x", "transform": "identity", "source-id": 1, "field-id": 1000}]}], "default-spec-id": 0, "last-partition-id": 1000, "properties": {}, "snapshots": [{"snapshot-id": 1925, "timestamp-ms": 1602638573822}], "snapshot-log": [], "metadata-log": [], "sort-orders": [{"order-id": 0, "fields": []}], "default-sort-order-id": 0, "refs": {}, "format-version": 1, "schema": {"fields": [{"id": 1, "name": "x", "type": "long", "required": true}, {"id": 2, "name": "y", "type": "long", "required": true, "doc": "comment"}, {"id": 3, "name": "z", "type": "long", "required": true}], "schema-id": 0, "identifier-field-ids": []}, "partition-spec": [{"name": "x", "transform": "identity", "source-id": 1, "field-id": 1000}]}""" + expected = """{"location": "s3://bucket/test/location", "table-uuid": "d20125c8-7284-442c-9aea-15fee620737c", "last-updated-ms": 1602638573874, "last-column-id": 3, "schemas": [{"fields": [{"id": 1, "name": "x", "type": "long", "required": true}, {"id": 2, "name": "y", "type": "long", "required": true, "doc": "comment"}, {"id": 3, "name": "z", "type": "long", "required": true}], "schema-id": 0, "identifier-field-ids": []}], "current-schema-id": 0, "partition-specs": [{"spec-id": 0, "fields": [{"source-id": 1, "field-id": 1000, "transform": "identity", "name": "x"}]}], "default-spec-id": 0, "last-partition-id": 1000, "properties": {}, "snapshots": [{"snapshot-id": 1925, "timestamp-ms": 1602638573822}], "snapshot-log": [], "metadata-log": [], "sort-orders": [{"order-id": 0, "fields": []}], "default-sort-order-id": 0, "refs": {}, "format-version": 1, "schema": {"fields": [{"id": 1, "name": "x", "type": "long", "required": true}, {"id": 2, "name": "y", "type": "long", "required": true, "doc": "comment"}, {"id": 3, "name": "z", "type": "long", "required": true}], "schema-id": 0, "identifier-field-ids": []}, "partition-spec": [{"name": "x", "transform": "identity", "source-id": 1, "field-id": 1000}]}""" assert table_metadata == expected def test_serialize_v2(): table_metadata = TableMetadataV2(**EXAMPLE_TABLE_METADATA_V2).json() - expected = """{"location": "s3://bucket/test/location", "table-uuid": "9c12d441-03fe-4693-9a96-a0705ddf69c1", "last-updated-ms": 1602638573590, "last-column-id": 3, "schemas": [{"fields": [{"id": 1, "name": "x", "type": "long", "required": true}], "schema-id": 0, "identifier-field-ids": []}, {"fields": [{"id": 1, "name": "x", "type": "long", "required": true}, {"id": 2, "name": "y", "type": "long", "required": true, "doc": "comment"}, {"id": 3, "name": "z", "type": "long", "required": true}], "schema-id": 1, "identifier-field-ids": [1, 2]}], "current-schema-id": 1, "partition-specs": [{"spec-id": 0, "fields": [{"name": "x", "transform": "identity", "source-id": 1, "field-id": 1000}]}], "default-spec-id": 0, "last-partition-id": 1000, "properties": {"read.split.target.size": "134217728"}, "current-snapshot-id": 3055729675574597004, "snapshots": [{"snapshot-id": 3051729675574597004, "sequence-number": 0, "timestamp-ms": 1515100955770, "manifest-list": "s3://a/b/1.avro", "summary": {"operation": "append"}}, {"snapshot-id": 3055729675574597004, "parent-snapshot-id": 3051729675574597004, "sequence-number": 1, "timestamp-ms": 1555100955770, "manifest-list": "s3://a/b/2.avro", "summary": {"operation": "append"}, "schema-id": 1}], "snapshot-log": [{"snapshot-id": 3051729675574597004, "timestamp-ms": 1515100955770}, {"snapshot-id": 3055729675574597004, "timestamp-ms": 1555100955770}], "metadata-log": [], "sort-orders": [{"order-id": 3, "fields": [{"source-id": 2, "transform": "identity", "direction": "asc", "null-order": "nulls-first"}, {"source-id": 3, "transform": "bucket[4]", "direction": "desc", "null-order": "nulls-last"}]}], "default-sort-order-id": 3, "refs": {"main": {"snapshot-id": 3055729675574597004, "type": "branch"}}, "format-version": 2, "last-sequence-number": 34}""" + expected = """{"location": "s3://bucket/test/location", "table-uuid": "9c12d441-03fe-4693-9a96-a0705ddf69c1", "last-updated-ms": 1602638573590, "last-column-id": 3, "schemas": [{"fields": [{"id": 1, "name": "x", "type": "long", "required": true}], "schema-id": 0, "identifier-field-ids": []}, {"fields": [{"id": 1, "name": "x", "type": "long", "required": true}, {"id": 2, "name": "y", "type": "long", "required": true, "doc": "comment"}, {"id": 3, "name": "z", "type": "long", "required": true}], "schema-id": 1, "identifier-field-ids": [1, 2]}], "current-schema-id": 1, "partition-specs": [{"spec-id": 0, "fields": [{"source-id": 1, "field-id": 1000, "transform": "identity", "name": "x"}]}], "default-spec-id": 0, "last-partition-id": 1000, "properties": {"read.split.target.size": "134217728"}, "current-snapshot-id": 3055729675574597004, "snapshots": [{"snapshot-id": 3051729675574597004, "sequence-number": 0, "timestamp-ms": 1515100955770, "manifest-list": "s3://a/b/1.avro", "summary": {"operation": "append"}}, {"snapshot-id": 3055729675574597004, "parent-snapshot-id": 3051729675574597004, "sequence-number": 1, "timestamp-ms": 1555100955770, "manifest-list": "s3://a/b/2.avro", "summary": {"operation": "append"}, "schema-id": 1}], "snapshot-log": [{"snapshot-id": 3051729675574597004, "timestamp-ms": 1515100955770}, {"snapshot-id": 3055729675574597004, "timestamp-ms": 1555100955770}], "metadata-log": [], "sort-orders": [{"order-id": 3, "fields": [{"source-id": 2, "transform": "identity", "direction": "asc", "null-order": "nulls-first"}, {"source-id": 3, "transform": "bucket[4]", "direction": "desc", "null-order": "nulls-last"}]}], "default-sort-order-id": 3, "refs": {"main": {"snapshot-id": 3055729675574597004, "type": "branch"}}, "format-version": 2, "last-sequence-number": 34}""" assert table_metadata == expected @@ -230,7 +232,11 @@ def test_migrate_v1_partition_specs(): assert len(table_metadata.partition_specs) == 1 # Spec ID gets added automatically assert table_metadata.partition_specs == [ - {"spec-id": 0, "fields": [{"field-id": 1000, "name": "x", "source-id": 1, "transform": "identity"}]} + PartitionSpec( + spec_id=0, + fields=(PartitionField(source_id=1, field_id=1000, transform=IdentityTransform(), name="x"),), + last_assigned_field_id=1000, + ), ] @@ -502,7 +508,10 @@ def test_v1_write_metadata_for_v2(): } ] assert metadata_v2["partition-specs"] == [ - {"spec-id": 0, "fields": [{"name": "x", "transform": "identity", "source-id": 1, "field-id": 1000}]} + { + "spec-id": 0, + "fields": [{"name": "x", "transform": "identity", "source-id": 1, "field-id": 1000}], + } ] assert metadata_v2["default-spec-id"] == 0 assert metadata_v2["last-partition-id"] == 1000 diff --git a/tests/table/test_partitioning.py b/tests/table/test_partitioning.py index 12c74625ee..1fe9dfeb01 100644 --- a/tests/table/test_partitioning.py +++ b/tests/table/test_partitioning.py @@ -14,10 +14,8 @@ # KIND, either express or implied. See the License for the # specific language governing permissions and limitations # under the License. - -from pyiceberg.schema import Schema from pyiceberg.table.partitioning import PartitionField, PartitionSpec -from pyiceberg.transforms import BucketTransform +from pyiceberg.transforms import BucketTransform, TruncateTransform def test_partition_field_init(): @@ -36,37 +34,70 @@ def test_partition_field_init(): ) -def test_partition_spec_init(table_schema_simple: Schema): +def test_partition_spec_init(): bucket_transform: BucketTransform = BucketTransform(4) + id_field1 = PartitionField(3, 1001, bucket_transform, "id") - partition_spec1 = PartitionSpec(table_schema_simple, 0, (id_field1,), 1001) + partition_spec1 = PartitionSpec(0, (id_field1,)) assert partition_spec1.spec_id == 0 - assert partition_spec1.schema == table_schema_simple assert partition_spec1 == partition_spec1 assert partition_spec1 != id_field1 assert str(partition_spec1) == f"[\n {str(id_field1)}\n]" assert not partition_spec1.is_unpartitioned() # only differ by PartitionField field_id id_field2 = PartitionField(3, 1002, bucket_transform, "id") - partition_spec2 = PartitionSpec(table_schema_simple, 0, (id_field2,), 1001) + partition_spec2 = PartitionSpec(0, (id_field2,)) assert partition_spec1 != partition_spec2 assert partition_spec1.compatible_with(partition_spec2) assert partition_spec1.fields_by_source_id(3) == [id_field1] -def test_partition_compatible_with(table_schema_simple: Schema): +def test_partition_compatible_with(): bucket_transform: BucketTransform = BucketTransform(4) field1 = PartitionField(3, 100, bucket_transform, "id") field2 = PartitionField(3, 102, bucket_transform, "id") - lhs = PartitionSpec(table_schema_simple, 0, (field1,), 1001) - rhs = PartitionSpec(table_schema_simple, 0, (field1, field2), 1001) + lhs = PartitionSpec(0, (field1,)) + rhs = PartitionSpec(0, (field1, field2)) assert not lhs.compatible_with(rhs) -def test_unpartitioned(table_schema_simple: Schema): - unpartitioned = PartitionSpec(table_schema_simple, 1, (), 1000) +def test_unpartitioned(): + unpartitioned = PartitionSpec(1, ()) assert not unpartitioned.fields assert unpartitioned.is_unpartitioned() assert str(unpartitioned) == "[]" + + +def test_serialize_unpartition_spec(): + unpartitioned = PartitionSpec(1, ()) + assert unpartitioned.json() == """{"spec-id": 1, "fields": []}""" + + +def test_serialize_partition_spec(): + partitioned = PartitionSpec( + spec_id=3, + fields=( + PartitionField(source_id=1, field_id=1000, transform=TruncateTransform(width=19), name="str_truncate"), + PartitionField(source_id=2, field_id=1001, transform=BucketTransform(num_buckets=25), name="int_bucket"), + ), + ) + assert ( + partitioned.json() + == """{"spec-id": 3, "fields": [{"source-id": 1, "field-id": 1000, "transform": "truncate[19]", "name": "str_truncate"}, {"source-id": 2, "field-id": 1001, "transform": "bucket[25]", "name": "int_bucket"}]}""" + ) + + +def test_deserialize_partition_spec(): + json_partition_spec = """{"spec-id": 3, "fields": [{"source-id": 1, "field-id": 1000, "transform": "truncate[19]", "name": "str_truncate"}, {"source-id": 2, "field-id": 1001, "transform": "bucket[25]", "name": "int_bucket"}]}""" + + spec = PartitionSpec.parse_raw(json_partition_spec) + + assert spec == PartitionSpec( + spec_id=3, + fields=( + PartitionField(source_id=1, field_id=1000, transform=TruncateTransform(width=19), name="str_truncate"), + PartitionField(source_id=2, field_id=1001, transform=BucketTransform(num_buckets=25), name="int_bucket"), + ), + ) From 6580147e25f2935379381380c9508d1c48e162ad Mon Sep 17 00:00:00 2001 From: Fokko Driesprong Date: Thu, 21 Jul 2022 17:33:52 +0200 Subject: [PATCH 137/642] Python: Add __version__ to the package (#5315) --- dev/RELEASE.md | 4 ++-- pyiceberg/__init__.py | 3 +++ pyproject.toml | 2 +- tests/test_version.py | 29 +++++++++++++++++++++++++++++ 4 files changed, 35 insertions(+), 3 deletions(-) create mode 100644 tests/test_version.py diff --git a/dev/RELEASE.md b/dev/RELEASE.md index d54fa0132d..03ec767808 100644 --- a/dev/RELEASE.md +++ b/dev/RELEASE.md @@ -28,8 +28,8 @@ First we're going to release a release candidate (RC) and publish it to the publ Make sure that you're on the version that you want to release. ```bash -export RC=rc1 -export VERSION=0.0.1${RC} +export RC=rc0 +export VERSION=0.0.1.${RC} export VERSION_WITHOUT_RC=${VERSION/rc?/} export VERSION_BRANCH=${VERSION_WITHOUT_RC//./-} diff --git a/pyiceberg/__init__.py b/pyiceberg/__init__.py index 13a83393a9..0c4a31db7a 100644 --- a/pyiceberg/__init__.py +++ b/pyiceberg/__init__.py @@ -14,3 +14,6 @@ # KIND, either express or implied. See the License for the # specific language governing permissions and limitations # under the License. +from importlib import metadata + +__version__ = metadata.version(__package__) diff --git a/pyproject.toml b/pyproject.toml index a3e35bfc94..3d080332ca 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -17,7 +17,7 @@ [tool.poetry] name = "pyiceberg" -version = "0.0.1rc1" +version = "0.1.0.dev0" readme = "README.md" homepage = "https://iceberg.apache.org/" repository = "https://github.com/apache/iceberg/" diff --git a/tests/test_version.py b/tests/test_version.py new file mode 100644 index 0000000000..12a058e3f3 --- /dev/null +++ b/tests/test_version.py @@ -0,0 +1,29 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +import re + +from pyiceberg import __version__ + +VERSION_REGEX = re.compile(r"^\d+.\d+.\d+(.(dev|rc)\d+)?$") + + +def test_version_format(): + # should be in the format of 0.14.0 or 0.14.0.dev0 + assert VERSION_REGEX.search(__version__) + + # RCs should work as well + assert VERSION_REGEX.search("0.1.0.rc0") From f0acbc6917dd4c6cdf19fc5f0a84a2cc7139b252 Mon Sep 17 00:00:00 2001 From: Fokko Driesprong Date: Thu, 21 Jul 2022 17:36:42 +0200 Subject: [PATCH 138/642] Python: Remove .python-version file (#5326) --- .python-version | 3 --- 1 file changed, 3 deletions(-) delete mode 100644 .python-version diff --git a/.python-version b/.python-version deleted file mode 100644 index 879a094044..0000000000 --- a/.python-version +++ /dev/null @@ -1,3 +0,0 @@ -3.8.12 -3.9.10 -3.10.4 From f4a455116457a66f87f793dbf5cadd783b3167ae Mon Sep 17 00:00:00 2001 From: Fokko Driesprong Date: Fri, 22 Jul 2022 17:14:34 +0200 Subject: [PATCH 139/642] Python: Bump dependencies to the latest version (#5325) --- poetry.lock | 270 ++++++++++++++++++---------------------------------- 1 file changed, 93 insertions(+), 177 deletions(-) diff --git a/poetry.lock b/poetry.lock index f2fc0d9cff..7103cb6fbf 100644 --- a/poetry.lock +++ b/poetry.lock @@ -1,6 +1,6 @@ [[package]] name = "atomicwrites" -version = "1.4.0" +version = "1.4.1" description = "Atomic file writes." category = "dev" optional = false @@ -22,7 +22,7 @@ tests_no_zope = ["coverage[toml] (>=5.0.2)", "hypothesis", "pympler", "pytest (> [[package]] name = "cffi" -version = "1.15.0" +version = "1.15.1" description = "Foreign Function Interface for Python calling C code." category = "main" optional = true @@ -49,7 +49,7 @@ python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*" [[package]] name = "coverage" -version = "6.4.1" +version = "6.4.2" description = "Code coverage measurement for Python" category = "dev" optional = false @@ -63,7 +63,7 @@ toml = ["tomli"] [[package]] name = "distlib" -version = "0.3.4" +version = "0.3.5" description = "Distribution utilities" category = "dev" optional = false @@ -71,15 +71,15 @@ python-versions = "*" [[package]] name = "docutils" -version = "0.18.1" +version = "0.19" description = "Docutils -- Python Documentation Utilities" category = "dev" optional = false -python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*" +python-versions = ">=3.7" [[package]] name = "fastavro" -version = "1.5.2" +version = "1.5.3" description = "Fast read/write of AVRO files" category = "dev" optional = false @@ -105,7 +105,7 @@ testing = ["covdefaults (>=1.2.0)", "coverage (>=4)", "pytest (>=4)", "pytest-co [[package]] name = "identify" -version = "2.5.1" +version = "2.5.2" description = "File identification library for Python" category = "dev" optional = false @@ -156,7 +156,7 @@ python-versions = ">=2.7,!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,!=3.4.*,!=3.5.*,!=3.6.* [[package]] name = "numpy" -version = "1.23.0" +version = "1.23.1" description = "NumPy is the fundamental package for array computing with Python." category = "main" optional = true @@ -210,7 +210,7 @@ testing = ["pytest", "pytest-benchmark"] [[package]] name = "pre-commit" -version = "2.19.0" +version = "2.20.0" description = "A framework for managing and maintaining multi-language pre-commit hooks." category = "dev" optional = false @@ -357,7 +357,7 @@ python-versions = ">=3.7" [[package]] name = "typing-extensions" -version = "4.2.0" +version = "4.3.0" description = "Backported and Experimental Type Hints for Python 3.7+" category = "main" optional = false @@ -365,7 +365,7 @@ python-versions = ">=3.7" [[package]] name = "virtualenv" -version = "20.15.0" +version = "20.15.1" description = "Virtual Python Environment builder" category = "dev" optional = false @@ -383,15 +383,15 @@ testing = ["coverage (>=4)", "coverage-enable-subprocess (>=1)", "flaky (>=3)", [[package]] name = "zipp" -version = "3.8.0" +version = "3.8.1" description = "Backport of pathlib-compatible object wrapper for zip files" category = "dev" optional = false python-versions = ">=3.7" [package.extras] -docs = ["sphinx", "jaraco.packaging (>=9)", "rst.linker (>=1.9)"] -testing = ["pytest (>=6)", "pytest-checkdocs (>=2.4)", "pytest-flake8", "pytest-cov", "pytest-enabler (>=1.0.1)", "jaraco.itertools", "func-timeout", "pytest-black (>=0.3.7)", "pytest-mypy (>=0.9.1)"] +docs = ["sphinx", "jaraco.packaging (>=9)", "rst.linker (>=1.9)", "jaraco.tidelift (>=1.4)"] +testing = ["pytest (>=6)", "pytest-checkdocs (>=2.4)", "pytest-flake8", "pytest-cov", "pytest-enabler (>=1.3)", "jaraco.itertools", "func-timeout", "pytest-black (>=0.3.7)", "pytest-mypy (>=0.9.1)"] [[package]] name = "zstandard" @@ -418,65 +418,76 @@ python-versions = "^3.8" content-hash = "c60cc02a89d6d22566af086e4f94dcf12662a59d0a9ceacbfbd90c65b5b34308" [metadata.files] -atomicwrites = [ - {file = "atomicwrites-1.4.0-py2.py3-none-any.whl", hash = "sha256:6d1784dea7c0c8d4a5172b6c620f40b6e4cbfdf96d783691f2e1302a7b88e197"}, - {file = "atomicwrites-1.4.0.tar.gz", hash = "sha256:ae70396ad1a434f9c7046fd2dd196fc04b12f9e91ffb859164193be8b6168a7a"}, -] +atomicwrites = [] attrs = [ {file = "attrs-21.4.0-py2.py3-none-any.whl", hash = "sha256:2d27e3784d7a565d36ab851fe94887c5eccd6a463168875832a1be79c82828b4"}, {file = "attrs-21.4.0.tar.gz", hash = "sha256:626ba8234211db98e869df76230a137c4c40a12d72445c45d5f5b716f076e2fd"}, ] cffi = [ - {file = "cffi-1.15.0-cp27-cp27m-macosx_10_9_x86_64.whl", hash = "sha256:c2502a1a03b6312837279c8c1bd3ebedf6c12c4228ddbad40912d671ccc8a962"}, - {file = "cffi-1.15.0-cp27-cp27m-manylinux1_i686.whl", hash = "sha256:23cfe892bd5dd8941608f93348c0737e369e51c100d03718f108bf1add7bd6d0"}, - {file = "cffi-1.15.0-cp27-cp27m-manylinux1_x86_64.whl", hash = "sha256:41d45de54cd277a7878919867c0f08b0cf817605e4eb94093e7516505d3c8d14"}, - {file = "cffi-1.15.0-cp27-cp27m-win32.whl", hash = "sha256:4a306fa632e8f0928956a41fa8e1d6243c71e7eb59ffbd165fc0b41e316b2474"}, - {file = "cffi-1.15.0-cp27-cp27m-win_amd64.whl", hash = "sha256:e7022a66d9b55e93e1a845d8c9eba2a1bebd4966cd8bfc25d9cd07d515b33fa6"}, - {file = "cffi-1.15.0-cp27-cp27mu-manylinux1_i686.whl", hash = "sha256:14cd121ea63ecdae71efa69c15c5543a4b5fbcd0bbe2aad864baca0063cecf27"}, - {file = "cffi-1.15.0-cp27-cp27mu-manylinux1_x86_64.whl", hash = "sha256:d4d692a89c5cf08a8557fdeb329b82e7bf609aadfaed6c0d79f5a449a3c7c023"}, - {file = "cffi-1.15.0-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:0104fb5ae2391d46a4cb082abdd5c69ea4eab79d8d44eaaf79f1b1fd806ee4c2"}, - {file = "cffi-1.15.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:91ec59c33514b7c7559a6acda53bbfe1b283949c34fe7440bcf917f96ac0723e"}, - {file = "cffi-1.15.0-cp310-cp310-manylinux_2_12_i686.manylinux2010_i686.whl", hash = "sha256:f5c7150ad32ba43a07c4479f40241756145a1f03b43480e058cfd862bf5041c7"}, - {file = "cffi-1.15.0-cp310-cp310-manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:00c878c90cb53ccfaae6b8bc18ad05d2036553e6d9d1d9dbcf323bbe83854ca3"}, - {file = "cffi-1.15.0-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:abb9a20a72ac4e0fdb50dae135ba5e77880518e742077ced47eb1499e29a443c"}, - {file = "cffi-1.15.0-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:a5263e363c27b653a90078143adb3d076c1a748ec9ecc78ea2fb916f9b861962"}, - {file = "cffi-1.15.0-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:f54a64f8b0c8ff0b64d18aa76675262e1700f3995182267998c31ae974fbc382"}, - {file = "cffi-1.15.0-cp310-cp310-win32.whl", hash = "sha256:c21c9e3896c23007803a875460fb786118f0cdd4434359577ea25eb556e34c55"}, - {file = "cffi-1.15.0-cp310-cp310-win_amd64.whl", hash = "sha256:5e069f72d497312b24fcc02073d70cb989045d1c91cbd53979366077959933e0"}, - {file = "cffi-1.15.0-cp36-cp36m-macosx_10_9_x86_64.whl", hash = "sha256:64d4ec9f448dfe041705426000cc13e34e6e5bb13736e9fd62e34a0b0c41566e"}, - {file = "cffi-1.15.0-cp36-cp36m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:2756c88cbb94231c7a147402476be2c4df2f6078099a6f4a480d239a8817ae39"}, - {file = "cffi-1.15.0-cp36-cp36m-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:3b96a311ac60a3f6be21d2572e46ce67f09abcf4d09344c49274eb9e0bf345fc"}, - {file = "cffi-1.15.0-cp36-cp36m-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:75e4024375654472cc27e91cbe9eaa08567f7fbdf822638be2814ce059f58032"}, - {file = "cffi-1.15.0-cp36-cp36m-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:59888172256cac5629e60e72e86598027aca6bf01fa2465bdb676d37636573e8"}, - {file = "cffi-1.15.0-cp36-cp36m-manylinux_2_5_x86_64.manylinux1_x86_64.whl", hash = "sha256:27c219baf94952ae9d50ec19651a687b826792055353d07648a5695413e0c605"}, - {file = "cffi-1.15.0-cp36-cp36m-win32.whl", hash = "sha256:4958391dbd6249d7ad855b9ca88fae690783a6be9e86df65865058ed81fc860e"}, - {file = "cffi-1.15.0-cp36-cp36m-win_amd64.whl", hash = "sha256:f6f824dc3bce0edab5f427efcfb1d63ee75b6fcb7282900ccaf925be84efb0fc"}, - {file = "cffi-1.15.0-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:06c48159c1abed75c2e721b1715c379fa3200c7784271b3c46df01383b593636"}, - {file = "cffi-1.15.0-cp37-cp37m-manylinux_2_12_i686.manylinux2010_i686.whl", hash = "sha256:c2051981a968d7de9dd2d7b87bcb9c939c74a34626a6e2f8181455dd49ed69e4"}, - {file = "cffi-1.15.0-cp37-cp37m-manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:fd8a250edc26254fe5b33be00402e6d287f562b6a5b2152dec302fa15bb3e997"}, - {file = "cffi-1.15.0-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:91d77d2a782be4274da750752bb1650a97bfd8f291022b379bb8e01c66b4e96b"}, - {file = "cffi-1.15.0-cp37-cp37m-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:45db3a33139e9c8f7c09234b5784a5e33d31fd6907800b316decad50af323ff2"}, - {file = "cffi-1.15.0-cp37-cp37m-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:263cc3d821c4ab2213cbe8cd8b355a7f72a8324577dc865ef98487c1aeee2bc7"}, - {file = "cffi-1.15.0-cp37-cp37m-win32.whl", hash = "sha256:17771976e82e9f94976180f76468546834d22a7cc404b17c22df2a2c81db0c66"}, - {file = "cffi-1.15.0-cp37-cp37m-win_amd64.whl", hash = "sha256:3415c89f9204ee60cd09b235810be700e993e343a408693e80ce7f6a40108029"}, - {file = "cffi-1.15.0-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:4238e6dab5d6a8ba812de994bbb0a79bddbdf80994e4ce802b6f6f3142fcc880"}, - {file = "cffi-1.15.0-cp38-cp38-manylinux_2_12_i686.manylinux2010_i686.whl", hash = "sha256:0808014eb713677ec1292301ea4c81ad277b6cdf2fdd90fd540af98c0b101d20"}, - {file = "cffi-1.15.0-cp38-cp38-manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:57e9ac9ccc3101fac9d6014fba037473e4358ef4e89f8e181f8951a2c0162024"}, - {file = "cffi-1.15.0-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:8b6c2ea03845c9f501ed1313e78de148cd3f6cad741a75d43a29b43da27f2e1e"}, - {file = "cffi-1.15.0-cp38-cp38-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:10dffb601ccfb65262a27233ac273d552ddc4d8ae1bf93b21c94b8511bffe728"}, - {file = "cffi-1.15.0-cp38-cp38-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:786902fb9ba7433aae840e0ed609f45c7bcd4e225ebb9c753aa39725bb3e6ad6"}, - {file = "cffi-1.15.0-cp38-cp38-win32.whl", hash = "sha256:da5db4e883f1ce37f55c667e5c0de439df76ac4cb55964655906306918e7363c"}, - {file = "cffi-1.15.0-cp38-cp38-win_amd64.whl", hash = "sha256:181dee03b1170ff1969489acf1c26533710231c58f95534e3edac87fff06c443"}, - {file = "cffi-1.15.0-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:45e8636704eacc432a206ac7345a5d3d2c62d95a507ec70d62f23cd91770482a"}, - {file = "cffi-1.15.0-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:31fb708d9d7c3f49a60f04cf5b119aeefe5644daba1cd2a0fe389b674fd1de37"}, - {file = "cffi-1.15.0-cp39-cp39-manylinux_2_12_i686.manylinux2010_i686.whl", hash = "sha256:6dc2737a3674b3e344847c8686cf29e500584ccad76204efea14f451d4cc669a"}, - {file = "cffi-1.15.0-cp39-cp39-manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:74fdfdbfdc48d3f47148976f49fab3251e550a8720bebc99bf1483f5bfb5db3e"}, - {file = "cffi-1.15.0-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:ffaa5c925128e29efbde7301d8ecaf35c8c60ffbcd6a1ffd3a552177c8e5e796"}, - {file = "cffi-1.15.0-cp39-cp39-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:3f7d084648d77af029acb79a0ff49a0ad7e9d09057a9bf46596dac9514dc07df"}, - {file = "cffi-1.15.0-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:ef1f279350da2c586a69d32fc8733092fd32cc8ac95139a00377841f59a3f8d8"}, - {file = "cffi-1.15.0-cp39-cp39-win32.whl", hash = "sha256:2a23af14f408d53d5e6cd4e3d9a24ff9e05906ad574822a10563efcef137979a"}, - {file = "cffi-1.15.0-cp39-cp39-win_amd64.whl", hash = "sha256:3773c4d81e6e818df2efbc7dd77325ca0dcb688116050fb2b3011218eda36139"}, - {file = "cffi-1.15.0.tar.gz", hash = "sha256:920f0d66a896c2d99f0adbb391f990a84091179542c205fa53ce5787aff87954"}, + {file = "cffi-1.15.1-cp27-cp27m-macosx_10_9_x86_64.whl", hash = "sha256:a66d3508133af6e8548451b25058d5812812ec3798c886bf38ed24a98216fab2"}, + {file = "cffi-1.15.1-cp27-cp27m-manylinux1_i686.whl", hash = "sha256:470c103ae716238bbe698d67ad020e1db9d9dba34fa5a899b5e21577e6d52ed2"}, + {file = "cffi-1.15.1-cp27-cp27m-manylinux1_x86_64.whl", hash = "sha256:9ad5db27f9cabae298d151c85cf2bad1d359a1b9c686a275df03385758e2f914"}, + {file = "cffi-1.15.1-cp27-cp27m-win32.whl", hash = "sha256:b3bbeb01c2b273cca1e1e0c5df57f12dce9a4dd331b4fa1635b8bec26350bde3"}, + {file = "cffi-1.15.1-cp27-cp27m-win_amd64.whl", hash = "sha256:e00b098126fd45523dd056d2efba6c5a63b71ffe9f2bbe1a4fe1716e1d0c331e"}, + {file = "cffi-1.15.1-cp27-cp27mu-manylinux1_i686.whl", hash = "sha256:d61f4695e6c866a23a21acab0509af1cdfd2c013cf256bbf5b6b5e2695827162"}, + {file = "cffi-1.15.1-cp27-cp27mu-manylinux1_x86_64.whl", hash = "sha256:ed9cb427ba5504c1dc15ede7d516b84757c3e3d7868ccc85121d9310d27eed0b"}, + {file = "cffi-1.15.1-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:39d39875251ca8f612b6f33e6b1195af86d1b3e60086068be9cc053aa4376e21"}, + {file = "cffi-1.15.1-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:285d29981935eb726a4399badae8f0ffdff4f5050eaa6d0cfc3f64b857b77185"}, + {file = "cffi-1.15.1-cp310-cp310-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:3eb6971dcff08619f8d91607cfc726518b6fa2a9eba42856be181c6d0d9515fd"}, + {file = "cffi-1.15.1-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:21157295583fe8943475029ed5abdcf71eb3911894724e360acff1d61c1d54bc"}, + {file = "cffi-1.15.1-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:5635bd9cb9731e6d4a1132a498dd34f764034a8ce60cef4f5319c0541159392f"}, + {file = "cffi-1.15.1-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:2012c72d854c2d03e45d06ae57f40d78e5770d252f195b93f581acf3ba44496e"}, + {file = "cffi-1.15.1-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:dd86c085fae2efd48ac91dd7ccffcfc0571387fe1193d33b6394db7ef31fe2a4"}, + {file = "cffi-1.15.1-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:fa6693661a4c91757f4412306191b6dc88c1703f780c8234035eac011922bc01"}, + {file = "cffi-1.15.1-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:59c0b02d0a6c384d453fece7566d1c7e6b7bae4fc5874ef2ef46d56776d61c9e"}, + {file = "cffi-1.15.1-cp310-cp310-win32.whl", hash = "sha256:cba9d6b9a7d64d4bd46167096fc9d2f835e25d7e4c121fb2ddfc6528fb0413b2"}, + {file = "cffi-1.15.1-cp310-cp310-win_amd64.whl", hash = "sha256:ce4bcc037df4fc5e3d184794f27bdaab018943698f4ca31630bc7f84a7b69c6d"}, + {file = "cffi-1.15.1-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:3d08afd128ddaa624a48cf2b859afef385b720bb4b43df214f85616922e6a5ac"}, + {file = "cffi-1.15.1-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:3799aecf2e17cf585d977b780ce79ff0dc9b78d799fc694221ce814c2c19db83"}, + {file = "cffi-1.15.1-cp311-cp311-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:a591fe9e525846e4d154205572a029f653ada1a78b93697f3b5a8f1f2bc055b9"}, + {file = "cffi-1.15.1-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:3548db281cd7d2561c9ad9984681c95f7b0e38881201e157833a2342c30d5e8c"}, + {file = "cffi-1.15.1-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:91fc98adde3d7881af9b59ed0294046f3806221863722ba7d8d120c575314325"}, + {file = "cffi-1.15.1-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:94411f22c3985acaec6f83c6df553f2dbe17b698cc7f8ae751ff2237d96b9e3c"}, + {file = "cffi-1.15.1-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:03425bdae262c76aad70202debd780501fabeaca237cdfddc008987c0e0f59ef"}, + {file = "cffi-1.15.1-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:cc4d65aeeaa04136a12677d3dd0b1c0c94dc43abac5860ab33cceb42b801c1e8"}, + {file = "cffi-1.15.1-cp311-cp311-win32.whl", hash = "sha256:a0f100c8912c114ff53e1202d0078b425bee3649ae34d7b070e9697f93c5d52d"}, + {file = "cffi-1.15.1-cp311-cp311-win_amd64.whl", hash = "sha256:04ed324bda3cda42b9b695d51bb7d54b680b9719cfab04227cdd1e04e5de3104"}, + {file = "cffi-1.15.1-cp36-cp36m-macosx_10_9_x86_64.whl", hash = "sha256:50a74364d85fd319352182ef59c5c790484a336f6db772c1a9231f1c3ed0cbd7"}, + {file = "cffi-1.15.1-cp36-cp36m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:e263d77ee3dd201c3a142934a086a4450861778baaeeb45db4591ef65550b0a6"}, + {file = "cffi-1.15.1-cp36-cp36m-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:cec7d9412a9102bdc577382c3929b337320c4c4c4849f2c5cdd14d7368c5562d"}, + {file = "cffi-1.15.1-cp36-cp36m-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:4289fc34b2f5316fbb762d75362931e351941fa95fa18789191b33fc4cf9504a"}, + {file = "cffi-1.15.1-cp36-cp36m-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:173379135477dc8cac4bc58f45db08ab45d228b3363adb7af79436135d028405"}, + {file = "cffi-1.15.1-cp36-cp36m-manylinux_2_5_x86_64.manylinux1_x86_64.whl", hash = "sha256:6975a3fac6bc83c4a65c9f9fcab9e47019a11d3d2cf7f3c0d03431bf145a941e"}, + {file = "cffi-1.15.1-cp36-cp36m-win32.whl", hash = "sha256:2470043b93ff09bf8fb1d46d1cb756ce6132c54826661a32d4e4d132e1977adf"}, + {file = "cffi-1.15.1-cp36-cp36m-win_amd64.whl", hash = "sha256:30d78fbc8ebf9c92c9b7823ee18eb92f2e6ef79b45ac84db507f52fbe3ec4497"}, + {file = "cffi-1.15.1-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:198caafb44239b60e252492445da556afafc7d1e3ab7a1fb3f0584ef6d742375"}, + {file = "cffi-1.15.1-cp37-cp37m-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:5ef34d190326c3b1f822a5b7a45f6c4535e2f47ed06fec77d3d799c450b2651e"}, + {file = "cffi-1.15.1-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:8102eaf27e1e448db915d08afa8b41d6c7ca7a04b7d73af6514df10a3e74bd82"}, + {file = "cffi-1.15.1-cp37-cp37m-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:5df2768244d19ab7f60546d0c7c63ce1581f7af8b5de3eb3004b9b6fc8a9f84b"}, + {file = "cffi-1.15.1-cp37-cp37m-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:a8c4917bd7ad33e8eb21e9a5bbba979b49d9a97acb3a803092cbc1133e20343c"}, + {file = "cffi-1.15.1-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:0e2642fe3142e4cc4af0799748233ad6da94c62a8bec3a6648bf8ee68b1c7426"}, + {file = "cffi-1.15.1-cp37-cp37m-win32.whl", hash = "sha256:e229a521186c75c8ad9490854fd8bbdd9a0c9aa3a524326b55be83b54d4e0ad9"}, + {file = "cffi-1.15.1-cp37-cp37m-win_amd64.whl", hash = "sha256:a0b71b1b8fbf2b96e41c4d990244165e2c9be83d54962a9a1d118fd8657d2045"}, + {file = "cffi-1.15.1-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:320dab6e7cb2eacdf0e658569d2575c4dad258c0fcc794f46215e1e39f90f2c3"}, + {file = "cffi-1.15.1-cp38-cp38-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:1e74c6b51a9ed6589199c787bf5f9875612ca4a8a0785fb2d4a84429badaf22a"}, + {file = "cffi-1.15.1-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a5c84c68147988265e60416b57fc83425a78058853509c1b0629c180094904a5"}, + {file = "cffi-1.15.1-cp38-cp38-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:3b926aa83d1edb5aa5b427b4053dc420ec295a08e40911296b9eb1b6170f6cca"}, + {file = "cffi-1.15.1-cp38-cp38-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:87c450779d0914f2861b8526e035c5e6da0a3199d8f1add1a665e1cbc6fc6d02"}, + {file = "cffi-1.15.1-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:4f2c9f67e9821cad2e5f480bc8d83b8742896f1242dba247911072d4fa94c192"}, + {file = "cffi-1.15.1-cp38-cp38-win32.whl", hash = "sha256:8b7ee99e510d7b66cdb6c593f21c043c248537a32e0bedf02e01e9553a172314"}, + {file = "cffi-1.15.1-cp38-cp38-win_amd64.whl", hash = "sha256:00a9ed42e88df81ffae7a8ab6d9356b371399b91dbdf0c3cb1e84c03a13aceb5"}, + {file = "cffi-1.15.1-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:54a2db7b78338edd780e7ef7f9f6c442500fb0d41a5a4ea24fff1c929d5af585"}, + {file = "cffi-1.15.1-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:fcd131dd944808b5bdb38e6f5b53013c5aa4f334c5cad0c72742f6eba4b73db0"}, + {file = "cffi-1.15.1-cp39-cp39-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:7473e861101c9e72452f9bf8acb984947aa1661a7704553a9f6e4baa5ba64415"}, + {file = "cffi-1.15.1-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:6c9a799e985904922a4d207a94eae35c78ebae90e128f0c4e521ce339396be9d"}, + {file = "cffi-1.15.1-cp39-cp39-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:3bcde07039e586f91b45c88f8583ea7cf7a0770df3a1649627bf598332cb6984"}, + {file = "cffi-1.15.1-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:33ab79603146aace82c2427da5ca6e58f2b3f2fb5da893ceac0c42218a40be35"}, + {file = "cffi-1.15.1-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:5d598b938678ebf3c67377cdd45e09d431369c3b1a5b331058c338e201f12b27"}, + {file = "cffi-1.15.1-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:db0fbb9c62743ce59a9ff687eb5f4afbe77e5e8403d6697f7446e5f609976f76"}, + {file = "cffi-1.15.1-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:98d85c6a2bef81588d9227dde12db8a7f47f639f4a17c9ae08e773aa9c697bf3"}, + {file = "cffi-1.15.1-cp39-cp39-win32.whl", hash = "sha256:40f4774f5a9d4f5e344f31a32b5096977b5d48560c5592e2f3d2c4374bd543ee"}, + {file = "cffi-1.15.1-cp39-cp39-win_amd64.whl", hash = "sha256:70df4e3b545a17496c9b3f41f5115e69a4f2e77e94e1d2a8e1070bc0c38c8a3c"}, + {file = "cffi-1.15.1.tar.gz", hash = "sha256:d400bfb9a37b1351253cb402671cea7e89bdecc294e8016a707f6d1d8ac934f9"}, ] cfgv = [ {file = "cfgv-3.3.1-py2.py3-none-any.whl", hash = "sha256:c6a0883f3917a037485059700b9e75da2464e6c27051014ad85ba6aaa5884426"}, @@ -486,84 +497,18 @@ colorama = [ {file = "colorama-0.4.5-py2.py3-none-any.whl", hash = "sha256:854bf444933e37f5824ae7bfc1e98d5bce2ebe4160d46b5edf346a89358e99da"}, {file = "colorama-0.4.5.tar.gz", hash = "sha256:e6c6b4334fc50988a639d9b98aa429a0b57da6e17b9a44f0451f930b6967b7a4"}, ] -coverage = [ - {file = "coverage-6.4.1-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:f1d5aa2703e1dab4ae6cf416eb0095304f49d004c39e9db1d86f57924f43006b"}, - {file = "coverage-6.4.1-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:4ce1b258493cbf8aec43e9b50d89982346b98e9ffdfaae8ae5793bc112fb0068"}, - {file = "coverage-6.4.1-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:83c4e737f60c6936460c5be330d296dd5b48b3963f48634c53b3f7deb0f34ec4"}, - {file = "coverage-6.4.1-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:84e65ef149028516c6d64461b95a8dbcfce95cfd5b9eb634320596173332ea84"}, - {file = "coverage-6.4.1-cp310-cp310-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:f69718750eaae75efe506406c490d6fc5a6161d047206cc63ce25527e8a3adad"}, - {file = "coverage-6.4.1-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:e57816f8ffe46b1df8f12e1b348f06d164fd5219beba7d9433ba79608ef011cc"}, - {file = "coverage-6.4.1-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:01c5615d13f3dd3aa8543afc069e5319cfa0c7d712f6e04b920431e5c564a749"}, - {file = "coverage-6.4.1-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:75ab269400706fab15981fd4bd5080c56bd5cc07c3bccb86aab5e1d5a88dc8f4"}, - {file = "coverage-6.4.1-cp310-cp310-win32.whl", hash = "sha256:a7f3049243783df2e6cc6deafc49ea123522b59f464831476d3d1448e30d72df"}, - {file = "coverage-6.4.1-cp310-cp310-win_amd64.whl", hash = "sha256:ee2ddcac99b2d2aec413e36d7a429ae9ebcadf912946b13ffa88e7d4c9b712d6"}, - {file = "coverage-6.4.1-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:fb73e0011b8793c053bfa85e53129ba5f0250fdc0392c1591fd35d915ec75c46"}, - {file = "coverage-6.4.1-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:106c16dfe494de3193ec55cac9640dd039b66e196e4641fa8ac396181578b982"}, - {file = "coverage-6.4.1-cp37-cp37m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:87f4f3df85aa39da00fd3ec4b5abeb7407e82b68c7c5ad181308b0e2526da5d4"}, - {file = "coverage-6.4.1-cp37-cp37m-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:961e2fb0680b4f5ad63234e0bf55dfb90d302740ae9c7ed0120677a94a1590cb"}, - {file = "coverage-6.4.1-cp37-cp37m-musllinux_1_1_aarch64.whl", hash = "sha256:cec3a0f75c8f1031825e19cd86ee787e87cf03e4fd2865c79c057092e69e3a3b"}, - {file = "coverage-6.4.1-cp37-cp37m-musllinux_1_1_i686.whl", hash = "sha256:129cd05ba6f0d08a766d942a9ed4b29283aff7b2cccf5b7ce279d50796860bb3"}, - {file = "coverage-6.4.1-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:bf5601c33213d3cb19d17a796f8a14a9eaa5e87629a53979a5981e3e3ae166f6"}, - {file = "coverage-6.4.1-cp37-cp37m-win32.whl", hash = "sha256:269eaa2c20a13a5bf17558d4dc91a8d078c4fa1872f25303dddcbba3a813085e"}, - {file = "coverage-6.4.1-cp37-cp37m-win_amd64.whl", hash = "sha256:f02cbbf8119db68455b9d763f2f8737bb7db7e43720afa07d8eb1604e5c5ae28"}, - {file = "coverage-6.4.1-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:ffa9297c3a453fba4717d06df579af42ab9a28022444cae7fa605af4df612d54"}, - {file = "coverage-6.4.1-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:145f296d00441ca703a659e8f3eb48ae39fb083baba2d7ce4482fb2723e050d9"}, - {file = "coverage-6.4.1-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:d67d44996140af8b84284e5e7d398e589574b376fb4de8ccd28d82ad8e3bea13"}, - {file = "coverage-6.4.1-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:2bd9a6fc18aab8d2e18f89b7ff91c0f34ff4d5e0ba0b33e989b3cd4194c81fd9"}, - {file = "coverage-6.4.1-cp38-cp38-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:3384f2a3652cef289e38100f2d037956194a837221edd520a7ee5b42d00cc605"}, - {file = "coverage-6.4.1-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:9b3e07152b4563722be523e8cd0b209e0d1a373022cfbde395ebb6575bf6790d"}, - {file = "coverage-6.4.1-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:1480ff858b4113db2718848d7b2d1b75bc79895a9c22e76a221b9d8d62496428"}, - {file = "coverage-6.4.1-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:865d69ae811a392f4d06bde506d531f6a28a00af36f5c8649684a9e5e4a85c83"}, - {file = "coverage-6.4.1-cp38-cp38-win32.whl", hash = "sha256:664a47ce62fe4bef9e2d2c430306e1428ecea207ffd68649e3b942fa8ea83b0b"}, - {file = "coverage-6.4.1-cp38-cp38-win_amd64.whl", hash = "sha256:26dff09fb0d82693ba9e6231248641d60ba606150d02ed45110f9ec26404ed1c"}, - {file = "coverage-6.4.1-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:d9c80df769f5ec05ad21ea34be7458d1dc51ff1fb4b2219e77fe24edf462d6df"}, - {file = "coverage-6.4.1-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:39ee53946bf009788108b4dd2894bf1349b4e0ca18c2016ffa7d26ce46b8f10d"}, - {file = "coverage-6.4.1-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:f5b66caa62922531059bc5ac04f836860412f7f88d38a476eda0a6f11d4724f4"}, - {file = "coverage-6.4.1-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:fd180ed867e289964404051a958f7cccabdeed423f91a899829264bb7974d3d3"}, - {file = "coverage-6.4.1-cp39-cp39-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:84631e81dd053e8a0d4967cedab6db94345f1c36107c71698f746cb2636c63e3"}, - {file = "coverage-6.4.1-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:8c08da0bd238f2970230c2a0d28ff0e99961598cb2e810245d7fc5afcf1254e8"}, - {file = "coverage-6.4.1-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:d42c549a8f41dc103a8004b9f0c433e2086add8a719da00e246e17cbe4056f72"}, - {file = "coverage-6.4.1-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:309ce4a522ed5fca432af4ebe0f32b21d6d7ccbb0f5fcc99290e71feba67c264"}, - {file = "coverage-6.4.1-cp39-cp39-win32.whl", hash = "sha256:fdb6f7bd51c2d1714cea40718f6149ad9be6a2ee7d93b19e9f00934c0f2a74d9"}, - {file = "coverage-6.4.1-cp39-cp39-win_amd64.whl", hash = "sha256:342d4aefd1c3e7f620a13f4fe563154d808b69cccef415415aece4c786665397"}, - {file = "coverage-6.4.1-pp36.pp37.pp38-none-any.whl", hash = "sha256:4803e7ccf93230accb928f3a68f00ffa80a88213af98ed338a57ad021ef06815"}, - {file = "coverage-6.4.1.tar.gz", hash = "sha256:4321f075095a096e70aff1d002030ee612b65a205a0a0f5b815280d5dc58100c"}, -] -distlib = [ - {file = "distlib-0.3.4-py2.py3-none-any.whl", hash = "sha256:6564fe0a8f51e734df6333d08b8b94d4ea8ee6b99b5ed50613f731fd4089f34b"}, - {file = "distlib-0.3.4.zip", hash = "sha256:e4b58818180336dc9c529bfb9a0b58728ffc09ad92027a3f30b7cd91e3458579"}, -] +coverage = [] +distlib = [] docutils = [ - {file = "docutils-0.18.1-py2.py3-none-any.whl", hash = "sha256:23010f129180089fbcd3bc08cfefccb3b890b0050e1ca00c867036e9d161b98c"}, - {file = "docutils-0.18.1.tar.gz", hash = "sha256:679987caf361a7539d76e584cbeddc311e3aee937877c87346f31debc63e9d06"}, -] -fastavro = [ - {file = "fastavro-1.5.2-cp310-cp310-macosx_10_14_x86_64.whl", hash = "sha256:4c447c81bf9002aa0258bb8e208698744b4e22362548d5423f718c92c22ce572"}, - {file = "fastavro-1.5.2-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:5de3473c759c884f88c0e4e078538b9199416183bd4f49a6baf39468d7c0900c"}, - {file = "fastavro-1.5.2-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:7e4ccdf1142a01b5a10b9451cc7f2e82b0bf4d2593921b9aa8d66a4758f42ee7"}, - {file = "fastavro-1.5.2-cp310-cp310-win_amd64.whl", hash = "sha256:cc162420a6619debb8aabffc961e8f9aa442dfb0246115c421a0fa108c5e0796"}, - {file = "fastavro-1.5.2-cp37-cp37m-macosx_10_15_x86_64.whl", hash = "sha256:8bfd03691fb57771b0f4d4a241bd5d5f2e622cbd55e88ab3718a58b7e8e00bdb"}, - {file = "fastavro-1.5.2-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:3600c5b8ccc0c5dfab5927486d71ba79ac155b0eeecce517d86e191a5186110c"}, - {file = "fastavro-1.5.2-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:2c4777ebd022aecb05829b52359351746fb8ac1f3958385e524addde22d212c4"}, - {file = "fastavro-1.5.2-cp37-cp37m-win_amd64.whl", hash = "sha256:faa8c9cd4920c2e4e4131dae2e5fb050cc64533c22384d5dbba55c6a6463771e"}, - {file = "fastavro-1.5.2-cp38-cp38-macosx_10_15_x86_64.whl", hash = "sha256:90517b77f9d8362efe8deaa3da06673f0ef961e973bd9daa49aba998d8ca849e"}, - {file = "fastavro-1.5.2-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:746d97e41d24c838c70a44cb4e6a7e4109f20923fda5a6de18482e31db06ccdd"}, - {file = "fastavro-1.5.2-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:49398a773fab076725abf8926525092c455f2cb2ae58127d69fcc132d970fd6e"}, - {file = "fastavro-1.5.2-cp38-cp38-win_amd64.whl", hash = "sha256:45b7bfabf42bb3882071599eef49228039febbb1b8801b3a65bf32d717eea075"}, - {file = "fastavro-1.5.2-cp39-cp39-macosx_10_15_x86_64.whl", hash = "sha256:e6f23f1f4e2726408a35d2f2b279c29da41a989351a0f8c28ec62348f01be977"}, - {file = "fastavro-1.5.2-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:7699d1283062f11a3bbfb705f1f2d3fae0f7a1238ae3ca6234c7c9208b0a11a6"}, - {file = "fastavro-1.5.2-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:778d134d144d025f72ed98f60b5a63d363f473c5b19a32f4173e2e11e7f30be9"}, - {file = "fastavro-1.5.2-cp39-cp39-win_amd64.whl", hash = "sha256:db4f1a65dfa6d26b4be74d9aacd13467a17928ada2c15096b401288671689e44"}, - {file = "fastavro-1.5.2.tar.gz", hash = "sha256:ee7ffc6882834499fd53877a08cdadbb107821a559bb78278dbbb3685ae92788"}, + {file = "docutils-0.19-py3-none-any.whl", hash = "sha256:5e1de4d849fee02c63b040a4a3fd567f4ab104defd8a5511fbbc24a8a017efbc"}, + {file = "docutils-0.19.tar.gz", hash = "sha256:33995a6753c30b7f577febfc2c50411fec6aac7f7ffeb7c4cfe5991072dcf9e6"}, ] +fastavro = [] filelock = [ {file = "filelock-3.7.1-py3-none-any.whl", hash = "sha256:37def7b658813cda163b56fc564cdc75e86d338246458c4c28ae84cabefa2404"}, {file = "filelock-3.7.1.tar.gz", hash = "sha256:3a0fd85166ad9dbab54c9aec96737b744106dc5f15c0b09a6744a445299fcf04"}, ] -identify = [ - {file = "identify-2.5.1-py2.py3-none-any.whl", hash = "sha256:0dca2ea3e4381c435ef9c33ba100a78a9b40c0bab11189c7cf121f75815efeaa"}, - {file = "identify-2.5.1.tar.gz", hash = "sha256:3d11b16f3fe19f52039fb7e39c9c884b21cb1b586988114fbe42671f03de3e82"}, -] +identify = [] importlib-metadata = [ {file = "importlib_metadata-4.12.0-py3-none-any.whl", hash = "sha256:7401a975809ea1fdc658c3aa4f78cc2195a0e019c5cbc4c06122884e9ae80c23"}, {file = "importlib_metadata-4.12.0.tar.gz", hash = "sha256:637245b8bab2b6502fcbc752cc4b7a6f6243bb02b31c5c26156ad103d3d45670"}, @@ -604,30 +549,7 @@ nodeenv = [ {file = "nodeenv-1.7.0-py2.py3-none-any.whl", hash = "sha256:27083a7b96a25f2f5e1d8cb4b6317ee8aeda3bdd121394e5ac54e498028a042e"}, {file = "nodeenv-1.7.0.tar.gz", hash = "sha256:e0e7f7dfb85fc5394c6fe1e8fa98131a2473e04311a45afb6508f7cf1836fa2b"}, ] -numpy = [ - {file = "numpy-1.23.0-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:58bfd40eb478f54ff7a5710dd61c8097e169bc36cc68333d00a9bcd8def53b38"}, - {file = "numpy-1.23.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:196cd074c3f97c4121601790955f915187736f9cf458d3ee1f1b46aff2b1ade0"}, - {file = "numpy-1.23.0-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:f1d88ef79e0a7fa631bb2c3dda1ea46b32b1fe614e10fedd611d3d5398447f2f"}, - {file = "numpy-1.23.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:d54b3b828d618a19779a84c3ad952e96e2c2311b16384e973e671aa5be1f6187"}, - {file = "numpy-1.23.0-cp310-cp310-win32.whl", hash = "sha256:2b2da66582f3a69c8ce25ed7921dcd8010d05e59ac8d89d126a299be60421171"}, - {file = "numpy-1.23.0-cp310-cp310-win_amd64.whl", hash = "sha256:97a76604d9b0e79f59baeca16593c711fddb44936e40310f78bfef79ee9a835f"}, - {file = "numpy-1.23.0-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:d8cc87bed09de55477dba9da370c1679bd534df9baa171dd01accbb09687dac3"}, - {file = "numpy-1.23.0-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:f0f18804df7370571fb65db9b98bf1378172bd4e962482b857e612d1fec0f53e"}, - {file = "numpy-1.23.0-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:ac86f407873b952679f5f9e6c0612687e51547af0e14ddea1eedfcb22466babd"}, - {file = "numpy-1.23.0-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:ae8adff4172692ce56233db04b7ce5792186f179c415c37d539c25de7298d25d"}, - {file = "numpy-1.23.0-cp38-cp38-win32.whl", hash = "sha256:fe8b9683eb26d2c4d5db32cd29b38fdcf8381324ab48313b5b69088e0e355379"}, - {file = "numpy-1.23.0-cp38-cp38-win_amd64.whl", hash = "sha256:5043bcd71fcc458dfb8a0fc5509bbc979da0131b9d08e3d5f50fb0bbb36f169a"}, - {file = "numpy-1.23.0-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:1c29b44905af288b3919803aceb6ec7fec77406d8b08aaa2e8b9e63d0fe2f160"}, - {file = "numpy-1.23.0-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:98e8e0d8d69ff4d3fa63e6c61e8cfe2d03c29b16b58dbef1f9baa175bbed7860"}, - {file = "numpy-1.23.0-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:79a506cacf2be3a74ead5467aee97b81fca00c9c4c8b3ba16dbab488cd99ba10"}, - {file = "numpy-1.23.0-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:092f5e6025813e64ad6d1b52b519165d08c730d099c114a9247c9bb635a2a450"}, - {file = "numpy-1.23.0-cp39-cp39-win32.whl", hash = "sha256:d6ca8dabe696c2785d0c8c9b0d8a9b6e5fdbe4f922bde70d57fa1a2848134f95"}, - {file = "numpy-1.23.0-cp39-cp39-win_amd64.whl", hash = "sha256:fc431493df245f3c627c0c05c2bd134535e7929dbe2e602b80e42bf52ff760bc"}, - {file = "numpy-1.23.0-pp38-pypy38_pp73-macosx_10_9_x86_64.whl", hash = "sha256:f9c3fc2adf67762c9fe1849c859942d23f8d3e0bee7b5ed3d4a9c3eeb50a2f07"}, - {file = "numpy-1.23.0-pp38-pypy38_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:d0d2094e8f4d760500394d77b383a1b06d3663e8892cdf5df3c592f55f3bff66"}, - {file = "numpy-1.23.0-pp38-pypy38_pp73-win_amd64.whl", hash = "sha256:94b170b4fa0168cd6be4becf37cb5b127bd12a795123984385b8cd4aca9857e5"}, - {file = "numpy-1.23.0.tar.gz", hash = "sha256:bd3fa4fe2e38533d5336e1272fc4e765cabbbde144309ccee8675509d5cd7b05"}, -] +numpy = [] packaging = [ {file = "packaging-21.3-py3-none-any.whl", hash = "sha256:ef103e05f519cdc783ae24ea4e2e0f508a9c99b2d4969652eed6a2e1ea5bd522"}, {file = "packaging-21.3.tar.gz", hash = "sha256:dd47c42927d89ab911e606518907cc2d3a1f38bbd026385970643f9c5b8ecfeb"}, @@ -644,10 +566,7 @@ pluggy = [ {file = "pluggy-1.0.0-py2.py3-none-any.whl", hash = "sha256:74134bbf457f031a36d68416e1509f34bd5ccc019f0bcc952c7b909d06b37bd3"}, {file = "pluggy-1.0.0.tar.gz", hash = "sha256:4224373bacce55f955a878bf9cfa763c1e360858e330072059e10bad68531159"}, ] -pre-commit = [ - {file = "pre_commit-2.19.0-py2.py3-none-any.whl", hash = "sha256:10c62741aa5704faea2ad69cb550ca78082efe5697d6f04e5710c3c229afdd10"}, - {file = "pre_commit-2.19.0.tar.gz", hash = "sha256:4233a1e38621c87d9dda9808c6606d7e7ba0e087cd56d3fe03202a01d2919615"}, -] +pre-commit = [] py = [ {file = "py-1.11.0-py2.py3-none-any.whl", hash = "sha256:607c53218732647dff4acdfcd50cb62615cedf612e72d1724fb1a0cc6405b378"}, {file = "py-1.11.0.tar.gz", hash = "sha256:51c75c4126074b472f746a24399ad32f6053d1b34b68d2fa41e558e6f4a98719"}, @@ -835,17 +754,14 @@ tomli = [ {file = "tomli-2.0.1.tar.gz", hash = "sha256:de526c12914f0c550d15924c62d72abc48d6fe7364aa87328337a31007fe8a4f"}, ] typing-extensions = [ - {file = "typing_extensions-4.2.0-py3-none-any.whl", hash = "sha256:6657594ee297170d19f67d55c05852a874e7eb634f4f753dbd667855e07c1708"}, - {file = "typing_extensions-4.2.0.tar.gz", hash = "sha256:f1c24655a0da0d1b67f07e17a5e6b2a105894e6824b92096378bb3668ef02376"}, + {file = "typing_extensions-4.3.0-py3-none-any.whl", hash = "sha256:25642c956049920a5aa49edcdd6ab1e06d7e5d467fc00e0506c44ac86fbfca02"}, + {file = "typing_extensions-4.3.0.tar.gz", hash = "sha256:e6d2677a32f47fc7eb2795db1dd15c1f34eff616bcaf2cfb5e997f854fa1c4a6"}, ] virtualenv = [ - {file = "virtualenv-20.15.0-py2.py3-none-any.whl", hash = "sha256:804cce4de5b8a322f099897e308eecc8f6e2951f1a8e7e2b3598dff865f01336"}, - {file = "virtualenv-20.15.0.tar.gz", hash = "sha256:4c44b1d77ca81f8368e2d7414f9b20c428ad16b343ac6d226206c5b84e2b4fcc"}, -] -zipp = [ - {file = "zipp-3.8.0-py3-none-any.whl", hash = "sha256:c4f6e5bbf48e74f7a38e7cc5b0480ff42b0ae5178957d564d18932525d5cf099"}, - {file = "zipp-3.8.0.tar.gz", hash = "sha256:56bf8aadb83c24db6c4b577e13de374ccfb67da2078beba1d037c17980bf43ad"}, + {file = "virtualenv-20.15.1-py2.py3-none-any.whl", hash = "sha256:b30aefac647e86af6d82bfc944c556f8f1a9c90427b2fb4e3bfbf338cb82becf"}, + {file = "virtualenv-20.15.1.tar.gz", hash = "sha256:288171134a2ff3bfb1a2f54f119e77cd1b81c29fc1265a2356f3e8d14c7d58c4"}, ] +zipp = [] zstandard = [ {file = "zstandard-0.17.0-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:a1991cdf2e81e643b53fb8d272931d2bdf5f4e70d56a457e1ef95bde147ae627"}, {file = "zstandard-0.17.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:4768449d8d1b0785309ace288e017cc5fa42e11a52bf08c90d9c3eb3a7a73cc6"}, From 83561c4465d2a1f73d14c88dcb933c90b5246915 Mon Sep 17 00:00:00 2001 From: Fokko Driesprong Date: Fri, 22 Jul 2022 17:15:02 +0200 Subject: [PATCH 140/642] Python: Bump pre-commit plugins to the latest version (#5324) --- .pre-commit-config.yaml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index efbc6cc7f2..9a219be8cc 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -37,22 +37,22 @@ repos: - id: isort args: [ --settings-path=python/pyproject.toml ] - repo: https://github.com/pre-commit/mirrors-mypy - rev: v0.961 + rev: v0.971 hooks: - id: mypy args: [--config=python/pyproject.toml] - repo: https://github.com/hadialqattan/pycln - rev: v2.0.1 + rev: v2.0.4 hooks: - id: pycln args: [--config=python/pyproject.toml] - repo: https://github.com/asottile/pyupgrade - rev: v2.34.0 + rev: v2.37.2 hooks: - id: pyupgrade args: [--py38-plus] - repo: https://github.com/pycqa/pylint - rev: v2.14.4 + rev: v2.14.5 hooks: - id: pylint args: [ --rcfile=python/pylintrc ] From 8c1b681dda23dd927102b003a7e84cec2b6ff40e Mon Sep 17 00:00:00 2001 From: Fokko Driesprong Date: Fri, 22 Jul 2022 17:15:26 +0200 Subject: [PATCH 141/642] Python: Remove unused function (#5296) Less is more --- pyiceberg/schema.py | 4 ---- 1 file changed, 4 deletions(-) diff --git a/pyiceberg/schema.py b/pyiceberg/schema.py index db2aaae35b..ea4f4cf6c0 100644 --- a/pyiceberg/schema.py +++ b/pyiceberg/schema.py @@ -570,10 +570,6 @@ class _BuildPositionAccessors(SchemaVisitor[Dict[Position, Accessor]]): True """ - @staticmethod - def _wrap_leaves(result: Dict[Position, Accessor], position: Position = 0) -> Dict[Position, Accessor]: - return {field_id: Accessor(position, inner=inner) for field_id, inner in result.items()} - def schema(self, schema: Schema, struct_result: Dict[Position, Accessor]) -> Dict[Position, Accessor]: return struct_result From e306cc567494f120cda5b76cab4dda821d898fb8 Mon Sep 17 00:00:00 2001 From: Fokko Driesprong Date: Fri, 22 Jul 2022 19:28:33 +0200 Subject: [PATCH 142/642] Python: Map Manifest onto Pydantic class (#5298) --- pyiceberg/manifest.py | 180 ++++++++++++++++++++++ tests/utils/test_manifest.py | 283 +++++++++++++++++++++++++++++++++++ 2 files changed, 463 insertions(+) create mode 100644 pyiceberg/manifest.py create mode 100644 tests/utils/test_manifest.py diff --git a/pyiceberg/manifest.py b/pyiceberg/manifest.py new file mode 100644 index 0000000000..b1349390d9 --- /dev/null +++ b/pyiceberg/manifest.py @@ -0,0 +1,180 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +from enum import Enum +from functools import singledispatch +from typing import ( + Any, + Dict, + Iterator, + List, + Optional, + Union, +) + +from pydantic import Field + +from pyiceberg.avro.file import AvroFile +from pyiceberg.avro.reader import AvroStruct +from pyiceberg.io.base import InputFile +from pyiceberg.schema import Schema +from pyiceberg.types import ( + IcebergType, + ListType, + MapType, + PrimitiveType, + StructType, +) +from pyiceberg.utils.iceberg_base_model import IcebergBaseModel + + +class DataFileContent(int, Enum): + DATA = 0 + POSITION_DELETES = 1 + EQUALITY_DELETES = 2 + + +class ManifestContent(int, Enum): + DATA = 0 + DELETES = 1 + + +class ManifestEntryStatus(int, Enum): + EXISTING = 0 + ADDED = 1 + DELETED = 2 + + +class FileFormat(str, Enum): + AVRO = "AVRO" + PARQUET = "PARQUET" + ORC = "ORC" + + +class DataFile(IcebergBaseModel): + content: DataFileContent = Field(default=DataFileContent.DATA) + file_path: str = Field() + file_format: FileFormat = Field() + partition: Dict[str, Any] = Field() + record_count: int = Field() + file_size_in_bytes: int = Field() + block_size_in_bytes: Optional[int] = Field() + column_sizes: Optional[Dict[int, int]] = Field() + value_counts: Optional[Dict[int, int]] = Field() + null_value_counts: Optional[Dict[int, int]] = Field() + nan_value_counts: Optional[Dict[int, int]] = Field() + distinct_counts: Optional[Dict[int, int]] = Field() + lower_bounds: Optional[Dict[int, bytes]] = Field() + upper_bounds: Optional[Dict[int, bytes]] = Field() + key_metadata: Optional[bytes] = Field() + split_offsets: Optional[List[int]] = Field() + equality_ids: Optional[List[int]] = Field() + sort_order_id: Optional[int] = Field() + + +class ManifestEntry(IcebergBaseModel): + status: ManifestEntryStatus = Field() + snapshot_id: Optional[int] = Field() + sequence_number: Optional[int] = Field() + data_file: DataFile = Field() + + +class FieldSummary(IcebergBaseModel): + contains_null: bool = Field() + contains_nan: Optional[bool] = Field() + lower_bound: Optional[bytes] = Field() + upper_bound: Optional[bytes] = Field() + + +class ManifestFile(IcebergBaseModel): + manifest_path: str = Field() + manifest_length: int = Field() + partition_spec_id: int = Field() + content: ManifestContent = Field(default=ManifestContent.DATA) + sequence_number: int = Field(default=0) + min_sequence_number: int = Field(default=0) + added_snapshot_id: Optional[int] = Field() + added_data_files_count: Optional[int] = Field() + existing_data_files_count: Optional[int] = Field() + deleted_data_files_count: Optional[int] = Field() + added_rows_count: Optional[int] = Field() + existing_rows_counts: Optional[int] = Field() + deleted_rows_count: Optional[int] = Field() + partitions: Optional[List[FieldSummary]] = Field() + key_metadata: Optional[bytes] = Field() + + +def read_manifest_entry(input_file: InputFile) -> Iterator[ManifestEntry]: + with AvroFile(input_file) as reader: + schema = reader.schema + for record in reader: + dict_repr = _convert_pos_to_dict(schema, record) + yield ManifestEntry(**dict_repr) + + +def read_manifest_list(input_file: InputFile) -> Iterator[ManifestFile]: + with AvroFile(input_file) as reader: + schema = reader.schema + for record in reader: + dict_repr = _convert_pos_to_dict(schema, record) + yield ManifestFile(**dict_repr) + + +@singledispatch +def _convert_pos_to_dict(schema: Union[Schema, IcebergType], struct: AvroStruct) -> Dict[str, Any]: + """Converts the positions in the field names + + This makes it easy to map it onto a Pydantic model. Might change later on depending on the performance + + Args: + schema (Schema | IcebergType): The schema of the file + struct (AvroStruct): The struct containing the data by positions + + Raises: + NotImplementedError: If attempting to handle an unknown type in the schema + """ + raise NotImplementedError(f"Cannot traverse non-type: {schema}") + + +@_convert_pos_to_dict.register +def _(schema: Schema, struct: AvroStruct) -> Dict[str, Any]: + return _convert_pos_to_dict(schema.as_struct(), struct) + + +@_convert_pos_to_dict.register +def _(struct_type: StructType, values: AvroStruct) -> Dict[str, Any]: + """Iterates over all the fields in the dict, and gets the data from the struct""" + return {field.name: _convert_pos_to_dict(field.field_type, values.get(pos)) for pos, field in enumerate(struct_type.fields)} + + +@_convert_pos_to_dict.register +def _(list_type: ListType, values: List[Any]) -> Any: + """In the case of a list, we'll go over the elements in the list to handle complex types""" + return [_convert_pos_to_dict(list_type.element_type, value) for value in values] + + +@_convert_pos_to_dict.register +def _(map_type: MapType, values: Dict) -> Dict: + """In the case of a map, we both traverse over the key and value to handle complex types""" + return { + _convert_pos_to_dict(map_type.key_type, key): _convert_pos_to_dict(map_type.value_type, value) + for key, value in values.items() + } + + +@_convert_pos_to_dict.register +def _(primitive: PrimitiveType, value: Any) -> Any: # pylint: disable=unused-argument + return value diff --git a/tests/utils/test_manifest.py b/tests/utils/test_manifest.py new file mode 100644 index 0000000000..289989bed3 --- /dev/null +++ b/tests/utils/test_manifest.py @@ -0,0 +1,283 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +from pyiceberg.manifest import ( + DataFile, + FieldSummary, + FileFormat, + ManifestEntry, + ManifestFile, + read_manifest_entry, + read_manifest_list, +) +from tests.io.test_io_base import LocalInputFile + + +def test_read_manifest_entry(generated_manifest_entry_file: str): + input_file = LocalInputFile(generated_manifest_entry_file) + assert list(read_manifest_entry(input_file)) == [ + ManifestEntry( + status=1, + snapshot_id=8744736658442914487, + data_file=DataFile( + file_path="/home/iceberg/warehouse/nyc/taxis_partitioned/data/VendorID=null/00000-633-d8a4223e-dc97-45a1-86e1-adaba6e8abd7-00001.parquet", + file_format=FileFormat.PARQUET, + partition={"VendorID": None}, + record_count=19513, + file_size_in_bytes=388872, + block_size_in_bytes=67108864, + column_sizes={ + 1: 53, + 2: 98153, + 3: 98693, + 4: 53, + 5: 53, + 6: 53, + 7: 17425, + 8: 18528, + 9: 53, + 10: 44788, + 11: 35571, + 12: 53, + 13: 1243, + 14: 2355, + 15: 12750, + 16: 4029, + 17: 110, + 18: 47194, + 19: 2948, + }, + value_counts={ + 1: 19513, + 2: 19513, + 3: 19513, + 4: 19513, + 5: 19513, + 6: 19513, + 7: 19513, + 8: 19513, + 9: 19513, + 10: 19513, + 11: 19513, + 12: 19513, + 13: 19513, + 14: 19513, + 15: 19513, + 16: 19513, + 17: 19513, + 18: 19513, + 19: 19513, + }, + null_value_counts={ + 1: 19513, + 2: 0, + 3: 0, + 4: 19513, + 5: 19513, + 6: 19513, + 7: 0, + 8: 0, + 9: 19513, + 10: 0, + 11: 0, + 12: 19513, + 13: 0, + 14: 0, + 15: 0, + 16: 0, + 17: 0, + 18: 0, + 19: 0, + }, + nan_value_counts={16: 0, 17: 0, 18: 0, 19: 0, 10: 0, 11: 0, 12: 0, 13: 0, 14: 0, 15: 0}, + lower_bounds={ + 2: b"2020-04-01 00:00", + 3: b"2020-04-01 00:12", + 7: b"\x03\x00\x00\x00", + 8: b"\x01\x00\x00\x00", + 10: b"\xf6(\\\x8f\xc2\x05S\xc0", + 11: b"\x00\x00\x00\x00\x00\x00\x00\x00", + 13: b"\x00\x00\x00\x00\x00\x00\x00\x00", + 14: b"\x00\x00\x00\x00\x00\x00\xe0\xbf", + 15: b")\\\x8f\xc2\xf5(\x08\xc0", + 16: b"\x00\x00\x00\x00\x00\x00\x00\x00", + 17: b"\x00\x00\x00\x00\x00\x00\x00\x00", + 18: b"\xf6(\\\x8f\xc2\xc5S\xc0", + 19: b"\x00\x00\x00\x00\x00\x00\x04\xc0", + }, + upper_bounds={ + 2: b"2020-04-30 23:5:", + 3: b"2020-05-01 00:41", + 7: b"\t\x01\x00\x00", + 8: b"\t\x01\x00\x00", + 10: b"\xcd\xcc\xcc\xcc\xcc,_@", + 11: b"\x1f\x85\xebQ\\\xe2\xfe@", + 13: b"\x00\x00\x00\x00\x00\x00\x12@", + 14: b"\x00\x00\x00\x00\x00\x00\xe0?", + 15: b"q=\n\xd7\xa3\xf01@", + 16: b"\x00\x00\x00\x00\x00`B@", + 17: b"333333\xd3?", + 18: b"\x00\x00\x00\x00\x00\x18b@", + 19: b"\x00\x00\x00\x00\x00\x00\x04@", + }, + key_metadata=None, + split_offsets=[4], + sort_order_id=0, + ), + ), + ManifestEntry( + status=1, + snapshot_id=8744736658442914487, + data_file=DataFile( + file_path="/home/iceberg/warehouse/nyc/taxis_partitioned/data/VendorID=1/00000-633-d8a4223e-dc97-45a1-86e1-adaba6e8abd7-00002.parquet", + file_format=FileFormat.PARQUET, + partition={"VendorID": 1}, + record_count=95050, + file_size_in_bytes=1265950, + block_size_in_bytes=67108864, + column_sizes={ + 1: 318, + 2: 329806, + 3: 331632, + 4: 15343, + 5: 2351, + 6: 3389, + 7: 71269, + 8: 76429, + 9: 16383, + 10: 86992, + 11: 89608, + 12: 265, + 13: 19377, + 14: 1692, + 15: 76162, + 16: 4354, + 17: 759, + 18: 120650, + 19: 11804, + }, + value_counts={ + 1: 95050, + 2: 95050, + 3: 95050, + 4: 95050, + 5: 95050, + 6: 95050, + 7: 95050, + 8: 95050, + 9: 95050, + 10: 95050, + 11: 95050, + 12: 95050, + 13: 95050, + 14: 95050, + 15: 95050, + 16: 95050, + 17: 95050, + 18: 95050, + 19: 95050, + }, + null_value_counts={ + 1: 0, + 2: 0, + 3: 0, + 4: 0, + 5: 0, + 6: 0, + 7: 0, + 8: 0, + 9: 0, + 10: 0, + 11: 0, + 12: 95050, + 13: 0, + 14: 0, + 15: 0, + 16: 0, + 17: 0, + 18: 0, + 19: 0, + }, + nan_value_counts={16: 0, 17: 0, 18: 0, 19: 0, 10: 0, 11: 0, 12: 0, 13: 0, 14: 0, 15: 0}, + lower_bounds={ + 1: b"\x01\x00\x00\x00", + 2: b"2020-04-01 00:00", + 3: b"2020-04-01 00:03", + 4: b"\x00\x00\x00\x00", + 5: b"\x01\x00\x00\x00", + 6: b"N", + 7: b"\x01\x00\x00\x00", + 8: b"\x01\x00\x00\x00", + 9: b"\x01\x00\x00\x00", + 10: b"\x00\x00\x00\x00\x00\x00\x00\x00", + 11: b"\x00\x00\x00\x00\x00\x00\x00\x00", + 13: b"\x00\x00\x00\x00\x00\x00\x00\x00", + 14: b"\x00\x00\x00\x00\x00\x00\x00\x00", + 15: b"\x00\x00\x00\x00\x00\x00\x00\x00", + 16: b"\x00\x00\x00\x00\x00\x00\x00\x00", + 17: b"\x00\x00\x00\x00\x00\x00\x00\x00", + 18: b"\x00\x00\x00\x00\x00\x00\x00\x00", + 19: b"\x00\x00\x00\x00\x00\x00\x00\x00", + }, + upper_bounds={ + 1: b"\x01\x00\x00\x00", + 2: b"2020-04-30 23:5:", + 3: b"2020-05-01 00:1:", + 4: b"\x06\x00\x00\x00", + 5: b"c\x00\x00\x00", + 6: b"Y", + 7: b"\t\x01\x00\x00", + 8: b"\t\x01\x00\x00", + 9: b"\x04\x00\x00\x00", + 10: b"\\\x8f\xc2\xf5(8\x8c@", + 11: b"\xcd\xcc\xcc\xcc\xcc,f@", + 13: b"\x00\x00\x00\x00\x00\x00\x1c@", + 14: b"\x9a\x99\x99\x99\x99\x99\xf1?", + 15: b"\x00\x00\x00\x00\x00\x00Y@", + 16: b"\x00\x00\x00\x00\x00\xb0X@", + 17: b"333333\xd3?", + 18: b"\xc3\xf5(\\\x8f:\x8c@", + 19: b"\x00\x00\x00\x00\x00\x00\x04@", + }, + key_metadata=None, + split_offsets=[4], + sort_order_id=0, + ), + ), + ] + + +def test_read_manifest_list(generated_manifest_file_file: str): + input_file = LocalInputFile(generated_manifest_file_file) + assert list(read_manifest_list(input_file)) == [ + ManifestFile( + manifest_path="/home/iceberg/warehouse/nyc/taxis_partitioned/metadata/0125c686-8aa6-4502-bdcc-b6d17ca41a3b-m0.avro", + manifest_length=7989, + partition_spec_id=0, + added_snapshot_id=9182715666859759686, + added_data_files_count=3, + existing_data_files_count=0, + deleted_data_files_count=0, + partitions=[ + FieldSummary( + contains_null=True, contains_nan=False, lower_bound=b"\x01\x00\x00\x00", upper_bound=b"\x02\x00\x00\x00" + ) + ], + added_rows_count=237993, + existing_rows_counts=None, + deleted_rows_count=0, + ) + ] From 8e0f615c674982cdd26fbed14372d1d67bf4fa1a Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Fri, 22 Jul 2022 13:18:29 -0700 Subject: [PATCH 143/642] Bump zstandard from 0.17.0 to 0.18.0 in /python (#5342) Bumps [zstandard](https://github.com/indygreg/python-zstandard) from 0.17.0 to 0.18.0. - [Release notes](https://github.com/indygreg/python-zstandard/releases) - [Changelog](https://github.com/indygreg/python-zstandard/blob/main/docs/news.rst) - [Commits](https://github.com/indygreg/python-zstandard/compare/0.17.0...0.18.0) Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- poetry.lock | 209 ++++++++++++++++++++++++++++++++++++------------- pyproject.toml | 2 +- 2 files changed, 154 insertions(+), 57 deletions(-) diff --git a/poetry.lock b/poetry.lock index 7103cb6fbf..2296e03625 100644 --- a/poetry.lock +++ b/poetry.lock @@ -205,8 +205,8 @@ optional = false python-versions = ">=3.6" [package.extras] -dev = ["pre-commit", "tox"] -testing = ["pytest", "pytest-benchmark"] +testing = ["pytest-benchmark", "pytest"] +dev = ["tox", "pre-commit"] [[package]] name = "pre-commit" @@ -395,7 +395,7 @@ testing = ["pytest (>=6)", "pytest-checkdocs (>=2.4)", "pytest-flake8", "pytest- [[package]] name = "zstandard" -version = "0.17.0" +version = "0.18.0" description = "Zstandard bindings for Python" category = "main" optional = true @@ -415,10 +415,12 @@ snappy = ["python-snappy"] [metadata] lock-version = "1.1" python-versions = "^3.8" -content-hash = "c60cc02a89d6d22566af086e4f94dcf12662a59d0a9ceacbfbd90c65b5b34308" +content-hash = "906929f3e98c56eb63e4175d071dd366746751e9357a45d18a223d68ece38498" [metadata.files] -atomicwrites = [] +atomicwrites = [ + {file = "atomicwrites-1.4.1.tar.gz", hash = "sha256:81b2c9071a49367a7f770170e5eec8cb66567cfbbc8c73d20ce5ca4a8d71cf11"}, +] attrs = [ {file = "attrs-21.4.0-py2.py3-none-any.whl", hash = "sha256:2d27e3784d7a565d36ab851fe94887c5eccd6a463168875832a1be79c82828b4"}, {file = "attrs-21.4.0.tar.gz", hash = "sha256:626ba8234211db98e869df76230a137c4c40a12d72445c45d5f5b716f076e2fd"}, @@ -497,18 +499,84 @@ colorama = [ {file = "colorama-0.4.5-py2.py3-none-any.whl", hash = "sha256:854bf444933e37f5824ae7bfc1e98d5bce2ebe4160d46b5edf346a89358e99da"}, {file = "colorama-0.4.5.tar.gz", hash = "sha256:e6c6b4334fc50988a639d9b98aa429a0b57da6e17b9a44f0451f930b6967b7a4"}, ] -coverage = [] -distlib = [] +coverage = [ + {file = "coverage-6.4.2-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:a9032f9b7d38bdf882ac9f66ebde3afb8145f0d4c24b2e600bc4c6304aafb87e"}, + {file = "coverage-6.4.2-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:e0524adb49c716ca763dbc1d27bedce36b14f33e6b8af6dba56886476b42957c"}, + {file = "coverage-6.4.2-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:d4548be38a1c810d79e097a38107b6bf2ff42151900e47d49635be69943763d8"}, + {file = "coverage-6.4.2-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:f23876b018dfa5d3e98e96f5644b109090f16a4acb22064e0f06933663005d39"}, + {file = "coverage-6.4.2-cp310-cp310-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:6fe75dcfcb889b6800f072f2af5a331342d63d0c1b3d2bf0f7b4f6c353e8c9c0"}, + {file = "coverage-6.4.2-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:2f8553878a24b00d5ab04b7a92a2af50409247ca5c4b7a2bf4eabe94ed20d3ee"}, + {file = "coverage-6.4.2-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:d774d9e97007b018a651eadc1b3970ed20237395527e22cbeb743d8e73e0563d"}, + {file = "coverage-6.4.2-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:d56f105592188ce7a797b2bd94b4a8cb2e36d5d9b0d8a1d2060ff2a71e6b9bbc"}, + {file = "coverage-6.4.2-cp310-cp310-win32.whl", hash = "sha256:d230d333b0be8042ac34808ad722eabba30036232e7a6fb3e317c49f61c93386"}, + {file = "coverage-6.4.2-cp310-cp310-win_amd64.whl", hash = "sha256:5ef42e1db047ca42827a85e34abe973971c635f83aed49611b7f3ab49d0130f0"}, + {file = "coverage-6.4.2-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:25b7ec944f114f70803d6529394b64f8749e93cbfac0fe6c5ea1b7e6c14e8a46"}, + {file = "coverage-6.4.2-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:7bb00521ab4f99fdce2d5c05a91bddc0280f0afaee0e0a00425e28e209d4af07"}, + {file = "coverage-6.4.2-cp37-cp37m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:2dff52b3e7f76ada36f82124703f4953186d9029d00d6287f17c68a75e2e6039"}, + {file = "coverage-6.4.2-cp37-cp37m-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:147605e1702d996279bb3cc3b164f408698850011210d133a2cb96a73a2f7996"}, + {file = "coverage-6.4.2-cp37-cp37m-musllinux_1_1_aarch64.whl", hash = "sha256:422fa44070b42fef9fb8dabd5af03861708cdd6deb69463adc2130b7bf81332f"}, + {file = "coverage-6.4.2-cp37-cp37m-musllinux_1_1_i686.whl", hash = "sha256:8af6c26ba8df6338e57bedbf916d76bdae6308e57fc8f14397f03b5da8622b4e"}, + {file = "coverage-6.4.2-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:5336e0352c0b12c7e72727d50ff02557005f79a0b8dcad9219c7c4940a930083"}, + {file = "coverage-6.4.2-cp37-cp37m-win32.whl", hash = "sha256:0f211df2cba951ffcae210ee00e54921ab42e2b64e0bf2c0befc977377fb09b7"}, + {file = "coverage-6.4.2-cp37-cp37m-win_amd64.whl", hash = "sha256:a13772c19619118903d65a91f1d5fea84be494d12fd406d06c849b00d31bf120"}, + {file = "coverage-6.4.2-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:f7bd0ffbcd03dc39490a1f40b2669cc414fae0c4e16b77bb26806a4d0b7d1452"}, + {file = "coverage-6.4.2-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:0895ea6e6f7f9939166cc835df8fa4599e2d9b759b02d1521b574e13b859ac32"}, + {file = "coverage-6.4.2-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:d4e7ced84a11c10160c0697a6cc0b214a5d7ab21dfec1cd46e89fbf77cc66fae"}, + {file = "coverage-6.4.2-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:80db4a47a199c4563d4a25919ff29c97c87569130375beca3483b41ad5f698e8"}, + {file = "coverage-6.4.2-cp38-cp38-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:3def6791adf580d66f025223078dc84c64696a26f174131059ce8e91452584e1"}, + {file = "coverage-6.4.2-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:4f89d8e03c8a3757aae65570d14033e8edf192ee9298303db15955cadcff0c63"}, + {file = "coverage-6.4.2-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:6d0b48aff8e9720bdec315d67723f0babd936a7211dc5df453ddf76f89c59933"}, + {file = "coverage-6.4.2-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:2b20286c2b726f94e766e86a3fddb7b7e37af5d0c635bdfa7e4399bc523563de"}, + {file = "coverage-6.4.2-cp38-cp38-win32.whl", hash = "sha256:d714af0bdba67739598849c9f18efdcc5a0412f4993914a0ec5ce0f1e864d783"}, + {file = "coverage-6.4.2-cp38-cp38-win_amd64.whl", hash = "sha256:5f65e5d3ff2d895dab76b1faca4586b970a99b5d4b24e9aafffc0ce94a6022d6"}, + {file = "coverage-6.4.2-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:a697977157adc052284a7160569b36a8bbec09db3c3220642e6323b47cec090f"}, + {file = "coverage-6.4.2-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:c77943ef768276b61c96a3eb854eba55633c7a3fddf0a79f82805f232326d33f"}, + {file = "coverage-6.4.2-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:54d8d0e073a7f238f0666d3c7c0d37469b2aa43311e4024c925ee14f5d5a1cbe"}, + {file = "coverage-6.4.2-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:f22325010d8824594820d6ce84fa830838f581a7fd86a9235f0d2ed6deb61e29"}, + {file = "coverage-6.4.2-cp39-cp39-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:24b04d305ea172ccb21bee5bacd559383cba2c6fcdef85b7701cf2de4188aa55"}, + {file = "coverage-6.4.2-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:866ebf42b4c5dbafd64455b0a1cd5aa7b4837a894809413b930026c91e18090b"}, + {file = "coverage-6.4.2-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:e36750fbbc422c1c46c9d13b937ab437138b998fe74a635ec88989afb57a3978"}, + {file = "coverage-6.4.2-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:79419370d6a637cb18553ecb25228893966bd7935a9120fa454e7076f13b627c"}, + {file = "coverage-6.4.2-cp39-cp39-win32.whl", hash = "sha256:b5e28db9199dd3833cc8a07fa6cf429a01227b5d429facb56eccd765050c26cd"}, + {file = "coverage-6.4.2-cp39-cp39-win_amd64.whl", hash = "sha256:edfdabe7aa4f97ed2b9dd5dde52d2bb29cb466993bb9d612ddd10d0085a683cf"}, + {file = "coverage-6.4.2-pp36.pp37.pp38-none-any.whl", hash = "sha256:e2618cb2cf5a7cc8d698306e42ebcacd02fb7ef8cfc18485c59394152c70be97"}, + {file = "coverage-6.4.2.tar.gz", hash = "sha256:6c3ccfe89c36f3e5b9837b9ee507472310164f352c9fe332120b764c9d60adbe"}, +] +distlib = [ + {file = "distlib-0.3.5-py2.py3-none-any.whl", hash = "sha256:b710088c59f06338ca514800ad795a132da19fda270e3ce4affc74abf955a26c"}, + {file = "distlib-0.3.5.tar.gz", hash = "sha256:a7f75737c70be3b25e2bee06288cec4e4c221de18455b2dd037fe2a795cab2fe"}, +] docutils = [ {file = "docutils-0.19-py3-none-any.whl", hash = "sha256:5e1de4d849fee02c63b040a4a3fd567f4ab104defd8a5511fbbc24a8a017efbc"}, {file = "docutils-0.19.tar.gz", hash = "sha256:33995a6753c30b7f577febfc2c50411fec6aac7f7ffeb7c4cfe5991072dcf9e6"}, ] -fastavro = [] +fastavro = [ + {file = "fastavro-1.5.3-cp310-cp310-macosx_10_14_x86_64.whl", hash = "sha256:bcaf1c9ead0aa1c7306cff229f2c207d714f7f765e0bb010e64c74d379008555"}, + {file = "fastavro-1.5.3-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:8ce66549bdbc9d43f40e1858796c1db59fa9f63f2368217071929c39f7ee7c1e"}, + {file = "fastavro-1.5.3-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:2c14c72f79bcb689edea06ec276bfd5c4d4f17b8e32f545e32a6ee108c5f09de"}, + {file = "fastavro-1.5.3-cp310-cp310-win_amd64.whl", hash = "sha256:6ad6af14d5e927e0c5fbc5dcb7c7030ef8759c8011620344a57252c263d6dd5a"}, + {file = "fastavro-1.5.3-cp37-cp37m-macosx_10_15_x86_64.whl", hash = "sha256:3a4fa9f72c9b134fcdb0ee30d4a838f1d30656ece72a95f18cc74db10148ee55"}, + {file = "fastavro-1.5.3-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:f8980f2bc1230aac509273938d23c544283a476bf84ac79b80540d503f941fe5"}, + {file = "fastavro-1.5.3-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:78ffb34790791a82edef9cce7e49e44faf1b6b0b1da12e9c7d45f31fd6655a7b"}, + {file = "fastavro-1.5.3-cp37-cp37m-win_amd64.whl", hash = "sha256:86a33b2f819cad67f002250e9c6a62909fc31e048679da5f2ac02f4bbf0a5998"}, + {file = "fastavro-1.5.3-cp38-cp38-macosx_10_15_x86_64.whl", hash = "sha256:c5043cfbdfed27f1331860f1d215db838b92d575e8a78915c52c3f6c64d42176"}, + {file = "fastavro-1.5.3-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:badcbd1c34f446f4b64ade84f9517ef7d52197e0ea9dd6b3775274f2271dba9a"}, + {file = "fastavro-1.5.3-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:259ade020a70870858b656219d25f566eb620e52967070f9e18b040f1c0cd482"}, + {file = "fastavro-1.5.3-cp38-cp38-win_amd64.whl", hash = "sha256:b0408d6b02abcbd4e8866b6b027d7cdd20d570e78177fb7c363fb3cc8ecb02a3"}, + {file = "fastavro-1.5.3-cp39-cp39-macosx_10_15_x86_64.whl", hash = "sha256:063f17e000b7a4174ad362ffc91b75de85970cc67fccfed2bd12e21908de6028"}, + {file = "fastavro-1.5.3-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:9b399317d4f559d8f3ef695ef8db8ea230e4b489437d2a1e50307120e181e831"}, + {file = "fastavro-1.5.3-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:298e98c55e98f579df66e8258a942494a64b5340212968ed1b6bd9a68c52db5f"}, + {file = "fastavro-1.5.3-cp39-cp39-win_amd64.whl", hash = "sha256:14c23dc870785d860f1523e7be7ce4079dc36070f1bad18139304a5f4d654180"}, + {file = "fastavro-1.5.3.tar.gz", hash = "sha256:262a9f629448a0d4941a4871fd958b70a01f48a0106644b40e4fbaf984f5e89e"}, +] filelock = [ {file = "filelock-3.7.1-py3-none-any.whl", hash = "sha256:37def7b658813cda163b56fc564cdc75e86d338246458c4c28ae84cabefa2404"}, {file = "filelock-3.7.1.tar.gz", hash = "sha256:3a0fd85166ad9dbab54c9aec96737b744106dc5f15c0b09a6744a445299fcf04"}, ] -identify = [] +identify = [ + {file = "identify-2.5.2-py2.py3-none-any.whl", hash = "sha256:feaa9db2dc0ce333b453ce171c0cf1247bbfde2c55fc6bb785022d411a1b78b5"}, + {file = "identify-2.5.2.tar.gz", hash = "sha256:a3d4c096b384d50d5e6dc5bc8b9bc44f1f61cefebd750a7b3e9f939b53fb214d"}, +] importlib-metadata = [ {file = "importlib_metadata-4.12.0-py3-none-any.whl", hash = "sha256:7401a975809ea1fdc658c3aa4f78cc2195a0e019c5cbc4c06122884e9ae80c23"}, {file = "importlib_metadata-4.12.0.tar.gz", hash = "sha256:637245b8bab2b6502fcbc752cc4b7a6f6243bb02b31c5c26156ad103d3d45670"}, @@ -549,7 +617,30 @@ nodeenv = [ {file = "nodeenv-1.7.0-py2.py3-none-any.whl", hash = "sha256:27083a7b96a25f2f5e1d8cb4b6317ee8aeda3bdd121394e5ac54e498028a042e"}, {file = "nodeenv-1.7.0.tar.gz", hash = "sha256:e0e7f7dfb85fc5394c6fe1e8fa98131a2473e04311a45afb6508f7cf1836fa2b"}, ] -numpy = [] +numpy = [ + {file = "numpy-1.23.1-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:b15c3f1ed08df4980e02cc79ee058b788a3d0bef2fb3c9ca90bb8cbd5b8a3a04"}, + {file = "numpy-1.23.1-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:9ce242162015b7e88092dccd0e854548c0926b75c7924a3495e02c6067aba1f5"}, + {file = "numpy-1.23.1-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:e0d7447679ae9a7124385ccf0ea990bb85bb869cef217e2ea6c844b6a6855073"}, + {file = "numpy-1.23.1-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:3119daed207e9410eaf57dcf9591fdc68045f60483d94956bee0bfdcba790953"}, + {file = "numpy-1.23.1-cp310-cp310-win32.whl", hash = "sha256:3ab67966c8d45d55a2bdf40701536af6443763907086c0a6d1232688e27e5447"}, + {file = "numpy-1.23.1-cp310-cp310-win_amd64.whl", hash = "sha256:1865fdf51446839ca3fffaab172461f2b781163f6f395f1aed256b1ddc253622"}, + {file = "numpy-1.23.1-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:aeba539285dcf0a1ba755945865ec61240ede5432df41d6e29fab305f4384db2"}, + {file = "numpy-1.23.1-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:7e8229f3687cdadba2c4faef39204feb51ef7c1a9b669247d49a24f3e2e1617c"}, + {file = "numpy-1.23.1-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:68b69f52e6545af010b76516f5daaef6173e73353e3295c5cb9f96c35d755641"}, + {file = "numpy-1.23.1-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:1408c3527a74a0209c781ac82bde2182b0f0bf54dea6e6a363fe0cc4488a7ce7"}, + {file = "numpy-1.23.1-cp38-cp38-win32.whl", hash = "sha256:47f10ab202fe4d8495ff484b5561c65dd59177949ca07975663f4494f7269e3e"}, + {file = "numpy-1.23.1-cp38-cp38-win_amd64.whl", hash = "sha256:37e5ebebb0eb54c5b4a9b04e6f3018e16b8ef257d26c8945925ba8105008e645"}, + {file = "numpy-1.23.1-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:173f28921b15d341afadf6c3898a34f20a0569e4ad5435297ba262ee8941e77b"}, + {file = "numpy-1.23.1-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:876f60de09734fbcb4e27a97c9a286b51284df1326b1ac5f1bf0ad3678236b22"}, + {file = "numpy-1.23.1-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:35590b9c33c0f1c9732b3231bb6a72d1e4f77872390c47d50a615686ae7ed3fd"}, + {file = "numpy-1.23.1-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:a35c4e64dfca659fe4d0f1421fc0f05b8ed1ca8c46fb73d9e5a7f175f85696bb"}, + {file = "numpy-1.23.1-cp39-cp39-win32.whl", hash = "sha256:c2f91f88230042a130ceb1b496932aa717dcbd665350beb821534c5c7e15881c"}, + {file = "numpy-1.23.1-cp39-cp39-win_amd64.whl", hash = "sha256:37ece2bd095e9781a7156852e43d18044fd0d742934833335599c583618181b9"}, + {file = "numpy-1.23.1-pp38-pypy38_pp73-macosx_10_9_x86_64.whl", hash = "sha256:8002574a6b46ac3b5739a003b5233376aeac5163e5dcd43dd7ad062f3e186129"}, + {file = "numpy-1.23.1-pp38-pypy38_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:5d732d17b8a9061540a10fda5bfeabca5785700ab5469a5e9b93aca5e2d3a5fb"}, + {file = "numpy-1.23.1-pp38-pypy38_pp73-win_amd64.whl", hash = "sha256:55df0f7483b822855af67e38fb3a526e787adf189383b4934305565d71c4b148"}, + {file = "numpy-1.23.1.tar.gz", hash = "sha256:d748ef349bfef2e1194b59da37ed5a29c19ea8d7e6342019921ba2ba4fd8b624"}, +] packaging = [ {file = "packaging-21.3-py3-none-any.whl", hash = "sha256:ef103e05f519cdc783ae24ea4e2e0f508a9c99b2d4969652eed6a2e1ea5bd522"}, {file = "packaging-21.3.tar.gz", hash = "sha256:dd47c42927d89ab911e606518907cc2d3a1f38bbd026385970643f9c5b8ecfeb"}, @@ -566,7 +657,10 @@ pluggy = [ {file = "pluggy-1.0.0-py2.py3-none-any.whl", hash = "sha256:74134bbf457f031a36d68416e1509f34bd5ccc019f0bcc952c7b909d06b37bd3"}, {file = "pluggy-1.0.0.tar.gz", hash = "sha256:4224373bacce55f955a878bf9cfa763c1e360858e330072059e10bad68531159"}, ] -pre-commit = [] +pre-commit = [ + {file = "pre_commit-2.20.0-py2.py3-none-any.whl", hash = "sha256:51a5ba7c480ae8072ecdb6933df22d2f812dc897d5fe848778116129a681aac7"}, + {file = "pre_commit-2.20.0.tar.gz", hash = "sha256:a978dac7bc9ec0bcee55c18a277d553b0f419d259dadb4b9418ff2d00eb43959"}, +] py = [ {file = "py-1.11.0-py2.py3-none-any.whl", hash = "sha256:607c53218732647dff4acdfcd50cb62615cedf612e72d1724fb1a0cc6405b378"}, {file = "py-1.11.0.tar.gz", hash = "sha256:51c75c4126074b472f746a24399ad32f6053d1b34b68d2fa41e558e6f4a98719"}, @@ -761,50 +855,53 @@ virtualenv = [ {file = "virtualenv-20.15.1-py2.py3-none-any.whl", hash = "sha256:b30aefac647e86af6d82bfc944c556f8f1a9c90427b2fb4e3bfbf338cb82becf"}, {file = "virtualenv-20.15.1.tar.gz", hash = "sha256:288171134a2ff3bfb1a2f54f119e77cd1b81c29fc1265a2356f3e8d14c7d58c4"}, ] -zipp = [] +zipp = [ + {file = "zipp-3.8.1-py3-none-any.whl", hash = "sha256:47c40d7fe183a6f21403a199b3e4192cca5774656965b0a4988ad2f8feb5f009"}, + {file = "zipp-3.8.1.tar.gz", hash = "sha256:05b45f1ee8f807d0cc928485ca40a07cb491cf092ff587c0df9cb1fd154848d2"}, +] zstandard = [ - {file = "zstandard-0.17.0-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:a1991cdf2e81e643b53fb8d272931d2bdf5f4e70d56a457e1ef95bde147ae627"}, - {file = "zstandard-0.17.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:4768449d8d1b0785309ace288e017cc5fa42e11a52bf08c90d9c3eb3a7a73cc6"}, - {file = "zstandard-0.17.0-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:b1ad6d2952b41d9a0ea702a474cc08c05210c6289e29dd496935c9ca3c7fb45c"}, - {file = "zstandard-0.17.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:90a9ba3a9c16b86afcb785b3c9418af39ccfb238fd5f6e429166e3ca8542b01f"}, - {file = "zstandard-0.17.0-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_12_i686.manylinux2010_i686.whl", hash = "sha256:9cf18c156b3a108197a8bf90b37d03c31c8ef35a7c18807b321d96b74e12c301"}, - {file = "zstandard-0.17.0-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:c81fd9386449df0ebf1ab3e01187bb30d61122c74df53ba4880a2454d866e55d"}, - {file = "zstandard-0.17.0-cp310-cp310-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:787efc741e61e00ffe5e65dac99b0dc5c88b9421012a207a91b869a8b1164921"}, - {file = "zstandard-0.17.0-cp310-cp310-win32.whl", hash = "sha256:49cd09ccbd1e3c0e2690dd62ebf95064d84aa42b9db381867e0b138631f969f2"}, - {file = "zstandard-0.17.0-cp310-cp310-win_amd64.whl", hash = "sha256:d78aac2ffc4e88ab1cbcad844669924c24e24c7c255de9628a18f14d832007c5"}, - {file = "zstandard-0.17.0-cp36-cp36m-macosx_10_9_x86_64.whl", hash = "sha256:c19d1e06569c277dcc872d80cbadf14a29e8199e013ff2a176d169f461439a40"}, - {file = "zstandard-0.17.0-cp36-cp36m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:d916018289d2f9a882e90d2e3bd41652861ce11b5ecd8515fa07ad31d97d56e5"}, - {file = "zstandard-0.17.0-cp36-cp36m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:f0c87f097d6867833a839b086eb8d03676bb87c2efa067a131099f04aa790683"}, - {file = "zstandard-0.17.0-cp36-cp36m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_12_i686.manylinux2010_i686.whl", hash = "sha256:60943f71e3117583655a1eb76188a7cc78a25267ef09cc74be4d25a0b0c8b947"}, - {file = "zstandard-0.17.0-cp36-cp36m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:208fa6bead577b2607205640078ee452e81fe20fe96321623c632bad9ebd7148"}, - {file = "zstandard-0.17.0-cp36-cp36m-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:42f3c02c7021073cafbc6cd152b288c56a25e585518861589bb08b063b6d2ad2"}, - {file = "zstandard-0.17.0-cp36-cp36m-win32.whl", hash = "sha256:2a2ac752162ba5cbc869c60c4a4e54e890b2ee2ffb57d3ff159feab1ae4518db"}, - {file = "zstandard-0.17.0-cp36-cp36m-win_amd64.whl", hash = "sha256:d1405caa964ba11b2396bd9fd19940440217345752e192c936d084ba5fe67dcb"}, - {file = "zstandard-0.17.0-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:ef62eb3bcfd6d786f439828bb544ebd3936432db669403e0b8f48e424f1d55f1"}, - {file = "zstandard-0.17.0-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:477f172807a9fa83467b30d7c58876af1410d20177c554c27525211edf535bae"}, - {file = "zstandard-0.17.0-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:de1aa618306a741e0497878b7f845fd6c397e52dd096fb76ed791e7268887176"}, - {file = "zstandard-0.17.0-cp37-cp37m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_12_i686.manylinux2010_i686.whl", hash = "sha256:a827b9c464ee966524f8e82ec1aabb4a77ff9514cae041667fa81ae2ec8bd3e9"}, - {file = "zstandard-0.17.0-cp37-cp37m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:3cf96ace804945e53bc3e5294097e5fa32a2d43bc52416c632b414b870ee0a21"}, - {file = "zstandard-0.17.0-cp37-cp37m-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:802109f67328c5b822d4fdac28e1cf65a24de2e2e99d76cdbeee9121cedb1b6c"}, - {file = "zstandard-0.17.0-cp37-cp37m-win32.whl", hash = "sha256:a628f20d019feb0f3a171c7a55cc4f75681f3b8c1bd7a5009165a487314887cd"}, - {file = "zstandard-0.17.0-cp37-cp37m-win_amd64.whl", hash = "sha256:7d2e7abac41d2b4b18f03575aca860d2cb647c343e13c23d6c769106a3db2f6f"}, - {file = "zstandard-0.17.0-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:f502fe79757434292174b04db114f9e25c767b2d5ca9e759d118b22a66f445f8"}, - {file = "zstandard-0.17.0-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:e37c4e21f696d6bcdbbc7caf98dffa505d04c0053909b9db0a6e8ca3b935eb07"}, - {file = "zstandard-0.17.0-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:8fd386d0ec1f9343f1776391d9e60d4eedced0a0b0e625bb89b91f6d05f70e83"}, - {file = "zstandard-0.17.0-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:91a228a077fc7cd8486c273788d4a006a37d060cb4293f471eb0325c3113af68"}, - {file = "zstandard-0.17.0-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_12_i686.manylinux2010_i686.whl", hash = "sha256:59eadb9f347d40e8f7ef77caffd0c04a31e82c1df82fe2d2a688032429d750ac"}, - {file = "zstandard-0.17.0-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:a71809ec062c5b7acf286ba6d4484e6fe8130fc2b93c25e596bb34e7810c79b2"}, - {file = "zstandard-0.17.0-cp38-cp38-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:8aedd38d357f6d5e2facd88ce62b4976afdc29db57216a23f14a0cd0ca05a8a3"}, - {file = "zstandard-0.17.0-cp38-cp38-win32.whl", hash = "sha256:bd842ae3dbb7cba88beb022161c819fa80ca7d0c5a4ddd209e7daae85d904e49"}, - {file = "zstandard-0.17.0-cp38-cp38-win_amd64.whl", hash = "sha256:d0e9fec68e304fb35c559c44530213adbc7d5918bdab906a45a0f40cd56c4de2"}, - {file = "zstandard-0.17.0-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:9ec62a4c2dbb0a86ee5138c16ef133e59a23ac108f8d7ac97aeb61d410ce6857"}, - {file = "zstandard-0.17.0-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:d5373a56b90052f171c8634fedc53a6ac371e6c742606e9825772a394bdbd4b0"}, - {file = "zstandard-0.17.0-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:f2e3ea5e4d5ecf3faefd4a5294acb6af1f0578b0cdd75d6b4529c45deaa54d6f"}, - {file = "zstandard-0.17.0-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:7a3a1aa9528087f6f4c47f4ece2d5e6a160527821263fb8174ff36429233e093"}, - {file = "zstandard-0.17.0-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_12_i686.manylinux2010_i686.whl", hash = "sha256:bdf691a205bc492956e6daef7a06fb38f8cbe8b2c1cb0386f35f4412c360c9e9"}, - {file = "zstandard-0.17.0-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:db993a56e21d903893933887984ca9b0d274f2b1db7b3cf21ba129783953864f"}, - {file = "zstandard-0.17.0-cp39-cp39-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:a7756a9446f83c81101f6c0a48c3bfd8d387a249933c57b0d095ca8b20541337"}, - {file = "zstandard-0.17.0-cp39-cp39-win32.whl", hash = "sha256:37e50501baaa935f13a1820ab2114f74313b5cb4cfff8146acb8c5b18cdced2a"}, - {file = "zstandard-0.17.0-cp39-cp39-win_amd64.whl", hash = "sha256:b4e671c4c0804cdf752be26f260058bb858fbdaaef1340af170635913ecca01e"}, - {file = "zstandard-0.17.0.tar.gz", hash = "sha256:fa9194cb91441df7242aa3ddc4cb184be38876cb10dd973674887f334bafbfb6"}, + {file = "zstandard-0.18.0-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:ef7e8a200e4c8ac9102ed3c90ed2aa379f6b880f63032200909c1be21951f556"}, + {file = "zstandard-0.18.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:2dc466207016564805e56d28375f4f533b525ff50d6776946980dff5465566ac"}, + {file = "zstandard-0.18.0-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:4a2ee1d4f98447f3e5183ecfce5626f983504a4a0c005fbe92e60fa8e5d547ec"}, + {file = "zstandard-0.18.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:d956e2f03c7200d7e61345e0880c292783ec26618d0d921dcad470cb195bbce2"}, + {file = "zstandard-0.18.0-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_12_i686.manylinux2010_i686.whl", hash = "sha256:ce6f59cba9854fd14da5bfe34217a1501143057313966637b7291d1b0267bd1e"}, + {file = "zstandard-0.18.0-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:a7fa67cba473623848b6e88acf8d799b1906178fd883fb3a1da24561c779593b"}, + {file = "zstandard-0.18.0-cp310-cp310-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:cdb44d7284c8c5dd1b66dfb86dda7f4560fa94bfbbc1d2da749ba44831335e32"}, + {file = "zstandard-0.18.0-cp310-cp310-win32.whl", hash = "sha256:63694a376cde0aa8b1971d06ca28e8f8b5f492779cb6ee1cc46bbc3f019a42a5"}, + {file = "zstandard-0.18.0-cp310-cp310-win_amd64.whl", hash = "sha256:702a8324cd90c74d9c8780d02bf55e79da3193c870c9665ad3a11647e3ad1435"}, + {file = "zstandard-0.18.0-cp36-cp36m-macosx_10_9_x86_64.whl", hash = "sha256:46f679bc5dfd938db4fb058218d9dc4db1336ffaf1ea774ff152ecadabd40805"}, + {file = "zstandard-0.18.0-cp36-cp36m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:dc2a4de9f363b3247d472362a65041fe4c0f59e01a2846b15d13046be866a885"}, + {file = "zstandard-0.18.0-cp36-cp36m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:bd3220d7627fd4d26397211cb3b560ec7cc4a94b75cfce89e847e8ce7fabe32d"}, + {file = "zstandard-0.18.0-cp36-cp36m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_12_i686.manylinux2010_i686.whl", hash = "sha256:39e98cf4773234bd9cebf9f9db730e451dfcfe435e220f8921242afda8321887"}, + {file = "zstandard-0.18.0-cp36-cp36m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:5228e596eb1554598c872a337bbe4e5afe41cd1f8b1b15f2e35b50d061e35244"}, + {file = "zstandard-0.18.0-cp36-cp36m-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:d4a8fd45746a6c31e729f35196e80b8f1e9987c59f5ccb8859d7c6a6fbeb9c63"}, + {file = "zstandard-0.18.0-cp36-cp36m-win32.whl", hash = "sha256:4cbb85f29a990c2fdbf7bc63246567061a362ddca886d7fae6f780267c0a9e67"}, + {file = "zstandard-0.18.0-cp36-cp36m-win_amd64.whl", hash = "sha256:bfa6c8549fa18e6497a738b7033c49f94a8e2e30c5fbe2d14d0b5aa8bbc1695d"}, + {file = "zstandard-0.18.0-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:e02043297c1832f2666cd2204f381bef43b10d56929e13c42c10c732c6e3b4ed"}, + {file = "zstandard-0.18.0-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:7231543d38d2b7e02ef7cc78ef7ffd86419437e1114ff08709fe25a160e24bd6"}, + {file = "zstandard-0.18.0-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:c86befac87445927488f5c8f205d11566f64c11519db223e9d282b945fa60dab"}, + {file = "zstandard-0.18.0-cp37-cp37m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_12_i686.manylinux2010_i686.whl", hash = "sha256:999a4e1768f219826ba3fa2064fab1c86dd72fdd47a42536235478c3bb3ca3e2"}, + {file = "zstandard-0.18.0-cp37-cp37m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:9df59cd1cf3c62075ee2a4da767089d19d874ac3ad42b04a71a167e91b384722"}, + {file = "zstandard-0.18.0-cp37-cp37m-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:1be31e9e3f7607ee0cdd60915410a5968b205d3e7aa83b7fcf3dd76dbbdb39e0"}, + {file = "zstandard-0.18.0-cp37-cp37m-win32.whl", hash = "sha256:490d11b705b8ae9dc845431bacc8dd1cef2408aede176620a5cd0cd411027936"}, + {file = "zstandard-0.18.0-cp37-cp37m-win_amd64.whl", hash = "sha256:266aba27fa9cc5e9091d3d325ebab1fa260f64e83e42516d5e73947c70216a5b"}, + {file = "zstandard-0.18.0-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:8b2260c4e07dd0723eadb586de7718b61acca4083a490dda69c5719d79bc715c"}, + {file = "zstandard-0.18.0-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:3af8c2383d02feb6650e9255491ec7d0824f6e6dd2bbe3e521c469c985f31fb1"}, + {file = "zstandard-0.18.0-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:28723a1d2e4df778573b76b321ebe9f3469ac98988104c2af116dd344802c3f8"}, + {file = "zstandard-0.18.0-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:19cac7108ff2c342317fad6dc97604b47a41f403c8f19d0bfc396dfadc3638b8"}, + {file = "zstandard-0.18.0-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_12_i686.manylinux2010_i686.whl", hash = "sha256:76725d1ee83a8915100a310bbad5d9c1fc6397410259c94033b8318d548d9990"}, + {file = "zstandard-0.18.0-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:d716a7694ce1fa60b20bc10f35c4a22be446ef7f514c8dbc8f858b61976de2fb"}, + {file = "zstandard-0.18.0-cp38-cp38-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:49685bf9a55d1ab34bd8423ea22db836ba43a181ac6b045ac4272093d5cb874e"}, + {file = "zstandard-0.18.0-cp38-cp38-win32.whl", hash = "sha256:1af1268a7dc870eb27515fb8db1f3e6c5a555d2b7bcc476fc3bab8886c7265ab"}, + {file = "zstandard-0.18.0-cp38-cp38-win_amd64.whl", hash = "sha256:1dc2d3809e763055a1a6c1a73f2b677320cc9a5aa1a7c6cfb35aee59bddc42d9"}, + {file = "zstandard-0.18.0-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:eea18c1e7442f2aa9aff1bb84550dbb6a1f711faf6e48e7319de8f2b2e923c2a"}, + {file = "zstandard-0.18.0-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:8677ffc6a6096cccbd892e558471c901fd821aba12b7fbc63833c7346f549224"}, + {file = "zstandard-0.18.0-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:083dc08abf03807af9beeb2b6a91c23ad78add2499f828176a3c7b742c44df02"}, + {file = "zstandard-0.18.0-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:c990063664c08169c84474acecc9251ee035871589025cac47c060ff4ec4bc1a"}, + {file = "zstandard-0.18.0-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_12_i686.manylinux2010_i686.whl", hash = "sha256:533db8a6fac6248b2cb2c935e7b92f994efbdeb72e1ffa0b354432e087bb5a3e"}, + {file = "zstandard-0.18.0-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:dbb3cb8a082d62b8a73af42291569d266b05605e017a3d8a06a0e5c30b5f10f0"}, + {file = "zstandard-0.18.0-cp39-cp39-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:d6c85ca5162049ede475b7ec98e87f9390501d44a3d6776ddd504e872464ec25"}, + {file = "zstandard-0.18.0-cp39-cp39-win32.whl", hash = "sha256:75479e7c2b3eebf402c59fbe57d21bc400cefa145ca356ee053b0a08908c5784"}, + {file = "zstandard-0.18.0-cp39-cp39-win_amd64.whl", hash = "sha256:d85bfabad444812133a92fc6fbe463e1d07581dba72f041f07a360e63808b23c"}, + {file = "zstandard-0.18.0.tar.gz", hash = "sha256:0ac0357a0d985b4ff31a854744040d7b5754385d1f98f7145c30e02c6865cb6f"}, ] diff --git a/pyproject.toml b/pyproject.toml index 3d080332ca..24cae0c813 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -44,7 +44,7 @@ pydantic = "^1.9.1" pyarrow = { version = "^8.0.0", optional = true } -zstandard = { version = "^0.17.0", optional = true } +zstandard = { version = "^0.18.0", optional = true } python-snappy = { version = "^0.6.1", optional = true } From ecbfbdecd570031242b302da82cd9ad5ee275868 Mon Sep 17 00:00:00 2001 From: Nick Ouellet <35903677+CircArgs@users.noreply.github.com> Date: Mon, 25 Jul 2022 15:37:26 -0400 Subject: [PATCH 144/642] Python: Add more expression classes (#5258) --- pyiceberg/expressions/base.py | 316 ++++++++++++++++-- tests/expressions/test_expressions_base.py | 361 ++++++++++++++++----- 2 files changed, 569 insertions(+), 108 deletions(-) diff --git a/pyiceberg/expressions/base.py b/pyiceberg/expressions/base.py index 876965ae71..4b4a487a4d 100644 --- a/pyiceberg/expressions/base.py +++ b/pyiceberg/expressions/base.py @@ -14,10 +14,12 @@ # KIND, either express or implied. See the License for the # specific language governing permissions and limitations # under the License. +from __future__ import annotations + from abc import ABC, abstractmethod from dataclasses import dataclass from functools import reduce, singledispatch -from typing import Generic, Tuple, TypeVar +from typing import Generic, TypeVar from pyiceberg.files import StructProtocol from pyiceberg.schema import Accessor, Schema @@ -41,7 +43,7 @@ def value(self) -> T: return self._value # type: ignore @abstractmethod - def to(self, type_var) -> "Literal": + def to(self, type_var) -> Literal: ... # pragma: no cover def __repr__(self): @@ -73,7 +75,7 @@ class BooleanExpression(ABC): """Represents a boolean expression tree.""" @abstractmethod - def __invert__(self) -> "BooleanExpression": + def __invert__(self) -> BooleanExpression: """Transform the Expression into its negated version.""" @@ -156,7 +158,7 @@ def bind(self, schema: Schema, case_sensitive: bool) -> BoundReference[T]: Returns: BoundReference: A reference bound to the specific field in the Iceberg schema """ - field = schema.find_field(name_or_id=self.name, case_sensitive=case_sensitive) + field = schema.find_field(name_or_id=self.name, case_sensitive=case_sensitive) # pylint: disable=redefined-outer-name if not field: raise ValueError(f"Cannot find field '{self.name}' in schema: {schema}") @@ -169,16 +171,70 @@ def bind(self, schema: Schema, case_sensitive: bool) -> BoundReference[T]: return BoundReference(field=field, accessor=accessor) -@dataclass(frozen=True) # type: ignore[misc] class BoundPredicate(Bound[T], BooleanExpression): - term: BoundReference[T] - literals: Tuple[Literal[T], ...] + _term: BoundTerm[T] + _literals: tuple[Literal[T], ...] + + def __init__(self, term: BoundTerm[T], *literals: Literal[T]): + self._term = term + self._literals = literals + self._validate_literals() + + def _validate_literals(self): + if len(self.literals) != 1: + raise AttributeError(f"{self.__class__.__name__} must have exactly 1 literal.") + + @property + def term(self) -> BoundTerm[T]: + return self._term + + @property + def literals(self) -> tuple[Literal[T], ...]: + return self._literals + + def __str__(self) -> str: + return f"{self.__class__.__name__}({str(self.term)}{self.literals and ', '+str(self.literals)})" + + def __repr__(self) -> str: + return f"{self.__class__.__name__}({repr(self.term)}{self.literals and ', '+repr(self.literals)})" + + def __eq__(self, other) -> bool: + return id(self) == id(other) or ( + type(self) == type(other) and self.term == other.term and self.literals == other.literals + ) -@dataclass(frozen=True) # type: ignore[misc] -class UnboundPredicate(Unbound[T, BooleanExpression], BooleanExpression): - term: Reference[T] - literals: Tuple[Literal[T], ...] +class UnboundPredicate(Unbound[T, BooleanExpression], BooleanExpression, ABC): + _term: UnboundTerm[T] + _literals: tuple[Literal[T], ...] + + def __init__(self, term: UnboundTerm[T], *literals: Literal[T]): + self._term = term + self._literals = literals + self._validate_literals() + + def _validate_literals(self): + if len(self.literals) != 1: + raise AttributeError(f"{self.__class__.__name__} must have exactly 1 literal.") + + @property + def term(self) -> UnboundTerm[T]: + return self._term + + @property + def literals(self) -> tuple[Literal[T], ...]: + return self._literals + + def __str__(self) -> str: + return f"{self.__class__.__name__}({str(self.term)}{self.literals and ', '+str(self.literals)})" + + def __repr__(self) -> str: + return f"{self.__class__.__name__}({repr(self.term)}{self.literals and ', '+repr(self.literals)})" + + def __eq__(self, other) -> bool: + return id(self) == id(other) or ( + type(self) == type(other) and self.term == other.term and self.literals == other.literals + ) class And(BooleanExpression): @@ -209,14 +265,14 @@ def right(self) -> BooleanExpression: def __eq__(self, other) -> bool: return id(self) == id(other) or (isinstance(other, And) and self.left == other.left and self.right == other.right) - def __invert__(self) -> "Or": + def __invert__(self) -> Or: return Or(~self.left, ~self.right) def __repr__(self) -> str: return f"And({repr(self.left)}, {repr(self.right)})" def __str__(self) -> str: - return f"({self.left} and {self.right})" + return f"And({str(self.left)}, {str(self.right)})" class Or(BooleanExpression): @@ -247,14 +303,14 @@ def right(self) -> BooleanExpression: def __eq__(self, other) -> bool: return id(self) == id(other) or (isinstance(other, Or) and self.left == other.left and self.right == other.right) - def __invert__(self) -> "And": + def __invert__(self) -> And: return And(~self.left, ~self.right) def __repr__(self) -> str: return f"Or({repr(self.left)}, {repr(self.right)})" def __str__(self) -> str: - return f"({self.left} or {self.right})" + return f"Or({str(self.left)}, {str(self.right)})" class Not(BooleanExpression): @@ -282,49 +338,239 @@ def __repr__(self) -> str: return f"Not({repr(self.child)})" def __str__(self) -> str: - return f"(not {self.child})" + return f"Not({str(self.child)})" +@dataclass(frozen=True) class AlwaysTrue(BooleanExpression, ABC, Singleton): """TRUE expression""" - def __invert__(self) -> "AlwaysFalse": + def __invert__(self) -> AlwaysFalse: return AlwaysFalse() - def __repr__(self) -> str: - return "AlwaysTrue()" - - def __str__(self) -> str: - return "true" - +@dataclass(frozen=True) class AlwaysFalse(BooleanExpression, ABC, Singleton): """FALSE expression""" - def __invert__(self) -> "AlwaysTrue": + def __invert__(self) -> AlwaysTrue: return AlwaysTrue() - def __repr__(self) -> str: - return "AlwaysFalse()" - def __str__(self) -> str: - return "false" +class IsNull(UnboundPredicate[T]): + def __invert__(self) -> NotNull: + return NotNull(self.term) + + def _validate_literals(self): # pylint: disable=W0238 + if self.literals is not None: + raise AttributeError("Null is a unary predicate and takes no Literals.") + + def bind(self, schema: Schema, case_sensitive: bool) -> BoundIsNull[T]: + bound_ref = self.term.bind(schema, case_sensitive) + return BoundIsNull(bound_ref) + + +class BoundIsNull(BoundPredicate[T]): + def __invert__(self) -> BoundNotNull: + return BoundNotNull(self.term) + + def _validate_literals(self): # pylint: disable=W0238 + if self.literals: + raise AttributeError("Null is a unary predicate and takes no Literals.") + + +class NotNull(UnboundPredicate[T]): + def __invert__(self) -> IsNull: + return IsNull(self.term) + + def _validate_literals(self): # pylint: disable=W0238 + if self.literals: + raise AttributeError("NotNull is a unary predicate and takes no Literals.") + + def bind(self, schema: Schema, case_sensitive: bool) -> BoundNotNull[T]: + bound_ref = self.term.bind(schema, case_sensitive) + return BoundNotNull(bound_ref) + + +class BoundNotNull(BoundPredicate[T]): + def __invert__(self) -> BoundIsNull: + return BoundIsNull(self.term) + + def _validate_literals(self): # pylint: disable=W0238 + if self.literals: + raise AttributeError("NotNull is a unary predicate and takes no Literals.") + + +class IsNaN(UnboundPredicate[T]): + def __invert__(self) -> NotNaN: + return NotNaN(self.term) + + def _validate_literals(self): # pylint: disable=W0238 + if self.literals: + raise AttributeError("IsNaN is a unary predicate and takes no Literals.") + + def bind(self, schema: Schema, case_sensitive: bool) -> BoundIsNaN[T]: + bound_ref = self.term.bind(schema, case_sensitive) + return BoundIsNaN(bound_ref) + + +class BoundIsNaN(BoundPredicate[T]): + def __invert__(self) -> BoundNotNaN: + return BoundNotNaN(self.term) + + def _validate_literals(self): # pylint: disable=W0238 + if self.literals: + raise AttributeError("IsNaN is a unary predicate and takes no Literals.") + + +class NotNaN(UnboundPredicate[T]): + def __invert__(self) -> IsNaN: + return IsNaN(self.term) + + def _validate_literals(self): # pylint: disable=W0238 + if self.literals: + raise AttributeError("NotNaN is a unary predicate and takes no Literals.") + + def bind(self, schema: Schema, case_sensitive: bool) -> BoundNotNaN[T]: + bound_ref = self.term.bind(schema, case_sensitive) + return BoundNotNaN(bound_ref) + + +class BoundNotNaN(BoundPredicate[T]): + def __invert__(self) -> BoundIsNaN: + return BoundIsNaN(self.term) + + def _validate_literals(self): # pylint: disable=W0238 + if self.literals: + raise AttributeError("NotNaN is a unary predicate and takes no Literals.") -@dataclass(frozen=True) class BoundIn(BoundPredicate[T]): - def __invert__(self): - raise TypeError("In expressions do not support negation.") + def _validate_literals(self): # pylint: disable=W0238 + if not self.literals: + raise AttributeError("BoundIn must contain at least 1 literal.") + + def __invert__(self) -> BoundNotIn[T]: + return BoundNotIn(self.term, *self.literals) -@dataclass(frozen=True) class In(UnboundPredicate[T]): - def __invert__(self): - raise TypeError("In expressions do not support negation.") + def _validate_literals(self): # pylint: disable=W0238 + if not self.literals: + raise AttributeError("In must contain at least 1 literal.") + + def __invert__(self) -> NotIn[T]: + return NotIn(self.term, *self.literals) def bind(self, schema: Schema, case_sensitive: bool) -> BoundIn[T]: bound_ref = self.term.bind(schema, case_sensitive) - return BoundIn(bound_ref, tuple(lit.to(bound_ref.field.field_type) for lit in self.literals)) # type: ignore + return BoundIn(bound_ref, *tuple(lit.to(bound_ref.field.field_type) for lit in self.literals)) # type: ignore + + +class BoundNotIn(BoundPredicate[T]): + def _validate_literals(self): # pylint: disable=W0238 + if not self.literals: + raise AttributeError("BoundNotIn must contain at least 1 literal.") + + def __invert__(self) -> BoundIn[T]: + return BoundIn(self.term, *self.literals) + + +class NotIn(UnboundPredicate[T]): + def _validate_literals(self): # pylint: disable=W0238 + if not self.literals: + raise AttributeError("NotIn must contain at least 1 literal.") + + def __invert__(self) -> In[T]: + return In(self.term, *self.literals) + + def bind(self, schema: Schema, case_sensitive: bool) -> BoundNotIn[T]: + bound_ref = self.term.bind(schema, case_sensitive) + return BoundNotIn(bound_ref, *tuple(lit.to(bound_ref.field.field_type) for lit in self.literals)) # type: ignore + + +class BoundEq(BoundPredicate[T]): + def __invert__(self) -> BoundNotEq[T]: + return BoundNotEq(self.term, *self.literals) + + +class Eq(UnboundPredicate[T]): + def __invert__(self) -> NotEq[T]: + return NotEq(self.term, *self.literals) + + def bind(self, schema: Schema, case_sensitive: bool) -> BoundEq[T]: + bound_ref = self.term.bind(schema, case_sensitive) + return BoundEq(bound_ref, self.literals[0].to(bound_ref.field.field_type)) # type: ignore + + +class BoundNotEq(BoundPredicate[T]): + def __invert__(self) -> BoundEq[T]: + return BoundEq(self.term, *self.literals) + + +class NotEq(UnboundPredicate[T]): + def __invert__(self) -> Eq[T]: + return Eq(self.term, *self.literals) + + def bind(self, schema: Schema, case_sensitive: bool) -> BoundNotEq[T]: + bound_ref = self.term.bind(schema, case_sensitive) + return BoundNotEq(bound_ref, self.literals[0].to(bound_ref.field.field_type)) # type: ignore + + +class BoundLt(BoundPredicate[T]): + def __invert__(self) -> BoundGtEq[T]: + return BoundGtEq(self.term, *self.literals) + + +class Lt(UnboundPredicate[T]): + def __invert__(self) -> GtEq[T]: + return GtEq(self.term, *self.literals) + + def bind(self, schema: Schema, case_sensitive: bool) -> BoundEq[T]: + bound_ref = self.term.bind(schema, case_sensitive) + return BoundLt(bound_ref, self.literals[0].to(bound_ref.field.field_type)) # type: ignore + + +class BoundGtEq(BoundPredicate[T]): + def __invert__(self) -> BoundLt[T]: + return BoundLt(self.term, *self.literals) + + +class GtEq(UnboundPredicate[T]): + def __invert__(self) -> Lt[T]: + return Lt(self.term, *self.literals) + + def bind(self, schema: Schema, case_sensitive: bool) -> BoundEq[T]: + bound_ref = self.term.bind(schema, case_sensitive) + return BoundGtEq(bound_ref, self.literals[0].to(bound_ref.field.field_type)) # type: ignore + + +class BoundGt(BoundPredicate[T]): + def __invert__(self) -> BoundLtEq[T]: + return BoundLtEq(self.term, *self.literals) + + +class Gt(UnboundPredicate[T]): + def __invert__(self) -> LtEq[T]: + return LtEq(self.term, *self.literals) + + def bind(self, schema: Schema, case_sensitive: bool) -> BoundEq[T]: + bound_ref = self.term.bind(schema, case_sensitive) + return BoundGt(bound_ref, self.literals[0].to(bound_ref.field.field_type)) # type: ignore + + +class BoundLtEq(BoundPredicate[T]): + def __invert__(self) -> BoundGt[T]: + return BoundGt(self.term, *self.literals) + + +class LtEq(UnboundPredicate[T]): + def __invert__(self) -> Gt[T]: + return Gt(self.term, *self.literals) + + def bind(self, schema: Schema, case_sensitive: bool) -> BoundEq[T]: + bound_ref = self.term.bind(schema, case_sensitive) + return BoundLtEq(bound_ref, self.literals[0].to(bound_ref.field.field_type)) # type: ignore class BooleanExpressionVisitor(Generic[T], ABC): diff --git a/tests/expressions/test_expressions_base.py b/tests/expressions/test_expressions_base.py index 51630d71e8..2fca17d6cc 100644 --- a/tests/expressions/test_expressions_base.py +++ b/tests/expressions/test_expressions_base.py @@ -130,9 +130,9 @@ def test_reprs(op, rep): @pytest.mark.parametrize( "op, string", [ - (base.And(ExpressionA(), ExpressionB()), "(testexpra and testexprb)"), - (base.Or(ExpressionA(), ExpressionB()), "(testexpra or testexprb)"), - (base.Not(ExpressionA()), "(not testexpra)"), + (base.And(ExpressionA(), ExpressionB()), "And(testexpra, testexprb)"), + (base.Or(ExpressionA(), ExpressionB()), "Or(testexpra, testexprb)"), + (base.Not(ExpressionA()), "Not(testexpra)"), ], ) def test_strs(op, string): @@ -143,25 +143,193 @@ def test_strs(op, string): "a, schema, case_sensitive, success", [ ( - base.In(base.Reference("foo"), (literal("hello"), literal("world"))), + base.In(base.Reference("foo"), literal("hello"), literal("world")), "table_schema_simple", True, True, ), ( - base.In(base.Reference("not_foo"), (literal("hello"), literal("world"))), + base.In(base.Reference("not_foo"), literal("hello"), literal("world")), "table_schema_simple", False, False, ), ( - base.In(base.Reference("Bar"), (literal("hello"), literal("world"))), + base.In(base.Reference("Bar"), literal(5), literal(2)), "table_schema_simple", False, True, ), ( - base.In(base.Reference("Bar"), (literal("hello"), literal("world"))), + base.In(base.Reference("Bar"), literal(5), literal(2)), + "table_schema_simple", + True, + False, + ), + ( + base.NotIn(base.Reference("foo"), literal("hello"), literal("world")), + "table_schema_simple", + True, + True, + ), + ( + base.NotIn(base.Reference("not_foo"), literal("hello"), literal("world")), + "table_schema_simple", + False, + False, + ), + ( + base.NotIn(base.Reference("Bar"), literal(5), literal(2)), + "table_schema_simple", + False, + True, + ), + ( + base.NotIn(base.Reference("Bar"), literal(5), literal(2)), + "table_schema_simple", + True, + False, + ), + ( + base.NotEq(base.Reference("foo"), literal("hello")), + "table_schema_simple", + True, + True, + ), + ( + base.NotEq(base.Reference("not_foo"), literal("hello")), + "table_schema_simple", + False, + False, + ), + ( + base.NotEq(base.Reference("Bar"), literal(5)), + "table_schema_simple", + False, + True, + ), + ( + base.NotEq(base.Reference("Bar"), literal(5)), + "table_schema_simple", + True, + False, + ), + ( + base.Eq(base.Reference("foo"), literal("hello")), + "table_schema_simple", + True, + True, + ), + ( + base.Eq(base.Reference("not_foo"), literal("hello")), + "table_schema_simple", + False, + False, + ), + ( + base.Eq(base.Reference("Bar"), literal(5)), + "table_schema_simple", + False, + True, + ), + ( + base.Eq(base.Reference("Bar"), literal(5)), + "table_schema_simple", + True, + False, + ), + ( + base.Gt(base.Reference("foo"), literal("hello")), + "table_schema_simple", + True, + True, + ), + ( + base.Gt(base.Reference("not_foo"), literal("hello")), + "table_schema_simple", + False, + False, + ), + ( + base.Gt(base.Reference("Bar"), literal(5)), + "table_schema_simple", + False, + True, + ), + ( + base.Gt(base.Reference("Bar"), literal(5)), + "table_schema_simple", + True, + False, + ), + ( + base.Lt(base.Reference("foo"), literal("hello")), + "table_schema_simple", + True, + True, + ), + ( + base.Lt(base.Reference("not_foo"), literal("hello")), + "table_schema_simple", + False, + False, + ), + ( + base.Lt(base.Reference("Bar"), literal(5)), + "table_schema_simple", + False, + True, + ), + ( + base.Lt(base.Reference("Bar"), literal(5)), + "table_schema_simple", + True, + False, + ), + ( + base.GtEq(base.Reference("foo"), literal("hello")), + "table_schema_simple", + True, + True, + ), + ( + base.GtEq(base.Reference("not_foo"), literal("hello")), + "table_schema_simple", + False, + False, + ), + ( + base.GtEq(base.Reference("Bar"), literal(5)), + "table_schema_simple", + False, + True, + ), + ( + base.GtEq(base.Reference("Bar"), literal(5)), + "table_schema_simple", + True, + False, + ), + ( + base.LtEq(base.Reference("foo"), literal("hello")), + "table_schema_simple", + True, + True, + ), + ( + base.LtEq(base.Reference("not_foo"), literal("hello")), + "table_schema_simple", + False, + False, + ), + ( + base.LtEq(base.Reference("Bar"), literal(5)), + "table_schema_simple", + False, + True, + ), + ( + base.LtEq(base.Reference("Bar"), literal(5)), "table_schema_simple", True, False, @@ -194,14 +362,14 @@ def test_bind(a, schema, case_sensitive, success, request): (ExpressionA(), ExpressionA(), ExpressionB()), (ExpressionB(), ExpressionB(), ExpressionA()), ( - base.In(base.Reference("foo"), (literal("hello"), literal("world"))), - base.In(base.Reference("foo"), (literal("hello"), literal("world"))), - base.In(base.Reference("not_foo"), (literal("hello"), literal("world"))), + base.In(base.Reference("foo"), literal("hello"), literal("world")), + base.In(base.Reference("foo"), literal("hello"), literal("world")), + base.In(base.Reference("not_foo"), literal("hello"), literal("world")), ), ( - base.In(base.Reference("foo"), (literal("hello"), literal("world"))), - base.In(base.Reference("foo"), (literal("hello"), literal("world"))), - base.In(base.Reference("foo"), (literal("goodbye"), literal("world"))), + base.In(base.Reference("foo"), literal("hello"), literal("world")), + base.In(base.Reference("foo"), literal("hello"), literal("world")), + base.In(base.Reference("foo"), literal("goodbye"), literal("world")), ), ], ) @@ -210,21 +378,39 @@ def test_eq(exp, testexpra, testexprb): @pytest.mark.parametrize( - "lhs, rhs, raises", + "lhs, rhs", [ - (base.And(ExpressionA(), ExpressionB()), base.Or(ExpressionB(), ExpressionA()), False), - (base.Or(ExpressionA(), ExpressionB()), base.And(ExpressionB(), ExpressionA()), False), - (base.Not(ExpressionA()), ExpressionA(), False), - (base.In(base.Reference("foo"), (literal("hello"), literal("world"))), None, True), - (ExpressionA(), ExpressionB(), False), + ( + base.And(ExpressionA(), ExpressionB()), + base.Or(ExpressionB(), ExpressionA()), + ), + ( + base.Or(ExpressionA(), ExpressionB()), + base.And(ExpressionB(), ExpressionA()), + ), + ( + base.Not(ExpressionA()), + ExpressionA(), + ), + ( + base.In(base.Reference("foo"), literal("hello"), literal("world")), + base.NotIn(base.Reference("foo"), literal("hello"), literal("world")), + ), + ( + base.NotIn(base.Reference("foo"), literal("hello"), literal("world")), + base.In(base.Reference("foo"), literal("hello"), literal("world")), + ), + (base.Gt(base.Reference("foo"), literal(5)), base.LtEq(base.Reference("foo"), literal(5))), + (base.Lt(base.Reference("foo"), literal(5)), base.GtEq(base.Reference("foo"), literal(5))), + (base.Eq(base.Reference("foo"), literal(5)), base.NotEq(base.Reference("foo"), literal(5))), + ( + ExpressionA(), + ExpressionB(), + ), ], ) -def test_negate(lhs, rhs, raises): - if not raises: - assert ~lhs == rhs - else: - with pytest.raises(TypeError): - ~lhs # pylint: disable=W0104 +def test_negate(lhs, rhs): + assert ~lhs == rhs @pytest.mark.parametrize( @@ -400,55 +586,65 @@ def test_always_false_or_always_true_expression_binding(table_schema_simple): [ ( base.And( - base.In(base.Reference("foo"), (literal("foo"), literal("bar"))), - base.In(base.Reference("bar"), (literal(1), literal(2), literal(3))), + base.In(base.Reference("foo"), literal("foo"), literal("bar")), + base.In(base.Reference("bar"), literal(1), literal(2), literal(3)), ), base.And( base.BoundIn[str]( - term=base.BoundReference( + base.BoundReference( field=NestedField(field_id=1, name="foo", field_type=StringType(), required=False), accessor=Accessor(position=0, inner=None), ), - literals=(StringLiteral("foo"), StringLiteral("bar")), + StringLiteral("foo"), + StringLiteral("bar"), ), base.BoundIn[int]( - term=base.BoundReference( + base.BoundReference( field=NestedField(field_id=2, name="bar", field_type=IntegerType(), required=True), accessor=Accessor(position=1, inner=None), ), - literals=(LongLiteral(1), LongLiteral(2), LongLiteral(3)), + LongLiteral(1), + LongLiteral(2), + LongLiteral(3), ), ), ), ( base.And( - base.In(base.Reference("foo"), (literal("bar"), literal("baz"))), - base.In(base.Reference("bar"), (literal(1),)), - base.In(base.Reference("foo"), (literal("baz"),)), + base.In(base.Reference("foo"), literal("bar"), literal("baz")), + base.In( + base.Reference("bar"), + literal(1), + ), + base.In( + base.Reference("foo"), + literal("baz"), + ), ), base.And( base.And( base.BoundIn[str]( - term=base.BoundReference( + base.BoundReference( field=NestedField(field_id=1, name="foo", field_type=StringType(), required=False), accessor=Accessor(position=0, inner=None), ), - literals=(StringLiteral("bar"), StringLiteral("baz")), + StringLiteral("bar"), + StringLiteral("baz"), ), base.BoundIn[int]( - term=base.BoundReference( + base.BoundReference( field=NestedField(field_id=2, name="bar", field_type=IntegerType(), required=True), accessor=Accessor(position=1, inner=None), ), - literals=(LongLiteral(1),), + LongLiteral(1), ), ), base.BoundIn[str]( - term=base.BoundReference( + base.BoundReference( field=NestedField(field_id=1, name="foo", field_type=StringType(), required=False), accessor=Accessor(position=0, inner=None), ), - literals=(StringLiteral("baz"),), + StringLiteral("baz"), ), ), ), @@ -465,55 +661,65 @@ def test_and_expression_binding(unbound_and_expression, expected_bound_expressio [ ( base.Or( - base.In(base.Reference("foo"), (literal("foo"), literal("bar"))), - base.In(base.Reference("bar"), (literal(1), literal(2), literal(3))), + base.In(base.Reference("foo"), literal("foo"), literal("bar")), + base.In(base.Reference("bar"), literal(1), literal(2), literal(3)), ), base.Or( base.BoundIn[str]( - term=base.BoundReference( + base.BoundReference( field=NestedField(field_id=1, name="foo", field_type=StringType(), required=False), accessor=Accessor(position=0, inner=None), ), - literals=(StringLiteral("foo"), StringLiteral("bar")), + StringLiteral("foo"), + StringLiteral("bar"), ), base.BoundIn[int]( - term=base.BoundReference( + base.BoundReference( field=NestedField(field_id=2, name="bar", field_type=IntegerType(), required=True), accessor=Accessor(position=1, inner=None), ), - literals=(LongLiteral(1), LongLiteral(2), LongLiteral(3)), + LongLiteral(1), + LongLiteral(2), + LongLiteral(3), ), ), ), ( base.Or( - base.In(base.Reference("foo"), (literal("bar"), literal("baz"))), - base.In(base.Reference("foo"), (literal("bar"),)), - base.In(base.Reference("foo"), (literal("baz"),)), + base.In(base.Reference("foo"), literal("bar"), literal("baz")), + base.In( + base.Reference("foo"), + literal("bar"), + ), + base.In( + base.Reference("foo"), + literal("baz"), + ), ), base.Or( base.Or( base.BoundIn[str]( - term=base.BoundReference( + base.BoundReference( field=NestedField(field_id=1, name="foo", field_type=StringType(), required=False), accessor=Accessor(position=0, inner=None), ), - literals=(StringLiteral("bar"), StringLiteral("baz")), + StringLiteral("bar"), + StringLiteral("baz"), ), base.BoundIn[str]( - term=base.BoundReference( + base.BoundReference( field=NestedField(field_id=1, name="foo", field_type=StringType(), required=False), accessor=Accessor(position=0, inner=None), ), - literals=(StringLiteral("bar"),), + StringLiteral("bar"), ), ), base.BoundIn[str]( - term=base.BoundReference( + base.BoundReference( field=NestedField(field_id=1, name="foo", field_type=StringType(), required=False), accessor=Accessor(position=0, inner=None), ), - literals=(StringLiteral("baz"),), + StringLiteral("baz"), ), ), ), @@ -550,33 +756,38 @@ def test_or_expression_binding(unbound_or_expression, expected_bound_expression, "unbound_in_expression,expected_bound_expression", [ ( - base.In(base.Reference("foo"), (literal("foo"), literal("bar"))), + base.In(base.Reference("foo"), literal("foo"), literal("bar")), base.BoundIn[str]( - term=base.BoundReference[str]( + base.BoundReference[str]( field=NestedField(field_id=1, name="foo", field_type=StringType(), required=False), accessor=Accessor(position=0, inner=None), ), - literals=(StringLiteral("foo"), StringLiteral("bar")), + StringLiteral("foo"), + StringLiteral("bar"), ), ), ( - base.In(base.Reference("foo"), (literal("bar"), literal("baz"))), + base.In(base.Reference("foo"), literal("bar"), literal("baz")), base.BoundIn[str]( - term=base.BoundReference[str]( + base.BoundReference[str]( field=NestedField(field_id=1, name="foo", field_type=StringType(), required=False), accessor=Accessor(position=0, inner=None), ), - literals=(StringLiteral("bar"), StringLiteral("baz")), + StringLiteral("bar"), + StringLiteral("baz"), ), ), ( - base.In(base.Reference("foo"), (literal("bar"),)), + base.In( + base.Reference("foo"), + literal("bar"), + ), base.BoundIn[str]( - term=base.BoundReference[str]( + base.BoundReference[str]( field=NestedField(field_id=1, name="foo", field_type=StringType(), required=False), accessor=Accessor(position=0, inner=None), ), - literals=(StringLiteral("bar"),), + StringLiteral("bar"), ), ), ], @@ -591,39 +802,43 @@ def test_in_expression_binding(unbound_in_expression, expected_bound_expression, "unbound_not_expression,expected_bound_expression", [ ( - base.Not(base.In(base.Reference("foo"), (literal("foo"), literal("bar")))), + base.Not(base.In(base.Reference("foo"), literal("foo"), literal("bar"))), base.Not( base.BoundIn[str]( - term=base.BoundReference[str]( + base.BoundReference[str]( field=NestedField(field_id=1, name="foo", field_type=StringType(), required=False), accessor=Accessor(position=0, inner=None), ), - literals=(StringLiteral("foo"), StringLiteral("bar")), + StringLiteral("foo"), + StringLiteral("bar"), ) ), ), ( base.Not( base.Or( - base.In(base.Reference("foo"), (literal("foo"), literal("bar"))), - base.In(base.Reference("foo"), (literal("foo"), literal("bar"), literal("baz"))), + base.In(base.Reference("foo"), literal("foo"), literal("bar")), + base.In(base.Reference("foo"), literal("foo"), literal("bar"), literal("baz")), ) ), base.Not( base.Or( base.BoundIn[str]( - term=base.BoundReference( + base.BoundReference( field=NestedField(field_id=1, name="foo", field_type=StringType(), required=False), accessor=Accessor(position=0, inner=None), ), - literals=(StringLiteral("foo"), StringLiteral("bar")), + StringLiteral("foo"), + StringLiteral("bar"), ), base.BoundIn[str]( - term=base.BoundReference( + base.BoundReference( field=NestedField(field_id=1, name="foo", field_type=StringType(), required=False), accessor=Accessor(position=0, inner=None), ), - literals=(StringLiteral("foo"), StringLiteral("bar"), StringLiteral("baz")), + StringLiteral("foo"), + StringLiteral("bar"), + StringLiteral("baz"), ), ), ), From 2f1dc0f8ffed22930a9195706e566a34a9fbcba1 Mon Sep 17 00:00:00 2001 From: Ryan Blue Date: Fri, 29 Jul 2022 11:17:49 -0700 Subject: [PATCH 145/642] Python: Refactor unary and set expressions (#5362) * Python: Refactor unary expressions. * Python: Add SetPredicate and BoundSetPredicate to simply in cases. * Update for review comments. * Fix type checks. * Fix linting. * Fix mypy issues. * Add LiteralPredicate and refactor. * Rename inequalities. * Python: Ignore false positive pylint W0221 for 3.8. --- pyiceberg/expressions/base.py | 491 ++++++++++----------- pyiceberg/expressions/literals.py | 71 ++- tests/expressions/test_expressions_base.py | 422 +++++++++--------- 3 files changed, 508 insertions(+), 476 deletions(-) diff --git a/pyiceberg/expressions/base.py b/pyiceberg/expressions/base.py index 4b4a487a4d..f91093b17a 100644 --- a/pyiceberg/expressions/base.py +++ b/pyiceberg/expressions/base.py @@ -19,58 +19,18 @@ from abc import ABC, abstractmethod from dataclasses import dataclass from functools import reduce, singledispatch -from typing import Generic, TypeVar +from typing import ClassVar, Generic, TypeVar +from pyiceberg.expressions.literals import Literal from pyiceberg.files import StructProtocol from pyiceberg.schema import Accessor, Schema -from pyiceberg.types import NestedField +from pyiceberg.types import DoubleType, FloatType, NestedField from pyiceberg.utils.singleton import Singleton T = TypeVar("T") B = TypeVar("B") -class Literal(Generic[T], ABC): - """Literal which has a value and can be converted between types""" - - def __init__(self, value: T, value_type: type): - if value is None or not isinstance(value, value_type): - raise TypeError(f"Invalid literal value: {value} (not a {value_type})") - self._value = value - - @property - def value(self) -> T: - return self._value # type: ignore - - @abstractmethod - def to(self, type_var) -> Literal: - ... # pragma: no cover - - def __repr__(self): - return f"{type(self).__name__}({self.value})" - - def __str__(self): - return str(self.value) - - def __eq__(self, other): - return self.value == other.value - - def __ne__(self, other): - return not self.__eq__(other) - - def __lt__(self, other): - return self.value < other.value - - def __gt__(self, other): - return self.value > other.value - - def __le__(self, other): - return self.value <= other.value - - def __ge__(self, other): - return self.value >= other.value - - class BooleanExpression(ABC): """Represents a boolean expression tree.""" @@ -105,6 +65,10 @@ class BaseReference(Generic[T], Term, ABC): class BoundTerm(Bound[T], Term): """Represents a bound term.""" + @abstractmethod + def ref(self) -> BoundReference[T]: + ... + class UnboundTerm(Unbound[T, BoundTerm[T]], Term): """Represents an unbound term.""" @@ -131,6 +95,9 @@ def eval(self, struct: StructProtocol) -> T: """ return self.accessor.get(struct) + def ref(self) -> BoundReference[T]: + return self + @dataclass(frozen=True) class Reference(UnboundTerm[T], BaseReference[T]): @@ -171,72 +138,6 @@ def bind(self, schema: Schema, case_sensitive: bool) -> BoundReference[T]: return BoundReference(field=field, accessor=accessor) -class BoundPredicate(Bound[T], BooleanExpression): - _term: BoundTerm[T] - _literals: tuple[Literal[T], ...] - - def __init__(self, term: BoundTerm[T], *literals: Literal[T]): - self._term = term - self._literals = literals - self._validate_literals() - - def _validate_literals(self): - if len(self.literals) != 1: - raise AttributeError(f"{self.__class__.__name__} must have exactly 1 literal.") - - @property - def term(self) -> BoundTerm[T]: - return self._term - - @property - def literals(self) -> tuple[Literal[T], ...]: - return self._literals - - def __str__(self) -> str: - return f"{self.__class__.__name__}({str(self.term)}{self.literals and ', '+str(self.literals)})" - - def __repr__(self) -> str: - return f"{self.__class__.__name__}({repr(self.term)}{self.literals and ', '+repr(self.literals)})" - - def __eq__(self, other) -> bool: - return id(self) == id(other) or ( - type(self) == type(other) and self.term == other.term and self.literals == other.literals - ) - - -class UnboundPredicate(Unbound[T, BooleanExpression], BooleanExpression, ABC): - _term: UnboundTerm[T] - _literals: tuple[Literal[T], ...] - - def __init__(self, term: UnboundTerm[T], *literals: Literal[T]): - self._term = term - self._literals = literals - self._validate_literals() - - def _validate_literals(self): - if len(self.literals) != 1: - raise AttributeError(f"{self.__class__.__name__} must have exactly 1 literal.") - - @property - def term(self) -> UnboundTerm[T]: - return self._term - - @property - def literals(self) -> tuple[Literal[T], ...]: - return self._literals - - def __str__(self) -> str: - return f"{self.__class__.__name__}({str(self.term)}{self.literals and ', '+str(self.literals)})" - - def __repr__(self) -> str: - return f"{self.__class__.__name__}({repr(self.term)}{self.literals and ', '+repr(self.literals)})" - - def __eq__(self, other) -> bool: - return id(self) == id(other) or ( - type(self) == type(other) and self.term == other.term and self.literals == other.literals - ) - - class And(BooleanExpression): """AND operation expression - logical conjunction""" @@ -342,7 +243,7 @@ def __str__(self) -> str: @dataclass(frozen=True) -class AlwaysTrue(BooleanExpression, ABC, Singleton): +class AlwaysTrue(BooleanExpression, Singleton): """TRUE expression""" def __invert__(self) -> AlwaysFalse: @@ -350,227 +251,313 @@ def __invert__(self) -> AlwaysFalse: @dataclass(frozen=True) -class AlwaysFalse(BooleanExpression, ABC, Singleton): +class AlwaysFalse(BooleanExpression, Singleton): """FALSE expression""" def __invert__(self) -> AlwaysTrue: return AlwaysTrue() -class IsNull(UnboundPredicate[T]): - def __invert__(self) -> NotNull: - return NotNull(self.term) +@dataclass(frozen=True) +class BoundPredicate(Bound[T], BooleanExpression): + term: BoundTerm[T] - def _validate_literals(self): # pylint: disable=W0238 - if self.literals is not None: - raise AttributeError("Null is a unary predicate and takes no Literals.") + def __invert__(self) -> BoundPredicate[T]: + raise NotImplementedError - def bind(self, schema: Schema, case_sensitive: bool) -> BoundIsNull[T]: - bound_ref = self.term.bind(schema, case_sensitive) - return BoundIsNull(bound_ref) +@dataclass(frozen=True) +class UnboundPredicate(Unbound[T, BooleanExpression], BooleanExpression): + as_bound: ClassVar[type] + term: UnboundTerm[T] -class BoundIsNull(BoundPredicate[T]): - def __invert__(self) -> BoundNotNull: - return BoundNotNull(self.term) + def __invert__(self) -> UnboundPredicate[T]: + raise NotImplementedError - def _validate_literals(self): # pylint: disable=W0238 - if self.literals: - raise AttributeError("Null is a unary predicate and takes no Literals.") + def bind(self, schema: Schema, case_sensitive: bool = True) -> BooleanExpression: + raise NotImplementedError -class NotNull(UnboundPredicate[T]): - def __invert__(self) -> IsNull: - return IsNull(self.term) +@dataclass(frozen=True) +class UnaryPredicate(UnboundPredicate[T]): + def bind(self, schema: Schema, case_sensitive: bool = True) -> BooleanExpression: + bound_term = self.term.bind(schema, case_sensitive) + return self.as_bound(bound_term) + + def __invert__(self) -> UnaryPredicate[T]: + raise NotImplementedError + - def _validate_literals(self): # pylint: disable=W0238 - if self.literals: - raise AttributeError("NotNull is a unary predicate and takes no Literals.") +@dataclass(frozen=True) +class BoundUnaryPredicate(BoundPredicate[T]): + def __invert__(self) -> BoundUnaryPredicate[T]: + raise NotImplementedError + + +@dataclass(frozen=True) +class BoundIsNull(BoundUnaryPredicate[T]): + def __new__(cls, term: BoundTerm[T]): # pylint: disable=W0221 + if term.ref().field.required: + return AlwaysFalse() + return super().__new__(cls) + + def __invert__(self) -> BoundNotNull[T]: + return BoundNotNull(self.term) - def bind(self, schema: Schema, case_sensitive: bool) -> BoundNotNull[T]: - bound_ref = self.term.bind(schema, case_sensitive) - return BoundNotNull(bound_ref) +@dataclass(frozen=True) +class BoundNotNull(BoundUnaryPredicate[T]): + def __new__(cls, term: BoundTerm[T]): # pylint: disable=W0221 + if term.ref().field.required: + return AlwaysTrue() + return super().__new__(cls) -class BoundNotNull(BoundPredicate[T]): def __invert__(self) -> BoundIsNull: return BoundIsNull(self.term) - def _validate_literals(self): # pylint: disable=W0238 - if self.literals: - raise AttributeError("NotNull is a unary predicate and takes no Literals.") +@dataclass(frozen=True) +class IsNull(UnaryPredicate[T]): + as_bound = BoundIsNull + + def __invert__(self) -> NotNull[T]: + return NotNull(self.term) -class IsNaN(UnboundPredicate[T]): - def __invert__(self) -> NotNaN: - return NotNaN(self.term) - def _validate_literals(self): # pylint: disable=W0238 - if self.literals: - raise AttributeError("IsNaN is a unary predicate and takes no Literals.") +@dataclass(frozen=True) +class NotNull(UnaryPredicate[T]): + as_bound = BoundNotNull - def bind(self, schema: Schema, case_sensitive: bool) -> BoundIsNaN[T]: - bound_ref = self.term.bind(schema, case_sensitive) - return BoundIsNaN(bound_ref) + def __invert__(self) -> IsNull[T]: + return IsNull(self.term) -class BoundIsNaN(BoundPredicate[T]): - def __invert__(self) -> BoundNotNaN: +@dataclass(frozen=True) +class BoundIsNaN(BoundUnaryPredicate[T]): + def __new__(cls, term: BoundTerm[T]): # pylint: disable=W0221 + bound_type = term.ref().field.field_type + if type(bound_type) in {FloatType, DoubleType}: + return super().__new__(cls) + return AlwaysFalse() + + def __invert__(self) -> BoundNotNaN[T]: return BoundNotNaN(self.term) - def _validate_literals(self): # pylint: disable=W0238 - if self.literals: - raise AttributeError("IsNaN is a unary predicate and takes no Literals.") + +@dataclass(frozen=True) +class BoundNotNaN(BoundUnaryPredicate[T]): + def __new__(cls, term: BoundTerm[T]): # pylint: disable=W0221 + bound_type = term.ref().field.field_type + if type(bound_type) in {FloatType, DoubleType}: + return super().__new__(cls) + return AlwaysTrue() + + def __invert__(self) -> BoundIsNaN[T]: + return BoundIsNaN(self.term) -class NotNaN(UnboundPredicate[T]): - def __invert__(self) -> IsNaN: +@dataclass(frozen=True) +class IsNaN(UnaryPredicate[T]): + as_bound = BoundIsNaN + + def __invert__(self) -> NotNaN[T]: + return NotNaN(self.term) + + +@dataclass(frozen=True) +class NotNaN(UnaryPredicate[T]): + as_bound = BoundNotNaN + + def __invert__(self) -> IsNaN[T]: return IsNaN(self.term) - def _validate_literals(self): # pylint: disable=W0238 - if self.literals: - raise AttributeError("NotNaN is a unary predicate and takes no Literals.") - def bind(self, schema: Schema, case_sensitive: bool) -> BoundNotNaN[T]: - bound_ref = self.term.bind(schema, case_sensitive) - return BoundNotNaN(bound_ref) +@dataclass(frozen=True) +class SetPredicate(UnboundPredicate[T]): + literals: tuple[Literal[T], ...] + + def __invert__(self) -> SetPredicate[T]: + raise NotImplementedError + def bind(self, schema: Schema, case_sensitive: bool = True) -> BooleanExpression: + bound_term = self.term.bind(schema, case_sensitive) + return self.as_bound(bound_term, {lit.to(bound_term.ref().field.field_type) for lit in self.literals}) -class BoundNotNaN(BoundPredicate[T]): - def __invert__(self) -> BoundIsNaN: - return BoundIsNaN(self.term) - def _validate_literals(self): # pylint: disable=W0238 - if self.literals: - raise AttributeError("NotNaN is a unary predicate and takes no Literals.") +@dataclass(frozen=True) +class BoundSetPredicate(BoundPredicate[T]): + literals: set[Literal[T]] + + def __invert__(self) -> BoundSetPredicate[T]: + raise NotImplementedError -class BoundIn(BoundPredicate[T]): - def _validate_literals(self): # pylint: disable=W0238 - if not self.literals: - raise AttributeError("BoundIn must contain at least 1 literal.") +@dataclass(frozen=True) +class BoundIn(BoundSetPredicate[T]): + def __new__(cls, term: BoundTerm[T], literals: set[Literal[T]]): # pylint: disable=W0221 + count = len(literals) + if count == 0: + return AlwaysFalse() + elif count == 1: + return BoundEqualTo(term, next(iter(literals))) + else: + return super().__new__(cls) def __invert__(self) -> BoundNotIn[T]: - return BoundNotIn(self.term, *self.literals) + return BoundNotIn(self.term, self.literals) + + +@dataclass(frozen=True) +class BoundNotIn(BoundSetPredicate[T]): + def __new__(cls, term: BoundTerm[T], literals: set[Literal[T]]): # pylint: disable=W0221 + count = len(literals) + if count == 0: + return AlwaysTrue() + elif count == 1: + return BoundNotEqualTo(term, next(iter(literals))) + else: + return super().__new__(cls) + def __invert__(self) -> BoundIn[T]: + return BoundIn(self.term, self.literals) -class In(UnboundPredicate[T]): - def _validate_literals(self): # pylint: disable=W0238 - if not self.literals: - raise AttributeError("In must contain at least 1 literal.") + +@dataclass(frozen=True) +class In(SetPredicate[T]): + as_bound = BoundIn + + def __new__(cls, term: UnboundTerm[T], literals: tuple[Literal[T], ...]): # pylint: disable=W0221 + count = len(literals) + if count == 0: + return AlwaysFalse() + elif count == 1: + return EqualTo(term, literals[0]) + else: + return super().__new__(cls) def __invert__(self) -> NotIn[T]: - return NotIn(self.term, *self.literals) + return NotIn(self.term, self.literals) - def bind(self, schema: Schema, case_sensitive: bool) -> BoundIn[T]: - bound_ref = self.term.bind(schema, case_sensitive) - return BoundIn(bound_ref, *tuple(lit.to(bound_ref.field.field_type) for lit in self.literals)) # type: ignore +@dataclass(frozen=True) +class NotIn(SetPredicate[T]): + as_bound = BoundNotIn -class BoundNotIn(BoundPredicate[T]): - def _validate_literals(self): # pylint: disable=W0238 - if not self.literals: - raise AttributeError("BoundNotIn must contain at least 1 literal.") + def __new__(cls, term: UnboundTerm[T], literals: tuple[Literal[T], ...]): # pylint: disable=W0221 + count = len(literals) + if count == 0: + return AlwaysTrue() + elif count == 1: + return NotEqualTo(term, literals[0]) + else: + return super().__new__(cls) - def __invert__(self) -> BoundIn[T]: - return BoundIn(self.term, *self.literals) + def __invert__(self) -> In[T]: + return In(self.term, self.literals) -class NotIn(UnboundPredicate[T]): - def _validate_literals(self): # pylint: disable=W0238 - if not self.literals: - raise AttributeError("NotIn must contain at least 1 literal.") +@dataclass(frozen=True) +class LiteralPredicate(UnboundPredicate[T]): + literal: Literal[T] - def __invert__(self) -> In[T]: - return In(self.term, *self.literals) + def bind(self, schema: Schema, case_sensitive: bool = True) -> BooleanExpression: + bound_term = self.term.bind(schema, case_sensitive) + return self.as_bound(bound_term, self.literal.to(bound_term.ref().field.field_type)) - def bind(self, schema: Schema, case_sensitive: bool) -> BoundNotIn[T]: - bound_ref = self.term.bind(schema, case_sensitive) - return BoundNotIn(bound_ref, *tuple(lit.to(bound_ref.field.field_type) for lit in self.literals)) # type: ignore + def __invert__(self) -> LiteralPredicate[T]: + raise NotImplementedError -class BoundEq(BoundPredicate[T]): - def __invert__(self) -> BoundNotEq[T]: - return BoundNotEq(self.term, *self.literals) +@dataclass(frozen=True) +class BoundLiteralPredicate(BoundPredicate[T]): + literal: Literal[T] + def __invert__(self) -> BoundLiteralPredicate[T]: + raise NotImplementedError -class Eq(UnboundPredicate[T]): - def __invert__(self) -> NotEq[T]: - return NotEq(self.term, *self.literals) - def bind(self, schema: Schema, case_sensitive: bool) -> BoundEq[T]: - bound_ref = self.term.bind(schema, case_sensitive) - return BoundEq(bound_ref, self.literals[0].to(bound_ref.field.field_type)) # type: ignore +@dataclass(frozen=True) +class BoundEqualTo(BoundLiteralPredicate[T]): + def __invert__(self) -> BoundNotEqualTo[T]: + return BoundNotEqualTo(self.term, self.literal) -class BoundNotEq(BoundPredicate[T]): - def __invert__(self) -> BoundEq[T]: - return BoundEq(self.term, *self.literals) +@dataclass(frozen=True) +class BoundNotEqualTo(BoundLiteralPredicate[T]): + def __invert__(self) -> BoundEqualTo[T]: + return BoundEqualTo(self.term, self.literal) -class NotEq(UnboundPredicate[T]): - def __invert__(self) -> Eq[T]: - return Eq(self.term, *self.literals) +@dataclass(frozen=True) +class BoundGreaterThanOrEqual(BoundLiteralPredicate[T]): + def __invert__(self) -> BoundLessThan[T]: + return BoundLessThan(self.term, self.literal) - def bind(self, schema: Schema, case_sensitive: bool) -> BoundNotEq[T]: - bound_ref = self.term.bind(schema, case_sensitive) - return BoundNotEq(bound_ref, self.literals[0].to(bound_ref.field.field_type)) # type: ignore +@dataclass(frozen=True) +class BoundGreaterThan(BoundLiteralPredicate[T]): + def __invert__(self) -> BoundLessThanOrEqual[T]: + return BoundLessThanOrEqual(self.term, self.literal) + + +@dataclass(frozen=True) +class BoundLessThan(BoundLiteralPredicate[T]): + def __invert__(self) -> BoundGreaterThanOrEqual[T]: + return BoundGreaterThanOrEqual(self.term, self.literal) -class BoundLt(BoundPredicate[T]): - def __invert__(self) -> BoundGtEq[T]: - return BoundGtEq(self.term, *self.literals) + +@dataclass(frozen=True) +class BoundLessThanOrEqual(BoundLiteralPredicate[T]): + def __invert__(self) -> BoundGreaterThan[T]: + return BoundGreaterThan(self.term, self.literal) -class Lt(UnboundPredicate[T]): - def __invert__(self) -> GtEq[T]: - return GtEq(self.term, *self.literals) +@dataclass(frozen=True) +class EqualTo(LiteralPredicate[T]): + as_bound = BoundEqualTo - def bind(self, schema: Schema, case_sensitive: bool) -> BoundEq[T]: - bound_ref = self.term.bind(schema, case_sensitive) - return BoundLt(bound_ref, self.literals[0].to(bound_ref.field.field_type)) # type: ignore + def __invert__(self) -> NotEqualTo[T]: + return NotEqualTo(self.term, self.literal) -class BoundGtEq(BoundPredicate[T]): - def __invert__(self) -> BoundLt[T]: - return BoundLt(self.term, *self.literals) +@dataclass(frozen=True) +class NotEqualTo(LiteralPredicate[T]): + as_bound = BoundNotEqualTo + def __invert__(self) -> EqualTo[T]: + return EqualTo(self.term, self.literal) -class GtEq(UnboundPredicate[T]): - def __invert__(self) -> Lt[T]: - return Lt(self.term, *self.literals) - def bind(self, schema: Schema, case_sensitive: bool) -> BoundEq[T]: - bound_ref = self.term.bind(schema, case_sensitive) - return BoundGtEq(bound_ref, self.literals[0].to(bound_ref.field.field_type)) # type: ignore +@dataclass(frozen=True) +class LessThan(LiteralPredicate[T]): + as_bound = BoundLessThan + def __invert__(self) -> GreaterThanOrEqual[T]: + return GreaterThanOrEqual(self.term, self.literal) -class BoundGt(BoundPredicate[T]): - def __invert__(self) -> BoundLtEq[T]: - return BoundLtEq(self.term, *self.literals) +@dataclass(frozen=True) +class GreaterThanOrEqual(LiteralPredicate[T]): + as_bound = BoundGreaterThanOrEqual -class Gt(UnboundPredicate[T]): - def __invert__(self) -> LtEq[T]: - return LtEq(self.term, *self.literals) + def __invert__(self) -> LessThan[T]: + return LessThan(self.term, self.literal) - def bind(self, schema: Schema, case_sensitive: bool) -> BoundEq[T]: - bound_ref = self.term.bind(schema, case_sensitive) - return BoundGt(bound_ref, self.literals[0].to(bound_ref.field.field_type)) # type: ignore +@dataclass(frozen=True) +class GreaterThan(LiteralPredicate[T]): + as_bound = BoundGreaterThan -class BoundLtEq(BoundPredicate[T]): - def __invert__(self) -> BoundGt[T]: - return BoundGt(self.term, *self.literals) + def __invert__(self) -> LessThanOrEqual[T]: + return LessThanOrEqual(self.term, self.literal) -class LtEq(UnboundPredicate[T]): - def __invert__(self) -> Gt[T]: - return Gt(self.term, *self.literals) +@dataclass(frozen=True) +class LessThanOrEqual(LiteralPredicate[T]): + as_bound = BoundLessThanOrEqual - def bind(self, schema: Schema, case_sensitive: bool) -> BoundEq[T]: - bound_ref = self.term.bind(schema, case_sensitive) - return BoundLtEq(bound_ref, self.literals[0].to(bound_ref.field.field_type)) # type: ignore + def __invert__(self) -> GreaterThan[T]: + return GreaterThan(self.term, self.literal) class BooleanExpressionVisitor(Generic[T], ABC): @@ -680,6 +667,12 @@ def _(obj: In, visitor: BooleanExpressionVisitor[T]) -> T: return visitor.visit_unbound_predicate(predicate=obj) +@visit.register(UnboundPredicate) +def _(obj: UnboundPredicate, visitor: BooleanExpressionVisitor[T]) -> T: + """Visit an In boolean expression with a concrete BooleanExpressionVisitor""" + return visitor.visit_unbound_predicate(predicate=obj) + + @visit.register(Or) def _(obj: Or, visitor: BooleanExpressionVisitor[T]) -> T: """Visit an Or boolean expression with a concrete BooleanExpressionVisitor""" diff --git a/pyiceberg/expressions/literals.py b/pyiceberg/expressions/literals.py index 53491854be..f49ee61805 100644 --- a/pyiceberg/expressions/literals.py +++ b/pyiceberg/expressions/literals.py @@ -19,14 +19,15 @@ # specific language governing permissions and limitations # under the License. # pylint: disable=W0613 +from __future__ import annotations import struct +from abc import ABC, abstractmethod from decimal import ROUND_HALF_UP, Decimal from functools import singledispatch, singledispatchmethod -from typing import Optional, Union +from typing import Generic, TypeVar from uuid import UUID -from pyiceberg.expressions.base import Literal from pyiceberg.types import ( BinaryType, BooleanType, @@ -52,6 +53,52 @@ ) from pyiceberg.utils.singleton import Singleton +T = TypeVar("T") + + +class Literal(Generic[T], ABC): + """Literal which has a value and can be converted between types""" + + def __init__(self, value: T, value_type: type): + if value is None or not isinstance(value, value_type): + raise TypeError(f"Invalid literal value: {value} (not a {value_type})") + self._value = value + + @property + def value(self) -> T: + return self._value # type: ignore + + @abstractmethod + def to(self, type_var) -> Literal: + ... # pragma: no cover + + def __repr__(self): + return f"{type(self).__name__}({self.value})" + + def __str__(self): + return str(self.value) + + def __hash__(self): + return hash(self.value) + + def __eq__(self, other): + return self.value == other.value + + def __ne__(self, other): + return not self.__eq__(other) + + def __lt__(self, other): + return self.value < other.value + + def __gt__(self, other): + return self.value > other.value + + def __le__(self, other): + return self.value <= other.value + + def __ge__(self, other): + return self.value >= other.value + @singledispatch def literal(value) -> Literal: @@ -170,7 +217,7 @@ def _(self, type_var: LongType) -> Literal[int]: return self @to.register(IntegerType) - def _(self, type_var: IntegerType) -> Union[AboveMax, BelowMin, Literal[int]]: + def _(self, type_var: IntegerType) -> AboveMax | BelowMin | Literal[int]: if IntegerType.max < self.value: return AboveMax() elif IntegerType.min > self.value: @@ -258,7 +305,7 @@ def _(self, type_var: DoubleType) -> Literal[float]: return self @to.register(FloatType) - def _(self, type_var: FloatType) -> Union[AboveMax, BelowMin, Literal[float]]: + def _(self, type_var: FloatType) -> AboveMax | BelowMin | Literal[float]: if FloatType.max < self.value: return AboveMax() elif FloatType.min > self.value: @@ -322,7 +369,7 @@ def to(self, type_var): return None @to.register(DecimalType) - def _(self, type_var: DecimalType) -> Optional[Literal[Decimal]]: + def _(self, type_var: DecimalType) -> Literal[Decimal] | None: if type_var.scale == abs(self.value.as_tuple().exponent): return self return None @@ -341,28 +388,28 @@ def _(self, type_var: StringType) -> Literal[str]: return self @to.register(DateType) - def _(self, type_var: DateType) -> Optional[Literal[int]]: + def _(self, type_var: DateType) -> Literal[int] | None: try: return DateLiteral(date_to_days(self.value)) except (TypeError, ValueError): return None @to.register(TimeType) - def _(self, type_var: TimeType) -> Optional[Literal[int]]: + def _(self, type_var: TimeType) -> Literal[int] | None: try: return TimeLiteral(time_to_micros(self.value)) except (TypeError, ValueError): return None @to.register(TimestampType) - def _(self, type_var: TimestampType) -> Optional[Literal[int]]: + def _(self, type_var: TimestampType) -> Literal[int] | None: try: return TimestampLiteral(timestamp_to_micros(self.value)) except (TypeError, ValueError): return None @to.register(TimestamptzType) - def _(self, type_var: TimestamptzType) -> Optional[Literal[int]]: + def _(self, type_var: TimestamptzType) -> Literal[int] | None: try: return TimestampLiteral(timestamptz_to_micros(self.value)) except (TypeError, ValueError): @@ -373,7 +420,7 @@ def _(self, type_var: UUIDType) -> Literal[UUID]: return UUIDLiteral(UUID(self.value)) @to.register(DecimalType) - def _(self, type_var: DecimalType) -> Optional[Literal[Decimal]]: + def _(self, type_var: DecimalType) -> Literal[Decimal] | None: dec = Decimal(self.value) if type_var.scale == abs(dec.as_tuple().exponent): return DecimalLiteral(dec) @@ -403,7 +450,7 @@ def to(self, type_var): return None @to.register(FixedType) - def _(self, type_var: FixedType) -> Optional[Literal[bytes]]: + def _(self, type_var: FixedType) -> Literal[bytes] | None: if len(self.value) == type_var.length: return self else: @@ -427,7 +474,7 @@ def _(self, type_var: BinaryType) -> Literal[bytes]: return self @to.register(FixedType) - def _(self, type_var: FixedType) -> Optional[Literal[bytes]]: + def _(self, type_var: FixedType) -> Literal[bytes] | None: if type_var.length == len(self.value): return FixedLiteral(self.value) else: diff --git a/tests/expressions/test_expressions_base.py b/tests/expressions/test_expressions_base.py index 2fca17d6cc..ba2850133b 100644 --- a/tests/expressions/test_expressions_base.py +++ b/tests/expressions/test_expressions_base.py @@ -23,8 +23,14 @@ from pyiceberg.expressions import base from pyiceberg.expressions.literals import LongLiteral, StringLiteral, literal -from pyiceberg.schema import Accessor -from pyiceberg.types import IntegerType, NestedField, StringType +from pyiceberg.schema import Accessor, Schema +from pyiceberg.types import ( + DoubleType, + FloatType, + IntegerType, + NestedField, + StringType, +) from pyiceberg.utils.singleton import Singleton @@ -127,6 +133,78 @@ def test_reprs(op, rep): assert repr(op) == rep +def test_isnull_inverse(): + assert ~base.IsNull(base.Reference("a")) == base.NotNull(base.Reference("a")) + + +def test_isnull_bind(): + schema = Schema(NestedField(2, "a", IntegerType()), schema_id=1) + bound = base.BoundIsNull(base.BoundReference(schema.find_field(2), schema.accessor_for_field(2))) + assert base.IsNull(base.Reference("a")).bind(schema) == bound + + +def test_isnull_bind_required(): + schema = Schema(NestedField(2, "a", IntegerType(), required=True), schema_id=1) + assert base.IsNull(base.Reference("a")).bind(schema) == base.AlwaysFalse() + + +def test_notnull_inverse(): + assert ~base.NotNull(base.Reference("a")) == base.IsNull(base.Reference("a")) + + +def test_notnull_bind(): + schema = Schema(NestedField(2, "a", IntegerType()), schema_id=1) + bound = base.BoundNotNull(base.BoundReference(schema.find_field(2), schema.accessor_for_field(2))) + assert base.NotNull(base.Reference("a")).bind(schema) == bound + + +def test_notnull_bind_required(): + schema = Schema(NestedField(2, "a", IntegerType(), required=True), schema_id=1) + assert base.NotNull(base.Reference("a")).bind(schema) == base.AlwaysTrue() + + +def test_isnan_inverse(): + assert ~base.IsNaN(base.Reference("f")) == base.NotNaN(base.Reference("f")) + + +def test_isnan_bind_float(): + schema = Schema(NestedField(2, "f", FloatType()), schema_id=1) + bound = base.BoundIsNaN(base.BoundReference(schema.find_field(2), schema.accessor_for_field(2))) + assert base.IsNaN(base.Reference("f")).bind(schema) == bound + + +def test_isnan_bind_double(): + schema = Schema(NestedField(2, "d", DoubleType()), schema_id=1) + bound = base.BoundIsNaN(base.BoundReference(schema.find_field(2), schema.accessor_for_field(2))) + assert base.IsNaN(base.Reference("d")).bind(schema) == bound + + +def test_isnan_bind_nonfloat(): + schema = Schema(NestedField(2, "i", IntegerType()), schema_id=1) + assert base.IsNaN(base.Reference("i")).bind(schema) == base.AlwaysFalse() + + +def test_notnan_inverse(): + assert ~base.NotNaN(base.Reference("f")) == base.IsNaN(base.Reference("f")) + + +def test_notnan_bind_float(): + schema = Schema(NestedField(2, "f", FloatType()), schema_id=1) + bound = base.BoundNotNaN(base.BoundReference(schema.find_field(2), schema.accessor_for_field(2))) + assert base.NotNaN(base.Reference("f")).bind(schema) == bound + + +def test_notnan_bind_double(): + schema = Schema(NestedField(2, "d", DoubleType()), schema_id=1) + bound = base.BoundNotNaN(base.BoundReference(schema.find_field(2), schema.accessor_for_field(2))) + assert base.NotNaN(base.Reference("d")).bind(schema) == bound + + +def test_notnan_bind_nonfloat(): + schema = Schema(NestedField(2, "i", IntegerType()), schema_id=1) + assert base.NotNaN(base.Reference("i")).bind(schema) == base.AlwaysTrue() + + @pytest.mark.parametrize( "op, string", [ @@ -139,210 +217,138 @@ def test_strs(op, string): assert str(op) == string +def test_ref_binding_case_sensitive(request): + schema = request.getfixturevalue("table_schema_simple") + ref = base.Reference("foo") + bound = base.BoundReference(schema.find_field(1), schema.accessor_for_field(1)) + assert ref.bind(schema, case_sensitive=True) == bound + + +def test_ref_binding_case_sensitive_failure(request): + schema = request.getfixturevalue("table_schema_simple") + ref = base.Reference("Foo") + with pytest.raises(ValueError): + ref.bind(schema, case_sensitive=True) + + +def test_ref_binding_case_insensitive(request): + schema = request.getfixturevalue("table_schema_simple") + ref = base.Reference("Foo") + bound = base.BoundReference(schema.find_field(1), schema.accessor_for_field(1)) + assert ref.bind(schema, case_sensitive=False) == bound + + +def test_ref_binding_case_insensitive_failure(request): + schema = request.getfixturevalue("table_schema_simple") + ref = base.Reference("Foot") + with pytest.raises(ValueError): + ref.bind(schema, case_sensitive=False) + + +def test_in_to_eq(): + assert base.In(base.Reference("x"), (literal(34.56),)) == base.EqualTo(base.Reference("x"), literal(34.56)) + + +def test_bind_in(request): + schema = request.getfixturevalue("table_schema_simple") + bound = base.BoundIn( + base.BoundReference(schema.find_field(1), schema.accessor_for_field(1)), {literal("hello"), literal("world")} + ) + assert base.In(base.Reference("foo"), (literal("hello"), literal("world"))).bind(schema) == bound + + +def test_bind_dedup(request): + schema = request.getfixturevalue("table_schema_simple") + bound = base.BoundIn( + base.BoundReference(schema.find_field(1), schema.accessor_for_field(1)), {literal("hello"), literal("world")} + ) + assert base.In(base.Reference("foo"), (literal("hello"), literal("world"), literal("world"))).bind(schema) == bound + + +def test_bind_dedup_to_eq(request): + schema = request.getfixturevalue("table_schema_simple") + bound = base.BoundEqualTo(base.BoundReference(schema.find_field(1), schema.accessor_for_field(1)), literal("hello")) + assert base.In(base.Reference("foo"), (literal("hello"), literal("hello"))).bind(schema) == bound + + @pytest.mark.parametrize( - "a, schema, case_sensitive, success", + "a, schema", [ ( - base.In(base.Reference("foo"), literal("hello"), literal("world")), - "table_schema_simple", - True, - True, - ), - ( - base.In(base.Reference("not_foo"), literal("hello"), literal("world")), - "table_schema_simple", - False, - False, - ), - ( - base.In(base.Reference("Bar"), literal(5), literal(2)), - "table_schema_simple", - False, - True, - ), - ( - base.In(base.Reference("Bar"), literal(5), literal(2)), - "table_schema_simple", - True, - False, - ), - ( - base.NotIn(base.Reference("foo"), literal("hello"), literal("world")), - "table_schema_simple", - True, - True, - ), - ( - base.NotIn(base.Reference("not_foo"), literal("hello"), literal("world")), - "table_schema_simple", - False, - False, - ), - ( - base.NotIn(base.Reference("Bar"), literal(5), literal(2)), - "table_schema_simple", - False, - True, - ), - ( - base.NotIn(base.Reference("Bar"), literal(5), literal(2)), - "table_schema_simple", - True, - False, - ), - ( - base.NotEq(base.Reference("foo"), literal("hello")), - "table_schema_simple", - True, - True, - ), - ( - base.NotEq(base.Reference("not_foo"), literal("hello")), - "table_schema_simple", - False, - False, - ), - ( - base.NotEq(base.Reference("Bar"), literal(5)), - "table_schema_simple", - False, - True, - ), - ( - base.NotEq(base.Reference("Bar"), literal(5)), - "table_schema_simple", - True, - False, - ), - ( - base.Eq(base.Reference("foo"), literal("hello")), + base.NotIn(base.Reference("foo"), (literal("hello"), literal("world"))), "table_schema_simple", - True, - True, ), ( - base.Eq(base.Reference("not_foo"), literal("hello")), + base.NotEqualTo(base.Reference("foo"), literal("hello")), "table_schema_simple", - False, - False, ), ( - base.Eq(base.Reference("Bar"), literal(5)), + base.EqualTo(base.Reference("foo"), literal("hello")), "table_schema_simple", - False, - True, ), ( - base.Eq(base.Reference("Bar"), literal(5)), + base.GreaterThan(base.Reference("foo"), literal("hello")), "table_schema_simple", - True, - False, ), ( - base.Gt(base.Reference("foo"), literal("hello")), + base.LessThan(base.Reference("foo"), literal("hello")), "table_schema_simple", - True, - True, ), ( - base.Gt(base.Reference("not_foo"), literal("hello")), + base.GreaterThanOrEqual(base.Reference("foo"), literal("hello")), "table_schema_simple", - False, - False, ), ( - base.Gt(base.Reference("Bar"), literal(5)), + base.LessThanOrEqual(base.Reference("foo"), literal("hello")), "table_schema_simple", - False, - True, - ), - ( - base.Gt(base.Reference("Bar"), literal(5)), - "table_schema_simple", - True, - False, - ), - ( - base.Lt(base.Reference("foo"), literal("hello")), - "table_schema_simple", - True, - True, - ), - ( - base.Lt(base.Reference("not_foo"), literal("hello")), - "table_schema_simple", - False, - False, - ), - ( - base.Lt(base.Reference("Bar"), literal(5)), - "table_schema_simple", - False, - True, - ), - ( - base.Lt(base.Reference("Bar"), literal(5)), - "table_schema_simple", - True, - False, ), + ], +) +def test_bind(a, schema, request): + schema = request.getfixturevalue(schema) + assert a.bind(schema, case_sensitive=True).term.field == schema.find_field(a.term.name, case_sensitive=True) + + +@pytest.mark.parametrize( + "a, schema", + [ ( - base.GtEq(base.Reference("foo"), literal("hello")), + base.In(base.Reference("Bar"), (literal(5), literal(2))), "table_schema_simple", - True, - True, ), ( - base.GtEq(base.Reference("not_foo"), literal("hello")), + base.NotIn(base.Reference("Bar"), (literal(5), literal(2))), "table_schema_simple", - False, - False, ), ( - base.GtEq(base.Reference("Bar"), literal(5)), + base.NotEqualTo(base.Reference("Bar"), literal(5)), "table_schema_simple", - False, - True, ), ( - base.GtEq(base.Reference("Bar"), literal(5)), + base.EqualTo(base.Reference("Bar"), literal(5)), "table_schema_simple", - True, - False, ), ( - base.LtEq(base.Reference("foo"), literal("hello")), + base.GreaterThan(base.Reference("Bar"), literal(5)), "table_schema_simple", - True, - True, ), ( - base.LtEq(base.Reference("not_foo"), literal("hello")), + base.LessThan(base.Reference("Bar"), literal(5)), "table_schema_simple", - False, - False, ), ( - base.LtEq(base.Reference("Bar"), literal(5)), + base.GreaterThanOrEqual(base.Reference("Bar"), literal(5)), "table_schema_simple", - False, - True, ), ( - base.LtEq(base.Reference("Bar"), literal(5)), + base.LessThanOrEqual(base.Reference("Bar"), literal(5)), "table_schema_simple", - True, - False, ), ], ) -def test_bind(a, schema, case_sensitive, success, request): +def test_bind_case_insensitive(a, schema, request): schema = request.getfixturevalue(schema) - if success: - assert a.bind(schema, case_sensitive).term.field == schema.find_field(a.term.name, case_sensitive) - else: - with pytest.raises(ValueError): - a.bind(schema, case_sensitive) + assert a.bind(schema, case_sensitive=False).term.field == schema.find_field(a.term.name, case_sensitive=False) @pytest.mark.parametrize( @@ -362,14 +368,14 @@ def test_bind(a, schema, case_sensitive, success, request): (ExpressionA(), ExpressionA(), ExpressionB()), (ExpressionB(), ExpressionB(), ExpressionA()), ( - base.In(base.Reference("foo"), literal("hello"), literal("world")), - base.In(base.Reference("foo"), literal("hello"), literal("world")), - base.In(base.Reference("not_foo"), literal("hello"), literal("world")), + base.In(base.Reference("foo"), (literal("hello"), literal("world"))), + base.In(base.Reference("foo"), (literal("hello"), literal("world"))), + base.In(base.Reference("not_foo"), (literal("hello"), literal("world"))), ), ( - base.In(base.Reference("foo"), literal("hello"), literal("world")), - base.In(base.Reference("foo"), literal("hello"), literal("world")), - base.In(base.Reference("foo"), literal("goodbye"), literal("world")), + base.In(base.Reference("foo"), (literal("hello"), literal("world"))), + base.In(base.Reference("foo"), (literal("hello"), literal("world"))), + base.In(base.Reference("foo"), (literal("goodbye"), literal("world"))), ), ], ) @@ -393,16 +399,16 @@ def test_eq(exp, testexpra, testexprb): ExpressionA(), ), ( - base.In(base.Reference("foo"), literal("hello"), literal("world")), - base.NotIn(base.Reference("foo"), literal("hello"), literal("world")), + base.In(base.Reference("foo"), (literal("hello"), literal("world"))), + base.NotIn(base.Reference("foo"), (literal("hello"), literal("world"))), ), ( - base.NotIn(base.Reference("foo"), literal("hello"), literal("world")), - base.In(base.Reference("foo"), literal("hello"), literal("world")), + base.NotIn(base.Reference("foo"), (literal("hello"), literal("world"))), + base.In(base.Reference("foo"), (literal("hello"), literal("world"))), ), - (base.Gt(base.Reference("foo"), literal(5)), base.LtEq(base.Reference("foo"), literal(5))), - (base.Lt(base.Reference("foo"), literal(5)), base.GtEq(base.Reference("foo"), literal(5))), - (base.Eq(base.Reference("foo"), literal(5)), base.NotEq(base.Reference("foo"), literal(5))), + (base.GreaterThan(base.Reference("foo"), literal(5)), base.LessThanOrEqual(base.Reference("foo"), literal(5))), + (base.LessThan(base.Reference("foo"), literal(5)), base.GreaterThanOrEqual(base.Reference("foo"), literal(5))), + (base.EqualTo(base.Reference("foo"), literal(5)), base.NotEqualTo(base.Reference("foo"), literal(5))), ( ExpressionA(), ExpressionB(), @@ -586,8 +592,8 @@ def test_always_false_or_always_true_expression_binding(table_schema_simple): [ ( base.And( - base.In(base.Reference("foo"), literal("foo"), literal("bar")), - base.In(base.Reference("bar"), literal(1), literal(2), literal(3)), + base.In(base.Reference("foo"), (literal("foo"), literal("bar"))), + base.In(base.Reference("bar"), (literal(1), literal(2), literal(3))), ), base.And( base.BoundIn[str]( @@ -595,30 +601,27 @@ def test_always_false_or_always_true_expression_binding(table_schema_simple): field=NestedField(field_id=1, name="foo", field_type=StringType(), required=False), accessor=Accessor(position=0, inner=None), ), - StringLiteral("foo"), - StringLiteral("bar"), + {StringLiteral("foo"), StringLiteral("bar")}, ), base.BoundIn[int]( base.BoundReference( field=NestedField(field_id=2, name="bar", field_type=IntegerType(), required=True), accessor=Accessor(position=1, inner=None), ), - LongLiteral(1), - LongLiteral(2), - LongLiteral(3), + {LongLiteral(1), LongLiteral(2), LongLiteral(3)}, ), ), ), ( base.And( - base.In(base.Reference("foo"), literal("bar"), literal("baz")), + base.In(base.Reference("foo"), (literal("bar"), literal("baz"))), base.In( base.Reference("bar"), - literal(1), + (literal(1),), ), base.In( base.Reference("foo"), - literal("baz"), + (literal("baz"),), ), ), base.And( @@ -628,10 +631,9 @@ def test_always_false_or_always_true_expression_binding(table_schema_simple): field=NestedField(field_id=1, name="foo", field_type=StringType(), required=False), accessor=Accessor(position=0, inner=None), ), - StringLiteral("bar"), - StringLiteral("baz"), + {StringLiteral("bar"), StringLiteral("baz")}, ), - base.BoundIn[int]( + base.BoundEqualTo[int]( base.BoundReference( field=NestedField(field_id=2, name="bar", field_type=IntegerType(), required=True), accessor=Accessor(position=1, inner=None), @@ -639,7 +641,7 @@ def test_always_false_or_always_true_expression_binding(table_schema_simple): LongLiteral(1), ), ), - base.BoundIn[str]( + base.BoundEqualTo[str]( base.BoundReference( field=NestedField(field_id=1, name="foo", field_type=StringType(), required=False), accessor=Accessor(position=0, inner=None), @@ -661,8 +663,8 @@ def test_and_expression_binding(unbound_and_expression, expected_bound_expressio [ ( base.Or( - base.In(base.Reference("foo"), literal("foo"), literal("bar")), - base.In(base.Reference("bar"), literal(1), literal(2), literal(3)), + base.In(base.Reference("foo"), (literal("foo"), literal("bar"))), + base.In(base.Reference("bar"), (literal(1), literal(2), literal(3))), ), base.Or( base.BoundIn[str]( @@ -670,30 +672,27 @@ def test_and_expression_binding(unbound_and_expression, expected_bound_expressio field=NestedField(field_id=1, name="foo", field_type=StringType(), required=False), accessor=Accessor(position=0, inner=None), ), - StringLiteral("foo"), - StringLiteral("bar"), + {StringLiteral("foo"), StringLiteral("bar")}, ), base.BoundIn[int]( base.BoundReference( field=NestedField(field_id=2, name="bar", field_type=IntegerType(), required=True), accessor=Accessor(position=1, inner=None), ), - LongLiteral(1), - LongLiteral(2), - LongLiteral(3), + {LongLiteral(1), LongLiteral(2), LongLiteral(3)}, ), ), ), ( base.Or( - base.In(base.Reference("foo"), literal("bar"), literal("baz")), + base.In(base.Reference("foo"), (literal("bar"), literal("baz"))), base.In( base.Reference("foo"), - literal("bar"), + (literal("bar"),), ), base.In( base.Reference("foo"), - literal("baz"), + (literal("baz"),), ), ), base.Or( @@ -703,15 +702,14 @@ def test_and_expression_binding(unbound_and_expression, expected_bound_expressio field=NestedField(field_id=1, name="foo", field_type=StringType(), required=False), accessor=Accessor(position=0, inner=None), ), - StringLiteral("bar"), - StringLiteral("baz"), + {StringLiteral("bar"), StringLiteral("baz")}, ), base.BoundIn[str]( base.BoundReference( field=NestedField(field_id=1, name="foo", field_type=StringType(), required=False), accessor=Accessor(position=0, inner=None), ), - StringLiteral("bar"), + {StringLiteral("bar")}, ), ), base.BoundIn[str]( @@ -719,7 +717,7 @@ def test_and_expression_binding(unbound_and_expression, expected_bound_expressio field=NestedField(field_id=1, name="foo", field_type=StringType(), required=False), accessor=Accessor(position=0, inner=None), ), - StringLiteral("baz"), + {StringLiteral("baz")}, ), ), ), @@ -756,33 +754,31 @@ def test_or_expression_binding(unbound_or_expression, expected_bound_expression, "unbound_in_expression,expected_bound_expression", [ ( - base.In(base.Reference("foo"), literal("foo"), literal("bar")), + base.In(base.Reference("foo"), (literal("foo"), literal("bar"))), base.BoundIn[str]( base.BoundReference[str]( field=NestedField(field_id=1, name="foo", field_type=StringType(), required=False), accessor=Accessor(position=0, inner=None), ), - StringLiteral("foo"), - StringLiteral("bar"), + {StringLiteral("foo"), StringLiteral("bar")}, ), ), ( - base.In(base.Reference("foo"), literal("bar"), literal("baz")), + base.In(base.Reference("foo"), (literal("bar"), literal("baz"))), base.BoundIn[str]( base.BoundReference[str]( field=NestedField(field_id=1, name="foo", field_type=StringType(), required=False), accessor=Accessor(position=0, inner=None), ), - StringLiteral("bar"), - StringLiteral("baz"), + {StringLiteral("bar"), StringLiteral("baz")}, ), ), ( base.In( base.Reference("foo"), - literal("bar"), + (literal("bar"),), ), - base.BoundIn[str]( + base.BoundEqualTo( base.BoundReference[str]( field=NestedField(field_id=1, name="foo", field_type=StringType(), required=False), accessor=Accessor(position=0, inner=None), @@ -802,23 +798,22 @@ def test_in_expression_binding(unbound_in_expression, expected_bound_expression, "unbound_not_expression,expected_bound_expression", [ ( - base.Not(base.In(base.Reference("foo"), literal("foo"), literal("bar"))), + base.Not(base.In(base.Reference("foo"), (literal("foo"), literal("bar")))), base.Not( base.BoundIn[str]( base.BoundReference[str]( field=NestedField(field_id=1, name="foo", field_type=StringType(), required=False), accessor=Accessor(position=0, inner=None), ), - StringLiteral("foo"), - StringLiteral("bar"), + {StringLiteral("foo"), StringLiteral("bar")}, ) ), ), ( base.Not( base.Or( - base.In(base.Reference("foo"), literal("foo"), literal("bar")), - base.In(base.Reference("foo"), literal("foo"), literal("bar"), literal("baz")), + base.In(base.Reference("foo"), (literal("foo"), literal("bar"))), + base.In(base.Reference("foo"), (literal("foo"), literal("bar"), literal("baz"))), ) ), base.Not( @@ -828,17 +823,14 @@ def test_in_expression_binding(unbound_in_expression, expected_bound_expression, field=NestedField(field_id=1, name="foo", field_type=StringType(), required=False), accessor=Accessor(position=0, inner=None), ), - StringLiteral("foo"), - StringLiteral("bar"), + {StringLiteral("foo"), StringLiteral("bar")}, ), base.BoundIn[str]( base.BoundReference( field=NestedField(field_id=1, name="foo", field_type=StringType(), required=False), accessor=Accessor(position=0, inner=None), ), - StringLiteral("foo"), - StringLiteral("bar"), - StringLiteral("baz"), + {StringLiteral("foo"), StringLiteral("bar"), StringLiteral("baz")}, ), ), ), From 1868a3d739b62846afab7579065c3c962a7f0101 Mon Sep 17 00:00:00 2001 From: Fokko Driesprong Date: Sat, 30 Jul 2022 21:40:53 +0200 Subject: [PATCH 146/642] Python: Add REST catalog implementation (#5287) This does not implement table commits, but does implement the catalog portion of the REST API. --- .pre-commit-config.yaml | 2 +- poetry.lock | 256 +++++------ pyiceberg/catalog/base.py | 19 +- pyiceberg/catalog/rest.py | 434 ++++++++++++++++++ pyiceberg/exceptions.py | 48 +- pyiceberg/schema.py | 2 + pyiceberg/table/base.py | 17 +- pyiceberg/table/metadata.py | 19 +- pyiceberg/table/partitioning.py | 5 +- pyiceberg/utils/iceberg_base_model.py | 1 + pyproject.toml | 7 + tests/catalog/test_base.py | 39 +- tests/catalog/test_rest.py | 612 ++++++++++++++++++++++++++ tests/conftest.py | 2 +- tests/table/test_metadata.py | 56 ++- tests/test_schema.py | 4 +- tests/utils/test_bin_packing.py | 15 - 17 files changed, 1322 insertions(+), 216 deletions(-) create mode 100644 pyiceberg/catalog/rest.py create mode 100644 tests/catalog/test_rest.py diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 9a219be8cc..717b429783 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -40,7 +40,7 @@ repos: rev: v0.971 hooks: - id: mypy - args: [--config=python/pyproject.toml] + args: [ --install-types, --non-interactive, --config=python/pyproject.toml] - repo: https://github.com/hadialqattan/pycln rev: v2.0.4 hooks: diff --git a/poetry.lock b/poetry.lock index 2296e03625..d95bd8729b 100644 --- a/poetry.lock +++ b/poetry.lock @@ -20,6 +20,14 @@ docs = ["furo", "sphinx", "zope.interface", "sphinx-notfound-page"] tests = ["coverage[toml] (>=5.0.2)", "hypothesis", "pympler", "pytest (>=4.3.0)", "six", "mypy", "pytest-mypy-plugins", "zope.interface", "cloudpickle"] tests_no_zope = ["coverage[toml] (>=5.0.2)", "hypothesis", "pympler", "pytest (>=4.3.0)", "six", "mypy", "pytest-mypy-plugins", "cloudpickle"] +[[package]] +name = "certifi" +version = "2022.6.15" +description = "Python package for providing Mozilla's CA Bundle." +category = "main" +optional = false +python-versions = ">=3.6" + [[package]] name = "cffi" version = "1.15.1" @@ -39,6 +47,17 @@ category = "dev" optional = false python-versions = ">=3.6.1" +[[package]] +name = "charset-normalizer" +version = "2.1.0" +description = "The Real First Universal Charset Detector. Open, modern and actively maintained alternative to Chardet." +category = "main" +optional = false +python-versions = ">=3.6.0" + +[package.extras] +unicode_backport = ["unicodedata2"] + [[package]] name = "colorama" version = "0.4.5" @@ -114,6 +133,14 @@ python-versions = ">=3.7" [package.extras] license = ["ukkonen"] +[[package]] +name = "idna" +version = "3.3" +description = "Internationalized Domain Names in Applications (IDNA)" +category = "main" +optional = false +python-versions = ">=3.5" + [[package]] name = "importlib-metadata" version = "4.12.0" @@ -205,8 +232,8 @@ optional = false python-versions = ">=3.6" [package.extras] -testing = ["pytest-benchmark", "pytest"] -dev = ["tox", "pre-commit"] +dev = ["pre-commit", "tox"] +testing = ["pytest", "pytest-benchmark"] [[package]] name = "pre-commit" @@ -331,6 +358,40 @@ category = "dev" optional = false python-versions = ">=3.6" +[[package]] +name = "requests" +version = "2.28.1" +description = "Python HTTP for Humans." +category = "main" +optional = false +python-versions = ">=3.7, <4" + +[package.dependencies] +certifi = ">=2017.4.17" +charset-normalizer = ">=2,<3" +idna = ">=2.5,<4" +urllib3 = ">=1.21.1,<1.27" + +[package.extras] +socks = ["PySocks (>=1.5.6,!=1.5.7)"] +use_chardet_on_py3 = ["chardet (>=3.0.2,<6)"] + +[[package]] +name = "requests-mock" +version = "1.9.3" +description = "Mock out responses from the requests package" +category = "dev" +optional = false +python-versions = "*" + +[package.dependencies] +requests = ">=2.3,<3" +six = "*" + +[package.extras] +fixture = ["fixtures"] +test = ["fixtures", "mock", "purl", "pytest", "sphinx", "testrepository (>=0.0.18)", "testtools"] + [[package]] name = "six" version = "1.16.0" @@ -363,6 +424,19 @@ category = "main" optional = false python-versions = ">=3.7" +[[package]] +name = "urllib3" +version = "1.26.10" +description = "HTTP library with thread-safe connection pooling, file post, and more." +category = "main" +optional = false +python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*, !=3.5.*, <4" + +[package.extras] +brotli = ["brotlicffi (>=0.8.0)", "brotli (>=1.0.9)", "brotlipy (>=0.6.0)"] +secure = ["pyOpenSSL (>=0.14)", "cryptography (>=1.3.4)", "idna (>=2.0.0)", "certifi", "ipaddress"] +socks = ["PySocks (>=1.5.6,!=1.5.7,<2.0)"] + [[package]] name = "virtualenv" version = "20.15.1" @@ -415,16 +489,18 @@ snappy = ["python-snappy"] [metadata] lock-version = "1.1" python-versions = "^3.8" -content-hash = "906929f3e98c56eb63e4175d071dd366746751e9357a45d18a223d68ece38498" +content-hash = "ff76c4a6349ee3a909543aa6587ace0ab6795e08e97e892a90553b6fd6a589af" [metadata.files] -atomicwrites = [ - {file = "atomicwrites-1.4.1.tar.gz", hash = "sha256:81b2c9071a49367a7f770170e5eec8cb66567cfbbc8c73d20ce5ca4a8d71cf11"}, -] +atomicwrites = [] attrs = [ {file = "attrs-21.4.0-py2.py3-none-any.whl", hash = "sha256:2d27e3784d7a565d36ab851fe94887c5eccd6a463168875832a1be79c82828b4"}, {file = "attrs-21.4.0.tar.gz", hash = "sha256:626ba8234211db98e869df76230a137c4c40a12d72445c45d5f5b716f076e2fd"}, ] +certifi = [ + {file = "certifi-2022.6.15-py3-none-any.whl", hash = "sha256:fe86415d55e84719d75f8b69414f6438ac3547d2078ab91b67e779ef69378412"}, + {file = "certifi-2022.6.15.tar.gz", hash = "sha256:84c85a9078b11105f04f3036a9482ae10e4621616db313fe045dd24743a0820d"}, +] cffi = [ {file = "cffi-1.15.1-cp27-cp27m-macosx_10_9_x86_64.whl", hash = "sha256:a66d3508133af6e8548451b25058d5812812ec3798c886bf38ed24a98216fab2"}, {file = "cffi-1.15.1-cp27-cp27m-manylinux1_i686.whl", hash = "sha256:470c103ae716238bbe698d67ad020e1db9d9dba34fa5a899b5e21577e6d52ed2"}, @@ -495,87 +571,29 @@ cfgv = [ {file = "cfgv-3.3.1-py2.py3-none-any.whl", hash = "sha256:c6a0883f3917a037485059700b9e75da2464e6c27051014ad85ba6aaa5884426"}, {file = "cfgv-3.3.1.tar.gz", hash = "sha256:f5a830efb9ce7a445376bb66ec94c638a9787422f96264c98edc6bdeed8ab736"}, ] +charset-normalizer = [ + {file = "charset-normalizer-2.1.0.tar.gz", hash = "sha256:575e708016ff3a5e3681541cb9d79312c416835686d054a23accb873b254f413"}, + {file = "charset_normalizer-2.1.0-py3-none-any.whl", hash = "sha256:5189b6f22b01957427f35b6a08d9a0bc45b46d3788ef5a92e978433c7a35f8a5"}, +] colorama = [ {file = "colorama-0.4.5-py2.py3-none-any.whl", hash = "sha256:854bf444933e37f5824ae7bfc1e98d5bce2ebe4160d46b5edf346a89358e99da"}, {file = "colorama-0.4.5.tar.gz", hash = "sha256:e6c6b4334fc50988a639d9b98aa429a0b57da6e17b9a44f0451f930b6967b7a4"}, ] -coverage = [ - {file = "coverage-6.4.2-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:a9032f9b7d38bdf882ac9f66ebde3afb8145f0d4c24b2e600bc4c6304aafb87e"}, - {file = "coverage-6.4.2-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:e0524adb49c716ca763dbc1d27bedce36b14f33e6b8af6dba56886476b42957c"}, - {file = "coverage-6.4.2-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:d4548be38a1c810d79e097a38107b6bf2ff42151900e47d49635be69943763d8"}, - {file = "coverage-6.4.2-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:f23876b018dfa5d3e98e96f5644b109090f16a4acb22064e0f06933663005d39"}, - {file = "coverage-6.4.2-cp310-cp310-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:6fe75dcfcb889b6800f072f2af5a331342d63d0c1b3d2bf0f7b4f6c353e8c9c0"}, - {file = "coverage-6.4.2-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:2f8553878a24b00d5ab04b7a92a2af50409247ca5c4b7a2bf4eabe94ed20d3ee"}, - {file = "coverage-6.4.2-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:d774d9e97007b018a651eadc1b3970ed20237395527e22cbeb743d8e73e0563d"}, - {file = "coverage-6.4.2-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:d56f105592188ce7a797b2bd94b4a8cb2e36d5d9b0d8a1d2060ff2a71e6b9bbc"}, - {file = "coverage-6.4.2-cp310-cp310-win32.whl", hash = "sha256:d230d333b0be8042ac34808ad722eabba30036232e7a6fb3e317c49f61c93386"}, - {file = "coverage-6.4.2-cp310-cp310-win_amd64.whl", hash = "sha256:5ef42e1db047ca42827a85e34abe973971c635f83aed49611b7f3ab49d0130f0"}, - {file = "coverage-6.4.2-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:25b7ec944f114f70803d6529394b64f8749e93cbfac0fe6c5ea1b7e6c14e8a46"}, - {file = "coverage-6.4.2-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:7bb00521ab4f99fdce2d5c05a91bddc0280f0afaee0e0a00425e28e209d4af07"}, - {file = "coverage-6.4.2-cp37-cp37m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:2dff52b3e7f76ada36f82124703f4953186d9029d00d6287f17c68a75e2e6039"}, - {file = "coverage-6.4.2-cp37-cp37m-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:147605e1702d996279bb3cc3b164f408698850011210d133a2cb96a73a2f7996"}, - {file = "coverage-6.4.2-cp37-cp37m-musllinux_1_1_aarch64.whl", hash = "sha256:422fa44070b42fef9fb8dabd5af03861708cdd6deb69463adc2130b7bf81332f"}, - {file = "coverage-6.4.2-cp37-cp37m-musllinux_1_1_i686.whl", hash = "sha256:8af6c26ba8df6338e57bedbf916d76bdae6308e57fc8f14397f03b5da8622b4e"}, - {file = "coverage-6.4.2-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:5336e0352c0b12c7e72727d50ff02557005f79a0b8dcad9219c7c4940a930083"}, - {file = "coverage-6.4.2-cp37-cp37m-win32.whl", hash = "sha256:0f211df2cba951ffcae210ee00e54921ab42e2b64e0bf2c0befc977377fb09b7"}, - {file = "coverage-6.4.2-cp37-cp37m-win_amd64.whl", hash = "sha256:a13772c19619118903d65a91f1d5fea84be494d12fd406d06c849b00d31bf120"}, - {file = "coverage-6.4.2-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:f7bd0ffbcd03dc39490a1f40b2669cc414fae0c4e16b77bb26806a4d0b7d1452"}, - {file = "coverage-6.4.2-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:0895ea6e6f7f9939166cc835df8fa4599e2d9b759b02d1521b574e13b859ac32"}, - {file = "coverage-6.4.2-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:d4e7ced84a11c10160c0697a6cc0b214a5d7ab21dfec1cd46e89fbf77cc66fae"}, - {file = "coverage-6.4.2-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:80db4a47a199c4563d4a25919ff29c97c87569130375beca3483b41ad5f698e8"}, - {file = "coverage-6.4.2-cp38-cp38-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:3def6791adf580d66f025223078dc84c64696a26f174131059ce8e91452584e1"}, - {file = "coverage-6.4.2-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:4f89d8e03c8a3757aae65570d14033e8edf192ee9298303db15955cadcff0c63"}, - {file = "coverage-6.4.2-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:6d0b48aff8e9720bdec315d67723f0babd936a7211dc5df453ddf76f89c59933"}, - {file = "coverage-6.4.2-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:2b20286c2b726f94e766e86a3fddb7b7e37af5d0c635bdfa7e4399bc523563de"}, - {file = "coverage-6.4.2-cp38-cp38-win32.whl", hash = "sha256:d714af0bdba67739598849c9f18efdcc5a0412f4993914a0ec5ce0f1e864d783"}, - {file = "coverage-6.4.2-cp38-cp38-win_amd64.whl", hash = "sha256:5f65e5d3ff2d895dab76b1faca4586b970a99b5d4b24e9aafffc0ce94a6022d6"}, - {file = "coverage-6.4.2-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:a697977157adc052284a7160569b36a8bbec09db3c3220642e6323b47cec090f"}, - {file = "coverage-6.4.2-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:c77943ef768276b61c96a3eb854eba55633c7a3fddf0a79f82805f232326d33f"}, - {file = "coverage-6.4.2-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:54d8d0e073a7f238f0666d3c7c0d37469b2aa43311e4024c925ee14f5d5a1cbe"}, - {file = "coverage-6.4.2-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:f22325010d8824594820d6ce84fa830838f581a7fd86a9235f0d2ed6deb61e29"}, - {file = "coverage-6.4.2-cp39-cp39-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:24b04d305ea172ccb21bee5bacd559383cba2c6fcdef85b7701cf2de4188aa55"}, - {file = "coverage-6.4.2-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:866ebf42b4c5dbafd64455b0a1cd5aa7b4837a894809413b930026c91e18090b"}, - {file = "coverage-6.4.2-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:e36750fbbc422c1c46c9d13b937ab437138b998fe74a635ec88989afb57a3978"}, - {file = "coverage-6.4.2-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:79419370d6a637cb18553ecb25228893966bd7935a9120fa454e7076f13b627c"}, - {file = "coverage-6.4.2-cp39-cp39-win32.whl", hash = "sha256:b5e28db9199dd3833cc8a07fa6cf429a01227b5d429facb56eccd765050c26cd"}, - {file = "coverage-6.4.2-cp39-cp39-win_amd64.whl", hash = "sha256:edfdabe7aa4f97ed2b9dd5dde52d2bb29cb466993bb9d612ddd10d0085a683cf"}, - {file = "coverage-6.4.2-pp36.pp37.pp38-none-any.whl", hash = "sha256:e2618cb2cf5a7cc8d698306e42ebcacd02fb7ef8cfc18485c59394152c70be97"}, - {file = "coverage-6.4.2.tar.gz", hash = "sha256:6c3ccfe89c36f3e5b9837b9ee507472310164f352c9fe332120b764c9d60adbe"}, -] -distlib = [ - {file = "distlib-0.3.5-py2.py3-none-any.whl", hash = "sha256:b710088c59f06338ca514800ad795a132da19fda270e3ce4affc74abf955a26c"}, - {file = "distlib-0.3.5.tar.gz", hash = "sha256:a7f75737c70be3b25e2bee06288cec4e4c221de18455b2dd037fe2a795cab2fe"}, -] +coverage = [] +distlib = [] docutils = [ {file = "docutils-0.19-py3-none-any.whl", hash = "sha256:5e1de4d849fee02c63b040a4a3fd567f4ab104defd8a5511fbbc24a8a017efbc"}, {file = "docutils-0.19.tar.gz", hash = "sha256:33995a6753c30b7f577febfc2c50411fec6aac7f7ffeb7c4cfe5991072dcf9e6"}, ] -fastavro = [ - {file = "fastavro-1.5.3-cp310-cp310-macosx_10_14_x86_64.whl", hash = "sha256:bcaf1c9ead0aa1c7306cff229f2c207d714f7f765e0bb010e64c74d379008555"}, - {file = "fastavro-1.5.3-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:8ce66549bdbc9d43f40e1858796c1db59fa9f63f2368217071929c39f7ee7c1e"}, - {file = "fastavro-1.5.3-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:2c14c72f79bcb689edea06ec276bfd5c4d4f17b8e32f545e32a6ee108c5f09de"}, - {file = "fastavro-1.5.3-cp310-cp310-win_amd64.whl", hash = "sha256:6ad6af14d5e927e0c5fbc5dcb7c7030ef8759c8011620344a57252c263d6dd5a"}, - {file = "fastavro-1.5.3-cp37-cp37m-macosx_10_15_x86_64.whl", hash = "sha256:3a4fa9f72c9b134fcdb0ee30d4a838f1d30656ece72a95f18cc74db10148ee55"}, - {file = "fastavro-1.5.3-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:f8980f2bc1230aac509273938d23c544283a476bf84ac79b80540d503f941fe5"}, - {file = "fastavro-1.5.3-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:78ffb34790791a82edef9cce7e49e44faf1b6b0b1da12e9c7d45f31fd6655a7b"}, - {file = "fastavro-1.5.3-cp37-cp37m-win_amd64.whl", hash = "sha256:86a33b2f819cad67f002250e9c6a62909fc31e048679da5f2ac02f4bbf0a5998"}, - {file = "fastavro-1.5.3-cp38-cp38-macosx_10_15_x86_64.whl", hash = "sha256:c5043cfbdfed27f1331860f1d215db838b92d575e8a78915c52c3f6c64d42176"}, - {file = "fastavro-1.5.3-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:badcbd1c34f446f4b64ade84f9517ef7d52197e0ea9dd6b3775274f2271dba9a"}, - {file = "fastavro-1.5.3-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:259ade020a70870858b656219d25f566eb620e52967070f9e18b040f1c0cd482"}, - {file = "fastavro-1.5.3-cp38-cp38-win_amd64.whl", hash = "sha256:b0408d6b02abcbd4e8866b6b027d7cdd20d570e78177fb7c363fb3cc8ecb02a3"}, - {file = "fastavro-1.5.3-cp39-cp39-macosx_10_15_x86_64.whl", hash = "sha256:063f17e000b7a4174ad362ffc91b75de85970cc67fccfed2bd12e21908de6028"}, - {file = "fastavro-1.5.3-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:9b399317d4f559d8f3ef695ef8db8ea230e4b489437d2a1e50307120e181e831"}, - {file = "fastavro-1.5.3-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:298e98c55e98f579df66e8258a942494a64b5340212968ed1b6bd9a68c52db5f"}, - {file = "fastavro-1.5.3-cp39-cp39-win_amd64.whl", hash = "sha256:14c23dc870785d860f1523e7be7ce4079dc36070f1bad18139304a5f4d654180"}, - {file = "fastavro-1.5.3.tar.gz", hash = "sha256:262a9f629448a0d4941a4871fd958b70a01f48a0106644b40e4fbaf984f5e89e"}, -] +fastavro = [] filelock = [ {file = "filelock-3.7.1-py3-none-any.whl", hash = "sha256:37def7b658813cda163b56fc564cdc75e86d338246458c4c28ae84cabefa2404"}, {file = "filelock-3.7.1.tar.gz", hash = "sha256:3a0fd85166ad9dbab54c9aec96737b744106dc5f15c0b09a6744a445299fcf04"}, ] -identify = [ - {file = "identify-2.5.2-py2.py3-none-any.whl", hash = "sha256:feaa9db2dc0ce333b453ce171c0cf1247bbfde2c55fc6bb785022d411a1b78b5"}, - {file = "identify-2.5.2.tar.gz", hash = "sha256:a3d4c096b384d50d5e6dc5bc8b9bc44f1f61cefebd750a7b3e9f939b53fb214d"}, +identify = [] +idna = [ + {file = "idna-3.3-py3-none-any.whl", hash = "sha256:84d9dd047ffa80596e0f246e2eab0b391788b0503584e8945f2368256d2735ff"}, + {file = "idna-3.3.tar.gz", hash = "sha256:9d643ff0a55b762d5cdb124b8eaa99c66322e2157b69160bc32796e824360e6d"}, ] importlib-metadata = [ {file = "importlib_metadata-4.12.0-py3-none-any.whl", hash = "sha256:7401a975809ea1fdc658c3aa4f78cc2195a0e019c5cbc4c06122884e9ae80c23"}, @@ -617,30 +635,7 @@ nodeenv = [ {file = "nodeenv-1.7.0-py2.py3-none-any.whl", hash = "sha256:27083a7b96a25f2f5e1d8cb4b6317ee8aeda3bdd121394e5ac54e498028a042e"}, {file = "nodeenv-1.7.0.tar.gz", hash = "sha256:e0e7f7dfb85fc5394c6fe1e8fa98131a2473e04311a45afb6508f7cf1836fa2b"}, ] -numpy = [ - {file = "numpy-1.23.1-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:b15c3f1ed08df4980e02cc79ee058b788a3d0bef2fb3c9ca90bb8cbd5b8a3a04"}, - {file = "numpy-1.23.1-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:9ce242162015b7e88092dccd0e854548c0926b75c7924a3495e02c6067aba1f5"}, - {file = "numpy-1.23.1-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:e0d7447679ae9a7124385ccf0ea990bb85bb869cef217e2ea6c844b6a6855073"}, - {file = "numpy-1.23.1-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:3119daed207e9410eaf57dcf9591fdc68045f60483d94956bee0bfdcba790953"}, - {file = "numpy-1.23.1-cp310-cp310-win32.whl", hash = "sha256:3ab67966c8d45d55a2bdf40701536af6443763907086c0a6d1232688e27e5447"}, - {file = "numpy-1.23.1-cp310-cp310-win_amd64.whl", hash = "sha256:1865fdf51446839ca3fffaab172461f2b781163f6f395f1aed256b1ddc253622"}, - {file = "numpy-1.23.1-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:aeba539285dcf0a1ba755945865ec61240ede5432df41d6e29fab305f4384db2"}, - {file = "numpy-1.23.1-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:7e8229f3687cdadba2c4faef39204feb51ef7c1a9b669247d49a24f3e2e1617c"}, - {file = "numpy-1.23.1-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:68b69f52e6545af010b76516f5daaef6173e73353e3295c5cb9f96c35d755641"}, - {file = "numpy-1.23.1-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:1408c3527a74a0209c781ac82bde2182b0f0bf54dea6e6a363fe0cc4488a7ce7"}, - {file = "numpy-1.23.1-cp38-cp38-win32.whl", hash = "sha256:47f10ab202fe4d8495ff484b5561c65dd59177949ca07975663f4494f7269e3e"}, - {file = "numpy-1.23.1-cp38-cp38-win_amd64.whl", hash = "sha256:37e5ebebb0eb54c5b4a9b04e6f3018e16b8ef257d26c8945925ba8105008e645"}, - {file = "numpy-1.23.1-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:173f28921b15d341afadf6c3898a34f20a0569e4ad5435297ba262ee8941e77b"}, - {file = "numpy-1.23.1-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:876f60de09734fbcb4e27a97c9a286b51284df1326b1ac5f1bf0ad3678236b22"}, - {file = "numpy-1.23.1-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:35590b9c33c0f1c9732b3231bb6a72d1e4f77872390c47d50a615686ae7ed3fd"}, - {file = "numpy-1.23.1-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:a35c4e64dfca659fe4d0f1421fc0f05b8ed1ca8c46fb73d9e5a7f175f85696bb"}, - {file = "numpy-1.23.1-cp39-cp39-win32.whl", hash = "sha256:c2f91f88230042a130ceb1b496932aa717dcbd665350beb821534c5c7e15881c"}, - {file = "numpy-1.23.1-cp39-cp39-win_amd64.whl", hash = "sha256:37ece2bd095e9781a7156852e43d18044fd0d742934833335599c583618181b9"}, - {file = "numpy-1.23.1-pp38-pypy38_pp73-macosx_10_9_x86_64.whl", hash = "sha256:8002574a6b46ac3b5739a003b5233376aeac5163e5dcd43dd7ad062f3e186129"}, - {file = "numpy-1.23.1-pp38-pypy38_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:5d732d17b8a9061540a10fda5bfeabca5785700ab5469a5e9b93aca5e2d3a5fb"}, - {file = "numpy-1.23.1-pp38-pypy38_pp73-win_amd64.whl", hash = "sha256:55df0f7483b822855af67e38fb3a526e787adf189383b4934305565d71c4b148"}, - {file = "numpy-1.23.1.tar.gz", hash = "sha256:d748ef349bfef2e1194b59da37ed5a29c19ea8d7e6342019921ba2ba4fd8b624"}, -] +numpy = [] packaging = [ {file = "packaging-21.3-py3-none-any.whl", hash = "sha256:ef103e05f519cdc783ae24ea4e2e0f508a9c99b2d4969652eed6a2e1ea5bd522"}, {file = "packaging-21.3.tar.gz", hash = "sha256:dd47c42927d89ab911e606518907cc2d3a1f38bbd026385970643f9c5b8ecfeb"}, @@ -657,10 +652,7 @@ pluggy = [ {file = "pluggy-1.0.0-py2.py3-none-any.whl", hash = "sha256:74134bbf457f031a36d68416e1509f34bd5ccc019f0bcc952c7b909d06b37bd3"}, {file = "pluggy-1.0.0.tar.gz", hash = "sha256:4224373bacce55f955a878bf9cfa763c1e360858e330072059e10bad68531159"}, ] -pre-commit = [ - {file = "pre_commit-2.20.0-py2.py3-none-any.whl", hash = "sha256:51a5ba7c480ae8072ecdb6933df22d2f812dc897d5fe848778116129a681aac7"}, - {file = "pre_commit-2.20.0.tar.gz", hash = "sha256:a978dac7bc9ec0bcee55c18a277d553b0f419d259dadb4b9418ff2d00eb43959"}, -] +pre-commit = [] py = [ {file = "py-1.11.0-py2.py3-none-any.whl", hash = "sha256:607c53218732647dff4acdfcd50cb62615cedf612e72d1724fb1a0cc6405b378"}, {file = "py-1.11.0.tar.gz", hash = "sha256:51c75c4126074b472f746a24399ad32f6053d1b34b68d2fa41e558e6f4a98719"}, @@ -835,6 +827,11 @@ pyyaml = [ {file = "PyYAML-6.0-cp39-cp39-win_amd64.whl", hash = "sha256:b3d267842bf12586ba6c734f89d1f5b871df0273157918b0ccefa29deb05c21c"}, {file = "PyYAML-6.0.tar.gz", hash = "sha256:68fb519c14306fec9720a2a5b45bc9f0c8d1b9c72adf45c37baedfcd949c35a2"}, ] +requests = [ + {file = "requests-2.28.1-py3-none-any.whl", hash = "sha256:8fefa2a1a1365bf5520aac41836fbee479da67864514bdb821f31ce07ce65349"}, + {file = "requests-2.28.1.tar.gz", hash = "sha256:7c5599b102feddaa661c826c56ab4fee28bfd17f5abca1ebbe3e7f19d7c97983"}, +] +requests-mock = [] six = [ {file = "six-1.16.0-py2.py3-none-any.whl", hash = "sha256:8abb2f1d86890a2dfb989f9a77cfcfd3e47c2a354b01111771326f8aa26e0254"}, {file = "six-1.16.0.tar.gz", hash = "sha256:1e61c37477a1626458e36f7b1d82aa5c9b094fa4802892072e49de9c60c4c926"}, @@ -851,57 +848,10 @@ typing-extensions = [ {file = "typing_extensions-4.3.0-py3-none-any.whl", hash = "sha256:25642c956049920a5aa49edcdd6ab1e06d7e5d467fc00e0506c44ac86fbfca02"}, {file = "typing_extensions-4.3.0.tar.gz", hash = "sha256:e6d2677a32f47fc7eb2795db1dd15c1f34eff616bcaf2cfb5e997f854fa1c4a6"}, ] +urllib3 = [] virtualenv = [ {file = "virtualenv-20.15.1-py2.py3-none-any.whl", hash = "sha256:b30aefac647e86af6d82bfc944c556f8f1a9c90427b2fb4e3bfbf338cb82becf"}, {file = "virtualenv-20.15.1.tar.gz", hash = "sha256:288171134a2ff3bfb1a2f54f119e77cd1b81c29fc1265a2356f3e8d14c7d58c4"}, ] -zipp = [ - {file = "zipp-3.8.1-py3-none-any.whl", hash = "sha256:47c40d7fe183a6f21403a199b3e4192cca5774656965b0a4988ad2f8feb5f009"}, - {file = "zipp-3.8.1.tar.gz", hash = "sha256:05b45f1ee8f807d0cc928485ca40a07cb491cf092ff587c0df9cb1fd154848d2"}, -] -zstandard = [ - {file = "zstandard-0.18.0-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:ef7e8a200e4c8ac9102ed3c90ed2aa379f6b880f63032200909c1be21951f556"}, - {file = "zstandard-0.18.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:2dc466207016564805e56d28375f4f533b525ff50d6776946980dff5465566ac"}, - {file = "zstandard-0.18.0-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:4a2ee1d4f98447f3e5183ecfce5626f983504a4a0c005fbe92e60fa8e5d547ec"}, - {file = "zstandard-0.18.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:d956e2f03c7200d7e61345e0880c292783ec26618d0d921dcad470cb195bbce2"}, - {file = "zstandard-0.18.0-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_12_i686.manylinux2010_i686.whl", hash = "sha256:ce6f59cba9854fd14da5bfe34217a1501143057313966637b7291d1b0267bd1e"}, - {file = "zstandard-0.18.0-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:a7fa67cba473623848b6e88acf8d799b1906178fd883fb3a1da24561c779593b"}, - {file = "zstandard-0.18.0-cp310-cp310-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:cdb44d7284c8c5dd1b66dfb86dda7f4560fa94bfbbc1d2da749ba44831335e32"}, - {file = "zstandard-0.18.0-cp310-cp310-win32.whl", hash = "sha256:63694a376cde0aa8b1971d06ca28e8f8b5f492779cb6ee1cc46bbc3f019a42a5"}, - {file = "zstandard-0.18.0-cp310-cp310-win_amd64.whl", hash = "sha256:702a8324cd90c74d9c8780d02bf55e79da3193c870c9665ad3a11647e3ad1435"}, - {file = "zstandard-0.18.0-cp36-cp36m-macosx_10_9_x86_64.whl", hash = "sha256:46f679bc5dfd938db4fb058218d9dc4db1336ffaf1ea774ff152ecadabd40805"}, - {file = "zstandard-0.18.0-cp36-cp36m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:dc2a4de9f363b3247d472362a65041fe4c0f59e01a2846b15d13046be866a885"}, - {file = "zstandard-0.18.0-cp36-cp36m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:bd3220d7627fd4d26397211cb3b560ec7cc4a94b75cfce89e847e8ce7fabe32d"}, - {file = "zstandard-0.18.0-cp36-cp36m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_12_i686.manylinux2010_i686.whl", hash = "sha256:39e98cf4773234bd9cebf9f9db730e451dfcfe435e220f8921242afda8321887"}, - {file = "zstandard-0.18.0-cp36-cp36m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:5228e596eb1554598c872a337bbe4e5afe41cd1f8b1b15f2e35b50d061e35244"}, - {file = "zstandard-0.18.0-cp36-cp36m-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:d4a8fd45746a6c31e729f35196e80b8f1e9987c59f5ccb8859d7c6a6fbeb9c63"}, - {file = "zstandard-0.18.0-cp36-cp36m-win32.whl", hash = "sha256:4cbb85f29a990c2fdbf7bc63246567061a362ddca886d7fae6f780267c0a9e67"}, - {file = "zstandard-0.18.0-cp36-cp36m-win_amd64.whl", hash = "sha256:bfa6c8549fa18e6497a738b7033c49f94a8e2e30c5fbe2d14d0b5aa8bbc1695d"}, - {file = "zstandard-0.18.0-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:e02043297c1832f2666cd2204f381bef43b10d56929e13c42c10c732c6e3b4ed"}, - {file = "zstandard-0.18.0-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:7231543d38d2b7e02ef7cc78ef7ffd86419437e1114ff08709fe25a160e24bd6"}, - {file = "zstandard-0.18.0-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:c86befac87445927488f5c8f205d11566f64c11519db223e9d282b945fa60dab"}, - {file = "zstandard-0.18.0-cp37-cp37m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_12_i686.manylinux2010_i686.whl", hash = "sha256:999a4e1768f219826ba3fa2064fab1c86dd72fdd47a42536235478c3bb3ca3e2"}, - {file = "zstandard-0.18.0-cp37-cp37m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:9df59cd1cf3c62075ee2a4da767089d19d874ac3ad42b04a71a167e91b384722"}, - {file = "zstandard-0.18.0-cp37-cp37m-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:1be31e9e3f7607ee0cdd60915410a5968b205d3e7aa83b7fcf3dd76dbbdb39e0"}, - {file = "zstandard-0.18.0-cp37-cp37m-win32.whl", hash = "sha256:490d11b705b8ae9dc845431bacc8dd1cef2408aede176620a5cd0cd411027936"}, - {file = "zstandard-0.18.0-cp37-cp37m-win_amd64.whl", hash = "sha256:266aba27fa9cc5e9091d3d325ebab1fa260f64e83e42516d5e73947c70216a5b"}, - {file = "zstandard-0.18.0-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:8b2260c4e07dd0723eadb586de7718b61acca4083a490dda69c5719d79bc715c"}, - {file = "zstandard-0.18.0-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:3af8c2383d02feb6650e9255491ec7d0824f6e6dd2bbe3e521c469c985f31fb1"}, - {file = "zstandard-0.18.0-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:28723a1d2e4df778573b76b321ebe9f3469ac98988104c2af116dd344802c3f8"}, - {file = "zstandard-0.18.0-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:19cac7108ff2c342317fad6dc97604b47a41f403c8f19d0bfc396dfadc3638b8"}, - {file = "zstandard-0.18.0-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_12_i686.manylinux2010_i686.whl", hash = "sha256:76725d1ee83a8915100a310bbad5d9c1fc6397410259c94033b8318d548d9990"}, - {file = "zstandard-0.18.0-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:d716a7694ce1fa60b20bc10f35c4a22be446ef7f514c8dbc8f858b61976de2fb"}, - {file = "zstandard-0.18.0-cp38-cp38-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:49685bf9a55d1ab34bd8423ea22db836ba43a181ac6b045ac4272093d5cb874e"}, - {file = "zstandard-0.18.0-cp38-cp38-win32.whl", hash = "sha256:1af1268a7dc870eb27515fb8db1f3e6c5a555d2b7bcc476fc3bab8886c7265ab"}, - {file = "zstandard-0.18.0-cp38-cp38-win_amd64.whl", hash = "sha256:1dc2d3809e763055a1a6c1a73f2b677320cc9a5aa1a7c6cfb35aee59bddc42d9"}, - {file = "zstandard-0.18.0-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:eea18c1e7442f2aa9aff1bb84550dbb6a1f711faf6e48e7319de8f2b2e923c2a"}, - {file = "zstandard-0.18.0-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:8677ffc6a6096cccbd892e558471c901fd821aba12b7fbc63833c7346f549224"}, - {file = "zstandard-0.18.0-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:083dc08abf03807af9beeb2b6a91c23ad78add2499f828176a3c7b742c44df02"}, - {file = "zstandard-0.18.0-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:c990063664c08169c84474acecc9251ee035871589025cac47c060ff4ec4bc1a"}, - {file = "zstandard-0.18.0-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_12_i686.manylinux2010_i686.whl", hash = "sha256:533db8a6fac6248b2cb2c935e7b92f994efbdeb72e1ffa0b354432e087bb5a3e"}, - {file = "zstandard-0.18.0-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:dbb3cb8a082d62b8a73af42291569d266b05605e017a3d8a06a0e5c30b5f10f0"}, - {file = "zstandard-0.18.0-cp39-cp39-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:d6c85ca5162049ede475b7ec98e87f9390501d44a3d6776ddd504e872464ec25"}, - {file = "zstandard-0.18.0-cp39-cp39-win32.whl", hash = "sha256:75479e7c2b3eebf402c59fbe57d21bc400cefa145ca356ee053b0a08908c5784"}, - {file = "zstandard-0.18.0-cp39-cp39-win_amd64.whl", hash = "sha256:d85bfabad444812133a92fc6fbe463e1d07581dba72f041f07a360e63808b23c"}, - {file = "zstandard-0.18.0.tar.gz", hash = "sha256:0ac0357a0d985b4ff31a854744040d7b5754385d1f98f7145c30e02c6865cb6f"}, -] +zipp = [] +zstandard = [] diff --git a/pyiceberg/catalog/base.py b/pyiceberg/catalog/base.py index 6a2dcf9e02..d70fd7b57c 100644 --- a/pyiceberg/catalog/base.py +++ b/pyiceberg/catalog/base.py @@ -18,11 +18,20 @@ from __future__ import annotations from abc import ABC, abstractmethod +from dataclasses import dataclass from pyiceberg.catalog import Identifier, Properties from pyiceberg.schema import Schema from pyiceberg.table.base import Table -from pyiceberg.table.partitioning import PartitionSpec +from pyiceberg.table.partitioning import UNPARTITIONED_PARTITION_SPEC, PartitionSpec +from pyiceberg.table.sorting import UNSORTED_SORT_ORDER, SortOrder + + +@dataclass +class PropertiesUpdateSummary: + removed: list[str] + updated: list[str] + missing: list[str] class Catalog(ABC): @@ -57,7 +66,8 @@ def create_table( identifier: str | Identifier, schema: Schema, location: str | None = None, - partition_spec: PartitionSpec | None = None, + partition_spec: PartitionSpec = UNPARTITIONED_PARTITION_SPEC, + sort_order: SortOrder = UNSORTED_SORT_ORDER, properties: Properties | None = None, ) -> Table: """Create a table @@ -66,7 +76,8 @@ def create_table( identifier: Table identifier. schema: Table's schema. location: Location for the table. Optional Argument. - partition_spec: PartitionSpec for the table. Optional Argument. + partition_spec: PartitionSpec for the table. + sort_order: SortOrder for the table. properties: Table properties that can be a string based dictionary. Optional Argument. Returns: @@ -195,7 +206,7 @@ def load_namespace_properties(self, namespace: str | Identifier) -> Properties: @abstractmethod def update_namespace_properties( self, namespace: str | Identifier, removals: set[str] | None = None, updates: Properties | None = None - ) -> None: + ) -> PropertiesUpdateSummary: """Removes provided property keys and updates properties for a namespace. Args: diff --git a/pyiceberg/catalog/rest.py b/pyiceberg/catalog/rest.py new file mode 100644 index 0000000000..fc228295c9 --- /dev/null +++ b/pyiceberg/catalog/rest.py @@ -0,0 +1,434 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +from json import JSONDecodeError +from typing import ( + Dict, + List, + Optional, + Set, + Tuple, + Type, + Union, +) + +import requests +from pydantic import Field +from requests import HTTPError + +from pyiceberg import __version__ +from pyiceberg.catalog import Identifier, Properties +from pyiceberg.catalog.base import Catalog, PropertiesUpdateSummary +from pyiceberg.exceptions import ( + AlreadyExistsError, + AuthorizationExpiredError, + BadCredentialsError, + BadRequestError, + ForbiddenError, + NoSuchNamespaceError, + NoSuchTableError, + RESTError, + ServerError, + ServiceUnavailableError, + TableAlreadyExistsError, + UnauthorizedError, +) +from pyiceberg.schema import Schema +from pyiceberg.table.base import Table +from pyiceberg.table.metadata import TableMetadataV1, TableMetadataV2 +from pyiceberg.table.partitioning import UNPARTITIONED_PARTITION_SPEC, PartitionSpec +from pyiceberg.table.sorting import UNSORTED_SORT_ORDER, SortOrder +from pyiceberg.utils.iceberg_base_model import IcebergBaseModel + + +class Endpoints: + get_config: str = "config" + list_namespaces: str = "namespaces" + create_namespace: str = "namespaces" + load_namespace_metadata: str = "namespaces/{namespace}" + drop_namespace: str = "namespaces/{namespace}" + update_properties: str = "namespaces/{namespace}/properties" + list_tables: str = "namespaces/{namespace}/tables" + create_table: str = "namespaces/{namespace}/tables" + load_table: str = "namespaces/{namespace}/tables/{table}" + update_table: str = "namespaces/{namespace}/tables/{table}" + drop_table: str = "namespaces/{namespace}/tables/{table}?purgeRequested={purge}" + table_exists: str = "namespaces/{namespace}/tables/{table}" + get_token: str = "oauth/tokens" + rename_table: str = "tables/rename" + + +AUTHORIZATION_HEADER = "Authorization" +BEARER_PREFIX = "Bearer" +CATALOG_SCOPE = "catalog" +CLIENT_ID = "client_id" +PREFIX = "prefix" +CLIENT_SECRET = "client_secret" +CLIENT_CREDENTIALS = "client_credentials" +CREDENTIAL = "credential" +GRANT_TYPE = "grant_type" +SCOPE = "scope" +TOKEN_EXCHANGE = "urn:ietf:params:oauth:grant-type:token-exchange" + +NAMESPACE_SEPARATOR = b"\x1F".decode("UTF-8") + + +class TableResponse(IcebergBaseModel): + metadata_location: Optional[str] = Field(alias="metadata-location", default=None) + metadata: Union[TableMetadataV1, TableMetadataV2] = Field() + config: Properties = Field(default_factory=dict) + + +class CreateTableRequest(IcebergBaseModel): + name: str = Field() + location: Optional[str] = Field() + table_schema: Schema = Field(alias="schema") + partition_spec: Optional[PartitionSpec] = Field(alias="partition-spec") + write_order: Optional[SortOrder] = Field(alias="write-order") + stage_create: bool = Field(alias="stage-create", default=False) + properties: Properties = Field(default_factory=dict) + + +class TokenResponse(IcebergBaseModel): + access_token: str = Field() + token_type: str = Field() + expires_in: int = Field() + issued_token_type: str = Field() + + +class ConfigResponse(IcebergBaseModel): + defaults: Properties = Field() + overrides: Properties = Field() + + +class ListNamespaceResponse(IcebergBaseModel): + namespaces: List[Identifier] = Field() + + +class NamespaceResponse(IcebergBaseModel): + namespace: Identifier = Field() + properties: Properties = Field() + + +class UpdateNamespacePropertiesResponse(IcebergBaseModel): + removed: List[str] = Field() + updated: List[str] = Field() + missing: List[str] = Field() + + +class ListTableResponseEntry(IcebergBaseModel): + name: str = Field() + namespace: Identifier = Field() + + +class ListTablesResponse(IcebergBaseModel): + identifiers: List[ListTableResponseEntry] = Field() + + +class ErrorResponseMessage(IcebergBaseModel): + message: str = Field() + type: str = Field() + code: int = Field() + + +class ErrorResponse(IcebergBaseModel): + error: ErrorResponseMessage = Field() + + +class RestCatalog(Catalog): + token: str + config: Properties + + host: str + + def __init__( + self, + name: str, + properties: Properties, + host: str, + client_id: Optional[str] = None, + client_secret: Optional[str] = None, + token: Optional[str] = None, + ): + """Rest Catalog + + You either need to provide a client_id and client_secret, or an already valid token. + + Args: + name: Name to identify the catalog + properties: Properties that are passed along to the configuration + host: The base-url of the REST Catalog endpoint + client_id: The id to identify the client + client_secret: The secret for the client + token: The bearer token + """ + self.host = host + if client_id and client_secret: + self.token = self._fetch_access_token(client_id, client_secret) + elif token: + self.token = token + else: + raise ValueError("Either set the client_id and client_secret, or provide a valid token") + self.config = self._fetch_config(properties) + super().__init__(name, properties) + + @staticmethod + def _split_credential(token: str) -> Tuple[str, str]: + """Splits the token in a client id and secret + + Args: + token: The token with a semicolon as a separator + + Returns: + The client id and secret + """ + client, secret = token.split(":") + return client, secret + + @property + def headers(self) -> Properties: + return { + AUTHORIZATION_HEADER: f"{BEARER_PREFIX} {self.token}", + "Content-type": "application/json", + "X-Client-Version": __version__, + } + + def url(self, endpoint: str, prefixed: bool = True, **kwargs) -> str: + """Constructs the endpoint + + Args: + prefixed: If the prefix return by the config needs to be appended + + Returns: + The base url of the rest catalog + """ + + url = self.host + url = url + "v1/" if url.endswith("/") else url + "/v1/" + + if prefixed: + url += self.config.get(PREFIX, "") + url = url if url.endswith("/") else url + "/" + + return url + endpoint.format(**kwargs) + + def _fetch_access_token(self, client_id: str, client_secret: str) -> str: + data = {GRANT_TYPE: CLIENT_CREDENTIALS, CLIENT_ID: client_id, CLIENT_SECRET: client_secret, SCOPE: CATALOG_SCOPE} + url = self.url(Endpoints.get_token, prefixed=False) + # Uses application/x-www-form-urlencoded by default + response = requests.post(url=url, data=data) + try: + response.raise_for_status() + except HTTPError as exc: + self._handle_non_200_response(exc, {401: BadCredentialsError}) + + return TokenResponse(**response.json()).access_token + + def _fetch_config(self, properties: Properties) -> Properties: + response = requests.get(self.url(Endpoints.get_config, prefixed=False), headers=self.headers) + response.raise_for_status() + config_response = ConfigResponse(**response.json()) + config = config_response.defaults + config.update(properties) + config.update(config_response.overrides) + return config + + def _split_identifier_for_path(self, identifier: Union[str, Identifier]) -> Properties: + identifier = self.identifier_to_tuple(identifier) + return {"namespace": NAMESPACE_SEPARATOR.join(identifier[:-1]), "table": identifier[-1]} + + def _split_identifier_for_json(self, identifier: Union[str, Identifier]) -> Dict[str, Union[Identifier, str]]: + identifier = self.identifier_to_tuple(identifier) + return {"namespace": identifier[:-1], "name": identifier[-1]} + + def _handle_non_200_response(self, exc: HTTPError, error_handler: Dict[int, Type[Exception]]): + try: + response = ErrorResponse(**exc.response.json()) + except JSONDecodeError: + # In the case we don't have a proper response + response = ErrorResponse( + error=ErrorResponseMessage( + message=f"Could not decode json payload: {exc.response.text}", + type="RESTError", + code=exc.response.status_code, + ) + ) + + code = exc.response.status_code + if code in error_handler: + raise error_handler[code](response.error.message) from exc + elif code == 400: + raise BadRequestError(response.error.message) from exc + elif code == 401: + raise UnauthorizedError(response.error.message) from exc + elif code == 403: + raise ForbiddenError(response.error.message) from exc + elif code == 422: + raise RESTError(response.error.message) from exc + elif code == 419: + raise AuthorizationExpiredError(response.error.message) + elif code == 501: + raise NotImplementedError(response.error.message) + elif code == 503: + raise ServiceUnavailableError(response.error.message) from exc + elif 500 <= code < 600: + raise ServerError(response.error.message) from exc + else: + raise RESTError(response.error.message) from exc + + def create_table( + self, + identifier: Union[str, Identifier], + schema: Schema, + location: Optional[str] = None, + partition_spec: PartitionSpec = UNPARTITIONED_PARTITION_SPEC, + sort_order: SortOrder = UNSORTED_SORT_ORDER, + properties: Optional[Properties] = None, + ) -> Table: + namespace_and_table = self._split_identifier_for_path(identifier) + properties = properties or {} + request = CreateTableRequest( + name=namespace_and_table["table"], + location=location, + table_schema=schema, + partition_spec=partition_spec, + write_order=sort_order, + properties=properties, + ) + serialized_json = request.json() + response = requests.post( + self.url(Endpoints.create_table, namespace=namespace_and_table["namespace"]), + data=serialized_json, + headers=self.headers, + ) + try: + response.raise_for_status() + except HTTPError as exc: + self._handle_non_200_response(exc, {409: TableAlreadyExistsError}) + + table_response = TableResponse(**response.json()) + + return Table( + identifier=(self.name,) + self.identifier_to_tuple(identifier), + metadata_location=table_response.metadata_location, + metadata=table_response.metadata, + ) + + def list_tables(self, namespace: Optional[Union[str, Identifier]] = None) -> List[Identifier]: + namespace_concat = NAMESPACE_SEPARATOR.join(self.identifier_to_tuple(namespace or "")) + response = requests.get( + self.url(Endpoints.list_tables, namespace=namespace_concat), + headers=self.headers, + ) + try: + response.raise_for_status() + except HTTPError as exc: + self._handle_non_200_response(exc, {404: NoSuchNamespaceError}) + return [(*table.namespace, table.name) for table in ListTablesResponse(**response.json()).identifiers] + + def load_table(self, identifier: Union[str, Identifier]) -> Table: + response = requests.get( + self.url(Endpoints.load_table, prefixed=True, **self._split_identifier_for_path(identifier)), headers=self.headers + ) + try: + response.raise_for_status() + except HTTPError as exc: + self._handle_non_200_response(exc, {404: NoSuchTableError}) + + table_response = TableResponse(**response.json()) + + return Table( + identifier=(self.name,) + self.identifier_to_tuple(identifier), + metadata_location=table_response.metadata_location, + metadata=table_response.metadata, + ) + + def drop_table(self, identifier: Union[str, Identifier], purge_requested: bool = False) -> None: + response = requests.delete( + self.url(Endpoints.drop_table, prefixed=True, purge=purge_requested, **self._split_identifier_for_path(identifier)), + headers=self.headers, + ) + try: + response.raise_for_status() + except HTTPError as exc: + self._handle_non_200_response(exc, {404: NoSuchTableError}) + + def purge_table(self, identifier: Union[str, Identifier]) -> None: + self.drop_table(identifier=identifier, purge_requested=True) + + def rename_table(self, from_identifier: Union[str, Identifier], to_identifier: Union[str, Identifier]): + payload = { + "source": self._split_identifier_for_json(from_identifier), + "destination": self._split_identifier_for_json(to_identifier), + } + response = requests.post(self.url(Endpoints.rename_table), json=payload, headers=self.headers) + try: + response.raise_for_status() + except HTTPError as exc: + self._handle_non_200_response(exc, {404: NoSuchTableError, 409: TableAlreadyExistsError}) + + def create_namespace(self, namespace: Union[str, Identifier], properties: Optional[Properties] = None) -> None: + payload = {"namespace": self.identifier_to_tuple(namespace), "properties": properties or {}} + response = requests.post(self.url(Endpoints.create_namespace), json=payload, headers=self.headers) + try: + response.raise_for_status() + except HTTPError as exc: + self._handle_non_200_response(exc, {404: NoSuchNamespaceError, 409: AlreadyExistsError}) + + def drop_namespace(self, namespace: Union[str, Identifier]) -> None: + namespace = NAMESPACE_SEPARATOR.join(self.identifier_to_tuple(namespace)) + response = requests.delete(self.url(Endpoints.drop_namespace, namespace=namespace), headers=self.headers) + try: + response.raise_for_status() + except HTTPError as exc: + self._handle_non_200_response(exc, {404: NoSuchNamespaceError}) + + def list_namespaces(self) -> List[Identifier]: + response = requests.get(self.url(Endpoints.list_namespaces), headers=self.headers) + response.raise_for_status() + namespaces = ListNamespaceResponse(**response.json()) + try: + response.raise_for_status() + except HTTPError as exc: + self._handle_non_200_response(exc, {}) + return namespaces.namespaces + + def load_namespace_properties(self, namespace: Union[str, Identifier]) -> Properties: + namespace = NAMESPACE_SEPARATOR.join(self.identifier_to_tuple(namespace)) + response = requests.get(self.url(Endpoints.load_namespace_metadata, namespace=namespace), headers=self.headers) + try: + response.raise_for_status() + except HTTPError as exc: + self._handle_non_200_response(exc, {404: NoSuchNamespaceError}) + + return NamespaceResponse(**response.json()).properties + + def update_namespace_properties( + self, namespace: Union[str, Identifier], removals: Optional[Set[str]] = None, updates: Optional[Properties] = None + ) -> PropertiesUpdateSummary: + namespace = NAMESPACE_SEPARATOR.join(self.identifier_to_tuple(namespace)) + payload = {"removals": list(removals or []), "updates": updates} + response = requests.post(self.url(Endpoints.update_properties, namespace=namespace), json=payload, headers=self.headers) + try: + response.raise_for_status() + except HTTPError as exc: + self._handle_non_200_response(exc, {404: NoSuchNamespaceError}) + parsed_response = UpdateNamespacePropertiesResponse(**response.json()) + return PropertiesUpdateSummary( + removed=parsed_response.removed, + updated=parsed_response.updated, + missing=parsed_response.missing, + ) diff --git a/pyiceberg/exceptions.py b/pyiceberg/exceptions.py index f9ac3333b1..f38cd0df67 100644 --- a/pyiceberg/exceptions.py +++ b/pyiceberg/exceptions.py @@ -16,12 +16,8 @@ # under the License. -class NoSuchTableError(Exception): - """Raised when a referenced table is not found""" - - -class NoSuchNamespaceError(Exception): - """Raised when a referenced name-space is not found""" +class TableAlreadyExistsError(Exception): + """Raised when creating a table with a name that already exists""" class NamespaceNotEmptyError(Exception): @@ -34,3 +30,43 @@ class AlreadyExistsError(Exception): class ValidationError(Exception): """Raises when there is an issue with the schema""" + + +class RESTError(Exception): + """Raises when there is an unknown response from the REST Catalog""" + + +class BadCredentialsError(RESTError): + """Raises when providing invalid credentials""" + + +class BadRequestError(RESTError): + """Raises when an invalid request is being made""" + + +class UnauthorizedError(RESTError): + """Raises when you don't have the proper authorization""" + + +class ServiceUnavailableError(RESTError): + """Raises when the service doesn't respond""" + + +class ServerError(RESTError): + """Raises when there is an unhandled exception on the server side""" + + +class ForbiddenError(RESTError): + """Raises when you don't have the credentials to perform the action on the REST catalog""" + + +class AuthorizationExpiredError(RESTError): + """When the credentials are expired when performing an action on the REST catalog""" + + +class NoSuchTableError(RESTError): + """Raises when the table can't be found in the REST catalog""" + + +class NoSuchNamespaceError(RESTError): + """Raised when a referenced name-space is not found""" diff --git a/pyiceberg/schema.py b/pyiceberg/schema.py index ea4f4cf6c0..5e5d4af56d 100644 --- a/pyiceberg/schema.py +++ b/pyiceberg/schema.py @@ -24,6 +24,7 @@ Dict, Generic, List, + Literal, Optional, Tuple, TypeVar, @@ -54,6 +55,7 @@ class Schema(IcebergBaseModel): >>> from pyiceberg import types """ + type: Literal["struct"] = "struct" fields: Tuple[NestedField, ...] = Field(default_factory=tuple) schema_id: int = Field(alias="schema-id") identifier_field_ids: List[int] = Field(alias="identifier-field-ids", default_factory=list) diff --git a/pyiceberg/table/base.py b/pyiceberg/table/base.py index 77821547e7..5e7c06a5b2 100644 --- a/pyiceberg/table/base.py +++ b/pyiceberg/table/base.py @@ -15,17 +15,16 @@ # specific language governing permissions and limitations # under the License. -from __future__ import annotations +from typing import Optional, Union -from abc import ABC +from pydantic import Field from pyiceberg.catalog.base import Identifier +from pyiceberg.table.metadata import TableMetadataV1, TableMetadataV2 +from pyiceberg.utils.iceberg_base_model import IcebergBaseModel -class Table(ABC): - """Placeholder for Table managed by the Catalog that points to the current Table Metadata. - - To be implemented by https://github.com/apache/iceberg/issues/3227 - """ - - identifier: str | Identifier +class Table(IcebergBaseModel): + identifier: Identifier = Field() + metadata_location: Optional[str] = Field() + metadata: Union[TableMetadataV1, TableMetadataV2] = Field() diff --git a/pyiceberg/table/metadata.py b/pyiceberg/table/metadata.py index 6543d4ce30..9608bf138b 100644 --- a/pyiceberg/table/metadata.py +++ b/pyiceberg/table/metadata.py @@ -38,6 +38,7 @@ INITIAL_SEQUENCE_NUMBER = 0 INITIAL_SPEC_ID = 0 DEFAULT_SCHEMA_ID = 0 +DEFAULT_LAST_PARTITION_ID = 1000 def check_schemas(values: Dict[str, Any]) -> Dict[str, Any]: @@ -103,7 +104,7 @@ def construct_refs(cls, data: Dict[str, Any]): """The table’s base location. This is used by writers to determine where to store data files, manifest files, and table metadata files.""" - table_uuid: Optional[UUID] = Field(alias="table-uuid") + table_uuid: Optional[UUID] = Field(alias="table-uuid", default_factory=uuid4) """A UUID that identifies the table, generated when the table is created. Implementations must throw an exception if a table’s UUID does not match the expected UUID after refreshing metadata.""" @@ -210,12 +211,10 @@ def set_v2_compatible_defaults(cls, data: Dict[str, Any]) -> Dict[str, Any]: Returns: The TableMetadata with the defaults applied """ - if "schema-id" not in data["schema"]: + if data.get("schema") and "schema-id" not in data["schema"]: data["schema"]["schema-id"] = DEFAULT_SCHEMA_ID - if "last-partition-id" not in data: + if data.get("partition-spec") and "last-partition-id" not in data: data["last-partition-id"] = max(spec["field-id"] for spec in data["partition-spec"]) - if "table-uuid" not in data: - data["table-uuid"] = uuid4() return data @root_validator(skip_on_failure=True) @@ -236,7 +235,7 @@ def construct_schemas(cls, data: Dict[str, Any]) -> Dict[str, Any]: schema = data["schema_"] data["schemas"] = [schema] else: - check_schemas(data["schemas"]) + check_schemas(data) return data @root_validator(skip_on_failure=True) @@ -257,7 +256,7 @@ def construct_partition_specs(cls, data: Dict[str, Any]) -> Dict[str, Any]: fields = data["partition_spec"] data["partition_specs"] = [PartitionSpec(spec_id=INITIAL_SPEC_ID, fields=fields)] else: - check_partition_specs(data["partition_specs"]) + check_partition_specs(data) return data @root_validator(skip_on_failure=True) @@ -273,10 +272,10 @@ def set_sort_orders(cls, data: Dict[str, Any]): Returns: The TableMetadata with the sort_orders set, if not provided """ - if sort_orders := data.get("sort_orders"): - check_sort_orders(sort_orders) - else: + if not data.get("sort_orders"): data["sort_orders"] = [UNSORTED_SORT_ORDER] + else: + check_sort_orders(data) return data def to_v2(self) -> "TableMetadataV2": diff --git a/pyiceberg/table/partitioning.py b/pyiceberg/table/partitioning.py index ef080bc307..f5222ad595 100644 --- a/pyiceberg/table/partitioning.py +++ b/pyiceberg/table/partitioning.py @@ -81,7 +81,7 @@ class PartitionSpec(IcebergBaseModel): """ spec_id: int = Field(alias="spec-id") - fields: Tuple[PartitionField, ...] = Field() + fields: Tuple[PartitionField, ...] = Field(default_factory=tuple) def __init__( self, @@ -154,3 +154,6 @@ def compatible_with(self, other: "PartitionSpec") -> bool: and this_field.name == that_field.name for this_field, that_field in zip(self.fields, other.fields) ) + + +UNPARTITIONED_PARTITION_SPEC = PartitionSpec(spec_id=0) diff --git a/pyiceberg/utils/iceberg_base_model.py b/pyiceberg/utils/iceberg_base_model.py index e218f9f89b..e3ad2eab0f 100644 --- a/pyiceberg/utils/iceberg_base_model.py +++ b/pyiceberg/utils/iceberg_base_model.py @@ -37,6 +37,7 @@ class IcebergBaseModel(BaseModel): class Config: keep_untouched = (cached_property,) allow_population_by_field_name = True + copy_on_model_validation = False frozen = True def dict(self, exclude_none: bool = True, **kwargs): diff --git a/pyproject.toml b/pyproject.toml index 24cae0c813..55011bcaaa 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -40,6 +40,8 @@ packages = [ [tool.poetry.dependencies] python = "^3.8" mmh3 = "^3.0.0" +requests = "^2.28.1" + pydantic = "^1.9.1" pyarrow = { version = "^8.0.0", optional = true } @@ -54,6 +56,7 @@ pytest-checkdocs = "^2.0.0" pre-commit = "^2.0.0" fastavro = "^1.5.1" coverage = { version = "^6.0.0", extras = ["toml"] } +requests-mock = "^1.9.3" [build-system] requires = ["poetry-core>=1.0.0"] @@ -111,5 +114,9 @@ ignore_missing_imports = true module = "mmh3.*" ignore_missing_imports = true +[[tool.mypy.overrides]] +module = "requests_mock.*" +ignore_missing_imports = true + [tool.coverage.run] source = ['pyiceberg/'] diff --git a/tests/catalog/test_base.py b/tests/catalog/test_base.py index e4dd4d3992..49790ecbf3 100644 --- a/tests/catalog/test_base.py +++ b/tests/catalog/test_base.py @@ -26,7 +26,7 @@ import pytest from pyiceberg.catalog import Identifier, Properties -from pyiceberg.catalog.base import Catalog +from pyiceberg.catalog.base import Catalog, PropertiesUpdateSummary from pyiceberg.exceptions import ( AlreadyExistsError, NamespaceNotEmptyError, @@ -36,7 +36,9 @@ from pyiceberg.schema import Schema from pyiceberg.table.base import Table from pyiceberg.table.metadata import INITIAL_SPEC_ID -from pyiceberg.table.partitioning import PartitionSpec +from pyiceberg.table.partitioning import UNPARTITIONED_PARTITION_SPEC, PartitionSpec +from pyiceberg.table.sorting import UNSORTED_SORT_ORDER, SortOrder +from tests.table.test_metadata import EXAMPLE_TABLE_METADATA_V1 class InMemoryCatalog(Catalog): @@ -55,7 +57,8 @@ def create_table( identifier: Union[str, Identifier], schema: Schema, location: Optional[str] = None, - partition_spec: Optional[PartitionSpec] = None, + partition_spec: PartitionSpec = UNPARTITIONED_PARTITION_SPEC, + sort_order: SortOrder = UNSORTED_SORT_ORDER, properties: Optional[Properties] = None, ) -> Table: @@ -68,8 +71,7 @@ def create_table( if namespace not in self.__namespaces: self.__namespaces[namespace] = {} - table = Table() - table.identifier = identifier + table = Table(identifier=identifier, metadata=EXAMPLE_TABLE_METADATA_V1) self.__tables[identifier] = table return table @@ -102,9 +104,8 @@ def rename_table(self, from_identifier: Union[str, Identifier], to_identifier: U if to_namespace not in self.__namespaces: self.__namespaces[to_namespace] = {} - table.identifier = to_identifier - self.__tables[to_identifier] = table - return table + self.__tables[to_identifier] = Table(identifier=to_identifier, metadata=table.metadata) + return self.__tables[to_identifier] def create_namespace(self, namespace: Union[str, Identifier], properties: Optional[Properties] = None) -> None: namespace = Catalog.identifier_to_tuple(namespace) @@ -143,18 +144,30 @@ def load_namespace_properties(self, namespace: Union[str, Identifier]) -> Proper def update_namespace_properties( self, namespace: Union[str, Identifier], removals: Optional[Set[str]] = None, updates: Optional[Properties] = None - ) -> None: + ) -> PropertiesUpdateSummary: + removed: Set[str] = set() + updated: Set[str] = set() + namespace = Catalog.identifier_to_tuple(namespace) if namespace in self.__namespaces: if removals: for key in removals: if key in self.__namespaces[namespace]: del self.__namespaces[namespace][key] + removed.add(key) if updates: - self.__namespaces[namespace].update(updates) + for key, value in updates.items(): + self.__namespaces[namespace][key] = value + updated.add(key) else: raise NoSuchNamespaceError(f"Namespace does not exist: {namespace}") + expected_to_change = removed.difference(removals or set()) + + return PropertiesUpdateSummary( + removed=list(removed or []), updated=list(updates.keys() if updates else []), missing=list(expected_to_change) + ) + TEST_TABLE_IDENTIFIER = ("com", "organization", "department", "my_table") TEST_TABLE_NAMESPACE = ("com", "organization", "department") @@ -383,11 +396,12 @@ def test_update_namespace_metadata(catalog: InMemoryCatalog): # When new_metadata = {"key3": "value3", "key4": "value4"} - catalog.update_namespace_properties(TEST_TABLE_NAMESPACE, updates=new_metadata) + summary = catalog.update_namespace_properties(TEST_TABLE_NAMESPACE, updates=new_metadata) # Then assert TEST_TABLE_NAMESPACE in catalog.list_namespaces() assert new_metadata.items() <= catalog.load_namespace_properties(TEST_TABLE_NAMESPACE).items() + assert summary == PropertiesUpdateSummary(removed=[], updated=["key3", "key4"], missing=[]) def test_update_namespace_metadata_removals(catalog: InMemoryCatalog): @@ -397,12 +411,13 @@ def test_update_namespace_metadata_removals(catalog: InMemoryCatalog): # When new_metadata = {"key3": "value3", "key4": "value4"} remove_metadata = {"key1"} - catalog.update_namespace_properties(TEST_TABLE_NAMESPACE, remove_metadata, new_metadata) + summary = catalog.update_namespace_properties(TEST_TABLE_NAMESPACE, remove_metadata, new_metadata) # Then assert TEST_TABLE_NAMESPACE in catalog.list_namespaces() assert new_metadata.items() <= catalog.load_namespace_properties(TEST_TABLE_NAMESPACE).items() assert remove_metadata.isdisjoint(catalog.load_namespace_properties(TEST_TABLE_NAMESPACE).keys()) + assert summary == PropertiesUpdateSummary(removed=["key1"], updated=["key3", "key4"], missing=[]) def test_update_namespace_metadata_raises_error_when_namespace_does_not_exist(catalog: InMemoryCatalog): diff --git a/tests/catalog/test_rest.py b/tests/catalog/test_rest.py new file mode 100644 index 0000000000..d5ccfe5b58 --- /dev/null +++ b/tests/catalog/test_rest.py @@ -0,0 +1,612 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# pylint: disable=redefined-outer-name +from uuid import UUID + +import pytest +from requests_mock import Mocker + +from pyiceberg.catalog.base import PropertiesUpdateSummary, Table +from pyiceberg.catalog.rest import RestCatalog +from pyiceberg.exceptions import ( + AlreadyExistsError, + BadCredentialsError, + NoSuchNamespaceError, + NoSuchTableError, + TableAlreadyExistsError, +) +from pyiceberg.schema import Schema +from pyiceberg.table.metadata import TableMetadataV1 +from pyiceberg.table.partitioning import PartitionField, PartitionSpec +from pyiceberg.table.refs import SnapshotRef, SnapshotRefType +from pyiceberg.table.snapshots import Operation, Snapshot, Summary +from pyiceberg.table.sorting import SortField, SortOrder +from pyiceberg.transforms import IdentityTransform, TruncateTransform +from pyiceberg.types import ( + BooleanType, + IntegerType, + NestedField, + StringType, +) + +TEST_HOST = "https://iceberg-test-catalog/" +TEST_CLIENT_ID = "client" +TEST_CLIENT_SECRET = "secret" +TEST_TOKEN = "some_jwt_token" + + +@pytest.fixture +def rest_mock(requests_mock: Mocker): + """Takes the default requests_mock and adds the config endpoint to it + + This endpoint is called when initializing the rest catalog + """ + requests_mock.get( + f"{TEST_HOST}v1/config", + json={"defaults": {}, "overrides": {}}, + status_code=200, + ) + return requests_mock + + +def test_token_200(rest_mock: Mocker): + rest_mock.post( + f"{TEST_HOST}v1/oauth/tokens", + json={ + "access_token": TEST_TOKEN, + "token_type": "Bearer", + "expires_in": 86400, + "issued_token_type": "urn:ietf:params:oauth:token-type:access_token", + }, + status_code=200, + ) + assert RestCatalog("rest", {}, TEST_HOST, TEST_CLIENT_ID, TEST_CLIENT_SECRET).token == TEST_TOKEN + + +def test_token_401(rest_mock: Mocker): + message = "Invalid client ID: abc" + rest_mock.post( + f"{TEST_HOST}v1/oauth/tokens", + json={ + "error": { + "message": message, + "type": "BadCredentialsException", + "code": 401, + } + }, + status_code=401, + ) + + with pytest.raises(BadCredentialsError) as e: + RestCatalog("rest", {}, TEST_HOST, client_id=TEST_CLIENT_ID, client_secret=TEST_CLIENT_SECRET) + assert message in str(e.value) + + +def test_list_tables_200(rest_mock: Mocker): + namespace = "examples" + rest_mock.get( + f"{TEST_HOST}v1/namespaces/{namespace}/tables", + json={"identifiers": [{"namespace": ["examples"], "name": "fooshare"}]}, + status_code=200, + ) + + assert RestCatalog("rest", {}, TEST_HOST, token=TEST_TOKEN).list_tables(namespace) == [("examples", "fooshare")] + + +def test_list_tables_404(rest_mock: Mocker): + namespace = "examples" + rest_mock.get( + f"{TEST_HOST}v1/namespaces/{namespace}/tables", + json={ + "error": { + "message": "Namespace does not exist: personal in warehouse 8bcb0838-50fc-472d-9ddb-8feb89ef5f1e", + "type": "NoSuchNamespaceException", + "code": 404, + } + }, + status_code=404, + ) + with pytest.raises(NoSuchNamespaceError) as e: + RestCatalog("rest", {}, TEST_HOST, token=TEST_TOKEN).list_tables(namespace) + assert "Namespace does not exist" in str(e.value) + + +def test_list_namespaces_200(rest_mock: Mocker): + rest_mock.get( + f"{TEST_HOST}v1/namespaces", + json={"namespaces": [["default"], ["examples"], ["fokko"], ["system"]]}, + status_code=200, + ) + assert RestCatalog("rest", {}, TEST_HOST, token=TEST_TOKEN).list_namespaces() == [ + ("default",), + ("examples",), + ("fokko",), + ("system",), + ] + + +def test_create_namespace_200(rest_mock: Mocker): + namespace = "leden" + rest_mock.post( + f"{TEST_HOST}v1/namespaces", + json={"namespace": [namespace], "properties": {}}, + status_code=200, + ) + RestCatalog("rest", {}, TEST_HOST, token=TEST_TOKEN).create_namespace(namespace) + + +def test_create_namespace_409(rest_mock: Mocker): + namespace = "examples" + rest_mock.post( + f"{TEST_HOST}v1/namespaces", + json={ + "error": { + "message": "Namespace already exists: fokko in warehouse 8bcb0838-50fc-472d-9ddb-8feb89ef5f1e", + "type": "AlreadyExistsException", + "code": 409, + } + }, + status_code=409, + ) + with pytest.raises(AlreadyExistsError) as e: + RestCatalog("rest", {}, TEST_HOST, token=TEST_TOKEN).create_namespace(namespace) + assert "Namespace already exists" in str(e.value) + + +def test_drop_namespace_404(rest_mock: Mocker): + namespace = "examples" + rest_mock.delete( + f"{TEST_HOST}v1/namespaces/{namespace}", + json={ + "error": { + "message": "Namespace does not exist: leden in warehouse 8bcb0838-50fc-472d-9ddb-8feb89ef5f1e", + "type": "NoSuchNamespaceException", + "code": 404, + } + }, + status_code=404, + ) + with pytest.raises(NoSuchNamespaceError) as e: + RestCatalog("rest", {}, TEST_HOST, token=TEST_TOKEN).drop_namespace(namespace) + assert "Namespace does not exist" in str(e.value) + + +def test_load_namespace_properties_200(rest_mock: Mocker): + namespace = "leden" + rest_mock.get( + f"{TEST_HOST}v1/namespaces/{namespace}", + json={"namespace": ["fokko"], "properties": {"prop": "yes"}}, + status_code=204, + ) + assert RestCatalog("rest", {}, TEST_HOST, token=TEST_TOKEN).load_namespace_properties(namespace) == {"prop": "yes"} + + +def test_load_namespace_properties_404(rest_mock: Mocker): + namespace = "leden" + rest_mock.get( + f"{TEST_HOST}v1/namespaces/{namespace}", + json={ + "error": { + "message": "Namespace does not exist: fokko22 in warehouse 8bcb0838-50fc-472d-9ddb-8feb89ef5f1e", + "type": "NoSuchNamespaceException", + "code": 404, + } + }, + status_code=404, + ) + with pytest.raises(NoSuchNamespaceError) as e: + RestCatalog("rest", {}, TEST_HOST, token=TEST_TOKEN).load_namespace_properties(namespace) + assert "Namespace does not exist" in str(e.value) + + +def test_update_namespace_properties_200(rest_mock: Mocker): + rest_mock.post( + f"{TEST_HOST}v1/namespaces/fokko/properties", + json={"removed": [], "updated": ["prop"], "missing": ["abc"]}, + status_code=200, + ) + response = RestCatalog("rest", {}, TEST_HOST, token=TEST_TOKEN).update_namespace_properties( + ("fokko",), {"abc"}, {"prop": "yes"} + ) + + assert response == PropertiesUpdateSummary(removed=[], updated=["prop"], missing=["abc"]) + + +def test_update_namespace_properties_404(rest_mock: Mocker): + rest_mock.post( + f"{TEST_HOST}v1/namespaces/fokko/properties", + json={ + "error": { + "message": "Namespace does not exist: does_not_exists in warehouse 8bcb0838-50fc-472d-9ddb-8feb89ef5f1e", + "type": "NoSuchNamespaceException", + "code": 404, + } + }, + status_code=404, + ) + with pytest.raises(NoSuchNamespaceError) as e: + RestCatalog("rest", {}, TEST_HOST, token=TEST_TOKEN).update_namespace_properties(("fokko",), {"abc"}, {"prop": "yes"}) + assert "Namespace does not exist" in str(e.value) + + +def test_load_table_200(rest_mock: Mocker): + rest_mock.get( + f"{TEST_HOST}v1/namespaces/fokko/tables/table", + json={ + "metadataLocation": "s3://tabular-public-us-west-2-dev/bb30733e-8769-4dab-aa1b-e76245bb2bd4/b55d9dda-6561-423a-8bfc-787980ce421f/metadata/00001-5f2f8166-244c-4eae-ac36-384ecdec81fc.gz.metadata.json", + "metadata": { + "format-version": 1, + "table-uuid": "b55d9dda-6561-423a-8bfc-787980ce421f", + "location": "s3://tabular-public-us-west-2-dev/bb30733e-8769-4dab-aa1b-e76245bb2bd4/b55d9dda-6561-423a-8bfc-787980ce421f", + "last-updated-ms": 1646787054459, + "last-column-id": 2, + "schema": { + "type": "struct", + "schema-id": 0, + "fields": [ + {"id": 1, "name": "id", "required": False, "type": "int"}, + {"id": 2, "name": "data", "required": False, "type": "string"}, + ], + }, + "current-schema-id": 0, + "schemas": [ + { + "type": "struct", + "schema-id": 0, + "fields": [ + {"id": 1, "name": "id", "required": False, "type": "int"}, + {"id": 2, "name": "data", "required": False, "type": "string"}, + ], + } + ], + "partition-spec": [], + "default-spec-id": 0, + "partition-specs": [{"spec-id": 0, "fields": []}], + "last-partition-id": 999, + "default-sort-order-id": 0, + "sort-orders": [{"order-id": 0, "fields": []}], + "properties": {"owner": "bryan", "write.metadata.compression-codec": "gzip"}, + "current-snapshot-id": 3497810964824022504, + "refs": {"main": {"snapshot-id": 3497810964824022504, "type": "branch"}}, + "snapshots": [ + { + "snapshot-id": 3497810964824022504, + "timestamp-ms": 1646787054459, + "summary": { + "operation": "append", + "spark.app.id": "local-1646787004168", + "added-data-files": "1", + "added-records": "1", + "added-files-size": "697", + "changed-partition-count": "1", + "total-records": "1", + "total-files-size": "697", + "total-data-files": "1", + "total-delete-files": "0", + "total-position-deletes": "0", + "total-equality-deletes": "0", + }, + "manifest-list": "s3://tabular-public-us-west-2-dev/bb30733e-8769-4dab-aa1b-e76245bb2bd4/b55d9dda-6561-423a-8bfc-787980ce421f/metadata/snap-3497810964824022504-1-c4f68204-666b-4e50-a9df-b10c34bf6b82.avro", + "schema-id": 0, + } + ], + "snapshot-log": [{"timestamp-ms": 1646787054459, "snapshot-id": 3497810964824022504}], + "metadata-log": [ + { + "timestamp-ms": 1646787031514, + "metadata-file": "s3://tabular-public-us-west-2-dev/bb30733e-8769-4dab-aa1b-e76245bb2bd4/b55d9dda-6561-423a-8bfc-787980ce421f/metadata/00000-88484a1c-00e5-4a07-a787-c0e7aeffa805.gz.metadata.json", + } + ], + }, + "config": {"client.factory": "io.tabular.iceberg.catalog.TabularAwsClientFactory", "region": "us-west-2"}, + }, + status_code=200, + ) + table = RestCatalog("rest", {}, TEST_HOST, token=TEST_TOKEN).load_table(("fokko", "table")) + assert table == Table( + identifier=("rest", "fokko", "table"), + metadata_location=None, + metadata=TableMetadataV1( + location="s3://tabular-public-us-west-2-dev/bb30733e-8769-4dab-aa1b-e76245bb2bd4/b55d9dda-6561-423a-8bfc-787980ce421f", + table_uuid=UUID("b55d9dda-6561-423a-8bfc-787980ce421f"), + last_updated_ms=1646787054459, + last_column_id=2, + schemas=[ + Schema( + NestedField(field_id=1, name="id", field_type=IntegerType(), required=False), + NestedField(field_id=2, name="data", field_type=StringType(), required=False), + schema_id=0, + identifier_field_ids=[], + ) + ], + current_schema_id=0, + partition_specs=[PartitionSpec(spec_id=0, fields=())], + default_spec_id=0, + last_partition_id=999, + properties={"owner": "bryan", "write.metadata.compression-codec": "gzip"}, + current_snapshot_id=3497810964824022504, + snapshots=[ + Snapshot( + snapshot_id=3497810964824022504, + parent_snapshot_id=None, + sequence_number=None, + timestamp_ms=1646787054459, + manifest_list="s3://tabular-public-us-west-2-dev/bb30733e-8769-4dab-aa1b-e76245bb2bd4/b55d9dda-6561-423a-8bfc-787980ce421f/metadata/snap-3497810964824022504-1-c4f68204-666b-4e50-a9df-b10c34bf6b82.avro", + summary=Summary( + operation=Operation.APPEND, + **{ # type: ignore + "spark.app.id": "local-1646787004168", + "added-data-files": "1", + "added-records": "1", + "added-files-size": "697", + "changed-partition-count": "1", + "total-records": "1", + "total-files-size": "697", + "total-data-files": "1", + "total-delete-files": "0", + "total-position-deletes": "0", + "total-equality-deletes": "0", + }, + ), + schema_id=0, + ) + ], + snapshot_log=[{"timestamp-ms": 1646787054459, "snapshot-id": 3497810964824022504}], + metadata_log=[ + { + "timestamp-ms": 1646787031514, + "metadata-file": "s3://tabular-public-us-west-2-dev/bb30733e-8769-4dab-aa1b-e76245bb2bd4/b55d9dda-6561-423a-8bfc-787980ce421f/metadata/00000-88484a1c-00e5-4a07-a787-c0e7aeffa805.gz.metadata.json", + } + ], + sort_orders=[SortOrder(order_id=0)], + default_sort_order_id=0, + refs={ + "main": SnapshotRef( + snapshot_id=3497810964824022504, + snapshot_ref_type=SnapshotRefType.BRANCH, + min_snapshots_to_keep=None, + max_snapshot_age_ms=None, + max_ref_age_ms=None, + ) + }, + format_version=1, + schema_=Schema( + NestedField(field_id=1, name="id", field_type=IntegerType(), required=False), + NestedField(field_id=2, name="data", field_type=StringType(), required=False), + schema_id=0, + identifier_field_ids=[], + ), + partition_spec=[], + ), + config={"client.factory": "io.tabular.iceberg.catalog.TabularAwsClientFactory", "region": "us-west-2"}, + ) + + +def test_load_table_404(rest_mock: Mocker): + rest_mock.get( + f"{TEST_HOST}v1/namespaces/fokko/tables/does_not_exists", + json={ + "error": { + "message": "Table does not exist: examples.does_not_exists in warehouse 8bcb0838-50fc-472d-9ddb-8feb89ef5f1e", + "type": "NoSuchNamespaceErrorException", + "code": 404, + } + }, + status_code=404, + ) + + with pytest.raises(NoSuchTableError) as e: + RestCatalog("rest", {}, TEST_HOST, token=TEST_TOKEN).load_table(("fokko", "does_not_exists")) + assert "Table does not exist" in str(e.value) + + +def test_drop_table_404(rest_mock: Mocker): + rest_mock.delete( + f"{TEST_HOST}v1/namespaces/fokko/tables/does_not_exists", + json={ + "error": { + "message": "Table does not exist: fokko.does_not_exists in warehouse 8bcb0838-50fc-472d-9ddb-8feb89ef5f1e", + "type": "NoSuchNamespaceErrorException", + "code": 404, + } + }, + status_code=404, + ) + + with pytest.raises(NoSuchTableError) as e: + RestCatalog("rest", {}, TEST_HOST, token=TEST_TOKEN).drop_table(("fokko", "does_not_exists")) + assert "Table does not exist" in str(e.value) + + +def test_create_table_200(rest_mock: Mocker, table_schema_simple: Schema): + rest_mock.post( + f"{TEST_HOST}v1/namespaces/fokko/tables", + json={ + "metadataLocation": None, + "metadata": { + "format-version": 1, + "table-uuid": "bf289591-dcc0-4234-ad4f-5c3eed811a29", + "location": "s3://tabular-wh-us-west-2-dev/8bcb0838-50fc-472d-9ddb-8feb89ef5f1e/bf289591-dcc0-4234-ad4f-5c3eed811a29", + "last-updated-ms": 1657810967051, + "last-column-id": 3, + "schema": { + "type": "struct", + "schema-id": 0, + "identifier-field-ids": [2], + "fields": [ + {"id": 1, "name": "foo", "required": False, "type": "string"}, + {"id": 2, "name": "bar", "required": True, "type": "int"}, + {"id": 3, "name": "baz", "required": False, "type": "boolean"}, + ], + }, + "current-schema-id": 0, + "schemas": [ + { + "type": "struct", + "schema-id": 0, + "identifier-field-ids": [2], + "fields": [ + {"id": 1, "name": "foo", "required": False, "type": "string"}, + {"id": 2, "name": "bar", "required": True, "type": "int"}, + {"id": 3, "name": "baz", "required": False, "type": "boolean"}, + ], + } + ], + "partition-spec": [], + "default-spec-id": 0, + "partition-specs": [{"spec-id": 0, "fields": []}], + "last-partition-id": 999, + "default-sort-order-id": 0, + "sort-orders": [{"order-id": 0, "fields": []}], + "properties": { + "write.delete.parquet.compression-codec": "zstd", + "write.metadata.compression-codec": "gzip", + "write.summary.partition-limit": "100", + "write.parquet.compression-codec": "zstd", + }, + "current-snapshot-id": -1, + "refs": {}, + "snapshots": [], + "snapshot-log": [], + "metadata-log": [], + }, + "config": { + "client.factory": "io.tabular.iceberg.catalog.TabularAwsClientFactory", + "region": "us-west-2", + }, + }, + status_code=200, + ) + table = RestCatalog("rest", {}, TEST_HOST, token=TEST_TOKEN).create_table( + identifier=("fokko", "fokko2"), + schema=table_schema_simple, + location=None, + partition_spec=PartitionSpec( + spec_id=1, fields=(PartitionField(source_id=1, field_id=1000, transform=TruncateTransform(width=3), name="id"),) + ), + sort_order=SortOrder(1, SortField(source_id=2, transform=IdentityTransform())), + properties={"owner": "fokko"}, + ) + assert table == Table( + identifier=("rest", "fokko", "fokko2"), + metadata_location=None, + metadata=TableMetadataV1( + location="s3://tabular-wh-us-west-2-dev/8bcb0838-50fc-472d-9ddb-8feb89ef5f1e/bf289591-dcc0-4234-ad4f-5c3eed811a29", + table_uuid=UUID("bf289591-dcc0-4234-ad4f-5c3eed811a29"), + last_updated_ms=1657810967051, + last_column_id=3, + schemas=[ + Schema( + NestedField(field_id=1, name="foo", field_type=StringType(), required=False), + NestedField(field_id=2, name="bar", field_type=IntegerType(), required=True), + NestedField(field_id=3, name="baz", field_type=BooleanType(), required=False), + schema_id=0, + identifier_field_ids=[2], + ) + ], + current_schema_id=0, + partition_specs=[PartitionSpec(spec_id=0, fields=())], + default_spec_id=0, + last_partition_id=999, + properties={ + "write.delete.parquet.compression-codec": "zstd", + "write.metadata.compression-codec": "gzip", + "write.summary.partition-limit": "100", + "write.parquet.compression-codec": "zstd", + }, + current_snapshot_id=None, + snapshots=[], + snapshot_log=[], + metadata_log=[], + sort_orders=[SortOrder(order_id=0)], + default_sort_order_id=0, + refs={}, + format_version=1, + schema_=Schema( + NestedField(field_id=1, name="foo", field_type=StringType(), required=False), + NestedField(field_id=2, name="bar", field_type=IntegerType(), required=True), + NestedField(field_id=3, name="baz", field_type=BooleanType(), required=False), + schema_id=0, + identifier_field_ids=[2], + ), + partition_spec=[], + ), + ) + + +def test_create_table_409(rest_mock, table_schema_simple: Schema): + rest_mock.post( + f"{TEST_HOST}v1/namespaces/fokko/tables", + json={ + "error": { + "message": "Table already exists: fokko.already_exists in warehouse 8bcb0838-50fc-472d-9ddb-8feb89ef5f1e", + "type": "AlreadyExistsException", + "code": 409, + } + }, + status_code=409, + ) + + with pytest.raises(TableAlreadyExistsError) as e: + RestCatalog("rest", {}, TEST_HOST, token=TEST_TOKEN).create_table( + identifier=("fokko", "fokko2"), + schema=table_schema_simple, + location=None, + partition_spec=PartitionSpec( + spec_id=1, + fields=(PartitionField(source_id=1, field_id=1000, transform=TruncateTransform(width=3), name="id"),), + ), + sort_order=SortOrder(1, SortField(source_id=2, transform=IdentityTransform())), + properties={"owner": "fokko"}, + ) + assert "Table already exists" in str(e.value) + + +def test_delete_namespace_204(rest_mock: Mocker): + namespace = "example" + rest_mock.delete( + f"{TEST_HOST}v1/namespaces/{namespace}", + json={}, + status_code=204, + ) + RestCatalog("rest", {}, TEST_HOST, token=TEST_TOKEN).drop_namespace(namespace) + + +def test_delete_table_204(rest_mock: Mocker): + rest_mock.delete( + f"{TEST_HOST}v1/namespaces/example/tables/fokko", + json={}, + status_code=204, + ) + RestCatalog("rest", {}, TEST_HOST, token=TEST_TOKEN).drop_table(("example", "fokko")) + + +def test_delete_table_404(rest_mock: Mocker): + rest_mock.delete( + f"{TEST_HOST}v1/namespaces/example/tables/fokko", + json={ + "error": { + "message": "Table does not exist: fokko.fokko2 in warehouse 8bcb0838-50fc-472d-9ddb-8feb89ef5f1e", + "type": "NoSuchNamespaceErrorException", + "code": 404, + } + }, + status_code=404, + ) + with pytest.raises(NoSuchTableError) as e: + RestCatalog("rest", {}, TEST_HOST, token=TEST_TOKEN).drop_table(("example", "fokko")) + assert "Table does not exist" in str(e.value) diff --git a/tests/conftest.py b/tests/conftest.py index 1b004ec8f8..7aa9290390 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -76,7 +76,7 @@ def table_schema_simple(): NestedField(field_id=2, name="bar", field_type=IntegerType(), required=True), NestedField(field_id=3, name="baz", field_type=BooleanType(), required=False), schema_id=1, - identifier_field_ids=[1], + identifier_field_ids=[2], ) diff --git a/tests/table/test_metadata.py b/tests/table/test_metadata.py index 0a58f201de..9c7352d511 100644 --- a/tests/table/test_metadata.py +++ b/tests/table/test_metadata.py @@ -207,13 +207,13 @@ def test_updating_metadata(): def test_serialize_v1(): table_metadata = TableMetadataV1(**EXAMPLE_TABLE_METADATA_V1).json() - expected = """{"location": "s3://bucket/test/location", "table-uuid": "d20125c8-7284-442c-9aea-15fee620737c", "last-updated-ms": 1602638573874, "last-column-id": 3, "schemas": [{"fields": [{"id": 1, "name": "x", "type": "long", "required": true}, {"id": 2, "name": "y", "type": "long", "required": true, "doc": "comment"}, {"id": 3, "name": "z", "type": "long", "required": true}], "schema-id": 0, "identifier-field-ids": []}], "current-schema-id": 0, "partition-specs": [{"spec-id": 0, "fields": [{"source-id": 1, "field-id": 1000, "transform": "identity", "name": "x"}]}], "default-spec-id": 0, "last-partition-id": 1000, "properties": {}, "snapshots": [{"snapshot-id": 1925, "timestamp-ms": 1602638573822}], "snapshot-log": [], "metadata-log": [], "sort-orders": [{"order-id": 0, "fields": []}], "default-sort-order-id": 0, "refs": {}, "format-version": 1, "schema": {"fields": [{"id": 1, "name": "x", "type": "long", "required": true}, {"id": 2, "name": "y", "type": "long", "required": true, "doc": "comment"}, {"id": 3, "name": "z", "type": "long", "required": true}], "schema-id": 0, "identifier-field-ids": []}, "partition-spec": [{"name": "x", "transform": "identity", "source-id": 1, "field-id": 1000}]}""" + expected = """{"location": "s3://bucket/test/location", "table-uuid": "d20125c8-7284-442c-9aea-15fee620737c", "last-updated-ms": 1602638573874, "last-column-id": 3, "schemas": [{"type": "struct", "fields": [{"id": 1, "name": "x", "type": "long", "required": true}, {"id": 2, "name": "y", "type": "long", "required": true, "doc": "comment"}, {"id": 3, "name": "z", "type": "long", "required": true}], "schema-id": 0, "identifier-field-ids": []}], "current-schema-id": 0, "partition-specs": [{"spec-id": 0, "fields": [{"source-id": 1, "field-id": 1000, "transform": "identity", "name": "x"}]}], "default-spec-id": 0, "last-partition-id": 1000, "properties": {}, "snapshots": [{"snapshot-id": 1925, "timestamp-ms": 1602638573822}], "snapshot-log": [], "metadata-log": [], "sort-orders": [{"order-id": 0, "fields": []}], "default-sort-order-id": 0, "refs": {}, "format-version": 1, "schema": {"type": "struct", "fields": [{"id": 1, "name": "x", "type": "long", "required": true}, {"id": 2, "name": "y", "type": "long", "required": true, "doc": "comment"}, {"id": 3, "name": "z", "type": "long", "required": true}], "schema-id": 0, "identifier-field-ids": []}, "partition-spec": [{"name": "x", "transform": "identity", "source-id": 1, "field-id": 1000}]}""" assert table_metadata == expected def test_serialize_v2(): table_metadata = TableMetadataV2(**EXAMPLE_TABLE_METADATA_V2).json() - expected = """{"location": "s3://bucket/test/location", "table-uuid": "9c12d441-03fe-4693-9a96-a0705ddf69c1", "last-updated-ms": 1602638573590, "last-column-id": 3, "schemas": [{"fields": [{"id": 1, "name": "x", "type": "long", "required": true}], "schema-id": 0, "identifier-field-ids": []}, {"fields": [{"id": 1, "name": "x", "type": "long", "required": true}, {"id": 2, "name": "y", "type": "long", "required": true, "doc": "comment"}, {"id": 3, "name": "z", "type": "long", "required": true}], "schema-id": 1, "identifier-field-ids": [1, 2]}], "current-schema-id": 1, "partition-specs": [{"spec-id": 0, "fields": [{"source-id": 1, "field-id": 1000, "transform": "identity", "name": "x"}]}], "default-spec-id": 0, "last-partition-id": 1000, "properties": {"read.split.target.size": "134217728"}, "current-snapshot-id": 3055729675574597004, "snapshots": [{"snapshot-id": 3051729675574597004, "sequence-number": 0, "timestamp-ms": 1515100955770, "manifest-list": "s3://a/b/1.avro", "summary": {"operation": "append"}}, {"snapshot-id": 3055729675574597004, "parent-snapshot-id": 3051729675574597004, "sequence-number": 1, "timestamp-ms": 1555100955770, "manifest-list": "s3://a/b/2.avro", "summary": {"operation": "append"}, "schema-id": 1}], "snapshot-log": [{"snapshot-id": 3051729675574597004, "timestamp-ms": 1515100955770}, {"snapshot-id": 3055729675574597004, "timestamp-ms": 1555100955770}], "metadata-log": [], "sort-orders": [{"order-id": 3, "fields": [{"source-id": 2, "transform": "identity", "direction": "asc", "null-order": "nulls-first"}, {"source-id": 3, "transform": "bucket[4]", "direction": "desc", "null-order": "nulls-last"}]}], "default-sort-order-id": 3, "refs": {"main": {"snapshot-id": 3055729675574597004, "type": "branch"}}, "format-version": 2, "last-sequence-number": 34}""" + expected = """{"location": "s3://bucket/test/location", "table-uuid": "9c12d441-03fe-4693-9a96-a0705ddf69c1", "last-updated-ms": 1602638573590, "last-column-id": 3, "schemas": [{"type": "struct", "fields": [{"id": 1, "name": "x", "type": "long", "required": true}], "schema-id": 0, "identifier-field-ids": []}, {"type": "struct", "fields": [{"id": 1, "name": "x", "type": "long", "required": true}, {"id": 2, "name": "y", "type": "long", "required": true, "doc": "comment"}, {"id": 3, "name": "z", "type": "long", "required": true}], "schema-id": 1, "identifier-field-ids": [1, 2]}], "current-schema-id": 1, "partition-specs": [{"spec-id": 0, "fields": [{"source-id": 1, "field-id": 1000, "transform": "identity", "name": "x"}]}], "default-spec-id": 0, "last-partition-id": 1000, "properties": {"read.split.target.size": "134217728"}, "current-snapshot-id": 3055729675574597004, "snapshots": [{"snapshot-id": 3051729675574597004, "sequence-number": 0, "timestamp-ms": 1515100955770, "manifest-list": "s3://a/b/1.avro", "summary": {"operation": "append"}}, {"snapshot-id": 3055729675574597004, "parent-snapshot-id": 3051729675574597004, "sequence-number": 1, "timestamp-ms": 1555100955770, "manifest-list": "s3://a/b/2.avro", "summary": {"operation": "append"}, "schema-id": 1}], "snapshot-log": [{"snapshot-id": 3051729675574597004, "timestamp-ms": 1515100955770}, {"snapshot-id": 3055729675574597004, "timestamp-ms": 1555100955770}], "metadata-log": [], "sort-orders": [{"order-id": 3, "fields": [{"source-id": 2, "transform": "identity", "direction": "asc", "null-order": "nulls-first"}, {"source-id": 3, "transform": "bucket[4]", "direction": "desc", "null-order": "nulls-last"}]}], "default-sort-order-id": 3, "refs": {"main": {"snapshot-id": 3055729675574597004, "type": "branch"}}, "format-version": 2, "last-sequence-number": 34}""" assert table_metadata == expected @@ -505,6 +505,7 @@ def test_v1_write_metadata_for_v2(): ], "identifier-field-ids": [], "schema-id": 0, + "type": "struct", } ] assert metadata_v2["partition-specs"] == [ @@ -525,3 +526,54 @@ def test_v1_write_metadata_for_v2(): def test_v2_ref_creation(): table_metadata = TableMetadataV2(**EXAMPLE_TABLE_METADATA_V2) assert table_metadata.refs == {"main": SnapshotRef(snapshot_id=3055729675574597004, snapshot_ref_type=SnapshotRefType.BRANCH)} + + +def test_metadata_v1(): + valid_v1 = { + "format-version": 1, + "table-uuid": "bf289591-dcc0-4234-ad4f-5c3eed811a29", + "location": "s3://tabular-wh-us-west-2-dev/8bcb0838-50fc-472d-9ddb-8feb89ef5f1e/bf289591-dcc0-4234-ad4f-5c3eed811a29", + "last-updated-ms": 1657810967051, + "last-column-id": 3, + "schema": { + "type": "struct", + "schema-id": 0, + "identifier-field-ids": [2], + "fields": [ + {"id": 1, "name": "foo", "required": False, "type": "string"}, + {"id": 2, "name": "bar", "required": True, "type": "int"}, + {"id": 3, "name": "baz", "required": False, "type": "boolean"}, + ], + }, + "current-schema-id": 0, + "schemas": [ + { + "type": "struct", + "schema-id": 0, + "identifier-field-ids": [2], + "fields": [ + {"id": 1, "name": "foo", "required": False, "type": "string"}, + {"id": 2, "name": "bar", "required": True, "type": "int"}, + {"id": 3, "name": "baz", "required": False, "type": "boolean"}, + ], + } + ], + "partition-spec": [], + "default-spec-id": 0, + "partition-specs": [{"spec-id": 0, "fields": []}], + "last-partition-id": 999, + "default-sort-order-id": 0, + "sort-orders": [{"order-id": 0, "fields": []}], + "properties": { + "write.delete.parquet.compression-codec": "zstd", + "write.metadata.compression-codec": "gzip", + "write.summary.partition-limit": "100", + "write.parquet.compression-codec": "zstd", + }, + "current-snapshot-id": -1, + "refs": {}, + "snapshots": [], + "snapshot-log": [], + "metadata-log": [], + } + TableMetadataV1(**valid_v1) diff --git a/tests/test_schema.py b/tests/test_schema.py index 3d61323f02..ad094ab934 100644 --- a/tests/test_schema.py +++ b/tests/test_schema.py @@ -406,13 +406,13 @@ def get(self, pos: int) -> Any: def test_serialize_schema(table_schema_simple: Schema): actual = table_schema_simple.json() - expected = """{"fields": [{"id": 1, "name": "foo", "type": "string", "required": false}, {"id": 2, "name": "bar", "type": "int", "required": true}, {"id": 3, "name": "baz", "type": "boolean", "required": false}], "schema-id": 1, "identifier-field-ids": [1]}""" + expected = """{"type": "struct", "fields": [{"id": 1, "name": "foo", "type": "string", "required": false}, {"id": 2, "name": "bar", "type": "int", "required": true}, {"id": 3, "name": "baz", "type": "boolean", "required": false}], "schema-id": 1, "identifier-field-ids": [2]}""" assert actual == expected def test_deserialize_schema(table_schema_simple: Schema): actual = Schema.parse_raw( - """{"fields": [{"id": 1, "name": "foo", "type": "string", "required": false}, {"id": 2, "name": "bar", "type": "int", "required": true}, {"id": 3, "name": "baz", "type": "boolean", "required": false}], "schema-id": 1, "identifier-field-ids": [1]}""" + """{"type": "struct", "fields": [{"id": 1, "name": "foo", "type": "string", "required": false}, {"id": 2, "name": "bar", "type": "int", "required": true}, {"id": 3, "name": "baz", "type": "boolean", "required": false}], "schema-id": 1, "identifier-field-ids": [2]}""" ) expected = table_schema_simple assert actual == expected diff --git a/tests/utils/test_bin_packing.py b/tests/utils/test_bin_packing.py index 71fd53c027..5447dfe928 100644 --- a/tests/utils/test_bin_packing.py +++ b/tests/utils/test_bin_packing.py @@ -19,7 +19,6 @@ import pytest -from pyiceberg.schema import Schema from pyiceberg.utils.bin_packing import PackingIterator @@ -82,17 +81,3 @@ def weight_func(x): return x assert list(PackingIterator(splits, target_weight, lookback, weight_func, largest_bin_first)) == expected_lists - - -def test_serialize_schema(table_schema_simple: Schema): - actual = table_schema_simple.json() - expected = """{"fields": [{"id": 1, "name": "foo", "type": "string", "required": false}, {"id": 2, "name": "bar", "type": "int", "required": true}, {"id": 3, "name": "baz", "type": "boolean", "required": false}], "schema-id": 1, "identifier-field-ids": [1]}""" - assert actual == expected - - -def test_deserialize_schema(table_schema_simple: Schema): - actual = Schema.parse_raw( - """{"fields": [{"id": 1, "name": "foo", "type": "string", "required": false}, {"id": 2, "name": "bar", "type": "int", "required": true}, {"id": 3, "name": "baz", "type": "boolean", "required": false}], "schema-id": 1, "identifier-field-ids": [1]}""" - ) - expected = table_schema_simple - assert actual == expected From b1bc8bc16f090c0d5fae43e9d480dde3c853823b Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Sun, 31 Jul 2022 09:44:13 -0700 Subject: [PATCH 147/642] Python: Bump fastavro from 1.5.3 to 1.5.4 in /python (#5396) Bumps [fastavro](https://github.com/fastavro/fastavro) from 1.5.3 to 1.5.4. - [Release notes](https://github.com/fastavro/fastavro/releases) - [Changelog](https://github.com/fastavro/fastavro/blob/master/ChangeLog) - [Commits](https://github.com/fastavro/fastavro/compare/1.5.3...1.5.4) Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- poetry.lock | 178 ++++++++++++++++++++++++++++++++++++++++++++----- pyproject.toml | 2 +- 2 files changed, 164 insertions(+), 16 deletions(-) diff --git a/poetry.lock b/poetry.lock index d95bd8729b..649dc074f5 100644 --- a/poetry.lock +++ b/poetry.lock @@ -98,7 +98,7 @@ python-versions = ">=3.7" [[package]] name = "fastavro" -version = "1.5.3" +version = "1.5.4" description = "Fast read/write of AVRO files" category = "dev" optional = false @@ -232,8 +232,8 @@ optional = false python-versions = ">=3.6" [package.extras] -dev = ["pre-commit", "tox"] -testing = ["pytest", "pytest-benchmark"] +testing = ["pytest-benchmark", "pytest"] +dev = ["tox", "pre-commit"] [[package]] name = "pre-commit" @@ -489,10 +489,12 @@ snappy = ["python-snappy"] [metadata] lock-version = "1.1" python-versions = "^3.8" -content-hash = "ff76c4a6349ee3a909543aa6587ace0ab6795e08e97e892a90553b6fd6a589af" +content-hash = "d8312a4f53fd87fc2dc2059d15a9aa8274512debe1c780a9e53a1108b6c12e55" [metadata.files] -atomicwrites = [] +atomicwrites = [ + {file = "atomicwrites-1.4.1.tar.gz", hash = "sha256:81b2c9071a49367a7f770170e5eec8cb66567cfbbc8c73d20ce5ca4a8d71cf11"}, +] attrs = [ {file = "attrs-21.4.0-py2.py3-none-any.whl", hash = "sha256:2d27e3784d7a565d36ab851fe94887c5eccd6a463168875832a1be79c82828b4"}, {file = "attrs-21.4.0.tar.gz", hash = "sha256:626ba8234211db98e869df76230a137c4c40a12d72445c45d5f5b716f076e2fd"}, @@ -579,18 +581,84 @@ colorama = [ {file = "colorama-0.4.5-py2.py3-none-any.whl", hash = "sha256:854bf444933e37f5824ae7bfc1e98d5bce2ebe4160d46b5edf346a89358e99da"}, {file = "colorama-0.4.5.tar.gz", hash = "sha256:e6c6b4334fc50988a639d9b98aa429a0b57da6e17b9a44f0451f930b6967b7a4"}, ] -coverage = [] -distlib = [] +coverage = [ + {file = "coverage-6.4.2-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:a9032f9b7d38bdf882ac9f66ebde3afb8145f0d4c24b2e600bc4c6304aafb87e"}, + {file = "coverage-6.4.2-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:e0524adb49c716ca763dbc1d27bedce36b14f33e6b8af6dba56886476b42957c"}, + {file = "coverage-6.4.2-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:d4548be38a1c810d79e097a38107b6bf2ff42151900e47d49635be69943763d8"}, + {file = "coverage-6.4.2-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:f23876b018dfa5d3e98e96f5644b109090f16a4acb22064e0f06933663005d39"}, + {file = "coverage-6.4.2-cp310-cp310-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:6fe75dcfcb889b6800f072f2af5a331342d63d0c1b3d2bf0f7b4f6c353e8c9c0"}, + {file = "coverage-6.4.2-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:2f8553878a24b00d5ab04b7a92a2af50409247ca5c4b7a2bf4eabe94ed20d3ee"}, + {file = "coverage-6.4.2-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:d774d9e97007b018a651eadc1b3970ed20237395527e22cbeb743d8e73e0563d"}, + {file = "coverage-6.4.2-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:d56f105592188ce7a797b2bd94b4a8cb2e36d5d9b0d8a1d2060ff2a71e6b9bbc"}, + {file = "coverage-6.4.2-cp310-cp310-win32.whl", hash = "sha256:d230d333b0be8042ac34808ad722eabba30036232e7a6fb3e317c49f61c93386"}, + {file = "coverage-6.4.2-cp310-cp310-win_amd64.whl", hash = "sha256:5ef42e1db047ca42827a85e34abe973971c635f83aed49611b7f3ab49d0130f0"}, + {file = "coverage-6.4.2-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:25b7ec944f114f70803d6529394b64f8749e93cbfac0fe6c5ea1b7e6c14e8a46"}, + {file = "coverage-6.4.2-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:7bb00521ab4f99fdce2d5c05a91bddc0280f0afaee0e0a00425e28e209d4af07"}, + {file = "coverage-6.4.2-cp37-cp37m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:2dff52b3e7f76ada36f82124703f4953186d9029d00d6287f17c68a75e2e6039"}, + {file = "coverage-6.4.2-cp37-cp37m-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:147605e1702d996279bb3cc3b164f408698850011210d133a2cb96a73a2f7996"}, + {file = "coverage-6.4.2-cp37-cp37m-musllinux_1_1_aarch64.whl", hash = "sha256:422fa44070b42fef9fb8dabd5af03861708cdd6deb69463adc2130b7bf81332f"}, + {file = "coverage-6.4.2-cp37-cp37m-musllinux_1_1_i686.whl", hash = "sha256:8af6c26ba8df6338e57bedbf916d76bdae6308e57fc8f14397f03b5da8622b4e"}, + {file = "coverage-6.4.2-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:5336e0352c0b12c7e72727d50ff02557005f79a0b8dcad9219c7c4940a930083"}, + {file = "coverage-6.4.2-cp37-cp37m-win32.whl", hash = "sha256:0f211df2cba951ffcae210ee00e54921ab42e2b64e0bf2c0befc977377fb09b7"}, + {file = "coverage-6.4.2-cp37-cp37m-win_amd64.whl", hash = "sha256:a13772c19619118903d65a91f1d5fea84be494d12fd406d06c849b00d31bf120"}, + {file = "coverage-6.4.2-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:f7bd0ffbcd03dc39490a1f40b2669cc414fae0c4e16b77bb26806a4d0b7d1452"}, + {file = "coverage-6.4.2-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:0895ea6e6f7f9939166cc835df8fa4599e2d9b759b02d1521b574e13b859ac32"}, + {file = "coverage-6.4.2-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:d4e7ced84a11c10160c0697a6cc0b214a5d7ab21dfec1cd46e89fbf77cc66fae"}, + {file = "coverage-6.4.2-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:80db4a47a199c4563d4a25919ff29c97c87569130375beca3483b41ad5f698e8"}, + {file = "coverage-6.4.2-cp38-cp38-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:3def6791adf580d66f025223078dc84c64696a26f174131059ce8e91452584e1"}, + {file = "coverage-6.4.2-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:4f89d8e03c8a3757aae65570d14033e8edf192ee9298303db15955cadcff0c63"}, + {file = "coverage-6.4.2-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:6d0b48aff8e9720bdec315d67723f0babd936a7211dc5df453ddf76f89c59933"}, + {file = "coverage-6.4.2-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:2b20286c2b726f94e766e86a3fddb7b7e37af5d0c635bdfa7e4399bc523563de"}, + {file = "coverage-6.4.2-cp38-cp38-win32.whl", hash = "sha256:d714af0bdba67739598849c9f18efdcc5a0412f4993914a0ec5ce0f1e864d783"}, + {file = "coverage-6.4.2-cp38-cp38-win_amd64.whl", hash = "sha256:5f65e5d3ff2d895dab76b1faca4586b970a99b5d4b24e9aafffc0ce94a6022d6"}, + {file = "coverage-6.4.2-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:a697977157adc052284a7160569b36a8bbec09db3c3220642e6323b47cec090f"}, + {file = "coverage-6.4.2-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:c77943ef768276b61c96a3eb854eba55633c7a3fddf0a79f82805f232326d33f"}, + {file = "coverage-6.4.2-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:54d8d0e073a7f238f0666d3c7c0d37469b2aa43311e4024c925ee14f5d5a1cbe"}, + {file = "coverage-6.4.2-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:f22325010d8824594820d6ce84fa830838f581a7fd86a9235f0d2ed6deb61e29"}, + {file = "coverage-6.4.2-cp39-cp39-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:24b04d305ea172ccb21bee5bacd559383cba2c6fcdef85b7701cf2de4188aa55"}, + {file = "coverage-6.4.2-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:866ebf42b4c5dbafd64455b0a1cd5aa7b4837a894809413b930026c91e18090b"}, + {file = "coverage-6.4.2-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:e36750fbbc422c1c46c9d13b937ab437138b998fe74a635ec88989afb57a3978"}, + {file = "coverage-6.4.2-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:79419370d6a637cb18553ecb25228893966bd7935a9120fa454e7076f13b627c"}, + {file = "coverage-6.4.2-cp39-cp39-win32.whl", hash = "sha256:b5e28db9199dd3833cc8a07fa6cf429a01227b5d429facb56eccd765050c26cd"}, + {file = "coverage-6.4.2-cp39-cp39-win_amd64.whl", hash = "sha256:edfdabe7aa4f97ed2b9dd5dde52d2bb29cb466993bb9d612ddd10d0085a683cf"}, + {file = "coverage-6.4.2-pp36.pp37.pp38-none-any.whl", hash = "sha256:e2618cb2cf5a7cc8d698306e42ebcacd02fb7ef8cfc18485c59394152c70be97"}, + {file = "coverage-6.4.2.tar.gz", hash = "sha256:6c3ccfe89c36f3e5b9837b9ee507472310164f352c9fe332120b764c9d60adbe"}, +] +distlib = [ + {file = "distlib-0.3.5-py2.py3-none-any.whl", hash = "sha256:b710088c59f06338ca514800ad795a132da19fda270e3ce4affc74abf955a26c"}, + {file = "distlib-0.3.5.tar.gz", hash = "sha256:a7f75737c70be3b25e2bee06288cec4e4c221de18455b2dd037fe2a795cab2fe"}, +] docutils = [ {file = "docutils-0.19-py3-none-any.whl", hash = "sha256:5e1de4d849fee02c63b040a4a3fd567f4ab104defd8a5511fbbc24a8a017efbc"}, {file = "docutils-0.19.tar.gz", hash = "sha256:33995a6753c30b7f577febfc2c50411fec6aac7f7ffeb7c4cfe5991072dcf9e6"}, ] -fastavro = [] +fastavro = [ + {file = "fastavro-1.5.4-cp310-cp310-macosx_10_14_x86_64.whl", hash = "sha256:d316cc476b2b24ef06402b8bfa047f8f72a9d6df2de777bb30d9ededda7e3a02"}, + {file = "fastavro-1.5.4-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:8459faec46e34f2dfeb9b70ee8c36e935e626cff8608d675724718987a5f9ce5"}, + {file = "fastavro-1.5.4-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:cd44636d7ff8365a57b88707b747371fffb676c8c1f68c0d423ec36623888668"}, + {file = "fastavro-1.5.4-cp310-cp310-win_amd64.whl", hash = "sha256:2402428b26d3c08a58acfa723833e19fb75077872bcb2475a4c81195cdae6a5d"}, + {file = "fastavro-1.5.4-cp37-cp37m-macosx_10_15_x86_64.whl", hash = "sha256:5afc14398f4191d1a807aa59d2fba5ed869b31343679ec43dbc289db0a8e35c5"}, + {file = "fastavro-1.5.4-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:5217e9713a3ea03205532394fba4d743749155b04b10b12a12fc26d225b89792"}, + {file = "fastavro-1.5.4-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:2e93a5eecb28cc35d670c9c4df70223fa9bcd6d9ca21b38b1b7ae13ece60c7fb"}, + {file = "fastavro-1.5.4-cp37-cp37m-win_amd64.whl", hash = "sha256:1a2f2465efd0e7de557c4034e8d4d88a132750cfa51e1582362a1b3a1a9fa911"}, + {file = "fastavro-1.5.4-cp38-cp38-macosx_10_15_x86_64.whl", hash = "sha256:f7d5bc76c03c692d9acea0e5d5baceec19e1c059b26cb8ae9f4481e842be95a5"}, + {file = "fastavro-1.5.4-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:80fe920229ab1f40eccb1b4918481cdd8a20e5e7dce19308ab38b23732da8a70"}, + {file = "fastavro-1.5.4-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:a3d190aee86ab73caa1aa550eba850be2ca5dd29d814b38720f4e300184e01d5"}, + {file = "fastavro-1.5.4-cp38-cp38-win_amd64.whl", hash = "sha256:b6c30299a49b11f42251cb81c8e15db67750642eac7ba5c194a5ee95c83ebb11"}, + {file = "fastavro-1.5.4-cp39-cp39-macosx_10_15_x86_64.whl", hash = "sha256:1f7685f3a3c38352abab432bad2f9f2229a0e5f5f8548831e887c30f8396f2e9"}, + {file = "fastavro-1.5.4-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:bd021ec850fd30020b7c4fa868466fb7f95450f1f06eac92bd2204cbd8e45fb8"}, + {file = "fastavro-1.5.4-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:06a7b5602dfa032c92f20ca90b8bde88251573773e501bedf5e8b76b9feb14a3"}, + {file = "fastavro-1.5.4-cp39-cp39-win_amd64.whl", hash = "sha256:18250aa2ab0f7a095b1865565cf9976ea4605c201129636e6defe24ec3ef112c"}, + {file = "fastavro-1.5.4.tar.gz", hash = "sha256:d86f72c966713fb699570a18f7960cf4110b069c70681d7538be8d671c9db7c8"}, +] filelock = [ {file = "filelock-3.7.1-py3-none-any.whl", hash = "sha256:37def7b658813cda163b56fc564cdc75e86d338246458c4c28ae84cabefa2404"}, {file = "filelock-3.7.1.tar.gz", hash = "sha256:3a0fd85166ad9dbab54c9aec96737b744106dc5f15c0b09a6744a445299fcf04"}, ] -identify = [] +identify = [ + {file = "identify-2.5.2-py2.py3-none-any.whl", hash = "sha256:feaa9db2dc0ce333b453ce171c0cf1247bbfde2c55fc6bb785022d411a1b78b5"}, + {file = "identify-2.5.2.tar.gz", hash = "sha256:a3d4c096b384d50d5e6dc5bc8b9bc44f1f61cefebd750a7b3e9f939b53fb214d"}, +] idna = [ {file = "idna-3.3-py3-none-any.whl", hash = "sha256:84d9dd047ffa80596e0f246e2eab0b391788b0503584e8945f2368256d2735ff"}, {file = "idna-3.3.tar.gz", hash = "sha256:9d643ff0a55b762d5cdb124b8eaa99c66322e2157b69160bc32796e824360e6d"}, @@ -635,7 +703,30 @@ nodeenv = [ {file = "nodeenv-1.7.0-py2.py3-none-any.whl", hash = "sha256:27083a7b96a25f2f5e1d8cb4b6317ee8aeda3bdd121394e5ac54e498028a042e"}, {file = "nodeenv-1.7.0.tar.gz", hash = "sha256:e0e7f7dfb85fc5394c6fe1e8fa98131a2473e04311a45afb6508f7cf1836fa2b"}, ] -numpy = [] +numpy = [ + {file = "numpy-1.23.1-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:b15c3f1ed08df4980e02cc79ee058b788a3d0bef2fb3c9ca90bb8cbd5b8a3a04"}, + {file = "numpy-1.23.1-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:9ce242162015b7e88092dccd0e854548c0926b75c7924a3495e02c6067aba1f5"}, + {file = "numpy-1.23.1-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:e0d7447679ae9a7124385ccf0ea990bb85bb869cef217e2ea6c844b6a6855073"}, + {file = "numpy-1.23.1-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:3119daed207e9410eaf57dcf9591fdc68045f60483d94956bee0bfdcba790953"}, + {file = "numpy-1.23.1-cp310-cp310-win32.whl", hash = "sha256:3ab67966c8d45d55a2bdf40701536af6443763907086c0a6d1232688e27e5447"}, + {file = "numpy-1.23.1-cp310-cp310-win_amd64.whl", hash = "sha256:1865fdf51446839ca3fffaab172461f2b781163f6f395f1aed256b1ddc253622"}, + {file = "numpy-1.23.1-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:aeba539285dcf0a1ba755945865ec61240ede5432df41d6e29fab305f4384db2"}, + {file = "numpy-1.23.1-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:7e8229f3687cdadba2c4faef39204feb51ef7c1a9b669247d49a24f3e2e1617c"}, + {file = "numpy-1.23.1-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:68b69f52e6545af010b76516f5daaef6173e73353e3295c5cb9f96c35d755641"}, + {file = "numpy-1.23.1-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:1408c3527a74a0209c781ac82bde2182b0f0bf54dea6e6a363fe0cc4488a7ce7"}, + {file = "numpy-1.23.1-cp38-cp38-win32.whl", hash = "sha256:47f10ab202fe4d8495ff484b5561c65dd59177949ca07975663f4494f7269e3e"}, + {file = "numpy-1.23.1-cp38-cp38-win_amd64.whl", hash = "sha256:37e5ebebb0eb54c5b4a9b04e6f3018e16b8ef257d26c8945925ba8105008e645"}, + {file = "numpy-1.23.1-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:173f28921b15d341afadf6c3898a34f20a0569e4ad5435297ba262ee8941e77b"}, + {file = "numpy-1.23.1-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:876f60de09734fbcb4e27a97c9a286b51284df1326b1ac5f1bf0ad3678236b22"}, + {file = "numpy-1.23.1-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:35590b9c33c0f1c9732b3231bb6a72d1e4f77872390c47d50a615686ae7ed3fd"}, + {file = "numpy-1.23.1-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:a35c4e64dfca659fe4d0f1421fc0f05b8ed1ca8c46fb73d9e5a7f175f85696bb"}, + {file = "numpy-1.23.1-cp39-cp39-win32.whl", hash = "sha256:c2f91f88230042a130ceb1b496932aa717dcbd665350beb821534c5c7e15881c"}, + {file = "numpy-1.23.1-cp39-cp39-win_amd64.whl", hash = "sha256:37ece2bd095e9781a7156852e43d18044fd0d742934833335599c583618181b9"}, + {file = "numpy-1.23.1-pp38-pypy38_pp73-macosx_10_9_x86_64.whl", hash = "sha256:8002574a6b46ac3b5739a003b5233376aeac5163e5dcd43dd7ad062f3e186129"}, + {file = "numpy-1.23.1-pp38-pypy38_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:5d732d17b8a9061540a10fda5bfeabca5785700ab5469a5e9b93aca5e2d3a5fb"}, + {file = "numpy-1.23.1-pp38-pypy38_pp73-win_amd64.whl", hash = "sha256:55df0f7483b822855af67e38fb3a526e787adf189383b4934305565d71c4b148"}, + {file = "numpy-1.23.1.tar.gz", hash = "sha256:d748ef349bfef2e1194b59da37ed5a29c19ea8d7e6342019921ba2ba4fd8b624"}, +] packaging = [ {file = "packaging-21.3-py3-none-any.whl", hash = "sha256:ef103e05f519cdc783ae24ea4e2e0f508a9c99b2d4969652eed6a2e1ea5bd522"}, {file = "packaging-21.3.tar.gz", hash = "sha256:dd47c42927d89ab911e606518907cc2d3a1f38bbd026385970643f9c5b8ecfeb"}, @@ -652,7 +743,10 @@ pluggy = [ {file = "pluggy-1.0.0-py2.py3-none-any.whl", hash = "sha256:74134bbf457f031a36d68416e1509f34bd5ccc019f0bcc952c7b909d06b37bd3"}, {file = "pluggy-1.0.0.tar.gz", hash = "sha256:4224373bacce55f955a878bf9cfa763c1e360858e330072059e10bad68531159"}, ] -pre-commit = [] +pre-commit = [ + {file = "pre_commit-2.20.0-py2.py3-none-any.whl", hash = "sha256:51a5ba7c480ae8072ecdb6933df22d2f812dc897d5fe848778116129a681aac7"}, + {file = "pre_commit-2.20.0.tar.gz", hash = "sha256:a978dac7bc9ec0bcee55c18a277d553b0f419d259dadb4b9418ff2d00eb43959"}, +] py = [ {file = "py-1.11.0-py2.py3-none-any.whl", hash = "sha256:607c53218732647dff4acdfcd50cb62615cedf612e72d1724fb1a0cc6405b378"}, {file = "py-1.11.0.tar.gz", hash = "sha256:51c75c4126074b472f746a24399ad32f6053d1b34b68d2fa41e558e6f4a98719"}, @@ -831,7 +925,10 @@ requests = [ {file = "requests-2.28.1-py3-none-any.whl", hash = "sha256:8fefa2a1a1365bf5520aac41836fbee479da67864514bdb821f31ce07ce65349"}, {file = "requests-2.28.1.tar.gz", hash = "sha256:7c5599b102feddaa661c826c56ab4fee28bfd17f5abca1ebbe3e7f19d7c97983"}, ] -requests-mock = [] +requests-mock = [ + {file = "requests-mock-1.9.3.tar.gz", hash = "sha256:8d72abe54546c1fc9696fa1516672f1031d72a55a1d66c85184f972a24ba0eba"}, + {file = "requests_mock-1.9.3-py2.py3-none-any.whl", hash = "sha256:0a2d38a117c08bb78939ec163522976ad59a6b7fdd82b709e23bb98004a44970"}, +] six = [ {file = "six-1.16.0-py2.py3-none-any.whl", hash = "sha256:8abb2f1d86890a2dfb989f9a77cfcfd3e47c2a354b01111771326f8aa26e0254"}, {file = "six-1.16.0.tar.gz", hash = "sha256:1e61c37477a1626458e36f7b1d82aa5c9b094fa4802892072e49de9c60c4c926"}, @@ -848,10 +945,61 @@ typing-extensions = [ {file = "typing_extensions-4.3.0-py3-none-any.whl", hash = "sha256:25642c956049920a5aa49edcdd6ab1e06d7e5d467fc00e0506c44ac86fbfca02"}, {file = "typing_extensions-4.3.0.tar.gz", hash = "sha256:e6d2677a32f47fc7eb2795db1dd15c1f34eff616bcaf2cfb5e997f854fa1c4a6"}, ] -urllib3 = [] +urllib3 = [ + {file = "urllib3-1.26.10-py2.py3-none-any.whl", hash = "sha256:8298d6d56d39be0e3bc13c1c97d133f9b45d797169a0e11cdd0e0489d786f7ec"}, + {file = "urllib3-1.26.10.tar.gz", hash = "sha256:879ba4d1e89654d9769ce13121e0f94310ea32e8d2f8cf587b77c08bbcdb30d6"}, +] virtualenv = [ {file = "virtualenv-20.15.1-py2.py3-none-any.whl", hash = "sha256:b30aefac647e86af6d82bfc944c556f8f1a9c90427b2fb4e3bfbf338cb82becf"}, {file = "virtualenv-20.15.1.tar.gz", hash = "sha256:288171134a2ff3bfb1a2f54f119e77cd1b81c29fc1265a2356f3e8d14c7d58c4"}, ] -zipp = [] -zstandard = [] +zipp = [ + {file = "zipp-3.8.1-py3-none-any.whl", hash = "sha256:47c40d7fe183a6f21403a199b3e4192cca5774656965b0a4988ad2f8feb5f009"}, + {file = "zipp-3.8.1.tar.gz", hash = "sha256:05b45f1ee8f807d0cc928485ca40a07cb491cf092ff587c0df9cb1fd154848d2"}, +] +zstandard = [ + {file = "zstandard-0.18.0-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:ef7e8a200e4c8ac9102ed3c90ed2aa379f6b880f63032200909c1be21951f556"}, + {file = "zstandard-0.18.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:2dc466207016564805e56d28375f4f533b525ff50d6776946980dff5465566ac"}, + {file = "zstandard-0.18.0-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:4a2ee1d4f98447f3e5183ecfce5626f983504a4a0c005fbe92e60fa8e5d547ec"}, + {file = "zstandard-0.18.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:d956e2f03c7200d7e61345e0880c292783ec26618d0d921dcad470cb195bbce2"}, + {file = "zstandard-0.18.0-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_12_i686.manylinux2010_i686.whl", hash = "sha256:ce6f59cba9854fd14da5bfe34217a1501143057313966637b7291d1b0267bd1e"}, + {file = "zstandard-0.18.0-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:a7fa67cba473623848b6e88acf8d799b1906178fd883fb3a1da24561c779593b"}, + {file = "zstandard-0.18.0-cp310-cp310-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:cdb44d7284c8c5dd1b66dfb86dda7f4560fa94bfbbc1d2da749ba44831335e32"}, + {file = "zstandard-0.18.0-cp310-cp310-win32.whl", hash = "sha256:63694a376cde0aa8b1971d06ca28e8f8b5f492779cb6ee1cc46bbc3f019a42a5"}, + {file = "zstandard-0.18.0-cp310-cp310-win_amd64.whl", hash = "sha256:702a8324cd90c74d9c8780d02bf55e79da3193c870c9665ad3a11647e3ad1435"}, + {file = "zstandard-0.18.0-cp36-cp36m-macosx_10_9_x86_64.whl", hash = "sha256:46f679bc5dfd938db4fb058218d9dc4db1336ffaf1ea774ff152ecadabd40805"}, + {file = "zstandard-0.18.0-cp36-cp36m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:dc2a4de9f363b3247d472362a65041fe4c0f59e01a2846b15d13046be866a885"}, + {file = "zstandard-0.18.0-cp36-cp36m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:bd3220d7627fd4d26397211cb3b560ec7cc4a94b75cfce89e847e8ce7fabe32d"}, + {file = "zstandard-0.18.0-cp36-cp36m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_12_i686.manylinux2010_i686.whl", hash = "sha256:39e98cf4773234bd9cebf9f9db730e451dfcfe435e220f8921242afda8321887"}, + {file = "zstandard-0.18.0-cp36-cp36m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:5228e596eb1554598c872a337bbe4e5afe41cd1f8b1b15f2e35b50d061e35244"}, + {file = "zstandard-0.18.0-cp36-cp36m-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:d4a8fd45746a6c31e729f35196e80b8f1e9987c59f5ccb8859d7c6a6fbeb9c63"}, + {file = "zstandard-0.18.0-cp36-cp36m-win32.whl", hash = "sha256:4cbb85f29a990c2fdbf7bc63246567061a362ddca886d7fae6f780267c0a9e67"}, + {file = "zstandard-0.18.0-cp36-cp36m-win_amd64.whl", hash = "sha256:bfa6c8549fa18e6497a738b7033c49f94a8e2e30c5fbe2d14d0b5aa8bbc1695d"}, + {file = "zstandard-0.18.0-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:e02043297c1832f2666cd2204f381bef43b10d56929e13c42c10c732c6e3b4ed"}, + {file = "zstandard-0.18.0-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:7231543d38d2b7e02ef7cc78ef7ffd86419437e1114ff08709fe25a160e24bd6"}, + {file = "zstandard-0.18.0-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:c86befac87445927488f5c8f205d11566f64c11519db223e9d282b945fa60dab"}, + {file = "zstandard-0.18.0-cp37-cp37m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_12_i686.manylinux2010_i686.whl", hash = "sha256:999a4e1768f219826ba3fa2064fab1c86dd72fdd47a42536235478c3bb3ca3e2"}, + {file = "zstandard-0.18.0-cp37-cp37m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:9df59cd1cf3c62075ee2a4da767089d19d874ac3ad42b04a71a167e91b384722"}, + {file = "zstandard-0.18.0-cp37-cp37m-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:1be31e9e3f7607ee0cdd60915410a5968b205d3e7aa83b7fcf3dd76dbbdb39e0"}, + {file = "zstandard-0.18.0-cp37-cp37m-win32.whl", hash = "sha256:490d11b705b8ae9dc845431bacc8dd1cef2408aede176620a5cd0cd411027936"}, + {file = "zstandard-0.18.0-cp37-cp37m-win_amd64.whl", hash = "sha256:266aba27fa9cc5e9091d3d325ebab1fa260f64e83e42516d5e73947c70216a5b"}, + {file = "zstandard-0.18.0-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:8b2260c4e07dd0723eadb586de7718b61acca4083a490dda69c5719d79bc715c"}, + {file = "zstandard-0.18.0-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:3af8c2383d02feb6650e9255491ec7d0824f6e6dd2bbe3e521c469c985f31fb1"}, + {file = "zstandard-0.18.0-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:28723a1d2e4df778573b76b321ebe9f3469ac98988104c2af116dd344802c3f8"}, + {file = "zstandard-0.18.0-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:19cac7108ff2c342317fad6dc97604b47a41f403c8f19d0bfc396dfadc3638b8"}, + {file = "zstandard-0.18.0-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_12_i686.manylinux2010_i686.whl", hash = "sha256:76725d1ee83a8915100a310bbad5d9c1fc6397410259c94033b8318d548d9990"}, + {file = "zstandard-0.18.0-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:d716a7694ce1fa60b20bc10f35c4a22be446ef7f514c8dbc8f858b61976de2fb"}, + {file = "zstandard-0.18.0-cp38-cp38-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:49685bf9a55d1ab34bd8423ea22db836ba43a181ac6b045ac4272093d5cb874e"}, + {file = "zstandard-0.18.0-cp38-cp38-win32.whl", hash = "sha256:1af1268a7dc870eb27515fb8db1f3e6c5a555d2b7bcc476fc3bab8886c7265ab"}, + {file = "zstandard-0.18.0-cp38-cp38-win_amd64.whl", hash = "sha256:1dc2d3809e763055a1a6c1a73f2b677320cc9a5aa1a7c6cfb35aee59bddc42d9"}, + {file = "zstandard-0.18.0-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:eea18c1e7442f2aa9aff1bb84550dbb6a1f711faf6e48e7319de8f2b2e923c2a"}, + {file = "zstandard-0.18.0-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:8677ffc6a6096cccbd892e558471c901fd821aba12b7fbc63833c7346f549224"}, + {file = "zstandard-0.18.0-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:083dc08abf03807af9beeb2b6a91c23ad78add2499f828176a3c7b742c44df02"}, + {file = "zstandard-0.18.0-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:c990063664c08169c84474acecc9251ee035871589025cac47c060ff4ec4bc1a"}, + {file = "zstandard-0.18.0-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_12_i686.manylinux2010_i686.whl", hash = "sha256:533db8a6fac6248b2cb2c935e7b92f994efbdeb72e1ffa0b354432e087bb5a3e"}, + {file = "zstandard-0.18.0-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:dbb3cb8a082d62b8a73af42291569d266b05605e017a3d8a06a0e5c30b5f10f0"}, + {file = "zstandard-0.18.0-cp39-cp39-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:d6c85ca5162049ede475b7ec98e87f9390501d44a3d6776ddd504e872464ec25"}, + {file = "zstandard-0.18.0-cp39-cp39-win32.whl", hash = "sha256:75479e7c2b3eebf402c59fbe57d21bc400cefa145ca356ee053b0a08908c5784"}, + {file = "zstandard-0.18.0-cp39-cp39-win_amd64.whl", hash = "sha256:d85bfabad444812133a92fc6fbe463e1d07581dba72f041f07a360e63808b23c"}, + {file = "zstandard-0.18.0.tar.gz", hash = "sha256:0ac0357a0d985b4ff31a854744040d7b5754385d1f98f7145c30e02c6865cb6f"}, +] diff --git a/pyproject.toml b/pyproject.toml index 55011bcaaa..b5f9b1e1d2 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -54,7 +54,7 @@ python-snappy = { version = "^0.6.1", optional = true } pytest = "^7.0.0" pytest-checkdocs = "^2.0.0" pre-commit = "^2.0.0" -fastavro = "^1.5.1" +fastavro = "^1.5.4" coverage = { version = "^6.0.0", extras = ["toml"] } requests-mock = "^1.9.3" From a04e3ad47cebd89e785b6ea24f76c22a58d1df6b Mon Sep 17 00:00:00 2001 From: Ryan Blue Date: Mon, 1 Aug 2022 11:49:15 -0700 Subject: [PATCH 148/642] Python: Refactor expression hierarchy (#5389) * Convert And, Or, and Not to dataclasses. * Refactor base expression types. --- pyiceberg/expressions/base.py | 139 ++++++++------------- tests/expressions/test_expressions_base.py | 12 +- 2 files changed, 55 insertions(+), 96 deletions(-) diff --git a/pyiceberg/expressions/base.py b/pyiceberg/expressions/base.py index f91093b17a..f4584fdb4c 100644 --- a/pyiceberg/expressions/base.py +++ b/pyiceberg/expressions/base.py @@ -32,50 +32,43 @@ class BooleanExpression(ABC): - """Represents a boolean expression tree.""" + """An expression that evaluates to a boolean""" @abstractmethod def __invert__(self) -> BooleanExpression: """Transform the Expression into its negated version.""" -class Bound(Generic[T], ABC): - """Represents a bound value expression.""" +class Term(Generic[T], ABC): + """A simple expression that evaluates to a value""" - def eval(self, struct: StructProtocol): # pylint: disable=W0613 - ... # pragma: no cover + +class Bound(ABC): + """Represents a bound value expression""" -class Unbound(Generic[T, B], ABC): - """Represents an unbound expression node.""" +class Unbound(Generic[B], ABC): + """Represents an unbound value expression""" @abstractmethod - def bind(self, schema: Schema, case_sensitive: bool) -> B: + def bind(self, schema: Schema, case_sensitive: bool = True) -> B: ... # pragma: no cover -class Term(ABC): - """An expression that evaluates to a value.""" - - -class BaseReference(Generic[T], Term, ABC): - """Represents a variable reference in an expression.""" - - -class BoundTerm(Bound[T], Term): - """Represents a bound term.""" +class BoundTerm(Term[T], Bound, ABC): + """Represents a bound term""" @abstractmethod def ref(self) -> BoundReference[T]: ... - -class UnboundTerm(Unbound[T, BoundTerm[T]], Term): - """Represents an unbound term.""" + @abstractmethod + def eval(self, struct: StructProtocol): # pylint: disable=W0613 + ... # pragma: no cover @dataclass(frozen=True) -class BoundReference(BoundTerm[T], BaseReference[T]): +class BoundReference(BoundTerm[T]): """A reference bound to a field in a schema Args: @@ -88,6 +81,7 @@ class BoundReference(BoundTerm[T], BaseReference[T]): def eval(self, struct: StructProtocol) -> T: """Returns the value at the referenced field's position in an object that abides by the StructProtocol + Args: struct (StructProtocol): A row object that abides by the StructProtocol and returns values given a position Returns: @@ -99,8 +93,12 @@ def ref(self) -> BoundReference[T]: return self +class UnboundTerm(Term[T], Unbound[BoundTerm[T]], ABC): + """Represents an unbound term.""" + + @dataclass(frozen=True) -class Reference(UnboundTerm[T], BaseReference[T]): +class Reference(UnboundTerm[T]): """A reference not yet bound to a field in a schema Args: @@ -112,7 +110,7 @@ class Reference(UnboundTerm[T], BaseReference[T]): name: str - def bind(self, schema: Schema, case_sensitive: bool) -> BoundReference[T]: + def bind(self, schema: Schema, case_sensitive: bool = True) -> BoundReference[T]: """Bind the reference to an Iceberg schema Args: @@ -125,22 +123,24 @@ def bind(self, schema: Schema, case_sensitive: bool) -> BoundReference[T]: Returns: BoundReference: A reference bound to the specific field in the Iceberg schema """ - field = schema.find_field(name_or_id=self.name, case_sensitive=case_sensitive) # pylint: disable=redefined-outer-name - + field = schema.find_field(name_or_id=self.name, case_sensitive=case_sensitive) if not field: raise ValueError(f"Cannot find field '{self.name}' in schema: {schema}") accessor = schema.accessor_for_field(field.field_id) - if not accessor: raise ValueError(f"Cannot find accessor for field '{self.name}' in schema: {schema}") return BoundReference(field=field, accessor=accessor) +@dataclass(frozen=True, init=False) class And(BooleanExpression): """AND operation expression - logical conjunction""" + left: BooleanExpression + right: BooleanExpression + def __new__(cls, left: BooleanExpression, right: BooleanExpression, *rest: BooleanExpression): if rest: return reduce(And, (left, right, *rest)) @@ -150,35 +150,23 @@ def __new__(cls, left: BooleanExpression, right: BooleanExpression, *rest: Boole return right elif right is AlwaysTrue(): return left - self = super().__new__(cls) - self._left = left # type: ignore - self._right = right # type: ignore - return self - - @property - def left(self) -> BooleanExpression: - return self._left # type: ignore - - @property - def right(self) -> BooleanExpression: - return self._right # type: ignore - - def __eq__(self, other) -> bool: - return id(self) == id(other) or (isinstance(other, And) and self.left == other.left and self.right == other.right) + else: + result = super().__new__(cls) + object.__setattr__(result, "left", left) + object.__setattr__(result, "right", right) + return result def __invert__(self) -> Or: return Or(~self.left, ~self.right) - def __repr__(self) -> str: - return f"And({repr(self.left)}, {repr(self.right)})" - - def __str__(self) -> str: - return f"And({str(self.left)}, {str(self.right)})" - +@dataclass(frozen=True, init=False) class Or(BooleanExpression): """OR operation expression - logical disjunction""" + left: BooleanExpression + right: BooleanExpression + def __new__(cls, left: BooleanExpression, right: BooleanExpression, *rest: BooleanExpression): if rest: return reduce(Or, (left, right, *rest)) @@ -188,35 +176,22 @@ def __new__(cls, left: BooleanExpression, right: BooleanExpression, *rest: Boole return right elif right is AlwaysFalse(): return left - self = super().__new__(cls) - self._left = left # type: ignore - self._right = right # type: ignore - return self - - @property - def left(self) -> BooleanExpression: - return self._left # type: ignore - - @property - def right(self) -> BooleanExpression: - return self._right # type: ignore - - def __eq__(self, other) -> bool: - return id(self) == id(other) or (isinstance(other, Or) and self.left == other.left and self.right == other.right) + else: + result = super().__new__(cls) + object.__setattr__(result, "left", left) + object.__setattr__(result, "right", right) + return result def __invert__(self) -> And: return And(~self.left, ~self.right) - def __repr__(self) -> str: - return f"Or({repr(self.left)}, {repr(self.right)})" - - def __str__(self) -> str: - return f"Or({str(self.left)}, {str(self.right)})" - +@dataclass(frozen=True, init=False) class Not(BooleanExpression): """NOT operation expression - logical negation""" + child: BooleanExpression + def __new__(cls, child: BooleanExpression): if child is AlwaysTrue(): return AlwaysFalse() @@ -224,23 +199,13 @@ def __new__(cls, child: BooleanExpression): return AlwaysTrue() elif isinstance(child, Not): return child.child - return super().__new__(cls) - - def __init__(self, child): - self.child = child - - def __eq__(self, other) -> bool: - return id(self) == id(other) or (isinstance(other, Not) and self.child == other.child) + result = super().__new__(cls) + object.__setattr__(result, "child", child) + return result def __invert__(self) -> BooleanExpression: return self.child - def __repr__(self) -> str: - return f"Not({repr(self.child)})" - - def __str__(self) -> str: - return f"Not({str(self.child)})" - @dataclass(frozen=True) class AlwaysTrue(BooleanExpression, Singleton): @@ -259,7 +224,7 @@ def __invert__(self) -> AlwaysTrue: @dataclass(frozen=True) -class BoundPredicate(Bound[T], BooleanExpression): +class BoundPredicate(Generic[T], Bound, BooleanExpression): term: BoundTerm[T] def __invert__(self) -> BoundPredicate[T]: @@ -267,7 +232,7 @@ def __invert__(self) -> BoundPredicate[T]: @dataclass(frozen=True) -class UnboundPredicate(Unbound[T, BooleanExpression], BooleanExpression): +class UnboundPredicate(Generic[T], Unbound[BooleanExpression], BooleanExpression): as_bound: ClassVar[type] term: UnboundTerm[T] @@ -661,12 +626,6 @@ def _(obj: And, visitor: BooleanExpressionVisitor[T]) -> T: return visitor.visit_and(left_result=left_result, right_result=right_result) -@visit.register(In) -def _(obj: In, visitor: BooleanExpressionVisitor[T]) -> T: - """Visit an In boolean expression with a concrete BooleanExpressionVisitor""" - return visitor.visit_unbound_predicate(predicate=obj) - - @visit.register(UnboundPredicate) def _(obj: UnboundPredicate, visitor: BooleanExpressionVisitor[T]) -> T: """Visit an In boolean expression with a concrete BooleanExpressionVisitor""" diff --git a/tests/expressions/test_expressions_base.py b/tests/expressions/test_expressions_base.py index ba2850133b..cf74298296 100644 --- a/tests/expressions/test_expressions_base.py +++ b/tests/expressions/test_expressions_base.py @@ -120,13 +120,13 @@ def _(obj: ExpressionB, visitor: BooleanExpressionVisitor) -> List: [ ( base.And(ExpressionA(), ExpressionB()), - "And(ExpressionA(), ExpressionB())", + "And(left=ExpressionA(), right=ExpressionB())", ), ( base.Or(ExpressionA(), ExpressionB()), - "Or(ExpressionA(), ExpressionB())", + "Or(left=ExpressionA(), right=ExpressionB())", ), - (base.Not(ExpressionA()), "Not(ExpressionA())"), + (base.Not(ExpressionA()), "Not(child=ExpressionA())"), ], ) def test_reprs(op, rep): @@ -208,9 +208,9 @@ def test_notnan_bind_nonfloat(): @pytest.mark.parametrize( "op, string", [ - (base.And(ExpressionA(), ExpressionB()), "And(testexpra, testexprb)"), - (base.Or(ExpressionA(), ExpressionB()), "Or(testexpra, testexprb)"), - (base.Not(ExpressionA()), "Not(testexpra)"), + (base.And(ExpressionA(), ExpressionB()), "And(left=ExpressionA(), right=ExpressionB())"), + (base.Or(ExpressionA(), ExpressionB()), "Or(left=ExpressionA(), right=ExpressionB())"), + (base.Not(ExpressionA()), "Not(child=ExpressionA())"), ], ) def test_strs(op, string): From d63f8bd1cc13d4097581e87bdadd4f71843147d0 Mon Sep 17 00:00:00 2001 From: Fokko Driesprong Date: Tue, 2 Aug 2022 01:30:06 +0200 Subject: [PATCH 149/642] Python: Minor REST catalog updates (#5402) --- pyiceberg/catalog/rest.py | 32 +++++++------- pyiceberg/exceptions.py | 16 +++---- tests/catalog/test_rest.py | 89 +++++++++++++++++++------------------- 3 files changed, 68 insertions(+), 69 deletions(-) diff --git a/pyiceberg/catalog/rest.py b/pyiceberg/catalog/rest.py index fc228295c9..188f5a8a6d 100644 --- a/pyiceberg/catalog/rest.py +++ b/pyiceberg/catalog/rest.py @@ -152,15 +152,14 @@ class RestCatalog(Catalog): token: str config: Properties - host: str + uri: str def __init__( self, name: str, properties: Properties, - host: str, - client_id: Optional[str] = None, - client_secret: Optional[str] = None, + uri: str, + credentials: Optional[str] = None, token: Optional[str] = None, ): """Rest Catalog @@ -170,18 +169,15 @@ def __init__( Args: name: Name to identify the catalog properties: Properties that are passed along to the configuration - host: The base-url of the REST Catalog endpoint - client_id: The id to identify the client - client_secret: The secret for the client + uri: The base-url of the REST Catalog endpoint + credentials: The credentials for authentication against the client token: The bearer token """ - self.host = host - if client_id and client_secret: - self.token = self._fetch_access_token(client_id, client_secret) + self.uri = uri + if credentials: + self.token = self._fetch_access_token(credentials) elif token: self.token = token - else: - raise ValueError("Either set the client_id and client_secret, or provide a valid token") self.config = self._fetch_config(properties) super().__init__(name, properties) @@ -200,23 +196,26 @@ def _split_credential(token: str) -> Tuple[str, str]: @property def headers(self) -> Properties: - return { - AUTHORIZATION_HEADER: f"{BEARER_PREFIX} {self.token}", + headers = { "Content-type": "application/json", "X-Client-Version": __version__, } + if self.token: + headers[AUTHORIZATION_HEADER] = f"{BEARER_PREFIX} {self.token}" + return headers def url(self, endpoint: str, prefixed: bool = True, **kwargs) -> str: """Constructs the endpoint Args: + endpoint: Resource identifier that points to the REST catalog prefixed: If the prefix return by the config needs to be appended Returns: The base url of the rest catalog """ - url = self.host + url = self.uri url = url + "v1/" if url.endswith("/") else url + "/v1/" if prefixed: @@ -225,7 +224,8 @@ def url(self, endpoint: str, prefixed: bool = True, **kwargs) -> str: return url + endpoint.format(**kwargs) - def _fetch_access_token(self, client_id: str, client_secret: str) -> str: + def _fetch_access_token(self, credentials: str) -> str: + client_id, client_secret = credentials.split(":") data = {GRANT_TYPE: CLIENT_CREDENTIALS, CLIENT_ID: client_id, CLIENT_SECRET: client_secret, SCOPE: CATALOG_SCOPE} url = self.url(Endpoints.get_token, prefixed=False) # Uses application/x-www-form-urlencoded by default diff --git a/pyiceberg/exceptions.py b/pyiceberg/exceptions.py index f38cd0df67..517252f9a1 100644 --- a/pyiceberg/exceptions.py +++ b/pyiceberg/exceptions.py @@ -32,6 +32,14 @@ class ValidationError(Exception): """Raises when there is an issue with the schema""" +class NoSuchTableError(Exception): + """Raises when the table can't be found in the REST catalog""" + + +class NoSuchNamespaceError(Exception): + """Raised when a referenced name-space is not found""" + + class RESTError(Exception): """Raises when there is an unknown response from the REST Catalog""" @@ -62,11 +70,3 @@ class ForbiddenError(RESTError): class AuthorizationExpiredError(RESTError): """When the credentials are expired when performing an action on the REST catalog""" - - -class NoSuchTableError(RESTError): - """Raises when the table can't be found in the REST catalog""" - - -class NoSuchNamespaceError(RESTError): - """Raised when a referenced name-space is not found""" diff --git a/tests/catalog/test_rest.py b/tests/catalog/test_rest.py index d5ccfe5b58..cec2023035 100644 --- a/tests/catalog/test_rest.py +++ b/tests/catalog/test_rest.py @@ -43,9 +43,8 @@ StringType, ) -TEST_HOST = "https://iceberg-test-catalog/" -TEST_CLIENT_ID = "client" -TEST_CLIENT_SECRET = "secret" +TEST_URI = "https://iceberg-test-catalog/" +TEST_CREDENTIALS = "client:secret" TEST_TOKEN = "some_jwt_token" @@ -56,7 +55,7 @@ def rest_mock(requests_mock: Mocker): This endpoint is called when initializing the rest catalog """ requests_mock.get( - f"{TEST_HOST}v1/config", + f"{TEST_URI}v1/config", json={"defaults": {}, "overrides": {}}, status_code=200, ) @@ -65,7 +64,7 @@ def rest_mock(requests_mock: Mocker): def test_token_200(rest_mock: Mocker): rest_mock.post( - f"{TEST_HOST}v1/oauth/tokens", + f"{TEST_URI}v1/oauth/tokens", json={ "access_token": TEST_TOKEN, "token_type": "Bearer", @@ -74,13 +73,13 @@ def test_token_200(rest_mock: Mocker): }, status_code=200, ) - assert RestCatalog("rest", {}, TEST_HOST, TEST_CLIENT_ID, TEST_CLIENT_SECRET).token == TEST_TOKEN + assert RestCatalog("rest", {}, TEST_URI, TEST_CREDENTIALS).token == TEST_TOKEN def test_token_401(rest_mock: Mocker): message = "Invalid client ID: abc" rest_mock.post( - f"{TEST_HOST}v1/oauth/tokens", + f"{TEST_URI}v1/oauth/tokens", json={ "error": { "message": message, @@ -92,25 +91,25 @@ def test_token_401(rest_mock: Mocker): ) with pytest.raises(BadCredentialsError) as e: - RestCatalog("rest", {}, TEST_HOST, client_id=TEST_CLIENT_ID, client_secret=TEST_CLIENT_SECRET) + RestCatalog("rest", {}, TEST_URI, credentials=TEST_CREDENTIALS) assert message in str(e.value) def test_list_tables_200(rest_mock: Mocker): namespace = "examples" rest_mock.get( - f"{TEST_HOST}v1/namespaces/{namespace}/tables", + f"{TEST_URI}v1/namespaces/{namespace}/tables", json={"identifiers": [{"namespace": ["examples"], "name": "fooshare"}]}, status_code=200, ) - assert RestCatalog("rest", {}, TEST_HOST, token=TEST_TOKEN).list_tables(namespace) == [("examples", "fooshare")] + assert RestCatalog("rest", {}, TEST_URI, token=TEST_TOKEN).list_tables(namespace) == [("examples", "fooshare")] def test_list_tables_404(rest_mock: Mocker): namespace = "examples" rest_mock.get( - f"{TEST_HOST}v1/namespaces/{namespace}/tables", + f"{TEST_URI}v1/namespaces/{namespace}/tables", json={ "error": { "message": "Namespace does not exist: personal in warehouse 8bcb0838-50fc-472d-9ddb-8feb89ef5f1e", @@ -121,17 +120,17 @@ def test_list_tables_404(rest_mock: Mocker): status_code=404, ) with pytest.raises(NoSuchNamespaceError) as e: - RestCatalog("rest", {}, TEST_HOST, token=TEST_TOKEN).list_tables(namespace) + RestCatalog("rest", {}, TEST_URI, token=TEST_TOKEN).list_tables(namespace) assert "Namespace does not exist" in str(e.value) def test_list_namespaces_200(rest_mock: Mocker): rest_mock.get( - f"{TEST_HOST}v1/namespaces", + f"{TEST_URI}v1/namespaces", json={"namespaces": [["default"], ["examples"], ["fokko"], ["system"]]}, status_code=200, ) - assert RestCatalog("rest", {}, TEST_HOST, token=TEST_TOKEN).list_namespaces() == [ + assert RestCatalog("rest", {}, TEST_URI, token=TEST_TOKEN).list_namespaces() == [ ("default",), ("examples",), ("fokko",), @@ -142,17 +141,17 @@ def test_list_namespaces_200(rest_mock: Mocker): def test_create_namespace_200(rest_mock: Mocker): namespace = "leden" rest_mock.post( - f"{TEST_HOST}v1/namespaces", + f"{TEST_URI}v1/namespaces", json={"namespace": [namespace], "properties": {}}, status_code=200, ) - RestCatalog("rest", {}, TEST_HOST, token=TEST_TOKEN).create_namespace(namespace) + RestCatalog("rest", {}, TEST_URI, token=TEST_TOKEN).create_namespace(namespace) def test_create_namespace_409(rest_mock: Mocker): namespace = "examples" rest_mock.post( - f"{TEST_HOST}v1/namespaces", + f"{TEST_URI}v1/namespaces", json={ "error": { "message": "Namespace already exists: fokko in warehouse 8bcb0838-50fc-472d-9ddb-8feb89ef5f1e", @@ -163,14 +162,14 @@ def test_create_namespace_409(rest_mock: Mocker): status_code=409, ) with pytest.raises(AlreadyExistsError) as e: - RestCatalog("rest", {}, TEST_HOST, token=TEST_TOKEN).create_namespace(namespace) + RestCatalog("rest", {}, TEST_URI, token=TEST_TOKEN).create_namespace(namespace) assert "Namespace already exists" in str(e.value) def test_drop_namespace_404(rest_mock: Mocker): namespace = "examples" rest_mock.delete( - f"{TEST_HOST}v1/namespaces/{namespace}", + f"{TEST_URI}v1/namespaces/{namespace}", json={ "error": { "message": "Namespace does not exist: leden in warehouse 8bcb0838-50fc-472d-9ddb-8feb89ef5f1e", @@ -181,24 +180,24 @@ def test_drop_namespace_404(rest_mock: Mocker): status_code=404, ) with pytest.raises(NoSuchNamespaceError) as e: - RestCatalog("rest", {}, TEST_HOST, token=TEST_TOKEN).drop_namespace(namespace) + RestCatalog("rest", {}, TEST_URI, token=TEST_TOKEN).drop_namespace(namespace) assert "Namespace does not exist" in str(e.value) def test_load_namespace_properties_200(rest_mock: Mocker): namespace = "leden" rest_mock.get( - f"{TEST_HOST}v1/namespaces/{namespace}", + f"{TEST_URI}v1/namespaces/{namespace}", json={"namespace": ["fokko"], "properties": {"prop": "yes"}}, status_code=204, ) - assert RestCatalog("rest", {}, TEST_HOST, token=TEST_TOKEN).load_namespace_properties(namespace) == {"prop": "yes"} + assert RestCatalog("rest", {}, TEST_URI, token=TEST_TOKEN).load_namespace_properties(namespace) == {"prop": "yes"} def test_load_namespace_properties_404(rest_mock: Mocker): namespace = "leden" rest_mock.get( - f"{TEST_HOST}v1/namespaces/{namespace}", + f"{TEST_URI}v1/namespaces/{namespace}", json={ "error": { "message": "Namespace does not exist: fokko22 in warehouse 8bcb0838-50fc-472d-9ddb-8feb89ef5f1e", @@ -209,17 +208,17 @@ def test_load_namespace_properties_404(rest_mock: Mocker): status_code=404, ) with pytest.raises(NoSuchNamespaceError) as e: - RestCatalog("rest", {}, TEST_HOST, token=TEST_TOKEN).load_namespace_properties(namespace) + RestCatalog("rest", {}, TEST_URI, token=TEST_TOKEN).load_namespace_properties(namespace) assert "Namespace does not exist" in str(e.value) def test_update_namespace_properties_200(rest_mock: Mocker): rest_mock.post( - f"{TEST_HOST}v1/namespaces/fokko/properties", + f"{TEST_URI}v1/namespaces/fokko/properties", json={"removed": [], "updated": ["prop"], "missing": ["abc"]}, status_code=200, ) - response = RestCatalog("rest", {}, TEST_HOST, token=TEST_TOKEN).update_namespace_properties( + response = RestCatalog("rest", {}, TEST_URI, token=TEST_TOKEN).update_namespace_properties( ("fokko",), {"abc"}, {"prop": "yes"} ) @@ -228,7 +227,7 @@ def test_update_namespace_properties_200(rest_mock: Mocker): def test_update_namespace_properties_404(rest_mock: Mocker): rest_mock.post( - f"{TEST_HOST}v1/namespaces/fokko/properties", + f"{TEST_URI}v1/namespaces/fokko/properties", json={ "error": { "message": "Namespace does not exist: does_not_exists in warehouse 8bcb0838-50fc-472d-9ddb-8feb89ef5f1e", @@ -239,13 +238,13 @@ def test_update_namespace_properties_404(rest_mock: Mocker): status_code=404, ) with pytest.raises(NoSuchNamespaceError) as e: - RestCatalog("rest", {}, TEST_HOST, token=TEST_TOKEN).update_namespace_properties(("fokko",), {"abc"}, {"prop": "yes"}) + RestCatalog("rest", {}, TEST_URI, token=TEST_TOKEN).update_namespace_properties(("fokko",), {"abc"}, {"prop": "yes"}) assert "Namespace does not exist" in str(e.value) def test_load_table_200(rest_mock: Mocker): rest_mock.get( - f"{TEST_HOST}v1/namespaces/fokko/tables/table", + f"{TEST_URI}v1/namespaces/fokko/tables/table", json={ "metadataLocation": "s3://tabular-public-us-west-2-dev/bb30733e-8769-4dab-aa1b-e76245bb2bd4/b55d9dda-6561-423a-8bfc-787980ce421f/metadata/00001-5f2f8166-244c-4eae-ac36-384ecdec81fc.gz.metadata.json", "metadata": { @@ -316,7 +315,7 @@ def test_load_table_200(rest_mock: Mocker): }, status_code=200, ) - table = RestCatalog("rest", {}, TEST_HOST, token=TEST_TOKEN).load_table(("fokko", "table")) + table = RestCatalog("rest", {}, TEST_URI, token=TEST_TOKEN).load_table(("fokko", "table")) assert table == Table( identifier=("rest", "fokko", "table"), metadata_location=None, @@ -398,7 +397,7 @@ def test_load_table_200(rest_mock: Mocker): def test_load_table_404(rest_mock: Mocker): rest_mock.get( - f"{TEST_HOST}v1/namespaces/fokko/tables/does_not_exists", + f"{TEST_URI}v1/namespaces/fokko/tables/does_not_exists", json={ "error": { "message": "Table does not exist: examples.does_not_exists in warehouse 8bcb0838-50fc-472d-9ddb-8feb89ef5f1e", @@ -410,13 +409,13 @@ def test_load_table_404(rest_mock: Mocker): ) with pytest.raises(NoSuchTableError) as e: - RestCatalog("rest", {}, TEST_HOST, token=TEST_TOKEN).load_table(("fokko", "does_not_exists")) + RestCatalog("rest", {}, TEST_URI, token=TEST_TOKEN).load_table(("fokko", "does_not_exists")) assert "Table does not exist" in str(e.value) def test_drop_table_404(rest_mock: Mocker): rest_mock.delete( - f"{TEST_HOST}v1/namespaces/fokko/tables/does_not_exists", + f"{TEST_URI}v1/namespaces/fokko/tables/does_not_exists", json={ "error": { "message": "Table does not exist: fokko.does_not_exists in warehouse 8bcb0838-50fc-472d-9ddb-8feb89ef5f1e", @@ -428,13 +427,13 @@ def test_drop_table_404(rest_mock: Mocker): ) with pytest.raises(NoSuchTableError) as e: - RestCatalog("rest", {}, TEST_HOST, token=TEST_TOKEN).drop_table(("fokko", "does_not_exists")) + RestCatalog("rest", {}, TEST_URI, token=TEST_TOKEN).drop_table(("fokko", "does_not_exists")) assert "Table does not exist" in str(e.value) def test_create_table_200(rest_mock: Mocker, table_schema_simple: Schema): rest_mock.post( - f"{TEST_HOST}v1/namespaces/fokko/tables", + f"{TEST_URI}v1/namespaces/fokko/tables", json={ "metadataLocation": None, "metadata": { @@ -491,7 +490,7 @@ def test_create_table_200(rest_mock: Mocker, table_schema_simple: Schema): }, status_code=200, ) - table = RestCatalog("rest", {}, TEST_HOST, token=TEST_TOKEN).create_table( + table = RestCatalog("rest", {}, TEST_URI, token=TEST_TOKEN).create_table( identifier=("fokko", "fokko2"), schema=table_schema_simple, location=None, @@ -550,7 +549,7 @@ def test_create_table_200(rest_mock: Mocker, table_schema_simple: Schema): def test_create_table_409(rest_mock, table_schema_simple: Schema): rest_mock.post( - f"{TEST_HOST}v1/namespaces/fokko/tables", + f"{TEST_URI}v1/namespaces/fokko/tables", json={ "error": { "message": "Table already exists: fokko.already_exists in warehouse 8bcb0838-50fc-472d-9ddb-8feb89ef5f1e", @@ -562,7 +561,7 @@ def test_create_table_409(rest_mock, table_schema_simple: Schema): ) with pytest.raises(TableAlreadyExistsError) as e: - RestCatalog("rest", {}, TEST_HOST, token=TEST_TOKEN).create_table( + RestCatalog("rest", {}, TEST_URI, token=TEST_TOKEN).create_table( identifier=("fokko", "fokko2"), schema=table_schema_simple, location=None, @@ -579,34 +578,34 @@ def test_create_table_409(rest_mock, table_schema_simple: Schema): def test_delete_namespace_204(rest_mock: Mocker): namespace = "example" rest_mock.delete( - f"{TEST_HOST}v1/namespaces/{namespace}", + f"{TEST_URI}v1/namespaces/{namespace}", json={}, status_code=204, ) - RestCatalog("rest", {}, TEST_HOST, token=TEST_TOKEN).drop_namespace(namespace) + RestCatalog("rest", {}, TEST_URI, token=TEST_TOKEN).drop_namespace(namespace) def test_delete_table_204(rest_mock: Mocker): rest_mock.delete( - f"{TEST_HOST}v1/namespaces/example/tables/fokko", + f"{TEST_URI}v1/namespaces/example/tables/fokko", json={}, status_code=204, ) - RestCatalog("rest", {}, TEST_HOST, token=TEST_TOKEN).drop_table(("example", "fokko")) + RestCatalog("rest", {}, TEST_URI, token=TEST_TOKEN).drop_table(("example", "fokko")) def test_delete_table_404(rest_mock: Mocker): rest_mock.delete( - f"{TEST_HOST}v1/namespaces/example/tables/fokko", + f"{TEST_URI}v1/namespaces/example/tables/fokko", json={ "error": { "message": "Table does not exist: fokko.fokko2 in warehouse 8bcb0838-50fc-472d-9ddb-8feb89ef5f1e", - "type": "NoSuchNamespaceErrorException", + "type": "NoSuchTableException", "code": 404, } }, status_code=404, ) with pytest.raises(NoSuchTableError) as e: - RestCatalog("rest", {}, TEST_HOST, token=TEST_TOKEN).drop_table(("example", "fokko")) + RestCatalog("rest", {}, TEST_URI, token=TEST_TOKEN).drop_table(("example", "fokko")) assert "Table does not exist" in str(e.value) From 9e34a649a26883d816b191b82234a50535323383 Mon Sep 17 00:00:00 2001 From: Fokko Driesprong Date: Wed, 3 Aug 2022 00:33:03 +0200 Subject: [PATCH 150/642] Python: Cleanup catalog docstring (#5421) --- pyiceberg/catalog/base.py | 71 ++++++++++++++++++++------------------ pyiceberg/catalog/rest.py | 4 +-- pyiceberg/exceptions.py | 4 +-- tests/catalog/test_base.py | 11 +++--- tests/catalog/test_rest.py | 4 +-- 5 files changed, 49 insertions(+), 45 deletions(-) diff --git a/pyiceberg/catalog/base.py b/pyiceberg/catalog/base.py index d70fd7b57c..a0e2e6296c 100644 --- a/pyiceberg/catalog/base.py +++ b/pyiceberg/catalog/base.py @@ -44,8 +44,8 @@ class Catalog(ABC): or tuple of strings. Attributes: - name(str): Name of the catalog - properties(Properties): Catalog properties + name (str): Name of the catalog + properties (Properties): Catalog properties """ def __init__(self, name: str, properties: Properties): @@ -73,18 +73,18 @@ def create_table( """Create a table Args: - identifier: Table identifier. - schema: Table's schema. - location: Location for the table. Optional Argument. - partition_spec: PartitionSpec for the table. - sort_order: SortOrder for the table. - properties: Table properties that can be a string based dictionary. Optional Argument. + identifier (str | Identifier): Table identifier. + schema (Schema): Table's schema. + location (str): Location for the table. Optional Argument. + partition_spec (PartitionSpec): PartitionSpec for the table. + sort_order (SortOrder): SortOrder for the table. + properties (Properties | None): Table properties that can be a string based dictionary. Optional Argument. Returns: Table: the created table instance Raises: - AlreadyExistsError: If a table with the name already exists + TableAlreadyExistsError: If a table with the name already exists """ @abstractmethod @@ -95,13 +95,13 @@ def load_table(self, identifier: str | Identifier) -> Table: Note: This method doesn't scan data stored in the table. Args: - identifier: Table identifier. + identifier (str | Identifier): Table identifier. Returns: Table: the table instance with its metadata Raises: - TableNotFoundError: If a table with the name does not exist + NoSuchTableError: If a table with the name does not exist """ @abstractmethod @@ -109,10 +109,10 @@ def drop_table(self, identifier: str | Identifier) -> None: """Drop a table. Args: - identifier: Table identifier. + identifier (str | Identifier): Table identifier. Raises: - TableNotFoundError: If a table with the name does not exist + NoSuchTableError: If a table with the name does not exist """ @abstractmethod @@ -120,10 +120,10 @@ def purge_table(self, identifier: str | Identifier) -> None: """Drop a table and purge all data and metadata files. Args: - identifier: Table identifier. + identifier (str | Identifier): Table identifier. Raises: - TableNotFoundError: If a table with the name does not exist + NoSuchTableError: If a table with the name does not exist """ @abstractmethod @@ -131,14 +131,14 @@ def rename_table(self, from_identifier: str | Identifier, to_identifier: str | I """Rename a fully classified table name Args: - from_identifier: Existing table identifier. - to_identifier: New table identifier. + from_identifier (str | Identifier): Existing table identifier. + to_identifier (str | Identifier): New table identifier. Returns: Table: the updated table instance with its metadata Raises: - TableNotFoundError: If a table with the name does not exist + NoSuchTableError: If a table with the name does not exist """ @abstractmethod @@ -146,11 +146,11 @@ def create_namespace(self, namespace: str | Identifier, properties: Properties | """Create a namespace in the catalog. Args: - namespace: Namespace identifier - properties: A string dictionary of properties for the given namespace + namespace (str | Identifier): Namespace identifier + properties (Properties | None): A string dictionary of properties for the given namespace Raises: - AlreadyExistsError: If a namespace with the given name already exists + NamespaceAlreadyExistsError: If a namespace with the given name already exists """ @abstractmethod @@ -158,10 +158,10 @@ def drop_namespace(self, namespace: str | Identifier) -> None: """Drop a namespace. Args: - namespace: Namespace identifier + namespace (str | Identifier): Namespace identifier Raises: - NamespaceNotFoundError: If a namespace with the given name does not exist + NoSuchNamespaceError: If a namespace with the given name does not exist NamespaceNotEmptyError: If the namespace is not empty """ @@ -172,13 +172,13 @@ def list_tables(self, namespace: str | Identifier | None = None) -> list[Identif If namespace not provided, will list all tables in the catalog. Args: - namespace: Namespace identifier to search. + namespace (str | Identifier | None): Namespace identifier to search. Returns: List[Identifier]: list of table identifiers. Raises: - NamespaceNotFoundError: If a namespace with the given name does not exist + NoSuchNamespaceError: If a namespace with the given name does not exist """ @abstractmethod @@ -187,6 +187,9 @@ def list_namespaces(self) -> list[Identifier]: Returns: List[Identifier]: a List of namespace identifiers + + Raises: + NoSuchNamespaceError: If a namespace with the given name does not exist """ @abstractmethod @@ -194,13 +197,13 @@ def load_namespace_properties(self, namespace: str | Identifier) -> Properties: """Get properties for a namespace. Args: - namespace: Namespace identifier + namespace (str | Identifier): Namespace identifier Returns: Properties: Properties for the given namespace Raises: - NamespaceNotFoundError: If a namespace with the given name does not exist + NoSuchNamespaceError: If a namespace with the given name does not exist """ @abstractmethod @@ -210,12 +213,12 @@ def update_namespace_properties( """Removes provided property keys and updates properties for a namespace. Args: - namespace: Namespace identifier - removals: Set of property keys that need to be removed. Optional Argument. - updates: Properties to be updated for the given namespace. Optional Argument. + namespace (str | Identifier): Namespace identifier + removals (Set[str]): Set of property keys that need to be removed. Optional Argument. + updates (Properties | None): Properties to be updated for the given namespace. Optional Argument. Raises: - NamespaceNotFoundError: If a namespace with the given name does not exist + NoSuchNamespaceError: If a namespace with the given name does not exist ValueError: If removals and updates have overlapping keys. """ @@ -226,7 +229,7 @@ def identifier_to_tuple(identifier: str | Identifier) -> Identifier: If the identifier is a string, it is split into a tuple on '.'. If it is a tuple, it is used as-is. Args: - identifier: an identifier, either a string or tuple of strings + identifier (str | Identifier: an identifier, either a string or tuple of strings Returns: Identifier: a tuple of strings @@ -238,7 +241,7 @@ def table_name_from(identifier: str | Identifier) -> str: """Extracts table name from a table identifier Args: - identifier: a table identifier + identifier (str | Identifier: a table identifier Returns: str: Table name @@ -250,7 +253,7 @@ def namespace_from(identifier: str | Identifier) -> Identifier: """Extracts table namespace from a table identifier Args: - identifier: a table identifier + identifier (str | Identifier: a table identifier Returns: Identifier: Namespace identifier diff --git a/pyiceberg/catalog/rest.py b/pyiceberg/catalog/rest.py index 188f5a8a6d..b5938267ba 100644 --- a/pyiceberg/catalog/rest.py +++ b/pyiceberg/catalog/rest.py @@ -33,11 +33,11 @@ from pyiceberg.catalog import Identifier, Properties from pyiceberg.catalog.base import Catalog, PropertiesUpdateSummary from pyiceberg.exceptions import ( - AlreadyExistsError, AuthorizationExpiredError, BadCredentialsError, BadRequestError, ForbiddenError, + NamespaceAlreadyExistsError, NoSuchNamespaceError, NoSuchTableError, RESTError, @@ -386,7 +386,7 @@ def create_namespace(self, namespace: Union[str, Identifier], properties: Option try: response.raise_for_status() except HTTPError as exc: - self._handle_non_200_response(exc, {404: NoSuchNamespaceError, 409: AlreadyExistsError}) + self._handle_non_200_response(exc, {404: NoSuchNamespaceError, 409: NamespaceAlreadyExistsError}) def drop_namespace(self, namespace: Union[str, Identifier]) -> None: namespace = NAMESPACE_SEPARATOR.join(self.identifier_to_tuple(namespace)) diff --git a/pyiceberg/exceptions.py b/pyiceberg/exceptions.py index 517252f9a1..432c25675f 100644 --- a/pyiceberg/exceptions.py +++ b/pyiceberg/exceptions.py @@ -24,8 +24,8 @@ class NamespaceNotEmptyError(Exception): """Raised when a name-space being dropped is not empty""" -class AlreadyExistsError(Exception): - """Raised when a table or name-space being created already exists in the catalog""" +class NamespaceAlreadyExistsError(Exception): + """Raised when a name-space being created already exists in the catalog""" class ValidationError(Exception): diff --git a/tests/catalog/test_base.py b/tests/catalog/test_base.py index 49790ecbf3..3ca41da083 100644 --- a/tests/catalog/test_base.py +++ b/tests/catalog/test_base.py @@ -28,10 +28,11 @@ from pyiceberg.catalog import Identifier, Properties from pyiceberg.catalog.base import Catalog, PropertiesUpdateSummary from pyiceberg.exceptions import ( - AlreadyExistsError, + NamespaceAlreadyExistsError, NamespaceNotEmptyError, NoSuchNamespaceError, NoSuchTableError, + TableAlreadyExistsError, ) from pyiceberg.schema import Schema from pyiceberg.table.base import Table @@ -66,7 +67,7 @@ def create_table( namespace = Catalog.namespace_from(identifier) if identifier in self.__tables: - raise AlreadyExistsError(f"Table already exists: {identifier}") + raise TableAlreadyExistsError(f"Table already exists: {identifier}") else: if namespace not in self.__namespaces: self.__namespaces[namespace] = {} @@ -110,7 +111,7 @@ def rename_table(self, from_identifier: Union[str, Identifier], to_identifier: U def create_namespace(self, namespace: Union[str, Identifier], properties: Optional[Properties] = None) -> None: namespace = Catalog.identifier_to_tuple(namespace) if namespace in self.__namespaces: - raise AlreadyExistsError(f"Namespace already exists: {namespace}") + raise NamespaceAlreadyExistsError(f"Namespace already exists: {namespace}") else: self.__namespaces[namespace] = properties if properties else {} @@ -244,7 +245,7 @@ def test_create_table_raises_error_when_table_already_exists(catalog: InMemoryCa # Given given_catalog_has_a_table(catalog) # When - with pytest.raises(AlreadyExistsError, match=TABLE_ALREADY_EXISTS_ERROR): + with pytest.raises(TableAlreadyExistsError, match=TABLE_ALREADY_EXISTS_ERROR): catalog.create_table( identifier=TEST_TABLE_IDENTIFIER, schema=TEST_TABLE_SCHEMA, @@ -326,7 +327,7 @@ def test_create_namespace_raises_error_on_existing_namespace(catalog: InMemoryCa # Given catalog.create_namespace(TEST_TABLE_NAMESPACE, TEST_TABLE_PROPERTIES) # When - with pytest.raises(AlreadyExistsError, match=NAMESPACE_ALREADY_EXISTS_ERROR): + with pytest.raises(NamespaceAlreadyExistsError, match=NAMESPACE_ALREADY_EXISTS_ERROR): catalog.create_namespace(TEST_TABLE_NAMESPACE, TEST_TABLE_PROPERTIES) diff --git a/tests/catalog/test_rest.py b/tests/catalog/test_rest.py index cec2023035..b3c7b661f2 100644 --- a/tests/catalog/test_rest.py +++ b/tests/catalog/test_rest.py @@ -23,8 +23,8 @@ from pyiceberg.catalog.base import PropertiesUpdateSummary, Table from pyiceberg.catalog.rest import RestCatalog from pyiceberg.exceptions import ( - AlreadyExistsError, BadCredentialsError, + NamespaceAlreadyExistsError, NoSuchNamespaceError, NoSuchTableError, TableAlreadyExistsError, @@ -161,7 +161,7 @@ def test_create_namespace_409(rest_mock: Mocker): }, status_code=409, ) - with pytest.raises(AlreadyExistsError) as e: + with pytest.raises(NamespaceAlreadyExistsError) as e: RestCatalog("rest", {}, TEST_URI, token=TEST_TOKEN).create_namespace(namespace) assert "Namespace already exists" in str(e.value) From ebf09589e17529c44b769220c045a1b1e26f4324 Mon Sep 17 00:00:00 2001 From: Fokko Driesprong Date: Wed, 3 Aug 2022 01:50:17 +0200 Subject: [PATCH 151/642] Python: Add Hive Catalog (#5391) --- .pre-commit-config.yaml | 1 + LICENSE | 20 + Makefile | 2 +- poetry.lock | 236 +- pyiceberg/catalog/base.py | 4 +- pyiceberg/catalog/hive.py | 459 + pyiceberg/catalog/rest.py | 4 +- pyiceberg/table/base.py | 2 +- pyproject.toml | 14 + tests/catalog/test_hive.py | 446 + vendor/README.md | 45 + vendor/fb303/FacebookService.py | 2420 + vendor/fb303/__init__.py | 18 + vendor/fb303/constants.py | 26 + vendor/fb303/ttypes.py | 64 + vendor/hive_metastore/ThriftHiveMetastore.py | 72960 +++++++++++++++++ vendor/hive_metastore/__init__.py | 17 + vendor/hive_metastore/constants.py | 66 + vendor/hive_metastore/ttypes.py | 42515 ++++++++++ 19 files changed, 119125 insertions(+), 194 deletions(-) create mode 100644 pyiceberg/catalog/hive.py create mode 100644 tests/catalog/test_hive.py create mode 100644 vendor/README.md create mode 100644 vendor/fb303/FacebookService.py create mode 100644 vendor/fb303/__init__.py create mode 100644 vendor/fb303/constants.py create mode 100644 vendor/fb303/ttypes.py create mode 100644 vendor/hive_metastore/ThriftHiveMetastore.py create mode 100644 vendor/hive_metastore/__init__.py create mode 100644 vendor/hive_metastore/constants.py create mode 100644 vendor/hive_metastore/ttypes.py diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 717b429783..7c9875a468 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -16,6 +16,7 @@ # under the License. --- files: ^python/ +exclude: ^python/vendor/ repos: - repo: https://github.com/pre-commit/pre-commit-hooks diff --git a/LICENSE b/LICENSE index 5823c32b65..ffdd12aad2 100644 --- a/LICENSE +++ b/LICENSE @@ -213,3 +213,23 @@ Home page: https://avro.apache.org/ License: https://www.apache.org/licenses/LICENSE-2.0 -------------------------------------------------------------------------------- + +This product includes code from Apache Thrift. + +* Uses the fb303.thrift file that's part of Hive's thrift service in vendor/fb303/ + +Copyright: 2006-2022 The Apache Software Foundation. +Home page: https://thrift.apache.org/ +License: https://www.apache.org/licenses/LICENSE-2.0 + +-------------------------------------------------------------------------------- + +This product includes code from Apache Hive. + +* Uses hive_metastore.thrift to generate the Hive Metastore client in vendor/hive_metastore/ + +Copyright: 2008-2022 The Apache Software Foundation. +Home page: https://hive.apache.org/ +License: https://www.apache.org/licenses/LICENSE-2.0 + +-------------------------------------------------------------------------------- diff --git a/Makefile b/Makefile index 66ce52cd8e..aef783db33 100644 --- a/Makefile +++ b/Makefile @@ -17,7 +17,7 @@ install: pip install poetry - poetry install -E pyarrow + poetry install -E pyarrow -E hive lint: poetry run pre-commit run --all-files diff --git a/poetry.lock b/poetry.lock index 649dc074f5..b7ecd0bd04 100644 --- a/poetry.lock +++ b/poetry.lock @@ -8,17 +8,17 @@ python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*" [[package]] name = "attrs" -version = "21.4.0" +version = "22.1.0" description = "Classes Without Boilerplate" category = "dev" optional = false -python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*" +python-versions = ">=3.5" [package.extras] -dev = ["coverage[toml] (>=5.0.2)", "hypothesis", "pympler", "pytest (>=4.3.0)", "six", "mypy", "pytest-mypy-plugins", "zope.interface", "furo", "sphinx", "sphinx-notfound-page", "pre-commit", "cloudpickle"] +dev = ["coverage[toml] (>=5.0.2)", "hypothesis", "pympler", "pytest (>=4.3.0)", "mypy (>=0.900,!=0.940)", "pytest-mypy-plugins", "zope.interface", "furo", "sphinx", "sphinx-notfound-page", "pre-commit", "cloudpickle"] docs = ["furo", "sphinx", "zope.interface", "sphinx-notfound-page"] -tests = ["coverage[toml] (>=5.0.2)", "hypothesis", "pympler", "pytest (>=4.3.0)", "six", "mypy", "pytest-mypy-plugins", "zope.interface", "cloudpickle"] -tests_no_zope = ["coverage[toml] (>=5.0.2)", "hypothesis", "pympler", "pytest (>=4.3.0)", "six", "mypy", "pytest-mypy-plugins", "cloudpickle"] +tests = ["coverage[toml] (>=5.0.2)", "hypothesis", "pympler", "pytest (>=4.3.0)", "mypy (>=0.900,!=0.940)", "pytest-mypy-plugins", "zope.interface", "cloudpickle"] +tests_no_zope = ["coverage[toml] (>=5.0.2)", "hypothesis", "pympler", "pytest (>=4.3.0)", "mypy (>=0.900,!=0.940)", "pytest-mypy-plugins", "cloudpickle"] [[package]] name = "certifi" @@ -202,14 +202,14 @@ pyparsing = ">=2.0.2,<3.0.5 || >3.0.5" [[package]] name = "pep517" -version = "0.12.0" +version = "0.13.0" description = "Wrappers to build Python packages using PEP 517 hooks" category = "dev" optional = false -python-versions = "*" +python-versions = ">=3.6" [package.dependencies] -tomli = {version = ">=1.1.0", markers = "python_version >= \"3.6\""} +tomli = {version = ">=1.1.0", markers = "python_version < \"3.11\""} [[package]] name = "platformdirs" @@ -232,8 +232,8 @@ optional = false python-versions = ">=3.6" [package.extras] -testing = ["pytest-benchmark", "pytest"] -dev = ["tox", "pre-commit"] +dev = ["pre-commit", "tox"] +testing = ["pytest", "pytest-benchmark"] [[package]] name = "pre-commit" @@ -396,10 +396,26 @@ test = ["fixtures", "mock", "purl", "pytest", "sphinx", "testrepository (>=0.0.1 name = "six" version = "1.16.0" description = "Python 2 and 3 compatibility utilities" -category = "dev" +category = "main" optional = false python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*" +[[package]] +name = "thrift" +version = "0.16.0" +description = "Python bindings for the Apache Thrift RPC system" +category = "main" +optional = true +python-versions = "*" + +[package.dependencies] +six = ">=1.7.2" + +[package.extras] +all = ["tornado (>=4.0)", "twisted"] +tornado = ["tornado (>=4.0)"] +twisted = ["twisted"] + [[package]] name = "toml" version = "0.10.2" @@ -426,7 +442,7 @@ python-versions = ">=3.7" [[package]] name = "urllib3" -version = "1.26.10" +version = "1.26.11" description = "HTTP library with thread-safe connection pooling, file post, and more." category = "main" optional = false @@ -439,21 +455,20 @@ socks = ["PySocks (>=1.5.6,!=1.5.7,<2.0)"] [[package]] name = "virtualenv" -version = "20.15.1" +version = "20.16.2" description = "Virtual Python Environment builder" category = "dev" optional = false -python-versions = "!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,!=3.4.*,>=2.7" +python-versions = ">=3.6" [package.dependencies] distlib = ">=0.3.1,<1" filelock = ">=3.2,<4" platformdirs = ">=2,<3" -six = ">=1.9.0,<2" [package.extras] docs = ["proselint (>=0.10.2)", "sphinx (>=3)", "sphinx-argparse (>=0.2.5)", "sphinx-rtd-theme (>=0.4.3)", "towncrier (>=21.3)"] -testing = ["coverage (>=4)", "coverage-enable-subprocess (>=1)", "flaky (>=3)", "pytest (>=4)", "pytest-env (>=0.6.2)", "pytest-freezegun (>=0.4.1)", "pytest-mock (>=2)", "pytest-randomly (>=1)", "pytest-timeout (>=1)", "packaging (>=20.0)"] +testing = ["coverage (>=4)", "coverage-enable-subprocess (>=1)", "flaky (>=3)", "packaging (>=20.0)", "pytest (>=4)", "pytest-env (>=0.6.2)", "pytest-freezegun (>=0.4.1)", "pytest-mock (>=2)", "pytest-randomly (>=1)", "pytest-timeout (>=1)"] [[package]] name = "zipp" @@ -482,6 +497,7 @@ cffi = {version = ">=1.11", markers = "platform_python_implementation == \"PyPy\ cffi = ["cffi (>=1.11)"] [extras] +hive = ["thrift"] pyarrow = ["pyarrow"] python-snappy = ["zstandard"] snappy = ["python-snappy"] @@ -489,16 +505,11 @@ snappy = ["python-snappy"] [metadata] lock-version = "1.1" python-versions = "^3.8" -content-hash = "d8312a4f53fd87fc2dc2059d15a9aa8274512debe1c780a9e53a1108b6c12e55" +content-hash = "fa4637615911260d5d031479bd2e2481d952e1ecb6feaddfe6be6c358132465c" [metadata.files] -atomicwrites = [ - {file = "atomicwrites-1.4.1.tar.gz", hash = "sha256:81b2c9071a49367a7f770170e5eec8cb66567cfbbc8c73d20ce5ca4a8d71cf11"}, -] -attrs = [ - {file = "attrs-21.4.0-py2.py3-none-any.whl", hash = "sha256:2d27e3784d7a565d36ab851fe94887c5eccd6a463168875832a1be79c82828b4"}, - {file = "attrs-21.4.0.tar.gz", hash = "sha256:626ba8234211db98e869df76230a137c4c40a12d72445c45d5f5b716f076e2fd"}, -] +atomicwrites = [] +attrs = [] certifi = [ {file = "certifi-2022.6.15-py3-none-any.whl", hash = "sha256:fe86415d55e84719d75f8b69414f6438ac3547d2078ab91b67e779ef69378412"}, {file = "certifi-2022.6.15.tar.gz", hash = "sha256:84c85a9078b11105f04f3036a9482ae10e4621616db313fe045dd24743a0820d"}, @@ -581,84 +592,18 @@ colorama = [ {file = "colorama-0.4.5-py2.py3-none-any.whl", hash = "sha256:854bf444933e37f5824ae7bfc1e98d5bce2ebe4160d46b5edf346a89358e99da"}, {file = "colorama-0.4.5.tar.gz", hash = "sha256:e6c6b4334fc50988a639d9b98aa429a0b57da6e17b9a44f0451f930b6967b7a4"}, ] -coverage = [ - {file = "coverage-6.4.2-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:a9032f9b7d38bdf882ac9f66ebde3afb8145f0d4c24b2e600bc4c6304aafb87e"}, - {file = "coverage-6.4.2-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:e0524adb49c716ca763dbc1d27bedce36b14f33e6b8af6dba56886476b42957c"}, - {file = "coverage-6.4.2-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:d4548be38a1c810d79e097a38107b6bf2ff42151900e47d49635be69943763d8"}, - {file = "coverage-6.4.2-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:f23876b018dfa5d3e98e96f5644b109090f16a4acb22064e0f06933663005d39"}, - {file = "coverage-6.4.2-cp310-cp310-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:6fe75dcfcb889b6800f072f2af5a331342d63d0c1b3d2bf0f7b4f6c353e8c9c0"}, - {file = "coverage-6.4.2-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:2f8553878a24b00d5ab04b7a92a2af50409247ca5c4b7a2bf4eabe94ed20d3ee"}, - {file = "coverage-6.4.2-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:d774d9e97007b018a651eadc1b3970ed20237395527e22cbeb743d8e73e0563d"}, - {file = "coverage-6.4.2-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:d56f105592188ce7a797b2bd94b4a8cb2e36d5d9b0d8a1d2060ff2a71e6b9bbc"}, - {file = "coverage-6.4.2-cp310-cp310-win32.whl", hash = "sha256:d230d333b0be8042ac34808ad722eabba30036232e7a6fb3e317c49f61c93386"}, - {file = "coverage-6.4.2-cp310-cp310-win_amd64.whl", hash = "sha256:5ef42e1db047ca42827a85e34abe973971c635f83aed49611b7f3ab49d0130f0"}, - {file = "coverage-6.4.2-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:25b7ec944f114f70803d6529394b64f8749e93cbfac0fe6c5ea1b7e6c14e8a46"}, - {file = "coverage-6.4.2-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:7bb00521ab4f99fdce2d5c05a91bddc0280f0afaee0e0a00425e28e209d4af07"}, - {file = "coverage-6.4.2-cp37-cp37m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:2dff52b3e7f76ada36f82124703f4953186d9029d00d6287f17c68a75e2e6039"}, - {file = "coverage-6.4.2-cp37-cp37m-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:147605e1702d996279bb3cc3b164f408698850011210d133a2cb96a73a2f7996"}, - {file = "coverage-6.4.2-cp37-cp37m-musllinux_1_1_aarch64.whl", hash = "sha256:422fa44070b42fef9fb8dabd5af03861708cdd6deb69463adc2130b7bf81332f"}, - {file = "coverage-6.4.2-cp37-cp37m-musllinux_1_1_i686.whl", hash = "sha256:8af6c26ba8df6338e57bedbf916d76bdae6308e57fc8f14397f03b5da8622b4e"}, - {file = "coverage-6.4.2-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:5336e0352c0b12c7e72727d50ff02557005f79a0b8dcad9219c7c4940a930083"}, - {file = "coverage-6.4.2-cp37-cp37m-win32.whl", hash = "sha256:0f211df2cba951ffcae210ee00e54921ab42e2b64e0bf2c0befc977377fb09b7"}, - {file = "coverage-6.4.2-cp37-cp37m-win_amd64.whl", hash = "sha256:a13772c19619118903d65a91f1d5fea84be494d12fd406d06c849b00d31bf120"}, - {file = "coverage-6.4.2-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:f7bd0ffbcd03dc39490a1f40b2669cc414fae0c4e16b77bb26806a4d0b7d1452"}, - {file = "coverage-6.4.2-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:0895ea6e6f7f9939166cc835df8fa4599e2d9b759b02d1521b574e13b859ac32"}, - {file = "coverage-6.4.2-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:d4e7ced84a11c10160c0697a6cc0b214a5d7ab21dfec1cd46e89fbf77cc66fae"}, - {file = "coverage-6.4.2-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:80db4a47a199c4563d4a25919ff29c97c87569130375beca3483b41ad5f698e8"}, - {file = "coverage-6.4.2-cp38-cp38-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:3def6791adf580d66f025223078dc84c64696a26f174131059ce8e91452584e1"}, - {file = "coverage-6.4.2-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:4f89d8e03c8a3757aae65570d14033e8edf192ee9298303db15955cadcff0c63"}, - {file = "coverage-6.4.2-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:6d0b48aff8e9720bdec315d67723f0babd936a7211dc5df453ddf76f89c59933"}, - {file = "coverage-6.4.2-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:2b20286c2b726f94e766e86a3fddb7b7e37af5d0c635bdfa7e4399bc523563de"}, - {file = "coverage-6.4.2-cp38-cp38-win32.whl", hash = "sha256:d714af0bdba67739598849c9f18efdcc5a0412f4993914a0ec5ce0f1e864d783"}, - {file = "coverage-6.4.2-cp38-cp38-win_amd64.whl", hash = "sha256:5f65e5d3ff2d895dab76b1faca4586b970a99b5d4b24e9aafffc0ce94a6022d6"}, - {file = "coverage-6.4.2-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:a697977157adc052284a7160569b36a8bbec09db3c3220642e6323b47cec090f"}, - {file = "coverage-6.4.2-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:c77943ef768276b61c96a3eb854eba55633c7a3fddf0a79f82805f232326d33f"}, - {file = "coverage-6.4.2-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:54d8d0e073a7f238f0666d3c7c0d37469b2aa43311e4024c925ee14f5d5a1cbe"}, - {file = "coverage-6.4.2-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:f22325010d8824594820d6ce84fa830838f581a7fd86a9235f0d2ed6deb61e29"}, - {file = "coverage-6.4.2-cp39-cp39-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:24b04d305ea172ccb21bee5bacd559383cba2c6fcdef85b7701cf2de4188aa55"}, - {file = "coverage-6.4.2-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:866ebf42b4c5dbafd64455b0a1cd5aa7b4837a894809413b930026c91e18090b"}, - {file = "coverage-6.4.2-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:e36750fbbc422c1c46c9d13b937ab437138b998fe74a635ec88989afb57a3978"}, - {file = "coverage-6.4.2-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:79419370d6a637cb18553ecb25228893966bd7935a9120fa454e7076f13b627c"}, - {file = "coverage-6.4.2-cp39-cp39-win32.whl", hash = "sha256:b5e28db9199dd3833cc8a07fa6cf429a01227b5d429facb56eccd765050c26cd"}, - {file = "coverage-6.4.2-cp39-cp39-win_amd64.whl", hash = "sha256:edfdabe7aa4f97ed2b9dd5dde52d2bb29cb466993bb9d612ddd10d0085a683cf"}, - {file = "coverage-6.4.2-pp36.pp37.pp38-none-any.whl", hash = "sha256:e2618cb2cf5a7cc8d698306e42ebcacd02fb7ef8cfc18485c59394152c70be97"}, - {file = "coverage-6.4.2.tar.gz", hash = "sha256:6c3ccfe89c36f3e5b9837b9ee507472310164f352c9fe332120b764c9d60adbe"}, -] -distlib = [ - {file = "distlib-0.3.5-py2.py3-none-any.whl", hash = "sha256:b710088c59f06338ca514800ad795a132da19fda270e3ce4affc74abf955a26c"}, - {file = "distlib-0.3.5.tar.gz", hash = "sha256:a7f75737c70be3b25e2bee06288cec4e4c221de18455b2dd037fe2a795cab2fe"}, -] +coverage = [] +distlib = [] docutils = [ {file = "docutils-0.19-py3-none-any.whl", hash = "sha256:5e1de4d849fee02c63b040a4a3fd567f4ab104defd8a5511fbbc24a8a017efbc"}, {file = "docutils-0.19.tar.gz", hash = "sha256:33995a6753c30b7f577febfc2c50411fec6aac7f7ffeb7c4cfe5991072dcf9e6"}, ] -fastavro = [ - {file = "fastavro-1.5.4-cp310-cp310-macosx_10_14_x86_64.whl", hash = "sha256:d316cc476b2b24ef06402b8bfa047f8f72a9d6df2de777bb30d9ededda7e3a02"}, - {file = "fastavro-1.5.4-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:8459faec46e34f2dfeb9b70ee8c36e935e626cff8608d675724718987a5f9ce5"}, - {file = "fastavro-1.5.4-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:cd44636d7ff8365a57b88707b747371fffb676c8c1f68c0d423ec36623888668"}, - {file = "fastavro-1.5.4-cp310-cp310-win_amd64.whl", hash = "sha256:2402428b26d3c08a58acfa723833e19fb75077872bcb2475a4c81195cdae6a5d"}, - {file = "fastavro-1.5.4-cp37-cp37m-macosx_10_15_x86_64.whl", hash = "sha256:5afc14398f4191d1a807aa59d2fba5ed869b31343679ec43dbc289db0a8e35c5"}, - {file = "fastavro-1.5.4-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:5217e9713a3ea03205532394fba4d743749155b04b10b12a12fc26d225b89792"}, - {file = "fastavro-1.5.4-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:2e93a5eecb28cc35d670c9c4df70223fa9bcd6d9ca21b38b1b7ae13ece60c7fb"}, - {file = "fastavro-1.5.4-cp37-cp37m-win_amd64.whl", hash = "sha256:1a2f2465efd0e7de557c4034e8d4d88a132750cfa51e1582362a1b3a1a9fa911"}, - {file = "fastavro-1.5.4-cp38-cp38-macosx_10_15_x86_64.whl", hash = "sha256:f7d5bc76c03c692d9acea0e5d5baceec19e1c059b26cb8ae9f4481e842be95a5"}, - {file = "fastavro-1.5.4-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:80fe920229ab1f40eccb1b4918481cdd8a20e5e7dce19308ab38b23732da8a70"}, - {file = "fastavro-1.5.4-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:a3d190aee86ab73caa1aa550eba850be2ca5dd29d814b38720f4e300184e01d5"}, - {file = "fastavro-1.5.4-cp38-cp38-win_amd64.whl", hash = "sha256:b6c30299a49b11f42251cb81c8e15db67750642eac7ba5c194a5ee95c83ebb11"}, - {file = "fastavro-1.5.4-cp39-cp39-macosx_10_15_x86_64.whl", hash = "sha256:1f7685f3a3c38352abab432bad2f9f2229a0e5f5f8548831e887c30f8396f2e9"}, - {file = "fastavro-1.5.4-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:bd021ec850fd30020b7c4fa868466fb7f95450f1f06eac92bd2204cbd8e45fb8"}, - {file = "fastavro-1.5.4-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:06a7b5602dfa032c92f20ca90b8bde88251573773e501bedf5e8b76b9feb14a3"}, - {file = "fastavro-1.5.4-cp39-cp39-win_amd64.whl", hash = "sha256:18250aa2ab0f7a095b1865565cf9976ea4605c201129636e6defe24ec3ef112c"}, - {file = "fastavro-1.5.4.tar.gz", hash = "sha256:d86f72c966713fb699570a18f7960cf4110b069c70681d7538be8d671c9db7c8"}, -] +fastavro = [] filelock = [ {file = "filelock-3.7.1-py3-none-any.whl", hash = "sha256:37def7b658813cda163b56fc564cdc75e86d338246458c4c28ae84cabefa2404"}, {file = "filelock-3.7.1.tar.gz", hash = "sha256:3a0fd85166ad9dbab54c9aec96737b744106dc5f15c0b09a6744a445299fcf04"}, ] -identify = [ - {file = "identify-2.5.2-py2.py3-none-any.whl", hash = "sha256:feaa9db2dc0ce333b453ce171c0cf1247bbfde2c55fc6bb785022d411a1b78b5"}, - {file = "identify-2.5.2.tar.gz", hash = "sha256:a3d4c096b384d50d5e6dc5bc8b9bc44f1f61cefebd750a7b3e9f939b53fb214d"}, -] +identify = [] idna = [ {file = "idna-3.3-py3-none-any.whl", hash = "sha256:84d9dd047ffa80596e0f246e2eab0b391788b0503584e8945f2368256d2735ff"}, {file = "idna-3.3.tar.gz", hash = "sha256:9d643ff0a55b762d5cdb124b8eaa99c66322e2157b69160bc32796e824360e6d"}, @@ -703,38 +648,12 @@ nodeenv = [ {file = "nodeenv-1.7.0-py2.py3-none-any.whl", hash = "sha256:27083a7b96a25f2f5e1d8cb4b6317ee8aeda3bdd121394e5ac54e498028a042e"}, {file = "nodeenv-1.7.0.tar.gz", hash = "sha256:e0e7f7dfb85fc5394c6fe1e8fa98131a2473e04311a45afb6508f7cf1836fa2b"}, ] -numpy = [ - {file = "numpy-1.23.1-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:b15c3f1ed08df4980e02cc79ee058b788a3d0bef2fb3c9ca90bb8cbd5b8a3a04"}, - {file = "numpy-1.23.1-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:9ce242162015b7e88092dccd0e854548c0926b75c7924a3495e02c6067aba1f5"}, - {file = "numpy-1.23.1-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:e0d7447679ae9a7124385ccf0ea990bb85bb869cef217e2ea6c844b6a6855073"}, - {file = "numpy-1.23.1-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:3119daed207e9410eaf57dcf9591fdc68045f60483d94956bee0bfdcba790953"}, - {file = "numpy-1.23.1-cp310-cp310-win32.whl", hash = "sha256:3ab67966c8d45d55a2bdf40701536af6443763907086c0a6d1232688e27e5447"}, - {file = "numpy-1.23.1-cp310-cp310-win_amd64.whl", hash = "sha256:1865fdf51446839ca3fffaab172461f2b781163f6f395f1aed256b1ddc253622"}, - {file = "numpy-1.23.1-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:aeba539285dcf0a1ba755945865ec61240ede5432df41d6e29fab305f4384db2"}, - {file = "numpy-1.23.1-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:7e8229f3687cdadba2c4faef39204feb51ef7c1a9b669247d49a24f3e2e1617c"}, - {file = "numpy-1.23.1-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:68b69f52e6545af010b76516f5daaef6173e73353e3295c5cb9f96c35d755641"}, - {file = "numpy-1.23.1-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:1408c3527a74a0209c781ac82bde2182b0f0bf54dea6e6a363fe0cc4488a7ce7"}, - {file = "numpy-1.23.1-cp38-cp38-win32.whl", hash = "sha256:47f10ab202fe4d8495ff484b5561c65dd59177949ca07975663f4494f7269e3e"}, - {file = "numpy-1.23.1-cp38-cp38-win_amd64.whl", hash = "sha256:37e5ebebb0eb54c5b4a9b04e6f3018e16b8ef257d26c8945925ba8105008e645"}, - {file = "numpy-1.23.1-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:173f28921b15d341afadf6c3898a34f20a0569e4ad5435297ba262ee8941e77b"}, - {file = "numpy-1.23.1-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:876f60de09734fbcb4e27a97c9a286b51284df1326b1ac5f1bf0ad3678236b22"}, - {file = "numpy-1.23.1-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:35590b9c33c0f1c9732b3231bb6a72d1e4f77872390c47d50a615686ae7ed3fd"}, - {file = "numpy-1.23.1-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:a35c4e64dfca659fe4d0f1421fc0f05b8ed1ca8c46fb73d9e5a7f175f85696bb"}, - {file = "numpy-1.23.1-cp39-cp39-win32.whl", hash = "sha256:c2f91f88230042a130ceb1b496932aa717dcbd665350beb821534c5c7e15881c"}, - {file = "numpy-1.23.1-cp39-cp39-win_amd64.whl", hash = "sha256:37ece2bd095e9781a7156852e43d18044fd0d742934833335599c583618181b9"}, - {file = "numpy-1.23.1-pp38-pypy38_pp73-macosx_10_9_x86_64.whl", hash = "sha256:8002574a6b46ac3b5739a003b5233376aeac5163e5dcd43dd7ad062f3e186129"}, - {file = "numpy-1.23.1-pp38-pypy38_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:5d732d17b8a9061540a10fda5bfeabca5785700ab5469a5e9b93aca5e2d3a5fb"}, - {file = "numpy-1.23.1-pp38-pypy38_pp73-win_amd64.whl", hash = "sha256:55df0f7483b822855af67e38fb3a526e787adf189383b4934305565d71c4b148"}, - {file = "numpy-1.23.1.tar.gz", hash = "sha256:d748ef349bfef2e1194b59da37ed5a29c19ea8d7e6342019921ba2ba4fd8b624"}, -] +numpy = [] packaging = [ {file = "packaging-21.3-py3-none-any.whl", hash = "sha256:ef103e05f519cdc783ae24ea4e2e0f508a9c99b2d4969652eed6a2e1ea5bd522"}, {file = "packaging-21.3.tar.gz", hash = "sha256:dd47c42927d89ab911e606518907cc2d3a1f38bbd026385970643f9c5b8ecfeb"}, ] -pep517 = [ - {file = "pep517-0.12.0-py2.py3-none-any.whl", hash = "sha256:dd884c326898e2c6e11f9e0b64940606a93eb10ea022a2e067959f3a110cf161"}, - {file = "pep517-0.12.0.tar.gz", hash = "sha256:931378d93d11b298cf511dd634cf5ea4cb249a28ef84160b3247ee9afb4e8ab0"}, -] +pep517 = [] platformdirs = [ {file = "platformdirs-2.5.2-py3-none-any.whl", hash = "sha256:027d8e83a2d7de06bbac4e5ef7e023c02b863d7ea5d079477e722bb41ab25788"}, {file = "platformdirs-2.5.2.tar.gz", hash = "sha256:58c8abb07dcb441e6ee4b11d8df0ac856038f944ab98b7be6b27b2a3c7feef19"}, @@ -743,10 +662,7 @@ pluggy = [ {file = "pluggy-1.0.0-py2.py3-none-any.whl", hash = "sha256:74134bbf457f031a36d68416e1509f34bd5ccc019f0bcc952c7b909d06b37bd3"}, {file = "pluggy-1.0.0.tar.gz", hash = "sha256:4224373bacce55f955a878bf9cfa763c1e360858e330072059e10bad68531159"}, ] -pre-commit = [ - {file = "pre_commit-2.20.0-py2.py3-none-any.whl", hash = "sha256:51a5ba7c480ae8072ecdb6933df22d2f812dc897d5fe848778116129a681aac7"}, - {file = "pre_commit-2.20.0.tar.gz", hash = "sha256:a978dac7bc9ec0bcee55c18a277d553b0f419d259dadb4b9418ff2d00eb43959"}, -] +pre-commit = [] py = [ {file = "py-1.11.0-py2.py3-none-any.whl", hash = "sha256:607c53218732647dff4acdfcd50cb62615cedf612e72d1724fb1a0cc6405b378"}, {file = "py-1.11.0.tar.gz", hash = "sha256:51c75c4126074b472f746a24399ad32f6053d1b34b68d2fa41e558e6f4a98719"}, @@ -925,14 +841,12 @@ requests = [ {file = "requests-2.28.1-py3-none-any.whl", hash = "sha256:8fefa2a1a1365bf5520aac41836fbee479da67864514bdb821f31ce07ce65349"}, {file = "requests-2.28.1.tar.gz", hash = "sha256:7c5599b102feddaa661c826c56ab4fee28bfd17f5abca1ebbe3e7f19d7c97983"}, ] -requests-mock = [ - {file = "requests-mock-1.9.3.tar.gz", hash = "sha256:8d72abe54546c1fc9696fa1516672f1031d72a55a1d66c85184f972a24ba0eba"}, - {file = "requests_mock-1.9.3-py2.py3-none-any.whl", hash = "sha256:0a2d38a117c08bb78939ec163522976ad59a6b7fdd82b709e23bb98004a44970"}, -] +requests-mock = [] six = [ {file = "six-1.16.0-py2.py3-none-any.whl", hash = "sha256:8abb2f1d86890a2dfb989f9a77cfcfd3e47c2a354b01111771326f8aa26e0254"}, {file = "six-1.16.0.tar.gz", hash = "sha256:1e61c37477a1626458e36f7b1d82aa5c9b094fa4802892072e49de9c60c4c926"}, ] +thrift = [] toml = [ {file = "toml-0.10.2-py2.py3-none-any.whl", hash = "sha256:806143ae5bfb6a3c6e736a764057db0e6a0e05e338b5630894a5f779cabb4f9b"}, {file = "toml-0.10.2.tar.gz", hash = "sha256:b3bda1d108d5dd99f4a20d24d9c348e91c4db7ab1b749200bded2f839ccbe68f"}, @@ -945,61 +859,7 @@ typing-extensions = [ {file = "typing_extensions-4.3.0-py3-none-any.whl", hash = "sha256:25642c956049920a5aa49edcdd6ab1e06d7e5d467fc00e0506c44ac86fbfca02"}, {file = "typing_extensions-4.3.0.tar.gz", hash = "sha256:e6d2677a32f47fc7eb2795db1dd15c1f34eff616bcaf2cfb5e997f854fa1c4a6"}, ] -urllib3 = [ - {file = "urllib3-1.26.10-py2.py3-none-any.whl", hash = "sha256:8298d6d56d39be0e3bc13c1c97d133f9b45d797169a0e11cdd0e0489d786f7ec"}, - {file = "urllib3-1.26.10.tar.gz", hash = "sha256:879ba4d1e89654d9769ce13121e0f94310ea32e8d2f8cf587b77c08bbcdb30d6"}, -] -virtualenv = [ - {file = "virtualenv-20.15.1-py2.py3-none-any.whl", hash = "sha256:b30aefac647e86af6d82bfc944c556f8f1a9c90427b2fb4e3bfbf338cb82becf"}, - {file = "virtualenv-20.15.1.tar.gz", hash = "sha256:288171134a2ff3bfb1a2f54f119e77cd1b81c29fc1265a2356f3e8d14c7d58c4"}, -] -zipp = [ - {file = "zipp-3.8.1-py3-none-any.whl", hash = "sha256:47c40d7fe183a6f21403a199b3e4192cca5774656965b0a4988ad2f8feb5f009"}, - {file = "zipp-3.8.1.tar.gz", hash = "sha256:05b45f1ee8f807d0cc928485ca40a07cb491cf092ff587c0df9cb1fd154848d2"}, -] -zstandard = [ - {file = "zstandard-0.18.0-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:ef7e8a200e4c8ac9102ed3c90ed2aa379f6b880f63032200909c1be21951f556"}, - {file = "zstandard-0.18.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:2dc466207016564805e56d28375f4f533b525ff50d6776946980dff5465566ac"}, - {file = "zstandard-0.18.0-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:4a2ee1d4f98447f3e5183ecfce5626f983504a4a0c005fbe92e60fa8e5d547ec"}, - {file = "zstandard-0.18.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:d956e2f03c7200d7e61345e0880c292783ec26618d0d921dcad470cb195bbce2"}, - {file = "zstandard-0.18.0-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_12_i686.manylinux2010_i686.whl", hash = "sha256:ce6f59cba9854fd14da5bfe34217a1501143057313966637b7291d1b0267bd1e"}, - {file = "zstandard-0.18.0-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:a7fa67cba473623848b6e88acf8d799b1906178fd883fb3a1da24561c779593b"}, - {file = "zstandard-0.18.0-cp310-cp310-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:cdb44d7284c8c5dd1b66dfb86dda7f4560fa94bfbbc1d2da749ba44831335e32"}, - {file = "zstandard-0.18.0-cp310-cp310-win32.whl", hash = "sha256:63694a376cde0aa8b1971d06ca28e8f8b5f492779cb6ee1cc46bbc3f019a42a5"}, - {file = "zstandard-0.18.0-cp310-cp310-win_amd64.whl", hash = "sha256:702a8324cd90c74d9c8780d02bf55e79da3193c870c9665ad3a11647e3ad1435"}, - {file = "zstandard-0.18.0-cp36-cp36m-macosx_10_9_x86_64.whl", hash = "sha256:46f679bc5dfd938db4fb058218d9dc4db1336ffaf1ea774ff152ecadabd40805"}, - {file = "zstandard-0.18.0-cp36-cp36m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:dc2a4de9f363b3247d472362a65041fe4c0f59e01a2846b15d13046be866a885"}, - {file = "zstandard-0.18.0-cp36-cp36m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:bd3220d7627fd4d26397211cb3b560ec7cc4a94b75cfce89e847e8ce7fabe32d"}, - {file = "zstandard-0.18.0-cp36-cp36m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_12_i686.manylinux2010_i686.whl", hash = "sha256:39e98cf4773234bd9cebf9f9db730e451dfcfe435e220f8921242afda8321887"}, - {file = "zstandard-0.18.0-cp36-cp36m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:5228e596eb1554598c872a337bbe4e5afe41cd1f8b1b15f2e35b50d061e35244"}, - {file = "zstandard-0.18.0-cp36-cp36m-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:d4a8fd45746a6c31e729f35196e80b8f1e9987c59f5ccb8859d7c6a6fbeb9c63"}, - {file = "zstandard-0.18.0-cp36-cp36m-win32.whl", hash = "sha256:4cbb85f29a990c2fdbf7bc63246567061a362ddca886d7fae6f780267c0a9e67"}, - {file = "zstandard-0.18.0-cp36-cp36m-win_amd64.whl", hash = "sha256:bfa6c8549fa18e6497a738b7033c49f94a8e2e30c5fbe2d14d0b5aa8bbc1695d"}, - {file = "zstandard-0.18.0-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:e02043297c1832f2666cd2204f381bef43b10d56929e13c42c10c732c6e3b4ed"}, - {file = "zstandard-0.18.0-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:7231543d38d2b7e02ef7cc78ef7ffd86419437e1114ff08709fe25a160e24bd6"}, - {file = "zstandard-0.18.0-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:c86befac87445927488f5c8f205d11566f64c11519db223e9d282b945fa60dab"}, - {file = "zstandard-0.18.0-cp37-cp37m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_12_i686.manylinux2010_i686.whl", hash = "sha256:999a4e1768f219826ba3fa2064fab1c86dd72fdd47a42536235478c3bb3ca3e2"}, - {file = "zstandard-0.18.0-cp37-cp37m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:9df59cd1cf3c62075ee2a4da767089d19d874ac3ad42b04a71a167e91b384722"}, - {file = "zstandard-0.18.0-cp37-cp37m-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:1be31e9e3f7607ee0cdd60915410a5968b205d3e7aa83b7fcf3dd76dbbdb39e0"}, - {file = "zstandard-0.18.0-cp37-cp37m-win32.whl", hash = "sha256:490d11b705b8ae9dc845431bacc8dd1cef2408aede176620a5cd0cd411027936"}, - {file = "zstandard-0.18.0-cp37-cp37m-win_amd64.whl", hash = "sha256:266aba27fa9cc5e9091d3d325ebab1fa260f64e83e42516d5e73947c70216a5b"}, - {file = "zstandard-0.18.0-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:8b2260c4e07dd0723eadb586de7718b61acca4083a490dda69c5719d79bc715c"}, - {file = "zstandard-0.18.0-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:3af8c2383d02feb6650e9255491ec7d0824f6e6dd2bbe3e521c469c985f31fb1"}, - {file = "zstandard-0.18.0-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:28723a1d2e4df778573b76b321ebe9f3469ac98988104c2af116dd344802c3f8"}, - {file = "zstandard-0.18.0-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:19cac7108ff2c342317fad6dc97604b47a41f403c8f19d0bfc396dfadc3638b8"}, - {file = "zstandard-0.18.0-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_12_i686.manylinux2010_i686.whl", hash = "sha256:76725d1ee83a8915100a310bbad5d9c1fc6397410259c94033b8318d548d9990"}, - {file = "zstandard-0.18.0-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:d716a7694ce1fa60b20bc10f35c4a22be446ef7f514c8dbc8f858b61976de2fb"}, - {file = "zstandard-0.18.0-cp38-cp38-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:49685bf9a55d1ab34bd8423ea22db836ba43a181ac6b045ac4272093d5cb874e"}, - {file = "zstandard-0.18.0-cp38-cp38-win32.whl", hash = "sha256:1af1268a7dc870eb27515fb8db1f3e6c5a555d2b7bcc476fc3bab8886c7265ab"}, - {file = "zstandard-0.18.0-cp38-cp38-win_amd64.whl", hash = "sha256:1dc2d3809e763055a1a6c1a73f2b677320cc9a5aa1a7c6cfb35aee59bddc42d9"}, - {file = "zstandard-0.18.0-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:eea18c1e7442f2aa9aff1bb84550dbb6a1f711faf6e48e7319de8f2b2e923c2a"}, - {file = "zstandard-0.18.0-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:8677ffc6a6096cccbd892e558471c901fd821aba12b7fbc63833c7346f549224"}, - {file = "zstandard-0.18.0-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:083dc08abf03807af9beeb2b6a91c23ad78add2499f828176a3c7b742c44df02"}, - {file = "zstandard-0.18.0-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:c990063664c08169c84474acecc9251ee035871589025cac47c060ff4ec4bc1a"}, - {file = "zstandard-0.18.0-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_12_i686.manylinux2010_i686.whl", hash = "sha256:533db8a6fac6248b2cb2c935e7b92f994efbdeb72e1ffa0b354432e087bb5a3e"}, - {file = "zstandard-0.18.0-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:dbb3cb8a082d62b8a73af42291569d266b05605e017a3d8a06a0e5c30b5f10f0"}, - {file = "zstandard-0.18.0-cp39-cp39-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:d6c85ca5162049ede475b7ec98e87f9390501d44a3d6776ddd504e872464ec25"}, - {file = "zstandard-0.18.0-cp39-cp39-win32.whl", hash = "sha256:75479e7c2b3eebf402c59fbe57d21bc400cefa145ca356ee053b0a08908c5784"}, - {file = "zstandard-0.18.0-cp39-cp39-win_amd64.whl", hash = "sha256:d85bfabad444812133a92fc6fbe463e1d07581dba72f041f07a360e63808b23c"}, - {file = "zstandard-0.18.0.tar.gz", hash = "sha256:0ac0357a0d985b4ff31a854744040d7b5754385d1f98f7145c30e02c6865cb6f"}, -] +urllib3 = [] +virtualenv = [] +zipp = [] +zstandard = [] diff --git a/pyiceberg/catalog/base.py b/pyiceberg/catalog/base.py index a0e2e6296c..49eca489bb 100644 --- a/pyiceberg/catalog/base.py +++ b/pyiceberg/catalog/base.py @@ -91,7 +91,7 @@ def create_table( def load_table(self, identifier: str | Identifier) -> Table: """Loads the table's metadata and returns the table instance. - You can also use this method to check for table existence using 'try catalog.table() except TableNotFoundError' + You can also use this method to check for table existence using 'try catalog.table() except NoSuchTableError' Note: This method doesn't scan data stored in the table. Args: @@ -166,7 +166,7 @@ def drop_namespace(self, namespace: str | Identifier) -> None: """ @abstractmethod - def list_tables(self, namespace: str | Identifier | None = None) -> list[Identifier]: + def list_tables(self, namespace: str | Identifier) -> list[Identifier]: """List tables under the given namespace in the catalog. If namespace not provided, will list all tables in the catalog. diff --git a/pyiceberg/catalog/hive.py b/pyiceberg/catalog/hive.py new file mode 100644 index 0000000000..f04fc22338 --- /dev/null +++ b/pyiceberg/catalog/hive.py @@ -0,0 +1,459 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +import getpass +import time +from typing import ( + Any, + Dict, + List, + Optional, + Set, + Tuple, + Type, + Union, +) +from urllib.parse import urlparse + +from hive_metastore.ThriftHiveMetastore import Client +from hive_metastore.ttypes import AlreadyExistsException +from hive_metastore.ttypes import Database as HiveDatabase +from hive_metastore.ttypes import ( + FieldSchema, + InvalidOperationException, + MetaException, + NoSuchObjectException, + SerDeInfo, + StorageDescriptor, +) +from hive_metastore.ttypes import Table as HiveTable +from thrift.protocol import TBinaryProtocol +from thrift.transport import TSocket, TTransport + +from pyiceberg.catalog import Identifier, Properties +from pyiceberg.catalog.base import Catalog, PropertiesUpdateSummary +from pyiceberg.exceptions import ( + NamespaceAlreadyExistsError, + NamespaceNotEmptyError, + NoSuchNamespaceError, + NoSuchTableError, + TableAlreadyExistsError, +) +from pyiceberg.schema import Schema +from pyiceberg.table.base import Table +from pyiceberg.table.partitioning import PartitionSpec +from pyiceberg.table.sorting import UNSORTED_SORT_ORDER, SortOrder +from pyiceberg.types import ( + BinaryType, + BooleanType, + DateType, + DecimalType, + DoubleType, + FixedType, + FloatType, + IcebergType, + IntegerType, + ListType, + LongType, + MapType, + StringType, + StructType, + TimestampType, + TimeType, + UUIDType, +) + +# Replace by visitor +hive_types = { + BooleanType: "boolean", + IntegerType: "int", + LongType: "bigint", + FloatType: "float", + DoubleType: "double", + DateType: "date", + TimeType: "string", + TimestampType: "timestamp", + StringType: "string", + UUIDType: "string", + BinaryType: "binary", + FixedType: "binary", + DecimalType: None, + StructType: None, + ListType: None, + MapType: None, +} + +OWNER = "owner" + + +class _HiveClient: + """Helper class to nicely open and close the transport""" + + _transport: TTransport + _client: Client + + def __init__(self, uri: str): + url_parts = urlparse(uri) + transport = TSocket.TSocket(url_parts.hostname, url_parts.port) + self._transport = TTransport.TBufferedTransport(transport) + protocol = TBinaryProtocol.TBinaryProtocol(transport) + + self._client = Client(protocol) + + def __enter__(self) -> Client: + self._transport.open() + return self._client + + def __exit__(self, exc_type, exc_val, exc_tb): + self._transport.close() + + +def _construct_hive_storage_descriptor(schema: Schema, location: Optional[str]) -> StorageDescriptor: + ser_de_info = SerDeInfo(serializationLib="org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe") + return StorageDescriptor( + _convert_schema_to_columns(schema), + location, + "org.apache.hadoop.mapred.FileInputFormat", + "org.apache.hadoop.mapred.FileOutputFormat", + serdeInfo=ser_de_info, + ) + + +def _convert_schema_to_columns(schema: Schema) -> List[FieldSchema]: + return [FieldSchema(field.name, _iceberg_type_to_hive_types(field.field_type), field.doc) for field in schema.fields] + + +def _iceberg_type_to_hive_types(col_type: IcebergType) -> str: + if hive_type := hive_types.get(type(col_type)): + return hive_type + raise NotImplementedError(f"Not yet implemented column type {col_type}") + + +PROP_EXTERNAL = "EXTERNAL" +PROP_TABLE_TYPE = "table_type" +PROP_METADATA_LOCATION = "metadata_location" +PROP_PREVIOUS_METADATA_LOCATION = "previous_metadata_location" + + +def _construct_parameters(metadata_location: str, previous_metadata_location: Optional[str] = None) -> Dict[str, Any]: + properties = {PROP_EXTERNAL: "TRUE", PROP_TABLE_TYPE: "ICEBERG", PROP_METADATA_LOCATION: metadata_location} + if previous_metadata_location: + properties[previous_metadata_location] = previous_metadata_location + + return properties + + +def _annotate_namespace(database: HiveDatabase, properties: Properties) -> HiveDatabase: + params = {} + for key, value in properties.items(): + if key == "comment": + database.description = value + elif key == "location": + database.locationUri = value + else: + params[key] = value + database.parameters = params + return database + + +class HiveCatalog(Catalog): + _client: _HiveClient + + @staticmethod + def identifier_to_database( + identifier: Union[str, Identifier], err: Union[Type[ValueError], Type[NoSuchNamespaceError]] = ValueError + ) -> str: + tuple_identifier = Catalog.identifier_to_tuple(identifier) + if len(tuple_identifier) != 1: + raise err(f"Invalid database, hierarchical namespaces are not supported: {identifier}") + + return tuple_identifier[0] + + @staticmethod + def identifier_to_database_and_table( + identifier: Union[str, Identifier], + err: Union[Type[ValueError], Type[NoSuchTableError], Type[NoSuchNamespaceError]] = ValueError, + ) -> Tuple[str, str]: + tuple_identifier = Catalog.identifier_to_tuple(identifier) + if len(tuple_identifier) != 2: + raise err(f"Invalid path, hierarchical namespaces are not supported: {identifier}") + + return tuple_identifier[0], tuple_identifier[1] + + def __init__(self, name: str, properties: Properties, uri: str): + super().__init__(name, properties) + self._client = _HiveClient(uri) + + def _convert_hive_into_iceberg(self, table: HiveTable) -> Table: + # Requires reading the manifest, will implement this in another PR + # Check the table type + return Table(identifier=(table.dbName, table.tableName)) + + def create_table( + self, + identifier: Union[str, Identifier], + schema: Schema, + location: Optional[str] = None, + partition_spec: Optional[PartitionSpec] = None, + sort_order: SortOrder = UNSORTED_SORT_ORDER, + properties: Optional[Properties] = None, + ) -> Table: + """Create a table + + Args: + identifier: Table identifier. + schema: Table's schema. + location: Location for the table. Optional Argument. + partition_spec: PartitionSpec for the table. + sort_order: SortOrder for the table. + properties: Table properties that can be a string based dictionary. Optional Argument. + + Returns: + Table: the created table instance + + Raises: + AlreadyExistsError: If a table with the name already exists + ValueError: If the identifier is invalid + """ + database_name, table_name = self.identifier_to_database_and_table(identifier) + current_time_millis = int(time.time()) + tbl = HiveTable( + dbName=database_name, + tableName=table_name, + owner=properties[OWNER] if properties and OWNER in properties else getpass.getuser(), + createTime=current_time_millis // 1000, + lastAccessTime=current_time_millis // 1000, + sd=_construct_hive_storage_descriptor(schema, location), + tableType="EXTERNAL_TABLE", + parameters=_construct_parameters("s3://"), + ) + try: + with self._client as open_client: + open_client.create_table(tbl) + hive_table = open_client.get_table(dbname=database_name, tbl_name=table_name) + except AlreadyExistsException as e: + raise TableAlreadyExistsError(f"Table {database_name}.{table_name} already exists") from e + return self._convert_hive_into_iceberg(hive_table) + + def load_table(self, identifier: Union[str, Identifier]) -> Table: + """Loads the table's metadata and returns the table instance. + + You can also use this method to check for table existence using 'try catalog.table() except TableNotFoundError' + Note: This method doesn't scan data stored in the table. + + Args: + identifier: Table identifier. + + Returns: + Table: the table instance with its metadata + + Raises: + NoSuchTableError: If a table with the name does not exist, or the identifier is invalid + """ + database_name, table_name = self.identifier_to_database_and_table(identifier, NoSuchTableError) + try: + with self._client as open_client: + hive_table = open_client.get_table(dbname=database_name, tbl_name=table_name) + except NoSuchObjectException as e: + raise NoSuchTableError(f"Table does not exists: {table_name}") from e + + return self._convert_hive_into_iceberg(hive_table) + + def drop_table(self, identifier: Union[str, Identifier]) -> None: + """Drop a table. + + Args: + identifier: Table identifier. + + Raises: + NoSuchTableError: If a table with the name does not exist, or the identifier is invalid + """ + database_name, table_name = self.identifier_to_database_and_table(identifier, NoSuchTableError) + try: + with self._client as open_client: + open_client.drop_table(dbname=database_name, name=table_name, deleteData=False) + except NoSuchObjectException as e: + # When the namespace doesn't exists, it throws the same error + raise NoSuchTableError(f"Table does not exists: {table_name}") from e + + def purge_table(self, identifier: Union[str, Identifier]) -> None: + # This requires to traverse the reachability set, and drop all the data files. + raise NotImplementedError("Not yet implemented") + + def rename_table(self, from_identifier: Union[str, Identifier], to_identifier: Union[str, Identifier]) -> Table: + """Rename a fully classified table name + + Args: + from_identifier: Existing table identifier. + to_identifier: New table identifier. + + Returns: + Table: the updated table instance with its metadata + + Raises: + ValueError: When the from table identifier is invalid + NoSuchTableError: When a table with the name does not exist + NoSuchNamespaceError: When the destination namespace doesn't exists + """ + from_database_name, from_table_name = self.identifier_to_database_and_table(from_identifier, NoSuchTableError) + to_database_name, to_table_name = self.identifier_to_database_and_table(to_identifier) + try: + with self._client as open_client: + tbl = open_client.get_table(dbname=from_database_name, tbl_name=from_table_name) + tbl.dbName = to_database_name + tbl.tableName = to_table_name + open_client.alter_table(dbname=from_database_name, tbl_name=from_table_name, new_tbl=tbl) + except NoSuchObjectException as e: + raise NoSuchTableError(f"Table does not exist: {from_table_name}") from e + except InvalidOperationException as e: + raise NoSuchNamespaceError(f"Database does not exists: {to_database_name}") from e + return Table() + + def create_namespace(self, namespace: Union[str, Identifier], properties: Optional[Properties] = None) -> None: + """Create a namespace in the catalog. + + Args: + namespace: Namespace identifier + properties: A string dictionary of properties for the given namespace + + Raises: + ValueError: If the identifier is invalid + AlreadyExistsError: If a namespace with the given name already exists + """ + database_name = self.identifier_to_database(namespace) + hive_database = HiveDatabase(name=database_name, parameters=properties) + + try: + with self._client as open_client: + open_client.create_database(_annotate_namespace(hive_database, properties or {})) + except AlreadyExistsException as e: + raise NamespaceAlreadyExistsError(f"Database {database_name} already exists") from e + + def drop_namespace(self, namespace: Union[str, Identifier]) -> None: + """Drop a namespace. + + Args: + namespace: Namespace identifier + + Raises: + NoSuchNamespaceError: If a namespace with the given name does not exist, or the identifier is invalid + NamespaceNotEmptyError: If the namespace is not empty + """ + database_name = self.identifier_to_database(namespace, NoSuchNamespaceError) + try: + with self._client as open_client: + open_client.drop_database(database_name, deleteData=False, cascade=False) + except InvalidOperationException as e: + raise NamespaceNotEmptyError(f"Database {database_name} is not empty") from e + except MetaException as e: + raise NoSuchNamespaceError(f"Database does not exists: {database_name}") from e + + def list_tables(self, namespace: Union[str, Identifier]) -> List[Identifier]: + """List tables under the given namespace in the catalog (including non-Iceberg tables) + + When the database doesn't exist, it will just return an empty list + + Args: + namespace: Database to list. + + Returns: + List[Identifier]: list of table identifiers. + + Raises: + NoSuchNamespaceError: If a namespace with the given name does not exist, or the identifier is invalid + """ + database_name = self.identifier_to_database(namespace, NoSuchNamespaceError) + with self._client as open_client: + return [(database_name, table_name) for table_name in open_client.get_all_tables(db_name=database_name)] + + def list_namespaces(self) -> List[Identifier]: + """List namespaces from the given namespace. If not given, list top-level namespaces from the catalog. + + Returns: + List[Identifier]: a List of namespace identifiers + """ + with self._client as open_client: + return list(map(self.identifier_to_tuple, open_client.get_all_databases())) + + def load_namespace_properties(self, namespace: Union[str, Identifier]) -> Properties: + """Get properties for a namespace. + + Args: + namespace: Namespace identifier + + Returns: + Properties: Properties for the given namespace + + Raises: + NoSuchNamespaceError: If a namespace with the given name does not exist, or identifier is invalid + """ + database_name = self.identifier_to_database(namespace, NoSuchNamespaceError) + try: + with self._client as open_client: + database = open_client.get_database(name=database_name) + properties = database.parameters + properties["location"] = database.locationUri + if comment := database.description: + properties["comment"] = comment + return properties + except NoSuchObjectException as e: + raise NoSuchNamespaceError(f"Database does not exists: {database_name}") from e + + def update_namespace_properties( + self, namespace: Union[str, Identifier], removals: Optional[Set[str]] = None, updates: Optional[Properties] = None + ) -> PropertiesUpdateSummary: + """Removes provided property keys and updates properties for a namespace. + + Args: + namespace: Namespace identifier + removals: Set of property keys that need to be removed. Optional Argument. + updates: Properties to be updated for the given namespace. Optional Argument. + + Raises: + NoSuchNamespaceError: If a namespace with the given name does not exist + ValueError: If removals and updates have overlapping keys. + """ + removed: Set[str] = set() + updated: Set[str] = set() + + if updates and removals: + overlap = set(removals) & set(updates.keys()) + if overlap: + raise ValueError(f"Updates and deletes have an overlap: {overlap}") + + database_name = self.identifier_to_database(namespace, NoSuchNamespaceError) + with self._client as open_client: + try: + database = open_client.get_database(database_name) + parameters = database.parameters + except NoSuchObjectException as e: + raise NoSuchNamespaceError(f"Database does not exists: {database_name}") from e + if removals: + for key in removals: + if key in parameters: + parameters[key] = None + removed.add(key) + if updates: + for key, value in updates.items(): + parameters[key] = value + updated.add(key) + open_client.alter_database(database_name, _annotate_namespace(database, parameters)) + + expected_to_change = (removals or set()).difference(removed) + + return PropertiesUpdateSummary( + removed=list(removed or []), updated=list(updates.keys() if updates else []), missing=list(expected_to_change) + ) diff --git a/pyiceberg/catalog/rest.py b/pyiceberg/catalog/rest.py index b5938267ba..bcedbc5cd4 100644 --- a/pyiceberg/catalog/rest.py +++ b/pyiceberg/catalog/rest.py @@ -327,8 +327,8 @@ def create_table( metadata=table_response.metadata, ) - def list_tables(self, namespace: Optional[Union[str, Identifier]] = None) -> List[Identifier]: - namespace_concat = NAMESPACE_SEPARATOR.join(self.identifier_to_tuple(namespace or "")) + def list_tables(self, namespace: Union[str, Identifier]) -> List[Identifier]: + namespace_concat = NAMESPACE_SEPARATOR.join(self.identifier_to_tuple(namespace)) response = requests.get( self.url(Endpoints.list_tables, namespace=namespace_concat), headers=self.headers, diff --git a/pyiceberg/table/base.py b/pyiceberg/table/base.py index 5e7c06a5b2..5ade9d7251 100644 --- a/pyiceberg/table/base.py +++ b/pyiceberg/table/base.py @@ -27,4 +27,4 @@ class Table(IcebergBaseModel): identifier: Identifier = Field() metadata_location: Optional[str] = Field() - metadata: Union[TableMetadataV1, TableMetadataV2] = Field() + metadata: Optional[Union[TableMetadataV1, TableMetadataV2]] = Field() diff --git a/pyproject.toml b/pyproject.toml index b5f9b1e1d2..438de24049 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -35,8 +35,11 @@ classifiers = [ packages = [ { include = "pyiceberg" }, + { from = "vendor", include = "fb303" }, + { from = "vendor", include = "hive_metastore" }, ] + [tool.poetry.dependencies] python = "^3.8" mmh3 = "^3.0.0" @@ -50,6 +53,8 @@ zstandard = { version = "^0.18.0", optional = true } python-snappy = { version = "^0.6.1", optional = true } +thrift = { version = "^0.16.0", optional = true } + [tool.poetry.dev-dependencies] pytest = "^7.0.0" pytest-checkdocs = "^2.0.0" @@ -66,6 +71,7 @@ build-backend = "poetry.core.masonry.api" pyarrow = ["pyarrow"] snappy = ["python-snappy"] python-snappy = ["zstandard"] +hive = ["thrift"] [tool.black] line-length = 130 @@ -114,6 +120,14 @@ ignore_missing_imports = true module = "mmh3.*" ignore_missing_imports = true +[[tool.mypy.overrides]] +module = "hive_metastore.*" +ignore_missing_imports = true + +[[tool.mypy.overrides]] +module = "thrift.*" +ignore_missing_imports = true + [[tool.mypy.overrides]] module = "requests_mock.*" ignore_missing_imports = true diff --git a/tests/catalog/test_hive.py b/tests/catalog/test_hive.py new file mode 100644 index 0000000000..77e0de02e7 --- /dev/null +++ b/tests/catalog/test_hive.py @@ -0,0 +1,446 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# pylint: disable=protected-access,redefined-outer-name +from unittest.mock import MagicMock, patch + +import pytest +from hive_metastore.ttypes import AlreadyExistsException +from hive_metastore.ttypes import Database as HiveDatabase +from hive_metastore.ttypes import ( + FieldSchema, + InvalidOperationException, + MetaException, + NoSuchObjectException, + SerDeInfo, + SkewedInfo, + StorageDescriptor, +) +from hive_metastore.ttypes import Table as HiveTable + +from pyiceberg.catalog.base import PropertiesUpdateSummary +from pyiceberg.catalog.hive import HiveCatalog +from pyiceberg.exceptions import ( + NamespaceAlreadyExistsError, + NamespaceNotEmptyError, + NoSuchNamespaceError, + NoSuchTableError, +) +from pyiceberg.schema import Schema + +HIVE_CATALOG_NAME = "hive" +HIVE_METASTORE_FAKE_URL = "thrift://unknown:9083" + + +@pytest.fixture +def hive_table() -> HiveTable: + return HiveTable( + tableName="new_tabl2e", + dbName="default", + owner="fokkodriesprong", + createTime=1659092339, + lastAccessTime=1659092, + retention=0, + sd=StorageDescriptor( + cols=[ + FieldSchema(name="foo", type="string", comment=None), + FieldSchema(name="bar", type="int", comment=None), + FieldSchema(name="baz", type="boolean", comment=None), + ], + location="file:/tmp/new_tabl2e", + inputFormat="org.apache.hadoop.mapred.FileInputFormat", + outputFormat="org.apache.hadoop.mapred.FileOutputFormat", + compressed=False, + numBuckets=0, + serdeInfo=SerDeInfo( + name=None, + serializationLib="org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe", + parameters={}, + description=None, + serializerClass=None, + deserializerClass=None, + serdeType=None, + ), + bucketCols=[], + sortCols=[], + parameters={}, + skewedInfo=SkewedInfo(skewedColNames=[], skewedColValues=[], skewedColValueLocationMaps={}), + storedAsSubDirectories=False, + ), + partitionKeys=[], + parameters={"EXTERNAL": "TRUE", "transient_lastDdlTime": "1659092339"}, + viewOriginalText=None, + viewExpandedText=None, + tableType="EXTERNAL_TABLE", + privileges=None, + temporary=False, + rewriteEnabled=False, + creationMetadata=None, + catName="hive", + ownerType=1, + writeId=-1, + isStatsCompliant=None, + colStats=None, + accessType=None, + requiredReadCapabilities=None, + requiredWriteCapabilities=None, + id=None, + fileMetadata=None, + dictionary=None, + txnId=None, + ) + + +@pytest.fixture +def hive_database() -> HiveDatabase: + return HiveDatabase( + name="default", + description=None, + locationUri="file:/tmp/default2.db", + parameters={"test": "property"}, + privileges=None, + ownerName=None, + ownerType=1, + catalogName="hive", + createTime=None, + managedLocationUri=None, + type=None, + connector_name=None, + remote_dbname=None, + ) + + +def test_check_number_of_namespaces(table_schema_simple: Schema): + catalog = HiveCatalog(HIVE_CATALOG_NAME, {}, uri=HIVE_METASTORE_FAKE_URL) + + with pytest.raises(ValueError): + catalog.create_table(("default", "namespace", "table"), schema=table_schema_simple) + + with pytest.raises(ValueError): + catalog.create_table("default.namespace.table", schema=table_schema_simple) + + with pytest.raises(ValueError): + catalog.create_table(("table",), schema=table_schema_simple) + + with pytest.raises(ValueError): + catalog.create_table("table", schema=table_schema_simple) + + +@patch("time.time", MagicMock(return_value=12345000)) +def test_create_table(table_schema_simple: Schema, hive_table: HiveTable): + catalog = HiveCatalog(HIVE_CATALOG_NAME, {}, uri=HIVE_METASTORE_FAKE_URL) + + catalog._client = MagicMock() + catalog._client.__enter__().create_table.return_value = hive_table + catalog._client.__enter__().get_table.return_value = hive_table + + catalog.create_table(("default", "table"), schema=table_schema_simple, properties={"owner": "javaberg"}) + + catalog._client.__enter__().create_table.assert_called_with( + HiveTable( + tableName="table", + dbName="default", + owner="javaberg", + createTime=12345, + lastAccessTime=12345, + retention=None, + sd=StorageDescriptor( + cols=[ + FieldSchema(name="foo", type="string", comment=None), + FieldSchema(name="bar", type="int", comment=None), + FieldSchema(name="baz", type="boolean", comment=None), + ], + location=None, + inputFormat="org.apache.hadoop.mapred.FileInputFormat", + outputFormat="org.apache.hadoop.mapred.FileOutputFormat", + compressed=None, + numBuckets=None, + serdeInfo=SerDeInfo( + name=None, + serializationLib="org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe", + parameters=None, + description=None, + serializerClass=None, + deserializerClass=None, + serdeType=None, + ), + bucketCols=None, + sortCols=None, + parameters=None, + skewedInfo=None, + storedAsSubDirectories=None, + ), + partitionKeys=None, + parameters={"EXTERNAL": "TRUE", "table_type": "ICEBERG", "metadata_location": "s3://"}, + viewOriginalText=None, + viewExpandedText=None, + tableType="EXTERNAL_TABLE", + privileges=None, + temporary=False, + rewriteEnabled=None, + creationMetadata=None, + catName=None, + ownerType=1, + writeId=-1, + isStatsCompliant=None, + colStats=None, + accessType=None, + requiredReadCapabilities=None, + requiredWriteCapabilities=None, + id=None, + fileMetadata=None, + dictionary=None, + txnId=None, + ) + ) + + +def test_load_table(hive_table: HiveTable): + catalog = HiveCatalog(HIVE_CATALOG_NAME, {}, uri=HIVE_METASTORE_FAKE_URL) + + catalog._client = MagicMock() + catalog._client.__enter__().get_table.return_value = hive_table + catalog.load_table(("default", "table")) + + catalog._client.__enter__().get_table.assert_called_with(dbname="default", tbl_name="table") + + +def test_rename_table_from_does_not_exists(): + catalog = HiveCatalog(HIVE_CATALOG_NAME, {}, uri=HIVE_METASTORE_FAKE_URL) + + catalog._client = MagicMock() + catalog._client.__enter__().alter_table.side_effect = NoSuchObjectException( + message="hive.default.does_not_exists table not found" + ) + + with pytest.raises(NoSuchTableError) as exc_info: + catalog.rename_table(("default", "does_not_exists"), ("default", "new_table")) + + assert "Table does not exist: does_not_exists" in str(exc_info.value) + + +def test_rename_table_to_namespace_does_not_exists(): + catalog = HiveCatalog(HIVE_CATALOG_NAME, {}, uri=HIVE_METASTORE_FAKE_URL) + + catalog._client = MagicMock() + catalog._client.__enter__().alter_table.side_effect = InvalidOperationException( + message="Unable to change partition or table. Database default does not exist Check metastore logs for detailed stack.does_not_exists" + ) + + with pytest.raises(NoSuchNamespaceError) as exc_info: + catalog.rename_table(("default", "does_exists"), ("default_does_not_exists", "new_table")) + + assert "Database does not exists: default_does_not_exists" in str(exc_info.value) + + +def test_drop_database_does_not_empty(): + catalog = HiveCatalog(HIVE_CATALOG_NAME, {}, uri=HIVE_METASTORE_FAKE_URL) + + catalog._client = MagicMock() + catalog._client.__enter__().drop_database.side_effect = InvalidOperationException( + message="Database not_empty is not empty. One or more tables exist." + ) + + with pytest.raises(NamespaceNotEmptyError) as exc_info: + catalog.drop_namespace(("not_empty",)) + + assert "Database not_empty is not empty" in str(exc_info.value) + + +def test_drop_database_does_not_exists(): + catalog = HiveCatalog(HIVE_CATALOG_NAME, {}, uri=HIVE_METASTORE_FAKE_URL) + + catalog._client = MagicMock() + catalog._client.__enter__().drop_database.side_effect = MetaException(message="java.lang.NullPointerException") + + with pytest.raises(NoSuchNamespaceError) as exc_info: + catalog.drop_namespace(("does_not_exists",)) + + assert "Database does not exists: does_not_exists" in str(exc_info.value) + + +def test_list_tables(): + catalog = HiveCatalog(HIVE_CATALOG_NAME, {}, uri=HIVE_METASTORE_FAKE_URL) + + catalog._client = MagicMock() + catalog._client.__enter__().get_all_tables.return_value = ["table1", "table2"] + + assert catalog.list_tables("database") == [ + ( + "database", + "table1", + ), + ( + "database", + "table2", + ), + ] + catalog._client.__enter__().get_all_tables.assert_called_with(db_name="database") + + +def test_list_namespaces(): + catalog = HiveCatalog(HIVE_CATALOG_NAME, {}, uri=HIVE_METASTORE_FAKE_URL) + + catalog._client = MagicMock() + catalog._client.__enter__().get_all_databases.return_value = ["namespace1", "namespace2"] + + assert catalog.list_namespaces() == [("namespace1",), ("namespace2",)] + + catalog._client.__enter__().get_all_databases.assert_called() + + +def test_drop_table(): + catalog = HiveCatalog(HIVE_CATALOG_NAME, {}, uri=HIVE_METASTORE_FAKE_URL) + + catalog._client = MagicMock() + catalog._client.__enter__().get_all_databases.return_value = ["namespace1", "namespace2"] + + catalog.drop_table(("default", "table")) + + catalog._client.__enter__().drop_table.assert_called_with(dbname="default", name="table", deleteData=False) + + +def test_drop_table_does_not_exists(): + catalog = HiveCatalog(HIVE_CATALOG_NAME, {}, uri=HIVE_METASTORE_FAKE_URL) + + catalog._client = MagicMock() + catalog._client.__enter__().drop_table.side_effect = NoSuchObjectException(message="does_not_exists") + + with pytest.raises(NoSuchTableError) as exc_info: + catalog.drop_table(("default", "does_not_exists")) + + assert "Table does not exists: does_not_exists" in str(exc_info.value) + + +def test_purge_table(): + catalog = HiveCatalog(HIVE_CATALOG_NAME, {}, uri=HIVE_METASTORE_FAKE_URL) + + with pytest.raises(NotImplementedError): + catalog.purge_table(("default", "does_not_exists")) + + +def test_create_database(): + catalog = HiveCatalog(HIVE_CATALOG_NAME, {}, uri=HIVE_METASTORE_FAKE_URL) + + catalog._client = MagicMock() + catalog._client.__enter__().create_database.return_value = None + + catalog.create_namespace("default", {"property": "true"}) + + catalog._client.__enter__().create_database.assert_called_with( + HiveDatabase( + name="default", + description=None, + locationUri=None, + parameters={"property": "true"}, + privileges=None, + ownerName=None, + ownerType=None, + catalogName=None, + createTime=None, + managedLocationUri=None, + type=None, + connector_name=None, + remote_dbname=None, + ) + ) + + +def test_create_database_already_exists(): + catalog = HiveCatalog(HIVE_CATALOG_NAME, {}, uri=HIVE_METASTORE_FAKE_URL) + + catalog._client = MagicMock() + catalog._client.__enter__().create_database.side_effect = AlreadyExistsException(message="Database default already exists") + + with pytest.raises(NamespaceAlreadyExistsError) as exc_info: + catalog.create_namespace("default") + + assert "Database default already exists" in str(exc_info.value) + + +def test_load_namespace_properties(hive_database: HiveDatabase): + catalog = HiveCatalog(HIVE_CATALOG_NAME, {}, uri=HIVE_METASTORE_FAKE_URL) + + catalog._client = MagicMock() + catalog._client.__enter__().get_database.return_value = hive_database + + assert catalog.load_namespace_properties("default2") == {"location": "file:/tmp/default2.db", "test": "property"} + + catalog._client.__enter__().get_database.assert_called_with(name="default2") + + +def test_load_namespace_properties_does_not_exists(): + catalog = HiveCatalog(HIVE_CATALOG_NAME, {}, uri=HIVE_METASTORE_FAKE_URL) + + catalog._client = MagicMock() + catalog._client.__enter__().get_database.side_effect = NoSuchObjectException(message="does_not_exists") + + with pytest.raises(NoSuchNamespaceError) as exc_info: + catalog.load_namespace_properties(("does_not_exists",)) + + assert "Database does not exists: does_not_exists" in str(exc_info.value) + + +def test_update_namespace_properties(hive_database: HiveDatabase): + catalog = HiveCatalog(HIVE_CATALOG_NAME, {}, uri=HIVE_METASTORE_FAKE_URL) + + catalog._client = MagicMock() + catalog._client.__enter__().get_database.return_value = hive_database + catalog._client.__enter__().alter_database.return_value = None + + assert catalog.update_namespace_properties( + namespace="default", removals={"test", "does_not_exists"}, updates={"label": "core"} + ) == PropertiesUpdateSummary(removed=["test"], updated=["label"], missing=["does_not_exists"]) + + catalog._client.__enter__().alter_database.assert_called_with( + "default", + HiveDatabase( + name="default", + description=None, + locationUri="file:/tmp/default2.db", + parameters={"test": None, "label": "core"}, + privileges=None, + ownerName=None, + ownerType=1, + catalogName="hive", + createTime=None, + managedLocationUri=None, + type=None, + connector_name=None, + remote_dbname=None, + ), + ) + + +def test_update_namespace_properties_namespace_does_not_exists(): + catalog = HiveCatalog(HIVE_CATALOG_NAME, {}, uri=HIVE_METASTORE_FAKE_URL) + + catalog._client = MagicMock() + catalog._client.__enter__().get_database.side_effect = NoSuchObjectException(message="does_not_exists") + + with pytest.raises(NoSuchNamespaceError) as exc_info: + catalog.update_namespace_properties(("does_not_exists",), removals=set(), updates={}) + + assert "Database does not exists: does_not_exists" in str(exc_info.value) + + +def test_update_namespace_properties_overlap(): + catalog = HiveCatalog(HIVE_CATALOG_NAME, {}, uri=HIVE_METASTORE_FAKE_URL) + + with pytest.raises(ValueError) as exc_info: + catalog.update_namespace_properties(("table",), removals=set("a"), updates={"a": "b"}) + + assert "Updates and deletes have an overlap: {'a'}" in str(exc_info.value) diff --git a/vendor/README.md b/vendor/README.md new file mode 100644 index 0000000000..0b55d9e5c6 --- /dev/null +++ b/vendor/README.md @@ -0,0 +1,45 @@ + +# Vendor packages + +Some packages we want to maintain in the repository itself, because there is no good 3rd party alternative. + +## FB303 Thrift client + +fb303 is a base Thrift service and a common set of functionality for querying stats, options, and other information from a service. + +```bash +rm -f /tmp/fb303.thrift +rm -rf fb303 +curl -s https://raw.githubusercontent.com/apache/thrift/master/contrib/fb303/if/fb303.thrift > /tmp/fb303.thrift +rm -rf /tmp/gen-py/ +thrift -gen py -o /tmp/ /tmp/fb303.thrift +mv /tmp/gen-py/fb303 fb303 +``` + +# Hive Metastore Thrift definition + +The thrift definition require the fb303 service as a dependency + +```bash +rm -rf /tmp/hive +mkdir -p /tmp/hive/share/fb303/if/ +curl -s https://raw.githubusercontent.com/apache/thrift/master/contrib/fb303/if/fb303.thrift > /tmp/hive/share/fb303/if/fb303.thrift +curl -s https://raw.githubusercontent.com/apache/hive/master/standalone-metastore/metastore-common/src/main/thrift/hive_metastore.thrift > /tmp/hive/hive_metastore.thrift +thrift -gen py -o /tmp/hive /tmp/hive/hive_metastore.thrift +mv /tmp/hive/gen-py/hive_metastore hive_metastore +``` diff --git a/vendor/fb303/FacebookService.py b/vendor/fb303/FacebookService.py new file mode 100644 index 0000000000..c46b0a82a2 --- /dev/null +++ b/vendor/fb303/FacebookService.py @@ -0,0 +1,2420 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# +# Autogenerated by Thrift Compiler (0.16.0) +# +# DO NOT EDIT UNLESS YOU ARE SURE THAT YOU KNOW WHAT YOU ARE DOING +# +# options string: py +# + +import logging +import sys + +from thrift.Thrift import ( + TApplicationException, + TMessageType, + TProcessor, + TType, +) +from thrift.transport import TTransport +from thrift.TRecursive import fix_spec + +from .ttypes import * + +all_structs = [] + + +class Iface: + """ + Standard base service + + """ + + def getName(self): + """ + Returns a descriptive name of the service + + """ + pass + + def getVersion(self): + """ + Returns the version of the service + + """ + pass + + def getStatus(self): + """ + Gets the status of this service + + """ + pass + + def getStatusDetails(self): + """ + User friendly description of status, such as why the service is in + the dead or warning state, or what is being started or stopped. + + """ + pass + + def getCounters(self): + """ + Gets the counters for this service + + """ + pass + + def getCounter(self, key): + """ + Gets the value of a single counter + + Parameters: + - key + + """ + pass + + def setOption(self, key, value): + """ + Sets an option + + Parameters: + - key + - value + + """ + pass + + def getOption(self, key): + """ + Gets an option + + Parameters: + - key + + """ + pass + + def getOptions(self): + """ + Gets all options + + """ + pass + + def getCpuProfile(self, profileDurationInSec): + """ + Returns a CPU profile over the given time interval (client and server + must agree on the profile format). + + Parameters: + - profileDurationInSec + + """ + pass + + def aliveSince(self): + """ + Returns the unix time that the server has been running since + + """ + pass + + def reinitialize(self): + """ + Tell the server to reload its configuration, reopen log files, etc + + """ + pass + + def shutdown(self): + """ + Suggest a shutdown to the server + + """ + pass + + +class Client(Iface): + """ + Standard base service + + """ + + def __init__(self, iprot, oprot=None): + self._iprot = self._oprot = iprot + if oprot is not None: + self._oprot = oprot + self._seqid = 0 + + def getName(self): + """ + Returns a descriptive name of the service + + """ + self.send_getName() + return self.recv_getName() + + def send_getName(self): + self._oprot.writeMessageBegin("getName", TMessageType.CALL, self._seqid) + args = getName_args() + args.write(self._oprot) + self._oprot.writeMessageEnd() + self._oprot.trans.flush() + + def recv_getName(self): + iprot = self._iprot + (fname, mtype, rseqid) = iprot.readMessageBegin() + if mtype == TMessageType.EXCEPTION: + x = TApplicationException() + x.read(iprot) + iprot.readMessageEnd() + raise x + result = getName_result() + result.read(iprot) + iprot.readMessageEnd() + if result.success is not None: + return result.success + raise TApplicationException(TApplicationException.MISSING_RESULT, "getName failed: unknown result") + + def getVersion(self): + """ + Returns the version of the service + + """ + self.send_getVersion() + return self.recv_getVersion() + + def send_getVersion(self): + self._oprot.writeMessageBegin("getVersion", TMessageType.CALL, self._seqid) + args = getVersion_args() + args.write(self._oprot) + self._oprot.writeMessageEnd() + self._oprot.trans.flush() + + def recv_getVersion(self): + iprot = self._iprot + (fname, mtype, rseqid) = iprot.readMessageBegin() + if mtype == TMessageType.EXCEPTION: + x = TApplicationException() + x.read(iprot) + iprot.readMessageEnd() + raise x + result = getVersion_result() + result.read(iprot) + iprot.readMessageEnd() + if result.success is not None: + return result.success + raise TApplicationException(TApplicationException.MISSING_RESULT, "getVersion failed: unknown result") + + def getStatus(self): + """ + Gets the status of this service + + """ + self.send_getStatus() + return self.recv_getStatus() + + def send_getStatus(self): + self._oprot.writeMessageBegin("getStatus", TMessageType.CALL, self._seqid) + args = getStatus_args() + args.write(self._oprot) + self._oprot.writeMessageEnd() + self._oprot.trans.flush() + + def recv_getStatus(self): + iprot = self._iprot + (fname, mtype, rseqid) = iprot.readMessageBegin() + if mtype == TMessageType.EXCEPTION: + x = TApplicationException() + x.read(iprot) + iprot.readMessageEnd() + raise x + result = getStatus_result() + result.read(iprot) + iprot.readMessageEnd() + if result.success is not None: + return result.success + raise TApplicationException(TApplicationException.MISSING_RESULT, "getStatus failed: unknown result") + + def getStatusDetails(self): + """ + User friendly description of status, such as why the service is in + the dead or warning state, or what is being started or stopped. + + """ + self.send_getStatusDetails() + return self.recv_getStatusDetails() + + def send_getStatusDetails(self): + self._oprot.writeMessageBegin("getStatusDetails", TMessageType.CALL, self._seqid) + args = getStatusDetails_args() + args.write(self._oprot) + self._oprot.writeMessageEnd() + self._oprot.trans.flush() + + def recv_getStatusDetails(self): + iprot = self._iprot + (fname, mtype, rseqid) = iprot.readMessageBegin() + if mtype == TMessageType.EXCEPTION: + x = TApplicationException() + x.read(iprot) + iprot.readMessageEnd() + raise x + result = getStatusDetails_result() + result.read(iprot) + iprot.readMessageEnd() + if result.success is not None: + return result.success + raise TApplicationException(TApplicationException.MISSING_RESULT, "getStatusDetails failed: unknown result") + + def getCounters(self): + """ + Gets the counters for this service + + """ + self.send_getCounters() + return self.recv_getCounters() + + def send_getCounters(self): + self._oprot.writeMessageBegin("getCounters", TMessageType.CALL, self._seqid) + args = getCounters_args() + args.write(self._oprot) + self._oprot.writeMessageEnd() + self._oprot.trans.flush() + + def recv_getCounters(self): + iprot = self._iprot + (fname, mtype, rseqid) = iprot.readMessageBegin() + if mtype == TMessageType.EXCEPTION: + x = TApplicationException() + x.read(iprot) + iprot.readMessageEnd() + raise x + result = getCounters_result() + result.read(iprot) + iprot.readMessageEnd() + if result.success is not None: + return result.success + raise TApplicationException(TApplicationException.MISSING_RESULT, "getCounters failed: unknown result") + + def getCounter(self, key): + """ + Gets the value of a single counter + + Parameters: + - key + + """ + self.send_getCounter(key) + return self.recv_getCounter() + + def send_getCounter(self, key): + self._oprot.writeMessageBegin("getCounter", TMessageType.CALL, self._seqid) + args = getCounter_args() + args.key = key + args.write(self._oprot) + self._oprot.writeMessageEnd() + self._oprot.trans.flush() + + def recv_getCounter(self): + iprot = self._iprot + (fname, mtype, rseqid) = iprot.readMessageBegin() + if mtype == TMessageType.EXCEPTION: + x = TApplicationException() + x.read(iprot) + iprot.readMessageEnd() + raise x + result = getCounter_result() + result.read(iprot) + iprot.readMessageEnd() + if result.success is not None: + return result.success + raise TApplicationException(TApplicationException.MISSING_RESULT, "getCounter failed: unknown result") + + def setOption(self, key, value): + """ + Sets an option + + Parameters: + - key + - value + + """ + self.send_setOption(key, value) + self.recv_setOption() + + def send_setOption(self, key, value): + self._oprot.writeMessageBegin("setOption", TMessageType.CALL, self._seqid) + args = setOption_args() + args.key = key + args.value = value + args.write(self._oprot) + self._oprot.writeMessageEnd() + self._oprot.trans.flush() + + def recv_setOption(self): + iprot = self._iprot + (fname, mtype, rseqid) = iprot.readMessageBegin() + if mtype == TMessageType.EXCEPTION: + x = TApplicationException() + x.read(iprot) + iprot.readMessageEnd() + raise x + result = setOption_result() + result.read(iprot) + iprot.readMessageEnd() + return + + def getOption(self, key): + """ + Gets an option + + Parameters: + - key + + """ + self.send_getOption(key) + return self.recv_getOption() + + def send_getOption(self, key): + self._oprot.writeMessageBegin("getOption", TMessageType.CALL, self._seqid) + args = getOption_args() + args.key = key + args.write(self._oprot) + self._oprot.writeMessageEnd() + self._oprot.trans.flush() + + def recv_getOption(self): + iprot = self._iprot + (fname, mtype, rseqid) = iprot.readMessageBegin() + if mtype == TMessageType.EXCEPTION: + x = TApplicationException() + x.read(iprot) + iprot.readMessageEnd() + raise x + result = getOption_result() + result.read(iprot) + iprot.readMessageEnd() + if result.success is not None: + return result.success + raise TApplicationException(TApplicationException.MISSING_RESULT, "getOption failed: unknown result") + + def getOptions(self): + """ + Gets all options + + """ + self.send_getOptions() + return self.recv_getOptions() + + def send_getOptions(self): + self._oprot.writeMessageBegin("getOptions", TMessageType.CALL, self._seqid) + args = getOptions_args() + args.write(self._oprot) + self._oprot.writeMessageEnd() + self._oprot.trans.flush() + + def recv_getOptions(self): + iprot = self._iprot + (fname, mtype, rseqid) = iprot.readMessageBegin() + if mtype == TMessageType.EXCEPTION: + x = TApplicationException() + x.read(iprot) + iprot.readMessageEnd() + raise x + result = getOptions_result() + result.read(iprot) + iprot.readMessageEnd() + if result.success is not None: + return result.success + raise TApplicationException(TApplicationException.MISSING_RESULT, "getOptions failed: unknown result") + + def getCpuProfile(self, profileDurationInSec): + """ + Returns a CPU profile over the given time interval (client and server + must agree on the profile format). + + Parameters: + - profileDurationInSec + + """ + self.send_getCpuProfile(profileDurationInSec) + return self.recv_getCpuProfile() + + def send_getCpuProfile(self, profileDurationInSec): + self._oprot.writeMessageBegin("getCpuProfile", TMessageType.CALL, self._seqid) + args = getCpuProfile_args() + args.profileDurationInSec = profileDurationInSec + args.write(self._oprot) + self._oprot.writeMessageEnd() + self._oprot.trans.flush() + + def recv_getCpuProfile(self): + iprot = self._iprot + (fname, mtype, rseqid) = iprot.readMessageBegin() + if mtype == TMessageType.EXCEPTION: + x = TApplicationException() + x.read(iprot) + iprot.readMessageEnd() + raise x + result = getCpuProfile_result() + result.read(iprot) + iprot.readMessageEnd() + if result.success is not None: + return result.success + raise TApplicationException(TApplicationException.MISSING_RESULT, "getCpuProfile failed: unknown result") + + def aliveSince(self): + """ + Returns the unix time that the server has been running since + + """ + self.send_aliveSince() + return self.recv_aliveSince() + + def send_aliveSince(self): + self._oprot.writeMessageBegin("aliveSince", TMessageType.CALL, self._seqid) + args = aliveSince_args() + args.write(self._oprot) + self._oprot.writeMessageEnd() + self._oprot.trans.flush() + + def recv_aliveSince(self): + iprot = self._iprot + (fname, mtype, rseqid) = iprot.readMessageBegin() + if mtype == TMessageType.EXCEPTION: + x = TApplicationException() + x.read(iprot) + iprot.readMessageEnd() + raise x + result = aliveSince_result() + result.read(iprot) + iprot.readMessageEnd() + if result.success is not None: + return result.success + raise TApplicationException(TApplicationException.MISSING_RESULT, "aliveSince failed: unknown result") + + def reinitialize(self): + """ + Tell the server to reload its configuration, reopen log files, etc + + """ + self.send_reinitialize() + + def send_reinitialize(self): + self._oprot.writeMessageBegin("reinitialize", TMessageType.ONEWAY, self._seqid) + args = reinitialize_args() + args.write(self._oprot) + self._oprot.writeMessageEnd() + self._oprot.trans.flush() + + def shutdown(self): + """ + Suggest a shutdown to the server + + """ + self.send_shutdown() + + def send_shutdown(self): + self._oprot.writeMessageBegin("shutdown", TMessageType.ONEWAY, self._seqid) + args = shutdown_args() + args.write(self._oprot) + self._oprot.writeMessageEnd() + self._oprot.trans.flush() + + +class Processor(Iface, TProcessor): + def __init__(self, handler): + self._handler = handler + self._processMap = {} + self._processMap["getName"] = Processor.process_getName + self._processMap["getVersion"] = Processor.process_getVersion + self._processMap["getStatus"] = Processor.process_getStatus + self._processMap["getStatusDetails"] = Processor.process_getStatusDetails + self._processMap["getCounters"] = Processor.process_getCounters + self._processMap["getCounter"] = Processor.process_getCounter + self._processMap["setOption"] = Processor.process_setOption + self._processMap["getOption"] = Processor.process_getOption + self._processMap["getOptions"] = Processor.process_getOptions + self._processMap["getCpuProfile"] = Processor.process_getCpuProfile + self._processMap["aliveSince"] = Processor.process_aliveSince + self._processMap["reinitialize"] = Processor.process_reinitialize + self._processMap["shutdown"] = Processor.process_shutdown + self._on_message_begin = None + + def on_message_begin(self, func): + self._on_message_begin = func + + def process(self, iprot, oprot): + (name, type, seqid) = iprot.readMessageBegin() + if self._on_message_begin: + self._on_message_begin(name, type, seqid) + if name not in self._processMap: + iprot.skip(TType.STRUCT) + iprot.readMessageEnd() + x = TApplicationException(TApplicationException.UNKNOWN_METHOD, "Unknown function %s" % (name)) + oprot.writeMessageBegin(name, TMessageType.EXCEPTION, seqid) + x.write(oprot) + oprot.writeMessageEnd() + oprot.trans.flush() + return + else: + self._processMap[name](self, seqid, iprot, oprot) + return True + + def process_getName(self, seqid, iprot, oprot): + args = getName_args() + args.read(iprot) + iprot.readMessageEnd() + result = getName_result() + try: + result.success = self._handler.getName() + msg_type = TMessageType.REPLY + except TTransport.TTransportException: + raise + except TApplicationException as ex: + logging.exception("TApplication exception in handler") + msg_type = TMessageType.EXCEPTION + result = ex + except Exception: + logging.exception("Unexpected exception in handler") + msg_type = TMessageType.EXCEPTION + result = TApplicationException(TApplicationException.INTERNAL_ERROR, "Internal error") + oprot.writeMessageBegin("getName", msg_type, seqid) + result.write(oprot) + oprot.writeMessageEnd() + oprot.trans.flush() + + def process_getVersion(self, seqid, iprot, oprot): + args = getVersion_args() + args.read(iprot) + iprot.readMessageEnd() + result = getVersion_result() + try: + result.success = self._handler.getVersion() + msg_type = TMessageType.REPLY + except TTransport.TTransportException: + raise + except TApplicationException as ex: + logging.exception("TApplication exception in handler") + msg_type = TMessageType.EXCEPTION + result = ex + except Exception: + logging.exception("Unexpected exception in handler") + msg_type = TMessageType.EXCEPTION + result = TApplicationException(TApplicationException.INTERNAL_ERROR, "Internal error") + oprot.writeMessageBegin("getVersion", msg_type, seqid) + result.write(oprot) + oprot.writeMessageEnd() + oprot.trans.flush() + + def process_getStatus(self, seqid, iprot, oprot): + args = getStatus_args() + args.read(iprot) + iprot.readMessageEnd() + result = getStatus_result() + try: + result.success = self._handler.getStatus() + msg_type = TMessageType.REPLY + except TTransport.TTransportException: + raise + except TApplicationException as ex: + logging.exception("TApplication exception in handler") + msg_type = TMessageType.EXCEPTION + result = ex + except Exception: + logging.exception("Unexpected exception in handler") + msg_type = TMessageType.EXCEPTION + result = TApplicationException(TApplicationException.INTERNAL_ERROR, "Internal error") + oprot.writeMessageBegin("getStatus", msg_type, seqid) + result.write(oprot) + oprot.writeMessageEnd() + oprot.trans.flush() + + def process_getStatusDetails(self, seqid, iprot, oprot): + args = getStatusDetails_args() + args.read(iprot) + iprot.readMessageEnd() + result = getStatusDetails_result() + try: + result.success = self._handler.getStatusDetails() + msg_type = TMessageType.REPLY + except TTransport.TTransportException: + raise + except TApplicationException as ex: + logging.exception("TApplication exception in handler") + msg_type = TMessageType.EXCEPTION + result = ex + except Exception: + logging.exception("Unexpected exception in handler") + msg_type = TMessageType.EXCEPTION + result = TApplicationException(TApplicationException.INTERNAL_ERROR, "Internal error") + oprot.writeMessageBegin("getStatusDetails", msg_type, seqid) + result.write(oprot) + oprot.writeMessageEnd() + oprot.trans.flush() + + def process_getCounters(self, seqid, iprot, oprot): + args = getCounters_args() + args.read(iprot) + iprot.readMessageEnd() + result = getCounters_result() + try: + result.success = self._handler.getCounters() + msg_type = TMessageType.REPLY + except TTransport.TTransportException: + raise + except TApplicationException as ex: + logging.exception("TApplication exception in handler") + msg_type = TMessageType.EXCEPTION + result = ex + except Exception: + logging.exception("Unexpected exception in handler") + msg_type = TMessageType.EXCEPTION + result = TApplicationException(TApplicationException.INTERNAL_ERROR, "Internal error") + oprot.writeMessageBegin("getCounters", msg_type, seqid) + result.write(oprot) + oprot.writeMessageEnd() + oprot.trans.flush() + + def process_getCounter(self, seqid, iprot, oprot): + args = getCounter_args() + args.read(iprot) + iprot.readMessageEnd() + result = getCounter_result() + try: + result.success = self._handler.getCounter(args.key) + msg_type = TMessageType.REPLY + except TTransport.TTransportException: + raise + except TApplicationException as ex: + logging.exception("TApplication exception in handler") + msg_type = TMessageType.EXCEPTION + result = ex + except Exception: + logging.exception("Unexpected exception in handler") + msg_type = TMessageType.EXCEPTION + result = TApplicationException(TApplicationException.INTERNAL_ERROR, "Internal error") + oprot.writeMessageBegin("getCounter", msg_type, seqid) + result.write(oprot) + oprot.writeMessageEnd() + oprot.trans.flush() + + def process_setOption(self, seqid, iprot, oprot): + args = setOption_args() + args.read(iprot) + iprot.readMessageEnd() + result = setOption_result() + try: + self._handler.setOption(args.key, args.value) + msg_type = TMessageType.REPLY + except TTransport.TTransportException: + raise + except TApplicationException as ex: + logging.exception("TApplication exception in handler") + msg_type = TMessageType.EXCEPTION + result = ex + except Exception: + logging.exception("Unexpected exception in handler") + msg_type = TMessageType.EXCEPTION + result = TApplicationException(TApplicationException.INTERNAL_ERROR, "Internal error") + oprot.writeMessageBegin("setOption", msg_type, seqid) + result.write(oprot) + oprot.writeMessageEnd() + oprot.trans.flush() + + def process_getOption(self, seqid, iprot, oprot): + args = getOption_args() + args.read(iprot) + iprot.readMessageEnd() + result = getOption_result() + try: + result.success = self._handler.getOption(args.key) + msg_type = TMessageType.REPLY + except TTransport.TTransportException: + raise + except TApplicationException as ex: + logging.exception("TApplication exception in handler") + msg_type = TMessageType.EXCEPTION + result = ex + except Exception: + logging.exception("Unexpected exception in handler") + msg_type = TMessageType.EXCEPTION + result = TApplicationException(TApplicationException.INTERNAL_ERROR, "Internal error") + oprot.writeMessageBegin("getOption", msg_type, seqid) + result.write(oprot) + oprot.writeMessageEnd() + oprot.trans.flush() + + def process_getOptions(self, seqid, iprot, oprot): + args = getOptions_args() + args.read(iprot) + iprot.readMessageEnd() + result = getOptions_result() + try: + result.success = self._handler.getOptions() + msg_type = TMessageType.REPLY + except TTransport.TTransportException: + raise + except TApplicationException as ex: + logging.exception("TApplication exception in handler") + msg_type = TMessageType.EXCEPTION + result = ex + except Exception: + logging.exception("Unexpected exception in handler") + msg_type = TMessageType.EXCEPTION + result = TApplicationException(TApplicationException.INTERNAL_ERROR, "Internal error") + oprot.writeMessageBegin("getOptions", msg_type, seqid) + result.write(oprot) + oprot.writeMessageEnd() + oprot.trans.flush() + + def process_getCpuProfile(self, seqid, iprot, oprot): + args = getCpuProfile_args() + args.read(iprot) + iprot.readMessageEnd() + result = getCpuProfile_result() + try: + result.success = self._handler.getCpuProfile(args.profileDurationInSec) + msg_type = TMessageType.REPLY + except TTransport.TTransportException: + raise + except TApplicationException as ex: + logging.exception("TApplication exception in handler") + msg_type = TMessageType.EXCEPTION + result = ex + except Exception: + logging.exception("Unexpected exception in handler") + msg_type = TMessageType.EXCEPTION + result = TApplicationException(TApplicationException.INTERNAL_ERROR, "Internal error") + oprot.writeMessageBegin("getCpuProfile", msg_type, seqid) + result.write(oprot) + oprot.writeMessageEnd() + oprot.trans.flush() + + def process_aliveSince(self, seqid, iprot, oprot): + args = aliveSince_args() + args.read(iprot) + iprot.readMessageEnd() + result = aliveSince_result() + try: + result.success = self._handler.aliveSince() + msg_type = TMessageType.REPLY + except TTransport.TTransportException: + raise + except TApplicationException as ex: + logging.exception("TApplication exception in handler") + msg_type = TMessageType.EXCEPTION + result = ex + except Exception: + logging.exception("Unexpected exception in handler") + msg_type = TMessageType.EXCEPTION + result = TApplicationException(TApplicationException.INTERNAL_ERROR, "Internal error") + oprot.writeMessageBegin("aliveSince", msg_type, seqid) + result.write(oprot) + oprot.writeMessageEnd() + oprot.trans.flush() + + def process_reinitialize(self, seqid, iprot, oprot): + args = reinitialize_args() + args.read(iprot) + iprot.readMessageEnd() + try: + self._handler.reinitialize() + except TTransport.TTransportException: + raise + except Exception: + logging.exception("Exception in oneway handler") + + def process_shutdown(self, seqid, iprot, oprot): + args = shutdown_args() + args.read(iprot) + iprot.readMessageEnd() + try: + self._handler.shutdown() + except TTransport.TTransportException: + raise + except Exception: + logging.exception("Exception in oneway handler") + + +# HELPER FUNCTIONS AND STRUCTURES + + +class getName_args: + def read(self, iprot): + if ( + iprot._fast_decode is not None + and isinstance(iprot.trans, TTransport.CReadableTransport) + and self.thrift_spec is not None + ): + iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec]) + return + iprot.readStructBegin() + while True: + (fname, ftype, fid) = iprot.readFieldBegin() + if ftype == TType.STOP: + break + else: + iprot.skip(ftype) + iprot.readFieldEnd() + iprot.readStructEnd() + + def write(self, oprot): + if oprot._fast_encode is not None and self.thrift_spec is not None: + oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec])) + return + oprot.writeStructBegin("getName_args") + oprot.writeFieldStop() + oprot.writeStructEnd() + + def validate(self): + return + + def __repr__(self): + L = ["{}={!r}".format(key, value) for key, value in self.__dict__.items()] + return "{}({})".format(self.__class__.__name__, ", ".join(L)) + + def __eq__(self, other): + return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ + + def __ne__(self, other): + return not (self == other) + + +all_structs.append(getName_args) +getName_args.thrift_spec = () + + +class getName_result: + """ + Attributes: + - success + + """ + + def __init__( + self, + success=None, + ): + self.success = success + + def read(self, iprot): + if ( + iprot._fast_decode is not None + and isinstance(iprot.trans, TTransport.CReadableTransport) + and self.thrift_spec is not None + ): + iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec]) + return + iprot.readStructBegin() + while True: + (fname, ftype, fid) = iprot.readFieldBegin() + if ftype == TType.STOP: + break + if fid == 0: + if ftype == TType.STRING: + self.success = ( + iprot.readString().decode("utf-8", errors="replace") if sys.version_info[0] == 2 else iprot.readString() + ) + else: + iprot.skip(ftype) + else: + iprot.skip(ftype) + iprot.readFieldEnd() + iprot.readStructEnd() + + def write(self, oprot): + if oprot._fast_encode is not None and self.thrift_spec is not None: + oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec])) + return + oprot.writeStructBegin("getName_result") + if self.success is not None: + oprot.writeFieldBegin("success", TType.STRING, 0) + oprot.writeString(self.success.encode("utf-8") if sys.version_info[0] == 2 else self.success) + oprot.writeFieldEnd() + oprot.writeFieldStop() + oprot.writeStructEnd() + + def validate(self): + return + + def __repr__(self): + L = ["{}={!r}".format(key, value) for key, value in self.__dict__.items()] + return "{}({})".format(self.__class__.__name__, ", ".join(L)) + + def __eq__(self, other): + return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ + + def __ne__(self, other): + return not (self == other) + + +all_structs.append(getName_result) +getName_result.thrift_spec = ( + ( + 0, + TType.STRING, + "success", + "UTF8", + None, + ), # 0 +) + + +class getVersion_args: + def read(self, iprot): + if ( + iprot._fast_decode is not None + and isinstance(iprot.trans, TTransport.CReadableTransport) + and self.thrift_spec is not None + ): + iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec]) + return + iprot.readStructBegin() + while True: + (fname, ftype, fid) = iprot.readFieldBegin() + if ftype == TType.STOP: + break + else: + iprot.skip(ftype) + iprot.readFieldEnd() + iprot.readStructEnd() + + def write(self, oprot): + if oprot._fast_encode is not None and self.thrift_spec is not None: + oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec])) + return + oprot.writeStructBegin("getVersion_args") + oprot.writeFieldStop() + oprot.writeStructEnd() + + def validate(self): + return + + def __repr__(self): + L = ["{}={!r}".format(key, value) for key, value in self.__dict__.items()] + return "{}({})".format(self.__class__.__name__, ", ".join(L)) + + def __eq__(self, other): + return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ + + def __ne__(self, other): + return not (self == other) + + +all_structs.append(getVersion_args) +getVersion_args.thrift_spec = () + + +class getVersion_result: + """ + Attributes: + - success + + """ + + def __init__( + self, + success=None, + ): + self.success = success + + def read(self, iprot): + if ( + iprot._fast_decode is not None + and isinstance(iprot.trans, TTransport.CReadableTransport) + and self.thrift_spec is not None + ): + iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec]) + return + iprot.readStructBegin() + while True: + (fname, ftype, fid) = iprot.readFieldBegin() + if ftype == TType.STOP: + break + if fid == 0: + if ftype == TType.STRING: + self.success = ( + iprot.readString().decode("utf-8", errors="replace") if sys.version_info[0] == 2 else iprot.readString() + ) + else: + iprot.skip(ftype) + else: + iprot.skip(ftype) + iprot.readFieldEnd() + iprot.readStructEnd() + + def write(self, oprot): + if oprot._fast_encode is not None and self.thrift_spec is not None: + oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec])) + return + oprot.writeStructBegin("getVersion_result") + if self.success is not None: + oprot.writeFieldBegin("success", TType.STRING, 0) + oprot.writeString(self.success.encode("utf-8") if sys.version_info[0] == 2 else self.success) + oprot.writeFieldEnd() + oprot.writeFieldStop() + oprot.writeStructEnd() + + def validate(self): + return + + def __repr__(self): + L = ["{}={!r}".format(key, value) for key, value in self.__dict__.items()] + return "{}({})".format(self.__class__.__name__, ", ".join(L)) + + def __eq__(self, other): + return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ + + def __ne__(self, other): + return not (self == other) + + +all_structs.append(getVersion_result) +getVersion_result.thrift_spec = ( + ( + 0, + TType.STRING, + "success", + "UTF8", + None, + ), # 0 +) + + +class getStatus_args: + def read(self, iprot): + if ( + iprot._fast_decode is not None + and isinstance(iprot.trans, TTransport.CReadableTransport) + and self.thrift_spec is not None + ): + iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec]) + return + iprot.readStructBegin() + while True: + (fname, ftype, fid) = iprot.readFieldBegin() + if ftype == TType.STOP: + break + else: + iprot.skip(ftype) + iprot.readFieldEnd() + iprot.readStructEnd() + + def write(self, oprot): + if oprot._fast_encode is not None and self.thrift_spec is not None: + oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec])) + return + oprot.writeStructBegin("getStatus_args") + oprot.writeFieldStop() + oprot.writeStructEnd() + + def validate(self): + return + + def __repr__(self): + L = ["{}={!r}".format(key, value) for key, value in self.__dict__.items()] + return "{}({})".format(self.__class__.__name__, ", ".join(L)) + + def __eq__(self, other): + return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ + + def __ne__(self, other): + return not (self == other) + + +all_structs.append(getStatus_args) +getStatus_args.thrift_spec = () + + +class getStatus_result: + """ + Attributes: + - success + + """ + + def __init__( + self, + success=None, + ): + self.success = success + + def read(self, iprot): + if ( + iprot._fast_decode is not None + and isinstance(iprot.trans, TTransport.CReadableTransport) + and self.thrift_spec is not None + ): + iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec]) + return + iprot.readStructBegin() + while True: + (fname, ftype, fid) = iprot.readFieldBegin() + if ftype == TType.STOP: + break + if fid == 0: + if ftype == TType.I32: + self.success = iprot.readI32() + else: + iprot.skip(ftype) + else: + iprot.skip(ftype) + iprot.readFieldEnd() + iprot.readStructEnd() + + def write(self, oprot): + if oprot._fast_encode is not None and self.thrift_spec is not None: + oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec])) + return + oprot.writeStructBegin("getStatus_result") + if self.success is not None: + oprot.writeFieldBegin("success", TType.I32, 0) + oprot.writeI32(self.success) + oprot.writeFieldEnd() + oprot.writeFieldStop() + oprot.writeStructEnd() + + def validate(self): + return + + def __repr__(self): + L = ["{}={!r}".format(key, value) for key, value in self.__dict__.items()] + return "{}({})".format(self.__class__.__name__, ", ".join(L)) + + def __eq__(self, other): + return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ + + def __ne__(self, other): + return not (self == other) + + +all_structs.append(getStatus_result) +getStatus_result.thrift_spec = ( + ( + 0, + TType.I32, + "success", + None, + None, + ), # 0 +) + + +class getStatusDetails_args: + def read(self, iprot): + if ( + iprot._fast_decode is not None + and isinstance(iprot.trans, TTransport.CReadableTransport) + and self.thrift_spec is not None + ): + iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec]) + return + iprot.readStructBegin() + while True: + (fname, ftype, fid) = iprot.readFieldBegin() + if ftype == TType.STOP: + break + else: + iprot.skip(ftype) + iprot.readFieldEnd() + iprot.readStructEnd() + + def write(self, oprot): + if oprot._fast_encode is not None and self.thrift_spec is not None: + oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec])) + return + oprot.writeStructBegin("getStatusDetails_args") + oprot.writeFieldStop() + oprot.writeStructEnd() + + def validate(self): + return + + def __repr__(self): + L = ["{}={!r}".format(key, value) for key, value in self.__dict__.items()] + return "{}({})".format(self.__class__.__name__, ", ".join(L)) + + def __eq__(self, other): + return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ + + def __ne__(self, other): + return not (self == other) + + +all_structs.append(getStatusDetails_args) +getStatusDetails_args.thrift_spec = () + + +class getStatusDetails_result: + """ + Attributes: + - success + + """ + + def __init__( + self, + success=None, + ): + self.success = success + + def read(self, iprot): + if ( + iprot._fast_decode is not None + and isinstance(iprot.trans, TTransport.CReadableTransport) + and self.thrift_spec is not None + ): + iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec]) + return + iprot.readStructBegin() + while True: + (fname, ftype, fid) = iprot.readFieldBegin() + if ftype == TType.STOP: + break + if fid == 0: + if ftype == TType.STRING: + self.success = ( + iprot.readString().decode("utf-8", errors="replace") if sys.version_info[0] == 2 else iprot.readString() + ) + else: + iprot.skip(ftype) + else: + iprot.skip(ftype) + iprot.readFieldEnd() + iprot.readStructEnd() + + def write(self, oprot): + if oprot._fast_encode is not None and self.thrift_spec is not None: + oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec])) + return + oprot.writeStructBegin("getStatusDetails_result") + if self.success is not None: + oprot.writeFieldBegin("success", TType.STRING, 0) + oprot.writeString(self.success.encode("utf-8") if sys.version_info[0] == 2 else self.success) + oprot.writeFieldEnd() + oprot.writeFieldStop() + oprot.writeStructEnd() + + def validate(self): + return + + def __repr__(self): + L = ["{}={!r}".format(key, value) for key, value in self.__dict__.items()] + return "{}({})".format(self.__class__.__name__, ", ".join(L)) + + def __eq__(self, other): + return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ + + def __ne__(self, other): + return not (self == other) + + +all_structs.append(getStatusDetails_result) +getStatusDetails_result.thrift_spec = ( + ( + 0, + TType.STRING, + "success", + "UTF8", + None, + ), # 0 +) + + +class getCounters_args: + def read(self, iprot): + if ( + iprot._fast_decode is not None + and isinstance(iprot.trans, TTransport.CReadableTransport) + and self.thrift_spec is not None + ): + iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec]) + return + iprot.readStructBegin() + while True: + (fname, ftype, fid) = iprot.readFieldBegin() + if ftype == TType.STOP: + break + else: + iprot.skip(ftype) + iprot.readFieldEnd() + iprot.readStructEnd() + + def write(self, oprot): + if oprot._fast_encode is not None and self.thrift_spec is not None: + oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec])) + return + oprot.writeStructBegin("getCounters_args") + oprot.writeFieldStop() + oprot.writeStructEnd() + + def validate(self): + return + + def __repr__(self): + L = ["{}={!r}".format(key, value) for key, value in self.__dict__.items()] + return "{}({})".format(self.__class__.__name__, ", ".join(L)) + + def __eq__(self, other): + return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ + + def __ne__(self, other): + return not (self == other) + + +all_structs.append(getCounters_args) +getCounters_args.thrift_spec = () + + +class getCounters_result: + """ + Attributes: + - success + + """ + + def __init__( + self, + success=None, + ): + self.success = success + + def read(self, iprot): + if ( + iprot._fast_decode is not None + and isinstance(iprot.trans, TTransport.CReadableTransport) + and self.thrift_spec is not None + ): + iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec]) + return + iprot.readStructBegin() + while True: + (fname, ftype, fid) = iprot.readFieldBegin() + if ftype == TType.STOP: + break + if fid == 0: + if ftype == TType.MAP: + self.success = {} + (_ktype1, _vtype2, _size0) = iprot.readMapBegin() + for _i4 in range(_size0): + _key5 = ( + iprot.readString().decode("utf-8", errors="replace") + if sys.version_info[0] == 2 + else iprot.readString() + ) + _val6 = iprot.readI64() + self.success[_key5] = _val6 + iprot.readMapEnd() + else: + iprot.skip(ftype) + else: + iprot.skip(ftype) + iprot.readFieldEnd() + iprot.readStructEnd() + + def write(self, oprot): + if oprot._fast_encode is not None and self.thrift_spec is not None: + oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec])) + return + oprot.writeStructBegin("getCounters_result") + if self.success is not None: + oprot.writeFieldBegin("success", TType.MAP, 0) + oprot.writeMapBegin(TType.STRING, TType.I64, len(self.success)) + for kiter7, viter8 in self.success.items(): + oprot.writeString(kiter7.encode("utf-8") if sys.version_info[0] == 2 else kiter7) + oprot.writeI64(viter8) + oprot.writeMapEnd() + oprot.writeFieldEnd() + oprot.writeFieldStop() + oprot.writeStructEnd() + + def validate(self): + return + + def __repr__(self): + L = ["{}={!r}".format(key, value) for key, value in self.__dict__.items()] + return "{}({})".format(self.__class__.__name__, ", ".join(L)) + + def __eq__(self, other): + return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ + + def __ne__(self, other): + return not (self == other) + + +all_structs.append(getCounters_result) +getCounters_result.thrift_spec = ( + ( + 0, + TType.MAP, + "success", + (TType.STRING, "UTF8", TType.I64, None, False), + None, + ), # 0 +) + + +class getCounter_args: + """ + Attributes: + - key + + """ + + def __init__( + self, + key=None, + ): + self.key = key + + def read(self, iprot): + if ( + iprot._fast_decode is not None + and isinstance(iprot.trans, TTransport.CReadableTransport) + and self.thrift_spec is not None + ): + iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec]) + return + iprot.readStructBegin() + while True: + (fname, ftype, fid) = iprot.readFieldBegin() + if ftype == TType.STOP: + break + if fid == 1: + if ftype == TType.STRING: + self.key = ( + iprot.readString().decode("utf-8", errors="replace") if sys.version_info[0] == 2 else iprot.readString() + ) + else: + iprot.skip(ftype) + else: + iprot.skip(ftype) + iprot.readFieldEnd() + iprot.readStructEnd() + + def write(self, oprot): + if oprot._fast_encode is not None and self.thrift_spec is not None: + oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec])) + return + oprot.writeStructBegin("getCounter_args") + if self.key is not None: + oprot.writeFieldBegin("key", TType.STRING, 1) + oprot.writeString(self.key.encode("utf-8") if sys.version_info[0] == 2 else self.key) + oprot.writeFieldEnd() + oprot.writeFieldStop() + oprot.writeStructEnd() + + def validate(self): + return + + def __repr__(self): + L = ["{}={!r}".format(key, value) for key, value in self.__dict__.items()] + return "{}({})".format(self.__class__.__name__, ", ".join(L)) + + def __eq__(self, other): + return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ + + def __ne__(self, other): + return not (self == other) + + +all_structs.append(getCounter_args) +getCounter_args.thrift_spec = ( + None, # 0 + ( + 1, + TType.STRING, + "key", + "UTF8", + None, + ), # 1 +) + + +class getCounter_result: + """ + Attributes: + - success + + """ + + def __init__( + self, + success=None, + ): + self.success = success + + def read(self, iprot): + if ( + iprot._fast_decode is not None + and isinstance(iprot.trans, TTransport.CReadableTransport) + and self.thrift_spec is not None + ): + iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec]) + return + iprot.readStructBegin() + while True: + (fname, ftype, fid) = iprot.readFieldBegin() + if ftype == TType.STOP: + break + if fid == 0: + if ftype == TType.I64: + self.success = iprot.readI64() + else: + iprot.skip(ftype) + else: + iprot.skip(ftype) + iprot.readFieldEnd() + iprot.readStructEnd() + + def write(self, oprot): + if oprot._fast_encode is not None and self.thrift_spec is not None: + oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec])) + return + oprot.writeStructBegin("getCounter_result") + if self.success is not None: + oprot.writeFieldBegin("success", TType.I64, 0) + oprot.writeI64(self.success) + oprot.writeFieldEnd() + oprot.writeFieldStop() + oprot.writeStructEnd() + + def validate(self): + return + + def __repr__(self): + L = ["{}={!r}".format(key, value) for key, value in self.__dict__.items()] + return "{}({})".format(self.__class__.__name__, ", ".join(L)) + + def __eq__(self, other): + return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ + + def __ne__(self, other): + return not (self == other) + + +all_structs.append(getCounter_result) +getCounter_result.thrift_spec = ( + ( + 0, + TType.I64, + "success", + None, + None, + ), # 0 +) + + +class setOption_args: + """ + Attributes: + - key + - value + + """ + + def __init__( + self, + key=None, + value=None, + ): + self.key = key + self.value = value + + def read(self, iprot): + if ( + iprot._fast_decode is not None + and isinstance(iprot.trans, TTransport.CReadableTransport) + and self.thrift_spec is not None + ): + iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec]) + return + iprot.readStructBegin() + while True: + (fname, ftype, fid) = iprot.readFieldBegin() + if ftype == TType.STOP: + break + if fid == 1: + if ftype == TType.STRING: + self.key = ( + iprot.readString().decode("utf-8", errors="replace") if sys.version_info[0] == 2 else iprot.readString() + ) + else: + iprot.skip(ftype) + elif fid == 2: + if ftype == TType.STRING: + self.value = ( + iprot.readString().decode("utf-8", errors="replace") if sys.version_info[0] == 2 else iprot.readString() + ) + else: + iprot.skip(ftype) + else: + iprot.skip(ftype) + iprot.readFieldEnd() + iprot.readStructEnd() + + def write(self, oprot): + if oprot._fast_encode is not None and self.thrift_spec is not None: + oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec])) + return + oprot.writeStructBegin("setOption_args") + if self.key is not None: + oprot.writeFieldBegin("key", TType.STRING, 1) + oprot.writeString(self.key.encode("utf-8") if sys.version_info[0] == 2 else self.key) + oprot.writeFieldEnd() + if self.value is not None: + oprot.writeFieldBegin("value", TType.STRING, 2) + oprot.writeString(self.value.encode("utf-8") if sys.version_info[0] == 2 else self.value) + oprot.writeFieldEnd() + oprot.writeFieldStop() + oprot.writeStructEnd() + + def validate(self): + return + + def __repr__(self): + L = ["{}={!r}".format(key, value) for key, value in self.__dict__.items()] + return "{}({})".format(self.__class__.__name__, ", ".join(L)) + + def __eq__(self, other): + return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ + + def __ne__(self, other): + return not (self == other) + + +all_structs.append(setOption_args) +setOption_args.thrift_spec = ( + None, # 0 + ( + 1, + TType.STRING, + "key", + "UTF8", + None, + ), # 1 + ( + 2, + TType.STRING, + "value", + "UTF8", + None, + ), # 2 +) + + +class setOption_result: + def read(self, iprot): + if ( + iprot._fast_decode is not None + and isinstance(iprot.trans, TTransport.CReadableTransport) + and self.thrift_spec is not None + ): + iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec]) + return + iprot.readStructBegin() + while True: + (fname, ftype, fid) = iprot.readFieldBegin() + if ftype == TType.STOP: + break + else: + iprot.skip(ftype) + iprot.readFieldEnd() + iprot.readStructEnd() + + def write(self, oprot): + if oprot._fast_encode is not None and self.thrift_spec is not None: + oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec])) + return + oprot.writeStructBegin("setOption_result") + oprot.writeFieldStop() + oprot.writeStructEnd() + + def validate(self): + return + + def __repr__(self): + L = ["{}={!r}".format(key, value) for key, value in self.__dict__.items()] + return "{}({})".format(self.__class__.__name__, ", ".join(L)) + + def __eq__(self, other): + return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ + + def __ne__(self, other): + return not (self == other) + + +all_structs.append(setOption_result) +setOption_result.thrift_spec = () + + +class getOption_args: + """ + Attributes: + - key + + """ + + def __init__( + self, + key=None, + ): + self.key = key + + def read(self, iprot): + if ( + iprot._fast_decode is not None + and isinstance(iprot.trans, TTransport.CReadableTransport) + and self.thrift_spec is not None + ): + iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec]) + return + iprot.readStructBegin() + while True: + (fname, ftype, fid) = iprot.readFieldBegin() + if ftype == TType.STOP: + break + if fid == 1: + if ftype == TType.STRING: + self.key = ( + iprot.readString().decode("utf-8", errors="replace") if sys.version_info[0] == 2 else iprot.readString() + ) + else: + iprot.skip(ftype) + else: + iprot.skip(ftype) + iprot.readFieldEnd() + iprot.readStructEnd() + + def write(self, oprot): + if oprot._fast_encode is not None and self.thrift_spec is not None: + oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec])) + return + oprot.writeStructBegin("getOption_args") + if self.key is not None: + oprot.writeFieldBegin("key", TType.STRING, 1) + oprot.writeString(self.key.encode("utf-8") if sys.version_info[0] == 2 else self.key) + oprot.writeFieldEnd() + oprot.writeFieldStop() + oprot.writeStructEnd() + + def validate(self): + return + + def __repr__(self): + L = ["{}={!r}".format(key, value) for key, value in self.__dict__.items()] + return "{}({})".format(self.__class__.__name__, ", ".join(L)) + + def __eq__(self, other): + return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ + + def __ne__(self, other): + return not (self == other) + + +all_structs.append(getOption_args) +getOption_args.thrift_spec = ( + None, # 0 + ( + 1, + TType.STRING, + "key", + "UTF8", + None, + ), # 1 +) + + +class getOption_result: + """ + Attributes: + - success + + """ + + def __init__( + self, + success=None, + ): + self.success = success + + def read(self, iprot): + if ( + iprot._fast_decode is not None + and isinstance(iprot.trans, TTransport.CReadableTransport) + and self.thrift_spec is not None + ): + iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec]) + return + iprot.readStructBegin() + while True: + (fname, ftype, fid) = iprot.readFieldBegin() + if ftype == TType.STOP: + break + if fid == 0: + if ftype == TType.STRING: + self.success = ( + iprot.readString().decode("utf-8", errors="replace") if sys.version_info[0] == 2 else iprot.readString() + ) + else: + iprot.skip(ftype) + else: + iprot.skip(ftype) + iprot.readFieldEnd() + iprot.readStructEnd() + + def write(self, oprot): + if oprot._fast_encode is not None and self.thrift_spec is not None: + oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec])) + return + oprot.writeStructBegin("getOption_result") + if self.success is not None: + oprot.writeFieldBegin("success", TType.STRING, 0) + oprot.writeString(self.success.encode("utf-8") if sys.version_info[0] == 2 else self.success) + oprot.writeFieldEnd() + oprot.writeFieldStop() + oprot.writeStructEnd() + + def validate(self): + return + + def __repr__(self): + L = ["{}={!r}".format(key, value) for key, value in self.__dict__.items()] + return "{}({})".format(self.__class__.__name__, ", ".join(L)) + + def __eq__(self, other): + return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ + + def __ne__(self, other): + return not (self == other) + + +all_structs.append(getOption_result) +getOption_result.thrift_spec = ( + ( + 0, + TType.STRING, + "success", + "UTF8", + None, + ), # 0 +) + + +class getOptions_args: + def read(self, iprot): + if ( + iprot._fast_decode is not None + and isinstance(iprot.trans, TTransport.CReadableTransport) + and self.thrift_spec is not None + ): + iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec]) + return + iprot.readStructBegin() + while True: + (fname, ftype, fid) = iprot.readFieldBegin() + if ftype == TType.STOP: + break + else: + iprot.skip(ftype) + iprot.readFieldEnd() + iprot.readStructEnd() + + def write(self, oprot): + if oprot._fast_encode is not None and self.thrift_spec is not None: + oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec])) + return + oprot.writeStructBegin("getOptions_args") + oprot.writeFieldStop() + oprot.writeStructEnd() + + def validate(self): + return + + def __repr__(self): + L = ["{}={!r}".format(key, value) for key, value in self.__dict__.items()] + return "{}({})".format(self.__class__.__name__, ", ".join(L)) + + def __eq__(self, other): + return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ + + def __ne__(self, other): + return not (self == other) + + +all_structs.append(getOptions_args) +getOptions_args.thrift_spec = () + + +class getOptions_result: + """ + Attributes: + - success + + """ + + def __init__( + self, + success=None, + ): + self.success = success + + def read(self, iprot): + if ( + iprot._fast_decode is not None + and isinstance(iprot.trans, TTransport.CReadableTransport) + and self.thrift_spec is not None + ): + iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec]) + return + iprot.readStructBegin() + while True: + (fname, ftype, fid) = iprot.readFieldBegin() + if ftype == TType.STOP: + break + if fid == 0: + if ftype == TType.MAP: + self.success = {} + (_ktype10, _vtype11, _size9) = iprot.readMapBegin() + for _i13 in range(_size9): + _key14 = ( + iprot.readString().decode("utf-8", errors="replace") + if sys.version_info[0] == 2 + else iprot.readString() + ) + _val15 = ( + iprot.readString().decode("utf-8", errors="replace") + if sys.version_info[0] == 2 + else iprot.readString() + ) + self.success[_key14] = _val15 + iprot.readMapEnd() + else: + iprot.skip(ftype) + else: + iprot.skip(ftype) + iprot.readFieldEnd() + iprot.readStructEnd() + + def write(self, oprot): + if oprot._fast_encode is not None and self.thrift_spec is not None: + oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec])) + return + oprot.writeStructBegin("getOptions_result") + if self.success is not None: + oprot.writeFieldBegin("success", TType.MAP, 0) + oprot.writeMapBegin(TType.STRING, TType.STRING, len(self.success)) + for kiter16, viter17 in self.success.items(): + oprot.writeString(kiter16.encode("utf-8") if sys.version_info[0] == 2 else kiter16) + oprot.writeString(viter17.encode("utf-8") if sys.version_info[0] == 2 else viter17) + oprot.writeMapEnd() + oprot.writeFieldEnd() + oprot.writeFieldStop() + oprot.writeStructEnd() + + def validate(self): + return + + def __repr__(self): + L = ["{}={!r}".format(key, value) for key, value in self.__dict__.items()] + return "{}({})".format(self.__class__.__name__, ", ".join(L)) + + def __eq__(self, other): + return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ + + def __ne__(self, other): + return not (self == other) + + +all_structs.append(getOptions_result) +getOptions_result.thrift_spec = ( + ( + 0, + TType.MAP, + "success", + (TType.STRING, "UTF8", TType.STRING, "UTF8", False), + None, + ), # 0 +) + + +class getCpuProfile_args: + """ + Attributes: + - profileDurationInSec + + """ + + def __init__( + self, + profileDurationInSec=None, + ): + self.profileDurationInSec = profileDurationInSec + + def read(self, iprot): + if ( + iprot._fast_decode is not None + and isinstance(iprot.trans, TTransport.CReadableTransport) + and self.thrift_spec is not None + ): + iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec]) + return + iprot.readStructBegin() + while True: + (fname, ftype, fid) = iprot.readFieldBegin() + if ftype == TType.STOP: + break + if fid == 1: + if ftype == TType.I32: + self.profileDurationInSec = iprot.readI32() + else: + iprot.skip(ftype) + else: + iprot.skip(ftype) + iprot.readFieldEnd() + iprot.readStructEnd() + + def write(self, oprot): + if oprot._fast_encode is not None and self.thrift_spec is not None: + oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec])) + return + oprot.writeStructBegin("getCpuProfile_args") + if self.profileDurationInSec is not None: + oprot.writeFieldBegin("profileDurationInSec", TType.I32, 1) + oprot.writeI32(self.profileDurationInSec) + oprot.writeFieldEnd() + oprot.writeFieldStop() + oprot.writeStructEnd() + + def validate(self): + return + + def __repr__(self): + L = ["{}={!r}".format(key, value) for key, value in self.__dict__.items()] + return "{}({})".format(self.__class__.__name__, ", ".join(L)) + + def __eq__(self, other): + return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ + + def __ne__(self, other): + return not (self == other) + + +all_structs.append(getCpuProfile_args) +getCpuProfile_args.thrift_spec = ( + None, # 0 + ( + 1, + TType.I32, + "profileDurationInSec", + None, + None, + ), # 1 +) + + +class getCpuProfile_result: + """ + Attributes: + - success + + """ + + def __init__( + self, + success=None, + ): + self.success = success + + def read(self, iprot): + if ( + iprot._fast_decode is not None + and isinstance(iprot.trans, TTransport.CReadableTransport) + and self.thrift_spec is not None + ): + iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec]) + return + iprot.readStructBegin() + while True: + (fname, ftype, fid) = iprot.readFieldBegin() + if ftype == TType.STOP: + break + if fid == 0: + if ftype == TType.STRING: + self.success = ( + iprot.readString().decode("utf-8", errors="replace") if sys.version_info[0] == 2 else iprot.readString() + ) + else: + iprot.skip(ftype) + else: + iprot.skip(ftype) + iprot.readFieldEnd() + iprot.readStructEnd() + + def write(self, oprot): + if oprot._fast_encode is not None and self.thrift_spec is not None: + oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec])) + return + oprot.writeStructBegin("getCpuProfile_result") + if self.success is not None: + oprot.writeFieldBegin("success", TType.STRING, 0) + oprot.writeString(self.success.encode("utf-8") if sys.version_info[0] == 2 else self.success) + oprot.writeFieldEnd() + oprot.writeFieldStop() + oprot.writeStructEnd() + + def validate(self): + return + + def __repr__(self): + L = ["{}={!r}".format(key, value) for key, value in self.__dict__.items()] + return "{}({})".format(self.__class__.__name__, ", ".join(L)) + + def __eq__(self, other): + return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ + + def __ne__(self, other): + return not (self == other) + + +all_structs.append(getCpuProfile_result) +getCpuProfile_result.thrift_spec = ( + ( + 0, + TType.STRING, + "success", + "UTF8", + None, + ), # 0 +) + + +class aliveSince_args: + def read(self, iprot): + if ( + iprot._fast_decode is not None + and isinstance(iprot.trans, TTransport.CReadableTransport) + and self.thrift_spec is not None + ): + iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec]) + return + iprot.readStructBegin() + while True: + (fname, ftype, fid) = iprot.readFieldBegin() + if ftype == TType.STOP: + break + else: + iprot.skip(ftype) + iprot.readFieldEnd() + iprot.readStructEnd() + + def write(self, oprot): + if oprot._fast_encode is not None and self.thrift_spec is not None: + oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec])) + return + oprot.writeStructBegin("aliveSince_args") + oprot.writeFieldStop() + oprot.writeStructEnd() + + def validate(self): + return + + def __repr__(self): + L = ["{}={!r}".format(key, value) for key, value in self.__dict__.items()] + return "{}({})".format(self.__class__.__name__, ", ".join(L)) + + def __eq__(self, other): + return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ + + def __ne__(self, other): + return not (self == other) + + +all_structs.append(aliveSince_args) +aliveSince_args.thrift_spec = () + + +class aliveSince_result: + """ + Attributes: + - success + + """ + + def __init__( + self, + success=None, + ): + self.success = success + + def read(self, iprot): + if ( + iprot._fast_decode is not None + and isinstance(iprot.trans, TTransport.CReadableTransport) + and self.thrift_spec is not None + ): + iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec]) + return + iprot.readStructBegin() + while True: + (fname, ftype, fid) = iprot.readFieldBegin() + if ftype == TType.STOP: + break + if fid == 0: + if ftype == TType.I64: + self.success = iprot.readI64() + else: + iprot.skip(ftype) + else: + iprot.skip(ftype) + iprot.readFieldEnd() + iprot.readStructEnd() + + def write(self, oprot): + if oprot._fast_encode is not None and self.thrift_spec is not None: + oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec])) + return + oprot.writeStructBegin("aliveSince_result") + if self.success is not None: + oprot.writeFieldBegin("success", TType.I64, 0) + oprot.writeI64(self.success) + oprot.writeFieldEnd() + oprot.writeFieldStop() + oprot.writeStructEnd() + + def validate(self): + return + + def __repr__(self): + L = ["{}={!r}".format(key, value) for key, value in self.__dict__.items()] + return "{}({})".format(self.__class__.__name__, ", ".join(L)) + + def __eq__(self, other): + return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ + + def __ne__(self, other): + return not (self == other) + + +all_structs.append(aliveSince_result) +aliveSince_result.thrift_spec = ( + ( + 0, + TType.I64, + "success", + None, + None, + ), # 0 +) + + +class reinitialize_args: + def read(self, iprot): + if ( + iprot._fast_decode is not None + and isinstance(iprot.trans, TTransport.CReadableTransport) + and self.thrift_spec is not None + ): + iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec]) + return + iprot.readStructBegin() + while True: + (fname, ftype, fid) = iprot.readFieldBegin() + if ftype == TType.STOP: + break + else: + iprot.skip(ftype) + iprot.readFieldEnd() + iprot.readStructEnd() + + def write(self, oprot): + if oprot._fast_encode is not None and self.thrift_spec is not None: + oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec])) + return + oprot.writeStructBegin("reinitialize_args") + oprot.writeFieldStop() + oprot.writeStructEnd() + + def validate(self): + return + + def __repr__(self): + L = ["{}={!r}".format(key, value) for key, value in self.__dict__.items()] + return "{}({})".format(self.__class__.__name__, ", ".join(L)) + + def __eq__(self, other): + return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ + + def __ne__(self, other): + return not (self == other) + + +all_structs.append(reinitialize_args) +reinitialize_args.thrift_spec = () + + +class shutdown_args: + def read(self, iprot): + if ( + iprot._fast_decode is not None + and isinstance(iprot.trans, TTransport.CReadableTransport) + and self.thrift_spec is not None + ): + iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec]) + return + iprot.readStructBegin() + while True: + (fname, ftype, fid) = iprot.readFieldBegin() + if ftype == TType.STOP: + break + else: + iprot.skip(ftype) + iprot.readFieldEnd() + iprot.readStructEnd() + + def write(self, oprot): + if oprot._fast_encode is not None and self.thrift_spec is not None: + oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec])) + return + oprot.writeStructBegin("shutdown_args") + oprot.writeFieldStop() + oprot.writeStructEnd() + + def validate(self): + return + + def __repr__(self): + L = ["{}={!r}".format(key, value) for key, value in self.__dict__.items()] + return "{}({})".format(self.__class__.__name__, ", ".join(L)) + + def __eq__(self, other): + return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ + + def __ne__(self, other): + return not (self == other) + + +all_structs.append(shutdown_args) +shutdown_args.thrift_spec = () +fix_spec(all_structs) +del all_structs diff --git a/vendor/fb303/__init__.py b/vendor/fb303/__init__.py new file mode 100644 index 0000000000..398041beaf --- /dev/null +++ b/vendor/fb303/__init__.py @@ -0,0 +1,18 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +__all__ = ["ttypes", "constants", "FacebookService"] diff --git a/vendor/fb303/constants.py b/vendor/fb303/constants.py new file mode 100644 index 0000000000..3361fd3904 --- /dev/null +++ b/vendor/fb303/constants.py @@ -0,0 +1,26 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# +# Autogenerated by Thrift Compiler (0.16.0) +# +# DO NOT EDIT UNLESS YOU ARE SURE THAT YOU KNOW WHAT YOU ARE DOING +# +# options string: py +# + + + diff --git a/vendor/fb303/ttypes.py b/vendor/fb303/ttypes.py new file mode 100644 index 0000000000..bf2c4e2955 --- /dev/null +++ b/vendor/fb303/ttypes.py @@ -0,0 +1,64 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# +# Autogenerated by Thrift Compiler (0.16.0) +# +# DO NOT EDIT UNLESS YOU ARE SURE THAT YOU KNOW WHAT YOU ARE DOING +# +# options string: py +# + + +from thrift.TRecursive import fix_spec + +all_structs = [] + + +class fb_status: + """ + Common status reporting mechanism across all services + + """ + + DEAD = 0 + STARTING = 1 + ALIVE = 2 + STOPPING = 3 + STOPPED = 4 + WARNING = 5 + + _VALUES_TO_NAMES = { + 0: "DEAD", + 1: "STARTING", + 2: "ALIVE", + 3: "STOPPING", + 4: "STOPPED", + 5: "WARNING", + } + + _NAMES_TO_VALUES = { + "DEAD": 0, + "STARTING": 1, + "ALIVE": 2, + "STOPPING": 3, + "STOPPED": 4, + "WARNING": 5, + } + + +fix_spec(all_structs) +del all_structs diff --git a/vendor/hive_metastore/ThriftHiveMetastore.py b/vendor/hive_metastore/ThriftHiveMetastore.py new file mode 100644 index 0000000000..4d25c087c7 --- /dev/null +++ b/vendor/hive_metastore/ThriftHiveMetastore.py @@ -0,0 +1,72960 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# +# Autogenerated by Thrift Compiler (0.16.0) +# +# DO NOT EDIT UNLESS YOU ARE SURE THAT YOU KNOW WHAT YOU ARE DOING +# +# options string: py +# + +import logging +import sys + +import fb303.FacebookService +from thrift.Thrift import ( + TApplicationException, + TMessageType, + TProcessor, + TType, +) +from thrift.transport import TTransport +from thrift.TRecursive import fix_spec + +from .ttypes import * + +all_structs = [] + + +class Iface(fb303.FacebookService.Iface): + """ + This interface is live. + + """ + + def getMetaConf(self, key): + """ + Parameters: + - key + + """ + pass + + def setMetaConf(self, key, value): + """ + Parameters: + - key + - value + + """ + pass + + def create_catalog(self, catalog): + """ + Parameters: + - catalog + + """ + pass + + def alter_catalog(self, rqst): + """ + Parameters: + - rqst + + """ + pass + + def get_catalog(self, catName): + """ + Parameters: + - catName + + """ + pass + + def get_catalogs(self): + pass + + def drop_catalog(self, catName): + """ + Parameters: + - catName + + """ + pass + + def create_database(self, database): + """ + Parameters: + - database + + """ + pass + + def get_database(self, name): + """ + Parameters: + - name + + """ + pass + + def get_database_req(self, request): + """ + Parameters: + - request + + """ + pass + + def drop_database(self, name, deleteData, cascade): + """ + Parameters: + - name + - deleteData + - cascade + + """ + pass + + def drop_database_req(self, req): + """ + Parameters: + - req + + """ + pass + + def get_databases(self, pattern): + """ + Parameters: + - pattern + + """ + pass + + def get_all_databases(self): + pass + + def alter_database(self, dbname, db): + """ + Parameters: + - dbname + - db + + """ + pass + + def create_dataconnector(self, connector): + """ + Parameters: + - connector + + """ + pass + + def get_dataconnector_req(self, request): + """ + Parameters: + - request + + """ + pass + + def drop_dataconnector(self, name, ifNotExists, checkReferences): + """ + Parameters: + - name + - ifNotExists + - checkReferences + + """ + pass + + def get_dataconnectors(self): + pass + + def alter_dataconnector(self, name, connector): + """ + Parameters: + - name + - connector + + """ + pass + + def get_type(self, name): + """ + Parameters: + - name + + """ + pass + + def create_type(self, type): + """ + Parameters: + - type + + """ + pass + + def drop_type(self, type): + """ + Parameters: + - type + + """ + pass + + def get_type_all(self, name): + """ + Parameters: + - name + + """ + pass + + def get_fields(self, db_name, table_name): + """ + Parameters: + - db_name + - table_name + + """ + pass + + def get_fields_with_environment_context(self, db_name, table_name, environment_context): + """ + Parameters: + - db_name + - table_name + - environment_context + + """ + pass + + def get_fields_req(self, req): + """ + Parameters: + - req + + """ + pass + + def get_schema(self, db_name, table_name): + """ + Parameters: + - db_name + - table_name + + """ + pass + + def get_schema_with_environment_context(self, db_name, table_name, environment_context): + """ + Parameters: + - db_name + - table_name + - environment_context + + """ + pass + + def get_schema_req(self, req): + """ + Parameters: + - req + + """ + pass + + def create_table(self, tbl): + """ + Parameters: + - tbl + + """ + pass + + def create_table_with_environment_context(self, tbl, environment_context): + """ + Parameters: + - tbl + - environment_context + + """ + pass + + def create_table_with_constraints( + self, tbl, primaryKeys, foreignKeys, uniqueConstraints, notNullConstraints, defaultConstraints, checkConstraints + ): + """ + Parameters: + - tbl + - primaryKeys + - foreignKeys + - uniqueConstraints + - notNullConstraints + - defaultConstraints + - checkConstraints + + """ + pass + + def create_table_req(self, request): + """ + Parameters: + - request + + """ + pass + + def drop_constraint(self, req): + """ + Parameters: + - req + + """ + pass + + def add_primary_key(self, req): + """ + Parameters: + - req + + """ + pass + + def add_foreign_key(self, req): + """ + Parameters: + - req + + """ + pass + + def add_unique_constraint(self, req): + """ + Parameters: + - req + + """ + pass + + def add_not_null_constraint(self, req): + """ + Parameters: + - req + + """ + pass + + def add_default_constraint(self, req): + """ + Parameters: + - req + + """ + pass + + def add_check_constraint(self, req): + """ + Parameters: + - req + + """ + pass + + def translate_table_dryrun(self, request): + """ + Parameters: + - request + + """ + pass + + def drop_table(self, dbname, name, deleteData): + """ + Parameters: + - dbname + - name + - deleteData + + """ + pass + + def drop_table_with_environment_context(self, dbname, name, deleteData, environment_context): + """ + Parameters: + - dbname + - name + - deleteData + - environment_context + + """ + pass + + def truncate_table(self, dbName, tableName, partNames): + """ + Parameters: + - dbName + - tableName + - partNames + + """ + pass + + def truncate_table_req(self, req): + """ + Parameters: + - req + + """ + pass + + def get_tables(self, db_name, pattern): + """ + Parameters: + - db_name + - pattern + + """ + pass + + def get_tables_by_type(self, db_name, pattern, tableType): + """ + Parameters: + - db_name + - pattern + - tableType + + """ + pass + + def get_all_materialized_view_objects_for_rewriting(self): + pass + + def get_materialized_views_for_rewriting(self, db_name): + """ + Parameters: + - db_name + + """ + pass + + def get_table_meta(self, db_patterns, tbl_patterns, tbl_types): + """ + Parameters: + - db_patterns + - tbl_patterns + - tbl_types + + """ + pass + + def get_all_tables(self, db_name): + """ + Parameters: + - db_name + + """ + pass + + def get_table(self, dbname, tbl_name): + """ + Parameters: + - dbname + - tbl_name + + """ + pass + + def get_table_objects_by_name(self, dbname, tbl_names): + """ + Parameters: + - dbname + - tbl_names + + """ + pass + + def get_tables_ext(self, req): + """ + Parameters: + - req + + """ + pass + + def get_table_req(self, req): + """ + Parameters: + - req + + """ + pass + + def get_table_objects_by_name_req(self, req): + """ + Parameters: + - req + + """ + pass + + def get_materialization_invalidation_info(self, creation_metadata, validTxnList): + """ + Parameters: + - creation_metadata + - validTxnList + + """ + pass + + def update_creation_metadata(self, catName, dbname, tbl_name, creation_metadata): + """ + Parameters: + - catName + - dbname + - tbl_name + - creation_metadata + + """ + pass + + def get_table_names_by_filter(self, dbname, filter, max_tables): + """ + Parameters: + - dbname + - filter + - max_tables + + """ + pass + + def alter_table(self, dbname, tbl_name, new_tbl): + """ + Parameters: + - dbname + - tbl_name + - new_tbl + + """ + pass + + def alter_table_with_environment_context(self, dbname, tbl_name, new_tbl, environment_context): + """ + Parameters: + - dbname + - tbl_name + - new_tbl + - environment_context + + """ + pass + + def alter_table_with_cascade(self, dbname, tbl_name, new_tbl, cascade): + """ + Parameters: + - dbname + - tbl_name + - new_tbl + - cascade + + """ + pass + + def alter_table_req(self, req): + """ + Parameters: + - req + + """ + pass + + def add_partition(self, new_part): + """ + Parameters: + - new_part + + """ + pass + + def add_partition_with_environment_context(self, new_part, environment_context): + """ + Parameters: + - new_part + - environment_context + + """ + pass + + def add_partitions(self, new_parts): + """ + Parameters: + - new_parts + + """ + pass + + def add_partitions_pspec(self, new_parts): + """ + Parameters: + - new_parts + + """ + pass + + def append_partition(self, db_name, tbl_name, part_vals): + """ + Parameters: + - db_name + - tbl_name + - part_vals + + """ + pass + + def add_partitions_req(self, request): + """ + Parameters: + - request + + """ + pass + + def append_partition_with_environment_context(self, db_name, tbl_name, part_vals, environment_context): + """ + Parameters: + - db_name + - tbl_name + - part_vals + - environment_context + + """ + pass + + def append_partition_by_name(self, db_name, tbl_name, part_name): + """ + Parameters: + - db_name + - tbl_name + - part_name + + """ + pass + + def append_partition_by_name_with_environment_context(self, db_name, tbl_name, part_name, environment_context): + """ + Parameters: + - db_name + - tbl_name + - part_name + - environment_context + + """ + pass + + def drop_partition(self, db_name, tbl_name, part_vals, deleteData): + """ + Parameters: + - db_name + - tbl_name + - part_vals + - deleteData + + """ + pass + + def drop_partition_with_environment_context(self, db_name, tbl_name, part_vals, deleteData, environment_context): + """ + Parameters: + - db_name + - tbl_name + - part_vals + - deleteData + - environment_context + + """ + pass + + def drop_partition_by_name(self, db_name, tbl_name, part_name, deleteData): + """ + Parameters: + - db_name + - tbl_name + - part_name + - deleteData + + """ + pass + + def drop_partition_by_name_with_environment_context(self, db_name, tbl_name, part_name, deleteData, environment_context): + """ + Parameters: + - db_name + - tbl_name + - part_name + - deleteData + - environment_context + + """ + pass + + def drop_partitions_req(self, req): + """ + Parameters: + - req + + """ + pass + + def get_partition(self, db_name, tbl_name, part_vals): + """ + Parameters: + - db_name + - tbl_name + - part_vals + + """ + pass + + def get_partition_req(self, req): + """ + Parameters: + - req + + """ + pass + + def exchange_partition(self, partitionSpecs, source_db, source_table_name, dest_db, dest_table_name): + """ + Parameters: + - partitionSpecs + - source_db + - source_table_name + - dest_db + - dest_table_name + + """ + pass + + def exchange_partitions(self, partitionSpecs, source_db, source_table_name, dest_db, dest_table_name): + """ + Parameters: + - partitionSpecs + - source_db + - source_table_name + - dest_db + - dest_table_name + + """ + pass + + def get_partition_with_auth(self, db_name, tbl_name, part_vals, user_name, group_names): + """ + Parameters: + - db_name + - tbl_name + - part_vals + - user_name + - group_names + + """ + pass + + def get_partition_by_name(self, db_name, tbl_name, part_name): + """ + Parameters: + - db_name + - tbl_name + - part_name + + """ + pass + + def get_partitions(self, db_name, tbl_name, max_parts): + """ + Parameters: + - db_name + - tbl_name + - max_parts + + """ + pass + + def get_partitions_req(self, req): + """ + Parameters: + - req + + """ + pass + + def get_partitions_with_auth(self, db_name, tbl_name, max_parts, user_name, group_names): + """ + Parameters: + - db_name + - tbl_name + - max_parts + - user_name + - group_names + + """ + pass + + def get_partitions_pspec(self, db_name, tbl_name, max_parts): + """ + Parameters: + - db_name + - tbl_name + - max_parts + + """ + pass + + def get_partition_names(self, db_name, tbl_name, max_parts): + """ + Parameters: + - db_name + - tbl_name + - max_parts + + """ + pass + + def get_partition_values(self, request): + """ + Parameters: + - request + + """ + pass + + def get_partitions_ps(self, db_name, tbl_name, part_vals, max_parts): + """ + Parameters: + - db_name + - tbl_name + - part_vals + - max_parts + + """ + pass + + def get_partitions_ps_with_auth(self, db_name, tbl_name, part_vals, max_parts, user_name, group_names): + """ + Parameters: + - db_name + - tbl_name + - part_vals + - max_parts + - user_name + - group_names + + """ + pass + + def get_partitions_ps_with_auth_req(self, req): + """ + Parameters: + - req + + """ + pass + + def get_partition_names_ps(self, db_name, tbl_name, part_vals, max_parts): + """ + Parameters: + - db_name + - tbl_name + - part_vals + - max_parts + + """ + pass + + def get_partition_names_ps_req(self, req): + """ + Parameters: + - req + + """ + pass + + def get_partition_names_req(self, req): + """ + Parameters: + - req + + """ + pass + + def get_partitions_by_filter(self, db_name, tbl_name, filter, max_parts): + """ + Parameters: + - db_name + - tbl_name + - filter + - max_parts + + """ + pass + + def get_part_specs_by_filter(self, db_name, tbl_name, filter, max_parts): + """ + Parameters: + - db_name + - tbl_name + - filter + - max_parts + + """ + pass + + def get_partitions_by_expr(self, req): + """ + Parameters: + - req + + """ + pass + + def get_partitions_spec_by_expr(self, req): + """ + Parameters: + - req + + """ + pass + + def get_num_partitions_by_filter(self, db_name, tbl_name, filter): + """ + Parameters: + - db_name + - tbl_name + - filter + + """ + pass + + def get_partitions_by_names(self, db_name, tbl_name, names): + """ + Parameters: + - db_name + - tbl_name + - names + + """ + pass + + def get_partitions_by_names_req(self, req): + """ + Parameters: + - req + + """ + pass + + def alter_partition(self, db_name, tbl_name, new_part): + """ + Parameters: + - db_name + - tbl_name + - new_part + + """ + pass + + def alter_partitions(self, db_name, tbl_name, new_parts): + """ + Parameters: + - db_name + - tbl_name + - new_parts + + """ + pass + + def alter_partitions_with_environment_context(self, db_name, tbl_name, new_parts, environment_context): + """ + Parameters: + - db_name + - tbl_name + - new_parts + - environment_context + + """ + pass + + def alter_partitions_req(self, req): + """ + Parameters: + - req + + """ + pass + + def alter_partition_with_environment_context(self, db_name, tbl_name, new_part, environment_context): + """ + Parameters: + - db_name + - tbl_name + - new_part + - environment_context + + """ + pass + + def rename_partition(self, db_name, tbl_name, part_vals, new_part): + """ + Parameters: + - db_name + - tbl_name + - part_vals + - new_part + + """ + pass + + def rename_partition_req(self, req): + """ + Parameters: + - req + + """ + pass + + def partition_name_has_valid_characters(self, part_vals, throw_exception): + """ + Parameters: + - part_vals + - throw_exception + + """ + pass + + def get_config_value(self, name, defaultValue): + """ + Parameters: + - name + - defaultValue + + """ + pass + + def partition_name_to_vals(self, part_name): + """ + Parameters: + - part_name + + """ + pass + + def partition_name_to_spec(self, part_name): + """ + Parameters: + - part_name + + """ + pass + + def markPartitionForEvent(self, db_name, tbl_name, part_vals, eventType): + """ + Parameters: + - db_name + - tbl_name + - part_vals + - eventType + + """ + pass + + def isPartitionMarkedForEvent(self, db_name, tbl_name, part_vals, eventType): + """ + Parameters: + - db_name + - tbl_name + - part_vals + - eventType + + """ + pass + + def get_primary_keys(self, request): + """ + Parameters: + - request + + """ + pass + + def get_foreign_keys(self, request): + """ + Parameters: + - request + + """ + pass + + def get_unique_constraints(self, request): + """ + Parameters: + - request + + """ + pass + + def get_not_null_constraints(self, request): + """ + Parameters: + - request + + """ + pass + + def get_default_constraints(self, request): + """ + Parameters: + - request + + """ + pass + + def get_check_constraints(self, request): + """ + Parameters: + - request + + """ + pass + + def get_all_table_constraints(self, request): + """ + Parameters: + - request + + """ + pass + + def update_table_column_statistics(self, stats_obj): + """ + Parameters: + - stats_obj + + """ + pass + + def update_partition_column_statistics(self, stats_obj): + """ + Parameters: + - stats_obj + + """ + pass + + def update_table_column_statistics_req(self, req): + """ + Parameters: + - req + + """ + pass + + def update_partition_column_statistics_req(self, req): + """ + Parameters: + - req + + """ + pass + + def update_transaction_statistics(self, req): + """ + Parameters: + - req + + """ + pass + + def get_table_column_statistics(self, db_name, tbl_name, col_name): + """ + Parameters: + - db_name + - tbl_name + - col_name + + """ + pass + + def get_partition_column_statistics(self, db_name, tbl_name, part_name, col_name): + """ + Parameters: + - db_name + - tbl_name + - part_name + - col_name + + """ + pass + + def get_table_statistics_req(self, request): + """ + Parameters: + - request + + """ + pass + + def get_partitions_statistics_req(self, request): + """ + Parameters: + - request + + """ + pass + + def get_aggr_stats_for(self, request): + """ + Parameters: + - request + + """ + pass + + def set_aggr_stats_for(self, request): + """ + Parameters: + - request + + """ + pass + + def delete_partition_column_statistics(self, db_name, tbl_name, part_name, col_name, engine): + """ + Parameters: + - db_name + - tbl_name + - part_name + - col_name + - engine + + """ + pass + + def delete_table_column_statistics(self, db_name, tbl_name, col_name, engine): + """ + Parameters: + - db_name + - tbl_name + - col_name + - engine + + """ + pass + + def create_function(self, func): + """ + Parameters: + - func + + """ + pass + + def drop_function(self, dbName, funcName): + """ + Parameters: + - dbName + - funcName + + """ + pass + + def alter_function(self, dbName, funcName, newFunc): + """ + Parameters: + - dbName + - funcName + - newFunc + + """ + pass + + def get_functions(self, dbName, pattern): + """ + Parameters: + - dbName + - pattern + + """ + pass + + def get_function(self, dbName, funcName): + """ + Parameters: + - dbName + - funcName + + """ + pass + + def get_all_functions(self): + pass + + def create_role(self, role): + """ + Parameters: + - role + + """ + pass + + def drop_role(self, role_name): + """ + Parameters: + - role_name + + """ + pass + + def get_role_names(self): + pass + + def grant_role(self, role_name, principal_name, principal_type, grantor, grantorType, grant_option): + """ + Parameters: + - role_name + - principal_name + - principal_type + - grantor + - grantorType + - grant_option + + """ + pass + + def revoke_role(self, role_name, principal_name, principal_type): + """ + Parameters: + - role_name + - principal_name + - principal_type + + """ + pass + + def list_roles(self, principal_name, principal_type): + """ + Parameters: + - principal_name + - principal_type + + """ + pass + + def grant_revoke_role(self, request): + """ + Parameters: + - request + + """ + pass + + def get_principals_in_role(self, request): + """ + Parameters: + - request + + """ + pass + + def get_role_grants_for_principal(self, request): + """ + Parameters: + - request + + """ + pass + + def get_privilege_set(self, hiveObject, user_name, group_names): + """ + Parameters: + - hiveObject + - user_name + - group_names + + """ + pass + + def list_privileges(self, principal_name, principal_type, hiveObject): + """ + Parameters: + - principal_name + - principal_type + - hiveObject + + """ + pass + + def grant_privileges(self, privileges): + """ + Parameters: + - privileges + + """ + pass + + def revoke_privileges(self, privileges): + """ + Parameters: + - privileges + + """ + pass + + def grant_revoke_privileges(self, request): + """ + Parameters: + - request + + """ + pass + + def refresh_privileges(self, objToRefresh, authorizer, grantRequest): + """ + Parameters: + - objToRefresh + - authorizer + - grantRequest + + """ + pass + + def set_ugi(self, user_name, group_names): + """ + Parameters: + - user_name + - group_names + + """ + pass + + def get_delegation_token(self, token_owner, renewer_kerberos_principal_name): + """ + Parameters: + - token_owner + - renewer_kerberos_principal_name + + """ + pass + + def renew_delegation_token(self, token_str_form): + """ + Parameters: + - token_str_form + + """ + pass + + def cancel_delegation_token(self, token_str_form): + """ + Parameters: + - token_str_form + + """ + pass + + def add_token(self, token_identifier, delegation_token): + """ + Parameters: + - token_identifier + - delegation_token + + """ + pass + + def remove_token(self, token_identifier): + """ + Parameters: + - token_identifier + + """ + pass + + def get_token(self, token_identifier): + """ + Parameters: + - token_identifier + + """ + pass + + def get_all_token_identifiers(self): + pass + + def add_master_key(self, key): + """ + Parameters: + - key + + """ + pass + + def update_master_key(self, seq_number, key): + """ + Parameters: + - seq_number + - key + + """ + pass + + def remove_master_key(self, key_seq): + """ + Parameters: + - key_seq + + """ + pass + + def get_master_keys(self): + pass + + def get_open_txns(self): + pass + + def get_open_txns_info(self): + pass + + def open_txns(self, rqst): + """ + Parameters: + - rqst + + """ + pass + + def abort_txn(self, rqst): + """ + Parameters: + - rqst + + """ + pass + + def abort_txns(self, rqst): + """ + Parameters: + - rqst + + """ + pass + + def commit_txn(self, rqst): + """ + Parameters: + - rqst + + """ + pass + + def get_latest_txnid_in_conflict(self, txnId): + """ + Parameters: + - txnId + + """ + pass + + def repl_tbl_writeid_state(self, rqst): + """ + Parameters: + - rqst + + """ + pass + + def get_valid_write_ids(self, rqst): + """ + Parameters: + - rqst + + """ + pass + + def allocate_table_write_ids(self, rqst): + """ + Parameters: + - rqst + + """ + pass + + def get_max_allocated_table_write_id(self, rqst): + """ + Parameters: + - rqst + + """ + pass + + def seed_write_id(self, rqst): + """ + Parameters: + - rqst + + """ + pass + + def seed_txn_id(self, rqst): + """ + Parameters: + - rqst + + """ + pass + + def lock(self, rqst): + """ + Parameters: + - rqst + + """ + pass + + def check_lock(self, rqst): + """ + Parameters: + - rqst + + """ + pass + + def unlock(self, rqst): + """ + Parameters: + - rqst + + """ + pass + + def show_locks(self, rqst): + """ + Parameters: + - rqst + + """ + pass + + def heartbeat(self, ids): + """ + Parameters: + - ids + + """ + pass + + def heartbeat_txn_range(self, txns): + """ + Parameters: + - txns + + """ + pass + + def compact(self, rqst): + """ + Parameters: + - rqst + + """ + pass + + def compact2(self, rqst): + """ + Parameters: + - rqst + + """ + pass + + def show_compact(self, rqst): + """ + Parameters: + - rqst + + """ + pass + + def add_dynamic_partitions(self, rqst): + """ + Parameters: + - rqst + + """ + pass + + def find_next_compact(self, workerId): + """ + Parameters: + - workerId + + """ + pass + + def find_next_compact2(self, rqst): + """ + Parameters: + - rqst + + """ + pass + + def update_compactor_state(self, cr, txn_id): + """ + Parameters: + - cr + - txn_id + + """ + pass + + def find_columns_with_stats(self, cr): + """ + Parameters: + - cr + + """ + pass + + def mark_cleaned(self, cr): + """ + Parameters: + - cr + + """ + pass + + def mark_compacted(self, cr): + """ + Parameters: + - cr + + """ + pass + + def mark_failed(self, cr): + """ + Parameters: + - cr + + """ + pass + + def mark_refused(self, cr): + """ + Parameters: + - cr + + """ + pass + + def update_compaction_metrics_data(self, data): + """ + Parameters: + - data + + """ + pass + + def remove_compaction_metrics_data(self, request): + """ + Parameters: + - request + + """ + pass + + def set_hadoop_jobid(self, jobId, cq_id): + """ + Parameters: + - jobId + - cq_id + + """ + pass + + def get_latest_committed_compaction_info(self, rqst): + """ + Parameters: + - rqst + + """ + pass + + def get_next_notification(self, rqst): + """ + Parameters: + - rqst + + """ + pass + + def get_current_notificationEventId(self): + pass + + def get_notification_events_count(self, rqst): + """ + Parameters: + - rqst + + """ + pass + + def fire_listener_event(self, rqst): + """ + Parameters: + - rqst + + """ + pass + + def flushCache(self): + pass + + def add_write_notification_log(self, rqst): + """ + Parameters: + - rqst + + """ + pass + + def add_write_notification_log_in_batch(self, rqst): + """ + Parameters: + - rqst + + """ + pass + + def cm_recycle(self, request): + """ + Parameters: + - request + + """ + pass + + def get_file_metadata_by_expr(self, req): + """ + Parameters: + - req + + """ + pass + + def get_file_metadata(self, req): + """ + Parameters: + - req + + """ + pass + + def put_file_metadata(self, req): + """ + Parameters: + - req + + """ + pass + + def clear_file_metadata(self, req): + """ + Parameters: + - req + + """ + pass + + def cache_file_metadata(self, req): + """ + Parameters: + - req + + """ + pass + + def get_metastore_db_uuid(self): + pass + + def create_resource_plan(self, request): + """ + Parameters: + - request + + """ + pass + + def get_resource_plan(self, request): + """ + Parameters: + - request + + """ + pass + + def get_active_resource_plan(self, request): + """ + Parameters: + - request + + """ + pass + + def get_all_resource_plans(self, request): + """ + Parameters: + - request + + """ + pass + + def alter_resource_plan(self, request): + """ + Parameters: + - request + + """ + pass + + def validate_resource_plan(self, request): + """ + Parameters: + - request + + """ + pass + + def drop_resource_plan(self, request): + """ + Parameters: + - request + + """ + pass + + def create_wm_trigger(self, request): + """ + Parameters: + - request + + """ + pass + + def alter_wm_trigger(self, request): + """ + Parameters: + - request + + """ + pass + + def drop_wm_trigger(self, request): + """ + Parameters: + - request + + """ + pass + + def get_triggers_for_resourceplan(self, request): + """ + Parameters: + - request + + """ + pass + + def create_wm_pool(self, request): + """ + Parameters: + - request + + """ + pass + + def alter_wm_pool(self, request): + """ + Parameters: + - request + + """ + pass + + def drop_wm_pool(self, request): + """ + Parameters: + - request + + """ + pass + + def create_or_update_wm_mapping(self, request): + """ + Parameters: + - request + + """ + pass + + def drop_wm_mapping(self, request): + """ + Parameters: + - request + + """ + pass + + def create_or_drop_wm_trigger_to_pool_mapping(self, request): + """ + Parameters: + - request + + """ + pass + + def create_ischema(self, schema): + """ + Parameters: + - schema + + """ + pass + + def alter_ischema(self, rqst): + """ + Parameters: + - rqst + + """ + pass + + def get_ischema(self, name): + """ + Parameters: + - name + + """ + pass + + def drop_ischema(self, name): + """ + Parameters: + - name + + """ + pass + + def add_schema_version(self, schemaVersion): + """ + Parameters: + - schemaVersion + + """ + pass + + def get_schema_version(self, schemaVersion): + """ + Parameters: + - schemaVersion + + """ + pass + + def get_schema_latest_version(self, schemaName): + """ + Parameters: + - schemaName + + """ + pass + + def get_schema_all_versions(self, schemaName): + """ + Parameters: + - schemaName + + """ + pass + + def drop_schema_version(self, schemaVersion): + """ + Parameters: + - schemaVersion + + """ + pass + + def get_schemas_by_cols(self, rqst): + """ + Parameters: + - rqst + + """ + pass + + def map_schema_version_to_serde(self, rqst): + """ + Parameters: + - rqst + + """ + pass + + def set_schema_version_state(self, rqst): + """ + Parameters: + - rqst + + """ + pass + + def add_serde(self, serde): + """ + Parameters: + - serde + + """ + pass + + def get_serde(self, rqst): + """ + Parameters: + - rqst + + """ + pass + + def get_lock_materialization_rebuild(self, dbName, tableName, txnId): + """ + Parameters: + - dbName + - tableName + - txnId + + """ + pass + + def heartbeat_lock_materialization_rebuild(self, dbName, tableName, txnId): + """ + Parameters: + - dbName + - tableName + - txnId + + """ + pass + + def add_runtime_stats(self, stat): + """ + Parameters: + - stat + + """ + pass + + def get_runtime_stats(self, rqst): + """ + Parameters: + - rqst + + """ + pass + + def get_partitions_with_specs(self, request): + """ + Parameters: + - request + + """ + pass + + def scheduled_query_poll(self, request): + """ + Parameters: + - request + + """ + pass + + def scheduled_query_maintenance(self, request): + """ + Parameters: + - request + + """ + pass + + def scheduled_query_progress(self, info): + """ + Parameters: + - info + + """ + pass + + def get_scheduled_query(self, scheduleKey): + """ + Parameters: + - scheduleKey + + """ + pass + + def add_replication_metrics(self, replicationMetricList): + """ + Parameters: + - replicationMetricList + + """ + pass + + def get_replication_metrics(self, rqst): + """ + Parameters: + - rqst + + """ + pass + + def get_open_txns_req(self, getOpenTxnsRequest): + """ + Parameters: + - getOpenTxnsRequest + + """ + pass + + def create_stored_procedure(self, proc): + """ + Parameters: + - proc + + """ + pass + + def get_stored_procedure(self, request): + """ + Parameters: + - request + + """ + pass + + def drop_stored_procedure(self, request): + """ + Parameters: + - request + + """ + pass + + def get_all_stored_procedures(self, request): + """ + Parameters: + - request + + """ + pass + + def find_package(self, request): + """ + Parameters: + - request + + """ + pass + + def add_package(self, request): + """ + Parameters: + - request + + """ + pass + + def get_all_packages(self, request): + """ + Parameters: + - request + + """ + pass + + def drop_package(self, request): + """ + Parameters: + - request + + """ + pass + + def get_all_write_event_info(self, request): + """ + Parameters: + - request + + """ + pass + + +class Client(fb303.FacebookService.Client, Iface): + """ + This interface is live. + + """ + + def __init__(self, iprot, oprot=None): + fb303.FacebookService.Client.__init__(self, iprot, oprot) + + def getMetaConf(self, key): + """ + Parameters: + - key + + """ + self.send_getMetaConf(key) + return self.recv_getMetaConf() + + def send_getMetaConf(self, key): + self._oprot.writeMessageBegin("getMetaConf", TMessageType.CALL, self._seqid) + args = getMetaConf_args() + args.key = key + args.write(self._oprot) + self._oprot.writeMessageEnd() + self._oprot.trans.flush() + + def recv_getMetaConf(self): + iprot = self._iprot + (fname, mtype, rseqid) = iprot.readMessageBegin() + if mtype == TMessageType.EXCEPTION: + x = TApplicationException() + x.read(iprot) + iprot.readMessageEnd() + raise x + result = getMetaConf_result() + result.read(iprot) + iprot.readMessageEnd() + if result.success is not None: + return result.success + if result.o1 is not None: + raise result.o1 + raise TApplicationException(TApplicationException.MISSING_RESULT, "getMetaConf failed: unknown result") + + def setMetaConf(self, key, value): + """ + Parameters: + - key + - value + + """ + self.send_setMetaConf(key, value) + self.recv_setMetaConf() + + def send_setMetaConf(self, key, value): + self._oprot.writeMessageBegin("setMetaConf", TMessageType.CALL, self._seqid) + args = setMetaConf_args() + args.key = key + args.value = value + args.write(self._oprot) + self._oprot.writeMessageEnd() + self._oprot.trans.flush() + + def recv_setMetaConf(self): + iprot = self._iprot + (fname, mtype, rseqid) = iprot.readMessageBegin() + if mtype == TMessageType.EXCEPTION: + x = TApplicationException() + x.read(iprot) + iprot.readMessageEnd() + raise x + result = setMetaConf_result() + result.read(iprot) + iprot.readMessageEnd() + if result.o1 is not None: + raise result.o1 + return + + def create_catalog(self, catalog): + """ + Parameters: + - catalog + + """ + self.send_create_catalog(catalog) + self.recv_create_catalog() + + def send_create_catalog(self, catalog): + self._oprot.writeMessageBegin("create_catalog", TMessageType.CALL, self._seqid) + args = create_catalog_args() + args.catalog = catalog + args.write(self._oprot) + self._oprot.writeMessageEnd() + self._oprot.trans.flush() + + def recv_create_catalog(self): + iprot = self._iprot + (fname, mtype, rseqid) = iprot.readMessageBegin() + if mtype == TMessageType.EXCEPTION: + x = TApplicationException() + x.read(iprot) + iprot.readMessageEnd() + raise x + result = create_catalog_result() + result.read(iprot) + iprot.readMessageEnd() + if result.o1 is not None: + raise result.o1 + if result.o2 is not None: + raise result.o2 + if result.o3 is not None: + raise result.o3 + return + + def alter_catalog(self, rqst): + """ + Parameters: + - rqst + + """ + self.send_alter_catalog(rqst) + self.recv_alter_catalog() + + def send_alter_catalog(self, rqst): + self._oprot.writeMessageBegin("alter_catalog", TMessageType.CALL, self._seqid) + args = alter_catalog_args() + args.rqst = rqst + args.write(self._oprot) + self._oprot.writeMessageEnd() + self._oprot.trans.flush() + + def recv_alter_catalog(self): + iprot = self._iprot + (fname, mtype, rseqid) = iprot.readMessageBegin() + if mtype == TMessageType.EXCEPTION: + x = TApplicationException() + x.read(iprot) + iprot.readMessageEnd() + raise x + result = alter_catalog_result() + result.read(iprot) + iprot.readMessageEnd() + if result.o1 is not None: + raise result.o1 + if result.o2 is not None: + raise result.o2 + if result.o3 is not None: + raise result.o3 + return + + def get_catalog(self, catName): + """ + Parameters: + - catName + + """ + self.send_get_catalog(catName) + return self.recv_get_catalog() + + def send_get_catalog(self, catName): + self._oprot.writeMessageBegin("get_catalog", TMessageType.CALL, self._seqid) + args = get_catalog_args() + args.catName = catName + args.write(self._oprot) + self._oprot.writeMessageEnd() + self._oprot.trans.flush() + + def recv_get_catalog(self): + iprot = self._iprot + (fname, mtype, rseqid) = iprot.readMessageBegin() + if mtype == TMessageType.EXCEPTION: + x = TApplicationException() + x.read(iprot) + iprot.readMessageEnd() + raise x + result = get_catalog_result() + result.read(iprot) + iprot.readMessageEnd() + if result.success is not None: + return result.success + if result.o1 is not None: + raise result.o1 + if result.o2 is not None: + raise result.o2 + raise TApplicationException(TApplicationException.MISSING_RESULT, "get_catalog failed: unknown result") + + def get_catalogs(self): + self.send_get_catalogs() + return self.recv_get_catalogs() + + def send_get_catalogs(self): + self._oprot.writeMessageBegin("get_catalogs", TMessageType.CALL, self._seqid) + args = get_catalogs_args() + args.write(self._oprot) + self._oprot.writeMessageEnd() + self._oprot.trans.flush() + + def recv_get_catalogs(self): + iprot = self._iprot + (fname, mtype, rseqid) = iprot.readMessageBegin() + if mtype == TMessageType.EXCEPTION: + x = TApplicationException() + x.read(iprot) + iprot.readMessageEnd() + raise x + result = get_catalogs_result() + result.read(iprot) + iprot.readMessageEnd() + if result.success is not None: + return result.success + if result.o1 is not None: + raise result.o1 + raise TApplicationException(TApplicationException.MISSING_RESULT, "get_catalogs failed: unknown result") + + def drop_catalog(self, catName): + """ + Parameters: + - catName + + """ + self.send_drop_catalog(catName) + self.recv_drop_catalog() + + def send_drop_catalog(self, catName): + self._oprot.writeMessageBegin("drop_catalog", TMessageType.CALL, self._seqid) + args = drop_catalog_args() + args.catName = catName + args.write(self._oprot) + self._oprot.writeMessageEnd() + self._oprot.trans.flush() + + def recv_drop_catalog(self): + iprot = self._iprot + (fname, mtype, rseqid) = iprot.readMessageBegin() + if mtype == TMessageType.EXCEPTION: + x = TApplicationException() + x.read(iprot) + iprot.readMessageEnd() + raise x + result = drop_catalog_result() + result.read(iprot) + iprot.readMessageEnd() + if result.o1 is not None: + raise result.o1 + if result.o2 is not None: + raise result.o2 + if result.o3 is not None: + raise result.o3 + return + + def create_database(self, database): + """ + Parameters: + - database + + """ + self.send_create_database(database) + self.recv_create_database() + + def send_create_database(self, database): + self._oprot.writeMessageBegin("create_database", TMessageType.CALL, self._seqid) + args = create_database_args() + args.database = database + args.write(self._oprot) + self._oprot.writeMessageEnd() + self._oprot.trans.flush() + + def recv_create_database(self): + iprot = self._iprot + (fname, mtype, rseqid) = iprot.readMessageBegin() + if mtype == TMessageType.EXCEPTION: + x = TApplicationException() + x.read(iprot) + iprot.readMessageEnd() + raise x + result = create_database_result() + result.read(iprot) + iprot.readMessageEnd() + if result.o1 is not None: + raise result.o1 + if result.o2 is not None: + raise result.o2 + if result.o3 is not None: + raise result.o3 + return + + def get_database(self, name): + """ + Parameters: + - name + + """ + self.send_get_database(name) + return self.recv_get_database() + + def send_get_database(self, name): + self._oprot.writeMessageBegin("get_database", TMessageType.CALL, self._seqid) + args = get_database_args() + args.name = name + args.write(self._oprot) + self._oprot.writeMessageEnd() + self._oprot.trans.flush() + + def recv_get_database(self): + iprot = self._iprot + (fname, mtype, rseqid) = iprot.readMessageBegin() + if mtype == TMessageType.EXCEPTION: + x = TApplicationException() + x.read(iprot) + iprot.readMessageEnd() + raise x + result = get_database_result() + result.read(iprot) + iprot.readMessageEnd() + if result.success is not None: + return result.success + if result.o1 is not None: + raise result.o1 + if result.o2 is not None: + raise result.o2 + raise TApplicationException(TApplicationException.MISSING_RESULT, "get_database failed: unknown result") + + def get_database_req(self, request): + """ + Parameters: + - request + + """ + self.send_get_database_req(request) + return self.recv_get_database_req() + + def send_get_database_req(self, request): + self._oprot.writeMessageBegin("get_database_req", TMessageType.CALL, self._seqid) + args = get_database_req_args() + args.request = request + args.write(self._oprot) + self._oprot.writeMessageEnd() + self._oprot.trans.flush() + + def recv_get_database_req(self): + iprot = self._iprot + (fname, mtype, rseqid) = iprot.readMessageBegin() + if mtype == TMessageType.EXCEPTION: + x = TApplicationException() + x.read(iprot) + iprot.readMessageEnd() + raise x + result = get_database_req_result() + result.read(iprot) + iprot.readMessageEnd() + if result.success is not None: + return result.success + if result.o1 is not None: + raise result.o1 + if result.o2 is not None: + raise result.o2 + raise TApplicationException(TApplicationException.MISSING_RESULT, "get_database_req failed: unknown result") + + def drop_database(self, name, deleteData, cascade): + """ + Parameters: + - name + - deleteData + - cascade + + """ + self.send_drop_database(name, deleteData, cascade) + self.recv_drop_database() + + def send_drop_database(self, name, deleteData, cascade): + self._oprot.writeMessageBegin("drop_database", TMessageType.CALL, self._seqid) + args = drop_database_args() + args.name = name + args.deleteData = deleteData + args.cascade = cascade + args.write(self._oprot) + self._oprot.writeMessageEnd() + self._oprot.trans.flush() + + def recv_drop_database(self): + iprot = self._iprot + (fname, mtype, rseqid) = iprot.readMessageBegin() + if mtype == TMessageType.EXCEPTION: + x = TApplicationException() + x.read(iprot) + iprot.readMessageEnd() + raise x + result = drop_database_result() + result.read(iprot) + iprot.readMessageEnd() + if result.o1 is not None: + raise result.o1 + if result.o2 is not None: + raise result.o2 + if result.o3 is not None: + raise result.o3 + return + + def drop_database_req(self, req): + """ + Parameters: + - req + + """ + self.send_drop_database_req(req) + self.recv_drop_database_req() + + def send_drop_database_req(self, req): + self._oprot.writeMessageBegin("drop_database_req", TMessageType.CALL, self._seqid) + args = drop_database_req_args() + args.req = req + args.write(self._oprot) + self._oprot.writeMessageEnd() + self._oprot.trans.flush() + + def recv_drop_database_req(self): + iprot = self._iprot + (fname, mtype, rseqid) = iprot.readMessageBegin() + if mtype == TMessageType.EXCEPTION: + x = TApplicationException() + x.read(iprot) + iprot.readMessageEnd() + raise x + result = drop_database_req_result() + result.read(iprot) + iprot.readMessageEnd() + if result.o1 is not None: + raise result.o1 + if result.o2 is not None: + raise result.o2 + if result.o3 is not None: + raise result.o3 + return + + def get_databases(self, pattern): + """ + Parameters: + - pattern + + """ + self.send_get_databases(pattern) + return self.recv_get_databases() + + def send_get_databases(self, pattern): + self._oprot.writeMessageBegin("get_databases", TMessageType.CALL, self._seqid) + args = get_databases_args() + args.pattern = pattern + args.write(self._oprot) + self._oprot.writeMessageEnd() + self._oprot.trans.flush() + + def recv_get_databases(self): + iprot = self._iprot + (fname, mtype, rseqid) = iprot.readMessageBegin() + if mtype == TMessageType.EXCEPTION: + x = TApplicationException() + x.read(iprot) + iprot.readMessageEnd() + raise x + result = get_databases_result() + result.read(iprot) + iprot.readMessageEnd() + if result.success is not None: + return result.success + if result.o1 is not None: + raise result.o1 + raise TApplicationException(TApplicationException.MISSING_RESULT, "get_databases failed: unknown result") + + def get_all_databases(self): + self.send_get_all_databases() + return self.recv_get_all_databases() + + def send_get_all_databases(self): + self._oprot.writeMessageBegin("get_all_databases", TMessageType.CALL, self._seqid) + args = get_all_databases_args() + args.write(self._oprot) + self._oprot.writeMessageEnd() + self._oprot.trans.flush() + + def recv_get_all_databases(self): + iprot = self._iprot + (fname, mtype, rseqid) = iprot.readMessageBegin() + if mtype == TMessageType.EXCEPTION: + x = TApplicationException() + x.read(iprot) + iprot.readMessageEnd() + raise x + result = get_all_databases_result() + result.read(iprot) + iprot.readMessageEnd() + if result.success is not None: + return result.success + if result.o1 is not None: + raise result.o1 + raise TApplicationException(TApplicationException.MISSING_RESULT, "get_all_databases failed: unknown result") + + def alter_database(self, dbname, db): + """ + Parameters: + - dbname + - db + + """ + self.send_alter_database(dbname, db) + self.recv_alter_database() + + def send_alter_database(self, dbname, db): + self._oprot.writeMessageBegin("alter_database", TMessageType.CALL, self._seqid) + args = alter_database_args() + args.dbname = dbname + args.db = db + args.write(self._oprot) + self._oprot.writeMessageEnd() + self._oprot.trans.flush() + + def recv_alter_database(self): + iprot = self._iprot + (fname, mtype, rseqid) = iprot.readMessageBegin() + if mtype == TMessageType.EXCEPTION: + x = TApplicationException() + x.read(iprot) + iprot.readMessageEnd() + raise x + result = alter_database_result() + result.read(iprot) + iprot.readMessageEnd() + if result.o1 is not None: + raise result.o1 + if result.o2 is not None: + raise result.o2 + return + + def create_dataconnector(self, connector): + """ + Parameters: + - connector + + """ + self.send_create_dataconnector(connector) + self.recv_create_dataconnector() + + def send_create_dataconnector(self, connector): + self._oprot.writeMessageBegin("create_dataconnector", TMessageType.CALL, self._seqid) + args = create_dataconnector_args() + args.connector = connector + args.write(self._oprot) + self._oprot.writeMessageEnd() + self._oprot.trans.flush() + + def recv_create_dataconnector(self): + iprot = self._iprot + (fname, mtype, rseqid) = iprot.readMessageBegin() + if mtype == TMessageType.EXCEPTION: + x = TApplicationException() + x.read(iprot) + iprot.readMessageEnd() + raise x + result = create_dataconnector_result() + result.read(iprot) + iprot.readMessageEnd() + if result.o1 is not None: + raise result.o1 + if result.o2 is not None: + raise result.o2 + if result.o3 is not None: + raise result.o3 + return + + def get_dataconnector_req(self, request): + """ + Parameters: + - request + + """ + self.send_get_dataconnector_req(request) + return self.recv_get_dataconnector_req() + + def send_get_dataconnector_req(self, request): + self._oprot.writeMessageBegin("get_dataconnector_req", TMessageType.CALL, self._seqid) + args = get_dataconnector_req_args() + args.request = request + args.write(self._oprot) + self._oprot.writeMessageEnd() + self._oprot.trans.flush() + + def recv_get_dataconnector_req(self): + iprot = self._iprot + (fname, mtype, rseqid) = iprot.readMessageBegin() + if mtype == TMessageType.EXCEPTION: + x = TApplicationException() + x.read(iprot) + iprot.readMessageEnd() + raise x + result = get_dataconnector_req_result() + result.read(iprot) + iprot.readMessageEnd() + if result.success is not None: + return result.success + if result.o1 is not None: + raise result.o1 + if result.o2 is not None: + raise result.o2 + raise TApplicationException(TApplicationException.MISSING_RESULT, "get_dataconnector_req failed: unknown result") + + def drop_dataconnector(self, name, ifNotExists, checkReferences): + """ + Parameters: + - name + - ifNotExists + - checkReferences + + """ + self.send_drop_dataconnector(name, ifNotExists, checkReferences) + self.recv_drop_dataconnector() + + def send_drop_dataconnector(self, name, ifNotExists, checkReferences): + self._oprot.writeMessageBegin("drop_dataconnector", TMessageType.CALL, self._seqid) + args = drop_dataconnector_args() + args.name = name + args.ifNotExists = ifNotExists + args.checkReferences = checkReferences + args.write(self._oprot) + self._oprot.writeMessageEnd() + self._oprot.trans.flush() + + def recv_drop_dataconnector(self): + iprot = self._iprot + (fname, mtype, rseqid) = iprot.readMessageBegin() + if mtype == TMessageType.EXCEPTION: + x = TApplicationException() + x.read(iprot) + iprot.readMessageEnd() + raise x + result = drop_dataconnector_result() + result.read(iprot) + iprot.readMessageEnd() + if result.o1 is not None: + raise result.o1 + if result.o2 is not None: + raise result.o2 + if result.o3 is not None: + raise result.o3 + return + + def get_dataconnectors(self): + self.send_get_dataconnectors() + return self.recv_get_dataconnectors() + + def send_get_dataconnectors(self): + self._oprot.writeMessageBegin("get_dataconnectors", TMessageType.CALL, self._seqid) + args = get_dataconnectors_args() + args.write(self._oprot) + self._oprot.writeMessageEnd() + self._oprot.trans.flush() + + def recv_get_dataconnectors(self): + iprot = self._iprot + (fname, mtype, rseqid) = iprot.readMessageBegin() + if mtype == TMessageType.EXCEPTION: + x = TApplicationException() + x.read(iprot) + iprot.readMessageEnd() + raise x + result = get_dataconnectors_result() + result.read(iprot) + iprot.readMessageEnd() + if result.success is not None: + return result.success + if result.o1 is not None: + raise result.o1 + raise TApplicationException(TApplicationException.MISSING_RESULT, "get_dataconnectors failed: unknown result") + + def alter_dataconnector(self, name, connector): + """ + Parameters: + - name + - connector + + """ + self.send_alter_dataconnector(name, connector) + self.recv_alter_dataconnector() + + def send_alter_dataconnector(self, name, connector): + self._oprot.writeMessageBegin("alter_dataconnector", TMessageType.CALL, self._seqid) + args = alter_dataconnector_args() + args.name = name + args.connector = connector + args.write(self._oprot) + self._oprot.writeMessageEnd() + self._oprot.trans.flush() + + def recv_alter_dataconnector(self): + iprot = self._iprot + (fname, mtype, rseqid) = iprot.readMessageBegin() + if mtype == TMessageType.EXCEPTION: + x = TApplicationException() + x.read(iprot) + iprot.readMessageEnd() + raise x + result = alter_dataconnector_result() + result.read(iprot) + iprot.readMessageEnd() + if result.o1 is not None: + raise result.o1 + if result.o2 is not None: + raise result.o2 + return + + def get_type(self, name): + """ + Parameters: + - name + + """ + self.send_get_type(name) + return self.recv_get_type() + + def send_get_type(self, name): + self._oprot.writeMessageBegin("get_type", TMessageType.CALL, self._seqid) + args = get_type_args() + args.name = name + args.write(self._oprot) + self._oprot.writeMessageEnd() + self._oprot.trans.flush() + + def recv_get_type(self): + iprot = self._iprot + (fname, mtype, rseqid) = iprot.readMessageBegin() + if mtype == TMessageType.EXCEPTION: + x = TApplicationException() + x.read(iprot) + iprot.readMessageEnd() + raise x + result = get_type_result() + result.read(iprot) + iprot.readMessageEnd() + if result.success is not None: + return result.success + if result.o1 is not None: + raise result.o1 + if result.o2 is not None: + raise result.o2 + raise TApplicationException(TApplicationException.MISSING_RESULT, "get_type failed: unknown result") + + def create_type(self, type): + """ + Parameters: + - type + + """ + self.send_create_type(type) + return self.recv_create_type() + + def send_create_type(self, type): + self._oprot.writeMessageBegin("create_type", TMessageType.CALL, self._seqid) + args = create_type_args() + args.type = type + args.write(self._oprot) + self._oprot.writeMessageEnd() + self._oprot.trans.flush() + + def recv_create_type(self): + iprot = self._iprot + (fname, mtype, rseqid) = iprot.readMessageBegin() + if mtype == TMessageType.EXCEPTION: + x = TApplicationException() + x.read(iprot) + iprot.readMessageEnd() + raise x + result = create_type_result() + result.read(iprot) + iprot.readMessageEnd() + if result.success is not None: + return result.success + if result.o1 is not None: + raise result.o1 + if result.o2 is not None: + raise result.o2 + if result.o3 is not None: + raise result.o3 + raise TApplicationException(TApplicationException.MISSING_RESULT, "create_type failed: unknown result") + + def drop_type(self, type): + """ + Parameters: + - type + + """ + self.send_drop_type(type) + return self.recv_drop_type() + + def send_drop_type(self, type): + self._oprot.writeMessageBegin("drop_type", TMessageType.CALL, self._seqid) + args = drop_type_args() + args.type = type + args.write(self._oprot) + self._oprot.writeMessageEnd() + self._oprot.trans.flush() + + def recv_drop_type(self): + iprot = self._iprot + (fname, mtype, rseqid) = iprot.readMessageBegin() + if mtype == TMessageType.EXCEPTION: + x = TApplicationException() + x.read(iprot) + iprot.readMessageEnd() + raise x + result = drop_type_result() + result.read(iprot) + iprot.readMessageEnd() + if result.success is not None: + return result.success + if result.o1 is not None: + raise result.o1 + if result.o2 is not None: + raise result.o2 + raise TApplicationException(TApplicationException.MISSING_RESULT, "drop_type failed: unknown result") + + def get_type_all(self, name): + """ + Parameters: + - name + + """ + self.send_get_type_all(name) + return self.recv_get_type_all() + + def send_get_type_all(self, name): + self._oprot.writeMessageBegin("get_type_all", TMessageType.CALL, self._seqid) + args = get_type_all_args() + args.name = name + args.write(self._oprot) + self._oprot.writeMessageEnd() + self._oprot.trans.flush() + + def recv_get_type_all(self): + iprot = self._iprot + (fname, mtype, rseqid) = iprot.readMessageBegin() + if mtype == TMessageType.EXCEPTION: + x = TApplicationException() + x.read(iprot) + iprot.readMessageEnd() + raise x + result = get_type_all_result() + result.read(iprot) + iprot.readMessageEnd() + if result.success is not None: + return result.success + if result.o2 is not None: + raise result.o2 + raise TApplicationException(TApplicationException.MISSING_RESULT, "get_type_all failed: unknown result") + + def get_fields(self, db_name, table_name): + """ + Parameters: + - db_name + - table_name + + """ + self.send_get_fields(db_name, table_name) + return self.recv_get_fields() + + def send_get_fields(self, db_name, table_name): + self._oprot.writeMessageBegin("get_fields", TMessageType.CALL, self._seqid) + args = get_fields_args() + args.db_name = db_name + args.table_name = table_name + args.write(self._oprot) + self._oprot.writeMessageEnd() + self._oprot.trans.flush() + + def recv_get_fields(self): + iprot = self._iprot + (fname, mtype, rseqid) = iprot.readMessageBegin() + if mtype == TMessageType.EXCEPTION: + x = TApplicationException() + x.read(iprot) + iprot.readMessageEnd() + raise x + result = get_fields_result() + result.read(iprot) + iprot.readMessageEnd() + if result.success is not None: + return result.success + if result.o1 is not None: + raise result.o1 + if result.o2 is not None: + raise result.o2 + if result.o3 is not None: + raise result.o3 + raise TApplicationException(TApplicationException.MISSING_RESULT, "get_fields failed: unknown result") + + def get_fields_with_environment_context(self, db_name, table_name, environment_context): + """ + Parameters: + - db_name + - table_name + - environment_context + + """ + self.send_get_fields_with_environment_context(db_name, table_name, environment_context) + return self.recv_get_fields_with_environment_context() + + def send_get_fields_with_environment_context(self, db_name, table_name, environment_context): + self._oprot.writeMessageBegin("get_fields_with_environment_context", TMessageType.CALL, self._seqid) + args = get_fields_with_environment_context_args() + args.db_name = db_name + args.table_name = table_name + args.environment_context = environment_context + args.write(self._oprot) + self._oprot.writeMessageEnd() + self._oprot.trans.flush() + + def recv_get_fields_with_environment_context(self): + iprot = self._iprot + (fname, mtype, rseqid) = iprot.readMessageBegin() + if mtype == TMessageType.EXCEPTION: + x = TApplicationException() + x.read(iprot) + iprot.readMessageEnd() + raise x + result = get_fields_with_environment_context_result() + result.read(iprot) + iprot.readMessageEnd() + if result.success is not None: + return result.success + if result.o1 is not None: + raise result.o1 + if result.o2 is not None: + raise result.o2 + if result.o3 is not None: + raise result.o3 + raise TApplicationException( + TApplicationException.MISSING_RESULT, "get_fields_with_environment_context failed: unknown result" + ) + + def get_fields_req(self, req): + """ + Parameters: + - req + + """ + self.send_get_fields_req(req) + return self.recv_get_fields_req() + + def send_get_fields_req(self, req): + self._oprot.writeMessageBegin("get_fields_req", TMessageType.CALL, self._seqid) + args = get_fields_req_args() + args.req = req + args.write(self._oprot) + self._oprot.writeMessageEnd() + self._oprot.trans.flush() + + def recv_get_fields_req(self): + iprot = self._iprot + (fname, mtype, rseqid) = iprot.readMessageBegin() + if mtype == TMessageType.EXCEPTION: + x = TApplicationException() + x.read(iprot) + iprot.readMessageEnd() + raise x + result = get_fields_req_result() + result.read(iprot) + iprot.readMessageEnd() + if result.success is not None: + return result.success + if result.o1 is not None: + raise result.o1 + if result.o2 is not None: + raise result.o2 + if result.o3 is not None: + raise result.o3 + raise TApplicationException(TApplicationException.MISSING_RESULT, "get_fields_req failed: unknown result") + + def get_schema(self, db_name, table_name): + """ + Parameters: + - db_name + - table_name + + """ + self.send_get_schema(db_name, table_name) + return self.recv_get_schema() + + def send_get_schema(self, db_name, table_name): + self._oprot.writeMessageBegin("get_schema", TMessageType.CALL, self._seqid) + args = get_schema_args() + args.db_name = db_name + args.table_name = table_name + args.write(self._oprot) + self._oprot.writeMessageEnd() + self._oprot.trans.flush() + + def recv_get_schema(self): + iprot = self._iprot + (fname, mtype, rseqid) = iprot.readMessageBegin() + if mtype == TMessageType.EXCEPTION: + x = TApplicationException() + x.read(iprot) + iprot.readMessageEnd() + raise x + result = get_schema_result() + result.read(iprot) + iprot.readMessageEnd() + if result.success is not None: + return result.success + if result.o1 is not None: + raise result.o1 + if result.o2 is not None: + raise result.o2 + if result.o3 is not None: + raise result.o3 + raise TApplicationException(TApplicationException.MISSING_RESULT, "get_schema failed: unknown result") + + def get_schema_with_environment_context(self, db_name, table_name, environment_context): + """ + Parameters: + - db_name + - table_name + - environment_context + + """ + self.send_get_schema_with_environment_context(db_name, table_name, environment_context) + return self.recv_get_schema_with_environment_context() + + def send_get_schema_with_environment_context(self, db_name, table_name, environment_context): + self._oprot.writeMessageBegin("get_schema_with_environment_context", TMessageType.CALL, self._seqid) + args = get_schema_with_environment_context_args() + args.db_name = db_name + args.table_name = table_name + args.environment_context = environment_context + args.write(self._oprot) + self._oprot.writeMessageEnd() + self._oprot.trans.flush() + + def recv_get_schema_with_environment_context(self): + iprot = self._iprot + (fname, mtype, rseqid) = iprot.readMessageBegin() + if mtype == TMessageType.EXCEPTION: + x = TApplicationException() + x.read(iprot) + iprot.readMessageEnd() + raise x + result = get_schema_with_environment_context_result() + result.read(iprot) + iprot.readMessageEnd() + if result.success is not None: + return result.success + if result.o1 is not None: + raise result.o1 + if result.o2 is not None: + raise result.o2 + if result.o3 is not None: + raise result.o3 + raise TApplicationException( + TApplicationException.MISSING_RESULT, "get_schema_with_environment_context failed: unknown result" + ) + + def get_schema_req(self, req): + """ + Parameters: + - req + + """ + self.send_get_schema_req(req) + return self.recv_get_schema_req() + + def send_get_schema_req(self, req): + self._oprot.writeMessageBegin("get_schema_req", TMessageType.CALL, self._seqid) + args = get_schema_req_args() + args.req = req + args.write(self._oprot) + self._oprot.writeMessageEnd() + self._oprot.trans.flush() + + def recv_get_schema_req(self): + iprot = self._iprot + (fname, mtype, rseqid) = iprot.readMessageBegin() + if mtype == TMessageType.EXCEPTION: + x = TApplicationException() + x.read(iprot) + iprot.readMessageEnd() + raise x + result = get_schema_req_result() + result.read(iprot) + iprot.readMessageEnd() + if result.success is not None: + return result.success + if result.o1 is not None: + raise result.o1 + if result.o2 is not None: + raise result.o2 + if result.o3 is not None: + raise result.o3 + raise TApplicationException(TApplicationException.MISSING_RESULT, "get_schema_req failed: unknown result") + + def create_table(self, tbl): + """ + Parameters: + - tbl + + """ + self.send_create_table(tbl) + self.recv_create_table() + + def send_create_table(self, tbl): + self._oprot.writeMessageBegin("create_table", TMessageType.CALL, self._seqid) + args = create_table_args() + args.tbl = tbl + args.write(self._oprot) + self._oprot.writeMessageEnd() + self._oprot.trans.flush() + + def recv_create_table(self): + iprot = self._iprot + (fname, mtype, rseqid) = iprot.readMessageBegin() + if mtype == TMessageType.EXCEPTION: + x = TApplicationException() + x.read(iprot) + iprot.readMessageEnd() + raise x + result = create_table_result() + result.read(iprot) + iprot.readMessageEnd() + if result.o1 is not None: + raise result.o1 + if result.o2 is not None: + raise result.o2 + if result.o3 is not None: + raise result.o3 + if result.o4 is not None: + raise result.o4 + return + + def create_table_with_environment_context(self, tbl, environment_context): + """ + Parameters: + - tbl + - environment_context + + """ + self.send_create_table_with_environment_context(tbl, environment_context) + self.recv_create_table_with_environment_context() + + def send_create_table_with_environment_context(self, tbl, environment_context): + self._oprot.writeMessageBegin("create_table_with_environment_context", TMessageType.CALL, self._seqid) + args = create_table_with_environment_context_args() + args.tbl = tbl + args.environment_context = environment_context + args.write(self._oprot) + self._oprot.writeMessageEnd() + self._oprot.trans.flush() + + def recv_create_table_with_environment_context(self): + iprot = self._iprot + (fname, mtype, rseqid) = iprot.readMessageBegin() + if mtype == TMessageType.EXCEPTION: + x = TApplicationException() + x.read(iprot) + iprot.readMessageEnd() + raise x + result = create_table_with_environment_context_result() + result.read(iprot) + iprot.readMessageEnd() + if result.o1 is not None: + raise result.o1 + if result.o2 is not None: + raise result.o2 + if result.o3 is not None: + raise result.o3 + if result.o4 is not None: + raise result.o4 + return + + def create_table_with_constraints( + self, tbl, primaryKeys, foreignKeys, uniqueConstraints, notNullConstraints, defaultConstraints, checkConstraints + ): + """ + Parameters: + - tbl + - primaryKeys + - foreignKeys + - uniqueConstraints + - notNullConstraints + - defaultConstraints + - checkConstraints + + """ + self.send_create_table_with_constraints( + tbl, primaryKeys, foreignKeys, uniqueConstraints, notNullConstraints, defaultConstraints, checkConstraints + ) + self.recv_create_table_with_constraints() + + def send_create_table_with_constraints( + self, tbl, primaryKeys, foreignKeys, uniqueConstraints, notNullConstraints, defaultConstraints, checkConstraints + ): + self._oprot.writeMessageBegin("create_table_with_constraints", TMessageType.CALL, self._seqid) + args = create_table_with_constraints_args() + args.tbl = tbl + args.primaryKeys = primaryKeys + args.foreignKeys = foreignKeys + args.uniqueConstraints = uniqueConstraints + args.notNullConstraints = notNullConstraints + args.defaultConstraints = defaultConstraints + args.checkConstraints = checkConstraints + args.write(self._oprot) + self._oprot.writeMessageEnd() + self._oprot.trans.flush() + + def recv_create_table_with_constraints(self): + iprot = self._iprot + (fname, mtype, rseqid) = iprot.readMessageBegin() + if mtype == TMessageType.EXCEPTION: + x = TApplicationException() + x.read(iprot) + iprot.readMessageEnd() + raise x + result = create_table_with_constraints_result() + result.read(iprot) + iprot.readMessageEnd() + if result.o1 is not None: + raise result.o1 + if result.o2 is not None: + raise result.o2 + if result.o3 is not None: + raise result.o3 + if result.o4 is not None: + raise result.o4 + return + + def create_table_req(self, request): + """ + Parameters: + - request + + """ + self.send_create_table_req(request) + self.recv_create_table_req() + + def send_create_table_req(self, request): + self._oprot.writeMessageBegin("create_table_req", TMessageType.CALL, self._seqid) + args = create_table_req_args() + args.request = request + args.write(self._oprot) + self._oprot.writeMessageEnd() + self._oprot.trans.flush() + + def recv_create_table_req(self): + iprot = self._iprot + (fname, mtype, rseqid) = iprot.readMessageBegin() + if mtype == TMessageType.EXCEPTION: + x = TApplicationException() + x.read(iprot) + iprot.readMessageEnd() + raise x + result = create_table_req_result() + result.read(iprot) + iprot.readMessageEnd() + if result.o1 is not None: + raise result.o1 + if result.o2 is not None: + raise result.o2 + if result.o3 is not None: + raise result.o3 + if result.o4 is not None: + raise result.o4 + return + + def drop_constraint(self, req): + """ + Parameters: + - req + + """ + self.send_drop_constraint(req) + self.recv_drop_constraint() + + def send_drop_constraint(self, req): + self._oprot.writeMessageBegin("drop_constraint", TMessageType.CALL, self._seqid) + args = drop_constraint_args() + args.req = req + args.write(self._oprot) + self._oprot.writeMessageEnd() + self._oprot.trans.flush() + + def recv_drop_constraint(self): + iprot = self._iprot + (fname, mtype, rseqid) = iprot.readMessageBegin() + if mtype == TMessageType.EXCEPTION: + x = TApplicationException() + x.read(iprot) + iprot.readMessageEnd() + raise x + result = drop_constraint_result() + result.read(iprot) + iprot.readMessageEnd() + if result.o1 is not None: + raise result.o1 + if result.o3 is not None: + raise result.o3 + return + + def add_primary_key(self, req): + """ + Parameters: + - req + + """ + self.send_add_primary_key(req) + self.recv_add_primary_key() + + def send_add_primary_key(self, req): + self._oprot.writeMessageBegin("add_primary_key", TMessageType.CALL, self._seqid) + args = add_primary_key_args() + args.req = req + args.write(self._oprot) + self._oprot.writeMessageEnd() + self._oprot.trans.flush() + + def recv_add_primary_key(self): + iprot = self._iprot + (fname, mtype, rseqid) = iprot.readMessageBegin() + if mtype == TMessageType.EXCEPTION: + x = TApplicationException() + x.read(iprot) + iprot.readMessageEnd() + raise x + result = add_primary_key_result() + result.read(iprot) + iprot.readMessageEnd() + if result.o1 is not None: + raise result.o1 + if result.o2 is not None: + raise result.o2 + return + + def add_foreign_key(self, req): + """ + Parameters: + - req + + """ + self.send_add_foreign_key(req) + self.recv_add_foreign_key() + + def send_add_foreign_key(self, req): + self._oprot.writeMessageBegin("add_foreign_key", TMessageType.CALL, self._seqid) + args = add_foreign_key_args() + args.req = req + args.write(self._oprot) + self._oprot.writeMessageEnd() + self._oprot.trans.flush() + + def recv_add_foreign_key(self): + iprot = self._iprot + (fname, mtype, rseqid) = iprot.readMessageBegin() + if mtype == TMessageType.EXCEPTION: + x = TApplicationException() + x.read(iprot) + iprot.readMessageEnd() + raise x + result = add_foreign_key_result() + result.read(iprot) + iprot.readMessageEnd() + if result.o1 is not None: + raise result.o1 + if result.o2 is not None: + raise result.o2 + return + + def add_unique_constraint(self, req): + """ + Parameters: + - req + + """ + self.send_add_unique_constraint(req) + self.recv_add_unique_constraint() + + def send_add_unique_constraint(self, req): + self._oprot.writeMessageBegin("add_unique_constraint", TMessageType.CALL, self._seqid) + args = add_unique_constraint_args() + args.req = req + args.write(self._oprot) + self._oprot.writeMessageEnd() + self._oprot.trans.flush() + + def recv_add_unique_constraint(self): + iprot = self._iprot + (fname, mtype, rseqid) = iprot.readMessageBegin() + if mtype == TMessageType.EXCEPTION: + x = TApplicationException() + x.read(iprot) + iprot.readMessageEnd() + raise x + result = add_unique_constraint_result() + result.read(iprot) + iprot.readMessageEnd() + if result.o1 is not None: + raise result.o1 + if result.o2 is not None: + raise result.o2 + return + + def add_not_null_constraint(self, req): + """ + Parameters: + - req + + """ + self.send_add_not_null_constraint(req) + self.recv_add_not_null_constraint() + + def send_add_not_null_constraint(self, req): + self._oprot.writeMessageBegin("add_not_null_constraint", TMessageType.CALL, self._seqid) + args = add_not_null_constraint_args() + args.req = req + args.write(self._oprot) + self._oprot.writeMessageEnd() + self._oprot.trans.flush() + + def recv_add_not_null_constraint(self): + iprot = self._iprot + (fname, mtype, rseqid) = iprot.readMessageBegin() + if mtype == TMessageType.EXCEPTION: + x = TApplicationException() + x.read(iprot) + iprot.readMessageEnd() + raise x + result = add_not_null_constraint_result() + result.read(iprot) + iprot.readMessageEnd() + if result.o1 is not None: + raise result.o1 + if result.o2 is not None: + raise result.o2 + return + + def add_default_constraint(self, req): + """ + Parameters: + - req + + """ + self.send_add_default_constraint(req) + self.recv_add_default_constraint() + + def send_add_default_constraint(self, req): + self._oprot.writeMessageBegin("add_default_constraint", TMessageType.CALL, self._seqid) + args = add_default_constraint_args() + args.req = req + args.write(self._oprot) + self._oprot.writeMessageEnd() + self._oprot.trans.flush() + + def recv_add_default_constraint(self): + iprot = self._iprot + (fname, mtype, rseqid) = iprot.readMessageBegin() + if mtype == TMessageType.EXCEPTION: + x = TApplicationException() + x.read(iprot) + iprot.readMessageEnd() + raise x + result = add_default_constraint_result() + result.read(iprot) + iprot.readMessageEnd() + if result.o1 is not None: + raise result.o1 + if result.o2 is not None: + raise result.o2 + return + + def add_check_constraint(self, req): + """ + Parameters: + - req + + """ + self.send_add_check_constraint(req) + self.recv_add_check_constraint() + + def send_add_check_constraint(self, req): + self._oprot.writeMessageBegin("add_check_constraint", TMessageType.CALL, self._seqid) + args = add_check_constraint_args() + args.req = req + args.write(self._oprot) + self._oprot.writeMessageEnd() + self._oprot.trans.flush() + + def recv_add_check_constraint(self): + iprot = self._iprot + (fname, mtype, rseqid) = iprot.readMessageBegin() + if mtype == TMessageType.EXCEPTION: + x = TApplicationException() + x.read(iprot) + iprot.readMessageEnd() + raise x + result = add_check_constraint_result() + result.read(iprot) + iprot.readMessageEnd() + if result.o1 is not None: + raise result.o1 + if result.o2 is not None: + raise result.o2 + return + + def translate_table_dryrun(self, request): + """ + Parameters: + - request + + """ + self.send_translate_table_dryrun(request) + return self.recv_translate_table_dryrun() + + def send_translate_table_dryrun(self, request): + self._oprot.writeMessageBegin("translate_table_dryrun", TMessageType.CALL, self._seqid) + args = translate_table_dryrun_args() + args.request = request + args.write(self._oprot) + self._oprot.writeMessageEnd() + self._oprot.trans.flush() + + def recv_translate_table_dryrun(self): + iprot = self._iprot + (fname, mtype, rseqid) = iprot.readMessageBegin() + if mtype == TMessageType.EXCEPTION: + x = TApplicationException() + x.read(iprot) + iprot.readMessageEnd() + raise x + result = translate_table_dryrun_result() + result.read(iprot) + iprot.readMessageEnd() + if result.success is not None: + return result.success + if result.o1 is not None: + raise result.o1 + if result.o2 is not None: + raise result.o2 + if result.o3 is not None: + raise result.o3 + if result.o4 is not None: + raise result.o4 + raise TApplicationException(TApplicationException.MISSING_RESULT, "translate_table_dryrun failed: unknown result") + + def drop_table(self, dbname, name, deleteData): + """ + Parameters: + - dbname + - name + - deleteData + + """ + self.send_drop_table(dbname, name, deleteData) + self.recv_drop_table() + + def send_drop_table(self, dbname, name, deleteData): + self._oprot.writeMessageBegin("drop_table", TMessageType.CALL, self._seqid) + args = drop_table_args() + args.dbname = dbname + args.name = name + args.deleteData = deleteData + args.write(self._oprot) + self._oprot.writeMessageEnd() + self._oprot.trans.flush() + + def recv_drop_table(self): + iprot = self._iprot + (fname, mtype, rseqid) = iprot.readMessageBegin() + if mtype == TMessageType.EXCEPTION: + x = TApplicationException() + x.read(iprot) + iprot.readMessageEnd() + raise x + result = drop_table_result() + result.read(iprot) + iprot.readMessageEnd() + if result.o1 is not None: + raise result.o1 + if result.o3 is not None: + raise result.o3 + return + + def drop_table_with_environment_context(self, dbname, name, deleteData, environment_context): + """ + Parameters: + - dbname + - name + - deleteData + - environment_context + + """ + self.send_drop_table_with_environment_context(dbname, name, deleteData, environment_context) + self.recv_drop_table_with_environment_context() + + def send_drop_table_with_environment_context(self, dbname, name, deleteData, environment_context): + self._oprot.writeMessageBegin("drop_table_with_environment_context", TMessageType.CALL, self._seqid) + args = drop_table_with_environment_context_args() + args.dbname = dbname + args.name = name + args.deleteData = deleteData + args.environment_context = environment_context + args.write(self._oprot) + self._oprot.writeMessageEnd() + self._oprot.trans.flush() + + def recv_drop_table_with_environment_context(self): + iprot = self._iprot + (fname, mtype, rseqid) = iprot.readMessageBegin() + if mtype == TMessageType.EXCEPTION: + x = TApplicationException() + x.read(iprot) + iprot.readMessageEnd() + raise x + result = drop_table_with_environment_context_result() + result.read(iprot) + iprot.readMessageEnd() + if result.o1 is not None: + raise result.o1 + if result.o3 is not None: + raise result.o3 + return + + def truncate_table(self, dbName, tableName, partNames): + """ + Parameters: + - dbName + - tableName + - partNames + + """ + self.send_truncate_table(dbName, tableName, partNames) + self.recv_truncate_table() + + def send_truncate_table(self, dbName, tableName, partNames): + self._oprot.writeMessageBegin("truncate_table", TMessageType.CALL, self._seqid) + args = truncate_table_args() + args.dbName = dbName + args.tableName = tableName + args.partNames = partNames + args.write(self._oprot) + self._oprot.writeMessageEnd() + self._oprot.trans.flush() + + def recv_truncate_table(self): + iprot = self._iprot + (fname, mtype, rseqid) = iprot.readMessageBegin() + if mtype == TMessageType.EXCEPTION: + x = TApplicationException() + x.read(iprot) + iprot.readMessageEnd() + raise x + result = truncate_table_result() + result.read(iprot) + iprot.readMessageEnd() + if result.o1 is not None: + raise result.o1 + return + + def truncate_table_req(self, req): + """ + Parameters: + - req + + """ + self.send_truncate_table_req(req) + return self.recv_truncate_table_req() + + def send_truncate_table_req(self, req): + self._oprot.writeMessageBegin("truncate_table_req", TMessageType.CALL, self._seqid) + args = truncate_table_req_args() + args.req = req + args.write(self._oprot) + self._oprot.writeMessageEnd() + self._oprot.trans.flush() + + def recv_truncate_table_req(self): + iprot = self._iprot + (fname, mtype, rseqid) = iprot.readMessageBegin() + if mtype == TMessageType.EXCEPTION: + x = TApplicationException() + x.read(iprot) + iprot.readMessageEnd() + raise x + result = truncate_table_req_result() + result.read(iprot) + iprot.readMessageEnd() + if result.success is not None: + return result.success + if result.o1 is not None: + raise result.o1 + raise TApplicationException(TApplicationException.MISSING_RESULT, "truncate_table_req failed: unknown result") + + def get_tables(self, db_name, pattern): + """ + Parameters: + - db_name + - pattern + + """ + self.send_get_tables(db_name, pattern) + return self.recv_get_tables() + + def send_get_tables(self, db_name, pattern): + self._oprot.writeMessageBegin("get_tables", TMessageType.CALL, self._seqid) + args = get_tables_args() + args.db_name = db_name + args.pattern = pattern + args.write(self._oprot) + self._oprot.writeMessageEnd() + self._oprot.trans.flush() + + def recv_get_tables(self): + iprot = self._iprot + (fname, mtype, rseqid) = iprot.readMessageBegin() + if mtype == TMessageType.EXCEPTION: + x = TApplicationException() + x.read(iprot) + iprot.readMessageEnd() + raise x + result = get_tables_result() + result.read(iprot) + iprot.readMessageEnd() + if result.success is not None: + return result.success + if result.o1 is not None: + raise result.o1 + raise TApplicationException(TApplicationException.MISSING_RESULT, "get_tables failed: unknown result") + + def get_tables_by_type(self, db_name, pattern, tableType): + """ + Parameters: + - db_name + - pattern + - tableType + + """ + self.send_get_tables_by_type(db_name, pattern, tableType) + return self.recv_get_tables_by_type() + + def send_get_tables_by_type(self, db_name, pattern, tableType): + self._oprot.writeMessageBegin("get_tables_by_type", TMessageType.CALL, self._seqid) + args = get_tables_by_type_args() + args.db_name = db_name + args.pattern = pattern + args.tableType = tableType + args.write(self._oprot) + self._oprot.writeMessageEnd() + self._oprot.trans.flush() + + def recv_get_tables_by_type(self): + iprot = self._iprot + (fname, mtype, rseqid) = iprot.readMessageBegin() + if mtype == TMessageType.EXCEPTION: + x = TApplicationException() + x.read(iprot) + iprot.readMessageEnd() + raise x + result = get_tables_by_type_result() + result.read(iprot) + iprot.readMessageEnd() + if result.success is not None: + return result.success + if result.o1 is not None: + raise result.o1 + raise TApplicationException(TApplicationException.MISSING_RESULT, "get_tables_by_type failed: unknown result") + + def get_all_materialized_view_objects_for_rewriting(self): + self.send_get_all_materialized_view_objects_for_rewriting() + return self.recv_get_all_materialized_view_objects_for_rewriting() + + def send_get_all_materialized_view_objects_for_rewriting(self): + self._oprot.writeMessageBegin("get_all_materialized_view_objects_for_rewriting", TMessageType.CALL, self._seqid) + args = get_all_materialized_view_objects_for_rewriting_args() + args.write(self._oprot) + self._oprot.writeMessageEnd() + self._oprot.trans.flush() + + def recv_get_all_materialized_view_objects_for_rewriting(self): + iprot = self._iprot + (fname, mtype, rseqid) = iprot.readMessageBegin() + if mtype == TMessageType.EXCEPTION: + x = TApplicationException() + x.read(iprot) + iprot.readMessageEnd() + raise x + result = get_all_materialized_view_objects_for_rewriting_result() + result.read(iprot) + iprot.readMessageEnd() + if result.success is not None: + return result.success + if result.o1 is not None: + raise result.o1 + raise TApplicationException( + TApplicationException.MISSING_RESULT, "get_all_materialized_view_objects_for_rewriting failed: unknown result" + ) + + def get_materialized_views_for_rewriting(self, db_name): + """ + Parameters: + - db_name + + """ + self.send_get_materialized_views_for_rewriting(db_name) + return self.recv_get_materialized_views_for_rewriting() + + def send_get_materialized_views_for_rewriting(self, db_name): + self._oprot.writeMessageBegin("get_materialized_views_for_rewriting", TMessageType.CALL, self._seqid) + args = get_materialized_views_for_rewriting_args() + args.db_name = db_name + args.write(self._oprot) + self._oprot.writeMessageEnd() + self._oprot.trans.flush() + + def recv_get_materialized_views_for_rewriting(self): + iprot = self._iprot + (fname, mtype, rseqid) = iprot.readMessageBegin() + if mtype == TMessageType.EXCEPTION: + x = TApplicationException() + x.read(iprot) + iprot.readMessageEnd() + raise x + result = get_materialized_views_for_rewriting_result() + result.read(iprot) + iprot.readMessageEnd() + if result.success is not None: + return result.success + if result.o1 is not None: + raise result.o1 + raise TApplicationException( + TApplicationException.MISSING_RESULT, "get_materialized_views_for_rewriting failed: unknown result" + ) + + def get_table_meta(self, db_patterns, tbl_patterns, tbl_types): + """ + Parameters: + - db_patterns + - tbl_patterns + - tbl_types + + """ + self.send_get_table_meta(db_patterns, tbl_patterns, tbl_types) + return self.recv_get_table_meta() + + def send_get_table_meta(self, db_patterns, tbl_patterns, tbl_types): + self._oprot.writeMessageBegin("get_table_meta", TMessageType.CALL, self._seqid) + args = get_table_meta_args() + args.db_patterns = db_patterns + args.tbl_patterns = tbl_patterns + args.tbl_types = tbl_types + args.write(self._oprot) + self._oprot.writeMessageEnd() + self._oprot.trans.flush() + + def recv_get_table_meta(self): + iprot = self._iprot + (fname, mtype, rseqid) = iprot.readMessageBegin() + if mtype == TMessageType.EXCEPTION: + x = TApplicationException() + x.read(iprot) + iprot.readMessageEnd() + raise x + result = get_table_meta_result() + result.read(iprot) + iprot.readMessageEnd() + if result.success is not None: + return result.success + if result.o1 is not None: + raise result.o1 + raise TApplicationException(TApplicationException.MISSING_RESULT, "get_table_meta failed: unknown result") + + def get_all_tables(self, db_name): + """ + Parameters: + - db_name + + """ + self.send_get_all_tables(db_name) + return self.recv_get_all_tables() + + def send_get_all_tables(self, db_name): + self._oprot.writeMessageBegin("get_all_tables", TMessageType.CALL, self._seqid) + args = get_all_tables_args() + args.db_name = db_name + args.write(self._oprot) + self._oprot.writeMessageEnd() + self._oprot.trans.flush() + + def recv_get_all_tables(self): + iprot = self._iprot + (fname, mtype, rseqid) = iprot.readMessageBegin() + if mtype == TMessageType.EXCEPTION: + x = TApplicationException() + x.read(iprot) + iprot.readMessageEnd() + raise x + result = get_all_tables_result() + result.read(iprot) + iprot.readMessageEnd() + if result.success is not None: + return result.success + if result.o1 is not None: + raise result.o1 + raise TApplicationException(TApplicationException.MISSING_RESULT, "get_all_tables failed: unknown result") + + def get_table(self, dbname, tbl_name): + """ + Parameters: + - dbname + - tbl_name + + """ + self.send_get_table(dbname, tbl_name) + return self.recv_get_table() + + def send_get_table(self, dbname, tbl_name): + self._oprot.writeMessageBegin("get_table", TMessageType.CALL, self._seqid) + args = get_table_args() + args.dbname = dbname + args.tbl_name = tbl_name + args.write(self._oprot) + self._oprot.writeMessageEnd() + self._oprot.trans.flush() + + def recv_get_table(self): + iprot = self._iprot + (fname, mtype, rseqid) = iprot.readMessageBegin() + if mtype == TMessageType.EXCEPTION: + x = TApplicationException() + x.read(iprot) + iprot.readMessageEnd() + raise x + result = get_table_result() + result.read(iprot) + iprot.readMessageEnd() + if result.success is not None: + return result.success + if result.o1 is not None: + raise result.o1 + if result.o2 is not None: + raise result.o2 + raise TApplicationException(TApplicationException.MISSING_RESULT, "get_table failed: unknown result") + + def get_table_objects_by_name(self, dbname, tbl_names): + """ + Parameters: + - dbname + - tbl_names + + """ + self.send_get_table_objects_by_name(dbname, tbl_names) + return self.recv_get_table_objects_by_name() + + def send_get_table_objects_by_name(self, dbname, tbl_names): + self._oprot.writeMessageBegin("get_table_objects_by_name", TMessageType.CALL, self._seqid) + args = get_table_objects_by_name_args() + args.dbname = dbname + args.tbl_names = tbl_names + args.write(self._oprot) + self._oprot.writeMessageEnd() + self._oprot.trans.flush() + + def recv_get_table_objects_by_name(self): + iprot = self._iprot + (fname, mtype, rseqid) = iprot.readMessageBegin() + if mtype == TMessageType.EXCEPTION: + x = TApplicationException() + x.read(iprot) + iprot.readMessageEnd() + raise x + result = get_table_objects_by_name_result() + result.read(iprot) + iprot.readMessageEnd() + if result.success is not None: + return result.success + raise TApplicationException(TApplicationException.MISSING_RESULT, "get_table_objects_by_name failed: unknown result") + + def get_tables_ext(self, req): + """ + Parameters: + - req + + """ + self.send_get_tables_ext(req) + return self.recv_get_tables_ext() + + def send_get_tables_ext(self, req): + self._oprot.writeMessageBegin("get_tables_ext", TMessageType.CALL, self._seqid) + args = get_tables_ext_args() + args.req = req + args.write(self._oprot) + self._oprot.writeMessageEnd() + self._oprot.trans.flush() + + def recv_get_tables_ext(self): + iprot = self._iprot + (fname, mtype, rseqid) = iprot.readMessageBegin() + if mtype == TMessageType.EXCEPTION: + x = TApplicationException() + x.read(iprot) + iprot.readMessageEnd() + raise x + result = get_tables_ext_result() + result.read(iprot) + iprot.readMessageEnd() + if result.success is not None: + return result.success + if result.o1 is not None: + raise result.o1 + raise TApplicationException(TApplicationException.MISSING_RESULT, "get_tables_ext failed: unknown result") + + def get_table_req(self, req): + """ + Parameters: + - req + + """ + self.send_get_table_req(req) + return self.recv_get_table_req() + + def send_get_table_req(self, req): + self._oprot.writeMessageBegin("get_table_req", TMessageType.CALL, self._seqid) + args = get_table_req_args() + args.req = req + args.write(self._oprot) + self._oprot.writeMessageEnd() + self._oprot.trans.flush() + + def recv_get_table_req(self): + iprot = self._iprot + (fname, mtype, rseqid) = iprot.readMessageBegin() + if mtype == TMessageType.EXCEPTION: + x = TApplicationException() + x.read(iprot) + iprot.readMessageEnd() + raise x + result = get_table_req_result() + result.read(iprot) + iprot.readMessageEnd() + if result.success is not None: + return result.success + if result.o1 is not None: + raise result.o1 + if result.o2 is not None: + raise result.o2 + raise TApplicationException(TApplicationException.MISSING_RESULT, "get_table_req failed: unknown result") + + def get_table_objects_by_name_req(self, req): + """ + Parameters: + - req + + """ + self.send_get_table_objects_by_name_req(req) + return self.recv_get_table_objects_by_name_req() + + def send_get_table_objects_by_name_req(self, req): + self._oprot.writeMessageBegin("get_table_objects_by_name_req", TMessageType.CALL, self._seqid) + args = get_table_objects_by_name_req_args() + args.req = req + args.write(self._oprot) + self._oprot.writeMessageEnd() + self._oprot.trans.flush() + + def recv_get_table_objects_by_name_req(self): + iprot = self._iprot + (fname, mtype, rseqid) = iprot.readMessageBegin() + if mtype == TMessageType.EXCEPTION: + x = TApplicationException() + x.read(iprot) + iprot.readMessageEnd() + raise x + result = get_table_objects_by_name_req_result() + result.read(iprot) + iprot.readMessageEnd() + if result.success is not None: + return result.success + if result.o1 is not None: + raise result.o1 + if result.o2 is not None: + raise result.o2 + if result.o3 is not None: + raise result.o3 + raise TApplicationException(TApplicationException.MISSING_RESULT, "get_table_objects_by_name_req failed: unknown result") + + def get_materialization_invalidation_info(self, creation_metadata, validTxnList): + """ + Parameters: + - creation_metadata + - validTxnList + + """ + self.send_get_materialization_invalidation_info(creation_metadata, validTxnList) + return self.recv_get_materialization_invalidation_info() + + def send_get_materialization_invalidation_info(self, creation_metadata, validTxnList): + self._oprot.writeMessageBegin("get_materialization_invalidation_info", TMessageType.CALL, self._seqid) + args = get_materialization_invalidation_info_args() + args.creation_metadata = creation_metadata + args.validTxnList = validTxnList + args.write(self._oprot) + self._oprot.writeMessageEnd() + self._oprot.trans.flush() + + def recv_get_materialization_invalidation_info(self): + iprot = self._iprot + (fname, mtype, rseqid) = iprot.readMessageBegin() + if mtype == TMessageType.EXCEPTION: + x = TApplicationException() + x.read(iprot) + iprot.readMessageEnd() + raise x + result = get_materialization_invalidation_info_result() + result.read(iprot) + iprot.readMessageEnd() + if result.success is not None: + return result.success + if result.o1 is not None: + raise result.o1 + if result.o2 is not None: + raise result.o2 + if result.o3 is not None: + raise result.o3 + raise TApplicationException( + TApplicationException.MISSING_RESULT, "get_materialization_invalidation_info failed: unknown result" + ) + + def update_creation_metadata(self, catName, dbname, tbl_name, creation_metadata): + """ + Parameters: + - catName + - dbname + - tbl_name + - creation_metadata + + """ + self.send_update_creation_metadata(catName, dbname, tbl_name, creation_metadata) + self.recv_update_creation_metadata() + + def send_update_creation_metadata(self, catName, dbname, tbl_name, creation_metadata): + self._oprot.writeMessageBegin("update_creation_metadata", TMessageType.CALL, self._seqid) + args = update_creation_metadata_args() + args.catName = catName + args.dbname = dbname + args.tbl_name = tbl_name + args.creation_metadata = creation_metadata + args.write(self._oprot) + self._oprot.writeMessageEnd() + self._oprot.trans.flush() + + def recv_update_creation_metadata(self): + iprot = self._iprot + (fname, mtype, rseqid) = iprot.readMessageBegin() + if mtype == TMessageType.EXCEPTION: + x = TApplicationException() + x.read(iprot) + iprot.readMessageEnd() + raise x + result = update_creation_metadata_result() + result.read(iprot) + iprot.readMessageEnd() + if result.o1 is not None: + raise result.o1 + if result.o2 is not None: + raise result.o2 + if result.o3 is not None: + raise result.o3 + return + + def get_table_names_by_filter(self, dbname, filter, max_tables): + """ + Parameters: + - dbname + - filter + - max_tables + + """ + self.send_get_table_names_by_filter(dbname, filter, max_tables) + return self.recv_get_table_names_by_filter() + + def send_get_table_names_by_filter(self, dbname, filter, max_tables): + self._oprot.writeMessageBegin("get_table_names_by_filter", TMessageType.CALL, self._seqid) + args = get_table_names_by_filter_args() + args.dbname = dbname + args.filter = filter + args.max_tables = max_tables + args.write(self._oprot) + self._oprot.writeMessageEnd() + self._oprot.trans.flush() + + def recv_get_table_names_by_filter(self): + iprot = self._iprot + (fname, mtype, rseqid) = iprot.readMessageBegin() + if mtype == TMessageType.EXCEPTION: + x = TApplicationException() + x.read(iprot) + iprot.readMessageEnd() + raise x + result = get_table_names_by_filter_result() + result.read(iprot) + iprot.readMessageEnd() + if result.success is not None: + return result.success + if result.o1 is not None: + raise result.o1 + if result.o2 is not None: + raise result.o2 + if result.o3 is not None: + raise result.o3 + raise TApplicationException(TApplicationException.MISSING_RESULT, "get_table_names_by_filter failed: unknown result") + + def alter_table(self, dbname, tbl_name, new_tbl): + """ + Parameters: + - dbname + - tbl_name + - new_tbl + + """ + self.send_alter_table(dbname, tbl_name, new_tbl) + self.recv_alter_table() + + def send_alter_table(self, dbname, tbl_name, new_tbl): + self._oprot.writeMessageBegin("alter_table", TMessageType.CALL, self._seqid) + args = alter_table_args() + args.dbname = dbname + args.tbl_name = tbl_name + args.new_tbl = new_tbl + args.write(self._oprot) + self._oprot.writeMessageEnd() + self._oprot.trans.flush() + + def recv_alter_table(self): + iprot = self._iprot + (fname, mtype, rseqid) = iprot.readMessageBegin() + if mtype == TMessageType.EXCEPTION: + x = TApplicationException() + x.read(iprot) + iprot.readMessageEnd() + raise x + result = alter_table_result() + result.read(iprot) + iprot.readMessageEnd() + if result.o1 is not None: + raise result.o1 + if result.o2 is not None: + raise result.o2 + return + + def alter_table_with_environment_context(self, dbname, tbl_name, new_tbl, environment_context): + """ + Parameters: + - dbname + - tbl_name + - new_tbl + - environment_context + + """ + self.send_alter_table_with_environment_context(dbname, tbl_name, new_tbl, environment_context) + self.recv_alter_table_with_environment_context() + + def send_alter_table_with_environment_context(self, dbname, tbl_name, new_tbl, environment_context): + self._oprot.writeMessageBegin("alter_table_with_environment_context", TMessageType.CALL, self._seqid) + args = alter_table_with_environment_context_args() + args.dbname = dbname + args.tbl_name = tbl_name + args.new_tbl = new_tbl + args.environment_context = environment_context + args.write(self._oprot) + self._oprot.writeMessageEnd() + self._oprot.trans.flush() + + def recv_alter_table_with_environment_context(self): + iprot = self._iprot + (fname, mtype, rseqid) = iprot.readMessageBegin() + if mtype == TMessageType.EXCEPTION: + x = TApplicationException() + x.read(iprot) + iprot.readMessageEnd() + raise x + result = alter_table_with_environment_context_result() + result.read(iprot) + iprot.readMessageEnd() + if result.o1 is not None: + raise result.o1 + if result.o2 is not None: + raise result.o2 + return + + def alter_table_with_cascade(self, dbname, tbl_name, new_tbl, cascade): + """ + Parameters: + - dbname + - tbl_name + - new_tbl + - cascade + + """ + self.send_alter_table_with_cascade(dbname, tbl_name, new_tbl, cascade) + self.recv_alter_table_with_cascade() + + def send_alter_table_with_cascade(self, dbname, tbl_name, new_tbl, cascade): + self._oprot.writeMessageBegin("alter_table_with_cascade", TMessageType.CALL, self._seqid) + args = alter_table_with_cascade_args() + args.dbname = dbname + args.tbl_name = tbl_name + args.new_tbl = new_tbl + args.cascade = cascade + args.write(self._oprot) + self._oprot.writeMessageEnd() + self._oprot.trans.flush() + + def recv_alter_table_with_cascade(self): + iprot = self._iprot + (fname, mtype, rseqid) = iprot.readMessageBegin() + if mtype == TMessageType.EXCEPTION: + x = TApplicationException() + x.read(iprot) + iprot.readMessageEnd() + raise x + result = alter_table_with_cascade_result() + result.read(iprot) + iprot.readMessageEnd() + if result.o1 is not None: + raise result.o1 + if result.o2 is not None: + raise result.o2 + return + + def alter_table_req(self, req): + """ + Parameters: + - req + + """ + self.send_alter_table_req(req) + return self.recv_alter_table_req() + + def send_alter_table_req(self, req): + self._oprot.writeMessageBegin("alter_table_req", TMessageType.CALL, self._seqid) + args = alter_table_req_args() + args.req = req + args.write(self._oprot) + self._oprot.writeMessageEnd() + self._oprot.trans.flush() + + def recv_alter_table_req(self): + iprot = self._iprot + (fname, mtype, rseqid) = iprot.readMessageBegin() + if mtype == TMessageType.EXCEPTION: + x = TApplicationException() + x.read(iprot) + iprot.readMessageEnd() + raise x + result = alter_table_req_result() + result.read(iprot) + iprot.readMessageEnd() + if result.success is not None: + return result.success + if result.o1 is not None: + raise result.o1 + if result.o2 is not None: + raise result.o2 + raise TApplicationException(TApplicationException.MISSING_RESULT, "alter_table_req failed: unknown result") + + def add_partition(self, new_part): + """ + Parameters: + - new_part + + """ + self.send_add_partition(new_part) + return self.recv_add_partition() + + def send_add_partition(self, new_part): + self._oprot.writeMessageBegin("add_partition", TMessageType.CALL, self._seqid) + args = add_partition_args() + args.new_part = new_part + args.write(self._oprot) + self._oprot.writeMessageEnd() + self._oprot.trans.flush() + + def recv_add_partition(self): + iprot = self._iprot + (fname, mtype, rseqid) = iprot.readMessageBegin() + if mtype == TMessageType.EXCEPTION: + x = TApplicationException() + x.read(iprot) + iprot.readMessageEnd() + raise x + result = add_partition_result() + result.read(iprot) + iprot.readMessageEnd() + if result.success is not None: + return result.success + if result.o1 is not None: + raise result.o1 + if result.o2 is not None: + raise result.o2 + if result.o3 is not None: + raise result.o3 + raise TApplicationException(TApplicationException.MISSING_RESULT, "add_partition failed: unknown result") + + def add_partition_with_environment_context(self, new_part, environment_context): + """ + Parameters: + - new_part + - environment_context + + """ + self.send_add_partition_with_environment_context(new_part, environment_context) + return self.recv_add_partition_with_environment_context() + + def send_add_partition_with_environment_context(self, new_part, environment_context): + self._oprot.writeMessageBegin("add_partition_with_environment_context", TMessageType.CALL, self._seqid) + args = add_partition_with_environment_context_args() + args.new_part = new_part + args.environment_context = environment_context + args.write(self._oprot) + self._oprot.writeMessageEnd() + self._oprot.trans.flush() + + def recv_add_partition_with_environment_context(self): + iprot = self._iprot + (fname, mtype, rseqid) = iprot.readMessageBegin() + if mtype == TMessageType.EXCEPTION: + x = TApplicationException() + x.read(iprot) + iprot.readMessageEnd() + raise x + result = add_partition_with_environment_context_result() + result.read(iprot) + iprot.readMessageEnd() + if result.success is not None: + return result.success + if result.o1 is not None: + raise result.o1 + if result.o2 is not None: + raise result.o2 + if result.o3 is not None: + raise result.o3 + raise TApplicationException( + TApplicationException.MISSING_RESULT, "add_partition_with_environment_context failed: unknown result" + ) + + def add_partitions(self, new_parts): + """ + Parameters: + - new_parts + + """ + self.send_add_partitions(new_parts) + return self.recv_add_partitions() + + def send_add_partitions(self, new_parts): + self._oprot.writeMessageBegin("add_partitions", TMessageType.CALL, self._seqid) + args = add_partitions_args() + args.new_parts = new_parts + args.write(self._oprot) + self._oprot.writeMessageEnd() + self._oprot.trans.flush() + + def recv_add_partitions(self): + iprot = self._iprot + (fname, mtype, rseqid) = iprot.readMessageBegin() + if mtype == TMessageType.EXCEPTION: + x = TApplicationException() + x.read(iprot) + iprot.readMessageEnd() + raise x + result = add_partitions_result() + result.read(iprot) + iprot.readMessageEnd() + if result.success is not None: + return result.success + if result.o1 is not None: + raise result.o1 + if result.o2 is not None: + raise result.o2 + if result.o3 is not None: + raise result.o3 + raise TApplicationException(TApplicationException.MISSING_RESULT, "add_partitions failed: unknown result") + + def add_partitions_pspec(self, new_parts): + """ + Parameters: + - new_parts + + """ + self.send_add_partitions_pspec(new_parts) + return self.recv_add_partitions_pspec() + + def send_add_partitions_pspec(self, new_parts): + self._oprot.writeMessageBegin("add_partitions_pspec", TMessageType.CALL, self._seqid) + args = add_partitions_pspec_args() + args.new_parts = new_parts + args.write(self._oprot) + self._oprot.writeMessageEnd() + self._oprot.trans.flush() + + def recv_add_partitions_pspec(self): + iprot = self._iprot + (fname, mtype, rseqid) = iprot.readMessageBegin() + if mtype == TMessageType.EXCEPTION: + x = TApplicationException() + x.read(iprot) + iprot.readMessageEnd() + raise x + result = add_partitions_pspec_result() + result.read(iprot) + iprot.readMessageEnd() + if result.success is not None: + return result.success + if result.o1 is not None: + raise result.o1 + if result.o2 is not None: + raise result.o2 + if result.o3 is not None: + raise result.o3 + raise TApplicationException(TApplicationException.MISSING_RESULT, "add_partitions_pspec failed: unknown result") + + def append_partition(self, db_name, tbl_name, part_vals): + """ + Parameters: + - db_name + - tbl_name + - part_vals + + """ + self.send_append_partition(db_name, tbl_name, part_vals) + return self.recv_append_partition() + + def send_append_partition(self, db_name, tbl_name, part_vals): + self._oprot.writeMessageBegin("append_partition", TMessageType.CALL, self._seqid) + args = append_partition_args() + args.db_name = db_name + args.tbl_name = tbl_name + args.part_vals = part_vals + args.write(self._oprot) + self._oprot.writeMessageEnd() + self._oprot.trans.flush() + + def recv_append_partition(self): + iprot = self._iprot + (fname, mtype, rseqid) = iprot.readMessageBegin() + if mtype == TMessageType.EXCEPTION: + x = TApplicationException() + x.read(iprot) + iprot.readMessageEnd() + raise x + result = append_partition_result() + result.read(iprot) + iprot.readMessageEnd() + if result.success is not None: + return result.success + if result.o1 is not None: + raise result.o1 + if result.o2 is not None: + raise result.o2 + if result.o3 is not None: + raise result.o3 + raise TApplicationException(TApplicationException.MISSING_RESULT, "append_partition failed: unknown result") + + def add_partitions_req(self, request): + """ + Parameters: + - request + + """ + self.send_add_partitions_req(request) + return self.recv_add_partitions_req() + + def send_add_partitions_req(self, request): + self._oprot.writeMessageBegin("add_partitions_req", TMessageType.CALL, self._seqid) + args = add_partitions_req_args() + args.request = request + args.write(self._oprot) + self._oprot.writeMessageEnd() + self._oprot.trans.flush() + + def recv_add_partitions_req(self): + iprot = self._iprot + (fname, mtype, rseqid) = iprot.readMessageBegin() + if mtype == TMessageType.EXCEPTION: + x = TApplicationException() + x.read(iprot) + iprot.readMessageEnd() + raise x + result = add_partitions_req_result() + result.read(iprot) + iprot.readMessageEnd() + if result.success is not None: + return result.success + if result.o1 is not None: + raise result.o1 + if result.o2 is not None: + raise result.o2 + if result.o3 is not None: + raise result.o3 + raise TApplicationException(TApplicationException.MISSING_RESULT, "add_partitions_req failed: unknown result") + + def append_partition_with_environment_context(self, db_name, tbl_name, part_vals, environment_context): + """ + Parameters: + - db_name + - tbl_name + - part_vals + - environment_context + + """ + self.send_append_partition_with_environment_context(db_name, tbl_name, part_vals, environment_context) + return self.recv_append_partition_with_environment_context() + + def send_append_partition_with_environment_context(self, db_name, tbl_name, part_vals, environment_context): + self._oprot.writeMessageBegin("append_partition_with_environment_context", TMessageType.CALL, self._seqid) + args = append_partition_with_environment_context_args() + args.db_name = db_name + args.tbl_name = tbl_name + args.part_vals = part_vals + args.environment_context = environment_context + args.write(self._oprot) + self._oprot.writeMessageEnd() + self._oprot.trans.flush() + + def recv_append_partition_with_environment_context(self): + iprot = self._iprot + (fname, mtype, rseqid) = iprot.readMessageBegin() + if mtype == TMessageType.EXCEPTION: + x = TApplicationException() + x.read(iprot) + iprot.readMessageEnd() + raise x + result = append_partition_with_environment_context_result() + result.read(iprot) + iprot.readMessageEnd() + if result.success is not None: + return result.success + if result.o1 is not None: + raise result.o1 + if result.o2 is not None: + raise result.o2 + if result.o3 is not None: + raise result.o3 + raise TApplicationException( + TApplicationException.MISSING_RESULT, "append_partition_with_environment_context failed: unknown result" + ) + + def append_partition_by_name(self, db_name, tbl_name, part_name): + """ + Parameters: + - db_name + - tbl_name + - part_name + + """ + self.send_append_partition_by_name(db_name, tbl_name, part_name) + return self.recv_append_partition_by_name() + + def send_append_partition_by_name(self, db_name, tbl_name, part_name): + self._oprot.writeMessageBegin("append_partition_by_name", TMessageType.CALL, self._seqid) + args = append_partition_by_name_args() + args.db_name = db_name + args.tbl_name = tbl_name + args.part_name = part_name + args.write(self._oprot) + self._oprot.writeMessageEnd() + self._oprot.trans.flush() + + def recv_append_partition_by_name(self): + iprot = self._iprot + (fname, mtype, rseqid) = iprot.readMessageBegin() + if mtype == TMessageType.EXCEPTION: + x = TApplicationException() + x.read(iprot) + iprot.readMessageEnd() + raise x + result = append_partition_by_name_result() + result.read(iprot) + iprot.readMessageEnd() + if result.success is not None: + return result.success + if result.o1 is not None: + raise result.o1 + if result.o2 is not None: + raise result.o2 + if result.o3 is not None: + raise result.o3 + raise TApplicationException(TApplicationException.MISSING_RESULT, "append_partition_by_name failed: unknown result") + + def append_partition_by_name_with_environment_context(self, db_name, tbl_name, part_name, environment_context): + """ + Parameters: + - db_name + - tbl_name + - part_name + - environment_context + + """ + self.send_append_partition_by_name_with_environment_context(db_name, tbl_name, part_name, environment_context) + return self.recv_append_partition_by_name_with_environment_context() + + def send_append_partition_by_name_with_environment_context(self, db_name, tbl_name, part_name, environment_context): + self._oprot.writeMessageBegin("append_partition_by_name_with_environment_context", TMessageType.CALL, self._seqid) + args = append_partition_by_name_with_environment_context_args() + args.db_name = db_name + args.tbl_name = tbl_name + args.part_name = part_name + args.environment_context = environment_context + args.write(self._oprot) + self._oprot.writeMessageEnd() + self._oprot.trans.flush() + + def recv_append_partition_by_name_with_environment_context(self): + iprot = self._iprot + (fname, mtype, rseqid) = iprot.readMessageBegin() + if mtype == TMessageType.EXCEPTION: + x = TApplicationException() + x.read(iprot) + iprot.readMessageEnd() + raise x + result = append_partition_by_name_with_environment_context_result() + result.read(iprot) + iprot.readMessageEnd() + if result.success is not None: + return result.success + if result.o1 is not None: + raise result.o1 + if result.o2 is not None: + raise result.o2 + if result.o3 is not None: + raise result.o3 + raise TApplicationException( + TApplicationException.MISSING_RESULT, "append_partition_by_name_with_environment_context failed: unknown result" + ) + + def drop_partition(self, db_name, tbl_name, part_vals, deleteData): + """ + Parameters: + - db_name + - tbl_name + - part_vals + - deleteData + + """ + self.send_drop_partition(db_name, tbl_name, part_vals, deleteData) + return self.recv_drop_partition() + + def send_drop_partition(self, db_name, tbl_name, part_vals, deleteData): + self._oprot.writeMessageBegin("drop_partition", TMessageType.CALL, self._seqid) + args = drop_partition_args() + args.db_name = db_name + args.tbl_name = tbl_name + args.part_vals = part_vals + args.deleteData = deleteData + args.write(self._oprot) + self._oprot.writeMessageEnd() + self._oprot.trans.flush() + + def recv_drop_partition(self): + iprot = self._iprot + (fname, mtype, rseqid) = iprot.readMessageBegin() + if mtype == TMessageType.EXCEPTION: + x = TApplicationException() + x.read(iprot) + iprot.readMessageEnd() + raise x + result = drop_partition_result() + result.read(iprot) + iprot.readMessageEnd() + if result.success is not None: + return result.success + if result.o1 is not None: + raise result.o1 + if result.o2 is not None: + raise result.o2 + raise TApplicationException(TApplicationException.MISSING_RESULT, "drop_partition failed: unknown result") + + def drop_partition_with_environment_context(self, db_name, tbl_name, part_vals, deleteData, environment_context): + """ + Parameters: + - db_name + - tbl_name + - part_vals + - deleteData + - environment_context + + """ + self.send_drop_partition_with_environment_context(db_name, tbl_name, part_vals, deleteData, environment_context) + return self.recv_drop_partition_with_environment_context() + + def send_drop_partition_with_environment_context(self, db_name, tbl_name, part_vals, deleteData, environment_context): + self._oprot.writeMessageBegin("drop_partition_with_environment_context", TMessageType.CALL, self._seqid) + args = drop_partition_with_environment_context_args() + args.db_name = db_name + args.tbl_name = tbl_name + args.part_vals = part_vals + args.deleteData = deleteData + args.environment_context = environment_context + args.write(self._oprot) + self._oprot.writeMessageEnd() + self._oprot.trans.flush() + + def recv_drop_partition_with_environment_context(self): + iprot = self._iprot + (fname, mtype, rseqid) = iprot.readMessageBegin() + if mtype == TMessageType.EXCEPTION: + x = TApplicationException() + x.read(iprot) + iprot.readMessageEnd() + raise x + result = drop_partition_with_environment_context_result() + result.read(iprot) + iprot.readMessageEnd() + if result.success is not None: + return result.success + if result.o1 is not None: + raise result.o1 + if result.o2 is not None: + raise result.o2 + raise TApplicationException( + TApplicationException.MISSING_RESULT, "drop_partition_with_environment_context failed: unknown result" + ) + + def drop_partition_by_name(self, db_name, tbl_name, part_name, deleteData): + """ + Parameters: + - db_name + - tbl_name + - part_name + - deleteData + + """ + self.send_drop_partition_by_name(db_name, tbl_name, part_name, deleteData) + return self.recv_drop_partition_by_name() + + def send_drop_partition_by_name(self, db_name, tbl_name, part_name, deleteData): + self._oprot.writeMessageBegin("drop_partition_by_name", TMessageType.CALL, self._seqid) + args = drop_partition_by_name_args() + args.db_name = db_name + args.tbl_name = tbl_name + args.part_name = part_name + args.deleteData = deleteData + args.write(self._oprot) + self._oprot.writeMessageEnd() + self._oprot.trans.flush() + + def recv_drop_partition_by_name(self): + iprot = self._iprot + (fname, mtype, rseqid) = iprot.readMessageBegin() + if mtype == TMessageType.EXCEPTION: + x = TApplicationException() + x.read(iprot) + iprot.readMessageEnd() + raise x + result = drop_partition_by_name_result() + result.read(iprot) + iprot.readMessageEnd() + if result.success is not None: + return result.success + if result.o1 is not None: + raise result.o1 + if result.o2 is not None: + raise result.o2 + raise TApplicationException(TApplicationException.MISSING_RESULT, "drop_partition_by_name failed: unknown result") + + def drop_partition_by_name_with_environment_context(self, db_name, tbl_name, part_name, deleteData, environment_context): + """ + Parameters: + - db_name + - tbl_name + - part_name + - deleteData + - environment_context + + """ + self.send_drop_partition_by_name_with_environment_context(db_name, tbl_name, part_name, deleteData, environment_context) + return self.recv_drop_partition_by_name_with_environment_context() + + def send_drop_partition_by_name_with_environment_context(self, db_name, tbl_name, part_name, deleteData, environment_context): + self._oprot.writeMessageBegin("drop_partition_by_name_with_environment_context", TMessageType.CALL, self._seqid) + args = drop_partition_by_name_with_environment_context_args() + args.db_name = db_name + args.tbl_name = tbl_name + args.part_name = part_name + args.deleteData = deleteData + args.environment_context = environment_context + args.write(self._oprot) + self._oprot.writeMessageEnd() + self._oprot.trans.flush() + + def recv_drop_partition_by_name_with_environment_context(self): + iprot = self._iprot + (fname, mtype, rseqid) = iprot.readMessageBegin() + if mtype == TMessageType.EXCEPTION: + x = TApplicationException() + x.read(iprot) + iprot.readMessageEnd() + raise x + result = drop_partition_by_name_with_environment_context_result() + result.read(iprot) + iprot.readMessageEnd() + if result.success is not None: + return result.success + if result.o1 is not None: + raise result.o1 + if result.o2 is not None: + raise result.o2 + raise TApplicationException( + TApplicationException.MISSING_RESULT, "drop_partition_by_name_with_environment_context failed: unknown result" + ) + + def drop_partitions_req(self, req): + """ + Parameters: + - req + + """ + self.send_drop_partitions_req(req) + return self.recv_drop_partitions_req() + + def send_drop_partitions_req(self, req): + self._oprot.writeMessageBegin("drop_partitions_req", TMessageType.CALL, self._seqid) + args = drop_partitions_req_args() + args.req = req + args.write(self._oprot) + self._oprot.writeMessageEnd() + self._oprot.trans.flush() + + def recv_drop_partitions_req(self): + iprot = self._iprot + (fname, mtype, rseqid) = iprot.readMessageBegin() + if mtype == TMessageType.EXCEPTION: + x = TApplicationException() + x.read(iprot) + iprot.readMessageEnd() + raise x + result = drop_partitions_req_result() + result.read(iprot) + iprot.readMessageEnd() + if result.success is not None: + return result.success + if result.o1 is not None: + raise result.o1 + if result.o2 is not None: + raise result.o2 + raise TApplicationException(TApplicationException.MISSING_RESULT, "drop_partitions_req failed: unknown result") + + def get_partition(self, db_name, tbl_name, part_vals): + """ + Parameters: + - db_name + - tbl_name + - part_vals + + """ + self.send_get_partition(db_name, tbl_name, part_vals) + return self.recv_get_partition() + + def send_get_partition(self, db_name, tbl_name, part_vals): + self._oprot.writeMessageBegin("get_partition", TMessageType.CALL, self._seqid) + args = get_partition_args() + args.db_name = db_name + args.tbl_name = tbl_name + args.part_vals = part_vals + args.write(self._oprot) + self._oprot.writeMessageEnd() + self._oprot.trans.flush() + + def recv_get_partition(self): + iprot = self._iprot + (fname, mtype, rseqid) = iprot.readMessageBegin() + if mtype == TMessageType.EXCEPTION: + x = TApplicationException() + x.read(iprot) + iprot.readMessageEnd() + raise x + result = get_partition_result() + result.read(iprot) + iprot.readMessageEnd() + if result.success is not None: + return result.success + if result.o1 is not None: + raise result.o1 + if result.o2 is not None: + raise result.o2 + raise TApplicationException(TApplicationException.MISSING_RESULT, "get_partition failed: unknown result") + + def get_partition_req(self, req): + """ + Parameters: + - req + + """ + self.send_get_partition_req(req) + return self.recv_get_partition_req() + + def send_get_partition_req(self, req): + self._oprot.writeMessageBegin("get_partition_req", TMessageType.CALL, self._seqid) + args = get_partition_req_args() + args.req = req + args.write(self._oprot) + self._oprot.writeMessageEnd() + self._oprot.trans.flush() + + def recv_get_partition_req(self): + iprot = self._iprot + (fname, mtype, rseqid) = iprot.readMessageBegin() + if mtype == TMessageType.EXCEPTION: + x = TApplicationException() + x.read(iprot) + iprot.readMessageEnd() + raise x + result = get_partition_req_result() + result.read(iprot) + iprot.readMessageEnd() + if result.success is not None: + return result.success + if result.o1 is not None: + raise result.o1 + if result.o2 is not None: + raise result.o2 + raise TApplicationException(TApplicationException.MISSING_RESULT, "get_partition_req failed: unknown result") + + def exchange_partition(self, partitionSpecs, source_db, source_table_name, dest_db, dest_table_name): + """ + Parameters: + - partitionSpecs + - source_db + - source_table_name + - dest_db + - dest_table_name + + """ + self.send_exchange_partition(partitionSpecs, source_db, source_table_name, dest_db, dest_table_name) + return self.recv_exchange_partition() + + def send_exchange_partition(self, partitionSpecs, source_db, source_table_name, dest_db, dest_table_name): + self._oprot.writeMessageBegin("exchange_partition", TMessageType.CALL, self._seqid) + args = exchange_partition_args() + args.partitionSpecs = partitionSpecs + args.source_db = source_db + args.source_table_name = source_table_name + args.dest_db = dest_db + args.dest_table_name = dest_table_name + args.write(self._oprot) + self._oprot.writeMessageEnd() + self._oprot.trans.flush() + + def recv_exchange_partition(self): + iprot = self._iprot + (fname, mtype, rseqid) = iprot.readMessageBegin() + if mtype == TMessageType.EXCEPTION: + x = TApplicationException() + x.read(iprot) + iprot.readMessageEnd() + raise x + result = exchange_partition_result() + result.read(iprot) + iprot.readMessageEnd() + if result.success is not None: + return result.success + if result.o1 is not None: + raise result.o1 + if result.o2 is not None: + raise result.o2 + if result.o3 is not None: + raise result.o3 + if result.o4 is not None: + raise result.o4 + raise TApplicationException(TApplicationException.MISSING_RESULT, "exchange_partition failed: unknown result") + + def exchange_partitions(self, partitionSpecs, source_db, source_table_name, dest_db, dest_table_name): + """ + Parameters: + - partitionSpecs + - source_db + - source_table_name + - dest_db + - dest_table_name + + """ + self.send_exchange_partitions(partitionSpecs, source_db, source_table_name, dest_db, dest_table_name) + return self.recv_exchange_partitions() + + def send_exchange_partitions(self, partitionSpecs, source_db, source_table_name, dest_db, dest_table_name): + self._oprot.writeMessageBegin("exchange_partitions", TMessageType.CALL, self._seqid) + args = exchange_partitions_args() + args.partitionSpecs = partitionSpecs + args.source_db = source_db + args.source_table_name = source_table_name + args.dest_db = dest_db + args.dest_table_name = dest_table_name + args.write(self._oprot) + self._oprot.writeMessageEnd() + self._oprot.trans.flush() + + def recv_exchange_partitions(self): + iprot = self._iprot + (fname, mtype, rseqid) = iprot.readMessageBegin() + if mtype == TMessageType.EXCEPTION: + x = TApplicationException() + x.read(iprot) + iprot.readMessageEnd() + raise x + result = exchange_partitions_result() + result.read(iprot) + iprot.readMessageEnd() + if result.success is not None: + return result.success + if result.o1 is not None: + raise result.o1 + if result.o2 is not None: + raise result.o2 + if result.o3 is not None: + raise result.o3 + if result.o4 is not None: + raise result.o4 + raise TApplicationException(TApplicationException.MISSING_RESULT, "exchange_partitions failed: unknown result") + + def get_partition_with_auth(self, db_name, tbl_name, part_vals, user_name, group_names): + """ + Parameters: + - db_name + - tbl_name + - part_vals + - user_name + - group_names + + """ + self.send_get_partition_with_auth(db_name, tbl_name, part_vals, user_name, group_names) + return self.recv_get_partition_with_auth() + + def send_get_partition_with_auth(self, db_name, tbl_name, part_vals, user_name, group_names): + self._oprot.writeMessageBegin("get_partition_with_auth", TMessageType.CALL, self._seqid) + args = get_partition_with_auth_args() + args.db_name = db_name + args.tbl_name = tbl_name + args.part_vals = part_vals + args.user_name = user_name + args.group_names = group_names + args.write(self._oprot) + self._oprot.writeMessageEnd() + self._oprot.trans.flush() + + def recv_get_partition_with_auth(self): + iprot = self._iprot + (fname, mtype, rseqid) = iprot.readMessageBegin() + if mtype == TMessageType.EXCEPTION: + x = TApplicationException() + x.read(iprot) + iprot.readMessageEnd() + raise x + result = get_partition_with_auth_result() + result.read(iprot) + iprot.readMessageEnd() + if result.success is not None: + return result.success + if result.o1 is not None: + raise result.o1 + if result.o2 is not None: + raise result.o2 + raise TApplicationException(TApplicationException.MISSING_RESULT, "get_partition_with_auth failed: unknown result") + + def get_partition_by_name(self, db_name, tbl_name, part_name): + """ + Parameters: + - db_name + - tbl_name + - part_name + + """ + self.send_get_partition_by_name(db_name, tbl_name, part_name) + return self.recv_get_partition_by_name() + + def send_get_partition_by_name(self, db_name, tbl_name, part_name): + self._oprot.writeMessageBegin("get_partition_by_name", TMessageType.CALL, self._seqid) + args = get_partition_by_name_args() + args.db_name = db_name + args.tbl_name = tbl_name + args.part_name = part_name + args.write(self._oprot) + self._oprot.writeMessageEnd() + self._oprot.trans.flush() + + def recv_get_partition_by_name(self): + iprot = self._iprot + (fname, mtype, rseqid) = iprot.readMessageBegin() + if mtype == TMessageType.EXCEPTION: + x = TApplicationException() + x.read(iprot) + iprot.readMessageEnd() + raise x + result = get_partition_by_name_result() + result.read(iprot) + iprot.readMessageEnd() + if result.success is not None: + return result.success + if result.o1 is not None: + raise result.o1 + if result.o2 is not None: + raise result.o2 + raise TApplicationException(TApplicationException.MISSING_RESULT, "get_partition_by_name failed: unknown result") + + def get_partitions(self, db_name, tbl_name, max_parts): + """ + Parameters: + - db_name + - tbl_name + - max_parts + + """ + self.send_get_partitions(db_name, tbl_name, max_parts) + return self.recv_get_partitions() + + def send_get_partitions(self, db_name, tbl_name, max_parts): + self._oprot.writeMessageBegin("get_partitions", TMessageType.CALL, self._seqid) + args = get_partitions_args() + args.db_name = db_name + args.tbl_name = tbl_name + args.max_parts = max_parts + args.write(self._oprot) + self._oprot.writeMessageEnd() + self._oprot.trans.flush() + + def recv_get_partitions(self): + iprot = self._iprot + (fname, mtype, rseqid) = iprot.readMessageBegin() + if mtype == TMessageType.EXCEPTION: + x = TApplicationException() + x.read(iprot) + iprot.readMessageEnd() + raise x + result = get_partitions_result() + result.read(iprot) + iprot.readMessageEnd() + if result.success is not None: + return result.success + if result.o1 is not None: + raise result.o1 + if result.o2 is not None: + raise result.o2 + raise TApplicationException(TApplicationException.MISSING_RESULT, "get_partitions failed: unknown result") + + def get_partitions_req(self, req): + """ + Parameters: + - req + + """ + self.send_get_partitions_req(req) + return self.recv_get_partitions_req() + + def send_get_partitions_req(self, req): + self._oprot.writeMessageBegin("get_partitions_req", TMessageType.CALL, self._seqid) + args = get_partitions_req_args() + args.req = req + args.write(self._oprot) + self._oprot.writeMessageEnd() + self._oprot.trans.flush() + + def recv_get_partitions_req(self): + iprot = self._iprot + (fname, mtype, rseqid) = iprot.readMessageBegin() + if mtype == TMessageType.EXCEPTION: + x = TApplicationException() + x.read(iprot) + iprot.readMessageEnd() + raise x + result = get_partitions_req_result() + result.read(iprot) + iprot.readMessageEnd() + if result.success is not None: + return result.success + if result.o1 is not None: + raise result.o1 + if result.o2 is not None: + raise result.o2 + raise TApplicationException(TApplicationException.MISSING_RESULT, "get_partitions_req failed: unknown result") + + def get_partitions_with_auth(self, db_name, tbl_name, max_parts, user_name, group_names): + """ + Parameters: + - db_name + - tbl_name + - max_parts + - user_name + - group_names + + """ + self.send_get_partitions_with_auth(db_name, tbl_name, max_parts, user_name, group_names) + return self.recv_get_partitions_with_auth() + + def send_get_partitions_with_auth(self, db_name, tbl_name, max_parts, user_name, group_names): + self._oprot.writeMessageBegin("get_partitions_with_auth", TMessageType.CALL, self._seqid) + args = get_partitions_with_auth_args() + args.db_name = db_name + args.tbl_name = tbl_name + args.max_parts = max_parts + args.user_name = user_name + args.group_names = group_names + args.write(self._oprot) + self._oprot.writeMessageEnd() + self._oprot.trans.flush() + + def recv_get_partitions_with_auth(self): + iprot = self._iprot + (fname, mtype, rseqid) = iprot.readMessageBegin() + if mtype == TMessageType.EXCEPTION: + x = TApplicationException() + x.read(iprot) + iprot.readMessageEnd() + raise x + result = get_partitions_with_auth_result() + result.read(iprot) + iprot.readMessageEnd() + if result.success is not None: + return result.success + if result.o1 is not None: + raise result.o1 + if result.o2 is not None: + raise result.o2 + raise TApplicationException(TApplicationException.MISSING_RESULT, "get_partitions_with_auth failed: unknown result") + + def get_partitions_pspec(self, db_name, tbl_name, max_parts): + """ + Parameters: + - db_name + - tbl_name + - max_parts + + """ + self.send_get_partitions_pspec(db_name, tbl_name, max_parts) + return self.recv_get_partitions_pspec() + + def send_get_partitions_pspec(self, db_name, tbl_name, max_parts): + self._oprot.writeMessageBegin("get_partitions_pspec", TMessageType.CALL, self._seqid) + args = get_partitions_pspec_args() + args.db_name = db_name + args.tbl_name = tbl_name + args.max_parts = max_parts + args.write(self._oprot) + self._oprot.writeMessageEnd() + self._oprot.trans.flush() + + def recv_get_partitions_pspec(self): + iprot = self._iprot + (fname, mtype, rseqid) = iprot.readMessageBegin() + if mtype == TMessageType.EXCEPTION: + x = TApplicationException() + x.read(iprot) + iprot.readMessageEnd() + raise x + result = get_partitions_pspec_result() + result.read(iprot) + iprot.readMessageEnd() + if result.success is not None: + return result.success + if result.o1 is not None: + raise result.o1 + if result.o2 is not None: + raise result.o2 + raise TApplicationException(TApplicationException.MISSING_RESULT, "get_partitions_pspec failed: unknown result") + + def get_partition_names(self, db_name, tbl_name, max_parts): + """ + Parameters: + - db_name + - tbl_name + - max_parts + + """ + self.send_get_partition_names(db_name, tbl_name, max_parts) + return self.recv_get_partition_names() + + def send_get_partition_names(self, db_name, tbl_name, max_parts): + self._oprot.writeMessageBegin("get_partition_names", TMessageType.CALL, self._seqid) + args = get_partition_names_args() + args.db_name = db_name + args.tbl_name = tbl_name + args.max_parts = max_parts + args.write(self._oprot) + self._oprot.writeMessageEnd() + self._oprot.trans.flush() + + def recv_get_partition_names(self): + iprot = self._iprot + (fname, mtype, rseqid) = iprot.readMessageBegin() + if mtype == TMessageType.EXCEPTION: + x = TApplicationException() + x.read(iprot) + iprot.readMessageEnd() + raise x + result = get_partition_names_result() + result.read(iprot) + iprot.readMessageEnd() + if result.success is not None: + return result.success + if result.o1 is not None: + raise result.o1 + if result.o2 is not None: + raise result.o2 + raise TApplicationException(TApplicationException.MISSING_RESULT, "get_partition_names failed: unknown result") + + def get_partition_values(self, request): + """ + Parameters: + - request + + """ + self.send_get_partition_values(request) + return self.recv_get_partition_values() + + def send_get_partition_values(self, request): + self._oprot.writeMessageBegin("get_partition_values", TMessageType.CALL, self._seqid) + args = get_partition_values_args() + args.request = request + args.write(self._oprot) + self._oprot.writeMessageEnd() + self._oprot.trans.flush() + + def recv_get_partition_values(self): + iprot = self._iprot + (fname, mtype, rseqid) = iprot.readMessageBegin() + if mtype == TMessageType.EXCEPTION: + x = TApplicationException() + x.read(iprot) + iprot.readMessageEnd() + raise x + result = get_partition_values_result() + result.read(iprot) + iprot.readMessageEnd() + if result.success is not None: + return result.success + if result.o1 is not None: + raise result.o1 + if result.o2 is not None: + raise result.o2 + raise TApplicationException(TApplicationException.MISSING_RESULT, "get_partition_values failed: unknown result") + + def get_partitions_ps(self, db_name, tbl_name, part_vals, max_parts): + """ + Parameters: + - db_name + - tbl_name + - part_vals + - max_parts + + """ + self.send_get_partitions_ps(db_name, tbl_name, part_vals, max_parts) + return self.recv_get_partitions_ps() + + def send_get_partitions_ps(self, db_name, tbl_name, part_vals, max_parts): + self._oprot.writeMessageBegin("get_partitions_ps", TMessageType.CALL, self._seqid) + args = get_partitions_ps_args() + args.db_name = db_name + args.tbl_name = tbl_name + args.part_vals = part_vals + args.max_parts = max_parts + args.write(self._oprot) + self._oprot.writeMessageEnd() + self._oprot.trans.flush() + + def recv_get_partitions_ps(self): + iprot = self._iprot + (fname, mtype, rseqid) = iprot.readMessageBegin() + if mtype == TMessageType.EXCEPTION: + x = TApplicationException() + x.read(iprot) + iprot.readMessageEnd() + raise x + result = get_partitions_ps_result() + result.read(iprot) + iprot.readMessageEnd() + if result.success is not None: + return result.success + if result.o1 is not None: + raise result.o1 + if result.o2 is not None: + raise result.o2 + raise TApplicationException(TApplicationException.MISSING_RESULT, "get_partitions_ps failed: unknown result") + + def get_partitions_ps_with_auth(self, db_name, tbl_name, part_vals, max_parts, user_name, group_names): + """ + Parameters: + - db_name + - tbl_name + - part_vals + - max_parts + - user_name + - group_names + + """ + self.send_get_partitions_ps_with_auth(db_name, tbl_name, part_vals, max_parts, user_name, group_names) + return self.recv_get_partitions_ps_with_auth() + + def send_get_partitions_ps_with_auth(self, db_name, tbl_name, part_vals, max_parts, user_name, group_names): + self._oprot.writeMessageBegin("get_partitions_ps_with_auth", TMessageType.CALL, self._seqid) + args = get_partitions_ps_with_auth_args() + args.db_name = db_name + args.tbl_name = tbl_name + args.part_vals = part_vals + args.max_parts = max_parts + args.user_name = user_name + args.group_names = group_names + args.write(self._oprot) + self._oprot.writeMessageEnd() + self._oprot.trans.flush() + + def recv_get_partitions_ps_with_auth(self): + iprot = self._iprot + (fname, mtype, rseqid) = iprot.readMessageBegin() + if mtype == TMessageType.EXCEPTION: + x = TApplicationException() + x.read(iprot) + iprot.readMessageEnd() + raise x + result = get_partitions_ps_with_auth_result() + result.read(iprot) + iprot.readMessageEnd() + if result.success is not None: + return result.success + if result.o1 is not None: + raise result.o1 + if result.o2 is not None: + raise result.o2 + raise TApplicationException(TApplicationException.MISSING_RESULT, "get_partitions_ps_with_auth failed: unknown result") + + def get_partitions_ps_with_auth_req(self, req): + """ + Parameters: + - req + + """ + self.send_get_partitions_ps_with_auth_req(req) + return self.recv_get_partitions_ps_with_auth_req() + + def send_get_partitions_ps_with_auth_req(self, req): + self._oprot.writeMessageBegin("get_partitions_ps_with_auth_req", TMessageType.CALL, self._seqid) + args = get_partitions_ps_with_auth_req_args() + args.req = req + args.write(self._oprot) + self._oprot.writeMessageEnd() + self._oprot.trans.flush() + + def recv_get_partitions_ps_with_auth_req(self): + iprot = self._iprot + (fname, mtype, rseqid) = iprot.readMessageBegin() + if mtype == TMessageType.EXCEPTION: + x = TApplicationException() + x.read(iprot) + iprot.readMessageEnd() + raise x + result = get_partitions_ps_with_auth_req_result() + result.read(iprot) + iprot.readMessageEnd() + if result.success is not None: + return result.success + if result.o1 is not None: + raise result.o1 + if result.o2 is not None: + raise result.o2 + raise TApplicationException( + TApplicationException.MISSING_RESULT, "get_partitions_ps_with_auth_req failed: unknown result" + ) + + def get_partition_names_ps(self, db_name, tbl_name, part_vals, max_parts): + """ + Parameters: + - db_name + - tbl_name + - part_vals + - max_parts + + """ + self.send_get_partition_names_ps(db_name, tbl_name, part_vals, max_parts) + return self.recv_get_partition_names_ps() + + def send_get_partition_names_ps(self, db_name, tbl_name, part_vals, max_parts): + self._oprot.writeMessageBegin("get_partition_names_ps", TMessageType.CALL, self._seqid) + args = get_partition_names_ps_args() + args.db_name = db_name + args.tbl_name = tbl_name + args.part_vals = part_vals + args.max_parts = max_parts + args.write(self._oprot) + self._oprot.writeMessageEnd() + self._oprot.trans.flush() + + def recv_get_partition_names_ps(self): + iprot = self._iprot + (fname, mtype, rseqid) = iprot.readMessageBegin() + if mtype == TMessageType.EXCEPTION: + x = TApplicationException() + x.read(iprot) + iprot.readMessageEnd() + raise x + result = get_partition_names_ps_result() + result.read(iprot) + iprot.readMessageEnd() + if result.success is not None: + return result.success + if result.o1 is not None: + raise result.o1 + if result.o2 is not None: + raise result.o2 + raise TApplicationException(TApplicationException.MISSING_RESULT, "get_partition_names_ps failed: unknown result") + + def get_partition_names_ps_req(self, req): + """ + Parameters: + - req + + """ + self.send_get_partition_names_ps_req(req) + return self.recv_get_partition_names_ps_req() + + def send_get_partition_names_ps_req(self, req): + self._oprot.writeMessageBegin("get_partition_names_ps_req", TMessageType.CALL, self._seqid) + args = get_partition_names_ps_req_args() + args.req = req + args.write(self._oprot) + self._oprot.writeMessageEnd() + self._oprot.trans.flush() + + def recv_get_partition_names_ps_req(self): + iprot = self._iprot + (fname, mtype, rseqid) = iprot.readMessageBegin() + if mtype == TMessageType.EXCEPTION: + x = TApplicationException() + x.read(iprot) + iprot.readMessageEnd() + raise x + result = get_partition_names_ps_req_result() + result.read(iprot) + iprot.readMessageEnd() + if result.success is not None: + return result.success + if result.o1 is not None: + raise result.o1 + if result.o2 is not None: + raise result.o2 + raise TApplicationException(TApplicationException.MISSING_RESULT, "get_partition_names_ps_req failed: unknown result") + + def get_partition_names_req(self, req): + """ + Parameters: + - req + + """ + self.send_get_partition_names_req(req) + return self.recv_get_partition_names_req() + + def send_get_partition_names_req(self, req): + self._oprot.writeMessageBegin("get_partition_names_req", TMessageType.CALL, self._seqid) + args = get_partition_names_req_args() + args.req = req + args.write(self._oprot) + self._oprot.writeMessageEnd() + self._oprot.trans.flush() + + def recv_get_partition_names_req(self): + iprot = self._iprot + (fname, mtype, rseqid) = iprot.readMessageBegin() + if mtype == TMessageType.EXCEPTION: + x = TApplicationException() + x.read(iprot) + iprot.readMessageEnd() + raise x + result = get_partition_names_req_result() + result.read(iprot) + iprot.readMessageEnd() + if result.success is not None: + return result.success + if result.o1 is not None: + raise result.o1 + if result.o2 is not None: + raise result.o2 + raise TApplicationException(TApplicationException.MISSING_RESULT, "get_partition_names_req failed: unknown result") + + def get_partitions_by_filter(self, db_name, tbl_name, filter, max_parts): + """ + Parameters: + - db_name + - tbl_name + - filter + - max_parts + + """ + self.send_get_partitions_by_filter(db_name, tbl_name, filter, max_parts) + return self.recv_get_partitions_by_filter() + + def send_get_partitions_by_filter(self, db_name, tbl_name, filter, max_parts): + self._oprot.writeMessageBegin("get_partitions_by_filter", TMessageType.CALL, self._seqid) + args = get_partitions_by_filter_args() + args.db_name = db_name + args.tbl_name = tbl_name + args.filter = filter + args.max_parts = max_parts + args.write(self._oprot) + self._oprot.writeMessageEnd() + self._oprot.trans.flush() + + def recv_get_partitions_by_filter(self): + iprot = self._iprot + (fname, mtype, rseqid) = iprot.readMessageBegin() + if mtype == TMessageType.EXCEPTION: + x = TApplicationException() + x.read(iprot) + iprot.readMessageEnd() + raise x + result = get_partitions_by_filter_result() + result.read(iprot) + iprot.readMessageEnd() + if result.success is not None: + return result.success + if result.o1 is not None: + raise result.o1 + if result.o2 is not None: + raise result.o2 + raise TApplicationException(TApplicationException.MISSING_RESULT, "get_partitions_by_filter failed: unknown result") + + def get_part_specs_by_filter(self, db_name, tbl_name, filter, max_parts): + """ + Parameters: + - db_name + - tbl_name + - filter + - max_parts + + """ + self.send_get_part_specs_by_filter(db_name, tbl_name, filter, max_parts) + return self.recv_get_part_specs_by_filter() + + def send_get_part_specs_by_filter(self, db_name, tbl_name, filter, max_parts): + self._oprot.writeMessageBegin("get_part_specs_by_filter", TMessageType.CALL, self._seqid) + args = get_part_specs_by_filter_args() + args.db_name = db_name + args.tbl_name = tbl_name + args.filter = filter + args.max_parts = max_parts + args.write(self._oprot) + self._oprot.writeMessageEnd() + self._oprot.trans.flush() + + def recv_get_part_specs_by_filter(self): + iprot = self._iprot + (fname, mtype, rseqid) = iprot.readMessageBegin() + if mtype == TMessageType.EXCEPTION: + x = TApplicationException() + x.read(iprot) + iprot.readMessageEnd() + raise x + result = get_part_specs_by_filter_result() + result.read(iprot) + iprot.readMessageEnd() + if result.success is not None: + return result.success + if result.o1 is not None: + raise result.o1 + if result.o2 is not None: + raise result.o2 + raise TApplicationException(TApplicationException.MISSING_RESULT, "get_part_specs_by_filter failed: unknown result") + + def get_partitions_by_expr(self, req): + """ + Parameters: + - req + + """ + self.send_get_partitions_by_expr(req) + return self.recv_get_partitions_by_expr() + + def send_get_partitions_by_expr(self, req): + self._oprot.writeMessageBegin("get_partitions_by_expr", TMessageType.CALL, self._seqid) + args = get_partitions_by_expr_args() + args.req = req + args.write(self._oprot) + self._oprot.writeMessageEnd() + self._oprot.trans.flush() + + def recv_get_partitions_by_expr(self): + iprot = self._iprot + (fname, mtype, rseqid) = iprot.readMessageBegin() + if mtype == TMessageType.EXCEPTION: + x = TApplicationException() + x.read(iprot) + iprot.readMessageEnd() + raise x + result = get_partitions_by_expr_result() + result.read(iprot) + iprot.readMessageEnd() + if result.success is not None: + return result.success + if result.o1 is not None: + raise result.o1 + if result.o2 is not None: + raise result.o2 + raise TApplicationException(TApplicationException.MISSING_RESULT, "get_partitions_by_expr failed: unknown result") + + def get_partitions_spec_by_expr(self, req): + """ + Parameters: + - req + + """ + self.send_get_partitions_spec_by_expr(req) + return self.recv_get_partitions_spec_by_expr() + + def send_get_partitions_spec_by_expr(self, req): + self._oprot.writeMessageBegin("get_partitions_spec_by_expr", TMessageType.CALL, self._seqid) + args = get_partitions_spec_by_expr_args() + args.req = req + args.write(self._oprot) + self._oprot.writeMessageEnd() + self._oprot.trans.flush() + + def recv_get_partitions_spec_by_expr(self): + iprot = self._iprot + (fname, mtype, rseqid) = iprot.readMessageBegin() + if mtype == TMessageType.EXCEPTION: + x = TApplicationException() + x.read(iprot) + iprot.readMessageEnd() + raise x + result = get_partitions_spec_by_expr_result() + result.read(iprot) + iprot.readMessageEnd() + if result.success is not None: + return result.success + if result.o1 is not None: + raise result.o1 + if result.o2 is not None: + raise result.o2 + raise TApplicationException(TApplicationException.MISSING_RESULT, "get_partitions_spec_by_expr failed: unknown result") + + def get_num_partitions_by_filter(self, db_name, tbl_name, filter): + """ + Parameters: + - db_name + - tbl_name + - filter + + """ + self.send_get_num_partitions_by_filter(db_name, tbl_name, filter) + return self.recv_get_num_partitions_by_filter() + + def send_get_num_partitions_by_filter(self, db_name, tbl_name, filter): + self._oprot.writeMessageBegin("get_num_partitions_by_filter", TMessageType.CALL, self._seqid) + args = get_num_partitions_by_filter_args() + args.db_name = db_name + args.tbl_name = tbl_name + args.filter = filter + args.write(self._oprot) + self._oprot.writeMessageEnd() + self._oprot.trans.flush() + + def recv_get_num_partitions_by_filter(self): + iprot = self._iprot + (fname, mtype, rseqid) = iprot.readMessageBegin() + if mtype == TMessageType.EXCEPTION: + x = TApplicationException() + x.read(iprot) + iprot.readMessageEnd() + raise x + result = get_num_partitions_by_filter_result() + result.read(iprot) + iprot.readMessageEnd() + if result.success is not None: + return result.success + if result.o1 is not None: + raise result.o1 + if result.o2 is not None: + raise result.o2 + raise TApplicationException(TApplicationException.MISSING_RESULT, "get_num_partitions_by_filter failed: unknown result") + + def get_partitions_by_names(self, db_name, tbl_name, names): + """ + Parameters: + - db_name + - tbl_name + - names + + """ + self.send_get_partitions_by_names(db_name, tbl_name, names) + return self.recv_get_partitions_by_names() + + def send_get_partitions_by_names(self, db_name, tbl_name, names): + self._oprot.writeMessageBegin("get_partitions_by_names", TMessageType.CALL, self._seqid) + args = get_partitions_by_names_args() + args.db_name = db_name + args.tbl_name = tbl_name + args.names = names + args.write(self._oprot) + self._oprot.writeMessageEnd() + self._oprot.trans.flush() + + def recv_get_partitions_by_names(self): + iprot = self._iprot + (fname, mtype, rseqid) = iprot.readMessageBegin() + if mtype == TMessageType.EXCEPTION: + x = TApplicationException() + x.read(iprot) + iprot.readMessageEnd() + raise x + result = get_partitions_by_names_result() + result.read(iprot) + iprot.readMessageEnd() + if result.success is not None: + return result.success + if result.o1 is not None: + raise result.o1 + if result.o2 is not None: + raise result.o2 + raise TApplicationException(TApplicationException.MISSING_RESULT, "get_partitions_by_names failed: unknown result") + + def get_partitions_by_names_req(self, req): + """ + Parameters: + - req + + """ + self.send_get_partitions_by_names_req(req) + return self.recv_get_partitions_by_names_req() + + def send_get_partitions_by_names_req(self, req): + self._oprot.writeMessageBegin("get_partitions_by_names_req", TMessageType.CALL, self._seqid) + args = get_partitions_by_names_req_args() + args.req = req + args.write(self._oprot) + self._oprot.writeMessageEnd() + self._oprot.trans.flush() + + def recv_get_partitions_by_names_req(self): + iprot = self._iprot + (fname, mtype, rseqid) = iprot.readMessageBegin() + if mtype == TMessageType.EXCEPTION: + x = TApplicationException() + x.read(iprot) + iprot.readMessageEnd() + raise x + result = get_partitions_by_names_req_result() + result.read(iprot) + iprot.readMessageEnd() + if result.success is not None: + return result.success + if result.o1 is not None: + raise result.o1 + if result.o2 is not None: + raise result.o2 + raise TApplicationException(TApplicationException.MISSING_RESULT, "get_partitions_by_names_req failed: unknown result") + + def alter_partition(self, db_name, tbl_name, new_part): + """ + Parameters: + - db_name + - tbl_name + - new_part + + """ + self.send_alter_partition(db_name, tbl_name, new_part) + self.recv_alter_partition() + + def send_alter_partition(self, db_name, tbl_name, new_part): + self._oprot.writeMessageBegin("alter_partition", TMessageType.CALL, self._seqid) + args = alter_partition_args() + args.db_name = db_name + args.tbl_name = tbl_name + args.new_part = new_part + args.write(self._oprot) + self._oprot.writeMessageEnd() + self._oprot.trans.flush() + + def recv_alter_partition(self): + iprot = self._iprot + (fname, mtype, rseqid) = iprot.readMessageBegin() + if mtype == TMessageType.EXCEPTION: + x = TApplicationException() + x.read(iprot) + iprot.readMessageEnd() + raise x + result = alter_partition_result() + result.read(iprot) + iprot.readMessageEnd() + if result.o1 is not None: + raise result.o1 + if result.o2 is not None: + raise result.o2 + return + + def alter_partitions(self, db_name, tbl_name, new_parts): + """ + Parameters: + - db_name + - tbl_name + - new_parts + + """ + self.send_alter_partitions(db_name, tbl_name, new_parts) + self.recv_alter_partitions() + + def send_alter_partitions(self, db_name, tbl_name, new_parts): + self._oprot.writeMessageBegin("alter_partitions", TMessageType.CALL, self._seqid) + args = alter_partitions_args() + args.db_name = db_name + args.tbl_name = tbl_name + args.new_parts = new_parts + args.write(self._oprot) + self._oprot.writeMessageEnd() + self._oprot.trans.flush() + + def recv_alter_partitions(self): + iprot = self._iprot + (fname, mtype, rseqid) = iprot.readMessageBegin() + if mtype == TMessageType.EXCEPTION: + x = TApplicationException() + x.read(iprot) + iprot.readMessageEnd() + raise x + result = alter_partitions_result() + result.read(iprot) + iprot.readMessageEnd() + if result.o1 is not None: + raise result.o1 + if result.o2 is not None: + raise result.o2 + return + + def alter_partitions_with_environment_context(self, db_name, tbl_name, new_parts, environment_context): + """ + Parameters: + - db_name + - tbl_name + - new_parts + - environment_context + + """ + self.send_alter_partitions_with_environment_context(db_name, tbl_name, new_parts, environment_context) + self.recv_alter_partitions_with_environment_context() + + def send_alter_partitions_with_environment_context(self, db_name, tbl_name, new_parts, environment_context): + self._oprot.writeMessageBegin("alter_partitions_with_environment_context", TMessageType.CALL, self._seqid) + args = alter_partitions_with_environment_context_args() + args.db_name = db_name + args.tbl_name = tbl_name + args.new_parts = new_parts + args.environment_context = environment_context + args.write(self._oprot) + self._oprot.writeMessageEnd() + self._oprot.trans.flush() + + def recv_alter_partitions_with_environment_context(self): + iprot = self._iprot + (fname, mtype, rseqid) = iprot.readMessageBegin() + if mtype == TMessageType.EXCEPTION: + x = TApplicationException() + x.read(iprot) + iprot.readMessageEnd() + raise x + result = alter_partitions_with_environment_context_result() + result.read(iprot) + iprot.readMessageEnd() + if result.o1 is not None: + raise result.o1 + if result.o2 is not None: + raise result.o2 + return + + def alter_partitions_req(self, req): + """ + Parameters: + - req + + """ + self.send_alter_partitions_req(req) + return self.recv_alter_partitions_req() + + def send_alter_partitions_req(self, req): + self._oprot.writeMessageBegin("alter_partitions_req", TMessageType.CALL, self._seqid) + args = alter_partitions_req_args() + args.req = req + args.write(self._oprot) + self._oprot.writeMessageEnd() + self._oprot.trans.flush() + + def recv_alter_partitions_req(self): + iprot = self._iprot + (fname, mtype, rseqid) = iprot.readMessageBegin() + if mtype == TMessageType.EXCEPTION: + x = TApplicationException() + x.read(iprot) + iprot.readMessageEnd() + raise x + result = alter_partitions_req_result() + result.read(iprot) + iprot.readMessageEnd() + if result.success is not None: + return result.success + if result.o1 is not None: + raise result.o1 + if result.o2 is not None: + raise result.o2 + raise TApplicationException(TApplicationException.MISSING_RESULT, "alter_partitions_req failed: unknown result") + + def alter_partition_with_environment_context(self, db_name, tbl_name, new_part, environment_context): + """ + Parameters: + - db_name + - tbl_name + - new_part + - environment_context + + """ + self.send_alter_partition_with_environment_context(db_name, tbl_name, new_part, environment_context) + self.recv_alter_partition_with_environment_context() + + def send_alter_partition_with_environment_context(self, db_name, tbl_name, new_part, environment_context): + self._oprot.writeMessageBegin("alter_partition_with_environment_context", TMessageType.CALL, self._seqid) + args = alter_partition_with_environment_context_args() + args.db_name = db_name + args.tbl_name = tbl_name + args.new_part = new_part + args.environment_context = environment_context + args.write(self._oprot) + self._oprot.writeMessageEnd() + self._oprot.trans.flush() + + def recv_alter_partition_with_environment_context(self): + iprot = self._iprot + (fname, mtype, rseqid) = iprot.readMessageBegin() + if mtype == TMessageType.EXCEPTION: + x = TApplicationException() + x.read(iprot) + iprot.readMessageEnd() + raise x + result = alter_partition_with_environment_context_result() + result.read(iprot) + iprot.readMessageEnd() + if result.o1 is not None: + raise result.o1 + if result.o2 is not None: + raise result.o2 + return + + def rename_partition(self, db_name, tbl_name, part_vals, new_part): + """ + Parameters: + - db_name + - tbl_name + - part_vals + - new_part + + """ + self.send_rename_partition(db_name, tbl_name, part_vals, new_part) + self.recv_rename_partition() + + def send_rename_partition(self, db_name, tbl_name, part_vals, new_part): + self._oprot.writeMessageBegin("rename_partition", TMessageType.CALL, self._seqid) + args = rename_partition_args() + args.db_name = db_name + args.tbl_name = tbl_name + args.part_vals = part_vals + args.new_part = new_part + args.write(self._oprot) + self._oprot.writeMessageEnd() + self._oprot.trans.flush() + + def recv_rename_partition(self): + iprot = self._iprot + (fname, mtype, rseqid) = iprot.readMessageBegin() + if mtype == TMessageType.EXCEPTION: + x = TApplicationException() + x.read(iprot) + iprot.readMessageEnd() + raise x + result = rename_partition_result() + result.read(iprot) + iprot.readMessageEnd() + if result.o1 is not None: + raise result.o1 + if result.o2 is not None: + raise result.o2 + return + + def rename_partition_req(self, req): + """ + Parameters: + - req + + """ + self.send_rename_partition_req(req) + return self.recv_rename_partition_req() + + def send_rename_partition_req(self, req): + self._oprot.writeMessageBegin("rename_partition_req", TMessageType.CALL, self._seqid) + args = rename_partition_req_args() + args.req = req + args.write(self._oprot) + self._oprot.writeMessageEnd() + self._oprot.trans.flush() + + def recv_rename_partition_req(self): + iprot = self._iprot + (fname, mtype, rseqid) = iprot.readMessageBegin() + if mtype == TMessageType.EXCEPTION: + x = TApplicationException() + x.read(iprot) + iprot.readMessageEnd() + raise x + result = rename_partition_req_result() + result.read(iprot) + iprot.readMessageEnd() + if result.success is not None: + return result.success + if result.o1 is not None: + raise result.o1 + if result.o2 is not None: + raise result.o2 + raise TApplicationException(TApplicationException.MISSING_RESULT, "rename_partition_req failed: unknown result") + + def partition_name_has_valid_characters(self, part_vals, throw_exception): + """ + Parameters: + - part_vals + - throw_exception + + """ + self.send_partition_name_has_valid_characters(part_vals, throw_exception) + return self.recv_partition_name_has_valid_characters() + + def send_partition_name_has_valid_characters(self, part_vals, throw_exception): + self._oprot.writeMessageBegin("partition_name_has_valid_characters", TMessageType.CALL, self._seqid) + args = partition_name_has_valid_characters_args() + args.part_vals = part_vals + args.throw_exception = throw_exception + args.write(self._oprot) + self._oprot.writeMessageEnd() + self._oprot.trans.flush() + + def recv_partition_name_has_valid_characters(self): + iprot = self._iprot + (fname, mtype, rseqid) = iprot.readMessageBegin() + if mtype == TMessageType.EXCEPTION: + x = TApplicationException() + x.read(iprot) + iprot.readMessageEnd() + raise x + result = partition_name_has_valid_characters_result() + result.read(iprot) + iprot.readMessageEnd() + if result.success is not None: + return result.success + if result.o1 is not None: + raise result.o1 + raise TApplicationException( + TApplicationException.MISSING_RESULT, "partition_name_has_valid_characters failed: unknown result" + ) + + def get_config_value(self, name, defaultValue): + """ + Parameters: + - name + - defaultValue + + """ + self.send_get_config_value(name, defaultValue) + return self.recv_get_config_value() + + def send_get_config_value(self, name, defaultValue): + self._oprot.writeMessageBegin("get_config_value", TMessageType.CALL, self._seqid) + args = get_config_value_args() + args.name = name + args.defaultValue = defaultValue + args.write(self._oprot) + self._oprot.writeMessageEnd() + self._oprot.trans.flush() + + def recv_get_config_value(self): + iprot = self._iprot + (fname, mtype, rseqid) = iprot.readMessageBegin() + if mtype == TMessageType.EXCEPTION: + x = TApplicationException() + x.read(iprot) + iprot.readMessageEnd() + raise x + result = get_config_value_result() + result.read(iprot) + iprot.readMessageEnd() + if result.success is not None: + return result.success + if result.o1 is not None: + raise result.o1 + raise TApplicationException(TApplicationException.MISSING_RESULT, "get_config_value failed: unknown result") + + def partition_name_to_vals(self, part_name): + """ + Parameters: + - part_name + + """ + self.send_partition_name_to_vals(part_name) + return self.recv_partition_name_to_vals() + + def send_partition_name_to_vals(self, part_name): + self._oprot.writeMessageBegin("partition_name_to_vals", TMessageType.CALL, self._seqid) + args = partition_name_to_vals_args() + args.part_name = part_name + args.write(self._oprot) + self._oprot.writeMessageEnd() + self._oprot.trans.flush() + + def recv_partition_name_to_vals(self): + iprot = self._iprot + (fname, mtype, rseqid) = iprot.readMessageBegin() + if mtype == TMessageType.EXCEPTION: + x = TApplicationException() + x.read(iprot) + iprot.readMessageEnd() + raise x + result = partition_name_to_vals_result() + result.read(iprot) + iprot.readMessageEnd() + if result.success is not None: + return result.success + if result.o1 is not None: + raise result.o1 + raise TApplicationException(TApplicationException.MISSING_RESULT, "partition_name_to_vals failed: unknown result") + + def partition_name_to_spec(self, part_name): + """ + Parameters: + - part_name + + """ + self.send_partition_name_to_spec(part_name) + return self.recv_partition_name_to_spec() + + def send_partition_name_to_spec(self, part_name): + self._oprot.writeMessageBegin("partition_name_to_spec", TMessageType.CALL, self._seqid) + args = partition_name_to_spec_args() + args.part_name = part_name + args.write(self._oprot) + self._oprot.writeMessageEnd() + self._oprot.trans.flush() + + def recv_partition_name_to_spec(self): + iprot = self._iprot + (fname, mtype, rseqid) = iprot.readMessageBegin() + if mtype == TMessageType.EXCEPTION: + x = TApplicationException() + x.read(iprot) + iprot.readMessageEnd() + raise x + result = partition_name_to_spec_result() + result.read(iprot) + iprot.readMessageEnd() + if result.success is not None: + return result.success + if result.o1 is not None: + raise result.o1 + raise TApplicationException(TApplicationException.MISSING_RESULT, "partition_name_to_spec failed: unknown result") + + def markPartitionForEvent(self, db_name, tbl_name, part_vals, eventType): + """ + Parameters: + - db_name + - tbl_name + - part_vals + - eventType + + """ + self.send_markPartitionForEvent(db_name, tbl_name, part_vals, eventType) + self.recv_markPartitionForEvent() + + def send_markPartitionForEvent(self, db_name, tbl_name, part_vals, eventType): + self._oprot.writeMessageBegin("markPartitionForEvent", TMessageType.CALL, self._seqid) + args = markPartitionForEvent_args() + args.db_name = db_name + args.tbl_name = tbl_name + args.part_vals = part_vals + args.eventType = eventType + args.write(self._oprot) + self._oprot.writeMessageEnd() + self._oprot.trans.flush() + + def recv_markPartitionForEvent(self): + iprot = self._iprot + (fname, mtype, rseqid) = iprot.readMessageBegin() + if mtype == TMessageType.EXCEPTION: + x = TApplicationException() + x.read(iprot) + iprot.readMessageEnd() + raise x + result = markPartitionForEvent_result() + result.read(iprot) + iprot.readMessageEnd() + if result.o1 is not None: + raise result.o1 + if result.o2 is not None: + raise result.o2 + if result.o3 is not None: + raise result.o3 + if result.o4 is not None: + raise result.o4 + if result.o5 is not None: + raise result.o5 + if result.o6 is not None: + raise result.o6 + return + + def isPartitionMarkedForEvent(self, db_name, tbl_name, part_vals, eventType): + """ + Parameters: + - db_name + - tbl_name + - part_vals + - eventType + + """ + self.send_isPartitionMarkedForEvent(db_name, tbl_name, part_vals, eventType) + return self.recv_isPartitionMarkedForEvent() + + def send_isPartitionMarkedForEvent(self, db_name, tbl_name, part_vals, eventType): + self._oprot.writeMessageBegin("isPartitionMarkedForEvent", TMessageType.CALL, self._seqid) + args = isPartitionMarkedForEvent_args() + args.db_name = db_name + args.tbl_name = tbl_name + args.part_vals = part_vals + args.eventType = eventType + args.write(self._oprot) + self._oprot.writeMessageEnd() + self._oprot.trans.flush() + + def recv_isPartitionMarkedForEvent(self): + iprot = self._iprot + (fname, mtype, rseqid) = iprot.readMessageBegin() + if mtype == TMessageType.EXCEPTION: + x = TApplicationException() + x.read(iprot) + iprot.readMessageEnd() + raise x + result = isPartitionMarkedForEvent_result() + result.read(iprot) + iprot.readMessageEnd() + if result.success is not None: + return result.success + if result.o1 is not None: + raise result.o1 + if result.o2 is not None: + raise result.o2 + if result.o3 is not None: + raise result.o3 + if result.o4 is not None: + raise result.o4 + if result.o5 is not None: + raise result.o5 + if result.o6 is not None: + raise result.o6 + raise TApplicationException(TApplicationException.MISSING_RESULT, "isPartitionMarkedForEvent failed: unknown result") + + def get_primary_keys(self, request): + """ + Parameters: + - request + + """ + self.send_get_primary_keys(request) + return self.recv_get_primary_keys() + + def send_get_primary_keys(self, request): + self._oprot.writeMessageBegin("get_primary_keys", TMessageType.CALL, self._seqid) + args = get_primary_keys_args() + args.request = request + args.write(self._oprot) + self._oprot.writeMessageEnd() + self._oprot.trans.flush() + + def recv_get_primary_keys(self): + iprot = self._iprot + (fname, mtype, rseqid) = iprot.readMessageBegin() + if mtype == TMessageType.EXCEPTION: + x = TApplicationException() + x.read(iprot) + iprot.readMessageEnd() + raise x + result = get_primary_keys_result() + result.read(iprot) + iprot.readMessageEnd() + if result.success is not None: + return result.success + if result.o1 is not None: + raise result.o1 + if result.o2 is not None: + raise result.o2 + raise TApplicationException(TApplicationException.MISSING_RESULT, "get_primary_keys failed: unknown result") + + def get_foreign_keys(self, request): + """ + Parameters: + - request + + """ + self.send_get_foreign_keys(request) + return self.recv_get_foreign_keys() + + def send_get_foreign_keys(self, request): + self._oprot.writeMessageBegin("get_foreign_keys", TMessageType.CALL, self._seqid) + args = get_foreign_keys_args() + args.request = request + args.write(self._oprot) + self._oprot.writeMessageEnd() + self._oprot.trans.flush() + + def recv_get_foreign_keys(self): + iprot = self._iprot + (fname, mtype, rseqid) = iprot.readMessageBegin() + if mtype == TMessageType.EXCEPTION: + x = TApplicationException() + x.read(iprot) + iprot.readMessageEnd() + raise x + result = get_foreign_keys_result() + result.read(iprot) + iprot.readMessageEnd() + if result.success is not None: + return result.success + if result.o1 is not None: + raise result.o1 + if result.o2 is not None: + raise result.o2 + raise TApplicationException(TApplicationException.MISSING_RESULT, "get_foreign_keys failed: unknown result") + + def get_unique_constraints(self, request): + """ + Parameters: + - request + + """ + self.send_get_unique_constraints(request) + return self.recv_get_unique_constraints() + + def send_get_unique_constraints(self, request): + self._oprot.writeMessageBegin("get_unique_constraints", TMessageType.CALL, self._seqid) + args = get_unique_constraints_args() + args.request = request + args.write(self._oprot) + self._oprot.writeMessageEnd() + self._oprot.trans.flush() + + def recv_get_unique_constraints(self): + iprot = self._iprot + (fname, mtype, rseqid) = iprot.readMessageBegin() + if mtype == TMessageType.EXCEPTION: + x = TApplicationException() + x.read(iprot) + iprot.readMessageEnd() + raise x + result = get_unique_constraints_result() + result.read(iprot) + iprot.readMessageEnd() + if result.success is not None: + return result.success + if result.o1 is not None: + raise result.o1 + if result.o2 is not None: + raise result.o2 + raise TApplicationException(TApplicationException.MISSING_RESULT, "get_unique_constraints failed: unknown result") + + def get_not_null_constraints(self, request): + """ + Parameters: + - request + + """ + self.send_get_not_null_constraints(request) + return self.recv_get_not_null_constraints() + + def send_get_not_null_constraints(self, request): + self._oprot.writeMessageBegin("get_not_null_constraints", TMessageType.CALL, self._seqid) + args = get_not_null_constraints_args() + args.request = request + args.write(self._oprot) + self._oprot.writeMessageEnd() + self._oprot.trans.flush() + + def recv_get_not_null_constraints(self): + iprot = self._iprot + (fname, mtype, rseqid) = iprot.readMessageBegin() + if mtype == TMessageType.EXCEPTION: + x = TApplicationException() + x.read(iprot) + iprot.readMessageEnd() + raise x + result = get_not_null_constraints_result() + result.read(iprot) + iprot.readMessageEnd() + if result.success is not None: + return result.success + if result.o1 is not None: + raise result.o1 + if result.o2 is not None: + raise result.o2 + raise TApplicationException(TApplicationException.MISSING_RESULT, "get_not_null_constraints failed: unknown result") + + def get_default_constraints(self, request): + """ + Parameters: + - request + + """ + self.send_get_default_constraints(request) + return self.recv_get_default_constraints() + + def send_get_default_constraints(self, request): + self._oprot.writeMessageBegin("get_default_constraints", TMessageType.CALL, self._seqid) + args = get_default_constraints_args() + args.request = request + args.write(self._oprot) + self._oprot.writeMessageEnd() + self._oprot.trans.flush() + + def recv_get_default_constraints(self): + iprot = self._iprot + (fname, mtype, rseqid) = iprot.readMessageBegin() + if mtype == TMessageType.EXCEPTION: + x = TApplicationException() + x.read(iprot) + iprot.readMessageEnd() + raise x + result = get_default_constraints_result() + result.read(iprot) + iprot.readMessageEnd() + if result.success is not None: + return result.success + if result.o1 is not None: + raise result.o1 + if result.o2 is not None: + raise result.o2 + raise TApplicationException(TApplicationException.MISSING_RESULT, "get_default_constraints failed: unknown result") + + def get_check_constraints(self, request): + """ + Parameters: + - request + + """ + self.send_get_check_constraints(request) + return self.recv_get_check_constraints() + + def send_get_check_constraints(self, request): + self._oprot.writeMessageBegin("get_check_constraints", TMessageType.CALL, self._seqid) + args = get_check_constraints_args() + args.request = request + args.write(self._oprot) + self._oprot.writeMessageEnd() + self._oprot.trans.flush() + + def recv_get_check_constraints(self): + iprot = self._iprot + (fname, mtype, rseqid) = iprot.readMessageBegin() + if mtype == TMessageType.EXCEPTION: + x = TApplicationException() + x.read(iprot) + iprot.readMessageEnd() + raise x + result = get_check_constraints_result() + result.read(iprot) + iprot.readMessageEnd() + if result.success is not None: + return result.success + if result.o1 is not None: + raise result.o1 + if result.o2 is not None: + raise result.o2 + raise TApplicationException(TApplicationException.MISSING_RESULT, "get_check_constraints failed: unknown result") + + def get_all_table_constraints(self, request): + """ + Parameters: + - request + + """ + self.send_get_all_table_constraints(request) + return self.recv_get_all_table_constraints() + + def send_get_all_table_constraints(self, request): + self._oprot.writeMessageBegin("get_all_table_constraints", TMessageType.CALL, self._seqid) + args = get_all_table_constraints_args() + args.request = request + args.write(self._oprot) + self._oprot.writeMessageEnd() + self._oprot.trans.flush() + + def recv_get_all_table_constraints(self): + iprot = self._iprot + (fname, mtype, rseqid) = iprot.readMessageBegin() + if mtype == TMessageType.EXCEPTION: + x = TApplicationException() + x.read(iprot) + iprot.readMessageEnd() + raise x + result = get_all_table_constraints_result() + result.read(iprot) + iprot.readMessageEnd() + if result.success is not None: + return result.success + if result.o1 is not None: + raise result.o1 + if result.o2 is not None: + raise result.o2 + raise TApplicationException(TApplicationException.MISSING_RESULT, "get_all_table_constraints failed: unknown result") + + def update_table_column_statistics(self, stats_obj): + """ + Parameters: + - stats_obj + + """ + self.send_update_table_column_statistics(stats_obj) + return self.recv_update_table_column_statistics() + + def send_update_table_column_statistics(self, stats_obj): + self._oprot.writeMessageBegin("update_table_column_statistics", TMessageType.CALL, self._seqid) + args = update_table_column_statistics_args() + args.stats_obj = stats_obj + args.write(self._oprot) + self._oprot.writeMessageEnd() + self._oprot.trans.flush() + + def recv_update_table_column_statistics(self): + iprot = self._iprot + (fname, mtype, rseqid) = iprot.readMessageBegin() + if mtype == TMessageType.EXCEPTION: + x = TApplicationException() + x.read(iprot) + iprot.readMessageEnd() + raise x + result = update_table_column_statistics_result() + result.read(iprot) + iprot.readMessageEnd() + if result.success is not None: + return result.success + if result.o1 is not None: + raise result.o1 + if result.o2 is not None: + raise result.o2 + if result.o3 is not None: + raise result.o3 + if result.o4 is not None: + raise result.o4 + raise TApplicationException(TApplicationException.MISSING_RESULT, "update_table_column_statistics failed: unknown result") + + def update_partition_column_statistics(self, stats_obj): + """ + Parameters: + - stats_obj + + """ + self.send_update_partition_column_statistics(stats_obj) + return self.recv_update_partition_column_statistics() + + def send_update_partition_column_statistics(self, stats_obj): + self._oprot.writeMessageBegin("update_partition_column_statistics", TMessageType.CALL, self._seqid) + args = update_partition_column_statistics_args() + args.stats_obj = stats_obj + args.write(self._oprot) + self._oprot.writeMessageEnd() + self._oprot.trans.flush() + + def recv_update_partition_column_statistics(self): + iprot = self._iprot + (fname, mtype, rseqid) = iprot.readMessageBegin() + if mtype == TMessageType.EXCEPTION: + x = TApplicationException() + x.read(iprot) + iprot.readMessageEnd() + raise x + result = update_partition_column_statistics_result() + result.read(iprot) + iprot.readMessageEnd() + if result.success is not None: + return result.success + if result.o1 is not None: + raise result.o1 + if result.o2 is not None: + raise result.o2 + if result.o3 is not None: + raise result.o3 + if result.o4 is not None: + raise result.o4 + raise TApplicationException( + TApplicationException.MISSING_RESULT, "update_partition_column_statistics failed: unknown result" + ) + + def update_table_column_statistics_req(self, req): + """ + Parameters: + - req + + """ + self.send_update_table_column_statistics_req(req) + return self.recv_update_table_column_statistics_req() + + def send_update_table_column_statistics_req(self, req): + self._oprot.writeMessageBegin("update_table_column_statistics_req", TMessageType.CALL, self._seqid) + args = update_table_column_statistics_req_args() + args.req = req + args.write(self._oprot) + self._oprot.writeMessageEnd() + self._oprot.trans.flush() + + def recv_update_table_column_statistics_req(self): + iprot = self._iprot + (fname, mtype, rseqid) = iprot.readMessageBegin() + if mtype == TMessageType.EXCEPTION: + x = TApplicationException() + x.read(iprot) + iprot.readMessageEnd() + raise x + result = update_table_column_statistics_req_result() + result.read(iprot) + iprot.readMessageEnd() + if result.success is not None: + return result.success + if result.o1 is not None: + raise result.o1 + if result.o2 is not None: + raise result.o2 + if result.o3 is not None: + raise result.o3 + if result.o4 is not None: + raise result.o4 + raise TApplicationException( + TApplicationException.MISSING_RESULT, "update_table_column_statistics_req failed: unknown result" + ) + + def update_partition_column_statistics_req(self, req): + """ + Parameters: + - req + + """ + self.send_update_partition_column_statistics_req(req) + return self.recv_update_partition_column_statistics_req() + + def send_update_partition_column_statistics_req(self, req): + self._oprot.writeMessageBegin("update_partition_column_statistics_req", TMessageType.CALL, self._seqid) + args = update_partition_column_statistics_req_args() + args.req = req + args.write(self._oprot) + self._oprot.writeMessageEnd() + self._oprot.trans.flush() + + def recv_update_partition_column_statistics_req(self): + iprot = self._iprot + (fname, mtype, rseqid) = iprot.readMessageBegin() + if mtype == TMessageType.EXCEPTION: + x = TApplicationException() + x.read(iprot) + iprot.readMessageEnd() + raise x + result = update_partition_column_statistics_req_result() + result.read(iprot) + iprot.readMessageEnd() + if result.success is not None: + return result.success + if result.o1 is not None: + raise result.o1 + if result.o2 is not None: + raise result.o2 + if result.o3 is not None: + raise result.o3 + if result.o4 is not None: + raise result.o4 + raise TApplicationException( + TApplicationException.MISSING_RESULT, "update_partition_column_statistics_req failed: unknown result" + ) + + def update_transaction_statistics(self, req): + """ + Parameters: + - req + + """ + self.send_update_transaction_statistics(req) + self.recv_update_transaction_statistics() + + def send_update_transaction_statistics(self, req): + self._oprot.writeMessageBegin("update_transaction_statistics", TMessageType.CALL, self._seqid) + args = update_transaction_statistics_args() + args.req = req + args.write(self._oprot) + self._oprot.writeMessageEnd() + self._oprot.trans.flush() + + def recv_update_transaction_statistics(self): + iprot = self._iprot + (fname, mtype, rseqid) = iprot.readMessageBegin() + if mtype == TMessageType.EXCEPTION: + x = TApplicationException() + x.read(iprot) + iprot.readMessageEnd() + raise x + result = update_transaction_statistics_result() + result.read(iprot) + iprot.readMessageEnd() + if result.o1 is not None: + raise result.o1 + return + + def get_table_column_statistics(self, db_name, tbl_name, col_name): + """ + Parameters: + - db_name + - tbl_name + - col_name + + """ + self.send_get_table_column_statistics(db_name, tbl_name, col_name) + return self.recv_get_table_column_statistics() + + def send_get_table_column_statistics(self, db_name, tbl_name, col_name): + self._oprot.writeMessageBegin("get_table_column_statistics", TMessageType.CALL, self._seqid) + args = get_table_column_statistics_args() + args.db_name = db_name + args.tbl_name = tbl_name + args.col_name = col_name + args.write(self._oprot) + self._oprot.writeMessageEnd() + self._oprot.trans.flush() + + def recv_get_table_column_statistics(self): + iprot = self._iprot + (fname, mtype, rseqid) = iprot.readMessageBegin() + if mtype == TMessageType.EXCEPTION: + x = TApplicationException() + x.read(iprot) + iprot.readMessageEnd() + raise x + result = get_table_column_statistics_result() + result.read(iprot) + iprot.readMessageEnd() + if result.success is not None: + return result.success + if result.o1 is not None: + raise result.o1 + if result.o2 is not None: + raise result.o2 + if result.o3 is not None: + raise result.o3 + if result.o4 is not None: + raise result.o4 + raise TApplicationException(TApplicationException.MISSING_RESULT, "get_table_column_statistics failed: unknown result") + + def get_partition_column_statistics(self, db_name, tbl_name, part_name, col_name): + """ + Parameters: + - db_name + - tbl_name + - part_name + - col_name + + """ + self.send_get_partition_column_statistics(db_name, tbl_name, part_name, col_name) + return self.recv_get_partition_column_statistics() + + def send_get_partition_column_statistics(self, db_name, tbl_name, part_name, col_name): + self._oprot.writeMessageBegin("get_partition_column_statistics", TMessageType.CALL, self._seqid) + args = get_partition_column_statistics_args() + args.db_name = db_name + args.tbl_name = tbl_name + args.part_name = part_name + args.col_name = col_name + args.write(self._oprot) + self._oprot.writeMessageEnd() + self._oprot.trans.flush() + + def recv_get_partition_column_statistics(self): + iprot = self._iprot + (fname, mtype, rseqid) = iprot.readMessageBegin() + if mtype == TMessageType.EXCEPTION: + x = TApplicationException() + x.read(iprot) + iprot.readMessageEnd() + raise x + result = get_partition_column_statistics_result() + result.read(iprot) + iprot.readMessageEnd() + if result.success is not None: + return result.success + if result.o1 is not None: + raise result.o1 + if result.o2 is not None: + raise result.o2 + if result.o3 is not None: + raise result.o3 + if result.o4 is not None: + raise result.o4 + raise TApplicationException( + TApplicationException.MISSING_RESULT, "get_partition_column_statistics failed: unknown result" + ) + + def get_table_statistics_req(self, request): + """ + Parameters: + - request + + """ + self.send_get_table_statistics_req(request) + return self.recv_get_table_statistics_req() + + def send_get_table_statistics_req(self, request): + self._oprot.writeMessageBegin("get_table_statistics_req", TMessageType.CALL, self._seqid) + args = get_table_statistics_req_args() + args.request = request + args.write(self._oprot) + self._oprot.writeMessageEnd() + self._oprot.trans.flush() + + def recv_get_table_statistics_req(self): + iprot = self._iprot + (fname, mtype, rseqid) = iprot.readMessageBegin() + if mtype == TMessageType.EXCEPTION: + x = TApplicationException() + x.read(iprot) + iprot.readMessageEnd() + raise x + result = get_table_statistics_req_result() + result.read(iprot) + iprot.readMessageEnd() + if result.success is not None: + return result.success + if result.o1 is not None: + raise result.o1 + if result.o2 is not None: + raise result.o2 + raise TApplicationException(TApplicationException.MISSING_RESULT, "get_table_statistics_req failed: unknown result") + + def get_partitions_statistics_req(self, request): + """ + Parameters: + - request + + """ + self.send_get_partitions_statistics_req(request) + return self.recv_get_partitions_statistics_req() + + def send_get_partitions_statistics_req(self, request): + self._oprot.writeMessageBegin("get_partitions_statistics_req", TMessageType.CALL, self._seqid) + args = get_partitions_statistics_req_args() + args.request = request + args.write(self._oprot) + self._oprot.writeMessageEnd() + self._oprot.trans.flush() + + def recv_get_partitions_statistics_req(self): + iprot = self._iprot + (fname, mtype, rseqid) = iprot.readMessageBegin() + if mtype == TMessageType.EXCEPTION: + x = TApplicationException() + x.read(iprot) + iprot.readMessageEnd() + raise x + result = get_partitions_statistics_req_result() + result.read(iprot) + iprot.readMessageEnd() + if result.success is not None: + return result.success + if result.o1 is not None: + raise result.o1 + if result.o2 is not None: + raise result.o2 + raise TApplicationException(TApplicationException.MISSING_RESULT, "get_partitions_statistics_req failed: unknown result") + + def get_aggr_stats_for(self, request): + """ + Parameters: + - request + + """ + self.send_get_aggr_stats_for(request) + return self.recv_get_aggr_stats_for() + + def send_get_aggr_stats_for(self, request): + self._oprot.writeMessageBegin("get_aggr_stats_for", TMessageType.CALL, self._seqid) + args = get_aggr_stats_for_args() + args.request = request + args.write(self._oprot) + self._oprot.writeMessageEnd() + self._oprot.trans.flush() + + def recv_get_aggr_stats_for(self): + iprot = self._iprot + (fname, mtype, rseqid) = iprot.readMessageBegin() + if mtype == TMessageType.EXCEPTION: + x = TApplicationException() + x.read(iprot) + iprot.readMessageEnd() + raise x + result = get_aggr_stats_for_result() + result.read(iprot) + iprot.readMessageEnd() + if result.success is not None: + return result.success + if result.o1 is not None: + raise result.o1 + if result.o2 is not None: + raise result.o2 + raise TApplicationException(TApplicationException.MISSING_RESULT, "get_aggr_stats_for failed: unknown result") + + def set_aggr_stats_for(self, request): + """ + Parameters: + - request + + """ + self.send_set_aggr_stats_for(request) + return self.recv_set_aggr_stats_for() + + def send_set_aggr_stats_for(self, request): + self._oprot.writeMessageBegin("set_aggr_stats_for", TMessageType.CALL, self._seqid) + args = set_aggr_stats_for_args() + args.request = request + args.write(self._oprot) + self._oprot.writeMessageEnd() + self._oprot.trans.flush() + + def recv_set_aggr_stats_for(self): + iprot = self._iprot + (fname, mtype, rseqid) = iprot.readMessageBegin() + if mtype == TMessageType.EXCEPTION: + x = TApplicationException() + x.read(iprot) + iprot.readMessageEnd() + raise x + result = set_aggr_stats_for_result() + result.read(iprot) + iprot.readMessageEnd() + if result.success is not None: + return result.success + if result.o1 is not None: + raise result.o1 + if result.o2 is not None: + raise result.o2 + if result.o3 is not None: + raise result.o3 + if result.o4 is not None: + raise result.o4 + raise TApplicationException(TApplicationException.MISSING_RESULT, "set_aggr_stats_for failed: unknown result") + + def delete_partition_column_statistics(self, db_name, tbl_name, part_name, col_name, engine): + """ + Parameters: + - db_name + - tbl_name + - part_name + - col_name + - engine + + """ + self.send_delete_partition_column_statistics(db_name, tbl_name, part_name, col_name, engine) + return self.recv_delete_partition_column_statistics() + + def send_delete_partition_column_statistics(self, db_name, tbl_name, part_name, col_name, engine): + self._oprot.writeMessageBegin("delete_partition_column_statistics", TMessageType.CALL, self._seqid) + args = delete_partition_column_statistics_args() + args.db_name = db_name + args.tbl_name = tbl_name + args.part_name = part_name + args.col_name = col_name + args.engine = engine + args.write(self._oprot) + self._oprot.writeMessageEnd() + self._oprot.trans.flush() + + def recv_delete_partition_column_statistics(self): + iprot = self._iprot + (fname, mtype, rseqid) = iprot.readMessageBegin() + if mtype == TMessageType.EXCEPTION: + x = TApplicationException() + x.read(iprot) + iprot.readMessageEnd() + raise x + result = delete_partition_column_statistics_result() + result.read(iprot) + iprot.readMessageEnd() + if result.success is not None: + return result.success + if result.o1 is not None: + raise result.o1 + if result.o2 is not None: + raise result.o2 + if result.o3 is not None: + raise result.o3 + if result.o4 is not None: + raise result.o4 + raise TApplicationException( + TApplicationException.MISSING_RESULT, "delete_partition_column_statistics failed: unknown result" + ) + + def delete_table_column_statistics(self, db_name, tbl_name, col_name, engine): + """ + Parameters: + - db_name + - tbl_name + - col_name + - engine + + """ + self.send_delete_table_column_statistics(db_name, tbl_name, col_name, engine) + return self.recv_delete_table_column_statistics() + + def send_delete_table_column_statistics(self, db_name, tbl_name, col_name, engine): + self._oprot.writeMessageBegin("delete_table_column_statistics", TMessageType.CALL, self._seqid) + args = delete_table_column_statistics_args() + args.db_name = db_name + args.tbl_name = tbl_name + args.col_name = col_name + args.engine = engine + args.write(self._oprot) + self._oprot.writeMessageEnd() + self._oprot.trans.flush() + + def recv_delete_table_column_statistics(self): + iprot = self._iprot + (fname, mtype, rseqid) = iprot.readMessageBegin() + if mtype == TMessageType.EXCEPTION: + x = TApplicationException() + x.read(iprot) + iprot.readMessageEnd() + raise x + result = delete_table_column_statistics_result() + result.read(iprot) + iprot.readMessageEnd() + if result.success is not None: + return result.success + if result.o1 is not None: + raise result.o1 + if result.o2 is not None: + raise result.o2 + if result.o3 is not None: + raise result.o3 + if result.o4 is not None: + raise result.o4 + raise TApplicationException(TApplicationException.MISSING_RESULT, "delete_table_column_statistics failed: unknown result") + + def create_function(self, func): + """ + Parameters: + - func + + """ + self.send_create_function(func) + self.recv_create_function() + + def send_create_function(self, func): + self._oprot.writeMessageBegin("create_function", TMessageType.CALL, self._seqid) + args = create_function_args() + args.func = func + args.write(self._oprot) + self._oprot.writeMessageEnd() + self._oprot.trans.flush() + + def recv_create_function(self): + iprot = self._iprot + (fname, mtype, rseqid) = iprot.readMessageBegin() + if mtype == TMessageType.EXCEPTION: + x = TApplicationException() + x.read(iprot) + iprot.readMessageEnd() + raise x + result = create_function_result() + result.read(iprot) + iprot.readMessageEnd() + if result.o1 is not None: + raise result.o1 + if result.o2 is not None: + raise result.o2 + if result.o3 is not None: + raise result.o3 + if result.o4 is not None: + raise result.o4 + return + + def drop_function(self, dbName, funcName): + """ + Parameters: + - dbName + - funcName + + """ + self.send_drop_function(dbName, funcName) + self.recv_drop_function() + + def send_drop_function(self, dbName, funcName): + self._oprot.writeMessageBegin("drop_function", TMessageType.CALL, self._seqid) + args = drop_function_args() + args.dbName = dbName + args.funcName = funcName + args.write(self._oprot) + self._oprot.writeMessageEnd() + self._oprot.trans.flush() + + def recv_drop_function(self): + iprot = self._iprot + (fname, mtype, rseqid) = iprot.readMessageBegin() + if mtype == TMessageType.EXCEPTION: + x = TApplicationException() + x.read(iprot) + iprot.readMessageEnd() + raise x + result = drop_function_result() + result.read(iprot) + iprot.readMessageEnd() + if result.o1 is not None: + raise result.o1 + if result.o3 is not None: + raise result.o3 + return + + def alter_function(self, dbName, funcName, newFunc): + """ + Parameters: + - dbName + - funcName + - newFunc + + """ + self.send_alter_function(dbName, funcName, newFunc) + self.recv_alter_function() + + def send_alter_function(self, dbName, funcName, newFunc): + self._oprot.writeMessageBegin("alter_function", TMessageType.CALL, self._seqid) + args = alter_function_args() + args.dbName = dbName + args.funcName = funcName + args.newFunc = newFunc + args.write(self._oprot) + self._oprot.writeMessageEnd() + self._oprot.trans.flush() + + def recv_alter_function(self): + iprot = self._iprot + (fname, mtype, rseqid) = iprot.readMessageBegin() + if mtype == TMessageType.EXCEPTION: + x = TApplicationException() + x.read(iprot) + iprot.readMessageEnd() + raise x + result = alter_function_result() + result.read(iprot) + iprot.readMessageEnd() + if result.o1 is not None: + raise result.o1 + if result.o2 is not None: + raise result.o2 + return + + def get_functions(self, dbName, pattern): + """ + Parameters: + - dbName + - pattern + + """ + self.send_get_functions(dbName, pattern) + return self.recv_get_functions() + + def send_get_functions(self, dbName, pattern): + self._oprot.writeMessageBegin("get_functions", TMessageType.CALL, self._seqid) + args = get_functions_args() + args.dbName = dbName + args.pattern = pattern + args.write(self._oprot) + self._oprot.writeMessageEnd() + self._oprot.trans.flush() + + def recv_get_functions(self): + iprot = self._iprot + (fname, mtype, rseqid) = iprot.readMessageBegin() + if mtype == TMessageType.EXCEPTION: + x = TApplicationException() + x.read(iprot) + iprot.readMessageEnd() + raise x + result = get_functions_result() + result.read(iprot) + iprot.readMessageEnd() + if result.success is not None: + return result.success + if result.o1 is not None: + raise result.o1 + raise TApplicationException(TApplicationException.MISSING_RESULT, "get_functions failed: unknown result") + + def get_function(self, dbName, funcName): + """ + Parameters: + - dbName + - funcName + + """ + self.send_get_function(dbName, funcName) + return self.recv_get_function() + + def send_get_function(self, dbName, funcName): + self._oprot.writeMessageBegin("get_function", TMessageType.CALL, self._seqid) + args = get_function_args() + args.dbName = dbName + args.funcName = funcName + args.write(self._oprot) + self._oprot.writeMessageEnd() + self._oprot.trans.flush() + + def recv_get_function(self): + iprot = self._iprot + (fname, mtype, rseqid) = iprot.readMessageBegin() + if mtype == TMessageType.EXCEPTION: + x = TApplicationException() + x.read(iprot) + iprot.readMessageEnd() + raise x + result = get_function_result() + result.read(iprot) + iprot.readMessageEnd() + if result.success is not None: + return result.success + if result.o1 is not None: + raise result.o1 + if result.o2 is not None: + raise result.o2 + raise TApplicationException(TApplicationException.MISSING_RESULT, "get_function failed: unknown result") + + def get_all_functions(self): + self.send_get_all_functions() + return self.recv_get_all_functions() + + def send_get_all_functions(self): + self._oprot.writeMessageBegin("get_all_functions", TMessageType.CALL, self._seqid) + args = get_all_functions_args() + args.write(self._oprot) + self._oprot.writeMessageEnd() + self._oprot.trans.flush() + + def recv_get_all_functions(self): + iprot = self._iprot + (fname, mtype, rseqid) = iprot.readMessageBegin() + if mtype == TMessageType.EXCEPTION: + x = TApplicationException() + x.read(iprot) + iprot.readMessageEnd() + raise x + result = get_all_functions_result() + result.read(iprot) + iprot.readMessageEnd() + if result.success is not None: + return result.success + if result.o1 is not None: + raise result.o1 + raise TApplicationException(TApplicationException.MISSING_RESULT, "get_all_functions failed: unknown result") + + def create_role(self, role): + """ + Parameters: + - role + + """ + self.send_create_role(role) + return self.recv_create_role() + + def send_create_role(self, role): + self._oprot.writeMessageBegin("create_role", TMessageType.CALL, self._seqid) + args = create_role_args() + args.role = role + args.write(self._oprot) + self._oprot.writeMessageEnd() + self._oprot.trans.flush() + + def recv_create_role(self): + iprot = self._iprot + (fname, mtype, rseqid) = iprot.readMessageBegin() + if mtype == TMessageType.EXCEPTION: + x = TApplicationException() + x.read(iprot) + iprot.readMessageEnd() + raise x + result = create_role_result() + result.read(iprot) + iprot.readMessageEnd() + if result.success is not None: + return result.success + if result.o1 is not None: + raise result.o1 + raise TApplicationException(TApplicationException.MISSING_RESULT, "create_role failed: unknown result") + + def drop_role(self, role_name): + """ + Parameters: + - role_name + + """ + self.send_drop_role(role_name) + return self.recv_drop_role() + + def send_drop_role(self, role_name): + self._oprot.writeMessageBegin("drop_role", TMessageType.CALL, self._seqid) + args = drop_role_args() + args.role_name = role_name + args.write(self._oprot) + self._oprot.writeMessageEnd() + self._oprot.trans.flush() + + def recv_drop_role(self): + iprot = self._iprot + (fname, mtype, rseqid) = iprot.readMessageBegin() + if mtype == TMessageType.EXCEPTION: + x = TApplicationException() + x.read(iprot) + iprot.readMessageEnd() + raise x + result = drop_role_result() + result.read(iprot) + iprot.readMessageEnd() + if result.success is not None: + return result.success + if result.o1 is not None: + raise result.o1 + raise TApplicationException(TApplicationException.MISSING_RESULT, "drop_role failed: unknown result") + + def get_role_names(self): + self.send_get_role_names() + return self.recv_get_role_names() + + def send_get_role_names(self): + self._oprot.writeMessageBegin("get_role_names", TMessageType.CALL, self._seqid) + args = get_role_names_args() + args.write(self._oprot) + self._oprot.writeMessageEnd() + self._oprot.trans.flush() + + def recv_get_role_names(self): + iprot = self._iprot + (fname, mtype, rseqid) = iprot.readMessageBegin() + if mtype == TMessageType.EXCEPTION: + x = TApplicationException() + x.read(iprot) + iprot.readMessageEnd() + raise x + result = get_role_names_result() + result.read(iprot) + iprot.readMessageEnd() + if result.success is not None: + return result.success + if result.o1 is not None: + raise result.o1 + raise TApplicationException(TApplicationException.MISSING_RESULT, "get_role_names failed: unknown result") + + def grant_role(self, role_name, principal_name, principal_type, grantor, grantorType, grant_option): + """ + Parameters: + - role_name + - principal_name + - principal_type + - grantor + - grantorType + - grant_option + + """ + self.send_grant_role(role_name, principal_name, principal_type, grantor, grantorType, grant_option) + return self.recv_grant_role() + + def send_grant_role(self, role_name, principal_name, principal_type, grantor, grantorType, grant_option): + self._oprot.writeMessageBegin("grant_role", TMessageType.CALL, self._seqid) + args = grant_role_args() + args.role_name = role_name + args.principal_name = principal_name + args.principal_type = principal_type + args.grantor = grantor + args.grantorType = grantorType + args.grant_option = grant_option + args.write(self._oprot) + self._oprot.writeMessageEnd() + self._oprot.trans.flush() + + def recv_grant_role(self): + iprot = self._iprot + (fname, mtype, rseqid) = iprot.readMessageBegin() + if mtype == TMessageType.EXCEPTION: + x = TApplicationException() + x.read(iprot) + iprot.readMessageEnd() + raise x + result = grant_role_result() + result.read(iprot) + iprot.readMessageEnd() + if result.success is not None: + return result.success + if result.o1 is not None: + raise result.o1 + raise TApplicationException(TApplicationException.MISSING_RESULT, "grant_role failed: unknown result") + + def revoke_role(self, role_name, principal_name, principal_type): + """ + Parameters: + - role_name + - principal_name + - principal_type + + """ + self.send_revoke_role(role_name, principal_name, principal_type) + return self.recv_revoke_role() + + def send_revoke_role(self, role_name, principal_name, principal_type): + self._oprot.writeMessageBegin("revoke_role", TMessageType.CALL, self._seqid) + args = revoke_role_args() + args.role_name = role_name + args.principal_name = principal_name + args.principal_type = principal_type + args.write(self._oprot) + self._oprot.writeMessageEnd() + self._oprot.trans.flush() + + def recv_revoke_role(self): + iprot = self._iprot + (fname, mtype, rseqid) = iprot.readMessageBegin() + if mtype == TMessageType.EXCEPTION: + x = TApplicationException() + x.read(iprot) + iprot.readMessageEnd() + raise x + result = revoke_role_result() + result.read(iprot) + iprot.readMessageEnd() + if result.success is not None: + return result.success + if result.o1 is not None: + raise result.o1 + raise TApplicationException(TApplicationException.MISSING_RESULT, "revoke_role failed: unknown result") + + def list_roles(self, principal_name, principal_type): + """ + Parameters: + - principal_name + - principal_type + + """ + self.send_list_roles(principal_name, principal_type) + return self.recv_list_roles() + + def send_list_roles(self, principal_name, principal_type): + self._oprot.writeMessageBegin("list_roles", TMessageType.CALL, self._seqid) + args = list_roles_args() + args.principal_name = principal_name + args.principal_type = principal_type + args.write(self._oprot) + self._oprot.writeMessageEnd() + self._oprot.trans.flush() + + def recv_list_roles(self): + iprot = self._iprot + (fname, mtype, rseqid) = iprot.readMessageBegin() + if mtype == TMessageType.EXCEPTION: + x = TApplicationException() + x.read(iprot) + iprot.readMessageEnd() + raise x + result = list_roles_result() + result.read(iprot) + iprot.readMessageEnd() + if result.success is not None: + return result.success + if result.o1 is not None: + raise result.o1 + raise TApplicationException(TApplicationException.MISSING_RESULT, "list_roles failed: unknown result") + + def grant_revoke_role(self, request): + """ + Parameters: + - request + + """ + self.send_grant_revoke_role(request) + return self.recv_grant_revoke_role() + + def send_grant_revoke_role(self, request): + self._oprot.writeMessageBegin("grant_revoke_role", TMessageType.CALL, self._seqid) + args = grant_revoke_role_args() + args.request = request + args.write(self._oprot) + self._oprot.writeMessageEnd() + self._oprot.trans.flush() + + def recv_grant_revoke_role(self): + iprot = self._iprot + (fname, mtype, rseqid) = iprot.readMessageBegin() + if mtype == TMessageType.EXCEPTION: + x = TApplicationException() + x.read(iprot) + iprot.readMessageEnd() + raise x + result = grant_revoke_role_result() + result.read(iprot) + iprot.readMessageEnd() + if result.success is not None: + return result.success + if result.o1 is not None: + raise result.o1 + raise TApplicationException(TApplicationException.MISSING_RESULT, "grant_revoke_role failed: unknown result") + + def get_principals_in_role(self, request): + """ + Parameters: + - request + + """ + self.send_get_principals_in_role(request) + return self.recv_get_principals_in_role() + + def send_get_principals_in_role(self, request): + self._oprot.writeMessageBegin("get_principals_in_role", TMessageType.CALL, self._seqid) + args = get_principals_in_role_args() + args.request = request + args.write(self._oprot) + self._oprot.writeMessageEnd() + self._oprot.trans.flush() + + def recv_get_principals_in_role(self): + iprot = self._iprot + (fname, mtype, rseqid) = iprot.readMessageBegin() + if mtype == TMessageType.EXCEPTION: + x = TApplicationException() + x.read(iprot) + iprot.readMessageEnd() + raise x + result = get_principals_in_role_result() + result.read(iprot) + iprot.readMessageEnd() + if result.success is not None: + return result.success + if result.o1 is not None: + raise result.o1 + raise TApplicationException(TApplicationException.MISSING_RESULT, "get_principals_in_role failed: unknown result") + + def get_role_grants_for_principal(self, request): + """ + Parameters: + - request + + """ + self.send_get_role_grants_for_principal(request) + return self.recv_get_role_grants_for_principal() + + def send_get_role_grants_for_principal(self, request): + self._oprot.writeMessageBegin("get_role_grants_for_principal", TMessageType.CALL, self._seqid) + args = get_role_grants_for_principal_args() + args.request = request + args.write(self._oprot) + self._oprot.writeMessageEnd() + self._oprot.trans.flush() + + def recv_get_role_grants_for_principal(self): + iprot = self._iprot + (fname, mtype, rseqid) = iprot.readMessageBegin() + if mtype == TMessageType.EXCEPTION: + x = TApplicationException() + x.read(iprot) + iprot.readMessageEnd() + raise x + result = get_role_grants_for_principal_result() + result.read(iprot) + iprot.readMessageEnd() + if result.success is not None: + return result.success + if result.o1 is not None: + raise result.o1 + raise TApplicationException(TApplicationException.MISSING_RESULT, "get_role_grants_for_principal failed: unknown result") + + def get_privilege_set(self, hiveObject, user_name, group_names): + """ + Parameters: + - hiveObject + - user_name + - group_names + + """ + self.send_get_privilege_set(hiveObject, user_name, group_names) + return self.recv_get_privilege_set() + + def send_get_privilege_set(self, hiveObject, user_name, group_names): + self._oprot.writeMessageBegin("get_privilege_set", TMessageType.CALL, self._seqid) + args = get_privilege_set_args() + args.hiveObject = hiveObject + args.user_name = user_name + args.group_names = group_names + args.write(self._oprot) + self._oprot.writeMessageEnd() + self._oprot.trans.flush() + + def recv_get_privilege_set(self): + iprot = self._iprot + (fname, mtype, rseqid) = iprot.readMessageBegin() + if mtype == TMessageType.EXCEPTION: + x = TApplicationException() + x.read(iprot) + iprot.readMessageEnd() + raise x + result = get_privilege_set_result() + result.read(iprot) + iprot.readMessageEnd() + if result.success is not None: + return result.success + if result.o1 is not None: + raise result.o1 + raise TApplicationException(TApplicationException.MISSING_RESULT, "get_privilege_set failed: unknown result") + + def list_privileges(self, principal_name, principal_type, hiveObject): + """ + Parameters: + - principal_name + - principal_type + - hiveObject + + """ + self.send_list_privileges(principal_name, principal_type, hiveObject) + return self.recv_list_privileges() + + def send_list_privileges(self, principal_name, principal_type, hiveObject): + self._oprot.writeMessageBegin("list_privileges", TMessageType.CALL, self._seqid) + args = list_privileges_args() + args.principal_name = principal_name + args.principal_type = principal_type + args.hiveObject = hiveObject + args.write(self._oprot) + self._oprot.writeMessageEnd() + self._oprot.trans.flush() + + def recv_list_privileges(self): + iprot = self._iprot + (fname, mtype, rseqid) = iprot.readMessageBegin() + if mtype == TMessageType.EXCEPTION: + x = TApplicationException() + x.read(iprot) + iprot.readMessageEnd() + raise x + result = list_privileges_result() + result.read(iprot) + iprot.readMessageEnd() + if result.success is not None: + return result.success + if result.o1 is not None: + raise result.o1 + raise TApplicationException(TApplicationException.MISSING_RESULT, "list_privileges failed: unknown result") + + def grant_privileges(self, privileges): + """ + Parameters: + - privileges + + """ + self.send_grant_privileges(privileges) + return self.recv_grant_privileges() + + def send_grant_privileges(self, privileges): + self._oprot.writeMessageBegin("grant_privileges", TMessageType.CALL, self._seqid) + args = grant_privileges_args() + args.privileges = privileges + args.write(self._oprot) + self._oprot.writeMessageEnd() + self._oprot.trans.flush() + + def recv_grant_privileges(self): + iprot = self._iprot + (fname, mtype, rseqid) = iprot.readMessageBegin() + if mtype == TMessageType.EXCEPTION: + x = TApplicationException() + x.read(iprot) + iprot.readMessageEnd() + raise x + result = grant_privileges_result() + result.read(iprot) + iprot.readMessageEnd() + if result.success is not None: + return result.success + if result.o1 is not None: + raise result.o1 + raise TApplicationException(TApplicationException.MISSING_RESULT, "grant_privileges failed: unknown result") + + def revoke_privileges(self, privileges): + """ + Parameters: + - privileges + + """ + self.send_revoke_privileges(privileges) + return self.recv_revoke_privileges() + + def send_revoke_privileges(self, privileges): + self._oprot.writeMessageBegin("revoke_privileges", TMessageType.CALL, self._seqid) + args = revoke_privileges_args() + args.privileges = privileges + args.write(self._oprot) + self._oprot.writeMessageEnd() + self._oprot.trans.flush() + + def recv_revoke_privileges(self): + iprot = self._iprot + (fname, mtype, rseqid) = iprot.readMessageBegin() + if mtype == TMessageType.EXCEPTION: + x = TApplicationException() + x.read(iprot) + iprot.readMessageEnd() + raise x + result = revoke_privileges_result() + result.read(iprot) + iprot.readMessageEnd() + if result.success is not None: + return result.success + if result.o1 is not None: + raise result.o1 + raise TApplicationException(TApplicationException.MISSING_RESULT, "revoke_privileges failed: unknown result") + + def grant_revoke_privileges(self, request): + """ + Parameters: + - request + + """ + self.send_grant_revoke_privileges(request) + return self.recv_grant_revoke_privileges() + + def send_grant_revoke_privileges(self, request): + self._oprot.writeMessageBegin("grant_revoke_privileges", TMessageType.CALL, self._seqid) + args = grant_revoke_privileges_args() + args.request = request + args.write(self._oprot) + self._oprot.writeMessageEnd() + self._oprot.trans.flush() + + def recv_grant_revoke_privileges(self): + iprot = self._iprot + (fname, mtype, rseqid) = iprot.readMessageBegin() + if mtype == TMessageType.EXCEPTION: + x = TApplicationException() + x.read(iprot) + iprot.readMessageEnd() + raise x + result = grant_revoke_privileges_result() + result.read(iprot) + iprot.readMessageEnd() + if result.success is not None: + return result.success + if result.o1 is not None: + raise result.o1 + raise TApplicationException(TApplicationException.MISSING_RESULT, "grant_revoke_privileges failed: unknown result") + + def refresh_privileges(self, objToRefresh, authorizer, grantRequest): + """ + Parameters: + - objToRefresh + - authorizer + - grantRequest + + """ + self.send_refresh_privileges(objToRefresh, authorizer, grantRequest) + return self.recv_refresh_privileges() + + def send_refresh_privileges(self, objToRefresh, authorizer, grantRequest): + self._oprot.writeMessageBegin("refresh_privileges", TMessageType.CALL, self._seqid) + args = refresh_privileges_args() + args.objToRefresh = objToRefresh + args.authorizer = authorizer + args.grantRequest = grantRequest + args.write(self._oprot) + self._oprot.writeMessageEnd() + self._oprot.trans.flush() + + def recv_refresh_privileges(self): + iprot = self._iprot + (fname, mtype, rseqid) = iprot.readMessageBegin() + if mtype == TMessageType.EXCEPTION: + x = TApplicationException() + x.read(iprot) + iprot.readMessageEnd() + raise x + result = refresh_privileges_result() + result.read(iprot) + iprot.readMessageEnd() + if result.success is not None: + return result.success + if result.o1 is not None: + raise result.o1 + raise TApplicationException(TApplicationException.MISSING_RESULT, "refresh_privileges failed: unknown result") + + def set_ugi(self, user_name, group_names): + """ + Parameters: + - user_name + - group_names + + """ + self.send_set_ugi(user_name, group_names) + return self.recv_set_ugi() + + def send_set_ugi(self, user_name, group_names): + self._oprot.writeMessageBegin("set_ugi", TMessageType.CALL, self._seqid) + args = set_ugi_args() + args.user_name = user_name + args.group_names = group_names + args.write(self._oprot) + self._oprot.writeMessageEnd() + self._oprot.trans.flush() + + def recv_set_ugi(self): + iprot = self._iprot + (fname, mtype, rseqid) = iprot.readMessageBegin() + if mtype == TMessageType.EXCEPTION: + x = TApplicationException() + x.read(iprot) + iprot.readMessageEnd() + raise x + result = set_ugi_result() + result.read(iprot) + iprot.readMessageEnd() + if result.success is not None: + return result.success + if result.o1 is not None: + raise result.o1 + raise TApplicationException(TApplicationException.MISSING_RESULT, "set_ugi failed: unknown result") + + def get_delegation_token(self, token_owner, renewer_kerberos_principal_name): + """ + Parameters: + - token_owner + - renewer_kerberos_principal_name + + """ + self.send_get_delegation_token(token_owner, renewer_kerberos_principal_name) + return self.recv_get_delegation_token() + + def send_get_delegation_token(self, token_owner, renewer_kerberos_principal_name): + self._oprot.writeMessageBegin("get_delegation_token", TMessageType.CALL, self._seqid) + args = get_delegation_token_args() + args.token_owner = token_owner + args.renewer_kerberos_principal_name = renewer_kerberos_principal_name + args.write(self._oprot) + self._oprot.writeMessageEnd() + self._oprot.trans.flush() + + def recv_get_delegation_token(self): + iprot = self._iprot + (fname, mtype, rseqid) = iprot.readMessageBegin() + if mtype == TMessageType.EXCEPTION: + x = TApplicationException() + x.read(iprot) + iprot.readMessageEnd() + raise x + result = get_delegation_token_result() + result.read(iprot) + iprot.readMessageEnd() + if result.success is not None: + return result.success + if result.o1 is not None: + raise result.o1 + raise TApplicationException(TApplicationException.MISSING_RESULT, "get_delegation_token failed: unknown result") + + def renew_delegation_token(self, token_str_form): + """ + Parameters: + - token_str_form + + """ + self.send_renew_delegation_token(token_str_form) + return self.recv_renew_delegation_token() + + def send_renew_delegation_token(self, token_str_form): + self._oprot.writeMessageBegin("renew_delegation_token", TMessageType.CALL, self._seqid) + args = renew_delegation_token_args() + args.token_str_form = token_str_form + args.write(self._oprot) + self._oprot.writeMessageEnd() + self._oprot.trans.flush() + + def recv_renew_delegation_token(self): + iprot = self._iprot + (fname, mtype, rseqid) = iprot.readMessageBegin() + if mtype == TMessageType.EXCEPTION: + x = TApplicationException() + x.read(iprot) + iprot.readMessageEnd() + raise x + result = renew_delegation_token_result() + result.read(iprot) + iprot.readMessageEnd() + if result.success is not None: + return result.success + if result.o1 is not None: + raise result.o1 + raise TApplicationException(TApplicationException.MISSING_RESULT, "renew_delegation_token failed: unknown result") + + def cancel_delegation_token(self, token_str_form): + """ + Parameters: + - token_str_form + + """ + self.send_cancel_delegation_token(token_str_form) + self.recv_cancel_delegation_token() + + def send_cancel_delegation_token(self, token_str_form): + self._oprot.writeMessageBegin("cancel_delegation_token", TMessageType.CALL, self._seqid) + args = cancel_delegation_token_args() + args.token_str_form = token_str_form + args.write(self._oprot) + self._oprot.writeMessageEnd() + self._oprot.trans.flush() + + def recv_cancel_delegation_token(self): + iprot = self._iprot + (fname, mtype, rseqid) = iprot.readMessageBegin() + if mtype == TMessageType.EXCEPTION: + x = TApplicationException() + x.read(iprot) + iprot.readMessageEnd() + raise x + result = cancel_delegation_token_result() + result.read(iprot) + iprot.readMessageEnd() + if result.o1 is not None: + raise result.o1 + return + + def add_token(self, token_identifier, delegation_token): + """ + Parameters: + - token_identifier + - delegation_token + + """ + self.send_add_token(token_identifier, delegation_token) + return self.recv_add_token() + + def send_add_token(self, token_identifier, delegation_token): + self._oprot.writeMessageBegin("add_token", TMessageType.CALL, self._seqid) + args = add_token_args() + args.token_identifier = token_identifier + args.delegation_token = delegation_token + args.write(self._oprot) + self._oprot.writeMessageEnd() + self._oprot.trans.flush() + + def recv_add_token(self): + iprot = self._iprot + (fname, mtype, rseqid) = iprot.readMessageBegin() + if mtype == TMessageType.EXCEPTION: + x = TApplicationException() + x.read(iprot) + iprot.readMessageEnd() + raise x + result = add_token_result() + result.read(iprot) + iprot.readMessageEnd() + if result.success is not None: + return result.success + raise TApplicationException(TApplicationException.MISSING_RESULT, "add_token failed: unknown result") + + def remove_token(self, token_identifier): + """ + Parameters: + - token_identifier + + """ + self.send_remove_token(token_identifier) + return self.recv_remove_token() + + def send_remove_token(self, token_identifier): + self._oprot.writeMessageBegin("remove_token", TMessageType.CALL, self._seqid) + args = remove_token_args() + args.token_identifier = token_identifier + args.write(self._oprot) + self._oprot.writeMessageEnd() + self._oprot.trans.flush() + + def recv_remove_token(self): + iprot = self._iprot + (fname, mtype, rseqid) = iprot.readMessageBegin() + if mtype == TMessageType.EXCEPTION: + x = TApplicationException() + x.read(iprot) + iprot.readMessageEnd() + raise x + result = remove_token_result() + result.read(iprot) + iprot.readMessageEnd() + if result.success is not None: + return result.success + raise TApplicationException(TApplicationException.MISSING_RESULT, "remove_token failed: unknown result") + + def get_token(self, token_identifier): + """ + Parameters: + - token_identifier + + """ + self.send_get_token(token_identifier) + return self.recv_get_token() + + def send_get_token(self, token_identifier): + self._oprot.writeMessageBegin("get_token", TMessageType.CALL, self._seqid) + args = get_token_args() + args.token_identifier = token_identifier + args.write(self._oprot) + self._oprot.writeMessageEnd() + self._oprot.trans.flush() + + def recv_get_token(self): + iprot = self._iprot + (fname, mtype, rseqid) = iprot.readMessageBegin() + if mtype == TMessageType.EXCEPTION: + x = TApplicationException() + x.read(iprot) + iprot.readMessageEnd() + raise x + result = get_token_result() + result.read(iprot) + iprot.readMessageEnd() + if result.success is not None: + return result.success + raise TApplicationException(TApplicationException.MISSING_RESULT, "get_token failed: unknown result") + + def get_all_token_identifiers(self): + self.send_get_all_token_identifiers() + return self.recv_get_all_token_identifiers() + + def send_get_all_token_identifiers(self): + self._oprot.writeMessageBegin("get_all_token_identifiers", TMessageType.CALL, self._seqid) + args = get_all_token_identifiers_args() + args.write(self._oprot) + self._oprot.writeMessageEnd() + self._oprot.trans.flush() + + def recv_get_all_token_identifiers(self): + iprot = self._iprot + (fname, mtype, rseqid) = iprot.readMessageBegin() + if mtype == TMessageType.EXCEPTION: + x = TApplicationException() + x.read(iprot) + iprot.readMessageEnd() + raise x + result = get_all_token_identifiers_result() + result.read(iprot) + iprot.readMessageEnd() + if result.success is not None: + return result.success + raise TApplicationException(TApplicationException.MISSING_RESULT, "get_all_token_identifiers failed: unknown result") + + def add_master_key(self, key): + """ + Parameters: + - key + + """ + self.send_add_master_key(key) + return self.recv_add_master_key() + + def send_add_master_key(self, key): + self._oprot.writeMessageBegin("add_master_key", TMessageType.CALL, self._seqid) + args = add_master_key_args() + args.key = key + args.write(self._oprot) + self._oprot.writeMessageEnd() + self._oprot.trans.flush() + + def recv_add_master_key(self): + iprot = self._iprot + (fname, mtype, rseqid) = iprot.readMessageBegin() + if mtype == TMessageType.EXCEPTION: + x = TApplicationException() + x.read(iprot) + iprot.readMessageEnd() + raise x + result = add_master_key_result() + result.read(iprot) + iprot.readMessageEnd() + if result.success is not None: + return result.success + if result.o1 is not None: + raise result.o1 + raise TApplicationException(TApplicationException.MISSING_RESULT, "add_master_key failed: unknown result") + + def update_master_key(self, seq_number, key): + """ + Parameters: + - seq_number + - key + + """ + self.send_update_master_key(seq_number, key) + self.recv_update_master_key() + + def send_update_master_key(self, seq_number, key): + self._oprot.writeMessageBegin("update_master_key", TMessageType.CALL, self._seqid) + args = update_master_key_args() + args.seq_number = seq_number + args.key = key + args.write(self._oprot) + self._oprot.writeMessageEnd() + self._oprot.trans.flush() + + def recv_update_master_key(self): + iprot = self._iprot + (fname, mtype, rseqid) = iprot.readMessageBegin() + if mtype == TMessageType.EXCEPTION: + x = TApplicationException() + x.read(iprot) + iprot.readMessageEnd() + raise x + result = update_master_key_result() + result.read(iprot) + iprot.readMessageEnd() + if result.o1 is not None: + raise result.o1 + if result.o2 is not None: + raise result.o2 + return + + def remove_master_key(self, key_seq): + """ + Parameters: + - key_seq + + """ + self.send_remove_master_key(key_seq) + return self.recv_remove_master_key() + + def send_remove_master_key(self, key_seq): + self._oprot.writeMessageBegin("remove_master_key", TMessageType.CALL, self._seqid) + args = remove_master_key_args() + args.key_seq = key_seq + args.write(self._oprot) + self._oprot.writeMessageEnd() + self._oprot.trans.flush() + + def recv_remove_master_key(self): + iprot = self._iprot + (fname, mtype, rseqid) = iprot.readMessageBegin() + if mtype == TMessageType.EXCEPTION: + x = TApplicationException() + x.read(iprot) + iprot.readMessageEnd() + raise x + result = remove_master_key_result() + result.read(iprot) + iprot.readMessageEnd() + if result.success is not None: + return result.success + raise TApplicationException(TApplicationException.MISSING_RESULT, "remove_master_key failed: unknown result") + + def get_master_keys(self): + self.send_get_master_keys() + return self.recv_get_master_keys() + + def send_get_master_keys(self): + self._oprot.writeMessageBegin("get_master_keys", TMessageType.CALL, self._seqid) + args = get_master_keys_args() + args.write(self._oprot) + self._oprot.writeMessageEnd() + self._oprot.trans.flush() + + def recv_get_master_keys(self): + iprot = self._iprot + (fname, mtype, rseqid) = iprot.readMessageBegin() + if mtype == TMessageType.EXCEPTION: + x = TApplicationException() + x.read(iprot) + iprot.readMessageEnd() + raise x + result = get_master_keys_result() + result.read(iprot) + iprot.readMessageEnd() + if result.success is not None: + return result.success + raise TApplicationException(TApplicationException.MISSING_RESULT, "get_master_keys failed: unknown result") + + def get_open_txns(self): + self.send_get_open_txns() + return self.recv_get_open_txns() + + def send_get_open_txns(self): + self._oprot.writeMessageBegin("get_open_txns", TMessageType.CALL, self._seqid) + args = get_open_txns_args() + args.write(self._oprot) + self._oprot.writeMessageEnd() + self._oprot.trans.flush() + + def recv_get_open_txns(self): + iprot = self._iprot + (fname, mtype, rseqid) = iprot.readMessageBegin() + if mtype == TMessageType.EXCEPTION: + x = TApplicationException() + x.read(iprot) + iprot.readMessageEnd() + raise x + result = get_open_txns_result() + result.read(iprot) + iprot.readMessageEnd() + if result.success is not None: + return result.success + raise TApplicationException(TApplicationException.MISSING_RESULT, "get_open_txns failed: unknown result") + + def get_open_txns_info(self): + self.send_get_open_txns_info() + return self.recv_get_open_txns_info() + + def send_get_open_txns_info(self): + self._oprot.writeMessageBegin("get_open_txns_info", TMessageType.CALL, self._seqid) + args = get_open_txns_info_args() + args.write(self._oprot) + self._oprot.writeMessageEnd() + self._oprot.trans.flush() + + def recv_get_open_txns_info(self): + iprot = self._iprot + (fname, mtype, rseqid) = iprot.readMessageBegin() + if mtype == TMessageType.EXCEPTION: + x = TApplicationException() + x.read(iprot) + iprot.readMessageEnd() + raise x + result = get_open_txns_info_result() + result.read(iprot) + iprot.readMessageEnd() + if result.success is not None: + return result.success + raise TApplicationException(TApplicationException.MISSING_RESULT, "get_open_txns_info failed: unknown result") + + def open_txns(self, rqst): + """ + Parameters: + - rqst + + """ + self.send_open_txns(rqst) + return self.recv_open_txns() + + def send_open_txns(self, rqst): + self._oprot.writeMessageBegin("open_txns", TMessageType.CALL, self._seqid) + args = open_txns_args() + args.rqst = rqst + args.write(self._oprot) + self._oprot.writeMessageEnd() + self._oprot.trans.flush() + + def recv_open_txns(self): + iprot = self._iprot + (fname, mtype, rseqid) = iprot.readMessageBegin() + if mtype == TMessageType.EXCEPTION: + x = TApplicationException() + x.read(iprot) + iprot.readMessageEnd() + raise x + result = open_txns_result() + result.read(iprot) + iprot.readMessageEnd() + if result.success is not None: + return result.success + raise TApplicationException(TApplicationException.MISSING_RESULT, "open_txns failed: unknown result") + + def abort_txn(self, rqst): + """ + Parameters: + - rqst + + """ + self.send_abort_txn(rqst) + self.recv_abort_txn() + + def send_abort_txn(self, rqst): + self._oprot.writeMessageBegin("abort_txn", TMessageType.CALL, self._seqid) + args = abort_txn_args() + args.rqst = rqst + args.write(self._oprot) + self._oprot.writeMessageEnd() + self._oprot.trans.flush() + + def recv_abort_txn(self): + iprot = self._iprot + (fname, mtype, rseqid) = iprot.readMessageBegin() + if mtype == TMessageType.EXCEPTION: + x = TApplicationException() + x.read(iprot) + iprot.readMessageEnd() + raise x + result = abort_txn_result() + result.read(iprot) + iprot.readMessageEnd() + if result.o1 is not None: + raise result.o1 + return + + def abort_txns(self, rqst): + """ + Parameters: + - rqst + + """ + self.send_abort_txns(rqst) + self.recv_abort_txns() + + def send_abort_txns(self, rqst): + self._oprot.writeMessageBegin("abort_txns", TMessageType.CALL, self._seqid) + args = abort_txns_args() + args.rqst = rqst + args.write(self._oprot) + self._oprot.writeMessageEnd() + self._oprot.trans.flush() + + def recv_abort_txns(self): + iprot = self._iprot + (fname, mtype, rseqid) = iprot.readMessageBegin() + if mtype == TMessageType.EXCEPTION: + x = TApplicationException() + x.read(iprot) + iprot.readMessageEnd() + raise x + result = abort_txns_result() + result.read(iprot) + iprot.readMessageEnd() + if result.o1 is not None: + raise result.o1 + return + + def commit_txn(self, rqst): + """ + Parameters: + - rqst + + """ + self.send_commit_txn(rqst) + self.recv_commit_txn() + + def send_commit_txn(self, rqst): + self._oprot.writeMessageBegin("commit_txn", TMessageType.CALL, self._seqid) + args = commit_txn_args() + args.rqst = rqst + args.write(self._oprot) + self._oprot.writeMessageEnd() + self._oprot.trans.flush() + + def recv_commit_txn(self): + iprot = self._iprot + (fname, mtype, rseqid) = iprot.readMessageBegin() + if mtype == TMessageType.EXCEPTION: + x = TApplicationException() + x.read(iprot) + iprot.readMessageEnd() + raise x + result = commit_txn_result() + result.read(iprot) + iprot.readMessageEnd() + if result.o1 is not None: + raise result.o1 + if result.o2 is not None: + raise result.o2 + return + + def get_latest_txnid_in_conflict(self, txnId): + """ + Parameters: + - txnId + + """ + self.send_get_latest_txnid_in_conflict(txnId) + return self.recv_get_latest_txnid_in_conflict() + + def send_get_latest_txnid_in_conflict(self, txnId): + self._oprot.writeMessageBegin("get_latest_txnid_in_conflict", TMessageType.CALL, self._seqid) + args = get_latest_txnid_in_conflict_args() + args.txnId = txnId + args.write(self._oprot) + self._oprot.writeMessageEnd() + self._oprot.trans.flush() + + def recv_get_latest_txnid_in_conflict(self): + iprot = self._iprot + (fname, mtype, rseqid) = iprot.readMessageBegin() + if mtype == TMessageType.EXCEPTION: + x = TApplicationException() + x.read(iprot) + iprot.readMessageEnd() + raise x + result = get_latest_txnid_in_conflict_result() + result.read(iprot) + iprot.readMessageEnd() + if result.success is not None: + return result.success + if result.o1 is not None: + raise result.o1 + raise TApplicationException(TApplicationException.MISSING_RESULT, "get_latest_txnid_in_conflict failed: unknown result") + + def repl_tbl_writeid_state(self, rqst): + """ + Parameters: + - rqst + + """ + self.send_repl_tbl_writeid_state(rqst) + self.recv_repl_tbl_writeid_state() + + def send_repl_tbl_writeid_state(self, rqst): + self._oprot.writeMessageBegin("repl_tbl_writeid_state", TMessageType.CALL, self._seqid) + args = repl_tbl_writeid_state_args() + args.rqst = rqst + args.write(self._oprot) + self._oprot.writeMessageEnd() + self._oprot.trans.flush() + + def recv_repl_tbl_writeid_state(self): + iprot = self._iprot + (fname, mtype, rseqid) = iprot.readMessageBegin() + if mtype == TMessageType.EXCEPTION: + x = TApplicationException() + x.read(iprot) + iprot.readMessageEnd() + raise x + result = repl_tbl_writeid_state_result() + result.read(iprot) + iprot.readMessageEnd() + return + + def get_valid_write_ids(self, rqst): + """ + Parameters: + - rqst + + """ + self.send_get_valid_write_ids(rqst) + return self.recv_get_valid_write_ids() + + def send_get_valid_write_ids(self, rqst): + self._oprot.writeMessageBegin("get_valid_write_ids", TMessageType.CALL, self._seqid) + args = get_valid_write_ids_args() + args.rqst = rqst + args.write(self._oprot) + self._oprot.writeMessageEnd() + self._oprot.trans.flush() + + def recv_get_valid_write_ids(self): + iprot = self._iprot + (fname, mtype, rseqid) = iprot.readMessageBegin() + if mtype == TMessageType.EXCEPTION: + x = TApplicationException() + x.read(iprot) + iprot.readMessageEnd() + raise x + result = get_valid_write_ids_result() + result.read(iprot) + iprot.readMessageEnd() + if result.success is not None: + return result.success + if result.o1 is not None: + raise result.o1 + if result.o2 is not None: + raise result.o2 + raise TApplicationException(TApplicationException.MISSING_RESULT, "get_valid_write_ids failed: unknown result") + + def allocate_table_write_ids(self, rqst): + """ + Parameters: + - rqst + + """ + self.send_allocate_table_write_ids(rqst) + return self.recv_allocate_table_write_ids() + + def send_allocate_table_write_ids(self, rqst): + self._oprot.writeMessageBegin("allocate_table_write_ids", TMessageType.CALL, self._seqid) + args = allocate_table_write_ids_args() + args.rqst = rqst + args.write(self._oprot) + self._oprot.writeMessageEnd() + self._oprot.trans.flush() + + def recv_allocate_table_write_ids(self): + iprot = self._iprot + (fname, mtype, rseqid) = iprot.readMessageBegin() + if mtype == TMessageType.EXCEPTION: + x = TApplicationException() + x.read(iprot) + iprot.readMessageEnd() + raise x + result = allocate_table_write_ids_result() + result.read(iprot) + iprot.readMessageEnd() + if result.success is not None: + return result.success + if result.o1 is not None: + raise result.o1 + if result.o2 is not None: + raise result.o2 + if result.o3 is not None: + raise result.o3 + raise TApplicationException(TApplicationException.MISSING_RESULT, "allocate_table_write_ids failed: unknown result") + + def get_max_allocated_table_write_id(self, rqst): + """ + Parameters: + - rqst + + """ + self.send_get_max_allocated_table_write_id(rqst) + return self.recv_get_max_allocated_table_write_id() + + def send_get_max_allocated_table_write_id(self, rqst): + self._oprot.writeMessageBegin("get_max_allocated_table_write_id", TMessageType.CALL, self._seqid) + args = get_max_allocated_table_write_id_args() + args.rqst = rqst + args.write(self._oprot) + self._oprot.writeMessageEnd() + self._oprot.trans.flush() + + def recv_get_max_allocated_table_write_id(self): + iprot = self._iprot + (fname, mtype, rseqid) = iprot.readMessageBegin() + if mtype == TMessageType.EXCEPTION: + x = TApplicationException() + x.read(iprot) + iprot.readMessageEnd() + raise x + result = get_max_allocated_table_write_id_result() + result.read(iprot) + iprot.readMessageEnd() + if result.success is not None: + return result.success + if result.o1 is not None: + raise result.o1 + raise TApplicationException( + TApplicationException.MISSING_RESULT, "get_max_allocated_table_write_id failed: unknown result" + ) + + def seed_write_id(self, rqst): + """ + Parameters: + - rqst + + """ + self.send_seed_write_id(rqst) + self.recv_seed_write_id() + + def send_seed_write_id(self, rqst): + self._oprot.writeMessageBegin("seed_write_id", TMessageType.CALL, self._seqid) + args = seed_write_id_args() + args.rqst = rqst + args.write(self._oprot) + self._oprot.writeMessageEnd() + self._oprot.trans.flush() + + def recv_seed_write_id(self): + iprot = self._iprot + (fname, mtype, rseqid) = iprot.readMessageBegin() + if mtype == TMessageType.EXCEPTION: + x = TApplicationException() + x.read(iprot) + iprot.readMessageEnd() + raise x + result = seed_write_id_result() + result.read(iprot) + iprot.readMessageEnd() + if result.o1 is not None: + raise result.o1 + return + + def seed_txn_id(self, rqst): + """ + Parameters: + - rqst + + """ + self.send_seed_txn_id(rqst) + self.recv_seed_txn_id() + + def send_seed_txn_id(self, rqst): + self._oprot.writeMessageBegin("seed_txn_id", TMessageType.CALL, self._seqid) + args = seed_txn_id_args() + args.rqst = rqst + args.write(self._oprot) + self._oprot.writeMessageEnd() + self._oprot.trans.flush() + + def recv_seed_txn_id(self): + iprot = self._iprot + (fname, mtype, rseqid) = iprot.readMessageBegin() + if mtype == TMessageType.EXCEPTION: + x = TApplicationException() + x.read(iprot) + iprot.readMessageEnd() + raise x + result = seed_txn_id_result() + result.read(iprot) + iprot.readMessageEnd() + if result.o1 is not None: + raise result.o1 + return + + def lock(self, rqst): + """ + Parameters: + - rqst + + """ + self.send_lock(rqst) + return self.recv_lock() + + def send_lock(self, rqst): + self._oprot.writeMessageBegin("lock", TMessageType.CALL, self._seqid) + args = lock_args() + args.rqst = rqst + args.write(self._oprot) + self._oprot.writeMessageEnd() + self._oprot.trans.flush() + + def recv_lock(self): + iprot = self._iprot + (fname, mtype, rseqid) = iprot.readMessageBegin() + if mtype == TMessageType.EXCEPTION: + x = TApplicationException() + x.read(iprot) + iprot.readMessageEnd() + raise x + result = lock_result() + result.read(iprot) + iprot.readMessageEnd() + if result.success is not None: + return result.success + if result.o1 is not None: + raise result.o1 + if result.o2 is not None: + raise result.o2 + raise TApplicationException(TApplicationException.MISSING_RESULT, "lock failed: unknown result") + + def check_lock(self, rqst): + """ + Parameters: + - rqst + + """ + self.send_check_lock(rqst) + return self.recv_check_lock() + + def send_check_lock(self, rqst): + self._oprot.writeMessageBegin("check_lock", TMessageType.CALL, self._seqid) + args = check_lock_args() + args.rqst = rqst + args.write(self._oprot) + self._oprot.writeMessageEnd() + self._oprot.trans.flush() + + def recv_check_lock(self): + iprot = self._iprot + (fname, mtype, rseqid) = iprot.readMessageBegin() + if mtype == TMessageType.EXCEPTION: + x = TApplicationException() + x.read(iprot) + iprot.readMessageEnd() + raise x + result = check_lock_result() + result.read(iprot) + iprot.readMessageEnd() + if result.success is not None: + return result.success + if result.o1 is not None: + raise result.o1 + if result.o2 is not None: + raise result.o2 + if result.o3 is not None: + raise result.o3 + raise TApplicationException(TApplicationException.MISSING_RESULT, "check_lock failed: unknown result") + + def unlock(self, rqst): + """ + Parameters: + - rqst + + """ + self.send_unlock(rqst) + self.recv_unlock() + + def send_unlock(self, rqst): + self._oprot.writeMessageBegin("unlock", TMessageType.CALL, self._seqid) + args = unlock_args() + args.rqst = rqst + args.write(self._oprot) + self._oprot.writeMessageEnd() + self._oprot.trans.flush() + + def recv_unlock(self): + iprot = self._iprot + (fname, mtype, rseqid) = iprot.readMessageBegin() + if mtype == TMessageType.EXCEPTION: + x = TApplicationException() + x.read(iprot) + iprot.readMessageEnd() + raise x + result = unlock_result() + result.read(iprot) + iprot.readMessageEnd() + if result.o1 is not None: + raise result.o1 + if result.o2 is not None: + raise result.o2 + return + + def show_locks(self, rqst): + """ + Parameters: + - rqst + + """ + self.send_show_locks(rqst) + return self.recv_show_locks() + + def send_show_locks(self, rqst): + self._oprot.writeMessageBegin("show_locks", TMessageType.CALL, self._seqid) + args = show_locks_args() + args.rqst = rqst + args.write(self._oprot) + self._oprot.writeMessageEnd() + self._oprot.trans.flush() + + def recv_show_locks(self): + iprot = self._iprot + (fname, mtype, rseqid) = iprot.readMessageBegin() + if mtype == TMessageType.EXCEPTION: + x = TApplicationException() + x.read(iprot) + iprot.readMessageEnd() + raise x + result = show_locks_result() + result.read(iprot) + iprot.readMessageEnd() + if result.success is not None: + return result.success + raise TApplicationException(TApplicationException.MISSING_RESULT, "show_locks failed: unknown result") + + def heartbeat(self, ids): + """ + Parameters: + - ids + + """ + self.send_heartbeat(ids) + self.recv_heartbeat() + + def send_heartbeat(self, ids): + self._oprot.writeMessageBegin("heartbeat", TMessageType.CALL, self._seqid) + args = heartbeat_args() + args.ids = ids + args.write(self._oprot) + self._oprot.writeMessageEnd() + self._oprot.trans.flush() + + def recv_heartbeat(self): + iprot = self._iprot + (fname, mtype, rseqid) = iprot.readMessageBegin() + if mtype == TMessageType.EXCEPTION: + x = TApplicationException() + x.read(iprot) + iprot.readMessageEnd() + raise x + result = heartbeat_result() + result.read(iprot) + iprot.readMessageEnd() + if result.o1 is not None: + raise result.o1 + if result.o2 is not None: + raise result.o2 + if result.o3 is not None: + raise result.o3 + return + + def heartbeat_txn_range(self, txns): + """ + Parameters: + - txns + + """ + self.send_heartbeat_txn_range(txns) + return self.recv_heartbeat_txn_range() + + def send_heartbeat_txn_range(self, txns): + self._oprot.writeMessageBegin("heartbeat_txn_range", TMessageType.CALL, self._seqid) + args = heartbeat_txn_range_args() + args.txns = txns + args.write(self._oprot) + self._oprot.writeMessageEnd() + self._oprot.trans.flush() + + def recv_heartbeat_txn_range(self): + iprot = self._iprot + (fname, mtype, rseqid) = iprot.readMessageBegin() + if mtype == TMessageType.EXCEPTION: + x = TApplicationException() + x.read(iprot) + iprot.readMessageEnd() + raise x + result = heartbeat_txn_range_result() + result.read(iprot) + iprot.readMessageEnd() + if result.success is not None: + return result.success + raise TApplicationException(TApplicationException.MISSING_RESULT, "heartbeat_txn_range failed: unknown result") + + def compact(self, rqst): + """ + Parameters: + - rqst + + """ + self.send_compact(rqst) + self.recv_compact() + + def send_compact(self, rqst): + self._oprot.writeMessageBegin("compact", TMessageType.CALL, self._seqid) + args = compact_args() + args.rqst = rqst + args.write(self._oprot) + self._oprot.writeMessageEnd() + self._oprot.trans.flush() + + def recv_compact(self): + iprot = self._iprot + (fname, mtype, rseqid) = iprot.readMessageBegin() + if mtype == TMessageType.EXCEPTION: + x = TApplicationException() + x.read(iprot) + iprot.readMessageEnd() + raise x + result = compact_result() + result.read(iprot) + iprot.readMessageEnd() + return + + def compact2(self, rqst): + """ + Parameters: + - rqst + + """ + self.send_compact2(rqst) + return self.recv_compact2() + + def send_compact2(self, rqst): + self._oprot.writeMessageBegin("compact2", TMessageType.CALL, self._seqid) + args = compact2_args() + args.rqst = rqst + args.write(self._oprot) + self._oprot.writeMessageEnd() + self._oprot.trans.flush() + + def recv_compact2(self): + iprot = self._iprot + (fname, mtype, rseqid) = iprot.readMessageBegin() + if mtype == TMessageType.EXCEPTION: + x = TApplicationException() + x.read(iprot) + iprot.readMessageEnd() + raise x + result = compact2_result() + result.read(iprot) + iprot.readMessageEnd() + if result.success is not None: + return result.success + raise TApplicationException(TApplicationException.MISSING_RESULT, "compact2 failed: unknown result") + + def show_compact(self, rqst): + """ + Parameters: + - rqst + + """ + self.send_show_compact(rqst) + return self.recv_show_compact() + + def send_show_compact(self, rqst): + self._oprot.writeMessageBegin("show_compact", TMessageType.CALL, self._seqid) + args = show_compact_args() + args.rqst = rqst + args.write(self._oprot) + self._oprot.writeMessageEnd() + self._oprot.trans.flush() + + def recv_show_compact(self): + iprot = self._iprot + (fname, mtype, rseqid) = iprot.readMessageBegin() + if mtype == TMessageType.EXCEPTION: + x = TApplicationException() + x.read(iprot) + iprot.readMessageEnd() + raise x + result = show_compact_result() + result.read(iprot) + iprot.readMessageEnd() + if result.success is not None: + return result.success + raise TApplicationException(TApplicationException.MISSING_RESULT, "show_compact failed: unknown result") + + def add_dynamic_partitions(self, rqst): + """ + Parameters: + - rqst + + """ + self.send_add_dynamic_partitions(rqst) + self.recv_add_dynamic_partitions() + + def send_add_dynamic_partitions(self, rqst): + self._oprot.writeMessageBegin("add_dynamic_partitions", TMessageType.CALL, self._seqid) + args = add_dynamic_partitions_args() + args.rqst = rqst + args.write(self._oprot) + self._oprot.writeMessageEnd() + self._oprot.trans.flush() + + def recv_add_dynamic_partitions(self): + iprot = self._iprot + (fname, mtype, rseqid) = iprot.readMessageBegin() + if mtype == TMessageType.EXCEPTION: + x = TApplicationException() + x.read(iprot) + iprot.readMessageEnd() + raise x + result = add_dynamic_partitions_result() + result.read(iprot) + iprot.readMessageEnd() + if result.o1 is not None: + raise result.o1 + if result.o2 is not None: + raise result.o2 + return + + def find_next_compact(self, workerId): + """ + Parameters: + - workerId + + """ + self.send_find_next_compact(workerId) + return self.recv_find_next_compact() + + def send_find_next_compact(self, workerId): + self._oprot.writeMessageBegin("find_next_compact", TMessageType.CALL, self._seqid) + args = find_next_compact_args() + args.workerId = workerId + args.write(self._oprot) + self._oprot.writeMessageEnd() + self._oprot.trans.flush() + + def recv_find_next_compact(self): + iprot = self._iprot + (fname, mtype, rseqid) = iprot.readMessageBegin() + if mtype == TMessageType.EXCEPTION: + x = TApplicationException() + x.read(iprot) + iprot.readMessageEnd() + raise x + result = find_next_compact_result() + result.read(iprot) + iprot.readMessageEnd() + if result.success is not None: + return result.success + if result.o1 is not None: + raise result.o1 + raise TApplicationException(TApplicationException.MISSING_RESULT, "find_next_compact failed: unknown result") + + def find_next_compact2(self, rqst): + """ + Parameters: + - rqst + + """ + self.send_find_next_compact2(rqst) + return self.recv_find_next_compact2() + + def send_find_next_compact2(self, rqst): + self._oprot.writeMessageBegin("find_next_compact2", TMessageType.CALL, self._seqid) + args = find_next_compact2_args() + args.rqst = rqst + args.write(self._oprot) + self._oprot.writeMessageEnd() + self._oprot.trans.flush() + + def recv_find_next_compact2(self): + iprot = self._iprot + (fname, mtype, rseqid) = iprot.readMessageBegin() + if mtype == TMessageType.EXCEPTION: + x = TApplicationException() + x.read(iprot) + iprot.readMessageEnd() + raise x + result = find_next_compact2_result() + result.read(iprot) + iprot.readMessageEnd() + if result.success is not None: + return result.success + if result.o1 is not None: + raise result.o1 + raise TApplicationException(TApplicationException.MISSING_RESULT, "find_next_compact2 failed: unknown result") + + def update_compactor_state(self, cr, txn_id): + """ + Parameters: + - cr + - txn_id + + """ + self.send_update_compactor_state(cr, txn_id) + self.recv_update_compactor_state() + + def send_update_compactor_state(self, cr, txn_id): + self._oprot.writeMessageBegin("update_compactor_state", TMessageType.CALL, self._seqid) + args = update_compactor_state_args() + args.cr = cr + args.txn_id = txn_id + args.write(self._oprot) + self._oprot.writeMessageEnd() + self._oprot.trans.flush() + + def recv_update_compactor_state(self): + iprot = self._iprot + (fname, mtype, rseqid) = iprot.readMessageBegin() + if mtype == TMessageType.EXCEPTION: + x = TApplicationException() + x.read(iprot) + iprot.readMessageEnd() + raise x + result = update_compactor_state_result() + result.read(iprot) + iprot.readMessageEnd() + return + + def find_columns_with_stats(self, cr): + """ + Parameters: + - cr + + """ + self.send_find_columns_with_stats(cr) + return self.recv_find_columns_with_stats() + + def send_find_columns_with_stats(self, cr): + self._oprot.writeMessageBegin("find_columns_with_stats", TMessageType.CALL, self._seqid) + args = find_columns_with_stats_args() + args.cr = cr + args.write(self._oprot) + self._oprot.writeMessageEnd() + self._oprot.trans.flush() + + def recv_find_columns_with_stats(self): + iprot = self._iprot + (fname, mtype, rseqid) = iprot.readMessageBegin() + if mtype == TMessageType.EXCEPTION: + x = TApplicationException() + x.read(iprot) + iprot.readMessageEnd() + raise x + result = find_columns_with_stats_result() + result.read(iprot) + iprot.readMessageEnd() + if result.success is not None: + return result.success + raise TApplicationException(TApplicationException.MISSING_RESULT, "find_columns_with_stats failed: unknown result") + + def mark_cleaned(self, cr): + """ + Parameters: + - cr + + """ + self.send_mark_cleaned(cr) + self.recv_mark_cleaned() + + def send_mark_cleaned(self, cr): + self._oprot.writeMessageBegin("mark_cleaned", TMessageType.CALL, self._seqid) + args = mark_cleaned_args() + args.cr = cr + args.write(self._oprot) + self._oprot.writeMessageEnd() + self._oprot.trans.flush() + + def recv_mark_cleaned(self): + iprot = self._iprot + (fname, mtype, rseqid) = iprot.readMessageBegin() + if mtype == TMessageType.EXCEPTION: + x = TApplicationException() + x.read(iprot) + iprot.readMessageEnd() + raise x + result = mark_cleaned_result() + result.read(iprot) + iprot.readMessageEnd() + if result.o1 is not None: + raise result.o1 + return + + def mark_compacted(self, cr): + """ + Parameters: + - cr + + """ + self.send_mark_compacted(cr) + self.recv_mark_compacted() + + def send_mark_compacted(self, cr): + self._oprot.writeMessageBegin("mark_compacted", TMessageType.CALL, self._seqid) + args = mark_compacted_args() + args.cr = cr + args.write(self._oprot) + self._oprot.writeMessageEnd() + self._oprot.trans.flush() + + def recv_mark_compacted(self): + iprot = self._iprot + (fname, mtype, rseqid) = iprot.readMessageBegin() + if mtype == TMessageType.EXCEPTION: + x = TApplicationException() + x.read(iprot) + iprot.readMessageEnd() + raise x + result = mark_compacted_result() + result.read(iprot) + iprot.readMessageEnd() + if result.o1 is not None: + raise result.o1 + return + + def mark_failed(self, cr): + """ + Parameters: + - cr + + """ + self.send_mark_failed(cr) + self.recv_mark_failed() + + def send_mark_failed(self, cr): + self._oprot.writeMessageBegin("mark_failed", TMessageType.CALL, self._seqid) + args = mark_failed_args() + args.cr = cr + args.write(self._oprot) + self._oprot.writeMessageEnd() + self._oprot.trans.flush() + + def recv_mark_failed(self): + iprot = self._iprot + (fname, mtype, rseqid) = iprot.readMessageBegin() + if mtype == TMessageType.EXCEPTION: + x = TApplicationException() + x.read(iprot) + iprot.readMessageEnd() + raise x + result = mark_failed_result() + result.read(iprot) + iprot.readMessageEnd() + if result.o1 is not None: + raise result.o1 + return + + def mark_refused(self, cr): + """ + Parameters: + - cr + + """ + self.send_mark_refused(cr) + self.recv_mark_refused() + + def send_mark_refused(self, cr): + self._oprot.writeMessageBegin("mark_refused", TMessageType.CALL, self._seqid) + args = mark_refused_args() + args.cr = cr + args.write(self._oprot) + self._oprot.writeMessageEnd() + self._oprot.trans.flush() + + def recv_mark_refused(self): + iprot = self._iprot + (fname, mtype, rseqid) = iprot.readMessageBegin() + if mtype == TMessageType.EXCEPTION: + x = TApplicationException() + x.read(iprot) + iprot.readMessageEnd() + raise x + result = mark_refused_result() + result.read(iprot) + iprot.readMessageEnd() + if result.o1 is not None: + raise result.o1 + return + + def update_compaction_metrics_data(self, data): + """ + Parameters: + - data + + """ + self.send_update_compaction_metrics_data(data) + return self.recv_update_compaction_metrics_data() + + def send_update_compaction_metrics_data(self, data): + self._oprot.writeMessageBegin("update_compaction_metrics_data", TMessageType.CALL, self._seqid) + args = update_compaction_metrics_data_args() + args.data = data + args.write(self._oprot) + self._oprot.writeMessageEnd() + self._oprot.trans.flush() + + def recv_update_compaction_metrics_data(self): + iprot = self._iprot + (fname, mtype, rseqid) = iprot.readMessageBegin() + if mtype == TMessageType.EXCEPTION: + x = TApplicationException() + x.read(iprot) + iprot.readMessageEnd() + raise x + result = update_compaction_metrics_data_result() + result.read(iprot) + iprot.readMessageEnd() + if result.success is not None: + return result.success + if result.o1 is not None: + raise result.o1 + raise TApplicationException(TApplicationException.MISSING_RESULT, "update_compaction_metrics_data failed: unknown result") + + def remove_compaction_metrics_data(self, request): + """ + Parameters: + - request + + """ + self.send_remove_compaction_metrics_data(request) + self.recv_remove_compaction_metrics_data() + + def send_remove_compaction_metrics_data(self, request): + self._oprot.writeMessageBegin("remove_compaction_metrics_data", TMessageType.CALL, self._seqid) + args = remove_compaction_metrics_data_args() + args.request = request + args.write(self._oprot) + self._oprot.writeMessageEnd() + self._oprot.trans.flush() + + def recv_remove_compaction_metrics_data(self): + iprot = self._iprot + (fname, mtype, rseqid) = iprot.readMessageBegin() + if mtype == TMessageType.EXCEPTION: + x = TApplicationException() + x.read(iprot) + iprot.readMessageEnd() + raise x + result = remove_compaction_metrics_data_result() + result.read(iprot) + iprot.readMessageEnd() + if result.o1 is not None: + raise result.o1 + return + + def set_hadoop_jobid(self, jobId, cq_id): + """ + Parameters: + - jobId + - cq_id + + """ + self.send_set_hadoop_jobid(jobId, cq_id) + self.recv_set_hadoop_jobid() + + def send_set_hadoop_jobid(self, jobId, cq_id): + self._oprot.writeMessageBegin("set_hadoop_jobid", TMessageType.CALL, self._seqid) + args = set_hadoop_jobid_args() + args.jobId = jobId + args.cq_id = cq_id + args.write(self._oprot) + self._oprot.writeMessageEnd() + self._oprot.trans.flush() + + def recv_set_hadoop_jobid(self): + iprot = self._iprot + (fname, mtype, rseqid) = iprot.readMessageBegin() + if mtype == TMessageType.EXCEPTION: + x = TApplicationException() + x.read(iprot) + iprot.readMessageEnd() + raise x + result = set_hadoop_jobid_result() + result.read(iprot) + iprot.readMessageEnd() + return + + def get_latest_committed_compaction_info(self, rqst): + """ + Parameters: + - rqst + + """ + self.send_get_latest_committed_compaction_info(rqst) + return self.recv_get_latest_committed_compaction_info() + + def send_get_latest_committed_compaction_info(self, rqst): + self._oprot.writeMessageBegin("get_latest_committed_compaction_info", TMessageType.CALL, self._seqid) + args = get_latest_committed_compaction_info_args() + args.rqst = rqst + args.write(self._oprot) + self._oprot.writeMessageEnd() + self._oprot.trans.flush() + + def recv_get_latest_committed_compaction_info(self): + iprot = self._iprot + (fname, mtype, rseqid) = iprot.readMessageBegin() + if mtype == TMessageType.EXCEPTION: + x = TApplicationException() + x.read(iprot) + iprot.readMessageEnd() + raise x + result = get_latest_committed_compaction_info_result() + result.read(iprot) + iprot.readMessageEnd() + if result.success is not None: + return result.success + raise TApplicationException( + TApplicationException.MISSING_RESULT, "get_latest_committed_compaction_info failed: unknown result" + ) + + def get_next_notification(self, rqst): + """ + Parameters: + - rqst + + """ + self.send_get_next_notification(rqst) + return self.recv_get_next_notification() + + def send_get_next_notification(self, rqst): + self._oprot.writeMessageBegin("get_next_notification", TMessageType.CALL, self._seqid) + args = get_next_notification_args() + args.rqst = rqst + args.write(self._oprot) + self._oprot.writeMessageEnd() + self._oprot.trans.flush() + + def recv_get_next_notification(self): + iprot = self._iprot + (fname, mtype, rseqid) = iprot.readMessageBegin() + if mtype == TMessageType.EXCEPTION: + x = TApplicationException() + x.read(iprot) + iprot.readMessageEnd() + raise x + result = get_next_notification_result() + result.read(iprot) + iprot.readMessageEnd() + if result.success is not None: + return result.success + raise TApplicationException(TApplicationException.MISSING_RESULT, "get_next_notification failed: unknown result") + + def get_current_notificationEventId(self): + self.send_get_current_notificationEventId() + return self.recv_get_current_notificationEventId() + + def send_get_current_notificationEventId(self): + self._oprot.writeMessageBegin("get_current_notificationEventId", TMessageType.CALL, self._seqid) + args = get_current_notificationEventId_args() + args.write(self._oprot) + self._oprot.writeMessageEnd() + self._oprot.trans.flush() + + def recv_get_current_notificationEventId(self): + iprot = self._iprot + (fname, mtype, rseqid) = iprot.readMessageBegin() + if mtype == TMessageType.EXCEPTION: + x = TApplicationException() + x.read(iprot) + iprot.readMessageEnd() + raise x + result = get_current_notificationEventId_result() + result.read(iprot) + iprot.readMessageEnd() + if result.success is not None: + return result.success + raise TApplicationException( + TApplicationException.MISSING_RESULT, "get_current_notificationEventId failed: unknown result" + ) + + def get_notification_events_count(self, rqst): + """ + Parameters: + - rqst + + """ + self.send_get_notification_events_count(rqst) + return self.recv_get_notification_events_count() + + def send_get_notification_events_count(self, rqst): + self._oprot.writeMessageBegin("get_notification_events_count", TMessageType.CALL, self._seqid) + args = get_notification_events_count_args() + args.rqst = rqst + args.write(self._oprot) + self._oprot.writeMessageEnd() + self._oprot.trans.flush() + + def recv_get_notification_events_count(self): + iprot = self._iprot + (fname, mtype, rseqid) = iprot.readMessageBegin() + if mtype == TMessageType.EXCEPTION: + x = TApplicationException() + x.read(iprot) + iprot.readMessageEnd() + raise x + result = get_notification_events_count_result() + result.read(iprot) + iprot.readMessageEnd() + if result.success is not None: + return result.success + raise TApplicationException(TApplicationException.MISSING_RESULT, "get_notification_events_count failed: unknown result") + + def fire_listener_event(self, rqst): + """ + Parameters: + - rqst + + """ + self.send_fire_listener_event(rqst) + return self.recv_fire_listener_event() + + def send_fire_listener_event(self, rqst): + self._oprot.writeMessageBegin("fire_listener_event", TMessageType.CALL, self._seqid) + args = fire_listener_event_args() + args.rqst = rqst + args.write(self._oprot) + self._oprot.writeMessageEnd() + self._oprot.trans.flush() + + def recv_fire_listener_event(self): + iprot = self._iprot + (fname, mtype, rseqid) = iprot.readMessageBegin() + if mtype == TMessageType.EXCEPTION: + x = TApplicationException() + x.read(iprot) + iprot.readMessageEnd() + raise x + result = fire_listener_event_result() + result.read(iprot) + iprot.readMessageEnd() + if result.success is not None: + return result.success + raise TApplicationException(TApplicationException.MISSING_RESULT, "fire_listener_event failed: unknown result") + + def flushCache(self): + self.send_flushCache() + self.recv_flushCache() + + def send_flushCache(self): + self._oprot.writeMessageBegin("flushCache", TMessageType.CALL, self._seqid) + args = flushCache_args() + args.write(self._oprot) + self._oprot.writeMessageEnd() + self._oprot.trans.flush() + + def recv_flushCache(self): + iprot = self._iprot + (fname, mtype, rseqid) = iprot.readMessageBegin() + if mtype == TMessageType.EXCEPTION: + x = TApplicationException() + x.read(iprot) + iprot.readMessageEnd() + raise x + result = flushCache_result() + result.read(iprot) + iprot.readMessageEnd() + return + + def add_write_notification_log(self, rqst): + """ + Parameters: + - rqst + + """ + self.send_add_write_notification_log(rqst) + return self.recv_add_write_notification_log() + + def send_add_write_notification_log(self, rqst): + self._oprot.writeMessageBegin("add_write_notification_log", TMessageType.CALL, self._seqid) + args = add_write_notification_log_args() + args.rqst = rqst + args.write(self._oprot) + self._oprot.writeMessageEnd() + self._oprot.trans.flush() + + def recv_add_write_notification_log(self): + iprot = self._iprot + (fname, mtype, rseqid) = iprot.readMessageBegin() + if mtype == TMessageType.EXCEPTION: + x = TApplicationException() + x.read(iprot) + iprot.readMessageEnd() + raise x + result = add_write_notification_log_result() + result.read(iprot) + iprot.readMessageEnd() + if result.success is not None: + return result.success + raise TApplicationException(TApplicationException.MISSING_RESULT, "add_write_notification_log failed: unknown result") + + def add_write_notification_log_in_batch(self, rqst): + """ + Parameters: + - rqst + + """ + self.send_add_write_notification_log_in_batch(rqst) + return self.recv_add_write_notification_log_in_batch() + + def send_add_write_notification_log_in_batch(self, rqst): + self._oprot.writeMessageBegin("add_write_notification_log_in_batch", TMessageType.CALL, self._seqid) + args = add_write_notification_log_in_batch_args() + args.rqst = rqst + args.write(self._oprot) + self._oprot.writeMessageEnd() + self._oprot.trans.flush() + + def recv_add_write_notification_log_in_batch(self): + iprot = self._iprot + (fname, mtype, rseqid) = iprot.readMessageBegin() + if mtype == TMessageType.EXCEPTION: + x = TApplicationException() + x.read(iprot) + iprot.readMessageEnd() + raise x + result = add_write_notification_log_in_batch_result() + result.read(iprot) + iprot.readMessageEnd() + if result.success is not None: + return result.success + raise TApplicationException( + TApplicationException.MISSING_RESULT, "add_write_notification_log_in_batch failed: unknown result" + ) + + def cm_recycle(self, request): + """ + Parameters: + - request + + """ + self.send_cm_recycle(request) + return self.recv_cm_recycle() + + def send_cm_recycle(self, request): + self._oprot.writeMessageBegin("cm_recycle", TMessageType.CALL, self._seqid) + args = cm_recycle_args() + args.request = request + args.write(self._oprot) + self._oprot.writeMessageEnd() + self._oprot.trans.flush() + + def recv_cm_recycle(self): + iprot = self._iprot + (fname, mtype, rseqid) = iprot.readMessageBegin() + if mtype == TMessageType.EXCEPTION: + x = TApplicationException() + x.read(iprot) + iprot.readMessageEnd() + raise x + result = cm_recycle_result() + result.read(iprot) + iprot.readMessageEnd() + if result.success is not None: + return result.success + if result.o1 is not None: + raise result.o1 + raise TApplicationException(TApplicationException.MISSING_RESULT, "cm_recycle failed: unknown result") + + def get_file_metadata_by_expr(self, req): + """ + Parameters: + - req + + """ + self.send_get_file_metadata_by_expr(req) + return self.recv_get_file_metadata_by_expr() + + def send_get_file_metadata_by_expr(self, req): + self._oprot.writeMessageBegin("get_file_metadata_by_expr", TMessageType.CALL, self._seqid) + args = get_file_metadata_by_expr_args() + args.req = req + args.write(self._oprot) + self._oprot.writeMessageEnd() + self._oprot.trans.flush() + + def recv_get_file_metadata_by_expr(self): + iprot = self._iprot + (fname, mtype, rseqid) = iprot.readMessageBegin() + if mtype == TMessageType.EXCEPTION: + x = TApplicationException() + x.read(iprot) + iprot.readMessageEnd() + raise x + result = get_file_metadata_by_expr_result() + result.read(iprot) + iprot.readMessageEnd() + if result.success is not None: + return result.success + raise TApplicationException(TApplicationException.MISSING_RESULT, "get_file_metadata_by_expr failed: unknown result") + + def get_file_metadata(self, req): + """ + Parameters: + - req + + """ + self.send_get_file_metadata(req) + return self.recv_get_file_metadata() + + def send_get_file_metadata(self, req): + self._oprot.writeMessageBegin("get_file_metadata", TMessageType.CALL, self._seqid) + args = get_file_metadata_args() + args.req = req + args.write(self._oprot) + self._oprot.writeMessageEnd() + self._oprot.trans.flush() + + def recv_get_file_metadata(self): + iprot = self._iprot + (fname, mtype, rseqid) = iprot.readMessageBegin() + if mtype == TMessageType.EXCEPTION: + x = TApplicationException() + x.read(iprot) + iprot.readMessageEnd() + raise x + result = get_file_metadata_result() + result.read(iprot) + iprot.readMessageEnd() + if result.success is not None: + return result.success + raise TApplicationException(TApplicationException.MISSING_RESULT, "get_file_metadata failed: unknown result") + + def put_file_metadata(self, req): + """ + Parameters: + - req + + """ + self.send_put_file_metadata(req) + return self.recv_put_file_metadata() + + def send_put_file_metadata(self, req): + self._oprot.writeMessageBegin("put_file_metadata", TMessageType.CALL, self._seqid) + args = put_file_metadata_args() + args.req = req + args.write(self._oprot) + self._oprot.writeMessageEnd() + self._oprot.trans.flush() + + def recv_put_file_metadata(self): + iprot = self._iprot + (fname, mtype, rseqid) = iprot.readMessageBegin() + if mtype == TMessageType.EXCEPTION: + x = TApplicationException() + x.read(iprot) + iprot.readMessageEnd() + raise x + result = put_file_metadata_result() + result.read(iprot) + iprot.readMessageEnd() + if result.success is not None: + return result.success + raise TApplicationException(TApplicationException.MISSING_RESULT, "put_file_metadata failed: unknown result") + + def clear_file_metadata(self, req): + """ + Parameters: + - req + + """ + self.send_clear_file_metadata(req) + return self.recv_clear_file_metadata() + + def send_clear_file_metadata(self, req): + self._oprot.writeMessageBegin("clear_file_metadata", TMessageType.CALL, self._seqid) + args = clear_file_metadata_args() + args.req = req + args.write(self._oprot) + self._oprot.writeMessageEnd() + self._oprot.trans.flush() + + def recv_clear_file_metadata(self): + iprot = self._iprot + (fname, mtype, rseqid) = iprot.readMessageBegin() + if mtype == TMessageType.EXCEPTION: + x = TApplicationException() + x.read(iprot) + iprot.readMessageEnd() + raise x + result = clear_file_metadata_result() + result.read(iprot) + iprot.readMessageEnd() + if result.success is not None: + return result.success + raise TApplicationException(TApplicationException.MISSING_RESULT, "clear_file_metadata failed: unknown result") + + def cache_file_metadata(self, req): + """ + Parameters: + - req + + """ + self.send_cache_file_metadata(req) + return self.recv_cache_file_metadata() + + def send_cache_file_metadata(self, req): + self._oprot.writeMessageBegin("cache_file_metadata", TMessageType.CALL, self._seqid) + args = cache_file_metadata_args() + args.req = req + args.write(self._oprot) + self._oprot.writeMessageEnd() + self._oprot.trans.flush() + + def recv_cache_file_metadata(self): + iprot = self._iprot + (fname, mtype, rseqid) = iprot.readMessageBegin() + if mtype == TMessageType.EXCEPTION: + x = TApplicationException() + x.read(iprot) + iprot.readMessageEnd() + raise x + result = cache_file_metadata_result() + result.read(iprot) + iprot.readMessageEnd() + if result.success is not None: + return result.success + raise TApplicationException(TApplicationException.MISSING_RESULT, "cache_file_metadata failed: unknown result") + + def get_metastore_db_uuid(self): + self.send_get_metastore_db_uuid() + return self.recv_get_metastore_db_uuid() + + def send_get_metastore_db_uuid(self): + self._oprot.writeMessageBegin("get_metastore_db_uuid", TMessageType.CALL, self._seqid) + args = get_metastore_db_uuid_args() + args.write(self._oprot) + self._oprot.writeMessageEnd() + self._oprot.trans.flush() + + def recv_get_metastore_db_uuid(self): + iprot = self._iprot + (fname, mtype, rseqid) = iprot.readMessageBegin() + if mtype == TMessageType.EXCEPTION: + x = TApplicationException() + x.read(iprot) + iprot.readMessageEnd() + raise x + result = get_metastore_db_uuid_result() + result.read(iprot) + iprot.readMessageEnd() + if result.success is not None: + return result.success + if result.o1 is not None: + raise result.o1 + raise TApplicationException(TApplicationException.MISSING_RESULT, "get_metastore_db_uuid failed: unknown result") + + def create_resource_plan(self, request): + """ + Parameters: + - request + + """ + self.send_create_resource_plan(request) + return self.recv_create_resource_plan() + + def send_create_resource_plan(self, request): + self._oprot.writeMessageBegin("create_resource_plan", TMessageType.CALL, self._seqid) + args = create_resource_plan_args() + args.request = request + args.write(self._oprot) + self._oprot.writeMessageEnd() + self._oprot.trans.flush() + + def recv_create_resource_plan(self): + iprot = self._iprot + (fname, mtype, rseqid) = iprot.readMessageBegin() + if mtype == TMessageType.EXCEPTION: + x = TApplicationException() + x.read(iprot) + iprot.readMessageEnd() + raise x + result = create_resource_plan_result() + result.read(iprot) + iprot.readMessageEnd() + if result.success is not None: + return result.success + if result.o1 is not None: + raise result.o1 + if result.o2 is not None: + raise result.o2 + if result.o3 is not None: + raise result.o3 + raise TApplicationException(TApplicationException.MISSING_RESULT, "create_resource_plan failed: unknown result") + + def get_resource_plan(self, request): + """ + Parameters: + - request + + """ + self.send_get_resource_plan(request) + return self.recv_get_resource_plan() + + def send_get_resource_plan(self, request): + self._oprot.writeMessageBegin("get_resource_plan", TMessageType.CALL, self._seqid) + args = get_resource_plan_args() + args.request = request + args.write(self._oprot) + self._oprot.writeMessageEnd() + self._oprot.trans.flush() + + def recv_get_resource_plan(self): + iprot = self._iprot + (fname, mtype, rseqid) = iprot.readMessageBegin() + if mtype == TMessageType.EXCEPTION: + x = TApplicationException() + x.read(iprot) + iprot.readMessageEnd() + raise x + result = get_resource_plan_result() + result.read(iprot) + iprot.readMessageEnd() + if result.success is not None: + return result.success + if result.o1 is not None: + raise result.o1 + if result.o2 is not None: + raise result.o2 + raise TApplicationException(TApplicationException.MISSING_RESULT, "get_resource_plan failed: unknown result") + + def get_active_resource_plan(self, request): + """ + Parameters: + - request + + """ + self.send_get_active_resource_plan(request) + return self.recv_get_active_resource_plan() + + def send_get_active_resource_plan(self, request): + self._oprot.writeMessageBegin("get_active_resource_plan", TMessageType.CALL, self._seqid) + args = get_active_resource_plan_args() + args.request = request + args.write(self._oprot) + self._oprot.writeMessageEnd() + self._oprot.trans.flush() + + def recv_get_active_resource_plan(self): + iprot = self._iprot + (fname, mtype, rseqid) = iprot.readMessageBegin() + if mtype == TMessageType.EXCEPTION: + x = TApplicationException() + x.read(iprot) + iprot.readMessageEnd() + raise x + result = get_active_resource_plan_result() + result.read(iprot) + iprot.readMessageEnd() + if result.success is not None: + return result.success + if result.o2 is not None: + raise result.o2 + raise TApplicationException(TApplicationException.MISSING_RESULT, "get_active_resource_plan failed: unknown result") + + def get_all_resource_plans(self, request): + """ + Parameters: + - request + + """ + self.send_get_all_resource_plans(request) + return self.recv_get_all_resource_plans() + + def send_get_all_resource_plans(self, request): + self._oprot.writeMessageBegin("get_all_resource_plans", TMessageType.CALL, self._seqid) + args = get_all_resource_plans_args() + args.request = request + args.write(self._oprot) + self._oprot.writeMessageEnd() + self._oprot.trans.flush() + + def recv_get_all_resource_plans(self): + iprot = self._iprot + (fname, mtype, rseqid) = iprot.readMessageBegin() + if mtype == TMessageType.EXCEPTION: + x = TApplicationException() + x.read(iprot) + iprot.readMessageEnd() + raise x + result = get_all_resource_plans_result() + result.read(iprot) + iprot.readMessageEnd() + if result.success is not None: + return result.success + if result.o1 is not None: + raise result.o1 + raise TApplicationException(TApplicationException.MISSING_RESULT, "get_all_resource_plans failed: unknown result") + + def alter_resource_plan(self, request): + """ + Parameters: + - request + + """ + self.send_alter_resource_plan(request) + return self.recv_alter_resource_plan() + + def send_alter_resource_plan(self, request): + self._oprot.writeMessageBegin("alter_resource_plan", TMessageType.CALL, self._seqid) + args = alter_resource_plan_args() + args.request = request + args.write(self._oprot) + self._oprot.writeMessageEnd() + self._oprot.trans.flush() + + def recv_alter_resource_plan(self): + iprot = self._iprot + (fname, mtype, rseqid) = iprot.readMessageBegin() + if mtype == TMessageType.EXCEPTION: + x = TApplicationException() + x.read(iprot) + iprot.readMessageEnd() + raise x + result = alter_resource_plan_result() + result.read(iprot) + iprot.readMessageEnd() + if result.success is not None: + return result.success + if result.o1 is not None: + raise result.o1 + if result.o2 is not None: + raise result.o2 + if result.o3 is not None: + raise result.o3 + raise TApplicationException(TApplicationException.MISSING_RESULT, "alter_resource_plan failed: unknown result") + + def validate_resource_plan(self, request): + """ + Parameters: + - request + + """ + self.send_validate_resource_plan(request) + return self.recv_validate_resource_plan() + + def send_validate_resource_plan(self, request): + self._oprot.writeMessageBegin("validate_resource_plan", TMessageType.CALL, self._seqid) + args = validate_resource_plan_args() + args.request = request + args.write(self._oprot) + self._oprot.writeMessageEnd() + self._oprot.trans.flush() + + def recv_validate_resource_plan(self): + iprot = self._iprot + (fname, mtype, rseqid) = iprot.readMessageBegin() + if mtype == TMessageType.EXCEPTION: + x = TApplicationException() + x.read(iprot) + iprot.readMessageEnd() + raise x + result = validate_resource_plan_result() + result.read(iprot) + iprot.readMessageEnd() + if result.success is not None: + return result.success + if result.o1 is not None: + raise result.o1 + if result.o2 is not None: + raise result.o2 + raise TApplicationException(TApplicationException.MISSING_RESULT, "validate_resource_plan failed: unknown result") + + def drop_resource_plan(self, request): + """ + Parameters: + - request + + """ + self.send_drop_resource_plan(request) + return self.recv_drop_resource_plan() + + def send_drop_resource_plan(self, request): + self._oprot.writeMessageBegin("drop_resource_plan", TMessageType.CALL, self._seqid) + args = drop_resource_plan_args() + args.request = request + args.write(self._oprot) + self._oprot.writeMessageEnd() + self._oprot.trans.flush() + + def recv_drop_resource_plan(self): + iprot = self._iprot + (fname, mtype, rseqid) = iprot.readMessageBegin() + if mtype == TMessageType.EXCEPTION: + x = TApplicationException() + x.read(iprot) + iprot.readMessageEnd() + raise x + result = drop_resource_plan_result() + result.read(iprot) + iprot.readMessageEnd() + if result.success is not None: + return result.success + if result.o1 is not None: + raise result.o1 + if result.o2 is not None: + raise result.o2 + if result.o3 is not None: + raise result.o3 + raise TApplicationException(TApplicationException.MISSING_RESULT, "drop_resource_plan failed: unknown result") + + def create_wm_trigger(self, request): + """ + Parameters: + - request + + """ + self.send_create_wm_trigger(request) + return self.recv_create_wm_trigger() + + def send_create_wm_trigger(self, request): + self._oprot.writeMessageBegin("create_wm_trigger", TMessageType.CALL, self._seqid) + args = create_wm_trigger_args() + args.request = request + args.write(self._oprot) + self._oprot.writeMessageEnd() + self._oprot.trans.flush() + + def recv_create_wm_trigger(self): + iprot = self._iprot + (fname, mtype, rseqid) = iprot.readMessageBegin() + if mtype == TMessageType.EXCEPTION: + x = TApplicationException() + x.read(iprot) + iprot.readMessageEnd() + raise x + result = create_wm_trigger_result() + result.read(iprot) + iprot.readMessageEnd() + if result.success is not None: + return result.success + if result.o1 is not None: + raise result.o1 + if result.o2 is not None: + raise result.o2 + if result.o3 is not None: + raise result.o3 + if result.o4 is not None: + raise result.o4 + raise TApplicationException(TApplicationException.MISSING_RESULT, "create_wm_trigger failed: unknown result") + + def alter_wm_trigger(self, request): + """ + Parameters: + - request + + """ + self.send_alter_wm_trigger(request) + return self.recv_alter_wm_trigger() + + def send_alter_wm_trigger(self, request): + self._oprot.writeMessageBegin("alter_wm_trigger", TMessageType.CALL, self._seqid) + args = alter_wm_trigger_args() + args.request = request + args.write(self._oprot) + self._oprot.writeMessageEnd() + self._oprot.trans.flush() + + def recv_alter_wm_trigger(self): + iprot = self._iprot + (fname, mtype, rseqid) = iprot.readMessageBegin() + if mtype == TMessageType.EXCEPTION: + x = TApplicationException() + x.read(iprot) + iprot.readMessageEnd() + raise x + result = alter_wm_trigger_result() + result.read(iprot) + iprot.readMessageEnd() + if result.success is not None: + return result.success + if result.o1 is not None: + raise result.o1 + if result.o2 is not None: + raise result.o2 + if result.o3 is not None: + raise result.o3 + raise TApplicationException(TApplicationException.MISSING_RESULT, "alter_wm_trigger failed: unknown result") + + def drop_wm_trigger(self, request): + """ + Parameters: + - request + + """ + self.send_drop_wm_trigger(request) + return self.recv_drop_wm_trigger() + + def send_drop_wm_trigger(self, request): + self._oprot.writeMessageBegin("drop_wm_trigger", TMessageType.CALL, self._seqid) + args = drop_wm_trigger_args() + args.request = request + args.write(self._oprot) + self._oprot.writeMessageEnd() + self._oprot.trans.flush() + + def recv_drop_wm_trigger(self): + iprot = self._iprot + (fname, mtype, rseqid) = iprot.readMessageBegin() + if mtype == TMessageType.EXCEPTION: + x = TApplicationException() + x.read(iprot) + iprot.readMessageEnd() + raise x + result = drop_wm_trigger_result() + result.read(iprot) + iprot.readMessageEnd() + if result.success is not None: + return result.success + if result.o1 is not None: + raise result.o1 + if result.o2 is not None: + raise result.o2 + if result.o3 is not None: + raise result.o3 + raise TApplicationException(TApplicationException.MISSING_RESULT, "drop_wm_trigger failed: unknown result") + + def get_triggers_for_resourceplan(self, request): + """ + Parameters: + - request + + """ + self.send_get_triggers_for_resourceplan(request) + return self.recv_get_triggers_for_resourceplan() + + def send_get_triggers_for_resourceplan(self, request): + self._oprot.writeMessageBegin("get_triggers_for_resourceplan", TMessageType.CALL, self._seqid) + args = get_triggers_for_resourceplan_args() + args.request = request + args.write(self._oprot) + self._oprot.writeMessageEnd() + self._oprot.trans.flush() + + def recv_get_triggers_for_resourceplan(self): + iprot = self._iprot + (fname, mtype, rseqid) = iprot.readMessageBegin() + if mtype == TMessageType.EXCEPTION: + x = TApplicationException() + x.read(iprot) + iprot.readMessageEnd() + raise x + result = get_triggers_for_resourceplan_result() + result.read(iprot) + iprot.readMessageEnd() + if result.success is not None: + return result.success + if result.o1 is not None: + raise result.o1 + if result.o2 is not None: + raise result.o2 + raise TApplicationException(TApplicationException.MISSING_RESULT, "get_triggers_for_resourceplan failed: unknown result") + + def create_wm_pool(self, request): + """ + Parameters: + - request + + """ + self.send_create_wm_pool(request) + return self.recv_create_wm_pool() + + def send_create_wm_pool(self, request): + self._oprot.writeMessageBegin("create_wm_pool", TMessageType.CALL, self._seqid) + args = create_wm_pool_args() + args.request = request + args.write(self._oprot) + self._oprot.writeMessageEnd() + self._oprot.trans.flush() + + def recv_create_wm_pool(self): + iprot = self._iprot + (fname, mtype, rseqid) = iprot.readMessageBegin() + if mtype == TMessageType.EXCEPTION: + x = TApplicationException() + x.read(iprot) + iprot.readMessageEnd() + raise x + result = create_wm_pool_result() + result.read(iprot) + iprot.readMessageEnd() + if result.success is not None: + return result.success + if result.o1 is not None: + raise result.o1 + if result.o2 is not None: + raise result.o2 + if result.o3 is not None: + raise result.o3 + if result.o4 is not None: + raise result.o4 + raise TApplicationException(TApplicationException.MISSING_RESULT, "create_wm_pool failed: unknown result") + + def alter_wm_pool(self, request): + """ + Parameters: + - request + + """ + self.send_alter_wm_pool(request) + return self.recv_alter_wm_pool() + + def send_alter_wm_pool(self, request): + self._oprot.writeMessageBegin("alter_wm_pool", TMessageType.CALL, self._seqid) + args = alter_wm_pool_args() + args.request = request + args.write(self._oprot) + self._oprot.writeMessageEnd() + self._oprot.trans.flush() + + def recv_alter_wm_pool(self): + iprot = self._iprot + (fname, mtype, rseqid) = iprot.readMessageBegin() + if mtype == TMessageType.EXCEPTION: + x = TApplicationException() + x.read(iprot) + iprot.readMessageEnd() + raise x + result = alter_wm_pool_result() + result.read(iprot) + iprot.readMessageEnd() + if result.success is not None: + return result.success + if result.o1 is not None: + raise result.o1 + if result.o2 is not None: + raise result.o2 + if result.o3 is not None: + raise result.o3 + if result.o4 is not None: + raise result.o4 + raise TApplicationException(TApplicationException.MISSING_RESULT, "alter_wm_pool failed: unknown result") + + def drop_wm_pool(self, request): + """ + Parameters: + - request + + """ + self.send_drop_wm_pool(request) + return self.recv_drop_wm_pool() + + def send_drop_wm_pool(self, request): + self._oprot.writeMessageBegin("drop_wm_pool", TMessageType.CALL, self._seqid) + args = drop_wm_pool_args() + args.request = request + args.write(self._oprot) + self._oprot.writeMessageEnd() + self._oprot.trans.flush() + + def recv_drop_wm_pool(self): + iprot = self._iprot + (fname, mtype, rseqid) = iprot.readMessageBegin() + if mtype == TMessageType.EXCEPTION: + x = TApplicationException() + x.read(iprot) + iprot.readMessageEnd() + raise x + result = drop_wm_pool_result() + result.read(iprot) + iprot.readMessageEnd() + if result.success is not None: + return result.success + if result.o1 is not None: + raise result.o1 + if result.o2 is not None: + raise result.o2 + if result.o3 is not None: + raise result.o3 + raise TApplicationException(TApplicationException.MISSING_RESULT, "drop_wm_pool failed: unknown result") + + def create_or_update_wm_mapping(self, request): + """ + Parameters: + - request + + """ + self.send_create_or_update_wm_mapping(request) + return self.recv_create_or_update_wm_mapping() + + def send_create_or_update_wm_mapping(self, request): + self._oprot.writeMessageBegin("create_or_update_wm_mapping", TMessageType.CALL, self._seqid) + args = create_or_update_wm_mapping_args() + args.request = request + args.write(self._oprot) + self._oprot.writeMessageEnd() + self._oprot.trans.flush() + + def recv_create_or_update_wm_mapping(self): + iprot = self._iprot + (fname, mtype, rseqid) = iprot.readMessageBegin() + if mtype == TMessageType.EXCEPTION: + x = TApplicationException() + x.read(iprot) + iprot.readMessageEnd() + raise x + result = create_or_update_wm_mapping_result() + result.read(iprot) + iprot.readMessageEnd() + if result.success is not None: + return result.success + if result.o1 is not None: + raise result.o1 + if result.o2 is not None: + raise result.o2 + if result.o3 is not None: + raise result.o3 + if result.o4 is not None: + raise result.o4 + raise TApplicationException(TApplicationException.MISSING_RESULT, "create_or_update_wm_mapping failed: unknown result") + + def drop_wm_mapping(self, request): + """ + Parameters: + - request + + """ + self.send_drop_wm_mapping(request) + return self.recv_drop_wm_mapping() + + def send_drop_wm_mapping(self, request): + self._oprot.writeMessageBegin("drop_wm_mapping", TMessageType.CALL, self._seqid) + args = drop_wm_mapping_args() + args.request = request + args.write(self._oprot) + self._oprot.writeMessageEnd() + self._oprot.trans.flush() + + def recv_drop_wm_mapping(self): + iprot = self._iprot + (fname, mtype, rseqid) = iprot.readMessageBegin() + if mtype == TMessageType.EXCEPTION: + x = TApplicationException() + x.read(iprot) + iprot.readMessageEnd() + raise x + result = drop_wm_mapping_result() + result.read(iprot) + iprot.readMessageEnd() + if result.success is not None: + return result.success + if result.o1 is not None: + raise result.o1 + if result.o2 is not None: + raise result.o2 + if result.o3 is not None: + raise result.o3 + raise TApplicationException(TApplicationException.MISSING_RESULT, "drop_wm_mapping failed: unknown result") + + def create_or_drop_wm_trigger_to_pool_mapping(self, request): + """ + Parameters: + - request + + """ + self.send_create_or_drop_wm_trigger_to_pool_mapping(request) + return self.recv_create_or_drop_wm_trigger_to_pool_mapping() + + def send_create_or_drop_wm_trigger_to_pool_mapping(self, request): + self._oprot.writeMessageBegin("create_or_drop_wm_trigger_to_pool_mapping", TMessageType.CALL, self._seqid) + args = create_or_drop_wm_trigger_to_pool_mapping_args() + args.request = request + args.write(self._oprot) + self._oprot.writeMessageEnd() + self._oprot.trans.flush() + + def recv_create_or_drop_wm_trigger_to_pool_mapping(self): + iprot = self._iprot + (fname, mtype, rseqid) = iprot.readMessageBegin() + if mtype == TMessageType.EXCEPTION: + x = TApplicationException() + x.read(iprot) + iprot.readMessageEnd() + raise x + result = create_or_drop_wm_trigger_to_pool_mapping_result() + result.read(iprot) + iprot.readMessageEnd() + if result.success is not None: + return result.success + if result.o1 is not None: + raise result.o1 + if result.o2 is not None: + raise result.o2 + if result.o3 is not None: + raise result.o3 + if result.o4 is not None: + raise result.o4 + raise TApplicationException( + TApplicationException.MISSING_RESULT, "create_or_drop_wm_trigger_to_pool_mapping failed: unknown result" + ) + + def create_ischema(self, schema): + """ + Parameters: + - schema + + """ + self.send_create_ischema(schema) + self.recv_create_ischema() + + def send_create_ischema(self, schema): + self._oprot.writeMessageBegin("create_ischema", TMessageType.CALL, self._seqid) + args = create_ischema_args() + args.schema = schema + args.write(self._oprot) + self._oprot.writeMessageEnd() + self._oprot.trans.flush() + + def recv_create_ischema(self): + iprot = self._iprot + (fname, mtype, rseqid) = iprot.readMessageBegin() + if mtype == TMessageType.EXCEPTION: + x = TApplicationException() + x.read(iprot) + iprot.readMessageEnd() + raise x + result = create_ischema_result() + result.read(iprot) + iprot.readMessageEnd() + if result.o1 is not None: + raise result.o1 + if result.o2 is not None: + raise result.o2 + if result.o3 is not None: + raise result.o3 + return + + def alter_ischema(self, rqst): + """ + Parameters: + - rqst + + """ + self.send_alter_ischema(rqst) + self.recv_alter_ischema() + + def send_alter_ischema(self, rqst): + self._oprot.writeMessageBegin("alter_ischema", TMessageType.CALL, self._seqid) + args = alter_ischema_args() + args.rqst = rqst + args.write(self._oprot) + self._oprot.writeMessageEnd() + self._oprot.trans.flush() + + def recv_alter_ischema(self): + iprot = self._iprot + (fname, mtype, rseqid) = iprot.readMessageBegin() + if mtype == TMessageType.EXCEPTION: + x = TApplicationException() + x.read(iprot) + iprot.readMessageEnd() + raise x + result = alter_ischema_result() + result.read(iprot) + iprot.readMessageEnd() + if result.o1 is not None: + raise result.o1 + if result.o2 is not None: + raise result.o2 + return + + def get_ischema(self, name): + """ + Parameters: + - name + + """ + self.send_get_ischema(name) + return self.recv_get_ischema() + + def send_get_ischema(self, name): + self._oprot.writeMessageBegin("get_ischema", TMessageType.CALL, self._seqid) + args = get_ischema_args() + args.name = name + args.write(self._oprot) + self._oprot.writeMessageEnd() + self._oprot.trans.flush() + + def recv_get_ischema(self): + iprot = self._iprot + (fname, mtype, rseqid) = iprot.readMessageBegin() + if mtype == TMessageType.EXCEPTION: + x = TApplicationException() + x.read(iprot) + iprot.readMessageEnd() + raise x + result = get_ischema_result() + result.read(iprot) + iprot.readMessageEnd() + if result.success is not None: + return result.success + if result.o1 is not None: + raise result.o1 + if result.o2 is not None: + raise result.o2 + raise TApplicationException(TApplicationException.MISSING_RESULT, "get_ischema failed: unknown result") + + def drop_ischema(self, name): + """ + Parameters: + - name + + """ + self.send_drop_ischema(name) + self.recv_drop_ischema() + + def send_drop_ischema(self, name): + self._oprot.writeMessageBegin("drop_ischema", TMessageType.CALL, self._seqid) + args = drop_ischema_args() + args.name = name + args.write(self._oprot) + self._oprot.writeMessageEnd() + self._oprot.trans.flush() + + def recv_drop_ischema(self): + iprot = self._iprot + (fname, mtype, rseqid) = iprot.readMessageBegin() + if mtype == TMessageType.EXCEPTION: + x = TApplicationException() + x.read(iprot) + iprot.readMessageEnd() + raise x + result = drop_ischema_result() + result.read(iprot) + iprot.readMessageEnd() + if result.o1 is not None: + raise result.o1 + if result.o2 is not None: + raise result.o2 + if result.o3 is not None: + raise result.o3 + return + + def add_schema_version(self, schemaVersion): + """ + Parameters: + - schemaVersion + + """ + self.send_add_schema_version(schemaVersion) + self.recv_add_schema_version() + + def send_add_schema_version(self, schemaVersion): + self._oprot.writeMessageBegin("add_schema_version", TMessageType.CALL, self._seqid) + args = add_schema_version_args() + args.schemaVersion = schemaVersion + args.write(self._oprot) + self._oprot.writeMessageEnd() + self._oprot.trans.flush() + + def recv_add_schema_version(self): + iprot = self._iprot + (fname, mtype, rseqid) = iprot.readMessageBegin() + if mtype == TMessageType.EXCEPTION: + x = TApplicationException() + x.read(iprot) + iprot.readMessageEnd() + raise x + result = add_schema_version_result() + result.read(iprot) + iprot.readMessageEnd() + if result.o1 is not None: + raise result.o1 + if result.o2 is not None: + raise result.o2 + if result.o3 is not None: + raise result.o3 + return + + def get_schema_version(self, schemaVersion): + """ + Parameters: + - schemaVersion + + """ + self.send_get_schema_version(schemaVersion) + return self.recv_get_schema_version() + + def send_get_schema_version(self, schemaVersion): + self._oprot.writeMessageBegin("get_schema_version", TMessageType.CALL, self._seqid) + args = get_schema_version_args() + args.schemaVersion = schemaVersion + args.write(self._oprot) + self._oprot.writeMessageEnd() + self._oprot.trans.flush() + + def recv_get_schema_version(self): + iprot = self._iprot + (fname, mtype, rseqid) = iprot.readMessageBegin() + if mtype == TMessageType.EXCEPTION: + x = TApplicationException() + x.read(iprot) + iprot.readMessageEnd() + raise x + result = get_schema_version_result() + result.read(iprot) + iprot.readMessageEnd() + if result.success is not None: + return result.success + if result.o1 is not None: + raise result.o1 + if result.o2 is not None: + raise result.o2 + raise TApplicationException(TApplicationException.MISSING_RESULT, "get_schema_version failed: unknown result") + + def get_schema_latest_version(self, schemaName): + """ + Parameters: + - schemaName + + """ + self.send_get_schema_latest_version(schemaName) + return self.recv_get_schema_latest_version() + + def send_get_schema_latest_version(self, schemaName): + self._oprot.writeMessageBegin("get_schema_latest_version", TMessageType.CALL, self._seqid) + args = get_schema_latest_version_args() + args.schemaName = schemaName + args.write(self._oprot) + self._oprot.writeMessageEnd() + self._oprot.trans.flush() + + def recv_get_schema_latest_version(self): + iprot = self._iprot + (fname, mtype, rseqid) = iprot.readMessageBegin() + if mtype == TMessageType.EXCEPTION: + x = TApplicationException() + x.read(iprot) + iprot.readMessageEnd() + raise x + result = get_schema_latest_version_result() + result.read(iprot) + iprot.readMessageEnd() + if result.success is not None: + return result.success + if result.o1 is not None: + raise result.o1 + if result.o2 is not None: + raise result.o2 + raise TApplicationException(TApplicationException.MISSING_RESULT, "get_schema_latest_version failed: unknown result") + + def get_schema_all_versions(self, schemaName): + """ + Parameters: + - schemaName + + """ + self.send_get_schema_all_versions(schemaName) + return self.recv_get_schema_all_versions() + + def send_get_schema_all_versions(self, schemaName): + self._oprot.writeMessageBegin("get_schema_all_versions", TMessageType.CALL, self._seqid) + args = get_schema_all_versions_args() + args.schemaName = schemaName + args.write(self._oprot) + self._oprot.writeMessageEnd() + self._oprot.trans.flush() + + def recv_get_schema_all_versions(self): + iprot = self._iprot + (fname, mtype, rseqid) = iprot.readMessageBegin() + if mtype == TMessageType.EXCEPTION: + x = TApplicationException() + x.read(iprot) + iprot.readMessageEnd() + raise x + result = get_schema_all_versions_result() + result.read(iprot) + iprot.readMessageEnd() + if result.success is not None: + return result.success + if result.o1 is not None: + raise result.o1 + if result.o2 is not None: + raise result.o2 + raise TApplicationException(TApplicationException.MISSING_RESULT, "get_schema_all_versions failed: unknown result") + + def drop_schema_version(self, schemaVersion): + """ + Parameters: + - schemaVersion + + """ + self.send_drop_schema_version(schemaVersion) + self.recv_drop_schema_version() + + def send_drop_schema_version(self, schemaVersion): + self._oprot.writeMessageBegin("drop_schema_version", TMessageType.CALL, self._seqid) + args = drop_schema_version_args() + args.schemaVersion = schemaVersion + args.write(self._oprot) + self._oprot.writeMessageEnd() + self._oprot.trans.flush() + + def recv_drop_schema_version(self): + iprot = self._iprot + (fname, mtype, rseqid) = iprot.readMessageBegin() + if mtype == TMessageType.EXCEPTION: + x = TApplicationException() + x.read(iprot) + iprot.readMessageEnd() + raise x + result = drop_schema_version_result() + result.read(iprot) + iprot.readMessageEnd() + if result.o1 is not None: + raise result.o1 + if result.o2 is not None: + raise result.o2 + return + + def get_schemas_by_cols(self, rqst): + """ + Parameters: + - rqst + + """ + self.send_get_schemas_by_cols(rqst) + return self.recv_get_schemas_by_cols() + + def send_get_schemas_by_cols(self, rqst): + self._oprot.writeMessageBegin("get_schemas_by_cols", TMessageType.CALL, self._seqid) + args = get_schemas_by_cols_args() + args.rqst = rqst + args.write(self._oprot) + self._oprot.writeMessageEnd() + self._oprot.trans.flush() + + def recv_get_schemas_by_cols(self): + iprot = self._iprot + (fname, mtype, rseqid) = iprot.readMessageBegin() + if mtype == TMessageType.EXCEPTION: + x = TApplicationException() + x.read(iprot) + iprot.readMessageEnd() + raise x + result = get_schemas_by_cols_result() + result.read(iprot) + iprot.readMessageEnd() + if result.success is not None: + return result.success + if result.o1 is not None: + raise result.o1 + raise TApplicationException(TApplicationException.MISSING_RESULT, "get_schemas_by_cols failed: unknown result") + + def map_schema_version_to_serde(self, rqst): + """ + Parameters: + - rqst + + """ + self.send_map_schema_version_to_serde(rqst) + self.recv_map_schema_version_to_serde() + + def send_map_schema_version_to_serde(self, rqst): + self._oprot.writeMessageBegin("map_schema_version_to_serde", TMessageType.CALL, self._seqid) + args = map_schema_version_to_serde_args() + args.rqst = rqst + args.write(self._oprot) + self._oprot.writeMessageEnd() + self._oprot.trans.flush() + + def recv_map_schema_version_to_serde(self): + iprot = self._iprot + (fname, mtype, rseqid) = iprot.readMessageBegin() + if mtype == TMessageType.EXCEPTION: + x = TApplicationException() + x.read(iprot) + iprot.readMessageEnd() + raise x + result = map_schema_version_to_serde_result() + result.read(iprot) + iprot.readMessageEnd() + if result.o1 is not None: + raise result.o1 + if result.o2 is not None: + raise result.o2 + return + + def set_schema_version_state(self, rqst): + """ + Parameters: + - rqst + + """ + self.send_set_schema_version_state(rqst) + self.recv_set_schema_version_state() + + def send_set_schema_version_state(self, rqst): + self._oprot.writeMessageBegin("set_schema_version_state", TMessageType.CALL, self._seqid) + args = set_schema_version_state_args() + args.rqst = rqst + args.write(self._oprot) + self._oprot.writeMessageEnd() + self._oprot.trans.flush() + + def recv_set_schema_version_state(self): + iprot = self._iprot + (fname, mtype, rseqid) = iprot.readMessageBegin() + if mtype == TMessageType.EXCEPTION: + x = TApplicationException() + x.read(iprot) + iprot.readMessageEnd() + raise x + result = set_schema_version_state_result() + result.read(iprot) + iprot.readMessageEnd() + if result.o1 is not None: + raise result.o1 + if result.o2 is not None: + raise result.o2 + if result.o3 is not None: + raise result.o3 + return + + def add_serde(self, serde): + """ + Parameters: + - serde + + """ + self.send_add_serde(serde) + self.recv_add_serde() + + def send_add_serde(self, serde): + self._oprot.writeMessageBegin("add_serde", TMessageType.CALL, self._seqid) + args = add_serde_args() + args.serde = serde + args.write(self._oprot) + self._oprot.writeMessageEnd() + self._oprot.trans.flush() + + def recv_add_serde(self): + iprot = self._iprot + (fname, mtype, rseqid) = iprot.readMessageBegin() + if mtype == TMessageType.EXCEPTION: + x = TApplicationException() + x.read(iprot) + iprot.readMessageEnd() + raise x + result = add_serde_result() + result.read(iprot) + iprot.readMessageEnd() + if result.o1 is not None: + raise result.o1 + if result.o2 is not None: + raise result.o2 + return + + def get_serde(self, rqst): + """ + Parameters: + - rqst + + """ + self.send_get_serde(rqst) + return self.recv_get_serde() + + def send_get_serde(self, rqst): + self._oprot.writeMessageBegin("get_serde", TMessageType.CALL, self._seqid) + args = get_serde_args() + args.rqst = rqst + args.write(self._oprot) + self._oprot.writeMessageEnd() + self._oprot.trans.flush() + + def recv_get_serde(self): + iprot = self._iprot + (fname, mtype, rseqid) = iprot.readMessageBegin() + if mtype == TMessageType.EXCEPTION: + x = TApplicationException() + x.read(iprot) + iprot.readMessageEnd() + raise x + result = get_serde_result() + result.read(iprot) + iprot.readMessageEnd() + if result.success is not None: + return result.success + if result.o1 is not None: + raise result.o1 + if result.o2 is not None: + raise result.o2 + raise TApplicationException(TApplicationException.MISSING_RESULT, "get_serde failed: unknown result") + + def get_lock_materialization_rebuild(self, dbName, tableName, txnId): + """ + Parameters: + - dbName + - tableName + - txnId + + """ + self.send_get_lock_materialization_rebuild(dbName, tableName, txnId) + return self.recv_get_lock_materialization_rebuild() + + def send_get_lock_materialization_rebuild(self, dbName, tableName, txnId): + self._oprot.writeMessageBegin("get_lock_materialization_rebuild", TMessageType.CALL, self._seqid) + args = get_lock_materialization_rebuild_args() + args.dbName = dbName + args.tableName = tableName + args.txnId = txnId + args.write(self._oprot) + self._oprot.writeMessageEnd() + self._oprot.trans.flush() + + def recv_get_lock_materialization_rebuild(self): + iprot = self._iprot + (fname, mtype, rseqid) = iprot.readMessageBegin() + if mtype == TMessageType.EXCEPTION: + x = TApplicationException() + x.read(iprot) + iprot.readMessageEnd() + raise x + result = get_lock_materialization_rebuild_result() + result.read(iprot) + iprot.readMessageEnd() + if result.success is not None: + return result.success + raise TApplicationException( + TApplicationException.MISSING_RESULT, "get_lock_materialization_rebuild failed: unknown result" + ) + + def heartbeat_lock_materialization_rebuild(self, dbName, tableName, txnId): + """ + Parameters: + - dbName + - tableName + - txnId + + """ + self.send_heartbeat_lock_materialization_rebuild(dbName, tableName, txnId) + return self.recv_heartbeat_lock_materialization_rebuild() + + def send_heartbeat_lock_materialization_rebuild(self, dbName, tableName, txnId): + self._oprot.writeMessageBegin("heartbeat_lock_materialization_rebuild", TMessageType.CALL, self._seqid) + args = heartbeat_lock_materialization_rebuild_args() + args.dbName = dbName + args.tableName = tableName + args.txnId = txnId + args.write(self._oprot) + self._oprot.writeMessageEnd() + self._oprot.trans.flush() + + def recv_heartbeat_lock_materialization_rebuild(self): + iprot = self._iprot + (fname, mtype, rseqid) = iprot.readMessageBegin() + if mtype == TMessageType.EXCEPTION: + x = TApplicationException() + x.read(iprot) + iprot.readMessageEnd() + raise x + result = heartbeat_lock_materialization_rebuild_result() + result.read(iprot) + iprot.readMessageEnd() + if result.success is not None: + return result.success + raise TApplicationException( + TApplicationException.MISSING_RESULT, "heartbeat_lock_materialization_rebuild failed: unknown result" + ) + + def add_runtime_stats(self, stat): + """ + Parameters: + - stat + + """ + self.send_add_runtime_stats(stat) + self.recv_add_runtime_stats() + + def send_add_runtime_stats(self, stat): + self._oprot.writeMessageBegin("add_runtime_stats", TMessageType.CALL, self._seqid) + args = add_runtime_stats_args() + args.stat = stat + args.write(self._oprot) + self._oprot.writeMessageEnd() + self._oprot.trans.flush() + + def recv_add_runtime_stats(self): + iprot = self._iprot + (fname, mtype, rseqid) = iprot.readMessageBegin() + if mtype == TMessageType.EXCEPTION: + x = TApplicationException() + x.read(iprot) + iprot.readMessageEnd() + raise x + result = add_runtime_stats_result() + result.read(iprot) + iprot.readMessageEnd() + if result.o1 is not None: + raise result.o1 + return + + def get_runtime_stats(self, rqst): + """ + Parameters: + - rqst + + """ + self.send_get_runtime_stats(rqst) + return self.recv_get_runtime_stats() + + def send_get_runtime_stats(self, rqst): + self._oprot.writeMessageBegin("get_runtime_stats", TMessageType.CALL, self._seqid) + args = get_runtime_stats_args() + args.rqst = rqst + args.write(self._oprot) + self._oprot.writeMessageEnd() + self._oprot.trans.flush() + + def recv_get_runtime_stats(self): + iprot = self._iprot + (fname, mtype, rseqid) = iprot.readMessageBegin() + if mtype == TMessageType.EXCEPTION: + x = TApplicationException() + x.read(iprot) + iprot.readMessageEnd() + raise x + result = get_runtime_stats_result() + result.read(iprot) + iprot.readMessageEnd() + if result.success is not None: + return result.success + if result.o1 is not None: + raise result.o1 + raise TApplicationException(TApplicationException.MISSING_RESULT, "get_runtime_stats failed: unknown result") + + def get_partitions_with_specs(self, request): + """ + Parameters: + - request + + """ + self.send_get_partitions_with_specs(request) + return self.recv_get_partitions_with_specs() + + def send_get_partitions_with_specs(self, request): + self._oprot.writeMessageBegin("get_partitions_with_specs", TMessageType.CALL, self._seqid) + args = get_partitions_with_specs_args() + args.request = request + args.write(self._oprot) + self._oprot.writeMessageEnd() + self._oprot.trans.flush() + + def recv_get_partitions_with_specs(self): + iprot = self._iprot + (fname, mtype, rseqid) = iprot.readMessageBegin() + if mtype == TMessageType.EXCEPTION: + x = TApplicationException() + x.read(iprot) + iprot.readMessageEnd() + raise x + result = get_partitions_with_specs_result() + result.read(iprot) + iprot.readMessageEnd() + if result.success is not None: + return result.success + if result.o1 is not None: + raise result.o1 + raise TApplicationException(TApplicationException.MISSING_RESULT, "get_partitions_with_specs failed: unknown result") + + def scheduled_query_poll(self, request): + """ + Parameters: + - request + + """ + self.send_scheduled_query_poll(request) + return self.recv_scheduled_query_poll() + + def send_scheduled_query_poll(self, request): + self._oprot.writeMessageBegin("scheduled_query_poll", TMessageType.CALL, self._seqid) + args = scheduled_query_poll_args() + args.request = request + args.write(self._oprot) + self._oprot.writeMessageEnd() + self._oprot.trans.flush() + + def recv_scheduled_query_poll(self): + iprot = self._iprot + (fname, mtype, rseqid) = iprot.readMessageBegin() + if mtype == TMessageType.EXCEPTION: + x = TApplicationException() + x.read(iprot) + iprot.readMessageEnd() + raise x + result = scheduled_query_poll_result() + result.read(iprot) + iprot.readMessageEnd() + if result.success is not None: + return result.success + if result.o1 is not None: + raise result.o1 + raise TApplicationException(TApplicationException.MISSING_RESULT, "scheduled_query_poll failed: unknown result") + + def scheduled_query_maintenance(self, request): + """ + Parameters: + - request + + """ + self.send_scheduled_query_maintenance(request) + self.recv_scheduled_query_maintenance() + + def send_scheduled_query_maintenance(self, request): + self._oprot.writeMessageBegin("scheduled_query_maintenance", TMessageType.CALL, self._seqid) + args = scheduled_query_maintenance_args() + args.request = request + args.write(self._oprot) + self._oprot.writeMessageEnd() + self._oprot.trans.flush() + + def recv_scheduled_query_maintenance(self): + iprot = self._iprot + (fname, mtype, rseqid) = iprot.readMessageBegin() + if mtype == TMessageType.EXCEPTION: + x = TApplicationException() + x.read(iprot) + iprot.readMessageEnd() + raise x + result = scheduled_query_maintenance_result() + result.read(iprot) + iprot.readMessageEnd() + if result.o1 is not None: + raise result.o1 + if result.o2 is not None: + raise result.o2 + if result.o3 is not None: + raise result.o3 + if result.o4 is not None: + raise result.o4 + return + + def scheduled_query_progress(self, info): + """ + Parameters: + - info + + """ + self.send_scheduled_query_progress(info) + self.recv_scheduled_query_progress() + + def send_scheduled_query_progress(self, info): + self._oprot.writeMessageBegin("scheduled_query_progress", TMessageType.CALL, self._seqid) + args = scheduled_query_progress_args() + args.info = info + args.write(self._oprot) + self._oprot.writeMessageEnd() + self._oprot.trans.flush() + + def recv_scheduled_query_progress(self): + iprot = self._iprot + (fname, mtype, rseqid) = iprot.readMessageBegin() + if mtype == TMessageType.EXCEPTION: + x = TApplicationException() + x.read(iprot) + iprot.readMessageEnd() + raise x + result = scheduled_query_progress_result() + result.read(iprot) + iprot.readMessageEnd() + if result.o1 is not None: + raise result.o1 + if result.o2 is not None: + raise result.o2 + return + + def get_scheduled_query(self, scheduleKey): + """ + Parameters: + - scheduleKey + + """ + self.send_get_scheduled_query(scheduleKey) + return self.recv_get_scheduled_query() + + def send_get_scheduled_query(self, scheduleKey): + self._oprot.writeMessageBegin("get_scheduled_query", TMessageType.CALL, self._seqid) + args = get_scheduled_query_args() + args.scheduleKey = scheduleKey + args.write(self._oprot) + self._oprot.writeMessageEnd() + self._oprot.trans.flush() + + def recv_get_scheduled_query(self): + iprot = self._iprot + (fname, mtype, rseqid) = iprot.readMessageBegin() + if mtype == TMessageType.EXCEPTION: + x = TApplicationException() + x.read(iprot) + iprot.readMessageEnd() + raise x + result = get_scheduled_query_result() + result.read(iprot) + iprot.readMessageEnd() + if result.success is not None: + return result.success + if result.o1 is not None: + raise result.o1 + if result.o2 is not None: + raise result.o2 + raise TApplicationException(TApplicationException.MISSING_RESULT, "get_scheduled_query failed: unknown result") + + def add_replication_metrics(self, replicationMetricList): + """ + Parameters: + - replicationMetricList + + """ + self.send_add_replication_metrics(replicationMetricList) + self.recv_add_replication_metrics() + + def send_add_replication_metrics(self, replicationMetricList): + self._oprot.writeMessageBegin("add_replication_metrics", TMessageType.CALL, self._seqid) + args = add_replication_metrics_args() + args.replicationMetricList = replicationMetricList + args.write(self._oprot) + self._oprot.writeMessageEnd() + self._oprot.trans.flush() + + def recv_add_replication_metrics(self): + iprot = self._iprot + (fname, mtype, rseqid) = iprot.readMessageBegin() + if mtype == TMessageType.EXCEPTION: + x = TApplicationException() + x.read(iprot) + iprot.readMessageEnd() + raise x + result = add_replication_metrics_result() + result.read(iprot) + iprot.readMessageEnd() + if result.o1 is not None: + raise result.o1 + return + + def get_replication_metrics(self, rqst): + """ + Parameters: + - rqst + + """ + self.send_get_replication_metrics(rqst) + return self.recv_get_replication_metrics() + + def send_get_replication_metrics(self, rqst): + self._oprot.writeMessageBegin("get_replication_metrics", TMessageType.CALL, self._seqid) + args = get_replication_metrics_args() + args.rqst = rqst + args.write(self._oprot) + self._oprot.writeMessageEnd() + self._oprot.trans.flush() + + def recv_get_replication_metrics(self): + iprot = self._iprot + (fname, mtype, rseqid) = iprot.readMessageBegin() + if mtype == TMessageType.EXCEPTION: + x = TApplicationException() + x.read(iprot) + iprot.readMessageEnd() + raise x + result = get_replication_metrics_result() + result.read(iprot) + iprot.readMessageEnd() + if result.success is not None: + return result.success + if result.o1 is not None: + raise result.o1 + raise TApplicationException(TApplicationException.MISSING_RESULT, "get_replication_metrics failed: unknown result") + + def get_open_txns_req(self, getOpenTxnsRequest): + """ + Parameters: + - getOpenTxnsRequest + + """ + self.send_get_open_txns_req(getOpenTxnsRequest) + return self.recv_get_open_txns_req() + + def send_get_open_txns_req(self, getOpenTxnsRequest): + self._oprot.writeMessageBegin("get_open_txns_req", TMessageType.CALL, self._seqid) + args = get_open_txns_req_args() + args.getOpenTxnsRequest = getOpenTxnsRequest + args.write(self._oprot) + self._oprot.writeMessageEnd() + self._oprot.trans.flush() + + def recv_get_open_txns_req(self): + iprot = self._iprot + (fname, mtype, rseqid) = iprot.readMessageBegin() + if mtype == TMessageType.EXCEPTION: + x = TApplicationException() + x.read(iprot) + iprot.readMessageEnd() + raise x + result = get_open_txns_req_result() + result.read(iprot) + iprot.readMessageEnd() + if result.success is not None: + return result.success + raise TApplicationException(TApplicationException.MISSING_RESULT, "get_open_txns_req failed: unknown result") + + def create_stored_procedure(self, proc): + """ + Parameters: + - proc + + """ + self.send_create_stored_procedure(proc) + self.recv_create_stored_procedure() + + def send_create_stored_procedure(self, proc): + self._oprot.writeMessageBegin("create_stored_procedure", TMessageType.CALL, self._seqid) + args = create_stored_procedure_args() + args.proc = proc + args.write(self._oprot) + self._oprot.writeMessageEnd() + self._oprot.trans.flush() + + def recv_create_stored_procedure(self): + iprot = self._iprot + (fname, mtype, rseqid) = iprot.readMessageBegin() + if mtype == TMessageType.EXCEPTION: + x = TApplicationException() + x.read(iprot) + iprot.readMessageEnd() + raise x + result = create_stored_procedure_result() + result.read(iprot) + iprot.readMessageEnd() + if result.o1 is not None: + raise result.o1 + if result.o2 is not None: + raise result.o2 + return + + def get_stored_procedure(self, request): + """ + Parameters: + - request + + """ + self.send_get_stored_procedure(request) + return self.recv_get_stored_procedure() + + def send_get_stored_procedure(self, request): + self._oprot.writeMessageBegin("get_stored_procedure", TMessageType.CALL, self._seqid) + args = get_stored_procedure_args() + args.request = request + args.write(self._oprot) + self._oprot.writeMessageEnd() + self._oprot.trans.flush() + + def recv_get_stored_procedure(self): + iprot = self._iprot + (fname, mtype, rseqid) = iprot.readMessageBegin() + if mtype == TMessageType.EXCEPTION: + x = TApplicationException() + x.read(iprot) + iprot.readMessageEnd() + raise x + result = get_stored_procedure_result() + result.read(iprot) + iprot.readMessageEnd() + if result.success is not None: + return result.success + if result.o1 is not None: + raise result.o1 + if result.o2 is not None: + raise result.o2 + raise TApplicationException(TApplicationException.MISSING_RESULT, "get_stored_procedure failed: unknown result") + + def drop_stored_procedure(self, request): + """ + Parameters: + - request + + """ + self.send_drop_stored_procedure(request) + self.recv_drop_stored_procedure() + + def send_drop_stored_procedure(self, request): + self._oprot.writeMessageBegin("drop_stored_procedure", TMessageType.CALL, self._seqid) + args = drop_stored_procedure_args() + args.request = request + args.write(self._oprot) + self._oprot.writeMessageEnd() + self._oprot.trans.flush() + + def recv_drop_stored_procedure(self): + iprot = self._iprot + (fname, mtype, rseqid) = iprot.readMessageBegin() + if mtype == TMessageType.EXCEPTION: + x = TApplicationException() + x.read(iprot) + iprot.readMessageEnd() + raise x + result = drop_stored_procedure_result() + result.read(iprot) + iprot.readMessageEnd() + if result.o1 is not None: + raise result.o1 + return + + def get_all_stored_procedures(self, request): + """ + Parameters: + - request + + """ + self.send_get_all_stored_procedures(request) + return self.recv_get_all_stored_procedures() + + def send_get_all_stored_procedures(self, request): + self._oprot.writeMessageBegin("get_all_stored_procedures", TMessageType.CALL, self._seqid) + args = get_all_stored_procedures_args() + args.request = request + args.write(self._oprot) + self._oprot.writeMessageEnd() + self._oprot.trans.flush() + + def recv_get_all_stored_procedures(self): + iprot = self._iprot + (fname, mtype, rseqid) = iprot.readMessageBegin() + if mtype == TMessageType.EXCEPTION: + x = TApplicationException() + x.read(iprot) + iprot.readMessageEnd() + raise x + result = get_all_stored_procedures_result() + result.read(iprot) + iprot.readMessageEnd() + if result.success is not None: + return result.success + if result.o1 is not None: + raise result.o1 + raise TApplicationException(TApplicationException.MISSING_RESULT, "get_all_stored_procedures failed: unknown result") + + def find_package(self, request): + """ + Parameters: + - request + + """ + self.send_find_package(request) + return self.recv_find_package() + + def send_find_package(self, request): + self._oprot.writeMessageBegin("find_package", TMessageType.CALL, self._seqid) + args = find_package_args() + args.request = request + args.write(self._oprot) + self._oprot.writeMessageEnd() + self._oprot.trans.flush() + + def recv_find_package(self): + iprot = self._iprot + (fname, mtype, rseqid) = iprot.readMessageBegin() + if mtype == TMessageType.EXCEPTION: + x = TApplicationException() + x.read(iprot) + iprot.readMessageEnd() + raise x + result = find_package_result() + result.read(iprot) + iprot.readMessageEnd() + if result.success is not None: + return result.success + if result.o1 is not None: + raise result.o1 + if result.o2 is not None: + raise result.o2 + raise TApplicationException(TApplicationException.MISSING_RESULT, "find_package failed: unknown result") + + def add_package(self, request): + """ + Parameters: + - request + + """ + self.send_add_package(request) + self.recv_add_package() + + def send_add_package(self, request): + self._oprot.writeMessageBegin("add_package", TMessageType.CALL, self._seqid) + args = add_package_args() + args.request = request + args.write(self._oprot) + self._oprot.writeMessageEnd() + self._oprot.trans.flush() + + def recv_add_package(self): + iprot = self._iprot + (fname, mtype, rseqid) = iprot.readMessageBegin() + if mtype == TMessageType.EXCEPTION: + x = TApplicationException() + x.read(iprot) + iprot.readMessageEnd() + raise x + result = add_package_result() + result.read(iprot) + iprot.readMessageEnd() + if result.o1 is not None: + raise result.o1 + return + + def get_all_packages(self, request): + """ + Parameters: + - request + + """ + self.send_get_all_packages(request) + return self.recv_get_all_packages() + + def send_get_all_packages(self, request): + self._oprot.writeMessageBegin("get_all_packages", TMessageType.CALL, self._seqid) + args = get_all_packages_args() + args.request = request + args.write(self._oprot) + self._oprot.writeMessageEnd() + self._oprot.trans.flush() + + def recv_get_all_packages(self): + iprot = self._iprot + (fname, mtype, rseqid) = iprot.readMessageBegin() + if mtype == TMessageType.EXCEPTION: + x = TApplicationException() + x.read(iprot) + iprot.readMessageEnd() + raise x + result = get_all_packages_result() + result.read(iprot) + iprot.readMessageEnd() + if result.success is not None: + return result.success + if result.o1 is not None: + raise result.o1 + raise TApplicationException(TApplicationException.MISSING_RESULT, "get_all_packages failed: unknown result") + + def drop_package(self, request): + """ + Parameters: + - request + + """ + self.send_drop_package(request) + self.recv_drop_package() + + def send_drop_package(self, request): + self._oprot.writeMessageBegin("drop_package", TMessageType.CALL, self._seqid) + args = drop_package_args() + args.request = request + args.write(self._oprot) + self._oprot.writeMessageEnd() + self._oprot.trans.flush() + + def recv_drop_package(self): + iprot = self._iprot + (fname, mtype, rseqid) = iprot.readMessageBegin() + if mtype == TMessageType.EXCEPTION: + x = TApplicationException() + x.read(iprot) + iprot.readMessageEnd() + raise x + result = drop_package_result() + result.read(iprot) + iprot.readMessageEnd() + if result.o1 is not None: + raise result.o1 + return + + def get_all_write_event_info(self, request): + """ + Parameters: + - request + + """ + self.send_get_all_write_event_info(request) + return self.recv_get_all_write_event_info() + + def send_get_all_write_event_info(self, request): + self._oprot.writeMessageBegin("get_all_write_event_info", TMessageType.CALL, self._seqid) + args = get_all_write_event_info_args() + args.request = request + args.write(self._oprot) + self._oprot.writeMessageEnd() + self._oprot.trans.flush() + + def recv_get_all_write_event_info(self): + iprot = self._iprot + (fname, mtype, rseqid) = iprot.readMessageBegin() + if mtype == TMessageType.EXCEPTION: + x = TApplicationException() + x.read(iprot) + iprot.readMessageEnd() + raise x + result = get_all_write_event_info_result() + result.read(iprot) + iprot.readMessageEnd() + if result.success is not None: + return result.success + if result.o1 is not None: + raise result.o1 + raise TApplicationException(TApplicationException.MISSING_RESULT, "get_all_write_event_info failed: unknown result") + + +class Processor(fb303.FacebookService.Processor, Iface, TProcessor): + def __init__(self, handler): + fb303.FacebookService.Processor.__init__(self, handler) + self._processMap["getMetaConf"] = Processor.process_getMetaConf + self._processMap["setMetaConf"] = Processor.process_setMetaConf + self._processMap["create_catalog"] = Processor.process_create_catalog + self._processMap["alter_catalog"] = Processor.process_alter_catalog + self._processMap["get_catalog"] = Processor.process_get_catalog + self._processMap["get_catalogs"] = Processor.process_get_catalogs + self._processMap["drop_catalog"] = Processor.process_drop_catalog + self._processMap["create_database"] = Processor.process_create_database + self._processMap["get_database"] = Processor.process_get_database + self._processMap["get_database_req"] = Processor.process_get_database_req + self._processMap["drop_database"] = Processor.process_drop_database + self._processMap["drop_database_req"] = Processor.process_drop_database_req + self._processMap["get_databases"] = Processor.process_get_databases + self._processMap["get_all_databases"] = Processor.process_get_all_databases + self._processMap["alter_database"] = Processor.process_alter_database + self._processMap["create_dataconnector"] = Processor.process_create_dataconnector + self._processMap["get_dataconnector_req"] = Processor.process_get_dataconnector_req + self._processMap["drop_dataconnector"] = Processor.process_drop_dataconnector + self._processMap["get_dataconnectors"] = Processor.process_get_dataconnectors + self._processMap["alter_dataconnector"] = Processor.process_alter_dataconnector + self._processMap["get_type"] = Processor.process_get_type + self._processMap["create_type"] = Processor.process_create_type + self._processMap["drop_type"] = Processor.process_drop_type + self._processMap["get_type_all"] = Processor.process_get_type_all + self._processMap["get_fields"] = Processor.process_get_fields + self._processMap["get_fields_with_environment_context"] = Processor.process_get_fields_with_environment_context + self._processMap["get_fields_req"] = Processor.process_get_fields_req + self._processMap["get_schema"] = Processor.process_get_schema + self._processMap["get_schema_with_environment_context"] = Processor.process_get_schema_with_environment_context + self._processMap["get_schema_req"] = Processor.process_get_schema_req + self._processMap["create_table"] = Processor.process_create_table + self._processMap["create_table_with_environment_context"] = Processor.process_create_table_with_environment_context + self._processMap["create_table_with_constraints"] = Processor.process_create_table_with_constraints + self._processMap["create_table_req"] = Processor.process_create_table_req + self._processMap["drop_constraint"] = Processor.process_drop_constraint + self._processMap["add_primary_key"] = Processor.process_add_primary_key + self._processMap["add_foreign_key"] = Processor.process_add_foreign_key + self._processMap["add_unique_constraint"] = Processor.process_add_unique_constraint + self._processMap["add_not_null_constraint"] = Processor.process_add_not_null_constraint + self._processMap["add_default_constraint"] = Processor.process_add_default_constraint + self._processMap["add_check_constraint"] = Processor.process_add_check_constraint + self._processMap["translate_table_dryrun"] = Processor.process_translate_table_dryrun + self._processMap["drop_table"] = Processor.process_drop_table + self._processMap["drop_table_with_environment_context"] = Processor.process_drop_table_with_environment_context + self._processMap["truncate_table"] = Processor.process_truncate_table + self._processMap["truncate_table_req"] = Processor.process_truncate_table_req + self._processMap["get_tables"] = Processor.process_get_tables + self._processMap["get_tables_by_type"] = Processor.process_get_tables_by_type + self._processMap[ + "get_all_materialized_view_objects_for_rewriting" + ] = Processor.process_get_all_materialized_view_objects_for_rewriting + self._processMap["get_materialized_views_for_rewriting"] = Processor.process_get_materialized_views_for_rewriting + self._processMap["get_table_meta"] = Processor.process_get_table_meta + self._processMap["get_all_tables"] = Processor.process_get_all_tables + self._processMap["get_table"] = Processor.process_get_table + self._processMap["get_table_objects_by_name"] = Processor.process_get_table_objects_by_name + self._processMap["get_tables_ext"] = Processor.process_get_tables_ext + self._processMap["get_table_req"] = Processor.process_get_table_req + self._processMap["get_table_objects_by_name_req"] = Processor.process_get_table_objects_by_name_req + self._processMap["get_materialization_invalidation_info"] = Processor.process_get_materialization_invalidation_info + self._processMap["update_creation_metadata"] = Processor.process_update_creation_metadata + self._processMap["get_table_names_by_filter"] = Processor.process_get_table_names_by_filter + self._processMap["alter_table"] = Processor.process_alter_table + self._processMap["alter_table_with_environment_context"] = Processor.process_alter_table_with_environment_context + self._processMap["alter_table_with_cascade"] = Processor.process_alter_table_with_cascade + self._processMap["alter_table_req"] = Processor.process_alter_table_req + self._processMap["add_partition"] = Processor.process_add_partition + self._processMap["add_partition_with_environment_context"] = Processor.process_add_partition_with_environment_context + self._processMap["add_partitions"] = Processor.process_add_partitions + self._processMap["add_partitions_pspec"] = Processor.process_add_partitions_pspec + self._processMap["append_partition"] = Processor.process_append_partition + self._processMap["add_partitions_req"] = Processor.process_add_partitions_req + self._processMap[ + "append_partition_with_environment_context" + ] = Processor.process_append_partition_with_environment_context + self._processMap["append_partition_by_name"] = Processor.process_append_partition_by_name + self._processMap[ + "append_partition_by_name_with_environment_context" + ] = Processor.process_append_partition_by_name_with_environment_context + self._processMap["drop_partition"] = Processor.process_drop_partition + self._processMap["drop_partition_with_environment_context"] = Processor.process_drop_partition_with_environment_context + self._processMap["drop_partition_by_name"] = Processor.process_drop_partition_by_name + self._processMap[ + "drop_partition_by_name_with_environment_context" + ] = Processor.process_drop_partition_by_name_with_environment_context + self._processMap["drop_partitions_req"] = Processor.process_drop_partitions_req + self._processMap["get_partition"] = Processor.process_get_partition + self._processMap["get_partition_req"] = Processor.process_get_partition_req + self._processMap["exchange_partition"] = Processor.process_exchange_partition + self._processMap["exchange_partitions"] = Processor.process_exchange_partitions + self._processMap["get_partition_with_auth"] = Processor.process_get_partition_with_auth + self._processMap["get_partition_by_name"] = Processor.process_get_partition_by_name + self._processMap["get_partitions"] = Processor.process_get_partitions + self._processMap["get_partitions_req"] = Processor.process_get_partitions_req + self._processMap["get_partitions_with_auth"] = Processor.process_get_partitions_with_auth + self._processMap["get_partitions_pspec"] = Processor.process_get_partitions_pspec + self._processMap["get_partition_names"] = Processor.process_get_partition_names + self._processMap["get_partition_values"] = Processor.process_get_partition_values + self._processMap["get_partitions_ps"] = Processor.process_get_partitions_ps + self._processMap["get_partitions_ps_with_auth"] = Processor.process_get_partitions_ps_with_auth + self._processMap["get_partitions_ps_with_auth_req"] = Processor.process_get_partitions_ps_with_auth_req + self._processMap["get_partition_names_ps"] = Processor.process_get_partition_names_ps + self._processMap["get_partition_names_ps_req"] = Processor.process_get_partition_names_ps_req + self._processMap["get_partition_names_req"] = Processor.process_get_partition_names_req + self._processMap["get_partitions_by_filter"] = Processor.process_get_partitions_by_filter + self._processMap["get_part_specs_by_filter"] = Processor.process_get_part_specs_by_filter + self._processMap["get_partitions_by_expr"] = Processor.process_get_partitions_by_expr + self._processMap["get_partitions_spec_by_expr"] = Processor.process_get_partitions_spec_by_expr + self._processMap["get_num_partitions_by_filter"] = Processor.process_get_num_partitions_by_filter + self._processMap["get_partitions_by_names"] = Processor.process_get_partitions_by_names + self._processMap["get_partitions_by_names_req"] = Processor.process_get_partitions_by_names_req + self._processMap["alter_partition"] = Processor.process_alter_partition + self._processMap["alter_partitions"] = Processor.process_alter_partitions + self._processMap[ + "alter_partitions_with_environment_context" + ] = Processor.process_alter_partitions_with_environment_context + self._processMap["alter_partitions_req"] = Processor.process_alter_partitions_req + self._processMap["alter_partition_with_environment_context"] = Processor.process_alter_partition_with_environment_context + self._processMap["rename_partition"] = Processor.process_rename_partition + self._processMap["rename_partition_req"] = Processor.process_rename_partition_req + self._processMap["partition_name_has_valid_characters"] = Processor.process_partition_name_has_valid_characters + self._processMap["get_config_value"] = Processor.process_get_config_value + self._processMap["partition_name_to_vals"] = Processor.process_partition_name_to_vals + self._processMap["partition_name_to_spec"] = Processor.process_partition_name_to_spec + self._processMap["markPartitionForEvent"] = Processor.process_markPartitionForEvent + self._processMap["isPartitionMarkedForEvent"] = Processor.process_isPartitionMarkedForEvent + self._processMap["get_primary_keys"] = Processor.process_get_primary_keys + self._processMap["get_foreign_keys"] = Processor.process_get_foreign_keys + self._processMap["get_unique_constraints"] = Processor.process_get_unique_constraints + self._processMap["get_not_null_constraints"] = Processor.process_get_not_null_constraints + self._processMap["get_default_constraints"] = Processor.process_get_default_constraints + self._processMap["get_check_constraints"] = Processor.process_get_check_constraints + self._processMap["get_all_table_constraints"] = Processor.process_get_all_table_constraints + self._processMap["update_table_column_statistics"] = Processor.process_update_table_column_statistics + self._processMap["update_partition_column_statistics"] = Processor.process_update_partition_column_statistics + self._processMap["update_table_column_statistics_req"] = Processor.process_update_table_column_statistics_req + self._processMap["update_partition_column_statistics_req"] = Processor.process_update_partition_column_statistics_req + self._processMap["update_transaction_statistics"] = Processor.process_update_transaction_statistics + self._processMap["get_table_column_statistics"] = Processor.process_get_table_column_statistics + self._processMap["get_partition_column_statistics"] = Processor.process_get_partition_column_statistics + self._processMap["get_table_statistics_req"] = Processor.process_get_table_statistics_req + self._processMap["get_partitions_statistics_req"] = Processor.process_get_partitions_statistics_req + self._processMap["get_aggr_stats_for"] = Processor.process_get_aggr_stats_for + self._processMap["set_aggr_stats_for"] = Processor.process_set_aggr_stats_for + self._processMap["delete_partition_column_statistics"] = Processor.process_delete_partition_column_statistics + self._processMap["delete_table_column_statistics"] = Processor.process_delete_table_column_statistics + self._processMap["create_function"] = Processor.process_create_function + self._processMap["drop_function"] = Processor.process_drop_function + self._processMap["alter_function"] = Processor.process_alter_function + self._processMap["get_functions"] = Processor.process_get_functions + self._processMap["get_function"] = Processor.process_get_function + self._processMap["get_all_functions"] = Processor.process_get_all_functions + self._processMap["create_role"] = Processor.process_create_role + self._processMap["drop_role"] = Processor.process_drop_role + self._processMap["get_role_names"] = Processor.process_get_role_names + self._processMap["grant_role"] = Processor.process_grant_role + self._processMap["revoke_role"] = Processor.process_revoke_role + self._processMap["list_roles"] = Processor.process_list_roles + self._processMap["grant_revoke_role"] = Processor.process_grant_revoke_role + self._processMap["get_principals_in_role"] = Processor.process_get_principals_in_role + self._processMap["get_role_grants_for_principal"] = Processor.process_get_role_grants_for_principal + self._processMap["get_privilege_set"] = Processor.process_get_privilege_set + self._processMap["list_privileges"] = Processor.process_list_privileges + self._processMap["grant_privileges"] = Processor.process_grant_privileges + self._processMap["revoke_privileges"] = Processor.process_revoke_privileges + self._processMap["grant_revoke_privileges"] = Processor.process_grant_revoke_privileges + self._processMap["refresh_privileges"] = Processor.process_refresh_privileges + self._processMap["set_ugi"] = Processor.process_set_ugi + self._processMap["get_delegation_token"] = Processor.process_get_delegation_token + self._processMap["renew_delegation_token"] = Processor.process_renew_delegation_token + self._processMap["cancel_delegation_token"] = Processor.process_cancel_delegation_token + self._processMap["add_token"] = Processor.process_add_token + self._processMap["remove_token"] = Processor.process_remove_token + self._processMap["get_token"] = Processor.process_get_token + self._processMap["get_all_token_identifiers"] = Processor.process_get_all_token_identifiers + self._processMap["add_master_key"] = Processor.process_add_master_key + self._processMap["update_master_key"] = Processor.process_update_master_key + self._processMap["remove_master_key"] = Processor.process_remove_master_key + self._processMap["get_master_keys"] = Processor.process_get_master_keys + self._processMap["get_open_txns"] = Processor.process_get_open_txns + self._processMap["get_open_txns_info"] = Processor.process_get_open_txns_info + self._processMap["open_txns"] = Processor.process_open_txns + self._processMap["abort_txn"] = Processor.process_abort_txn + self._processMap["abort_txns"] = Processor.process_abort_txns + self._processMap["commit_txn"] = Processor.process_commit_txn + self._processMap["get_latest_txnid_in_conflict"] = Processor.process_get_latest_txnid_in_conflict + self._processMap["repl_tbl_writeid_state"] = Processor.process_repl_tbl_writeid_state + self._processMap["get_valid_write_ids"] = Processor.process_get_valid_write_ids + self._processMap["allocate_table_write_ids"] = Processor.process_allocate_table_write_ids + self._processMap["get_max_allocated_table_write_id"] = Processor.process_get_max_allocated_table_write_id + self._processMap["seed_write_id"] = Processor.process_seed_write_id + self._processMap["seed_txn_id"] = Processor.process_seed_txn_id + self._processMap["lock"] = Processor.process_lock + self._processMap["check_lock"] = Processor.process_check_lock + self._processMap["unlock"] = Processor.process_unlock + self._processMap["show_locks"] = Processor.process_show_locks + self._processMap["heartbeat"] = Processor.process_heartbeat + self._processMap["heartbeat_txn_range"] = Processor.process_heartbeat_txn_range + self._processMap["compact"] = Processor.process_compact + self._processMap["compact2"] = Processor.process_compact2 + self._processMap["show_compact"] = Processor.process_show_compact + self._processMap["add_dynamic_partitions"] = Processor.process_add_dynamic_partitions + self._processMap["find_next_compact"] = Processor.process_find_next_compact + self._processMap["find_next_compact2"] = Processor.process_find_next_compact2 + self._processMap["update_compactor_state"] = Processor.process_update_compactor_state + self._processMap["find_columns_with_stats"] = Processor.process_find_columns_with_stats + self._processMap["mark_cleaned"] = Processor.process_mark_cleaned + self._processMap["mark_compacted"] = Processor.process_mark_compacted + self._processMap["mark_failed"] = Processor.process_mark_failed + self._processMap["mark_refused"] = Processor.process_mark_refused + self._processMap["update_compaction_metrics_data"] = Processor.process_update_compaction_metrics_data + self._processMap["remove_compaction_metrics_data"] = Processor.process_remove_compaction_metrics_data + self._processMap["set_hadoop_jobid"] = Processor.process_set_hadoop_jobid + self._processMap["get_latest_committed_compaction_info"] = Processor.process_get_latest_committed_compaction_info + self._processMap["get_next_notification"] = Processor.process_get_next_notification + self._processMap["get_current_notificationEventId"] = Processor.process_get_current_notificationEventId + self._processMap["get_notification_events_count"] = Processor.process_get_notification_events_count + self._processMap["fire_listener_event"] = Processor.process_fire_listener_event + self._processMap["flushCache"] = Processor.process_flushCache + self._processMap["add_write_notification_log"] = Processor.process_add_write_notification_log + self._processMap["add_write_notification_log_in_batch"] = Processor.process_add_write_notification_log_in_batch + self._processMap["cm_recycle"] = Processor.process_cm_recycle + self._processMap["get_file_metadata_by_expr"] = Processor.process_get_file_metadata_by_expr + self._processMap["get_file_metadata"] = Processor.process_get_file_metadata + self._processMap["put_file_metadata"] = Processor.process_put_file_metadata + self._processMap["clear_file_metadata"] = Processor.process_clear_file_metadata + self._processMap["cache_file_metadata"] = Processor.process_cache_file_metadata + self._processMap["get_metastore_db_uuid"] = Processor.process_get_metastore_db_uuid + self._processMap["create_resource_plan"] = Processor.process_create_resource_plan + self._processMap["get_resource_plan"] = Processor.process_get_resource_plan + self._processMap["get_active_resource_plan"] = Processor.process_get_active_resource_plan + self._processMap["get_all_resource_plans"] = Processor.process_get_all_resource_plans + self._processMap["alter_resource_plan"] = Processor.process_alter_resource_plan + self._processMap["validate_resource_plan"] = Processor.process_validate_resource_plan + self._processMap["drop_resource_plan"] = Processor.process_drop_resource_plan + self._processMap["create_wm_trigger"] = Processor.process_create_wm_trigger + self._processMap["alter_wm_trigger"] = Processor.process_alter_wm_trigger + self._processMap["drop_wm_trigger"] = Processor.process_drop_wm_trigger + self._processMap["get_triggers_for_resourceplan"] = Processor.process_get_triggers_for_resourceplan + self._processMap["create_wm_pool"] = Processor.process_create_wm_pool + self._processMap["alter_wm_pool"] = Processor.process_alter_wm_pool + self._processMap["drop_wm_pool"] = Processor.process_drop_wm_pool + self._processMap["create_or_update_wm_mapping"] = Processor.process_create_or_update_wm_mapping + self._processMap["drop_wm_mapping"] = Processor.process_drop_wm_mapping + self._processMap[ + "create_or_drop_wm_trigger_to_pool_mapping" + ] = Processor.process_create_or_drop_wm_trigger_to_pool_mapping + self._processMap["create_ischema"] = Processor.process_create_ischema + self._processMap["alter_ischema"] = Processor.process_alter_ischema + self._processMap["get_ischema"] = Processor.process_get_ischema + self._processMap["drop_ischema"] = Processor.process_drop_ischema + self._processMap["add_schema_version"] = Processor.process_add_schema_version + self._processMap["get_schema_version"] = Processor.process_get_schema_version + self._processMap["get_schema_latest_version"] = Processor.process_get_schema_latest_version + self._processMap["get_schema_all_versions"] = Processor.process_get_schema_all_versions + self._processMap["drop_schema_version"] = Processor.process_drop_schema_version + self._processMap["get_schemas_by_cols"] = Processor.process_get_schemas_by_cols + self._processMap["map_schema_version_to_serde"] = Processor.process_map_schema_version_to_serde + self._processMap["set_schema_version_state"] = Processor.process_set_schema_version_state + self._processMap["add_serde"] = Processor.process_add_serde + self._processMap["get_serde"] = Processor.process_get_serde + self._processMap["get_lock_materialization_rebuild"] = Processor.process_get_lock_materialization_rebuild + self._processMap["heartbeat_lock_materialization_rebuild"] = Processor.process_heartbeat_lock_materialization_rebuild + self._processMap["add_runtime_stats"] = Processor.process_add_runtime_stats + self._processMap["get_runtime_stats"] = Processor.process_get_runtime_stats + self._processMap["get_partitions_with_specs"] = Processor.process_get_partitions_with_specs + self._processMap["scheduled_query_poll"] = Processor.process_scheduled_query_poll + self._processMap["scheduled_query_maintenance"] = Processor.process_scheduled_query_maintenance + self._processMap["scheduled_query_progress"] = Processor.process_scheduled_query_progress + self._processMap["get_scheduled_query"] = Processor.process_get_scheduled_query + self._processMap["add_replication_metrics"] = Processor.process_add_replication_metrics + self._processMap["get_replication_metrics"] = Processor.process_get_replication_metrics + self._processMap["get_open_txns_req"] = Processor.process_get_open_txns_req + self._processMap["create_stored_procedure"] = Processor.process_create_stored_procedure + self._processMap["get_stored_procedure"] = Processor.process_get_stored_procedure + self._processMap["drop_stored_procedure"] = Processor.process_drop_stored_procedure + self._processMap["get_all_stored_procedures"] = Processor.process_get_all_stored_procedures + self._processMap["find_package"] = Processor.process_find_package + self._processMap["add_package"] = Processor.process_add_package + self._processMap["get_all_packages"] = Processor.process_get_all_packages + self._processMap["drop_package"] = Processor.process_drop_package + self._processMap["get_all_write_event_info"] = Processor.process_get_all_write_event_info + self._on_message_begin = None + + def on_message_begin(self, func): + self._on_message_begin = func + + def process(self, iprot, oprot): + (name, type, seqid) = iprot.readMessageBegin() + if self._on_message_begin: + self._on_message_begin(name, type, seqid) + if name not in self._processMap: + iprot.skip(TType.STRUCT) + iprot.readMessageEnd() + x = TApplicationException(TApplicationException.UNKNOWN_METHOD, "Unknown function %s" % (name)) + oprot.writeMessageBegin(name, TMessageType.EXCEPTION, seqid) + x.write(oprot) + oprot.writeMessageEnd() + oprot.trans.flush() + return + else: + self._processMap[name](self, seqid, iprot, oprot) + return True + + def process_getMetaConf(self, seqid, iprot, oprot): + args = getMetaConf_args() + args.read(iprot) + iprot.readMessageEnd() + result = getMetaConf_result() + try: + result.success = self._handler.getMetaConf(args.key) + msg_type = TMessageType.REPLY + except TTransport.TTransportException: + raise + except MetaException as o1: + msg_type = TMessageType.REPLY + result.o1 = o1 + except TApplicationException as ex: + logging.exception("TApplication exception in handler") + msg_type = TMessageType.EXCEPTION + result = ex + except Exception: + logging.exception("Unexpected exception in handler") + msg_type = TMessageType.EXCEPTION + result = TApplicationException(TApplicationException.INTERNAL_ERROR, "Internal error") + oprot.writeMessageBegin("getMetaConf", msg_type, seqid) + result.write(oprot) + oprot.writeMessageEnd() + oprot.trans.flush() + + def process_setMetaConf(self, seqid, iprot, oprot): + args = setMetaConf_args() + args.read(iprot) + iprot.readMessageEnd() + result = setMetaConf_result() + try: + self._handler.setMetaConf(args.key, args.value) + msg_type = TMessageType.REPLY + except TTransport.TTransportException: + raise + except MetaException as o1: + msg_type = TMessageType.REPLY + result.o1 = o1 + except TApplicationException as ex: + logging.exception("TApplication exception in handler") + msg_type = TMessageType.EXCEPTION + result = ex + except Exception: + logging.exception("Unexpected exception in handler") + msg_type = TMessageType.EXCEPTION + result = TApplicationException(TApplicationException.INTERNAL_ERROR, "Internal error") + oprot.writeMessageBegin("setMetaConf", msg_type, seqid) + result.write(oprot) + oprot.writeMessageEnd() + oprot.trans.flush() + + def process_create_catalog(self, seqid, iprot, oprot): + args = create_catalog_args() + args.read(iprot) + iprot.readMessageEnd() + result = create_catalog_result() + try: + self._handler.create_catalog(args.catalog) + msg_type = TMessageType.REPLY + except TTransport.TTransportException: + raise + except AlreadyExistsException as o1: + msg_type = TMessageType.REPLY + result.o1 = o1 + except InvalidObjectException as o2: + msg_type = TMessageType.REPLY + result.o2 = o2 + except MetaException as o3: + msg_type = TMessageType.REPLY + result.o3 = o3 + except TApplicationException as ex: + logging.exception("TApplication exception in handler") + msg_type = TMessageType.EXCEPTION + result = ex + except Exception: + logging.exception("Unexpected exception in handler") + msg_type = TMessageType.EXCEPTION + result = TApplicationException(TApplicationException.INTERNAL_ERROR, "Internal error") + oprot.writeMessageBegin("create_catalog", msg_type, seqid) + result.write(oprot) + oprot.writeMessageEnd() + oprot.trans.flush() + + def process_alter_catalog(self, seqid, iprot, oprot): + args = alter_catalog_args() + args.read(iprot) + iprot.readMessageEnd() + result = alter_catalog_result() + try: + self._handler.alter_catalog(args.rqst) + msg_type = TMessageType.REPLY + except TTransport.TTransportException: + raise + except NoSuchObjectException as o1: + msg_type = TMessageType.REPLY + result.o1 = o1 + except InvalidOperationException as o2: + msg_type = TMessageType.REPLY + result.o2 = o2 + except MetaException as o3: + msg_type = TMessageType.REPLY + result.o3 = o3 + except TApplicationException as ex: + logging.exception("TApplication exception in handler") + msg_type = TMessageType.EXCEPTION + result = ex + except Exception: + logging.exception("Unexpected exception in handler") + msg_type = TMessageType.EXCEPTION + result = TApplicationException(TApplicationException.INTERNAL_ERROR, "Internal error") + oprot.writeMessageBegin("alter_catalog", msg_type, seqid) + result.write(oprot) + oprot.writeMessageEnd() + oprot.trans.flush() + + def process_get_catalog(self, seqid, iprot, oprot): + args = get_catalog_args() + args.read(iprot) + iprot.readMessageEnd() + result = get_catalog_result() + try: + result.success = self._handler.get_catalog(args.catName) + msg_type = TMessageType.REPLY + except TTransport.TTransportException: + raise + except NoSuchObjectException as o1: + msg_type = TMessageType.REPLY + result.o1 = o1 + except MetaException as o2: + msg_type = TMessageType.REPLY + result.o2 = o2 + except TApplicationException as ex: + logging.exception("TApplication exception in handler") + msg_type = TMessageType.EXCEPTION + result = ex + except Exception: + logging.exception("Unexpected exception in handler") + msg_type = TMessageType.EXCEPTION + result = TApplicationException(TApplicationException.INTERNAL_ERROR, "Internal error") + oprot.writeMessageBegin("get_catalog", msg_type, seqid) + result.write(oprot) + oprot.writeMessageEnd() + oprot.trans.flush() + + def process_get_catalogs(self, seqid, iprot, oprot): + args = get_catalogs_args() + args.read(iprot) + iprot.readMessageEnd() + result = get_catalogs_result() + try: + result.success = self._handler.get_catalogs() + msg_type = TMessageType.REPLY + except TTransport.TTransportException: + raise + except MetaException as o1: + msg_type = TMessageType.REPLY + result.o1 = o1 + except TApplicationException as ex: + logging.exception("TApplication exception in handler") + msg_type = TMessageType.EXCEPTION + result = ex + except Exception: + logging.exception("Unexpected exception in handler") + msg_type = TMessageType.EXCEPTION + result = TApplicationException(TApplicationException.INTERNAL_ERROR, "Internal error") + oprot.writeMessageBegin("get_catalogs", msg_type, seqid) + result.write(oprot) + oprot.writeMessageEnd() + oprot.trans.flush() + + def process_drop_catalog(self, seqid, iprot, oprot): + args = drop_catalog_args() + args.read(iprot) + iprot.readMessageEnd() + result = drop_catalog_result() + try: + self._handler.drop_catalog(args.catName) + msg_type = TMessageType.REPLY + except TTransport.TTransportException: + raise + except NoSuchObjectException as o1: + msg_type = TMessageType.REPLY + result.o1 = o1 + except InvalidOperationException as o2: + msg_type = TMessageType.REPLY + result.o2 = o2 + except MetaException as o3: + msg_type = TMessageType.REPLY + result.o3 = o3 + except TApplicationException as ex: + logging.exception("TApplication exception in handler") + msg_type = TMessageType.EXCEPTION + result = ex + except Exception: + logging.exception("Unexpected exception in handler") + msg_type = TMessageType.EXCEPTION + result = TApplicationException(TApplicationException.INTERNAL_ERROR, "Internal error") + oprot.writeMessageBegin("drop_catalog", msg_type, seqid) + result.write(oprot) + oprot.writeMessageEnd() + oprot.trans.flush() + + def process_create_database(self, seqid, iprot, oprot): + args = create_database_args() + args.read(iprot) + iprot.readMessageEnd() + result = create_database_result() + try: + self._handler.create_database(args.database) + msg_type = TMessageType.REPLY + except TTransport.TTransportException: + raise + except AlreadyExistsException as o1: + msg_type = TMessageType.REPLY + result.o1 = o1 + except InvalidObjectException as o2: + msg_type = TMessageType.REPLY + result.o2 = o2 + except MetaException as o3: + msg_type = TMessageType.REPLY + result.o3 = o3 + except TApplicationException as ex: + logging.exception("TApplication exception in handler") + msg_type = TMessageType.EXCEPTION + result = ex + except Exception: + logging.exception("Unexpected exception in handler") + msg_type = TMessageType.EXCEPTION + result = TApplicationException(TApplicationException.INTERNAL_ERROR, "Internal error") + oprot.writeMessageBegin("create_database", msg_type, seqid) + result.write(oprot) + oprot.writeMessageEnd() + oprot.trans.flush() + + def process_get_database(self, seqid, iprot, oprot): + args = get_database_args() + args.read(iprot) + iprot.readMessageEnd() + result = get_database_result() + try: + result.success = self._handler.get_database(args.name) + msg_type = TMessageType.REPLY + except TTransport.TTransportException: + raise + except NoSuchObjectException as o1: + msg_type = TMessageType.REPLY + result.o1 = o1 + except MetaException as o2: + msg_type = TMessageType.REPLY + result.o2 = o2 + except TApplicationException as ex: + logging.exception("TApplication exception in handler") + msg_type = TMessageType.EXCEPTION + result = ex + except Exception: + logging.exception("Unexpected exception in handler") + msg_type = TMessageType.EXCEPTION + result = TApplicationException(TApplicationException.INTERNAL_ERROR, "Internal error") + oprot.writeMessageBegin("get_database", msg_type, seqid) + result.write(oprot) + oprot.writeMessageEnd() + oprot.trans.flush() + + def process_get_database_req(self, seqid, iprot, oprot): + args = get_database_req_args() + args.read(iprot) + iprot.readMessageEnd() + result = get_database_req_result() + try: + result.success = self._handler.get_database_req(args.request) + msg_type = TMessageType.REPLY + except TTransport.TTransportException: + raise + except NoSuchObjectException as o1: + msg_type = TMessageType.REPLY + result.o1 = o1 + except MetaException as o2: + msg_type = TMessageType.REPLY + result.o2 = o2 + except TApplicationException as ex: + logging.exception("TApplication exception in handler") + msg_type = TMessageType.EXCEPTION + result = ex + except Exception: + logging.exception("Unexpected exception in handler") + msg_type = TMessageType.EXCEPTION + result = TApplicationException(TApplicationException.INTERNAL_ERROR, "Internal error") + oprot.writeMessageBegin("get_database_req", msg_type, seqid) + result.write(oprot) + oprot.writeMessageEnd() + oprot.trans.flush() + + def process_drop_database(self, seqid, iprot, oprot): + args = drop_database_args() + args.read(iprot) + iprot.readMessageEnd() + result = drop_database_result() + try: + self._handler.drop_database(args.name, args.deleteData, args.cascade) + msg_type = TMessageType.REPLY + except TTransport.TTransportException: + raise + except NoSuchObjectException as o1: + msg_type = TMessageType.REPLY + result.o1 = o1 + except InvalidOperationException as o2: + msg_type = TMessageType.REPLY + result.o2 = o2 + except MetaException as o3: + msg_type = TMessageType.REPLY + result.o3 = o3 + except TApplicationException as ex: + logging.exception("TApplication exception in handler") + msg_type = TMessageType.EXCEPTION + result = ex + except Exception: + logging.exception("Unexpected exception in handler") + msg_type = TMessageType.EXCEPTION + result = TApplicationException(TApplicationException.INTERNAL_ERROR, "Internal error") + oprot.writeMessageBegin("drop_database", msg_type, seqid) + result.write(oprot) + oprot.writeMessageEnd() + oprot.trans.flush() + + def process_drop_database_req(self, seqid, iprot, oprot): + args = drop_database_req_args() + args.read(iprot) + iprot.readMessageEnd() + result = drop_database_req_result() + try: + self._handler.drop_database_req(args.req) + msg_type = TMessageType.REPLY + except TTransport.TTransportException: + raise + except NoSuchObjectException as o1: + msg_type = TMessageType.REPLY + result.o1 = o1 + except InvalidOperationException as o2: + msg_type = TMessageType.REPLY + result.o2 = o2 + except MetaException as o3: + msg_type = TMessageType.REPLY + result.o3 = o3 + except TApplicationException as ex: + logging.exception("TApplication exception in handler") + msg_type = TMessageType.EXCEPTION + result = ex + except Exception: + logging.exception("Unexpected exception in handler") + msg_type = TMessageType.EXCEPTION + result = TApplicationException(TApplicationException.INTERNAL_ERROR, "Internal error") + oprot.writeMessageBegin("drop_database_req", msg_type, seqid) + result.write(oprot) + oprot.writeMessageEnd() + oprot.trans.flush() + + def process_get_databases(self, seqid, iprot, oprot): + args = get_databases_args() + args.read(iprot) + iprot.readMessageEnd() + result = get_databases_result() + try: + result.success = self._handler.get_databases(args.pattern) + msg_type = TMessageType.REPLY + except TTransport.TTransportException: + raise + except MetaException as o1: + msg_type = TMessageType.REPLY + result.o1 = o1 + except TApplicationException as ex: + logging.exception("TApplication exception in handler") + msg_type = TMessageType.EXCEPTION + result = ex + except Exception: + logging.exception("Unexpected exception in handler") + msg_type = TMessageType.EXCEPTION + result = TApplicationException(TApplicationException.INTERNAL_ERROR, "Internal error") + oprot.writeMessageBegin("get_databases", msg_type, seqid) + result.write(oprot) + oprot.writeMessageEnd() + oprot.trans.flush() + + def process_get_all_databases(self, seqid, iprot, oprot): + args = get_all_databases_args() + args.read(iprot) + iprot.readMessageEnd() + result = get_all_databases_result() + try: + result.success = self._handler.get_all_databases() + msg_type = TMessageType.REPLY + except TTransport.TTransportException: + raise + except MetaException as o1: + msg_type = TMessageType.REPLY + result.o1 = o1 + except TApplicationException as ex: + logging.exception("TApplication exception in handler") + msg_type = TMessageType.EXCEPTION + result = ex + except Exception: + logging.exception("Unexpected exception in handler") + msg_type = TMessageType.EXCEPTION + result = TApplicationException(TApplicationException.INTERNAL_ERROR, "Internal error") + oprot.writeMessageBegin("get_all_databases", msg_type, seqid) + result.write(oprot) + oprot.writeMessageEnd() + oprot.trans.flush() + + def process_alter_database(self, seqid, iprot, oprot): + args = alter_database_args() + args.read(iprot) + iprot.readMessageEnd() + result = alter_database_result() + try: + self._handler.alter_database(args.dbname, args.db) + msg_type = TMessageType.REPLY + except TTransport.TTransportException: + raise + except MetaException as o1: + msg_type = TMessageType.REPLY + result.o1 = o1 + except NoSuchObjectException as o2: + msg_type = TMessageType.REPLY + result.o2 = o2 + except TApplicationException as ex: + logging.exception("TApplication exception in handler") + msg_type = TMessageType.EXCEPTION + result = ex + except Exception: + logging.exception("Unexpected exception in handler") + msg_type = TMessageType.EXCEPTION + result = TApplicationException(TApplicationException.INTERNAL_ERROR, "Internal error") + oprot.writeMessageBegin("alter_database", msg_type, seqid) + result.write(oprot) + oprot.writeMessageEnd() + oprot.trans.flush() + + def process_create_dataconnector(self, seqid, iprot, oprot): + args = create_dataconnector_args() + args.read(iprot) + iprot.readMessageEnd() + result = create_dataconnector_result() + try: + self._handler.create_dataconnector(args.connector) + msg_type = TMessageType.REPLY + except TTransport.TTransportException: + raise + except AlreadyExistsException as o1: + msg_type = TMessageType.REPLY + result.o1 = o1 + except InvalidObjectException as o2: + msg_type = TMessageType.REPLY + result.o2 = o2 + except MetaException as o3: + msg_type = TMessageType.REPLY + result.o3 = o3 + except TApplicationException as ex: + logging.exception("TApplication exception in handler") + msg_type = TMessageType.EXCEPTION + result = ex + except Exception: + logging.exception("Unexpected exception in handler") + msg_type = TMessageType.EXCEPTION + result = TApplicationException(TApplicationException.INTERNAL_ERROR, "Internal error") + oprot.writeMessageBegin("create_dataconnector", msg_type, seqid) + result.write(oprot) + oprot.writeMessageEnd() + oprot.trans.flush() + + def process_get_dataconnector_req(self, seqid, iprot, oprot): + args = get_dataconnector_req_args() + args.read(iprot) + iprot.readMessageEnd() + result = get_dataconnector_req_result() + try: + result.success = self._handler.get_dataconnector_req(args.request) + msg_type = TMessageType.REPLY + except TTransport.TTransportException: + raise + except NoSuchObjectException as o1: + msg_type = TMessageType.REPLY + result.o1 = o1 + except MetaException as o2: + msg_type = TMessageType.REPLY + result.o2 = o2 + except TApplicationException as ex: + logging.exception("TApplication exception in handler") + msg_type = TMessageType.EXCEPTION + result = ex + except Exception: + logging.exception("Unexpected exception in handler") + msg_type = TMessageType.EXCEPTION + result = TApplicationException(TApplicationException.INTERNAL_ERROR, "Internal error") + oprot.writeMessageBegin("get_dataconnector_req", msg_type, seqid) + result.write(oprot) + oprot.writeMessageEnd() + oprot.trans.flush() + + def process_drop_dataconnector(self, seqid, iprot, oprot): + args = drop_dataconnector_args() + args.read(iprot) + iprot.readMessageEnd() + result = drop_dataconnector_result() + try: + self._handler.drop_dataconnector(args.name, args.ifNotExists, args.checkReferences) + msg_type = TMessageType.REPLY + except TTransport.TTransportException: + raise + except NoSuchObjectException as o1: + msg_type = TMessageType.REPLY + result.o1 = o1 + except InvalidOperationException as o2: + msg_type = TMessageType.REPLY + result.o2 = o2 + except MetaException as o3: + msg_type = TMessageType.REPLY + result.o3 = o3 + except TApplicationException as ex: + logging.exception("TApplication exception in handler") + msg_type = TMessageType.EXCEPTION + result = ex + except Exception: + logging.exception("Unexpected exception in handler") + msg_type = TMessageType.EXCEPTION + result = TApplicationException(TApplicationException.INTERNAL_ERROR, "Internal error") + oprot.writeMessageBegin("drop_dataconnector", msg_type, seqid) + result.write(oprot) + oprot.writeMessageEnd() + oprot.trans.flush() + + def process_get_dataconnectors(self, seqid, iprot, oprot): + args = get_dataconnectors_args() + args.read(iprot) + iprot.readMessageEnd() + result = get_dataconnectors_result() + try: + result.success = self._handler.get_dataconnectors() + msg_type = TMessageType.REPLY + except TTransport.TTransportException: + raise + except MetaException as o1: + msg_type = TMessageType.REPLY + result.o1 = o1 + except TApplicationException as ex: + logging.exception("TApplication exception in handler") + msg_type = TMessageType.EXCEPTION + result = ex + except Exception: + logging.exception("Unexpected exception in handler") + msg_type = TMessageType.EXCEPTION + result = TApplicationException(TApplicationException.INTERNAL_ERROR, "Internal error") + oprot.writeMessageBegin("get_dataconnectors", msg_type, seqid) + result.write(oprot) + oprot.writeMessageEnd() + oprot.trans.flush() + + def process_alter_dataconnector(self, seqid, iprot, oprot): + args = alter_dataconnector_args() + args.read(iprot) + iprot.readMessageEnd() + result = alter_dataconnector_result() + try: + self._handler.alter_dataconnector(args.name, args.connector) + msg_type = TMessageType.REPLY + except TTransport.TTransportException: + raise + except MetaException as o1: + msg_type = TMessageType.REPLY + result.o1 = o1 + except NoSuchObjectException as o2: + msg_type = TMessageType.REPLY + result.o2 = o2 + except TApplicationException as ex: + logging.exception("TApplication exception in handler") + msg_type = TMessageType.EXCEPTION + result = ex + except Exception: + logging.exception("Unexpected exception in handler") + msg_type = TMessageType.EXCEPTION + result = TApplicationException(TApplicationException.INTERNAL_ERROR, "Internal error") + oprot.writeMessageBegin("alter_dataconnector", msg_type, seqid) + result.write(oprot) + oprot.writeMessageEnd() + oprot.trans.flush() + + def process_get_type(self, seqid, iprot, oprot): + args = get_type_args() + args.read(iprot) + iprot.readMessageEnd() + result = get_type_result() + try: + result.success = self._handler.get_type(args.name) + msg_type = TMessageType.REPLY + except TTransport.TTransportException: + raise + except MetaException as o1: + msg_type = TMessageType.REPLY + result.o1 = o1 + except NoSuchObjectException as o2: + msg_type = TMessageType.REPLY + result.o2 = o2 + except TApplicationException as ex: + logging.exception("TApplication exception in handler") + msg_type = TMessageType.EXCEPTION + result = ex + except Exception: + logging.exception("Unexpected exception in handler") + msg_type = TMessageType.EXCEPTION + result = TApplicationException(TApplicationException.INTERNAL_ERROR, "Internal error") + oprot.writeMessageBegin("get_type", msg_type, seqid) + result.write(oprot) + oprot.writeMessageEnd() + oprot.trans.flush() + + def process_create_type(self, seqid, iprot, oprot): + args = create_type_args() + args.read(iprot) + iprot.readMessageEnd() + result = create_type_result() + try: + result.success = self._handler.create_type(args.type) + msg_type = TMessageType.REPLY + except TTransport.TTransportException: + raise + except AlreadyExistsException as o1: + msg_type = TMessageType.REPLY + result.o1 = o1 + except InvalidObjectException as o2: + msg_type = TMessageType.REPLY + result.o2 = o2 + except MetaException as o3: + msg_type = TMessageType.REPLY + result.o3 = o3 + except TApplicationException as ex: + logging.exception("TApplication exception in handler") + msg_type = TMessageType.EXCEPTION + result = ex + except Exception: + logging.exception("Unexpected exception in handler") + msg_type = TMessageType.EXCEPTION + result = TApplicationException(TApplicationException.INTERNAL_ERROR, "Internal error") + oprot.writeMessageBegin("create_type", msg_type, seqid) + result.write(oprot) + oprot.writeMessageEnd() + oprot.trans.flush() + + def process_drop_type(self, seqid, iprot, oprot): + args = drop_type_args() + args.read(iprot) + iprot.readMessageEnd() + result = drop_type_result() + try: + result.success = self._handler.drop_type(args.type) + msg_type = TMessageType.REPLY + except TTransport.TTransportException: + raise + except MetaException as o1: + msg_type = TMessageType.REPLY + result.o1 = o1 + except NoSuchObjectException as o2: + msg_type = TMessageType.REPLY + result.o2 = o2 + except TApplicationException as ex: + logging.exception("TApplication exception in handler") + msg_type = TMessageType.EXCEPTION + result = ex + except Exception: + logging.exception("Unexpected exception in handler") + msg_type = TMessageType.EXCEPTION + result = TApplicationException(TApplicationException.INTERNAL_ERROR, "Internal error") + oprot.writeMessageBegin("drop_type", msg_type, seqid) + result.write(oprot) + oprot.writeMessageEnd() + oprot.trans.flush() + + def process_get_type_all(self, seqid, iprot, oprot): + args = get_type_all_args() + args.read(iprot) + iprot.readMessageEnd() + result = get_type_all_result() + try: + result.success = self._handler.get_type_all(args.name) + msg_type = TMessageType.REPLY + except TTransport.TTransportException: + raise + except MetaException as o2: + msg_type = TMessageType.REPLY + result.o2 = o2 + except TApplicationException as ex: + logging.exception("TApplication exception in handler") + msg_type = TMessageType.EXCEPTION + result = ex + except Exception: + logging.exception("Unexpected exception in handler") + msg_type = TMessageType.EXCEPTION + result = TApplicationException(TApplicationException.INTERNAL_ERROR, "Internal error") + oprot.writeMessageBegin("get_type_all", msg_type, seqid) + result.write(oprot) + oprot.writeMessageEnd() + oprot.trans.flush() + + def process_get_fields(self, seqid, iprot, oprot): + args = get_fields_args() + args.read(iprot) + iprot.readMessageEnd() + result = get_fields_result() + try: + result.success = self._handler.get_fields(args.db_name, args.table_name) + msg_type = TMessageType.REPLY + except TTransport.TTransportException: + raise + except MetaException as o1: + msg_type = TMessageType.REPLY + result.o1 = o1 + except UnknownTableException as o2: + msg_type = TMessageType.REPLY + result.o2 = o2 + except UnknownDBException as o3: + msg_type = TMessageType.REPLY + result.o3 = o3 + except TApplicationException as ex: + logging.exception("TApplication exception in handler") + msg_type = TMessageType.EXCEPTION + result = ex + except Exception: + logging.exception("Unexpected exception in handler") + msg_type = TMessageType.EXCEPTION + result = TApplicationException(TApplicationException.INTERNAL_ERROR, "Internal error") + oprot.writeMessageBegin("get_fields", msg_type, seqid) + result.write(oprot) + oprot.writeMessageEnd() + oprot.trans.flush() + + def process_get_fields_with_environment_context(self, seqid, iprot, oprot): + args = get_fields_with_environment_context_args() + args.read(iprot) + iprot.readMessageEnd() + result = get_fields_with_environment_context_result() + try: + result.success = self._handler.get_fields_with_environment_context( + args.db_name, args.table_name, args.environment_context + ) + msg_type = TMessageType.REPLY + except TTransport.TTransportException: + raise + except MetaException as o1: + msg_type = TMessageType.REPLY + result.o1 = o1 + except UnknownTableException as o2: + msg_type = TMessageType.REPLY + result.o2 = o2 + except UnknownDBException as o3: + msg_type = TMessageType.REPLY + result.o3 = o3 + except TApplicationException as ex: + logging.exception("TApplication exception in handler") + msg_type = TMessageType.EXCEPTION + result = ex + except Exception: + logging.exception("Unexpected exception in handler") + msg_type = TMessageType.EXCEPTION + result = TApplicationException(TApplicationException.INTERNAL_ERROR, "Internal error") + oprot.writeMessageBegin("get_fields_with_environment_context", msg_type, seqid) + result.write(oprot) + oprot.writeMessageEnd() + oprot.trans.flush() + + def process_get_fields_req(self, seqid, iprot, oprot): + args = get_fields_req_args() + args.read(iprot) + iprot.readMessageEnd() + result = get_fields_req_result() + try: + result.success = self._handler.get_fields_req(args.req) + msg_type = TMessageType.REPLY + except TTransport.TTransportException: + raise + except MetaException as o1: + msg_type = TMessageType.REPLY + result.o1 = o1 + except UnknownTableException as o2: + msg_type = TMessageType.REPLY + result.o2 = o2 + except UnknownDBException as o3: + msg_type = TMessageType.REPLY + result.o3 = o3 + except TApplicationException as ex: + logging.exception("TApplication exception in handler") + msg_type = TMessageType.EXCEPTION + result = ex + except Exception: + logging.exception("Unexpected exception in handler") + msg_type = TMessageType.EXCEPTION + result = TApplicationException(TApplicationException.INTERNAL_ERROR, "Internal error") + oprot.writeMessageBegin("get_fields_req", msg_type, seqid) + result.write(oprot) + oprot.writeMessageEnd() + oprot.trans.flush() + + def process_get_schema(self, seqid, iprot, oprot): + args = get_schema_args() + args.read(iprot) + iprot.readMessageEnd() + result = get_schema_result() + try: + result.success = self._handler.get_schema(args.db_name, args.table_name) + msg_type = TMessageType.REPLY + except TTransport.TTransportException: + raise + except MetaException as o1: + msg_type = TMessageType.REPLY + result.o1 = o1 + except UnknownTableException as o2: + msg_type = TMessageType.REPLY + result.o2 = o2 + except UnknownDBException as o3: + msg_type = TMessageType.REPLY + result.o3 = o3 + except TApplicationException as ex: + logging.exception("TApplication exception in handler") + msg_type = TMessageType.EXCEPTION + result = ex + except Exception: + logging.exception("Unexpected exception in handler") + msg_type = TMessageType.EXCEPTION + result = TApplicationException(TApplicationException.INTERNAL_ERROR, "Internal error") + oprot.writeMessageBegin("get_schema", msg_type, seqid) + result.write(oprot) + oprot.writeMessageEnd() + oprot.trans.flush() + + def process_get_schema_with_environment_context(self, seqid, iprot, oprot): + args = get_schema_with_environment_context_args() + args.read(iprot) + iprot.readMessageEnd() + result = get_schema_with_environment_context_result() + try: + result.success = self._handler.get_schema_with_environment_context( + args.db_name, args.table_name, args.environment_context + ) + msg_type = TMessageType.REPLY + except TTransport.TTransportException: + raise + except MetaException as o1: + msg_type = TMessageType.REPLY + result.o1 = o1 + except UnknownTableException as o2: + msg_type = TMessageType.REPLY + result.o2 = o2 + except UnknownDBException as o3: + msg_type = TMessageType.REPLY + result.o3 = o3 + except TApplicationException as ex: + logging.exception("TApplication exception in handler") + msg_type = TMessageType.EXCEPTION + result = ex + except Exception: + logging.exception("Unexpected exception in handler") + msg_type = TMessageType.EXCEPTION + result = TApplicationException(TApplicationException.INTERNAL_ERROR, "Internal error") + oprot.writeMessageBegin("get_schema_with_environment_context", msg_type, seqid) + result.write(oprot) + oprot.writeMessageEnd() + oprot.trans.flush() + + def process_get_schema_req(self, seqid, iprot, oprot): + args = get_schema_req_args() + args.read(iprot) + iprot.readMessageEnd() + result = get_schema_req_result() + try: + result.success = self._handler.get_schema_req(args.req) + msg_type = TMessageType.REPLY + except TTransport.TTransportException: + raise + except MetaException as o1: + msg_type = TMessageType.REPLY + result.o1 = o1 + except UnknownTableException as o2: + msg_type = TMessageType.REPLY + result.o2 = o2 + except UnknownDBException as o3: + msg_type = TMessageType.REPLY + result.o3 = o3 + except TApplicationException as ex: + logging.exception("TApplication exception in handler") + msg_type = TMessageType.EXCEPTION + result = ex + except Exception: + logging.exception("Unexpected exception in handler") + msg_type = TMessageType.EXCEPTION + result = TApplicationException(TApplicationException.INTERNAL_ERROR, "Internal error") + oprot.writeMessageBegin("get_schema_req", msg_type, seqid) + result.write(oprot) + oprot.writeMessageEnd() + oprot.trans.flush() + + def process_create_table(self, seqid, iprot, oprot): + args = create_table_args() + args.read(iprot) + iprot.readMessageEnd() + result = create_table_result() + try: + self._handler.create_table(args.tbl) + msg_type = TMessageType.REPLY + except TTransport.TTransportException: + raise + except AlreadyExistsException as o1: + msg_type = TMessageType.REPLY + result.o1 = o1 + except InvalidObjectException as o2: + msg_type = TMessageType.REPLY + result.o2 = o2 + except MetaException as o3: + msg_type = TMessageType.REPLY + result.o3 = o3 + except NoSuchObjectException as o4: + msg_type = TMessageType.REPLY + result.o4 = o4 + except TApplicationException as ex: + logging.exception("TApplication exception in handler") + msg_type = TMessageType.EXCEPTION + result = ex + except Exception: + logging.exception("Unexpected exception in handler") + msg_type = TMessageType.EXCEPTION + result = TApplicationException(TApplicationException.INTERNAL_ERROR, "Internal error") + oprot.writeMessageBegin("create_table", msg_type, seqid) + result.write(oprot) + oprot.writeMessageEnd() + oprot.trans.flush() + + def process_create_table_with_environment_context(self, seqid, iprot, oprot): + args = create_table_with_environment_context_args() + args.read(iprot) + iprot.readMessageEnd() + result = create_table_with_environment_context_result() + try: + self._handler.create_table_with_environment_context(args.tbl, args.environment_context) + msg_type = TMessageType.REPLY + except TTransport.TTransportException: + raise + except AlreadyExistsException as o1: + msg_type = TMessageType.REPLY + result.o1 = o1 + except InvalidObjectException as o2: + msg_type = TMessageType.REPLY + result.o2 = o2 + except MetaException as o3: + msg_type = TMessageType.REPLY + result.o3 = o3 + except NoSuchObjectException as o4: + msg_type = TMessageType.REPLY + result.o4 = o4 + except TApplicationException as ex: + logging.exception("TApplication exception in handler") + msg_type = TMessageType.EXCEPTION + result = ex + except Exception: + logging.exception("Unexpected exception in handler") + msg_type = TMessageType.EXCEPTION + result = TApplicationException(TApplicationException.INTERNAL_ERROR, "Internal error") + oprot.writeMessageBegin("create_table_with_environment_context", msg_type, seqid) + result.write(oprot) + oprot.writeMessageEnd() + oprot.trans.flush() + + def process_create_table_with_constraints(self, seqid, iprot, oprot): + args = create_table_with_constraints_args() + args.read(iprot) + iprot.readMessageEnd() + result = create_table_with_constraints_result() + try: + self._handler.create_table_with_constraints( + args.tbl, + args.primaryKeys, + args.foreignKeys, + args.uniqueConstraints, + args.notNullConstraints, + args.defaultConstraints, + args.checkConstraints, + ) + msg_type = TMessageType.REPLY + except TTransport.TTransportException: + raise + except AlreadyExistsException as o1: + msg_type = TMessageType.REPLY + result.o1 = o1 + except InvalidObjectException as o2: + msg_type = TMessageType.REPLY + result.o2 = o2 + except MetaException as o3: + msg_type = TMessageType.REPLY + result.o3 = o3 + except NoSuchObjectException as o4: + msg_type = TMessageType.REPLY + result.o4 = o4 + except TApplicationException as ex: + logging.exception("TApplication exception in handler") + msg_type = TMessageType.EXCEPTION + result = ex + except Exception: + logging.exception("Unexpected exception in handler") + msg_type = TMessageType.EXCEPTION + result = TApplicationException(TApplicationException.INTERNAL_ERROR, "Internal error") + oprot.writeMessageBegin("create_table_with_constraints", msg_type, seqid) + result.write(oprot) + oprot.writeMessageEnd() + oprot.trans.flush() + + def process_create_table_req(self, seqid, iprot, oprot): + args = create_table_req_args() + args.read(iprot) + iprot.readMessageEnd() + result = create_table_req_result() + try: + self._handler.create_table_req(args.request) + msg_type = TMessageType.REPLY + except TTransport.TTransportException: + raise + except AlreadyExistsException as o1: + msg_type = TMessageType.REPLY + result.o1 = o1 + except InvalidObjectException as o2: + msg_type = TMessageType.REPLY + result.o2 = o2 + except MetaException as o3: + msg_type = TMessageType.REPLY + result.o3 = o3 + except NoSuchObjectException as o4: + msg_type = TMessageType.REPLY + result.o4 = o4 + except TApplicationException as ex: + logging.exception("TApplication exception in handler") + msg_type = TMessageType.EXCEPTION + result = ex + except Exception: + logging.exception("Unexpected exception in handler") + msg_type = TMessageType.EXCEPTION + result = TApplicationException(TApplicationException.INTERNAL_ERROR, "Internal error") + oprot.writeMessageBegin("create_table_req", msg_type, seqid) + result.write(oprot) + oprot.writeMessageEnd() + oprot.trans.flush() + + def process_drop_constraint(self, seqid, iprot, oprot): + args = drop_constraint_args() + args.read(iprot) + iprot.readMessageEnd() + result = drop_constraint_result() + try: + self._handler.drop_constraint(args.req) + msg_type = TMessageType.REPLY + except TTransport.TTransportException: + raise + except NoSuchObjectException as o1: + msg_type = TMessageType.REPLY + result.o1 = o1 + except MetaException as o3: + msg_type = TMessageType.REPLY + result.o3 = o3 + except TApplicationException as ex: + logging.exception("TApplication exception in handler") + msg_type = TMessageType.EXCEPTION + result = ex + except Exception: + logging.exception("Unexpected exception in handler") + msg_type = TMessageType.EXCEPTION + result = TApplicationException(TApplicationException.INTERNAL_ERROR, "Internal error") + oprot.writeMessageBegin("drop_constraint", msg_type, seqid) + result.write(oprot) + oprot.writeMessageEnd() + oprot.trans.flush() + + def process_add_primary_key(self, seqid, iprot, oprot): + args = add_primary_key_args() + args.read(iprot) + iprot.readMessageEnd() + result = add_primary_key_result() + try: + self._handler.add_primary_key(args.req) + msg_type = TMessageType.REPLY + except TTransport.TTransportException: + raise + except NoSuchObjectException as o1: + msg_type = TMessageType.REPLY + result.o1 = o1 + except MetaException as o2: + msg_type = TMessageType.REPLY + result.o2 = o2 + except TApplicationException as ex: + logging.exception("TApplication exception in handler") + msg_type = TMessageType.EXCEPTION + result = ex + except Exception: + logging.exception("Unexpected exception in handler") + msg_type = TMessageType.EXCEPTION + result = TApplicationException(TApplicationException.INTERNAL_ERROR, "Internal error") + oprot.writeMessageBegin("add_primary_key", msg_type, seqid) + result.write(oprot) + oprot.writeMessageEnd() + oprot.trans.flush() + + def process_add_foreign_key(self, seqid, iprot, oprot): + args = add_foreign_key_args() + args.read(iprot) + iprot.readMessageEnd() + result = add_foreign_key_result() + try: + self._handler.add_foreign_key(args.req) + msg_type = TMessageType.REPLY + except TTransport.TTransportException: + raise + except NoSuchObjectException as o1: + msg_type = TMessageType.REPLY + result.o1 = o1 + except MetaException as o2: + msg_type = TMessageType.REPLY + result.o2 = o2 + except TApplicationException as ex: + logging.exception("TApplication exception in handler") + msg_type = TMessageType.EXCEPTION + result = ex + except Exception: + logging.exception("Unexpected exception in handler") + msg_type = TMessageType.EXCEPTION + result = TApplicationException(TApplicationException.INTERNAL_ERROR, "Internal error") + oprot.writeMessageBegin("add_foreign_key", msg_type, seqid) + result.write(oprot) + oprot.writeMessageEnd() + oprot.trans.flush() + + def process_add_unique_constraint(self, seqid, iprot, oprot): + args = add_unique_constraint_args() + args.read(iprot) + iprot.readMessageEnd() + result = add_unique_constraint_result() + try: + self._handler.add_unique_constraint(args.req) + msg_type = TMessageType.REPLY + except TTransport.TTransportException: + raise + except NoSuchObjectException as o1: + msg_type = TMessageType.REPLY + result.o1 = o1 + except MetaException as o2: + msg_type = TMessageType.REPLY + result.o2 = o2 + except TApplicationException as ex: + logging.exception("TApplication exception in handler") + msg_type = TMessageType.EXCEPTION + result = ex + except Exception: + logging.exception("Unexpected exception in handler") + msg_type = TMessageType.EXCEPTION + result = TApplicationException(TApplicationException.INTERNAL_ERROR, "Internal error") + oprot.writeMessageBegin("add_unique_constraint", msg_type, seqid) + result.write(oprot) + oprot.writeMessageEnd() + oprot.trans.flush() + + def process_add_not_null_constraint(self, seqid, iprot, oprot): + args = add_not_null_constraint_args() + args.read(iprot) + iprot.readMessageEnd() + result = add_not_null_constraint_result() + try: + self._handler.add_not_null_constraint(args.req) + msg_type = TMessageType.REPLY + except TTransport.TTransportException: + raise + except NoSuchObjectException as o1: + msg_type = TMessageType.REPLY + result.o1 = o1 + except MetaException as o2: + msg_type = TMessageType.REPLY + result.o2 = o2 + except TApplicationException as ex: + logging.exception("TApplication exception in handler") + msg_type = TMessageType.EXCEPTION + result = ex + except Exception: + logging.exception("Unexpected exception in handler") + msg_type = TMessageType.EXCEPTION + result = TApplicationException(TApplicationException.INTERNAL_ERROR, "Internal error") + oprot.writeMessageBegin("add_not_null_constraint", msg_type, seqid) + result.write(oprot) + oprot.writeMessageEnd() + oprot.trans.flush() + + def process_add_default_constraint(self, seqid, iprot, oprot): + args = add_default_constraint_args() + args.read(iprot) + iprot.readMessageEnd() + result = add_default_constraint_result() + try: + self._handler.add_default_constraint(args.req) + msg_type = TMessageType.REPLY + except TTransport.TTransportException: + raise + except NoSuchObjectException as o1: + msg_type = TMessageType.REPLY + result.o1 = o1 + except MetaException as o2: + msg_type = TMessageType.REPLY + result.o2 = o2 + except TApplicationException as ex: + logging.exception("TApplication exception in handler") + msg_type = TMessageType.EXCEPTION + result = ex + except Exception: + logging.exception("Unexpected exception in handler") + msg_type = TMessageType.EXCEPTION + result = TApplicationException(TApplicationException.INTERNAL_ERROR, "Internal error") + oprot.writeMessageBegin("add_default_constraint", msg_type, seqid) + result.write(oprot) + oprot.writeMessageEnd() + oprot.trans.flush() + + def process_add_check_constraint(self, seqid, iprot, oprot): + args = add_check_constraint_args() + args.read(iprot) + iprot.readMessageEnd() + result = add_check_constraint_result() + try: + self._handler.add_check_constraint(args.req) + msg_type = TMessageType.REPLY + except TTransport.TTransportException: + raise + except NoSuchObjectException as o1: + msg_type = TMessageType.REPLY + result.o1 = o1 + except MetaException as o2: + msg_type = TMessageType.REPLY + result.o2 = o2 + except TApplicationException as ex: + logging.exception("TApplication exception in handler") + msg_type = TMessageType.EXCEPTION + result = ex + except Exception: + logging.exception("Unexpected exception in handler") + msg_type = TMessageType.EXCEPTION + result = TApplicationException(TApplicationException.INTERNAL_ERROR, "Internal error") + oprot.writeMessageBegin("add_check_constraint", msg_type, seqid) + result.write(oprot) + oprot.writeMessageEnd() + oprot.trans.flush() + + def process_translate_table_dryrun(self, seqid, iprot, oprot): + args = translate_table_dryrun_args() + args.read(iprot) + iprot.readMessageEnd() + result = translate_table_dryrun_result() + try: + result.success = self._handler.translate_table_dryrun(args.request) + msg_type = TMessageType.REPLY + except TTransport.TTransportException: + raise + except AlreadyExistsException as o1: + msg_type = TMessageType.REPLY + result.o1 = o1 + except InvalidObjectException as o2: + msg_type = TMessageType.REPLY + result.o2 = o2 + except MetaException as o3: + msg_type = TMessageType.REPLY + result.o3 = o3 + except NoSuchObjectException as o4: + msg_type = TMessageType.REPLY + result.o4 = o4 + except TApplicationException as ex: + logging.exception("TApplication exception in handler") + msg_type = TMessageType.EXCEPTION + result = ex + except Exception: + logging.exception("Unexpected exception in handler") + msg_type = TMessageType.EXCEPTION + result = TApplicationException(TApplicationException.INTERNAL_ERROR, "Internal error") + oprot.writeMessageBegin("translate_table_dryrun", msg_type, seqid) + result.write(oprot) + oprot.writeMessageEnd() + oprot.trans.flush() + + def process_drop_table(self, seqid, iprot, oprot): + args = drop_table_args() + args.read(iprot) + iprot.readMessageEnd() + result = drop_table_result() + try: + self._handler.drop_table(args.dbname, args.name, args.deleteData) + msg_type = TMessageType.REPLY + except TTransport.TTransportException: + raise + except NoSuchObjectException as o1: + msg_type = TMessageType.REPLY + result.o1 = o1 + except MetaException as o3: + msg_type = TMessageType.REPLY + result.o3 = o3 + except TApplicationException as ex: + logging.exception("TApplication exception in handler") + msg_type = TMessageType.EXCEPTION + result = ex + except Exception: + logging.exception("Unexpected exception in handler") + msg_type = TMessageType.EXCEPTION + result = TApplicationException(TApplicationException.INTERNAL_ERROR, "Internal error") + oprot.writeMessageBegin("drop_table", msg_type, seqid) + result.write(oprot) + oprot.writeMessageEnd() + oprot.trans.flush() + + def process_drop_table_with_environment_context(self, seqid, iprot, oprot): + args = drop_table_with_environment_context_args() + args.read(iprot) + iprot.readMessageEnd() + result = drop_table_with_environment_context_result() + try: + self._handler.drop_table_with_environment_context(args.dbname, args.name, args.deleteData, args.environment_context) + msg_type = TMessageType.REPLY + except TTransport.TTransportException: + raise + except NoSuchObjectException as o1: + msg_type = TMessageType.REPLY + result.o1 = o1 + except MetaException as o3: + msg_type = TMessageType.REPLY + result.o3 = o3 + except TApplicationException as ex: + logging.exception("TApplication exception in handler") + msg_type = TMessageType.EXCEPTION + result = ex + except Exception: + logging.exception("Unexpected exception in handler") + msg_type = TMessageType.EXCEPTION + result = TApplicationException(TApplicationException.INTERNAL_ERROR, "Internal error") + oprot.writeMessageBegin("drop_table_with_environment_context", msg_type, seqid) + result.write(oprot) + oprot.writeMessageEnd() + oprot.trans.flush() + + def process_truncate_table(self, seqid, iprot, oprot): + args = truncate_table_args() + args.read(iprot) + iprot.readMessageEnd() + result = truncate_table_result() + try: + self._handler.truncate_table(args.dbName, args.tableName, args.partNames) + msg_type = TMessageType.REPLY + except TTransport.TTransportException: + raise + except MetaException as o1: + msg_type = TMessageType.REPLY + result.o1 = o1 + except TApplicationException as ex: + logging.exception("TApplication exception in handler") + msg_type = TMessageType.EXCEPTION + result = ex + except Exception: + logging.exception("Unexpected exception in handler") + msg_type = TMessageType.EXCEPTION + result = TApplicationException(TApplicationException.INTERNAL_ERROR, "Internal error") + oprot.writeMessageBegin("truncate_table", msg_type, seqid) + result.write(oprot) + oprot.writeMessageEnd() + oprot.trans.flush() + + def process_truncate_table_req(self, seqid, iprot, oprot): + args = truncate_table_req_args() + args.read(iprot) + iprot.readMessageEnd() + result = truncate_table_req_result() + try: + result.success = self._handler.truncate_table_req(args.req) + msg_type = TMessageType.REPLY + except TTransport.TTransportException: + raise + except MetaException as o1: + msg_type = TMessageType.REPLY + result.o1 = o1 + except TApplicationException as ex: + logging.exception("TApplication exception in handler") + msg_type = TMessageType.EXCEPTION + result = ex + except Exception: + logging.exception("Unexpected exception in handler") + msg_type = TMessageType.EXCEPTION + result = TApplicationException(TApplicationException.INTERNAL_ERROR, "Internal error") + oprot.writeMessageBegin("truncate_table_req", msg_type, seqid) + result.write(oprot) + oprot.writeMessageEnd() + oprot.trans.flush() + + def process_get_tables(self, seqid, iprot, oprot): + args = get_tables_args() + args.read(iprot) + iprot.readMessageEnd() + result = get_tables_result() + try: + result.success = self._handler.get_tables(args.db_name, args.pattern) + msg_type = TMessageType.REPLY + except TTransport.TTransportException: + raise + except MetaException as o1: + msg_type = TMessageType.REPLY + result.o1 = o1 + except TApplicationException as ex: + logging.exception("TApplication exception in handler") + msg_type = TMessageType.EXCEPTION + result = ex + except Exception: + logging.exception("Unexpected exception in handler") + msg_type = TMessageType.EXCEPTION + result = TApplicationException(TApplicationException.INTERNAL_ERROR, "Internal error") + oprot.writeMessageBegin("get_tables", msg_type, seqid) + result.write(oprot) + oprot.writeMessageEnd() + oprot.trans.flush() + + def process_get_tables_by_type(self, seqid, iprot, oprot): + args = get_tables_by_type_args() + args.read(iprot) + iprot.readMessageEnd() + result = get_tables_by_type_result() + try: + result.success = self._handler.get_tables_by_type(args.db_name, args.pattern, args.tableType) + msg_type = TMessageType.REPLY + except TTransport.TTransportException: + raise + except MetaException as o1: + msg_type = TMessageType.REPLY + result.o1 = o1 + except TApplicationException as ex: + logging.exception("TApplication exception in handler") + msg_type = TMessageType.EXCEPTION + result = ex + except Exception: + logging.exception("Unexpected exception in handler") + msg_type = TMessageType.EXCEPTION + result = TApplicationException(TApplicationException.INTERNAL_ERROR, "Internal error") + oprot.writeMessageBegin("get_tables_by_type", msg_type, seqid) + result.write(oprot) + oprot.writeMessageEnd() + oprot.trans.flush() + + def process_get_all_materialized_view_objects_for_rewriting(self, seqid, iprot, oprot): + args = get_all_materialized_view_objects_for_rewriting_args() + args.read(iprot) + iprot.readMessageEnd() + result = get_all_materialized_view_objects_for_rewriting_result() + try: + result.success = self._handler.get_all_materialized_view_objects_for_rewriting() + msg_type = TMessageType.REPLY + except TTransport.TTransportException: + raise + except MetaException as o1: + msg_type = TMessageType.REPLY + result.o1 = o1 + except TApplicationException as ex: + logging.exception("TApplication exception in handler") + msg_type = TMessageType.EXCEPTION + result = ex + except Exception: + logging.exception("Unexpected exception in handler") + msg_type = TMessageType.EXCEPTION + result = TApplicationException(TApplicationException.INTERNAL_ERROR, "Internal error") + oprot.writeMessageBegin("get_all_materialized_view_objects_for_rewriting", msg_type, seqid) + result.write(oprot) + oprot.writeMessageEnd() + oprot.trans.flush() + + def process_get_materialized_views_for_rewriting(self, seqid, iprot, oprot): + args = get_materialized_views_for_rewriting_args() + args.read(iprot) + iprot.readMessageEnd() + result = get_materialized_views_for_rewriting_result() + try: + result.success = self._handler.get_materialized_views_for_rewriting(args.db_name) + msg_type = TMessageType.REPLY + except TTransport.TTransportException: + raise + except MetaException as o1: + msg_type = TMessageType.REPLY + result.o1 = o1 + except TApplicationException as ex: + logging.exception("TApplication exception in handler") + msg_type = TMessageType.EXCEPTION + result = ex + except Exception: + logging.exception("Unexpected exception in handler") + msg_type = TMessageType.EXCEPTION + result = TApplicationException(TApplicationException.INTERNAL_ERROR, "Internal error") + oprot.writeMessageBegin("get_materialized_views_for_rewriting", msg_type, seqid) + result.write(oprot) + oprot.writeMessageEnd() + oprot.trans.flush() + + def process_get_table_meta(self, seqid, iprot, oprot): + args = get_table_meta_args() + args.read(iprot) + iprot.readMessageEnd() + result = get_table_meta_result() + try: + result.success = self._handler.get_table_meta(args.db_patterns, args.tbl_patterns, args.tbl_types) + msg_type = TMessageType.REPLY + except TTransport.TTransportException: + raise + except MetaException as o1: + msg_type = TMessageType.REPLY + result.o1 = o1 + except TApplicationException as ex: + logging.exception("TApplication exception in handler") + msg_type = TMessageType.EXCEPTION + result = ex + except Exception: + logging.exception("Unexpected exception in handler") + msg_type = TMessageType.EXCEPTION + result = TApplicationException(TApplicationException.INTERNAL_ERROR, "Internal error") + oprot.writeMessageBegin("get_table_meta", msg_type, seqid) + result.write(oprot) + oprot.writeMessageEnd() + oprot.trans.flush() + + def process_get_all_tables(self, seqid, iprot, oprot): + args = get_all_tables_args() + args.read(iprot) + iprot.readMessageEnd() + result = get_all_tables_result() + try: + result.success = self._handler.get_all_tables(args.db_name) + msg_type = TMessageType.REPLY + except TTransport.TTransportException: + raise + except MetaException as o1: + msg_type = TMessageType.REPLY + result.o1 = o1 + except TApplicationException as ex: + logging.exception("TApplication exception in handler") + msg_type = TMessageType.EXCEPTION + result = ex + except Exception: + logging.exception("Unexpected exception in handler") + msg_type = TMessageType.EXCEPTION + result = TApplicationException(TApplicationException.INTERNAL_ERROR, "Internal error") + oprot.writeMessageBegin("get_all_tables", msg_type, seqid) + result.write(oprot) + oprot.writeMessageEnd() + oprot.trans.flush() + + def process_get_table(self, seqid, iprot, oprot): + args = get_table_args() + args.read(iprot) + iprot.readMessageEnd() + result = get_table_result() + try: + result.success = self._handler.get_table(args.dbname, args.tbl_name) + msg_type = TMessageType.REPLY + except TTransport.TTransportException: + raise + except MetaException as o1: + msg_type = TMessageType.REPLY + result.o1 = o1 + except NoSuchObjectException as o2: + msg_type = TMessageType.REPLY + result.o2 = o2 + except TApplicationException as ex: + logging.exception("TApplication exception in handler") + msg_type = TMessageType.EXCEPTION + result = ex + except Exception: + logging.exception("Unexpected exception in handler") + msg_type = TMessageType.EXCEPTION + result = TApplicationException(TApplicationException.INTERNAL_ERROR, "Internal error") + oprot.writeMessageBegin("get_table", msg_type, seqid) + result.write(oprot) + oprot.writeMessageEnd() + oprot.trans.flush() + + def process_get_table_objects_by_name(self, seqid, iprot, oprot): + args = get_table_objects_by_name_args() + args.read(iprot) + iprot.readMessageEnd() + result = get_table_objects_by_name_result() + try: + result.success = self._handler.get_table_objects_by_name(args.dbname, args.tbl_names) + msg_type = TMessageType.REPLY + except TTransport.TTransportException: + raise + except TApplicationException as ex: + logging.exception("TApplication exception in handler") + msg_type = TMessageType.EXCEPTION + result = ex + except Exception: + logging.exception("Unexpected exception in handler") + msg_type = TMessageType.EXCEPTION + result = TApplicationException(TApplicationException.INTERNAL_ERROR, "Internal error") + oprot.writeMessageBegin("get_table_objects_by_name", msg_type, seqid) + result.write(oprot) + oprot.writeMessageEnd() + oprot.trans.flush() + + def process_get_tables_ext(self, seqid, iprot, oprot): + args = get_tables_ext_args() + args.read(iprot) + iprot.readMessageEnd() + result = get_tables_ext_result() + try: + result.success = self._handler.get_tables_ext(args.req) + msg_type = TMessageType.REPLY + except TTransport.TTransportException: + raise + except MetaException as o1: + msg_type = TMessageType.REPLY + result.o1 = o1 + except TApplicationException as ex: + logging.exception("TApplication exception in handler") + msg_type = TMessageType.EXCEPTION + result = ex + except Exception: + logging.exception("Unexpected exception in handler") + msg_type = TMessageType.EXCEPTION + result = TApplicationException(TApplicationException.INTERNAL_ERROR, "Internal error") + oprot.writeMessageBegin("get_tables_ext", msg_type, seqid) + result.write(oprot) + oprot.writeMessageEnd() + oprot.trans.flush() + + def process_get_table_req(self, seqid, iprot, oprot): + args = get_table_req_args() + args.read(iprot) + iprot.readMessageEnd() + result = get_table_req_result() + try: + result.success = self._handler.get_table_req(args.req) + msg_type = TMessageType.REPLY + except TTransport.TTransportException: + raise + except MetaException as o1: + msg_type = TMessageType.REPLY + result.o1 = o1 + except NoSuchObjectException as o2: + msg_type = TMessageType.REPLY + result.o2 = o2 + except TApplicationException as ex: + logging.exception("TApplication exception in handler") + msg_type = TMessageType.EXCEPTION + result = ex + except Exception: + logging.exception("Unexpected exception in handler") + msg_type = TMessageType.EXCEPTION + result = TApplicationException(TApplicationException.INTERNAL_ERROR, "Internal error") + oprot.writeMessageBegin("get_table_req", msg_type, seqid) + result.write(oprot) + oprot.writeMessageEnd() + oprot.trans.flush() + + def process_get_table_objects_by_name_req(self, seqid, iprot, oprot): + args = get_table_objects_by_name_req_args() + args.read(iprot) + iprot.readMessageEnd() + result = get_table_objects_by_name_req_result() + try: + result.success = self._handler.get_table_objects_by_name_req(args.req) + msg_type = TMessageType.REPLY + except TTransport.TTransportException: + raise + except MetaException as o1: + msg_type = TMessageType.REPLY + result.o1 = o1 + except InvalidOperationException as o2: + msg_type = TMessageType.REPLY + result.o2 = o2 + except UnknownDBException as o3: + msg_type = TMessageType.REPLY + result.o3 = o3 + except TApplicationException as ex: + logging.exception("TApplication exception in handler") + msg_type = TMessageType.EXCEPTION + result = ex + except Exception: + logging.exception("Unexpected exception in handler") + msg_type = TMessageType.EXCEPTION + result = TApplicationException(TApplicationException.INTERNAL_ERROR, "Internal error") + oprot.writeMessageBegin("get_table_objects_by_name_req", msg_type, seqid) + result.write(oprot) + oprot.writeMessageEnd() + oprot.trans.flush() + + def process_get_materialization_invalidation_info(self, seqid, iprot, oprot): + args = get_materialization_invalidation_info_args() + args.read(iprot) + iprot.readMessageEnd() + result = get_materialization_invalidation_info_result() + try: + result.success = self._handler.get_materialization_invalidation_info(args.creation_metadata, args.validTxnList) + msg_type = TMessageType.REPLY + except TTransport.TTransportException: + raise + except MetaException as o1: + msg_type = TMessageType.REPLY + result.o1 = o1 + except InvalidOperationException as o2: + msg_type = TMessageType.REPLY + result.o2 = o2 + except UnknownDBException as o3: + msg_type = TMessageType.REPLY + result.o3 = o3 + except TApplicationException as ex: + logging.exception("TApplication exception in handler") + msg_type = TMessageType.EXCEPTION + result = ex + except Exception: + logging.exception("Unexpected exception in handler") + msg_type = TMessageType.EXCEPTION + result = TApplicationException(TApplicationException.INTERNAL_ERROR, "Internal error") + oprot.writeMessageBegin("get_materialization_invalidation_info", msg_type, seqid) + result.write(oprot) + oprot.writeMessageEnd() + oprot.trans.flush() + + def process_update_creation_metadata(self, seqid, iprot, oprot): + args = update_creation_metadata_args() + args.read(iprot) + iprot.readMessageEnd() + result = update_creation_metadata_result() + try: + self._handler.update_creation_metadata(args.catName, args.dbname, args.tbl_name, args.creation_metadata) + msg_type = TMessageType.REPLY + except TTransport.TTransportException: + raise + except MetaException as o1: + msg_type = TMessageType.REPLY + result.o1 = o1 + except InvalidOperationException as o2: + msg_type = TMessageType.REPLY + result.o2 = o2 + except UnknownDBException as o3: + msg_type = TMessageType.REPLY + result.o3 = o3 + except TApplicationException as ex: + logging.exception("TApplication exception in handler") + msg_type = TMessageType.EXCEPTION + result = ex + except Exception: + logging.exception("Unexpected exception in handler") + msg_type = TMessageType.EXCEPTION + result = TApplicationException(TApplicationException.INTERNAL_ERROR, "Internal error") + oprot.writeMessageBegin("update_creation_metadata", msg_type, seqid) + result.write(oprot) + oprot.writeMessageEnd() + oprot.trans.flush() + + def process_get_table_names_by_filter(self, seqid, iprot, oprot): + args = get_table_names_by_filter_args() + args.read(iprot) + iprot.readMessageEnd() + result = get_table_names_by_filter_result() + try: + result.success = self._handler.get_table_names_by_filter(args.dbname, args.filter, args.max_tables) + msg_type = TMessageType.REPLY + except TTransport.TTransportException: + raise + except MetaException as o1: + msg_type = TMessageType.REPLY + result.o1 = o1 + except InvalidOperationException as o2: + msg_type = TMessageType.REPLY + result.o2 = o2 + except UnknownDBException as o3: + msg_type = TMessageType.REPLY + result.o3 = o3 + except TApplicationException as ex: + logging.exception("TApplication exception in handler") + msg_type = TMessageType.EXCEPTION + result = ex + except Exception: + logging.exception("Unexpected exception in handler") + msg_type = TMessageType.EXCEPTION + result = TApplicationException(TApplicationException.INTERNAL_ERROR, "Internal error") + oprot.writeMessageBegin("get_table_names_by_filter", msg_type, seqid) + result.write(oprot) + oprot.writeMessageEnd() + oprot.trans.flush() + + def process_alter_table(self, seqid, iprot, oprot): + args = alter_table_args() + args.read(iprot) + iprot.readMessageEnd() + result = alter_table_result() + try: + self._handler.alter_table(args.dbname, args.tbl_name, args.new_tbl) + msg_type = TMessageType.REPLY + except TTransport.TTransportException: + raise + except InvalidOperationException as o1: + msg_type = TMessageType.REPLY + result.o1 = o1 + except MetaException as o2: + msg_type = TMessageType.REPLY + result.o2 = o2 + except TApplicationException as ex: + logging.exception("TApplication exception in handler") + msg_type = TMessageType.EXCEPTION + result = ex + except Exception: + logging.exception("Unexpected exception in handler") + msg_type = TMessageType.EXCEPTION + result = TApplicationException(TApplicationException.INTERNAL_ERROR, "Internal error") + oprot.writeMessageBegin("alter_table", msg_type, seqid) + result.write(oprot) + oprot.writeMessageEnd() + oprot.trans.flush() + + def process_alter_table_with_environment_context(self, seqid, iprot, oprot): + args = alter_table_with_environment_context_args() + args.read(iprot) + iprot.readMessageEnd() + result = alter_table_with_environment_context_result() + try: + self._handler.alter_table_with_environment_context(args.dbname, args.tbl_name, args.new_tbl, args.environment_context) + msg_type = TMessageType.REPLY + except TTransport.TTransportException: + raise + except InvalidOperationException as o1: + msg_type = TMessageType.REPLY + result.o1 = o1 + except MetaException as o2: + msg_type = TMessageType.REPLY + result.o2 = o2 + except TApplicationException as ex: + logging.exception("TApplication exception in handler") + msg_type = TMessageType.EXCEPTION + result = ex + except Exception: + logging.exception("Unexpected exception in handler") + msg_type = TMessageType.EXCEPTION + result = TApplicationException(TApplicationException.INTERNAL_ERROR, "Internal error") + oprot.writeMessageBegin("alter_table_with_environment_context", msg_type, seqid) + result.write(oprot) + oprot.writeMessageEnd() + oprot.trans.flush() + + def process_alter_table_with_cascade(self, seqid, iprot, oprot): + args = alter_table_with_cascade_args() + args.read(iprot) + iprot.readMessageEnd() + result = alter_table_with_cascade_result() + try: + self._handler.alter_table_with_cascade(args.dbname, args.tbl_name, args.new_tbl, args.cascade) + msg_type = TMessageType.REPLY + except TTransport.TTransportException: + raise + except InvalidOperationException as o1: + msg_type = TMessageType.REPLY + result.o1 = o1 + except MetaException as o2: + msg_type = TMessageType.REPLY + result.o2 = o2 + except TApplicationException as ex: + logging.exception("TApplication exception in handler") + msg_type = TMessageType.EXCEPTION + result = ex + except Exception: + logging.exception("Unexpected exception in handler") + msg_type = TMessageType.EXCEPTION + result = TApplicationException(TApplicationException.INTERNAL_ERROR, "Internal error") + oprot.writeMessageBegin("alter_table_with_cascade", msg_type, seqid) + result.write(oprot) + oprot.writeMessageEnd() + oprot.trans.flush() + + def process_alter_table_req(self, seqid, iprot, oprot): + args = alter_table_req_args() + args.read(iprot) + iprot.readMessageEnd() + result = alter_table_req_result() + try: + result.success = self._handler.alter_table_req(args.req) + msg_type = TMessageType.REPLY + except TTransport.TTransportException: + raise + except InvalidOperationException as o1: + msg_type = TMessageType.REPLY + result.o1 = o1 + except MetaException as o2: + msg_type = TMessageType.REPLY + result.o2 = o2 + except TApplicationException as ex: + logging.exception("TApplication exception in handler") + msg_type = TMessageType.EXCEPTION + result = ex + except Exception: + logging.exception("Unexpected exception in handler") + msg_type = TMessageType.EXCEPTION + result = TApplicationException(TApplicationException.INTERNAL_ERROR, "Internal error") + oprot.writeMessageBegin("alter_table_req", msg_type, seqid) + result.write(oprot) + oprot.writeMessageEnd() + oprot.trans.flush() + + def process_add_partition(self, seqid, iprot, oprot): + args = add_partition_args() + args.read(iprot) + iprot.readMessageEnd() + result = add_partition_result() + try: + result.success = self._handler.add_partition(args.new_part) + msg_type = TMessageType.REPLY + except TTransport.TTransportException: + raise + except InvalidObjectException as o1: + msg_type = TMessageType.REPLY + result.o1 = o1 + except AlreadyExistsException as o2: + msg_type = TMessageType.REPLY + result.o2 = o2 + except MetaException as o3: + msg_type = TMessageType.REPLY + result.o3 = o3 + except TApplicationException as ex: + logging.exception("TApplication exception in handler") + msg_type = TMessageType.EXCEPTION + result = ex + except Exception: + logging.exception("Unexpected exception in handler") + msg_type = TMessageType.EXCEPTION + result = TApplicationException(TApplicationException.INTERNAL_ERROR, "Internal error") + oprot.writeMessageBegin("add_partition", msg_type, seqid) + result.write(oprot) + oprot.writeMessageEnd() + oprot.trans.flush() + + def process_add_partition_with_environment_context(self, seqid, iprot, oprot): + args = add_partition_with_environment_context_args() + args.read(iprot) + iprot.readMessageEnd() + result = add_partition_with_environment_context_result() + try: + result.success = self._handler.add_partition_with_environment_context(args.new_part, args.environment_context) + msg_type = TMessageType.REPLY + except TTransport.TTransportException: + raise + except InvalidObjectException as o1: + msg_type = TMessageType.REPLY + result.o1 = o1 + except AlreadyExistsException as o2: + msg_type = TMessageType.REPLY + result.o2 = o2 + except MetaException as o3: + msg_type = TMessageType.REPLY + result.o3 = o3 + except TApplicationException as ex: + logging.exception("TApplication exception in handler") + msg_type = TMessageType.EXCEPTION + result = ex + except Exception: + logging.exception("Unexpected exception in handler") + msg_type = TMessageType.EXCEPTION + result = TApplicationException(TApplicationException.INTERNAL_ERROR, "Internal error") + oprot.writeMessageBegin("add_partition_with_environment_context", msg_type, seqid) + result.write(oprot) + oprot.writeMessageEnd() + oprot.trans.flush() + + def process_add_partitions(self, seqid, iprot, oprot): + args = add_partitions_args() + args.read(iprot) + iprot.readMessageEnd() + result = add_partitions_result() + try: + result.success = self._handler.add_partitions(args.new_parts) + msg_type = TMessageType.REPLY + except TTransport.TTransportException: + raise + except InvalidObjectException as o1: + msg_type = TMessageType.REPLY + result.o1 = o1 + except AlreadyExistsException as o2: + msg_type = TMessageType.REPLY + result.o2 = o2 + except MetaException as o3: + msg_type = TMessageType.REPLY + result.o3 = o3 + except TApplicationException as ex: + logging.exception("TApplication exception in handler") + msg_type = TMessageType.EXCEPTION + result = ex + except Exception: + logging.exception("Unexpected exception in handler") + msg_type = TMessageType.EXCEPTION + result = TApplicationException(TApplicationException.INTERNAL_ERROR, "Internal error") + oprot.writeMessageBegin("add_partitions", msg_type, seqid) + result.write(oprot) + oprot.writeMessageEnd() + oprot.trans.flush() + + def process_add_partitions_pspec(self, seqid, iprot, oprot): + args = add_partitions_pspec_args() + args.read(iprot) + iprot.readMessageEnd() + result = add_partitions_pspec_result() + try: + result.success = self._handler.add_partitions_pspec(args.new_parts) + msg_type = TMessageType.REPLY + except TTransport.TTransportException: + raise + except InvalidObjectException as o1: + msg_type = TMessageType.REPLY + result.o1 = o1 + except AlreadyExistsException as o2: + msg_type = TMessageType.REPLY + result.o2 = o2 + except MetaException as o3: + msg_type = TMessageType.REPLY + result.o3 = o3 + except TApplicationException as ex: + logging.exception("TApplication exception in handler") + msg_type = TMessageType.EXCEPTION + result = ex + except Exception: + logging.exception("Unexpected exception in handler") + msg_type = TMessageType.EXCEPTION + result = TApplicationException(TApplicationException.INTERNAL_ERROR, "Internal error") + oprot.writeMessageBegin("add_partitions_pspec", msg_type, seqid) + result.write(oprot) + oprot.writeMessageEnd() + oprot.trans.flush() + + def process_append_partition(self, seqid, iprot, oprot): + args = append_partition_args() + args.read(iprot) + iprot.readMessageEnd() + result = append_partition_result() + try: + result.success = self._handler.append_partition(args.db_name, args.tbl_name, args.part_vals) + msg_type = TMessageType.REPLY + except TTransport.TTransportException: + raise + except InvalidObjectException as o1: + msg_type = TMessageType.REPLY + result.o1 = o1 + except AlreadyExistsException as o2: + msg_type = TMessageType.REPLY + result.o2 = o2 + except MetaException as o3: + msg_type = TMessageType.REPLY + result.o3 = o3 + except TApplicationException as ex: + logging.exception("TApplication exception in handler") + msg_type = TMessageType.EXCEPTION + result = ex + except Exception: + logging.exception("Unexpected exception in handler") + msg_type = TMessageType.EXCEPTION + result = TApplicationException(TApplicationException.INTERNAL_ERROR, "Internal error") + oprot.writeMessageBegin("append_partition", msg_type, seqid) + result.write(oprot) + oprot.writeMessageEnd() + oprot.trans.flush() + + def process_add_partitions_req(self, seqid, iprot, oprot): + args = add_partitions_req_args() + args.read(iprot) + iprot.readMessageEnd() + result = add_partitions_req_result() + try: + result.success = self._handler.add_partitions_req(args.request) + msg_type = TMessageType.REPLY + except TTransport.TTransportException: + raise + except InvalidObjectException as o1: + msg_type = TMessageType.REPLY + result.o1 = o1 + except AlreadyExistsException as o2: + msg_type = TMessageType.REPLY + result.o2 = o2 + except MetaException as o3: + msg_type = TMessageType.REPLY + result.o3 = o3 + except TApplicationException as ex: + logging.exception("TApplication exception in handler") + msg_type = TMessageType.EXCEPTION + result = ex + except Exception: + logging.exception("Unexpected exception in handler") + msg_type = TMessageType.EXCEPTION + result = TApplicationException(TApplicationException.INTERNAL_ERROR, "Internal error") + oprot.writeMessageBegin("add_partitions_req", msg_type, seqid) + result.write(oprot) + oprot.writeMessageEnd() + oprot.trans.flush() + + def process_append_partition_with_environment_context(self, seqid, iprot, oprot): + args = append_partition_with_environment_context_args() + args.read(iprot) + iprot.readMessageEnd() + result = append_partition_with_environment_context_result() + try: + result.success = self._handler.append_partition_with_environment_context( + args.db_name, args.tbl_name, args.part_vals, args.environment_context + ) + msg_type = TMessageType.REPLY + except TTransport.TTransportException: + raise + except InvalidObjectException as o1: + msg_type = TMessageType.REPLY + result.o1 = o1 + except AlreadyExistsException as o2: + msg_type = TMessageType.REPLY + result.o2 = o2 + except MetaException as o3: + msg_type = TMessageType.REPLY + result.o3 = o3 + except TApplicationException as ex: + logging.exception("TApplication exception in handler") + msg_type = TMessageType.EXCEPTION + result = ex + except Exception: + logging.exception("Unexpected exception in handler") + msg_type = TMessageType.EXCEPTION + result = TApplicationException(TApplicationException.INTERNAL_ERROR, "Internal error") + oprot.writeMessageBegin("append_partition_with_environment_context", msg_type, seqid) + result.write(oprot) + oprot.writeMessageEnd() + oprot.trans.flush() + + def process_append_partition_by_name(self, seqid, iprot, oprot): + args = append_partition_by_name_args() + args.read(iprot) + iprot.readMessageEnd() + result = append_partition_by_name_result() + try: + result.success = self._handler.append_partition_by_name(args.db_name, args.tbl_name, args.part_name) + msg_type = TMessageType.REPLY + except TTransport.TTransportException: + raise + except InvalidObjectException as o1: + msg_type = TMessageType.REPLY + result.o1 = o1 + except AlreadyExistsException as o2: + msg_type = TMessageType.REPLY + result.o2 = o2 + except MetaException as o3: + msg_type = TMessageType.REPLY + result.o3 = o3 + except TApplicationException as ex: + logging.exception("TApplication exception in handler") + msg_type = TMessageType.EXCEPTION + result = ex + except Exception: + logging.exception("Unexpected exception in handler") + msg_type = TMessageType.EXCEPTION + result = TApplicationException(TApplicationException.INTERNAL_ERROR, "Internal error") + oprot.writeMessageBegin("append_partition_by_name", msg_type, seqid) + result.write(oprot) + oprot.writeMessageEnd() + oprot.trans.flush() + + def process_append_partition_by_name_with_environment_context(self, seqid, iprot, oprot): + args = append_partition_by_name_with_environment_context_args() + args.read(iprot) + iprot.readMessageEnd() + result = append_partition_by_name_with_environment_context_result() + try: + result.success = self._handler.append_partition_by_name_with_environment_context( + args.db_name, args.tbl_name, args.part_name, args.environment_context + ) + msg_type = TMessageType.REPLY + except TTransport.TTransportException: + raise + except InvalidObjectException as o1: + msg_type = TMessageType.REPLY + result.o1 = o1 + except AlreadyExistsException as o2: + msg_type = TMessageType.REPLY + result.o2 = o2 + except MetaException as o3: + msg_type = TMessageType.REPLY + result.o3 = o3 + except TApplicationException as ex: + logging.exception("TApplication exception in handler") + msg_type = TMessageType.EXCEPTION + result = ex + except Exception: + logging.exception("Unexpected exception in handler") + msg_type = TMessageType.EXCEPTION + result = TApplicationException(TApplicationException.INTERNAL_ERROR, "Internal error") + oprot.writeMessageBegin("append_partition_by_name_with_environment_context", msg_type, seqid) + result.write(oprot) + oprot.writeMessageEnd() + oprot.trans.flush() + + def process_drop_partition(self, seqid, iprot, oprot): + args = drop_partition_args() + args.read(iprot) + iprot.readMessageEnd() + result = drop_partition_result() + try: + result.success = self._handler.drop_partition(args.db_name, args.tbl_name, args.part_vals, args.deleteData) + msg_type = TMessageType.REPLY + except TTransport.TTransportException: + raise + except NoSuchObjectException as o1: + msg_type = TMessageType.REPLY + result.o1 = o1 + except MetaException as o2: + msg_type = TMessageType.REPLY + result.o2 = o2 + except TApplicationException as ex: + logging.exception("TApplication exception in handler") + msg_type = TMessageType.EXCEPTION + result = ex + except Exception: + logging.exception("Unexpected exception in handler") + msg_type = TMessageType.EXCEPTION + result = TApplicationException(TApplicationException.INTERNAL_ERROR, "Internal error") + oprot.writeMessageBegin("drop_partition", msg_type, seqid) + result.write(oprot) + oprot.writeMessageEnd() + oprot.trans.flush() + + def process_drop_partition_with_environment_context(self, seqid, iprot, oprot): + args = drop_partition_with_environment_context_args() + args.read(iprot) + iprot.readMessageEnd() + result = drop_partition_with_environment_context_result() + try: + result.success = self._handler.drop_partition_with_environment_context( + args.db_name, args.tbl_name, args.part_vals, args.deleteData, args.environment_context + ) + msg_type = TMessageType.REPLY + except TTransport.TTransportException: + raise + except NoSuchObjectException as o1: + msg_type = TMessageType.REPLY + result.o1 = o1 + except MetaException as o2: + msg_type = TMessageType.REPLY + result.o2 = o2 + except TApplicationException as ex: + logging.exception("TApplication exception in handler") + msg_type = TMessageType.EXCEPTION + result = ex + except Exception: + logging.exception("Unexpected exception in handler") + msg_type = TMessageType.EXCEPTION + result = TApplicationException(TApplicationException.INTERNAL_ERROR, "Internal error") + oprot.writeMessageBegin("drop_partition_with_environment_context", msg_type, seqid) + result.write(oprot) + oprot.writeMessageEnd() + oprot.trans.flush() + + def process_drop_partition_by_name(self, seqid, iprot, oprot): + args = drop_partition_by_name_args() + args.read(iprot) + iprot.readMessageEnd() + result = drop_partition_by_name_result() + try: + result.success = self._handler.drop_partition_by_name(args.db_name, args.tbl_name, args.part_name, args.deleteData) + msg_type = TMessageType.REPLY + except TTransport.TTransportException: + raise + except NoSuchObjectException as o1: + msg_type = TMessageType.REPLY + result.o1 = o1 + except MetaException as o2: + msg_type = TMessageType.REPLY + result.o2 = o2 + except TApplicationException as ex: + logging.exception("TApplication exception in handler") + msg_type = TMessageType.EXCEPTION + result = ex + except Exception: + logging.exception("Unexpected exception in handler") + msg_type = TMessageType.EXCEPTION + result = TApplicationException(TApplicationException.INTERNAL_ERROR, "Internal error") + oprot.writeMessageBegin("drop_partition_by_name", msg_type, seqid) + result.write(oprot) + oprot.writeMessageEnd() + oprot.trans.flush() + + def process_drop_partition_by_name_with_environment_context(self, seqid, iprot, oprot): + args = drop_partition_by_name_with_environment_context_args() + args.read(iprot) + iprot.readMessageEnd() + result = drop_partition_by_name_with_environment_context_result() + try: + result.success = self._handler.drop_partition_by_name_with_environment_context( + args.db_name, args.tbl_name, args.part_name, args.deleteData, args.environment_context + ) + msg_type = TMessageType.REPLY + except TTransport.TTransportException: + raise + except NoSuchObjectException as o1: + msg_type = TMessageType.REPLY + result.o1 = o1 + except MetaException as o2: + msg_type = TMessageType.REPLY + result.o2 = o2 + except TApplicationException as ex: + logging.exception("TApplication exception in handler") + msg_type = TMessageType.EXCEPTION + result = ex + except Exception: + logging.exception("Unexpected exception in handler") + msg_type = TMessageType.EXCEPTION + result = TApplicationException(TApplicationException.INTERNAL_ERROR, "Internal error") + oprot.writeMessageBegin("drop_partition_by_name_with_environment_context", msg_type, seqid) + result.write(oprot) + oprot.writeMessageEnd() + oprot.trans.flush() + + def process_drop_partitions_req(self, seqid, iprot, oprot): + args = drop_partitions_req_args() + args.read(iprot) + iprot.readMessageEnd() + result = drop_partitions_req_result() + try: + result.success = self._handler.drop_partitions_req(args.req) + msg_type = TMessageType.REPLY + except TTransport.TTransportException: + raise + except NoSuchObjectException as o1: + msg_type = TMessageType.REPLY + result.o1 = o1 + except MetaException as o2: + msg_type = TMessageType.REPLY + result.o2 = o2 + except TApplicationException as ex: + logging.exception("TApplication exception in handler") + msg_type = TMessageType.EXCEPTION + result = ex + except Exception: + logging.exception("Unexpected exception in handler") + msg_type = TMessageType.EXCEPTION + result = TApplicationException(TApplicationException.INTERNAL_ERROR, "Internal error") + oprot.writeMessageBegin("drop_partitions_req", msg_type, seqid) + result.write(oprot) + oprot.writeMessageEnd() + oprot.trans.flush() + + def process_get_partition(self, seqid, iprot, oprot): + args = get_partition_args() + args.read(iprot) + iprot.readMessageEnd() + result = get_partition_result() + try: + result.success = self._handler.get_partition(args.db_name, args.tbl_name, args.part_vals) + msg_type = TMessageType.REPLY + except TTransport.TTransportException: + raise + except MetaException as o1: + msg_type = TMessageType.REPLY + result.o1 = o1 + except NoSuchObjectException as o2: + msg_type = TMessageType.REPLY + result.o2 = o2 + except TApplicationException as ex: + logging.exception("TApplication exception in handler") + msg_type = TMessageType.EXCEPTION + result = ex + except Exception: + logging.exception("Unexpected exception in handler") + msg_type = TMessageType.EXCEPTION + result = TApplicationException(TApplicationException.INTERNAL_ERROR, "Internal error") + oprot.writeMessageBegin("get_partition", msg_type, seqid) + result.write(oprot) + oprot.writeMessageEnd() + oprot.trans.flush() + + def process_get_partition_req(self, seqid, iprot, oprot): + args = get_partition_req_args() + args.read(iprot) + iprot.readMessageEnd() + result = get_partition_req_result() + try: + result.success = self._handler.get_partition_req(args.req) + msg_type = TMessageType.REPLY + except TTransport.TTransportException: + raise + except MetaException as o1: + msg_type = TMessageType.REPLY + result.o1 = o1 + except NoSuchObjectException as o2: + msg_type = TMessageType.REPLY + result.o2 = o2 + except TApplicationException as ex: + logging.exception("TApplication exception in handler") + msg_type = TMessageType.EXCEPTION + result = ex + except Exception: + logging.exception("Unexpected exception in handler") + msg_type = TMessageType.EXCEPTION + result = TApplicationException(TApplicationException.INTERNAL_ERROR, "Internal error") + oprot.writeMessageBegin("get_partition_req", msg_type, seqid) + result.write(oprot) + oprot.writeMessageEnd() + oprot.trans.flush() + + def process_exchange_partition(self, seqid, iprot, oprot): + args = exchange_partition_args() + args.read(iprot) + iprot.readMessageEnd() + result = exchange_partition_result() + try: + result.success = self._handler.exchange_partition( + args.partitionSpecs, args.source_db, args.source_table_name, args.dest_db, args.dest_table_name + ) + msg_type = TMessageType.REPLY + except TTransport.TTransportException: + raise + except MetaException as o1: + msg_type = TMessageType.REPLY + result.o1 = o1 + except NoSuchObjectException as o2: + msg_type = TMessageType.REPLY + result.o2 = o2 + except InvalidObjectException as o3: + msg_type = TMessageType.REPLY + result.o3 = o3 + except InvalidInputException as o4: + msg_type = TMessageType.REPLY + result.o4 = o4 + except TApplicationException as ex: + logging.exception("TApplication exception in handler") + msg_type = TMessageType.EXCEPTION + result = ex + except Exception: + logging.exception("Unexpected exception in handler") + msg_type = TMessageType.EXCEPTION + result = TApplicationException(TApplicationException.INTERNAL_ERROR, "Internal error") + oprot.writeMessageBegin("exchange_partition", msg_type, seqid) + result.write(oprot) + oprot.writeMessageEnd() + oprot.trans.flush() + + def process_exchange_partitions(self, seqid, iprot, oprot): + args = exchange_partitions_args() + args.read(iprot) + iprot.readMessageEnd() + result = exchange_partitions_result() + try: + result.success = self._handler.exchange_partitions( + args.partitionSpecs, args.source_db, args.source_table_name, args.dest_db, args.dest_table_name + ) + msg_type = TMessageType.REPLY + except TTransport.TTransportException: + raise + except MetaException as o1: + msg_type = TMessageType.REPLY + result.o1 = o1 + except NoSuchObjectException as o2: + msg_type = TMessageType.REPLY + result.o2 = o2 + except InvalidObjectException as o3: + msg_type = TMessageType.REPLY + result.o3 = o3 + except InvalidInputException as o4: + msg_type = TMessageType.REPLY + result.o4 = o4 + except TApplicationException as ex: + logging.exception("TApplication exception in handler") + msg_type = TMessageType.EXCEPTION + result = ex + except Exception: + logging.exception("Unexpected exception in handler") + msg_type = TMessageType.EXCEPTION + result = TApplicationException(TApplicationException.INTERNAL_ERROR, "Internal error") + oprot.writeMessageBegin("exchange_partitions", msg_type, seqid) + result.write(oprot) + oprot.writeMessageEnd() + oprot.trans.flush() + + def process_get_partition_with_auth(self, seqid, iprot, oprot): + args = get_partition_with_auth_args() + args.read(iprot) + iprot.readMessageEnd() + result = get_partition_with_auth_result() + try: + result.success = self._handler.get_partition_with_auth( + args.db_name, args.tbl_name, args.part_vals, args.user_name, args.group_names + ) + msg_type = TMessageType.REPLY + except TTransport.TTransportException: + raise + except MetaException as o1: + msg_type = TMessageType.REPLY + result.o1 = o1 + except NoSuchObjectException as o2: + msg_type = TMessageType.REPLY + result.o2 = o2 + except TApplicationException as ex: + logging.exception("TApplication exception in handler") + msg_type = TMessageType.EXCEPTION + result = ex + except Exception: + logging.exception("Unexpected exception in handler") + msg_type = TMessageType.EXCEPTION + result = TApplicationException(TApplicationException.INTERNAL_ERROR, "Internal error") + oprot.writeMessageBegin("get_partition_with_auth", msg_type, seqid) + result.write(oprot) + oprot.writeMessageEnd() + oprot.trans.flush() + + def process_get_partition_by_name(self, seqid, iprot, oprot): + args = get_partition_by_name_args() + args.read(iprot) + iprot.readMessageEnd() + result = get_partition_by_name_result() + try: + result.success = self._handler.get_partition_by_name(args.db_name, args.tbl_name, args.part_name) + msg_type = TMessageType.REPLY + except TTransport.TTransportException: + raise + except MetaException as o1: + msg_type = TMessageType.REPLY + result.o1 = o1 + except NoSuchObjectException as o2: + msg_type = TMessageType.REPLY + result.o2 = o2 + except TApplicationException as ex: + logging.exception("TApplication exception in handler") + msg_type = TMessageType.EXCEPTION + result = ex + except Exception: + logging.exception("Unexpected exception in handler") + msg_type = TMessageType.EXCEPTION + result = TApplicationException(TApplicationException.INTERNAL_ERROR, "Internal error") + oprot.writeMessageBegin("get_partition_by_name", msg_type, seqid) + result.write(oprot) + oprot.writeMessageEnd() + oprot.trans.flush() + + def process_get_partitions(self, seqid, iprot, oprot): + args = get_partitions_args() + args.read(iprot) + iprot.readMessageEnd() + result = get_partitions_result() + try: + result.success = self._handler.get_partitions(args.db_name, args.tbl_name, args.max_parts) + msg_type = TMessageType.REPLY + except TTransport.TTransportException: + raise + except NoSuchObjectException as o1: + msg_type = TMessageType.REPLY + result.o1 = o1 + except MetaException as o2: + msg_type = TMessageType.REPLY + result.o2 = o2 + except TApplicationException as ex: + logging.exception("TApplication exception in handler") + msg_type = TMessageType.EXCEPTION + result = ex + except Exception: + logging.exception("Unexpected exception in handler") + msg_type = TMessageType.EXCEPTION + result = TApplicationException(TApplicationException.INTERNAL_ERROR, "Internal error") + oprot.writeMessageBegin("get_partitions", msg_type, seqid) + result.write(oprot) + oprot.writeMessageEnd() + oprot.trans.flush() + + def process_get_partitions_req(self, seqid, iprot, oprot): + args = get_partitions_req_args() + args.read(iprot) + iprot.readMessageEnd() + result = get_partitions_req_result() + try: + result.success = self._handler.get_partitions_req(args.req) + msg_type = TMessageType.REPLY + except TTransport.TTransportException: + raise + except NoSuchObjectException as o1: + msg_type = TMessageType.REPLY + result.o1 = o1 + except MetaException as o2: + msg_type = TMessageType.REPLY + result.o2 = o2 + except TApplicationException as ex: + logging.exception("TApplication exception in handler") + msg_type = TMessageType.EXCEPTION + result = ex + except Exception: + logging.exception("Unexpected exception in handler") + msg_type = TMessageType.EXCEPTION + result = TApplicationException(TApplicationException.INTERNAL_ERROR, "Internal error") + oprot.writeMessageBegin("get_partitions_req", msg_type, seqid) + result.write(oprot) + oprot.writeMessageEnd() + oprot.trans.flush() + + def process_get_partitions_with_auth(self, seqid, iprot, oprot): + args = get_partitions_with_auth_args() + args.read(iprot) + iprot.readMessageEnd() + result = get_partitions_with_auth_result() + try: + result.success = self._handler.get_partitions_with_auth( + args.db_name, args.tbl_name, args.max_parts, args.user_name, args.group_names + ) + msg_type = TMessageType.REPLY + except TTransport.TTransportException: + raise + except NoSuchObjectException as o1: + msg_type = TMessageType.REPLY + result.o1 = o1 + except MetaException as o2: + msg_type = TMessageType.REPLY + result.o2 = o2 + except TApplicationException as ex: + logging.exception("TApplication exception in handler") + msg_type = TMessageType.EXCEPTION + result = ex + except Exception: + logging.exception("Unexpected exception in handler") + msg_type = TMessageType.EXCEPTION + result = TApplicationException(TApplicationException.INTERNAL_ERROR, "Internal error") + oprot.writeMessageBegin("get_partitions_with_auth", msg_type, seqid) + result.write(oprot) + oprot.writeMessageEnd() + oprot.trans.flush() + + def process_get_partitions_pspec(self, seqid, iprot, oprot): + args = get_partitions_pspec_args() + args.read(iprot) + iprot.readMessageEnd() + result = get_partitions_pspec_result() + try: + result.success = self._handler.get_partitions_pspec(args.db_name, args.tbl_name, args.max_parts) + msg_type = TMessageType.REPLY + except TTransport.TTransportException: + raise + except NoSuchObjectException as o1: + msg_type = TMessageType.REPLY + result.o1 = o1 + except MetaException as o2: + msg_type = TMessageType.REPLY + result.o2 = o2 + except TApplicationException as ex: + logging.exception("TApplication exception in handler") + msg_type = TMessageType.EXCEPTION + result = ex + except Exception: + logging.exception("Unexpected exception in handler") + msg_type = TMessageType.EXCEPTION + result = TApplicationException(TApplicationException.INTERNAL_ERROR, "Internal error") + oprot.writeMessageBegin("get_partitions_pspec", msg_type, seqid) + result.write(oprot) + oprot.writeMessageEnd() + oprot.trans.flush() + + def process_get_partition_names(self, seqid, iprot, oprot): + args = get_partition_names_args() + args.read(iprot) + iprot.readMessageEnd() + result = get_partition_names_result() + try: + result.success = self._handler.get_partition_names(args.db_name, args.tbl_name, args.max_parts) + msg_type = TMessageType.REPLY + except TTransport.TTransportException: + raise + except NoSuchObjectException as o1: + msg_type = TMessageType.REPLY + result.o1 = o1 + except MetaException as o2: + msg_type = TMessageType.REPLY + result.o2 = o2 + except TApplicationException as ex: + logging.exception("TApplication exception in handler") + msg_type = TMessageType.EXCEPTION + result = ex + except Exception: + logging.exception("Unexpected exception in handler") + msg_type = TMessageType.EXCEPTION + result = TApplicationException(TApplicationException.INTERNAL_ERROR, "Internal error") + oprot.writeMessageBegin("get_partition_names", msg_type, seqid) + result.write(oprot) + oprot.writeMessageEnd() + oprot.trans.flush() + + def process_get_partition_values(self, seqid, iprot, oprot): + args = get_partition_values_args() + args.read(iprot) + iprot.readMessageEnd() + result = get_partition_values_result() + try: + result.success = self._handler.get_partition_values(args.request) + msg_type = TMessageType.REPLY + except TTransport.TTransportException: + raise + except MetaException as o1: + msg_type = TMessageType.REPLY + result.o1 = o1 + except NoSuchObjectException as o2: + msg_type = TMessageType.REPLY + result.o2 = o2 + except TApplicationException as ex: + logging.exception("TApplication exception in handler") + msg_type = TMessageType.EXCEPTION + result = ex + except Exception: + logging.exception("Unexpected exception in handler") + msg_type = TMessageType.EXCEPTION + result = TApplicationException(TApplicationException.INTERNAL_ERROR, "Internal error") + oprot.writeMessageBegin("get_partition_values", msg_type, seqid) + result.write(oprot) + oprot.writeMessageEnd() + oprot.trans.flush() + + def process_get_partitions_ps(self, seqid, iprot, oprot): + args = get_partitions_ps_args() + args.read(iprot) + iprot.readMessageEnd() + result = get_partitions_ps_result() + try: + result.success = self._handler.get_partitions_ps(args.db_name, args.tbl_name, args.part_vals, args.max_parts) + msg_type = TMessageType.REPLY + except TTransport.TTransportException: + raise + except MetaException as o1: + msg_type = TMessageType.REPLY + result.o1 = o1 + except NoSuchObjectException as o2: + msg_type = TMessageType.REPLY + result.o2 = o2 + except TApplicationException as ex: + logging.exception("TApplication exception in handler") + msg_type = TMessageType.EXCEPTION + result = ex + except Exception: + logging.exception("Unexpected exception in handler") + msg_type = TMessageType.EXCEPTION + result = TApplicationException(TApplicationException.INTERNAL_ERROR, "Internal error") + oprot.writeMessageBegin("get_partitions_ps", msg_type, seqid) + result.write(oprot) + oprot.writeMessageEnd() + oprot.trans.flush() + + def process_get_partitions_ps_with_auth(self, seqid, iprot, oprot): + args = get_partitions_ps_with_auth_args() + args.read(iprot) + iprot.readMessageEnd() + result = get_partitions_ps_with_auth_result() + try: + result.success = self._handler.get_partitions_ps_with_auth( + args.db_name, args.tbl_name, args.part_vals, args.max_parts, args.user_name, args.group_names + ) + msg_type = TMessageType.REPLY + except TTransport.TTransportException: + raise + except NoSuchObjectException as o1: + msg_type = TMessageType.REPLY + result.o1 = o1 + except MetaException as o2: + msg_type = TMessageType.REPLY + result.o2 = o2 + except TApplicationException as ex: + logging.exception("TApplication exception in handler") + msg_type = TMessageType.EXCEPTION + result = ex + except Exception: + logging.exception("Unexpected exception in handler") + msg_type = TMessageType.EXCEPTION + result = TApplicationException(TApplicationException.INTERNAL_ERROR, "Internal error") + oprot.writeMessageBegin("get_partitions_ps_with_auth", msg_type, seqid) + result.write(oprot) + oprot.writeMessageEnd() + oprot.trans.flush() + + def process_get_partitions_ps_with_auth_req(self, seqid, iprot, oprot): + args = get_partitions_ps_with_auth_req_args() + args.read(iprot) + iprot.readMessageEnd() + result = get_partitions_ps_with_auth_req_result() + try: + result.success = self._handler.get_partitions_ps_with_auth_req(args.req) + msg_type = TMessageType.REPLY + except TTransport.TTransportException: + raise + except MetaException as o1: + msg_type = TMessageType.REPLY + result.o1 = o1 + except NoSuchObjectException as o2: + msg_type = TMessageType.REPLY + result.o2 = o2 + except TApplicationException as ex: + logging.exception("TApplication exception in handler") + msg_type = TMessageType.EXCEPTION + result = ex + except Exception: + logging.exception("Unexpected exception in handler") + msg_type = TMessageType.EXCEPTION + result = TApplicationException(TApplicationException.INTERNAL_ERROR, "Internal error") + oprot.writeMessageBegin("get_partitions_ps_with_auth_req", msg_type, seqid) + result.write(oprot) + oprot.writeMessageEnd() + oprot.trans.flush() + + def process_get_partition_names_ps(self, seqid, iprot, oprot): + args = get_partition_names_ps_args() + args.read(iprot) + iprot.readMessageEnd() + result = get_partition_names_ps_result() + try: + result.success = self._handler.get_partition_names_ps(args.db_name, args.tbl_name, args.part_vals, args.max_parts) + msg_type = TMessageType.REPLY + except TTransport.TTransportException: + raise + except MetaException as o1: + msg_type = TMessageType.REPLY + result.o1 = o1 + except NoSuchObjectException as o2: + msg_type = TMessageType.REPLY + result.o2 = o2 + except TApplicationException as ex: + logging.exception("TApplication exception in handler") + msg_type = TMessageType.EXCEPTION + result = ex + except Exception: + logging.exception("Unexpected exception in handler") + msg_type = TMessageType.EXCEPTION + result = TApplicationException(TApplicationException.INTERNAL_ERROR, "Internal error") + oprot.writeMessageBegin("get_partition_names_ps", msg_type, seqid) + result.write(oprot) + oprot.writeMessageEnd() + oprot.trans.flush() + + def process_get_partition_names_ps_req(self, seqid, iprot, oprot): + args = get_partition_names_ps_req_args() + args.read(iprot) + iprot.readMessageEnd() + result = get_partition_names_ps_req_result() + try: + result.success = self._handler.get_partition_names_ps_req(args.req) + msg_type = TMessageType.REPLY + except TTransport.TTransportException: + raise + except MetaException as o1: + msg_type = TMessageType.REPLY + result.o1 = o1 + except NoSuchObjectException as o2: + msg_type = TMessageType.REPLY + result.o2 = o2 + except TApplicationException as ex: + logging.exception("TApplication exception in handler") + msg_type = TMessageType.EXCEPTION + result = ex + except Exception: + logging.exception("Unexpected exception in handler") + msg_type = TMessageType.EXCEPTION + result = TApplicationException(TApplicationException.INTERNAL_ERROR, "Internal error") + oprot.writeMessageBegin("get_partition_names_ps_req", msg_type, seqid) + result.write(oprot) + oprot.writeMessageEnd() + oprot.trans.flush() + + def process_get_partition_names_req(self, seqid, iprot, oprot): + args = get_partition_names_req_args() + args.read(iprot) + iprot.readMessageEnd() + result = get_partition_names_req_result() + try: + result.success = self._handler.get_partition_names_req(args.req) + msg_type = TMessageType.REPLY + except TTransport.TTransportException: + raise + except MetaException as o1: + msg_type = TMessageType.REPLY + result.o1 = o1 + except NoSuchObjectException as o2: + msg_type = TMessageType.REPLY + result.o2 = o2 + except TApplicationException as ex: + logging.exception("TApplication exception in handler") + msg_type = TMessageType.EXCEPTION + result = ex + except Exception: + logging.exception("Unexpected exception in handler") + msg_type = TMessageType.EXCEPTION + result = TApplicationException(TApplicationException.INTERNAL_ERROR, "Internal error") + oprot.writeMessageBegin("get_partition_names_req", msg_type, seqid) + result.write(oprot) + oprot.writeMessageEnd() + oprot.trans.flush() + + def process_get_partitions_by_filter(self, seqid, iprot, oprot): + args = get_partitions_by_filter_args() + args.read(iprot) + iprot.readMessageEnd() + result = get_partitions_by_filter_result() + try: + result.success = self._handler.get_partitions_by_filter(args.db_name, args.tbl_name, args.filter, args.max_parts) + msg_type = TMessageType.REPLY + except TTransport.TTransportException: + raise + except MetaException as o1: + msg_type = TMessageType.REPLY + result.o1 = o1 + except NoSuchObjectException as o2: + msg_type = TMessageType.REPLY + result.o2 = o2 + except TApplicationException as ex: + logging.exception("TApplication exception in handler") + msg_type = TMessageType.EXCEPTION + result = ex + except Exception: + logging.exception("Unexpected exception in handler") + msg_type = TMessageType.EXCEPTION + result = TApplicationException(TApplicationException.INTERNAL_ERROR, "Internal error") + oprot.writeMessageBegin("get_partitions_by_filter", msg_type, seqid) + result.write(oprot) + oprot.writeMessageEnd() + oprot.trans.flush() + + def process_get_part_specs_by_filter(self, seqid, iprot, oprot): + args = get_part_specs_by_filter_args() + args.read(iprot) + iprot.readMessageEnd() + result = get_part_specs_by_filter_result() + try: + result.success = self._handler.get_part_specs_by_filter(args.db_name, args.tbl_name, args.filter, args.max_parts) + msg_type = TMessageType.REPLY + except TTransport.TTransportException: + raise + except MetaException as o1: + msg_type = TMessageType.REPLY + result.o1 = o1 + except NoSuchObjectException as o2: + msg_type = TMessageType.REPLY + result.o2 = o2 + except TApplicationException as ex: + logging.exception("TApplication exception in handler") + msg_type = TMessageType.EXCEPTION + result = ex + except Exception: + logging.exception("Unexpected exception in handler") + msg_type = TMessageType.EXCEPTION + result = TApplicationException(TApplicationException.INTERNAL_ERROR, "Internal error") + oprot.writeMessageBegin("get_part_specs_by_filter", msg_type, seqid) + result.write(oprot) + oprot.writeMessageEnd() + oprot.trans.flush() + + def process_get_partitions_by_expr(self, seqid, iprot, oprot): + args = get_partitions_by_expr_args() + args.read(iprot) + iprot.readMessageEnd() + result = get_partitions_by_expr_result() + try: + result.success = self._handler.get_partitions_by_expr(args.req) + msg_type = TMessageType.REPLY + except TTransport.TTransportException: + raise + except MetaException as o1: + msg_type = TMessageType.REPLY + result.o1 = o1 + except NoSuchObjectException as o2: + msg_type = TMessageType.REPLY + result.o2 = o2 + except TApplicationException as ex: + logging.exception("TApplication exception in handler") + msg_type = TMessageType.EXCEPTION + result = ex + except Exception: + logging.exception("Unexpected exception in handler") + msg_type = TMessageType.EXCEPTION + result = TApplicationException(TApplicationException.INTERNAL_ERROR, "Internal error") + oprot.writeMessageBegin("get_partitions_by_expr", msg_type, seqid) + result.write(oprot) + oprot.writeMessageEnd() + oprot.trans.flush() + + def process_get_partitions_spec_by_expr(self, seqid, iprot, oprot): + args = get_partitions_spec_by_expr_args() + args.read(iprot) + iprot.readMessageEnd() + result = get_partitions_spec_by_expr_result() + try: + result.success = self._handler.get_partitions_spec_by_expr(args.req) + msg_type = TMessageType.REPLY + except TTransport.TTransportException: + raise + except MetaException as o1: + msg_type = TMessageType.REPLY + result.o1 = o1 + except NoSuchObjectException as o2: + msg_type = TMessageType.REPLY + result.o2 = o2 + except TApplicationException as ex: + logging.exception("TApplication exception in handler") + msg_type = TMessageType.EXCEPTION + result = ex + except Exception: + logging.exception("Unexpected exception in handler") + msg_type = TMessageType.EXCEPTION + result = TApplicationException(TApplicationException.INTERNAL_ERROR, "Internal error") + oprot.writeMessageBegin("get_partitions_spec_by_expr", msg_type, seqid) + result.write(oprot) + oprot.writeMessageEnd() + oprot.trans.flush() + + def process_get_num_partitions_by_filter(self, seqid, iprot, oprot): + args = get_num_partitions_by_filter_args() + args.read(iprot) + iprot.readMessageEnd() + result = get_num_partitions_by_filter_result() + try: + result.success = self._handler.get_num_partitions_by_filter(args.db_name, args.tbl_name, args.filter) + msg_type = TMessageType.REPLY + except TTransport.TTransportException: + raise + except MetaException as o1: + msg_type = TMessageType.REPLY + result.o1 = o1 + except NoSuchObjectException as o2: + msg_type = TMessageType.REPLY + result.o2 = o2 + except TApplicationException as ex: + logging.exception("TApplication exception in handler") + msg_type = TMessageType.EXCEPTION + result = ex + except Exception: + logging.exception("Unexpected exception in handler") + msg_type = TMessageType.EXCEPTION + result = TApplicationException(TApplicationException.INTERNAL_ERROR, "Internal error") + oprot.writeMessageBegin("get_num_partitions_by_filter", msg_type, seqid) + result.write(oprot) + oprot.writeMessageEnd() + oprot.trans.flush() + + def process_get_partitions_by_names(self, seqid, iprot, oprot): + args = get_partitions_by_names_args() + args.read(iprot) + iprot.readMessageEnd() + result = get_partitions_by_names_result() + try: + result.success = self._handler.get_partitions_by_names(args.db_name, args.tbl_name, args.names) + msg_type = TMessageType.REPLY + except TTransport.TTransportException: + raise + except MetaException as o1: + msg_type = TMessageType.REPLY + result.o1 = o1 + except NoSuchObjectException as o2: + msg_type = TMessageType.REPLY + result.o2 = o2 + except TApplicationException as ex: + logging.exception("TApplication exception in handler") + msg_type = TMessageType.EXCEPTION + result = ex + except Exception: + logging.exception("Unexpected exception in handler") + msg_type = TMessageType.EXCEPTION + result = TApplicationException(TApplicationException.INTERNAL_ERROR, "Internal error") + oprot.writeMessageBegin("get_partitions_by_names", msg_type, seqid) + result.write(oprot) + oprot.writeMessageEnd() + oprot.trans.flush() + + def process_get_partitions_by_names_req(self, seqid, iprot, oprot): + args = get_partitions_by_names_req_args() + args.read(iprot) + iprot.readMessageEnd() + result = get_partitions_by_names_req_result() + try: + result.success = self._handler.get_partitions_by_names_req(args.req) + msg_type = TMessageType.REPLY + except TTransport.TTransportException: + raise + except MetaException as o1: + msg_type = TMessageType.REPLY + result.o1 = o1 + except NoSuchObjectException as o2: + msg_type = TMessageType.REPLY + result.o2 = o2 + except TApplicationException as ex: + logging.exception("TApplication exception in handler") + msg_type = TMessageType.EXCEPTION + result = ex + except Exception: + logging.exception("Unexpected exception in handler") + msg_type = TMessageType.EXCEPTION + result = TApplicationException(TApplicationException.INTERNAL_ERROR, "Internal error") + oprot.writeMessageBegin("get_partitions_by_names_req", msg_type, seqid) + result.write(oprot) + oprot.writeMessageEnd() + oprot.trans.flush() + + def process_alter_partition(self, seqid, iprot, oprot): + args = alter_partition_args() + args.read(iprot) + iprot.readMessageEnd() + result = alter_partition_result() + try: + self._handler.alter_partition(args.db_name, args.tbl_name, args.new_part) + msg_type = TMessageType.REPLY + except TTransport.TTransportException: + raise + except InvalidOperationException as o1: + msg_type = TMessageType.REPLY + result.o1 = o1 + except MetaException as o2: + msg_type = TMessageType.REPLY + result.o2 = o2 + except TApplicationException as ex: + logging.exception("TApplication exception in handler") + msg_type = TMessageType.EXCEPTION + result = ex + except Exception: + logging.exception("Unexpected exception in handler") + msg_type = TMessageType.EXCEPTION + result = TApplicationException(TApplicationException.INTERNAL_ERROR, "Internal error") + oprot.writeMessageBegin("alter_partition", msg_type, seqid) + result.write(oprot) + oprot.writeMessageEnd() + oprot.trans.flush() + + def process_alter_partitions(self, seqid, iprot, oprot): + args = alter_partitions_args() + args.read(iprot) + iprot.readMessageEnd() + result = alter_partitions_result() + try: + self._handler.alter_partitions(args.db_name, args.tbl_name, args.new_parts) + msg_type = TMessageType.REPLY + except TTransport.TTransportException: + raise + except InvalidOperationException as o1: + msg_type = TMessageType.REPLY + result.o1 = o1 + except MetaException as o2: + msg_type = TMessageType.REPLY + result.o2 = o2 + except TApplicationException as ex: + logging.exception("TApplication exception in handler") + msg_type = TMessageType.EXCEPTION + result = ex + except Exception: + logging.exception("Unexpected exception in handler") + msg_type = TMessageType.EXCEPTION + result = TApplicationException(TApplicationException.INTERNAL_ERROR, "Internal error") + oprot.writeMessageBegin("alter_partitions", msg_type, seqid) + result.write(oprot) + oprot.writeMessageEnd() + oprot.trans.flush() + + def process_alter_partitions_with_environment_context(self, seqid, iprot, oprot): + args = alter_partitions_with_environment_context_args() + args.read(iprot) + iprot.readMessageEnd() + result = alter_partitions_with_environment_context_result() + try: + self._handler.alter_partitions_with_environment_context( + args.db_name, args.tbl_name, args.new_parts, args.environment_context + ) + msg_type = TMessageType.REPLY + except TTransport.TTransportException: + raise + except InvalidOperationException as o1: + msg_type = TMessageType.REPLY + result.o1 = o1 + except MetaException as o2: + msg_type = TMessageType.REPLY + result.o2 = o2 + except TApplicationException as ex: + logging.exception("TApplication exception in handler") + msg_type = TMessageType.EXCEPTION + result = ex + except Exception: + logging.exception("Unexpected exception in handler") + msg_type = TMessageType.EXCEPTION + result = TApplicationException(TApplicationException.INTERNAL_ERROR, "Internal error") + oprot.writeMessageBegin("alter_partitions_with_environment_context", msg_type, seqid) + result.write(oprot) + oprot.writeMessageEnd() + oprot.trans.flush() + + def process_alter_partitions_req(self, seqid, iprot, oprot): + args = alter_partitions_req_args() + args.read(iprot) + iprot.readMessageEnd() + result = alter_partitions_req_result() + try: + result.success = self._handler.alter_partitions_req(args.req) + msg_type = TMessageType.REPLY + except TTransport.TTransportException: + raise + except InvalidOperationException as o1: + msg_type = TMessageType.REPLY + result.o1 = o1 + except MetaException as o2: + msg_type = TMessageType.REPLY + result.o2 = o2 + except TApplicationException as ex: + logging.exception("TApplication exception in handler") + msg_type = TMessageType.EXCEPTION + result = ex + except Exception: + logging.exception("Unexpected exception in handler") + msg_type = TMessageType.EXCEPTION + result = TApplicationException(TApplicationException.INTERNAL_ERROR, "Internal error") + oprot.writeMessageBegin("alter_partitions_req", msg_type, seqid) + result.write(oprot) + oprot.writeMessageEnd() + oprot.trans.flush() + + def process_alter_partition_with_environment_context(self, seqid, iprot, oprot): + args = alter_partition_with_environment_context_args() + args.read(iprot) + iprot.readMessageEnd() + result = alter_partition_with_environment_context_result() + try: + self._handler.alter_partition_with_environment_context( + args.db_name, args.tbl_name, args.new_part, args.environment_context + ) + msg_type = TMessageType.REPLY + except TTransport.TTransportException: + raise + except InvalidOperationException as o1: + msg_type = TMessageType.REPLY + result.o1 = o1 + except MetaException as o2: + msg_type = TMessageType.REPLY + result.o2 = o2 + except TApplicationException as ex: + logging.exception("TApplication exception in handler") + msg_type = TMessageType.EXCEPTION + result = ex + except Exception: + logging.exception("Unexpected exception in handler") + msg_type = TMessageType.EXCEPTION + result = TApplicationException(TApplicationException.INTERNAL_ERROR, "Internal error") + oprot.writeMessageBegin("alter_partition_with_environment_context", msg_type, seqid) + result.write(oprot) + oprot.writeMessageEnd() + oprot.trans.flush() + + def process_rename_partition(self, seqid, iprot, oprot): + args = rename_partition_args() + args.read(iprot) + iprot.readMessageEnd() + result = rename_partition_result() + try: + self._handler.rename_partition(args.db_name, args.tbl_name, args.part_vals, args.new_part) + msg_type = TMessageType.REPLY + except TTransport.TTransportException: + raise + except InvalidOperationException as o1: + msg_type = TMessageType.REPLY + result.o1 = o1 + except MetaException as o2: + msg_type = TMessageType.REPLY + result.o2 = o2 + except TApplicationException as ex: + logging.exception("TApplication exception in handler") + msg_type = TMessageType.EXCEPTION + result = ex + except Exception: + logging.exception("Unexpected exception in handler") + msg_type = TMessageType.EXCEPTION + result = TApplicationException(TApplicationException.INTERNAL_ERROR, "Internal error") + oprot.writeMessageBegin("rename_partition", msg_type, seqid) + result.write(oprot) + oprot.writeMessageEnd() + oprot.trans.flush() + + def process_rename_partition_req(self, seqid, iprot, oprot): + args = rename_partition_req_args() + args.read(iprot) + iprot.readMessageEnd() + result = rename_partition_req_result() + try: + result.success = self._handler.rename_partition_req(args.req) + msg_type = TMessageType.REPLY + except TTransport.TTransportException: + raise + except InvalidOperationException as o1: + msg_type = TMessageType.REPLY + result.o1 = o1 + except MetaException as o2: + msg_type = TMessageType.REPLY + result.o2 = o2 + except TApplicationException as ex: + logging.exception("TApplication exception in handler") + msg_type = TMessageType.EXCEPTION + result = ex + except Exception: + logging.exception("Unexpected exception in handler") + msg_type = TMessageType.EXCEPTION + result = TApplicationException(TApplicationException.INTERNAL_ERROR, "Internal error") + oprot.writeMessageBegin("rename_partition_req", msg_type, seqid) + result.write(oprot) + oprot.writeMessageEnd() + oprot.trans.flush() + + def process_partition_name_has_valid_characters(self, seqid, iprot, oprot): + args = partition_name_has_valid_characters_args() + args.read(iprot) + iprot.readMessageEnd() + result = partition_name_has_valid_characters_result() + try: + result.success = self._handler.partition_name_has_valid_characters(args.part_vals, args.throw_exception) + msg_type = TMessageType.REPLY + except TTransport.TTransportException: + raise + except MetaException as o1: + msg_type = TMessageType.REPLY + result.o1 = o1 + except TApplicationException as ex: + logging.exception("TApplication exception in handler") + msg_type = TMessageType.EXCEPTION + result = ex + except Exception: + logging.exception("Unexpected exception in handler") + msg_type = TMessageType.EXCEPTION + result = TApplicationException(TApplicationException.INTERNAL_ERROR, "Internal error") + oprot.writeMessageBegin("partition_name_has_valid_characters", msg_type, seqid) + result.write(oprot) + oprot.writeMessageEnd() + oprot.trans.flush() + + def process_get_config_value(self, seqid, iprot, oprot): + args = get_config_value_args() + args.read(iprot) + iprot.readMessageEnd() + result = get_config_value_result() + try: + result.success = self._handler.get_config_value(args.name, args.defaultValue) + msg_type = TMessageType.REPLY + except TTransport.TTransportException: + raise + except ConfigValSecurityException as o1: + msg_type = TMessageType.REPLY + result.o1 = o1 + except TApplicationException as ex: + logging.exception("TApplication exception in handler") + msg_type = TMessageType.EXCEPTION + result = ex + except Exception: + logging.exception("Unexpected exception in handler") + msg_type = TMessageType.EXCEPTION + result = TApplicationException(TApplicationException.INTERNAL_ERROR, "Internal error") + oprot.writeMessageBegin("get_config_value", msg_type, seqid) + result.write(oprot) + oprot.writeMessageEnd() + oprot.trans.flush() + + def process_partition_name_to_vals(self, seqid, iprot, oprot): + args = partition_name_to_vals_args() + args.read(iprot) + iprot.readMessageEnd() + result = partition_name_to_vals_result() + try: + result.success = self._handler.partition_name_to_vals(args.part_name) + msg_type = TMessageType.REPLY + except TTransport.TTransportException: + raise + except MetaException as o1: + msg_type = TMessageType.REPLY + result.o1 = o1 + except TApplicationException as ex: + logging.exception("TApplication exception in handler") + msg_type = TMessageType.EXCEPTION + result = ex + except Exception: + logging.exception("Unexpected exception in handler") + msg_type = TMessageType.EXCEPTION + result = TApplicationException(TApplicationException.INTERNAL_ERROR, "Internal error") + oprot.writeMessageBegin("partition_name_to_vals", msg_type, seqid) + result.write(oprot) + oprot.writeMessageEnd() + oprot.trans.flush() + + def process_partition_name_to_spec(self, seqid, iprot, oprot): + args = partition_name_to_spec_args() + args.read(iprot) + iprot.readMessageEnd() + result = partition_name_to_spec_result() + try: + result.success = self._handler.partition_name_to_spec(args.part_name) + msg_type = TMessageType.REPLY + except TTransport.TTransportException: + raise + except MetaException as o1: + msg_type = TMessageType.REPLY + result.o1 = o1 + except TApplicationException as ex: + logging.exception("TApplication exception in handler") + msg_type = TMessageType.EXCEPTION + result = ex + except Exception: + logging.exception("Unexpected exception in handler") + msg_type = TMessageType.EXCEPTION + result = TApplicationException(TApplicationException.INTERNAL_ERROR, "Internal error") + oprot.writeMessageBegin("partition_name_to_spec", msg_type, seqid) + result.write(oprot) + oprot.writeMessageEnd() + oprot.trans.flush() + + def process_markPartitionForEvent(self, seqid, iprot, oprot): + args = markPartitionForEvent_args() + args.read(iprot) + iprot.readMessageEnd() + result = markPartitionForEvent_result() + try: + self._handler.markPartitionForEvent(args.db_name, args.tbl_name, args.part_vals, args.eventType) + msg_type = TMessageType.REPLY + except TTransport.TTransportException: + raise + except MetaException as o1: + msg_type = TMessageType.REPLY + result.o1 = o1 + except NoSuchObjectException as o2: + msg_type = TMessageType.REPLY + result.o2 = o2 + except UnknownDBException as o3: + msg_type = TMessageType.REPLY + result.o3 = o3 + except UnknownTableException as o4: + msg_type = TMessageType.REPLY + result.o4 = o4 + except UnknownPartitionException as o5: + msg_type = TMessageType.REPLY + result.o5 = o5 + except InvalidPartitionException as o6: + msg_type = TMessageType.REPLY + result.o6 = o6 + except TApplicationException as ex: + logging.exception("TApplication exception in handler") + msg_type = TMessageType.EXCEPTION + result = ex + except Exception: + logging.exception("Unexpected exception in handler") + msg_type = TMessageType.EXCEPTION + result = TApplicationException(TApplicationException.INTERNAL_ERROR, "Internal error") + oprot.writeMessageBegin("markPartitionForEvent", msg_type, seqid) + result.write(oprot) + oprot.writeMessageEnd() + oprot.trans.flush() + + def process_isPartitionMarkedForEvent(self, seqid, iprot, oprot): + args = isPartitionMarkedForEvent_args() + args.read(iprot) + iprot.readMessageEnd() + result = isPartitionMarkedForEvent_result() + try: + result.success = self._handler.isPartitionMarkedForEvent(args.db_name, args.tbl_name, args.part_vals, args.eventType) + msg_type = TMessageType.REPLY + except TTransport.TTransportException: + raise + except MetaException as o1: + msg_type = TMessageType.REPLY + result.o1 = o1 + except NoSuchObjectException as o2: + msg_type = TMessageType.REPLY + result.o2 = o2 + except UnknownDBException as o3: + msg_type = TMessageType.REPLY + result.o3 = o3 + except UnknownTableException as o4: + msg_type = TMessageType.REPLY + result.o4 = o4 + except UnknownPartitionException as o5: + msg_type = TMessageType.REPLY + result.o5 = o5 + except InvalidPartitionException as o6: + msg_type = TMessageType.REPLY + result.o6 = o6 + except TApplicationException as ex: + logging.exception("TApplication exception in handler") + msg_type = TMessageType.EXCEPTION + result = ex + except Exception: + logging.exception("Unexpected exception in handler") + msg_type = TMessageType.EXCEPTION + result = TApplicationException(TApplicationException.INTERNAL_ERROR, "Internal error") + oprot.writeMessageBegin("isPartitionMarkedForEvent", msg_type, seqid) + result.write(oprot) + oprot.writeMessageEnd() + oprot.trans.flush() + + def process_get_primary_keys(self, seqid, iprot, oprot): + args = get_primary_keys_args() + args.read(iprot) + iprot.readMessageEnd() + result = get_primary_keys_result() + try: + result.success = self._handler.get_primary_keys(args.request) + msg_type = TMessageType.REPLY + except TTransport.TTransportException: + raise + except MetaException as o1: + msg_type = TMessageType.REPLY + result.o1 = o1 + except NoSuchObjectException as o2: + msg_type = TMessageType.REPLY + result.o2 = o2 + except TApplicationException as ex: + logging.exception("TApplication exception in handler") + msg_type = TMessageType.EXCEPTION + result = ex + except Exception: + logging.exception("Unexpected exception in handler") + msg_type = TMessageType.EXCEPTION + result = TApplicationException(TApplicationException.INTERNAL_ERROR, "Internal error") + oprot.writeMessageBegin("get_primary_keys", msg_type, seqid) + result.write(oprot) + oprot.writeMessageEnd() + oprot.trans.flush() + + def process_get_foreign_keys(self, seqid, iprot, oprot): + args = get_foreign_keys_args() + args.read(iprot) + iprot.readMessageEnd() + result = get_foreign_keys_result() + try: + result.success = self._handler.get_foreign_keys(args.request) + msg_type = TMessageType.REPLY + except TTransport.TTransportException: + raise + except MetaException as o1: + msg_type = TMessageType.REPLY + result.o1 = o1 + except NoSuchObjectException as o2: + msg_type = TMessageType.REPLY + result.o2 = o2 + except TApplicationException as ex: + logging.exception("TApplication exception in handler") + msg_type = TMessageType.EXCEPTION + result = ex + except Exception: + logging.exception("Unexpected exception in handler") + msg_type = TMessageType.EXCEPTION + result = TApplicationException(TApplicationException.INTERNAL_ERROR, "Internal error") + oprot.writeMessageBegin("get_foreign_keys", msg_type, seqid) + result.write(oprot) + oprot.writeMessageEnd() + oprot.trans.flush() + + def process_get_unique_constraints(self, seqid, iprot, oprot): + args = get_unique_constraints_args() + args.read(iprot) + iprot.readMessageEnd() + result = get_unique_constraints_result() + try: + result.success = self._handler.get_unique_constraints(args.request) + msg_type = TMessageType.REPLY + except TTransport.TTransportException: + raise + except MetaException as o1: + msg_type = TMessageType.REPLY + result.o1 = o1 + except NoSuchObjectException as o2: + msg_type = TMessageType.REPLY + result.o2 = o2 + except TApplicationException as ex: + logging.exception("TApplication exception in handler") + msg_type = TMessageType.EXCEPTION + result = ex + except Exception: + logging.exception("Unexpected exception in handler") + msg_type = TMessageType.EXCEPTION + result = TApplicationException(TApplicationException.INTERNAL_ERROR, "Internal error") + oprot.writeMessageBegin("get_unique_constraints", msg_type, seqid) + result.write(oprot) + oprot.writeMessageEnd() + oprot.trans.flush() + + def process_get_not_null_constraints(self, seqid, iprot, oprot): + args = get_not_null_constraints_args() + args.read(iprot) + iprot.readMessageEnd() + result = get_not_null_constraints_result() + try: + result.success = self._handler.get_not_null_constraints(args.request) + msg_type = TMessageType.REPLY + except TTransport.TTransportException: + raise + except MetaException as o1: + msg_type = TMessageType.REPLY + result.o1 = o1 + except NoSuchObjectException as o2: + msg_type = TMessageType.REPLY + result.o2 = o2 + except TApplicationException as ex: + logging.exception("TApplication exception in handler") + msg_type = TMessageType.EXCEPTION + result = ex + except Exception: + logging.exception("Unexpected exception in handler") + msg_type = TMessageType.EXCEPTION + result = TApplicationException(TApplicationException.INTERNAL_ERROR, "Internal error") + oprot.writeMessageBegin("get_not_null_constraints", msg_type, seqid) + result.write(oprot) + oprot.writeMessageEnd() + oprot.trans.flush() + + def process_get_default_constraints(self, seqid, iprot, oprot): + args = get_default_constraints_args() + args.read(iprot) + iprot.readMessageEnd() + result = get_default_constraints_result() + try: + result.success = self._handler.get_default_constraints(args.request) + msg_type = TMessageType.REPLY + except TTransport.TTransportException: + raise + except MetaException as o1: + msg_type = TMessageType.REPLY + result.o1 = o1 + except NoSuchObjectException as o2: + msg_type = TMessageType.REPLY + result.o2 = o2 + except TApplicationException as ex: + logging.exception("TApplication exception in handler") + msg_type = TMessageType.EXCEPTION + result = ex + except Exception: + logging.exception("Unexpected exception in handler") + msg_type = TMessageType.EXCEPTION + result = TApplicationException(TApplicationException.INTERNAL_ERROR, "Internal error") + oprot.writeMessageBegin("get_default_constraints", msg_type, seqid) + result.write(oprot) + oprot.writeMessageEnd() + oprot.trans.flush() + + def process_get_check_constraints(self, seqid, iprot, oprot): + args = get_check_constraints_args() + args.read(iprot) + iprot.readMessageEnd() + result = get_check_constraints_result() + try: + result.success = self._handler.get_check_constraints(args.request) + msg_type = TMessageType.REPLY + except TTransport.TTransportException: + raise + except MetaException as o1: + msg_type = TMessageType.REPLY + result.o1 = o1 + except NoSuchObjectException as o2: + msg_type = TMessageType.REPLY + result.o2 = o2 + except TApplicationException as ex: + logging.exception("TApplication exception in handler") + msg_type = TMessageType.EXCEPTION + result = ex + except Exception: + logging.exception("Unexpected exception in handler") + msg_type = TMessageType.EXCEPTION + result = TApplicationException(TApplicationException.INTERNAL_ERROR, "Internal error") + oprot.writeMessageBegin("get_check_constraints", msg_type, seqid) + result.write(oprot) + oprot.writeMessageEnd() + oprot.trans.flush() + + def process_get_all_table_constraints(self, seqid, iprot, oprot): + args = get_all_table_constraints_args() + args.read(iprot) + iprot.readMessageEnd() + result = get_all_table_constraints_result() + try: + result.success = self._handler.get_all_table_constraints(args.request) + msg_type = TMessageType.REPLY + except TTransport.TTransportException: + raise + except MetaException as o1: + msg_type = TMessageType.REPLY + result.o1 = o1 + except NoSuchObjectException as o2: + msg_type = TMessageType.REPLY + result.o2 = o2 + except TApplicationException as ex: + logging.exception("TApplication exception in handler") + msg_type = TMessageType.EXCEPTION + result = ex + except Exception: + logging.exception("Unexpected exception in handler") + msg_type = TMessageType.EXCEPTION + result = TApplicationException(TApplicationException.INTERNAL_ERROR, "Internal error") + oprot.writeMessageBegin("get_all_table_constraints", msg_type, seqid) + result.write(oprot) + oprot.writeMessageEnd() + oprot.trans.flush() + + def process_update_table_column_statistics(self, seqid, iprot, oprot): + args = update_table_column_statistics_args() + args.read(iprot) + iprot.readMessageEnd() + result = update_table_column_statistics_result() + try: + result.success = self._handler.update_table_column_statistics(args.stats_obj) + msg_type = TMessageType.REPLY + except TTransport.TTransportException: + raise + except NoSuchObjectException as o1: + msg_type = TMessageType.REPLY + result.o1 = o1 + except InvalidObjectException as o2: + msg_type = TMessageType.REPLY + result.o2 = o2 + except MetaException as o3: + msg_type = TMessageType.REPLY + result.o3 = o3 + except InvalidInputException as o4: + msg_type = TMessageType.REPLY + result.o4 = o4 + except TApplicationException as ex: + logging.exception("TApplication exception in handler") + msg_type = TMessageType.EXCEPTION + result = ex + except Exception: + logging.exception("Unexpected exception in handler") + msg_type = TMessageType.EXCEPTION + result = TApplicationException(TApplicationException.INTERNAL_ERROR, "Internal error") + oprot.writeMessageBegin("update_table_column_statistics", msg_type, seqid) + result.write(oprot) + oprot.writeMessageEnd() + oprot.trans.flush() + + def process_update_partition_column_statistics(self, seqid, iprot, oprot): + args = update_partition_column_statistics_args() + args.read(iprot) + iprot.readMessageEnd() + result = update_partition_column_statistics_result() + try: + result.success = self._handler.update_partition_column_statistics(args.stats_obj) + msg_type = TMessageType.REPLY + except TTransport.TTransportException: + raise + except NoSuchObjectException as o1: + msg_type = TMessageType.REPLY + result.o1 = o1 + except InvalidObjectException as o2: + msg_type = TMessageType.REPLY + result.o2 = o2 + except MetaException as o3: + msg_type = TMessageType.REPLY + result.o3 = o3 + except InvalidInputException as o4: + msg_type = TMessageType.REPLY + result.o4 = o4 + except TApplicationException as ex: + logging.exception("TApplication exception in handler") + msg_type = TMessageType.EXCEPTION + result = ex + except Exception: + logging.exception("Unexpected exception in handler") + msg_type = TMessageType.EXCEPTION + result = TApplicationException(TApplicationException.INTERNAL_ERROR, "Internal error") + oprot.writeMessageBegin("update_partition_column_statistics", msg_type, seqid) + result.write(oprot) + oprot.writeMessageEnd() + oprot.trans.flush() + + def process_update_table_column_statistics_req(self, seqid, iprot, oprot): + args = update_table_column_statistics_req_args() + args.read(iprot) + iprot.readMessageEnd() + result = update_table_column_statistics_req_result() + try: + result.success = self._handler.update_table_column_statistics_req(args.req) + msg_type = TMessageType.REPLY + except TTransport.TTransportException: + raise + except NoSuchObjectException as o1: + msg_type = TMessageType.REPLY + result.o1 = o1 + except InvalidObjectException as o2: + msg_type = TMessageType.REPLY + result.o2 = o2 + except MetaException as o3: + msg_type = TMessageType.REPLY + result.o3 = o3 + except InvalidInputException as o4: + msg_type = TMessageType.REPLY + result.o4 = o4 + except TApplicationException as ex: + logging.exception("TApplication exception in handler") + msg_type = TMessageType.EXCEPTION + result = ex + except Exception: + logging.exception("Unexpected exception in handler") + msg_type = TMessageType.EXCEPTION + result = TApplicationException(TApplicationException.INTERNAL_ERROR, "Internal error") + oprot.writeMessageBegin("update_table_column_statistics_req", msg_type, seqid) + result.write(oprot) + oprot.writeMessageEnd() + oprot.trans.flush() + + def process_update_partition_column_statistics_req(self, seqid, iprot, oprot): + args = update_partition_column_statistics_req_args() + args.read(iprot) + iprot.readMessageEnd() + result = update_partition_column_statistics_req_result() + try: + result.success = self._handler.update_partition_column_statistics_req(args.req) + msg_type = TMessageType.REPLY + except TTransport.TTransportException: + raise + except NoSuchObjectException as o1: + msg_type = TMessageType.REPLY + result.o1 = o1 + except InvalidObjectException as o2: + msg_type = TMessageType.REPLY + result.o2 = o2 + except MetaException as o3: + msg_type = TMessageType.REPLY + result.o3 = o3 + except InvalidInputException as o4: + msg_type = TMessageType.REPLY + result.o4 = o4 + except TApplicationException as ex: + logging.exception("TApplication exception in handler") + msg_type = TMessageType.EXCEPTION + result = ex + except Exception: + logging.exception("Unexpected exception in handler") + msg_type = TMessageType.EXCEPTION + result = TApplicationException(TApplicationException.INTERNAL_ERROR, "Internal error") + oprot.writeMessageBegin("update_partition_column_statistics_req", msg_type, seqid) + result.write(oprot) + oprot.writeMessageEnd() + oprot.trans.flush() + + def process_update_transaction_statistics(self, seqid, iprot, oprot): + args = update_transaction_statistics_args() + args.read(iprot) + iprot.readMessageEnd() + result = update_transaction_statistics_result() + try: + self._handler.update_transaction_statistics(args.req) + msg_type = TMessageType.REPLY + except TTransport.TTransportException: + raise + except MetaException as o1: + msg_type = TMessageType.REPLY + result.o1 = o1 + except TApplicationException as ex: + logging.exception("TApplication exception in handler") + msg_type = TMessageType.EXCEPTION + result = ex + except Exception: + logging.exception("Unexpected exception in handler") + msg_type = TMessageType.EXCEPTION + result = TApplicationException(TApplicationException.INTERNAL_ERROR, "Internal error") + oprot.writeMessageBegin("update_transaction_statistics", msg_type, seqid) + result.write(oprot) + oprot.writeMessageEnd() + oprot.trans.flush() + + def process_get_table_column_statistics(self, seqid, iprot, oprot): + args = get_table_column_statistics_args() + args.read(iprot) + iprot.readMessageEnd() + result = get_table_column_statistics_result() + try: + result.success = self._handler.get_table_column_statistics(args.db_name, args.tbl_name, args.col_name) + msg_type = TMessageType.REPLY + except TTransport.TTransportException: + raise + except NoSuchObjectException as o1: + msg_type = TMessageType.REPLY + result.o1 = o1 + except MetaException as o2: + msg_type = TMessageType.REPLY + result.o2 = o2 + except InvalidInputException as o3: + msg_type = TMessageType.REPLY + result.o3 = o3 + except InvalidObjectException as o4: + msg_type = TMessageType.REPLY + result.o4 = o4 + except TApplicationException as ex: + logging.exception("TApplication exception in handler") + msg_type = TMessageType.EXCEPTION + result = ex + except Exception: + logging.exception("Unexpected exception in handler") + msg_type = TMessageType.EXCEPTION + result = TApplicationException(TApplicationException.INTERNAL_ERROR, "Internal error") + oprot.writeMessageBegin("get_table_column_statistics", msg_type, seqid) + result.write(oprot) + oprot.writeMessageEnd() + oprot.trans.flush() + + def process_get_partition_column_statistics(self, seqid, iprot, oprot): + args = get_partition_column_statistics_args() + args.read(iprot) + iprot.readMessageEnd() + result = get_partition_column_statistics_result() + try: + result.success = self._handler.get_partition_column_statistics( + args.db_name, args.tbl_name, args.part_name, args.col_name + ) + msg_type = TMessageType.REPLY + except TTransport.TTransportException: + raise + except NoSuchObjectException as o1: + msg_type = TMessageType.REPLY + result.o1 = o1 + except MetaException as o2: + msg_type = TMessageType.REPLY + result.o2 = o2 + except InvalidInputException as o3: + msg_type = TMessageType.REPLY + result.o3 = o3 + except InvalidObjectException as o4: + msg_type = TMessageType.REPLY + result.o4 = o4 + except TApplicationException as ex: + logging.exception("TApplication exception in handler") + msg_type = TMessageType.EXCEPTION + result = ex + except Exception: + logging.exception("Unexpected exception in handler") + msg_type = TMessageType.EXCEPTION + result = TApplicationException(TApplicationException.INTERNAL_ERROR, "Internal error") + oprot.writeMessageBegin("get_partition_column_statistics", msg_type, seqid) + result.write(oprot) + oprot.writeMessageEnd() + oprot.trans.flush() + + def process_get_table_statistics_req(self, seqid, iprot, oprot): + args = get_table_statistics_req_args() + args.read(iprot) + iprot.readMessageEnd() + result = get_table_statistics_req_result() + try: + result.success = self._handler.get_table_statistics_req(args.request) + msg_type = TMessageType.REPLY + except TTransport.TTransportException: + raise + except NoSuchObjectException as o1: + msg_type = TMessageType.REPLY + result.o1 = o1 + except MetaException as o2: + msg_type = TMessageType.REPLY + result.o2 = o2 + except TApplicationException as ex: + logging.exception("TApplication exception in handler") + msg_type = TMessageType.EXCEPTION + result = ex + except Exception: + logging.exception("Unexpected exception in handler") + msg_type = TMessageType.EXCEPTION + result = TApplicationException(TApplicationException.INTERNAL_ERROR, "Internal error") + oprot.writeMessageBegin("get_table_statistics_req", msg_type, seqid) + result.write(oprot) + oprot.writeMessageEnd() + oprot.trans.flush() + + def process_get_partitions_statistics_req(self, seqid, iprot, oprot): + args = get_partitions_statistics_req_args() + args.read(iprot) + iprot.readMessageEnd() + result = get_partitions_statistics_req_result() + try: + result.success = self._handler.get_partitions_statistics_req(args.request) + msg_type = TMessageType.REPLY + except TTransport.TTransportException: + raise + except NoSuchObjectException as o1: + msg_type = TMessageType.REPLY + result.o1 = o1 + except MetaException as o2: + msg_type = TMessageType.REPLY + result.o2 = o2 + except TApplicationException as ex: + logging.exception("TApplication exception in handler") + msg_type = TMessageType.EXCEPTION + result = ex + except Exception: + logging.exception("Unexpected exception in handler") + msg_type = TMessageType.EXCEPTION + result = TApplicationException(TApplicationException.INTERNAL_ERROR, "Internal error") + oprot.writeMessageBegin("get_partitions_statistics_req", msg_type, seqid) + result.write(oprot) + oprot.writeMessageEnd() + oprot.trans.flush() + + def process_get_aggr_stats_for(self, seqid, iprot, oprot): + args = get_aggr_stats_for_args() + args.read(iprot) + iprot.readMessageEnd() + result = get_aggr_stats_for_result() + try: + result.success = self._handler.get_aggr_stats_for(args.request) + msg_type = TMessageType.REPLY + except TTransport.TTransportException: + raise + except NoSuchObjectException as o1: + msg_type = TMessageType.REPLY + result.o1 = o1 + except MetaException as o2: + msg_type = TMessageType.REPLY + result.o2 = o2 + except TApplicationException as ex: + logging.exception("TApplication exception in handler") + msg_type = TMessageType.EXCEPTION + result = ex + except Exception: + logging.exception("Unexpected exception in handler") + msg_type = TMessageType.EXCEPTION + result = TApplicationException(TApplicationException.INTERNAL_ERROR, "Internal error") + oprot.writeMessageBegin("get_aggr_stats_for", msg_type, seqid) + result.write(oprot) + oprot.writeMessageEnd() + oprot.trans.flush() + + def process_set_aggr_stats_for(self, seqid, iprot, oprot): + args = set_aggr_stats_for_args() + args.read(iprot) + iprot.readMessageEnd() + result = set_aggr_stats_for_result() + try: + result.success = self._handler.set_aggr_stats_for(args.request) + msg_type = TMessageType.REPLY + except TTransport.TTransportException: + raise + except NoSuchObjectException as o1: + msg_type = TMessageType.REPLY + result.o1 = o1 + except InvalidObjectException as o2: + msg_type = TMessageType.REPLY + result.o2 = o2 + except MetaException as o3: + msg_type = TMessageType.REPLY + result.o3 = o3 + except InvalidInputException as o4: + msg_type = TMessageType.REPLY + result.o4 = o4 + except TApplicationException as ex: + logging.exception("TApplication exception in handler") + msg_type = TMessageType.EXCEPTION + result = ex + except Exception: + logging.exception("Unexpected exception in handler") + msg_type = TMessageType.EXCEPTION + result = TApplicationException(TApplicationException.INTERNAL_ERROR, "Internal error") + oprot.writeMessageBegin("set_aggr_stats_for", msg_type, seqid) + result.write(oprot) + oprot.writeMessageEnd() + oprot.trans.flush() + + def process_delete_partition_column_statistics(self, seqid, iprot, oprot): + args = delete_partition_column_statistics_args() + args.read(iprot) + iprot.readMessageEnd() + result = delete_partition_column_statistics_result() + try: + result.success = self._handler.delete_partition_column_statistics( + args.db_name, args.tbl_name, args.part_name, args.col_name, args.engine + ) + msg_type = TMessageType.REPLY + except TTransport.TTransportException: + raise + except NoSuchObjectException as o1: + msg_type = TMessageType.REPLY + result.o1 = o1 + except MetaException as o2: + msg_type = TMessageType.REPLY + result.o2 = o2 + except InvalidObjectException as o3: + msg_type = TMessageType.REPLY + result.o3 = o3 + except InvalidInputException as o4: + msg_type = TMessageType.REPLY + result.o4 = o4 + except TApplicationException as ex: + logging.exception("TApplication exception in handler") + msg_type = TMessageType.EXCEPTION + result = ex + except Exception: + logging.exception("Unexpected exception in handler") + msg_type = TMessageType.EXCEPTION + result = TApplicationException(TApplicationException.INTERNAL_ERROR, "Internal error") + oprot.writeMessageBegin("delete_partition_column_statistics", msg_type, seqid) + result.write(oprot) + oprot.writeMessageEnd() + oprot.trans.flush() + + def process_delete_table_column_statistics(self, seqid, iprot, oprot): + args = delete_table_column_statistics_args() + args.read(iprot) + iprot.readMessageEnd() + result = delete_table_column_statistics_result() + try: + result.success = self._handler.delete_table_column_statistics(args.db_name, args.tbl_name, args.col_name, args.engine) + msg_type = TMessageType.REPLY + except TTransport.TTransportException: + raise + except NoSuchObjectException as o1: + msg_type = TMessageType.REPLY + result.o1 = o1 + except MetaException as o2: + msg_type = TMessageType.REPLY + result.o2 = o2 + except InvalidObjectException as o3: + msg_type = TMessageType.REPLY + result.o3 = o3 + except InvalidInputException as o4: + msg_type = TMessageType.REPLY + result.o4 = o4 + except TApplicationException as ex: + logging.exception("TApplication exception in handler") + msg_type = TMessageType.EXCEPTION + result = ex + except Exception: + logging.exception("Unexpected exception in handler") + msg_type = TMessageType.EXCEPTION + result = TApplicationException(TApplicationException.INTERNAL_ERROR, "Internal error") + oprot.writeMessageBegin("delete_table_column_statistics", msg_type, seqid) + result.write(oprot) + oprot.writeMessageEnd() + oprot.trans.flush() + + def process_create_function(self, seqid, iprot, oprot): + args = create_function_args() + args.read(iprot) + iprot.readMessageEnd() + result = create_function_result() + try: + self._handler.create_function(args.func) + msg_type = TMessageType.REPLY + except TTransport.TTransportException: + raise + except AlreadyExistsException as o1: + msg_type = TMessageType.REPLY + result.o1 = o1 + except InvalidObjectException as o2: + msg_type = TMessageType.REPLY + result.o2 = o2 + except MetaException as o3: + msg_type = TMessageType.REPLY + result.o3 = o3 + except NoSuchObjectException as o4: + msg_type = TMessageType.REPLY + result.o4 = o4 + except TApplicationException as ex: + logging.exception("TApplication exception in handler") + msg_type = TMessageType.EXCEPTION + result = ex + except Exception: + logging.exception("Unexpected exception in handler") + msg_type = TMessageType.EXCEPTION + result = TApplicationException(TApplicationException.INTERNAL_ERROR, "Internal error") + oprot.writeMessageBegin("create_function", msg_type, seqid) + result.write(oprot) + oprot.writeMessageEnd() + oprot.trans.flush() + + def process_drop_function(self, seqid, iprot, oprot): + args = drop_function_args() + args.read(iprot) + iprot.readMessageEnd() + result = drop_function_result() + try: + self._handler.drop_function(args.dbName, args.funcName) + msg_type = TMessageType.REPLY + except TTransport.TTransportException: + raise + except NoSuchObjectException as o1: + msg_type = TMessageType.REPLY + result.o1 = o1 + except MetaException as o3: + msg_type = TMessageType.REPLY + result.o3 = o3 + except TApplicationException as ex: + logging.exception("TApplication exception in handler") + msg_type = TMessageType.EXCEPTION + result = ex + except Exception: + logging.exception("Unexpected exception in handler") + msg_type = TMessageType.EXCEPTION + result = TApplicationException(TApplicationException.INTERNAL_ERROR, "Internal error") + oprot.writeMessageBegin("drop_function", msg_type, seqid) + result.write(oprot) + oprot.writeMessageEnd() + oprot.trans.flush() + + def process_alter_function(self, seqid, iprot, oprot): + args = alter_function_args() + args.read(iprot) + iprot.readMessageEnd() + result = alter_function_result() + try: + self._handler.alter_function(args.dbName, args.funcName, args.newFunc) + msg_type = TMessageType.REPLY + except TTransport.TTransportException: + raise + except InvalidOperationException as o1: + msg_type = TMessageType.REPLY + result.o1 = o1 + except MetaException as o2: + msg_type = TMessageType.REPLY + result.o2 = o2 + except TApplicationException as ex: + logging.exception("TApplication exception in handler") + msg_type = TMessageType.EXCEPTION + result = ex + except Exception: + logging.exception("Unexpected exception in handler") + msg_type = TMessageType.EXCEPTION + result = TApplicationException(TApplicationException.INTERNAL_ERROR, "Internal error") + oprot.writeMessageBegin("alter_function", msg_type, seqid) + result.write(oprot) + oprot.writeMessageEnd() + oprot.trans.flush() + + def process_get_functions(self, seqid, iprot, oprot): + args = get_functions_args() + args.read(iprot) + iprot.readMessageEnd() + result = get_functions_result() + try: + result.success = self._handler.get_functions(args.dbName, args.pattern) + msg_type = TMessageType.REPLY + except TTransport.TTransportException: + raise + except MetaException as o1: + msg_type = TMessageType.REPLY + result.o1 = o1 + except TApplicationException as ex: + logging.exception("TApplication exception in handler") + msg_type = TMessageType.EXCEPTION + result = ex + except Exception: + logging.exception("Unexpected exception in handler") + msg_type = TMessageType.EXCEPTION + result = TApplicationException(TApplicationException.INTERNAL_ERROR, "Internal error") + oprot.writeMessageBegin("get_functions", msg_type, seqid) + result.write(oprot) + oprot.writeMessageEnd() + oprot.trans.flush() + + def process_get_function(self, seqid, iprot, oprot): + args = get_function_args() + args.read(iprot) + iprot.readMessageEnd() + result = get_function_result() + try: + result.success = self._handler.get_function(args.dbName, args.funcName) + msg_type = TMessageType.REPLY + except TTransport.TTransportException: + raise + except MetaException as o1: + msg_type = TMessageType.REPLY + result.o1 = o1 + except NoSuchObjectException as o2: + msg_type = TMessageType.REPLY + result.o2 = o2 + except TApplicationException as ex: + logging.exception("TApplication exception in handler") + msg_type = TMessageType.EXCEPTION + result = ex + except Exception: + logging.exception("Unexpected exception in handler") + msg_type = TMessageType.EXCEPTION + result = TApplicationException(TApplicationException.INTERNAL_ERROR, "Internal error") + oprot.writeMessageBegin("get_function", msg_type, seqid) + result.write(oprot) + oprot.writeMessageEnd() + oprot.trans.flush() + + def process_get_all_functions(self, seqid, iprot, oprot): + args = get_all_functions_args() + args.read(iprot) + iprot.readMessageEnd() + result = get_all_functions_result() + try: + result.success = self._handler.get_all_functions() + msg_type = TMessageType.REPLY + except TTransport.TTransportException: + raise + except MetaException as o1: + msg_type = TMessageType.REPLY + result.o1 = o1 + except TApplicationException as ex: + logging.exception("TApplication exception in handler") + msg_type = TMessageType.EXCEPTION + result = ex + except Exception: + logging.exception("Unexpected exception in handler") + msg_type = TMessageType.EXCEPTION + result = TApplicationException(TApplicationException.INTERNAL_ERROR, "Internal error") + oprot.writeMessageBegin("get_all_functions", msg_type, seqid) + result.write(oprot) + oprot.writeMessageEnd() + oprot.trans.flush() + + def process_create_role(self, seqid, iprot, oprot): + args = create_role_args() + args.read(iprot) + iprot.readMessageEnd() + result = create_role_result() + try: + result.success = self._handler.create_role(args.role) + msg_type = TMessageType.REPLY + except TTransport.TTransportException: + raise + except MetaException as o1: + msg_type = TMessageType.REPLY + result.o1 = o1 + except TApplicationException as ex: + logging.exception("TApplication exception in handler") + msg_type = TMessageType.EXCEPTION + result = ex + except Exception: + logging.exception("Unexpected exception in handler") + msg_type = TMessageType.EXCEPTION + result = TApplicationException(TApplicationException.INTERNAL_ERROR, "Internal error") + oprot.writeMessageBegin("create_role", msg_type, seqid) + result.write(oprot) + oprot.writeMessageEnd() + oprot.trans.flush() + + def process_drop_role(self, seqid, iprot, oprot): + args = drop_role_args() + args.read(iprot) + iprot.readMessageEnd() + result = drop_role_result() + try: + result.success = self._handler.drop_role(args.role_name) + msg_type = TMessageType.REPLY + except TTransport.TTransportException: + raise + except MetaException as o1: + msg_type = TMessageType.REPLY + result.o1 = o1 + except TApplicationException as ex: + logging.exception("TApplication exception in handler") + msg_type = TMessageType.EXCEPTION + result = ex + except Exception: + logging.exception("Unexpected exception in handler") + msg_type = TMessageType.EXCEPTION + result = TApplicationException(TApplicationException.INTERNAL_ERROR, "Internal error") + oprot.writeMessageBegin("drop_role", msg_type, seqid) + result.write(oprot) + oprot.writeMessageEnd() + oprot.trans.flush() + + def process_get_role_names(self, seqid, iprot, oprot): + args = get_role_names_args() + args.read(iprot) + iprot.readMessageEnd() + result = get_role_names_result() + try: + result.success = self._handler.get_role_names() + msg_type = TMessageType.REPLY + except TTransport.TTransportException: + raise + except MetaException as o1: + msg_type = TMessageType.REPLY + result.o1 = o1 + except TApplicationException as ex: + logging.exception("TApplication exception in handler") + msg_type = TMessageType.EXCEPTION + result = ex + except Exception: + logging.exception("Unexpected exception in handler") + msg_type = TMessageType.EXCEPTION + result = TApplicationException(TApplicationException.INTERNAL_ERROR, "Internal error") + oprot.writeMessageBegin("get_role_names", msg_type, seqid) + result.write(oprot) + oprot.writeMessageEnd() + oprot.trans.flush() + + def process_grant_role(self, seqid, iprot, oprot): + args = grant_role_args() + args.read(iprot) + iprot.readMessageEnd() + result = grant_role_result() + try: + result.success = self._handler.grant_role( + args.role_name, args.principal_name, args.principal_type, args.grantor, args.grantorType, args.grant_option + ) + msg_type = TMessageType.REPLY + except TTransport.TTransportException: + raise + except MetaException as o1: + msg_type = TMessageType.REPLY + result.o1 = o1 + except TApplicationException as ex: + logging.exception("TApplication exception in handler") + msg_type = TMessageType.EXCEPTION + result = ex + except Exception: + logging.exception("Unexpected exception in handler") + msg_type = TMessageType.EXCEPTION + result = TApplicationException(TApplicationException.INTERNAL_ERROR, "Internal error") + oprot.writeMessageBegin("grant_role", msg_type, seqid) + result.write(oprot) + oprot.writeMessageEnd() + oprot.trans.flush() + + def process_revoke_role(self, seqid, iprot, oprot): + args = revoke_role_args() + args.read(iprot) + iprot.readMessageEnd() + result = revoke_role_result() + try: + result.success = self._handler.revoke_role(args.role_name, args.principal_name, args.principal_type) + msg_type = TMessageType.REPLY + except TTransport.TTransportException: + raise + except MetaException as o1: + msg_type = TMessageType.REPLY + result.o1 = o1 + except TApplicationException as ex: + logging.exception("TApplication exception in handler") + msg_type = TMessageType.EXCEPTION + result = ex + except Exception: + logging.exception("Unexpected exception in handler") + msg_type = TMessageType.EXCEPTION + result = TApplicationException(TApplicationException.INTERNAL_ERROR, "Internal error") + oprot.writeMessageBegin("revoke_role", msg_type, seqid) + result.write(oprot) + oprot.writeMessageEnd() + oprot.trans.flush() + + def process_list_roles(self, seqid, iprot, oprot): + args = list_roles_args() + args.read(iprot) + iprot.readMessageEnd() + result = list_roles_result() + try: + result.success = self._handler.list_roles(args.principal_name, args.principal_type) + msg_type = TMessageType.REPLY + except TTransport.TTransportException: + raise + except MetaException as o1: + msg_type = TMessageType.REPLY + result.o1 = o1 + except TApplicationException as ex: + logging.exception("TApplication exception in handler") + msg_type = TMessageType.EXCEPTION + result = ex + except Exception: + logging.exception("Unexpected exception in handler") + msg_type = TMessageType.EXCEPTION + result = TApplicationException(TApplicationException.INTERNAL_ERROR, "Internal error") + oprot.writeMessageBegin("list_roles", msg_type, seqid) + result.write(oprot) + oprot.writeMessageEnd() + oprot.trans.flush() + + def process_grant_revoke_role(self, seqid, iprot, oprot): + args = grant_revoke_role_args() + args.read(iprot) + iprot.readMessageEnd() + result = grant_revoke_role_result() + try: + result.success = self._handler.grant_revoke_role(args.request) + msg_type = TMessageType.REPLY + except TTransport.TTransportException: + raise + except MetaException as o1: + msg_type = TMessageType.REPLY + result.o1 = o1 + except TApplicationException as ex: + logging.exception("TApplication exception in handler") + msg_type = TMessageType.EXCEPTION + result = ex + except Exception: + logging.exception("Unexpected exception in handler") + msg_type = TMessageType.EXCEPTION + result = TApplicationException(TApplicationException.INTERNAL_ERROR, "Internal error") + oprot.writeMessageBegin("grant_revoke_role", msg_type, seqid) + result.write(oprot) + oprot.writeMessageEnd() + oprot.trans.flush() + + def process_get_principals_in_role(self, seqid, iprot, oprot): + args = get_principals_in_role_args() + args.read(iprot) + iprot.readMessageEnd() + result = get_principals_in_role_result() + try: + result.success = self._handler.get_principals_in_role(args.request) + msg_type = TMessageType.REPLY + except TTransport.TTransportException: + raise + except MetaException as o1: + msg_type = TMessageType.REPLY + result.o1 = o1 + except TApplicationException as ex: + logging.exception("TApplication exception in handler") + msg_type = TMessageType.EXCEPTION + result = ex + except Exception: + logging.exception("Unexpected exception in handler") + msg_type = TMessageType.EXCEPTION + result = TApplicationException(TApplicationException.INTERNAL_ERROR, "Internal error") + oprot.writeMessageBegin("get_principals_in_role", msg_type, seqid) + result.write(oprot) + oprot.writeMessageEnd() + oprot.trans.flush() + + def process_get_role_grants_for_principal(self, seqid, iprot, oprot): + args = get_role_grants_for_principal_args() + args.read(iprot) + iprot.readMessageEnd() + result = get_role_grants_for_principal_result() + try: + result.success = self._handler.get_role_grants_for_principal(args.request) + msg_type = TMessageType.REPLY + except TTransport.TTransportException: + raise + except MetaException as o1: + msg_type = TMessageType.REPLY + result.o1 = o1 + except TApplicationException as ex: + logging.exception("TApplication exception in handler") + msg_type = TMessageType.EXCEPTION + result = ex + except Exception: + logging.exception("Unexpected exception in handler") + msg_type = TMessageType.EXCEPTION + result = TApplicationException(TApplicationException.INTERNAL_ERROR, "Internal error") + oprot.writeMessageBegin("get_role_grants_for_principal", msg_type, seqid) + result.write(oprot) + oprot.writeMessageEnd() + oprot.trans.flush() + + def process_get_privilege_set(self, seqid, iprot, oprot): + args = get_privilege_set_args() + args.read(iprot) + iprot.readMessageEnd() + result = get_privilege_set_result() + try: + result.success = self._handler.get_privilege_set(args.hiveObject, args.user_name, args.group_names) + msg_type = TMessageType.REPLY + except TTransport.TTransportException: + raise + except MetaException as o1: + msg_type = TMessageType.REPLY + result.o1 = o1 + except TApplicationException as ex: + logging.exception("TApplication exception in handler") + msg_type = TMessageType.EXCEPTION + result = ex + except Exception: + logging.exception("Unexpected exception in handler") + msg_type = TMessageType.EXCEPTION + result = TApplicationException(TApplicationException.INTERNAL_ERROR, "Internal error") + oprot.writeMessageBegin("get_privilege_set", msg_type, seqid) + result.write(oprot) + oprot.writeMessageEnd() + oprot.trans.flush() + + def process_list_privileges(self, seqid, iprot, oprot): + args = list_privileges_args() + args.read(iprot) + iprot.readMessageEnd() + result = list_privileges_result() + try: + result.success = self._handler.list_privileges(args.principal_name, args.principal_type, args.hiveObject) + msg_type = TMessageType.REPLY + except TTransport.TTransportException: + raise + except MetaException as o1: + msg_type = TMessageType.REPLY + result.o1 = o1 + except TApplicationException as ex: + logging.exception("TApplication exception in handler") + msg_type = TMessageType.EXCEPTION + result = ex + except Exception: + logging.exception("Unexpected exception in handler") + msg_type = TMessageType.EXCEPTION + result = TApplicationException(TApplicationException.INTERNAL_ERROR, "Internal error") + oprot.writeMessageBegin("list_privileges", msg_type, seqid) + result.write(oprot) + oprot.writeMessageEnd() + oprot.trans.flush() + + def process_grant_privileges(self, seqid, iprot, oprot): + args = grant_privileges_args() + args.read(iprot) + iprot.readMessageEnd() + result = grant_privileges_result() + try: + result.success = self._handler.grant_privileges(args.privileges) + msg_type = TMessageType.REPLY + except TTransport.TTransportException: + raise + except MetaException as o1: + msg_type = TMessageType.REPLY + result.o1 = o1 + except TApplicationException as ex: + logging.exception("TApplication exception in handler") + msg_type = TMessageType.EXCEPTION + result = ex + except Exception: + logging.exception("Unexpected exception in handler") + msg_type = TMessageType.EXCEPTION + result = TApplicationException(TApplicationException.INTERNAL_ERROR, "Internal error") + oprot.writeMessageBegin("grant_privileges", msg_type, seqid) + result.write(oprot) + oprot.writeMessageEnd() + oprot.trans.flush() + + def process_revoke_privileges(self, seqid, iprot, oprot): + args = revoke_privileges_args() + args.read(iprot) + iprot.readMessageEnd() + result = revoke_privileges_result() + try: + result.success = self._handler.revoke_privileges(args.privileges) + msg_type = TMessageType.REPLY + except TTransport.TTransportException: + raise + except MetaException as o1: + msg_type = TMessageType.REPLY + result.o1 = o1 + except TApplicationException as ex: + logging.exception("TApplication exception in handler") + msg_type = TMessageType.EXCEPTION + result = ex + except Exception: + logging.exception("Unexpected exception in handler") + msg_type = TMessageType.EXCEPTION + result = TApplicationException(TApplicationException.INTERNAL_ERROR, "Internal error") + oprot.writeMessageBegin("revoke_privileges", msg_type, seqid) + result.write(oprot) + oprot.writeMessageEnd() + oprot.trans.flush() + + def process_grant_revoke_privileges(self, seqid, iprot, oprot): + args = grant_revoke_privileges_args() + args.read(iprot) + iprot.readMessageEnd() + result = grant_revoke_privileges_result() + try: + result.success = self._handler.grant_revoke_privileges(args.request) + msg_type = TMessageType.REPLY + except TTransport.TTransportException: + raise + except MetaException as o1: + msg_type = TMessageType.REPLY + result.o1 = o1 + except TApplicationException as ex: + logging.exception("TApplication exception in handler") + msg_type = TMessageType.EXCEPTION + result = ex + except Exception: + logging.exception("Unexpected exception in handler") + msg_type = TMessageType.EXCEPTION + result = TApplicationException(TApplicationException.INTERNAL_ERROR, "Internal error") + oprot.writeMessageBegin("grant_revoke_privileges", msg_type, seqid) + result.write(oprot) + oprot.writeMessageEnd() + oprot.trans.flush() + + def process_refresh_privileges(self, seqid, iprot, oprot): + args = refresh_privileges_args() + args.read(iprot) + iprot.readMessageEnd() + result = refresh_privileges_result() + try: + result.success = self._handler.refresh_privileges(args.objToRefresh, args.authorizer, args.grantRequest) + msg_type = TMessageType.REPLY + except TTransport.TTransportException: + raise + except MetaException as o1: + msg_type = TMessageType.REPLY + result.o1 = o1 + except TApplicationException as ex: + logging.exception("TApplication exception in handler") + msg_type = TMessageType.EXCEPTION + result = ex + except Exception: + logging.exception("Unexpected exception in handler") + msg_type = TMessageType.EXCEPTION + result = TApplicationException(TApplicationException.INTERNAL_ERROR, "Internal error") + oprot.writeMessageBegin("refresh_privileges", msg_type, seqid) + result.write(oprot) + oprot.writeMessageEnd() + oprot.trans.flush() + + def process_set_ugi(self, seqid, iprot, oprot): + args = set_ugi_args() + args.read(iprot) + iprot.readMessageEnd() + result = set_ugi_result() + try: + result.success = self._handler.set_ugi(args.user_name, args.group_names) + msg_type = TMessageType.REPLY + except TTransport.TTransportException: + raise + except MetaException as o1: + msg_type = TMessageType.REPLY + result.o1 = o1 + except TApplicationException as ex: + logging.exception("TApplication exception in handler") + msg_type = TMessageType.EXCEPTION + result = ex + except Exception: + logging.exception("Unexpected exception in handler") + msg_type = TMessageType.EXCEPTION + result = TApplicationException(TApplicationException.INTERNAL_ERROR, "Internal error") + oprot.writeMessageBegin("set_ugi", msg_type, seqid) + result.write(oprot) + oprot.writeMessageEnd() + oprot.trans.flush() + + def process_get_delegation_token(self, seqid, iprot, oprot): + args = get_delegation_token_args() + args.read(iprot) + iprot.readMessageEnd() + result = get_delegation_token_result() + try: + result.success = self._handler.get_delegation_token(args.token_owner, args.renewer_kerberos_principal_name) + msg_type = TMessageType.REPLY + except TTransport.TTransportException: + raise + except MetaException as o1: + msg_type = TMessageType.REPLY + result.o1 = o1 + except TApplicationException as ex: + logging.exception("TApplication exception in handler") + msg_type = TMessageType.EXCEPTION + result = ex + except Exception: + logging.exception("Unexpected exception in handler") + msg_type = TMessageType.EXCEPTION + result = TApplicationException(TApplicationException.INTERNAL_ERROR, "Internal error") + oprot.writeMessageBegin("get_delegation_token", msg_type, seqid) + result.write(oprot) + oprot.writeMessageEnd() + oprot.trans.flush() + + def process_renew_delegation_token(self, seqid, iprot, oprot): + args = renew_delegation_token_args() + args.read(iprot) + iprot.readMessageEnd() + result = renew_delegation_token_result() + try: + result.success = self._handler.renew_delegation_token(args.token_str_form) + msg_type = TMessageType.REPLY + except TTransport.TTransportException: + raise + except MetaException as o1: + msg_type = TMessageType.REPLY + result.o1 = o1 + except TApplicationException as ex: + logging.exception("TApplication exception in handler") + msg_type = TMessageType.EXCEPTION + result = ex + except Exception: + logging.exception("Unexpected exception in handler") + msg_type = TMessageType.EXCEPTION + result = TApplicationException(TApplicationException.INTERNAL_ERROR, "Internal error") + oprot.writeMessageBegin("renew_delegation_token", msg_type, seqid) + result.write(oprot) + oprot.writeMessageEnd() + oprot.trans.flush() + + def process_cancel_delegation_token(self, seqid, iprot, oprot): + args = cancel_delegation_token_args() + args.read(iprot) + iprot.readMessageEnd() + result = cancel_delegation_token_result() + try: + self._handler.cancel_delegation_token(args.token_str_form) + msg_type = TMessageType.REPLY + except TTransport.TTransportException: + raise + except MetaException as o1: + msg_type = TMessageType.REPLY + result.o1 = o1 + except TApplicationException as ex: + logging.exception("TApplication exception in handler") + msg_type = TMessageType.EXCEPTION + result = ex + except Exception: + logging.exception("Unexpected exception in handler") + msg_type = TMessageType.EXCEPTION + result = TApplicationException(TApplicationException.INTERNAL_ERROR, "Internal error") + oprot.writeMessageBegin("cancel_delegation_token", msg_type, seqid) + result.write(oprot) + oprot.writeMessageEnd() + oprot.trans.flush() + + def process_add_token(self, seqid, iprot, oprot): + args = add_token_args() + args.read(iprot) + iprot.readMessageEnd() + result = add_token_result() + try: + result.success = self._handler.add_token(args.token_identifier, args.delegation_token) + msg_type = TMessageType.REPLY + except TTransport.TTransportException: + raise + except TApplicationException as ex: + logging.exception("TApplication exception in handler") + msg_type = TMessageType.EXCEPTION + result = ex + except Exception: + logging.exception("Unexpected exception in handler") + msg_type = TMessageType.EXCEPTION + result = TApplicationException(TApplicationException.INTERNAL_ERROR, "Internal error") + oprot.writeMessageBegin("add_token", msg_type, seqid) + result.write(oprot) + oprot.writeMessageEnd() + oprot.trans.flush() + + def process_remove_token(self, seqid, iprot, oprot): + args = remove_token_args() + args.read(iprot) + iprot.readMessageEnd() + result = remove_token_result() + try: + result.success = self._handler.remove_token(args.token_identifier) + msg_type = TMessageType.REPLY + except TTransport.TTransportException: + raise + except TApplicationException as ex: + logging.exception("TApplication exception in handler") + msg_type = TMessageType.EXCEPTION + result = ex + except Exception: + logging.exception("Unexpected exception in handler") + msg_type = TMessageType.EXCEPTION + result = TApplicationException(TApplicationException.INTERNAL_ERROR, "Internal error") + oprot.writeMessageBegin("remove_token", msg_type, seqid) + result.write(oprot) + oprot.writeMessageEnd() + oprot.trans.flush() + + def process_get_token(self, seqid, iprot, oprot): + args = get_token_args() + args.read(iprot) + iprot.readMessageEnd() + result = get_token_result() + try: + result.success = self._handler.get_token(args.token_identifier) + msg_type = TMessageType.REPLY + except TTransport.TTransportException: + raise + except TApplicationException as ex: + logging.exception("TApplication exception in handler") + msg_type = TMessageType.EXCEPTION + result = ex + except Exception: + logging.exception("Unexpected exception in handler") + msg_type = TMessageType.EXCEPTION + result = TApplicationException(TApplicationException.INTERNAL_ERROR, "Internal error") + oprot.writeMessageBegin("get_token", msg_type, seqid) + result.write(oprot) + oprot.writeMessageEnd() + oprot.trans.flush() + + def process_get_all_token_identifiers(self, seqid, iprot, oprot): + args = get_all_token_identifiers_args() + args.read(iprot) + iprot.readMessageEnd() + result = get_all_token_identifiers_result() + try: + result.success = self._handler.get_all_token_identifiers() + msg_type = TMessageType.REPLY + except TTransport.TTransportException: + raise + except TApplicationException as ex: + logging.exception("TApplication exception in handler") + msg_type = TMessageType.EXCEPTION + result = ex + except Exception: + logging.exception("Unexpected exception in handler") + msg_type = TMessageType.EXCEPTION + result = TApplicationException(TApplicationException.INTERNAL_ERROR, "Internal error") + oprot.writeMessageBegin("get_all_token_identifiers", msg_type, seqid) + result.write(oprot) + oprot.writeMessageEnd() + oprot.trans.flush() + + def process_add_master_key(self, seqid, iprot, oprot): + args = add_master_key_args() + args.read(iprot) + iprot.readMessageEnd() + result = add_master_key_result() + try: + result.success = self._handler.add_master_key(args.key) + msg_type = TMessageType.REPLY + except TTransport.TTransportException: + raise + except MetaException as o1: + msg_type = TMessageType.REPLY + result.o1 = o1 + except TApplicationException as ex: + logging.exception("TApplication exception in handler") + msg_type = TMessageType.EXCEPTION + result = ex + except Exception: + logging.exception("Unexpected exception in handler") + msg_type = TMessageType.EXCEPTION + result = TApplicationException(TApplicationException.INTERNAL_ERROR, "Internal error") + oprot.writeMessageBegin("add_master_key", msg_type, seqid) + result.write(oprot) + oprot.writeMessageEnd() + oprot.trans.flush() + + def process_update_master_key(self, seqid, iprot, oprot): + args = update_master_key_args() + args.read(iprot) + iprot.readMessageEnd() + result = update_master_key_result() + try: + self._handler.update_master_key(args.seq_number, args.key) + msg_type = TMessageType.REPLY + except TTransport.TTransportException: + raise + except NoSuchObjectException as o1: + msg_type = TMessageType.REPLY + result.o1 = o1 + except MetaException as o2: + msg_type = TMessageType.REPLY + result.o2 = o2 + except TApplicationException as ex: + logging.exception("TApplication exception in handler") + msg_type = TMessageType.EXCEPTION + result = ex + except Exception: + logging.exception("Unexpected exception in handler") + msg_type = TMessageType.EXCEPTION + result = TApplicationException(TApplicationException.INTERNAL_ERROR, "Internal error") + oprot.writeMessageBegin("update_master_key", msg_type, seqid) + result.write(oprot) + oprot.writeMessageEnd() + oprot.trans.flush() + + def process_remove_master_key(self, seqid, iprot, oprot): + args = remove_master_key_args() + args.read(iprot) + iprot.readMessageEnd() + result = remove_master_key_result() + try: + result.success = self._handler.remove_master_key(args.key_seq) + msg_type = TMessageType.REPLY + except TTransport.TTransportException: + raise + except TApplicationException as ex: + logging.exception("TApplication exception in handler") + msg_type = TMessageType.EXCEPTION + result = ex + except Exception: + logging.exception("Unexpected exception in handler") + msg_type = TMessageType.EXCEPTION + result = TApplicationException(TApplicationException.INTERNAL_ERROR, "Internal error") + oprot.writeMessageBegin("remove_master_key", msg_type, seqid) + result.write(oprot) + oprot.writeMessageEnd() + oprot.trans.flush() + + def process_get_master_keys(self, seqid, iprot, oprot): + args = get_master_keys_args() + args.read(iprot) + iprot.readMessageEnd() + result = get_master_keys_result() + try: + result.success = self._handler.get_master_keys() + msg_type = TMessageType.REPLY + except TTransport.TTransportException: + raise + except TApplicationException as ex: + logging.exception("TApplication exception in handler") + msg_type = TMessageType.EXCEPTION + result = ex + except Exception: + logging.exception("Unexpected exception in handler") + msg_type = TMessageType.EXCEPTION + result = TApplicationException(TApplicationException.INTERNAL_ERROR, "Internal error") + oprot.writeMessageBegin("get_master_keys", msg_type, seqid) + result.write(oprot) + oprot.writeMessageEnd() + oprot.trans.flush() + + def process_get_open_txns(self, seqid, iprot, oprot): + args = get_open_txns_args() + args.read(iprot) + iprot.readMessageEnd() + result = get_open_txns_result() + try: + result.success = self._handler.get_open_txns() + msg_type = TMessageType.REPLY + except TTransport.TTransportException: + raise + except TApplicationException as ex: + logging.exception("TApplication exception in handler") + msg_type = TMessageType.EXCEPTION + result = ex + except Exception: + logging.exception("Unexpected exception in handler") + msg_type = TMessageType.EXCEPTION + result = TApplicationException(TApplicationException.INTERNAL_ERROR, "Internal error") + oprot.writeMessageBegin("get_open_txns", msg_type, seqid) + result.write(oprot) + oprot.writeMessageEnd() + oprot.trans.flush() + + def process_get_open_txns_info(self, seqid, iprot, oprot): + args = get_open_txns_info_args() + args.read(iprot) + iprot.readMessageEnd() + result = get_open_txns_info_result() + try: + result.success = self._handler.get_open_txns_info() + msg_type = TMessageType.REPLY + except TTransport.TTransportException: + raise + except TApplicationException as ex: + logging.exception("TApplication exception in handler") + msg_type = TMessageType.EXCEPTION + result = ex + except Exception: + logging.exception("Unexpected exception in handler") + msg_type = TMessageType.EXCEPTION + result = TApplicationException(TApplicationException.INTERNAL_ERROR, "Internal error") + oprot.writeMessageBegin("get_open_txns_info", msg_type, seqid) + result.write(oprot) + oprot.writeMessageEnd() + oprot.trans.flush() + + def process_open_txns(self, seqid, iprot, oprot): + args = open_txns_args() + args.read(iprot) + iprot.readMessageEnd() + result = open_txns_result() + try: + result.success = self._handler.open_txns(args.rqst) + msg_type = TMessageType.REPLY + except TTransport.TTransportException: + raise + except TApplicationException as ex: + logging.exception("TApplication exception in handler") + msg_type = TMessageType.EXCEPTION + result = ex + except Exception: + logging.exception("Unexpected exception in handler") + msg_type = TMessageType.EXCEPTION + result = TApplicationException(TApplicationException.INTERNAL_ERROR, "Internal error") + oprot.writeMessageBegin("open_txns", msg_type, seqid) + result.write(oprot) + oprot.writeMessageEnd() + oprot.trans.flush() + + def process_abort_txn(self, seqid, iprot, oprot): + args = abort_txn_args() + args.read(iprot) + iprot.readMessageEnd() + result = abort_txn_result() + try: + self._handler.abort_txn(args.rqst) + msg_type = TMessageType.REPLY + except TTransport.TTransportException: + raise + except NoSuchTxnException as o1: + msg_type = TMessageType.REPLY + result.o1 = o1 + except TApplicationException as ex: + logging.exception("TApplication exception in handler") + msg_type = TMessageType.EXCEPTION + result = ex + except Exception: + logging.exception("Unexpected exception in handler") + msg_type = TMessageType.EXCEPTION + result = TApplicationException(TApplicationException.INTERNAL_ERROR, "Internal error") + oprot.writeMessageBegin("abort_txn", msg_type, seqid) + result.write(oprot) + oprot.writeMessageEnd() + oprot.trans.flush() + + def process_abort_txns(self, seqid, iprot, oprot): + args = abort_txns_args() + args.read(iprot) + iprot.readMessageEnd() + result = abort_txns_result() + try: + self._handler.abort_txns(args.rqst) + msg_type = TMessageType.REPLY + except TTransport.TTransportException: + raise + except NoSuchTxnException as o1: + msg_type = TMessageType.REPLY + result.o1 = o1 + except TApplicationException as ex: + logging.exception("TApplication exception in handler") + msg_type = TMessageType.EXCEPTION + result = ex + except Exception: + logging.exception("Unexpected exception in handler") + msg_type = TMessageType.EXCEPTION + result = TApplicationException(TApplicationException.INTERNAL_ERROR, "Internal error") + oprot.writeMessageBegin("abort_txns", msg_type, seqid) + result.write(oprot) + oprot.writeMessageEnd() + oprot.trans.flush() + + def process_commit_txn(self, seqid, iprot, oprot): + args = commit_txn_args() + args.read(iprot) + iprot.readMessageEnd() + result = commit_txn_result() + try: + self._handler.commit_txn(args.rqst) + msg_type = TMessageType.REPLY + except TTransport.TTransportException: + raise + except NoSuchTxnException as o1: + msg_type = TMessageType.REPLY + result.o1 = o1 + except TxnAbortedException as o2: + msg_type = TMessageType.REPLY + result.o2 = o2 + except TApplicationException as ex: + logging.exception("TApplication exception in handler") + msg_type = TMessageType.EXCEPTION + result = ex + except Exception: + logging.exception("Unexpected exception in handler") + msg_type = TMessageType.EXCEPTION + result = TApplicationException(TApplicationException.INTERNAL_ERROR, "Internal error") + oprot.writeMessageBegin("commit_txn", msg_type, seqid) + result.write(oprot) + oprot.writeMessageEnd() + oprot.trans.flush() + + def process_get_latest_txnid_in_conflict(self, seqid, iprot, oprot): + args = get_latest_txnid_in_conflict_args() + args.read(iprot) + iprot.readMessageEnd() + result = get_latest_txnid_in_conflict_result() + try: + result.success = self._handler.get_latest_txnid_in_conflict(args.txnId) + msg_type = TMessageType.REPLY + except TTransport.TTransportException: + raise + except MetaException as o1: + msg_type = TMessageType.REPLY + result.o1 = o1 + except TApplicationException as ex: + logging.exception("TApplication exception in handler") + msg_type = TMessageType.EXCEPTION + result = ex + except Exception: + logging.exception("Unexpected exception in handler") + msg_type = TMessageType.EXCEPTION + result = TApplicationException(TApplicationException.INTERNAL_ERROR, "Internal error") + oprot.writeMessageBegin("get_latest_txnid_in_conflict", msg_type, seqid) + result.write(oprot) + oprot.writeMessageEnd() + oprot.trans.flush() + + def process_repl_tbl_writeid_state(self, seqid, iprot, oprot): + args = repl_tbl_writeid_state_args() + args.read(iprot) + iprot.readMessageEnd() + result = repl_tbl_writeid_state_result() + try: + self._handler.repl_tbl_writeid_state(args.rqst) + msg_type = TMessageType.REPLY + except TTransport.TTransportException: + raise + except TApplicationException as ex: + logging.exception("TApplication exception in handler") + msg_type = TMessageType.EXCEPTION + result = ex + except Exception: + logging.exception("Unexpected exception in handler") + msg_type = TMessageType.EXCEPTION + result = TApplicationException(TApplicationException.INTERNAL_ERROR, "Internal error") + oprot.writeMessageBegin("repl_tbl_writeid_state", msg_type, seqid) + result.write(oprot) + oprot.writeMessageEnd() + oprot.trans.flush() + + def process_get_valid_write_ids(self, seqid, iprot, oprot): + args = get_valid_write_ids_args() + args.read(iprot) + iprot.readMessageEnd() + result = get_valid_write_ids_result() + try: + result.success = self._handler.get_valid_write_ids(args.rqst) + msg_type = TMessageType.REPLY + except TTransport.TTransportException: + raise + except NoSuchTxnException as o1: + msg_type = TMessageType.REPLY + result.o1 = o1 + except MetaException as o2: + msg_type = TMessageType.REPLY + result.o2 = o2 + except TApplicationException as ex: + logging.exception("TApplication exception in handler") + msg_type = TMessageType.EXCEPTION + result = ex + except Exception: + logging.exception("Unexpected exception in handler") + msg_type = TMessageType.EXCEPTION + result = TApplicationException(TApplicationException.INTERNAL_ERROR, "Internal error") + oprot.writeMessageBegin("get_valid_write_ids", msg_type, seqid) + result.write(oprot) + oprot.writeMessageEnd() + oprot.trans.flush() + + def process_allocate_table_write_ids(self, seqid, iprot, oprot): + args = allocate_table_write_ids_args() + args.read(iprot) + iprot.readMessageEnd() + result = allocate_table_write_ids_result() + try: + result.success = self._handler.allocate_table_write_ids(args.rqst) + msg_type = TMessageType.REPLY + except TTransport.TTransportException: + raise + except NoSuchTxnException as o1: + msg_type = TMessageType.REPLY + result.o1 = o1 + except TxnAbortedException as o2: + msg_type = TMessageType.REPLY + result.o2 = o2 + except MetaException as o3: + msg_type = TMessageType.REPLY + result.o3 = o3 + except TApplicationException as ex: + logging.exception("TApplication exception in handler") + msg_type = TMessageType.EXCEPTION + result = ex + except Exception: + logging.exception("Unexpected exception in handler") + msg_type = TMessageType.EXCEPTION + result = TApplicationException(TApplicationException.INTERNAL_ERROR, "Internal error") + oprot.writeMessageBegin("allocate_table_write_ids", msg_type, seqid) + result.write(oprot) + oprot.writeMessageEnd() + oprot.trans.flush() + + def process_get_max_allocated_table_write_id(self, seqid, iprot, oprot): + args = get_max_allocated_table_write_id_args() + args.read(iprot) + iprot.readMessageEnd() + result = get_max_allocated_table_write_id_result() + try: + result.success = self._handler.get_max_allocated_table_write_id(args.rqst) + msg_type = TMessageType.REPLY + except TTransport.TTransportException: + raise + except MetaException as o1: + msg_type = TMessageType.REPLY + result.o1 = o1 + except TApplicationException as ex: + logging.exception("TApplication exception in handler") + msg_type = TMessageType.EXCEPTION + result = ex + except Exception: + logging.exception("Unexpected exception in handler") + msg_type = TMessageType.EXCEPTION + result = TApplicationException(TApplicationException.INTERNAL_ERROR, "Internal error") + oprot.writeMessageBegin("get_max_allocated_table_write_id", msg_type, seqid) + result.write(oprot) + oprot.writeMessageEnd() + oprot.trans.flush() + + def process_seed_write_id(self, seqid, iprot, oprot): + args = seed_write_id_args() + args.read(iprot) + iprot.readMessageEnd() + result = seed_write_id_result() + try: + self._handler.seed_write_id(args.rqst) + msg_type = TMessageType.REPLY + except TTransport.TTransportException: + raise + except MetaException as o1: + msg_type = TMessageType.REPLY + result.o1 = o1 + except TApplicationException as ex: + logging.exception("TApplication exception in handler") + msg_type = TMessageType.EXCEPTION + result = ex + except Exception: + logging.exception("Unexpected exception in handler") + msg_type = TMessageType.EXCEPTION + result = TApplicationException(TApplicationException.INTERNAL_ERROR, "Internal error") + oprot.writeMessageBegin("seed_write_id", msg_type, seqid) + result.write(oprot) + oprot.writeMessageEnd() + oprot.trans.flush() + + def process_seed_txn_id(self, seqid, iprot, oprot): + args = seed_txn_id_args() + args.read(iprot) + iprot.readMessageEnd() + result = seed_txn_id_result() + try: + self._handler.seed_txn_id(args.rqst) + msg_type = TMessageType.REPLY + except TTransport.TTransportException: + raise + except MetaException as o1: + msg_type = TMessageType.REPLY + result.o1 = o1 + except TApplicationException as ex: + logging.exception("TApplication exception in handler") + msg_type = TMessageType.EXCEPTION + result = ex + except Exception: + logging.exception("Unexpected exception in handler") + msg_type = TMessageType.EXCEPTION + result = TApplicationException(TApplicationException.INTERNAL_ERROR, "Internal error") + oprot.writeMessageBegin("seed_txn_id", msg_type, seqid) + result.write(oprot) + oprot.writeMessageEnd() + oprot.trans.flush() + + def process_lock(self, seqid, iprot, oprot): + args = lock_args() + args.read(iprot) + iprot.readMessageEnd() + result = lock_result() + try: + result.success = self._handler.lock(args.rqst) + msg_type = TMessageType.REPLY + except TTransport.TTransportException: + raise + except NoSuchTxnException as o1: + msg_type = TMessageType.REPLY + result.o1 = o1 + except TxnAbortedException as o2: + msg_type = TMessageType.REPLY + result.o2 = o2 + except TApplicationException as ex: + logging.exception("TApplication exception in handler") + msg_type = TMessageType.EXCEPTION + result = ex + except Exception: + logging.exception("Unexpected exception in handler") + msg_type = TMessageType.EXCEPTION + result = TApplicationException(TApplicationException.INTERNAL_ERROR, "Internal error") + oprot.writeMessageBegin("lock", msg_type, seqid) + result.write(oprot) + oprot.writeMessageEnd() + oprot.trans.flush() + + def process_check_lock(self, seqid, iprot, oprot): + args = check_lock_args() + args.read(iprot) + iprot.readMessageEnd() + result = check_lock_result() + try: + result.success = self._handler.check_lock(args.rqst) + msg_type = TMessageType.REPLY + except TTransport.TTransportException: + raise + except NoSuchTxnException as o1: + msg_type = TMessageType.REPLY + result.o1 = o1 + except TxnAbortedException as o2: + msg_type = TMessageType.REPLY + result.o2 = o2 + except NoSuchLockException as o3: + msg_type = TMessageType.REPLY + result.o3 = o3 + except TApplicationException as ex: + logging.exception("TApplication exception in handler") + msg_type = TMessageType.EXCEPTION + result = ex + except Exception: + logging.exception("Unexpected exception in handler") + msg_type = TMessageType.EXCEPTION + result = TApplicationException(TApplicationException.INTERNAL_ERROR, "Internal error") + oprot.writeMessageBegin("check_lock", msg_type, seqid) + result.write(oprot) + oprot.writeMessageEnd() + oprot.trans.flush() + + def process_unlock(self, seqid, iprot, oprot): + args = unlock_args() + args.read(iprot) + iprot.readMessageEnd() + result = unlock_result() + try: + self._handler.unlock(args.rqst) + msg_type = TMessageType.REPLY + except TTransport.TTransportException: + raise + except NoSuchLockException as o1: + msg_type = TMessageType.REPLY + result.o1 = o1 + except TxnOpenException as o2: + msg_type = TMessageType.REPLY + result.o2 = o2 + except TApplicationException as ex: + logging.exception("TApplication exception in handler") + msg_type = TMessageType.EXCEPTION + result = ex + except Exception: + logging.exception("Unexpected exception in handler") + msg_type = TMessageType.EXCEPTION + result = TApplicationException(TApplicationException.INTERNAL_ERROR, "Internal error") + oprot.writeMessageBegin("unlock", msg_type, seqid) + result.write(oprot) + oprot.writeMessageEnd() + oprot.trans.flush() + + def process_show_locks(self, seqid, iprot, oprot): + args = show_locks_args() + args.read(iprot) + iprot.readMessageEnd() + result = show_locks_result() + try: + result.success = self._handler.show_locks(args.rqst) + msg_type = TMessageType.REPLY + except TTransport.TTransportException: + raise + except TApplicationException as ex: + logging.exception("TApplication exception in handler") + msg_type = TMessageType.EXCEPTION + result = ex + except Exception: + logging.exception("Unexpected exception in handler") + msg_type = TMessageType.EXCEPTION + result = TApplicationException(TApplicationException.INTERNAL_ERROR, "Internal error") + oprot.writeMessageBegin("show_locks", msg_type, seqid) + result.write(oprot) + oprot.writeMessageEnd() + oprot.trans.flush() + + def process_heartbeat(self, seqid, iprot, oprot): + args = heartbeat_args() + args.read(iprot) + iprot.readMessageEnd() + result = heartbeat_result() + try: + self._handler.heartbeat(args.ids) + msg_type = TMessageType.REPLY + except TTransport.TTransportException: + raise + except NoSuchLockException as o1: + msg_type = TMessageType.REPLY + result.o1 = o1 + except NoSuchTxnException as o2: + msg_type = TMessageType.REPLY + result.o2 = o2 + except TxnAbortedException as o3: + msg_type = TMessageType.REPLY + result.o3 = o3 + except TApplicationException as ex: + logging.exception("TApplication exception in handler") + msg_type = TMessageType.EXCEPTION + result = ex + except Exception: + logging.exception("Unexpected exception in handler") + msg_type = TMessageType.EXCEPTION + result = TApplicationException(TApplicationException.INTERNAL_ERROR, "Internal error") + oprot.writeMessageBegin("heartbeat", msg_type, seqid) + result.write(oprot) + oprot.writeMessageEnd() + oprot.trans.flush() + + def process_heartbeat_txn_range(self, seqid, iprot, oprot): + args = heartbeat_txn_range_args() + args.read(iprot) + iprot.readMessageEnd() + result = heartbeat_txn_range_result() + try: + result.success = self._handler.heartbeat_txn_range(args.txns) + msg_type = TMessageType.REPLY + except TTransport.TTransportException: + raise + except TApplicationException as ex: + logging.exception("TApplication exception in handler") + msg_type = TMessageType.EXCEPTION + result = ex + except Exception: + logging.exception("Unexpected exception in handler") + msg_type = TMessageType.EXCEPTION + result = TApplicationException(TApplicationException.INTERNAL_ERROR, "Internal error") + oprot.writeMessageBegin("heartbeat_txn_range", msg_type, seqid) + result.write(oprot) + oprot.writeMessageEnd() + oprot.trans.flush() + + def process_compact(self, seqid, iprot, oprot): + args = compact_args() + args.read(iprot) + iprot.readMessageEnd() + result = compact_result() + try: + self._handler.compact(args.rqst) + msg_type = TMessageType.REPLY + except TTransport.TTransportException: + raise + except TApplicationException as ex: + logging.exception("TApplication exception in handler") + msg_type = TMessageType.EXCEPTION + result = ex + except Exception: + logging.exception("Unexpected exception in handler") + msg_type = TMessageType.EXCEPTION + result = TApplicationException(TApplicationException.INTERNAL_ERROR, "Internal error") + oprot.writeMessageBegin("compact", msg_type, seqid) + result.write(oprot) + oprot.writeMessageEnd() + oprot.trans.flush() + + def process_compact2(self, seqid, iprot, oprot): + args = compact2_args() + args.read(iprot) + iprot.readMessageEnd() + result = compact2_result() + try: + result.success = self._handler.compact2(args.rqst) + msg_type = TMessageType.REPLY + except TTransport.TTransportException: + raise + except TApplicationException as ex: + logging.exception("TApplication exception in handler") + msg_type = TMessageType.EXCEPTION + result = ex + except Exception: + logging.exception("Unexpected exception in handler") + msg_type = TMessageType.EXCEPTION + result = TApplicationException(TApplicationException.INTERNAL_ERROR, "Internal error") + oprot.writeMessageBegin("compact2", msg_type, seqid) + result.write(oprot) + oprot.writeMessageEnd() + oprot.trans.flush() + + def process_show_compact(self, seqid, iprot, oprot): + args = show_compact_args() + args.read(iprot) + iprot.readMessageEnd() + result = show_compact_result() + try: + result.success = self._handler.show_compact(args.rqst) + msg_type = TMessageType.REPLY + except TTransport.TTransportException: + raise + except TApplicationException as ex: + logging.exception("TApplication exception in handler") + msg_type = TMessageType.EXCEPTION + result = ex + except Exception: + logging.exception("Unexpected exception in handler") + msg_type = TMessageType.EXCEPTION + result = TApplicationException(TApplicationException.INTERNAL_ERROR, "Internal error") + oprot.writeMessageBegin("show_compact", msg_type, seqid) + result.write(oprot) + oprot.writeMessageEnd() + oprot.trans.flush() + + def process_add_dynamic_partitions(self, seqid, iprot, oprot): + args = add_dynamic_partitions_args() + args.read(iprot) + iprot.readMessageEnd() + result = add_dynamic_partitions_result() + try: + self._handler.add_dynamic_partitions(args.rqst) + msg_type = TMessageType.REPLY + except TTransport.TTransportException: + raise + except NoSuchTxnException as o1: + msg_type = TMessageType.REPLY + result.o1 = o1 + except TxnAbortedException as o2: + msg_type = TMessageType.REPLY + result.o2 = o2 + except TApplicationException as ex: + logging.exception("TApplication exception in handler") + msg_type = TMessageType.EXCEPTION + result = ex + except Exception: + logging.exception("Unexpected exception in handler") + msg_type = TMessageType.EXCEPTION + result = TApplicationException(TApplicationException.INTERNAL_ERROR, "Internal error") + oprot.writeMessageBegin("add_dynamic_partitions", msg_type, seqid) + result.write(oprot) + oprot.writeMessageEnd() + oprot.trans.flush() + + def process_find_next_compact(self, seqid, iprot, oprot): + args = find_next_compact_args() + args.read(iprot) + iprot.readMessageEnd() + result = find_next_compact_result() + try: + result.success = self._handler.find_next_compact(args.workerId) + msg_type = TMessageType.REPLY + except TTransport.TTransportException: + raise + except MetaException as o1: + msg_type = TMessageType.REPLY + result.o1 = o1 + except TApplicationException as ex: + logging.exception("TApplication exception in handler") + msg_type = TMessageType.EXCEPTION + result = ex + except Exception: + logging.exception("Unexpected exception in handler") + msg_type = TMessageType.EXCEPTION + result = TApplicationException(TApplicationException.INTERNAL_ERROR, "Internal error") + oprot.writeMessageBegin("find_next_compact", msg_type, seqid) + result.write(oprot) + oprot.writeMessageEnd() + oprot.trans.flush() + + def process_find_next_compact2(self, seqid, iprot, oprot): + args = find_next_compact2_args() + args.read(iprot) + iprot.readMessageEnd() + result = find_next_compact2_result() + try: + result.success = self._handler.find_next_compact2(args.rqst) + msg_type = TMessageType.REPLY + except TTransport.TTransportException: + raise + except MetaException as o1: + msg_type = TMessageType.REPLY + result.o1 = o1 + except TApplicationException as ex: + logging.exception("TApplication exception in handler") + msg_type = TMessageType.EXCEPTION + result = ex + except Exception: + logging.exception("Unexpected exception in handler") + msg_type = TMessageType.EXCEPTION + result = TApplicationException(TApplicationException.INTERNAL_ERROR, "Internal error") + oprot.writeMessageBegin("find_next_compact2", msg_type, seqid) + result.write(oprot) + oprot.writeMessageEnd() + oprot.trans.flush() + + def process_update_compactor_state(self, seqid, iprot, oprot): + args = update_compactor_state_args() + args.read(iprot) + iprot.readMessageEnd() + result = update_compactor_state_result() + try: + self._handler.update_compactor_state(args.cr, args.txn_id) + msg_type = TMessageType.REPLY + except TTransport.TTransportException: + raise + except TApplicationException as ex: + logging.exception("TApplication exception in handler") + msg_type = TMessageType.EXCEPTION + result = ex + except Exception: + logging.exception("Unexpected exception in handler") + msg_type = TMessageType.EXCEPTION + result = TApplicationException(TApplicationException.INTERNAL_ERROR, "Internal error") + oprot.writeMessageBegin("update_compactor_state", msg_type, seqid) + result.write(oprot) + oprot.writeMessageEnd() + oprot.trans.flush() + + def process_find_columns_with_stats(self, seqid, iprot, oprot): + args = find_columns_with_stats_args() + args.read(iprot) + iprot.readMessageEnd() + result = find_columns_with_stats_result() + try: + result.success = self._handler.find_columns_with_stats(args.cr) + msg_type = TMessageType.REPLY + except TTransport.TTransportException: + raise + except TApplicationException as ex: + logging.exception("TApplication exception in handler") + msg_type = TMessageType.EXCEPTION + result = ex + except Exception: + logging.exception("Unexpected exception in handler") + msg_type = TMessageType.EXCEPTION + result = TApplicationException(TApplicationException.INTERNAL_ERROR, "Internal error") + oprot.writeMessageBegin("find_columns_with_stats", msg_type, seqid) + result.write(oprot) + oprot.writeMessageEnd() + oprot.trans.flush() + + def process_mark_cleaned(self, seqid, iprot, oprot): + args = mark_cleaned_args() + args.read(iprot) + iprot.readMessageEnd() + result = mark_cleaned_result() + try: + self._handler.mark_cleaned(args.cr) + msg_type = TMessageType.REPLY + except TTransport.TTransportException: + raise + except MetaException as o1: + msg_type = TMessageType.REPLY + result.o1 = o1 + except TApplicationException as ex: + logging.exception("TApplication exception in handler") + msg_type = TMessageType.EXCEPTION + result = ex + except Exception: + logging.exception("Unexpected exception in handler") + msg_type = TMessageType.EXCEPTION + result = TApplicationException(TApplicationException.INTERNAL_ERROR, "Internal error") + oprot.writeMessageBegin("mark_cleaned", msg_type, seqid) + result.write(oprot) + oprot.writeMessageEnd() + oprot.trans.flush() + + def process_mark_compacted(self, seqid, iprot, oprot): + args = mark_compacted_args() + args.read(iprot) + iprot.readMessageEnd() + result = mark_compacted_result() + try: + self._handler.mark_compacted(args.cr) + msg_type = TMessageType.REPLY + except TTransport.TTransportException: + raise + except MetaException as o1: + msg_type = TMessageType.REPLY + result.o1 = o1 + except TApplicationException as ex: + logging.exception("TApplication exception in handler") + msg_type = TMessageType.EXCEPTION + result = ex + except Exception: + logging.exception("Unexpected exception in handler") + msg_type = TMessageType.EXCEPTION + result = TApplicationException(TApplicationException.INTERNAL_ERROR, "Internal error") + oprot.writeMessageBegin("mark_compacted", msg_type, seqid) + result.write(oprot) + oprot.writeMessageEnd() + oprot.trans.flush() + + def process_mark_failed(self, seqid, iprot, oprot): + args = mark_failed_args() + args.read(iprot) + iprot.readMessageEnd() + result = mark_failed_result() + try: + self._handler.mark_failed(args.cr) + msg_type = TMessageType.REPLY + except TTransport.TTransportException: + raise + except MetaException as o1: + msg_type = TMessageType.REPLY + result.o1 = o1 + except TApplicationException as ex: + logging.exception("TApplication exception in handler") + msg_type = TMessageType.EXCEPTION + result = ex + except Exception: + logging.exception("Unexpected exception in handler") + msg_type = TMessageType.EXCEPTION + result = TApplicationException(TApplicationException.INTERNAL_ERROR, "Internal error") + oprot.writeMessageBegin("mark_failed", msg_type, seqid) + result.write(oprot) + oprot.writeMessageEnd() + oprot.trans.flush() + + def process_mark_refused(self, seqid, iprot, oprot): + args = mark_refused_args() + args.read(iprot) + iprot.readMessageEnd() + result = mark_refused_result() + try: + self._handler.mark_refused(args.cr) + msg_type = TMessageType.REPLY + except TTransport.TTransportException: + raise + except MetaException as o1: + msg_type = TMessageType.REPLY + result.o1 = o1 + except TApplicationException as ex: + logging.exception("TApplication exception in handler") + msg_type = TMessageType.EXCEPTION + result = ex + except Exception: + logging.exception("Unexpected exception in handler") + msg_type = TMessageType.EXCEPTION + result = TApplicationException(TApplicationException.INTERNAL_ERROR, "Internal error") + oprot.writeMessageBegin("mark_refused", msg_type, seqid) + result.write(oprot) + oprot.writeMessageEnd() + oprot.trans.flush() + + def process_update_compaction_metrics_data(self, seqid, iprot, oprot): + args = update_compaction_metrics_data_args() + args.read(iprot) + iprot.readMessageEnd() + result = update_compaction_metrics_data_result() + try: + result.success = self._handler.update_compaction_metrics_data(args.data) + msg_type = TMessageType.REPLY + except TTransport.TTransportException: + raise + except MetaException as o1: + msg_type = TMessageType.REPLY + result.o1 = o1 + except TApplicationException as ex: + logging.exception("TApplication exception in handler") + msg_type = TMessageType.EXCEPTION + result = ex + except Exception: + logging.exception("Unexpected exception in handler") + msg_type = TMessageType.EXCEPTION + result = TApplicationException(TApplicationException.INTERNAL_ERROR, "Internal error") + oprot.writeMessageBegin("update_compaction_metrics_data", msg_type, seqid) + result.write(oprot) + oprot.writeMessageEnd() + oprot.trans.flush() + + def process_remove_compaction_metrics_data(self, seqid, iprot, oprot): + args = remove_compaction_metrics_data_args() + args.read(iprot) + iprot.readMessageEnd() + result = remove_compaction_metrics_data_result() + try: + self._handler.remove_compaction_metrics_data(args.request) + msg_type = TMessageType.REPLY + except TTransport.TTransportException: + raise + except MetaException as o1: + msg_type = TMessageType.REPLY + result.o1 = o1 + except TApplicationException as ex: + logging.exception("TApplication exception in handler") + msg_type = TMessageType.EXCEPTION + result = ex + except Exception: + logging.exception("Unexpected exception in handler") + msg_type = TMessageType.EXCEPTION + result = TApplicationException(TApplicationException.INTERNAL_ERROR, "Internal error") + oprot.writeMessageBegin("remove_compaction_metrics_data", msg_type, seqid) + result.write(oprot) + oprot.writeMessageEnd() + oprot.trans.flush() + + def process_set_hadoop_jobid(self, seqid, iprot, oprot): + args = set_hadoop_jobid_args() + args.read(iprot) + iprot.readMessageEnd() + result = set_hadoop_jobid_result() + try: + self._handler.set_hadoop_jobid(args.jobId, args.cq_id) + msg_type = TMessageType.REPLY + except TTransport.TTransportException: + raise + except TApplicationException as ex: + logging.exception("TApplication exception in handler") + msg_type = TMessageType.EXCEPTION + result = ex + except Exception: + logging.exception("Unexpected exception in handler") + msg_type = TMessageType.EXCEPTION + result = TApplicationException(TApplicationException.INTERNAL_ERROR, "Internal error") + oprot.writeMessageBegin("set_hadoop_jobid", msg_type, seqid) + result.write(oprot) + oprot.writeMessageEnd() + oprot.trans.flush() + + def process_get_latest_committed_compaction_info(self, seqid, iprot, oprot): + args = get_latest_committed_compaction_info_args() + args.read(iprot) + iprot.readMessageEnd() + result = get_latest_committed_compaction_info_result() + try: + result.success = self._handler.get_latest_committed_compaction_info(args.rqst) + msg_type = TMessageType.REPLY + except TTransport.TTransportException: + raise + except TApplicationException as ex: + logging.exception("TApplication exception in handler") + msg_type = TMessageType.EXCEPTION + result = ex + except Exception: + logging.exception("Unexpected exception in handler") + msg_type = TMessageType.EXCEPTION + result = TApplicationException(TApplicationException.INTERNAL_ERROR, "Internal error") + oprot.writeMessageBegin("get_latest_committed_compaction_info", msg_type, seqid) + result.write(oprot) + oprot.writeMessageEnd() + oprot.trans.flush() + + def process_get_next_notification(self, seqid, iprot, oprot): + args = get_next_notification_args() + args.read(iprot) + iprot.readMessageEnd() + result = get_next_notification_result() + try: + result.success = self._handler.get_next_notification(args.rqst) + msg_type = TMessageType.REPLY + except TTransport.TTransportException: + raise + except TApplicationException as ex: + logging.exception("TApplication exception in handler") + msg_type = TMessageType.EXCEPTION + result = ex + except Exception: + logging.exception("Unexpected exception in handler") + msg_type = TMessageType.EXCEPTION + result = TApplicationException(TApplicationException.INTERNAL_ERROR, "Internal error") + oprot.writeMessageBegin("get_next_notification", msg_type, seqid) + result.write(oprot) + oprot.writeMessageEnd() + oprot.trans.flush() + + def process_get_current_notificationEventId(self, seqid, iprot, oprot): + args = get_current_notificationEventId_args() + args.read(iprot) + iprot.readMessageEnd() + result = get_current_notificationEventId_result() + try: + result.success = self._handler.get_current_notificationEventId() + msg_type = TMessageType.REPLY + except TTransport.TTransportException: + raise + except TApplicationException as ex: + logging.exception("TApplication exception in handler") + msg_type = TMessageType.EXCEPTION + result = ex + except Exception: + logging.exception("Unexpected exception in handler") + msg_type = TMessageType.EXCEPTION + result = TApplicationException(TApplicationException.INTERNAL_ERROR, "Internal error") + oprot.writeMessageBegin("get_current_notificationEventId", msg_type, seqid) + result.write(oprot) + oprot.writeMessageEnd() + oprot.trans.flush() + + def process_get_notification_events_count(self, seqid, iprot, oprot): + args = get_notification_events_count_args() + args.read(iprot) + iprot.readMessageEnd() + result = get_notification_events_count_result() + try: + result.success = self._handler.get_notification_events_count(args.rqst) + msg_type = TMessageType.REPLY + except TTransport.TTransportException: + raise + except TApplicationException as ex: + logging.exception("TApplication exception in handler") + msg_type = TMessageType.EXCEPTION + result = ex + except Exception: + logging.exception("Unexpected exception in handler") + msg_type = TMessageType.EXCEPTION + result = TApplicationException(TApplicationException.INTERNAL_ERROR, "Internal error") + oprot.writeMessageBegin("get_notification_events_count", msg_type, seqid) + result.write(oprot) + oprot.writeMessageEnd() + oprot.trans.flush() + + def process_fire_listener_event(self, seqid, iprot, oprot): + args = fire_listener_event_args() + args.read(iprot) + iprot.readMessageEnd() + result = fire_listener_event_result() + try: + result.success = self._handler.fire_listener_event(args.rqst) + msg_type = TMessageType.REPLY + except TTransport.TTransportException: + raise + except TApplicationException as ex: + logging.exception("TApplication exception in handler") + msg_type = TMessageType.EXCEPTION + result = ex + except Exception: + logging.exception("Unexpected exception in handler") + msg_type = TMessageType.EXCEPTION + result = TApplicationException(TApplicationException.INTERNAL_ERROR, "Internal error") + oprot.writeMessageBegin("fire_listener_event", msg_type, seqid) + result.write(oprot) + oprot.writeMessageEnd() + oprot.trans.flush() + + def process_flushCache(self, seqid, iprot, oprot): + args = flushCache_args() + args.read(iprot) + iprot.readMessageEnd() + result = flushCache_result() + try: + self._handler.flushCache() + msg_type = TMessageType.REPLY + except TTransport.TTransportException: + raise + except TApplicationException as ex: + logging.exception("TApplication exception in handler") + msg_type = TMessageType.EXCEPTION + result = ex + except Exception: + logging.exception("Unexpected exception in handler") + msg_type = TMessageType.EXCEPTION + result = TApplicationException(TApplicationException.INTERNAL_ERROR, "Internal error") + oprot.writeMessageBegin("flushCache", msg_type, seqid) + result.write(oprot) + oprot.writeMessageEnd() + oprot.trans.flush() + + def process_add_write_notification_log(self, seqid, iprot, oprot): + args = add_write_notification_log_args() + args.read(iprot) + iprot.readMessageEnd() + result = add_write_notification_log_result() + try: + result.success = self._handler.add_write_notification_log(args.rqst) + msg_type = TMessageType.REPLY + except TTransport.TTransportException: + raise + except TApplicationException as ex: + logging.exception("TApplication exception in handler") + msg_type = TMessageType.EXCEPTION + result = ex + except Exception: + logging.exception("Unexpected exception in handler") + msg_type = TMessageType.EXCEPTION + result = TApplicationException(TApplicationException.INTERNAL_ERROR, "Internal error") + oprot.writeMessageBegin("add_write_notification_log", msg_type, seqid) + result.write(oprot) + oprot.writeMessageEnd() + oprot.trans.flush() + + def process_add_write_notification_log_in_batch(self, seqid, iprot, oprot): + args = add_write_notification_log_in_batch_args() + args.read(iprot) + iprot.readMessageEnd() + result = add_write_notification_log_in_batch_result() + try: + result.success = self._handler.add_write_notification_log_in_batch(args.rqst) + msg_type = TMessageType.REPLY + except TTransport.TTransportException: + raise + except TApplicationException as ex: + logging.exception("TApplication exception in handler") + msg_type = TMessageType.EXCEPTION + result = ex + except Exception: + logging.exception("Unexpected exception in handler") + msg_type = TMessageType.EXCEPTION + result = TApplicationException(TApplicationException.INTERNAL_ERROR, "Internal error") + oprot.writeMessageBegin("add_write_notification_log_in_batch", msg_type, seqid) + result.write(oprot) + oprot.writeMessageEnd() + oprot.trans.flush() + + def process_cm_recycle(self, seqid, iprot, oprot): + args = cm_recycle_args() + args.read(iprot) + iprot.readMessageEnd() + result = cm_recycle_result() + try: + result.success = self._handler.cm_recycle(args.request) + msg_type = TMessageType.REPLY + except TTransport.TTransportException: + raise + except MetaException as o1: + msg_type = TMessageType.REPLY + result.o1 = o1 + except TApplicationException as ex: + logging.exception("TApplication exception in handler") + msg_type = TMessageType.EXCEPTION + result = ex + except Exception: + logging.exception("Unexpected exception in handler") + msg_type = TMessageType.EXCEPTION + result = TApplicationException(TApplicationException.INTERNAL_ERROR, "Internal error") + oprot.writeMessageBegin("cm_recycle", msg_type, seqid) + result.write(oprot) + oprot.writeMessageEnd() + oprot.trans.flush() + + def process_get_file_metadata_by_expr(self, seqid, iprot, oprot): + args = get_file_metadata_by_expr_args() + args.read(iprot) + iprot.readMessageEnd() + result = get_file_metadata_by_expr_result() + try: + result.success = self._handler.get_file_metadata_by_expr(args.req) + msg_type = TMessageType.REPLY + except TTransport.TTransportException: + raise + except TApplicationException as ex: + logging.exception("TApplication exception in handler") + msg_type = TMessageType.EXCEPTION + result = ex + except Exception: + logging.exception("Unexpected exception in handler") + msg_type = TMessageType.EXCEPTION + result = TApplicationException(TApplicationException.INTERNAL_ERROR, "Internal error") + oprot.writeMessageBegin("get_file_metadata_by_expr", msg_type, seqid) + result.write(oprot) + oprot.writeMessageEnd() + oprot.trans.flush() + + def process_get_file_metadata(self, seqid, iprot, oprot): + args = get_file_metadata_args() + args.read(iprot) + iprot.readMessageEnd() + result = get_file_metadata_result() + try: + result.success = self._handler.get_file_metadata(args.req) + msg_type = TMessageType.REPLY + except TTransport.TTransportException: + raise + except TApplicationException as ex: + logging.exception("TApplication exception in handler") + msg_type = TMessageType.EXCEPTION + result = ex + except Exception: + logging.exception("Unexpected exception in handler") + msg_type = TMessageType.EXCEPTION + result = TApplicationException(TApplicationException.INTERNAL_ERROR, "Internal error") + oprot.writeMessageBegin("get_file_metadata", msg_type, seqid) + result.write(oprot) + oprot.writeMessageEnd() + oprot.trans.flush() + + def process_put_file_metadata(self, seqid, iprot, oprot): + args = put_file_metadata_args() + args.read(iprot) + iprot.readMessageEnd() + result = put_file_metadata_result() + try: + result.success = self._handler.put_file_metadata(args.req) + msg_type = TMessageType.REPLY + except TTransport.TTransportException: + raise + except TApplicationException as ex: + logging.exception("TApplication exception in handler") + msg_type = TMessageType.EXCEPTION + result = ex + except Exception: + logging.exception("Unexpected exception in handler") + msg_type = TMessageType.EXCEPTION + result = TApplicationException(TApplicationException.INTERNAL_ERROR, "Internal error") + oprot.writeMessageBegin("put_file_metadata", msg_type, seqid) + result.write(oprot) + oprot.writeMessageEnd() + oprot.trans.flush() + + def process_clear_file_metadata(self, seqid, iprot, oprot): + args = clear_file_metadata_args() + args.read(iprot) + iprot.readMessageEnd() + result = clear_file_metadata_result() + try: + result.success = self._handler.clear_file_metadata(args.req) + msg_type = TMessageType.REPLY + except TTransport.TTransportException: + raise + except TApplicationException as ex: + logging.exception("TApplication exception in handler") + msg_type = TMessageType.EXCEPTION + result = ex + except Exception: + logging.exception("Unexpected exception in handler") + msg_type = TMessageType.EXCEPTION + result = TApplicationException(TApplicationException.INTERNAL_ERROR, "Internal error") + oprot.writeMessageBegin("clear_file_metadata", msg_type, seqid) + result.write(oprot) + oprot.writeMessageEnd() + oprot.trans.flush() + + def process_cache_file_metadata(self, seqid, iprot, oprot): + args = cache_file_metadata_args() + args.read(iprot) + iprot.readMessageEnd() + result = cache_file_metadata_result() + try: + result.success = self._handler.cache_file_metadata(args.req) + msg_type = TMessageType.REPLY + except TTransport.TTransportException: + raise + except TApplicationException as ex: + logging.exception("TApplication exception in handler") + msg_type = TMessageType.EXCEPTION + result = ex + except Exception: + logging.exception("Unexpected exception in handler") + msg_type = TMessageType.EXCEPTION + result = TApplicationException(TApplicationException.INTERNAL_ERROR, "Internal error") + oprot.writeMessageBegin("cache_file_metadata", msg_type, seqid) + result.write(oprot) + oprot.writeMessageEnd() + oprot.trans.flush() + + def process_get_metastore_db_uuid(self, seqid, iprot, oprot): + args = get_metastore_db_uuid_args() + args.read(iprot) + iprot.readMessageEnd() + result = get_metastore_db_uuid_result() + try: + result.success = self._handler.get_metastore_db_uuid() + msg_type = TMessageType.REPLY + except TTransport.TTransportException: + raise + except MetaException as o1: + msg_type = TMessageType.REPLY + result.o1 = o1 + except TApplicationException as ex: + logging.exception("TApplication exception in handler") + msg_type = TMessageType.EXCEPTION + result = ex + except Exception: + logging.exception("Unexpected exception in handler") + msg_type = TMessageType.EXCEPTION + result = TApplicationException(TApplicationException.INTERNAL_ERROR, "Internal error") + oprot.writeMessageBegin("get_metastore_db_uuid", msg_type, seqid) + result.write(oprot) + oprot.writeMessageEnd() + oprot.trans.flush() + + def process_create_resource_plan(self, seqid, iprot, oprot): + args = create_resource_plan_args() + args.read(iprot) + iprot.readMessageEnd() + result = create_resource_plan_result() + try: + result.success = self._handler.create_resource_plan(args.request) + msg_type = TMessageType.REPLY + except TTransport.TTransportException: + raise + except AlreadyExistsException as o1: + msg_type = TMessageType.REPLY + result.o1 = o1 + except InvalidObjectException as o2: + msg_type = TMessageType.REPLY + result.o2 = o2 + except MetaException as o3: + msg_type = TMessageType.REPLY + result.o3 = o3 + except TApplicationException as ex: + logging.exception("TApplication exception in handler") + msg_type = TMessageType.EXCEPTION + result = ex + except Exception: + logging.exception("Unexpected exception in handler") + msg_type = TMessageType.EXCEPTION + result = TApplicationException(TApplicationException.INTERNAL_ERROR, "Internal error") + oprot.writeMessageBegin("create_resource_plan", msg_type, seqid) + result.write(oprot) + oprot.writeMessageEnd() + oprot.trans.flush() + + def process_get_resource_plan(self, seqid, iprot, oprot): + args = get_resource_plan_args() + args.read(iprot) + iprot.readMessageEnd() + result = get_resource_plan_result() + try: + result.success = self._handler.get_resource_plan(args.request) + msg_type = TMessageType.REPLY + except TTransport.TTransportException: + raise + except NoSuchObjectException as o1: + msg_type = TMessageType.REPLY + result.o1 = o1 + except MetaException as o2: + msg_type = TMessageType.REPLY + result.o2 = o2 + except TApplicationException as ex: + logging.exception("TApplication exception in handler") + msg_type = TMessageType.EXCEPTION + result = ex + except Exception: + logging.exception("Unexpected exception in handler") + msg_type = TMessageType.EXCEPTION + result = TApplicationException(TApplicationException.INTERNAL_ERROR, "Internal error") + oprot.writeMessageBegin("get_resource_plan", msg_type, seqid) + result.write(oprot) + oprot.writeMessageEnd() + oprot.trans.flush() + + def process_get_active_resource_plan(self, seqid, iprot, oprot): + args = get_active_resource_plan_args() + args.read(iprot) + iprot.readMessageEnd() + result = get_active_resource_plan_result() + try: + result.success = self._handler.get_active_resource_plan(args.request) + msg_type = TMessageType.REPLY + except TTransport.TTransportException: + raise + except MetaException as o2: + msg_type = TMessageType.REPLY + result.o2 = o2 + except TApplicationException as ex: + logging.exception("TApplication exception in handler") + msg_type = TMessageType.EXCEPTION + result = ex + except Exception: + logging.exception("Unexpected exception in handler") + msg_type = TMessageType.EXCEPTION + result = TApplicationException(TApplicationException.INTERNAL_ERROR, "Internal error") + oprot.writeMessageBegin("get_active_resource_plan", msg_type, seqid) + result.write(oprot) + oprot.writeMessageEnd() + oprot.trans.flush() + + def process_get_all_resource_plans(self, seqid, iprot, oprot): + args = get_all_resource_plans_args() + args.read(iprot) + iprot.readMessageEnd() + result = get_all_resource_plans_result() + try: + result.success = self._handler.get_all_resource_plans(args.request) + msg_type = TMessageType.REPLY + except TTransport.TTransportException: + raise + except MetaException as o1: + msg_type = TMessageType.REPLY + result.o1 = o1 + except TApplicationException as ex: + logging.exception("TApplication exception in handler") + msg_type = TMessageType.EXCEPTION + result = ex + except Exception: + logging.exception("Unexpected exception in handler") + msg_type = TMessageType.EXCEPTION + result = TApplicationException(TApplicationException.INTERNAL_ERROR, "Internal error") + oprot.writeMessageBegin("get_all_resource_plans", msg_type, seqid) + result.write(oprot) + oprot.writeMessageEnd() + oprot.trans.flush() + + def process_alter_resource_plan(self, seqid, iprot, oprot): + args = alter_resource_plan_args() + args.read(iprot) + iprot.readMessageEnd() + result = alter_resource_plan_result() + try: + result.success = self._handler.alter_resource_plan(args.request) + msg_type = TMessageType.REPLY + except TTransport.TTransportException: + raise + except NoSuchObjectException as o1: + msg_type = TMessageType.REPLY + result.o1 = o1 + except InvalidOperationException as o2: + msg_type = TMessageType.REPLY + result.o2 = o2 + except MetaException as o3: + msg_type = TMessageType.REPLY + result.o3 = o3 + except TApplicationException as ex: + logging.exception("TApplication exception in handler") + msg_type = TMessageType.EXCEPTION + result = ex + except Exception: + logging.exception("Unexpected exception in handler") + msg_type = TMessageType.EXCEPTION + result = TApplicationException(TApplicationException.INTERNAL_ERROR, "Internal error") + oprot.writeMessageBegin("alter_resource_plan", msg_type, seqid) + result.write(oprot) + oprot.writeMessageEnd() + oprot.trans.flush() + + def process_validate_resource_plan(self, seqid, iprot, oprot): + args = validate_resource_plan_args() + args.read(iprot) + iprot.readMessageEnd() + result = validate_resource_plan_result() + try: + result.success = self._handler.validate_resource_plan(args.request) + msg_type = TMessageType.REPLY + except TTransport.TTransportException: + raise + except NoSuchObjectException as o1: + msg_type = TMessageType.REPLY + result.o1 = o1 + except MetaException as o2: + msg_type = TMessageType.REPLY + result.o2 = o2 + except TApplicationException as ex: + logging.exception("TApplication exception in handler") + msg_type = TMessageType.EXCEPTION + result = ex + except Exception: + logging.exception("Unexpected exception in handler") + msg_type = TMessageType.EXCEPTION + result = TApplicationException(TApplicationException.INTERNAL_ERROR, "Internal error") + oprot.writeMessageBegin("validate_resource_plan", msg_type, seqid) + result.write(oprot) + oprot.writeMessageEnd() + oprot.trans.flush() + + def process_drop_resource_plan(self, seqid, iprot, oprot): + args = drop_resource_plan_args() + args.read(iprot) + iprot.readMessageEnd() + result = drop_resource_plan_result() + try: + result.success = self._handler.drop_resource_plan(args.request) + msg_type = TMessageType.REPLY + except TTransport.TTransportException: + raise + except NoSuchObjectException as o1: + msg_type = TMessageType.REPLY + result.o1 = o1 + except InvalidOperationException as o2: + msg_type = TMessageType.REPLY + result.o2 = o2 + except MetaException as o3: + msg_type = TMessageType.REPLY + result.o3 = o3 + except TApplicationException as ex: + logging.exception("TApplication exception in handler") + msg_type = TMessageType.EXCEPTION + result = ex + except Exception: + logging.exception("Unexpected exception in handler") + msg_type = TMessageType.EXCEPTION + result = TApplicationException(TApplicationException.INTERNAL_ERROR, "Internal error") + oprot.writeMessageBegin("drop_resource_plan", msg_type, seqid) + result.write(oprot) + oprot.writeMessageEnd() + oprot.trans.flush() + + def process_create_wm_trigger(self, seqid, iprot, oprot): + args = create_wm_trigger_args() + args.read(iprot) + iprot.readMessageEnd() + result = create_wm_trigger_result() + try: + result.success = self._handler.create_wm_trigger(args.request) + msg_type = TMessageType.REPLY + except TTransport.TTransportException: + raise + except AlreadyExistsException as o1: + msg_type = TMessageType.REPLY + result.o1 = o1 + except NoSuchObjectException as o2: + msg_type = TMessageType.REPLY + result.o2 = o2 + except InvalidObjectException as o3: + msg_type = TMessageType.REPLY + result.o3 = o3 + except MetaException as o4: + msg_type = TMessageType.REPLY + result.o4 = o4 + except TApplicationException as ex: + logging.exception("TApplication exception in handler") + msg_type = TMessageType.EXCEPTION + result = ex + except Exception: + logging.exception("Unexpected exception in handler") + msg_type = TMessageType.EXCEPTION + result = TApplicationException(TApplicationException.INTERNAL_ERROR, "Internal error") + oprot.writeMessageBegin("create_wm_trigger", msg_type, seqid) + result.write(oprot) + oprot.writeMessageEnd() + oprot.trans.flush() + + def process_alter_wm_trigger(self, seqid, iprot, oprot): + args = alter_wm_trigger_args() + args.read(iprot) + iprot.readMessageEnd() + result = alter_wm_trigger_result() + try: + result.success = self._handler.alter_wm_trigger(args.request) + msg_type = TMessageType.REPLY + except TTransport.TTransportException: + raise + except NoSuchObjectException as o1: + msg_type = TMessageType.REPLY + result.o1 = o1 + except InvalidObjectException as o2: + msg_type = TMessageType.REPLY + result.o2 = o2 + except MetaException as o3: + msg_type = TMessageType.REPLY + result.o3 = o3 + except TApplicationException as ex: + logging.exception("TApplication exception in handler") + msg_type = TMessageType.EXCEPTION + result = ex + except Exception: + logging.exception("Unexpected exception in handler") + msg_type = TMessageType.EXCEPTION + result = TApplicationException(TApplicationException.INTERNAL_ERROR, "Internal error") + oprot.writeMessageBegin("alter_wm_trigger", msg_type, seqid) + result.write(oprot) + oprot.writeMessageEnd() + oprot.trans.flush() + + def process_drop_wm_trigger(self, seqid, iprot, oprot): + args = drop_wm_trigger_args() + args.read(iprot) + iprot.readMessageEnd() + result = drop_wm_trigger_result() + try: + result.success = self._handler.drop_wm_trigger(args.request) + msg_type = TMessageType.REPLY + except TTransport.TTransportException: + raise + except NoSuchObjectException as o1: + msg_type = TMessageType.REPLY + result.o1 = o1 + except InvalidOperationException as o2: + msg_type = TMessageType.REPLY + result.o2 = o2 + except MetaException as o3: + msg_type = TMessageType.REPLY + result.o3 = o3 + except TApplicationException as ex: + logging.exception("TApplication exception in handler") + msg_type = TMessageType.EXCEPTION + result = ex + except Exception: + logging.exception("Unexpected exception in handler") + msg_type = TMessageType.EXCEPTION + result = TApplicationException(TApplicationException.INTERNAL_ERROR, "Internal error") + oprot.writeMessageBegin("drop_wm_trigger", msg_type, seqid) + result.write(oprot) + oprot.writeMessageEnd() + oprot.trans.flush() + + def process_get_triggers_for_resourceplan(self, seqid, iprot, oprot): + args = get_triggers_for_resourceplan_args() + args.read(iprot) + iprot.readMessageEnd() + result = get_triggers_for_resourceplan_result() + try: + result.success = self._handler.get_triggers_for_resourceplan(args.request) + msg_type = TMessageType.REPLY + except TTransport.TTransportException: + raise + except NoSuchObjectException as o1: + msg_type = TMessageType.REPLY + result.o1 = o1 + except MetaException as o2: + msg_type = TMessageType.REPLY + result.o2 = o2 + except TApplicationException as ex: + logging.exception("TApplication exception in handler") + msg_type = TMessageType.EXCEPTION + result = ex + except Exception: + logging.exception("Unexpected exception in handler") + msg_type = TMessageType.EXCEPTION + result = TApplicationException(TApplicationException.INTERNAL_ERROR, "Internal error") + oprot.writeMessageBegin("get_triggers_for_resourceplan", msg_type, seqid) + result.write(oprot) + oprot.writeMessageEnd() + oprot.trans.flush() + + def process_create_wm_pool(self, seqid, iprot, oprot): + args = create_wm_pool_args() + args.read(iprot) + iprot.readMessageEnd() + result = create_wm_pool_result() + try: + result.success = self._handler.create_wm_pool(args.request) + msg_type = TMessageType.REPLY + except TTransport.TTransportException: + raise + except AlreadyExistsException as o1: + msg_type = TMessageType.REPLY + result.o1 = o1 + except NoSuchObjectException as o2: + msg_type = TMessageType.REPLY + result.o2 = o2 + except InvalidObjectException as o3: + msg_type = TMessageType.REPLY + result.o3 = o3 + except MetaException as o4: + msg_type = TMessageType.REPLY + result.o4 = o4 + except TApplicationException as ex: + logging.exception("TApplication exception in handler") + msg_type = TMessageType.EXCEPTION + result = ex + except Exception: + logging.exception("Unexpected exception in handler") + msg_type = TMessageType.EXCEPTION + result = TApplicationException(TApplicationException.INTERNAL_ERROR, "Internal error") + oprot.writeMessageBegin("create_wm_pool", msg_type, seqid) + result.write(oprot) + oprot.writeMessageEnd() + oprot.trans.flush() + + def process_alter_wm_pool(self, seqid, iprot, oprot): + args = alter_wm_pool_args() + args.read(iprot) + iprot.readMessageEnd() + result = alter_wm_pool_result() + try: + result.success = self._handler.alter_wm_pool(args.request) + msg_type = TMessageType.REPLY + except TTransport.TTransportException: + raise + except AlreadyExistsException as o1: + msg_type = TMessageType.REPLY + result.o1 = o1 + except NoSuchObjectException as o2: + msg_type = TMessageType.REPLY + result.o2 = o2 + except InvalidObjectException as o3: + msg_type = TMessageType.REPLY + result.o3 = o3 + except MetaException as o4: + msg_type = TMessageType.REPLY + result.o4 = o4 + except TApplicationException as ex: + logging.exception("TApplication exception in handler") + msg_type = TMessageType.EXCEPTION + result = ex + except Exception: + logging.exception("Unexpected exception in handler") + msg_type = TMessageType.EXCEPTION + result = TApplicationException(TApplicationException.INTERNAL_ERROR, "Internal error") + oprot.writeMessageBegin("alter_wm_pool", msg_type, seqid) + result.write(oprot) + oprot.writeMessageEnd() + oprot.trans.flush() + + def process_drop_wm_pool(self, seqid, iprot, oprot): + args = drop_wm_pool_args() + args.read(iprot) + iprot.readMessageEnd() + result = drop_wm_pool_result() + try: + result.success = self._handler.drop_wm_pool(args.request) + msg_type = TMessageType.REPLY + except TTransport.TTransportException: + raise + except NoSuchObjectException as o1: + msg_type = TMessageType.REPLY + result.o1 = o1 + except InvalidOperationException as o2: + msg_type = TMessageType.REPLY + result.o2 = o2 + except MetaException as o3: + msg_type = TMessageType.REPLY + result.o3 = o3 + except TApplicationException as ex: + logging.exception("TApplication exception in handler") + msg_type = TMessageType.EXCEPTION + result = ex + except Exception: + logging.exception("Unexpected exception in handler") + msg_type = TMessageType.EXCEPTION + result = TApplicationException(TApplicationException.INTERNAL_ERROR, "Internal error") + oprot.writeMessageBegin("drop_wm_pool", msg_type, seqid) + result.write(oprot) + oprot.writeMessageEnd() + oprot.trans.flush() + + def process_create_or_update_wm_mapping(self, seqid, iprot, oprot): + args = create_or_update_wm_mapping_args() + args.read(iprot) + iprot.readMessageEnd() + result = create_or_update_wm_mapping_result() + try: + result.success = self._handler.create_or_update_wm_mapping(args.request) + msg_type = TMessageType.REPLY + except TTransport.TTransportException: + raise + except AlreadyExistsException as o1: + msg_type = TMessageType.REPLY + result.o1 = o1 + except NoSuchObjectException as o2: + msg_type = TMessageType.REPLY + result.o2 = o2 + except InvalidObjectException as o3: + msg_type = TMessageType.REPLY + result.o3 = o3 + except MetaException as o4: + msg_type = TMessageType.REPLY + result.o4 = o4 + except TApplicationException as ex: + logging.exception("TApplication exception in handler") + msg_type = TMessageType.EXCEPTION + result = ex + except Exception: + logging.exception("Unexpected exception in handler") + msg_type = TMessageType.EXCEPTION + result = TApplicationException(TApplicationException.INTERNAL_ERROR, "Internal error") + oprot.writeMessageBegin("create_or_update_wm_mapping", msg_type, seqid) + result.write(oprot) + oprot.writeMessageEnd() + oprot.trans.flush() + + def process_drop_wm_mapping(self, seqid, iprot, oprot): + args = drop_wm_mapping_args() + args.read(iprot) + iprot.readMessageEnd() + result = drop_wm_mapping_result() + try: + result.success = self._handler.drop_wm_mapping(args.request) + msg_type = TMessageType.REPLY + except TTransport.TTransportException: + raise + except NoSuchObjectException as o1: + msg_type = TMessageType.REPLY + result.o1 = o1 + except InvalidOperationException as o2: + msg_type = TMessageType.REPLY + result.o2 = o2 + except MetaException as o3: + msg_type = TMessageType.REPLY + result.o3 = o3 + except TApplicationException as ex: + logging.exception("TApplication exception in handler") + msg_type = TMessageType.EXCEPTION + result = ex + except Exception: + logging.exception("Unexpected exception in handler") + msg_type = TMessageType.EXCEPTION + result = TApplicationException(TApplicationException.INTERNAL_ERROR, "Internal error") + oprot.writeMessageBegin("drop_wm_mapping", msg_type, seqid) + result.write(oprot) + oprot.writeMessageEnd() + oprot.trans.flush() + + def process_create_or_drop_wm_trigger_to_pool_mapping(self, seqid, iprot, oprot): + args = create_or_drop_wm_trigger_to_pool_mapping_args() + args.read(iprot) + iprot.readMessageEnd() + result = create_or_drop_wm_trigger_to_pool_mapping_result() + try: + result.success = self._handler.create_or_drop_wm_trigger_to_pool_mapping(args.request) + msg_type = TMessageType.REPLY + except TTransport.TTransportException: + raise + except AlreadyExistsException as o1: + msg_type = TMessageType.REPLY + result.o1 = o1 + except NoSuchObjectException as o2: + msg_type = TMessageType.REPLY + result.o2 = o2 + except InvalidObjectException as o3: + msg_type = TMessageType.REPLY + result.o3 = o3 + except MetaException as o4: + msg_type = TMessageType.REPLY + result.o4 = o4 + except TApplicationException as ex: + logging.exception("TApplication exception in handler") + msg_type = TMessageType.EXCEPTION + result = ex + except Exception: + logging.exception("Unexpected exception in handler") + msg_type = TMessageType.EXCEPTION + result = TApplicationException(TApplicationException.INTERNAL_ERROR, "Internal error") + oprot.writeMessageBegin("create_or_drop_wm_trigger_to_pool_mapping", msg_type, seqid) + result.write(oprot) + oprot.writeMessageEnd() + oprot.trans.flush() + + def process_create_ischema(self, seqid, iprot, oprot): + args = create_ischema_args() + args.read(iprot) + iprot.readMessageEnd() + result = create_ischema_result() + try: + self._handler.create_ischema(args.schema) + msg_type = TMessageType.REPLY + except TTransport.TTransportException: + raise + except AlreadyExistsException as o1: + msg_type = TMessageType.REPLY + result.o1 = o1 + except NoSuchObjectException as o2: + msg_type = TMessageType.REPLY + result.o2 = o2 + except MetaException as o3: + msg_type = TMessageType.REPLY + result.o3 = o3 + except TApplicationException as ex: + logging.exception("TApplication exception in handler") + msg_type = TMessageType.EXCEPTION + result = ex + except Exception: + logging.exception("Unexpected exception in handler") + msg_type = TMessageType.EXCEPTION + result = TApplicationException(TApplicationException.INTERNAL_ERROR, "Internal error") + oprot.writeMessageBegin("create_ischema", msg_type, seqid) + result.write(oprot) + oprot.writeMessageEnd() + oprot.trans.flush() + + def process_alter_ischema(self, seqid, iprot, oprot): + args = alter_ischema_args() + args.read(iprot) + iprot.readMessageEnd() + result = alter_ischema_result() + try: + self._handler.alter_ischema(args.rqst) + msg_type = TMessageType.REPLY + except TTransport.TTransportException: + raise + except NoSuchObjectException as o1: + msg_type = TMessageType.REPLY + result.o1 = o1 + except MetaException as o2: + msg_type = TMessageType.REPLY + result.o2 = o2 + except TApplicationException as ex: + logging.exception("TApplication exception in handler") + msg_type = TMessageType.EXCEPTION + result = ex + except Exception: + logging.exception("Unexpected exception in handler") + msg_type = TMessageType.EXCEPTION + result = TApplicationException(TApplicationException.INTERNAL_ERROR, "Internal error") + oprot.writeMessageBegin("alter_ischema", msg_type, seqid) + result.write(oprot) + oprot.writeMessageEnd() + oprot.trans.flush() + + def process_get_ischema(self, seqid, iprot, oprot): + args = get_ischema_args() + args.read(iprot) + iprot.readMessageEnd() + result = get_ischema_result() + try: + result.success = self._handler.get_ischema(args.name) + msg_type = TMessageType.REPLY + except TTransport.TTransportException: + raise + except NoSuchObjectException as o1: + msg_type = TMessageType.REPLY + result.o1 = o1 + except MetaException as o2: + msg_type = TMessageType.REPLY + result.o2 = o2 + except TApplicationException as ex: + logging.exception("TApplication exception in handler") + msg_type = TMessageType.EXCEPTION + result = ex + except Exception: + logging.exception("Unexpected exception in handler") + msg_type = TMessageType.EXCEPTION + result = TApplicationException(TApplicationException.INTERNAL_ERROR, "Internal error") + oprot.writeMessageBegin("get_ischema", msg_type, seqid) + result.write(oprot) + oprot.writeMessageEnd() + oprot.trans.flush() + + def process_drop_ischema(self, seqid, iprot, oprot): + args = drop_ischema_args() + args.read(iprot) + iprot.readMessageEnd() + result = drop_ischema_result() + try: + self._handler.drop_ischema(args.name) + msg_type = TMessageType.REPLY + except TTransport.TTransportException: + raise + except NoSuchObjectException as o1: + msg_type = TMessageType.REPLY + result.o1 = o1 + except InvalidOperationException as o2: + msg_type = TMessageType.REPLY + result.o2 = o2 + except MetaException as o3: + msg_type = TMessageType.REPLY + result.o3 = o3 + except TApplicationException as ex: + logging.exception("TApplication exception in handler") + msg_type = TMessageType.EXCEPTION + result = ex + except Exception: + logging.exception("Unexpected exception in handler") + msg_type = TMessageType.EXCEPTION + result = TApplicationException(TApplicationException.INTERNAL_ERROR, "Internal error") + oprot.writeMessageBegin("drop_ischema", msg_type, seqid) + result.write(oprot) + oprot.writeMessageEnd() + oprot.trans.flush() + + def process_add_schema_version(self, seqid, iprot, oprot): + args = add_schema_version_args() + args.read(iprot) + iprot.readMessageEnd() + result = add_schema_version_result() + try: + self._handler.add_schema_version(args.schemaVersion) + msg_type = TMessageType.REPLY + except TTransport.TTransportException: + raise + except AlreadyExistsException as o1: + msg_type = TMessageType.REPLY + result.o1 = o1 + except NoSuchObjectException as o2: + msg_type = TMessageType.REPLY + result.o2 = o2 + except MetaException as o3: + msg_type = TMessageType.REPLY + result.o3 = o3 + except TApplicationException as ex: + logging.exception("TApplication exception in handler") + msg_type = TMessageType.EXCEPTION + result = ex + except Exception: + logging.exception("Unexpected exception in handler") + msg_type = TMessageType.EXCEPTION + result = TApplicationException(TApplicationException.INTERNAL_ERROR, "Internal error") + oprot.writeMessageBegin("add_schema_version", msg_type, seqid) + result.write(oprot) + oprot.writeMessageEnd() + oprot.trans.flush() + + def process_get_schema_version(self, seqid, iprot, oprot): + args = get_schema_version_args() + args.read(iprot) + iprot.readMessageEnd() + result = get_schema_version_result() + try: + result.success = self._handler.get_schema_version(args.schemaVersion) + msg_type = TMessageType.REPLY + except TTransport.TTransportException: + raise + except NoSuchObjectException as o1: + msg_type = TMessageType.REPLY + result.o1 = o1 + except MetaException as o2: + msg_type = TMessageType.REPLY + result.o2 = o2 + except TApplicationException as ex: + logging.exception("TApplication exception in handler") + msg_type = TMessageType.EXCEPTION + result = ex + except Exception: + logging.exception("Unexpected exception in handler") + msg_type = TMessageType.EXCEPTION + result = TApplicationException(TApplicationException.INTERNAL_ERROR, "Internal error") + oprot.writeMessageBegin("get_schema_version", msg_type, seqid) + result.write(oprot) + oprot.writeMessageEnd() + oprot.trans.flush() + + def process_get_schema_latest_version(self, seqid, iprot, oprot): + args = get_schema_latest_version_args() + args.read(iprot) + iprot.readMessageEnd() + result = get_schema_latest_version_result() + try: + result.success = self._handler.get_schema_latest_version(args.schemaName) + msg_type = TMessageType.REPLY + except TTransport.TTransportException: + raise + except NoSuchObjectException as o1: + msg_type = TMessageType.REPLY + result.o1 = o1 + except MetaException as o2: + msg_type = TMessageType.REPLY + result.o2 = o2 + except TApplicationException as ex: + logging.exception("TApplication exception in handler") + msg_type = TMessageType.EXCEPTION + result = ex + except Exception: + logging.exception("Unexpected exception in handler") + msg_type = TMessageType.EXCEPTION + result = TApplicationException(TApplicationException.INTERNAL_ERROR, "Internal error") + oprot.writeMessageBegin("get_schema_latest_version", msg_type, seqid) + result.write(oprot) + oprot.writeMessageEnd() + oprot.trans.flush() + + def process_get_schema_all_versions(self, seqid, iprot, oprot): + args = get_schema_all_versions_args() + args.read(iprot) + iprot.readMessageEnd() + result = get_schema_all_versions_result() + try: + result.success = self._handler.get_schema_all_versions(args.schemaName) + msg_type = TMessageType.REPLY + except TTransport.TTransportException: + raise + except NoSuchObjectException as o1: + msg_type = TMessageType.REPLY + result.o1 = o1 + except MetaException as o2: + msg_type = TMessageType.REPLY + result.o2 = o2 + except TApplicationException as ex: + logging.exception("TApplication exception in handler") + msg_type = TMessageType.EXCEPTION + result = ex + except Exception: + logging.exception("Unexpected exception in handler") + msg_type = TMessageType.EXCEPTION + result = TApplicationException(TApplicationException.INTERNAL_ERROR, "Internal error") + oprot.writeMessageBegin("get_schema_all_versions", msg_type, seqid) + result.write(oprot) + oprot.writeMessageEnd() + oprot.trans.flush() + + def process_drop_schema_version(self, seqid, iprot, oprot): + args = drop_schema_version_args() + args.read(iprot) + iprot.readMessageEnd() + result = drop_schema_version_result() + try: + self._handler.drop_schema_version(args.schemaVersion) + msg_type = TMessageType.REPLY + except TTransport.TTransportException: + raise + except NoSuchObjectException as o1: + msg_type = TMessageType.REPLY + result.o1 = o1 + except MetaException as o2: + msg_type = TMessageType.REPLY + result.o2 = o2 + except TApplicationException as ex: + logging.exception("TApplication exception in handler") + msg_type = TMessageType.EXCEPTION + result = ex + except Exception: + logging.exception("Unexpected exception in handler") + msg_type = TMessageType.EXCEPTION + result = TApplicationException(TApplicationException.INTERNAL_ERROR, "Internal error") + oprot.writeMessageBegin("drop_schema_version", msg_type, seqid) + result.write(oprot) + oprot.writeMessageEnd() + oprot.trans.flush() + + def process_get_schemas_by_cols(self, seqid, iprot, oprot): + args = get_schemas_by_cols_args() + args.read(iprot) + iprot.readMessageEnd() + result = get_schemas_by_cols_result() + try: + result.success = self._handler.get_schemas_by_cols(args.rqst) + msg_type = TMessageType.REPLY + except TTransport.TTransportException: + raise + except MetaException as o1: + msg_type = TMessageType.REPLY + result.o1 = o1 + except TApplicationException as ex: + logging.exception("TApplication exception in handler") + msg_type = TMessageType.EXCEPTION + result = ex + except Exception: + logging.exception("Unexpected exception in handler") + msg_type = TMessageType.EXCEPTION + result = TApplicationException(TApplicationException.INTERNAL_ERROR, "Internal error") + oprot.writeMessageBegin("get_schemas_by_cols", msg_type, seqid) + result.write(oprot) + oprot.writeMessageEnd() + oprot.trans.flush() + + def process_map_schema_version_to_serde(self, seqid, iprot, oprot): + args = map_schema_version_to_serde_args() + args.read(iprot) + iprot.readMessageEnd() + result = map_schema_version_to_serde_result() + try: + self._handler.map_schema_version_to_serde(args.rqst) + msg_type = TMessageType.REPLY + except TTransport.TTransportException: + raise + except NoSuchObjectException as o1: + msg_type = TMessageType.REPLY + result.o1 = o1 + except MetaException as o2: + msg_type = TMessageType.REPLY + result.o2 = o2 + except TApplicationException as ex: + logging.exception("TApplication exception in handler") + msg_type = TMessageType.EXCEPTION + result = ex + except Exception: + logging.exception("Unexpected exception in handler") + msg_type = TMessageType.EXCEPTION + result = TApplicationException(TApplicationException.INTERNAL_ERROR, "Internal error") + oprot.writeMessageBegin("map_schema_version_to_serde", msg_type, seqid) + result.write(oprot) + oprot.writeMessageEnd() + oprot.trans.flush() + + def process_set_schema_version_state(self, seqid, iprot, oprot): + args = set_schema_version_state_args() + args.read(iprot) + iprot.readMessageEnd() + result = set_schema_version_state_result() + try: + self._handler.set_schema_version_state(args.rqst) + msg_type = TMessageType.REPLY + except TTransport.TTransportException: + raise + except NoSuchObjectException as o1: + msg_type = TMessageType.REPLY + result.o1 = o1 + except InvalidOperationException as o2: + msg_type = TMessageType.REPLY + result.o2 = o2 + except MetaException as o3: + msg_type = TMessageType.REPLY + result.o3 = o3 + except TApplicationException as ex: + logging.exception("TApplication exception in handler") + msg_type = TMessageType.EXCEPTION + result = ex + except Exception: + logging.exception("Unexpected exception in handler") + msg_type = TMessageType.EXCEPTION + result = TApplicationException(TApplicationException.INTERNAL_ERROR, "Internal error") + oprot.writeMessageBegin("set_schema_version_state", msg_type, seqid) + result.write(oprot) + oprot.writeMessageEnd() + oprot.trans.flush() + + def process_add_serde(self, seqid, iprot, oprot): + args = add_serde_args() + args.read(iprot) + iprot.readMessageEnd() + result = add_serde_result() + try: + self._handler.add_serde(args.serde) + msg_type = TMessageType.REPLY + except TTransport.TTransportException: + raise + except AlreadyExistsException as o1: + msg_type = TMessageType.REPLY + result.o1 = o1 + except MetaException as o2: + msg_type = TMessageType.REPLY + result.o2 = o2 + except TApplicationException as ex: + logging.exception("TApplication exception in handler") + msg_type = TMessageType.EXCEPTION + result = ex + except Exception: + logging.exception("Unexpected exception in handler") + msg_type = TMessageType.EXCEPTION + result = TApplicationException(TApplicationException.INTERNAL_ERROR, "Internal error") + oprot.writeMessageBegin("add_serde", msg_type, seqid) + result.write(oprot) + oprot.writeMessageEnd() + oprot.trans.flush() + + def process_get_serde(self, seqid, iprot, oprot): + args = get_serde_args() + args.read(iprot) + iprot.readMessageEnd() + result = get_serde_result() + try: + result.success = self._handler.get_serde(args.rqst) + msg_type = TMessageType.REPLY + except TTransport.TTransportException: + raise + except NoSuchObjectException as o1: + msg_type = TMessageType.REPLY + result.o1 = o1 + except MetaException as o2: + msg_type = TMessageType.REPLY + result.o2 = o2 + except TApplicationException as ex: + logging.exception("TApplication exception in handler") + msg_type = TMessageType.EXCEPTION + result = ex + except Exception: + logging.exception("Unexpected exception in handler") + msg_type = TMessageType.EXCEPTION + result = TApplicationException(TApplicationException.INTERNAL_ERROR, "Internal error") + oprot.writeMessageBegin("get_serde", msg_type, seqid) + result.write(oprot) + oprot.writeMessageEnd() + oprot.trans.flush() + + def process_get_lock_materialization_rebuild(self, seqid, iprot, oprot): + args = get_lock_materialization_rebuild_args() + args.read(iprot) + iprot.readMessageEnd() + result = get_lock_materialization_rebuild_result() + try: + result.success = self._handler.get_lock_materialization_rebuild(args.dbName, args.tableName, args.txnId) + msg_type = TMessageType.REPLY + except TTransport.TTransportException: + raise + except TApplicationException as ex: + logging.exception("TApplication exception in handler") + msg_type = TMessageType.EXCEPTION + result = ex + except Exception: + logging.exception("Unexpected exception in handler") + msg_type = TMessageType.EXCEPTION + result = TApplicationException(TApplicationException.INTERNAL_ERROR, "Internal error") + oprot.writeMessageBegin("get_lock_materialization_rebuild", msg_type, seqid) + result.write(oprot) + oprot.writeMessageEnd() + oprot.trans.flush() + + def process_heartbeat_lock_materialization_rebuild(self, seqid, iprot, oprot): + args = heartbeat_lock_materialization_rebuild_args() + args.read(iprot) + iprot.readMessageEnd() + result = heartbeat_lock_materialization_rebuild_result() + try: + result.success = self._handler.heartbeat_lock_materialization_rebuild(args.dbName, args.tableName, args.txnId) + msg_type = TMessageType.REPLY + except TTransport.TTransportException: + raise + except TApplicationException as ex: + logging.exception("TApplication exception in handler") + msg_type = TMessageType.EXCEPTION + result = ex + except Exception: + logging.exception("Unexpected exception in handler") + msg_type = TMessageType.EXCEPTION + result = TApplicationException(TApplicationException.INTERNAL_ERROR, "Internal error") + oprot.writeMessageBegin("heartbeat_lock_materialization_rebuild", msg_type, seqid) + result.write(oprot) + oprot.writeMessageEnd() + oprot.trans.flush() + + def process_add_runtime_stats(self, seqid, iprot, oprot): + args = add_runtime_stats_args() + args.read(iprot) + iprot.readMessageEnd() + result = add_runtime_stats_result() + try: + self._handler.add_runtime_stats(args.stat) + msg_type = TMessageType.REPLY + except TTransport.TTransportException: + raise + except MetaException as o1: + msg_type = TMessageType.REPLY + result.o1 = o1 + except TApplicationException as ex: + logging.exception("TApplication exception in handler") + msg_type = TMessageType.EXCEPTION + result = ex + except Exception: + logging.exception("Unexpected exception in handler") + msg_type = TMessageType.EXCEPTION + result = TApplicationException(TApplicationException.INTERNAL_ERROR, "Internal error") + oprot.writeMessageBegin("add_runtime_stats", msg_type, seqid) + result.write(oprot) + oprot.writeMessageEnd() + oprot.trans.flush() + + def process_get_runtime_stats(self, seqid, iprot, oprot): + args = get_runtime_stats_args() + args.read(iprot) + iprot.readMessageEnd() + result = get_runtime_stats_result() + try: + result.success = self._handler.get_runtime_stats(args.rqst) + msg_type = TMessageType.REPLY + except TTransport.TTransportException: + raise + except MetaException as o1: + msg_type = TMessageType.REPLY + result.o1 = o1 + except TApplicationException as ex: + logging.exception("TApplication exception in handler") + msg_type = TMessageType.EXCEPTION + result = ex + except Exception: + logging.exception("Unexpected exception in handler") + msg_type = TMessageType.EXCEPTION + result = TApplicationException(TApplicationException.INTERNAL_ERROR, "Internal error") + oprot.writeMessageBegin("get_runtime_stats", msg_type, seqid) + result.write(oprot) + oprot.writeMessageEnd() + oprot.trans.flush() + + def process_get_partitions_with_specs(self, seqid, iprot, oprot): + args = get_partitions_with_specs_args() + args.read(iprot) + iprot.readMessageEnd() + result = get_partitions_with_specs_result() + try: + result.success = self._handler.get_partitions_with_specs(args.request) + msg_type = TMessageType.REPLY + except TTransport.TTransportException: + raise + except MetaException as o1: + msg_type = TMessageType.REPLY + result.o1 = o1 + except TApplicationException as ex: + logging.exception("TApplication exception in handler") + msg_type = TMessageType.EXCEPTION + result = ex + except Exception: + logging.exception("Unexpected exception in handler") + msg_type = TMessageType.EXCEPTION + result = TApplicationException(TApplicationException.INTERNAL_ERROR, "Internal error") + oprot.writeMessageBegin("get_partitions_with_specs", msg_type, seqid) + result.write(oprot) + oprot.writeMessageEnd() + oprot.trans.flush() + + def process_scheduled_query_poll(self, seqid, iprot, oprot): + args = scheduled_query_poll_args() + args.read(iprot) + iprot.readMessageEnd() + result = scheduled_query_poll_result() + try: + result.success = self._handler.scheduled_query_poll(args.request) + msg_type = TMessageType.REPLY + except TTransport.TTransportException: + raise + except MetaException as o1: + msg_type = TMessageType.REPLY + result.o1 = o1 + except TApplicationException as ex: + logging.exception("TApplication exception in handler") + msg_type = TMessageType.EXCEPTION + result = ex + except Exception: + logging.exception("Unexpected exception in handler") + msg_type = TMessageType.EXCEPTION + result = TApplicationException(TApplicationException.INTERNAL_ERROR, "Internal error") + oprot.writeMessageBegin("scheduled_query_poll", msg_type, seqid) + result.write(oprot) + oprot.writeMessageEnd() + oprot.trans.flush() + + def process_scheduled_query_maintenance(self, seqid, iprot, oprot): + args = scheduled_query_maintenance_args() + args.read(iprot) + iprot.readMessageEnd() + result = scheduled_query_maintenance_result() + try: + self._handler.scheduled_query_maintenance(args.request) + msg_type = TMessageType.REPLY + except TTransport.TTransportException: + raise + except MetaException as o1: + msg_type = TMessageType.REPLY + result.o1 = o1 + except NoSuchObjectException as o2: + msg_type = TMessageType.REPLY + result.o2 = o2 + except AlreadyExistsException as o3: + msg_type = TMessageType.REPLY + result.o3 = o3 + except InvalidInputException as o4: + msg_type = TMessageType.REPLY + result.o4 = o4 + except TApplicationException as ex: + logging.exception("TApplication exception in handler") + msg_type = TMessageType.EXCEPTION + result = ex + except Exception: + logging.exception("Unexpected exception in handler") + msg_type = TMessageType.EXCEPTION + result = TApplicationException(TApplicationException.INTERNAL_ERROR, "Internal error") + oprot.writeMessageBegin("scheduled_query_maintenance", msg_type, seqid) + result.write(oprot) + oprot.writeMessageEnd() + oprot.trans.flush() + + def process_scheduled_query_progress(self, seqid, iprot, oprot): + args = scheduled_query_progress_args() + args.read(iprot) + iprot.readMessageEnd() + result = scheduled_query_progress_result() + try: + self._handler.scheduled_query_progress(args.info) + msg_type = TMessageType.REPLY + except TTransport.TTransportException: + raise + except MetaException as o1: + msg_type = TMessageType.REPLY + result.o1 = o1 + except InvalidOperationException as o2: + msg_type = TMessageType.REPLY + result.o2 = o2 + except TApplicationException as ex: + logging.exception("TApplication exception in handler") + msg_type = TMessageType.EXCEPTION + result = ex + except Exception: + logging.exception("Unexpected exception in handler") + msg_type = TMessageType.EXCEPTION + result = TApplicationException(TApplicationException.INTERNAL_ERROR, "Internal error") + oprot.writeMessageBegin("scheduled_query_progress", msg_type, seqid) + result.write(oprot) + oprot.writeMessageEnd() + oprot.trans.flush() + + def process_get_scheduled_query(self, seqid, iprot, oprot): + args = get_scheduled_query_args() + args.read(iprot) + iprot.readMessageEnd() + result = get_scheduled_query_result() + try: + result.success = self._handler.get_scheduled_query(args.scheduleKey) + msg_type = TMessageType.REPLY + except TTransport.TTransportException: + raise + except MetaException as o1: + msg_type = TMessageType.REPLY + result.o1 = o1 + except NoSuchObjectException as o2: + msg_type = TMessageType.REPLY + result.o2 = o2 + except TApplicationException as ex: + logging.exception("TApplication exception in handler") + msg_type = TMessageType.EXCEPTION + result = ex + except Exception: + logging.exception("Unexpected exception in handler") + msg_type = TMessageType.EXCEPTION + result = TApplicationException(TApplicationException.INTERNAL_ERROR, "Internal error") + oprot.writeMessageBegin("get_scheduled_query", msg_type, seqid) + result.write(oprot) + oprot.writeMessageEnd() + oprot.trans.flush() + + def process_add_replication_metrics(self, seqid, iprot, oprot): + args = add_replication_metrics_args() + args.read(iprot) + iprot.readMessageEnd() + result = add_replication_metrics_result() + try: + self._handler.add_replication_metrics(args.replicationMetricList) + msg_type = TMessageType.REPLY + except TTransport.TTransportException: + raise + except MetaException as o1: + msg_type = TMessageType.REPLY + result.o1 = o1 + except TApplicationException as ex: + logging.exception("TApplication exception in handler") + msg_type = TMessageType.EXCEPTION + result = ex + except Exception: + logging.exception("Unexpected exception in handler") + msg_type = TMessageType.EXCEPTION + result = TApplicationException(TApplicationException.INTERNAL_ERROR, "Internal error") + oprot.writeMessageBegin("add_replication_metrics", msg_type, seqid) + result.write(oprot) + oprot.writeMessageEnd() + oprot.trans.flush() + + def process_get_replication_metrics(self, seqid, iprot, oprot): + args = get_replication_metrics_args() + args.read(iprot) + iprot.readMessageEnd() + result = get_replication_metrics_result() + try: + result.success = self._handler.get_replication_metrics(args.rqst) + msg_type = TMessageType.REPLY + except TTransport.TTransportException: + raise + except MetaException as o1: + msg_type = TMessageType.REPLY + result.o1 = o1 + except TApplicationException as ex: + logging.exception("TApplication exception in handler") + msg_type = TMessageType.EXCEPTION + result = ex + except Exception: + logging.exception("Unexpected exception in handler") + msg_type = TMessageType.EXCEPTION + result = TApplicationException(TApplicationException.INTERNAL_ERROR, "Internal error") + oprot.writeMessageBegin("get_replication_metrics", msg_type, seqid) + result.write(oprot) + oprot.writeMessageEnd() + oprot.trans.flush() + + def process_get_open_txns_req(self, seqid, iprot, oprot): + args = get_open_txns_req_args() + args.read(iprot) + iprot.readMessageEnd() + result = get_open_txns_req_result() + try: + result.success = self._handler.get_open_txns_req(args.getOpenTxnsRequest) + msg_type = TMessageType.REPLY + except TTransport.TTransportException: + raise + except TApplicationException as ex: + logging.exception("TApplication exception in handler") + msg_type = TMessageType.EXCEPTION + result = ex + except Exception: + logging.exception("Unexpected exception in handler") + msg_type = TMessageType.EXCEPTION + result = TApplicationException(TApplicationException.INTERNAL_ERROR, "Internal error") + oprot.writeMessageBegin("get_open_txns_req", msg_type, seqid) + result.write(oprot) + oprot.writeMessageEnd() + oprot.trans.flush() + + def process_create_stored_procedure(self, seqid, iprot, oprot): + args = create_stored_procedure_args() + args.read(iprot) + iprot.readMessageEnd() + result = create_stored_procedure_result() + try: + self._handler.create_stored_procedure(args.proc) + msg_type = TMessageType.REPLY + except TTransport.TTransportException: + raise + except NoSuchObjectException as o1: + msg_type = TMessageType.REPLY + result.o1 = o1 + except MetaException as o2: + msg_type = TMessageType.REPLY + result.o2 = o2 + except TApplicationException as ex: + logging.exception("TApplication exception in handler") + msg_type = TMessageType.EXCEPTION + result = ex + except Exception: + logging.exception("Unexpected exception in handler") + msg_type = TMessageType.EXCEPTION + result = TApplicationException(TApplicationException.INTERNAL_ERROR, "Internal error") + oprot.writeMessageBegin("create_stored_procedure", msg_type, seqid) + result.write(oprot) + oprot.writeMessageEnd() + oprot.trans.flush() + + def process_get_stored_procedure(self, seqid, iprot, oprot): + args = get_stored_procedure_args() + args.read(iprot) + iprot.readMessageEnd() + result = get_stored_procedure_result() + try: + result.success = self._handler.get_stored_procedure(args.request) + msg_type = TMessageType.REPLY + except TTransport.TTransportException: + raise + except MetaException as o1: + msg_type = TMessageType.REPLY + result.o1 = o1 + except NoSuchObjectException as o2: + msg_type = TMessageType.REPLY + result.o2 = o2 + except TApplicationException as ex: + logging.exception("TApplication exception in handler") + msg_type = TMessageType.EXCEPTION + result = ex + except Exception: + logging.exception("Unexpected exception in handler") + msg_type = TMessageType.EXCEPTION + result = TApplicationException(TApplicationException.INTERNAL_ERROR, "Internal error") + oprot.writeMessageBegin("get_stored_procedure", msg_type, seqid) + result.write(oprot) + oprot.writeMessageEnd() + oprot.trans.flush() + + def process_drop_stored_procedure(self, seqid, iprot, oprot): + args = drop_stored_procedure_args() + args.read(iprot) + iprot.readMessageEnd() + result = drop_stored_procedure_result() + try: + self._handler.drop_stored_procedure(args.request) + msg_type = TMessageType.REPLY + except TTransport.TTransportException: + raise + except MetaException as o1: + msg_type = TMessageType.REPLY + result.o1 = o1 + except TApplicationException as ex: + logging.exception("TApplication exception in handler") + msg_type = TMessageType.EXCEPTION + result = ex + except Exception: + logging.exception("Unexpected exception in handler") + msg_type = TMessageType.EXCEPTION + result = TApplicationException(TApplicationException.INTERNAL_ERROR, "Internal error") + oprot.writeMessageBegin("drop_stored_procedure", msg_type, seqid) + result.write(oprot) + oprot.writeMessageEnd() + oprot.trans.flush() + + def process_get_all_stored_procedures(self, seqid, iprot, oprot): + args = get_all_stored_procedures_args() + args.read(iprot) + iprot.readMessageEnd() + result = get_all_stored_procedures_result() + try: + result.success = self._handler.get_all_stored_procedures(args.request) + msg_type = TMessageType.REPLY + except TTransport.TTransportException: + raise + except MetaException as o1: + msg_type = TMessageType.REPLY + result.o1 = o1 + except TApplicationException as ex: + logging.exception("TApplication exception in handler") + msg_type = TMessageType.EXCEPTION + result = ex + except Exception: + logging.exception("Unexpected exception in handler") + msg_type = TMessageType.EXCEPTION + result = TApplicationException(TApplicationException.INTERNAL_ERROR, "Internal error") + oprot.writeMessageBegin("get_all_stored_procedures", msg_type, seqid) + result.write(oprot) + oprot.writeMessageEnd() + oprot.trans.flush() + + def process_find_package(self, seqid, iprot, oprot): + args = find_package_args() + args.read(iprot) + iprot.readMessageEnd() + result = find_package_result() + try: + result.success = self._handler.find_package(args.request) + msg_type = TMessageType.REPLY + except TTransport.TTransportException: + raise + except MetaException as o1: + msg_type = TMessageType.REPLY + result.o1 = o1 + except NoSuchObjectException as o2: + msg_type = TMessageType.REPLY + result.o2 = o2 + except TApplicationException as ex: + logging.exception("TApplication exception in handler") + msg_type = TMessageType.EXCEPTION + result = ex + except Exception: + logging.exception("Unexpected exception in handler") + msg_type = TMessageType.EXCEPTION + result = TApplicationException(TApplicationException.INTERNAL_ERROR, "Internal error") + oprot.writeMessageBegin("find_package", msg_type, seqid) + result.write(oprot) + oprot.writeMessageEnd() + oprot.trans.flush() + + def process_add_package(self, seqid, iprot, oprot): + args = add_package_args() + args.read(iprot) + iprot.readMessageEnd() + result = add_package_result() + try: + self._handler.add_package(args.request) + msg_type = TMessageType.REPLY + except TTransport.TTransportException: + raise + except MetaException as o1: + msg_type = TMessageType.REPLY + result.o1 = o1 + except TApplicationException as ex: + logging.exception("TApplication exception in handler") + msg_type = TMessageType.EXCEPTION + result = ex + except Exception: + logging.exception("Unexpected exception in handler") + msg_type = TMessageType.EXCEPTION + result = TApplicationException(TApplicationException.INTERNAL_ERROR, "Internal error") + oprot.writeMessageBegin("add_package", msg_type, seqid) + result.write(oprot) + oprot.writeMessageEnd() + oprot.trans.flush() + + def process_get_all_packages(self, seqid, iprot, oprot): + args = get_all_packages_args() + args.read(iprot) + iprot.readMessageEnd() + result = get_all_packages_result() + try: + result.success = self._handler.get_all_packages(args.request) + msg_type = TMessageType.REPLY + except TTransport.TTransportException: + raise + except MetaException as o1: + msg_type = TMessageType.REPLY + result.o1 = o1 + except TApplicationException as ex: + logging.exception("TApplication exception in handler") + msg_type = TMessageType.EXCEPTION + result = ex + except Exception: + logging.exception("Unexpected exception in handler") + msg_type = TMessageType.EXCEPTION + result = TApplicationException(TApplicationException.INTERNAL_ERROR, "Internal error") + oprot.writeMessageBegin("get_all_packages", msg_type, seqid) + result.write(oprot) + oprot.writeMessageEnd() + oprot.trans.flush() + + def process_drop_package(self, seqid, iprot, oprot): + args = drop_package_args() + args.read(iprot) + iprot.readMessageEnd() + result = drop_package_result() + try: + self._handler.drop_package(args.request) + msg_type = TMessageType.REPLY + except TTransport.TTransportException: + raise + except MetaException as o1: + msg_type = TMessageType.REPLY + result.o1 = o1 + except TApplicationException as ex: + logging.exception("TApplication exception in handler") + msg_type = TMessageType.EXCEPTION + result = ex + except Exception: + logging.exception("Unexpected exception in handler") + msg_type = TMessageType.EXCEPTION + result = TApplicationException(TApplicationException.INTERNAL_ERROR, "Internal error") + oprot.writeMessageBegin("drop_package", msg_type, seqid) + result.write(oprot) + oprot.writeMessageEnd() + oprot.trans.flush() + + def process_get_all_write_event_info(self, seqid, iprot, oprot): + args = get_all_write_event_info_args() + args.read(iprot) + iprot.readMessageEnd() + result = get_all_write_event_info_result() + try: + result.success = self._handler.get_all_write_event_info(args.request) + msg_type = TMessageType.REPLY + except TTransport.TTransportException: + raise + except MetaException as o1: + msg_type = TMessageType.REPLY + result.o1 = o1 + except TApplicationException as ex: + logging.exception("TApplication exception in handler") + msg_type = TMessageType.EXCEPTION + result = ex + except Exception: + logging.exception("Unexpected exception in handler") + msg_type = TMessageType.EXCEPTION + result = TApplicationException(TApplicationException.INTERNAL_ERROR, "Internal error") + oprot.writeMessageBegin("get_all_write_event_info", msg_type, seqid) + result.write(oprot) + oprot.writeMessageEnd() + oprot.trans.flush() + + +# HELPER FUNCTIONS AND STRUCTURES + + +class getMetaConf_args: + """ + Attributes: + - key + + """ + + def __init__( + self, + key=None, + ): + self.key = key + + def read(self, iprot): + if ( + iprot._fast_decode is not None + and isinstance(iprot.trans, TTransport.CReadableTransport) + and self.thrift_spec is not None + ): + iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec]) + return + iprot.readStructBegin() + while True: + (fname, ftype, fid) = iprot.readFieldBegin() + if ftype == TType.STOP: + break + if fid == 1: + if ftype == TType.STRING: + self.key = ( + iprot.readString().decode("utf-8", errors="replace") if sys.version_info[0] == 2 else iprot.readString() + ) + else: + iprot.skip(ftype) + else: + iprot.skip(ftype) + iprot.readFieldEnd() + iprot.readStructEnd() + + def write(self, oprot): + if oprot._fast_encode is not None and self.thrift_spec is not None: + oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec])) + return + oprot.writeStructBegin("getMetaConf_args") + if self.key is not None: + oprot.writeFieldBegin("key", TType.STRING, 1) + oprot.writeString(self.key.encode("utf-8") if sys.version_info[0] == 2 else self.key) + oprot.writeFieldEnd() + oprot.writeFieldStop() + oprot.writeStructEnd() + + def validate(self): + return + + def __repr__(self): + L = ["{}={!r}".format(key, value) for key, value in self.__dict__.items()] + return "{}({})".format(self.__class__.__name__, ", ".join(L)) + + def __eq__(self, other): + return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ + + def __ne__(self, other): + return not (self == other) + + +all_structs.append(getMetaConf_args) +getMetaConf_args.thrift_spec = ( + None, # 0 + ( + 1, + TType.STRING, + "key", + "UTF8", + None, + ), # 1 +) + + +class getMetaConf_result: + """ + Attributes: + - success + - o1 + + """ + + def __init__( + self, + success=None, + o1=None, + ): + self.success = success + self.o1 = o1 + + def read(self, iprot): + if ( + iprot._fast_decode is not None + and isinstance(iprot.trans, TTransport.CReadableTransport) + and self.thrift_spec is not None + ): + iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec]) + return + iprot.readStructBegin() + while True: + (fname, ftype, fid) = iprot.readFieldBegin() + if ftype == TType.STOP: + break + if fid == 0: + if ftype == TType.STRING: + self.success = ( + iprot.readString().decode("utf-8", errors="replace") if sys.version_info[0] == 2 else iprot.readString() + ) + else: + iprot.skip(ftype) + elif fid == 1: + if ftype == TType.STRUCT: + self.o1 = MetaException.read(iprot) + else: + iprot.skip(ftype) + else: + iprot.skip(ftype) + iprot.readFieldEnd() + iprot.readStructEnd() + + def write(self, oprot): + if oprot._fast_encode is not None and self.thrift_spec is not None: + oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec])) + return + oprot.writeStructBegin("getMetaConf_result") + if self.success is not None: + oprot.writeFieldBegin("success", TType.STRING, 0) + oprot.writeString(self.success.encode("utf-8") if sys.version_info[0] == 2 else self.success) + oprot.writeFieldEnd() + if self.o1 is not None: + oprot.writeFieldBegin("o1", TType.STRUCT, 1) + self.o1.write(oprot) + oprot.writeFieldEnd() + oprot.writeFieldStop() + oprot.writeStructEnd() + + def validate(self): + return + + def __repr__(self): + L = ["{}={!r}".format(key, value) for key, value in self.__dict__.items()] + return "{}({})".format(self.__class__.__name__, ", ".join(L)) + + def __eq__(self, other): + return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ + + def __ne__(self, other): + return not (self == other) + + +all_structs.append(getMetaConf_result) +getMetaConf_result.thrift_spec = ( + ( + 0, + TType.STRING, + "success", + "UTF8", + None, + ), # 0 + ( + 1, + TType.STRUCT, + "o1", + [MetaException, None], + None, + ), # 1 +) + + +class setMetaConf_args: + """ + Attributes: + - key + - value + + """ + + def __init__( + self, + key=None, + value=None, + ): + self.key = key + self.value = value + + def read(self, iprot): + if ( + iprot._fast_decode is not None + and isinstance(iprot.trans, TTransport.CReadableTransport) + and self.thrift_spec is not None + ): + iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec]) + return + iprot.readStructBegin() + while True: + (fname, ftype, fid) = iprot.readFieldBegin() + if ftype == TType.STOP: + break + if fid == 1: + if ftype == TType.STRING: + self.key = ( + iprot.readString().decode("utf-8", errors="replace") if sys.version_info[0] == 2 else iprot.readString() + ) + else: + iprot.skip(ftype) + elif fid == 2: + if ftype == TType.STRING: + self.value = ( + iprot.readString().decode("utf-8", errors="replace") if sys.version_info[0] == 2 else iprot.readString() + ) + else: + iprot.skip(ftype) + else: + iprot.skip(ftype) + iprot.readFieldEnd() + iprot.readStructEnd() + + def write(self, oprot): + if oprot._fast_encode is not None and self.thrift_spec is not None: + oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec])) + return + oprot.writeStructBegin("setMetaConf_args") + if self.key is not None: + oprot.writeFieldBegin("key", TType.STRING, 1) + oprot.writeString(self.key.encode("utf-8") if sys.version_info[0] == 2 else self.key) + oprot.writeFieldEnd() + if self.value is not None: + oprot.writeFieldBegin("value", TType.STRING, 2) + oprot.writeString(self.value.encode("utf-8") if sys.version_info[0] == 2 else self.value) + oprot.writeFieldEnd() + oprot.writeFieldStop() + oprot.writeStructEnd() + + def validate(self): + return + + def __repr__(self): + L = ["{}={!r}".format(key, value) for key, value in self.__dict__.items()] + return "{}({})".format(self.__class__.__name__, ", ".join(L)) + + def __eq__(self, other): + return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ + + def __ne__(self, other): + return not (self == other) + + +all_structs.append(setMetaConf_args) +setMetaConf_args.thrift_spec = ( + None, # 0 + ( + 1, + TType.STRING, + "key", + "UTF8", + None, + ), # 1 + ( + 2, + TType.STRING, + "value", + "UTF8", + None, + ), # 2 +) + + +class setMetaConf_result: + """ + Attributes: + - o1 + + """ + + def __init__( + self, + o1=None, + ): + self.o1 = o1 + + def read(self, iprot): + if ( + iprot._fast_decode is not None + and isinstance(iprot.trans, TTransport.CReadableTransport) + and self.thrift_spec is not None + ): + iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec]) + return + iprot.readStructBegin() + while True: + (fname, ftype, fid) = iprot.readFieldBegin() + if ftype == TType.STOP: + break + if fid == 1: + if ftype == TType.STRUCT: + self.o1 = MetaException.read(iprot) + else: + iprot.skip(ftype) + else: + iprot.skip(ftype) + iprot.readFieldEnd() + iprot.readStructEnd() + + def write(self, oprot): + if oprot._fast_encode is not None and self.thrift_spec is not None: + oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec])) + return + oprot.writeStructBegin("setMetaConf_result") + if self.o1 is not None: + oprot.writeFieldBegin("o1", TType.STRUCT, 1) + self.o1.write(oprot) + oprot.writeFieldEnd() + oprot.writeFieldStop() + oprot.writeStructEnd() + + def validate(self): + return + + def __repr__(self): + L = ["{}={!r}".format(key, value) for key, value in self.__dict__.items()] + return "{}({})".format(self.__class__.__name__, ", ".join(L)) + + def __eq__(self, other): + return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ + + def __ne__(self, other): + return not (self == other) + + +all_structs.append(setMetaConf_result) +setMetaConf_result.thrift_spec = ( + None, # 0 + ( + 1, + TType.STRUCT, + "o1", + [MetaException, None], + None, + ), # 1 +) + + +class create_catalog_args: + """ + Attributes: + - catalog + + """ + + def __init__( + self, + catalog=None, + ): + self.catalog = catalog + + def read(self, iprot): + if ( + iprot._fast_decode is not None + and isinstance(iprot.trans, TTransport.CReadableTransport) + and self.thrift_spec is not None + ): + iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec]) + return + iprot.readStructBegin() + while True: + (fname, ftype, fid) = iprot.readFieldBegin() + if ftype == TType.STOP: + break + if fid == 1: + if ftype == TType.STRUCT: + self.catalog = CreateCatalogRequest() + self.catalog.read(iprot) + else: + iprot.skip(ftype) + else: + iprot.skip(ftype) + iprot.readFieldEnd() + iprot.readStructEnd() + + def write(self, oprot): + if oprot._fast_encode is not None and self.thrift_spec is not None: + oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec])) + return + oprot.writeStructBegin("create_catalog_args") + if self.catalog is not None: + oprot.writeFieldBegin("catalog", TType.STRUCT, 1) + self.catalog.write(oprot) + oprot.writeFieldEnd() + oprot.writeFieldStop() + oprot.writeStructEnd() + + def validate(self): + return + + def __repr__(self): + L = ["{}={!r}".format(key, value) for key, value in self.__dict__.items()] + return "{}({})".format(self.__class__.__name__, ", ".join(L)) + + def __eq__(self, other): + return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ + + def __ne__(self, other): + return not (self == other) + + +all_structs.append(create_catalog_args) +create_catalog_args.thrift_spec = ( + None, # 0 + ( + 1, + TType.STRUCT, + "catalog", + [CreateCatalogRequest, None], + None, + ), # 1 +) + + +class create_catalog_result: + """ + Attributes: + - o1 + - o2 + - o3 + + """ + + def __init__( + self, + o1=None, + o2=None, + o3=None, + ): + self.o1 = o1 + self.o2 = o2 + self.o3 = o3 + + def read(self, iprot): + if ( + iprot._fast_decode is not None + and isinstance(iprot.trans, TTransport.CReadableTransport) + and self.thrift_spec is not None + ): + iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec]) + return + iprot.readStructBegin() + while True: + (fname, ftype, fid) = iprot.readFieldBegin() + if ftype == TType.STOP: + break + if fid == 1: + if ftype == TType.STRUCT: + self.o1 = AlreadyExistsException.read(iprot) + else: + iprot.skip(ftype) + elif fid == 2: + if ftype == TType.STRUCT: + self.o2 = InvalidObjectException.read(iprot) + else: + iprot.skip(ftype) + elif fid == 3: + if ftype == TType.STRUCT: + self.o3 = MetaException.read(iprot) + else: + iprot.skip(ftype) + else: + iprot.skip(ftype) + iprot.readFieldEnd() + iprot.readStructEnd() + + def write(self, oprot): + if oprot._fast_encode is not None and self.thrift_spec is not None: + oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec])) + return + oprot.writeStructBegin("create_catalog_result") + if self.o1 is not None: + oprot.writeFieldBegin("o1", TType.STRUCT, 1) + self.o1.write(oprot) + oprot.writeFieldEnd() + if self.o2 is not None: + oprot.writeFieldBegin("o2", TType.STRUCT, 2) + self.o2.write(oprot) + oprot.writeFieldEnd() + if self.o3 is not None: + oprot.writeFieldBegin("o3", TType.STRUCT, 3) + self.o3.write(oprot) + oprot.writeFieldEnd() + oprot.writeFieldStop() + oprot.writeStructEnd() + + def validate(self): + return + + def __repr__(self): + L = ["{}={!r}".format(key, value) for key, value in self.__dict__.items()] + return "{}({})".format(self.__class__.__name__, ", ".join(L)) + + def __eq__(self, other): + return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ + + def __ne__(self, other): + return not (self == other) + + +all_structs.append(create_catalog_result) +create_catalog_result.thrift_spec = ( + None, # 0 + ( + 1, + TType.STRUCT, + "o1", + [AlreadyExistsException, None], + None, + ), # 1 + ( + 2, + TType.STRUCT, + "o2", + [InvalidObjectException, None], + None, + ), # 2 + ( + 3, + TType.STRUCT, + "o3", + [MetaException, None], + None, + ), # 3 +) + + +class alter_catalog_args: + """ + Attributes: + - rqst + + """ + + def __init__( + self, + rqst=None, + ): + self.rqst = rqst + + def read(self, iprot): + if ( + iprot._fast_decode is not None + and isinstance(iprot.trans, TTransport.CReadableTransport) + and self.thrift_spec is not None + ): + iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec]) + return + iprot.readStructBegin() + while True: + (fname, ftype, fid) = iprot.readFieldBegin() + if ftype == TType.STOP: + break + if fid == 1: + if ftype == TType.STRUCT: + self.rqst = AlterCatalogRequest() + self.rqst.read(iprot) + else: + iprot.skip(ftype) + else: + iprot.skip(ftype) + iprot.readFieldEnd() + iprot.readStructEnd() + + def write(self, oprot): + if oprot._fast_encode is not None and self.thrift_spec is not None: + oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec])) + return + oprot.writeStructBegin("alter_catalog_args") + if self.rqst is not None: + oprot.writeFieldBegin("rqst", TType.STRUCT, 1) + self.rqst.write(oprot) + oprot.writeFieldEnd() + oprot.writeFieldStop() + oprot.writeStructEnd() + + def validate(self): + return + + def __repr__(self): + L = ["{}={!r}".format(key, value) for key, value in self.__dict__.items()] + return "{}({})".format(self.__class__.__name__, ", ".join(L)) + + def __eq__(self, other): + return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ + + def __ne__(self, other): + return not (self == other) + + +all_structs.append(alter_catalog_args) +alter_catalog_args.thrift_spec = ( + None, # 0 + ( + 1, + TType.STRUCT, + "rqst", + [AlterCatalogRequest, None], + None, + ), # 1 +) + + +class alter_catalog_result: + """ + Attributes: + - o1 + - o2 + - o3 + + """ + + def __init__( + self, + o1=None, + o2=None, + o3=None, + ): + self.o1 = o1 + self.o2 = o2 + self.o3 = o3 + + def read(self, iprot): + if ( + iprot._fast_decode is not None + and isinstance(iprot.trans, TTransport.CReadableTransport) + and self.thrift_spec is not None + ): + iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec]) + return + iprot.readStructBegin() + while True: + (fname, ftype, fid) = iprot.readFieldBegin() + if ftype == TType.STOP: + break + if fid == 1: + if ftype == TType.STRUCT: + self.o1 = NoSuchObjectException.read(iprot) + else: + iprot.skip(ftype) + elif fid == 2: + if ftype == TType.STRUCT: + self.o2 = InvalidOperationException.read(iprot) + else: + iprot.skip(ftype) + elif fid == 3: + if ftype == TType.STRUCT: + self.o3 = MetaException.read(iprot) + else: + iprot.skip(ftype) + else: + iprot.skip(ftype) + iprot.readFieldEnd() + iprot.readStructEnd() + + def write(self, oprot): + if oprot._fast_encode is not None and self.thrift_spec is not None: + oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec])) + return + oprot.writeStructBegin("alter_catalog_result") + if self.o1 is not None: + oprot.writeFieldBegin("o1", TType.STRUCT, 1) + self.o1.write(oprot) + oprot.writeFieldEnd() + if self.o2 is not None: + oprot.writeFieldBegin("o2", TType.STRUCT, 2) + self.o2.write(oprot) + oprot.writeFieldEnd() + if self.o3 is not None: + oprot.writeFieldBegin("o3", TType.STRUCT, 3) + self.o3.write(oprot) + oprot.writeFieldEnd() + oprot.writeFieldStop() + oprot.writeStructEnd() + + def validate(self): + return + + def __repr__(self): + L = ["{}={!r}".format(key, value) for key, value in self.__dict__.items()] + return "{}({})".format(self.__class__.__name__, ", ".join(L)) + + def __eq__(self, other): + return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ + + def __ne__(self, other): + return not (self == other) + + +all_structs.append(alter_catalog_result) +alter_catalog_result.thrift_spec = ( + None, # 0 + ( + 1, + TType.STRUCT, + "o1", + [NoSuchObjectException, None], + None, + ), # 1 + ( + 2, + TType.STRUCT, + "o2", + [InvalidOperationException, None], + None, + ), # 2 + ( + 3, + TType.STRUCT, + "o3", + [MetaException, None], + None, + ), # 3 +) + + +class get_catalog_args: + """ + Attributes: + - catName + + """ + + def __init__( + self, + catName=None, + ): + self.catName = catName + + def read(self, iprot): + if ( + iprot._fast_decode is not None + and isinstance(iprot.trans, TTransport.CReadableTransport) + and self.thrift_spec is not None + ): + iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec]) + return + iprot.readStructBegin() + while True: + (fname, ftype, fid) = iprot.readFieldBegin() + if ftype == TType.STOP: + break + if fid == 1: + if ftype == TType.STRUCT: + self.catName = GetCatalogRequest() + self.catName.read(iprot) + else: + iprot.skip(ftype) + else: + iprot.skip(ftype) + iprot.readFieldEnd() + iprot.readStructEnd() + + def write(self, oprot): + if oprot._fast_encode is not None and self.thrift_spec is not None: + oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec])) + return + oprot.writeStructBegin("get_catalog_args") + if self.catName is not None: + oprot.writeFieldBegin("catName", TType.STRUCT, 1) + self.catName.write(oprot) + oprot.writeFieldEnd() + oprot.writeFieldStop() + oprot.writeStructEnd() + + def validate(self): + return + + def __repr__(self): + L = ["{}={!r}".format(key, value) for key, value in self.__dict__.items()] + return "{}({})".format(self.__class__.__name__, ", ".join(L)) + + def __eq__(self, other): + return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ + + def __ne__(self, other): + return not (self == other) + + +all_structs.append(get_catalog_args) +get_catalog_args.thrift_spec = ( + None, # 0 + ( + 1, + TType.STRUCT, + "catName", + [GetCatalogRequest, None], + None, + ), # 1 +) + + +class get_catalog_result: + """ + Attributes: + - success + - o1 + - o2 + + """ + + def __init__( + self, + success=None, + o1=None, + o2=None, + ): + self.success = success + self.o1 = o1 + self.o2 = o2 + + def read(self, iprot): + if ( + iprot._fast_decode is not None + and isinstance(iprot.trans, TTransport.CReadableTransport) + and self.thrift_spec is not None + ): + iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec]) + return + iprot.readStructBegin() + while True: + (fname, ftype, fid) = iprot.readFieldBegin() + if ftype == TType.STOP: + break + if fid == 0: + if ftype == TType.STRUCT: + self.success = GetCatalogResponse() + self.success.read(iprot) + else: + iprot.skip(ftype) + elif fid == 1: + if ftype == TType.STRUCT: + self.o1 = NoSuchObjectException.read(iprot) + else: + iprot.skip(ftype) + elif fid == 2: + if ftype == TType.STRUCT: + self.o2 = MetaException.read(iprot) + else: + iprot.skip(ftype) + else: + iprot.skip(ftype) + iprot.readFieldEnd() + iprot.readStructEnd() + + def write(self, oprot): + if oprot._fast_encode is not None and self.thrift_spec is not None: + oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec])) + return + oprot.writeStructBegin("get_catalog_result") + if self.success is not None: + oprot.writeFieldBegin("success", TType.STRUCT, 0) + self.success.write(oprot) + oprot.writeFieldEnd() + if self.o1 is not None: + oprot.writeFieldBegin("o1", TType.STRUCT, 1) + self.o1.write(oprot) + oprot.writeFieldEnd() + if self.o2 is not None: + oprot.writeFieldBegin("o2", TType.STRUCT, 2) + self.o2.write(oprot) + oprot.writeFieldEnd() + oprot.writeFieldStop() + oprot.writeStructEnd() + + def validate(self): + return + + def __repr__(self): + L = ["{}={!r}".format(key, value) for key, value in self.__dict__.items()] + return "{}({})".format(self.__class__.__name__, ", ".join(L)) + + def __eq__(self, other): + return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ + + def __ne__(self, other): + return not (self == other) + + +all_structs.append(get_catalog_result) +get_catalog_result.thrift_spec = ( + ( + 0, + TType.STRUCT, + "success", + [GetCatalogResponse, None], + None, + ), # 0 + ( + 1, + TType.STRUCT, + "o1", + [NoSuchObjectException, None], + None, + ), # 1 + ( + 2, + TType.STRUCT, + "o2", + [MetaException, None], + None, + ), # 2 +) + + +class get_catalogs_args: + def read(self, iprot): + if ( + iprot._fast_decode is not None + and isinstance(iprot.trans, TTransport.CReadableTransport) + and self.thrift_spec is not None + ): + iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec]) + return + iprot.readStructBegin() + while True: + (fname, ftype, fid) = iprot.readFieldBegin() + if ftype == TType.STOP: + break + else: + iprot.skip(ftype) + iprot.readFieldEnd() + iprot.readStructEnd() + + def write(self, oprot): + if oprot._fast_encode is not None and self.thrift_spec is not None: + oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec])) + return + oprot.writeStructBegin("get_catalogs_args") + oprot.writeFieldStop() + oprot.writeStructEnd() + + def validate(self): + return + + def __repr__(self): + L = ["{}={!r}".format(key, value) for key, value in self.__dict__.items()] + return "{}({})".format(self.__class__.__name__, ", ".join(L)) + + def __eq__(self, other): + return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ + + def __ne__(self, other): + return not (self == other) + + +all_structs.append(get_catalogs_args) +get_catalogs_args.thrift_spec = () + + +class get_catalogs_result: + """ + Attributes: + - success + - o1 + + """ + + def __init__( + self, + success=None, + o1=None, + ): + self.success = success + self.o1 = o1 + + def read(self, iprot): + if ( + iprot._fast_decode is not None + and isinstance(iprot.trans, TTransport.CReadableTransport) + and self.thrift_spec is not None + ): + iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec]) + return + iprot.readStructBegin() + while True: + (fname, ftype, fid) = iprot.readFieldBegin() + if ftype == TType.STOP: + break + if fid == 0: + if ftype == TType.STRUCT: + self.success = GetCatalogsResponse() + self.success.read(iprot) + else: + iprot.skip(ftype) + elif fid == 1: + if ftype == TType.STRUCT: + self.o1 = MetaException.read(iprot) + else: + iprot.skip(ftype) + else: + iprot.skip(ftype) + iprot.readFieldEnd() + iprot.readStructEnd() + + def write(self, oprot): + if oprot._fast_encode is not None and self.thrift_spec is not None: + oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec])) + return + oprot.writeStructBegin("get_catalogs_result") + if self.success is not None: + oprot.writeFieldBegin("success", TType.STRUCT, 0) + self.success.write(oprot) + oprot.writeFieldEnd() + if self.o1 is not None: + oprot.writeFieldBegin("o1", TType.STRUCT, 1) + self.o1.write(oprot) + oprot.writeFieldEnd() + oprot.writeFieldStop() + oprot.writeStructEnd() + + def validate(self): + return + + def __repr__(self): + L = ["{}={!r}".format(key, value) for key, value in self.__dict__.items()] + return "{}({})".format(self.__class__.__name__, ", ".join(L)) + + def __eq__(self, other): + return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ + + def __ne__(self, other): + return not (self == other) + + +all_structs.append(get_catalogs_result) +get_catalogs_result.thrift_spec = ( + ( + 0, + TType.STRUCT, + "success", + [GetCatalogsResponse, None], + None, + ), # 0 + ( + 1, + TType.STRUCT, + "o1", + [MetaException, None], + None, + ), # 1 +) + + +class drop_catalog_args: + """ + Attributes: + - catName + + """ + + def __init__( + self, + catName=None, + ): + self.catName = catName + + def read(self, iprot): + if ( + iprot._fast_decode is not None + and isinstance(iprot.trans, TTransport.CReadableTransport) + and self.thrift_spec is not None + ): + iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec]) + return + iprot.readStructBegin() + while True: + (fname, ftype, fid) = iprot.readFieldBegin() + if ftype == TType.STOP: + break + if fid == 1: + if ftype == TType.STRUCT: + self.catName = DropCatalogRequest() + self.catName.read(iprot) + else: + iprot.skip(ftype) + else: + iprot.skip(ftype) + iprot.readFieldEnd() + iprot.readStructEnd() + + def write(self, oprot): + if oprot._fast_encode is not None and self.thrift_spec is not None: + oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec])) + return + oprot.writeStructBegin("drop_catalog_args") + if self.catName is not None: + oprot.writeFieldBegin("catName", TType.STRUCT, 1) + self.catName.write(oprot) + oprot.writeFieldEnd() + oprot.writeFieldStop() + oprot.writeStructEnd() + + def validate(self): + return + + def __repr__(self): + L = ["{}={!r}".format(key, value) for key, value in self.__dict__.items()] + return "{}({})".format(self.__class__.__name__, ", ".join(L)) + + def __eq__(self, other): + return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ + + def __ne__(self, other): + return not (self == other) + + +all_structs.append(drop_catalog_args) +drop_catalog_args.thrift_spec = ( + None, # 0 + ( + 1, + TType.STRUCT, + "catName", + [DropCatalogRequest, None], + None, + ), # 1 +) + + +class drop_catalog_result: + """ + Attributes: + - o1 + - o2 + - o3 + + """ + + def __init__( + self, + o1=None, + o2=None, + o3=None, + ): + self.o1 = o1 + self.o2 = o2 + self.o3 = o3 + + def read(self, iprot): + if ( + iprot._fast_decode is not None + and isinstance(iprot.trans, TTransport.CReadableTransport) + and self.thrift_spec is not None + ): + iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec]) + return + iprot.readStructBegin() + while True: + (fname, ftype, fid) = iprot.readFieldBegin() + if ftype == TType.STOP: + break + if fid == 1: + if ftype == TType.STRUCT: + self.o1 = NoSuchObjectException.read(iprot) + else: + iprot.skip(ftype) + elif fid == 2: + if ftype == TType.STRUCT: + self.o2 = InvalidOperationException.read(iprot) + else: + iprot.skip(ftype) + elif fid == 3: + if ftype == TType.STRUCT: + self.o3 = MetaException.read(iprot) + else: + iprot.skip(ftype) + else: + iprot.skip(ftype) + iprot.readFieldEnd() + iprot.readStructEnd() + + def write(self, oprot): + if oprot._fast_encode is not None and self.thrift_spec is not None: + oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec])) + return + oprot.writeStructBegin("drop_catalog_result") + if self.o1 is not None: + oprot.writeFieldBegin("o1", TType.STRUCT, 1) + self.o1.write(oprot) + oprot.writeFieldEnd() + if self.o2 is not None: + oprot.writeFieldBegin("o2", TType.STRUCT, 2) + self.o2.write(oprot) + oprot.writeFieldEnd() + if self.o3 is not None: + oprot.writeFieldBegin("o3", TType.STRUCT, 3) + self.o3.write(oprot) + oprot.writeFieldEnd() + oprot.writeFieldStop() + oprot.writeStructEnd() + + def validate(self): + return + + def __repr__(self): + L = ["{}={!r}".format(key, value) for key, value in self.__dict__.items()] + return "{}({})".format(self.__class__.__name__, ", ".join(L)) + + def __eq__(self, other): + return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ + + def __ne__(self, other): + return not (self == other) + + +all_structs.append(drop_catalog_result) +drop_catalog_result.thrift_spec = ( + None, # 0 + ( + 1, + TType.STRUCT, + "o1", + [NoSuchObjectException, None], + None, + ), # 1 + ( + 2, + TType.STRUCT, + "o2", + [InvalidOperationException, None], + None, + ), # 2 + ( + 3, + TType.STRUCT, + "o3", + [MetaException, None], + None, + ), # 3 +) + + +class create_database_args: + """ + Attributes: + - database + + """ + + def __init__( + self, + database=None, + ): + self.database = database + + def read(self, iprot): + if ( + iprot._fast_decode is not None + and isinstance(iprot.trans, TTransport.CReadableTransport) + and self.thrift_spec is not None + ): + iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec]) + return + iprot.readStructBegin() + while True: + (fname, ftype, fid) = iprot.readFieldBegin() + if ftype == TType.STOP: + break + if fid == 1: + if ftype == TType.STRUCT: + self.database = Database() + self.database.read(iprot) + else: + iprot.skip(ftype) + else: + iprot.skip(ftype) + iprot.readFieldEnd() + iprot.readStructEnd() + + def write(self, oprot): + if oprot._fast_encode is not None and self.thrift_spec is not None: + oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec])) + return + oprot.writeStructBegin("create_database_args") + if self.database is not None: + oprot.writeFieldBegin("database", TType.STRUCT, 1) + self.database.write(oprot) + oprot.writeFieldEnd() + oprot.writeFieldStop() + oprot.writeStructEnd() + + def validate(self): + return + + def __repr__(self): + L = ["{}={!r}".format(key, value) for key, value in self.__dict__.items()] + return "{}({})".format(self.__class__.__name__, ", ".join(L)) + + def __eq__(self, other): + return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ + + def __ne__(self, other): + return not (self == other) + + +all_structs.append(create_database_args) +create_database_args.thrift_spec = ( + None, # 0 + ( + 1, + TType.STRUCT, + "database", + [Database, None], + None, + ), # 1 +) + + +class create_database_result: + """ + Attributes: + - o1 + - o2 + - o3 + + """ + + def __init__( + self, + o1=None, + o2=None, + o3=None, + ): + self.o1 = o1 + self.o2 = o2 + self.o3 = o3 + + def read(self, iprot): + if ( + iprot._fast_decode is not None + and isinstance(iprot.trans, TTransport.CReadableTransport) + and self.thrift_spec is not None + ): + iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec]) + return + iprot.readStructBegin() + while True: + (fname, ftype, fid) = iprot.readFieldBegin() + if ftype == TType.STOP: + break + if fid == 1: + if ftype == TType.STRUCT: + self.o1 = AlreadyExistsException.read(iprot) + else: + iprot.skip(ftype) + elif fid == 2: + if ftype == TType.STRUCT: + self.o2 = InvalidObjectException.read(iprot) + else: + iprot.skip(ftype) + elif fid == 3: + if ftype == TType.STRUCT: + self.o3 = MetaException.read(iprot) + else: + iprot.skip(ftype) + else: + iprot.skip(ftype) + iprot.readFieldEnd() + iprot.readStructEnd() + + def write(self, oprot): + if oprot._fast_encode is not None and self.thrift_spec is not None: + oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec])) + return + oprot.writeStructBegin("create_database_result") + if self.o1 is not None: + oprot.writeFieldBegin("o1", TType.STRUCT, 1) + self.o1.write(oprot) + oprot.writeFieldEnd() + if self.o2 is not None: + oprot.writeFieldBegin("o2", TType.STRUCT, 2) + self.o2.write(oprot) + oprot.writeFieldEnd() + if self.o3 is not None: + oprot.writeFieldBegin("o3", TType.STRUCT, 3) + self.o3.write(oprot) + oprot.writeFieldEnd() + oprot.writeFieldStop() + oprot.writeStructEnd() + + def validate(self): + return + + def __repr__(self): + L = ["{}={!r}".format(key, value) for key, value in self.__dict__.items()] + return "{}({})".format(self.__class__.__name__, ", ".join(L)) + + def __eq__(self, other): + return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ + + def __ne__(self, other): + return not (self == other) + + +all_structs.append(create_database_result) +create_database_result.thrift_spec = ( + None, # 0 + ( + 1, + TType.STRUCT, + "o1", + [AlreadyExistsException, None], + None, + ), # 1 + ( + 2, + TType.STRUCT, + "o2", + [InvalidObjectException, None], + None, + ), # 2 + ( + 3, + TType.STRUCT, + "o3", + [MetaException, None], + None, + ), # 3 +) + + +class get_database_args: + """ + Attributes: + - name + + """ + + def __init__( + self, + name=None, + ): + self.name = name + + def read(self, iprot): + if ( + iprot._fast_decode is not None + and isinstance(iprot.trans, TTransport.CReadableTransport) + and self.thrift_spec is not None + ): + iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec]) + return + iprot.readStructBegin() + while True: + (fname, ftype, fid) = iprot.readFieldBegin() + if ftype == TType.STOP: + break + if fid == 1: + if ftype == TType.STRING: + self.name = ( + iprot.readString().decode("utf-8", errors="replace") if sys.version_info[0] == 2 else iprot.readString() + ) + else: + iprot.skip(ftype) + else: + iprot.skip(ftype) + iprot.readFieldEnd() + iprot.readStructEnd() + + def write(self, oprot): + if oprot._fast_encode is not None and self.thrift_spec is not None: + oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec])) + return + oprot.writeStructBegin("get_database_args") + if self.name is not None: + oprot.writeFieldBegin("name", TType.STRING, 1) + oprot.writeString(self.name.encode("utf-8") if sys.version_info[0] == 2 else self.name) + oprot.writeFieldEnd() + oprot.writeFieldStop() + oprot.writeStructEnd() + + def validate(self): + return + + def __repr__(self): + L = ["{}={!r}".format(key, value) for key, value in self.__dict__.items()] + return "{}({})".format(self.__class__.__name__, ", ".join(L)) + + def __eq__(self, other): + return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ + + def __ne__(self, other): + return not (self == other) + + +all_structs.append(get_database_args) +get_database_args.thrift_spec = ( + None, # 0 + ( + 1, + TType.STRING, + "name", + "UTF8", + None, + ), # 1 +) + + +class get_database_result: + """ + Attributes: + - success + - o1 + - o2 + + """ + + def __init__( + self, + success=None, + o1=None, + o2=None, + ): + self.success = success + self.o1 = o1 + self.o2 = o2 + + def read(self, iprot): + if ( + iprot._fast_decode is not None + and isinstance(iprot.trans, TTransport.CReadableTransport) + and self.thrift_spec is not None + ): + iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec]) + return + iprot.readStructBegin() + while True: + (fname, ftype, fid) = iprot.readFieldBegin() + if ftype == TType.STOP: + break + if fid == 0: + if ftype == TType.STRUCT: + self.success = Database() + self.success.read(iprot) + else: + iprot.skip(ftype) + elif fid == 1: + if ftype == TType.STRUCT: + self.o1 = NoSuchObjectException.read(iprot) + else: + iprot.skip(ftype) + elif fid == 2: + if ftype == TType.STRUCT: + self.o2 = MetaException.read(iprot) + else: + iprot.skip(ftype) + else: + iprot.skip(ftype) + iprot.readFieldEnd() + iprot.readStructEnd() + + def write(self, oprot): + if oprot._fast_encode is not None and self.thrift_spec is not None: + oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec])) + return + oprot.writeStructBegin("get_database_result") + if self.success is not None: + oprot.writeFieldBegin("success", TType.STRUCT, 0) + self.success.write(oprot) + oprot.writeFieldEnd() + if self.o1 is not None: + oprot.writeFieldBegin("o1", TType.STRUCT, 1) + self.o1.write(oprot) + oprot.writeFieldEnd() + if self.o2 is not None: + oprot.writeFieldBegin("o2", TType.STRUCT, 2) + self.o2.write(oprot) + oprot.writeFieldEnd() + oprot.writeFieldStop() + oprot.writeStructEnd() + + def validate(self): + return + + def __repr__(self): + L = ["{}={!r}".format(key, value) for key, value in self.__dict__.items()] + return "{}({})".format(self.__class__.__name__, ", ".join(L)) + + def __eq__(self, other): + return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ + + def __ne__(self, other): + return not (self == other) + + +all_structs.append(get_database_result) +get_database_result.thrift_spec = ( + ( + 0, + TType.STRUCT, + "success", + [Database, None], + None, + ), # 0 + ( + 1, + TType.STRUCT, + "o1", + [NoSuchObjectException, None], + None, + ), # 1 + ( + 2, + TType.STRUCT, + "o2", + [MetaException, None], + None, + ), # 2 +) + + +class get_database_req_args: + """ + Attributes: + - request + + """ + + def __init__( + self, + request=None, + ): + self.request = request + + def read(self, iprot): + if ( + iprot._fast_decode is not None + and isinstance(iprot.trans, TTransport.CReadableTransport) + and self.thrift_spec is not None + ): + iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec]) + return + iprot.readStructBegin() + while True: + (fname, ftype, fid) = iprot.readFieldBegin() + if ftype == TType.STOP: + break + if fid == 1: + if ftype == TType.STRUCT: + self.request = GetDatabaseRequest() + self.request.read(iprot) + else: + iprot.skip(ftype) + else: + iprot.skip(ftype) + iprot.readFieldEnd() + iprot.readStructEnd() + + def write(self, oprot): + if oprot._fast_encode is not None and self.thrift_spec is not None: + oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec])) + return + oprot.writeStructBegin("get_database_req_args") + if self.request is not None: + oprot.writeFieldBegin("request", TType.STRUCT, 1) + self.request.write(oprot) + oprot.writeFieldEnd() + oprot.writeFieldStop() + oprot.writeStructEnd() + + def validate(self): + return + + def __repr__(self): + L = ["{}={!r}".format(key, value) for key, value in self.__dict__.items()] + return "{}({})".format(self.__class__.__name__, ", ".join(L)) + + def __eq__(self, other): + return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ + + def __ne__(self, other): + return not (self == other) + + +all_structs.append(get_database_req_args) +get_database_req_args.thrift_spec = ( + None, # 0 + ( + 1, + TType.STRUCT, + "request", + [GetDatabaseRequest, None], + None, + ), # 1 +) + + +class get_database_req_result: + """ + Attributes: + - success + - o1 + - o2 + + """ + + def __init__( + self, + success=None, + o1=None, + o2=None, + ): + self.success = success + self.o1 = o1 + self.o2 = o2 + + def read(self, iprot): + if ( + iprot._fast_decode is not None + and isinstance(iprot.trans, TTransport.CReadableTransport) + and self.thrift_spec is not None + ): + iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec]) + return + iprot.readStructBegin() + while True: + (fname, ftype, fid) = iprot.readFieldBegin() + if ftype == TType.STOP: + break + if fid == 0: + if ftype == TType.STRUCT: + self.success = Database() + self.success.read(iprot) + else: + iprot.skip(ftype) + elif fid == 1: + if ftype == TType.STRUCT: + self.o1 = NoSuchObjectException.read(iprot) + else: + iprot.skip(ftype) + elif fid == 2: + if ftype == TType.STRUCT: + self.o2 = MetaException.read(iprot) + else: + iprot.skip(ftype) + else: + iprot.skip(ftype) + iprot.readFieldEnd() + iprot.readStructEnd() + + def write(self, oprot): + if oprot._fast_encode is not None and self.thrift_spec is not None: + oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec])) + return + oprot.writeStructBegin("get_database_req_result") + if self.success is not None: + oprot.writeFieldBegin("success", TType.STRUCT, 0) + self.success.write(oprot) + oprot.writeFieldEnd() + if self.o1 is not None: + oprot.writeFieldBegin("o1", TType.STRUCT, 1) + self.o1.write(oprot) + oprot.writeFieldEnd() + if self.o2 is not None: + oprot.writeFieldBegin("o2", TType.STRUCT, 2) + self.o2.write(oprot) + oprot.writeFieldEnd() + oprot.writeFieldStop() + oprot.writeStructEnd() + + def validate(self): + return + + def __repr__(self): + L = ["{}={!r}".format(key, value) for key, value in self.__dict__.items()] + return "{}({})".format(self.__class__.__name__, ", ".join(L)) + + def __eq__(self, other): + return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ + + def __ne__(self, other): + return not (self == other) + + +all_structs.append(get_database_req_result) +get_database_req_result.thrift_spec = ( + ( + 0, + TType.STRUCT, + "success", + [Database, None], + None, + ), # 0 + ( + 1, + TType.STRUCT, + "o1", + [NoSuchObjectException, None], + None, + ), # 1 + ( + 2, + TType.STRUCT, + "o2", + [MetaException, None], + None, + ), # 2 +) + + +class drop_database_args: + """ + Attributes: + - name + - deleteData + - cascade + + """ + + def __init__( + self, + name=None, + deleteData=None, + cascade=None, + ): + self.name = name + self.deleteData = deleteData + self.cascade = cascade + + def read(self, iprot): + if ( + iprot._fast_decode is not None + and isinstance(iprot.trans, TTransport.CReadableTransport) + and self.thrift_spec is not None + ): + iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec]) + return + iprot.readStructBegin() + while True: + (fname, ftype, fid) = iprot.readFieldBegin() + if ftype == TType.STOP: + break + if fid == 1: + if ftype == TType.STRING: + self.name = ( + iprot.readString().decode("utf-8", errors="replace") if sys.version_info[0] == 2 else iprot.readString() + ) + else: + iprot.skip(ftype) + elif fid == 2: + if ftype == TType.BOOL: + self.deleteData = iprot.readBool() + else: + iprot.skip(ftype) + elif fid == 3: + if ftype == TType.BOOL: + self.cascade = iprot.readBool() + else: + iprot.skip(ftype) + else: + iprot.skip(ftype) + iprot.readFieldEnd() + iprot.readStructEnd() + + def write(self, oprot): + if oprot._fast_encode is not None and self.thrift_spec is not None: + oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec])) + return + oprot.writeStructBegin("drop_database_args") + if self.name is not None: + oprot.writeFieldBegin("name", TType.STRING, 1) + oprot.writeString(self.name.encode("utf-8") if sys.version_info[0] == 2 else self.name) + oprot.writeFieldEnd() + if self.deleteData is not None: + oprot.writeFieldBegin("deleteData", TType.BOOL, 2) + oprot.writeBool(self.deleteData) + oprot.writeFieldEnd() + if self.cascade is not None: + oprot.writeFieldBegin("cascade", TType.BOOL, 3) + oprot.writeBool(self.cascade) + oprot.writeFieldEnd() + oprot.writeFieldStop() + oprot.writeStructEnd() + + def validate(self): + return + + def __repr__(self): + L = ["{}={!r}".format(key, value) for key, value in self.__dict__.items()] + return "{}({})".format(self.__class__.__name__, ", ".join(L)) + + def __eq__(self, other): + return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ + + def __ne__(self, other): + return not (self == other) + + +all_structs.append(drop_database_args) +drop_database_args.thrift_spec = ( + None, # 0 + ( + 1, + TType.STRING, + "name", + "UTF8", + None, + ), # 1 + ( + 2, + TType.BOOL, + "deleteData", + None, + None, + ), # 2 + ( + 3, + TType.BOOL, + "cascade", + None, + None, + ), # 3 +) + + +class drop_database_result: + """ + Attributes: + - o1 + - o2 + - o3 + + """ + + def __init__( + self, + o1=None, + o2=None, + o3=None, + ): + self.o1 = o1 + self.o2 = o2 + self.o3 = o3 + + def read(self, iprot): + if ( + iprot._fast_decode is not None + and isinstance(iprot.trans, TTransport.CReadableTransport) + and self.thrift_spec is not None + ): + iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec]) + return + iprot.readStructBegin() + while True: + (fname, ftype, fid) = iprot.readFieldBegin() + if ftype == TType.STOP: + break + if fid == 1: + if ftype == TType.STRUCT: + self.o1 = NoSuchObjectException.read(iprot) + else: + iprot.skip(ftype) + elif fid == 2: + if ftype == TType.STRUCT: + self.o2 = InvalidOperationException.read(iprot) + else: + iprot.skip(ftype) + elif fid == 3: + if ftype == TType.STRUCT: + self.o3 = MetaException.read(iprot) + else: + iprot.skip(ftype) + else: + iprot.skip(ftype) + iprot.readFieldEnd() + iprot.readStructEnd() + + def write(self, oprot): + if oprot._fast_encode is not None and self.thrift_spec is not None: + oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec])) + return + oprot.writeStructBegin("drop_database_result") + if self.o1 is not None: + oprot.writeFieldBegin("o1", TType.STRUCT, 1) + self.o1.write(oprot) + oprot.writeFieldEnd() + if self.o2 is not None: + oprot.writeFieldBegin("o2", TType.STRUCT, 2) + self.o2.write(oprot) + oprot.writeFieldEnd() + if self.o3 is not None: + oprot.writeFieldBegin("o3", TType.STRUCT, 3) + self.o3.write(oprot) + oprot.writeFieldEnd() + oprot.writeFieldStop() + oprot.writeStructEnd() + + def validate(self): + return + + def __repr__(self): + L = ["{}={!r}".format(key, value) for key, value in self.__dict__.items()] + return "{}({})".format(self.__class__.__name__, ", ".join(L)) + + def __eq__(self, other): + return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ + + def __ne__(self, other): + return not (self == other) + + +all_structs.append(drop_database_result) +drop_database_result.thrift_spec = ( + None, # 0 + ( + 1, + TType.STRUCT, + "o1", + [NoSuchObjectException, None], + None, + ), # 1 + ( + 2, + TType.STRUCT, + "o2", + [InvalidOperationException, None], + None, + ), # 2 + ( + 3, + TType.STRUCT, + "o3", + [MetaException, None], + None, + ), # 3 +) + + +class drop_database_req_args: + """ + Attributes: + - req + + """ + + def __init__( + self, + req=None, + ): + self.req = req + + def read(self, iprot): + if ( + iprot._fast_decode is not None + and isinstance(iprot.trans, TTransport.CReadableTransport) + and self.thrift_spec is not None + ): + iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec]) + return + iprot.readStructBegin() + while True: + (fname, ftype, fid) = iprot.readFieldBegin() + if ftype == TType.STOP: + break + if fid == 1: + if ftype == TType.STRUCT: + self.req = DropDatabaseRequest() + self.req.read(iprot) + else: + iprot.skip(ftype) + else: + iprot.skip(ftype) + iprot.readFieldEnd() + iprot.readStructEnd() + + def write(self, oprot): + if oprot._fast_encode is not None and self.thrift_spec is not None: + oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec])) + return + oprot.writeStructBegin("drop_database_req_args") + if self.req is not None: + oprot.writeFieldBegin("req", TType.STRUCT, 1) + self.req.write(oprot) + oprot.writeFieldEnd() + oprot.writeFieldStop() + oprot.writeStructEnd() + + def validate(self): + return + + def __repr__(self): + L = ["{}={!r}".format(key, value) for key, value in self.__dict__.items()] + return "{}({})".format(self.__class__.__name__, ", ".join(L)) + + def __eq__(self, other): + return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ + + def __ne__(self, other): + return not (self == other) + + +all_structs.append(drop_database_req_args) +drop_database_req_args.thrift_spec = ( + None, # 0 + ( + 1, + TType.STRUCT, + "req", + [DropDatabaseRequest, None], + None, + ), # 1 +) + + +class drop_database_req_result: + """ + Attributes: + - o1 + - o2 + - o3 + + """ + + def __init__( + self, + o1=None, + o2=None, + o3=None, + ): + self.o1 = o1 + self.o2 = o2 + self.o3 = o3 + + def read(self, iprot): + if ( + iprot._fast_decode is not None + and isinstance(iprot.trans, TTransport.CReadableTransport) + and self.thrift_spec is not None + ): + iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec]) + return + iprot.readStructBegin() + while True: + (fname, ftype, fid) = iprot.readFieldBegin() + if ftype == TType.STOP: + break + if fid == 1: + if ftype == TType.STRUCT: + self.o1 = NoSuchObjectException.read(iprot) + else: + iprot.skip(ftype) + elif fid == 2: + if ftype == TType.STRUCT: + self.o2 = InvalidOperationException.read(iprot) + else: + iprot.skip(ftype) + elif fid == 3: + if ftype == TType.STRUCT: + self.o3 = MetaException.read(iprot) + else: + iprot.skip(ftype) + else: + iprot.skip(ftype) + iprot.readFieldEnd() + iprot.readStructEnd() + + def write(self, oprot): + if oprot._fast_encode is not None and self.thrift_spec is not None: + oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec])) + return + oprot.writeStructBegin("drop_database_req_result") + if self.o1 is not None: + oprot.writeFieldBegin("o1", TType.STRUCT, 1) + self.o1.write(oprot) + oprot.writeFieldEnd() + if self.o2 is not None: + oprot.writeFieldBegin("o2", TType.STRUCT, 2) + self.o2.write(oprot) + oprot.writeFieldEnd() + if self.o3 is not None: + oprot.writeFieldBegin("o3", TType.STRUCT, 3) + self.o3.write(oprot) + oprot.writeFieldEnd() + oprot.writeFieldStop() + oprot.writeStructEnd() + + def validate(self): + return + + def __repr__(self): + L = ["{}={!r}".format(key, value) for key, value in self.__dict__.items()] + return "{}({})".format(self.__class__.__name__, ", ".join(L)) + + def __eq__(self, other): + return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ + + def __ne__(self, other): + return not (self == other) + + +all_structs.append(drop_database_req_result) +drop_database_req_result.thrift_spec = ( + None, # 0 + ( + 1, + TType.STRUCT, + "o1", + [NoSuchObjectException, None], + None, + ), # 1 + ( + 2, + TType.STRUCT, + "o2", + [InvalidOperationException, None], + None, + ), # 2 + ( + 3, + TType.STRUCT, + "o3", + [MetaException, None], + None, + ), # 3 +) + + +class get_databases_args: + """ + Attributes: + - pattern + + """ + + def __init__( + self, + pattern=None, + ): + self.pattern = pattern + + def read(self, iprot): + if ( + iprot._fast_decode is not None + and isinstance(iprot.trans, TTransport.CReadableTransport) + and self.thrift_spec is not None + ): + iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec]) + return + iprot.readStructBegin() + while True: + (fname, ftype, fid) = iprot.readFieldBegin() + if ftype == TType.STOP: + break + if fid == 1: + if ftype == TType.STRING: + self.pattern = ( + iprot.readString().decode("utf-8", errors="replace") if sys.version_info[0] == 2 else iprot.readString() + ) + else: + iprot.skip(ftype) + else: + iprot.skip(ftype) + iprot.readFieldEnd() + iprot.readStructEnd() + + def write(self, oprot): + if oprot._fast_encode is not None and self.thrift_spec is not None: + oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec])) + return + oprot.writeStructBegin("get_databases_args") + if self.pattern is not None: + oprot.writeFieldBegin("pattern", TType.STRING, 1) + oprot.writeString(self.pattern.encode("utf-8") if sys.version_info[0] == 2 else self.pattern) + oprot.writeFieldEnd() + oprot.writeFieldStop() + oprot.writeStructEnd() + + def validate(self): + return + + def __repr__(self): + L = ["{}={!r}".format(key, value) for key, value in self.__dict__.items()] + return "{}({})".format(self.__class__.__name__, ", ".join(L)) + + def __eq__(self, other): + return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ + + def __ne__(self, other): + return not (self == other) + + +all_structs.append(get_databases_args) +get_databases_args.thrift_spec = ( + None, # 0 + ( + 1, + TType.STRING, + "pattern", + "UTF8", + None, + ), # 1 +) + + +class get_databases_result: + """ + Attributes: + - success + - o1 + + """ + + def __init__( + self, + success=None, + o1=None, + ): + self.success = success + self.o1 = o1 + + def read(self, iprot): + if ( + iprot._fast_decode is not None + and isinstance(iprot.trans, TTransport.CReadableTransport) + and self.thrift_spec is not None + ): + iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec]) + return + iprot.readStructBegin() + while True: + (fname, ftype, fid) = iprot.readFieldBegin() + if ftype == TType.STOP: + break + if fid == 0: + if ftype == TType.LIST: + self.success = [] + (_etype1252, _size1249) = iprot.readListBegin() + for _i1253 in range(_size1249): + _elem1254 = ( + iprot.readString().decode("utf-8", errors="replace") + if sys.version_info[0] == 2 + else iprot.readString() + ) + self.success.append(_elem1254) + iprot.readListEnd() + else: + iprot.skip(ftype) + elif fid == 1: + if ftype == TType.STRUCT: + self.o1 = MetaException.read(iprot) + else: + iprot.skip(ftype) + else: + iprot.skip(ftype) + iprot.readFieldEnd() + iprot.readStructEnd() + + def write(self, oprot): + if oprot._fast_encode is not None and self.thrift_spec is not None: + oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec])) + return + oprot.writeStructBegin("get_databases_result") + if self.success is not None: + oprot.writeFieldBegin("success", TType.LIST, 0) + oprot.writeListBegin(TType.STRING, len(self.success)) + for iter1255 in self.success: + oprot.writeString(iter1255.encode("utf-8") if sys.version_info[0] == 2 else iter1255) + oprot.writeListEnd() + oprot.writeFieldEnd() + if self.o1 is not None: + oprot.writeFieldBegin("o1", TType.STRUCT, 1) + self.o1.write(oprot) + oprot.writeFieldEnd() + oprot.writeFieldStop() + oprot.writeStructEnd() + + def validate(self): + return + + def __repr__(self): + L = ["{}={!r}".format(key, value) for key, value in self.__dict__.items()] + return "{}({})".format(self.__class__.__name__, ", ".join(L)) + + def __eq__(self, other): + return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ + + def __ne__(self, other): + return not (self == other) + + +all_structs.append(get_databases_result) +get_databases_result.thrift_spec = ( + ( + 0, + TType.LIST, + "success", + (TType.STRING, "UTF8", False), + None, + ), # 0 + ( + 1, + TType.STRUCT, + "o1", + [MetaException, None], + None, + ), # 1 +) + + +class get_all_databases_args: + def read(self, iprot): + if ( + iprot._fast_decode is not None + and isinstance(iprot.trans, TTransport.CReadableTransport) + and self.thrift_spec is not None + ): + iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec]) + return + iprot.readStructBegin() + while True: + (fname, ftype, fid) = iprot.readFieldBegin() + if ftype == TType.STOP: + break + else: + iprot.skip(ftype) + iprot.readFieldEnd() + iprot.readStructEnd() + + def write(self, oprot): + if oprot._fast_encode is not None and self.thrift_spec is not None: + oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec])) + return + oprot.writeStructBegin("get_all_databases_args") + oprot.writeFieldStop() + oprot.writeStructEnd() + + def validate(self): + return + + def __repr__(self): + L = ["{}={!r}".format(key, value) for key, value in self.__dict__.items()] + return "{}({})".format(self.__class__.__name__, ", ".join(L)) + + def __eq__(self, other): + return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ + + def __ne__(self, other): + return not (self == other) + + +all_structs.append(get_all_databases_args) +get_all_databases_args.thrift_spec = () + + +class get_all_databases_result: + """ + Attributes: + - success + - o1 + + """ + + def __init__( + self, + success=None, + o1=None, + ): + self.success = success + self.o1 = o1 + + def read(self, iprot): + if ( + iprot._fast_decode is not None + and isinstance(iprot.trans, TTransport.CReadableTransport) + and self.thrift_spec is not None + ): + iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec]) + return + iprot.readStructBegin() + while True: + (fname, ftype, fid) = iprot.readFieldBegin() + if ftype == TType.STOP: + break + if fid == 0: + if ftype == TType.LIST: + self.success = [] + (_etype1259, _size1256) = iprot.readListBegin() + for _i1260 in range(_size1256): + _elem1261 = ( + iprot.readString().decode("utf-8", errors="replace") + if sys.version_info[0] == 2 + else iprot.readString() + ) + self.success.append(_elem1261) + iprot.readListEnd() + else: + iprot.skip(ftype) + elif fid == 1: + if ftype == TType.STRUCT: + self.o1 = MetaException.read(iprot) + else: + iprot.skip(ftype) + else: + iprot.skip(ftype) + iprot.readFieldEnd() + iprot.readStructEnd() + + def write(self, oprot): + if oprot._fast_encode is not None and self.thrift_spec is not None: + oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec])) + return + oprot.writeStructBegin("get_all_databases_result") + if self.success is not None: + oprot.writeFieldBegin("success", TType.LIST, 0) + oprot.writeListBegin(TType.STRING, len(self.success)) + for iter1262 in self.success: + oprot.writeString(iter1262.encode("utf-8") if sys.version_info[0] == 2 else iter1262) + oprot.writeListEnd() + oprot.writeFieldEnd() + if self.o1 is not None: + oprot.writeFieldBegin("o1", TType.STRUCT, 1) + self.o1.write(oprot) + oprot.writeFieldEnd() + oprot.writeFieldStop() + oprot.writeStructEnd() + + def validate(self): + return + + def __repr__(self): + L = ["{}={!r}".format(key, value) for key, value in self.__dict__.items()] + return "{}({})".format(self.__class__.__name__, ", ".join(L)) + + def __eq__(self, other): + return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ + + def __ne__(self, other): + return not (self == other) + + +all_structs.append(get_all_databases_result) +get_all_databases_result.thrift_spec = ( + ( + 0, + TType.LIST, + "success", + (TType.STRING, "UTF8", False), + None, + ), # 0 + ( + 1, + TType.STRUCT, + "o1", + [MetaException, None], + None, + ), # 1 +) + + +class alter_database_args: + """ + Attributes: + - dbname + - db + + """ + + def __init__( + self, + dbname=None, + db=None, + ): + self.dbname = dbname + self.db = db + + def read(self, iprot): + if ( + iprot._fast_decode is not None + and isinstance(iprot.trans, TTransport.CReadableTransport) + and self.thrift_spec is not None + ): + iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec]) + return + iprot.readStructBegin() + while True: + (fname, ftype, fid) = iprot.readFieldBegin() + if ftype == TType.STOP: + break + if fid == 1: + if ftype == TType.STRING: + self.dbname = ( + iprot.readString().decode("utf-8", errors="replace") if sys.version_info[0] == 2 else iprot.readString() + ) + else: + iprot.skip(ftype) + elif fid == 2: + if ftype == TType.STRUCT: + self.db = Database() + self.db.read(iprot) + else: + iprot.skip(ftype) + else: + iprot.skip(ftype) + iprot.readFieldEnd() + iprot.readStructEnd() + + def write(self, oprot): + if oprot._fast_encode is not None and self.thrift_spec is not None: + oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec])) + return + oprot.writeStructBegin("alter_database_args") + if self.dbname is not None: + oprot.writeFieldBegin("dbname", TType.STRING, 1) + oprot.writeString(self.dbname.encode("utf-8") if sys.version_info[0] == 2 else self.dbname) + oprot.writeFieldEnd() + if self.db is not None: + oprot.writeFieldBegin("db", TType.STRUCT, 2) + self.db.write(oprot) + oprot.writeFieldEnd() + oprot.writeFieldStop() + oprot.writeStructEnd() + + def validate(self): + return + + def __repr__(self): + L = ["{}={!r}".format(key, value) for key, value in self.__dict__.items()] + return "{}({})".format(self.__class__.__name__, ", ".join(L)) + + def __eq__(self, other): + return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ + + def __ne__(self, other): + return not (self == other) + + +all_structs.append(alter_database_args) +alter_database_args.thrift_spec = ( + None, # 0 + ( + 1, + TType.STRING, + "dbname", + "UTF8", + None, + ), # 1 + ( + 2, + TType.STRUCT, + "db", + [Database, None], + None, + ), # 2 +) + + +class alter_database_result: + """ + Attributes: + - o1 + - o2 + + """ + + def __init__( + self, + o1=None, + o2=None, + ): + self.o1 = o1 + self.o2 = o2 + + def read(self, iprot): + if ( + iprot._fast_decode is not None + and isinstance(iprot.trans, TTransport.CReadableTransport) + and self.thrift_spec is not None + ): + iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec]) + return + iprot.readStructBegin() + while True: + (fname, ftype, fid) = iprot.readFieldBegin() + if ftype == TType.STOP: + break + if fid == 1: + if ftype == TType.STRUCT: + self.o1 = MetaException.read(iprot) + else: + iprot.skip(ftype) + elif fid == 2: + if ftype == TType.STRUCT: + self.o2 = NoSuchObjectException.read(iprot) + else: + iprot.skip(ftype) + else: + iprot.skip(ftype) + iprot.readFieldEnd() + iprot.readStructEnd() + + def write(self, oprot): + if oprot._fast_encode is not None and self.thrift_spec is not None: + oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec])) + return + oprot.writeStructBegin("alter_database_result") + if self.o1 is not None: + oprot.writeFieldBegin("o1", TType.STRUCT, 1) + self.o1.write(oprot) + oprot.writeFieldEnd() + if self.o2 is not None: + oprot.writeFieldBegin("o2", TType.STRUCT, 2) + self.o2.write(oprot) + oprot.writeFieldEnd() + oprot.writeFieldStop() + oprot.writeStructEnd() + + def validate(self): + return + + def __repr__(self): + L = ["{}={!r}".format(key, value) for key, value in self.__dict__.items()] + return "{}({})".format(self.__class__.__name__, ", ".join(L)) + + def __eq__(self, other): + return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ + + def __ne__(self, other): + return not (self == other) + + +all_structs.append(alter_database_result) +alter_database_result.thrift_spec = ( + None, # 0 + ( + 1, + TType.STRUCT, + "o1", + [MetaException, None], + None, + ), # 1 + ( + 2, + TType.STRUCT, + "o2", + [NoSuchObjectException, None], + None, + ), # 2 +) + + +class create_dataconnector_args: + """ + Attributes: + - connector + + """ + + def __init__( + self, + connector=None, + ): + self.connector = connector + + def read(self, iprot): + if ( + iprot._fast_decode is not None + and isinstance(iprot.trans, TTransport.CReadableTransport) + and self.thrift_spec is not None + ): + iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec]) + return + iprot.readStructBegin() + while True: + (fname, ftype, fid) = iprot.readFieldBegin() + if ftype == TType.STOP: + break + if fid == 1: + if ftype == TType.STRUCT: + self.connector = DataConnector() + self.connector.read(iprot) + else: + iprot.skip(ftype) + else: + iprot.skip(ftype) + iprot.readFieldEnd() + iprot.readStructEnd() + + def write(self, oprot): + if oprot._fast_encode is not None and self.thrift_spec is not None: + oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec])) + return + oprot.writeStructBegin("create_dataconnector_args") + if self.connector is not None: + oprot.writeFieldBegin("connector", TType.STRUCT, 1) + self.connector.write(oprot) + oprot.writeFieldEnd() + oprot.writeFieldStop() + oprot.writeStructEnd() + + def validate(self): + return + + def __repr__(self): + L = ["{}={!r}".format(key, value) for key, value in self.__dict__.items()] + return "{}({})".format(self.__class__.__name__, ", ".join(L)) + + def __eq__(self, other): + return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ + + def __ne__(self, other): + return not (self == other) + + +all_structs.append(create_dataconnector_args) +create_dataconnector_args.thrift_spec = ( + None, # 0 + ( + 1, + TType.STRUCT, + "connector", + [DataConnector, None], + None, + ), # 1 +) + + +class create_dataconnector_result: + """ + Attributes: + - o1 + - o2 + - o3 + + """ + + def __init__( + self, + o1=None, + o2=None, + o3=None, + ): + self.o1 = o1 + self.o2 = o2 + self.o3 = o3 + + def read(self, iprot): + if ( + iprot._fast_decode is not None + and isinstance(iprot.trans, TTransport.CReadableTransport) + and self.thrift_spec is not None + ): + iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec]) + return + iprot.readStructBegin() + while True: + (fname, ftype, fid) = iprot.readFieldBegin() + if ftype == TType.STOP: + break + if fid == 1: + if ftype == TType.STRUCT: + self.o1 = AlreadyExistsException.read(iprot) + else: + iprot.skip(ftype) + elif fid == 2: + if ftype == TType.STRUCT: + self.o2 = InvalidObjectException.read(iprot) + else: + iprot.skip(ftype) + elif fid == 3: + if ftype == TType.STRUCT: + self.o3 = MetaException.read(iprot) + else: + iprot.skip(ftype) + else: + iprot.skip(ftype) + iprot.readFieldEnd() + iprot.readStructEnd() + + def write(self, oprot): + if oprot._fast_encode is not None and self.thrift_spec is not None: + oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec])) + return + oprot.writeStructBegin("create_dataconnector_result") + if self.o1 is not None: + oprot.writeFieldBegin("o1", TType.STRUCT, 1) + self.o1.write(oprot) + oprot.writeFieldEnd() + if self.o2 is not None: + oprot.writeFieldBegin("o2", TType.STRUCT, 2) + self.o2.write(oprot) + oprot.writeFieldEnd() + if self.o3 is not None: + oprot.writeFieldBegin("o3", TType.STRUCT, 3) + self.o3.write(oprot) + oprot.writeFieldEnd() + oprot.writeFieldStop() + oprot.writeStructEnd() + + def validate(self): + return + + def __repr__(self): + L = ["{}={!r}".format(key, value) for key, value in self.__dict__.items()] + return "{}({})".format(self.__class__.__name__, ", ".join(L)) + + def __eq__(self, other): + return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ + + def __ne__(self, other): + return not (self == other) + + +all_structs.append(create_dataconnector_result) +create_dataconnector_result.thrift_spec = ( + None, # 0 + ( + 1, + TType.STRUCT, + "o1", + [AlreadyExistsException, None], + None, + ), # 1 + ( + 2, + TType.STRUCT, + "o2", + [InvalidObjectException, None], + None, + ), # 2 + ( + 3, + TType.STRUCT, + "o3", + [MetaException, None], + None, + ), # 3 +) + + +class get_dataconnector_req_args: + """ + Attributes: + - request + + """ + + def __init__( + self, + request=None, + ): + self.request = request + + def read(self, iprot): + if ( + iprot._fast_decode is not None + and isinstance(iprot.trans, TTransport.CReadableTransport) + and self.thrift_spec is not None + ): + iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec]) + return + iprot.readStructBegin() + while True: + (fname, ftype, fid) = iprot.readFieldBegin() + if ftype == TType.STOP: + break + if fid == 1: + if ftype == TType.STRUCT: + self.request = GetDataConnectorRequest() + self.request.read(iprot) + else: + iprot.skip(ftype) + else: + iprot.skip(ftype) + iprot.readFieldEnd() + iprot.readStructEnd() + + def write(self, oprot): + if oprot._fast_encode is not None and self.thrift_spec is not None: + oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec])) + return + oprot.writeStructBegin("get_dataconnector_req_args") + if self.request is not None: + oprot.writeFieldBegin("request", TType.STRUCT, 1) + self.request.write(oprot) + oprot.writeFieldEnd() + oprot.writeFieldStop() + oprot.writeStructEnd() + + def validate(self): + return + + def __repr__(self): + L = ["{}={!r}".format(key, value) for key, value in self.__dict__.items()] + return "{}({})".format(self.__class__.__name__, ", ".join(L)) + + def __eq__(self, other): + return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ + + def __ne__(self, other): + return not (self == other) + + +all_structs.append(get_dataconnector_req_args) +get_dataconnector_req_args.thrift_spec = ( + None, # 0 + ( + 1, + TType.STRUCT, + "request", + [GetDataConnectorRequest, None], + None, + ), # 1 +) + + +class get_dataconnector_req_result: + """ + Attributes: + - success + - o1 + - o2 + + """ + + def __init__( + self, + success=None, + o1=None, + o2=None, + ): + self.success = success + self.o1 = o1 + self.o2 = o2 + + def read(self, iprot): + if ( + iprot._fast_decode is not None + and isinstance(iprot.trans, TTransport.CReadableTransport) + and self.thrift_spec is not None + ): + iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec]) + return + iprot.readStructBegin() + while True: + (fname, ftype, fid) = iprot.readFieldBegin() + if ftype == TType.STOP: + break + if fid == 0: + if ftype == TType.STRUCT: + self.success = DataConnector() + self.success.read(iprot) + else: + iprot.skip(ftype) + elif fid == 1: + if ftype == TType.STRUCT: + self.o1 = NoSuchObjectException.read(iprot) + else: + iprot.skip(ftype) + elif fid == 2: + if ftype == TType.STRUCT: + self.o2 = MetaException.read(iprot) + else: + iprot.skip(ftype) + else: + iprot.skip(ftype) + iprot.readFieldEnd() + iprot.readStructEnd() + + def write(self, oprot): + if oprot._fast_encode is not None and self.thrift_spec is not None: + oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec])) + return + oprot.writeStructBegin("get_dataconnector_req_result") + if self.success is not None: + oprot.writeFieldBegin("success", TType.STRUCT, 0) + self.success.write(oprot) + oprot.writeFieldEnd() + if self.o1 is not None: + oprot.writeFieldBegin("o1", TType.STRUCT, 1) + self.o1.write(oprot) + oprot.writeFieldEnd() + if self.o2 is not None: + oprot.writeFieldBegin("o2", TType.STRUCT, 2) + self.o2.write(oprot) + oprot.writeFieldEnd() + oprot.writeFieldStop() + oprot.writeStructEnd() + + def validate(self): + return + + def __repr__(self): + L = ["{}={!r}".format(key, value) for key, value in self.__dict__.items()] + return "{}({})".format(self.__class__.__name__, ", ".join(L)) + + def __eq__(self, other): + return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ + + def __ne__(self, other): + return not (self == other) + + +all_structs.append(get_dataconnector_req_result) +get_dataconnector_req_result.thrift_spec = ( + ( + 0, + TType.STRUCT, + "success", + [DataConnector, None], + None, + ), # 0 + ( + 1, + TType.STRUCT, + "o1", + [NoSuchObjectException, None], + None, + ), # 1 + ( + 2, + TType.STRUCT, + "o2", + [MetaException, None], + None, + ), # 2 +) + + +class drop_dataconnector_args: + """ + Attributes: + - name + - ifNotExists + - checkReferences + + """ + + def __init__( + self, + name=None, + ifNotExists=None, + checkReferences=None, + ): + self.name = name + self.ifNotExists = ifNotExists + self.checkReferences = checkReferences + + def read(self, iprot): + if ( + iprot._fast_decode is not None + and isinstance(iprot.trans, TTransport.CReadableTransport) + and self.thrift_spec is not None + ): + iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec]) + return + iprot.readStructBegin() + while True: + (fname, ftype, fid) = iprot.readFieldBegin() + if ftype == TType.STOP: + break + if fid == 1: + if ftype == TType.STRING: + self.name = ( + iprot.readString().decode("utf-8", errors="replace") if sys.version_info[0] == 2 else iprot.readString() + ) + else: + iprot.skip(ftype) + elif fid == -1: + if ftype == TType.BOOL: + self.ifNotExists = iprot.readBool() + else: + iprot.skip(ftype) + elif fid == -2: + if ftype == TType.BOOL: + self.checkReferences = iprot.readBool() + else: + iprot.skip(ftype) + else: + iprot.skip(ftype) + iprot.readFieldEnd() + iprot.readStructEnd() + + def write(self, oprot): + if oprot._fast_encode is not None and self.thrift_spec is not None: + oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec])) + return + oprot.writeStructBegin("drop_dataconnector_args") + if self.checkReferences is not None: + oprot.writeFieldBegin("checkReferences", TType.BOOL, -2) + oprot.writeBool(self.checkReferences) + oprot.writeFieldEnd() + if self.ifNotExists is not None: + oprot.writeFieldBegin("ifNotExists", TType.BOOL, -1) + oprot.writeBool(self.ifNotExists) + oprot.writeFieldEnd() + if self.name is not None: + oprot.writeFieldBegin("name", TType.STRING, 1) + oprot.writeString(self.name.encode("utf-8") if sys.version_info[0] == 2 else self.name) + oprot.writeFieldEnd() + oprot.writeFieldStop() + oprot.writeStructEnd() + + def validate(self): + return + + def __repr__(self): + L = ["{}={!r}".format(key, value) for key, value in self.__dict__.items()] + return "{}({})".format(self.__class__.__name__, ", ".join(L)) + + def __eq__(self, other): + return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ + + def __ne__(self, other): + return not (self == other) + + +all_structs.append(drop_dataconnector_args) +drop_dataconnector_args.thrift_spec = () + + +class drop_dataconnector_result: + """ + Attributes: + - o1 + - o2 + - o3 + + """ + + def __init__( + self, + o1=None, + o2=None, + o3=None, + ): + self.o1 = o1 + self.o2 = o2 + self.o3 = o3 + + def read(self, iprot): + if ( + iprot._fast_decode is not None + and isinstance(iprot.trans, TTransport.CReadableTransport) + and self.thrift_spec is not None + ): + iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec]) + return + iprot.readStructBegin() + while True: + (fname, ftype, fid) = iprot.readFieldBegin() + if ftype == TType.STOP: + break + if fid == 1: + if ftype == TType.STRUCT: + self.o1 = NoSuchObjectException.read(iprot) + else: + iprot.skip(ftype) + elif fid == 2: + if ftype == TType.STRUCT: + self.o2 = InvalidOperationException.read(iprot) + else: + iprot.skip(ftype) + elif fid == 3: + if ftype == TType.STRUCT: + self.o3 = MetaException.read(iprot) + else: + iprot.skip(ftype) + else: + iprot.skip(ftype) + iprot.readFieldEnd() + iprot.readStructEnd() + + def write(self, oprot): + if oprot._fast_encode is not None and self.thrift_spec is not None: + oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec])) + return + oprot.writeStructBegin("drop_dataconnector_result") + if self.o1 is not None: + oprot.writeFieldBegin("o1", TType.STRUCT, 1) + self.o1.write(oprot) + oprot.writeFieldEnd() + if self.o2 is not None: + oprot.writeFieldBegin("o2", TType.STRUCT, 2) + self.o2.write(oprot) + oprot.writeFieldEnd() + if self.o3 is not None: + oprot.writeFieldBegin("o3", TType.STRUCT, 3) + self.o3.write(oprot) + oprot.writeFieldEnd() + oprot.writeFieldStop() + oprot.writeStructEnd() + + def validate(self): + return + + def __repr__(self): + L = ["{}={!r}".format(key, value) for key, value in self.__dict__.items()] + return "{}({})".format(self.__class__.__name__, ", ".join(L)) + + def __eq__(self, other): + return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ + + def __ne__(self, other): + return not (self == other) + + +all_structs.append(drop_dataconnector_result) +drop_dataconnector_result.thrift_spec = ( + None, # 0 + ( + 1, + TType.STRUCT, + "o1", + [NoSuchObjectException, None], + None, + ), # 1 + ( + 2, + TType.STRUCT, + "o2", + [InvalidOperationException, None], + None, + ), # 2 + ( + 3, + TType.STRUCT, + "o3", + [MetaException, None], + None, + ), # 3 +) + + +class get_dataconnectors_args: + def read(self, iprot): + if ( + iprot._fast_decode is not None + and isinstance(iprot.trans, TTransport.CReadableTransport) + and self.thrift_spec is not None + ): + iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec]) + return + iprot.readStructBegin() + while True: + (fname, ftype, fid) = iprot.readFieldBegin() + if ftype == TType.STOP: + break + else: + iprot.skip(ftype) + iprot.readFieldEnd() + iprot.readStructEnd() + + def write(self, oprot): + if oprot._fast_encode is not None and self.thrift_spec is not None: + oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec])) + return + oprot.writeStructBegin("get_dataconnectors_args") + oprot.writeFieldStop() + oprot.writeStructEnd() + + def validate(self): + return + + def __repr__(self): + L = ["{}={!r}".format(key, value) for key, value in self.__dict__.items()] + return "{}({})".format(self.__class__.__name__, ", ".join(L)) + + def __eq__(self, other): + return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ + + def __ne__(self, other): + return not (self == other) + + +all_structs.append(get_dataconnectors_args) +get_dataconnectors_args.thrift_spec = () + + +class get_dataconnectors_result: + """ + Attributes: + - success + - o1 + + """ + + def __init__( + self, + success=None, + o1=None, + ): + self.success = success + self.o1 = o1 + + def read(self, iprot): + if ( + iprot._fast_decode is not None + and isinstance(iprot.trans, TTransport.CReadableTransport) + and self.thrift_spec is not None + ): + iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec]) + return + iprot.readStructBegin() + while True: + (fname, ftype, fid) = iprot.readFieldBegin() + if ftype == TType.STOP: + break + if fid == 0: + if ftype == TType.LIST: + self.success = [] + (_etype1266, _size1263) = iprot.readListBegin() + for _i1267 in range(_size1263): + _elem1268 = ( + iprot.readString().decode("utf-8", errors="replace") + if sys.version_info[0] == 2 + else iprot.readString() + ) + self.success.append(_elem1268) + iprot.readListEnd() + else: + iprot.skip(ftype) + elif fid == 1: + if ftype == TType.STRUCT: + self.o1 = MetaException.read(iprot) + else: + iprot.skip(ftype) + else: + iprot.skip(ftype) + iprot.readFieldEnd() + iprot.readStructEnd() + + def write(self, oprot): + if oprot._fast_encode is not None and self.thrift_spec is not None: + oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec])) + return + oprot.writeStructBegin("get_dataconnectors_result") + if self.success is not None: + oprot.writeFieldBegin("success", TType.LIST, 0) + oprot.writeListBegin(TType.STRING, len(self.success)) + for iter1269 in self.success: + oprot.writeString(iter1269.encode("utf-8") if sys.version_info[0] == 2 else iter1269) + oprot.writeListEnd() + oprot.writeFieldEnd() + if self.o1 is not None: + oprot.writeFieldBegin("o1", TType.STRUCT, 1) + self.o1.write(oprot) + oprot.writeFieldEnd() + oprot.writeFieldStop() + oprot.writeStructEnd() + + def validate(self): + return + + def __repr__(self): + L = ["{}={!r}".format(key, value) for key, value in self.__dict__.items()] + return "{}({})".format(self.__class__.__name__, ", ".join(L)) + + def __eq__(self, other): + return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ + + def __ne__(self, other): + return not (self == other) + + +all_structs.append(get_dataconnectors_result) +get_dataconnectors_result.thrift_spec = ( + ( + 0, + TType.LIST, + "success", + (TType.STRING, "UTF8", False), + None, + ), # 0 + ( + 1, + TType.STRUCT, + "o1", + [MetaException, None], + None, + ), # 1 +) + + +class alter_dataconnector_args: + """ + Attributes: + - name + - connector + + """ + + def __init__( + self, + name=None, + connector=None, + ): + self.name = name + self.connector = connector + + def read(self, iprot): + if ( + iprot._fast_decode is not None + and isinstance(iprot.trans, TTransport.CReadableTransport) + and self.thrift_spec is not None + ): + iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec]) + return + iprot.readStructBegin() + while True: + (fname, ftype, fid) = iprot.readFieldBegin() + if ftype == TType.STOP: + break + if fid == 1: + if ftype == TType.STRING: + self.name = ( + iprot.readString().decode("utf-8", errors="replace") if sys.version_info[0] == 2 else iprot.readString() + ) + else: + iprot.skip(ftype) + elif fid == 2: + if ftype == TType.STRUCT: + self.connector = DataConnector() + self.connector.read(iprot) + else: + iprot.skip(ftype) + else: + iprot.skip(ftype) + iprot.readFieldEnd() + iprot.readStructEnd() + + def write(self, oprot): + if oprot._fast_encode is not None and self.thrift_spec is not None: + oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec])) + return + oprot.writeStructBegin("alter_dataconnector_args") + if self.name is not None: + oprot.writeFieldBegin("name", TType.STRING, 1) + oprot.writeString(self.name.encode("utf-8") if sys.version_info[0] == 2 else self.name) + oprot.writeFieldEnd() + if self.connector is not None: + oprot.writeFieldBegin("connector", TType.STRUCT, 2) + self.connector.write(oprot) + oprot.writeFieldEnd() + oprot.writeFieldStop() + oprot.writeStructEnd() + + def validate(self): + return + + def __repr__(self): + L = ["{}={!r}".format(key, value) for key, value in self.__dict__.items()] + return "{}({})".format(self.__class__.__name__, ", ".join(L)) + + def __eq__(self, other): + return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ + + def __ne__(self, other): + return not (self == other) + + +all_structs.append(alter_dataconnector_args) +alter_dataconnector_args.thrift_spec = ( + None, # 0 + ( + 1, + TType.STRING, + "name", + "UTF8", + None, + ), # 1 + ( + 2, + TType.STRUCT, + "connector", + [DataConnector, None], + None, + ), # 2 +) + + +class alter_dataconnector_result: + """ + Attributes: + - o1 + - o2 + + """ + + def __init__( + self, + o1=None, + o2=None, + ): + self.o1 = o1 + self.o2 = o2 + + def read(self, iprot): + if ( + iprot._fast_decode is not None + and isinstance(iprot.trans, TTransport.CReadableTransport) + and self.thrift_spec is not None + ): + iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec]) + return + iprot.readStructBegin() + while True: + (fname, ftype, fid) = iprot.readFieldBegin() + if ftype == TType.STOP: + break + if fid == 1: + if ftype == TType.STRUCT: + self.o1 = MetaException.read(iprot) + else: + iprot.skip(ftype) + elif fid == 2: + if ftype == TType.STRUCT: + self.o2 = NoSuchObjectException.read(iprot) + else: + iprot.skip(ftype) + else: + iprot.skip(ftype) + iprot.readFieldEnd() + iprot.readStructEnd() + + def write(self, oprot): + if oprot._fast_encode is not None and self.thrift_spec is not None: + oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec])) + return + oprot.writeStructBegin("alter_dataconnector_result") + if self.o1 is not None: + oprot.writeFieldBegin("o1", TType.STRUCT, 1) + self.o1.write(oprot) + oprot.writeFieldEnd() + if self.o2 is not None: + oprot.writeFieldBegin("o2", TType.STRUCT, 2) + self.o2.write(oprot) + oprot.writeFieldEnd() + oprot.writeFieldStop() + oprot.writeStructEnd() + + def validate(self): + return + + def __repr__(self): + L = ["{}={!r}".format(key, value) for key, value in self.__dict__.items()] + return "{}({})".format(self.__class__.__name__, ", ".join(L)) + + def __eq__(self, other): + return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ + + def __ne__(self, other): + return not (self == other) + + +all_structs.append(alter_dataconnector_result) +alter_dataconnector_result.thrift_spec = ( + None, # 0 + ( + 1, + TType.STRUCT, + "o1", + [MetaException, None], + None, + ), # 1 + ( + 2, + TType.STRUCT, + "o2", + [NoSuchObjectException, None], + None, + ), # 2 +) + + +class get_type_args: + """ + Attributes: + - name + + """ + + def __init__( + self, + name=None, + ): + self.name = name + + def read(self, iprot): + if ( + iprot._fast_decode is not None + and isinstance(iprot.trans, TTransport.CReadableTransport) + and self.thrift_spec is not None + ): + iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec]) + return + iprot.readStructBegin() + while True: + (fname, ftype, fid) = iprot.readFieldBegin() + if ftype == TType.STOP: + break + if fid == 1: + if ftype == TType.STRING: + self.name = ( + iprot.readString().decode("utf-8", errors="replace") if sys.version_info[0] == 2 else iprot.readString() + ) + else: + iprot.skip(ftype) + else: + iprot.skip(ftype) + iprot.readFieldEnd() + iprot.readStructEnd() + + def write(self, oprot): + if oprot._fast_encode is not None and self.thrift_spec is not None: + oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec])) + return + oprot.writeStructBegin("get_type_args") + if self.name is not None: + oprot.writeFieldBegin("name", TType.STRING, 1) + oprot.writeString(self.name.encode("utf-8") if sys.version_info[0] == 2 else self.name) + oprot.writeFieldEnd() + oprot.writeFieldStop() + oprot.writeStructEnd() + + def validate(self): + return + + def __repr__(self): + L = ["{}={!r}".format(key, value) for key, value in self.__dict__.items()] + return "{}({})".format(self.__class__.__name__, ", ".join(L)) + + def __eq__(self, other): + return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ + + def __ne__(self, other): + return not (self == other) + + +all_structs.append(get_type_args) +get_type_args.thrift_spec = ( + None, # 0 + ( + 1, + TType.STRING, + "name", + "UTF8", + None, + ), # 1 +) + + +class get_type_result: + """ + Attributes: + - success + - o1 + - o2 + + """ + + def __init__( + self, + success=None, + o1=None, + o2=None, + ): + self.success = success + self.o1 = o1 + self.o2 = o2 + + def read(self, iprot): + if ( + iprot._fast_decode is not None + and isinstance(iprot.trans, TTransport.CReadableTransport) + and self.thrift_spec is not None + ): + iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec]) + return + iprot.readStructBegin() + while True: + (fname, ftype, fid) = iprot.readFieldBegin() + if ftype == TType.STOP: + break + if fid == 0: + if ftype == TType.STRUCT: + self.success = Type() + self.success.read(iprot) + else: + iprot.skip(ftype) + elif fid == 1: + if ftype == TType.STRUCT: + self.o1 = MetaException.read(iprot) + else: + iprot.skip(ftype) + elif fid == 2: + if ftype == TType.STRUCT: + self.o2 = NoSuchObjectException.read(iprot) + else: + iprot.skip(ftype) + else: + iprot.skip(ftype) + iprot.readFieldEnd() + iprot.readStructEnd() + + def write(self, oprot): + if oprot._fast_encode is not None and self.thrift_spec is not None: + oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec])) + return + oprot.writeStructBegin("get_type_result") + if self.success is not None: + oprot.writeFieldBegin("success", TType.STRUCT, 0) + self.success.write(oprot) + oprot.writeFieldEnd() + if self.o1 is not None: + oprot.writeFieldBegin("o1", TType.STRUCT, 1) + self.o1.write(oprot) + oprot.writeFieldEnd() + if self.o2 is not None: + oprot.writeFieldBegin("o2", TType.STRUCT, 2) + self.o2.write(oprot) + oprot.writeFieldEnd() + oprot.writeFieldStop() + oprot.writeStructEnd() + + def validate(self): + return + + def __repr__(self): + L = ["{}={!r}".format(key, value) for key, value in self.__dict__.items()] + return "{}({})".format(self.__class__.__name__, ", ".join(L)) + + def __eq__(self, other): + return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ + + def __ne__(self, other): + return not (self == other) + + +all_structs.append(get_type_result) +get_type_result.thrift_spec = ( + ( + 0, + TType.STRUCT, + "success", + [Type, None], + None, + ), # 0 + ( + 1, + TType.STRUCT, + "o1", + [MetaException, None], + None, + ), # 1 + ( + 2, + TType.STRUCT, + "o2", + [NoSuchObjectException, None], + None, + ), # 2 +) + + +class create_type_args: + """ + Attributes: + - type + + """ + + def __init__( + self, + type=None, + ): + self.type = type + + def read(self, iprot): + if ( + iprot._fast_decode is not None + and isinstance(iprot.trans, TTransport.CReadableTransport) + and self.thrift_spec is not None + ): + iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec]) + return + iprot.readStructBegin() + while True: + (fname, ftype, fid) = iprot.readFieldBegin() + if ftype == TType.STOP: + break + if fid == 1: + if ftype == TType.STRUCT: + self.type = Type() + self.type.read(iprot) + else: + iprot.skip(ftype) + else: + iprot.skip(ftype) + iprot.readFieldEnd() + iprot.readStructEnd() + + def write(self, oprot): + if oprot._fast_encode is not None and self.thrift_spec is not None: + oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec])) + return + oprot.writeStructBegin("create_type_args") + if self.type is not None: + oprot.writeFieldBegin("type", TType.STRUCT, 1) + self.type.write(oprot) + oprot.writeFieldEnd() + oprot.writeFieldStop() + oprot.writeStructEnd() + + def validate(self): + return + + def __repr__(self): + L = ["{}={!r}".format(key, value) for key, value in self.__dict__.items()] + return "{}({})".format(self.__class__.__name__, ", ".join(L)) + + def __eq__(self, other): + return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ + + def __ne__(self, other): + return not (self == other) + + +all_structs.append(create_type_args) +create_type_args.thrift_spec = ( + None, # 0 + ( + 1, + TType.STRUCT, + "type", + [Type, None], + None, + ), # 1 +) + + +class create_type_result: + """ + Attributes: + - success + - o1 + - o2 + - o3 + + """ + + def __init__( + self, + success=None, + o1=None, + o2=None, + o3=None, + ): + self.success = success + self.o1 = o1 + self.o2 = o2 + self.o3 = o3 + + def read(self, iprot): + if ( + iprot._fast_decode is not None + and isinstance(iprot.trans, TTransport.CReadableTransport) + and self.thrift_spec is not None + ): + iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec]) + return + iprot.readStructBegin() + while True: + (fname, ftype, fid) = iprot.readFieldBegin() + if ftype == TType.STOP: + break + if fid == 0: + if ftype == TType.BOOL: + self.success = iprot.readBool() + else: + iprot.skip(ftype) + elif fid == 1: + if ftype == TType.STRUCT: + self.o1 = AlreadyExistsException.read(iprot) + else: + iprot.skip(ftype) + elif fid == 2: + if ftype == TType.STRUCT: + self.o2 = InvalidObjectException.read(iprot) + else: + iprot.skip(ftype) + elif fid == 3: + if ftype == TType.STRUCT: + self.o3 = MetaException.read(iprot) + else: + iprot.skip(ftype) + else: + iprot.skip(ftype) + iprot.readFieldEnd() + iprot.readStructEnd() + + def write(self, oprot): + if oprot._fast_encode is not None and self.thrift_spec is not None: + oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec])) + return + oprot.writeStructBegin("create_type_result") + if self.success is not None: + oprot.writeFieldBegin("success", TType.BOOL, 0) + oprot.writeBool(self.success) + oprot.writeFieldEnd() + if self.o1 is not None: + oprot.writeFieldBegin("o1", TType.STRUCT, 1) + self.o1.write(oprot) + oprot.writeFieldEnd() + if self.o2 is not None: + oprot.writeFieldBegin("o2", TType.STRUCT, 2) + self.o2.write(oprot) + oprot.writeFieldEnd() + if self.o3 is not None: + oprot.writeFieldBegin("o3", TType.STRUCT, 3) + self.o3.write(oprot) + oprot.writeFieldEnd() + oprot.writeFieldStop() + oprot.writeStructEnd() + + def validate(self): + return + + def __repr__(self): + L = ["{}={!r}".format(key, value) for key, value in self.__dict__.items()] + return "{}({})".format(self.__class__.__name__, ", ".join(L)) + + def __eq__(self, other): + return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ + + def __ne__(self, other): + return not (self == other) + + +all_structs.append(create_type_result) +create_type_result.thrift_spec = ( + ( + 0, + TType.BOOL, + "success", + None, + None, + ), # 0 + ( + 1, + TType.STRUCT, + "o1", + [AlreadyExistsException, None], + None, + ), # 1 + ( + 2, + TType.STRUCT, + "o2", + [InvalidObjectException, None], + None, + ), # 2 + ( + 3, + TType.STRUCT, + "o3", + [MetaException, None], + None, + ), # 3 +) + + +class drop_type_args: + """ + Attributes: + - type + + """ + + def __init__( + self, + type=None, + ): + self.type = type + + def read(self, iprot): + if ( + iprot._fast_decode is not None + and isinstance(iprot.trans, TTransport.CReadableTransport) + and self.thrift_spec is not None + ): + iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec]) + return + iprot.readStructBegin() + while True: + (fname, ftype, fid) = iprot.readFieldBegin() + if ftype == TType.STOP: + break + if fid == 1: + if ftype == TType.STRING: + self.type = ( + iprot.readString().decode("utf-8", errors="replace") if sys.version_info[0] == 2 else iprot.readString() + ) + else: + iprot.skip(ftype) + else: + iprot.skip(ftype) + iprot.readFieldEnd() + iprot.readStructEnd() + + def write(self, oprot): + if oprot._fast_encode is not None and self.thrift_spec is not None: + oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec])) + return + oprot.writeStructBegin("drop_type_args") + if self.type is not None: + oprot.writeFieldBegin("type", TType.STRING, 1) + oprot.writeString(self.type.encode("utf-8") if sys.version_info[0] == 2 else self.type) + oprot.writeFieldEnd() + oprot.writeFieldStop() + oprot.writeStructEnd() + + def validate(self): + return + + def __repr__(self): + L = ["{}={!r}".format(key, value) for key, value in self.__dict__.items()] + return "{}({})".format(self.__class__.__name__, ", ".join(L)) + + def __eq__(self, other): + return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ + + def __ne__(self, other): + return not (self == other) + + +all_structs.append(drop_type_args) +drop_type_args.thrift_spec = ( + None, # 0 + ( + 1, + TType.STRING, + "type", + "UTF8", + None, + ), # 1 +) + + +class drop_type_result: + """ + Attributes: + - success + - o1 + - o2 + + """ + + def __init__( + self, + success=None, + o1=None, + o2=None, + ): + self.success = success + self.o1 = o1 + self.o2 = o2 + + def read(self, iprot): + if ( + iprot._fast_decode is not None + and isinstance(iprot.trans, TTransport.CReadableTransport) + and self.thrift_spec is not None + ): + iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec]) + return + iprot.readStructBegin() + while True: + (fname, ftype, fid) = iprot.readFieldBegin() + if ftype == TType.STOP: + break + if fid == 0: + if ftype == TType.BOOL: + self.success = iprot.readBool() + else: + iprot.skip(ftype) + elif fid == 1: + if ftype == TType.STRUCT: + self.o1 = MetaException.read(iprot) + else: + iprot.skip(ftype) + elif fid == 2: + if ftype == TType.STRUCT: + self.o2 = NoSuchObjectException.read(iprot) + else: + iprot.skip(ftype) + else: + iprot.skip(ftype) + iprot.readFieldEnd() + iprot.readStructEnd() + + def write(self, oprot): + if oprot._fast_encode is not None and self.thrift_spec is not None: + oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec])) + return + oprot.writeStructBegin("drop_type_result") + if self.success is not None: + oprot.writeFieldBegin("success", TType.BOOL, 0) + oprot.writeBool(self.success) + oprot.writeFieldEnd() + if self.o1 is not None: + oprot.writeFieldBegin("o1", TType.STRUCT, 1) + self.o1.write(oprot) + oprot.writeFieldEnd() + if self.o2 is not None: + oprot.writeFieldBegin("o2", TType.STRUCT, 2) + self.o2.write(oprot) + oprot.writeFieldEnd() + oprot.writeFieldStop() + oprot.writeStructEnd() + + def validate(self): + return + + def __repr__(self): + L = ["{}={!r}".format(key, value) for key, value in self.__dict__.items()] + return "{}({})".format(self.__class__.__name__, ", ".join(L)) + + def __eq__(self, other): + return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ + + def __ne__(self, other): + return not (self == other) + + +all_structs.append(drop_type_result) +drop_type_result.thrift_spec = ( + ( + 0, + TType.BOOL, + "success", + None, + None, + ), # 0 + ( + 1, + TType.STRUCT, + "o1", + [MetaException, None], + None, + ), # 1 + ( + 2, + TType.STRUCT, + "o2", + [NoSuchObjectException, None], + None, + ), # 2 +) + + +class get_type_all_args: + """ + Attributes: + - name + + """ + + def __init__( + self, + name=None, + ): + self.name = name + + def read(self, iprot): + if ( + iprot._fast_decode is not None + and isinstance(iprot.trans, TTransport.CReadableTransport) + and self.thrift_spec is not None + ): + iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec]) + return + iprot.readStructBegin() + while True: + (fname, ftype, fid) = iprot.readFieldBegin() + if ftype == TType.STOP: + break + if fid == 1: + if ftype == TType.STRING: + self.name = ( + iprot.readString().decode("utf-8", errors="replace") if sys.version_info[0] == 2 else iprot.readString() + ) + else: + iprot.skip(ftype) + else: + iprot.skip(ftype) + iprot.readFieldEnd() + iprot.readStructEnd() + + def write(self, oprot): + if oprot._fast_encode is not None and self.thrift_spec is not None: + oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec])) + return + oprot.writeStructBegin("get_type_all_args") + if self.name is not None: + oprot.writeFieldBegin("name", TType.STRING, 1) + oprot.writeString(self.name.encode("utf-8") if sys.version_info[0] == 2 else self.name) + oprot.writeFieldEnd() + oprot.writeFieldStop() + oprot.writeStructEnd() + + def validate(self): + return + + def __repr__(self): + L = ["{}={!r}".format(key, value) for key, value in self.__dict__.items()] + return "{}({})".format(self.__class__.__name__, ", ".join(L)) + + def __eq__(self, other): + return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ + + def __ne__(self, other): + return not (self == other) + + +all_structs.append(get_type_all_args) +get_type_all_args.thrift_spec = ( + None, # 0 + ( + 1, + TType.STRING, + "name", + "UTF8", + None, + ), # 1 +) + + +class get_type_all_result: + """ + Attributes: + - success + - o2 + + """ + + def __init__( + self, + success=None, + o2=None, + ): + self.success = success + self.o2 = o2 + + def read(self, iprot): + if ( + iprot._fast_decode is not None + and isinstance(iprot.trans, TTransport.CReadableTransport) + and self.thrift_spec is not None + ): + iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec]) + return + iprot.readStructBegin() + while True: + (fname, ftype, fid) = iprot.readFieldBegin() + if ftype == TType.STOP: + break + if fid == 0: + if ftype == TType.MAP: + self.success = {} + (_ktype1271, _vtype1272, _size1270) = iprot.readMapBegin() + for _i1274 in range(_size1270): + _key1275 = ( + iprot.readString().decode("utf-8", errors="replace") + if sys.version_info[0] == 2 + else iprot.readString() + ) + _val1276 = Type() + _val1276.read(iprot) + self.success[_key1275] = _val1276 + iprot.readMapEnd() + else: + iprot.skip(ftype) + elif fid == 1: + if ftype == TType.STRUCT: + self.o2 = MetaException.read(iprot) + else: + iprot.skip(ftype) + else: + iprot.skip(ftype) + iprot.readFieldEnd() + iprot.readStructEnd() + + def write(self, oprot): + if oprot._fast_encode is not None and self.thrift_spec is not None: + oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec])) + return + oprot.writeStructBegin("get_type_all_result") + if self.success is not None: + oprot.writeFieldBegin("success", TType.MAP, 0) + oprot.writeMapBegin(TType.STRING, TType.STRUCT, len(self.success)) + for kiter1277, viter1278 in self.success.items(): + oprot.writeString(kiter1277.encode("utf-8") if sys.version_info[0] == 2 else kiter1277) + viter1278.write(oprot) + oprot.writeMapEnd() + oprot.writeFieldEnd() + if self.o2 is not None: + oprot.writeFieldBegin("o2", TType.STRUCT, 1) + self.o2.write(oprot) + oprot.writeFieldEnd() + oprot.writeFieldStop() + oprot.writeStructEnd() + + def validate(self): + return + + def __repr__(self): + L = ["{}={!r}".format(key, value) for key, value in self.__dict__.items()] + return "{}({})".format(self.__class__.__name__, ", ".join(L)) + + def __eq__(self, other): + return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ + + def __ne__(self, other): + return not (self == other) + + +all_structs.append(get_type_all_result) +get_type_all_result.thrift_spec = ( + ( + 0, + TType.MAP, + "success", + (TType.STRING, "UTF8", TType.STRUCT, [Type, None], False), + None, + ), # 0 + ( + 1, + TType.STRUCT, + "o2", + [MetaException, None], + None, + ), # 1 +) + + +class get_fields_args: + """ + Attributes: + - db_name + - table_name + + """ + + def __init__( + self, + db_name=None, + table_name=None, + ): + self.db_name = db_name + self.table_name = table_name + + def read(self, iprot): + if ( + iprot._fast_decode is not None + and isinstance(iprot.trans, TTransport.CReadableTransport) + and self.thrift_spec is not None + ): + iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec]) + return + iprot.readStructBegin() + while True: + (fname, ftype, fid) = iprot.readFieldBegin() + if ftype == TType.STOP: + break + if fid == 1: + if ftype == TType.STRING: + self.db_name = ( + iprot.readString().decode("utf-8", errors="replace") if sys.version_info[0] == 2 else iprot.readString() + ) + else: + iprot.skip(ftype) + elif fid == 2: + if ftype == TType.STRING: + self.table_name = ( + iprot.readString().decode("utf-8", errors="replace") if sys.version_info[0] == 2 else iprot.readString() + ) + else: + iprot.skip(ftype) + else: + iprot.skip(ftype) + iprot.readFieldEnd() + iprot.readStructEnd() + + def write(self, oprot): + if oprot._fast_encode is not None and self.thrift_spec is not None: + oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec])) + return + oprot.writeStructBegin("get_fields_args") + if self.db_name is not None: + oprot.writeFieldBegin("db_name", TType.STRING, 1) + oprot.writeString(self.db_name.encode("utf-8") if sys.version_info[0] == 2 else self.db_name) + oprot.writeFieldEnd() + if self.table_name is not None: + oprot.writeFieldBegin("table_name", TType.STRING, 2) + oprot.writeString(self.table_name.encode("utf-8") if sys.version_info[0] == 2 else self.table_name) + oprot.writeFieldEnd() + oprot.writeFieldStop() + oprot.writeStructEnd() + + def validate(self): + return + + def __repr__(self): + L = ["{}={!r}".format(key, value) for key, value in self.__dict__.items()] + return "{}({})".format(self.__class__.__name__, ", ".join(L)) + + def __eq__(self, other): + return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ + + def __ne__(self, other): + return not (self == other) + + +all_structs.append(get_fields_args) +get_fields_args.thrift_spec = ( + None, # 0 + ( + 1, + TType.STRING, + "db_name", + "UTF8", + None, + ), # 1 + ( + 2, + TType.STRING, + "table_name", + "UTF8", + None, + ), # 2 +) + + +class get_fields_result: + """ + Attributes: + - success + - o1 + - o2 + - o3 + + """ + + def __init__( + self, + success=None, + o1=None, + o2=None, + o3=None, + ): + self.success = success + self.o1 = o1 + self.o2 = o2 + self.o3 = o3 + + def read(self, iprot): + if ( + iprot._fast_decode is not None + and isinstance(iprot.trans, TTransport.CReadableTransport) + and self.thrift_spec is not None + ): + iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec]) + return + iprot.readStructBegin() + while True: + (fname, ftype, fid) = iprot.readFieldBegin() + if ftype == TType.STOP: + break + if fid == 0: + if ftype == TType.LIST: + self.success = [] + (_etype1282, _size1279) = iprot.readListBegin() + for _i1283 in range(_size1279): + _elem1284 = FieldSchema() + _elem1284.read(iprot) + self.success.append(_elem1284) + iprot.readListEnd() + else: + iprot.skip(ftype) + elif fid == 1: + if ftype == TType.STRUCT: + self.o1 = MetaException.read(iprot) + else: + iprot.skip(ftype) + elif fid == 2: + if ftype == TType.STRUCT: + self.o2 = UnknownTableException.read(iprot) + else: + iprot.skip(ftype) + elif fid == 3: + if ftype == TType.STRUCT: + self.o3 = UnknownDBException.read(iprot) + else: + iprot.skip(ftype) + else: + iprot.skip(ftype) + iprot.readFieldEnd() + iprot.readStructEnd() + + def write(self, oprot): + if oprot._fast_encode is not None and self.thrift_spec is not None: + oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec])) + return + oprot.writeStructBegin("get_fields_result") + if self.success is not None: + oprot.writeFieldBegin("success", TType.LIST, 0) + oprot.writeListBegin(TType.STRUCT, len(self.success)) + for iter1285 in self.success: + iter1285.write(oprot) + oprot.writeListEnd() + oprot.writeFieldEnd() + if self.o1 is not None: + oprot.writeFieldBegin("o1", TType.STRUCT, 1) + self.o1.write(oprot) + oprot.writeFieldEnd() + if self.o2 is not None: + oprot.writeFieldBegin("o2", TType.STRUCT, 2) + self.o2.write(oprot) + oprot.writeFieldEnd() + if self.o3 is not None: + oprot.writeFieldBegin("o3", TType.STRUCT, 3) + self.o3.write(oprot) + oprot.writeFieldEnd() + oprot.writeFieldStop() + oprot.writeStructEnd() + + def validate(self): + return + + def __repr__(self): + L = ["{}={!r}".format(key, value) for key, value in self.__dict__.items()] + return "{}({})".format(self.__class__.__name__, ", ".join(L)) + + def __eq__(self, other): + return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ + + def __ne__(self, other): + return not (self == other) + + +all_structs.append(get_fields_result) +get_fields_result.thrift_spec = ( + ( + 0, + TType.LIST, + "success", + (TType.STRUCT, [FieldSchema, None], False), + None, + ), # 0 + ( + 1, + TType.STRUCT, + "o1", + [MetaException, None], + None, + ), # 1 + ( + 2, + TType.STRUCT, + "o2", + [UnknownTableException, None], + None, + ), # 2 + ( + 3, + TType.STRUCT, + "o3", + [UnknownDBException, None], + None, + ), # 3 +) + + +class get_fields_with_environment_context_args: + """ + Attributes: + - db_name + - table_name + - environment_context + + """ + + def __init__( + self, + db_name=None, + table_name=None, + environment_context=None, + ): + self.db_name = db_name + self.table_name = table_name + self.environment_context = environment_context + + def read(self, iprot): + if ( + iprot._fast_decode is not None + and isinstance(iprot.trans, TTransport.CReadableTransport) + and self.thrift_spec is not None + ): + iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec]) + return + iprot.readStructBegin() + while True: + (fname, ftype, fid) = iprot.readFieldBegin() + if ftype == TType.STOP: + break + if fid == 1: + if ftype == TType.STRING: + self.db_name = ( + iprot.readString().decode("utf-8", errors="replace") if sys.version_info[0] == 2 else iprot.readString() + ) + else: + iprot.skip(ftype) + elif fid == 2: + if ftype == TType.STRING: + self.table_name = ( + iprot.readString().decode("utf-8", errors="replace") if sys.version_info[0] == 2 else iprot.readString() + ) + else: + iprot.skip(ftype) + elif fid == 3: + if ftype == TType.STRUCT: + self.environment_context = EnvironmentContext() + self.environment_context.read(iprot) + else: + iprot.skip(ftype) + else: + iprot.skip(ftype) + iprot.readFieldEnd() + iprot.readStructEnd() + + def write(self, oprot): + if oprot._fast_encode is not None and self.thrift_spec is not None: + oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec])) + return + oprot.writeStructBegin("get_fields_with_environment_context_args") + if self.db_name is not None: + oprot.writeFieldBegin("db_name", TType.STRING, 1) + oprot.writeString(self.db_name.encode("utf-8") if sys.version_info[0] == 2 else self.db_name) + oprot.writeFieldEnd() + if self.table_name is not None: + oprot.writeFieldBegin("table_name", TType.STRING, 2) + oprot.writeString(self.table_name.encode("utf-8") if sys.version_info[0] == 2 else self.table_name) + oprot.writeFieldEnd() + if self.environment_context is not None: + oprot.writeFieldBegin("environment_context", TType.STRUCT, 3) + self.environment_context.write(oprot) + oprot.writeFieldEnd() + oprot.writeFieldStop() + oprot.writeStructEnd() + + def validate(self): + return + + def __repr__(self): + L = ["{}={!r}".format(key, value) for key, value in self.__dict__.items()] + return "{}({})".format(self.__class__.__name__, ", ".join(L)) + + def __eq__(self, other): + return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ + + def __ne__(self, other): + return not (self == other) + + +all_structs.append(get_fields_with_environment_context_args) +get_fields_with_environment_context_args.thrift_spec = ( + None, # 0 + ( + 1, + TType.STRING, + "db_name", + "UTF8", + None, + ), # 1 + ( + 2, + TType.STRING, + "table_name", + "UTF8", + None, + ), # 2 + ( + 3, + TType.STRUCT, + "environment_context", + [EnvironmentContext, None], + None, + ), # 3 +) + + +class get_fields_with_environment_context_result: + """ + Attributes: + - success + - o1 + - o2 + - o3 + + """ + + def __init__( + self, + success=None, + o1=None, + o2=None, + o3=None, + ): + self.success = success + self.o1 = o1 + self.o2 = o2 + self.o3 = o3 + + def read(self, iprot): + if ( + iprot._fast_decode is not None + and isinstance(iprot.trans, TTransport.CReadableTransport) + and self.thrift_spec is not None + ): + iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec]) + return + iprot.readStructBegin() + while True: + (fname, ftype, fid) = iprot.readFieldBegin() + if ftype == TType.STOP: + break + if fid == 0: + if ftype == TType.LIST: + self.success = [] + (_etype1289, _size1286) = iprot.readListBegin() + for _i1290 in range(_size1286): + _elem1291 = FieldSchema() + _elem1291.read(iprot) + self.success.append(_elem1291) + iprot.readListEnd() + else: + iprot.skip(ftype) + elif fid == 1: + if ftype == TType.STRUCT: + self.o1 = MetaException.read(iprot) + else: + iprot.skip(ftype) + elif fid == 2: + if ftype == TType.STRUCT: + self.o2 = UnknownTableException.read(iprot) + else: + iprot.skip(ftype) + elif fid == 3: + if ftype == TType.STRUCT: + self.o3 = UnknownDBException.read(iprot) + else: + iprot.skip(ftype) + else: + iprot.skip(ftype) + iprot.readFieldEnd() + iprot.readStructEnd() + + def write(self, oprot): + if oprot._fast_encode is not None and self.thrift_spec is not None: + oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec])) + return + oprot.writeStructBegin("get_fields_with_environment_context_result") + if self.success is not None: + oprot.writeFieldBegin("success", TType.LIST, 0) + oprot.writeListBegin(TType.STRUCT, len(self.success)) + for iter1292 in self.success: + iter1292.write(oprot) + oprot.writeListEnd() + oprot.writeFieldEnd() + if self.o1 is not None: + oprot.writeFieldBegin("o1", TType.STRUCT, 1) + self.o1.write(oprot) + oprot.writeFieldEnd() + if self.o2 is not None: + oprot.writeFieldBegin("o2", TType.STRUCT, 2) + self.o2.write(oprot) + oprot.writeFieldEnd() + if self.o3 is not None: + oprot.writeFieldBegin("o3", TType.STRUCT, 3) + self.o3.write(oprot) + oprot.writeFieldEnd() + oprot.writeFieldStop() + oprot.writeStructEnd() + + def validate(self): + return + + def __repr__(self): + L = ["{}={!r}".format(key, value) for key, value in self.__dict__.items()] + return "{}({})".format(self.__class__.__name__, ", ".join(L)) + + def __eq__(self, other): + return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ + + def __ne__(self, other): + return not (self == other) + + +all_structs.append(get_fields_with_environment_context_result) +get_fields_with_environment_context_result.thrift_spec = ( + ( + 0, + TType.LIST, + "success", + (TType.STRUCT, [FieldSchema, None], False), + None, + ), # 0 + ( + 1, + TType.STRUCT, + "o1", + [MetaException, None], + None, + ), # 1 + ( + 2, + TType.STRUCT, + "o2", + [UnknownTableException, None], + None, + ), # 2 + ( + 3, + TType.STRUCT, + "o3", + [UnknownDBException, None], + None, + ), # 3 +) + + +class get_fields_req_args: + """ + Attributes: + - req + + """ + + def __init__( + self, + req=None, + ): + self.req = req + + def read(self, iprot): + if ( + iprot._fast_decode is not None + and isinstance(iprot.trans, TTransport.CReadableTransport) + and self.thrift_spec is not None + ): + iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec]) + return + iprot.readStructBegin() + while True: + (fname, ftype, fid) = iprot.readFieldBegin() + if ftype == TType.STOP: + break + if fid == 1: + if ftype == TType.STRUCT: + self.req = GetFieldsRequest() + self.req.read(iprot) + else: + iprot.skip(ftype) + else: + iprot.skip(ftype) + iprot.readFieldEnd() + iprot.readStructEnd() + + def write(self, oprot): + if oprot._fast_encode is not None and self.thrift_spec is not None: + oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec])) + return + oprot.writeStructBegin("get_fields_req_args") + if self.req is not None: + oprot.writeFieldBegin("req", TType.STRUCT, 1) + self.req.write(oprot) + oprot.writeFieldEnd() + oprot.writeFieldStop() + oprot.writeStructEnd() + + def validate(self): + return + + def __repr__(self): + L = ["{}={!r}".format(key, value) for key, value in self.__dict__.items()] + return "{}({})".format(self.__class__.__name__, ", ".join(L)) + + def __eq__(self, other): + return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ + + def __ne__(self, other): + return not (self == other) + + +all_structs.append(get_fields_req_args) +get_fields_req_args.thrift_spec = ( + None, # 0 + ( + 1, + TType.STRUCT, + "req", + [GetFieldsRequest, None], + None, + ), # 1 +) + + +class get_fields_req_result: + """ + Attributes: + - success + - o1 + - o2 + - o3 + + """ + + def __init__( + self, + success=None, + o1=None, + o2=None, + o3=None, + ): + self.success = success + self.o1 = o1 + self.o2 = o2 + self.o3 = o3 + + def read(self, iprot): + if ( + iprot._fast_decode is not None + and isinstance(iprot.trans, TTransport.CReadableTransport) + and self.thrift_spec is not None + ): + iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec]) + return + iprot.readStructBegin() + while True: + (fname, ftype, fid) = iprot.readFieldBegin() + if ftype == TType.STOP: + break + if fid == 0: + if ftype == TType.STRUCT: + self.success = GetFieldsResponse() + self.success.read(iprot) + else: + iprot.skip(ftype) + elif fid == 1: + if ftype == TType.STRUCT: + self.o1 = MetaException.read(iprot) + else: + iprot.skip(ftype) + elif fid == 2: + if ftype == TType.STRUCT: + self.o2 = UnknownTableException.read(iprot) + else: + iprot.skip(ftype) + elif fid == 3: + if ftype == TType.STRUCT: + self.o3 = UnknownDBException.read(iprot) + else: + iprot.skip(ftype) + else: + iprot.skip(ftype) + iprot.readFieldEnd() + iprot.readStructEnd() + + def write(self, oprot): + if oprot._fast_encode is not None and self.thrift_spec is not None: + oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec])) + return + oprot.writeStructBegin("get_fields_req_result") + if self.success is not None: + oprot.writeFieldBegin("success", TType.STRUCT, 0) + self.success.write(oprot) + oprot.writeFieldEnd() + if self.o1 is not None: + oprot.writeFieldBegin("o1", TType.STRUCT, 1) + self.o1.write(oprot) + oprot.writeFieldEnd() + if self.o2 is not None: + oprot.writeFieldBegin("o2", TType.STRUCT, 2) + self.o2.write(oprot) + oprot.writeFieldEnd() + if self.o3 is not None: + oprot.writeFieldBegin("o3", TType.STRUCT, 3) + self.o3.write(oprot) + oprot.writeFieldEnd() + oprot.writeFieldStop() + oprot.writeStructEnd() + + def validate(self): + return + + def __repr__(self): + L = ["{}={!r}".format(key, value) for key, value in self.__dict__.items()] + return "{}({})".format(self.__class__.__name__, ", ".join(L)) + + def __eq__(self, other): + return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ + + def __ne__(self, other): + return not (self == other) + + +all_structs.append(get_fields_req_result) +get_fields_req_result.thrift_spec = ( + ( + 0, + TType.STRUCT, + "success", + [GetFieldsResponse, None], + None, + ), # 0 + ( + 1, + TType.STRUCT, + "o1", + [MetaException, None], + None, + ), # 1 + ( + 2, + TType.STRUCT, + "o2", + [UnknownTableException, None], + None, + ), # 2 + ( + 3, + TType.STRUCT, + "o3", + [UnknownDBException, None], + None, + ), # 3 +) + + +class get_schema_args: + """ + Attributes: + - db_name + - table_name + + """ + + def __init__( + self, + db_name=None, + table_name=None, + ): + self.db_name = db_name + self.table_name = table_name + + def read(self, iprot): + if ( + iprot._fast_decode is not None + and isinstance(iprot.trans, TTransport.CReadableTransport) + and self.thrift_spec is not None + ): + iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec]) + return + iprot.readStructBegin() + while True: + (fname, ftype, fid) = iprot.readFieldBegin() + if ftype == TType.STOP: + break + if fid == 1: + if ftype == TType.STRING: + self.db_name = ( + iprot.readString().decode("utf-8", errors="replace") if sys.version_info[0] == 2 else iprot.readString() + ) + else: + iprot.skip(ftype) + elif fid == 2: + if ftype == TType.STRING: + self.table_name = ( + iprot.readString().decode("utf-8", errors="replace") if sys.version_info[0] == 2 else iprot.readString() + ) + else: + iprot.skip(ftype) + else: + iprot.skip(ftype) + iprot.readFieldEnd() + iprot.readStructEnd() + + def write(self, oprot): + if oprot._fast_encode is not None and self.thrift_spec is not None: + oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec])) + return + oprot.writeStructBegin("get_schema_args") + if self.db_name is not None: + oprot.writeFieldBegin("db_name", TType.STRING, 1) + oprot.writeString(self.db_name.encode("utf-8") if sys.version_info[0] == 2 else self.db_name) + oprot.writeFieldEnd() + if self.table_name is not None: + oprot.writeFieldBegin("table_name", TType.STRING, 2) + oprot.writeString(self.table_name.encode("utf-8") if sys.version_info[0] == 2 else self.table_name) + oprot.writeFieldEnd() + oprot.writeFieldStop() + oprot.writeStructEnd() + + def validate(self): + return + + def __repr__(self): + L = ["{}={!r}".format(key, value) for key, value in self.__dict__.items()] + return "{}({})".format(self.__class__.__name__, ", ".join(L)) + + def __eq__(self, other): + return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ + + def __ne__(self, other): + return not (self == other) + + +all_structs.append(get_schema_args) +get_schema_args.thrift_spec = ( + None, # 0 + ( + 1, + TType.STRING, + "db_name", + "UTF8", + None, + ), # 1 + ( + 2, + TType.STRING, + "table_name", + "UTF8", + None, + ), # 2 +) + + +class get_schema_result: + """ + Attributes: + - success + - o1 + - o2 + - o3 + + """ + + def __init__( + self, + success=None, + o1=None, + o2=None, + o3=None, + ): + self.success = success + self.o1 = o1 + self.o2 = o2 + self.o3 = o3 + + def read(self, iprot): + if ( + iprot._fast_decode is not None + and isinstance(iprot.trans, TTransport.CReadableTransport) + and self.thrift_spec is not None + ): + iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec]) + return + iprot.readStructBegin() + while True: + (fname, ftype, fid) = iprot.readFieldBegin() + if ftype == TType.STOP: + break + if fid == 0: + if ftype == TType.LIST: + self.success = [] + (_etype1296, _size1293) = iprot.readListBegin() + for _i1297 in range(_size1293): + _elem1298 = FieldSchema() + _elem1298.read(iprot) + self.success.append(_elem1298) + iprot.readListEnd() + else: + iprot.skip(ftype) + elif fid == 1: + if ftype == TType.STRUCT: + self.o1 = MetaException.read(iprot) + else: + iprot.skip(ftype) + elif fid == 2: + if ftype == TType.STRUCT: + self.o2 = UnknownTableException.read(iprot) + else: + iprot.skip(ftype) + elif fid == 3: + if ftype == TType.STRUCT: + self.o3 = UnknownDBException.read(iprot) + else: + iprot.skip(ftype) + else: + iprot.skip(ftype) + iprot.readFieldEnd() + iprot.readStructEnd() + + def write(self, oprot): + if oprot._fast_encode is not None and self.thrift_spec is not None: + oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec])) + return + oprot.writeStructBegin("get_schema_result") + if self.success is not None: + oprot.writeFieldBegin("success", TType.LIST, 0) + oprot.writeListBegin(TType.STRUCT, len(self.success)) + for iter1299 in self.success: + iter1299.write(oprot) + oprot.writeListEnd() + oprot.writeFieldEnd() + if self.o1 is not None: + oprot.writeFieldBegin("o1", TType.STRUCT, 1) + self.o1.write(oprot) + oprot.writeFieldEnd() + if self.o2 is not None: + oprot.writeFieldBegin("o2", TType.STRUCT, 2) + self.o2.write(oprot) + oprot.writeFieldEnd() + if self.o3 is not None: + oprot.writeFieldBegin("o3", TType.STRUCT, 3) + self.o3.write(oprot) + oprot.writeFieldEnd() + oprot.writeFieldStop() + oprot.writeStructEnd() + + def validate(self): + return + + def __repr__(self): + L = ["{}={!r}".format(key, value) for key, value in self.__dict__.items()] + return "{}({})".format(self.__class__.__name__, ", ".join(L)) + + def __eq__(self, other): + return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ + + def __ne__(self, other): + return not (self == other) + + +all_structs.append(get_schema_result) +get_schema_result.thrift_spec = ( + ( + 0, + TType.LIST, + "success", + (TType.STRUCT, [FieldSchema, None], False), + None, + ), # 0 + ( + 1, + TType.STRUCT, + "o1", + [MetaException, None], + None, + ), # 1 + ( + 2, + TType.STRUCT, + "o2", + [UnknownTableException, None], + None, + ), # 2 + ( + 3, + TType.STRUCT, + "o3", + [UnknownDBException, None], + None, + ), # 3 +) + + +class get_schema_with_environment_context_args: + """ + Attributes: + - db_name + - table_name + - environment_context + + """ + + def __init__( + self, + db_name=None, + table_name=None, + environment_context=None, + ): + self.db_name = db_name + self.table_name = table_name + self.environment_context = environment_context + + def read(self, iprot): + if ( + iprot._fast_decode is not None + and isinstance(iprot.trans, TTransport.CReadableTransport) + and self.thrift_spec is not None + ): + iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec]) + return + iprot.readStructBegin() + while True: + (fname, ftype, fid) = iprot.readFieldBegin() + if ftype == TType.STOP: + break + if fid == 1: + if ftype == TType.STRING: + self.db_name = ( + iprot.readString().decode("utf-8", errors="replace") if sys.version_info[0] == 2 else iprot.readString() + ) + else: + iprot.skip(ftype) + elif fid == 2: + if ftype == TType.STRING: + self.table_name = ( + iprot.readString().decode("utf-8", errors="replace") if sys.version_info[0] == 2 else iprot.readString() + ) + else: + iprot.skip(ftype) + elif fid == 3: + if ftype == TType.STRUCT: + self.environment_context = EnvironmentContext() + self.environment_context.read(iprot) + else: + iprot.skip(ftype) + else: + iprot.skip(ftype) + iprot.readFieldEnd() + iprot.readStructEnd() + + def write(self, oprot): + if oprot._fast_encode is not None and self.thrift_spec is not None: + oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec])) + return + oprot.writeStructBegin("get_schema_with_environment_context_args") + if self.db_name is not None: + oprot.writeFieldBegin("db_name", TType.STRING, 1) + oprot.writeString(self.db_name.encode("utf-8") if sys.version_info[0] == 2 else self.db_name) + oprot.writeFieldEnd() + if self.table_name is not None: + oprot.writeFieldBegin("table_name", TType.STRING, 2) + oprot.writeString(self.table_name.encode("utf-8") if sys.version_info[0] == 2 else self.table_name) + oprot.writeFieldEnd() + if self.environment_context is not None: + oprot.writeFieldBegin("environment_context", TType.STRUCT, 3) + self.environment_context.write(oprot) + oprot.writeFieldEnd() + oprot.writeFieldStop() + oprot.writeStructEnd() + + def validate(self): + return + + def __repr__(self): + L = ["{}={!r}".format(key, value) for key, value in self.__dict__.items()] + return "{}({})".format(self.__class__.__name__, ", ".join(L)) + + def __eq__(self, other): + return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ + + def __ne__(self, other): + return not (self == other) + + +all_structs.append(get_schema_with_environment_context_args) +get_schema_with_environment_context_args.thrift_spec = ( + None, # 0 + ( + 1, + TType.STRING, + "db_name", + "UTF8", + None, + ), # 1 + ( + 2, + TType.STRING, + "table_name", + "UTF8", + None, + ), # 2 + ( + 3, + TType.STRUCT, + "environment_context", + [EnvironmentContext, None], + None, + ), # 3 +) + + +class get_schema_with_environment_context_result: + """ + Attributes: + - success + - o1 + - o2 + - o3 + + """ + + def __init__( + self, + success=None, + o1=None, + o2=None, + o3=None, + ): + self.success = success + self.o1 = o1 + self.o2 = o2 + self.o3 = o3 + + def read(self, iprot): + if ( + iprot._fast_decode is not None + and isinstance(iprot.trans, TTransport.CReadableTransport) + and self.thrift_spec is not None + ): + iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec]) + return + iprot.readStructBegin() + while True: + (fname, ftype, fid) = iprot.readFieldBegin() + if ftype == TType.STOP: + break + if fid == 0: + if ftype == TType.LIST: + self.success = [] + (_etype1303, _size1300) = iprot.readListBegin() + for _i1304 in range(_size1300): + _elem1305 = FieldSchema() + _elem1305.read(iprot) + self.success.append(_elem1305) + iprot.readListEnd() + else: + iprot.skip(ftype) + elif fid == 1: + if ftype == TType.STRUCT: + self.o1 = MetaException.read(iprot) + else: + iprot.skip(ftype) + elif fid == 2: + if ftype == TType.STRUCT: + self.o2 = UnknownTableException.read(iprot) + else: + iprot.skip(ftype) + elif fid == 3: + if ftype == TType.STRUCT: + self.o3 = UnknownDBException.read(iprot) + else: + iprot.skip(ftype) + else: + iprot.skip(ftype) + iprot.readFieldEnd() + iprot.readStructEnd() + + def write(self, oprot): + if oprot._fast_encode is not None and self.thrift_spec is not None: + oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec])) + return + oprot.writeStructBegin("get_schema_with_environment_context_result") + if self.success is not None: + oprot.writeFieldBegin("success", TType.LIST, 0) + oprot.writeListBegin(TType.STRUCT, len(self.success)) + for iter1306 in self.success: + iter1306.write(oprot) + oprot.writeListEnd() + oprot.writeFieldEnd() + if self.o1 is not None: + oprot.writeFieldBegin("o1", TType.STRUCT, 1) + self.o1.write(oprot) + oprot.writeFieldEnd() + if self.o2 is not None: + oprot.writeFieldBegin("o2", TType.STRUCT, 2) + self.o2.write(oprot) + oprot.writeFieldEnd() + if self.o3 is not None: + oprot.writeFieldBegin("o3", TType.STRUCT, 3) + self.o3.write(oprot) + oprot.writeFieldEnd() + oprot.writeFieldStop() + oprot.writeStructEnd() + + def validate(self): + return + + def __repr__(self): + L = ["{}={!r}".format(key, value) for key, value in self.__dict__.items()] + return "{}({})".format(self.__class__.__name__, ", ".join(L)) + + def __eq__(self, other): + return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ + + def __ne__(self, other): + return not (self == other) + + +all_structs.append(get_schema_with_environment_context_result) +get_schema_with_environment_context_result.thrift_spec = ( + ( + 0, + TType.LIST, + "success", + (TType.STRUCT, [FieldSchema, None], False), + None, + ), # 0 + ( + 1, + TType.STRUCT, + "o1", + [MetaException, None], + None, + ), # 1 + ( + 2, + TType.STRUCT, + "o2", + [UnknownTableException, None], + None, + ), # 2 + ( + 3, + TType.STRUCT, + "o3", + [UnknownDBException, None], + None, + ), # 3 +) + + +class get_schema_req_args: + """ + Attributes: + - req + + """ + + def __init__( + self, + req=None, + ): + self.req = req + + def read(self, iprot): + if ( + iprot._fast_decode is not None + and isinstance(iprot.trans, TTransport.CReadableTransport) + and self.thrift_spec is not None + ): + iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec]) + return + iprot.readStructBegin() + while True: + (fname, ftype, fid) = iprot.readFieldBegin() + if ftype == TType.STOP: + break + if fid == 1: + if ftype == TType.STRUCT: + self.req = GetSchemaRequest() + self.req.read(iprot) + else: + iprot.skip(ftype) + else: + iprot.skip(ftype) + iprot.readFieldEnd() + iprot.readStructEnd() + + def write(self, oprot): + if oprot._fast_encode is not None and self.thrift_spec is not None: + oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec])) + return + oprot.writeStructBegin("get_schema_req_args") + if self.req is not None: + oprot.writeFieldBegin("req", TType.STRUCT, 1) + self.req.write(oprot) + oprot.writeFieldEnd() + oprot.writeFieldStop() + oprot.writeStructEnd() + + def validate(self): + return + + def __repr__(self): + L = ["{}={!r}".format(key, value) for key, value in self.__dict__.items()] + return "{}({})".format(self.__class__.__name__, ", ".join(L)) + + def __eq__(self, other): + return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ + + def __ne__(self, other): + return not (self == other) + + +all_structs.append(get_schema_req_args) +get_schema_req_args.thrift_spec = ( + None, # 0 + ( + 1, + TType.STRUCT, + "req", + [GetSchemaRequest, None], + None, + ), # 1 +) + + +class get_schema_req_result: + """ + Attributes: + - success + - o1 + - o2 + - o3 + + """ + + def __init__( + self, + success=None, + o1=None, + o2=None, + o3=None, + ): + self.success = success + self.o1 = o1 + self.o2 = o2 + self.o3 = o3 + + def read(self, iprot): + if ( + iprot._fast_decode is not None + and isinstance(iprot.trans, TTransport.CReadableTransport) + and self.thrift_spec is not None + ): + iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec]) + return + iprot.readStructBegin() + while True: + (fname, ftype, fid) = iprot.readFieldBegin() + if ftype == TType.STOP: + break + if fid == 0: + if ftype == TType.STRUCT: + self.success = GetSchemaResponse() + self.success.read(iprot) + else: + iprot.skip(ftype) + elif fid == 1: + if ftype == TType.STRUCT: + self.o1 = MetaException.read(iprot) + else: + iprot.skip(ftype) + elif fid == 2: + if ftype == TType.STRUCT: + self.o2 = UnknownTableException.read(iprot) + else: + iprot.skip(ftype) + elif fid == 3: + if ftype == TType.STRUCT: + self.o3 = UnknownDBException.read(iprot) + else: + iprot.skip(ftype) + else: + iprot.skip(ftype) + iprot.readFieldEnd() + iprot.readStructEnd() + + def write(self, oprot): + if oprot._fast_encode is not None and self.thrift_spec is not None: + oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec])) + return + oprot.writeStructBegin("get_schema_req_result") + if self.success is not None: + oprot.writeFieldBegin("success", TType.STRUCT, 0) + self.success.write(oprot) + oprot.writeFieldEnd() + if self.o1 is not None: + oprot.writeFieldBegin("o1", TType.STRUCT, 1) + self.o1.write(oprot) + oprot.writeFieldEnd() + if self.o2 is not None: + oprot.writeFieldBegin("o2", TType.STRUCT, 2) + self.o2.write(oprot) + oprot.writeFieldEnd() + if self.o3 is not None: + oprot.writeFieldBegin("o3", TType.STRUCT, 3) + self.o3.write(oprot) + oprot.writeFieldEnd() + oprot.writeFieldStop() + oprot.writeStructEnd() + + def validate(self): + return + + def __repr__(self): + L = ["{}={!r}".format(key, value) for key, value in self.__dict__.items()] + return "{}({})".format(self.__class__.__name__, ", ".join(L)) + + def __eq__(self, other): + return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ + + def __ne__(self, other): + return not (self == other) + + +all_structs.append(get_schema_req_result) +get_schema_req_result.thrift_spec = ( + ( + 0, + TType.STRUCT, + "success", + [GetSchemaResponse, None], + None, + ), # 0 + ( + 1, + TType.STRUCT, + "o1", + [MetaException, None], + None, + ), # 1 + ( + 2, + TType.STRUCT, + "o2", + [UnknownTableException, None], + None, + ), # 2 + ( + 3, + TType.STRUCT, + "o3", + [UnknownDBException, None], + None, + ), # 3 +) + + +class create_table_args: + """ + Attributes: + - tbl + + """ + + def __init__( + self, + tbl=None, + ): + self.tbl = tbl + + def read(self, iprot): + if ( + iprot._fast_decode is not None + and isinstance(iprot.trans, TTransport.CReadableTransport) + and self.thrift_spec is not None + ): + iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec]) + return + iprot.readStructBegin() + while True: + (fname, ftype, fid) = iprot.readFieldBegin() + if ftype == TType.STOP: + break + if fid == 1: + if ftype == TType.STRUCT: + self.tbl = Table() + self.tbl.read(iprot) + else: + iprot.skip(ftype) + else: + iprot.skip(ftype) + iprot.readFieldEnd() + iprot.readStructEnd() + + def write(self, oprot): + if oprot._fast_encode is not None and self.thrift_spec is not None: + oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec])) + return + oprot.writeStructBegin("create_table_args") + if self.tbl is not None: + oprot.writeFieldBegin("tbl", TType.STRUCT, 1) + self.tbl.write(oprot) + oprot.writeFieldEnd() + oprot.writeFieldStop() + oprot.writeStructEnd() + + def validate(self): + return + + def __repr__(self): + L = ["{}={!r}".format(key, value) for key, value in self.__dict__.items()] + return "{}({})".format(self.__class__.__name__, ", ".join(L)) + + def __eq__(self, other): + return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ + + def __ne__(self, other): + return not (self == other) + + +all_structs.append(create_table_args) +create_table_args.thrift_spec = ( + None, # 0 + ( + 1, + TType.STRUCT, + "tbl", + [Table, None], + None, + ), # 1 +) + + +class create_table_result: + """ + Attributes: + - o1 + - o2 + - o3 + - o4 + + """ + + def __init__( + self, + o1=None, + o2=None, + o3=None, + o4=None, + ): + self.o1 = o1 + self.o2 = o2 + self.o3 = o3 + self.o4 = o4 + + def read(self, iprot): + if ( + iprot._fast_decode is not None + and isinstance(iprot.trans, TTransport.CReadableTransport) + and self.thrift_spec is not None + ): + iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec]) + return + iprot.readStructBegin() + while True: + (fname, ftype, fid) = iprot.readFieldBegin() + if ftype == TType.STOP: + break + if fid == 1: + if ftype == TType.STRUCT: + self.o1 = AlreadyExistsException.read(iprot) + else: + iprot.skip(ftype) + elif fid == 2: + if ftype == TType.STRUCT: + self.o2 = InvalidObjectException.read(iprot) + else: + iprot.skip(ftype) + elif fid == 3: + if ftype == TType.STRUCT: + self.o3 = MetaException.read(iprot) + else: + iprot.skip(ftype) + elif fid == 4: + if ftype == TType.STRUCT: + self.o4 = NoSuchObjectException.read(iprot) + else: + iprot.skip(ftype) + else: + iprot.skip(ftype) + iprot.readFieldEnd() + iprot.readStructEnd() + + def write(self, oprot): + if oprot._fast_encode is not None and self.thrift_spec is not None: + oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec])) + return + oprot.writeStructBegin("create_table_result") + if self.o1 is not None: + oprot.writeFieldBegin("o1", TType.STRUCT, 1) + self.o1.write(oprot) + oprot.writeFieldEnd() + if self.o2 is not None: + oprot.writeFieldBegin("o2", TType.STRUCT, 2) + self.o2.write(oprot) + oprot.writeFieldEnd() + if self.o3 is not None: + oprot.writeFieldBegin("o3", TType.STRUCT, 3) + self.o3.write(oprot) + oprot.writeFieldEnd() + if self.o4 is not None: + oprot.writeFieldBegin("o4", TType.STRUCT, 4) + self.o4.write(oprot) + oprot.writeFieldEnd() + oprot.writeFieldStop() + oprot.writeStructEnd() + + def validate(self): + return + + def __repr__(self): + L = ["{}={!r}".format(key, value) for key, value in self.__dict__.items()] + return "{}({})".format(self.__class__.__name__, ", ".join(L)) + + def __eq__(self, other): + return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ + + def __ne__(self, other): + return not (self == other) + + +all_structs.append(create_table_result) +create_table_result.thrift_spec = ( + None, # 0 + ( + 1, + TType.STRUCT, + "o1", + [AlreadyExistsException, None], + None, + ), # 1 + ( + 2, + TType.STRUCT, + "o2", + [InvalidObjectException, None], + None, + ), # 2 + ( + 3, + TType.STRUCT, + "o3", + [MetaException, None], + None, + ), # 3 + ( + 4, + TType.STRUCT, + "o4", + [NoSuchObjectException, None], + None, + ), # 4 +) + + +class create_table_with_environment_context_args: + """ + Attributes: + - tbl + - environment_context + + """ + + def __init__( + self, + tbl=None, + environment_context=None, + ): + self.tbl = tbl + self.environment_context = environment_context + + def read(self, iprot): + if ( + iprot._fast_decode is not None + and isinstance(iprot.trans, TTransport.CReadableTransport) + and self.thrift_spec is not None + ): + iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec]) + return + iprot.readStructBegin() + while True: + (fname, ftype, fid) = iprot.readFieldBegin() + if ftype == TType.STOP: + break + if fid == 1: + if ftype == TType.STRUCT: + self.tbl = Table() + self.tbl.read(iprot) + else: + iprot.skip(ftype) + elif fid == 2: + if ftype == TType.STRUCT: + self.environment_context = EnvironmentContext() + self.environment_context.read(iprot) + else: + iprot.skip(ftype) + else: + iprot.skip(ftype) + iprot.readFieldEnd() + iprot.readStructEnd() + + def write(self, oprot): + if oprot._fast_encode is not None and self.thrift_spec is not None: + oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec])) + return + oprot.writeStructBegin("create_table_with_environment_context_args") + if self.tbl is not None: + oprot.writeFieldBegin("tbl", TType.STRUCT, 1) + self.tbl.write(oprot) + oprot.writeFieldEnd() + if self.environment_context is not None: + oprot.writeFieldBegin("environment_context", TType.STRUCT, 2) + self.environment_context.write(oprot) + oprot.writeFieldEnd() + oprot.writeFieldStop() + oprot.writeStructEnd() + + def validate(self): + return + + def __repr__(self): + L = ["{}={!r}".format(key, value) for key, value in self.__dict__.items()] + return "{}({})".format(self.__class__.__name__, ", ".join(L)) + + def __eq__(self, other): + return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ + + def __ne__(self, other): + return not (self == other) + + +all_structs.append(create_table_with_environment_context_args) +create_table_with_environment_context_args.thrift_spec = ( + None, # 0 + ( + 1, + TType.STRUCT, + "tbl", + [Table, None], + None, + ), # 1 + ( + 2, + TType.STRUCT, + "environment_context", + [EnvironmentContext, None], + None, + ), # 2 +) + + +class create_table_with_environment_context_result: + """ + Attributes: + - o1 + - o2 + - o3 + - o4 + + """ + + def __init__( + self, + o1=None, + o2=None, + o3=None, + o4=None, + ): + self.o1 = o1 + self.o2 = o2 + self.o3 = o3 + self.o4 = o4 + + def read(self, iprot): + if ( + iprot._fast_decode is not None + and isinstance(iprot.trans, TTransport.CReadableTransport) + and self.thrift_spec is not None + ): + iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec]) + return + iprot.readStructBegin() + while True: + (fname, ftype, fid) = iprot.readFieldBegin() + if ftype == TType.STOP: + break + if fid == 1: + if ftype == TType.STRUCT: + self.o1 = AlreadyExistsException.read(iprot) + else: + iprot.skip(ftype) + elif fid == 2: + if ftype == TType.STRUCT: + self.o2 = InvalidObjectException.read(iprot) + else: + iprot.skip(ftype) + elif fid == 3: + if ftype == TType.STRUCT: + self.o3 = MetaException.read(iprot) + else: + iprot.skip(ftype) + elif fid == 4: + if ftype == TType.STRUCT: + self.o4 = NoSuchObjectException.read(iprot) + else: + iprot.skip(ftype) + else: + iprot.skip(ftype) + iprot.readFieldEnd() + iprot.readStructEnd() + + def write(self, oprot): + if oprot._fast_encode is not None and self.thrift_spec is not None: + oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec])) + return + oprot.writeStructBegin("create_table_with_environment_context_result") + if self.o1 is not None: + oprot.writeFieldBegin("o1", TType.STRUCT, 1) + self.o1.write(oprot) + oprot.writeFieldEnd() + if self.o2 is not None: + oprot.writeFieldBegin("o2", TType.STRUCT, 2) + self.o2.write(oprot) + oprot.writeFieldEnd() + if self.o3 is not None: + oprot.writeFieldBegin("o3", TType.STRUCT, 3) + self.o3.write(oprot) + oprot.writeFieldEnd() + if self.o4 is not None: + oprot.writeFieldBegin("o4", TType.STRUCT, 4) + self.o4.write(oprot) + oprot.writeFieldEnd() + oprot.writeFieldStop() + oprot.writeStructEnd() + + def validate(self): + return + + def __repr__(self): + L = ["{}={!r}".format(key, value) for key, value in self.__dict__.items()] + return "{}({})".format(self.__class__.__name__, ", ".join(L)) + + def __eq__(self, other): + return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ + + def __ne__(self, other): + return not (self == other) + + +all_structs.append(create_table_with_environment_context_result) +create_table_with_environment_context_result.thrift_spec = ( + None, # 0 + ( + 1, + TType.STRUCT, + "o1", + [AlreadyExistsException, None], + None, + ), # 1 + ( + 2, + TType.STRUCT, + "o2", + [InvalidObjectException, None], + None, + ), # 2 + ( + 3, + TType.STRUCT, + "o3", + [MetaException, None], + None, + ), # 3 + ( + 4, + TType.STRUCT, + "o4", + [NoSuchObjectException, None], + None, + ), # 4 +) + + +class create_table_with_constraints_args: + """ + Attributes: + - tbl + - primaryKeys + - foreignKeys + - uniqueConstraints + - notNullConstraints + - defaultConstraints + - checkConstraints + + """ + + def __init__( + self, + tbl=None, + primaryKeys=None, + foreignKeys=None, + uniqueConstraints=None, + notNullConstraints=None, + defaultConstraints=None, + checkConstraints=None, + ): + self.tbl = tbl + self.primaryKeys = primaryKeys + self.foreignKeys = foreignKeys + self.uniqueConstraints = uniqueConstraints + self.notNullConstraints = notNullConstraints + self.defaultConstraints = defaultConstraints + self.checkConstraints = checkConstraints + + def read(self, iprot): + if ( + iprot._fast_decode is not None + and isinstance(iprot.trans, TTransport.CReadableTransport) + and self.thrift_spec is not None + ): + iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec]) + return + iprot.readStructBegin() + while True: + (fname, ftype, fid) = iprot.readFieldBegin() + if ftype == TType.STOP: + break + if fid == 1: + if ftype == TType.STRUCT: + self.tbl = Table() + self.tbl.read(iprot) + else: + iprot.skip(ftype) + elif fid == 2: + if ftype == TType.LIST: + self.primaryKeys = [] + (_etype1310, _size1307) = iprot.readListBegin() + for _i1311 in range(_size1307): + _elem1312 = SQLPrimaryKey() + _elem1312.read(iprot) + self.primaryKeys.append(_elem1312) + iprot.readListEnd() + else: + iprot.skip(ftype) + elif fid == 3: + if ftype == TType.LIST: + self.foreignKeys = [] + (_etype1316, _size1313) = iprot.readListBegin() + for _i1317 in range(_size1313): + _elem1318 = SQLForeignKey() + _elem1318.read(iprot) + self.foreignKeys.append(_elem1318) + iprot.readListEnd() + else: + iprot.skip(ftype) + elif fid == 4: + if ftype == TType.LIST: + self.uniqueConstraints = [] + (_etype1322, _size1319) = iprot.readListBegin() + for _i1323 in range(_size1319): + _elem1324 = SQLUniqueConstraint() + _elem1324.read(iprot) + self.uniqueConstraints.append(_elem1324) + iprot.readListEnd() + else: + iprot.skip(ftype) + elif fid == 5: + if ftype == TType.LIST: + self.notNullConstraints = [] + (_etype1328, _size1325) = iprot.readListBegin() + for _i1329 in range(_size1325): + _elem1330 = SQLNotNullConstraint() + _elem1330.read(iprot) + self.notNullConstraints.append(_elem1330) + iprot.readListEnd() + else: + iprot.skip(ftype) + elif fid == 6: + if ftype == TType.LIST: + self.defaultConstraints = [] + (_etype1334, _size1331) = iprot.readListBegin() + for _i1335 in range(_size1331): + _elem1336 = SQLDefaultConstraint() + _elem1336.read(iprot) + self.defaultConstraints.append(_elem1336) + iprot.readListEnd() + else: + iprot.skip(ftype) + elif fid == 7: + if ftype == TType.LIST: + self.checkConstraints = [] + (_etype1340, _size1337) = iprot.readListBegin() + for _i1341 in range(_size1337): + _elem1342 = SQLCheckConstraint() + _elem1342.read(iprot) + self.checkConstraints.append(_elem1342) + iprot.readListEnd() + else: + iprot.skip(ftype) + else: + iprot.skip(ftype) + iprot.readFieldEnd() + iprot.readStructEnd() + + def write(self, oprot): + if oprot._fast_encode is not None and self.thrift_spec is not None: + oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec])) + return + oprot.writeStructBegin("create_table_with_constraints_args") + if self.tbl is not None: + oprot.writeFieldBegin("tbl", TType.STRUCT, 1) + self.tbl.write(oprot) + oprot.writeFieldEnd() + if self.primaryKeys is not None: + oprot.writeFieldBegin("primaryKeys", TType.LIST, 2) + oprot.writeListBegin(TType.STRUCT, len(self.primaryKeys)) + for iter1343 in self.primaryKeys: + iter1343.write(oprot) + oprot.writeListEnd() + oprot.writeFieldEnd() + if self.foreignKeys is not None: + oprot.writeFieldBegin("foreignKeys", TType.LIST, 3) + oprot.writeListBegin(TType.STRUCT, len(self.foreignKeys)) + for iter1344 in self.foreignKeys: + iter1344.write(oprot) + oprot.writeListEnd() + oprot.writeFieldEnd() + if self.uniqueConstraints is not None: + oprot.writeFieldBegin("uniqueConstraints", TType.LIST, 4) + oprot.writeListBegin(TType.STRUCT, len(self.uniqueConstraints)) + for iter1345 in self.uniqueConstraints: + iter1345.write(oprot) + oprot.writeListEnd() + oprot.writeFieldEnd() + if self.notNullConstraints is not None: + oprot.writeFieldBegin("notNullConstraints", TType.LIST, 5) + oprot.writeListBegin(TType.STRUCT, len(self.notNullConstraints)) + for iter1346 in self.notNullConstraints: + iter1346.write(oprot) + oprot.writeListEnd() + oprot.writeFieldEnd() + if self.defaultConstraints is not None: + oprot.writeFieldBegin("defaultConstraints", TType.LIST, 6) + oprot.writeListBegin(TType.STRUCT, len(self.defaultConstraints)) + for iter1347 in self.defaultConstraints: + iter1347.write(oprot) + oprot.writeListEnd() + oprot.writeFieldEnd() + if self.checkConstraints is not None: + oprot.writeFieldBegin("checkConstraints", TType.LIST, 7) + oprot.writeListBegin(TType.STRUCT, len(self.checkConstraints)) + for iter1348 in self.checkConstraints: + iter1348.write(oprot) + oprot.writeListEnd() + oprot.writeFieldEnd() + oprot.writeFieldStop() + oprot.writeStructEnd() + + def validate(self): + return + + def __repr__(self): + L = ["{}={!r}".format(key, value) for key, value in self.__dict__.items()] + return "{}({})".format(self.__class__.__name__, ", ".join(L)) + + def __eq__(self, other): + return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ + + def __ne__(self, other): + return not (self == other) + + +all_structs.append(create_table_with_constraints_args) +create_table_with_constraints_args.thrift_spec = ( + None, # 0 + ( + 1, + TType.STRUCT, + "tbl", + [Table, None], + None, + ), # 1 + ( + 2, + TType.LIST, + "primaryKeys", + (TType.STRUCT, [SQLPrimaryKey, None], False), + None, + ), # 2 + ( + 3, + TType.LIST, + "foreignKeys", + (TType.STRUCT, [SQLForeignKey, None], False), + None, + ), # 3 + ( + 4, + TType.LIST, + "uniqueConstraints", + (TType.STRUCT, [SQLUniqueConstraint, None], False), + None, + ), # 4 + ( + 5, + TType.LIST, + "notNullConstraints", + (TType.STRUCT, [SQLNotNullConstraint, None], False), + None, + ), # 5 + ( + 6, + TType.LIST, + "defaultConstraints", + (TType.STRUCT, [SQLDefaultConstraint, None], False), + None, + ), # 6 + ( + 7, + TType.LIST, + "checkConstraints", + (TType.STRUCT, [SQLCheckConstraint, None], False), + None, + ), # 7 +) + + +class create_table_with_constraints_result: + """ + Attributes: + - o1 + - o2 + - o3 + - o4 + + """ + + def __init__( + self, + o1=None, + o2=None, + o3=None, + o4=None, + ): + self.o1 = o1 + self.o2 = o2 + self.o3 = o3 + self.o4 = o4 + + def read(self, iprot): + if ( + iprot._fast_decode is not None + and isinstance(iprot.trans, TTransport.CReadableTransport) + and self.thrift_spec is not None + ): + iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec]) + return + iprot.readStructBegin() + while True: + (fname, ftype, fid) = iprot.readFieldBegin() + if ftype == TType.STOP: + break + if fid == 1: + if ftype == TType.STRUCT: + self.o1 = AlreadyExistsException.read(iprot) + else: + iprot.skip(ftype) + elif fid == 2: + if ftype == TType.STRUCT: + self.o2 = InvalidObjectException.read(iprot) + else: + iprot.skip(ftype) + elif fid == 3: + if ftype == TType.STRUCT: + self.o3 = MetaException.read(iprot) + else: + iprot.skip(ftype) + elif fid == 4: + if ftype == TType.STRUCT: + self.o4 = NoSuchObjectException.read(iprot) + else: + iprot.skip(ftype) + else: + iprot.skip(ftype) + iprot.readFieldEnd() + iprot.readStructEnd() + + def write(self, oprot): + if oprot._fast_encode is not None and self.thrift_spec is not None: + oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec])) + return + oprot.writeStructBegin("create_table_with_constraints_result") + if self.o1 is not None: + oprot.writeFieldBegin("o1", TType.STRUCT, 1) + self.o1.write(oprot) + oprot.writeFieldEnd() + if self.o2 is not None: + oprot.writeFieldBegin("o2", TType.STRUCT, 2) + self.o2.write(oprot) + oprot.writeFieldEnd() + if self.o3 is not None: + oprot.writeFieldBegin("o3", TType.STRUCT, 3) + self.o3.write(oprot) + oprot.writeFieldEnd() + if self.o4 is not None: + oprot.writeFieldBegin("o4", TType.STRUCT, 4) + self.o4.write(oprot) + oprot.writeFieldEnd() + oprot.writeFieldStop() + oprot.writeStructEnd() + + def validate(self): + return + + def __repr__(self): + L = ["{}={!r}".format(key, value) for key, value in self.__dict__.items()] + return "{}({})".format(self.__class__.__name__, ", ".join(L)) + + def __eq__(self, other): + return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ + + def __ne__(self, other): + return not (self == other) + + +all_structs.append(create_table_with_constraints_result) +create_table_with_constraints_result.thrift_spec = ( + None, # 0 + ( + 1, + TType.STRUCT, + "o1", + [AlreadyExistsException, None], + None, + ), # 1 + ( + 2, + TType.STRUCT, + "o2", + [InvalidObjectException, None], + None, + ), # 2 + ( + 3, + TType.STRUCT, + "o3", + [MetaException, None], + None, + ), # 3 + ( + 4, + TType.STRUCT, + "o4", + [NoSuchObjectException, None], + None, + ), # 4 +) + + +class create_table_req_args: + """ + Attributes: + - request + + """ + + def __init__( + self, + request=None, + ): + self.request = request + + def read(self, iprot): + if ( + iprot._fast_decode is not None + and isinstance(iprot.trans, TTransport.CReadableTransport) + and self.thrift_spec is not None + ): + iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec]) + return + iprot.readStructBegin() + while True: + (fname, ftype, fid) = iprot.readFieldBegin() + if ftype == TType.STOP: + break + if fid == 1: + if ftype == TType.STRUCT: + self.request = CreateTableRequest() + self.request.read(iprot) + else: + iprot.skip(ftype) + else: + iprot.skip(ftype) + iprot.readFieldEnd() + iprot.readStructEnd() + + def write(self, oprot): + if oprot._fast_encode is not None and self.thrift_spec is not None: + oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec])) + return + oprot.writeStructBegin("create_table_req_args") + if self.request is not None: + oprot.writeFieldBegin("request", TType.STRUCT, 1) + self.request.write(oprot) + oprot.writeFieldEnd() + oprot.writeFieldStop() + oprot.writeStructEnd() + + def validate(self): + return + + def __repr__(self): + L = ["{}={!r}".format(key, value) for key, value in self.__dict__.items()] + return "{}({})".format(self.__class__.__name__, ", ".join(L)) + + def __eq__(self, other): + return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ + + def __ne__(self, other): + return not (self == other) + + +all_structs.append(create_table_req_args) +create_table_req_args.thrift_spec = ( + None, # 0 + ( + 1, + TType.STRUCT, + "request", + [CreateTableRequest, None], + None, + ), # 1 +) + + +class create_table_req_result: + """ + Attributes: + - o1 + - o2 + - o3 + - o4 + + """ + + def __init__( + self, + o1=None, + o2=None, + o3=None, + o4=None, + ): + self.o1 = o1 + self.o2 = o2 + self.o3 = o3 + self.o4 = o4 + + def read(self, iprot): + if ( + iprot._fast_decode is not None + and isinstance(iprot.trans, TTransport.CReadableTransport) + and self.thrift_spec is not None + ): + iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec]) + return + iprot.readStructBegin() + while True: + (fname, ftype, fid) = iprot.readFieldBegin() + if ftype == TType.STOP: + break + if fid == 1: + if ftype == TType.STRUCT: + self.o1 = AlreadyExistsException.read(iprot) + else: + iprot.skip(ftype) + elif fid == 2: + if ftype == TType.STRUCT: + self.o2 = InvalidObjectException.read(iprot) + else: + iprot.skip(ftype) + elif fid == 3: + if ftype == TType.STRUCT: + self.o3 = MetaException.read(iprot) + else: + iprot.skip(ftype) + elif fid == 4: + if ftype == TType.STRUCT: + self.o4 = NoSuchObjectException.read(iprot) + else: + iprot.skip(ftype) + else: + iprot.skip(ftype) + iprot.readFieldEnd() + iprot.readStructEnd() + + def write(self, oprot): + if oprot._fast_encode is not None and self.thrift_spec is not None: + oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec])) + return + oprot.writeStructBegin("create_table_req_result") + if self.o1 is not None: + oprot.writeFieldBegin("o1", TType.STRUCT, 1) + self.o1.write(oprot) + oprot.writeFieldEnd() + if self.o2 is not None: + oprot.writeFieldBegin("o2", TType.STRUCT, 2) + self.o2.write(oprot) + oprot.writeFieldEnd() + if self.o3 is not None: + oprot.writeFieldBegin("o3", TType.STRUCT, 3) + self.o3.write(oprot) + oprot.writeFieldEnd() + if self.o4 is not None: + oprot.writeFieldBegin("o4", TType.STRUCT, 4) + self.o4.write(oprot) + oprot.writeFieldEnd() + oprot.writeFieldStop() + oprot.writeStructEnd() + + def validate(self): + return + + def __repr__(self): + L = ["{}={!r}".format(key, value) for key, value in self.__dict__.items()] + return "{}({})".format(self.__class__.__name__, ", ".join(L)) + + def __eq__(self, other): + return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ + + def __ne__(self, other): + return not (self == other) + + +all_structs.append(create_table_req_result) +create_table_req_result.thrift_spec = ( + None, # 0 + ( + 1, + TType.STRUCT, + "o1", + [AlreadyExistsException, None], + None, + ), # 1 + ( + 2, + TType.STRUCT, + "o2", + [InvalidObjectException, None], + None, + ), # 2 + ( + 3, + TType.STRUCT, + "o3", + [MetaException, None], + None, + ), # 3 + ( + 4, + TType.STRUCT, + "o4", + [NoSuchObjectException, None], + None, + ), # 4 +) + + +class drop_constraint_args: + """ + Attributes: + - req + + """ + + def __init__( + self, + req=None, + ): + self.req = req + + def read(self, iprot): + if ( + iprot._fast_decode is not None + and isinstance(iprot.trans, TTransport.CReadableTransport) + and self.thrift_spec is not None + ): + iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec]) + return + iprot.readStructBegin() + while True: + (fname, ftype, fid) = iprot.readFieldBegin() + if ftype == TType.STOP: + break + if fid == 1: + if ftype == TType.STRUCT: + self.req = DropConstraintRequest() + self.req.read(iprot) + else: + iprot.skip(ftype) + else: + iprot.skip(ftype) + iprot.readFieldEnd() + iprot.readStructEnd() + + def write(self, oprot): + if oprot._fast_encode is not None and self.thrift_spec is not None: + oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec])) + return + oprot.writeStructBegin("drop_constraint_args") + if self.req is not None: + oprot.writeFieldBegin("req", TType.STRUCT, 1) + self.req.write(oprot) + oprot.writeFieldEnd() + oprot.writeFieldStop() + oprot.writeStructEnd() + + def validate(self): + return + + def __repr__(self): + L = ["{}={!r}".format(key, value) for key, value in self.__dict__.items()] + return "{}({})".format(self.__class__.__name__, ", ".join(L)) + + def __eq__(self, other): + return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ + + def __ne__(self, other): + return not (self == other) + + +all_structs.append(drop_constraint_args) +drop_constraint_args.thrift_spec = ( + None, # 0 + ( + 1, + TType.STRUCT, + "req", + [DropConstraintRequest, None], + None, + ), # 1 +) + + +class drop_constraint_result: + """ + Attributes: + - o1 + - o3 + + """ + + def __init__( + self, + o1=None, + o3=None, + ): + self.o1 = o1 + self.o3 = o3 + + def read(self, iprot): + if ( + iprot._fast_decode is not None + and isinstance(iprot.trans, TTransport.CReadableTransport) + and self.thrift_spec is not None + ): + iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec]) + return + iprot.readStructBegin() + while True: + (fname, ftype, fid) = iprot.readFieldBegin() + if ftype == TType.STOP: + break + if fid == 1: + if ftype == TType.STRUCT: + self.o1 = NoSuchObjectException.read(iprot) + else: + iprot.skip(ftype) + elif fid == 2: + if ftype == TType.STRUCT: + self.o3 = MetaException.read(iprot) + else: + iprot.skip(ftype) + else: + iprot.skip(ftype) + iprot.readFieldEnd() + iprot.readStructEnd() + + def write(self, oprot): + if oprot._fast_encode is not None and self.thrift_spec is not None: + oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec])) + return + oprot.writeStructBegin("drop_constraint_result") + if self.o1 is not None: + oprot.writeFieldBegin("o1", TType.STRUCT, 1) + self.o1.write(oprot) + oprot.writeFieldEnd() + if self.o3 is not None: + oprot.writeFieldBegin("o3", TType.STRUCT, 2) + self.o3.write(oprot) + oprot.writeFieldEnd() + oprot.writeFieldStop() + oprot.writeStructEnd() + + def validate(self): + return + + def __repr__(self): + L = ["{}={!r}".format(key, value) for key, value in self.__dict__.items()] + return "{}({})".format(self.__class__.__name__, ", ".join(L)) + + def __eq__(self, other): + return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ + + def __ne__(self, other): + return not (self == other) + + +all_structs.append(drop_constraint_result) +drop_constraint_result.thrift_spec = ( + None, # 0 + ( + 1, + TType.STRUCT, + "o1", + [NoSuchObjectException, None], + None, + ), # 1 + ( + 2, + TType.STRUCT, + "o3", + [MetaException, None], + None, + ), # 2 +) + + +class add_primary_key_args: + """ + Attributes: + - req + + """ + + def __init__( + self, + req=None, + ): + self.req = req + + def read(self, iprot): + if ( + iprot._fast_decode is not None + and isinstance(iprot.trans, TTransport.CReadableTransport) + and self.thrift_spec is not None + ): + iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec]) + return + iprot.readStructBegin() + while True: + (fname, ftype, fid) = iprot.readFieldBegin() + if ftype == TType.STOP: + break + if fid == 1: + if ftype == TType.STRUCT: + self.req = AddPrimaryKeyRequest() + self.req.read(iprot) + else: + iprot.skip(ftype) + else: + iprot.skip(ftype) + iprot.readFieldEnd() + iprot.readStructEnd() + + def write(self, oprot): + if oprot._fast_encode is not None and self.thrift_spec is not None: + oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec])) + return + oprot.writeStructBegin("add_primary_key_args") + if self.req is not None: + oprot.writeFieldBegin("req", TType.STRUCT, 1) + self.req.write(oprot) + oprot.writeFieldEnd() + oprot.writeFieldStop() + oprot.writeStructEnd() + + def validate(self): + return + + def __repr__(self): + L = ["{}={!r}".format(key, value) for key, value in self.__dict__.items()] + return "{}({})".format(self.__class__.__name__, ", ".join(L)) + + def __eq__(self, other): + return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ + + def __ne__(self, other): + return not (self == other) + + +all_structs.append(add_primary_key_args) +add_primary_key_args.thrift_spec = ( + None, # 0 + ( + 1, + TType.STRUCT, + "req", + [AddPrimaryKeyRequest, None], + None, + ), # 1 +) + + +class add_primary_key_result: + """ + Attributes: + - o1 + - o2 + + """ + + def __init__( + self, + o1=None, + o2=None, + ): + self.o1 = o1 + self.o2 = o2 + + def read(self, iprot): + if ( + iprot._fast_decode is not None + and isinstance(iprot.trans, TTransport.CReadableTransport) + and self.thrift_spec is not None + ): + iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec]) + return + iprot.readStructBegin() + while True: + (fname, ftype, fid) = iprot.readFieldBegin() + if ftype == TType.STOP: + break + if fid == 1: + if ftype == TType.STRUCT: + self.o1 = NoSuchObjectException.read(iprot) + else: + iprot.skip(ftype) + elif fid == 2: + if ftype == TType.STRUCT: + self.o2 = MetaException.read(iprot) + else: + iprot.skip(ftype) + else: + iprot.skip(ftype) + iprot.readFieldEnd() + iprot.readStructEnd() + + def write(self, oprot): + if oprot._fast_encode is not None and self.thrift_spec is not None: + oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec])) + return + oprot.writeStructBegin("add_primary_key_result") + if self.o1 is not None: + oprot.writeFieldBegin("o1", TType.STRUCT, 1) + self.o1.write(oprot) + oprot.writeFieldEnd() + if self.o2 is not None: + oprot.writeFieldBegin("o2", TType.STRUCT, 2) + self.o2.write(oprot) + oprot.writeFieldEnd() + oprot.writeFieldStop() + oprot.writeStructEnd() + + def validate(self): + return + + def __repr__(self): + L = ["{}={!r}".format(key, value) for key, value in self.__dict__.items()] + return "{}({})".format(self.__class__.__name__, ", ".join(L)) + + def __eq__(self, other): + return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ + + def __ne__(self, other): + return not (self == other) + + +all_structs.append(add_primary_key_result) +add_primary_key_result.thrift_spec = ( + None, # 0 + ( + 1, + TType.STRUCT, + "o1", + [NoSuchObjectException, None], + None, + ), # 1 + ( + 2, + TType.STRUCT, + "o2", + [MetaException, None], + None, + ), # 2 +) + + +class add_foreign_key_args: + """ + Attributes: + - req + + """ + + def __init__( + self, + req=None, + ): + self.req = req + + def read(self, iprot): + if ( + iprot._fast_decode is not None + and isinstance(iprot.trans, TTransport.CReadableTransport) + and self.thrift_spec is not None + ): + iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec]) + return + iprot.readStructBegin() + while True: + (fname, ftype, fid) = iprot.readFieldBegin() + if ftype == TType.STOP: + break + if fid == 1: + if ftype == TType.STRUCT: + self.req = AddForeignKeyRequest() + self.req.read(iprot) + else: + iprot.skip(ftype) + else: + iprot.skip(ftype) + iprot.readFieldEnd() + iprot.readStructEnd() + + def write(self, oprot): + if oprot._fast_encode is not None and self.thrift_spec is not None: + oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec])) + return + oprot.writeStructBegin("add_foreign_key_args") + if self.req is not None: + oprot.writeFieldBegin("req", TType.STRUCT, 1) + self.req.write(oprot) + oprot.writeFieldEnd() + oprot.writeFieldStop() + oprot.writeStructEnd() + + def validate(self): + return + + def __repr__(self): + L = ["{}={!r}".format(key, value) for key, value in self.__dict__.items()] + return "{}({})".format(self.__class__.__name__, ", ".join(L)) + + def __eq__(self, other): + return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ + + def __ne__(self, other): + return not (self == other) + + +all_structs.append(add_foreign_key_args) +add_foreign_key_args.thrift_spec = ( + None, # 0 + ( + 1, + TType.STRUCT, + "req", + [AddForeignKeyRequest, None], + None, + ), # 1 +) + + +class add_foreign_key_result: + """ + Attributes: + - o1 + - o2 + + """ + + def __init__( + self, + o1=None, + o2=None, + ): + self.o1 = o1 + self.o2 = o2 + + def read(self, iprot): + if ( + iprot._fast_decode is not None + and isinstance(iprot.trans, TTransport.CReadableTransport) + and self.thrift_spec is not None + ): + iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec]) + return + iprot.readStructBegin() + while True: + (fname, ftype, fid) = iprot.readFieldBegin() + if ftype == TType.STOP: + break + if fid == 1: + if ftype == TType.STRUCT: + self.o1 = NoSuchObjectException.read(iprot) + else: + iprot.skip(ftype) + elif fid == 2: + if ftype == TType.STRUCT: + self.o2 = MetaException.read(iprot) + else: + iprot.skip(ftype) + else: + iprot.skip(ftype) + iprot.readFieldEnd() + iprot.readStructEnd() + + def write(self, oprot): + if oprot._fast_encode is not None and self.thrift_spec is not None: + oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec])) + return + oprot.writeStructBegin("add_foreign_key_result") + if self.o1 is not None: + oprot.writeFieldBegin("o1", TType.STRUCT, 1) + self.o1.write(oprot) + oprot.writeFieldEnd() + if self.o2 is not None: + oprot.writeFieldBegin("o2", TType.STRUCT, 2) + self.o2.write(oprot) + oprot.writeFieldEnd() + oprot.writeFieldStop() + oprot.writeStructEnd() + + def validate(self): + return + + def __repr__(self): + L = ["{}={!r}".format(key, value) for key, value in self.__dict__.items()] + return "{}({})".format(self.__class__.__name__, ", ".join(L)) + + def __eq__(self, other): + return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ + + def __ne__(self, other): + return not (self == other) + + +all_structs.append(add_foreign_key_result) +add_foreign_key_result.thrift_spec = ( + None, # 0 + ( + 1, + TType.STRUCT, + "o1", + [NoSuchObjectException, None], + None, + ), # 1 + ( + 2, + TType.STRUCT, + "o2", + [MetaException, None], + None, + ), # 2 +) + + +class add_unique_constraint_args: + """ + Attributes: + - req + + """ + + def __init__( + self, + req=None, + ): + self.req = req + + def read(self, iprot): + if ( + iprot._fast_decode is not None + and isinstance(iprot.trans, TTransport.CReadableTransport) + and self.thrift_spec is not None + ): + iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec]) + return + iprot.readStructBegin() + while True: + (fname, ftype, fid) = iprot.readFieldBegin() + if ftype == TType.STOP: + break + if fid == 1: + if ftype == TType.STRUCT: + self.req = AddUniqueConstraintRequest() + self.req.read(iprot) + else: + iprot.skip(ftype) + else: + iprot.skip(ftype) + iprot.readFieldEnd() + iprot.readStructEnd() + + def write(self, oprot): + if oprot._fast_encode is not None and self.thrift_spec is not None: + oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec])) + return + oprot.writeStructBegin("add_unique_constraint_args") + if self.req is not None: + oprot.writeFieldBegin("req", TType.STRUCT, 1) + self.req.write(oprot) + oprot.writeFieldEnd() + oprot.writeFieldStop() + oprot.writeStructEnd() + + def validate(self): + return + + def __repr__(self): + L = ["{}={!r}".format(key, value) for key, value in self.__dict__.items()] + return "{}({})".format(self.__class__.__name__, ", ".join(L)) + + def __eq__(self, other): + return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ + + def __ne__(self, other): + return not (self == other) + + +all_structs.append(add_unique_constraint_args) +add_unique_constraint_args.thrift_spec = ( + None, # 0 + ( + 1, + TType.STRUCT, + "req", + [AddUniqueConstraintRequest, None], + None, + ), # 1 +) + + +class add_unique_constraint_result: + """ + Attributes: + - o1 + - o2 + + """ + + def __init__( + self, + o1=None, + o2=None, + ): + self.o1 = o1 + self.o2 = o2 + + def read(self, iprot): + if ( + iprot._fast_decode is not None + and isinstance(iprot.trans, TTransport.CReadableTransport) + and self.thrift_spec is not None + ): + iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec]) + return + iprot.readStructBegin() + while True: + (fname, ftype, fid) = iprot.readFieldBegin() + if ftype == TType.STOP: + break + if fid == 1: + if ftype == TType.STRUCT: + self.o1 = NoSuchObjectException.read(iprot) + else: + iprot.skip(ftype) + elif fid == 2: + if ftype == TType.STRUCT: + self.o2 = MetaException.read(iprot) + else: + iprot.skip(ftype) + else: + iprot.skip(ftype) + iprot.readFieldEnd() + iprot.readStructEnd() + + def write(self, oprot): + if oprot._fast_encode is not None and self.thrift_spec is not None: + oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec])) + return + oprot.writeStructBegin("add_unique_constraint_result") + if self.o1 is not None: + oprot.writeFieldBegin("o1", TType.STRUCT, 1) + self.o1.write(oprot) + oprot.writeFieldEnd() + if self.o2 is not None: + oprot.writeFieldBegin("o2", TType.STRUCT, 2) + self.o2.write(oprot) + oprot.writeFieldEnd() + oprot.writeFieldStop() + oprot.writeStructEnd() + + def validate(self): + return + + def __repr__(self): + L = ["{}={!r}".format(key, value) for key, value in self.__dict__.items()] + return "{}({})".format(self.__class__.__name__, ", ".join(L)) + + def __eq__(self, other): + return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ + + def __ne__(self, other): + return not (self == other) + + +all_structs.append(add_unique_constraint_result) +add_unique_constraint_result.thrift_spec = ( + None, # 0 + ( + 1, + TType.STRUCT, + "o1", + [NoSuchObjectException, None], + None, + ), # 1 + ( + 2, + TType.STRUCT, + "o2", + [MetaException, None], + None, + ), # 2 +) + + +class add_not_null_constraint_args: + """ + Attributes: + - req + + """ + + def __init__( + self, + req=None, + ): + self.req = req + + def read(self, iprot): + if ( + iprot._fast_decode is not None + and isinstance(iprot.trans, TTransport.CReadableTransport) + and self.thrift_spec is not None + ): + iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec]) + return + iprot.readStructBegin() + while True: + (fname, ftype, fid) = iprot.readFieldBegin() + if ftype == TType.STOP: + break + if fid == 1: + if ftype == TType.STRUCT: + self.req = AddNotNullConstraintRequest() + self.req.read(iprot) + else: + iprot.skip(ftype) + else: + iprot.skip(ftype) + iprot.readFieldEnd() + iprot.readStructEnd() + + def write(self, oprot): + if oprot._fast_encode is not None and self.thrift_spec is not None: + oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec])) + return + oprot.writeStructBegin("add_not_null_constraint_args") + if self.req is not None: + oprot.writeFieldBegin("req", TType.STRUCT, 1) + self.req.write(oprot) + oprot.writeFieldEnd() + oprot.writeFieldStop() + oprot.writeStructEnd() + + def validate(self): + return + + def __repr__(self): + L = ["{}={!r}".format(key, value) for key, value in self.__dict__.items()] + return "{}({})".format(self.__class__.__name__, ", ".join(L)) + + def __eq__(self, other): + return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ + + def __ne__(self, other): + return not (self == other) + + +all_structs.append(add_not_null_constraint_args) +add_not_null_constraint_args.thrift_spec = ( + None, # 0 + ( + 1, + TType.STRUCT, + "req", + [AddNotNullConstraintRequest, None], + None, + ), # 1 +) + + +class add_not_null_constraint_result: + """ + Attributes: + - o1 + - o2 + + """ + + def __init__( + self, + o1=None, + o2=None, + ): + self.o1 = o1 + self.o2 = o2 + + def read(self, iprot): + if ( + iprot._fast_decode is not None + and isinstance(iprot.trans, TTransport.CReadableTransport) + and self.thrift_spec is not None + ): + iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec]) + return + iprot.readStructBegin() + while True: + (fname, ftype, fid) = iprot.readFieldBegin() + if ftype == TType.STOP: + break + if fid == 1: + if ftype == TType.STRUCT: + self.o1 = NoSuchObjectException.read(iprot) + else: + iprot.skip(ftype) + elif fid == 2: + if ftype == TType.STRUCT: + self.o2 = MetaException.read(iprot) + else: + iprot.skip(ftype) + else: + iprot.skip(ftype) + iprot.readFieldEnd() + iprot.readStructEnd() + + def write(self, oprot): + if oprot._fast_encode is not None and self.thrift_spec is not None: + oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec])) + return + oprot.writeStructBegin("add_not_null_constraint_result") + if self.o1 is not None: + oprot.writeFieldBegin("o1", TType.STRUCT, 1) + self.o1.write(oprot) + oprot.writeFieldEnd() + if self.o2 is not None: + oprot.writeFieldBegin("o2", TType.STRUCT, 2) + self.o2.write(oprot) + oprot.writeFieldEnd() + oprot.writeFieldStop() + oprot.writeStructEnd() + + def validate(self): + return + + def __repr__(self): + L = ["{}={!r}".format(key, value) for key, value in self.__dict__.items()] + return "{}({})".format(self.__class__.__name__, ", ".join(L)) + + def __eq__(self, other): + return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ + + def __ne__(self, other): + return not (self == other) + + +all_structs.append(add_not_null_constraint_result) +add_not_null_constraint_result.thrift_spec = ( + None, # 0 + ( + 1, + TType.STRUCT, + "o1", + [NoSuchObjectException, None], + None, + ), # 1 + ( + 2, + TType.STRUCT, + "o2", + [MetaException, None], + None, + ), # 2 +) + + +class add_default_constraint_args: + """ + Attributes: + - req + + """ + + def __init__( + self, + req=None, + ): + self.req = req + + def read(self, iprot): + if ( + iprot._fast_decode is not None + and isinstance(iprot.trans, TTransport.CReadableTransport) + and self.thrift_spec is not None + ): + iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec]) + return + iprot.readStructBegin() + while True: + (fname, ftype, fid) = iprot.readFieldBegin() + if ftype == TType.STOP: + break + if fid == 1: + if ftype == TType.STRUCT: + self.req = AddDefaultConstraintRequest() + self.req.read(iprot) + else: + iprot.skip(ftype) + else: + iprot.skip(ftype) + iprot.readFieldEnd() + iprot.readStructEnd() + + def write(self, oprot): + if oprot._fast_encode is not None and self.thrift_spec is not None: + oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec])) + return + oprot.writeStructBegin("add_default_constraint_args") + if self.req is not None: + oprot.writeFieldBegin("req", TType.STRUCT, 1) + self.req.write(oprot) + oprot.writeFieldEnd() + oprot.writeFieldStop() + oprot.writeStructEnd() + + def validate(self): + return + + def __repr__(self): + L = ["{}={!r}".format(key, value) for key, value in self.__dict__.items()] + return "{}({})".format(self.__class__.__name__, ", ".join(L)) + + def __eq__(self, other): + return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ + + def __ne__(self, other): + return not (self == other) + + +all_structs.append(add_default_constraint_args) +add_default_constraint_args.thrift_spec = ( + None, # 0 + ( + 1, + TType.STRUCT, + "req", + [AddDefaultConstraintRequest, None], + None, + ), # 1 +) + + +class add_default_constraint_result: + """ + Attributes: + - o1 + - o2 + + """ + + def __init__( + self, + o1=None, + o2=None, + ): + self.o1 = o1 + self.o2 = o2 + + def read(self, iprot): + if ( + iprot._fast_decode is not None + and isinstance(iprot.trans, TTransport.CReadableTransport) + and self.thrift_spec is not None + ): + iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec]) + return + iprot.readStructBegin() + while True: + (fname, ftype, fid) = iprot.readFieldBegin() + if ftype == TType.STOP: + break + if fid == 1: + if ftype == TType.STRUCT: + self.o1 = NoSuchObjectException.read(iprot) + else: + iprot.skip(ftype) + elif fid == 2: + if ftype == TType.STRUCT: + self.o2 = MetaException.read(iprot) + else: + iprot.skip(ftype) + else: + iprot.skip(ftype) + iprot.readFieldEnd() + iprot.readStructEnd() + + def write(self, oprot): + if oprot._fast_encode is not None and self.thrift_spec is not None: + oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec])) + return + oprot.writeStructBegin("add_default_constraint_result") + if self.o1 is not None: + oprot.writeFieldBegin("o1", TType.STRUCT, 1) + self.o1.write(oprot) + oprot.writeFieldEnd() + if self.o2 is not None: + oprot.writeFieldBegin("o2", TType.STRUCT, 2) + self.o2.write(oprot) + oprot.writeFieldEnd() + oprot.writeFieldStop() + oprot.writeStructEnd() + + def validate(self): + return + + def __repr__(self): + L = ["{}={!r}".format(key, value) for key, value in self.__dict__.items()] + return "{}({})".format(self.__class__.__name__, ", ".join(L)) + + def __eq__(self, other): + return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ + + def __ne__(self, other): + return not (self == other) + + +all_structs.append(add_default_constraint_result) +add_default_constraint_result.thrift_spec = ( + None, # 0 + ( + 1, + TType.STRUCT, + "o1", + [NoSuchObjectException, None], + None, + ), # 1 + ( + 2, + TType.STRUCT, + "o2", + [MetaException, None], + None, + ), # 2 +) + + +class add_check_constraint_args: + """ + Attributes: + - req + + """ + + def __init__( + self, + req=None, + ): + self.req = req + + def read(self, iprot): + if ( + iprot._fast_decode is not None + and isinstance(iprot.trans, TTransport.CReadableTransport) + and self.thrift_spec is not None + ): + iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec]) + return + iprot.readStructBegin() + while True: + (fname, ftype, fid) = iprot.readFieldBegin() + if ftype == TType.STOP: + break + if fid == 1: + if ftype == TType.STRUCT: + self.req = AddCheckConstraintRequest() + self.req.read(iprot) + else: + iprot.skip(ftype) + else: + iprot.skip(ftype) + iprot.readFieldEnd() + iprot.readStructEnd() + + def write(self, oprot): + if oprot._fast_encode is not None and self.thrift_spec is not None: + oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec])) + return + oprot.writeStructBegin("add_check_constraint_args") + if self.req is not None: + oprot.writeFieldBegin("req", TType.STRUCT, 1) + self.req.write(oprot) + oprot.writeFieldEnd() + oprot.writeFieldStop() + oprot.writeStructEnd() + + def validate(self): + return + + def __repr__(self): + L = ["{}={!r}".format(key, value) for key, value in self.__dict__.items()] + return "{}({})".format(self.__class__.__name__, ", ".join(L)) + + def __eq__(self, other): + return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ + + def __ne__(self, other): + return not (self == other) + + +all_structs.append(add_check_constraint_args) +add_check_constraint_args.thrift_spec = ( + None, # 0 + ( + 1, + TType.STRUCT, + "req", + [AddCheckConstraintRequest, None], + None, + ), # 1 +) + + +class add_check_constraint_result: + """ + Attributes: + - o1 + - o2 + + """ + + def __init__( + self, + o1=None, + o2=None, + ): + self.o1 = o1 + self.o2 = o2 + + def read(self, iprot): + if ( + iprot._fast_decode is not None + and isinstance(iprot.trans, TTransport.CReadableTransport) + and self.thrift_spec is not None + ): + iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec]) + return + iprot.readStructBegin() + while True: + (fname, ftype, fid) = iprot.readFieldBegin() + if ftype == TType.STOP: + break + if fid == 1: + if ftype == TType.STRUCT: + self.o1 = NoSuchObjectException.read(iprot) + else: + iprot.skip(ftype) + elif fid == 2: + if ftype == TType.STRUCT: + self.o2 = MetaException.read(iprot) + else: + iprot.skip(ftype) + else: + iprot.skip(ftype) + iprot.readFieldEnd() + iprot.readStructEnd() + + def write(self, oprot): + if oprot._fast_encode is not None and self.thrift_spec is not None: + oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec])) + return + oprot.writeStructBegin("add_check_constraint_result") + if self.o1 is not None: + oprot.writeFieldBegin("o1", TType.STRUCT, 1) + self.o1.write(oprot) + oprot.writeFieldEnd() + if self.o2 is not None: + oprot.writeFieldBegin("o2", TType.STRUCT, 2) + self.o2.write(oprot) + oprot.writeFieldEnd() + oprot.writeFieldStop() + oprot.writeStructEnd() + + def validate(self): + return + + def __repr__(self): + L = ["{}={!r}".format(key, value) for key, value in self.__dict__.items()] + return "{}({})".format(self.__class__.__name__, ", ".join(L)) + + def __eq__(self, other): + return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ + + def __ne__(self, other): + return not (self == other) + + +all_structs.append(add_check_constraint_result) +add_check_constraint_result.thrift_spec = ( + None, # 0 + ( + 1, + TType.STRUCT, + "o1", + [NoSuchObjectException, None], + None, + ), # 1 + ( + 2, + TType.STRUCT, + "o2", + [MetaException, None], + None, + ), # 2 +) + + +class translate_table_dryrun_args: + """ + Attributes: + - request + + """ + + def __init__( + self, + request=None, + ): + self.request = request + + def read(self, iprot): + if ( + iprot._fast_decode is not None + and isinstance(iprot.trans, TTransport.CReadableTransport) + and self.thrift_spec is not None + ): + iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec]) + return + iprot.readStructBegin() + while True: + (fname, ftype, fid) = iprot.readFieldBegin() + if ftype == TType.STOP: + break + if fid == 1: + if ftype == TType.STRUCT: + self.request = CreateTableRequest() + self.request.read(iprot) + else: + iprot.skip(ftype) + else: + iprot.skip(ftype) + iprot.readFieldEnd() + iprot.readStructEnd() + + def write(self, oprot): + if oprot._fast_encode is not None and self.thrift_spec is not None: + oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec])) + return + oprot.writeStructBegin("translate_table_dryrun_args") + if self.request is not None: + oprot.writeFieldBegin("request", TType.STRUCT, 1) + self.request.write(oprot) + oprot.writeFieldEnd() + oprot.writeFieldStop() + oprot.writeStructEnd() + + def validate(self): + return + + def __repr__(self): + L = ["{}={!r}".format(key, value) for key, value in self.__dict__.items()] + return "{}({})".format(self.__class__.__name__, ", ".join(L)) + + def __eq__(self, other): + return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ + + def __ne__(self, other): + return not (self == other) + + +all_structs.append(translate_table_dryrun_args) +translate_table_dryrun_args.thrift_spec = ( + None, # 0 + ( + 1, + TType.STRUCT, + "request", + [CreateTableRequest, None], + None, + ), # 1 +) + + +class translate_table_dryrun_result: + """ + Attributes: + - success + - o1 + - o2 + - o3 + - o4 + + """ + + def __init__( + self, + success=None, + o1=None, + o2=None, + o3=None, + o4=None, + ): + self.success = success + self.o1 = o1 + self.o2 = o2 + self.o3 = o3 + self.o4 = o4 + + def read(self, iprot): + if ( + iprot._fast_decode is not None + and isinstance(iprot.trans, TTransport.CReadableTransport) + and self.thrift_spec is not None + ): + iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec]) + return + iprot.readStructBegin() + while True: + (fname, ftype, fid) = iprot.readFieldBegin() + if ftype == TType.STOP: + break + if fid == 0: + if ftype == TType.STRUCT: + self.success = Table() + self.success.read(iprot) + else: + iprot.skip(ftype) + elif fid == 1: + if ftype == TType.STRUCT: + self.o1 = AlreadyExistsException.read(iprot) + else: + iprot.skip(ftype) + elif fid == 2: + if ftype == TType.STRUCT: + self.o2 = InvalidObjectException.read(iprot) + else: + iprot.skip(ftype) + elif fid == 3: + if ftype == TType.STRUCT: + self.o3 = MetaException.read(iprot) + else: + iprot.skip(ftype) + elif fid == 4: + if ftype == TType.STRUCT: + self.o4 = NoSuchObjectException.read(iprot) + else: + iprot.skip(ftype) + else: + iprot.skip(ftype) + iprot.readFieldEnd() + iprot.readStructEnd() + + def write(self, oprot): + if oprot._fast_encode is not None and self.thrift_spec is not None: + oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec])) + return + oprot.writeStructBegin("translate_table_dryrun_result") + if self.success is not None: + oprot.writeFieldBegin("success", TType.STRUCT, 0) + self.success.write(oprot) + oprot.writeFieldEnd() + if self.o1 is not None: + oprot.writeFieldBegin("o1", TType.STRUCT, 1) + self.o1.write(oprot) + oprot.writeFieldEnd() + if self.o2 is not None: + oprot.writeFieldBegin("o2", TType.STRUCT, 2) + self.o2.write(oprot) + oprot.writeFieldEnd() + if self.o3 is not None: + oprot.writeFieldBegin("o3", TType.STRUCT, 3) + self.o3.write(oprot) + oprot.writeFieldEnd() + if self.o4 is not None: + oprot.writeFieldBegin("o4", TType.STRUCT, 4) + self.o4.write(oprot) + oprot.writeFieldEnd() + oprot.writeFieldStop() + oprot.writeStructEnd() + + def validate(self): + return + + def __repr__(self): + L = ["{}={!r}".format(key, value) for key, value in self.__dict__.items()] + return "{}({})".format(self.__class__.__name__, ", ".join(L)) + + def __eq__(self, other): + return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ + + def __ne__(self, other): + return not (self == other) + + +all_structs.append(translate_table_dryrun_result) +translate_table_dryrun_result.thrift_spec = ( + ( + 0, + TType.STRUCT, + "success", + [Table, None], + None, + ), # 0 + ( + 1, + TType.STRUCT, + "o1", + [AlreadyExistsException, None], + None, + ), # 1 + ( + 2, + TType.STRUCT, + "o2", + [InvalidObjectException, None], + None, + ), # 2 + ( + 3, + TType.STRUCT, + "o3", + [MetaException, None], + None, + ), # 3 + ( + 4, + TType.STRUCT, + "o4", + [NoSuchObjectException, None], + None, + ), # 4 +) + + +class drop_table_args: + """ + Attributes: + - dbname + - name + - deleteData + + """ + + def __init__( + self, + dbname=None, + name=None, + deleteData=None, + ): + self.dbname = dbname + self.name = name + self.deleteData = deleteData + + def read(self, iprot): + if ( + iprot._fast_decode is not None + and isinstance(iprot.trans, TTransport.CReadableTransport) + and self.thrift_spec is not None + ): + iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec]) + return + iprot.readStructBegin() + while True: + (fname, ftype, fid) = iprot.readFieldBegin() + if ftype == TType.STOP: + break + if fid == 1: + if ftype == TType.STRING: + self.dbname = ( + iprot.readString().decode("utf-8", errors="replace") if sys.version_info[0] == 2 else iprot.readString() + ) + else: + iprot.skip(ftype) + elif fid == 2: + if ftype == TType.STRING: + self.name = ( + iprot.readString().decode("utf-8", errors="replace") if sys.version_info[0] == 2 else iprot.readString() + ) + else: + iprot.skip(ftype) + elif fid == 3: + if ftype == TType.BOOL: + self.deleteData = iprot.readBool() + else: + iprot.skip(ftype) + else: + iprot.skip(ftype) + iprot.readFieldEnd() + iprot.readStructEnd() + + def write(self, oprot): + if oprot._fast_encode is not None and self.thrift_spec is not None: + oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec])) + return + oprot.writeStructBegin("drop_table_args") + if self.dbname is not None: + oprot.writeFieldBegin("dbname", TType.STRING, 1) + oprot.writeString(self.dbname.encode("utf-8") if sys.version_info[0] == 2 else self.dbname) + oprot.writeFieldEnd() + if self.name is not None: + oprot.writeFieldBegin("name", TType.STRING, 2) + oprot.writeString(self.name.encode("utf-8") if sys.version_info[0] == 2 else self.name) + oprot.writeFieldEnd() + if self.deleteData is not None: + oprot.writeFieldBegin("deleteData", TType.BOOL, 3) + oprot.writeBool(self.deleteData) + oprot.writeFieldEnd() + oprot.writeFieldStop() + oprot.writeStructEnd() + + def validate(self): + return + + def __repr__(self): + L = ["{}={!r}".format(key, value) for key, value in self.__dict__.items()] + return "{}({})".format(self.__class__.__name__, ", ".join(L)) + + def __eq__(self, other): + return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ + + def __ne__(self, other): + return not (self == other) + + +all_structs.append(drop_table_args) +drop_table_args.thrift_spec = ( + None, # 0 + ( + 1, + TType.STRING, + "dbname", + "UTF8", + None, + ), # 1 + ( + 2, + TType.STRING, + "name", + "UTF8", + None, + ), # 2 + ( + 3, + TType.BOOL, + "deleteData", + None, + None, + ), # 3 +) + + +class drop_table_result: + """ + Attributes: + - o1 + - o3 + + """ + + def __init__( + self, + o1=None, + o3=None, + ): + self.o1 = o1 + self.o3 = o3 + + def read(self, iprot): + if ( + iprot._fast_decode is not None + and isinstance(iprot.trans, TTransport.CReadableTransport) + and self.thrift_spec is not None + ): + iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec]) + return + iprot.readStructBegin() + while True: + (fname, ftype, fid) = iprot.readFieldBegin() + if ftype == TType.STOP: + break + if fid == 1: + if ftype == TType.STRUCT: + self.o1 = NoSuchObjectException.read(iprot) + else: + iprot.skip(ftype) + elif fid == 2: + if ftype == TType.STRUCT: + self.o3 = MetaException.read(iprot) + else: + iprot.skip(ftype) + else: + iprot.skip(ftype) + iprot.readFieldEnd() + iprot.readStructEnd() + + def write(self, oprot): + if oprot._fast_encode is not None and self.thrift_spec is not None: + oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec])) + return + oprot.writeStructBegin("drop_table_result") + if self.o1 is not None: + oprot.writeFieldBegin("o1", TType.STRUCT, 1) + self.o1.write(oprot) + oprot.writeFieldEnd() + if self.o3 is not None: + oprot.writeFieldBegin("o3", TType.STRUCT, 2) + self.o3.write(oprot) + oprot.writeFieldEnd() + oprot.writeFieldStop() + oprot.writeStructEnd() + + def validate(self): + return + + def __repr__(self): + L = ["{}={!r}".format(key, value) for key, value in self.__dict__.items()] + return "{}({})".format(self.__class__.__name__, ", ".join(L)) + + def __eq__(self, other): + return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ + + def __ne__(self, other): + return not (self == other) + + +all_structs.append(drop_table_result) +drop_table_result.thrift_spec = ( + None, # 0 + ( + 1, + TType.STRUCT, + "o1", + [NoSuchObjectException, None], + None, + ), # 1 + ( + 2, + TType.STRUCT, + "o3", + [MetaException, None], + None, + ), # 2 +) + + +class drop_table_with_environment_context_args: + """ + Attributes: + - dbname + - name + - deleteData + - environment_context + + """ + + def __init__( + self, + dbname=None, + name=None, + deleteData=None, + environment_context=None, + ): + self.dbname = dbname + self.name = name + self.deleteData = deleteData + self.environment_context = environment_context + + def read(self, iprot): + if ( + iprot._fast_decode is not None + and isinstance(iprot.trans, TTransport.CReadableTransport) + and self.thrift_spec is not None + ): + iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec]) + return + iprot.readStructBegin() + while True: + (fname, ftype, fid) = iprot.readFieldBegin() + if ftype == TType.STOP: + break + if fid == 1: + if ftype == TType.STRING: + self.dbname = ( + iprot.readString().decode("utf-8", errors="replace") if sys.version_info[0] == 2 else iprot.readString() + ) + else: + iprot.skip(ftype) + elif fid == 2: + if ftype == TType.STRING: + self.name = ( + iprot.readString().decode("utf-8", errors="replace") if sys.version_info[0] == 2 else iprot.readString() + ) + else: + iprot.skip(ftype) + elif fid == 3: + if ftype == TType.BOOL: + self.deleteData = iprot.readBool() + else: + iprot.skip(ftype) + elif fid == 4: + if ftype == TType.STRUCT: + self.environment_context = EnvironmentContext() + self.environment_context.read(iprot) + else: + iprot.skip(ftype) + else: + iprot.skip(ftype) + iprot.readFieldEnd() + iprot.readStructEnd() + + def write(self, oprot): + if oprot._fast_encode is not None and self.thrift_spec is not None: + oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec])) + return + oprot.writeStructBegin("drop_table_with_environment_context_args") + if self.dbname is not None: + oprot.writeFieldBegin("dbname", TType.STRING, 1) + oprot.writeString(self.dbname.encode("utf-8") if sys.version_info[0] == 2 else self.dbname) + oprot.writeFieldEnd() + if self.name is not None: + oprot.writeFieldBegin("name", TType.STRING, 2) + oprot.writeString(self.name.encode("utf-8") if sys.version_info[0] == 2 else self.name) + oprot.writeFieldEnd() + if self.deleteData is not None: + oprot.writeFieldBegin("deleteData", TType.BOOL, 3) + oprot.writeBool(self.deleteData) + oprot.writeFieldEnd() + if self.environment_context is not None: + oprot.writeFieldBegin("environment_context", TType.STRUCT, 4) + self.environment_context.write(oprot) + oprot.writeFieldEnd() + oprot.writeFieldStop() + oprot.writeStructEnd() + + def validate(self): + return + + def __repr__(self): + L = ["{}={!r}".format(key, value) for key, value in self.__dict__.items()] + return "{}({})".format(self.__class__.__name__, ", ".join(L)) + + def __eq__(self, other): + return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ + + def __ne__(self, other): + return not (self == other) + + +all_structs.append(drop_table_with_environment_context_args) +drop_table_with_environment_context_args.thrift_spec = ( + None, # 0 + ( + 1, + TType.STRING, + "dbname", + "UTF8", + None, + ), # 1 + ( + 2, + TType.STRING, + "name", + "UTF8", + None, + ), # 2 + ( + 3, + TType.BOOL, + "deleteData", + None, + None, + ), # 3 + ( + 4, + TType.STRUCT, + "environment_context", + [EnvironmentContext, None], + None, + ), # 4 +) + + +class drop_table_with_environment_context_result: + """ + Attributes: + - o1 + - o3 + + """ + + def __init__( + self, + o1=None, + o3=None, + ): + self.o1 = o1 + self.o3 = o3 + + def read(self, iprot): + if ( + iprot._fast_decode is not None + and isinstance(iprot.trans, TTransport.CReadableTransport) + and self.thrift_spec is not None + ): + iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec]) + return + iprot.readStructBegin() + while True: + (fname, ftype, fid) = iprot.readFieldBegin() + if ftype == TType.STOP: + break + if fid == 1: + if ftype == TType.STRUCT: + self.o1 = NoSuchObjectException.read(iprot) + else: + iprot.skip(ftype) + elif fid == 2: + if ftype == TType.STRUCT: + self.o3 = MetaException.read(iprot) + else: + iprot.skip(ftype) + else: + iprot.skip(ftype) + iprot.readFieldEnd() + iprot.readStructEnd() + + def write(self, oprot): + if oprot._fast_encode is not None and self.thrift_spec is not None: + oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec])) + return + oprot.writeStructBegin("drop_table_with_environment_context_result") + if self.o1 is not None: + oprot.writeFieldBegin("o1", TType.STRUCT, 1) + self.o1.write(oprot) + oprot.writeFieldEnd() + if self.o3 is not None: + oprot.writeFieldBegin("o3", TType.STRUCT, 2) + self.o3.write(oprot) + oprot.writeFieldEnd() + oprot.writeFieldStop() + oprot.writeStructEnd() + + def validate(self): + return + + def __repr__(self): + L = ["{}={!r}".format(key, value) for key, value in self.__dict__.items()] + return "{}({})".format(self.__class__.__name__, ", ".join(L)) + + def __eq__(self, other): + return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ + + def __ne__(self, other): + return not (self == other) + + +all_structs.append(drop_table_with_environment_context_result) +drop_table_with_environment_context_result.thrift_spec = ( + None, # 0 + ( + 1, + TType.STRUCT, + "o1", + [NoSuchObjectException, None], + None, + ), # 1 + ( + 2, + TType.STRUCT, + "o3", + [MetaException, None], + None, + ), # 2 +) + + +class truncate_table_args: + """ + Attributes: + - dbName + - tableName + - partNames + + """ + + def __init__( + self, + dbName=None, + tableName=None, + partNames=None, + ): + self.dbName = dbName + self.tableName = tableName + self.partNames = partNames + + def read(self, iprot): + if ( + iprot._fast_decode is not None + and isinstance(iprot.trans, TTransport.CReadableTransport) + and self.thrift_spec is not None + ): + iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec]) + return + iprot.readStructBegin() + while True: + (fname, ftype, fid) = iprot.readFieldBegin() + if ftype == TType.STOP: + break + if fid == 1: + if ftype == TType.STRING: + self.dbName = ( + iprot.readString().decode("utf-8", errors="replace") if sys.version_info[0] == 2 else iprot.readString() + ) + else: + iprot.skip(ftype) + elif fid == 2: + if ftype == TType.STRING: + self.tableName = ( + iprot.readString().decode("utf-8", errors="replace") if sys.version_info[0] == 2 else iprot.readString() + ) + else: + iprot.skip(ftype) + elif fid == 3: + if ftype == TType.LIST: + self.partNames = [] + (_etype1352, _size1349) = iprot.readListBegin() + for _i1353 in range(_size1349): + _elem1354 = ( + iprot.readString().decode("utf-8", errors="replace") + if sys.version_info[0] == 2 + else iprot.readString() + ) + self.partNames.append(_elem1354) + iprot.readListEnd() + else: + iprot.skip(ftype) + else: + iprot.skip(ftype) + iprot.readFieldEnd() + iprot.readStructEnd() + + def write(self, oprot): + if oprot._fast_encode is not None and self.thrift_spec is not None: + oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec])) + return + oprot.writeStructBegin("truncate_table_args") + if self.dbName is not None: + oprot.writeFieldBegin("dbName", TType.STRING, 1) + oprot.writeString(self.dbName.encode("utf-8") if sys.version_info[0] == 2 else self.dbName) + oprot.writeFieldEnd() + if self.tableName is not None: + oprot.writeFieldBegin("tableName", TType.STRING, 2) + oprot.writeString(self.tableName.encode("utf-8") if sys.version_info[0] == 2 else self.tableName) + oprot.writeFieldEnd() + if self.partNames is not None: + oprot.writeFieldBegin("partNames", TType.LIST, 3) + oprot.writeListBegin(TType.STRING, len(self.partNames)) + for iter1355 in self.partNames: + oprot.writeString(iter1355.encode("utf-8") if sys.version_info[0] == 2 else iter1355) + oprot.writeListEnd() + oprot.writeFieldEnd() + oprot.writeFieldStop() + oprot.writeStructEnd() + + def validate(self): + return + + def __repr__(self): + L = ["{}={!r}".format(key, value) for key, value in self.__dict__.items()] + return "{}({})".format(self.__class__.__name__, ", ".join(L)) + + def __eq__(self, other): + return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ + + def __ne__(self, other): + return not (self == other) + + +all_structs.append(truncate_table_args) +truncate_table_args.thrift_spec = ( + None, # 0 + ( + 1, + TType.STRING, + "dbName", + "UTF8", + None, + ), # 1 + ( + 2, + TType.STRING, + "tableName", + "UTF8", + None, + ), # 2 + ( + 3, + TType.LIST, + "partNames", + (TType.STRING, "UTF8", False), + None, + ), # 3 +) + + +class truncate_table_result: + """ + Attributes: + - o1 + + """ + + def __init__( + self, + o1=None, + ): + self.o1 = o1 + + def read(self, iprot): + if ( + iprot._fast_decode is not None + and isinstance(iprot.trans, TTransport.CReadableTransport) + and self.thrift_spec is not None + ): + iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec]) + return + iprot.readStructBegin() + while True: + (fname, ftype, fid) = iprot.readFieldBegin() + if ftype == TType.STOP: + break + if fid == 1: + if ftype == TType.STRUCT: + self.o1 = MetaException.read(iprot) + else: + iprot.skip(ftype) + else: + iprot.skip(ftype) + iprot.readFieldEnd() + iprot.readStructEnd() + + def write(self, oprot): + if oprot._fast_encode is not None and self.thrift_spec is not None: + oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec])) + return + oprot.writeStructBegin("truncate_table_result") + if self.o1 is not None: + oprot.writeFieldBegin("o1", TType.STRUCT, 1) + self.o1.write(oprot) + oprot.writeFieldEnd() + oprot.writeFieldStop() + oprot.writeStructEnd() + + def validate(self): + return + + def __repr__(self): + L = ["{}={!r}".format(key, value) for key, value in self.__dict__.items()] + return "{}({})".format(self.__class__.__name__, ", ".join(L)) + + def __eq__(self, other): + return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ + + def __ne__(self, other): + return not (self == other) + + +all_structs.append(truncate_table_result) +truncate_table_result.thrift_spec = ( + None, # 0 + ( + 1, + TType.STRUCT, + "o1", + [MetaException, None], + None, + ), # 1 +) + + +class truncate_table_req_args: + """ + Attributes: + - req + + """ + + def __init__( + self, + req=None, + ): + self.req = req + + def read(self, iprot): + if ( + iprot._fast_decode is not None + and isinstance(iprot.trans, TTransport.CReadableTransport) + and self.thrift_spec is not None + ): + iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec]) + return + iprot.readStructBegin() + while True: + (fname, ftype, fid) = iprot.readFieldBegin() + if ftype == TType.STOP: + break + if fid == 1: + if ftype == TType.STRUCT: + self.req = TruncateTableRequest() + self.req.read(iprot) + else: + iprot.skip(ftype) + else: + iprot.skip(ftype) + iprot.readFieldEnd() + iprot.readStructEnd() + + def write(self, oprot): + if oprot._fast_encode is not None and self.thrift_spec is not None: + oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec])) + return + oprot.writeStructBegin("truncate_table_req_args") + if self.req is not None: + oprot.writeFieldBegin("req", TType.STRUCT, 1) + self.req.write(oprot) + oprot.writeFieldEnd() + oprot.writeFieldStop() + oprot.writeStructEnd() + + def validate(self): + return + + def __repr__(self): + L = ["{}={!r}".format(key, value) for key, value in self.__dict__.items()] + return "{}({})".format(self.__class__.__name__, ", ".join(L)) + + def __eq__(self, other): + return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ + + def __ne__(self, other): + return not (self == other) + + +all_structs.append(truncate_table_req_args) +truncate_table_req_args.thrift_spec = ( + None, # 0 + ( + 1, + TType.STRUCT, + "req", + [TruncateTableRequest, None], + None, + ), # 1 +) + + +class truncate_table_req_result: + """ + Attributes: + - success + - o1 + + """ + + def __init__( + self, + success=None, + o1=None, + ): + self.success = success + self.o1 = o1 + + def read(self, iprot): + if ( + iprot._fast_decode is not None + and isinstance(iprot.trans, TTransport.CReadableTransport) + and self.thrift_spec is not None + ): + iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec]) + return + iprot.readStructBegin() + while True: + (fname, ftype, fid) = iprot.readFieldBegin() + if ftype == TType.STOP: + break + if fid == 0: + if ftype == TType.STRUCT: + self.success = TruncateTableResponse() + self.success.read(iprot) + else: + iprot.skip(ftype) + elif fid == 1: + if ftype == TType.STRUCT: + self.o1 = MetaException.read(iprot) + else: + iprot.skip(ftype) + else: + iprot.skip(ftype) + iprot.readFieldEnd() + iprot.readStructEnd() + + def write(self, oprot): + if oprot._fast_encode is not None and self.thrift_spec is not None: + oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec])) + return + oprot.writeStructBegin("truncate_table_req_result") + if self.success is not None: + oprot.writeFieldBegin("success", TType.STRUCT, 0) + self.success.write(oprot) + oprot.writeFieldEnd() + if self.o1 is not None: + oprot.writeFieldBegin("o1", TType.STRUCT, 1) + self.o1.write(oprot) + oprot.writeFieldEnd() + oprot.writeFieldStop() + oprot.writeStructEnd() + + def validate(self): + return + + def __repr__(self): + L = ["{}={!r}".format(key, value) for key, value in self.__dict__.items()] + return "{}({})".format(self.__class__.__name__, ", ".join(L)) + + def __eq__(self, other): + return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ + + def __ne__(self, other): + return not (self == other) + + +all_structs.append(truncate_table_req_result) +truncate_table_req_result.thrift_spec = ( + ( + 0, + TType.STRUCT, + "success", + [TruncateTableResponse, None], + None, + ), # 0 + ( + 1, + TType.STRUCT, + "o1", + [MetaException, None], + None, + ), # 1 +) + + +class get_tables_args: + """ + Attributes: + - db_name + - pattern + + """ + + def __init__( + self, + db_name=None, + pattern=None, + ): + self.db_name = db_name + self.pattern = pattern + + def read(self, iprot): + if ( + iprot._fast_decode is not None + and isinstance(iprot.trans, TTransport.CReadableTransport) + and self.thrift_spec is not None + ): + iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec]) + return + iprot.readStructBegin() + while True: + (fname, ftype, fid) = iprot.readFieldBegin() + if ftype == TType.STOP: + break + if fid == 1: + if ftype == TType.STRING: + self.db_name = ( + iprot.readString().decode("utf-8", errors="replace") if sys.version_info[0] == 2 else iprot.readString() + ) + else: + iprot.skip(ftype) + elif fid == 2: + if ftype == TType.STRING: + self.pattern = ( + iprot.readString().decode("utf-8", errors="replace") if sys.version_info[0] == 2 else iprot.readString() + ) + else: + iprot.skip(ftype) + else: + iprot.skip(ftype) + iprot.readFieldEnd() + iprot.readStructEnd() + + def write(self, oprot): + if oprot._fast_encode is not None and self.thrift_spec is not None: + oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec])) + return + oprot.writeStructBegin("get_tables_args") + if self.db_name is not None: + oprot.writeFieldBegin("db_name", TType.STRING, 1) + oprot.writeString(self.db_name.encode("utf-8") if sys.version_info[0] == 2 else self.db_name) + oprot.writeFieldEnd() + if self.pattern is not None: + oprot.writeFieldBegin("pattern", TType.STRING, 2) + oprot.writeString(self.pattern.encode("utf-8") if sys.version_info[0] == 2 else self.pattern) + oprot.writeFieldEnd() + oprot.writeFieldStop() + oprot.writeStructEnd() + + def validate(self): + return + + def __repr__(self): + L = ["{}={!r}".format(key, value) for key, value in self.__dict__.items()] + return "{}({})".format(self.__class__.__name__, ", ".join(L)) + + def __eq__(self, other): + return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ + + def __ne__(self, other): + return not (self == other) + + +all_structs.append(get_tables_args) +get_tables_args.thrift_spec = ( + None, # 0 + ( + 1, + TType.STRING, + "db_name", + "UTF8", + None, + ), # 1 + ( + 2, + TType.STRING, + "pattern", + "UTF8", + None, + ), # 2 +) + + +class get_tables_result: + """ + Attributes: + - success + - o1 + + """ + + def __init__( + self, + success=None, + o1=None, + ): + self.success = success + self.o1 = o1 + + def read(self, iprot): + if ( + iprot._fast_decode is not None + and isinstance(iprot.trans, TTransport.CReadableTransport) + and self.thrift_spec is not None + ): + iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec]) + return + iprot.readStructBegin() + while True: + (fname, ftype, fid) = iprot.readFieldBegin() + if ftype == TType.STOP: + break + if fid == 0: + if ftype == TType.LIST: + self.success = [] + (_etype1359, _size1356) = iprot.readListBegin() + for _i1360 in range(_size1356): + _elem1361 = ( + iprot.readString().decode("utf-8", errors="replace") + if sys.version_info[0] == 2 + else iprot.readString() + ) + self.success.append(_elem1361) + iprot.readListEnd() + else: + iprot.skip(ftype) + elif fid == 1: + if ftype == TType.STRUCT: + self.o1 = MetaException.read(iprot) + else: + iprot.skip(ftype) + else: + iprot.skip(ftype) + iprot.readFieldEnd() + iprot.readStructEnd() + + def write(self, oprot): + if oprot._fast_encode is not None and self.thrift_spec is not None: + oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec])) + return + oprot.writeStructBegin("get_tables_result") + if self.success is not None: + oprot.writeFieldBegin("success", TType.LIST, 0) + oprot.writeListBegin(TType.STRING, len(self.success)) + for iter1362 in self.success: + oprot.writeString(iter1362.encode("utf-8") if sys.version_info[0] == 2 else iter1362) + oprot.writeListEnd() + oprot.writeFieldEnd() + if self.o1 is not None: + oprot.writeFieldBegin("o1", TType.STRUCT, 1) + self.o1.write(oprot) + oprot.writeFieldEnd() + oprot.writeFieldStop() + oprot.writeStructEnd() + + def validate(self): + return + + def __repr__(self): + L = ["{}={!r}".format(key, value) for key, value in self.__dict__.items()] + return "{}({})".format(self.__class__.__name__, ", ".join(L)) + + def __eq__(self, other): + return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ + + def __ne__(self, other): + return not (self == other) + + +all_structs.append(get_tables_result) +get_tables_result.thrift_spec = ( + ( + 0, + TType.LIST, + "success", + (TType.STRING, "UTF8", False), + None, + ), # 0 + ( + 1, + TType.STRUCT, + "o1", + [MetaException, None], + None, + ), # 1 +) + + +class get_tables_by_type_args: + """ + Attributes: + - db_name + - pattern + - tableType + + """ + + def __init__( + self, + db_name=None, + pattern=None, + tableType=None, + ): + self.db_name = db_name + self.pattern = pattern + self.tableType = tableType + + def read(self, iprot): + if ( + iprot._fast_decode is not None + and isinstance(iprot.trans, TTransport.CReadableTransport) + and self.thrift_spec is not None + ): + iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec]) + return + iprot.readStructBegin() + while True: + (fname, ftype, fid) = iprot.readFieldBegin() + if ftype == TType.STOP: + break + if fid == 1: + if ftype == TType.STRING: + self.db_name = ( + iprot.readString().decode("utf-8", errors="replace") if sys.version_info[0] == 2 else iprot.readString() + ) + else: + iprot.skip(ftype) + elif fid == 2: + if ftype == TType.STRING: + self.pattern = ( + iprot.readString().decode("utf-8", errors="replace") if sys.version_info[0] == 2 else iprot.readString() + ) + else: + iprot.skip(ftype) + elif fid == 3: + if ftype == TType.STRING: + self.tableType = ( + iprot.readString().decode("utf-8", errors="replace") if sys.version_info[0] == 2 else iprot.readString() + ) + else: + iprot.skip(ftype) + else: + iprot.skip(ftype) + iprot.readFieldEnd() + iprot.readStructEnd() + + def write(self, oprot): + if oprot._fast_encode is not None and self.thrift_spec is not None: + oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec])) + return + oprot.writeStructBegin("get_tables_by_type_args") + if self.db_name is not None: + oprot.writeFieldBegin("db_name", TType.STRING, 1) + oprot.writeString(self.db_name.encode("utf-8") if sys.version_info[0] == 2 else self.db_name) + oprot.writeFieldEnd() + if self.pattern is not None: + oprot.writeFieldBegin("pattern", TType.STRING, 2) + oprot.writeString(self.pattern.encode("utf-8") if sys.version_info[0] == 2 else self.pattern) + oprot.writeFieldEnd() + if self.tableType is not None: + oprot.writeFieldBegin("tableType", TType.STRING, 3) + oprot.writeString(self.tableType.encode("utf-8") if sys.version_info[0] == 2 else self.tableType) + oprot.writeFieldEnd() + oprot.writeFieldStop() + oprot.writeStructEnd() + + def validate(self): + return + + def __repr__(self): + L = ["{}={!r}".format(key, value) for key, value in self.__dict__.items()] + return "{}({})".format(self.__class__.__name__, ", ".join(L)) + + def __eq__(self, other): + return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ + + def __ne__(self, other): + return not (self == other) + + +all_structs.append(get_tables_by_type_args) +get_tables_by_type_args.thrift_spec = ( + None, # 0 + ( + 1, + TType.STRING, + "db_name", + "UTF8", + None, + ), # 1 + ( + 2, + TType.STRING, + "pattern", + "UTF8", + None, + ), # 2 + ( + 3, + TType.STRING, + "tableType", + "UTF8", + None, + ), # 3 +) + + +class get_tables_by_type_result: + """ + Attributes: + - success + - o1 + + """ + + def __init__( + self, + success=None, + o1=None, + ): + self.success = success + self.o1 = o1 + + def read(self, iprot): + if ( + iprot._fast_decode is not None + and isinstance(iprot.trans, TTransport.CReadableTransport) + and self.thrift_spec is not None + ): + iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec]) + return + iprot.readStructBegin() + while True: + (fname, ftype, fid) = iprot.readFieldBegin() + if ftype == TType.STOP: + break + if fid == 0: + if ftype == TType.LIST: + self.success = [] + (_etype1366, _size1363) = iprot.readListBegin() + for _i1367 in range(_size1363): + _elem1368 = ( + iprot.readString().decode("utf-8", errors="replace") + if sys.version_info[0] == 2 + else iprot.readString() + ) + self.success.append(_elem1368) + iprot.readListEnd() + else: + iprot.skip(ftype) + elif fid == 1: + if ftype == TType.STRUCT: + self.o1 = MetaException.read(iprot) + else: + iprot.skip(ftype) + else: + iprot.skip(ftype) + iprot.readFieldEnd() + iprot.readStructEnd() + + def write(self, oprot): + if oprot._fast_encode is not None and self.thrift_spec is not None: + oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec])) + return + oprot.writeStructBegin("get_tables_by_type_result") + if self.success is not None: + oprot.writeFieldBegin("success", TType.LIST, 0) + oprot.writeListBegin(TType.STRING, len(self.success)) + for iter1369 in self.success: + oprot.writeString(iter1369.encode("utf-8") if sys.version_info[0] == 2 else iter1369) + oprot.writeListEnd() + oprot.writeFieldEnd() + if self.o1 is not None: + oprot.writeFieldBegin("o1", TType.STRUCT, 1) + self.o1.write(oprot) + oprot.writeFieldEnd() + oprot.writeFieldStop() + oprot.writeStructEnd() + + def validate(self): + return + + def __repr__(self): + L = ["{}={!r}".format(key, value) for key, value in self.__dict__.items()] + return "{}({})".format(self.__class__.__name__, ", ".join(L)) + + def __eq__(self, other): + return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ + + def __ne__(self, other): + return not (self == other) + + +all_structs.append(get_tables_by_type_result) +get_tables_by_type_result.thrift_spec = ( + ( + 0, + TType.LIST, + "success", + (TType.STRING, "UTF8", False), + None, + ), # 0 + ( + 1, + TType.STRUCT, + "o1", + [MetaException, None], + None, + ), # 1 +) + + +class get_all_materialized_view_objects_for_rewriting_args: + def read(self, iprot): + if ( + iprot._fast_decode is not None + and isinstance(iprot.trans, TTransport.CReadableTransport) + and self.thrift_spec is not None + ): + iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec]) + return + iprot.readStructBegin() + while True: + (fname, ftype, fid) = iprot.readFieldBegin() + if ftype == TType.STOP: + break + else: + iprot.skip(ftype) + iprot.readFieldEnd() + iprot.readStructEnd() + + def write(self, oprot): + if oprot._fast_encode is not None and self.thrift_spec is not None: + oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec])) + return + oprot.writeStructBegin("get_all_materialized_view_objects_for_rewriting_args") + oprot.writeFieldStop() + oprot.writeStructEnd() + + def validate(self): + return + + def __repr__(self): + L = ["{}={!r}".format(key, value) for key, value in self.__dict__.items()] + return "{}({})".format(self.__class__.__name__, ", ".join(L)) + + def __eq__(self, other): + return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ + + def __ne__(self, other): + return not (self == other) + + +all_structs.append(get_all_materialized_view_objects_for_rewriting_args) +get_all_materialized_view_objects_for_rewriting_args.thrift_spec = () + + +class get_all_materialized_view_objects_for_rewriting_result: + """ + Attributes: + - success + - o1 + + """ + + def __init__( + self, + success=None, + o1=None, + ): + self.success = success + self.o1 = o1 + + def read(self, iprot): + if ( + iprot._fast_decode is not None + and isinstance(iprot.trans, TTransport.CReadableTransport) + and self.thrift_spec is not None + ): + iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec]) + return + iprot.readStructBegin() + while True: + (fname, ftype, fid) = iprot.readFieldBegin() + if ftype == TType.STOP: + break + if fid == 0: + if ftype == TType.LIST: + self.success = [] + (_etype1373, _size1370) = iprot.readListBegin() + for _i1374 in range(_size1370): + _elem1375 = Table() + _elem1375.read(iprot) + self.success.append(_elem1375) + iprot.readListEnd() + else: + iprot.skip(ftype) + elif fid == 1: + if ftype == TType.STRUCT: + self.o1 = MetaException.read(iprot) + else: + iprot.skip(ftype) + else: + iprot.skip(ftype) + iprot.readFieldEnd() + iprot.readStructEnd() + + def write(self, oprot): + if oprot._fast_encode is not None and self.thrift_spec is not None: + oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec])) + return + oprot.writeStructBegin("get_all_materialized_view_objects_for_rewriting_result") + if self.success is not None: + oprot.writeFieldBegin("success", TType.LIST, 0) + oprot.writeListBegin(TType.STRUCT, len(self.success)) + for iter1376 in self.success: + iter1376.write(oprot) + oprot.writeListEnd() + oprot.writeFieldEnd() + if self.o1 is not None: + oprot.writeFieldBegin("o1", TType.STRUCT, 1) + self.o1.write(oprot) + oprot.writeFieldEnd() + oprot.writeFieldStop() + oprot.writeStructEnd() + + def validate(self): + return + + def __repr__(self): + L = ["{}={!r}".format(key, value) for key, value in self.__dict__.items()] + return "{}({})".format(self.__class__.__name__, ", ".join(L)) + + def __eq__(self, other): + return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ + + def __ne__(self, other): + return not (self == other) + + +all_structs.append(get_all_materialized_view_objects_for_rewriting_result) +get_all_materialized_view_objects_for_rewriting_result.thrift_spec = ( + ( + 0, + TType.LIST, + "success", + (TType.STRUCT, [Table, None], False), + None, + ), # 0 + ( + 1, + TType.STRUCT, + "o1", + [MetaException, None], + None, + ), # 1 +) + + +class get_materialized_views_for_rewriting_args: + """ + Attributes: + - db_name + + """ + + def __init__( + self, + db_name=None, + ): + self.db_name = db_name + + def read(self, iprot): + if ( + iprot._fast_decode is not None + and isinstance(iprot.trans, TTransport.CReadableTransport) + and self.thrift_spec is not None + ): + iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec]) + return + iprot.readStructBegin() + while True: + (fname, ftype, fid) = iprot.readFieldBegin() + if ftype == TType.STOP: + break + if fid == 1: + if ftype == TType.STRING: + self.db_name = ( + iprot.readString().decode("utf-8", errors="replace") if sys.version_info[0] == 2 else iprot.readString() + ) + else: + iprot.skip(ftype) + else: + iprot.skip(ftype) + iprot.readFieldEnd() + iprot.readStructEnd() + + def write(self, oprot): + if oprot._fast_encode is not None and self.thrift_spec is not None: + oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec])) + return + oprot.writeStructBegin("get_materialized_views_for_rewriting_args") + if self.db_name is not None: + oprot.writeFieldBegin("db_name", TType.STRING, 1) + oprot.writeString(self.db_name.encode("utf-8") if sys.version_info[0] == 2 else self.db_name) + oprot.writeFieldEnd() + oprot.writeFieldStop() + oprot.writeStructEnd() + + def validate(self): + return + + def __repr__(self): + L = ["{}={!r}".format(key, value) for key, value in self.__dict__.items()] + return "{}({})".format(self.__class__.__name__, ", ".join(L)) + + def __eq__(self, other): + return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ + + def __ne__(self, other): + return not (self == other) + + +all_structs.append(get_materialized_views_for_rewriting_args) +get_materialized_views_for_rewriting_args.thrift_spec = ( + None, # 0 + ( + 1, + TType.STRING, + "db_name", + "UTF8", + None, + ), # 1 +) + + +class get_materialized_views_for_rewriting_result: + """ + Attributes: + - success + - o1 + + """ + + def __init__( + self, + success=None, + o1=None, + ): + self.success = success + self.o1 = o1 + + def read(self, iprot): + if ( + iprot._fast_decode is not None + and isinstance(iprot.trans, TTransport.CReadableTransport) + and self.thrift_spec is not None + ): + iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec]) + return + iprot.readStructBegin() + while True: + (fname, ftype, fid) = iprot.readFieldBegin() + if ftype == TType.STOP: + break + if fid == 0: + if ftype == TType.LIST: + self.success = [] + (_etype1380, _size1377) = iprot.readListBegin() + for _i1381 in range(_size1377): + _elem1382 = ( + iprot.readString().decode("utf-8", errors="replace") + if sys.version_info[0] == 2 + else iprot.readString() + ) + self.success.append(_elem1382) + iprot.readListEnd() + else: + iprot.skip(ftype) + elif fid == 1: + if ftype == TType.STRUCT: + self.o1 = MetaException.read(iprot) + else: + iprot.skip(ftype) + else: + iprot.skip(ftype) + iprot.readFieldEnd() + iprot.readStructEnd() + + def write(self, oprot): + if oprot._fast_encode is not None and self.thrift_spec is not None: + oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec])) + return + oprot.writeStructBegin("get_materialized_views_for_rewriting_result") + if self.success is not None: + oprot.writeFieldBegin("success", TType.LIST, 0) + oprot.writeListBegin(TType.STRING, len(self.success)) + for iter1383 in self.success: + oprot.writeString(iter1383.encode("utf-8") if sys.version_info[0] == 2 else iter1383) + oprot.writeListEnd() + oprot.writeFieldEnd() + if self.o1 is not None: + oprot.writeFieldBegin("o1", TType.STRUCT, 1) + self.o1.write(oprot) + oprot.writeFieldEnd() + oprot.writeFieldStop() + oprot.writeStructEnd() + + def validate(self): + return + + def __repr__(self): + L = ["{}={!r}".format(key, value) for key, value in self.__dict__.items()] + return "{}({})".format(self.__class__.__name__, ", ".join(L)) + + def __eq__(self, other): + return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ + + def __ne__(self, other): + return not (self == other) + + +all_structs.append(get_materialized_views_for_rewriting_result) +get_materialized_views_for_rewriting_result.thrift_spec = ( + ( + 0, + TType.LIST, + "success", + (TType.STRING, "UTF8", False), + None, + ), # 0 + ( + 1, + TType.STRUCT, + "o1", + [MetaException, None], + None, + ), # 1 +) + + +class get_table_meta_args: + """ + Attributes: + - db_patterns + - tbl_patterns + - tbl_types + + """ + + def __init__( + self, + db_patterns=None, + tbl_patterns=None, + tbl_types=None, + ): + self.db_patterns = db_patterns + self.tbl_patterns = tbl_patterns + self.tbl_types = tbl_types + + def read(self, iprot): + if ( + iprot._fast_decode is not None + and isinstance(iprot.trans, TTransport.CReadableTransport) + and self.thrift_spec is not None + ): + iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec]) + return + iprot.readStructBegin() + while True: + (fname, ftype, fid) = iprot.readFieldBegin() + if ftype == TType.STOP: + break + if fid == 1: + if ftype == TType.STRING: + self.db_patterns = ( + iprot.readString().decode("utf-8", errors="replace") if sys.version_info[0] == 2 else iprot.readString() + ) + else: + iprot.skip(ftype) + elif fid == 2: + if ftype == TType.STRING: + self.tbl_patterns = ( + iprot.readString().decode("utf-8", errors="replace") if sys.version_info[0] == 2 else iprot.readString() + ) + else: + iprot.skip(ftype) + elif fid == 3: + if ftype == TType.LIST: + self.tbl_types = [] + (_etype1387, _size1384) = iprot.readListBegin() + for _i1388 in range(_size1384): + _elem1389 = ( + iprot.readString().decode("utf-8", errors="replace") + if sys.version_info[0] == 2 + else iprot.readString() + ) + self.tbl_types.append(_elem1389) + iprot.readListEnd() + else: + iprot.skip(ftype) + else: + iprot.skip(ftype) + iprot.readFieldEnd() + iprot.readStructEnd() + + def write(self, oprot): + if oprot._fast_encode is not None and self.thrift_spec is not None: + oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec])) + return + oprot.writeStructBegin("get_table_meta_args") + if self.db_patterns is not None: + oprot.writeFieldBegin("db_patterns", TType.STRING, 1) + oprot.writeString(self.db_patterns.encode("utf-8") if sys.version_info[0] == 2 else self.db_patterns) + oprot.writeFieldEnd() + if self.tbl_patterns is not None: + oprot.writeFieldBegin("tbl_patterns", TType.STRING, 2) + oprot.writeString(self.tbl_patterns.encode("utf-8") if sys.version_info[0] == 2 else self.tbl_patterns) + oprot.writeFieldEnd() + if self.tbl_types is not None: + oprot.writeFieldBegin("tbl_types", TType.LIST, 3) + oprot.writeListBegin(TType.STRING, len(self.tbl_types)) + for iter1390 in self.tbl_types: + oprot.writeString(iter1390.encode("utf-8") if sys.version_info[0] == 2 else iter1390) + oprot.writeListEnd() + oprot.writeFieldEnd() + oprot.writeFieldStop() + oprot.writeStructEnd() + + def validate(self): + return + + def __repr__(self): + L = ["{}={!r}".format(key, value) for key, value in self.__dict__.items()] + return "{}({})".format(self.__class__.__name__, ", ".join(L)) + + def __eq__(self, other): + return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ + + def __ne__(self, other): + return not (self == other) + + +all_structs.append(get_table_meta_args) +get_table_meta_args.thrift_spec = ( + None, # 0 + ( + 1, + TType.STRING, + "db_patterns", + "UTF8", + None, + ), # 1 + ( + 2, + TType.STRING, + "tbl_patterns", + "UTF8", + None, + ), # 2 + ( + 3, + TType.LIST, + "tbl_types", + (TType.STRING, "UTF8", False), + None, + ), # 3 +) + + +class get_table_meta_result: + """ + Attributes: + - success + - o1 + + """ + + def __init__( + self, + success=None, + o1=None, + ): + self.success = success + self.o1 = o1 + + def read(self, iprot): + if ( + iprot._fast_decode is not None + and isinstance(iprot.trans, TTransport.CReadableTransport) + and self.thrift_spec is not None + ): + iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec]) + return + iprot.readStructBegin() + while True: + (fname, ftype, fid) = iprot.readFieldBegin() + if ftype == TType.STOP: + break + if fid == 0: + if ftype == TType.LIST: + self.success = [] + (_etype1394, _size1391) = iprot.readListBegin() + for _i1395 in range(_size1391): + _elem1396 = TableMeta() + _elem1396.read(iprot) + self.success.append(_elem1396) + iprot.readListEnd() + else: + iprot.skip(ftype) + elif fid == 1: + if ftype == TType.STRUCT: + self.o1 = MetaException.read(iprot) + else: + iprot.skip(ftype) + else: + iprot.skip(ftype) + iprot.readFieldEnd() + iprot.readStructEnd() + + def write(self, oprot): + if oprot._fast_encode is not None and self.thrift_spec is not None: + oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec])) + return + oprot.writeStructBegin("get_table_meta_result") + if self.success is not None: + oprot.writeFieldBegin("success", TType.LIST, 0) + oprot.writeListBegin(TType.STRUCT, len(self.success)) + for iter1397 in self.success: + iter1397.write(oprot) + oprot.writeListEnd() + oprot.writeFieldEnd() + if self.o1 is not None: + oprot.writeFieldBegin("o1", TType.STRUCT, 1) + self.o1.write(oprot) + oprot.writeFieldEnd() + oprot.writeFieldStop() + oprot.writeStructEnd() + + def validate(self): + return + + def __repr__(self): + L = ["{}={!r}".format(key, value) for key, value in self.__dict__.items()] + return "{}({})".format(self.__class__.__name__, ", ".join(L)) + + def __eq__(self, other): + return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ + + def __ne__(self, other): + return not (self == other) + + +all_structs.append(get_table_meta_result) +get_table_meta_result.thrift_spec = ( + ( + 0, + TType.LIST, + "success", + (TType.STRUCT, [TableMeta, None], False), + None, + ), # 0 + ( + 1, + TType.STRUCT, + "o1", + [MetaException, None], + None, + ), # 1 +) + + +class get_all_tables_args: + """ + Attributes: + - db_name + + """ + + def __init__( + self, + db_name=None, + ): + self.db_name = db_name + + def read(self, iprot): + if ( + iprot._fast_decode is not None + and isinstance(iprot.trans, TTransport.CReadableTransport) + and self.thrift_spec is not None + ): + iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec]) + return + iprot.readStructBegin() + while True: + (fname, ftype, fid) = iprot.readFieldBegin() + if ftype == TType.STOP: + break + if fid == 1: + if ftype == TType.STRING: + self.db_name = ( + iprot.readString().decode("utf-8", errors="replace") if sys.version_info[0] == 2 else iprot.readString() + ) + else: + iprot.skip(ftype) + else: + iprot.skip(ftype) + iprot.readFieldEnd() + iprot.readStructEnd() + + def write(self, oprot): + if oprot._fast_encode is not None and self.thrift_spec is not None: + oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec])) + return + oprot.writeStructBegin("get_all_tables_args") + if self.db_name is not None: + oprot.writeFieldBegin("db_name", TType.STRING, 1) + oprot.writeString(self.db_name.encode("utf-8") if sys.version_info[0] == 2 else self.db_name) + oprot.writeFieldEnd() + oprot.writeFieldStop() + oprot.writeStructEnd() + + def validate(self): + return + + def __repr__(self): + L = ["{}={!r}".format(key, value) for key, value in self.__dict__.items()] + return "{}({})".format(self.__class__.__name__, ", ".join(L)) + + def __eq__(self, other): + return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ + + def __ne__(self, other): + return not (self == other) + + +all_structs.append(get_all_tables_args) +get_all_tables_args.thrift_spec = ( + None, # 0 + ( + 1, + TType.STRING, + "db_name", + "UTF8", + None, + ), # 1 +) + + +class get_all_tables_result: + """ + Attributes: + - success + - o1 + + """ + + def __init__( + self, + success=None, + o1=None, + ): + self.success = success + self.o1 = o1 + + def read(self, iprot): + if ( + iprot._fast_decode is not None + and isinstance(iprot.trans, TTransport.CReadableTransport) + and self.thrift_spec is not None + ): + iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec]) + return + iprot.readStructBegin() + while True: + (fname, ftype, fid) = iprot.readFieldBegin() + if ftype == TType.STOP: + break + if fid == 0: + if ftype == TType.LIST: + self.success = [] + (_etype1401, _size1398) = iprot.readListBegin() + for _i1402 in range(_size1398): + _elem1403 = ( + iprot.readString().decode("utf-8", errors="replace") + if sys.version_info[0] == 2 + else iprot.readString() + ) + self.success.append(_elem1403) + iprot.readListEnd() + else: + iprot.skip(ftype) + elif fid == 1: + if ftype == TType.STRUCT: + self.o1 = MetaException.read(iprot) + else: + iprot.skip(ftype) + else: + iprot.skip(ftype) + iprot.readFieldEnd() + iprot.readStructEnd() + + def write(self, oprot): + if oprot._fast_encode is not None and self.thrift_spec is not None: + oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec])) + return + oprot.writeStructBegin("get_all_tables_result") + if self.success is not None: + oprot.writeFieldBegin("success", TType.LIST, 0) + oprot.writeListBegin(TType.STRING, len(self.success)) + for iter1404 in self.success: + oprot.writeString(iter1404.encode("utf-8") if sys.version_info[0] == 2 else iter1404) + oprot.writeListEnd() + oprot.writeFieldEnd() + if self.o1 is not None: + oprot.writeFieldBegin("o1", TType.STRUCT, 1) + self.o1.write(oprot) + oprot.writeFieldEnd() + oprot.writeFieldStop() + oprot.writeStructEnd() + + def validate(self): + return + + def __repr__(self): + L = ["{}={!r}".format(key, value) for key, value in self.__dict__.items()] + return "{}({})".format(self.__class__.__name__, ", ".join(L)) + + def __eq__(self, other): + return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ + + def __ne__(self, other): + return not (self == other) + + +all_structs.append(get_all_tables_result) +get_all_tables_result.thrift_spec = ( + ( + 0, + TType.LIST, + "success", + (TType.STRING, "UTF8", False), + None, + ), # 0 + ( + 1, + TType.STRUCT, + "o1", + [MetaException, None], + None, + ), # 1 +) + + +class get_table_args: + """ + Attributes: + - dbname + - tbl_name + + """ + + def __init__( + self, + dbname=None, + tbl_name=None, + ): + self.dbname = dbname + self.tbl_name = tbl_name + + def read(self, iprot): + if ( + iprot._fast_decode is not None + and isinstance(iprot.trans, TTransport.CReadableTransport) + and self.thrift_spec is not None + ): + iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec]) + return + iprot.readStructBegin() + while True: + (fname, ftype, fid) = iprot.readFieldBegin() + if ftype == TType.STOP: + break + if fid == 1: + if ftype == TType.STRING: + self.dbname = ( + iprot.readString().decode("utf-8", errors="replace") if sys.version_info[0] == 2 else iprot.readString() + ) + else: + iprot.skip(ftype) + elif fid == 2: + if ftype == TType.STRING: + self.tbl_name = ( + iprot.readString().decode("utf-8", errors="replace") if sys.version_info[0] == 2 else iprot.readString() + ) + else: + iprot.skip(ftype) + else: + iprot.skip(ftype) + iprot.readFieldEnd() + iprot.readStructEnd() + + def write(self, oprot): + if oprot._fast_encode is not None and self.thrift_spec is not None: + oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec])) + return + oprot.writeStructBegin("get_table_args") + if self.dbname is not None: + oprot.writeFieldBegin("dbname", TType.STRING, 1) + oprot.writeString(self.dbname.encode("utf-8") if sys.version_info[0] == 2 else self.dbname) + oprot.writeFieldEnd() + if self.tbl_name is not None: + oprot.writeFieldBegin("tbl_name", TType.STRING, 2) + oprot.writeString(self.tbl_name.encode("utf-8") if sys.version_info[0] == 2 else self.tbl_name) + oprot.writeFieldEnd() + oprot.writeFieldStop() + oprot.writeStructEnd() + + def validate(self): + return + + def __repr__(self): + L = ["{}={!r}".format(key, value) for key, value in self.__dict__.items()] + return "{}({})".format(self.__class__.__name__, ", ".join(L)) + + def __eq__(self, other): + return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ + + def __ne__(self, other): + return not (self == other) + + +all_structs.append(get_table_args) +get_table_args.thrift_spec = ( + None, # 0 + ( + 1, + TType.STRING, + "dbname", + "UTF8", + None, + ), # 1 + ( + 2, + TType.STRING, + "tbl_name", + "UTF8", + None, + ), # 2 +) + + +class get_table_result: + """ + Attributes: + - success + - o1 + - o2 + + """ + + def __init__( + self, + success=None, + o1=None, + o2=None, + ): + self.success = success + self.o1 = o1 + self.o2 = o2 + + def read(self, iprot): + if ( + iprot._fast_decode is not None + and isinstance(iprot.trans, TTransport.CReadableTransport) + and self.thrift_spec is not None + ): + iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec]) + return + iprot.readStructBegin() + while True: + (fname, ftype, fid) = iprot.readFieldBegin() + if ftype == TType.STOP: + break + if fid == 0: + if ftype == TType.STRUCT: + self.success = Table() + self.success.read(iprot) + else: + iprot.skip(ftype) + elif fid == 1: + if ftype == TType.STRUCT: + self.o1 = MetaException.read(iprot) + else: + iprot.skip(ftype) + elif fid == 2: + if ftype == TType.STRUCT: + self.o2 = NoSuchObjectException.read(iprot) + else: + iprot.skip(ftype) + else: + iprot.skip(ftype) + iprot.readFieldEnd() + iprot.readStructEnd() + + def write(self, oprot): + if oprot._fast_encode is not None and self.thrift_spec is not None: + oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec])) + return + oprot.writeStructBegin("get_table_result") + if self.success is not None: + oprot.writeFieldBegin("success", TType.STRUCT, 0) + self.success.write(oprot) + oprot.writeFieldEnd() + if self.o1 is not None: + oprot.writeFieldBegin("o1", TType.STRUCT, 1) + self.o1.write(oprot) + oprot.writeFieldEnd() + if self.o2 is not None: + oprot.writeFieldBegin("o2", TType.STRUCT, 2) + self.o2.write(oprot) + oprot.writeFieldEnd() + oprot.writeFieldStop() + oprot.writeStructEnd() + + def validate(self): + return + + def __repr__(self): + L = ["{}={!r}".format(key, value) for key, value in self.__dict__.items()] + return "{}({})".format(self.__class__.__name__, ", ".join(L)) + + def __eq__(self, other): + return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ + + def __ne__(self, other): + return not (self == other) + + +all_structs.append(get_table_result) +get_table_result.thrift_spec = ( + ( + 0, + TType.STRUCT, + "success", + [Table, None], + None, + ), # 0 + ( + 1, + TType.STRUCT, + "o1", + [MetaException, None], + None, + ), # 1 + ( + 2, + TType.STRUCT, + "o2", + [NoSuchObjectException, None], + None, + ), # 2 +) + + +class get_table_objects_by_name_args: + """ + Attributes: + - dbname + - tbl_names + + """ + + def __init__( + self, + dbname=None, + tbl_names=None, + ): + self.dbname = dbname + self.tbl_names = tbl_names + + def read(self, iprot): + if ( + iprot._fast_decode is not None + and isinstance(iprot.trans, TTransport.CReadableTransport) + and self.thrift_spec is not None + ): + iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec]) + return + iprot.readStructBegin() + while True: + (fname, ftype, fid) = iprot.readFieldBegin() + if ftype == TType.STOP: + break + if fid == 1: + if ftype == TType.STRING: + self.dbname = ( + iprot.readString().decode("utf-8", errors="replace") if sys.version_info[0] == 2 else iprot.readString() + ) + else: + iprot.skip(ftype) + elif fid == 2: + if ftype == TType.LIST: + self.tbl_names = [] + (_etype1408, _size1405) = iprot.readListBegin() + for _i1409 in range(_size1405): + _elem1410 = ( + iprot.readString().decode("utf-8", errors="replace") + if sys.version_info[0] == 2 + else iprot.readString() + ) + self.tbl_names.append(_elem1410) + iprot.readListEnd() + else: + iprot.skip(ftype) + else: + iprot.skip(ftype) + iprot.readFieldEnd() + iprot.readStructEnd() + + def write(self, oprot): + if oprot._fast_encode is not None and self.thrift_spec is not None: + oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec])) + return + oprot.writeStructBegin("get_table_objects_by_name_args") + if self.dbname is not None: + oprot.writeFieldBegin("dbname", TType.STRING, 1) + oprot.writeString(self.dbname.encode("utf-8") if sys.version_info[0] == 2 else self.dbname) + oprot.writeFieldEnd() + if self.tbl_names is not None: + oprot.writeFieldBegin("tbl_names", TType.LIST, 2) + oprot.writeListBegin(TType.STRING, len(self.tbl_names)) + for iter1411 in self.tbl_names: + oprot.writeString(iter1411.encode("utf-8") if sys.version_info[0] == 2 else iter1411) + oprot.writeListEnd() + oprot.writeFieldEnd() + oprot.writeFieldStop() + oprot.writeStructEnd() + + def validate(self): + return + + def __repr__(self): + L = ["{}={!r}".format(key, value) for key, value in self.__dict__.items()] + return "{}({})".format(self.__class__.__name__, ", ".join(L)) + + def __eq__(self, other): + return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ + + def __ne__(self, other): + return not (self == other) + + +all_structs.append(get_table_objects_by_name_args) +get_table_objects_by_name_args.thrift_spec = ( + None, # 0 + ( + 1, + TType.STRING, + "dbname", + "UTF8", + None, + ), # 1 + ( + 2, + TType.LIST, + "tbl_names", + (TType.STRING, "UTF8", False), + None, + ), # 2 +) + + +class get_table_objects_by_name_result: + """ + Attributes: + - success + + """ + + def __init__( + self, + success=None, + ): + self.success = success + + def read(self, iprot): + if ( + iprot._fast_decode is not None + and isinstance(iprot.trans, TTransport.CReadableTransport) + and self.thrift_spec is not None + ): + iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec]) + return + iprot.readStructBegin() + while True: + (fname, ftype, fid) = iprot.readFieldBegin() + if ftype == TType.STOP: + break + if fid == 0: + if ftype == TType.LIST: + self.success = [] + (_etype1415, _size1412) = iprot.readListBegin() + for _i1416 in range(_size1412): + _elem1417 = Table() + _elem1417.read(iprot) + self.success.append(_elem1417) + iprot.readListEnd() + else: + iprot.skip(ftype) + else: + iprot.skip(ftype) + iprot.readFieldEnd() + iprot.readStructEnd() + + def write(self, oprot): + if oprot._fast_encode is not None and self.thrift_spec is not None: + oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec])) + return + oprot.writeStructBegin("get_table_objects_by_name_result") + if self.success is not None: + oprot.writeFieldBegin("success", TType.LIST, 0) + oprot.writeListBegin(TType.STRUCT, len(self.success)) + for iter1418 in self.success: + iter1418.write(oprot) + oprot.writeListEnd() + oprot.writeFieldEnd() + oprot.writeFieldStop() + oprot.writeStructEnd() + + def validate(self): + return + + def __repr__(self): + L = ["{}={!r}".format(key, value) for key, value in self.__dict__.items()] + return "{}({})".format(self.__class__.__name__, ", ".join(L)) + + def __eq__(self, other): + return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ + + def __ne__(self, other): + return not (self == other) + + +all_structs.append(get_table_objects_by_name_result) +get_table_objects_by_name_result.thrift_spec = ( + ( + 0, + TType.LIST, + "success", + (TType.STRUCT, [Table, None], False), + None, + ), # 0 +) + + +class get_tables_ext_args: + """ + Attributes: + - req + + """ + + def __init__( + self, + req=None, + ): + self.req = req + + def read(self, iprot): + if ( + iprot._fast_decode is not None + and isinstance(iprot.trans, TTransport.CReadableTransport) + and self.thrift_spec is not None + ): + iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec]) + return + iprot.readStructBegin() + while True: + (fname, ftype, fid) = iprot.readFieldBegin() + if ftype == TType.STOP: + break + if fid == 1: + if ftype == TType.STRUCT: + self.req = GetTablesExtRequest() + self.req.read(iprot) + else: + iprot.skip(ftype) + else: + iprot.skip(ftype) + iprot.readFieldEnd() + iprot.readStructEnd() + + def write(self, oprot): + if oprot._fast_encode is not None and self.thrift_spec is not None: + oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec])) + return + oprot.writeStructBegin("get_tables_ext_args") + if self.req is not None: + oprot.writeFieldBegin("req", TType.STRUCT, 1) + self.req.write(oprot) + oprot.writeFieldEnd() + oprot.writeFieldStop() + oprot.writeStructEnd() + + def validate(self): + return + + def __repr__(self): + L = ["{}={!r}".format(key, value) for key, value in self.__dict__.items()] + return "{}({})".format(self.__class__.__name__, ", ".join(L)) + + def __eq__(self, other): + return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ + + def __ne__(self, other): + return not (self == other) + + +all_structs.append(get_tables_ext_args) +get_tables_ext_args.thrift_spec = ( + None, # 0 + ( + 1, + TType.STRUCT, + "req", + [GetTablesExtRequest, None], + None, + ), # 1 +) + + +class get_tables_ext_result: + """ + Attributes: + - success + - o1 + + """ + + def __init__( + self, + success=None, + o1=None, + ): + self.success = success + self.o1 = o1 + + def read(self, iprot): + if ( + iprot._fast_decode is not None + and isinstance(iprot.trans, TTransport.CReadableTransport) + and self.thrift_spec is not None + ): + iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec]) + return + iprot.readStructBegin() + while True: + (fname, ftype, fid) = iprot.readFieldBegin() + if ftype == TType.STOP: + break + if fid == 0: + if ftype == TType.LIST: + self.success = [] + (_etype1422, _size1419) = iprot.readListBegin() + for _i1423 in range(_size1419): + _elem1424 = ExtendedTableInfo() + _elem1424.read(iprot) + self.success.append(_elem1424) + iprot.readListEnd() + else: + iprot.skip(ftype) + elif fid == 1: + if ftype == TType.STRUCT: + self.o1 = MetaException.read(iprot) + else: + iprot.skip(ftype) + else: + iprot.skip(ftype) + iprot.readFieldEnd() + iprot.readStructEnd() + + def write(self, oprot): + if oprot._fast_encode is not None and self.thrift_spec is not None: + oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec])) + return + oprot.writeStructBegin("get_tables_ext_result") + if self.success is not None: + oprot.writeFieldBegin("success", TType.LIST, 0) + oprot.writeListBegin(TType.STRUCT, len(self.success)) + for iter1425 in self.success: + iter1425.write(oprot) + oprot.writeListEnd() + oprot.writeFieldEnd() + if self.o1 is not None: + oprot.writeFieldBegin("o1", TType.STRUCT, 1) + self.o1.write(oprot) + oprot.writeFieldEnd() + oprot.writeFieldStop() + oprot.writeStructEnd() + + def validate(self): + return + + def __repr__(self): + L = ["{}={!r}".format(key, value) for key, value in self.__dict__.items()] + return "{}({})".format(self.__class__.__name__, ", ".join(L)) + + def __eq__(self, other): + return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ + + def __ne__(self, other): + return not (self == other) + + +all_structs.append(get_tables_ext_result) +get_tables_ext_result.thrift_spec = ( + ( + 0, + TType.LIST, + "success", + (TType.STRUCT, [ExtendedTableInfo, None], False), + None, + ), # 0 + ( + 1, + TType.STRUCT, + "o1", + [MetaException, None], + None, + ), # 1 +) + + +class get_table_req_args: + """ + Attributes: + - req + + """ + + def __init__( + self, + req=None, + ): + self.req = req + + def read(self, iprot): + if ( + iprot._fast_decode is not None + and isinstance(iprot.trans, TTransport.CReadableTransport) + and self.thrift_spec is not None + ): + iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec]) + return + iprot.readStructBegin() + while True: + (fname, ftype, fid) = iprot.readFieldBegin() + if ftype == TType.STOP: + break + if fid == 1: + if ftype == TType.STRUCT: + self.req = GetTableRequest() + self.req.read(iprot) + else: + iprot.skip(ftype) + else: + iprot.skip(ftype) + iprot.readFieldEnd() + iprot.readStructEnd() + + def write(self, oprot): + if oprot._fast_encode is not None and self.thrift_spec is not None: + oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec])) + return + oprot.writeStructBegin("get_table_req_args") + if self.req is not None: + oprot.writeFieldBegin("req", TType.STRUCT, 1) + self.req.write(oprot) + oprot.writeFieldEnd() + oprot.writeFieldStop() + oprot.writeStructEnd() + + def validate(self): + return + + def __repr__(self): + L = ["{}={!r}".format(key, value) for key, value in self.__dict__.items()] + return "{}({})".format(self.__class__.__name__, ", ".join(L)) + + def __eq__(self, other): + return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ + + def __ne__(self, other): + return not (self == other) + + +all_structs.append(get_table_req_args) +get_table_req_args.thrift_spec = ( + None, # 0 + ( + 1, + TType.STRUCT, + "req", + [GetTableRequest, None], + None, + ), # 1 +) + + +class get_table_req_result: + """ + Attributes: + - success + - o1 + - o2 + + """ + + def __init__( + self, + success=None, + o1=None, + o2=None, + ): + self.success = success + self.o1 = o1 + self.o2 = o2 + + def read(self, iprot): + if ( + iprot._fast_decode is not None + and isinstance(iprot.trans, TTransport.CReadableTransport) + and self.thrift_spec is not None + ): + iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec]) + return + iprot.readStructBegin() + while True: + (fname, ftype, fid) = iprot.readFieldBegin() + if ftype == TType.STOP: + break + if fid == 0: + if ftype == TType.STRUCT: + self.success = GetTableResult() + self.success.read(iprot) + else: + iprot.skip(ftype) + elif fid == 1: + if ftype == TType.STRUCT: + self.o1 = MetaException.read(iprot) + else: + iprot.skip(ftype) + elif fid == 2: + if ftype == TType.STRUCT: + self.o2 = NoSuchObjectException.read(iprot) + else: + iprot.skip(ftype) + else: + iprot.skip(ftype) + iprot.readFieldEnd() + iprot.readStructEnd() + + def write(self, oprot): + if oprot._fast_encode is not None and self.thrift_spec is not None: + oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec])) + return + oprot.writeStructBegin("get_table_req_result") + if self.success is not None: + oprot.writeFieldBegin("success", TType.STRUCT, 0) + self.success.write(oprot) + oprot.writeFieldEnd() + if self.o1 is not None: + oprot.writeFieldBegin("o1", TType.STRUCT, 1) + self.o1.write(oprot) + oprot.writeFieldEnd() + if self.o2 is not None: + oprot.writeFieldBegin("o2", TType.STRUCT, 2) + self.o2.write(oprot) + oprot.writeFieldEnd() + oprot.writeFieldStop() + oprot.writeStructEnd() + + def validate(self): + return + + def __repr__(self): + L = ["{}={!r}".format(key, value) for key, value in self.__dict__.items()] + return "{}({})".format(self.__class__.__name__, ", ".join(L)) + + def __eq__(self, other): + return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ + + def __ne__(self, other): + return not (self == other) + + +all_structs.append(get_table_req_result) +get_table_req_result.thrift_spec = ( + ( + 0, + TType.STRUCT, + "success", + [GetTableResult, None], + None, + ), # 0 + ( + 1, + TType.STRUCT, + "o1", + [MetaException, None], + None, + ), # 1 + ( + 2, + TType.STRUCT, + "o2", + [NoSuchObjectException, None], + None, + ), # 2 +) + + +class get_table_objects_by_name_req_args: + """ + Attributes: + - req + + """ + + def __init__( + self, + req=None, + ): + self.req = req + + def read(self, iprot): + if ( + iprot._fast_decode is not None + and isinstance(iprot.trans, TTransport.CReadableTransport) + and self.thrift_spec is not None + ): + iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec]) + return + iprot.readStructBegin() + while True: + (fname, ftype, fid) = iprot.readFieldBegin() + if ftype == TType.STOP: + break + if fid == 1: + if ftype == TType.STRUCT: + self.req = GetTablesRequest() + self.req.read(iprot) + else: + iprot.skip(ftype) + else: + iprot.skip(ftype) + iprot.readFieldEnd() + iprot.readStructEnd() + + def write(self, oprot): + if oprot._fast_encode is not None and self.thrift_spec is not None: + oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec])) + return + oprot.writeStructBegin("get_table_objects_by_name_req_args") + if self.req is not None: + oprot.writeFieldBegin("req", TType.STRUCT, 1) + self.req.write(oprot) + oprot.writeFieldEnd() + oprot.writeFieldStop() + oprot.writeStructEnd() + + def validate(self): + return + + def __repr__(self): + L = ["{}={!r}".format(key, value) for key, value in self.__dict__.items()] + return "{}({})".format(self.__class__.__name__, ", ".join(L)) + + def __eq__(self, other): + return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ + + def __ne__(self, other): + return not (self == other) + + +all_structs.append(get_table_objects_by_name_req_args) +get_table_objects_by_name_req_args.thrift_spec = ( + None, # 0 + ( + 1, + TType.STRUCT, + "req", + [GetTablesRequest, None], + None, + ), # 1 +) + + +class get_table_objects_by_name_req_result: + """ + Attributes: + - success + - o1 + - o2 + - o3 + + """ + + def __init__( + self, + success=None, + o1=None, + o2=None, + o3=None, + ): + self.success = success + self.o1 = o1 + self.o2 = o2 + self.o3 = o3 + + def read(self, iprot): + if ( + iprot._fast_decode is not None + and isinstance(iprot.trans, TTransport.CReadableTransport) + and self.thrift_spec is not None + ): + iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec]) + return + iprot.readStructBegin() + while True: + (fname, ftype, fid) = iprot.readFieldBegin() + if ftype == TType.STOP: + break + if fid == 0: + if ftype == TType.STRUCT: + self.success = GetTablesResult() + self.success.read(iprot) + else: + iprot.skip(ftype) + elif fid == 1: + if ftype == TType.STRUCT: + self.o1 = MetaException.read(iprot) + else: + iprot.skip(ftype) + elif fid == 2: + if ftype == TType.STRUCT: + self.o2 = InvalidOperationException.read(iprot) + else: + iprot.skip(ftype) + elif fid == 3: + if ftype == TType.STRUCT: + self.o3 = UnknownDBException.read(iprot) + else: + iprot.skip(ftype) + else: + iprot.skip(ftype) + iprot.readFieldEnd() + iprot.readStructEnd() + + def write(self, oprot): + if oprot._fast_encode is not None and self.thrift_spec is not None: + oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec])) + return + oprot.writeStructBegin("get_table_objects_by_name_req_result") + if self.success is not None: + oprot.writeFieldBegin("success", TType.STRUCT, 0) + self.success.write(oprot) + oprot.writeFieldEnd() + if self.o1 is not None: + oprot.writeFieldBegin("o1", TType.STRUCT, 1) + self.o1.write(oprot) + oprot.writeFieldEnd() + if self.o2 is not None: + oprot.writeFieldBegin("o2", TType.STRUCT, 2) + self.o2.write(oprot) + oprot.writeFieldEnd() + if self.o3 is not None: + oprot.writeFieldBegin("o3", TType.STRUCT, 3) + self.o3.write(oprot) + oprot.writeFieldEnd() + oprot.writeFieldStop() + oprot.writeStructEnd() + + def validate(self): + return + + def __repr__(self): + L = ["{}={!r}".format(key, value) for key, value in self.__dict__.items()] + return "{}({})".format(self.__class__.__name__, ", ".join(L)) + + def __eq__(self, other): + return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ + + def __ne__(self, other): + return not (self == other) + + +all_structs.append(get_table_objects_by_name_req_result) +get_table_objects_by_name_req_result.thrift_spec = ( + ( + 0, + TType.STRUCT, + "success", + [GetTablesResult, None], + None, + ), # 0 + ( + 1, + TType.STRUCT, + "o1", + [MetaException, None], + None, + ), # 1 + ( + 2, + TType.STRUCT, + "o2", + [InvalidOperationException, None], + None, + ), # 2 + ( + 3, + TType.STRUCT, + "o3", + [UnknownDBException, None], + None, + ), # 3 +) + + +class get_materialization_invalidation_info_args: + """ + Attributes: + - creation_metadata + - validTxnList + + """ + + def __init__( + self, + creation_metadata=None, + validTxnList=None, + ): + self.creation_metadata = creation_metadata + self.validTxnList = validTxnList + + def read(self, iprot): + if ( + iprot._fast_decode is not None + and isinstance(iprot.trans, TTransport.CReadableTransport) + and self.thrift_spec is not None + ): + iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec]) + return + iprot.readStructBegin() + while True: + (fname, ftype, fid) = iprot.readFieldBegin() + if ftype == TType.STOP: + break + if fid == 1: + if ftype == TType.STRUCT: + self.creation_metadata = CreationMetadata() + self.creation_metadata.read(iprot) + else: + iprot.skip(ftype) + elif fid == 2: + if ftype == TType.STRING: + self.validTxnList = ( + iprot.readString().decode("utf-8", errors="replace") if sys.version_info[0] == 2 else iprot.readString() + ) + else: + iprot.skip(ftype) + else: + iprot.skip(ftype) + iprot.readFieldEnd() + iprot.readStructEnd() + + def write(self, oprot): + if oprot._fast_encode is not None and self.thrift_spec is not None: + oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec])) + return + oprot.writeStructBegin("get_materialization_invalidation_info_args") + if self.creation_metadata is not None: + oprot.writeFieldBegin("creation_metadata", TType.STRUCT, 1) + self.creation_metadata.write(oprot) + oprot.writeFieldEnd() + if self.validTxnList is not None: + oprot.writeFieldBegin("validTxnList", TType.STRING, 2) + oprot.writeString(self.validTxnList.encode("utf-8") if sys.version_info[0] == 2 else self.validTxnList) + oprot.writeFieldEnd() + oprot.writeFieldStop() + oprot.writeStructEnd() + + def validate(self): + return + + def __repr__(self): + L = ["{}={!r}".format(key, value) for key, value in self.__dict__.items()] + return "{}({})".format(self.__class__.__name__, ", ".join(L)) + + def __eq__(self, other): + return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ + + def __ne__(self, other): + return not (self == other) + + +all_structs.append(get_materialization_invalidation_info_args) +get_materialization_invalidation_info_args.thrift_spec = ( + None, # 0 + ( + 1, + TType.STRUCT, + "creation_metadata", + [CreationMetadata, None], + None, + ), # 1 + ( + 2, + TType.STRING, + "validTxnList", + "UTF8", + None, + ), # 2 +) + + +class get_materialization_invalidation_info_result: + """ + Attributes: + - success + - o1 + - o2 + - o3 + + """ + + def __init__( + self, + success=None, + o1=None, + o2=None, + o3=None, + ): + self.success = success + self.o1 = o1 + self.o2 = o2 + self.o3 = o3 + + def read(self, iprot): + if ( + iprot._fast_decode is not None + and isinstance(iprot.trans, TTransport.CReadableTransport) + and self.thrift_spec is not None + ): + iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec]) + return + iprot.readStructBegin() + while True: + (fname, ftype, fid) = iprot.readFieldBegin() + if ftype == TType.STOP: + break + if fid == 0: + if ftype == TType.STRUCT: + self.success = Materialization() + self.success.read(iprot) + else: + iprot.skip(ftype) + elif fid == 1: + if ftype == TType.STRUCT: + self.o1 = MetaException.read(iprot) + else: + iprot.skip(ftype) + elif fid == 2: + if ftype == TType.STRUCT: + self.o2 = InvalidOperationException.read(iprot) + else: + iprot.skip(ftype) + elif fid == 3: + if ftype == TType.STRUCT: + self.o3 = UnknownDBException.read(iprot) + else: + iprot.skip(ftype) + else: + iprot.skip(ftype) + iprot.readFieldEnd() + iprot.readStructEnd() + + def write(self, oprot): + if oprot._fast_encode is not None and self.thrift_spec is not None: + oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec])) + return + oprot.writeStructBegin("get_materialization_invalidation_info_result") + if self.success is not None: + oprot.writeFieldBegin("success", TType.STRUCT, 0) + self.success.write(oprot) + oprot.writeFieldEnd() + if self.o1 is not None: + oprot.writeFieldBegin("o1", TType.STRUCT, 1) + self.o1.write(oprot) + oprot.writeFieldEnd() + if self.o2 is not None: + oprot.writeFieldBegin("o2", TType.STRUCT, 2) + self.o2.write(oprot) + oprot.writeFieldEnd() + if self.o3 is not None: + oprot.writeFieldBegin("o3", TType.STRUCT, 3) + self.o3.write(oprot) + oprot.writeFieldEnd() + oprot.writeFieldStop() + oprot.writeStructEnd() + + def validate(self): + return + + def __repr__(self): + L = ["{}={!r}".format(key, value) for key, value in self.__dict__.items()] + return "{}({})".format(self.__class__.__name__, ", ".join(L)) + + def __eq__(self, other): + return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ + + def __ne__(self, other): + return not (self == other) + + +all_structs.append(get_materialization_invalidation_info_result) +get_materialization_invalidation_info_result.thrift_spec = ( + ( + 0, + TType.STRUCT, + "success", + [Materialization, None], + None, + ), # 0 + ( + 1, + TType.STRUCT, + "o1", + [MetaException, None], + None, + ), # 1 + ( + 2, + TType.STRUCT, + "o2", + [InvalidOperationException, None], + None, + ), # 2 + ( + 3, + TType.STRUCT, + "o3", + [UnknownDBException, None], + None, + ), # 3 +) + + +class update_creation_metadata_args: + """ + Attributes: + - catName + - dbname + - tbl_name + - creation_metadata + + """ + + def __init__( + self, + catName=None, + dbname=None, + tbl_name=None, + creation_metadata=None, + ): + self.catName = catName + self.dbname = dbname + self.tbl_name = tbl_name + self.creation_metadata = creation_metadata + + def read(self, iprot): + if ( + iprot._fast_decode is not None + and isinstance(iprot.trans, TTransport.CReadableTransport) + and self.thrift_spec is not None + ): + iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec]) + return + iprot.readStructBegin() + while True: + (fname, ftype, fid) = iprot.readFieldBegin() + if ftype == TType.STOP: + break + if fid == 1: + if ftype == TType.STRING: + self.catName = ( + iprot.readString().decode("utf-8", errors="replace") if sys.version_info[0] == 2 else iprot.readString() + ) + else: + iprot.skip(ftype) + elif fid == 2: + if ftype == TType.STRING: + self.dbname = ( + iprot.readString().decode("utf-8", errors="replace") if sys.version_info[0] == 2 else iprot.readString() + ) + else: + iprot.skip(ftype) + elif fid == 3: + if ftype == TType.STRING: + self.tbl_name = ( + iprot.readString().decode("utf-8", errors="replace") if sys.version_info[0] == 2 else iprot.readString() + ) + else: + iprot.skip(ftype) + elif fid == 4: + if ftype == TType.STRUCT: + self.creation_metadata = CreationMetadata() + self.creation_metadata.read(iprot) + else: + iprot.skip(ftype) + else: + iprot.skip(ftype) + iprot.readFieldEnd() + iprot.readStructEnd() + + def write(self, oprot): + if oprot._fast_encode is not None and self.thrift_spec is not None: + oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec])) + return + oprot.writeStructBegin("update_creation_metadata_args") + if self.catName is not None: + oprot.writeFieldBegin("catName", TType.STRING, 1) + oprot.writeString(self.catName.encode("utf-8") if sys.version_info[0] == 2 else self.catName) + oprot.writeFieldEnd() + if self.dbname is not None: + oprot.writeFieldBegin("dbname", TType.STRING, 2) + oprot.writeString(self.dbname.encode("utf-8") if sys.version_info[0] == 2 else self.dbname) + oprot.writeFieldEnd() + if self.tbl_name is not None: + oprot.writeFieldBegin("tbl_name", TType.STRING, 3) + oprot.writeString(self.tbl_name.encode("utf-8") if sys.version_info[0] == 2 else self.tbl_name) + oprot.writeFieldEnd() + if self.creation_metadata is not None: + oprot.writeFieldBegin("creation_metadata", TType.STRUCT, 4) + self.creation_metadata.write(oprot) + oprot.writeFieldEnd() + oprot.writeFieldStop() + oprot.writeStructEnd() + + def validate(self): + return + + def __repr__(self): + L = ["{}={!r}".format(key, value) for key, value in self.__dict__.items()] + return "{}({})".format(self.__class__.__name__, ", ".join(L)) + + def __eq__(self, other): + return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ + + def __ne__(self, other): + return not (self == other) + + +all_structs.append(update_creation_metadata_args) +update_creation_metadata_args.thrift_spec = ( + None, # 0 + ( + 1, + TType.STRING, + "catName", + "UTF8", + None, + ), # 1 + ( + 2, + TType.STRING, + "dbname", + "UTF8", + None, + ), # 2 + ( + 3, + TType.STRING, + "tbl_name", + "UTF8", + None, + ), # 3 + ( + 4, + TType.STRUCT, + "creation_metadata", + [CreationMetadata, None], + None, + ), # 4 +) + + +class update_creation_metadata_result: + """ + Attributes: + - o1 + - o2 + - o3 + + """ + + def __init__( + self, + o1=None, + o2=None, + o3=None, + ): + self.o1 = o1 + self.o2 = o2 + self.o3 = o3 + + def read(self, iprot): + if ( + iprot._fast_decode is not None + and isinstance(iprot.trans, TTransport.CReadableTransport) + and self.thrift_spec is not None + ): + iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec]) + return + iprot.readStructBegin() + while True: + (fname, ftype, fid) = iprot.readFieldBegin() + if ftype == TType.STOP: + break + if fid == 1: + if ftype == TType.STRUCT: + self.o1 = MetaException.read(iprot) + else: + iprot.skip(ftype) + elif fid == 2: + if ftype == TType.STRUCT: + self.o2 = InvalidOperationException.read(iprot) + else: + iprot.skip(ftype) + elif fid == 3: + if ftype == TType.STRUCT: + self.o3 = UnknownDBException.read(iprot) + else: + iprot.skip(ftype) + else: + iprot.skip(ftype) + iprot.readFieldEnd() + iprot.readStructEnd() + + def write(self, oprot): + if oprot._fast_encode is not None and self.thrift_spec is not None: + oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec])) + return + oprot.writeStructBegin("update_creation_metadata_result") + if self.o1 is not None: + oprot.writeFieldBegin("o1", TType.STRUCT, 1) + self.o1.write(oprot) + oprot.writeFieldEnd() + if self.o2 is not None: + oprot.writeFieldBegin("o2", TType.STRUCT, 2) + self.o2.write(oprot) + oprot.writeFieldEnd() + if self.o3 is not None: + oprot.writeFieldBegin("o3", TType.STRUCT, 3) + self.o3.write(oprot) + oprot.writeFieldEnd() + oprot.writeFieldStop() + oprot.writeStructEnd() + + def validate(self): + return + + def __repr__(self): + L = ["{}={!r}".format(key, value) for key, value in self.__dict__.items()] + return "{}({})".format(self.__class__.__name__, ", ".join(L)) + + def __eq__(self, other): + return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ + + def __ne__(self, other): + return not (self == other) + + +all_structs.append(update_creation_metadata_result) +update_creation_metadata_result.thrift_spec = ( + None, # 0 + ( + 1, + TType.STRUCT, + "o1", + [MetaException, None], + None, + ), # 1 + ( + 2, + TType.STRUCT, + "o2", + [InvalidOperationException, None], + None, + ), # 2 + ( + 3, + TType.STRUCT, + "o3", + [UnknownDBException, None], + None, + ), # 3 +) + + +class get_table_names_by_filter_args: + """ + Attributes: + - dbname + - filter + - max_tables + + """ + + def __init__( + self, + dbname=None, + filter=None, + max_tables=-1, + ): + self.dbname = dbname + self.filter = filter + self.max_tables = max_tables + + def read(self, iprot): + if ( + iprot._fast_decode is not None + and isinstance(iprot.trans, TTransport.CReadableTransport) + and self.thrift_spec is not None + ): + iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec]) + return + iprot.readStructBegin() + while True: + (fname, ftype, fid) = iprot.readFieldBegin() + if ftype == TType.STOP: + break + if fid == 1: + if ftype == TType.STRING: + self.dbname = ( + iprot.readString().decode("utf-8", errors="replace") if sys.version_info[0] == 2 else iprot.readString() + ) + else: + iprot.skip(ftype) + elif fid == 2: + if ftype == TType.STRING: + self.filter = ( + iprot.readString().decode("utf-8", errors="replace") if sys.version_info[0] == 2 else iprot.readString() + ) + else: + iprot.skip(ftype) + elif fid == 3: + if ftype == TType.I16: + self.max_tables = iprot.readI16() + else: + iprot.skip(ftype) + else: + iprot.skip(ftype) + iprot.readFieldEnd() + iprot.readStructEnd() + + def write(self, oprot): + if oprot._fast_encode is not None and self.thrift_spec is not None: + oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec])) + return + oprot.writeStructBegin("get_table_names_by_filter_args") + if self.dbname is not None: + oprot.writeFieldBegin("dbname", TType.STRING, 1) + oprot.writeString(self.dbname.encode("utf-8") if sys.version_info[0] == 2 else self.dbname) + oprot.writeFieldEnd() + if self.filter is not None: + oprot.writeFieldBegin("filter", TType.STRING, 2) + oprot.writeString(self.filter.encode("utf-8") if sys.version_info[0] == 2 else self.filter) + oprot.writeFieldEnd() + if self.max_tables is not None: + oprot.writeFieldBegin("max_tables", TType.I16, 3) + oprot.writeI16(self.max_tables) + oprot.writeFieldEnd() + oprot.writeFieldStop() + oprot.writeStructEnd() + + def validate(self): + return + + def __repr__(self): + L = ["{}={!r}".format(key, value) for key, value in self.__dict__.items()] + return "{}({})".format(self.__class__.__name__, ", ".join(L)) + + def __eq__(self, other): + return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ + + def __ne__(self, other): + return not (self == other) + + +all_structs.append(get_table_names_by_filter_args) +get_table_names_by_filter_args.thrift_spec = ( + None, # 0 + ( + 1, + TType.STRING, + "dbname", + "UTF8", + None, + ), # 1 + ( + 2, + TType.STRING, + "filter", + "UTF8", + None, + ), # 2 + ( + 3, + TType.I16, + "max_tables", + None, + -1, + ), # 3 +) + + +class get_table_names_by_filter_result: + """ + Attributes: + - success + - o1 + - o2 + - o3 + + """ + + def __init__( + self, + success=None, + o1=None, + o2=None, + o3=None, + ): + self.success = success + self.o1 = o1 + self.o2 = o2 + self.o3 = o3 + + def read(self, iprot): + if ( + iprot._fast_decode is not None + and isinstance(iprot.trans, TTransport.CReadableTransport) + and self.thrift_spec is not None + ): + iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec]) + return + iprot.readStructBegin() + while True: + (fname, ftype, fid) = iprot.readFieldBegin() + if ftype == TType.STOP: + break + if fid == 0: + if ftype == TType.LIST: + self.success = [] + (_etype1429, _size1426) = iprot.readListBegin() + for _i1430 in range(_size1426): + _elem1431 = ( + iprot.readString().decode("utf-8", errors="replace") + if sys.version_info[0] == 2 + else iprot.readString() + ) + self.success.append(_elem1431) + iprot.readListEnd() + else: + iprot.skip(ftype) + elif fid == 1: + if ftype == TType.STRUCT: + self.o1 = MetaException.read(iprot) + else: + iprot.skip(ftype) + elif fid == 2: + if ftype == TType.STRUCT: + self.o2 = InvalidOperationException.read(iprot) + else: + iprot.skip(ftype) + elif fid == 3: + if ftype == TType.STRUCT: + self.o3 = UnknownDBException.read(iprot) + else: + iprot.skip(ftype) + else: + iprot.skip(ftype) + iprot.readFieldEnd() + iprot.readStructEnd() + + def write(self, oprot): + if oprot._fast_encode is not None and self.thrift_spec is not None: + oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec])) + return + oprot.writeStructBegin("get_table_names_by_filter_result") + if self.success is not None: + oprot.writeFieldBegin("success", TType.LIST, 0) + oprot.writeListBegin(TType.STRING, len(self.success)) + for iter1432 in self.success: + oprot.writeString(iter1432.encode("utf-8") if sys.version_info[0] == 2 else iter1432) + oprot.writeListEnd() + oprot.writeFieldEnd() + if self.o1 is not None: + oprot.writeFieldBegin("o1", TType.STRUCT, 1) + self.o1.write(oprot) + oprot.writeFieldEnd() + if self.o2 is not None: + oprot.writeFieldBegin("o2", TType.STRUCT, 2) + self.o2.write(oprot) + oprot.writeFieldEnd() + if self.o3 is not None: + oprot.writeFieldBegin("o3", TType.STRUCT, 3) + self.o3.write(oprot) + oprot.writeFieldEnd() + oprot.writeFieldStop() + oprot.writeStructEnd() + + def validate(self): + return + + def __repr__(self): + L = ["{}={!r}".format(key, value) for key, value in self.__dict__.items()] + return "{}({})".format(self.__class__.__name__, ", ".join(L)) + + def __eq__(self, other): + return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ + + def __ne__(self, other): + return not (self == other) + + +all_structs.append(get_table_names_by_filter_result) +get_table_names_by_filter_result.thrift_spec = ( + ( + 0, + TType.LIST, + "success", + (TType.STRING, "UTF8", False), + None, + ), # 0 + ( + 1, + TType.STRUCT, + "o1", + [MetaException, None], + None, + ), # 1 + ( + 2, + TType.STRUCT, + "o2", + [InvalidOperationException, None], + None, + ), # 2 + ( + 3, + TType.STRUCT, + "o3", + [UnknownDBException, None], + None, + ), # 3 +) + + +class alter_table_args: + """ + Attributes: + - dbname + - tbl_name + - new_tbl + + """ + + def __init__( + self, + dbname=None, + tbl_name=None, + new_tbl=None, + ): + self.dbname = dbname + self.tbl_name = tbl_name + self.new_tbl = new_tbl + + def read(self, iprot): + if ( + iprot._fast_decode is not None + and isinstance(iprot.trans, TTransport.CReadableTransport) + and self.thrift_spec is not None + ): + iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec]) + return + iprot.readStructBegin() + while True: + (fname, ftype, fid) = iprot.readFieldBegin() + if ftype == TType.STOP: + break + if fid == 1: + if ftype == TType.STRING: + self.dbname = ( + iprot.readString().decode("utf-8", errors="replace") if sys.version_info[0] == 2 else iprot.readString() + ) + else: + iprot.skip(ftype) + elif fid == 2: + if ftype == TType.STRING: + self.tbl_name = ( + iprot.readString().decode("utf-8", errors="replace") if sys.version_info[0] == 2 else iprot.readString() + ) + else: + iprot.skip(ftype) + elif fid == 3: + if ftype == TType.STRUCT: + self.new_tbl = Table() + self.new_tbl.read(iprot) + else: + iprot.skip(ftype) + else: + iprot.skip(ftype) + iprot.readFieldEnd() + iprot.readStructEnd() + + def write(self, oprot): + if oprot._fast_encode is not None and self.thrift_spec is not None: + oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec])) + return + oprot.writeStructBegin("alter_table_args") + if self.dbname is not None: + oprot.writeFieldBegin("dbname", TType.STRING, 1) + oprot.writeString(self.dbname.encode("utf-8") if sys.version_info[0] == 2 else self.dbname) + oprot.writeFieldEnd() + if self.tbl_name is not None: + oprot.writeFieldBegin("tbl_name", TType.STRING, 2) + oprot.writeString(self.tbl_name.encode("utf-8") if sys.version_info[0] == 2 else self.tbl_name) + oprot.writeFieldEnd() + if self.new_tbl is not None: + oprot.writeFieldBegin("new_tbl", TType.STRUCT, 3) + self.new_tbl.write(oprot) + oprot.writeFieldEnd() + oprot.writeFieldStop() + oprot.writeStructEnd() + + def validate(self): + return + + def __repr__(self): + L = ["{}={!r}".format(key, value) for key, value in self.__dict__.items()] + return "{}({})".format(self.__class__.__name__, ", ".join(L)) + + def __eq__(self, other): + return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ + + def __ne__(self, other): + return not (self == other) + + +all_structs.append(alter_table_args) +alter_table_args.thrift_spec = ( + None, # 0 + ( + 1, + TType.STRING, + "dbname", + "UTF8", + None, + ), # 1 + ( + 2, + TType.STRING, + "tbl_name", + "UTF8", + None, + ), # 2 + ( + 3, + TType.STRUCT, + "new_tbl", + [Table, None], + None, + ), # 3 +) + + +class alter_table_result: + """ + Attributes: + - o1 + - o2 + + """ + + def __init__( + self, + o1=None, + o2=None, + ): + self.o1 = o1 + self.o2 = o2 + + def read(self, iprot): + if ( + iprot._fast_decode is not None + and isinstance(iprot.trans, TTransport.CReadableTransport) + and self.thrift_spec is not None + ): + iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec]) + return + iprot.readStructBegin() + while True: + (fname, ftype, fid) = iprot.readFieldBegin() + if ftype == TType.STOP: + break + if fid == 1: + if ftype == TType.STRUCT: + self.o1 = InvalidOperationException.read(iprot) + else: + iprot.skip(ftype) + elif fid == 2: + if ftype == TType.STRUCT: + self.o2 = MetaException.read(iprot) + else: + iprot.skip(ftype) + else: + iprot.skip(ftype) + iprot.readFieldEnd() + iprot.readStructEnd() + + def write(self, oprot): + if oprot._fast_encode is not None and self.thrift_spec is not None: + oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec])) + return + oprot.writeStructBegin("alter_table_result") + if self.o1 is not None: + oprot.writeFieldBegin("o1", TType.STRUCT, 1) + self.o1.write(oprot) + oprot.writeFieldEnd() + if self.o2 is not None: + oprot.writeFieldBegin("o2", TType.STRUCT, 2) + self.o2.write(oprot) + oprot.writeFieldEnd() + oprot.writeFieldStop() + oprot.writeStructEnd() + + def validate(self): + return + + def __repr__(self): + L = ["{}={!r}".format(key, value) for key, value in self.__dict__.items()] + return "{}({})".format(self.__class__.__name__, ", ".join(L)) + + def __eq__(self, other): + return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ + + def __ne__(self, other): + return not (self == other) + + +all_structs.append(alter_table_result) +alter_table_result.thrift_spec = ( + None, # 0 + ( + 1, + TType.STRUCT, + "o1", + [InvalidOperationException, None], + None, + ), # 1 + ( + 2, + TType.STRUCT, + "o2", + [MetaException, None], + None, + ), # 2 +) + + +class alter_table_with_environment_context_args: + """ + Attributes: + - dbname + - tbl_name + - new_tbl + - environment_context + + """ + + def __init__( + self, + dbname=None, + tbl_name=None, + new_tbl=None, + environment_context=None, + ): + self.dbname = dbname + self.tbl_name = tbl_name + self.new_tbl = new_tbl + self.environment_context = environment_context + + def read(self, iprot): + if ( + iprot._fast_decode is not None + and isinstance(iprot.trans, TTransport.CReadableTransport) + and self.thrift_spec is not None + ): + iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec]) + return + iprot.readStructBegin() + while True: + (fname, ftype, fid) = iprot.readFieldBegin() + if ftype == TType.STOP: + break + if fid == 1: + if ftype == TType.STRING: + self.dbname = ( + iprot.readString().decode("utf-8", errors="replace") if sys.version_info[0] == 2 else iprot.readString() + ) + else: + iprot.skip(ftype) + elif fid == 2: + if ftype == TType.STRING: + self.tbl_name = ( + iprot.readString().decode("utf-8", errors="replace") if sys.version_info[0] == 2 else iprot.readString() + ) + else: + iprot.skip(ftype) + elif fid == 3: + if ftype == TType.STRUCT: + self.new_tbl = Table() + self.new_tbl.read(iprot) + else: + iprot.skip(ftype) + elif fid == 4: + if ftype == TType.STRUCT: + self.environment_context = EnvironmentContext() + self.environment_context.read(iprot) + else: + iprot.skip(ftype) + else: + iprot.skip(ftype) + iprot.readFieldEnd() + iprot.readStructEnd() + + def write(self, oprot): + if oprot._fast_encode is not None and self.thrift_spec is not None: + oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec])) + return + oprot.writeStructBegin("alter_table_with_environment_context_args") + if self.dbname is not None: + oprot.writeFieldBegin("dbname", TType.STRING, 1) + oprot.writeString(self.dbname.encode("utf-8") if sys.version_info[0] == 2 else self.dbname) + oprot.writeFieldEnd() + if self.tbl_name is not None: + oprot.writeFieldBegin("tbl_name", TType.STRING, 2) + oprot.writeString(self.tbl_name.encode("utf-8") if sys.version_info[0] == 2 else self.tbl_name) + oprot.writeFieldEnd() + if self.new_tbl is not None: + oprot.writeFieldBegin("new_tbl", TType.STRUCT, 3) + self.new_tbl.write(oprot) + oprot.writeFieldEnd() + if self.environment_context is not None: + oprot.writeFieldBegin("environment_context", TType.STRUCT, 4) + self.environment_context.write(oprot) + oprot.writeFieldEnd() + oprot.writeFieldStop() + oprot.writeStructEnd() + + def validate(self): + return + + def __repr__(self): + L = ["{}={!r}".format(key, value) for key, value in self.__dict__.items()] + return "{}({})".format(self.__class__.__name__, ", ".join(L)) + + def __eq__(self, other): + return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ + + def __ne__(self, other): + return not (self == other) + + +all_structs.append(alter_table_with_environment_context_args) +alter_table_with_environment_context_args.thrift_spec = ( + None, # 0 + ( + 1, + TType.STRING, + "dbname", + "UTF8", + None, + ), # 1 + ( + 2, + TType.STRING, + "tbl_name", + "UTF8", + None, + ), # 2 + ( + 3, + TType.STRUCT, + "new_tbl", + [Table, None], + None, + ), # 3 + ( + 4, + TType.STRUCT, + "environment_context", + [EnvironmentContext, None], + None, + ), # 4 +) + + +class alter_table_with_environment_context_result: + """ + Attributes: + - o1 + - o2 + + """ + + def __init__( + self, + o1=None, + o2=None, + ): + self.o1 = o1 + self.o2 = o2 + + def read(self, iprot): + if ( + iprot._fast_decode is not None + and isinstance(iprot.trans, TTransport.CReadableTransport) + and self.thrift_spec is not None + ): + iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec]) + return + iprot.readStructBegin() + while True: + (fname, ftype, fid) = iprot.readFieldBegin() + if ftype == TType.STOP: + break + if fid == 1: + if ftype == TType.STRUCT: + self.o1 = InvalidOperationException.read(iprot) + else: + iprot.skip(ftype) + elif fid == 2: + if ftype == TType.STRUCT: + self.o2 = MetaException.read(iprot) + else: + iprot.skip(ftype) + else: + iprot.skip(ftype) + iprot.readFieldEnd() + iprot.readStructEnd() + + def write(self, oprot): + if oprot._fast_encode is not None and self.thrift_spec is not None: + oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec])) + return + oprot.writeStructBegin("alter_table_with_environment_context_result") + if self.o1 is not None: + oprot.writeFieldBegin("o1", TType.STRUCT, 1) + self.o1.write(oprot) + oprot.writeFieldEnd() + if self.o2 is not None: + oprot.writeFieldBegin("o2", TType.STRUCT, 2) + self.o2.write(oprot) + oprot.writeFieldEnd() + oprot.writeFieldStop() + oprot.writeStructEnd() + + def validate(self): + return + + def __repr__(self): + L = ["{}={!r}".format(key, value) for key, value in self.__dict__.items()] + return "{}({})".format(self.__class__.__name__, ", ".join(L)) + + def __eq__(self, other): + return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ + + def __ne__(self, other): + return not (self == other) + + +all_structs.append(alter_table_with_environment_context_result) +alter_table_with_environment_context_result.thrift_spec = ( + None, # 0 + ( + 1, + TType.STRUCT, + "o1", + [InvalidOperationException, None], + None, + ), # 1 + ( + 2, + TType.STRUCT, + "o2", + [MetaException, None], + None, + ), # 2 +) + + +class alter_table_with_cascade_args: + """ + Attributes: + - dbname + - tbl_name + - new_tbl + - cascade + + """ + + def __init__( + self, + dbname=None, + tbl_name=None, + new_tbl=None, + cascade=None, + ): + self.dbname = dbname + self.tbl_name = tbl_name + self.new_tbl = new_tbl + self.cascade = cascade + + def read(self, iprot): + if ( + iprot._fast_decode is not None + and isinstance(iprot.trans, TTransport.CReadableTransport) + and self.thrift_spec is not None + ): + iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec]) + return + iprot.readStructBegin() + while True: + (fname, ftype, fid) = iprot.readFieldBegin() + if ftype == TType.STOP: + break + if fid == 1: + if ftype == TType.STRING: + self.dbname = ( + iprot.readString().decode("utf-8", errors="replace") if sys.version_info[0] == 2 else iprot.readString() + ) + else: + iprot.skip(ftype) + elif fid == 2: + if ftype == TType.STRING: + self.tbl_name = ( + iprot.readString().decode("utf-8", errors="replace") if sys.version_info[0] == 2 else iprot.readString() + ) + else: + iprot.skip(ftype) + elif fid == 3: + if ftype == TType.STRUCT: + self.new_tbl = Table() + self.new_tbl.read(iprot) + else: + iprot.skip(ftype) + elif fid == 4: + if ftype == TType.BOOL: + self.cascade = iprot.readBool() + else: + iprot.skip(ftype) + else: + iprot.skip(ftype) + iprot.readFieldEnd() + iprot.readStructEnd() + + def write(self, oprot): + if oprot._fast_encode is not None and self.thrift_spec is not None: + oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec])) + return + oprot.writeStructBegin("alter_table_with_cascade_args") + if self.dbname is not None: + oprot.writeFieldBegin("dbname", TType.STRING, 1) + oprot.writeString(self.dbname.encode("utf-8") if sys.version_info[0] == 2 else self.dbname) + oprot.writeFieldEnd() + if self.tbl_name is not None: + oprot.writeFieldBegin("tbl_name", TType.STRING, 2) + oprot.writeString(self.tbl_name.encode("utf-8") if sys.version_info[0] == 2 else self.tbl_name) + oprot.writeFieldEnd() + if self.new_tbl is not None: + oprot.writeFieldBegin("new_tbl", TType.STRUCT, 3) + self.new_tbl.write(oprot) + oprot.writeFieldEnd() + if self.cascade is not None: + oprot.writeFieldBegin("cascade", TType.BOOL, 4) + oprot.writeBool(self.cascade) + oprot.writeFieldEnd() + oprot.writeFieldStop() + oprot.writeStructEnd() + + def validate(self): + return + + def __repr__(self): + L = ["{}={!r}".format(key, value) for key, value in self.__dict__.items()] + return "{}({})".format(self.__class__.__name__, ", ".join(L)) + + def __eq__(self, other): + return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ + + def __ne__(self, other): + return not (self == other) + + +all_structs.append(alter_table_with_cascade_args) +alter_table_with_cascade_args.thrift_spec = ( + None, # 0 + ( + 1, + TType.STRING, + "dbname", + "UTF8", + None, + ), # 1 + ( + 2, + TType.STRING, + "tbl_name", + "UTF8", + None, + ), # 2 + ( + 3, + TType.STRUCT, + "new_tbl", + [Table, None], + None, + ), # 3 + ( + 4, + TType.BOOL, + "cascade", + None, + None, + ), # 4 +) + + +class alter_table_with_cascade_result: + """ + Attributes: + - o1 + - o2 + + """ + + def __init__( + self, + o1=None, + o2=None, + ): + self.o1 = o1 + self.o2 = o2 + + def read(self, iprot): + if ( + iprot._fast_decode is not None + and isinstance(iprot.trans, TTransport.CReadableTransport) + and self.thrift_spec is not None + ): + iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec]) + return + iprot.readStructBegin() + while True: + (fname, ftype, fid) = iprot.readFieldBegin() + if ftype == TType.STOP: + break + if fid == 1: + if ftype == TType.STRUCT: + self.o1 = InvalidOperationException.read(iprot) + else: + iprot.skip(ftype) + elif fid == 2: + if ftype == TType.STRUCT: + self.o2 = MetaException.read(iprot) + else: + iprot.skip(ftype) + else: + iprot.skip(ftype) + iprot.readFieldEnd() + iprot.readStructEnd() + + def write(self, oprot): + if oprot._fast_encode is not None and self.thrift_spec is not None: + oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec])) + return + oprot.writeStructBegin("alter_table_with_cascade_result") + if self.o1 is not None: + oprot.writeFieldBegin("o1", TType.STRUCT, 1) + self.o1.write(oprot) + oprot.writeFieldEnd() + if self.o2 is not None: + oprot.writeFieldBegin("o2", TType.STRUCT, 2) + self.o2.write(oprot) + oprot.writeFieldEnd() + oprot.writeFieldStop() + oprot.writeStructEnd() + + def validate(self): + return + + def __repr__(self): + L = ["{}={!r}".format(key, value) for key, value in self.__dict__.items()] + return "{}({})".format(self.__class__.__name__, ", ".join(L)) + + def __eq__(self, other): + return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ + + def __ne__(self, other): + return not (self == other) + + +all_structs.append(alter_table_with_cascade_result) +alter_table_with_cascade_result.thrift_spec = ( + None, # 0 + ( + 1, + TType.STRUCT, + "o1", + [InvalidOperationException, None], + None, + ), # 1 + ( + 2, + TType.STRUCT, + "o2", + [MetaException, None], + None, + ), # 2 +) + + +class alter_table_req_args: + """ + Attributes: + - req + + """ + + def __init__( + self, + req=None, + ): + self.req = req + + def read(self, iprot): + if ( + iprot._fast_decode is not None + and isinstance(iprot.trans, TTransport.CReadableTransport) + and self.thrift_spec is not None + ): + iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec]) + return + iprot.readStructBegin() + while True: + (fname, ftype, fid) = iprot.readFieldBegin() + if ftype == TType.STOP: + break + if fid == 1: + if ftype == TType.STRUCT: + self.req = AlterTableRequest() + self.req.read(iprot) + else: + iprot.skip(ftype) + else: + iprot.skip(ftype) + iprot.readFieldEnd() + iprot.readStructEnd() + + def write(self, oprot): + if oprot._fast_encode is not None and self.thrift_spec is not None: + oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec])) + return + oprot.writeStructBegin("alter_table_req_args") + if self.req is not None: + oprot.writeFieldBegin("req", TType.STRUCT, 1) + self.req.write(oprot) + oprot.writeFieldEnd() + oprot.writeFieldStop() + oprot.writeStructEnd() + + def validate(self): + return + + def __repr__(self): + L = ["{}={!r}".format(key, value) for key, value in self.__dict__.items()] + return "{}({})".format(self.__class__.__name__, ", ".join(L)) + + def __eq__(self, other): + return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ + + def __ne__(self, other): + return not (self == other) + + +all_structs.append(alter_table_req_args) +alter_table_req_args.thrift_spec = ( + None, # 0 + ( + 1, + TType.STRUCT, + "req", + [AlterTableRequest, None], + None, + ), # 1 +) + + +class alter_table_req_result: + """ + Attributes: + - success + - o1 + - o2 + + """ + + def __init__( + self, + success=None, + o1=None, + o2=None, + ): + self.success = success + self.o1 = o1 + self.o2 = o2 + + def read(self, iprot): + if ( + iprot._fast_decode is not None + and isinstance(iprot.trans, TTransport.CReadableTransport) + and self.thrift_spec is not None + ): + iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec]) + return + iprot.readStructBegin() + while True: + (fname, ftype, fid) = iprot.readFieldBegin() + if ftype == TType.STOP: + break + if fid == 0: + if ftype == TType.STRUCT: + self.success = AlterTableResponse() + self.success.read(iprot) + else: + iprot.skip(ftype) + elif fid == 1: + if ftype == TType.STRUCT: + self.o1 = InvalidOperationException.read(iprot) + else: + iprot.skip(ftype) + elif fid == 2: + if ftype == TType.STRUCT: + self.o2 = MetaException.read(iprot) + else: + iprot.skip(ftype) + else: + iprot.skip(ftype) + iprot.readFieldEnd() + iprot.readStructEnd() + + def write(self, oprot): + if oprot._fast_encode is not None and self.thrift_spec is not None: + oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec])) + return + oprot.writeStructBegin("alter_table_req_result") + if self.success is not None: + oprot.writeFieldBegin("success", TType.STRUCT, 0) + self.success.write(oprot) + oprot.writeFieldEnd() + if self.o1 is not None: + oprot.writeFieldBegin("o1", TType.STRUCT, 1) + self.o1.write(oprot) + oprot.writeFieldEnd() + if self.o2 is not None: + oprot.writeFieldBegin("o2", TType.STRUCT, 2) + self.o2.write(oprot) + oprot.writeFieldEnd() + oprot.writeFieldStop() + oprot.writeStructEnd() + + def validate(self): + return + + def __repr__(self): + L = ["{}={!r}".format(key, value) for key, value in self.__dict__.items()] + return "{}({})".format(self.__class__.__name__, ", ".join(L)) + + def __eq__(self, other): + return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ + + def __ne__(self, other): + return not (self == other) + + +all_structs.append(alter_table_req_result) +alter_table_req_result.thrift_spec = ( + ( + 0, + TType.STRUCT, + "success", + [AlterTableResponse, None], + None, + ), # 0 + ( + 1, + TType.STRUCT, + "o1", + [InvalidOperationException, None], + None, + ), # 1 + ( + 2, + TType.STRUCT, + "o2", + [MetaException, None], + None, + ), # 2 +) + + +class add_partition_args: + """ + Attributes: + - new_part + + """ + + def __init__( + self, + new_part=None, + ): + self.new_part = new_part + + def read(self, iprot): + if ( + iprot._fast_decode is not None + and isinstance(iprot.trans, TTransport.CReadableTransport) + and self.thrift_spec is not None + ): + iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec]) + return + iprot.readStructBegin() + while True: + (fname, ftype, fid) = iprot.readFieldBegin() + if ftype == TType.STOP: + break + if fid == 1: + if ftype == TType.STRUCT: + self.new_part = Partition() + self.new_part.read(iprot) + else: + iprot.skip(ftype) + else: + iprot.skip(ftype) + iprot.readFieldEnd() + iprot.readStructEnd() + + def write(self, oprot): + if oprot._fast_encode is not None and self.thrift_spec is not None: + oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec])) + return + oprot.writeStructBegin("add_partition_args") + if self.new_part is not None: + oprot.writeFieldBegin("new_part", TType.STRUCT, 1) + self.new_part.write(oprot) + oprot.writeFieldEnd() + oprot.writeFieldStop() + oprot.writeStructEnd() + + def validate(self): + return + + def __repr__(self): + L = ["{}={!r}".format(key, value) for key, value in self.__dict__.items()] + return "{}({})".format(self.__class__.__name__, ", ".join(L)) + + def __eq__(self, other): + return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ + + def __ne__(self, other): + return not (self == other) + + +all_structs.append(add_partition_args) +add_partition_args.thrift_spec = ( + None, # 0 + ( + 1, + TType.STRUCT, + "new_part", + [Partition, None], + None, + ), # 1 +) + + +class add_partition_result: + """ + Attributes: + - success + - o1 + - o2 + - o3 + + """ + + def __init__( + self, + success=None, + o1=None, + o2=None, + o3=None, + ): + self.success = success + self.o1 = o1 + self.o2 = o2 + self.o3 = o3 + + def read(self, iprot): + if ( + iprot._fast_decode is not None + and isinstance(iprot.trans, TTransport.CReadableTransport) + and self.thrift_spec is not None + ): + iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec]) + return + iprot.readStructBegin() + while True: + (fname, ftype, fid) = iprot.readFieldBegin() + if ftype == TType.STOP: + break + if fid == 0: + if ftype == TType.STRUCT: + self.success = Partition() + self.success.read(iprot) + else: + iprot.skip(ftype) + elif fid == 1: + if ftype == TType.STRUCT: + self.o1 = InvalidObjectException.read(iprot) + else: + iprot.skip(ftype) + elif fid == 2: + if ftype == TType.STRUCT: + self.o2 = AlreadyExistsException.read(iprot) + else: + iprot.skip(ftype) + elif fid == 3: + if ftype == TType.STRUCT: + self.o3 = MetaException.read(iprot) + else: + iprot.skip(ftype) + else: + iprot.skip(ftype) + iprot.readFieldEnd() + iprot.readStructEnd() + + def write(self, oprot): + if oprot._fast_encode is not None and self.thrift_spec is not None: + oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec])) + return + oprot.writeStructBegin("add_partition_result") + if self.success is not None: + oprot.writeFieldBegin("success", TType.STRUCT, 0) + self.success.write(oprot) + oprot.writeFieldEnd() + if self.o1 is not None: + oprot.writeFieldBegin("o1", TType.STRUCT, 1) + self.o1.write(oprot) + oprot.writeFieldEnd() + if self.o2 is not None: + oprot.writeFieldBegin("o2", TType.STRUCT, 2) + self.o2.write(oprot) + oprot.writeFieldEnd() + if self.o3 is not None: + oprot.writeFieldBegin("o3", TType.STRUCT, 3) + self.o3.write(oprot) + oprot.writeFieldEnd() + oprot.writeFieldStop() + oprot.writeStructEnd() + + def validate(self): + return + + def __repr__(self): + L = ["{}={!r}".format(key, value) for key, value in self.__dict__.items()] + return "{}({})".format(self.__class__.__name__, ", ".join(L)) + + def __eq__(self, other): + return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ + + def __ne__(self, other): + return not (self == other) + + +all_structs.append(add_partition_result) +add_partition_result.thrift_spec = ( + ( + 0, + TType.STRUCT, + "success", + [Partition, None], + None, + ), # 0 + ( + 1, + TType.STRUCT, + "o1", + [InvalidObjectException, None], + None, + ), # 1 + ( + 2, + TType.STRUCT, + "o2", + [AlreadyExistsException, None], + None, + ), # 2 + ( + 3, + TType.STRUCT, + "o3", + [MetaException, None], + None, + ), # 3 +) + + +class add_partition_with_environment_context_args: + """ + Attributes: + - new_part + - environment_context + + """ + + def __init__( + self, + new_part=None, + environment_context=None, + ): + self.new_part = new_part + self.environment_context = environment_context + + def read(self, iprot): + if ( + iprot._fast_decode is not None + and isinstance(iprot.trans, TTransport.CReadableTransport) + and self.thrift_spec is not None + ): + iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec]) + return + iprot.readStructBegin() + while True: + (fname, ftype, fid) = iprot.readFieldBegin() + if ftype == TType.STOP: + break + if fid == 1: + if ftype == TType.STRUCT: + self.new_part = Partition() + self.new_part.read(iprot) + else: + iprot.skip(ftype) + elif fid == 2: + if ftype == TType.STRUCT: + self.environment_context = EnvironmentContext() + self.environment_context.read(iprot) + else: + iprot.skip(ftype) + else: + iprot.skip(ftype) + iprot.readFieldEnd() + iprot.readStructEnd() + + def write(self, oprot): + if oprot._fast_encode is not None and self.thrift_spec is not None: + oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec])) + return + oprot.writeStructBegin("add_partition_with_environment_context_args") + if self.new_part is not None: + oprot.writeFieldBegin("new_part", TType.STRUCT, 1) + self.new_part.write(oprot) + oprot.writeFieldEnd() + if self.environment_context is not None: + oprot.writeFieldBegin("environment_context", TType.STRUCT, 2) + self.environment_context.write(oprot) + oprot.writeFieldEnd() + oprot.writeFieldStop() + oprot.writeStructEnd() + + def validate(self): + return + + def __repr__(self): + L = ["{}={!r}".format(key, value) for key, value in self.__dict__.items()] + return "{}({})".format(self.__class__.__name__, ", ".join(L)) + + def __eq__(self, other): + return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ + + def __ne__(self, other): + return not (self == other) + + +all_structs.append(add_partition_with_environment_context_args) +add_partition_with_environment_context_args.thrift_spec = ( + None, # 0 + ( + 1, + TType.STRUCT, + "new_part", + [Partition, None], + None, + ), # 1 + ( + 2, + TType.STRUCT, + "environment_context", + [EnvironmentContext, None], + None, + ), # 2 +) + + +class add_partition_with_environment_context_result: + """ + Attributes: + - success + - o1 + - o2 + - o3 + + """ + + def __init__( + self, + success=None, + o1=None, + o2=None, + o3=None, + ): + self.success = success + self.o1 = o1 + self.o2 = o2 + self.o3 = o3 + + def read(self, iprot): + if ( + iprot._fast_decode is not None + and isinstance(iprot.trans, TTransport.CReadableTransport) + and self.thrift_spec is not None + ): + iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec]) + return + iprot.readStructBegin() + while True: + (fname, ftype, fid) = iprot.readFieldBegin() + if ftype == TType.STOP: + break + if fid == 0: + if ftype == TType.STRUCT: + self.success = Partition() + self.success.read(iprot) + else: + iprot.skip(ftype) + elif fid == 1: + if ftype == TType.STRUCT: + self.o1 = InvalidObjectException.read(iprot) + else: + iprot.skip(ftype) + elif fid == 2: + if ftype == TType.STRUCT: + self.o2 = AlreadyExistsException.read(iprot) + else: + iprot.skip(ftype) + elif fid == 3: + if ftype == TType.STRUCT: + self.o3 = MetaException.read(iprot) + else: + iprot.skip(ftype) + else: + iprot.skip(ftype) + iprot.readFieldEnd() + iprot.readStructEnd() + + def write(self, oprot): + if oprot._fast_encode is not None and self.thrift_spec is not None: + oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec])) + return + oprot.writeStructBegin("add_partition_with_environment_context_result") + if self.success is not None: + oprot.writeFieldBegin("success", TType.STRUCT, 0) + self.success.write(oprot) + oprot.writeFieldEnd() + if self.o1 is not None: + oprot.writeFieldBegin("o1", TType.STRUCT, 1) + self.o1.write(oprot) + oprot.writeFieldEnd() + if self.o2 is not None: + oprot.writeFieldBegin("o2", TType.STRUCT, 2) + self.o2.write(oprot) + oprot.writeFieldEnd() + if self.o3 is not None: + oprot.writeFieldBegin("o3", TType.STRUCT, 3) + self.o3.write(oprot) + oprot.writeFieldEnd() + oprot.writeFieldStop() + oprot.writeStructEnd() + + def validate(self): + return + + def __repr__(self): + L = ["{}={!r}".format(key, value) for key, value in self.__dict__.items()] + return "{}({})".format(self.__class__.__name__, ", ".join(L)) + + def __eq__(self, other): + return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ + + def __ne__(self, other): + return not (self == other) + + +all_structs.append(add_partition_with_environment_context_result) +add_partition_with_environment_context_result.thrift_spec = ( + ( + 0, + TType.STRUCT, + "success", + [Partition, None], + None, + ), # 0 + ( + 1, + TType.STRUCT, + "o1", + [InvalidObjectException, None], + None, + ), # 1 + ( + 2, + TType.STRUCT, + "o2", + [AlreadyExistsException, None], + None, + ), # 2 + ( + 3, + TType.STRUCT, + "o3", + [MetaException, None], + None, + ), # 3 +) + + +class add_partitions_args: + """ + Attributes: + - new_parts + + """ + + def __init__( + self, + new_parts=None, + ): + self.new_parts = new_parts + + def read(self, iprot): + if ( + iprot._fast_decode is not None + and isinstance(iprot.trans, TTransport.CReadableTransport) + and self.thrift_spec is not None + ): + iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec]) + return + iprot.readStructBegin() + while True: + (fname, ftype, fid) = iprot.readFieldBegin() + if ftype == TType.STOP: + break + if fid == 1: + if ftype == TType.LIST: + self.new_parts = [] + (_etype1436, _size1433) = iprot.readListBegin() + for _i1437 in range(_size1433): + _elem1438 = Partition() + _elem1438.read(iprot) + self.new_parts.append(_elem1438) + iprot.readListEnd() + else: + iprot.skip(ftype) + else: + iprot.skip(ftype) + iprot.readFieldEnd() + iprot.readStructEnd() + + def write(self, oprot): + if oprot._fast_encode is not None and self.thrift_spec is not None: + oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec])) + return + oprot.writeStructBegin("add_partitions_args") + if self.new_parts is not None: + oprot.writeFieldBegin("new_parts", TType.LIST, 1) + oprot.writeListBegin(TType.STRUCT, len(self.new_parts)) + for iter1439 in self.new_parts: + iter1439.write(oprot) + oprot.writeListEnd() + oprot.writeFieldEnd() + oprot.writeFieldStop() + oprot.writeStructEnd() + + def validate(self): + return + + def __repr__(self): + L = ["{}={!r}".format(key, value) for key, value in self.__dict__.items()] + return "{}({})".format(self.__class__.__name__, ", ".join(L)) + + def __eq__(self, other): + return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ + + def __ne__(self, other): + return not (self == other) + + +all_structs.append(add_partitions_args) +add_partitions_args.thrift_spec = ( + None, # 0 + ( + 1, + TType.LIST, + "new_parts", + (TType.STRUCT, [Partition, None], False), + None, + ), # 1 +) + + +class add_partitions_result: + """ + Attributes: + - success + - o1 + - o2 + - o3 + + """ + + def __init__( + self, + success=None, + o1=None, + o2=None, + o3=None, + ): + self.success = success + self.o1 = o1 + self.o2 = o2 + self.o3 = o3 + + def read(self, iprot): + if ( + iprot._fast_decode is not None + and isinstance(iprot.trans, TTransport.CReadableTransport) + and self.thrift_spec is not None + ): + iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec]) + return + iprot.readStructBegin() + while True: + (fname, ftype, fid) = iprot.readFieldBegin() + if ftype == TType.STOP: + break + if fid == 0: + if ftype == TType.I32: + self.success = iprot.readI32() + else: + iprot.skip(ftype) + elif fid == 1: + if ftype == TType.STRUCT: + self.o1 = InvalidObjectException.read(iprot) + else: + iprot.skip(ftype) + elif fid == 2: + if ftype == TType.STRUCT: + self.o2 = AlreadyExistsException.read(iprot) + else: + iprot.skip(ftype) + elif fid == 3: + if ftype == TType.STRUCT: + self.o3 = MetaException.read(iprot) + else: + iprot.skip(ftype) + else: + iprot.skip(ftype) + iprot.readFieldEnd() + iprot.readStructEnd() + + def write(self, oprot): + if oprot._fast_encode is not None and self.thrift_spec is not None: + oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec])) + return + oprot.writeStructBegin("add_partitions_result") + if self.success is not None: + oprot.writeFieldBegin("success", TType.I32, 0) + oprot.writeI32(self.success) + oprot.writeFieldEnd() + if self.o1 is not None: + oprot.writeFieldBegin("o1", TType.STRUCT, 1) + self.o1.write(oprot) + oprot.writeFieldEnd() + if self.o2 is not None: + oprot.writeFieldBegin("o2", TType.STRUCT, 2) + self.o2.write(oprot) + oprot.writeFieldEnd() + if self.o3 is not None: + oprot.writeFieldBegin("o3", TType.STRUCT, 3) + self.o3.write(oprot) + oprot.writeFieldEnd() + oprot.writeFieldStop() + oprot.writeStructEnd() + + def validate(self): + return + + def __repr__(self): + L = ["{}={!r}".format(key, value) for key, value in self.__dict__.items()] + return "{}({})".format(self.__class__.__name__, ", ".join(L)) + + def __eq__(self, other): + return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ + + def __ne__(self, other): + return not (self == other) + + +all_structs.append(add_partitions_result) +add_partitions_result.thrift_spec = ( + ( + 0, + TType.I32, + "success", + None, + None, + ), # 0 + ( + 1, + TType.STRUCT, + "o1", + [InvalidObjectException, None], + None, + ), # 1 + ( + 2, + TType.STRUCT, + "o2", + [AlreadyExistsException, None], + None, + ), # 2 + ( + 3, + TType.STRUCT, + "o3", + [MetaException, None], + None, + ), # 3 +) + + +class add_partitions_pspec_args: + """ + Attributes: + - new_parts + + """ + + def __init__( + self, + new_parts=None, + ): + self.new_parts = new_parts + + def read(self, iprot): + if ( + iprot._fast_decode is not None + and isinstance(iprot.trans, TTransport.CReadableTransport) + and self.thrift_spec is not None + ): + iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec]) + return + iprot.readStructBegin() + while True: + (fname, ftype, fid) = iprot.readFieldBegin() + if ftype == TType.STOP: + break + if fid == 1: + if ftype == TType.LIST: + self.new_parts = [] + (_etype1443, _size1440) = iprot.readListBegin() + for _i1444 in range(_size1440): + _elem1445 = PartitionSpec() + _elem1445.read(iprot) + self.new_parts.append(_elem1445) + iprot.readListEnd() + else: + iprot.skip(ftype) + else: + iprot.skip(ftype) + iprot.readFieldEnd() + iprot.readStructEnd() + + def write(self, oprot): + if oprot._fast_encode is not None and self.thrift_spec is not None: + oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec])) + return + oprot.writeStructBegin("add_partitions_pspec_args") + if self.new_parts is not None: + oprot.writeFieldBegin("new_parts", TType.LIST, 1) + oprot.writeListBegin(TType.STRUCT, len(self.new_parts)) + for iter1446 in self.new_parts: + iter1446.write(oprot) + oprot.writeListEnd() + oprot.writeFieldEnd() + oprot.writeFieldStop() + oprot.writeStructEnd() + + def validate(self): + return + + def __repr__(self): + L = ["{}={!r}".format(key, value) for key, value in self.__dict__.items()] + return "{}({})".format(self.__class__.__name__, ", ".join(L)) + + def __eq__(self, other): + return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ + + def __ne__(self, other): + return not (self == other) + + +all_structs.append(add_partitions_pspec_args) +add_partitions_pspec_args.thrift_spec = ( + None, # 0 + ( + 1, + TType.LIST, + "new_parts", + (TType.STRUCT, [PartitionSpec, None], False), + None, + ), # 1 +) + + +class add_partitions_pspec_result: + """ + Attributes: + - success + - o1 + - o2 + - o3 + + """ + + def __init__( + self, + success=None, + o1=None, + o2=None, + o3=None, + ): + self.success = success + self.o1 = o1 + self.o2 = o2 + self.o3 = o3 + + def read(self, iprot): + if ( + iprot._fast_decode is not None + and isinstance(iprot.trans, TTransport.CReadableTransport) + and self.thrift_spec is not None + ): + iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec]) + return + iprot.readStructBegin() + while True: + (fname, ftype, fid) = iprot.readFieldBegin() + if ftype == TType.STOP: + break + if fid == 0: + if ftype == TType.I32: + self.success = iprot.readI32() + else: + iprot.skip(ftype) + elif fid == 1: + if ftype == TType.STRUCT: + self.o1 = InvalidObjectException.read(iprot) + else: + iprot.skip(ftype) + elif fid == 2: + if ftype == TType.STRUCT: + self.o2 = AlreadyExistsException.read(iprot) + else: + iprot.skip(ftype) + elif fid == 3: + if ftype == TType.STRUCT: + self.o3 = MetaException.read(iprot) + else: + iprot.skip(ftype) + else: + iprot.skip(ftype) + iprot.readFieldEnd() + iprot.readStructEnd() + + def write(self, oprot): + if oprot._fast_encode is not None and self.thrift_spec is not None: + oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec])) + return + oprot.writeStructBegin("add_partitions_pspec_result") + if self.success is not None: + oprot.writeFieldBegin("success", TType.I32, 0) + oprot.writeI32(self.success) + oprot.writeFieldEnd() + if self.o1 is not None: + oprot.writeFieldBegin("o1", TType.STRUCT, 1) + self.o1.write(oprot) + oprot.writeFieldEnd() + if self.o2 is not None: + oprot.writeFieldBegin("o2", TType.STRUCT, 2) + self.o2.write(oprot) + oprot.writeFieldEnd() + if self.o3 is not None: + oprot.writeFieldBegin("o3", TType.STRUCT, 3) + self.o3.write(oprot) + oprot.writeFieldEnd() + oprot.writeFieldStop() + oprot.writeStructEnd() + + def validate(self): + return + + def __repr__(self): + L = ["{}={!r}".format(key, value) for key, value in self.__dict__.items()] + return "{}({})".format(self.__class__.__name__, ", ".join(L)) + + def __eq__(self, other): + return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ + + def __ne__(self, other): + return not (self == other) + + +all_structs.append(add_partitions_pspec_result) +add_partitions_pspec_result.thrift_spec = ( + ( + 0, + TType.I32, + "success", + None, + None, + ), # 0 + ( + 1, + TType.STRUCT, + "o1", + [InvalidObjectException, None], + None, + ), # 1 + ( + 2, + TType.STRUCT, + "o2", + [AlreadyExistsException, None], + None, + ), # 2 + ( + 3, + TType.STRUCT, + "o3", + [MetaException, None], + None, + ), # 3 +) + + +class append_partition_args: + """ + Attributes: + - db_name + - tbl_name + - part_vals + + """ + + def __init__( + self, + db_name=None, + tbl_name=None, + part_vals=None, + ): + self.db_name = db_name + self.tbl_name = tbl_name + self.part_vals = part_vals + + def read(self, iprot): + if ( + iprot._fast_decode is not None + and isinstance(iprot.trans, TTransport.CReadableTransport) + and self.thrift_spec is not None + ): + iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec]) + return + iprot.readStructBegin() + while True: + (fname, ftype, fid) = iprot.readFieldBegin() + if ftype == TType.STOP: + break + if fid == 1: + if ftype == TType.STRING: + self.db_name = ( + iprot.readString().decode("utf-8", errors="replace") if sys.version_info[0] == 2 else iprot.readString() + ) + else: + iprot.skip(ftype) + elif fid == 2: + if ftype == TType.STRING: + self.tbl_name = ( + iprot.readString().decode("utf-8", errors="replace") if sys.version_info[0] == 2 else iprot.readString() + ) + else: + iprot.skip(ftype) + elif fid == 3: + if ftype == TType.LIST: + self.part_vals = [] + (_etype1450, _size1447) = iprot.readListBegin() + for _i1451 in range(_size1447): + _elem1452 = ( + iprot.readString().decode("utf-8", errors="replace") + if sys.version_info[0] == 2 + else iprot.readString() + ) + self.part_vals.append(_elem1452) + iprot.readListEnd() + else: + iprot.skip(ftype) + else: + iprot.skip(ftype) + iprot.readFieldEnd() + iprot.readStructEnd() + + def write(self, oprot): + if oprot._fast_encode is not None and self.thrift_spec is not None: + oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec])) + return + oprot.writeStructBegin("append_partition_args") + if self.db_name is not None: + oprot.writeFieldBegin("db_name", TType.STRING, 1) + oprot.writeString(self.db_name.encode("utf-8") if sys.version_info[0] == 2 else self.db_name) + oprot.writeFieldEnd() + if self.tbl_name is not None: + oprot.writeFieldBegin("tbl_name", TType.STRING, 2) + oprot.writeString(self.tbl_name.encode("utf-8") if sys.version_info[0] == 2 else self.tbl_name) + oprot.writeFieldEnd() + if self.part_vals is not None: + oprot.writeFieldBegin("part_vals", TType.LIST, 3) + oprot.writeListBegin(TType.STRING, len(self.part_vals)) + for iter1453 in self.part_vals: + oprot.writeString(iter1453.encode("utf-8") if sys.version_info[0] == 2 else iter1453) + oprot.writeListEnd() + oprot.writeFieldEnd() + oprot.writeFieldStop() + oprot.writeStructEnd() + + def validate(self): + return + + def __repr__(self): + L = ["{}={!r}".format(key, value) for key, value in self.__dict__.items()] + return "{}({})".format(self.__class__.__name__, ", ".join(L)) + + def __eq__(self, other): + return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ + + def __ne__(self, other): + return not (self == other) + + +all_structs.append(append_partition_args) +append_partition_args.thrift_spec = ( + None, # 0 + ( + 1, + TType.STRING, + "db_name", + "UTF8", + None, + ), # 1 + ( + 2, + TType.STRING, + "tbl_name", + "UTF8", + None, + ), # 2 + ( + 3, + TType.LIST, + "part_vals", + (TType.STRING, "UTF8", False), + None, + ), # 3 +) + + +class append_partition_result: + """ + Attributes: + - success + - o1 + - o2 + - o3 + + """ + + def __init__( + self, + success=None, + o1=None, + o2=None, + o3=None, + ): + self.success = success + self.o1 = o1 + self.o2 = o2 + self.o3 = o3 + + def read(self, iprot): + if ( + iprot._fast_decode is not None + and isinstance(iprot.trans, TTransport.CReadableTransport) + and self.thrift_spec is not None + ): + iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec]) + return + iprot.readStructBegin() + while True: + (fname, ftype, fid) = iprot.readFieldBegin() + if ftype == TType.STOP: + break + if fid == 0: + if ftype == TType.STRUCT: + self.success = Partition() + self.success.read(iprot) + else: + iprot.skip(ftype) + elif fid == 1: + if ftype == TType.STRUCT: + self.o1 = InvalidObjectException.read(iprot) + else: + iprot.skip(ftype) + elif fid == 2: + if ftype == TType.STRUCT: + self.o2 = AlreadyExistsException.read(iprot) + else: + iprot.skip(ftype) + elif fid == 3: + if ftype == TType.STRUCT: + self.o3 = MetaException.read(iprot) + else: + iprot.skip(ftype) + else: + iprot.skip(ftype) + iprot.readFieldEnd() + iprot.readStructEnd() + + def write(self, oprot): + if oprot._fast_encode is not None and self.thrift_spec is not None: + oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec])) + return + oprot.writeStructBegin("append_partition_result") + if self.success is not None: + oprot.writeFieldBegin("success", TType.STRUCT, 0) + self.success.write(oprot) + oprot.writeFieldEnd() + if self.o1 is not None: + oprot.writeFieldBegin("o1", TType.STRUCT, 1) + self.o1.write(oprot) + oprot.writeFieldEnd() + if self.o2 is not None: + oprot.writeFieldBegin("o2", TType.STRUCT, 2) + self.o2.write(oprot) + oprot.writeFieldEnd() + if self.o3 is not None: + oprot.writeFieldBegin("o3", TType.STRUCT, 3) + self.o3.write(oprot) + oprot.writeFieldEnd() + oprot.writeFieldStop() + oprot.writeStructEnd() + + def validate(self): + return + + def __repr__(self): + L = ["{}={!r}".format(key, value) for key, value in self.__dict__.items()] + return "{}({})".format(self.__class__.__name__, ", ".join(L)) + + def __eq__(self, other): + return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ + + def __ne__(self, other): + return not (self == other) + + +all_structs.append(append_partition_result) +append_partition_result.thrift_spec = ( + ( + 0, + TType.STRUCT, + "success", + [Partition, None], + None, + ), # 0 + ( + 1, + TType.STRUCT, + "o1", + [InvalidObjectException, None], + None, + ), # 1 + ( + 2, + TType.STRUCT, + "o2", + [AlreadyExistsException, None], + None, + ), # 2 + ( + 3, + TType.STRUCT, + "o3", + [MetaException, None], + None, + ), # 3 +) + + +class add_partitions_req_args: + """ + Attributes: + - request + + """ + + def __init__( + self, + request=None, + ): + self.request = request + + def read(self, iprot): + if ( + iprot._fast_decode is not None + and isinstance(iprot.trans, TTransport.CReadableTransport) + and self.thrift_spec is not None + ): + iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec]) + return + iprot.readStructBegin() + while True: + (fname, ftype, fid) = iprot.readFieldBegin() + if ftype == TType.STOP: + break + if fid == 1: + if ftype == TType.STRUCT: + self.request = AddPartitionsRequest() + self.request.read(iprot) + else: + iprot.skip(ftype) + else: + iprot.skip(ftype) + iprot.readFieldEnd() + iprot.readStructEnd() + + def write(self, oprot): + if oprot._fast_encode is not None and self.thrift_spec is not None: + oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec])) + return + oprot.writeStructBegin("add_partitions_req_args") + if self.request is not None: + oprot.writeFieldBegin("request", TType.STRUCT, 1) + self.request.write(oprot) + oprot.writeFieldEnd() + oprot.writeFieldStop() + oprot.writeStructEnd() + + def validate(self): + return + + def __repr__(self): + L = ["{}={!r}".format(key, value) for key, value in self.__dict__.items()] + return "{}({})".format(self.__class__.__name__, ", ".join(L)) + + def __eq__(self, other): + return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ + + def __ne__(self, other): + return not (self == other) + + +all_structs.append(add_partitions_req_args) +add_partitions_req_args.thrift_spec = ( + None, # 0 + ( + 1, + TType.STRUCT, + "request", + [AddPartitionsRequest, None], + None, + ), # 1 +) + + +class add_partitions_req_result: + """ + Attributes: + - success + - o1 + - o2 + - o3 + + """ + + def __init__( + self, + success=None, + o1=None, + o2=None, + o3=None, + ): + self.success = success + self.o1 = o1 + self.o2 = o2 + self.o3 = o3 + + def read(self, iprot): + if ( + iprot._fast_decode is not None + and isinstance(iprot.trans, TTransport.CReadableTransport) + and self.thrift_spec is not None + ): + iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec]) + return + iprot.readStructBegin() + while True: + (fname, ftype, fid) = iprot.readFieldBegin() + if ftype == TType.STOP: + break + if fid == 0: + if ftype == TType.STRUCT: + self.success = AddPartitionsResult() + self.success.read(iprot) + else: + iprot.skip(ftype) + elif fid == 1: + if ftype == TType.STRUCT: + self.o1 = InvalidObjectException.read(iprot) + else: + iprot.skip(ftype) + elif fid == 2: + if ftype == TType.STRUCT: + self.o2 = AlreadyExistsException.read(iprot) + else: + iprot.skip(ftype) + elif fid == 3: + if ftype == TType.STRUCT: + self.o3 = MetaException.read(iprot) + else: + iprot.skip(ftype) + else: + iprot.skip(ftype) + iprot.readFieldEnd() + iprot.readStructEnd() + + def write(self, oprot): + if oprot._fast_encode is not None and self.thrift_spec is not None: + oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec])) + return + oprot.writeStructBegin("add_partitions_req_result") + if self.success is not None: + oprot.writeFieldBegin("success", TType.STRUCT, 0) + self.success.write(oprot) + oprot.writeFieldEnd() + if self.o1 is not None: + oprot.writeFieldBegin("o1", TType.STRUCT, 1) + self.o1.write(oprot) + oprot.writeFieldEnd() + if self.o2 is not None: + oprot.writeFieldBegin("o2", TType.STRUCT, 2) + self.o2.write(oprot) + oprot.writeFieldEnd() + if self.o3 is not None: + oprot.writeFieldBegin("o3", TType.STRUCT, 3) + self.o3.write(oprot) + oprot.writeFieldEnd() + oprot.writeFieldStop() + oprot.writeStructEnd() + + def validate(self): + return + + def __repr__(self): + L = ["{}={!r}".format(key, value) for key, value in self.__dict__.items()] + return "{}({})".format(self.__class__.__name__, ", ".join(L)) + + def __eq__(self, other): + return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ + + def __ne__(self, other): + return not (self == other) + + +all_structs.append(add_partitions_req_result) +add_partitions_req_result.thrift_spec = ( + ( + 0, + TType.STRUCT, + "success", + [AddPartitionsResult, None], + None, + ), # 0 + ( + 1, + TType.STRUCT, + "o1", + [InvalidObjectException, None], + None, + ), # 1 + ( + 2, + TType.STRUCT, + "o2", + [AlreadyExistsException, None], + None, + ), # 2 + ( + 3, + TType.STRUCT, + "o3", + [MetaException, None], + None, + ), # 3 +) + + +class append_partition_with_environment_context_args: + """ + Attributes: + - db_name + - tbl_name + - part_vals + - environment_context + + """ + + def __init__( + self, + db_name=None, + tbl_name=None, + part_vals=None, + environment_context=None, + ): + self.db_name = db_name + self.tbl_name = tbl_name + self.part_vals = part_vals + self.environment_context = environment_context + + def read(self, iprot): + if ( + iprot._fast_decode is not None + and isinstance(iprot.trans, TTransport.CReadableTransport) + and self.thrift_spec is not None + ): + iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec]) + return + iprot.readStructBegin() + while True: + (fname, ftype, fid) = iprot.readFieldBegin() + if ftype == TType.STOP: + break + if fid == 1: + if ftype == TType.STRING: + self.db_name = ( + iprot.readString().decode("utf-8", errors="replace") if sys.version_info[0] == 2 else iprot.readString() + ) + else: + iprot.skip(ftype) + elif fid == 2: + if ftype == TType.STRING: + self.tbl_name = ( + iprot.readString().decode("utf-8", errors="replace") if sys.version_info[0] == 2 else iprot.readString() + ) + else: + iprot.skip(ftype) + elif fid == 3: + if ftype == TType.LIST: + self.part_vals = [] + (_etype1457, _size1454) = iprot.readListBegin() + for _i1458 in range(_size1454): + _elem1459 = ( + iprot.readString().decode("utf-8", errors="replace") + if sys.version_info[0] == 2 + else iprot.readString() + ) + self.part_vals.append(_elem1459) + iprot.readListEnd() + else: + iprot.skip(ftype) + elif fid == 4: + if ftype == TType.STRUCT: + self.environment_context = EnvironmentContext() + self.environment_context.read(iprot) + else: + iprot.skip(ftype) + else: + iprot.skip(ftype) + iprot.readFieldEnd() + iprot.readStructEnd() + + def write(self, oprot): + if oprot._fast_encode is not None and self.thrift_spec is not None: + oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec])) + return + oprot.writeStructBegin("append_partition_with_environment_context_args") + if self.db_name is not None: + oprot.writeFieldBegin("db_name", TType.STRING, 1) + oprot.writeString(self.db_name.encode("utf-8") if sys.version_info[0] == 2 else self.db_name) + oprot.writeFieldEnd() + if self.tbl_name is not None: + oprot.writeFieldBegin("tbl_name", TType.STRING, 2) + oprot.writeString(self.tbl_name.encode("utf-8") if sys.version_info[0] == 2 else self.tbl_name) + oprot.writeFieldEnd() + if self.part_vals is not None: + oprot.writeFieldBegin("part_vals", TType.LIST, 3) + oprot.writeListBegin(TType.STRING, len(self.part_vals)) + for iter1460 in self.part_vals: + oprot.writeString(iter1460.encode("utf-8") if sys.version_info[0] == 2 else iter1460) + oprot.writeListEnd() + oprot.writeFieldEnd() + if self.environment_context is not None: + oprot.writeFieldBegin("environment_context", TType.STRUCT, 4) + self.environment_context.write(oprot) + oprot.writeFieldEnd() + oprot.writeFieldStop() + oprot.writeStructEnd() + + def validate(self): + return + + def __repr__(self): + L = ["{}={!r}".format(key, value) for key, value in self.__dict__.items()] + return "{}({})".format(self.__class__.__name__, ", ".join(L)) + + def __eq__(self, other): + return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ + + def __ne__(self, other): + return not (self == other) + + +all_structs.append(append_partition_with_environment_context_args) +append_partition_with_environment_context_args.thrift_spec = ( + None, # 0 + ( + 1, + TType.STRING, + "db_name", + "UTF8", + None, + ), # 1 + ( + 2, + TType.STRING, + "tbl_name", + "UTF8", + None, + ), # 2 + ( + 3, + TType.LIST, + "part_vals", + (TType.STRING, "UTF8", False), + None, + ), # 3 + ( + 4, + TType.STRUCT, + "environment_context", + [EnvironmentContext, None], + None, + ), # 4 +) + + +class append_partition_with_environment_context_result: + """ + Attributes: + - success + - o1 + - o2 + - o3 + + """ + + def __init__( + self, + success=None, + o1=None, + o2=None, + o3=None, + ): + self.success = success + self.o1 = o1 + self.o2 = o2 + self.o3 = o3 + + def read(self, iprot): + if ( + iprot._fast_decode is not None + and isinstance(iprot.trans, TTransport.CReadableTransport) + and self.thrift_spec is not None + ): + iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec]) + return + iprot.readStructBegin() + while True: + (fname, ftype, fid) = iprot.readFieldBegin() + if ftype == TType.STOP: + break + if fid == 0: + if ftype == TType.STRUCT: + self.success = Partition() + self.success.read(iprot) + else: + iprot.skip(ftype) + elif fid == 1: + if ftype == TType.STRUCT: + self.o1 = InvalidObjectException.read(iprot) + else: + iprot.skip(ftype) + elif fid == 2: + if ftype == TType.STRUCT: + self.o2 = AlreadyExistsException.read(iprot) + else: + iprot.skip(ftype) + elif fid == 3: + if ftype == TType.STRUCT: + self.o3 = MetaException.read(iprot) + else: + iprot.skip(ftype) + else: + iprot.skip(ftype) + iprot.readFieldEnd() + iprot.readStructEnd() + + def write(self, oprot): + if oprot._fast_encode is not None and self.thrift_spec is not None: + oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec])) + return + oprot.writeStructBegin("append_partition_with_environment_context_result") + if self.success is not None: + oprot.writeFieldBegin("success", TType.STRUCT, 0) + self.success.write(oprot) + oprot.writeFieldEnd() + if self.o1 is not None: + oprot.writeFieldBegin("o1", TType.STRUCT, 1) + self.o1.write(oprot) + oprot.writeFieldEnd() + if self.o2 is not None: + oprot.writeFieldBegin("o2", TType.STRUCT, 2) + self.o2.write(oprot) + oprot.writeFieldEnd() + if self.o3 is not None: + oprot.writeFieldBegin("o3", TType.STRUCT, 3) + self.o3.write(oprot) + oprot.writeFieldEnd() + oprot.writeFieldStop() + oprot.writeStructEnd() + + def validate(self): + return + + def __repr__(self): + L = ["{}={!r}".format(key, value) for key, value in self.__dict__.items()] + return "{}({})".format(self.__class__.__name__, ", ".join(L)) + + def __eq__(self, other): + return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ + + def __ne__(self, other): + return not (self == other) + + +all_structs.append(append_partition_with_environment_context_result) +append_partition_with_environment_context_result.thrift_spec = ( + ( + 0, + TType.STRUCT, + "success", + [Partition, None], + None, + ), # 0 + ( + 1, + TType.STRUCT, + "o1", + [InvalidObjectException, None], + None, + ), # 1 + ( + 2, + TType.STRUCT, + "o2", + [AlreadyExistsException, None], + None, + ), # 2 + ( + 3, + TType.STRUCT, + "o3", + [MetaException, None], + None, + ), # 3 +) + + +class append_partition_by_name_args: + """ + Attributes: + - db_name + - tbl_name + - part_name + + """ + + def __init__( + self, + db_name=None, + tbl_name=None, + part_name=None, + ): + self.db_name = db_name + self.tbl_name = tbl_name + self.part_name = part_name + + def read(self, iprot): + if ( + iprot._fast_decode is not None + and isinstance(iprot.trans, TTransport.CReadableTransport) + and self.thrift_spec is not None + ): + iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec]) + return + iprot.readStructBegin() + while True: + (fname, ftype, fid) = iprot.readFieldBegin() + if ftype == TType.STOP: + break + if fid == 1: + if ftype == TType.STRING: + self.db_name = ( + iprot.readString().decode("utf-8", errors="replace") if sys.version_info[0] == 2 else iprot.readString() + ) + else: + iprot.skip(ftype) + elif fid == 2: + if ftype == TType.STRING: + self.tbl_name = ( + iprot.readString().decode("utf-8", errors="replace") if sys.version_info[0] == 2 else iprot.readString() + ) + else: + iprot.skip(ftype) + elif fid == 3: + if ftype == TType.STRING: + self.part_name = ( + iprot.readString().decode("utf-8", errors="replace") if sys.version_info[0] == 2 else iprot.readString() + ) + else: + iprot.skip(ftype) + else: + iprot.skip(ftype) + iprot.readFieldEnd() + iprot.readStructEnd() + + def write(self, oprot): + if oprot._fast_encode is not None and self.thrift_spec is not None: + oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec])) + return + oprot.writeStructBegin("append_partition_by_name_args") + if self.db_name is not None: + oprot.writeFieldBegin("db_name", TType.STRING, 1) + oprot.writeString(self.db_name.encode("utf-8") if sys.version_info[0] == 2 else self.db_name) + oprot.writeFieldEnd() + if self.tbl_name is not None: + oprot.writeFieldBegin("tbl_name", TType.STRING, 2) + oprot.writeString(self.tbl_name.encode("utf-8") if sys.version_info[0] == 2 else self.tbl_name) + oprot.writeFieldEnd() + if self.part_name is not None: + oprot.writeFieldBegin("part_name", TType.STRING, 3) + oprot.writeString(self.part_name.encode("utf-8") if sys.version_info[0] == 2 else self.part_name) + oprot.writeFieldEnd() + oprot.writeFieldStop() + oprot.writeStructEnd() + + def validate(self): + return + + def __repr__(self): + L = ["{}={!r}".format(key, value) for key, value in self.__dict__.items()] + return "{}({})".format(self.__class__.__name__, ", ".join(L)) + + def __eq__(self, other): + return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ + + def __ne__(self, other): + return not (self == other) + + +all_structs.append(append_partition_by_name_args) +append_partition_by_name_args.thrift_spec = ( + None, # 0 + ( + 1, + TType.STRING, + "db_name", + "UTF8", + None, + ), # 1 + ( + 2, + TType.STRING, + "tbl_name", + "UTF8", + None, + ), # 2 + ( + 3, + TType.STRING, + "part_name", + "UTF8", + None, + ), # 3 +) + + +class append_partition_by_name_result: + """ + Attributes: + - success + - o1 + - o2 + - o3 + + """ + + def __init__( + self, + success=None, + o1=None, + o2=None, + o3=None, + ): + self.success = success + self.o1 = o1 + self.o2 = o2 + self.o3 = o3 + + def read(self, iprot): + if ( + iprot._fast_decode is not None + and isinstance(iprot.trans, TTransport.CReadableTransport) + and self.thrift_spec is not None + ): + iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec]) + return + iprot.readStructBegin() + while True: + (fname, ftype, fid) = iprot.readFieldBegin() + if ftype == TType.STOP: + break + if fid == 0: + if ftype == TType.STRUCT: + self.success = Partition() + self.success.read(iprot) + else: + iprot.skip(ftype) + elif fid == 1: + if ftype == TType.STRUCT: + self.o1 = InvalidObjectException.read(iprot) + else: + iprot.skip(ftype) + elif fid == 2: + if ftype == TType.STRUCT: + self.o2 = AlreadyExistsException.read(iprot) + else: + iprot.skip(ftype) + elif fid == 3: + if ftype == TType.STRUCT: + self.o3 = MetaException.read(iprot) + else: + iprot.skip(ftype) + else: + iprot.skip(ftype) + iprot.readFieldEnd() + iprot.readStructEnd() + + def write(self, oprot): + if oprot._fast_encode is not None and self.thrift_spec is not None: + oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec])) + return + oprot.writeStructBegin("append_partition_by_name_result") + if self.success is not None: + oprot.writeFieldBegin("success", TType.STRUCT, 0) + self.success.write(oprot) + oprot.writeFieldEnd() + if self.o1 is not None: + oprot.writeFieldBegin("o1", TType.STRUCT, 1) + self.o1.write(oprot) + oprot.writeFieldEnd() + if self.o2 is not None: + oprot.writeFieldBegin("o2", TType.STRUCT, 2) + self.o2.write(oprot) + oprot.writeFieldEnd() + if self.o3 is not None: + oprot.writeFieldBegin("o3", TType.STRUCT, 3) + self.o3.write(oprot) + oprot.writeFieldEnd() + oprot.writeFieldStop() + oprot.writeStructEnd() + + def validate(self): + return + + def __repr__(self): + L = ["{}={!r}".format(key, value) for key, value in self.__dict__.items()] + return "{}({})".format(self.__class__.__name__, ", ".join(L)) + + def __eq__(self, other): + return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ + + def __ne__(self, other): + return not (self == other) + + +all_structs.append(append_partition_by_name_result) +append_partition_by_name_result.thrift_spec = ( + ( + 0, + TType.STRUCT, + "success", + [Partition, None], + None, + ), # 0 + ( + 1, + TType.STRUCT, + "o1", + [InvalidObjectException, None], + None, + ), # 1 + ( + 2, + TType.STRUCT, + "o2", + [AlreadyExistsException, None], + None, + ), # 2 + ( + 3, + TType.STRUCT, + "o3", + [MetaException, None], + None, + ), # 3 +) + + +class append_partition_by_name_with_environment_context_args: + """ + Attributes: + - db_name + - tbl_name + - part_name + - environment_context + + """ + + def __init__( + self, + db_name=None, + tbl_name=None, + part_name=None, + environment_context=None, + ): + self.db_name = db_name + self.tbl_name = tbl_name + self.part_name = part_name + self.environment_context = environment_context + + def read(self, iprot): + if ( + iprot._fast_decode is not None + and isinstance(iprot.trans, TTransport.CReadableTransport) + and self.thrift_spec is not None + ): + iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec]) + return + iprot.readStructBegin() + while True: + (fname, ftype, fid) = iprot.readFieldBegin() + if ftype == TType.STOP: + break + if fid == 1: + if ftype == TType.STRING: + self.db_name = ( + iprot.readString().decode("utf-8", errors="replace") if sys.version_info[0] == 2 else iprot.readString() + ) + else: + iprot.skip(ftype) + elif fid == 2: + if ftype == TType.STRING: + self.tbl_name = ( + iprot.readString().decode("utf-8", errors="replace") if sys.version_info[0] == 2 else iprot.readString() + ) + else: + iprot.skip(ftype) + elif fid == 3: + if ftype == TType.STRING: + self.part_name = ( + iprot.readString().decode("utf-8", errors="replace") if sys.version_info[0] == 2 else iprot.readString() + ) + else: + iprot.skip(ftype) + elif fid == 4: + if ftype == TType.STRUCT: + self.environment_context = EnvironmentContext() + self.environment_context.read(iprot) + else: + iprot.skip(ftype) + else: + iprot.skip(ftype) + iprot.readFieldEnd() + iprot.readStructEnd() + + def write(self, oprot): + if oprot._fast_encode is not None and self.thrift_spec is not None: + oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec])) + return + oprot.writeStructBegin("append_partition_by_name_with_environment_context_args") + if self.db_name is not None: + oprot.writeFieldBegin("db_name", TType.STRING, 1) + oprot.writeString(self.db_name.encode("utf-8") if sys.version_info[0] == 2 else self.db_name) + oprot.writeFieldEnd() + if self.tbl_name is not None: + oprot.writeFieldBegin("tbl_name", TType.STRING, 2) + oprot.writeString(self.tbl_name.encode("utf-8") if sys.version_info[0] == 2 else self.tbl_name) + oprot.writeFieldEnd() + if self.part_name is not None: + oprot.writeFieldBegin("part_name", TType.STRING, 3) + oprot.writeString(self.part_name.encode("utf-8") if sys.version_info[0] == 2 else self.part_name) + oprot.writeFieldEnd() + if self.environment_context is not None: + oprot.writeFieldBegin("environment_context", TType.STRUCT, 4) + self.environment_context.write(oprot) + oprot.writeFieldEnd() + oprot.writeFieldStop() + oprot.writeStructEnd() + + def validate(self): + return + + def __repr__(self): + L = ["{}={!r}".format(key, value) for key, value in self.__dict__.items()] + return "{}({})".format(self.__class__.__name__, ", ".join(L)) + + def __eq__(self, other): + return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ + + def __ne__(self, other): + return not (self == other) + + +all_structs.append(append_partition_by_name_with_environment_context_args) +append_partition_by_name_with_environment_context_args.thrift_spec = ( + None, # 0 + ( + 1, + TType.STRING, + "db_name", + "UTF8", + None, + ), # 1 + ( + 2, + TType.STRING, + "tbl_name", + "UTF8", + None, + ), # 2 + ( + 3, + TType.STRING, + "part_name", + "UTF8", + None, + ), # 3 + ( + 4, + TType.STRUCT, + "environment_context", + [EnvironmentContext, None], + None, + ), # 4 +) + + +class append_partition_by_name_with_environment_context_result: + """ + Attributes: + - success + - o1 + - o2 + - o3 + + """ + + def __init__( + self, + success=None, + o1=None, + o2=None, + o3=None, + ): + self.success = success + self.o1 = o1 + self.o2 = o2 + self.o3 = o3 + + def read(self, iprot): + if ( + iprot._fast_decode is not None + and isinstance(iprot.trans, TTransport.CReadableTransport) + and self.thrift_spec is not None + ): + iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec]) + return + iprot.readStructBegin() + while True: + (fname, ftype, fid) = iprot.readFieldBegin() + if ftype == TType.STOP: + break + if fid == 0: + if ftype == TType.STRUCT: + self.success = Partition() + self.success.read(iprot) + else: + iprot.skip(ftype) + elif fid == 1: + if ftype == TType.STRUCT: + self.o1 = InvalidObjectException.read(iprot) + else: + iprot.skip(ftype) + elif fid == 2: + if ftype == TType.STRUCT: + self.o2 = AlreadyExistsException.read(iprot) + else: + iprot.skip(ftype) + elif fid == 3: + if ftype == TType.STRUCT: + self.o3 = MetaException.read(iprot) + else: + iprot.skip(ftype) + else: + iprot.skip(ftype) + iprot.readFieldEnd() + iprot.readStructEnd() + + def write(self, oprot): + if oprot._fast_encode is not None and self.thrift_spec is not None: + oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec])) + return + oprot.writeStructBegin("append_partition_by_name_with_environment_context_result") + if self.success is not None: + oprot.writeFieldBegin("success", TType.STRUCT, 0) + self.success.write(oprot) + oprot.writeFieldEnd() + if self.o1 is not None: + oprot.writeFieldBegin("o1", TType.STRUCT, 1) + self.o1.write(oprot) + oprot.writeFieldEnd() + if self.o2 is not None: + oprot.writeFieldBegin("o2", TType.STRUCT, 2) + self.o2.write(oprot) + oprot.writeFieldEnd() + if self.o3 is not None: + oprot.writeFieldBegin("o3", TType.STRUCT, 3) + self.o3.write(oprot) + oprot.writeFieldEnd() + oprot.writeFieldStop() + oprot.writeStructEnd() + + def validate(self): + return + + def __repr__(self): + L = ["{}={!r}".format(key, value) for key, value in self.__dict__.items()] + return "{}({})".format(self.__class__.__name__, ", ".join(L)) + + def __eq__(self, other): + return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ + + def __ne__(self, other): + return not (self == other) + + +all_structs.append(append_partition_by_name_with_environment_context_result) +append_partition_by_name_with_environment_context_result.thrift_spec = ( + ( + 0, + TType.STRUCT, + "success", + [Partition, None], + None, + ), # 0 + ( + 1, + TType.STRUCT, + "o1", + [InvalidObjectException, None], + None, + ), # 1 + ( + 2, + TType.STRUCT, + "o2", + [AlreadyExistsException, None], + None, + ), # 2 + ( + 3, + TType.STRUCT, + "o3", + [MetaException, None], + None, + ), # 3 +) + + +class drop_partition_args: + """ + Attributes: + - db_name + - tbl_name + - part_vals + - deleteData + + """ + + def __init__( + self, + db_name=None, + tbl_name=None, + part_vals=None, + deleteData=None, + ): + self.db_name = db_name + self.tbl_name = tbl_name + self.part_vals = part_vals + self.deleteData = deleteData + + def read(self, iprot): + if ( + iprot._fast_decode is not None + and isinstance(iprot.trans, TTransport.CReadableTransport) + and self.thrift_spec is not None + ): + iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec]) + return + iprot.readStructBegin() + while True: + (fname, ftype, fid) = iprot.readFieldBegin() + if ftype == TType.STOP: + break + if fid == 1: + if ftype == TType.STRING: + self.db_name = ( + iprot.readString().decode("utf-8", errors="replace") if sys.version_info[0] == 2 else iprot.readString() + ) + else: + iprot.skip(ftype) + elif fid == 2: + if ftype == TType.STRING: + self.tbl_name = ( + iprot.readString().decode("utf-8", errors="replace") if sys.version_info[0] == 2 else iprot.readString() + ) + else: + iprot.skip(ftype) + elif fid == 3: + if ftype == TType.LIST: + self.part_vals = [] + (_etype1464, _size1461) = iprot.readListBegin() + for _i1465 in range(_size1461): + _elem1466 = ( + iprot.readString().decode("utf-8", errors="replace") + if sys.version_info[0] == 2 + else iprot.readString() + ) + self.part_vals.append(_elem1466) + iprot.readListEnd() + else: + iprot.skip(ftype) + elif fid == 4: + if ftype == TType.BOOL: + self.deleteData = iprot.readBool() + else: + iprot.skip(ftype) + else: + iprot.skip(ftype) + iprot.readFieldEnd() + iprot.readStructEnd() + + def write(self, oprot): + if oprot._fast_encode is not None and self.thrift_spec is not None: + oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec])) + return + oprot.writeStructBegin("drop_partition_args") + if self.db_name is not None: + oprot.writeFieldBegin("db_name", TType.STRING, 1) + oprot.writeString(self.db_name.encode("utf-8") if sys.version_info[0] == 2 else self.db_name) + oprot.writeFieldEnd() + if self.tbl_name is not None: + oprot.writeFieldBegin("tbl_name", TType.STRING, 2) + oprot.writeString(self.tbl_name.encode("utf-8") if sys.version_info[0] == 2 else self.tbl_name) + oprot.writeFieldEnd() + if self.part_vals is not None: + oprot.writeFieldBegin("part_vals", TType.LIST, 3) + oprot.writeListBegin(TType.STRING, len(self.part_vals)) + for iter1467 in self.part_vals: + oprot.writeString(iter1467.encode("utf-8") if sys.version_info[0] == 2 else iter1467) + oprot.writeListEnd() + oprot.writeFieldEnd() + if self.deleteData is not None: + oprot.writeFieldBegin("deleteData", TType.BOOL, 4) + oprot.writeBool(self.deleteData) + oprot.writeFieldEnd() + oprot.writeFieldStop() + oprot.writeStructEnd() + + def validate(self): + return + + def __repr__(self): + L = ["{}={!r}".format(key, value) for key, value in self.__dict__.items()] + return "{}({})".format(self.__class__.__name__, ", ".join(L)) + + def __eq__(self, other): + return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ + + def __ne__(self, other): + return not (self == other) + + +all_structs.append(drop_partition_args) +drop_partition_args.thrift_spec = ( + None, # 0 + ( + 1, + TType.STRING, + "db_name", + "UTF8", + None, + ), # 1 + ( + 2, + TType.STRING, + "tbl_name", + "UTF8", + None, + ), # 2 + ( + 3, + TType.LIST, + "part_vals", + (TType.STRING, "UTF8", False), + None, + ), # 3 + ( + 4, + TType.BOOL, + "deleteData", + None, + None, + ), # 4 +) + + +class drop_partition_result: + """ + Attributes: + - success + - o1 + - o2 + + """ + + def __init__( + self, + success=None, + o1=None, + o2=None, + ): + self.success = success + self.o1 = o1 + self.o2 = o2 + + def read(self, iprot): + if ( + iprot._fast_decode is not None + and isinstance(iprot.trans, TTransport.CReadableTransport) + and self.thrift_spec is not None + ): + iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec]) + return + iprot.readStructBegin() + while True: + (fname, ftype, fid) = iprot.readFieldBegin() + if ftype == TType.STOP: + break + if fid == 0: + if ftype == TType.BOOL: + self.success = iprot.readBool() + else: + iprot.skip(ftype) + elif fid == 1: + if ftype == TType.STRUCT: + self.o1 = NoSuchObjectException.read(iprot) + else: + iprot.skip(ftype) + elif fid == 2: + if ftype == TType.STRUCT: + self.o2 = MetaException.read(iprot) + else: + iprot.skip(ftype) + else: + iprot.skip(ftype) + iprot.readFieldEnd() + iprot.readStructEnd() + + def write(self, oprot): + if oprot._fast_encode is not None and self.thrift_spec is not None: + oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec])) + return + oprot.writeStructBegin("drop_partition_result") + if self.success is not None: + oprot.writeFieldBegin("success", TType.BOOL, 0) + oprot.writeBool(self.success) + oprot.writeFieldEnd() + if self.o1 is not None: + oprot.writeFieldBegin("o1", TType.STRUCT, 1) + self.o1.write(oprot) + oprot.writeFieldEnd() + if self.o2 is not None: + oprot.writeFieldBegin("o2", TType.STRUCT, 2) + self.o2.write(oprot) + oprot.writeFieldEnd() + oprot.writeFieldStop() + oprot.writeStructEnd() + + def validate(self): + return + + def __repr__(self): + L = ["{}={!r}".format(key, value) for key, value in self.__dict__.items()] + return "{}({})".format(self.__class__.__name__, ", ".join(L)) + + def __eq__(self, other): + return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ + + def __ne__(self, other): + return not (self == other) + + +all_structs.append(drop_partition_result) +drop_partition_result.thrift_spec = ( + ( + 0, + TType.BOOL, + "success", + None, + None, + ), # 0 + ( + 1, + TType.STRUCT, + "o1", + [NoSuchObjectException, None], + None, + ), # 1 + ( + 2, + TType.STRUCT, + "o2", + [MetaException, None], + None, + ), # 2 +) + + +class drop_partition_with_environment_context_args: + """ + Attributes: + - db_name + - tbl_name + - part_vals + - deleteData + - environment_context + + """ + + def __init__( + self, + db_name=None, + tbl_name=None, + part_vals=None, + deleteData=None, + environment_context=None, + ): + self.db_name = db_name + self.tbl_name = tbl_name + self.part_vals = part_vals + self.deleteData = deleteData + self.environment_context = environment_context + + def read(self, iprot): + if ( + iprot._fast_decode is not None + and isinstance(iprot.trans, TTransport.CReadableTransport) + and self.thrift_spec is not None + ): + iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec]) + return + iprot.readStructBegin() + while True: + (fname, ftype, fid) = iprot.readFieldBegin() + if ftype == TType.STOP: + break + if fid == 1: + if ftype == TType.STRING: + self.db_name = ( + iprot.readString().decode("utf-8", errors="replace") if sys.version_info[0] == 2 else iprot.readString() + ) + else: + iprot.skip(ftype) + elif fid == 2: + if ftype == TType.STRING: + self.tbl_name = ( + iprot.readString().decode("utf-8", errors="replace") if sys.version_info[0] == 2 else iprot.readString() + ) + else: + iprot.skip(ftype) + elif fid == 3: + if ftype == TType.LIST: + self.part_vals = [] + (_etype1471, _size1468) = iprot.readListBegin() + for _i1472 in range(_size1468): + _elem1473 = ( + iprot.readString().decode("utf-8", errors="replace") + if sys.version_info[0] == 2 + else iprot.readString() + ) + self.part_vals.append(_elem1473) + iprot.readListEnd() + else: + iprot.skip(ftype) + elif fid == 4: + if ftype == TType.BOOL: + self.deleteData = iprot.readBool() + else: + iprot.skip(ftype) + elif fid == 5: + if ftype == TType.STRUCT: + self.environment_context = EnvironmentContext() + self.environment_context.read(iprot) + else: + iprot.skip(ftype) + else: + iprot.skip(ftype) + iprot.readFieldEnd() + iprot.readStructEnd() + + def write(self, oprot): + if oprot._fast_encode is not None and self.thrift_spec is not None: + oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec])) + return + oprot.writeStructBegin("drop_partition_with_environment_context_args") + if self.db_name is not None: + oprot.writeFieldBegin("db_name", TType.STRING, 1) + oprot.writeString(self.db_name.encode("utf-8") if sys.version_info[0] == 2 else self.db_name) + oprot.writeFieldEnd() + if self.tbl_name is not None: + oprot.writeFieldBegin("tbl_name", TType.STRING, 2) + oprot.writeString(self.tbl_name.encode("utf-8") if sys.version_info[0] == 2 else self.tbl_name) + oprot.writeFieldEnd() + if self.part_vals is not None: + oprot.writeFieldBegin("part_vals", TType.LIST, 3) + oprot.writeListBegin(TType.STRING, len(self.part_vals)) + for iter1474 in self.part_vals: + oprot.writeString(iter1474.encode("utf-8") if sys.version_info[0] == 2 else iter1474) + oprot.writeListEnd() + oprot.writeFieldEnd() + if self.deleteData is not None: + oprot.writeFieldBegin("deleteData", TType.BOOL, 4) + oprot.writeBool(self.deleteData) + oprot.writeFieldEnd() + if self.environment_context is not None: + oprot.writeFieldBegin("environment_context", TType.STRUCT, 5) + self.environment_context.write(oprot) + oprot.writeFieldEnd() + oprot.writeFieldStop() + oprot.writeStructEnd() + + def validate(self): + return + + def __repr__(self): + L = ["{}={!r}".format(key, value) for key, value in self.__dict__.items()] + return "{}({})".format(self.__class__.__name__, ", ".join(L)) + + def __eq__(self, other): + return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ + + def __ne__(self, other): + return not (self == other) + + +all_structs.append(drop_partition_with_environment_context_args) +drop_partition_with_environment_context_args.thrift_spec = ( + None, # 0 + ( + 1, + TType.STRING, + "db_name", + "UTF8", + None, + ), # 1 + ( + 2, + TType.STRING, + "tbl_name", + "UTF8", + None, + ), # 2 + ( + 3, + TType.LIST, + "part_vals", + (TType.STRING, "UTF8", False), + None, + ), # 3 + ( + 4, + TType.BOOL, + "deleteData", + None, + None, + ), # 4 + ( + 5, + TType.STRUCT, + "environment_context", + [EnvironmentContext, None], + None, + ), # 5 +) + + +class drop_partition_with_environment_context_result: + """ + Attributes: + - success + - o1 + - o2 + + """ + + def __init__( + self, + success=None, + o1=None, + o2=None, + ): + self.success = success + self.o1 = o1 + self.o2 = o2 + + def read(self, iprot): + if ( + iprot._fast_decode is not None + and isinstance(iprot.trans, TTransport.CReadableTransport) + and self.thrift_spec is not None + ): + iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec]) + return + iprot.readStructBegin() + while True: + (fname, ftype, fid) = iprot.readFieldBegin() + if ftype == TType.STOP: + break + if fid == 0: + if ftype == TType.BOOL: + self.success = iprot.readBool() + else: + iprot.skip(ftype) + elif fid == 1: + if ftype == TType.STRUCT: + self.o1 = NoSuchObjectException.read(iprot) + else: + iprot.skip(ftype) + elif fid == 2: + if ftype == TType.STRUCT: + self.o2 = MetaException.read(iprot) + else: + iprot.skip(ftype) + else: + iprot.skip(ftype) + iprot.readFieldEnd() + iprot.readStructEnd() + + def write(self, oprot): + if oprot._fast_encode is not None and self.thrift_spec is not None: + oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec])) + return + oprot.writeStructBegin("drop_partition_with_environment_context_result") + if self.success is not None: + oprot.writeFieldBegin("success", TType.BOOL, 0) + oprot.writeBool(self.success) + oprot.writeFieldEnd() + if self.o1 is not None: + oprot.writeFieldBegin("o1", TType.STRUCT, 1) + self.o1.write(oprot) + oprot.writeFieldEnd() + if self.o2 is not None: + oprot.writeFieldBegin("o2", TType.STRUCT, 2) + self.o2.write(oprot) + oprot.writeFieldEnd() + oprot.writeFieldStop() + oprot.writeStructEnd() + + def validate(self): + return + + def __repr__(self): + L = ["{}={!r}".format(key, value) for key, value in self.__dict__.items()] + return "{}({})".format(self.__class__.__name__, ", ".join(L)) + + def __eq__(self, other): + return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ + + def __ne__(self, other): + return not (self == other) + + +all_structs.append(drop_partition_with_environment_context_result) +drop_partition_with_environment_context_result.thrift_spec = ( + ( + 0, + TType.BOOL, + "success", + None, + None, + ), # 0 + ( + 1, + TType.STRUCT, + "o1", + [NoSuchObjectException, None], + None, + ), # 1 + ( + 2, + TType.STRUCT, + "o2", + [MetaException, None], + None, + ), # 2 +) + + +class drop_partition_by_name_args: + """ + Attributes: + - db_name + - tbl_name + - part_name + - deleteData + + """ + + def __init__( + self, + db_name=None, + tbl_name=None, + part_name=None, + deleteData=None, + ): + self.db_name = db_name + self.tbl_name = tbl_name + self.part_name = part_name + self.deleteData = deleteData + + def read(self, iprot): + if ( + iprot._fast_decode is not None + and isinstance(iprot.trans, TTransport.CReadableTransport) + and self.thrift_spec is not None + ): + iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec]) + return + iprot.readStructBegin() + while True: + (fname, ftype, fid) = iprot.readFieldBegin() + if ftype == TType.STOP: + break + if fid == 1: + if ftype == TType.STRING: + self.db_name = ( + iprot.readString().decode("utf-8", errors="replace") if sys.version_info[0] == 2 else iprot.readString() + ) + else: + iprot.skip(ftype) + elif fid == 2: + if ftype == TType.STRING: + self.tbl_name = ( + iprot.readString().decode("utf-8", errors="replace") if sys.version_info[0] == 2 else iprot.readString() + ) + else: + iprot.skip(ftype) + elif fid == 3: + if ftype == TType.STRING: + self.part_name = ( + iprot.readString().decode("utf-8", errors="replace") if sys.version_info[0] == 2 else iprot.readString() + ) + else: + iprot.skip(ftype) + elif fid == 4: + if ftype == TType.BOOL: + self.deleteData = iprot.readBool() + else: + iprot.skip(ftype) + else: + iprot.skip(ftype) + iprot.readFieldEnd() + iprot.readStructEnd() + + def write(self, oprot): + if oprot._fast_encode is not None and self.thrift_spec is not None: + oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec])) + return + oprot.writeStructBegin("drop_partition_by_name_args") + if self.db_name is not None: + oprot.writeFieldBegin("db_name", TType.STRING, 1) + oprot.writeString(self.db_name.encode("utf-8") if sys.version_info[0] == 2 else self.db_name) + oprot.writeFieldEnd() + if self.tbl_name is not None: + oprot.writeFieldBegin("tbl_name", TType.STRING, 2) + oprot.writeString(self.tbl_name.encode("utf-8") if sys.version_info[0] == 2 else self.tbl_name) + oprot.writeFieldEnd() + if self.part_name is not None: + oprot.writeFieldBegin("part_name", TType.STRING, 3) + oprot.writeString(self.part_name.encode("utf-8") if sys.version_info[0] == 2 else self.part_name) + oprot.writeFieldEnd() + if self.deleteData is not None: + oprot.writeFieldBegin("deleteData", TType.BOOL, 4) + oprot.writeBool(self.deleteData) + oprot.writeFieldEnd() + oprot.writeFieldStop() + oprot.writeStructEnd() + + def validate(self): + return + + def __repr__(self): + L = ["{}={!r}".format(key, value) for key, value in self.__dict__.items()] + return "{}({})".format(self.__class__.__name__, ", ".join(L)) + + def __eq__(self, other): + return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ + + def __ne__(self, other): + return not (self == other) + + +all_structs.append(drop_partition_by_name_args) +drop_partition_by_name_args.thrift_spec = ( + None, # 0 + ( + 1, + TType.STRING, + "db_name", + "UTF8", + None, + ), # 1 + ( + 2, + TType.STRING, + "tbl_name", + "UTF8", + None, + ), # 2 + ( + 3, + TType.STRING, + "part_name", + "UTF8", + None, + ), # 3 + ( + 4, + TType.BOOL, + "deleteData", + None, + None, + ), # 4 +) + + +class drop_partition_by_name_result: + """ + Attributes: + - success + - o1 + - o2 + + """ + + def __init__( + self, + success=None, + o1=None, + o2=None, + ): + self.success = success + self.o1 = o1 + self.o2 = o2 + + def read(self, iprot): + if ( + iprot._fast_decode is not None + and isinstance(iprot.trans, TTransport.CReadableTransport) + and self.thrift_spec is not None + ): + iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec]) + return + iprot.readStructBegin() + while True: + (fname, ftype, fid) = iprot.readFieldBegin() + if ftype == TType.STOP: + break + if fid == 0: + if ftype == TType.BOOL: + self.success = iprot.readBool() + else: + iprot.skip(ftype) + elif fid == 1: + if ftype == TType.STRUCT: + self.o1 = NoSuchObjectException.read(iprot) + else: + iprot.skip(ftype) + elif fid == 2: + if ftype == TType.STRUCT: + self.o2 = MetaException.read(iprot) + else: + iprot.skip(ftype) + else: + iprot.skip(ftype) + iprot.readFieldEnd() + iprot.readStructEnd() + + def write(self, oprot): + if oprot._fast_encode is not None and self.thrift_spec is not None: + oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec])) + return + oprot.writeStructBegin("drop_partition_by_name_result") + if self.success is not None: + oprot.writeFieldBegin("success", TType.BOOL, 0) + oprot.writeBool(self.success) + oprot.writeFieldEnd() + if self.o1 is not None: + oprot.writeFieldBegin("o1", TType.STRUCT, 1) + self.o1.write(oprot) + oprot.writeFieldEnd() + if self.o2 is not None: + oprot.writeFieldBegin("o2", TType.STRUCT, 2) + self.o2.write(oprot) + oprot.writeFieldEnd() + oprot.writeFieldStop() + oprot.writeStructEnd() + + def validate(self): + return + + def __repr__(self): + L = ["{}={!r}".format(key, value) for key, value in self.__dict__.items()] + return "{}({})".format(self.__class__.__name__, ", ".join(L)) + + def __eq__(self, other): + return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ + + def __ne__(self, other): + return not (self == other) + + +all_structs.append(drop_partition_by_name_result) +drop_partition_by_name_result.thrift_spec = ( + ( + 0, + TType.BOOL, + "success", + None, + None, + ), # 0 + ( + 1, + TType.STRUCT, + "o1", + [NoSuchObjectException, None], + None, + ), # 1 + ( + 2, + TType.STRUCT, + "o2", + [MetaException, None], + None, + ), # 2 +) + + +class drop_partition_by_name_with_environment_context_args: + """ + Attributes: + - db_name + - tbl_name + - part_name + - deleteData + - environment_context + + """ + + def __init__( + self, + db_name=None, + tbl_name=None, + part_name=None, + deleteData=None, + environment_context=None, + ): + self.db_name = db_name + self.tbl_name = tbl_name + self.part_name = part_name + self.deleteData = deleteData + self.environment_context = environment_context + + def read(self, iprot): + if ( + iprot._fast_decode is not None + and isinstance(iprot.trans, TTransport.CReadableTransport) + and self.thrift_spec is not None + ): + iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec]) + return + iprot.readStructBegin() + while True: + (fname, ftype, fid) = iprot.readFieldBegin() + if ftype == TType.STOP: + break + if fid == 1: + if ftype == TType.STRING: + self.db_name = ( + iprot.readString().decode("utf-8", errors="replace") if sys.version_info[0] == 2 else iprot.readString() + ) + else: + iprot.skip(ftype) + elif fid == 2: + if ftype == TType.STRING: + self.tbl_name = ( + iprot.readString().decode("utf-8", errors="replace") if sys.version_info[0] == 2 else iprot.readString() + ) + else: + iprot.skip(ftype) + elif fid == 3: + if ftype == TType.STRING: + self.part_name = ( + iprot.readString().decode("utf-8", errors="replace") if sys.version_info[0] == 2 else iprot.readString() + ) + else: + iprot.skip(ftype) + elif fid == 4: + if ftype == TType.BOOL: + self.deleteData = iprot.readBool() + else: + iprot.skip(ftype) + elif fid == 5: + if ftype == TType.STRUCT: + self.environment_context = EnvironmentContext() + self.environment_context.read(iprot) + else: + iprot.skip(ftype) + else: + iprot.skip(ftype) + iprot.readFieldEnd() + iprot.readStructEnd() + + def write(self, oprot): + if oprot._fast_encode is not None and self.thrift_spec is not None: + oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec])) + return + oprot.writeStructBegin("drop_partition_by_name_with_environment_context_args") + if self.db_name is not None: + oprot.writeFieldBegin("db_name", TType.STRING, 1) + oprot.writeString(self.db_name.encode("utf-8") if sys.version_info[0] == 2 else self.db_name) + oprot.writeFieldEnd() + if self.tbl_name is not None: + oprot.writeFieldBegin("tbl_name", TType.STRING, 2) + oprot.writeString(self.tbl_name.encode("utf-8") if sys.version_info[0] == 2 else self.tbl_name) + oprot.writeFieldEnd() + if self.part_name is not None: + oprot.writeFieldBegin("part_name", TType.STRING, 3) + oprot.writeString(self.part_name.encode("utf-8") if sys.version_info[0] == 2 else self.part_name) + oprot.writeFieldEnd() + if self.deleteData is not None: + oprot.writeFieldBegin("deleteData", TType.BOOL, 4) + oprot.writeBool(self.deleteData) + oprot.writeFieldEnd() + if self.environment_context is not None: + oprot.writeFieldBegin("environment_context", TType.STRUCT, 5) + self.environment_context.write(oprot) + oprot.writeFieldEnd() + oprot.writeFieldStop() + oprot.writeStructEnd() + + def validate(self): + return + + def __repr__(self): + L = ["{}={!r}".format(key, value) for key, value in self.__dict__.items()] + return "{}({})".format(self.__class__.__name__, ", ".join(L)) + + def __eq__(self, other): + return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ + + def __ne__(self, other): + return not (self == other) + + +all_structs.append(drop_partition_by_name_with_environment_context_args) +drop_partition_by_name_with_environment_context_args.thrift_spec = ( + None, # 0 + ( + 1, + TType.STRING, + "db_name", + "UTF8", + None, + ), # 1 + ( + 2, + TType.STRING, + "tbl_name", + "UTF8", + None, + ), # 2 + ( + 3, + TType.STRING, + "part_name", + "UTF8", + None, + ), # 3 + ( + 4, + TType.BOOL, + "deleteData", + None, + None, + ), # 4 + ( + 5, + TType.STRUCT, + "environment_context", + [EnvironmentContext, None], + None, + ), # 5 +) + + +class drop_partition_by_name_with_environment_context_result: + """ + Attributes: + - success + - o1 + - o2 + + """ + + def __init__( + self, + success=None, + o1=None, + o2=None, + ): + self.success = success + self.o1 = o1 + self.o2 = o2 + + def read(self, iprot): + if ( + iprot._fast_decode is not None + and isinstance(iprot.trans, TTransport.CReadableTransport) + and self.thrift_spec is not None + ): + iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec]) + return + iprot.readStructBegin() + while True: + (fname, ftype, fid) = iprot.readFieldBegin() + if ftype == TType.STOP: + break + if fid == 0: + if ftype == TType.BOOL: + self.success = iprot.readBool() + else: + iprot.skip(ftype) + elif fid == 1: + if ftype == TType.STRUCT: + self.o1 = NoSuchObjectException.read(iprot) + else: + iprot.skip(ftype) + elif fid == 2: + if ftype == TType.STRUCT: + self.o2 = MetaException.read(iprot) + else: + iprot.skip(ftype) + else: + iprot.skip(ftype) + iprot.readFieldEnd() + iprot.readStructEnd() + + def write(self, oprot): + if oprot._fast_encode is not None and self.thrift_spec is not None: + oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec])) + return + oprot.writeStructBegin("drop_partition_by_name_with_environment_context_result") + if self.success is not None: + oprot.writeFieldBegin("success", TType.BOOL, 0) + oprot.writeBool(self.success) + oprot.writeFieldEnd() + if self.o1 is not None: + oprot.writeFieldBegin("o1", TType.STRUCT, 1) + self.o1.write(oprot) + oprot.writeFieldEnd() + if self.o2 is not None: + oprot.writeFieldBegin("o2", TType.STRUCT, 2) + self.o2.write(oprot) + oprot.writeFieldEnd() + oprot.writeFieldStop() + oprot.writeStructEnd() + + def validate(self): + return + + def __repr__(self): + L = ["{}={!r}".format(key, value) for key, value in self.__dict__.items()] + return "{}({})".format(self.__class__.__name__, ", ".join(L)) + + def __eq__(self, other): + return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ + + def __ne__(self, other): + return not (self == other) + + +all_structs.append(drop_partition_by_name_with_environment_context_result) +drop_partition_by_name_with_environment_context_result.thrift_spec = ( + ( + 0, + TType.BOOL, + "success", + None, + None, + ), # 0 + ( + 1, + TType.STRUCT, + "o1", + [NoSuchObjectException, None], + None, + ), # 1 + ( + 2, + TType.STRUCT, + "o2", + [MetaException, None], + None, + ), # 2 +) + + +class drop_partitions_req_args: + """ + Attributes: + - req + + """ + + def __init__( + self, + req=None, + ): + self.req = req + + def read(self, iprot): + if ( + iprot._fast_decode is not None + and isinstance(iprot.trans, TTransport.CReadableTransport) + and self.thrift_spec is not None + ): + iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec]) + return + iprot.readStructBegin() + while True: + (fname, ftype, fid) = iprot.readFieldBegin() + if ftype == TType.STOP: + break + if fid == 1: + if ftype == TType.STRUCT: + self.req = DropPartitionsRequest() + self.req.read(iprot) + else: + iprot.skip(ftype) + else: + iprot.skip(ftype) + iprot.readFieldEnd() + iprot.readStructEnd() + + def write(self, oprot): + if oprot._fast_encode is not None and self.thrift_spec is not None: + oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec])) + return + oprot.writeStructBegin("drop_partitions_req_args") + if self.req is not None: + oprot.writeFieldBegin("req", TType.STRUCT, 1) + self.req.write(oprot) + oprot.writeFieldEnd() + oprot.writeFieldStop() + oprot.writeStructEnd() + + def validate(self): + return + + def __repr__(self): + L = ["{}={!r}".format(key, value) for key, value in self.__dict__.items()] + return "{}({})".format(self.__class__.__name__, ", ".join(L)) + + def __eq__(self, other): + return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ + + def __ne__(self, other): + return not (self == other) + + +all_structs.append(drop_partitions_req_args) +drop_partitions_req_args.thrift_spec = ( + None, # 0 + ( + 1, + TType.STRUCT, + "req", + [DropPartitionsRequest, None], + None, + ), # 1 +) + + +class drop_partitions_req_result: + """ + Attributes: + - success + - o1 + - o2 + + """ + + def __init__( + self, + success=None, + o1=None, + o2=None, + ): + self.success = success + self.o1 = o1 + self.o2 = o2 + + def read(self, iprot): + if ( + iprot._fast_decode is not None + and isinstance(iprot.trans, TTransport.CReadableTransport) + and self.thrift_spec is not None + ): + iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec]) + return + iprot.readStructBegin() + while True: + (fname, ftype, fid) = iprot.readFieldBegin() + if ftype == TType.STOP: + break + if fid == 0: + if ftype == TType.STRUCT: + self.success = DropPartitionsResult() + self.success.read(iprot) + else: + iprot.skip(ftype) + elif fid == 1: + if ftype == TType.STRUCT: + self.o1 = NoSuchObjectException.read(iprot) + else: + iprot.skip(ftype) + elif fid == 2: + if ftype == TType.STRUCT: + self.o2 = MetaException.read(iprot) + else: + iprot.skip(ftype) + else: + iprot.skip(ftype) + iprot.readFieldEnd() + iprot.readStructEnd() + + def write(self, oprot): + if oprot._fast_encode is not None and self.thrift_spec is not None: + oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec])) + return + oprot.writeStructBegin("drop_partitions_req_result") + if self.success is not None: + oprot.writeFieldBegin("success", TType.STRUCT, 0) + self.success.write(oprot) + oprot.writeFieldEnd() + if self.o1 is not None: + oprot.writeFieldBegin("o1", TType.STRUCT, 1) + self.o1.write(oprot) + oprot.writeFieldEnd() + if self.o2 is not None: + oprot.writeFieldBegin("o2", TType.STRUCT, 2) + self.o2.write(oprot) + oprot.writeFieldEnd() + oprot.writeFieldStop() + oprot.writeStructEnd() + + def validate(self): + return + + def __repr__(self): + L = ["{}={!r}".format(key, value) for key, value in self.__dict__.items()] + return "{}({})".format(self.__class__.__name__, ", ".join(L)) + + def __eq__(self, other): + return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ + + def __ne__(self, other): + return not (self == other) + + +all_structs.append(drop_partitions_req_result) +drop_partitions_req_result.thrift_spec = ( + ( + 0, + TType.STRUCT, + "success", + [DropPartitionsResult, None], + None, + ), # 0 + ( + 1, + TType.STRUCT, + "o1", + [NoSuchObjectException, None], + None, + ), # 1 + ( + 2, + TType.STRUCT, + "o2", + [MetaException, None], + None, + ), # 2 +) + + +class get_partition_args: + """ + Attributes: + - db_name + - tbl_name + - part_vals + + """ + + def __init__( + self, + db_name=None, + tbl_name=None, + part_vals=None, + ): + self.db_name = db_name + self.tbl_name = tbl_name + self.part_vals = part_vals + + def read(self, iprot): + if ( + iprot._fast_decode is not None + and isinstance(iprot.trans, TTransport.CReadableTransport) + and self.thrift_spec is not None + ): + iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec]) + return + iprot.readStructBegin() + while True: + (fname, ftype, fid) = iprot.readFieldBegin() + if ftype == TType.STOP: + break + if fid == 1: + if ftype == TType.STRING: + self.db_name = ( + iprot.readString().decode("utf-8", errors="replace") if sys.version_info[0] == 2 else iprot.readString() + ) + else: + iprot.skip(ftype) + elif fid == 2: + if ftype == TType.STRING: + self.tbl_name = ( + iprot.readString().decode("utf-8", errors="replace") if sys.version_info[0] == 2 else iprot.readString() + ) + else: + iprot.skip(ftype) + elif fid == 3: + if ftype == TType.LIST: + self.part_vals = [] + (_etype1478, _size1475) = iprot.readListBegin() + for _i1479 in range(_size1475): + _elem1480 = ( + iprot.readString().decode("utf-8", errors="replace") + if sys.version_info[0] == 2 + else iprot.readString() + ) + self.part_vals.append(_elem1480) + iprot.readListEnd() + else: + iprot.skip(ftype) + else: + iprot.skip(ftype) + iprot.readFieldEnd() + iprot.readStructEnd() + + def write(self, oprot): + if oprot._fast_encode is not None and self.thrift_spec is not None: + oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec])) + return + oprot.writeStructBegin("get_partition_args") + if self.db_name is not None: + oprot.writeFieldBegin("db_name", TType.STRING, 1) + oprot.writeString(self.db_name.encode("utf-8") if sys.version_info[0] == 2 else self.db_name) + oprot.writeFieldEnd() + if self.tbl_name is not None: + oprot.writeFieldBegin("tbl_name", TType.STRING, 2) + oprot.writeString(self.tbl_name.encode("utf-8") if sys.version_info[0] == 2 else self.tbl_name) + oprot.writeFieldEnd() + if self.part_vals is not None: + oprot.writeFieldBegin("part_vals", TType.LIST, 3) + oprot.writeListBegin(TType.STRING, len(self.part_vals)) + for iter1481 in self.part_vals: + oprot.writeString(iter1481.encode("utf-8") if sys.version_info[0] == 2 else iter1481) + oprot.writeListEnd() + oprot.writeFieldEnd() + oprot.writeFieldStop() + oprot.writeStructEnd() + + def validate(self): + return + + def __repr__(self): + L = ["{}={!r}".format(key, value) for key, value in self.__dict__.items()] + return "{}({})".format(self.__class__.__name__, ", ".join(L)) + + def __eq__(self, other): + return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ + + def __ne__(self, other): + return not (self == other) + + +all_structs.append(get_partition_args) +get_partition_args.thrift_spec = ( + None, # 0 + ( + 1, + TType.STRING, + "db_name", + "UTF8", + None, + ), # 1 + ( + 2, + TType.STRING, + "tbl_name", + "UTF8", + None, + ), # 2 + ( + 3, + TType.LIST, + "part_vals", + (TType.STRING, "UTF8", False), + None, + ), # 3 +) + + +class get_partition_result: + """ + Attributes: + - success + - o1 + - o2 + + """ + + def __init__( + self, + success=None, + o1=None, + o2=None, + ): + self.success = success + self.o1 = o1 + self.o2 = o2 + + def read(self, iprot): + if ( + iprot._fast_decode is not None + and isinstance(iprot.trans, TTransport.CReadableTransport) + and self.thrift_spec is not None + ): + iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec]) + return + iprot.readStructBegin() + while True: + (fname, ftype, fid) = iprot.readFieldBegin() + if ftype == TType.STOP: + break + if fid == 0: + if ftype == TType.STRUCT: + self.success = Partition() + self.success.read(iprot) + else: + iprot.skip(ftype) + elif fid == 1: + if ftype == TType.STRUCT: + self.o1 = MetaException.read(iprot) + else: + iprot.skip(ftype) + elif fid == 2: + if ftype == TType.STRUCT: + self.o2 = NoSuchObjectException.read(iprot) + else: + iprot.skip(ftype) + else: + iprot.skip(ftype) + iprot.readFieldEnd() + iprot.readStructEnd() + + def write(self, oprot): + if oprot._fast_encode is not None and self.thrift_spec is not None: + oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec])) + return + oprot.writeStructBegin("get_partition_result") + if self.success is not None: + oprot.writeFieldBegin("success", TType.STRUCT, 0) + self.success.write(oprot) + oprot.writeFieldEnd() + if self.o1 is not None: + oprot.writeFieldBegin("o1", TType.STRUCT, 1) + self.o1.write(oprot) + oprot.writeFieldEnd() + if self.o2 is not None: + oprot.writeFieldBegin("o2", TType.STRUCT, 2) + self.o2.write(oprot) + oprot.writeFieldEnd() + oprot.writeFieldStop() + oprot.writeStructEnd() + + def validate(self): + return + + def __repr__(self): + L = ["{}={!r}".format(key, value) for key, value in self.__dict__.items()] + return "{}({})".format(self.__class__.__name__, ", ".join(L)) + + def __eq__(self, other): + return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ + + def __ne__(self, other): + return not (self == other) + + +all_structs.append(get_partition_result) +get_partition_result.thrift_spec = ( + ( + 0, + TType.STRUCT, + "success", + [Partition, None], + None, + ), # 0 + ( + 1, + TType.STRUCT, + "o1", + [MetaException, None], + None, + ), # 1 + ( + 2, + TType.STRUCT, + "o2", + [NoSuchObjectException, None], + None, + ), # 2 +) + + +class get_partition_req_args: + """ + Attributes: + - req + + """ + + def __init__( + self, + req=None, + ): + self.req = req + + def read(self, iprot): + if ( + iprot._fast_decode is not None + and isinstance(iprot.trans, TTransport.CReadableTransport) + and self.thrift_spec is not None + ): + iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec]) + return + iprot.readStructBegin() + while True: + (fname, ftype, fid) = iprot.readFieldBegin() + if ftype == TType.STOP: + break + if fid == 1: + if ftype == TType.STRUCT: + self.req = GetPartitionRequest() + self.req.read(iprot) + else: + iprot.skip(ftype) + else: + iprot.skip(ftype) + iprot.readFieldEnd() + iprot.readStructEnd() + + def write(self, oprot): + if oprot._fast_encode is not None and self.thrift_spec is not None: + oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec])) + return + oprot.writeStructBegin("get_partition_req_args") + if self.req is not None: + oprot.writeFieldBegin("req", TType.STRUCT, 1) + self.req.write(oprot) + oprot.writeFieldEnd() + oprot.writeFieldStop() + oprot.writeStructEnd() + + def validate(self): + return + + def __repr__(self): + L = ["{}={!r}".format(key, value) for key, value in self.__dict__.items()] + return "{}({})".format(self.__class__.__name__, ", ".join(L)) + + def __eq__(self, other): + return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ + + def __ne__(self, other): + return not (self == other) + + +all_structs.append(get_partition_req_args) +get_partition_req_args.thrift_spec = ( + None, # 0 + ( + 1, + TType.STRUCT, + "req", + [GetPartitionRequest, None], + None, + ), # 1 +) + + +class get_partition_req_result: + """ + Attributes: + - success + - o1 + - o2 + + """ + + def __init__( + self, + success=None, + o1=None, + o2=None, + ): + self.success = success + self.o1 = o1 + self.o2 = o2 + + def read(self, iprot): + if ( + iprot._fast_decode is not None + and isinstance(iprot.trans, TTransport.CReadableTransport) + and self.thrift_spec is not None + ): + iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec]) + return + iprot.readStructBegin() + while True: + (fname, ftype, fid) = iprot.readFieldBegin() + if ftype == TType.STOP: + break + if fid == 0: + if ftype == TType.STRUCT: + self.success = GetPartitionResponse() + self.success.read(iprot) + else: + iprot.skip(ftype) + elif fid == 1: + if ftype == TType.STRUCT: + self.o1 = MetaException.read(iprot) + else: + iprot.skip(ftype) + elif fid == 2: + if ftype == TType.STRUCT: + self.o2 = NoSuchObjectException.read(iprot) + else: + iprot.skip(ftype) + else: + iprot.skip(ftype) + iprot.readFieldEnd() + iprot.readStructEnd() + + def write(self, oprot): + if oprot._fast_encode is not None and self.thrift_spec is not None: + oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec])) + return + oprot.writeStructBegin("get_partition_req_result") + if self.success is not None: + oprot.writeFieldBegin("success", TType.STRUCT, 0) + self.success.write(oprot) + oprot.writeFieldEnd() + if self.o1 is not None: + oprot.writeFieldBegin("o1", TType.STRUCT, 1) + self.o1.write(oprot) + oprot.writeFieldEnd() + if self.o2 is not None: + oprot.writeFieldBegin("o2", TType.STRUCT, 2) + self.o2.write(oprot) + oprot.writeFieldEnd() + oprot.writeFieldStop() + oprot.writeStructEnd() + + def validate(self): + return + + def __repr__(self): + L = ["{}={!r}".format(key, value) for key, value in self.__dict__.items()] + return "{}({})".format(self.__class__.__name__, ", ".join(L)) + + def __eq__(self, other): + return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ + + def __ne__(self, other): + return not (self == other) + + +all_structs.append(get_partition_req_result) +get_partition_req_result.thrift_spec = ( + ( + 0, + TType.STRUCT, + "success", + [GetPartitionResponse, None], + None, + ), # 0 + ( + 1, + TType.STRUCT, + "o1", + [MetaException, None], + None, + ), # 1 + ( + 2, + TType.STRUCT, + "o2", + [NoSuchObjectException, None], + None, + ), # 2 +) + + +class exchange_partition_args: + """ + Attributes: + - partitionSpecs + - source_db + - source_table_name + - dest_db + - dest_table_name + + """ + + def __init__( + self, + partitionSpecs=None, + source_db=None, + source_table_name=None, + dest_db=None, + dest_table_name=None, + ): + self.partitionSpecs = partitionSpecs + self.source_db = source_db + self.source_table_name = source_table_name + self.dest_db = dest_db + self.dest_table_name = dest_table_name + + def read(self, iprot): + if ( + iprot._fast_decode is not None + and isinstance(iprot.trans, TTransport.CReadableTransport) + and self.thrift_spec is not None + ): + iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec]) + return + iprot.readStructBegin() + while True: + (fname, ftype, fid) = iprot.readFieldBegin() + if ftype == TType.STOP: + break + if fid == 1: + if ftype == TType.MAP: + self.partitionSpecs = {} + (_ktype1483, _vtype1484, _size1482) = iprot.readMapBegin() + for _i1486 in range(_size1482): + _key1487 = ( + iprot.readString().decode("utf-8", errors="replace") + if sys.version_info[0] == 2 + else iprot.readString() + ) + _val1488 = ( + iprot.readString().decode("utf-8", errors="replace") + if sys.version_info[0] == 2 + else iprot.readString() + ) + self.partitionSpecs[_key1487] = _val1488 + iprot.readMapEnd() + else: + iprot.skip(ftype) + elif fid == 2: + if ftype == TType.STRING: + self.source_db = ( + iprot.readString().decode("utf-8", errors="replace") if sys.version_info[0] == 2 else iprot.readString() + ) + else: + iprot.skip(ftype) + elif fid == 3: + if ftype == TType.STRING: + self.source_table_name = ( + iprot.readString().decode("utf-8", errors="replace") if sys.version_info[0] == 2 else iprot.readString() + ) + else: + iprot.skip(ftype) + elif fid == 4: + if ftype == TType.STRING: + self.dest_db = ( + iprot.readString().decode("utf-8", errors="replace") if sys.version_info[0] == 2 else iprot.readString() + ) + else: + iprot.skip(ftype) + elif fid == 5: + if ftype == TType.STRING: + self.dest_table_name = ( + iprot.readString().decode("utf-8", errors="replace") if sys.version_info[0] == 2 else iprot.readString() + ) + else: + iprot.skip(ftype) + else: + iprot.skip(ftype) + iprot.readFieldEnd() + iprot.readStructEnd() + + def write(self, oprot): + if oprot._fast_encode is not None and self.thrift_spec is not None: + oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec])) + return + oprot.writeStructBegin("exchange_partition_args") + if self.partitionSpecs is not None: + oprot.writeFieldBegin("partitionSpecs", TType.MAP, 1) + oprot.writeMapBegin(TType.STRING, TType.STRING, len(self.partitionSpecs)) + for kiter1489, viter1490 in self.partitionSpecs.items(): + oprot.writeString(kiter1489.encode("utf-8") if sys.version_info[0] == 2 else kiter1489) + oprot.writeString(viter1490.encode("utf-8") if sys.version_info[0] == 2 else viter1490) + oprot.writeMapEnd() + oprot.writeFieldEnd() + if self.source_db is not None: + oprot.writeFieldBegin("source_db", TType.STRING, 2) + oprot.writeString(self.source_db.encode("utf-8") if sys.version_info[0] == 2 else self.source_db) + oprot.writeFieldEnd() + if self.source_table_name is not None: + oprot.writeFieldBegin("source_table_name", TType.STRING, 3) + oprot.writeString(self.source_table_name.encode("utf-8") if sys.version_info[0] == 2 else self.source_table_name) + oprot.writeFieldEnd() + if self.dest_db is not None: + oprot.writeFieldBegin("dest_db", TType.STRING, 4) + oprot.writeString(self.dest_db.encode("utf-8") if sys.version_info[0] == 2 else self.dest_db) + oprot.writeFieldEnd() + if self.dest_table_name is not None: + oprot.writeFieldBegin("dest_table_name", TType.STRING, 5) + oprot.writeString(self.dest_table_name.encode("utf-8") if sys.version_info[0] == 2 else self.dest_table_name) + oprot.writeFieldEnd() + oprot.writeFieldStop() + oprot.writeStructEnd() + + def validate(self): + return + + def __repr__(self): + L = ["{}={!r}".format(key, value) for key, value in self.__dict__.items()] + return "{}({})".format(self.__class__.__name__, ", ".join(L)) + + def __eq__(self, other): + return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ + + def __ne__(self, other): + return not (self == other) + + +all_structs.append(exchange_partition_args) +exchange_partition_args.thrift_spec = ( + None, # 0 + ( + 1, + TType.MAP, + "partitionSpecs", + (TType.STRING, "UTF8", TType.STRING, "UTF8", False), + None, + ), # 1 + ( + 2, + TType.STRING, + "source_db", + "UTF8", + None, + ), # 2 + ( + 3, + TType.STRING, + "source_table_name", + "UTF8", + None, + ), # 3 + ( + 4, + TType.STRING, + "dest_db", + "UTF8", + None, + ), # 4 + ( + 5, + TType.STRING, + "dest_table_name", + "UTF8", + None, + ), # 5 +) + + +class exchange_partition_result: + """ + Attributes: + - success + - o1 + - o2 + - o3 + - o4 + + """ + + def __init__( + self, + success=None, + o1=None, + o2=None, + o3=None, + o4=None, + ): + self.success = success + self.o1 = o1 + self.o2 = o2 + self.o3 = o3 + self.o4 = o4 + + def read(self, iprot): + if ( + iprot._fast_decode is not None + and isinstance(iprot.trans, TTransport.CReadableTransport) + and self.thrift_spec is not None + ): + iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec]) + return + iprot.readStructBegin() + while True: + (fname, ftype, fid) = iprot.readFieldBegin() + if ftype == TType.STOP: + break + if fid == 0: + if ftype == TType.STRUCT: + self.success = Partition() + self.success.read(iprot) + else: + iprot.skip(ftype) + elif fid == 1: + if ftype == TType.STRUCT: + self.o1 = MetaException.read(iprot) + else: + iprot.skip(ftype) + elif fid == 2: + if ftype == TType.STRUCT: + self.o2 = NoSuchObjectException.read(iprot) + else: + iprot.skip(ftype) + elif fid == 3: + if ftype == TType.STRUCT: + self.o3 = InvalidObjectException.read(iprot) + else: + iprot.skip(ftype) + elif fid == 4: + if ftype == TType.STRUCT: + self.o4 = InvalidInputException.read(iprot) + else: + iprot.skip(ftype) + else: + iprot.skip(ftype) + iprot.readFieldEnd() + iprot.readStructEnd() + + def write(self, oprot): + if oprot._fast_encode is not None and self.thrift_spec is not None: + oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec])) + return + oprot.writeStructBegin("exchange_partition_result") + if self.success is not None: + oprot.writeFieldBegin("success", TType.STRUCT, 0) + self.success.write(oprot) + oprot.writeFieldEnd() + if self.o1 is not None: + oprot.writeFieldBegin("o1", TType.STRUCT, 1) + self.o1.write(oprot) + oprot.writeFieldEnd() + if self.o2 is not None: + oprot.writeFieldBegin("o2", TType.STRUCT, 2) + self.o2.write(oprot) + oprot.writeFieldEnd() + if self.o3 is not None: + oprot.writeFieldBegin("o3", TType.STRUCT, 3) + self.o3.write(oprot) + oprot.writeFieldEnd() + if self.o4 is not None: + oprot.writeFieldBegin("o4", TType.STRUCT, 4) + self.o4.write(oprot) + oprot.writeFieldEnd() + oprot.writeFieldStop() + oprot.writeStructEnd() + + def validate(self): + return + + def __repr__(self): + L = ["{}={!r}".format(key, value) for key, value in self.__dict__.items()] + return "{}({})".format(self.__class__.__name__, ", ".join(L)) + + def __eq__(self, other): + return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ + + def __ne__(self, other): + return not (self == other) + + +all_structs.append(exchange_partition_result) +exchange_partition_result.thrift_spec = ( + ( + 0, + TType.STRUCT, + "success", + [Partition, None], + None, + ), # 0 + ( + 1, + TType.STRUCT, + "o1", + [MetaException, None], + None, + ), # 1 + ( + 2, + TType.STRUCT, + "o2", + [NoSuchObjectException, None], + None, + ), # 2 + ( + 3, + TType.STRUCT, + "o3", + [InvalidObjectException, None], + None, + ), # 3 + ( + 4, + TType.STRUCT, + "o4", + [InvalidInputException, None], + None, + ), # 4 +) + + +class exchange_partitions_args: + """ + Attributes: + - partitionSpecs + - source_db + - source_table_name + - dest_db + - dest_table_name + + """ + + def __init__( + self, + partitionSpecs=None, + source_db=None, + source_table_name=None, + dest_db=None, + dest_table_name=None, + ): + self.partitionSpecs = partitionSpecs + self.source_db = source_db + self.source_table_name = source_table_name + self.dest_db = dest_db + self.dest_table_name = dest_table_name + + def read(self, iprot): + if ( + iprot._fast_decode is not None + and isinstance(iprot.trans, TTransport.CReadableTransport) + and self.thrift_spec is not None + ): + iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec]) + return + iprot.readStructBegin() + while True: + (fname, ftype, fid) = iprot.readFieldBegin() + if ftype == TType.STOP: + break + if fid == 1: + if ftype == TType.MAP: + self.partitionSpecs = {} + (_ktype1492, _vtype1493, _size1491) = iprot.readMapBegin() + for _i1495 in range(_size1491): + _key1496 = ( + iprot.readString().decode("utf-8", errors="replace") + if sys.version_info[0] == 2 + else iprot.readString() + ) + _val1497 = ( + iprot.readString().decode("utf-8", errors="replace") + if sys.version_info[0] == 2 + else iprot.readString() + ) + self.partitionSpecs[_key1496] = _val1497 + iprot.readMapEnd() + else: + iprot.skip(ftype) + elif fid == 2: + if ftype == TType.STRING: + self.source_db = ( + iprot.readString().decode("utf-8", errors="replace") if sys.version_info[0] == 2 else iprot.readString() + ) + else: + iprot.skip(ftype) + elif fid == 3: + if ftype == TType.STRING: + self.source_table_name = ( + iprot.readString().decode("utf-8", errors="replace") if sys.version_info[0] == 2 else iprot.readString() + ) + else: + iprot.skip(ftype) + elif fid == 4: + if ftype == TType.STRING: + self.dest_db = ( + iprot.readString().decode("utf-8", errors="replace") if sys.version_info[0] == 2 else iprot.readString() + ) + else: + iprot.skip(ftype) + elif fid == 5: + if ftype == TType.STRING: + self.dest_table_name = ( + iprot.readString().decode("utf-8", errors="replace") if sys.version_info[0] == 2 else iprot.readString() + ) + else: + iprot.skip(ftype) + else: + iprot.skip(ftype) + iprot.readFieldEnd() + iprot.readStructEnd() + + def write(self, oprot): + if oprot._fast_encode is not None and self.thrift_spec is not None: + oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec])) + return + oprot.writeStructBegin("exchange_partitions_args") + if self.partitionSpecs is not None: + oprot.writeFieldBegin("partitionSpecs", TType.MAP, 1) + oprot.writeMapBegin(TType.STRING, TType.STRING, len(self.partitionSpecs)) + for kiter1498, viter1499 in self.partitionSpecs.items(): + oprot.writeString(kiter1498.encode("utf-8") if sys.version_info[0] == 2 else kiter1498) + oprot.writeString(viter1499.encode("utf-8") if sys.version_info[0] == 2 else viter1499) + oprot.writeMapEnd() + oprot.writeFieldEnd() + if self.source_db is not None: + oprot.writeFieldBegin("source_db", TType.STRING, 2) + oprot.writeString(self.source_db.encode("utf-8") if sys.version_info[0] == 2 else self.source_db) + oprot.writeFieldEnd() + if self.source_table_name is not None: + oprot.writeFieldBegin("source_table_name", TType.STRING, 3) + oprot.writeString(self.source_table_name.encode("utf-8") if sys.version_info[0] == 2 else self.source_table_name) + oprot.writeFieldEnd() + if self.dest_db is not None: + oprot.writeFieldBegin("dest_db", TType.STRING, 4) + oprot.writeString(self.dest_db.encode("utf-8") if sys.version_info[0] == 2 else self.dest_db) + oprot.writeFieldEnd() + if self.dest_table_name is not None: + oprot.writeFieldBegin("dest_table_name", TType.STRING, 5) + oprot.writeString(self.dest_table_name.encode("utf-8") if sys.version_info[0] == 2 else self.dest_table_name) + oprot.writeFieldEnd() + oprot.writeFieldStop() + oprot.writeStructEnd() + + def validate(self): + return + + def __repr__(self): + L = ["{}={!r}".format(key, value) for key, value in self.__dict__.items()] + return "{}({})".format(self.__class__.__name__, ", ".join(L)) + + def __eq__(self, other): + return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ + + def __ne__(self, other): + return not (self == other) + + +all_structs.append(exchange_partitions_args) +exchange_partitions_args.thrift_spec = ( + None, # 0 + ( + 1, + TType.MAP, + "partitionSpecs", + (TType.STRING, "UTF8", TType.STRING, "UTF8", False), + None, + ), # 1 + ( + 2, + TType.STRING, + "source_db", + "UTF8", + None, + ), # 2 + ( + 3, + TType.STRING, + "source_table_name", + "UTF8", + None, + ), # 3 + ( + 4, + TType.STRING, + "dest_db", + "UTF8", + None, + ), # 4 + ( + 5, + TType.STRING, + "dest_table_name", + "UTF8", + None, + ), # 5 +) + + +class exchange_partitions_result: + """ + Attributes: + - success + - o1 + - o2 + - o3 + - o4 + + """ + + def __init__( + self, + success=None, + o1=None, + o2=None, + o3=None, + o4=None, + ): + self.success = success + self.o1 = o1 + self.o2 = o2 + self.o3 = o3 + self.o4 = o4 + + def read(self, iprot): + if ( + iprot._fast_decode is not None + and isinstance(iprot.trans, TTransport.CReadableTransport) + and self.thrift_spec is not None + ): + iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec]) + return + iprot.readStructBegin() + while True: + (fname, ftype, fid) = iprot.readFieldBegin() + if ftype == TType.STOP: + break + if fid == 0: + if ftype == TType.LIST: + self.success = [] + (_etype1503, _size1500) = iprot.readListBegin() + for _i1504 in range(_size1500): + _elem1505 = Partition() + _elem1505.read(iprot) + self.success.append(_elem1505) + iprot.readListEnd() + else: + iprot.skip(ftype) + elif fid == 1: + if ftype == TType.STRUCT: + self.o1 = MetaException.read(iprot) + else: + iprot.skip(ftype) + elif fid == 2: + if ftype == TType.STRUCT: + self.o2 = NoSuchObjectException.read(iprot) + else: + iprot.skip(ftype) + elif fid == 3: + if ftype == TType.STRUCT: + self.o3 = InvalidObjectException.read(iprot) + else: + iprot.skip(ftype) + elif fid == 4: + if ftype == TType.STRUCT: + self.o4 = InvalidInputException.read(iprot) + else: + iprot.skip(ftype) + else: + iprot.skip(ftype) + iprot.readFieldEnd() + iprot.readStructEnd() + + def write(self, oprot): + if oprot._fast_encode is not None and self.thrift_spec is not None: + oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec])) + return + oprot.writeStructBegin("exchange_partitions_result") + if self.success is not None: + oprot.writeFieldBegin("success", TType.LIST, 0) + oprot.writeListBegin(TType.STRUCT, len(self.success)) + for iter1506 in self.success: + iter1506.write(oprot) + oprot.writeListEnd() + oprot.writeFieldEnd() + if self.o1 is not None: + oprot.writeFieldBegin("o1", TType.STRUCT, 1) + self.o1.write(oprot) + oprot.writeFieldEnd() + if self.o2 is not None: + oprot.writeFieldBegin("o2", TType.STRUCT, 2) + self.o2.write(oprot) + oprot.writeFieldEnd() + if self.o3 is not None: + oprot.writeFieldBegin("o3", TType.STRUCT, 3) + self.o3.write(oprot) + oprot.writeFieldEnd() + if self.o4 is not None: + oprot.writeFieldBegin("o4", TType.STRUCT, 4) + self.o4.write(oprot) + oprot.writeFieldEnd() + oprot.writeFieldStop() + oprot.writeStructEnd() + + def validate(self): + return + + def __repr__(self): + L = ["{}={!r}".format(key, value) for key, value in self.__dict__.items()] + return "{}({})".format(self.__class__.__name__, ", ".join(L)) + + def __eq__(self, other): + return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ + + def __ne__(self, other): + return not (self == other) + + +all_structs.append(exchange_partitions_result) +exchange_partitions_result.thrift_spec = ( + ( + 0, + TType.LIST, + "success", + (TType.STRUCT, [Partition, None], False), + None, + ), # 0 + ( + 1, + TType.STRUCT, + "o1", + [MetaException, None], + None, + ), # 1 + ( + 2, + TType.STRUCT, + "o2", + [NoSuchObjectException, None], + None, + ), # 2 + ( + 3, + TType.STRUCT, + "o3", + [InvalidObjectException, None], + None, + ), # 3 + ( + 4, + TType.STRUCT, + "o4", + [InvalidInputException, None], + None, + ), # 4 +) + + +class get_partition_with_auth_args: + """ + Attributes: + - db_name + - tbl_name + - part_vals + - user_name + - group_names + + """ + + def __init__( + self, + db_name=None, + tbl_name=None, + part_vals=None, + user_name=None, + group_names=None, + ): + self.db_name = db_name + self.tbl_name = tbl_name + self.part_vals = part_vals + self.user_name = user_name + self.group_names = group_names + + def read(self, iprot): + if ( + iprot._fast_decode is not None + and isinstance(iprot.trans, TTransport.CReadableTransport) + and self.thrift_spec is not None + ): + iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec]) + return + iprot.readStructBegin() + while True: + (fname, ftype, fid) = iprot.readFieldBegin() + if ftype == TType.STOP: + break + if fid == 1: + if ftype == TType.STRING: + self.db_name = ( + iprot.readString().decode("utf-8", errors="replace") if sys.version_info[0] == 2 else iprot.readString() + ) + else: + iprot.skip(ftype) + elif fid == 2: + if ftype == TType.STRING: + self.tbl_name = ( + iprot.readString().decode("utf-8", errors="replace") if sys.version_info[0] == 2 else iprot.readString() + ) + else: + iprot.skip(ftype) + elif fid == 3: + if ftype == TType.LIST: + self.part_vals = [] + (_etype1510, _size1507) = iprot.readListBegin() + for _i1511 in range(_size1507): + _elem1512 = ( + iprot.readString().decode("utf-8", errors="replace") + if sys.version_info[0] == 2 + else iprot.readString() + ) + self.part_vals.append(_elem1512) + iprot.readListEnd() + else: + iprot.skip(ftype) + elif fid == 4: + if ftype == TType.STRING: + self.user_name = ( + iprot.readString().decode("utf-8", errors="replace") if sys.version_info[0] == 2 else iprot.readString() + ) + else: + iprot.skip(ftype) + elif fid == 5: + if ftype == TType.LIST: + self.group_names = [] + (_etype1516, _size1513) = iprot.readListBegin() + for _i1517 in range(_size1513): + _elem1518 = ( + iprot.readString().decode("utf-8", errors="replace") + if sys.version_info[0] == 2 + else iprot.readString() + ) + self.group_names.append(_elem1518) + iprot.readListEnd() + else: + iprot.skip(ftype) + else: + iprot.skip(ftype) + iprot.readFieldEnd() + iprot.readStructEnd() + + def write(self, oprot): + if oprot._fast_encode is not None and self.thrift_spec is not None: + oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec])) + return + oprot.writeStructBegin("get_partition_with_auth_args") + if self.db_name is not None: + oprot.writeFieldBegin("db_name", TType.STRING, 1) + oprot.writeString(self.db_name.encode("utf-8") if sys.version_info[0] == 2 else self.db_name) + oprot.writeFieldEnd() + if self.tbl_name is not None: + oprot.writeFieldBegin("tbl_name", TType.STRING, 2) + oprot.writeString(self.tbl_name.encode("utf-8") if sys.version_info[0] == 2 else self.tbl_name) + oprot.writeFieldEnd() + if self.part_vals is not None: + oprot.writeFieldBegin("part_vals", TType.LIST, 3) + oprot.writeListBegin(TType.STRING, len(self.part_vals)) + for iter1519 in self.part_vals: + oprot.writeString(iter1519.encode("utf-8") if sys.version_info[0] == 2 else iter1519) + oprot.writeListEnd() + oprot.writeFieldEnd() + if self.user_name is not None: + oprot.writeFieldBegin("user_name", TType.STRING, 4) + oprot.writeString(self.user_name.encode("utf-8") if sys.version_info[0] == 2 else self.user_name) + oprot.writeFieldEnd() + if self.group_names is not None: + oprot.writeFieldBegin("group_names", TType.LIST, 5) + oprot.writeListBegin(TType.STRING, len(self.group_names)) + for iter1520 in self.group_names: + oprot.writeString(iter1520.encode("utf-8") if sys.version_info[0] == 2 else iter1520) + oprot.writeListEnd() + oprot.writeFieldEnd() + oprot.writeFieldStop() + oprot.writeStructEnd() + + def validate(self): + return + + def __repr__(self): + L = ["{}={!r}".format(key, value) for key, value in self.__dict__.items()] + return "{}({})".format(self.__class__.__name__, ", ".join(L)) + + def __eq__(self, other): + return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ + + def __ne__(self, other): + return not (self == other) + + +all_structs.append(get_partition_with_auth_args) +get_partition_with_auth_args.thrift_spec = ( + None, # 0 + ( + 1, + TType.STRING, + "db_name", + "UTF8", + None, + ), # 1 + ( + 2, + TType.STRING, + "tbl_name", + "UTF8", + None, + ), # 2 + ( + 3, + TType.LIST, + "part_vals", + (TType.STRING, "UTF8", False), + None, + ), # 3 + ( + 4, + TType.STRING, + "user_name", + "UTF8", + None, + ), # 4 + ( + 5, + TType.LIST, + "group_names", + (TType.STRING, "UTF8", False), + None, + ), # 5 +) + + +class get_partition_with_auth_result: + """ + Attributes: + - success + - o1 + - o2 + + """ + + def __init__( + self, + success=None, + o1=None, + o2=None, + ): + self.success = success + self.o1 = o1 + self.o2 = o2 + + def read(self, iprot): + if ( + iprot._fast_decode is not None + and isinstance(iprot.trans, TTransport.CReadableTransport) + and self.thrift_spec is not None + ): + iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec]) + return + iprot.readStructBegin() + while True: + (fname, ftype, fid) = iprot.readFieldBegin() + if ftype == TType.STOP: + break + if fid == 0: + if ftype == TType.STRUCT: + self.success = Partition() + self.success.read(iprot) + else: + iprot.skip(ftype) + elif fid == 1: + if ftype == TType.STRUCT: + self.o1 = MetaException.read(iprot) + else: + iprot.skip(ftype) + elif fid == 2: + if ftype == TType.STRUCT: + self.o2 = NoSuchObjectException.read(iprot) + else: + iprot.skip(ftype) + else: + iprot.skip(ftype) + iprot.readFieldEnd() + iprot.readStructEnd() + + def write(self, oprot): + if oprot._fast_encode is not None and self.thrift_spec is not None: + oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec])) + return + oprot.writeStructBegin("get_partition_with_auth_result") + if self.success is not None: + oprot.writeFieldBegin("success", TType.STRUCT, 0) + self.success.write(oprot) + oprot.writeFieldEnd() + if self.o1 is not None: + oprot.writeFieldBegin("o1", TType.STRUCT, 1) + self.o1.write(oprot) + oprot.writeFieldEnd() + if self.o2 is not None: + oprot.writeFieldBegin("o2", TType.STRUCT, 2) + self.o2.write(oprot) + oprot.writeFieldEnd() + oprot.writeFieldStop() + oprot.writeStructEnd() + + def validate(self): + return + + def __repr__(self): + L = ["{}={!r}".format(key, value) for key, value in self.__dict__.items()] + return "{}({})".format(self.__class__.__name__, ", ".join(L)) + + def __eq__(self, other): + return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ + + def __ne__(self, other): + return not (self == other) + + +all_structs.append(get_partition_with_auth_result) +get_partition_with_auth_result.thrift_spec = ( + ( + 0, + TType.STRUCT, + "success", + [Partition, None], + None, + ), # 0 + ( + 1, + TType.STRUCT, + "o1", + [MetaException, None], + None, + ), # 1 + ( + 2, + TType.STRUCT, + "o2", + [NoSuchObjectException, None], + None, + ), # 2 +) + + +class get_partition_by_name_args: + """ + Attributes: + - db_name + - tbl_name + - part_name + + """ + + def __init__( + self, + db_name=None, + tbl_name=None, + part_name=None, + ): + self.db_name = db_name + self.tbl_name = tbl_name + self.part_name = part_name + + def read(self, iprot): + if ( + iprot._fast_decode is not None + and isinstance(iprot.trans, TTransport.CReadableTransport) + and self.thrift_spec is not None + ): + iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec]) + return + iprot.readStructBegin() + while True: + (fname, ftype, fid) = iprot.readFieldBegin() + if ftype == TType.STOP: + break + if fid == 1: + if ftype == TType.STRING: + self.db_name = ( + iprot.readString().decode("utf-8", errors="replace") if sys.version_info[0] == 2 else iprot.readString() + ) + else: + iprot.skip(ftype) + elif fid == 2: + if ftype == TType.STRING: + self.tbl_name = ( + iprot.readString().decode("utf-8", errors="replace") if sys.version_info[0] == 2 else iprot.readString() + ) + else: + iprot.skip(ftype) + elif fid == 3: + if ftype == TType.STRING: + self.part_name = ( + iprot.readString().decode("utf-8", errors="replace") if sys.version_info[0] == 2 else iprot.readString() + ) + else: + iprot.skip(ftype) + else: + iprot.skip(ftype) + iprot.readFieldEnd() + iprot.readStructEnd() + + def write(self, oprot): + if oprot._fast_encode is not None and self.thrift_spec is not None: + oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec])) + return + oprot.writeStructBegin("get_partition_by_name_args") + if self.db_name is not None: + oprot.writeFieldBegin("db_name", TType.STRING, 1) + oprot.writeString(self.db_name.encode("utf-8") if sys.version_info[0] == 2 else self.db_name) + oprot.writeFieldEnd() + if self.tbl_name is not None: + oprot.writeFieldBegin("tbl_name", TType.STRING, 2) + oprot.writeString(self.tbl_name.encode("utf-8") if sys.version_info[0] == 2 else self.tbl_name) + oprot.writeFieldEnd() + if self.part_name is not None: + oprot.writeFieldBegin("part_name", TType.STRING, 3) + oprot.writeString(self.part_name.encode("utf-8") if sys.version_info[0] == 2 else self.part_name) + oprot.writeFieldEnd() + oprot.writeFieldStop() + oprot.writeStructEnd() + + def validate(self): + return + + def __repr__(self): + L = ["{}={!r}".format(key, value) for key, value in self.__dict__.items()] + return "{}({})".format(self.__class__.__name__, ", ".join(L)) + + def __eq__(self, other): + return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ + + def __ne__(self, other): + return not (self == other) + + +all_structs.append(get_partition_by_name_args) +get_partition_by_name_args.thrift_spec = ( + None, # 0 + ( + 1, + TType.STRING, + "db_name", + "UTF8", + None, + ), # 1 + ( + 2, + TType.STRING, + "tbl_name", + "UTF8", + None, + ), # 2 + ( + 3, + TType.STRING, + "part_name", + "UTF8", + None, + ), # 3 +) + + +class get_partition_by_name_result: + """ + Attributes: + - success + - o1 + - o2 + + """ + + def __init__( + self, + success=None, + o1=None, + o2=None, + ): + self.success = success + self.o1 = o1 + self.o2 = o2 + + def read(self, iprot): + if ( + iprot._fast_decode is not None + and isinstance(iprot.trans, TTransport.CReadableTransport) + and self.thrift_spec is not None + ): + iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec]) + return + iprot.readStructBegin() + while True: + (fname, ftype, fid) = iprot.readFieldBegin() + if ftype == TType.STOP: + break + if fid == 0: + if ftype == TType.STRUCT: + self.success = Partition() + self.success.read(iprot) + else: + iprot.skip(ftype) + elif fid == 1: + if ftype == TType.STRUCT: + self.o1 = MetaException.read(iprot) + else: + iprot.skip(ftype) + elif fid == 2: + if ftype == TType.STRUCT: + self.o2 = NoSuchObjectException.read(iprot) + else: + iprot.skip(ftype) + else: + iprot.skip(ftype) + iprot.readFieldEnd() + iprot.readStructEnd() + + def write(self, oprot): + if oprot._fast_encode is not None and self.thrift_spec is not None: + oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec])) + return + oprot.writeStructBegin("get_partition_by_name_result") + if self.success is not None: + oprot.writeFieldBegin("success", TType.STRUCT, 0) + self.success.write(oprot) + oprot.writeFieldEnd() + if self.o1 is not None: + oprot.writeFieldBegin("o1", TType.STRUCT, 1) + self.o1.write(oprot) + oprot.writeFieldEnd() + if self.o2 is not None: + oprot.writeFieldBegin("o2", TType.STRUCT, 2) + self.o2.write(oprot) + oprot.writeFieldEnd() + oprot.writeFieldStop() + oprot.writeStructEnd() + + def validate(self): + return + + def __repr__(self): + L = ["{}={!r}".format(key, value) for key, value in self.__dict__.items()] + return "{}({})".format(self.__class__.__name__, ", ".join(L)) + + def __eq__(self, other): + return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ + + def __ne__(self, other): + return not (self == other) + + +all_structs.append(get_partition_by_name_result) +get_partition_by_name_result.thrift_spec = ( + ( + 0, + TType.STRUCT, + "success", + [Partition, None], + None, + ), # 0 + ( + 1, + TType.STRUCT, + "o1", + [MetaException, None], + None, + ), # 1 + ( + 2, + TType.STRUCT, + "o2", + [NoSuchObjectException, None], + None, + ), # 2 +) + + +class get_partitions_args: + """ + Attributes: + - db_name + - tbl_name + - max_parts + + """ + + def __init__( + self, + db_name=None, + tbl_name=None, + max_parts=-1, + ): + self.db_name = db_name + self.tbl_name = tbl_name + self.max_parts = max_parts + + def read(self, iprot): + if ( + iprot._fast_decode is not None + and isinstance(iprot.trans, TTransport.CReadableTransport) + and self.thrift_spec is not None + ): + iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec]) + return + iprot.readStructBegin() + while True: + (fname, ftype, fid) = iprot.readFieldBegin() + if ftype == TType.STOP: + break + if fid == 1: + if ftype == TType.STRING: + self.db_name = ( + iprot.readString().decode("utf-8", errors="replace") if sys.version_info[0] == 2 else iprot.readString() + ) + else: + iprot.skip(ftype) + elif fid == 2: + if ftype == TType.STRING: + self.tbl_name = ( + iprot.readString().decode("utf-8", errors="replace") if sys.version_info[0] == 2 else iprot.readString() + ) + else: + iprot.skip(ftype) + elif fid == 3: + if ftype == TType.I16: + self.max_parts = iprot.readI16() + else: + iprot.skip(ftype) + else: + iprot.skip(ftype) + iprot.readFieldEnd() + iprot.readStructEnd() + + def write(self, oprot): + if oprot._fast_encode is not None and self.thrift_spec is not None: + oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec])) + return + oprot.writeStructBegin("get_partitions_args") + if self.db_name is not None: + oprot.writeFieldBegin("db_name", TType.STRING, 1) + oprot.writeString(self.db_name.encode("utf-8") if sys.version_info[0] == 2 else self.db_name) + oprot.writeFieldEnd() + if self.tbl_name is not None: + oprot.writeFieldBegin("tbl_name", TType.STRING, 2) + oprot.writeString(self.tbl_name.encode("utf-8") if sys.version_info[0] == 2 else self.tbl_name) + oprot.writeFieldEnd() + if self.max_parts is not None: + oprot.writeFieldBegin("max_parts", TType.I16, 3) + oprot.writeI16(self.max_parts) + oprot.writeFieldEnd() + oprot.writeFieldStop() + oprot.writeStructEnd() + + def validate(self): + return + + def __repr__(self): + L = ["{}={!r}".format(key, value) for key, value in self.__dict__.items()] + return "{}({})".format(self.__class__.__name__, ", ".join(L)) + + def __eq__(self, other): + return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ + + def __ne__(self, other): + return not (self == other) + + +all_structs.append(get_partitions_args) +get_partitions_args.thrift_spec = ( + None, # 0 + ( + 1, + TType.STRING, + "db_name", + "UTF8", + None, + ), # 1 + ( + 2, + TType.STRING, + "tbl_name", + "UTF8", + None, + ), # 2 + ( + 3, + TType.I16, + "max_parts", + None, + -1, + ), # 3 +) + + +class get_partitions_result: + """ + Attributes: + - success + - o1 + - o2 + + """ + + def __init__( + self, + success=None, + o1=None, + o2=None, + ): + self.success = success + self.o1 = o1 + self.o2 = o2 + + def read(self, iprot): + if ( + iprot._fast_decode is not None + and isinstance(iprot.trans, TTransport.CReadableTransport) + and self.thrift_spec is not None + ): + iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec]) + return + iprot.readStructBegin() + while True: + (fname, ftype, fid) = iprot.readFieldBegin() + if ftype == TType.STOP: + break + if fid == 0: + if ftype == TType.LIST: + self.success = [] + (_etype1524, _size1521) = iprot.readListBegin() + for _i1525 in range(_size1521): + _elem1526 = Partition() + _elem1526.read(iprot) + self.success.append(_elem1526) + iprot.readListEnd() + else: + iprot.skip(ftype) + elif fid == 1: + if ftype == TType.STRUCT: + self.o1 = NoSuchObjectException.read(iprot) + else: + iprot.skip(ftype) + elif fid == 2: + if ftype == TType.STRUCT: + self.o2 = MetaException.read(iprot) + else: + iprot.skip(ftype) + else: + iprot.skip(ftype) + iprot.readFieldEnd() + iprot.readStructEnd() + + def write(self, oprot): + if oprot._fast_encode is not None and self.thrift_spec is not None: + oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec])) + return + oprot.writeStructBegin("get_partitions_result") + if self.success is not None: + oprot.writeFieldBegin("success", TType.LIST, 0) + oprot.writeListBegin(TType.STRUCT, len(self.success)) + for iter1527 in self.success: + iter1527.write(oprot) + oprot.writeListEnd() + oprot.writeFieldEnd() + if self.o1 is not None: + oprot.writeFieldBegin("o1", TType.STRUCT, 1) + self.o1.write(oprot) + oprot.writeFieldEnd() + if self.o2 is not None: + oprot.writeFieldBegin("o2", TType.STRUCT, 2) + self.o2.write(oprot) + oprot.writeFieldEnd() + oprot.writeFieldStop() + oprot.writeStructEnd() + + def validate(self): + return + + def __repr__(self): + L = ["{}={!r}".format(key, value) for key, value in self.__dict__.items()] + return "{}({})".format(self.__class__.__name__, ", ".join(L)) + + def __eq__(self, other): + return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ + + def __ne__(self, other): + return not (self == other) + + +all_structs.append(get_partitions_result) +get_partitions_result.thrift_spec = ( + ( + 0, + TType.LIST, + "success", + (TType.STRUCT, [Partition, None], False), + None, + ), # 0 + ( + 1, + TType.STRUCT, + "o1", + [NoSuchObjectException, None], + None, + ), # 1 + ( + 2, + TType.STRUCT, + "o2", + [MetaException, None], + None, + ), # 2 +) + + +class get_partitions_req_args: + """ + Attributes: + - req + + """ + + def __init__( + self, + req=None, + ): + self.req = req + + def read(self, iprot): + if ( + iprot._fast_decode is not None + and isinstance(iprot.trans, TTransport.CReadableTransport) + and self.thrift_spec is not None + ): + iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec]) + return + iprot.readStructBegin() + while True: + (fname, ftype, fid) = iprot.readFieldBegin() + if ftype == TType.STOP: + break + if fid == 1: + if ftype == TType.STRUCT: + self.req = PartitionsRequest() + self.req.read(iprot) + else: + iprot.skip(ftype) + else: + iprot.skip(ftype) + iprot.readFieldEnd() + iprot.readStructEnd() + + def write(self, oprot): + if oprot._fast_encode is not None and self.thrift_spec is not None: + oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec])) + return + oprot.writeStructBegin("get_partitions_req_args") + if self.req is not None: + oprot.writeFieldBegin("req", TType.STRUCT, 1) + self.req.write(oprot) + oprot.writeFieldEnd() + oprot.writeFieldStop() + oprot.writeStructEnd() + + def validate(self): + return + + def __repr__(self): + L = ["{}={!r}".format(key, value) for key, value in self.__dict__.items()] + return "{}({})".format(self.__class__.__name__, ", ".join(L)) + + def __eq__(self, other): + return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ + + def __ne__(self, other): + return not (self == other) + + +all_structs.append(get_partitions_req_args) +get_partitions_req_args.thrift_spec = ( + None, # 0 + ( + 1, + TType.STRUCT, + "req", + [PartitionsRequest, None], + None, + ), # 1 +) + + +class get_partitions_req_result: + """ + Attributes: + - success + - o1 + - o2 + + """ + + def __init__( + self, + success=None, + o1=None, + o2=None, + ): + self.success = success + self.o1 = o1 + self.o2 = o2 + + def read(self, iprot): + if ( + iprot._fast_decode is not None + and isinstance(iprot.trans, TTransport.CReadableTransport) + and self.thrift_spec is not None + ): + iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec]) + return + iprot.readStructBegin() + while True: + (fname, ftype, fid) = iprot.readFieldBegin() + if ftype == TType.STOP: + break + if fid == 0: + if ftype == TType.STRUCT: + self.success = PartitionsResponse() + self.success.read(iprot) + else: + iprot.skip(ftype) + elif fid == 1: + if ftype == TType.STRUCT: + self.o1 = NoSuchObjectException.read(iprot) + else: + iprot.skip(ftype) + elif fid == 2: + if ftype == TType.STRUCT: + self.o2 = MetaException.read(iprot) + else: + iprot.skip(ftype) + else: + iprot.skip(ftype) + iprot.readFieldEnd() + iprot.readStructEnd() + + def write(self, oprot): + if oprot._fast_encode is not None and self.thrift_spec is not None: + oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec])) + return + oprot.writeStructBegin("get_partitions_req_result") + if self.success is not None: + oprot.writeFieldBegin("success", TType.STRUCT, 0) + self.success.write(oprot) + oprot.writeFieldEnd() + if self.o1 is not None: + oprot.writeFieldBegin("o1", TType.STRUCT, 1) + self.o1.write(oprot) + oprot.writeFieldEnd() + if self.o2 is not None: + oprot.writeFieldBegin("o2", TType.STRUCT, 2) + self.o2.write(oprot) + oprot.writeFieldEnd() + oprot.writeFieldStop() + oprot.writeStructEnd() + + def validate(self): + return + + def __repr__(self): + L = ["{}={!r}".format(key, value) for key, value in self.__dict__.items()] + return "{}({})".format(self.__class__.__name__, ", ".join(L)) + + def __eq__(self, other): + return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ + + def __ne__(self, other): + return not (self == other) + + +all_structs.append(get_partitions_req_result) +get_partitions_req_result.thrift_spec = ( + ( + 0, + TType.STRUCT, + "success", + [PartitionsResponse, None], + None, + ), # 0 + ( + 1, + TType.STRUCT, + "o1", + [NoSuchObjectException, None], + None, + ), # 1 + ( + 2, + TType.STRUCT, + "o2", + [MetaException, None], + None, + ), # 2 +) + + +class get_partitions_with_auth_args: + """ + Attributes: + - db_name + - tbl_name + - max_parts + - user_name + - group_names + + """ + + def __init__( + self, + db_name=None, + tbl_name=None, + max_parts=-1, + user_name=None, + group_names=None, + ): + self.db_name = db_name + self.tbl_name = tbl_name + self.max_parts = max_parts + self.user_name = user_name + self.group_names = group_names + + def read(self, iprot): + if ( + iprot._fast_decode is not None + and isinstance(iprot.trans, TTransport.CReadableTransport) + and self.thrift_spec is not None + ): + iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec]) + return + iprot.readStructBegin() + while True: + (fname, ftype, fid) = iprot.readFieldBegin() + if ftype == TType.STOP: + break + if fid == 1: + if ftype == TType.STRING: + self.db_name = ( + iprot.readString().decode("utf-8", errors="replace") if sys.version_info[0] == 2 else iprot.readString() + ) + else: + iprot.skip(ftype) + elif fid == 2: + if ftype == TType.STRING: + self.tbl_name = ( + iprot.readString().decode("utf-8", errors="replace") if sys.version_info[0] == 2 else iprot.readString() + ) + else: + iprot.skip(ftype) + elif fid == 3: + if ftype == TType.I16: + self.max_parts = iprot.readI16() + else: + iprot.skip(ftype) + elif fid == 4: + if ftype == TType.STRING: + self.user_name = ( + iprot.readString().decode("utf-8", errors="replace") if sys.version_info[0] == 2 else iprot.readString() + ) + else: + iprot.skip(ftype) + elif fid == 5: + if ftype == TType.LIST: + self.group_names = [] + (_etype1531, _size1528) = iprot.readListBegin() + for _i1532 in range(_size1528): + _elem1533 = ( + iprot.readString().decode("utf-8", errors="replace") + if sys.version_info[0] == 2 + else iprot.readString() + ) + self.group_names.append(_elem1533) + iprot.readListEnd() + else: + iprot.skip(ftype) + else: + iprot.skip(ftype) + iprot.readFieldEnd() + iprot.readStructEnd() + + def write(self, oprot): + if oprot._fast_encode is not None and self.thrift_spec is not None: + oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec])) + return + oprot.writeStructBegin("get_partitions_with_auth_args") + if self.db_name is not None: + oprot.writeFieldBegin("db_name", TType.STRING, 1) + oprot.writeString(self.db_name.encode("utf-8") if sys.version_info[0] == 2 else self.db_name) + oprot.writeFieldEnd() + if self.tbl_name is not None: + oprot.writeFieldBegin("tbl_name", TType.STRING, 2) + oprot.writeString(self.tbl_name.encode("utf-8") if sys.version_info[0] == 2 else self.tbl_name) + oprot.writeFieldEnd() + if self.max_parts is not None: + oprot.writeFieldBegin("max_parts", TType.I16, 3) + oprot.writeI16(self.max_parts) + oprot.writeFieldEnd() + if self.user_name is not None: + oprot.writeFieldBegin("user_name", TType.STRING, 4) + oprot.writeString(self.user_name.encode("utf-8") if sys.version_info[0] == 2 else self.user_name) + oprot.writeFieldEnd() + if self.group_names is not None: + oprot.writeFieldBegin("group_names", TType.LIST, 5) + oprot.writeListBegin(TType.STRING, len(self.group_names)) + for iter1534 in self.group_names: + oprot.writeString(iter1534.encode("utf-8") if sys.version_info[0] == 2 else iter1534) + oprot.writeListEnd() + oprot.writeFieldEnd() + oprot.writeFieldStop() + oprot.writeStructEnd() + + def validate(self): + return + + def __repr__(self): + L = ["{}={!r}".format(key, value) for key, value in self.__dict__.items()] + return "{}({})".format(self.__class__.__name__, ", ".join(L)) + + def __eq__(self, other): + return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ + + def __ne__(self, other): + return not (self == other) + + +all_structs.append(get_partitions_with_auth_args) +get_partitions_with_auth_args.thrift_spec = ( + None, # 0 + ( + 1, + TType.STRING, + "db_name", + "UTF8", + None, + ), # 1 + ( + 2, + TType.STRING, + "tbl_name", + "UTF8", + None, + ), # 2 + ( + 3, + TType.I16, + "max_parts", + None, + -1, + ), # 3 + ( + 4, + TType.STRING, + "user_name", + "UTF8", + None, + ), # 4 + ( + 5, + TType.LIST, + "group_names", + (TType.STRING, "UTF8", False), + None, + ), # 5 +) + + +class get_partitions_with_auth_result: + """ + Attributes: + - success + - o1 + - o2 + + """ + + def __init__( + self, + success=None, + o1=None, + o2=None, + ): + self.success = success + self.o1 = o1 + self.o2 = o2 + + def read(self, iprot): + if ( + iprot._fast_decode is not None + and isinstance(iprot.trans, TTransport.CReadableTransport) + and self.thrift_spec is not None + ): + iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec]) + return + iprot.readStructBegin() + while True: + (fname, ftype, fid) = iprot.readFieldBegin() + if ftype == TType.STOP: + break + if fid == 0: + if ftype == TType.LIST: + self.success = [] + (_etype1538, _size1535) = iprot.readListBegin() + for _i1539 in range(_size1535): + _elem1540 = Partition() + _elem1540.read(iprot) + self.success.append(_elem1540) + iprot.readListEnd() + else: + iprot.skip(ftype) + elif fid == 1: + if ftype == TType.STRUCT: + self.o1 = NoSuchObjectException.read(iprot) + else: + iprot.skip(ftype) + elif fid == 2: + if ftype == TType.STRUCT: + self.o2 = MetaException.read(iprot) + else: + iprot.skip(ftype) + else: + iprot.skip(ftype) + iprot.readFieldEnd() + iprot.readStructEnd() + + def write(self, oprot): + if oprot._fast_encode is not None and self.thrift_spec is not None: + oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec])) + return + oprot.writeStructBegin("get_partitions_with_auth_result") + if self.success is not None: + oprot.writeFieldBegin("success", TType.LIST, 0) + oprot.writeListBegin(TType.STRUCT, len(self.success)) + for iter1541 in self.success: + iter1541.write(oprot) + oprot.writeListEnd() + oprot.writeFieldEnd() + if self.o1 is not None: + oprot.writeFieldBegin("o1", TType.STRUCT, 1) + self.o1.write(oprot) + oprot.writeFieldEnd() + if self.o2 is not None: + oprot.writeFieldBegin("o2", TType.STRUCT, 2) + self.o2.write(oprot) + oprot.writeFieldEnd() + oprot.writeFieldStop() + oprot.writeStructEnd() + + def validate(self): + return + + def __repr__(self): + L = ["{}={!r}".format(key, value) for key, value in self.__dict__.items()] + return "{}({})".format(self.__class__.__name__, ", ".join(L)) + + def __eq__(self, other): + return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ + + def __ne__(self, other): + return not (self == other) + + +all_structs.append(get_partitions_with_auth_result) +get_partitions_with_auth_result.thrift_spec = ( + ( + 0, + TType.LIST, + "success", + (TType.STRUCT, [Partition, None], False), + None, + ), # 0 + ( + 1, + TType.STRUCT, + "o1", + [NoSuchObjectException, None], + None, + ), # 1 + ( + 2, + TType.STRUCT, + "o2", + [MetaException, None], + None, + ), # 2 +) + + +class get_partitions_pspec_args: + """ + Attributes: + - db_name + - tbl_name + - max_parts + + """ + + def __init__( + self, + db_name=None, + tbl_name=None, + max_parts=-1, + ): + self.db_name = db_name + self.tbl_name = tbl_name + self.max_parts = max_parts + + def read(self, iprot): + if ( + iprot._fast_decode is not None + and isinstance(iprot.trans, TTransport.CReadableTransport) + and self.thrift_spec is not None + ): + iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec]) + return + iprot.readStructBegin() + while True: + (fname, ftype, fid) = iprot.readFieldBegin() + if ftype == TType.STOP: + break + if fid == 1: + if ftype == TType.STRING: + self.db_name = ( + iprot.readString().decode("utf-8", errors="replace") if sys.version_info[0] == 2 else iprot.readString() + ) + else: + iprot.skip(ftype) + elif fid == 2: + if ftype == TType.STRING: + self.tbl_name = ( + iprot.readString().decode("utf-8", errors="replace") if sys.version_info[0] == 2 else iprot.readString() + ) + else: + iprot.skip(ftype) + elif fid == 3: + if ftype == TType.I32: + self.max_parts = iprot.readI32() + else: + iprot.skip(ftype) + else: + iprot.skip(ftype) + iprot.readFieldEnd() + iprot.readStructEnd() + + def write(self, oprot): + if oprot._fast_encode is not None and self.thrift_spec is not None: + oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec])) + return + oprot.writeStructBegin("get_partitions_pspec_args") + if self.db_name is not None: + oprot.writeFieldBegin("db_name", TType.STRING, 1) + oprot.writeString(self.db_name.encode("utf-8") if sys.version_info[0] == 2 else self.db_name) + oprot.writeFieldEnd() + if self.tbl_name is not None: + oprot.writeFieldBegin("tbl_name", TType.STRING, 2) + oprot.writeString(self.tbl_name.encode("utf-8") if sys.version_info[0] == 2 else self.tbl_name) + oprot.writeFieldEnd() + if self.max_parts is not None: + oprot.writeFieldBegin("max_parts", TType.I32, 3) + oprot.writeI32(self.max_parts) + oprot.writeFieldEnd() + oprot.writeFieldStop() + oprot.writeStructEnd() + + def validate(self): + return + + def __repr__(self): + L = ["{}={!r}".format(key, value) for key, value in self.__dict__.items()] + return "{}({})".format(self.__class__.__name__, ", ".join(L)) + + def __eq__(self, other): + return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ + + def __ne__(self, other): + return not (self == other) + + +all_structs.append(get_partitions_pspec_args) +get_partitions_pspec_args.thrift_spec = ( + None, # 0 + ( + 1, + TType.STRING, + "db_name", + "UTF8", + None, + ), # 1 + ( + 2, + TType.STRING, + "tbl_name", + "UTF8", + None, + ), # 2 + ( + 3, + TType.I32, + "max_parts", + None, + -1, + ), # 3 +) + + +class get_partitions_pspec_result: + """ + Attributes: + - success + - o1 + - o2 + + """ + + def __init__( + self, + success=None, + o1=None, + o2=None, + ): + self.success = success + self.o1 = o1 + self.o2 = o2 + + def read(self, iprot): + if ( + iprot._fast_decode is not None + and isinstance(iprot.trans, TTransport.CReadableTransport) + and self.thrift_spec is not None + ): + iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec]) + return + iprot.readStructBegin() + while True: + (fname, ftype, fid) = iprot.readFieldBegin() + if ftype == TType.STOP: + break + if fid == 0: + if ftype == TType.LIST: + self.success = [] + (_etype1545, _size1542) = iprot.readListBegin() + for _i1546 in range(_size1542): + _elem1547 = PartitionSpec() + _elem1547.read(iprot) + self.success.append(_elem1547) + iprot.readListEnd() + else: + iprot.skip(ftype) + elif fid == 1: + if ftype == TType.STRUCT: + self.o1 = NoSuchObjectException.read(iprot) + else: + iprot.skip(ftype) + elif fid == 2: + if ftype == TType.STRUCT: + self.o2 = MetaException.read(iprot) + else: + iprot.skip(ftype) + else: + iprot.skip(ftype) + iprot.readFieldEnd() + iprot.readStructEnd() + + def write(self, oprot): + if oprot._fast_encode is not None and self.thrift_spec is not None: + oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec])) + return + oprot.writeStructBegin("get_partitions_pspec_result") + if self.success is not None: + oprot.writeFieldBegin("success", TType.LIST, 0) + oprot.writeListBegin(TType.STRUCT, len(self.success)) + for iter1548 in self.success: + iter1548.write(oprot) + oprot.writeListEnd() + oprot.writeFieldEnd() + if self.o1 is not None: + oprot.writeFieldBegin("o1", TType.STRUCT, 1) + self.o1.write(oprot) + oprot.writeFieldEnd() + if self.o2 is not None: + oprot.writeFieldBegin("o2", TType.STRUCT, 2) + self.o2.write(oprot) + oprot.writeFieldEnd() + oprot.writeFieldStop() + oprot.writeStructEnd() + + def validate(self): + return + + def __repr__(self): + L = ["{}={!r}".format(key, value) for key, value in self.__dict__.items()] + return "{}({})".format(self.__class__.__name__, ", ".join(L)) + + def __eq__(self, other): + return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ + + def __ne__(self, other): + return not (self == other) + + +all_structs.append(get_partitions_pspec_result) +get_partitions_pspec_result.thrift_spec = ( + ( + 0, + TType.LIST, + "success", + (TType.STRUCT, [PartitionSpec, None], False), + None, + ), # 0 + ( + 1, + TType.STRUCT, + "o1", + [NoSuchObjectException, None], + None, + ), # 1 + ( + 2, + TType.STRUCT, + "o2", + [MetaException, None], + None, + ), # 2 +) + + +class get_partition_names_args: + """ + Attributes: + - db_name + - tbl_name + - max_parts + + """ + + def __init__( + self, + db_name=None, + tbl_name=None, + max_parts=-1, + ): + self.db_name = db_name + self.tbl_name = tbl_name + self.max_parts = max_parts + + def read(self, iprot): + if ( + iprot._fast_decode is not None + and isinstance(iprot.trans, TTransport.CReadableTransport) + and self.thrift_spec is not None + ): + iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec]) + return + iprot.readStructBegin() + while True: + (fname, ftype, fid) = iprot.readFieldBegin() + if ftype == TType.STOP: + break + if fid == 1: + if ftype == TType.STRING: + self.db_name = ( + iprot.readString().decode("utf-8", errors="replace") if sys.version_info[0] == 2 else iprot.readString() + ) + else: + iprot.skip(ftype) + elif fid == 2: + if ftype == TType.STRING: + self.tbl_name = ( + iprot.readString().decode("utf-8", errors="replace") if sys.version_info[0] == 2 else iprot.readString() + ) + else: + iprot.skip(ftype) + elif fid == 3: + if ftype == TType.I16: + self.max_parts = iprot.readI16() + else: + iprot.skip(ftype) + else: + iprot.skip(ftype) + iprot.readFieldEnd() + iprot.readStructEnd() + + def write(self, oprot): + if oprot._fast_encode is not None and self.thrift_spec is not None: + oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec])) + return + oprot.writeStructBegin("get_partition_names_args") + if self.db_name is not None: + oprot.writeFieldBegin("db_name", TType.STRING, 1) + oprot.writeString(self.db_name.encode("utf-8") if sys.version_info[0] == 2 else self.db_name) + oprot.writeFieldEnd() + if self.tbl_name is not None: + oprot.writeFieldBegin("tbl_name", TType.STRING, 2) + oprot.writeString(self.tbl_name.encode("utf-8") if sys.version_info[0] == 2 else self.tbl_name) + oprot.writeFieldEnd() + if self.max_parts is not None: + oprot.writeFieldBegin("max_parts", TType.I16, 3) + oprot.writeI16(self.max_parts) + oprot.writeFieldEnd() + oprot.writeFieldStop() + oprot.writeStructEnd() + + def validate(self): + return + + def __repr__(self): + L = ["{}={!r}".format(key, value) for key, value in self.__dict__.items()] + return "{}({})".format(self.__class__.__name__, ", ".join(L)) + + def __eq__(self, other): + return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ + + def __ne__(self, other): + return not (self == other) + + +all_structs.append(get_partition_names_args) +get_partition_names_args.thrift_spec = ( + None, # 0 + ( + 1, + TType.STRING, + "db_name", + "UTF8", + None, + ), # 1 + ( + 2, + TType.STRING, + "tbl_name", + "UTF8", + None, + ), # 2 + ( + 3, + TType.I16, + "max_parts", + None, + -1, + ), # 3 +) + + +class get_partition_names_result: + """ + Attributes: + - success + - o1 + - o2 + + """ + + def __init__( + self, + success=None, + o1=None, + o2=None, + ): + self.success = success + self.o1 = o1 + self.o2 = o2 + + def read(self, iprot): + if ( + iprot._fast_decode is not None + and isinstance(iprot.trans, TTransport.CReadableTransport) + and self.thrift_spec is not None + ): + iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec]) + return + iprot.readStructBegin() + while True: + (fname, ftype, fid) = iprot.readFieldBegin() + if ftype == TType.STOP: + break + if fid == 0: + if ftype == TType.LIST: + self.success = [] + (_etype1552, _size1549) = iprot.readListBegin() + for _i1553 in range(_size1549): + _elem1554 = ( + iprot.readString().decode("utf-8", errors="replace") + if sys.version_info[0] == 2 + else iprot.readString() + ) + self.success.append(_elem1554) + iprot.readListEnd() + else: + iprot.skip(ftype) + elif fid == 1: + if ftype == TType.STRUCT: + self.o1 = NoSuchObjectException.read(iprot) + else: + iprot.skip(ftype) + elif fid == 2: + if ftype == TType.STRUCT: + self.o2 = MetaException.read(iprot) + else: + iprot.skip(ftype) + else: + iprot.skip(ftype) + iprot.readFieldEnd() + iprot.readStructEnd() + + def write(self, oprot): + if oprot._fast_encode is not None and self.thrift_spec is not None: + oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec])) + return + oprot.writeStructBegin("get_partition_names_result") + if self.success is not None: + oprot.writeFieldBegin("success", TType.LIST, 0) + oprot.writeListBegin(TType.STRING, len(self.success)) + for iter1555 in self.success: + oprot.writeString(iter1555.encode("utf-8") if sys.version_info[0] == 2 else iter1555) + oprot.writeListEnd() + oprot.writeFieldEnd() + if self.o1 is not None: + oprot.writeFieldBegin("o1", TType.STRUCT, 1) + self.o1.write(oprot) + oprot.writeFieldEnd() + if self.o2 is not None: + oprot.writeFieldBegin("o2", TType.STRUCT, 2) + self.o2.write(oprot) + oprot.writeFieldEnd() + oprot.writeFieldStop() + oprot.writeStructEnd() + + def validate(self): + return + + def __repr__(self): + L = ["{}={!r}".format(key, value) for key, value in self.__dict__.items()] + return "{}({})".format(self.__class__.__name__, ", ".join(L)) + + def __eq__(self, other): + return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ + + def __ne__(self, other): + return not (self == other) + + +all_structs.append(get_partition_names_result) +get_partition_names_result.thrift_spec = ( + ( + 0, + TType.LIST, + "success", + (TType.STRING, "UTF8", False), + None, + ), # 0 + ( + 1, + TType.STRUCT, + "o1", + [NoSuchObjectException, None], + None, + ), # 1 + ( + 2, + TType.STRUCT, + "o2", + [MetaException, None], + None, + ), # 2 +) + + +class get_partition_values_args: + """ + Attributes: + - request + + """ + + def __init__( + self, + request=None, + ): + self.request = request + + def read(self, iprot): + if ( + iprot._fast_decode is not None + and isinstance(iprot.trans, TTransport.CReadableTransport) + and self.thrift_spec is not None + ): + iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec]) + return + iprot.readStructBegin() + while True: + (fname, ftype, fid) = iprot.readFieldBegin() + if ftype == TType.STOP: + break + if fid == 1: + if ftype == TType.STRUCT: + self.request = PartitionValuesRequest() + self.request.read(iprot) + else: + iprot.skip(ftype) + else: + iprot.skip(ftype) + iprot.readFieldEnd() + iprot.readStructEnd() + + def write(self, oprot): + if oprot._fast_encode is not None and self.thrift_spec is not None: + oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec])) + return + oprot.writeStructBegin("get_partition_values_args") + if self.request is not None: + oprot.writeFieldBegin("request", TType.STRUCT, 1) + self.request.write(oprot) + oprot.writeFieldEnd() + oprot.writeFieldStop() + oprot.writeStructEnd() + + def validate(self): + return + + def __repr__(self): + L = ["{}={!r}".format(key, value) for key, value in self.__dict__.items()] + return "{}({})".format(self.__class__.__name__, ", ".join(L)) + + def __eq__(self, other): + return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ + + def __ne__(self, other): + return not (self == other) + + +all_structs.append(get_partition_values_args) +get_partition_values_args.thrift_spec = ( + None, # 0 + ( + 1, + TType.STRUCT, + "request", + [PartitionValuesRequest, None], + None, + ), # 1 +) + + +class get_partition_values_result: + """ + Attributes: + - success + - o1 + - o2 + + """ + + def __init__( + self, + success=None, + o1=None, + o2=None, + ): + self.success = success + self.o1 = o1 + self.o2 = o2 + + def read(self, iprot): + if ( + iprot._fast_decode is not None + and isinstance(iprot.trans, TTransport.CReadableTransport) + and self.thrift_spec is not None + ): + iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec]) + return + iprot.readStructBegin() + while True: + (fname, ftype, fid) = iprot.readFieldBegin() + if ftype == TType.STOP: + break + if fid == 0: + if ftype == TType.STRUCT: + self.success = PartitionValuesResponse() + self.success.read(iprot) + else: + iprot.skip(ftype) + elif fid == 1: + if ftype == TType.STRUCT: + self.o1 = MetaException.read(iprot) + else: + iprot.skip(ftype) + elif fid == 2: + if ftype == TType.STRUCT: + self.o2 = NoSuchObjectException.read(iprot) + else: + iprot.skip(ftype) + else: + iprot.skip(ftype) + iprot.readFieldEnd() + iprot.readStructEnd() + + def write(self, oprot): + if oprot._fast_encode is not None and self.thrift_spec is not None: + oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec])) + return + oprot.writeStructBegin("get_partition_values_result") + if self.success is not None: + oprot.writeFieldBegin("success", TType.STRUCT, 0) + self.success.write(oprot) + oprot.writeFieldEnd() + if self.o1 is not None: + oprot.writeFieldBegin("o1", TType.STRUCT, 1) + self.o1.write(oprot) + oprot.writeFieldEnd() + if self.o2 is not None: + oprot.writeFieldBegin("o2", TType.STRUCT, 2) + self.o2.write(oprot) + oprot.writeFieldEnd() + oprot.writeFieldStop() + oprot.writeStructEnd() + + def validate(self): + return + + def __repr__(self): + L = ["{}={!r}".format(key, value) for key, value in self.__dict__.items()] + return "{}({})".format(self.__class__.__name__, ", ".join(L)) + + def __eq__(self, other): + return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ + + def __ne__(self, other): + return not (self == other) + + +all_structs.append(get_partition_values_result) +get_partition_values_result.thrift_spec = ( + ( + 0, + TType.STRUCT, + "success", + [PartitionValuesResponse, None], + None, + ), # 0 + ( + 1, + TType.STRUCT, + "o1", + [MetaException, None], + None, + ), # 1 + ( + 2, + TType.STRUCT, + "o2", + [NoSuchObjectException, None], + None, + ), # 2 +) + + +class get_partitions_ps_args: + """ + Attributes: + - db_name + - tbl_name + - part_vals + - max_parts + + """ + + def __init__( + self, + db_name=None, + tbl_name=None, + part_vals=None, + max_parts=-1, + ): + self.db_name = db_name + self.tbl_name = tbl_name + self.part_vals = part_vals + self.max_parts = max_parts + + def read(self, iprot): + if ( + iprot._fast_decode is not None + and isinstance(iprot.trans, TTransport.CReadableTransport) + and self.thrift_spec is not None + ): + iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec]) + return + iprot.readStructBegin() + while True: + (fname, ftype, fid) = iprot.readFieldBegin() + if ftype == TType.STOP: + break + if fid == 1: + if ftype == TType.STRING: + self.db_name = ( + iprot.readString().decode("utf-8", errors="replace") if sys.version_info[0] == 2 else iprot.readString() + ) + else: + iprot.skip(ftype) + elif fid == 2: + if ftype == TType.STRING: + self.tbl_name = ( + iprot.readString().decode("utf-8", errors="replace") if sys.version_info[0] == 2 else iprot.readString() + ) + else: + iprot.skip(ftype) + elif fid == 3: + if ftype == TType.LIST: + self.part_vals = [] + (_etype1559, _size1556) = iprot.readListBegin() + for _i1560 in range(_size1556): + _elem1561 = ( + iprot.readString().decode("utf-8", errors="replace") + if sys.version_info[0] == 2 + else iprot.readString() + ) + self.part_vals.append(_elem1561) + iprot.readListEnd() + else: + iprot.skip(ftype) + elif fid == 4: + if ftype == TType.I16: + self.max_parts = iprot.readI16() + else: + iprot.skip(ftype) + else: + iprot.skip(ftype) + iprot.readFieldEnd() + iprot.readStructEnd() + + def write(self, oprot): + if oprot._fast_encode is not None and self.thrift_spec is not None: + oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec])) + return + oprot.writeStructBegin("get_partitions_ps_args") + if self.db_name is not None: + oprot.writeFieldBegin("db_name", TType.STRING, 1) + oprot.writeString(self.db_name.encode("utf-8") if sys.version_info[0] == 2 else self.db_name) + oprot.writeFieldEnd() + if self.tbl_name is not None: + oprot.writeFieldBegin("tbl_name", TType.STRING, 2) + oprot.writeString(self.tbl_name.encode("utf-8") if sys.version_info[0] == 2 else self.tbl_name) + oprot.writeFieldEnd() + if self.part_vals is not None: + oprot.writeFieldBegin("part_vals", TType.LIST, 3) + oprot.writeListBegin(TType.STRING, len(self.part_vals)) + for iter1562 in self.part_vals: + oprot.writeString(iter1562.encode("utf-8") if sys.version_info[0] == 2 else iter1562) + oprot.writeListEnd() + oprot.writeFieldEnd() + if self.max_parts is not None: + oprot.writeFieldBegin("max_parts", TType.I16, 4) + oprot.writeI16(self.max_parts) + oprot.writeFieldEnd() + oprot.writeFieldStop() + oprot.writeStructEnd() + + def validate(self): + return + + def __repr__(self): + L = ["{}={!r}".format(key, value) for key, value in self.__dict__.items()] + return "{}({})".format(self.__class__.__name__, ", ".join(L)) + + def __eq__(self, other): + return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ + + def __ne__(self, other): + return not (self == other) + + +all_structs.append(get_partitions_ps_args) +get_partitions_ps_args.thrift_spec = ( + None, # 0 + ( + 1, + TType.STRING, + "db_name", + "UTF8", + None, + ), # 1 + ( + 2, + TType.STRING, + "tbl_name", + "UTF8", + None, + ), # 2 + ( + 3, + TType.LIST, + "part_vals", + (TType.STRING, "UTF8", False), + None, + ), # 3 + ( + 4, + TType.I16, + "max_parts", + None, + -1, + ), # 4 +) + + +class get_partitions_ps_result: + """ + Attributes: + - success + - o1 + - o2 + + """ + + def __init__( + self, + success=None, + o1=None, + o2=None, + ): + self.success = success + self.o1 = o1 + self.o2 = o2 + + def read(self, iprot): + if ( + iprot._fast_decode is not None + and isinstance(iprot.trans, TTransport.CReadableTransport) + and self.thrift_spec is not None + ): + iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec]) + return + iprot.readStructBegin() + while True: + (fname, ftype, fid) = iprot.readFieldBegin() + if ftype == TType.STOP: + break + if fid == 0: + if ftype == TType.LIST: + self.success = [] + (_etype1566, _size1563) = iprot.readListBegin() + for _i1567 in range(_size1563): + _elem1568 = Partition() + _elem1568.read(iprot) + self.success.append(_elem1568) + iprot.readListEnd() + else: + iprot.skip(ftype) + elif fid == 1: + if ftype == TType.STRUCT: + self.o1 = MetaException.read(iprot) + else: + iprot.skip(ftype) + elif fid == 2: + if ftype == TType.STRUCT: + self.o2 = NoSuchObjectException.read(iprot) + else: + iprot.skip(ftype) + else: + iprot.skip(ftype) + iprot.readFieldEnd() + iprot.readStructEnd() + + def write(self, oprot): + if oprot._fast_encode is not None and self.thrift_spec is not None: + oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec])) + return + oprot.writeStructBegin("get_partitions_ps_result") + if self.success is not None: + oprot.writeFieldBegin("success", TType.LIST, 0) + oprot.writeListBegin(TType.STRUCT, len(self.success)) + for iter1569 in self.success: + iter1569.write(oprot) + oprot.writeListEnd() + oprot.writeFieldEnd() + if self.o1 is not None: + oprot.writeFieldBegin("o1", TType.STRUCT, 1) + self.o1.write(oprot) + oprot.writeFieldEnd() + if self.o2 is not None: + oprot.writeFieldBegin("o2", TType.STRUCT, 2) + self.o2.write(oprot) + oprot.writeFieldEnd() + oprot.writeFieldStop() + oprot.writeStructEnd() + + def validate(self): + return + + def __repr__(self): + L = ["{}={!r}".format(key, value) for key, value in self.__dict__.items()] + return "{}({})".format(self.__class__.__name__, ", ".join(L)) + + def __eq__(self, other): + return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ + + def __ne__(self, other): + return not (self == other) + + +all_structs.append(get_partitions_ps_result) +get_partitions_ps_result.thrift_spec = ( + ( + 0, + TType.LIST, + "success", + (TType.STRUCT, [Partition, None], False), + None, + ), # 0 + ( + 1, + TType.STRUCT, + "o1", + [MetaException, None], + None, + ), # 1 + ( + 2, + TType.STRUCT, + "o2", + [NoSuchObjectException, None], + None, + ), # 2 +) + + +class get_partitions_ps_with_auth_args: + """ + Attributes: + - db_name + - tbl_name + - part_vals + - max_parts + - user_name + - group_names + + """ + + def __init__( + self, + db_name=None, + tbl_name=None, + part_vals=None, + max_parts=-1, + user_name=None, + group_names=None, + ): + self.db_name = db_name + self.tbl_name = tbl_name + self.part_vals = part_vals + self.max_parts = max_parts + self.user_name = user_name + self.group_names = group_names + + def read(self, iprot): + if ( + iprot._fast_decode is not None + and isinstance(iprot.trans, TTransport.CReadableTransport) + and self.thrift_spec is not None + ): + iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec]) + return + iprot.readStructBegin() + while True: + (fname, ftype, fid) = iprot.readFieldBegin() + if ftype == TType.STOP: + break + if fid == 1: + if ftype == TType.STRING: + self.db_name = ( + iprot.readString().decode("utf-8", errors="replace") if sys.version_info[0] == 2 else iprot.readString() + ) + else: + iprot.skip(ftype) + elif fid == 2: + if ftype == TType.STRING: + self.tbl_name = ( + iprot.readString().decode("utf-8", errors="replace") if sys.version_info[0] == 2 else iprot.readString() + ) + else: + iprot.skip(ftype) + elif fid == 3: + if ftype == TType.LIST: + self.part_vals = [] + (_etype1573, _size1570) = iprot.readListBegin() + for _i1574 in range(_size1570): + _elem1575 = ( + iprot.readString().decode("utf-8", errors="replace") + if sys.version_info[0] == 2 + else iprot.readString() + ) + self.part_vals.append(_elem1575) + iprot.readListEnd() + else: + iprot.skip(ftype) + elif fid == 4: + if ftype == TType.I16: + self.max_parts = iprot.readI16() + else: + iprot.skip(ftype) + elif fid == 5: + if ftype == TType.STRING: + self.user_name = ( + iprot.readString().decode("utf-8", errors="replace") if sys.version_info[0] == 2 else iprot.readString() + ) + else: + iprot.skip(ftype) + elif fid == 6: + if ftype == TType.LIST: + self.group_names = [] + (_etype1579, _size1576) = iprot.readListBegin() + for _i1580 in range(_size1576): + _elem1581 = ( + iprot.readString().decode("utf-8", errors="replace") + if sys.version_info[0] == 2 + else iprot.readString() + ) + self.group_names.append(_elem1581) + iprot.readListEnd() + else: + iprot.skip(ftype) + else: + iprot.skip(ftype) + iprot.readFieldEnd() + iprot.readStructEnd() + + def write(self, oprot): + if oprot._fast_encode is not None and self.thrift_spec is not None: + oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec])) + return + oprot.writeStructBegin("get_partitions_ps_with_auth_args") + if self.db_name is not None: + oprot.writeFieldBegin("db_name", TType.STRING, 1) + oprot.writeString(self.db_name.encode("utf-8") if sys.version_info[0] == 2 else self.db_name) + oprot.writeFieldEnd() + if self.tbl_name is not None: + oprot.writeFieldBegin("tbl_name", TType.STRING, 2) + oprot.writeString(self.tbl_name.encode("utf-8") if sys.version_info[0] == 2 else self.tbl_name) + oprot.writeFieldEnd() + if self.part_vals is not None: + oprot.writeFieldBegin("part_vals", TType.LIST, 3) + oprot.writeListBegin(TType.STRING, len(self.part_vals)) + for iter1582 in self.part_vals: + oprot.writeString(iter1582.encode("utf-8") if sys.version_info[0] == 2 else iter1582) + oprot.writeListEnd() + oprot.writeFieldEnd() + if self.max_parts is not None: + oprot.writeFieldBegin("max_parts", TType.I16, 4) + oprot.writeI16(self.max_parts) + oprot.writeFieldEnd() + if self.user_name is not None: + oprot.writeFieldBegin("user_name", TType.STRING, 5) + oprot.writeString(self.user_name.encode("utf-8") if sys.version_info[0] == 2 else self.user_name) + oprot.writeFieldEnd() + if self.group_names is not None: + oprot.writeFieldBegin("group_names", TType.LIST, 6) + oprot.writeListBegin(TType.STRING, len(self.group_names)) + for iter1583 in self.group_names: + oprot.writeString(iter1583.encode("utf-8") if sys.version_info[0] == 2 else iter1583) + oprot.writeListEnd() + oprot.writeFieldEnd() + oprot.writeFieldStop() + oprot.writeStructEnd() + + def validate(self): + return + + def __repr__(self): + L = ["{}={!r}".format(key, value) for key, value in self.__dict__.items()] + return "{}({})".format(self.__class__.__name__, ", ".join(L)) + + def __eq__(self, other): + return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ + + def __ne__(self, other): + return not (self == other) + + +all_structs.append(get_partitions_ps_with_auth_args) +get_partitions_ps_with_auth_args.thrift_spec = ( + None, # 0 + ( + 1, + TType.STRING, + "db_name", + "UTF8", + None, + ), # 1 + ( + 2, + TType.STRING, + "tbl_name", + "UTF8", + None, + ), # 2 + ( + 3, + TType.LIST, + "part_vals", + (TType.STRING, "UTF8", False), + None, + ), # 3 + ( + 4, + TType.I16, + "max_parts", + None, + -1, + ), # 4 + ( + 5, + TType.STRING, + "user_name", + "UTF8", + None, + ), # 5 + ( + 6, + TType.LIST, + "group_names", + (TType.STRING, "UTF8", False), + None, + ), # 6 +) + + +class get_partitions_ps_with_auth_result: + """ + Attributes: + - success + - o1 + - o2 + + """ + + def __init__( + self, + success=None, + o1=None, + o2=None, + ): + self.success = success + self.o1 = o1 + self.o2 = o2 + + def read(self, iprot): + if ( + iprot._fast_decode is not None + and isinstance(iprot.trans, TTransport.CReadableTransport) + and self.thrift_spec is not None + ): + iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec]) + return + iprot.readStructBegin() + while True: + (fname, ftype, fid) = iprot.readFieldBegin() + if ftype == TType.STOP: + break + if fid == 0: + if ftype == TType.LIST: + self.success = [] + (_etype1587, _size1584) = iprot.readListBegin() + for _i1588 in range(_size1584): + _elem1589 = Partition() + _elem1589.read(iprot) + self.success.append(_elem1589) + iprot.readListEnd() + else: + iprot.skip(ftype) + elif fid == 1: + if ftype == TType.STRUCT: + self.o1 = NoSuchObjectException.read(iprot) + else: + iprot.skip(ftype) + elif fid == 2: + if ftype == TType.STRUCT: + self.o2 = MetaException.read(iprot) + else: + iprot.skip(ftype) + else: + iprot.skip(ftype) + iprot.readFieldEnd() + iprot.readStructEnd() + + def write(self, oprot): + if oprot._fast_encode is not None and self.thrift_spec is not None: + oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec])) + return + oprot.writeStructBegin("get_partitions_ps_with_auth_result") + if self.success is not None: + oprot.writeFieldBegin("success", TType.LIST, 0) + oprot.writeListBegin(TType.STRUCT, len(self.success)) + for iter1590 in self.success: + iter1590.write(oprot) + oprot.writeListEnd() + oprot.writeFieldEnd() + if self.o1 is not None: + oprot.writeFieldBegin("o1", TType.STRUCT, 1) + self.o1.write(oprot) + oprot.writeFieldEnd() + if self.o2 is not None: + oprot.writeFieldBegin("o2", TType.STRUCT, 2) + self.o2.write(oprot) + oprot.writeFieldEnd() + oprot.writeFieldStop() + oprot.writeStructEnd() + + def validate(self): + return + + def __repr__(self): + L = ["{}={!r}".format(key, value) for key, value in self.__dict__.items()] + return "{}({})".format(self.__class__.__name__, ", ".join(L)) + + def __eq__(self, other): + return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ + + def __ne__(self, other): + return not (self == other) + + +all_structs.append(get_partitions_ps_with_auth_result) +get_partitions_ps_with_auth_result.thrift_spec = ( + ( + 0, + TType.LIST, + "success", + (TType.STRUCT, [Partition, None], False), + None, + ), # 0 + ( + 1, + TType.STRUCT, + "o1", + [NoSuchObjectException, None], + None, + ), # 1 + ( + 2, + TType.STRUCT, + "o2", + [MetaException, None], + None, + ), # 2 +) + + +class get_partitions_ps_with_auth_req_args: + """ + Attributes: + - req + + """ + + def __init__( + self, + req=None, + ): + self.req = req + + def read(self, iprot): + if ( + iprot._fast_decode is not None + and isinstance(iprot.trans, TTransport.CReadableTransport) + and self.thrift_spec is not None + ): + iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec]) + return + iprot.readStructBegin() + while True: + (fname, ftype, fid) = iprot.readFieldBegin() + if ftype == TType.STOP: + break + if fid == 1: + if ftype == TType.STRUCT: + self.req = GetPartitionsPsWithAuthRequest() + self.req.read(iprot) + else: + iprot.skip(ftype) + else: + iprot.skip(ftype) + iprot.readFieldEnd() + iprot.readStructEnd() + + def write(self, oprot): + if oprot._fast_encode is not None and self.thrift_spec is not None: + oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec])) + return + oprot.writeStructBegin("get_partitions_ps_with_auth_req_args") + if self.req is not None: + oprot.writeFieldBegin("req", TType.STRUCT, 1) + self.req.write(oprot) + oprot.writeFieldEnd() + oprot.writeFieldStop() + oprot.writeStructEnd() + + def validate(self): + return + + def __repr__(self): + L = ["{}={!r}".format(key, value) for key, value in self.__dict__.items()] + return "{}({})".format(self.__class__.__name__, ", ".join(L)) + + def __eq__(self, other): + return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ + + def __ne__(self, other): + return not (self == other) + + +all_structs.append(get_partitions_ps_with_auth_req_args) +get_partitions_ps_with_auth_req_args.thrift_spec = ( + None, # 0 + ( + 1, + TType.STRUCT, + "req", + [GetPartitionsPsWithAuthRequest, None], + None, + ), # 1 +) + + +class get_partitions_ps_with_auth_req_result: + """ + Attributes: + - success + - o1 + - o2 + + """ + + def __init__( + self, + success=None, + o1=None, + o2=None, + ): + self.success = success + self.o1 = o1 + self.o2 = o2 + + def read(self, iprot): + if ( + iprot._fast_decode is not None + and isinstance(iprot.trans, TTransport.CReadableTransport) + and self.thrift_spec is not None + ): + iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec]) + return + iprot.readStructBegin() + while True: + (fname, ftype, fid) = iprot.readFieldBegin() + if ftype == TType.STOP: + break + if fid == 0: + if ftype == TType.STRUCT: + self.success = GetPartitionsPsWithAuthResponse() + self.success.read(iprot) + else: + iprot.skip(ftype) + elif fid == 1: + if ftype == TType.STRUCT: + self.o1 = MetaException.read(iprot) + else: + iprot.skip(ftype) + elif fid == 2: + if ftype == TType.STRUCT: + self.o2 = NoSuchObjectException.read(iprot) + else: + iprot.skip(ftype) + else: + iprot.skip(ftype) + iprot.readFieldEnd() + iprot.readStructEnd() + + def write(self, oprot): + if oprot._fast_encode is not None and self.thrift_spec is not None: + oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec])) + return + oprot.writeStructBegin("get_partitions_ps_with_auth_req_result") + if self.success is not None: + oprot.writeFieldBegin("success", TType.STRUCT, 0) + self.success.write(oprot) + oprot.writeFieldEnd() + if self.o1 is not None: + oprot.writeFieldBegin("o1", TType.STRUCT, 1) + self.o1.write(oprot) + oprot.writeFieldEnd() + if self.o2 is not None: + oprot.writeFieldBegin("o2", TType.STRUCT, 2) + self.o2.write(oprot) + oprot.writeFieldEnd() + oprot.writeFieldStop() + oprot.writeStructEnd() + + def validate(self): + return + + def __repr__(self): + L = ["{}={!r}".format(key, value) for key, value in self.__dict__.items()] + return "{}({})".format(self.__class__.__name__, ", ".join(L)) + + def __eq__(self, other): + return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ + + def __ne__(self, other): + return not (self == other) + + +all_structs.append(get_partitions_ps_with_auth_req_result) +get_partitions_ps_with_auth_req_result.thrift_spec = ( + ( + 0, + TType.STRUCT, + "success", + [GetPartitionsPsWithAuthResponse, None], + None, + ), # 0 + ( + 1, + TType.STRUCT, + "o1", + [MetaException, None], + None, + ), # 1 + ( + 2, + TType.STRUCT, + "o2", + [NoSuchObjectException, None], + None, + ), # 2 +) + + +class get_partition_names_ps_args: + """ + Attributes: + - db_name + - tbl_name + - part_vals + - max_parts + + """ + + def __init__( + self, + db_name=None, + tbl_name=None, + part_vals=None, + max_parts=-1, + ): + self.db_name = db_name + self.tbl_name = tbl_name + self.part_vals = part_vals + self.max_parts = max_parts + + def read(self, iprot): + if ( + iprot._fast_decode is not None + and isinstance(iprot.trans, TTransport.CReadableTransport) + and self.thrift_spec is not None + ): + iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec]) + return + iprot.readStructBegin() + while True: + (fname, ftype, fid) = iprot.readFieldBegin() + if ftype == TType.STOP: + break + if fid == 1: + if ftype == TType.STRING: + self.db_name = ( + iprot.readString().decode("utf-8", errors="replace") if sys.version_info[0] == 2 else iprot.readString() + ) + else: + iprot.skip(ftype) + elif fid == 2: + if ftype == TType.STRING: + self.tbl_name = ( + iprot.readString().decode("utf-8", errors="replace") if sys.version_info[0] == 2 else iprot.readString() + ) + else: + iprot.skip(ftype) + elif fid == 3: + if ftype == TType.LIST: + self.part_vals = [] + (_etype1594, _size1591) = iprot.readListBegin() + for _i1595 in range(_size1591): + _elem1596 = ( + iprot.readString().decode("utf-8", errors="replace") + if sys.version_info[0] == 2 + else iprot.readString() + ) + self.part_vals.append(_elem1596) + iprot.readListEnd() + else: + iprot.skip(ftype) + elif fid == 4: + if ftype == TType.I16: + self.max_parts = iprot.readI16() + else: + iprot.skip(ftype) + else: + iprot.skip(ftype) + iprot.readFieldEnd() + iprot.readStructEnd() + + def write(self, oprot): + if oprot._fast_encode is not None and self.thrift_spec is not None: + oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec])) + return + oprot.writeStructBegin("get_partition_names_ps_args") + if self.db_name is not None: + oprot.writeFieldBegin("db_name", TType.STRING, 1) + oprot.writeString(self.db_name.encode("utf-8") if sys.version_info[0] == 2 else self.db_name) + oprot.writeFieldEnd() + if self.tbl_name is not None: + oprot.writeFieldBegin("tbl_name", TType.STRING, 2) + oprot.writeString(self.tbl_name.encode("utf-8") if sys.version_info[0] == 2 else self.tbl_name) + oprot.writeFieldEnd() + if self.part_vals is not None: + oprot.writeFieldBegin("part_vals", TType.LIST, 3) + oprot.writeListBegin(TType.STRING, len(self.part_vals)) + for iter1597 in self.part_vals: + oprot.writeString(iter1597.encode("utf-8") if sys.version_info[0] == 2 else iter1597) + oprot.writeListEnd() + oprot.writeFieldEnd() + if self.max_parts is not None: + oprot.writeFieldBegin("max_parts", TType.I16, 4) + oprot.writeI16(self.max_parts) + oprot.writeFieldEnd() + oprot.writeFieldStop() + oprot.writeStructEnd() + + def validate(self): + return + + def __repr__(self): + L = ["{}={!r}".format(key, value) for key, value in self.__dict__.items()] + return "{}({})".format(self.__class__.__name__, ", ".join(L)) + + def __eq__(self, other): + return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ + + def __ne__(self, other): + return not (self == other) + + +all_structs.append(get_partition_names_ps_args) +get_partition_names_ps_args.thrift_spec = ( + None, # 0 + ( + 1, + TType.STRING, + "db_name", + "UTF8", + None, + ), # 1 + ( + 2, + TType.STRING, + "tbl_name", + "UTF8", + None, + ), # 2 + ( + 3, + TType.LIST, + "part_vals", + (TType.STRING, "UTF8", False), + None, + ), # 3 + ( + 4, + TType.I16, + "max_parts", + None, + -1, + ), # 4 +) + + +class get_partition_names_ps_result: + """ + Attributes: + - success + - o1 + - o2 + + """ + + def __init__( + self, + success=None, + o1=None, + o2=None, + ): + self.success = success + self.o1 = o1 + self.o2 = o2 + + def read(self, iprot): + if ( + iprot._fast_decode is not None + and isinstance(iprot.trans, TTransport.CReadableTransport) + and self.thrift_spec is not None + ): + iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec]) + return + iprot.readStructBegin() + while True: + (fname, ftype, fid) = iprot.readFieldBegin() + if ftype == TType.STOP: + break + if fid == 0: + if ftype == TType.LIST: + self.success = [] + (_etype1601, _size1598) = iprot.readListBegin() + for _i1602 in range(_size1598): + _elem1603 = ( + iprot.readString().decode("utf-8", errors="replace") + if sys.version_info[0] == 2 + else iprot.readString() + ) + self.success.append(_elem1603) + iprot.readListEnd() + else: + iprot.skip(ftype) + elif fid == 1: + if ftype == TType.STRUCT: + self.o1 = MetaException.read(iprot) + else: + iprot.skip(ftype) + elif fid == 2: + if ftype == TType.STRUCT: + self.o2 = NoSuchObjectException.read(iprot) + else: + iprot.skip(ftype) + else: + iprot.skip(ftype) + iprot.readFieldEnd() + iprot.readStructEnd() + + def write(self, oprot): + if oprot._fast_encode is not None and self.thrift_spec is not None: + oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec])) + return + oprot.writeStructBegin("get_partition_names_ps_result") + if self.success is not None: + oprot.writeFieldBegin("success", TType.LIST, 0) + oprot.writeListBegin(TType.STRING, len(self.success)) + for iter1604 in self.success: + oprot.writeString(iter1604.encode("utf-8") if sys.version_info[0] == 2 else iter1604) + oprot.writeListEnd() + oprot.writeFieldEnd() + if self.o1 is not None: + oprot.writeFieldBegin("o1", TType.STRUCT, 1) + self.o1.write(oprot) + oprot.writeFieldEnd() + if self.o2 is not None: + oprot.writeFieldBegin("o2", TType.STRUCT, 2) + self.o2.write(oprot) + oprot.writeFieldEnd() + oprot.writeFieldStop() + oprot.writeStructEnd() + + def validate(self): + return + + def __repr__(self): + L = ["{}={!r}".format(key, value) for key, value in self.__dict__.items()] + return "{}({})".format(self.__class__.__name__, ", ".join(L)) + + def __eq__(self, other): + return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ + + def __ne__(self, other): + return not (self == other) + + +all_structs.append(get_partition_names_ps_result) +get_partition_names_ps_result.thrift_spec = ( + ( + 0, + TType.LIST, + "success", + (TType.STRING, "UTF8", False), + None, + ), # 0 + ( + 1, + TType.STRUCT, + "o1", + [MetaException, None], + None, + ), # 1 + ( + 2, + TType.STRUCT, + "o2", + [NoSuchObjectException, None], + None, + ), # 2 +) + + +class get_partition_names_ps_req_args: + """ + Attributes: + - req + + """ + + def __init__( + self, + req=None, + ): + self.req = req + + def read(self, iprot): + if ( + iprot._fast_decode is not None + and isinstance(iprot.trans, TTransport.CReadableTransport) + and self.thrift_spec is not None + ): + iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec]) + return + iprot.readStructBegin() + while True: + (fname, ftype, fid) = iprot.readFieldBegin() + if ftype == TType.STOP: + break + if fid == 1: + if ftype == TType.STRUCT: + self.req = GetPartitionNamesPsRequest() + self.req.read(iprot) + else: + iprot.skip(ftype) + else: + iprot.skip(ftype) + iprot.readFieldEnd() + iprot.readStructEnd() + + def write(self, oprot): + if oprot._fast_encode is not None and self.thrift_spec is not None: + oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec])) + return + oprot.writeStructBegin("get_partition_names_ps_req_args") + if self.req is not None: + oprot.writeFieldBegin("req", TType.STRUCT, 1) + self.req.write(oprot) + oprot.writeFieldEnd() + oprot.writeFieldStop() + oprot.writeStructEnd() + + def validate(self): + return + + def __repr__(self): + L = ["{}={!r}".format(key, value) for key, value in self.__dict__.items()] + return "{}({})".format(self.__class__.__name__, ", ".join(L)) + + def __eq__(self, other): + return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ + + def __ne__(self, other): + return not (self == other) + + +all_structs.append(get_partition_names_ps_req_args) +get_partition_names_ps_req_args.thrift_spec = ( + None, # 0 + ( + 1, + TType.STRUCT, + "req", + [GetPartitionNamesPsRequest, None], + None, + ), # 1 +) + + +class get_partition_names_ps_req_result: + """ + Attributes: + - success + - o1 + - o2 + + """ + + def __init__( + self, + success=None, + o1=None, + o2=None, + ): + self.success = success + self.o1 = o1 + self.o2 = o2 + + def read(self, iprot): + if ( + iprot._fast_decode is not None + and isinstance(iprot.trans, TTransport.CReadableTransport) + and self.thrift_spec is not None + ): + iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec]) + return + iprot.readStructBegin() + while True: + (fname, ftype, fid) = iprot.readFieldBegin() + if ftype == TType.STOP: + break + if fid == 0: + if ftype == TType.STRUCT: + self.success = GetPartitionNamesPsResponse() + self.success.read(iprot) + else: + iprot.skip(ftype) + elif fid == 1: + if ftype == TType.STRUCT: + self.o1 = MetaException.read(iprot) + else: + iprot.skip(ftype) + elif fid == 2: + if ftype == TType.STRUCT: + self.o2 = NoSuchObjectException.read(iprot) + else: + iprot.skip(ftype) + else: + iprot.skip(ftype) + iprot.readFieldEnd() + iprot.readStructEnd() + + def write(self, oprot): + if oprot._fast_encode is not None and self.thrift_spec is not None: + oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec])) + return + oprot.writeStructBegin("get_partition_names_ps_req_result") + if self.success is not None: + oprot.writeFieldBegin("success", TType.STRUCT, 0) + self.success.write(oprot) + oprot.writeFieldEnd() + if self.o1 is not None: + oprot.writeFieldBegin("o1", TType.STRUCT, 1) + self.o1.write(oprot) + oprot.writeFieldEnd() + if self.o2 is not None: + oprot.writeFieldBegin("o2", TType.STRUCT, 2) + self.o2.write(oprot) + oprot.writeFieldEnd() + oprot.writeFieldStop() + oprot.writeStructEnd() + + def validate(self): + return + + def __repr__(self): + L = ["{}={!r}".format(key, value) for key, value in self.__dict__.items()] + return "{}({})".format(self.__class__.__name__, ", ".join(L)) + + def __eq__(self, other): + return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ + + def __ne__(self, other): + return not (self == other) + + +all_structs.append(get_partition_names_ps_req_result) +get_partition_names_ps_req_result.thrift_spec = ( + ( + 0, + TType.STRUCT, + "success", + [GetPartitionNamesPsResponse, None], + None, + ), # 0 + ( + 1, + TType.STRUCT, + "o1", + [MetaException, None], + None, + ), # 1 + ( + 2, + TType.STRUCT, + "o2", + [NoSuchObjectException, None], + None, + ), # 2 +) + + +class get_partition_names_req_args: + """ + Attributes: + - req + + """ + + def __init__( + self, + req=None, + ): + self.req = req + + def read(self, iprot): + if ( + iprot._fast_decode is not None + and isinstance(iprot.trans, TTransport.CReadableTransport) + and self.thrift_spec is not None + ): + iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec]) + return + iprot.readStructBegin() + while True: + (fname, ftype, fid) = iprot.readFieldBegin() + if ftype == TType.STOP: + break + if fid == 1: + if ftype == TType.STRUCT: + self.req = PartitionsByExprRequest() + self.req.read(iprot) + else: + iprot.skip(ftype) + else: + iprot.skip(ftype) + iprot.readFieldEnd() + iprot.readStructEnd() + + def write(self, oprot): + if oprot._fast_encode is not None and self.thrift_spec is not None: + oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec])) + return + oprot.writeStructBegin("get_partition_names_req_args") + if self.req is not None: + oprot.writeFieldBegin("req", TType.STRUCT, 1) + self.req.write(oprot) + oprot.writeFieldEnd() + oprot.writeFieldStop() + oprot.writeStructEnd() + + def validate(self): + return + + def __repr__(self): + L = ["{}={!r}".format(key, value) for key, value in self.__dict__.items()] + return "{}({})".format(self.__class__.__name__, ", ".join(L)) + + def __eq__(self, other): + return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ + + def __ne__(self, other): + return not (self == other) + + +all_structs.append(get_partition_names_req_args) +get_partition_names_req_args.thrift_spec = ( + None, # 0 + ( + 1, + TType.STRUCT, + "req", + [PartitionsByExprRequest, None], + None, + ), # 1 +) + + +class get_partition_names_req_result: + """ + Attributes: + - success + - o1 + - o2 + + """ + + def __init__( + self, + success=None, + o1=None, + o2=None, + ): + self.success = success + self.o1 = o1 + self.o2 = o2 + + def read(self, iprot): + if ( + iprot._fast_decode is not None + and isinstance(iprot.trans, TTransport.CReadableTransport) + and self.thrift_spec is not None + ): + iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec]) + return + iprot.readStructBegin() + while True: + (fname, ftype, fid) = iprot.readFieldBegin() + if ftype == TType.STOP: + break + if fid == 0: + if ftype == TType.LIST: + self.success = [] + (_etype1608, _size1605) = iprot.readListBegin() + for _i1609 in range(_size1605): + _elem1610 = ( + iprot.readString().decode("utf-8", errors="replace") + if sys.version_info[0] == 2 + else iprot.readString() + ) + self.success.append(_elem1610) + iprot.readListEnd() + else: + iprot.skip(ftype) + elif fid == 1: + if ftype == TType.STRUCT: + self.o1 = MetaException.read(iprot) + else: + iprot.skip(ftype) + elif fid == 2: + if ftype == TType.STRUCT: + self.o2 = NoSuchObjectException.read(iprot) + else: + iprot.skip(ftype) + else: + iprot.skip(ftype) + iprot.readFieldEnd() + iprot.readStructEnd() + + def write(self, oprot): + if oprot._fast_encode is not None and self.thrift_spec is not None: + oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec])) + return + oprot.writeStructBegin("get_partition_names_req_result") + if self.success is not None: + oprot.writeFieldBegin("success", TType.LIST, 0) + oprot.writeListBegin(TType.STRING, len(self.success)) + for iter1611 in self.success: + oprot.writeString(iter1611.encode("utf-8") if sys.version_info[0] == 2 else iter1611) + oprot.writeListEnd() + oprot.writeFieldEnd() + if self.o1 is not None: + oprot.writeFieldBegin("o1", TType.STRUCT, 1) + self.o1.write(oprot) + oprot.writeFieldEnd() + if self.o2 is not None: + oprot.writeFieldBegin("o2", TType.STRUCT, 2) + self.o2.write(oprot) + oprot.writeFieldEnd() + oprot.writeFieldStop() + oprot.writeStructEnd() + + def validate(self): + return + + def __repr__(self): + L = ["{}={!r}".format(key, value) for key, value in self.__dict__.items()] + return "{}({})".format(self.__class__.__name__, ", ".join(L)) + + def __eq__(self, other): + return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ + + def __ne__(self, other): + return not (self == other) + + +all_structs.append(get_partition_names_req_result) +get_partition_names_req_result.thrift_spec = ( + ( + 0, + TType.LIST, + "success", + (TType.STRING, "UTF8", False), + None, + ), # 0 + ( + 1, + TType.STRUCT, + "o1", + [MetaException, None], + None, + ), # 1 + ( + 2, + TType.STRUCT, + "o2", + [NoSuchObjectException, None], + None, + ), # 2 +) + + +class get_partitions_by_filter_args: + """ + Attributes: + - db_name + - tbl_name + - filter + - max_parts + + """ + + def __init__( + self, + db_name=None, + tbl_name=None, + filter=None, + max_parts=-1, + ): + self.db_name = db_name + self.tbl_name = tbl_name + self.filter = filter + self.max_parts = max_parts + + def read(self, iprot): + if ( + iprot._fast_decode is not None + and isinstance(iprot.trans, TTransport.CReadableTransport) + and self.thrift_spec is not None + ): + iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec]) + return + iprot.readStructBegin() + while True: + (fname, ftype, fid) = iprot.readFieldBegin() + if ftype == TType.STOP: + break + if fid == 1: + if ftype == TType.STRING: + self.db_name = ( + iprot.readString().decode("utf-8", errors="replace") if sys.version_info[0] == 2 else iprot.readString() + ) + else: + iprot.skip(ftype) + elif fid == 2: + if ftype == TType.STRING: + self.tbl_name = ( + iprot.readString().decode("utf-8", errors="replace") if sys.version_info[0] == 2 else iprot.readString() + ) + else: + iprot.skip(ftype) + elif fid == 3: + if ftype == TType.STRING: + self.filter = ( + iprot.readString().decode("utf-8", errors="replace") if sys.version_info[0] == 2 else iprot.readString() + ) + else: + iprot.skip(ftype) + elif fid == 4: + if ftype == TType.I16: + self.max_parts = iprot.readI16() + else: + iprot.skip(ftype) + else: + iprot.skip(ftype) + iprot.readFieldEnd() + iprot.readStructEnd() + + def write(self, oprot): + if oprot._fast_encode is not None and self.thrift_spec is not None: + oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec])) + return + oprot.writeStructBegin("get_partitions_by_filter_args") + if self.db_name is not None: + oprot.writeFieldBegin("db_name", TType.STRING, 1) + oprot.writeString(self.db_name.encode("utf-8") if sys.version_info[0] == 2 else self.db_name) + oprot.writeFieldEnd() + if self.tbl_name is not None: + oprot.writeFieldBegin("tbl_name", TType.STRING, 2) + oprot.writeString(self.tbl_name.encode("utf-8") if sys.version_info[0] == 2 else self.tbl_name) + oprot.writeFieldEnd() + if self.filter is not None: + oprot.writeFieldBegin("filter", TType.STRING, 3) + oprot.writeString(self.filter.encode("utf-8") if sys.version_info[0] == 2 else self.filter) + oprot.writeFieldEnd() + if self.max_parts is not None: + oprot.writeFieldBegin("max_parts", TType.I16, 4) + oprot.writeI16(self.max_parts) + oprot.writeFieldEnd() + oprot.writeFieldStop() + oprot.writeStructEnd() + + def validate(self): + return + + def __repr__(self): + L = ["{}={!r}".format(key, value) for key, value in self.__dict__.items()] + return "{}({})".format(self.__class__.__name__, ", ".join(L)) + + def __eq__(self, other): + return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ + + def __ne__(self, other): + return not (self == other) + + +all_structs.append(get_partitions_by_filter_args) +get_partitions_by_filter_args.thrift_spec = ( + None, # 0 + ( + 1, + TType.STRING, + "db_name", + "UTF8", + None, + ), # 1 + ( + 2, + TType.STRING, + "tbl_name", + "UTF8", + None, + ), # 2 + ( + 3, + TType.STRING, + "filter", + "UTF8", + None, + ), # 3 + ( + 4, + TType.I16, + "max_parts", + None, + -1, + ), # 4 +) + + +class get_partitions_by_filter_result: + """ + Attributes: + - success + - o1 + - o2 + + """ + + def __init__( + self, + success=None, + o1=None, + o2=None, + ): + self.success = success + self.o1 = o1 + self.o2 = o2 + + def read(self, iprot): + if ( + iprot._fast_decode is not None + and isinstance(iprot.trans, TTransport.CReadableTransport) + and self.thrift_spec is not None + ): + iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec]) + return + iprot.readStructBegin() + while True: + (fname, ftype, fid) = iprot.readFieldBegin() + if ftype == TType.STOP: + break + if fid == 0: + if ftype == TType.LIST: + self.success = [] + (_etype1615, _size1612) = iprot.readListBegin() + for _i1616 in range(_size1612): + _elem1617 = Partition() + _elem1617.read(iprot) + self.success.append(_elem1617) + iprot.readListEnd() + else: + iprot.skip(ftype) + elif fid == 1: + if ftype == TType.STRUCT: + self.o1 = MetaException.read(iprot) + else: + iprot.skip(ftype) + elif fid == 2: + if ftype == TType.STRUCT: + self.o2 = NoSuchObjectException.read(iprot) + else: + iprot.skip(ftype) + else: + iprot.skip(ftype) + iprot.readFieldEnd() + iprot.readStructEnd() + + def write(self, oprot): + if oprot._fast_encode is not None and self.thrift_spec is not None: + oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec])) + return + oprot.writeStructBegin("get_partitions_by_filter_result") + if self.success is not None: + oprot.writeFieldBegin("success", TType.LIST, 0) + oprot.writeListBegin(TType.STRUCT, len(self.success)) + for iter1618 in self.success: + iter1618.write(oprot) + oprot.writeListEnd() + oprot.writeFieldEnd() + if self.o1 is not None: + oprot.writeFieldBegin("o1", TType.STRUCT, 1) + self.o1.write(oprot) + oprot.writeFieldEnd() + if self.o2 is not None: + oprot.writeFieldBegin("o2", TType.STRUCT, 2) + self.o2.write(oprot) + oprot.writeFieldEnd() + oprot.writeFieldStop() + oprot.writeStructEnd() + + def validate(self): + return + + def __repr__(self): + L = ["{}={!r}".format(key, value) for key, value in self.__dict__.items()] + return "{}({})".format(self.__class__.__name__, ", ".join(L)) + + def __eq__(self, other): + return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ + + def __ne__(self, other): + return not (self == other) + + +all_structs.append(get_partitions_by_filter_result) +get_partitions_by_filter_result.thrift_spec = ( + ( + 0, + TType.LIST, + "success", + (TType.STRUCT, [Partition, None], False), + None, + ), # 0 + ( + 1, + TType.STRUCT, + "o1", + [MetaException, None], + None, + ), # 1 + ( + 2, + TType.STRUCT, + "o2", + [NoSuchObjectException, None], + None, + ), # 2 +) + + +class get_part_specs_by_filter_args: + """ + Attributes: + - db_name + - tbl_name + - filter + - max_parts + + """ + + def __init__( + self, + db_name=None, + tbl_name=None, + filter=None, + max_parts=-1, + ): + self.db_name = db_name + self.tbl_name = tbl_name + self.filter = filter + self.max_parts = max_parts + + def read(self, iprot): + if ( + iprot._fast_decode is not None + and isinstance(iprot.trans, TTransport.CReadableTransport) + and self.thrift_spec is not None + ): + iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec]) + return + iprot.readStructBegin() + while True: + (fname, ftype, fid) = iprot.readFieldBegin() + if ftype == TType.STOP: + break + if fid == 1: + if ftype == TType.STRING: + self.db_name = ( + iprot.readString().decode("utf-8", errors="replace") if sys.version_info[0] == 2 else iprot.readString() + ) + else: + iprot.skip(ftype) + elif fid == 2: + if ftype == TType.STRING: + self.tbl_name = ( + iprot.readString().decode("utf-8", errors="replace") if sys.version_info[0] == 2 else iprot.readString() + ) + else: + iprot.skip(ftype) + elif fid == 3: + if ftype == TType.STRING: + self.filter = ( + iprot.readString().decode("utf-8", errors="replace") if sys.version_info[0] == 2 else iprot.readString() + ) + else: + iprot.skip(ftype) + elif fid == 4: + if ftype == TType.I32: + self.max_parts = iprot.readI32() + else: + iprot.skip(ftype) + else: + iprot.skip(ftype) + iprot.readFieldEnd() + iprot.readStructEnd() + + def write(self, oprot): + if oprot._fast_encode is not None and self.thrift_spec is not None: + oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec])) + return + oprot.writeStructBegin("get_part_specs_by_filter_args") + if self.db_name is not None: + oprot.writeFieldBegin("db_name", TType.STRING, 1) + oprot.writeString(self.db_name.encode("utf-8") if sys.version_info[0] == 2 else self.db_name) + oprot.writeFieldEnd() + if self.tbl_name is not None: + oprot.writeFieldBegin("tbl_name", TType.STRING, 2) + oprot.writeString(self.tbl_name.encode("utf-8") if sys.version_info[0] == 2 else self.tbl_name) + oprot.writeFieldEnd() + if self.filter is not None: + oprot.writeFieldBegin("filter", TType.STRING, 3) + oprot.writeString(self.filter.encode("utf-8") if sys.version_info[0] == 2 else self.filter) + oprot.writeFieldEnd() + if self.max_parts is not None: + oprot.writeFieldBegin("max_parts", TType.I32, 4) + oprot.writeI32(self.max_parts) + oprot.writeFieldEnd() + oprot.writeFieldStop() + oprot.writeStructEnd() + + def validate(self): + return + + def __repr__(self): + L = ["{}={!r}".format(key, value) for key, value in self.__dict__.items()] + return "{}({})".format(self.__class__.__name__, ", ".join(L)) + + def __eq__(self, other): + return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ + + def __ne__(self, other): + return not (self == other) + + +all_structs.append(get_part_specs_by_filter_args) +get_part_specs_by_filter_args.thrift_spec = ( + None, # 0 + ( + 1, + TType.STRING, + "db_name", + "UTF8", + None, + ), # 1 + ( + 2, + TType.STRING, + "tbl_name", + "UTF8", + None, + ), # 2 + ( + 3, + TType.STRING, + "filter", + "UTF8", + None, + ), # 3 + ( + 4, + TType.I32, + "max_parts", + None, + -1, + ), # 4 +) + + +class get_part_specs_by_filter_result: + """ + Attributes: + - success + - o1 + - o2 + + """ + + def __init__( + self, + success=None, + o1=None, + o2=None, + ): + self.success = success + self.o1 = o1 + self.o2 = o2 + + def read(self, iprot): + if ( + iprot._fast_decode is not None + and isinstance(iprot.trans, TTransport.CReadableTransport) + and self.thrift_spec is not None + ): + iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec]) + return + iprot.readStructBegin() + while True: + (fname, ftype, fid) = iprot.readFieldBegin() + if ftype == TType.STOP: + break + if fid == 0: + if ftype == TType.LIST: + self.success = [] + (_etype1622, _size1619) = iprot.readListBegin() + for _i1623 in range(_size1619): + _elem1624 = PartitionSpec() + _elem1624.read(iprot) + self.success.append(_elem1624) + iprot.readListEnd() + else: + iprot.skip(ftype) + elif fid == 1: + if ftype == TType.STRUCT: + self.o1 = MetaException.read(iprot) + else: + iprot.skip(ftype) + elif fid == 2: + if ftype == TType.STRUCT: + self.o2 = NoSuchObjectException.read(iprot) + else: + iprot.skip(ftype) + else: + iprot.skip(ftype) + iprot.readFieldEnd() + iprot.readStructEnd() + + def write(self, oprot): + if oprot._fast_encode is not None and self.thrift_spec is not None: + oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec])) + return + oprot.writeStructBegin("get_part_specs_by_filter_result") + if self.success is not None: + oprot.writeFieldBegin("success", TType.LIST, 0) + oprot.writeListBegin(TType.STRUCT, len(self.success)) + for iter1625 in self.success: + iter1625.write(oprot) + oprot.writeListEnd() + oprot.writeFieldEnd() + if self.o1 is not None: + oprot.writeFieldBegin("o1", TType.STRUCT, 1) + self.o1.write(oprot) + oprot.writeFieldEnd() + if self.o2 is not None: + oprot.writeFieldBegin("o2", TType.STRUCT, 2) + self.o2.write(oprot) + oprot.writeFieldEnd() + oprot.writeFieldStop() + oprot.writeStructEnd() + + def validate(self): + return + + def __repr__(self): + L = ["{}={!r}".format(key, value) for key, value in self.__dict__.items()] + return "{}({})".format(self.__class__.__name__, ", ".join(L)) + + def __eq__(self, other): + return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ + + def __ne__(self, other): + return not (self == other) + + +all_structs.append(get_part_specs_by_filter_result) +get_part_specs_by_filter_result.thrift_spec = ( + ( + 0, + TType.LIST, + "success", + (TType.STRUCT, [PartitionSpec, None], False), + None, + ), # 0 + ( + 1, + TType.STRUCT, + "o1", + [MetaException, None], + None, + ), # 1 + ( + 2, + TType.STRUCT, + "o2", + [NoSuchObjectException, None], + None, + ), # 2 +) + + +class get_partitions_by_expr_args: + """ + Attributes: + - req + + """ + + def __init__( + self, + req=None, + ): + self.req = req + + def read(self, iprot): + if ( + iprot._fast_decode is not None + and isinstance(iprot.trans, TTransport.CReadableTransport) + and self.thrift_spec is not None + ): + iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec]) + return + iprot.readStructBegin() + while True: + (fname, ftype, fid) = iprot.readFieldBegin() + if ftype == TType.STOP: + break + if fid == 1: + if ftype == TType.STRUCT: + self.req = PartitionsByExprRequest() + self.req.read(iprot) + else: + iprot.skip(ftype) + else: + iprot.skip(ftype) + iprot.readFieldEnd() + iprot.readStructEnd() + + def write(self, oprot): + if oprot._fast_encode is not None and self.thrift_spec is not None: + oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec])) + return + oprot.writeStructBegin("get_partitions_by_expr_args") + if self.req is not None: + oprot.writeFieldBegin("req", TType.STRUCT, 1) + self.req.write(oprot) + oprot.writeFieldEnd() + oprot.writeFieldStop() + oprot.writeStructEnd() + + def validate(self): + return + + def __repr__(self): + L = ["{}={!r}".format(key, value) for key, value in self.__dict__.items()] + return "{}({})".format(self.__class__.__name__, ", ".join(L)) + + def __eq__(self, other): + return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ + + def __ne__(self, other): + return not (self == other) + + +all_structs.append(get_partitions_by_expr_args) +get_partitions_by_expr_args.thrift_spec = ( + None, # 0 + ( + 1, + TType.STRUCT, + "req", + [PartitionsByExprRequest, None], + None, + ), # 1 +) + + +class get_partitions_by_expr_result: + """ + Attributes: + - success + - o1 + - o2 + + """ + + def __init__( + self, + success=None, + o1=None, + o2=None, + ): + self.success = success + self.o1 = o1 + self.o2 = o2 + + def read(self, iprot): + if ( + iprot._fast_decode is not None + and isinstance(iprot.trans, TTransport.CReadableTransport) + and self.thrift_spec is not None + ): + iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec]) + return + iprot.readStructBegin() + while True: + (fname, ftype, fid) = iprot.readFieldBegin() + if ftype == TType.STOP: + break + if fid == 0: + if ftype == TType.STRUCT: + self.success = PartitionsByExprResult() + self.success.read(iprot) + else: + iprot.skip(ftype) + elif fid == 1: + if ftype == TType.STRUCT: + self.o1 = MetaException.read(iprot) + else: + iprot.skip(ftype) + elif fid == 2: + if ftype == TType.STRUCT: + self.o2 = NoSuchObjectException.read(iprot) + else: + iprot.skip(ftype) + else: + iprot.skip(ftype) + iprot.readFieldEnd() + iprot.readStructEnd() + + def write(self, oprot): + if oprot._fast_encode is not None and self.thrift_spec is not None: + oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec])) + return + oprot.writeStructBegin("get_partitions_by_expr_result") + if self.success is not None: + oprot.writeFieldBegin("success", TType.STRUCT, 0) + self.success.write(oprot) + oprot.writeFieldEnd() + if self.o1 is not None: + oprot.writeFieldBegin("o1", TType.STRUCT, 1) + self.o1.write(oprot) + oprot.writeFieldEnd() + if self.o2 is not None: + oprot.writeFieldBegin("o2", TType.STRUCT, 2) + self.o2.write(oprot) + oprot.writeFieldEnd() + oprot.writeFieldStop() + oprot.writeStructEnd() + + def validate(self): + return + + def __repr__(self): + L = ["{}={!r}".format(key, value) for key, value in self.__dict__.items()] + return "{}({})".format(self.__class__.__name__, ", ".join(L)) + + def __eq__(self, other): + return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ + + def __ne__(self, other): + return not (self == other) + + +all_structs.append(get_partitions_by_expr_result) +get_partitions_by_expr_result.thrift_spec = ( + ( + 0, + TType.STRUCT, + "success", + [PartitionsByExprResult, None], + None, + ), # 0 + ( + 1, + TType.STRUCT, + "o1", + [MetaException, None], + None, + ), # 1 + ( + 2, + TType.STRUCT, + "o2", + [NoSuchObjectException, None], + None, + ), # 2 +) + + +class get_partitions_spec_by_expr_args: + """ + Attributes: + - req + + """ + + def __init__( + self, + req=None, + ): + self.req = req + + def read(self, iprot): + if ( + iprot._fast_decode is not None + and isinstance(iprot.trans, TTransport.CReadableTransport) + and self.thrift_spec is not None + ): + iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec]) + return + iprot.readStructBegin() + while True: + (fname, ftype, fid) = iprot.readFieldBegin() + if ftype == TType.STOP: + break + if fid == 1: + if ftype == TType.STRUCT: + self.req = PartitionsByExprRequest() + self.req.read(iprot) + else: + iprot.skip(ftype) + else: + iprot.skip(ftype) + iprot.readFieldEnd() + iprot.readStructEnd() + + def write(self, oprot): + if oprot._fast_encode is not None and self.thrift_spec is not None: + oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec])) + return + oprot.writeStructBegin("get_partitions_spec_by_expr_args") + if self.req is not None: + oprot.writeFieldBegin("req", TType.STRUCT, 1) + self.req.write(oprot) + oprot.writeFieldEnd() + oprot.writeFieldStop() + oprot.writeStructEnd() + + def validate(self): + return + + def __repr__(self): + L = ["{}={!r}".format(key, value) for key, value in self.__dict__.items()] + return "{}({})".format(self.__class__.__name__, ", ".join(L)) + + def __eq__(self, other): + return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ + + def __ne__(self, other): + return not (self == other) + + +all_structs.append(get_partitions_spec_by_expr_args) +get_partitions_spec_by_expr_args.thrift_spec = ( + None, # 0 + ( + 1, + TType.STRUCT, + "req", + [PartitionsByExprRequest, None], + None, + ), # 1 +) + + +class get_partitions_spec_by_expr_result: + """ + Attributes: + - success + - o1 + - o2 + + """ + + def __init__( + self, + success=None, + o1=None, + o2=None, + ): + self.success = success + self.o1 = o1 + self.o2 = o2 + + def read(self, iprot): + if ( + iprot._fast_decode is not None + and isinstance(iprot.trans, TTransport.CReadableTransport) + and self.thrift_spec is not None + ): + iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec]) + return + iprot.readStructBegin() + while True: + (fname, ftype, fid) = iprot.readFieldBegin() + if ftype == TType.STOP: + break + if fid == 0: + if ftype == TType.STRUCT: + self.success = PartitionsSpecByExprResult() + self.success.read(iprot) + else: + iprot.skip(ftype) + elif fid == 1: + if ftype == TType.STRUCT: + self.o1 = MetaException.read(iprot) + else: + iprot.skip(ftype) + elif fid == 2: + if ftype == TType.STRUCT: + self.o2 = NoSuchObjectException.read(iprot) + else: + iprot.skip(ftype) + else: + iprot.skip(ftype) + iprot.readFieldEnd() + iprot.readStructEnd() + + def write(self, oprot): + if oprot._fast_encode is not None and self.thrift_spec is not None: + oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec])) + return + oprot.writeStructBegin("get_partitions_spec_by_expr_result") + if self.success is not None: + oprot.writeFieldBegin("success", TType.STRUCT, 0) + self.success.write(oprot) + oprot.writeFieldEnd() + if self.o1 is not None: + oprot.writeFieldBegin("o1", TType.STRUCT, 1) + self.o1.write(oprot) + oprot.writeFieldEnd() + if self.o2 is not None: + oprot.writeFieldBegin("o2", TType.STRUCT, 2) + self.o2.write(oprot) + oprot.writeFieldEnd() + oprot.writeFieldStop() + oprot.writeStructEnd() + + def validate(self): + return + + def __repr__(self): + L = ["{}={!r}".format(key, value) for key, value in self.__dict__.items()] + return "{}({})".format(self.__class__.__name__, ", ".join(L)) + + def __eq__(self, other): + return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ + + def __ne__(self, other): + return not (self == other) + + +all_structs.append(get_partitions_spec_by_expr_result) +get_partitions_spec_by_expr_result.thrift_spec = ( + ( + 0, + TType.STRUCT, + "success", + [PartitionsSpecByExprResult, None], + None, + ), # 0 + ( + 1, + TType.STRUCT, + "o1", + [MetaException, None], + None, + ), # 1 + ( + 2, + TType.STRUCT, + "o2", + [NoSuchObjectException, None], + None, + ), # 2 +) + + +class get_num_partitions_by_filter_args: + """ + Attributes: + - db_name + - tbl_name + - filter + + """ + + def __init__( + self, + db_name=None, + tbl_name=None, + filter=None, + ): + self.db_name = db_name + self.tbl_name = tbl_name + self.filter = filter + + def read(self, iprot): + if ( + iprot._fast_decode is not None + and isinstance(iprot.trans, TTransport.CReadableTransport) + and self.thrift_spec is not None + ): + iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec]) + return + iprot.readStructBegin() + while True: + (fname, ftype, fid) = iprot.readFieldBegin() + if ftype == TType.STOP: + break + if fid == 1: + if ftype == TType.STRING: + self.db_name = ( + iprot.readString().decode("utf-8", errors="replace") if sys.version_info[0] == 2 else iprot.readString() + ) + else: + iprot.skip(ftype) + elif fid == 2: + if ftype == TType.STRING: + self.tbl_name = ( + iprot.readString().decode("utf-8", errors="replace") if sys.version_info[0] == 2 else iprot.readString() + ) + else: + iprot.skip(ftype) + elif fid == 3: + if ftype == TType.STRING: + self.filter = ( + iprot.readString().decode("utf-8", errors="replace") if sys.version_info[0] == 2 else iprot.readString() + ) + else: + iprot.skip(ftype) + else: + iprot.skip(ftype) + iprot.readFieldEnd() + iprot.readStructEnd() + + def write(self, oprot): + if oprot._fast_encode is not None and self.thrift_spec is not None: + oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec])) + return + oprot.writeStructBegin("get_num_partitions_by_filter_args") + if self.db_name is not None: + oprot.writeFieldBegin("db_name", TType.STRING, 1) + oprot.writeString(self.db_name.encode("utf-8") if sys.version_info[0] == 2 else self.db_name) + oprot.writeFieldEnd() + if self.tbl_name is not None: + oprot.writeFieldBegin("tbl_name", TType.STRING, 2) + oprot.writeString(self.tbl_name.encode("utf-8") if sys.version_info[0] == 2 else self.tbl_name) + oprot.writeFieldEnd() + if self.filter is not None: + oprot.writeFieldBegin("filter", TType.STRING, 3) + oprot.writeString(self.filter.encode("utf-8") if sys.version_info[0] == 2 else self.filter) + oprot.writeFieldEnd() + oprot.writeFieldStop() + oprot.writeStructEnd() + + def validate(self): + return + + def __repr__(self): + L = ["{}={!r}".format(key, value) for key, value in self.__dict__.items()] + return "{}({})".format(self.__class__.__name__, ", ".join(L)) + + def __eq__(self, other): + return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ + + def __ne__(self, other): + return not (self == other) + + +all_structs.append(get_num_partitions_by_filter_args) +get_num_partitions_by_filter_args.thrift_spec = ( + None, # 0 + ( + 1, + TType.STRING, + "db_name", + "UTF8", + None, + ), # 1 + ( + 2, + TType.STRING, + "tbl_name", + "UTF8", + None, + ), # 2 + ( + 3, + TType.STRING, + "filter", + "UTF8", + None, + ), # 3 +) + + +class get_num_partitions_by_filter_result: + """ + Attributes: + - success + - o1 + - o2 + + """ + + def __init__( + self, + success=None, + o1=None, + o2=None, + ): + self.success = success + self.o1 = o1 + self.o2 = o2 + + def read(self, iprot): + if ( + iprot._fast_decode is not None + and isinstance(iprot.trans, TTransport.CReadableTransport) + and self.thrift_spec is not None + ): + iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec]) + return + iprot.readStructBegin() + while True: + (fname, ftype, fid) = iprot.readFieldBegin() + if ftype == TType.STOP: + break + if fid == 0: + if ftype == TType.I32: + self.success = iprot.readI32() + else: + iprot.skip(ftype) + elif fid == 1: + if ftype == TType.STRUCT: + self.o1 = MetaException.read(iprot) + else: + iprot.skip(ftype) + elif fid == 2: + if ftype == TType.STRUCT: + self.o2 = NoSuchObjectException.read(iprot) + else: + iprot.skip(ftype) + else: + iprot.skip(ftype) + iprot.readFieldEnd() + iprot.readStructEnd() + + def write(self, oprot): + if oprot._fast_encode is not None and self.thrift_spec is not None: + oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec])) + return + oprot.writeStructBegin("get_num_partitions_by_filter_result") + if self.success is not None: + oprot.writeFieldBegin("success", TType.I32, 0) + oprot.writeI32(self.success) + oprot.writeFieldEnd() + if self.o1 is not None: + oprot.writeFieldBegin("o1", TType.STRUCT, 1) + self.o1.write(oprot) + oprot.writeFieldEnd() + if self.o2 is not None: + oprot.writeFieldBegin("o2", TType.STRUCT, 2) + self.o2.write(oprot) + oprot.writeFieldEnd() + oprot.writeFieldStop() + oprot.writeStructEnd() + + def validate(self): + return + + def __repr__(self): + L = ["{}={!r}".format(key, value) for key, value in self.__dict__.items()] + return "{}({})".format(self.__class__.__name__, ", ".join(L)) + + def __eq__(self, other): + return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ + + def __ne__(self, other): + return not (self == other) + + +all_structs.append(get_num_partitions_by_filter_result) +get_num_partitions_by_filter_result.thrift_spec = ( + ( + 0, + TType.I32, + "success", + None, + None, + ), # 0 + ( + 1, + TType.STRUCT, + "o1", + [MetaException, None], + None, + ), # 1 + ( + 2, + TType.STRUCT, + "o2", + [NoSuchObjectException, None], + None, + ), # 2 +) + + +class get_partitions_by_names_args: + """ + Attributes: + - db_name + - tbl_name + - names + + """ + + def __init__( + self, + db_name=None, + tbl_name=None, + names=None, + ): + self.db_name = db_name + self.tbl_name = tbl_name + self.names = names + + def read(self, iprot): + if ( + iprot._fast_decode is not None + and isinstance(iprot.trans, TTransport.CReadableTransport) + and self.thrift_spec is not None + ): + iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec]) + return + iprot.readStructBegin() + while True: + (fname, ftype, fid) = iprot.readFieldBegin() + if ftype == TType.STOP: + break + if fid == 1: + if ftype == TType.STRING: + self.db_name = ( + iprot.readString().decode("utf-8", errors="replace") if sys.version_info[0] == 2 else iprot.readString() + ) + else: + iprot.skip(ftype) + elif fid == 2: + if ftype == TType.STRING: + self.tbl_name = ( + iprot.readString().decode("utf-8", errors="replace") if sys.version_info[0] == 2 else iprot.readString() + ) + else: + iprot.skip(ftype) + elif fid == 3: + if ftype == TType.LIST: + self.names = [] + (_etype1629, _size1626) = iprot.readListBegin() + for _i1630 in range(_size1626): + _elem1631 = ( + iprot.readString().decode("utf-8", errors="replace") + if sys.version_info[0] == 2 + else iprot.readString() + ) + self.names.append(_elem1631) + iprot.readListEnd() + else: + iprot.skip(ftype) + else: + iprot.skip(ftype) + iprot.readFieldEnd() + iprot.readStructEnd() + + def write(self, oprot): + if oprot._fast_encode is not None and self.thrift_spec is not None: + oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec])) + return + oprot.writeStructBegin("get_partitions_by_names_args") + if self.db_name is not None: + oprot.writeFieldBegin("db_name", TType.STRING, 1) + oprot.writeString(self.db_name.encode("utf-8") if sys.version_info[0] == 2 else self.db_name) + oprot.writeFieldEnd() + if self.tbl_name is not None: + oprot.writeFieldBegin("tbl_name", TType.STRING, 2) + oprot.writeString(self.tbl_name.encode("utf-8") if sys.version_info[0] == 2 else self.tbl_name) + oprot.writeFieldEnd() + if self.names is not None: + oprot.writeFieldBegin("names", TType.LIST, 3) + oprot.writeListBegin(TType.STRING, len(self.names)) + for iter1632 in self.names: + oprot.writeString(iter1632.encode("utf-8") if sys.version_info[0] == 2 else iter1632) + oprot.writeListEnd() + oprot.writeFieldEnd() + oprot.writeFieldStop() + oprot.writeStructEnd() + + def validate(self): + return + + def __repr__(self): + L = ["{}={!r}".format(key, value) for key, value in self.__dict__.items()] + return "{}({})".format(self.__class__.__name__, ", ".join(L)) + + def __eq__(self, other): + return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ + + def __ne__(self, other): + return not (self == other) + + +all_structs.append(get_partitions_by_names_args) +get_partitions_by_names_args.thrift_spec = ( + None, # 0 + ( + 1, + TType.STRING, + "db_name", + "UTF8", + None, + ), # 1 + ( + 2, + TType.STRING, + "tbl_name", + "UTF8", + None, + ), # 2 + ( + 3, + TType.LIST, + "names", + (TType.STRING, "UTF8", False), + None, + ), # 3 +) + + +class get_partitions_by_names_result: + """ + Attributes: + - success + - o1 + - o2 + + """ + + def __init__( + self, + success=None, + o1=None, + o2=None, + ): + self.success = success + self.o1 = o1 + self.o2 = o2 + + def read(self, iprot): + if ( + iprot._fast_decode is not None + and isinstance(iprot.trans, TTransport.CReadableTransport) + and self.thrift_spec is not None + ): + iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec]) + return + iprot.readStructBegin() + while True: + (fname, ftype, fid) = iprot.readFieldBegin() + if ftype == TType.STOP: + break + if fid == 0: + if ftype == TType.LIST: + self.success = [] + (_etype1636, _size1633) = iprot.readListBegin() + for _i1637 in range(_size1633): + _elem1638 = Partition() + _elem1638.read(iprot) + self.success.append(_elem1638) + iprot.readListEnd() + else: + iprot.skip(ftype) + elif fid == 1: + if ftype == TType.STRUCT: + self.o1 = MetaException.read(iprot) + else: + iprot.skip(ftype) + elif fid == 2: + if ftype == TType.STRUCT: + self.o2 = NoSuchObjectException.read(iprot) + else: + iprot.skip(ftype) + else: + iprot.skip(ftype) + iprot.readFieldEnd() + iprot.readStructEnd() + + def write(self, oprot): + if oprot._fast_encode is not None and self.thrift_spec is not None: + oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec])) + return + oprot.writeStructBegin("get_partitions_by_names_result") + if self.success is not None: + oprot.writeFieldBegin("success", TType.LIST, 0) + oprot.writeListBegin(TType.STRUCT, len(self.success)) + for iter1639 in self.success: + iter1639.write(oprot) + oprot.writeListEnd() + oprot.writeFieldEnd() + if self.o1 is not None: + oprot.writeFieldBegin("o1", TType.STRUCT, 1) + self.o1.write(oprot) + oprot.writeFieldEnd() + if self.o2 is not None: + oprot.writeFieldBegin("o2", TType.STRUCT, 2) + self.o2.write(oprot) + oprot.writeFieldEnd() + oprot.writeFieldStop() + oprot.writeStructEnd() + + def validate(self): + return + + def __repr__(self): + L = ["{}={!r}".format(key, value) for key, value in self.__dict__.items()] + return "{}({})".format(self.__class__.__name__, ", ".join(L)) + + def __eq__(self, other): + return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ + + def __ne__(self, other): + return not (self == other) + + +all_structs.append(get_partitions_by_names_result) +get_partitions_by_names_result.thrift_spec = ( + ( + 0, + TType.LIST, + "success", + (TType.STRUCT, [Partition, None], False), + None, + ), # 0 + ( + 1, + TType.STRUCT, + "o1", + [MetaException, None], + None, + ), # 1 + ( + 2, + TType.STRUCT, + "o2", + [NoSuchObjectException, None], + None, + ), # 2 +) + + +class get_partitions_by_names_req_args: + """ + Attributes: + - req + + """ + + def __init__( + self, + req=None, + ): + self.req = req + + def read(self, iprot): + if ( + iprot._fast_decode is not None + and isinstance(iprot.trans, TTransport.CReadableTransport) + and self.thrift_spec is not None + ): + iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec]) + return + iprot.readStructBegin() + while True: + (fname, ftype, fid) = iprot.readFieldBegin() + if ftype == TType.STOP: + break + if fid == 1: + if ftype == TType.STRUCT: + self.req = GetPartitionsByNamesRequest() + self.req.read(iprot) + else: + iprot.skip(ftype) + else: + iprot.skip(ftype) + iprot.readFieldEnd() + iprot.readStructEnd() + + def write(self, oprot): + if oprot._fast_encode is not None and self.thrift_spec is not None: + oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec])) + return + oprot.writeStructBegin("get_partitions_by_names_req_args") + if self.req is not None: + oprot.writeFieldBegin("req", TType.STRUCT, 1) + self.req.write(oprot) + oprot.writeFieldEnd() + oprot.writeFieldStop() + oprot.writeStructEnd() + + def validate(self): + return + + def __repr__(self): + L = ["{}={!r}".format(key, value) for key, value in self.__dict__.items()] + return "{}({})".format(self.__class__.__name__, ", ".join(L)) + + def __eq__(self, other): + return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ + + def __ne__(self, other): + return not (self == other) + + +all_structs.append(get_partitions_by_names_req_args) +get_partitions_by_names_req_args.thrift_spec = ( + None, # 0 + ( + 1, + TType.STRUCT, + "req", + [GetPartitionsByNamesRequest, None], + None, + ), # 1 +) + + +class get_partitions_by_names_req_result: + """ + Attributes: + - success + - o1 + - o2 + + """ + + def __init__( + self, + success=None, + o1=None, + o2=None, + ): + self.success = success + self.o1 = o1 + self.o2 = o2 + + def read(self, iprot): + if ( + iprot._fast_decode is not None + and isinstance(iprot.trans, TTransport.CReadableTransport) + and self.thrift_spec is not None + ): + iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec]) + return + iprot.readStructBegin() + while True: + (fname, ftype, fid) = iprot.readFieldBegin() + if ftype == TType.STOP: + break + if fid == 0: + if ftype == TType.STRUCT: + self.success = GetPartitionsByNamesResult() + self.success.read(iprot) + else: + iprot.skip(ftype) + elif fid == 1: + if ftype == TType.STRUCT: + self.o1 = MetaException.read(iprot) + else: + iprot.skip(ftype) + elif fid == 2: + if ftype == TType.STRUCT: + self.o2 = NoSuchObjectException.read(iprot) + else: + iprot.skip(ftype) + else: + iprot.skip(ftype) + iprot.readFieldEnd() + iprot.readStructEnd() + + def write(self, oprot): + if oprot._fast_encode is not None and self.thrift_spec is not None: + oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec])) + return + oprot.writeStructBegin("get_partitions_by_names_req_result") + if self.success is not None: + oprot.writeFieldBegin("success", TType.STRUCT, 0) + self.success.write(oprot) + oprot.writeFieldEnd() + if self.o1 is not None: + oprot.writeFieldBegin("o1", TType.STRUCT, 1) + self.o1.write(oprot) + oprot.writeFieldEnd() + if self.o2 is not None: + oprot.writeFieldBegin("o2", TType.STRUCT, 2) + self.o2.write(oprot) + oprot.writeFieldEnd() + oprot.writeFieldStop() + oprot.writeStructEnd() + + def validate(self): + return + + def __repr__(self): + L = ["{}={!r}".format(key, value) for key, value in self.__dict__.items()] + return "{}({})".format(self.__class__.__name__, ", ".join(L)) + + def __eq__(self, other): + return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ + + def __ne__(self, other): + return not (self == other) + + +all_structs.append(get_partitions_by_names_req_result) +get_partitions_by_names_req_result.thrift_spec = ( + ( + 0, + TType.STRUCT, + "success", + [GetPartitionsByNamesResult, None], + None, + ), # 0 + ( + 1, + TType.STRUCT, + "o1", + [MetaException, None], + None, + ), # 1 + ( + 2, + TType.STRUCT, + "o2", + [NoSuchObjectException, None], + None, + ), # 2 +) + + +class alter_partition_args: + """ + Attributes: + - db_name + - tbl_name + - new_part + + """ + + def __init__( + self, + db_name=None, + tbl_name=None, + new_part=None, + ): + self.db_name = db_name + self.tbl_name = tbl_name + self.new_part = new_part + + def read(self, iprot): + if ( + iprot._fast_decode is not None + and isinstance(iprot.trans, TTransport.CReadableTransport) + and self.thrift_spec is not None + ): + iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec]) + return + iprot.readStructBegin() + while True: + (fname, ftype, fid) = iprot.readFieldBegin() + if ftype == TType.STOP: + break + if fid == 1: + if ftype == TType.STRING: + self.db_name = ( + iprot.readString().decode("utf-8", errors="replace") if sys.version_info[0] == 2 else iprot.readString() + ) + else: + iprot.skip(ftype) + elif fid == 2: + if ftype == TType.STRING: + self.tbl_name = ( + iprot.readString().decode("utf-8", errors="replace") if sys.version_info[0] == 2 else iprot.readString() + ) + else: + iprot.skip(ftype) + elif fid == 3: + if ftype == TType.STRUCT: + self.new_part = Partition() + self.new_part.read(iprot) + else: + iprot.skip(ftype) + else: + iprot.skip(ftype) + iprot.readFieldEnd() + iprot.readStructEnd() + + def write(self, oprot): + if oprot._fast_encode is not None and self.thrift_spec is not None: + oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec])) + return + oprot.writeStructBegin("alter_partition_args") + if self.db_name is not None: + oprot.writeFieldBegin("db_name", TType.STRING, 1) + oprot.writeString(self.db_name.encode("utf-8") if sys.version_info[0] == 2 else self.db_name) + oprot.writeFieldEnd() + if self.tbl_name is not None: + oprot.writeFieldBegin("tbl_name", TType.STRING, 2) + oprot.writeString(self.tbl_name.encode("utf-8") if sys.version_info[0] == 2 else self.tbl_name) + oprot.writeFieldEnd() + if self.new_part is not None: + oprot.writeFieldBegin("new_part", TType.STRUCT, 3) + self.new_part.write(oprot) + oprot.writeFieldEnd() + oprot.writeFieldStop() + oprot.writeStructEnd() + + def validate(self): + return + + def __repr__(self): + L = ["{}={!r}".format(key, value) for key, value in self.__dict__.items()] + return "{}({})".format(self.__class__.__name__, ", ".join(L)) + + def __eq__(self, other): + return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ + + def __ne__(self, other): + return not (self == other) + + +all_structs.append(alter_partition_args) +alter_partition_args.thrift_spec = ( + None, # 0 + ( + 1, + TType.STRING, + "db_name", + "UTF8", + None, + ), # 1 + ( + 2, + TType.STRING, + "tbl_name", + "UTF8", + None, + ), # 2 + ( + 3, + TType.STRUCT, + "new_part", + [Partition, None], + None, + ), # 3 +) + + +class alter_partition_result: + """ + Attributes: + - o1 + - o2 + + """ + + def __init__( + self, + o1=None, + o2=None, + ): + self.o1 = o1 + self.o2 = o2 + + def read(self, iprot): + if ( + iprot._fast_decode is not None + and isinstance(iprot.trans, TTransport.CReadableTransport) + and self.thrift_spec is not None + ): + iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec]) + return + iprot.readStructBegin() + while True: + (fname, ftype, fid) = iprot.readFieldBegin() + if ftype == TType.STOP: + break + if fid == 1: + if ftype == TType.STRUCT: + self.o1 = InvalidOperationException.read(iprot) + else: + iprot.skip(ftype) + elif fid == 2: + if ftype == TType.STRUCT: + self.o2 = MetaException.read(iprot) + else: + iprot.skip(ftype) + else: + iprot.skip(ftype) + iprot.readFieldEnd() + iprot.readStructEnd() + + def write(self, oprot): + if oprot._fast_encode is not None and self.thrift_spec is not None: + oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec])) + return + oprot.writeStructBegin("alter_partition_result") + if self.o1 is not None: + oprot.writeFieldBegin("o1", TType.STRUCT, 1) + self.o1.write(oprot) + oprot.writeFieldEnd() + if self.o2 is not None: + oprot.writeFieldBegin("o2", TType.STRUCT, 2) + self.o2.write(oprot) + oprot.writeFieldEnd() + oprot.writeFieldStop() + oprot.writeStructEnd() + + def validate(self): + return + + def __repr__(self): + L = ["{}={!r}".format(key, value) for key, value in self.__dict__.items()] + return "{}({})".format(self.__class__.__name__, ", ".join(L)) + + def __eq__(self, other): + return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ + + def __ne__(self, other): + return not (self == other) + + +all_structs.append(alter_partition_result) +alter_partition_result.thrift_spec = ( + None, # 0 + ( + 1, + TType.STRUCT, + "o1", + [InvalidOperationException, None], + None, + ), # 1 + ( + 2, + TType.STRUCT, + "o2", + [MetaException, None], + None, + ), # 2 +) + + +class alter_partitions_args: + """ + Attributes: + - db_name + - tbl_name + - new_parts + + """ + + def __init__( + self, + db_name=None, + tbl_name=None, + new_parts=None, + ): + self.db_name = db_name + self.tbl_name = tbl_name + self.new_parts = new_parts + + def read(self, iprot): + if ( + iprot._fast_decode is not None + and isinstance(iprot.trans, TTransport.CReadableTransport) + and self.thrift_spec is not None + ): + iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec]) + return + iprot.readStructBegin() + while True: + (fname, ftype, fid) = iprot.readFieldBegin() + if ftype == TType.STOP: + break + if fid == 1: + if ftype == TType.STRING: + self.db_name = ( + iprot.readString().decode("utf-8", errors="replace") if sys.version_info[0] == 2 else iprot.readString() + ) + else: + iprot.skip(ftype) + elif fid == 2: + if ftype == TType.STRING: + self.tbl_name = ( + iprot.readString().decode("utf-8", errors="replace") if sys.version_info[0] == 2 else iprot.readString() + ) + else: + iprot.skip(ftype) + elif fid == 3: + if ftype == TType.LIST: + self.new_parts = [] + (_etype1643, _size1640) = iprot.readListBegin() + for _i1644 in range(_size1640): + _elem1645 = Partition() + _elem1645.read(iprot) + self.new_parts.append(_elem1645) + iprot.readListEnd() + else: + iprot.skip(ftype) + else: + iprot.skip(ftype) + iprot.readFieldEnd() + iprot.readStructEnd() + + def write(self, oprot): + if oprot._fast_encode is not None and self.thrift_spec is not None: + oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec])) + return + oprot.writeStructBegin("alter_partitions_args") + if self.db_name is not None: + oprot.writeFieldBegin("db_name", TType.STRING, 1) + oprot.writeString(self.db_name.encode("utf-8") if sys.version_info[0] == 2 else self.db_name) + oprot.writeFieldEnd() + if self.tbl_name is not None: + oprot.writeFieldBegin("tbl_name", TType.STRING, 2) + oprot.writeString(self.tbl_name.encode("utf-8") if sys.version_info[0] == 2 else self.tbl_name) + oprot.writeFieldEnd() + if self.new_parts is not None: + oprot.writeFieldBegin("new_parts", TType.LIST, 3) + oprot.writeListBegin(TType.STRUCT, len(self.new_parts)) + for iter1646 in self.new_parts: + iter1646.write(oprot) + oprot.writeListEnd() + oprot.writeFieldEnd() + oprot.writeFieldStop() + oprot.writeStructEnd() + + def validate(self): + return + + def __repr__(self): + L = ["{}={!r}".format(key, value) for key, value in self.__dict__.items()] + return "{}({})".format(self.__class__.__name__, ", ".join(L)) + + def __eq__(self, other): + return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ + + def __ne__(self, other): + return not (self == other) + + +all_structs.append(alter_partitions_args) +alter_partitions_args.thrift_spec = ( + None, # 0 + ( + 1, + TType.STRING, + "db_name", + "UTF8", + None, + ), # 1 + ( + 2, + TType.STRING, + "tbl_name", + "UTF8", + None, + ), # 2 + ( + 3, + TType.LIST, + "new_parts", + (TType.STRUCT, [Partition, None], False), + None, + ), # 3 +) + + +class alter_partitions_result: + """ + Attributes: + - o1 + - o2 + + """ + + def __init__( + self, + o1=None, + o2=None, + ): + self.o1 = o1 + self.o2 = o2 + + def read(self, iprot): + if ( + iprot._fast_decode is not None + and isinstance(iprot.trans, TTransport.CReadableTransport) + and self.thrift_spec is not None + ): + iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec]) + return + iprot.readStructBegin() + while True: + (fname, ftype, fid) = iprot.readFieldBegin() + if ftype == TType.STOP: + break + if fid == 1: + if ftype == TType.STRUCT: + self.o1 = InvalidOperationException.read(iprot) + else: + iprot.skip(ftype) + elif fid == 2: + if ftype == TType.STRUCT: + self.o2 = MetaException.read(iprot) + else: + iprot.skip(ftype) + else: + iprot.skip(ftype) + iprot.readFieldEnd() + iprot.readStructEnd() + + def write(self, oprot): + if oprot._fast_encode is not None and self.thrift_spec is not None: + oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec])) + return + oprot.writeStructBegin("alter_partitions_result") + if self.o1 is not None: + oprot.writeFieldBegin("o1", TType.STRUCT, 1) + self.o1.write(oprot) + oprot.writeFieldEnd() + if self.o2 is not None: + oprot.writeFieldBegin("o2", TType.STRUCT, 2) + self.o2.write(oprot) + oprot.writeFieldEnd() + oprot.writeFieldStop() + oprot.writeStructEnd() + + def validate(self): + return + + def __repr__(self): + L = ["{}={!r}".format(key, value) for key, value in self.__dict__.items()] + return "{}({})".format(self.__class__.__name__, ", ".join(L)) + + def __eq__(self, other): + return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ + + def __ne__(self, other): + return not (self == other) + + +all_structs.append(alter_partitions_result) +alter_partitions_result.thrift_spec = ( + None, # 0 + ( + 1, + TType.STRUCT, + "o1", + [InvalidOperationException, None], + None, + ), # 1 + ( + 2, + TType.STRUCT, + "o2", + [MetaException, None], + None, + ), # 2 +) + + +class alter_partitions_with_environment_context_args: + """ + Attributes: + - db_name + - tbl_name + - new_parts + - environment_context + + """ + + def __init__( + self, + db_name=None, + tbl_name=None, + new_parts=None, + environment_context=None, + ): + self.db_name = db_name + self.tbl_name = tbl_name + self.new_parts = new_parts + self.environment_context = environment_context + + def read(self, iprot): + if ( + iprot._fast_decode is not None + and isinstance(iprot.trans, TTransport.CReadableTransport) + and self.thrift_spec is not None + ): + iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec]) + return + iprot.readStructBegin() + while True: + (fname, ftype, fid) = iprot.readFieldBegin() + if ftype == TType.STOP: + break + if fid == 1: + if ftype == TType.STRING: + self.db_name = ( + iprot.readString().decode("utf-8", errors="replace") if sys.version_info[0] == 2 else iprot.readString() + ) + else: + iprot.skip(ftype) + elif fid == 2: + if ftype == TType.STRING: + self.tbl_name = ( + iprot.readString().decode("utf-8", errors="replace") if sys.version_info[0] == 2 else iprot.readString() + ) + else: + iprot.skip(ftype) + elif fid == 3: + if ftype == TType.LIST: + self.new_parts = [] + (_etype1650, _size1647) = iprot.readListBegin() + for _i1651 in range(_size1647): + _elem1652 = Partition() + _elem1652.read(iprot) + self.new_parts.append(_elem1652) + iprot.readListEnd() + else: + iprot.skip(ftype) + elif fid == 4: + if ftype == TType.STRUCT: + self.environment_context = EnvironmentContext() + self.environment_context.read(iprot) + else: + iprot.skip(ftype) + else: + iprot.skip(ftype) + iprot.readFieldEnd() + iprot.readStructEnd() + + def write(self, oprot): + if oprot._fast_encode is not None and self.thrift_spec is not None: + oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec])) + return + oprot.writeStructBegin("alter_partitions_with_environment_context_args") + if self.db_name is not None: + oprot.writeFieldBegin("db_name", TType.STRING, 1) + oprot.writeString(self.db_name.encode("utf-8") if sys.version_info[0] == 2 else self.db_name) + oprot.writeFieldEnd() + if self.tbl_name is not None: + oprot.writeFieldBegin("tbl_name", TType.STRING, 2) + oprot.writeString(self.tbl_name.encode("utf-8") if sys.version_info[0] == 2 else self.tbl_name) + oprot.writeFieldEnd() + if self.new_parts is not None: + oprot.writeFieldBegin("new_parts", TType.LIST, 3) + oprot.writeListBegin(TType.STRUCT, len(self.new_parts)) + for iter1653 in self.new_parts: + iter1653.write(oprot) + oprot.writeListEnd() + oprot.writeFieldEnd() + if self.environment_context is not None: + oprot.writeFieldBegin("environment_context", TType.STRUCT, 4) + self.environment_context.write(oprot) + oprot.writeFieldEnd() + oprot.writeFieldStop() + oprot.writeStructEnd() + + def validate(self): + return + + def __repr__(self): + L = ["{}={!r}".format(key, value) for key, value in self.__dict__.items()] + return "{}({})".format(self.__class__.__name__, ", ".join(L)) + + def __eq__(self, other): + return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ + + def __ne__(self, other): + return not (self == other) + + +all_structs.append(alter_partitions_with_environment_context_args) +alter_partitions_with_environment_context_args.thrift_spec = ( + None, # 0 + ( + 1, + TType.STRING, + "db_name", + "UTF8", + None, + ), # 1 + ( + 2, + TType.STRING, + "tbl_name", + "UTF8", + None, + ), # 2 + ( + 3, + TType.LIST, + "new_parts", + (TType.STRUCT, [Partition, None], False), + None, + ), # 3 + ( + 4, + TType.STRUCT, + "environment_context", + [EnvironmentContext, None], + None, + ), # 4 +) + + +class alter_partitions_with_environment_context_result: + """ + Attributes: + - o1 + - o2 + + """ + + def __init__( + self, + o1=None, + o2=None, + ): + self.o1 = o1 + self.o2 = o2 + + def read(self, iprot): + if ( + iprot._fast_decode is not None + and isinstance(iprot.trans, TTransport.CReadableTransport) + and self.thrift_spec is not None + ): + iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec]) + return + iprot.readStructBegin() + while True: + (fname, ftype, fid) = iprot.readFieldBegin() + if ftype == TType.STOP: + break + if fid == 1: + if ftype == TType.STRUCT: + self.o1 = InvalidOperationException.read(iprot) + else: + iprot.skip(ftype) + elif fid == 2: + if ftype == TType.STRUCT: + self.o2 = MetaException.read(iprot) + else: + iprot.skip(ftype) + else: + iprot.skip(ftype) + iprot.readFieldEnd() + iprot.readStructEnd() + + def write(self, oprot): + if oprot._fast_encode is not None and self.thrift_spec is not None: + oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec])) + return + oprot.writeStructBegin("alter_partitions_with_environment_context_result") + if self.o1 is not None: + oprot.writeFieldBegin("o1", TType.STRUCT, 1) + self.o1.write(oprot) + oprot.writeFieldEnd() + if self.o2 is not None: + oprot.writeFieldBegin("o2", TType.STRUCT, 2) + self.o2.write(oprot) + oprot.writeFieldEnd() + oprot.writeFieldStop() + oprot.writeStructEnd() + + def validate(self): + return + + def __repr__(self): + L = ["{}={!r}".format(key, value) for key, value in self.__dict__.items()] + return "{}({})".format(self.__class__.__name__, ", ".join(L)) + + def __eq__(self, other): + return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ + + def __ne__(self, other): + return not (self == other) + + +all_structs.append(alter_partitions_with_environment_context_result) +alter_partitions_with_environment_context_result.thrift_spec = ( + None, # 0 + ( + 1, + TType.STRUCT, + "o1", + [InvalidOperationException, None], + None, + ), # 1 + ( + 2, + TType.STRUCT, + "o2", + [MetaException, None], + None, + ), # 2 +) + + +class alter_partitions_req_args: + """ + Attributes: + - req + + """ + + def __init__( + self, + req=None, + ): + self.req = req + + def read(self, iprot): + if ( + iprot._fast_decode is not None + and isinstance(iprot.trans, TTransport.CReadableTransport) + and self.thrift_spec is not None + ): + iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec]) + return + iprot.readStructBegin() + while True: + (fname, ftype, fid) = iprot.readFieldBegin() + if ftype == TType.STOP: + break + if fid == 1: + if ftype == TType.STRUCT: + self.req = AlterPartitionsRequest() + self.req.read(iprot) + else: + iprot.skip(ftype) + else: + iprot.skip(ftype) + iprot.readFieldEnd() + iprot.readStructEnd() + + def write(self, oprot): + if oprot._fast_encode is not None and self.thrift_spec is not None: + oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec])) + return + oprot.writeStructBegin("alter_partitions_req_args") + if self.req is not None: + oprot.writeFieldBegin("req", TType.STRUCT, 1) + self.req.write(oprot) + oprot.writeFieldEnd() + oprot.writeFieldStop() + oprot.writeStructEnd() + + def validate(self): + return + + def __repr__(self): + L = ["{}={!r}".format(key, value) for key, value in self.__dict__.items()] + return "{}({})".format(self.__class__.__name__, ", ".join(L)) + + def __eq__(self, other): + return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ + + def __ne__(self, other): + return not (self == other) + + +all_structs.append(alter_partitions_req_args) +alter_partitions_req_args.thrift_spec = ( + None, # 0 + ( + 1, + TType.STRUCT, + "req", + [AlterPartitionsRequest, None], + None, + ), # 1 +) + + +class alter_partitions_req_result: + """ + Attributes: + - success + - o1 + - o2 + + """ + + def __init__( + self, + success=None, + o1=None, + o2=None, + ): + self.success = success + self.o1 = o1 + self.o2 = o2 + + def read(self, iprot): + if ( + iprot._fast_decode is not None + and isinstance(iprot.trans, TTransport.CReadableTransport) + and self.thrift_spec is not None + ): + iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec]) + return + iprot.readStructBegin() + while True: + (fname, ftype, fid) = iprot.readFieldBegin() + if ftype == TType.STOP: + break + if fid == 0: + if ftype == TType.STRUCT: + self.success = AlterPartitionsResponse() + self.success.read(iprot) + else: + iprot.skip(ftype) + elif fid == 1: + if ftype == TType.STRUCT: + self.o1 = InvalidOperationException.read(iprot) + else: + iprot.skip(ftype) + elif fid == 2: + if ftype == TType.STRUCT: + self.o2 = MetaException.read(iprot) + else: + iprot.skip(ftype) + else: + iprot.skip(ftype) + iprot.readFieldEnd() + iprot.readStructEnd() + + def write(self, oprot): + if oprot._fast_encode is not None and self.thrift_spec is not None: + oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec])) + return + oprot.writeStructBegin("alter_partitions_req_result") + if self.success is not None: + oprot.writeFieldBegin("success", TType.STRUCT, 0) + self.success.write(oprot) + oprot.writeFieldEnd() + if self.o1 is not None: + oprot.writeFieldBegin("o1", TType.STRUCT, 1) + self.o1.write(oprot) + oprot.writeFieldEnd() + if self.o2 is not None: + oprot.writeFieldBegin("o2", TType.STRUCT, 2) + self.o2.write(oprot) + oprot.writeFieldEnd() + oprot.writeFieldStop() + oprot.writeStructEnd() + + def validate(self): + return + + def __repr__(self): + L = ["{}={!r}".format(key, value) for key, value in self.__dict__.items()] + return "{}({})".format(self.__class__.__name__, ", ".join(L)) + + def __eq__(self, other): + return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ + + def __ne__(self, other): + return not (self == other) + + +all_structs.append(alter_partitions_req_result) +alter_partitions_req_result.thrift_spec = ( + ( + 0, + TType.STRUCT, + "success", + [AlterPartitionsResponse, None], + None, + ), # 0 + ( + 1, + TType.STRUCT, + "o1", + [InvalidOperationException, None], + None, + ), # 1 + ( + 2, + TType.STRUCT, + "o2", + [MetaException, None], + None, + ), # 2 +) + + +class alter_partition_with_environment_context_args: + """ + Attributes: + - db_name + - tbl_name + - new_part + - environment_context + + """ + + def __init__( + self, + db_name=None, + tbl_name=None, + new_part=None, + environment_context=None, + ): + self.db_name = db_name + self.tbl_name = tbl_name + self.new_part = new_part + self.environment_context = environment_context + + def read(self, iprot): + if ( + iprot._fast_decode is not None + and isinstance(iprot.trans, TTransport.CReadableTransport) + and self.thrift_spec is not None + ): + iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec]) + return + iprot.readStructBegin() + while True: + (fname, ftype, fid) = iprot.readFieldBegin() + if ftype == TType.STOP: + break + if fid == 1: + if ftype == TType.STRING: + self.db_name = ( + iprot.readString().decode("utf-8", errors="replace") if sys.version_info[0] == 2 else iprot.readString() + ) + else: + iprot.skip(ftype) + elif fid == 2: + if ftype == TType.STRING: + self.tbl_name = ( + iprot.readString().decode("utf-8", errors="replace") if sys.version_info[0] == 2 else iprot.readString() + ) + else: + iprot.skip(ftype) + elif fid == 3: + if ftype == TType.STRUCT: + self.new_part = Partition() + self.new_part.read(iprot) + else: + iprot.skip(ftype) + elif fid == 4: + if ftype == TType.STRUCT: + self.environment_context = EnvironmentContext() + self.environment_context.read(iprot) + else: + iprot.skip(ftype) + else: + iprot.skip(ftype) + iprot.readFieldEnd() + iprot.readStructEnd() + + def write(self, oprot): + if oprot._fast_encode is not None and self.thrift_spec is not None: + oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec])) + return + oprot.writeStructBegin("alter_partition_with_environment_context_args") + if self.db_name is not None: + oprot.writeFieldBegin("db_name", TType.STRING, 1) + oprot.writeString(self.db_name.encode("utf-8") if sys.version_info[0] == 2 else self.db_name) + oprot.writeFieldEnd() + if self.tbl_name is not None: + oprot.writeFieldBegin("tbl_name", TType.STRING, 2) + oprot.writeString(self.tbl_name.encode("utf-8") if sys.version_info[0] == 2 else self.tbl_name) + oprot.writeFieldEnd() + if self.new_part is not None: + oprot.writeFieldBegin("new_part", TType.STRUCT, 3) + self.new_part.write(oprot) + oprot.writeFieldEnd() + if self.environment_context is not None: + oprot.writeFieldBegin("environment_context", TType.STRUCT, 4) + self.environment_context.write(oprot) + oprot.writeFieldEnd() + oprot.writeFieldStop() + oprot.writeStructEnd() + + def validate(self): + return + + def __repr__(self): + L = ["{}={!r}".format(key, value) for key, value in self.__dict__.items()] + return "{}({})".format(self.__class__.__name__, ", ".join(L)) + + def __eq__(self, other): + return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ + + def __ne__(self, other): + return not (self == other) + + +all_structs.append(alter_partition_with_environment_context_args) +alter_partition_with_environment_context_args.thrift_spec = ( + None, # 0 + ( + 1, + TType.STRING, + "db_name", + "UTF8", + None, + ), # 1 + ( + 2, + TType.STRING, + "tbl_name", + "UTF8", + None, + ), # 2 + ( + 3, + TType.STRUCT, + "new_part", + [Partition, None], + None, + ), # 3 + ( + 4, + TType.STRUCT, + "environment_context", + [EnvironmentContext, None], + None, + ), # 4 +) + + +class alter_partition_with_environment_context_result: + """ + Attributes: + - o1 + - o2 + + """ + + def __init__( + self, + o1=None, + o2=None, + ): + self.o1 = o1 + self.o2 = o2 + + def read(self, iprot): + if ( + iprot._fast_decode is not None + and isinstance(iprot.trans, TTransport.CReadableTransport) + and self.thrift_spec is not None + ): + iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec]) + return + iprot.readStructBegin() + while True: + (fname, ftype, fid) = iprot.readFieldBegin() + if ftype == TType.STOP: + break + if fid == 1: + if ftype == TType.STRUCT: + self.o1 = InvalidOperationException.read(iprot) + else: + iprot.skip(ftype) + elif fid == 2: + if ftype == TType.STRUCT: + self.o2 = MetaException.read(iprot) + else: + iprot.skip(ftype) + else: + iprot.skip(ftype) + iprot.readFieldEnd() + iprot.readStructEnd() + + def write(self, oprot): + if oprot._fast_encode is not None and self.thrift_spec is not None: + oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec])) + return + oprot.writeStructBegin("alter_partition_with_environment_context_result") + if self.o1 is not None: + oprot.writeFieldBegin("o1", TType.STRUCT, 1) + self.o1.write(oprot) + oprot.writeFieldEnd() + if self.o2 is not None: + oprot.writeFieldBegin("o2", TType.STRUCT, 2) + self.o2.write(oprot) + oprot.writeFieldEnd() + oprot.writeFieldStop() + oprot.writeStructEnd() + + def validate(self): + return + + def __repr__(self): + L = ["{}={!r}".format(key, value) for key, value in self.__dict__.items()] + return "{}({})".format(self.__class__.__name__, ", ".join(L)) + + def __eq__(self, other): + return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ + + def __ne__(self, other): + return not (self == other) + + +all_structs.append(alter_partition_with_environment_context_result) +alter_partition_with_environment_context_result.thrift_spec = ( + None, # 0 + ( + 1, + TType.STRUCT, + "o1", + [InvalidOperationException, None], + None, + ), # 1 + ( + 2, + TType.STRUCT, + "o2", + [MetaException, None], + None, + ), # 2 +) + + +class rename_partition_args: + """ + Attributes: + - db_name + - tbl_name + - part_vals + - new_part + + """ + + def __init__( + self, + db_name=None, + tbl_name=None, + part_vals=None, + new_part=None, + ): + self.db_name = db_name + self.tbl_name = tbl_name + self.part_vals = part_vals + self.new_part = new_part + + def read(self, iprot): + if ( + iprot._fast_decode is not None + and isinstance(iprot.trans, TTransport.CReadableTransport) + and self.thrift_spec is not None + ): + iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec]) + return + iprot.readStructBegin() + while True: + (fname, ftype, fid) = iprot.readFieldBegin() + if ftype == TType.STOP: + break + if fid == 1: + if ftype == TType.STRING: + self.db_name = ( + iprot.readString().decode("utf-8", errors="replace") if sys.version_info[0] == 2 else iprot.readString() + ) + else: + iprot.skip(ftype) + elif fid == 2: + if ftype == TType.STRING: + self.tbl_name = ( + iprot.readString().decode("utf-8", errors="replace") if sys.version_info[0] == 2 else iprot.readString() + ) + else: + iprot.skip(ftype) + elif fid == 3: + if ftype == TType.LIST: + self.part_vals = [] + (_etype1657, _size1654) = iprot.readListBegin() + for _i1658 in range(_size1654): + _elem1659 = ( + iprot.readString().decode("utf-8", errors="replace") + if sys.version_info[0] == 2 + else iprot.readString() + ) + self.part_vals.append(_elem1659) + iprot.readListEnd() + else: + iprot.skip(ftype) + elif fid == 4: + if ftype == TType.STRUCT: + self.new_part = Partition() + self.new_part.read(iprot) + else: + iprot.skip(ftype) + else: + iprot.skip(ftype) + iprot.readFieldEnd() + iprot.readStructEnd() + + def write(self, oprot): + if oprot._fast_encode is not None and self.thrift_spec is not None: + oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec])) + return + oprot.writeStructBegin("rename_partition_args") + if self.db_name is not None: + oprot.writeFieldBegin("db_name", TType.STRING, 1) + oprot.writeString(self.db_name.encode("utf-8") if sys.version_info[0] == 2 else self.db_name) + oprot.writeFieldEnd() + if self.tbl_name is not None: + oprot.writeFieldBegin("tbl_name", TType.STRING, 2) + oprot.writeString(self.tbl_name.encode("utf-8") if sys.version_info[0] == 2 else self.tbl_name) + oprot.writeFieldEnd() + if self.part_vals is not None: + oprot.writeFieldBegin("part_vals", TType.LIST, 3) + oprot.writeListBegin(TType.STRING, len(self.part_vals)) + for iter1660 in self.part_vals: + oprot.writeString(iter1660.encode("utf-8") if sys.version_info[0] == 2 else iter1660) + oprot.writeListEnd() + oprot.writeFieldEnd() + if self.new_part is not None: + oprot.writeFieldBegin("new_part", TType.STRUCT, 4) + self.new_part.write(oprot) + oprot.writeFieldEnd() + oprot.writeFieldStop() + oprot.writeStructEnd() + + def validate(self): + return + + def __repr__(self): + L = ["{}={!r}".format(key, value) for key, value in self.__dict__.items()] + return "{}({})".format(self.__class__.__name__, ", ".join(L)) + + def __eq__(self, other): + return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ + + def __ne__(self, other): + return not (self == other) + + +all_structs.append(rename_partition_args) +rename_partition_args.thrift_spec = ( + None, # 0 + ( + 1, + TType.STRING, + "db_name", + "UTF8", + None, + ), # 1 + ( + 2, + TType.STRING, + "tbl_name", + "UTF8", + None, + ), # 2 + ( + 3, + TType.LIST, + "part_vals", + (TType.STRING, "UTF8", False), + None, + ), # 3 + ( + 4, + TType.STRUCT, + "new_part", + [Partition, None], + None, + ), # 4 +) + + +class rename_partition_result: + """ + Attributes: + - o1 + - o2 + + """ + + def __init__( + self, + o1=None, + o2=None, + ): + self.o1 = o1 + self.o2 = o2 + + def read(self, iprot): + if ( + iprot._fast_decode is not None + and isinstance(iprot.trans, TTransport.CReadableTransport) + and self.thrift_spec is not None + ): + iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec]) + return + iprot.readStructBegin() + while True: + (fname, ftype, fid) = iprot.readFieldBegin() + if ftype == TType.STOP: + break + if fid == 1: + if ftype == TType.STRUCT: + self.o1 = InvalidOperationException.read(iprot) + else: + iprot.skip(ftype) + elif fid == 2: + if ftype == TType.STRUCT: + self.o2 = MetaException.read(iprot) + else: + iprot.skip(ftype) + else: + iprot.skip(ftype) + iprot.readFieldEnd() + iprot.readStructEnd() + + def write(self, oprot): + if oprot._fast_encode is not None and self.thrift_spec is not None: + oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec])) + return + oprot.writeStructBegin("rename_partition_result") + if self.o1 is not None: + oprot.writeFieldBegin("o1", TType.STRUCT, 1) + self.o1.write(oprot) + oprot.writeFieldEnd() + if self.o2 is not None: + oprot.writeFieldBegin("o2", TType.STRUCT, 2) + self.o2.write(oprot) + oprot.writeFieldEnd() + oprot.writeFieldStop() + oprot.writeStructEnd() + + def validate(self): + return + + def __repr__(self): + L = ["{}={!r}".format(key, value) for key, value in self.__dict__.items()] + return "{}({})".format(self.__class__.__name__, ", ".join(L)) + + def __eq__(self, other): + return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ + + def __ne__(self, other): + return not (self == other) + + +all_structs.append(rename_partition_result) +rename_partition_result.thrift_spec = ( + None, # 0 + ( + 1, + TType.STRUCT, + "o1", + [InvalidOperationException, None], + None, + ), # 1 + ( + 2, + TType.STRUCT, + "o2", + [MetaException, None], + None, + ), # 2 +) + + +class rename_partition_req_args: + """ + Attributes: + - req + + """ + + def __init__( + self, + req=None, + ): + self.req = req + + def read(self, iprot): + if ( + iprot._fast_decode is not None + and isinstance(iprot.trans, TTransport.CReadableTransport) + and self.thrift_spec is not None + ): + iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec]) + return + iprot.readStructBegin() + while True: + (fname, ftype, fid) = iprot.readFieldBegin() + if ftype == TType.STOP: + break + if fid == 1: + if ftype == TType.STRUCT: + self.req = RenamePartitionRequest() + self.req.read(iprot) + else: + iprot.skip(ftype) + else: + iprot.skip(ftype) + iprot.readFieldEnd() + iprot.readStructEnd() + + def write(self, oprot): + if oprot._fast_encode is not None and self.thrift_spec is not None: + oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec])) + return + oprot.writeStructBegin("rename_partition_req_args") + if self.req is not None: + oprot.writeFieldBegin("req", TType.STRUCT, 1) + self.req.write(oprot) + oprot.writeFieldEnd() + oprot.writeFieldStop() + oprot.writeStructEnd() + + def validate(self): + return + + def __repr__(self): + L = ["{}={!r}".format(key, value) for key, value in self.__dict__.items()] + return "{}({})".format(self.__class__.__name__, ", ".join(L)) + + def __eq__(self, other): + return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ + + def __ne__(self, other): + return not (self == other) + + +all_structs.append(rename_partition_req_args) +rename_partition_req_args.thrift_spec = ( + None, # 0 + ( + 1, + TType.STRUCT, + "req", + [RenamePartitionRequest, None], + None, + ), # 1 +) + + +class rename_partition_req_result: + """ + Attributes: + - success + - o1 + - o2 + + """ + + def __init__( + self, + success=None, + o1=None, + o2=None, + ): + self.success = success + self.o1 = o1 + self.o2 = o2 + + def read(self, iprot): + if ( + iprot._fast_decode is not None + and isinstance(iprot.trans, TTransport.CReadableTransport) + and self.thrift_spec is not None + ): + iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec]) + return + iprot.readStructBegin() + while True: + (fname, ftype, fid) = iprot.readFieldBegin() + if ftype == TType.STOP: + break + if fid == 0: + if ftype == TType.STRUCT: + self.success = RenamePartitionResponse() + self.success.read(iprot) + else: + iprot.skip(ftype) + elif fid == 1: + if ftype == TType.STRUCT: + self.o1 = InvalidOperationException.read(iprot) + else: + iprot.skip(ftype) + elif fid == 2: + if ftype == TType.STRUCT: + self.o2 = MetaException.read(iprot) + else: + iprot.skip(ftype) + else: + iprot.skip(ftype) + iprot.readFieldEnd() + iprot.readStructEnd() + + def write(self, oprot): + if oprot._fast_encode is not None and self.thrift_spec is not None: + oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec])) + return + oprot.writeStructBegin("rename_partition_req_result") + if self.success is not None: + oprot.writeFieldBegin("success", TType.STRUCT, 0) + self.success.write(oprot) + oprot.writeFieldEnd() + if self.o1 is not None: + oprot.writeFieldBegin("o1", TType.STRUCT, 1) + self.o1.write(oprot) + oprot.writeFieldEnd() + if self.o2 is not None: + oprot.writeFieldBegin("o2", TType.STRUCT, 2) + self.o2.write(oprot) + oprot.writeFieldEnd() + oprot.writeFieldStop() + oprot.writeStructEnd() + + def validate(self): + return + + def __repr__(self): + L = ["{}={!r}".format(key, value) for key, value in self.__dict__.items()] + return "{}({})".format(self.__class__.__name__, ", ".join(L)) + + def __eq__(self, other): + return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ + + def __ne__(self, other): + return not (self == other) + + +all_structs.append(rename_partition_req_result) +rename_partition_req_result.thrift_spec = ( + ( + 0, + TType.STRUCT, + "success", + [RenamePartitionResponse, None], + None, + ), # 0 + ( + 1, + TType.STRUCT, + "o1", + [InvalidOperationException, None], + None, + ), # 1 + ( + 2, + TType.STRUCT, + "o2", + [MetaException, None], + None, + ), # 2 +) + + +class partition_name_has_valid_characters_args: + """ + Attributes: + - part_vals + - throw_exception + + """ + + def __init__( + self, + part_vals=None, + throw_exception=None, + ): + self.part_vals = part_vals + self.throw_exception = throw_exception + + def read(self, iprot): + if ( + iprot._fast_decode is not None + and isinstance(iprot.trans, TTransport.CReadableTransport) + and self.thrift_spec is not None + ): + iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec]) + return + iprot.readStructBegin() + while True: + (fname, ftype, fid) = iprot.readFieldBegin() + if ftype == TType.STOP: + break + if fid == 1: + if ftype == TType.LIST: + self.part_vals = [] + (_etype1664, _size1661) = iprot.readListBegin() + for _i1665 in range(_size1661): + _elem1666 = ( + iprot.readString().decode("utf-8", errors="replace") + if sys.version_info[0] == 2 + else iprot.readString() + ) + self.part_vals.append(_elem1666) + iprot.readListEnd() + else: + iprot.skip(ftype) + elif fid == 2: + if ftype == TType.BOOL: + self.throw_exception = iprot.readBool() + else: + iprot.skip(ftype) + else: + iprot.skip(ftype) + iprot.readFieldEnd() + iprot.readStructEnd() + + def write(self, oprot): + if oprot._fast_encode is not None and self.thrift_spec is not None: + oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec])) + return + oprot.writeStructBegin("partition_name_has_valid_characters_args") + if self.part_vals is not None: + oprot.writeFieldBegin("part_vals", TType.LIST, 1) + oprot.writeListBegin(TType.STRING, len(self.part_vals)) + for iter1667 in self.part_vals: + oprot.writeString(iter1667.encode("utf-8") if sys.version_info[0] == 2 else iter1667) + oprot.writeListEnd() + oprot.writeFieldEnd() + if self.throw_exception is not None: + oprot.writeFieldBegin("throw_exception", TType.BOOL, 2) + oprot.writeBool(self.throw_exception) + oprot.writeFieldEnd() + oprot.writeFieldStop() + oprot.writeStructEnd() + + def validate(self): + return + + def __repr__(self): + L = ["{}={!r}".format(key, value) for key, value in self.__dict__.items()] + return "{}({})".format(self.__class__.__name__, ", ".join(L)) + + def __eq__(self, other): + return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ + + def __ne__(self, other): + return not (self == other) + + +all_structs.append(partition_name_has_valid_characters_args) +partition_name_has_valid_characters_args.thrift_spec = ( + None, # 0 + ( + 1, + TType.LIST, + "part_vals", + (TType.STRING, "UTF8", False), + None, + ), # 1 + ( + 2, + TType.BOOL, + "throw_exception", + None, + None, + ), # 2 +) + + +class partition_name_has_valid_characters_result: + """ + Attributes: + - success + - o1 + + """ + + def __init__( + self, + success=None, + o1=None, + ): + self.success = success + self.o1 = o1 + + def read(self, iprot): + if ( + iprot._fast_decode is not None + and isinstance(iprot.trans, TTransport.CReadableTransport) + and self.thrift_spec is not None + ): + iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec]) + return + iprot.readStructBegin() + while True: + (fname, ftype, fid) = iprot.readFieldBegin() + if ftype == TType.STOP: + break + if fid == 0: + if ftype == TType.BOOL: + self.success = iprot.readBool() + else: + iprot.skip(ftype) + elif fid == 1: + if ftype == TType.STRUCT: + self.o1 = MetaException.read(iprot) + else: + iprot.skip(ftype) + else: + iprot.skip(ftype) + iprot.readFieldEnd() + iprot.readStructEnd() + + def write(self, oprot): + if oprot._fast_encode is not None and self.thrift_spec is not None: + oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec])) + return + oprot.writeStructBegin("partition_name_has_valid_characters_result") + if self.success is not None: + oprot.writeFieldBegin("success", TType.BOOL, 0) + oprot.writeBool(self.success) + oprot.writeFieldEnd() + if self.o1 is not None: + oprot.writeFieldBegin("o1", TType.STRUCT, 1) + self.o1.write(oprot) + oprot.writeFieldEnd() + oprot.writeFieldStop() + oprot.writeStructEnd() + + def validate(self): + return + + def __repr__(self): + L = ["{}={!r}".format(key, value) for key, value in self.__dict__.items()] + return "{}({})".format(self.__class__.__name__, ", ".join(L)) + + def __eq__(self, other): + return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ + + def __ne__(self, other): + return not (self == other) + + +all_structs.append(partition_name_has_valid_characters_result) +partition_name_has_valid_characters_result.thrift_spec = ( + ( + 0, + TType.BOOL, + "success", + None, + None, + ), # 0 + ( + 1, + TType.STRUCT, + "o1", + [MetaException, None], + None, + ), # 1 +) + + +class get_config_value_args: + """ + Attributes: + - name + - defaultValue + + """ + + def __init__( + self, + name=None, + defaultValue=None, + ): + self.name = name + self.defaultValue = defaultValue + + def read(self, iprot): + if ( + iprot._fast_decode is not None + and isinstance(iprot.trans, TTransport.CReadableTransport) + and self.thrift_spec is not None + ): + iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec]) + return + iprot.readStructBegin() + while True: + (fname, ftype, fid) = iprot.readFieldBegin() + if ftype == TType.STOP: + break + if fid == 1: + if ftype == TType.STRING: + self.name = ( + iprot.readString().decode("utf-8", errors="replace") if sys.version_info[0] == 2 else iprot.readString() + ) + else: + iprot.skip(ftype) + elif fid == 2: + if ftype == TType.STRING: + self.defaultValue = ( + iprot.readString().decode("utf-8", errors="replace") if sys.version_info[0] == 2 else iprot.readString() + ) + else: + iprot.skip(ftype) + else: + iprot.skip(ftype) + iprot.readFieldEnd() + iprot.readStructEnd() + + def write(self, oprot): + if oprot._fast_encode is not None and self.thrift_spec is not None: + oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec])) + return + oprot.writeStructBegin("get_config_value_args") + if self.name is not None: + oprot.writeFieldBegin("name", TType.STRING, 1) + oprot.writeString(self.name.encode("utf-8") if sys.version_info[0] == 2 else self.name) + oprot.writeFieldEnd() + if self.defaultValue is not None: + oprot.writeFieldBegin("defaultValue", TType.STRING, 2) + oprot.writeString(self.defaultValue.encode("utf-8") if sys.version_info[0] == 2 else self.defaultValue) + oprot.writeFieldEnd() + oprot.writeFieldStop() + oprot.writeStructEnd() + + def validate(self): + return + + def __repr__(self): + L = ["{}={!r}".format(key, value) for key, value in self.__dict__.items()] + return "{}({})".format(self.__class__.__name__, ", ".join(L)) + + def __eq__(self, other): + return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ + + def __ne__(self, other): + return not (self == other) + + +all_structs.append(get_config_value_args) +get_config_value_args.thrift_spec = ( + None, # 0 + ( + 1, + TType.STRING, + "name", + "UTF8", + None, + ), # 1 + ( + 2, + TType.STRING, + "defaultValue", + "UTF8", + None, + ), # 2 +) + + +class get_config_value_result: + """ + Attributes: + - success + - o1 + + """ + + def __init__( + self, + success=None, + o1=None, + ): + self.success = success + self.o1 = o1 + + def read(self, iprot): + if ( + iprot._fast_decode is not None + and isinstance(iprot.trans, TTransport.CReadableTransport) + and self.thrift_spec is not None + ): + iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec]) + return + iprot.readStructBegin() + while True: + (fname, ftype, fid) = iprot.readFieldBegin() + if ftype == TType.STOP: + break + if fid == 0: + if ftype == TType.STRING: + self.success = ( + iprot.readString().decode("utf-8", errors="replace") if sys.version_info[0] == 2 else iprot.readString() + ) + else: + iprot.skip(ftype) + elif fid == 1: + if ftype == TType.STRUCT: + self.o1 = ConfigValSecurityException.read(iprot) + else: + iprot.skip(ftype) + else: + iprot.skip(ftype) + iprot.readFieldEnd() + iprot.readStructEnd() + + def write(self, oprot): + if oprot._fast_encode is not None and self.thrift_spec is not None: + oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec])) + return + oprot.writeStructBegin("get_config_value_result") + if self.success is not None: + oprot.writeFieldBegin("success", TType.STRING, 0) + oprot.writeString(self.success.encode("utf-8") if sys.version_info[0] == 2 else self.success) + oprot.writeFieldEnd() + if self.o1 is not None: + oprot.writeFieldBegin("o1", TType.STRUCT, 1) + self.o1.write(oprot) + oprot.writeFieldEnd() + oprot.writeFieldStop() + oprot.writeStructEnd() + + def validate(self): + return + + def __repr__(self): + L = ["{}={!r}".format(key, value) for key, value in self.__dict__.items()] + return "{}({})".format(self.__class__.__name__, ", ".join(L)) + + def __eq__(self, other): + return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ + + def __ne__(self, other): + return not (self == other) + + +all_structs.append(get_config_value_result) +get_config_value_result.thrift_spec = ( + ( + 0, + TType.STRING, + "success", + "UTF8", + None, + ), # 0 + ( + 1, + TType.STRUCT, + "o1", + [ConfigValSecurityException, None], + None, + ), # 1 +) + + +class partition_name_to_vals_args: + """ + Attributes: + - part_name + + """ + + def __init__( + self, + part_name=None, + ): + self.part_name = part_name + + def read(self, iprot): + if ( + iprot._fast_decode is not None + and isinstance(iprot.trans, TTransport.CReadableTransport) + and self.thrift_spec is not None + ): + iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec]) + return + iprot.readStructBegin() + while True: + (fname, ftype, fid) = iprot.readFieldBegin() + if ftype == TType.STOP: + break + if fid == 1: + if ftype == TType.STRING: + self.part_name = ( + iprot.readString().decode("utf-8", errors="replace") if sys.version_info[0] == 2 else iprot.readString() + ) + else: + iprot.skip(ftype) + else: + iprot.skip(ftype) + iprot.readFieldEnd() + iprot.readStructEnd() + + def write(self, oprot): + if oprot._fast_encode is not None and self.thrift_spec is not None: + oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec])) + return + oprot.writeStructBegin("partition_name_to_vals_args") + if self.part_name is not None: + oprot.writeFieldBegin("part_name", TType.STRING, 1) + oprot.writeString(self.part_name.encode("utf-8") if sys.version_info[0] == 2 else self.part_name) + oprot.writeFieldEnd() + oprot.writeFieldStop() + oprot.writeStructEnd() + + def validate(self): + return + + def __repr__(self): + L = ["{}={!r}".format(key, value) for key, value in self.__dict__.items()] + return "{}({})".format(self.__class__.__name__, ", ".join(L)) + + def __eq__(self, other): + return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ + + def __ne__(self, other): + return not (self == other) + + +all_structs.append(partition_name_to_vals_args) +partition_name_to_vals_args.thrift_spec = ( + None, # 0 + ( + 1, + TType.STRING, + "part_name", + "UTF8", + None, + ), # 1 +) + + +class partition_name_to_vals_result: + """ + Attributes: + - success + - o1 + + """ + + def __init__( + self, + success=None, + o1=None, + ): + self.success = success + self.o1 = o1 + + def read(self, iprot): + if ( + iprot._fast_decode is not None + and isinstance(iprot.trans, TTransport.CReadableTransport) + and self.thrift_spec is not None + ): + iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec]) + return + iprot.readStructBegin() + while True: + (fname, ftype, fid) = iprot.readFieldBegin() + if ftype == TType.STOP: + break + if fid == 0: + if ftype == TType.LIST: + self.success = [] + (_etype1671, _size1668) = iprot.readListBegin() + for _i1672 in range(_size1668): + _elem1673 = ( + iprot.readString().decode("utf-8", errors="replace") + if sys.version_info[0] == 2 + else iprot.readString() + ) + self.success.append(_elem1673) + iprot.readListEnd() + else: + iprot.skip(ftype) + elif fid == 1: + if ftype == TType.STRUCT: + self.o1 = MetaException.read(iprot) + else: + iprot.skip(ftype) + else: + iprot.skip(ftype) + iprot.readFieldEnd() + iprot.readStructEnd() + + def write(self, oprot): + if oprot._fast_encode is not None and self.thrift_spec is not None: + oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec])) + return + oprot.writeStructBegin("partition_name_to_vals_result") + if self.success is not None: + oprot.writeFieldBegin("success", TType.LIST, 0) + oprot.writeListBegin(TType.STRING, len(self.success)) + for iter1674 in self.success: + oprot.writeString(iter1674.encode("utf-8") if sys.version_info[0] == 2 else iter1674) + oprot.writeListEnd() + oprot.writeFieldEnd() + if self.o1 is not None: + oprot.writeFieldBegin("o1", TType.STRUCT, 1) + self.o1.write(oprot) + oprot.writeFieldEnd() + oprot.writeFieldStop() + oprot.writeStructEnd() + + def validate(self): + return + + def __repr__(self): + L = ["{}={!r}".format(key, value) for key, value in self.__dict__.items()] + return "{}({})".format(self.__class__.__name__, ", ".join(L)) + + def __eq__(self, other): + return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ + + def __ne__(self, other): + return not (self == other) + + +all_structs.append(partition_name_to_vals_result) +partition_name_to_vals_result.thrift_spec = ( + ( + 0, + TType.LIST, + "success", + (TType.STRING, "UTF8", False), + None, + ), # 0 + ( + 1, + TType.STRUCT, + "o1", + [MetaException, None], + None, + ), # 1 +) + + +class partition_name_to_spec_args: + """ + Attributes: + - part_name + + """ + + def __init__( + self, + part_name=None, + ): + self.part_name = part_name + + def read(self, iprot): + if ( + iprot._fast_decode is not None + and isinstance(iprot.trans, TTransport.CReadableTransport) + and self.thrift_spec is not None + ): + iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec]) + return + iprot.readStructBegin() + while True: + (fname, ftype, fid) = iprot.readFieldBegin() + if ftype == TType.STOP: + break + if fid == 1: + if ftype == TType.STRING: + self.part_name = ( + iprot.readString().decode("utf-8", errors="replace") if sys.version_info[0] == 2 else iprot.readString() + ) + else: + iprot.skip(ftype) + else: + iprot.skip(ftype) + iprot.readFieldEnd() + iprot.readStructEnd() + + def write(self, oprot): + if oprot._fast_encode is not None and self.thrift_spec is not None: + oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec])) + return + oprot.writeStructBegin("partition_name_to_spec_args") + if self.part_name is not None: + oprot.writeFieldBegin("part_name", TType.STRING, 1) + oprot.writeString(self.part_name.encode("utf-8") if sys.version_info[0] == 2 else self.part_name) + oprot.writeFieldEnd() + oprot.writeFieldStop() + oprot.writeStructEnd() + + def validate(self): + return + + def __repr__(self): + L = ["{}={!r}".format(key, value) for key, value in self.__dict__.items()] + return "{}({})".format(self.__class__.__name__, ", ".join(L)) + + def __eq__(self, other): + return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ + + def __ne__(self, other): + return not (self == other) + + +all_structs.append(partition_name_to_spec_args) +partition_name_to_spec_args.thrift_spec = ( + None, # 0 + ( + 1, + TType.STRING, + "part_name", + "UTF8", + None, + ), # 1 +) + + +class partition_name_to_spec_result: + """ + Attributes: + - success + - o1 + + """ + + def __init__( + self, + success=None, + o1=None, + ): + self.success = success + self.o1 = o1 + + def read(self, iprot): + if ( + iprot._fast_decode is not None + and isinstance(iprot.trans, TTransport.CReadableTransport) + and self.thrift_spec is not None + ): + iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec]) + return + iprot.readStructBegin() + while True: + (fname, ftype, fid) = iprot.readFieldBegin() + if ftype == TType.STOP: + break + if fid == 0: + if ftype == TType.MAP: + self.success = {} + (_ktype1676, _vtype1677, _size1675) = iprot.readMapBegin() + for _i1679 in range(_size1675): + _key1680 = ( + iprot.readString().decode("utf-8", errors="replace") + if sys.version_info[0] == 2 + else iprot.readString() + ) + _val1681 = ( + iprot.readString().decode("utf-8", errors="replace") + if sys.version_info[0] == 2 + else iprot.readString() + ) + self.success[_key1680] = _val1681 + iprot.readMapEnd() + else: + iprot.skip(ftype) + elif fid == 1: + if ftype == TType.STRUCT: + self.o1 = MetaException.read(iprot) + else: + iprot.skip(ftype) + else: + iprot.skip(ftype) + iprot.readFieldEnd() + iprot.readStructEnd() + + def write(self, oprot): + if oprot._fast_encode is not None and self.thrift_spec is not None: + oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec])) + return + oprot.writeStructBegin("partition_name_to_spec_result") + if self.success is not None: + oprot.writeFieldBegin("success", TType.MAP, 0) + oprot.writeMapBegin(TType.STRING, TType.STRING, len(self.success)) + for kiter1682, viter1683 in self.success.items(): + oprot.writeString(kiter1682.encode("utf-8") if sys.version_info[0] == 2 else kiter1682) + oprot.writeString(viter1683.encode("utf-8") if sys.version_info[0] == 2 else viter1683) + oprot.writeMapEnd() + oprot.writeFieldEnd() + if self.o1 is not None: + oprot.writeFieldBegin("o1", TType.STRUCT, 1) + self.o1.write(oprot) + oprot.writeFieldEnd() + oprot.writeFieldStop() + oprot.writeStructEnd() + + def validate(self): + return + + def __repr__(self): + L = ["{}={!r}".format(key, value) for key, value in self.__dict__.items()] + return "{}({})".format(self.__class__.__name__, ", ".join(L)) + + def __eq__(self, other): + return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ + + def __ne__(self, other): + return not (self == other) + + +all_structs.append(partition_name_to_spec_result) +partition_name_to_spec_result.thrift_spec = ( + ( + 0, + TType.MAP, + "success", + (TType.STRING, "UTF8", TType.STRING, "UTF8", False), + None, + ), # 0 + ( + 1, + TType.STRUCT, + "o1", + [MetaException, None], + None, + ), # 1 +) + + +class markPartitionForEvent_args: + """ + Attributes: + - db_name + - tbl_name + - part_vals + - eventType + + """ + + def __init__( + self, + db_name=None, + tbl_name=None, + part_vals=None, + eventType=None, + ): + self.db_name = db_name + self.tbl_name = tbl_name + self.part_vals = part_vals + self.eventType = eventType + + def read(self, iprot): + if ( + iprot._fast_decode is not None + and isinstance(iprot.trans, TTransport.CReadableTransport) + and self.thrift_spec is not None + ): + iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec]) + return + iprot.readStructBegin() + while True: + (fname, ftype, fid) = iprot.readFieldBegin() + if ftype == TType.STOP: + break + if fid == 1: + if ftype == TType.STRING: + self.db_name = ( + iprot.readString().decode("utf-8", errors="replace") if sys.version_info[0] == 2 else iprot.readString() + ) + else: + iprot.skip(ftype) + elif fid == 2: + if ftype == TType.STRING: + self.tbl_name = ( + iprot.readString().decode("utf-8", errors="replace") if sys.version_info[0] == 2 else iprot.readString() + ) + else: + iprot.skip(ftype) + elif fid == 3: + if ftype == TType.MAP: + self.part_vals = {} + (_ktype1685, _vtype1686, _size1684) = iprot.readMapBegin() + for _i1688 in range(_size1684): + _key1689 = ( + iprot.readString().decode("utf-8", errors="replace") + if sys.version_info[0] == 2 + else iprot.readString() + ) + _val1690 = ( + iprot.readString().decode("utf-8", errors="replace") + if sys.version_info[0] == 2 + else iprot.readString() + ) + self.part_vals[_key1689] = _val1690 + iprot.readMapEnd() + else: + iprot.skip(ftype) + elif fid == 4: + if ftype == TType.I32: + self.eventType = iprot.readI32() + else: + iprot.skip(ftype) + else: + iprot.skip(ftype) + iprot.readFieldEnd() + iprot.readStructEnd() + + def write(self, oprot): + if oprot._fast_encode is not None and self.thrift_spec is not None: + oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec])) + return + oprot.writeStructBegin("markPartitionForEvent_args") + if self.db_name is not None: + oprot.writeFieldBegin("db_name", TType.STRING, 1) + oprot.writeString(self.db_name.encode("utf-8") if sys.version_info[0] == 2 else self.db_name) + oprot.writeFieldEnd() + if self.tbl_name is not None: + oprot.writeFieldBegin("tbl_name", TType.STRING, 2) + oprot.writeString(self.tbl_name.encode("utf-8") if sys.version_info[0] == 2 else self.tbl_name) + oprot.writeFieldEnd() + if self.part_vals is not None: + oprot.writeFieldBegin("part_vals", TType.MAP, 3) + oprot.writeMapBegin(TType.STRING, TType.STRING, len(self.part_vals)) + for kiter1691, viter1692 in self.part_vals.items(): + oprot.writeString(kiter1691.encode("utf-8") if sys.version_info[0] == 2 else kiter1691) + oprot.writeString(viter1692.encode("utf-8") if sys.version_info[0] == 2 else viter1692) + oprot.writeMapEnd() + oprot.writeFieldEnd() + if self.eventType is not None: + oprot.writeFieldBegin("eventType", TType.I32, 4) + oprot.writeI32(self.eventType) + oprot.writeFieldEnd() + oprot.writeFieldStop() + oprot.writeStructEnd() + + def validate(self): + return + + def __repr__(self): + L = ["{}={!r}".format(key, value) for key, value in self.__dict__.items()] + return "{}({})".format(self.__class__.__name__, ", ".join(L)) + + def __eq__(self, other): + return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ + + def __ne__(self, other): + return not (self == other) + + +all_structs.append(markPartitionForEvent_args) +markPartitionForEvent_args.thrift_spec = ( + None, # 0 + ( + 1, + TType.STRING, + "db_name", + "UTF8", + None, + ), # 1 + ( + 2, + TType.STRING, + "tbl_name", + "UTF8", + None, + ), # 2 + ( + 3, + TType.MAP, + "part_vals", + (TType.STRING, "UTF8", TType.STRING, "UTF8", False), + None, + ), # 3 + ( + 4, + TType.I32, + "eventType", + None, + None, + ), # 4 +) + + +class markPartitionForEvent_result: + """ + Attributes: + - o1 + - o2 + - o3 + - o4 + - o5 + - o6 + + """ + + def __init__( + self, + o1=None, + o2=None, + o3=None, + o4=None, + o5=None, + o6=None, + ): + self.o1 = o1 + self.o2 = o2 + self.o3 = o3 + self.o4 = o4 + self.o5 = o5 + self.o6 = o6 + + def read(self, iprot): + if ( + iprot._fast_decode is not None + and isinstance(iprot.trans, TTransport.CReadableTransport) + and self.thrift_spec is not None + ): + iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec]) + return + iprot.readStructBegin() + while True: + (fname, ftype, fid) = iprot.readFieldBegin() + if ftype == TType.STOP: + break + if fid == 1: + if ftype == TType.STRUCT: + self.o1 = MetaException.read(iprot) + else: + iprot.skip(ftype) + elif fid == 2: + if ftype == TType.STRUCT: + self.o2 = NoSuchObjectException.read(iprot) + else: + iprot.skip(ftype) + elif fid == 3: + if ftype == TType.STRUCT: + self.o3 = UnknownDBException.read(iprot) + else: + iprot.skip(ftype) + elif fid == 4: + if ftype == TType.STRUCT: + self.o4 = UnknownTableException.read(iprot) + else: + iprot.skip(ftype) + elif fid == 5: + if ftype == TType.STRUCT: + self.o5 = UnknownPartitionException.read(iprot) + else: + iprot.skip(ftype) + elif fid == 6: + if ftype == TType.STRUCT: + self.o6 = InvalidPartitionException.read(iprot) + else: + iprot.skip(ftype) + else: + iprot.skip(ftype) + iprot.readFieldEnd() + iprot.readStructEnd() + + def write(self, oprot): + if oprot._fast_encode is not None and self.thrift_spec is not None: + oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec])) + return + oprot.writeStructBegin("markPartitionForEvent_result") + if self.o1 is not None: + oprot.writeFieldBegin("o1", TType.STRUCT, 1) + self.o1.write(oprot) + oprot.writeFieldEnd() + if self.o2 is not None: + oprot.writeFieldBegin("o2", TType.STRUCT, 2) + self.o2.write(oprot) + oprot.writeFieldEnd() + if self.o3 is not None: + oprot.writeFieldBegin("o3", TType.STRUCT, 3) + self.o3.write(oprot) + oprot.writeFieldEnd() + if self.o4 is not None: + oprot.writeFieldBegin("o4", TType.STRUCT, 4) + self.o4.write(oprot) + oprot.writeFieldEnd() + if self.o5 is not None: + oprot.writeFieldBegin("o5", TType.STRUCT, 5) + self.o5.write(oprot) + oprot.writeFieldEnd() + if self.o6 is not None: + oprot.writeFieldBegin("o6", TType.STRUCT, 6) + self.o6.write(oprot) + oprot.writeFieldEnd() + oprot.writeFieldStop() + oprot.writeStructEnd() + + def validate(self): + return + + def __repr__(self): + L = ["{}={!r}".format(key, value) for key, value in self.__dict__.items()] + return "{}({})".format(self.__class__.__name__, ", ".join(L)) + + def __eq__(self, other): + return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ + + def __ne__(self, other): + return not (self == other) + + +all_structs.append(markPartitionForEvent_result) +markPartitionForEvent_result.thrift_spec = ( + None, # 0 + ( + 1, + TType.STRUCT, + "o1", + [MetaException, None], + None, + ), # 1 + ( + 2, + TType.STRUCT, + "o2", + [NoSuchObjectException, None], + None, + ), # 2 + ( + 3, + TType.STRUCT, + "o3", + [UnknownDBException, None], + None, + ), # 3 + ( + 4, + TType.STRUCT, + "o4", + [UnknownTableException, None], + None, + ), # 4 + ( + 5, + TType.STRUCT, + "o5", + [UnknownPartitionException, None], + None, + ), # 5 + ( + 6, + TType.STRUCT, + "o6", + [InvalidPartitionException, None], + None, + ), # 6 +) + + +class isPartitionMarkedForEvent_args: + """ + Attributes: + - db_name + - tbl_name + - part_vals + - eventType + + """ + + def __init__( + self, + db_name=None, + tbl_name=None, + part_vals=None, + eventType=None, + ): + self.db_name = db_name + self.tbl_name = tbl_name + self.part_vals = part_vals + self.eventType = eventType + + def read(self, iprot): + if ( + iprot._fast_decode is not None + and isinstance(iprot.trans, TTransport.CReadableTransport) + and self.thrift_spec is not None + ): + iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec]) + return + iprot.readStructBegin() + while True: + (fname, ftype, fid) = iprot.readFieldBegin() + if ftype == TType.STOP: + break + if fid == 1: + if ftype == TType.STRING: + self.db_name = ( + iprot.readString().decode("utf-8", errors="replace") if sys.version_info[0] == 2 else iprot.readString() + ) + else: + iprot.skip(ftype) + elif fid == 2: + if ftype == TType.STRING: + self.tbl_name = ( + iprot.readString().decode("utf-8", errors="replace") if sys.version_info[0] == 2 else iprot.readString() + ) + else: + iprot.skip(ftype) + elif fid == 3: + if ftype == TType.MAP: + self.part_vals = {} + (_ktype1694, _vtype1695, _size1693) = iprot.readMapBegin() + for _i1697 in range(_size1693): + _key1698 = ( + iprot.readString().decode("utf-8", errors="replace") + if sys.version_info[0] == 2 + else iprot.readString() + ) + _val1699 = ( + iprot.readString().decode("utf-8", errors="replace") + if sys.version_info[0] == 2 + else iprot.readString() + ) + self.part_vals[_key1698] = _val1699 + iprot.readMapEnd() + else: + iprot.skip(ftype) + elif fid == 4: + if ftype == TType.I32: + self.eventType = iprot.readI32() + else: + iprot.skip(ftype) + else: + iprot.skip(ftype) + iprot.readFieldEnd() + iprot.readStructEnd() + + def write(self, oprot): + if oprot._fast_encode is not None and self.thrift_spec is not None: + oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec])) + return + oprot.writeStructBegin("isPartitionMarkedForEvent_args") + if self.db_name is not None: + oprot.writeFieldBegin("db_name", TType.STRING, 1) + oprot.writeString(self.db_name.encode("utf-8") if sys.version_info[0] == 2 else self.db_name) + oprot.writeFieldEnd() + if self.tbl_name is not None: + oprot.writeFieldBegin("tbl_name", TType.STRING, 2) + oprot.writeString(self.tbl_name.encode("utf-8") if sys.version_info[0] == 2 else self.tbl_name) + oprot.writeFieldEnd() + if self.part_vals is not None: + oprot.writeFieldBegin("part_vals", TType.MAP, 3) + oprot.writeMapBegin(TType.STRING, TType.STRING, len(self.part_vals)) + for kiter1700, viter1701 in self.part_vals.items(): + oprot.writeString(kiter1700.encode("utf-8") if sys.version_info[0] == 2 else kiter1700) + oprot.writeString(viter1701.encode("utf-8") if sys.version_info[0] == 2 else viter1701) + oprot.writeMapEnd() + oprot.writeFieldEnd() + if self.eventType is not None: + oprot.writeFieldBegin("eventType", TType.I32, 4) + oprot.writeI32(self.eventType) + oprot.writeFieldEnd() + oprot.writeFieldStop() + oprot.writeStructEnd() + + def validate(self): + return + + def __repr__(self): + L = ["{}={!r}".format(key, value) for key, value in self.__dict__.items()] + return "{}({})".format(self.__class__.__name__, ", ".join(L)) + + def __eq__(self, other): + return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ + + def __ne__(self, other): + return not (self == other) + + +all_structs.append(isPartitionMarkedForEvent_args) +isPartitionMarkedForEvent_args.thrift_spec = ( + None, # 0 + ( + 1, + TType.STRING, + "db_name", + "UTF8", + None, + ), # 1 + ( + 2, + TType.STRING, + "tbl_name", + "UTF8", + None, + ), # 2 + ( + 3, + TType.MAP, + "part_vals", + (TType.STRING, "UTF8", TType.STRING, "UTF8", False), + None, + ), # 3 + ( + 4, + TType.I32, + "eventType", + None, + None, + ), # 4 +) + + +class isPartitionMarkedForEvent_result: + """ + Attributes: + - success + - o1 + - o2 + - o3 + - o4 + - o5 + - o6 + + """ + + def __init__( + self, + success=None, + o1=None, + o2=None, + o3=None, + o4=None, + o5=None, + o6=None, + ): + self.success = success + self.o1 = o1 + self.o2 = o2 + self.o3 = o3 + self.o4 = o4 + self.o5 = o5 + self.o6 = o6 + + def read(self, iprot): + if ( + iprot._fast_decode is not None + and isinstance(iprot.trans, TTransport.CReadableTransport) + and self.thrift_spec is not None + ): + iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec]) + return + iprot.readStructBegin() + while True: + (fname, ftype, fid) = iprot.readFieldBegin() + if ftype == TType.STOP: + break + if fid == 0: + if ftype == TType.BOOL: + self.success = iprot.readBool() + else: + iprot.skip(ftype) + elif fid == 1: + if ftype == TType.STRUCT: + self.o1 = MetaException.read(iprot) + else: + iprot.skip(ftype) + elif fid == 2: + if ftype == TType.STRUCT: + self.o2 = NoSuchObjectException.read(iprot) + else: + iprot.skip(ftype) + elif fid == 3: + if ftype == TType.STRUCT: + self.o3 = UnknownDBException.read(iprot) + else: + iprot.skip(ftype) + elif fid == 4: + if ftype == TType.STRUCT: + self.o4 = UnknownTableException.read(iprot) + else: + iprot.skip(ftype) + elif fid == 5: + if ftype == TType.STRUCT: + self.o5 = UnknownPartitionException.read(iprot) + else: + iprot.skip(ftype) + elif fid == 6: + if ftype == TType.STRUCT: + self.o6 = InvalidPartitionException.read(iprot) + else: + iprot.skip(ftype) + else: + iprot.skip(ftype) + iprot.readFieldEnd() + iprot.readStructEnd() + + def write(self, oprot): + if oprot._fast_encode is not None and self.thrift_spec is not None: + oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec])) + return + oprot.writeStructBegin("isPartitionMarkedForEvent_result") + if self.success is not None: + oprot.writeFieldBegin("success", TType.BOOL, 0) + oprot.writeBool(self.success) + oprot.writeFieldEnd() + if self.o1 is not None: + oprot.writeFieldBegin("o1", TType.STRUCT, 1) + self.o1.write(oprot) + oprot.writeFieldEnd() + if self.o2 is not None: + oprot.writeFieldBegin("o2", TType.STRUCT, 2) + self.o2.write(oprot) + oprot.writeFieldEnd() + if self.o3 is not None: + oprot.writeFieldBegin("o3", TType.STRUCT, 3) + self.o3.write(oprot) + oprot.writeFieldEnd() + if self.o4 is not None: + oprot.writeFieldBegin("o4", TType.STRUCT, 4) + self.o4.write(oprot) + oprot.writeFieldEnd() + if self.o5 is not None: + oprot.writeFieldBegin("o5", TType.STRUCT, 5) + self.o5.write(oprot) + oprot.writeFieldEnd() + if self.o6 is not None: + oprot.writeFieldBegin("o6", TType.STRUCT, 6) + self.o6.write(oprot) + oprot.writeFieldEnd() + oprot.writeFieldStop() + oprot.writeStructEnd() + + def validate(self): + return + + def __repr__(self): + L = ["{}={!r}".format(key, value) for key, value in self.__dict__.items()] + return "{}({})".format(self.__class__.__name__, ", ".join(L)) + + def __eq__(self, other): + return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ + + def __ne__(self, other): + return not (self == other) + + +all_structs.append(isPartitionMarkedForEvent_result) +isPartitionMarkedForEvent_result.thrift_spec = ( + ( + 0, + TType.BOOL, + "success", + None, + None, + ), # 0 + ( + 1, + TType.STRUCT, + "o1", + [MetaException, None], + None, + ), # 1 + ( + 2, + TType.STRUCT, + "o2", + [NoSuchObjectException, None], + None, + ), # 2 + ( + 3, + TType.STRUCT, + "o3", + [UnknownDBException, None], + None, + ), # 3 + ( + 4, + TType.STRUCT, + "o4", + [UnknownTableException, None], + None, + ), # 4 + ( + 5, + TType.STRUCT, + "o5", + [UnknownPartitionException, None], + None, + ), # 5 + ( + 6, + TType.STRUCT, + "o6", + [InvalidPartitionException, None], + None, + ), # 6 +) + + +class get_primary_keys_args: + """ + Attributes: + - request + + """ + + def __init__( + self, + request=None, + ): + self.request = request + + def read(self, iprot): + if ( + iprot._fast_decode is not None + and isinstance(iprot.trans, TTransport.CReadableTransport) + and self.thrift_spec is not None + ): + iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec]) + return + iprot.readStructBegin() + while True: + (fname, ftype, fid) = iprot.readFieldBegin() + if ftype == TType.STOP: + break + if fid == 1: + if ftype == TType.STRUCT: + self.request = PrimaryKeysRequest() + self.request.read(iprot) + else: + iprot.skip(ftype) + else: + iprot.skip(ftype) + iprot.readFieldEnd() + iprot.readStructEnd() + + def write(self, oprot): + if oprot._fast_encode is not None and self.thrift_spec is not None: + oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec])) + return + oprot.writeStructBegin("get_primary_keys_args") + if self.request is not None: + oprot.writeFieldBegin("request", TType.STRUCT, 1) + self.request.write(oprot) + oprot.writeFieldEnd() + oprot.writeFieldStop() + oprot.writeStructEnd() + + def validate(self): + return + + def __repr__(self): + L = ["{}={!r}".format(key, value) for key, value in self.__dict__.items()] + return "{}({})".format(self.__class__.__name__, ", ".join(L)) + + def __eq__(self, other): + return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ + + def __ne__(self, other): + return not (self == other) + + +all_structs.append(get_primary_keys_args) +get_primary_keys_args.thrift_spec = ( + None, # 0 + ( + 1, + TType.STRUCT, + "request", + [PrimaryKeysRequest, None], + None, + ), # 1 +) + + +class get_primary_keys_result: + """ + Attributes: + - success + - o1 + - o2 + + """ + + def __init__( + self, + success=None, + o1=None, + o2=None, + ): + self.success = success + self.o1 = o1 + self.o2 = o2 + + def read(self, iprot): + if ( + iprot._fast_decode is not None + and isinstance(iprot.trans, TTransport.CReadableTransport) + and self.thrift_spec is not None + ): + iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec]) + return + iprot.readStructBegin() + while True: + (fname, ftype, fid) = iprot.readFieldBegin() + if ftype == TType.STOP: + break + if fid == 0: + if ftype == TType.STRUCT: + self.success = PrimaryKeysResponse() + self.success.read(iprot) + else: + iprot.skip(ftype) + elif fid == 1: + if ftype == TType.STRUCT: + self.o1 = MetaException.read(iprot) + else: + iprot.skip(ftype) + elif fid == 2: + if ftype == TType.STRUCT: + self.o2 = NoSuchObjectException.read(iprot) + else: + iprot.skip(ftype) + else: + iprot.skip(ftype) + iprot.readFieldEnd() + iprot.readStructEnd() + + def write(self, oprot): + if oprot._fast_encode is not None and self.thrift_spec is not None: + oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec])) + return + oprot.writeStructBegin("get_primary_keys_result") + if self.success is not None: + oprot.writeFieldBegin("success", TType.STRUCT, 0) + self.success.write(oprot) + oprot.writeFieldEnd() + if self.o1 is not None: + oprot.writeFieldBegin("o1", TType.STRUCT, 1) + self.o1.write(oprot) + oprot.writeFieldEnd() + if self.o2 is not None: + oprot.writeFieldBegin("o2", TType.STRUCT, 2) + self.o2.write(oprot) + oprot.writeFieldEnd() + oprot.writeFieldStop() + oprot.writeStructEnd() + + def validate(self): + return + + def __repr__(self): + L = ["{}={!r}".format(key, value) for key, value in self.__dict__.items()] + return "{}({})".format(self.__class__.__name__, ", ".join(L)) + + def __eq__(self, other): + return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ + + def __ne__(self, other): + return not (self == other) + + +all_structs.append(get_primary_keys_result) +get_primary_keys_result.thrift_spec = ( + ( + 0, + TType.STRUCT, + "success", + [PrimaryKeysResponse, None], + None, + ), # 0 + ( + 1, + TType.STRUCT, + "o1", + [MetaException, None], + None, + ), # 1 + ( + 2, + TType.STRUCT, + "o2", + [NoSuchObjectException, None], + None, + ), # 2 +) + + +class get_foreign_keys_args: + """ + Attributes: + - request + + """ + + def __init__( + self, + request=None, + ): + self.request = request + + def read(self, iprot): + if ( + iprot._fast_decode is not None + and isinstance(iprot.trans, TTransport.CReadableTransport) + and self.thrift_spec is not None + ): + iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec]) + return + iprot.readStructBegin() + while True: + (fname, ftype, fid) = iprot.readFieldBegin() + if ftype == TType.STOP: + break + if fid == 1: + if ftype == TType.STRUCT: + self.request = ForeignKeysRequest() + self.request.read(iprot) + else: + iprot.skip(ftype) + else: + iprot.skip(ftype) + iprot.readFieldEnd() + iprot.readStructEnd() + + def write(self, oprot): + if oprot._fast_encode is not None and self.thrift_spec is not None: + oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec])) + return + oprot.writeStructBegin("get_foreign_keys_args") + if self.request is not None: + oprot.writeFieldBegin("request", TType.STRUCT, 1) + self.request.write(oprot) + oprot.writeFieldEnd() + oprot.writeFieldStop() + oprot.writeStructEnd() + + def validate(self): + return + + def __repr__(self): + L = ["{}={!r}".format(key, value) for key, value in self.__dict__.items()] + return "{}({})".format(self.__class__.__name__, ", ".join(L)) + + def __eq__(self, other): + return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ + + def __ne__(self, other): + return not (self == other) + + +all_structs.append(get_foreign_keys_args) +get_foreign_keys_args.thrift_spec = ( + None, # 0 + ( + 1, + TType.STRUCT, + "request", + [ForeignKeysRequest, None], + None, + ), # 1 +) + + +class get_foreign_keys_result: + """ + Attributes: + - success + - o1 + - o2 + + """ + + def __init__( + self, + success=None, + o1=None, + o2=None, + ): + self.success = success + self.o1 = o1 + self.o2 = o2 + + def read(self, iprot): + if ( + iprot._fast_decode is not None + and isinstance(iprot.trans, TTransport.CReadableTransport) + and self.thrift_spec is not None + ): + iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec]) + return + iprot.readStructBegin() + while True: + (fname, ftype, fid) = iprot.readFieldBegin() + if ftype == TType.STOP: + break + if fid == 0: + if ftype == TType.STRUCT: + self.success = ForeignKeysResponse() + self.success.read(iprot) + else: + iprot.skip(ftype) + elif fid == 1: + if ftype == TType.STRUCT: + self.o1 = MetaException.read(iprot) + else: + iprot.skip(ftype) + elif fid == 2: + if ftype == TType.STRUCT: + self.o2 = NoSuchObjectException.read(iprot) + else: + iprot.skip(ftype) + else: + iprot.skip(ftype) + iprot.readFieldEnd() + iprot.readStructEnd() + + def write(self, oprot): + if oprot._fast_encode is not None and self.thrift_spec is not None: + oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec])) + return + oprot.writeStructBegin("get_foreign_keys_result") + if self.success is not None: + oprot.writeFieldBegin("success", TType.STRUCT, 0) + self.success.write(oprot) + oprot.writeFieldEnd() + if self.o1 is not None: + oprot.writeFieldBegin("o1", TType.STRUCT, 1) + self.o1.write(oprot) + oprot.writeFieldEnd() + if self.o2 is not None: + oprot.writeFieldBegin("o2", TType.STRUCT, 2) + self.o2.write(oprot) + oprot.writeFieldEnd() + oprot.writeFieldStop() + oprot.writeStructEnd() + + def validate(self): + return + + def __repr__(self): + L = ["{}={!r}".format(key, value) for key, value in self.__dict__.items()] + return "{}({})".format(self.__class__.__name__, ", ".join(L)) + + def __eq__(self, other): + return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ + + def __ne__(self, other): + return not (self == other) + + +all_structs.append(get_foreign_keys_result) +get_foreign_keys_result.thrift_spec = ( + ( + 0, + TType.STRUCT, + "success", + [ForeignKeysResponse, None], + None, + ), # 0 + ( + 1, + TType.STRUCT, + "o1", + [MetaException, None], + None, + ), # 1 + ( + 2, + TType.STRUCT, + "o2", + [NoSuchObjectException, None], + None, + ), # 2 +) + + +class get_unique_constraints_args: + """ + Attributes: + - request + + """ + + def __init__( + self, + request=None, + ): + self.request = request + + def read(self, iprot): + if ( + iprot._fast_decode is not None + and isinstance(iprot.trans, TTransport.CReadableTransport) + and self.thrift_spec is not None + ): + iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec]) + return + iprot.readStructBegin() + while True: + (fname, ftype, fid) = iprot.readFieldBegin() + if ftype == TType.STOP: + break + if fid == 1: + if ftype == TType.STRUCT: + self.request = UniqueConstraintsRequest() + self.request.read(iprot) + else: + iprot.skip(ftype) + else: + iprot.skip(ftype) + iprot.readFieldEnd() + iprot.readStructEnd() + + def write(self, oprot): + if oprot._fast_encode is not None and self.thrift_spec is not None: + oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec])) + return + oprot.writeStructBegin("get_unique_constraints_args") + if self.request is not None: + oprot.writeFieldBegin("request", TType.STRUCT, 1) + self.request.write(oprot) + oprot.writeFieldEnd() + oprot.writeFieldStop() + oprot.writeStructEnd() + + def validate(self): + return + + def __repr__(self): + L = ["{}={!r}".format(key, value) for key, value in self.__dict__.items()] + return "{}({})".format(self.__class__.__name__, ", ".join(L)) + + def __eq__(self, other): + return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ + + def __ne__(self, other): + return not (self == other) + + +all_structs.append(get_unique_constraints_args) +get_unique_constraints_args.thrift_spec = ( + None, # 0 + ( + 1, + TType.STRUCT, + "request", + [UniqueConstraintsRequest, None], + None, + ), # 1 +) + + +class get_unique_constraints_result: + """ + Attributes: + - success + - o1 + - o2 + + """ + + def __init__( + self, + success=None, + o1=None, + o2=None, + ): + self.success = success + self.o1 = o1 + self.o2 = o2 + + def read(self, iprot): + if ( + iprot._fast_decode is not None + and isinstance(iprot.trans, TTransport.CReadableTransport) + and self.thrift_spec is not None + ): + iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec]) + return + iprot.readStructBegin() + while True: + (fname, ftype, fid) = iprot.readFieldBegin() + if ftype == TType.STOP: + break + if fid == 0: + if ftype == TType.STRUCT: + self.success = UniqueConstraintsResponse() + self.success.read(iprot) + else: + iprot.skip(ftype) + elif fid == 1: + if ftype == TType.STRUCT: + self.o1 = MetaException.read(iprot) + else: + iprot.skip(ftype) + elif fid == 2: + if ftype == TType.STRUCT: + self.o2 = NoSuchObjectException.read(iprot) + else: + iprot.skip(ftype) + else: + iprot.skip(ftype) + iprot.readFieldEnd() + iprot.readStructEnd() + + def write(self, oprot): + if oprot._fast_encode is not None and self.thrift_spec is not None: + oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec])) + return + oprot.writeStructBegin("get_unique_constraints_result") + if self.success is not None: + oprot.writeFieldBegin("success", TType.STRUCT, 0) + self.success.write(oprot) + oprot.writeFieldEnd() + if self.o1 is not None: + oprot.writeFieldBegin("o1", TType.STRUCT, 1) + self.o1.write(oprot) + oprot.writeFieldEnd() + if self.o2 is not None: + oprot.writeFieldBegin("o2", TType.STRUCT, 2) + self.o2.write(oprot) + oprot.writeFieldEnd() + oprot.writeFieldStop() + oprot.writeStructEnd() + + def validate(self): + return + + def __repr__(self): + L = ["{}={!r}".format(key, value) for key, value in self.__dict__.items()] + return "{}({})".format(self.__class__.__name__, ", ".join(L)) + + def __eq__(self, other): + return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ + + def __ne__(self, other): + return not (self == other) + + +all_structs.append(get_unique_constraints_result) +get_unique_constraints_result.thrift_spec = ( + ( + 0, + TType.STRUCT, + "success", + [UniqueConstraintsResponse, None], + None, + ), # 0 + ( + 1, + TType.STRUCT, + "o1", + [MetaException, None], + None, + ), # 1 + ( + 2, + TType.STRUCT, + "o2", + [NoSuchObjectException, None], + None, + ), # 2 +) + + +class get_not_null_constraints_args: + """ + Attributes: + - request + + """ + + def __init__( + self, + request=None, + ): + self.request = request + + def read(self, iprot): + if ( + iprot._fast_decode is not None + and isinstance(iprot.trans, TTransport.CReadableTransport) + and self.thrift_spec is not None + ): + iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec]) + return + iprot.readStructBegin() + while True: + (fname, ftype, fid) = iprot.readFieldBegin() + if ftype == TType.STOP: + break + if fid == 1: + if ftype == TType.STRUCT: + self.request = NotNullConstraintsRequest() + self.request.read(iprot) + else: + iprot.skip(ftype) + else: + iprot.skip(ftype) + iprot.readFieldEnd() + iprot.readStructEnd() + + def write(self, oprot): + if oprot._fast_encode is not None and self.thrift_spec is not None: + oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec])) + return + oprot.writeStructBegin("get_not_null_constraints_args") + if self.request is not None: + oprot.writeFieldBegin("request", TType.STRUCT, 1) + self.request.write(oprot) + oprot.writeFieldEnd() + oprot.writeFieldStop() + oprot.writeStructEnd() + + def validate(self): + return + + def __repr__(self): + L = ["{}={!r}".format(key, value) for key, value in self.__dict__.items()] + return "{}({})".format(self.__class__.__name__, ", ".join(L)) + + def __eq__(self, other): + return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ + + def __ne__(self, other): + return not (self == other) + + +all_structs.append(get_not_null_constraints_args) +get_not_null_constraints_args.thrift_spec = ( + None, # 0 + ( + 1, + TType.STRUCT, + "request", + [NotNullConstraintsRequest, None], + None, + ), # 1 +) + + +class get_not_null_constraints_result: + """ + Attributes: + - success + - o1 + - o2 + + """ + + def __init__( + self, + success=None, + o1=None, + o2=None, + ): + self.success = success + self.o1 = o1 + self.o2 = o2 + + def read(self, iprot): + if ( + iprot._fast_decode is not None + and isinstance(iprot.trans, TTransport.CReadableTransport) + and self.thrift_spec is not None + ): + iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec]) + return + iprot.readStructBegin() + while True: + (fname, ftype, fid) = iprot.readFieldBegin() + if ftype == TType.STOP: + break + if fid == 0: + if ftype == TType.STRUCT: + self.success = NotNullConstraintsResponse() + self.success.read(iprot) + else: + iprot.skip(ftype) + elif fid == 1: + if ftype == TType.STRUCT: + self.o1 = MetaException.read(iprot) + else: + iprot.skip(ftype) + elif fid == 2: + if ftype == TType.STRUCT: + self.o2 = NoSuchObjectException.read(iprot) + else: + iprot.skip(ftype) + else: + iprot.skip(ftype) + iprot.readFieldEnd() + iprot.readStructEnd() + + def write(self, oprot): + if oprot._fast_encode is not None and self.thrift_spec is not None: + oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec])) + return + oprot.writeStructBegin("get_not_null_constraints_result") + if self.success is not None: + oprot.writeFieldBegin("success", TType.STRUCT, 0) + self.success.write(oprot) + oprot.writeFieldEnd() + if self.o1 is not None: + oprot.writeFieldBegin("o1", TType.STRUCT, 1) + self.o1.write(oprot) + oprot.writeFieldEnd() + if self.o2 is not None: + oprot.writeFieldBegin("o2", TType.STRUCT, 2) + self.o2.write(oprot) + oprot.writeFieldEnd() + oprot.writeFieldStop() + oprot.writeStructEnd() + + def validate(self): + return + + def __repr__(self): + L = ["{}={!r}".format(key, value) for key, value in self.__dict__.items()] + return "{}({})".format(self.__class__.__name__, ", ".join(L)) + + def __eq__(self, other): + return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ + + def __ne__(self, other): + return not (self == other) + + +all_structs.append(get_not_null_constraints_result) +get_not_null_constraints_result.thrift_spec = ( + ( + 0, + TType.STRUCT, + "success", + [NotNullConstraintsResponse, None], + None, + ), # 0 + ( + 1, + TType.STRUCT, + "o1", + [MetaException, None], + None, + ), # 1 + ( + 2, + TType.STRUCT, + "o2", + [NoSuchObjectException, None], + None, + ), # 2 +) + + +class get_default_constraints_args: + """ + Attributes: + - request + + """ + + def __init__( + self, + request=None, + ): + self.request = request + + def read(self, iprot): + if ( + iprot._fast_decode is not None + and isinstance(iprot.trans, TTransport.CReadableTransport) + and self.thrift_spec is not None + ): + iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec]) + return + iprot.readStructBegin() + while True: + (fname, ftype, fid) = iprot.readFieldBegin() + if ftype == TType.STOP: + break + if fid == 1: + if ftype == TType.STRUCT: + self.request = DefaultConstraintsRequest() + self.request.read(iprot) + else: + iprot.skip(ftype) + else: + iprot.skip(ftype) + iprot.readFieldEnd() + iprot.readStructEnd() + + def write(self, oprot): + if oprot._fast_encode is not None and self.thrift_spec is not None: + oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec])) + return + oprot.writeStructBegin("get_default_constraints_args") + if self.request is not None: + oprot.writeFieldBegin("request", TType.STRUCT, 1) + self.request.write(oprot) + oprot.writeFieldEnd() + oprot.writeFieldStop() + oprot.writeStructEnd() + + def validate(self): + return + + def __repr__(self): + L = ["{}={!r}".format(key, value) for key, value in self.__dict__.items()] + return "{}({})".format(self.__class__.__name__, ", ".join(L)) + + def __eq__(self, other): + return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ + + def __ne__(self, other): + return not (self == other) + + +all_structs.append(get_default_constraints_args) +get_default_constraints_args.thrift_spec = ( + None, # 0 + ( + 1, + TType.STRUCT, + "request", + [DefaultConstraintsRequest, None], + None, + ), # 1 +) + + +class get_default_constraints_result: + """ + Attributes: + - success + - o1 + - o2 + + """ + + def __init__( + self, + success=None, + o1=None, + o2=None, + ): + self.success = success + self.o1 = o1 + self.o2 = o2 + + def read(self, iprot): + if ( + iprot._fast_decode is not None + and isinstance(iprot.trans, TTransport.CReadableTransport) + and self.thrift_spec is not None + ): + iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec]) + return + iprot.readStructBegin() + while True: + (fname, ftype, fid) = iprot.readFieldBegin() + if ftype == TType.STOP: + break + if fid == 0: + if ftype == TType.STRUCT: + self.success = DefaultConstraintsResponse() + self.success.read(iprot) + else: + iprot.skip(ftype) + elif fid == 1: + if ftype == TType.STRUCT: + self.o1 = MetaException.read(iprot) + else: + iprot.skip(ftype) + elif fid == 2: + if ftype == TType.STRUCT: + self.o2 = NoSuchObjectException.read(iprot) + else: + iprot.skip(ftype) + else: + iprot.skip(ftype) + iprot.readFieldEnd() + iprot.readStructEnd() + + def write(self, oprot): + if oprot._fast_encode is not None and self.thrift_spec is not None: + oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec])) + return + oprot.writeStructBegin("get_default_constraints_result") + if self.success is not None: + oprot.writeFieldBegin("success", TType.STRUCT, 0) + self.success.write(oprot) + oprot.writeFieldEnd() + if self.o1 is not None: + oprot.writeFieldBegin("o1", TType.STRUCT, 1) + self.o1.write(oprot) + oprot.writeFieldEnd() + if self.o2 is not None: + oprot.writeFieldBegin("o2", TType.STRUCT, 2) + self.o2.write(oprot) + oprot.writeFieldEnd() + oprot.writeFieldStop() + oprot.writeStructEnd() + + def validate(self): + return + + def __repr__(self): + L = ["{}={!r}".format(key, value) for key, value in self.__dict__.items()] + return "{}({})".format(self.__class__.__name__, ", ".join(L)) + + def __eq__(self, other): + return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ + + def __ne__(self, other): + return not (self == other) + + +all_structs.append(get_default_constraints_result) +get_default_constraints_result.thrift_spec = ( + ( + 0, + TType.STRUCT, + "success", + [DefaultConstraintsResponse, None], + None, + ), # 0 + ( + 1, + TType.STRUCT, + "o1", + [MetaException, None], + None, + ), # 1 + ( + 2, + TType.STRUCT, + "o2", + [NoSuchObjectException, None], + None, + ), # 2 +) + + +class get_check_constraints_args: + """ + Attributes: + - request + + """ + + def __init__( + self, + request=None, + ): + self.request = request + + def read(self, iprot): + if ( + iprot._fast_decode is not None + and isinstance(iprot.trans, TTransport.CReadableTransport) + and self.thrift_spec is not None + ): + iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec]) + return + iprot.readStructBegin() + while True: + (fname, ftype, fid) = iprot.readFieldBegin() + if ftype == TType.STOP: + break + if fid == 1: + if ftype == TType.STRUCT: + self.request = CheckConstraintsRequest() + self.request.read(iprot) + else: + iprot.skip(ftype) + else: + iprot.skip(ftype) + iprot.readFieldEnd() + iprot.readStructEnd() + + def write(self, oprot): + if oprot._fast_encode is not None and self.thrift_spec is not None: + oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec])) + return + oprot.writeStructBegin("get_check_constraints_args") + if self.request is not None: + oprot.writeFieldBegin("request", TType.STRUCT, 1) + self.request.write(oprot) + oprot.writeFieldEnd() + oprot.writeFieldStop() + oprot.writeStructEnd() + + def validate(self): + return + + def __repr__(self): + L = ["{}={!r}".format(key, value) for key, value in self.__dict__.items()] + return "{}({})".format(self.__class__.__name__, ", ".join(L)) + + def __eq__(self, other): + return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ + + def __ne__(self, other): + return not (self == other) + + +all_structs.append(get_check_constraints_args) +get_check_constraints_args.thrift_spec = ( + None, # 0 + ( + 1, + TType.STRUCT, + "request", + [CheckConstraintsRequest, None], + None, + ), # 1 +) + + +class get_check_constraints_result: + """ + Attributes: + - success + - o1 + - o2 + + """ + + def __init__( + self, + success=None, + o1=None, + o2=None, + ): + self.success = success + self.o1 = o1 + self.o2 = o2 + + def read(self, iprot): + if ( + iprot._fast_decode is not None + and isinstance(iprot.trans, TTransport.CReadableTransport) + and self.thrift_spec is not None + ): + iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec]) + return + iprot.readStructBegin() + while True: + (fname, ftype, fid) = iprot.readFieldBegin() + if ftype == TType.STOP: + break + if fid == 0: + if ftype == TType.STRUCT: + self.success = CheckConstraintsResponse() + self.success.read(iprot) + else: + iprot.skip(ftype) + elif fid == 1: + if ftype == TType.STRUCT: + self.o1 = MetaException.read(iprot) + else: + iprot.skip(ftype) + elif fid == 2: + if ftype == TType.STRUCT: + self.o2 = NoSuchObjectException.read(iprot) + else: + iprot.skip(ftype) + else: + iprot.skip(ftype) + iprot.readFieldEnd() + iprot.readStructEnd() + + def write(self, oprot): + if oprot._fast_encode is not None and self.thrift_spec is not None: + oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec])) + return + oprot.writeStructBegin("get_check_constraints_result") + if self.success is not None: + oprot.writeFieldBegin("success", TType.STRUCT, 0) + self.success.write(oprot) + oprot.writeFieldEnd() + if self.o1 is not None: + oprot.writeFieldBegin("o1", TType.STRUCT, 1) + self.o1.write(oprot) + oprot.writeFieldEnd() + if self.o2 is not None: + oprot.writeFieldBegin("o2", TType.STRUCT, 2) + self.o2.write(oprot) + oprot.writeFieldEnd() + oprot.writeFieldStop() + oprot.writeStructEnd() + + def validate(self): + return + + def __repr__(self): + L = ["{}={!r}".format(key, value) for key, value in self.__dict__.items()] + return "{}({})".format(self.__class__.__name__, ", ".join(L)) + + def __eq__(self, other): + return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ + + def __ne__(self, other): + return not (self == other) + + +all_structs.append(get_check_constraints_result) +get_check_constraints_result.thrift_spec = ( + ( + 0, + TType.STRUCT, + "success", + [CheckConstraintsResponse, None], + None, + ), # 0 + ( + 1, + TType.STRUCT, + "o1", + [MetaException, None], + None, + ), # 1 + ( + 2, + TType.STRUCT, + "o2", + [NoSuchObjectException, None], + None, + ), # 2 +) + + +class get_all_table_constraints_args: + """ + Attributes: + - request + + """ + + def __init__( + self, + request=None, + ): + self.request = request + + def read(self, iprot): + if ( + iprot._fast_decode is not None + and isinstance(iprot.trans, TTransport.CReadableTransport) + and self.thrift_spec is not None + ): + iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec]) + return + iprot.readStructBegin() + while True: + (fname, ftype, fid) = iprot.readFieldBegin() + if ftype == TType.STOP: + break + if fid == 1: + if ftype == TType.STRUCT: + self.request = AllTableConstraintsRequest() + self.request.read(iprot) + else: + iprot.skip(ftype) + else: + iprot.skip(ftype) + iprot.readFieldEnd() + iprot.readStructEnd() + + def write(self, oprot): + if oprot._fast_encode is not None and self.thrift_spec is not None: + oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec])) + return + oprot.writeStructBegin("get_all_table_constraints_args") + if self.request is not None: + oprot.writeFieldBegin("request", TType.STRUCT, 1) + self.request.write(oprot) + oprot.writeFieldEnd() + oprot.writeFieldStop() + oprot.writeStructEnd() + + def validate(self): + return + + def __repr__(self): + L = ["{}={!r}".format(key, value) for key, value in self.__dict__.items()] + return "{}({})".format(self.__class__.__name__, ", ".join(L)) + + def __eq__(self, other): + return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ + + def __ne__(self, other): + return not (self == other) + + +all_structs.append(get_all_table_constraints_args) +get_all_table_constraints_args.thrift_spec = ( + None, # 0 + ( + 1, + TType.STRUCT, + "request", + [AllTableConstraintsRequest, None], + None, + ), # 1 +) + + +class get_all_table_constraints_result: + """ + Attributes: + - success + - o1 + - o2 + + """ + + def __init__( + self, + success=None, + o1=None, + o2=None, + ): + self.success = success + self.o1 = o1 + self.o2 = o2 + + def read(self, iprot): + if ( + iprot._fast_decode is not None + and isinstance(iprot.trans, TTransport.CReadableTransport) + and self.thrift_spec is not None + ): + iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec]) + return + iprot.readStructBegin() + while True: + (fname, ftype, fid) = iprot.readFieldBegin() + if ftype == TType.STOP: + break + if fid == 0: + if ftype == TType.STRUCT: + self.success = AllTableConstraintsResponse() + self.success.read(iprot) + else: + iprot.skip(ftype) + elif fid == 1: + if ftype == TType.STRUCT: + self.o1 = MetaException.read(iprot) + else: + iprot.skip(ftype) + elif fid == 2: + if ftype == TType.STRUCT: + self.o2 = NoSuchObjectException.read(iprot) + else: + iprot.skip(ftype) + else: + iprot.skip(ftype) + iprot.readFieldEnd() + iprot.readStructEnd() + + def write(self, oprot): + if oprot._fast_encode is not None and self.thrift_spec is not None: + oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec])) + return + oprot.writeStructBegin("get_all_table_constraints_result") + if self.success is not None: + oprot.writeFieldBegin("success", TType.STRUCT, 0) + self.success.write(oprot) + oprot.writeFieldEnd() + if self.o1 is not None: + oprot.writeFieldBegin("o1", TType.STRUCT, 1) + self.o1.write(oprot) + oprot.writeFieldEnd() + if self.o2 is not None: + oprot.writeFieldBegin("o2", TType.STRUCT, 2) + self.o2.write(oprot) + oprot.writeFieldEnd() + oprot.writeFieldStop() + oprot.writeStructEnd() + + def validate(self): + return + + def __repr__(self): + L = ["{}={!r}".format(key, value) for key, value in self.__dict__.items()] + return "{}({})".format(self.__class__.__name__, ", ".join(L)) + + def __eq__(self, other): + return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ + + def __ne__(self, other): + return not (self == other) + + +all_structs.append(get_all_table_constraints_result) +get_all_table_constraints_result.thrift_spec = ( + ( + 0, + TType.STRUCT, + "success", + [AllTableConstraintsResponse, None], + None, + ), # 0 + ( + 1, + TType.STRUCT, + "o1", + [MetaException, None], + None, + ), # 1 + ( + 2, + TType.STRUCT, + "o2", + [NoSuchObjectException, None], + None, + ), # 2 +) + + +class update_table_column_statistics_args: + """ + Attributes: + - stats_obj + + """ + + def __init__( + self, + stats_obj=None, + ): + self.stats_obj = stats_obj + + def read(self, iprot): + if ( + iprot._fast_decode is not None + and isinstance(iprot.trans, TTransport.CReadableTransport) + and self.thrift_spec is not None + ): + iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec]) + return + iprot.readStructBegin() + while True: + (fname, ftype, fid) = iprot.readFieldBegin() + if ftype == TType.STOP: + break + if fid == 1: + if ftype == TType.STRUCT: + self.stats_obj = ColumnStatistics() + self.stats_obj.read(iprot) + else: + iprot.skip(ftype) + else: + iprot.skip(ftype) + iprot.readFieldEnd() + iprot.readStructEnd() + + def write(self, oprot): + if oprot._fast_encode is not None and self.thrift_spec is not None: + oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec])) + return + oprot.writeStructBegin("update_table_column_statistics_args") + if self.stats_obj is not None: + oprot.writeFieldBegin("stats_obj", TType.STRUCT, 1) + self.stats_obj.write(oprot) + oprot.writeFieldEnd() + oprot.writeFieldStop() + oprot.writeStructEnd() + + def validate(self): + return + + def __repr__(self): + L = ["{}={!r}".format(key, value) for key, value in self.__dict__.items()] + return "{}({})".format(self.__class__.__name__, ", ".join(L)) + + def __eq__(self, other): + return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ + + def __ne__(self, other): + return not (self == other) + + +all_structs.append(update_table_column_statistics_args) +update_table_column_statistics_args.thrift_spec = ( + None, # 0 + ( + 1, + TType.STRUCT, + "stats_obj", + [ColumnStatistics, None], + None, + ), # 1 +) + + +class update_table_column_statistics_result: + """ + Attributes: + - success + - o1 + - o2 + - o3 + - o4 + + """ + + def __init__( + self, + success=None, + o1=None, + o2=None, + o3=None, + o4=None, + ): + self.success = success + self.o1 = o1 + self.o2 = o2 + self.o3 = o3 + self.o4 = o4 + + def read(self, iprot): + if ( + iprot._fast_decode is not None + and isinstance(iprot.trans, TTransport.CReadableTransport) + and self.thrift_spec is not None + ): + iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec]) + return + iprot.readStructBegin() + while True: + (fname, ftype, fid) = iprot.readFieldBegin() + if ftype == TType.STOP: + break + if fid == 0: + if ftype == TType.BOOL: + self.success = iprot.readBool() + else: + iprot.skip(ftype) + elif fid == 1: + if ftype == TType.STRUCT: + self.o1 = NoSuchObjectException.read(iprot) + else: + iprot.skip(ftype) + elif fid == 2: + if ftype == TType.STRUCT: + self.o2 = InvalidObjectException.read(iprot) + else: + iprot.skip(ftype) + elif fid == 3: + if ftype == TType.STRUCT: + self.o3 = MetaException.read(iprot) + else: + iprot.skip(ftype) + elif fid == 4: + if ftype == TType.STRUCT: + self.o4 = InvalidInputException.read(iprot) + else: + iprot.skip(ftype) + else: + iprot.skip(ftype) + iprot.readFieldEnd() + iprot.readStructEnd() + + def write(self, oprot): + if oprot._fast_encode is not None and self.thrift_spec is not None: + oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec])) + return + oprot.writeStructBegin("update_table_column_statistics_result") + if self.success is not None: + oprot.writeFieldBegin("success", TType.BOOL, 0) + oprot.writeBool(self.success) + oprot.writeFieldEnd() + if self.o1 is not None: + oprot.writeFieldBegin("o1", TType.STRUCT, 1) + self.o1.write(oprot) + oprot.writeFieldEnd() + if self.o2 is not None: + oprot.writeFieldBegin("o2", TType.STRUCT, 2) + self.o2.write(oprot) + oprot.writeFieldEnd() + if self.o3 is not None: + oprot.writeFieldBegin("o3", TType.STRUCT, 3) + self.o3.write(oprot) + oprot.writeFieldEnd() + if self.o4 is not None: + oprot.writeFieldBegin("o4", TType.STRUCT, 4) + self.o4.write(oprot) + oprot.writeFieldEnd() + oprot.writeFieldStop() + oprot.writeStructEnd() + + def validate(self): + return + + def __repr__(self): + L = ["{}={!r}".format(key, value) for key, value in self.__dict__.items()] + return "{}({})".format(self.__class__.__name__, ", ".join(L)) + + def __eq__(self, other): + return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ + + def __ne__(self, other): + return not (self == other) + + +all_structs.append(update_table_column_statistics_result) +update_table_column_statistics_result.thrift_spec = ( + ( + 0, + TType.BOOL, + "success", + None, + None, + ), # 0 + ( + 1, + TType.STRUCT, + "o1", + [NoSuchObjectException, None], + None, + ), # 1 + ( + 2, + TType.STRUCT, + "o2", + [InvalidObjectException, None], + None, + ), # 2 + ( + 3, + TType.STRUCT, + "o3", + [MetaException, None], + None, + ), # 3 + ( + 4, + TType.STRUCT, + "o4", + [InvalidInputException, None], + None, + ), # 4 +) + + +class update_partition_column_statistics_args: + """ + Attributes: + - stats_obj + + """ + + def __init__( + self, + stats_obj=None, + ): + self.stats_obj = stats_obj + + def read(self, iprot): + if ( + iprot._fast_decode is not None + and isinstance(iprot.trans, TTransport.CReadableTransport) + and self.thrift_spec is not None + ): + iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec]) + return + iprot.readStructBegin() + while True: + (fname, ftype, fid) = iprot.readFieldBegin() + if ftype == TType.STOP: + break + if fid == 1: + if ftype == TType.STRUCT: + self.stats_obj = ColumnStatistics() + self.stats_obj.read(iprot) + else: + iprot.skip(ftype) + else: + iprot.skip(ftype) + iprot.readFieldEnd() + iprot.readStructEnd() + + def write(self, oprot): + if oprot._fast_encode is not None and self.thrift_spec is not None: + oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec])) + return + oprot.writeStructBegin("update_partition_column_statistics_args") + if self.stats_obj is not None: + oprot.writeFieldBegin("stats_obj", TType.STRUCT, 1) + self.stats_obj.write(oprot) + oprot.writeFieldEnd() + oprot.writeFieldStop() + oprot.writeStructEnd() + + def validate(self): + return + + def __repr__(self): + L = ["{}={!r}".format(key, value) for key, value in self.__dict__.items()] + return "{}({})".format(self.__class__.__name__, ", ".join(L)) + + def __eq__(self, other): + return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ + + def __ne__(self, other): + return not (self == other) + + +all_structs.append(update_partition_column_statistics_args) +update_partition_column_statistics_args.thrift_spec = ( + None, # 0 + ( + 1, + TType.STRUCT, + "stats_obj", + [ColumnStatistics, None], + None, + ), # 1 +) + + +class update_partition_column_statistics_result: + """ + Attributes: + - success + - o1 + - o2 + - o3 + - o4 + + """ + + def __init__( + self, + success=None, + o1=None, + o2=None, + o3=None, + o4=None, + ): + self.success = success + self.o1 = o1 + self.o2 = o2 + self.o3 = o3 + self.o4 = o4 + + def read(self, iprot): + if ( + iprot._fast_decode is not None + and isinstance(iprot.trans, TTransport.CReadableTransport) + and self.thrift_spec is not None + ): + iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec]) + return + iprot.readStructBegin() + while True: + (fname, ftype, fid) = iprot.readFieldBegin() + if ftype == TType.STOP: + break + if fid == 0: + if ftype == TType.BOOL: + self.success = iprot.readBool() + else: + iprot.skip(ftype) + elif fid == 1: + if ftype == TType.STRUCT: + self.o1 = NoSuchObjectException.read(iprot) + else: + iprot.skip(ftype) + elif fid == 2: + if ftype == TType.STRUCT: + self.o2 = InvalidObjectException.read(iprot) + else: + iprot.skip(ftype) + elif fid == 3: + if ftype == TType.STRUCT: + self.o3 = MetaException.read(iprot) + else: + iprot.skip(ftype) + elif fid == 4: + if ftype == TType.STRUCT: + self.o4 = InvalidInputException.read(iprot) + else: + iprot.skip(ftype) + else: + iprot.skip(ftype) + iprot.readFieldEnd() + iprot.readStructEnd() + + def write(self, oprot): + if oprot._fast_encode is not None and self.thrift_spec is not None: + oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec])) + return + oprot.writeStructBegin("update_partition_column_statistics_result") + if self.success is not None: + oprot.writeFieldBegin("success", TType.BOOL, 0) + oprot.writeBool(self.success) + oprot.writeFieldEnd() + if self.o1 is not None: + oprot.writeFieldBegin("o1", TType.STRUCT, 1) + self.o1.write(oprot) + oprot.writeFieldEnd() + if self.o2 is not None: + oprot.writeFieldBegin("o2", TType.STRUCT, 2) + self.o2.write(oprot) + oprot.writeFieldEnd() + if self.o3 is not None: + oprot.writeFieldBegin("o3", TType.STRUCT, 3) + self.o3.write(oprot) + oprot.writeFieldEnd() + if self.o4 is not None: + oprot.writeFieldBegin("o4", TType.STRUCT, 4) + self.o4.write(oprot) + oprot.writeFieldEnd() + oprot.writeFieldStop() + oprot.writeStructEnd() + + def validate(self): + return + + def __repr__(self): + L = ["{}={!r}".format(key, value) for key, value in self.__dict__.items()] + return "{}({})".format(self.__class__.__name__, ", ".join(L)) + + def __eq__(self, other): + return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ + + def __ne__(self, other): + return not (self == other) + + +all_structs.append(update_partition_column_statistics_result) +update_partition_column_statistics_result.thrift_spec = ( + ( + 0, + TType.BOOL, + "success", + None, + None, + ), # 0 + ( + 1, + TType.STRUCT, + "o1", + [NoSuchObjectException, None], + None, + ), # 1 + ( + 2, + TType.STRUCT, + "o2", + [InvalidObjectException, None], + None, + ), # 2 + ( + 3, + TType.STRUCT, + "o3", + [MetaException, None], + None, + ), # 3 + ( + 4, + TType.STRUCT, + "o4", + [InvalidInputException, None], + None, + ), # 4 +) + + +class update_table_column_statistics_req_args: + """ + Attributes: + - req + + """ + + def __init__( + self, + req=None, + ): + self.req = req + + def read(self, iprot): + if ( + iprot._fast_decode is not None + and isinstance(iprot.trans, TTransport.CReadableTransport) + and self.thrift_spec is not None + ): + iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec]) + return + iprot.readStructBegin() + while True: + (fname, ftype, fid) = iprot.readFieldBegin() + if ftype == TType.STOP: + break + if fid == 1: + if ftype == TType.STRUCT: + self.req = SetPartitionsStatsRequest() + self.req.read(iprot) + else: + iprot.skip(ftype) + else: + iprot.skip(ftype) + iprot.readFieldEnd() + iprot.readStructEnd() + + def write(self, oprot): + if oprot._fast_encode is not None and self.thrift_spec is not None: + oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec])) + return + oprot.writeStructBegin("update_table_column_statistics_req_args") + if self.req is not None: + oprot.writeFieldBegin("req", TType.STRUCT, 1) + self.req.write(oprot) + oprot.writeFieldEnd() + oprot.writeFieldStop() + oprot.writeStructEnd() + + def validate(self): + return + + def __repr__(self): + L = ["{}={!r}".format(key, value) for key, value in self.__dict__.items()] + return "{}({})".format(self.__class__.__name__, ", ".join(L)) + + def __eq__(self, other): + return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ + + def __ne__(self, other): + return not (self == other) + + +all_structs.append(update_table_column_statistics_req_args) +update_table_column_statistics_req_args.thrift_spec = ( + None, # 0 + ( + 1, + TType.STRUCT, + "req", + [SetPartitionsStatsRequest, None], + None, + ), # 1 +) + + +class update_table_column_statistics_req_result: + """ + Attributes: + - success + - o1 + - o2 + - o3 + - o4 + + """ + + def __init__( + self, + success=None, + o1=None, + o2=None, + o3=None, + o4=None, + ): + self.success = success + self.o1 = o1 + self.o2 = o2 + self.o3 = o3 + self.o4 = o4 + + def read(self, iprot): + if ( + iprot._fast_decode is not None + and isinstance(iprot.trans, TTransport.CReadableTransport) + and self.thrift_spec is not None + ): + iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec]) + return + iprot.readStructBegin() + while True: + (fname, ftype, fid) = iprot.readFieldBegin() + if ftype == TType.STOP: + break + if fid == 0: + if ftype == TType.STRUCT: + self.success = SetPartitionsStatsResponse() + self.success.read(iprot) + else: + iprot.skip(ftype) + elif fid == 1: + if ftype == TType.STRUCT: + self.o1 = NoSuchObjectException.read(iprot) + else: + iprot.skip(ftype) + elif fid == 2: + if ftype == TType.STRUCT: + self.o2 = InvalidObjectException.read(iprot) + else: + iprot.skip(ftype) + elif fid == 3: + if ftype == TType.STRUCT: + self.o3 = MetaException.read(iprot) + else: + iprot.skip(ftype) + elif fid == 4: + if ftype == TType.STRUCT: + self.o4 = InvalidInputException.read(iprot) + else: + iprot.skip(ftype) + else: + iprot.skip(ftype) + iprot.readFieldEnd() + iprot.readStructEnd() + + def write(self, oprot): + if oprot._fast_encode is not None and self.thrift_spec is not None: + oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec])) + return + oprot.writeStructBegin("update_table_column_statistics_req_result") + if self.success is not None: + oprot.writeFieldBegin("success", TType.STRUCT, 0) + self.success.write(oprot) + oprot.writeFieldEnd() + if self.o1 is not None: + oprot.writeFieldBegin("o1", TType.STRUCT, 1) + self.o1.write(oprot) + oprot.writeFieldEnd() + if self.o2 is not None: + oprot.writeFieldBegin("o2", TType.STRUCT, 2) + self.o2.write(oprot) + oprot.writeFieldEnd() + if self.o3 is not None: + oprot.writeFieldBegin("o3", TType.STRUCT, 3) + self.o3.write(oprot) + oprot.writeFieldEnd() + if self.o4 is not None: + oprot.writeFieldBegin("o4", TType.STRUCT, 4) + self.o4.write(oprot) + oprot.writeFieldEnd() + oprot.writeFieldStop() + oprot.writeStructEnd() + + def validate(self): + return + + def __repr__(self): + L = ["{}={!r}".format(key, value) for key, value in self.__dict__.items()] + return "{}({})".format(self.__class__.__name__, ", ".join(L)) + + def __eq__(self, other): + return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ + + def __ne__(self, other): + return not (self == other) + + +all_structs.append(update_table_column_statistics_req_result) +update_table_column_statistics_req_result.thrift_spec = ( + ( + 0, + TType.STRUCT, + "success", + [SetPartitionsStatsResponse, None], + None, + ), # 0 + ( + 1, + TType.STRUCT, + "o1", + [NoSuchObjectException, None], + None, + ), # 1 + ( + 2, + TType.STRUCT, + "o2", + [InvalidObjectException, None], + None, + ), # 2 + ( + 3, + TType.STRUCT, + "o3", + [MetaException, None], + None, + ), # 3 + ( + 4, + TType.STRUCT, + "o4", + [InvalidInputException, None], + None, + ), # 4 +) + + +class update_partition_column_statistics_req_args: + """ + Attributes: + - req + + """ + + def __init__( + self, + req=None, + ): + self.req = req + + def read(self, iprot): + if ( + iprot._fast_decode is not None + and isinstance(iprot.trans, TTransport.CReadableTransport) + and self.thrift_spec is not None + ): + iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec]) + return + iprot.readStructBegin() + while True: + (fname, ftype, fid) = iprot.readFieldBegin() + if ftype == TType.STOP: + break + if fid == 1: + if ftype == TType.STRUCT: + self.req = SetPartitionsStatsRequest() + self.req.read(iprot) + else: + iprot.skip(ftype) + else: + iprot.skip(ftype) + iprot.readFieldEnd() + iprot.readStructEnd() + + def write(self, oprot): + if oprot._fast_encode is not None and self.thrift_spec is not None: + oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec])) + return + oprot.writeStructBegin("update_partition_column_statistics_req_args") + if self.req is not None: + oprot.writeFieldBegin("req", TType.STRUCT, 1) + self.req.write(oprot) + oprot.writeFieldEnd() + oprot.writeFieldStop() + oprot.writeStructEnd() + + def validate(self): + return + + def __repr__(self): + L = ["{}={!r}".format(key, value) for key, value in self.__dict__.items()] + return "{}({})".format(self.__class__.__name__, ", ".join(L)) + + def __eq__(self, other): + return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ + + def __ne__(self, other): + return not (self == other) + + +all_structs.append(update_partition_column_statistics_req_args) +update_partition_column_statistics_req_args.thrift_spec = ( + None, # 0 + ( + 1, + TType.STRUCT, + "req", + [SetPartitionsStatsRequest, None], + None, + ), # 1 +) + + +class update_partition_column_statistics_req_result: + """ + Attributes: + - success + - o1 + - o2 + - o3 + - o4 + + """ + + def __init__( + self, + success=None, + o1=None, + o2=None, + o3=None, + o4=None, + ): + self.success = success + self.o1 = o1 + self.o2 = o2 + self.o3 = o3 + self.o4 = o4 + + def read(self, iprot): + if ( + iprot._fast_decode is not None + and isinstance(iprot.trans, TTransport.CReadableTransport) + and self.thrift_spec is not None + ): + iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec]) + return + iprot.readStructBegin() + while True: + (fname, ftype, fid) = iprot.readFieldBegin() + if ftype == TType.STOP: + break + if fid == 0: + if ftype == TType.STRUCT: + self.success = SetPartitionsStatsResponse() + self.success.read(iprot) + else: + iprot.skip(ftype) + elif fid == 1: + if ftype == TType.STRUCT: + self.o1 = NoSuchObjectException.read(iprot) + else: + iprot.skip(ftype) + elif fid == 2: + if ftype == TType.STRUCT: + self.o2 = InvalidObjectException.read(iprot) + else: + iprot.skip(ftype) + elif fid == 3: + if ftype == TType.STRUCT: + self.o3 = MetaException.read(iprot) + else: + iprot.skip(ftype) + elif fid == 4: + if ftype == TType.STRUCT: + self.o4 = InvalidInputException.read(iprot) + else: + iprot.skip(ftype) + else: + iprot.skip(ftype) + iprot.readFieldEnd() + iprot.readStructEnd() + + def write(self, oprot): + if oprot._fast_encode is not None and self.thrift_spec is not None: + oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec])) + return + oprot.writeStructBegin("update_partition_column_statistics_req_result") + if self.success is not None: + oprot.writeFieldBegin("success", TType.STRUCT, 0) + self.success.write(oprot) + oprot.writeFieldEnd() + if self.o1 is not None: + oprot.writeFieldBegin("o1", TType.STRUCT, 1) + self.o1.write(oprot) + oprot.writeFieldEnd() + if self.o2 is not None: + oprot.writeFieldBegin("o2", TType.STRUCT, 2) + self.o2.write(oprot) + oprot.writeFieldEnd() + if self.o3 is not None: + oprot.writeFieldBegin("o3", TType.STRUCT, 3) + self.o3.write(oprot) + oprot.writeFieldEnd() + if self.o4 is not None: + oprot.writeFieldBegin("o4", TType.STRUCT, 4) + self.o4.write(oprot) + oprot.writeFieldEnd() + oprot.writeFieldStop() + oprot.writeStructEnd() + + def validate(self): + return + + def __repr__(self): + L = ["{}={!r}".format(key, value) for key, value in self.__dict__.items()] + return "{}({})".format(self.__class__.__name__, ", ".join(L)) + + def __eq__(self, other): + return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ + + def __ne__(self, other): + return not (self == other) + + +all_structs.append(update_partition_column_statistics_req_result) +update_partition_column_statistics_req_result.thrift_spec = ( + ( + 0, + TType.STRUCT, + "success", + [SetPartitionsStatsResponse, None], + None, + ), # 0 + ( + 1, + TType.STRUCT, + "o1", + [NoSuchObjectException, None], + None, + ), # 1 + ( + 2, + TType.STRUCT, + "o2", + [InvalidObjectException, None], + None, + ), # 2 + ( + 3, + TType.STRUCT, + "o3", + [MetaException, None], + None, + ), # 3 + ( + 4, + TType.STRUCT, + "o4", + [InvalidInputException, None], + None, + ), # 4 +) + + +class update_transaction_statistics_args: + """ + Attributes: + - req + + """ + + def __init__( + self, + req=None, + ): + self.req = req + + def read(self, iprot): + if ( + iprot._fast_decode is not None + and isinstance(iprot.trans, TTransport.CReadableTransport) + and self.thrift_spec is not None + ): + iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec]) + return + iprot.readStructBegin() + while True: + (fname, ftype, fid) = iprot.readFieldBegin() + if ftype == TType.STOP: + break + if fid == 1: + if ftype == TType.STRUCT: + self.req = UpdateTransactionalStatsRequest() + self.req.read(iprot) + else: + iprot.skip(ftype) + else: + iprot.skip(ftype) + iprot.readFieldEnd() + iprot.readStructEnd() + + def write(self, oprot): + if oprot._fast_encode is not None and self.thrift_spec is not None: + oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec])) + return + oprot.writeStructBegin("update_transaction_statistics_args") + if self.req is not None: + oprot.writeFieldBegin("req", TType.STRUCT, 1) + self.req.write(oprot) + oprot.writeFieldEnd() + oprot.writeFieldStop() + oprot.writeStructEnd() + + def validate(self): + return + + def __repr__(self): + L = ["{}={!r}".format(key, value) for key, value in self.__dict__.items()] + return "{}({})".format(self.__class__.__name__, ", ".join(L)) + + def __eq__(self, other): + return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ + + def __ne__(self, other): + return not (self == other) + + +all_structs.append(update_transaction_statistics_args) +update_transaction_statistics_args.thrift_spec = ( + None, # 0 + ( + 1, + TType.STRUCT, + "req", + [UpdateTransactionalStatsRequest, None], + None, + ), # 1 +) + + +class update_transaction_statistics_result: + """ + Attributes: + - o1 + + """ + + def __init__( + self, + o1=None, + ): + self.o1 = o1 + + def read(self, iprot): + if ( + iprot._fast_decode is not None + and isinstance(iprot.trans, TTransport.CReadableTransport) + and self.thrift_spec is not None + ): + iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec]) + return + iprot.readStructBegin() + while True: + (fname, ftype, fid) = iprot.readFieldBegin() + if ftype == TType.STOP: + break + if fid == 1: + if ftype == TType.STRUCT: + self.o1 = MetaException.read(iprot) + else: + iprot.skip(ftype) + else: + iprot.skip(ftype) + iprot.readFieldEnd() + iprot.readStructEnd() + + def write(self, oprot): + if oprot._fast_encode is not None and self.thrift_spec is not None: + oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec])) + return + oprot.writeStructBegin("update_transaction_statistics_result") + if self.o1 is not None: + oprot.writeFieldBegin("o1", TType.STRUCT, 1) + self.o1.write(oprot) + oprot.writeFieldEnd() + oprot.writeFieldStop() + oprot.writeStructEnd() + + def validate(self): + return + + def __repr__(self): + L = ["{}={!r}".format(key, value) for key, value in self.__dict__.items()] + return "{}({})".format(self.__class__.__name__, ", ".join(L)) + + def __eq__(self, other): + return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ + + def __ne__(self, other): + return not (self == other) + + +all_structs.append(update_transaction_statistics_result) +update_transaction_statistics_result.thrift_spec = ( + None, # 0 + ( + 1, + TType.STRUCT, + "o1", + [MetaException, None], + None, + ), # 1 +) + + +class get_table_column_statistics_args: + """ + Attributes: + - db_name + - tbl_name + - col_name + + """ + + def __init__( + self, + db_name=None, + tbl_name=None, + col_name=None, + ): + self.db_name = db_name + self.tbl_name = tbl_name + self.col_name = col_name + + def read(self, iprot): + if ( + iprot._fast_decode is not None + and isinstance(iprot.trans, TTransport.CReadableTransport) + and self.thrift_spec is not None + ): + iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec]) + return + iprot.readStructBegin() + while True: + (fname, ftype, fid) = iprot.readFieldBegin() + if ftype == TType.STOP: + break + if fid == 1: + if ftype == TType.STRING: + self.db_name = ( + iprot.readString().decode("utf-8", errors="replace") if sys.version_info[0] == 2 else iprot.readString() + ) + else: + iprot.skip(ftype) + elif fid == 2: + if ftype == TType.STRING: + self.tbl_name = ( + iprot.readString().decode("utf-8", errors="replace") if sys.version_info[0] == 2 else iprot.readString() + ) + else: + iprot.skip(ftype) + elif fid == 3: + if ftype == TType.STRING: + self.col_name = ( + iprot.readString().decode("utf-8", errors="replace") if sys.version_info[0] == 2 else iprot.readString() + ) + else: + iprot.skip(ftype) + else: + iprot.skip(ftype) + iprot.readFieldEnd() + iprot.readStructEnd() + + def write(self, oprot): + if oprot._fast_encode is not None and self.thrift_spec is not None: + oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec])) + return + oprot.writeStructBegin("get_table_column_statistics_args") + if self.db_name is not None: + oprot.writeFieldBegin("db_name", TType.STRING, 1) + oprot.writeString(self.db_name.encode("utf-8") if sys.version_info[0] == 2 else self.db_name) + oprot.writeFieldEnd() + if self.tbl_name is not None: + oprot.writeFieldBegin("tbl_name", TType.STRING, 2) + oprot.writeString(self.tbl_name.encode("utf-8") if sys.version_info[0] == 2 else self.tbl_name) + oprot.writeFieldEnd() + if self.col_name is not None: + oprot.writeFieldBegin("col_name", TType.STRING, 3) + oprot.writeString(self.col_name.encode("utf-8") if sys.version_info[0] == 2 else self.col_name) + oprot.writeFieldEnd() + oprot.writeFieldStop() + oprot.writeStructEnd() + + def validate(self): + return + + def __repr__(self): + L = ["{}={!r}".format(key, value) for key, value in self.__dict__.items()] + return "{}({})".format(self.__class__.__name__, ", ".join(L)) + + def __eq__(self, other): + return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ + + def __ne__(self, other): + return not (self == other) + + +all_structs.append(get_table_column_statistics_args) +get_table_column_statistics_args.thrift_spec = ( + None, # 0 + ( + 1, + TType.STRING, + "db_name", + "UTF8", + None, + ), # 1 + ( + 2, + TType.STRING, + "tbl_name", + "UTF8", + None, + ), # 2 + ( + 3, + TType.STRING, + "col_name", + "UTF8", + None, + ), # 3 +) + + +class get_table_column_statistics_result: + """ + Attributes: + - success + - o1 + - o2 + - o3 + - o4 + + """ + + def __init__( + self, + success=None, + o1=None, + o2=None, + o3=None, + o4=None, + ): + self.success = success + self.o1 = o1 + self.o2 = o2 + self.o3 = o3 + self.o4 = o4 + + def read(self, iprot): + if ( + iprot._fast_decode is not None + and isinstance(iprot.trans, TTransport.CReadableTransport) + and self.thrift_spec is not None + ): + iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec]) + return + iprot.readStructBegin() + while True: + (fname, ftype, fid) = iprot.readFieldBegin() + if ftype == TType.STOP: + break + if fid == 0: + if ftype == TType.STRUCT: + self.success = ColumnStatistics() + self.success.read(iprot) + else: + iprot.skip(ftype) + elif fid == 1: + if ftype == TType.STRUCT: + self.o1 = NoSuchObjectException.read(iprot) + else: + iprot.skip(ftype) + elif fid == 2: + if ftype == TType.STRUCT: + self.o2 = MetaException.read(iprot) + else: + iprot.skip(ftype) + elif fid == 3: + if ftype == TType.STRUCT: + self.o3 = InvalidInputException.read(iprot) + else: + iprot.skip(ftype) + elif fid == 4: + if ftype == TType.STRUCT: + self.o4 = InvalidObjectException.read(iprot) + else: + iprot.skip(ftype) + else: + iprot.skip(ftype) + iprot.readFieldEnd() + iprot.readStructEnd() + + def write(self, oprot): + if oprot._fast_encode is not None and self.thrift_spec is not None: + oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec])) + return + oprot.writeStructBegin("get_table_column_statistics_result") + if self.success is not None: + oprot.writeFieldBegin("success", TType.STRUCT, 0) + self.success.write(oprot) + oprot.writeFieldEnd() + if self.o1 is not None: + oprot.writeFieldBegin("o1", TType.STRUCT, 1) + self.o1.write(oprot) + oprot.writeFieldEnd() + if self.o2 is not None: + oprot.writeFieldBegin("o2", TType.STRUCT, 2) + self.o2.write(oprot) + oprot.writeFieldEnd() + if self.o3 is not None: + oprot.writeFieldBegin("o3", TType.STRUCT, 3) + self.o3.write(oprot) + oprot.writeFieldEnd() + if self.o4 is not None: + oprot.writeFieldBegin("o4", TType.STRUCT, 4) + self.o4.write(oprot) + oprot.writeFieldEnd() + oprot.writeFieldStop() + oprot.writeStructEnd() + + def validate(self): + return + + def __repr__(self): + L = ["{}={!r}".format(key, value) for key, value in self.__dict__.items()] + return "{}({})".format(self.__class__.__name__, ", ".join(L)) + + def __eq__(self, other): + return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ + + def __ne__(self, other): + return not (self == other) + + +all_structs.append(get_table_column_statistics_result) +get_table_column_statistics_result.thrift_spec = ( + ( + 0, + TType.STRUCT, + "success", + [ColumnStatistics, None], + None, + ), # 0 + ( + 1, + TType.STRUCT, + "o1", + [NoSuchObjectException, None], + None, + ), # 1 + ( + 2, + TType.STRUCT, + "o2", + [MetaException, None], + None, + ), # 2 + ( + 3, + TType.STRUCT, + "o3", + [InvalidInputException, None], + None, + ), # 3 + ( + 4, + TType.STRUCT, + "o4", + [InvalidObjectException, None], + None, + ), # 4 +) + + +class get_partition_column_statistics_args: + """ + Attributes: + - db_name + - tbl_name + - part_name + - col_name + + """ + + def __init__( + self, + db_name=None, + tbl_name=None, + part_name=None, + col_name=None, + ): + self.db_name = db_name + self.tbl_name = tbl_name + self.part_name = part_name + self.col_name = col_name + + def read(self, iprot): + if ( + iprot._fast_decode is not None + and isinstance(iprot.trans, TTransport.CReadableTransport) + and self.thrift_spec is not None + ): + iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec]) + return + iprot.readStructBegin() + while True: + (fname, ftype, fid) = iprot.readFieldBegin() + if ftype == TType.STOP: + break + if fid == 1: + if ftype == TType.STRING: + self.db_name = ( + iprot.readString().decode("utf-8", errors="replace") if sys.version_info[0] == 2 else iprot.readString() + ) + else: + iprot.skip(ftype) + elif fid == 2: + if ftype == TType.STRING: + self.tbl_name = ( + iprot.readString().decode("utf-8", errors="replace") if sys.version_info[0] == 2 else iprot.readString() + ) + else: + iprot.skip(ftype) + elif fid == 3: + if ftype == TType.STRING: + self.part_name = ( + iprot.readString().decode("utf-8", errors="replace") if sys.version_info[0] == 2 else iprot.readString() + ) + else: + iprot.skip(ftype) + elif fid == 4: + if ftype == TType.STRING: + self.col_name = ( + iprot.readString().decode("utf-8", errors="replace") if sys.version_info[0] == 2 else iprot.readString() + ) + else: + iprot.skip(ftype) + else: + iprot.skip(ftype) + iprot.readFieldEnd() + iprot.readStructEnd() + + def write(self, oprot): + if oprot._fast_encode is not None and self.thrift_spec is not None: + oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec])) + return + oprot.writeStructBegin("get_partition_column_statistics_args") + if self.db_name is not None: + oprot.writeFieldBegin("db_name", TType.STRING, 1) + oprot.writeString(self.db_name.encode("utf-8") if sys.version_info[0] == 2 else self.db_name) + oprot.writeFieldEnd() + if self.tbl_name is not None: + oprot.writeFieldBegin("tbl_name", TType.STRING, 2) + oprot.writeString(self.tbl_name.encode("utf-8") if sys.version_info[0] == 2 else self.tbl_name) + oprot.writeFieldEnd() + if self.part_name is not None: + oprot.writeFieldBegin("part_name", TType.STRING, 3) + oprot.writeString(self.part_name.encode("utf-8") if sys.version_info[0] == 2 else self.part_name) + oprot.writeFieldEnd() + if self.col_name is not None: + oprot.writeFieldBegin("col_name", TType.STRING, 4) + oprot.writeString(self.col_name.encode("utf-8") if sys.version_info[0] == 2 else self.col_name) + oprot.writeFieldEnd() + oprot.writeFieldStop() + oprot.writeStructEnd() + + def validate(self): + return + + def __repr__(self): + L = ["{}={!r}".format(key, value) for key, value in self.__dict__.items()] + return "{}({})".format(self.__class__.__name__, ", ".join(L)) + + def __eq__(self, other): + return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ + + def __ne__(self, other): + return not (self == other) + + +all_structs.append(get_partition_column_statistics_args) +get_partition_column_statistics_args.thrift_spec = ( + None, # 0 + ( + 1, + TType.STRING, + "db_name", + "UTF8", + None, + ), # 1 + ( + 2, + TType.STRING, + "tbl_name", + "UTF8", + None, + ), # 2 + ( + 3, + TType.STRING, + "part_name", + "UTF8", + None, + ), # 3 + ( + 4, + TType.STRING, + "col_name", + "UTF8", + None, + ), # 4 +) + + +class get_partition_column_statistics_result: + """ + Attributes: + - success + - o1 + - o2 + - o3 + - o4 + + """ + + def __init__( + self, + success=None, + o1=None, + o2=None, + o3=None, + o4=None, + ): + self.success = success + self.o1 = o1 + self.o2 = o2 + self.o3 = o3 + self.o4 = o4 + + def read(self, iprot): + if ( + iprot._fast_decode is not None + and isinstance(iprot.trans, TTransport.CReadableTransport) + and self.thrift_spec is not None + ): + iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec]) + return + iprot.readStructBegin() + while True: + (fname, ftype, fid) = iprot.readFieldBegin() + if ftype == TType.STOP: + break + if fid == 0: + if ftype == TType.STRUCT: + self.success = ColumnStatistics() + self.success.read(iprot) + else: + iprot.skip(ftype) + elif fid == 1: + if ftype == TType.STRUCT: + self.o1 = NoSuchObjectException.read(iprot) + else: + iprot.skip(ftype) + elif fid == 2: + if ftype == TType.STRUCT: + self.o2 = MetaException.read(iprot) + else: + iprot.skip(ftype) + elif fid == 3: + if ftype == TType.STRUCT: + self.o3 = InvalidInputException.read(iprot) + else: + iprot.skip(ftype) + elif fid == 4: + if ftype == TType.STRUCT: + self.o4 = InvalidObjectException.read(iprot) + else: + iprot.skip(ftype) + else: + iprot.skip(ftype) + iprot.readFieldEnd() + iprot.readStructEnd() + + def write(self, oprot): + if oprot._fast_encode is not None and self.thrift_spec is not None: + oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec])) + return + oprot.writeStructBegin("get_partition_column_statistics_result") + if self.success is not None: + oprot.writeFieldBegin("success", TType.STRUCT, 0) + self.success.write(oprot) + oprot.writeFieldEnd() + if self.o1 is not None: + oprot.writeFieldBegin("o1", TType.STRUCT, 1) + self.o1.write(oprot) + oprot.writeFieldEnd() + if self.o2 is not None: + oprot.writeFieldBegin("o2", TType.STRUCT, 2) + self.o2.write(oprot) + oprot.writeFieldEnd() + if self.o3 is not None: + oprot.writeFieldBegin("o3", TType.STRUCT, 3) + self.o3.write(oprot) + oprot.writeFieldEnd() + if self.o4 is not None: + oprot.writeFieldBegin("o4", TType.STRUCT, 4) + self.o4.write(oprot) + oprot.writeFieldEnd() + oprot.writeFieldStop() + oprot.writeStructEnd() + + def validate(self): + return + + def __repr__(self): + L = ["{}={!r}".format(key, value) for key, value in self.__dict__.items()] + return "{}({})".format(self.__class__.__name__, ", ".join(L)) + + def __eq__(self, other): + return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ + + def __ne__(self, other): + return not (self == other) + + +all_structs.append(get_partition_column_statistics_result) +get_partition_column_statistics_result.thrift_spec = ( + ( + 0, + TType.STRUCT, + "success", + [ColumnStatistics, None], + None, + ), # 0 + ( + 1, + TType.STRUCT, + "o1", + [NoSuchObjectException, None], + None, + ), # 1 + ( + 2, + TType.STRUCT, + "o2", + [MetaException, None], + None, + ), # 2 + ( + 3, + TType.STRUCT, + "o3", + [InvalidInputException, None], + None, + ), # 3 + ( + 4, + TType.STRUCT, + "o4", + [InvalidObjectException, None], + None, + ), # 4 +) + + +class get_table_statistics_req_args: + """ + Attributes: + - request + + """ + + def __init__( + self, + request=None, + ): + self.request = request + + def read(self, iprot): + if ( + iprot._fast_decode is not None + and isinstance(iprot.trans, TTransport.CReadableTransport) + and self.thrift_spec is not None + ): + iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec]) + return + iprot.readStructBegin() + while True: + (fname, ftype, fid) = iprot.readFieldBegin() + if ftype == TType.STOP: + break + if fid == 1: + if ftype == TType.STRUCT: + self.request = TableStatsRequest() + self.request.read(iprot) + else: + iprot.skip(ftype) + else: + iprot.skip(ftype) + iprot.readFieldEnd() + iprot.readStructEnd() + + def write(self, oprot): + if oprot._fast_encode is not None and self.thrift_spec is not None: + oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec])) + return + oprot.writeStructBegin("get_table_statistics_req_args") + if self.request is not None: + oprot.writeFieldBegin("request", TType.STRUCT, 1) + self.request.write(oprot) + oprot.writeFieldEnd() + oprot.writeFieldStop() + oprot.writeStructEnd() + + def validate(self): + return + + def __repr__(self): + L = ["{}={!r}".format(key, value) for key, value in self.__dict__.items()] + return "{}({})".format(self.__class__.__name__, ", ".join(L)) + + def __eq__(self, other): + return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ + + def __ne__(self, other): + return not (self == other) + + +all_structs.append(get_table_statistics_req_args) +get_table_statistics_req_args.thrift_spec = ( + None, # 0 + ( + 1, + TType.STRUCT, + "request", + [TableStatsRequest, None], + None, + ), # 1 +) + + +class get_table_statistics_req_result: + """ + Attributes: + - success + - o1 + - o2 + + """ + + def __init__( + self, + success=None, + o1=None, + o2=None, + ): + self.success = success + self.o1 = o1 + self.o2 = o2 + + def read(self, iprot): + if ( + iprot._fast_decode is not None + and isinstance(iprot.trans, TTransport.CReadableTransport) + and self.thrift_spec is not None + ): + iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec]) + return + iprot.readStructBegin() + while True: + (fname, ftype, fid) = iprot.readFieldBegin() + if ftype == TType.STOP: + break + if fid == 0: + if ftype == TType.STRUCT: + self.success = TableStatsResult() + self.success.read(iprot) + else: + iprot.skip(ftype) + elif fid == 1: + if ftype == TType.STRUCT: + self.o1 = NoSuchObjectException.read(iprot) + else: + iprot.skip(ftype) + elif fid == 2: + if ftype == TType.STRUCT: + self.o2 = MetaException.read(iprot) + else: + iprot.skip(ftype) + else: + iprot.skip(ftype) + iprot.readFieldEnd() + iprot.readStructEnd() + + def write(self, oprot): + if oprot._fast_encode is not None and self.thrift_spec is not None: + oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec])) + return + oprot.writeStructBegin("get_table_statistics_req_result") + if self.success is not None: + oprot.writeFieldBegin("success", TType.STRUCT, 0) + self.success.write(oprot) + oprot.writeFieldEnd() + if self.o1 is not None: + oprot.writeFieldBegin("o1", TType.STRUCT, 1) + self.o1.write(oprot) + oprot.writeFieldEnd() + if self.o2 is not None: + oprot.writeFieldBegin("o2", TType.STRUCT, 2) + self.o2.write(oprot) + oprot.writeFieldEnd() + oprot.writeFieldStop() + oprot.writeStructEnd() + + def validate(self): + return + + def __repr__(self): + L = ["{}={!r}".format(key, value) for key, value in self.__dict__.items()] + return "{}({})".format(self.__class__.__name__, ", ".join(L)) + + def __eq__(self, other): + return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ + + def __ne__(self, other): + return not (self == other) + + +all_structs.append(get_table_statistics_req_result) +get_table_statistics_req_result.thrift_spec = ( + ( + 0, + TType.STRUCT, + "success", + [TableStatsResult, None], + None, + ), # 0 + ( + 1, + TType.STRUCT, + "o1", + [NoSuchObjectException, None], + None, + ), # 1 + ( + 2, + TType.STRUCT, + "o2", + [MetaException, None], + None, + ), # 2 +) + + +class get_partitions_statistics_req_args: + """ + Attributes: + - request + + """ + + def __init__( + self, + request=None, + ): + self.request = request + + def read(self, iprot): + if ( + iprot._fast_decode is not None + and isinstance(iprot.trans, TTransport.CReadableTransport) + and self.thrift_spec is not None + ): + iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec]) + return + iprot.readStructBegin() + while True: + (fname, ftype, fid) = iprot.readFieldBegin() + if ftype == TType.STOP: + break + if fid == 1: + if ftype == TType.STRUCT: + self.request = PartitionsStatsRequest() + self.request.read(iprot) + else: + iprot.skip(ftype) + else: + iprot.skip(ftype) + iprot.readFieldEnd() + iprot.readStructEnd() + + def write(self, oprot): + if oprot._fast_encode is not None and self.thrift_spec is not None: + oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec])) + return + oprot.writeStructBegin("get_partitions_statistics_req_args") + if self.request is not None: + oprot.writeFieldBegin("request", TType.STRUCT, 1) + self.request.write(oprot) + oprot.writeFieldEnd() + oprot.writeFieldStop() + oprot.writeStructEnd() + + def validate(self): + return + + def __repr__(self): + L = ["{}={!r}".format(key, value) for key, value in self.__dict__.items()] + return "{}({})".format(self.__class__.__name__, ", ".join(L)) + + def __eq__(self, other): + return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ + + def __ne__(self, other): + return not (self == other) + + +all_structs.append(get_partitions_statistics_req_args) +get_partitions_statistics_req_args.thrift_spec = ( + None, # 0 + ( + 1, + TType.STRUCT, + "request", + [PartitionsStatsRequest, None], + None, + ), # 1 +) + + +class get_partitions_statistics_req_result: + """ + Attributes: + - success + - o1 + - o2 + + """ + + def __init__( + self, + success=None, + o1=None, + o2=None, + ): + self.success = success + self.o1 = o1 + self.o2 = o2 + + def read(self, iprot): + if ( + iprot._fast_decode is not None + and isinstance(iprot.trans, TTransport.CReadableTransport) + and self.thrift_spec is not None + ): + iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec]) + return + iprot.readStructBegin() + while True: + (fname, ftype, fid) = iprot.readFieldBegin() + if ftype == TType.STOP: + break + if fid == 0: + if ftype == TType.STRUCT: + self.success = PartitionsStatsResult() + self.success.read(iprot) + else: + iprot.skip(ftype) + elif fid == 1: + if ftype == TType.STRUCT: + self.o1 = NoSuchObjectException.read(iprot) + else: + iprot.skip(ftype) + elif fid == 2: + if ftype == TType.STRUCT: + self.o2 = MetaException.read(iprot) + else: + iprot.skip(ftype) + else: + iprot.skip(ftype) + iprot.readFieldEnd() + iprot.readStructEnd() + + def write(self, oprot): + if oprot._fast_encode is not None and self.thrift_spec is not None: + oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec])) + return + oprot.writeStructBegin("get_partitions_statistics_req_result") + if self.success is not None: + oprot.writeFieldBegin("success", TType.STRUCT, 0) + self.success.write(oprot) + oprot.writeFieldEnd() + if self.o1 is not None: + oprot.writeFieldBegin("o1", TType.STRUCT, 1) + self.o1.write(oprot) + oprot.writeFieldEnd() + if self.o2 is not None: + oprot.writeFieldBegin("o2", TType.STRUCT, 2) + self.o2.write(oprot) + oprot.writeFieldEnd() + oprot.writeFieldStop() + oprot.writeStructEnd() + + def validate(self): + return + + def __repr__(self): + L = ["{}={!r}".format(key, value) for key, value in self.__dict__.items()] + return "{}({})".format(self.__class__.__name__, ", ".join(L)) + + def __eq__(self, other): + return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ + + def __ne__(self, other): + return not (self == other) + + +all_structs.append(get_partitions_statistics_req_result) +get_partitions_statistics_req_result.thrift_spec = ( + ( + 0, + TType.STRUCT, + "success", + [PartitionsStatsResult, None], + None, + ), # 0 + ( + 1, + TType.STRUCT, + "o1", + [NoSuchObjectException, None], + None, + ), # 1 + ( + 2, + TType.STRUCT, + "o2", + [MetaException, None], + None, + ), # 2 +) + + +class get_aggr_stats_for_args: + """ + Attributes: + - request + + """ + + def __init__( + self, + request=None, + ): + self.request = request + + def read(self, iprot): + if ( + iprot._fast_decode is not None + and isinstance(iprot.trans, TTransport.CReadableTransport) + and self.thrift_spec is not None + ): + iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec]) + return + iprot.readStructBegin() + while True: + (fname, ftype, fid) = iprot.readFieldBegin() + if ftype == TType.STOP: + break + if fid == 1: + if ftype == TType.STRUCT: + self.request = PartitionsStatsRequest() + self.request.read(iprot) + else: + iprot.skip(ftype) + else: + iprot.skip(ftype) + iprot.readFieldEnd() + iprot.readStructEnd() + + def write(self, oprot): + if oprot._fast_encode is not None and self.thrift_spec is not None: + oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec])) + return + oprot.writeStructBegin("get_aggr_stats_for_args") + if self.request is not None: + oprot.writeFieldBegin("request", TType.STRUCT, 1) + self.request.write(oprot) + oprot.writeFieldEnd() + oprot.writeFieldStop() + oprot.writeStructEnd() + + def validate(self): + return + + def __repr__(self): + L = ["{}={!r}".format(key, value) for key, value in self.__dict__.items()] + return "{}({})".format(self.__class__.__name__, ", ".join(L)) + + def __eq__(self, other): + return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ + + def __ne__(self, other): + return not (self == other) + + +all_structs.append(get_aggr_stats_for_args) +get_aggr_stats_for_args.thrift_spec = ( + None, # 0 + ( + 1, + TType.STRUCT, + "request", + [PartitionsStatsRequest, None], + None, + ), # 1 +) + + +class get_aggr_stats_for_result: + """ + Attributes: + - success + - o1 + - o2 + + """ + + def __init__( + self, + success=None, + o1=None, + o2=None, + ): + self.success = success + self.o1 = o1 + self.o2 = o2 + + def read(self, iprot): + if ( + iprot._fast_decode is not None + and isinstance(iprot.trans, TTransport.CReadableTransport) + and self.thrift_spec is not None + ): + iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec]) + return + iprot.readStructBegin() + while True: + (fname, ftype, fid) = iprot.readFieldBegin() + if ftype == TType.STOP: + break + if fid == 0: + if ftype == TType.STRUCT: + self.success = AggrStats() + self.success.read(iprot) + else: + iprot.skip(ftype) + elif fid == 1: + if ftype == TType.STRUCT: + self.o1 = NoSuchObjectException.read(iprot) + else: + iprot.skip(ftype) + elif fid == 2: + if ftype == TType.STRUCT: + self.o2 = MetaException.read(iprot) + else: + iprot.skip(ftype) + else: + iprot.skip(ftype) + iprot.readFieldEnd() + iprot.readStructEnd() + + def write(self, oprot): + if oprot._fast_encode is not None and self.thrift_spec is not None: + oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec])) + return + oprot.writeStructBegin("get_aggr_stats_for_result") + if self.success is not None: + oprot.writeFieldBegin("success", TType.STRUCT, 0) + self.success.write(oprot) + oprot.writeFieldEnd() + if self.o1 is not None: + oprot.writeFieldBegin("o1", TType.STRUCT, 1) + self.o1.write(oprot) + oprot.writeFieldEnd() + if self.o2 is not None: + oprot.writeFieldBegin("o2", TType.STRUCT, 2) + self.o2.write(oprot) + oprot.writeFieldEnd() + oprot.writeFieldStop() + oprot.writeStructEnd() + + def validate(self): + return + + def __repr__(self): + L = ["{}={!r}".format(key, value) for key, value in self.__dict__.items()] + return "{}({})".format(self.__class__.__name__, ", ".join(L)) + + def __eq__(self, other): + return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ + + def __ne__(self, other): + return not (self == other) + + +all_structs.append(get_aggr_stats_for_result) +get_aggr_stats_for_result.thrift_spec = ( + ( + 0, + TType.STRUCT, + "success", + [AggrStats, None], + None, + ), # 0 + ( + 1, + TType.STRUCT, + "o1", + [NoSuchObjectException, None], + None, + ), # 1 + ( + 2, + TType.STRUCT, + "o2", + [MetaException, None], + None, + ), # 2 +) + + +class set_aggr_stats_for_args: + """ + Attributes: + - request + + """ + + def __init__( + self, + request=None, + ): + self.request = request + + def read(self, iprot): + if ( + iprot._fast_decode is not None + and isinstance(iprot.trans, TTransport.CReadableTransport) + and self.thrift_spec is not None + ): + iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec]) + return + iprot.readStructBegin() + while True: + (fname, ftype, fid) = iprot.readFieldBegin() + if ftype == TType.STOP: + break + if fid == 1: + if ftype == TType.STRUCT: + self.request = SetPartitionsStatsRequest() + self.request.read(iprot) + else: + iprot.skip(ftype) + else: + iprot.skip(ftype) + iprot.readFieldEnd() + iprot.readStructEnd() + + def write(self, oprot): + if oprot._fast_encode is not None and self.thrift_spec is not None: + oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec])) + return + oprot.writeStructBegin("set_aggr_stats_for_args") + if self.request is not None: + oprot.writeFieldBegin("request", TType.STRUCT, 1) + self.request.write(oprot) + oprot.writeFieldEnd() + oprot.writeFieldStop() + oprot.writeStructEnd() + + def validate(self): + return + + def __repr__(self): + L = ["{}={!r}".format(key, value) for key, value in self.__dict__.items()] + return "{}({})".format(self.__class__.__name__, ", ".join(L)) + + def __eq__(self, other): + return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ + + def __ne__(self, other): + return not (self == other) + + +all_structs.append(set_aggr_stats_for_args) +set_aggr_stats_for_args.thrift_spec = ( + None, # 0 + ( + 1, + TType.STRUCT, + "request", + [SetPartitionsStatsRequest, None], + None, + ), # 1 +) + + +class set_aggr_stats_for_result: + """ + Attributes: + - success + - o1 + - o2 + - o3 + - o4 + + """ + + def __init__( + self, + success=None, + o1=None, + o2=None, + o3=None, + o4=None, + ): + self.success = success + self.o1 = o1 + self.o2 = o2 + self.o3 = o3 + self.o4 = o4 + + def read(self, iprot): + if ( + iprot._fast_decode is not None + and isinstance(iprot.trans, TTransport.CReadableTransport) + and self.thrift_spec is not None + ): + iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec]) + return + iprot.readStructBegin() + while True: + (fname, ftype, fid) = iprot.readFieldBegin() + if ftype == TType.STOP: + break + if fid == 0: + if ftype == TType.BOOL: + self.success = iprot.readBool() + else: + iprot.skip(ftype) + elif fid == 1: + if ftype == TType.STRUCT: + self.o1 = NoSuchObjectException.read(iprot) + else: + iprot.skip(ftype) + elif fid == 2: + if ftype == TType.STRUCT: + self.o2 = InvalidObjectException.read(iprot) + else: + iprot.skip(ftype) + elif fid == 3: + if ftype == TType.STRUCT: + self.o3 = MetaException.read(iprot) + else: + iprot.skip(ftype) + elif fid == 4: + if ftype == TType.STRUCT: + self.o4 = InvalidInputException.read(iprot) + else: + iprot.skip(ftype) + else: + iprot.skip(ftype) + iprot.readFieldEnd() + iprot.readStructEnd() + + def write(self, oprot): + if oprot._fast_encode is not None and self.thrift_spec is not None: + oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec])) + return + oprot.writeStructBegin("set_aggr_stats_for_result") + if self.success is not None: + oprot.writeFieldBegin("success", TType.BOOL, 0) + oprot.writeBool(self.success) + oprot.writeFieldEnd() + if self.o1 is not None: + oprot.writeFieldBegin("o1", TType.STRUCT, 1) + self.o1.write(oprot) + oprot.writeFieldEnd() + if self.o2 is not None: + oprot.writeFieldBegin("o2", TType.STRUCT, 2) + self.o2.write(oprot) + oprot.writeFieldEnd() + if self.o3 is not None: + oprot.writeFieldBegin("o3", TType.STRUCT, 3) + self.o3.write(oprot) + oprot.writeFieldEnd() + if self.o4 is not None: + oprot.writeFieldBegin("o4", TType.STRUCT, 4) + self.o4.write(oprot) + oprot.writeFieldEnd() + oprot.writeFieldStop() + oprot.writeStructEnd() + + def validate(self): + return + + def __repr__(self): + L = ["{}={!r}".format(key, value) for key, value in self.__dict__.items()] + return "{}({})".format(self.__class__.__name__, ", ".join(L)) + + def __eq__(self, other): + return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ + + def __ne__(self, other): + return not (self == other) + + +all_structs.append(set_aggr_stats_for_result) +set_aggr_stats_for_result.thrift_spec = ( + ( + 0, + TType.BOOL, + "success", + None, + None, + ), # 0 + ( + 1, + TType.STRUCT, + "o1", + [NoSuchObjectException, None], + None, + ), # 1 + ( + 2, + TType.STRUCT, + "o2", + [InvalidObjectException, None], + None, + ), # 2 + ( + 3, + TType.STRUCT, + "o3", + [MetaException, None], + None, + ), # 3 + ( + 4, + TType.STRUCT, + "o4", + [InvalidInputException, None], + None, + ), # 4 +) + + +class delete_partition_column_statistics_args: + """ + Attributes: + - db_name + - tbl_name + - part_name + - col_name + - engine + + """ + + def __init__( + self, + db_name=None, + tbl_name=None, + part_name=None, + col_name=None, + engine=None, + ): + self.db_name = db_name + self.tbl_name = tbl_name + self.part_name = part_name + self.col_name = col_name + self.engine = engine + + def read(self, iprot): + if ( + iprot._fast_decode is not None + and isinstance(iprot.trans, TTransport.CReadableTransport) + and self.thrift_spec is not None + ): + iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec]) + return + iprot.readStructBegin() + while True: + (fname, ftype, fid) = iprot.readFieldBegin() + if ftype == TType.STOP: + break + if fid == 1: + if ftype == TType.STRING: + self.db_name = ( + iprot.readString().decode("utf-8", errors="replace") if sys.version_info[0] == 2 else iprot.readString() + ) + else: + iprot.skip(ftype) + elif fid == 2: + if ftype == TType.STRING: + self.tbl_name = ( + iprot.readString().decode("utf-8", errors="replace") if sys.version_info[0] == 2 else iprot.readString() + ) + else: + iprot.skip(ftype) + elif fid == 3: + if ftype == TType.STRING: + self.part_name = ( + iprot.readString().decode("utf-8", errors="replace") if sys.version_info[0] == 2 else iprot.readString() + ) + else: + iprot.skip(ftype) + elif fid == 4: + if ftype == TType.STRING: + self.col_name = ( + iprot.readString().decode("utf-8", errors="replace") if sys.version_info[0] == 2 else iprot.readString() + ) + else: + iprot.skip(ftype) + elif fid == 5: + if ftype == TType.STRING: + self.engine = ( + iprot.readString().decode("utf-8", errors="replace") if sys.version_info[0] == 2 else iprot.readString() + ) + else: + iprot.skip(ftype) + else: + iprot.skip(ftype) + iprot.readFieldEnd() + iprot.readStructEnd() + + def write(self, oprot): + if oprot._fast_encode is not None and self.thrift_spec is not None: + oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec])) + return + oprot.writeStructBegin("delete_partition_column_statistics_args") + if self.db_name is not None: + oprot.writeFieldBegin("db_name", TType.STRING, 1) + oprot.writeString(self.db_name.encode("utf-8") if sys.version_info[0] == 2 else self.db_name) + oprot.writeFieldEnd() + if self.tbl_name is not None: + oprot.writeFieldBegin("tbl_name", TType.STRING, 2) + oprot.writeString(self.tbl_name.encode("utf-8") if sys.version_info[0] == 2 else self.tbl_name) + oprot.writeFieldEnd() + if self.part_name is not None: + oprot.writeFieldBegin("part_name", TType.STRING, 3) + oprot.writeString(self.part_name.encode("utf-8") if sys.version_info[0] == 2 else self.part_name) + oprot.writeFieldEnd() + if self.col_name is not None: + oprot.writeFieldBegin("col_name", TType.STRING, 4) + oprot.writeString(self.col_name.encode("utf-8") if sys.version_info[0] == 2 else self.col_name) + oprot.writeFieldEnd() + if self.engine is not None: + oprot.writeFieldBegin("engine", TType.STRING, 5) + oprot.writeString(self.engine.encode("utf-8") if sys.version_info[0] == 2 else self.engine) + oprot.writeFieldEnd() + oprot.writeFieldStop() + oprot.writeStructEnd() + + def validate(self): + return + + def __repr__(self): + L = ["{}={!r}".format(key, value) for key, value in self.__dict__.items()] + return "{}({})".format(self.__class__.__name__, ", ".join(L)) + + def __eq__(self, other): + return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ + + def __ne__(self, other): + return not (self == other) + + +all_structs.append(delete_partition_column_statistics_args) +delete_partition_column_statistics_args.thrift_spec = ( + None, # 0 + ( + 1, + TType.STRING, + "db_name", + "UTF8", + None, + ), # 1 + ( + 2, + TType.STRING, + "tbl_name", + "UTF8", + None, + ), # 2 + ( + 3, + TType.STRING, + "part_name", + "UTF8", + None, + ), # 3 + ( + 4, + TType.STRING, + "col_name", + "UTF8", + None, + ), # 4 + ( + 5, + TType.STRING, + "engine", + "UTF8", + None, + ), # 5 +) + + +class delete_partition_column_statistics_result: + """ + Attributes: + - success + - o1 + - o2 + - o3 + - o4 + + """ + + def __init__( + self, + success=None, + o1=None, + o2=None, + o3=None, + o4=None, + ): + self.success = success + self.o1 = o1 + self.o2 = o2 + self.o3 = o3 + self.o4 = o4 + + def read(self, iprot): + if ( + iprot._fast_decode is not None + and isinstance(iprot.trans, TTransport.CReadableTransport) + and self.thrift_spec is not None + ): + iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec]) + return + iprot.readStructBegin() + while True: + (fname, ftype, fid) = iprot.readFieldBegin() + if ftype == TType.STOP: + break + if fid == 0: + if ftype == TType.BOOL: + self.success = iprot.readBool() + else: + iprot.skip(ftype) + elif fid == 1: + if ftype == TType.STRUCT: + self.o1 = NoSuchObjectException.read(iprot) + else: + iprot.skip(ftype) + elif fid == 2: + if ftype == TType.STRUCT: + self.o2 = MetaException.read(iprot) + else: + iprot.skip(ftype) + elif fid == 3: + if ftype == TType.STRUCT: + self.o3 = InvalidObjectException.read(iprot) + else: + iprot.skip(ftype) + elif fid == 4: + if ftype == TType.STRUCT: + self.o4 = InvalidInputException.read(iprot) + else: + iprot.skip(ftype) + else: + iprot.skip(ftype) + iprot.readFieldEnd() + iprot.readStructEnd() + + def write(self, oprot): + if oprot._fast_encode is not None and self.thrift_spec is not None: + oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec])) + return + oprot.writeStructBegin("delete_partition_column_statistics_result") + if self.success is not None: + oprot.writeFieldBegin("success", TType.BOOL, 0) + oprot.writeBool(self.success) + oprot.writeFieldEnd() + if self.o1 is not None: + oprot.writeFieldBegin("o1", TType.STRUCT, 1) + self.o1.write(oprot) + oprot.writeFieldEnd() + if self.o2 is not None: + oprot.writeFieldBegin("o2", TType.STRUCT, 2) + self.o2.write(oprot) + oprot.writeFieldEnd() + if self.o3 is not None: + oprot.writeFieldBegin("o3", TType.STRUCT, 3) + self.o3.write(oprot) + oprot.writeFieldEnd() + if self.o4 is not None: + oprot.writeFieldBegin("o4", TType.STRUCT, 4) + self.o4.write(oprot) + oprot.writeFieldEnd() + oprot.writeFieldStop() + oprot.writeStructEnd() + + def validate(self): + return + + def __repr__(self): + L = ["{}={!r}".format(key, value) for key, value in self.__dict__.items()] + return "{}({})".format(self.__class__.__name__, ", ".join(L)) + + def __eq__(self, other): + return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ + + def __ne__(self, other): + return not (self == other) + + +all_structs.append(delete_partition_column_statistics_result) +delete_partition_column_statistics_result.thrift_spec = ( + ( + 0, + TType.BOOL, + "success", + None, + None, + ), # 0 + ( + 1, + TType.STRUCT, + "o1", + [NoSuchObjectException, None], + None, + ), # 1 + ( + 2, + TType.STRUCT, + "o2", + [MetaException, None], + None, + ), # 2 + ( + 3, + TType.STRUCT, + "o3", + [InvalidObjectException, None], + None, + ), # 3 + ( + 4, + TType.STRUCT, + "o4", + [InvalidInputException, None], + None, + ), # 4 +) + + +class delete_table_column_statistics_args: + """ + Attributes: + - db_name + - tbl_name + - col_name + - engine + + """ + + def __init__( + self, + db_name=None, + tbl_name=None, + col_name=None, + engine=None, + ): + self.db_name = db_name + self.tbl_name = tbl_name + self.col_name = col_name + self.engine = engine + + def read(self, iprot): + if ( + iprot._fast_decode is not None + and isinstance(iprot.trans, TTransport.CReadableTransport) + and self.thrift_spec is not None + ): + iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec]) + return + iprot.readStructBegin() + while True: + (fname, ftype, fid) = iprot.readFieldBegin() + if ftype == TType.STOP: + break + if fid == 1: + if ftype == TType.STRING: + self.db_name = ( + iprot.readString().decode("utf-8", errors="replace") if sys.version_info[0] == 2 else iprot.readString() + ) + else: + iprot.skip(ftype) + elif fid == 2: + if ftype == TType.STRING: + self.tbl_name = ( + iprot.readString().decode("utf-8", errors="replace") if sys.version_info[0] == 2 else iprot.readString() + ) + else: + iprot.skip(ftype) + elif fid == 3: + if ftype == TType.STRING: + self.col_name = ( + iprot.readString().decode("utf-8", errors="replace") if sys.version_info[0] == 2 else iprot.readString() + ) + else: + iprot.skip(ftype) + elif fid == 4: + if ftype == TType.STRING: + self.engine = ( + iprot.readString().decode("utf-8", errors="replace") if sys.version_info[0] == 2 else iprot.readString() + ) + else: + iprot.skip(ftype) + else: + iprot.skip(ftype) + iprot.readFieldEnd() + iprot.readStructEnd() + + def write(self, oprot): + if oprot._fast_encode is not None and self.thrift_spec is not None: + oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec])) + return + oprot.writeStructBegin("delete_table_column_statistics_args") + if self.db_name is not None: + oprot.writeFieldBegin("db_name", TType.STRING, 1) + oprot.writeString(self.db_name.encode("utf-8") if sys.version_info[0] == 2 else self.db_name) + oprot.writeFieldEnd() + if self.tbl_name is not None: + oprot.writeFieldBegin("tbl_name", TType.STRING, 2) + oprot.writeString(self.tbl_name.encode("utf-8") if sys.version_info[0] == 2 else self.tbl_name) + oprot.writeFieldEnd() + if self.col_name is not None: + oprot.writeFieldBegin("col_name", TType.STRING, 3) + oprot.writeString(self.col_name.encode("utf-8") if sys.version_info[0] == 2 else self.col_name) + oprot.writeFieldEnd() + if self.engine is not None: + oprot.writeFieldBegin("engine", TType.STRING, 4) + oprot.writeString(self.engine.encode("utf-8") if sys.version_info[0] == 2 else self.engine) + oprot.writeFieldEnd() + oprot.writeFieldStop() + oprot.writeStructEnd() + + def validate(self): + return + + def __repr__(self): + L = ["{}={!r}".format(key, value) for key, value in self.__dict__.items()] + return "{}({})".format(self.__class__.__name__, ", ".join(L)) + + def __eq__(self, other): + return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ + + def __ne__(self, other): + return not (self == other) + + +all_structs.append(delete_table_column_statistics_args) +delete_table_column_statistics_args.thrift_spec = ( + None, # 0 + ( + 1, + TType.STRING, + "db_name", + "UTF8", + None, + ), # 1 + ( + 2, + TType.STRING, + "tbl_name", + "UTF8", + None, + ), # 2 + ( + 3, + TType.STRING, + "col_name", + "UTF8", + None, + ), # 3 + ( + 4, + TType.STRING, + "engine", + "UTF8", + None, + ), # 4 +) + + +class delete_table_column_statistics_result: + """ + Attributes: + - success + - o1 + - o2 + - o3 + - o4 + + """ + + def __init__( + self, + success=None, + o1=None, + o2=None, + o3=None, + o4=None, + ): + self.success = success + self.o1 = o1 + self.o2 = o2 + self.o3 = o3 + self.o4 = o4 + + def read(self, iprot): + if ( + iprot._fast_decode is not None + and isinstance(iprot.trans, TTransport.CReadableTransport) + and self.thrift_spec is not None + ): + iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec]) + return + iprot.readStructBegin() + while True: + (fname, ftype, fid) = iprot.readFieldBegin() + if ftype == TType.STOP: + break + if fid == 0: + if ftype == TType.BOOL: + self.success = iprot.readBool() + else: + iprot.skip(ftype) + elif fid == 1: + if ftype == TType.STRUCT: + self.o1 = NoSuchObjectException.read(iprot) + else: + iprot.skip(ftype) + elif fid == 2: + if ftype == TType.STRUCT: + self.o2 = MetaException.read(iprot) + else: + iprot.skip(ftype) + elif fid == 3: + if ftype == TType.STRUCT: + self.o3 = InvalidObjectException.read(iprot) + else: + iprot.skip(ftype) + elif fid == 4: + if ftype == TType.STRUCT: + self.o4 = InvalidInputException.read(iprot) + else: + iprot.skip(ftype) + else: + iprot.skip(ftype) + iprot.readFieldEnd() + iprot.readStructEnd() + + def write(self, oprot): + if oprot._fast_encode is not None and self.thrift_spec is not None: + oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec])) + return + oprot.writeStructBegin("delete_table_column_statistics_result") + if self.success is not None: + oprot.writeFieldBegin("success", TType.BOOL, 0) + oprot.writeBool(self.success) + oprot.writeFieldEnd() + if self.o1 is not None: + oprot.writeFieldBegin("o1", TType.STRUCT, 1) + self.o1.write(oprot) + oprot.writeFieldEnd() + if self.o2 is not None: + oprot.writeFieldBegin("o2", TType.STRUCT, 2) + self.o2.write(oprot) + oprot.writeFieldEnd() + if self.o3 is not None: + oprot.writeFieldBegin("o3", TType.STRUCT, 3) + self.o3.write(oprot) + oprot.writeFieldEnd() + if self.o4 is not None: + oprot.writeFieldBegin("o4", TType.STRUCT, 4) + self.o4.write(oprot) + oprot.writeFieldEnd() + oprot.writeFieldStop() + oprot.writeStructEnd() + + def validate(self): + return + + def __repr__(self): + L = ["{}={!r}".format(key, value) for key, value in self.__dict__.items()] + return "{}({})".format(self.__class__.__name__, ", ".join(L)) + + def __eq__(self, other): + return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ + + def __ne__(self, other): + return not (self == other) + + +all_structs.append(delete_table_column_statistics_result) +delete_table_column_statistics_result.thrift_spec = ( + ( + 0, + TType.BOOL, + "success", + None, + None, + ), # 0 + ( + 1, + TType.STRUCT, + "o1", + [NoSuchObjectException, None], + None, + ), # 1 + ( + 2, + TType.STRUCT, + "o2", + [MetaException, None], + None, + ), # 2 + ( + 3, + TType.STRUCT, + "o3", + [InvalidObjectException, None], + None, + ), # 3 + ( + 4, + TType.STRUCT, + "o4", + [InvalidInputException, None], + None, + ), # 4 +) + + +class create_function_args: + """ + Attributes: + - func + + """ + + def __init__( + self, + func=None, + ): + self.func = func + + def read(self, iprot): + if ( + iprot._fast_decode is not None + and isinstance(iprot.trans, TTransport.CReadableTransport) + and self.thrift_spec is not None + ): + iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec]) + return + iprot.readStructBegin() + while True: + (fname, ftype, fid) = iprot.readFieldBegin() + if ftype == TType.STOP: + break + if fid == 1: + if ftype == TType.STRUCT: + self.func = Function() + self.func.read(iprot) + else: + iprot.skip(ftype) + else: + iprot.skip(ftype) + iprot.readFieldEnd() + iprot.readStructEnd() + + def write(self, oprot): + if oprot._fast_encode is not None and self.thrift_spec is not None: + oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec])) + return + oprot.writeStructBegin("create_function_args") + if self.func is not None: + oprot.writeFieldBegin("func", TType.STRUCT, 1) + self.func.write(oprot) + oprot.writeFieldEnd() + oprot.writeFieldStop() + oprot.writeStructEnd() + + def validate(self): + return + + def __repr__(self): + L = ["{}={!r}".format(key, value) for key, value in self.__dict__.items()] + return "{}({})".format(self.__class__.__name__, ", ".join(L)) + + def __eq__(self, other): + return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ + + def __ne__(self, other): + return not (self == other) + + +all_structs.append(create_function_args) +create_function_args.thrift_spec = ( + None, # 0 + ( + 1, + TType.STRUCT, + "func", + [Function, None], + None, + ), # 1 +) + + +class create_function_result: + """ + Attributes: + - o1 + - o2 + - o3 + - o4 + + """ + + def __init__( + self, + o1=None, + o2=None, + o3=None, + o4=None, + ): + self.o1 = o1 + self.o2 = o2 + self.o3 = o3 + self.o4 = o4 + + def read(self, iprot): + if ( + iprot._fast_decode is not None + and isinstance(iprot.trans, TTransport.CReadableTransport) + and self.thrift_spec is not None + ): + iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec]) + return + iprot.readStructBegin() + while True: + (fname, ftype, fid) = iprot.readFieldBegin() + if ftype == TType.STOP: + break + if fid == 1: + if ftype == TType.STRUCT: + self.o1 = AlreadyExistsException.read(iprot) + else: + iprot.skip(ftype) + elif fid == 2: + if ftype == TType.STRUCT: + self.o2 = InvalidObjectException.read(iprot) + else: + iprot.skip(ftype) + elif fid == 3: + if ftype == TType.STRUCT: + self.o3 = MetaException.read(iprot) + else: + iprot.skip(ftype) + elif fid == 4: + if ftype == TType.STRUCT: + self.o4 = NoSuchObjectException.read(iprot) + else: + iprot.skip(ftype) + else: + iprot.skip(ftype) + iprot.readFieldEnd() + iprot.readStructEnd() + + def write(self, oprot): + if oprot._fast_encode is not None and self.thrift_spec is not None: + oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec])) + return + oprot.writeStructBegin("create_function_result") + if self.o1 is not None: + oprot.writeFieldBegin("o1", TType.STRUCT, 1) + self.o1.write(oprot) + oprot.writeFieldEnd() + if self.o2 is not None: + oprot.writeFieldBegin("o2", TType.STRUCT, 2) + self.o2.write(oprot) + oprot.writeFieldEnd() + if self.o3 is not None: + oprot.writeFieldBegin("o3", TType.STRUCT, 3) + self.o3.write(oprot) + oprot.writeFieldEnd() + if self.o4 is not None: + oprot.writeFieldBegin("o4", TType.STRUCT, 4) + self.o4.write(oprot) + oprot.writeFieldEnd() + oprot.writeFieldStop() + oprot.writeStructEnd() + + def validate(self): + return + + def __repr__(self): + L = ["{}={!r}".format(key, value) for key, value in self.__dict__.items()] + return "{}({})".format(self.__class__.__name__, ", ".join(L)) + + def __eq__(self, other): + return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ + + def __ne__(self, other): + return not (self == other) + + +all_structs.append(create_function_result) +create_function_result.thrift_spec = ( + None, # 0 + ( + 1, + TType.STRUCT, + "o1", + [AlreadyExistsException, None], + None, + ), # 1 + ( + 2, + TType.STRUCT, + "o2", + [InvalidObjectException, None], + None, + ), # 2 + ( + 3, + TType.STRUCT, + "o3", + [MetaException, None], + None, + ), # 3 + ( + 4, + TType.STRUCT, + "o4", + [NoSuchObjectException, None], + None, + ), # 4 +) + + +class drop_function_args: + """ + Attributes: + - dbName + - funcName + + """ + + def __init__( + self, + dbName=None, + funcName=None, + ): + self.dbName = dbName + self.funcName = funcName + + def read(self, iprot): + if ( + iprot._fast_decode is not None + and isinstance(iprot.trans, TTransport.CReadableTransport) + and self.thrift_spec is not None + ): + iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec]) + return + iprot.readStructBegin() + while True: + (fname, ftype, fid) = iprot.readFieldBegin() + if ftype == TType.STOP: + break + if fid == 1: + if ftype == TType.STRING: + self.dbName = ( + iprot.readString().decode("utf-8", errors="replace") if sys.version_info[0] == 2 else iprot.readString() + ) + else: + iprot.skip(ftype) + elif fid == 2: + if ftype == TType.STRING: + self.funcName = ( + iprot.readString().decode("utf-8", errors="replace") if sys.version_info[0] == 2 else iprot.readString() + ) + else: + iprot.skip(ftype) + else: + iprot.skip(ftype) + iprot.readFieldEnd() + iprot.readStructEnd() + + def write(self, oprot): + if oprot._fast_encode is not None and self.thrift_spec is not None: + oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec])) + return + oprot.writeStructBegin("drop_function_args") + if self.dbName is not None: + oprot.writeFieldBegin("dbName", TType.STRING, 1) + oprot.writeString(self.dbName.encode("utf-8") if sys.version_info[0] == 2 else self.dbName) + oprot.writeFieldEnd() + if self.funcName is not None: + oprot.writeFieldBegin("funcName", TType.STRING, 2) + oprot.writeString(self.funcName.encode("utf-8") if sys.version_info[0] == 2 else self.funcName) + oprot.writeFieldEnd() + oprot.writeFieldStop() + oprot.writeStructEnd() + + def validate(self): + return + + def __repr__(self): + L = ["{}={!r}".format(key, value) for key, value in self.__dict__.items()] + return "{}({})".format(self.__class__.__name__, ", ".join(L)) + + def __eq__(self, other): + return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ + + def __ne__(self, other): + return not (self == other) + + +all_structs.append(drop_function_args) +drop_function_args.thrift_spec = ( + None, # 0 + ( + 1, + TType.STRING, + "dbName", + "UTF8", + None, + ), # 1 + ( + 2, + TType.STRING, + "funcName", + "UTF8", + None, + ), # 2 +) + + +class drop_function_result: + """ + Attributes: + - o1 + - o3 + + """ + + def __init__( + self, + o1=None, + o3=None, + ): + self.o1 = o1 + self.o3 = o3 + + def read(self, iprot): + if ( + iprot._fast_decode is not None + and isinstance(iprot.trans, TTransport.CReadableTransport) + and self.thrift_spec is not None + ): + iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec]) + return + iprot.readStructBegin() + while True: + (fname, ftype, fid) = iprot.readFieldBegin() + if ftype == TType.STOP: + break + if fid == 1: + if ftype == TType.STRUCT: + self.o1 = NoSuchObjectException.read(iprot) + else: + iprot.skip(ftype) + elif fid == 2: + if ftype == TType.STRUCT: + self.o3 = MetaException.read(iprot) + else: + iprot.skip(ftype) + else: + iprot.skip(ftype) + iprot.readFieldEnd() + iprot.readStructEnd() + + def write(self, oprot): + if oprot._fast_encode is not None and self.thrift_spec is not None: + oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec])) + return + oprot.writeStructBegin("drop_function_result") + if self.o1 is not None: + oprot.writeFieldBegin("o1", TType.STRUCT, 1) + self.o1.write(oprot) + oprot.writeFieldEnd() + if self.o3 is not None: + oprot.writeFieldBegin("o3", TType.STRUCT, 2) + self.o3.write(oprot) + oprot.writeFieldEnd() + oprot.writeFieldStop() + oprot.writeStructEnd() + + def validate(self): + return + + def __repr__(self): + L = ["{}={!r}".format(key, value) for key, value in self.__dict__.items()] + return "{}({})".format(self.__class__.__name__, ", ".join(L)) + + def __eq__(self, other): + return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ + + def __ne__(self, other): + return not (self == other) + + +all_structs.append(drop_function_result) +drop_function_result.thrift_spec = ( + None, # 0 + ( + 1, + TType.STRUCT, + "o1", + [NoSuchObjectException, None], + None, + ), # 1 + ( + 2, + TType.STRUCT, + "o3", + [MetaException, None], + None, + ), # 2 +) + + +class alter_function_args: + """ + Attributes: + - dbName + - funcName + - newFunc + + """ + + def __init__( + self, + dbName=None, + funcName=None, + newFunc=None, + ): + self.dbName = dbName + self.funcName = funcName + self.newFunc = newFunc + + def read(self, iprot): + if ( + iprot._fast_decode is not None + and isinstance(iprot.trans, TTransport.CReadableTransport) + and self.thrift_spec is not None + ): + iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec]) + return + iprot.readStructBegin() + while True: + (fname, ftype, fid) = iprot.readFieldBegin() + if ftype == TType.STOP: + break + if fid == 1: + if ftype == TType.STRING: + self.dbName = ( + iprot.readString().decode("utf-8", errors="replace") if sys.version_info[0] == 2 else iprot.readString() + ) + else: + iprot.skip(ftype) + elif fid == 2: + if ftype == TType.STRING: + self.funcName = ( + iprot.readString().decode("utf-8", errors="replace") if sys.version_info[0] == 2 else iprot.readString() + ) + else: + iprot.skip(ftype) + elif fid == 3: + if ftype == TType.STRUCT: + self.newFunc = Function() + self.newFunc.read(iprot) + else: + iprot.skip(ftype) + else: + iprot.skip(ftype) + iprot.readFieldEnd() + iprot.readStructEnd() + + def write(self, oprot): + if oprot._fast_encode is not None and self.thrift_spec is not None: + oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec])) + return + oprot.writeStructBegin("alter_function_args") + if self.dbName is not None: + oprot.writeFieldBegin("dbName", TType.STRING, 1) + oprot.writeString(self.dbName.encode("utf-8") if sys.version_info[0] == 2 else self.dbName) + oprot.writeFieldEnd() + if self.funcName is not None: + oprot.writeFieldBegin("funcName", TType.STRING, 2) + oprot.writeString(self.funcName.encode("utf-8") if sys.version_info[0] == 2 else self.funcName) + oprot.writeFieldEnd() + if self.newFunc is not None: + oprot.writeFieldBegin("newFunc", TType.STRUCT, 3) + self.newFunc.write(oprot) + oprot.writeFieldEnd() + oprot.writeFieldStop() + oprot.writeStructEnd() + + def validate(self): + return + + def __repr__(self): + L = ["{}={!r}".format(key, value) for key, value in self.__dict__.items()] + return "{}({})".format(self.__class__.__name__, ", ".join(L)) + + def __eq__(self, other): + return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ + + def __ne__(self, other): + return not (self == other) + + +all_structs.append(alter_function_args) +alter_function_args.thrift_spec = ( + None, # 0 + ( + 1, + TType.STRING, + "dbName", + "UTF8", + None, + ), # 1 + ( + 2, + TType.STRING, + "funcName", + "UTF8", + None, + ), # 2 + ( + 3, + TType.STRUCT, + "newFunc", + [Function, None], + None, + ), # 3 +) + + +class alter_function_result: + """ + Attributes: + - o1 + - o2 + + """ + + def __init__( + self, + o1=None, + o2=None, + ): + self.o1 = o1 + self.o2 = o2 + + def read(self, iprot): + if ( + iprot._fast_decode is not None + and isinstance(iprot.trans, TTransport.CReadableTransport) + and self.thrift_spec is not None + ): + iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec]) + return + iprot.readStructBegin() + while True: + (fname, ftype, fid) = iprot.readFieldBegin() + if ftype == TType.STOP: + break + if fid == 1: + if ftype == TType.STRUCT: + self.o1 = InvalidOperationException.read(iprot) + else: + iprot.skip(ftype) + elif fid == 2: + if ftype == TType.STRUCT: + self.o2 = MetaException.read(iprot) + else: + iprot.skip(ftype) + else: + iprot.skip(ftype) + iprot.readFieldEnd() + iprot.readStructEnd() + + def write(self, oprot): + if oprot._fast_encode is not None and self.thrift_spec is not None: + oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec])) + return + oprot.writeStructBegin("alter_function_result") + if self.o1 is not None: + oprot.writeFieldBegin("o1", TType.STRUCT, 1) + self.o1.write(oprot) + oprot.writeFieldEnd() + if self.o2 is not None: + oprot.writeFieldBegin("o2", TType.STRUCT, 2) + self.o2.write(oprot) + oprot.writeFieldEnd() + oprot.writeFieldStop() + oprot.writeStructEnd() + + def validate(self): + return + + def __repr__(self): + L = ["{}={!r}".format(key, value) for key, value in self.__dict__.items()] + return "{}({})".format(self.__class__.__name__, ", ".join(L)) + + def __eq__(self, other): + return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ + + def __ne__(self, other): + return not (self == other) + + +all_structs.append(alter_function_result) +alter_function_result.thrift_spec = ( + None, # 0 + ( + 1, + TType.STRUCT, + "o1", + [InvalidOperationException, None], + None, + ), # 1 + ( + 2, + TType.STRUCT, + "o2", + [MetaException, None], + None, + ), # 2 +) + + +class get_functions_args: + """ + Attributes: + - dbName + - pattern + + """ + + def __init__( + self, + dbName=None, + pattern=None, + ): + self.dbName = dbName + self.pattern = pattern + + def read(self, iprot): + if ( + iprot._fast_decode is not None + and isinstance(iprot.trans, TTransport.CReadableTransport) + and self.thrift_spec is not None + ): + iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec]) + return + iprot.readStructBegin() + while True: + (fname, ftype, fid) = iprot.readFieldBegin() + if ftype == TType.STOP: + break + if fid == 1: + if ftype == TType.STRING: + self.dbName = ( + iprot.readString().decode("utf-8", errors="replace") if sys.version_info[0] == 2 else iprot.readString() + ) + else: + iprot.skip(ftype) + elif fid == 2: + if ftype == TType.STRING: + self.pattern = ( + iprot.readString().decode("utf-8", errors="replace") if sys.version_info[0] == 2 else iprot.readString() + ) + else: + iprot.skip(ftype) + else: + iprot.skip(ftype) + iprot.readFieldEnd() + iprot.readStructEnd() + + def write(self, oprot): + if oprot._fast_encode is not None and self.thrift_spec is not None: + oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec])) + return + oprot.writeStructBegin("get_functions_args") + if self.dbName is not None: + oprot.writeFieldBegin("dbName", TType.STRING, 1) + oprot.writeString(self.dbName.encode("utf-8") if sys.version_info[0] == 2 else self.dbName) + oprot.writeFieldEnd() + if self.pattern is not None: + oprot.writeFieldBegin("pattern", TType.STRING, 2) + oprot.writeString(self.pattern.encode("utf-8") if sys.version_info[0] == 2 else self.pattern) + oprot.writeFieldEnd() + oprot.writeFieldStop() + oprot.writeStructEnd() + + def validate(self): + return + + def __repr__(self): + L = ["{}={!r}".format(key, value) for key, value in self.__dict__.items()] + return "{}({})".format(self.__class__.__name__, ", ".join(L)) + + def __eq__(self, other): + return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ + + def __ne__(self, other): + return not (self == other) + + +all_structs.append(get_functions_args) +get_functions_args.thrift_spec = ( + None, # 0 + ( + 1, + TType.STRING, + "dbName", + "UTF8", + None, + ), # 1 + ( + 2, + TType.STRING, + "pattern", + "UTF8", + None, + ), # 2 +) + + +class get_functions_result: + """ + Attributes: + - success + - o1 + + """ + + def __init__( + self, + success=None, + o1=None, + ): + self.success = success + self.o1 = o1 + + def read(self, iprot): + if ( + iprot._fast_decode is not None + and isinstance(iprot.trans, TTransport.CReadableTransport) + and self.thrift_spec is not None + ): + iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec]) + return + iprot.readStructBegin() + while True: + (fname, ftype, fid) = iprot.readFieldBegin() + if ftype == TType.STOP: + break + if fid == 0: + if ftype == TType.LIST: + self.success = [] + (_etype1705, _size1702) = iprot.readListBegin() + for _i1706 in range(_size1702): + _elem1707 = ( + iprot.readString().decode("utf-8", errors="replace") + if sys.version_info[0] == 2 + else iprot.readString() + ) + self.success.append(_elem1707) + iprot.readListEnd() + else: + iprot.skip(ftype) + elif fid == 1: + if ftype == TType.STRUCT: + self.o1 = MetaException.read(iprot) + else: + iprot.skip(ftype) + else: + iprot.skip(ftype) + iprot.readFieldEnd() + iprot.readStructEnd() + + def write(self, oprot): + if oprot._fast_encode is not None and self.thrift_spec is not None: + oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec])) + return + oprot.writeStructBegin("get_functions_result") + if self.success is not None: + oprot.writeFieldBegin("success", TType.LIST, 0) + oprot.writeListBegin(TType.STRING, len(self.success)) + for iter1708 in self.success: + oprot.writeString(iter1708.encode("utf-8") if sys.version_info[0] == 2 else iter1708) + oprot.writeListEnd() + oprot.writeFieldEnd() + if self.o1 is not None: + oprot.writeFieldBegin("o1", TType.STRUCT, 1) + self.o1.write(oprot) + oprot.writeFieldEnd() + oprot.writeFieldStop() + oprot.writeStructEnd() + + def validate(self): + return + + def __repr__(self): + L = ["{}={!r}".format(key, value) for key, value in self.__dict__.items()] + return "{}({})".format(self.__class__.__name__, ", ".join(L)) + + def __eq__(self, other): + return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ + + def __ne__(self, other): + return not (self == other) + + +all_structs.append(get_functions_result) +get_functions_result.thrift_spec = ( + ( + 0, + TType.LIST, + "success", + (TType.STRING, "UTF8", False), + None, + ), # 0 + ( + 1, + TType.STRUCT, + "o1", + [MetaException, None], + None, + ), # 1 +) + + +class get_function_args: + """ + Attributes: + - dbName + - funcName + + """ + + def __init__( + self, + dbName=None, + funcName=None, + ): + self.dbName = dbName + self.funcName = funcName + + def read(self, iprot): + if ( + iprot._fast_decode is not None + and isinstance(iprot.trans, TTransport.CReadableTransport) + and self.thrift_spec is not None + ): + iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec]) + return + iprot.readStructBegin() + while True: + (fname, ftype, fid) = iprot.readFieldBegin() + if ftype == TType.STOP: + break + if fid == 1: + if ftype == TType.STRING: + self.dbName = ( + iprot.readString().decode("utf-8", errors="replace") if sys.version_info[0] == 2 else iprot.readString() + ) + else: + iprot.skip(ftype) + elif fid == 2: + if ftype == TType.STRING: + self.funcName = ( + iprot.readString().decode("utf-8", errors="replace") if sys.version_info[0] == 2 else iprot.readString() + ) + else: + iprot.skip(ftype) + else: + iprot.skip(ftype) + iprot.readFieldEnd() + iprot.readStructEnd() + + def write(self, oprot): + if oprot._fast_encode is not None and self.thrift_spec is not None: + oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec])) + return + oprot.writeStructBegin("get_function_args") + if self.dbName is not None: + oprot.writeFieldBegin("dbName", TType.STRING, 1) + oprot.writeString(self.dbName.encode("utf-8") if sys.version_info[0] == 2 else self.dbName) + oprot.writeFieldEnd() + if self.funcName is not None: + oprot.writeFieldBegin("funcName", TType.STRING, 2) + oprot.writeString(self.funcName.encode("utf-8") if sys.version_info[0] == 2 else self.funcName) + oprot.writeFieldEnd() + oprot.writeFieldStop() + oprot.writeStructEnd() + + def validate(self): + return + + def __repr__(self): + L = ["{}={!r}".format(key, value) for key, value in self.__dict__.items()] + return "{}({})".format(self.__class__.__name__, ", ".join(L)) + + def __eq__(self, other): + return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ + + def __ne__(self, other): + return not (self == other) + + +all_structs.append(get_function_args) +get_function_args.thrift_spec = ( + None, # 0 + ( + 1, + TType.STRING, + "dbName", + "UTF8", + None, + ), # 1 + ( + 2, + TType.STRING, + "funcName", + "UTF8", + None, + ), # 2 +) + + +class get_function_result: + """ + Attributes: + - success + - o1 + - o2 + + """ + + def __init__( + self, + success=None, + o1=None, + o2=None, + ): + self.success = success + self.o1 = o1 + self.o2 = o2 + + def read(self, iprot): + if ( + iprot._fast_decode is not None + and isinstance(iprot.trans, TTransport.CReadableTransport) + and self.thrift_spec is not None + ): + iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec]) + return + iprot.readStructBegin() + while True: + (fname, ftype, fid) = iprot.readFieldBegin() + if ftype == TType.STOP: + break + if fid == 0: + if ftype == TType.STRUCT: + self.success = Function() + self.success.read(iprot) + else: + iprot.skip(ftype) + elif fid == 1: + if ftype == TType.STRUCT: + self.o1 = MetaException.read(iprot) + else: + iprot.skip(ftype) + elif fid == 2: + if ftype == TType.STRUCT: + self.o2 = NoSuchObjectException.read(iprot) + else: + iprot.skip(ftype) + else: + iprot.skip(ftype) + iprot.readFieldEnd() + iprot.readStructEnd() + + def write(self, oprot): + if oprot._fast_encode is not None and self.thrift_spec is not None: + oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec])) + return + oprot.writeStructBegin("get_function_result") + if self.success is not None: + oprot.writeFieldBegin("success", TType.STRUCT, 0) + self.success.write(oprot) + oprot.writeFieldEnd() + if self.o1 is not None: + oprot.writeFieldBegin("o1", TType.STRUCT, 1) + self.o1.write(oprot) + oprot.writeFieldEnd() + if self.o2 is not None: + oprot.writeFieldBegin("o2", TType.STRUCT, 2) + self.o2.write(oprot) + oprot.writeFieldEnd() + oprot.writeFieldStop() + oprot.writeStructEnd() + + def validate(self): + return + + def __repr__(self): + L = ["{}={!r}".format(key, value) for key, value in self.__dict__.items()] + return "{}({})".format(self.__class__.__name__, ", ".join(L)) + + def __eq__(self, other): + return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ + + def __ne__(self, other): + return not (self == other) + + +all_structs.append(get_function_result) +get_function_result.thrift_spec = ( + ( + 0, + TType.STRUCT, + "success", + [Function, None], + None, + ), # 0 + ( + 1, + TType.STRUCT, + "o1", + [MetaException, None], + None, + ), # 1 + ( + 2, + TType.STRUCT, + "o2", + [NoSuchObjectException, None], + None, + ), # 2 +) + + +class get_all_functions_args: + def read(self, iprot): + if ( + iprot._fast_decode is not None + and isinstance(iprot.trans, TTransport.CReadableTransport) + and self.thrift_spec is not None + ): + iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec]) + return + iprot.readStructBegin() + while True: + (fname, ftype, fid) = iprot.readFieldBegin() + if ftype == TType.STOP: + break + else: + iprot.skip(ftype) + iprot.readFieldEnd() + iprot.readStructEnd() + + def write(self, oprot): + if oprot._fast_encode is not None and self.thrift_spec is not None: + oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec])) + return + oprot.writeStructBegin("get_all_functions_args") + oprot.writeFieldStop() + oprot.writeStructEnd() + + def validate(self): + return + + def __repr__(self): + L = ["{}={!r}".format(key, value) for key, value in self.__dict__.items()] + return "{}({})".format(self.__class__.__name__, ", ".join(L)) + + def __eq__(self, other): + return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ + + def __ne__(self, other): + return not (self == other) + + +all_structs.append(get_all_functions_args) +get_all_functions_args.thrift_spec = () + + +class get_all_functions_result: + """ + Attributes: + - success + - o1 + + """ + + def __init__( + self, + success=None, + o1=None, + ): + self.success = success + self.o1 = o1 + + def read(self, iprot): + if ( + iprot._fast_decode is not None + and isinstance(iprot.trans, TTransport.CReadableTransport) + and self.thrift_spec is not None + ): + iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec]) + return + iprot.readStructBegin() + while True: + (fname, ftype, fid) = iprot.readFieldBegin() + if ftype == TType.STOP: + break + if fid == 0: + if ftype == TType.STRUCT: + self.success = GetAllFunctionsResponse() + self.success.read(iprot) + else: + iprot.skip(ftype) + elif fid == 1: + if ftype == TType.STRUCT: + self.o1 = MetaException.read(iprot) + else: + iprot.skip(ftype) + else: + iprot.skip(ftype) + iprot.readFieldEnd() + iprot.readStructEnd() + + def write(self, oprot): + if oprot._fast_encode is not None and self.thrift_spec is not None: + oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec])) + return + oprot.writeStructBegin("get_all_functions_result") + if self.success is not None: + oprot.writeFieldBegin("success", TType.STRUCT, 0) + self.success.write(oprot) + oprot.writeFieldEnd() + if self.o1 is not None: + oprot.writeFieldBegin("o1", TType.STRUCT, 1) + self.o1.write(oprot) + oprot.writeFieldEnd() + oprot.writeFieldStop() + oprot.writeStructEnd() + + def validate(self): + return + + def __repr__(self): + L = ["{}={!r}".format(key, value) for key, value in self.__dict__.items()] + return "{}({})".format(self.__class__.__name__, ", ".join(L)) + + def __eq__(self, other): + return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ + + def __ne__(self, other): + return not (self == other) + + +all_structs.append(get_all_functions_result) +get_all_functions_result.thrift_spec = ( + ( + 0, + TType.STRUCT, + "success", + [GetAllFunctionsResponse, None], + None, + ), # 0 + ( + 1, + TType.STRUCT, + "o1", + [MetaException, None], + None, + ), # 1 +) + + +class create_role_args: + """ + Attributes: + - role + + """ + + def __init__( + self, + role=None, + ): + self.role = role + + def read(self, iprot): + if ( + iprot._fast_decode is not None + and isinstance(iprot.trans, TTransport.CReadableTransport) + and self.thrift_spec is not None + ): + iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec]) + return + iprot.readStructBegin() + while True: + (fname, ftype, fid) = iprot.readFieldBegin() + if ftype == TType.STOP: + break + if fid == 1: + if ftype == TType.STRUCT: + self.role = Role() + self.role.read(iprot) + else: + iprot.skip(ftype) + else: + iprot.skip(ftype) + iprot.readFieldEnd() + iprot.readStructEnd() + + def write(self, oprot): + if oprot._fast_encode is not None and self.thrift_spec is not None: + oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec])) + return + oprot.writeStructBegin("create_role_args") + if self.role is not None: + oprot.writeFieldBegin("role", TType.STRUCT, 1) + self.role.write(oprot) + oprot.writeFieldEnd() + oprot.writeFieldStop() + oprot.writeStructEnd() + + def validate(self): + return + + def __repr__(self): + L = ["{}={!r}".format(key, value) for key, value in self.__dict__.items()] + return "{}({})".format(self.__class__.__name__, ", ".join(L)) + + def __eq__(self, other): + return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ + + def __ne__(self, other): + return not (self == other) + + +all_structs.append(create_role_args) +create_role_args.thrift_spec = ( + None, # 0 + ( + 1, + TType.STRUCT, + "role", + [Role, None], + None, + ), # 1 +) + + +class create_role_result: + """ + Attributes: + - success + - o1 + + """ + + def __init__( + self, + success=None, + o1=None, + ): + self.success = success + self.o1 = o1 + + def read(self, iprot): + if ( + iprot._fast_decode is not None + and isinstance(iprot.trans, TTransport.CReadableTransport) + and self.thrift_spec is not None + ): + iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec]) + return + iprot.readStructBegin() + while True: + (fname, ftype, fid) = iprot.readFieldBegin() + if ftype == TType.STOP: + break + if fid == 0: + if ftype == TType.BOOL: + self.success = iprot.readBool() + else: + iprot.skip(ftype) + elif fid == 1: + if ftype == TType.STRUCT: + self.o1 = MetaException.read(iprot) + else: + iprot.skip(ftype) + else: + iprot.skip(ftype) + iprot.readFieldEnd() + iprot.readStructEnd() + + def write(self, oprot): + if oprot._fast_encode is not None and self.thrift_spec is not None: + oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec])) + return + oprot.writeStructBegin("create_role_result") + if self.success is not None: + oprot.writeFieldBegin("success", TType.BOOL, 0) + oprot.writeBool(self.success) + oprot.writeFieldEnd() + if self.o1 is not None: + oprot.writeFieldBegin("o1", TType.STRUCT, 1) + self.o1.write(oprot) + oprot.writeFieldEnd() + oprot.writeFieldStop() + oprot.writeStructEnd() + + def validate(self): + return + + def __repr__(self): + L = ["{}={!r}".format(key, value) for key, value in self.__dict__.items()] + return "{}({})".format(self.__class__.__name__, ", ".join(L)) + + def __eq__(self, other): + return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ + + def __ne__(self, other): + return not (self == other) + + +all_structs.append(create_role_result) +create_role_result.thrift_spec = ( + ( + 0, + TType.BOOL, + "success", + None, + None, + ), # 0 + ( + 1, + TType.STRUCT, + "o1", + [MetaException, None], + None, + ), # 1 +) + + +class drop_role_args: + """ + Attributes: + - role_name + + """ + + def __init__( + self, + role_name=None, + ): + self.role_name = role_name + + def read(self, iprot): + if ( + iprot._fast_decode is not None + and isinstance(iprot.trans, TTransport.CReadableTransport) + and self.thrift_spec is not None + ): + iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec]) + return + iprot.readStructBegin() + while True: + (fname, ftype, fid) = iprot.readFieldBegin() + if ftype == TType.STOP: + break + if fid == 1: + if ftype == TType.STRING: + self.role_name = ( + iprot.readString().decode("utf-8", errors="replace") if sys.version_info[0] == 2 else iprot.readString() + ) + else: + iprot.skip(ftype) + else: + iprot.skip(ftype) + iprot.readFieldEnd() + iprot.readStructEnd() + + def write(self, oprot): + if oprot._fast_encode is not None and self.thrift_spec is not None: + oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec])) + return + oprot.writeStructBegin("drop_role_args") + if self.role_name is not None: + oprot.writeFieldBegin("role_name", TType.STRING, 1) + oprot.writeString(self.role_name.encode("utf-8") if sys.version_info[0] == 2 else self.role_name) + oprot.writeFieldEnd() + oprot.writeFieldStop() + oprot.writeStructEnd() + + def validate(self): + return + + def __repr__(self): + L = ["{}={!r}".format(key, value) for key, value in self.__dict__.items()] + return "{}({})".format(self.__class__.__name__, ", ".join(L)) + + def __eq__(self, other): + return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ + + def __ne__(self, other): + return not (self == other) + + +all_structs.append(drop_role_args) +drop_role_args.thrift_spec = ( + None, # 0 + ( + 1, + TType.STRING, + "role_name", + "UTF8", + None, + ), # 1 +) + + +class drop_role_result: + """ + Attributes: + - success + - o1 + + """ + + def __init__( + self, + success=None, + o1=None, + ): + self.success = success + self.o1 = o1 + + def read(self, iprot): + if ( + iprot._fast_decode is not None + and isinstance(iprot.trans, TTransport.CReadableTransport) + and self.thrift_spec is not None + ): + iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec]) + return + iprot.readStructBegin() + while True: + (fname, ftype, fid) = iprot.readFieldBegin() + if ftype == TType.STOP: + break + if fid == 0: + if ftype == TType.BOOL: + self.success = iprot.readBool() + else: + iprot.skip(ftype) + elif fid == 1: + if ftype == TType.STRUCT: + self.o1 = MetaException.read(iprot) + else: + iprot.skip(ftype) + else: + iprot.skip(ftype) + iprot.readFieldEnd() + iprot.readStructEnd() + + def write(self, oprot): + if oprot._fast_encode is not None and self.thrift_spec is not None: + oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec])) + return + oprot.writeStructBegin("drop_role_result") + if self.success is not None: + oprot.writeFieldBegin("success", TType.BOOL, 0) + oprot.writeBool(self.success) + oprot.writeFieldEnd() + if self.o1 is not None: + oprot.writeFieldBegin("o1", TType.STRUCT, 1) + self.o1.write(oprot) + oprot.writeFieldEnd() + oprot.writeFieldStop() + oprot.writeStructEnd() + + def validate(self): + return + + def __repr__(self): + L = ["{}={!r}".format(key, value) for key, value in self.__dict__.items()] + return "{}({})".format(self.__class__.__name__, ", ".join(L)) + + def __eq__(self, other): + return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ + + def __ne__(self, other): + return not (self == other) + + +all_structs.append(drop_role_result) +drop_role_result.thrift_spec = ( + ( + 0, + TType.BOOL, + "success", + None, + None, + ), # 0 + ( + 1, + TType.STRUCT, + "o1", + [MetaException, None], + None, + ), # 1 +) + + +class get_role_names_args: + def read(self, iprot): + if ( + iprot._fast_decode is not None + and isinstance(iprot.trans, TTransport.CReadableTransport) + and self.thrift_spec is not None + ): + iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec]) + return + iprot.readStructBegin() + while True: + (fname, ftype, fid) = iprot.readFieldBegin() + if ftype == TType.STOP: + break + else: + iprot.skip(ftype) + iprot.readFieldEnd() + iprot.readStructEnd() + + def write(self, oprot): + if oprot._fast_encode is not None and self.thrift_spec is not None: + oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec])) + return + oprot.writeStructBegin("get_role_names_args") + oprot.writeFieldStop() + oprot.writeStructEnd() + + def validate(self): + return + + def __repr__(self): + L = ["{}={!r}".format(key, value) for key, value in self.__dict__.items()] + return "{}({})".format(self.__class__.__name__, ", ".join(L)) + + def __eq__(self, other): + return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ + + def __ne__(self, other): + return not (self == other) + + +all_structs.append(get_role_names_args) +get_role_names_args.thrift_spec = () + + +class get_role_names_result: + """ + Attributes: + - success + - o1 + + """ + + def __init__( + self, + success=None, + o1=None, + ): + self.success = success + self.o1 = o1 + + def read(self, iprot): + if ( + iprot._fast_decode is not None + and isinstance(iprot.trans, TTransport.CReadableTransport) + and self.thrift_spec is not None + ): + iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec]) + return + iprot.readStructBegin() + while True: + (fname, ftype, fid) = iprot.readFieldBegin() + if ftype == TType.STOP: + break + if fid == 0: + if ftype == TType.LIST: + self.success = [] + (_etype1712, _size1709) = iprot.readListBegin() + for _i1713 in range(_size1709): + _elem1714 = ( + iprot.readString().decode("utf-8", errors="replace") + if sys.version_info[0] == 2 + else iprot.readString() + ) + self.success.append(_elem1714) + iprot.readListEnd() + else: + iprot.skip(ftype) + elif fid == 1: + if ftype == TType.STRUCT: + self.o1 = MetaException.read(iprot) + else: + iprot.skip(ftype) + else: + iprot.skip(ftype) + iprot.readFieldEnd() + iprot.readStructEnd() + + def write(self, oprot): + if oprot._fast_encode is not None and self.thrift_spec is not None: + oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec])) + return + oprot.writeStructBegin("get_role_names_result") + if self.success is not None: + oprot.writeFieldBegin("success", TType.LIST, 0) + oprot.writeListBegin(TType.STRING, len(self.success)) + for iter1715 in self.success: + oprot.writeString(iter1715.encode("utf-8") if sys.version_info[0] == 2 else iter1715) + oprot.writeListEnd() + oprot.writeFieldEnd() + if self.o1 is not None: + oprot.writeFieldBegin("o1", TType.STRUCT, 1) + self.o1.write(oprot) + oprot.writeFieldEnd() + oprot.writeFieldStop() + oprot.writeStructEnd() + + def validate(self): + return + + def __repr__(self): + L = ["{}={!r}".format(key, value) for key, value in self.__dict__.items()] + return "{}({})".format(self.__class__.__name__, ", ".join(L)) + + def __eq__(self, other): + return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ + + def __ne__(self, other): + return not (self == other) + + +all_structs.append(get_role_names_result) +get_role_names_result.thrift_spec = ( + ( + 0, + TType.LIST, + "success", + (TType.STRING, "UTF8", False), + None, + ), # 0 + ( + 1, + TType.STRUCT, + "o1", + [MetaException, None], + None, + ), # 1 +) + + +class grant_role_args: + """ + Attributes: + - role_name + - principal_name + - principal_type + - grantor + - grantorType + - grant_option + + """ + + def __init__( + self, + role_name=None, + principal_name=None, + principal_type=None, + grantor=None, + grantorType=None, + grant_option=None, + ): + self.role_name = role_name + self.principal_name = principal_name + self.principal_type = principal_type + self.grantor = grantor + self.grantorType = grantorType + self.grant_option = grant_option + + def read(self, iprot): + if ( + iprot._fast_decode is not None + and isinstance(iprot.trans, TTransport.CReadableTransport) + and self.thrift_spec is not None + ): + iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec]) + return + iprot.readStructBegin() + while True: + (fname, ftype, fid) = iprot.readFieldBegin() + if ftype == TType.STOP: + break + if fid == 1: + if ftype == TType.STRING: + self.role_name = ( + iprot.readString().decode("utf-8", errors="replace") if sys.version_info[0] == 2 else iprot.readString() + ) + else: + iprot.skip(ftype) + elif fid == 2: + if ftype == TType.STRING: + self.principal_name = ( + iprot.readString().decode("utf-8", errors="replace") if sys.version_info[0] == 2 else iprot.readString() + ) + else: + iprot.skip(ftype) + elif fid == 3: + if ftype == TType.I32: + self.principal_type = iprot.readI32() + else: + iprot.skip(ftype) + elif fid == 4: + if ftype == TType.STRING: + self.grantor = ( + iprot.readString().decode("utf-8", errors="replace") if sys.version_info[0] == 2 else iprot.readString() + ) + else: + iprot.skip(ftype) + elif fid == 5: + if ftype == TType.I32: + self.grantorType = iprot.readI32() + else: + iprot.skip(ftype) + elif fid == 6: + if ftype == TType.BOOL: + self.grant_option = iprot.readBool() + else: + iprot.skip(ftype) + else: + iprot.skip(ftype) + iprot.readFieldEnd() + iprot.readStructEnd() + + def write(self, oprot): + if oprot._fast_encode is not None and self.thrift_spec is not None: + oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec])) + return + oprot.writeStructBegin("grant_role_args") + if self.role_name is not None: + oprot.writeFieldBegin("role_name", TType.STRING, 1) + oprot.writeString(self.role_name.encode("utf-8") if sys.version_info[0] == 2 else self.role_name) + oprot.writeFieldEnd() + if self.principal_name is not None: + oprot.writeFieldBegin("principal_name", TType.STRING, 2) + oprot.writeString(self.principal_name.encode("utf-8") if sys.version_info[0] == 2 else self.principal_name) + oprot.writeFieldEnd() + if self.principal_type is not None: + oprot.writeFieldBegin("principal_type", TType.I32, 3) + oprot.writeI32(self.principal_type) + oprot.writeFieldEnd() + if self.grantor is not None: + oprot.writeFieldBegin("grantor", TType.STRING, 4) + oprot.writeString(self.grantor.encode("utf-8") if sys.version_info[0] == 2 else self.grantor) + oprot.writeFieldEnd() + if self.grantorType is not None: + oprot.writeFieldBegin("grantorType", TType.I32, 5) + oprot.writeI32(self.grantorType) + oprot.writeFieldEnd() + if self.grant_option is not None: + oprot.writeFieldBegin("grant_option", TType.BOOL, 6) + oprot.writeBool(self.grant_option) + oprot.writeFieldEnd() + oprot.writeFieldStop() + oprot.writeStructEnd() + + def validate(self): + return + + def __repr__(self): + L = ["{}={!r}".format(key, value) for key, value in self.__dict__.items()] + return "{}({})".format(self.__class__.__name__, ", ".join(L)) + + def __eq__(self, other): + return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ + + def __ne__(self, other): + return not (self == other) + + +all_structs.append(grant_role_args) +grant_role_args.thrift_spec = ( + None, # 0 + ( + 1, + TType.STRING, + "role_name", + "UTF8", + None, + ), # 1 + ( + 2, + TType.STRING, + "principal_name", + "UTF8", + None, + ), # 2 + ( + 3, + TType.I32, + "principal_type", + None, + None, + ), # 3 + ( + 4, + TType.STRING, + "grantor", + "UTF8", + None, + ), # 4 + ( + 5, + TType.I32, + "grantorType", + None, + None, + ), # 5 + ( + 6, + TType.BOOL, + "grant_option", + None, + None, + ), # 6 +) + + +class grant_role_result: + """ + Attributes: + - success + - o1 + + """ + + def __init__( + self, + success=None, + o1=None, + ): + self.success = success + self.o1 = o1 + + def read(self, iprot): + if ( + iprot._fast_decode is not None + and isinstance(iprot.trans, TTransport.CReadableTransport) + and self.thrift_spec is not None + ): + iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec]) + return + iprot.readStructBegin() + while True: + (fname, ftype, fid) = iprot.readFieldBegin() + if ftype == TType.STOP: + break + if fid == 0: + if ftype == TType.BOOL: + self.success = iprot.readBool() + else: + iprot.skip(ftype) + elif fid == 1: + if ftype == TType.STRUCT: + self.o1 = MetaException.read(iprot) + else: + iprot.skip(ftype) + else: + iprot.skip(ftype) + iprot.readFieldEnd() + iprot.readStructEnd() + + def write(self, oprot): + if oprot._fast_encode is not None and self.thrift_spec is not None: + oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec])) + return + oprot.writeStructBegin("grant_role_result") + if self.success is not None: + oprot.writeFieldBegin("success", TType.BOOL, 0) + oprot.writeBool(self.success) + oprot.writeFieldEnd() + if self.o1 is not None: + oprot.writeFieldBegin("o1", TType.STRUCT, 1) + self.o1.write(oprot) + oprot.writeFieldEnd() + oprot.writeFieldStop() + oprot.writeStructEnd() + + def validate(self): + return + + def __repr__(self): + L = ["{}={!r}".format(key, value) for key, value in self.__dict__.items()] + return "{}({})".format(self.__class__.__name__, ", ".join(L)) + + def __eq__(self, other): + return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ + + def __ne__(self, other): + return not (self == other) + + +all_structs.append(grant_role_result) +grant_role_result.thrift_spec = ( + ( + 0, + TType.BOOL, + "success", + None, + None, + ), # 0 + ( + 1, + TType.STRUCT, + "o1", + [MetaException, None], + None, + ), # 1 +) + + +class revoke_role_args: + """ + Attributes: + - role_name + - principal_name + - principal_type + + """ + + def __init__( + self, + role_name=None, + principal_name=None, + principal_type=None, + ): + self.role_name = role_name + self.principal_name = principal_name + self.principal_type = principal_type + + def read(self, iprot): + if ( + iprot._fast_decode is not None + and isinstance(iprot.trans, TTransport.CReadableTransport) + and self.thrift_spec is not None + ): + iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec]) + return + iprot.readStructBegin() + while True: + (fname, ftype, fid) = iprot.readFieldBegin() + if ftype == TType.STOP: + break + if fid == 1: + if ftype == TType.STRING: + self.role_name = ( + iprot.readString().decode("utf-8", errors="replace") if sys.version_info[0] == 2 else iprot.readString() + ) + else: + iprot.skip(ftype) + elif fid == 2: + if ftype == TType.STRING: + self.principal_name = ( + iprot.readString().decode("utf-8", errors="replace") if sys.version_info[0] == 2 else iprot.readString() + ) + else: + iprot.skip(ftype) + elif fid == 3: + if ftype == TType.I32: + self.principal_type = iprot.readI32() + else: + iprot.skip(ftype) + else: + iprot.skip(ftype) + iprot.readFieldEnd() + iprot.readStructEnd() + + def write(self, oprot): + if oprot._fast_encode is not None and self.thrift_spec is not None: + oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec])) + return + oprot.writeStructBegin("revoke_role_args") + if self.role_name is not None: + oprot.writeFieldBegin("role_name", TType.STRING, 1) + oprot.writeString(self.role_name.encode("utf-8") if sys.version_info[0] == 2 else self.role_name) + oprot.writeFieldEnd() + if self.principal_name is not None: + oprot.writeFieldBegin("principal_name", TType.STRING, 2) + oprot.writeString(self.principal_name.encode("utf-8") if sys.version_info[0] == 2 else self.principal_name) + oprot.writeFieldEnd() + if self.principal_type is not None: + oprot.writeFieldBegin("principal_type", TType.I32, 3) + oprot.writeI32(self.principal_type) + oprot.writeFieldEnd() + oprot.writeFieldStop() + oprot.writeStructEnd() + + def validate(self): + return + + def __repr__(self): + L = ["{}={!r}".format(key, value) for key, value in self.__dict__.items()] + return "{}({})".format(self.__class__.__name__, ", ".join(L)) + + def __eq__(self, other): + return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ + + def __ne__(self, other): + return not (self == other) + + +all_structs.append(revoke_role_args) +revoke_role_args.thrift_spec = ( + None, # 0 + ( + 1, + TType.STRING, + "role_name", + "UTF8", + None, + ), # 1 + ( + 2, + TType.STRING, + "principal_name", + "UTF8", + None, + ), # 2 + ( + 3, + TType.I32, + "principal_type", + None, + None, + ), # 3 +) + + +class revoke_role_result: + """ + Attributes: + - success + - o1 + + """ + + def __init__( + self, + success=None, + o1=None, + ): + self.success = success + self.o1 = o1 + + def read(self, iprot): + if ( + iprot._fast_decode is not None + and isinstance(iprot.trans, TTransport.CReadableTransport) + and self.thrift_spec is not None + ): + iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec]) + return + iprot.readStructBegin() + while True: + (fname, ftype, fid) = iprot.readFieldBegin() + if ftype == TType.STOP: + break + if fid == 0: + if ftype == TType.BOOL: + self.success = iprot.readBool() + else: + iprot.skip(ftype) + elif fid == 1: + if ftype == TType.STRUCT: + self.o1 = MetaException.read(iprot) + else: + iprot.skip(ftype) + else: + iprot.skip(ftype) + iprot.readFieldEnd() + iprot.readStructEnd() + + def write(self, oprot): + if oprot._fast_encode is not None and self.thrift_spec is not None: + oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec])) + return + oprot.writeStructBegin("revoke_role_result") + if self.success is not None: + oprot.writeFieldBegin("success", TType.BOOL, 0) + oprot.writeBool(self.success) + oprot.writeFieldEnd() + if self.o1 is not None: + oprot.writeFieldBegin("o1", TType.STRUCT, 1) + self.o1.write(oprot) + oprot.writeFieldEnd() + oprot.writeFieldStop() + oprot.writeStructEnd() + + def validate(self): + return + + def __repr__(self): + L = ["{}={!r}".format(key, value) for key, value in self.__dict__.items()] + return "{}({})".format(self.__class__.__name__, ", ".join(L)) + + def __eq__(self, other): + return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ + + def __ne__(self, other): + return not (self == other) + + +all_structs.append(revoke_role_result) +revoke_role_result.thrift_spec = ( + ( + 0, + TType.BOOL, + "success", + None, + None, + ), # 0 + ( + 1, + TType.STRUCT, + "o1", + [MetaException, None], + None, + ), # 1 +) + + +class list_roles_args: + """ + Attributes: + - principal_name + - principal_type + + """ + + def __init__( + self, + principal_name=None, + principal_type=None, + ): + self.principal_name = principal_name + self.principal_type = principal_type + + def read(self, iprot): + if ( + iprot._fast_decode is not None + and isinstance(iprot.trans, TTransport.CReadableTransport) + and self.thrift_spec is not None + ): + iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec]) + return + iprot.readStructBegin() + while True: + (fname, ftype, fid) = iprot.readFieldBegin() + if ftype == TType.STOP: + break + if fid == 1: + if ftype == TType.STRING: + self.principal_name = ( + iprot.readString().decode("utf-8", errors="replace") if sys.version_info[0] == 2 else iprot.readString() + ) + else: + iprot.skip(ftype) + elif fid == 2: + if ftype == TType.I32: + self.principal_type = iprot.readI32() + else: + iprot.skip(ftype) + else: + iprot.skip(ftype) + iprot.readFieldEnd() + iprot.readStructEnd() + + def write(self, oprot): + if oprot._fast_encode is not None and self.thrift_spec is not None: + oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec])) + return + oprot.writeStructBegin("list_roles_args") + if self.principal_name is not None: + oprot.writeFieldBegin("principal_name", TType.STRING, 1) + oprot.writeString(self.principal_name.encode("utf-8") if sys.version_info[0] == 2 else self.principal_name) + oprot.writeFieldEnd() + if self.principal_type is not None: + oprot.writeFieldBegin("principal_type", TType.I32, 2) + oprot.writeI32(self.principal_type) + oprot.writeFieldEnd() + oprot.writeFieldStop() + oprot.writeStructEnd() + + def validate(self): + return + + def __repr__(self): + L = ["{}={!r}".format(key, value) for key, value in self.__dict__.items()] + return "{}({})".format(self.__class__.__name__, ", ".join(L)) + + def __eq__(self, other): + return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ + + def __ne__(self, other): + return not (self == other) + + +all_structs.append(list_roles_args) +list_roles_args.thrift_spec = ( + None, # 0 + ( + 1, + TType.STRING, + "principal_name", + "UTF8", + None, + ), # 1 + ( + 2, + TType.I32, + "principal_type", + None, + None, + ), # 2 +) + + +class list_roles_result: + """ + Attributes: + - success + - o1 + + """ + + def __init__( + self, + success=None, + o1=None, + ): + self.success = success + self.o1 = o1 + + def read(self, iprot): + if ( + iprot._fast_decode is not None + and isinstance(iprot.trans, TTransport.CReadableTransport) + and self.thrift_spec is not None + ): + iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec]) + return + iprot.readStructBegin() + while True: + (fname, ftype, fid) = iprot.readFieldBegin() + if ftype == TType.STOP: + break + if fid == 0: + if ftype == TType.LIST: + self.success = [] + (_etype1719, _size1716) = iprot.readListBegin() + for _i1720 in range(_size1716): + _elem1721 = Role() + _elem1721.read(iprot) + self.success.append(_elem1721) + iprot.readListEnd() + else: + iprot.skip(ftype) + elif fid == 1: + if ftype == TType.STRUCT: + self.o1 = MetaException.read(iprot) + else: + iprot.skip(ftype) + else: + iprot.skip(ftype) + iprot.readFieldEnd() + iprot.readStructEnd() + + def write(self, oprot): + if oprot._fast_encode is not None and self.thrift_spec is not None: + oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec])) + return + oprot.writeStructBegin("list_roles_result") + if self.success is not None: + oprot.writeFieldBegin("success", TType.LIST, 0) + oprot.writeListBegin(TType.STRUCT, len(self.success)) + for iter1722 in self.success: + iter1722.write(oprot) + oprot.writeListEnd() + oprot.writeFieldEnd() + if self.o1 is not None: + oprot.writeFieldBegin("o1", TType.STRUCT, 1) + self.o1.write(oprot) + oprot.writeFieldEnd() + oprot.writeFieldStop() + oprot.writeStructEnd() + + def validate(self): + return + + def __repr__(self): + L = ["{}={!r}".format(key, value) for key, value in self.__dict__.items()] + return "{}({})".format(self.__class__.__name__, ", ".join(L)) + + def __eq__(self, other): + return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ + + def __ne__(self, other): + return not (self == other) + + +all_structs.append(list_roles_result) +list_roles_result.thrift_spec = ( + ( + 0, + TType.LIST, + "success", + (TType.STRUCT, [Role, None], False), + None, + ), # 0 + ( + 1, + TType.STRUCT, + "o1", + [MetaException, None], + None, + ), # 1 +) + + +class grant_revoke_role_args: + """ + Attributes: + - request + + """ + + def __init__( + self, + request=None, + ): + self.request = request + + def read(self, iprot): + if ( + iprot._fast_decode is not None + and isinstance(iprot.trans, TTransport.CReadableTransport) + and self.thrift_spec is not None + ): + iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec]) + return + iprot.readStructBegin() + while True: + (fname, ftype, fid) = iprot.readFieldBegin() + if ftype == TType.STOP: + break + if fid == 1: + if ftype == TType.STRUCT: + self.request = GrantRevokeRoleRequest() + self.request.read(iprot) + else: + iprot.skip(ftype) + else: + iprot.skip(ftype) + iprot.readFieldEnd() + iprot.readStructEnd() + + def write(self, oprot): + if oprot._fast_encode is not None and self.thrift_spec is not None: + oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec])) + return + oprot.writeStructBegin("grant_revoke_role_args") + if self.request is not None: + oprot.writeFieldBegin("request", TType.STRUCT, 1) + self.request.write(oprot) + oprot.writeFieldEnd() + oprot.writeFieldStop() + oprot.writeStructEnd() + + def validate(self): + return + + def __repr__(self): + L = ["{}={!r}".format(key, value) for key, value in self.__dict__.items()] + return "{}({})".format(self.__class__.__name__, ", ".join(L)) + + def __eq__(self, other): + return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ + + def __ne__(self, other): + return not (self == other) + + +all_structs.append(grant_revoke_role_args) +grant_revoke_role_args.thrift_spec = ( + None, # 0 + ( + 1, + TType.STRUCT, + "request", + [GrantRevokeRoleRequest, None], + None, + ), # 1 +) + + +class grant_revoke_role_result: + """ + Attributes: + - success + - o1 + + """ + + def __init__( + self, + success=None, + o1=None, + ): + self.success = success + self.o1 = o1 + + def read(self, iprot): + if ( + iprot._fast_decode is not None + and isinstance(iprot.trans, TTransport.CReadableTransport) + and self.thrift_spec is not None + ): + iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec]) + return + iprot.readStructBegin() + while True: + (fname, ftype, fid) = iprot.readFieldBegin() + if ftype == TType.STOP: + break + if fid == 0: + if ftype == TType.STRUCT: + self.success = GrantRevokeRoleResponse() + self.success.read(iprot) + else: + iprot.skip(ftype) + elif fid == 1: + if ftype == TType.STRUCT: + self.o1 = MetaException.read(iprot) + else: + iprot.skip(ftype) + else: + iprot.skip(ftype) + iprot.readFieldEnd() + iprot.readStructEnd() + + def write(self, oprot): + if oprot._fast_encode is not None and self.thrift_spec is not None: + oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec])) + return + oprot.writeStructBegin("grant_revoke_role_result") + if self.success is not None: + oprot.writeFieldBegin("success", TType.STRUCT, 0) + self.success.write(oprot) + oprot.writeFieldEnd() + if self.o1 is not None: + oprot.writeFieldBegin("o1", TType.STRUCT, 1) + self.o1.write(oprot) + oprot.writeFieldEnd() + oprot.writeFieldStop() + oprot.writeStructEnd() + + def validate(self): + return + + def __repr__(self): + L = ["{}={!r}".format(key, value) for key, value in self.__dict__.items()] + return "{}({})".format(self.__class__.__name__, ", ".join(L)) + + def __eq__(self, other): + return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ + + def __ne__(self, other): + return not (self == other) + + +all_structs.append(grant_revoke_role_result) +grant_revoke_role_result.thrift_spec = ( + ( + 0, + TType.STRUCT, + "success", + [GrantRevokeRoleResponse, None], + None, + ), # 0 + ( + 1, + TType.STRUCT, + "o1", + [MetaException, None], + None, + ), # 1 +) + + +class get_principals_in_role_args: + """ + Attributes: + - request + + """ + + def __init__( + self, + request=None, + ): + self.request = request + + def read(self, iprot): + if ( + iprot._fast_decode is not None + and isinstance(iprot.trans, TTransport.CReadableTransport) + and self.thrift_spec is not None + ): + iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec]) + return + iprot.readStructBegin() + while True: + (fname, ftype, fid) = iprot.readFieldBegin() + if ftype == TType.STOP: + break + if fid == 1: + if ftype == TType.STRUCT: + self.request = GetPrincipalsInRoleRequest() + self.request.read(iprot) + else: + iprot.skip(ftype) + else: + iprot.skip(ftype) + iprot.readFieldEnd() + iprot.readStructEnd() + + def write(self, oprot): + if oprot._fast_encode is not None and self.thrift_spec is not None: + oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec])) + return + oprot.writeStructBegin("get_principals_in_role_args") + if self.request is not None: + oprot.writeFieldBegin("request", TType.STRUCT, 1) + self.request.write(oprot) + oprot.writeFieldEnd() + oprot.writeFieldStop() + oprot.writeStructEnd() + + def validate(self): + return + + def __repr__(self): + L = ["{}={!r}".format(key, value) for key, value in self.__dict__.items()] + return "{}({})".format(self.__class__.__name__, ", ".join(L)) + + def __eq__(self, other): + return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ + + def __ne__(self, other): + return not (self == other) + + +all_structs.append(get_principals_in_role_args) +get_principals_in_role_args.thrift_spec = ( + None, # 0 + ( + 1, + TType.STRUCT, + "request", + [GetPrincipalsInRoleRequest, None], + None, + ), # 1 +) + + +class get_principals_in_role_result: + """ + Attributes: + - success + - o1 + + """ + + def __init__( + self, + success=None, + o1=None, + ): + self.success = success + self.o1 = o1 + + def read(self, iprot): + if ( + iprot._fast_decode is not None + and isinstance(iprot.trans, TTransport.CReadableTransport) + and self.thrift_spec is not None + ): + iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec]) + return + iprot.readStructBegin() + while True: + (fname, ftype, fid) = iprot.readFieldBegin() + if ftype == TType.STOP: + break + if fid == 0: + if ftype == TType.STRUCT: + self.success = GetPrincipalsInRoleResponse() + self.success.read(iprot) + else: + iprot.skip(ftype) + elif fid == 1: + if ftype == TType.STRUCT: + self.o1 = MetaException.read(iprot) + else: + iprot.skip(ftype) + else: + iprot.skip(ftype) + iprot.readFieldEnd() + iprot.readStructEnd() + + def write(self, oprot): + if oprot._fast_encode is not None and self.thrift_spec is not None: + oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec])) + return + oprot.writeStructBegin("get_principals_in_role_result") + if self.success is not None: + oprot.writeFieldBegin("success", TType.STRUCT, 0) + self.success.write(oprot) + oprot.writeFieldEnd() + if self.o1 is not None: + oprot.writeFieldBegin("o1", TType.STRUCT, 1) + self.o1.write(oprot) + oprot.writeFieldEnd() + oprot.writeFieldStop() + oprot.writeStructEnd() + + def validate(self): + return + + def __repr__(self): + L = ["{}={!r}".format(key, value) for key, value in self.__dict__.items()] + return "{}({})".format(self.__class__.__name__, ", ".join(L)) + + def __eq__(self, other): + return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ + + def __ne__(self, other): + return not (self == other) + + +all_structs.append(get_principals_in_role_result) +get_principals_in_role_result.thrift_spec = ( + ( + 0, + TType.STRUCT, + "success", + [GetPrincipalsInRoleResponse, None], + None, + ), # 0 + ( + 1, + TType.STRUCT, + "o1", + [MetaException, None], + None, + ), # 1 +) + + +class get_role_grants_for_principal_args: + """ + Attributes: + - request + + """ + + def __init__( + self, + request=None, + ): + self.request = request + + def read(self, iprot): + if ( + iprot._fast_decode is not None + and isinstance(iprot.trans, TTransport.CReadableTransport) + and self.thrift_spec is not None + ): + iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec]) + return + iprot.readStructBegin() + while True: + (fname, ftype, fid) = iprot.readFieldBegin() + if ftype == TType.STOP: + break + if fid == 1: + if ftype == TType.STRUCT: + self.request = GetRoleGrantsForPrincipalRequest() + self.request.read(iprot) + else: + iprot.skip(ftype) + else: + iprot.skip(ftype) + iprot.readFieldEnd() + iprot.readStructEnd() + + def write(self, oprot): + if oprot._fast_encode is not None and self.thrift_spec is not None: + oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec])) + return + oprot.writeStructBegin("get_role_grants_for_principal_args") + if self.request is not None: + oprot.writeFieldBegin("request", TType.STRUCT, 1) + self.request.write(oprot) + oprot.writeFieldEnd() + oprot.writeFieldStop() + oprot.writeStructEnd() + + def validate(self): + return + + def __repr__(self): + L = ["{}={!r}".format(key, value) for key, value in self.__dict__.items()] + return "{}({})".format(self.__class__.__name__, ", ".join(L)) + + def __eq__(self, other): + return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ + + def __ne__(self, other): + return not (self == other) + + +all_structs.append(get_role_grants_for_principal_args) +get_role_grants_for_principal_args.thrift_spec = ( + None, # 0 + ( + 1, + TType.STRUCT, + "request", + [GetRoleGrantsForPrincipalRequest, None], + None, + ), # 1 +) + + +class get_role_grants_for_principal_result: + """ + Attributes: + - success + - o1 + + """ + + def __init__( + self, + success=None, + o1=None, + ): + self.success = success + self.o1 = o1 + + def read(self, iprot): + if ( + iprot._fast_decode is not None + and isinstance(iprot.trans, TTransport.CReadableTransport) + and self.thrift_spec is not None + ): + iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec]) + return + iprot.readStructBegin() + while True: + (fname, ftype, fid) = iprot.readFieldBegin() + if ftype == TType.STOP: + break + if fid == 0: + if ftype == TType.STRUCT: + self.success = GetRoleGrantsForPrincipalResponse() + self.success.read(iprot) + else: + iprot.skip(ftype) + elif fid == 1: + if ftype == TType.STRUCT: + self.o1 = MetaException.read(iprot) + else: + iprot.skip(ftype) + else: + iprot.skip(ftype) + iprot.readFieldEnd() + iprot.readStructEnd() + + def write(self, oprot): + if oprot._fast_encode is not None and self.thrift_spec is not None: + oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec])) + return + oprot.writeStructBegin("get_role_grants_for_principal_result") + if self.success is not None: + oprot.writeFieldBegin("success", TType.STRUCT, 0) + self.success.write(oprot) + oprot.writeFieldEnd() + if self.o1 is not None: + oprot.writeFieldBegin("o1", TType.STRUCT, 1) + self.o1.write(oprot) + oprot.writeFieldEnd() + oprot.writeFieldStop() + oprot.writeStructEnd() + + def validate(self): + return + + def __repr__(self): + L = ["{}={!r}".format(key, value) for key, value in self.__dict__.items()] + return "{}({})".format(self.__class__.__name__, ", ".join(L)) + + def __eq__(self, other): + return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ + + def __ne__(self, other): + return not (self == other) + + +all_structs.append(get_role_grants_for_principal_result) +get_role_grants_for_principal_result.thrift_spec = ( + ( + 0, + TType.STRUCT, + "success", + [GetRoleGrantsForPrincipalResponse, None], + None, + ), # 0 + ( + 1, + TType.STRUCT, + "o1", + [MetaException, None], + None, + ), # 1 +) + + +class get_privilege_set_args: + """ + Attributes: + - hiveObject + - user_name + - group_names + + """ + + def __init__( + self, + hiveObject=None, + user_name=None, + group_names=None, + ): + self.hiveObject = hiveObject + self.user_name = user_name + self.group_names = group_names + + def read(self, iprot): + if ( + iprot._fast_decode is not None + and isinstance(iprot.trans, TTransport.CReadableTransport) + and self.thrift_spec is not None + ): + iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec]) + return + iprot.readStructBegin() + while True: + (fname, ftype, fid) = iprot.readFieldBegin() + if ftype == TType.STOP: + break + if fid == 1: + if ftype == TType.STRUCT: + self.hiveObject = HiveObjectRef() + self.hiveObject.read(iprot) + else: + iprot.skip(ftype) + elif fid == 2: + if ftype == TType.STRING: + self.user_name = ( + iprot.readString().decode("utf-8", errors="replace") if sys.version_info[0] == 2 else iprot.readString() + ) + else: + iprot.skip(ftype) + elif fid == 3: + if ftype == TType.LIST: + self.group_names = [] + (_etype1726, _size1723) = iprot.readListBegin() + for _i1727 in range(_size1723): + _elem1728 = ( + iprot.readString().decode("utf-8", errors="replace") + if sys.version_info[0] == 2 + else iprot.readString() + ) + self.group_names.append(_elem1728) + iprot.readListEnd() + else: + iprot.skip(ftype) + else: + iprot.skip(ftype) + iprot.readFieldEnd() + iprot.readStructEnd() + + def write(self, oprot): + if oprot._fast_encode is not None and self.thrift_spec is not None: + oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec])) + return + oprot.writeStructBegin("get_privilege_set_args") + if self.hiveObject is not None: + oprot.writeFieldBegin("hiveObject", TType.STRUCT, 1) + self.hiveObject.write(oprot) + oprot.writeFieldEnd() + if self.user_name is not None: + oprot.writeFieldBegin("user_name", TType.STRING, 2) + oprot.writeString(self.user_name.encode("utf-8") if sys.version_info[0] == 2 else self.user_name) + oprot.writeFieldEnd() + if self.group_names is not None: + oprot.writeFieldBegin("group_names", TType.LIST, 3) + oprot.writeListBegin(TType.STRING, len(self.group_names)) + for iter1729 in self.group_names: + oprot.writeString(iter1729.encode("utf-8") if sys.version_info[0] == 2 else iter1729) + oprot.writeListEnd() + oprot.writeFieldEnd() + oprot.writeFieldStop() + oprot.writeStructEnd() + + def validate(self): + return + + def __repr__(self): + L = ["{}={!r}".format(key, value) for key, value in self.__dict__.items()] + return "{}({})".format(self.__class__.__name__, ", ".join(L)) + + def __eq__(self, other): + return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ + + def __ne__(self, other): + return not (self == other) + + +all_structs.append(get_privilege_set_args) +get_privilege_set_args.thrift_spec = ( + None, # 0 + ( + 1, + TType.STRUCT, + "hiveObject", + [HiveObjectRef, None], + None, + ), # 1 + ( + 2, + TType.STRING, + "user_name", + "UTF8", + None, + ), # 2 + ( + 3, + TType.LIST, + "group_names", + (TType.STRING, "UTF8", False), + None, + ), # 3 +) + + +class get_privilege_set_result: + """ + Attributes: + - success + - o1 + + """ + + def __init__( + self, + success=None, + o1=None, + ): + self.success = success + self.o1 = o1 + + def read(self, iprot): + if ( + iprot._fast_decode is not None + and isinstance(iprot.trans, TTransport.CReadableTransport) + and self.thrift_spec is not None + ): + iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec]) + return + iprot.readStructBegin() + while True: + (fname, ftype, fid) = iprot.readFieldBegin() + if ftype == TType.STOP: + break + if fid == 0: + if ftype == TType.STRUCT: + self.success = PrincipalPrivilegeSet() + self.success.read(iprot) + else: + iprot.skip(ftype) + elif fid == 1: + if ftype == TType.STRUCT: + self.o1 = MetaException.read(iprot) + else: + iprot.skip(ftype) + else: + iprot.skip(ftype) + iprot.readFieldEnd() + iprot.readStructEnd() + + def write(self, oprot): + if oprot._fast_encode is not None and self.thrift_spec is not None: + oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec])) + return + oprot.writeStructBegin("get_privilege_set_result") + if self.success is not None: + oprot.writeFieldBegin("success", TType.STRUCT, 0) + self.success.write(oprot) + oprot.writeFieldEnd() + if self.o1 is not None: + oprot.writeFieldBegin("o1", TType.STRUCT, 1) + self.o1.write(oprot) + oprot.writeFieldEnd() + oprot.writeFieldStop() + oprot.writeStructEnd() + + def validate(self): + return + + def __repr__(self): + L = ["{}={!r}".format(key, value) for key, value in self.__dict__.items()] + return "{}({})".format(self.__class__.__name__, ", ".join(L)) + + def __eq__(self, other): + return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ + + def __ne__(self, other): + return not (self == other) + + +all_structs.append(get_privilege_set_result) +get_privilege_set_result.thrift_spec = ( + ( + 0, + TType.STRUCT, + "success", + [PrincipalPrivilegeSet, None], + None, + ), # 0 + ( + 1, + TType.STRUCT, + "o1", + [MetaException, None], + None, + ), # 1 +) + + +class list_privileges_args: + """ + Attributes: + - principal_name + - principal_type + - hiveObject + + """ + + def __init__( + self, + principal_name=None, + principal_type=None, + hiveObject=None, + ): + self.principal_name = principal_name + self.principal_type = principal_type + self.hiveObject = hiveObject + + def read(self, iprot): + if ( + iprot._fast_decode is not None + and isinstance(iprot.trans, TTransport.CReadableTransport) + and self.thrift_spec is not None + ): + iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec]) + return + iprot.readStructBegin() + while True: + (fname, ftype, fid) = iprot.readFieldBegin() + if ftype == TType.STOP: + break + if fid == 1: + if ftype == TType.STRING: + self.principal_name = ( + iprot.readString().decode("utf-8", errors="replace") if sys.version_info[0] == 2 else iprot.readString() + ) + else: + iprot.skip(ftype) + elif fid == 2: + if ftype == TType.I32: + self.principal_type = iprot.readI32() + else: + iprot.skip(ftype) + elif fid == 3: + if ftype == TType.STRUCT: + self.hiveObject = HiveObjectRef() + self.hiveObject.read(iprot) + else: + iprot.skip(ftype) + else: + iprot.skip(ftype) + iprot.readFieldEnd() + iprot.readStructEnd() + + def write(self, oprot): + if oprot._fast_encode is not None and self.thrift_spec is not None: + oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec])) + return + oprot.writeStructBegin("list_privileges_args") + if self.principal_name is not None: + oprot.writeFieldBegin("principal_name", TType.STRING, 1) + oprot.writeString(self.principal_name.encode("utf-8") if sys.version_info[0] == 2 else self.principal_name) + oprot.writeFieldEnd() + if self.principal_type is not None: + oprot.writeFieldBegin("principal_type", TType.I32, 2) + oprot.writeI32(self.principal_type) + oprot.writeFieldEnd() + if self.hiveObject is not None: + oprot.writeFieldBegin("hiveObject", TType.STRUCT, 3) + self.hiveObject.write(oprot) + oprot.writeFieldEnd() + oprot.writeFieldStop() + oprot.writeStructEnd() + + def validate(self): + return + + def __repr__(self): + L = ["{}={!r}".format(key, value) for key, value in self.__dict__.items()] + return "{}({})".format(self.__class__.__name__, ", ".join(L)) + + def __eq__(self, other): + return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ + + def __ne__(self, other): + return not (self == other) + + +all_structs.append(list_privileges_args) +list_privileges_args.thrift_spec = ( + None, # 0 + ( + 1, + TType.STRING, + "principal_name", + "UTF8", + None, + ), # 1 + ( + 2, + TType.I32, + "principal_type", + None, + None, + ), # 2 + ( + 3, + TType.STRUCT, + "hiveObject", + [HiveObjectRef, None], + None, + ), # 3 +) + + +class list_privileges_result: + """ + Attributes: + - success + - o1 + + """ + + def __init__( + self, + success=None, + o1=None, + ): + self.success = success + self.o1 = o1 + + def read(self, iprot): + if ( + iprot._fast_decode is not None + and isinstance(iprot.trans, TTransport.CReadableTransport) + and self.thrift_spec is not None + ): + iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec]) + return + iprot.readStructBegin() + while True: + (fname, ftype, fid) = iprot.readFieldBegin() + if ftype == TType.STOP: + break + if fid == 0: + if ftype == TType.LIST: + self.success = [] + (_etype1733, _size1730) = iprot.readListBegin() + for _i1734 in range(_size1730): + _elem1735 = HiveObjectPrivilege() + _elem1735.read(iprot) + self.success.append(_elem1735) + iprot.readListEnd() + else: + iprot.skip(ftype) + elif fid == 1: + if ftype == TType.STRUCT: + self.o1 = MetaException.read(iprot) + else: + iprot.skip(ftype) + else: + iprot.skip(ftype) + iprot.readFieldEnd() + iprot.readStructEnd() + + def write(self, oprot): + if oprot._fast_encode is not None and self.thrift_spec is not None: + oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec])) + return + oprot.writeStructBegin("list_privileges_result") + if self.success is not None: + oprot.writeFieldBegin("success", TType.LIST, 0) + oprot.writeListBegin(TType.STRUCT, len(self.success)) + for iter1736 in self.success: + iter1736.write(oprot) + oprot.writeListEnd() + oprot.writeFieldEnd() + if self.o1 is not None: + oprot.writeFieldBegin("o1", TType.STRUCT, 1) + self.o1.write(oprot) + oprot.writeFieldEnd() + oprot.writeFieldStop() + oprot.writeStructEnd() + + def validate(self): + return + + def __repr__(self): + L = ["{}={!r}".format(key, value) for key, value in self.__dict__.items()] + return "{}({})".format(self.__class__.__name__, ", ".join(L)) + + def __eq__(self, other): + return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ + + def __ne__(self, other): + return not (self == other) + + +all_structs.append(list_privileges_result) +list_privileges_result.thrift_spec = ( + ( + 0, + TType.LIST, + "success", + (TType.STRUCT, [HiveObjectPrivilege, None], False), + None, + ), # 0 + ( + 1, + TType.STRUCT, + "o1", + [MetaException, None], + None, + ), # 1 +) + + +class grant_privileges_args: + """ + Attributes: + - privileges + + """ + + def __init__( + self, + privileges=None, + ): + self.privileges = privileges + + def read(self, iprot): + if ( + iprot._fast_decode is not None + and isinstance(iprot.trans, TTransport.CReadableTransport) + and self.thrift_spec is not None + ): + iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec]) + return + iprot.readStructBegin() + while True: + (fname, ftype, fid) = iprot.readFieldBegin() + if ftype == TType.STOP: + break + if fid == 1: + if ftype == TType.STRUCT: + self.privileges = PrivilegeBag() + self.privileges.read(iprot) + else: + iprot.skip(ftype) + else: + iprot.skip(ftype) + iprot.readFieldEnd() + iprot.readStructEnd() + + def write(self, oprot): + if oprot._fast_encode is not None and self.thrift_spec is not None: + oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec])) + return + oprot.writeStructBegin("grant_privileges_args") + if self.privileges is not None: + oprot.writeFieldBegin("privileges", TType.STRUCT, 1) + self.privileges.write(oprot) + oprot.writeFieldEnd() + oprot.writeFieldStop() + oprot.writeStructEnd() + + def validate(self): + return + + def __repr__(self): + L = ["{}={!r}".format(key, value) for key, value in self.__dict__.items()] + return "{}({})".format(self.__class__.__name__, ", ".join(L)) + + def __eq__(self, other): + return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ + + def __ne__(self, other): + return not (self == other) + + +all_structs.append(grant_privileges_args) +grant_privileges_args.thrift_spec = ( + None, # 0 + ( + 1, + TType.STRUCT, + "privileges", + [PrivilegeBag, None], + None, + ), # 1 +) + + +class grant_privileges_result: + """ + Attributes: + - success + - o1 + + """ + + def __init__( + self, + success=None, + o1=None, + ): + self.success = success + self.o1 = o1 + + def read(self, iprot): + if ( + iprot._fast_decode is not None + and isinstance(iprot.trans, TTransport.CReadableTransport) + and self.thrift_spec is not None + ): + iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec]) + return + iprot.readStructBegin() + while True: + (fname, ftype, fid) = iprot.readFieldBegin() + if ftype == TType.STOP: + break + if fid == 0: + if ftype == TType.BOOL: + self.success = iprot.readBool() + else: + iprot.skip(ftype) + elif fid == 1: + if ftype == TType.STRUCT: + self.o1 = MetaException.read(iprot) + else: + iprot.skip(ftype) + else: + iprot.skip(ftype) + iprot.readFieldEnd() + iprot.readStructEnd() + + def write(self, oprot): + if oprot._fast_encode is not None and self.thrift_spec is not None: + oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec])) + return + oprot.writeStructBegin("grant_privileges_result") + if self.success is not None: + oprot.writeFieldBegin("success", TType.BOOL, 0) + oprot.writeBool(self.success) + oprot.writeFieldEnd() + if self.o1 is not None: + oprot.writeFieldBegin("o1", TType.STRUCT, 1) + self.o1.write(oprot) + oprot.writeFieldEnd() + oprot.writeFieldStop() + oprot.writeStructEnd() + + def validate(self): + return + + def __repr__(self): + L = ["{}={!r}".format(key, value) for key, value in self.__dict__.items()] + return "{}({})".format(self.__class__.__name__, ", ".join(L)) + + def __eq__(self, other): + return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ + + def __ne__(self, other): + return not (self == other) + + +all_structs.append(grant_privileges_result) +grant_privileges_result.thrift_spec = ( + ( + 0, + TType.BOOL, + "success", + None, + None, + ), # 0 + ( + 1, + TType.STRUCT, + "o1", + [MetaException, None], + None, + ), # 1 +) + + +class revoke_privileges_args: + """ + Attributes: + - privileges + + """ + + def __init__( + self, + privileges=None, + ): + self.privileges = privileges + + def read(self, iprot): + if ( + iprot._fast_decode is not None + and isinstance(iprot.trans, TTransport.CReadableTransport) + and self.thrift_spec is not None + ): + iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec]) + return + iprot.readStructBegin() + while True: + (fname, ftype, fid) = iprot.readFieldBegin() + if ftype == TType.STOP: + break + if fid == 1: + if ftype == TType.STRUCT: + self.privileges = PrivilegeBag() + self.privileges.read(iprot) + else: + iprot.skip(ftype) + else: + iprot.skip(ftype) + iprot.readFieldEnd() + iprot.readStructEnd() + + def write(self, oprot): + if oprot._fast_encode is not None and self.thrift_spec is not None: + oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec])) + return + oprot.writeStructBegin("revoke_privileges_args") + if self.privileges is not None: + oprot.writeFieldBegin("privileges", TType.STRUCT, 1) + self.privileges.write(oprot) + oprot.writeFieldEnd() + oprot.writeFieldStop() + oprot.writeStructEnd() + + def validate(self): + return + + def __repr__(self): + L = ["{}={!r}".format(key, value) for key, value in self.__dict__.items()] + return "{}({})".format(self.__class__.__name__, ", ".join(L)) + + def __eq__(self, other): + return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ + + def __ne__(self, other): + return not (self == other) + + +all_structs.append(revoke_privileges_args) +revoke_privileges_args.thrift_spec = ( + None, # 0 + ( + 1, + TType.STRUCT, + "privileges", + [PrivilegeBag, None], + None, + ), # 1 +) + + +class revoke_privileges_result: + """ + Attributes: + - success + - o1 + + """ + + def __init__( + self, + success=None, + o1=None, + ): + self.success = success + self.o1 = o1 + + def read(self, iprot): + if ( + iprot._fast_decode is not None + and isinstance(iprot.trans, TTransport.CReadableTransport) + and self.thrift_spec is not None + ): + iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec]) + return + iprot.readStructBegin() + while True: + (fname, ftype, fid) = iprot.readFieldBegin() + if ftype == TType.STOP: + break + if fid == 0: + if ftype == TType.BOOL: + self.success = iprot.readBool() + else: + iprot.skip(ftype) + elif fid == 1: + if ftype == TType.STRUCT: + self.o1 = MetaException.read(iprot) + else: + iprot.skip(ftype) + else: + iprot.skip(ftype) + iprot.readFieldEnd() + iprot.readStructEnd() + + def write(self, oprot): + if oprot._fast_encode is not None and self.thrift_spec is not None: + oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec])) + return + oprot.writeStructBegin("revoke_privileges_result") + if self.success is not None: + oprot.writeFieldBegin("success", TType.BOOL, 0) + oprot.writeBool(self.success) + oprot.writeFieldEnd() + if self.o1 is not None: + oprot.writeFieldBegin("o1", TType.STRUCT, 1) + self.o1.write(oprot) + oprot.writeFieldEnd() + oprot.writeFieldStop() + oprot.writeStructEnd() + + def validate(self): + return + + def __repr__(self): + L = ["{}={!r}".format(key, value) for key, value in self.__dict__.items()] + return "{}({})".format(self.__class__.__name__, ", ".join(L)) + + def __eq__(self, other): + return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ + + def __ne__(self, other): + return not (self == other) + + +all_structs.append(revoke_privileges_result) +revoke_privileges_result.thrift_spec = ( + ( + 0, + TType.BOOL, + "success", + None, + None, + ), # 0 + ( + 1, + TType.STRUCT, + "o1", + [MetaException, None], + None, + ), # 1 +) + + +class grant_revoke_privileges_args: + """ + Attributes: + - request + + """ + + def __init__( + self, + request=None, + ): + self.request = request + + def read(self, iprot): + if ( + iprot._fast_decode is not None + and isinstance(iprot.trans, TTransport.CReadableTransport) + and self.thrift_spec is not None + ): + iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec]) + return + iprot.readStructBegin() + while True: + (fname, ftype, fid) = iprot.readFieldBegin() + if ftype == TType.STOP: + break + if fid == 1: + if ftype == TType.STRUCT: + self.request = GrantRevokePrivilegeRequest() + self.request.read(iprot) + else: + iprot.skip(ftype) + else: + iprot.skip(ftype) + iprot.readFieldEnd() + iprot.readStructEnd() + + def write(self, oprot): + if oprot._fast_encode is not None and self.thrift_spec is not None: + oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec])) + return + oprot.writeStructBegin("grant_revoke_privileges_args") + if self.request is not None: + oprot.writeFieldBegin("request", TType.STRUCT, 1) + self.request.write(oprot) + oprot.writeFieldEnd() + oprot.writeFieldStop() + oprot.writeStructEnd() + + def validate(self): + return + + def __repr__(self): + L = ["{}={!r}".format(key, value) for key, value in self.__dict__.items()] + return "{}({})".format(self.__class__.__name__, ", ".join(L)) + + def __eq__(self, other): + return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ + + def __ne__(self, other): + return not (self == other) + + +all_structs.append(grant_revoke_privileges_args) +grant_revoke_privileges_args.thrift_spec = ( + None, # 0 + ( + 1, + TType.STRUCT, + "request", + [GrantRevokePrivilegeRequest, None], + None, + ), # 1 +) + + +class grant_revoke_privileges_result: + """ + Attributes: + - success + - o1 + + """ + + def __init__( + self, + success=None, + o1=None, + ): + self.success = success + self.o1 = o1 + + def read(self, iprot): + if ( + iprot._fast_decode is not None + and isinstance(iprot.trans, TTransport.CReadableTransport) + and self.thrift_spec is not None + ): + iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec]) + return + iprot.readStructBegin() + while True: + (fname, ftype, fid) = iprot.readFieldBegin() + if ftype == TType.STOP: + break + if fid == 0: + if ftype == TType.STRUCT: + self.success = GrantRevokePrivilegeResponse() + self.success.read(iprot) + else: + iprot.skip(ftype) + elif fid == 1: + if ftype == TType.STRUCT: + self.o1 = MetaException.read(iprot) + else: + iprot.skip(ftype) + else: + iprot.skip(ftype) + iprot.readFieldEnd() + iprot.readStructEnd() + + def write(self, oprot): + if oprot._fast_encode is not None and self.thrift_spec is not None: + oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec])) + return + oprot.writeStructBegin("grant_revoke_privileges_result") + if self.success is not None: + oprot.writeFieldBegin("success", TType.STRUCT, 0) + self.success.write(oprot) + oprot.writeFieldEnd() + if self.o1 is not None: + oprot.writeFieldBegin("o1", TType.STRUCT, 1) + self.o1.write(oprot) + oprot.writeFieldEnd() + oprot.writeFieldStop() + oprot.writeStructEnd() + + def validate(self): + return + + def __repr__(self): + L = ["{}={!r}".format(key, value) for key, value in self.__dict__.items()] + return "{}({})".format(self.__class__.__name__, ", ".join(L)) + + def __eq__(self, other): + return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ + + def __ne__(self, other): + return not (self == other) + + +all_structs.append(grant_revoke_privileges_result) +grant_revoke_privileges_result.thrift_spec = ( + ( + 0, + TType.STRUCT, + "success", + [GrantRevokePrivilegeResponse, None], + None, + ), # 0 + ( + 1, + TType.STRUCT, + "o1", + [MetaException, None], + None, + ), # 1 +) + + +class refresh_privileges_args: + """ + Attributes: + - objToRefresh + - authorizer + - grantRequest + + """ + + def __init__( + self, + objToRefresh=None, + authorizer=None, + grantRequest=None, + ): + self.objToRefresh = objToRefresh + self.authorizer = authorizer + self.grantRequest = grantRequest + + def read(self, iprot): + if ( + iprot._fast_decode is not None + and isinstance(iprot.trans, TTransport.CReadableTransport) + and self.thrift_spec is not None + ): + iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec]) + return + iprot.readStructBegin() + while True: + (fname, ftype, fid) = iprot.readFieldBegin() + if ftype == TType.STOP: + break + if fid == 1: + if ftype == TType.STRUCT: + self.objToRefresh = HiveObjectRef() + self.objToRefresh.read(iprot) + else: + iprot.skip(ftype) + elif fid == 2: + if ftype == TType.STRING: + self.authorizer = ( + iprot.readString().decode("utf-8", errors="replace") if sys.version_info[0] == 2 else iprot.readString() + ) + else: + iprot.skip(ftype) + elif fid == 3: + if ftype == TType.STRUCT: + self.grantRequest = GrantRevokePrivilegeRequest() + self.grantRequest.read(iprot) + else: + iprot.skip(ftype) + else: + iprot.skip(ftype) + iprot.readFieldEnd() + iprot.readStructEnd() + + def write(self, oprot): + if oprot._fast_encode is not None and self.thrift_spec is not None: + oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec])) + return + oprot.writeStructBegin("refresh_privileges_args") + if self.objToRefresh is not None: + oprot.writeFieldBegin("objToRefresh", TType.STRUCT, 1) + self.objToRefresh.write(oprot) + oprot.writeFieldEnd() + if self.authorizer is not None: + oprot.writeFieldBegin("authorizer", TType.STRING, 2) + oprot.writeString(self.authorizer.encode("utf-8") if sys.version_info[0] == 2 else self.authorizer) + oprot.writeFieldEnd() + if self.grantRequest is not None: + oprot.writeFieldBegin("grantRequest", TType.STRUCT, 3) + self.grantRequest.write(oprot) + oprot.writeFieldEnd() + oprot.writeFieldStop() + oprot.writeStructEnd() + + def validate(self): + return + + def __repr__(self): + L = ["{}={!r}".format(key, value) for key, value in self.__dict__.items()] + return "{}({})".format(self.__class__.__name__, ", ".join(L)) + + def __eq__(self, other): + return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ + + def __ne__(self, other): + return not (self == other) + + +all_structs.append(refresh_privileges_args) +refresh_privileges_args.thrift_spec = ( + None, # 0 + ( + 1, + TType.STRUCT, + "objToRefresh", + [HiveObjectRef, None], + None, + ), # 1 + ( + 2, + TType.STRING, + "authorizer", + "UTF8", + None, + ), # 2 + ( + 3, + TType.STRUCT, + "grantRequest", + [GrantRevokePrivilegeRequest, None], + None, + ), # 3 +) + + +class refresh_privileges_result: + """ + Attributes: + - success + - o1 + + """ + + def __init__( + self, + success=None, + o1=None, + ): + self.success = success + self.o1 = o1 + + def read(self, iprot): + if ( + iprot._fast_decode is not None + and isinstance(iprot.trans, TTransport.CReadableTransport) + and self.thrift_spec is not None + ): + iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec]) + return + iprot.readStructBegin() + while True: + (fname, ftype, fid) = iprot.readFieldBegin() + if ftype == TType.STOP: + break + if fid == 0: + if ftype == TType.STRUCT: + self.success = GrantRevokePrivilegeResponse() + self.success.read(iprot) + else: + iprot.skip(ftype) + elif fid == 1: + if ftype == TType.STRUCT: + self.o1 = MetaException.read(iprot) + else: + iprot.skip(ftype) + else: + iprot.skip(ftype) + iprot.readFieldEnd() + iprot.readStructEnd() + + def write(self, oprot): + if oprot._fast_encode is not None and self.thrift_spec is not None: + oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec])) + return + oprot.writeStructBegin("refresh_privileges_result") + if self.success is not None: + oprot.writeFieldBegin("success", TType.STRUCT, 0) + self.success.write(oprot) + oprot.writeFieldEnd() + if self.o1 is not None: + oprot.writeFieldBegin("o1", TType.STRUCT, 1) + self.o1.write(oprot) + oprot.writeFieldEnd() + oprot.writeFieldStop() + oprot.writeStructEnd() + + def validate(self): + return + + def __repr__(self): + L = ["{}={!r}".format(key, value) for key, value in self.__dict__.items()] + return "{}({})".format(self.__class__.__name__, ", ".join(L)) + + def __eq__(self, other): + return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ + + def __ne__(self, other): + return not (self == other) + + +all_structs.append(refresh_privileges_result) +refresh_privileges_result.thrift_spec = ( + ( + 0, + TType.STRUCT, + "success", + [GrantRevokePrivilegeResponse, None], + None, + ), # 0 + ( + 1, + TType.STRUCT, + "o1", + [MetaException, None], + None, + ), # 1 +) + + +class set_ugi_args: + """ + Attributes: + - user_name + - group_names + + """ + + def __init__( + self, + user_name=None, + group_names=None, + ): + self.user_name = user_name + self.group_names = group_names + + def read(self, iprot): + if ( + iprot._fast_decode is not None + and isinstance(iprot.trans, TTransport.CReadableTransport) + and self.thrift_spec is not None + ): + iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec]) + return + iprot.readStructBegin() + while True: + (fname, ftype, fid) = iprot.readFieldBegin() + if ftype == TType.STOP: + break + if fid == 1: + if ftype == TType.STRING: + self.user_name = ( + iprot.readString().decode("utf-8", errors="replace") if sys.version_info[0] == 2 else iprot.readString() + ) + else: + iprot.skip(ftype) + elif fid == 2: + if ftype == TType.LIST: + self.group_names = [] + (_etype1740, _size1737) = iprot.readListBegin() + for _i1741 in range(_size1737): + _elem1742 = ( + iprot.readString().decode("utf-8", errors="replace") + if sys.version_info[0] == 2 + else iprot.readString() + ) + self.group_names.append(_elem1742) + iprot.readListEnd() + else: + iprot.skip(ftype) + else: + iprot.skip(ftype) + iprot.readFieldEnd() + iprot.readStructEnd() + + def write(self, oprot): + if oprot._fast_encode is not None and self.thrift_spec is not None: + oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec])) + return + oprot.writeStructBegin("set_ugi_args") + if self.user_name is not None: + oprot.writeFieldBegin("user_name", TType.STRING, 1) + oprot.writeString(self.user_name.encode("utf-8") if sys.version_info[0] == 2 else self.user_name) + oprot.writeFieldEnd() + if self.group_names is not None: + oprot.writeFieldBegin("group_names", TType.LIST, 2) + oprot.writeListBegin(TType.STRING, len(self.group_names)) + for iter1743 in self.group_names: + oprot.writeString(iter1743.encode("utf-8") if sys.version_info[0] == 2 else iter1743) + oprot.writeListEnd() + oprot.writeFieldEnd() + oprot.writeFieldStop() + oprot.writeStructEnd() + + def validate(self): + return + + def __repr__(self): + L = ["{}={!r}".format(key, value) for key, value in self.__dict__.items()] + return "{}({})".format(self.__class__.__name__, ", ".join(L)) + + def __eq__(self, other): + return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ + + def __ne__(self, other): + return not (self == other) + + +all_structs.append(set_ugi_args) +set_ugi_args.thrift_spec = ( + None, # 0 + ( + 1, + TType.STRING, + "user_name", + "UTF8", + None, + ), # 1 + ( + 2, + TType.LIST, + "group_names", + (TType.STRING, "UTF8", False), + None, + ), # 2 +) + + +class set_ugi_result: + """ + Attributes: + - success + - o1 + + """ + + def __init__( + self, + success=None, + o1=None, + ): + self.success = success + self.o1 = o1 + + def read(self, iprot): + if ( + iprot._fast_decode is not None + and isinstance(iprot.trans, TTransport.CReadableTransport) + and self.thrift_spec is not None + ): + iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec]) + return + iprot.readStructBegin() + while True: + (fname, ftype, fid) = iprot.readFieldBegin() + if ftype == TType.STOP: + break + if fid == 0: + if ftype == TType.LIST: + self.success = [] + (_etype1747, _size1744) = iprot.readListBegin() + for _i1748 in range(_size1744): + _elem1749 = ( + iprot.readString().decode("utf-8", errors="replace") + if sys.version_info[0] == 2 + else iprot.readString() + ) + self.success.append(_elem1749) + iprot.readListEnd() + else: + iprot.skip(ftype) + elif fid == 1: + if ftype == TType.STRUCT: + self.o1 = MetaException.read(iprot) + else: + iprot.skip(ftype) + else: + iprot.skip(ftype) + iprot.readFieldEnd() + iprot.readStructEnd() + + def write(self, oprot): + if oprot._fast_encode is not None and self.thrift_spec is not None: + oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec])) + return + oprot.writeStructBegin("set_ugi_result") + if self.success is not None: + oprot.writeFieldBegin("success", TType.LIST, 0) + oprot.writeListBegin(TType.STRING, len(self.success)) + for iter1750 in self.success: + oprot.writeString(iter1750.encode("utf-8") if sys.version_info[0] == 2 else iter1750) + oprot.writeListEnd() + oprot.writeFieldEnd() + if self.o1 is not None: + oprot.writeFieldBegin("o1", TType.STRUCT, 1) + self.o1.write(oprot) + oprot.writeFieldEnd() + oprot.writeFieldStop() + oprot.writeStructEnd() + + def validate(self): + return + + def __repr__(self): + L = ["{}={!r}".format(key, value) for key, value in self.__dict__.items()] + return "{}({})".format(self.__class__.__name__, ", ".join(L)) + + def __eq__(self, other): + return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ + + def __ne__(self, other): + return not (self == other) + + +all_structs.append(set_ugi_result) +set_ugi_result.thrift_spec = ( + ( + 0, + TType.LIST, + "success", + (TType.STRING, "UTF8", False), + None, + ), # 0 + ( + 1, + TType.STRUCT, + "o1", + [MetaException, None], + None, + ), # 1 +) + + +class get_delegation_token_args: + """ + Attributes: + - token_owner + - renewer_kerberos_principal_name + + """ + + def __init__( + self, + token_owner=None, + renewer_kerberos_principal_name=None, + ): + self.token_owner = token_owner + self.renewer_kerberos_principal_name = renewer_kerberos_principal_name + + def read(self, iprot): + if ( + iprot._fast_decode is not None + and isinstance(iprot.trans, TTransport.CReadableTransport) + and self.thrift_spec is not None + ): + iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec]) + return + iprot.readStructBegin() + while True: + (fname, ftype, fid) = iprot.readFieldBegin() + if ftype == TType.STOP: + break + if fid == 1: + if ftype == TType.STRING: + self.token_owner = ( + iprot.readString().decode("utf-8", errors="replace") if sys.version_info[0] == 2 else iprot.readString() + ) + else: + iprot.skip(ftype) + elif fid == 2: + if ftype == TType.STRING: + self.renewer_kerberos_principal_name = ( + iprot.readString().decode("utf-8", errors="replace") if sys.version_info[0] == 2 else iprot.readString() + ) + else: + iprot.skip(ftype) + else: + iprot.skip(ftype) + iprot.readFieldEnd() + iprot.readStructEnd() + + def write(self, oprot): + if oprot._fast_encode is not None and self.thrift_spec is not None: + oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec])) + return + oprot.writeStructBegin("get_delegation_token_args") + if self.token_owner is not None: + oprot.writeFieldBegin("token_owner", TType.STRING, 1) + oprot.writeString(self.token_owner.encode("utf-8") if sys.version_info[0] == 2 else self.token_owner) + oprot.writeFieldEnd() + if self.renewer_kerberos_principal_name is not None: + oprot.writeFieldBegin("renewer_kerberos_principal_name", TType.STRING, 2) + oprot.writeString( + self.renewer_kerberos_principal_name.encode("utf-8") + if sys.version_info[0] == 2 + else self.renewer_kerberos_principal_name + ) + oprot.writeFieldEnd() + oprot.writeFieldStop() + oprot.writeStructEnd() + + def validate(self): + return + + def __repr__(self): + L = ["{}={!r}".format(key, value) for key, value in self.__dict__.items()] + return "{}({})".format(self.__class__.__name__, ", ".join(L)) + + def __eq__(self, other): + return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ + + def __ne__(self, other): + return not (self == other) + + +all_structs.append(get_delegation_token_args) +get_delegation_token_args.thrift_spec = ( + None, # 0 + ( + 1, + TType.STRING, + "token_owner", + "UTF8", + None, + ), # 1 + ( + 2, + TType.STRING, + "renewer_kerberos_principal_name", + "UTF8", + None, + ), # 2 +) + + +class get_delegation_token_result: + """ + Attributes: + - success + - o1 + + """ + + def __init__( + self, + success=None, + o1=None, + ): + self.success = success + self.o1 = o1 + + def read(self, iprot): + if ( + iprot._fast_decode is not None + and isinstance(iprot.trans, TTransport.CReadableTransport) + and self.thrift_spec is not None + ): + iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec]) + return + iprot.readStructBegin() + while True: + (fname, ftype, fid) = iprot.readFieldBegin() + if ftype == TType.STOP: + break + if fid == 0: + if ftype == TType.STRING: + self.success = ( + iprot.readString().decode("utf-8", errors="replace") if sys.version_info[0] == 2 else iprot.readString() + ) + else: + iprot.skip(ftype) + elif fid == 1: + if ftype == TType.STRUCT: + self.o1 = MetaException.read(iprot) + else: + iprot.skip(ftype) + else: + iprot.skip(ftype) + iprot.readFieldEnd() + iprot.readStructEnd() + + def write(self, oprot): + if oprot._fast_encode is not None and self.thrift_spec is not None: + oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec])) + return + oprot.writeStructBegin("get_delegation_token_result") + if self.success is not None: + oprot.writeFieldBegin("success", TType.STRING, 0) + oprot.writeString(self.success.encode("utf-8") if sys.version_info[0] == 2 else self.success) + oprot.writeFieldEnd() + if self.o1 is not None: + oprot.writeFieldBegin("o1", TType.STRUCT, 1) + self.o1.write(oprot) + oprot.writeFieldEnd() + oprot.writeFieldStop() + oprot.writeStructEnd() + + def validate(self): + return + + def __repr__(self): + L = ["{}={!r}".format(key, value) for key, value in self.__dict__.items()] + return "{}({})".format(self.__class__.__name__, ", ".join(L)) + + def __eq__(self, other): + return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ + + def __ne__(self, other): + return not (self == other) + + +all_structs.append(get_delegation_token_result) +get_delegation_token_result.thrift_spec = ( + ( + 0, + TType.STRING, + "success", + "UTF8", + None, + ), # 0 + ( + 1, + TType.STRUCT, + "o1", + [MetaException, None], + None, + ), # 1 +) + + +class renew_delegation_token_args: + """ + Attributes: + - token_str_form + + """ + + def __init__( + self, + token_str_form=None, + ): + self.token_str_form = token_str_form + + def read(self, iprot): + if ( + iprot._fast_decode is not None + and isinstance(iprot.trans, TTransport.CReadableTransport) + and self.thrift_spec is not None + ): + iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec]) + return + iprot.readStructBegin() + while True: + (fname, ftype, fid) = iprot.readFieldBegin() + if ftype == TType.STOP: + break + if fid == 1: + if ftype == TType.STRING: + self.token_str_form = ( + iprot.readString().decode("utf-8", errors="replace") if sys.version_info[0] == 2 else iprot.readString() + ) + else: + iprot.skip(ftype) + else: + iprot.skip(ftype) + iprot.readFieldEnd() + iprot.readStructEnd() + + def write(self, oprot): + if oprot._fast_encode is not None and self.thrift_spec is not None: + oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec])) + return + oprot.writeStructBegin("renew_delegation_token_args") + if self.token_str_form is not None: + oprot.writeFieldBegin("token_str_form", TType.STRING, 1) + oprot.writeString(self.token_str_form.encode("utf-8") if sys.version_info[0] == 2 else self.token_str_form) + oprot.writeFieldEnd() + oprot.writeFieldStop() + oprot.writeStructEnd() + + def validate(self): + return + + def __repr__(self): + L = ["{}={!r}".format(key, value) for key, value in self.__dict__.items()] + return "{}({})".format(self.__class__.__name__, ", ".join(L)) + + def __eq__(self, other): + return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ + + def __ne__(self, other): + return not (self == other) + + +all_structs.append(renew_delegation_token_args) +renew_delegation_token_args.thrift_spec = ( + None, # 0 + ( + 1, + TType.STRING, + "token_str_form", + "UTF8", + None, + ), # 1 +) + + +class renew_delegation_token_result: + """ + Attributes: + - success + - o1 + + """ + + def __init__( + self, + success=None, + o1=None, + ): + self.success = success + self.o1 = o1 + + def read(self, iprot): + if ( + iprot._fast_decode is not None + and isinstance(iprot.trans, TTransport.CReadableTransport) + and self.thrift_spec is not None + ): + iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec]) + return + iprot.readStructBegin() + while True: + (fname, ftype, fid) = iprot.readFieldBegin() + if ftype == TType.STOP: + break + if fid == 0: + if ftype == TType.I64: + self.success = iprot.readI64() + else: + iprot.skip(ftype) + elif fid == 1: + if ftype == TType.STRUCT: + self.o1 = MetaException.read(iprot) + else: + iprot.skip(ftype) + else: + iprot.skip(ftype) + iprot.readFieldEnd() + iprot.readStructEnd() + + def write(self, oprot): + if oprot._fast_encode is not None and self.thrift_spec is not None: + oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec])) + return + oprot.writeStructBegin("renew_delegation_token_result") + if self.success is not None: + oprot.writeFieldBegin("success", TType.I64, 0) + oprot.writeI64(self.success) + oprot.writeFieldEnd() + if self.o1 is not None: + oprot.writeFieldBegin("o1", TType.STRUCT, 1) + self.o1.write(oprot) + oprot.writeFieldEnd() + oprot.writeFieldStop() + oprot.writeStructEnd() + + def validate(self): + return + + def __repr__(self): + L = ["{}={!r}".format(key, value) for key, value in self.__dict__.items()] + return "{}({})".format(self.__class__.__name__, ", ".join(L)) + + def __eq__(self, other): + return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ + + def __ne__(self, other): + return not (self == other) + + +all_structs.append(renew_delegation_token_result) +renew_delegation_token_result.thrift_spec = ( + ( + 0, + TType.I64, + "success", + None, + None, + ), # 0 + ( + 1, + TType.STRUCT, + "o1", + [MetaException, None], + None, + ), # 1 +) + + +class cancel_delegation_token_args: + """ + Attributes: + - token_str_form + + """ + + def __init__( + self, + token_str_form=None, + ): + self.token_str_form = token_str_form + + def read(self, iprot): + if ( + iprot._fast_decode is not None + and isinstance(iprot.trans, TTransport.CReadableTransport) + and self.thrift_spec is not None + ): + iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec]) + return + iprot.readStructBegin() + while True: + (fname, ftype, fid) = iprot.readFieldBegin() + if ftype == TType.STOP: + break + if fid == 1: + if ftype == TType.STRING: + self.token_str_form = ( + iprot.readString().decode("utf-8", errors="replace") if sys.version_info[0] == 2 else iprot.readString() + ) + else: + iprot.skip(ftype) + else: + iprot.skip(ftype) + iprot.readFieldEnd() + iprot.readStructEnd() + + def write(self, oprot): + if oprot._fast_encode is not None and self.thrift_spec is not None: + oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec])) + return + oprot.writeStructBegin("cancel_delegation_token_args") + if self.token_str_form is not None: + oprot.writeFieldBegin("token_str_form", TType.STRING, 1) + oprot.writeString(self.token_str_form.encode("utf-8") if sys.version_info[0] == 2 else self.token_str_form) + oprot.writeFieldEnd() + oprot.writeFieldStop() + oprot.writeStructEnd() + + def validate(self): + return + + def __repr__(self): + L = ["{}={!r}".format(key, value) for key, value in self.__dict__.items()] + return "{}({})".format(self.__class__.__name__, ", ".join(L)) + + def __eq__(self, other): + return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ + + def __ne__(self, other): + return not (self == other) + + +all_structs.append(cancel_delegation_token_args) +cancel_delegation_token_args.thrift_spec = ( + None, # 0 + ( + 1, + TType.STRING, + "token_str_form", + "UTF8", + None, + ), # 1 +) + + +class cancel_delegation_token_result: + """ + Attributes: + - o1 + + """ + + def __init__( + self, + o1=None, + ): + self.o1 = o1 + + def read(self, iprot): + if ( + iprot._fast_decode is not None + and isinstance(iprot.trans, TTransport.CReadableTransport) + and self.thrift_spec is not None + ): + iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec]) + return + iprot.readStructBegin() + while True: + (fname, ftype, fid) = iprot.readFieldBegin() + if ftype == TType.STOP: + break + if fid == 1: + if ftype == TType.STRUCT: + self.o1 = MetaException.read(iprot) + else: + iprot.skip(ftype) + else: + iprot.skip(ftype) + iprot.readFieldEnd() + iprot.readStructEnd() + + def write(self, oprot): + if oprot._fast_encode is not None and self.thrift_spec is not None: + oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec])) + return + oprot.writeStructBegin("cancel_delegation_token_result") + if self.o1 is not None: + oprot.writeFieldBegin("o1", TType.STRUCT, 1) + self.o1.write(oprot) + oprot.writeFieldEnd() + oprot.writeFieldStop() + oprot.writeStructEnd() + + def validate(self): + return + + def __repr__(self): + L = ["{}={!r}".format(key, value) for key, value in self.__dict__.items()] + return "{}({})".format(self.__class__.__name__, ", ".join(L)) + + def __eq__(self, other): + return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ + + def __ne__(self, other): + return not (self == other) + + +all_structs.append(cancel_delegation_token_result) +cancel_delegation_token_result.thrift_spec = ( + None, # 0 + ( + 1, + TType.STRUCT, + "o1", + [MetaException, None], + None, + ), # 1 +) + + +class add_token_args: + """ + Attributes: + - token_identifier + - delegation_token + + """ + + def __init__( + self, + token_identifier=None, + delegation_token=None, + ): + self.token_identifier = token_identifier + self.delegation_token = delegation_token + + def read(self, iprot): + if ( + iprot._fast_decode is not None + and isinstance(iprot.trans, TTransport.CReadableTransport) + and self.thrift_spec is not None + ): + iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec]) + return + iprot.readStructBegin() + while True: + (fname, ftype, fid) = iprot.readFieldBegin() + if ftype == TType.STOP: + break + if fid == 1: + if ftype == TType.STRING: + self.token_identifier = ( + iprot.readString().decode("utf-8", errors="replace") if sys.version_info[0] == 2 else iprot.readString() + ) + else: + iprot.skip(ftype) + elif fid == 2: + if ftype == TType.STRING: + self.delegation_token = ( + iprot.readString().decode("utf-8", errors="replace") if sys.version_info[0] == 2 else iprot.readString() + ) + else: + iprot.skip(ftype) + else: + iprot.skip(ftype) + iprot.readFieldEnd() + iprot.readStructEnd() + + def write(self, oprot): + if oprot._fast_encode is not None and self.thrift_spec is not None: + oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec])) + return + oprot.writeStructBegin("add_token_args") + if self.token_identifier is not None: + oprot.writeFieldBegin("token_identifier", TType.STRING, 1) + oprot.writeString(self.token_identifier.encode("utf-8") if sys.version_info[0] == 2 else self.token_identifier) + oprot.writeFieldEnd() + if self.delegation_token is not None: + oprot.writeFieldBegin("delegation_token", TType.STRING, 2) + oprot.writeString(self.delegation_token.encode("utf-8") if sys.version_info[0] == 2 else self.delegation_token) + oprot.writeFieldEnd() + oprot.writeFieldStop() + oprot.writeStructEnd() + + def validate(self): + return + + def __repr__(self): + L = ["{}={!r}".format(key, value) for key, value in self.__dict__.items()] + return "{}({})".format(self.__class__.__name__, ", ".join(L)) + + def __eq__(self, other): + return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ + + def __ne__(self, other): + return not (self == other) + + +all_structs.append(add_token_args) +add_token_args.thrift_spec = ( + None, # 0 + ( + 1, + TType.STRING, + "token_identifier", + "UTF8", + None, + ), # 1 + ( + 2, + TType.STRING, + "delegation_token", + "UTF8", + None, + ), # 2 +) + + +class add_token_result: + """ + Attributes: + - success + + """ + + def __init__( + self, + success=None, + ): + self.success = success + + def read(self, iprot): + if ( + iprot._fast_decode is not None + and isinstance(iprot.trans, TTransport.CReadableTransport) + and self.thrift_spec is not None + ): + iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec]) + return + iprot.readStructBegin() + while True: + (fname, ftype, fid) = iprot.readFieldBegin() + if ftype == TType.STOP: + break + if fid == 0: + if ftype == TType.BOOL: + self.success = iprot.readBool() + else: + iprot.skip(ftype) + else: + iprot.skip(ftype) + iprot.readFieldEnd() + iprot.readStructEnd() + + def write(self, oprot): + if oprot._fast_encode is not None and self.thrift_spec is not None: + oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec])) + return + oprot.writeStructBegin("add_token_result") + if self.success is not None: + oprot.writeFieldBegin("success", TType.BOOL, 0) + oprot.writeBool(self.success) + oprot.writeFieldEnd() + oprot.writeFieldStop() + oprot.writeStructEnd() + + def validate(self): + return + + def __repr__(self): + L = ["{}={!r}".format(key, value) for key, value in self.__dict__.items()] + return "{}({})".format(self.__class__.__name__, ", ".join(L)) + + def __eq__(self, other): + return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ + + def __ne__(self, other): + return not (self == other) + + +all_structs.append(add_token_result) +add_token_result.thrift_spec = ( + ( + 0, + TType.BOOL, + "success", + None, + None, + ), # 0 +) + + +class remove_token_args: + """ + Attributes: + - token_identifier + + """ + + def __init__( + self, + token_identifier=None, + ): + self.token_identifier = token_identifier + + def read(self, iprot): + if ( + iprot._fast_decode is not None + and isinstance(iprot.trans, TTransport.CReadableTransport) + and self.thrift_spec is not None + ): + iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec]) + return + iprot.readStructBegin() + while True: + (fname, ftype, fid) = iprot.readFieldBegin() + if ftype == TType.STOP: + break + if fid == 1: + if ftype == TType.STRING: + self.token_identifier = ( + iprot.readString().decode("utf-8", errors="replace") if sys.version_info[0] == 2 else iprot.readString() + ) + else: + iprot.skip(ftype) + else: + iprot.skip(ftype) + iprot.readFieldEnd() + iprot.readStructEnd() + + def write(self, oprot): + if oprot._fast_encode is not None and self.thrift_spec is not None: + oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec])) + return + oprot.writeStructBegin("remove_token_args") + if self.token_identifier is not None: + oprot.writeFieldBegin("token_identifier", TType.STRING, 1) + oprot.writeString(self.token_identifier.encode("utf-8") if sys.version_info[0] == 2 else self.token_identifier) + oprot.writeFieldEnd() + oprot.writeFieldStop() + oprot.writeStructEnd() + + def validate(self): + return + + def __repr__(self): + L = ["{}={!r}".format(key, value) for key, value in self.__dict__.items()] + return "{}({})".format(self.__class__.__name__, ", ".join(L)) + + def __eq__(self, other): + return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ + + def __ne__(self, other): + return not (self == other) + + +all_structs.append(remove_token_args) +remove_token_args.thrift_spec = ( + None, # 0 + ( + 1, + TType.STRING, + "token_identifier", + "UTF8", + None, + ), # 1 +) + + +class remove_token_result: + """ + Attributes: + - success + + """ + + def __init__( + self, + success=None, + ): + self.success = success + + def read(self, iprot): + if ( + iprot._fast_decode is not None + and isinstance(iprot.trans, TTransport.CReadableTransport) + and self.thrift_spec is not None + ): + iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec]) + return + iprot.readStructBegin() + while True: + (fname, ftype, fid) = iprot.readFieldBegin() + if ftype == TType.STOP: + break + if fid == 0: + if ftype == TType.BOOL: + self.success = iprot.readBool() + else: + iprot.skip(ftype) + else: + iprot.skip(ftype) + iprot.readFieldEnd() + iprot.readStructEnd() + + def write(self, oprot): + if oprot._fast_encode is not None and self.thrift_spec is not None: + oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec])) + return + oprot.writeStructBegin("remove_token_result") + if self.success is not None: + oprot.writeFieldBegin("success", TType.BOOL, 0) + oprot.writeBool(self.success) + oprot.writeFieldEnd() + oprot.writeFieldStop() + oprot.writeStructEnd() + + def validate(self): + return + + def __repr__(self): + L = ["{}={!r}".format(key, value) for key, value in self.__dict__.items()] + return "{}({})".format(self.__class__.__name__, ", ".join(L)) + + def __eq__(self, other): + return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ + + def __ne__(self, other): + return not (self == other) + + +all_structs.append(remove_token_result) +remove_token_result.thrift_spec = ( + ( + 0, + TType.BOOL, + "success", + None, + None, + ), # 0 +) + + +class get_token_args: + """ + Attributes: + - token_identifier + + """ + + def __init__( + self, + token_identifier=None, + ): + self.token_identifier = token_identifier + + def read(self, iprot): + if ( + iprot._fast_decode is not None + and isinstance(iprot.trans, TTransport.CReadableTransport) + and self.thrift_spec is not None + ): + iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec]) + return + iprot.readStructBegin() + while True: + (fname, ftype, fid) = iprot.readFieldBegin() + if ftype == TType.STOP: + break + if fid == 1: + if ftype == TType.STRING: + self.token_identifier = ( + iprot.readString().decode("utf-8", errors="replace") if sys.version_info[0] == 2 else iprot.readString() + ) + else: + iprot.skip(ftype) + else: + iprot.skip(ftype) + iprot.readFieldEnd() + iprot.readStructEnd() + + def write(self, oprot): + if oprot._fast_encode is not None and self.thrift_spec is not None: + oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec])) + return + oprot.writeStructBegin("get_token_args") + if self.token_identifier is not None: + oprot.writeFieldBegin("token_identifier", TType.STRING, 1) + oprot.writeString(self.token_identifier.encode("utf-8") if sys.version_info[0] == 2 else self.token_identifier) + oprot.writeFieldEnd() + oprot.writeFieldStop() + oprot.writeStructEnd() + + def validate(self): + return + + def __repr__(self): + L = ["{}={!r}".format(key, value) for key, value in self.__dict__.items()] + return "{}({})".format(self.__class__.__name__, ", ".join(L)) + + def __eq__(self, other): + return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ + + def __ne__(self, other): + return not (self == other) + + +all_structs.append(get_token_args) +get_token_args.thrift_spec = ( + None, # 0 + ( + 1, + TType.STRING, + "token_identifier", + "UTF8", + None, + ), # 1 +) + + +class get_token_result: + """ + Attributes: + - success + + """ + + def __init__( + self, + success=None, + ): + self.success = success + + def read(self, iprot): + if ( + iprot._fast_decode is not None + and isinstance(iprot.trans, TTransport.CReadableTransport) + and self.thrift_spec is not None + ): + iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec]) + return + iprot.readStructBegin() + while True: + (fname, ftype, fid) = iprot.readFieldBegin() + if ftype == TType.STOP: + break + if fid == 0: + if ftype == TType.STRING: + self.success = ( + iprot.readString().decode("utf-8", errors="replace") if sys.version_info[0] == 2 else iprot.readString() + ) + else: + iprot.skip(ftype) + else: + iprot.skip(ftype) + iprot.readFieldEnd() + iprot.readStructEnd() + + def write(self, oprot): + if oprot._fast_encode is not None and self.thrift_spec is not None: + oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec])) + return + oprot.writeStructBegin("get_token_result") + if self.success is not None: + oprot.writeFieldBegin("success", TType.STRING, 0) + oprot.writeString(self.success.encode("utf-8") if sys.version_info[0] == 2 else self.success) + oprot.writeFieldEnd() + oprot.writeFieldStop() + oprot.writeStructEnd() + + def validate(self): + return + + def __repr__(self): + L = ["{}={!r}".format(key, value) for key, value in self.__dict__.items()] + return "{}({})".format(self.__class__.__name__, ", ".join(L)) + + def __eq__(self, other): + return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ + + def __ne__(self, other): + return not (self == other) + + +all_structs.append(get_token_result) +get_token_result.thrift_spec = ( + ( + 0, + TType.STRING, + "success", + "UTF8", + None, + ), # 0 +) + + +class get_all_token_identifiers_args: + def read(self, iprot): + if ( + iprot._fast_decode is not None + and isinstance(iprot.trans, TTransport.CReadableTransport) + and self.thrift_spec is not None + ): + iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec]) + return + iprot.readStructBegin() + while True: + (fname, ftype, fid) = iprot.readFieldBegin() + if ftype == TType.STOP: + break + else: + iprot.skip(ftype) + iprot.readFieldEnd() + iprot.readStructEnd() + + def write(self, oprot): + if oprot._fast_encode is not None and self.thrift_spec is not None: + oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec])) + return + oprot.writeStructBegin("get_all_token_identifiers_args") + oprot.writeFieldStop() + oprot.writeStructEnd() + + def validate(self): + return + + def __repr__(self): + L = ["{}={!r}".format(key, value) for key, value in self.__dict__.items()] + return "{}({})".format(self.__class__.__name__, ", ".join(L)) + + def __eq__(self, other): + return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ + + def __ne__(self, other): + return not (self == other) + + +all_structs.append(get_all_token_identifiers_args) +get_all_token_identifiers_args.thrift_spec = () + + +class get_all_token_identifiers_result: + """ + Attributes: + - success + + """ + + def __init__( + self, + success=None, + ): + self.success = success + + def read(self, iprot): + if ( + iprot._fast_decode is not None + and isinstance(iprot.trans, TTransport.CReadableTransport) + and self.thrift_spec is not None + ): + iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec]) + return + iprot.readStructBegin() + while True: + (fname, ftype, fid) = iprot.readFieldBegin() + if ftype == TType.STOP: + break + if fid == 0: + if ftype == TType.LIST: + self.success = [] + (_etype1754, _size1751) = iprot.readListBegin() + for _i1755 in range(_size1751): + _elem1756 = ( + iprot.readString().decode("utf-8", errors="replace") + if sys.version_info[0] == 2 + else iprot.readString() + ) + self.success.append(_elem1756) + iprot.readListEnd() + else: + iprot.skip(ftype) + else: + iprot.skip(ftype) + iprot.readFieldEnd() + iprot.readStructEnd() + + def write(self, oprot): + if oprot._fast_encode is not None and self.thrift_spec is not None: + oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec])) + return + oprot.writeStructBegin("get_all_token_identifiers_result") + if self.success is not None: + oprot.writeFieldBegin("success", TType.LIST, 0) + oprot.writeListBegin(TType.STRING, len(self.success)) + for iter1757 in self.success: + oprot.writeString(iter1757.encode("utf-8") if sys.version_info[0] == 2 else iter1757) + oprot.writeListEnd() + oprot.writeFieldEnd() + oprot.writeFieldStop() + oprot.writeStructEnd() + + def validate(self): + return + + def __repr__(self): + L = ["{}={!r}".format(key, value) for key, value in self.__dict__.items()] + return "{}({})".format(self.__class__.__name__, ", ".join(L)) + + def __eq__(self, other): + return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ + + def __ne__(self, other): + return not (self == other) + + +all_structs.append(get_all_token_identifiers_result) +get_all_token_identifiers_result.thrift_spec = ( + ( + 0, + TType.LIST, + "success", + (TType.STRING, "UTF8", False), + None, + ), # 0 +) + + +class add_master_key_args: + """ + Attributes: + - key + + """ + + def __init__( + self, + key=None, + ): + self.key = key + + def read(self, iprot): + if ( + iprot._fast_decode is not None + and isinstance(iprot.trans, TTransport.CReadableTransport) + and self.thrift_spec is not None + ): + iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec]) + return + iprot.readStructBegin() + while True: + (fname, ftype, fid) = iprot.readFieldBegin() + if ftype == TType.STOP: + break + if fid == 1: + if ftype == TType.STRING: + self.key = ( + iprot.readString().decode("utf-8", errors="replace") if sys.version_info[0] == 2 else iprot.readString() + ) + else: + iprot.skip(ftype) + else: + iprot.skip(ftype) + iprot.readFieldEnd() + iprot.readStructEnd() + + def write(self, oprot): + if oprot._fast_encode is not None and self.thrift_spec is not None: + oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec])) + return + oprot.writeStructBegin("add_master_key_args") + if self.key is not None: + oprot.writeFieldBegin("key", TType.STRING, 1) + oprot.writeString(self.key.encode("utf-8") if sys.version_info[0] == 2 else self.key) + oprot.writeFieldEnd() + oprot.writeFieldStop() + oprot.writeStructEnd() + + def validate(self): + return + + def __repr__(self): + L = ["{}={!r}".format(key, value) for key, value in self.__dict__.items()] + return "{}({})".format(self.__class__.__name__, ", ".join(L)) + + def __eq__(self, other): + return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ + + def __ne__(self, other): + return not (self == other) + + +all_structs.append(add_master_key_args) +add_master_key_args.thrift_spec = ( + None, # 0 + ( + 1, + TType.STRING, + "key", + "UTF8", + None, + ), # 1 +) + + +class add_master_key_result: + """ + Attributes: + - success + - o1 + + """ + + def __init__( + self, + success=None, + o1=None, + ): + self.success = success + self.o1 = o1 + + def read(self, iprot): + if ( + iprot._fast_decode is not None + and isinstance(iprot.trans, TTransport.CReadableTransport) + and self.thrift_spec is not None + ): + iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec]) + return + iprot.readStructBegin() + while True: + (fname, ftype, fid) = iprot.readFieldBegin() + if ftype == TType.STOP: + break + if fid == 0: + if ftype == TType.I32: + self.success = iprot.readI32() + else: + iprot.skip(ftype) + elif fid == 1: + if ftype == TType.STRUCT: + self.o1 = MetaException.read(iprot) + else: + iprot.skip(ftype) + else: + iprot.skip(ftype) + iprot.readFieldEnd() + iprot.readStructEnd() + + def write(self, oprot): + if oprot._fast_encode is not None and self.thrift_spec is not None: + oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec])) + return + oprot.writeStructBegin("add_master_key_result") + if self.success is not None: + oprot.writeFieldBegin("success", TType.I32, 0) + oprot.writeI32(self.success) + oprot.writeFieldEnd() + if self.o1 is not None: + oprot.writeFieldBegin("o1", TType.STRUCT, 1) + self.o1.write(oprot) + oprot.writeFieldEnd() + oprot.writeFieldStop() + oprot.writeStructEnd() + + def validate(self): + return + + def __repr__(self): + L = ["{}={!r}".format(key, value) for key, value in self.__dict__.items()] + return "{}({})".format(self.__class__.__name__, ", ".join(L)) + + def __eq__(self, other): + return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ + + def __ne__(self, other): + return not (self == other) + + +all_structs.append(add_master_key_result) +add_master_key_result.thrift_spec = ( + ( + 0, + TType.I32, + "success", + None, + None, + ), # 0 + ( + 1, + TType.STRUCT, + "o1", + [MetaException, None], + None, + ), # 1 +) + + +class update_master_key_args: + """ + Attributes: + - seq_number + - key + + """ + + def __init__( + self, + seq_number=None, + key=None, + ): + self.seq_number = seq_number + self.key = key + + def read(self, iprot): + if ( + iprot._fast_decode is not None + and isinstance(iprot.trans, TTransport.CReadableTransport) + and self.thrift_spec is not None + ): + iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec]) + return + iprot.readStructBegin() + while True: + (fname, ftype, fid) = iprot.readFieldBegin() + if ftype == TType.STOP: + break + if fid == 1: + if ftype == TType.I32: + self.seq_number = iprot.readI32() + else: + iprot.skip(ftype) + elif fid == 2: + if ftype == TType.STRING: + self.key = ( + iprot.readString().decode("utf-8", errors="replace") if sys.version_info[0] == 2 else iprot.readString() + ) + else: + iprot.skip(ftype) + else: + iprot.skip(ftype) + iprot.readFieldEnd() + iprot.readStructEnd() + + def write(self, oprot): + if oprot._fast_encode is not None and self.thrift_spec is not None: + oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec])) + return + oprot.writeStructBegin("update_master_key_args") + if self.seq_number is not None: + oprot.writeFieldBegin("seq_number", TType.I32, 1) + oprot.writeI32(self.seq_number) + oprot.writeFieldEnd() + if self.key is not None: + oprot.writeFieldBegin("key", TType.STRING, 2) + oprot.writeString(self.key.encode("utf-8") if sys.version_info[0] == 2 else self.key) + oprot.writeFieldEnd() + oprot.writeFieldStop() + oprot.writeStructEnd() + + def validate(self): + return + + def __repr__(self): + L = ["{}={!r}".format(key, value) for key, value in self.__dict__.items()] + return "{}({})".format(self.__class__.__name__, ", ".join(L)) + + def __eq__(self, other): + return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ + + def __ne__(self, other): + return not (self == other) + + +all_structs.append(update_master_key_args) +update_master_key_args.thrift_spec = ( + None, # 0 + ( + 1, + TType.I32, + "seq_number", + None, + None, + ), # 1 + ( + 2, + TType.STRING, + "key", + "UTF8", + None, + ), # 2 +) + + +class update_master_key_result: + """ + Attributes: + - o1 + - o2 + + """ + + def __init__( + self, + o1=None, + o2=None, + ): + self.o1 = o1 + self.o2 = o2 + + def read(self, iprot): + if ( + iprot._fast_decode is not None + and isinstance(iprot.trans, TTransport.CReadableTransport) + and self.thrift_spec is not None + ): + iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec]) + return + iprot.readStructBegin() + while True: + (fname, ftype, fid) = iprot.readFieldBegin() + if ftype == TType.STOP: + break + if fid == 1: + if ftype == TType.STRUCT: + self.o1 = NoSuchObjectException.read(iprot) + else: + iprot.skip(ftype) + elif fid == 2: + if ftype == TType.STRUCT: + self.o2 = MetaException.read(iprot) + else: + iprot.skip(ftype) + else: + iprot.skip(ftype) + iprot.readFieldEnd() + iprot.readStructEnd() + + def write(self, oprot): + if oprot._fast_encode is not None and self.thrift_spec is not None: + oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec])) + return + oprot.writeStructBegin("update_master_key_result") + if self.o1 is not None: + oprot.writeFieldBegin("o1", TType.STRUCT, 1) + self.o1.write(oprot) + oprot.writeFieldEnd() + if self.o2 is not None: + oprot.writeFieldBegin("o2", TType.STRUCT, 2) + self.o2.write(oprot) + oprot.writeFieldEnd() + oprot.writeFieldStop() + oprot.writeStructEnd() + + def validate(self): + return + + def __repr__(self): + L = ["{}={!r}".format(key, value) for key, value in self.__dict__.items()] + return "{}({})".format(self.__class__.__name__, ", ".join(L)) + + def __eq__(self, other): + return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ + + def __ne__(self, other): + return not (self == other) + + +all_structs.append(update_master_key_result) +update_master_key_result.thrift_spec = ( + None, # 0 + ( + 1, + TType.STRUCT, + "o1", + [NoSuchObjectException, None], + None, + ), # 1 + ( + 2, + TType.STRUCT, + "o2", + [MetaException, None], + None, + ), # 2 +) + + +class remove_master_key_args: + """ + Attributes: + - key_seq + + """ + + def __init__( + self, + key_seq=None, + ): + self.key_seq = key_seq + + def read(self, iprot): + if ( + iprot._fast_decode is not None + and isinstance(iprot.trans, TTransport.CReadableTransport) + and self.thrift_spec is not None + ): + iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec]) + return + iprot.readStructBegin() + while True: + (fname, ftype, fid) = iprot.readFieldBegin() + if ftype == TType.STOP: + break + if fid == 1: + if ftype == TType.I32: + self.key_seq = iprot.readI32() + else: + iprot.skip(ftype) + else: + iprot.skip(ftype) + iprot.readFieldEnd() + iprot.readStructEnd() + + def write(self, oprot): + if oprot._fast_encode is not None and self.thrift_spec is not None: + oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec])) + return + oprot.writeStructBegin("remove_master_key_args") + if self.key_seq is not None: + oprot.writeFieldBegin("key_seq", TType.I32, 1) + oprot.writeI32(self.key_seq) + oprot.writeFieldEnd() + oprot.writeFieldStop() + oprot.writeStructEnd() + + def validate(self): + return + + def __repr__(self): + L = ["{}={!r}".format(key, value) for key, value in self.__dict__.items()] + return "{}({})".format(self.__class__.__name__, ", ".join(L)) + + def __eq__(self, other): + return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ + + def __ne__(self, other): + return not (self == other) + + +all_structs.append(remove_master_key_args) +remove_master_key_args.thrift_spec = ( + None, # 0 + ( + 1, + TType.I32, + "key_seq", + None, + None, + ), # 1 +) + + +class remove_master_key_result: + """ + Attributes: + - success + + """ + + def __init__( + self, + success=None, + ): + self.success = success + + def read(self, iprot): + if ( + iprot._fast_decode is not None + and isinstance(iprot.trans, TTransport.CReadableTransport) + and self.thrift_spec is not None + ): + iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec]) + return + iprot.readStructBegin() + while True: + (fname, ftype, fid) = iprot.readFieldBegin() + if ftype == TType.STOP: + break + if fid == 0: + if ftype == TType.BOOL: + self.success = iprot.readBool() + else: + iprot.skip(ftype) + else: + iprot.skip(ftype) + iprot.readFieldEnd() + iprot.readStructEnd() + + def write(self, oprot): + if oprot._fast_encode is not None and self.thrift_spec is not None: + oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec])) + return + oprot.writeStructBegin("remove_master_key_result") + if self.success is not None: + oprot.writeFieldBegin("success", TType.BOOL, 0) + oprot.writeBool(self.success) + oprot.writeFieldEnd() + oprot.writeFieldStop() + oprot.writeStructEnd() + + def validate(self): + return + + def __repr__(self): + L = ["{}={!r}".format(key, value) for key, value in self.__dict__.items()] + return "{}({})".format(self.__class__.__name__, ", ".join(L)) + + def __eq__(self, other): + return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ + + def __ne__(self, other): + return not (self == other) + + +all_structs.append(remove_master_key_result) +remove_master_key_result.thrift_spec = ( + ( + 0, + TType.BOOL, + "success", + None, + None, + ), # 0 +) + + +class get_master_keys_args: + def read(self, iprot): + if ( + iprot._fast_decode is not None + and isinstance(iprot.trans, TTransport.CReadableTransport) + and self.thrift_spec is not None + ): + iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec]) + return + iprot.readStructBegin() + while True: + (fname, ftype, fid) = iprot.readFieldBegin() + if ftype == TType.STOP: + break + else: + iprot.skip(ftype) + iprot.readFieldEnd() + iprot.readStructEnd() + + def write(self, oprot): + if oprot._fast_encode is not None and self.thrift_spec is not None: + oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec])) + return + oprot.writeStructBegin("get_master_keys_args") + oprot.writeFieldStop() + oprot.writeStructEnd() + + def validate(self): + return + + def __repr__(self): + L = ["{}={!r}".format(key, value) for key, value in self.__dict__.items()] + return "{}({})".format(self.__class__.__name__, ", ".join(L)) + + def __eq__(self, other): + return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ + + def __ne__(self, other): + return not (self == other) + + +all_structs.append(get_master_keys_args) +get_master_keys_args.thrift_spec = () + + +class get_master_keys_result: + """ + Attributes: + - success + + """ + + def __init__( + self, + success=None, + ): + self.success = success + + def read(self, iprot): + if ( + iprot._fast_decode is not None + and isinstance(iprot.trans, TTransport.CReadableTransport) + and self.thrift_spec is not None + ): + iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec]) + return + iprot.readStructBegin() + while True: + (fname, ftype, fid) = iprot.readFieldBegin() + if ftype == TType.STOP: + break + if fid == 0: + if ftype == TType.LIST: + self.success = [] + (_etype1761, _size1758) = iprot.readListBegin() + for _i1762 in range(_size1758): + _elem1763 = ( + iprot.readString().decode("utf-8", errors="replace") + if sys.version_info[0] == 2 + else iprot.readString() + ) + self.success.append(_elem1763) + iprot.readListEnd() + else: + iprot.skip(ftype) + else: + iprot.skip(ftype) + iprot.readFieldEnd() + iprot.readStructEnd() + + def write(self, oprot): + if oprot._fast_encode is not None and self.thrift_spec is not None: + oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec])) + return + oprot.writeStructBegin("get_master_keys_result") + if self.success is not None: + oprot.writeFieldBegin("success", TType.LIST, 0) + oprot.writeListBegin(TType.STRING, len(self.success)) + for iter1764 in self.success: + oprot.writeString(iter1764.encode("utf-8") if sys.version_info[0] == 2 else iter1764) + oprot.writeListEnd() + oprot.writeFieldEnd() + oprot.writeFieldStop() + oprot.writeStructEnd() + + def validate(self): + return + + def __repr__(self): + L = ["{}={!r}".format(key, value) for key, value in self.__dict__.items()] + return "{}({})".format(self.__class__.__name__, ", ".join(L)) + + def __eq__(self, other): + return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ + + def __ne__(self, other): + return not (self == other) + + +all_structs.append(get_master_keys_result) +get_master_keys_result.thrift_spec = ( + ( + 0, + TType.LIST, + "success", + (TType.STRING, "UTF8", False), + None, + ), # 0 +) + + +class get_open_txns_args: + def read(self, iprot): + if ( + iprot._fast_decode is not None + and isinstance(iprot.trans, TTransport.CReadableTransport) + and self.thrift_spec is not None + ): + iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec]) + return + iprot.readStructBegin() + while True: + (fname, ftype, fid) = iprot.readFieldBegin() + if ftype == TType.STOP: + break + else: + iprot.skip(ftype) + iprot.readFieldEnd() + iprot.readStructEnd() + + def write(self, oprot): + if oprot._fast_encode is not None and self.thrift_spec is not None: + oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec])) + return + oprot.writeStructBegin("get_open_txns_args") + oprot.writeFieldStop() + oprot.writeStructEnd() + + def validate(self): + return + + def __repr__(self): + L = ["{}={!r}".format(key, value) for key, value in self.__dict__.items()] + return "{}({})".format(self.__class__.__name__, ", ".join(L)) + + def __eq__(self, other): + return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ + + def __ne__(self, other): + return not (self == other) + + +all_structs.append(get_open_txns_args) +get_open_txns_args.thrift_spec = () + + +class get_open_txns_result: + """ + Attributes: + - success + + """ + + def __init__( + self, + success=None, + ): + self.success = success + + def read(self, iprot): + if ( + iprot._fast_decode is not None + and isinstance(iprot.trans, TTransport.CReadableTransport) + and self.thrift_spec is not None + ): + iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec]) + return + iprot.readStructBegin() + while True: + (fname, ftype, fid) = iprot.readFieldBegin() + if ftype == TType.STOP: + break + if fid == 0: + if ftype == TType.STRUCT: + self.success = GetOpenTxnsResponse() + self.success.read(iprot) + else: + iprot.skip(ftype) + else: + iprot.skip(ftype) + iprot.readFieldEnd() + iprot.readStructEnd() + + def write(self, oprot): + if oprot._fast_encode is not None and self.thrift_spec is not None: + oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec])) + return + oprot.writeStructBegin("get_open_txns_result") + if self.success is not None: + oprot.writeFieldBegin("success", TType.STRUCT, 0) + self.success.write(oprot) + oprot.writeFieldEnd() + oprot.writeFieldStop() + oprot.writeStructEnd() + + def validate(self): + return + + def __repr__(self): + L = ["{}={!r}".format(key, value) for key, value in self.__dict__.items()] + return "{}({})".format(self.__class__.__name__, ", ".join(L)) + + def __eq__(self, other): + return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ + + def __ne__(self, other): + return not (self == other) + + +all_structs.append(get_open_txns_result) +get_open_txns_result.thrift_spec = ( + ( + 0, + TType.STRUCT, + "success", + [GetOpenTxnsResponse, None], + None, + ), # 0 +) + + +class get_open_txns_info_args: + def read(self, iprot): + if ( + iprot._fast_decode is not None + and isinstance(iprot.trans, TTransport.CReadableTransport) + and self.thrift_spec is not None + ): + iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec]) + return + iprot.readStructBegin() + while True: + (fname, ftype, fid) = iprot.readFieldBegin() + if ftype == TType.STOP: + break + else: + iprot.skip(ftype) + iprot.readFieldEnd() + iprot.readStructEnd() + + def write(self, oprot): + if oprot._fast_encode is not None and self.thrift_spec is not None: + oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec])) + return + oprot.writeStructBegin("get_open_txns_info_args") + oprot.writeFieldStop() + oprot.writeStructEnd() + + def validate(self): + return + + def __repr__(self): + L = ["{}={!r}".format(key, value) for key, value in self.__dict__.items()] + return "{}({})".format(self.__class__.__name__, ", ".join(L)) + + def __eq__(self, other): + return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ + + def __ne__(self, other): + return not (self == other) + + +all_structs.append(get_open_txns_info_args) +get_open_txns_info_args.thrift_spec = () + + +class get_open_txns_info_result: + """ + Attributes: + - success + + """ + + def __init__( + self, + success=None, + ): + self.success = success + + def read(self, iprot): + if ( + iprot._fast_decode is not None + and isinstance(iprot.trans, TTransport.CReadableTransport) + and self.thrift_spec is not None + ): + iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec]) + return + iprot.readStructBegin() + while True: + (fname, ftype, fid) = iprot.readFieldBegin() + if ftype == TType.STOP: + break + if fid == 0: + if ftype == TType.STRUCT: + self.success = GetOpenTxnsInfoResponse() + self.success.read(iprot) + else: + iprot.skip(ftype) + else: + iprot.skip(ftype) + iprot.readFieldEnd() + iprot.readStructEnd() + + def write(self, oprot): + if oprot._fast_encode is not None and self.thrift_spec is not None: + oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec])) + return + oprot.writeStructBegin("get_open_txns_info_result") + if self.success is not None: + oprot.writeFieldBegin("success", TType.STRUCT, 0) + self.success.write(oprot) + oprot.writeFieldEnd() + oprot.writeFieldStop() + oprot.writeStructEnd() + + def validate(self): + return + + def __repr__(self): + L = ["{}={!r}".format(key, value) for key, value in self.__dict__.items()] + return "{}({})".format(self.__class__.__name__, ", ".join(L)) + + def __eq__(self, other): + return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ + + def __ne__(self, other): + return not (self == other) + + +all_structs.append(get_open_txns_info_result) +get_open_txns_info_result.thrift_spec = ( + ( + 0, + TType.STRUCT, + "success", + [GetOpenTxnsInfoResponse, None], + None, + ), # 0 +) + + +class open_txns_args: + """ + Attributes: + - rqst + + """ + + def __init__( + self, + rqst=None, + ): + self.rqst = rqst + + def read(self, iprot): + if ( + iprot._fast_decode is not None + and isinstance(iprot.trans, TTransport.CReadableTransport) + and self.thrift_spec is not None + ): + iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec]) + return + iprot.readStructBegin() + while True: + (fname, ftype, fid) = iprot.readFieldBegin() + if ftype == TType.STOP: + break + if fid == 1: + if ftype == TType.STRUCT: + self.rqst = OpenTxnRequest() + self.rqst.read(iprot) + else: + iprot.skip(ftype) + else: + iprot.skip(ftype) + iprot.readFieldEnd() + iprot.readStructEnd() + + def write(self, oprot): + if oprot._fast_encode is not None and self.thrift_spec is not None: + oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec])) + return + oprot.writeStructBegin("open_txns_args") + if self.rqst is not None: + oprot.writeFieldBegin("rqst", TType.STRUCT, 1) + self.rqst.write(oprot) + oprot.writeFieldEnd() + oprot.writeFieldStop() + oprot.writeStructEnd() + + def validate(self): + return + + def __repr__(self): + L = ["{}={!r}".format(key, value) for key, value in self.__dict__.items()] + return "{}({})".format(self.__class__.__name__, ", ".join(L)) + + def __eq__(self, other): + return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ + + def __ne__(self, other): + return not (self == other) + + +all_structs.append(open_txns_args) +open_txns_args.thrift_spec = ( + None, # 0 + ( + 1, + TType.STRUCT, + "rqst", + [OpenTxnRequest, None], + None, + ), # 1 +) + + +class open_txns_result: + """ + Attributes: + - success + + """ + + def __init__( + self, + success=None, + ): + self.success = success + + def read(self, iprot): + if ( + iprot._fast_decode is not None + and isinstance(iprot.trans, TTransport.CReadableTransport) + and self.thrift_spec is not None + ): + iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec]) + return + iprot.readStructBegin() + while True: + (fname, ftype, fid) = iprot.readFieldBegin() + if ftype == TType.STOP: + break + if fid == 0: + if ftype == TType.STRUCT: + self.success = OpenTxnsResponse() + self.success.read(iprot) + else: + iprot.skip(ftype) + else: + iprot.skip(ftype) + iprot.readFieldEnd() + iprot.readStructEnd() + + def write(self, oprot): + if oprot._fast_encode is not None and self.thrift_spec is not None: + oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec])) + return + oprot.writeStructBegin("open_txns_result") + if self.success is not None: + oprot.writeFieldBegin("success", TType.STRUCT, 0) + self.success.write(oprot) + oprot.writeFieldEnd() + oprot.writeFieldStop() + oprot.writeStructEnd() + + def validate(self): + return + + def __repr__(self): + L = ["{}={!r}".format(key, value) for key, value in self.__dict__.items()] + return "{}({})".format(self.__class__.__name__, ", ".join(L)) + + def __eq__(self, other): + return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ + + def __ne__(self, other): + return not (self == other) + + +all_structs.append(open_txns_result) +open_txns_result.thrift_spec = ( + ( + 0, + TType.STRUCT, + "success", + [OpenTxnsResponse, None], + None, + ), # 0 +) + + +class abort_txn_args: + """ + Attributes: + - rqst + + """ + + def __init__( + self, + rqst=None, + ): + self.rqst = rqst + + def read(self, iprot): + if ( + iprot._fast_decode is not None + and isinstance(iprot.trans, TTransport.CReadableTransport) + and self.thrift_spec is not None + ): + iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec]) + return + iprot.readStructBegin() + while True: + (fname, ftype, fid) = iprot.readFieldBegin() + if ftype == TType.STOP: + break + if fid == 1: + if ftype == TType.STRUCT: + self.rqst = AbortTxnRequest() + self.rqst.read(iprot) + else: + iprot.skip(ftype) + else: + iprot.skip(ftype) + iprot.readFieldEnd() + iprot.readStructEnd() + + def write(self, oprot): + if oprot._fast_encode is not None and self.thrift_spec is not None: + oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec])) + return + oprot.writeStructBegin("abort_txn_args") + if self.rqst is not None: + oprot.writeFieldBegin("rqst", TType.STRUCT, 1) + self.rqst.write(oprot) + oprot.writeFieldEnd() + oprot.writeFieldStop() + oprot.writeStructEnd() + + def validate(self): + return + + def __repr__(self): + L = ["{}={!r}".format(key, value) for key, value in self.__dict__.items()] + return "{}({})".format(self.__class__.__name__, ", ".join(L)) + + def __eq__(self, other): + return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ + + def __ne__(self, other): + return not (self == other) + + +all_structs.append(abort_txn_args) +abort_txn_args.thrift_spec = ( + None, # 0 + ( + 1, + TType.STRUCT, + "rqst", + [AbortTxnRequest, None], + None, + ), # 1 +) + + +class abort_txn_result: + """ + Attributes: + - o1 + + """ + + def __init__( + self, + o1=None, + ): + self.o1 = o1 + + def read(self, iprot): + if ( + iprot._fast_decode is not None + and isinstance(iprot.trans, TTransport.CReadableTransport) + and self.thrift_spec is not None + ): + iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec]) + return + iprot.readStructBegin() + while True: + (fname, ftype, fid) = iprot.readFieldBegin() + if ftype == TType.STOP: + break + if fid == 1: + if ftype == TType.STRUCT: + self.o1 = NoSuchTxnException.read(iprot) + else: + iprot.skip(ftype) + else: + iprot.skip(ftype) + iprot.readFieldEnd() + iprot.readStructEnd() + + def write(self, oprot): + if oprot._fast_encode is not None and self.thrift_spec is not None: + oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec])) + return + oprot.writeStructBegin("abort_txn_result") + if self.o1 is not None: + oprot.writeFieldBegin("o1", TType.STRUCT, 1) + self.o1.write(oprot) + oprot.writeFieldEnd() + oprot.writeFieldStop() + oprot.writeStructEnd() + + def validate(self): + return + + def __repr__(self): + L = ["{}={!r}".format(key, value) for key, value in self.__dict__.items()] + return "{}({})".format(self.__class__.__name__, ", ".join(L)) + + def __eq__(self, other): + return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ + + def __ne__(self, other): + return not (self == other) + + +all_structs.append(abort_txn_result) +abort_txn_result.thrift_spec = ( + None, # 0 + ( + 1, + TType.STRUCT, + "o1", + [NoSuchTxnException, None], + None, + ), # 1 +) + + +class abort_txns_args: + """ + Attributes: + - rqst + + """ + + def __init__( + self, + rqst=None, + ): + self.rqst = rqst + + def read(self, iprot): + if ( + iprot._fast_decode is not None + and isinstance(iprot.trans, TTransport.CReadableTransport) + and self.thrift_spec is not None + ): + iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec]) + return + iprot.readStructBegin() + while True: + (fname, ftype, fid) = iprot.readFieldBegin() + if ftype == TType.STOP: + break + if fid == 1: + if ftype == TType.STRUCT: + self.rqst = AbortTxnsRequest() + self.rqst.read(iprot) + else: + iprot.skip(ftype) + else: + iprot.skip(ftype) + iprot.readFieldEnd() + iprot.readStructEnd() + + def write(self, oprot): + if oprot._fast_encode is not None and self.thrift_spec is not None: + oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec])) + return + oprot.writeStructBegin("abort_txns_args") + if self.rqst is not None: + oprot.writeFieldBegin("rqst", TType.STRUCT, 1) + self.rqst.write(oprot) + oprot.writeFieldEnd() + oprot.writeFieldStop() + oprot.writeStructEnd() + + def validate(self): + return + + def __repr__(self): + L = ["{}={!r}".format(key, value) for key, value in self.__dict__.items()] + return "{}({})".format(self.__class__.__name__, ", ".join(L)) + + def __eq__(self, other): + return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ + + def __ne__(self, other): + return not (self == other) + + +all_structs.append(abort_txns_args) +abort_txns_args.thrift_spec = ( + None, # 0 + ( + 1, + TType.STRUCT, + "rqst", + [AbortTxnsRequest, None], + None, + ), # 1 +) + + +class abort_txns_result: + """ + Attributes: + - o1 + + """ + + def __init__( + self, + o1=None, + ): + self.o1 = o1 + + def read(self, iprot): + if ( + iprot._fast_decode is not None + and isinstance(iprot.trans, TTransport.CReadableTransport) + and self.thrift_spec is not None + ): + iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec]) + return + iprot.readStructBegin() + while True: + (fname, ftype, fid) = iprot.readFieldBegin() + if ftype == TType.STOP: + break + if fid == 1: + if ftype == TType.STRUCT: + self.o1 = NoSuchTxnException.read(iprot) + else: + iprot.skip(ftype) + else: + iprot.skip(ftype) + iprot.readFieldEnd() + iprot.readStructEnd() + + def write(self, oprot): + if oprot._fast_encode is not None and self.thrift_spec is not None: + oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec])) + return + oprot.writeStructBegin("abort_txns_result") + if self.o1 is not None: + oprot.writeFieldBegin("o1", TType.STRUCT, 1) + self.o1.write(oprot) + oprot.writeFieldEnd() + oprot.writeFieldStop() + oprot.writeStructEnd() + + def validate(self): + return + + def __repr__(self): + L = ["{}={!r}".format(key, value) for key, value in self.__dict__.items()] + return "{}({})".format(self.__class__.__name__, ", ".join(L)) + + def __eq__(self, other): + return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ + + def __ne__(self, other): + return not (self == other) + + +all_structs.append(abort_txns_result) +abort_txns_result.thrift_spec = ( + None, # 0 + ( + 1, + TType.STRUCT, + "o1", + [NoSuchTxnException, None], + None, + ), # 1 +) + + +class commit_txn_args: + """ + Attributes: + - rqst + + """ + + def __init__( + self, + rqst=None, + ): + self.rqst = rqst + + def read(self, iprot): + if ( + iprot._fast_decode is not None + and isinstance(iprot.trans, TTransport.CReadableTransport) + and self.thrift_spec is not None + ): + iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec]) + return + iprot.readStructBegin() + while True: + (fname, ftype, fid) = iprot.readFieldBegin() + if ftype == TType.STOP: + break + if fid == 1: + if ftype == TType.STRUCT: + self.rqst = CommitTxnRequest() + self.rqst.read(iprot) + else: + iprot.skip(ftype) + else: + iprot.skip(ftype) + iprot.readFieldEnd() + iprot.readStructEnd() + + def write(self, oprot): + if oprot._fast_encode is not None and self.thrift_spec is not None: + oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec])) + return + oprot.writeStructBegin("commit_txn_args") + if self.rqst is not None: + oprot.writeFieldBegin("rqst", TType.STRUCT, 1) + self.rqst.write(oprot) + oprot.writeFieldEnd() + oprot.writeFieldStop() + oprot.writeStructEnd() + + def validate(self): + return + + def __repr__(self): + L = ["{}={!r}".format(key, value) for key, value in self.__dict__.items()] + return "{}({})".format(self.__class__.__name__, ", ".join(L)) + + def __eq__(self, other): + return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ + + def __ne__(self, other): + return not (self == other) + + +all_structs.append(commit_txn_args) +commit_txn_args.thrift_spec = ( + None, # 0 + ( + 1, + TType.STRUCT, + "rqst", + [CommitTxnRequest, None], + None, + ), # 1 +) + + +class commit_txn_result: + """ + Attributes: + - o1 + - o2 + + """ + + def __init__( + self, + o1=None, + o2=None, + ): + self.o1 = o1 + self.o2 = o2 + + def read(self, iprot): + if ( + iprot._fast_decode is not None + and isinstance(iprot.trans, TTransport.CReadableTransport) + and self.thrift_spec is not None + ): + iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec]) + return + iprot.readStructBegin() + while True: + (fname, ftype, fid) = iprot.readFieldBegin() + if ftype == TType.STOP: + break + if fid == 1: + if ftype == TType.STRUCT: + self.o1 = NoSuchTxnException.read(iprot) + else: + iprot.skip(ftype) + elif fid == 2: + if ftype == TType.STRUCT: + self.o2 = TxnAbortedException.read(iprot) + else: + iprot.skip(ftype) + else: + iprot.skip(ftype) + iprot.readFieldEnd() + iprot.readStructEnd() + + def write(self, oprot): + if oprot._fast_encode is not None and self.thrift_spec is not None: + oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec])) + return + oprot.writeStructBegin("commit_txn_result") + if self.o1 is not None: + oprot.writeFieldBegin("o1", TType.STRUCT, 1) + self.o1.write(oprot) + oprot.writeFieldEnd() + if self.o2 is not None: + oprot.writeFieldBegin("o2", TType.STRUCT, 2) + self.o2.write(oprot) + oprot.writeFieldEnd() + oprot.writeFieldStop() + oprot.writeStructEnd() + + def validate(self): + return + + def __repr__(self): + L = ["{}={!r}".format(key, value) for key, value in self.__dict__.items()] + return "{}({})".format(self.__class__.__name__, ", ".join(L)) + + def __eq__(self, other): + return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ + + def __ne__(self, other): + return not (self == other) + + +all_structs.append(commit_txn_result) +commit_txn_result.thrift_spec = ( + None, # 0 + ( + 1, + TType.STRUCT, + "o1", + [NoSuchTxnException, None], + None, + ), # 1 + ( + 2, + TType.STRUCT, + "o2", + [TxnAbortedException, None], + None, + ), # 2 +) + + +class get_latest_txnid_in_conflict_args: + """ + Attributes: + - txnId + + """ + + def __init__( + self, + txnId=None, + ): + self.txnId = txnId + + def read(self, iprot): + if ( + iprot._fast_decode is not None + and isinstance(iprot.trans, TTransport.CReadableTransport) + and self.thrift_spec is not None + ): + iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec]) + return + iprot.readStructBegin() + while True: + (fname, ftype, fid) = iprot.readFieldBegin() + if ftype == TType.STOP: + break + if fid == 1: + if ftype == TType.I64: + self.txnId = iprot.readI64() + else: + iprot.skip(ftype) + else: + iprot.skip(ftype) + iprot.readFieldEnd() + iprot.readStructEnd() + + def write(self, oprot): + if oprot._fast_encode is not None and self.thrift_spec is not None: + oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec])) + return + oprot.writeStructBegin("get_latest_txnid_in_conflict_args") + if self.txnId is not None: + oprot.writeFieldBegin("txnId", TType.I64, 1) + oprot.writeI64(self.txnId) + oprot.writeFieldEnd() + oprot.writeFieldStop() + oprot.writeStructEnd() + + def validate(self): + return + + def __repr__(self): + L = ["{}={!r}".format(key, value) for key, value in self.__dict__.items()] + return "{}({})".format(self.__class__.__name__, ", ".join(L)) + + def __eq__(self, other): + return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ + + def __ne__(self, other): + return not (self == other) + + +all_structs.append(get_latest_txnid_in_conflict_args) +get_latest_txnid_in_conflict_args.thrift_spec = ( + None, # 0 + ( + 1, + TType.I64, + "txnId", + None, + None, + ), # 1 +) + + +class get_latest_txnid_in_conflict_result: + """ + Attributes: + - success + - o1 + + """ + + def __init__( + self, + success=None, + o1=None, + ): + self.success = success + self.o1 = o1 + + def read(self, iprot): + if ( + iprot._fast_decode is not None + and isinstance(iprot.trans, TTransport.CReadableTransport) + and self.thrift_spec is not None + ): + iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec]) + return + iprot.readStructBegin() + while True: + (fname, ftype, fid) = iprot.readFieldBegin() + if ftype == TType.STOP: + break + if fid == 0: + if ftype == TType.I64: + self.success = iprot.readI64() + else: + iprot.skip(ftype) + elif fid == 1: + if ftype == TType.STRUCT: + self.o1 = MetaException.read(iprot) + else: + iprot.skip(ftype) + else: + iprot.skip(ftype) + iprot.readFieldEnd() + iprot.readStructEnd() + + def write(self, oprot): + if oprot._fast_encode is not None and self.thrift_spec is not None: + oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec])) + return + oprot.writeStructBegin("get_latest_txnid_in_conflict_result") + if self.success is not None: + oprot.writeFieldBegin("success", TType.I64, 0) + oprot.writeI64(self.success) + oprot.writeFieldEnd() + if self.o1 is not None: + oprot.writeFieldBegin("o1", TType.STRUCT, 1) + self.o1.write(oprot) + oprot.writeFieldEnd() + oprot.writeFieldStop() + oprot.writeStructEnd() + + def validate(self): + return + + def __repr__(self): + L = ["{}={!r}".format(key, value) for key, value in self.__dict__.items()] + return "{}({})".format(self.__class__.__name__, ", ".join(L)) + + def __eq__(self, other): + return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ + + def __ne__(self, other): + return not (self == other) + + +all_structs.append(get_latest_txnid_in_conflict_result) +get_latest_txnid_in_conflict_result.thrift_spec = ( + ( + 0, + TType.I64, + "success", + None, + None, + ), # 0 + ( + 1, + TType.STRUCT, + "o1", + [MetaException, None], + None, + ), # 1 +) + + +class repl_tbl_writeid_state_args: + """ + Attributes: + - rqst + + """ + + def __init__( + self, + rqst=None, + ): + self.rqst = rqst + + def read(self, iprot): + if ( + iprot._fast_decode is not None + and isinstance(iprot.trans, TTransport.CReadableTransport) + and self.thrift_spec is not None + ): + iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec]) + return + iprot.readStructBegin() + while True: + (fname, ftype, fid) = iprot.readFieldBegin() + if ftype == TType.STOP: + break + if fid == 1: + if ftype == TType.STRUCT: + self.rqst = ReplTblWriteIdStateRequest() + self.rqst.read(iprot) + else: + iprot.skip(ftype) + else: + iprot.skip(ftype) + iprot.readFieldEnd() + iprot.readStructEnd() + + def write(self, oprot): + if oprot._fast_encode is not None and self.thrift_spec is not None: + oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec])) + return + oprot.writeStructBegin("repl_tbl_writeid_state_args") + if self.rqst is not None: + oprot.writeFieldBegin("rqst", TType.STRUCT, 1) + self.rqst.write(oprot) + oprot.writeFieldEnd() + oprot.writeFieldStop() + oprot.writeStructEnd() + + def validate(self): + return + + def __repr__(self): + L = ["{}={!r}".format(key, value) for key, value in self.__dict__.items()] + return "{}({})".format(self.__class__.__name__, ", ".join(L)) + + def __eq__(self, other): + return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ + + def __ne__(self, other): + return not (self == other) + + +all_structs.append(repl_tbl_writeid_state_args) +repl_tbl_writeid_state_args.thrift_spec = ( + None, # 0 + ( + 1, + TType.STRUCT, + "rqst", + [ReplTblWriteIdStateRequest, None], + None, + ), # 1 +) + + +class repl_tbl_writeid_state_result: + def read(self, iprot): + if ( + iprot._fast_decode is not None + and isinstance(iprot.trans, TTransport.CReadableTransport) + and self.thrift_spec is not None + ): + iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec]) + return + iprot.readStructBegin() + while True: + (fname, ftype, fid) = iprot.readFieldBegin() + if ftype == TType.STOP: + break + else: + iprot.skip(ftype) + iprot.readFieldEnd() + iprot.readStructEnd() + + def write(self, oprot): + if oprot._fast_encode is not None and self.thrift_spec is not None: + oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec])) + return + oprot.writeStructBegin("repl_tbl_writeid_state_result") + oprot.writeFieldStop() + oprot.writeStructEnd() + + def validate(self): + return + + def __repr__(self): + L = ["{}={!r}".format(key, value) for key, value in self.__dict__.items()] + return "{}({})".format(self.__class__.__name__, ", ".join(L)) + + def __eq__(self, other): + return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ + + def __ne__(self, other): + return not (self == other) + + +all_structs.append(repl_tbl_writeid_state_result) +repl_tbl_writeid_state_result.thrift_spec = () + + +class get_valid_write_ids_args: + """ + Attributes: + - rqst + + """ + + def __init__( + self, + rqst=None, + ): + self.rqst = rqst + + def read(self, iprot): + if ( + iprot._fast_decode is not None + and isinstance(iprot.trans, TTransport.CReadableTransport) + and self.thrift_spec is not None + ): + iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec]) + return + iprot.readStructBegin() + while True: + (fname, ftype, fid) = iprot.readFieldBegin() + if ftype == TType.STOP: + break + if fid == 1: + if ftype == TType.STRUCT: + self.rqst = GetValidWriteIdsRequest() + self.rqst.read(iprot) + else: + iprot.skip(ftype) + else: + iprot.skip(ftype) + iprot.readFieldEnd() + iprot.readStructEnd() + + def write(self, oprot): + if oprot._fast_encode is not None and self.thrift_spec is not None: + oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec])) + return + oprot.writeStructBegin("get_valid_write_ids_args") + if self.rqst is not None: + oprot.writeFieldBegin("rqst", TType.STRUCT, 1) + self.rqst.write(oprot) + oprot.writeFieldEnd() + oprot.writeFieldStop() + oprot.writeStructEnd() + + def validate(self): + return + + def __repr__(self): + L = ["{}={!r}".format(key, value) for key, value in self.__dict__.items()] + return "{}({})".format(self.__class__.__name__, ", ".join(L)) + + def __eq__(self, other): + return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ + + def __ne__(self, other): + return not (self == other) + + +all_structs.append(get_valid_write_ids_args) +get_valid_write_ids_args.thrift_spec = ( + None, # 0 + ( + 1, + TType.STRUCT, + "rqst", + [GetValidWriteIdsRequest, None], + None, + ), # 1 +) + + +class get_valid_write_ids_result: + """ + Attributes: + - success + - o1 + - o2 + + """ + + def __init__( + self, + success=None, + o1=None, + o2=None, + ): + self.success = success + self.o1 = o1 + self.o2 = o2 + + def read(self, iprot): + if ( + iprot._fast_decode is not None + and isinstance(iprot.trans, TTransport.CReadableTransport) + and self.thrift_spec is not None + ): + iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec]) + return + iprot.readStructBegin() + while True: + (fname, ftype, fid) = iprot.readFieldBegin() + if ftype == TType.STOP: + break + if fid == 0: + if ftype == TType.STRUCT: + self.success = GetValidWriteIdsResponse() + self.success.read(iprot) + else: + iprot.skip(ftype) + elif fid == 1: + if ftype == TType.STRUCT: + self.o1 = NoSuchTxnException.read(iprot) + else: + iprot.skip(ftype) + elif fid == 2: + if ftype == TType.STRUCT: + self.o2 = MetaException.read(iprot) + else: + iprot.skip(ftype) + else: + iprot.skip(ftype) + iprot.readFieldEnd() + iprot.readStructEnd() + + def write(self, oprot): + if oprot._fast_encode is not None and self.thrift_spec is not None: + oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec])) + return + oprot.writeStructBegin("get_valid_write_ids_result") + if self.success is not None: + oprot.writeFieldBegin("success", TType.STRUCT, 0) + self.success.write(oprot) + oprot.writeFieldEnd() + if self.o1 is not None: + oprot.writeFieldBegin("o1", TType.STRUCT, 1) + self.o1.write(oprot) + oprot.writeFieldEnd() + if self.o2 is not None: + oprot.writeFieldBegin("o2", TType.STRUCT, 2) + self.o2.write(oprot) + oprot.writeFieldEnd() + oprot.writeFieldStop() + oprot.writeStructEnd() + + def validate(self): + return + + def __repr__(self): + L = ["{}={!r}".format(key, value) for key, value in self.__dict__.items()] + return "{}({})".format(self.__class__.__name__, ", ".join(L)) + + def __eq__(self, other): + return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ + + def __ne__(self, other): + return not (self == other) + + +all_structs.append(get_valid_write_ids_result) +get_valid_write_ids_result.thrift_spec = ( + ( + 0, + TType.STRUCT, + "success", + [GetValidWriteIdsResponse, None], + None, + ), # 0 + ( + 1, + TType.STRUCT, + "o1", + [NoSuchTxnException, None], + None, + ), # 1 + ( + 2, + TType.STRUCT, + "o2", + [MetaException, None], + None, + ), # 2 +) + + +class allocate_table_write_ids_args: + """ + Attributes: + - rqst + + """ + + def __init__( + self, + rqst=None, + ): + self.rqst = rqst + + def read(self, iprot): + if ( + iprot._fast_decode is not None + and isinstance(iprot.trans, TTransport.CReadableTransport) + and self.thrift_spec is not None + ): + iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec]) + return + iprot.readStructBegin() + while True: + (fname, ftype, fid) = iprot.readFieldBegin() + if ftype == TType.STOP: + break + if fid == 1: + if ftype == TType.STRUCT: + self.rqst = AllocateTableWriteIdsRequest() + self.rqst.read(iprot) + else: + iprot.skip(ftype) + else: + iprot.skip(ftype) + iprot.readFieldEnd() + iprot.readStructEnd() + + def write(self, oprot): + if oprot._fast_encode is not None and self.thrift_spec is not None: + oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec])) + return + oprot.writeStructBegin("allocate_table_write_ids_args") + if self.rqst is not None: + oprot.writeFieldBegin("rqst", TType.STRUCT, 1) + self.rqst.write(oprot) + oprot.writeFieldEnd() + oprot.writeFieldStop() + oprot.writeStructEnd() + + def validate(self): + return + + def __repr__(self): + L = ["{}={!r}".format(key, value) for key, value in self.__dict__.items()] + return "{}({})".format(self.__class__.__name__, ", ".join(L)) + + def __eq__(self, other): + return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ + + def __ne__(self, other): + return not (self == other) + + +all_structs.append(allocate_table_write_ids_args) +allocate_table_write_ids_args.thrift_spec = ( + None, # 0 + ( + 1, + TType.STRUCT, + "rqst", + [AllocateTableWriteIdsRequest, None], + None, + ), # 1 +) + + +class allocate_table_write_ids_result: + """ + Attributes: + - success + - o1 + - o2 + - o3 + + """ + + def __init__( + self, + success=None, + o1=None, + o2=None, + o3=None, + ): + self.success = success + self.o1 = o1 + self.o2 = o2 + self.o3 = o3 + + def read(self, iprot): + if ( + iprot._fast_decode is not None + and isinstance(iprot.trans, TTransport.CReadableTransport) + and self.thrift_spec is not None + ): + iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec]) + return + iprot.readStructBegin() + while True: + (fname, ftype, fid) = iprot.readFieldBegin() + if ftype == TType.STOP: + break + if fid == 0: + if ftype == TType.STRUCT: + self.success = AllocateTableWriteIdsResponse() + self.success.read(iprot) + else: + iprot.skip(ftype) + elif fid == 1: + if ftype == TType.STRUCT: + self.o1 = NoSuchTxnException.read(iprot) + else: + iprot.skip(ftype) + elif fid == 2: + if ftype == TType.STRUCT: + self.o2 = TxnAbortedException.read(iprot) + else: + iprot.skip(ftype) + elif fid == 3: + if ftype == TType.STRUCT: + self.o3 = MetaException.read(iprot) + else: + iprot.skip(ftype) + else: + iprot.skip(ftype) + iprot.readFieldEnd() + iprot.readStructEnd() + + def write(self, oprot): + if oprot._fast_encode is not None and self.thrift_spec is not None: + oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec])) + return + oprot.writeStructBegin("allocate_table_write_ids_result") + if self.success is not None: + oprot.writeFieldBegin("success", TType.STRUCT, 0) + self.success.write(oprot) + oprot.writeFieldEnd() + if self.o1 is not None: + oprot.writeFieldBegin("o1", TType.STRUCT, 1) + self.o1.write(oprot) + oprot.writeFieldEnd() + if self.o2 is not None: + oprot.writeFieldBegin("o2", TType.STRUCT, 2) + self.o2.write(oprot) + oprot.writeFieldEnd() + if self.o3 is not None: + oprot.writeFieldBegin("o3", TType.STRUCT, 3) + self.o3.write(oprot) + oprot.writeFieldEnd() + oprot.writeFieldStop() + oprot.writeStructEnd() + + def validate(self): + return + + def __repr__(self): + L = ["{}={!r}".format(key, value) for key, value in self.__dict__.items()] + return "{}({})".format(self.__class__.__name__, ", ".join(L)) + + def __eq__(self, other): + return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ + + def __ne__(self, other): + return not (self == other) + + +all_structs.append(allocate_table_write_ids_result) +allocate_table_write_ids_result.thrift_spec = ( + ( + 0, + TType.STRUCT, + "success", + [AllocateTableWriteIdsResponse, None], + None, + ), # 0 + ( + 1, + TType.STRUCT, + "o1", + [NoSuchTxnException, None], + None, + ), # 1 + ( + 2, + TType.STRUCT, + "o2", + [TxnAbortedException, None], + None, + ), # 2 + ( + 3, + TType.STRUCT, + "o3", + [MetaException, None], + None, + ), # 3 +) + + +class get_max_allocated_table_write_id_args: + """ + Attributes: + - rqst + + """ + + def __init__( + self, + rqst=None, + ): + self.rqst = rqst + + def read(self, iprot): + if ( + iprot._fast_decode is not None + and isinstance(iprot.trans, TTransport.CReadableTransport) + and self.thrift_spec is not None + ): + iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec]) + return + iprot.readStructBegin() + while True: + (fname, ftype, fid) = iprot.readFieldBegin() + if ftype == TType.STOP: + break + if fid == 1: + if ftype == TType.STRUCT: + self.rqst = MaxAllocatedTableWriteIdRequest() + self.rqst.read(iprot) + else: + iprot.skip(ftype) + else: + iprot.skip(ftype) + iprot.readFieldEnd() + iprot.readStructEnd() + + def write(self, oprot): + if oprot._fast_encode is not None and self.thrift_spec is not None: + oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec])) + return + oprot.writeStructBegin("get_max_allocated_table_write_id_args") + if self.rqst is not None: + oprot.writeFieldBegin("rqst", TType.STRUCT, 1) + self.rqst.write(oprot) + oprot.writeFieldEnd() + oprot.writeFieldStop() + oprot.writeStructEnd() + + def validate(self): + return + + def __repr__(self): + L = ["{}={!r}".format(key, value) for key, value in self.__dict__.items()] + return "{}({})".format(self.__class__.__name__, ", ".join(L)) + + def __eq__(self, other): + return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ + + def __ne__(self, other): + return not (self == other) + + +all_structs.append(get_max_allocated_table_write_id_args) +get_max_allocated_table_write_id_args.thrift_spec = ( + None, # 0 + ( + 1, + TType.STRUCT, + "rqst", + [MaxAllocatedTableWriteIdRequest, None], + None, + ), # 1 +) + + +class get_max_allocated_table_write_id_result: + """ + Attributes: + - success + - o1 + + """ + + def __init__( + self, + success=None, + o1=None, + ): + self.success = success + self.o1 = o1 + + def read(self, iprot): + if ( + iprot._fast_decode is not None + and isinstance(iprot.trans, TTransport.CReadableTransport) + and self.thrift_spec is not None + ): + iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec]) + return + iprot.readStructBegin() + while True: + (fname, ftype, fid) = iprot.readFieldBegin() + if ftype == TType.STOP: + break + if fid == 0: + if ftype == TType.STRUCT: + self.success = MaxAllocatedTableWriteIdResponse() + self.success.read(iprot) + else: + iprot.skip(ftype) + elif fid == 1: + if ftype == TType.STRUCT: + self.o1 = MetaException.read(iprot) + else: + iprot.skip(ftype) + else: + iprot.skip(ftype) + iprot.readFieldEnd() + iprot.readStructEnd() + + def write(self, oprot): + if oprot._fast_encode is not None and self.thrift_spec is not None: + oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec])) + return + oprot.writeStructBegin("get_max_allocated_table_write_id_result") + if self.success is not None: + oprot.writeFieldBegin("success", TType.STRUCT, 0) + self.success.write(oprot) + oprot.writeFieldEnd() + if self.o1 is not None: + oprot.writeFieldBegin("o1", TType.STRUCT, 1) + self.o1.write(oprot) + oprot.writeFieldEnd() + oprot.writeFieldStop() + oprot.writeStructEnd() + + def validate(self): + return + + def __repr__(self): + L = ["{}={!r}".format(key, value) for key, value in self.__dict__.items()] + return "{}({})".format(self.__class__.__name__, ", ".join(L)) + + def __eq__(self, other): + return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ + + def __ne__(self, other): + return not (self == other) + + +all_structs.append(get_max_allocated_table_write_id_result) +get_max_allocated_table_write_id_result.thrift_spec = ( + ( + 0, + TType.STRUCT, + "success", + [MaxAllocatedTableWriteIdResponse, None], + None, + ), # 0 + ( + 1, + TType.STRUCT, + "o1", + [MetaException, None], + None, + ), # 1 +) + + +class seed_write_id_args: + """ + Attributes: + - rqst + + """ + + def __init__( + self, + rqst=None, + ): + self.rqst = rqst + + def read(self, iprot): + if ( + iprot._fast_decode is not None + and isinstance(iprot.trans, TTransport.CReadableTransport) + and self.thrift_spec is not None + ): + iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec]) + return + iprot.readStructBegin() + while True: + (fname, ftype, fid) = iprot.readFieldBegin() + if ftype == TType.STOP: + break + if fid == 1: + if ftype == TType.STRUCT: + self.rqst = SeedTableWriteIdsRequest() + self.rqst.read(iprot) + else: + iprot.skip(ftype) + else: + iprot.skip(ftype) + iprot.readFieldEnd() + iprot.readStructEnd() + + def write(self, oprot): + if oprot._fast_encode is not None and self.thrift_spec is not None: + oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec])) + return + oprot.writeStructBegin("seed_write_id_args") + if self.rqst is not None: + oprot.writeFieldBegin("rqst", TType.STRUCT, 1) + self.rqst.write(oprot) + oprot.writeFieldEnd() + oprot.writeFieldStop() + oprot.writeStructEnd() + + def validate(self): + return + + def __repr__(self): + L = ["{}={!r}".format(key, value) for key, value in self.__dict__.items()] + return "{}({})".format(self.__class__.__name__, ", ".join(L)) + + def __eq__(self, other): + return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ + + def __ne__(self, other): + return not (self == other) + + +all_structs.append(seed_write_id_args) +seed_write_id_args.thrift_spec = ( + None, # 0 + ( + 1, + TType.STRUCT, + "rqst", + [SeedTableWriteIdsRequest, None], + None, + ), # 1 +) + + +class seed_write_id_result: + """ + Attributes: + - o1 + + """ + + def __init__( + self, + o1=None, + ): + self.o1 = o1 + + def read(self, iprot): + if ( + iprot._fast_decode is not None + and isinstance(iprot.trans, TTransport.CReadableTransport) + and self.thrift_spec is not None + ): + iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec]) + return + iprot.readStructBegin() + while True: + (fname, ftype, fid) = iprot.readFieldBegin() + if ftype == TType.STOP: + break + if fid == 1: + if ftype == TType.STRUCT: + self.o1 = MetaException.read(iprot) + else: + iprot.skip(ftype) + else: + iprot.skip(ftype) + iprot.readFieldEnd() + iprot.readStructEnd() + + def write(self, oprot): + if oprot._fast_encode is not None and self.thrift_spec is not None: + oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec])) + return + oprot.writeStructBegin("seed_write_id_result") + if self.o1 is not None: + oprot.writeFieldBegin("o1", TType.STRUCT, 1) + self.o1.write(oprot) + oprot.writeFieldEnd() + oprot.writeFieldStop() + oprot.writeStructEnd() + + def validate(self): + return + + def __repr__(self): + L = ["{}={!r}".format(key, value) for key, value in self.__dict__.items()] + return "{}({})".format(self.__class__.__name__, ", ".join(L)) + + def __eq__(self, other): + return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ + + def __ne__(self, other): + return not (self == other) + + +all_structs.append(seed_write_id_result) +seed_write_id_result.thrift_spec = ( + None, # 0 + ( + 1, + TType.STRUCT, + "o1", + [MetaException, None], + None, + ), # 1 +) + + +class seed_txn_id_args: + """ + Attributes: + - rqst + + """ + + def __init__( + self, + rqst=None, + ): + self.rqst = rqst + + def read(self, iprot): + if ( + iprot._fast_decode is not None + and isinstance(iprot.trans, TTransport.CReadableTransport) + and self.thrift_spec is not None + ): + iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec]) + return + iprot.readStructBegin() + while True: + (fname, ftype, fid) = iprot.readFieldBegin() + if ftype == TType.STOP: + break + if fid == 1: + if ftype == TType.STRUCT: + self.rqst = SeedTxnIdRequest() + self.rqst.read(iprot) + else: + iprot.skip(ftype) + else: + iprot.skip(ftype) + iprot.readFieldEnd() + iprot.readStructEnd() + + def write(self, oprot): + if oprot._fast_encode is not None and self.thrift_spec is not None: + oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec])) + return + oprot.writeStructBegin("seed_txn_id_args") + if self.rqst is not None: + oprot.writeFieldBegin("rqst", TType.STRUCT, 1) + self.rqst.write(oprot) + oprot.writeFieldEnd() + oprot.writeFieldStop() + oprot.writeStructEnd() + + def validate(self): + return + + def __repr__(self): + L = ["{}={!r}".format(key, value) for key, value in self.__dict__.items()] + return "{}({})".format(self.__class__.__name__, ", ".join(L)) + + def __eq__(self, other): + return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ + + def __ne__(self, other): + return not (self == other) + + +all_structs.append(seed_txn_id_args) +seed_txn_id_args.thrift_spec = ( + None, # 0 + ( + 1, + TType.STRUCT, + "rqst", + [SeedTxnIdRequest, None], + None, + ), # 1 +) + + +class seed_txn_id_result: + """ + Attributes: + - o1 + + """ + + def __init__( + self, + o1=None, + ): + self.o1 = o1 + + def read(self, iprot): + if ( + iprot._fast_decode is not None + and isinstance(iprot.trans, TTransport.CReadableTransport) + and self.thrift_spec is not None + ): + iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec]) + return + iprot.readStructBegin() + while True: + (fname, ftype, fid) = iprot.readFieldBegin() + if ftype == TType.STOP: + break + if fid == 1: + if ftype == TType.STRUCT: + self.o1 = MetaException.read(iprot) + else: + iprot.skip(ftype) + else: + iprot.skip(ftype) + iprot.readFieldEnd() + iprot.readStructEnd() + + def write(self, oprot): + if oprot._fast_encode is not None and self.thrift_spec is not None: + oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec])) + return + oprot.writeStructBegin("seed_txn_id_result") + if self.o1 is not None: + oprot.writeFieldBegin("o1", TType.STRUCT, 1) + self.o1.write(oprot) + oprot.writeFieldEnd() + oprot.writeFieldStop() + oprot.writeStructEnd() + + def validate(self): + return + + def __repr__(self): + L = ["{}={!r}".format(key, value) for key, value in self.__dict__.items()] + return "{}({})".format(self.__class__.__name__, ", ".join(L)) + + def __eq__(self, other): + return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ + + def __ne__(self, other): + return not (self == other) + + +all_structs.append(seed_txn_id_result) +seed_txn_id_result.thrift_spec = ( + None, # 0 + ( + 1, + TType.STRUCT, + "o1", + [MetaException, None], + None, + ), # 1 +) + + +class lock_args: + """ + Attributes: + - rqst + + """ + + def __init__( + self, + rqst=None, + ): + self.rqst = rqst + + def read(self, iprot): + if ( + iprot._fast_decode is not None + and isinstance(iprot.trans, TTransport.CReadableTransport) + and self.thrift_spec is not None + ): + iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec]) + return + iprot.readStructBegin() + while True: + (fname, ftype, fid) = iprot.readFieldBegin() + if ftype == TType.STOP: + break + if fid == 1: + if ftype == TType.STRUCT: + self.rqst = LockRequest() + self.rqst.read(iprot) + else: + iprot.skip(ftype) + else: + iprot.skip(ftype) + iprot.readFieldEnd() + iprot.readStructEnd() + + def write(self, oprot): + if oprot._fast_encode is not None and self.thrift_spec is not None: + oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec])) + return + oprot.writeStructBegin("lock_args") + if self.rqst is not None: + oprot.writeFieldBegin("rqst", TType.STRUCT, 1) + self.rqst.write(oprot) + oprot.writeFieldEnd() + oprot.writeFieldStop() + oprot.writeStructEnd() + + def validate(self): + return + + def __repr__(self): + L = ["{}={!r}".format(key, value) for key, value in self.__dict__.items()] + return "{}({})".format(self.__class__.__name__, ", ".join(L)) + + def __eq__(self, other): + return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ + + def __ne__(self, other): + return not (self == other) + + +all_structs.append(lock_args) +lock_args.thrift_spec = ( + None, # 0 + ( + 1, + TType.STRUCT, + "rqst", + [LockRequest, None], + None, + ), # 1 +) + + +class lock_result: + """ + Attributes: + - success + - o1 + - o2 + + """ + + def __init__( + self, + success=None, + o1=None, + o2=None, + ): + self.success = success + self.o1 = o1 + self.o2 = o2 + + def read(self, iprot): + if ( + iprot._fast_decode is not None + and isinstance(iprot.trans, TTransport.CReadableTransport) + and self.thrift_spec is not None + ): + iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec]) + return + iprot.readStructBegin() + while True: + (fname, ftype, fid) = iprot.readFieldBegin() + if ftype == TType.STOP: + break + if fid == 0: + if ftype == TType.STRUCT: + self.success = LockResponse() + self.success.read(iprot) + else: + iprot.skip(ftype) + elif fid == 1: + if ftype == TType.STRUCT: + self.o1 = NoSuchTxnException.read(iprot) + else: + iprot.skip(ftype) + elif fid == 2: + if ftype == TType.STRUCT: + self.o2 = TxnAbortedException.read(iprot) + else: + iprot.skip(ftype) + else: + iprot.skip(ftype) + iprot.readFieldEnd() + iprot.readStructEnd() + + def write(self, oprot): + if oprot._fast_encode is not None and self.thrift_spec is not None: + oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec])) + return + oprot.writeStructBegin("lock_result") + if self.success is not None: + oprot.writeFieldBegin("success", TType.STRUCT, 0) + self.success.write(oprot) + oprot.writeFieldEnd() + if self.o1 is not None: + oprot.writeFieldBegin("o1", TType.STRUCT, 1) + self.o1.write(oprot) + oprot.writeFieldEnd() + if self.o2 is not None: + oprot.writeFieldBegin("o2", TType.STRUCT, 2) + self.o2.write(oprot) + oprot.writeFieldEnd() + oprot.writeFieldStop() + oprot.writeStructEnd() + + def validate(self): + return + + def __repr__(self): + L = ["{}={!r}".format(key, value) for key, value in self.__dict__.items()] + return "{}({})".format(self.__class__.__name__, ", ".join(L)) + + def __eq__(self, other): + return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ + + def __ne__(self, other): + return not (self == other) + + +all_structs.append(lock_result) +lock_result.thrift_spec = ( + ( + 0, + TType.STRUCT, + "success", + [LockResponse, None], + None, + ), # 0 + ( + 1, + TType.STRUCT, + "o1", + [NoSuchTxnException, None], + None, + ), # 1 + ( + 2, + TType.STRUCT, + "o2", + [TxnAbortedException, None], + None, + ), # 2 +) + + +class check_lock_args: + """ + Attributes: + - rqst + + """ + + def __init__( + self, + rqst=None, + ): + self.rqst = rqst + + def read(self, iprot): + if ( + iprot._fast_decode is not None + and isinstance(iprot.trans, TTransport.CReadableTransport) + and self.thrift_spec is not None + ): + iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec]) + return + iprot.readStructBegin() + while True: + (fname, ftype, fid) = iprot.readFieldBegin() + if ftype == TType.STOP: + break + if fid == 1: + if ftype == TType.STRUCT: + self.rqst = CheckLockRequest() + self.rqst.read(iprot) + else: + iprot.skip(ftype) + else: + iprot.skip(ftype) + iprot.readFieldEnd() + iprot.readStructEnd() + + def write(self, oprot): + if oprot._fast_encode is not None and self.thrift_spec is not None: + oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec])) + return + oprot.writeStructBegin("check_lock_args") + if self.rqst is not None: + oprot.writeFieldBegin("rqst", TType.STRUCT, 1) + self.rqst.write(oprot) + oprot.writeFieldEnd() + oprot.writeFieldStop() + oprot.writeStructEnd() + + def validate(self): + return + + def __repr__(self): + L = ["{}={!r}".format(key, value) for key, value in self.__dict__.items()] + return "{}({})".format(self.__class__.__name__, ", ".join(L)) + + def __eq__(self, other): + return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ + + def __ne__(self, other): + return not (self == other) + + +all_structs.append(check_lock_args) +check_lock_args.thrift_spec = ( + None, # 0 + ( + 1, + TType.STRUCT, + "rqst", + [CheckLockRequest, None], + None, + ), # 1 +) + + +class check_lock_result: + """ + Attributes: + - success + - o1 + - o2 + - o3 + + """ + + def __init__( + self, + success=None, + o1=None, + o2=None, + o3=None, + ): + self.success = success + self.o1 = o1 + self.o2 = o2 + self.o3 = o3 + + def read(self, iprot): + if ( + iprot._fast_decode is not None + and isinstance(iprot.trans, TTransport.CReadableTransport) + and self.thrift_spec is not None + ): + iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec]) + return + iprot.readStructBegin() + while True: + (fname, ftype, fid) = iprot.readFieldBegin() + if ftype == TType.STOP: + break + if fid == 0: + if ftype == TType.STRUCT: + self.success = LockResponse() + self.success.read(iprot) + else: + iprot.skip(ftype) + elif fid == 1: + if ftype == TType.STRUCT: + self.o1 = NoSuchTxnException.read(iprot) + else: + iprot.skip(ftype) + elif fid == 2: + if ftype == TType.STRUCT: + self.o2 = TxnAbortedException.read(iprot) + else: + iprot.skip(ftype) + elif fid == 3: + if ftype == TType.STRUCT: + self.o3 = NoSuchLockException.read(iprot) + else: + iprot.skip(ftype) + else: + iprot.skip(ftype) + iprot.readFieldEnd() + iprot.readStructEnd() + + def write(self, oprot): + if oprot._fast_encode is not None and self.thrift_spec is not None: + oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec])) + return + oprot.writeStructBegin("check_lock_result") + if self.success is not None: + oprot.writeFieldBegin("success", TType.STRUCT, 0) + self.success.write(oprot) + oprot.writeFieldEnd() + if self.o1 is not None: + oprot.writeFieldBegin("o1", TType.STRUCT, 1) + self.o1.write(oprot) + oprot.writeFieldEnd() + if self.o2 is not None: + oprot.writeFieldBegin("o2", TType.STRUCT, 2) + self.o2.write(oprot) + oprot.writeFieldEnd() + if self.o3 is not None: + oprot.writeFieldBegin("o3", TType.STRUCT, 3) + self.o3.write(oprot) + oprot.writeFieldEnd() + oprot.writeFieldStop() + oprot.writeStructEnd() + + def validate(self): + return + + def __repr__(self): + L = ["{}={!r}".format(key, value) for key, value in self.__dict__.items()] + return "{}({})".format(self.__class__.__name__, ", ".join(L)) + + def __eq__(self, other): + return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ + + def __ne__(self, other): + return not (self == other) + + +all_structs.append(check_lock_result) +check_lock_result.thrift_spec = ( + ( + 0, + TType.STRUCT, + "success", + [LockResponse, None], + None, + ), # 0 + ( + 1, + TType.STRUCT, + "o1", + [NoSuchTxnException, None], + None, + ), # 1 + ( + 2, + TType.STRUCT, + "o2", + [TxnAbortedException, None], + None, + ), # 2 + ( + 3, + TType.STRUCT, + "o3", + [NoSuchLockException, None], + None, + ), # 3 +) + + +class unlock_args: + """ + Attributes: + - rqst + + """ + + def __init__( + self, + rqst=None, + ): + self.rqst = rqst + + def read(self, iprot): + if ( + iprot._fast_decode is not None + and isinstance(iprot.trans, TTransport.CReadableTransport) + and self.thrift_spec is not None + ): + iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec]) + return + iprot.readStructBegin() + while True: + (fname, ftype, fid) = iprot.readFieldBegin() + if ftype == TType.STOP: + break + if fid == 1: + if ftype == TType.STRUCT: + self.rqst = UnlockRequest() + self.rqst.read(iprot) + else: + iprot.skip(ftype) + else: + iprot.skip(ftype) + iprot.readFieldEnd() + iprot.readStructEnd() + + def write(self, oprot): + if oprot._fast_encode is not None and self.thrift_spec is not None: + oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec])) + return + oprot.writeStructBegin("unlock_args") + if self.rqst is not None: + oprot.writeFieldBegin("rqst", TType.STRUCT, 1) + self.rqst.write(oprot) + oprot.writeFieldEnd() + oprot.writeFieldStop() + oprot.writeStructEnd() + + def validate(self): + return + + def __repr__(self): + L = ["{}={!r}".format(key, value) for key, value in self.__dict__.items()] + return "{}({})".format(self.__class__.__name__, ", ".join(L)) + + def __eq__(self, other): + return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ + + def __ne__(self, other): + return not (self == other) + + +all_structs.append(unlock_args) +unlock_args.thrift_spec = ( + None, # 0 + ( + 1, + TType.STRUCT, + "rqst", + [UnlockRequest, None], + None, + ), # 1 +) + + +class unlock_result: + """ + Attributes: + - o1 + - o2 + + """ + + def __init__( + self, + o1=None, + o2=None, + ): + self.o1 = o1 + self.o2 = o2 + + def read(self, iprot): + if ( + iprot._fast_decode is not None + and isinstance(iprot.trans, TTransport.CReadableTransport) + and self.thrift_spec is not None + ): + iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec]) + return + iprot.readStructBegin() + while True: + (fname, ftype, fid) = iprot.readFieldBegin() + if ftype == TType.STOP: + break + if fid == 1: + if ftype == TType.STRUCT: + self.o1 = NoSuchLockException.read(iprot) + else: + iprot.skip(ftype) + elif fid == 2: + if ftype == TType.STRUCT: + self.o2 = TxnOpenException.read(iprot) + else: + iprot.skip(ftype) + else: + iprot.skip(ftype) + iprot.readFieldEnd() + iprot.readStructEnd() + + def write(self, oprot): + if oprot._fast_encode is not None and self.thrift_spec is not None: + oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec])) + return + oprot.writeStructBegin("unlock_result") + if self.o1 is not None: + oprot.writeFieldBegin("o1", TType.STRUCT, 1) + self.o1.write(oprot) + oprot.writeFieldEnd() + if self.o2 is not None: + oprot.writeFieldBegin("o2", TType.STRUCT, 2) + self.o2.write(oprot) + oprot.writeFieldEnd() + oprot.writeFieldStop() + oprot.writeStructEnd() + + def validate(self): + return + + def __repr__(self): + L = ["{}={!r}".format(key, value) for key, value in self.__dict__.items()] + return "{}({})".format(self.__class__.__name__, ", ".join(L)) + + def __eq__(self, other): + return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ + + def __ne__(self, other): + return not (self == other) + + +all_structs.append(unlock_result) +unlock_result.thrift_spec = ( + None, # 0 + ( + 1, + TType.STRUCT, + "o1", + [NoSuchLockException, None], + None, + ), # 1 + ( + 2, + TType.STRUCT, + "o2", + [TxnOpenException, None], + None, + ), # 2 +) + + +class show_locks_args: + """ + Attributes: + - rqst + + """ + + def __init__( + self, + rqst=None, + ): + self.rqst = rqst + + def read(self, iprot): + if ( + iprot._fast_decode is not None + and isinstance(iprot.trans, TTransport.CReadableTransport) + and self.thrift_spec is not None + ): + iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec]) + return + iprot.readStructBegin() + while True: + (fname, ftype, fid) = iprot.readFieldBegin() + if ftype == TType.STOP: + break + if fid == 1: + if ftype == TType.STRUCT: + self.rqst = ShowLocksRequest() + self.rqst.read(iprot) + else: + iprot.skip(ftype) + else: + iprot.skip(ftype) + iprot.readFieldEnd() + iprot.readStructEnd() + + def write(self, oprot): + if oprot._fast_encode is not None and self.thrift_spec is not None: + oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec])) + return + oprot.writeStructBegin("show_locks_args") + if self.rqst is not None: + oprot.writeFieldBegin("rqst", TType.STRUCT, 1) + self.rqst.write(oprot) + oprot.writeFieldEnd() + oprot.writeFieldStop() + oprot.writeStructEnd() + + def validate(self): + return + + def __repr__(self): + L = ["{}={!r}".format(key, value) for key, value in self.__dict__.items()] + return "{}({})".format(self.__class__.__name__, ", ".join(L)) + + def __eq__(self, other): + return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ + + def __ne__(self, other): + return not (self == other) + + +all_structs.append(show_locks_args) +show_locks_args.thrift_spec = ( + None, # 0 + ( + 1, + TType.STRUCT, + "rqst", + [ShowLocksRequest, None], + None, + ), # 1 +) + + +class show_locks_result: + """ + Attributes: + - success + + """ + + def __init__( + self, + success=None, + ): + self.success = success + + def read(self, iprot): + if ( + iprot._fast_decode is not None + and isinstance(iprot.trans, TTransport.CReadableTransport) + and self.thrift_spec is not None + ): + iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec]) + return + iprot.readStructBegin() + while True: + (fname, ftype, fid) = iprot.readFieldBegin() + if ftype == TType.STOP: + break + if fid == 0: + if ftype == TType.STRUCT: + self.success = ShowLocksResponse() + self.success.read(iprot) + else: + iprot.skip(ftype) + else: + iprot.skip(ftype) + iprot.readFieldEnd() + iprot.readStructEnd() + + def write(self, oprot): + if oprot._fast_encode is not None and self.thrift_spec is not None: + oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec])) + return + oprot.writeStructBegin("show_locks_result") + if self.success is not None: + oprot.writeFieldBegin("success", TType.STRUCT, 0) + self.success.write(oprot) + oprot.writeFieldEnd() + oprot.writeFieldStop() + oprot.writeStructEnd() + + def validate(self): + return + + def __repr__(self): + L = ["{}={!r}".format(key, value) for key, value in self.__dict__.items()] + return "{}({})".format(self.__class__.__name__, ", ".join(L)) + + def __eq__(self, other): + return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ + + def __ne__(self, other): + return not (self == other) + + +all_structs.append(show_locks_result) +show_locks_result.thrift_spec = ( + ( + 0, + TType.STRUCT, + "success", + [ShowLocksResponse, None], + None, + ), # 0 +) + + +class heartbeat_args: + """ + Attributes: + - ids + + """ + + def __init__( + self, + ids=None, + ): + self.ids = ids + + def read(self, iprot): + if ( + iprot._fast_decode is not None + and isinstance(iprot.trans, TTransport.CReadableTransport) + and self.thrift_spec is not None + ): + iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec]) + return + iprot.readStructBegin() + while True: + (fname, ftype, fid) = iprot.readFieldBegin() + if ftype == TType.STOP: + break + if fid == 1: + if ftype == TType.STRUCT: + self.ids = HeartbeatRequest() + self.ids.read(iprot) + else: + iprot.skip(ftype) + else: + iprot.skip(ftype) + iprot.readFieldEnd() + iprot.readStructEnd() + + def write(self, oprot): + if oprot._fast_encode is not None and self.thrift_spec is not None: + oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec])) + return + oprot.writeStructBegin("heartbeat_args") + if self.ids is not None: + oprot.writeFieldBegin("ids", TType.STRUCT, 1) + self.ids.write(oprot) + oprot.writeFieldEnd() + oprot.writeFieldStop() + oprot.writeStructEnd() + + def validate(self): + return + + def __repr__(self): + L = ["{}={!r}".format(key, value) for key, value in self.__dict__.items()] + return "{}({})".format(self.__class__.__name__, ", ".join(L)) + + def __eq__(self, other): + return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ + + def __ne__(self, other): + return not (self == other) + + +all_structs.append(heartbeat_args) +heartbeat_args.thrift_spec = ( + None, # 0 + ( + 1, + TType.STRUCT, + "ids", + [HeartbeatRequest, None], + None, + ), # 1 +) + + +class heartbeat_result: + """ + Attributes: + - o1 + - o2 + - o3 + + """ + + def __init__( + self, + o1=None, + o2=None, + o3=None, + ): + self.o1 = o1 + self.o2 = o2 + self.o3 = o3 + + def read(self, iprot): + if ( + iprot._fast_decode is not None + and isinstance(iprot.trans, TTransport.CReadableTransport) + and self.thrift_spec is not None + ): + iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec]) + return + iprot.readStructBegin() + while True: + (fname, ftype, fid) = iprot.readFieldBegin() + if ftype == TType.STOP: + break + if fid == 1: + if ftype == TType.STRUCT: + self.o1 = NoSuchLockException.read(iprot) + else: + iprot.skip(ftype) + elif fid == 2: + if ftype == TType.STRUCT: + self.o2 = NoSuchTxnException.read(iprot) + else: + iprot.skip(ftype) + elif fid == 3: + if ftype == TType.STRUCT: + self.o3 = TxnAbortedException.read(iprot) + else: + iprot.skip(ftype) + else: + iprot.skip(ftype) + iprot.readFieldEnd() + iprot.readStructEnd() + + def write(self, oprot): + if oprot._fast_encode is not None and self.thrift_spec is not None: + oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec])) + return + oprot.writeStructBegin("heartbeat_result") + if self.o1 is not None: + oprot.writeFieldBegin("o1", TType.STRUCT, 1) + self.o1.write(oprot) + oprot.writeFieldEnd() + if self.o2 is not None: + oprot.writeFieldBegin("o2", TType.STRUCT, 2) + self.o2.write(oprot) + oprot.writeFieldEnd() + if self.o3 is not None: + oprot.writeFieldBegin("o3", TType.STRUCT, 3) + self.o3.write(oprot) + oprot.writeFieldEnd() + oprot.writeFieldStop() + oprot.writeStructEnd() + + def validate(self): + return + + def __repr__(self): + L = ["{}={!r}".format(key, value) for key, value in self.__dict__.items()] + return "{}({})".format(self.__class__.__name__, ", ".join(L)) + + def __eq__(self, other): + return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ + + def __ne__(self, other): + return not (self == other) + + +all_structs.append(heartbeat_result) +heartbeat_result.thrift_spec = ( + None, # 0 + ( + 1, + TType.STRUCT, + "o1", + [NoSuchLockException, None], + None, + ), # 1 + ( + 2, + TType.STRUCT, + "o2", + [NoSuchTxnException, None], + None, + ), # 2 + ( + 3, + TType.STRUCT, + "o3", + [TxnAbortedException, None], + None, + ), # 3 +) + + +class heartbeat_txn_range_args: + """ + Attributes: + - txns + + """ + + def __init__( + self, + txns=None, + ): + self.txns = txns + + def read(self, iprot): + if ( + iprot._fast_decode is not None + and isinstance(iprot.trans, TTransport.CReadableTransport) + and self.thrift_spec is not None + ): + iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec]) + return + iprot.readStructBegin() + while True: + (fname, ftype, fid) = iprot.readFieldBegin() + if ftype == TType.STOP: + break + if fid == 1: + if ftype == TType.STRUCT: + self.txns = HeartbeatTxnRangeRequest() + self.txns.read(iprot) + else: + iprot.skip(ftype) + else: + iprot.skip(ftype) + iprot.readFieldEnd() + iprot.readStructEnd() + + def write(self, oprot): + if oprot._fast_encode is not None and self.thrift_spec is not None: + oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec])) + return + oprot.writeStructBegin("heartbeat_txn_range_args") + if self.txns is not None: + oprot.writeFieldBegin("txns", TType.STRUCT, 1) + self.txns.write(oprot) + oprot.writeFieldEnd() + oprot.writeFieldStop() + oprot.writeStructEnd() + + def validate(self): + return + + def __repr__(self): + L = ["{}={!r}".format(key, value) for key, value in self.__dict__.items()] + return "{}({})".format(self.__class__.__name__, ", ".join(L)) + + def __eq__(self, other): + return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ + + def __ne__(self, other): + return not (self == other) + + +all_structs.append(heartbeat_txn_range_args) +heartbeat_txn_range_args.thrift_spec = ( + None, # 0 + ( + 1, + TType.STRUCT, + "txns", + [HeartbeatTxnRangeRequest, None], + None, + ), # 1 +) + + +class heartbeat_txn_range_result: + """ + Attributes: + - success + + """ + + def __init__( + self, + success=None, + ): + self.success = success + + def read(self, iprot): + if ( + iprot._fast_decode is not None + and isinstance(iprot.trans, TTransport.CReadableTransport) + and self.thrift_spec is not None + ): + iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec]) + return + iprot.readStructBegin() + while True: + (fname, ftype, fid) = iprot.readFieldBegin() + if ftype == TType.STOP: + break + if fid == 0: + if ftype == TType.STRUCT: + self.success = HeartbeatTxnRangeResponse() + self.success.read(iprot) + else: + iprot.skip(ftype) + else: + iprot.skip(ftype) + iprot.readFieldEnd() + iprot.readStructEnd() + + def write(self, oprot): + if oprot._fast_encode is not None and self.thrift_spec is not None: + oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec])) + return + oprot.writeStructBegin("heartbeat_txn_range_result") + if self.success is not None: + oprot.writeFieldBegin("success", TType.STRUCT, 0) + self.success.write(oprot) + oprot.writeFieldEnd() + oprot.writeFieldStop() + oprot.writeStructEnd() + + def validate(self): + return + + def __repr__(self): + L = ["{}={!r}".format(key, value) for key, value in self.__dict__.items()] + return "{}({})".format(self.__class__.__name__, ", ".join(L)) + + def __eq__(self, other): + return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ + + def __ne__(self, other): + return not (self == other) + + +all_structs.append(heartbeat_txn_range_result) +heartbeat_txn_range_result.thrift_spec = ( + ( + 0, + TType.STRUCT, + "success", + [HeartbeatTxnRangeResponse, None], + None, + ), # 0 +) + + +class compact_args: + """ + Attributes: + - rqst + + """ + + def __init__( + self, + rqst=None, + ): + self.rqst = rqst + + def read(self, iprot): + if ( + iprot._fast_decode is not None + and isinstance(iprot.trans, TTransport.CReadableTransport) + and self.thrift_spec is not None + ): + iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec]) + return + iprot.readStructBegin() + while True: + (fname, ftype, fid) = iprot.readFieldBegin() + if ftype == TType.STOP: + break + if fid == 1: + if ftype == TType.STRUCT: + self.rqst = CompactionRequest() + self.rqst.read(iprot) + else: + iprot.skip(ftype) + else: + iprot.skip(ftype) + iprot.readFieldEnd() + iprot.readStructEnd() + + def write(self, oprot): + if oprot._fast_encode is not None and self.thrift_spec is not None: + oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec])) + return + oprot.writeStructBegin("compact_args") + if self.rqst is not None: + oprot.writeFieldBegin("rqst", TType.STRUCT, 1) + self.rqst.write(oprot) + oprot.writeFieldEnd() + oprot.writeFieldStop() + oprot.writeStructEnd() + + def validate(self): + return + + def __repr__(self): + L = ["{}={!r}".format(key, value) for key, value in self.__dict__.items()] + return "{}({})".format(self.__class__.__name__, ", ".join(L)) + + def __eq__(self, other): + return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ + + def __ne__(self, other): + return not (self == other) + + +all_structs.append(compact_args) +compact_args.thrift_spec = ( + None, # 0 + ( + 1, + TType.STRUCT, + "rqst", + [CompactionRequest, None], + None, + ), # 1 +) + + +class compact_result: + def read(self, iprot): + if ( + iprot._fast_decode is not None + and isinstance(iprot.trans, TTransport.CReadableTransport) + and self.thrift_spec is not None + ): + iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec]) + return + iprot.readStructBegin() + while True: + (fname, ftype, fid) = iprot.readFieldBegin() + if ftype == TType.STOP: + break + else: + iprot.skip(ftype) + iprot.readFieldEnd() + iprot.readStructEnd() + + def write(self, oprot): + if oprot._fast_encode is not None and self.thrift_spec is not None: + oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec])) + return + oprot.writeStructBegin("compact_result") + oprot.writeFieldStop() + oprot.writeStructEnd() + + def validate(self): + return + + def __repr__(self): + L = ["{}={!r}".format(key, value) for key, value in self.__dict__.items()] + return "{}({})".format(self.__class__.__name__, ", ".join(L)) + + def __eq__(self, other): + return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ + + def __ne__(self, other): + return not (self == other) + + +all_structs.append(compact_result) +compact_result.thrift_spec = () + + +class compact2_args: + """ + Attributes: + - rqst + + """ + + def __init__( + self, + rqst=None, + ): + self.rqst = rqst + + def read(self, iprot): + if ( + iprot._fast_decode is not None + and isinstance(iprot.trans, TTransport.CReadableTransport) + and self.thrift_spec is not None + ): + iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec]) + return + iprot.readStructBegin() + while True: + (fname, ftype, fid) = iprot.readFieldBegin() + if ftype == TType.STOP: + break + if fid == 1: + if ftype == TType.STRUCT: + self.rqst = CompactionRequest() + self.rqst.read(iprot) + else: + iprot.skip(ftype) + else: + iprot.skip(ftype) + iprot.readFieldEnd() + iprot.readStructEnd() + + def write(self, oprot): + if oprot._fast_encode is not None and self.thrift_spec is not None: + oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec])) + return + oprot.writeStructBegin("compact2_args") + if self.rqst is not None: + oprot.writeFieldBegin("rqst", TType.STRUCT, 1) + self.rqst.write(oprot) + oprot.writeFieldEnd() + oprot.writeFieldStop() + oprot.writeStructEnd() + + def validate(self): + return + + def __repr__(self): + L = ["{}={!r}".format(key, value) for key, value in self.__dict__.items()] + return "{}({})".format(self.__class__.__name__, ", ".join(L)) + + def __eq__(self, other): + return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ + + def __ne__(self, other): + return not (self == other) + + +all_structs.append(compact2_args) +compact2_args.thrift_spec = ( + None, # 0 + ( + 1, + TType.STRUCT, + "rqst", + [CompactionRequest, None], + None, + ), # 1 +) + + +class compact2_result: + """ + Attributes: + - success + + """ + + def __init__( + self, + success=None, + ): + self.success = success + + def read(self, iprot): + if ( + iprot._fast_decode is not None + and isinstance(iprot.trans, TTransport.CReadableTransport) + and self.thrift_spec is not None + ): + iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec]) + return + iprot.readStructBegin() + while True: + (fname, ftype, fid) = iprot.readFieldBegin() + if ftype == TType.STOP: + break + if fid == 0: + if ftype == TType.STRUCT: + self.success = CompactionResponse() + self.success.read(iprot) + else: + iprot.skip(ftype) + else: + iprot.skip(ftype) + iprot.readFieldEnd() + iprot.readStructEnd() + + def write(self, oprot): + if oprot._fast_encode is not None and self.thrift_spec is not None: + oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec])) + return + oprot.writeStructBegin("compact2_result") + if self.success is not None: + oprot.writeFieldBegin("success", TType.STRUCT, 0) + self.success.write(oprot) + oprot.writeFieldEnd() + oprot.writeFieldStop() + oprot.writeStructEnd() + + def validate(self): + return + + def __repr__(self): + L = ["{}={!r}".format(key, value) for key, value in self.__dict__.items()] + return "{}({})".format(self.__class__.__name__, ", ".join(L)) + + def __eq__(self, other): + return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ + + def __ne__(self, other): + return not (self == other) + + +all_structs.append(compact2_result) +compact2_result.thrift_spec = ( + ( + 0, + TType.STRUCT, + "success", + [CompactionResponse, None], + None, + ), # 0 +) + + +class show_compact_args: + """ + Attributes: + - rqst + + """ + + def __init__( + self, + rqst=None, + ): + self.rqst = rqst + + def read(self, iprot): + if ( + iprot._fast_decode is not None + and isinstance(iprot.trans, TTransport.CReadableTransport) + and self.thrift_spec is not None + ): + iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec]) + return + iprot.readStructBegin() + while True: + (fname, ftype, fid) = iprot.readFieldBegin() + if ftype == TType.STOP: + break + if fid == 1: + if ftype == TType.STRUCT: + self.rqst = ShowCompactRequest() + self.rqst.read(iprot) + else: + iprot.skip(ftype) + else: + iprot.skip(ftype) + iprot.readFieldEnd() + iprot.readStructEnd() + + def write(self, oprot): + if oprot._fast_encode is not None and self.thrift_spec is not None: + oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec])) + return + oprot.writeStructBegin("show_compact_args") + if self.rqst is not None: + oprot.writeFieldBegin("rqst", TType.STRUCT, 1) + self.rqst.write(oprot) + oprot.writeFieldEnd() + oprot.writeFieldStop() + oprot.writeStructEnd() + + def validate(self): + return + + def __repr__(self): + L = ["{}={!r}".format(key, value) for key, value in self.__dict__.items()] + return "{}({})".format(self.__class__.__name__, ", ".join(L)) + + def __eq__(self, other): + return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ + + def __ne__(self, other): + return not (self == other) + + +all_structs.append(show_compact_args) +show_compact_args.thrift_spec = ( + None, # 0 + ( + 1, + TType.STRUCT, + "rqst", + [ShowCompactRequest, None], + None, + ), # 1 +) + + +class show_compact_result: + """ + Attributes: + - success + + """ + + def __init__( + self, + success=None, + ): + self.success = success + + def read(self, iprot): + if ( + iprot._fast_decode is not None + and isinstance(iprot.trans, TTransport.CReadableTransport) + and self.thrift_spec is not None + ): + iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec]) + return + iprot.readStructBegin() + while True: + (fname, ftype, fid) = iprot.readFieldBegin() + if ftype == TType.STOP: + break + if fid == 0: + if ftype == TType.STRUCT: + self.success = ShowCompactResponse() + self.success.read(iprot) + else: + iprot.skip(ftype) + else: + iprot.skip(ftype) + iprot.readFieldEnd() + iprot.readStructEnd() + + def write(self, oprot): + if oprot._fast_encode is not None and self.thrift_spec is not None: + oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec])) + return + oprot.writeStructBegin("show_compact_result") + if self.success is not None: + oprot.writeFieldBegin("success", TType.STRUCT, 0) + self.success.write(oprot) + oprot.writeFieldEnd() + oprot.writeFieldStop() + oprot.writeStructEnd() + + def validate(self): + return + + def __repr__(self): + L = ["{}={!r}".format(key, value) for key, value in self.__dict__.items()] + return "{}({})".format(self.__class__.__name__, ", ".join(L)) + + def __eq__(self, other): + return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ + + def __ne__(self, other): + return not (self == other) + + +all_structs.append(show_compact_result) +show_compact_result.thrift_spec = ( + ( + 0, + TType.STRUCT, + "success", + [ShowCompactResponse, None], + None, + ), # 0 +) + + +class add_dynamic_partitions_args: + """ + Attributes: + - rqst + + """ + + def __init__( + self, + rqst=None, + ): + self.rqst = rqst + + def read(self, iprot): + if ( + iprot._fast_decode is not None + and isinstance(iprot.trans, TTransport.CReadableTransport) + and self.thrift_spec is not None + ): + iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec]) + return + iprot.readStructBegin() + while True: + (fname, ftype, fid) = iprot.readFieldBegin() + if ftype == TType.STOP: + break + if fid == 1: + if ftype == TType.STRUCT: + self.rqst = AddDynamicPartitions() + self.rqst.read(iprot) + else: + iprot.skip(ftype) + else: + iprot.skip(ftype) + iprot.readFieldEnd() + iprot.readStructEnd() + + def write(self, oprot): + if oprot._fast_encode is not None and self.thrift_spec is not None: + oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec])) + return + oprot.writeStructBegin("add_dynamic_partitions_args") + if self.rqst is not None: + oprot.writeFieldBegin("rqst", TType.STRUCT, 1) + self.rqst.write(oprot) + oprot.writeFieldEnd() + oprot.writeFieldStop() + oprot.writeStructEnd() + + def validate(self): + return + + def __repr__(self): + L = ["{}={!r}".format(key, value) for key, value in self.__dict__.items()] + return "{}({})".format(self.__class__.__name__, ", ".join(L)) + + def __eq__(self, other): + return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ + + def __ne__(self, other): + return not (self == other) + + +all_structs.append(add_dynamic_partitions_args) +add_dynamic_partitions_args.thrift_spec = ( + None, # 0 + ( + 1, + TType.STRUCT, + "rqst", + [AddDynamicPartitions, None], + None, + ), # 1 +) + + +class add_dynamic_partitions_result: + """ + Attributes: + - o1 + - o2 + + """ + + def __init__( + self, + o1=None, + o2=None, + ): + self.o1 = o1 + self.o2 = o2 + + def read(self, iprot): + if ( + iprot._fast_decode is not None + and isinstance(iprot.trans, TTransport.CReadableTransport) + and self.thrift_spec is not None + ): + iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec]) + return + iprot.readStructBegin() + while True: + (fname, ftype, fid) = iprot.readFieldBegin() + if ftype == TType.STOP: + break + if fid == 1: + if ftype == TType.STRUCT: + self.o1 = NoSuchTxnException.read(iprot) + else: + iprot.skip(ftype) + elif fid == 2: + if ftype == TType.STRUCT: + self.o2 = TxnAbortedException.read(iprot) + else: + iprot.skip(ftype) + else: + iprot.skip(ftype) + iprot.readFieldEnd() + iprot.readStructEnd() + + def write(self, oprot): + if oprot._fast_encode is not None and self.thrift_spec is not None: + oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec])) + return + oprot.writeStructBegin("add_dynamic_partitions_result") + if self.o1 is not None: + oprot.writeFieldBegin("o1", TType.STRUCT, 1) + self.o1.write(oprot) + oprot.writeFieldEnd() + if self.o2 is not None: + oprot.writeFieldBegin("o2", TType.STRUCT, 2) + self.o2.write(oprot) + oprot.writeFieldEnd() + oprot.writeFieldStop() + oprot.writeStructEnd() + + def validate(self): + return + + def __repr__(self): + L = ["{}={!r}".format(key, value) for key, value in self.__dict__.items()] + return "{}({})".format(self.__class__.__name__, ", ".join(L)) + + def __eq__(self, other): + return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ + + def __ne__(self, other): + return not (self == other) + + +all_structs.append(add_dynamic_partitions_result) +add_dynamic_partitions_result.thrift_spec = ( + None, # 0 + ( + 1, + TType.STRUCT, + "o1", + [NoSuchTxnException, None], + None, + ), # 1 + ( + 2, + TType.STRUCT, + "o2", + [TxnAbortedException, None], + None, + ), # 2 +) + + +class find_next_compact_args: + """ + Attributes: + - workerId + + """ + + def __init__( + self, + workerId=None, + ): + self.workerId = workerId + + def read(self, iprot): + if ( + iprot._fast_decode is not None + and isinstance(iprot.trans, TTransport.CReadableTransport) + and self.thrift_spec is not None + ): + iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec]) + return + iprot.readStructBegin() + while True: + (fname, ftype, fid) = iprot.readFieldBegin() + if ftype == TType.STOP: + break + if fid == 1: + if ftype == TType.STRING: + self.workerId = ( + iprot.readString().decode("utf-8", errors="replace") if sys.version_info[0] == 2 else iprot.readString() + ) + else: + iprot.skip(ftype) + else: + iprot.skip(ftype) + iprot.readFieldEnd() + iprot.readStructEnd() + + def write(self, oprot): + if oprot._fast_encode is not None and self.thrift_spec is not None: + oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec])) + return + oprot.writeStructBegin("find_next_compact_args") + if self.workerId is not None: + oprot.writeFieldBegin("workerId", TType.STRING, 1) + oprot.writeString(self.workerId.encode("utf-8") if sys.version_info[0] == 2 else self.workerId) + oprot.writeFieldEnd() + oprot.writeFieldStop() + oprot.writeStructEnd() + + def validate(self): + return + + def __repr__(self): + L = ["{}={!r}".format(key, value) for key, value in self.__dict__.items()] + return "{}({})".format(self.__class__.__name__, ", ".join(L)) + + def __eq__(self, other): + return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ + + def __ne__(self, other): + return not (self == other) + + +all_structs.append(find_next_compact_args) +find_next_compact_args.thrift_spec = ( + None, # 0 + ( + 1, + TType.STRING, + "workerId", + "UTF8", + None, + ), # 1 +) + + +class find_next_compact_result: + """ + Attributes: + - success + - o1 + + """ + + def __init__( + self, + success=None, + o1=None, + ): + self.success = success + self.o1 = o1 + + def read(self, iprot): + if ( + iprot._fast_decode is not None + and isinstance(iprot.trans, TTransport.CReadableTransport) + and self.thrift_spec is not None + ): + iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec]) + return + iprot.readStructBegin() + while True: + (fname, ftype, fid) = iprot.readFieldBegin() + if ftype == TType.STOP: + break + if fid == 0: + if ftype == TType.STRUCT: + self.success = OptionalCompactionInfoStruct() + self.success.read(iprot) + else: + iprot.skip(ftype) + elif fid == 1: + if ftype == TType.STRUCT: + self.o1 = MetaException.read(iprot) + else: + iprot.skip(ftype) + else: + iprot.skip(ftype) + iprot.readFieldEnd() + iprot.readStructEnd() + + def write(self, oprot): + if oprot._fast_encode is not None and self.thrift_spec is not None: + oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec])) + return + oprot.writeStructBegin("find_next_compact_result") + if self.success is not None: + oprot.writeFieldBegin("success", TType.STRUCT, 0) + self.success.write(oprot) + oprot.writeFieldEnd() + if self.o1 is not None: + oprot.writeFieldBegin("o1", TType.STRUCT, 1) + self.o1.write(oprot) + oprot.writeFieldEnd() + oprot.writeFieldStop() + oprot.writeStructEnd() + + def validate(self): + return + + def __repr__(self): + L = ["{}={!r}".format(key, value) for key, value in self.__dict__.items()] + return "{}({})".format(self.__class__.__name__, ", ".join(L)) + + def __eq__(self, other): + return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ + + def __ne__(self, other): + return not (self == other) + + +all_structs.append(find_next_compact_result) +find_next_compact_result.thrift_spec = ( + ( + 0, + TType.STRUCT, + "success", + [OptionalCompactionInfoStruct, None], + None, + ), # 0 + ( + 1, + TType.STRUCT, + "o1", + [MetaException, None], + None, + ), # 1 +) + + +class find_next_compact2_args: + """ + Attributes: + - rqst + + """ + + def __init__( + self, + rqst=None, + ): + self.rqst = rqst + + def read(self, iprot): + if ( + iprot._fast_decode is not None + and isinstance(iprot.trans, TTransport.CReadableTransport) + and self.thrift_spec is not None + ): + iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec]) + return + iprot.readStructBegin() + while True: + (fname, ftype, fid) = iprot.readFieldBegin() + if ftype == TType.STOP: + break + if fid == 1: + if ftype == TType.STRUCT: + self.rqst = FindNextCompactRequest() + self.rqst.read(iprot) + else: + iprot.skip(ftype) + else: + iprot.skip(ftype) + iprot.readFieldEnd() + iprot.readStructEnd() + + def write(self, oprot): + if oprot._fast_encode is not None and self.thrift_spec is not None: + oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec])) + return + oprot.writeStructBegin("find_next_compact2_args") + if self.rqst is not None: + oprot.writeFieldBegin("rqst", TType.STRUCT, 1) + self.rqst.write(oprot) + oprot.writeFieldEnd() + oprot.writeFieldStop() + oprot.writeStructEnd() + + def validate(self): + return + + def __repr__(self): + L = ["{}={!r}".format(key, value) for key, value in self.__dict__.items()] + return "{}({})".format(self.__class__.__name__, ", ".join(L)) + + def __eq__(self, other): + return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ + + def __ne__(self, other): + return not (self == other) + + +all_structs.append(find_next_compact2_args) +find_next_compact2_args.thrift_spec = ( + None, # 0 + ( + 1, + TType.STRUCT, + "rqst", + [FindNextCompactRequest, None], + None, + ), # 1 +) + + +class find_next_compact2_result: + """ + Attributes: + - success + - o1 + + """ + + def __init__( + self, + success=None, + o1=None, + ): + self.success = success + self.o1 = o1 + + def read(self, iprot): + if ( + iprot._fast_decode is not None + and isinstance(iprot.trans, TTransport.CReadableTransport) + and self.thrift_spec is not None + ): + iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec]) + return + iprot.readStructBegin() + while True: + (fname, ftype, fid) = iprot.readFieldBegin() + if ftype == TType.STOP: + break + if fid == 0: + if ftype == TType.STRUCT: + self.success = OptionalCompactionInfoStruct() + self.success.read(iprot) + else: + iprot.skip(ftype) + elif fid == 1: + if ftype == TType.STRUCT: + self.o1 = MetaException.read(iprot) + else: + iprot.skip(ftype) + else: + iprot.skip(ftype) + iprot.readFieldEnd() + iprot.readStructEnd() + + def write(self, oprot): + if oprot._fast_encode is not None and self.thrift_spec is not None: + oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec])) + return + oprot.writeStructBegin("find_next_compact2_result") + if self.success is not None: + oprot.writeFieldBegin("success", TType.STRUCT, 0) + self.success.write(oprot) + oprot.writeFieldEnd() + if self.o1 is not None: + oprot.writeFieldBegin("o1", TType.STRUCT, 1) + self.o1.write(oprot) + oprot.writeFieldEnd() + oprot.writeFieldStop() + oprot.writeStructEnd() + + def validate(self): + return + + def __repr__(self): + L = ["{}={!r}".format(key, value) for key, value in self.__dict__.items()] + return "{}({})".format(self.__class__.__name__, ", ".join(L)) + + def __eq__(self, other): + return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ + + def __ne__(self, other): + return not (self == other) + + +all_structs.append(find_next_compact2_result) +find_next_compact2_result.thrift_spec = ( + ( + 0, + TType.STRUCT, + "success", + [OptionalCompactionInfoStruct, None], + None, + ), # 0 + ( + 1, + TType.STRUCT, + "o1", + [MetaException, None], + None, + ), # 1 +) + + +class update_compactor_state_args: + """ + Attributes: + - cr + - txn_id + + """ + + def __init__( + self, + cr=None, + txn_id=None, + ): + self.cr = cr + self.txn_id = txn_id + + def read(self, iprot): + if ( + iprot._fast_decode is not None + and isinstance(iprot.trans, TTransport.CReadableTransport) + and self.thrift_spec is not None + ): + iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec]) + return + iprot.readStructBegin() + while True: + (fname, ftype, fid) = iprot.readFieldBegin() + if ftype == TType.STOP: + break + if fid == 1: + if ftype == TType.STRUCT: + self.cr = CompactionInfoStruct() + self.cr.read(iprot) + else: + iprot.skip(ftype) + elif fid == 2: + if ftype == TType.I64: + self.txn_id = iprot.readI64() + else: + iprot.skip(ftype) + else: + iprot.skip(ftype) + iprot.readFieldEnd() + iprot.readStructEnd() + + def write(self, oprot): + if oprot._fast_encode is not None and self.thrift_spec is not None: + oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec])) + return + oprot.writeStructBegin("update_compactor_state_args") + if self.cr is not None: + oprot.writeFieldBegin("cr", TType.STRUCT, 1) + self.cr.write(oprot) + oprot.writeFieldEnd() + if self.txn_id is not None: + oprot.writeFieldBegin("txn_id", TType.I64, 2) + oprot.writeI64(self.txn_id) + oprot.writeFieldEnd() + oprot.writeFieldStop() + oprot.writeStructEnd() + + def validate(self): + return + + def __repr__(self): + L = ["{}={!r}".format(key, value) for key, value in self.__dict__.items()] + return "{}({})".format(self.__class__.__name__, ", ".join(L)) + + def __eq__(self, other): + return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ + + def __ne__(self, other): + return not (self == other) + + +all_structs.append(update_compactor_state_args) +update_compactor_state_args.thrift_spec = ( + None, # 0 + ( + 1, + TType.STRUCT, + "cr", + [CompactionInfoStruct, None], + None, + ), # 1 + ( + 2, + TType.I64, + "txn_id", + None, + None, + ), # 2 +) + + +class update_compactor_state_result: + def read(self, iprot): + if ( + iprot._fast_decode is not None + and isinstance(iprot.trans, TTransport.CReadableTransport) + and self.thrift_spec is not None + ): + iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec]) + return + iprot.readStructBegin() + while True: + (fname, ftype, fid) = iprot.readFieldBegin() + if ftype == TType.STOP: + break + else: + iprot.skip(ftype) + iprot.readFieldEnd() + iprot.readStructEnd() + + def write(self, oprot): + if oprot._fast_encode is not None and self.thrift_spec is not None: + oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec])) + return + oprot.writeStructBegin("update_compactor_state_result") + oprot.writeFieldStop() + oprot.writeStructEnd() + + def validate(self): + return + + def __repr__(self): + L = ["{}={!r}".format(key, value) for key, value in self.__dict__.items()] + return "{}({})".format(self.__class__.__name__, ", ".join(L)) + + def __eq__(self, other): + return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ + + def __ne__(self, other): + return not (self == other) + + +all_structs.append(update_compactor_state_result) +update_compactor_state_result.thrift_spec = () + + +class find_columns_with_stats_args: + """ + Attributes: + - cr + + """ + + def __init__( + self, + cr=None, + ): + self.cr = cr + + def read(self, iprot): + if ( + iprot._fast_decode is not None + and isinstance(iprot.trans, TTransport.CReadableTransport) + and self.thrift_spec is not None + ): + iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec]) + return + iprot.readStructBegin() + while True: + (fname, ftype, fid) = iprot.readFieldBegin() + if ftype == TType.STOP: + break + if fid == 1: + if ftype == TType.STRUCT: + self.cr = CompactionInfoStruct() + self.cr.read(iprot) + else: + iprot.skip(ftype) + else: + iprot.skip(ftype) + iprot.readFieldEnd() + iprot.readStructEnd() + + def write(self, oprot): + if oprot._fast_encode is not None and self.thrift_spec is not None: + oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec])) + return + oprot.writeStructBegin("find_columns_with_stats_args") + if self.cr is not None: + oprot.writeFieldBegin("cr", TType.STRUCT, 1) + self.cr.write(oprot) + oprot.writeFieldEnd() + oprot.writeFieldStop() + oprot.writeStructEnd() + + def validate(self): + return + + def __repr__(self): + L = ["{}={!r}".format(key, value) for key, value in self.__dict__.items()] + return "{}({})".format(self.__class__.__name__, ", ".join(L)) + + def __eq__(self, other): + return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ + + def __ne__(self, other): + return not (self == other) + + +all_structs.append(find_columns_with_stats_args) +find_columns_with_stats_args.thrift_spec = ( + None, # 0 + ( + 1, + TType.STRUCT, + "cr", + [CompactionInfoStruct, None], + None, + ), # 1 +) + + +class find_columns_with_stats_result: + """ + Attributes: + - success + + """ + + def __init__( + self, + success=None, + ): + self.success = success + + def read(self, iprot): + if ( + iprot._fast_decode is not None + and isinstance(iprot.trans, TTransport.CReadableTransport) + and self.thrift_spec is not None + ): + iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec]) + return + iprot.readStructBegin() + while True: + (fname, ftype, fid) = iprot.readFieldBegin() + if ftype == TType.STOP: + break + if fid == 0: + if ftype == TType.LIST: + self.success = [] + (_etype1768, _size1765) = iprot.readListBegin() + for _i1769 in range(_size1765): + _elem1770 = ( + iprot.readString().decode("utf-8", errors="replace") + if sys.version_info[0] == 2 + else iprot.readString() + ) + self.success.append(_elem1770) + iprot.readListEnd() + else: + iprot.skip(ftype) + else: + iprot.skip(ftype) + iprot.readFieldEnd() + iprot.readStructEnd() + + def write(self, oprot): + if oprot._fast_encode is not None and self.thrift_spec is not None: + oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec])) + return + oprot.writeStructBegin("find_columns_with_stats_result") + if self.success is not None: + oprot.writeFieldBegin("success", TType.LIST, 0) + oprot.writeListBegin(TType.STRING, len(self.success)) + for iter1771 in self.success: + oprot.writeString(iter1771.encode("utf-8") if sys.version_info[0] == 2 else iter1771) + oprot.writeListEnd() + oprot.writeFieldEnd() + oprot.writeFieldStop() + oprot.writeStructEnd() + + def validate(self): + return + + def __repr__(self): + L = ["{}={!r}".format(key, value) for key, value in self.__dict__.items()] + return "{}({})".format(self.__class__.__name__, ", ".join(L)) + + def __eq__(self, other): + return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ + + def __ne__(self, other): + return not (self == other) + + +all_structs.append(find_columns_with_stats_result) +find_columns_with_stats_result.thrift_spec = ( + ( + 0, + TType.LIST, + "success", + (TType.STRING, "UTF8", False), + None, + ), # 0 +) + + +class mark_cleaned_args: + """ + Attributes: + - cr + + """ + + def __init__( + self, + cr=None, + ): + self.cr = cr + + def read(self, iprot): + if ( + iprot._fast_decode is not None + and isinstance(iprot.trans, TTransport.CReadableTransport) + and self.thrift_spec is not None + ): + iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec]) + return + iprot.readStructBegin() + while True: + (fname, ftype, fid) = iprot.readFieldBegin() + if ftype == TType.STOP: + break + if fid == 1: + if ftype == TType.STRUCT: + self.cr = CompactionInfoStruct() + self.cr.read(iprot) + else: + iprot.skip(ftype) + else: + iprot.skip(ftype) + iprot.readFieldEnd() + iprot.readStructEnd() + + def write(self, oprot): + if oprot._fast_encode is not None and self.thrift_spec is not None: + oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec])) + return + oprot.writeStructBegin("mark_cleaned_args") + if self.cr is not None: + oprot.writeFieldBegin("cr", TType.STRUCT, 1) + self.cr.write(oprot) + oprot.writeFieldEnd() + oprot.writeFieldStop() + oprot.writeStructEnd() + + def validate(self): + return + + def __repr__(self): + L = ["{}={!r}".format(key, value) for key, value in self.__dict__.items()] + return "{}({})".format(self.__class__.__name__, ", ".join(L)) + + def __eq__(self, other): + return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ + + def __ne__(self, other): + return not (self == other) + + +all_structs.append(mark_cleaned_args) +mark_cleaned_args.thrift_spec = ( + None, # 0 + ( + 1, + TType.STRUCT, + "cr", + [CompactionInfoStruct, None], + None, + ), # 1 +) + + +class mark_cleaned_result: + """ + Attributes: + - o1 + + """ + + def __init__( + self, + o1=None, + ): + self.o1 = o1 + + def read(self, iprot): + if ( + iprot._fast_decode is not None + and isinstance(iprot.trans, TTransport.CReadableTransport) + and self.thrift_spec is not None + ): + iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec]) + return + iprot.readStructBegin() + while True: + (fname, ftype, fid) = iprot.readFieldBegin() + if ftype == TType.STOP: + break + if fid == 1: + if ftype == TType.STRUCT: + self.o1 = MetaException.read(iprot) + else: + iprot.skip(ftype) + else: + iprot.skip(ftype) + iprot.readFieldEnd() + iprot.readStructEnd() + + def write(self, oprot): + if oprot._fast_encode is not None and self.thrift_spec is not None: + oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec])) + return + oprot.writeStructBegin("mark_cleaned_result") + if self.o1 is not None: + oprot.writeFieldBegin("o1", TType.STRUCT, 1) + self.o1.write(oprot) + oprot.writeFieldEnd() + oprot.writeFieldStop() + oprot.writeStructEnd() + + def validate(self): + return + + def __repr__(self): + L = ["{}={!r}".format(key, value) for key, value in self.__dict__.items()] + return "{}({})".format(self.__class__.__name__, ", ".join(L)) + + def __eq__(self, other): + return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ + + def __ne__(self, other): + return not (self == other) + + +all_structs.append(mark_cleaned_result) +mark_cleaned_result.thrift_spec = ( + None, # 0 + ( + 1, + TType.STRUCT, + "o1", + [MetaException, None], + None, + ), # 1 +) + + +class mark_compacted_args: + """ + Attributes: + - cr + + """ + + def __init__( + self, + cr=None, + ): + self.cr = cr + + def read(self, iprot): + if ( + iprot._fast_decode is not None + and isinstance(iprot.trans, TTransport.CReadableTransport) + and self.thrift_spec is not None + ): + iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec]) + return + iprot.readStructBegin() + while True: + (fname, ftype, fid) = iprot.readFieldBegin() + if ftype == TType.STOP: + break + if fid == 1: + if ftype == TType.STRUCT: + self.cr = CompactionInfoStruct() + self.cr.read(iprot) + else: + iprot.skip(ftype) + else: + iprot.skip(ftype) + iprot.readFieldEnd() + iprot.readStructEnd() + + def write(self, oprot): + if oprot._fast_encode is not None and self.thrift_spec is not None: + oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec])) + return + oprot.writeStructBegin("mark_compacted_args") + if self.cr is not None: + oprot.writeFieldBegin("cr", TType.STRUCT, 1) + self.cr.write(oprot) + oprot.writeFieldEnd() + oprot.writeFieldStop() + oprot.writeStructEnd() + + def validate(self): + return + + def __repr__(self): + L = ["{}={!r}".format(key, value) for key, value in self.__dict__.items()] + return "{}({})".format(self.__class__.__name__, ", ".join(L)) + + def __eq__(self, other): + return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ + + def __ne__(self, other): + return not (self == other) + + +all_structs.append(mark_compacted_args) +mark_compacted_args.thrift_spec = ( + None, # 0 + ( + 1, + TType.STRUCT, + "cr", + [CompactionInfoStruct, None], + None, + ), # 1 +) + + +class mark_compacted_result: + """ + Attributes: + - o1 + + """ + + def __init__( + self, + o1=None, + ): + self.o1 = o1 + + def read(self, iprot): + if ( + iprot._fast_decode is not None + and isinstance(iprot.trans, TTransport.CReadableTransport) + and self.thrift_spec is not None + ): + iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec]) + return + iprot.readStructBegin() + while True: + (fname, ftype, fid) = iprot.readFieldBegin() + if ftype == TType.STOP: + break + if fid == 1: + if ftype == TType.STRUCT: + self.o1 = MetaException.read(iprot) + else: + iprot.skip(ftype) + else: + iprot.skip(ftype) + iprot.readFieldEnd() + iprot.readStructEnd() + + def write(self, oprot): + if oprot._fast_encode is not None and self.thrift_spec is not None: + oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec])) + return + oprot.writeStructBegin("mark_compacted_result") + if self.o1 is not None: + oprot.writeFieldBegin("o1", TType.STRUCT, 1) + self.o1.write(oprot) + oprot.writeFieldEnd() + oprot.writeFieldStop() + oprot.writeStructEnd() + + def validate(self): + return + + def __repr__(self): + L = ["{}={!r}".format(key, value) for key, value in self.__dict__.items()] + return "{}({})".format(self.__class__.__name__, ", ".join(L)) + + def __eq__(self, other): + return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ + + def __ne__(self, other): + return not (self == other) + + +all_structs.append(mark_compacted_result) +mark_compacted_result.thrift_spec = ( + None, # 0 + ( + 1, + TType.STRUCT, + "o1", + [MetaException, None], + None, + ), # 1 +) + + +class mark_failed_args: + """ + Attributes: + - cr + + """ + + def __init__( + self, + cr=None, + ): + self.cr = cr + + def read(self, iprot): + if ( + iprot._fast_decode is not None + and isinstance(iprot.trans, TTransport.CReadableTransport) + and self.thrift_spec is not None + ): + iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec]) + return + iprot.readStructBegin() + while True: + (fname, ftype, fid) = iprot.readFieldBegin() + if ftype == TType.STOP: + break + if fid == 1: + if ftype == TType.STRUCT: + self.cr = CompactionInfoStruct() + self.cr.read(iprot) + else: + iprot.skip(ftype) + else: + iprot.skip(ftype) + iprot.readFieldEnd() + iprot.readStructEnd() + + def write(self, oprot): + if oprot._fast_encode is not None and self.thrift_spec is not None: + oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec])) + return + oprot.writeStructBegin("mark_failed_args") + if self.cr is not None: + oprot.writeFieldBegin("cr", TType.STRUCT, 1) + self.cr.write(oprot) + oprot.writeFieldEnd() + oprot.writeFieldStop() + oprot.writeStructEnd() + + def validate(self): + return + + def __repr__(self): + L = ["{}={!r}".format(key, value) for key, value in self.__dict__.items()] + return "{}({})".format(self.__class__.__name__, ", ".join(L)) + + def __eq__(self, other): + return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ + + def __ne__(self, other): + return not (self == other) + + +all_structs.append(mark_failed_args) +mark_failed_args.thrift_spec = ( + None, # 0 + ( + 1, + TType.STRUCT, + "cr", + [CompactionInfoStruct, None], + None, + ), # 1 +) + + +class mark_failed_result: + """ + Attributes: + - o1 + + """ + + def __init__( + self, + o1=None, + ): + self.o1 = o1 + + def read(self, iprot): + if ( + iprot._fast_decode is not None + and isinstance(iprot.trans, TTransport.CReadableTransport) + and self.thrift_spec is not None + ): + iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec]) + return + iprot.readStructBegin() + while True: + (fname, ftype, fid) = iprot.readFieldBegin() + if ftype == TType.STOP: + break + if fid == 1: + if ftype == TType.STRUCT: + self.o1 = MetaException.read(iprot) + else: + iprot.skip(ftype) + else: + iprot.skip(ftype) + iprot.readFieldEnd() + iprot.readStructEnd() + + def write(self, oprot): + if oprot._fast_encode is not None and self.thrift_spec is not None: + oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec])) + return + oprot.writeStructBegin("mark_failed_result") + if self.o1 is not None: + oprot.writeFieldBegin("o1", TType.STRUCT, 1) + self.o1.write(oprot) + oprot.writeFieldEnd() + oprot.writeFieldStop() + oprot.writeStructEnd() + + def validate(self): + return + + def __repr__(self): + L = ["{}={!r}".format(key, value) for key, value in self.__dict__.items()] + return "{}({})".format(self.__class__.__name__, ", ".join(L)) + + def __eq__(self, other): + return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ + + def __ne__(self, other): + return not (self == other) + + +all_structs.append(mark_failed_result) +mark_failed_result.thrift_spec = ( + None, # 0 + ( + 1, + TType.STRUCT, + "o1", + [MetaException, None], + None, + ), # 1 +) + + +class mark_refused_args: + """ + Attributes: + - cr + + """ + + def __init__( + self, + cr=None, + ): + self.cr = cr + + def read(self, iprot): + if ( + iprot._fast_decode is not None + and isinstance(iprot.trans, TTransport.CReadableTransport) + and self.thrift_spec is not None + ): + iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec]) + return + iprot.readStructBegin() + while True: + (fname, ftype, fid) = iprot.readFieldBegin() + if ftype == TType.STOP: + break + if fid == 1: + if ftype == TType.STRUCT: + self.cr = CompactionInfoStruct() + self.cr.read(iprot) + else: + iprot.skip(ftype) + else: + iprot.skip(ftype) + iprot.readFieldEnd() + iprot.readStructEnd() + + def write(self, oprot): + if oprot._fast_encode is not None and self.thrift_spec is not None: + oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec])) + return + oprot.writeStructBegin("mark_refused_args") + if self.cr is not None: + oprot.writeFieldBegin("cr", TType.STRUCT, 1) + self.cr.write(oprot) + oprot.writeFieldEnd() + oprot.writeFieldStop() + oprot.writeStructEnd() + + def validate(self): + return + + def __repr__(self): + L = ["{}={!r}".format(key, value) for key, value in self.__dict__.items()] + return "{}({})".format(self.__class__.__name__, ", ".join(L)) + + def __eq__(self, other): + return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ + + def __ne__(self, other): + return not (self == other) + + +all_structs.append(mark_refused_args) +mark_refused_args.thrift_spec = ( + None, # 0 + ( + 1, + TType.STRUCT, + "cr", + [CompactionInfoStruct, None], + None, + ), # 1 +) + + +class mark_refused_result: + """ + Attributes: + - o1 + + """ + + def __init__( + self, + o1=None, + ): + self.o1 = o1 + + def read(self, iprot): + if ( + iprot._fast_decode is not None + and isinstance(iprot.trans, TTransport.CReadableTransport) + and self.thrift_spec is not None + ): + iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec]) + return + iprot.readStructBegin() + while True: + (fname, ftype, fid) = iprot.readFieldBegin() + if ftype == TType.STOP: + break + if fid == 1: + if ftype == TType.STRUCT: + self.o1 = MetaException.read(iprot) + else: + iprot.skip(ftype) + else: + iprot.skip(ftype) + iprot.readFieldEnd() + iprot.readStructEnd() + + def write(self, oprot): + if oprot._fast_encode is not None and self.thrift_spec is not None: + oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec])) + return + oprot.writeStructBegin("mark_refused_result") + if self.o1 is not None: + oprot.writeFieldBegin("o1", TType.STRUCT, 1) + self.o1.write(oprot) + oprot.writeFieldEnd() + oprot.writeFieldStop() + oprot.writeStructEnd() + + def validate(self): + return + + def __repr__(self): + L = ["{}={!r}".format(key, value) for key, value in self.__dict__.items()] + return "{}({})".format(self.__class__.__name__, ", ".join(L)) + + def __eq__(self, other): + return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ + + def __ne__(self, other): + return not (self == other) + + +all_structs.append(mark_refused_result) +mark_refused_result.thrift_spec = ( + None, # 0 + ( + 1, + TType.STRUCT, + "o1", + [MetaException, None], + None, + ), # 1 +) + + +class update_compaction_metrics_data_args: + """ + Attributes: + - data + + """ + + def __init__( + self, + data=None, + ): + self.data = data + + def read(self, iprot): + if ( + iprot._fast_decode is not None + and isinstance(iprot.trans, TTransport.CReadableTransport) + and self.thrift_spec is not None + ): + iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec]) + return + iprot.readStructBegin() + while True: + (fname, ftype, fid) = iprot.readFieldBegin() + if ftype == TType.STOP: + break + if fid == 1: + if ftype == TType.STRUCT: + self.data = CompactionMetricsDataStruct() + self.data.read(iprot) + else: + iprot.skip(ftype) + else: + iprot.skip(ftype) + iprot.readFieldEnd() + iprot.readStructEnd() + + def write(self, oprot): + if oprot._fast_encode is not None and self.thrift_spec is not None: + oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec])) + return + oprot.writeStructBegin("update_compaction_metrics_data_args") + if self.data is not None: + oprot.writeFieldBegin("data", TType.STRUCT, 1) + self.data.write(oprot) + oprot.writeFieldEnd() + oprot.writeFieldStop() + oprot.writeStructEnd() + + def validate(self): + return + + def __repr__(self): + L = ["{}={!r}".format(key, value) for key, value in self.__dict__.items()] + return "{}({})".format(self.__class__.__name__, ", ".join(L)) + + def __eq__(self, other): + return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ + + def __ne__(self, other): + return not (self == other) + + +all_structs.append(update_compaction_metrics_data_args) +update_compaction_metrics_data_args.thrift_spec = ( + None, # 0 + ( + 1, + TType.STRUCT, + "data", + [CompactionMetricsDataStruct, None], + None, + ), # 1 +) + + +class update_compaction_metrics_data_result: + """ + Attributes: + - success + - o1 + + """ + + def __init__( + self, + success=None, + o1=None, + ): + self.success = success + self.o1 = o1 + + def read(self, iprot): + if ( + iprot._fast_decode is not None + and isinstance(iprot.trans, TTransport.CReadableTransport) + and self.thrift_spec is not None + ): + iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec]) + return + iprot.readStructBegin() + while True: + (fname, ftype, fid) = iprot.readFieldBegin() + if ftype == TType.STOP: + break + if fid == 0: + if ftype == TType.BOOL: + self.success = iprot.readBool() + else: + iprot.skip(ftype) + elif fid == 1: + if ftype == TType.STRUCT: + self.o1 = MetaException.read(iprot) + else: + iprot.skip(ftype) + else: + iprot.skip(ftype) + iprot.readFieldEnd() + iprot.readStructEnd() + + def write(self, oprot): + if oprot._fast_encode is not None and self.thrift_spec is not None: + oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec])) + return + oprot.writeStructBegin("update_compaction_metrics_data_result") + if self.success is not None: + oprot.writeFieldBegin("success", TType.BOOL, 0) + oprot.writeBool(self.success) + oprot.writeFieldEnd() + if self.o1 is not None: + oprot.writeFieldBegin("o1", TType.STRUCT, 1) + self.o1.write(oprot) + oprot.writeFieldEnd() + oprot.writeFieldStop() + oprot.writeStructEnd() + + def validate(self): + return + + def __repr__(self): + L = ["{}={!r}".format(key, value) for key, value in self.__dict__.items()] + return "{}({})".format(self.__class__.__name__, ", ".join(L)) + + def __eq__(self, other): + return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ + + def __ne__(self, other): + return not (self == other) + + +all_structs.append(update_compaction_metrics_data_result) +update_compaction_metrics_data_result.thrift_spec = ( + ( + 0, + TType.BOOL, + "success", + None, + None, + ), # 0 + ( + 1, + TType.STRUCT, + "o1", + [MetaException, None], + None, + ), # 1 +) + + +class remove_compaction_metrics_data_args: + """ + Attributes: + - request + + """ + + def __init__( + self, + request=None, + ): + self.request = request + + def read(self, iprot): + if ( + iprot._fast_decode is not None + and isinstance(iprot.trans, TTransport.CReadableTransport) + and self.thrift_spec is not None + ): + iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec]) + return + iprot.readStructBegin() + while True: + (fname, ftype, fid) = iprot.readFieldBegin() + if ftype == TType.STOP: + break + if fid == 1: + if ftype == TType.STRUCT: + self.request = CompactionMetricsDataRequest() + self.request.read(iprot) + else: + iprot.skip(ftype) + else: + iprot.skip(ftype) + iprot.readFieldEnd() + iprot.readStructEnd() + + def write(self, oprot): + if oprot._fast_encode is not None and self.thrift_spec is not None: + oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec])) + return + oprot.writeStructBegin("remove_compaction_metrics_data_args") + if self.request is not None: + oprot.writeFieldBegin("request", TType.STRUCT, 1) + self.request.write(oprot) + oprot.writeFieldEnd() + oprot.writeFieldStop() + oprot.writeStructEnd() + + def validate(self): + return + + def __repr__(self): + L = ["{}={!r}".format(key, value) for key, value in self.__dict__.items()] + return "{}({})".format(self.__class__.__name__, ", ".join(L)) + + def __eq__(self, other): + return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ + + def __ne__(self, other): + return not (self == other) + + +all_structs.append(remove_compaction_metrics_data_args) +remove_compaction_metrics_data_args.thrift_spec = ( + None, # 0 + ( + 1, + TType.STRUCT, + "request", + [CompactionMetricsDataRequest, None], + None, + ), # 1 +) + + +class remove_compaction_metrics_data_result: + """ + Attributes: + - o1 + + """ + + def __init__( + self, + o1=None, + ): + self.o1 = o1 + + def read(self, iprot): + if ( + iprot._fast_decode is not None + and isinstance(iprot.trans, TTransport.CReadableTransport) + and self.thrift_spec is not None + ): + iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec]) + return + iprot.readStructBegin() + while True: + (fname, ftype, fid) = iprot.readFieldBegin() + if ftype == TType.STOP: + break + if fid == 1: + if ftype == TType.STRUCT: + self.o1 = MetaException.read(iprot) + else: + iprot.skip(ftype) + else: + iprot.skip(ftype) + iprot.readFieldEnd() + iprot.readStructEnd() + + def write(self, oprot): + if oprot._fast_encode is not None and self.thrift_spec is not None: + oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec])) + return + oprot.writeStructBegin("remove_compaction_metrics_data_result") + if self.o1 is not None: + oprot.writeFieldBegin("o1", TType.STRUCT, 1) + self.o1.write(oprot) + oprot.writeFieldEnd() + oprot.writeFieldStop() + oprot.writeStructEnd() + + def validate(self): + return + + def __repr__(self): + L = ["{}={!r}".format(key, value) for key, value in self.__dict__.items()] + return "{}({})".format(self.__class__.__name__, ", ".join(L)) + + def __eq__(self, other): + return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ + + def __ne__(self, other): + return not (self == other) + + +all_structs.append(remove_compaction_metrics_data_result) +remove_compaction_metrics_data_result.thrift_spec = ( + None, # 0 + ( + 1, + TType.STRUCT, + "o1", + [MetaException, None], + None, + ), # 1 +) + + +class set_hadoop_jobid_args: + """ + Attributes: + - jobId + - cq_id + + """ + + def __init__( + self, + jobId=None, + cq_id=None, + ): + self.jobId = jobId + self.cq_id = cq_id + + def read(self, iprot): + if ( + iprot._fast_decode is not None + and isinstance(iprot.trans, TTransport.CReadableTransport) + and self.thrift_spec is not None + ): + iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec]) + return + iprot.readStructBegin() + while True: + (fname, ftype, fid) = iprot.readFieldBegin() + if ftype == TType.STOP: + break + if fid == 1: + if ftype == TType.STRING: + self.jobId = ( + iprot.readString().decode("utf-8", errors="replace") if sys.version_info[0] == 2 else iprot.readString() + ) + else: + iprot.skip(ftype) + elif fid == 2: + if ftype == TType.I64: + self.cq_id = iprot.readI64() + else: + iprot.skip(ftype) + else: + iprot.skip(ftype) + iprot.readFieldEnd() + iprot.readStructEnd() + + def write(self, oprot): + if oprot._fast_encode is not None and self.thrift_spec is not None: + oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec])) + return + oprot.writeStructBegin("set_hadoop_jobid_args") + if self.jobId is not None: + oprot.writeFieldBegin("jobId", TType.STRING, 1) + oprot.writeString(self.jobId.encode("utf-8") if sys.version_info[0] == 2 else self.jobId) + oprot.writeFieldEnd() + if self.cq_id is not None: + oprot.writeFieldBegin("cq_id", TType.I64, 2) + oprot.writeI64(self.cq_id) + oprot.writeFieldEnd() + oprot.writeFieldStop() + oprot.writeStructEnd() + + def validate(self): + return + + def __repr__(self): + L = ["{}={!r}".format(key, value) for key, value in self.__dict__.items()] + return "{}({})".format(self.__class__.__name__, ", ".join(L)) + + def __eq__(self, other): + return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ + + def __ne__(self, other): + return not (self == other) + + +all_structs.append(set_hadoop_jobid_args) +set_hadoop_jobid_args.thrift_spec = ( + None, # 0 + ( + 1, + TType.STRING, + "jobId", + "UTF8", + None, + ), # 1 + ( + 2, + TType.I64, + "cq_id", + None, + None, + ), # 2 +) + + +class set_hadoop_jobid_result: + def read(self, iprot): + if ( + iprot._fast_decode is not None + and isinstance(iprot.trans, TTransport.CReadableTransport) + and self.thrift_spec is not None + ): + iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec]) + return + iprot.readStructBegin() + while True: + (fname, ftype, fid) = iprot.readFieldBegin() + if ftype == TType.STOP: + break + else: + iprot.skip(ftype) + iprot.readFieldEnd() + iprot.readStructEnd() + + def write(self, oprot): + if oprot._fast_encode is not None and self.thrift_spec is not None: + oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec])) + return + oprot.writeStructBegin("set_hadoop_jobid_result") + oprot.writeFieldStop() + oprot.writeStructEnd() + + def validate(self): + return + + def __repr__(self): + L = ["{}={!r}".format(key, value) for key, value in self.__dict__.items()] + return "{}({})".format(self.__class__.__name__, ", ".join(L)) + + def __eq__(self, other): + return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ + + def __ne__(self, other): + return not (self == other) + + +all_structs.append(set_hadoop_jobid_result) +set_hadoop_jobid_result.thrift_spec = () + + +class get_latest_committed_compaction_info_args: + """ + Attributes: + - rqst + + """ + + def __init__( + self, + rqst=None, + ): + self.rqst = rqst + + def read(self, iprot): + if ( + iprot._fast_decode is not None + and isinstance(iprot.trans, TTransport.CReadableTransport) + and self.thrift_spec is not None + ): + iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec]) + return + iprot.readStructBegin() + while True: + (fname, ftype, fid) = iprot.readFieldBegin() + if ftype == TType.STOP: + break + if fid == 1: + if ftype == TType.STRUCT: + self.rqst = GetLatestCommittedCompactionInfoRequest() + self.rqst.read(iprot) + else: + iprot.skip(ftype) + else: + iprot.skip(ftype) + iprot.readFieldEnd() + iprot.readStructEnd() + + def write(self, oprot): + if oprot._fast_encode is not None and self.thrift_spec is not None: + oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec])) + return + oprot.writeStructBegin("get_latest_committed_compaction_info_args") + if self.rqst is not None: + oprot.writeFieldBegin("rqst", TType.STRUCT, 1) + self.rqst.write(oprot) + oprot.writeFieldEnd() + oprot.writeFieldStop() + oprot.writeStructEnd() + + def validate(self): + return + + def __repr__(self): + L = ["{}={!r}".format(key, value) for key, value in self.__dict__.items()] + return "{}({})".format(self.__class__.__name__, ", ".join(L)) + + def __eq__(self, other): + return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ + + def __ne__(self, other): + return not (self == other) + + +all_structs.append(get_latest_committed_compaction_info_args) +get_latest_committed_compaction_info_args.thrift_spec = ( + None, # 0 + ( + 1, + TType.STRUCT, + "rqst", + [GetLatestCommittedCompactionInfoRequest, None], + None, + ), # 1 +) + + +class get_latest_committed_compaction_info_result: + """ + Attributes: + - success + + """ + + def __init__( + self, + success=None, + ): + self.success = success + + def read(self, iprot): + if ( + iprot._fast_decode is not None + and isinstance(iprot.trans, TTransport.CReadableTransport) + and self.thrift_spec is not None + ): + iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec]) + return + iprot.readStructBegin() + while True: + (fname, ftype, fid) = iprot.readFieldBegin() + if ftype == TType.STOP: + break + if fid == 0: + if ftype == TType.STRUCT: + self.success = GetLatestCommittedCompactionInfoResponse() + self.success.read(iprot) + else: + iprot.skip(ftype) + else: + iprot.skip(ftype) + iprot.readFieldEnd() + iprot.readStructEnd() + + def write(self, oprot): + if oprot._fast_encode is not None and self.thrift_spec is not None: + oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec])) + return + oprot.writeStructBegin("get_latest_committed_compaction_info_result") + if self.success is not None: + oprot.writeFieldBegin("success", TType.STRUCT, 0) + self.success.write(oprot) + oprot.writeFieldEnd() + oprot.writeFieldStop() + oprot.writeStructEnd() + + def validate(self): + return + + def __repr__(self): + L = ["{}={!r}".format(key, value) for key, value in self.__dict__.items()] + return "{}({})".format(self.__class__.__name__, ", ".join(L)) + + def __eq__(self, other): + return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ + + def __ne__(self, other): + return not (self == other) + + +all_structs.append(get_latest_committed_compaction_info_result) +get_latest_committed_compaction_info_result.thrift_spec = ( + ( + 0, + TType.STRUCT, + "success", + [GetLatestCommittedCompactionInfoResponse, None], + None, + ), # 0 +) + + +class get_next_notification_args: + """ + Attributes: + - rqst + + """ + + def __init__( + self, + rqst=None, + ): + self.rqst = rqst + + def read(self, iprot): + if ( + iprot._fast_decode is not None + and isinstance(iprot.trans, TTransport.CReadableTransport) + and self.thrift_spec is not None + ): + iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec]) + return + iprot.readStructBegin() + while True: + (fname, ftype, fid) = iprot.readFieldBegin() + if ftype == TType.STOP: + break + if fid == 1: + if ftype == TType.STRUCT: + self.rqst = NotificationEventRequest() + self.rqst.read(iprot) + else: + iprot.skip(ftype) + else: + iprot.skip(ftype) + iprot.readFieldEnd() + iprot.readStructEnd() + + def write(self, oprot): + if oprot._fast_encode is not None and self.thrift_spec is not None: + oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec])) + return + oprot.writeStructBegin("get_next_notification_args") + if self.rqst is not None: + oprot.writeFieldBegin("rqst", TType.STRUCT, 1) + self.rqst.write(oprot) + oprot.writeFieldEnd() + oprot.writeFieldStop() + oprot.writeStructEnd() + + def validate(self): + return + + def __repr__(self): + L = ["{}={!r}".format(key, value) for key, value in self.__dict__.items()] + return "{}({})".format(self.__class__.__name__, ", ".join(L)) + + def __eq__(self, other): + return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ + + def __ne__(self, other): + return not (self == other) + + +all_structs.append(get_next_notification_args) +get_next_notification_args.thrift_spec = ( + None, # 0 + ( + 1, + TType.STRUCT, + "rqst", + [NotificationEventRequest, None], + None, + ), # 1 +) + + +class get_next_notification_result: + """ + Attributes: + - success + + """ + + def __init__( + self, + success=None, + ): + self.success = success + + def read(self, iprot): + if ( + iprot._fast_decode is not None + and isinstance(iprot.trans, TTransport.CReadableTransport) + and self.thrift_spec is not None + ): + iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec]) + return + iprot.readStructBegin() + while True: + (fname, ftype, fid) = iprot.readFieldBegin() + if ftype == TType.STOP: + break + if fid == 0: + if ftype == TType.STRUCT: + self.success = NotificationEventResponse() + self.success.read(iprot) + else: + iprot.skip(ftype) + else: + iprot.skip(ftype) + iprot.readFieldEnd() + iprot.readStructEnd() + + def write(self, oprot): + if oprot._fast_encode is not None and self.thrift_spec is not None: + oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec])) + return + oprot.writeStructBegin("get_next_notification_result") + if self.success is not None: + oprot.writeFieldBegin("success", TType.STRUCT, 0) + self.success.write(oprot) + oprot.writeFieldEnd() + oprot.writeFieldStop() + oprot.writeStructEnd() + + def validate(self): + return + + def __repr__(self): + L = ["{}={!r}".format(key, value) for key, value in self.__dict__.items()] + return "{}({})".format(self.__class__.__name__, ", ".join(L)) + + def __eq__(self, other): + return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ + + def __ne__(self, other): + return not (self == other) + + +all_structs.append(get_next_notification_result) +get_next_notification_result.thrift_spec = ( + ( + 0, + TType.STRUCT, + "success", + [NotificationEventResponse, None], + None, + ), # 0 +) + + +class get_current_notificationEventId_args: + def read(self, iprot): + if ( + iprot._fast_decode is not None + and isinstance(iprot.trans, TTransport.CReadableTransport) + and self.thrift_spec is not None + ): + iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec]) + return + iprot.readStructBegin() + while True: + (fname, ftype, fid) = iprot.readFieldBegin() + if ftype == TType.STOP: + break + else: + iprot.skip(ftype) + iprot.readFieldEnd() + iprot.readStructEnd() + + def write(self, oprot): + if oprot._fast_encode is not None and self.thrift_spec is not None: + oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec])) + return + oprot.writeStructBegin("get_current_notificationEventId_args") + oprot.writeFieldStop() + oprot.writeStructEnd() + + def validate(self): + return + + def __repr__(self): + L = ["{}={!r}".format(key, value) for key, value in self.__dict__.items()] + return "{}({})".format(self.__class__.__name__, ", ".join(L)) + + def __eq__(self, other): + return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ + + def __ne__(self, other): + return not (self == other) + + +all_structs.append(get_current_notificationEventId_args) +get_current_notificationEventId_args.thrift_spec = () + + +class get_current_notificationEventId_result: + """ + Attributes: + - success + + """ + + def __init__( + self, + success=None, + ): + self.success = success + + def read(self, iprot): + if ( + iprot._fast_decode is not None + and isinstance(iprot.trans, TTransport.CReadableTransport) + and self.thrift_spec is not None + ): + iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec]) + return + iprot.readStructBegin() + while True: + (fname, ftype, fid) = iprot.readFieldBegin() + if ftype == TType.STOP: + break + if fid == 0: + if ftype == TType.STRUCT: + self.success = CurrentNotificationEventId() + self.success.read(iprot) + else: + iprot.skip(ftype) + else: + iprot.skip(ftype) + iprot.readFieldEnd() + iprot.readStructEnd() + + def write(self, oprot): + if oprot._fast_encode is not None and self.thrift_spec is not None: + oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec])) + return + oprot.writeStructBegin("get_current_notificationEventId_result") + if self.success is not None: + oprot.writeFieldBegin("success", TType.STRUCT, 0) + self.success.write(oprot) + oprot.writeFieldEnd() + oprot.writeFieldStop() + oprot.writeStructEnd() + + def validate(self): + return + + def __repr__(self): + L = ["{}={!r}".format(key, value) for key, value in self.__dict__.items()] + return "{}({})".format(self.__class__.__name__, ", ".join(L)) + + def __eq__(self, other): + return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ + + def __ne__(self, other): + return not (self == other) + + +all_structs.append(get_current_notificationEventId_result) +get_current_notificationEventId_result.thrift_spec = ( + ( + 0, + TType.STRUCT, + "success", + [CurrentNotificationEventId, None], + None, + ), # 0 +) + + +class get_notification_events_count_args: + """ + Attributes: + - rqst + + """ + + def __init__( + self, + rqst=None, + ): + self.rqst = rqst + + def read(self, iprot): + if ( + iprot._fast_decode is not None + and isinstance(iprot.trans, TTransport.CReadableTransport) + and self.thrift_spec is not None + ): + iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec]) + return + iprot.readStructBegin() + while True: + (fname, ftype, fid) = iprot.readFieldBegin() + if ftype == TType.STOP: + break + if fid == 1: + if ftype == TType.STRUCT: + self.rqst = NotificationEventsCountRequest() + self.rqst.read(iprot) + else: + iprot.skip(ftype) + else: + iprot.skip(ftype) + iprot.readFieldEnd() + iprot.readStructEnd() + + def write(self, oprot): + if oprot._fast_encode is not None and self.thrift_spec is not None: + oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec])) + return + oprot.writeStructBegin("get_notification_events_count_args") + if self.rqst is not None: + oprot.writeFieldBegin("rqst", TType.STRUCT, 1) + self.rqst.write(oprot) + oprot.writeFieldEnd() + oprot.writeFieldStop() + oprot.writeStructEnd() + + def validate(self): + return + + def __repr__(self): + L = ["{}={!r}".format(key, value) for key, value in self.__dict__.items()] + return "{}({})".format(self.__class__.__name__, ", ".join(L)) + + def __eq__(self, other): + return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ + + def __ne__(self, other): + return not (self == other) + + +all_structs.append(get_notification_events_count_args) +get_notification_events_count_args.thrift_spec = ( + None, # 0 + ( + 1, + TType.STRUCT, + "rqst", + [NotificationEventsCountRequest, None], + None, + ), # 1 +) + + +class get_notification_events_count_result: + """ + Attributes: + - success + + """ + + def __init__( + self, + success=None, + ): + self.success = success + + def read(self, iprot): + if ( + iprot._fast_decode is not None + and isinstance(iprot.trans, TTransport.CReadableTransport) + and self.thrift_spec is not None + ): + iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec]) + return + iprot.readStructBegin() + while True: + (fname, ftype, fid) = iprot.readFieldBegin() + if ftype == TType.STOP: + break + if fid == 0: + if ftype == TType.STRUCT: + self.success = NotificationEventsCountResponse() + self.success.read(iprot) + else: + iprot.skip(ftype) + else: + iprot.skip(ftype) + iprot.readFieldEnd() + iprot.readStructEnd() + + def write(self, oprot): + if oprot._fast_encode is not None and self.thrift_spec is not None: + oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec])) + return + oprot.writeStructBegin("get_notification_events_count_result") + if self.success is not None: + oprot.writeFieldBegin("success", TType.STRUCT, 0) + self.success.write(oprot) + oprot.writeFieldEnd() + oprot.writeFieldStop() + oprot.writeStructEnd() + + def validate(self): + return + + def __repr__(self): + L = ["{}={!r}".format(key, value) for key, value in self.__dict__.items()] + return "{}({})".format(self.__class__.__name__, ", ".join(L)) + + def __eq__(self, other): + return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ + + def __ne__(self, other): + return not (self == other) + + +all_structs.append(get_notification_events_count_result) +get_notification_events_count_result.thrift_spec = ( + ( + 0, + TType.STRUCT, + "success", + [NotificationEventsCountResponse, None], + None, + ), # 0 +) + + +class fire_listener_event_args: + """ + Attributes: + - rqst + + """ + + def __init__( + self, + rqst=None, + ): + self.rqst = rqst + + def read(self, iprot): + if ( + iprot._fast_decode is not None + and isinstance(iprot.trans, TTransport.CReadableTransport) + and self.thrift_spec is not None + ): + iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec]) + return + iprot.readStructBegin() + while True: + (fname, ftype, fid) = iprot.readFieldBegin() + if ftype == TType.STOP: + break + if fid == 1: + if ftype == TType.STRUCT: + self.rqst = FireEventRequest() + self.rqst.read(iprot) + else: + iprot.skip(ftype) + else: + iprot.skip(ftype) + iprot.readFieldEnd() + iprot.readStructEnd() + + def write(self, oprot): + if oprot._fast_encode is not None and self.thrift_spec is not None: + oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec])) + return + oprot.writeStructBegin("fire_listener_event_args") + if self.rqst is not None: + oprot.writeFieldBegin("rqst", TType.STRUCT, 1) + self.rqst.write(oprot) + oprot.writeFieldEnd() + oprot.writeFieldStop() + oprot.writeStructEnd() + + def validate(self): + return + + def __repr__(self): + L = ["{}={!r}".format(key, value) for key, value in self.__dict__.items()] + return "{}({})".format(self.__class__.__name__, ", ".join(L)) + + def __eq__(self, other): + return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ + + def __ne__(self, other): + return not (self == other) + + +all_structs.append(fire_listener_event_args) +fire_listener_event_args.thrift_spec = ( + None, # 0 + ( + 1, + TType.STRUCT, + "rqst", + [FireEventRequest, None], + None, + ), # 1 +) + + +class fire_listener_event_result: + """ + Attributes: + - success + + """ + + def __init__( + self, + success=None, + ): + self.success = success + + def read(self, iprot): + if ( + iprot._fast_decode is not None + and isinstance(iprot.trans, TTransport.CReadableTransport) + and self.thrift_spec is not None + ): + iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec]) + return + iprot.readStructBegin() + while True: + (fname, ftype, fid) = iprot.readFieldBegin() + if ftype == TType.STOP: + break + if fid == 0: + if ftype == TType.STRUCT: + self.success = FireEventResponse() + self.success.read(iprot) + else: + iprot.skip(ftype) + else: + iprot.skip(ftype) + iprot.readFieldEnd() + iprot.readStructEnd() + + def write(self, oprot): + if oprot._fast_encode is not None and self.thrift_spec is not None: + oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec])) + return + oprot.writeStructBegin("fire_listener_event_result") + if self.success is not None: + oprot.writeFieldBegin("success", TType.STRUCT, 0) + self.success.write(oprot) + oprot.writeFieldEnd() + oprot.writeFieldStop() + oprot.writeStructEnd() + + def validate(self): + return + + def __repr__(self): + L = ["{}={!r}".format(key, value) for key, value in self.__dict__.items()] + return "{}({})".format(self.__class__.__name__, ", ".join(L)) + + def __eq__(self, other): + return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ + + def __ne__(self, other): + return not (self == other) + + +all_structs.append(fire_listener_event_result) +fire_listener_event_result.thrift_spec = ( + ( + 0, + TType.STRUCT, + "success", + [FireEventResponse, None], + None, + ), # 0 +) + + +class flushCache_args: + def read(self, iprot): + if ( + iprot._fast_decode is not None + and isinstance(iprot.trans, TTransport.CReadableTransport) + and self.thrift_spec is not None + ): + iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec]) + return + iprot.readStructBegin() + while True: + (fname, ftype, fid) = iprot.readFieldBegin() + if ftype == TType.STOP: + break + else: + iprot.skip(ftype) + iprot.readFieldEnd() + iprot.readStructEnd() + + def write(self, oprot): + if oprot._fast_encode is not None and self.thrift_spec is not None: + oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec])) + return + oprot.writeStructBegin("flushCache_args") + oprot.writeFieldStop() + oprot.writeStructEnd() + + def validate(self): + return + + def __repr__(self): + L = ["{}={!r}".format(key, value) for key, value in self.__dict__.items()] + return "{}({})".format(self.__class__.__name__, ", ".join(L)) + + def __eq__(self, other): + return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ + + def __ne__(self, other): + return not (self == other) + + +all_structs.append(flushCache_args) +flushCache_args.thrift_spec = () + + +class flushCache_result: + def read(self, iprot): + if ( + iprot._fast_decode is not None + and isinstance(iprot.trans, TTransport.CReadableTransport) + and self.thrift_spec is not None + ): + iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec]) + return + iprot.readStructBegin() + while True: + (fname, ftype, fid) = iprot.readFieldBegin() + if ftype == TType.STOP: + break + else: + iprot.skip(ftype) + iprot.readFieldEnd() + iprot.readStructEnd() + + def write(self, oprot): + if oprot._fast_encode is not None and self.thrift_spec is not None: + oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec])) + return + oprot.writeStructBegin("flushCache_result") + oprot.writeFieldStop() + oprot.writeStructEnd() + + def validate(self): + return + + def __repr__(self): + L = ["{}={!r}".format(key, value) for key, value in self.__dict__.items()] + return "{}({})".format(self.__class__.__name__, ", ".join(L)) + + def __eq__(self, other): + return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ + + def __ne__(self, other): + return not (self == other) + + +all_structs.append(flushCache_result) +flushCache_result.thrift_spec = () + + +class add_write_notification_log_args: + """ + Attributes: + - rqst + + """ + + def __init__( + self, + rqst=None, + ): + self.rqst = rqst + + def read(self, iprot): + if ( + iprot._fast_decode is not None + and isinstance(iprot.trans, TTransport.CReadableTransport) + and self.thrift_spec is not None + ): + iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec]) + return + iprot.readStructBegin() + while True: + (fname, ftype, fid) = iprot.readFieldBegin() + if ftype == TType.STOP: + break + if fid == 1: + if ftype == TType.STRUCT: + self.rqst = WriteNotificationLogRequest() + self.rqst.read(iprot) + else: + iprot.skip(ftype) + else: + iprot.skip(ftype) + iprot.readFieldEnd() + iprot.readStructEnd() + + def write(self, oprot): + if oprot._fast_encode is not None and self.thrift_spec is not None: + oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec])) + return + oprot.writeStructBegin("add_write_notification_log_args") + if self.rqst is not None: + oprot.writeFieldBegin("rqst", TType.STRUCT, 1) + self.rqst.write(oprot) + oprot.writeFieldEnd() + oprot.writeFieldStop() + oprot.writeStructEnd() + + def validate(self): + return + + def __repr__(self): + L = ["{}={!r}".format(key, value) for key, value in self.__dict__.items()] + return "{}({})".format(self.__class__.__name__, ", ".join(L)) + + def __eq__(self, other): + return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ + + def __ne__(self, other): + return not (self == other) + + +all_structs.append(add_write_notification_log_args) +add_write_notification_log_args.thrift_spec = ( + None, # 0 + ( + 1, + TType.STRUCT, + "rqst", + [WriteNotificationLogRequest, None], + None, + ), # 1 +) + + +class add_write_notification_log_result: + """ + Attributes: + - success + + """ + + def __init__( + self, + success=None, + ): + self.success = success + + def read(self, iprot): + if ( + iprot._fast_decode is not None + and isinstance(iprot.trans, TTransport.CReadableTransport) + and self.thrift_spec is not None + ): + iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec]) + return + iprot.readStructBegin() + while True: + (fname, ftype, fid) = iprot.readFieldBegin() + if ftype == TType.STOP: + break + if fid == 0: + if ftype == TType.STRUCT: + self.success = WriteNotificationLogResponse() + self.success.read(iprot) + else: + iprot.skip(ftype) + else: + iprot.skip(ftype) + iprot.readFieldEnd() + iprot.readStructEnd() + + def write(self, oprot): + if oprot._fast_encode is not None and self.thrift_spec is not None: + oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec])) + return + oprot.writeStructBegin("add_write_notification_log_result") + if self.success is not None: + oprot.writeFieldBegin("success", TType.STRUCT, 0) + self.success.write(oprot) + oprot.writeFieldEnd() + oprot.writeFieldStop() + oprot.writeStructEnd() + + def validate(self): + return + + def __repr__(self): + L = ["{}={!r}".format(key, value) for key, value in self.__dict__.items()] + return "{}({})".format(self.__class__.__name__, ", ".join(L)) + + def __eq__(self, other): + return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ + + def __ne__(self, other): + return not (self == other) + + +all_structs.append(add_write_notification_log_result) +add_write_notification_log_result.thrift_spec = ( + ( + 0, + TType.STRUCT, + "success", + [WriteNotificationLogResponse, None], + None, + ), # 0 +) + + +class add_write_notification_log_in_batch_args: + """ + Attributes: + - rqst + + """ + + def __init__( + self, + rqst=None, + ): + self.rqst = rqst + + def read(self, iprot): + if ( + iprot._fast_decode is not None + and isinstance(iprot.trans, TTransport.CReadableTransport) + and self.thrift_spec is not None + ): + iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec]) + return + iprot.readStructBegin() + while True: + (fname, ftype, fid) = iprot.readFieldBegin() + if ftype == TType.STOP: + break + if fid == 1: + if ftype == TType.STRUCT: + self.rqst = WriteNotificationLogBatchRequest() + self.rqst.read(iprot) + else: + iprot.skip(ftype) + else: + iprot.skip(ftype) + iprot.readFieldEnd() + iprot.readStructEnd() + + def write(self, oprot): + if oprot._fast_encode is not None and self.thrift_spec is not None: + oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec])) + return + oprot.writeStructBegin("add_write_notification_log_in_batch_args") + if self.rqst is not None: + oprot.writeFieldBegin("rqst", TType.STRUCT, 1) + self.rqst.write(oprot) + oprot.writeFieldEnd() + oprot.writeFieldStop() + oprot.writeStructEnd() + + def validate(self): + return + + def __repr__(self): + L = ["{}={!r}".format(key, value) for key, value in self.__dict__.items()] + return "{}({})".format(self.__class__.__name__, ", ".join(L)) + + def __eq__(self, other): + return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ + + def __ne__(self, other): + return not (self == other) + + +all_structs.append(add_write_notification_log_in_batch_args) +add_write_notification_log_in_batch_args.thrift_spec = ( + None, # 0 + ( + 1, + TType.STRUCT, + "rqst", + [WriteNotificationLogBatchRequest, None], + None, + ), # 1 +) + + +class add_write_notification_log_in_batch_result: + """ + Attributes: + - success + + """ + + def __init__( + self, + success=None, + ): + self.success = success + + def read(self, iprot): + if ( + iprot._fast_decode is not None + and isinstance(iprot.trans, TTransport.CReadableTransport) + and self.thrift_spec is not None + ): + iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec]) + return + iprot.readStructBegin() + while True: + (fname, ftype, fid) = iprot.readFieldBegin() + if ftype == TType.STOP: + break + if fid == 0: + if ftype == TType.STRUCT: + self.success = WriteNotificationLogBatchResponse() + self.success.read(iprot) + else: + iprot.skip(ftype) + else: + iprot.skip(ftype) + iprot.readFieldEnd() + iprot.readStructEnd() + + def write(self, oprot): + if oprot._fast_encode is not None and self.thrift_spec is not None: + oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec])) + return + oprot.writeStructBegin("add_write_notification_log_in_batch_result") + if self.success is not None: + oprot.writeFieldBegin("success", TType.STRUCT, 0) + self.success.write(oprot) + oprot.writeFieldEnd() + oprot.writeFieldStop() + oprot.writeStructEnd() + + def validate(self): + return + + def __repr__(self): + L = ["{}={!r}".format(key, value) for key, value in self.__dict__.items()] + return "{}({})".format(self.__class__.__name__, ", ".join(L)) + + def __eq__(self, other): + return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ + + def __ne__(self, other): + return not (self == other) + + +all_structs.append(add_write_notification_log_in_batch_result) +add_write_notification_log_in_batch_result.thrift_spec = ( + ( + 0, + TType.STRUCT, + "success", + [WriteNotificationLogBatchResponse, None], + None, + ), # 0 +) + + +class cm_recycle_args: + """ + Attributes: + - request + + """ + + def __init__( + self, + request=None, + ): + self.request = request + + def read(self, iprot): + if ( + iprot._fast_decode is not None + and isinstance(iprot.trans, TTransport.CReadableTransport) + and self.thrift_spec is not None + ): + iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec]) + return + iprot.readStructBegin() + while True: + (fname, ftype, fid) = iprot.readFieldBegin() + if ftype == TType.STOP: + break + if fid == 1: + if ftype == TType.STRUCT: + self.request = CmRecycleRequest() + self.request.read(iprot) + else: + iprot.skip(ftype) + else: + iprot.skip(ftype) + iprot.readFieldEnd() + iprot.readStructEnd() + + def write(self, oprot): + if oprot._fast_encode is not None and self.thrift_spec is not None: + oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec])) + return + oprot.writeStructBegin("cm_recycle_args") + if self.request is not None: + oprot.writeFieldBegin("request", TType.STRUCT, 1) + self.request.write(oprot) + oprot.writeFieldEnd() + oprot.writeFieldStop() + oprot.writeStructEnd() + + def validate(self): + return + + def __repr__(self): + L = ["{}={!r}".format(key, value) for key, value in self.__dict__.items()] + return "{}({})".format(self.__class__.__name__, ", ".join(L)) + + def __eq__(self, other): + return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ + + def __ne__(self, other): + return not (self == other) + + +all_structs.append(cm_recycle_args) +cm_recycle_args.thrift_spec = ( + None, # 0 + ( + 1, + TType.STRUCT, + "request", + [CmRecycleRequest, None], + None, + ), # 1 +) + + +class cm_recycle_result: + """ + Attributes: + - success + - o1 + + """ + + def __init__( + self, + success=None, + o1=None, + ): + self.success = success + self.o1 = o1 + + def read(self, iprot): + if ( + iprot._fast_decode is not None + and isinstance(iprot.trans, TTransport.CReadableTransport) + and self.thrift_spec is not None + ): + iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec]) + return + iprot.readStructBegin() + while True: + (fname, ftype, fid) = iprot.readFieldBegin() + if ftype == TType.STOP: + break + if fid == 0: + if ftype == TType.STRUCT: + self.success = CmRecycleResponse() + self.success.read(iprot) + else: + iprot.skip(ftype) + elif fid == 1: + if ftype == TType.STRUCT: + self.o1 = MetaException.read(iprot) + else: + iprot.skip(ftype) + else: + iprot.skip(ftype) + iprot.readFieldEnd() + iprot.readStructEnd() + + def write(self, oprot): + if oprot._fast_encode is not None and self.thrift_spec is not None: + oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec])) + return + oprot.writeStructBegin("cm_recycle_result") + if self.success is not None: + oprot.writeFieldBegin("success", TType.STRUCT, 0) + self.success.write(oprot) + oprot.writeFieldEnd() + if self.o1 is not None: + oprot.writeFieldBegin("o1", TType.STRUCT, 1) + self.o1.write(oprot) + oprot.writeFieldEnd() + oprot.writeFieldStop() + oprot.writeStructEnd() + + def validate(self): + return + + def __repr__(self): + L = ["{}={!r}".format(key, value) for key, value in self.__dict__.items()] + return "{}({})".format(self.__class__.__name__, ", ".join(L)) + + def __eq__(self, other): + return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ + + def __ne__(self, other): + return not (self == other) + + +all_structs.append(cm_recycle_result) +cm_recycle_result.thrift_spec = ( + ( + 0, + TType.STRUCT, + "success", + [CmRecycleResponse, None], + None, + ), # 0 + ( + 1, + TType.STRUCT, + "o1", + [MetaException, None], + None, + ), # 1 +) + + +class get_file_metadata_by_expr_args: + """ + Attributes: + - req + + """ + + def __init__( + self, + req=None, + ): + self.req = req + + def read(self, iprot): + if ( + iprot._fast_decode is not None + and isinstance(iprot.trans, TTransport.CReadableTransport) + and self.thrift_spec is not None + ): + iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec]) + return + iprot.readStructBegin() + while True: + (fname, ftype, fid) = iprot.readFieldBegin() + if ftype == TType.STOP: + break + if fid == 1: + if ftype == TType.STRUCT: + self.req = GetFileMetadataByExprRequest() + self.req.read(iprot) + else: + iprot.skip(ftype) + else: + iprot.skip(ftype) + iprot.readFieldEnd() + iprot.readStructEnd() + + def write(self, oprot): + if oprot._fast_encode is not None and self.thrift_spec is not None: + oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec])) + return + oprot.writeStructBegin("get_file_metadata_by_expr_args") + if self.req is not None: + oprot.writeFieldBegin("req", TType.STRUCT, 1) + self.req.write(oprot) + oprot.writeFieldEnd() + oprot.writeFieldStop() + oprot.writeStructEnd() + + def validate(self): + return + + def __repr__(self): + L = ["{}={!r}".format(key, value) for key, value in self.__dict__.items()] + return "{}({})".format(self.__class__.__name__, ", ".join(L)) + + def __eq__(self, other): + return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ + + def __ne__(self, other): + return not (self == other) + + +all_structs.append(get_file_metadata_by_expr_args) +get_file_metadata_by_expr_args.thrift_spec = ( + None, # 0 + ( + 1, + TType.STRUCT, + "req", + [GetFileMetadataByExprRequest, None], + None, + ), # 1 +) + + +class get_file_metadata_by_expr_result: + """ + Attributes: + - success + + """ + + def __init__( + self, + success=None, + ): + self.success = success + + def read(self, iprot): + if ( + iprot._fast_decode is not None + and isinstance(iprot.trans, TTransport.CReadableTransport) + and self.thrift_spec is not None + ): + iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec]) + return + iprot.readStructBegin() + while True: + (fname, ftype, fid) = iprot.readFieldBegin() + if ftype == TType.STOP: + break + if fid == 0: + if ftype == TType.STRUCT: + self.success = GetFileMetadataByExprResult() + self.success.read(iprot) + else: + iprot.skip(ftype) + else: + iprot.skip(ftype) + iprot.readFieldEnd() + iprot.readStructEnd() + + def write(self, oprot): + if oprot._fast_encode is not None and self.thrift_spec is not None: + oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec])) + return + oprot.writeStructBegin("get_file_metadata_by_expr_result") + if self.success is not None: + oprot.writeFieldBegin("success", TType.STRUCT, 0) + self.success.write(oprot) + oprot.writeFieldEnd() + oprot.writeFieldStop() + oprot.writeStructEnd() + + def validate(self): + return + + def __repr__(self): + L = ["{}={!r}".format(key, value) for key, value in self.__dict__.items()] + return "{}({})".format(self.__class__.__name__, ", ".join(L)) + + def __eq__(self, other): + return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ + + def __ne__(self, other): + return not (self == other) + + +all_structs.append(get_file_metadata_by_expr_result) +get_file_metadata_by_expr_result.thrift_spec = ( + ( + 0, + TType.STRUCT, + "success", + [GetFileMetadataByExprResult, None], + None, + ), # 0 +) + + +class get_file_metadata_args: + """ + Attributes: + - req + + """ + + def __init__( + self, + req=None, + ): + self.req = req + + def read(self, iprot): + if ( + iprot._fast_decode is not None + and isinstance(iprot.trans, TTransport.CReadableTransport) + and self.thrift_spec is not None + ): + iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec]) + return + iprot.readStructBegin() + while True: + (fname, ftype, fid) = iprot.readFieldBegin() + if ftype == TType.STOP: + break + if fid == 1: + if ftype == TType.STRUCT: + self.req = GetFileMetadataRequest() + self.req.read(iprot) + else: + iprot.skip(ftype) + else: + iprot.skip(ftype) + iprot.readFieldEnd() + iprot.readStructEnd() + + def write(self, oprot): + if oprot._fast_encode is not None and self.thrift_spec is not None: + oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec])) + return + oprot.writeStructBegin("get_file_metadata_args") + if self.req is not None: + oprot.writeFieldBegin("req", TType.STRUCT, 1) + self.req.write(oprot) + oprot.writeFieldEnd() + oprot.writeFieldStop() + oprot.writeStructEnd() + + def validate(self): + return + + def __repr__(self): + L = ["{}={!r}".format(key, value) for key, value in self.__dict__.items()] + return "{}({})".format(self.__class__.__name__, ", ".join(L)) + + def __eq__(self, other): + return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ + + def __ne__(self, other): + return not (self == other) + + +all_structs.append(get_file_metadata_args) +get_file_metadata_args.thrift_spec = ( + None, # 0 + ( + 1, + TType.STRUCT, + "req", + [GetFileMetadataRequest, None], + None, + ), # 1 +) + + +class get_file_metadata_result: + """ + Attributes: + - success + + """ + + def __init__( + self, + success=None, + ): + self.success = success + + def read(self, iprot): + if ( + iprot._fast_decode is not None + and isinstance(iprot.trans, TTransport.CReadableTransport) + and self.thrift_spec is not None + ): + iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec]) + return + iprot.readStructBegin() + while True: + (fname, ftype, fid) = iprot.readFieldBegin() + if ftype == TType.STOP: + break + if fid == 0: + if ftype == TType.STRUCT: + self.success = GetFileMetadataResult() + self.success.read(iprot) + else: + iprot.skip(ftype) + else: + iprot.skip(ftype) + iprot.readFieldEnd() + iprot.readStructEnd() + + def write(self, oprot): + if oprot._fast_encode is not None and self.thrift_spec is not None: + oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec])) + return + oprot.writeStructBegin("get_file_metadata_result") + if self.success is not None: + oprot.writeFieldBegin("success", TType.STRUCT, 0) + self.success.write(oprot) + oprot.writeFieldEnd() + oprot.writeFieldStop() + oprot.writeStructEnd() + + def validate(self): + return + + def __repr__(self): + L = ["{}={!r}".format(key, value) for key, value in self.__dict__.items()] + return "{}({})".format(self.__class__.__name__, ", ".join(L)) + + def __eq__(self, other): + return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ + + def __ne__(self, other): + return not (self == other) + + +all_structs.append(get_file_metadata_result) +get_file_metadata_result.thrift_spec = ( + ( + 0, + TType.STRUCT, + "success", + [GetFileMetadataResult, None], + None, + ), # 0 +) + + +class put_file_metadata_args: + """ + Attributes: + - req + + """ + + def __init__( + self, + req=None, + ): + self.req = req + + def read(self, iprot): + if ( + iprot._fast_decode is not None + and isinstance(iprot.trans, TTransport.CReadableTransport) + and self.thrift_spec is not None + ): + iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec]) + return + iprot.readStructBegin() + while True: + (fname, ftype, fid) = iprot.readFieldBegin() + if ftype == TType.STOP: + break + if fid == 1: + if ftype == TType.STRUCT: + self.req = PutFileMetadataRequest() + self.req.read(iprot) + else: + iprot.skip(ftype) + else: + iprot.skip(ftype) + iprot.readFieldEnd() + iprot.readStructEnd() + + def write(self, oprot): + if oprot._fast_encode is not None and self.thrift_spec is not None: + oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec])) + return + oprot.writeStructBegin("put_file_metadata_args") + if self.req is not None: + oprot.writeFieldBegin("req", TType.STRUCT, 1) + self.req.write(oprot) + oprot.writeFieldEnd() + oprot.writeFieldStop() + oprot.writeStructEnd() + + def validate(self): + return + + def __repr__(self): + L = ["{}={!r}".format(key, value) for key, value in self.__dict__.items()] + return "{}({})".format(self.__class__.__name__, ", ".join(L)) + + def __eq__(self, other): + return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ + + def __ne__(self, other): + return not (self == other) + + +all_structs.append(put_file_metadata_args) +put_file_metadata_args.thrift_spec = ( + None, # 0 + ( + 1, + TType.STRUCT, + "req", + [PutFileMetadataRequest, None], + None, + ), # 1 +) + + +class put_file_metadata_result: + """ + Attributes: + - success + + """ + + def __init__( + self, + success=None, + ): + self.success = success + + def read(self, iprot): + if ( + iprot._fast_decode is not None + and isinstance(iprot.trans, TTransport.CReadableTransport) + and self.thrift_spec is not None + ): + iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec]) + return + iprot.readStructBegin() + while True: + (fname, ftype, fid) = iprot.readFieldBegin() + if ftype == TType.STOP: + break + if fid == 0: + if ftype == TType.STRUCT: + self.success = PutFileMetadataResult() + self.success.read(iprot) + else: + iprot.skip(ftype) + else: + iprot.skip(ftype) + iprot.readFieldEnd() + iprot.readStructEnd() + + def write(self, oprot): + if oprot._fast_encode is not None and self.thrift_spec is not None: + oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec])) + return + oprot.writeStructBegin("put_file_metadata_result") + if self.success is not None: + oprot.writeFieldBegin("success", TType.STRUCT, 0) + self.success.write(oprot) + oprot.writeFieldEnd() + oprot.writeFieldStop() + oprot.writeStructEnd() + + def validate(self): + return + + def __repr__(self): + L = ["{}={!r}".format(key, value) for key, value in self.__dict__.items()] + return "{}({})".format(self.__class__.__name__, ", ".join(L)) + + def __eq__(self, other): + return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ + + def __ne__(self, other): + return not (self == other) + + +all_structs.append(put_file_metadata_result) +put_file_metadata_result.thrift_spec = ( + ( + 0, + TType.STRUCT, + "success", + [PutFileMetadataResult, None], + None, + ), # 0 +) + + +class clear_file_metadata_args: + """ + Attributes: + - req + + """ + + def __init__( + self, + req=None, + ): + self.req = req + + def read(self, iprot): + if ( + iprot._fast_decode is not None + and isinstance(iprot.trans, TTransport.CReadableTransport) + and self.thrift_spec is not None + ): + iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec]) + return + iprot.readStructBegin() + while True: + (fname, ftype, fid) = iprot.readFieldBegin() + if ftype == TType.STOP: + break + if fid == 1: + if ftype == TType.STRUCT: + self.req = ClearFileMetadataRequest() + self.req.read(iprot) + else: + iprot.skip(ftype) + else: + iprot.skip(ftype) + iprot.readFieldEnd() + iprot.readStructEnd() + + def write(self, oprot): + if oprot._fast_encode is not None and self.thrift_spec is not None: + oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec])) + return + oprot.writeStructBegin("clear_file_metadata_args") + if self.req is not None: + oprot.writeFieldBegin("req", TType.STRUCT, 1) + self.req.write(oprot) + oprot.writeFieldEnd() + oprot.writeFieldStop() + oprot.writeStructEnd() + + def validate(self): + return + + def __repr__(self): + L = ["{}={!r}".format(key, value) for key, value in self.__dict__.items()] + return "{}({})".format(self.__class__.__name__, ", ".join(L)) + + def __eq__(self, other): + return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ + + def __ne__(self, other): + return not (self == other) + + +all_structs.append(clear_file_metadata_args) +clear_file_metadata_args.thrift_spec = ( + None, # 0 + ( + 1, + TType.STRUCT, + "req", + [ClearFileMetadataRequest, None], + None, + ), # 1 +) + + +class clear_file_metadata_result: + """ + Attributes: + - success + + """ + + def __init__( + self, + success=None, + ): + self.success = success + + def read(self, iprot): + if ( + iprot._fast_decode is not None + and isinstance(iprot.trans, TTransport.CReadableTransport) + and self.thrift_spec is not None + ): + iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec]) + return + iprot.readStructBegin() + while True: + (fname, ftype, fid) = iprot.readFieldBegin() + if ftype == TType.STOP: + break + if fid == 0: + if ftype == TType.STRUCT: + self.success = ClearFileMetadataResult() + self.success.read(iprot) + else: + iprot.skip(ftype) + else: + iprot.skip(ftype) + iprot.readFieldEnd() + iprot.readStructEnd() + + def write(self, oprot): + if oprot._fast_encode is not None and self.thrift_spec is not None: + oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec])) + return + oprot.writeStructBegin("clear_file_metadata_result") + if self.success is not None: + oprot.writeFieldBegin("success", TType.STRUCT, 0) + self.success.write(oprot) + oprot.writeFieldEnd() + oprot.writeFieldStop() + oprot.writeStructEnd() + + def validate(self): + return + + def __repr__(self): + L = ["{}={!r}".format(key, value) for key, value in self.__dict__.items()] + return "{}({})".format(self.__class__.__name__, ", ".join(L)) + + def __eq__(self, other): + return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ + + def __ne__(self, other): + return not (self == other) + + +all_structs.append(clear_file_metadata_result) +clear_file_metadata_result.thrift_spec = ( + ( + 0, + TType.STRUCT, + "success", + [ClearFileMetadataResult, None], + None, + ), # 0 +) + + +class cache_file_metadata_args: + """ + Attributes: + - req + + """ + + def __init__( + self, + req=None, + ): + self.req = req + + def read(self, iprot): + if ( + iprot._fast_decode is not None + and isinstance(iprot.trans, TTransport.CReadableTransport) + and self.thrift_spec is not None + ): + iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec]) + return + iprot.readStructBegin() + while True: + (fname, ftype, fid) = iprot.readFieldBegin() + if ftype == TType.STOP: + break + if fid == 1: + if ftype == TType.STRUCT: + self.req = CacheFileMetadataRequest() + self.req.read(iprot) + else: + iprot.skip(ftype) + else: + iprot.skip(ftype) + iprot.readFieldEnd() + iprot.readStructEnd() + + def write(self, oprot): + if oprot._fast_encode is not None and self.thrift_spec is not None: + oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec])) + return + oprot.writeStructBegin("cache_file_metadata_args") + if self.req is not None: + oprot.writeFieldBegin("req", TType.STRUCT, 1) + self.req.write(oprot) + oprot.writeFieldEnd() + oprot.writeFieldStop() + oprot.writeStructEnd() + + def validate(self): + return + + def __repr__(self): + L = ["{}={!r}".format(key, value) for key, value in self.__dict__.items()] + return "{}({})".format(self.__class__.__name__, ", ".join(L)) + + def __eq__(self, other): + return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ + + def __ne__(self, other): + return not (self == other) + + +all_structs.append(cache_file_metadata_args) +cache_file_metadata_args.thrift_spec = ( + None, # 0 + ( + 1, + TType.STRUCT, + "req", + [CacheFileMetadataRequest, None], + None, + ), # 1 +) + + +class cache_file_metadata_result: + """ + Attributes: + - success + + """ + + def __init__( + self, + success=None, + ): + self.success = success + + def read(self, iprot): + if ( + iprot._fast_decode is not None + and isinstance(iprot.trans, TTransport.CReadableTransport) + and self.thrift_spec is not None + ): + iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec]) + return + iprot.readStructBegin() + while True: + (fname, ftype, fid) = iprot.readFieldBegin() + if ftype == TType.STOP: + break + if fid == 0: + if ftype == TType.STRUCT: + self.success = CacheFileMetadataResult() + self.success.read(iprot) + else: + iprot.skip(ftype) + else: + iprot.skip(ftype) + iprot.readFieldEnd() + iprot.readStructEnd() + + def write(self, oprot): + if oprot._fast_encode is not None and self.thrift_spec is not None: + oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec])) + return + oprot.writeStructBegin("cache_file_metadata_result") + if self.success is not None: + oprot.writeFieldBegin("success", TType.STRUCT, 0) + self.success.write(oprot) + oprot.writeFieldEnd() + oprot.writeFieldStop() + oprot.writeStructEnd() + + def validate(self): + return + + def __repr__(self): + L = ["{}={!r}".format(key, value) for key, value in self.__dict__.items()] + return "{}({})".format(self.__class__.__name__, ", ".join(L)) + + def __eq__(self, other): + return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ + + def __ne__(self, other): + return not (self == other) + + +all_structs.append(cache_file_metadata_result) +cache_file_metadata_result.thrift_spec = ( + ( + 0, + TType.STRUCT, + "success", + [CacheFileMetadataResult, None], + None, + ), # 0 +) + + +class get_metastore_db_uuid_args: + def read(self, iprot): + if ( + iprot._fast_decode is not None + and isinstance(iprot.trans, TTransport.CReadableTransport) + and self.thrift_spec is not None + ): + iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec]) + return + iprot.readStructBegin() + while True: + (fname, ftype, fid) = iprot.readFieldBegin() + if ftype == TType.STOP: + break + else: + iprot.skip(ftype) + iprot.readFieldEnd() + iprot.readStructEnd() + + def write(self, oprot): + if oprot._fast_encode is not None and self.thrift_spec is not None: + oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec])) + return + oprot.writeStructBegin("get_metastore_db_uuid_args") + oprot.writeFieldStop() + oprot.writeStructEnd() + + def validate(self): + return + + def __repr__(self): + L = ["{}={!r}".format(key, value) for key, value in self.__dict__.items()] + return "{}({})".format(self.__class__.__name__, ", ".join(L)) + + def __eq__(self, other): + return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ + + def __ne__(self, other): + return not (self == other) + + +all_structs.append(get_metastore_db_uuid_args) +get_metastore_db_uuid_args.thrift_spec = () + + +class get_metastore_db_uuid_result: + """ + Attributes: + - success + - o1 + + """ + + def __init__( + self, + success=None, + o1=None, + ): + self.success = success + self.o1 = o1 + + def read(self, iprot): + if ( + iprot._fast_decode is not None + and isinstance(iprot.trans, TTransport.CReadableTransport) + and self.thrift_spec is not None + ): + iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec]) + return + iprot.readStructBegin() + while True: + (fname, ftype, fid) = iprot.readFieldBegin() + if ftype == TType.STOP: + break + if fid == 0: + if ftype == TType.STRING: + self.success = ( + iprot.readString().decode("utf-8", errors="replace") if sys.version_info[0] == 2 else iprot.readString() + ) + else: + iprot.skip(ftype) + elif fid == 1: + if ftype == TType.STRUCT: + self.o1 = MetaException.read(iprot) + else: + iprot.skip(ftype) + else: + iprot.skip(ftype) + iprot.readFieldEnd() + iprot.readStructEnd() + + def write(self, oprot): + if oprot._fast_encode is not None and self.thrift_spec is not None: + oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec])) + return + oprot.writeStructBegin("get_metastore_db_uuid_result") + if self.success is not None: + oprot.writeFieldBegin("success", TType.STRING, 0) + oprot.writeString(self.success.encode("utf-8") if sys.version_info[0] == 2 else self.success) + oprot.writeFieldEnd() + if self.o1 is not None: + oprot.writeFieldBegin("o1", TType.STRUCT, 1) + self.o1.write(oprot) + oprot.writeFieldEnd() + oprot.writeFieldStop() + oprot.writeStructEnd() + + def validate(self): + return + + def __repr__(self): + L = ["{}={!r}".format(key, value) for key, value in self.__dict__.items()] + return "{}({})".format(self.__class__.__name__, ", ".join(L)) + + def __eq__(self, other): + return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ + + def __ne__(self, other): + return not (self == other) + + +all_structs.append(get_metastore_db_uuid_result) +get_metastore_db_uuid_result.thrift_spec = ( + ( + 0, + TType.STRING, + "success", + "UTF8", + None, + ), # 0 + ( + 1, + TType.STRUCT, + "o1", + [MetaException, None], + None, + ), # 1 +) + + +class create_resource_plan_args: + """ + Attributes: + - request + + """ + + def __init__( + self, + request=None, + ): + self.request = request + + def read(self, iprot): + if ( + iprot._fast_decode is not None + and isinstance(iprot.trans, TTransport.CReadableTransport) + and self.thrift_spec is not None + ): + iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec]) + return + iprot.readStructBegin() + while True: + (fname, ftype, fid) = iprot.readFieldBegin() + if ftype == TType.STOP: + break + if fid == 1: + if ftype == TType.STRUCT: + self.request = WMCreateResourcePlanRequest() + self.request.read(iprot) + else: + iprot.skip(ftype) + else: + iprot.skip(ftype) + iprot.readFieldEnd() + iprot.readStructEnd() + + def write(self, oprot): + if oprot._fast_encode is not None and self.thrift_spec is not None: + oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec])) + return + oprot.writeStructBegin("create_resource_plan_args") + if self.request is not None: + oprot.writeFieldBegin("request", TType.STRUCT, 1) + self.request.write(oprot) + oprot.writeFieldEnd() + oprot.writeFieldStop() + oprot.writeStructEnd() + + def validate(self): + return + + def __repr__(self): + L = ["{}={!r}".format(key, value) for key, value in self.__dict__.items()] + return "{}({})".format(self.__class__.__name__, ", ".join(L)) + + def __eq__(self, other): + return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ + + def __ne__(self, other): + return not (self == other) + + +all_structs.append(create_resource_plan_args) +create_resource_plan_args.thrift_spec = ( + None, # 0 + ( + 1, + TType.STRUCT, + "request", + [WMCreateResourcePlanRequest, None], + None, + ), # 1 +) + + +class create_resource_plan_result: + """ + Attributes: + - success + - o1 + - o2 + - o3 + + """ + + def __init__( + self, + success=None, + o1=None, + o2=None, + o3=None, + ): + self.success = success + self.o1 = o1 + self.o2 = o2 + self.o3 = o3 + + def read(self, iprot): + if ( + iprot._fast_decode is not None + and isinstance(iprot.trans, TTransport.CReadableTransport) + and self.thrift_spec is not None + ): + iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec]) + return + iprot.readStructBegin() + while True: + (fname, ftype, fid) = iprot.readFieldBegin() + if ftype == TType.STOP: + break + if fid == 0: + if ftype == TType.STRUCT: + self.success = WMCreateResourcePlanResponse() + self.success.read(iprot) + else: + iprot.skip(ftype) + elif fid == 1: + if ftype == TType.STRUCT: + self.o1 = AlreadyExistsException.read(iprot) + else: + iprot.skip(ftype) + elif fid == 2: + if ftype == TType.STRUCT: + self.o2 = InvalidObjectException.read(iprot) + else: + iprot.skip(ftype) + elif fid == 3: + if ftype == TType.STRUCT: + self.o3 = MetaException.read(iprot) + else: + iprot.skip(ftype) + else: + iprot.skip(ftype) + iprot.readFieldEnd() + iprot.readStructEnd() + + def write(self, oprot): + if oprot._fast_encode is not None and self.thrift_spec is not None: + oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec])) + return + oprot.writeStructBegin("create_resource_plan_result") + if self.success is not None: + oprot.writeFieldBegin("success", TType.STRUCT, 0) + self.success.write(oprot) + oprot.writeFieldEnd() + if self.o1 is not None: + oprot.writeFieldBegin("o1", TType.STRUCT, 1) + self.o1.write(oprot) + oprot.writeFieldEnd() + if self.o2 is not None: + oprot.writeFieldBegin("o2", TType.STRUCT, 2) + self.o2.write(oprot) + oprot.writeFieldEnd() + if self.o3 is not None: + oprot.writeFieldBegin("o3", TType.STRUCT, 3) + self.o3.write(oprot) + oprot.writeFieldEnd() + oprot.writeFieldStop() + oprot.writeStructEnd() + + def validate(self): + return + + def __repr__(self): + L = ["{}={!r}".format(key, value) for key, value in self.__dict__.items()] + return "{}({})".format(self.__class__.__name__, ", ".join(L)) + + def __eq__(self, other): + return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ + + def __ne__(self, other): + return not (self == other) + + +all_structs.append(create_resource_plan_result) +create_resource_plan_result.thrift_spec = ( + ( + 0, + TType.STRUCT, + "success", + [WMCreateResourcePlanResponse, None], + None, + ), # 0 + ( + 1, + TType.STRUCT, + "o1", + [AlreadyExistsException, None], + None, + ), # 1 + ( + 2, + TType.STRUCT, + "o2", + [InvalidObjectException, None], + None, + ), # 2 + ( + 3, + TType.STRUCT, + "o3", + [MetaException, None], + None, + ), # 3 +) + + +class get_resource_plan_args: + """ + Attributes: + - request + + """ + + def __init__( + self, + request=None, + ): + self.request = request + + def read(self, iprot): + if ( + iprot._fast_decode is not None + and isinstance(iprot.trans, TTransport.CReadableTransport) + and self.thrift_spec is not None + ): + iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec]) + return + iprot.readStructBegin() + while True: + (fname, ftype, fid) = iprot.readFieldBegin() + if ftype == TType.STOP: + break + if fid == 1: + if ftype == TType.STRUCT: + self.request = WMGetResourcePlanRequest() + self.request.read(iprot) + else: + iprot.skip(ftype) + else: + iprot.skip(ftype) + iprot.readFieldEnd() + iprot.readStructEnd() + + def write(self, oprot): + if oprot._fast_encode is not None and self.thrift_spec is not None: + oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec])) + return + oprot.writeStructBegin("get_resource_plan_args") + if self.request is not None: + oprot.writeFieldBegin("request", TType.STRUCT, 1) + self.request.write(oprot) + oprot.writeFieldEnd() + oprot.writeFieldStop() + oprot.writeStructEnd() + + def validate(self): + return + + def __repr__(self): + L = ["{}={!r}".format(key, value) for key, value in self.__dict__.items()] + return "{}({})".format(self.__class__.__name__, ", ".join(L)) + + def __eq__(self, other): + return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ + + def __ne__(self, other): + return not (self == other) + + +all_structs.append(get_resource_plan_args) +get_resource_plan_args.thrift_spec = ( + None, # 0 + ( + 1, + TType.STRUCT, + "request", + [WMGetResourcePlanRequest, None], + None, + ), # 1 +) + + +class get_resource_plan_result: + """ + Attributes: + - success + - o1 + - o2 + + """ + + def __init__( + self, + success=None, + o1=None, + o2=None, + ): + self.success = success + self.o1 = o1 + self.o2 = o2 + + def read(self, iprot): + if ( + iprot._fast_decode is not None + and isinstance(iprot.trans, TTransport.CReadableTransport) + and self.thrift_spec is not None + ): + iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec]) + return + iprot.readStructBegin() + while True: + (fname, ftype, fid) = iprot.readFieldBegin() + if ftype == TType.STOP: + break + if fid == 0: + if ftype == TType.STRUCT: + self.success = WMGetResourcePlanResponse() + self.success.read(iprot) + else: + iprot.skip(ftype) + elif fid == 1: + if ftype == TType.STRUCT: + self.o1 = NoSuchObjectException.read(iprot) + else: + iprot.skip(ftype) + elif fid == 2: + if ftype == TType.STRUCT: + self.o2 = MetaException.read(iprot) + else: + iprot.skip(ftype) + else: + iprot.skip(ftype) + iprot.readFieldEnd() + iprot.readStructEnd() + + def write(self, oprot): + if oprot._fast_encode is not None and self.thrift_spec is not None: + oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec])) + return + oprot.writeStructBegin("get_resource_plan_result") + if self.success is not None: + oprot.writeFieldBegin("success", TType.STRUCT, 0) + self.success.write(oprot) + oprot.writeFieldEnd() + if self.o1 is not None: + oprot.writeFieldBegin("o1", TType.STRUCT, 1) + self.o1.write(oprot) + oprot.writeFieldEnd() + if self.o2 is not None: + oprot.writeFieldBegin("o2", TType.STRUCT, 2) + self.o2.write(oprot) + oprot.writeFieldEnd() + oprot.writeFieldStop() + oprot.writeStructEnd() + + def validate(self): + return + + def __repr__(self): + L = ["{}={!r}".format(key, value) for key, value in self.__dict__.items()] + return "{}({})".format(self.__class__.__name__, ", ".join(L)) + + def __eq__(self, other): + return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ + + def __ne__(self, other): + return not (self == other) + + +all_structs.append(get_resource_plan_result) +get_resource_plan_result.thrift_spec = ( + ( + 0, + TType.STRUCT, + "success", + [WMGetResourcePlanResponse, None], + None, + ), # 0 + ( + 1, + TType.STRUCT, + "o1", + [NoSuchObjectException, None], + None, + ), # 1 + ( + 2, + TType.STRUCT, + "o2", + [MetaException, None], + None, + ), # 2 +) + + +class get_active_resource_plan_args: + """ + Attributes: + - request + + """ + + def __init__( + self, + request=None, + ): + self.request = request + + def read(self, iprot): + if ( + iprot._fast_decode is not None + and isinstance(iprot.trans, TTransport.CReadableTransport) + and self.thrift_spec is not None + ): + iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec]) + return + iprot.readStructBegin() + while True: + (fname, ftype, fid) = iprot.readFieldBegin() + if ftype == TType.STOP: + break + if fid == 1: + if ftype == TType.STRUCT: + self.request = WMGetActiveResourcePlanRequest() + self.request.read(iprot) + else: + iprot.skip(ftype) + else: + iprot.skip(ftype) + iprot.readFieldEnd() + iprot.readStructEnd() + + def write(self, oprot): + if oprot._fast_encode is not None and self.thrift_spec is not None: + oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec])) + return + oprot.writeStructBegin("get_active_resource_plan_args") + if self.request is not None: + oprot.writeFieldBegin("request", TType.STRUCT, 1) + self.request.write(oprot) + oprot.writeFieldEnd() + oprot.writeFieldStop() + oprot.writeStructEnd() + + def validate(self): + return + + def __repr__(self): + L = ["{}={!r}".format(key, value) for key, value in self.__dict__.items()] + return "{}({})".format(self.__class__.__name__, ", ".join(L)) + + def __eq__(self, other): + return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ + + def __ne__(self, other): + return not (self == other) + + +all_structs.append(get_active_resource_plan_args) +get_active_resource_plan_args.thrift_spec = ( + None, # 0 + ( + 1, + TType.STRUCT, + "request", + [WMGetActiveResourcePlanRequest, None], + None, + ), # 1 +) + + +class get_active_resource_plan_result: + """ + Attributes: + - success + - o2 + + """ + + def __init__( + self, + success=None, + o2=None, + ): + self.success = success + self.o2 = o2 + + def read(self, iprot): + if ( + iprot._fast_decode is not None + and isinstance(iprot.trans, TTransport.CReadableTransport) + and self.thrift_spec is not None + ): + iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec]) + return + iprot.readStructBegin() + while True: + (fname, ftype, fid) = iprot.readFieldBegin() + if ftype == TType.STOP: + break + if fid == 0: + if ftype == TType.STRUCT: + self.success = WMGetActiveResourcePlanResponse() + self.success.read(iprot) + else: + iprot.skip(ftype) + elif fid == 1: + if ftype == TType.STRUCT: + self.o2 = MetaException.read(iprot) + else: + iprot.skip(ftype) + else: + iprot.skip(ftype) + iprot.readFieldEnd() + iprot.readStructEnd() + + def write(self, oprot): + if oprot._fast_encode is not None and self.thrift_spec is not None: + oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec])) + return + oprot.writeStructBegin("get_active_resource_plan_result") + if self.success is not None: + oprot.writeFieldBegin("success", TType.STRUCT, 0) + self.success.write(oprot) + oprot.writeFieldEnd() + if self.o2 is not None: + oprot.writeFieldBegin("o2", TType.STRUCT, 1) + self.o2.write(oprot) + oprot.writeFieldEnd() + oprot.writeFieldStop() + oprot.writeStructEnd() + + def validate(self): + return + + def __repr__(self): + L = ["{}={!r}".format(key, value) for key, value in self.__dict__.items()] + return "{}({})".format(self.__class__.__name__, ", ".join(L)) + + def __eq__(self, other): + return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ + + def __ne__(self, other): + return not (self == other) + + +all_structs.append(get_active_resource_plan_result) +get_active_resource_plan_result.thrift_spec = ( + ( + 0, + TType.STRUCT, + "success", + [WMGetActiveResourcePlanResponse, None], + None, + ), # 0 + ( + 1, + TType.STRUCT, + "o2", + [MetaException, None], + None, + ), # 1 +) + + +class get_all_resource_plans_args: + """ + Attributes: + - request + + """ + + def __init__( + self, + request=None, + ): + self.request = request + + def read(self, iprot): + if ( + iprot._fast_decode is not None + and isinstance(iprot.trans, TTransport.CReadableTransport) + and self.thrift_spec is not None + ): + iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec]) + return + iprot.readStructBegin() + while True: + (fname, ftype, fid) = iprot.readFieldBegin() + if ftype == TType.STOP: + break + if fid == 1: + if ftype == TType.STRUCT: + self.request = WMGetAllResourcePlanRequest() + self.request.read(iprot) + else: + iprot.skip(ftype) + else: + iprot.skip(ftype) + iprot.readFieldEnd() + iprot.readStructEnd() + + def write(self, oprot): + if oprot._fast_encode is not None and self.thrift_spec is not None: + oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec])) + return + oprot.writeStructBegin("get_all_resource_plans_args") + if self.request is not None: + oprot.writeFieldBegin("request", TType.STRUCT, 1) + self.request.write(oprot) + oprot.writeFieldEnd() + oprot.writeFieldStop() + oprot.writeStructEnd() + + def validate(self): + return + + def __repr__(self): + L = ["{}={!r}".format(key, value) for key, value in self.__dict__.items()] + return "{}({})".format(self.__class__.__name__, ", ".join(L)) + + def __eq__(self, other): + return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ + + def __ne__(self, other): + return not (self == other) + + +all_structs.append(get_all_resource_plans_args) +get_all_resource_plans_args.thrift_spec = ( + None, # 0 + ( + 1, + TType.STRUCT, + "request", + [WMGetAllResourcePlanRequest, None], + None, + ), # 1 +) + + +class get_all_resource_plans_result: + """ + Attributes: + - success + - o1 + + """ + + def __init__( + self, + success=None, + o1=None, + ): + self.success = success + self.o1 = o1 + + def read(self, iprot): + if ( + iprot._fast_decode is not None + and isinstance(iprot.trans, TTransport.CReadableTransport) + and self.thrift_spec is not None + ): + iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec]) + return + iprot.readStructBegin() + while True: + (fname, ftype, fid) = iprot.readFieldBegin() + if ftype == TType.STOP: + break + if fid == 0: + if ftype == TType.STRUCT: + self.success = WMGetAllResourcePlanResponse() + self.success.read(iprot) + else: + iprot.skip(ftype) + elif fid == 1: + if ftype == TType.STRUCT: + self.o1 = MetaException.read(iprot) + else: + iprot.skip(ftype) + else: + iprot.skip(ftype) + iprot.readFieldEnd() + iprot.readStructEnd() + + def write(self, oprot): + if oprot._fast_encode is not None and self.thrift_spec is not None: + oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec])) + return + oprot.writeStructBegin("get_all_resource_plans_result") + if self.success is not None: + oprot.writeFieldBegin("success", TType.STRUCT, 0) + self.success.write(oprot) + oprot.writeFieldEnd() + if self.o1 is not None: + oprot.writeFieldBegin("o1", TType.STRUCT, 1) + self.o1.write(oprot) + oprot.writeFieldEnd() + oprot.writeFieldStop() + oprot.writeStructEnd() + + def validate(self): + return + + def __repr__(self): + L = ["{}={!r}".format(key, value) for key, value in self.__dict__.items()] + return "{}({})".format(self.__class__.__name__, ", ".join(L)) + + def __eq__(self, other): + return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ + + def __ne__(self, other): + return not (self == other) + + +all_structs.append(get_all_resource_plans_result) +get_all_resource_plans_result.thrift_spec = ( + ( + 0, + TType.STRUCT, + "success", + [WMGetAllResourcePlanResponse, None], + None, + ), # 0 + ( + 1, + TType.STRUCT, + "o1", + [MetaException, None], + None, + ), # 1 +) + + +class alter_resource_plan_args: + """ + Attributes: + - request + + """ + + def __init__( + self, + request=None, + ): + self.request = request + + def read(self, iprot): + if ( + iprot._fast_decode is not None + and isinstance(iprot.trans, TTransport.CReadableTransport) + and self.thrift_spec is not None + ): + iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec]) + return + iprot.readStructBegin() + while True: + (fname, ftype, fid) = iprot.readFieldBegin() + if ftype == TType.STOP: + break + if fid == 1: + if ftype == TType.STRUCT: + self.request = WMAlterResourcePlanRequest() + self.request.read(iprot) + else: + iprot.skip(ftype) + else: + iprot.skip(ftype) + iprot.readFieldEnd() + iprot.readStructEnd() + + def write(self, oprot): + if oprot._fast_encode is not None and self.thrift_spec is not None: + oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec])) + return + oprot.writeStructBegin("alter_resource_plan_args") + if self.request is not None: + oprot.writeFieldBegin("request", TType.STRUCT, 1) + self.request.write(oprot) + oprot.writeFieldEnd() + oprot.writeFieldStop() + oprot.writeStructEnd() + + def validate(self): + return + + def __repr__(self): + L = ["{}={!r}".format(key, value) for key, value in self.__dict__.items()] + return "{}({})".format(self.__class__.__name__, ", ".join(L)) + + def __eq__(self, other): + return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ + + def __ne__(self, other): + return not (self == other) + + +all_structs.append(alter_resource_plan_args) +alter_resource_plan_args.thrift_spec = ( + None, # 0 + ( + 1, + TType.STRUCT, + "request", + [WMAlterResourcePlanRequest, None], + None, + ), # 1 +) + + +class alter_resource_plan_result: + """ + Attributes: + - success + - o1 + - o2 + - o3 + + """ + + def __init__( + self, + success=None, + o1=None, + o2=None, + o3=None, + ): + self.success = success + self.o1 = o1 + self.o2 = o2 + self.o3 = o3 + + def read(self, iprot): + if ( + iprot._fast_decode is not None + and isinstance(iprot.trans, TTransport.CReadableTransport) + and self.thrift_spec is not None + ): + iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec]) + return + iprot.readStructBegin() + while True: + (fname, ftype, fid) = iprot.readFieldBegin() + if ftype == TType.STOP: + break + if fid == 0: + if ftype == TType.STRUCT: + self.success = WMAlterResourcePlanResponse() + self.success.read(iprot) + else: + iprot.skip(ftype) + elif fid == 1: + if ftype == TType.STRUCT: + self.o1 = NoSuchObjectException.read(iprot) + else: + iprot.skip(ftype) + elif fid == 2: + if ftype == TType.STRUCT: + self.o2 = InvalidOperationException.read(iprot) + else: + iprot.skip(ftype) + elif fid == 3: + if ftype == TType.STRUCT: + self.o3 = MetaException.read(iprot) + else: + iprot.skip(ftype) + else: + iprot.skip(ftype) + iprot.readFieldEnd() + iprot.readStructEnd() + + def write(self, oprot): + if oprot._fast_encode is not None and self.thrift_spec is not None: + oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec])) + return + oprot.writeStructBegin("alter_resource_plan_result") + if self.success is not None: + oprot.writeFieldBegin("success", TType.STRUCT, 0) + self.success.write(oprot) + oprot.writeFieldEnd() + if self.o1 is not None: + oprot.writeFieldBegin("o1", TType.STRUCT, 1) + self.o1.write(oprot) + oprot.writeFieldEnd() + if self.o2 is not None: + oprot.writeFieldBegin("o2", TType.STRUCT, 2) + self.o2.write(oprot) + oprot.writeFieldEnd() + if self.o3 is not None: + oprot.writeFieldBegin("o3", TType.STRUCT, 3) + self.o3.write(oprot) + oprot.writeFieldEnd() + oprot.writeFieldStop() + oprot.writeStructEnd() + + def validate(self): + return + + def __repr__(self): + L = ["{}={!r}".format(key, value) for key, value in self.__dict__.items()] + return "{}({})".format(self.__class__.__name__, ", ".join(L)) + + def __eq__(self, other): + return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ + + def __ne__(self, other): + return not (self == other) + + +all_structs.append(alter_resource_plan_result) +alter_resource_plan_result.thrift_spec = ( + ( + 0, + TType.STRUCT, + "success", + [WMAlterResourcePlanResponse, None], + None, + ), # 0 + ( + 1, + TType.STRUCT, + "o1", + [NoSuchObjectException, None], + None, + ), # 1 + ( + 2, + TType.STRUCT, + "o2", + [InvalidOperationException, None], + None, + ), # 2 + ( + 3, + TType.STRUCT, + "o3", + [MetaException, None], + None, + ), # 3 +) + + +class validate_resource_plan_args: + """ + Attributes: + - request + + """ + + def __init__( + self, + request=None, + ): + self.request = request + + def read(self, iprot): + if ( + iprot._fast_decode is not None + and isinstance(iprot.trans, TTransport.CReadableTransport) + and self.thrift_spec is not None + ): + iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec]) + return + iprot.readStructBegin() + while True: + (fname, ftype, fid) = iprot.readFieldBegin() + if ftype == TType.STOP: + break + if fid == 1: + if ftype == TType.STRUCT: + self.request = WMValidateResourcePlanRequest() + self.request.read(iprot) + else: + iprot.skip(ftype) + else: + iprot.skip(ftype) + iprot.readFieldEnd() + iprot.readStructEnd() + + def write(self, oprot): + if oprot._fast_encode is not None and self.thrift_spec is not None: + oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec])) + return + oprot.writeStructBegin("validate_resource_plan_args") + if self.request is not None: + oprot.writeFieldBegin("request", TType.STRUCT, 1) + self.request.write(oprot) + oprot.writeFieldEnd() + oprot.writeFieldStop() + oprot.writeStructEnd() + + def validate(self): + return + + def __repr__(self): + L = ["{}={!r}".format(key, value) for key, value in self.__dict__.items()] + return "{}({})".format(self.__class__.__name__, ", ".join(L)) + + def __eq__(self, other): + return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ + + def __ne__(self, other): + return not (self == other) + + +all_structs.append(validate_resource_plan_args) +validate_resource_plan_args.thrift_spec = ( + None, # 0 + ( + 1, + TType.STRUCT, + "request", + [WMValidateResourcePlanRequest, None], + None, + ), # 1 +) + + +class validate_resource_plan_result: + """ + Attributes: + - success + - o1 + - o2 + + """ + + def __init__( + self, + success=None, + o1=None, + o2=None, + ): + self.success = success + self.o1 = o1 + self.o2 = o2 + + def read(self, iprot): + if ( + iprot._fast_decode is not None + and isinstance(iprot.trans, TTransport.CReadableTransport) + and self.thrift_spec is not None + ): + iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec]) + return + iprot.readStructBegin() + while True: + (fname, ftype, fid) = iprot.readFieldBegin() + if ftype == TType.STOP: + break + if fid == 0: + if ftype == TType.STRUCT: + self.success = WMValidateResourcePlanResponse() + self.success.read(iprot) + else: + iprot.skip(ftype) + elif fid == 1: + if ftype == TType.STRUCT: + self.o1 = NoSuchObjectException.read(iprot) + else: + iprot.skip(ftype) + elif fid == 2: + if ftype == TType.STRUCT: + self.o2 = MetaException.read(iprot) + else: + iprot.skip(ftype) + else: + iprot.skip(ftype) + iprot.readFieldEnd() + iprot.readStructEnd() + + def write(self, oprot): + if oprot._fast_encode is not None and self.thrift_spec is not None: + oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec])) + return + oprot.writeStructBegin("validate_resource_plan_result") + if self.success is not None: + oprot.writeFieldBegin("success", TType.STRUCT, 0) + self.success.write(oprot) + oprot.writeFieldEnd() + if self.o1 is not None: + oprot.writeFieldBegin("o1", TType.STRUCT, 1) + self.o1.write(oprot) + oprot.writeFieldEnd() + if self.o2 is not None: + oprot.writeFieldBegin("o2", TType.STRUCT, 2) + self.o2.write(oprot) + oprot.writeFieldEnd() + oprot.writeFieldStop() + oprot.writeStructEnd() + + def validate(self): + return + + def __repr__(self): + L = ["{}={!r}".format(key, value) for key, value in self.__dict__.items()] + return "{}({})".format(self.__class__.__name__, ", ".join(L)) + + def __eq__(self, other): + return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ + + def __ne__(self, other): + return not (self == other) + + +all_structs.append(validate_resource_plan_result) +validate_resource_plan_result.thrift_spec = ( + ( + 0, + TType.STRUCT, + "success", + [WMValidateResourcePlanResponse, None], + None, + ), # 0 + ( + 1, + TType.STRUCT, + "o1", + [NoSuchObjectException, None], + None, + ), # 1 + ( + 2, + TType.STRUCT, + "o2", + [MetaException, None], + None, + ), # 2 +) + + +class drop_resource_plan_args: + """ + Attributes: + - request + + """ + + def __init__( + self, + request=None, + ): + self.request = request + + def read(self, iprot): + if ( + iprot._fast_decode is not None + and isinstance(iprot.trans, TTransport.CReadableTransport) + and self.thrift_spec is not None + ): + iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec]) + return + iprot.readStructBegin() + while True: + (fname, ftype, fid) = iprot.readFieldBegin() + if ftype == TType.STOP: + break + if fid == 1: + if ftype == TType.STRUCT: + self.request = WMDropResourcePlanRequest() + self.request.read(iprot) + else: + iprot.skip(ftype) + else: + iprot.skip(ftype) + iprot.readFieldEnd() + iprot.readStructEnd() + + def write(self, oprot): + if oprot._fast_encode is not None and self.thrift_spec is not None: + oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec])) + return + oprot.writeStructBegin("drop_resource_plan_args") + if self.request is not None: + oprot.writeFieldBegin("request", TType.STRUCT, 1) + self.request.write(oprot) + oprot.writeFieldEnd() + oprot.writeFieldStop() + oprot.writeStructEnd() + + def validate(self): + return + + def __repr__(self): + L = ["{}={!r}".format(key, value) for key, value in self.__dict__.items()] + return "{}({})".format(self.__class__.__name__, ", ".join(L)) + + def __eq__(self, other): + return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ + + def __ne__(self, other): + return not (self == other) + + +all_structs.append(drop_resource_plan_args) +drop_resource_plan_args.thrift_spec = ( + None, # 0 + ( + 1, + TType.STRUCT, + "request", + [WMDropResourcePlanRequest, None], + None, + ), # 1 +) + + +class drop_resource_plan_result: + """ + Attributes: + - success + - o1 + - o2 + - o3 + + """ + + def __init__( + self, + success=None, + o1=None, + o2=None, + o3=None, + ): + self.success = success + self.o1 = o1 + self.o2 = o2 + self.o3 = o3 + + def read(self, iprot): + if ( + iprot._fast_decode is not None + and isinstance(iprot.trans, TTransport.CReadableTransport) + and self.thrift_spec is not None + ): + iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec]) + return + iprot.readStructBegin() + while True: + (fname, ftype, fid) = iprot.readFieldBegin() + if ftype == TType.STOP: + break + if fid == 0: + if ftype == TType.STRUCT: + self.success = WMDropResourcePlanResponse() + self.success.read(iprot) + else: + iprot.skip(ftype) + elif fid == 1: + if ftype == TType.STRUCT: + self.o1 = NoSuchObjectException.read(iprot) + else: + iprot.skip(ftype) + elif fid == 2: + if ftype == TType.STRUCT: + self.o2 = InvalidOperationException.read(iprot) + else: + iprot.skip(ftype) + elif fid == 3: + if ftype == TType.STRUCT: + self.o3 = MetaException.read(iprot) + else: + iprot.skip(ftype) + else: + iprot.skip(ftype) + iprot.readFieldEnd() + iprot.readStructEnd() + + def write(self, oprot): + if oprot._fast_encode is not None and self.thrift_spec is not None: + oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec])) + return + oprot.writeStructBegin("drop_resource_plan_result") + if self.success is not None: + oprot.writeFieldBegin("success", TType.STRUCT, 0) + self.success.write(oprot) + oprot.writeFieldEnd() + if self.o1 is not None: + oprot.writeFieldBegin("o1", TType.STRUCT, 1) + self.o1.write(oprot) + oprot.writeFieldEnd() + if self.o2 is not None: + oprot.writeFieldBegin("o2", TType.STRUCT, 2) + self.o2.write(oprot) + oprot.writeFieldEnd() + if self.o3 is not None: + oprot.writeFieldBegin("o3", TType.STRUCT, 3) + self.o3.write(oprot) + oprot.writeFieldEnd() + oprot.writeFieldStop() + oprot.writeStructEnd() + + def validate(self): + return + + def __repr__(self): + L = ["{}={!r}".format(key, value) for key, value in self.__dict__.items()] + return "{}({})".format(self.__class__.__name__, ", ".join(L)) + + def __eq__(self, other): + return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ + + def __ne__(self, other): + return not (self == other) + + +all_structs.append(drop_resource_plan_result) +drop_resource_plan_result.thrift_spec = ( + ( + 0, + TType.STRUCT, + "success", + [WMDropResourcePlanResponse, None], + None, + ), # 0 + ( + 1, + TType.STRUCT, + "o1", + [NoSuchObjectException, None], + None, + ), # 1 + ( + 2, + TType.STRUCT, + "o2", + [InvalidOperationException, None], + None, + ), # 2 + ( + 3, + TType.STRUCT, + "o3", + [MetaException, None], + None, + ), # 3 +) + + +class create_wm_trigger_args: + """ + Attributes: + - request + + """ + + def __init__( + self, + request=None, + ): + self.request = request + + def read(self, iprot): + if ( + iprot._fast_decode is not None + and isinstance(iprot.trans, TTransport.CReadableTransport) + and self.thrift_spec is not None + ): + iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec]) + return + iprot.readStructBegin() + while True: + (fname, ftype, fid) = iprot.readFieldBegin() + if ftype == TType.STOP: + break + if fid == 1: + if ftype == TType.STRUCT: + self.request = WMCreateTriggerRequest() + self.request.read(iprot) + else: + iprot.skip(ftype) + else: + iprot.skip(ftype) + iprot.readFieldEnd() + iprot.readStructEnd() + + def write(self, oprot): + if oprot._fast_encode is not None and self.thrift_spec is not None: + oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec])) + return + oprot.writeStructBegin("create_wm_trigger_args") + if self.request is not None: + oprot.writeFieldBegin("request", TType.STRUCT, 1) + self.request.write(oprot) + oprot.writeFieldEnd() + oprot.writeFieldStop() + oprot.writeStructEnd() + + def validate(self): + return + + def __repr__(self): + L = ["{}={!r}".format(key, value) for key, value in self.__dict__.items()] + return "{}({})".format(self.__class__.__name__, ", ".join(L)) + + def __eq__(self, other): + return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ + + def __ne__(self, other): + return not (self == other) + + +all_structs.append(create_wm_trigger_args) +create_wm_trigger_args.thrift_spec = ( + None, # 0 + ( + 1, + TType.STRUCT, + "request", + [WMCreateTriggerRequest, None], + None, + ), # 1 +) + + +class create_wm_trigger_result: + """ + Attributes: + - success + - o1 + - o2 + - o3 + - o4 + + """ + + def __init__( + self, + success=None, + o1=None, + o2=None, + o3=None, + o4=None, + ): + self.success = success + self.o1 = o1 + self.o2 = o2 + self.o3 = o3 + self.o4 = o4 + + def read(self, iprot): + if ( + iprot._fast_decode is not None + and isinstance(iprot.trans, TTransport.CReadableTransport) + and self.thrift_spec is not None + ): + iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec]) + return + iprot.readStructBegin() + while True: + (fname, ftype, fid) = iprot.readFieldBegin() + if ftype == TType.STOP: + break + if fid == 0: + if ftype == TType.STRUCT: + self.success = WMCreateTriggerResponse() + self.success.read(iprot) + else: + iprot.skip(ftype) + elif fid == 1: + if ftype == TType.STRUCT: + self.o1 = AlreadyExistsException.read(iprot) + else: + iprot.skip(ftype) + elif fid == 2: + if ftype == TType.STRUCT: + self.o2 = NoSuchObjectException.read(iprot) + else: + iprot.skip(ftype) + elif fid == 3: + if ftype == TType.STRUCT: + self.o3 = InvalidObjectException.read(iprot) + else: + iprot.skip(ftype) + elif fid == 4: + if ftype == TType.STRUCT: + self.o4 = MetaException.read(iprot) + else: + iprot.skip(ftype) + else: + iprot.skip(ftype) + iprot.readFieldEnd() + iprot.readStructEnd() + + def write(self, oprot): + if oprot._fast_encode is not None and self.thrift_spec is not None: + oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec])) + return + oprot.writeStructBegin("create_wm_trigger_result") + if self.success is not None: + oprot.writeFieldBegin("success", TType.STRUCT, 0) + self.success.write(oprot) + oprot.writeFieldEnd() + if self.o1 is not None: + oprot.writeFieldBegin("o1", TType.STRUCT, 1) + self.o1.write(oprot) + oprot.writeFieldEnd() + if self.o2 is not None: + oprot.writeFieldBegin("o2", TType.STRUCT, 2) + self.o2.write(oprot) + oprot.writeFieldEnd() + if self.o3 is not None: + oprot.writeFieldBegin("o3", TType.STRUCT, 3) + self.o3.write(oprot) + oprot.writeFieldEnd() + if self.o4 is not None: + oprot.writeFieldBegin("o4", TType.STRUCT, 4) + self.o4.write(oprot) + oprot.writeFieldEnd() + oprot.writeFieldStop() + oprot.writeStructEnd() + + def validate(self): + return + + def __repr__(self): + L = ["{}={!r}".format(key, value) for key, value in self.__dict__.items()] + return "{}({})".format(self.__class__.__name__, ", ".join(L)) + + def __eq__(self, other): + return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ + + def __ne__(self, other): + return not (self == other) + + +all_structs.append(create_wm_trigger_result) +create_wm_trigger_result.thrift_spec = ( + ( + 0, + TType.STRUCT, + "success", + [WMCreateTriggerResponse, None], + None, + ), # 0 + ( + 1, + TType.STRUCT, + "o1", + [AlreadyExistsException, None], + None, + ), # 1 + ( + 2, + TType.STRUCT, + "o2", + [NoSuchObjectException, None], + None, + ), # 2 + ( + 3, + TType.STRUCT, + "o3", + [InvalidObjectException, None], + None, + ), # 3 + ( + 4, + TType.STRUCT, + "o4", + [MetaException, None], + None, + ), # 4 +) + + +class alter_wm_trigger_args: + """ + Attributes: + - request + + """ + + def __init__( + self, + request=None, + ): + self.request = request + + def read(self, iprot): + if ( + iprot._fast_decode is not None + and isinstance(iprot.trans, TTransport.CReadableTransport) + and self.thrift_spec is not None + ): + iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec]) + return + iprot.readStructBegin() + while True: + (fname, ftype, fid) = iprot.readFieldBegin() + if ftype == TType.STOP: + break + if fid == 1: + if ftype == TType.STRUCT: + self.request = WMAlterTriggerRequest() + self.request.read(iprot) + else: + iprot.skip(ftype) + else: + iprot.skip(ftype) + iprot.readFieldEnd() + iprot.readStructEnd() + + def write(self, oprot): + if oprot._fast_encode is not None and self.thrift_spec is not None: + oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec])) + return + oprot.writeStructBegin("alter_wm_trigger_args") + if self.request is not None: + oprot.writeFieldBegin("request", TType.STRUCT, 1) + self.request.write(oprot) + oprot.writeFieldEnd() + oprot.writeFieldStop() + oprot.writeStructEnd() + + def validate(self): + return + + def __repr__(self): + L = ["{}={!r}".format(key, value) for key, value in self.__dict__.items()] + return "{}({})".format(self.__class__.__name__, ", ".join(L)) + + def __eq__(self, other): + return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ + + def __ne__(self, other): + return not (self == other) + + +all_structs.append(alter_wm_trigger_args) +alter_wm_trigger_args.thrift_spec = ( + None, # 0 + ( + 1, + TType.STRUCT, + "request", + [WMAlterTriggerRequest, None], + None, + ), # 1 +) + + +class alter_wm_trigger_result: + """ + Attributes: + - success + - o1 + - o2 + - o3 + + """ + + def __init__( + self, + success=None, + o1=None, + o2=None, + o3=None, + ): + self.success = success + self.o1 = o1 + self.o2 = o2 + self.o3 = o3 + + def read(self, iprot): + if ( + iprot._fast_decode is not None + and isinstance(iprot.trans, TTransport.CReadableTransport) + and self.thrift_spec is not None + ): + iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec]) + return + iprot.readStructBegin() + while True: + (fname, ftype, fid) = iprot.readFieldBegin() + if ftype == TType.STOP: + break + if fid == 0: + if ftype == TType.STRUCT: + self.success = WMAlterTriggerResponse() + self.success.read(iprot) + else: + iprot.skip(ftype) + elif fid == 1: + if ftype == TType.STRUCT: + self.o1 = NoSuchObjectException.read(iprot) + else: + iprot.skip(ftype) + elif fid == 2: + if ftype == TType.STRUCT: + self.o2 = InvalidObjectException.read(iprot) + else: + iprot.skip(ftype) + elif fid == 3: + if ftype == TType.STRUCT: + self.o3 = MetaException.read(iprot) + else: + iprot.skip(ftype) + else: + iprot.skip(ftype) + iprot.readFieldEnd() + iprot.readStructEnd() + + def write(self, oprot): + if oprot._fast_encode is not None and self.thrift_spec is not None: + oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec])) + return + oprot.writeStructBegin("alter_wm_trigger_result") + if self.success is not None: + oprot.writeFieldBegin("success", TType.STRUCT, 0) + self.success.write(oprot) + oprot.writeFieldEnd() + if self.o1 is not None: + oprot.writeFieldBegin("o1", TType.STRUCT, 1) + self.o1.write(oprot) + oprot.writeFieldEnd() + if self.o2 is not None: + oprot.writeFieldBegin("o2", TType.STRUCT, 2) + self.o2.write(oprot) + oprot.writeFieldEnd() + if self.o3 is not None: + oprot.writeFieldBegin("o3", TType.STRUCT, 3) + self.o3.write(oprot) + oprot.writeFieldEnd() + oprot.writeFieldStop() + oprot.writeStructEnd() + + def validate(self): + return + + def __repr__(self): + L = ["{}={!r}".format(key, value) for key, value in self.__dict__.items()] + return "{}({})".format(self.__class__.__name__, ", ".join(L)) + + def __eq__(self, other): + return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ + + def __ne__(self, other): + return not (self == other) + + +all_structs.append(alter_wm_trigger_result) +alter_wm_trigger_result.thrift_spec = ( + ( + 0, + TType.STRUCT, + "success", + [WMAlterTriggerResponse, None], + None, + ), # 0 + ( + 1, + TType.STRUCT, + "o1", + [NoSuchObjectException, None], + None, + ), # 1 + ( + 2, + TType.STRUCT, + "o2", + [InvalidObjectException, None], + None, + ), # 2 + ( + 3, + TType.STRUCT, + "o3", + [MetaException, None], + None, + ), # 3 +) + + +class drop_wm_trigger_args: + """ + Attributes: + - request + + """ + + def __init__( + self, + request=None, + ): + self.request = request + + def read(self, iprot): + if ( + iprot._fast_decode is not None + and isinstance(iprot.trans, TTransport.CReadableTransport) + and self.thrift_spec is not None + ): + iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec]) + return + iprot.readStructBegin() + while True: + (fname, ftype, fid) = iprot.readFieldBegin() + if ftype == TType.STOP: + break + if fid == 1: + if ftype == TType.STRUCT: + self.request = WMDropTriggerRequest() + self.request.read(iprot) + else: + iprot.skip(ftype) + else: + iprot.skip(ftype) + iprot.readFieldEnd() + iprot.readStructEnd() + + def write(self, oprot): + if oprot._fast_encode is not None and self.thrift_spec is not None: + oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec])) + return + oprot.writeStructBegin("drop_wm_trigger_args") + if self.request is not None: + oprot.writeFieldBegin("request", TType.STRUCT, 1) + self.request.write(oprot) + oprot.writeFieldEnd() + oprot.writeFieldStop() + oprot.writeStructEnd() + + def validate(self): + return + + def __repr__(self): + L = ["{}={!r}".format(key, value) for key, value in self.__dict__.items()] + return "{}({})".format(self.__class__.__name__, ", ".join(L)) + + def __eq__(self, other): + return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ + + def __ne__(self, other): + return not (self == other) + + +all_structs.append(drop_wm_trigger_args) +drop_wm_trigger_args.thrift_spec = ( + None, # 0 + ( + 1, + TType.STRUCT, + "request", + [WMDropTriggerRequest, None], + None, + ), # 1 +) + + +class drop_wm_trigger_result: + """ + Attributes: + - success + - o1 + - o2 + - o3 + + """ + + def __init__( + self, + success=None, + o1=None, + o2=None, + o3=None, + ): + self.success = success + self.o1 = o1 + self.o2 = o2 + self.o3 = o3 + + def read(self, iprot): + if ( + iprot._fast_decode is not None + and isinstance(iprot.trans, TTransport.CReadableTransport) + and self.thrift_spec is not None + ): + iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec]) + return + iprot.readStructBegin() + while True: + (fname, ftype, fid) = iprot.readFieldBegin() + if ftype == TType.STOP: + break + if fid == 0: + if ftype == TType.STRUCT: + self.success = WMDropTriggerResponse() + self.success.read(iprot) + else: + iprot.skip(ftype) + elif fid == 1: + if ftype == TType.STRUCT: + self.o1 = NoSuchObjectException.read(iprot) + else: + iprot.skip(ftype) + elif fid == 2: + if ftype == TType.STRUCT: + self.o2 = InvalidOperationException.read(iprot) + else: + iprot.skip(ftype) + elif fid == 3: + if ftype == TType.STRUCT: + self.o3 = MetaException.read(iprot) + else: + iprot.skip(ftype) + else: + iprot.skip(ftype) + iprot.readFieldEnd() + iprot.readStructEnd() + + def write(self, oprot): + if oprot._fast_encode is not None and self.thrift_spec is not None: + oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec])) + return + oprot.writeStructBegin("drop_wm_trigger_result") + if self.success is not None: + oprot.writeFieldBegin("success", TType.STRUCT, 0) + self.success.write(oprot) + oprot.writeFieldEnd() + if self.o1 is not None: + oprot.writeFieldBegin("o1", TType.STRUCT, 1) + self.o1.write(oprot) + oprot.writeFieldEnd() + if self.o2 is not None: + oprot.writeFieldBegin("o2", TType.STRUCT, 2) + self.o2.write(oprot) + oprot.writeFieldEnd() + if self.o3 is not None: + oprot.writeFieldBegin("o3", TType.STRUCT, 3) + self.o3.write(oprot) + oprot.writeFieldEnd() + oprot.writeFieldStop() + oprot.writeStructEnd() + + def validate(self): + return + + def __repr__(self): + L = ["{}={!r}".format(key, value) for key, value in self.__dict__.items()] + return "{}({})".format(self.__class__.__name__, ", ".join(L)) + + def __eq__(self, other): + return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ + + def __ne__(self, other): + return not (self == other) + + +all_structs.append(drop_wm_trigger_result) +drop_wm_trigger_result.thrift_spec = ( + ( + 0, + TType.STRUCT, + "success", + [WMDropTriggerResponse, None], + None, + ), # 0 + ( + 1, + TType.STRUCT, + "o1", + [NoSuchObjectException, None], + None, + ), # 1 + ( + 2, + TType.STRUCT, + "o2", + [InvalidOperationException, None], + None, + ), # 2 + ( + 3, + TType.STRUCT, + "o3", + [MetaException, None], + None, + ), # 3 +) + + +class get_triggers_for_resourceplan_args: + """ + Attributes: + - request + + """ + + def __init__( + self, + request=None, + ): + self.request = request + + def read(self, iprot): + if ( + iprot._fast_decode is not None + and isinstance(iprot.trans, TTransport.CReadableTransport) + and self.thrift_spec is not None + ): + iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec]) + return + iprot.readStructBegin() + while True: + (fname, ftype, fid) = iprot.readFieldBegin() + if ftype == TType.STOP: + break + if fid == 1: + if ftype == TType.STRUCT: + self.request = WMGetTriggersForResourePlanRequest() + self.request.read(iprot) + else: + iprot.skip(ftype) + else: + iprot.skip(ftype) + iprot.readFieldEnd() + iprot.readStructEnd() + + def write(self, oprot): + if oprot._fast_encode is not None and self.thrift_spec is not None: + oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec])) + return + oprot.writeStructBegin("get_triggers_for_resourceplan_args") + if self.request is not None: + oprot.writeFieldBegin("request", TType.STRUCT, 1) + self.request.write(oprot) + oprot.writeFieldEnd() + oprot.writeFieldStop() + oprot.writeStructEnd() + + def validate(self): + return + + def __repr__(self): + L = ["{}={!r}".format(key, value) for key, value in self.__dict__.items()] + return "{}({})".format(self.__class__.__name__, ", ".join(L)) + + def __eq__(self, other): + return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ + + def __ne__(self, other): + return not (self == other) + + +all_structs.append(get_triggers_for_resourceplan_args) +get_triggers_for_resourceplan_args.thrift_spec = ( + None, # 0 + ( + 1, + TType.STRUCT, + "request", + [WMGetTriggersForResourePlanRequest, None], + None, + ), # 1 +) + + +class get_triggers_for_resourceplan_result: + """ + Attributes: + - success + - o1 + - o2 + + """ + + def __init__( + self, + success=None, + o1=None, + o2=None, + ): + self.success = success + self.o1 = o1 + self.o2 = o2 + + def read(self, iprot): + if ( + iprot._fast_decode is not None + and isinstance(iprot.trans, TTransport.CReadableTransport) + and self.thrift_spec is not None + ): + iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec]) + return + iprot.readStructBegin() + while True: + (fname, ftype, fid) = iprot.readFieldBegin() + if ftype == TType.STOP: + break + if fid == 0: + if ftype == TType.STRUCT: + self.success = WMGetTriggersForResourePlanResponse() + self.success.read(iprot) + else: + iprot.skip(ftype) + elif fid == 1: + if ftype == TType.STRUCT: + self.o1 = NoSuchObjectException.read(iprot) + else: + iprot.skip(ftype) + elif fid == 2: + if ftype == TType.STRUCT: + self.o2 = MetaException.read(iprot) + else: + iprot.skip(ftype) + else: + iprot.skip(ftype) + iprot.readFieldEnd() + iprot.readStructEnd() + + def write(self, oprot): + if oprot._fast_encode is not None and self.thrift_spec is not None: + oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec])) + return + oprot.writeStructBegin("get_triggers_for_resourceplan_result") + if self.success is not None: + oprot.writeFieldBegin("success", TType.STRUCT, 0) + self.success.write(oprot) + oprot.writeFieldEnd() + if self.o1 is not None: + oprot.writeFieldBegin("o1", TType.STRUCT, 1) + self.o1.write(oprot) + oprot.writeFieldEnd() + if self.o2 is not None: + oprot.writeFieldBegin("o2", TType.STRUCT, 2) + self.o2.write(oprot) + oprot.writeFieldEnd() + oprot.writeFieldStop() + oprot.writeStructEnd() + + def validate(self): + return + + def __repr__(self): + L = ["{}={!r}".format(key, value) for key, value in self.__dict__.items()] + return "{}({})".format(self.__class__.__name__, ", ".join(L)) + + def __eq__(self, other): + return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ + + def __ne__(self, other): + return not (self == other) + + +all_structs.append(get_triggers_for_resourceplan_result) +get_triggers_for_resourceplan_result.thrift_spec = ( + ( + 0, + TType.STRUCT, + "success", + [WMGetTriggersForResourePlanResponse, None], + None, + ), # 0 + ( + 1, + TType.STRUCT, + "o1", + [NoSuchObjectException, None], + None, + ), # 1 + ( + 2, + TType.STRUCT, + "o2", + [MetaException, None], + None, + ), # 2 +) + + +class create_wm_pool_args: + """ + Attributes: + - request + + """ + + def __init__( + self, + request=None, + ): + self.request = request + + def read(self, iprot): + if ( + iprot._fast_decode is not None + and isinstance(iprot.trans, TTransport.CReadableTransport) + and self.thrift_spec is not None + ): + iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec]) + return + iprot.readStructBegin() + while True: + (fname, ftype, fid) = iprot.readFieldBegin() + if ftype == TType.STOP: + break + if fid == 1: + if ftype == TType.STRUCT: + self.request = WMCreatePoolRequest() + self.request.read(iprot) + else: + iprot.skip(ftype) + else: + iprot.skip(ftype) + iprot.readFieldEnd() + iprot.readStructEnd() + + def write(self, oprot): + if oprot._fast_encode is not None and self.thrift_spec is not None: + oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec])) + return + oprot.writeStructBegin("create_wm_pool_args") + if self.request is not None: + oprot.writeFieldBegin("request", TType.STRUCT, 1) + self.request.write(oprot) + oprot.writeFieldEnd() + oprot.writeFieldStop() + oprot.writeStructEnd() + + def validate(self): + return + + def __repr__(self): + L = ["{}={!r}".format(key, value) for key, value in self.__dict__.items()] + return "{}({})".format(self.__class__.__name__, ", ".join(L)) + + def __eq__(self, other): + return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ + + def __ne__(self, other): + return not (self == other) + + +all_structs.append(create_wm_pool_args) +create_wm_pool_args.thrift_spec = ( + None, # 0 + ( + 1, + TType.STRUCT, + "request", + [WMCreatePoolRequest, None], + None, + ), # 1 +) + + +class create_wm_pool_result: + """ + Attributes: + - success + - o1 + - o2 + - o3 + - o4 + + """ + + def __init__( + self, + success=None, + o1=None, + o2=None, + o3=None, + o4=None, + ): + self.success = success + self.o1 = o1 + self.o2 = o2 + self.o3 = o3 + self.o4 = o4 + + def read(self, iprot): + if ( + iprot._fast_decode is not None + and isinstance(iprot.trans, TTransport.CReadableTransport) + and self.thrift_spec is not None + ): + iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec]) + return + iprot.readStructBegin() + while True: + (fname, ftype, fid) = iprot.readFieldBegin() + if ftype == TType.STOP: + break + if fid == 0: + if ftype == TType.STRUCT: + self.success = WMCreatePoolResponse() + self.success.read(iprot) + else: + iprot.skip(ftype) + elif fid == 1: + if ftype == TType.STRUCT: + self.o1 = AlreadyExistsException.read(iprot) + else: + iprot.skip(ftype) + elif fid == 2: + if ftype == TType.STRUCT: + self.o2 = NoSuchObjectException.read(iprot) + else: + iprot.skip(ftype) + elif fid == 3: + if ftype == TType.STRUCT: + self.o3 = InvalidObjectException.read(iprot) + else: + iprot.skip(ftype) + elif fid == 4: + if ftype == TType.STRUCT: + self.o4 = MetaException.read(iprot) + else: + iprot.skip(ftype) + else: + iprot.skip(ftype) + iprot.readFieldEnd() + iprot.readStructEnd() + + def write(self, oprot): + if oprot._fast_encode is not None and self.thrift_spec is not None: + oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec])) + return + oprot.writeStructBegin("create_wm_pool_result") + if self.success is not None: + oprot.writeFieldBegin("success", TType.STRUCT, 0) + self.success.write(oprot) + oprot.writeFieldEnd() + if self.o1 is not None: + oprot.writeFieldBegin("o1", TType.STRUCT, 1) + self.o1.write(oprot) + oprot.writeFieldEnd() + if self.o2 is not None: + oprot.writeFieldBegin("o2", TType.STRUCT, 2) + self.o2.write(oprot) + oprot.writeFieldEnd() + if self.o3 is not None: + oprot.writeFieldBegin("o3", TType.STRUCT, 3) + self.o3.write(oprot) + oprot.writeFieldEnd() + if self.o4 is not None: + oprot.writeFieldBegin("o4", TType.STRUCT, 4) + self.o4.write(oprot) + oprot.writeFieldEnd() + oprot.writeFieldStop() + oprot.writeStructEnd() + + def validate(self): + return + + def __repr__(self): + L = ["{}={!r}".format(key, value) for key, value in self.__dict__.items()] + return "{}({})".format(self.__class__.__name__, ", ".join(L)) + + def __eq__(self, other): + return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ + + def __ne__(self, other): + return not (self == other) + + +all_structs.append(create_wm_pool_result) +create_wm_pool_result.thrift_spec = ( + ( + 0, + TType.STRUCT, + "success", + [WMCreatePoolResponse, None], + None, + ), # 0 + ( + 1, + TType.STRUCT, + "o1", + [AlreadyExistsException, None], + None, + ), # 1 + ( + 2, + TType.STRUCT, + "o2", + [NoSuchObjectException, None], + None, + ), # 2 + ( + 3, + TType.STRUCT, + "o3", + [InvalidObjectException, None], + None, + ), # 3 + ( + 4, + TType.STRUCT, + "o4", + [MetaException, None], + None, + ), # 4 +) + + +class alter_wm_pool_args: + """ + Attributes: + - request + + """ + + def __init__( + self, + request=None, + ): + self.request = request + + def read(self, iprot): + if ( + iprot._fast_decode is not None + and isinstance(iprot.trans, TTransport.CReadableTransport) + and self.thrift_spec is not None + ): + iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec]) + return + iprot.readStructBegin() + while True: + (fname, ftype, fid) = iprot.readFieldBegin() + if ftype == TType.STOP: + break + if fid == 1: + if ftype == TType.STRUCT: + self.request = WMAlterPoolRequest() + self.request.read(iprot) + else: + iprot.skip(ftype) + else: + iprot.skip(ftype) + iprot.readFieldEnd() + iprot.readStructEnd() + + def write(self, oprot): + if oprot._fast_encode is not None and self.thrift_spec is not None: + oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec])) + return + oprot.writeStructBegin("alter_wm_pool_args") + if self.request is not None: + oprot.writeFieldBegin("request", TType.STRUCT, 1) + self.request.write(oprot) + oprot.writeFieldEnd() + oprot.writeFieldStop() + oprot.writeStructEnd() + + def validate(self): + return + + def __repr__(self): + L = ["{}={!r}".format(key, value) for key, value in self.__dict__.items()] + return "{}({})".format(self.__class__.__name__, ", ".join(L)) + + def __eq__(self, other): + return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ + + def __ne__(self, other): + return not (self == other) + + +all_structs.append(alter_wm_pool_args) +alter_wm_pool_args.thrift_spec = ( + None, # 0 + ( + 1, + TType.STRUCT, + "request", + [WMAlterPoolRequest, None], + None, + ), # 1 +) + + +class alter_wm_pool_result: + """ + Attributes: + - success + - o1 + - o2 + - o3 + - o4 + + """ + + def __init__( + self, + success=None, + o1=None, + o2=None, + o3=None, + o4=None, + ): + self.success = success + self.o1 = o1 + self.o2 = o2 + self.o3 = o3 + self.o4 = o4 + + def read(self, iprot): + if ( + iprot._fast_decode is not None + and isinstance(iprot.trans, TTransport.CReadableTransport) + and self.thrift_spec is not None + ): + iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec]) + return + iprot.readStructBegin() + while True: + (fname, ftype, fid) = iprot.readFieldBegin() + if ftype == TType.STOP: + break + if fid == 0: + if ftype == TType.STRUCT: + self.success = WMAlterPoolResponse() + self.success.read(iprot) + else: + iprot.skip(ftype) + elif fid == 1: + if ftype == TType.STRUCT: + self.o1 = AlreadyExistsException.read(iprot) + else: + iprot.skip(ftype) + elif fid == 2: + if ftype == TType.STRUCT: + self.o2 = NoSuchObjectException.read(iprot) + else: + iprot.skip(ftype) + elif fid == 3: + if ftype == TType.STRUCT: + self.o3 = InvalidObjectException.read(iprot) + else: + iprot.skip(ftype) + elif fid == 4: + if ftype == TType.STRUCT: + self.o4 = MetaException.read(iprot) + else: + iprot.skip(ftype) + else: + iprot.skip(ftype) + iprot.readFieldEnd() + iprot.readStructEnd() + + def write(self, oprot): + if oprot._fast_encode is not None and self.thrift_spec is not None: + oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec])) + return + oprot.writeStructBegin("alter_wm_pool_result") + if self.success is not None: + oprot.writeFieldBegin("success", TType.STRUCT, 0) + self.success.write(oprot) + oprot.writeFieldEnd() + if self.o1 is not None: + oprot.writeFieldBegin("o1", TType.STRUCT, 1) + self.o1.write(oprot) + oprot.writeFieldEnd() + if self.o2 is not None: + oprot.writeFieldBegin("o2", TType.STRUCT, 2) + self.o2.write(oprot) + oprot.writeFieldEnd() + if self.o3 is not None: + oprot.writeFieldBegin("o3", TType.STRUCT, 3) + self.o3.write(oprot) + oprot.writeFieldEnd() + if self.o4 is not None: + oprot.writeFieldBegin("o4", TType.STRUCT, 4) + self.o4.write(oprot) + oprot.writeFieldEnd() + oprot.writeFieldStop() + oprot.writeStructEnd() + + def validate(self): + return + + def __repr__(self): + L = ["{}={!r}".format(key, value) for key, value in self.__dict__.items()] + return "{}({})".format(self.__class__.__name__, ", ".join(L)) + + def __eq__(self, other): + return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ + + def __ne__(self, other): + return not (self == other) + + +all_structs.append(alter_wm_pool_result) +alter_wm_pool_result.thrift_spec = ( + ( + 0, + TType.STRUCT, + "success", + [WMAlterPoolResponse, None], + None, + ), # 0 + ( + 1, + TType.STRUCT, + "o1", + [AlreadyExistsException, None], + None, + ), # 1 + ( + 2, + TType.STRUCT, + "o2", + [NoSuchObjectException, None], + None, + ), # 2 + ( + 3, + TType.STRUCT, + "o3", + [InvalidObjectException, None], + None, + ), # 3 + ( + 4, + TType.STRUCT, + "o4", + [MetaException, None], + None, + ), # 4 +) + + +class drop_wm_pool_args: + """ + Attributes: + - request + + """ + + def __init__( + self, + request=None, + ): + self.request = request + + def read(self, iprot): + if ( + iprot._fast_decode is not None + and isinstance(iprot.trans, TTransport.CReadableTransport) + and self.thrift_spec is not None + ): + iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec]) + return + iprot.readStructBegin() + while True: + (fname, ftype, fid) = iprot.readFieldBegin() + if ftype == TType.STOP: + break + if fid == 1: + if ftype == TType.STRUCT: + self.request = WMDropPoolRequest() + self.request.read(iprot) + else: + iprot.skip(ftype) + else: + iprot.skip(ftype) + iprot.readFieldEnd() + iprot.readStructEnd() + + def write(self, oprot): + if oprot._fast_encode is not None and self.thrift_spec is not None: + oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec])) + return + oprot.writeStructBegin("drop_wm_pool_args") + if self.request is not None: + oprot.writeFieldBegin("request", TType.STRUCT, 1) + self.request.write(oprot) + oprot.writeFieldEnd() + oprot.writeFieldStop() + oprot.writeStructEnd() + + def validate(self): + return + + def __repr__(self): + L = ["{}={!r}".format(key, value) for key, value in self.__dict__.items()] + return "{}({})".format(self.__class__.__name__, ", ".join(L)) + + def __eq__(self, other): + return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ + + def __ne__(self, other): + return not (self == other) + + +all_structs.append(drop_wm_pool_args) +drop_wm_pool_args.thrift_spec = ( + None, # 0 + ( + 1, + TType.STRUCT, + "request", + [WMDropPoolRequest, None], + None, + ), # 1 +) + + +class drop_wm_pool_result: + """ + Attributes: + - success + - o1 + - o2 + - o3 + + """ + + def __init__( + self, + success=None, + o1=None, + o2=None, + o3=None, + ): + self.success = success + self.o1 = o1 + self.o2 = o2 + self.o3 = o3 + + def read(self, iprot): + if ( + iprot._fast_decode is not None + and isinstance(iprot.trans, TTransport.CReadableTransport) + and self.thrift_spec is not None + ): + iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec]) + return + iprot.readStructBegin() + while True: + (fname, ftype, fid) = iprot.readFieldBegin() + if ftype == TType.STOP: + break + if fid == 0: + if ftype == TType.STRUCT: + self.success = WMDropPoolResponse() + self.success.read(iprot) + else: + iprot.skip(ftype) + elif fid == 1: + if ftype == TType.STRUCT: + self.o1 = NoSuchObjectException.read(iprot) + else: + iprot.skip(ftype) + elif fid == 2: + if ftype == TType.STRUCT: + self.o2 = InvalidOperationException.read(iprot) + else: + iprot.skip(ftype) + elif fid == 3: + if ftype == TType.STRUCT: + self.o3 = MetaException.read(iprot) + else: + iprot.skip(ftype) + else: + iprot.skip(ftype) + iprot.readFieldEnd() + iprot.readStructEnd() + + def write(self, oprot): + if oprot._fast_encode is not None and self.thrift_spec is not None: + oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec])) + return + oprot.writeStructBegin("drop_wm_pool_result") + if self.success is not None: + oprot.writeFieldBegin("success", TType.STRUCT, 0) + self.success.write(oprot) + oprot.writeFieldEnd() + if self.o1 is not None: + oprot.writeFieldBegin("o1", TType.STRUCT, 1) + self.o1.write(oprot) + oprot.writeFieldEnd() + if self.o2 is not None: + oprot.writeFieldBegin("o2", TType.STRUCT, 2) + self.o2.write(oprot) + oprot.writeFieldEnd() + if self.o3 is not None: + oprot.writeFieldBegin("o3", TType.STRUCT, 3) + self.o3.write(oprot) + oprot.writeFieldEnd() + oprot.writeFieldStop() + oprot.writeStructEnd() + + def validate(self): + return + + def __repr__(self): + L = ["{}={!r}".format(key, value) for key, value in self.__dict__.items()] + return "{}({})".format(self.__class__.__name__, ", ".join(L)) + + def __eq__(self, other): + return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ + + def __ne__(self, other): + return not (self == other) + + +all_structs.append(drop_wm_pool_result) +drop_wm_pool_result.thrift_spec = ( + ( + 0, + TType.STRUCT, + "success", + [WMDropPoolResponse, None], + None, + ), # 0 + ( + 1, + TType.STRUCT, + "o1", + [NoSuchObjectException, None], + None, + ), # 1 + ( + 2, + TType.STRUCT, + "o2", + [InvalidOperationException, None], + None, + ), # 2 + ( + 3, + TType.STRUCT, + "o3", + [MetaException, None], + None, + ), # 3 +) + + +class create_or_update_wm_mapping_args: + """ + Attributes: + - request + + """ + + def __init__( + self, + request=None, + ): + self.request = request + + def read(self, iprot): + if ( + iprot._fast_decode is not None + and isinstance(iprot.trans, TTransport.CReadableTransport) + and self.thrift_spec is not None + ): + iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec]) + return + iprot.readStructBegin() + while True: + (fname, ftype, fid) = iprot.readFieldBegin() + if ftype == TType.STOP: + break + if fid == 1: + if ftype == TType.STRUCT: + self.request = WMCreateOrUpdateMappingRequest() + self.request.read(iprot) + else: + iprot.skip(ftype) + else: + iprot.skip(ftype) + iprot.readFieldEnd() + iprot.readStructEnd() + + def write(self, oprot): + if oprot._fast_encode is not None and self.thrift_spec is not None: + oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec])) + return + oprot.writeStructBegin("create_or_update_wm_mapping_args") + if self.request is not None: + oprot.writeFieldBegin("request", TType.STRUCT, 1) + self.request.write(oprot) + oprot.writeFieldEnd() + oprot.writeFieldStop() + oprot.writeStructEnd() + + def validate(self): + return + + def __repr__(self): + L = ["{}={!r}".format(key, value) for key, value in self.__dict__.items()] + return "{}({})".format(self.__class__.__name__, ", ".join(L)) + + def __eq__(self, other): + return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ + + def __ne__(self, other): + return not (self == other) + + +all_structs.append(create_or_update_wm_mapping_args) +create_or_update_wm_mapping_args.thrift_spec = ( + None, # 0 + ( + 1, + TType.STRUCT, + "request", + [WMCreateOrUpdateMappingRequest, None], + None, + ), # 1 +) + + +class create_or_update_wm_mapping_result: + """ + Attributes: + - success + - o1 + - o2 + - o3 + - o4 + + """ + + def __init__( + self, + success=None, + o1=None, + o2=None, + o3=None, + o4=None, + ): + self.success = success + self.o1 = o1 + self.o2 = o2 + self.o3 = o3 + self.o4 = o4 + + def read(self, iprot): + if ( + iprot._fast_decode is not None + and isinstance(iprot.trans, TTransport.CReadableTransport) + and self.thrift_spec is not None + ): + iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec]) + return + iprot.readStructBegin() + while True: + (fname, ftype, fid) = iprot.readFieldBegin() + if ftype == TType.STOP: + break + if fid == 0: + if ftype == TType.STRUCT: + self.success = WMCreateOrUpdateMappingResponse() + self.success.read(iprot) + else: + iprot.skip(ftype) + elif fid == 1: + if ftype == TType.STRUCT: + self.o1 = AlreadyExistsException.read(iprot) + else: + iprot.skip(ftype) + elif fid == 2: + if ftype == TType.STRUCT: + self.o2 = NoSuchObjectException.read(iprot) + else: + iprot.skip(ftype) + elif fid == 3: + if ftype == TType.STRUCT: + self.o3 = InvalidObjectException.read(iprot) + else: + iprot.skip(ftype) + elif fid == 4: + if ftype == TType.STRUCT: + self.o4 = MetaException.read(iprot) + else: + iprot.skip(ftype) + else: + iprot.skip(ftype) + iprot.readFieldEnd() + iprot.readStructEnd() + + def write(self, oprot): + if oprot._fast_encode is not None and self.thrift_spec is not None: + oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec])) + return + oprot.writeStructBegin("create_or_update_wm_mapping_result") + if self.success is not None: + oprot.writeFieldBegin("success", TType.STRUCT, 0) + self.success.write(oprot) + oprot.writeFieldEnd() + if self.o1 is not None: + oprot.writeFieldBegin("o1", TType.STRUCT, 1) + self.o1.write(oprot) + oprot.writeFieldEnd() + if self.o2 is not None: + oprot.writeFieldBegin("o2", TType.STRUCT, 2) + self.o2.write(oprot) + oprot.writeFieldEnd() + if self.o3 is not None: + oprot.writeFieldBegin("o3", TType.STRUCT, 3) + self.o3.write(oprot) + oprot.writeFieldEnd() + if self.o4 is not None: + oprot.writeFieldBegin("o4", TType.STRUCT, 4) + self.o4.write(oprot) + oprot.writeFieldEnd() + oprot.writeFieldStop() + oprot.writeStructEnd() + + def validate(self): + return + + def __repr__(self): + L = ["{}={!r}".format(key, value) for key, value in self.__dict__.items()] + return "{}({})".format(self.__class__.__name__, ", ".join(L)) + + def __eq__(self, other): + return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ + + def __ne__(self, other): + return not (self == other) + + +all_structs.append(create_or_update_wm_mapping_result) +create_or_update_wm_mapping_result.thrift_spec = ( + ( + 0, + TType.STRUCT, + "success", + [WMCreateOrUpdateMappingResponse, None], + None, + ), # 0 + ( + 1, + TType.STRUCT, + "o1", + [AlreadyExistsException, None], + None, + ), # 1 + ( + 2, + TType.STRUCT, + "o2", + [NoSuchObjectException, None], + None, + ), # 2 + ( + 3, + TType.STRUCT, + "o3", + [InvalidObjectException, None], + None, + ), # 3 + ( + 4, + TType.STRUCT, + "o4", + [MetaException, None], + None, + ), # 4 +) + + +class drop_wm_mapping_args: + """ + Attributes: + - request + + """ + + def __init__( + self, + request=None, + ): + self.request = request + + def read(self, iprot): + if ( + iprot._fast_decode is not None + and isinstance(iprot.trans, TTransport.CReadableTransport) + and self.thrift_spec is not None + ): + iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec]) + return + iprot.readStructBegin() + while True: + (fname, ftype, fid) = iprot.readFieldBegin() + if ftype == TType.STOP: + break + if fid == 1: + if ftype == TType.STRUCT: + self.request = WMDropMappingRequest() + self.request.read(iprot) + else: + iprot.skip(ftype) + else: + iprot.skip(ftype) + iprot.readFieldEnd() + iprot.readStructEnd() + + def write(self, oprot): + if oprot._fast_encode is not None and self.thrift_spec is not None: + oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec])) + return + oprot.writeStructBegin("drop_wm_mapping_args") + if self.request is not None: + oprot.writeFieldBegin("request", TType.STRUCT, 1) + self.request.write(oprot) + oprot.writeFieldEnd() + oprot.writeFieldStop() + oprot.writeStructEnd() + + def validate(self): + return + + def __repr__(self): + L = ["{}={!r}".format(key, value) for key, value in self.__dict__.items()] + return "{}({})".format(self.__class__.__name__, ", ".join(L)) + + def __eq__(self, other): + return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ + + def __ne__(self, other): + return not (self == other) + + +all_structs.append(drop_wm_mapping_args) +drop_wm_mapping_args.thrift_spec = ( + None, # 0 + ( + 1, + TType.STRUCT, + "request", + [WMDropMappingRequest, None], + None, + ), # 1 +) + + +class drop_wm_mapping_result: + """ + Attributes: + - success + - o1 + - o2 + - o3 + + """ + + def __init__( + self, + success=None, + o1=None, + o2=None, + o3=None, + ): + self.success = success + self.o1 = o1 + self.o2 = o2 + self.o3 = o3 + + def read(self, iprot): + if ( + iprot._fast_decode is not None + and isinstance(iprot.trans, TTransport.CReadableTransport) + and self.thrift_spec is not None + ): + iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec]) + return + iprot.readStructBegin() + while True: + (fname, ftype, fid) = iprot.readFieldBegin() + if ftype == TType.STOP: + break + if fid == 0: + if ftype == TType.STRUCT: + self.success = WMDropMappingResponse() + self.success.read(iprot) + else: + iprot.skip(ftype) + elif fid == 1: + if ftype == TType.STRUCT: + self.o1 = NoSuchObjectException.read(iprot) + else: + iprot.skip(ftype) + elif fid == 2: + if ftype == TType.STRUCT: + self.o2 = InvalidOperationException.read(iprot) + else: + iprot.skip(ftype) + elif fid == 3: + if ftype == TType.STRUCT: + self.o3 = MetaException.read(iprot) + else: + iprot.skip(ftype) + else: + iprot.skip(ftype) + iprot.readFieldEnd() + iprot.readStructEnd() + + def write(self, oprot): + if oprot._fast_encode is not None and self.thrift_spec is not None: + oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec])) + return + oprot.writeStructBegin("drop_wm_mapping_result") + if self.success is not None: + oprot.writeFieldBegin("success", TType.STRUCT, 0) + self.success.write(oprot) + oprot.writeFieldEnd() + if self.o1 is not None: + oprot.writeFieldBegin("o1", TType.STRUCT, 1) + self.o1.write(oprot) + oprot.writeFieldEnd() + if self.o2 is not None: + oprot.writeFieldBegin("o2", TType.STRUCT, 2) + self.o2.write(oprot) + oprot.writeFieldEnd() + if self.o3 is not None: + oprot.writeFieldBegin("o3", TType.STRUCT, 3) + self.o3.write(oprot) + oprot.writeFieldEnd() + oprot.writeFieldStop() + oprot.writeStructEnd() + + def validate(self): + return + + def __repr__(self): + L = ["{}={!r}".format(key, value) for key, value in self.__dict__.items()] + return "{}({})".format(self.__class__.__name__, ", ".join(L)) + + def __eq__(self, other): + return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ + + def __ne__(self, other): + return not (self == other) + + +all_structs.append(drop_wm_mapping_result) +drop_wm_mapping_result.thrift_spec = ( + ( + 0, + TType.STRUCT, + "success", + [WMDropMappingResponse, None], + None, + ), # 0 + ( + 1, + TType.STRUCT, + "o1", + [NoSuchObjectException, None], + None, + ), # 1 + ( + 2, + TType.STRUCT, + "o2", + [InvalidOperationException, None], + None, + ), # 2 + ( + 3, + TType.STRUCT, + "o3", + [MetaException, None], + None, + ), # 3 +) + + +class create_or_drop_wm_trigger_to_pool_mapping_args: + """ + Attributes: + - request + + """ + + def __init__( + self, + request=None, + ): + self.request = request + + def read(self, iprot): + if ( + iprot._fast_decode is not None + and isinstance(iprot.trans, TTransport.CReadableTransport) + and self.thrift_spec is not None + ): + iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec]) + return + iprot.readStructBegin() + while True: + (fname, ftype, fid) = iprot.readFieldBegin() + if ftype == TType.STOP: + break + if fid == 1: + if ftype == TType.STRUCT: + self.request = WMCreateOrDropTriggerToPoolMappingRequest() + self.request.read(iprot) + else: + iprot.skip(ftype) + else: + iprot.skip(ftype) + iprot.readFieldEnd() + iprot.readStructEnd() + + def write(self, oprot): + if oprot._fast_encode is not None and self.thrift_spec is not None: + oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec])) + return + oprot.writeStructBegin("create_or_drop_wm_trigger_to_pool_mapping_args") + if self.request is not None: + oprot.writeFieldBegin("request", TType.STRUCT, 1) + self.request.write(oprot) + oprot.writeFieldEnd() + oprot.writeFieldStop() + oprot.writeStructEnd() + + def validate(self): + return + + def __repr__(self): + L = ["{}={!r}".format(key, value) for key, value in self.__dict__.items()] + return "{}({})".format(self.__class__.__name__, ", ".join(L)) + + def __eq__(self, other): + return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ + + def __ne__(self, other): + return not (self == other) + + +all_structs.append(create_or_drop_wm_trigger_to_pool_mapping_args) +create_or_drop_wm_trigger_to_pool_mapping_args.thrift_spec = ( + None, # 0 + ( + 1, + TType.STRUCT, + "request", + [WMCreateOrDropTriggerToPoolMappingRequest, None], + None, + ), # 1 +) + + +class create_or_drop_wm_trigger_to_pool_mapping_result: + """ + Attributes: + - success + - o1 + - o2 + - o3 + - o4 + + """ + + def __init__( + self, + success=None, + o1=None, + o2=None, + o3=None, + o4=None, + ): + self.success = success + self.o1 = o1 + self.o2 = o2 + self.o3 = o3 + self.o4 = o4 + + def read(self, iprot): + if ( + iprot._fast_decode is not None + and isinstance(iprot.trans, TTransport.CReadableTransport) + and self.thrift_spec is not None + ): + iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec]) + return + iprot.readStructBegin() + while True: + (fname, ftype, fid) = iprot.readFieldBegin() + if ftype == TType.STOP: + break + if fid == 0: + if ftype == TType.STRUCT: + self.success = WMCreateOrDropTriggerToPoolMappingResponse() + self.success.read(iprot) + else: + iprot.skip(ftype) + elif fid == 1: + if ftype == TType.STRUCT: + self.o1 = AlreadyExistsException.read(iprot) + else: + iprot.skip(ftype) + elif fid == 2: + if ftype == TType.STRUCT: + self.o2 = NoSuchObjectException.read(iprot) + else: + iprot.skip(ftype) + elif fid == 3: + if ftype == TType.STRUCT: + self.o3 = InvalidObjectException.read(iprot) + else: + iprot.skip(ftype) + elif fid == 4: + if ftype == TType.STRUCT: + self.o4 = MetaException.read(iprot) + else: + iprot.skip(ftype) + else: + iprot.skip(ftype) + iprot.readFieldEnd() + iprot.readStructEnd() + + def write(self, oprot): + if oprot._fast_encode is not None and self.thrift_spec is not None: + oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec])) + return + oprot.writeStructBegin("create_or_drop_wm_trigger_to_pool_mapping_result") + if self.success is not None: + oprot.writeFieldBegin("success", TType.STRUCT, 0) + self.success.write(oprot) + oprot.writeFieldEnd() + if self.o1 is not None: + oprot.writeFieldBegin("o1", TType.STRUCT, 1) + self.o1.write(oprot) + oprot.writeFieldEnd() + if self.o2 is not None: + oprot.writeFieldBegin("o2", TType.STRUCT, 2) + self.o2.write(oprot) + oprot.writeFieldEnd() + if self.o3 is not None: + oprot.writeFieldBegin("o3", TType.STRUCT, 3) + self.o3.write(oprot) + oprot.writeFieldEnd() + if self.o4 is not None: + oprot.writeFieldBegin("o4", TType.STRUCT, 4) + self.o4.write(oprot) + oprot.writeFieldEnd() + oprot.writeFieldStop() + oprot.writeStructEnd() + + def validate(self): + return + + def __repr__(self): + L = ["{}={!r}".format(key, value) for key, value in self.__dict__.items()] + return "{}({})".format(self.__class__.__name__, ", ".join(L)) + + def __eq__(self, other): + return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ + + def __ne__(self, other): + return not (self == other) + + +all_structs.append(create_or_drop_wm_trigger_to_pool_mapping_result) +create_or_drop_wm_trigger_to_pool_mapping_result.thrift_spec = ( + ( + 0, + TType.STRUCT, + "success", + [WMCreateOrDropTriggerToPoolMappingResponse, None], + None, + ), # 0 + ( + 1, + TType.STRUCT, + "o1", + [AlreadyExistsException, None], + None, + ), # 1 + ( + 2, + TType.STRUCT, + "o2", + [NoSuchObjectException, None], + None, + ), # 2 + ( + 3, + TType.STRUCT, + "o3", + [InvalidObjectException, None], + None, + ), # 3 + ( + 4, + TType.STRUCT, + "o4", + [MetaException, None], + None, + ), # 4 +) + + +class create_ischema_args: + """ + Attributes: + - schema + + """ + + def __init__( + self, + schema=None, + ): + self.schema = schema + + def read(self, iprot): + if ( + iprot._fast_decode is not None + and isinstance(iprot.trans, TTransport.CReadableTransport) + and self.thrift_spec is not None + ): + iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec]) + return + iprot.readStructBegin() + while True: + (fname, ftype, fid) = iprot.readFieldBegin() + if ftype == TType.STOP: + break + if fid == 1: + if ftype == TType.STRUCT: + self.schema = ISchema() + self.schema.read(iprot) + else: + iprot.skip(ftype) + else: + iprot.skip(ftype) + iprot.readFieldEnd() + iprot.readStructEnd() + + def write(self, oprot): + if oprot._fast_encode is not None and self.thrift_spec is not None: + oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec])) + return + oprot.writeStructBegin("create_ischema_args") + if self.schema is not None: + oprot.writeFieldBegin("schema", TType.STRUCT, 1) + self.schema.write(oprot) + oprot.writeFieldEnd() + oprot.writeFieldStop() + oprot.writeStructEnd() + + def validate(self): + return + + def __repr__(self): + L = ["{}={!r}".format(key, value) for key, value in self.__dict__.items()] + return "{}({})".format(self.__class__.__name__, ", ".join(L)) + + def __eq__(self, other): + return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ + + def __ne__(self, other): + return not (self == other) + + +all_structs.append(create_ischema_args) +create_ischema_args.thrift_spec = ( + None, # 0 + ( + 1, + TType.STRUCT, + "schema", + [ISchema, None], + None, + ), # 1 +) + + +class create_ischema_result: + """ + Attributes: + - o1 + - o2 + - o3 + + """ + + def __init__( + self, + o1=None, + o2=None, + o3=None, + ): + self.o1 = o1 + self.o2 = o2 + self.o3 = o3 + + def read(self, iprot): + if ( + iprot._fast_decode is not None + and isinstance(iprot.trans, TTransport.CReadableTransport) + and self.thrift_spec is not None + ): + iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec]) + return + iprot.readStructBegin() + while True: + (fname, ftype, fid) = iprot.readFieldBegin() + if ftype == TType.STOP: + break + if fid == 1: + if ftype == TType.STRUCT: + self.o1 = AlreadyExistsException.read(iprot) + else: + iprot.skip(ftype) + elif fid == 2: + if ftype == TType.STRUCT: + self.o2 = NoSuchObjectException.read(iprot) + else: + iprot.skip(ftype) + elif fid == 3: + if ftype == TType.STRUCT: + self.o3 = MetaException.read(iprot) + else: + iprot.skip(ftype) + else: + iprot.skip(ftype) + iprot.readFieldEnd() + iprot.readStructEnd() + + def write(self, oprot): + if oprot._fast_encode is not None and self.thrift_spec is not None: + oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec])) + return + oprot.writeStructBegin("create_ischema_result") + if self.o1 is not None: + oprot.writeFieldBegin("o1", TType.STRUCT, 1) + self.o1.write(oprot) + oprot.writeFieldEnd() + if self.o2 is not None: + oprot.writeFieldBegin("o2", TType.STRUCT, 2) + self.o2.write(oprot) + oprot.writeFieldEnd() + if self.o3 is not None: + oprot.writeFieldBegin("o3", TType.STRUCT, 3) + self.o3.write(oprot) + oprot.writeFieldEnd() + oprot.writeFieldStop() + oprot.writeStructEnd() + + def validate(self): + return + + def __repr__(self): + L = ["{}={!r}".format(key, value) for key, value in self.__dict__.items()] + return "{}({})".format(self.__class__.__name__, ", ".join(L)) + + def __eq__(self, other): + return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ + + def __ne__(self, other): + return not (self == other) + + +all_structs.append(create_ischema_result) +create_ischema_result.thrift_spec = ( + None, # 0 + ( + 1, + TType.STRUCT, + "o1", + [AlreadyExistsException, None], + None, + ), # 1 + ( + 2, + TType.STRUCT, + "o2", + [NoSuchObjectException, None], + None, + ), # 2 + ( + 3, + TType.STRUCT, + "o3", + [MetaException, None], + None, + ), # 3 +) + + +class alter_ischema_args: + """ + Attributes: + - rqst + + """ + + def __init__( + self, + rqst=None, + ): + self.rqst = rqst + + def read(self, iprot): + if ( + iprot._fast_decode is not None + and isinstance(iprot.trans, TTransport.CReadableTransport) + and self.thrift_spec is not None + ): + iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec]) + return + iprot.readStructBegin() + while True: + (fname, ftype, fid) = iprot.readFieldBegin() + if ftype == TType.STOP: + break + if fid == 1: + if ftype == TType.STRUCT: + self.rqst = AlterISchemaRequest() + self.rqst.read(iprot) + else: + iprot.skip(ftype) + else: + iprot.skip(ftype) + iprot.readFieldEnd() + iprot.readStructEnd() + + def write(self, oprot): + if oprot._fast_encode is not None and self.thrift_spec is not None: + oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec])) + return + oprot.writeStructBegin("alter_ischema_args") + if self.rqst is not None: + oprot.writeFieldBegin("rqst", TType.STRUCT, 1) + self.rqst.write(oprot) + oprot.writeFieldEnd() + oprot.writeFieldStop() + oprot.writeStructEnd() + + def validate(self): + return + + def __repr__(self): + L = ["{}={!r}".format(key, value) for key, value in self.__dict__.items()] + return "{}({})".format(self.__class__.__name__, ", ".join(L)) + + def __eq__(self, other): + return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ + + def __ne__(self, other): + return not (self == other) + + +all_structs.append(alter_ischema_args) +alter_ischema_args.thrift_spec = ( + None, # 0 + ( + 1, + TType.STRUCT, + "rqst", + [AlterISchemaRequest, None], + None, + ), # 1 +) + + +class alter_ischema_result: + """ + Attributes: + - o1 + - o2 + + """ + + def __init__( + self, + o1=None, + o2=None, + ): + self.o1 = o1 + self.o2 = o2 + + def read(self, iprot): + if ( + iprot._fast_decode is not None + and isinstance(iprot.trans, TTransport.CReadableTransport) + and self.thrift_spec is not None + ): + iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec]) + return + iprot.readStructBegin() + while True: + (fname, ftype, fid) = iprot.readFieldBegin() + if ftype == TType.STOP: + break + if fid == 1: + if ftype == TType.STRUCT: + self.o1 = NoSuchObjectException.read(iprot) + else: + iprot.skip(ftype) + elif fid == 2: + if ftype == TType.STRUCT: + self.o2 = MetaException.read(iprot) + else: + iprot.skip(ftype) + else: + iprot.skip(ftype) + iprot.readFieldEnd() + iprot.readStructEnd() + + def write(self, oprot): + if oprot._fast_encode is not None and self.thrift_spec is not None: + oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec])) + return + oprot.writeStructBegin("alter_ischema_result") + if self.o1 is not None: + oprot.writeFieldBegin("o1", TType.STRUCT, 1) + self.o1.write(oprot) + oprot.writeFieldEnd() + if self.o2 is not None: + oprot.writeFieldBegin("o2", TType.STRUCT, 2) + self.o2.write(oprot) + oprot.writeFieldEnd() + oprot.writeFieldStop() + oprot.writeStructEnd() + + def validate(self): + return + + def __repr__(self): + L = ["{}={!r}".format(key, value) for key, value in self.__dict__.items()] + return "{}({})".format(self.__class__.__name__, ", ".join(L)) + + def __eq__(self, other): + return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ + + def __ne__(self, other): + return not (self == other) + + +all_structs.append(alter_ischema_result) +alter_ischema_result.thrift_spec = ( + None, # 0 + ( + 1, + TType.STRUCT, + "o1", + [NoSuchObjectException, None], + None, + ), # 1 + ( + 2, + TType.STRUCT, + "o2", + [MetaException, None], + None, + ), # 2 +) + + +class get_ischema_args: + """ + Attributes: + - name + + """ + + def __init__( + self, + name=None, + ): + self.name = name + + def read(self, iprot): + if ( + iprot._fast_decode is not None + and isinstance(iprot.trans, TTransport.CReadableTransport) + and self.thrift_spec is not None + ): + iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec]) + return + iprot.readStructBegin() + while True: + (fname, ftype, fid) = iprot.readFieldBegin() + if ftype == TType.STOP: + break + if fid == 1: + if ftype == TType.STRUCT: + self.name = ISchemaName() + self.name.read(iprot) + else: + iprot.skip(ftype) + else: + iprot.skip(ftype) + iprot.readFieldEnd() + iprot.readStructEnd() + + def write(self, oprot): + if oprot._fast_encode is not None and self.thrift_spec is not None: + oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec])) + return + oprot.writeStructBegin("get_ischema_args") + if self.name is not None: + oprot.writeFieldBegin("name", TType.STRUCT, 1) + self.name.write(oprot) + oprot.writeFieldEnd() + oprot.writeFieldStop() + oprot.writeStructEnd() + + def validate(self): + return + + def __repr__(self): + L = ["{}={!r}".format(key, value) for key, value in self.__dict__.items()] + return "{}({})".format(self.__class__.__name__, ", ".join(L)) + + def __eq__(self, other): + return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ + + def __ne__(self, other): + return not (self == other) + + +all_structs.append(get_ischema_args) +get_ischema_args.thrift_spec = ( + None, # 0 + ( + 1, + TType.STRUCT, + "name", + [ISchemaName, None], + None, + ), # 1 +) + + +class get_ischema_result: + """ + Attributes: + - success + - o1 + - o2 + + """ + + def __init__( + self, + success=None, + o1=None, + o2=None, + ): + self.success = success + self.o1 = o1 + self.o2 = o2 + + def read(self, iprot): + if ( + iprot._fast_decode is not None + and isinstance(iprot.trans, TTransport.CReadableTransport) + and self.thrift_spec is not None + ): + iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec]) + return + iprot.readStructBegin() + while True: + (fname, ftype, fid) = iprot.readFieldBegin() + if ftype == TType.STOP: + break + if fid == 0: + if ftype == TType.STRUCT: + self.success = ISchema() + self.success.read(iprot) + else: + iprot.skip(ftype) + elif fid == 1: + if ftype == TType.STRUCT: + self.o1 = NoSuchObjectException.read(iprot) + else: + iprot.skip(ftype) + elif fid == 2: + if ftype == TType.STRUCT: + self.o2 = MetaException.read(iprot) + else: + iprot.skip(ftype) + else: + iprot.skip(ftype) + iprot.readFieldEnd() + iprot.readStructEnd() + + def write(self, oprot): + if oprot._fast_encode is not None and self.thrift_spec is not None: + oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec])) + return + oprot.writeStructBegin("get_ischema_result") + if self.success is not None: + oprot.writeFieldBegin("success", TType.STRUCT, 0) + self.success.write(oprot) + oprot.writeFieldEnd() + if self.o1 is not None: + oprot.writeFieldBegin("o1", TType.STRUCT, 1) + self.o1.write(oprot) + oprot.writeFieldEnd() + if self.o2 is not None: + oprot.writeFieldBegin("o2", TType.STRUCT, 2) + self.o2.write(oprot) + oprot.writeFieldEnd() + oprot.writeFieldStop() + oprot.writeStructEnd() + + def validate(self): + return + + def __repr__(self): + L = ["{}={!r}".format(key, value) for key, value in self.__dict__.items()] + return "{}({})".format(self.__class__.__name__, ", ".join(L)) + + def __eq__(self, other): + return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ + + def __ne__(self, other): + return not (self == other) + + +all_structs.append(get_ischema_result) +get_ischema_result.thrift_spec = ( + ( + 0, + TType.STRUCT, + "success", + [ISchema, None], + None, + ), # 0 + ( + 1, + TType.STRUCT, + "o1", + [NoSuchObjectException, None], + None, + ), # 1 + ( + 2, + TType.STRUCT, + "o2", + [MetaException, None], + None, + ), # 2 +) + + +class drop_ischema_args: + """ + Attributes: + - name + + """ + + def __init__( + self, + name=None, + ): + self.name = name + + def read(self, iprot): + if ( + iprot._fast_decode is not None + and isinstance(iprot.trans, TTransport.CReadableTransport) + and self.thrift_spec is not None + ): + iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec]) + return + iprot.readStructBegin() + while True: + (fname, ftype, fid) = iprot.readFieldBegin() + if ftype == TType.STOP: + break + if fid == 1: + if ftype == TType.STRUCT: + self.name = ISchemaName() + self.name.read(iprot) + else: + iprot.skip(ftype) + else: + iprot.skip(ftype) + iprot.readFieldEnd() + iprot.readStructEnd() + + def write(self, oprot): + if oprot._fast_encode is not None and self.thrift_spec is not None: + oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec])) + return + oprot.writeStructBegin("drop_ischema_args") + if self.name is not None: + oprot.writeFieldBegin("name", TType.STRUCT, 1) + self.name.write(oprot) + oprot.writeFieldEnd() + oprot.writeFieldStop() + oprot.writeStructEnd() + + def validate(self): + return + + def __repr__(self): + L = ["{}={!r}".format(key, value) for key, value in self.__dict__.items()] + return "{}({})".format(self.__class__.__name__, ", ".join(L)) + + def __eq__(self, other): + return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ + + def __ne__(self, other): + return not (self == other) + + +all_structs.append(drop_ischema_args) +drop_ischema_args.thrift_spec = ( + None, # 0 + ( + 1, + TType.STRUCT, + "name", + [ISchemaName, None], + None, + ), # 1 +) + + +class drop_ischema_result: + """ + Attributes: + - o1 + - o2 + - o3 + + """ + + def __init__( + self, + o1=None, + o2=None, + o3=None, + ): + self.o1 = o1 + self.o2 = o2 + self.o3 = o3 + + def read(self, iprot): + if ( + iprot._fast_decode is not None + and isinstance(iprot.trans, TTransport.CReadableTransport) + and self.thrift_spec is not None + ): + iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec]) + return + iprot.readStructBegin() + while True: + (fname, ftype, fid) = iprot.readFieldBegin() + if ftype == TType.STOP: + break + if fid == 1: + if ftype == TType.STRUCT: + self.o1 = NoSuchObjectException.read(iprot) + else: + iprot.skip(ftype) + elif fid == 2: + if ftype == TType.STRUCT: + self.o2 = InvalidOperationException.read(iprot) + else: + iprot.skip(ftype) + elif fid == 3: + if ftype == TType.STRUCT: + self.o3 = MetaException.read(iprot) + else: + iprot.skip(ftype) + else: + iprot.skip(ftype) + iprot.readFieldEnd() + iprot.readStructEnd() + + def write(self, oprot): + if oprot._fast_encode is not None and self.thrift_spec is not None: + oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec])) + return + oprot.writeStructBegin("drop_ischema_result") + if self.o1 is not None: + oprot.writeFieldBegin("o1", TType.STRUCT, 1) + self.o1.write(oprot) + oprot.writeFieldEnd() + if self.o2 is not None: + oprot.writeFieldBegin("o2", TType.STRUCT, 2) + self.o2.write(oprot) + oprot.writeFieldEnd() + if self.o3 is not None: + oprot.writeFieldBegin("o3", TType.STRUCT, 3) + self.o3.write(oprot) + oprot.writeFieldEnd() + oprot.writeFieldStop() + oprot.writeStructEnd() + + def validate(self): + return + + def __repr__(self): + L = ["{}={!r}".format(key, value) for key, value in self.__dict__.items()] + return "{}({})".format(self.__class__.__name__, ", ".join(L)) + + def __eq__(self, other): + return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ + + def __ne__(self, other): + return not (self == other) + + +all_structs.append(drop_ischema_result) +drop_ischema_result.thrift_spec = ( + None, # 0 + ( + 1, + TType.STRUCT, + "o1", + [NoSuchObjectException, None], + None, + ), # 1 + ( + 2, + TType.STRUCT, + "o2", + [InvalidOperationException, None], + None, + ), # 2 + ( + 3, + TType.STRUCT, + "o3", + [MetaException, None], + None, + ), # 3 +) + + +class add_schema_version_args: + """ + Attributes: + - schemaVersion + + """ + + def __init__( + self, + schemaVersion=None, + ): + self.schemaVersion = schemaVersion + + def read(self, iprot): + if ( + iprot._fast_decode is not None + and isinstance(iprot.trans, TTransport.CReadableTransport) + and self.thrift_spec is not None + ): + iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec]) + return + iprot.readStructBegin() + while True: + (fname, ftype, fid) = iprot.readFieldBegin() + if ftype == TType.STOP: + break + if fid == 1: + if ftype == TType.STRUCT: + self.schemaVersion = SchemaVersion() + self.schemaVersion.read(iprot) + else: + iprot.skip(ftype) + else: + iprot.skip(ftype) + iprot.readFieldEnd() + iprot.readStructEnd() + + def write(self, oprot): + if oprot._fast_encode is not None and self.thrift_spec is not None: + oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec])) + return + oprot.writeStructBegin("add_schema_version_args") + if self.schemaVersion is not None: + oprot.writeFieldBegin("schemaVersion", TType.STRUCT, 1) + self.schemaVersion.write(oprot) + oprot.writeFieldEnd() + oprot.writeFieldStop() + oprot.writeStructEnd() + + def validate(self): + return + + def __repr__(self): + L = ["{}={!r}".format(key, value) for key, value in self.__dict__.items()] + return "{}({})".format(self.__class__.__name__, ", ".join(L)) + + def __eq__(self, other): + return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ + + def __ne__(self, other): + return not (self == other) + + +all_structs.append(add_schema_version_args) +add_schema_version_args.thrift_spec = ( + None, # 0 + ( + 1, + TType.STRUCT, + "schemaVersion", + [SchemaVersion, None], + None, + ), # 1 +) + + +class add_schema_version_result: + """ + Attributes: + - o1 + - o2 + - o3 + + """ + + def __init__( + self, + o1=None, + o2=None, + o3=None, + ): + self.o1 = o1 + self.o2 = o2 + self.o3 = o3 + + def read(self, iprot): + if ( + iprot._fast_decode is not None + and isinstance(iprot.trans, TTransport.CReadableTransport) + and self.thrift_spec is not None + ): + iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec]) + return + iprot.readStructBegin() + while True: + (fname, ftype, fid) = iprot.readFieldBegin() + if ftype == TType.STOP: + break + if fid == 1: + if ftype == TType.STRUCT: + self.o1 = AlreadyExistsException.read(iprot) + else: + iprot.skip(ftype) + elif fid == 2: + if ftype == TType.STRUCT: + self.o2 = NoSuchObjectException.read(iprot) + else: + iprot.skip(ftype) + elif fid == 3: + if ftype == TType.STRUCT: + self.o3 = MetaException.read(iprot) + else: + iprot.skip(ftype) + else: + iprot.skip(ftype) + iprot.readFieldEnd() + iprot.readStructEnd() + + def write(self, oprot): + if oprot._fast_encode is not None and self.thrift_spec is not None: + oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec])) + return + oprot.writeStructBegin("add_schema_version_result") + if self.o1 is not None: + oprot.writeFieldBegin("o1", TType.STRUCT, 1) + self.o1.write(oprot) + oprot.writeFieldEnd() + if self.o2 is not None: + oprot.writeFieldBegin("o2", TType.STRUCT, 2) + self.o2.write(oprot) + oprot.writeFieldEnd() + if self.o3 is not None: + oprot.writeFieldBegin("o3", TType.STRUCT, 3) + self.o3.write(oprot) + oprot.writeFieldEnd() + oprot.writeFieldStop() + oprot.writeStructEnd() + + def validate(self): + return + + def __repr__(self): + L = ["{}={!r}".format(key, value) for key, value in self.__dict__.items()] + return "{}({})".format(self.__class__.__name__, ", ".join(L)) + + def __eq__(self, other): + return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ + + def __ne__(self, other): + return not (self == other) + + +all_structs.append(add_schema_version_result) +add_schema_version_result.thrift_spec = ( + None, # 0 + ( + 1, + TType.STRUCT, + "o1", + [AlreadyExistsException, None], + None, + ), # 1 + ( + 2, + TType.STRUCT, + "o2", + [NoSuchObjectException, None], + None, + ), # 2 + ( + 3, + TType.STRUCT, + "o3", + [MetaException, None], + None, + ), # 3 +) + + +class get_schema_version_args: + """ + Attributes: + - schemaVersion + + """ + + def __init__( + self, + schemaVersion=None, + ): + self.schemaVersion = schemaVersion + + def read(self, iprot): + if ( + iprot._fast_decode is not None + and isinstance(iprot.trans, TTransport.CReadableTransport) + and self.thrift_spec is not None + ): + iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec]) + return + iprot.readStructBegin() + while True: + (fname, ftype, fid) = iprot.readFieldBegin() + if ftype == TType.STOP: + break + if fid == 1: + if ftype == TType.STRUCT: + self.schemaVersion = SchemaVersionDescriptor() + self.schemaVersion.read(iprot) + else: + iprot.skip(ftype) + else: + iprot.skip(ftype) + iprot.readFieldEnd() + iprot.readStructEnd() + + def write(self, oprot): + if oprot._fast_encode is not None and self.thrift_spec is not None: + oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec])) + return + oprot.writeStructBegin("get_schema_version_args") + if self.schemaVersion is not None: + oprot.writeFieldBegin("schemaVersion", TType.STRUCT, 1) + self.schemaVersion.write(oprot) + oprot.writeFieldEnd() + oprot.writeFieldStop() + oprot.writeStructEnd() + + def validate(self): + return + + def __repr__(self): + L = ["{}={!r}".format(key, value) for key, value in self.__dict__.items()] + return "{}({})".format(self.__class__.__name__, ", ".join(L)) + + def __eq__(self, other): + return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ + + def __ne__(self, other): + return not (self == other) + + +all_structs.append(get_schema_version_args) +get_schema_version_args.thrift_spec = ( + None, # 0 + ( + 1, + TType.STRUCT, + "schemaVersion", + [SchemaVersionDescriptor, None], + None, + ), # 1 +) + + +class get_schema_version_result: + """ + Attributes: + - success + - o1 + - o2 + + """ + + def __init__( + self, + success=None, + o1=None, + o2=None, + ): + self.success = success + self.o1 = o1 + self.o2 = o2 + + def read(self, iprot): + if ( + iprot._fast_decode is not None + and isinstance(iprot.trans, TTransport.CReadableTransport) + and self.thrift_spec is not None + ): + iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec]) + return + iprot.readStructBegin() + while True: + (fname, ftype, fid) = iprot.readFieldBegin() + if ftype == TType.STOP: + break + if fid == 0: + if ftype == TType.STRUCT: + self.success = SchemaVersion() + self.success.read(iprot) + else: + iprot.skip(ftype) + elif fid == 1: + if ftype == TType.STRUCT: + self.o1 = NoSuchObjectException.read(iprot) + else: + iprot.skip(ftype) + elif fid == 2: + if ftype == TType.STRUCT: + self.o2 = MetaException.read(iprot) + else: + iprot.skip(ftype) + else: + iprot.skip(ftype) + iprot.readFieldEnd() + iprot.readStructEnd() + + def write(self, oprot): + if oprot._fast_encode is not None and self.thrift_spec is not None: + oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec])) + return + oprot.writeStructBegin("get_schema_version_result") + if self.success is not None: + oprot.writeFieldBegin("success", TType.STRUCT, 0) + self.success.write(oprot) + oprot.writeFieldEnd() + if self.o1 is not None: + oprot.writeFieldBegin("o1", TType.STRUCT, 1) + self.o1.write(oprot) + oprot.writeFieldEnd() + if self.o2 is not None: + oprot.writeFieldBegin("o2", TType.STRUCT, 2) + self.o2.write(oprot) + oprot.writeFieldEnd() + oprot.writeFieldStop() + oprot.writeStructEnd() + + def validate(self): + return + + def __repr__(self): + L = ["{}={!r}".format(key, value) for key, value in self.__dict__.items()] + return "{}({})".format(self.__class__.__name__, ", ".join(L)) + + def __eq__(self, other): + return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ + + def __ne__(self, other): + return not (self == other) + + +all_structs.append(get_schema_version_result) +get_schema_version_result.thrift_spec = ( + ( + 0, + TType.STRUCT, + "success", + [SchemaVersion, None], + None, + ), # 0 + ( + 1, + TType.STRUCT, + "o1", + [NoSuchObjectException, None], + None, + ), # 1 + ( + 2, + TType.STRUCT, + "o2", + [MetaException, None], + None, + ), # 2 +) + + +class get_schema_latest_version_args: + """ + Attributes: + - schemaName + + """ + + def __init__( + self, + schemaName=None, + ): + self.schemaName = schemaName + + def read(self, iprot): + if ( + iprot._fast_decode is not None + and isinstance(iprot.trans, TTransport.CReadableTransport) + and self.thrift_spec is not None + ): + iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec]) + return + iprot.readStructBegin() + while True: + (fname, ftype, fid) = iprot.readFieldBegin() + if ftype == TType.STOP: + break + if fid == 1: + if ftype == TType.STRUCT: + self.schemaName = ISchemaName() + self.schemaName.read(iprot) + else: + iprot.skip(ftype) + else: + iprot.skip(ftype) + iprot.readFieldEnd() + iprot.readStructEnd() + + def write(self, oprot): + if oprot._fast_encode is not None and self.thrift_spec is not None: + oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec])) + return + oprot.writeStructBegin("get_schema_latest_version_args") + if self.schemaName is not None: + oprot.writeFieldBegin("schemaName", TType.STRUCT, 1) + self.schemaName.write(oprot) + oprot.writeFieldEnd() + oprot.writeFieldStop() + oprot.writeStructEnd() + + def validate(self): + return + + def __repr__(self): + L = ["{}={!r}".format(key, value) for key, value in self.__dict__.items()] + return "{}({})".format(self.__class__.__name__, ", ".join(L)) + + def __eq__(self, other): + return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ + + def __ne__(self, other): + return not (self == other) + + +all_structs.append(get_schema_latest_version_args) +get_schema_latest_version_args.thrift_spec = ( + None, # 0 + ( + 1, + TType.STRUCT, + "schemaName", + [ISchemaName, None], + None, + ), # 1 +) + + +class get_schema_latest_version_result: + """ + Attributes: + - success + - o1 + - o2 + + """ + + def __init__( + self, + success=None, + o1=None, + o2=None, + ): + self.success = success + self.o1 = o1 + self.o2 = o2 + + def read(self, iprot): + if ( + iprot._fast_decode is not None + and isinstance(iprot.trans, TTransport.CReadableTransport) + and self.thrift_spec is not None + ): + iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec]) + return + iprot.readStructBegin() + while True: + (fname, ftype, fid) = iprot.readFieldBegin() + if ftype == TType.STOP: + break + if fid == 0: + if ftype == TType.STRUCT: + self.success = SchemaVersion() + self.success.read(iprot) + else: + iprot.skip(ftype) + elif fid == 1: + if ftype == TType.STRUCT: + self.o1 = NoSuchObjectException.read(iprot) + else: + iprot.skip(ftype) + elif fid == 2: + if ftype == TType.STRUCT: + self.o2 = MetaException.read(iprot) + else: + iprot.skip(ftype) + else: + iprot.skip(ftype) + iprot.readFieldEnd() + iprot.readStructEnd() + + def write(self, oprot): + if oprot._fast_encode is not None and self.thrift_spec is not None: + oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec])) + return + oprot.writeStructBegin("get_schema_latest_version_result") + if self.success is not None: + oprot.writeFieldBegin("success", TType.STRUCT, 0) + self.success.write(oprot) + oprot.writeFieldEnd() + if self.o1 is not None: + oprot.writeFieldBegin("o1", TType.STRUCT, 1) + self.o1.write(oprot) + oprot.writeFieldEnd() + if self.o2 is not None: + oprot.writeFieldBegin("o2", TType.STRUCT, 2) + self.o2.write(oprot) + oprot.writeFieldEnd() + oprot.writeFieldStop() + oprot.writeStructEnd() + + def validate(self): + return + + def __repr__(self): + L = ["{}={!r}".format(key, value) for key, value in self.__dict__.items()] + return "{}({})".format(self.__class__.__name__, ", ".join(L)) + + def __eq__(self, other): + return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ + + def __ne__(self, other): + return not (self == other) + + +all_structs.append(get_schema_latest_version_result) +get_schema_latest_version_result.thrift_spec = ( + ( + 0, + TType.STRUCT, + "success", + [SchemaVersion, None], + None, + ), # 0 + ( + 1, + TType.STRUCT, + "o1", + [NoSuchObjectException, None], + None, + ), # 1 + ( + 2, + TType.STRUCT, + "o2", + [MetaException, None], + None, + ), # 2 +) + + +class get_schema_all_versions_args: + """ + Attributes: + - schemaName + + """ + + def __init__( + self, + schemaName=None, + ): + self.schemaName = schemaName + + def read(self, iprot): + if ( + iprot._fast_decode is not None + and isinstance(iprot.trans, TTransport.CReadableTransport) + and self.thrift_spec is not None + ): + iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec]) + return + iprot.readStructBegin() + while True: + (fname, ftype, fid) = iprot.readFieldBegin() + if ftype == TType.STOP: + break + if fid == 1: + if ftype == TType.STRUCT: + self.schemaName = ISchemaName() + self.schemaName.read(iprot) + else: + iprot.skip(ftype) + else: + iprot.skip(ftype) + iprot.readFieldEnd() + iprot.readStructEnd() + + def write(self, oprot): + if oprot._fast_encode is not None and self.thrift_spec is not None: + oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec])) + return + oprot.writeStructBegin("get_schema_all_versions_args") + if self.schemaName is not None: + oprot.writeFieldBegin("schemaName", TType.STRUCT, 1) + self.schemaName.write(oprot) + oprot.writeFieldEnd() + oprot.writeFieldStop() + oprot.writeStructEnd() + + def validate(self): + return + + def __repr__(self): + L = ["{}={!r}".format(key, value) for key, value in self.__dict__.items()] + return "{}({})".format(self.__class__.__name__, ", ".join(L)) + + def __eq__(self, other): + return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ + + def __ne__(self, other): + return not (self == other) + + +all_structs.append(get_schema_all_versions_args) +get_schema_all_versions_args.thrift_spec = ( + None, # 0 + ( + 1, + TType.STRUCT, + "schemaName", + [ISchemaName, None], + None, + ), # 1 +) + + +class get_schema_all_versions_result: + """ + Attributes: + - success + - o1 + - o2 + + """ + + def __init__( + self, + success=None, + o1=None, + o2=None, + ): + self.success = success + self.o1 = o1 + self.o2 = o2 + + def read(self, iprot): + if ( + iprot._fast_decode is not None + and isinstance(iprot.trans, TTransport.CReadableTransport) + and self.thrift_spec is not None + ): + iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec]) + return + iprot.readStructBegin() + while True: + (fname, ftype, fid) = iprot.readFieldBegin() + if ftype == TType.STOP: + break + if fid == 0: + if ftype == TType.LIST: + self.success = [] + (_etype1775, _size1772) = iprot.readListBegin() + for _i1776 in range(_size1772): + _elem1777 = SchemaVersion() + _elem1777.read(iprot) + self.success.append(_elem1777) + iprot.readListEnd() + else: + iprot.skip(ftype) + elif fid == 1: + if ftype == TType.STRUCT: + self.o1 = NoSuchObjectException.read(iprot) + else: + iprot.skip(ftype) + elif fid == 2: + if ftype == TType.STRUCT: + self.o2 = MetaException.read(iprot) + else: + iprot.skip(ftype) + else: + iprot.skip(ftype) + iprot.readFieldEnd() + iprot.readStructEnd() + + def write(self, oprot): + if oprot._fast_encode is not None and self.thrift_spec is not None: + oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec])) + return + oprot.writeStructBegin("get_schema_all_versions_result") + if self.success is not None: + oprot.writeFieldBegin("success", TType.LIST, 0) + oprot.writeListBegin(TType.STRUCT, len(self.success)) + for iter1778 in self.success: + iter1778.write(oprot) + oprot.writeListEnd() + oprot.writeFieldEnd() + if self.o1 is not None: + oprot.writeFieldBegin("o1", TType.STRUCT, 1) + self.o1.write(oprot) + oprot.writeFieldEnd() + if self.o2 is not None: + oprot.writeFieldBegin("o2", TType.STRUCT, 2) + self.o2.write(oprot) + oprot.writeFieldEnd() + oprot.writeFieldStop() + oprot.writeStructEnd() + + def validate(self): + return + + def __repr__(self): + L = ["{}={!r}".format(key, value) for key, value in self.__dict__.items()] + return "{}({})".format(self.__class__.__name__, ", ".join(L)) + + def __eq__(self, other): + return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ + + def __ne__(self, other): + return not (self == other) + + +all_structs.append(get_schema_all_versions_result) +get_schema_all_versions_result.thrift_spec = ( + ( + 0, + TType.LIST, + "success", + (TType.STRUCT, [SchemaVersion, None], False), + None, + ), # 0 + ( + 1, + TType.STRUCT, + "o1", + [NoSuchObjectException, None], + None, + ), # 1 + ( + 2, + TType.STRUCT, + "o2", + [MetaException, None], + None, + ), # 2 +) + + +class drop_schema_version_args: + """ + Attributes: + - schemaVersion + + """ + + def __init__( + self, + schemaVersion=None, + ): + self.schemaVersion = schemaVersion + + def read(self, iprot): + if ( + iprot._fast_decode is not None + and isinstance(iprot.trans, TTransport.CReadableTransport) + and self.thrift_spec is not None + ): + iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec]) + return + iprot.readStructBegin() + while True: + (fname, ftype, fid) = iprot.readFieldBegin() + if ftype == TType.STOP: + break + if fid == 1: + if ftype == TType.STRUCT: + self.schemaVersion = SchemaVersionDescriptor() + self.schemaVersion.read(iprot) + else: + iprot.skip(ftype) + else: + iprot.skip(ftype) + iprot.readFieldEnd() + iprot.readStructEnd() + + def write(self, oprot): + if oprot._fast_encode is not None and self.thrift_spec is not None: + oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec])) + return + oprot.writeStructBegin("drop_schema_version_args") + if self.schemaVersion is not None: + oprot.writeFieldBegin("schemaVersion", TType.STRUCT, 1) + self.schemaVersion.write(oprot) + oprot.writeFieldEnd() + oprot.writeFieldStop() + oprot.writeStructEnd() + + def validate(self): + return + + def __repr__(self): + L = ["{}={!r}".format(key, value) for key, value in self.__dict__.items()] + return "{}({})".format(self.__class__.__name__, ", ".join(L)) + + def __eq__(self, other): + return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ + + def __ne__(self, other): + return not (self == other) + + +all_structs.append(drop_schema_version_args) +drop_schema_version_args.thrift_spec = ( + None, # 0 + ( + 1, + TType.STRUCT, + "schemaVersion", + [SchemaVersionDescriptor, None], + None, + ), # 1 +) + + +class drop_schema_version_result: + """ + Attributes: + - o1 + - o2 + + """ + + def __init__( + self, + o1=None, + o2=None, + ): + self.o1 = o1 + self.o2 = o2 + + def read(self, iprot): + if ( + iprot._fast_decode is not None + and isinstance(iprot.trans, TTransport.CReadableTransport) + and self.thrift_spec is not None + ): + iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec]) + return + iprot.readStructBegin() + while True: + (fname, ftype, fid) = iprot.readFieldBegin() + if ftype == TType.STOP: + break + if fid == 1: + if ftype == TType.STRUCT: + self.o1 = NoSuchObjectException.read(iprot) + else: + iprot.skip(ftype) + elif fid == 2: + if ftype == TType.STRUCT: + self.o2 = MetaException.read(iprot) + else: + iprot.skip(ftype) + else: + iprot.skip(ftype) + iprot.readFieldEnd() + iprot.readStructEnd() + + def write(self, oprot): + if oprot._fast_encode is not None and self.thrift_spec is not None: + oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec])) + return + oprot.writeStructBegin("drop_schema_version_result") + if self.o1 is not None: + oprot.writeFieldBegin("o1", TType.STRUCT, 1) + self.o1.write(oprot) + oprot.writeFieldEnd() + if self.o2 is not None: + oprot.writeFieldBegin("o2", TType.STRUCT, 2) + self.o2.write(oprot) + oprot.writeFieldEnd() + oprot.writeFieldStop() + oprot.writeStructEnd() + + def validate(self): + return + + def __repr__(self): + L = ["{}={!r}".format(key, value) for key, value in self.__dict__.items()] + return "{}({})".format(self.__class__.__name__, ", ".join(L)) + + def __eq__(self, other): + return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ + + def __ne__(self, other): + return not (self == other) + + +all_structs.append(drop_schema_version_result) +drop_schema_version_result.thrift_spec = ( + None, # 0 + ( + 1, + TType.STRUCT, + "o1", + [NoSuchObjectException, None], + None, + ), # 1 + ( + 2, + TType.STRUCT, + "o2", + [MetaException, None], + None, + ), # 2 +) + + +class get_schemas_by_cols_args: + """ + Attributes: + - rqst + + """ + + def __init__( + self, + rqst=None, + ): + self.rqst = rqst + + def read(self, iprot): + if ( + iprot._fast_decode is not None + and isinstance(iprot.trans, TTransport.CReadableTransport) + and self.thrift_spec is not None + ): + iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec]) + return + iprot.readStructBegin() + while True: + (fname, ftype, fid) = iprot.readFieldBegin() + if ftype == TType.STOP: + break + if fid == 1: + if ftype == TType.STRUCT: + self.rqst = FindSchemasByColsRqst() + self.rqst.read(iprot) + else: + iprot.skip(ftype) + else: + iprot.skip(ftype) + iprot.readFieldEnd() + iprot.readStructEnd() + + def write(self, oprot): + if oprot._fast_encode is not None and self.thrift_spec is not None: + oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec])) + return + oprot.writeStructBegin("get_schemas_by_cols_args") + if self.rqst is not None: + oprot.writeFieldBegin("rqst", TType.STRUCT, 1) + self.rqst.write(oprot) + oprot.writeFieldEnd() + oprot.writeFieldStop() + oprot.writeStructEnd() + + def validate(self): + return + + def __repr__(self): + L = ["{}={!r}".format(key, value) for key, value in self.__dict__.items()] + return "{}({})".format(self.__class__.__name__, ", ".join(L)) + + def __eq__(self, other): + return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ + + def __ne__(self, other): + return not (self == other) + + +all_structs.append(get_schemas_by_cols_args) +get_schemas_by_cols_args.thrift_spec = ( + None, # 0 + ( + 1, + TType.STRUCT, + "rqst", + [FindSchemasByColsRqst, None], + None, + ), # 1 +) + + +class get_schemas_by_cols_result: + """ + Attributes: + - success + - o1 + + """ + + def __init__( + self, + success=None, + o1=None, + ): + self.success = success + self.o1 = o1 + + def read(self, iprot): + if ( + iprot._fast_decode is not None + and isinstance(iprot.trans, TTransport.CReadableTransport) + and self.thrift_spec is not None + ): + iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec]) + return + iprot.readStructBegin() + while True: + (fname, ftype, fid) = iprot.readFieldBegin() + if ftype == TType.STOP: + break + if fid == 0: + if ftype == TType.STRUCT: + self.success = FindSchemasByColsResp() + self.success.read(iprot) + else: + iprot.skip(ftype) + elif fid == 1: + if ftype == TType.STRUCT: + self.o1 = MetaException.read(iprot) + else: + iprot.skip(ftype) + else: + iprot.skip(ftype) + iprot.readFieldEnd() + iprot.readStructEnd() + + def write(self, oprot): + if oprot._fast_encode is not None and self.thrift_spec is not None: + oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec])) + return + oprot.writeStructBegin("get_schemas_by_cols_result") + if self.success is not None: + oprot.writeFieldBegin("success", TType.STRUCT, 0) + self.success.write(oprot) + oprot.writeFieldEnd() + if self.o1 is not None: + oprot.writeFieldBegin("o1", TType.STRUCT, 1) + self.o1.write(oprot) + oprot.writeFieldEnd() + oprot.writeFieldStop() + oprot.writeStructEnd() + + def validate(self): + return + + def __repr__(self): + L = ["{}={!r}".format(key, value) for key, value in self.__dict__.items()] + return "{}({})".format(self.__class__.__name__, ", ".join(L)) + + def __eq__(self, other): + return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ + + def __ne__(self, other): + return not (self == other) + + +all_structs.append(get_schemas_by_cols_result) +get_schemas_by_cols_result.thrift_spec = ( + ( + 0, + TType.STRUCT, + "success", + [FindSchemasByColsResp, None], + None, + ), # 0 + ( + 1, + TType.STRUCT, + "o1", + [MetaException, None], + None, + ), # 1 +) + + +class map_schema_version_to_serde_args: + """ + Attributes: + - rqst + + """ + + def __init__( + self, + rqst=None, + ): + self.rqst = rqst + + def read(self, iprot): + if ( + iprot._fast_decode is not None + and isinstance(iprot.trans, TTransport.CReadableTransport) + and self.thrift_spec is not None + ): + iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec]) + return + iprot.readStructBegin() + while True: + (fname, ftype, fid) = iprot.readFieldBegin() + if ftype == TType.STOP: + break + if fid == 1: + if ftype == TType.STRUCT: + self.rqst = MapSchemaVersionToSerdeRequest() + self.rqst.read(iprot) + else: + iprot.skip(ftype) + else: + iprot.skip(ftype) + iprot.readFieldEnd() + iprot.readStructEnd() + + def write(self, oprot): + if oprot._fast_encode is not None and self.thrift_spec is not None: + oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec])) + return + oprot.writeStructBegin("map_schema_version_to_serde_args") + if self.rqst is not None: + oprot.writeFieldBegin("rqst", TType.STRUCT, 1) + self.rqst.write(oprot) + oprot.writeFieldEnd() + oprot.writeFieldStop() + oprot.writeStructEnd() + + def validate(self): + return + + def __repr__(self): + L = ["{}={!r}".format(key, value) for key, value in self.__dict__.items()] + return "{}({})".format(self.__class__.__name__, ", ".join(L)) + + def __eq__(self, other): + return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ + + def __ne__(self, other): + return not (self == other) + + +all_structs.append(map_schema_version_to_serde_args) +map_schema_version_to_serde_args.thrift_spec = ( + None, # 0 + ( + 1, + TType.STRUCT, + "rqst", + [MapSchemaVersionToSerdeRequest, None], + None, + ), # 1 +) + + +class map_schema_version_to_serde_result: + """ + Attributes: + - o1 + - o2 + + """ + + def __init__( + self, + o1=None, + o2=None, + ): + self.o1 = o1 + self.o2 = o2 + + def read(self, iprot): + if ( + iprot._fast_decode is not None + and isinstance(iprot.trans, TTransport.CReadableTransport) + and self.thrift_spec is not None + ): + iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec]) + return + iprot.readStructBegin() + while True: + (fname, ftype, fid) = iprot.readFieldBegin() + if ftype == TType.STOP: + break + if fid == 1: + if ftype == TType.STRUCT: + self.o1 = NoSuchObjectException.read(iprot) + else: + iprot.skip(ftype) + elif fid == 2: + if ftype == TType.STRUCT: + self.o2 = MetaException.read(iprot) + else: + iprot.skip(ftype) + else: + iprot.skip(ftype) + iprot.readFieldEnd() + iprot.readStructEnd() + + def write(self, oprot): + if oprot._fast_encode is not None and self.thrift_spec is not None: + oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec])) + return + oprot.writeStructBegin("map_schema_version_to_serde_result") + if self.o1 is not None: + oprot.writeFieldBegin("o1", TType.STRUCT, 1) + self.o1.write(oprot) + oprot.writeFieldEnd() + if self.o2 is not None: + oprot.writeFieldBegin("o2", TType.STRUCT, 2) + self.o2.write(oprot) + oprot.writeFieldEnd() + oprot.writeFieldStop() + oprot.writeStructEnd() + + def validate(self): + return + + def __repr__(self): + L = ["{}={!r}".format(key, value) for key, value in self.__dict__.items()] + return "{}({})".format(self.__class__.__name__, ", ".join(L)) + + def __eq__(self, other): + return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ + + def __ne__(self, other): + return not (self == other) + + +all_structs.append(map_schema_version_to_serde_result) +map_schema_version_to_serde_result.thrift_spec = ( + None, # 0 + ( + 1, + TType.STRUCT, + "o1", + [NoSuchObjectException, None], + None, + ), # 1 + ( + 2, + TType.STRUCT, + "o2", + [MetaException, None], + None, + ), # 2 +) + + +class set_schema_version_state_args: + """ + Attributes: + - rqst + + """ + + def __init__( + self, + rqst=None, + ): + self.rqst = rqst + + def read(self, iprot): + if ( + iprot._fast_decode is not None + and isinstance(iprot.trans, TTransport.CReadableTransport) + and self.thrift_spec is not None + ): + iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec]) + return + iprot.readStructBegin() + while True: + (fname, ftype, fid) = iprot.readFieldBegin() + if ftype == TType.STOP: + break + if fid == 1: + if ftype == TType.STRUCT: + self.rqst = SetSchemaVersionStateRequest() + self.rqst.read(iprot) + else: + iprot.skip(ftype) + else: + iprot.skip(ftype) + iprot.readFieldEnd() + iprot.readStructEnd() + + def write(self, oprot): + if oprot._fast_encode is not None and self.thrift_spec is not None: + oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec])) + return + oprot.writeStructBegin("set_schema_version_state_args") + if self.rqst is not None: + oprot.writeFieldBegin("rqst", TType.STRUCT, 1) + self.rqst.write(oprot) + oprot.writeFieldEnd() + oprot.writeFieldStop() + oprot.writeStructEnd() + + def validate(self): + return + + def __repr__(self): + L = ["{}={!r}".format(key, value) for key, value in self.__dict__.items()] + return "{}({})".format(self.__class__.__name__, ", ".join(L)) + + def __eq__(self, other): + return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ + + def __ne__(self, other): + return not (self == other) + + +all_structs.append(set_schema_version_state_args) +set_schema_version_state_args.thrift_spec = ( + None, # 0 + ( + 1, + TType.STRUCT, + "rqst", + [SetSchemaVersionStateRequest, None], + None, + ), # 1 +) + + +class set_schema_version_state_result: + """ + Attributes: + - o1 + - o2 + - o3 + + """ + + def __init__( + self, + o1=None, + o2=None, + o3=None, + ): + self.o1 = o1 + self.o2 = o2 + self.o3 = o3 + + def read(self, iprot): + if ( + iprot._fast_decode is not None + and isinstance(iprot.trans, TTransport.CReadableTransport) + and self.thrift_spec is not None + ): + iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec]) + return + iprot.readStructBegin() + while True: + (fname, ftype, fid) = iprot.readFieldBegin() + if ftype == TType.STOP: + break + if fid == 1: + if ftype == TType.STRUCT: + self.o1 = NoSuchObjectException.read(iprot) + else: + iprot.skip(ftype) + elif fid == 2: + if ftype == TType.STRUCT: + self.o2 = InvalidOperationException.read(iprot) + else: + iprot.skip(ftype) + elif fid == 3: + if ftype == TType.STRUCT: + self.o3 = MetaException.read(iprot) + else: + iprot.skip(ftype) + else: + iprot.skip(ftype) + iprot.readFieldEnd() + iprot.readStructEnd() + + def write(self, oprot): + if oprot._fast_encode is not None and self.thrift_spec is not None: + oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec])) + return + oprot.writeStructBegin("set_schema_version_state_result") + if self.o1 is not None: + oprot.writeFieldBegin("o1", TType.STRUCT, 1) + self.o1.write(oprot) + oprot.writeFieldEnd() + if self.o2 is not None: + oprot.writeFieldBegin("o2", TType.STRUCT, 2) + self.o2.write(oprot) + oprot.writeFieldEnd() + if self.o3 is not None: + oprot.writeFieldBegin("o3", TType.STRUCT, 3) + self.o3.write(oprot) + oprot.writeFieldEnd() + oprot.writeFieldStop() + oprot.writeStructEnd() + + def validate(self): + return + + def __repr__(self): + L = ["{}={!r}".format(key, value) for key, value in self.__dict__.items()] + return "{}({})".format(self.__class__.__name__, ", ".join(L)) + + def __eq__(self, other): + return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ + + def __ne__(self, other): + return not (self == other) + + +all_structs.append(set_schema_version_state_result) +set_schema_version_state_result.thrift_spec = ( + None, # 0 + ( + 1, + TType.STRUCT, + "o1", + [NoSuchObjectException, None], + None, + ), # 1 + ( + 2, + TType.STRUCT, + "o2", + [InvalidOperationException, None], + None, + ), # 2 + ( + 3, + TType.STRUCT, + "o3", + [MetaException, None], + None, + ), # 3 +) + + +class add_serde_args: + """ + Attributes: + - serde + + """ + + def __init__( + self, + serde=None, + ): + self.serde = serde + + def read(self, iprot): + if ( + iprot._fast_decode is not None + and isinstance(iprot.trans, TTransport.CReadableTransport) + and self.thrift_spec is not None + ): + iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec]) + return + iprot.readStructBegin() + while True: + (fname, ftype, fid) = iprot.readFieldBegin() + if ftype == TType.STOP: + break + if fid == 1: + if ftype == TType.STRUCT: + self.serde = SerDeInfo() + self.serde.read(iprot) + else: + iprot.skip(ftype) + else: + iprot.skip(ftype) + iprot.readFieldEnd() + iprot.readStructEnd() + + def write(self, oprot): + if oprot._fast_encode is not None and self.thrift_spec is not None: + oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec])) + return + oprot.writeStructBegin("add_serde_args") + if self.serde is not None: + oprot.writeFieldBegin("serde", TType.STRUCT, 1) + self.serde.write(oprot) + oprot.writeFieldEnd() + oprot.writeFieldStop() + oprot.writeStructEnd() + + def validate(self): + return + + def __repr__(self): + L = ["{}={!r}".format(key, value) for key, value in self.__dict__.items()] + return "{}({})".format(self.__class__.__name__, ", ".join(L)) + + def __eq__(self, other): + return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ + + def __ne__(self, other): + return not (self == other) + + +all_structs.append(add_serde_args) +add_serde_args.thrift_spec = ( + None, # 0 + ( + 1, + TType.STRUCT, + "serde", + [SerDeInfo, None], + None, + ), # 1 +) + + +class add_serde_result: + """ + Attributes: + - o1 + - o2 + + """ + + def __init__( + self, + o1=None, + o2=None, + ): + self.o1 = o1 + self.o2 = o2 + + def read(self, iprot): + if ( + iprot._fast_decode is not None + and isinstance(iprot.trans, TTransport.CReadableTransport) + and self.thrift_spec is not None + ): + iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec]) + return + iprot.readStructBegin() + while True: + (fname, ftype, fid) = iprot.readFieldBegin() + if ftype == TType.STOP: + break + if fid == 1: + if ftype == TType.STRUCT: + self.o1 = AlreadyExistsException.read(iprot) + else: + iprot.skip(ftype) + elif fid == 2: + if ftype == TType.STRUCT: + self.o2 = MetaException.read(iprot) + else: + iprot.skip(ftype) + else: + iprot.skip(ftype) + iprot.readFieldEnd() + iprot.readStructEnd() + + def write(self, oprot): + if oprot._fast_encode is not None and self.thrift_spec is not None: + oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec])) + return + oprot.writeStructBegin("add_serde_result") + if self.o1 is not None: + oprot.writeFieldBegin("o1", TType.STRUCT, 1) + self.o1.write(oprot) + oprot.writeFieldEnd() + if self.o2 is not None: + oprot.writeFieldBegin("o2", TType.STRUCT, 2) + self.o2.write(oprot) + oprot.writeFieldEnd() + oprot.writeFieldStop() + oprot.writeStructEnd() + + def validate(self): + return + + def __repr__(self): + L = ["{}={!r}".format(key, value) for key, value in self.__dict__.items()] + return "{}({})".format(self.__class__.__name__, ", ".join(L)) + + def __eq__(self, other): + return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ + + def __ne__(self, other): + return not (self == other) + + +all_structs.append(add_serde_result) +add_serde_result.thrift_spec = ( + None, # 0 + ( + 1, + TType.STRUCT, + "o1", + [AlreadyExistsException, None], + None, + ), # 1 + ( + 2, + TType.STRUCT, + "o2", + [MetaException, None], + None, + ), # 2 +) + + +class get_serde_args: + """ + Attributes: + - rqst + + """ + + def __init__( + self, + rqst=None, + ): + self.rqst = rqst + + def read(self, iprot): + if ( + iprot._fast_decode is not None + and isinstance(iprot.trans, TTransport.CReadableTransport) + and self.thrift_spec is not None + ): + iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec]) + return + iprot.readStructBegin() + while True: + (fname, ftype, fid) = iprot.readFieldBegin() + if ftype == TType.STOP: + break + if fid == 1: + if ftype == TType.STRUCT: + self.rqst = GetSerdeRequest() + self.rqst.read(iprot) + else: + iprot.skip(ftype) + else: + iprot.skip(ftype) + iprot.readFieldEnd() + iprot.readStructEnd() + + def write(self, oprot): + if oprot._fast_encode is not None and self.thrift_spec is not None: + oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec])) + return + oprot.writeStructBegin("get_serde_args") + if self.rqst is not None: + oprot.writeFieldBegin("rqst", TType.STRUCT, 1) + self.rqst.write(oprot) + oprot.writeFieldEnd() + oprot.writeFieldStop() + oprot.writeStructEnd() + + def validate(self): + return + + def __repr__(self): + L = ["{}={!r}".format(key, value) for key, value in self.__dict__.items()] + return "{}({})".format(self.__class__.__name__, ", ".join(L)) + + def __eq__(self, other): + return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ + + def __ne__(self, other): + return not (self == other) + + +all_structs.append(get_serde_args) +get_serde_args.thrift_spec = ( + None, # 0 + ( + 1, + TType.STRUCT, + "rqst", + [GetSerdeRequest, None], + None, + ), # 1 +) + + +class get_serde_result: + """ + Attributes: + - success + - o1 + - o2 + + """ + + def __init__( + self, + success=None, + o1=None, + o2=None, + ): + self.success = success + self.o1 = o1 + self.o2 = o2 + + def read(self, iprot): + if ( + iprot._fast_decode is not None + and isinstance(iprot.trans, TTransport.CReadableTransport) + and self.thrift_spec is not None + ): + iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec]) + return + iprot.readStructBegin() + while True: + (fname, ftype, fid) = iprot.readFieldBegin() + if ftype == TType.STOP: + break + if fid == 0: + if ftype == TType.STRUCT: + self.success = SerDeInfo() + self.success.read(iprot) + else: + iprot.skip(ftype) + elif fid == 1: + if ftype == TType.STRUCT: + self.o1 = NoSuchObjectException.read(iprot) + else: + iprot.skip(ftype) + elif fid == 2: + if ftype == TType.STRUCT: + self.o2 = MetaException.read(iprot) + else: + iprot.skip(ftype) + else: + iprot.skip(ftype) + iprot.readFieldEnd() + iprot.readStructEnd() + + def write(self, oprot): + if oprot._fast_encode is not None and self.thrift_spec is not None: + oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec])) + return + oprot.writeStructBegin("get_serde_result") + if self.success is not None: + oprot.writeFieldBegin("success", TType.STRUCT, 0) + self.success.write(oprot) + oprot.writeFieldEnd() + if self.o1 is not None: + oprot.writeFieldBegin("o1", TType.STRUCT, 1) + self.o1.write(oprot) + oprot.writeFieldEnd() + if self.o2 is not None: + oprot.writeFieldBegin("o2", TType.STRUCT, 2) + self.o2.write(oprot) + oprot.writeFieldEnd() + oprot.writeFieldStop() + oprot.writeStructEnd() + + def validate(self): + return + + def __repr__(self): + L = ["{}={!r}".format(key, value) for key, value in self.__dict__.items()] + return "{}({})".format(self.__class__.__name__, ", ".join(L)) + + def __eq__(self, other): + return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ + + def __ne__(self, other): + return not (self == other) + + +all_structs.append(get_serde_result) +get_serde_result.thrift_spec = ( + ( + 0, + TType.STRUCT, + "success", + [SerDeInfo, None], + None, + ), # 0 + ( + 1, + TType.STRUCT, + "o1", + [NoSuchObjectException, None], + None, + ), # 1 + ( + 2, + TType.STRUCT, + "o2", + [MetaException, None], + None, + ), # 2 +) + + +class get_lock_materialization_rebuild_args: + """ + Attributes: + - dbName + - tableName + - txnId + + """ + + def __init__( + self, + dbName=None, + tableName=None, + txnId=None, + ): + self.dbName = dbName + self.tableName = tableName + self.txnId = txnId + + def read(self, iprot): + if ( + iprot._fast_decode is not None + and isinstance(iprot.trans, TTransport.CReadableTransport) + and self.thrift_spec is not None + ): + iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec]) + return + iprot.readStructBegin() + while True: + (fname, ftype, fid) = iprot.readFieldBegin() + if ftype == TType.STOP: + break + if fid == 1: + if ftype == TType.STRING: + self.dbName = ( + iprot.readString().decode("utf-8", errors="replace") if sys.version_info[0] == 2 else iprot.readString() + ) + else: + iprot.skip(ftype) + elif fid == 2: + if ftype == TType.STRING: + self.tableName = ( + iprot.readString().decode("utf-8", errors="replace") if sys.version_info[0] == 2 else iprot.readString() + ) + else: + iprot.skip(ftype) + elif fid == 3: + if ftype == TType.I64: + self.txnId = iprot.readI64() + else: + iprot.skip(ftype) + else: + iprot.skip(ftype) + iprot.readFieldEnd() + iprot.readStructEnd() + + def write(self, oprot): + if oprot._fast_encode is not None and self.thrift_spec is not None: + oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec])) + return + oprot.writeStructBegin("get_lock_materialization_rebuild_args") + if self.dbName is not None: + oprot.writeFieldBegin("dbName", TType.STRING, 1) + oprot.writeString(self.dbName.encode("utf-8") if sys.version_info[0] == 2 else self.dbName) + oprot.writeFieldEnd() + if self.tableName is not None: + oprot.writeFieldBegin("tableName", TType.STRING, 2) + oprot.writeString(self.tableName.encode("utf-8") if sys.version_info[0] == 2 else self.tableName) + oprot.writeFieldEnd() + if self.txnId is not None: + oprot.writeFieldBegin("txnId", TType.I64, 3) + oprot.writeI64(self.txnId) + oprot.writeFieldEnd() + oprot.writeFieldStop() + oprot.writeStructEnd() + + def validate(self): + return + + def __repr__(self): + L = ["{}={!r}".format(key, value) for key, value in self.__dict__.items()] + return "{}({})".format(self.__class__.__name__, ", ".join(L)) + + def __eq__(self, other): + return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ + + def __ne__(self, other): + return not (self == other) + + +all_structs.append(get_lock_materialization_rebuild_args) +get_lock_materialization_rebuild_args.thrift_spec = ( + None, # 0 + ( + 1, + TType.STRING, + "dbName", + "UTF8", + None, + ), # 1 + ( + 2, + TType.STRING, + "tableName", + "UTF8", + None, + ), # 2 + ( + 3, + TType.I64, + "txnId", + None, + None, + ), # 3 +) + + +class get_lock_materialization_rebuild_result: + """ + Attributes: + - success + + """ + + def __init__( + self, + success=None, + ): + self.success = success + + def read(self, iprot): + if ( + iprot._fast_decode is not None + and isinstance(iprot.trans, TTransport.CReadableTransport) + and self.thrift_spec is not None + ): + iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec]) + return + iprot.readStructBegin() + while True: + (fname, ftype, fid) = iprot.readFieldBegin() + if ftype == TType.STOP: + break + if fid == 0: + if ftype == TType.STRUCT: + self.success = LockResponse() + self.success.read(iprot) + else: + iprot.skip(ftype) + else: + iprot.skip(ftype) + iprot.readFieldEnd() + iprot.readStructEnd() + + def write(self, oprot): + if oprot._fast_encode is not None and self.thrift_spec is not None: + oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec])) + return + oprot.writeStructBegin("get_lock_materialization_rebuild_result") + if self.success is not None: + oprot.writeFieldBegin("success", TType.STRUCT, 0) + self.success.write(oprot) + oprot.writeFieldEnd() + oprot.writeFieldStop() + oprot.writeStructEnd() + + def validate(self): + return + + def __repr__(self): + L = ["{}={!r}".format(key, value) for key, value in self.__dict__.items()] + return "{}({})".format(self.__class__.__name__, ", ".join(L)) + + def __eq__(self, other): + return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ + + def __ne__(self, other): + return not (self == other) + + +all_structs.append(get_lock_materialization_rebuild_result) +get_lock_materialization_rebuild_result.thrift_spec = ( + ( + 0, + TType.STRUCT, + "success", + [LockResponse, None], + None, + ), # 0 +) + + +class heartbeat_lock_materialization_rebuild_args: + """ + Attributes: + - dbName + - tableName + - txnId + + """ + + def __init__( + self, + dbName=None, + tableName=None, + txnId=None, + ): + self.dbName = dbName + self.tableName = tableName + self.txnId = txnId + + def read(self, iprot): + if ( + iprot._fast_decode is not None + and isinstance(iprot.trans, TTransport.CReadableTransport) + and self.thrift_spec is not None + ): + iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec]) + return + iprot.readStructBegin() + while True: + (fname, ftype, fid) = iprot.readFieldBegin() + if ftype == TType.STOP: + break + if fid == 1: + if ftype == TType.STRING: + self.dbName = ( + iprot.readString().decode("utf-8", errors="replace") if sys.version_info[0] == 2 else iprot.readString() + ) + else: + iprot.skip(ftype) + elif fid == 2: + if ftype == TType.STRING: + self.tableName = ( + iprot.readString().decode("utf-8", errors="replace") if sys.version_info[0] == 2 else iprot.readString() + ) + else: + iprot.skip(ftype) + elif fid == 3: + if ftype == TType.I64: + self.txnId = iprot.readI64() + else: + iprot.skip(ftype) + else: + iprot.skip(ftype) + iprot.readFieldEnd() + iprot.readStructEnd() + + def write(self, oprot): + if oprot._fast_encode is not None and self.thrift_spec is not None: + oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec])) + return + oprot.writeStructBegin("heartbeat_lock_materialization_rebuild_args") + if self.dbName is not None: + oprot.writeFieldBegin("dbName", TType.STRING, 1) + oprot.writeString(self.dbName.encode("utf-8") if sys.version_info[0] == 2 else self.dbName) + oprot.writeFieldEnd() + if self.tableName is not None: + oprot.writeFieldBegin("tableName", TType.STRING, 2) + oprot.writeString(self.tableName.encode("utf-8") if sys.version_info[0] == 2 else self.tableName) + oprot.writeFieldEnd() + if self.txnId is not None: + oprot.writeFieldBegin("txnId", TType.I64, 3) + oprot.writeI64(self.txnId) + oprot.writeFieldEnd() + oprot.writeFieldStop() + oprot.writeStructEnd() + + def validate(self): + return + + def __repr__(self): + L = ["{}={!r}".format(key, value) for key, value in self.__dict__.items()] + return "{}({})".format(self.__class__.__name__, ", ".join(L)) + + def __eq__(self, other): + return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ + + def __ne__(self, other): + return not (self == other) + + +all_structs.append(heartbeat_lock_materialization_rebuild_args) +heartbeat_lock_materialization_rebuild_args.thrift_spec = ( + None, # 0 + ( + 1, + TType.STRING, + "dbName", + "UTF8", + None, + ), # 1 + ( + 2, + TType.STRING, + "tableName", + "UTF8", + None, + ), # 2 + ( + 3, + TType.I64, + "txnId", + None, + None, + ), # 3 +) + + +class heartbeat_lock_materialization_rebuild_result: + """ + Attributes: + - success + + """ + + def __init__( + self, + success=None, + ): + self.success = success + + def read(self, iprot): + if ( + iprot._fast_decode is not None + and isinstance(iprot.trans, TTransport.CReadableTransport) + and self.thrift_spec is not None + ): + iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec]) + return + iprot.readStructBegin() + while True: + (fname, ftype, fid) = iprot.readFieldBegin() + if ftype == TType.STOP: + break + if fid == 0: + if ftype == TType.BOOL: + self.success = iprot.readBool() + else: + iprot.skip(ftype) + else: + iprot.skip(ftype) + iprot.readFieldEnd() + iprot.readStructEnd() + + def write(self, oprot): + if oprot._fast_encode is not None and self.thrift_spec is not None: + oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec])) + return + oprot.writeStructBegin("heartbeat_lock_materialization_rebuild_result") + if self.success is not None: + oprot.writeFieldBegin("success", TType.BOOL, 0) + oprot.writeBool(self.success) + oprot.writeFieldEnd() + oprot.writeFieldStop() + oprot.writeStructEnd() + + def validate(self): + return + + def __repr__(self): + L = ["{}={!r}".format(key, value) for key, value in self.__dict__.items()] + return "{}({})".format(self.__class__.__name__, ", ".join(L)) + + def __eq__(self, other): + return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ + + def __ne__(self, other): + return not (self == other) + + +all_structs.append(heartbeat_lock_materialization_rebuild_result) +heartbeat_lock_materialization_rebuild_result.thrift_spec = ( + ( + 0, + TType.BOOL, + "success", + None, + None, + ), # 0 +) + + +class add_runtime_stats_args: + """ + Attributes: + - stat + + """ + + def __init__( + self, + stat=None, + ): + self.stat = stat + + def read(self, iprot): + if ( + iprot._fast_decode is not None + and isinstance(iprot.trans, TTransport.CReadableTransport) + and self.thrift_spec is not None + ): + iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec]) + return + iprot.readStructBegin() + while True: + (fname, ftype, fid) = iprot.readFieldBegin() + if ftype == TType.STOP: + break + if fid == 1: + if ftype == TType.STRUCT: + self.stat = RuntimeStat() + self.stat.read(iprot) + else: + iprot.skip(ftype) + else: + iprot.skip(ftype) + iprot.readFieldEnd() + iprot.readStructEnd() + + def write(self, oprot): + if oprot._fast_encode is not None and self.thrift_spec is not None: + oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec])) + return + oprot.writeStructBegin("add_runtime_stats_args") + if self.stat is not None: + oprot.writeFieldBegin("stat", TType.STRUCT, 1) + self.stat.write(oprot) + oprot.writeFieldEnd() + oprot.writeFieldStop() + oprot.writeStructEnd() + + def validate(self): + return + + def __repr__(self): + L = ["{}={!r}".format(key, value) for key, value in self.__dict__.items()] + return "{}({})".format(self.__class__.__name__, ", ".join(L)) + + def __eq__(self, other): + return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ + + def __ne__(self, other): + return not (self == other) + + +all_structs.append(add_runtime_stats_args) +add_runtime_stats_args.thrift_spec = ( + None, # 0 + ( + 1, + TType.STRUCT, + "stat", + [RuntimeStat, None], + None, + ), # 1 +) + + +class add_runtime_stats_result: + """ + Attributes: + - o1 + + """ + + def __init__( + self, + o1=None, + ): + self.o1 = o1 + + def read(self, iprot): + if ( + iprot._fast_decode is not None + and isinstance(iprot.trans, TTransport.CReadableTransport) + and self.thrift_spec is not None + ): + iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec]) + return + iprot.readStructBegin() + while True: + (fname, ftype, fid) = iprot.readFieldBegin() + if ftype == TType.STOP: + break + if fid == 1: + if ftype == TType.STRUCT: + self.o1 = MetaException.read(iprot) + else: + iprot.skip(ftype) + else: + iprot.skip(ftype) + iprot.readFieldEnd() + iprot.readStructEnd() + + def write(self, oprot): + if oprot._fast_encode is not None and self.thrift_spec is not None: + oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec])) + return + oprot.writeStructBegin("add_runtime_stats_result") + if self.o1 is not None: + oprot.writeFieldBegin("o1", TType.STRUCT, 1) + self.o1.write(oprot) + oprot.writeFieldEnd() + oprot.writeFieldStop() + oprot.writeStructEnd() + + def validate(self): + return + + def __repr__(self): + L = ["{}={!r}".format(key, value) for key, value in self.__dict__.items()] + return "{}({})".format(self.__class__.__name__, ", ".join(L)) + + def __eq__(self, other): + return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ + + def __ne__(self, other): + return not (self == other) + + +all_structs.append(add_runtime_stats_result) +add_runtime_stats_result.thrift_spec = ( + None, # 0 + ( + 1, + TType.STRUCT, + "o1", + [MetaException, None], + None, + ), # 1 +) + + +class get_runtime_stats_args: + """ + Attributes: + - rqst + + """ + + def __init__( + self, + rqst=None, + ): + self.rqst = rqst + + def read(self, iprot): + if ( + iprot._fast_decode is not None + and isinstance(iprot.trans, TTransport.CReadableTransport) + and self.thrift_spec is not None + ): + iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec]) + return + iprot.readStructBegin() + while True: + (fname, ftype, fid) = iprot.readFieldBegin() + if ftype == TType.STOP: + break + if fid == 1: + if ftype == TType.STRUCT: + self.rqst = GetRuntimeStatsRequest() + self.rqst.read(iprot) + else: + iprot.skip(ftype) + else: + iprot.skip(ftype) + iprot.readFieldEnd() + iprot.readStructEnd() + + def write(self, oprot): + if oprot._fast_encode is not None and self.thrift_spec is not None: + oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec])) + return + oprot.writeStructBegin("get_runtime_stats_args") + if self.rqst is not None: + oprot.writeFieldBegin("rqst", TType.STRUCT, 1) + self.rqst.write(oprot) + oprot.writeFieldEnd() + oprot.writeFieldStop() + oprot.writeStructEnd() + + def validate(self): + return + + def __repr__(self): + L = ["{}={!r}".format(key, value) for key, value in self.__dict__.items()] + return "{}({})".format(self.__class__.__name__, ", ".join(L)) + + def __eq__(self, other): + return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ + + def __ne__(self, other): + return not (self == other) + + +all_structs.append(get_runtime_stats_args) +get_runtime_stats_args.thrift_spec = ( + None, # 0 + ( + 1, + TType.STRUCT, + "rqst", + [GetRuntimeStatsRequest, None], + None, + ), # 1 +) + + +class get_runtime_stats_result: + """ + Attributes: + - success + - o1 + + """ + + def __init__( + self, + success=None, + o1=None, + ): + self.success = success + self.o1 = o1 + + def read(self, iprot): + if ( + iprot._fast_decode is not None + and isinstance(iprot.trans, TTransport.CReadableTransport) + and self.thrift_spec is not None + ): + iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec]) + return + iprot.readStructBegin() + while True: + (fname, ftype, fid) = iprot.readFieldBegin() + if ftype == TType.STOP: + break + if fid == 0: + if ftype == TType.LIST: + self.success = [] + (_etype1782, _size1779) = iprot.readListBegin() + for _i1783 in range(_size1779): + _elem1784 = RuntimeStat() + _elem1784.read(iprot) + self.success.append(_elem1784) + iprot.readListEnd() + else: + iprot.skip(ftype) + elif fid == 1: + if ftype == TType.STRUCT: + self.o1 = MetaException.read(iprot) + else: + iprot.skip(ftype) + else: + iprot.skip(ftype) + iprot.readFieldEnd() + iprot.readStructEnd() + + def write(self, oprot): + if oprot._fast_encode is not None and self.thrift_spec is not None: + oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec])) + return + oprot.writeStructBegin("get_runtime_stats_result") + if self.success is not None: + oprot.writeFieldBegin("success", TType.LIST, 0) + oprot.writeListBegin(TType.STRUCT, len(self.success)) + for iter1785 in self.success: + iter1785.write(oprot) + oprot.writeListEnd() + oprot.writeFieldEnd() + if self.o1 is not None: + oprot.writeFieldBegin("o1", TType.STRUCT, 1) + self.o1.write(oprot) + oprot.writeFieldEnd() + oprot.writeFieldStop() + oprot.writeStructEnd() + + def validate(self): + return + + def __repr__(self): + L = ["{}={!r}".format(key, value) for key, value in self.__dict__.items()] + return "{}({})".format(self.__class__.__name__, ", ".join(L)) + + def __eq__(self, other): + return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ + + def __ne__(self, other): + return not (self == other) + + +all_structs.append(get_runtime_stats_result) +get_runtime_stats_result.thrift_spec = ( + ( + 0, + TType.LIST, + "success", + (TType.STRUCT, [RuntimeStat, None], False), + None, + ), # 0 + ( + 1, + TType.STRUCT, + "o1", + [MetaException, None], + None, + ), # 1 +) + + +class get_partitions_with_specs_args: + """ + Attributes: + - request + + """ + + def __init__( + self, + request=None, + ): + self.request = request + + def read(self, iprot): + if ( + iprot._fast_decode is not None + and isinstance(iprot.trans, TTransport.CReadableTransport) + and self.thrift_spec is not None + ): + iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec]) + return + iprot.readStructBegin() + while True: + (fname, ftype, fid) = iprot.readFieldBegin() + if ftype == TType.STOP: + break + if fid == 1: + if ftype == TType.STRUCT: + self.request = GetPartitionsRequest() + self.request.read(iprot) + else: + iprot.skip(ftype) + else: + iprot.skip(ftype) + iprot.readFieldEnd() + iprot.readStructEnd() + + def write(self, oprot): + if oprot._fast_encode is not None and self.thrift_spec is not None: + oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec])) + return + oprot.writeStructBegin("get_partitions_with_specs_args") + if self.request is not None: + oprot.writeFieldBegin("request", TType.STRUCT, 1) + self.request.write(oprot) + oprot.writeFieldEnd() + oprot.writeFieldStop() + oprot.writeStructEnd() + + def validate(self): + return + + def __repr__(self): + L = ["{}={!r}".format(key, value) for key, value in self.__dict__.items()] + return "{}({})".format(self.__class__.__name__, ", ".join(L)) + + def __eq__(self, other): + return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ + + def __ne__(self, other): + return not (self == other) + + +all_structs.append(get_partitions_with_specs_args) +get_partitions_with_specs_args.thrift_spec = ( + None, # 0 + ( + 1, + TType.STRUCT, + "request", + [GetPartitionsRequest, None], + None, + ), # 1 +) + + +class get_partitions_with_specs_result: + """ + Attributes: + - success + - o1 + + """ + + def __init__( + self, + success=None, + o1=None, + ): + self.success = success + self.o1 = o1 + + def read(self, iprot): + if ( + iprot._fast_decode is not None + and isinstance(iprot.trans, TTransport.CReadableTransport) + and self.thrift_spec is not None + ): + iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec]) + return + iprot.readStructBegin() + while True: + (fname, ftype, fid) = iprot.readFieldBegin() + if ftype == TType.STOP: + break + if fid == 0: + if ftype == TType.STRUCT: + self.success = GetPartitionsResponse() + self.success.read(iprot) + else: + iprot.skip(ftype) + elif fid == 1: + if ftype == TType.STRUCT: + self.o1 = MetaException.read(iprot) + else: + iprot.skip(ftype) + else: + iprot.skip(ftype) + iprot.readFieldEnd() + iprot.readStructEnd() + + def write(self, oprot): + if oprot._fast_encode is not None and self.thrift_spec is not None: + oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec])) + return + oprot.writeStructBegin("get_partitions_with_specs_result") + if self.success is not None: + oprot.writeFieldBegin("success", TType.STRUCT, 0) + self.success.write(oprot) + oprot.writeFieldEnd() + if self.o1 is not None: + oprot.writeFieldBegin("o1", TType.STRUCT, 1) + self.o1.write(oprot) + oprot.writeFieldEnd() + oprot.writeFieldStop() + oprot.writeStructEnd() + + def validate(self): + return + + def __repr__(self): + L = ["{}={!r}".format(key, value) for key, value in self.__dict__.items()] + return "{}({})".format(self.__class__.__name__, ", ".join(L)) + + def __eq__(self, other): + return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ + + def __ne__(self, other): + return not (self == other) + + +all_structs.append(get_partitions_with_specs_result) +get_partitions_with_specs_result.thrift_spec = ( + ( + 0, + TType.STRUCT, + "success", + [GetPartitionsResponse, None], + None, + ), # 0 + ( + 1, + TType.STRUCT, + "o1", + [MetaException, None], + None, + ), # 1 +) + + +class scheduled_query_poll_args: + """ + Attributes: + - request + + """ + + def __init__( + self, + request=None, + ): + self.request = request + + def read(self, iprot): + if ( + iprot._fast_decode is not None + and isinstance(iprot.trans, TTransport.CReadableTransport) + and self.thrift_spec is not None + ): + iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec]) + return + iprot.readStructBegin() + while True: + (fname, ftype, fid) = iprot.readFieldBegin() + if ftype == TType.STOP: + break + if fid == 1: + if ftype == TType.STRUCT: + self.request = ScheduledQueryPollRequest() + self.request.read(iprot) + else: + iprot.skip(ftype) + else: + iprot.skip(ftype) + iprot.readFieldEnd() + iprot.readStructEnd() + + def write(self, oprot): + if oprot._fast_encode is not None and self.thrift_spec is not None: + oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec])) + return + oprot.writeStructBegin("scheduled_query_poll_args") + if self.request is not None: + oprot.writeFieldBegin("request", TType.STRUCT, 1) + self.request.write(oprot) + oprot.writeFieldEnd() + oprot.writeFieldStop() + oprot.writeStructEnd() + + def validate(self): + return + + def __repr__(self): + L = ["{}={!r}".format(key, value) for key, value in self.__dict__.items()] + return "{}({})".format(self.__class__.__name__, ", ".join(L)) + + def __eq__(self, other): + return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ + + def __ne__(self, other): + return not (self == other) + + +all_structs.append(scheduled_query_poll_args) +scheduled_query_poll_args.thrift_spec = ( + None, # 0 + ( + 1, + TType.STRUCT, + "request", + [ScheduledQueryPollRequest, None], + None, + ), # 1 +) + + +class scheduled_query_poll_result: + """ + Attributes: + - success + - o1 + + """ + + def __init__( + self, + success=None, + o1=None, + ): + self.success = success + self.o1 = o1 + + def read(self, iprot): + if ( + iprot._fast_decode is not None + and isinstance(iprot.trans, TTransport.CReadableTransport) + and self.thrift_spec is not None + ): + iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec]) + return + iprot.readStructBegin() + while True: + (fname, ftype, fid) = iprot.readFieldBegin() + if ftype == TType.STOP: + break + if fid == 0: + if ftype == TType.STRUCT: + self.success = ScheduledQueryPollResponse() + self.success.read(iprot) + else: + iprot.skip(ftype) + elif fid == 1: + if ftype == TType.STRUCT: + self.o1 = MetaException.read(iprot) + else: + iprot.skip(ftype) + else: + iprot.skip(ftype) + iprot.readFieldEnd() + iprot.readStructEnd() + + def write(self, oprot): + if oprot._fast_encode is not None and self.thrift_spec is not None: + oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec])) + return + oprot.writeStructBegin("scheduled_query_poll_result") + if self.success is not None: + oprot.writeFieldBegin("success", TType.STRUCT, 0) + self.success.write(oprot) + oprot.writeFieldEnd() + if self.o1 is not None: + oprot.writeFieldBegin("o1", TType.STRUCT, 1) + self.o1.write(oprot) + oprot.writeFieldEnd() + oprot.writeFieldStop() + oprot.writeStructEnd() + + def validate(self): + return + + def __repr__(self): + L = ["{}={!r}".format(key, value) for key, value in self.__dict__.items()] + return "{}({})".format(self.__class__.__name__, ", ".join(L)) + + def __eq__(self, other): + return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ + + def __ne__(self, other): + return not (self == other) + + +all_structs.append(scheduled_query_poll_result) +scheduled_query_poll_result.thrift_spec = ( + ( + 0, + TType.STRUCT, + "success", + [ScheduledQueryPollResponse, None], + None, + ), # 0 + ( + 1, + TType.STRUCT, + "o1", + [MetaException, None], + None, + ), # 1 +) + + +class scheduled_query_maintenance_args: + """ + Attributes: + - request + + """ + + def __init__( + self, + request=None, + ): + self.request = request + + def read(self, iprot): + if ( + iprot._fast_decode is not None + and isinstance(iprot.trans, TTransport.CReadableTransport) + and self.thrift_spec is not None + ): + iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec]) + return + iprot.readStructBegin() + while True: + (fname, ftype, fid) = iprot.readFieldBegin() + if ftype == TType.STOP: + break + if fid == 1: + if ftype == TType.STRUCT: + self.request = ScheduledQueryMaintenanceRequest() + self.request.read(iprot) + else: + iprot.skip(ftype) + else: + iprot.skip(ftype) + iprot.readFieldEnd() + iprot.readStructEnd() + + def write(self, oprot): + if oprot._fast_encode is not None and self.thrift_spec is not None: + oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec])) + return + oprot.writeStructBegin("scheduled_query_maintenance_args") + if self.request is not None: + oprot.writeFieldBegin("request", TType.STRUCT, 1) + self.request.write(oprot) + oprot.writeFieldEnd() + oprot.writeFieldStop() + oprot.writeStructEnd() + + def validate(self): + return + + def __repr__(self): + L = ["{}={!r}".format(key, value) for key, value in self.__dict__.items()] + return "{}({})".format(self.__class__.__name__, ", ".join(L)) + + def __eq__(self, other): + return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ + + def __ne__(self, other): + return not (self == other) + + +all_structs.append(scheduled_query_maintenance_args) +scheduled_query_maintenance_args.thrift_spec = ( + None, # 0 + ( + 1, + TType.STRUCT, + "request", + [ScheduledQueryMaintenanceRequest, None], + None, + ), # 1 +) + + +class scheduled_query_maintenance_result: + """ + Attributes: + - o1 + - o2 + - o3 + - o4 + + """ + + def __init__( + self, + o1=None, + o2=None, + o3=None, + o4=None, + ): + self.o1 = o1 + self.o2 = o2 + self.o3 = o3 + self.o4 = o4 + + def read(self, iprot): + if ( + iprot._fast_decode is not None + and isinstance(iprot.trans, TTransport.CReadableTransport) + and self.thrift_spec is not None + ): + iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec]) + return + iprot.readStructBegin() + while True: + (fname, ftype, fid) = iprot.readFieldBegin() + if ftype == TType.STOP: + break + if fid == 1: + if ftype == TType.STRUCT: + self.o1 = MetaException.read(iprot) + else: + iprot.skip(ftype) + elif fid == 2: + if ftype == TType.STRUCT: + self.o2 = NoSuchObjectException.read(iprot) + else: + iprot.skip(ftype) + elif fid == 3: + if ftype == TType.STRUCT: + self.o3 = AlreadyExistsException.read(iprot) + else: + iprot.skip(ftype) + elif fid == 4: + if ftype == TType.STRUCT: + self.o4 = InvalidInputException.read(iprot) + else: + iprot.skip(ftype) + else: + iprot.skip(ftype) + iprot.readFieldEnd() + iprot.readStructEnd() + + def write(self, oprot): + if oprot._fast_encode is not None and self.thrift_spec is not None: + oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec])) + return + oprot.writeStructBegin("scheduled_query_maintenance_result") + if self.o1 is not None: + oprot.writeFieldBegin("o1", TType.STRUCT, 1) + self.o1.write(oprot) + oprot.writeFieldEnd() + if self.o2 is not None: + oprot.writeFieldBegin("o2", TType.STRUCT, 2) + self.o2.write(oprot) + oprot.writeFieldEnd() + if self.o3 is not None: + oprot.writeFieldBegin("o3", TType.STRUCT, 3) + self.o3.write(oprot) + oprot.writeFieldEnd() + if self.o4 is not None: + oprot.writeFieldBegin("o4", TType.STRUCT, 4) + self.o4.write(oprot) + oprot.writeFieldEnd() + oprot.writeFieldStop() + oprot.writeStructEnd() + + def validate(self): + return + + def __repr__(self): + L = ["{}={!r}".format(key, value) for key, value in self.__dict__.items()] + return "{}({})".format(self.__class__.__name__, ", ".join(L)) + + def __eq__(self, other): + return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ + + def __ne__(self, other): + return not (self == other) + + +all_structs.append(scheduled_query_maintenance_result) +scheduled_query_maintenance_result.thrift_spec = ( + None, # 0 + ( + 1, + TType.STRUCT, + "o1", + [MetaException, None], + None, + ), # 1 + ( + 2, + TType.STRUCT, + "o2", + [NoSuchObjectException, None], + None, + ), # 2 + ( + 3, + TType.STRUCT, + "o3", + [AlreadyExistsException, None], + None, + ), # 3 + ( + 4, + TType.STRUCT, + "o4", + [InvalidInputException, None], + None, + ), # 4 +) + + +class scheduled_query_progress_args: + """ + Attributes: + - info + + """ + + def __init__( + self, + info=None, + ): + self.info = info + + def read(self, iprot): + if ( + iprot._fast_decode is not None + and isinstance(iprot.trans, TTransport.CReadableTransport) + and self.thrift_spec is not None + ): + iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec]) + return + iprot.readStructBegin() + while True: + (fname, ftype, fid) = iprot.readFieldBegin() + if ftype == TType.STOP: + break + if fid == 1: + if ftype == TType.STRUCT: + self.info = ScheduledQueryProgressInfo() + self.info.read(iprot) + else: + iprot.skip(ftype) + else: + iprot.skip(ftype) + iprot.readFieldEnd() + iprot.readStructEnd() + + def write(self, oprot): + if oprot._fast_encode is not None and self.thrift_spec is not None: + oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec])) + return + oprot.writeStructBegin("scheduled_query_progress_args") + if self.info is not None: + oprot.writeFieldBegin("info", TType.STRUCT, 1) + self.info.write(oprot) + oprot.writeFieldEnd() + oprot.writeFieldStop() + oprot.writeStructEnd() + + def validate(self): + return + + def __repr__(self): + L = ["{}={!r}".format(key, value) for key, value in self.__dict__.items()] + return "{}({})".format(self.__class__.__name__, ", ".join(L)) + + def __eq__(self, other): + return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ + + def __ne__(self, other): + return not (self == other) + + +all_structs.append(scheduled_query_progress_args) +scheduled_query_progress_args.thrift_spec = ( + None, # 0 + ( + 1, + TType.STRUCT, + "info", + [ScheduledQueryProgressInfo, None], + None, + ), # 1 +) + + +class scheduled_query_progress_result: + """ + Attributes: + - o1 + - o2 + + """ + + def __init__( + self, + o1=None, + o2=None, + ): + self.o1 = o1 + self.o2 = o2 + + def read(self, iprot): + if ( + iprot._fast_decode is not None + and isinstance(iprot.trans, TTransport.CReadableTransport) + and self.thrift_spec is not None + ): + iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec]) + return + iprot.readStructBegin() + while True: + (fname, ftype, fid) = iprot.readFieldBegin() + if ftype == TType.STOP: + break + if fid == 1: + if ftype == TType.STRUCT: + self.o1 = MetaException.read(iprot) + else: + iprot.skip(ftype) + elif fid == 2: + if ftype == TType.STRUCT: + self.o2 = InvalidOperationException.read(iprot) + else: + iprot.skip(ftype) + else: + iprot.skip(ftype) + iprot.readFieldEnd() + iprot.readStructEnd() + + def write(self, oprot): + if oprot._fast_encode is not None and self.thrift_spec is not None: + oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec])) + return + oprot.writeStructBegin("scheduled_query_progress_result") + if self.o1 is not None: + oprot.writeFieldBegin("o1", TType.STRUCT, 1) + self.o1.write(oprot) + oprot.writeFieldEnd() + if self.o2 is not None: + oprot.writeFieldBegin("o2", TType.STRUCT, 2) + self.o2.write(oprot) + oprot.writeFieldEnd() + oprot.writeFieldStop() + oprot.writeStructEnd() + + def validate(self): + return + + def __repr__(self): + L = ["{}={!r}".format(key, value) for key, value in self.__dict__.items()] + return "{}({})".format(self.__class__.__name__, ", ".join(L)) + + def __eq__(self, other): + return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ + + def __ne__(self, other): + return not (self == other) + + +all_structs.append(scheduled_query_progress_result) +scheduled_query_progress_result.thrift_spec = ( + None, # 0 + ( + 1, + TType.STRUCT, + "o1", + [MetaException, None], + None, + ), # 1 + ( + 2, + TType.STRUCT, + "o2", + [InvalidOperationException, None], + None, + ), # 2 +) + + +class get_scheduled_query_args: + """ + Attributes: + - scheduleKey + + """ + + def __init__( + self, + scheduleKey=None, + ): + self.scheduleKey = scheduleKey + + def read(self, iprot): + if ( + iprot._fast_decode is not None + and isinstance(iprot.trans, TTransport.CReadableTransport) + and self.thrift_spec is not None + ): + iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec]) + return + iprot.readStructBegin() + while True: + (fname, ftype, fid) = iprot.readFieldBegin() + if ftype == TType.STOP: + break + if fid == 1: + if ftype == TType.STRUCT: + self.scheduleKey = ScheduledQueryKey() + self.scheduleKey.read(iprot) + else: + iprot.skip(ftype) + else: + iprot.skip(ftype) + iprot.readFieldEnd() + iprot.readStructEnd() + + def write(self, oprot): + if oprot._fast_encode is not None and self.thrift_spec is not None: + oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec])) + return + oprot.writeStructBegin("get_scheduled_query_args") + if self.scheduleKey is not None: + oprot.writeFieldBegin("scheduleKey", TType.STRUCT, 1) + self.scheduleKey.write(oprot) + oprot.writeFieldEnd() + oprot.writeFieldStop() + oprot.writeStructEnd() + + def validate(self): + return + + def __repr__(self): + L = ["{}={!r}".format(key, value) for key, value in self.__dict__.items()] + return "{}({})".format(self.__class__.__name__, ", ".join(L)) + + def __eq__(self, other): + return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ + + def __ne__(self, other): + return not (self == other) + + +all_structs.append(get_scheduled_query_args) +get_scheduled_query_args.thrift_spec = ( + None, # 0 + ( + 1, + TType.STRUCT, + "scheduleKey", + [ScheduledQueryKey, None], + None, + ), # 1 +) + + +class get_scheduled_query_result: + """ + Attributes: + - success + - o1 + - o2 + + """ + + def __init__( + self, + success=None, + o1=None, + o2=None, + ): + self.success = success + self.o1 = o1 + self.o2 = o2 + + def read(self, iprot): + if ( + iprot._fast_decode is not None + and isinstance(iprot.trans, TTransport.CReadableTransport) + and self.thrift_spec is not None + ): + iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec]) + return + iprot.readStructBegin() + while True: + (fname, ftype, fid) = iprot.readFieldBegin() + if ftype == TType.STOP: + break + if fid == 0: + if ftype == TType.STRUCT: + self.success = ScheduledQuery() + self.success.read(iprot) + else: + iprot.skip(ftype) + elif fid == 1: + if ftype == TType.STRUCT: + self.o1 = MetaException.read(iprot) + else: + iprot.skip(ftype) + elif fid == 2: + if ftype == TType.STRUCT: + self.o2 = NoSuchObjectException.read(iprot) + else: + iprot.skip(ftype) + else: + iprot.skip(ftype) + iprot.readFieldEnd() + iprot.readStructEnd() + + def write(self, oprot): + if oprot._fast_encode is not None and self.thrift_spec is not None: + oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec])) + return + oprot.writeStructBegin("get_scheduled_query_result") + if self.success is not None: + oprot.writeFieldBegin("success", TType.STRUCT, 0) + self.success.write(oprot) + oprot.writeFieldEnd() + if self.o1 is not None: + oprot.writeFieldBegin("o1", TType.STRUCT, 1) + self.o1.write(oprot) + oprot.writeFieldEnd() + if self.o2 is not None: + oprot.writeFieldBegin("o2", TType.STRUCT, 2) + self.o2.write(oprot) + oprot.writeFieldEnd() + oprot.writeFieldStop() + oprot.writeStructEnd() + + def validate(self): + return + + def __repr__(self): + L = ["{}={!r}".format(key, value) for key, value in self.__dict__.items()] + return "{}({})".format(self.__class__.__name__, ", ".join(L)) + + def __eq__(self, other): + return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ + + def __ne__(self, other): + return not (self == other) + + +all_structs.append(get_scheduled_query_result) +get_scheduled_query_result.thrift_spec = ( + ( + 0, + TType.STRUCT, + "success", + [ScheduledQuery, None], + None, + ), # 0 + ( + 1, + TType.STRUCT, + "o1", + [MetaException, None], + None, + ), # 1 + ( + 2, + TType.STRUCT, + "o2", + [NoSuchObjectException, None], + None, + ), # 2 +) + + +class add_replication_metrics_args: + """ + Attributes: + - replicationMetricList + + """ + + def __init__( + self, + replicationMetricList=None, + ): + self.replicationMetricList = replicationMetricList + + def read(self, iprot): + if ( + iprot._fast_decode is not None + and isinstance(iprot.trans, TTransport.CReadableTransport) + and self.thrift_spec is not None + ): + iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec]) + return + iprot.readStructBegin() + while True: + (fname, ftype, fid) = iprot.readFieldBegin() + if ftype == TType.STOP: + break + if fid == 1: + if ftype == TType.STRUCT: + self.replicationMetricList = ReplicationMetricList() + self.replicationMetricList.read(iprot) + else: + iprot.skip(ftype) + else: + iprot.skip(ftype) + iprot.readFieldEnd() + iprot.readStructEnd() + + def write(self, oprot): + if oprot._fast_encode is not None and self.thrift_spec is not None: + oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec])) + return + oprot.writeStructBegin("add_replication_metrics_args") + if self.replicationMetricList is not None: + oprot.writeFieldBegin("replicationMetricList", TType.STRUCT, 1) + self.replicationMetricList.write(oprot) + oprot.writeFieldEnd() + oprot.writeFieldStop() + oprot.writeStructEnd() + + def validate(self): + return + + def __repr__(self): + L = ["{}={!r}".format(key, value) for key, value in self.__dict__.items()] + return "{}({})".format(self.__class__.__name__, ", ".join(L)) + + def __eq__(self, other): + return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ + + def __ne__(self, other): + return not (self == other) + + +all_structs.append(add_replication_metrics_args) +add_replication_metrics_args.thrift_spec = ( + None, # 0 + ( + 1, + TType.STRUCT, + "replicationMetricList", + [ReplicationMetricList, None], + None, + ), # 1 +) + + +class add_replication_metrics_result: + """ + Attributes: + - o1 + + """ + + def __init__( + self, + o1=None, + ): + self.o1 = o1 + + def read(self, iprot): + if ( + iprot._fast_decode is not None + and isinstance(iprot.trans, TTransport.CReadableTransport) + and self.thrift_spec is not None + ): + iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec]) + return + iprot.readStructBegin() + while True: + (fname, ftype, fid) = iprot.readFieldBegin() + if ftype == TType.STOP: + break + if fid == 1: + if ftype == TType.STRUCT: + self.o1 = MetaException.read(iprot) + else: + iprot.skip(ftype) + else: + iprot.skip(ftype) + iprot.readFieldEnd() + iprot.readStructEnd() + + def write(self, oprot): + if oprot._fast_encode is not None and self.thrift_spec is not None: + oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec])) + return + oprot.writeStructBegin("add_replication_metrics_result") + if self.o1 is not None: + oprot.writeFieldBegin("o1", TType.STRUCT, 1) + self.o1.write(oprot) + oprot.writeFieldEnd() + oprot.writeFieldStop() + oprot.writeStructEnd() + + def validate(self): + return + + def __repr__(self): + L = ["{}={!r}".format(key, value) for key, value in self.__dict__.items()] + return "{}({})".format(self.__class__.__name__, ", ".join(L)) + + def __eq__(self, other): + return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ + + def __ne__(self, other): + return not (self == other) + + +all_structs.append(add_replication_metrics_result) +add_replication_metrics_result.thrift_spec = ( + None, # 0 + ( + 1, + TType.STRUCT, + "o1", + [MetaException, None], + None, + ), # 1 +) + + +class get_replication_metrics_args: + """ + Attributes: + - rqst + + """ + + def __init__( + self, + rqst=None, + ): + self.rqst = rqst + + def read(self, iprot): + if ( + iprot._fast_decode is not None + and isinstance(iprot.trans, TTransport.CReadableTransport) + and self.thrift_spec is not None + ): + iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec]) + return + iprot.readStructBegin() + while True: + (fname, ftype, fid) = iprot.readFieldBegin() + if ftype == TType.STOP: + break + if fid == 1: + if ftype == TType.STRUCT: + self.rqst = GetReplicationMetricsRequest() + self.rqst.read(iprot) + else: + iprot.skip(ftype) + else: + iprot.skip(ftype) + iprot.readFieldEnd() + iprot.readStructEnd() + + def write(self, oprot): + if oprot._fast_encode is not None and self.thrift_spec is not None: + oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec])) + return + oprot.writeStructBegin("get_replication_metrics_args") + if self.rqst is not None: + oprot.writeFieldBegin("rqst", TType.STRUCT, 1) + self.rqst.write(oprot) + oprot.writeFieldEnd() + oprot.writeFieldStop() + oprot.writeStructEnd() + + def validate(self): + return + + def __repr__(self): + L = ["{}={!r}".format(key, value) for key, value in self.__dict__.items()] + return "{}({})".format(self.__class__.__name__, ", ".join(L)) + + def __eq__(self, other): + return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ + + def __ne__(self, other): + return not (self == other) + + +all_structs.append(get_replication_metrics_args) +get_replication_metrics_args.thrift_spec = ( + None, # 0 + ( + 1, + TType.STRUCT, + "rqst", + [GetReplicationMetricsRequest, None], + None, + ), # 1 +) + + +class get_replication_metrics_result: + """ + Attributes: + - success + - o1 + + """ + + def __init__( + self, + success=None, + o1=None, + ): + self.success = success + self.o1 = o1 + + def read(self, iprot): + if ( + iprot._fast_decode is not None + and isinstance(iprot.trans, TTransport.CReadableTransport) + and self.thrift_spec is not None + ): + iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec]) + return + iprot.readStructBegin() + while True: + (fname, ftype, fid) = iprot.readFieldBegin() + if ftype == TType.STOP: + break + if fid == 0: + if ftype == TType.STRUCT: + self.success = ReplicationMetricList() + self.success.read(iprot) + else: + iprot.skip(ftype) + elif fid == 1: + if ftype == TType.STRUCT: + self.o1 = MetaException.read(iprot) + else: + iprot.skip(ftype) + else: + iprot.skip(ftype) + iprot.readFieldEnd() + iprot.readStructEnd() + + def write(self, oprot): + if oprot._fast_encode is not None and self.thrift_spec is not None: + oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec])) + return + oprot.writeStructBegin("get_replication_metrics_result") + if self.success is not None: + oprot.writeFieldBegin("success", TType.STRUCT, 0) + self.success.write(oprot) + oprot.writeFieldEnd() + if self.o1 is not None: + oprot.writeFieldBegin("o1", TType.STRUCT, 1) + self.o1.write(oprot) + oprot.writeFieldEnd() + oprot.writeFieldStop() + oprot.writeStructEnd() + + def validate(self): + return + + def __repr__(self): + L = ["{}={!r}".format(key, value) for key, value in self.__dict__.items()] + return "{}({})".format(self.__class__.__name__, ", ".join(L)) + + def __eq__(self, other): + return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ + + def __ne__(self, other): + return not (self == other) + + +all_structs.append(get_replication_metrics_result) +get_replication_metrics_result.thrift_spec = ( + ( + 0, + TType.STRUCT, + "success", + [ReplicationMetricList, None], + None, + ), # 0 + ( + 1, + TType.STRUCT, + "o1", + [MetaException, None], + None, + ), # 1 +) + + +class get_open_txns_req_args: + """ + Attributes: + - getOpenTxnsRequest + + """ + + def __init__( + self, + getOpenTxnsRequest=None, + ): + self.getOpenTxnsRequest = getOpenTxnsRequest + + def read(self, iprot): + if ( + iprot._fast_decode is not None + and isinstance(iprot.trans, TTransport.CReadableTransport) + and self.thrift_spec is not None + ): + iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec]) + return + iprot.readStructBegin() + while True: + (fname, ftype, fid) = iprot.readFieldBegin() + if ftype == TType.STOP: + break + if fid == 1: + if ftype == TType.STRUCT: + self.getOpenTxnsRequest = GetOpenTxnsRequest() + self.getOpenTxnsRequest.read(iprot) + else: + iprot.skip(ftype) + else: + iprot.skip(ftype) + iprot.readFieldEnd() + iprot.readStructEnd() + + def write(self, oprot): + if oprot._fast_encode is not None and self.thrift_spec is not None: + oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec])) + return + oprot.writeStructBegin("get_open_txns_req_args") + if self.getOpenTxnsRequest is not None: + oprot.writeFieldBegin("getOpenTxnsRequest", TType.STRUCT, 1) + self.getOpenTxnsRequest.write(oprot) + oprot.writeFieldEnd() + oprot.writeFieldStop() + oprot.writeStructEnd() + + def validate(self): + return + + def __repr__(self): + L = ["{}={!r}".format(key, value) for key, value in self.__dict__.items()] + return "{}({})".format(self.__class__.__name__, ", ".join(L)) + + def __eq__(self, other): + return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ + + def __ne__(self, other): + return not (self == other) + + +all_structs.append(get_open_txns_req_args) +get_open_txns_req_args.thrift_spec = ( + None, # 0 + ( + 1, + TType.STRUCT, + "getOpenTxnsRequest", + [GetOpenTxnsRequest, None], + None, + ), # 1 +) + + +class get_open_txns_req_result: + """ + Attributes: + - success + + """ + + def __init__( + self, + success=None, + ): + self.success = success + + def read(self, iprot): + if ( + iprot._fast_decode is not None + and isinstance(iprot.trans, TTransport.CReadableTransport) + and self.thrift_spec is not None + ): + iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec]) + return + iprot.readStructBegin() + while True: + (fname, ftype, fid) = iprot.readFieldBegin() + if ftype == TType.STOP: + break + if fid == 0: + if ftype == TType.STRUCT: + self.success = GetOpenTxnsResponse() + self.success.read(iprot) + else: + iprot.skip(ftype) + else: + iprot.skip(ftype) + iprot.readFieldEnd() + iprot.readStructEnd() + + def write(self, oprot): + if oprot._fast_encode is not None and self.thrift_spec is not None: + oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec])) + return + oprot.writeStructBegin("get_open_txns_req_result") + if self.success is not None: + oprot.writeFieldBegin("success", TType.STRUCT, 0) + self.success.write(oprot) + oprot.writeFieldEnd() + oprot.writeFieldStop() + oprot.writeStructEnd() + + def validate(self): + return + + def __repr__(self): + L = ["{}={!r}".format(key, value) for key, value in self.__dict__.items()] + return "{}({})".format(self.__class__.__name__, ", ".join(L)) + + def __eq__(self, other): + return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ + + def __ne__(self, other): + return not (self == other) + + +all_structs.append(get_open_txns_req_result) +get_open_txns_req_result.thrift_spec = ( + ( + 0, + TType.STRUCT, + "success", + [GetOpenTxnsResponse, None], + None, + ), # 0 +) + + +class create_stored_procedure_args: + """ + Attributes: + - proc + + """ + + def __init__( + self, + proc=None, + ): + self.proc = proc + + def read(self, iprot): + if ( + iprot._fast_decode is not None + and isinstance(iprot.trans, TTransport.CReadableTransport) + and self.thrift_spec is not None + ): + iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec]) + return + iprot.readStructBegin() + while True: + (fname, ftype, fid) = iprot.readFieldBegin() + if ftype == TType.STOP: + break + if fid == 1: + if ftype == TType.STRUCT: + self.proc = StoredProcedure() + self.proc.read(iprot) + else: + iprot.skip(ftype) + else: + iprot.skip(ftype) + iprot.readFieldEnd() + iprot.readStructEnd() + + def write(self, oprot): + if oprot._fast_encode is not None and self.thrift_spec is not None: + oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec])) + return + oprot.writeStructBegin("create_stored_procedure_args") + if self.proc is not None: + oprot.writeFieldBegin("proc", TType.STRUCT, 1) + self.proc.write(oprot) + oprot.writeFieldEnd() + oprot.writeFieldStop() + oprot.writeStructEnd() + + def validate(self): + return + + def __repr__(self): + L = ["{}={!r}".format(key, value) for key, value in self.__dict__.items()] + return "{}({})".format(self.__class__.__name__, ", ".join(L)) + + def __eq__(self, other): + return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ + + def __ne__(self, other): + return not (self == other) + + +all_structs.append(create_stored_procedure_args) +create_stored_procedure_args.thrift_spec = ( + None, # 0 + ( + 1, + TType.STRUCT, + "proc", + [StoredProcedure, None], + None, + ), # 1 +) + + +class create_stored_procedure_result: + """ + Attributes: + - o1 + - o2 + + """ + + def __init__( + self, + o1=None, + o2=None, + ): + self.o1 = o1 + self.o2 = o2 + + def read(self, iprot): + if ( + iprot._fast_decode is not None + and isinstance(iprot.trans, TTransport.CReadableTransport) + and self.thrift_spec is not None + ): + iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec]) + return + iprot.readStructBegin() + while True: + (fname, ftype, fid) = iprot.readFieldBegin() + if ftype == TType.STOP: + break + if fid == 1: + if ftype == TType.STRUCT: + self.o1 = NoSuchObjectException.read(iprot) + else: + iprot.skip(ftype) + elif fid == 2: + if ftype == TType.STRUCT: + self.o2 = MetaException.read(iprot) + else: + iprot.skip(ftype) + else: + iprot.skip(ftype) + iprot.readFieldEnd() + iprot.readStructEnd() + + def write(self, oprot): + if oprot._fast_encode is not None and self.thrift_spec is not None: + oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec])) + return + oprot.writeStructBegin("create_stored_procedure_result") + if self.o1 is not None: + oprot.writeFieldBegin("o1", TType.STRUCT, 1) + self.o1.write(oprot) + oprot.writeFieldEnd() + if self.o2 is not None: + oprot.writeFieldBegin("o2", TType.STRUCT, 2) + self.o2.write(oprot) + oprot.writeFieldEnd() + oprot.writeFieldStop() + oprot.writeStructEnd() + + def validate(self): + return + + def __repr__(self): + L = ["{}={!r}".format(key, value) for key, value in self.__dict__.items()] + return "{}({})".format(self.__class__.__name__, ", ".join(L)) + + def __eq__(self, other): + return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ + + def __ne__(self, other): + return not (self == other) + + +all_structs.append(create_stored_procedure_result) +create_stored_procedure_result.thrift_spec = ( + None, # 0 + ( + 1, + TType.STRUCT, + "o1", + [NoSuchObjectException, None], + None, + ), # 1 + ( + 2, + TType.STRUCT, + "o2", + [MetaException, None], + None, + ), # 2 +) + + +class get_stored_procedure_args: + """ + Attributes: + - request + + """ + + def __init__( + self, + request=None, + ): + self.request = request + + def read(self, iprot): + if ( + iprot._fast_decode is not None + and isinstance(iprot.trans, TTransport.CReadableTransport) + and self.thrift_spec is not None + ): + iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec]) + return + iprot.readStructBegin() + while True: + (fname, ftype, fid) = iprot.readFieldBegin() + if ftype == TType.STOP: + break + if fid == 1: + if ftype == TType.STRUCT: + self.request = StoredProcedureRequest() + self.request.read(iprot) + else: + iprot.skip(ftype) + else: + iprot.skip(ftype) + iprot.readFieldEnd() + iprot.readStructEnd() + + def write(self, oprot): + if oprot._fast_encode is not None and self.thrift_spec is not None: + oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec])) + return + oprot.writeStructBegin("get_stored_procedure_args") + if self.request is not None: + oprot.writeFieldBegin("request", TType.STRUCT, 1) + self.request.write(oprot) + oprot.writeFieldEnd() + oprot.writeFieldStop() + oprot.writeStructEnd() + + def validate(self): + return + + def __repr__(self): + L = ["{}={!r}".format(key, value) for key, value in self.__dict__.items()] + return "{}({})".format(self.__class__.__name__, ", ".join(L)) + + def __eq__(self, other): + return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ + + def __ne__(self, other): + return not (self == other) + + +all_structs.append(get_stored_procedure_args) +get_stored_procedure_args.thrift_spec = ( + None, # 0 + ( + 1, + TType.STRUCT, + "request", + [StoredProcedureRequest, None], + None, + ), # 1 +) + + +class get_stored_procedure_result: + """ + Attributes: + - success + - o1 + - o2 + + """ + + def __init__( + self, + success=None, + o1=None, + o2=None, + ): + self.success = success + self.o1 = o1 + self.o2 = o2 + + def read(self, iprot): + if ( + iprot._fast_decode is not None + and isinstance(iprot.trans, TTransport.CReadableTransport) + and self.thrift_spec is not None + ): + iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec]) + return + iprot.readStructBegin() + while True: + (fname, ftype, fid) = iprot.readFieldBegin() + if ftype == TType.STOP: + break + if fid == 0: + if ftype == TType.STRUCT: + self.success = StoredProcedure() + self.success.read(iprot) + else: + iprot.skip(ftype) + elif fid == 1: + if ftype == TType.STRUCT: + self.o1 = MetaException.read(iprot) + else: + iprot.skip(ftype) + elif fid == 2: + if ftype == TType.STRUCT: + self.o2 = NoSuchObjectException.read(iprot) + else: + iprot.skip(ftype) + else: + iprot.skip(ftype) + iprot.readFieldEnd() + iprot.readStructEnd() + + def write(self, oprot): + if oprot._fast_encode is not None and self.thrift_spec is not None: + oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec])) + return + oprot.writeStructBegin("get_stored_procedure_result") + if self.success is not None: + oprot.writeFieldBegin("success", TType.STRUCT, 0) + self.success.write(oprot) + oprot.writeFieldEnd() + if self.o1 is not None: + oprot.writeFieldBegin("o1", TType.STRUCT, 1) + self.o1.write(oprot) + oprot.writeFieldEnd() + if self.o2 is not None: + oprot.writeFieldBegin("o2", TType.STRUCT, 2) + self.o2.write(oprot) + oprot.writeFieldEnd() + oprot.writeFieldStop() + oprot.writeStructEnd() + + def validate(self): + return + + def __repr__(self): + L = ["{}={!r}".format(key, value) for key, value in self.__dict__.items()] + return "{}({})".format(self.__class__.__name__, ", ".join(L)) + + def __eq__(self, other): + return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ + + def __ne__(self, other): + return not (self == other) + + +all_structs.append(get_stored_procedure_result) +get_stored_procedure_result.thrift_spec = ( + ( + 0, + TType.STRUCT, + "success", + [StoredProcedure, None], + None, + ), # 0 + ( + 1, + TType.STRUCT, + "o1", + [MetaException, None], + None, + ), # 1 + ( + 2, + TType.STRUCT, + "o2", + [NoSuchObjectException, None], + None, + ), # 2 +) + + +class drop_stored_procedure_args: + """ + Attributes: + - request + + """ + + def __init__( + self, + request=None, + ): + self.request = request + + def read(self, iprot): + if ( + iprot._fast_decode is not None + and isinstance(iprot.trans, TTransport.CReadableTransport) + and self.thrift_spec is not None + ): + iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec]) + return + iprot.readStructBegin() + while True: + (fname, ftype, fid) = iprot.readFieldBegin() + if ftype == TType.STOP: + break + if fid == 1: + if ftype == TType.STRUCT: + self.request = StoredProcedureRequest() + self.request.read(iprot) + else: + iprot.skip(ftype) + else: + iprot.skip(ftype) + iprot.readFieldEnd() + iprot.readStructEnd() + + def write(self, oprot): + if oprot._fast_encode is not None and self.thrift_spec is not None: + oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec])) + return + oprot.writeStructBegin("drop_stored_procedure_args") + if self.request is not None: + oprot.writeFieldBegin("request", TType.STRUCT, 1) + self.request.write(oprot) + oprot.writeFieldEnd() + oprot.writeFieldStop() + oprot.writeStructEnd() + + def validate(self): + return + + def __repr__(self): + L = ["{}={!r}".format(key, value) for key, value in self.__dict__.items()] + return "{}({})".format(self.__class__.__name__, ", ".join(L)) + + def __eq__(self, other): + return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ + + def __ne__(self, other): + return not (self == other) + + +all_structs.append(drop_stored_procedure_args) +drop_stored_procedure_args.thrift_spec = ( + None, # 0 + ( + 1, + TType.STRUCT, + "request", + [StoredProcedureRequest, None], + None, + ), # 1 +) + + +class drop_stored_procedure_result: + """ + Attributes: + - o1 + + """ + + def __init__( + self, + o1=None, + ): + self.o1 = o1 + + def read(self, iprot): + if ( + iprot._fast_decode is not None + and isinstance(iprot.trans, TTransport.CReadableTransport) + and self.thrift_spec is not None + ): + iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec]) + return + iprot.readStructBegin() + while True: + (fname, ftype, fid) = iprot.readFieldBegin() + if ftype == TType.STOP: + break + if fid == 1: + if ftype == TType.STRUCT: + self.o1 = MetaException.read(iprot) + else: + iprot.skip(ftype) + else: + iprot.skip(ftype) + iprot.readFieldEnd() + iprot.readStructEnd() + + def write(self, oprot): + if oprot._fast_encode is not None and self.thrift_spec is not None: + oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec])) + return + oprot.writeStructBegin("drop_stored_procedure_result") + if self.o1 is not None: + oprot.writeFieldBegin("o1", TType.STRUCT, 1) + self.o1.write(oprot) + oprot.writeFieldEnd() + oprot.writeFieldStop() + oprot.writeStructEnd() + + def validate(self): + return + + def __repr__(self): + L = ["{}={!r}".format(key, value) for key, value in self.__dict__.items()] + return "{}({})".format(self.__class__.__name__, ", ".join(L)) + + def __eq__(self, other): + return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ + + def __ne__(self, other): + return not (self == other) + + +all_structs.append(drop_stored_procedure_result) +drop_stored_procedure_result.thrift_spec = ( + None, # 0 + ( + 1, + TType.STRUCT, + "o1", + [MetaException, None], + None, + ), # 1 +) + + +class get_all_stored_procedures_args: + """ + Attributes: + - request + + """ + + def __init__( + self, + request=None, + ): + self.request = request + + def read(self, iprot): + if ( + iprot._fast_decode is not None + and isinstance(iprot.trans, TTransport.CReadableTransport) + and self.thrift_spec is not None + ): + iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec]) + return + iprot.readStructBegin() + while True: + (fname, ftype, fid) = iprot.readFieldBegin() + if ftype == TType.STOP: + break + if fid == 1: + if ftype == TType.STRUCT: + self.request = ListStoredProcedureRequest() + self.request.read(iprot) + else: + iprot.skip(ftype) + else: + iprot.skip(ftype) + iprot.readFieldEnd() + iprot.readStructEnd() + + def write(self, oprot): + if oprot._fast_encode is not None and self.thrift_spec is not None: + oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec])) + return + oprot.writeStructBegin("get_all_stored_procedures_args") + if self.request is not None: + oprot.writeFieldBegin("request", TType.STRUCT, 1) + self.request.write(oprot) + oprot.writeFieldEnd() + oprot.writeFieldStop() + oprot.writeStructEnd() + + def validate(self): + return + + def __repr__(self): + L = ["{}={!r}".format(key, value) for key, value in self.__dict__.items()] + return "{}({})".format(self.__class__.__name__, ", ".join(L)) + + def __eq__(self, other): + return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ + + def __ne__(self, other): + return not (self == other) + + +all_structs.append(get_all_stored_procedures_args) +get_all_stored_procedures_args.thrift_spec = ( + None, # 0 + ( + 1, + TType.STRUCT, + "request", + [ListStoredProcedureRequest, None], + None, + ), # 1 +) + + +class get_all_stored_procedures_result: + """ + Attributes: + - success + - o1 + + """ + + def __init__( + self, + success=None, + o1=None, + ): + self.success = success + self.o1 = o1 + + def read(self, iprot): + if ( + iprot._fast_decode is not None + and isinstance(iprot.trans, TTransport.CReadableTransport) + and self.thrift_spec is not None + ): + iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec]) + return + iprot.readStructBegin() + while True: + (fname, ftype, fid) = iprot.readFieldBegin() + if ftype == TType.STOP: + break + if fid == 0: + if ftype == TType.LIST: + self.success = [] + (_etype1789, _size1786) = iprot.readListBegin() + for _i1790 in range(_size1786): + _elem1791 = ( + iprot.readString().decode("utf-8", errors="replace") + if sys.version_info[0] == 2 + else iprot.readString() + ) + self.success.append(_elem1791) + iprot.readListEnd() + else: + iprot.skip(ftype) + elif fid == 1: + if ftype == TType.STRUCT: + self.o1 = MetaException.read(iprot) + else: + iprot.skip(ftype) + else: + iprot.skip(ftype) + iprot.readFieldEnd() + iprot.readStructEnd() + + def write(self, oprot): + if oprot._fast_encode is not None and self.thrift_spec is not None: + oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec])) + return + oprot.writeStructBegin("get_all_stored_procedures_result") + if self.success is not None: + oprot.writeFieldBegin("success", TType.LIST, 0) + oprot.writeListBegin(TType.STRING, len(self.success)) + for iter1792 in self.success: + oprot.writeString(iter1792.encode("utf-8") if sys.version_info[0] == 2 else iter1792) + oprot.writeListEnd() + oprot.writeFieldEnd() + if self.o1 is not None: + oprot.writeFieldBegin("o1", TType.STRUCT, 1) + self.o1.write(oprot) + oprot.writeFieldEnd() + oprot.writeFieldStop() + oprot.writeStructEnd() + + def validate(self): + return + + def __repr__(self): + L = ["{}={!r}".format(key, value) for key, value in self.__dict__.items()] + return "{}({})".format(self.__class__.__name__, ", ".join(L)) + + def __eq__(self, other): + return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ + + def __ne__(self, other): + return not (self == other) + + +all_structs.append(get_all_stored_procedures_result) +get_all_stored_procedures_result.thrift_spec = ( + ( + 0, + TType.LIST, + "success", + (TType.STRING, "UTF8", False), + None, + ), # 0 + ( + 1, + TType.STRUCT, + "o1", + [MetaException, None], + None, + ), # 1 +) + + +class find_package_args: + """ + Attributes: + - request + + """ + + def __init__( + self, + request=None, + ): + self.request = request + + def read(self, iprot): + if ( + iprot._fast_decode is not None + and isinstance(iprot.trans, TTransport.CReadableTransport) + and self.thrift_spec is not None + ): + iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec]) + return + iprot.readStructBegin() + while True: + (fname, ftype, fid) = iprot.readFieldBegin() + if ftype == TType.STOP: + break + if fid == 1: + if ftype == TType.STRUCT: + self.request = GetPackageRequest() + self.request.read(iprot) + else: + iprot.skip(ftype) + else: + iprot.skip(ftype) + iprot.readFieldEnd() + iprot.readStructEnd() + + def write(self, oprot): + if oprot._fast_encode is not None and self.thrift_spec is not None: + oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec])) + return + oprot.writeStructBegin("find_package_args") + if self.request is not None: + oprot.writeFieldBegin("request", TType.STRUCT, 1) + self.request.write(oprot) + oprot.writeFieldEnd() + oprot.writeFieldStop() + oprot.writeStructEnd() + + def validate(self): + return + + def __repr__(self): + L = ["{}={!r}".format(key, value) for key, value in self.__dict__.items()] + return "{}({})".format(self.__class__.__name__, ", ".join(L)) + + def __eq__(self, other): + return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ + + def __ne__(self, other): + return not (self == other) + + +all_structs.append(find_package_args) +find_package_args.thrift_spec = ( + None, # 0 + ( + 1, + TType.STRUCT, + "request", + [GetPackageRequest, None], + None, + ), # 1 +) + + +class find_package_result: + """ + Attributes: + - success + - o1 + - o2 + + """ + + def __init__( + self, + success=None, + o1=None, + o2=None, + ): + self.success = success + self.o1 = o1 + self.o2 = o2 + + def read(self, iprot): + if ( + iprot._fast_decode is not None + and isinstance(iprot.trans, TTransport.CReadableTransport) + and self.thrift_spec is not None + ): + iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec]) + return + iprot.readStructBegin() + while True: + (fname, ftype, fid) = iprot.readFieldBegin() + if ftype == TType.STOP: + break + if fid == 0: + if ftype == TType.STRUCT: + self.success = Package() + self.success.read(iprot) + else: + iprot.skip(ftype) + elif fid == 1: + if ftype == TType.STRUCT: + self.o1 = MetaException.read(iprot) + else: + iprot.skip(ftype) + elif fid == 2: + if ftype == TType.STRUCT: + self.o2 = NoSuchObjectException.read(iprot) + else: + iprot.skip(ftype) + else: + iprot.skip(ftype) + iprot.readFieldEnd() + iprot.readStructEnd() + + def write(self, oprot): + if oprot._fast_encode is not None and self.thrift_spec is not None: + oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec])) + return + oprot.writeStructBegin("find_package_result") + if self.success is not None: + oprot.writeFieldBegin("success", TType.STRUCT, 0) + self.success.write(oprot) + oprot.writeFieldEnd() + if self.o1 is not None: + oprot.writeFieldBegin("o1", TType.STRUCT, 1) + self.o1.write(oprot) + oprot.writeFieldEnd() + if self.o2 is not None: + oprot.writeFieldBegin("o2", TType.STRUCT, 2) + self.o2.write(oprot) + oprot.writeFieldEnd() + oprot.writeFieldStop() + oprot.writeStructEnd() + + def validate(self): + return + + def __repr__(self): + L = ["{}={!r}".format(key, value) for key, value in self.__dict__.items()] + return "{}({})".format(self.__class__.__name__, ", ".join(L)) + + def __eq__(self, other): + return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ + + def __ne__(self, other): + return not (self == other) + + +all_structs.append(find_package_result) +find_package_result.thrift_spec = ( + ( + 0, + TType.STRUCT, + "success", + [Package, None], + None, + ), # 0 + ( + 1, + TType.STRUCT, + "o1", + [MetaException, None], + None, + ), # 1 + ( + 2, + TType.STRUCT, + "o2", + [NoSuchObjectException, None], + None, + ), # 2 +) + + +class add_package_args: + """ + Attributes: + - request + + """ + + def __init__( + self, + request=None, + ): + self.request = request + + def read(self, iprot): + if ( + iprot._fast_decode is not None + and isinstance(iprot.trans, TTransport.CReadableTransport) + and self.thrift_spec is not None + ): + iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec]) + return + iprot.readStructBegin() + while True: + (fname, ftype, fid) = iprot.readFieldBegin() + if ftype == TType.STOP: + break + if fid == 1: + if ftype == TType.STRUCT: + self.request = AddPackageRequest() + self.request.read(iprot) + else: + iprot.skip(ftype) + else: + iprot.skip(ftype) + iprot.readFieldEnd() + iprot.readStructEnd() + + def write(self, oprot): + if oprot._fast_encode is not None and self.thrift_spec is not None: + oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec])) + return + oprot.writeStructBegin("add_package_args") + if self.request is not None: + oprot.writeFieldBegin("request", TType.STRUCT, 1) + self.request.write(oprot) + oprot.writeFieldEnd() + oprot.writeFieldStop() + oprot.writeStructEnd() + + def validate(self): + return + + def __repr__(self): + L = ["{}={!r}".format(key, value) for key, value in self.__dict__.items()] + return "{}({})".format(self.__class__.__name__, ", ".join(L)) + + def __eq__(self, other): + return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ + + def __ne__(self, other): + return not (self == other) + + +all_structs.append(add_package_args) +add_package_args.thrift_spec = ( + None, # 0 + ( + 1, + TType.STRUCT, + "request", + [AddPackageRequest, None], + None, + ), # 1 +) + + +class add_package_result: + """ + Attributes: + - o1 + + """ + + def __init__( + self, + o1=None, + ): + self.o1 = o1 + + def read(self, iprot): + if ( + iprot._fast_decode is not None + and isinstance(iprot.trans, TTransport.CReadableTransport) + and self.thrift_spec is not None + ): + iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec]) + return + iprot.readStructBegin() + while True: + (fname, ftype, fid) = iprot.readFieldBegin() + if ftype == TType.STOP: + break + if fid == 1: + if ftype == TType.STRUCT: + self.o1 = MetaException.read(iprot) + else: + iprot.skip(ftype) + else: + iprot.skip(ftype) + iprot.readFieldEnd() + iprot.readStructEnd() + + def write(self, oprot): + if oprot._fast_encode is not None and self.thrift_spec is not None: + oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec])) + return + oprot.writeStructBegin("add_package_result") + if self.o1 is not None: + oprot.writeFieldBegin("o1", TType.STRUCT, 1) + self.o1.write(oprot) + oprot.writeFieldEnd() + oprot.writeFieldStop() + oprot.writeStructEnd() + + def validate(self): + return + + def __repr__(self): + L = ["{}={!r}".format(key, value) for key, value in self.__dict__.items()] + return "{}({})".format(self.__class__.__name__, ", ".join(L)) + + def __eq__(self, other): + return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ + + def __ne__(self, other): + return not (self == other) + + +all_structs.append(add_package_result) +add_package_result.thrift_spec = ( + None, # 0 + ( + 1, + TType.STRUCT, + "o1", + [MetaException, None], + None, + ), # 1 +) + + +class get_all_packages_args: + """ + Attributes: + - request + + """ + + def __init__( + self, + request=None, + ): + self.request = request + + def read(self, iprot): + if ( + iprot._fast_decode is not None + and isinstance(iprot.trans, TTransport.CReadableTransport) + and self.thrift_spec is not None + ): + iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec]) + return + iprot.readStructBegin() + while True: + (fname, ftype, fid) = iprot.readFieldBegin() + if ftype == TType.STOP: + break + if fid == 1: + if ftype == TType.STRUCT: + self.request = ListPackageRequest() + self.request.read(iprot) + else: + iprot.skip(ftype) + else: + iprot.skip(ftype) + iprot.readFieldEnd() + iprot.readStructEnd() + + def write(self, oprot): + if oprot._fast_encode is not None and self.thrift_spec is not None: + oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec])) + return + oprot.writeStructBegin("get_all_packages_args") + if self.request is not None: + oprot.writeFieldBegin("request", TType.STRUCT, 1) + self.request.write(oprot) + oprot.writeFieldEnd() + oprot.writeFieldStop() + oprot.writeStructEnd() + + def validate(self): + return + + def __repr__(self): + L = ["{}={!r}".format(key, value) for key, value in self.__dict__.items()] + return "{}({})".format(self.__class__.__name__, ", ".join(L)) + + def __eq__(self, other): + return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ + + def __ne__(self, other): + return not (self == other) + + +all_structs.append(get_all_packages_args) +get_all_packages_args.thrift_spec = ( + None, # 0 + ( + 1, + TType.STRUCT, + "request", + [ListPackageRequest, None], + None, + ), # 1 +) + + +class get_all_packages_result: + """ + Attributes: + - success + - o1 + + """ + + def __init__( + self, + success=None, + o1=None, + ): + self.success = success + self.o1 = o1 + + def read(self, iprot): + if ( + iprot._fast_decode is not None + and isinstance(iprot.trans, TTransport.CReadableTransport) + and self.thrift_spec is not None + ): + iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec]) + return + iprot.readStructBegin() + while True: + (fname, ftype, fid) = iprot.readFieldBegin() + if ftype == TType.STOP: + break + if fid == 0: + if ftype == TType.LIST: + self.success = [] + (_etype1796, _size1793) = iprot.readListBegin() + for _i1797 in range(_size1793): + _elem1798 = ( + iprot.readString().decode("utf-8", errors="replace") + if sys.version_info[0] == 2 + else iprot.readString() + ) + self.success.append(_elem1798) + iprot.readListEnd() + else: + iprot.skip(ftype) + elif fid == 1: + if ftype == TType.STRUCT: + self.o1 = MetaException.read(iprot) + else: + iprot.skip(ftype) + else: + iprot.skip(ftype) + iprot.readFieldEnd() + iprot.readStructEnd() + + def write(self, oprot): + if oprot._fast_encode is not None and self.thrift_spec is not None: + oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec])) + return + oprot.writeStructBegin("get_all_packages_result") + if self.success is not None: + oprot.writeFieldBegin("success", TType.LIST, 0) + oprot.writeListBegin(TType.STRING, len(self.success)) + for iter1799 in self.success: + oprot.writeString(iter1799.encode("utf-8") if sys.version_info[0] == 2 else iter1799) + oprot.writeListEnd() + oprot.writeFieldEnd() + if self.o1 is not None: + oprot.writeFieldBegin("o1", TType.STRUCT, 1) + self.o1.write(oprot) + oprot.writeFieldEnd() + oprot.writeFieldStop() + oprot.writeStructEnd() + + def validate(self): + return + + def __repr__(self): + L = ["{}={!r}".format(key, value) for key, value in self.__dict__.items()] + return "{}({})".format(self.__class__.__name__, ", ".join(L)) + + def __eq__(self, other): + return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ + + def __ne__(self, other): + return not (self == other) + + +all_structs.append(get_all_packages_result) +get_all_packages_result.thrift_spec = ( + ( + 0, + TType.LIST, + "success", + (TType.STRING, "UTF8", False), + None, + ), # 0 + ( + 1, + TType.STRUCT, + "o1", + [MetaException, None], + None, + ), # 1 +) + + +class drop_package_args: + """ + Attributes: + - request + + """ + + def __init__( + self, + request=None, + ): + self.request = request + + def read(self, iprot): + if ( + iprot._fast_decode is not None + and isinstance(iprot.trans, TTransport.CReadableTransport) + and self.thrift_spec is not None + ): + iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec]) + return + iprot.readStructBegin() + while True: + (fname, ftype, fid) = iprot.readFieldBegin() + if ftype == TType.STOP: + break + if fid == 1: + if ftype == TType.STRUCT: + self.request = DropPackageRequest() + self.request.read(iprot) + else: + iprot.skip(ftype) + else: + iprot.skip(ftype) + iprot.readFieldEnd() + iprot.readStructEnd() + + def write(self, oprot): + if oprot._fast_encode is not None and self.thrift_spec is not None: + oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec])) + return + oprot.writeStructBegin("drop_package_args") + if self.request is not None: + oprot.writeFieldBegin("request", TType.STRUCT, 1) + self.request.write(oprot) + oprot.writeFieldEnd() + oprot.writeFieldStop() + oprot.writeStructEnd() + + def validate(self): + return + + def __repr__(self): + L = ["{}={!r}".format(key, value) for key, value in self.__dict__.items()] + return "{}({})".format(self.__class__.__name__, ", ".join(L)) + + def __eq__(self, other): + return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ + + def __ne__(self, other): + return not (self == other) + + +all_structs.append(drop_package_args) +drop_package_args.thrift_spec = ( + None, # 0 + ( + 1, + TType.STRUCT, + "request", + [DropPackageRequest, None], + None, + ), # 1 +) + + +class drop_package_result: + """ + Attributes: + - o1 + + """ + + def __init__( + self, + o1=None, + ): + self.o1 = o1 + + def read(self, iprot): + if ( + iprot._fast_decode is not None + and isinstance(iprot.trans, TTransport.CReadableTransport) + and self.thrift_spec is not None + ): + iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec]) + return + iprot.readStructBegin() + while True: + (fname, ftype, fid) = iprot.readFieldBegin() + if ftype == TType.STOP: + break + if fid == 1: + if ftype == TType.STRUCT: + self.o1 = MetaException.read(iprot) + else: + iprot.skip(ftype) + else: + iprot.skip(ftype) + iprot.readFieldEnd() + iprot.readStructEnd() + + def write(self, oprot): + if oprot._fast_encode is not None and self.thrift_spec is not None: + oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec])) + return + oprot.writeStructBegin("drop_package_result") + if self.o1 is not None: + oprot.writeFieldBegin("o1", TType.STRUCT, 1) + self.o1.write(oprot) + oprot.writeFieldEnd() + oprot.writeFieldStop() + oprot.writeStructEnd() + + def validate(self): + return + + def __repr__(self): + L = ["{}={!r}".format(key, value) for key, value in self.__dict__.items()] + return "{}({})".format(self.__class__.__name__, ", ".join(L)) + + def __eq__(self, other): + return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ + + def __ne__(self, other): + return not (self == other) + + +all_structs.append(drop_package_result) +drop_package_result.thrift_spec = ( + None, # 0 + ( + 1, + TType.STRUCT, + "o1", + [MetaException, None], + None, + ), # 1 +) + + +class get_all_write_event_info_args: + """ + Attributes: + - request + + """ + + def __init__( + self, + request=None, + ): + self.request = request + + def read(self, iprot): + if ( + iprot._fast_decode is not None + and isinstance(iprot.trans, TTransport.CReadableTransport) + and self.thrift_spec is not None + ): + iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec]) + return + iprot.readStructBegin() + while True: + (fname, ftype, fid) = iprot.readFieldBegin() + if ftype == TType.STOP: + break + if fid == 1: + if ftype == TType.STRUCT: + self.request = GetAllWriteEventInfoRequest() + self.request.read(iprot) + else: + iprot.skip(ftype) + else: + iprot.skip(ftype) + iprot.readFieldEnd() + iprot.readStructEnd() + + def write(self, oprot): + if oprot._fast_encode is not None and self.thrift_spec is not None: + oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec])) + return + oprot.writeStructBegin("get_all_write_event_info_args") + if self.request is not None: + oprot.writeFieldBegin("request", TType.STRUCT, 1) + self.request.write(oprot) + oprot.writeFieldEnd() + oprot.writeFieldStop() + oprot.writeStructEnd() + + def validate(self): + return + + def __repr__(self): + L = ["{}={!r}".format(key, value) for key, value in self.__dict__.items()] + return "{}({})".format(self.__class__.__name__, ", ".join(L)) + + def __eq__(self, other): + return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ + + def __ne__(self, other): + return not (self == other) + + +all_structs.append(get_all_write_event_info_args) +get_all_write_event_info_args.thrift_spec = ( + None, # 0 + ( + 1, + TType.STRUCT, + "request", + [GetAllWriteEventInfoRequest, None], + None, + ), # 1 +) + + +class get_all_write_event_info_result: + """ + Attributes: + - success + - o1 + + """ + + def __init__( + self, + success=None, + o1=None, + ): + self.success = success + self.o1 = o1 + + def read(self, iprot): + if ( + iprot._fast_decode is not None + and isinstance(iprot.trans, TTransport.CReadableTransport) + and self.thrift_spec is not None + ): + iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec]) + return + iprot.readStructBegin() + while True: + (fname, ftype, fid) = iprot.readFieldBegin() + if ftype == TType.STOP: + break + if fid == 0: + if ftype == TType.LIST: + self.success = [] + (_etype1803, _size1800) = iprot.readListBegin() + for _i1804 in range(_size1800): + _elem1805 = WriteEventInfo() + _elem1805.read(iprot) + self.success.append(_elem1805) + iprot.readListEnd() + else: + iprot.skip(ftype) + elif fid == 1: + if ftype == TType.STRUCT: + self.o1 = MetaException.read(iprot) + else: + iprot.skip(ftype) + else: + iprot.skip(ftype) + iprot.readFieldEnd() + iprot.readStructEnd() + + def write(self, oprot): + if oprot._fast_encode is not None and self.thrift_spec is not None: + oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec])) + return + oprot.writeStructBegin("get_all_write_event_info_result") + if self.success is not None: + oprot.writeFieldBegin("success", TType.LIST, 0) + oprot.writeListBegin(TType.STRUCT, len(self.success)) + for iter1806 in self.success: + iter1806.write(oprot) + oprot.writeListEnd() + oprot.writeFieldEnd() + if self.o1 is not None: + oprot.writeFieldBegin("o1", TType.STRUCT, 1) + self.o1.write(oprot) + oprot.writeFieldEnd() + oprot.writeFieldStop() + oprot.writeStructEnd() + + def validate(self): + return + + def __repr__(self): + L = ["{}={!r}".format(key, value) for key, value in self.__dict__.items()] + return "{}({})".format(self.__class__.__name__, ", ".join(L)) + + def __eq__(self, other): + return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ + + def __ne__(self, other): + return not (self == other) + + +all_structs.append(get_all_write_event_info_result) +get_all_write_event_info_result.thrift_spec = ( + ( + 0, + TType.LIST, + "success", + (TType.STRUCT, [WriteEventInfo, None], False), + None, + ), # 0 + ( + 1, + TType.STRUCT, + "o1", + [MetaException, None], + None, + ), # 1 +) +fix_spec(all_structs) +del all_structs diff --git a/vendor/hive_metastore/__init__.py b/vendor/hive_metastore/__init__.py new file mode 100644 index 0000000000..178d664d81 --- /dev/null +++ b/vendor/hive_metastore/__init__.py @@ -0,0 +1,17 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +__all__ = ["ttypes", "constants", "ThriftHiveMetastore"] diff --git a/vendor/hive_metastore/constants.py b/vendor/hive_metastore/constants.py new file mode 100644 index 0000000000..218e527553 --- /dev/null +++ b/vendor/hive_metastore/constants.py @@ -0,0 +1,66 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# +# Autogenerated by Thrift Compiler (0.16.0) +# +# DO NOT EDIT UNLESS YOU ARE SURE THAT YOU KNOW WHAT YOU ARE DOING +# +# options string: py +# + + + + +DDL_TIME = "transient_lastDdlTime" +ACCESSTYPE_NONE = 1 +ACCESSTYPE_READONLY = 2 +ACCESSTYPE_WRITEONLY = 4 +ACCESSTYPE_READWRITE = 8 +HIVE_FILTER_FIELD_OWNER = "hive_filter_field_owner__" +HIVE_FILTER_FIELD_PARAMS = "hive_filter_field_params__" +HIVE_FILTER_FIELD_LAST_ACCESS = "hive_filter_field_last_access__" +IS_ARCHIVED = "is_archived" +ORIGINAL_LOCATION = "original_location" +IS_IMMUTABLE = "immutable" +META_TABLE_COLUMNS = "columns" +META_TABLE_COLUMN_TYPES = "columns.types" +BUCKET_FIELD_NAME = "bucket_field_name" +BUCKET_COUNT = "bucket_count" +FIELD_TO_DIMENSION = "field_to_dimension" +META_TABLE_NAME = "name" +META_TABLE_DB = "db" +META_TABLE_LOCATION = "location" +META_TABLE_SERDE = "serde" +META_TABLE_PARTITION_COLUMNS = "partition_columns" +META_TABLE_PARTITION_COLUMN_TYPES = "partition_columns.types" +FILE_INPUT_FORMAT = "file.inputformat" +FILE_OUTPUT_FORMAT = "file.outputformat" +META_TABLE_STORAGE = "storage_handler" +TABLE_IS_TRANSACTIONAL = "transactional" +TABLE_NO_AUTO_COMPACT = "no_auto_compaction" +TABLE_TRANSACTIONAL_PROPERTIES = "transactional_properties" +TABLE_BUCKETING_VERSION = "bucketing_version" +DRUID_CONFIG_PREFIX = "druid." +JDBC_CONFIG_PREFIX = "hive.sql." +TABLE_IS_CTAS = "created_with_ctas" +TABLE_IS_CTLT = "created_with_ctlt" +PARTITION_TRANSFORM_SPEC = "partition_transform_spec" +NO_CLEANUP = "no_cleanup" +CTAS_LEGACY_CONFIG = "create_table_as_external" +DEFAULT_TABLE_TYPE = "defaultTableType" +TXN_ID = "txnId" +WRITE_ID = "writeId" diff --git a/vendor/hive_metastore/ttypes.py b/vendor/hive_metastore/ttypes.py new file mode 100644 index 0000000000..dca7aaadc7 --- /dev/null +++ b/vendor/hive_metastore/ttypes.py @@ -0,0 +1,42515 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# +# Autogenerated by Thrift Compiler (0.16.0) +# +# DO NOT EDIT UNLESS YOU ARE SURE THAT YOU KNOW WHAT YOU ARE DOING +# +# options string: py +# + +import sys + +from thrift.protocol.TProtocol import TProtocolException +from thrift.Thrift import ( + TException, + TType, +) +from thrift.transport import TTransport +from thrift.TRecursive import fix_spec + +all_structs = [] + + +class HiveObjectType: + GLOBAL = 1 + DATABASE = 2 + TABLE = 3 + PARTITION = 4 + COLUMN = 5 + DATACONNECTOR = 6 + + _VALUES_TO_NAMES = { + 1: "GLOBAL", + 2: "DATABASE", + 3: "TABLE", + 4: "PARTITION", + 5: "COLUMN", + 6: "DATACONNECTOR", + } + + _NAMES_TO_VALUES = { + "GLOBAL": 1, + "DATABASE": 2, + "TABLE": 3, + "PARTITION": 4, + "COLUMN": 5, + "DATACONNECTOR": 6, + } + + +class PrincipalType: + USER = 1 + ROLE = 2 + GROUP = 3 + + _VALUES_TO_NAMES = { + 1: "USER", + 2: "ROLE", + 3: "GROUP", + } + + _NAMES_TO_VALUES = { + "USER": 1, + "ROLE": 2, + "GROUP": 3, + } + + +class PartitionEventType: + LOAD_DONE = 1 + + _VALUES_TO_NAMES = { + 1: "LOAD_DONE", + } + + _NAMES_TO_VALUES = { + "LOAD_DONE": 1, + } + + +class TxnState: + COMMITTED = 1 + ABORTED = 2 + OPEN = 3 + + _VALUES_TO_NAMES = { + 1: "COMMITTED", + 2: "ABORTED", + 3: "OPEN", + } + + _NAMES_TO_VALUES = { + "COMMITTED": 1, + "ABORTED": 2, + "OPEN": 3, + } + + +class LockLevel: + DB = 1 + TABLE = 2 + PARTITION = 3 + + _VALUES_TO_NAMES = { + 1: "DB", + 2: "TABLE", + 3: "PARTITION", + } + + _NAMES_TO_VALUES = { + "DB": 1, + "TABLE": 2, + "PARTITION": 3, + } + + +class LockState: + ACQUIRED = 1 + WAITING = 2 + ABORT = 3 + NOT_ACQUIRED = 4 + + _VALUES_TO_NAMES = { + 1: "ACQUIRED", + 2: "WAITING", + 3: "ABORT", + 4: "NOT_ACQUIRED", + } + + _NAMES_TO_VALUES = { + "ACQUIRED": 1, + "WAITING": 2, + "ABORT": 3, + "NOT_ACQUIRED": 4, + } + + +class LockType: + SHARED_READ = 1 + SHARED_WRITE = 2 + EXCLUSIVE = 3 + EXCL_WRITE = 4 + + _VALUES_TO_NAMES = { + 1: "SHARED_READ", + 2: "SHARED_WRITE", + 3: "EXCLUSIVE", + 4: "EXCL_WRITE", + } + + _NAMES_TO_VALUES = { + "SHARED_READ": 1, + "SHARED_WRITE": 2, + "EXCLUSIVE": 3, + "EXCL_WRITE": 4, + } + + +class CompactionType: + MINOR = 1 + MAJOR = 2 + + _VALUES_TO_NAMES = { + 1: "MINOR", + 2: "MAJOR", + } + + _NAMES_TO_VALUES = { + "MINOR": 1, + "MAJOR": 2, + } + + +class GrantRevokeType: + GRANT = 1 + REVOKE = 2 + + _VALUES_TO_NAMES = { + 1: "GRANT", + 2: "REVOKE", + } + + _NAMES_TO_VALUES = { + "GRANT": 1, + "REVOKE": 2, + } + + +class DataOperationType: + SELECT = 1 + INSERT = 2 + UPDATE = 3 + DELETE = 4 + UNSET = 5 + NO_TXN = 6 + + _VALUES_TO_NAMES = { + 1: "SELECT", + 2: "INSERT", + 3: "UPDATE", + 4: "DELETE", + 5: "UNSET", + 6: "NO_TXN", + } + + _NAMES_TO_VALUES = { + "SELECT": 1, + "INSERT": 2, + "UPDATE": 3, + "DELETE": 4, + "UNSET": 5, + "NO_TXN": 6, + } + + +class EventRequestType: + INSERT = 1 + UPDATE = 2 + DELETE = 3 + + _VALUES_TO_NAMES = { + 1: "INSERT", + 2: "UPDATE", + 3: "DELETE", + } + + _NAMES_TO_VALUES = { + "INSERT": 1, + "UPDATE": 2, + "DELETE": 3, + } + + +class SerdeType: + HIVE = 1 + SCHEMA_REGISTRY = 2 + + _VALUES_TO_NAMES = { + 1: "HIVE", + 2: "SCHEMA_REGISTRY", + } + + _NAMES_TO_VALUES = { + "HIVE": 1, + "SCHEMA_REGISTRY": 2, + } + + +class SchemaType: + HIVE = 1 + AVRO = 2 + + _VALUES_TO_NAMES = { + 1: "HIVE", + 2: "AVRO", + } + + _NAMES_TO_VALUES = { + "HIVE": 1, + "AVRO": 2, + } + + +class SchemaCompatibility: + NONE = 1 + BACKWARD = 2 + FORWARD = 3 + BOTH = 4 + + _VALUES_TO_NAMES = { + 1: "NONE", + 2: "BACKWARD", + 3: "FORWARD", + 4: "BOTH", + } + + _NAMES_TO_VALUES = { + "NONE": 1, + "BACKWARD": 2, + "FORWARD": 3, + "BOTH": 4, + } + + +class SchemaValidation: + LATEST = 1 + ALL = 2 + + _VALUES_TO_NAMES = { + 1: "LATEST", + 2: "ALL", + } + + _NAMES_TO_VALUES = { + "LATEST": 1, + "ALL": 2, + } + + +class SchemaVersionState: + INITIATED = 1 + START_REVIEW = 2 + CHANGES_REQUIRED = 3 + REVIEWED = 4 + ENABLED = 5 + DISABLED = 6 + ARCHIVED = 7 + DELETED = 8 + + _VALUES_TO_NAMES = { + 1: "INITIATED", + 2: "START_REVIEW", + 3: "CHANGES_REQUIRED", + 4: "REVIEWED", + 5: "ENABLED", + 6: "DISABLED", + 7: "ARCHIVED", + 8: "DELETED", + } + + _NAMES_TO_VALUES = { + "INITIATED": 1, + "START_REVIEW": 2, + "CHANGES_REQUIRED": 3, + "REVIEWED": 4, + "ENABLED": 5, + "DISABLED": 6, + "ARCHIVED": 7, + "DELETED": 8, + } + + +class DatabaseType: + NATIVE = 1 + REMOTE = 2 + + _VALUES_TO_NAMES = { + 1: "NATIVE", + 2: "REMOTE", + } + + _NAMES_TO_VALUES = { + "NATIVE": 1, + "REMOTE": 2, + } + + +class FunctionType: + JAVA = 1 + + _VALUES_TO_NAMES = { + 1: "JAVA", + } + + _NAMES_TO_VALUES = { + "JAVA": 1, + } + + +class ResourceType: + JAR = 1 + FILE = 2 + ARCHIVE = 3 + + _VALUES_TO_NAMES = { + 1: "JAR", + 2: "FILE", + 3: "ARCHIVE", + } + + _NAMES_TO_VALUES = { + "JAR": 1, + "FILE": 2, + "ARCHIVE": 3, + } + + +class TxnType: + DEFAULT = 0 + REPL_CREATED = 1 + READ_ONLY = 2 + COMPACTION = 3 + MATER_VIEW_REBUILD = 4 + SOFT_DELETE = 5 + + _VALUES_TO_NAMES = { + 0: "DEFAULT", + 1: "REPL_CREATED", + 2: "READ_ONLY", + 3: "COMPACTION", + 4: "MATER_VIEW_REBUILD", + 5: "SOFT_DELETE", + } + + _NAMES_TO_VALUES = { + "DEFAULT": 0, + "REPL_CREATED": 1, + "READ_ONLY": 2, + "COMPACTION": 3, + "MATER_VIEW_REBUILD": 4, + "SOFT_DELETE": 5, + } + + +class GetTablesExtRequestFields: + ACCESS_TYPE = 1 + PROCESSOR_CAPABILITIES = 2 + ALL = 2147483647 + + _VALUES_TO_NAMES = { + 1: "ACCESS_TYPE", + 2: "PROCESSOR_CAPABILITIES", + 2147483647: "ALL", + } + + _NAMES_TO_VALUES = { + "ACCESS_TYPE": 1, + "PROCESSOR_CAPABILITIES": 2, + "ALL": 2147483647, + } + + +class CompactionMetricsMetricType: + NUM_OBSOLETE_DELTAS = 0 + NUM_DELTAS = 1 + NUM_SMALL_DELTAS = 2 + + _VALUES_TO_NAMES = { + 0: "NUM_OBSOLETE_DELTAS", + 1: "NUM_DELTAS", + 2: "NUM_SMALL_DELTAS", + } + + _NAMES_TO_VALUES = { + "NUM_OBSOLETE_DELTAS": 0, + "NUM_DELTAS": 1, + "NUM_SMALL_DELTAS": 2, + } + + +class FileMetadataExprType: + ORC_SARG = 1 + + _VALUES_TO_NAMES = { + 1: "ORC_SARG", + } + + _NAMES_TO_VALUES = { + "ORC_SARG": 1, + } + + +class ClientCapability: + TEST_CAPABILITY = 1 + INSERT_ONLY_TABLES = 2 + + _VALUES_TO_NAMES = { + 1: "TEST_CAPABILITY", + 2: "INSERT_ONLY_TABLES", + } + + _NAMES_TO_VALUES = { + "TEST_CAPABILITY": 1, + "INSERT_ONLY_TABLES": 2, + } + + +class WMResourcePlanStatus: + ACTIVE = 1 + ENABLED = 2 + DISABLED = 3 + + _VALUES_TO_NAMES = { + 1: "ACTIVE", + 2: "ENABLED", + 3: "DISABLED", + } + + _NAMES_TO_VALUES = { + "ACTIVE": 1, + "ENABLED": 2, + "DISABLED": 3, + } + + +class WMPoolSchedulingPolicy: + FAIR = 1 + FIFO = 2 + + _VALUES_TO_NAMES = { + 1: "FAIR", + 2: "FIFO", + } + + _NAMES_TO_VALUES = { + "FAIR": 1, + "FIFO": 2, + } + + +class ScheduledQueryMaintenanceRequestType: + CREATE = 1 + ALTER = 2 + DROP = 3 + + _VALUES_TO_NAMES = { + 1: "CREATE", + 2: "ALTER", + 3: "DROP", + } + + _NAMES_TO_VALUES = { + "CREATE": 1, + "ALTER": 2, + "DROP": 3, + } + + +class QueryState: + INITED = 0 + EXECUTING = 1 + FAILED = 2 + FINISHED = 3 + TIMED_OUT = 4 + AUTO_DISABLED = 5 + + _VALUES_TO_NAMES = { + 0: "INITED", + 1: "EXECUTING", + 2: "FAILED", + 3: "FINISHED", + 4: "TIMED_OUT", + 5: "AUTO_DISABLED", + } + + _NAMES_TO_VALUES = { + "INITED": 0, + "EXECUTING": 1, + "FAILED": 2, + "FINISHED": 3, + "TIMED_OUT": 4, + "AUTO_DISABLED": 5, + } + + +class PartitionFilterMode: + BY_NAMES = 0 + BY_VALUES = 1 + BY_EXPR = 2 + + _VALUES_TO_NAMES = { + 0: "BY_NAMES", + 1: "BY_VALUES", + 2: "BY_EXPR", + } + + _NAMES_TO_VALUES = { + "BY_NAMES": 0, + "BY_VALUES": 1, + "BY_EXPR": 2, + } + + +class Version: + """ + Attributes: + - version + - comments + + """ + + def __init__( + self, + version=None, + comments=None, + ): + self.version = version + self.comments = comments + + def read(self, iprot): + if ( + iprot._fast_decode is not None + and isinstance(iprot.trans, TTransport.CReadableTransport) + and self.thrift_spec is not None + ): + iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec]) + return + iprot.readStructBegin() + while True: + (fname, ftype, fid) = iprot.readFieldBegin() + if ftype == TType.STOP: + break + if fid == 1: + if ftype == TType.STRING: + self.version = ( + iprot.readString().decode("utf-8", errors="replace") if sys.version_info[0] == 2 else iprot.readString() + ) + else: + iprot.skip(ftype) + elif fid == 2: + if ftype == TType.STRING: + self.comments = ( + iprot.readString().decode("utf-8", errors="replace") if sys.version_info[0] == 2 else iprot.readString() + ) + else: + iprot.skip(ftype) + else: + iprot.skip(ftype) + iprot.readFieldEnd() + iprot.readStructEnd() + + def write(self, oprot): + if oprot._fast_encode is not None and self.thrift_spec is not None: + oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec])) + return + oprot.writeStructBegin("Version") + if self.version is not None: + oprot.writeFieldBegin("version", TType.STRING, 1) + oprot.writeString(self.version.encode("utf-8") if sys.version_info[0] == 2 else self.version) + oprot.writeFieldEnd() + if self.comments is not None: + oprot.writeFieldBegin("comments", TType.STRING, 2) + oprot.writeString(self.comments.encode("utf-8") if sys.version_info[0] == 2 else self.comments) + oprot.writeFieldEnd() + oprot.writeFieldStop() + oprot.writeStructEnd() + + def validate(self): + return + + def __repr__(self): + L = ["{}={!r}".format(key, value) for key, value in self.__dict__.items()] + return "{}({})".format(self.__class__.__name__, ", ".join(L)) + + def __eq__(self, other): + return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ + + def __ne__(self, other): + return not (self == other) + + +class FieldSchema: + """ + Attributes: + - name + - type + - comment + + """ + + def __init__( + self, + name=None, + type=None, + comment=None, + ): + self.name = name + self.type = type + self.comment = comment + + def read(self, iprot): + if ( + iprot._fast_decode is not None + and isinstance(iprot.trans, TTransport.CReadableTransport) + and self.thrift_spec is not None + ): + iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec]) + return + iprot.readStructBegin() + while True: + (fname, ftype, fid) = iprot.readFieldBegin() + if ftype == TType.STOP: + break + if fid == 1: + if ftype == TType.STRING: + self.name = ( + iprot.readString().decode("utf-8", errors="replace") if sys.version_info[0] == 2 else iprot.readString() + ) + else: + iprot.skip(ftype) + elif fid == 2: + if ftype == TType.STRING: + self.type = ( + iprot.readString().decode("utf-8", errors="replace") if sys.version_info[0] == 2 else iprot.readString() + ) + else: + iprot.skip(ftype) + elif fid == 3: + if ftype == TType.STRING: + self.comment = ( + iprot.readString().decode("utf-8", errors="replace") if sys.version_info[0] == 2 else iprot.readString() + ) + else: + iprot.skip(ftype) + else: + iprot.skip(ftype) + iprot.readFieldEnd() + iprot.readStructEnd() + + def write(self, oprot): + if oprot._fast_encode is not None and self.thrift_spec is not None: + oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec])) + return + oprot.writeStructBegin("FieldSchema") + if self.name is not None: + oprot.writeFieldBegin("name", TType.STRING, 1) + oprot.writeString(self.name.encode("utf-8") if sys.version_info[0] == 2 else self.name) + oprot.writeFieldEnd() + if self.type is not None: + oprot.writeFieldBegin("type", TType.STRING, 2) + oprot.writeString(self.type.encode("utf-8") if sys.version_info[0] == 2 else self.type) + oprot.writeFieldEnd() + if self.comment is not None: + oprot.writeFieldBegin("comment", TType.STRING, 3) + oprot.writeString(self.comment.encode("utf-8") if sys.version_info[0] == 2 else self.comment) + oprot.writeFieldEnd() + oprot.writeFieldStop() + oprot.writeStructEnd() + + def validate(self): + return + + def __repr__(self): + L = ["{}={!r}".format(key, value) for key, value in self.__dict__.items()] + return "{}({})".format(self.__class__.__name__, ", ".join(L)) + + def __eq__(self, other): + return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ + + def __ne__(self, other): + return not (self == other) + + +class EnvironmentContext: + """ + Attributes: + - properties + + """ + + def __init__( + self, + properties=None, + ): + self.properties = properties + + def read(self, iprot): + if ( + iprot._fast_decode is not None + and isinstance(iprot.trans, TTransport.CReadableTransport) + and self.thrift_spec is not None + ): + iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec]) + return + iprot.readStructBegin() + while True: + (fname, ftype, fid) = iprot.readFieldBegin() + if ftype == TType.STOP: + break + if fid == 1: + if ftype == TType.MAP: + self.properties = {} + (_ktype1, _vtype2, _size0) = iprot.readMapBegin() + for _i4 in range(_size0): + _key5 = ( + iprot.readString().decode("utf-8", errors="replace") + if sys.version_info[0] == 2 + else iprot.readString() + ) + _val6 = ( + iprot.readString().decode("utf-8", errors="replace") + if sys.version_info[0] == 2 + else iprot.readString() + ) + self.properties[_key5] = _val6 + iprot.readMapEnd() + else: + iprot.skip(ftype) + else: + iprot.skip(ftype) + iprot.readFieldEnd() + iprot.readStructEnd() + + def write(self, oprot): + if oprot._fast_encode is not None and self.thrift_spec is not None: + oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec])) + return + oprot.writeStructBegin("EnvironmentContext") + if self.properties is not None: + oprot.writeFieldBegin("properties", TType.MAP, 1) + oprot.writeMapBegin(TType.STRING, TType.STRING, len(self.properties)) + for kiter7, viter8 in self.properties.items(): + oprot.writeString(kiter7.encode("utf-8") if sys.version_info[0] == 2 else kiter7) + oprot.writeString(viter8.encode("utf-8") if sys.version_info[0] == 2 else viter8) + oprot.writeMapEnd() + oprot.writeFieldEnd() + oprot.writeFieldStop() + oprot.writeStructEnd() + + def validate(self): + return + + def __repr__(self): + L = ["{}={!r}".format(key, value) for key, value in self.__dict__.items()] + return "{}({})".format(self.__class__.__name__, ", ".join(L)) + + def __eq__(self, other): + return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ + + def __ne__(self, other): + return not (self == other) + + +class SQLPrimaryKey: + """ + Attributes: + - table_db + - table_name + - column_name + - key_seq + - pk_name + - enable_cstr + - validate_cstr + - rely_cstr + - catName + + """ + + def __init__( + self, + table_db=None, + table_name=None, + column_name=None, + key_seq=None, + pk_name=None, + enable_cstr=None, + validate_cstr=None, + rely_cstr=None, + catName=None, + ): + self.table_db = table_db + self.table_name = table_name + self.column_name = column_name + self.key_seq = key_seq + self.pk_name = pk_name + self.enable_cstr = enable_cstr + self.validate_cstr = validate_cstr + self.rely_cstr = rely_cstr + self.catName = catName + + def read(self, iprot): + if ( + iprot._fast_decode is not None + and isinstance(iprot.trans, TTransport.CReadableTransport) + and self.thrift_spec is not None + ): + iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec]) + return + iprot.readStructBegin() + while True: + (fname, ftype, fid) = iprot.readFieldBegin() + if ftype == TType.STOP: + break + if fid == 1: + if ftype == TType.STRING: + self.table_db = ( + iprot.readString().decode("utf-8", errors="replace") if sys.version_info[0] == 2 else iprot.readString() + ) + else: + iprot.skip(ftype) + elif fid == 2: + if ftype == TType.STRING: + self.table_name = ( + iprot.readString().decode("utf-8", errors="replace") if sys.version_info[0] == 2 else iprot.readString() + ) + else: + iprot.skip(ftype) + elif fid == 3: + if ftype == TType.STRING: + self.column_name = ( + iprot.readString().decode("utf-8", errors="replace") if sys.version_info[0] == 2 else iprot.readString() + ) + else: + iprot.skip(ftype) + elif fid == 4: + if ftype == TType.I32: + self.key_seq = iprot.readI32() + else: + iprot.skip(ftype) + elif fid == 5: + if ftype == TType.STRING: + self.pk_name = ( + iprot.readString().decode("utf-8", errors="replace") if sys.version_info[0] == 2 else iprot.readString() + ) + else: + iprot.skip(ftype) + elif fid == 6: + if ftype == TType.BOOL: + self.enable_cstr = iprot.readBool() + else: + iprot.skip(ftype) + elif fid == 7: + if ftype == TType.BOOL: + self.validate_cstr = iprot.readBool() + else: + iprot.skip(ftype) + elif fid == 8: + if ftype == TType.BOOL: + self.rely_cstr = iprot.readBool() + else: + iprot.skip(ftype) + elif fid == 9: + if ftype == TType.STRING: + self.catName = ( + iprot.readString().decode("utf-8", errors="replace") if sys.version_info[0] == 2 else iprot.readString() + ) + else: + iprot.skip(ftype) + else: + iprot.skip(ftype) + iprot.readFieldEnd() + iprot.readStructEnd() + + def write(self, oprot): + if oprot._fast_encode is not None and self.thrift_spec is not None: + oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec])) + return + oprot.writeStructBegin("SQLPrimaryKey") + if self.table_db is not None: + oprot.writeFieldBegin("table_db", TType.STRING, 1) + oprot.writeString(self.table_db.encode("utf-8") if sys.version_info[0] == 2 else self.table_db) + oprot.writeFieldEnd() + if self.table_name is not None: + oprot.writeFieldBegin("table_name", TType.STRING, 2) + oprot.writeString(self.table_name.encode("utf-8") if sys.version_info[0] == 2 else self.table_name) + oprot.writeFieldEnd() + if self.column_name is not None: + oprot.writeFieldBegin("column_name", TType.STRING, 3) + oprot.writeString(self.column_name.encode("utf-8") if sys.version_info[0] == 2 else self.column_name) + oprot.writeFieldEnd() + if self.key_seq is not None: + oprot.writeFieldBegin("key_seq", TType.I32, 4) + oprot.writeI32(self.key_seq) + oprot.writeFieldEnd() + if self.pk_name is not None: + oprot.writeFieldBegin("pk_name", TType.STRING, 5) + oprot.writeString(self.pk_name.encode("utf-8") if sys.version_info[0] == 2 else self.pk_name) + oprot.writeFieldEnd() + if self.enable_cstr is not None: + oprot.writeFieldBegin("enable_cstr", TType.BOOL, 6) + oprot.writeBool(self.enable_cstr) + oprot.writeFieldEnd() + if self.validate_cstr is not None: + oprot.writeFieldBegin("validate_cstr", TType.BOOL, 7) + oprot.writeBool(self.validate_cstr) + oprot.writeFieldEnd() + if self.rely_cstr is not None: + oprot.writeFieldBegin("rely_cstr", TType.BOOL, 8) + oprot.writeBool(self.rely_cstr) + oprot.writeFieldEnd() + if self.catName is not None: + oprot.writeFieldBegin("catName", TType.STRING, 9) + oprot.writeString(self.catName.encode("utf-8") if sys.version_info[0] == 2 else self.catName) + oprot.writeFieldEnd() + oprot.writeFieldStop() + oprot.writeStructEnd() + + def validate(self): + return + + def __repr__(self): + L = ["{}={!r}".format(key, value) for key, value in self.__dict__.items()] + return "{}({})".format(self.__class__.__name__, ", ".join(L)) + + def __eq__(self, other): + return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ + + def __ne__(self, other): + return not (self == other) + + +class SQLForeignKey: + """ + Attributes: + - pktable_db + - pktable_name + - pkcolumn_name + - fktable_db + - fktable_name + - fkcolumn_name + - key_seq + - update_rule + - delete_rule + - fk_name + - pk_name + - enable_cstr + - validate_cstr + - rely_cstr + - catName + + """ + + def __init__( + self, + pktable_db=None, + pktable_name=None, + pkcolumn_name=None, + fktable_db=None, + fktable_name=None, + fkcolumn_name=None, + key_seq=None, + update_rule=None, + delete_rule=None, + fk_name=None, + pk_name=None, + enable_cstr=None, + validate_cstr=None, + rely_cstr=None, + catName=None, + ): + self.pktable_db = pktable_db + self.pktable_name = pktable_name + self.pkcolumn_name = pkcolumn_name + self.fktable_db = fktable_db + self.fktable_name = fktable_name + self.fkcolumn_name = fkcolumn_name + self.key_seq = key_seq + self.update_rule = update_rule + self.delete_rule = delete_rule + self.fk_name = fk_name + self.pk_name = pk_name + self.enable_cstr = enable_cstr + self.validate_cstr = validate_cstr + self.rely_cstr = rely_cstr + self.catName = catName + + def read(self, iprot): + if ( + iprot._fast_decode is not None + and isinstance(iprot.trans, TTransport.CReadableTransport) + and self.thrift_spec is not None + ): + iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec]) + return + iprot.readStructBegin() + while True: + (fname, ftype, fid) = iprot.readFieldBegin() + if ftype == TType.STOP: + break + if fid == 1: + if ftype == TType.STRING: + self.pktable_db = ( + iprot.readString().decode("utf-8", errors="replace") if sys.version_info[0] == 2 else iprot.readString() + ) + else: + iprot.skip(ftype) + elif fid == 2: + if ftype == TType.STRING: + self.pktable_name = ( + iprot.readString().decode("utf-8", errors="replace") if sys.version_info[0] == 2 else iprot.readString() + ) + else: + iprot.skip(ftype) + elif fid == 3: + if ftype == TType.STRING: + self.pkcolumn_name = ( + iprot.readString().decode("utf-8", errors="replace") if sys.version_info[0] == 2 else iprot.readString() + ) + else: + iprot.skip(ftype) + elif fid == 4: + if ftype == TType.STRING: + self.fktable_db = ( + iprot.readString().decode("utf-8", errors="replace") if sys.version_info[0] == 2 else iprot.readString() + ) + else: + iprot.skip(ftype) + elif fid == 5: + if ftype == TType.STRING: + self.fktable_name = ( + iprot.readString().decode("utf-8", errors="replace") if sys.version_info[0] == 2 else iprot.readString() + ) + else: + iprot.skip(ftype) + elif fid == 6: + if ftype == TType.STRING: + self.fkcolumn_name = ( + iprot.readString().decode("utf-8", errors="replace") if sys.version_info[0] == 2 else iprot.readString() + ) + else: + iprot.skip(ftype) + elif fid == 7: + if ftype == TType.I32: + self.key_seq = iprot.readI32() + else: + iprot.skip(ftype) + elif fid == 8: + if ftype == TType.I32: + self.update_rule = iprot.readI32() + else: + iprot.skip(ftype) + elif fid == 9: + if ftype == TType.I32: + self.delete_rule = iprot.readI32() + else: + iprot.skip(ftype) + elif fid == 10: + if ftype == TType.STRING: + self.fk_name = ( + iprot.readString().decode("utf-8", errors="replace") if sys.version_info[0] == 2 else iprot.readString() + ) + else: + iprot.skip(ftype) + elif fid == 11: + if ftype == TType.STRING: + self.pk_name = ( + iprot.readString().decode("utf-8", errors="replace") if sys.version_info[0] == 2 else iprot.readString() + ) + else: + iprot.skip(ftype) + elif fid == 12: + if ftype == TType.BOOL: + self.enable_cstr = iprot.readBool() + else: + iprot.skip(ftype) + elif fid == 13: + if ftype == TType.BOOL: + self.validate_cstr = iprot.readBool() + else: + iprot.skip(ftype) + elif fid == 14: + if ftype == TType.BOOL: + self.rely_cstr = iprot.readBool() + else: + iprot.skip(ftype) + elif fid == 15: + if ftype == TType.STRING: + self.catName = ( + iprot.readString().decode("utf-8", errors="replace") if sys.version_info[0] == 2 else iprot.readString() + ) + else: + iprot.skip(ftype) + else: + iprot.skip(ftype) + iprot.readFieldEnd() + iprot.readStructEnd() + + def write(self, oprot): + if oprot._fast_encode is not None and self.thrift_spec is not None: + oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec])) + return + oprot.writeStructBegin("SQLForeignKey") + if self.pktable_db is not None: + oprot.writeFieldBegin("pktable_db", TType.STRING, 1) + oprot.writeString(self.pktable_db.encode("utf-8") if sys.version_info[0] == 2 else self.pktable_db) + oprot.writeFieldEnd() + if self.pktable_name is not None: + oprot.writeFieldBegin("pktable_name", TType.STRING, 2) + oprot.writeString(self.pktable_name.encode("utf-8") if sys.version_info[0] == 2 else self.pktable_name) + oprot.writeFieldEnd() + if self.pkcolumn_name is not None: + oprot.writeFieldBegin("pkcolumn_name", TType.STRING, 3) + oprot.writeString(self.pkcolumn_name.encode("utf-8") if sys.version_info[0] == 2 else self.pkcolumn_name) + oprot.writeFieldEnd() + if self.fktable_db is not None: + oprot.writeFieldBegin("fktable_db", TType.STRING, 4) + oprot.writeString(self.fktable_db.encode("utf-8") if sys.version_info[0] == 2 else self.fktable_db) + oprot.writeFieldEnd() + if self.fktable_name is not None: + oprot.writeFieldBegin("fktable_name", TType.STRING, 5) + oprot.writeString(self.fktable_name.encode("utf-8") if sys.version_info[0] == 2 else self.fktable_name) + oprot.writeFieldEnd() + if self.fkcolumn_name is not None: + oprot.writeFieldBegin("fkcolumn_name", TType.STRING, 6) + oprot.writeString(self.fkcolumn_name.encode("utf-8") if sys.version_info[0] == 2 else self.fkcolumn_name) + oprot.writeFieldEnd() + if self.key_seq is not None: + oprot.writeFieldBegin("key_seq", TType.I32, 7) + oprot.writeI32(self.key_seq) + oprot.writeFieldEnd() + if self.update_rule is not None: + oprot.writeFieldBegin("update_rule", TType.I32, 8) + oprot.writeI32(self.update_rule) + oprot.writeFieldEnd() + if self.delete_rule is not None: + oprot.writeFieldBegin("delete_rule", TType.I32, 9) + oprot.writeI32(self.delete_rule) + oprot.writeFieldEnd() + if self.fk_name is not None: + oprot.writeFieldBegin("fk_name", TType.STRING, 10) + oprot.writeString(self.fk_name.encode("utf-8") if sys.version_info[0] == 2 else self.fk_name) + oprot.writeFieldEnd() + if self.pk_name is not None: + oprot.writeFieldBegin("pk_name", TType.STRING, 11) + oprot.writeString(self.pk_name.encode("utf-8") if sys.version_info[0] == 2 else self.pk_name) + oprot.writeFieldEnd() + if self.enable_cstr is not None: + oprot.writeFieldBegin("enable_cstr", TType.BOOL, 12) + oprot.writeBool(self.enable_cstr) + oprot.writeFieldEnd() + if self.validate_cstr is not None: + oprot.writeFieldBegin("validate_cstr", TType.BOOL, 13) + oprot.writeBool(self.validate_cstr) + oprot.writeFieldEnd() + if self.rely_cstr is not None: + oprot.writeFieldBegin("rely_cstr", TType.BOOL, 14) + oprot.writeBool(self.rely_cstr) + oprot.writeFieldEnd() + if self.catName is not None: + oprot.writeFieldBegin("catName", TType.STRING, 15) + oprot.writeString(self.catName.encode("utf-8") if sys.version_info[0] == 2 else self.catName) + oprot.writeFieldEnd() + oprot.writeFieldStop() + oprot.writeStructEnd() + + def validate(self): + return + + def __repr__(self): + L = ["{}={!r}".format(key, value) for key, value in self.__dict__.items()] + return "{}({})".format(self.__class__.__name__, ", ".join(L)) + + def __eq__(self, other): + return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ + + def __ne__(self, other): + return not (self == other) + + +class SQLUniqueConstraint: + """ + Attributes: + - catName + - table_db + - table_name + - column_name + - key_seq + - uk_name + - enable_cstr + - validate_cstr + - rely_cstr + + """ + + def __init__( + self, + catName=None, + table_db=None, + table_name=None, + column_name=None, + key_seq=None, + uk_name=None, + enable_cstr=None, + validate_cstr=None, + rely_cstr=None, + ): + self.catName = catName + self.table_db = table_db + self.table_name = table_name + self.column_name = column_name + self.key_seq = key_seq + self.uk_name = uk_name + self.enable_cstr = enable_cstr + self.validate_cstr = validate_cstr + self.rely_cstr = rely_cstr + + def read(self, iprot): + if ( + iprot._fast_decode is not None + and isinstance(iprot.trans, TTransport.CReadableTransport) + and self.thrift_spec is not None + ): + iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec]) + return + iprot.readStructBegin() + while True: + (fname, ftype, fid) = iprot.readFieldBegin() + if ftype == TType.STOP: + break + if fid == 1: + if ftype == TType.STRING: + self.catName = ( + iprot.readString().decode("utf-8", errors="replace") if sys.version_info[0] == 2 else iprot.readString() + ) + else: + iprot.skip(ftype) + elif fid == 2: + if ftype == TType.STRING: + self.table_db = ( + iprot.readString().decode("utf-8", errors="replace") if sys.version_info[0] == 2 else iprot.readString() + ) + else: + iprot.skip(ftype) + elif fid == 3: + if ftype == TType.STRING: + self.table_name = ( + iprot.readString().decode("utf-8", errors="replace") if sys.version_info[0] == 2 else iprot.readString() + ) + else: + iprot.skip(ftype) + elif fid == 4: + if ftype == TType.STRING: + self.column_name = ( + iprot.readString().decode("utf-8", errors="replace") if sys.version_info[0] == 2 else iprot.readString() + ) + else: + iprot.skip(ftype) + elif fid == 5: + if ftype == TType.I32: + self.key_seq = iprot.readI32() + else: + iprot.skip(ftype) + elif fid == 6: + if ftype == TType.STRING: + self.uk_name = ( + iprot.readString().decode("utf-8", errors="replace") if sys.version_info[0] == 2 else iprot.readString() + ) + else: + iprot.skip(ftype) + elif fid == 7: + if ftype == TType.BOOL: + self.enable_cstr = iprot.readBool() + else: + iprot.skip(ftype) + elif fid == 8: + if ftype == TType.BOOL: + self.validate_cstr = iprot.readBool() + else: + iprot.skip(ftype) + elif fid == 9: + if ftype == TType.BOOL: + self.rely_cstr = iprot.readBool() + else: + iprot.skip(ftype) + else: + iprot.skip(ftype) + iprot.readFieldEnd() + iprot.readStructEnd() + + def write(self, oprot): + if oprot._fast_encode is not None and self.thrift_spec is not None: + oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec])) + return + oprot.writeStructBegin("SQLUniqueConstraint") + if self.catName is not None: + oprot.writeFieldBegin("catName", TType.STRING, 1) + oprot.writeString(self.catName.encode("utf-8") if sys.version_info[0] == 2 else self.catName) + oprot.writeFieldEnd() + if self.table_db is not None: + oprot.writeFieldBegin("table_db", TType.STRING, 2) + oprot.writeString(self.table_db.encode("utf-8") if sys.version_info[0] == 2 else self.table_db) + oprot.writeFieldEnd() + if self.table_name is not None: + oprot.writeFieldBegin("table_name", TType.STRING, 3) + oprot.writeString(self.table_name.encode("utf-8") if sys.version_info[0] == 2 else self.table_name) + oprot.writeFieldEnd() + if self.column_name is not None: + oprot.writeFieldBegin("column_name", TType.STRING, 4) + oprot.writeString(self.column_name.encode("utf-8") if sys.version_info[0] == 2 else self.column_name) + oprot.writeFieldEnd() + if self.key_seq is not None: + oprot.writeFieldBegin("key_seq", TType.I32, 5) + oprot.writeI32(self.key_seq) + oprot.writeFieldEnd() + if self.uk_name is not None: + oprot.writeFieldBegin("uk_name", TType.STRING, 6) + oprot.writeString(self.uk_name.encode("utf-8") if sys.version_info[0] == 2 else self.uk_name) + oprot.writeFieldEnd() + if self.enable_cstr is not None: + oprot.writeFieldBegin("enable_cstr", TType.BOOL, 7) + oprot.writeBool(self.enable_cstr) + oprot.writeFieldEnd() + if self.validate_cstr is not None: + oprot.writeFieldBegin("validate_cstr", TType.BOOL, 8) + oprot.writeBool(self.validate_cstr) + oprot.writeFieldEnd() + if self.rely_cstr is not None: + oprot.writeFieldBegin("rely_cstr", TType.BOOL, 9) + oprot.writeBool(self.rely_cstr) + oprot.writeFieldEnd() + oprot.writeFieldStop() + oprot.writeStructEnd() + + def validate(self): + return + + def __repr__(self): + L = ["{}={!r}".format(key, value) for key, value in self.__dict__.items()] + return "{}({})".format(self.__class__.__name__, ", ".join(L)) + + def __eq__(self, other): + return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ + + def __ne__(self, other): + return not (self == other) + + +class SQLNotNullConstraint: + """ + Attributes: + - catName + - table_db + - table_name + - column_name + - nn_name + - enable_cstr + - validate_cstr + - rely_cstr + + """ + + def __init__( + self, + catName=None, + table_db=None, + table_name=None, + column_name=None, + nn_name=None, + enable_cstr=None, + validate_cstr=None, + rely_cstr=None, + ): + self.catName = catName + self.table_db = table_db + self.table_name = table_name + self.column_name = column_name + self.nn_name = nn_name + self.enable_cstr = enable_cstr + self.validate_cstr = validate_cstr + self.rely_cstr = rely_cstr + + def read(self, iprot): + if ( + iprot._fast_decode is not None + and isinstance(iprot.trans, TTransport.CReadableTransport) + and self.thrift_spec is not None + ): + iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec]) + return + iprot.readStructBegin() + while True: + (fname, ftype, fid) = iprot.readFieldBegin() + if ftype == TType.STOP: + break + if fid == 1: + if ftype == TType.STRING: + self.catName = ( + iprot.readString().decode("utf-8", errors="replace") if sys.version_info[0] == 2 else iprot.readString() + ) + else: + iprot.skip(ftype) + elif fid == 2: + if ftype == TType.STRING: + self.table_db = ( + iprot.readString().decode("utf-8", errors="replace") if sys.version_info[0] == 2 else iprot.readString() + ) + else: + iprot.skip(ftype) + elif fid == 3: + if ftype == TType.STRING: + self.table_name = ( + iprot.readString().decode("utf-8", errors="replace") if sys.version_info[0] == 2 else iprot.readString() + ) + else: + iprot.skip(ftype) + elif fid == 4: + if ftype == TType.STRING: + self.column_name = ( + iprot.readString().decode("utf-8", errors="replace") if sys.version_info[0] == 2 else iprot.readString() + ) + else: + iprot.skip(ftype) + elif fid == 5: + if ftype == TType.STRING: + self.nn_name = ( + iprot.readString().decode("utf-8", errors="replace") if sys.version_info[0] == 2 else iprot.readString() + ) + else: + iprot.skip(ftype) + elif fid == 6: + if ftype == TType.BOOL: + self.enable_cstr = iprot.readBool() + else: + iprot.skip(ftype) + elif fid == 7: + if ftype == TType.BOOL: + self.validate_cstr = iprot.readBool() + else: + iprot.skip(ftype) + elif fid == 8: + if ftype == TType.BOOL: + self.rely_cstr = iprot.readBool() + else: + iprot.skip(ftype) + else: + iprot.skip(ftype) + iprot.readFieldEnd() + iprot.readStructEnd() + + def write(self, oprot): + if oprot._fast_encode is not None and self.thrift_spec is not None: + oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec])) + return + oprot.writeStructBegin("SQLNotNullConstraint") + if self.catName is not None: + oprot.writeFieldBegin("catName", TType.STRING, 1) + oprot.writeString(self.catName.encode("utf-8") if sys.version_info[0] == 2 else self.catName) + oprot.writeFieldEnd() + if self.table_db is not None: + oprot.writeFieldBegin("table_db", TType.STRING, 2) + oprot.writeString(self.table_db.encode("utf-8") if sys.version_info[0] == 2 else self.table_db) + oprot.writeFieldEnd() + if self.table_name is not None: + oprot.writeFieldBegin("table_name", TType.STRING, 3) + oprot.writeString(self.table_name.encode("utf-8") if sys.version_info[0] == 2 else self.table_name) + oprot.writeFieldEnd() + if self.column_name is not None: + oprot.writeFieldBegin("column_name", TType.STRING, 4) + oprot.writeString(self.column_name.encode("utf-8") if sys.version_info[0] == 2 else self.column_name) + oprot.writeFieldEnd() + if self.nn_name is not None: + oprot.writeFieldBegin("nn_name", TType.STRING, 5) + oprot.writeString(self.nn_name.encode("utf-8") if sys.version_info[0] == 2 else self.nn_name) + oprot.writeFieldEnd() + if self.enable_cstr is not None: + oprot.writeFieldBegin("enable_cstr", TType.BOOL, 6) + oprot.writeBool(self.enable_cstr) + oprot.writeFieldEnd() + if self.validate_cstr is not None: + oprot.writeFieldBegin("validate_cstr", TType.BOOL, 7) + oprot.writeBool(self.validate_cstr) + oprot.writeFieldEnd() + if self.rely_cstr is not None: + oprot.writeFieldBegin("rely_cstr", TType.BOOL, 8) + oprot.writeBool(self.rely_cstr) + oprot.writeFieldEnd() + oprot.writeFieldStop() + oprot.writeStructEnd() + + def validate(self): + return + + def __repr__(self): + L = ["{}={!r}".format(key, value) for key, value in self.__dict__.items()] + return "{}({})".format(self.__class__.__name__, ", ".join(L)) + + def __eq__(self, other): + return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ + + def __ne__(self, other): + return not (self == other) + + +class SQLDefaultConstraint: + """ + Attributes: + - catName + - table_db + - table_name + - column_name + - default_value + - dc_name + - enable_cstr + - validate_cstr + - rely_cstr + + """ + + def __init__( + self, + catName=None, + table_db=None, + table_name=None, + column_name=None, + default_value=None, + dc_name=None, + enable_cstr=None, + validate_cstr=None, + rely_cstr=None, + ): + self.catName = catName + self.table_db = table_db + self.table_name = table_name + self.column_name = column_name + self.default_value = default_value + self.dc_name = dc_name + self.enable_cstr = enable_cstr + self.validate_cstr = validate_cstr + self.rely_cstr = rely_cstr + + def read(self, iprot): + if ( + iprot._fast_decode is not None + and isinstance(iprot.trans, TTransport.CReadableTransport) + and self.thrift_spec is not None + ): + iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec]) + return + iprot.readStructBegin() + while True: + (fname, ftype, fid) = iprot.readFieldBegin() + if ftype == TType.STOP: + break + if fid == 1: + if ftype == TType.STRING: + self.catName = ( + iprot.readString().decode("utf-8", errors="replace") if sys.version_info[0] == 2 else iprot.readString() + ) + else: + iprot.skip(ftype) + elif fid == 2: + if ftype == TType.STRING: + self.table_db = ( + iprot.readString().decode("utf-8", errors="replace") if sys.version_info[0] == 2 else iprot.readString() + ) + else: + iprot.skip(ftype) + elif fid == 3: + if ftype == TType.STRING: + self.table_name = ( + iprot.readString().decode("utf-8", errors="replace") if sys.version_info[0] == 2 else iprot.readString() + ) + else: + iprot.skip(ftype) + elif fid == 4: + if ftype == TType.STRING: + self.column_name = ( + iprot.readString().decode("utf-8", errors="replace") if sys.version_info[0] == 2 else iprot.readString() + ) + else: + iprot.skip(ftype) + elif fid == 5: + if ftype == TType.STRING: + self.default_value = ( + iprot.readString().decode("utf-8", errors="replace") if sys.version_info[0] == 2 else iprot.readString() + ) + else: + iprot.skip(ftype) + elif fid == 6: + if ftype == TType.STRING: + self.dc_name = ( + iprot.readString().decode("utf-8", errors="replace") if sys.version_info[0] == 2 else iprot.readString() + ) + else: + iprot.skip(ftype) + elif fid == 7: + if ftype == TType.BOOL: + self.enable_cstr = iprot.readBool() + else: + iprot.skip(ftype) + elif fid == 8: + if ftype == TType.BOOL: + self.validate_cstr = iprot.readBool() + else: + iprot.skip(ftype) + elif fid == 9: + if ftype == TType.BOOL: + self.rely_cstr = iprot.readBool() + else: + iprot.skip(ftype) + else: + iprot.skip(ftype) + iprot.readFieldEnd() + iprot.readStructEnd() + + def write(self, oprot): + if oprot._fast_encode is not None and self.thrift_spec is not None: + oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec])) + return + oprot.writeStructBegin("SQLDefaultConstraint") + if self.catName is not None: + oprot.writeFieldBegin("catName", TType.STRING, 1) + oprot.writeString(self.catName.encode("utf-8") if sys.version_info[0] == 2 else self.catName) + oprot.writeFieldEnd() + if self.table_db is not None: + oprot.writeFieldBegin("table_db", TType.STRING, 2) + oprot.writeString(self.table_db.encode("utf-8") if sys.version_info[0] == 2 else self.table_db) + oprot.writeFieldEnd() + if self.table_name is not None: + oprot.writeFieldBegin("table_name", TType.STRING, 3) + oprot.writeString(self.table_name.encode("utf-8") if sys.version_info[0] == 2 else self.table_name) + oprot.writeFieldEnd() + if self.column_name is not None: + oprot.writeFieldBegin("column_name", TType.STRING, 4) + oprot.writeString(self.column_name.encode("utf-8") if sys.version_info[0] == 2 else self.column_name) + oprot.writeFieldEnd() + if self.default_value is not None: + oprot.writeFieldBegin("default_value", TType.STRING, 5) + oprot.writeString(self.default_value.encode("utf-8") if sys.version_info[0] == 2 else self.default_value) + oprot.writeFieldEnd() + if self.dc_name is not None: + oprot.writeFieldBegin("dc_name", TType.STRING, 6) + oprot.writeString(self.dc_name.encode("utf-8") if sys.version_info[0] == 2 else self.dc_name) + oprot.writeFieldEnd() + if self.enable_cstr is not None: + oprot.writeFieldBegin("enable_cstr", TType.BOOL, 7) + oprot.writeBool(self.enable_cstr) + oprot.writeFieldEnd() + if self.validate_cstr is not None: + oprot.writeFieldBegin("validate_cstr", TType.BOOL, 8) + oprot.writeBool(self.validate_cstr) + oprot.writeFieldEnd() + if self.rely_cstr is not None: + oprot.writeFieldBegin("rely_cstr", TType.BOOL, 9) + oprot.writeBool(self.rely_cstr) + oprot.writeFieldEnd() + oprot.writeFieldStop() + oprot.writeStructEnd() + + def validate(self): + return + + def __repr__(self): + L = ["{}={!r}".format(key, value) for key, value in self.__dict__.items()] + return "{}({})".format(self.__class__.__name__, ", ".join(L)) + + def __eq__(self, other): + return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ + + def __ne__(self, other): + return not (self == other) + + +class SQLCheckConstraint: + """ + Attributes: + - catName + - table_db + - table_name + - column_name + - check_expression + - dc_name + - enable_cstr + - validate_cstr + - rely_cstr + + """ + + def __init__( + self, + catName=None, + table_db=None, + table_name=None, + column_name=None, + check_expression=None, + dc_name=None, + enable_cstr=None, + validate_cstr=None, + rely_cstr=None, + ): + self.catName = catName + self.table_db = table_db + self.table_name = table_name + self.column_name = column_name + self.check_expression = check_expression + self.dc_name = dc_name + self.enable_cstr = enable_cstr + self.validate_cstr = validate_cstr + self.rely_cstr = rely_cstr + + def read(self, iprot): + if ( + iprot._fast_decode is not None + and isinstance(iprot.trans, TTransport.CReadableTransport) + and self.thrift_spec is not None + ): + iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec]) + return + iprot.readStructBegin() + while True: + (fname, ftype, fid) = iprot.readFieldBegin() + if ftype == TType.STOP: + break + if fid == 1: + if ftype == TType.STRING: + self.catName = ( + iprot.readString().decode("utf-8", errors="replace") if sys.version_info[0] == 2 else iprot.readString() + ) + else: + iprot.skip(ftype) + elif fid == 2: + if ftype == TType.STRING: + self.table_db = ( + iprot.readString().decode("utf-8", errors="replace") if sys.version_info[0] == 2 else iprot.readString() + ) + else: + iprot.skip(ftype) + elif fid == 3: + if ftype == TType.STRING: + self.table_name = ( + iprot.readString().decode("utf-8", errors="replace") if sys.version_info[0] == 2 else iprot.readString() + ) + else: + iprot.skip(ftype) + elif fid == 4: + if ftype == TType.STRING: + self.column_name = ( + iprot.readString().decode("utf-8", errors="replace") if sys.version_info[0] == 2 else iprot.readString() + ) + else: + iprot.skip(ftype) + elif fid == 5: + if ftype == TType.STRING: + self.check_expression = ( + iprot.readString().decode("utf-8", errors="replace") if sys.version_info[0] == 2 else iprot.readString() + ) + else: + iprot.skip(ftype) + elif fid == 6: + if ftype == TType.STRING: + self.dc_name = ( + iprot.readString().decode("utf-8", errors="replace") if sys.version_info[0] == 2 else iprot.readString() + ) + else: + iprot.skip(ftype) + elif fid == 7: + if ftype == TType.BOOL: + self.enable_cstr = iprot.readBool() + else: + iprot.skip(ftype) + elif fid == 8: + if ftype == TType.BOOL: + self.validate_cstr = iprot.readBool() + else: + iprot.skip(ftype) + elif fid == 9: + if ftype == TType.BOOL: + self.rely_cstr = iprot.readBool() + else: + iprot.skip(ftype) + else: + iprot.skip(ftype) + iprot.readFieldEnd() + iprot.readStructEnd() + + def write(self, oprot): + if oprot._fast_encode is not None and self.thrift_spec is not None: + oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec])) + return + oprot.writeStructBegin("SQLCheckConstraint") + if self.catName is not None: + oprot.writeFieldBegin("catName", TType.STRING, 1) + oprot.writeString(self.catName.encode("utf-8") if sys.version_info[0] == 2 else self.catName) + oprot.writeFieldEnd() + if self.table_db is not None: + oprot.writeFieldBegin("table_db", TType.STRING, 2) + oprot.writeString(self.table_db.encode("utf-8") if sys.version_info[0] == 2 else self.table_db) + oprot.writeFieldEnd() + if self.table_name is not None: + oprot.writeFieldBegin("table_name", TType.STRING, 3) + oprot.writeString(self.table_name.encode("utf-8") if sys.version_info[0] == 2 else self.table_name) + oprot.writeFieldEnd() + if self.column_name is not None: + oprot.writeFieldBegin("column_name", TType.STRING, 4) + oprot.writeString(self.column_name.encode("utf-8") if sys.version_info[0] == 2 else self.column_name) + oprot.writeFieldEnd() + if self.check_expression is not None: + oprot.writeFieldBegin("check_expression", TType.STRING, 5) + oprot.writeString(self.check_expression.encode("utf-8") if sys.version_info[0] == 2 else self.check_expression) + oprot.writeFieldEnd() + if self.dc_name is not None: + oprot.writeFieldBegin("dc_name", TType.STRING, 6) + oprot.writeString(self.dc_name.encode("utf-8") if sys.version_info[0] == 2 else self.dc_name) + oprot.writeFieldEnd() + if self.enable_cstr is not None: + oprot.writeFieldBegin("enable_cstr", TType.BOOL, 7) + oprot.writeBool(self.enable_cstr) + oprot.writeFieldEnd() + if self.validate_cstr is not None: + oprot.writeFieldBegin("validate_cstr", TType.BOOL, 8) + oprot.writeBool(self.validate_cstr) + oprot.writeFieldEnd() + if self.rely_cstr is not None: + oprot.writeFieldBegin("rely_cstr", TType.BOOL, 9) + oprot.writeBool(self.rely_cstr) + oprot.writeFieldEnd() + oprot.writeFieldStop() + oprot.writeStructEnd() + + def validate(self): + return + + def __repr__(self): + L = ["{}={!r}".format(key, value) for key, value in self.__dict__.items()] + return "{}({})".format(self.__class__.__name__, ", ".join(L)) + + def __eq__(self, other): + return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ + + def __ne__(self, other): + return not (self == other) + + +class SQLAllTableConstraints: + """ + Attributes: + - primaryKeys + - foreignKeys + - uniqueConstraints + - notNullConstraints + - defaultConstraints + - checkConstraints + + """ + + def __init__( + self, + primaryKeys=None, + foreignKeys=None, + uniqueConstraints=None, + notNullConstraints=None, + defaultConstraints=None, + checkConstraints=None, + ): + self.primaryKeys = primaryKeys + self.foreignKeys = foreignKeys + self.uniqueConstraints = uniqueConstraints + self.notNullConstraints = notNullConstraints + self.defaultConstraints = defaultConstraints + self.checkConstraints = checkConstraints + + def read(self, iprot): + if ( + iprot._fast_decode is not None + and isinstance(iprot.trans, TTransport.CReadableTransport) + and self.thrift_spec is not None + ): + iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec]) + return + iprot.readStructBegin() + while True: + (fname, ftype, fid) = iprot.readFieldBegin() + if ftype == TType.STOP: + break + if fid == 1: + if ftype == TType.LIST: + self.primaryKeys = [] + (_etype12, _size9) = iprot.readListBegin() + for _i13 in range(_size9): + _elem14 = SQLPrimaryKey() + _elem14.read(iprot) + self.primaryKeys.append(_elem14) + iprot.readListEnd() + else: + iprot.skip(ftype) + elif fid == 2: + if ftype == TType.LIST: + self.foreignKeys = [] + (_etype18, _size15) = iprot.readListBegin() + for _i19 in range(_size15): + _elem20 = SQLForeignKey() + _elem20.read(iprot) + self.foreignKeys.append(_elem20) + iprot.readListEnd() + else: + iprot.skip(ftype) + elif fid == 3: + if ftype == TType.LIST: + self.uniqueConstraints = [] + (_etype24, _size21) = iprot.readListBegin() + for _i25 in range(_size21): + _elem26 = SQLUniqueConstraint() + _elem26.read(iprot) + self.uniqueConstraints.append(_elem26) + iprot.readListEnd() + else: + iprot.skip(ftype) + elif fid == 4: + if ftype == TType.LIST: + self.notNullConstraints = [] + (_etype30, _size27) = iprot.readListBegin() + for _i31 in range(_size27): + _elem32 = SQLNotNullConstraint() + _elem32.read(iprot) + self.notNullConstraints.append(_elem32) + iprot.readListEnd() + else: + iprot.skip(ftype) + elif fid == 5: + if ftype == TType.LIST: + self.defaultConstraints = [] + (_etype36, _size33) = iprot.readListBegin() + for _i37 in range(_size33): + _elem38 = SQLDefaultConstraint() + _elem38.read(iprot) + self.defaultConstraints.append(_elem38) + iprot.readListEnd() + else: + iprot.skip(ftype) + elif fid == 6: + if ftype == TType.LIST: + self.checkConstraints = [] + (_etype42, _size39) = iprot.readListBegin() + for _i43 in range(_size39): + _elem44 = SQLCheckConstraint() + _elem44.read(iprot) + self.checkConstraints.append(_elem44) + iprot.readListEnd() + else: + iprot.skip(ftype) + else: + iprot.skip(ftype) + iprot.readFieldEnd() + iprot.readStructEnd() + + def write(self, oprot): + if oprot._fast_encode is not None and self.thrift_spec is not None: + oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec])) + return + oprot.writeStructBegin("SQLAllTableConstraints") + if self.primaryKeys is not None: + oprot.writeFieldBegin("primaryKeys", TType.LIST, 1) + oprot.writeListBegin(TType.STRUCT, len(self.primaryKeys)) + for iter45 in self.primaryKeys: + iter45.write(oprot) + oprot.writeListEnd() + oprot.writeFieldEnd() + if self.foreignKeys is not None: + oprot.writeFieldBegin("foreignKeys", TType.LIST, 2) + oprot.writeListBegin(TType.STRUCT, len(self.foreignKeys)) + for iter46 in self.foreignKeys: + iter46.write(oprot) + oprot.writeListEnd() + oprot.writeFieldEnd() + if self.uniqueConstraints is not None: + oprot.writeFieldBegin("uniqueConstraints", TType.LIST, 3) + oprot.writeListBegin(TType.STRUCT, len(self.uniqueConstraints)) + for iter47 in self.uniqueConstraints: + iter47.write(oprot) + oprot.writeListEnd() + oprot.writeFieldEnd() + if self.notNullConstraints is not None: + oprot.writeFieldBegin("notNullConstraints", TType.LIST, 4) + oprot.writeListBegin(TType.STRUCT, len(self.notNullConstraints)) + for iter48 in self.notNullConstraints: + iter48.write(oprot) + oprot.writeListEnd() + oprot.writeFieldEnd() + if self.defaultConstraints is not None: + oprot.writeFieldBegin("defaultConstraints", TType.LIST, 5) + oprot.writeListBegin(TType.STRUCT, len(self.defaultConstraints)) + for iter49 in self.defaultConstraints: + iter49.write(oprot) + oprot.writeListEnd() + oprot.writeFieldEnd() + if self.checkConstraints is not None: + oprot.writeFieldBegin("checkConstraints", TType.LIST, 6) + oprot.writeListBegin(TType.STRUCT, len(self.checkConstraints)) + for iter50 in self.checkConstraints: + iter50.write(oprot) + oprot.writeListEnd() + oprot.writeFieldEnd() + oprot.writeFieldStop() + oprot.writeStructEnd() + + def validate(self): + return + + def __repr__(self): + L = ["{}={!r}".format(key, value) for key, value in self.__dict__.items()] + return "{}({})".format(self.__class__.__name__, ", ".join(L)) + + def __eq__(self, other): + return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ + + def __ne__(self, other): + return not (self == other) + + +class Type: + """ + Attributes: + - name + - type1 + - type2 + - fields + + """ + + def __init__( + self, + name=None, + type1=None, + type2=None, + fields=None, + ): + self.name = name + self.type1 = type1 + self.type2 = type2 + self.fields = fields + + def read(self, iprot): + if ( + iprot._fast_decode is not None + and isinstance(iprot.trans, TTransport.CReadableTransport) + and self.thrift_spec is not None + ): + iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec]) + return + iprot.readStructBegin() + while True: + (fname, ftype, fid) = iprot.readFieldBegin() + if ftype == TType.STOP: + break + if fid == 1: + if ftype == TType.STRING: + self.name = ( + iprot.readString().decode("utf-8", errors="replace") if sys.version_info[0] == 2 else iprot.readString() + ) + else: + iprot.skip(ftype) + elif fid == 2: + if ftype == TType.STRING: + self.type1 = ( + iprot.readString().decode("utf-8", errors="replace") if sys.version_info[0] == 2 else iprot.readString() + ) + else: + iprot.skip(ftype) + elif fid == 3: + if ftype == TType.STRING: + self.type2 = ( + iprot.readString().decode("utf-8", errors="replace") if sys.version_info[0] == 2 else iprot.readString() + ) + else: + iprot.skip(ftype) + elif fid == 4: + if ftype == TType.LIST: + self.fields = [] + (_etype54, _size51) = iprot.readListBegin() + for _i55 in range(_size51): + _elem56 = FieldSchema() + _elem56.read(iprot) + self.fields.append(_elem56) + iprot.readListEnd() + else: + iprot.skip(ftype) + else: + iprot.skip(ftype) + iprot.readFieldEnd() + iprot.readStructEnd() + + def write(self, oprot): + if oprot._fast_encode is not None and self.thrift_spec is not None: + oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec])) + return + oprot.writeStructBegin("Type") + if self.name is not None: + oprot.writeFieldBegin("name", TType.STRING, 1) + oprot.writeString(self.name.encode("utf-8") if sys.version_info[0] == 2 else self.name) + oprot.writeFieldEnd() + if self.type1 is not None: + oprot.writeFieldBegin("type1", TType.STRING, 2) + oprot.writeString(self.type1.encode("utf-8") if sys.version_info[0] == 2 else self.type1) + oprot.writeFieldEnd() + if self.type2 is not None: + oprot.writeFieldBegin("type2", TType.STRING, 3) + oprot.writeString(self.type2.encode("utf-8") if sys.version_info[0] == 2 else self.type2) + oprot.writeFieldEnd() + if self.fields is not None: + oprot.writeFieldBegin("fields", TType.LIST, 4) + oprot.writeListBegin(TType.STRUCT, len(self.fields)) + for iter57 in self.fields: + iter57.write(oprot) + oprot.writeListEnd() + oprot.writeFieldEnd() + oprot.writeFieldStop() + oprot.writeStructEnd() + + def validate(self): + return + + def __repr__(self): + L = ["{}={!r}".format(key, value) for key, value in self.__dict__.items()] + return "{}({})".format(self.__class__.__name__, ", ".join(L)) + + def __eq__(self, other): + return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ + + def __ne__(self, other): + return not (self == other) + + +class HiveObjectRef: + """ + Attributes: + - objectType + - dbName + - objectName + - partValues + - columnName + - catName + + """ + + def __init__( + self, + objectType=None, + dbName=None, + objectName=None, + partValues=None, + columnName=None, + catName=None, + ): + self.objectType = objectType + self.dbName = dbName + self.objectName = objectName + self.partValues = partValues + self.columnName = columnName + self.catName = catName + + def read(self, iprot): + if ( + iprot._fast_decode is not None + and isinstance(iprot.trans, TTransport.CReadableTransport) + and self.thrift_spec is not None + ): + iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec]) + return + iprot.readStructBegin() + while True: + (fname, ftype, fid) = iprot.readFieldBegin() + if ftype == TType.STOP: + break + if fid == 1: + if ftype == TType.I32: + self.objectType = iprot.readI32() + else: + iprot.skip(ftype) + elif fid == 2: + if ftype == TType.STRING: + self.dbName = ( + iprot.readString().decode("utf-8", errors="replace") if sys.version_info[0] == 2 else iprot.readString() + ) + else: + iprot.skip(ftype) + elif fid == 3: + if ftype == TType.STRING: + self.objectName = ( + iprot.readString().decode("utf-8", errors="replace") if sys.version_info[0] == 2 else iprot.readString() + ) + else: + iprot.skip(ftype) + elif fid == 4: + if ftype == TType.LIST: + self.partValues = [] + (_etype61, _size58) = iprot.readListBegin() + for _i62 in range(_size58): + _elem63 = ( + iprot.readString().decode("utf-8", errors="replace") + if sys.version_info[0] == 2 + else iprot.readString() + ) + self.partValues.append(_elem63) + iprot.readListEnd() + else: + iprot.skip(ftype) + elif fid == 5: + if ftype == TType.STRING: + self.columnName = ( + iprot.readString().decode("utf-8", errors="replace") if sys.version_info[0] == 2 else iprot.readString() + ) + else: + iprot.skip(ftype) + elif fid == 6: + if ftype == TType.STRING: + self.catName = ( + iprot.readString().decode("utf-8", errors="replace") if sys.version_info[0] == 2 else iprot.readString() + ) + else: + iprot.skip(ftype) + else: + iprot.skip(ftype) + iprot.readFieldEnd() + iprot.readStructEnd() + + def write(self, oprot): + if oprot._fast_encode is not None and self.thrift_spec is not None: + oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec])) + return + oprot.writeStructBegin("HiveObjectRef") + if self.objectType is not None: + oprot.writeFieldBegin("objectType", TType.I32, 1) + oprot.writeI32(self.objectType) + oprot.writeFieldEnd() + if self.dbName is not None: + oprot.writeFieldBegin("dbName", TType.STRING, 2) + oprot.writeString(self.dbName.encode("utf-8") if sys.version_info[0] == 2 else self.dbName) + oprot.writeFieldEnd() + if self.objectName is not None: + oprot.writeFieldBegin("objectName", TType.STRING, 3) + oprot.writeString(self.objectName.encode("utf-8") if sys.version_info[0] == 2 else self.objectName) + oprot.writeFieldEnd() + if self.partValues is not None: + oprot.writeFieldBegin("partValues", TType.LIST, 4) + oprot.writeListBegin(TType.STRING, len(self.partValues)) + for iter64 in self.partValues: + oprot.writeString(iter64.encode("utf-8") if sys.version_info[0] == 2 else iter64) + oprot.writeListEnd() + oprot.writeFieldEnd() + if self.columnName is not None: + oprot.writeFieldBegin("columnName", TType.STRING, 5) + oprot.writeString(self.columnName.encode("utf-8") if sys.version_info[0] == 2 else self.columnName) + oprot.writeFieldEnd() + if self.catName is not None: + oprot.writeFieldBegin("catName", TType.STRING, 6) + oprot.writeString(self.catName.encode("utf-8") if sys.version_info[0] == 2 else self.catName) + oprot.writeFieldEnd() + oprot.writeFieldStop() + oprot.writeStructEnd() + + def validate(self): + return + + def __repr__(self): + L = ["{}={!r}".format(key, value) for key, value in self.__dict__.items()] + return "{}({})".format(self.__class__.__name__, ", ".join(L)) + + def __eq__(self, other): + return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ + + def __ne__(self, other): + return not (self == other) + + +class PrivilegeGrantInfo: + """ + Attributes: + - privilege + - createTime + - grantor + - grantorType + - grantOption + + """ + + def __init__( + self, + privilege=None, + createTime=None, + grantor=None, + grantorType=None, + grantOption=None, + ): + self.privilege = privilege + self.createTime = createTime + self.grantor = grantor + self.grantorType = grantorType + self.grantOption = grantOption + + def read(self, iprot): + if ( + iprot._fast_decode is not None + and isinstance(iprot.trans, TTransport.CReadableTransport) + and self.thrift_spec is not None + ): + iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec]) + return + iprot.readStructBegin() + while True: + (fname, ftype, fid) = iprot.readFieldBegin() + if ftype == TType.STOP: + break + if fid == 1: + if ftype == TType.STRING: + self.privilege = ( + iprot.readString().decode("utf-8", errors="replace") if sys.version_info[0] == 2 else iprot.readString() + ) + else: + iprot.skip(ftype) + elif fid == 2: + if ftype == TType.I32: + self.createTime = iprot.readI32() + else: + iprot.skip(ftype) + elif fid == 3: + if ftype == TType.STRING: + self.grantor = ( + iprot.readString().decode("utf-8", errors="replace") if sys.version_info[0] == 2 else iprot.readString() + ) + else: + iprot.skip(ftype) + elif fid == 4: + if ftype == TType.I32: + self.grantorType = iprot.readI32() + else: + iprot.skip(ftype) + elif fid == 5: + if ftype == TType.BOOL: + self.grantOption = iprot.readBool() + else: + iprot.skip(ftype) + else: + iprot.skip(ftype) + iprot.readFieldEnd() + iprot.readStructEnd() + + def write(self, oprot): + if oprot._fast_encode is not None and self.thrift_spec is not None: + oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec])) + return + oprot.writeStructBegin("PrivilegeGrantInfo") + if self.privilege is not None: + oprot.writeFieldBegin("privilege", TType.STRING, 1) + oprot.writeString(self.privilege.encode("utf-8") if sys.version_info[0] == 2 else self.privilege) + oprot.writeFieldEnd() + if self.createTime is not None: + oprot.writeFieldBegin("createTime", TType.I32, 2) + oprot.writeI32(self.createTime) + oprot.writeFieldEnd() + if self.grantor is not None: + oprot.writeFieldBegin("grantor", TType.STRING, 3) + oprot.writeString(self.grantor.encode("utf-8") if sys.version_info[0] == 2 else self.grantor) + oprot.writeFieldEnd() + if self.grantorType is not None: + oprot.writeFieldBegin("grantorType", TType.I32, 4) + oprot.writeI32(self.grantorType) + oprot.writeFieldEnd() + if self.grantOption is not None: + oprot.writeFieldBegin("grantOption", TType.BOOL, 5) + oprot.writeBool(self.grantOption) + oprot.writeFieldEnd() + oprot.writeFieldStop() + oprot.writeStructEnd() + + def validate(self): + return + + def __repr__(self): + L = ["{}={!r}".format(key, value) for key, value in self.__dict__.items()] + return "{}({})".format(self.__class__.__name__, ", ".join(L)) + + def __eq__(self, other): + return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ + + def __ne__(self, other): + return not (self == other) + + +class HiveObjectPrivilege: + """ + Attributes: + - hiveObject + - principalName + - principalType + - grantInfo + - authorizer + + """ + + def __init__( + self, + hiveObject=None, + principalName=None, + principalType=None, + grantInfo=None, + authorizer=None, + ): + self.hiveObject = hiveObject + self.principalName = principalName + self.principalType = principalType + self.grantInfo = grantInfo + self.authorizer = authorizer + + def read(self, iprot): + if ( + iprot._fast_decode is not None + and isinstance(iprot.trans, TTransport.CReadableTransport) + and self.thrift_spec is not None + ): + iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec]) + return + iprot.readStructBegin() + while True: + (fname, ftype, fid) = iprot.readFieldBegin() + if ftype == TType.STOP: + break + if fid == 1: + if ftype == TType.STRUCT: + self.hiveObject = HiveObjectRef() + self.hiveObject.read(iprot) + else: + iprot.skip(ftype) + elif fid == 2: + if ftype == TType.STRING: + self.principalName = ( + iprot.readString().decode("utf-8", errors="replace") if sys.version_info[0] == 2 else iprot.readString() + ) + else: + iprot.skip(ftype) + elif fid == 3: + if ftype == TType.I32: + self.principalType = iprot.readI32() + else: + iprot.skip(ftype) + elif fid == 4: + if ftype == TType.STRUCT: + self.grantInfo = PrivilegeGrantInfo() + self.grantInfo.read(iprot) + else: + iprot.skip(ftype) + elif fid == 5: + if ftype == TType.STRING: + self.authorizer = ( + iprot.readString().decode("utf-8", errors="replace") if sys.version_info[0] == 2 else iprot.readString() + ) + else: + iprot.skip(ftype) + else: + iprot.skip(ftype) + iprot.readFieldEnd() + iprot.readStructEnd() + + def write(self, oprot): + if oprot._fast_encode is not None and self.thrift_spec is not None: + oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec])) + return + oprot.writeStructBegin("HiveObjectPrivilege") + if self.hiveObject is not None: + oprot.writeFieldBegin("hiveObject", TType.STRUCT, 1) + self.hiveObject.write(oprot) + oprot.writeFieldEnd() + if self.principalName is not None: + oprot.writeFieldBegin("principalName", TType.STRING, 2) + oprot.writeString(self.principalName.encode("utf-8") if sys.version_info[0] == 2 else self.principalName) + oprot.writeFieldEnd() + if self.principalType is not None: + oprot.writeFieldBegin("principalType", TType.I32, 3) + oprot.writeI32(self.principalType) + oprot.writeFieldEnd() + if self.grantInfo is not None: + oprot.writeFieldBegin("grantInfo", TType.STRUCT, 4) + self.grantInfo.write(oprot) + oprot.writeFieldEnd() + if self.authorizer is not None: + oprot.writeFieldBegin("authorizer", TType.STRING, 5) + oprot.writeString(self.authorizer.encode("utf-8") if sys.version_info[0] == 2 else self.authorizer) + oprot.writeFieldEnd() + oprot.writeFieldStop() + oprot.writeStructEnd() + + def validate(self): + return + + def __repr__(self): + L = ["{}={!r}".format(key, value) for key, value in self.__dict__.items()] + return "{}({})".format(self.__class__.__name__, ", ".join(L)) + + def __eq__(self, other): + return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ + + def __ne__(self, other): + return not (self == other) + + +class PrivilegeBag: + """ + Attributes: + - privileges + + """ + + def __init__( + self, + privileges=None, + ): + self.privileges = privileges + + def read(self, iprot): + if ( + iprot._fast_decode is not None + and isinstance(iprot.trans, TTransport.CReadableTransport) + and self.thrift_spec is not None + ): + iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec]) + return + iprot.readStructBegin() + while True: + (fname, ftype, fid) = iprot.readFieldBegin() + if ftype == TType.STOP: + break + if fid == 1: + if ftype == TType.LIST: + self.privileges = [] + (_etype68, _size65) = iprot.readListBegin() + for _i69 in range(_size65): + _elem70 = HiveObjectPrivilege() + _elem70.read(iprot) + self.privileges.append(_elem70) + iprot.readListEnd() + else: + iprot.skip(ftype) + else: + iprot.skip(ftype) + iprot.readFieldEnd() + iprot.readStructEnd() + + def write(self, oprot): + if oprot._fast_encode is not None and self.thrift_spec is not None: + oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec])) + return + oprot.writeStructBegin("PrivilegeBag") + if self.privileges is not None: + oprot.writeFieldBegin("privileges", TType.LIST, 1) + oprot.writeListBegin(TType.STRUCT, len(self.privileges)) + for iter71 in self.privileges: + iter71.write(oprot) + oprot.writeListEnd() + oprot.writeFieldEnd() + oprot.writeFieldStop() + oprot.writeStructEnd() + + def validate(self): + return + + def __repr__(self): + L = ["{}={!r}".format(key, value) for key, value in self.__dict__.items()] + return "{}({})".format(self.__class__.__name__, ", ".join(L)) + + def __eq__(self, other): + return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ + + def __ne__(self, other): + return not (self == other) + + +class PrincipalPrivilegeSet: + """ + Attributes: + - userPrivileges + - groupPrivileges + - rolePrivileges + + """ + + def __init__( + self, + userPrivileges=None, + groupPrivileges=None, + rolePrivileges=None, + ): + self.userPrivileges = userPrivileges + self.groupPrivileges = groupPrivileges + self.rolePrivileges = rolePrivileges + + def read(self, iprot): + if ( + iprot._fast_decode is not None + and isinstance(iprot.trans, TTransport.CReadableTransport) + and self.thrift_spec is not None + ): + iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec]) + return + iprot.readStructBegin() + while True: + (fname, ftype, fid) = iprot.readFieldBegin() + if ftype == TType.STOP: + break + if fid == 1: + if ftype == TType.MAP: + self.userPrivileges = {} + (_ktype73, _vtype74, _size72) = iprot.readMapBegin() + for _i76 in range(_size72): + _key77 = ( + iprot.readString().decode("utf-8", errors="replace") + if sys.version_info[0] == 2 + else iprot.readString() + ) + _val78 = [] + (_etype82, _size79) = iprot.readListBegin() + for _i83 in range(_size79): + _elem84 = PrivilegeGrantInfo() + _elem84.read(iprot) + _val78.append(_elem84) + iprot.readListEnd() + self.userPrivileges[_key77] = _val78 + iprot.readMapEnd() + else: + iprot.skip(ftype) + elif fid == 2: + if ftype == TType.MAP: + self.groupPrivileges = {} + (_ktype86, _vtype87, _size85) = iprot.readMapBegin() + for _i89 in range(_size85): + _key90 = ( + iprot.readString().decode("utf-8", errors="replace") + if sys.version_info[0] == 2 + else iprot.readString() + ) + _val91 = [] + (_etype95, _size92) = iprot.readListBegin() + for _i96 in range(_size92): + _elem97 = PrivilegeGrantInfo() + _elem97.read(iprot) + _val91.append(_elem97) + iprot.readListEnd() + self.groupPrivileges[_key90] = _val91 + iprot.readMapEnd() + else: + iprot.skip(ftype) + elif fid == 3: + if ftype == TType.MAP: + self.rolePrivileges = {} + (_ktype99, _vtype100, _size98) = iprot.readMapBegin() + for _i102 in range(_size98): + _key103 = ( + iprot.readString().decode("utf-8", errors="replace") + if sys.version_info[0] == 2 + else iprot.readString() + ) + _val104 = [] + (_etype108, _size105) = iprot.readListBegin() + for _i109 in range(_size105): + _elem110 = PrivilegeGrantInfo() + _elem110.read(iprot) + _val104.append(_elem110) + iprot.readListEnd() + self.rolePrivileges[_key103] = _val104 + iprot.readMapEnd() + else: + iprot.skip(ftype) + else: + iprot.skip(ftype) + iprot.readFieldEnd() + iprot.readStructEnd() + + def write(self, oprot): + if oprot._fast_encode is not None and self.thrift_spec is not None: + oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec])) + return + oprot.writeStructBegin("PrincipalPrivilegeSet") + if self.userPrivileges is not None: + oprot.writeFieldBegin("userPrivileges", TType.MAP, 1) + oprot.writeMapBegin(TType.STRING, TType.LIST, len(self.userPrivileges)) + for kiter111, viter112 in self.userPrivileges.items(): + oprot.writeString(kiter111.encode("utf-8") if sys.version_info[0] == 2 else kiter111) + oprot.writeListBegin(TType.STRUCT, len(viter112)) + for iter113 in viter112: + iter113.write(oprot) + oprot.writeListEnd() + oprot.writeMapEnd() + oprot.writeFieldEnd() + if self.groupPrivileges is not None: + oprot.writeFieldBegin("groupPrivileges", TType.MAP, 2) + oprot.writeMapBegin(TType.STRING, TType.LIST, len(self.groupPrivileges)) + for kiter114, viter115 in self.groupPrivileges.items(): + oprot.writeString(kiter114.encode("utf-8") if sys.version_info[0] == 2 else kiter114) + oprot.writeListBegin(TType.STRUCT, len(viter115)) + for iter116 in viter115: + iter116.write(oprot) + oprot.writeListEnd() + oprot.writeMapEnd() + oprot.writeFieldEnd() + if self.rolePrivileges is not None: + oprot.writeFieldBegin("rolePrivileges", TType.MAP, 3) + oprot.writeMapBegin(TType.STRING, TType.LIST, len(self.rolePrivileges)) + for kiter117, viter118 in self.rolePrivileges.items(): + oprot.writeString(kiter117.encode("utf-8") if sys.version_info[0] == 2 else kiter117) + oprot.writeListBegin(TType.STRUCT, len(viter118)) + for iter119 in viter118: + iter119.write(oprot) + oprot.writeListEnd() + oprot.writeMapEnd() + oprot.writeFieldEnd() + oprot.writeFieldStop() + oprot.writeStructEnd() + + def validate(self): + return + + def __repr__(self): + L = ["{}={!r}".format(key, value) for key, value in self.__dict__.items()] + return "{}({})".format(self.__class__.__name__, ", ".join(L)) + + def __eq__(self, other): + return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ + + def __ne__(self, other): + return not (self == other) + + +class GrantRevokePrivilegeRequest: + """ + Attributes: + - requestType + - privileges + - revokeGrantOption + + """ + + def __init__( + self, + requestType=None, + privileges=None, + revokeGrantOption=None, + ): + self.requestType = requestType + self.privileges = privileges + self.revokeGrantOption = revokeGrantOption + + def read(self, iprot): + if ( + iprot._fast_decode is not None + and isinstance(iprot.trans, TTransport.CReadableTransport) + and self.thrift_spec is not None + ): + iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec]) + return + iprot.readStructBegin() + while True: + (fname, ftype, fid) = iprot.readFieldBegin() + if ftype == TType.STOP: + break + if fid == 1: + if ftype == TType.I32: + self.requestType = iprot.readI32() + else: + iprot.skip(ftype) + elif fid == 2: + if ftype == TType.STRUCT: + self.privileges = PrivilegeBag() + self.privileges.read(iprot) + else: + iprot.skip(ftype) + elif fid == 3: + if ftype == TType.BOOL: + self.revokeGrantOption = iprot.readBool() + else: + iprot.skip(ftype) + else: + iprot.skip(ftype) + iprot.readFieldEnd() + iprot.readStructEnd() + + def write(self, oprot): + if oprot._fast_encode is not None and self.thrift_spec is not None: + oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec])) + return + oprot.writeStructBegin("GrantRevokePrivilegeRequest") + if self.requestType is not None: + oprot.writeFieldBegin("requestType", TType.I32, 1) + oprot.writeI32(self.requestType) + oprot.writeFieldEnd() + if self.privileges is not None: + oprot.writeFieldBegin("privileges", TType.STRUCT, 2) + self.privileges.write(oprot) + oprot.writeFieldEnd() + if self.revokeGrantOption is not None: + oprot.writeFieldBegin("revokeGrantOption", TType.BOOL, 3) + oprot.writeBool(self.revokeGrantOption) + oprot.writeFieldEnd() + oprot.writeFieldStop() + oprot.writeStructEnd() + + def validate(self): + return + + def __repr__(self): + L = ["{}={!r}".format(key, value) for key, value in self.__dict__.items()] + return "{}({})".format(self.__class__.__name__, ", ".join(L)) + + def __eq__(self, other): + return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ + + def __ne__(self, other): + return not (self == other) + + +class GrantRevokePrivilegeResponse: + """ + Attributes: + - success + + """ + + def __init__( + self, + success=None, + ): + self.success = success + + def read(self, iprot): + if ( + iprot._fast_decode is not None + and isinstance(iprot.trans, TTransport.CReadableTransport) + and self.thrift_spec is not None + ): + iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec]) + return + iprot.readStructBegin() + while True: + (fname, ftype, fid) = iprot.readFieldBegin() + if ftype == TType.STOP: + break + if fid == 1: + if ftype == TType.BOOL: + self.success = iprot.readBool() + else: + iprot.skip(ftype) + else: + iprot.skip(ftype) + iprot.readFieldEnd() + iprot.readStructEnd() + + def write(self, oprot): + if oprot._fast_encode is not None and self.thrift_spec is not None: + oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec])) + return + oprot.writeStructBegin("GrantRevokePrivilegeResponse") + if self.success is not None: + oprot.writeFieldBegin("success", TType.BOOL, 1) + oprot.writeBool(self.success) + oprot.writeFieldEnd() + oprot.writeFieldStop() + oprot.writeStructEnd() + + def validate(self): + return + + def __repr__(self): + L = ["{}={!r}".format(key, value) for key, value in self.__dict__.items()] + return "{}({})".format(self.__class__.__name__, ", ".join(L)) + + def __eq__(self, other): + return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ + + def __ne__(self, other): + return not (self == other) + + +class TruncateTableRequest: + """ + Attributes: + - dbName + - tableName + - partNames + - writeId + - validWriteIdList + - environmentContext + + """ + + def __init__( + self, + dbName=None, + tableName=None, + partNames=None, + writeId=-1, + validWriteIdList=None, + environmentContext=None, + ): + self.dbName = dbName + self.tableName = tableName + self.partNames = partNames + self.writeId = writeId + self.validWriteIdList = validWriteIdList + self.environmentContext = environmentContext + + def read(self, iprot): + if ( + iprot._fast_decode is not None + and isinstance(iprot.trans, TTransport.CReadableTransport) + and self.thrift_spec is not None + ): + iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec]) + return + iprot.readStructBegin() + while True: + (fname, ftype, fid) = iprot.readFieldBegin() + if ftype == TType.STOP: + break + if fid == 1: + if ftype == TType.STRING: + self.dbName = ( + iprot.readString().decode("utf-8", errors="replace") if sys.version_info[0] == 2 else iprot.readString() + ) + else: + iprot.skip(ftype) + elif fid == 2: + if ftype == TType.STRING: + self.tableName = ( + iprot.readString().decode("utf-8", errors="replace") if sys.version_info[0] == 2 else iprot.readString() + ) + else: + iprot.skip(ftype) + elif fid == 3: + if ftype == TType.LIST: + self.partNames = [] + (_etype123, _size120) = iprot.readListBegin() + for _i124 in range(_size120): + _elem125 = ( + iprot.readString().decode("utf-8", errors="replace") + if sys.version_info[0] == 2 + else iprot.readString() + ) + self.partNames.append(_elem125) + iprot.readListEnd() + else: + iprot.skip(ftype) + elif fid == 4: + if ftype == TType.I64: + self.writeId = iprot.readI64() + else: + iprot.skip(ftype) + elif fid == 5: + if ftype == TType.STRING: + self.validWriteIdList = ( + iprot.readString().decode("utf-8", errors="replace") if sys.version_info[0] == 2 else iprot.readString() + ) + else: + iprot.skip(ftype) + elif fid == 6: + if ftype == TType.STRUCT: + self.environmentContext = EnvironmentContext() + self.environmentContext.read(iprot) + else: + iprot.skip(ftype) + else: + iprot.skip(ftype) + iprot.readFieldEnd() + iprot.readStructEnd() + + def write(self, oprot): + if oprot._fast_encode is not None and self.thrift_spec is not None: + oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec])) + return + oprot.writeStructBegin("TruncateTableRequest") + if self.dbName is not None: + oprot.writeFieldBegin("dbName", TType.STRING, 1) + oprot.writeString(self.dbName.encode("utf-8") if sys.version_info[0] == 2 else self.dbName) + oprot.writeFieldEnd() + if self.tableName is not None: + oprot.writeFieldBegin("tableName", TType.STRING, 2) + oprot.writeString(self.tableName.encode("utf-8") if sys.version_info[0] == 2 else self.tableName) + oprot.writeFieldEnd() + if self.partNames is not None: + oprot.writeFieldBegin("partNames", TType.LIST, 3) + oprot.writeListBegin(TType.STRING, len(self.partNames)) + for iter126 in self.partNames: + oprot.writeString(iter126.encode("utf-8") if sys.version_info[0] == 2 else iter126) + oprot.writeListEnd() + oprot.writeFieldEnd() + if self.writeId is not None: + oprot.writeFieldBegin("writeId", TType.I64, 4) + oprot.writeI64(self.writeId) + oprot.writeFieldEnd() + if self.validWriteIdList is not None: + oprot.writeFieldBegin("validWriteIdList", TType.STRING, 5) + oprot.writeString(self.validWriteIdList.encode("utf-8") if sys.version_info[0] == 2 else self.validWriteIdList) + oprot.writeFieldEnd() + if self.environmentContext is not None: + oprot.writeFieldBegin("environmentContext", TType.STRUCT, 6) + self.environmentContext.write(oprot) + oprot.writeFieldEnd() + oprot.writeFieldStop() + oprot.writeStructEnd() + + def validate(self): + if self.dbName is None: + raise TProtocolException(message="Required field dbName is unset!") + if self.tableName is None: + raise TProtocolException(message="Required field tableName is unset!") + return + + def __repr__(self): + L = ["{}={!r}".format(key, value) for key, value in self.__dict__.items()] + return "{}({})".format(self.__class__.__name__, ", ".join(L)) + + def __eq__(self, other): + return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ + + def __ne__(self, other): + return not (self == other) + + +class TruncateTableResponse: + def read(self, iprot): + if ( + iprot._fast_decode is not None + and isinstance(iprot.trans, TTransport.CReadableTransport) + and self.thrift_spec is not None + ): + iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec]) + return + iprot.readStructBegin() + while True: + (fname, ftype, fid) = iprot.readFieldBegin() + if ftype == TType.STOP: + break + else: + iprot.skip(ftype) + iprot.readFieldEnd() + iprot.readStructEnd() + + def write(self, oprot): + if oprot._fast_encode is not None and self.thrift_spec is not None: + oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec])) + return + oprot.writeStructBegin("TruncateTableResponse") + oprot.writeFieldStop() + oprot.writeStructEnd() + + def validate(self): + return + + def __repr__(self): + L = ["{}={!r}".format(key, value) for key, value in self.__dict__.items()] + return "{}({})".format(self.__class__.__name__, ", ".join(L)) + + def __eq__(self, other): + return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ + + def __ne__(self, other): + return not (self == other) + + +class Role: + """ + Attributes: + - roleName + - createTime + - ownerName + + """ + + def __init__( + self, + roleName=None, + createTime=None, + ownerName=None, + ): + self.roleName = roleName + self.createTime = createTime + self.ownerName = ownerName + + def read(self, iprot): + if ( + iprot._fast_decode is not None + and isinstance(iprot.trans, TTransport.CReadableTransport) + and self.thrift_spec is not None + ): + iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec]) + return + iprot.readStructBegin() + while True: + (fname, ftype, fid) = iprot.readFieldBegin() + if ftype == TType.STOP: + break + if fid == 1: + if ftype == TType.STRING: + self.roleName = ( + iprot.readString().decode("utf-8", errors="replace") if sys.version_info[0] == 2 else iprot.readString() + ) + else: + iprot.skip(ftype) + elif fid == 2: + if ftype == TType.I32: + self.createTime = iprot.readI32() + else: + iprot.skip(ftype) + elif fid == 3: + if ftype == TType.STRING: + self.ownerName = ( + iprot.readString().decode("utf-8", errors="replace") if sys.version_info[0] == 2 else iprot.readString() + ) + else: + iprot.skip(ftype) + else: + iprot.skip(ftype) + iprot.readFieldEnd() + iprot.readStructEnd() + + def write(self, oprot): + if oprot._fast_encode is not None and self.thrift_spec is not None: + oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec])) + return + oprot.writeStructBegin("Role") + if self.roleName is not None: + oprot.writeFieldBegin("roleName", TType.STRING, 1) + oprot.writeString(self.roleName.encode("utf-8") if sys.version_info[0] == 2 else self.roleName) + oprot.writeFieldEnd() + if self.createTime is not None: + oprot.writeFieldBegin("createTime", TType.I32, 2) + oprot.writeI32(self.createTime) + oprot.writeFieldEnd() + if self.ownerName is not None: + oprot.writeFieldBegin("ownerName", TType.STRING, 3) + oprot.writeString(self.ownerName.encode("utf-8") if sys.version_info[0] == 2 else self.ownerName) + oprot.writeFieldEnd() + oprot.writeFieldStop() + oprot.writeStructEnd() + + def validate(self): + return + + def __repr__(self): + L = ["{}={!r}".format(key, value) for key, value in self.__dict__.items()] + return "{}({})".format(self.__class__.__name__, ", ".join(L)) + + def __eq__(self, other): + return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ + + def __ne__(self, other): + return not (self == other) + + +class RolePrincipalGrant: + """ + Attributes: + - roleName + - principalName + - principalType + - grantOption + - grantTime + - grantorName + - grantorPrincipalType + + """ + + def __init__( + self, + roleName=None, + principalName=None, + principalType=None, + grantOption=None, + grantTime=None, + grantorName=None, + grantorPrincipalType=None, + ): + self.roleName = roleName + self.principalName = principalName + self.principalType = principalType + self.grantOption = grantOption + self.grantTime = grantTime + self.grantorName = grantorName + self.grantorPrincipalType = grantorPrincipalType + + def read(self, iprot): + if ( + iprot._fast_decode is not None + and isinstance(iprot.trans, TTransport.CReadableTransport) + and self.thrift_spec is not None + ): + iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec]) + return + iprot.readStructBegin() + while True: + (fname, ftype, fid) = iprot.readFieldBegin() + if ftype == TType.STOP: + break + if fid == 1: + if ftype == TType.STRING: + self.roleName = ( + iprot.readString().decode("utf-8", errors="replace") if sys.version_info[0] == 2 else iprot.readString() + ) + else: + iprot.skip(ftype) + elif fid == 2: + if ftype == TType.STRING: + self.principalName = ( + iprot.readString().decode("utf-8", errors="replace") if sys.version_info[0] == 2 else iprot.readString() + ) + else: + iprot.skip(ftype) + elif fid == 3: + if ftype == TType.I32: + self.principalType = iprot.readI32() + else: + iprot.skip(ftype) + elif fid == 4: + if ftype == TType.BOOL: + self.grantOption = iprot.readBool() + else: + iprot.skip(ftype) + elif fid == 5: + if ftype == TType.I32: + self.grantTime = iprot.readI32() + else: + iprot.skip(ftype) + elif fid == 6: + if ftype == TType.STRING: + self.grantorName = ( + iprot.readString().decode("utf-8", errors="replace") if sys.version_info[0] == 2 else iprot.readString() + ) + else: + iprot.skip(ftype) + elif fid == 7: + if ftype == TType.I32: + self.grantorPrincipalType = iprot.readI32() + else: + iprot.skip(ftype) + else: + iprot.skip(ftype) + iprot.readFieldEnd() + iprot.readStructEnd() + + def write(self, oprot): + if oprot._fast_encode is not None and self.thrift_spec is not None: + oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec])) + return + oprot.writeStructBegin("RolePrincipalGrant") + if self.roleName is not None: + oprot.writeFieldBegin("roleName", TType.STRING, 1) + oprot.writeString(self.roleName.encode("utf-8") if sys.version_info[0] == 2 else self.roleName) + oprot.writeFieldEnd() + if self.principalName is not None: + oprot.writeFieldBegin("principalName", TType.STRING, 2) + oprot.writeString(self.principalName.encode("utf-8") if sys.version_info[0] == 2 else self.principalName) + oprot.writeFieldEnd() + if self.principalType is not None: + oprot.writeFieldBegin("principalType", TType.I32, 3) + oprot.writeI32(self.principalType) + oprot.writeFieldEnd() + if self.grantOption is not None: + oprot.writeFieldBegin("grantOption", TType.BOOL, 4) + oprot.writeBool(self.grantOption) + oprot.writeFieldEnd() + if self.grantTime is not None: + oprot.writeFieldBegin("grantTime", TType.I32, 5) + oprot.writeI32(self.grantTime) + oprot.writeFieldEnd() + if self.grantorName is not None: + oprot.writeFieldBegin("grantorName", TType.STRING, 6) + oprot.writeString(self.grantorName.encode("utf-8") if sys.version_info[0] == 2 else self.grantorName) + oprot.writeFieldEnd() + if self.grantorPrincipalType is not None: + oprot.writeFieldBegin("grantorPrincipalType", TType.I32, 7) + oprot.writeI32(self.grantorPrincipalType) + oprot.writeFieldEnd() + oprot.writeFieldStop() + oprot.writeStructEnd() + + def validate(self): + return + + def __repr__(self): + L = ["{}={!r}".format(key, value) for key, value in self.__dict__.items()] + return "{}({})".format(self.__class__.__name__, ", ".join(L)) + + def __eq__(self, other): + return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ + + def __ne__(self, other): + return not (self == other) + + +class GetRoleGrantsForPrincipalRequest: + """ + Attributes: + - principal_name + - principal_type + + """ + + def __init__( + self, + principal_name=None, + principal_type=None, + ): + self.principal_name = principal_name + self.principal_type = principal_type + + def read(self, iprot): + if ( + iprot._fast_decode is not None + and isinstance(iprot.trans, TTransport.CReadableTransport) + and self.thrift_spec is not None + ): + iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec]) + return + iprot.readStructBegin() + while True: + (fname, ftype, fid) = iprot.readFieldBegin() + if ftype == TType.STOP: + break + if fid == 1: + if ftype == TType.STRING: + self.principal_name = ( + iprot.readString().decode("utf-8", errors="replace") if sys.version_info[0] == 2 else iprot.readString() + ) + else: + iprot.skip(ftype) + elif fid == 2: + if ftype == TType.I32: + self.principal_type = iprot.readI32() + else: + iprot.skip(ftype) + else: + iprot.skip(ftype) + iprot.readFieldEnd() + iprot.readStructEnd() + + def write(self, oprot): + if oprot._fast_encode is not None and self.thrift_spec is not None: + oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec])) + return + oprot.writeStructBegin("GetRoleGrantsForPrincipalRequest") + if self.principal_name is not None: + oprot.writeFieldBegin("principal_name", TType.STRING, 1) + oprot.writeString(self.principal_name.encode("utf-8") if sys.version_info[0] == 2 else self.principal_name) + oprot.writeFieldEnd() + if self.principal_type is not None: + oprot.writeFieldBegin("principal_type", TType.I32, 2) + oprot.writeI32(self.principal_type) + oprot.writeFieldEnd() + oprot.writeFieldStop() + oprot.writeStructEnd() + + def validate(self): + if self.principal_name is None: + raise TProtocolException(message="Required field principal_name is unset!") + if self.principal_type is None: + raise TProtocolException(message="Required field principal_type is unset!") + return + + def __repr__(self): + L = ["{}={!r}".format(key, value) for key, value in self.__dict__.items()] + return "{}({})".format(self.__class__.__name__, ", ".join(L)) + + def __eq__(self, other): + return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ + + def __ne__(self, other): + return not (self == other) + + +class GetRoleGrantsForPrincipalResponse: + """ + Attributes: + - principalGrants + + """ + + def __init__( + self, + principalGrants=None, + ): + self.principalGrants = principalGrants + + def read(self, iprot): + if ( + iprot._fast_decode is not None + and isinstance(iprot.trans, TTransport.CReadableTransport) + and self.thrift_spec is not None + ): + iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec]) + return + iprot.readStructBegin() + while True: + (fname, ftype, fid) = iprot.readFieldBegin() + if ftype == TType.STOP: + break + if fid == 1: + if ftype == TType.LIST: + self.principalGrants = [] + (_etype130, _size127) = iprot.readListBegin() + for _i131 in range(_size127): + _elem132 = RolePrincipalGrant() + _elem132.read(iprot) + self.principalGrants.append(_elem132) + iprot.readListEnd() + else: + iprot.skip(ftype) + else: + iprot.skip(ftype) + iprot.readFieldEnd() + iprot.readStructEnd() + + def write(self, oprot): + if oprot._fast_encode is not None and self.thrift_spec is not None: + oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec])) + return + oprot.writeStructBegin("GetRoleGrantsForPrincipalResponse") + if self.principalGrants is not None: + oprot.writeFieldBegin("principalGrants", TType.LIST, 1) + oprot.writeListBegin(TType.STRUCT, len(self.principalGrants)) + for iter133 in self.principalGrants: + iter133.write(oprot) + oprot.writeListEnd() + oprot.writeFieldEnd() + oprot.writeFieldStop() + oprot.writeStructEnd() + + def validate(self): + if self.principalGrants is None: + raise TProtocolException(message="Required field principalGrants is unset!") + return + + def __repr__(self): + L = ["{}={!r}".format(key, value) for key, value in self.__dict__.items()] + return "{}({})".format(self.__class__.__name__, ", ".join(L)) + + def __eq__(self, other): + return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ + + def __ne__(self, other): + return not (self == other) + + +class GetPrincipalsInRoleRequest: + """ + Attributes: + - roleName + + """ + + def __init__( + self, + roleName=None, + ): + self.roleName = roleName + + def read(self, iprot): + if ( + iprot._fast_decode is not None + and isinstance(iprot.trans, TTransport.CReadableTransport) + and self.thrift_spec is not None + ): + iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec]) + return + iprot.readStructBegin() + while True: + (fname, ftype, fid) = iprot.readFieldBegin() + if ftype == TType.STOP: + break + if fid == 1: + if ftype == TType.STRING: + self.roleName = ( + iprot.readString().decode("utf-8", errors="replace") if sys.version_info[0] == 2 else iprot.readString() + ) + else: + iprot.skip(ftype) + else: + iprot.skip(ftype) + iprot.readFieldEnd() + iprot.readStructEnd() + + def write(self, oprot): + if oprot._fast_encode is not None and self.thrift_spec is not None: + oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec])) + return + oprot.writeStructBegin("GetPrincipalsInRoleRequest") + if self.roleName is not None: + oprot.writeFieldBegin("roleName", TType.STRING, 1) + oprot.writeString(self.roleName.encode("utf-8") if sys.version_info[0] == 2 else self.roleName) + oprot.writeFieldEnd() + oprot.writeFieldStop() + oprot.writeStructEnd() + + def validate(self): + if self.roleName is None: + raise TProtocolException(message="Required field roleName is unset!") + return + + def __repr__(self): + L = ["{}={!r}".format(key, value) for key, value in self.__dict__.items()] + return "{}({})".format(self.__class__.__name__, ", ".join(L)) + + def __eq__(self, other): + return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ + + def __ne__(self, other): + return not (self == other) + + +class GetPrincipalsInRoleResponse: + """ + Attributes: + - principalGrants + + """ + + def __init__( + self, + principalGrants=None, + ): + self.principalGrants = principalGrants + + def read(self, iprot): + if ( + iprot._fast_decode is not None + and isinstance(iprot.trans, TTransport.CReadableTransport) + and self.thrift_spec is not None + ): + iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec]) + return + iprot.readStructBegin() + while True: + (fname, ftype, fid) = iprot.readFieldBegin() + if ftype == TType.STOP: + break + if fid == 1: + if ftype == TType.LIST: + self.principalGrants = [] + (_etype137, _size134) = iprot.readListBegin() + for _i138 in range(_size134): + _elem139 = RolePrincipalGrant() + _elem139.read(iprot) + self.principalGrants.append(_elem139) + iprot.readListEnd() + else: + iprot.skip(ftype) + else: + iprot.skip(ftype) + iprot.readFieldEnd() + iprot.readStructEnd() + + def write(self, oprot): + if oprot._fast_encode is not None and self.thrift_spec is not None: + oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec])) + return + oprot.writeStructBegin("GetPrincipalsInRoleResponse") + if self.principalGrants is not None: + oprot.writeFieldBegin("principalGrants", TType.LIST, 1) + oprot.writeListBegin(TType.STRUCT, len(self.principalGrants)) + for iter140 in self.principalGrants: + iter140.write(oprot) + oprot.writeListEnd() + oprot.writeFieldEnd() + oprot.writeFieldStop() + oprot.writeStructEnd() + + def validate(self): + if self.principalGrants is None: + raise TProtocolException(message="Required field principalGrants is unset!") + return + + def __repr__(self): + L = ["{}={!r}".format(key, value) for key, value in self.__dict__.items()] + return "{}({})".format(self.__class__.__name__, ", ".join(L)) + + def __eq__(self, other): + return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ + + def __ne__(self, other): + return not (self == other) + + +class GrantRevokeRoleRequest: + """ + Attributes: + - requestType + - roleName + - principalName + - principalType + - grantor + - grantorType + - grantOption + + """ + + def __init__( + self, + requestType=None, + roleName=None, + principalName=None, + principalType=None, + grantor=None, + grantorType=None, + grantOption=None, + ): + self.requestType = requestType + self.roleName = roleName + self.principalName = principalName + self.principalType = principalType + self.grantor = grantor + self.grantorType = grantorType + self.grantOption = grantOption + + def read(self, iprot): + if ( + iprot._fast_decode is not None + and isinstance(iprot.trans, TTransport.CReadableTransport) + and self.thrift_spec is not None + ): + iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec]) + return + iprot.readStructBegin() + while True: + (fname, ftype, fid) = iprot.readFieldBegin() + if ftype == TType.STOP: + break + if fid == 1: + if ftype == TType.I32: + self.requestType = iprot.readI32() + else: + iprot.skip(ftype) + elif fid == 2: + if ftype == TType.STRING: + self.roleName = ( + iprot.readString().decode("utf-8", errors="replace") if sys.version_info[0] == 2 else iprot.readString() + ) + else: + iprot.skip(ftype) + elif fid == 3: + if ftype == TType.STRING: + self.principalName = ( + iprot.readString().decode("utf-8", errors="replace") if sys.version_info[0] == 2 else iprot.readString() + ) + else: + iprot.skip(ftype) + elif fid == 4: + if ftype == TType.I32: + self.principalType = iprot.readI32() + else: + iprot.skip(ftype) + elif fid == 5: + if ftype == TType.STRING: + self.grantor = ( + iprot.readString().decode("utf-8", errors="replace") if sys.version_info[0] == 2 else iprot.readString() + ) + else: + iprot.skip(ftype) + elif fid == 6: + if ftype == TType.I32: + self.grantorType = iprot.readI32() + else: + iprot.skip(ftype) + elif fid == 7: + if ftype == TType.BOOL: + self.grantOption = iprot.readBool() + else: + iprot.skip(ftype) + else: + iprot.skip(ftype) + iprot.readFieldEnd() + iprot.readStructEnd() + + def write(self, oprot): + if oprot._fast_encode is not None and self.thrift_spec is not None: + oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec])) + return + oprot.writeStructBegin("GrantRevokeRoleRequest") + if self.requestType is not None: + oprot.writeFieldBegin("requestType", TType.I32, 1) + oprot.writeI32(self.requestType) + oprot.writeFieldEnd() + if self.roleName is not None: + oprot.writeFieldBegin("roleName", TType.STRING, 2) + oprot.writeString(self.roleName.encode("utf-8") if sys.version_info[0] == 2 else self.roleName) + oprot.writeFieldEnd() + if self.principalName is not None: + oprot.writeFieldBegin("principalName", TType.STRING, 3) + oprot.writeString(self.principalName.encode("utf-8") if sys.version_info[0] == 2 else self.principalName) + oprot.writeFieldEnd() + if self.principalType is not None: + oprot.writeFieldBegin("principalType", TType.I32, 4) + oprot.writeI32(self.principalType) + oprot.writeFieldEnd() + if self.grantor is not None: + oprot.writeFieldBegin("grantor", TType.STRING, 5) + oprot.writeString(self.grantor.encode("utf-8") if sys.version_info[0] == 2 else self.grantor) + oprot.writeFieldEnd() + if self.grantorType is not None: + oprot.writeFieldBegin("grantorType", TType.I32, 6) + oprot.writeI32(self.grantorType) + oprot.writeFieldEnd() + if self.grantOption is not None: + oprot.writeFieldBegin("grantOption", TType.BOOL, 7) + oprot.writeBool(self.grantOption) + oprot.writeFieldEnd() + oprot.writeFieldStop() + oprot.writeStructEnd() + + def validate(self): + return + + def __repr__(self): + L = ["{}={!r}".format(key, value) for key, value in self.__dict__.items()] + return "{}({})".format(self.__class__.__name__, ", ".join(L)) + + def __eq__(self, other): + return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ + + def __ne__(self, other): + return not (self == other) + + +class GrantRevokeRoleResponse: + """ + Attributes: + - success + + """ + + def __init__( + self, + success=None, + ): + self.success = success + + def read(self, iprot): + if ( + iprot._fast_decode is not None + and isinstance(iprot.trans, TTransport.CReadableTransport) + and self.thrift_spec is not None + ): + iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec]) + return + iprot.readStructBegin() + while True: + (fname, ftype, fid) = iprot.readFieldBegin() + if ftype == TType.STOP: + break + if fid == 1: + if ftype == TType.BOOL: + self.success = iprot.readBool() + else: + iprot.skip(ftype) + else: + iprot.skip(ftype) + iprot.readFieldEnd() + iprot.readStructEnd() + + def write(self, oprot): + if oprot._fast_encode is not None and self.thrift_spec is not None: + oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec])) + return + oprot.writeStructBegin("GrantRevokeRoleResponse") + if self.success is not None: + oprot.writeFieldBegin("success", TType.BOOL, 1) + oprot.writeBool(self.success) + oprot.writeFieldEnd() + oprot.writeFieldStop() + oprot.writeStructEnd() + + def validate(self): + return + + def __repr__(self): + L = ["{}={!r}".format(key, value) for key, value in self.__dict__.items()] + return "{}({})".format(self.__class__.__name__, ", ".join(L)) + + def __eq__(self, other): + return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ + + def __ne__(self, other): + return not (self == other) + + +class Catalog: + """ + Attributes: + - name + - description + - locationUri + - createTime + + """ + + def __init__( + self, + name=None, + description=None, + locationUri=None, + createTime=None, + ): + self.name = name + self.description = description + self.locationUri = locationUri + self.createTime = createTime + + def read(self, iprot): + if ( + iprot._fast_decode is not None + and isinstance(iprot.trans, TTransport.CReadableTransport) + and self.thrift_spec is not None + ): + iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec]) + return + iprot.readStructBegin() + while True: + (fname, ftype, fid) = iprot.readFieldBegin() + if ftype == TType.STOP: + break + if fid == 1: + if ftype == TType.STRING: + self.name = ( + iprot.readString().decode("utf-8", errors="replace") if sys.version_info[0] == 2 else iprot.readString() + ) + else: + iprot.skip(ftype) + elif fid == 2: + if ftype == TType.STRING: + self.description = ( + iprot.readString().decode("utf-8", errors="replace") if sys.version_info[0] == 2 else iprot.readString() + ) + else: + iprot.skip(ftype) + elif fid == 3: + if ftype == TType.STRING: + self.locationUri = ( + iprot.readString().decode("utf-8", errors="replace") if sys.version_info[0] == 2 else iprot.readString() + ) + else: + iprot.skip(ftype) + elif fid == 4: + if ftype == TType.I32: + self.createTime = iprot.readI32() + else: + iprot.skip(ftype) + else: + iprot.skip(ftype) + iprot.readFieldEnd() + iprot.readStructEnd() + + def write(self, oprot): + if oprot._fast_encode is not None and self.thrift_spec is not None: + oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec])) + return + oprot.writeStructBegin("Catalog") + if self.name is not None: + oprot.writeFieldBegin("name", TType.STRING, 1) + oprot.writeString(self.name.encode("utf-8") if sys.version_info[0] == 2 else self.name) + oprot.writeFieldEnd() + if self.description is not None: + oprot.writeFieldBegin("description", TType.STRING, 2) + oprot.writeString(self.description.encode("utf-8") if sys.version_info[0] == 2 else self.description) + oprot.writeFieldEnd() + if self.locationUri is not None: + oprot.writeFieldBegin("locationUri", TType.STRING, 3) + oprot.writeString(self.locationUri.encode("utf-8") if sys.version_info[0] == 2 else self.locationUri) + oprot.writeFieldEnd() + if self.createTime is not None: + oprot.writeFieldBegin("createTime", TType.I32, 4) + oprot.writeI32(self.createTime) + oprot.writeFieldEnd() + oprot.writeFieldStop() + oprot.writeStructEnd() + + def validate(self): + return + + def __repr__(self): + L = ["{}={!r}".format(key, value) for key, value in self.__dict__.items()] + return "{}({})".format(self.__class__.__name__, ", ".join(L)) + + def __eq__(self, other): + return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ + + def __ne__(self, other): + return not (self == other) + + +class CreateCatalogRequest: + """ + Attributes: + - catalog + + """ + + def __init__( + self, + catalog=None, + ): + self.catalog = catalog + + def read(self, iprot): + if ( + iprot._fast_decode is not None + and isinstance(iprot.trans, TTransport.CReadableTransport) + and self.thrift_spec is not None + ): + iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec]) + return + iprot.readStructBegin() + while True: + (fname, ftype, fid) = iprot.readFieldBegin() + if ftype == TType.STOP: + break + if fid == 1: + if ftype == TType.STRUCT: + self.catalog = Catalog() + self.catalog.read(iprot) + else: + iprot.skip(ftype) + else: + iprot.skip(ftype) + iprot.readFieldEnd() + iprot.readStructEnd() + + def write(self, oprot): + if oprot._fast_encode is not None and self.thrift_spec is not None: + oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec])) + return + oprot.writeStructBegin("CreateCatalogRequest") + if self.catalog is not None: + oprot.writeFieldBegin("catalog", TType.STRUCT, 1) + self.catalog.write(oprot) + oprot.writeFieldEnd() + oprot.writeFieldStop() + oprot.writeStructEnd() + + def validate(self): + return + + def __repr__(self): + L = ["{}={!r}".format(key, value) for key, value in self.__dict__.items()] + return "{}({})".format(self.__class__.__name__, ", ".join(L)) + + def __eq__(self, other): + return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ + + def __ne__(self, other): + return not (self == other) + + +class AlterCatalogRequest: + """ + Attributes: + - name + - newCat + + """ + + def __init__( + self, + name=None, + newCat=None, + ): + self.name = name + self.newCat = newCat + + def read(self, iprot): + if ( + iprot._fast_decode is not None + and isinstance(iprot.trans, TTransport.CReadableTransport) + and self.thrift_spec is not None + ): + iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec]) + return + iprot.readStructBegin() + while True: + (fname, ftype, fid) = iprot.readFieldBegin() + if ftype == TType.STOP: + break + if fid == 1: + if ftype == TType.STRING: + self.name = ( + iprot.readString().decode("utf-8", errors="replace") if sys.version_info[0] == 2 else iprot.readString() + ) + else: + iprot.skip(ftype) + elif fid == 2: + if ftype == TType.STRUCT: + self.newCat = Catalog() + self.newCat.read(iprot) + else: + iprot.skip(ftype) + else: + iprot.skip(ftype) + iprot.readFieldEnd() + iprot.readStructEnd() + + def write(self, oprot): + if oprot._fast_encode is not None and self.thrift_spec is not None: + oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec])) + return + oprot.writeStructBegin("AlterCatalogRequest") + if self.name is not None: + oprot.writeFieldBegin("name", TType.STRING, 1) + oprot.writeString(self.name.encode("utf-8") if sys.version_info[0] == 2 else self.name) + oprot.writeFieldEnd() + if self.newCat is not None: + oprot.writeFieldBegin("newCat", TType.STRUCT, 2) + self.newCat.write(oprot) + oprot.writeFieldEnd() + oprot.writeFieldStop() + oprot.writeStructEnd() + + def validate(self): + return + + def __repr__(self): + L = ["{}={!r}".format(key, value) for key, value in self.__dict__.items()] + return "{}({})".format(self.__class__.__name__, ", ".join(L)) + + def __eq__(self, other): + return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ + + def __ne__(self, other): + return not (self == other) + + +class GetCatalogRequest: + """ + Attributes: + - name + + """ + + def __init__( + self, + name=None, + ): + self.name = name + + def read(self, iprot): + if ( + iprot._fast_decode is not None + and isinstance(iprot.trans, TTransport.CReadableTransport) + and self.thrift_spec is not None + ): + iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec]) + return + iprot.readStructBegin() + while True: + (fname, ftype, fid) = iprot.readFieldBegin() + if ftype == TType.STOP: + break + if fid == 1: + if ftype == TType.STRING: + self.name = ( + iprot.readString().decode("utf-8", errors="replace") if sys.version_info[0] == 2 else iprot.readString() + ) + else: + iprot.skip(ftype) + else: + iprot.skip(ftype) + iprot.readFieldEnd() + iprot.readStructEnd() + + def write(self, oprot): + if oprot._fast_encode is not None and self.thrift_spec is not None: + oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec])) + return + oprot.writeStructBegin("GetCatalogRequest") + if self.name is not None: + oprot.writeFieldBegin("name", TType.STRING, 1) + oprot.writeString(self.name.encode("utf-8") if sys.version_info[0] == 2 else self.name) + oprot.writeFieldEnd() + oprot.writeFieldStop() + oprot.writeStructEnd() + + def validate(self): + return + + def __repr__(self): + L = ["{}={!r}".format(key, value) for key, value in self.__dict__.items()] + return "{}({})".format(self.__class__.__name__, ", ".join(L)) + + def __eq__(self, other): + return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ + + def __ne__(self, other): + return not (self == other) + + +class GetCatalogResponse: + """ + Attributes: + - catalog + + """ + + def __init__( + self, + catalog=None, + ): + self.catalog = catalog + + def read(self, iprot): + if ( + iprot._fast_decode is not None + and isinstance(iprot.trans, TTransport.CReadableTransport) + and self.thrift_spec is not None + ): + iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec]) + return + iprot.readStructBegin() + while True: + (fname, ftype, fid) = iprot.readFieldBegin() + if ftype == TType.STOP: + break + if fid == 1: + if ftype == TType.STRUCT: + self.catalog = Catalog() + self.catalog.read(iprot) + else: + iprot.skip(ftype) + else: + iprot.skip(ftype) + iprot.readFieldEnd() + iprot.readStructEnd() + + def write(self, oprot): + if oprot._fast_encode is not None and self.thrift_spec is not None: + oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec])) + return + oprot.writeStructBegin("GetCatalogResponse") + if self.catalog is not None: + oprot.writeFieldBegin("catalog", TType.STRUCT, 1) + self.catalog.write(oprot) + oprot.writeFieldEnd() + oprot.writeFieldStop() + oprot.writeStructEnd() + + def validate(self): + return + + def __repr__(self): + L = ["{}={!r}".format(key, value) for key, value in self.__dict__.items()] + return "{}({})".format(self.__class__.__name__, ", ".join(L)) + + def __eq__(self, other): + return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ + + def __ne__(self, other): + return not (self == other) + + +class GetCatalogsResponse: + """ + Attributes: + - names + + """ + + def __init__( + self, + names=None, + ): + self.names = names + + def read(self, iprot): + if ( + iprot._fast_decode is not None + and isinstance(iprot.trans, TTransport.CReadableTransport) + and self.thrift_spec is not None + ): + iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec]) + return + iprot.readStructBegin() + while True: + (fname, ftype, fid) = iprot.readFieldBegin() + if ftype == TType.STOP: + break + if fid == 1: + if ftype == TType.LIST: + self.names = [] + (_etype144, _size141) = iprot.readListBegin() + for _i145 in range(_size141): + _elem146 = ( + iprot.readString().decode("utf-8", errors="replace") + if sys.version_info[0] == 2 + else iprot.readString() + ) + self.names.append(_elem146) + iprot.readListEnd() + else: + iprot.skip(ftype) + else: + iprot.skip(ftype) + iprot.readFieldEnd() + iprot.readStructEnd() + + def write(self, oprot): + if oprot._fast_encode is not None and self.thrift_spec is not None: + oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec])) + return + oprot.writeStructBegin("GetCatalogsResponse") + if self.names is not None: + oprot.writeFieldBegin("names", TType.LIST, 1) + oprot.writeListBegin(TType.STRING, len(self.names)) + for iter147 in self.names: + oprot.writeString(iter147.encode("utf-8") if sys.version_info[0] == 2 else iter147) + oprot.writeListEnd() + oprot.writeFieldEnd() + oprot.writeFieldStop() + oprot.writeStructEnd() + + def validate(self): + return + + def __repr__(self): + L = ["{}={!r}".format(key, value) for key, value in self.__dict__.items()] + return "{}({})".format(self.__class__.__name__, ", ".join(L)) + + def __eq__(self, other): + return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ + + def __ne__(self, other): + return not (self == other) + + +class DropCatalogRequest: + """ + Attributes: + - name + + """ + + def __init__( + self, + name=None, + ): + self.name = name + + def read(self, iprot): + if ( + iprot._fast_decode is not None + and isinstance(iprot.trans, TTransport.CReadableTransport) + and self.thrift_spec is not None + ): + iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec]) + return + iprot.readStructBegin() + while True: + (fname, ftype, fid) = iprot.readFieldBegin() + if ftype == TType.STOP: + break + if fid == 1: + if ftype == TType.STRING: + self.name = ( + iprot.readString().decode("utf-8", errors="replace") if sys.version_info[0] == 2 else iprot.readString() + ) + else: + iprot.skip(ftype) + else: + iprot.skip(ftype) + iprot.readFieldEnd() + iprot.readStructEnd() + + def write(self, oprot): + if oprot._fast_encode is not None and self.thrift_spec is not None: + oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec])) + return + oprot.writeStructBegin("DropCatalogRequest") + if self.name is not None: + oprot.writeFieldBegin("name", TType.STRING, 1) + oprot.writeString(self.name.encode("utf-8") if sys.version_info[0] == 2 else self.name) + oprot.writeFieldEnd() + oprot.writeFieldStop() + oprot.writeStructEnd() + + def validate(self): + return + + def __repr__(self): + L = ["{}={!r}".format(key, value) for key, value in self.__dict__.items()] + return "{}({})".format(self.__class__.__name__, ", ".join(L)) + + def __eq__(self, other): + return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ + + def __ne__(self, other): + return not (self == other) + + +class Database: + """ + Attributes: + - name + - description + - locationUri + - parameters + - privileges + - ownerName + - ownerType + - catalogName + - createTime + - managedLocationUri + - type + - connector_name + - remote_dbname + + """ + + def __init__( + self, + name=None, + description=None, + locationUri=None, + parameters=None, + privileges=None, + ownerName=None, + ownerType=None, + catalogName=None, + createTime=None, + managedLocationUri=None, + type=None, + connector_name=None, + remote_dbname=None, + ): + self.name = name + self.description = description + self.locationUri = locationUri + self.parameters = parameters + self.privileges = privileges + self.ownerName = ownerName + self.ownerType = ownerType + self.catalogName = catalogName + self.createTime = createTime + self.managedLocationUri = managedLocationUri + self.type = type + self.connector_name = connector_name + self.remote_dbname = remote_dbname + + def read(self, iprot): + if ( + iprot._fast_decode is not None + and isinstance(iprot.trans, TTransport.CReadableTransport) + and self.thrift_spec is not None + ): + iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec]) + return + iprot.readStructBegin() + while True: + (fname, ftype, fid) = iprot.readFieldBegin() + if ftype == TType.STOP: + break + if fid == 1: + if ftype == TType.STRING: + self.name = ( + iprot.readString().decode("utf-8", errors="replace") if sys.version_info[0] == 2 else iprot.readString() + ) + else: + iprot.skip(ftype) + elif fid == 2: + if ftype == TType.STRING: + self.description = ( + iprot.readString().decode("utf-8", errors="replace") if sys.version_info[0] == 2 else iprot.readString() + ) + else: + iprot.skip(ftype) + elif fid == 3: + if ftype == TType.STRING: + self.locationUri = ( + iprot.readString().decode("utf-8", errors="replace") if sys.version_info[0] == 2 else iprot.readString() + ) + else: + iprot.skip(ftype) + elif fid == 4: + if ftype == TType.MAP: + self.parameters = {} + (_ktype149, _vtype150, _size148) = iprot.readMapBegin() + for _i152 in range(_size148): + _key153 = ( + iprot.readString().decode("utf-8", errors="replace") + if sys.version_info[0] == 2 + else iprot.readString() + ) + _val154 = ( + iprot.readString().decode("utf-8", errors="replace") + if sys.version_info[0] == 2 + else iprot.readString() + ) + self.parameters[_key153] = _val154 + iprot.readMapEnd() + else: + iprot.skip(ftype) + elif fid == 5: + if ftype == TType.STRUCT: + self.privileges = PrincipalPrivilegeSet() + self.privileges.read(iprot) + else: + iprot.skip(ftype) + elif fid == 6: + if ftype == TType.STRING: + self.ownerName = ( + iprot.readString().decode("utf-8", errors="replace") if sys.version_info[0] == 2 else iprot.readString() + ) + else: + iprot.skip(ftype) + elif fid == 7: + if ftype == TType.I32: + self.ownerType = iprot.readI32() + else: + iprot.skip(ftype) + elif fid == 8: + if ftype == TType.STRING: + self.catalogName = ( + iprot.readString().decode("utf-8", errors="replace") if sys.version_info[0] == 2 else iprot.readString() + ) + else: + iprot.skip(ftype) + elif fid == 9: + if ftype == TType.I32: + self.createTime = iprot.readI32() + else: + iprot.skip(ftype) + elif fid == 10: + if ftype == TType.STRING: + self.managedLocationUri = ( + iprot.readString().decode("utf-8", errors="replace") if sys.version_info[0] == 2 else iprot.readString() + ) + else: + iprot.skip(ftype) + elif fid == 11: + if ftype == TType.I32: + self.type = iprot.readI32() + else: + iprot.skip(ftype) + elif fid == 12: + if ftype == TType.STRING: + self.connector_name = ( + iprot.readString().decode("utf-8", errors="replace") if sys.version_info[0] == 2 else iprot.readString() + ) + else: + iprot.skip(ftype) + elif fid == 13: + if ftype == TType.STRING: + self.remote_dbname = ( + iprot.readString().decode("utf-8", errors="replace") if sys.version_info[0] == 2 else iprot.readString() + ) + else: + iprot.skip(ftype) + else: + iprot.skip(ftype) + iprot.readFieldEnd() + iprot.readStructEnd() + + def write(self, oprot): + if oprot._fast_encode is not None and self.thrift_spec is not None: + oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec])) + return + oprot.writeStructBegin("Database") + if self.name is not None: + oprot.writeFieldBegin("name", TType.STRING, 1) + oprot.writeString(self.name.encode("utf-8") if sys.version_info[0] == 2 else self.name) + oprot.writeFieldEnd() + if self.description is not None: + oprot.writeFieldBegin("description", TType.STRING, 2) + oprot.writeString(self.description.encode("utf-8") if sys.version_info[0] == 2 else self.description) + oprot.writeFieldEnd() + if self.locationUri is not None: + oprot.writeFieldBegin("locationUri", TType.STRING, 3) + oprot.writeString(self.locationUri.encode("utf-8") if sys.version_info[0] == 2 else self.locationUri) + oprot.writeFieldEnd() + if self.parameters is not None: + oprot.writeFieldBegin("parameters", TType.MAP, 4) + oprot.writeMapBegin(TType.STRING, TType.STRING, len(self.parameters)) + for kiter155, viter156 in self.parameters.items(): + oprot.writeString(kiter155.encode("utf-8") if sys.version_info[0] == 2 else kiter155) + oprot.writeString(viter156.encode("utf-8") if sys.version_info[0] == 2 else viter156) + oprot.writeMapEnd() + oprot.writeFieldEnd() + if self.privileges is not None: + oprot.writeFieldBegin("privileges", TType.STRUCT, 5) + self.privileges.write(oprot) + oprot.writeFieldEnd() + if self.ownerName is not None: + oprot.writeFieldBegin("ownerName", TType.STRING, 6) + oprot.writeString(self.ownerName.encode("utf-8") if sys.version_info[0] == 2 else self.ownerName) + oprot.writeFieldEnd() + if self.ownerType is not None: + oprot.writeFieldBegin("ownerType", TType.I32, 7) + oprot.writeI32(self.ownerType) + oprot.writeFieldEnd() + if self.catalogName is not None: + oprot.writeFieldBegin("catalogName", TType.STRING, 8) + oprot.writeString(self.catalogName.encode("utf-8") if sys.version_info[0] == 2 else self.catalogName) + oprot.writeFieldEnd() + if self.createTime is not None: + oprot.writeFieldBegin("createTime", TType.I32, 9) + oprot.writeI32(self.createTime) + oprot.writeFieldEnd() + if self.managedLocationUri is not None: + oprot.writeFieldBegin("managedLocationUri", TType.STRING, 10) + oprot.writeString(self.managedLocationUri.encode("utf-8") if sys.version_info[0] == 2 else self.managedLocationUri) + oprot.writeFieldEnd() + if self.type is not None: + oprot.writeFieldBegin("type", TType.I32, 11) + oprot.writeI32(self.type) + oprot.writeFieldEnd() + if self.connector_name is not None: + oprot.writeFieldBegin("connector_name", TType.STRING, 12) + oprot.writeString(self.connector_name.encode("utf-8") if sys.version_info[0] == 2 else self.connector_name) + oprot.writeFieldEnd() + if self.remote_dbname is not None: + oprot.writeFieldBegin("remote_dbname", TType.STRING, 13) + oprot.writeString(self.remote_dbname.encode("utf-8") if sys.version_info[0] == 2 else self.remote_dbname) + oprot.writeFieldEnd() + oprot.writeFieldStop() + oprot.writeStructEnd() + + def validate(self): + return + + def __repr__(self): + L = ["{}={!r}".format(key, value) for key, value in self.__dict__.items()] + return "{}({})".format(self.__class__.__name__, ", ".join(L)) + + def __eq__(self, other): + return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ + + def __ne__(self, other): + return not (self == other) + + +class SerDeInfo: + """ + Attributes: + - name + - serializationLib + - parameters + - description + - serializerClass + - deserializerClass + - serdeType + + """ + + def __init__( + self, + name=None, + serializationLib=None, + parameters=None, + description=None, + serializerClass=None, + deserializerClass=None, + serdeType=None, + ): + self.name = name + self.serializationLib = serializationLib + self.parameters = parameters + self.description = description + self.serializerClass = serializerClass + self.deserializerClass = deserializerClass + self.serdeType = serdeType + + def read(self, iprot): + if ( + iprot._fast_decode is not None + and isinstance(iprot.trans, TTransport.CReadableTransport) + and self.thrift_spec is not None + ): + iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec]) + return + iprot.readStructBegin() + while True: + (fname, ftype, fid) = iprot.readFieldBegin() + if ftype == TType.STOP: + break + if fid == 1: + if ftype == TType.STRING: + self.name = ( + iprot.readString().decode("utf-8", errors="replace") if sys.version_info[0] == 2 else iprot.readString() + ) + else: + iprot.skip(ftype) + elif fid == 2: + if ftype == TType.STRING: + self.serializationLib = ( + iprot.readString().decode("utf-8", errors="replace") if sys.version_info[0] == 2 else iprot.readString() + ) + else: + iprot.skip(ftype) + elif fid == 3: + if ftype == TType.MAP: + self.parameters = {} + (_ktype158, _vtype159, _size157) = iprot.readMapBegin() + for _i161 in range(_size157): + _key162 = ( + iprot.readString().decode("utf-8", errors="replace") + if sys.version_info[0] == 2 + else iprot.readString() + ) + _val163 = ( + iprot.readString().decode("utf-8", errors="replace") + if sys.version_info[0] == 2 + else iprot.readString() + ) + self.parameters[_key162] = _val163 + iprot.readMapEnd() + else: + iprot.skip(ftype) + elif fid == 4: + if ftype == TType.STRING: + self.description = ( + iprot.readString().decode("utf-8", errors="replace") if sys.version_info[0] == 2 else iprot.readString() + ) + else: + iprot.skip(ftype) + elif fid == 5: + if ftype == TType.STRING: + self.serializerClass = ( + iprot.readString().decode("utf-8", errors="replace") if sys.version_info[0] == 2 else iprot.readString() + ) + else: + iprot.skip(ftype) + elif fid == 6: + if ftype == TType.STRING: + self.deserializerClass = ( + iprot.readString().decode("utf-8", errors="replace") if sys.version_info[0] == 2 else iprot.readString() + ) + else: + iprot.skip(ftype) + elif fid == 7: + if ftype == TType.I32: + self.serdeType = iprot.readI32() + else: + iprot.skip(ftype) + else: + iprot.skip(ftype) + iprot.readFieldEnd() + iprot.readStructEnd() + + def write(self, oprot): + if oprot._fast_encode is not None and self.thrift_spec is not None: + oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec])) + return + oprot.writeStructBegin("SerDeInfo") + if self.name is not None: + oprot.writeFieldBegin("name", TType.STRING, 1) + oprot.writeString(self.name.encode("utf-8") if sys.version_info[0] == 2 else self.name) + oprot.writeFieldEnd() + if self.serializationLib is not None: + oprot.writeFieldBegin("serializationLib", TType.STRING, 2) + oprot.writeString(self.serializationLib.encode("utf-8") if sys.version_info[0] == 2 else self.serializationLib) + oprot.writeFieldEnd() + if self.parameters is not None: + oprot.writeFieldBegin("parameters", TType.MAP, 3) + oprot.writeMapBegin(TType.STRING, TType.STRING, len(self.parameters)) + for kiter164, viter165 in self.parameters.items(): + oprot.writeString(kiter164.encode("utf-8") if sys.version_info[0] == 2 else kiter164) + oprot.writeString(viter165.encode("utf-8") if sys.version_info[0] == 2 else viter165) + oprot.writeMapEnd() + oprot.writeFieldEnd() + if self.description is not None: + oprot.writeFieldBegin("description", TType.STRING, 4) + oprot.writeString(self.description.encode("utf-8") if sys.version_info[0] == 2 else self.description) + oprot.writeFieldEnd() + if self.serializerClass is not None: + oprot.writeFieldBegin("serializerClass", TType.STRING, 5) + oprot.writeString(self.serializerClass.encode("utf-8") if sys.version_info[0] == 2 else self.serializerClass) + oprot.writeFieldEnd() + if self.deserializerClass is not None: + oprot.writeFieldBegin("deserializerClass", TType.STRING, 6) + oprot.writeString(self.deserializerClass.encode("utf-8") if sys.version_info[0] == 2 else self.deserializerClass) + oprot.writeFieldEnd() + if self.serdeType is not None: + oprot.writeFieldBegin("serdeType", TType.I32, 7) + oprot.writeI32(self.serdeType) + oprot.writeFieldEnd() + oprot.writeFieldStop() + oprot.writeStructEnd() + + def validate(self): + return + + def __repr__(self): + L = ["{}={!r}".format(key, value) for key, value in self.__dict__.items()] + return "{}({})".format(self.__class__.__name__, ", ".join(L)) + + def __eq__(self, other): + return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ + + def __ne__(self, other): + return not (self == other) + + +class Order: + """ + Attributes: + - col + - order + + """ + + def __init__( + self, + col=None, + order=None, + ): + self.col = col + self.order = order + + def read(self, iprot): + if ( + iprot._fast_decode is not None + and isinstance(iprot.trans, TTransport.CReadableTransport) + and self.thrift_spec is not None + ): + iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec]) + return + iprot.readStructBegin() + while True: + (fname, ftype, fid) = iprot.readFieldBegin() + if ftype == TType.STOP: + break + if fid == 1: + if ftype == TType.STRING: + self.col = ( + iprot.readString().decode("utf-8", errors="replace") if sys.version_info[0] == 2 else iprot.readString() + ) + else: + iprot.skip(ftype) + elif fid == 2: + if ftype == TType.I32: + self.order = iprot.readI32() + else: + iprot.skip(ftype) + else: + iprot.skip(ftype) + iprot.readFieldEnd() + iprot.readStructEnd() + + def write(self, oprot): + if oprot._fast_encode is not None and self.thrift_spec is not None: + oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec])) + return + oprot.writeStructBegin("Order") + if self.col is not None: + oprot.writeFieldBegin("col", TType.STRING, 1) + oprot.writeString(self.col.encode("utf-8") if sys.version_info[0] == 2 else self.col) + oprot.writeFieldEnd() + if self.order is not None: + oprot.writeFieldBegin("order", TType.I32, 2) + oprot.writeI32(self.order) + oprot.writeFieldEnd() + oprot.writeFieldStop() + oprot.writeStructEnd() + + def validate(self): + return + + def __repr__(self): + L = ["{}={!r}".format(key, value) for key, value in self.__dict__.items()] + return "{}({})".format(self.__class__.__name__, ", ".join(L)) + + def __eq__(self, other): + return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ + + def __ne__(self, other): + return not (self == other) + + +class SkewedInfo: + """ + Attributes: + - skewedColNames + - skewedColValues + - skewedColValueLocationMaps + + """ + + def __init__( + self, + skewedColNames=None, + skewedColValues=None, + skewedColValueLocationMaps=None, + ): + self.skewedColNames = skewedColNames + self.skewedColValues = skewedColValues + self.skewedColValueLocationMaps = skewedColValueLocationMaps + + def read(self, iprot): + if ( + iprot._fast_decode is not None + and isinstance(iprot.trans, TTransport.CReadableTransport) + and self.thrift_spec is not None + ): + iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec]) + return + iprot.readStructBegin() + while True: + (fname, ftype, fid) = iprot.readFieldBegin() + if ftype == TType.STOP: + break + if fid == 1: + if ftype == TType.LIST: + self.skewedColNames = [] + (_etype169, _size166) = iprot.readListBegin() + for _i170 in range(_size166): + _elem171 = ( + iprot.readString().decode("utf-8", errors="replace") + if sys.version_info[0] == 2 + else iprot.readString() + ) + self.skewedColNames.append(_elem171) + iprot.readListEnd() + else: + iprot.skip(ftype) + elif fid == 2: + if ftype == TType.LIST: + self.skewedColValues = [] + (_etype175, _size172) = iprot.readListBegin() + for _i176 in range(_size172): + _elem177 = [] + (_etype181, _size178) = iprot.readListBegin() + for _i182 in range(_size178): + _elem183 = ( + iprot.readString().decode("utf-8", errors="replace") + if sys.version_info[0] == 2 + else iprot.readString() + ) + _elem177.append(_elem183) + iprot.readListEnd() + self.skewedColValues.append(_elem177) + iprot.readListEnd() + else: + iprot.skip(ftype) + elif fid == 3: + if ftype == TType.MAP: + self.skewedColValueLocationMaps = {} + (_ktype185, _vtype186, _size184) = iprot.readMapBegin() + for _i188 in range(_size184): + _key189 = [] + (_etype194, _size191) = iprot.readListBegin() + for _i195 in range(_size191): + _elem196 = ( + iprot.readString().decode("utf-8", errors="replace") + if sys.version_info[0] == 2 + else iprot.readString() + ) + _key189.append(_elem196) + iprot.readListEnd() + _val190 = ( + iprot.readString().decode("utf-8", errors="replace") + if sys.version_info[0] == 2 + else iprot.readString() + ) + self.skewedColValueLocationMaps[_key189] = _val190 + iprot.readMapEnd() + else: + iprot.skip(ftype) + else: + iprot.skip(ftype) + iprot.readFieldEnd() + iprot.readStructEnd() + + def write(self, oprot): + if oprot._fast_encode is not None and self.thrift_spec is not None: + oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec])) + return + oprot.writeStructBegin("SkewedInfo") + if self.skewedColNames is not None: + oprot.writeFieldBegin("skewedColNames", TType.LIST, 1) + oprot.writeListBegin(TType.STRING, len(self.skewedColNames)) + for iter197 in self.skewedColNames: + oprot.writeString(iter197.encode("utf-8") if sys.version_info[0] == 2 else iter197) + oprot.writeListEnd() + oprot.writeFieldEnd() + if self.skewedColValues is not None: + oprot.writeFieldBegin("skewedColValues", TType.LIST, 2) + oprot.writeListBegin(TType.LIST, len(self.skewedColValues)) + for iter198 in self.skewedColValues: + oprot.writeListBegin(TType.STRING, len(iter198)) + for iter199 in iter198: + oprot.writeString(iter199.encode("utf-8") if sys.version_info[0] == 2 else iter199) + oprot.writeListEnd() + oprot.writeListEnd() + oprot.writeFieldEnd() + if self.skewedColValueLocationMaps is not None: + oprot.writeFieldBegin("skewedColValueLocationMaps", TType.MAP, 3) + oprot.writeMapBegin(TType.LIST, TType.STRING, len(self.skewedColValueLocationMaps)) + for kiter200, viter201 in self.skewedColValueLocationMaps.items(): + oprot.writeListBegin(TType.STRING, len(kiter200)) + for iter202 in kiter200: + oprot.writeString(iter202.encode("utf-8") if sys.version_info[0] == 2 else iter202) + oprot.writeListEnd() + oprot.writeString(viter201.encode("utf-8") if sys.version_info[0] == 2 else viter201) + oprot.writeMapEnd() + oprot.writeFieldEnd() + oprot.writeFieldStop() + oprot.writeStructEnd() + + def validate(self): + return + + def __repr__(self): + L = ["{}={!r}".format(key, value) for key, value in self.__dict__.items()] + return "{}({})".format(self.__class__.__name__, ", ".join(L)) + + def __eq__(self, other): + return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ + + def __ne__(self, other): + return not (self == other) + + +class StorageDescriptor: + """ + Attributes: + - cols + - location + - inputFormat + - outputFormat + - compressed + - numBuckets + - serdeInfo + - bucketCols + - sortCols + - parameters + - skewedInfo + - storedAsSubDirectories + + """ + + def __init__( + self, + cols=None, + location=None, + inputFormat=None, + outputFormat=None, + compressed=None, + numBuckets=None, + serdeInfo=None, + bucketCols=None, + sortCols=None, + parameters=None, + skewedInfo=None, + storedAsSubDirectories=None, + ): + self.cols = cols + self.location = location + self.inputFormat = inputFormat + self.outputFormat = outputFormat + self.compressed = compressed + self.numBuckets = numBuckets + self.serdeInfo = serdeInfo + self.bucketCols = bucketCols + self.sortCols = sortCols + self.parameters = parameters + self.skewedInfo = skewedInfo + self.storedAsSubDirectories = storedAsSubDirectories + + def read(self, iprot): + if ( + iprot._fast_decode is not None + and isinstance(iprot.trans, TTransport.CReadableTransport) + and self.thrift_spec is not None + ): + iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec]) + return + iprot.readStructBegin() + while True: + (fname, ftype, fid) = iprot.readFieldBegin() + if ftype == TType.STOP: + break + if fid == 1: + if ftype == TType.LIST: + self.cols = [] + (_etype206, _size203) = iprot.readListBegin() + for _i207 in range(_size203): + _elem208 = FieldSchema() + _elem208.read(iprot) + self.cols.append(_elem208) + iprot.readListEnd() + else: + iprot.skip(ftype) + elif fid == 2: + if ftype == TType.STRING: + self.location = ( + iprot.readString().decode("utf-8", errors="replace") if sys.version_info[0] == 2 else iprot.readString() + ) + else: + iprot.skip(ftype) + elif fid == 3: + if ftype == TType.STRING: + self.inputFormat = ( + iprot.readString().decode("utf-8", errors="replace") if sys.version_info[0] == 2 else iprot.readString() + ) + else: + iprot.skip(ftype) + elif fid == 4: + if ftype == TType.STRING: + self.outputFormat = ( + iprot.readString().decode("utf-8", errors="replace") if sys.version_info[0] == 2 else iprot.readString() + ) + else: + iprot.skip(ftype) + elif fid == 5: + if ftype == TType.BOOL: + self.compressed = iprot.readBool() + else: + iprot.skip(ftype) + elif fid == 6: + if ftype == TType.I32: + self.numBuckets = iprot.readI32() + else: + iprot.skip(ftype) + elif fid == 7: + if ftype == TType.STRUCT: + self.serdeInfo = SerDeInfo() + self.serdeInfo.read(iprot) + else: + iprot.skip(ftype) + elif fid == 8: + if ftype == TType.LIST: + self.bucketCols = [] + (_etype212, _size209) = iprot.readListBegin() + for _i213 in range(_size209): + _elem214 = ( + iprot.readString().decode("utf-8", errors="replace") + if sys.version_info[0] == 2 + else iprot.readString() + ) + self.bucketCols.append(_elem214) + iprot.readListEnd() + else: + iprot.skip(ftype) + elif fid == 9: + if ftype == TType.LIST: + self.sortCols = [] + (_etype218, _size215) = iprot.readListBegin() + for _i219 in range(_size215): + _elem220 = Order() + _elem220.read(iprot) + self.sortCols.append(_elem220) + iprot.readListEnd() + else: + iprot.skip(ftype) + elif fid == 10: + if ftype == TType.MAP: + self.parameters = {} + (_ktype222, _vtype223, _size221) = iprot.readMapBegin() + for _i225 in range(_size221): + _key226 = ( + iprot.readString().decode("utf-8", errors="replace") + if sys.version_info[0] == 2 + else iprot.readString() + ) + _val227 = ( + iprot.readString().decode("utf-8", errors="replace") + if sys.version_info[0] == 2 + else iprot.readString() + ) + self.parameters[_key226] = _val227 + iprot.readMapEnd() + else: + iprot.skip(ftype) + elif fid == 11: + if ftype == TType.STRUCT: + self.skewedInfo = SkewedInfo() + self.skewedInfo.read(iprot) + else: + iprot.skip(ftype) + elif fid == 12: + if ftype == TType.BOOL: + self.storedAsSubDirectories = iprot.readBool() + else: + iprot.skip(ftype) + else: + iprot.skip(ftype) + iprot.readFieldEnd() + iprot.readStructEnd() + + def write(self, oprot): + if oprot._fast_encode is not None and self.thrift_spec is not None: + oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec])) + return + oprot.writeStructBegin("StorageDescriptor") + if self.cols is not None: + oprot.writeFieldBegin("cols", TType.LIST, 1) + oprot.writeListBegin(TType.STRUCT, len(self.cols)) + for iter228 in self.cols: + iter228.write(oprot) + oprot.writeListEnd() + oprot.writeFieldEnd() + if self.location is not None: + oprot.writeFieldBegin("location", TType.STRING, 2) + oprot.writeString(self.location.encode("utf-8") if sys.version_info[0] == 2 else self.location) + oprot.writeFieldEnd() + if self.inputFormat is not None: + oprot.writeFieldBegin("inputFormat", TType.STRING, 3) + oprot.writeString(self.inputFormat.encode("utf-8") if sys.version_info[0] == 2 else self.inputFormat) + oprot.writeFieldEnd() + if self.outputFormat is not None: + oprot.writeFieldBegin("outputFormat", TType.STRING, 4) + oprot.writeString(self.outputFormat.encode("utf-8") if sys.version_info[0] == 2 else self.outputFormat) + oprot.writeFieldEnd() + if self.compressed is not None: + oprot.writeFieldBegin("compressed", TType.BOOL, 5) + oprot.writeBool(self.compressed) + oprot.writeFieldEnd() + if self.numBuckets is not None: + oprot.writeFieldBegin("numBuckets", TType.I32, 6) + oprot.writeI32(self.numBuckets) + oprot.writeFieldEnd() + if self.serdeInfo is not None: + oprot.writeFieldBegin("serdeInfo", TType.STRUCT, 7) + self.serdeInfo.write(oprot) + oprot.writeFieldEnd() + if self.bucketCols is not None: + oprot.writeFieldBegin("bucketCols", TType.LIST, 8) + oprot.writeListBegin(TType.STRING, len(self.bucketCols)) + for iter229 in self.bucketCols: + oprot.writeString(iter229.encode("utf-8") if sys.version_info[0] == 2 else iter229) + oprot.writeListEnd() + oprot.writeFieldEnd() + if self.sortCols is not None: + oprot.writeFieldBegin("sortCols", TType.LIST, 9) + oprot.writeListBegin(TType.STRUCT, len(self.sortCols)) + for iter230 in self.sortCols: + iter230.write(oprot) + oprot.writeListEnd() + oprot.writeFieldEnd() + if self.parameters is not None: + oprot.writeFieldBegin("parameters", TType.MAP, 10) + oprot.writeMapBegin(TType.STRING, TType.STRING, len(self.parameters)) + for kiter231, viter232 in self.parameters.items(): + oprot.writeString(kiter231.encode("utf-8") if sys.version_info[0] == 2 else kiter231) + oprot.writeString(viter232.encode("utf-8") if sys.version_info[0] == 2 else viter232) + oprot.writeMapEnd() + oprot.writeFieldEnd() + if self.skewedInfo is not None: + oprot.writeFieldBegin("skewedInfo", TType.STRUCT, 11) + self.skewedInfo.write(oprot) + oprot.writeFieldEnd() + if self.storedAsSubDirectories is not None: + oprot.writeFieldBegin("storedAsSubDirectories", TType.BOOL, 12) + oprot.writeBool(self.storedAsSubDirectories) + oprot.writeFieldEnd() + oprot.writeFieldStop() + oprot.writeStructEnd() + + def validate(self): + return + + def __repr__(self): + L = ["{}={!r}".format(key, value) for key, value in self.__dict__.items()] + return "{}({})".format(self.__class__.__name__, ", ".join(L)) + + def __eq__(self, other): + return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ + + def __ne__(self, other): + return not (self == other) + + +class CreationMetadata: + """ + Attributes: + - catName + - dbName + - tblName + - tablesUsed + - validTxnList + - materializationTime + - sourceTables + + """ + + def __init__( + self, + catName=None, + dbName=None, + tblName=None, + tablesUsed=None, + validTxnList=None, + materializationTime=None, + sourceTables=None, + ): + self.catName = catName + self.dbName = dbName + self.tblName = tblName + self.tablesUsed = tablesUsed + self.validTxnList = validTxnList + self.materializationTime = materializationTime + self.sourceTables = sourceTables + + def read(self, iprot): + if ( + iprot._fast_decode is not None + and isinstance(iprot.trans, TTransport.CReadableTransport) + and self.thrift_spec is not None + ): + iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec]) + return + iprot.readStructBegin() + while True: + (fname, ftype, fid) = iprot.readFieldBegin() + if ftype == TType.STOP: + break + if fid == 1: + if ftype == TType.STRING: + self.catName = ( + iprot.readString().decode("utf-8", errors="replace") if sys.version_info[0] == 2 else iprot.readString() + ) + else: + iprot.skip(ftype) + elif fid == 2: + if ftype == TType.STRING: + self.dbName = ( + iprot.readString().decode("utf-8", errors="replace") if sys.version_info[0] == 2 else iprot.readString() + ) + else: + iprot.skip(ftype) + elif fid == 3: + if ftype == TType.STRING: + self.tblName = ( + iprot.readString().decode("utf-8", errors="replace") if sys.version_info[0] == 2 else iprot.readString() + ) + else: + iprot.skip(ftype) + elif fid == 4: + if ftype == TType.SET: + self.tablesUsed = set() + (_etype236, _size233) = iprot.readSetBegin() + for _i237 in range(_size233): + _elem238 = ( + iprot.readString().decode("utf-8", errors="replace") + if sys.version_info[0] == 2 + else iprot.readString() + ) + self.tablesUsed.add(_elem238) + iprot.readSetEnd() + else: + iprot.skip(ftype) + elif fid == 5: + if ftype == TType.STRING: + self.validTxnList = ( + iprot.readString().decode("utf-8", errors="replace") if sys.version_info[0] == 2 else iprot.readString() + ) + else: + iprot.skip(ftype) + elif fid == 6: + if ftype == TType.I64: + self.materializationTime = iprot.readI64() + else: + iprot.skip(ftype) + elif fid == 7: + if ftype == TType.LIST: + self.sourceTables = [] + (_etype242, _size239) = iprot.readListBegin() + for _i243 in range(_size239): + _elem244 = SourceTable() + _elem244.read(iprot) + self.sourceTables.append(_elem244) + iprot.readListEnd() + else: + iprot.skip(ftype) + else: + iprot.skip(ftype) + iprot.readFieldEnd() + iprot.readStructEnd() + + def write(self, oprot): + if oprot._fast_encode is not None and self.thrift_spec is not None: + oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec])) + return + oprot.writeStructBegin("CreationMetadata") + if self.catName is not None: + oprot.writeFieldBegin("catName", TType.STRING, 1) + oprot.writeString(self.catName.encode("utf-8") if sys.version_info[0] == 2 else self.catName) + oprot.writeFieldEnd() + if self.dbName is not None: + oprot.writeFieldBegin("dbName", TType.STRING, 2) + oprot.writeString(self.dbName.encode("utf-8") if sys.version_info[0] == 2 else self.dbName) + oprot.writeFieldEnd() + if self.tblName is not None: + oprot.writeFieldBegin("tblName", TType.STRING, 3) + oprot.writeString(self.tblName.encode("utf-8") if sys.version_info[0] == 2 else self.tblName) + oprot.writeFieldEnd() + if self.tablesUsed is not None: + oprot.writeFieldBegin("tablesUsed", TType.SET, 4) + oprot.writeSetBegin(TType.STRING, len(self.tablesUsed)) + for iter245 in self.tablesUsed: + oprot.writeString(iter245.encode("utf-8") if sys.version_info[0] == 2 else iter245) + oprot.writeSetEnd() + oprot.writeFieldEnd() + if self.validTxnList is not None: + oprot.writeFieldBegin("validTxnList", TType.STRING, 5) + oprot.writeString(self.validTxnList.encode("utf-8") if sys.version_info[0] == 2 else self.validTxnList) + oprot.writeFieldEnd() + if self.materializationTime is not None: + oprot.writeFieldBegin("materializationTime", TType.I64, 6) + oprot.writeI64(self.materializationTime) + oprot.writeFieldEnd() + if self.sourceTables is not None: + oprot.writeFieldBegin("sourceTables", TType.LIST, 7) + oprot.writeListBegin(TType.STRUCT, len(self.sourceTables)) + for iter246 in self.sourceTables: + iter246.write(oprot) + oprot.writeListEnd() + oprot.writeFieldEnd() + oprot.writeFieldStop() + oprot.writeStructEnd() + + def validate(self): + if self.catName is None: + raise TProtocolException(message="Required field catName is unset!") + if self.dbName is None: + raise TProtocolException(message="Required field dbName is unset!") + if self.tblName is None: + raise TProtocolException(message="Required field tblName is unset!") + if self.tablesUsed is None: + raise TProtocolException(message="Required field tablesUsed is unset!") + return + + def __repr__(self): + L = ["{}={!r}".format(key, value) for key, value in self.__dict__.items()] + return "{}({})".format(self.__class__.__name__, ", ".join(L)) + + def __eq__(self, other): + return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ + + def __ne__(self, other): + return not (self == other) + + +class BooleanColumnStatsData: + """ + Attributes: + - numTrues + - numFalses + - numNulls + - bitVectors + + """ + + def __init__( + self, + numTrues=None, + numFalses=None, + numNulls=None, + bitVectors=None, + ): + self.numTrues = numTrues + self.numFalses = numFalses + self.numNulls = numNulls + self.bitVectors = bitVectors + + def read(self, iprot): + if ( + iprot._fast_decode is not None + and isinstance(iprot.trans, TTransport.CReadableTransport) + and self.thrift_spec is not None + ): + iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec]) + return + iprot.readStructBegin() + while True: + (fname, ftype, fid) = iprot.readFieldBegin() + if ftype == TType.STOP: + break + if fid == 1: + if ftype == TType.I64: + self.numTrues = iprot.readI64() + else: + iprot.skip(ftype) + elif fid == 2: + if ftype == TType.I64: + self.numFalses = iprot.readI64() + else: + iprot.skip(ftype) + elif fid == 3: + if ftype == TType.I64: + self.numNulls = iprot.readI64() + else: + iprot.skip(ftype) + elif fid == 4: + if ftype == TType.STRING: + self.bitVectors = iprot.readBinary() + else: + iprot.skip(ftype) + else: + iprot.skip(ftype) + iprot.readFieldEnd() + iprot.readStructEnd() + + def write(self, oprot): + if oprot._fast_encode is not None and self.thrift_spec is not None: + oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec])) + return + oprot.writeStructBegin("BooleanColumnStatsData") + if self.numTrues is not None: + oprot.writeFieldBegin("numTrues", TType.I64, 1) + oprot.writeI64(self.numTrues) + oprot.writeFieldEnd() + if self.numFalses is not None: + oprot.writeFieldBegin("numFalses", TType.I64, 2) + oprot.writeI64(self.numFalses) + oprot.writeFieldEnd() + if self.numNulls is not None: + oprot.writeFieldBegin("numNulls", TType.I64, 3) + oprot.writeI64(self.numNulls) + oprot.writeFieldEnd() + if self.bitVectors is not None: + oprot.writeFieldBegin("bitVectors", TType.STRING, 4) + oprot.writeBinary(self.bitVectors) + oprot.writeFieldEnd() + oprot.writeFieldStop() + oprot.writeStructEnd() + + def validate(self): + if self.numTrues is None: + raise TProtocolException(message="Required field numTrues is unset!") + if self.numFalses is None: + raise TProtocolException(message="Required field numFalses is unset!") + if self.numNulls is None: + raise TProtocolException(message="Required field numNulls is unset!") + return + + def __repr__(self): + L = ["{}={!r}".format(key, value) for key, value in self.__dict__.items()] + return "{}({})".format(self.__class__.__name__, ", ".join(L)) + + def __eq__(self, other): + return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ + + def __ne__(self, other): + return not (self == other) + + +class DoubleColumnStatsData: + """ + Attributes: + - lowValue + - highValue + - numNulls + - numDVs + - bitVectors + + """ + + def __init__( + self, + lowValue=None, + highValue=None, + numNulls=None, + numDVs=None, + bitVectors=None, + ): + self.lowValue = lowValue + self.highValue = highValue + self.numNulls = numNulls + self.numDVs = numDVs + self.bitVectors = bitVectors + + def read(self, iprot): + if ( + iprot._fast_decode is not None + and isinstance(iprot.trans, TTransport.CReadableTransport) + and self.thrift_spec is not None + ): + iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec]) + return + iprot.readStructBegin() + while True: + (fname, ftype, fid) = iprot.readFieldBegin() + if ftype == TType.STOP: + break + if fid == 1: + if ftype == TType.DOUBLE: + self.lowValue = iprot.readDouble() + else: + iprot.skip(ftype) + elif fid == 2: + if ftype == TType.DOUBLE: + self.highValue = iprot.readDouble() + else: + iprot.skip(ftype) + elif fid == 3: + if ftype == TType.I64: + self.numNulls = iprot.readI64() + else: + iprot.skip(ftype) + elif fid == 4: + if ftype == TType.I64: + self.numDVs = iprot.readI64() + else: + iprot.skip(ftype) + elif fid == 5: + if ftype == TType.STRING: + self.bitVectors = iprot.readBinary() + else: + iprot.skip(ftype) + else: + iprot.skip(ftype) + iprot.readFieldEnd() + iprot.readStructEnd() + + def write(self, oprot): + if oprot._fast_encode is not None and self.thrift_spec is not None: + oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec])) + return + oprot.writeStructBegin("DoubleColumnStatsData") + if self.lowValue is not None: + oprot.writeFieldBegin("lowValue", TType.DOUBLE, 1) + oprot.writeDouble(self.lowValue) + oprot.writeFieldEnd() + if self.highValue is not None: + oprot.writeFieldBegin("highValue", TType.DOUBLE, 2) + oprot.writeDouble(self.highValue) + oprot.writeFieldEnd() + if self.numNulls is not None: + oprot.writeFieldBegin("numNulls", TType.I64, 3) + oprot.writeI64(self.numNulls) + oprot.writeFieldEnd() + if self.numDVs is not None: + oprot.writeFieldBegin("numDVs", TType.I64, 4) + oprot.writeI64(self.numDVs) + oprot.writeFieldEnd() + if self.bitVectors is not None: + oprot.writeFieldBegin("bitVectors", TType.STRING, 5) + oprot.writeBinary(self.bitVectors) + oprot.writeFieldEnd() + oprot.writeFieldStop() + oprot.writeStructEnd() + + def validate(self): + if self.numNulls is None: + raise TProtocolException(message="Required field numNulls is unset!") + if self.numDVs is None: + raise TProtocolException(message="Required field numDVs is unset!") + return + + def __repr__(self): + L = ["{}={!r}".format(key, value) for key, value in self.__dict__.items()] + return "{}({})".format(self.__class__.__name__, ", ".join(L)) + + def __eq__(self, other): + return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ + + def __ne__(self, other): + return not (self == other) + + +class LongColumnStatsData: + """ + Attributes: + - lowValue + - highValue + - numNulls + - numDVs + - bitVectors + + """ + + def __init__( + self, + lowValue=None, + highValue=None, + numNulls=None, + numDVs=None, + bitVectors=None, + ): + self.lowValue = lowValue + self.highValue = highValue + self.numNulls = numNulls + self.numDVs = numDVs + self.bitVectors = bitVectors + + def read(self, iprot): + if ( + iprot._fast_decode is not None + and isinstance(iprot.trans, TTransport.CReadableTransport) + and self.thrift_spec is not None + ): + iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec]) + return + iprot.readStructBegin() + while True: + (fname, ftype, fid) = iprot.readFieldBegin() + if ftype == TType.STOP: + break + if fid == 1: + if ftype == TType.I64: + self.lowValue = iprot.readI64() + else: + iprot.skip(ftype) + elif fid == 2: + if ftype == TType.I64: + self.highValue = iprot.readI64() + else: + iprot.skip(ftype) + elif fid == 3: + if ftype == TType.I64: + self.numNulls = iprot.readI64() + else: + iprot.skip(ftype) + elif fid == 4: + if ftype == TType.I64: + self.numDVs = iprot.readI64() + else: + iprot.skip(ftype) + elif fid == 5: + if ftype == TType.STRING: + self.bitVectors = iprot.readBinary() + else: + iprot.skip(ftype) + else: + iprot.skip(ftype) + iprot.readFieldEnd() + iprot.readStructEnd() + + def write(self, oprot): + if oprot._fast_encode is not None and self.thrift_spec is not None: + oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec])) + return + oprot.writeStructBegin("LongColumnStatsData") + if self.lowValue is not None: + oprot.writeFieldBegin("lowValue", TType.I64, 1) + oprot.writeI64(self.lowValue) + oprot.writeFieldEnd() + if self.highValue is not None: + oprot.writeFieldBegin("highValue", TType.I64, 2) + oprot.writeI64(self.highValue) + oprot.writeFieldEnd() + if self.numNulls is not None: + oprot.writeFieldBegin("numNulls", TType.I64, 3) + oprot.writeI64(self.numNulls) + oprot.writeFieldEnd() + if self.numDVs is not None: + oprot.writeFieldBegin("numDVs", TType.I64, 4) + oprot.writeI64(self.numDVs) + oprot.writeFieldEnd() + if self.bitVectors is not None: + oprot.writeFieldBegin("bitVectors", TType.STRING, 5) + oprot.writeBinary(self.bitVectors) + oprot.writeFieldEnd() + oprot.writeFieldStop() + oprot.writeStructEnd() + + def validate(self): + if self.numNulls is None: + raise TProtocolException(message="Required field numNulls is unset!") + if self.numDVs is None: + raise TProtocolException(message="Required field numDVs is unset!") + return + + def __repr__(self): + L = ["{}={!r}".format(key, value) for key, value in self.__dict__.items()] + return "{}({})".format(self.__class__.__name__, ", ".join(L)) + + def __eq__(self, other): + return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ + + def __ne__(self, other): + return not (self == other) + + +class StringColumnStatsData: + """ + Attributes: + - maxColLen + - avgColLen + - numNulls + - numDVs + - bitVectors + + """ + + def __init__( + self, + maxColLen=None, + avgColLen=None, + numNulls=None, + numDVs=None, + bitVectors=None, + ): + self.maxColLen = maxColLen + self.avgColLen = avgColLen + self.numNulls = numNulls + self.numDVs = numDVs + self.bitVectors = bitVectors + + def read(self, iprot): + if ( + iprot._fast_decode is not None + and isinstance(iprot.trans, TTransport.CReadableTransport) + and self.thrift_spec is not None + ): + iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec]) + return + iprot.readStructBegin() + while True: + (fname, ftype, fid) = iprot.readFieldBegin() + if ftype == TType.STOP: + break + if fid == 1: + if ftype == TType.I64: + self.maxColLen = iprot.readI64() + else: + iprot.skip(ftype) + elif fid == 2: + if ftype == TType.DOUBLE: + self.avgColLen = iprot.readDouble() + else: + iprot.skip(ftype) + elif fid == 3: + if ftype == TType.I64: + self.numNulls = iprot.readI64() + else: + iprot.skip(ftype) + elif fid == 4: + if ftype == TType.I64: + self.numDVs = iprot.readI64() + else: + iprot.skip(ftype) + elif fid == 5: + if ftype == TType.STRING: + self.bitVectors = iprot.readBinary() + else: + iprot.skip(ftype) + else: + iprot.skip(ftype) + iprot.readFieldEnd() + iprot.readStructEnd() + + def write(self, oprot): + if oprot._fast_encode is not None and self.thrift_spec is not None: + oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec])) + return + oprot.writeStructBegin("StringColumnStatsData") + if self.maxColLen is not None: + oprot.writeFieldBegin("maxColLen", TType.I64, 1) + oprot.writeI64(self.maxColLen) + oprot.writeFieldEnd() + if self.avgColLen is not None: + oprot.writeFieldBegin("avgColLen", TType.DOUBLE, 2) + oprot.writeDouble(self.avgColLen) + oprot.writeFieldEnd() + if self.numNulls is not None: + oprot.writeFieldBegin("numNulls", TType.I64, 3) + oprot.writeI64(self.numNulls) + oprot.writeFieldEnd() + if self.numDVs is not None: + oprot.writeFieldBegin("numDVs", TType.I64, 4) + oprot.writeI64(self.numDVs) + oprot.writeFieldEnd() + if self.bitVectors is not None: + oprot.writeFieldBegin("bitVectors", TType.STRING, 5) + oprot.writeBinary(self.bitVectors) + oprot.writeFieldEnd() + oprot.writeFieldStop() + oprot.writeStructEnd() + + def validate(self): + if self.maxColLen is None: + raise TProtocolException(message="Required field maxColLen is unset!") + if self.avgColLen is None: + raise TProtocolException(message="Required field avgColLen is unset!") + if self.numNulls is None: + raise TProtocolException(message="Required field numNulls is unset!") + if self.numDVs is None: + raise TProtocolException(message="Required field numDVs is unset!") + return + + def __repr__(self): + L = ["{}={!r}".format(key, value) for key, value in self.__dict__.items()] + return "{}({})".format(self.__class__.__name__, ", ".join(L)) + + def __eq__(self, other): + return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ + + def __ne__(self, other): + return not (self == other) + + +class BinaryColumnStatsData: + """ + Attributes: + - maxColLen + - avgColLen + - numNulls + - bitVectors + + """ + + def __init__( + self, + maxColLen=None, + avgColLen=None, + numNulls=None, + bitVectors=None, + ): + self.maxColLen = maxColLen + self.avgColLen = avgColLen + self.numNulls = numNulls + self.bitVectors = bitVectors + + def read(self, iprot): + if ( + iprot._fast_decode is not None + and isinstance(iprot.trans, TTransport.CReadableTransport) + and self.thrift_spec is not None + ): + iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec]) + return + iprot.readStructBegin() + while True: + (fname, ftype, fid) = iprot.readFieldBegin() + if ftype == TType.STOP: + break + if fid == 1: + if ftype == TType.I64: + self.maxColLen = iprot.readI64() + else: + iprot.skip(ftype) + elif fid == 2: + if ftype == TType.DOUBLE: + self.avgColLen = iprot.readDouble() + else: + iprot.skip(ftype) + elif fid == 3: + if ftype == TType.I64: + self.numNulls = iprot.readI64() + else: + iprot.skip(ftype) + elif fid == 4: + if ftype == TType.STRING: + self.bitVectors = iprot.readBinary() + else: + iprot.skip(ftype) + else: + iprot.skip(ftype) + iprot.readFieldEnd() + iprot.readStructEnd() + + def write(self, oprot): + if oprot._fast_encode is not None and self.thrift_spec is not None: + oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec])) + return + oprot.writeStructBegin("BinaryColumnStatsData") + if self.maxColLen is not None: + oprot.writeFieldBegin("maxColLen", TType.I64, 1) + oprot.writeI64(self.maxColLen) + oprot.writeFieldEnd() + if self.avgColLen is not None: + oprot.writeFieldBegin("avgColLen", TType.DOUBLE, 2) + oprot.writeDouble(self.avgColLen) + oprot.writeFieldEnd() + if self.numNulls is not None: + oprot.writeFieldBegin("numNulls", TType.I64, 3) + oprot.writeI64(self.numNulls) + oprot.writeFieldEnd() + if self.bitVectors is not None: + oprot.writeFieldBegin("bitVectors", TType.STRING, 4) + oprot.writeBinary(self.bitVectors) + oprot.writeFieldEnd() + oprot.writeFieldStop() + oprot.writeStructEnd() + + def validate(self): + if self.maxColLen is None: + raise TProtocolException(message="Required field maxColLen is unset!") + if self.avgColLen is None: + raise TProtocolException(message="Required field avgColLen is unset!") + if self.numNulls is None: + raise TProtocolException(message="Required field numNulls is unset!") + return + + def __repr__(self): + L = ["{}={!r}".format(key, value) for key, value in self.__dict__.items()] + return "{}({})".format(self.__class__.__name__, ", ".join(L)) + + def __eq__(self, other): + return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ + + def __ne__(self, other): + return not (self == other) + + +class Decimal: + """ + Attributes: + - scale + - unscaled + + """ + + def __init__( + self, + scale=None, + unscaled=None, + ): + self.scale = scale + self.unscaled = unscaled + + def read(self, iprot): + if ( + iprot._fast_decode is not None + and isinstance(iprot.trans, TTransport.CReadableTransport) + and self.thrift_spec is not None + ): + iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec]) + return + iprot.readStructBegin() + while True: + (fname, ftype, fid) = iprot.readFieldBegin() + if ftype == TType.STOP: + break + if fid == 3: + if ftype == TType.I16: + self.scale = iprot.readI16() + else: + iprot.skip(ftype) + elif fid == 1: + if ftype == TType.STRING: + self.unscaled = iprot.readBinary() + else: + iprot.skip(ftype) + else: + iprot.skip(ftype) + iprot.readFieldEnd() + iprot.readStructEnd() + + def write(self, oprot): + if oprot._fast_encode is not None and self.thrift_spec is not None: + oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec])) + return + oprot.writeStructBegin("Decimal") + if self.unscaled is not None: + oprot.writeFieldBegin("unscaled", TType.STRING, 1) + oprot.writeBinary(self.unscaled) + oprot.writeFieldEnd() + if self.scale is not None: + oprot.writeFieldBegin("scale", TType.I16, 3) + oprot.writeI16(self.scale) + oprot.writeFieldEnd() + oprot.writeFieldStop() + oprot.writeStructEnd() + + def validate(self): + if self.scale is None: + raise TProtocolException(message="Required field scale is unset!") + if self.unscaled is None: + raise TProtocolException(message="Required field unscaled is unset!") + return + + def __repr__(self): + L = ["{}={!r}".format(key, value) for key, value in self.__dict__.items()] + return "{}({})".format(self.__class__.__name__, ", ".join(L)) + + def __eq__(self, other): + return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ + + def __ne__(self, other): + return not (self == other) + + +class DecimalColumnStatsData: + """ + Attributes: + - lowValue + - highValue + - numNulls + - numDVs + - bitVectors + + """ + + def __init__( + self, + lowValue=None, + highValue=None, + numNulls=None, + numDVs=None, + bitVectors=None, + ): + self.lowValue = lowValue + self.highValue = highValue + self.numNulls = numNulls + self.numDVs = numDVs + self.bitVectors = bitVectors + + def read(self, iprot): + if ( + iprot._fast_decode is not None + and isinstance(iprot.trans, TTransport.CReadableTransport) + and self.thrift_spec is not None + ): + iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec]) + return + iprot.readStructBegin() + while True: + (fname, ftype, fid) = iprot.readFieldBegin() + if ftype == TType.STOP: + break + if fid == 1: + if ftype == TType.STRUCT: + self.lowValue = Decimal() + self.lowValue.read(iprot) + else: + iprot.skip(ftype) + elif fid == 2: + if ftype == TType.STRUCT: + self.highValue = Decimal() + self.highValue.read(iprot) + else: + iprot.skip(ftype) + elif fid == 3: + if ftype == TType.I64: + self.numNulls = iprot.readI64() + else: + iprot.skip(ftype) + elif fid == 4: + if ftype == TType.I64: + self.numDVs = iprot.readI64() + else: + iprot.skip(ftype) + elif fid == 5: + if ftype == TType.STRING: + self.bitVectors = iprot.readBinary() + else: + iprot.skip(ftype) + else: + iprot.skip(ftype) + iprot.readFieldEnd() + iprot.readStructEnd() + + def write(self, oprot): + if oprot._fast_encode is not None and self.thrift_spec is not None: + oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec])) + return + oprot.writeStructBegin("DecimalColumnStatsData") + if self.lowValue is not None: + oprot.writeFieldBegin("lowValue", TType.STRUCT, 1) + self.lowValue.write(oprot) + oprot.writeFieldEnd() + if self.highValue is not None: + oprot.writeFieldBegin("highValue", TType.STRUCT, 2) + self.highValue.write(oprot) + oprot.writeFieldEnd() + if self.numNulls is not None: + oprot.writeFieldBegin("numNulls", TType.I64, 3) + oprot.writeI64(self.numNulls) + oprot.writeFieldEnd() + if self.numDVs is not None: + oprot.writeFieldBegin("numDVs", TType.I64, 4) + oprot.writeI64(self.numDVs) + oprot.writeFieldEnd() + if self.bitVectors is not None: + oprot.writeFieldBegin("bitVectors", TType.STRING, 5) + oprot.writeBinary(self.bitVectors) + oprot.writeFieldEnd() + oprot.writeFieldStop() + oprot.writeStructEnd() + + def validate(self): + if self.numNulls is None: + raise TProtocolException(message="Required field numNulls is unset!") + if self.numDVs is None: + raise TProtocolException(message="Required field numDVs is unset!") + return + + def __repr__(self): + L = ["{}={!r}".format(key, value) for key, value in self.__dict__.items()] + return "{}({})".format(self.__class__.__name__, ", ".join(L)) + + def __eq__(self, other): + return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ + + def __ne__(self, other): + return not (self == other) + + +class Date: + """ + Attributes: + - daysSinceEpoch + + """ + + def __init__( + self, + daysSinceEpoch=None, + ): + self.daysSinceEpoch = daysSinceEpoch + + def read(self, iprot): + if ( + iprot._fast_decode is not None + and isinstance(iprot.trans, TTransport.CReadableTransport) + and self.thrift_spec is not None + ): + iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec]) + return + iprot.readStructBegin() + while True: + (fname, ftype, fid) = iprot.readFieldBegin() + if ftype == TType.STOP: + break + if fid == 1: + if ftype == TType.I64: + self.daysSinceEpoch = iprot.readI64() + else: + iprot.skip(ftype) + else: + iprot.skip(ftype) + iprot.readFieldEnd() + iprot.readStructEnd() + + def write(self, oprot): + if oprot._fast_encode is not None and self.thrift_spec is not None: + oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec])) + return + oprot.writeStructBegin("Date") + if self.daysSinceEpoch is not None: + oprot.writeFieldBegin("daysSinceEpoch", TType.I64, 1) + oprot.writeI64(self.daysSinceEpoch) + oprot.writeFieldEnd() + oprot.writeFieldStop() + oprot.writeStructEnd() + + def validate(self): + if self.daysSinceEpoch is None: + raise TProtocolException(message="Required field daysSinceEpoch is unset!") + return + + def __repr__(self): + L = ["{}={!r}".format(key, value) for key, value in self.__dict__.items()] + return "{}({})".format(self.__class__.__name__, ", ".join(L)) + + def __eq__(self, other): + return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ + + def __ne__(self, other): + return not (self == other) + + +class DateColumnStatsData: + """ + Attributes: + - lowValue + - highValue + - numNulls + - numDVs + - bitVectors + + """ + + def __init__( + self, + lowValue=None, + highValue=None, + numNulls=None, + numDVs=None, + bitVectors=None, + ): + self.lowValue = lowValue + self.highValue = highValue + self.numNulls = numNulls + self.numDVs = numDVs + self.bitVectors = bitVectors + + def read(self, iprot): + if ( + iprot._fast_decode is not None + and isinstance(iprot.trans, TTransport.CReadableTransport) + and self.thrift_spec is not None + ): + iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec]) + return + iprot.readStructBegin() + while True: + (fname, ftype, fid) = iprot.readFieldBegin() + if ftype == TType.STOP: + break + if fid == 1: + if ftype == TType.STRUCT: + self.lowValue = Date() + self.lowValue.read(iprot) + else: + iprot.skip(ftype) + elif fid == 2: + if ftype == TType.STRUCT: + self.highValue = Date() + self.highValue.read(iprot) + else: + iprot.skip(ftype) + elif fid == 3: + if ftype == TType.I64: + self.numNulls = iprot.readI64() + else: + iprot.skip(ftype) + elif fid == 4: + if ftype == TType.I64: + self.numDVs = iprot.readI64() + else: + iprot.skip(ftype) + elif fid == 5: + if ftype == TType.STRING: + self.bitVectors = iprot.readBinary() + else: + iprot.skip(ftype) + else: + iprot.skip(ftype) + iprot.readFieldEnd() + iprot.readStructEnd() + + def write(self, oprot): + if oprot._fast_encode is not None and self.thrift_spec is not None: + oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec])) + return + oprot.writeStructBegin("DateColumnStatsData") + if self.lowValue is not None: + oprot.writeFieldBegin("lowValue", TType.STRUCT, 1) + self.lowValue.write(oprot) + oprot.writeFieldEnd() + if self.highValue is not None: + oprot.writeFieldBegin("highValue", TType.STRUCT, 2) + self.highValue.write(oprot) + oprot.writeFieldEnd() + if self.numNulls is not None: + oprot.writeFieldBegin("numNulls", TType.I64, 3) + oprot.writeI64(self.numNulls) + oprot.writeFieldEnd() + if self.numDVs is not None: + oprot.writeFieldBegin("numDVs", TType.I64, 4) + oprot.writeI64(self.numDVs) + oprot.writeFieldEnd() + if self.bitVectors is not None: + oprot.writeFieldBegin("bitVectors", TType.STRING, 5) + oprot.writeBinary(self.bitVectors) + oprot.writeFieldEnd() + oprot.writeFieldStop() + oprot.writeStructEnd() + + def validate(self): + if self.numNulls is None: + raise TProtocolException(message="Required field numNulls is unset!") + if self.numDVs is None: + raise TProtocolException(message="Required field numDVs is unset!") + return + + def __repr__(self): + L = ["{}={!r}".format(key, value) for key, value in self.__dict__.items()] + return "{}({})".format(self.__class__.__name__, ", ".join(L)) + + def __eq__(self, other): + return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ + + def __ne__(self, other): + return not (self == other) + + +class Timestamp: + """ + Attributes: + - secondsSinceEpoch + + """ + + def __init__( + self, + secondsSinceEpoch=None, + ): + self.secondsSinceEpoch = secondsSinceEpoch + + def read(self, iprot): + if ( + iprot._fast_decode is not None + and isinstance(iprot.trans, TTransport.CReadableTransport) + and self.thrift_spec is not None + ): + iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec]) + return + iprot.readStructBegin() + while True: + (fname, ftype, fid) = iprot.readFieldBegin() + if ftype == TType.STOP: + break + if fid == 1: + if ftype == TType.I64: + self.secondsSinceEpoch = iprot.readI64() + else: + iprot.skip(ftype) + else: + iprot.skip(ftype) + iprot.readFieldEnd() + iprot.readStructEnd() + + def write(self, oprot): + if oprot._fast_encode is not None and self.thrift_spec is not None: + oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec])) + return + oprot.writeStructBegin("Timestamp") + if self.secondsSinceEpoch is not None: + oprot.writeFieldBegin("secondsSinceEpoch", TType.I64, 1) + oprot.writeI64(self.secondsSinceEpoch) + oprot.writeFieldEnd() + oprot.writeFieldStop() + oprot.writeStructEnd() + + def validate(self): + if self.secondsSinceEpoch is None: + raise TProtocolException(message="Required field secondsSinceEpoch is unset!") + return + + def __repr__(self): + L = ["{}={!r}".format(key, value) for key, value in self.__dict__.items()] + return "{}({})".format(self.__class__.__name__, ", ".join(L)) + + def __eq__(self, other): + return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ + + def __ne__(self, other): + return not (self == other) + + +class TimestampColumnStatsData: + """ + Attributes: + - lowValue + - highValue + - numNulls + - numDVs + - bitVectors + + """ + + def __init__( + self, + lowValue=None, + highValue=None, + numNulls=None, + numDVs=None, + bitVectors=None, + ): + self.lowValue = lowValue + self.highValue = highValue + self.numNulls = numNulls + self.numDVs = numDVs + self.bitVectors = bitVectors + + def read(self, iprot): + if ( + iprot._fast_decode is not None + and isinstance(iprot.trans, TTransport.CReadableTransport) + and self.thrift_spec is not None + ): + iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec]) + return + iprot.readStructBegin() + while True: + (fname, ftype, fid) = iprot.readFieldBegin() + if ftype == TType.STOP: + break + if fid == 1: + if ftype == TType.STRUCT: + self.lowValue = Timestamp() + self.lowValue.read(iprot) + else: + iprot.skip(ftype) + elif fid == 2: + if ftype == TType.STRUCT: + self.highValue = Timestamp() + self.highValue.read(iprot) + else: + iprot.skip(ftype) + elif fid == 3: + if ftype == TType.I64: + self.numNulls = iprot.readI64() + else: + iprot.skip(ftype) + elif fid == 4: + if ftype == TType.I64: + self.numDVs = iprot.readI64() + else: + iprot.skip(ftype) + elif fid == 5: + if ftype == TType.STRING: + self.bitVectors = iprot.readBinary() + else: + iprot.skip(ftype) + else: + iprot.skip(ftype) + iprot.readFieldEnd() + iprot.readStructEnd() + + def write(self, oprot): + if oprot._fast_encode is not None and self.thrift_spec is not None: + oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec])) + return + oprot.writeStructBegin("TimestampColumnStatsData") + if self.lowValue is not None: + oprot.writeFieldBegin("lowValue", TType.STRUCT, 1) + self.lowValue.write(oprot) + oprot.writeFieldEnd() + if self.highValue is not None: + oprot.writeFieldBegin("highValue", TType.STRUCT, 2) + self.highValue.write(oprot) + oprot.writeFieldEnd() + if self.numNulls is not None: + oprot.writeFieldBegin("numNulls", TType.I64, 3) + oprot.writeI64(self.numNulls) + oprot.writeFieldEnd() + if self.numDVs is not None: + oprot.writeFieldBegin("numDVs", TType.I64, 4) + oprot.writeI64(self.numDVs) + oprot.writeFieldEnd() + if self.bitVectors is not None: + oprot.writeFieldBegin("bitVectors", TType.STRING, 5) + oprot.writeBinary(self.bitVectors) + oprot.writeFieldEnd() + oprot.writeFieldStop() + oprot.writeStructEnd() + + def validate(self): + if self.numNulls is None: + raise TProtocolException(message="Required field numNulls is unset!") + if self.numDVs is None: + raise TProtocolException(message="Required field numDVs is unset!") + return + + def __repr__(self): + L = ["{}={!r}".format(key, value) for key, value in self.__dict__.items()] + return "{}({})".format(self.__class__.__name__, ", ".join(L)) + + def __eq__(self, other): + return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ + + def __ne__(self, other): + return not (self == other) + + +class ColumnStatisticsData: + """ + Attributes: + - booleanStats + - longStats + - doubleStats + - stringStats + - binaryStats + - decimalStats + - dateStats + - timestampStats + + """ + + def __init__( + self, + booleanStats=None, + longStats=None, + doubleStats=None, + stringStats=None, + binaryStats=None, + decimalStats=None, + dateStats=None, + timestampStats=None, + ): + self.booleanStats = booleanStats + self.longStats = longStats + self.doubleStats = doubleStats + self.stringStats = stringStats + self.binaryStats = binaryStats + self.decimalStats = decimalStats + self.dateStats = dateStats + self.timestampStats = timestampStats + + def read(self, iprot): + if ( + iprot._fast_decode is not None + and isinstance(iprot.trans, TTransport.CReadableTransport) + and self.thrift_spec is not None + ): + iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec]) + return + iprot.readStructBegin() + while True: + (fname, ftype, fid) = iprot.readFieldBegin() + if ftype == TType.STOP: + break + if fid == 1: + if ftype == TType.STRUCT: + self.booleanStats = BooleanColumnStatsData() + self.booleanStats.read(iprot) + else: + iprot.skip(ftype) + elif fid == 2: + if ftype == TType.STRUCT: + self.longStats = LongColumnStatsData() + self.longStats.read(iprot) + else: + iprot.skip(ftype) + elif fid == 3: + if ftype == TType.STRUCT: + self.doubleStats = DoubleColumnStatsData() + self.doubleStats.read(iprot) + else: + iprot.skip(ftype) + elif fid == 4: + if ftype == TType.STRUCT: + self.stringStats = StringColumnStatsData() + self.stringStats.read(iprot) + else: + iprot.skip(ftype) + elif fid == 5: + if ftype == TType.STRUCT: + self.binaryStats = BinaryColumnStatsData() + self.binaryStats.read(iprot) + else: + iprot.skip(ftype) + elif fid == 6: + if ftype == TType.STRUCT: + self.decimalStats = DecimalColumnStatsData() + self.decimalStats.read(iprot) + else: + iprot.skip(ftype) + elif fid == 7: + if ftype == TType.STRUCT: + self.dateStats = DateColumnStatsData() + self.dateStats.read(iprot) + else: + iprot.skip(ftype) + elif fid == 8: + if ftype == TType.STRUCT: + self.timestampStats = TimestampColumnStatsData() + self.timestampStats.read(iprot) + else: + iprot.skip(ftype) + else: + iprot.skip(ftype) + iprot.readFieldEnd() + iprot.readStructEnd() + + def write(self, oprot): + if oprot._fast_encode is not None and self.thrift_spec is not None: + oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec])) + return + oprot.writeStructBegin("ColumnStatisticsData") + if self.booleanStats is not None: + oprot.writeFieldBegin("booleanStats", TType.STRUCT, 1) + self.booleanStats.write(oprot) + oprot.writeFieldEnd() + if self.longStats is not None: + oprot.writeFieldBegin("longStats", TType.STRUCT, 2) + self.longStats.write(oprot) + oprot.writeFieldEnd() + if self.doubleStats is not None: + oprot.writeFieldBegin("doubleStats", TType.STRUCT, 3) + self.doubleStats.write(oprot) + oprot.writeFieldEnd() + if self.stringStats is not None: + oprot.writeFieldBegin("stringStats", TType.STRUCT, 4) + self.stringStats.write(oprot) + oprot.writeFieldEnd() + if self.binaryStats is not None: + oprot.writeFieldBegin("binaryStats", TType.STRUCT, 5) + self.binaryStats.write(oprot) + oprot.writeFieldEnd() + if self.decimalStats is not None: + oprot.writeFieldBegin("decimalStats", TType.STRUCT, 6) + self.decimalStats.write(oprot) + oprot.writeFieldEnd() + if self.dateStats is not None: + oprot.writeFieldBegin("dateStats", TType.STRUCT, 7) + self.dateStats.write(oprot) + oprot.writeFieldEnd() + if self.timestampStats is not None: + oprot.writeFieldBegin("timestampStats", TType.STRUCT, 8) + self.timestampStats.write(oprot) + oprot.writeFieldEnd() + oprot.writeFieldStop() + oprot.writeStructEnd() + + def validate(self): + return + + def __repr__(self): + L = ["{}={!r}".format(key, value) for key, value in self.__dict__.items()] + return "{}({})".format(self.__class__.__name__, ", ".join(L)) + + def __eq__(self, other): + return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ + + def __ne__(self, other): + return not (self == other) + + +class ColumnStatisticsObj: + """ + Attributes: + - colName + - colType + - statsData + + """ + + def __init__( + self, + colName=None, + colType=None, + statsData=None, + ): + self.colName = colName + self.colType = colType + self.statsData = statsData + + def read(self, iprot): + if ( + iprot._fast_decode is not None + and isinstance(iprot.trans, TTransport.CReadableTransport) + and self.thrift_spec is not None + ): + iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec]) + return + iprot.readStructBegin() + while True: + (fname, ftype, fid) = iprot.readFieldBegin() + if ftype == TType.STOP: + break + if fid == 1: + if ftype == TType.STRING: + self.colName = ( + iprot.readString().decode("utf-8", errors="replace") if sys.version_info[0] == 2 else iprot.readString() + ) + else: + iprot.skip(ftype) + elif fid == 2: + if ftype == TType.STRING: + self.colType = ( + iprot.readString().decode("utf-8", errors="replace") if sys.version_info[0] == 2 else iprot.readString() + ) + else: + iprot.skip(ftype) + elif fid == 3: + if ftype == TType.STRUCT: + self.statsData = ColumnStatisticsData() + self.statsData.read(iprot) + else: + iprot.skip(ftype) + else: + iprot.skip(ftype) + iprot.readFieldEnd() + iprot.readStructEnd() + + def write(self, oprot): + if oprot._fast_encode is not None and self.thrift_spec is not None: + oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec])) + return + oprot.writeStructBegin("ColumnStatisticsObj") + if self.colName is not None: + oprot.writeFieldBegin("colName", TType.STRING, 1) + oprot.writeString(self.colName.encode("utf-8") if sys.version_info[0] == 2 else self.colName) + oprot.writeFieldEnd() + if self.colType is not None: + oprot.writeFieldBegin("colType", TType.STRING, 2) + oprot.writeString(self.colType.encode("utf-8") if sys.version_info[0] == 2 else self.colType) + oprot.writeFieldEnd() + if self.statsData is not None: + oprot.writeFieldBegin("statsData", TType.STRUCT, 3) + self.statsData.write(oprot) + oprot.writeFieldEnd() + oprot.writeFieldStop() + oprot.writeStructEnd() + + def validate(self): + if self.colName is None: + raise TProtocolException(message="Required field colName is unset!") + if self.colType is None: + raise TProtocolException(message="Required field colType is unset!") + if self.statsData is None: + raise TProtocolException(message="Required field statsData is unset!") + return + + def __repr__(self): + L = ["{}={!r}".format(key, value) for key, value in self.__dict__.items()] + return "{}({})".format(self.__class__.__name__, ", ".join(L)) + + def __eq__(self, other): + return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ + + def __ne__(self, other): + return not (self == other) + + +class ColumnStatisticsDesc: + """ + Attributes: + - isTblLevel + - dbName + - tableName + - partName + - lastAnalyzed + - catName + + """ + + def __init__( + self, + isTblLevel=None, + dbName=None, + tableName=None, + partName=None, + lastAnalyzed=None, + catName=None, + ): + self.isTblLevel = isTblLevel + self.dbName = dbName + self.tableName = tableName + self.partName = partName + self.lastAnalyzed = lastAnalyzed + self.catName = catName + + def read(self, iprot): + if ( + iprot._fast_decode is not None + and isinstance(iprot.trans, TTransport.CReadableTransport) + and self.thrift_spec is not None + ): + iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec]) + return + iprot.readStructBegin() + while True: + (fname, ftype, fid) = iprot.readFieldBegin() + if ftype == TType.STOP: + break + if fid == 1: + if ftype == TType.BOOL: + self.isTblLevel = iprot.readBool() + else: + iprot.skip(ftype) + elif fid == 2: + if ftype == TType.STRING: + self.dbName = ( + iprot.readString().decode("utf-8", errors="replace") if sys.version_info[0] == 2 else iprot.readString() + ) + else: + iprot.skip(ftype) + elif fid == 3: + if ftype == TType.STRING: + self.tableName = ( + iprot.readString().decode("utf-8", errors="replace") if sys.version_info[0] == 2 else iprot.readString() + ) + else: + iprot.skip(ftype) + elif fid == 4: + if ftype == TType.STRING: + self.partName = ( + iprot.readString().decode("utf-8", errors="replace") if sys.version_info[0] == 2 else iprot.readString() + ) + else: + iprot.skip(ftype) + elif fid == 5: + if ftype == TType.I64: + self.lastAnalyzed = iprot.readI64() + else: + iprot.skip(ftype) + elif fid == 6: + if ftype == TType.STRING: + self.catName = ( + iprot.readString().decode("utf-8", errors="replace") if sys.version_info[0] == 2 else iprot.readString() + ) + else: + iprot.skip(ftype) + else: + iprot.skip(ftype) + iprot.readFieldEnd() + iprot.readStructEnd() + + def write(self, oprot): + if oprot._fast_encode is not None and self.thrift_spec is not None: + oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec])) + return + oprot.writeStructBegin("ColumnStatisticsDesc") + if self.isTblLevel is not None: + oprot.writeFieldBegin("isTblLevel", TType.BOOL, 1) + oprot.writeBool(self.isTblLevel) + oprot.writeFieldEnd() + if self.dbName is not None: + oprot.writeFieldBegin("dbName", TType.STRING, 2) + oprot.writeString(self.dbName.encode("utf-8") if sys.version_info[0] == 2 else self.dbName) + oprot.writeFieldEnd() + if self.tableName is not None: + oprot.writeFieldBegin("tableName", TType.STRING, 3) + oprot.writeString(self.tableName.encode("utf-8") if sys.version_info[0] == 2 else self.tableName) + oprot.writeFieldEnd() + if self.partName is not None: + oprot.writeFieldBegin("partName", TType.STRING, 4) + oprot.writeString(self.partName.encode("utf-8") if sys.version_info[0] == 2 else self.partName) + oprot.writeFieldEnd() + if self.lastAnalyzed is not None: + oprot.writeFieldBegin("lastAnalyzed", TType.I64, 5) + oprot.writeI64(self.lastAnalyzed) + oprot.writeFieldEnd() + if self.catName is not None: + oprot.writeFieldBegin("catName", TType.STRING, 6) + oprot.writeString(self.catName.encode("utf-8") if sys.version_info[0] == 2 else self.catName) + oprot.writeFieldEnd() + oprot.writeFieldStop() + oprot.writeStructEnd() + + def validate(self): + if self.isTblLevel is None: + raise TProtocolException(message="Required field isTblLevel is unset!") + if self.dbName is None: + raise TProtocolException(message="Required field dbName is unset!") + if self.tableName is None: + raise TProtocolException(message="Required field tableName is unset!") + return + + def __repr__(self): + L = ["{}={!r}".format(key, value) for key, value in self.__dict__.items()] + return "{}({})".format(self.__class__.__name__, ", ".join(L)) + + def __eq__(self, other): + return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ + + def __ne__(self, other): + return not (self == other) + + +class ColumnStatistics: + """ + Attributes: + - statsDesc + - statsObj + - isStatsCompliant + - engine + + """ + + def __init__( + self, + statsDesc=None, + statsObj=None, + isStatsCompliant=None, + engine=None, + ): + self.statsDesc = statsDesc + self.statsObj = statsObj + self.isStatsCompliant = isStatsCompliant + self.engine = engine + + def read(self, iprot): + if ( + iprot._fast_decode is not None + and isinstance(iprot.trans, TTransport.CReadableTransport) + and self.thrift_spec is not None + ): + iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec]) + return + iprot.readStructBegin() + while True: + (fname, ftype, fid) = iprot.readFieldBegin() + if ftype == TType.STOP: + break + if fid == 1: + if ftype == TType.STRUCT: + self.statsDesc = ColumnStatisticsDesc() + self.statsDesc.read(iprot) + else: + iprot.skip(ftype) + elif fid == 2: + if ftype == TType.LIST: + self.statsObj = [] + (_etype250, _size247) = iprot.readListBegin() + for _i251 in range(_size247): + _elem252 = ColumnStatisticsObj() + _elem252.read(iprot) + self.statsObj.append(_elem252) + iprot.readListEnd() + else: + iprot.skip(ftype) + elif fid == 3: + if ftype == TType.BOOL: + self.isStatsCompliant = iprot.readBool() + else: + iprot.skip(ftype) + elif fid == 4: + if ftype == TType.STRING: + self.engine = ( + iprot.readString().decode("utf-8", errors="replace") if sys.version_info[0] == 2 else iprot.readString() + ) + else: + iprot.skip(ftype) + else: + iprot.skip(ftype) + iprot.readFieldEnd() + iprot.readStructEnd() + + def write(self, oprot): + if oprot._fast_encode is not None and self.thrift_spec is not None: + oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec])) + return + oprot.writeStructBegin("ColumnStatistics") + if self.statsDesc is not None: + oprot.writeFieldBegin("statsDesc", TType.STRUCT, 1) + self.statsDesc.write(oprot) + oprot.writeFieldEnd() + if self.statsObj is not None: + oprot.writeFieldBegin("statsObj", TType.LIST, 2) + oprot.writeListBegin(TType.STRUCT, len(self.statsObj)) + for iter253 in self.statsObj: + iter253.write(oprot) + oprot.writeListEnd() + oprot.writeFieldEnd() + if self.isStatsCompliant is not None: + oprot.writeFieldBegin("isStatsCompliant", TType.BOOL, 3) + oprot.writeBool(self.isStatsCompliant) + oprot.writeFieldEnd() + if self.engine is not None: + oprot.writeFieldBegin("engine", TType.STRING, 4) + oprot.writeString(self.engine.encode("utf-8") if sys.version_info[0] == 2 else self.engine) + oprot.writeFieldEnd() + oprot.writeFieldStop() + oprot.writeStructEnd() + + def validate(self): + if self.statsDesc is None: + raise TProtocolException(message="Required field statsDesc is unset!") + if self.statsObj is None: + raise TProtocolException(message="Required field statsObj is unset!") + return + + def __repr__(self): + L = ["{}={!r}".format(key, value) for key, value in self.__dict__.items()] + return "{}({})".format(self.__class__.__name__, ", ".join(L)) + + def __eq__(self, other): + return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ + + def __ne__(self, other): + return not (self == other) + + +class FileMetadata: + """ + Attributes: + - type + - version + - data + + """ + + def __init__( + self, + type=1, + version=1, + data=None, + ): + self.type = type + self.version = version + self.data = data + + def read(self, iprot): + if ( + iprot._fast_decode is not None + and isinstance(iprot.trans, TTransport.CReadableTransport) + and self.thrift_spec is not None + ): + iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec]) + return + iprot.readStructBegin() + while True: + (fname, ftype, fid) = iprot.readFieldBegin() + if ftype == TType.STOP: + break + if fid == 1: + if ftype == TType.BYTE: + self.type = iprot.readByte() + else: + iprot.skip(ftype) + elif fid == 2: + if ftype == TType.BYTE: + self.version = iprot.readByte() + else: + iprot.skip(ftype) + elif fid == 3: + if ftype == TType.LIST: + self.data = [] + (_etype257, _size254) = iprot.readListBegin() + for _i258 in range(_size254): + _elem259 = iprot.readBinary() + self.data.append(_elem259) + iprot.readListEnd() + else: + iprot.skip(ftype) + else: + iprot.skip(ftype) + iprot.readFieldEnd() + iprot.readStructEnd() + + def write(self, oprot): + if oprot._fast_encode is not None and self.thrift_spec is not None: + oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec])) + return + oprot.writeStructBegin("FileMetadata") + if self.type is not None: + oprot.writeFieldBegin("type", TType.BYTE, 1) + oprot.writeByte(self.type) + oprot.writeFieldEnd() + if self.version is not None: + oprot.writeFieldBegin("version", TType.BYTE, 2) + oprot.writeByte(self.version) + oprot.writeFieldEnd() + if self.data is not None: + oprot.writeFieldBegin("data", TType.LIST, 3) + oprot.writeListBegin(TType.STRING, len(self.data)) + for iter260 in self.data: + oprot.writeBinary(iter260) + oprot.writeListEnd() + oprot.writeFieldEnd() + oprot.writeFieldStop() + oprot.writeStructEnd() + + def validate(self): + return + + def __repr__(self): + L = ["{}={!r}".format(key, value) for key, value in self.__dict__.items()] + return "{}({})".format(self.__class__.__name__, ", ".join(L)) + + def __eq__(self, other): + return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ + + def __ne__(self, other): + return not (self == other) + + +class ObjectDictionary: + """ + Attributes: + - values + + """ + + def __init__( + self, + values=None, + ): + self.values = values + + def read(self, iprot): + if ( + iprot._fast_decode is not None + and isinstance(iprot.trans, TTransport.CReadableTransport) + and self.thrift_spec is not None + ): + iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec]) + return + iprot.readStructBegin() + while True: + (fname, ftype, fid) = iprot.readFieldBegin() + if ftype == TType.STOP: + break + if fid == 1: + if ftype == TType.MAP: + self.values = {} + (_ktype262, _vtype263, _size261) = iprot.readMapBegin() + for _i265 in range(_size261): + _key266 = ( + iprot.readString().decode("utf-8", errors="replace") + if sys.version_info[0] == 2 + else iprot.readString() + ) + _val267 = [] + (_etype271, _size268) = iprot.readListBegin() + for _i272 in range(_size268): + _elem273 = iprot.readBinary() + _val267.append(_elem273) + iprot.readListEnd() + self.values[_key266] = _val267 + iprot.readMapEnd() + else: + iprot.skip(ftype) + else: + iprot.skip(ftype) + iprot.readFieldEnd() + iprot.readStructEnd() + + def write(self, oprot): + if oprot._fast_encode is not None and self.thrift_spec is not None: + oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec])) + return + oprot.writeStructBegin("ObjectDictionary") + if self.values is not None: + oprot.writeFieldBegin("values", TType.MAP, 1) + oprot.writeMapBegin(TType.STRING, TType.LIST, len(self.values)) + for kiter274, viter275 in self.values.items(): + oprot.writeString(kiter274.encode("utf-8") if sys.version_info[0] == 2 else kiter274) + oprot.writeListBegin(TType.STRING, len(viter275)) + for iter276 in viter275: + oprot.writeBinary(iter276) + oprot.writeListEnd() + oprot.writeMapEnd() + oprot.writeFieldEnd() + oprot.writeFieldStop() + oprot.writeStructEnd() + + def validate(self): + if self.values is None: + raise TProtocolException(message="Required field values is unset!") + return + + def __repr__(self): + L = ["{}={!r}".format(key, value) for key, value in self.__dict__.items()] + return "{}({})".format(self.__class__.__name__, ", ".join(L)) + + def __eq__(self, other): + return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ + + def __ne__(self, other): + return not (self == other) + + +class Table: + """ + Attributes: + - tableName + - dbName + - owner + - createTime + - lastAccessTime + - retention + - sd + - partitionKeys + - parameters + - viewOriginalText + - viewExpandedText + - tableType + - privileges + - temporary + - rewriteEnabled + - creationMetadata + - catName + - ownerType + - writeId + - isStatsCompliant + - colStats + - accessType + - requiredReadCapabilities + - requiredWriteCapabilities + - id + - fileMetadata + - dictionary + - txnId + + """ + + def __init__( + self, + tableName=None, + dbName=None, + owner=None, + createTime=None, + lastAccessTime=None, + retention=None, + sd=None, + partitionKeys=None, + parameters=None, + viewOriginalText=None, + viewExpandedText=None, + tableType=None, + privileges=None, + temporary=False, + rewriteEnabled=None, + creationMetadata=None, + catName=None, + ownerType=1, + writeId=-1, + isStatsCompliant=None, + colStats=None, + accessType=None, + requiredReadCapabilities=None, + requiredWriteCapabilities=None, + id=None, + fileMetadata=None, + dictionary=None, + txnId=None, + ): + self.tableName = tableName + self.dbName = dbName + self.owner = owner + self.createTime = createTime + self.lastAccessTime = lastAccessTime + self.retention = retention + self.sd = sd + self.partitionKeys = partitionKeys + self.parameters = parameters + self.viewOriginalText = viewOriginalText + self.viewExpandedText = viewExpandedText + self.tableType = tableType + self.privileges = privileges + self.temporary = temporary + self.rewriteEnabled = rewriteEnabled + self.creationMetadata = creationMetadata + self.catName = catName + self.ownerType = ownerType + self.writeId = writeId + self.isStatsCompliant = isStatsCompliant + self.colStats = colStats + self.accessType = accessType + self.requiredReadCapabilities = requiredReadCapabilities + self.requiredWriteCapabilities = requiredWriteCapabilities + self.id = id + self.fileMetadata = fileMetadata + self.dictionary = dictionary + self.txnId = txnId + + def read(self, iprot): + if ( + iprot._fast_decode is not None + and isinstance(iprot.trans, TTransport.CReadableTransport) + and self.thrift_spec is not None + ): + iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec]) + return + iprot.readStructBegin() + while True: + (fname, ftype, fid) = iprot.readFieldBegin() + if ftype == TType.STOP: + break + if fid == 1: + if ftype == TType.STRING: + self.tableName = ( + iprot.readString().decode("utf-8", errors="replace") if sys.version_info[0] == 2 else iprot.readString() + ) + else: + iprot.skip(ftype) + elif fid == 2: + if ftype == TType.STRING: + self.dbName = ( + iprot.readString().decode("utf-8", errors="replace") if sys.version_info[0] == 2 else iprot.readString() + ) + else: + iprot.skip(ftype) + elif fid == 3: + if ftype == TType.STRING: + self.owner = ( + iprot.readString().decode("utf-8", errors="replace") if sys.version_info[0] == 2 else iprot.readString() + ) + else: + iprot.skip(ftype) + elif fid == 4: + if ftype == TType.I32: + self.createTime = iprot.readI32() + else: + iprot.skip(ftype) + elif fid == 5: + if ftype == TType.I32: + self.lastAccessTime = iprot.readI32() + else: + iprot.skip(ftype) + elif fid == 6: + if ftype == TType.I32: + self.retention = iprot.readI32() + else: + iprot.skip(ftype) + elif fid == 7: + if ftype == TType.STRUCT: + self.sd = StorageDescriptor() + self.sd.read(iprot) + else: + iprot.skip(ftype) + elif fid == 8: + if ftype == TType.LIST: + self.partitionKeys = [] + (_etype280, _size277) = iprot.readListBegin() + for _i281 in range(_size277): + _elem282 = FieldSchema() + _elem282.read(iprot) + self.partitionKeys.append(_elem282) + iprot.readListEnd() + else: + iprot.skip(ftype) + elif fid == 9: + if ftype == TType.MAP: + self.parameters = {} + (_ktype284, _vtype285, _size283) = iprot.readMapBegin() + for _i287 in range(_size283): + _key288 = ( + iprot.readString().decode("utf-8", errors="replace") + if sys.version_info[0] == 2 + else iprot.readString() + ) + _val289 = ( + iprot.readString().decode("utf-8", errors="replace") + if sys.version_info[0] == 2 + else iprot.readString() + ) + self.parameters[_key288] = _val289 + iprot.readMapEnd() + else: + iprot.skip(ftype) + elif fid == 10: + if ftype == TType.STRING: + self.viewOriginalText = ( + iprot.readString().decode("utf-8", errors="replace") if sys.version_info[0] == 2 else iprot.readString() + ) + else: + iprot.skip(ftype) + elif fid == 11: + if ftype == TType.STRING: + self.viewExpandedText = ( + iprot.readString().decode("utf-8", errors="replace") if sys.version_info[0] == 2 else iprot.readString() + ) + else: + iprot.skip(ftype) + elif fid == 12: + if ftype == TType.STRING: + self.tableType = ( + iprot.readString().decode("utf-8", errors="replace") if sys.version_info[0] == 2 else iprot.readString() + ) + else: + iprot.skip(ftype) + elif fid == 13: + if ftype == TType.STRUCT: + self.privileges = PrincipalPrivilegeSet() + self.privileges.read(iprot) + else: + iprot.skip(ftype) + elif fid == 14: + if ftype == TType.BOOL: + self.temporary = iprot.readBool() + else: + iprot.skip(ftype) + elif fid == 15: + if ftype == TType.BOOL: + self.rewriteEnabled = iprot.readBool() + else: + iprot.skip(ftype) + elif fid == 16: + if ftype == TType.STRUCT: + self.creationMetadata = CreationMetadata() + self.creationMetadata.read(iprot) + else: + iprot.skip(ftype) + elif fid == 17: + if ftype == TType.STRING: + self.catName = ( + iprot.readString().decode("utf-8", errors="replace") if sys.version_info[0] == 2 else iprot.readString() + ) + else: + iprot.skip(ftype) + elif fid == 18: + if ftype == TType.I32: + self.ownerType = iprot.readI32() + else: + iprot.skip(ftype) + elif fid == 19: + if ftype == TType.I64: + self.writeId = iprot.readI64() + else: + iprot.skip(ftype) + elif fid == 20: + if ftype == TType.BOOL: + self.isStatsCompliant = iprot.readBool() + else: + iprot.skip(ftype) + elif fid == 21: + if ftype == TType.STRUCT: + self.colStats = ColumnStatistics() + self.colStats.read(iprot) + else: + iprot.skip(ftype) + elif fid == 22: + if ftype == TType.BYTE: + self.accessType = iprot.readByte() + else: + iprot.skip(ftype) + elif fid == 23: + if ftype == TType.LIST: + self.requiredReadCapabilities = [] + (_etype293, _size290) = iprot.readListBegin() + for _i294 in range(_size290): + _elem295 = ( + iprot.readString().decode("utf-8", errors="replace") + if sys.version_info[0] == 2 + else iprot.readString() + ) + self.requiredReadCapabilities.append(_elem295) + iprot.readListEnd() + else: + iprot.skip(ftype) + elif fid == 24: + if ftype == TType.LIST: + self.requiredWriteCapabilities = [] + (_etype299, _size296) = iprot.readListBegin() + for _i300 in range(_size296): + _elem301 = ( + iprot.readString().decode("utf-8", errors="replace") + if sys.version_info[0] == 2 + else iprot.readString() + ) + self.requiredWriteCapabilities.append(_elem301) + iprot.readListEnd() + else: + iprot.skip(ftype) + elif fid == 25: + if ftype == TType.I64: + self.id = iprot.readI64() + else: + iprot.skip(ftype) + elif fid == 26: + if ftype == TType.STRUCT: + self.fileMetadata = FileMetadata() + self.fileMetadata.read(iprot) + else: + iprot.skip(ftype) + elif fid == 27: + if ftype == TType.STRUCT: + self.dictionary = ObjectDictionary() + self.dictionary.read(iprot) + else: + iprot.skip(ftype) + elif fid == 28: + if ftype == TType.I64: + self.txnId = iprot.readI64() + else: + iprot.skip(ftype) + else: + iprot.skip(ftype) + iprot.readFieldEnd() + iprot.readStructEnd() + + def write(self, oprot): + if oprot._fast_encode is not None and self.thrift_spec is not None: + oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec])) + return + oprot.writeStructBegin("Table") + if self.tableName is not None: + oprot.writeFieldBegin("tableName", TType.STRING, 1) + oprot.writeString(self.tableName.encode("utf-8") if sys.version_info[0] == 2 else self.tableName) + oprot.writeFieldEnd() + if self.dbName is not None: + oprot.writeFieldBegin("dbName", TType.STRING, 2) + oprot.writeString(self.dbName.encode("utf-8") if sys.version_info[0] == 2 else self.dbName) + oprot.writeFieldEnd() + if self.owner is not None: + oprot.writeFieldBegin("owner", TType.STRING, 3) + oprot.writeString(self.owner.encode("utf-8") if sys.version_info[0] == 2 else self.owner) + oprot.writeFieldEnd() + if self.createTime is not None: + oprot.writeFieldBegin("createTime", TType.I32, 4) + oprot.writeI32(self.createTime) + oprot.writeFieldEnd() + if self.lastAccessTime is not None: + oprot.writeFieldBegin("lastAccessTime", TType.I32, 5) + oprot.writeI32(self.lastAccessTime) + oprot.writeFieldEnd() + if self.retention is not None: + oprot.writeFieldBegin("retention", TType.I32, 6) + oprot.writeI32(self.retention) + oprot.writeFieldEnd() + if self.sd is not None: + oprot.writeFieldBegin("sd", TType.STRUCT, 7) + self.sd.write(oprot) + oprot.writeFieldEnd() + if self.partitionKeys is not None: + oprot.writeFieldBegin("partitionKeys", TType.LIST, 8) + oprot.writeListBegin(TType.STRUCT, len(self.partitionKeys)) + for iter302 in self.partitionKeys: + iter302.write(oprot) + oprot.writeListEnd() + oprot.writeFieldEnd() + if self.parameters is not None: + oprot.writeFieldBegin("parameters", TType.MAP, 9) + oprot.writeMapBegin(TType.STRING, TType.STRING, len(self.parameters)) + for kiter303, viter304 in self.parameters.items(): + oprot.writeString(kiter303.encode("utf-8") if sys.version_info[0] == 2 else kiter303) + oprot.writeString(viter304.encode("utf-8") if sys.version_info[0] == 2 else viter304) + oprot.writeMapEnd() + oprot.writeFieldEnd() + if self.viewOriginalText is not None: + oprot.writeFieldBegin("viewOriginalText", TType.STRING, 10) + oprot.writeString(self.viewOriginalText.encode("utf-8") if sys.version_info[0] == 2 else self.viewOriginalText) + oprot.writeFieldEnd() + if self.viewExpandedText is not None: + oprot.writeFieldBegin("viewExpandedText", TType.STRING, 11) + oprot.writeString(self.viewExpandedText.encode("utf-8") if sys.version_info[0] == 2 else self.viewExpandedText) + oprot.writeFieldEnd() + if self.tableType is not None: + oprot.writeFieldBegin("tableType", TType.STRING, 12) + oprot.writeString(self.tableType.encode("utf-8") if sys.version_info[0] == 2 else self.tableType) + oprot.writeFieldEnd() + if self.privileges is not None: + oprot.writeFieldBegin("privileges", TType.STRUCT, 13) + self.privileges.write(oprot) + oprot.writeFieldEnd() + if self.temporary is not None: + oprot.writeFieldBegin("temporary", TType.BOOL, 14) + oprot.writeBool(self.temporary) + oprot.writeFieldEnd() + if self.rewriteEnabled is not None: + oprot.writeFieldBegin("rewriteEnabled", TType.BOOL, 15) + oprot.writeBool(self.rewriteEnabled) + oprot.writeFieldEnd() + if self.creationMetadata is not None: + oprot.writeFieldBegin("creationMetadata", TType.STRUCT, 16) + self.creationMetadata.write(oprot) + oprot.writeFieldEnd() + if self.catName is not None: + oprot.writeFieldBegin("catName", TType.STRING, 17) + oprot.writeString(self.catName.encode("utf-8") if sys.version_info[0] == 2 else self.catName) + oprot.writeFieldEnd() + if self.ownerType is not None: + oprot.writeFieldBegin("ownerType", TType.I32, 18) + oprot.writeI32(self.ownerType) + oprot.writeFieldEnd() + if self.writeId is not None: + oprot.writeFieldBegin("writeId", TType.I64, 19) + oprot.writeI64(self.writeId) + oprot.writeFieldEnd() + if self.isStatsCompliant is not None: + oprot.writeFieldBegin("isStatsCompliant", TType.BOOL, 20) + oprot.writeBool(self.isStatsCompliant) + oprot.writeFieldEnd() + if self.colStats is not None: + oprot.writeFieldBegin("colStats", TType.STRUCT, 21) + self.colStats.write(oprot) + oprot.writeFieldEnd() + if self.accessType is not None: + oprot.writeFieldBegin("accessType", TType.BYTE, 22) + oprot.writeByte(self.accessType) + oprot.writeFieldEnd() + if self.requiredReadCapabilities is not None: + oprot.writeFieldBegin("requiredReadCapabilities", TType.LIST, 23) + oprot.writeListBegin(TType.STRING, len(self.requiredReadCapabilities)) + for iter305 in self.requiredReadCapabilities: + oprot.writeString(iter305.encode("utf-8") if sys.version_info[0] == 2 else iter305) + oprot.writeListEnd() + oprot.writeFieldEnd() + if self.requiredWriteCapabilities is not None: + oprot.writeFieldBegin("requiredWriteCapabilities", TType.LIST, 24) + oprot.writeListBegin(TType.STRING, len(self.requiredWriteCapabilities)) + for iter306 in self.requiredWriteCapabilities: + oprot.writeString(iter306.encode("utf-8") if sys.version_info[0] == 2 else iter306) + oprot.writeListEnd() + oprot.writeFieldEnd() + if self.id is not None: + oprot.writeFieldBegin("id", TType.I64, 25) + oprot.writeI64(self.id) + oprot.writeFieldEnd() + if self.fileMetadata is not None: + oprot.writeFieldBegin("fileMetadata", TType.STRUCT, 26) + self.fileMetadata.write(oprot) + oprot.writeFieldEnd() + if self.dictionary is not None: + oprot.writeFieldBegin("dictionary", TType.STRUCT, 27) + self.dictionary.write(oprot) + oprot.writeFieldEnd() + if self.txnId is not None: + oprot.writeFieldBegin("txnId", TType.I64, 28) + oprot.writeI64(self.txnId) + oprot.writeFieldEnd() + oprot.writeFieldStop() + oprot.writeStructEnd() + + def validate(self): + return + + def __repr__(self): + L = ["{}={!r}".format(key, value) for key, value in self.__dict__.items()] + return "{}({})".format(self.__class__.__name__, ", ".join(L)) + + def __eq__(self, other): + return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ + + def __ne__(self, other): + return not (self == other) + + +class SourceTable: + """ + Attributes: + - table + - insertedCount + - updatedCount + - deletedCount + + """ + + def __init__( + self, + table=None, + insertedCount=None, + updatedCount=None, + deletedCount=None, + ): + self.table = table + self.insertedCount = insertedCount + self.updatedCount = updatedCount + self.deletedCount = deletedCount + + def read(self, iprot): + if ( + iprot._fast_decode is not None + and isinstance(iprot.trans, TTransport.CReadableTransport) + and self.thrift_spec is not None + ): + iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec]) + return + iprot.readStructBegin() + while True: + (fname, ftype, fid) = iprot.readFieldBegin() + if ftype == TType.STOP: + break + if fid == 1: + if ftype == TType.STRUCT: + self.table = Table() + self.table.read(iprot) + else: + iprot.skip(ftype) + elif fid == 2: + if ftype == TType.I64: + self.insertedCount = iprot.readI64() + else: + iprot.skip(ftype) + elif fid == 3: + if ftype == TType.I64: + self.updatedCount = iprot.readI64() + else: + iprot.skip(ftype) + elif fid == 4: + if ftype == TType.I64: + self.deletedCount = iprot.readI64() + else: + iprot.skip(ftype) + else: + iprot.skip(ftype) + iprot.readFieldEnd() + iprot.readStructEnd() + + def write(self, oprot): + if oprot._fast_encode is not None and self.thrift_spec is not None: + oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec])) + return + oprot.writeStructBegin("SourceTable") + if self.table is not None: + oprot.writeFieldBegin("table", TType.STRUCT, 1) + self.table.write(oprot) + oprot.writeFieldEnd() + if self.insertedCount is not None: + oprot.writeFieldBegin("insertedCount", TType.I64, 2) + oprot.writeI64(self.insertedCount) + oprot.writeFieldEnd() + if self.updatedCount is not None: + oprot.writeFieldBegin("updatedCount", TType.I64, 3) + oprot.writeI64(self.updatedCount) + oprot.writeFieldEnd() + if self.deletedCount is not None: + oprot.writeFieldBegin("deletedCount", TType.I64, 4) + oprot.writeI64(self.deletedCount) + oprot.writeFieldEnd() + oprot.writeFieldStop() + oprot.writeStructEnd() + + def validate(self): + if self.table is None: + raise TProtocolException(message="Required field table is unset!") + if self.insertedCount is None: + raise TProtocolException(message="Required field insertedCount is unset!") + if self.updatedCount is None: + raise TProtocolException(message="Required field updatedCount is unset!") + if self.deletedCount is None: + raise TProtocolException(message="Required field deletedCount is unset!") + return + + def __repr__(self): + L = ["{}={!r}".format(key, value) for key, value in self.__dict__.items()] + return "{}({})".format(self.__class__.__name__, ", ".join(L)) + + def __eq__(self, other): + return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ + + def __ne__(self, other): + return not (self == other) + + +class Partition: + """ + Attributes: + - values + - dbName + - tableName + - createTime + - lastAccessTime + - sd + - parameters + - privileges + - catName + - writeId + - isStatsCompliant + - colStats + - fileMetadata + + """ + + def __init__( + self, + values=None, + dbName=None, + tableName=None, + createTime=None, + lastAccessTime=None, + sd=None, + parameters=None, + privileges=None, + catName=None, + writeId=-1, + isStatsCompliant=None, + colStats=None, + fileMetadata=None, + ): + self.values = values + self.dbName = dbName + self.tableName = tableName + self.createTime = createTime + self.lastAccessTime = lastAccessTime + self.sd = sd + self.parameters = parameters + self.privileges = privileges + self.catName = catName + self.writeId = writeId + self.isStatsCompliant = isStatsCompliant + self.colStats = colStats + self.fileMetadata = fileMetadata + + def read(self, iprot): + if ( + iprot._fast_decode is not None + and isinstance(iprot.trans, TTransport.CReadableTransport) + and self.thrift_spec is not None + ): + iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec]) + return + iprot.readStructBegin() + while True: + (fname, ftype, fid) = iprot.readFieldBegin() + if ftype == TType.STOP: + break + if fid == 1: + if ftype == TType.LIST: + self.values = [] + (_etype310, _size307) = iprot.readListBegin() + for _i311 in range(_size307): + _elem312 = ( + iprot.readString().decode("utf-8", errors="replace") + if sys.version_info[0] == 2 + else iprot.readString() + ) + self.values.append(_elem312) + iprot.readListEnd() + else: + iprot.skip(ftype) + elif fid == 2: + if ftype == TType.STRING: + self.dbName = ( + iprot.readString().decode("utf-8", errors="replace") if sys.version_info[0] == 2 else iprot.readString() + ) + else: + iprot.skip(ftype) + elif fid == 3: + if ftype == TType.STRING: + self.tableName = ( + iprot.readString().decode("utf-8", errors="replace") if sys.version_info[0] == 2 else iprot.readString() + ) + else: + iprot.skip(ftype) + elif fid == 4: + if ftype == TType.I32: + self.createTime = iprot.readI32() + else: + iprot.skip(ftype) + elif fid == 5: + if ftype == TType.I32: + self.lastAccessTime = iprot.readI32() + else: + iprot.skip(ftype) + elif fid == 6: + if ftype == TType.STRUCT: + self.sd = StorageDescriptor() + self.sd.read(iprot) + else: + iprot.skip(ftype) + elif fid == 7: + if ftype == TType.MAP: + self.parameters = {} + (_ktype314, _vtype315, _size313) = iprot.readMapBegin() + for _i317 in range(_size313): + _key318 = ( + iprot.readString().decode("utf-8", errors="replace") + if sys.version_info[0] == 2 + else iprot.readString() + ) + _val319 = ( + iprot.readString().decode("utf-8", errors="replace") + if sys.version_info[0] == 2 + else iprot.readString() + ) + self.parameters[_key318] = _val319 + iprot.readMapEnd() + else: + iprot.skip(ftype) + elif fid == 8: + if ftype == TType.STRUCT: + self.privileges = PrincipalPrivilegeSet() + self.privileges.read(iprot) + else: + iprot.skip(ftype) + elif fid == 9: + if ftype == TType.STRING: + self.catName = ( + iprot.readString().decode("utf-8", errors="replace") if sys.version_info[0] == 2 else iprot.readString() + ) + else: + iprot.skip(ftype) + elif fid == 10: + if ftype == TType.I64: + self.writeId = iprot.readI64() + else: + iprot.skip(ftype) + elif fid == 11: + if ftype == TType.BOOL: + self.isStatsCompliant = iprot.readBool() + else: + iprot.skip(ftype) + elif fid == 12: + if ftype == TType.STRUCT: + self.colStats = ColumnStatistics() + self.colStats.read(iprot) + else: + iprot.skip(ftype) + elif fid == 13: + if ftype == TType.STRUCT: + self.fileMetadata = FileMetadata() + self.fileMetadata.read(iprot) + else: + iprot.skip(ftype) + else: + iprot.skip(ftype) + iprot.readFieldEnd() + iprot.readStructEnd() + + def write(self, oprot): + if oprot._fast_encode is not None and self.thrift_spec is not None: + oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec])) + return + oprot.writeStructBegin("Partition") + if self.values is not None: + oprot.writeFieldBegin("values", TType.LIST, 1) + oprot.writeListBegin(TType.STRING, len(self.values)) + for iter320 in self.values: + oprot.writeString(iter320.encode("utf-8") if sys.version_info[0] == 2 else iter320) + oprot.writeListEnd() + oprot.writeFieldEnd() + if self.dbName is not None: + oprot.writeFieldBegin("dbName", TType.STRING, 2) + oprot.writeString(self.dbName.encode("utf-8") if sys.version_info[0] == 2 else self.dbName) + oprot.writeFieldEnd() + if self.tableName is not None: + oprot.writeFieldBegin("tableName", TType.STRING, 3) + oprot.writeString(self.tableName.encode("utf-8") if sys.version_info[0] == 2 else self.tableName) + oprot.writeFieldEnd() + if self.createTime is not None: + oprot.writeFieldBegin("createTime", TType.I32, 4) + oprot.writeI32(self.createTime) + oprot.writeFieldEnd() + if self.lastAccessTime is not None: + oprot.writeFieldBegin("lastAccessTime", TType.I32, 5) + oprot.writeI32(self.lastAccessTime) + oprot.writeFieldEnd() + if self.sd is not None: + oprot.writeFieldBegin("sd", TType.STRUCT, 6) + self.sd.write(oprot) + oprot.writeFieldEnd() + if self.parameters is not None: + oprot.writeFieldBegin("parameters", TType.MAP, 7) + oprot.writeMapBegin(TType.STRING, TType.STRING, len(self.parameters)) + for kiter321, viter322 in self.parameters.items(): + oprot.writeString(kiter321.encode("utf-8") if sys.version_info[0] == 2 else kiter321) + oprot.writeString(viter322.encode("utf-8") if sys.version_info[0] == 2 else viter322) + oprot.writeMapEnd() + oprot.writeFieldEnd() + if self.privileges is not None: + oprot.writeFieldBegin("privileges", TType.STRUCT, 8) + self.privileges.write(oprot) + oprot.writeFieldEnd() + if self.catName is not None: + oprot.writeFieldBegin("catName", TType.STRING, 9) + oprot.writeString(self.catName.encode("utf-8") if sys.version_info[0] == 2 else self.catName) + oprot.writeFieldEnd() + if self.writeId is not None: + oprot.writeFieldBegin("writeId", TType.I64, 10) + oprot.writeI64(self.writeId) + oprot.writeFieldEnd() + if self.isStatsCompliant is not None: + oprot.writeFieldBegin("isStatsCompliant", TType.BOOL, 11) + oprot.writeBool(self.isStatsCompliant) + oprot.writeFieldEnd() + if self.colStats is not None: + oprot.writeFieldBegin("colStats", TType.STRUCT, 12) + self.colStats.write(oprot) + oprot.writeFieldEnd() + if self.fileMetadata is not None: + oprot.writeFieldBegin("fileMetadata", TType.STRUCT, 13) + self.fileMetadata.write(oprot) + oprot.writeFieldEnd() + oprot.writeFieldStop() + oprot.writeStructEnd() + + def validate(self): + return + + def __repr__(self): + L = ["{}={!r}".format(key, value) for key, value in self.__dict__.items()] + return "{}({})".format(self.__class__.__name__, ", ".join(L)) + + def __eq__(self, other): + return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ + + def __ne__(self, other): + return not (self == other) + + +class PartitionWithoutSD: + """ + Attributes: + - values + - createTime + - lastAccessTime + - relativePath + - parameters + - privileges + + """ + + def __init__( + self, + values=None, + createTime=None, + lastAccessTime=None, + relativePath=None, + parameters=None, + privileges=None, + ): + self.values = values + self.createTime = createTime + self.lastAccessTime = lastAccessTime + self.relativePath = relativePath + self.parameters = parameters + self.privileges = privileges + + def read(self, iprot): + if ( + iprot._fast_decode is not None + and isinstance(iprot.trans, TTransport.CReadableTransport) + and self.thrift_spec is not None + ): + iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec]) + return + iprot.readStructBegin() + while True: + (fname, ftype, fid) = iprot.readFieldBegin() + if ftype == TType.STOP: + break + if fid == 1: + if ftype == TType.LIST: + self.values = [] + (_etype326, _size323) = iprot.readListBegin() + for _i327 in range(_size323): + _elem328 = ( + iprot.readString().decode("utf-8", errors="replace") + if sys.version_info[0] == 2 + else iprot.readString() + ) + self.values.append(_elem328) + iprot.readListEnd() + else: + iprot.skip(ftype) + elif fid == 2: + if ftype == TType.I32: + self.createTime = iprot.readI32() + else: + iprot.skip(ftype) + elif fid == 3: + if ftype == TType.I32: + self.lastAccessTime = iprot.readI32() + else: + iprot.skip(ftype) + elif fid == 4: + if ftype == TType.STRING: + self.relativePath = ( + iprot.readString().decode("utf-8", errors="replace") if sys.version_info[0] == 2 else iprot.readString() + ) + else: + iprot.skip(ftype) + elif fid == 5: + if ftype == TType.MAP: + self.parameters = {} + (_ktype330, _vtype331, _size329) = iprot.readMapBegin() + for _i333 in range(_size329): + _key334 = ( + iprot.readString().decode("utf-8", errors="replace") + if sys.version_info[0] == 2 + else iprot.readString() + ) + _val335 = ( + iprot.readString().decode("utf-8", errors="replace") + if sys.version_info[0] == 2 + else iprot.readString() + ) + self.parameters[_key334] = _val335 + iprot.readMapEnd() + else: + iprot.skip(ftype) + elif fid == 6: + if ftype == TType.STRUCT: + self.privileges = PrincipalPrivilegeSet() + self.privileges.read(iprot) + else: + iprot.skip(ftype) + else: + iprot.skip(ftype) + iprot.readFieldEnd() + iprot.readStructEnd() + + def write(self, oprot): + if oprot._fast_encode is not None and self.thrift_spec is not None: + oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec])) + return + oprot.writeStructBegin("PartitionWithoutSD") + if self.values is not None: + oprot.writeFieldBegin("values", TType.LIST, 1) + oprot.writeListBegin(TType.STRING, len(self.values)) + for iter336 in self.values: + oprot.writeString(iter336.encode("utf-8") if sys.version_info[0] == 2 else iter336) + oprot.writeListEnd() + oprot.writeFieldEnd() + if self.createTime is not None: + oprot.writeFieldBegin("createTime", TType.I32, 2) + oprot.writeI32(self.createTime) + oprot.writeFieldEnd() + if self.lastAccessTime is not None: + oprot.writeFieldBegin("lastAccessTime", TType.I32, 3) + oprot.writeI32(self.lastAccessTime) + oprot.writeFieldEnd() + if self.relativePath is not None: + oprot.writeFieldBegin("relativePath", TType.STRING, 4) + oprot.writeString(self.relativePath.encode("utf-8") if sys.version_info[0] == 2 else self.relativePath) + oprot.writeFieldEnd() + if self.parameters is not None: + oprot.writeFieldBegin("parameters", TType.MAP, 5) + oprot.writeMapBegin(TType.STRING, TType.STRING, len(self.parameters)) + for kiter337, viter338 in self.parameters.items(): + oprot.writeString(kiter337.encode("utf-8") if sys.version_info[0] == 2 else kiter337) + oprot.writeString(viter338.encode("utf-8") if sys.version_info[0] == 2 else viter338) + oprot.writeMapEnd() + oprot.writeFieldEnd() + if self.privileges is not None: + oprot.writeFieldBegin("privileges", TType.STRUCT, 6) + self.privileges.write(oprot) + oprot.writeFieldEnd() + oprot.writeFieldStop() + oprot.writeStructEnd() + + def validate(self): + return + + def __repr__(self): + L = ["{}={!r}".format(key, value) for key, value in self.__dict__.items()] + return "{}({})".format(self.__class__.__name__, ", ".join(L)) + + def __eq__(self, other): + return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ + + def __ne__(self, other): + return not (self == other) + + +class PartitionSpecWithSharedSD: + """ + Attributes: + - partitions + - sd + + """ + + def __init__( + self, + partitions=None, + sd=None, + ): + self.partitions = partitions + self.sd = sd + + def read(self, iprot): + if ( + iprot._fast_decode is not None + and isinstance(iprot.trans, TTransport.CReadableTransport) + and self.thrift_spec is not None + ): + iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec]) + return + iprot.readStructBegin() + while True: + (fname, ftype, fid) = iprot.readFieldBegin() + if ftype == TType.STOP: + break + if fid == 1: + if ftype == TType.LIST: + self.partitions = [] + (_etype342, _size339) = iprot.readListBegin() + for _i343 in range(_size339): + _elem344 = PartitionWithoutSD() + _elem344.read(iprot) + self.partitions.append(_elem344) + iprot.readListEnd() + else: + iprot.skip(ftype) + elif fid == 2: + if ftype == TType.STRUCT: + self.sd = StorageDescriptor() + self.sd.read(iprot) + else: + iprot.skip(ftype) + else: + iprot.skip(ftype) + iprot.readFieldEnd() + iprot.readStructEnd() + + def write(self, oprot): + if oprot._fast_encode is not None and self.thrift_spec is not None: + oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec])) + return + oprot.writeStructBegin("PartitionSpecWithSharedSD") + if self.partitions is not None: + oprot.writeFieldBegin("partitions", TType.LIST, 1) + oprot.writeListBegin(TType.STRUCT, len(self.partitions)) + for iter345 in self.partitions: + iter345.write(oprot) + oprot.writeListEnd() + oprot.writeFieldEnd() + if self.sd is not None: + oprot.writeFieldBegin("sd", TType.STRUCT, 2) + self.sd.write(oprot) + oprot.writeFieldEnd() + oprot.writeFieldStop() + oprot.writeStructEnd() + + def validate(self): + return + + def __repr__(self): + L = ["{}={!r}".format(key, value) for key, value in self.__dict__.items()] + return "{}({})".format(self.__class__.__name__, ", ".join(L)) + + def __eq__(self, other): + return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ + + def __ne__(self, other): + return not (self == other) + + +class PartitionListComposingSpec: + """ + Attributes: + - partitions + + """ + + def __init__( + self, + partitions=None, + ): + self.partitions = partitions + + def read(self, iprot): + if ( + iprot._fast_decode is not None + and isinstance(iprot.trans, TTransport.CReadableTransport) + and self.thrift_spec is not None + ): + iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec]) + return + iprot.readStructBegin() + while True: + (fname, ftype, fid) = iprot.readFieldBegin() + if ftype == TType.STOP: + break + if fid == 1: + if ftype == TType.LIST: + self.partitions = [] + (_etype349, _size346) = iprot.readListBegin() + for _i350 in range(_size346): + _elem351 = Partition() + _elem351.read(iprot) + self.partitions.append(_elem351) + iprot.readListEnd() + else: + iprot.skip(ftype) + else: + iprot.skip(ftype) + iprot.readFieldEnd() + iprot.readStructEnd() + + def write(self, oprot): + if oprot._fast_encode is not None and self.thrift_spec is not None: + oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec])) + return + oprot.writeStructBegin("PartitionListComposingSpec") + if self.partitions is not None: + oprot.writeFieldBegin("partitions", TType.LIST, 1) + oprot.writeListBegin(TType.STRUCT, len(self.partitions)) + for iter352 in self.partitions: + iter352.write(oprot) + oprot.writeListEnd() + oprot.writeFieldEnd() + oprot.writeFieldStop() + oprot.writeStructEnd() + + def validate(self): + return + + def __repr__(self): + L = ["{}={!r}".format(key, value) for key, value in self.__dict__.items()] + return "{}({})".format(self.__class__.__name__, ", ".join(L)) + + def __eq__(self, other): + return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ + + def __ne__(self, other): + return not (self == other) + + +class PartitionSpec: + """ + Attributes: + - dbName + - tableName + - rootPath + - sharedSDPartitionSpec + - partitionList + - catName + - writeId + - isStatsCompliant + + """ + + def __init__( + self, + dbName=None, + tableName=None, + rootPath=None, + sharedSDPartitionSpec=None, + partitionList=None, + catName=None, + writeId=-1, + isStatsCompliant=None, + ): + self.dbName = dbName + self.tableName = tableName + self.rootPath = rootPath + self.sharedSDPartitionSpec = sharedSDPartitionSpec + self.partitionList = partitionList + self.catName = catName + self.writeId = writeId + self.isStatsCompliant = isStatsCompliant + + def read(self, iprot): + if ( + iprot._fast_decode is not None + and isinstance(iprot.trans, TTransport.CReadableTransport) + and self.thrift_spec is not None + ): + iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec]) + return + iprot.readStructBegin() + while True: + (fname, ftype, fid) = iprot.readFieldBegin() + if ftype == TType.STOP: + break + if fid == 1: + if ftype == TType.STRING: + self.dbName = ( + iprot.readString().decode("utf-8", errors="replace") if sys.version_info[0] == 2 else iprot.readString() + ) + else: + iprot.skip(ftype) + elif fid == 2: + if ftype == TType.STRING: + self.tableName = ( + iprot.readString().decode("utf-8", errors="replace") if sys.version_info[0] == 2 else iprot.readString() + ) + else: + iprot.skip(ftype) + elif fid == 3: + if ftype == TType.STRING: + self.rootPath = ( + iprot.readString().decode("utf-8", errors="replace") if sys.version_info[0] == 2 else iprot.readString() + ) + else: + iprot.skip(ftype) + elif fid == 4: + if ftype == TType.STRUCT: + self.sharedSDPartitionSpec = PartitionSpecWithSharedSD() + self.sharedSDPartitionSpec.read(iprot) + else: + iprot.skip(ftype) + elif fid == 5: + if ftype == TType.STRUCT: + self.partitionList = PartitionListComposingSpec() + self.partitionList.read(iprot) + else: + iprot.skip(ftype) + elif fid == 6: + if ftype == TType.STRING: + self.catName = ( + iprot.readString().decode("utf-8", errors="replace") if sys.version_info[0] == 2 else iprot.readString() + ) + else: + iprot.skip(ftype) + elif fid == 7: + if ftype == TType.I64: + self.writeId = iprot.readI64() + else: + iprot.skip(ftype) + elif fid == 8: + if ftype == TType.BOOL: + self.isStatsCompliant = iprot.readBool() + else: + iprot.skip(ftype) + else: + iprot.skip(ftype) + iprot.readFieldEnd() + iprot.readStructEnd() + + def write(self, oprot): + if oprot._fast_encode is not None and self.thrift_spec is not None: + oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec])) + return + oprot.writeStructBegin("PartitionSpec") + if self.dbName is not None: + oprot.writeFieldBegin("dbName", TType.STRING, 1) + oprot.writeString(self.dbName.encode("utf-8") if sys.version_info[0] == 2 else self.dbName) + oprot.writeFieldEnd() + if self.tableName is not None: + oprot.writeFieldBegin("tableName", TType.STRING, 2) + oprot.writeString(self.tableName.encode("utf-8") if sys.version_info[0] == 2 else self.tableName) + oprot.writeFieldEnd() + if self.rootPath is not None: + oprot.writeFieldBegin("rootPath", TType.STRING, 3) + oprot.writeString(self.rootPath.encode("utf-8") if sys.version_info[0] == 2 else self.rootPath) + oprot.writeFieldEnd() + if self.sharedSDPartitionSpec is not None: + oprot.writeFieldBegin("sharedSDPartitionSpec", TType.STRUCT, 4) + self.sharedSDPartitionSpec.write(oprot) + oprot.writeFieldEnd() + if self.partitionList is not None: + oprot.writeFieldBegin("partitionList", TType.STRUCT, 5) + self.partitionList.write(oprot) + oprot.writeFieldEnd() + if self.catName is not None: + oprot.writeFieldBegin("catName", TType.STRING, 6) + oprot.writeString(self.catName.encode("utf-8") if sys.version_info[0] == 2 else self.catName) + oprot.writeFieldEnd() + if self.writeId is not None: + oprot.writeFieldBegin("writeId", TType.I64, 7) + oprot.writeI64(self.writeId) + oprot.writeFieldEnd() + if self.isStatsCompliant is not None: + oprot.writeFieldBegin("isStatsCompliant", TType.BOOL, 8) + oprot.writeBool(self.isStatsCompliant) + oprot.writeFieldEnd() + oprot.writeFieldStop() + oprot.writeStructEnd() + + def validate(self): + return + + def __repr__(self): + L = ["{}={!r}".format(key, value) for key, value in self.__dict__.items()] + return "{}({})".format(self.__class__.__name__, ", ".join(L)) + + def __eq__(self, other): + return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ + + def __ne__(self, other): + return not (self == other) + + +class AggrStats: + """ + Attributes: + - colStats + - partsFound + - isStatsCompliant + + """ + + def __init__( + self, + colStats=None, + partsFound=None, + isStatsCompliant=None, + ): + self.colStats = colStats + self.partsFound = partsFound + self.isStatsCompliant = isStatsCompliant + + def read(self, iprot): + if ( + iprot._fast_decode is not None + and isinstance(iprot.trans, TTransport.CReadableTransport) + and self.thrift_spec is not None + ): + iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec]) + return + iprot.readStructBegin() + while True: + (fname, ftype, fid) = iprot.readFieldBegin() + if ftype == TType.STOP: + break + if fid == 1: + if ftype == TType.LIST: + self.colStats = [] + (_etype356, _size353) = iprot.readListBegin() + for _i357 in range(_size353): + _elem358 = ColumnStatisticsObj() + _elem358.read(iprot) + self.colStats.append(_elem358) + iprot.readListEnd() + else: + iprot.skip(ftype) + elif fid == 2: + if ftype == TType.I64: + self.partsFound = iprot.readI64() + else: + iprot.skip(ftype) + elif fid == 3: + if ftype == TType.BOOL: + self.isStatsCompliant = iprot.readBool() + else: + iprot.skip(ftype) + else: + iprot.skip(ftype) + iprot.readFieldEnd() + iprot.readStructEnd() + + def write(self, oprot): + if oprot._fast_encode is not None and self.thrift_spec is not None: + oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec])) + return + oprot.writeStructBegin("AggrStats") + if self.colStats is not None: + oprot.writeFieldBegin("colStats", TType.LIST, 1) + oprot.writeListBegin(TType.STRUCT, len(self.colStats)) + for iter359 in self.colStats: + iter359.write(oprot) + oprot.writeListEnd() + oprot.writeFieldEnd() + if self.partsFound is not None: + oprot.writeFieldBegin("partsFound", TType.I64, 2) + oprot.writeI64(self.partsFound) + oprot.writeFieldEnd() + if self.isStatsCompliant is not None: + oprot.writeFieldBegin("isStatsCompliant", TType.BOOL, 3) + oprot.writeBool(self.isStatsCompliant) + oprot.writeFieldEnd() + oprot.writeFieldStop() + oprot.writeStructEnd() + + def validate(self): + if self.colStats is None: + raise TProtocolException(message="Required field colStats is unset!") + if self.partsFound is None: + raise TProtocolException(message="Required field partsFound is unset!") + return + + def __repr__(self): + L = ["{}={!r}".format(key, value) for key, value in self.__dict__.items()] + return "{}({})".format(self.__class__.__name__, ", ".join(L)) + + def __eq__(self, other): + return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ + + def __ne__(self, other): + return not (self == other) + + +class SetPartitionsStatsRequest: + """ + Attributes: + - colStats + - needMerge + - writeId + - validWriteIdList + - engine + + """ + + def __init__( + self, + colStats=None, + needMerge=None, + writeId=-1, + validWriteIdList=None, + engine=None, + ): + self.colStats = colStats + self.needMerge = needMerge + self.writeId = writeId + self.validWriteIdList = validWriteIdList + self.engine = engine + + def read(self, iprot): + if ( + iprot._fast_decode is not None + and isinstance(iprot.trans, TTransport.CReadableTransport) + and self.thrift_spec is not None + ): + iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec]) + return + iprot.readStructBegin() + while True: + (fname, ftype, fid) = iprot.readFieldBegin() + if ftype == TType.STOP: + break + if fid == 1: + if ftype == TType.LIST: + self.colStats = [] + (_etype363, _size360) = iprot.readListBegin() + for _i364 in range(_size360): + _elem365 = ColumnStatistics() + _elem365.read(iprot) + self.colStats.append(_elem365) + iprot.readListEnd() + else: + iprot.skip(ftype) + elif fid == 2: + if ftype == TType.BOOL: + self.needMerge = iprot.readBool() + else: + iprot.skip(ftype) + elif fid == 3: + if ftype == TType.I64: + self.writeId = iprot.readI64() + else: + iprot.skip(ftype) + elif fid == 4: + if ftype == TType.STRING: + self.validWriteIdList = ( + iprot.readString().decode("utf-8", errors="replace") if sys.version_info[0] == 2 else iprot.readString() + ) + else: + iprot.skip(ftype) + elif fid == 5: + if ftype == TType.STRING: + self.engine = ( + iprot.readString().decode("utf-8", errors="replace") if sys.version_info[0] == 2 else iprot.readString() + ) + else: + iprot.skip(ftype) + else: + iprot.skip(ftype) + iprot.readFieldEnd() + iprot.readStructEnd() + + def write(self, oprot): + if oprot._fast_encode is not None and self.thrift_spec is not None: + oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec])) + return + oprot.writeStructBegin("SetPartitionsStatsRequest") + if self.colStats is not None: + oprot.writeFieldBegin("colStats", TType.LIST, 1) + oprot.writeListBegin(TType.STRUCT, len(self.colStats)) + for iter366 in self.colStats: + iter366.write(oprot) + oprot.writeListEnd() + oprot.writeFieldEnd() + if self.needMerge is not None: + oprot.writeFieldBegin("needMerge", TType.BOOL, 2) + oprot.writeBool(self.needMerge) + oprot.writeFieldEnd() + if self.writeId is not None: + oprot.writeFieldBegin("writeId", TType.I64, 3) + oprot.writeI64(self.writeId) + oprot.writeFieldEnd() + if self.validWriteIdList is not None: + oprot.writeFieldBegin("validWriteIdList", TType.STRING, 4) + oprot.writeString(self.validWriteIdList.encode("utf-8") if sys.version_info[0] == 2 else self.validWriteIdList) + oprot.writeFieldEnd() + if self.engine is not None: + oprot.writeFieldBegin("engine", TType.STRING, 5) + oprot.writeString(self.engine.encode("utf-8") if sys.version_info[0] == 2 else self.engine) + oprot.writeFieldEnd() + oprot.writeFieldStop() + oprot.writeStructEnd() + + def validate(self): + if self.colStats is None: + raise TProtocolException(message="Required field colStats is unset!") + if self.engine is None: + raise TProtocolException(message="Required field engine is unset!") + return + + def __repr__(self): + L = ["{}={!r}".format(key, value) for key, value in self.__dict__.items()] + return "{}({})".format(self.__class__.__name__, ", ".join(L)) + + def __eq__(self, other): + return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ + + def __ne__(self, other): + return not (self == other) + + +class SetPartitionsStatsResponse: + """ + Attributes: + - result + + """ + + def __init__( + self, + result=None, + ): + self.result = result + + def read(self, iprot): + if ( + iprot._fast_decode is not None + and isinstance(iprot.trans, TTransport.CReadableTransport) + and self.thrift_spec is not None + ): + iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec]) + return + iprot.readStructBegin() + while True: + (fname, ftype, fid) = iprot.readFieldBegin() + if ftype == TType.STOP: + break + if fid == 1: + if ftype == TType.BOOL: + self.result = iprot.readBool() + else: + iprot.skip(ftype) + else: + iprot.skip(ftype) + iprot.readFieldEnd() + iprot.readStructEnd() + + def write(self, oprot): + if oprot._fast_encode is not None and self.thrift_spec is not None: + oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec])) + return + oprot.writeStructBegin("SetPartitionsStatsResponse") + if self.result is not None: + oprot.writeFieldBegin("result", TType.BOOL, 1) + oprot.writeBool(self.result) + oprot.writeFieldEnd() + oprot.writeFieldStop() + oprot.writeStructEnd() + + def validate(self): + if self.result is None: + raise TProtocolException(message="Required field result is unset!") + return + + def __repr__(self): + L = ["{}={!r}".format(key, value) for key, value in self.__dict__.items()] + return "{}({})".format(self.__class__.__name__, ", ".join(L)) + + def __eq__(self, other): + return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ + + def __ne__(self, other): + return not (self == other) + + +class Schema: + """ + Attributes: + - fieldSchemas + - properties + + """ + + def __init__( + self, + fieldSchemas=None, + properties=None, + ): + self.fieldSchemas = fieldSchemas + self.properties = properties + + def read(self, iprot): + if ( + iprot._fast_decode is not None + and isinstance(iprot.trans, TTransport.CReadableTransport) + and self.thrift_spec is not None + ): + iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec]) + return + iprot.readStructBegin() + while True: + (fname, ftype, fid) = iprot.readFieldBegin() + if ftype == TType.STOP: + break + if fid == 1: + if ftype == TType.LIST: + self.fieldSchemas = [] + (_etype370, _size367) = iprot.readListBegin() + for _i371 in range(_size367): + _elem372 = FieldSchema() + _elem372.read(iprot) + self.fieldSchemas.append(_elem372) + iprot.readListEnd() + else: + iprot.skip(ftype) + elif fid == 2: + if ftype == TType.MAP: + self.properties = {} + (_ktype374, _vtype375, _size373) = iprot.readMapBegin() + for _i377 in range(_size373): + _key378 = ( + iprot.readString().decode("utf-8", errors="replace") + if sys.version_info[0] == 2 + else iprot.readString() + ) + _val379 = ( + iprot.readString().decode("utf-8", errors="replace") + if sys.version_info[0] == 2 + else iprot.readString() + ) + self.properties[_key378] = _val379 + iprot.readMapEnd() + else: + iprot.skip(ftype) + else: + iprot.skip(ftype) + iprot.readFieldEnd() + iprot.readStructEnd() + + def write(self, oprot): + if oprot._fast_encode is not None and self.thrift_spec is not None: + oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec])) + return + oprot.writeStructBegin("Schema") + if self.fieldSchemas is not None: + oprot.writeFieldBegin("fieldSchemas", TType.LIST, 1) + oprot.writeListBegin(TType.STRUCT, len(self.fieldSchemas)) + for iter380 in self.fieldSchemas: + iter380.write(oprot) + oprot.writeListEnd() + oprot.writeFieldEnd() + if self.properties is not None: + oprot.writeFieldBegin("properties", TType.MAP, 2) + oprot.writeMapBegin(TType.STRING, TType.STRING, len(self.properties)) + for kiter381, viter382 in self.properties.items(): + oprot.writeString(kiter381.encode("utf-8") if sys.version_info[0] == 2 else kiter381) + oprot.writeString(viter382.encode("utf-8") if sys.version_info[0] == 2 else viter382) + oprot.writeMapEnd() + oprot.writeFieldEnd() + oprot.writeFieldStop() + oprot.writeStructEnd() + + def validate(self): + return + + def __repr__(self): + L = ["{}={!r}".format(key, value) for key, value in self.__dict__.items()] + return "{}({})".format(self.__class__.__name__, ", ".join(L)) + + def __eq__(self, other): + return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ + + def __ne__(self, other): + return not (self == other) + + +class PrimaryKeysRequest: + """ + Attributes: + - db_name + - tbl_name + - catName + - validWriteIdList + - tableId + + """ + + def __init__( + self, + db_name=None, + tbl_name=None, + catName=None, + validWriteIdList=None, + tableId=-1, + ): + self.db_name = db_name + self.tbl_name = tbl_name + self.catName = catName + self.validWriteIdList = validWriteIdList + self.tableId = tableId + + def read(self, iprot): + if ( + iprot._fast_decode is not None + and isinstance(iprot.trans, TTransport.CReadableTransport) + and self.thrift_spec is not None + ): + iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec]) + return + iprot.readStructBegin() + while True: + (fname, ftype, fid) = iprot.readFieldBegin() + if ftype == TType.STOP: + break + if fid == 1: + if ftype == TType.STRING: + self.db_name = ( + iprot.readString().decode("utf-8", errors="replace") if sys.version_info[0] == 2 else iprot.readString() + ) + else: + iprot.skip(ftype) + elif fid == 2: + if ftype == TType.STRING: + self.tbl_name = ( + iprot.readString().decode("utf-8", errors="replace") if sys.version_info[0] == 2 else iprot.readString() + ) + else: + iprot.skip(ftype) + elif fid == 3: + if ftype == TType.STRING: + self.catName = ( + iprot.readString().decode("utf-8", errors="replace") if sys.version_info[0] == 2 else iprot.readString() + ) + else: + iprot.skip(ftype) + elif fid == 4: + if ftype == TType.STRING: + self.validWriteIdList = ( + iprot.readString().decode("utf-8", errors="replace") if sys.version_info[0] == 2 else iprot.readString() + ) + else: + iprot.skip(ftype) + elif fid == 5: + if ftype == TType.I64: + self.tableId = iprot.readI64() + else: + iprot.skip(ftype) + else: + iprot.skip(ftype) + iprot.readFieldEnd() + iprot.readStructEnd() + + def write(self, oprot): + if oprot._fast_encode is not None and self.thrift_spec is not None: + oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec])) + return + oprot.writeStructBegin("PrimaryKeysRequest") + if self.db_name is not None: + oprot.writeFieldBegin("db_name", TType.STRING, 1) + oprot.writeString(self.db_name.encode("utf-8") if sys.version_info[0] == 2 else self.db_name) + oprot.writeFieldEnd() + if self.tbl_name is not None: + oprot.writeFieldBegin("tbl_name", TType.STRING, 2) + oprot.writeString(self.tbl_name.encode("utf-8") if sys.version_info[0] == 2 else self.tbl_name) + oprot.writeFieldEnd() + if self.catName is not None: + oprot.writeFieldBegin("catName", TType.STRING, 3) + oprot.writeString(self.catName.encode("utf-8") if sys.version_info[0] == 2 else self.catName) + oprot.writeFieldEnd() + if self.validWriteIdList is not None: + oprot.writeFieldBegin("validWriteIdList", TType.STRING, 4) + oprot.writeString(self.validWriteIdList.encode("utf-8") if sys.version_info[0] == 2 else self.validWriteIdList) + oprot.writeFieldEnd() + if self.tableId is not None: + oprot.writeFieldBegin("tableId", TType.I64, 5) + oprot.writeI64(self.tableId) + oprot.writeFieldEnd() + oprot.writeFieldStop() + oprot.writeStructEnd() + + def validate(self): + if self.db_name is None: + raise TProtocolException(message="Required field db_name is unset!") + if self.tbl_name is None: + raise TProtocolException(message="Required field tbl_name is unset!") + return + + def __repr__(self): + L = ["{}={!r}".format(key, value) for key, value in self.__dict__.items()] + return "{}({})".format(self.__class__.__name__, ", ".join(L)) + + def __eq__(self, other): + return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ + + def __ne__(self, other): + return not (self == other) + + +class PrimaryKeysResponse: + """ + Attributes: + - primaryKeys + + """ + + def __init__( + self, + primaryKeys=None, + ): + self.primaryKeys = primaryKeys + + def read(self, iprot): + if ( + iprot._fast_decode is not None + and isinstance(iprot.trans, TTransport.CReadableTransport) + and self.thrift_spec is not None + ): + iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec]) + return + iprot.readStructBegin() + while True: + (fname, ftype, fid) = iprot.readFieldBegin() + if ftype == TType.STOP: + break + if fid == 1: + if ftype == TType.LIST: + self.primaryKeys = [] + (_etype386, _size383) = iprot.readListBegin() + for _i387 in range(_size383): + _elem388 = SQLPrimaryKey() + _elem388.read(iprot) + self.primaryKeys.append(_elem388) + iprot.readListEnd() + else: + iprot.skip(ftype) + else: + iprot.skip(ftype) + iprot.readFieldEnd() + iprot.readStructEnd() + + def write(self, oprot): + if oprot._fast_encode is not None and self.thrift_spec is not None: + oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec])) + return + oprot.writeStructBegin("PrimaryKeysResponse") + if self.primaryKeys is not None: + oprot.writeFieldBegin("primaryKeys", TType.LIST, 1) + oprot.writeListBegin(TType.STRUCT, len(self.primaryKeys)) + for iter389 in self.primaryKeys: + iter389.write(oprot) + oprot.writeListEnd() + oprot.writeFieldEnd() + oprot.writeFieldStop() + oprot.writeStructEnd() + + def validate(self): + if self.primaryKeys is None: + raise TProtocolException(message="Required field primaryKeys is unset!") + return + + def __repr__(self): + L = ["{}={!r}".format(key, value) for key, value in self.__dict__.items()] + return "{}({})".format(self.__class__.__name__, ", ".join(L)) + + def __eq__(self, other): + return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ + + def __ne__(self, other): + return not (self == other) + + +class ForeignKeysRequest: + """ + Attributes: + - parent_db_name + - parent_tbl_name + - foreign_db_name + - foreign_tbl_name + - catName + - validWriteIdList + - tableId + + """ + + def __init__( + self, + parent_db_name=None, + parent_tbl_name=None, + foreign_db_name=None, + foreign_tbl_name=None, + catName=None, + validWriteIdList=None, + tableId=-1, + ): + self.parent_db_name = parent_db_name + self.parent_tbl_name = parent_tbl_name + self.foreign_db_name = foreign_db_name + self.foreign_tbl_name = foreign_tbl_name + self.catName = catName + self.validWriteIdList = validWriteIdList + self.tableId = tableId + + def read(self, iprot): + if ( + iprot._fast_decode is not None + and isinstance(iprot.trans, TTransport.CReadableTransport) + and self.thrift_spec is not None + ): + iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec]) + return + iprot.readStructBegin() + while True: + (fname, ftype, fid) = iprot.readFieldBegin() + if ftype == TType.STOP: + break + if fid == 1: + if ftype == TType.STRING: + self.parent_db_name = ( + iprot.readString().decode("utf-8", errors="replace") if sys.version_info[0] == 2 else iprot.readString() + ) + else: + iprot.skip(ftype) + elif fid == 2: + if ftype == TType.STRING: + self.parent_tbl_name = ( + iprot.readString().decode("utf-8", errors="replace") if sys.version_info[0] == 2 else iprot.readString() + ) + else: + iprot.skip(ftype) + elif fid == 3: + if ftype == TType.STRING: + self.foreign_db_name = ( + iprot.readString().decode("utf-8", errors="replace") if sys.version_info[0] == 2 else iprot.readString() + ) + else: + iprot.skip(ftype) + elif fid == 4: + if ftype == TType.STRING: + self.foreign_tbl_name = ( + iprot.readString().decode("utf-8", errors="replace") if sys.version_info[0] == 2 else iprot.readString() + ) + else: + iprot.skip(ftype) + elif fid == 5: + if ftype == TType.STRING: + self.catName = ( + iprot.readString().decode("utf-8", errors="replace") if sys.version_info[0] == 2 else iprot.readString() + ) + else: + iprot.skip(ftype) + elif fid == 6: + if ftype == TType.STRING: + self.validWriteIdList = ( + iprot.readString().decode("utf-8", errors="replace") if sys.version_info[0] == 2 else iprot.readString() + ) + else: + iprot.skip(ftype) + elif fid == 7: + if ftype == TType.I64: + self.tableId = iprot.readI64() + else: + iprot.skip(ftype) + else: + iprot.skip(ftype) + iprot.readFieldEnd() + iprot.readStructEnd() + + def write(self, oprot): + if oprot._fast_encode is not None and self.thrift_spec is not None: + oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec])) + return + oprot.writeStructBegin("ForeignKeysRequest") + if self.parent_db_name is not None: + oprot.writeFieldBegin("parent_db_name", TType.STRING, 1) + oprot.writeString(self.parent_db_name.encode("utf-8") if sys.version_info[0] == 2 else self.parent_db_name) + oprot.writeFieldEnd() + if self.parent_tbl_name is not None: + oprot.writeFieldBegin("parent_tbl_name", TType.STRING, 2) + oprot.writeString(self.parent_tbl_name.encode("utf-8") if sys.version_info[0] == 2 else self.parent_tbl_name) + oprot.writeFieldEnd() + if self.foreign_db_name is not None: + oprot.writeFieldBegin("foreign_db_name", TType.STRING, 3) + oprot.writeString(self.foreign_db_name.encode("utf-8") if sys.version_info[0] == 2 else self.foreign_db_name) + oprot.writeFieldEnd() + if self.foreign_tbl_name is not None: + oprot.writeFieldBegin("foreign_tbl_name", TType.STRING, 4) + oprot.writeString(self.foreign_tbl_name.encode("utf-8") if sys.version_info[0] == 2 else self.foreign_tbl_name) + oprot.writeFieldEnd() + if self.catName is not None: + oprot.writeFieldBegin("catName", TType.STRING, 5) + oprot.writeString(self.catName.encode("utf-8") if sys.version_info[0] == 2 else self.catName) + oprot.writeFieldEnd() + if self.validWriteIdList is not None: + oprot.writeFieldBegin("validWriteIdList", TType.STRING, 6) + oprot.writeString(self.validWriteIdList.encode("utf-8") if sys.version_info[0] == 2 else self.validWriteIdList) + oprot.writeFieldEnd() + if self.tableId is not None: + oprot.writeFieldBegin("tableId", TType.I64, 7) + oprot.writeI64(self.tableId) + oprot.writeFieldEnd() + oprot.writeFieldStop() + oprot.writeStructEnd() + + def validate(self): + return + + def __repr__(self): + L = ["{}={!r}".format(key, value) for key, value in self.__dict__.items()] + return "{}({})".format(self.__class__.__name__, ", ".join(L)) + + def __eq__(self, other): + return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ + + def __ne__(self, other): + return not (self == other) + + +class ForeignKeysResponse: + """ + Attributes: + - foreignKeys + + """ + + def __init__( + self, + foreignKeys=None, + ): + self.foreignKeys = foreignKeys + + def read(self, iprot): + if ( + iprot._fast_decode is not None + and isinstance(iprot.trans, TTransport.CReadableTransport) + and self.thrift_spec is not None + ): + iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec]) + return + iprot.readStructBegin() + while True: + (fname, ftype, fid) = iprot.readFieldBegin() + if ftype == TType.STOP: + break + if fid == 1: + if ftype == TType.LIST: + self.foreignKeys = [] + (_etype393, _size390) = iprot.readListBegin() + for _i394 in range(_size390): + _elem395 = SQLForeignKey() + _elem395.read(iprot) + self.foreignKeys.append(_elem395) + iprot.readListEnd() + else: + iprot.skip(ftype) + else: + iprot.skip(ftype) + iprot.readFieldEnd() + iprot.readStructEnd() + + def write(self, oprot): + if oprot._fast_encode is not None and self.thrift_spec is not None: + oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec])) + return + oprot.writeStructBegin("ForeignKeysResponse") + if self.foreignKeys is not None: + oprot.writeFieldBegin("foreignKeys", TType.LIST, 1) + oprot.writeListBegin(TType.STRUCT, len(self.foreignKeys)) + for iter396 in self.foreignKeys: + iter396.write(oprot) + oprot.writeListEnd() + oprot.writeFieldEnd() + oprot.writeFieldStop() + oprot.writeStructEnd() + + def validate(self): + if self.foreignKeys is None: + raise TProtocolException(message="Required field foreignKeys is unset!") + return + + def __repr__(self): + L = ["{}={!r}".format(key, value) for key, value in self.__dict__.items()] + return "{}({})".format(self.__class__.__name__, ", ".join(L)) + + def __eq__(self, other): + return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ + + def __ne__(self, other): + return not (self == other) + + +class UniqueConstraintsRequest: + """ + Attributes: + - catName + - db_name + - tbl_name + - validWriteIdList + - tableId + + """ + + def __init__( + self, + catName=None, + db_name=None, + tbl_name=None, + validWriteIdList=None, + tableId=-1, + ): + self.catName = catName + self.db_name = db_name + self.tbl_name = tbl_name + self.validWriteIdList = validWriteIdList + self.tableId = tableId + + def read(self, iprot): + if ( + iprot._fast_decode is not None + and isinstance(iprot.trans, TTransport.CReadableTransport) + and self.thrift_spec is not None + ): + iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec]) + return + iprot.readStructBegin() + while True: + (fname, ftype, fid) = iprot.readFieldBegin() + if ftype == TType.STOP: + break + if fid == 1: + if ftype == TType.STRING: + self.catName = ( + iprot.readString().decode("utf-8", errors="replace") if sys.version_info[0] == 2 else iprot.readString() + ) + else: + iprot.skip(ftype) + elif fid == 2: + if ftype == TType.STRING: + self.db_name = ( + iprot.readString().decode("utf-8", errors="replace") if sys.version_info[0] == 2 else iprot.readString() + ) + else: + iprot.skip(ftype) + elif fid == 3: + if ftype == TType.STRING: + self.tbl_name = ( + iprot.readString().decode("utf-8", errors="replace") if sys.version_info[0] == 2 else iprot.readString() + ) + else: + iprot.skip(ftype) + elif fid == 4: + if ftype == TType.STRING: + self.validWriteIdList = ( + iprot.readString().decode("utf-8", errors="replace") if sys.version_info[0] == 2 else iprot.readString() + ) + else: + iprot.skip(ftype) + elif fid == 5: + if ftype == TType.I64: + self.tableId = iprot.readI64() + else: + iprot.skip(ftype) + else: + iprot.skip(ftype) + iprot.readFieldEnd() + iprot.readStructEnd() + + def write(self, oprot): + if oprot._fast_encode is not None and self.thrift_spec is not None: + oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec])) + return + oprot.writeStructBegin("UniqueConstraintsRequest") + if self.catName is not None: + oprot.writeFieldBegin("catName", TType.STRING, 1) + oprot.writeString(self.catName.encode("utf-8") if sys.version_info[0] == 2 else self.catName) + oprot.writeFieldEnd() + if self.db_name is not None: + oprot.writeFieldBegin("db_name", TType.STRING, 2) + oprot.writeString(self.db_name.encode("utf-8") if sys.version_info[0] == 2 else self.db_name) + oprot.writeFieldEnd() + if self.tbl_name is not None: + oprot.writeFieldBegin("tbl_name", TType.STRING, 3) + oprot.writeString(self.tbl_name.encode("utf-8") if sys.version_info[0] == 2 else self.tbl_name) + oprot.writeFieldEnd() + if self.validWriteIdList is not None: + oprot.writeFieldBegin("validWriteIdList", TType.STRING, 4) + oprot.writeString(self.validWriteIdList.encode("utf-8") if sys.version_info[0] == 2 else self.validWriteIdList) + oprot.writeFieldEnd() + if self.tableId is not None: + oprot.writeFieldBegin("tableId", TType.I64, 5) + oprot.writeI64(self.tableId) + oprot.writeFieldEnd() + oprot.writeFieldStop() + oprot.writeStructEnd() + + def validate(self): + if self.catName is None: + raise TProtocolException(message="Required field catName is unset!") + if self.db_name is None: + raise TProtocolException(message="Required field db_name is unset!") + if self.tbl_name is None: + raise TProtocolException(message="Required field tbl_name is unset!") + return + + def __repr__(self): + L = ["{}={!r}".format(key, value) for key, value in self.__dict__.items()] + return "{}({})".format(self.__class__.__name__, ", ".join(L)) + + def __eq__(self, other): + return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ + + def __ne__(self, other): + return not (self == other) + + +class UniqueConstraintsResponse: + """ + Attributes: + - uniqueConstraints + + """ + + def __init__( + self, + uniqueConstraints=None, + ): + self.uniqueConstraints = uniqueConstraints + + def read(self, iprot): + if ( + iprot._fast_decode is not None + and isinstance(iprot.trans, TTransport.CReadableTransport) + and self.thrift_spec is not None + ): + iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec]) + return + iprot.readStructBegin() + while True: + (fname, ftype, fid) = iprot.readFieldBegin() + if ftype == TType.STOP: + break + if fid == 1: + if ftype == TType.LIST: + self.uniqueConstraints = [] + (_etype400, _size397) = iprot.readListBegin() + for _i401 in range(_size397): + _elem402 = SQLUniqueConstraint() + _elem402.read(iprot) + self.uniqueConstraints.append(_elem402) + iprot.readListEnd() + else: + iprot.skip(ftype) + else: + iprot.skip(ftype) + iprot.readFieldEnd() + iprot.readStructEnd() + + def write(self, oprot): + if oprot._fast_encode is not None and self.thrift_spec is not None: + oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec])) + return + oprot.writeStructBegin("UniqueConstraintsResponse") + if self.uniqueConstraints is not None: + oprot.writeFieldBegin("uniqueConstraints", TType.LIST, 1) + oprot.writeListBegin(TType.STRUCT, len(self.uniqueConstraints)) + for iter403 in self.uniqueConstraints: + iter403.write(oprot) + oprot.writeListEnd() + oprot.writeFieldEnd() + oprot.writeFieldStop() + oprot.writeStructEnd() + + def validate(self): + if self.uniqueConstraints is None: + raise TProtocolException(message="Required field uniqueConstraints is unset!") + return + + def __repr__(self): + L = ["{}={!r}".format(key, value) for key, value in self.__dict__.items()] + return "{}({})".format(self.__class__.__name__, ", ".join(L)) + + def __eq__(self, other): + return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ + + def __ne__(self, other): + return not (self == other) + + +class NotNullConstraintsRequest: + """ + Attributes: + - catName + - db_name + - tbl_name + - validWriteIdList + - tableId + + """ + + def __init__( + self, + catName=None, + db_name=None, + tbl_name=None, + validWriteIdList=None, + tableId=-1, + ): + self.catName = catName + self.db_name = db_name + self.tbl_name = tbl_name + self.validWriteIdList = validWriteIdList + self.tableId = tableId + + def read(self, iprot): + if ( + iprot._fast_decode is not None + and isinstance(iprot.trans, TTransport.CReadableTransport) + and self.thrift_spec is not None + ): + iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec]) + return + iprot.readStructBegin() + while True: + (fname, ftype, fid) = iprot.readFieldBegin() + if ftype == TType.STOP: + break + if fid == 1: + if ftype == TType.STRING: + self.catName = ( + iprot.readString().decode("utf-8", errors="replace") if sys.version_info[0] == 2 else iprot.readString() + ) + else: + iprot.skip(ftype) + elif fid == 2: + if ftype == TType.STRING: + self.db_name = ( + iprot.readString().decode("utf-8", errors="replace") if sys.version_info[0] == 2 else iprot.readString() + ) + else: + iprot.skip(ftype) + elif fid == 3: + if ftype == TType.STRING: + self.tbl_name = ( + iprot.readString().decode("utf-8", errors="replace") if sys.version_info[0] == 2 else iprot.readString() + ) + else: + iprot.skip(ftype) + elif fid == 4: + if ftype == TType.STRING: + self.validWriteIdList = ( + iprot.readString().decode("utf-8", errors="replace") if sys.version_info[0] == 2 else iprot.readString() + ) + else: + iprot.skip(ftype) + elif fid == 5: + if ftype == TType.I64: + self.tableId = iprot.readI64() + else: + iprot.skip(ftype) + else: + iprot.skip(ftype) + iprot.readFieldEnd() + iprot.readStructEnd() + + def write(self, oprot): + if oprot._fast_encode is not None and self.thrift_spec is not None: + oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec])) + return + oprot.writeStructBegin("NotNullConstraintsRequest") + if self.catName is not None: + oprot.writeFieldBegin("catName", TType.STRING, 1) + oprot.writeString(self.catName.encode("utf-8") if sys.version_info[0] == 2 else self.catName) + oprot.writeFieldEnd() + if self.db_name is not None: + oprot.writeFieldBegin("db_name", TType.STRING, 2) + oprot.writeString(self.db_name.encode("utf-8") if sys.version_info[0] == 2 else self.db_name) + oprot.writeFieldEnd() + if self.tbl_name is not None: + oprot.writeFieldBegin("tbl_name", TType.STRING, 3) + oprot.writeString(self.tbl_name.encode("utf-8") if sys.version_info[0] == 2 else self.tbl_name) + oprot.writeFieldEnd() + if self.validWriteIdList is not None: + oprot.writeFieldBegin("validWriteIdList", TType.STRING, 4) + oprot.writeString(self.validWriteIdList.encode("utf-8") if sys.version_info[0] == 2 else self.validWriteIdList) + oprot.writeFieldEnd() + if self.tableId is not None: + oprot.writeFieldBegin("tableId", TType.I64, 5) + oprot.writeI64(self.tableId) + oprot.writeFieldEnd() + oprot.writeFieldStop() + oprot.writeStructEnd() + + def validate(self): + if self.catName is None: + raise TProtocolException(message="Required field catName is unset!") + if self.db_name is None: + raise TProtocolException(message="Required field db_name is unset!") + if self.tbl_name is None: + raise TProtocolException(message="Required field tbl_name is unset!") + return + + def __repr__(self): + L = ["{}={!r}".format(key, value) for key, value in self.__dict__.items()] + return "{}({})".format(self.__class__.__name__, ", ".join(L)) + + def __eq__(self, other): + return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ + + def __ne__(self, other): + return not (self == other) + + +class NotNullConstraintsResponse: + """ + Attributes: + - notNullConstraints + + """ + + def __init__( + self, + notNullConstraints=None, + ): + self.notNullConstraints = notNullConstraints + + def read(self, iprot): + if ( + iprot._fast_decode is not None + and isinstance(iprot.trans, TTransport.CReadableTransport) + and self.thrift_spec is not None + ): + iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec]) + return + iprot.readStructBegin() + while True: + (fname, ftype, fid) = iprot.readFieldBegin() + if ftype == TType.STOP: + break + if fid == 1: + if ftype == TType.LIST: + self.notNullConstraints = [] + (_etype407, _size404) = iprot.readListBegin() + for _i408 in range(_size404): + _elem409 = SQLNotNullConstraint() + _elem409.read(iprot) + self.notNullConstraints.append(_elem409) + iprot.readListEnd() + else: + iprot.skip(ftype) + else: + iprot.skip(ftype) + iprot.readFieldEnd() + iprot.readStructEnd() + + def write(self, oprot): + if oprot._fast_encode is not None and self.thrift_spec is not None: + oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec])) + return + oprot.writeStructBegin("NotNullConstraintsResponse") + if self.notNullConstraints is not None: + oprot.writeFieldBegin("notNullConstraints", TType.LIST, 1) + oprot.writeListBegin(TType.STRUCT, len(self.notNullConstraints)) + for iter410 in self.notNullConstraints: + iter410.write(oprot) + oprot.writeListEnd() + oprot.writeFieldEnd() + oprot.writeFieldStop() + oprot.writeStructEnd() + + def validate(self): + if self.notNullConstraints is None: + raise TProtocolException(message="Required field notNullConstraints is unset!") + return + + def __repr__(self): + L = ["{}={!r}".format(key, value) for key, value in self.__dict__.items()] + return "{}({})".format(self.__class__.__name__, ", ".join(L)) + + def __eq__(self, other): + return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ + + def __ne__(self, other): + return not (self == other) + + +class DefaultConstraintsRequest: + """ + Attributes: + - catName + - db_name + - tbl_name + - validWriteIdList + - tableId + + """ + + def __init__( + self, + catName=None, + db_name=None, + tbl_name=None, + validWriteIdList=None, + tableId=-1, + ): + self.catName = catName + self.db_name = db_name + self.tbl_name = tbl_name + self.validWriteIdList = validWriteIdList + self.tableId = tableId + + def read(self, iprot): + if ( + iprot._fast_decode is not None + and isinstance(iprot.trans, TTransport.CReadableTransport) + and self.thrift_spec is not None + ): + iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec]) + return + iprot.readStructBegin() + while True: + (fname, ftype, fid) = iprot.readFieldBegin() + if ftype == TType.STOP: + break + if fid == 1: + if ftype == TType.STRING: + self.catName = ( + iprot.readString().decode("utf-8", errors="replace") if sys.version_info[0] == 2 else iprot.readString() + ) + else: + iprot.skip(ftype) + elif fid == 2: + if ftype == TType.STRING: + self.db_name = ( + iprot.readString().decode("utf-8", errors="replace") if sys.version_info[0] == 2 else iprot.readString() + ) + else: + iprot.skip(ftype) + elif fid == 3: + if ftype == TType.STRING: + self.tbl_name = ( + iprot.readString().decode("utf-8", errors="replace") if sys.version_info[0] == 2 else iprot.readString() + ) + else: + iprot.skip(ftype) + elif fid == 4: + if ftype == TType.STRING: + self.validWriteIdList = ( + iprot.readString().decode("utf-8", errors="replace") if sys.version_info[0] == 2 else iprot.readString() + ) + else: + iprot.skip(ftype) + elif fid == 5: + if ftype == TType.I64: + self.tableId = iprot.readI64() + else: + iprot.skip(ftype) + else: + iprot.skip(ftype) + iprot.readFieldEnd() + iprot.readStructEnd() + + def write(self, oprot): + if oprot._fast_encode is not None and self.thrift_spec is not None: + oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec])) + return + oprot.writeStructBegin("DefaultConstraintsRequest") + if self.catName is not None: + oprot.writeFieldBegin("catName", TType.STRING, 1) + oprot.writeString(self.catName.encode("utf-8") if sys.version_info[0] == 2 else self.catName) + oprot.writeFieldEnd() + if self.db_name is not None: + oprot.writeFieldBegin("db_name", TType.STRING, 2) + oprot.writeString(self.db_name.encode("utf-8") if sys.version_info[0] == 2 else self.db_name) + oprot.writeFieldEnd() + if self.tbl_name is not None: + oprot.writeFieldBegin("tbl_name", TType.STRING, 3) + oprot.writeString(self.tbl_name.encode("utf-8") if sys.version_info[0] == 2 else self.tbl_name) + oprot.writeFieldEnd() + if self.validWriteIdList is not None: + oprot.writeFieldBegin("validWriteIdList", TType.STRING, 4) + oprot.writeString(self.validWriteIdList.encode("utf-8") if sys.version_info[0] == 2 else self.validWriteIdList) + oprot.writeFieldEnd() + if self.tableId is not None: + oprot.writeFieldBegin("tableId", TType.I64, 5) + oprot.writeI64(self.tableId) + oprot.writeFieldEnd() + oprot.writeFieldStop() + oprot.writeStructEnd() + + def validate(self): + if self.catName is None: + raise TProtocolException(message="Required field catName is unset!") + if self.db_name is None: + raise TProtocolException(message="Required field db_name is unset!") + if self.tbl_name is None: + raise TProtocolException(message="Required field tbl_name is unset!") + return + + def __repr__(self): + L = ["{}={!r}".format(key, value) for key, value in self.__dict__.items()] + return "{}({})".format(self.__class__.__name__, ", ".join(L)) + + def __eq__(self, other): + return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ + + def __ne__(self, other): + return not (self == other) + + +class DefaultConstraintsResponse: + """ + Attributes: + - defaultConstraints + + """ + + def __init__( + self, + defaultConstraints=None, + ): + self.defaultConstraints = defaultConstraints + + def read(self, iprot): + if ( + iprot._fast_decode is not None + and isinstance(iprot.trans, TTransport.CReadableTransport) + and self.thrift_spec is not None + ): + iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec]) + return + iprot.readStructBegin() + while True: + (fname, ftype, fid) = iprot.readFieldBegin() + if ftype == TType.STOP: + break + if fid == 1: + if ftype == TType.LIST: + self.defaultConstraints = [] + (_etype414, _size411) = iprot.readListBegin() + for _i415 in range(_size411): + _elem416 = SQLDefaultConstraint() + _elem416.read(iprot) + self.defaultConstraints.append(_elem416) + iprot.readListEnd() + else: + iprot.skip(ftype) + else: + iprot.skip(ftype) + iprot.readFieldEnd() + iprot.readStructEnd() + + def write(self, oprot): + if oprot._fast_encode is not None and self.thrift_spec is not None: + oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec])) + return + oprot.writeStructBegin("DefaultConstraintsResponse") + if self.defaultConstraints is not None: + oprot.writeFieldBegin("defaultConstraints", TType.LIST, 1) + oprot.writeListBegin(TType.STRUCT, len(self.defaultConstraints)) + for iter417 in self.defaultConstraints: + iter417.write(oprot) + oprot.writeListEnd() + oprot.writeFieldEnd() + oprot.writeFieldStop() + oprot.writeStructEnd() + + def validate(self): + if self.defaultConstraints is None: + raise TProtocolException(message="Required field defaultConstraints is unset!") + return + + def __repr__(self): + L = ["{}={!r}".format(key, value) for key, value in self.__dict__.items()] + return "{}({})".format(self.__class__.__name__, ", ".join(L)) + + def __eq__(self, other): + return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ + + def __ne__(self, other): + return not (self == other) + + +class CheckConstraintsRequest: + """ + Attributes: + - catName + - db_name + - tbl_name + - validWriteIdList + - tableId + + """ + + def __init__( + self, + catName=None, + db_name=None, + tbl_name=None, + validWriteIdList=None, + tableId=-1, + ): + self.catName = catName + self.db_name = db_name + self.tbl_name = tbl_name + self.validWriteIdList = validWriteIdList + self.tableId = tableId + + def read(self, iprot): + if ( + iprot._fast_decode is not None + and isinstance(iprot.trans, TTransport.CReadableTransport) + and self.thrift_spec is not None + ): + iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec]) + return + iprot.readStructBegin() + while True: + (fname, ftype, fid) = iprot.readFieldBegin() + if ftype == TType.STOP: + break + if fid == 1: + if ftype == TType.STRING: + self.catName = ( + iprot.readString().decode("utf-8", errors="replace") if sys.version_info[0] == 2 else iprot.readString() + ) + else: + iprot.skip(ftype) + elif fid == 2: + if ftype == TType.STRING: + self.db_name = ( + iprot.readString().decode("utf-8", errors="replace") if sys.version_info[0] == 2 else iprot.readString() + ) + else: + iprot.skip(ftype) + elif fid == 3: + if ftype == TType.STRING: + self.tbl_name = ( + iprot.readString().decode("utf-8", errors="replace") if sys.version_info[0] == 2 else iprot.readString() + ) + else: + iprot.skip(ftype) + elif fid == 4: + if ftype == TType.STRING: + self.validWriteIdList = ( + iprot.readString().decode("utf-8", errors="replace") if sys.version_info[0] == 2 else iprot.readString() + ) + else: + iprot.skip(ftype) + elif fid == 5: + if ftype == TType.I64: + self.tableId = iprot.readI64() + else: + iprot.skip(ftype) + else: + iprot.skip(ftype) + iprot.readFieldEnd() + iprot.readStructEnd() + + def write(self, oprot): + if oprot._fast_encode is not None and self.thrift_spec is not None: + oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec])) + return + oprot.writeStructBegin("CheckConstraintsRequest") + if self.catName is not None: + oprot.writeFieldBegin("catName", TType.STRING, 1) + oprot.writeString(self.catName.encode("utf-8") if sys.version_info[0] == 2 else self.catName) + oprot.writeFieldEnd() + if self.db_name is not None: + oprot.writeFieldBegin("db_name", TType.STRING, 2) + oprot.writeString(self.db_name.encode("utf-8") if sys.version_info[0] == 2 else self.db_name) + oprot.writeFieldEnd() + if self.tbl_name is not None: + oprot.writeFieldBegin("tbl_name", TType.STRING, 3) + oprot.writeString(self.tbl_name.encode("utf-8") if sys.version_info[0] == 2 else self.tbl_name) + oprot.writeFieldEnd() + if self.validWriteIdList is not None: + oprot.writeFieldBegin("validWriteIdList", TType.STRING, 4) + oprot.writeString(self.validWriteIdList.encode("utf-8") if sys.version_info[0] == 2 else self.validWriteIdList) + oprot.writeFieldEnd() + if self.tableId is not None: + oprot.writeFieldBegin("tableId", TType.I64, 5) + oprot.writeI64(self.tableId) + oprot.writeFieldEnd() + oprot.writeFieldStop() + oprot.writeStructEnd() + + def validate(self): + if self.catName is None: + raise TProtocolException(message="Required field catName is unset!") + if self.db_name is None: + raise TProtocolException(message="Required field db_name is unset!") + if self.tbl_name is None: + raise TProtocolException(message="Required field tbl_name is unset!") + return + + def __repr__(self): + L = ["{}={!r}".format(key, value) for key, value in self.__dict__.items()] + return "{}({})".format(self.__class__.__name__, ", ".join(L)) + + def __eq__(self, other): + return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ + + def __ne__(self, other): + return not (self == other) + + +class CheckConstraintsResponse: + """ + Attributes: + - checkConstraints + + """ + + def __init__( + self, + checkConstraints=None, + ): + self.checkConstraints = checkConstraints + + def read(self, iprot): + if ( + iprot._fast_decode is not None + and isinstance(iprot.trans, TTransport.CReadableTransport) + and self.thrift_spec is not None + ): + iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec]) + return + iprot.readStructBegin() + while True: + (fname, ftype, fid) = iprot.readFieldBegin() + if ftype == TType.STOP: + break + if fid == 1: + if ftype == TType.LIST: + self.checkConstraints = [] + (_etype421, _size418) = iprot.readListBegin() + for _i422 in range(_size418): + _elem423 = SQLCheckConstraint() + _elem423.read(iprot) + self.checkConstraints.append(_elem423) + iprot.readListEnd() + else: + iprot.skip(ftype) + else: + iprot.skip(ftype) + iprot.readFieldEnd() + iprot.readStructEnd() + + def write(self, oprot): + if oprot._fast_encode is not None and self.thrift_spec is not None: + oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec])) + return + oprot.writeStructBegin("CheckConstraintsResponse") + if self.checkConstraints is not None: + oprot.writeFieldBegin("checkConstraints", TType.LIST, 1) + oprot.writeListBegin(TType.STRUCT, len(self.checkConstraints)) + for iter424 in self.checkConstraints: + iter424.write(oprot) + oprot.writeListEnd() + oprot.writeFieldEnd() + oprot.writeFieldStop() + oprot.writeStructEnd() + + def validate(self): + if self.checkConstraints is None: + raise TProtocolException(message="Required field checkConstraints is unset!") + return + + def __repr__(self): + L = ["{}={!r}".format(key, value) for key, value in self.__dict__.items()] + return "{}({})".format(self.__class__.__name__, ", ".join(L)) + + def __eq__(self, other): + return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ + + def __ne__(self, other): + return not (self == other) + + +class AllTableConstraintsRequest: + """ + Attributes: + - dbName + - tblName + - catName + - validWriteIdList + - tableId + + """ + + def __init__( + self, + dbName=None, + tblName=None, + catName=None, + validWriteIdList=None, + tableId=-1, + ): + self.dbName = dbName + self.tblName = tblName + self.catName = catName + self.validWriteIdList = validWriteIdList + self.tableId = tableId + + def read(self, iprot): + if ( + iprot._fast_decode is not None + and isinstance(iprot.trans, TTransport.CReadableTransport) + and self.thrift_spec is not None + ): + iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec]) + return + iprot.readStructBegin() + while True: + (fname, ftype, fid) = iprot.readFieldBegin() + if ftype == TType.STOP: + break + if fid == 1: + if ftype == TType.STRING: + self.dbName = ( + iprot.readString().decode("utf-8", errors="replace") if sys.version_info[0] == 2 else iprot.readString() + ) + else: + iprot.skip(ftype) + elif fid == 2: + if ftype == TType.STRING: + self.tblName = ( + iprot.readString().decode("utf-8", errors="replace") if sys.version_info[0] == 2 else iprot.readString() + ) + else: + iprot.skip(ftype) + elif fid == 3: + if ftype == TType.STRING: + self.catName = ( + iprot.readString().decode("utf-8", errors="replace") if sys.version_info[0] == 2 else iprot.readString() + ) + else: + iprot.skip(ftype) + elif fid == 4: + if ftype == TType.STRING: + self.validWriteIdList = ( + iprot.readString().decode("utf-8", errors="replace") if sys.version_info[0] == 2 else iprot.readString() + ) + else: + iprot.skip(ftype) + elif fid == 5: + if ftype == TType.I64: + self.tableId = iprot.readI64() + else: + iprot.skip(ftype) + else: + iprot.skip(ftype) + iprot.readFieldEnd() + iprot.readStructEnd() + + def write(self, oprot): + if oprot._fast_encode is not None and self.thrift_spec is not None: + oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec])) + return + oprot.writeStructBegin("AllTableConstraintsRequest") + if self.dbName is not None: + oprot.writeFieldBegin("dbName", TType.STRING, 1) + oprot.writeString(self.dbName.encode("utf-8") if sys.version_info[0] == 2 else self.dbName) + oprot.writeFieldEnd() + if self.tblName is not None: + oprot.writeFieldBegin("tblName", TType.STRING, 2) + oprot.writeString(self.tblName.encode("utf-8") if sys.version_info[0] == 2 else self.tblName) + oprot.writeFieldEnd() + if self.catName is not None: + oprot.writeFieldBegin("catName", TType.STRING, 3) + oprot.writeString(self.catName.encode("utf-8") if sys.version_info[0] == 2 else self.catName) + oprot.writeFieldEnd() + if self.validWriteIdList is not None: + oprot.writeFieldBegin("validWriteIdList", TType.STRING, 4) + oprot.writeString(self.validWriteIdList.encode("utf-8") if sys.version_info[0] == 2 else self.validWriteIdList) + oprot.writeFieldEnd() + if self.tableId is not None: + oprot.writeFieldBegin("tableId", TType.I64, 5) + oprot.writeI64(self.tableId) + oprot.writeFieldEnd() + oprot.writeFieldStop() + oprot.writeStructEnd() + + def validate(self): + if self.dbName is None: + raise TProtocolException(message="Required field dbName is unset!") + if self.tblName is None: + raise TProtocolException(message="Required field tblName is unset!") + if self.catName is None: + raise TProtocolException(message="Required field catName is unset!") + return + + def __repr__(self): + L = ["{}={!r}".format(key, value) for key, value in self.__dict__.items()] + return "{}({})".format(self.__class__.__name__, ", ".join(L)) + + def __eq__(self, other): + return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ + + def __ne__(self, other): + return not (self == other) + + +class AllTableConstraintsResponse: + """ + Attributes: + - allTableConstraints + + """ + + def __init__( + self, + allTableConstraints=None, + ): + self.allTableConstraints = allTableConstraints + + def read(self, iprot): + if ( + iprot._fast_decode is not None + and isinstance(iprot.trans, TTransport.CReadableTransport) + and self.thrift_spec is not None + ): + iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec]) + return + iprot.readStructBegin() + while True: + (fname, ftype, fid) = iprot.readFieldBegin() + if ftype == TType.STOP: + break + if fid == 1: + if ftype == TType.STRUCT: + self.allTableConstraints = SQLAllTableConstraints() + self.allTableConstraints.read(iprot) + else: + iprot.skip(ftype) + else: + iprot.skip(ftype) + iprot.readFieldEnd() + iprot.readStructEnd() + + def write(self, oprot): + if oprot._fast_encode is not None and self.thrift_spec is not None: + oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec])) + return + oprot.writeStructBegin("AllTableConstraintsResponse") + if self.allTableConstraints is not None: + oprot.writeFieldBegin("allTableConstraints", TType.STRUCT, 1) + self.allTableConstraints.write(oprot) + oprot.writeFieldEnd() + oprot.writeFieldStop() + oprot.writeStructEnd() + + def validate(self): + if self.allTableConstraints is None: + raise TProtocolException(message="Required field allTableConstraints is unset!") + return + + def __repr__(self): + L = ["{}={!r}".format(key, value) for key, value in self.__dict__.items()] + return "{}({})".format(self.__class__.__name__, ", ".join(L)) + + def __eq__(self, other): + return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ + + def __ne__(self, other): + return not (self == other) + + +class DropConstraintRequest: + """ + Attributes: + - dbname + - tablename + - constraintname + - catName + + """ + + def __init__( + self, + dbname=None, + tablename=None, + constraintname=None, + catName=None, + ): + self.dbname = dbname + self.tablename = tablename + self.constraintname = constraintname + self.catName = catName + + def read(self, iprot): + if ( + iprot._fast_decode is not None + and isinstance(iprot.trans, TTransport.CReadableTransport) + and self.thrift_spec is not None + ): + iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec]) + return + iprot.readStructBegin() + while True: + (fname, ftype, fid) = iprot.readFieldBegin() + if ftype == TType.STOP: + break + if fid == 1: + if ftype == TType.STRING: + self.dbname = ( + iprot.readString().decode("utf-8", errors="replace") if sys.version_info[0] == 2 else iprot.readString() + ) + else: + iprot.skip(ftype) + elif fid == 2: + if ftype == TType.STRING: + self.tablename = ( + iprot.readString().decode("utf-8", errors="replace") if sys.version_info[0] == 2 else iprot.readString() + ) + else: + iprot.skip(ftype) + elif fid == 3: + if ftype == TType.STRING: + self.constraintname = ( + iprot.readString().decode("utf-8", errors="replace") if sys.version_info[0] == 2 else iprot.readString() + ) + else: + iprot.skip(ftype) + elif fid == 4: + if ftype == TType.STRING: + self.catName = ( + iprot.readString().decode("utf-8", errors="replace") if sys.version_info[0] == 2 else iprot.readString() + ) + else: + iprot.skip(ftype) + else: + iprot.skip(ftype) + iprot.readFieldEnd() + iprot.readStructEnd() + + def write(self, oprot): + if oprot._fast_encode is not None and self.thrift_spec is not None: + oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec])) + return + oprot.writeStructBegin("DropConstraintRequest") + if self.dbname is not None: + oprot.writeFieldBegin("dbname", TType.STRING, 1) + oprot.writeString(self.dbname.encode("utf-8") if sys.version_info[0] == 2 else self.dbname) + oprot.writeFieldEnd() + if self.tablename is not None: + oprot.writeFieldBegin("tablename", TType.STRING, 2) + oprot.writeString(self.tablename.encode("utf-8") if sys.version_info[0] == 2 else self.tablename) + oprot.writeFieldEnd() + if self.constraintname is not None: + oprot.writeFieldBegin("constraintname", TType.STRING, 3) + oprot.writeString(self.constraintname.encode("utf-8") if sys.version_info[0] == 2 else self.constraintname) + oprot.writeFieldEnd() + if self.catName is not None: + oprot.writeFieldBegin("catName", TType.STRING, 4) + oprot.writeString(self.catName.encode("utf-8") if sys.version_info[0] == 2 else self.catName) + oprot.writeFieldEnd() + oprot.writeFieldStop() + oprot.writeStructEnd() + + def validate(self): + if self.dbname is None: + raise TProtocolException(message="Required field dbname is unset!") + if self.tablename is None: + raise TProtocolException(message="Required field tablename is unset!") + if self.constraintname is None: + raise TProtocolException(message="Required field constraintname is unset!") + return + + def __repr__(self): + L = ["{}={!r}".format(key, value) for key, value in self.__dict__.items()] + return "{}({})".format(self.__class__.__name__, ", ".join(L)) + + def __eq__(self, other): + return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ + + def __ne__(self, other): + return not (self == other) + + +class AddPrimaryKeyRequest: + """ + Attributes: + - primaryKeyCols + + """ + + def __init__( + self, + primaryKeyCols=None, + ): + self.primaryKeyCols = primaryKeyCols + + def read(self, iprot): + if ( + iprot._fast_decode is not None + and isinstance(iprot.trans, TTransport.CReadableTransport) + and self.thrift_spec is not None + ): + iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec]) + return + iprot.readStructBegin() + while True: + (fname, ftype, fid) = iprot.readFieldBegin() + if ftype == TType.STOP: + break + if fid == 1: + if ftype == TType.LIST: + self.primaryKeyCols = [] + (_etype428, _size425) = iprot.readListBegin() + for _i429 in range(_size425): + _elem430 = SQLPrimaryKey() + _elem430.read(iprot) + self.primaryKeyCols.append(_elem430) + iprot.readListEnd() + else: + iprot.skip(ftype) + else: + iprot.skip(ftype) + iprot.readFieldEnd() + iprot.readStructEnd() + + def write(self, oprot): + if oprot._fast_encode is not None and self.thrift_spec is not None: + oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec])) + return + oprot.writeStructBegin("AddPrimaryKeyRequest") + if self.primaryKeyCols is not None: + oprot.writeFieldBegin("primaryKeyCols", TType.LIST, 1) + oprot.writeListBegin(TType.STRUCT, len(self.primaryKeyCols)) + for iter431 in self.primaryKeyCols: + iter431.write(oprot) + oprot.writeListEnd() + oprot.writeFieldEnd() + oprot.writeFieldStop() + oprot.writeStructEnd() + + def validate(self): + if self.primaryKeyCols is None: + raise TProtocolException(message="Required field primaryKeyCols is unset!") + return + + def __repr__(self): + L = ["{}={!r}".format(key, value) for key, value in self.__dict__.items()] + return "{}({})".format(self.__class__.__name__, ", ".join(L)) + + def __eq__(self, other): + return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ + + def __ne__(self, other): + return not (self == other) + + +class AddForeignKeyRequest: + """ + Attributes: + - foreignKeyCols + + """ + + def __init__( + self, + foreignKeyCols=None, + ): + self.foreignKeyCols = foreignKeyCols + + def read(self, iprot): + if ( + iprot._fast_decode is not None + and isinstance(iprot.trans, TTransport.CReadableTransport) + and self.thrift_spec is not None + ): + iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec]) + return + iprot.readStructBegin() + while True: + (fname, ftype, fid) = iprot.readFieldBegin() + if ftype == TType.STOP: + break + if fid == 1: + if ftype == TType.LIST: + self.foreignKeyCols = [] + (_etype435, _size432) = iprot.readListBegin() + for _i436 in range(_size432): + _elem437 = SQLForeignKey() + _elem437.read(iprot) + self.foreignKeyCols.append(_elem437) + iprot.readListEnd() + else: + iprot.skip(ftype) + else: + iprot.skip(ftype) + iprot.readFieldEnd() + iprot.readStructEnd() + + def write(self, oprot): + if oprot._fast_encode is not None and self.thrift_spec is not None: + oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec])) + return + oprot.writeStructBegin("AddForeignKeyRequest") + if self.foreignKeyCols is not None: + oprot.writeFieldBegin("foreignKeyCols", TType.LIST, 1) + oprot.writeListBegin(TType.STRUCT, len(self.foreignKeyCols)) + for iter438 in self.foreignKeyCols: + iter438.write(oprot) + oprot.writeListEnd() + oprot.writeFieldEnd() + oprot.writeFieldStop() + oprot.writeStructEnd() + + def validate(self): + if self.foreignKeyCols is None: + raise TProtocolException(message="Required field foreignKeyCols is unset!") + return + + def __repr__(self): + L = ["{}={!r}".format(key, value) for key, value in self.__dict__.items()] + return "{}({})".format(self.__class__.__name__, ", ".join(L)) + + def __eq__(self, other): + return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ + + def __ne__(self, other): + return not (self == other) + + +class AddUniqueConstraintRequest: + """ + Attributes: + - uniqueConstraintCols + + """ + + def __init__( + self, + uniqueConstraintCols=None, + ): + self.uniqueConstraintCols = uniqueConstraintCols + + def read(self, iprot): + if ( + iprot._fast_decode is not None + and isinstance(iprot.trans, TTransport.CReadableTransport) + and self.thrift_spec is not None + ): + iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec]) + return + iprot.readStructBegin() + while True: + (fname, ftype, fid) = iprot.readFieldBegin() + if ftype == TType.STOP: + break + if fid == 1: + if ftype == TType.LIST: + self.uniqueConstraintCols = [] + (_etype442, _size439) = iprot.readListBegin() + for _i443 in range(_size439): + _elem444 = SQLUniqueConstraint() + _elem444.read(iprot) + self.uniqueConstraintCols.append(_elem444) + iprot.readListEnd() + else: + iprot.skip(ftype) + else: + iprot.skip(ftype) + iprot.readFieldEnd() + iprot.readStructEnd() + + def write(self, oprot): + if oprot._fast_encode is not None and self.thrift_spec is not None: + oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec])) + return + oprot.writeStructBegin("AddUniqueConstraintRequest") + if self.uniqueConstraintCols is not None: + oprot.writeFieldBegin("uniqueConstraintCols", TType.LIST, 1) + oprot.writeListBegin(TType.STRUCT, len(self.uniqueConstraintCols)) + for iter445 in self.uniqueConstraintCols: + iter445.write(oprot) + oprot.writeListEnd() + oprot.writeFieldEnd() + oprot.writeFieldStop() + oprot.writeStructEnd() + + def validate(self): + if self.uniqueConstraintCols is None: + raise TProtocolException(message="Required field uniqueConstraintCols is unset!") + return + + def __repr__(self): + L = ["{}={!r}".format(key, value) for key, value in self.__dict__.items()] + return "{}({})".format(self.__class__.__name__, ", ".join(L)) + + def __eq__(self, other): + return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ + + def __ne__(self, other): + return not (self == other) + + +class AddNotNullConstraintRequest: + """ + Attributes: + - notNullConstraintCols + + """ + + def __init__( + self, + notNullConstraintCols=None, + ): + self.notNullConstraintCols = notNullConstraintCols + + def read(self, iprot): + if ( + iprot._fast_decode is not None + and isinstance(iprot.trans, TTransport.CReadableTransport) + and self.thrift_spec is not None + ): + iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec]) + return + iprot.readStructBegin() + while True: + (fname, ftype, fid) = iprot.readFieldBegin() + if ftype == TType.STOP: + break + if fid == 1: + if ftype == TType.LIST: + self.notNullConstraintCols = [] + (_etype449, _size446) = iprot.readListBegin() + for _i450 in range(_size446): + _elem451 = SQLNotNullConstraint() + _elem451.read(iprot) + self.notNullConstraintCols.append(_elem451) + iprot.readListEnd() + else: + iprot.skip(ftype) + else: + iprot.skip(ftype) + iprot.readFieldEnd() + iprot.readStructEnd() + + def write(self, oprot): + if oprot._fast_encode is not None and self.thrift_spec is not None: + oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec])) + return + oprot.writeStructBegin("AddNotNullConstraintRequest") + if self.notNullConstraintCols is not None: + oprot.writeFieldBegin("notNullConstraintCols", TType.LIST, 1) + oprot.writeListBegin(TType.STRUCT, len(self.notNullConstraintCols)) + for iter452 in self.notNullConstraintCols: + iter452.write(oprot) + oprot.writeListEnd() + oprot.writeFieldEnd() + oprot.writeFieldStop() + oprot.writeStructEnd() + + def validate(self): + if self.notNullConstraintCols is None: + raise TProtocolException(message="Required field notNullConstraintCols is unset!") + return + + def __repr__(self): + L = ["{}={!r}".format(key, value) for key, value in self.__dict__.items()] + return "{}({})".format(self.__class__.__name__, ", ".join(L)) + + def __eq__(self, other): + return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ + + def __ne__(self, other): + return not (self == other) + + +class AddDefaultConstraintRequest: + """ + Attributes: + - defaultConstraintCols + + """ + + def __init__( + self, + defaultConstraintCols=None, + ): + self.defaultConstraintCols = defaultConstraintCols + + def read(self, iprot): + if ( + iprot._fast_decode is not None + and isinstance(iprot.trans, TTransport.CReadableTransport) + and self.thrift_spec is not None + ): + iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec]) + return + iprot.readStructBegin() + while True: + (fname, ftype, fid) = iprot.readFieldBegin() + if ftype == TType.STOP: + break + if fid == 1: + if ftype == TType.LIST: + self.defaultConstraintCols = [] + (_etype456, _size453) = iprot.readListBegin() + for _i457 in range(_size453): + _elem458 = SQLDefaultConstraint() + _elem458.read(iprot) + self.defaultConstraintCols.append(_elem458) + iprot.readListEnd() + else: + iprot.skip(ftype) + else: + iprot.skip(ftype) + iprot.readFieldEnd() + iprot.readStructEnd() + + def write(self, oprot): + if oprot._fast_encode is not None and self.thrift_spec is not None: + oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec])) + return + oprot.writeStructBegin("AddDefaultConstraintRequest") + if self.defaultConstraintCols is not None: + oprot.writeFieldBegin("defaultConstraintCols", TType.LIST, 1) + oprot.writeListBegin(TType.STRUCT, len(self.defaultConstraintCols)) + for iter459 in self.defaultConstraintCols: + iter459.write(oprot) + oprot.writeListEnd() + oprot.writeFieldEnd() + oprot.writeFieldStop() + oprot.writeStructEnd() + + def validate(self): + if self.defaultConstraintCols is None: + raise TProtocolException(message="Required field defaultConstraintCols is unset!") + return + + def __repr__(self): + L = ["{}={!r}".format(key, value) for key, value in self.__dict__.items()] + return "{}({})".format(self.__class__.__name__, ", ".join(L)) + + def __eq__(self, other): + return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ + + def __ne__(self, other): + return not (self == other) + + +class AddCheckConstraintRequest: + """ + Attributes: + - checkConstraintCols + + """ + + def __init__( + self, + checkConstraintCols=None, + ): + self.checkConstraintCols = checkConstraintCols + + def read(self, iprot): + if ( + iprot._fast_decode is not None + and isinstance(iprot.trans, TTransport.CReadableTransport) + and self.thrift_spec is not None + ): + iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec]) + return + iprot.readStructBegin() + while True: + (fname, ftype, fid) = iprot.readFieldBegin() + if ftype == TType.STOP: + break + if fid == 1: + if ftype == TType.LIST: + self.checkConstraintCols = [] + (_etype463, _size460) = iprot.readListBegin() + for _i464 in range(_size460): + _elem465 = SQLCheckConstraint() + _elem465.read(iprot) + self.checkConstraintCols.append(_elem465) + iprot.readListEnd() + else: + iprot.skip(ftype) + else: + iprot.skip(ftype) + iprot.readFieldEnd() + iprot.readStructEnd() + + def write(self, oprot): + if oprot._fast_encode is not None and self.thrift_spec is not None: + oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec])) + return + oprot.writeStructBegin("AddCheckConstraintRequest") + if self.checkConstraintCols is not None: + oprot.writeFieldBegin("checkConstraintCols", TType.LIST, 1) + oprot.writeListBegin(TType.STRUCT, len(self.checkConstraintCols)) + for iter466 in self.checkConstraintCols: + iter466.write(oprot) + oprot.writeListEnd() + oprot.writeFieldEnd() + oprot.writeFieldStop() + oprot.writeStructEnd() + + def validate(self): + if self.checkConstraintCols is None: + raise TProtocolException(message="Required field checkConstraintCols is unset!") + return + + def __repr__(self): + L = ["{}={!r}".format(key, value) for key, value in self.__dict__.items()] + return "{}({})".format(self.__class__.__name__, ", ".join(L)) + + def __eq__(self, other): + return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ + + def __ne__(self, other): + return not (self == other) + + +class PartitionsByExprResult: + """ + Attributes: + - partitions + - hasUnknownPartitions + + """ + + def __init__( + self, + partitions=None, + hasUnknownPartitions=None, + ): + self.partitions = partitions + self.hasUnknownPartitions = hasUnknownPartitions + + def read(self, iprot): + if ( + iprot._fast_decode is not None + and isinstance(iprot.trans, TTransport.CReadableTransport) + and self.thrift_spec is not None + ): + iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec]) + return + iprot.readStructBegin() + while True: + (fname, ftype, fid) = iprot.readFieldBegin() + if ftype == TType.STOP: + break + if fid == 1: + if ftype == TType.LIST: + self.partitions = [] + (_etype470, _size467) = iprot.readListBegin() + for _i471 in range(_size467): + _elem472 = Partition() + _elem472.read(iprot) + self.partitions.append(_elem472) + iprot.readListEnd() + else: + iprot.skip(ftype) + elif fid == 2: + if ftype == TType.BOOL: + self.hasUnknownPartitions = iprot.readBool() + else: + iprot.skip(ftype) + else: + iprot.skip(ftype) + iprot.readFieldEnd() + iprot.readStructEnd() + + def write(self, oprot): + if oprot._fast_encode is not None and self.thrift_spec is not None: + oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec])) + return + oprot.writeStructBegin("PartitionsByExprResult") + if self.partitions is not None: + oprot.writeFieldBegin("partitions", TType.LIST, 1) + oprot.writeListBegin(TType.STRUCT, len(self.partitions)) + for iter473 in self.partitions: + iter473.write(oprot) + oprot.writeListEnd() + oprot.writeFieldEnd() + if self.hasUnknownPartitions is not None: + oprot.writeFieldBegin("hasUnknownPartitions", TType.BOOL, 2) + oprot.writeBool(self.hasUnknownPartitions) + oprot.writeFieldEnd() + oprot.writeFieldStop() + oprot.writeStructEnd() + + def validate(self): + if self.partitions is None: + raise TProtocolException(message="Required field partitions is unset!") + if self.hasUnknownPartitions is None: + raise TProtocolException(message="Required field hasUnknownPartitions is unset!") + return + + def __repr__(self): + L = ["{}={!r}".format(key, value) for key, value in self.__dict__.items()] + return "{}({})".format(self.__class__.__name__, ", ".join(L)) + + def __eq__(self, other): + return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ + + def __ne__(self, other): + return not (self == other) + + +class PartitionsSpecByExprResult: + """ + Attributes: + - partitionsSpec + - hasUnknownPartitions + + """ + + def __init__( + self, + partitionsSpec=None, + hasUnknownPartitions=None, + ): + self.partitionsSpec = partitionsSpec + self.hasUnknownPartitions = hasUnknownPartitions + + def read(self, iprot): + if ( + iprot._fast_decode is not None + and isinstance(iprot.trans, TTransport.CReadableTransport) + and self.thrift_spec is not None + ): + iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec]) + return + iprot.readStructBegin() + while True: + (fname, ftype, fid) = iprot.readFieldBegin() + if ftype == TType.STOP: + break + if fid == 1: + if ftype == TType.LIST: + self.partitionsSpec = [] + (_etype477, _size474) = iprot.readListBegin() + for _i478 in range(_size474): + _elem479 = PartitionSpec() + _elem479.read(iprot) + self.partitionsSpec.append(_elem479) + iprot.readListEnd() + else: + iprot.skip(ftype) + elif fid == 2: + if ftype == TType.BOOL: + self.hasUnknownPartitions = iprot.readBool() + else: + iprot.skip(ftype) + else: + iprot.skip(ftype) + iprot.readFieldEnd() + iprot.readStructEnd() + + def write(self, oprot): + if oprot._fast_encode is not None and self.thrift_spec is not None: + oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec])) + return + oprot.writeStructBegin("PartitionsSpecByExprResult") + if self.partitionsSpec is not None: + oprot.writeFieldBegin("partitionsSpec", TType.LIST, 1) + oprot.writeListBegin(TType.STRUCT, len(self.partitionsSpec)) + for iter480 in self.partitionsSpec: + iter480.write(oprot) + oprot.writeListEnd() + oprot.writeFieldEnd() + if self.hasUnknownPartitions is not None: + oprot.writeFieldBegin("hasUnknownPartitions", TType.BOOL, 2) + oprot.writeBool(self.hasUnknownPartitions) + oprot.writeFieldEnd() + oprot.writeFieldStop() + oprot.writeStructEnd() + + def validate(self): + if self.partitionsSpec is None: + raise TProtocolException(message="Required field partitionsSpec is unset!") + if self.hasUnknownPartitions is None: + raise TProtocolException(message="Required field hasUnknownPartitions is unset!") + return + + def __repr__(self): + L = ["{}={!r}".format(key, value) for key, value in self.__dict__.items()] + return "{}({})".format(self.__class__.__name__, ", ".join(L)) + + def __eq__(self, other): + return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ + + def __ne__(self, other): + return not (self == other) + + +class PartitionsByExprRequest: + """ + Attributes: + - dbName + - tblName + - expr + - defaultPartitionName + - maxParts + - catName + - order + - validWriteIdList + - id + + """ + + def __init__( + self, + dbName=None, + tblName=None, + expr=None, + defaultPartitionName=None, + maxParts=-1, + catName=None, + order=None, + validWriteIdList=None, + id=-1, + ): + self.dbName = dbName + self.tblName = tblName + self.expr = expr + self.defaultPartitionName = defaultPartitionName + self.maxParts = maxParts + self.catName = catName + self.order = order + self.validWriteIdList = validWriteIdList + self.id = id + + def read(self, iprot): + if ( + iprot._fast_decode is not None + and isinstance(iprot.trans, TTransport.CReadableTransport) + and self.thrift_spec is not None + ): + iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec]) + return + iprot.readStructBegin() + while True: + (fname, ftype, fid) = iprot.readFieldBegin() + if ftype == TType.STOP: + break + if fid == 1: + if ftype == TType.STRING: + self.dbName = ( + iprot.readString().decode("utf-8", errors="replace") if sys.version_info[0] == 2 else iprot.readString() + ) + else: + iprot.skip(ftype) + elif fid == 2: + if ftype == TType.STRING: + self.tblName = ( + iprot.readString().decode("utf-8", errors="replace") if sys.version_info[0] == 2 else iprot.readString() + ) + else: + iprot.skip(ftype) + elif fid == 3: + if ftype == TType.STRING: + self.expr = iprot.readBinary() + else: + iprot.skip(ftype) + elif fid == 4: + if ftype == TType.STRING: + self.defaultPartitionName = ( + iprot.readString().decode("utf-8", errors="replace") if sys.version_info[0] == 2 else iprot.readString() + ) + else: + iprot.skip(ftype) + elif fid == 5: + if ftype == TType.I16: + self.maxParts = iprot.readI16() + else: + iprot.skip(ftype) + elif fid == 6: + if ftype == TType.STRING: + self.catName = ( + iprot.readString().decode("utf-8", errors="replace") if sys.version_info[0] == 2 else iprot.readString() + ) + else: + iprot.skip(ftype) + elif fid == 7: + if ftype == TType.STRING: + self.order = ( + iprot.readString().decode("utf-8", errors="replace") if sys.version_info[0] == 2 else iprot.readString() + ) + else: + iprot.skip(ftype) + elif fid == 8: + if ftype == TType.STRING: + self.validWriteIdList = ( + iprot.readString().decode("utf-8", errors="replace") if sys.version_info[0] == 2 else iprot.readString() + ) + else: + iprot.skip(ftype) + elif fid == 9: + if ftype == TType.I64: + self.id = iprot.readI64() + else: + iprot.skip(ftype) + else: + iprot.skip(ftype) + iprot.readFieldEnd() + iprot.readStructEnd() + + def write(self, oprot): + if oprot._fast_encode is not None and self.thrift_spec is not None: + oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec])) + return + oprot.writeStructBegin("PartitionsByExprRequest") + if self.dbName is not None: + oprot.writeFieldBegin("dbName", TType.STRING, 1) + oprot.writeString(self.dbName.encode("utf-8") if sys.version_info[0] == 2 else self.dbName) + oprot.writeFieldEnd() + if self.tblName is not None: + oprot.writeFieldBegin("tblName", TType.STRING, 2) + oprot.writeString(self.tblName.encode("utf-8") if sys.version_info[0] == 2 else self.tblName) + oprot.writeFieldEnd() + if self.expr is not None: + oprot.writeFieldBegin("expr", TType.STRING, 3) + oprot.writeBinary(self.expr) + oprot.writeFieldEnd() + if self.defaultPartitionName is not None: + oprot.writeFieldBegin("defaultPartitionName", TType.STRING, 4) + oprot.writeString( + self.defaultPartitionName.encode("utf-8") if sys.version_info[0] == 2 else self.defaultPartitionName + ) + oprot.writeFieldEnd() + if self.maxParts is not None: + oprot.writeFieldBegin("maxParts", TType.I16, 5) + oprot.writeI16(self.maxParts) + oprot.writeFieldEnd() + if self.catName is not None: + oprot.writeFieldBegin("catName", TType.STRING, 6) + oprot.writeString(self.catName.encode("utf-8") if sys.version_info[0] == 2 else self.catName) + oprot.writeFieldEnd() + if self.order is not None: + oprot.writeFieldBegin("order", TType.STRING, 7) + oprot.writeString(self.order.encode("utf-8") if sys.version_info[0] == 2 else self.order) + oprot.writeFieldEnd() + if self.validWriteIdList is not None: + oprot.writeFieldBegin("validWriteIdList", TType.STRING, 8) + oprot.writeString(self.validWriteIdList.encode("utf-8") if sys.version_info[0] == 2 else self.validWriteIdList) + oprot.writeFieldEnd() + if self.id is not None: + oprot.writeFieldBegin("id", TType.I64, 9) + oprot.writeI64(self.id) + oprot.writeFieldEnd() + oprot.writeFieldStop() + oprot.writeStructEnd() + + def validate(self): + if self.dbName is None: + raise TProtocolException(message="Required field dbName is unset!") + if self.tblName is None: + raise TProtocolException(message="Required field tblName is unset!") + if self.expr is None: + raise TProtocolException(message="Required field expr is unset!") + return + + def __repr__(self): + L = ["{}={!r}".format(key, value) for key, value in self.__dict__.items()] + return "{}({})".format(self.__class__.__name__, ", ".join(L)) + + def __eq__(self, other): + return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ + + def __ne__(self, other): + return not (self == other) + + +class TableStatsResult: + """ + Attributes: + - tableStats + - isStatsCompliant + + """ + + def __init__( + self, + tableStats=None, + isStatsCompliant=None, + ): + self.tableStats = tableStats + self.isStatsCompliant = isStatsCompliant + + def read(self, iprot): + if ( + iprot._fast_decode is not None + and isinstance(iprot.trans, TTransport.CReadableTransport) + and self.thrift_spec is not None + ): + iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec]) + return + iprot.readStructBegin() + while True: + (fname, ftype, fid) = iprot.readFieldBegin() + if ftype == TType.STOP: + break + if fid == 1: + if ftype == TType.LIST: + self.tableStats = [] + (_etype484, _size481) = iprot.readListBegin() + for _i485 in range(_size481): + _elem486 = ColumnStatisticsObj() + _elem486.read(iprot) + self.tableStats.append(_elem486) + iprot.readListEnd() + else: + iprot.skip(ftype) + elif fid == 2: + if ftype == TType.BOOL: + self.isStatsCompliant = iprot.readBool() + else: + iprot.skip(ftype) + else: + iprot.skip(ftype) + iprot.readFieldEnd() + iprot.readStructEnd() + + def write(self, oprot): + if oprot._fast_encode is not None and self.thrift_spec is not None: + oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec])) + return + oprot.writeStructBegin("TableStatsResult") + if self.tableStats is not None: + oprot.writeFieldBegin("tableStats", TType.LIST, 1) + oprot.writeListBegin(TType.STRUCT, len(self.tableStats)) + for iter487 in self.tableStats: + iter487.write(oprot) + oprot.writeListEnd() + oprot.writeFieldEnd() + if self.isStatsCompliant is not None: + oprot.writeFieldBegin("isStatsCompliant", TType.BOOL, 2) + oprot.writeBool(self.isStatsCompliant) + oprot.writeFieldEnd() + oprot.writeFieldStop() + oprot.writeStructEnd() + + def validate(self): + if self.tableStats is None: + raise TProtocolException(message="Required field tableStats is unset!") + return + + def __repr__(self): + L = ["{}={!r}".format(key, value) for key, value in self.__dict__.items()] + return "{}({})".format(self.__class__.__name__, ", ".join(L)) + + def __eq__(self, other): + return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ + + def __ne__(self, other): + return not (self == other) + + +class PartitionsStatsResult: + """ + Attributes: + - partStats + - isStatsCompliant + + """ + + def __init__( + self, + partStats=None, + isStatsCompliant=None, + ): + self.partStats = partStats + self.isStatsCompliant = isStatsCompliant + + def read(self, iprot): + if ( + iprot._fast_decode is not None + and isinstance(iprot.trans, TTransport.CReadableTransport) + and self.thrift_spec is not None + ): + iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec]) + return + iprot.readStructBegin() + while True: + (fname, ftype, fid) = iprot.readFieldBegin() + if ftype == TType.STOP: + break + if fid == 1: + if ftype == TType.MAP: + self.partStats = {} + (_ktype489, _vtype490, _size488) = iprot.readMapBegin() + for _i492 in range(_size488): + _key493 = ( + iprot.readString().decode("utf-8", errors="replace") + if sys.version_info[0] == 2 + else iprot.readString() + ) + _val494 = [] + (_etype498, _size495) = iprot.readListBegin() + for _i499 in range(_size495): + _elem500 = ColumnStatisticsObj() + _elem500.read(iprot) + _val494.append(_elem500) + iprot.readListEnd() + self.partStats[_key493] = _val494 + iprot.readMapEnd() + else: + iprot.skip(ftype) + elif fid == 2: + if ftype == TType.BOOL: + self.isStatsCompliant = iprot.readBool() + else: + iprot.skip(ftype) + else: + iprot.skip(ftype) + iprot.readFieldEnd() + iprot.readStructEnd() + + def write(self, oprot): + if oprot._fast_encode is not None and self.thrift_spec is not None: + oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec])) + return + oprot.writeStructBegin("PartitionsStatsResult") + if self.partStats is not None: + oprot.writeFieldBegin("partStats", TType.MAP, 1) + oprot.writeMapBegin(TType.STRING, TType.LIST, len(self.partStats)) + for kiter501, viter502 in self.partStats.items(): + oprot.writeString(kiter501.encode("utf-8") if sys.version_info[0] == 2 else kiter501) + oprot.writeListBegin(TType.STRUCT, len(viter502)) + for iter503 in viter502: + iter503.write(oprot) + oprot.writeListEnd() + oprot.writeMapEnd() + oprot.writeFieldEnd() + if self.isStatsCompliant is not None: + oprot.writeFieldBegin("isStatsCompliant", TType.BOOL, 2) + oprot.writeBool(self.isStatsCompliant) + oprot.writeFieldEnd() + oprot.writeFieldStop() + oprot.writeStructEnd() + + def validate(self): + if self.partStats is None: + raise TProtocolException(message="Required field partStats is unset!") + return + + def __repr__(self): + L = ["{}={!r}".format(key, value) for key, value in self.__dict__.items()] + return "{}({})".format(self.__class__.__name__, ", ".join(L)) + + def __eq__(self, other): + return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ + + def __ne__(self, other): + return not (self == other) + + +class TableStatsRequest: + """ + Attributes: + - dbName + - tblName + - colNames + - catName + - validWriteIdList + - engine + - id + + """ + + def __init__( + self, + dbName=None, + tblName=None, + colNames=None, + catName=None, + validWriteIdList=None, + engine=None, + id=-1, + ): + self.dbName = dbName + self.tblName = tblName + self.colNames = colNames + self.catName = catName + self.validWriteIdList = validWriteIdList + self.engine = engine + self.id = id + + def read(self, iprot): + if ( + iprot._fast_decode is not None + and isinstance(iprot.trans, TTransport.CReadableTransport) + and self.thrift_spec is not None + ): + iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec]) + return + iprot.readStructBegin() + while True: + (fname, ftype, fid) = iprot.readFieldBegin() + if ftype == TType.STOP: + break + if fid == 1: + if ftype == TType.STRING: + self.dbName = ( + iprot.readString().decode("utf-8", errors="replace") if sys.version_info[0] == 2 else iprot.readString() + ) + else: + iprot.skip(ftype) + elif fid == 2: + if ftype == TType.STRING: + self.tblName = ( + iprot.readString().decode("utf-8", errors="replace") if sys.version_info[0] == 2 else iprot.readString() + ) + else: + iprot.skip(ftype) + elif fid == 3: + if ftype == TType.LIST: + self.colNames = [] + (_etype507, _size504) = iprot.readListBegin() + for _i508 in range(_size504): + _elem509 = ( + iprot.readString().decode("utf-8", errors="replace") + if sys.version_info[0] == 2 + else iprot.readString() + ) + self.colNames.append(_elem509) + iprot.readListEnd() + else: + iprot.skip(ftype) + elif fid == 4: + if ftype == TType.STRING: + self.catName = ( + iprot.readString().decode("utf-8", errors="replace") if sys.version_info[0] == 2 else iprot.readString() + ) + else: + iprot.skip(ftype) + elif fid == 5: + if ftype == TType.STRING: + self.validWriteIdList = ( + iprot.readString().decode("utf-8", errors="replace") if sys.version_info[0] == 2 else iprot.readString() + ) + else: + iprot.skip(ftype) + elif fid == 6: + if ftype == TType.STRING: + self.engine = ( + iprot.readString().decode("utf-8", errors="replace") if sys.version_info[0] == 2 else iprot.readString() + ) + else: + iprot.skip(ftype) + elif fid == 7: + if ftype == TType.I64: + self.id = iprot.readI64() + else: + iprot.skip(ftype) + else: + iprot.skip(ftype) + iprot.readFieldEnd() + iprot.readStructEnd() + + def write(self, oprot): + if oprot._fast_encode is not None and self.thrift_spec is not None: + oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec])) + return + oprot.writeStructBegin("TableStatsRequest") + if self.dbName is not None: + oprot.writeFieldBegin("dbName", TType.STRING, 1) + oprot.writeString(self.dbName.encode("utf-8") if sys.version_info[0] == 2 else self.dbName) + oprot.writeFieldEnd() + if self.tblName is not None: + oprot.writeFieldBegin("tblName", TType.STRING, 2) + oprot.writeString(self.tblName.encode("utf-8") if sys.version_info[0] == 2 else self.tblName) + oprot.writeFieldEnd() + if self.colNames is not None: + oprot.writeFieldBegin("colNames", TType.LIST, 3) + oprot.writeListBegin(TType.STRING, len(self.colNames)) + for iter510 in self.colNames: + oprot.writeString(iter510.encode("utf-8") if sys.version_info[0] == 2 else iter510) + oprot.writeListEnd() + oprot.writeFieldEnd() + if self.catName is not None: + oprot.writeFieldBegin("catName", TType.STRING, 4) + oprot.writeString(self.catName.encode("utf-8") if sys.version_info[0] == 2 else self.catName) + oprot.writeFieldEnd() + if self.validWriteIdList is not None: + oprot.writeFieldBegin("validWriteIdList", TType.STRING, 5) + oprot.writeString(self.validWriteIdList.encode("utf-8") if sys.version_info[0] == 2 else self.validWriteIdList) + oprot.writeFieldEnd() + if self.engine is not None: + oprot.writeFieldBegin("engine", TType.STRING, 6) + oprot.writeString(self.engine.encode("utf-8") if sys.version_info[0] == 2 else self.engine) + oprot.writeFieldEnd() + if self.id is not None: + oprot.writeFieldBegin("id", TType.I64, 7) + oprot.writeI64(self.id) + oprot.writeFieldEnd() + oprot.writeFieldStop() + oprot.writeStructEnd() + + def validate(self): + if self.dbName is None: + raise TProtocolException(message="Required field dbName is unset!") + if self.tblName is None: + raise TProtocolException(message="Required field tblName is unset!") + if self.colNames is None: + raise TProtocolException(message="Required field colNames is unset!") + if self.engine is None: + raise TProtocolException(message="Required field engine is unset!") + return + + def __repr__(self): + L = ["{}={!r}".format(key, value) for key, value in self.__dict__.items()] + return "{}({})".format(self.__class__.__name__, ", ".join(L)) + + def __eq__(self, other): + return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ + + def __ne__(self, other): + return not (self == other) + + +class PartitionsStatsRequest: + """ + Attributes: + - dbName + - tblName + - colNames + - partNames + - catName + - validWriteIdList + - engine + + """ + + def __init__( + self, + dbName=None, + tblName=None, + colNames=None, + partNames=None, + catName=None, + validWriteIdList=None, + engine=None, + ): + self.dbName = dbName + self.tblName = tblName + self.colNames = colNames + self.partNames = partNames + self.catName = catName + self.validWriteIdList = validWriteIdList + self.engine = engine + + def read(self, iprot): + if ( + iprot._fast_decode is not None + and isinstance(iprot.trans, TTransport.CReadableTransport) + and self.thrift_spec is not None + ): + iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec]) + return + iprot.readStructBegin() + while True: + (fname, ftype, fid) = iprot.readFieldBegin() + if ftype == TType.STOP: + break + if fid == 1: + if ftype == TType.STRING: + self.dbName = ( + iprot.readString().decode("utf-8", errors="replace") if sys.version_info[0] == 2 else iprot.readString() + ) + else: + iprot.skip(ftype) + elif fid == 2: + if ftype == TType.STRING: + self.tblName = ( + iprot.readString().decode("utf-8", errors="replace") if sys.version_info[0] == 2 else iprot.readString() + ) + else: + iprot.skip(ftype) + elif fid == 3: + if ftype == TType.LIST: + self.colNames = [] + (_etype514, _size511) = iprot.readListBegin() + for _i515 in range(_size511): + _elem516 = ( + iprot.readString().decode("utf-8", errors="replace") + if sys.version_info[0] == 2 + else iprot.readString() + ) + self.colNames.append(_elem516) + iprot.readListEnd() + else: + iprot.skip(ftype) + elif fid == 4: + if ftype == TType.LIST: + self.partNames = [] + (_etype520, _size517) = iprot.readListBegin() + for _i521 in range(_size517): + _elem522 = ( + iprot.readString().decode("utf-8", errors="replace") + if sys.version_info[0] == 2 + else iprot.readString() + ) + self.partNames.append(_elem522) + iprot.readListEnd() + else: + iprot.skip(ftype) + elif fid == 5: + if ftype == TType.STRING: + self.catName = ( + iprot.readString().decode("utf-8", errors="replace") if sys.version_info[0] == 2 else iprot.readString() + ) + else: + iprot.skip(ftype) + elif fid == 6: + if ftype == TType.STRING: + self.validWriteIdList = ( + iprot.readString().decode("utf-8", errors="replace") if sys.version_info[0] == 2 else iprot.readString() + ) + else: + iprot.skip(ftype) + elif fid == 7: + if ftype == TType.STRING: + self.engine = ( + iprot.readString().decode("utf-8", errors="replace") if sys.version_info[0] == 2 else iprot.readString() + ) + else: + iprot.skip(ftype) + else: + iprot.skip(ftype) + iprot.readFieldEnd() + iprot.readStructEnd() + + def write(self, oprot): + if oprot._fast_encode is not None and self.thrift_spec is not None: + oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec])) + return + oprot.writeStructBegin("PartitionsStatsRequest") + if self.dbName is not None: + oprot.writeFieldBegin("dbName", TType.STRING, 1) + oprot.writeString(self.dbName.encode("utf-8") if sys.version_info[0] == 2 else self.dbName) + oprot.writeFieldEnd() + if self.tblName is not None: + oprot.writeFieldBegin("tblName", TType.STRING, 2) + oprot.writeString(self.tblName.encode("utf-8") if sys.version_info[0] == 2 else self.tblName) + oprot.writeFieldEnd() + if self.colNames is not None: + oprot.writeFieldBegin("colNames", TType.LIST, 3) + oprot.writeListBegin(TType.STRING, len(self.colNames)) + for iter523 in self.colNames: + oprot.writeString(iter523.encode("utf-8") if sys.version_info[0] == 2 else iter523) + oprot.writeListEnd() + oprot.writeFieldEnd() + if self.partNames is not None: + oprot.writeFieldBegin("partNames", TType.LIST, 4) + oprot.writeListBegin(TType.STRING, len(self.partNames)) + for iter524 in self.partNames: + oprot.writeString(iter524.encode("utf-8") if sys.version_info[0] == 2 else iter524) + oprot.writeListEnd() + oprot.writeFieldEnd() + if self.catName is not None: + oprot.writeFieldBegin("catName", TType.STRING, 5) + oprot.writeString(self.catName.encode("utf-8") if sys.version_info[0] == 2 else self.catName) + oprot.writeFieldEnd() + if self.validWriteIdList is not None: + oprot.writeFieldBegin("validWriteIdList", TType.STRING, 6) + oprot.writeString(self.validWriteIdList.encode("utf-8") if sys.version_info[0] == 2 else self.validWriteIdList) + oprot.writeFieldEnd() + if self.engine is not None: + oprot.writeFieldBegin("engine", TType.STRING, 7) + oprot.writeString(self.engine.encode("utf-8") if sys.version_info[0] == 2 else self.engine) + oprot.writeFieldEnd() + oprot.writeFieldStop() + oprot.writeStructEnd() + + def validate(self): + if self.dbName is None: + raise TProtocolException(message="Required field dbName is unset!") + if self.tblName is None: + raise TProtocolException(message="Required field tblName is unset!") + if self.colNames is None: + raise TProtocolException(message="Required field colNames is unset!") + if self.partNames is None: + raise TProtocolException(message="Required field partNames is unset!") + if self.engine is None: + raise TProtocolException(message="Required field engine is unset!") + return + + def __repr__(self): + L = ["{}={!r}".format(key, value) for key, value in self.__dict__.items()] + return "{}({})".format(self.__class__.__name__, ", ".join(L)) + + def __eq__(self, other): + return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ + + def __ne__(self, other): + return not (self == other) + + +class AddPartitionsResult: + """ + Attributes: + - partitions + - isStatsCompliant + + """ + + def __init__( + self, + partitions=None, + isStatsCompliant=None, + ): + self.partitions = partitions + self.isStatsCompliant = isStatsCompliant + + def read(self, iprot): + if ( + iprot._fast_decode is not None + and isinstance(iprot.trans, TTransport.CReadableTransport) + and self.thrift_spec is not None + ): + iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec]) + return + iprot.readStructBegin() + while True: + (fname, ftype, fid) = iprot.readFieldBegin() + if ftype == TType.STOP: + break + if fid == 1: + if ftype == TType.LIST: + self.partitions = [] + (_etype528, _size525) = iprot.readListBegin() + for _i529 in range(_size525): + _elem530 = Partition() + _elem530.read(iprot) + self.partitions.append(_elem530) + iprot.readListEnd() + else: + iprot.skip(ftype) + elif fid == 2: + if ftype == TType.BOOL: + self.isStatsCompliant = iprot.readBool() + else: + iprot.skip(ftype) + else: + iprot.skip(ftype) + iprot.readFieldEnd() + iprot.readStructEnd() + + def write(self, oprot): + if oprot._fast_encode is not None and self.thrift_spec is not None: + oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec])) + return + oprot.writeStructBegin("AddPartitionsResult") + if self.partitions is not None: + oprot.writeFieldBegin("partitions", TType.LIST, 1) + oprot.writeListBegin(TType.STRUCT, len(self.partitions)) + for iter531 in self.partitions: + iter531.write(oprot) + oprot.writeListEnd() + oprot.writeFieldEnd() + if self.isStatsCompliant is not None: + oprot.writeFieldBegin("isStatsCompliant", TType.BOOL, 2) + oprot.writeBool(self.isStatsCompliant) + oprot.writeFieldEnd() + oprot.writeFieldStop() + oprot.writeStructEnd() + + def validate(self): + return + + def __repr__(self): + L = ["{}={!r}".format(key, value) for key, value in self.__dict__.items()] + return "{}({})".format(self.__class__.__name__, ", ".join(L)) + + def __eq__(self, other): + return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ + + def __ne__(self, other): + return not (self == other) + + +class AddPartitionsRequest: + """ + Attributes: + - dbName + - tblName + - parts + - ifNotExists + - needResult + - catName + - validWriteIdList + + """ + + def __init__( + self, + dbName=None, + tblName=None, + parts=None, + ifNotExists=None, + needResult=True, + catName=None, + validWriteIdList=None, + ): + self.dbName = dbName + self.tblName = tblName + self.parts = parts + self.ifNotExists = ifNotExists + self.needResult = needResult + self.catName = catName + self.validWriteIdList = validWriteIdList + + def read(self, iprot): + if ( + iprot._fast_decode is not None + and isinstance(iprot.trans, TTransport.CReadableTransport) + and self.thrift_spec is not None + ): + iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec]) + return + iprot.readStructBegin() + while True: + (fname, ftype, fid) = iprot.readFieldBegin() + if ftype == TType.STOP: + break + if fid == 1: + if ftype == TType.STRING: + self.dbName = ( + iprot.readString().decode("utf-8", errors="replace") if sys.version_info[0] == 2 else iprot.readString() + ) + else: + iprot.skip(ftype) + elif fid == 2: + if ftype == TType.STRING: + self.tblName = ( + iprot.readString().decode("utf-8", errors="replace") if sys.version_info[0] == 2 else iprot.readString() + ) + else: + iprot.skip(ftype) + elif fid == 3: + if ftype == TType.LIST: + self.parts = [] + (_etype535, _size532) = iprot.readListBegin() + for _i536 in range(_size532): + _elem537 = Partition() + _elem537.read(iprot) + self.parts.append(_elem537) + iprot.readListEnd() + else: + iprot.skip(ftype) + elif fid == 4: + if ftype == TType.BOOL: + self.ifNotExists = iprot.readBool() + else: + iprot.skip(ftype) + elif fid == 5: + if ftype == TType.BOOL: + self.needResult = iprot.readBool() + else: + iprot.skip(ftype) + elif fid == 6: + if ftype == TType.STRING: + self.catName = ( + iprot.readString().decode("utf-8", errors="replace") if sys.version_info[0] == 2 else iprot.readString() + ) + else: + iprot.skip(ftype) + elif fid == 7: + if ftype == TType.STRING: + self.validWriteIdList = ( + iprot.readString().decode("utf-8", errors="replace") if sys.version_info[0] == 2 else iprot.readString() + ) + else: + iprot.skip(ftype) + else: + iprot.skip(ftype) + iprot.readFieldEnd() + iprot.readStructEnd() + + def write(self, oprot): + if oprot._fast_encode is not None and self.thrift_spec is not None: + oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec])) + return + oprot.writeStructBegin("AddPartitionsRequest") + if self.dbName is not None: + oprot.writeFieldBegin("dbName", TType.STRING, 1) + oprot.writeString(self.dbName.encode("utf-8") if sys.version_info[0] == 2 else self.dbName) + oprot.writeFieldEnd() + if self.tblName is not None: + oprot.writeFieldBegin("tblName", TType.STRING, 2) + oprot.writeString(self.tblName.encode("utf-8") if sys.version_info[0] == 2 else self.tblName) + oprot.writeFieldEnd() + if self.parts is not None: + oprot.writeFieldBegin("parts", TType.LIST, 3) + oprot.writeListBegin(TType.STRUCT, len(self.parts)) + for iter538 in self.parts: + iter538.write(oprot) + oprot.writeListEnd() + oprot.writeFieldEnd() + if self.ifNotExists is not None: + oprot.writeFieldBegin("ifNotExists", TType.BOOL, 4) + oprot.writeBool(self.ifNotExists) + oprot.writeFieldEnd() + if self.needResult is not None: + oprot.writeFieldBegin("needResult", TType.BOOL, 5) + oprot.writeBool(self.needResult) + oprot.writeFieldEnd() + if self.catName is not None: + oprot.writeFieldBegin("catName", TType.STRING, 6) + oprot.writeString(self.catName.encode("utf-8") if sys.version_info[0] == 2 else self.catName) + oprot.writeFieldEnd() + if self.validWriteIdList is not None: + oprot.writeFieldBegin("validWriteIdList", TType.STRING, 7) + oprot.writeString(self.validWriteIdList.encode("utf-8") if sys.version_info[0] == 2 else self.validWriteIdList) + oprot.writeFieldEnd() + oprot.writeFieldStop() + oprot.writeStructEnd() + + def validate(self): + if self.dbName is None: + raise TProtocolException(message="Required field dbName is unset!") + if self.tblName is None: + raise TProtocolException(message="Required field tblName is unset!") + if self.parts is None: + raise TProtocolException(message="Required field parts is unset!") + if self.ifNotExists is None: + raise TProtocolException(message="Required field ifNotExists is unset!") + return + + def __repr__(self): + L = ["{}={!r}".format(key, value) for key, value in self.__dict__.items()] + return "{}({})".format(self.__class__.__name__, ", ".join(L)) + + def __eq__(self, other): + return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ + + def __ne__(self, other): + return not (self == other) + + +class DropPartitionsResult: + """ + Attributes: + - partitions + + """ + + def __init__( + self, + partitions=None, + ): + self.partitions = partitions + + def read(self, iprot): + if ( + iprot._fast_decode is not None + and isinstance(iprot.trans, TTransport.CReadableTransport) + and self.thrift_spec is not None + ): + iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec]) + return + iprot.readStructBegin() + while True: + (fname, ftype, fid) = iprot.readFieldBegin() + if ftype == TType.STOP: + break + if fid == 1: + if ftype == TType.LIST: + self.partitions = [] + (_etype542, _size539) = iprot.readListBegin() + for _i543 in range(_size539): + _elem544 = Partition() + _elem544.read(iprot) + self.partitions.append(_elem544) + iprot.readListEnd() + else: + iprot.skip(ftype) + else: + iprot.skip(ftype) + iprot.readFieldEnd() + iprot.readStructEnd() + + def write(self, oprot): + if oprot._fast_encode is not None and self.thrift_spec is not None: + oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec])) + return + oprot.writeStructBegin("DropPartitionsResult") + if self.partitions is not None: + oprot.writeFieldBegin("partitions", TType.LIST, 1) + oprot.writeListBegin(TType.STRUCT, len(self.partitions)) + for iter545 in self.partitions: + iter545.write(oprot) + oprot.writeListEnd() + oprot.writeFieldEnd() + oprot.writeFieldStop() + oprot.writeStructEnd() + + def validate(self): + return + + def __repr__(self): + L = ["{}={!r}".format(key, value) for key, value in self.__dict__.items()] + return "{}({})".format(self.__class__.__name__, ", ".join(L)) + + def __eq__(self, other): + return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ + + def __ne__(self, other): + return not (self == other) + + +class DropPartitionsExpr: + """ + Attributes: + - expr + - partArchiveLevel + + """ + + def __init__( + self, + expr=None, + partArchiveLevel=None, + ): + self.expr = expr + self.partArchiveLevel = partArchiveLevel + + def read(self, iprot): + if ( + iprot._fast_decode is not None + and isinstance(iprot.trans, TTransport.CReadableTransport) + and self.thrift_spec is not None + ): + iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec]) + return + iprot.readStructBegin() + while True: + (fname, ftype, fid) = iprot.readFieldBegin() + if ftype == TType.STOP: + break + if fid == 1: + if ftype == TType.STRING: + self.expr = iprot.readBinary() + else: + iprot.skip(ftype) + elif fid == 2: + if ftype == TType.I32: + self.partArchiveLevel = iprot.readI32() + else: + iprot.skip(ftype) + else: + iprot.skip(ftype) + iprot.readFieldEnd() + iprot.readStructEnd() + + def write(self, oprot): + if oprot._fast_encode is not None and self.thrift_spec is not None: + oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec])) + return + oprot.writeStructBegin("DropPartitionsExpr") + if self.expr is not None: + oprot.writeFieldBegin("expr", TType.STRING, 1) + oprot.writeBinary(self.expr) + oprot.writeFieldEnd() + if self.partArchiveLevel is not None: + oprot.writeFieldBegin("partArchiveLevel", TType.I32, 2) + oprot.writeI32(self.partArchiveLevel) + oprot.writeFieldEnd() + oprot.writeFieldStop() + oprot.writeStructEnd() + + def validate(self): + if self.expr is None: + raise TProtocolException(message="Required field expr is unset!") + return + + def __repr__(self): + L = ["{}={!r}".format(key, value) for key, value in self.__dict__.items()] + return "{}({})".format(self.__class__.__name__, ", ".join(L)) + + def __eq__(self, other): + return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ + + def __ne__(self, other): + return not (self == other) + + +class RequestPartsSpec: + """ + Attributes: + - names + - exprs + + """ + + def __init__( + self, + names=None, + exprs=None, + ): + self.names = names + self.exprs = exprs + + def read(self, iprot): + if ( + iprot._fast_decode is not None + and isinstance(iprot.trans, TTransport.CReadableTransport) + and self.thrift_spec is not None + ): + iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec]) + return + iprot.readStructBegin() + while True: + (fname, ftype, fid) = iprot.readFieldBegin() + if ftype == TType.STOP: + break + if fid == 1: + if ftype == TType.LIST: + self.names = [] + (_etype549, _size546) = iprot.readListBegin() + for _i550 in range(_size546): + _elem551 = ( + iprot.readString().decode("utf-8", errors="replace") + if sys.version_info[0] == 2 + else iprot.readString() + ) + self.names.append(_elem551) + iprot.readListEnd() + else: + iprot.skip(ftype) + elif fid == 2: + if ftype == TType.LIST: + self.exprs = [] + (_etype555, _size552) = iprot.readListBegin() + for _i556 in range(_size552): + _elem557 = DropPartitionsExpr() + _elem557.read(iprot) + self.exprs.append(_elem557) + iprot.readListEnd() + else: + iprot.skip(ftype) + else: + iprot.skip(ftype) + iprot.readFieldEnd() + iprot.readStructEnd() + + def write(self, oprot): + if oprot._fast_encode is not None and self.thrift_spec is not None: + oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec])) + return + oprot.writeStructBegin("RequestPartsSpec") + if self.names is not None: + oprot.writeFieldBegin("names", TType.LIST, 1) + oprot.writeListBegin(TType.STRING, len(self.names)) + for iter558 in self.names: + oprot.writeString(iter558.encode("utf-8") if sys.version_info[0] == 2 else iter558) + oprot.writeListEnd() + oprot.writeFieldEnd() + if self.exprs is not None: + oprot.writeFieldBegin("exprs", TType.LIST, 2) + oprot.writeListBegin(TType.STRUCT, len(self.exprs)) + for iter559 in self.exprs: + iter559.write(oprot) + oprot.writeListEnd() + oprot.writeFieldEnd() + oprot.writeFieldStop() + oprot.writeStructEnd() + + def validate(self): + return + + def __repr__(self): + L = ["{}={!r}".format(key, value) for key, value in self.__dict__.items()] + return "{}({})".format(self.__class__.__name__, ", ".join(L)) + + def __eq__(self, other): + return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ + + def __ne__(self, other): + return not (self == other) + + +class DropPartitionsRequest: + """ + Attributes: + - dbName + - tblName + - parts + - deleteData + - ifExists + - ignoreProtection + - environmentContext + - needResult + - catName + + """ + + def __init__( + self, + dbName=None, + tblName=None, + parts=None, + deleteData=None, + ifExists=True, + ignoreProtection=None, + environmentContext=None, + needResult=True, + catName=None, + ): + self.dbName = dbName + self.tblName = tblName + self.parts = parts + self.deleteData = deleteData + self.ifExists = ifExists + self.ignoreProtection = ignoreProtection + self.environmentContext = environmentContext + self.needResult = needResult + self.catName = catName + + def read(self, iprot): + if ( + iprot._fast_decode is not None + and isinstance(iprot.trans, TTransport.CReadableTransport) + and self.thrift_spec is not None + ): + iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec]) + return + iprot.readStructBegin() + while True: + (fname, ftype, fid) = iprot.readFieldBegin() + if ftype == TType.STOP: + break + if fid == 1: + if ftype == TType.STRING: + self.dbName = ( + iprot.readString().decode("utf-8", errors="replace") if sys.version_info[0] == 2 else iprot.readString() + ) + else: + iprot.skip(ftype) + elif fid == 2: + if ftype == TType.STRING: + self.tblName = ( + iprot.readString().decode("utf-8", errors="replace") if sys.version_info[0] == 2 else iprot.readString() + ) + else: + iprot.skip(ftype) + elif fid == 3: + if ftype == TType.STRUCT: + self.parts = RequestPartsSpec() + self.parts.read(iprot) + else: + iprot.skip(ftype) + elif fid == 4: + if ftype == TType.BOOL: + self.deleteData = iprot.readBool() + else: + iprot.skip(ftype) + elif fid == 5: + if ftype == TType.BOOL: + self.ifExists = iprot.readBool() + else: + iprot.skip(ftype) + elif fid == 6: + if ftype == TType.BOOL: + self.ignoreProtection = iprot.readBool() + else: + iprot.skip(ftype) + elif fid == 7: + if ftype == TType.STRUCT: + self.environmentContext = EnvironmentContext() + self.environmentContext.read(iprot) + else: + iprot.skip(ftype) + elif fid == 8: + if ftype == TType.BOOL: + self.needResult = iprot.readBool() + else: + iprot.skip(ftype) + elif fid == 9: + if ftype == TType.STRING: + self.catName = ( + iprot.readString().decode("utf-8", errors="replace") if sys.version_info[0] == 2 else iprot.readString() + ) + else: + iprot.skip(ftype) + else: + iprot.skip(ftype) + iprot.readFieldEnd() + iprot.readStructEnd() + + def write(self, oprot): + if oprot._fast_encode is not None and self.thrift_spec is not None: + oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec])) + return + oprot.writeStructBegin("DropPartitionsRequest") + if self.dbName is not None: + oprot.writeFieldBegin("dbName", TType.STRING, 1) + oprot.writeString(self.dbName.encode("utf-8") if sys.version_info[0] == 2 else self.dbName) + oprot.writeFieldEnd() + if self.tblName is not None: + oprot.writeFieldBegin("tblName", TType.STRING, 2) + oprot.writeString(self.tblName.encode("utf-8") if sys.version_info[0] == 2 else self.tblName) + oprot.writeFieldEnd() + if self.parts is not None: + oprot.writeFieldBegin("parts", TType.STRUCT, 3) + self.parts.write(oprot) + oprot.writeFieldEnd() + if self.deleteData is not None: + oprot.writeFieldBegin("deleteData", TType.BOOL, 4) + oprot.writeBool(self.deleteData) + oprot.writeFieldEnd() + if self.ifExists is not None: + oprot.writeFieldBegin("ifExists", TType.BOOL, 5) + oprot.writeBool(self.ifExists) + oprot.writeFieldEnd() + if self.ignoreProtection is not None: + oprot.writeFieldBegin("ignoreProtection", TType.BOOL, 6) + oprot.writeBool(self.ignoreProtection) + oprot.writeFieldEnd() + if self.environmentContext is not None: + oprot.writeFieldBegin("environmentContext", TType.STRUCT, 7) + self.environmentContext.write(oprot) + oprot.writeFieldEnd() + if self.needResult is not None: + oprot.writeFieldBegin("needResult", TType.BOOL, 8) + oprot.writeBool(self.needResult) + oprot.writeFieldEnd() + if self.catName is not None: + oprot.writeFieldBegin("catName", TType.STRING, 9) + oprot.writeString(self.catName.encode("utf-8") if sys.version_info[0] == 2 else self.catName) + oprot.writeFieldEnd() + oprot.writeFieldStop() + oprot.writeStructEnd() + + def validate(self): + if self.dbName is None: + raise TProtocolException(message="Required field dbName is unset!") + if self.tblName is None: + raise TProtocolException(message="Required field tblName is unset!") + if self.parts is None: + raise TProtocolException(message="Required field parts is unset!") + return + + def __repr__(self): + L = ["{}={!r}".format(key, value) for key, value in self.__dict__.items()] + return "{}({})".format(self.__class__.__name__, ", ".join(L)) + + def __eq__(self, other): + return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ + + def __ne__(self, other): + return not (self == other) + + +class PartitionValuesRequest: + """ + Attributes: + - dbName + - tblName + - partitionKeys + - applyDistinct + - filter + - partitionOrder + - ascending + - maxParts + - catName + - validWriteIdList + + """ + + def __init__( + self, + dbName=None, + tblName=None, + partitionKeys=None, + applyDistinct=True, + filter=None, + partitionOrder=None, + ascending=True, + maxParts=-1, + catName=None, + validWriteIdList=None, + ): + self.dbName = dbName + self.tblName = tblName + self.partitionKeys = partitionKeys + self.applyDistinct = applyDistinct + self.filter = filter + self.partitionOrder = partitionOrder + self.ascending = ascending + self.maxParts = maxParts + self.catName = catName + self.validWriteIdList = validWriteIdList + + def read(self, iprot): + if ( + iprot._fast_decode is not None + and isinstance(iprot.trans, TTransport.CReadableTransport) + and self.thrift_spec is not None + ): + iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec]) + return + iprot.readStructBegin() + while True: + (fname, ftype, fid) = iprot.readFieldBegin() + if ftype == TType.STOP: + break + if fid == 1: + if ftype == TType.STRING: + self.dbName = ( + iprot.readString().decode("utf-8", errors="replace") if sys.version_info[0] == 2 else iprot.readString() + ) + else: + iprot.skip(ftype) + elif fid == 2: + if ftype == TType.STRING: + self.tblName = ( + iprot.readString().decode("utf-8", errors="replace") if sys.version_info[0] == 2 else iprot.readString() + ) + else: + iprot.skip(ftype) + elif fid == 3: + if ftype == TType.LIST: + self.partitionKeys = [] + (_etype563, _size560) = iprot.readListBegin() + for _i564 in range(_size560): + _elem565 = FieldSchema() + _elem565.read(iprot) + self.partitionKeys.append(_elem565) + iprot.readListEnd() + else: + iprot.skip(ftype) + elif fid == 4: + if ftype == TType.BOOL: + self.applyDistinct = iprot.readBool() + else: + iprot.skip(ftype) + elif fid == 5: + if ftype == TType.STRING: + self.filter = ( + iprot.readString().decode("utf-8", errors="replace") if sys.version_info[0] == 2 else iprot.readString() + ) + else: + iprot.skip(ftype) + elif fid == 6: + if ftype == TType.LIST: + self.partitionOrder = [] + (_etype569, _size566) = iprot.readListBegin() + for _i570 in range(_size566): + _elem571 = FieldSchema() + _elem571.read(iprot) + self.partitionOrder.append(_elem571) + iprot.readListEnd() + else: + iprot.skip(ftype) + elif fid == 7: + if ftype == TType.BOOL: + self.ascending = iprot.readBool() + else: + iprot.skip(ftype) + elif fid == 8: + if ftype == TType.I64: + self.maxParts = iprot.readI64() + else: + iprot.skip(ftype) + elif fid == 9: + if ftype == TType.STRING: + self.catName = ( + iprot.readString().decode("utf-8", errors="replace") if sys.version_info[0] == 2 else iprot.readString() + ) + else: + iprot.skip(ftype) + elif fid == 10: + if ftype == TType.STRING: + self.validWriteIdList = ( + iprot.readString().decode("utf-8", errors="replace") if sys.version_info[0] == 2 else iprot.readString() + ) + else: + iprot.skip(ftype) + else: + iprot.skip(ftype) + iprot.readFieldEnd() + iprot.readStructEnd() + + def write(self, oprot): + if oprot._fast_encode is not None and self.thrift_spec is not None: + oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec])) + return + oprot.writeStructBegin("PartitionValuesRequest") + if self.dbName is not None: + oprot.writeFieldBegin("dbName", TType.STRING, 1) + oprot.writeString(self.dbName.encode("utf-8") if sys.version_info[0] == 2 else self.dbName) + oprot.writeFieldEnd() + if self.tblName is not None: + oprot.writeFieldBegin("tblName", TType.STRING, 2) + oprot.writeString(self.tblName.encode("utf-8") if sys.version_info[0] == 2 else self.tblName) + oprot.writeFieldEnd() + if self.partitionKeys is not None: + oprot.writeFieldBegin("partitionKeys", TType.LIST, 3) + oprot.writeListBegin(TType.STRUCT, len(self.partitionKeys)) + for iter572 in self.partitionKeys: + iter572.write(oprot) + oprot.writeListEnd() + oprot.writeFieldEnd() + if self.applyDistinct is not None: + oprot.writeFieldBegin("applyDistinct", TType.BOOL, 4) + oprot.writeBool(self.applyDistinct) + oprot.writeFieldEnd() + if self.filter is not None: + oprot.writeFieldBegin("filter", TType.STRING, 5) + oprot.writeString(self.filter.encode("utf-8") if sys.version_info[0] == 2 else self.filter) + oprot.writeFieldEnd() + if self.partitionOrder is not None: + oprot.writeFieldBegin("partitionOrder", TType.LIST, 6) + oprot.writeListBegin(TType.STRUCT, len(self.partitionOrder)) + for iter573 in self.partitionOrder: + iter573.write(oprot) + oprot.writeListEnd() + oprot.writeFieldEnd() + if self.ascending is not None: + oprot.writeFieldBegin("ascending", TType.BOOL, 7) + oprot.writeBool(self.ascending) + oprot.writeFieldEnd() + if self.maxParts is not None: + oprot.writeFieldBegin("maxParts", TType.I64, 8) + oprot.writeI64(self.maxParts) + oprot.writeFieldEnd() + if self.catName is not None: + oprot.writeFieldBegin("catName", TType.STRING, 9) + oprot.writeString(self.catName.encode("utf-8") if sys.version_info[0] == 2 else self.catName) + oprot.writeFieldEnd() + if self.validWriteIdList is not None: + oprot.writeFieldBegin("validWriteIdList", TType.STRING, 10) + oprot.writeString(self.validWriteIdList.encode("utf-8") if sys.version_info[0] == 2 else self.validWriteIdList) + oprot.writeFieldEnd() + oprot.writeFieldStop() + oprot.writeStructEnd() + + def validate(self): + if self.dbName is None: + raise TProtocolException(message="Required field dbName is unset!") + if self.tblName is None: + raise TProtocolException(message="Required field tblName is unset!") + if self.partitionKeys is None: + raise TProtocolException(message="Required field partitionKeys is unset!") + return + + def __repr__(self): + L = ["{}={!r}".format(key, value) for key, value in self.__dict__.items()] + return "{}({})".format(self.__class__.__name__, ", ".join(L)) + + def __eq__(self, other): + return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ + + def __ne__(self, other): + return not (self == other) + + +class PartitionValuesRow: + """ + Attributes: + - row + + """ + + def __init__( + self, + row=None, + ): + self.row = row + + def read(self, iprot): + if ( + iprot._fast_decode is not None + and isinstance(iprot.trans, TTransport.CReadableTransport) + and self.thrift_spec is not None + ): + iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec]) + return + iprot.readStructBegin() + while True: + (fname, ftype, fid) = iprot.readFieldBegin() + if ftype == TType.STOP: + break + if fid == 1: + if ftype == TType.LIST: + self.row = [] + (_etype577, _size574) = iprot.readListBegin() + for _i578 in range(_size574): + _elem579 = ( + iprot.readString().decode("utf-8", errors="replace") + if sys.version_info[0] == 2 + else iprot.readString() + ) + self.row.append(_elem579) + iprot.readListEnd() + else: + iprot.skip(ftype) + else: + iprot.skip(ftype) + iprot.readFieldEnd() + iprot.readStructEnd() + + def write(self, oprot): + if oprot._fast_encode is not None and self.thrift_spec is not None: + oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec])) + return + oprot.writeStructBegin("PartitionValuesRow") + if self.row is not None: + oprot.writeFieldBegin("row", TType.LIST, 1) + oprot.writeListBegin(TType.STRING, len(self.row)) + for iter580 in self.row: + oprot.writeString(iter580.encode("utf-8") if sys.version_info[0] == 2 else iter580) + oprot.writeListEnd() + oprot.writeFieldEnd() + oprot.writeFieldStop() + oprot.writeStructEnd() + + def validate(self): + if self.row is None: + raise TProtocolException(message="Required field row is unset!") + return + + def __repr__(self): + L = ["{}={!r}".format(key, value) for key, value in self.__dict__.items()] + return "{}({})".format(self.__class__.__name__, ", ".join(L)) + + def __eq__(self, other): + return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ + + def __ne__(self, other): + return not (self == other) + + +class PartitionValuesResponse: + """ + Attributes: + - partitionValues + + """ + + def __init__( + self, + partitionValues=None, + ): + self.partitionValues = partitionValues + + def read(self, iprot): + if ( + iprot._fast_decode is not None + and isinstance(iprot.trans, TTransport.CReadableTransport) + and self.thrift_spec is not None + ): + iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec]) + return + iprot.readStructBegin() + while True: + (fname, ftype, fid) = iprot.readFieldBegin() + if ftype == TType.STOP: + break + if fid == 1: + if ftype == TType.LIST: + self.partitionValues = [] + (_etype584, _size581) = iprot.readListBegin() + for _i585 in range(_size581): + _elem586 = PartitionValuesRow() + _elem586.read(iprot) + self.partitionValues.append(_elem586) + iprot.readListEnd() + else: + iprot.skip(ftype) + else: + iprot.skip(ftype) + iprot.readFieldEnd() + iprot.readStructEnd() + + def write(self, oprot): + if oprot._fast_encode is not None and self.thrift_spec is not None: + oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec])) + return + oprot.writeStructBegin("PartitionValuesResponse") + if self.partitionValues is not None: + oprot.writeFieldBegin("partitionValues", TType.LIST, 1) + oprot.writeListBegin(TType.STRUCT, len(self.partitionValues)) + for iter587 in self.partitionValues: + iter587.write(oprot) + oprot.writeListEnd() + oprot.writeFieldEnd() + oprot.writeFieldStop() + oprot.writeStructEnd() + + def validate(self): + if self.partitionValues is None: + raise TProtocolException(message="Required field partitionValues is unset!") + return + + def __repr__(self): + L = ["{}={!r}".format(key, value) for key, value in self.__dict__.items()] + return "{}({})".format(self.__class__.__name__, ", ".join(L)) + + def __eq__(self, other): + return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ + + def __ne__(self, other): + return not (self == other) + + +class GetPartitionsByNamesRequest: + """ + Attributes: + - db_name + - tbl_name + - names + - get_col_stats + - processorCapabilities + - processorIdentifier + - engine + - validWriteIdList + - getFileMetadata + - id + + """ + + def __init__( + self, + db_name=None, + tbl_name=None, + names=None, + get_col_stats=None, + processorCapabilities=None, + processorIdentifier=None, + engine=None, + validWriteIdList=None, + getFileMetadata=None, + id=-1, + ): + self.db_name = db_name + self.tbl_name = tbl_name + self.names = names + self.get_col_stats = get_col_stats + self.processorCapabilities = processorCapabilities + self.processorIdentifier = processorIdentifier + self.engine = engine + self.validWriteIdList = validWriteIdList + self.getFileMetadata = getFileMetadata + self.id = id + + def read(self, iprot): + if ( + iprot._fast_decode is not None + and isinstance(iprot.trans, TTransport.CReadableTransport) + and self.thrift_spec is not None + ): + iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec]) + return + iprot.readStructBegin() + while True: + (fname, ftype, fid) = iprot.readFieldBegin() + if ftype == TType.STOP: + break + if fid == 1: + if ftype == TType.STRING: + self.db_name = ( + iprot.readString().decode("utf-8", errors="replace") if sys.version_info[0] == 2 else iprot.readString() + ) + else: + iprot.skip(ftype) + elif fid == 2: + if ftype == TType.STRING: + self.tbl_name = ( + iprot.readString().decode("utf-8", errors="replace") if sys.version_info[0] == 2 else iprot.readString() + ) + else: + iprot.skip(ftype) + elif fid == 3: + if ftype == TType.LIST: + self.names = [] + (_etype591, _size588) = iprot.readListBegin() + for _i592 in range(_size588): + _elem593 = ( + iprot.readString().decode("utf-8", errors="replace") + if sys.version_info[0] == 2 + else iprot.readString() + ) + self.names.append(_elem593) + iprot.readListEnd() + else: + iprot.skip(ftype) + elif fid == 4: + if ftype == TType.BOOL: + self.get_col_stats = iprot.readBool() + else: + iprot.skip(ftype) + elif fid == 5: + if ftype == TType.LIST: + self.processorCapabilities = [] + (_etype597, _size594) = iprot.readListBegin() + for _i598 in range(_size594): + _elem599 = ( + iprot.readString().decode("utf-8", errors="replace") + if sys.version_info[0] == 2 + else iprot.readString() + ) + self.processorCapabilities.append(_elem599) + iprot.readListEnd() + else: + iprot.skip(ftype) + elif fid == 6: + if ftype == TType.STRING: + self.processorIdentifier = ( + iprot.readString().decode("utf-8", errors="replace") if sys.version_info[0] == 2 else iprot.readString() + ) + else: + iprot.skip(ftype) + elif fid == 7: + if ftype == TType.STRING: + self.engine = ( + iprot.readString().decode("utf-8", errors="replace") if sys.version_info[0] == 2 else iprot.readString() + ) + else: + iprot.skip(ftype) + elif fid == 8: + if ftype == TType.STRING: + self.validWriteIdList = ( + iprot.readString().decode("utf-8", errors="replace") if sys.version_info[0] == 2 else iprot.readString() + ) + else: + iprot.skip(ftype) + elif fid == 9: + if ftype == TType.BOOL: + self.getFileMetadata = iprot.readBool() + else: + iprot.skip(ftype) + elif fid == 10: + if ftype == TType.I64: + self.id = iprot.readI64() + else: + iprot.skip(ftype) + else: + iprot.skip(ftype) + iprot.readFieldEnd() + iprot.readStructEnd() + + def write(self, oprot): + if oprot._fast_encode is not None and self.thrift_spec is not None: + oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec])) + return + oprot.writeStructBegin("GetPartitionsByNamesRequest") + if self.db_name is not None: + oprot.writeFieldBegin("db_name", TType.STRING, 1) + oprot.writeString(self.db_name.encode("utf-8") if sys.version_info[0] == 2 else self.db_name) + oprot.writeFieldEnd() + if self.tbl_name is not None: + oprot.writeFieldBegin("tbl_name", TType.STRING, 2) + oprot.writeString(self.tbl_name.encode("utf-8") if sys.version_info[0] == 2 else self.tbl_name) + oprot.writeFieldEnd() + if self.names is not None: + oprot.writeFieldBegin("names", TType.LIST, 3) + oprot.writeListBegin(TType.STRING, len(self.names)) + for iter600 in self.names: + oprot.writeString(iter600.encode("utf-8") if sys.version_info[0] == 2 else iter600) + oprot.writeListEnd() + oprot.writeFieldEnd() + if self.get_col_stats is not None: + oprot.writeFieldBegin("get_col_stats", TType.BOOL, 4) + oprot.writeBool(self.get_col_stats) + oprot.writeFieldEnd() + if self.processorCapabilities is not None: + oprot.writeFieldBegin("processorCapabilities", TType.LIST, 5) + oprot.writeListBegin(TType.STRING, len(self.processorCapabilities)) + for iter601 in self.processorCapabilities: + oprot.writeString(iter601.encode("utf-8") if sys.version_info[0] == 2 else iter601) + oprot.writeListEnd() + oprot.writeFieldEnd() + if self.processorIdentifier is not None: + oprot.writeFieldBegin("processorIdentifier", TType.STRING, 6) + oprot.writeString(self.processorIdentifier.encode("utf-8") if sys.version_info[0] == 2 else self.processorIdentifier) + oprot.writeFieldEnd() + if self.engine is not None: + oprot.writeFieldBegin("engine", TType.STRING, 7) + oprot.writeString(self.engine.encode("utf-8") if sys.version_info[0] == 2 else self.engine) + oprot.writeFieldEnd() + if self.validWriteIdList is not None: + oprot.writeFieldBegin("validWriteIdList", TType.STRING, 8) + oprot.writeString(self.validWriteIdList.encode("utf-8") if sys.version_info[0] == 2 else self.validWriteIdList) + oprot.writeFieldEnd() + if self.getFileMetadata is not None: + oprot.writeFieldBegin("getFileMetadata", TType.BOOL, 9) + oprot.writeBool(self.getFileMetadata) + oprot.writeFieldEnd() + if self.id is not None: + oprot.writeFieldBegin("id", TType.I64, 10) + oprot.writeI64(self.id) + oprot.writeFieldEnd() + oprot.writeFieldStop() + oprot.writeStructEnd() + + def validate(self): + if self.db_name is None: + raise TProtocolException(message="Required field db_name is unset!") + if self.tbl_name is None: + raise TProtocolException(message="Required field tbl_name is unset!") + return + + def __repr__(self): + L = ["{}={!r}".format(key, value) for key, value in self.__dict__.items()] + return "{}({})".format(self.__class__.__name__, ", ".join(L)) + + def __eq__(self, other): + return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ + + def __ne__(self, other): + return not (self == other) + + +class GetPartitionsByNamesResult: + """ + Attributes: + - partitions + - dictionary + + """ + + def __init__( + self, + partitions=None, + dictionary=None, + ): + self.partitions = partitions + self.dictionary = dictionary + + def read(self, iprot): + if ( + iprot._fast_decode is not None + and isinstance(iprot.trans, TTransport.CReadableTransport) + and self.thrift_spec is not None + ): + iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec]) + return + iprot.readStructBegin() + while True: + (fname, ftype, fid) = iprot.readFieldBegin() + if ftype == TType.STOP: + break + if fid == 1: + if ftype == TType.LIST: + self.partitions = [] + (_etype605, _size602) = iprot.readListBegin() + for _i606 in range(_size602): + _elem607 = Partition() + _elem607.read(iprot) + self.partitions.append(_elem607) + iprot.readListEnd() + else: + iprot.skip(ftype) + elif fid == 2: + if ftype == TType.STRUCT: + self.dictionary = ObjectDictionary() + self.dictionary.read(iprot) + else: + iprot.skip(ftype) + else: + iprot.skip(ftype) + iprot.readFieldEnd() + iprot.readStructEnd() + + def write(self, oprot): + if oprot._fast_encode is not None and self.thrift_spec is not None: + oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec])) + return + oprot.writeStructBegin("GetPartitionsByNamesResult") + if self.partitions is not None: + oprot.writeFieldBegin("partitions", TType.LIST, 1) + oprot.writeListBegin(TType.STRUCT, len(self.partitions)) + for iter608 in self.partitions: + iter608.write(oprot) + oprot.writeListEnd() + oprot.writeFieldEnd() + if self.dictionary is not None: + oprot.writeFieldBegin("dictionary", TType.STRUCT, 2) + self.dictionary.write(oprot) + oprot.writeFieldEnd() + oprot.writeFieldStop() + oprot.writeStructEnd() + + def validate(self): + if self.partitions is None: + raise TProtocolException(message="Required field partitions is unset!") + return + + def __repr__(self): + L = ["{}={!r}".format(key, value) for key, value in self.__dict__.items()] + return "{}({})".format(self.__class__.__name__, ", ".join(L)) + + def __eq__(self, other): + return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ + + def __ne__(self, other): + return not (self == other) + + +class DataConnector: + """ + Attributes: + - name + - type + - url + - description + - parameters + - ownerName + - ownerType + - createTime + + """ + + def __init__( + self, + name=None, + type=None, + url=None, + description=None, + parameters=None, + ownerName=None, + ownerType=None, + createTime=None, + ): + self.name = name + self.type = type + self.url = url + self.description = description + self.parameters = parameters + self.ownerName = ownerName + self.ownerType = ownerType + self.createTime = createTime + + def read(self, iprot): + if ( + iprot._fast_decode is not None + and isinstance(iprot.trans, TTransport.CReadableTransport) + and self.thrift_spec is not None + ): + iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec]) + return + iprot.readStructBegin() + while True: + (fname, ftype, fid) = iprot.readFieldBegin() + if ftype == TType.STOP: + break + if fid == 1: + if ftype == TType.STRING: + self.name = ( + iprot.readString().decode("utf-8", errors="replace") if sys.version_info[0] == 2 else iprot.readString() + ) + else: + iprot.skip(ftype) + elif fid == 2: + if ftype == TType.STRING: + self.type = ( + iprot.readString().decode("utf-8", errors="replace") if sys.version_info[0] == 2 else iprot.readString() + ) + else: + iprot.skip(ftype) + elif fid == 3: + if ftype == TType.STRING: + self.url = ( + iprot.readString().decode("utf-8", errors="replace") if sys.version_info[0] == 2 else iprot.readString() + ) + else: + iprot.skip(ftype) + elif fid == 4: + if ftype == TType.STRING: + self.description = ( + iprot.readString().decode("utf-8", errors="replace") if sys.version_info[0] == 2 else iprot.readString() + ) + else: + iprot.skip(ftype) + elif fid == 5: + if ftype == TType.MAP: + self.parameters = {} + (_ktype610, _vtype611, _size609) = iprot.readMapBegin() + for _i613 in range(_size609): + _key614 = ( + iprot.readString().decode("utf-8", errors="replace") + if sys.version_info[0] == 2 + else iprot.readString() + ) + _val615 = ( + iprot.readString().decode("utf-8", errors="replace") + if sys.version_info[0] == 2 + else iprot.readString() + ) + self.parameters[_key614] = _val615 + iprot.readMapEnd() + else: + iprot.skip(ftype) + elif fid == 6: + if ftype == TType.STRING: + self.ownerName = ( + iprot.readString().decode("utf-8", errors="replace") if sys.version_info[0] == 2 else iprot.readString() + ) + else: + iprot.skip(ftype) + elif fid == 7: + if ftype == TType.I32: + self.ownerType = iprot.readI32() + else: + iprot.skip(ftype) + elif fid == 8: + if ftype == TType.I32: + self.createTime = iprot.readI32() + else: + iprot.skip(ftype) + else: + iprot.skip(ftype) + iprot.readFieldEnd() + iprot.readStructEnd() + + def write(self, oprot): + if oprot._fast_encode is not None and self.thrift_spec is not None: + oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec])) + return + oprot.writeStructBegin("DataConnector") + if self.name is not None: + oprot.writeFieldBegin("name", TType.STRING, 1) + oprot.writeString(self.name.encode("utf-8") if sys.version_info[0] == 2 else self.name) + oprot.writeFieldEnd() + if self.type is not None: + oprot.writeFieldBegin("type", TType.STRING, 2) + oprot.writeString(self.type.encode("utf-8") if sys.version_info[0] == 2 else self.type) + oprot.writeFieldEnd() + if self.url is not None: + oprot.writeFieldBegin("url", TType.STRING, 3) + oprot.writeString(self.url.encode("utf-8") if sys.version_info[0] == 2 else self.url) + oprot.writeFieldEnd() + if self.description is not None: + oprot.writeFieldBegin("description", TType.STRING, 4) + oprot.writeString(self.description.encode("utf-8") if sys.version_info[0] == 2 else self.description) + oprot.writeFieldEnd() + if self.parameters is not None: + oprot.writeFieldBegin("parameters", TType.MAP, 5) + oprot.writeMapBegin(TType.STRING, TType.STRING, len(self.parameters)) + for kiter616, viter617 in self.parameters.items(): + oprot.writeString(kiter616.encode("utf-8") if sys.version_info[0] == 2 else kiter616) + oprot.writeString(viter617.encode("utf-8") if sys.version_info[0] == 2 else viter617) + oprot.writeMapEnd() + oprot.writeFieldEnd() + if self.ownerName is not None: + oprot.writeFieldBegin("ownerName", TType.STRING, 6) + oprot.writeString(self.ownerName.encode("utf-8") if sys.version_info[0] == 2 else self.ownerName) + oprot.writeFieldEnd() + if self.ownerType is not None: + oprot.writeFieldBegin("ownerType", TType.I32, 7) + oprot.writeI32(self.ownerType) + oprot.writeFieldEnd() + if self.createTime is not None: + oprot.writeFieldBegin("createTime", TType.I32, 8) + oprot.writeI32(self.createTime) + oprot.writeFieldEnd() + oprot.writeFieldStop() + oprot.writeStructEnd() + + def validate(self): + return + + def __repr__(self): + L = ["{}={!r}".format(key, value) for key, value in self.__dict__.items()] + return "{}({})".format(self.__class__.__name__, ", ".join(L)) + + def __eq__(self, other): + return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ + + def __ne__(self, other): + return not (self == other) + + +class ResourceUri: + """ + Attributes: + - resourceType + - uri + + """ + + def __init__( + self, + resourceType=None, + uri=None, + ): + self.resourceType = resourceType + self.uri = uri + + def read(self, iprot): + if ( + iprot._fast_decode is not None + and isinstance(iprot.trans, TTransport.CReadableTransport) + and self.thrift_spec is not None + ): + iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec]) + return + iprot.readStructBegin() + while True: + (fname, ftype, fid) = iprot.readFieldBegin() + if ftype == TType.STOP: + break + if fid == 1: + if ftype == TType.I32: + self.resourceType = iprot.readI32() + else: + iprot.skip(ftype) + elif fid == 2: + if ftype == TType.STRING: + self.uri = ( + iprot.readString().decode("utf-8", errors="replace") if sys.version_info[0] == 2 else iprot.readString() + ) + else: + iprot.skip(ftype) + else: + iprot.skip(ftype) + iprot.readFieldEnd() + iprot.readStructEnd() + + def write(self, oprot): + if oprot._fast_encode is not None and self.thrift_spec is not None: + oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec])) + return + oprot.writeStructBegin("ResourceUri") + if self.resourceType is not None: + oprot.writeFieldBegin("resourceType", TType.I32, 1) + oprot.writeI32(self.resourceType) + oprot.writeFieldEnd() + if self.uri is not None: + oprot.writeFieldBegin("uri", TType.STRING, 2) + oprot.writeString(self.uri.encode("utf-8") if sys.version_info[0] == 2 else self.uri) + oprot.writeFieldEnd() + oprot.writeFieldStop() + oprot.writeStructEnd() + + def validate(self): + return + + def __repr__(self): + L = ["{}={!r}".format(key, value) for key, value in self.__dict__.items()] + return "{}({})".format(self.__class__.__name__, ", ".join(L)) + + def __eq__(self, other): + return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ + + def __ne__(self, other): + return not (self == other) + + +class Function: + """ + Attributes: + - functionName + - dbName + - className + - ownerName + - ownerType + - createTime + - functionType + - resourceUris + - catName + + """ + + def __init__( + self, + functionName=None, + dbName=None, + className=None, + ownerName=None, + ownerType=None, + createTime=None, + functionType=None, + resourceUris=None, + catName=None, + ): + self.functionName = functionName + self.dbName = dbName + self.className = className + self.ownerName = ownerName + self.ownerType = ownerType + self.createTime = createTime + self.functionType = functionType + self.resourceUris = resourceUris + self.catName = catName + + def read(self, iprot): + if ( + iprot._fast_decode is not None + and isinstance(iprot.trans, TTransport.CReadableTransport) + and self.thrift_spec is not None + ): + iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec]) + return + iprot.readStructBegin() + while True: + (fname, ftype, fid) = iprot.readFieldBegin() + if ftype == TType.STOP: + break + if fid == 1: + if ftype == TType.STRING: + self.functionName = ( + iprot.readString().decode("utf-8", errors="replace") if sys.version_info[0] == 2 else iprot.readString() + ) + else: + iprot.skip(ftype) + elif fid == 2: + if ftype == TType.STRING: + self.dbName = ( + iprot.readString().decode("utf-8", errors="replace") if sys.version_info[0] == 2 else iprot.readString() + ) + else: + iprot.skip(ftype) + elif fid == 3: + if ftype == TType.STRING: + self.className = ( + iprot.readString().decode("utf-8", errors="replace") if sys.version_info[0] == 2 else iprot.readString() + ) + else: + iprot.skip(ftype) + elif fid == 4: + if ftype == TType.STRING: + self.ownerName = ( + iprot.readString().decode("utf-8", errors="replace") if sys.version_info[0] == 2 else iprot.readString() + ) + else: + iprot.skip(ftype) + elif fid == 5: + if ftype == TType.I32: + self.ownerType = iprot.readI32() + else: + iprot.skip(ftype) + elif fid == 6: + if ftype == TType.I32: + self.createTime = iprot.readI32() + else: + iprot.skip(ftype) + elif fid == 7: + if ftype == TType.I32: + self.functionType = iprot.readI32() + else: + iprot.skip(ftype) + elif fid == 8: + if ftype == TType.LIST: + self.resourceUris = [] + (_etype621, _size618) = iprot.readListBegin() + for _i622 in range(_size618): + _elem623 = ResourceUri() + _elem623.read(iprot) + self.resourceUris.append(_elem623) + iprot.readListEnd() + else: + iprot.skip(ftype) + elif fid == 9: + if ftype == TType.STRING: + self.catName = ( + iprot.readString().decode("utf-8", errors="replace") if sys.version_info[0] == 2 else iprot.readString() + ) + else: + iprot.skip(ftype) + else: + iprot.skip(ftype) + iprot.readFieldEnd() + iprot.readStructEnd() + + def write(self, oprot): + if oprot._fast_encode is not None and self.thrift_spec is not None: + oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec])) + return + oprot.writeStructBegin("Function") + if self.functionName is not None: + oprot.writeFieldBegin("functionName", TType.STRING, 1) + oprot.writeString(self.functionName.encode("utf-8") if sys.version_info[0] == 2 else self.functionName) + oprot.writeFieldEnd() + if self.dbName is not None: + oprot.writeFieldBegin("dbName", TType.STRING, 2) + oprot.writeString(self.dbName.encode("utf-8") if sys.version_info[0] == 2 else self.dbName) + oprot.writeFieldEnd() + if self.className is not None: + oprot.writeFieldBegin("className", TType.STRING, 3) + oprot.writeString(self.className.encode("utf-8") if sys.version_info[0] == 2 else self.className) + oprot.writeFieldEnd() + if self.ownerName is not None: + oprot.writeFieldBegin("ownerName", TType.STRING, 4) + oprot.writeString(self.ownerName.encode("utf-8") if sys.version_info[0] == 2 else self.ownerName) + oprot.writeFieldEnd() + if self.ownerType is not None: + oprot.writeFieldBegin("ownerType", TType.I32, 5) + oprot.writeI32(self.ownerType) + oprot.writeFieldEnd() + if self.createTime is not None: + oprot.writeFieldBegin("createTime", TType.I32, 6) + oprot.writeI32(self.createTime) + oprot.writeFieldEnd() + if self.functionType is not None: + oprot.writeFieldBegin("functionType", TType.I32, 7) + oprot.writeI32(self.functionType) + oprot.writeFieldEnd() + if self.resourceUris is not None: + oprot.writeFieldBegin("resourceUris", TType.LIST, 8) + oprot.writeListBegin(TType.STRUCT, len(self.resourceUris)) + for iter624 in self.resourceUris: + iter624.write(oprot) + oprot.writeListEnd() + oprot.writeFieldEnd() + if self.catName is not None: + oprot.writeFieldBegin("catName", TType.STRING, 9) + oprot.writeString(self.catName.encode("utf-8") if sys.version_info[0] == 2 else self.catName) + oprot.writeFieldEnd() + oprot.writeFieldStop() + oprot.writeStructEnd() + + def validate(self): + return + + def __repr__(self): + L = ["{}={!r}".format(key, value) for key, value in self.__dict__.items()] + return "{}({})".format(self.__class__.__name__, ", ".join(L)) + + def __eq__(self, other): + return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ + + def __ne__(self, other): + return not (self == other) + + +class TxnInfo: + """ + Attributes: + - id + - state + - user + - hostname + - agentInfo + - heartbeatCount + - metaInfo + - startedTime + - lastHeartbeatTime + + """ + + def __init__( + self, + id=None, + state=None, + user=None, + hostname=None, + agentInfo="Unknown", + heartbeatCount=0, + metaInfo=None, + startedTime=None, + lastHeartbeatTime=None, + ): + self.id = id + self.state = state + self.user = user + self.hostname = hostname + self.agentInfo = agentInfo + self.heartbeatCount = heartbeatCount + self.metaInfo = metaInfo + self.startedTime = startedTime + self.lastHeartbeatTime = lastHeartbeatTime + + def read(self, iprot): + if ( + iprot._fast_decode is not None + and isinstance(iprot.trans, TTransport.CReadableTransport) + and self.thrift_spec is not None + ): + iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec]) + return + iprot.readStructBegin() + while True: + (fname, ftype, fid) = iprot.readFieldBegin() + if ftype == TType.STOP: + break + if fid == 1: + if ftype == TType.I64: + self.id = iprot.readI64() + else: + iprot.skip(ftype) + elif fid == 2: + if ftype == TType.I32: + self.state = iprot.readI32() + else: + iprot.skip(ftype) + elif fid == 3: + if ftype == TType.STRING: + self.user = ( + iprot.readString().decode("utf-8", errors="replace") if sys.version_info[0] == 2 else iprot.readString() + ) + else: + iprot.skip(ftype) + elif fid == 4: + if ftype == TType.STRING: + self.hostname = ( + iprot.readString().decode("utf-8", errors="replace") if sys.version_info[0] == 2 else iprot.readString() + ) + else: + iprot.skip(ftype) + elif fid == 5: + if ftype == TType.STRING: + self.agentInfo = ( + iprot.readString().decode("utf-8", errors="replace") if sys.version_info[0] == 2 else iprot.readString() + ) + else: + iprot.skip(ftype) + elif fid == 6: + if ftype == TType.I32: + self.heartbeatCount = iprot.readI32() + else: + iprot.skip(ftype) + elif fid == 7: + if ftype == TType.STRING: + self.metaInfo = ( + iprot.readString().decode("utf-8", errors="replace") if sys.version_info[0] == 2 else iprot.readString() + ) + else: + iprot.skip(ftype) + elif fid == 8: + if ftype == TType.I64: + self.startedTime = iprot.readI64() + else: + iprot.skip(ftype) + elif fid == 9: + if ftype == TType.I64: + self.lastHeartbeatTime = iprot.readI64() + else: + iprot.skip(ftype) + else: + iprot.skip(ftype) + iprot.readFieldEnd() + iprot.readStructEnd() + + def write(self, oprot): + if oprot._fast_encode is not None and self.thrift_spec is not None: + oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec])) + return + oprot.writeStructBegin("TxnInfo") + if self.id is not None: + oprot.writeFieldBegin("id", TType.I64, 1) + oprot.writeI64(self.id) + oprot.writeFieldEnd() + if self.state is not None: + oprot.writeFieldBegin("state", TType.I32, 2) + oprot.writeI32(self.state) + oprot.writeFieldEnd() + if self.user is not None: + oprot.writeFieldBegin("user", TType.STRING, 3) + oprot.writeString(self.user.encode("utf-8") if sys.version_info[0] == 2 else self.user) + oprot.writeFieldEnd() + if self.hostname is not None: + oprot.writeFieldBegin("hostname", TType.STRING, 4) + oprot.writeString(self.hostname.encode("utf-8") if sys.version_info[0] == 2 else self.hostname) + oprot.writeFieldEnd() + if self.agentInfo is not None: + oprot.writeFieldBegin("agentInfo", TType.STRING, 5) + oprot.writeString(self.agentInfo.encode("utf-8") if sys.version_info[0] == 2 else self.agentInfo) + oprot.writeFieldEnd() + if self.heartbeatCount is not None: + oprot.writeFieldBegin("heartbeatCount", TType.I32, 6) + oprot.writeI32(self.heartbeatCount) + oprot.writeFieldEnd() + if self.metaInfo is not None: + oprot.writeFieldBegin("metaInfo", TType.STRING, 7) + oprot.writeString(self.metaInfo.encode("utf-8") if sys.version_info[0] == 2 else self.metaInfo) + oprot.writeFieldEnd() + if self.startedTime is not None: + oprot.writeFieldBegin("startedTime", TType.I64, 8) + oprot.writeI64(self.startedTime) + oprot.writeFieldEnd() + if self.lastHeartbeatTime is not None: + oprot.writeFieldBegin("lastHeartbeatTime", TType.I64, 9) + oprot.writeI64(self.lastHeartbeatTime) + oprot.writeFieldEnd() + oprot.writeFieldStop() + oprot.writeStructEnd() + + def validate(self): + if self.id is None: + raise TProtocolException(message="Required field id is unset!") + if self.state is None: + raise TProtocolException(message="Required field state is unset!") + if self.user is None: + raise TProtocolException(message="Required field user is unset!") + if self.hostname is None: + raise TProtocolException(message="Required field hostname is unset!") + return + + def __repr__(self): + L = ["{}={!r}".format(key, value) for key, value in self.__dict__.items()] + return "{}({})".format(self.__class__.__name__, ", ".join(L)) + + def __eq__(self, other): + return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ + + def __ne__(self, other): + return not (self == other) + + +class GetOpenTxnsInfoResponse: + """ + Attributes: + - txn_high_water_mark + - open_txns + + """ + + def __init__( + self, + txn_high_water_mark=None, + open_txns=None, + ): + self.txn_high_water_mark = txn_high_water_mark + self.open_txns = open_txns + + def read(self, iprot): + if ( + iprot._fast_decode is not None + and isinstance(iprot.trans, TTransport.CReadableTransport) + and self.thrift_spec is not None + ): + iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec]) + return + iprot.readStructBegin() + while True: + (fname, ftype, fid) = iprot.readFieldBegin() + if ftype == TType.STOP: + break + if fid == 1: + if ftype == TType.I64: + self.txn_high_water_mark = iprot.readI64() + else: + iprot.skip(ftype) + elif fid == 2: + if ftype == TType.LIST: + self.open_txns = [] + (_etype628, _size625) = iprot.readListBegin() + for _i629 in range(_size625): + _elem630 = TxnInfo() + _elem630.read(iprot) + self.open_txns.append(_elem630) + iprot.readListEnd() + else: + iprot.skip(ftype) + else: + iprot.skip(ftype) + iprot.readFieldEnd() + iprot.readStructEnd() + + def write(self, oprot): + if oprot._fast_encode is not None and self.thrift_spec is not None: + oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec])) + return + oprot.writeStructBegin("GetOpenTxnsInfoResponse") + if self.txn_high_water_mark is not None: + oprot.writeFieldBegin("txn_high_water_mark", TType.I64, 1) + oprot.writeI64(self.txn_high_water_mark) + oprot.writeFieldEnd() + if self.open_txns is not None: + oprot.writeFieldBegin("open_txns", TType.LIST, 2) + oprot.writeListBegin(TType.STRUCT, len(self.open_txns)) + for iter631 in self.open_txns: + iter631.write(oprot) + oprot.writeListEnd() + oprot.writeFieldEnd() + oprot.writeFieldStop() + oprot.writeStructEnd() + + def validate(self): + if self.txn_high_water_mark is None: + raise TProtocolException(message="Required field txn_high_water_mark is unset!") + if self.open_txns is None: + raise TProtocolException(message="Required field open_txns is unset!") + return + + def __repr__(self): + L = ["{}={!r}".format(key, value) for key, value in self.__dict__.items()] + return "{}({})".format(self.__class__.__name__, ", ".join(L)) + + def __eq__(self, other): + return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ + + def __ne__(self, other): + return not (self == other) + + +class GetOpenTxnsResponse: + """ + Attributes: + - txn_high_water_mark + - open_txns + - min_open_txn + - abortedBits + + """ + + def __init__( + self, + txn_high_water_mark=None, + open_txns=None, + min_open_txn=None, + abortedBits=None, + ): + self.txn_high_water_mark = txn_high_water_mark + self.open_txns = open_txns + self.min_open_txn = min_open_txn + self.abortedBits = abortedBits + + def read(self, iprot): + if ( + iprot._fast_decode is not None + and isinstance(iprot.trans, TTransport.CReadableTransport) + and self.thrift_spec is not None + ): + iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec]) + return + iprot.readStructBegin() + while True: + (fname, ftype, fid) = iprot.readFieldBegin() + if ftype == TType.STOP: + break + if fid == 1: + if ftype == TType.I64: + self.txn_high_water_mark = iprot.readI64() + else: + iprot.skip(ftype) + elif fid == 2: + if ftype == TType.LIST: + self.open_txns = [] + (_etype635, _size632) = iprot.readListBegin() + for _i636 in range(_size632): + _elem637 = iprot.readI64() + self.open_txns.append(_elem637) + iprot.readListEnd() + else: + iprot.skip(ftype) + elif fid == 3: + if ftype == TType.I64: + self.min_open_txn = iprot.readI64() + else: + iprot.skip(ftype) + elif fid == 4: + if ftype == TType.STRING: + self.abortedBits = iprot.readBinary() + else: + iprot.skip(ftype) + else: + iprot.skip(ftype) + iprot.readFieldEnd() + iprot.readStructEnd() + + def write(self, oprot): + if oprot._fast_encode is not None and self.thrift_spec is not None: + oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec])) + return + oprot.writeStructBegin("GetOpenTxnsResponse") + if self.txn_high_water_mark is not None: + oprot.writeFieldBegin("txn_high_water_mark", TType.I64, 1) + oprot.writeI64(self.txn_high_water_mark) + oprot.writeFieldEnd() + if self.open_txns is not None: + oprot.writeFieldBegin("open_txns", TType.LIST, 2) + oprot.writeListBegin(TType.I64, len(self.open_txns)) + for iter638 in self.open_txns: + oprot.writeI64(iter638) + oprot.writeListEnd() + oprot.writeFieldEnd() + if self.min_open_txn is not None: + oprot.writeFieldBegin("min_open_txn", TType.I64, 3) + oprot.writeI64(self.min_open_txn) + oprot.writeFieldEnd() + if self.abortedBits is not None: + oprot.writeFieldBegin("abortedBits", TType.STRING, 4) + oprot.writeBinary(self.abortedBits) + oprot.writeFieldEnd() + oprot.writeFieldStop() + oprot.writeStructEnd() + + def validate(self): + if self.txn_high_water_mark is None: + raise TProtocolException(message="Required field txn_high_water_mark is unset!") + if self.open_txns is None: + raise TProtocolException(message="Required field open_txns is unset!") + if self.abortedBits is None: + raise TProtocolException(message="Required field abortedBits is unset!") + return + + def __repr__(self): + L = ["{}={!r}".format(key, value) for key, value in self.__dict__.items()] + return "{}({})".format(self.__class__.__name__, ", ".join(L)) + + def __eq__(self, other): + return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ + + def __ne__(self, other): + return not (self == other) + + +class OpenTxnRequest: + """ + Attributes: + - num_txns + - user + - hostname + - agentInfo + - replPolicy + - replSrcTxnIds + - txn_type + + """ + + def __init__( + self, + num_txns=None, + user=None, + hostname=None, + agentInfo="Unknown", + replPolicy=None, + replSrcTxnIds=None, + txn_type=0, + ): + self.num_txns = num_txns + self.user = user + self.hostname = hostname + self.agentInfo = agentInfo + self.replPolicy = replPolicy + self.replSrcTxnIds = replSrcTxnIds + self.txn_type = txn_type + + def read(self, iprot): + if ( + iprot._fast_decode is not None + and isinstance(iprot.trans, TTransport.CReadableTransport) + and self.thrift_spec is not None + ): + iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec]) + return + iprot.readStructBegin() + while True: + (fname, ftype, fid) = iprot.readFieldBegin() + if ftype == TType.STOP: + break + if fid == 1: + if ftype == TType.I32: + self.num_txns = iprot.readI32() + else: + iprot.skip(ftype) + elif fid == 2: + if ftype == TType.STRING: + self.user = ( + iprot.readString().decode("utf-8", errors="replace") if sys.version_info[0] == 2 else iprot.readString() + ) + else: + iprot.skip(ftype) + elif fid == 3: + if ftype == TType.STRING: + self.hostname = ( + iprot.readString().decode("utf-8", errors="replace") if sys.version_info[0] == 2 else iprot.readString() + ) + else: + iprot.skip(ftype) + elif fid == 4: + if ftype == TType.STRING: + self.agentInfo = ( + iprot.readString().decode("utf-8", errors="replace") if sys.version_info[0] == 2 else iprot.readString() + ) + else: + iprot.skip(ftype) + elif fid == 5: + if ftype == TType.STRING: + self.replPolicy = ( + iprot.readString().decode("utf-8", errors="replace") if sys.version_info[0] == 2 else iprot.readString() + ) + else: + iprot.skip(ftype) + elif fid == 6: + if ftype == TType.LIST: + self.replSrcTxnIds = [] + (_etype642, _size639) = iprot.readListBegin() + for _i643 in range(_size639): + _elem644 = iprot.readI64() + self.replSrcTxnIds.append(_elem644) + iprot.readListEnd() + else: + iprot.skip(ftype) + elif fid == 7: + if ftype == TType.I32: + self.txn_type = iprot.readI32() + else: + iprot.skip(ftype) + else: + iprot.skip(ftype) + iprot.readFieldEnd() + iprot.readStructEnd() + + def write(self, oprot): + if oprot._fast_encode is not None and self.thrift_spec is not None: + oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec])) + return + oprot.writeStructBegin("OpenTxnRequest") + if self.num_txns is not None: + oprot.writeFieldBegin("num_txns", TType.I32, 1) + oprot.writeI32(self.num_txns) + oprot.writeFieldEnd() + if self.user is not None: + oprot.writeFieldBegin("user", TType.STRING, 2) + oprot.writeString(self.user.encode("utf-8") if sys.version_info[0] == 2 else self.user) + oprot.writeFieldEnd() + if self.hostname is not None: + oprot.writeFieldBegin("hostname", TType.STRING, 3) + oprot.writeString(self.hostname.encode("utf-8") if sys.version_info[0] == 2 else self.hostname) + oprot.writeFieldEnd() + if self.agentInfo is not None: + oprot.writeFieldBegin("agentInfo", TType.STRING, 4) + oprot.writeString(self.agentInfo.encode("utf-8") if sys.version_info[0] == 2 else self.agentInfo) + oprot.writeFieldEnd() + if self.replPolicy is not None: + oprot.writeFieldBegin("replPolicy", TType.STRING, 5) + oprot.writeString(self.replPolicy.encode("utf-8") if sys.version_info[0] == 2 else self.replPolicy) + oprot.writeFieldEnd() + if self.replSrcTxnIds is not None: + oprot.writeFieldBegin("replSrcTxnIds", TType.LIST, 6) + oprot.writeListBegin(TType.I64, len(self.replSrcTxnIds)) + for iter645 in self.replSrcTxnIds: + oprot.writeI64(iter645) + oprot.writeListEnd() + oprot.writeFieldEnd() + if self.txn_type is not None: + oprot.writeFieldBegin("txn_type", TType.I32, 7) + oprot.writeI32(self.txn_type) + oprot.writeFieldEnd() + oprot.writeFieldStop() + oprot.writeStructEnd() + + def validate(self): + if self.num_txns is None: + raise TProtocolException(message="Required field num_txns is unset!") + if self.user is None: + raise TProtocolException(message="Required field user is unset!") + if self.hostname is None: + raise TProtocolException(message="Required field hostname is unset!") + return + + def __repr__(self): + L = ["{}={!r}".format(key, value) for key, value in self.__dict__.items()] + return "{}({})".format(self.__class__.__name__, ", ".join(L)) + + def __eq__(self, other): + return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ + + def __ne__(self, other): + return not (self == other) + + +class OpenTxnsResponse: + """ + Attributes: + - txn_ids + + """ + + def __init__( + self, + txn_ids=None, + ): + self.txn_ids = txn_ids + + def read(self, iprot): + if ( + iprot._fast_decode is not None + and isinstance(iprot.trans, TTransport.CReadableTransport) + and self.thrift_spec is not None + ): + iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec]) + return + iprot.readStructBegin() + while True: + (fname, ftype, fid) = iprot.readFieldBegin() + if ftype == TType.STOP: + break + if fid == 1: + if ftype == TType.LIST: + self.txn_ids = [] + (_etype649, _size646) = iprot.readListBegin() + for _i650 in range(_size646): + _elem651 = iprot.readI64() + self.txn_ids.append(_elem651) + iprot.readListEnd() + else: + iprot.skip(ftype) + else: + iprot.skip(ftype) + iprot.readFieldEnd() + iprot.readStructEnd() + + def write(self, oprot): + if oprot._fast_encode is not None and self.thrift_spec is not None: + oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec])) + return + oprot.writeStructBegin("OpenTxnsResponse") + if self.txn_ids is not None: + oprot.writeFieldBegin("txn_ids", TType.LIST, 1) + oprot.writeListBegin(TType.I64, len(self.txn_ids)) + for iter652 in self.txn_ids: + oprot.writeI64(iter652) + oprot.writeListEnd() + oprot.writeFieldEnd() + oprot.writeFieldStop() + oprot.writeStructEnd() + + def validate(self): + if self.txn_ids is None: + raise TProtocolException(message="Required field txn_ids is unset!") + return + + def __repr__(self): + L = ["{}={!r}".format(key, value) for key, value in self.__dict__.items()] + return "{}({})".format(self.__class__.__name__, ", ".join(L)) + + def __eq__(self, other): + return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ + + def __ne__(self, other): + return not (self == other) + + +class AbortTxnRequest: + """ + Attributes: + - txnid + - replPolicy + - txn_type + + """ + + def __init__( + self, + txnid=None, + replPolicy=None, + txn_type=None, + ): + self.txnid = txnid + self.replPolicy = replPolicy + self.txn_type = txn_type + + def read(self, iprot): + if ( + iprot._fast_decode is not None + and isinstance(iprot.trans, TTransport.CReadableTransport) + and self.thrift_spec is not None + ): + iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec]) + return + iprot.readStructBegin() + while True: + (fname, ftype, fid) = iprot.readFieldBegin() + if ftype == TType.STOP: + break + if fid == 1: + if ftype == TType.I64: + self.txnid = iprot.readI64() + else: + iprot.skip(ftype) + elif fid == 2: + if ftype == TType.STRING: + self.replPolicy = ( + iprot.readString().decode("utf-8", errors="replace") if sys.version_info[0] == 2 else iprot.readString() + ) + else: + iprot.skip(ftype) + elif fid == 3: + if ftype == TType.I32: + self.txn_type = iprot.readI32() + else: + iprot.skip(ftype) + else: + iprot.skip(ftype) + iprot.readFieldEnd() + iprot.readStructEnd() + + def write(self, oprot): + if oprot._fast_encode is not None and self.thrift_spec is not None: + oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec])) + return + oprot.writeStructBegin("AbortTxnRequest") + if self.txnid is not None: + oprot.writeFieldBegin("txnid", TType.I64, 1) + oprot.writeI64(self.txnid) + oprot.writeFieldEnd() + if self.replPolicy is not None: + oprot.writeFieldBegin("replPolicy", TType.STRING, 2) + oprot.writeString(self.replPolicy.encode("utf-8") if sys.version_info[0] == 2 else self.replPolicy) + oprot.writeFieldEnd() + if self.txn_type is not None: + oprot.writeFieldBegin("txn_type", TType.I32, 3) + oprot.writeI32(self.txn_type) + oprot.writeFieldEnd() + oprot.writeFieldStop() + oprot.writeStructEnd() + + def validate(self): + if self.txnid is None: + raise TProtocolException(message="Required field txnid is unset!") + return + + def __repr__(self): + L = ["{}={!r}".format(key, value) for key, value in self.__dict__.items()] + return "{}({})".format(self.__class__.__name__, ", ".join(L)) + + def __eq__(self, other): + return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ + + def __ne__(self, other): + return not (self == other) + + +class AbortTxnsRequest: + """ + Attributes: + - txn_ids + + """ + + def __init__( + self, + txn_ids=None, + ): + self.txn_ids = txn_ids + + def read(self, iprot): + if ( + iprot._fast_decode is not None + and isinstance(iprot.trans, TTransport.CReadableTransport) + and self.thrift_spec is not None + ): + iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec]) + return + iprot.readStructBegin() + while True: + (fname, ftype, fid) = iprot.readFieldBegin() + if ftype == TType.STOP: + break + if fid == 1: + if ftype == TType.LIST: + self.txn_ids = [] + (_etype656, _size653) = iprot.readListBegin() + for _i657 in range(_size653): + _elem658 = iprot.readI64() + self.txn_ids.append(_elem658) + iprot.readListEnd() + else: + iprot.skip(ftype) + else: + iprot.skip(ftype) + iprot.readFieldEnd() + iprot.readStructEnd() + + def write(self, oprot): + if oprot._fast_encode is not None and self.thrift_spec is not None: + oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec])) + return + oprot.writeStructBegin("AbortTxnsRequest") + if self.txn_ids is not None: + oprot.writeFieldBegin("txn_ids", TType.LIST, 1) + oprot.writeListBegin(TType.I64, len(self.txn_ids)) + for iter659 in self.txn_ids: + oprot.writeI64(iter659) + oprot.writeListEnd() + oprot.writeFieldEnd() + oprot.writeFieldStop() + oprot.writeStructEnd() + + def validate(self): + if self.txn_ids is None: + raise TProtocolException(message="Required field txn_ids is unset!") + return + + def __repr__(self): + L = ["{}={!r}".format(key, value) for key, value in self.__dict__.items()] + return "{}({})".format(self.__class__.__name__, ", ".join(L)) + + def __eq__(self, other): + return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ + + def __ne__(self, other): + return not (self == other) + + +class CommitTxnKeyValue: + """ + Attributes: + - tableId + - key + - value + + """ + + def __init__( + self, + tableId=None, + key=None, + value=None, + ): + self.tableId = tableId + self.key = key + self.value = value + + def read(self, iprot): + if ( + iprot._fast_decode is not None + and isinstance(iprot.trans, TTransport.CReadableTransport) + and self.thrift_spec is not None + ): + iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec]) + return + iprot.readStructBegin() + while True: + (fname, ftype, fid) = iprot.readFieldBegin() + if ftype == TType.STOP: + break + if fid == 1: + if ftype == TType.I64: + self.tableId = iprot.readI64() + else: + iprot.skip(ftype) + elif fid == 2: + if ftype == TType.STRING: + self.key = ( + iprot.readString().decode("utf-8", errors="replace") if sys.version_info[0] == 2 else iprot.readString() + ) + else: + iprot.skip(ftype) + elif fid == 3: + if ftype == TType.STRING: + self.value = ( + iprot.readString().decode("utf-8", errors="replace") if sys.version_info[0] == 2 else iprot.readString() + ) + else: + iprot.skip(ftype) + else: + iprot.skip(ftype) + iprot.readFieldEnd() + iprot.readStructEnd() + + def write(self, oprot): + if oprot._fast_encode is not None and self.thrift_spec is not None: + oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec])) + return + oprot.writeStructBegin("CommitTxnKeyValue") + if self.tableId is not None: + oprot.writeFieldBegin("tableId", TType.I64, 1) + oprot.writeI64(self.tableId) + oprot.writeFieldEnd() + if self.key is not None: + oprot.writeFieldBegin("key", TType.STRING, 2) + oprot.writeString(self.key.encode("utf-8") if sys.version_info[0] == 2 else self.key) + oprot.writeFieldEnd() + if self.value is not None: + oprot.writeFieldBegin("value", TType.STRING, 3) + oprot.writeString(self.value.encode("utf-8") if sys.version_info[0] == 2 else self.value) + oprot.writeFieldEnd() + oprot.writeFieldStop() + oprot.writeStructEnd() + + def validate(self): + if self.tableId is None: + raise TProtocolException(message="Required field tableId is unset!") + if self.key is None: + raise TProtocolException(message="Required field key is unset!") + if self.value is None: + raise TProtocolException(message="Required field value is unset!") + return + + def __repr__(self): + L = ["{}={!r}".format(key, value) for key, value in self.__dict__.items()] + return "{}({})".format(self.__class__.__name__, ", ".join(L)) + + def __eq__(self, other): + return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ + + def __ne__(self, other): + return not (self == other) + + +class WriteEventInfo: + """ + Attributes: + - writeId + - database + - table + - files + - partition + - tableObj + - partitionObj + + """ + + def __init__( + self, + writeId=None, + database=None, + table=None, + files=None, + partition=None, + tableObj=None, + partitionObj=None, + ): + self.writeId = writeId + self.database = database + self.table = table + self.files = files + self.partition = partition + self.tableObj = tableObj + self.partitionObj = partitionObj + + def read(self, iprot): + if ( + iprot._fast_decode is not None + and isinstance(iprot.trans, TTransport.CReadableTransport) + and self.thrift_spec is not None + ): + iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec]) + return + iprot.readStructBegin() + while True: + (fname, ftype, fid) = iprot.readFieldBegin() + if ftype == TType.STOP: + break + if fid == 1: + if ftype == TType.I64: + self.writeId = iprot.readI64() + else: + iprot.skip(ftype) + elif fid == 2: + if ftype == TType.STRING: + self.database = ( + iprot.readString().decode("utf-8", errors="replace") if sys.version_info[0] == 2 else iprot.readString() + ) + else: + iprot.skip(ftype) + elif fid == 3: + if ftype == TType.STRING: + self.table = ( + iprot.readString().decode("utf-8", errors="replace") if sys.version_info[0] == 2 else iprot.readString() + ) + else: + iprot.skip(ftype) + elif fid == 4: + if ftype == TType.STRING: + self.files = ( + iprot.readString().decode("utf-8", errors="replace") if sys.version_info[0] == 2 else iprot.readString() + ) + else: + iprot.skip(ftype) + elif fid == 5: + if ftype == TType.STRING: + self.partition = ( + iprot.readString().decode("utf-8", errors="replace") if sys.version_info[0] == 2 else iprot.readString() + ) + else: + iprot.skip(ftype) + elif fid == 6: + if ftype == TType.STRING: + self.tableObj = ( + iprot.readString().decode("utf-8", errors="replace") if sys.version_info[0] == 2 else iprot.readString() + ) + else: + iprot.skip(ftype) + elif fid == 7: + if ftype == TType.STRING: + self.partitionObj = ( + iprot.readString().decode("utf-8", errors="replace") if sys.version_info[0] == 2 else iprot.readString() + ) + else: + iprot.skip(ftype) + else: + iprot.skip(ftype) + iprot.readFieldEnd() + iprot.readStructEnd() + + def write(self, oprot): + if oprot._fast_encode is not None and self.thrift_spec is not None: + oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec])) + return + oprot.writeStructBegin("WriteEventInfo") + if self.writeId is not None: + oprot.writeFieldBegin("writeId", TType.I64, 1) + oprot.writeI64(self.writeId) + oprot.writeFieldEnd() + if self.database is not None: + oprot.writeFieldBegin("database", TType.STRING, 2) + oprot.writeString(self.database.encode("utf-8") if sys.version_info[0] == 2 else self.database) + oprot.writeFieldEnd() + if self.table is not None: + oprot.writeFieldBegin("table", TType.STRING, 3) + oprot.writeString(self.table.encode("utf-8") if sys.version_info[0] == 2 else self.table) + oprot.writeFieldEnd() + if self.files is not None: + oprot.writeFieldBegin("files", TType.STRING, 4) + oprot.writeString(self.files.encode("utf-8") if sys.version_info[0] == 2 else self.files) + oprot.writeFieldEnd() + if self.partition is not None: + oprot.writeFieldBegin("partition", TType.STRING, 5) + oprot.writeString(self.partition.encode("utf-8") if sys.version_info[0] == 2 else self.partition) + oprot.writeFieldEnd() + if self.tableObj is not None: + oprot.writeFieldBegin("tableObj", TType.STRING, 6) + oprot.writeString(self.tableObj.encode("utf-8") if sys.version_info[0] == 2 else self.tableObj) + oprot.writeFieldEnd() + if self.partitionObj is not None: + oprot.writeFieldBegin("partitionObj", TType.STRING, 7) + oprot.writeString(self.partitionObj.encode("utf-8") if sys.version_info[0] == 2 else self.partitionObj) + oprot.writeFieldEnd() + oprot.writeFieldStop() + oprot.writeStructEnd() + + def validate(self): + if self.writeId is None: + raise TProtocolException(message="Required field writeId is unset!") + if self.database is None: + raise TProtocolException(message="Required field database is unset!") + if self.table is None: + raise TProtocolException(message="Required field table is unset!") + if self.files is None: + raise TProtocolException(message="Required field files is unset!") + return + + def __repr__(self): + L = ["{}={!r}".format(key, value) for key, value in self.__dict__.items()] + return "{}({})".format(self.__class__.__name__, ", ".join(L)) + + def __eq__(self, other): + return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ + + def __ne__(self, other): + return not (self == other) + + +class ReplLastIdInfo: + """ + Attributes: + - database + - lastReplId + - table + - catalog + - partitionList + + """ + + def __init__( + self, + database=None, + lastReplId=None, + table=None, + catalog=None, + partitionList=None, + ): + self.database = database + self.lastReplId = lastReplId + self.table = table + self.catalog = catalog + self.partitionList = partitionList + + def read(self, iprot): + if ( + iprot._fast_decode is not None + and isinstance(iprot.trans, TTransport.CReadableTransport) + and self.thrift_spec is not None + ): + iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec]) + return + iprot.readStructBegin() + while True: + (fname, ftype, fid) = iprot.readFieldBegin() + if ftype == TType.STOP: + break + if fid == 1: + if ftype == TType.STRING: + self.database = ( + iprot.readString().decode("utf-8", errors="replace") if sys.version_info[0] == 2 else iprot.readString() + ) + else: + iprot.skip(ftype) + elif fid == 2: + if ftype == TType.I64: + self.lastReplId = iprot.readI64() + else: + iprot.skip(ftype) + elif fid == 3: + if ftype == TType.STRING: + self.table = ( + iprot.readString().decode("utf-8", errors="replace") if sys.version_info[0] == 2 else iprot.readString() + ) + else: + iprot.skip(ftype) + elif fid == 4: + if ftype == TType.STRING: + self.catalog = ( + iprot.readString().decode("utf-8", errors="replace") if sys.version_info[0] == 2 else iprot.readString() + ) + else: + iprot.skip(ftype) + elif fid == 5: + if ftype == TType.LIST: + self.partitionList = [] + (_etype663, _size660) = iprot.readListBegin() + for _i664 in range(_size660): + _elem665 = ( + iprot.readString().decode("utf-8", errors="replace") + if sys.version_info[0] == 2 + else iprot.readString() + ) + self.partitionList.append(_elem665) + iprot.readListEnd() + else: + iprot.skip(ftype) + else: + iprot.skip(ftype) + iprot.readFieldEnd() + iprot.readStructEnd() + + def write(self, oprot): + if oprot._fast_encode is not None and self.thrift_spec is not None: + oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec])) + return + oprot.writeStructBegin("ReplLastIdInfo") + if self.database is not None: + oprot.writeFieldBegin("database", TType.STRING, 1) + oprot.writeString(self.database.encode("utf-8") if sys.version_info[0] == 2 else self.database) + oprot.writeFieldEnd() + if self.lastReplId is not None: + oprot.writeFieldBegin("lastReplId", TType.I64, 2) + oprot.writeI64(self.lastReplId) + oprot.writeFieldEnd() + if self.table is not None: + oprot.writeFieldBegin("table", TType.STRING, 3) + oprot.writeString(self.table.encode("utf-8") if sys.version_info[0] == 2 else self.table) + oprot.writeFieldEnd() + if self.catalog is not None: + oprot.writeFieldBegin("catalog", TType.STRING, 4) + oprot.writeString(self.catalog.encode("utf-8") if sys.version_info[0] == 2 else self.catalog) + oprot.writeFieldEnd() + if self.partitionList is not None: + oprot.writeFieldBegin("partitionList", TType.LIST, 5) + oprot.writeListBegin(TType.STRING, len(self.partitionList)) + for iter666 in self.partitionList: + oprot.writeString(iter666.encode("utf-8") if sys.version_info[0] == 2 else iter666) + oprot.writeListEnd() + oprot.writeFieldEnd() + oprot.writeFieldStop() + oprot.writeStructEnd() + + def validate(self): + if self.database is None: + raise TProtocolException(message="Required field database is unset!") + if self.lastReplId is None: + raise TProtocolException(message="Required field lastReplId is unset!") + return + + def __repr__(self): + L = ["{}={!r}".format(key, value) for key, value in self.__dict__.items()] + return "{}({})".format(self.__class__.__name__, ", ".join(L)) + + def __eq__(self, other): + return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ + + def __ne__(self, other): + return not (self == other) + + +class UpdateTransactionalStatsRequest: + """ + Attributes: + - tableId + - insertCount + - updatedCount + - deletedCount + + """ + + def __init__( + self, + tableId=None, + insertCount=None, + updatedCount=None, + deletedCount=None, + ): + self.tableId = tableId + self.insertCount = insertCount + self.updatedCount = updatedCount + self.deletedCount = deletedCount + + def read(self, iprot): + if ( + iprot._fast_decode is not None + and isinstance(iprot.trans, TTransport.CReadableTransport) + and self.thrift_spec is not None + ): + iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec]) + return + iprot.readStructBegin() + while True: + (fname, ftype, fid) = iprot.readFieldBegin() + if ftype == TType.STOP: + break + if fid == 1: + if ftype == TType.I64: + self.tableId = iprot.readI64() + else: + iprot.skip(ftype) + elif fid == 2: + if ftype == TType.I64: + self.insertCount = iprot.readI64() + else: + iprot.skip(ftype) + elif fid == 3: + if ftype == TType.I64: + self.updatedCount = iprot.readI64() + else: + iprot.skip(ftype) + elif fid == 4: + if ftype == TType.I64: + self.deletedCount = iprot.readI64() + else: + iprot.skip(ftype) + else: + iprot.skip(ftype) + iprot.readFieldEnd() + iprot.readStructEnd() + + def write(self, oprot): + if oprot._fast_encode is not None and self.thrift_spec is not None: + oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec])) + return + oprot.writeStructBegin("UpdateTransactionalStatsRequest") + if self.tableId is not None: + oprot.writeFieldBegin("tableId", TType.I64, 1) + oprot.writeI64(self.tableId) + oprot.writeFieldEnd() + if self.insertCount is not None: + oprot.writeFieldBegin("insertCount", TType.I64, 2) + oprot.writeI64(self.insertCount) + oprot.writeFieldEnd() + if self.updatedCount is not None: + oprot.writeFieldBegin("updatedCount", TType.I64, 3) + oprot.writeI64(self.updatedCount) + oprot.writeFieldEnd() + if self.deletedCount is not None: + oprot.writeFieldBegin("deletedCount", TType.I64, 4) + oprot.writeI64(self.deletedCount) + oprot.writeFieldEnd() + oprot.writeFieldStop() + oprot.writeStructEnd() + + def validate(self): + if self.tableId is None: + raise TProtocolException(message="Required field tableId is unset!") + if self.insertCount is None: + raise TProtocolException(message="Required field insertCount is unset!") + if self.updatedCount is None: + raise TProtocolException(message="Required field updatedCount is unset!") + if self.deletedCount is None: + raise TProtocolException(message="Required field deletedCount is unset!") + return + + def __repr__(self): + L = ["{}={!r}".format(key, value) for key, value in self.__dict__.items()] + return "{}({})".format(self.__class__.__name__, ", ".join(L)) + + def __eq__(self, other): + return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ + + def __ne__(self, other): + return not (self == other) + + +class CommitTxnRequest: + """ + Attributes: + - txnid + - replPolicy + - writeEventInfos + - replLastIdInfo + - keyValue + - exclWriteEnabled + - txn_type + + """ + + def __init__( + self, + txnid=None, + replPolicy=None, + writeEventInfos=None, + replLastIdInfo=None, + keyValue=None, + exclWriteEnabled=True, + txn_type=None, + ): + self.txnid = txnid + self.replPolicy = replPolicy + self.writeEventInfos = writeEventInfos + self.replLastIdInfo = replLastIdInfo + self.keyValue = keyValue + self.exclWriteEnabled = exclWriteEnabled + self.txn_type = txn_type + + def read(self, iprot): + if ( + iprot._fast_decode is not None + and isinstance(iprot.trans, TTransport.CReadableTransport) + and self.thrift_spec is not None + ): + iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec]) + return + iprot.readStructBegin() + while True: + (fname, ftype, fid) = iprot.readFieldBegin() + if ftype == TType.STOP: + break + if fid == 1: + if ftype == TType.I64: + self.txnid = iprot.readI64() + else: + iprot.skip(ftype) + elif fid == 2: + if ftype == TType.STRING: + self.replPolicy = ( + iprot.readString().decode("utf-8", errors="replace") if sys.version_info[0] == 2 else iprot.readString() + ) + else: + iprot.skip(ftype) + elif fid == 3: + if ftype == TType.LIST: + self.writeEventInfos = [] + (_etype670, _size667) = iprot.readListBegin() + for _i671 in range(_size667): + _elem672 = WriteEventInfo() + _elem672.read(iprot) + self.writeEventInfos.append(_elem672) + iprot.readListEnd() + else: + iprot.skip(ftype) + elif fid == 4: + if ftype == TType.STRUCT: + self.replLastIdInfo = ReplLastIdInfo() + self.replLastIdInfo.read(iprot) + else: + iprot.skip(ftype) + elif fid == 5: + if ftype == TType.STRUCT: + self.keyValue = CommitTxnKeyValue() + self.keyValue.read(iprot) + else: + iprot.skip(ftype) + elif fid == 6: + if ftype == TType.BOOL: + self.exclWriteEnabled = iprot.readBool() + else: + iprot.skip(ftype) + elif fid == 7: + if ftype == TType.I32: + self.txn_type = iprot.readI32() + else: + iprot.skip(ftype) + else: + iprot.skip(ftype) + iprot.readFieldEnd() + iprot.readStructEnd() + + def write(self, oprot): + if oprot._fast_encode is not None and self.thrift_spec is not None: + oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec])) + return + oprot.writeStructBegin("CommitTxnRequest") + if self.txnid is not None: + oprot.writeFieldBegin("txnid", TType.I64, 1) + oprot.writeI64(self.txnid) + oprot.writeFieldEnd() + if self.replPolicy is not None: + oprot.writeFieldBegin("replPolicy", TType.STRING, 2) + oprot.writeString(self.replPolicy.encode("utf-8") if sys.version_info[0] == 2 else self.replPolicy) + oprot.writeFieldEnd() + if self.writeEventInfos is not None: + oprot.writeFieldBegin("writeEventInfos", TType.LIST, 3) + oprot.writeListBegin(TType.STRUCT, len(self.writeEventInfos)) + for iter673 in self.writeEventInfos: + iter673.write(oprot) + oprot.writeListEnd() + oprot.writeFieldEnd() + if self.replLastIdInfo is not None: + oprot.writeFieldBegin("replLastIdInfo", TType.STRUCT, 4) + self.replLastIdInfo.write(oprot) + oprot.writeFieldEnd() + if self.keyValue is not None: + oprot.writeFieldBegin("keyValue", TType.STRUCT, 5) + self.keyValue.write(oprot) + oprot.writeFieldEnd() + if self.exclWriteEnabled is not None: + oprot.writeFieldBegin("exclWriteEnabled", TType.BOOL, 6) + oprot.writeBool(self.exclWriteEnabled) + oprot.writeFieldEnd() + if self.txn_type is not None: + oprot.writeFieldBegin("txn_type", TType.I32, 7) + oprot.writeI32(self.txn_type) + oprot.writeFieldEnd() + oprot.writeFieldStop() + oprot.writeStructEnd() + + def validate(self): + if self.txnid is None: + raise TProtocolException(message="Required field txnid is unset!") + return + + def __repr__(self): + L = ["{}={!r}".format(key, value) for key, value in self.__dict__.items()] + return "{}({})".format(self.__class__.__name__, ", ".join(L)) + + def __eq__(self, other): + return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ + + def __ne__(self, other): + return not (self == other) + + +class ReplTblWriteIdStateRequest: + """ + Attributes: + - validWriteIdlist + - user + - hostName + - dbName + - tableName + - partNames + + """ + + def __init__( + self, + validWriteIdlist=None, + user=None, + hostName=None, + dbName=None, + tableName=None, + partNames=None, + ): + self.validWriteIdlist = validWriteIdlist + self.user = user + self.hostName = hostName + self.dbName = dbName + self.tableName = tableName + self.partNames = partNames + + def read(self, iprot): + if ( + iprot._fast_decode is not None + and isinstance(iprot.trans, TTransport.CReadableTransport) + and self.thrift_spec is not None + ): + iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec]) + return + iprot.readStructBegin() + while True: + (fname, ftype, fid) = iprot.readFieldBegin() + if ftype == TType.STOP: + break + if fid == 1: + if ftype == TType.STRING: + self.validWriteIdlist = ( + iprot.readString().decode("utf-8", errors="replace") if sys.version_info[0] == 2 else iprot.readString() + ) + else: + iprot.skip(ftype) + elif fid == 2: + if ftype == TType.STRING: + self.user = ( + iprot.readString().decode("utf-8", errors="replace") if sys.version_info[0] == 2 else iprot.readString() + ) + else: + iprot.skip(ftype) + elif fid == 3: + if ftype == TType.STRING: + self.hostName = ( + iprot.readString().decode("utf-8", errors="replace") if sys.version_info[0] == 2 else iprot.readString() + ) + else: + iprot.skip(ftype) + elif fid == 4: + if ftype == TType.STRING: + self.dbName = ( + iprot.readString().decode("utf-8", errors="replace") if sys.version_info[0] == 2 else iprot.readString() + ) + else: + iprot.skip(ftype) + elif fid == 5: + if ftype == TType.STRING: + self.tableName = ( + iprot.readString().decode("utf-8", errors="replace") if sys.version_info[0] == 2 else iprot.readString() + ) + else: + iprot.skip(ftype) + elif fid == 6: + if ftype == TType.LIST: + self.partNames = [] + (_etype677, _size674) = iprot.readListBegin() + for _i678 in range(_size674): + _elem679 = ( + iprot.readString().decode("utf-8", errors="replace") + if sys.version_info[0] == 2 + else iprot.readString() + ) + self.partNames.append(_elem679) + iprot.readListEnd() + else: + iprot.skip(ftype) + else: + iprot.skip(ftype) + iprot.readFieldEnd() + iprot.readStructEnd() + + def write(self, oprot): + if oprot._fast_encode is not None and self.thrift_spec is not None: + oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec])) + return + oprot.writeStructBegin("ReplTblWriteIdStateRequest") + if self.validWriteIdlist is not None: + oprot.writeFieldBegin("validWriteIdlist", TType.STRING, 1) + oprot.writeString(self.validWriteIdlist.encode("utf-8") if sys.version_info[0] == 2 else self.validWriteIdlist) + oprot.writeFieldEnd() + if self.user is not None: + oprot.writeFieldBegin("user", TType.STRING, 2) + oprot.writeString(self.user.encode("utf-8") if sys.version_info[0] == 2 else self.user) + oprot.writeFieldEnd() + if self.hostName is not None: + oprot.writeFieldBegin("hostName", TType.STRING, 3) + oprot.writeString(self.hostName.encode("utf-8") if sys.version_info[0] == 2 else self.hostName) + oprot.writeFieldEnd() + if self.dbName is not None: + oprot.writeFieldBegin("dbName", TType.STRING, 4) + oprot.writeString(self.dbName.encode("utf-8") if sys.version_info[0] == 2 else self.dbName) + oprot.writeFieldEnd() + if self.tableName is not None: + oprot.writeFieldBegin("tableName", TType.STRING, 5) + oprot.writeString(self.tableName.encode("utf-8") if sys.version_info[0] == 2 else self.tableName) + oprot.writeFieldEnd() + if self.partNames is not None: + oprot.writeFieldBegin("partNames", TType.LIST, 6) + oprot.writeListBegin(TType.STRING, len(self.partNames)) + for iter680 in self.partNames: + oprot.writeString(iter680.encode("utf-8") if sys.version_info[0] == 2 else iter680) + oprot.writeListEnd() + oprot.writeFieldEnd() + oprot.writeFieldStop() + oprot.writeStructEnd() + + def validate(self): + if self.validWriteIdlist is None: + raise TProtocolException(message="Required field validWriteIdlist is unset!") + if self.user is None: + raise TProtocolException(message="Required field user is unset!") + if self.hostName is None: + raise TProtocolException(message="Required field hostName is unset!") + if self.dbName is None: + raise TProtocolException(message="Required field dbName is unset!") + if self.tableName is None: + raise TProtocolException(message="Required field tableName is unset!") + return + + def __repr__(self): + L = ["{}={!r}".format(key, value) for key, value in self.__dict__.items()] + return "{}({})".format(self.__class__.__name__, ", ".join(L)) + + def __eq__(self, other): + return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ + + def __ne__(self, other): + return not (self == other) + + +class GetValidWriteIdsRequest: + """ + Attributes: + - fullTableNames + - validTxnList + - writeId + + """ + + def __init__( + self, + fullTableNames=None, + validTxnList=None, + writeId=None, + ): + self.fullTableNames = fullTableNames + self.validTxnList = validTxnList + self.writeId = writeId + + def read(self, iprot): + if ( + iprot._fast_decode is not None + and isinstance(iprot.trans, TTransport.CReadableTransport) + and self.thrift_spec is not None + ): + iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec]) + return + iprot.readStructBegin() + while True: + (fname, ftype, fid) = iprot.readFieldBegin() + if ftype == TType.STOP: + break + if fid == 1: + if ftype == TType.LIST: + self.fullTableNames = [] + (_etype684, _size681) = iprot.readListBegin() + for _i685 in range(_size681): + _elem686 = ( + iprot.readString().decode("utf-8", errors="replace") + if sys.version_info[0] == 2 + else iprot.readString() + ) + self.fullTableNames.append(_elem686) + iprot.readListEnd() + else: + iprot.skip(ftype) + elif fid == 2: + if ftype == TType.STRING: + self.validTxnList = ( + iprot.readString().decode("utf-8", errors="replace") if sys.version_info[0] == 2 else iprot.readString() + ) + else: + iprot.skip(ftype) + elif fid == 3: + if ftype == TType.I64: + self.writeId = iprot.readI64() + else: + iprot.skip(ftype) + else: + iprot.skip(ftype) + iprot.readFieldEnd() + iprot.readStructEnd() + + def write(self, oprot): + if oprot._fast_encode is not None and self.thrift_spec is not None: + oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec])) + return + oprot.writeStructBegin("GetValidWriteIdsRequest") + if self.fullTableNames is not None: + oprot.writeFieldBegin("fullTableNames", TType.LIST, 1) + oprot.writeListBegin(TType.STRING, len(self.fullTableNames)) + for iter687 in self.fullTableNames: + oprot.writeString(iter687.encode("utf-8") if sys.version_info[0] == 2 else iter687) + oprot.writeListEnd() + oprot.writeFieldEnd() + if self.validTxnList is not None: + oprot.writeFieldBegin("validTxnList", TType.STRING, 2) + oprot.writeString(self.validTxnList.encode("utf-8") if sys.version_info[0] == 2 else self.validTxnList) + oprot.writeFieldEnd() + if self.writeId is not None: + oprot.writeFieldBegin("writeId", TType.I64, 3) + oprot.writeI64(self.writeId) + oprot.writeFieldEnd() + oprot.writeFieldStop() + oprot.writeStructEnd() + + def validate(self): + if self.fullTableNames is None: + raise TProtocolException(message="Required field fullTableNames is unset!") + return + + def __repr__(self): + L = ["{}={!r}".format(key, value) for key, value in self.__dict__.items()] + return "{}({})".format(self.__class__.__name__, ", ".join(L)) + + def __eq__(self, other): + return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ + + def __ne__(self, other): + return not (self == other) + + +class TableValidWriteIds: + """ + Attributes: + - fullTableName + - writeIdHighWaterMark + - invalidWriteIds + - minOpenWriteId + - abortedBits + + """ + + def __init__( + self, + fullTableName=None, + writeIdHighWaterMark=None, + invalidWriteIds=None, + minOpenWriteId=None, + abortedBits=None, + ): + self.fullTableName = fullTableName + self.writeIdHighWaterMark = writeIdHighWaterMark + self.invalidWriteIds = invalidWriteIds + self.minOpenWriteId = minOpenWriteId + self.abortedBits = abortedBits + + def read(self, iprot): + if ( + iprot._fast_decode is not None + and isinstance(iprot.trans, TTransport.CReadableTransport) + and self.thrift_spec is not None + ): + iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec]) + return + iprot.readStructBegin() + while True: + (fname, ftype, fid) = iprot.readFieldBegin() + if ftype == TType.STOP: + break + if fid == 1: + if ftype == TType.STRING: + self.fullTableName = ( + iprot.readString().decode("utf-8", errors="replace") if sys.version_info[0] == 2 else iprot.readString() + ) + else: + iprot.skip(ftype) + elif fid == 2: + if ftype == TType.I64: + self.writeIdHighWaterMark = iprot.readI64() + else: + iprot.skip(ftype) + elif fid == 3: + if ftype == TType.LIST: + self.invalidWriteIds = [] + (_etype691, _size688) = iprot.readListBegin() + for _i692 in range(_size688): + _elem693 = iprot.readI64() + self.invalidWriteIds.append(_elem693) + iprot.readListEnd() + else: + iprot.skip(ftype) + elif fid == 4: + if ftype == TType.I64: + self.minOpenWriteId = iprot.readI64() + else: + iprot.skip(ftype) + elif fid == 5: + if ftype == TType.STRING: + self.abortedBits = iprot.readBinary() + else: + iprot.skip(ftype) + else: + iprot.skip(ftype) + iprot.readFieldEnd() + iprot.readStructEnd() + + def write(self, oprot): + if oprot._fast_encode is not None and self.thrift_spec is not None: + oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec])) + return + oprot.writeStructBegin("TableValidWriteIds") + if self.fullTableName is not None: + oprot.writeFieldBegin("fullTableName", TType.STRING, 1) + oprot.writeString(self.fullTableName.encode("utf-8") if sys.version_info[0] == 2 else self.fullTableName) + oprot.writeFieldEnd() + if self.writeIdHighWaterMark is not None: + oprot.writeFieldBegin("writeIdHighWaterMark", TType.I64, 2) + oprot.writeI64(self.writeIdHighWaterMark) + oprot.writeFieldEnd() + if self.invalidWriteIds is not None: + oprot.writeFieldBegin("invalidWriteIds", TType.LIST, 3) + oprot.writeListBegin(TType.I64, len(self.invalidWriteIds)) + for iter694 in self.invalidWriteIds: + oprot.writeI64(iter694) + oprot.writeListEnd() + oprot.writeFieldEnd() + if self.minOpenWriteId is not None: + oprot.writeFieldBegin("minOpenWriteId", TType.I64, 4) + oprot.writeI64(self.minOpenWriteId) + oprot.writeFieldEnd() + if self.abortedBits is not None: + oprot.writeFieldBegin("abortedBits", TType.STRING, 5) + oprot.writeBinary(self.abortedBits) + oprot.writeFieldEnd() + oprot.writeFieldStop() + oprot.writeStructEnd() + + def validate(self): + if self.fullTableName is None: + raise TProtocolException(message="Required field fullTableName is unset!") + if self.writeIdHighWaterMark is None: + raise TProtocolException(message="Required field writeIdHighWaterMark is unset!") + if self.invalidWriteIds is None: + raise TProtocolException(message="Required field invalidWriteIds is unset!") + if self.abortedBits is None: + raise TProtocolException(message="Required field abortedBits is unset!") + return + + def __repr__(self): + L = ["{}={!r}".format(key, value) for key, value in self.__dict__.items()] + return "{}({})".format(self.__class__.__name__, ", ".join(L)) + + def __eq__(self, other): + return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ + + def __ne__(self, other): + return not (self == other) + + +class GetValidWriteIdsResponse: + """ + Attributes: + - tblValidWriteIds + + """ + + def __init__( + self, + tblValidWriteIds=None, + ): + self.tblValidWriteIds = tblValidWriteIds + + def read(self, iprot): + if ( + iprot._fast_decode is not None + and isinstance(iprot.trans, TTransport.CReadableTransport) + and self.thrift_spec is not None + ): + iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec]) + return + iprot.readStructBegin() + while True: + (fname, ftype, fid) = iprot.readFieldBegin() + if ftype == TType.STOP: + break + if fid == 1: + if ftype == TType.LIST: + self.tblValidWriteIds = [] + (_etype698, _size695) = iprot.readListBegin() + for _i699 in range(_size695): + _elem700 = TableValidWriteIds() + _elem700.read(iprot) + self.tblValidWriteIds.append(_elem700) + iprot.readListEnd() + else: + iprot.skip(ftype) + else: + iprot.skip(ftype) + iprot.readFieldEnd() + iprot.readStructEnd() + + def write(self, oprot): + if oprot._fast_encode is not None and self.thrift_spec is not None: + oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec])) + return + oprot.writeStructBegin("GetValidWriteIdsResponse") + if self.tblValidWriteIds is not None: + oprot.writeFieldBegin("tblValidWriteIds", TType.LIST, 1) + oprot.writeListBegin(TType.STRUCT, len(self.tblValidWriteIds)) + for iter701 in self.tblValidWriteIds: + iter701.write(oprot) + oprot.writeListEnd() + oprot.writeFieldEnd() + oprot.writeFieldStop() + oprot.writeStructEnd() + + def validate(self): + if self.tblValidWriteIds is None: + raise TProtocolException(message="Required field tblValidWriteIds is unset!") + return + + def __repr__(self): + L = ["{}={!r}".format(key, value) for key, value in self.__dict__.items()] + return "{}({})".format(self.__class__.__name__, ", ".join(L)) + + def __eq__(self, other): + return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ + + def __ne__(self, other): + return not (self == other) + + +class TxnToWriteId: + """ + Attributes: + - txnId + - writeId + + """ + + def __init__( + self, + txnId=None, + writeId=None, + ): + self.txnId = txnId + self.writeId = writeId + + def read(self, iprot): + if ( + iprot._fast_decode is not None + and isinstance(iprot.trans, TTransport.CReadableTransport) + and self.thrift_spec is not None + ): + iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec]) + return + iprot.readStructBegin() + while True: + (fname, ftype, fid) = iprot.readFieldBegin() + if ftype == TType.STOP: + break + if fid == 1: + if ftype == TType.I64: + self.txnId = iprot.readI64() + else: + iprot.skip(ftype) + elif fid == 2: + if ftype == TType.I64: + self.writeId = iprot.readI64() + else: + iprot.skip(ftype) + else: + iprot.skip(ftype) + iprot.readFieldEnd() + iprot.readStructEnd() + + def write(self, oprot): + if oprot._fast_encode is not None and self.thrift_spec is not None: + oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec])) + return + oprot.writeStructBegin("TxnToWriteId") + if self.txnId is not None: + oprot.writeFieldBegin("txnId", TType.I64, 1) + oprot.writeI64(self.txnId) + oprot.writeFieldEnd() + if self.writeId is not None: + oprot.writeFieldBegin("writeId", TType.I64, 2) + oprot.writeI64(self.writeId) + oprot.writeFieldEnd() + oprot.writeFieldStop() + oprot.writeStructEnd() + + def validate(self): + if self.txnId is None: + raise TProtocolException(message="Required field txnId is unset!") + if self.writeId is None: + raise TProtocolException(message="Required field writeId is unset!") + return + + def __repr__(self): + L = ["{}={!r}".format(key, value) for key, value in self.__dict__.items()] + return "{}({})".format(self.__class__.__name__, ", ".join(L)) + + def __eq__(self, other): + return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ + + def __ne__(self, other): + return not (self == other) + + +class AllocateTableWriteIdsRequest: + """ + Attributes: + - dbName + - tableName + - txnIds + - replPolicy + - srcTxnToWriteIdList + + """ + + def __init__( + self, + dbName=None, + tableName=None, + txnIds=None, + replPolicy=None, + srcTxnToWriteIdList=None, + ): + self.dbName = dbName + self.tableName = tableName + self.txnIds = txnIds + self.replPolicy = replPolicy + self.srcTxnToWriteIdList = srcTxnToWriteIdList + + def read(self, iprot): + if ( + iprot._fast_decode is not None + and isinstance(iprot.trans, TTransport.CReadableTransport) + and self.thrift_spec is not None + ): + iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec]) + return + iprot.readStructBegin() + while True: + (fname, ftype, fid) = iprot.readFieldBegin() + if ftype == TType.STOP: + break + if fid == 1: + if ftype == TType.STRING: + self.dbName = ( + iprot.readString().decode("utf-8", errors="replace") if sys.version_info[0] == 2 else iprot.readString() + ) + else: + iprot.skip(ftype) + elif fid == 2: + if ftype == TType.STRING: + self.tableName = ( + iprot.readString().decode("utf-8", errors="replace") if sys.version_info[0] == 2 else iprot.readString() + ) + else: + iprot.skip(ftype) + elif fid == 3: + if ftype == TType.LIST: + self.txnIds = [] + (_etype705, _size702) = iprot.readListBegin() + for _i706 in range(_size702): + _elem707 = iprot.readI64() + self.txnIds.append(_elem707) + iprot.readListEnd() + else: + iprot.skip(ftype) + elif fid == 4: + if ftype == TType.STRING: + self.replPolicy = ( + iprot.readString().decode("utf-8", errors="replace") if sys.version_info[0] == 2 else iprot.readString() + ) + else: + iprot.skip(ftype) + elif fid == 5: + if ftype == TType.LIST: + self.srcTxnToWriteIdList = [] + (_etype711, _size708) = iprot.readListBegin() + for _i712 in range(_size708): + _elem713 = TxnToWriteId() + _elem713.read(iprot) + self.srcTxnToWriteIdList.append(_elem713) + iprot.readListEnd() + else: + iprot.skip(ftype) + else: + iprot.skip(ftype) + iprot.readFieldEnd() + iprot.readStructEnd() + + def write(self, oprot): + if oprot._fast_encode is not None and self.thrift_spec is not None: + oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec])) + return + oprot.writeStructBegin("AllocateTableWriteIdsRequest") + if self.dbName is not None: + oprot.writeFieldBegin("dbName", TType.STRING, 1) + oprot.writeString(self.dbName.encode("utf-8") if sys.version_info[0] == 2 else self.dbName) + oprot.writeFieldEnd() + if self.tableName is not None: + oprot.writeFieldBegin("tableName", TType.STRING, 2) + oprot.writeString(self.tableName.encode("utf-8") if sys.version_info[0] == 2 else self.tableName) + oprot.writeFieldEnd() + if self.txnIds is not None: + oprot.writeFieldBegin("txnIds", TType.LIST, 3) + oprot.writeListBegin(TType.I64, len(self.txnIds)) + for iter714 in self.txnIds: + oprot.writeI64(iter714) + oprot.writeListEnd() + oprot.writeFieldEnd() + if self.replPolicy is not None: + oprot.writeFieldBegin("replPolicy", TType.STRING, 4) + oprot.writeString(self.replPolicy.encode("utf-8") if sys.version_info[0] == 2 else self.replPolicy) + oprot.writeFieldEnd() + if self.srcTxnToWriteIdList is not None: + oprot.writeFieldBegin("srcTxnToWriteIdList", TType.LIST, 5) + oprot.writeListBegin(TType.STRUCT, len(self.srcTxnToWriteIdList)) + for iter715 in self.srcTxnToWriteIdList: + iter715.write(oprot) + oprot.writeListEnd() + oprot.writeFieldEnd() + oprot.writeFieldStop() + oprot.writeStructEnd() + + def validate(self): + if self.dbName is None: + raise TProtocolException(message="Required field dbName is unset!") + if self.tableName is None: + raise TProtocolException(message="Required field tableName is unset!") + return + + def __repr__(self): + L = ["{}={!r}".format(key, value) for key, value in self.__dict__.items()] + return "{}({})".format(self.__class__.__name__, ", ".join(L)) + + def __eq__(self, other): + return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ + + def __ne__(self, other): + return not (self == other) + + +class AllocateTableWriteIdsResponse: + """ + Attributes: + - txnToWriteIds + + """ + + def __init__( + self, + txnToWriteIds=None, + ): + self.txnToWriteIds = txnToWriteIds + + def read(self, iprot): + if ( + iprot._fast_decode is not None + and isinstance(iprot.trans, TTransport.CReadableTransport) + and self.thrift_spec is not None + ): + iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec]) + return + iprot.readStructBegin() + while True: + (fname, ftype, fid) = iprot.readFieldBegin() + if ftype == TType.STOP: + break + if fid == 1: + if ftype == TType.LIST: + self.txnToWriteIds = [] + (_etype719, _size716) = iprot.readListBegin() + for _i720 in range(_size716): + _elem721 = TxnToWriteId() + _elem721.read(iprot) + self.txnToWriteIds.append(_elem721) + iprot.readListEnd() + else: + iprot.skip(ftype) + else: + iprot.skip(ftype) + iprot.readFieldEnd() + iprot.readStructEnd() + + def write(self, oprot): + if oprot._fast_encode is not None and self.thrift_spec is not None: + oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec])) + return + oprot.writeStructBegin("AllocateTableWriteIdsResponse") + if self.txnToWriteIds is not None: + oprot.writeFieldBegin("txnToWriteIds", TType.LIST, 1) + oprot.writeListBegin(TType.STRUCT, len(self.txnToWriteIds)) + for iter722 in self.txnToWriteIds: + iter722.write(oprot) + oprot.writeListEnd() + oprot.writeFieldEnd() + oprot.writeFieldStop() + oprot.writeStructEnd() + + def validate(self): + if self.txnToWriteIds is None: + raise TProtocolException(message="Required field txnToWriteIds is unset!") + return + + def __repr__(self): + L = ["{}={!r}".format(key, value) for key, value in self.__dict__.items()] + return "{}({})".format(self.__class__.__name__, ", ".join(L)) + + def __eq__(self, other): + return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ + + def __ne__(self, other): + return not (self == other) + + +class MaxAllocatedTableWriteIdRequest: + """ + Attributes: + - dbName + - tableName + + """ + + def __init__( + self, + dbName=None, + tableName=None, + ): + self.dbName = dbName + self.tableName = tableName + + def read(self, iprot): + if ( + iprot._fast_decode is not None + and isinstance(iprot.trans, TTransport.CReadableTransport) + and self.thrift_spec is not None + ): + iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec]) + return + iprot.readStructBegin() + while True: + (fname, ftype, fid) = iprot.readFieldBegin() + if ftype == TType.STOP: + break + if fid == 1: + if ftype == TType.STRING: + self.dbName = ( + iprot.readString().decode("utf-8", errors="replace") if sys.version_info[0] == 2 else iprot.readString() + ) + else: + iprot.skip(ftype) + elif fid == 2: + if ftype == TType.STRING: + self.tableName = ( + iprot.readString().decode("utf-8", errors="replace") if sys.version_info[0] == 2 else iprot.readString() + ) + else: + iprot.skip(ftype) + else: + iprot.skip(ftype) + iprot.readFieldEnd() + iprot.readStructEnd() + + def write(self, oprot): + if oprot._fast_encode is not None and self.thrift_spec is not None: + oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec])) + return + oprot.writeStructBegin("MaxAllocatedTableWriteIdRequest") + if self.dbName is not None: + oprot.writeFieldBegin("dbName", TType.STRING, 1) + oprot.writeString(self.dbName.encode("utf-8") if sys.version_info[0] == 2 else self.dbName) + oprot.writeFieldEnd() + if self.tableName is not None: + oprot.writeFieldBegin("tableName", TType.STRING, 2) + oprot.writeString(self.tableName.encode("utf-8") if sys.version_info[0] == 2 else self.tableName) + oprot.writeFieldEnd() + oprot.writeFieldStop() + oprot.writeStructEnd() + + def validate(self): + if self.dbName is None: + raise TProtocolException(message="Required field dbName is unset!") + if self.tableName is None: + raise TProtocolException(message="Required field tableName is unset!") + return + + def __repr__(self): + L = ["{}={!r}".format(key, value) for key, value in self.__dict__.items()] + return "{}({})".format(self.__class__.__name__, ", ".join(L)) + + def __eq__(self, other): + return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ + + def __ne__(self, other): + return not (self == other) + + +class MaxAllocatedTableWriteIdResponse: + """ + Attributes: + - maxWriteId + + """ + + def __init__( + self, + maxWriteId=None, + ): + self.maxWriteId = maxWriteId + + def read(self, iprot): + if ( + iprot._fast_decode is not None + and isinstance(iprot.trans, TTransport.CReadableTransport) + and self.thrift_spec is not None + ): + iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec]) + return + iprot.readStructBegin() + while True: + (fname, ftype, fid) = iprot.readFieldBegin() + if ftype == TType.STOP: + break + if fid == 1: + if ftype == TType.I64: + self.maxWriteId = iprot.readI64() + else: + iprot.skip(ftype) + else: + iprot.skip(ftype) + iprot.readFieldEnd() + iprot.readStructEnd() + + def write(self, oprot): + if oprot._fast_encode is not None and self.thrift_spec is not None: + oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec])) + return + oprot.writeStructBegin("MaxAllocatedTableWriteIdResponse") + if self.maxWriteId is not None: + oprot.writeFieldBegin("maxWriteId", TType.I64, 1) + oprot.writeI64(self.maxWriteId) + oprot.writeFieldEnd() + oprot.writeFieldStop() + oprot.writeStructEnd() + + def validate(self): + if self.maxWriteId is None: + raise TProtocolException(message="Required field maxWriteId is unset!") + return + + def __repr__(self): + L = ["{}={!r}".format(key, value) for key, value in self.__dict__.items()] + return "{}({})".format(self.__class__.__name__, ", ".join(L)) + + def __eq__(self, other): + return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ + + def __ne__(self, other): + return not (self == other) + + +class SeedTableWriteIdsRequest: + """ + Attributes: + - dbName + - tableName + - seedWriteId + + """ + + def __init__( + self, + dbName=None, + tableName=None, + seedWriteId=None, + ): + self.dbName = dbName + self.tableName = tableName + self.seedWriteId = seedWriteId + + def read(self, iprot): + if ( + iprot._fast_decode is not None + and isinstance(iprot.trans, TTransport.CReadableTransport) + and self.thrift_spec is not None + ): + iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec]) + return + iprot.readStructBegin() + while True: + (fname, ftype, fid) = iprot.readFieldBegin() + if ftype == TType.STOP: + break + if fid == 1: + if ftype == TType.STRING: + self.dbName = ( + iprot.readString().decode("utf-8", errors="replace") if sys.version_info[0] == 2 else iprot.readString() + ) + else: + iprot.skip(ftype) + elif fid == 2: + if ftype == TType.STRING: + self.tableName = ( + iprot.readString().decode("utf-8", errors="replace") if sys.version_info[0] == 2 else iprot.readString() + ) + else: + iprot.skip(ftype) + elif fid == 3: + if ftype == TType.I64: + self.seedWriteId = iprot.readI64() + else: + iprot.skip(ftype) + else: + iprot.skip(ftype) + iprot.readFieldEnd() + iprot.readStructEnd() + + def write(self, oprot): + if oprot._fast_encode is not None and self.thrift_spec is not None: + oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec])) + return + oprot.writeStructBegin("SeedTableWriteIdsRequest") + if self.dbName is not None: + oprot.writeFieldBegin("dbName", TType.STRING, 1) + oprot.writeString(self.dbName.encode("utf-8") if sys.version_info[0] == 2 else self.dbName) + oprot.writeFieldEnd() + if self.tableName is not None: + oprot.writeFieldBegin("tableName", TType.STRING, 2) + oprot.writeString(self.tableName.encode("utf-8") if sys.version_info[0] == 2 else self.tableName) + oprot.writeFieldEnd() + if self.seedWriteId is not None: + oprot.writeFieldBegin("seedWriteId", TType.I64, 3) + oprot.writeI64(self.seedWriteId) + oprot.writeFieldEnd() + oprot.writeFieldStop() + oprot.writeStructEnd() + + def validate(self): + if self.dbName is None: + raise TProtocolException(message="Required field dbName is unset!") + if self.tableName is None: + raise TProtocolException(message="Required field tableName is unset!") + if self.seedWriteId is None: + raise TProtocolException(message="Required field seedWriteId is unset!") + return + + def __repr__(self): + L = ["{}={!r}".format(key, value) for key, value in self.__dict__.items()] + return "{}({})".format(self.__class__.__name__, ", ".join(L)) + + def __eq__(self, other): + return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ + + def __ne__(self, other): + return not (self == other) + + +class SeedTxnIdRequest: + """ + Attributes: + - seedTxnId + + """ + + def __init__( + self, + seedTxnId=None, + ): + self.seedTxnId = seedTxnId + + def read(self, iprot): + if ( + iprot._fast_decode is not None + and isinstance(iprot.trans, TTransport.CReadableTransport) + and self.thrift_spec is not None + ): + iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec]) + return + iprot.readStructBegin() + while True: + (fname, ftype, fid) = iprot.readFieldBegin() + if ftype == TType.STOP: + break + if fid == 1: + if ftype == TType.I64: + self.seedTxnId = iprot.readI64() + else: + iprot.skip(ftype) + else: + iprot.skip(ftype) + iprot.readFieldEnd() + iprot.readStructEnd() + + def write(self, oprot): + if oprot._fast_encode is not None and self.thrift_spec is not None: + oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec])) + return + oprot.writeStructBegin("SeedTxnIdRequest") + if self.seedTxnId is not None: + oprot.writeFieldBegin("seedTxnId", TType.I64, 1) + oprot.writeI64(self.seedTxnId) + oprot.writeFieldEnd() + oprot.writeFieldStop() + oprot.writeStructEnd() + + def validate(self): + if self.seedTxnId is None: + raise TProtocolException(message="Required field seedTxnId is unset!") + return + + def __repr__(self): + L = ["{}={!r}".format(key, value) for key, value in self.__dict__.items()] + return "{}({})".format(self.__class__.__name__, ", ".join(L)) + + def __eq__(self, other): + return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ + + def __ne__(self, other): + return not (self == other) + + +class LockComponent: + """ + Attributes: + - type + - level + - dbname + - tablename + - partitionname + - operationType + - isTransactional + - isDynamicPartitionWrite + + """ + + def __init__( + self, + type=None, + level=None, + dbname=None, + tablename=None, + partitionname=None, + operationType=5, + isTransactional=False, + isDynamicPartitionWrite=False, + ): + self.type = type + self.level = level + self.dbname = dbname + self.tablename = tablename + self.partitionname = partitionname + self.operationType = operationType + self.isTransactional = isTransactional + self.isDynamicPartitionWrite = isDynamicPartitionWrite + + def read(self, iprot): + if ( + iprot._fast_decode is not None + and isinstance(iprot.trans, TTransport.CReadableTransport) + and self.thrift_spec is not None + ): + iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec]) + return + iprot.readStructBegin() + while True: + (fname, ftype, fid) = iprot.readFieldBegin() + if ftype == TType.STOP: + break + if fid == 1: + if ftype == TType.I32: + self.type = iprot.readI32() + else: + iprot.skip(ftype) + elif fid == 2: + if ftype == TType.I32: + self.level = iprot.readI32() + else: + iprot.skip(ftype) + elif fid == 3: + if ftype == TType.STRING: + self.dbname = ( + iprot.readString().decode("utf-8", errors="replace") if sys.version_info[0] == 2 else iprot.readString() + ) + else: + iprot.skip(ftype) + elif fid == 4: + if ftype == TType.STRING: + self.tablename = ( + iprot.readString().decode("utf-8", errors="replace") if sys.version_info[0] == 2 else iprot.readString() + ) + else: + iprot.skip(ftype) + elif fid == 5: + if ftype == TType.STRING: + self.partitionname = ( + iprot.readString().decode("utf-8", errors="replace") if sys.version_info[0] == 2 else iprot.readString() + ) + else: + iprot.skip(ftype) + elif fid == 6: + if ftype == TType.I32: + self.operationType = iprot.readI32() + else: + iprot.skip(ftype) + elif fid == 7: + if ftype == TType.BOOL: + self.isTransactional = iprot.readBool() + else: + iprot.skip(ftype) + elif fid == 8: + if ftype == TType.BOOL: + self.isDynamicPartitionWrite = iprot.readBool() + else: + iprot.skip(ftype) + else: + iprot.skip(ftype) + iprot.readFieldEnd() + iprot.readStructEnd() + + def write(self, oprot): + if oprot._fast_encode is not None and self.thrift_spec is not None: + oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec])) + return + oprot.writeStructBegin("LockComponent") + if self.type is not None: + oprot.writeFieldBegin("type", TType.I32, 1) + oprot.writeI32(self.type) + oprot.writeFieldEnd() + if self.level is not None: + oprot.writeFieldBegin("level", TType.I32, 2) + oprot.writeI32(self.level) + oprot.writeFieldEnd() + if self.dbname is not None: + oprot.writeFieldBegin("dbname", TType.STRING, 3) + oprot.writeString(self.dbname.encode("utf-8") if sys.version_info[0] == 2 else self.dbname) + oprot.writeFieldEnd() + if self.tablename is not None: + oprot.writeFieldBegin("tablename", TType.STRING, 4) + oprot.writeString(self.tablename.encode("utf-8") if sys.version_info[0] == 2 else self.tablename) + oprot.writeFieldEnd() + if self.partitionname is not None: + oprot.writeFieldBegin("partitionname", TType.STRING, 5) + oprot.writeString(self.partitionname.encode("utf-8") if sys.version_info[0] == 2 else self.partitionname) + oprot.writeFieldEnd() + if self.operationType is not None: + oprot.writeFieldBegin("operationType", TType.I32, 6) + oprot.writeI32(self.operationType) + oprot.writeFieldEnd() + if self.isTransactional is not None: + oprot.writeFieldBegin("isTransactional", TType.BOOL, 7) + oprot.writeBool(self.isTransactional) + oprot.writeFieldEnd() + if self.isDynamicPartitionWrite is not None: + oprot.writeFieldBegin("isDynamicPartitionWrite", TType.BOOL, 8) + oprot.writeBool(self.isDynamicPartitionWrite) + oprot.writeFieldEnd() + oprot.writeFieldStop() + oprot.writeStructEnd() + + def validate(self): + if self.type is None: + raise TProtocolException(message="Required field type is unset!") + if self.level is None: + raise TProtocolException(message="Required field level is unset!") + if self.dbname is None: + raise TProtocolException(message="Required field dbname is unset!") + return + + def __repr__(self): + L = ["{}={!r}".format(key, value) for key, value in self.__dict__.items()] + return "{}({})".format(self.__class__.__name__, ", ".join(L)) + + def __eq__(self, other): + return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ + + def __ne__(self, other): + return not (self == other) + + +class LockRequest: + """ + Attributes: + - component + - txnid + - user + - hostname + - agentInfo + - zeroWaitReadEnabled + - exclusiveCTAS + + """ + + def __init__( + self, + component=None, + txnid=None, + user=None, + hostname=None, + agentInfo="Unknown", + zeroWaitReadEnabled=False, + exclusiveCTAS=False, + ): + self.component = component + self.txnid = txnid + self.user = user + self.hostname = hostname + self.agentInfo = agentInfo + self.zeroWaitReadEnabled = zeroWaitReadEnabled + self.exclusiveCTAS = exclusiveCTAS + + def read(self, iprot): + if ( + iprot._fast_decode is not None + and isinstance(iprot.trans, TTransport.CReadableTransport) + and self.thrift_spec is not None + ): + iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec]) + return + iprot.readStructBegin() + while True: + (fname, ftype, fid) = iprot.readFieldBegin() + if ftype == TType.STOP: + break + if fid == 1: + if ftype == TType.LIST: + self.component = [] + (_etype726, _size723) = iprot.readListBegin() + for _i727 in range(_size723): + _elem728 = LockComponent() + _elem728.read(iprot) + self.component.append(_elem728) + iprot.readListEnd() + else: + iprot.skip(ftype) + elif fid == 2: + if ftype == TType.I64: + self.txnid = iprot.readI64() + else: + iprot.skip(ftype) + elif fid == 3: + if ftype == TType.STRING: + self.user = ( + iprot.readString().decode("utf-8", errors="replace") if sys.version_info[0] == 2 else iprot.readString() + ) + else: + iprot.skip(ftype) + elif fid == 4: + if ftype == TType.STRING: + self.hostname = ( + iprot.readString().decode("utf-8", errors="replace") if sys.version_info[0] == 2 else iprot.readString() + ) + else: + iprot.skip(ftype) + elif fid == 5: + if ftype == TType.STRING: + self.agentInfo = ( + iprot.readString().decode("utf-8", errors="replace") if sys.version_info[0] == 2 else iprot.readString() + ) + else: + iprot.skip(ftype) + elif fid == 6: + if ftype == TType.BOOL: + self.zeroWaitReadEnabled = iprot.readBool() + else: + iprot.skip(ftype) + elif fid == 7: + if ftype == TType.BOOL: + self.exclusiveCTAS = iprot.readBool() + else: + iprot.skip(ftype) + else: + iprot.skip(ftype) + iprot.readFieldEnd() + iprot.readStructEnd() + + def write(self, oprot): + if oprot._fast_encode is not None and self.thrift_spec is not None: + oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec])) + return + oprot.writeStructBegin("LockRequest") + if self.component is not None: + oprot.writeFieldBegin("component", TType.LIST, 1) + oprot.writeListBegin(TType.STRUCT, len(self.component)) + for iter729 in self.component: + iter729.write(oprot) + oprot.writeListEnd() + oprot.writeFieldEnd() + if self.txnid is not None: + oprot.writeFieldBegin("txnid", TType.I64, 2) + oprot.writeI64(self.txnid) + oprot.writeFieldEnd() + if self.user is not None: + oprot.writeFieldBegin("user", TType.STRING, 3) + oprot.writeString(self.user.encode("utf-8") if sys.version_info[0] == 2 else self.user) + oprot.writeFieldEnd() + if self.hostname is not None: + oprot.writeFieldBegin("hostname", TType.STRING, 4) + oprot.writeString(self.hostname.encode("utf-8") if sys.version_info[0] == 2 else self.hostname) + oprot.writeFieldEnd() + if self.agentInfo is not None: + oprot.writeFieldBegin("agentInfo", TType.STRING, 5) + oprot.writeString(self.agentInfo.encode("utf-8") if sys.version_info[0] == 2 else self.agentInfo) + oprot.writeFieldEnd() + if self.zeroWaitReadEnabled is not None: + oprot.writeFieldBegin("zeroWaitReadEnabled", TType.BOOL, 6) + oprot.writeBool(self.zeroWaitReadEnabled) + oprot.writeFieldEnd() + if self.exclusiveCTAS is not None: + oprot.writeFieldBegin("exclusiveCTAS", TType.BOOL, 7) + oprot.writeBool(self.exclusiveCTAS) + oprot.writeFieldEnd() + oprot.writeFieldStop() + oprot.writeStructEnd() + + def validate(self): + if self.component is None: + raise TProtocolException(message="Required field component is unset!") + if self.user is None: + raise TProtocolException(message="Required field user is unset!") + if self.hostname is None: + raise TProtocolException(message="Required field hostname is unset!") + return + + def __repr__(self): + L = ["{}={!r}".format(key, value) for key, value in self.__dict__.items()] + return "{}({})".format(self.__class__.__name__, ", ".join(L)) + + def __eq__(self, other): + return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ + + def __ne__(self, other): + return not (self == other) + + +class LockResponse: + """ + Attributes: + - lockid + - state + - errorMessage + + """ + + def __init__( + self, + lockid=None, + state=None, + errorMessage=None, + ): + self.lockid = lockid + self.state = state + self.errorMessage = errorMessage + + def read(self, iprot): + if ( + iprot._fast_decode is not None + and isinstance(iprot.trans, TTransport.CReadableTransport) + and self.thrift_spec is not None + ): + iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec]) + return + iprot.readStructBegin() + while True: + (fname, ftype, fid) = iprot.readFieldBegin() + if ftype == TType.STOP: + break + if fid == 1: + if ftype == TType.I64: + self.lockid = iprot.readI64() + else: + iprot.skip(ftype) + elif fid == 2: + if ftype == TType.I32: + self.state = iprot.readI32() + else: + iprot.skip(ftype) + elif fid == 3: + if ftype == TType.STRING: + self.errorMessage = ( + iprot.readString().decode("utf-8", errors="replace") if sys.version_info[0] == 2 else iprot.readString() + ) + else: + iprot.skip(ftype) + else: + iprot.skip(ftype) + iprot.readFieldEnd() + iprot.readStructEnd() + + def write(self, oprot): + if oprot._fast_encode is not None and self.thrift_spec is not None: + oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec])) + return + oprot.writeStructBegin("LockResponse") + if self.lockid is not None: + oprot.writeFieldBegin("lockid", TType.I64, 1) + oprot.writeI64(self.lockid) + oprot.writeFieldEnd() + if self.state is not None: + oprot.writeFieldBegin("state", TType.I32, 2) + oprot.writeI32(self.state) + oprot.writeFieldEnd() + if self.errorMessage is not None: + oprot.writeFieldBegin("errorMessage", TType.STRING, 3) + oprot.writeString(self.errorMessage.encode("utf-8") if sys.version_info[0] == 2 else self.errorMessage) + oprot.writeFieldEnd() + oprot.writeFieldStop() + oprot.writeStructEnd() + + def validate(self): + if self.lockid is None: + raise TProtocolException(message="Required field lockid is unset!") + if self.state is None: + raise TProtocolException(message="Required field state is unset!") + return + + def __repr__(self): + L = ["{}={!r}".format(key, value) for key, value in self.__dict__.items()] + return "{}({})".format(self.__class__.__name__, ", ".join(L)) + + def __eq__(self, other): + return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ + + def __ne__(self, other): + return not (self == other) + + +class CheckLockRequest: + """ + Attributes: + - lockid + - txnid + - elapsed_ms + + """ + + def __init__( + self, + lockid=None, + txnid=None, + elapsed_ms=None, + ): + self.lockid = lockid + self.txnid = txnid + self.elapsed_ms = elapsed_ms + + def read(self, iprot): + if ( + iprot._fast_decode is not None + and isinstance(iprot.trans, TTransport.CReadableTransport) + and self.thrift_spec is not None + ): + iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec]) + return + iprot.readStructBegin() + while True: + (fname, ftype, fid) = iprot.readFieldBegin() + if ftype == TType.STOP: + break + if fid == 1: + if ftype == TType.I64: + self.lockid = iprot.readI64() + else: + iprot.skip(ftype) + elif fid == 2: + if ftype == TType.I64: + self.txnid = iprot.readI64() + else: + iprot.skip(ftype) + elif fid == 3: + if ftype == TType.I64: + self.elapsed_ms = iprot.readI64() + else: + iprot.skip(ftype) + else: + iprot.skip(ftype) + iprot.readFieldEnd() + iprot.readStructEnd() + + def write(self, oprot): + if oprot._fast_encode is not None and self.thrift_spec is not None: + oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec])) + return + oprot.writeStructBegin("CheckLockRequest") + if self.lockid is not None: + oprot.writeFieldBegin("lockid", TType.I64, 1) + oprot.writeI64(self.lockid) + oprot.writeFieldEnd() + if self.txnid is not None: + oprot.writeFieldBegin("txnid", TType.I64, 2) + oprot.writeI64(self.txnid) + oprot.writeFieldEnd() + if self.elapsed_ms is not None: + oprot.writeFieldBegin("elapsed_ms", TType.I64, 3) + oprot.writeI64(self.elapsed_ms) + oprot.writeFieldEnd() + oprot.writeFieldStop() + oprot.writeStructEnd() + + def validate(self): + if self.lockid is None: + raise TProtocolException(message="Required field lockid is unset!") + return + + def __repr__(self): + L = ["{}={!r}".format(key, value) for key, value in self.__dict__.items()] + return "{}({})".format(self.__class__.__name__, ", ".join(L)) + + def __eq__(self, other): + return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ + + def __ne__(self, other): + return not (self == other) + + +class UnlockRequest: + """ + Attributes: + - lockid + + """ + + def __init__( + self, + lockid=None, + ): + self.lockid = lockid + + def read(self, iprot): + if ( + iprot._fast_decode is not None + and isinstance(iprot.trans, TTransport.CReadableTransport) + and self.thrift_spec is not None + ): + iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec]) + return + iprot.readStructBegin() + while True: + (fname, ftype, fid) = iprot.readFieldBegin() + if ftype == TType.STOP: + break + if fid == 1: + if ftype == TType.I64: + self.lockid = iprot.readI64() + else: + iprot.skip(ftype) + else: + iprot.skip(ftype) + iprot.readFieldEnd() + iprot.readStructEnd() + + def write(self, oprot): + if oprot._fast_encode is not None and self.thrift_spec is not None: + oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec])) + return + oprot.writeStructBegin("UnlockRequest") + if self.lockid is not None: + oprot.writeFieldBegin("lockid", TType.I64, 1) + oprot.writeI64(self.lockid) + oprot.writeFieldEnd() + oprot.writeFieldStop() + oprot.writeStructEnd() + + def validate(self): + if self.lockid is None: + raise TProtocolException(message="Required field lockid is unset!") + return + + def __repr__(self): + L = ["{}={!r}".format(key, value) for key, value in self.__dict__.items()] + return "{}({})".format(self.__class__.__name__, ", ".join(L)) + + def __eq__(self, other): + return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ + + def __ne__(self, other): + return not (self == other) + + +class ShowLocksRequest: + """ + Attributes: + - dbname + - tablename + - partname + - isExtended + - txnid + + """ + + def __init__( + self, + dbname=None, + tablename=None, + partname=None, + isExtended=False, + txnid=None, + ): + self.dbname = dbname + self.tablename = tablename + self.partname = partname + self.isExtended = isExtended + self.txnid = txnid + + def read(self, iprot): + if ( + iprot._fast_decode is not None + and isinstance(iprot.trans, TTransport.CReadableTransport) + and self.thrift_spec is not None + ): + iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec]) + return + iprot.readStructBegin() + while True: + (fname, ftype, fid) = iprot.readFieldBegin() + if ftype == TType.STOP: + break + if fid == 1: + if ftype == TType.STRING: + self.dbname = ( + iprot.readString().decode("utf-8", errors="replace") if sys.version_info[0] == 2 else iprot.readString() + ) + else: + iprot.skip(ftype) + elif fid == 2: + if ftype == TType.STRING: + self.tablename = ( + iprot.readString().decode("utf-8", errors="replace") if sys.version_info[0] == 2 else iprot.readString() + ) + else: + iprot.skip(ftype) + elif fid == 3: + if ftype == TType.STRING: + self.partname = ( + iprot.readString().decode("utf-8", errors="replace") if sys.version_info[0] == 2 else iprot.readString() + ) + else: + iprot.skip(ftype) + elif fid == 4: + if ftype == TType.BOOL: + self.isExtended = iprot.readBool() + else: + iprot.skip(ftype) + elif fid == 5: + if ftype == TType.I64: + self.txnid = iprot.readI64() + else: + iprot.skip(ftype) + else: + iprot.skip(ftype) + iprot.readFieldEnd() + iprot.readStructEnd() + + def write(self, oprot): + if oprot._fast_encode is not None and self.thrift_spec is not None: + oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec])) + return + oprot.writeStructBegin("ShowLocksRequest") + if self.dbname is not None: + oprot.writeFieldBegin("dbname", TType.STRING, 1) + oprot.writeString(self.dbname.encode("utf-8") if sys.version_info[0] == 2 else self.dbname) + oprot.writeFieldEnd() + if self.tablename is not None: + oprot.writeFieldBegin("tablename", TType.STRING, 2) + oprot.writeString(self.tablename.encode("utf-8") if sys.version_info[0] == 2 else self.tablename) + oprot.writeFieldEnd() + if self.partname is not None: + oprot.writeFieldBegin("partname", TType.STRING, 3) + oprot.writeString(self.partname.encode("utf-8") if sys.version_info[0] == 2 else self.partname) + oprot.writeFieldEnd() + if self.isExtended is not None: + oprot.writeFieldBegin("isExtended", TType.BOOL, 4) + oprot.writeBool(self.isExtended) + oprot.writeFieldEnd() + if self.txnid is not None: + oprot.writeFieldBegin("txnid", TType.I64, 5) + oprot.writeI64(self.txnid) + oprot.writeFieldEnd() + oprot.writeFieldStop() + oprot.writeStructEnd() + + def validate(self): + return + + def __repr__(self): + L = ["{}={!r}".format(key, value) for key, value in self.__dict__.items()] + return "{}({})".format(self.__class__.__name__, ", ".join(L)) + + def __eq__(self, other): + return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ + + def __ne__(self, other): + return not (self == other) + + +class ShowLocksResponseElement: + """ + Attributes: + - lockid + - dbname + - tablename + - partname + - state + - type + - txnid + - lastheartbeat + - acquiredat + - user + - hostname + - heartbeatCount + - agentInfo + - blockedByExtId + - blockedByIntId + - lockIdInternal + + """ + + def __init__( + self, + lockid=None, + dbname=None, + tablename=None, + partname=None, + state=None, + type=None, + txnid=None, + lastheartbeat=None, + acquiredat=None, + user=None, + hostname=None, + heartbeatCount=0, + agentInfo=None, + blockedByExtId=None, + blockedByIntId=None, + lockIdInternal=None, + ): + self.lockid = lockid + self.dbname = dbname + self.tablename = tablename + self.partname = partname + self.state = state + self.type = type + self.txnid = txnid + self.lastheartbeat = lastheartbeat + self.acquiredat = acquiredat + self.user = user + self.hostname = hostname + self.heartbeatCount = heartbeatCount + self.agentInfo = agentInfo + self.blockedByExtId = blockedByExtId + self.blockedByIntId = blockedByIntId + self.lockIdInternal = lockIdInternal + + def read(self, iprot): + if ( + iprot._fast_decode is not None + and isinstance(iprot.trans, TTransport.CReadableTransport) + and self.thrift_spec is not None + ): + iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec]) + return + iprot.readStructBegin() + while True: + (fname, ftype, fid) = iprot.readFieldBegin() + if ftype == TType.STOP: + break + if fid == 1: + if ftype == TType.I64: + self.lockid = iprot.readI64() + else: + iprot.skip(ftype) + elif fid == 2: + if ftype == TType.STRING: + self.dbname = ( + iprot.readString().decode("utf-8", errors="replace") if sys.version_info[0] == 2 else iprot.readString() + ) + else: + iprot.skip(ftype) + elif fid == 3: + if ftype == TType.STRING: + self.tablename = ( + iprot.readString().decode("utf-8", errors="replace") if sys.version_info[0] == 2 else iprot.readString() + ) + else: + iprot.skip(ftype) + elif fid == 4: + if ftype == TType.STRING: + self.partname = ( + iprot.readString().decode("utf-8", errors="replace") if sys.version_info[0] == 2 else iprot.readString() + ) + else: + iprot.skip(ftype) + elif fid == 5: + if ftype == TType.I32: + self.state = iprot.readI32() + else: + iprot.skip(ftype) + elif fid == 6: + if ftype == TType.I32: + self.type = iprot.readI32() + else: + iprot.skip(ftype) + elif fid == 7: + if ftype == TType.I64: + self.txnid = iprot.readI64() + else: + iprot.skip(ftype) + elif fid == 8: + if ftype == TType.I64: + self.lastheartbeat = iprot.readI64() + else: + iprot.skip(ftype) + elif fid == 9: + if ftype == TType.I64: + self.acquiredat = iprot.readI64() + else: + iprot.skip(ftype) + elif fid == 10: + if ftype == TType.STRING: + self.user = ( + iprot.readString().decode("utf-8", errors="replace") if sys.version_info[0] == 2 else iprot.readString() + ) + else: + iprot.skip(ftype) + elif fid == 11: + if ftype == TType.STRING: + self.hostname = ( + iprot.readString().decode("utf-8", errors="replace") if sys.version_info[0] == 2 else iprot.readString() + ) + else: + iprot.skip(ftype) + elif fid == 12: + if ftype == TType.I32: + self.heartbeatCount = iprot.readI32() + else: + iprot.skip(ftype) + elif fid == 13: + if ftype == TType.STRING: + self.agentInfo = ( + iprot.readString().decode("utf-8", errors="replace") if sys.version_info[0] == 2 else iprot.readString() + ) + else: + iprot.skip(ftype) + elif fid == 14: + if ftype == TType.I64: + self.blockedByExtId = iprot.readI64() + else: + iprot.skip(ftype) + elif fid == 15: + if ftype == TType.I64: + self.blockedByIntId = iprot.readI64() + else: + iprot.skip(ftype) + elif fid == 16: + if ftype == TType.I64: + self.lockIdInternal = iprot.readI64() + else: + iprot.skip(ftype) + else: + iprot.skip(ftype) + iprot.readFieldEnd() + iprot.readStructEnd() + + def write(self, oprot): + if oprot._fast_encode is not None and self.thrift_spec is not None: + oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec])) + return + oprot.writeStructBegin("ShowLocksResponseElement") + if self.lockid is not None: + oprot.writeFieldBegin("lockid", TType.I64, 1) + oprot.writeI64(self.lockid) + oprot.writeFieldEnd() + if self.dbname is not None: + oprot.writeFieldBegin("dbname", TType.STRING, 2) + oprot.writeString(self.dbname.encode("utf-8") if sys.version_info[0] == 2 else self.dbname) + oprot.writeFieldEnd() + if self.tablename is not None: + oprot.writeFieldBegin("tablename", TType.STRING, 3) + oprot.writeString(self.tablename.encode("utf-8") if sys.version_info[0] == 2 else self.tablename) + oprot.writeFieldEnd() + if self.partname is not None: + oprot.writeFieldBegin("partname", TType.STRING, 4) + oprot.writeString(self.partname.encode("utf-8") if sys.version_info[0] == 2 else self.partname) + oprot.writeFieldEnd() + if self.state is not None: + oprot.writeFieldBegin("state", TType.I32, 5) + oprot.writeI32(self.state) + oprot.writeFieldEnd() + if self.type is not None: + oprot.writeFieldBegin("type", TType.I32, 6) + oprot.writeI32(self.type) + oprot.writeFieldEnd() + if self.txnid is not None: + oprot.writeFieldBegin("txnid", TType.I64, 7) + oprot.writeI64(self.txnid) + oprot.writeFieldEnd() + if self.lastheartbeat is not None: + oprot.writeFieldBegin("lastheartbeat", TType.I64, 8) + oprot.writeI64(self.lastheartbeat) + oprot.writeFieldEnd() + if self.acquiredat is not None: + oprot.writeFieldBegin("acquiredat", TType.I64, 9) + oprot.writeI64(self.acquiredat) + oprot.writeFieldEnd() + if self.user is not None: + oprot.writeFieldBegin("user", TType.STRING, 10) + oprot.writeString(self.user.encode("utf-8") if sys.version_info[0] == 2 else self.user) + oprot.writeFieldEnd() + if self.hostname is not None: + oprot.writeFieldBegin("hostname", TType.STRING, 11) + oprot.writeString(self.hostname.encode("utf-8") if sys.version_info[0] == 2 else self.hostname) + oprot.writeFieldEnd() + if self.heartbeatCount is not None: + oprot.writeFieldBegin("heartbeatCount", TType.I32, 12) + oprot.writeI32(self.heartbeatCount) + oprot.writeFieldEnd() + if self.agentInfo is not None: + oprot.writeFieldBegin("agentInfo", TType.STRING, 13) + oprot.writeString(self.agentInfo.encode("utf-8") if sys.version_info[0] == 2 else self.agentInfo) + oprot.writeFieldEnd() + if self.blockedByExtId is not None: + oprot.writeFieldBegin("blockedByExtId", TType.I64, 14) + oprot.writeI64(self.blockedByExtId) + oprot.writeFieldEnd() + if self.blockedByIntId is not None: + oprot.writeFieldBegin("blockedByIntId", TType.I64, 15) + oprot.writeI64(self.blockedByIntId) + oprot.writeFieldEnd() + if self.lockIdInternal is not None: + oprot.writeFieldBegin("lockIdInternal", TType.I64, 16) + oprot.writeI64(self.lockIdInternal) + oprot.writeFieldEnd() + oprot.writeFieldStop() + oprot.writeStructEnd() + + def validate(self): + if self.lockid is None: + raise TProtocolException(message="Required field lockid is unset!") + if self.dbname is None: + raise TProtocolException(message="Required field dbname is unset!") + if self.state is None: + raise TProtocolException(message="Required field state is unset!") + if self.type is None: + raise TProtocolException(message="Required field type is unset!") + if self.lastheartbeat is None: + raise TProtocolException(message="Required field lastheartbeat is unset!") + if self.user is None: + raise TProtocolException(message="Required field user is unset!") + if self.hostname is None: + raise TProtocolException(message="Required field hostname is unset!") + return + + def __repr__(self): + L = ["{}={!r}".format(key, value) for key, value in self.__dict__.items()] + return "{}({})".format(self.__class__.__name__, ", ".join(L)) + + def __eq__(self, other): + return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ + + def __ne__(self, other): + return not (self == other) + + +class ShowLocksResponse: + """ + Attributes: + - locks + + """ + + def __init__( + self, + locks=None, + ): + self.locks = locks + + def read(self, iprot): + if ( + iprot._fast_decode is not None + and isinstance(iprot.trans, TTransport.CReadableTransport) + and self.thrift_spec is not None + ): + iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec]) + return + iprot.readStructBegin() + while True: + (fname, ftype, fid) = iprot.readFieldBegin() + if ftype == TType.STOP: + break + if fid == 1: + if ftype == TType.LIST: + self.locks = [] + (_etype733, _size730) = iprot.readListBegin() + for _i734 in range(_size730): + _elem735 = ShowLocksResponseElement() + _elem735.read(iprot) + self.locks.append(_elem735) + iprot.readListEnd() + else: + iprot.skip(ftype) + else: + iprot.skip(ftype) + iprot.readFieldEnd() + iprot.readStructEnd() + + def write(self, oprot): + if oprot._fast_encode is not None and self.thrift_spec is not None: + oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec])) + return + oprot.writeStructBegin("ShowLocksResponse") + if self.locks is not None: + oprot.writeFieldBegin("locks", TType.LIST, 1) + oprot.writeListBegin(TType.STRUCT, len(self.locks)) + for iter736 in self.locks: + iter736.write(oprot) + oprot.writeListEnd() + oprot.writeFieldEnd() + oprot.writeFieldStop() + oprot.writeStructEnd() + + def validate(self): + return + + def __repr__(self): + L = ["{}={!r}".format(key, value) for key, value in self.__dict__.items()] + return "{}({})".format(self.__class__.__name__, ", ".join(L)) + + def __eq__(self, other): + return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ + + def __ne__(self, other): + return not (self == other) + + +class HeartbeatRequest: + """ + Attributes: + - lockid + - txnid + + """ + + def __init__( + self, + lockid=None, + txnid=None, + ): + self.lockid = lockid + self.txnid = txnid + + def read(self, iprot): + if ( + iprot._fast_decode is not None + and isinstance(iprot.trans, TTransport.CReadableTransport) + and self.thrift_spec is not None + ): + iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec]) + return + iprot.readStructBegin() + while True: + (fname, ftype, fid) = iprot.readFieldBegin() + if ftype == TType.STOP: + break + if fid == 1: + if ftype == TType.I64: + self.lockid = iprot.readI64() + else: + iprot.skip(ftype) + elif fid == 2: + if ftype == TType.I64: + self.txnid = iprot.readI64() + else: + iprot.skip(ftype) + else: + iprot.skip(ftype) + iprot.readFieldEnd() + iprot.readStructEnd() + + def write(self, oprot): + if oprot._fast_encode is not None and self.thrift_spec is not None: + oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec])) + return + oprot.writeStructBegin("HeartbeatRequest") + if self.lockid is not None: + oprot.writeFieldBegin("lockid", TType.I64, 1) + oprot.writeI64(self.lockid) + oprot.writeFieldEnd() + if self.txnid is not None: + oprot.writeFieldBegin("txnid", TType.I64, 2) + oprot.writeI64(self.txnid) + oprot.writeFieldEnd() + oprot.writeFieldStop() + oprot.writeStructEnd() + + def validate(self): + return + + def __repr__(self): + L = ["{}={!r}".format(key, value) for key, value in self.__dict__.items()] + return "{}({})".format(self.__class__.__name__, ", ".join(L)) + + def __eq__(self, other): + return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ + + def __ne__(self, other): + return not (self == other) + + +class HeartbeatTxnRangeRequest: + """ + Attributes: + - min + - max + + """ + + def __init__( + self, + min=None, + max=None, + ): + self.min = min + self.max = max + + def read(self, iprot): + if ( + iprot._fast_decode is not None + and isinstance(iprot.trans, TTransport.CReadableTransport) + and self.thrift_spec is not None + ): + iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec]) + return + iprot.readStructBegin() + while True: + (fname, ftype, fid) = iprot.readFieldBegin() + if ftype == TType.STOP: + break + if fid == 1: + if ftype == TType.I64: + self.min = iprot.readI64() + else: + iprot.skip(ftype) + elif fid == 2: + if ftype == TType.I64: + self.max = iprot.readI64() + else: + iprot.skip(ftype) + else: + iprot.skip(ftype) + iprot.readFieldEnd() + iprot.readStructEnd() + + def write(self, oprot): + if oprot._fast_encode is not None and self.thrift_spec is not None: + oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec])) + return + oprot.writeStructBegin("HeartbeatTxnRangeRequest") + if self.min is not None: + oprot.writeFieldBegin("min", TType.I64, 1) + oprot.writeI64(self.min) + oprot.writeFieldEnd() + if self.max is not None: + oprot.writeFieldBegin("max", TType.I64, 2) + oprot.writeI64(self.max) + oprot.writeFieldEnd() + oprot.writeFieldStop() + oprot.writeStructEnd() + + def validate(self): + if self.min is None: + raise TProtocolException(message="Required field min is unset!") + if self.max is None: + raise TProtocolException(message="Required field max is unset!") + return + + def __repr__(self): + L = ["{}={!r}".format(key, value) for key, value in self.__dict__.items()] + return "{}({})".format(self.__class__.__name__, ", ".join(L)) + + def __eq__(self, other): + return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ + + def __ne__(self, other): + return not (self == other) + + +class HeartbeatTxnRangeResponse: + """ + Attributes: + - aborted + - nosuch + + """ + + def __init__( + self, + aborted=None, + nosuch=None, + ): + self.aborted = aborted + self.nosuch = nosuch + + def read(self, iprot): + if ( + iprot._fast_decode is not None + and isinstance(iprot.trans, TTransport.CReadableTransport) + and self.thrift_spec is not None + ): + iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec]) + return + iprot.readStructBegin() + while True: + (fname, ftype, fid) = iprot.readFieldBegin() + if ftype == TType.STOP: + break + if fid == 1: + if ftype == TType.SET: + self.aborted = set() + (_etype740, _size737) = iprot.readSetBegin() + for _i741 in range(_size737): + _elem742 = iprot.readI64() + self.aborted.add(_elem742) + iprot.readSetEnd() + else: + iprot.skip(ftype) + elif fid == 2: + if ftype == TType.SET: + self.nosuch = set() + (_etype746, _size743) = iprot.readSetBegin() + for _i747 in range(_size743): + _elem748 = iprot.readI64() + self.nosuch.add(_elem748) + iprot.readSetEnd() + else: + iprot.skip(ftype) + else: + iprot.skip(ftype) + iprot.readFieldEnd() + iprot.readStructEnd() + + def write(self, oprot): + if oprot._fast_encode is not None and self.thrift_spec is not None: + oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec])) + return + oprot.writeStructBegin("HeartbeatTxnRangeResponse") + if self.aborted is not None: + oprot.writeFieldBegin("aborted", TType.SET, 1) + oprot.writeSetBegin(TType.I64, len(self.aborted)) + for iter749 in self.aborted: + oprot.writeI64(iter749) + oprot.writeSetEnd() + oprot.writeFieldEnd() + if self.nosuch is not None: + oprot.writeFieldBegin("nosuch", TType.SET, 2) + oprot.writeSetBegin(TType.I64, len(self.nosuch)) + for iter750 in self.nosuch: + oprot.writeI64(iter750) + oprot.writeSetEnd() + oprot.writeFieldEnd() + oprot.writeFieldStop() + oprot.writeStructEnd() + + def validate(self): + if self.aborted is None: + raise TProtocolException(message="Required field aborted is unset!") + if self.nosuch is None: + raise TProtocolException(message="Required field nosuch is unset!") + return + + def __repr__(self): + L = ["{}={!r}".format(key, value) for key, value in self.__dict__.items()] + return "{}({})".format(self.__class__.__name__, ", ".join(L)) + + def __eq__(self, other): + return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ + + def __ne__(self, other): + return not (self == other) + + +class CompactionRequest: + """ + Attributes: + - dbname + - tablename + - partitionname + - type + - runas + - properties + - initiatorId + - initiatorVersion + + """ + + def __init__( + self, + dbname=None, + tablename=None, + partitionname=None, + type=None, + runas=None, + properties=None, + initiatorId=None, + initiatorVersion=None, + ): + self.dbname = dbname + self.tablename = tablename + self.partitionname = partitionname + self.type = type + self.runas = runas + self.properties = properties + self.initiatorId = initiatorId + self.initiatorVersion = initiatorVersion + + def read(self, iprot): + if ( + iprot._fast_decode is not None + and isinstance(iprot.trans, TTransport.CReadableTransport) + and self.thrift_spec is not None + ): + iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec]) + return + iprot.readStructBegin() + while True: + (fname, ftype, fid) = iprot.readFieldBegin() + if ftype == TType.STOP: + break + if fid == 1: + if ftype == TType.STRING: + self.dbname = ( + iprot.readString().decode("utf-8", errors="replace") if sys.version_info[0] == 2 else iprot.readString() + ) + else: + iprot.skip(ftype) + elif fid == 2: + if ftype == TType.STRING: + self.tablename = ( + iprot.readString().decode("utf-8", errors="replace") if sys.version_info[0] == 2 else iprot.readString() + ) + else: + iprot.skip(ftype) + elif fid == 3: + if ftype == TType.STRING: + self.partitionname = ( + iprot.readString().decode("utf-8", errors="replace") if sys.version_info[0] == 2 else iprot.readString() + ) + else: + iprot.skip(ftype) + elif fid == 4: + if ftype == TType.I32: + self.type = iprot.readI32() + else: + iprot.skip(ftype) + elif fid == 5: + if ftype == TType.STRING: + self.runas = ( + iprot.readString().decode("utf-8", errors="replace") if sys.version_info[0] == 2 else iprot.readString() + ) + else: + iprot.skip(ftype) + elif fid == 6: + if ftype == TType.MAP: + self.properties = {} + (_ktype752, _vtype753, _size751) = iprot.readMapBegin() + for _i755 in range(_size751): + _key756 = ( + iprot.readString().decode("utf-8", errors="replace") + if sys.version_info[0] == 2 + else iprot.readString() + ) + _val757 = ( + iprot.readString().decode("utf-8", errors="replace") + if sys.version_info[0] == 2 + else iprot.readString() + ) + self.properties[_key756] = _val757 + iprot.readMapEnd() + else: + iprot.skip(ftype) + elif fid == 7: + if ftype == TType.STRING: + self.initiatorId = ( + iprot.readString().decode("utf-8", errors="replace") if sys.version_info[0] == 2 else iprot.readString() + ) + else: + iprot.skip(ftype) + elif fid == 8: + if ftype == TType.STRING: + self.initiatorVersion = ( + iprot.readString().decode("utf-8", errors="replace") if sys.version_info[0] == 2 else iprot.readString() + ) + else: + iprot.skip(ftype) + else: + iprot.skip(ftype) + iprot.readFieldEnd() + iprot.readStructEnd() + + def write(self, oprot): + if oprot._fast_encode is not None and self.thrift_spec is not None: + oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec])) + return + oprot.writeStructBegin("CompactionRequest") + if self.dbname is not None: + oprot.writeFieldBegin("dbname", TType.STRING, 1) + oprot.writeString(self.dbname.encode("utf-8") if sys.version_info[0] == 2 else self.dbname) + oprot.writeFieldEnd() + if self.tablename is not None: + oprot.writeFieldBegin("tablename", TType.STRING, 2) + oprot.writeString(self.tablename.encode("utf-8") if sys.version_info[0] == 2 else self.tablename) + oprot.writeFieldEnd() + if self.partitionname is not None: + oprot.writeFieldBegin("partitionname", TType.STRING, 3) + oprot.writeString(self.partitionname.encode("utf-8") if sys.version_info[0] == 2 else self.partitionname) + oprot.writeFieldEnd() + if self.type is not None: + oprot.writeFieldBegin("type", TType.I32, 4) + oprot.writeI32(self.type) + oprot.writeFieldEnd() + if self.runas is not None: + oprot.writeFieldBegin("runas", TType.STRING, 5) + oprot.writeString(self.runas.encode("utf-8") if sys.version_info[0] == 2 else self.runas) + oprot.writeFieldEnd() + if self.properties is not None: + oprot.writeFieldBegin("properties", TType.MAP, 6) + oprot.writeMapBegin(TType.STRING, TType.STRING, len(self.properties)) + for kiter758, viter759 in self.properties.items(): + oprot.writeString(kiter758.encode("utf-8") if sys.version_info[0] == 2 else kiter758) + oprot.writeString(viter759.encode("utf-8") if sys.version_info[0] == 2 else viter759) + oprot.writeMapEnd() + oprot.writeFieldEnd() + if self.initiatorId is not None: + oprot.writeFieldBegin("initiatorId", TType.STRING, 7) + oprot.writeString(self.initiatorId.encode("utf-8") if sys.version_info[0] == 2 else self.initiatorId) + oprot.writeFieldEnd() + if self.initiatorVersion is not None: + oprot.writeFieldBegin("initiatorVersion", TType.STRING, 8) + oprot.writeString(self.initiatorVersion.encode("utf-8") if sys.version_info[0] == 2 else self.initiatorVersion) + oprot.writeFieldEnd() + oprot.writeFieldStop() + oprot.writeStructEnd() + + def validate(self): + if self.dbname is None: + raise TProtocolException(message="Required field dbname is unset!") + if self.tablename is None: + raise TProtocolException(message="Required field tablename is unset!") + if self.type is None: + raise TProtocolException(message="Required field type is unset!") + return + + def __repr__(self): + L = ["{}={!r}".format(key, value) for key, value in self.__dict__.items()] + return "{}({})".format(self.__class__.__name__, ", ".join(L)) + + def __eq__(self, other): + return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ + + def __ne__(self, other): + return not (self == other) + + +class CompactionInfoStruct: + """ + Attributes: + - id + - dbname + - tablename + - partitionname + - type + - runas + - properties + - toomanyaborts + - state + - workerId + - start + - highestWriteId + - errorMessage + - hasoldabort + - enqueueTime + - retryRetention + + """ + + def __init__( + self, + id=None, + dbname=None, + tablename=None, + partitionname=None, + type=None, + runas=None, + properties=None, + toomanyaborts=None, + state=None, + workerId=None, + start=None, + highestWriteId=None, + errorMessage=None, + hasoldabort=None, + enqueueTime=None, + retryRetention=None, + ): + self.id = id + self.dbname = dbname + self.tablename = tablename + self.partitionname = partitionname + self.type = type + self.runas = runas + self.properties = properties + self.toomanyaborts = toomanyaborts + self.state = state + self.workerId = workerId + self.start = start + self.highestWriteId = highestWriteId + self.errorMessage = errorMessage + self.hasoldabort = hasoldabort + self.enqueueTime = enqueueTime + self.retryRetention = retryRetention + + def read(self, iprot): + if ( + iprot._fast_decode is not None + and isinstance(iprot.trans, TTransport.CReadableTransport) + and self.thrift_spec is not None + ): + iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec]) + return + iprot.readStructBegin() + while True: + (fname, ftype, fid) = iprot.readFieldBegin() + if ftype == TType.STOP: + break + if fid == 1: + if ftype == TType.I64: + self.id = iprot.readI64() + else: + iprot.skip(ftype) + elif fid == 2: + if ftype == TType.STRING: + self.dbname = ( + iprot.readString().decode("utf-8", errors="replace") if sys.version_info[0] == 2 else iprot.readString() + ) + else: + iprot.skip(ftype) + elif fid == 3: + if ftype == TType.STRING: + self.tablename = ( + iprot.readString().decode("utf-8", errors="replace") if sys.version_info[0] == 2 else iprot.readString() + ) + else: + iprot.skip(ftype) + elif fid == 4: + if ftype == TType.STRING: + self.partitionname = ( + iprot.readString().decode("utf-8", errors="replace") if sys.version_info[0] == 2 else iprot.readString() + ) + else: + iprot.skip(ftype) + elif fid == 5: + if ftype == TType.I32: + self.type = iprot.readI32() + else: + iprot.skip(ftype) + elif fid == 6: + if ftype == TType.STRING: + self.runas = ( + iprot.readString().decode("utf-8", errors="replace") if sys.version_info[0] == 2 else iprot.readString() + ) + else: + iprot.skip(ftype) + elif fid == 7: + if ftype == TType.STRING: + self.properties = ( + iprot.readString().decode("utf-8", errors="replace") if sys.version_info[0] == 2 else iprot.readString() + ) + else: + iprot.skip(ftype) + elif fid == 8: + if ftype == TType.BOOL: + self.toomanyaborts = iprot.readBool() + else: + iprot.skip(ftype) + elif fid == 9: + if ftype == TType.STRING: + self.state = ( + iprot.readString().decode("utf-8", errors="replace") if sys.version_info[0] == 2 else iprot.readString() + ) + else: + iprot.skip(ftype) + elif fid == 10: + if ftype == TType.STRING: + self.workerId = ( + iprot.readString().decode("utf-8", errors="replace") if sys.version_info[0] == 2 else iprot.readString() + ) + else: + iprot.skip(ftype) + elif fid == 11: + if ftype == TType.I64: + self.start = iprot.readI64() + else: + iprot.skip(ftype) + elif fid == 12: + if ftype == TType.I64: + self.highestWriteId = iprot.readI64() + else: + iprot.skip(ftype) + elif fid == 13: + if ftype == TType.STRING: + self.errorMessage = ( + iprot.readString().decode("utf-8", errors="replace") if sys.version_info[0] == 2 else iprot.readString() + ) + else: + iprot.skip(ftype) + elif fid == 14: + if ftype == TType.BOOL: + self.hasoldabort = iprot.readBool() + else: + iprot.skip(ftype) + elif fid == 15: + if ftype == TType.I64: + self.enqueueTime = iprot.readI64() + else: + iprot.skip(ftype) + elif fid == 16: + if ftype == TType.I64: + self.retryRetention = iprot.readI64() + else: + iprot.skip(ftype) + else: + iprot.skip(ftype) + iprot.readFieldEnd() + iprot.readStructEnd() + + def write(self, oprot): + if oprot._fast_encode is not None and self.thrift_spec is not None: + oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec])) + return + oprot.writeStructBegin("CompactionInfoStruct") + if self.id is not None: + oprot.writeFieldBegin("id", TType.I64, 1) + oprot.writeI64(self.id) + oprot.writeFieldEnd() + if self.dbname is not None: + oprot.writeFieldBegin("dbname", TType.STRING, 2) + oprot.writeString(self.dbname.encode("utf-8") if sys.version_info[0] == 2 else self.dbname) + oprot.writeFieldEnd() + if self.tablename is not None: + oprot.writeFieldBegin("tablename", TType.STRING, 3) + oprot.writeString(self.tablename.encode("utf-8") if sys.version_info[0] == 2 else self.tablename) + oprot.writeFieldEnd() + if self.partitionname is not None: + oprot.writeFieldBegin("partitionname", TType.STRING, 4) + oprot.writeString(self.partitionname.encode("utf-8") if sys.version_info[0] == 2 else self.partitionname) + oprot.writeFieldEnd() + if self.type is not None: + oprot.writeFieldBegin("type", TType.I32, 5) + oprot.writeI32(self.type) + oprot.writeFieldEnd() + if self.runas is not None: + oprot.writeFieldBegin("runas", TType.STRING, 6) + oprot.writeString(self.runas.encode("utf-8") if sys.version_info[0] == 2 else self.runas) + oprot.writeFieldEnd() + if self.properties is not None: + oprot.writeFieldBegin("properties", TType.STRING, 7) + oprot.writeString(self.properties.encode("utf-8") if sys.version_info[0] == 2 else self.properties) + oprot.writeFieldEnd() + if self.toomanyaborts is not None: + oprot.writeFieldBegin("toomanyaborts", TType.BOOL, 8) + oprot.writeBool(self.toomanyaborts) + oprot.writeFieldEnd() + if self.state is not None: + oprot.writeFieldBegin("state", TType.STRING, 9) + oprot.writeString(self.state.encode("utf-8") if sys.version_info[0] == 2 else self.state) + oprot.writeFieldEnd() + if self.workerId is not None: + oprot.writeFieldBegin("workerId", TType.STRING, 10) + oprot.writeString(self.workerId.encode("utf-8") if sys.version_info[0] == 2 else self.workerId) + oprot.writeFieldEnd() + if self.start is not None: + oprot.writeFieldBegin("start", TType.I64, 11) + oprot.writeI64(self.start) + oprot.writeFieldEnd() + if self.highestWriteId is not None: + oprot.writeFieldBegin("highestWriteId", TType.I64, 12) + oprot.writeI64(self.highestWriteId) + oprot.writeFieldEnd() + if self.errorMessage is not None: + oprot.writeFieldBegin("errorMessage", TType.STRING, 13) + oprot.writeString(self.errorMessage.encode("utf-8") if sys.version_info[0] == 2 else self.errorMessage) + oprot.writeFieldEnd() + if self.hasoldabort is not None: + oprot.writeFieldBegin("hasoldabort", TType.BOOL, 14) + oprot.writeBool(self.hasoldabort) + oprot.writeFieldEnd() + if self.enqueueTime is not None: + oprot.writeFieldBegin("enqueueTime", TType.I64, 15) + oprot.writeI64(self.enqueueTime) + oprot.writeFieldEnd() + if self.retryRetention is not None: + oprot.writeFieldBegin("retryRetention", TType.I64, 16) + oprot.writeI64(self.retryRetention) + oprot.writeFieldEnd() + oprot.writeFieldStop() + oprot.writeStructEnd() + + def validate(self): + if self.id is None: + raise TProtocolException(message="Required field id is unset!") + if self.dbname is None: + raise TProtocolException(message="Required field dbname is unset!") + if self.tablename is None: + raise TProtocolException(message="Required field tablename is unset!") + if self.type is None: + raise TProtocolException(message="Required field type is unset!") + return + + def __repr__(self): + L = ["{}={!r}".format(key, value) for key, value in self.__dict__.items()] + return "{}({})".format(self.__class__.__name__, ", ".join(L)) + + def __eq__(self, other): + return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ + + def __ne__(self, other): + return not (self == other) + + +class OptionalCompactionInfoStruct: + """ + Attributes: + - ci + + """ + + def __init__( + self, + ci=None, + ): + self.ci = ci + + def read(self, iprot): + if ( + iprot._fast_decode is not None + and isinstance(iprot.trans, TTransport.CReadableTransport) + and self.thrift_spec is not None + ): + iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec]) + return + iprot.readStructBegin() + while True: + (fname, ftype, fid) = iprot.readFieldBegin() + if ftype == TType.STOP: + break + if fid == 1: + if ftype == TType.STRUCT: + self.ci = CompactionInfoStruct() + self.ci.read(iprot) + else: + iprot.skip(ftype) + else: + iprot.skip(ftype) + iprot.readFieldEnd() + iprot.readStructEnd() + + def write(self, oprot): + if oprot._fast_encode is not None and self.thrift_spec is not None: + oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec])) + return + oprot.writeStructBegin("OptionalCompactionInfoStruct") + if self.ci is not None: + oprot.writeFieldBegin("ci", TType.STRUCT, 1) + self.ci.write(oprot) + oprot.writeFieldEnd() + oprot.writeFieldStop() + oprot.writeStructEnd() + + def validate(self): + return + + def __repr__(self): + L = ["{}={!r}".format(key, value) for key, value in self.__dict__.items()] + return "{}({})".format(self.__class__.__name__, ", ".join(L)) + + def __eq__(self, other): + return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ + + def __ne__(self, other): + return not (self == other) + + +class CompactionMetricsDataStruct: + """ + Attributes: + - dbname + - tblname + - partitionname + - type + - metricvalue + - version + - threshold + + """ + + def __init__( + self, + dbname=None, + tblname=None, + partitionname=None, + type=None, + metricvalue=None, + version=None, + threshold=None, + ): + self.dbname = dbname + self.tblname = tblname + self.partitionname = partitionname + self.type = type + self.metricvalue = metricvalue + self.version = version + self.threshold = threshold + + def read(self, iprot): + if ( + iprot._fast_decode is not None + and isinstance(iprot.trans, TTransport.CReadableTransport) + and self.thrift_spec is not None + ): + iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec]) + return + iprot.readStructBegin() + while True: + (fname, ftype, fid) = iprot.readFieldBegin() + if ftype == TType.STOP: + break + if fid == 1: + if ftype == TType.STRING: + self.dbname = ( + iprot.readString().decode("utf-8", errors="replace") if sys.version_info[0] == 2 else iprot.readString() + ) + else: + iprot.skip(ftype) + elif fid == 2: + if ftype == TType.STRING: + self.tblname = ( + iprot.readString().decode("utf-8", errors="replace") if sys.version_info[0] == 2 else iprot.readString() + ) + else: + iprot.skip(ftype) + elif fid == 3: + if ftype == TType.STRING: + self.partitionname = ( + iprot.readString().decode("utf-8", errors="replace") if sys.version_info[0] == 2 else iprot.readString() + ) + else: + iprot.skip(ftype) + elif fid == 4: + if ftype == TType.I32: + self.type = iprot.readI32() + else: + iprot.skip(ftype) + elif fid == 5: + if ftype == TType.I32: + self.metricvalue = iprot.readI32() + else: + iprot.skip(ftype) + elif fid == 6: + if ftype == TType.I32: + self.version = iprot.readI32() + else: + iprot.skip(ftype) + elif fid == 7: + if ftype == TType.I32: + self.threshold = iprot.readI32() + else: + iprot.skip(ftype) + else: + iprot.skip(ftype) + iprot.readFieldEnd() + iprot.readStructEnd() + + def write(self, oprot): + if oprot._fast_encode is not None and self.thrift_spec is not None: + oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec])) + return + oprot.writeStructBegin("CompactionMetricsDataStruct") + if self.dbname is not None: + oprot.writeFieldBegin("dbname", TType.STRING, 1) + oprot.writeString(self.dbname.encode("utf-8") if sys.version_info[0] == 2 else self.dbname) + oprot.writeFieldEnd() + if self.tblname is not None: + oprot.writeFieldBegin("tblname", TType.STRING, 2) + oprot.writeString(self.tblname.encode("utf-8") if sys.version_info[0] == 2 else self.tblname) + oprot.writeFieldEnd() + if self.partitionname is not None: + oprot.writeFieldBegin("partitionname", TType.STRING, 3) + oprot.writeString(self.partitionname.encode("utf-8") if sys.version_info[0] == 2 else self.partitionname) + oprot.writeFieldEnd() + if self.type is not None: + oprot.writeFieldBegin("type", TType.I32, 4) + oprot.writeI32(self.type) + oprot.writeFieldEnd() + if self.metricvalue is not None: + oprot.writeFieldBegin("metricvalue", TType.I32, 5) + oprot.writeI32(self.metricvalue) + oprot.writeFieldEnd() + if self.version is not None: + oprot.writeFieldBegin("version", TType.I32, 6) + oprot.writeI32(self.version) + oprot.writeFieldEnd() + if self.threshold is not None: + oprot.writeFieldBegin("threshold", TType.I32, 7) + oprot.writeI32(self.threshold) + oprot.writeFieldEnd() + oprot.writeFieldStop() + oprot.writeStructEnd() + + def validate(self): + if self.dbname is None: + raise TProtocolException(message="Required field dbname is unset!") + if self.tblname is None: + raise TProtocolException(message="Required field tblname is unset!") + if self.type is None: + raise TProtocolException(message="Required field type is unset!") + if self.metricvalue is None: + raise TProtocolException(message="Required field metricvalue is unset!") + if self.version is None: + raise TProtocolException(message="Required field version is unset!") + if self.threshold is None: + raise TProtocolException(message="Required field threshold is unset!") + return + + def __repr__(self): + L = ["{}={!r}".format(key, value) for key, value in self.__dict__.items()] + return "{}({})".format(self.__class__.__name__, ", ".join(L)) + + def __eq__(self, other): + return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ + + def __ne__(self, other): + return not (self == other) + + +class CompactionMetricsDataResponse: + """ + Attributes: + - data + + """ + + def __init__( + self, + data=None, + ): + self.data = data + + def read(self, iprot): + if ( + iprot._fast_decode is not None + and isinstance(iprot.trans, TTransport.CReadableTransport) + and self.thrift_spec is not None + ): + iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec]) + return + iprot.readStructBegin() + while True: + (fname, ftype, fid) = iprot.readFieldBegin() + if ftype == TType.STOP: + break + if fid == 1: + if ftype == TType.STRUCT: + self.data = CompactionMetricsDataStruct() + self.data.read(iprot) + else: + iprot.skip(ftype) + else: + iprot.skip(ftype) + iprot.readFieldEnd() + iprot.readStructEnd() + + def write(self, oprot): + if oprot._fast_encode is not None and self.thrift_spec is not None: + oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec])) + return + oprot.writeStructBegin("CompactionMetricsDataResponse") + if self.data is not None: + oprot.writeFieldBegin("data", TType.STRUCT, 1) + self.data.write(oprot) + oprot.writeFieldEnd() + oprot.writeFieldStop() + oprot.writeStructEnd() + + def validate(self): + return + + def __repr__(self): + L = ["{}={!r}".format(key, value) for key, value in self.__dict__.items()] + return "{}({})".format(self.__class__.__name__, ", ".join(L)) + + def __eq__(self, other): + return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ + + def __ne__(self, other): + return not (self == other) + + +class CompactionMetricsDataRequest: + """ + Attributes: + - dbName + - tblName + - partitionName + - type + + """ + + def __init__( + self, + dbName=None, + tblName=None, + partitionName=None, + type=None, + ): + self.dbName = dbName + self.tblName = tblName + self.partitionName = partitionName + self.type = type + + def read(self, iprot): + if ( + iprot._fast_decode is not None + and isinstance(iprot.trans, TTransport.CReadableTransport) + and self.thrift_spec is not None + ): + iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec]) + return + iprot.readStructBegin() + while True: + (fname, ftype, fid) = iprot.readFieldBegin() + if ftype == TType.STOP: + break + if fid == 1: + if ftype == TType.STRING: + self.dbName = ( + iprot.readString().decode("utf-8", errors="replace") if sys.version_info[0] == 2 else iprot.readString() + ) + else: + iprot.skip(ftype) + elif fid == 2: + if ftype == TType.STRING: + self.tblName = ( + iprot.readString().decode("utf-8", errors="replace") if sys.version_info[0] == 2 else iprot.readString() + ) + else: + iprot.skip(ftype) + elif fid == 3: + if ftype == TType.STRING: + self.partitionName = ( + iprot.readString().decode("utf-8", errors="replace") if sys.version_info[0] == 2 else iprot.readString() + ) + else: + iprot.skip(ftype) + elif fid == 4: + if ftype == TType.I32: + self.type = iprot.readI32() + else: + iprot.skip(ftype) + else: + iprot.skip(ftype) + iprot.readFieldEnd() + iprot.readStructEnd() + + def write(self, oprot): + if oprot._fast_encode is not None and self.thrift_spec is not None: + oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec])) + return + oprot.writeStructBegin("CompactionMetricsDataRequest") + if self.dbName is not None: + oprot.writeFieldBegin("dbName", TType.STRING, 1) + oprot.writeString(self.dbName.encode("utf-8") if sys.version_info[0] == 2 else self.dbName) + oprot.writeFieldEnd() + if self.tblName is not None: + oprot.writeFieldBegin("tblName", TType.STRING, 2) + oprot.writeString(self.tblName.encode("utf-8") if sys.version_info[0] == 2 else self.tblName) + oprot.writeFieldEnd() + if self.partitionName is not None: + oprot.writeFieldBegin("partitionName", TType.STRING, 3) + oprot.writeString(self.partitionName.encode("utf-8") if sys.version_info[0] == 2 else self.partitionName) + oprot.writeFieldEnd() + if self.type is not None: + oprot.writeFieldBegin("type", TType.I32, 4) + oprot.writeI32(self.type) + oprot.writeFieldEnd() + oprot.writeFieldStop() + oprot.writeStructEnd() + + def validate(self): + if self.dbName is None: + raise TProtocolException(message="Required field dbName is unset!") + if self.tblName is None: + raise TProtocolException(message="Required field tblName is unset!") + if self.type is None: + raise TProtocolException(message="Required field type is unset!") + return + + def __repr__(self): + L = ["{}={!r}".format(key, value) for key, value in self.__dict__.items()] + return "{}({})".format(self.__class__.__name__, ", ".join(L)) + + def __eq__(self, other): + return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ + + def __ne__(self, other): + return not (self == other) + + +class CompactionResponse: + """ + Attributes: + - id + - state + - accepted + - errormessage + + """ + + def __init__( + self, + id=None, + state=None, + accepted=None, + errormessage=None, + ): + self.id = id + self.state = state + self.accepted = accepted + self.errormessage = errormessage + + def read(self, iprot): + if ( + iprot._fast_decode is not None + and isinstance(iprot.trans, TTransport.CReadableTransport) + and self.thrift_spec is not None + ): + iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec]) + return + iprot.readStructBegin() + while True: + (fname, ftype, fid) = iprot.readFieldBegin() + if ftype == TType.STOP: + break + if fid == 1: + if ftype == TType.I64: + self.id = iprot.readI64() + else: + iprot.skip(ftype) + elif fid == 2: + if ftype == TType.STRING: + self.state = ( + iprot.readString().decode("utf-8", errors="replace") if sys.version_info[0] == 2 else iprot.readString() + ) + else: + iprot.skip(ftype) + elif fid == 3: + if ftype == TType.BOOL: + self.accepted = iprot.readBool() + else: + iprot.skip(ftype) + elif fid == 4: + if ftype == TType.STRING: + self.errormessage = ( + iprot.readString().decode("utf-8", errors="replace") if sys.version_info[0] == 2 else iprot.readString() + ) + else: + iprot.skip(ftype) + else: + iprot.skip(ftype) + iprot.readFieldEnd() + iprot.readStructEnd() + + def write(self, oprot): + if oprot._fast_encode is not None and self.thrift_spec is not None: + oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec])) + return + oprot.writeStructBegin("CompactionResponse") + if self.id is not None: + oprot.writeFieldBegin("id", TType.I64, 1) + oprot.writeI64(self.id) + oprot.writeFieldEnd() + if self.state is not None: + oprot.writeFieldBegin("state", TType.STRING, 2) + oprot.writeString(self.state.encode("utf-8") if sys.version_info[0] == 2 else self.state) + oprot.writeFieldEnd() + if self.accepted is not None: + oprot.writeFieldBegin("accepted", TType.BOOL, 3) + oprot.writeBool(self.accepted) + oprot.writeFieldEnd() + if self.errormessage is not None: + oprot.writeFieldBegin("errormessage", TType.STRING, 4) + oprot.writeString(self.errormessage.encode("utf-8") if sys.version_info[0] == 2 else self.errormessage) + oprot.writeFieldEnd() + oprot.writeFieldStop() + oprot.writeStructEnd() + + def validate(self): + if self.id is None: + raise TProtocolException(message="Required field id is unset!") + if self.state is None: + raise TProtocolException(message="Required field state is unset!") + if self.accepted is None: + raise TProtocolException(message="Required field accepted is unset!") + return + + def __repr__(self): + L = ["{}={!r}".format(key, value) for key, value in self.__dict__.items()] + return "{}({})".format(self.__class__.__name__, ", ".join(L)) + + def __eq__(self, other): + return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ + + def __ne__(self, other): + return not (self == other) + + +class ShowCompactRequest: + def read(self, iprot): + if ( + iprot._fast_decode is not None + and isinstance(iprot.trans, TTransport.CReadableTransport) + and self.thrift_spec is not None + ): + iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec]) + return + iprot.readStructBegin() + while True: + (fname, ftype, fid) = iprot.readFieldBegin() + if ftype == TType.STOP: + break + else: + iprot.skip(ftype) + iprot.readFieldEnd() + iprot.readStructEnd() + + def write(self, oprot): + if oprot._fast_encode is not None and self.thrift_spec is not None: + oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec])) + return + oprot.writeStructBegin("ShowCompactRequest") + oprot.writeFieldStop() + oprot.writeStructEnd() + + def validate(self): + return + + def __repr__(self): + L = ["{}={!r}".format(key, value) for key, value in self.__dict__.items()] + return "{}({})".format(self.__class__.__name__, ", ".join(L)) + + def __eq__(self, other): + return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ + + def __ne__(self, other): + return not (self == other) + + +class ShowCompactResponseElement: + """ + Attributes: + - dbname + - tablename + - partitionname + - type + - state + - workerid + - start + - runAs + - hightestTxnId + - metaInfo + - endTime + - hadoopJobId + - id + - errorMessage + - enqueueTime + - workerVersion + - initiatorId + - initiatorVersion + - cleanerStart + + """ + + def __init__( + self, + dbname=None, + tablename=None, + partitionname=None, + type=None, + state=None, + workerid=None, + start=None, + runAs=None, + hightestTxnId=None, + metaInfo=None, + endTime=None, + hadoopJobId="None", + id=None, + errorMessage=None, + enqueueTime=None, + workerVersion=None, + initiatorId=None, + initiatorVersion=None, + cleanerStart=None, + ): + self.dbname = dbname + self.tablename = tablename + self.partitionname = partitionname + self.type = type + self.state = state + self.workerid = workerid + self.start = start + self.runAs = runAs + self.hightestTxnId = hightestTxnId + self.metaInfo = metaInfo + self.endTime = endTime + self.hadoopJobId = hadoopJobId + self.id = id + self.errorMessage = errorMessage + self.enqueueTime = enqueueTime + self.workerVersion = workerVersion + self.initiatorId = initiatorId + self.initiatorVersion = initiatorVersion + self.cleanerStart = cleanerStart + + def read(self, iprot): + if ( + iprot._fast_decode is not None + and isinstance(iprot.trans, TTransport.CReadableTransport) + and self.thrift_spec is not None + ): + iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec]) + return + iprot.readStructBegin() + while True: + (fname, ftype, fid) = iprot.readFieldBegin() + if ftype == TType.STOP: + break + if fid == 1: + if ftype == TType.STRING: + self.dbname = ( + iprot.readString().decode("utf-8", errors="replace") if sys.version_info[0] == 2 else iprot.readString() + ) + else: + iprot.skip(ftype) + elif fid == 2: + if ftype == TType.STRING: + self.tablename = ( + iprot.readString().decode("utf-8", errors="replace") if sys.version_info[0] == 2 else iprot.readString() + ) + else: + iprot.skip(ftype) + elif fid == 3: + if ftype == TType.STRING: + self.partitionname = ( + iprot.readString().decode("utf-8", errors="replace") if sys.version_info[0] == 2 else iprot.readString() + ) + else: + iprot.skip(ftype) + elif fid == 4: + if ftype == TType.I32: + self.type = iprot.readI32() + else: + iprot.skip(ftype) + elif fid == 5: + if ftype == TType.STRING: + self.state = ( + iprot.readString().decode("utf-8", errors="replace") if sys.version_info[0] == 2 else iprot.readString() + ) + else: + iprot.skip(ftype) + elif fid == 6: + if ftype == TType.STRING: + self.workerid = ( + iprot.readString().decode("utf-8", errors="replace") if sys.version_info[0] == 2 else iprot.readString() + ) + else: + iprot.skip(ftype) + elif fid == 7: + if ftype == TType.I64: + self.start = iprot.readI64() + else: + iprot.skip(ftype) + elif fid == 8: + if ftype == TType.STRING: + self.runAs = ( + iprot.readString().decode("utf-8", errors="replace") if sys.version_info[0] == 2 else iprot.readString() + ) + else: + iprot.skip(ftype) + elif fid == 9: + if ftype == TType.I64: + self.hightestTxnId = iprot.readI64() + else: + iprot.skip(ftype) + elif fid == 10: + if ftype == TType.STRING: + self.metaInfo = ( + iprot.readString().decode("utf-8", errors="replace") if sys.version_info[0] == 2 else iprot.readString() + ) + else: + iprot.skip(ftype) + elif fid == 11: + if ftype == TType.I64: + self.endTime = iprot.readI64() + else: + iprot.skip(ftype) + elif fid == 12: + if ftype == TType.STRING: + self.hadoopJobId = ( + iprot.readString().decode("utf-8", errors="replace") if sys.version_info[0] == 2 else iprot.readString() + ) + else: + iprot.skip(ftype) + elif fid == 13: + if ftype == TType.I64: + self.id = iprot.readI64() + else: + iprot.skip(ftype) + elif fid == 14: + if ftype == TType.STRING: + self.errorMessage = ( + iprot.readString().decode("utf-8", errors="replace") if sys.version_info[0] == 2 else iprot.readString() + ) + else: + iprot.skip(ftype) + elif fid == 15: + if ftype == TType.I64: + self.enqueueTime = iprot.readI64() + else: + iprot.skip(ftype) + elif fid == 16: + if ftype == TType.STRING: + self.workerVersion = ( + iprot.readString().decode("utf-8", errors="replace") if sys.version_info[0] == 2 else iprot.readString() + ) + else: + iprot.skip(ftype) + elif fid == 17: + if ftype == TType.STRING: + self.initiatorId = ( + iprot.readString().decode("utf-8", errors="replace") if sys.version_info[0] == 2 else iprot.readString() + ) + else: + iprot.skip(ftype) + elif fid == 18: + if ftype == TType.STRING: + self.initiatorVersion = ( + iprot.readString().decode("utf-8", errors="replace") if sys.version_info[0] == 2 else iprot.readString() + ) + else: + iprot.skip(ftype) + elif fid == 19: + if ftype == TType.I64: + self.cleanerStart = iprot.readI64() + else: + iprot.skip(ftype) + else: + iprot.skip(ftype) + iprot.readFieldEnd() + iprot.readStructEnd() + + def write(self, oprot): + if oprot._fast_encode is not None and self.thrift_spec is not None: + oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec])) + return + oprot.writeStructBegin("ShowCompactResponseElement") + if self.dbname is not None: + oprot.writeFieldBegin("dbname", TType.STRING, 1) + oprot.writeString(self.dbname.encode("utf-8") if sys.version_info[0] == 2 else self.dbname) + oprot.writeFieldEnd() + if self.tablename is not None: + oprot.writeFieldBegin("tablename", TType.STRING, 2) + oprot.writeString(self.tablename.encode("utf-8") if sys.version_info[0] == 2 else self.tablename) + oprot.writeFieldEnd() + if self.partitionname is not None: + oprot.writeFieldBegin("partitionname", TType.STRING, 3) + oprot.writeString(self.partitionname.encode("utf-8") if sys.version_info[0] == 2 else self.partitionname) + oprot.writeFieldEnd() + if self.type is not None: + oprot.writeFieldBegin("type", TType.I32, 4) + oprot.writeI32(self.type) + oprot.writeFieldEnd() + if self.state is not None: + oprot.writeFieldBegin("state", TType.STRING, 5) + oprot.writeString(self.state.encode("utf-8") if sys.version_info[0] == 2 else self.state) + oprot.writeFieldEnd() + if self.workerid is not None: + oprot.writeFieldBegin("workerid", TType.STRING, 6) + oprot.writeString(self.workerid.encode("utf-8") if sys.version_info[0] == 2 else self.workerid) + oprot.writeFieldEnd() + if self.start is not None: + oprot.writeFieldBegin("start", TType.I64, 7) + oprot.writeI64(self.start) + oprot.writeFieldEnd() + if self.runAs is not None: + oprot.writeFieldBegin("runAs", TType.STRING, 8) + oprot.writeString(self.runAs.encode("utf-8") if sys.version_info[0] == 2 else self.runAs) + oprot.writeFieldEnd() + if self.hightestTxnId is not None: + oprot.writeFieldBegin("hightestTxnId", TType.I64, 9) + oprot.writeI64(self.hightestTxnId) + oprot.writeFieldEnd() + if self.metaInfo is not None: + oprot.writeFieldBegin("metaInfo", TType.STRING, 10) + oprot.writeString(self.metaInfo.encode("utf-8") if sys.version_info[0] == 2 else self.metaInfo) + oprot.writeFieldEnd() + if self.endTime is not None: + oprot.writeFieldBegin("endTime", TType.I64, 11) + oprot.writeI64(self.endTime) + oprot.writeFieldEnd() + if self.hadoopJobId is not None: + oprot.writeFieldBegin("hadoopJobId", TType.STRING, 12) + oprot.writeString(self.hadoopJobId.encode("utf-8") if sys.version_info[0] == 2 else self.hadoopJobId) + oprot.writeFieldEnd() + if self.id is not None: + oprot.writeFieldBegin("id", TType.I64, 13) + oprot.writeI64(self.id) + oprot.writeFieldEnd() + if self.errorMessage is not None: + oprot.writeFieldBegin("errorMessage", TType.STRING, 14) + oprot.writeString(self.errorMessage.encode("utf-8") if sys.version_info[0] == 2 else self.errorMessage) + oprot.writeFieldEnd() + if self.enqueueTime is not None: + oprot.writeFieldBegin("enqueueTime", TType.I64, 15) + oprot.writeI64(self.enqueueTime) + oprot.writeFieldEnd() + if self.workerVersion is not None: + oprot.writeFieldBegin("workerVersion", TType.STRING, 16) + oprot.writeString(self.workerVersion.encode("utf-8") if sys.version_info[0] == 2 else self.workerVersion) + oprot.writeFieldEnd() + if self.initiatorId is not None: + oprot.writeFieldBegin("initiatorId", TType.STRING, 17) + oprot.writeString(self.initiatorId.encode("utf-8") if sys.version_info[0] == 2 else self.initiatorId) + oprot.writeFieldEnd() + if self.initiatorVersion is not None: + oprot.writeFieldBegin("initiatorVersion", TType.STRING, 18) + oprot.writeString(self.initiatorVersion.encode("utf-8") if sys.version_info[0] == 2 else self.initiatorVersion) + oprot.writeFieldEnd() + if self.cleanerStart is not None: + oprot.writeFieldBegin("cleanerStart", TType.I64, 19) + oprot.writeI64(self.cleanerStart) + oprot.writeFieldEnd() + oprot.writeFieldStop() + oprot.writeStructEnd() + + def validate(self): + if self.dbname is None: + raise TProtocolException(message="Required field dbname is unset!") + if self.tablename is None: + raise TProtocolException(message="Required field tablename is unset!") + if self.type is None: + raise TProtocolException(message="Required field type is unset!") + if self.state is None: + raise TProtocolException(message="Required field state is unset!") + return + + def __repr__(self): + L = ["{}={!r}".format(key, value) for key, value in self.__dict__.items()] + return "{}({})".format(self.__class__.__name__, ", ".join(L)) + + def __eq__(self, other): + return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ + + def __ne__(self, other): + return not (self == other) + + +class ShowCompactResponse: + """ + Attributes: + - compacts + + """ + + def __init__( + self, + compacts=None, + ): + self.compacts = compacts + + def read(self, iprot): + if ( + iprot._fast_decode is not None + and isinstance(iprot.trans, TTransport.CReadableTransport) + and self.thrift_spec is not None + ): + iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec]) + return + iprot.readStructBegin() + while True: + (fname, ftype, fid) = iprot.readFieldBegin() + if ftype == TType.STOP: + break + if fid == 1: + if ftype == TType.LIST: + self.compacts = [] + (_etype763, _size760) = iprot.readListBegin() + for _i764 in range(_size760): + _elem765 = ShowCompactResponseElement() + _elem765.read(iprot) + self.compacts.append(_elem765) + iprot.readListEnd() + else: + iprot.skip(ftype) + else: + iprot.skip(ftype) + iprot.readFieldEnd() + iprot.readStructEnd() + + def write(self, oprot): + if oprot._fast_encode is not None and self.thrift_spec is not None: + oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec])) + return + oprot.writeStructBegin("ShowCompactResponse") + if self.compacts is not None: + oprot.writeFieldBegin("compacts", TType.LIST, 1) + oprot.writeListBegin(TType.STRUCT, len(self.compacts)) + for iter766 in self.compacts: + iter766.write(oprot) + oprot.writeListEnd() + oprot.writeFieldEnd() + oprot.writeFieldStop() + oprot.writeStructEnd() + + def validate(self): + if self.compacts is None: + raise TProtocolException(message="Required field compacts is unset!") + return + + def __repr__(self): + L = ["{}={!r}".format(key, value) for key, value in self.__dict__.items()] + return "{}({})".format(self.__class__.__name__, ", ".join(L)) + + def __eq__(self, other): + return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ + + def __ne__(self, other): + return not (self == other) + + +class GetLatestCommittedCompactionInfoRequest: + """ + Attributes: + - dbname + - tablename + - partitionnames + - lastCompactionId + + """ + + def __init__( + self, + dbname=None, + tablename=None, + partitionnames=None, + lastCompactionId=None, + ): + self.dbname = dbname + self.tablename = tablename + self.partitionnames = partitionnames + self.lastCompactionId = lastCompactionId + + def read(self, iprot): + if ( + iprot._fast_decode is not None + and isinstance(iprot.trans, TTransport.CReadableTransport) + and self.thrift_spec is not None + ): + iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec]) + return + iprot.readStructBegin() + while True: + (fname, ftype, fid) = iprot.readFieldBegin() + if ftype == TType.STOP: + break + if fid == 1: + if ftype == TType.STRING: + self.dbname = ( + iprot.readString().decode("utf-8", errors="replace") if sys.version_info[0] == 2 else iprot.readString() + ) + else: + iprot.skip(ftype) + elif fid == 2: + if ftype == TType.STRING: + self.tablename = ( + iprot.readString().decode("utf-8", errors="replace") if sys.version_info[0] == 2 else iprot.readString() + ) + else: + iprot.skip(ftype) + elif fid == 3: + if ftype == TType.LIST: + self.partitionnames = [] + (_etype770, _size767) = iprot.readListBegin() + for _i771 in range(_size767): + _elem772 = ( + iprot.readString().decode("utf-8", errors="replace") + if sys.version_info[0] == 2 + else iprot.readString() + ) + self.partitionnames.append(_elem772) + iprot.readListEnd() + else: + iprot.skip(ftype) + elif fid == 4: + if ftype == TType.I64: + self.lastCompactionId = iprot.readI64() + else: + iprot.skip(ftype) + else: + iprot.skip(ftype) + iprot.readFieldEnd() + iprot.readStructEnd() + + def write(self, oprot): + if oprot._fast_encode is not None and self.thrift_spec is not None: + oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec])) + return + oprot.writeStructBegin("GetLatestCommittedCompactionInfoRequest") + if self.dbname is not None: + oprot.writeFieldBegin("dbname", TType.STRING, 1) + oprot.writeString(self.dbname.encode("utf-8") if sys.version_info[0] == 2 else self.dbname) + oprot.writeFieldEnd() + if self.tablename is not None: + oprot.writeFieldBegin("tablename", TType.STRING, 2) + oprot.writeString(self.tablename.encode("utf-8") if sys.version_info[0] == 2 else self.tablename) + oprot.writeFieldEnd() + if self.partitionnames is not None: + oprot.writeFieldBegin("partitionnames", TType.LIST, 3) + oprot.writeListBegin(TType.STRING, len(self.partitionnames)) + for iter773 in self.partitionnames: + oprot.writeString(iter773.encode("utf-8") if sys.version_info[0] == 2 else iter773) + oprot.writeListEnd() + oprot.writeFieldEnd() + if self.lastCompactionId is not None: + oprot.writeFieldBegin("lastCompactionId", TType.I64, 4) + oprot.writeI64(self.lastCompactionId) + oprot.writeFieldEnd() + oprot.writeFieldStop() + oprot.writeStructEnd() + + def validate(self): + if self.dbname is None: + raise TProtocolException(message="Required field dbname is unset!") + if self.tablename is None: + raise TProtocolException(message="Required field tablename is unset!") + return + + def __repr__(self): + L = ["{}={!r}".format(key, value) for key, value in self.__dict__.items()] + return "{}({})".format(self.__class__.__name__, ", ".join(L)) + + def __eq__(self, other): + return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ + + def __ne__(self, other): + return not (self == other) + + +class GetLatestCommittedCompactionInfoResponse: + """ + Attributes: + - compactions + + """ + + def __init__( + self, + compactions=None, + ): + self.compactions = compactions + + def read(self, iprot): + if ( + iprot._fast_decode is not None + and isinstance(iprot.trans, TTransport.CReadableTransport) + and self.thrift_spec is not None + ): + iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec]) + return + iprot.readStructBegin() + while True: + (fname, ftype, fid) = iprot.readFieldBegin() + if ftype == TType.STOP: + break + if fid == 1: + if ftype == TType.LIST: + self.compactions = [] + (_etype777, _size774) = iprot.readListBegin() + for _i778 in range(_size774): + _elem779 = CompactionInfoStruct() + _elem779.read(iprot) + self.compactions.append(_elem779) + iprot.readListEnd() + else: + iprot.skip(ftype) + else: + iprot.skip(ftype) + iprot.readFieldEnd() + iprot.readStructEnd() + + def write(self, oprot): + if oprot._fast_encode is not None and self.thrift_spec is not None: + oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec])) + return + oprot.writeStructBegin("GetLatestCommittedCompactionInfoResponse") + if self.compactions is not None: + oprot.writeFieldBegin("compactions", TType.LIST, 1) + oprot.writeListBegin(TType.STRUCT, len(self.compactions)) + for iter780 in self.compactions: + iter780.write(oprot) + oprot.writeListEnd() + oprot.writeFieldEnd() + oprot.writeFieldStop() + oprot.writeStructEnd() + + def validate(self): + if self.compactions is None: + raise TProtocolException(message="Required field compactions is unset!") + return + + def __repr__(self): + L = ["{}={!r}".format(key, value) for key, value in self.__dict__.items()] + return "{}({})".format(self.__class__.__name__, ", ".join(L)) + + def __eq__(self, other): + return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ + + def __ne__(self, other): + return not (self == other) + + +class FindNextCompactRequest: + """ + Attributes: + - workerId + - workerVersion + + """ + + def __init__( + self, + workerId=None, + workerVersion=None, + ): + self.workerId = workerId + self.workerVersion = workerVersion + + def read(self, iprot): + if ( + iprot._fast_decode is not None + and isinstance(iprot.trans, TTransport.CReadableTransport) + and self.thrift_spec is not None + ): + iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec]) + return + iprot.readStructBegin() + while True: + (fname, ftype, fid) = iprot.readFieldBegin() + if ftype == TType.STOP: + break + if fid == 1: + if ftype == TType.STRING: + self.workerId = ( + iprot.readString().decode("utf-8", errors="replace") if sys.version_info[0] == 2 else iprot.readString() + ) + else: + iprot.skip(ftype) + elif fid == 2: + if ftype == TType.STRING: + self.workerVersion = ( + iprot.readString().decode("utf-8", errors="replace") if sys.version_info[0] == 2 else iprot.readString() + ) + else: + iprot.skip(ftype) + else: + iprot.skip(ftype) + iprot.readFieldEnd() + iprot.readStructEnd() + + def write(self, oprot): + if oprot._fast_encode is not None and self.thrift_spec is not None: + oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec])) + return + oprot.writeStructBegin("FindNextCompactRequest") + if self.workerId is not None: + oprot.writeFieldBegin("workerId", TType.STRING, 1) + oprot.writeString(self.workerId.encode("utf-8") if sys.version_info[0] == 2 else self.workerId) + oprot.writeFieldEnd() + if self.workerVersion is not None: + oprot.writeFieldBegin("workerVersion", TType.STRING, 2) + oprot.writeString(self.workerVersion.encode("utf-8") if sys.version_info[0] == 2 else self.workerVersion) + oprot.writeFieldEnd() + oprot.writeFieldStop() + oprot.writeStructEnd() + + def validate(self): + return + + def __repr__(self): + L = ["{}={!r}".format(key, value) for key, value in self.__dict__.items()] + return "{}({})".format(self.__class__.__name__, ", ".join(L)) + + def __eq__(self, other): + return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ + + def __ne__(self, other): + return not (self == other) + + +class AddDynamicPartitions: + """ + Attributes: + - txnid + - writeid + - dbname + - tablename + - partitionnames + - operationType + + """ + + def __init__( + self, + txnid=None, + writeid=None, + dbname=None, + tablename=None, + partitionnames=None, + operationType=5, + ): + self.txnid = txnid + self.writeid = writeid + self.dbname = dbname + self.tablename = tablename + self.partitionnames = partitionnames + self.operationType = operationType + + def read(self, iprot): + if ( + iprot._fast_decode is not None + and isinstance(iprot.trans, TTransport.CReadableTransport) + and self.thrift_spec is not None + ): + iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec]) + return + iprot.readStructBegin() + while True: + (fname, ftype, fid) = iprot.readFieldBegin() + if ftype == TType.STOP: + break + if fid == 1: + if ftype == TType.I64: + self.txnid = iprot.readI64() + else: + iprot.skip(ftype) + elif fid == 2: + if ftype == TType.I64: + self.writeid = iprot.readI64() + else: + iprot.skip(ftype) + elif fid == 3: + if ftype == TType.STRING: + self.dbname = ( + iprot.readString().decode("utf-8", errors="replace") if sys.version_info[0] == 2 else iprot.readString() + ) + else: + iprot.skip(ftype) + elif fid == 4: + if ftype == TType.STRING: + self.tablename = ( + iprot.readString().decode("utf-8", errors="replace") if sys.version_info[0] == 2 else iprot.readString() + ) + else: + iprot.skip(ftype) + elif fid == 5: + if ftype == TType.LIST: + self.partitionnames = [] + (_etype784, _size781) = iprot.readListBegin() + for _i785 in range(_size781): + _elem786 = ( + iprot.readString().decode("utf-8", errors="replace") + if sys.version_info[0] == 2 + else iprot.readString() + ) + self.partitionnames.append(_elem786) + iprot.readListEnd() + else: + iprot.skip(ftype) + elif fid == 6: + if ftype == TType.I32: + self.operationType = iprot.readI32() + else: + iprot.skip(ftype) + else: + iprot.skip(ftype) + iprot.readFieldEnd() + iprot.readStructEnd() + + def write(self, oprot): + if oprot._fast_encode is not None and self.thrift_spec is not None: + oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec])) + return + oprot.writeStructBegin("AddDynamicPartitions") + if self.txnid is not None: + oprot.writeFieldBegin("txnid", TType.I64, 1) + oprot.writeI64(self.txnid) + oprot.writeFieldEnd() + if self.writeid is not None: + oprot.writeFieldBegin("writeid", TType.I64, 2) + oprot.writeI64(self.writeid) + oprot.writeFieldEnd() + if self.dbname is not None: + oprot.writeFieldBegin("dbname", TType.STRING, 3) + oprot.writeString(self.dbname.encode("utf-8") if sys.version_info[0] == 2 else self.dbname) + oprot.writeFieldEnd() + if self.tablename is not None: + oprot.writeFieldBegin("tablename", TType.STRING, 4) + oprot.writeString(self.tablename.encode("utf-8") if sys.version_info[0] == 2 else self.tablename) + oprot.writeFieldEnd() + if self.partitionnames is not None: + oprot.writeFieldBegin("partitionnames", TType.LIST, 5) + oprot.writeListBegin(TType.STRING, len(self.partitionnames)) + for iter787 in self.partitionnames: + oprot.writeString(iter787.encode("utf-8") if sys.version_info[0] == 2 else iter787) + oprot.writeListEnd() + oprot.writeFieldEnd() + if self.operationType is not None: + oprot.writeFieldBegin("operationType", TType.I32, 6) + oprot.writeI32(self.operationType) + oprot.writeFieldEnd() + oprot.writeFieldStop() + oprot.writeStructEnd() + + def validate(self): + if self.txnid is None: + raise TProtocolException(message="Required field txnid is unset!") + if self.writeid is None: + raise TProtocolException(message="Required field writeid is unset!") + if self.dbname is None: + raise TProtocolException(message="Required field dbname is unset!") + if self.tablename is None: + raise TProtocolException(message="Required field tablename is unset!") + if self.partitionnames is None: + raise TProtocolException(message="Required field partitionnames is unset!") + return + + def __repr__(self): + L = ["{}={!r}".format(key, value) for key, value in self.__dict__.items()] + return "{}({})".format(self.__class__.__name__, ", ".join(L)) + + def __eq__(self, other): + return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ + + def __ne__(self, other): + return not (self == other) + + +class BasicTxnInfo: + """ + Attributes: + - isnull + - time + - txnid + - dbname + - tablename + - partitionname + + """ + + def __init__( + self, + isnull=None, + time=None, + txnid=None, + dbname=None, + tablename=None, + partitionname=None, + ): + self.isnull = isnull + self.time = time + self.txnid = txnid + self.dbname = dbname + self.tablename = tablename + self.partitionname = partitionname + + def read(self, iprot): + if ( + iprot._fast_decode is not None + and isinstance(iprot.trans, TTransport.CReadableTransport) + and self.thrift_spec is not None + ): + iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec]) + return + iprot.readStructBegin() + while True: + (fname, ftype, fid) = iprot.readFieldBegin() + if ftype == TType.STOP: + break + if fid == 1: + if ftype == TType.BOOL: + self.isnull = iprot.readBool() + else: + iprot.skip(ftype) + elif fid == 2: + if ftype == TType.I64: + self.time = iprot.readI64() + else: + iprot.skip(ftype) + elif fid == 3: + if ftype == TType.I64: + self.txnid = iprot.readI64() + else: + iprot.skip(ftype) + elif fid == 4: + if ftype == TType.STRING: + self.dbname = ( + iprot.readString().decode("utf-8", errors="replace") if sys.version_info[0] == 2 else iprot.readString() + ) + else: + iprot.skip(ftype) + elif fid == 5: + if ftype == TType.STRING: + self.tablename = ( + iprot.readString().decode("utf-8", errors="replace") if sys.version_info[0] == 2 else iprot.readString() + ) + else: + iprot.skip(ftype) + elif fid == 6: + if ftype == TType.STRING: + self.partitionname = ( + iprot.readString().decode("utf-8", errors="replace") if sys.version_info[0] == 2 else iprot.readString() + ) + else: + iprot.skip(ftype) + else: + iprot.skip(ftype) + iprot.readFieldEnd() + iprot.readStructEnd() + + def write(self, oprot): + if oprot._fast_encode is not None and self.thrift_spec is not None: + oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec])) + return + oprot.writeStructBegin("BasicTxnInfo") + if self.isnull is not None: + oprot.writeFieldBegin("isnull", TType.BOOL, 1) + oprot.writeBool(self.isnull) + oprot.writeFieldEnd() + if self.time is not None: + oprot.writeFieldBegin("time", TType.I64, 2) + oprot.writeI64(self.time) + oprot.writeFieldEnd() + if self.txnid is not None: + oprot.writeFieldBegin("txnid", TType.I64, 3) + oprot.writeI64(self.txnid) + oprot.writeFieldEnd() + if self.dbname is not None: + oprot.writeFieldBegin("dbname", TType.STRING, 4) + oprot.writeString(self.dbname.encode("utf-8") if sys.version_info[0] == 2 else self.dbname) + oprot.writeFieldEnd() + if self.tablename is not None: + oprot.writeFieldBegin("tablename", TType.STRING, 5) + oprot.writeString(self.tablename.encode("utf-8") if sys.version_info[0] == 2 else self.tablename) + oprot.writeFieldEnd() + if self.partitionname is not None: + oprot.writeFieldBegin("partitionname", TType.STRING, 6) + oprot.writeString(self.partitionname.encode("utf-8") if sys.version_info[0] == 2 else self.partitionname) + oprot.writeFieldEnd() + oprot.writeFieldStop() + oprot.writeStructEnd() + + def validate(self): + if self.isnull is None: + raise TProtocolException(message="Required field isnull is unset!") + return + + def __repr__(self): + L = ["{}={!r}".format(key, value) for key, value in self.__dict__.items()] + return "{}({})".format(self.__class__.__name__, ", ".join(L)) + + def __eq__(self, other): + return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ + + def __ne__(self, other): + return not (self == other) + + +class NotificationEventRequest: + """ + Attributes: + - lastEvent + - maxEvents + - eventTypeSkipList + + """ + + def __init__( + self, + lastEvent=None, + maxEvents=None, + eventTypeSkipList=None, + ): + self.lastEvent = lastEvent + self.maxEvents = maxEvents + self.eventTypeSkipList = eventTypeSkipList + + def read(self, iprot): + if ( + iprot._fast_decode is not None + and isinstance(iprot.trans, TTransport.CReadableTransport) + and self.thrift_spec is not None + ): + iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec]) + return + iprot.readStructBegin() + while True: + (fname, ftype, fid) = iprot.readFieldBegin() + if ftype == TType.STOP: + break + if fid == 1: + if ftype == TType.I64: + self.lastEvent = iprot.readI64() + else: + iprot.skip(ftype) + elif fid == 2: + if ftype == TType.I32: + self.maxEvents = iprot.readI32() + else: + iprot.skip(ftype) + elif fid == 3: + if ftype == TType.LIST: + self.eventTypeSkipList = [] + (_etype791, _size788) = iprot.readListBegin() + for _i792 in range(_size788): + _elem793 = ( + iprot.readString().decode("utf-8", errors="replace") + if sys.version_info[0] == 2 + else iprot.readString() + ) + self.eventTypeSkipList.append(_elem793) + iprot.readListEnd() + else: + iprot.skip(ftype) + else: + iprot.skip(ftype) + iprot.readFieldEnd() + iprot.readStructEnd() + + def write(self, oprot): + if oprot._fast_encode is not None and self.thrift_spec is not None: + oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec])) + return + oprot.writeStructBegin("NotificationEventRequest") + if self.lastEvent is not None: + oprot.writeFieldBegin("lastEvent", TType.I64, 1) + oprot.writeI64(self.lastEvent) + oprot.writeFieldEnd() + if self.maxEvents is not None: + oprot.writeFieldBegin("maxEvents", TType.I32, 2) + oprot.writeI32(self.maxEvents) + oprot.writeFieldEnd() + if self.eventTypeSkipList is not None: + oprot.writeFieldBegin("eventTypeSkipList", TType.LIST, 3) + oprot.writeListBegin(TType.STRING, len(self.eventTypeSkipList)) + for iter794 in self.eventTypeSkipList: + oprot.writeString(iter794.encode("utf-8") if sys.version_info[0] == 2 else iter794) + oprot.writeListEnd() + oprot.writeFieldEnd() + oprot.writeFieldStop() + oprot.writeStructEnd() + + def validate(self): + if self.lastEvent is None: + raise TProtocolException(message="Required field lastEvent is unset!") + return + + def __repr__(self): + L = ["{}={!r}".format(key, value) for key, value in self.__dict__.items()] + return "{}({})".format(self.__class__.__name__, ", ".join(L)) + + def __eq__(self, other): + return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ + + def __ne__(self, other): + return not (self == other) + + +class NotificationEvent: + """ + Attributes: + - eventId + - eventTime + - eventType + - dbName + - tableName + - message + - messageFormat + - catName + + """ + + def __init__( + self, + eventId=None, + eventTime=None, + eventType=None, + dbName=None, + tableName=None, + message=None, + messageFormat=None, + catName=None, + ): + self.eventId = eventId + self.eventTime = eventTime + self.eventType = eventType + self.dbName = dbName + self.tableName = tableName + self.message = message + self.messageFormat = messageFormat + self.catName = catName + + def read(self, iprot): + if ( + iprot._fast_decode is not None + and isinstance(iprot.trans, TTransport.CReadableTransport) + and self.thrift_spec is not None + ): + iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec]) + return + iprot.readStructBegin() + while True: + (fname, ftype, fid) = iprot.readFieldBegin() + if ftype == TType.STOP: + break + if fid == 1: + if ftype == TType.I64: + self.eventId = iprot.readI64() + else: + iprot.skip(ftype) + elif fid == 2: + if ftype == TType.I32: + self.eventTime = iprot.readI32() + else: + iprot.skip(ftype) + elif fid == 3: + if ftype == TType.STRING: + self.eventType = ( + iprot.readString().decode("utf-8", errors="replace") if sys.version_info[0] == 2 else iprot.readString() + ) + else: + iprot.skip(ftype) + elif fid == 4: + if ftype == TType.STRING: + self.dbName = ( + iprot.readString().decode("utf-8", errors="replace") if sys.version_info[0] == 2 else iprot.readString() + ) + else: + iprot.skip(ftype) + elif fid == 5: + if ftype == TType.STRING: + self.tableName = ( + iprot.readString().decode("utf-8", errors="replace") if sys.version_info[0] == 2 else iprot.readString() + ) + else: + iprot.skip(ftype) + elif fid == 6: + if ftype == TType.STRING: + self.message = ( + iprot.readString().decode("utf-8", errors="replace") if sys.version_info[0] == 2 else iprot.readString() + ) + else: + iprot.skip(ftype) + elif fid == 7: + if ftype == TType.STRING: + self.messageFormat = ( + iprot.readString().decode("utf-8", errors="replace") if sys.version_info[0] == 2 else iprot.readString() + ) + else: + iprot.skip(ftype) + elif fid == 8: + if ftype == TType.STRING: + self.catName = ( + iprot.readString().decode("utf-8", errors="replace") if sys.version_info[0] == 2 else iprot.readString() + ) + else: + iprot.skip(ftype) + else: + iprot.skip(ftype) + iprot.readFieldEnd() + iprot.readStructEnd() + + def write(self, oprot): + if oprot._fast_encode is not None and self.thrift_spec is not None: + oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec])) + return + oprot.writeStructBegin("NotificationEvent") + if self.eventId is not None: + oprot.writeFieldBegin("eventId", TType.I64, 1) + oprot.writeI64(self.eventId) + oprot.writeFieldEnd() + if self.eventTime is not None: + oprot.writeFieldBegin("eventTime", TType.I32, 2) + oprot.writeI32(self.eventTime) + oprot.writeFieldEnd() + if self.eventType is not None: + oprot.writeFieldBegin("eventType", TType.STRING, 3) + oprot.writeString(self.eventType.encode("utf-8") if sys.version_info[0] == 2 else self.eventType) + oprot.writeFieldEnd() + if self.dbName is not None: + oprot.writeFieldBegin("dbName", TType.STRING, 4) + oprot.writeString(self.dbName.encode("utf-8") if sys.version_info[0] == 2 else self.dbName) + oprot.writeFieldEnd() + if self.tableName is not None: + oprot.writeFieldBegin("tableName", TType.STRING, 5) + oprot.writeString(self.tableName.encode("utf-8") if sys.version_info[0] == 2 else self.tableName) + oprot.writeFieldEnd() + if self.message is not None: + oprot.writeFieldBegin("message", TType.STRING, 6) + oprot.writeString(self.message.encode("utf-8") if sys.version_info[0] == 2 else self.message) + oprot.writeFieldEnd() + if self.messageFormat is not None: + oprot.writeFieldBegin("messageFormat", TType.STRING, 7) + oprot.writeString(self.messageFormat.encode("utf-8") if sys.version_info[0] == 2 else self.messageFormat) + oprot.writeFieldEnd() + if self.catName is not None: + oprot.writeFieldBegin("catName", TType.STRING, 8) + oprot.writeString(self.catName.encode("utf-8") if sys.version_info[0] == 2 else self.catName) + oprot.writeFieldEnd() + oprot.writeFieldStop() + oprot.writeStructEnd() + + def validate(self): + if self.eventId is None: + raise TProtocolException(message="Required field eventId is unset!") + if self.eventTime is None: + raise TProtocolException(message="Required field eventTime is unset!") + if self.eventType is None: + raise TProtocolException(message="Required field eventType is unset!") + if self.message is None: + raise TProtocolException(message="Required field message is unset!") + return + + def __repr__(self): + L = ["{}={!r}".format(key, value) for key, value in self.__dict__.items()] + return "{}({})".format(self.__class__.__name__, ", ".join(L)) + + def __eq__(self, other): + return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ + + def __ne__(self, other): + return not (self == other) + + +class NotificationEventResponse: + """ + Attributes: + - events + + """ + + def __init__( + self, + events=None, + ): + self.events = events + + def read(self, iprot): + if ( + iprot._fast_decode is not None + and isinstance(iprot.trans, TTransport.CReadableTransport) + and self.thrift_spec is not None + ): + iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec]) + return + iprot.readStructBegin() + while True: + (fname, ftype, fid) = iprot.readFieldBegin() + if ftype == TType.STOP: + break + if fid == 1: + if ftype == TType.LIST: + self.events = [] + (_etype798, _size795) = iprot.readListBegin() + for _i799 in range(_size795): + _elem800 = NotificationEvent() + _elem800.read(iprot) + self.events.append(_elem800) + iprot.readListEnd() + else: + iprot.skip(ftype) + else: + iprot.skip(ftype) + iprot.readFieldEnd() + iprot.readStructEnd() + + def write(self, oprot): + if oprot._fast_encode is not None and self.thrift_spec is not None: + oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec])) + return + oprot.writeStructBegin("NotificationEventResponse") + if self.events is not None: + oprot.writeFieldBegin("events", TType.LIST, 1) + oprot.writeListBegin(TType.STRUCT, len(self.events)) + for iter801 in self.events: + iter801.write(oprot) + oprot.writeListEnd() + oprot.writeFieldEnd() + oprot.writeFieldStop() + oprot.writeStructEnd() + + def validate(self): + if self.events is None: + raise TProtocolException(message="Required field events is unset!") + return + + def __repr__(self): + L = ["{}={!r}".format(key, value) for key, value in self.__dict__.items()] + return "{}({})".format(self.__class__.__name__, ", ".join(L)) + + def __eq__(self, other): + return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ + + def __ne__(self, other): + return not (self == other) + + +class CurrentNotificationEventId: + """ + Attributes: + - eventId + + """ + + def __init__( + self, + eventId=None, + ): + self.eventId = eventId + + def read(self, iprot): + if ( + iprot._fast_decode is not None + and isinstance(iprot.trans, TTransport.CReadableTransport) + and self.thrift_spec is not None + ): + iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec]) + return + iprot.readStructBegin() + while True: + (fname, ftype, fid) = iprot.readFieldBegin() + if ftype == TType.STOP: + break + if fid == 1: + if ftype == TType.I64: + self.eventId = iprot.readI64() + else: + iprot.skip(ftype) + else: + iprot.skip(ftype) + iprot.readFieldEnd() + iprot.readStructEnd() + + def write(self, oprot): + if oprot._fast_encode is not None and self.thrift_spec is not None: + oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec])) + return + oprot.writeStructBegin("CurrentNotificationEventId") + if self.eventId is not None: + oprot.writeFieldBegin("eventId", TType.I64, 1) + oprot.writeI64(self.eventId) + oprot.writeFieldEnd() + oprot.writeFieldStop() + oprot.writeStructEnd() + + def validate(self): + if self.eventId is None: + raise TProtocolException(message="Required field eventId is unset!") + return + + def __repr__(self): + L = ["{}={!r}".format(key, value) for key, value in self.__dict__.items()] + return "{}({})".format(self.__class__.__name__, ", ".join(L)) + + def __eq__(self, other): + return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ + + def __ne__(self, other): + return not (self == other) + + +class NotificationEventsCountRequest: + """ + Attributes: + - fromEventId + - dbName + - catName + - toEventId + - limit + + """ + + def __init__( + self, + fromEventId=None, + dbName=None, + catName=None, + toEventId=None, + limit=None, + ): + self.fromEventId = fromEventId + self.dbName = dbName + self.catName = catName + self.toEventId = toEventId + self.limit = limit + + def read(self, iprot): + if ( + iprot._fast_decode is not None + and isinstance(iprot.trans, TTransport.CReadableTransport) + and self.thrift_spec is not None + ): + iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec]) + return + iprot.readStructBegin() + while True: + (fname, ftype, fid) = iprot.readFieldBegin() + if ftype == TType.STOP: + break + if fid == 1: + if ftype == TType.I64: + self.fromEventId = iprot.readI64() + else: + iprot.skip(ftype) + elif fid == 2: + if ftype == TType.STRING: + self.dbName = ( + iprot.readString().decode("utf-8", errors="replace") if sys.version_info[0] == 2 else iprot.readString() + ) + else: + iprot.skip(ftype) + elif fid == 3: + if ftype == TType.STRING: + self.catName = ( + iprot.readString().decode("utf-8", errors="replace") if sys.version_info[0] == 2 else iprot.readString() + ) + else: + iprot.skip(ftype) + elif fid == 4: + if ftype == TType.I64: + self.toEventId = iprot.readI64() + else: + iprot.skip(ftype) + elif fid == 5: + if ftype == TType.I64: + self.limit = iprot.readI64() + else: + iprot.skip(ftype) + else: + iprot.skip(ftype) + iprot.readFieldEnd() + iprot.readStructEnd() + + def write(self, oprot): + if oprot._fast_encode is not None and self.thrift_spec is not None: + oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec])) + return + oprot.writeStructBegin("NotificationEventsCountRequest") + if self.fromEventId is not None: + oprot.writeFieldBegin("fromEventId", TType.I64, 1) + oprot.writeI64(self.fromEventId) + oprot.writeFieldEnd() + if self.dbName is not None: + oprot.writeFieldBegin("dbName", TType.STRING, 2) + oprot.writeString(self.dbName.encode("utf-8") if sys.version_info[0] == 2 else self.dbName) + oprot.writeFieldEnd() + if self.catName is not None: + oprot.writeFieldBegin("catName", TType.STRING, 3) + oprot.writeString(self.catName.encode("utf-8") if sys.version_info[0] == 2 else self.catName) + oprot.writeFieldEnd() + if self.toEventId is not None: + oprot.writeFieldBegin("toEventId", TType.I64, 4) + oprot.writeI64(self.toEventId) + oprot.writeFieldEnd() + if self.limit is not None: + oprot.writeFieldBegin("limit", TType.I64, 5) + oprot.writeI64(self.limit) + oprot.writeFieldEnd() + oprot.writeFieldStop() + oprot.writeStructEnd() + + def validate(self): + if self.fromEventId is None: + raise TProtocolException(message="Required field fromEventId is unset!") + if self.dbName is None: + raise TProtocolException(message="Required field dbName is unset!") + return + + def __repr__(self): + L = ["{}={!r}".format(key, value) for key, value in self.__dict__.items()] + return "{}({})".format(self.__class__.__name__, ", ".join(L)) + + def __eq__(self, other): + return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ + + def __ne__(self, other): + return not (self == other) + + +class NotificationEventsCountResponse: + """ + Attributes: + - eventsCount + + """ + + def __init__( + self, + eventsCount=None, + ): + self.eventsCount = eventsCount + + def read(self, iprot): + if ( + iprot._fast_decode is not None + and isinstance(iprot.trans, TTransport.CReadableTransport) + and self.thrift_spec is not None + ): + iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec]) + return + iprot.readStructBegin() + while True: + (fname, ftype, fid) = iprot.readFieldBegin() + if ftype == TType.STOP: + break + if fid == 1: + if ftype == TType.I64: + self.eventsCount = iprot.readI64() + else: + iprot.skip(ftype) + else: + iprot.skip(ftype) + iprot.readFieldEnd() + iprot.readStructEnd() + + def write(self, oprot): + if oprot._fast_encode is not None and self.thrift_spec is not None: + oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec])) + return + oprot.writeStructBegin("NotificationEventsCountResponse") + if self.eventsCount is not None: + oprot.writeFieldBegin("eventsCount", TType.I64, 1) + oprot.writeI64(self.eventsCount) + oprot.writeFieldEnd() + oprot.writeFieldStop() + oprot.writeStructEnd() + + def validate(self): + if self.eventsCount is None: + raise TProtocolException(message="Required field eventsCount is unset!") + return + + def __repr__(self): + L = ["{}={!r}".format(key, value) for key, value in self.__dict__.items()] + return "{}({})".format(self.__class__.__name__, ", ".join(L)) + + def __eq__(self, other): + return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ + + def __ne__(self, other): + return not (self == other) + + +class InsertEventRequestData: + """ + Attributes: + - replace + - filesAdded + - filesAddedChecksum + - subDirectoryList + - partitionVal + + """ + + def __init__( + self, + replace=None, + filesAdded=None, + filesAddedChecksum=None, + subDirectoryList=None, + partitionVal=None, + ): + self.replace = replace + self.filesAdded = filesAdded + self.filesAddedChecksum = filesAddedChecksum + self.subDirectoryList = subDirectoryList + self.partitionVal = partitionVal + + def read(self, iprot): + if ( + iprot._fast_decode is not None + and isinstance(iprot.trans, TTransport.CReadableTransport) + and self.thrift_spec is not None + ): + iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec]) + return + iprot.readStructBegin() + while True: + (fname, ftype, fid) = iprot.readFieldBegin() + if ftype == TType.STOP: + break + if fid == 1: + if ftype == TType.BOOL: + self.replace = iprot.readBool() + else: + iprot.skip(ftype) + elif fid == 2: + if ftype == TType.LIST: + self.filesAdded = [] + (_etype805, _size802) = iprot.readListBegin() + for _i806 in range(_size802): + _elem807 = ( + iprot.readString().decode("utf-8", errors="replace") + if sys.version_info[0] == 2 + else iprot.readString() + ) + self.filesAdded.append(_elem807) + iprot.readListEnd() + else: + iprot.skip(ftype) + elif fid == 3: + if ftype == TType.LIST: + self.filesAddedChecksum = [] + (_etype811, _size808) = iprot.readListBegin() + for _i812 in range(_size808): + _elem813 = ( + iprot.readString().decode("utf-8", errors="replace") + if sys.version_info[0] == 2 + else iprot.readString() + ) + self.filesAddedChecksum.append(_elem813) + iprot.readListEnd() + else: + iprot.skip(ftype) + elif fid == 4: + if ftype == TType.LIST: + self.subDirectoryList = [] + (_etype817, _size814) = iprot.readListBegin() + for _i818 in range(_size814): + _elem819 = ( + iprot.readString().decode("utf-8", errors="replace") + if sys.version_info[0] == 2 + else iprot.readString() + ) + self.subDirectoryList.append(_elem819) + iprot.readListEnd() + else: + iprot.skip(ftype) + elif fid == 5: + if ftype == TType.LIST: + self.partitionVal = [] + (_etype823, _size820) = iprot.readListBegin() + for _i824 in range(_size820): + _elem825 = ( + iprot.readString().decode("utf-8", errors="replace") + if sys.version_info[0] == 2 + else iprot.readString() + ) + self.partitionVal.append(_elem825) + iprot.readListEnd() + else: + iprot.skip(ftype) + else: + iprot.skip(ftype) + iprot.readFieldEnd() + iprot.readStructEnd() + + def write(self, oprot): + if oprot._fast_encode is not None and self.thrift_spec is not None: + oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec])) + return + oprot.writeStructBegin("InsertEventRequestData") + if self.replace is not None: + oprot.writeFieldBegin("replace", TType.BOOL, 1) + oprot.writeBool(self.replace) + oprot.writeFieldEnd() + if self.filesAdded is not None: + oprot.writeFieldBegin("filesAdded", TType.LIST, 2) + oprot.writeListBegin(TType.STRING, len(self.filesAdded)) + for iter826 in self.filesAdded: + oprot.writeString(iter826.encode("utf-8") if sys.version_info[0] == 2 else iter826) + oprot.writeListEnd() + oprot.writeFieldEnd() + if self.filesAddedChecksum is not None: + oprot.writeFieldBegin("filesAddedChecksum", TType.LIST, 3) + oprot.writeListBegin(TType.STRING, len(self.filesAddedChecksum)) + for iter827 in self.filesAddedChecksum: + oprot.writeString(iter827.encode("utf-8") if sys.version_info[0] == 2 else iter827) + oprot.writeListEnd() + oprot.writeFieldEnd() + if self.subDirectoryList is not None: + oprot.writeFieldBegin("subDirectoryList", TType.LIST, 4) + oprot.writeListBegin(TType.STRING, len(self.subDirectoryList)) + for iter828 in self.subDirectoryList: + oprot.writeString(iter828.encode("utf-8") if sys.version_info[0] == 2 else iter828) + oprot.writeListEnd() + oprot.writeFieldEnd() + if self.partitionVal is not None: + oprot.writeFieldBegin("partitionVal", TType.LIST, 5) + oprot.writeListBegin(TType.STRING, len(self.partitionVal)) + for iter829 in self.partitionVal: + oprot.writeString(iter829.encode("utf-8") if sys.version_info[0] == 2 else iter829) + oprot.writeListEnd() + oprot.writeFieldEnd() + oprot.writeFieldStop() + oprot.writeStructEnd() + + def validate(self): + if self.filesAdded is None: + raise TProtocolException(message="Required field filesAdded is unset!") + return + + def __repr__(self): + L = ["{}={!r}".format(key, value) for key, value in self.__dict__.items()] + return "{}({})".format(self.__class__.__name__, ", ".join(L)) + + def __eq__(self, other): + return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ + + def __ne__(self, other): + return not (self == other) + + +class FireEventRequestData: + """ + Attributes: + - insertData + - insertDatas + + """ + + def __init__( + self, + insertData=None, + insertDatas=None, + ): + self.insertData = insertData + self.insertDatas = insertDatas + + def read(self, iprot): + if ( + iprot._fast_decode is not None + and isinstance(iprot.trans, TTransport.CReadableTransport) + and self.thrift_spec is not None + ): + iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec]) + return + iprot.readStructBegin() + while True: + (fname, ftype, fid) = iprot.readFieldBegin() + if ftype == TType.STOP: + break + if fid == 1: + if ftype == TType.STRUCT: + self.insertData = InsertEventRequestData() + self.insertData.read(iprot) + else: + iprot.skip(ftype) + elif fid == 2: + if ftype == TType.LIST: + self.insertDatas = [] + (_etype833, _size830) = iprot.readListBegin() + for _i834 in range(_size830): + _elem835 = InsertEventRequestData() + _elem835.read(iprot) + self.insertDatas.append(_elem835) + iprot.readListEnd() + else: + iprot.skip(ftype) + else: + iprot.skip(ftype) + iprot.readFieldEnd() + iprot.readStructEnd() + + def write(self, oprot): + if oprot._fast_encode is not None and self.thrift_spec is not None: + oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec])) + return + oprot.writeStructBegin("FireEventRequestData") + if self.insertData is not None: + oprot.writeFieldBegin("insertData", TType.STRUCT, 1) + self.insertData.write(oprot) + oprot.writeFieldEnd() + if self.insertDatas is not None: + oprot.writeFieldBegin("insertDatas", TType.LIST, 2) + oprot.writeListBegin(TType.STRUCT, len(self.insertDatas)) + for iter836 in self.insertDatas: + iter836.write(oprot) + oprot.writeListEnd() + oprot.writeFieldEnd() + oprot.writeFieldStop() + oprot.writeStructEnd() + + def validate(self): + return + + def __repr__(self): + L = ["{}={!r}".format(key, value) for key, value in self.__dict__.items()] + return "{}({})".format(self.__class__.__name__, ", ".join(L)) + + def __eq__(self, other): + return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ + + def __ne__(self, other): + return not (self == other) + + +class FireEventRequest: + """ + Attributes: + - successful + - data + - dbName + - tableName + - partitionVals + - catName + + """ + + def __init__( + self, + successful=None, + data=None, + dbName=None, + tableName=None, + partitionVals=None, + catName=None, + ): + self.successful = successful + self.data = data + self.dbName = dbName + self.tableName = tableName + self.partitionVals = partitionVals + self.catName = catName + + def read(self, iprot): + if ( + iprot._fast_decode is not None + and isinstance(iprot.trans, TTransport.CReadableTransport) + and self.thrift_spec is not None + ): + iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec]) + return + iprot.readStructBegin() + while True: + (fname, ftype, fid) = iprot.readFieldBegin() + if ftype == TType.STOP: + break + if fid == 1: + if ftype == TType.BOOL: + self.successful = iprot.readBool() + else: + iprot.skip(ftype) + elif fid == 2: + if ftype == TType.STRUCT: + self.data = FireEventRequestData() + self.data.read(iprot) + else: + iprot.skip(ftype) + elif fid == 3: + if ftype == TType.STRING: + self.dbName = ( + iprot.readString().decode("utf-8", errors="replace") if sys.version_info[0] == 2 else iprot.readString() + ) + else: + iprot.skip(ftype) + elif fid == 4: + if ftype == TType.STRING: + self.tableName = ( + iprot.readString().decode("utf-8", errors="replace") if sys.version_info[0] == 2 else iprot.readString() + ) + else: + iprot.skip(ftype) + elif fid == 5: + if ftype == TType.LIST: + self.partitionVals = [] + (_etype840, _size837) = iprot.readListBegin() + for _i841 in range(_size837): + _elem842 = ( + iprot.readString().decode("utf-8", errors="replace") + if sys.version_info[0] == 2 + else iprot.readString() + ) + self.partitionVals.append(_elem842) + iprot.readListEnd() + else: + iprot.skip(ftype) + elif fid == 6: + if ftype == TType.STRING: + self.catName = ( + iprot.readString().decode("utf-8", errors="replace") if sys.version_info[0] == 2 else iprot.readString() + ) + else: + iprot.skip(ftype) + else: + iprot.skip(ftype) + iprot.readFieldEnd() + iprot.readStructEnd() + + def write(self, oprot): + if oprot._fast_encode is not None and self.thrift_spec is not None: + oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec])) + return + oprot.writeStructBegin("FireEventRequest") + if self.successful is not None: + oprot.writeFieldBegin("successful", TType.BOOL, 1) + oprot.writeBool(self.successful) + oprot.writeFieldEnd() + if self.data is not None: + oprot.writeFieldBegin("data", TType.STRUCT, 2) + self.data.write(oprot) + oprot.writeFieldEnd() + if self.dbName is not None: + oprot.writeFieldBegin("dbName", TType.STRING, 3) + oprot.writeString(self.dbName.encode("utf-8") if sys.version_info[0] == 2 else self.dbName) + oprot.writeFieldEnd() + if self.tableName is not None: + oprot.writeFieldBegin("tableName", TType.STRING, 4) + oprot.writeString(self.tableName.encode("utf-8") if sys.version_info[0] == 2 else self.tableName) + oprot.writeFieldEnd() + if self.partitionVals is not None: + oprot.writeFieldBegin("partitionVals", TType.LIST, 5) + oprot.writeListBegin(TType.STRING, len(self.partitionVals)) + for iter843 in self.partitionVals: + oprot.writeString(iter843.encode("utf-8") if sys.version_info[0] == 2 else iter843) + oprot.writeListEnd() + oprot.writeFieldEnd() + if self.catName is not None: + oprot.writeFieldBegin("catName", TType.STRING, 6) + oprot.writeString(self.catName.encode("utf-8") if sys.version_info[0] == 2 else self.catName) + oprot.writeFieldEnd() + oprot.writeFieldStop() + oprot.writeStructEnd() + + def validate(self): + if self.successful is None: + raise TProtocolException(message="Required field successful is unset!") + if self.data is None: + raise TProtocolException(message="Required field data is unset!") + return + + def __repr__(self): + L = ["{}={!r}".format(key, value) for key, value in self.__dict__.items()] + return "{}({})".format(self.__class__.__name__, ", ".join(L)) + + def __eq__(self, other): + return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ + + def __ne__(self, other): + return not (self == other) + + +class FireEventResponse: + """ + Attributes: + - eventIds + + """ + + def __init__( + self, + eventIds=None, + ): + self.eventIds = eventIds + + def read(self, iprot): + if ( + iprot._fast_decode is not None + and isinstance(iprot.trans, TTransport.CReadableTransport) + and self.thrift_spec is not None + ): + iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec]) + return + iprot.readStructBegin() + while True: + (fname, ftype, fid) = iprot.readFieldBegin() + if ftype == TType.STOP: + break + if fid == 1: + if ftype == TType.LIST: + self.eventIds = [] + (_etype847, _size844) = iprot.readListBegin() + for _i848 in range(_size844): + _elem849 = iprot.readI64() + self.eventIds.append(_elem849) + iprot.readListEnd() + else: + iprot.skip(ftype) + else: + iprot.skip(ftype) + iprot.readFieldEnd() + iprot.readStructEnd() + + def write(self, oprot): + if oprot._fast_encode is not None and self.thrift_spec is not None: + oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec])) + return + oprot.writeStructBegin("FireEventResponse") + if self.eventIds is not None: + oprot.writeFieldBegin("eventIds", TType.LIST, 1) + oprot.writeListBegin(TType.I64, len(self.eventIds)) + for iter850 in self.eventIds: + oprot.writeI64(iter850) + oprot.writeListEnd() + oprot.writeFieldEnd() + oprot.writeFieldStop() + oprot.writeStructEnd() + + def validate(self): + return + + def __repr__(self): + L = ["{}={!r}".format(key, value) for key, value in self.__dict__.items()] + return "{}({})".format(self.__class__.__name__, ", ".join(L)) + + def __eq__(self, other): + return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ + + def __ne__(self, other): + return not (self == other) + + +class WriteNotificationLogRequest: + """ + Attributes: + - txnId + - writeId + - db + - table + - fileInfo + - partitionVals + + """ + + def __init__( + self, + txnId=None, + writeId=None, + db=None, + table=None, + fileInfo=None, + partitionVals=None, + ): + self.txnId = txnId + self.writeId = writeId + self.db = db + self.table = table + self.fileInfo = fileInfo + self.partitionVals = partitionVals + + def read(self, iprot): + if ( + iprot._fast_decode is not None + and isinstance(iprot.trans, TTransport.CReadableTransport) + and self.thrift_spec is not None + ): + iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec]) + return + iprot.readStructBegin() + while True: + (fname, ftype, fid) = iprot.readFieldBegin() + if ftype == TType.STOP: + break + if fid == 1: + if ftype == TType.I64: + self.txnId = iprot.readI64() + else: + iprot.skip(ftype) + elif fid == 2: + if ftype == TType.I64: + self.writeId = iprot.readI64() + else: + iprot.skip(ftype) + elif fid == 3: + if ftype == TType.STRING: + self.db = ( + iprot.readString().decode("utf-8", errors="replace") if sys.version_info[0] == 2 else iprot.readString() + ) + else: + iprot.skip(ftype) + elif fid == 4: + if ftype == TType.STRING: + self.table = ( + iprot.readString().decode("utf-8", errors="replace") if sys.version_info[0] == 2 else iprot.readString() + ) + else: + iprot.skip(ftype) + elif fid == 5: + if ftype == TType.STRUCT: + self.fileInfo = InsertEventRequestData() + self.fileInfo.read(iprot) + else: + iprot.skip(ftype) + elif fid == 6: + if ftype == TType.LIST: + self.partitionVals = [] + (_etype854, _size851) = iprot.readListBegin() + for _i855 in range(_size851): + _elem856 = ( + iprot.readString().decode("utf-8", errors="replace") + if sys.version_info[0] == 2 + else iprot.readString() + ) + self.partitionVals.append(_elem856) + iprot.readListEnd() + else: + iprot.skip(ftype) + else: + iprot.skip(ftype) + iprot.readFieldEnd() + iprot.readStructEnd() + + def write(self, oprot): + if oprot._fast_encode is not None and self.thrift_spec is not None: + oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec])) + return + oprot.writeStructBegin("WriteNotificationLogRequest") + if self.txnId is not None: + oprot.writeFieldBegin("txnId", TType.I64, 1) + oprot.writeI64(self.txnId) + oprot.writeFieldEnd() + if self.writeId is not None: + oprot.writeFieldBegin("writeId", TType.I64, 2) + oprot.writeI64(self.writeId) + oprot.writeFieldEnd() + if self.db is not None: + oprot.writeFieldBegin("db", TType.STRING, 3) + oprot.writeString(self.db.encode("utf-8") if sys.version_info[0] == 2 else self.db) + oprot.writeFieldEnd() + if self.table is not None: + oprot.writeFieldBegin("table", TType.STRING, 4) + oprot.writeString(self.table.encode("utf-8") if sys.version_info[0] == 2 else self.table) + oprot.writeFieldEnd() + if self.fileInfo is not None: + oprot.writeFieldBegin("fileInfo", TType.STRUCT, 5) + self.fileInfo.write(oprot) + oprot.writeFieldEnd() + if self.partitionVals is not None: + oprot.writeFieldBegin("partitionVals", TType.LIST, 6) + oprot.writeListBegin(TType.STRING, len(self.partitionVals)) + for iter857 in self.partitionVals: + oprot.writeString(iter857.encode("utf-8") if sys.version_info[0] == 2 else iter857) + oprot.writeListEnd() + oprot.writeFieldEnd() + oprot.writeFieldStop() + oprot.writeStructEnd() + + def validate(self): + if self.txnId is None: + raise TProtocolException(message="Required field txnId is unset!") + if self.writeId is None: + raise TProtocolException(message="Required field writeId is unset!") + if self.db is None: + raise TProtocolException(message="Required field db is unset!") + if self.table is None: + raise TProtocolException(message="Required field table is unset!") + if self.fileInfo is None: + raise TProtocolException(message="Required field fileInfo is unset!") + return + + def __repr__(self): + L = ["{}={!r}".format(key, value) for key, value in self.__dict__.items()] + return "{}({})".format(self.__class__.__name__, ", ".join(L)) + + def __eq__(self, other): + return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ + + def __ne__(self, other): + return not (self == other) + + +class WriteNotificationLogResponse: + def read(self, iprot): + if ( + iprot._fast_decode is not None + and isinstance(iprot.trans, TTransport.CReadableTransport) + and self.thrift_spec is not None + ): + iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec]) + return + iprot.readStructBegin() + while True: + (fname, ftype, fid) = iprot.readFieldBegin() + if ftype == TType.STOP: + break + else: + iprot.skip(ftype) + iprot.readFieldEnd() + iprot.readStructEnd() + + def write(self, oprot): + if oprot._fast_encode is not None and self.thrift_spec is not None: + oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec])) + return + oprot.writeStructBegin("WriteNotificationLogResponse") + oprot.writeFieldStop() + oprot.writeStructEnd() + + def validate(self): + return + + def __repr__(self): + L = ["{}={!r}".format(key, value) for key, value in self.__dict__.items()] + return "{}({})".format(self.__class__.__name__, ", ".join(L)) + + def __eq__(self, other): + return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ + + def __ne__(self, other): + return not (self == other) + + +class WriteNotificationLogBatchRequest: + """ + Attributes: + - catalog + - db + - table + - requestList + + """ + + def __init__( + self, + catalog=None, + db=None, + table=None, + requestList=None, + ): + self.catalog = catalog + self.db = db + self.table = table + self.requestList = requestList + + def read(self, iprot): + if ( + iprot._fast_decode is not None + and isinstance(iprot.trans, TTransport.CReadableTransport) + and self.thrift_spec is not None + ): + iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec]) + return + iprot.readStructBegin() + while True: + (fname, ftype, fid) = iprot.readFieldBegin() + if ftype == TType.STOP: + break + if fid == 1: + if ftype == TType.STRING: + self.catalog = ( + iprot.readString().decode("utf-8", errors="replace") if sys.version_info[0] == 2 else iprot.readString() + ) + else: + iprot.skip(ftype) + elif fid == 2: + if ftype == TType.STRING: + self.db = ( + iprot.readString().decode("utf-8", errors="replace") if sys.version_info[0] == 2 else iprot.readString() + ) + else: + iprot.skip(ftype) + elif fid == 3: + if ftype == TType.STRING: + self.table = ( + iprot.readString().decode("utf-8", errors="replace") if sys.version_info[0] == 2 else iprot.readString() + ) + else: + iprot.skip(ftype) + elif fid == 4: + if ftype == TType.LIST: + self.requestList = [] + (_etype861, _size858) = iprot.readListBegin() + for _i862 in range(_size858): + _elem863 = WriteNotificationLogRequest() + _elem863.read(iprot) + self.requestList.append(_elem863) + iprot.readListEnd() + else: + iprot.skip(ftype) + else: + iprot.skip(ftype) + iprot.readFieldEnd() + iprot.readStructEnd() + + def write(self, oprot): + if oprot._fast_encode is not None and self.thrift_spec is not None: + oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec])) + return + oprot.writeStructBegin("WriteNotificationLogBatchRequest") + if self.catalog is not None: + oprot.writeFieldBegin("catalog", TType.STRING, 1) + oprot.writeString(self.catalog.encode("utf-8") if sys.version_info[0] == 2 else self.catalog) + oprot.writeFieldEnd() + if self.db is not None: + oprot.writeFieldBegin("db", TType.STRING, 2) + oprot.writeString(self.db.encode("utf-8") if sys.version_info[0] == 2 else self.db) + oprot.writeFieldEnd() + if self.table is not None: + oprot.writeFieldBegin("table", TType.STRING, 3) + oprot.writeString(self.table.encode("utf-8") if sys.version_info[0] == 2 else self.table) + oprot.writeFieldEnd() + if self.requestList is not None: + oprot.writeFieldBegin("requestList", TType.LIST, 4) + oprot.writeListBegin(TType.STRUCT, len(self.requestList)) + for iter864 in self.requestList: + iter864.write(oprot) + oprot.writeListEnd() + oprot.writeFieldEnd() + oprot.writeFieldStop() + oprot.writeStructEnd() + + def validate(self): + if self.catalog is None: + raise TProtocolException(message="Required field catalog is unset!") + if self.db is None: + raise TProtocolException(message="Required field db is unset!") + if self.table is None: + raise TProtocolException(message="Required field table is unset!") + if self.requestList is None: + raise TProtocolException(message="Required field requestList is unset!") + return + + def __repr__(self): + L = ["{}={!r}".format(key, value) for key, value in self.__dict__.items()] + return "{}({})".format(self.__class__.__name__, ", ".join(L)) + + def __eq__(self, other): + return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ + + def __ne__(self, other): + return not (self == other) + + +class WriteNotificationLogBatchResponse: + def read(self, iprot): + if ( + iprot._fast_decode is not None + and isinstance(iprot.trans, TTransport.CReadableTransport) + and self.thrift_spec is not None + ): + iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec]) + return + iprot.readStructBegin() + while True: + (fname, ftype, fid) = iprot.readFieldBegin() + if ftype == TType.STOP: + break + else: + iprot.skip(ftype) + iprot.readFieldEnd() + iprot.readStructEnd() + + def write(self, oprot): + if oprot._fast_encode is not None and self.thrift_spec is not None: + oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec])) + return + oprot.writeStructBegin("WriteNotificationLogBatchResponse") + oprot.writeFieldStop() + oprot.writeStructEnd() + + def validate(self): + return + + def __repr__(self): + L = ["{}={!r}".format(key, value) for key, value in self.__dict__.items()] + return "{}({})".format(self.__class__.__name__, ", ".join(L)) + + def __eq__(self, other): + return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ + + def __ne__(self, other): + return not (self == other) + + +class MetadataPpdResult: + """ + Attributes: + - metadata + - includeBitset + + """ + + def __init__( + self, + metadata=None, + includeBitset=None, + ): + self.metadata = metadata + self.includeBitset = includeBitset + + def read(self, iprot): + if ( + iprot._fast_decode is not None + and isinstance(iprot.trans, TTransport.CReadableTransport) + and self.thrift_spec is not None + ): + iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec]) + return + iprot.readStructBegin() + while True: + (fname, ftype, fid) = iprot.readFieldBegin() + if ftype == TType.STOP: + break + if fid == 1: + if ftype == TType.STRING: + self.metadata = iprot.readBinary() + else: + iprot.skip(ftype) + elif fid == 2: + if ftype == TType.STRING: + self.includeBitset = iprot.readBinary() + else: + iprot.skip(ftype) + else: + iprot.skip(ftype) + iprot.readFieldEnd() + iprot.readStructEnd() + + def write(self, oprot): + if oprot._fast_encode is not None and self.thrift_spec is not None: + oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec])) + return + oprot.writeStructBegin("MetadataPpdResult") + if self.metadata is not None: + oprot.writeFieldBegin("metadata", TType.STRING, 1) + oprot.writeBinary(self.metadata) + oprot.writeFieldEnd() + if self.includeBitset is not None: + oprot.writeFieldBegin("includeBitset", TType.STRING, 2) + oprot.writeBinary(self.includeBitset) + oprot.writeFieldEnd() + oprot.writeFieldStop() + oprot.writeStructEnd() + + def validate(self): + return + + def __repr__(self): + L = ["{}={!r}".format(key, value) for key, value in self.__dict__.items()] + return "{}({})".format(self.__class__.__name__, ", ".join(L)) + + def __eq__(self, other): + return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ + + def __ne__(self, other): + return not (self == other) + + +class GetFileMetadataByExprResult: + """ + Attributes: + - metadata + - isSupported + + """ + + def __init__( + self, + metadata=None, + isSupported=None, + ): + self.metadata = metadata + self.isSupported = isSupported + + def read(self, iprot): + if ( + iprot._fast_decode is not None + and isinstance(iprot.trans, TTransport.CReadableTransport) + and self.thrift_spec is not None + ): + iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec]) + return + iprot.readStructBegin() + while True: + (fname, ftype, fid) = iprot.readFieldBegin() + if ftype == TType.STOP: + break + if fid == 1: + if ftype == TType.MAP: + self.metadata = {} + (_ktype866, _vtype867, _size865) = iprot.readMapBegin() + for _i869 in range(_size865): + _key870 = iprot.readI64() + _val871 = MetadataPpdResult() + _val871.read(iprot) + self.metadata[_key870] = _val871 + iprot.readMapEnd() + else: + iprot.skip(ftype) + elif fid == 2: + if ftype == TType.BOOL: + self.isSupported = iprot.readBool() + else: + iprot.skip(ftype) + else: + iprot.skip(ftype) + iprot.readFieldEnd() + iprot.readStructEnd() + + def write(self, oprot): + if oprot._fast_encode is not None and self.thrift_spec is not None: + oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec])) + return + oprot.writeStructBegin("GetFileMetadataByExprResult") + if self.metadata is not None: + oprot.writeFieldBegin("metadata", TType.MAP, 1) + oprot.writeMapBegin(TType.I64, TType.STRUCT, len(self.metadata)) + for kiter872, viter873 in self.metadata.items(): + oprot.writeI64(kiter872) + viter873.write(oprot) + oprot.writeMapEnd() + oprot.writeFieldEnd() + if self.isSupported is not None: + oprot.writeFieldBegin("isSupported", TType.BOOL, 2) + oprot.writeBool(self.isSupported) + oprot.writeFieldEnd() + oprot.writeFieldStop() + oprot.writeStructEnd() + + def validate(self): + if self.metadata is None: + raise TProtocolException(message="Required field metadata is unset!") + if self.isSupported is None: + raise TProtocolException(message="Required field isSupported is unset!") + return + + def __repr__(self): + L = ["{}={!r}".format(key, value) for key, value in self.__dict__.items()] + return "{}({})".format(self.__class__.__name__, ", ".join(L)) + + def __eq__(self, other): + return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ + + def __ne__(self, other): + return not (self == other) + + +class GetFileMetadataByExprRequest: + """ + Attributes: + - fileIds + - expr + - doGetFooters + - type + + """ + + def __init__( + self, + fileIds=None, + expr=None, + doGetFooters=None, + type=None, + ): + self.fileIds = fileIds + self.expr = expr + self.doGetFooters = doGetFooters + self.type = type + + def read(self, iprot): + if ( + iprot._fast_decode is not None + and isinstance(iprot.trans, TTransport.CReadableTransport) + and self.thrift_spec is not None + ): + iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec]) + return + iprot.readStructBegin() + while True: + (fname, ftype, fid) = iprot.readFieldBegin() + if ftype == TType.STOP: + break + if fid == 1: + if ftype == TType.LIST: + self.fileIds = [] + (_etype877, _size874) = iprot.readListBegin() + for _i878 in range(_size874): + _elem879 = iprot.readI64() + self.fileIds.append(_elem879) + iprot.readListEnd() + else: + iprot.skip(ftype) + elif fid == 2: + if ftype == TType.STRING: + self.expr = iprot.readBinary() + else: + iprot.skip(ftype) + elif fid == 3: + if ftype == TType.BOOL: + self.doGetFooters = iprot.readBool() + else: + iprot.skip(ftype) + elif fid == 4: + if ftype == TType.I32: + self.type = iprot.readI32() + else: + iprot.skip(ftype) + else: + iprot.skip(ftype) + iprot.readFieldEnd() + iprot.readStructEnd() + + def write(self, oprot): + if oprot._fast_encode is not None and self.thrift_spec is not None: + oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec])) + return + oprot.writeStructBegin("GetFileMetadataByExprRequest") + if self.fileIds is not None: + oprot.writeFieldBegin("fileIds", TType.LIST, 1) + oprot.writeListBegin(TType.I64, len(self.fileIds)) + for iter880 in self.fileIds: + oprot.writeI64(iter880) + oprot.writeListEnd() + oprot.writeFieldEnd() + if self.expr is not None: + oprot.writeFieldBegin("expr", TType.STRING, 2) + oprot.writeBinary(self.expr) + oprot.writeFieldEnd() + if self.doGetFooters is not None: + oprot.writeFieldBegin("doGetFooters", TType.BOOL, 3) + oprot.writeBool(self.doGetFooters) + oprot.writeFieldEnd() + if self.type is not None: + oprot.writeFieldBegin("type", TType.I32, 4) + oprot.writeI32(self.type) + oprot.writeFieldEnd() + oprot.writeFieldStop() + oprot.writeStructEnd() + + def validate(self): + if self.fileIds is None: + raise TProtocolException(message="Required field fileIds is unset!") + if self.expr is None: + raise TProtocolException(message="Required field expr is unset!") + return + + def __repr__(self): + L = ["{}={!r}".format(key, value) for key, value in self.__dict__.items()] + return "{}({})".format(self.__class__.__name__, ", ".join(L)) + + def __eq__(self, other): + return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ + + def __ne__(self, other): + return not (self == other) + + +class GetFileMetadataResult: + """ + Attributes: + - metadata + - isSupported + + """ + + def __init__( + self, + metadata=None, + isSupported=None, + ): + self.metadata = metadata + self.isSupported = isSupported + + def read(self, iprot): + if ( + iprot._fast_decode is not None + and isinstance(iprot.trans, TTransport.CReadableTransport) + and self.thrift_spec is not None + ): + iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec]) + return + iprot.readStructBegin() + while True: + (fname, ftype, fid) = iprot.readFieldBegin() + if ftype == TType.STOP: + break + if fid == 1: + if ftype == TType.MAP: + self.metadata = {} + (_ktype882, _vtype883, _size881) = iprot.readMapBegin() + for _i885 in range(_size881): + _key886 = iprot.readI64() + _val887 = iprot.readBinary() + self.metadata[_key886] = _val887 + iprot.readMapEnd() + else: + iprot.skip(ftype) + elif fid == 2: + if ftype == TType.BOOL: + self.isSupported = iprot.readBool() + else: + iprot.skip(ftype) + else: + iprot.skip(ftype) + iprot.readFieldEnd() + iprot.readStructEnd() + + def write(self, oprot): + if oprot._fast_encode is not None and self.thrift_spec is not None: + oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec])) + return + oprot.writeStructBegin("GetFileMetadataResult") + if self.metadata is not None: + oprot.writeFieldBegin("metadata", TType.MAP, 1) + oprot.writeMapBegin(TType.I64, TType.STRING, len(self.metadata)) + for kiter888, viter889 in self.metadata.items(): + oprot.writeI64(kiter888) + oprot.writeBinary(viter889) + oprot.writeMapEnd() + oprot.writeFieldEnd() + if self.isSupported is not None: + oprot.writeFieldBegin("isSupported", TType.BOOL, 2) + oprot.writeBool(self.isSupported) + oprot.writeFieldEnd() + oprot.writeFieldStop() + oprot.writeStructEnd() + + def validate(self): + if self.metadata is None: + raise TProtocolException(message="Required field metadata is unset!") + if self.isSupported is None: + raise TProtocolException(message="Required field isSupported is unset!") + return + + def __repr__(self): + L = ["{}={!r}".format(key, value) for key, value in self.__dict__.items()] + return "{}({})".format(self.__class__.__name__, ", ".join(L)) + + def __eq__(self, other): + return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ + + def __ne__(self, other): + return not (self == other) + + +class GetFileMetadataRequest: + """ + Attributes: + - fileIds + + """ + + def __init__( + self, + fileIds=None, + ): + self.fileIds = fileIds + + def read(self, iprot): + if ( + iprot._fast_decode is not None + and isinstance(iprot.trans, TTransport.CReadableTransport) + and self.thrift_spec is not None + ): + iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec]) + return + iprot.readStructBegin() + while True: + (fname, ftype, fid) = iprot.readFieldBegin() + if ftype == TType.STOP: + break + if fid == 1: + if ftype == TType.LIST: + self.fileIds = [] + (_etype893, _size890) = iprot.readListBegin() + for _i894 in range(_size890): + _elem895 = iprot.readI64() + self.fileIds.append(_elem895) + iprot.readListEnd() + else: + iprot.skip(ftype) + else: + iprot.skip(ftype) + iprot.readFieldEnd() + iprot.readStructEnd() + + def write(self, oprot): + if oprot._fast_encode is not None and self.thrift_spec is not None: + oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec])) + return + oprot.writeStructBegin("GetFileMetadataRequest") + if self.fileIds is not None: + oprot.writeFieldBegin("fileIds", TType.LIST, 1) + oprot.writeListBegin(TType.I64, len(self.fileIds)) + for iter896 in self.fileIds: + oprot.writeI64(iter896) + oprot.writeListEnd() + oprot.writeFieldEnd() + oprot.writeFieldStop() + oprot.writeStructEnd() + + def validate(self): + if self.fileIds is None: + raise TProtocolException(message="Required field fileIds is unset!") + return + + def __repr__(self): + L = ["{}={!r}".format(key, value) for key, value in self.__dict__.items()] + return "{}({})".format(self.__class__.__name__, ", ".join(L)) + + def __eq__(self, other): + return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ + + def __ne__(self, other): + return not (self == other) + + +class PutFileMetadataResult: + def read(self, iprot): + if ( + iprot._fast_decode is not None + and isinstance(iprot.trans, TTransport.CReadableTransport) + and self.thrift_spec is not None + ): + iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec]) + return + iprot.readStructBegin() + while True: + (fname, ftype, fid) = iprot.readFieldBegin() + if ftype == TType.STOP: + break + else: + iprot.skip(ftype) + iprot.readFieldEnd() + iprot.readStructEnd() + + def write(self, oprot): + if oprot._fast_encode is not None and self.thrift_spec is not None: + oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec])) + return + oprot.writeStructBegin("PutFileMetadataResult") + oprot.writeFieldStop() + oprot.writeStructEnd() + + def validate(self): + return + + def __repr__(self): + L = ["{}={!r}".format(key, value) for key, value in self.__dict__.items()] + return "{}({})".format(self.__class__.__name__, ", ".join(L)) + + def __eq__(self, other): + return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ + + def __ne__(self, other): + return not (self == other) + + +class PutFileMetadataRequest: + """ + Attributes: + - fileIds + - metadata + - type + + """ + + def __init__( + self, + fileIds=None, + metadata=None, + type=None, + ): + self.fileIds = fileIds + self.metadata = metadata + self.type = type + + def read(self, iprot): + if ( + iprot._fast_decode is not None + and isinstance(iprot.trans, TTransport.CReadableTransport) + and self.thrift_spec is not None + ): + iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec]) + return + iprot.readStructBegin() + while True: + (fname, ftype, fid) = iprot.readFieldBegin() + if ftype == TType.STOP: + break + if fid == 1: + if ftype == TType.LIST: + self.fileIds = [] + (_etype900, _size897) = iprot.readListBegin() + for _i901 in range(_size897): + _elem902 = iprot.readI64() + self.fileIds.append(_elem902) + iprot.readListEnd() + else: + iprot.skip(ftype) + elif fid == 2: + if ftype == TType.LIST: + self.metadata = [] + (_etype906, _size903) = iprot.readListBegin() + for _i907 in range(_size903): + _elem908 = iprot.readBinary() + self.metadata.append(_elem908) + iprot.readListEnd() + else: + iprot.skip(ftype) + elif fid == 3: + if ftype == TType.I32: + self.type = iprot.readI32() + else: + iprot.skip(ftype) + else: + iprot.skip(ftype) + iprot.readFieldEnd() + iprot.readStructEnd() + + def write(self, oprot): + if oprot._fast_encode is not None and self.thrift_spec is not None: + oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec])) + return + oprot.writeStructBegin("PutFileMetadataRequest") + if self.fileIds is not None: + oprot.writeFieldBegin("fileIds", TType.LIST, 1) + oprot.writeListBegin(TType.I64, len(self.fileIds)) + for iter909 in self.fileIds: + oprot.writeI64(iter909) + oprot.writeListEnd() + oprot.writeFieldEnd() + if self.metadata is not None: + oprot.writeFieldBegin("metadata", TType.LIST, 2) + oprot.writeListBegin(TType.STRING, len(self.metadata)) + for iter910 in self.metadata: + oprot.writeBinary(iter910) + oprot.writeListEnd() + oprot.writeFieldEnd() + if self.type is not None: + oprot.writeFieldBegin("type", TType.I32, 3) + oprot.writeI32(self.type) + oprot.writeFieldEnd() + oprot.writeFieldStop() + oprot.writeStructEnd() + + def validate(self): + if self.fileIds is None: + raise TProtocolException(message="Required field fileIds is unset!") + if self.metadata is None: + raise TProtocolException(message="Required field metadata is unset!") + return + + def __repr__(self): + L = ["{}={!r}".format(key, value) for key, value in self.__dict__.items()] + return "{}({})".format(self.__class__.__name__, ", ".join(L)) + + def __eq__(self, other): + return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ + + def __ne__(self, other): + return not (self == other) + + +class ClearFileMetadataResult: + def read(self, iprot): + if ( + iprot._fast_decode is not None + and isinstance(iprot.trans, TTransport.CReadableTransport) + and self.thrift_spec is not None + ): + iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec]) + return + iprot.readStructBegin() + while True: + (fname, ftype, fid) = iprot.readFieldBegin() + if ftype == TType.STOP: + break + else: + iprot.skip(ftype) + iprot.readFieldEnd() + iprot.readStructEnd() + + def write(self, oprot): + if oprot._fast_encode is not None and self.thrift_spec is not None: + oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec])) + return + oprot.writeStructBegin("ClearFileMetadataResult") + oprot.writeFieldStop() + oprot.writeStructEnd() + + def validate(self): + return + + def __repr__(self): + L = ["{}={!r}".format(key, value) for key, value in self.__dict__.items()] + return "{}({})".format(self.__class__.__name__, ", ".join(L)) + + def __eq__(self, other): + return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ + + def __ne__(self, other): + return not (self == other) + + +class ClearFileMetadataRequest: + """ + Attributes: + - fileIds + + """ + + def __init__( + self, + fileIds=None, + ): + self.fileIds = fileIds + + def read(self, iprot): + if ( + iprot._fast_decode is not None + and isinstance(iprot.trans, TTransport.CReadableTransport) + and self.thrift_spec is not None + ): + iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec]) + return + iprot.readStructBegin() + while True: + (fname, ftype, fid) = iprot.readFieldBegin() + if ftype == TType.STOP: + break + if fid == 1: + if ftype == TType.LIST: + self.fileIds = [] + (_etype914, _size911) = iprot.readListBegin() + for _i915 in range(_size911): + _elem916 = iprot.readI64() + self.fileIds.append(_elem916) + iprot.readListEnd() + else: + iprot.skip(ftype) + else: + iprot.skip(ftype) + iprot.readFieldEnd() + iprot.readStructEnd() + + def write(self, oprot): + if oprot._fast_encode is not None and self.thrift_spec is not None: + oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec])) + return + oprot.writeStructBegin("ClearFileMetadataRequest") + if self.fileIds is not None: + oprot.writeFieldBegin("fileIds", TType.LIST, 1) + oprot.writeListBegin(TType.I64, len(self.fileIds)) + for iter917 in self.fileIds: + oprot.writeI64(iter917) + oprot.writeListEnd() + oprot.writeFieldEnd() + oprot.writeFieldStop() + oprot.writeStructEnd() + + def validate(self): + if self.fileIds is None: + raise TProtocolException(message="Required field fileIds is unset!") + return + + def __repr__(self): + L = ["{}={!r}".format(key, value) for key, value in self.__dict__.items()] + return "{}({})".format(self.__class__.__name__, ", ".join(L)) + + def __eq__(self, other): + return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ + + def __ne__(self, other): + return not (self == other) + + +class CacheFileMetadataResult: + """ + Attributes: + - isSupported + + """ + + def __init__( + self, + isSupported=None, + ): + self.isSupported = isSupported + + def read(self, iprot): + if ( + iprot._fast_decode is not None + and isinstance(iprot.trans, TTransport.CReadableTransport) + and self.thrift_spec is not None + ): + iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec]) + return + iprot.readStructBegin() + while True: + (fname, ftype, fid) = iprot.readFieldBegin() + if ftype == TType.STOP: + break + if fid == 1: + if ftype == TType.BOOL: + self.isSupported = iprot.readBool() + else: + iprot.skip(ftype) + else: + iprot.skip(ftype) + iprot.readFieldEnd() + iprot.readStructEnd() + + def write(self, oprot): + if oprot._fast_encode is not None and self.thrift_spec is not None: + oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec])) + return + oprot.writeStructBegin("CacheFileMetadataResult") + if self.isSupported is not None: + oprot.writeFieldBegin("isSupported", TType.BOOL, 1) + oprot.writeBool(self.isSupported) + oprot.writeFieldEnd() + oprot.writeFieldStop() + oprot.writeStructEnd() + + def validate(self): + if self.isSupported is None: + raise TProtocolException(message="Required field isSupported is unset!") + return + + def __repr__(self): + L = ["{}={!r}".format(key, value) for key, value in self.__dict__.items()] + return "{}({})".format(self.__class__.__name__, ", ".join(L)) + + def __eq__(self, other): + return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ + + def __ne__(self, other): + return not (self == other) + + +class CacheFileMetadataRequest: + """ + Attributes: + - dbName + - tblName + - partName + - isAllParts + + """ + + def __init__( + self, + dbName=None, + tblName=None, + partName=None, + isAllParts=None, + ): + self.dbName = dbName + self.tblName = tblName + self.partName = partName + self.isAllParts = isAllParts + + def read(self, iprot): + if ( + iprot._fast_decode is not None + and isinstance(iprot.trans, TTransport.CReadableTransport) + and self.thrift_spec is not None + ): + iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec]) + return + iprot.readStructBegin() + while True: + (fname, ftype, fid) = iprot.readFieldBegin() + if ftype == TType.STOP: + break + if fid == 1: + if ftype == TType.STRING: + self.dbName = ( + iprot.readString().decode("utf-8", errors="replace") if sys.version_info[0] == 2 else iprot.readString() + ) + else: + iprot.skip(ftype) + elif fid == 2: + if ftype == TType.STRING: + self.tblName = ( + iprot.readString().decode("utf-8", errors="replace") if sys.version_info[0] == 2 else iprot.readString() + ) + else: + iprot.skip(ftype) + elif fid == 3: + if ftype == TType.STRING: + self.partName = ( + iprot.readString().decode("utf-8", errors="replace") if sys.version_info[0] == 2 else iprot.readString() + ) + else: + iprot.skip(ftype) + elif fid == 4: + if ftype == TType.BOOL: + self.isAllParts = iprot.readBool() + else: + iprot.skip(ftype) + else: + iprot.skip(ftype) + iprot.readFieldEnd() + iprot.readStructEnd() + + def write(self, oprot): + if oprot._fast_encode is not None and self.thrift_spec is not None: + oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec])) + return + oprot.writeStructBegin("CacheFileMetadataRequest") + if self.dbName is not None: + oprot.writeFieldBegin("dbName", TType.STRING, 1) + oprot.writeString(self.dbName.encode("utf-8") if sys.version_info[0] == 2 else self.dbName) + oprot.writeFieldEnd() + if self.tblName is not None: + oprot.writeFieldBegin("tblName", TType.STRING, 2) + oprot.writeString(self.tblName.encode("utf-8") if sys.version_info[0] == 2 else self.tblName) + oprot.writeFieldEnd() + if self.partName is not None: + oprot.writeFieldBegin("partName", TType.STRING, 3) + oprot.writeString(self.partName.encode("utf-8") if sys.version_info[0] == 2 else self.partName) + oprot.writeFieldEnd() + if self.isAllParts is not None: + oprot.writeFieldBegin("isAllParts", TType.BOOL, 4) + oprot.writeBool(self.isAllParts) + oprot.writeFieldEnd() + oprot.writeFieldStop() + oprot.writeStructEnd() + + def validate(self): + if self.dbName is None: + raise TProtocolException(message="Required field dbName is unset!") + if self.tblName is None: + raise TProtocolException(message="Required field tblName is unset!") + return + + def __repr__(self): + L = ["{}={!r}".format(key, value) for key, value in self.__dict__.items()] + return "{}({})".format(self.__class__.__name__, ", ".join(L)) + + def __eq__(self, other): + return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ + + def __ne__(self, other): + return not (self == other) + + +class GetAllFunctionsResponse: + """ + Attributes: + - functions + + """ + + def __init__( + self, + functions=None, + ): + self.functions = functions + + def read(self, iprot): + if ( + iprot._fast_decode is not None + and isinstance(iprot.trans, TTransport.CReadableTransport) + and self.thrift_spec is not None + ): + iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec]) + return + iprot.readStructBegin() + while True: + (fname, ftype, fid) = iprot.readFieldBegin() + if ftype == TType.STOP: + break + if fid == 1: + if ftype == TType.LIST: + self.functions = [] + (_etype921, _size918) = iprot.readListBegin() + for _i922 in range(_size918): + _elem923 = Function() + _elem923.read(iprot) + self.functions.append(_elem923) + iprot.readListEnd() + else: + iprot.skip(ftype) + else: + iprot.skip(ftype) + iprot.readFieldEnd() + iprot.readStructEnd() + + def write(self, oprot): + if oprot._fast_encode is not None and self.thrift_spec is not None: + oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec])) + return + oprot.writeStructBegin("GetAllFunctionsResponse") + if self.functions is not None: + oprot.writeFieldBegin("functions", TType.LIST, 1) + oprot.writeListBegin(TType.STRUCT, len(self.functions)) + for iter924 in self.functions: + iter924.write(oprot) + oprot.writeListEnd() + oprot.writeFieldEnd() + oprot.writeFieldStop() + oprot.writeStructEnd() + + def validate(self): + return + + def __repr__(self): + L = ["{}={!r}".format(key, value) for key, value in self.__dict__.items()] + return "{}({})".format(self.__class__.__name__, ", ".join(L)) + + def __eq__(self, other): + return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ + + def __ne__(self, other): + return not (self == other) + + +class ClientCapabilities: + """ + Attributes: + - values + + """ + + def __init__( + self, + values=None, + ): + self.values = values + + def read(self, iprot): + if ( + iprot._fast_decode is not None + and isinstance(iprot.trans, TTransport.CReadableTransport) + and self.thrift_spec is not None + ): + iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec]) + return + iprot.readStructBegin() + while True: + (fname, ftype, fid) = iprot.readFieldBegin() + if ftype == TType.STOP: + break + if fid == 1: + if ftype == TType.LIST: + self.values = [] + (_etype928, _size925) = iprot.readListBegin() + for _i929 in range(_size925): + _elem930 = iprot.readI32() + self.values.append(_elem930) + iprot.readListEnd() + else: + iprot.skip(ftype) + else: + iprot.skip(ftype) + iprot.readFieldEnd() + iprot.readStructEnd() + + def write(self, oprot): + if oprot._fast_encode is not None and self.thrift_spec is not None: + oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec])) + return + oprot.writeStructBegin("ClientCapabilities") + if self.values is not None: + oprot.writeFieldBegin("values", TType.LIST, 1) + oprot.writeListBegin(TType.I32, len(self.values)) + for iter931 in self.values: + oprot.writeI32(iter931) + oprot.writeListEnd() + oprot.writeFieldEnd() + oprot.writeFieldStop() + oprot.writeStructEnd() + + def validate(self): + if self.values is None: + raise TProtocolException(message="Required field values is unset!") + return + + def __repr__(self): + L = ["{}={!r}".format(key, value) for key, value in self.__dict__.items()] + return "{}({})".format(self.__class__.__name__, ", ".join(L)) + + def __eq__(self, other): + return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ + + def __ne__(self, other): + return not (self == other) + + +class GetProjectionsSpec: + """ + Attributes: + - fieldList + - includeParamKeyPattern + - excludeParamKeyPattern + + """ + + def __init__( + self, + fieldList=None, + includeParamKeyPattern=None, + excludeParamKeyPattern=None, + ): + self.fieldList = fieldList + self.includeParamKeyPattern = includeParamKeyPattern + self.excludeParamKeyPattern = excludeParamKeyPattern + + def read(self, iprot): + if ( + iprot._fast_decode is not None + and isinstance(iprot.trans, TTransport.CReadableTransport) + and self.thrift_spec is not None + ): + iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec]) + return + iprot.readStructBegin() + while True: + (fname, ftype, fid) = iprot.readFieldBegin() + if ftype == TType.STOP: + break + if fid == 1: + if ftype == TType.LIST: + self.fieldList = [] + (_etype935, _size932) = iprot.readListBegin() + for _i936 in range(_size932): + _elem937 = ( + iprot.readString().decode("utf-8", errors="replace") + if sys.version_info[0] == 2 + else iprot.readString() + ) + self.fieldList.append(_elem937) + iprot.readListEnd() + else: + iprot.skip(ftype) + elif fid == 2: + if ftype == TType.STRING: + self.includeParamKeyPattern = ( + iprot.readString().decode("utf-8", errors="replace") if sys.version_info[0] == 2 else iprot.readString() + ) + else: + iprot.skip(ftype) + elif fid == 3: + if ftype == TType.STRING: + self.excludeParamKeyPattern = ( + iprot.readString().decode("utf-8", errors="replace") if sys.version_info[0] == 2 else iprot.readString() + ) + else: + iprot.skip(ftype) + else: + iprot.skip(ftype) + iprot.readFieldEnd() + iprot.readStructEnd() + + def write(self, oprot): + if oprot._fast_encode is not None and self.thrift_spec is not None: + oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec])) + return + oprot.writeStructBegin("GetProjectionsSpec") + if self.fieldList is not None: + oprot.writeFieldBegin("fieldList", TType.LIST, 1) + oprot.writeListBegin(TType.STRING, len(self.fieldList)) + for iter938 in self.fieldList: + oprot.writeString(iter938.encode("utf-8") if sys.version_info[0] == 2 else iter938) + oprot.writeListEnd() + oprot.writeFieldEnd() + if self.includeParamKeyPattern is not None: + oprot.writeFieldBegin("includeParamKeyPattern", TType.STRING, 2) + oprot.writeString( + self.includeParamKeyPattern.encode("utf-8") if sys.version_info[0] == 2 else self.includeParamKeyPattern + ) + oprot.writeFieldEnd() + if self.excludeParamKeyPattern is not None: + oprot.writeFieldBegin("excludeParamKeyPattern", TType.STRING, 3) + oprot.writeString( + self.excludeParamKeyPattern.encode("utf-8") if sys.version_info[0] == 2 else self.excludeParamKeyPattern + ) + oprot.writeFieldEnd() + oprot.writeFieldStop() + oprot.writeStructEnd() + + def validate(self): + return + + def __repr__(self): + L = ["{}={!r}".format(key, value) for key, value in self.__dict__.items()] + return "{}({})".format(self.__class__.__name__, ", ".join(L)) + + def __eq__(self, other): + return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ + + def __ne__(self, other): + return not (self == other) + + +class GetTableRequest: + """ + Attributes: + - dbName + - tblName + - capabilities + - catName + - validWriteIdList + - getColumnStats + - processorCapabilities + - processorIdentifier + - engine + - id + + """ + + def __init__( + self, + dbName=None, + tblName=None, + capabilities=None, + catName=None, + validWriteIdList=None, + getColumnStats=None, + processorCapabilities=None, + processorIdentifier=None, + engine=None, + id=-1, + ): + self.dbName = dbName + self.tblName = tblName + self.capabilities = capabilities + self.catName = catName + self.validWriteIdList = validWriteIdList + self.getColumnStats = getColumnStats + self.processorCapabilities = processorCapabilities + self.processorIdentifier = processorIdentifier + self.engine = engine + self.id = id + + def read(self, iprot): + if ( + iprot._fast_decode is not None + and isinstance(iprot.trans, TTransport.CReadableTransport) + and self.thrift_spec is not None + ): + iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec]) + return + iprot.readStructBegin() + while True: + (fname, ftype, fid) = iprot.readFieldBegin() + if ftype == TType.STOP: + break + if fid == 1: + if ftype == TType.STRING: + self.dbName = ( + iprot.readString().decode("utf-8", errors="replace") if sys.version_info[0] == 2 else iprot.readString() + ) + else: + iprot.skip(ftype) + elif fid == 2: + if ftype == TType.STRING: + self.tblName = ( + iprot.readString().decode("utf-8", errors="replace") if sys.version_info[0] == 2 else iprot.readString() + ) + else: + iprot.skip(ftype) + elif fid == 3: + if ftype == TType.STRUCT: + self.capabilities = ClientCapabilities() + self.capabilities.read(iprot) + else: + iprot.skip(ftype) + elif fid == 4: + if ftype == TType.STRING: + self.catName = ( + iprot.readString().decode("utf-8", errors="replace") if sys.version_info[0] == 2 else iprot.readString() + ) + else: + iprot.skip(ftype) + elif fid == 6: + if ftype == TType.STRING: + self.validWriteIdList = ( + iprot.readString().decode("utf-8", errors="replace") if sys.version_info[0] == 2 else iprot.readString() + ) + else: + iprot.skip(ftype) + elif fid == 7: + if ftype == TType.BOOL: + self.getColumnStats = iprot.readBool() + else: + iprot.skip(ftype) + elif fid == 8: + if ftype == TType.LIST: + self.processorCapabilities = [] + (_etype942, _size939) = iprot.readListBegin() + for _i943 in range(_size939): + _elem944 = ( + iprot.readString().decode("utf-8", errors="replace") + if sys.version_info[0] == 2 + else iprot.readString() + ) + self.processorCapabilities.append(_elem944) + iprot.readListEnd() + else: + iprot.skip(ftype) + elif fid == 9: + if ftype == TType.STRING: + self.processorIdentifier = ( + iprot.readString().decode("utf-8", errors="replace") if sys.version_info[0] == 2 else iprot.readString() + ) + else: + iprot.skip(ftype) + elif fid == 10: + if ftype == TType.STRING: + self.engine = ( + iprot.readString().decode("utf-8", errors="replace") if sys.version_info[0] == 2 else iprot.readString() + ) + else: + iprot.skip(ftype) + elif fid == 11: + if ftype == TType.I64: + self.id = iprot.readI64() + else: + iprot.skip(ftype) + else: + iprot.skip(ftype) + iprot.readFieldEnd() + iprot.readStructEnd() + + def write(self, oprot): + if oprot._fast_encode is not None and self.thrift_spec is not None: + oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec])) + return + oprot.writeStructBegin("GetTableRequest") + if self.dbName is not None: + oprot.writeFieldBegin("dbName", TType.STRING, 1) + oprot.writeString(self.dbName.encode("utf-8") if sys.version_info[0] == 2 else self.dbName) + oprot.writeFieldEnd() + if self.tblName is not None: + oprot.writeFieldBegin("tblName", TType.STRING, 2) + oprot.writeString(self.tblName.encode("utf-8") if sys.version_info[0] == 2 else self.tblName) + oprot.writeFieldEnd() + if self.capabilities is not None: + oprot.writeFieldBegin("capabilities", TType.STRUCT, 3) + self.capabilities.write(oprot) + oprot.writeFieldEnd() + if self.catName is not None: + oprot.writeFieldBegin("catName", TType.STRING, 4) + oprot.writeString(self.catName.encode("utf-8") if sys.version_info[0] == 2 else self.catName) + oprot.writeFieldEnd() + if self.validWriteIdList is not None: + oprot.writeFieldBegin("validWriteIdList", TType.STRING, 6) + oprot.writeString(self.validWriteIdList.encode("utf-8") if sys.version_info[0] == 2 else self.validWriteIdList) + oprot.writeFieldEnd() + if self.getColumnStats is not None: + oprot.writeFieldBegin("getColumnStats", TType.BOOL, 7) + oprot.writeBool(self.getColumnStats) + oprot.writeFieldEnd() + if self.processorCapabilities is not None: + oprot.writeFieldBegin("processorCapabilities", TType.LIST, 8) + oprot.writeListBegin(TType.STRING, len(self.processorCapabilities)) + for iter945 in self.processorCapabilities: + oprot.writeString(iter945.encode("utf-8") if sys.version_info[0] == 2 else iter945) + oprot.writeListEnd() + oprot.writeFieldEnd() + if self.processorIdentifier is not None: + oprot.writeFieldBegin("processorIdentifier", TType.STRING, 9) + oprot.writeString(self.processorIdentifier.encode("utf-8") if sys.version_info[0] == 2 else self.processorIdentifier) + oprot.writeFieldEnd() + if self.engine is not None: + oprot.writeFieldBegin("engine", TType.STRING, 10) + oprot.writeString(self.engine.encode("utf-8") if sys.version_info[0] == 2 else self.engine) + oprot.writeFieldEnd() + if self.id is not None: + oprot.writeFieldBegin("id", TType.I64, 11) + oprot.writeI64(self.id) + oprot.writeFieldEnd() + oprot.writeFieldStop() + oprot.writeStructEnd() + + def validate(self): + if self.dbName is None: + raise TProtocolException(message="Required field dbName is unset!") + if self.tblName is None: + raise TProtocolException(message="Required field tblName is unset!") + return + + def __repr__(self): + L = ["{}={!r}".format(key, value) for key, value in self.__dict__.items()] + return "{}({})".format(self.__class__.__name__, ", ".join(L)) + + def __eq__(self, other): + return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ + + def __ne__(self, other): + return not (self == other) + + +class GetTableResult: + """ + Attributes: + - table + - isStatsCompliant + + """ + + def __init__( + self, + table=None, + isStatsCompliant=None, + ): + self.table = table + self.isStatsCompliant = isStatsCompliant + + def read(self, iprot): + if ( + iprot._fast_decode is not None + and isinstance(iprot.trans, TTransport.CReadableTransport) + and self.thrift_spec is not None + ): + iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec]) + return + iprot.readStructBegin() + while True: + (fname, ftype, fid) = iprot.readFieldBegin() + if ftype == TType.STOP: + break + if fid == 1: + if ftype == TType.STRUCT: + self.table = Table() + self.table.read(iprot) + else: + iprot.skip(ftype) + elif fid == 2: + if ftype == TType.BOOL: + self.isStatsCompliant = iprot.readBool() + else: + iprot.skip(ftype) + else: + iprot.skip(ftype) + iprot.readFieldEnd() + iprot.readStructEnd() + + def write(self, oprot): + if oprot._fast_encode is not None and self.thrift_spec is not None: + oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec])) + return + oprot.writeStructBegin("GetTableResult") + if self.table is not None: + oprot.writeFieldBegin("table", TType.STRUCT, 1) + self.table.write(oprot) + oprot.writeFieldEnd() + if self.isStatsCompliant is not None: + oprot.writeFieldBegin("isStatsCompliant", TType.BOOL, 2) + oprot.writeBool(self.isStatsCompliant) + oprot.writeFieldEnd() + oprot.writeFieldStop() + oprot.writeStructEnd() + + def validate(self): + if self.table is None: + raise TProtocolException(message="Required field table is unset!") + return + + def __repr__(self): + L = ["{}={!r}".format(key, value) for key, value in self.__dict__.items()] + return "{}({})".format(self.__class__.__name__, ", ".join(L)) + + def __eq__(self, other): + return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ + + def __ne__(self, other): + return not (self == other) + + +class GetTablesRequest: + """ + Attributes: + - dbName + - tblNames + - capabilities + - catName + - processorCapabilities + - processorIdentifier + - projectionSpec + - tablesPattern + + """ + + def __init__( + self, + dbName=None, + tblNames=None, + capabilities=None, + catName=None, + processorCapabilities=None, + processorIdentifier=None, + projectionSpec=None, + tablesPattern=None, + ): + self.dbName = dbName + self.tblNames = tblNames + self.capabilities = capabilities + self.catName = catName + self.processorCapabilities = processorCapabilities + self.processorIdentifier = processorIdentifier + self.projectionSpec = projectionSpec + self.tablesPattern = tablesPattern + + def read(self, iprot): + if ( + iprot._fast_decode is not None + and isinstance(iprot.trans, TTransport.CReadableTransport) + and self.thrift_spec is not None + ): + iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec]) + return + iprot.readStructBegin() + while True: + (fname, ftype, fid) = iprot.readFieldBegin() + if ftype == TType.STOP: + break + if fid == 1: + if ftype == TType.STRING: + self.dbName = ( + iprot.readString().decode("utf-8", errors="replace") if sys.version_info[0] == 2 else iprot.readString() + ) + else: + iprot.skip(ftype) + elif fid == 2: + if ftype == TType.LIST: + self.tblNames = [] + (_etype949, _size946) = iprot.readListBegin() + for _i950 in range(_size946): + _elem951 = ( + iprot.readString().decode("utf-8", errors="replace") + if sys.version_info[0] == 2 + else iprot.readString() + ) + self.tblNames.append(_elem951) + iprot.readListEnd() + else: + iprot.skip(ftype) + elif fid == 3: + if ftype == TType.STRUCT: + self.capabilities = ClientCapabilities() + self.capabilities.read(iprot) + else: + iprot.skip(ftype) + elif fid == 4: + if ftype == TType.STRING: + self.catName = ( + iprot.readString().decode("utf-8", errors="replace") if sys.version_info[0] == 2 else iprot.readString() + ) + else: + iprot.skip(ftype) + elif fid == 5: + if ftype == TType.LIST: + self.processorCapabilities = [] + (_etype955, _size952) = iprot.readListBegin() + for _i956 in range(_size952): + _elem957 = ( + iprot.readString().decode("utf-8", errors="replace") + if sys.version_info[0] == 2 + else iprot.readString() + ) + self.processorCapabilities.append(_elem957) + iprot.readListEnd() + else: + iprot.skip(ftype) + elif fid == 6: + if ftype == TType.STRING: + self.processorIdentifier = ( + iprot.readString().decode("utf-8", errors="replace") if sys.version_info[0] == 2 else iprot.readString() + ) + else: + iprot.skip(ftype) + elif fid == 7: + if ftype == TType.STRUCT: + self.projectionSpec = GetProjectionsSpec() + self.projectionSpec.read(iprot) + else: + iprot.skip(ftype) + elif fid == 8: + if ftype == TType.STRING: + self.tablesPattern = ( + iprot.readString().decode("utf-8", errors="replace") if sys.version_info[0] == 2 else iprot.readString() + ) + else: + iprot.skip(ftype) + else: + iprot.skip(ftype) + iprot.readFieldEnd() + iprot.readStructEnd() + + def write(self, oprot): + if oprot._fast_encode is not None and self.thrift_spec is not None: + oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec])) + return + oprot.writeStructBegin("GetTablesRequest") + if self.dbName is not None: + oprot.writeFieldBegin("dbName", TType.STRING, 1) + oprot.writeString(self.dbName.encode("utf-8") if sys.version_info[0] == 2 else self.dbName) + oprot.writeFieldEnd() + if self.tblNames is not None: + oprot.writeFieldBegin("tblNames", TType.LIST, 2) + oprot.writeListBegin(TType.STRING, len(self.tblNames)) + for iter958 in self.tblNames: + oprot.writeString(iter958.encode("utf-8") if sys.version_info[0] == 2 else iter958) + oprot.writeListEnd() + oprot.writeFieldEnd() + if self.capabilities is not None: + oprot.writeFieldBegin("capabilities", TType.STRUCT, 3) + self.capabilities.write(oprot) + oprot.writeFieldEnd() + if self.catName is not None: + oprot.writeFieldBegin("catName", TType.STRING, 4) + oprot.writeString(self.catName.encode("utf-8") if sys.version_info[0] == 2 else self.catName) + oprot.writeFieldEnd() + if self.processorCapabilities is not None: + oprot.writeFieldBegin("processorCapabilities", TType.LIST, 5) + oprot.writeListBegin(TType.STRING, len(self.processorCapabilities)) + for iter959 in self.processorCapabilities: + oprot.writeString(iter959.encode("utf-8") if sys.version_info[0] == 2 else iter959) + oprot.writeListEnd() + oprot.writeFieldEnd() + if self.processorIdentifier is not None: + oprot.writeFieldBegin("processorIdentifier", TType.STRING, 6) + oprot.writeString(self.processorIdentifier.encode("utf-8") if sys.version_info[0] == 2 else self.processorIdentifier) + oprot.writeFieldEnd() + if self.projectionSpec is not None: + oprot.writeFieldBegin("projectionSpec", TType.STRUCT, 7) + self.projectionSpec.write(oprot) + oprot.writeFieldEnd() + if self.tablesPattern is not None: + oprot.writeFieldBegin("tablesPattern", TType.STRING, 8) + oprot.writeString(self.tablesPattern.encode("utf-8") if sys.version_info[0] == 2 else self.tablesPattern) + oprot.writeFieldEnd() + oprot.writeFieldStop() + oprot.writeStructEnd() + + def validate(self): + if self.dbName is None: + raise TProtocolException(message="Required field dbName is unset!") + return + + def __repr__(self): + L = ["{}={!r}".format(key, value) for key, value in self.__dict__.items()] + return "{}({})".format(self.__class__.__name__, ", ".join(L)) + + def __eq__(self, other): + return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ + + def __ne__(self, other): + return not (self == other) + + +class GetTablesResult: + """ + Attributes: + - tables + + """ + + def __init__( + self, + tables=None, + ): + self.tables = tables + + def read(self, iprot): + if ( + iprot._fast_decode is not None + and isinstance(iprot.trans, TTransport.CReadableTransport) + and self.thrift_spec is not None + ): + iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec]) + return + iprot.readStructBegin() + while True: + (fname, ftype, fid) = iprot.readFieldBegin() + if ftype == TType.STOP: + break + if fid == 1: + if ftype == TType.LIST: + self.tables = [] + (_etype963, _size960) = iprot.readListBegin() + for _i964 in range(_size960): + _elem965 = Table() + _elem965.read(iprot) + self.tables.append(_elem965) + iprot.readListEnd() + else: + iprot.skip(ftype) + else: + iprot.skip(ftype) + iprot.readFieldEnd() + iprot.readStructEnd() + + def write(self, oprot): + if oprot._fast_encode is not None and self.thrift_spec is not None: + oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec])) + return + oprot.writeStructBegin("GetTablesResult") + if self.tables is not None: + oprot.writeFieldBegin("tables", TType.LIST, 1) + oprot.writeListBegin(TType.STRUCT, len(self.tables)) + for iter966 in self.tables: + iter966.write(oprot) + oprot.writeListEnd() + oprot.writeFieldEnd() + oprot.writeFieldStop() + oprot.writeStructEnd() + + def validate(self): + if self.tables is None: + raise TProtocolException(message="Required field tables is unset!") + return + + def __repr__(self): + L = ["{}={!r}".format(key, value) for key, value in self.__dict__.items()] + return "{}({})".format(self.__class__.__name__, ", ".join(L)) + + def __eq__(self, other): + return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ + + def __ne__(self, other): + return not (self == other) + + +class GetTablesExtRequest: + """ + Attributes: + - catalog + - database + - tableNamePattern + - requestedFields + - limit + - processorCapabilities + - processorIdentifier + + """ + + def __init__( + self, + catalog=None, + database=None, + tableNamePattern=None, + requestedFields=None, + limit=None, + processorCapabilities=None, + processorIdentifier=None, + ): + self.catalog = catalog + self.database = database + self.tableNamePattern = tableNamePattern + self.requestedFields = requestedFields + self.limit = limit + self.processorCapabilities = processorCapabilities + self.processorIdentifier = processorIdentifier + + def read(self, iprot): + if ( + iprot._fast_decode is not None + and isinstance(iprot.trans, TTransport.CReadableTransport) + and self.thrift_spec is not None + ): + iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec]) + return + iprot.readStructBegin() + while True: + (fname, ftype, fid) = iprot.readFieldBegin() + if ftype == TType.STOP: + break + if fid == 1: + if ftype == TType.STRING: + self.catalog = ( + iprot.readString().decode("utf-8", errors="replace") if sys.version_info[0] == 2 else iprot.readString() + ) + else: + iprot.skip(ftype) + elif fid == 2: + if ftype == TType.STRING: + self.database = ( + iprot.readString().decode("utf-8", errors="replace") if sys.version_info[0] == 2 else iprot.readString() + ) + else: + iprot.skip(ftype) + elif fid == 3: + if ftype == TType.STRING: + self.tableNamePattern = ( + iprot.readString().decode("utf-8", errors="replace") if sys.version_info[0] == 2 else iprot.readString() + ) + else: + iprot.skip(ftype) + elif fid == 4: + if ftype == TType.I32: + self.requestedFields = iprot.readI32() + else: + iprot.skip(ftype) + elif fid == 5: + if ftype == TType.I32: + self.limit = iprot.readI32() + else: + iprot.skip(ftype) + elif fid == 6: + if ftype == TType.LIST: + self.processorCapabilities = [] + (_etype970, _size967) = iprot.readListBegin() + for _i971 in range(_size967): + _elem972 = ( + iprot.readString().decode("utf-8", errors="replace") + if sys.version_info[0] == 2 + else iprot.readString() + ) + self.processorCapabilities.append(_elem972) + iprot.readListEnd() + else: + iprot.skip(ftype) + elif fid == 7: + if ftype == TType.STRING: + self.processorIdentifier = ( + iprot.readString().decode("utf-8", errors="replace") if sys.version_info[0] == 2 else iprot.readString() + ) + else: + iprot.skip(ftype) + else: + iprot.skip(ftype) + iprot.readFieldEnd() + iprot.readStructEnd() + + def write(self, oprot): + if oprot._fast_encode is not None and self.thrift_spec is not None: + oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec])) + return + oprot.writeStructBegin("GetTablesExtRequest") + if self.catalog is not None: + oprot.writeFieldBegin("catalog", TType.STRING, 1) + oprot.writeString(self.catalog.encode("utf-8") if sys.version_info[0] == 2 else self.catalog) + oprot.writeFieldEnd() + if self.database is not None: + oprot.writeFieldBegin("database", TType.STRING, 2) + oprot.writeString(self.database.encode("utf-8") if sys.version_info[0] == 2 else self.database) + oprot.writeFieldEnd() + if self.tableNamePattern is not None: + oprot.writeFieldBegin("tableNamePattern", TType.STRING, 3) + oprot.writeString(self.tableNamePattern.encode("utf-8") if sys.version_info[0] == 2 else self.tableNamePattern) + oprot.writeFieldEnd() + if self.requestedFields is not None: + oprot.writeFieldBegin("requestedFields", TType.I32, 4) + oprot.writeI32(self.requestedFields) + oprot.writeFieldEnd() + if self.limit is not None: + oprot.writeFieldBegin("limit", TType.I32, 5) + oprot.writeI32(self.limit) + oprot.writeFieldEnd() + if self.processorCapabilities is not None: + oprot.writeFieldBegin("processorCapabilities", TType.LIST, 6) + oprot.writeListBegin(TType.STRING, len(self.processorCapabilities)) + for iter973 in self.processorCapabilities: + oprot.writeString(iter973.encode("utf-8") if sys.version_info[0] == 2 else iter973) + oprot.writeListEnd() + oprot.writeFieldEnd() + if self.processorIdentifier is not None: + oprot.writeFieldBegin("processorIdentifier", TType.STRING, 7) + oprot.writeString(self.processorIdentifier.encode("utf-8") if sys.version_info[0] == 2 else self.processorIdentifier) + oprot.writeFieldEnd() + oprot.writeFieldStop() + oprot.writeStructEnd() + + def validate(self): + if self.catalog is None: + raise TProtocolException(message="Required field catalog is unset!") + if self.database is None: + raise TProtocolException(message="Required field database is unset!") + if self.tableNamePattern is None: + raise TProtocolException(message="Required field tableNamePattern is unset!") + if self.requestedFields is None: + raise TProtocolException(message="Required field requestedFields is unset!") + return + + def __repr__(self): + L = ["{}={!r}".format(key, value) for key, value in self.__dict__.items()] + return "{}({})".format(self.__class__.__name__, ", ".join(L)) + + def __eq__(self, other): + return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ + + def __ne__(self, other): + return not (self == other) + + +class ExtendedTableInfo: + """ + Attributes: + - tblName + - accessType + - requiredReadCapabilities + - requiredWriteCapabilities + + """ + + def __init__( + self, + tblName=None, + accessType=None, + requiredReadCapabilities=None, + requiredWriteCapabilities=None, + ): + self.tblName = tblName + self.accessType = accessType + self.requiredReadCapabilities = requiredReadCapabilities + self.requiredWriteCapabilities = requiredWriteCapabilities + + def read(self, iprot): + if ( + iprot._fast_decode is not None + and isinstance(iprot.trans, TTransport.CReadableTransport) + and self.thrift_spec is not None + ): + iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec]) + return + iprot.readStructBegin() + while True: + (fname, ftype, fid) = iprot.readFieldBegin() + if ftype == TType.STOP: + break + if fid == 1: + if ftype == TType.STRING: + self.tblName = ( + iprot.readString().decode("utf-8", errors="replace") if sys.version_info[0] == 2 else iprot.readString() + ) + else: + iprot.skip(ftype) + elif fid == 2: + if ftype == TType.I32: + self.accessType = iprot.readI32() + else: + iprot.skip(ftype) + elif fid == 3: + if ftype == TType.LIST: + self.requiredReadCapabilities = [] + (_etype977, _size974) = iprot.readListBegin() + for _i978 in range(_size974): + _elem979 = ( + iprot.readString().decode("utf-8", errors="replace") + if sys.version_info[0] == 2 + else iprot.readString() + ) + self.requiredReadCapabilities.append(_elem979) + iprot.readListEnd() + else: + iprot.skip(ftype) + elif fid == 4: + if ftype == TType.LIST: + self.requiredWriteCapabilities = [] + (_etype983, _size980) = iprot.readListBegin() + for _i984 in range(_size980): + _elem985 = ( + iprot.readString().decode("utf-8", errors="replace") + if sys.version_info[0] == 2 + else iprot.readString() + ) + self.requiredWriteCapabilities.append(_elem985) + iprot.readListEnd() + else: + iprot.skip(ftype) + else: + iprot.skip(ftype) + iprot.readFieldEnd() + iprot.readStructEnd() + + def write(self, oprot): + if oprot._fast_encode is not None and self.thrift_spec is not None: + oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec])) + return + oprot.writeStructBegin("ExtendedTableInfo") + if self.tblName is not None: + oprot.writeFieldBegin("tblName", TType.STRING, 1) + oprot.writeString(self.tblName.encode("utf-8") if sys.version_info[0] == 2 else self.tblName) + oprot.writeFieldEnd() + if self.accessType is not None: + oprot.writeFieldBegin("accessType", TType.I32, 2) + oprot.writeI32(self.accessType) + oprot.writeFieldEnd() + if self.requiredReadCapabilities is not None: + oprot.writeFieldBegin("requiredReadCapabilities", TType.LIST, 3) + oprot.writeListBegin(TType.STRING, len(self.requiredReadCapabilities)) + for iter986 in self.requiredReadCapabilities: + oprot.writeString(iter986.encode("utf-8") if sys.version_info[0] == 2 else iter986) + oprot.writeListEnd() + oprot.writeFieldEnd() + if self.requiredWriteCapabilities is not None: + oprot.writeFieldBegin("requiredWriteCapabilities", TType.LIST, 4) + oprot.writeListBegin(TType.STRING, len(self.requiredWriteCapabilities)) + for iter987 in self.requiredWriteCapabilities: + oprot.writeString(iter987.encode("utf-8") if sys.version_info[0] == 2 else iter987) + oprot.writeListEnd() + oprot.writeFieldEnd() + oprot.writeFieldStop() + oprot.writeStructEnd() + + def validate(self): + if self.tblName is None: + raise TProtocolException(message="Required field tblName is unset!") + return + + def __repr__(self): + L = ["{}={!r}".format(key, value) for key, value in self.__dict__.items()] + return "{}({})".format(self.__class__.__name__, ", ".join(L)) + + def __eq__(self, other): + return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ + + def __ne__(self, other): + return not (self == other) + + +class GetDatabaseRequest: + """ + Attributes: + - name + - catalogName + - processorCapabilities + - processorIdentifier + + """ + + def __init__( + self, + name=None, + catalogName=None, + processorCapabilities=None, + processorIdentifier=None, + ): + self.name = name + self.catalogName = catalogName + self.processorCapabilities = processorCapabilities + self.processorIdentifier = processorIdentifier + + def read(self, iprot): + if ( + iprot._fast_decode is not None + and isinstance(iprot.trans, TTransport.CReadableTransport) + and self.thrift_spec is not None + ): + iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec]) + return + iprot.readStructBegin() + while True: + (fname, ftype, fid) = iprot.readFieldBegin() + if ftype == TType.STOP: + break + if fid == 1: + if ftype == TType.STRING: + self.name = ( + iprot.readString().decode("utf-8", errors="replace") if sys.version_info[0] == 2 else iprot.readString() + ) + else: + iprot.skip(ftype) + elif fid == 2: + if ftype == TType.STRING: + self.catalogName = ( + iprot.readString().decode("utf-8", errors="replace") if sys.version_info[0] == 2 else iprot.readString() + ) + else: + iprot.skip(ftype) + elif fid == 3: + if ftype == TType.LIST: + self.processorCapabilities = [] + (_etype991, _size988) = iprot.readListBegin() + for _i992 in range(_size988): + _elem993 = ( + iprot.readString().decode("utf-8", errors="replace") + if sys.version_info[0] == 2 + else iprot.readString() + ) + self.processorCapabilities.append(_elem993) + iprot.readListEnd() + else: + iprot.skip(ftype) + elif fid == 4: + if ftype == TType.STRING: + self.processorIdentifier = ( + iprot.readString().decode("utf-8", errors="replace") if sys.version_info[0] == 2 else iprot.readString() + ) + else: + iprot.skip(ftype) + else: + iprot.skip(ftype) + iprot.readFieldEnd() + iprot.readStructEnd() + + def write(self, oprot): + if oprot._fast_encode is not None and self.thrift_spec is not None: + oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec])) + return + oprot.writeStructBegin("GetDatabaseRequest") + if self.name is not None: + oprot.writeFieldBegin("name", TType.STRING, 1) + oprot.writeString(self.name.encode("utf-8") if sys.version_info[0] == 2 else self.name) + oprot.writeFieldEnd() + if self.catalogName is not None: + oprot.writeFieldBegin("catalogName", TType.STRING, 2) + oprot.writeString(self.catalogName.encode("utf-8") if sys.version_info[0] == 2 else self.catalogName) + oprot.writeFieldEnd() + if self.processorCapabilities is not None: + oprot.writeFieldBegin("processorCapabilities", TType.LIST, 3) + oprot.writeListBegin(TType.STRING, len(self.processorCapabilities)) + for iter994 in self.processorCapabilities: + oprot.writeString(iter994.encode("utf-8") if sys.version_info[0] == 2 else iter994) + oprot.writeListEnd() + oprot.writeFieldEnd() + if self.processorIdentifier is not None: + oprot.writeFieldBegin("processorIdentifier", TType.STRING, 4) + oprot.writeString(self.processorIdentifier.encode("utf-8") if sys.version_info[0] == 2 else self.processorIdentifier) + oprot.writeFieldEnd() + oprot.writeFieldStop() + oprot.writeStructEnd() + + def validate(self): + return + + def __repr__(self): + L = ["{}={!r}".format(key, value) for key, value in self.__dict__.items()] + return "{}({})".format(self.__class__.__name__, ", ".join(L)) + + def __eq__(self, other): + return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ + + def __ne__(self, other): + return not (self == other) + + +class DropDatabaseRequest: + """ + Attributes: + - name + - catalogName + - ignoreUnknownDb + - deleteData + - cascade + - softDelete + - txnId + - deleteManagedDir + + """ + + def __init__( + self, + name=None, + catalogName=None, + ignoreUnknownDb=None, + deleteData=None, + cascade=None, + softDelete=False, + txnId=0, + deleteManagedDir=True, + ): + self.name = name + self.catalogName = catalogName + self.ignoreUnknownDb = ignoreUnknownDb + self.deleteData = deleteData + self.cascade = cascade + self.softDelete = softDelete + self.txnId = txnId + self.deleteManagedDir = deleteManagedDir + + def read(self, iprot): + if ( + iprot._fast_decode is not None + and isinstance(iprot.trans, TTransport.CReadableTransport) + and self.thrift_spec is not None + ): + iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec]) + return + iprot.readStructBegin() + while True: + (fname, ftype, fid) = iprot.readFieldBegin() + if ftype == TType.STOP: + break + if fid == 1: + if ftype == TType.STRING: + self.name = ( + iprot.readString().decode("utf-8", errors="replace") if sys.version_info[0] == 2 else iprot.readString() + ) + else: + iprot.skip(ftype) + elif fid == 2: + if ftype == TType.STRING: + self.catalogName = ( + iprot.readString().decode("utf-8", errors="replace") if sys.version_info[0] == 2 else iprot.readString() + ) + else: + iprot.skip(ftype) + elif fid == 3: + if ftype == TType.BOOL: + self.ignoreUnknownDb = iprot.readBool() + else: + iprot.skip(ftype) + elif fid == 4: + if ftype == TType.BOOL: + self.deleteData = iprot.readBool() + else: + iprot.skip(ftype) + elif fid == 5: + if ftype == TType.BOOL: + self.cascade = iprot.readBool() + else: + iprot.skip(ftype) + elif fid == 6: + if ftype == TType.BOOL: + self.softDelete = iprot.readBool() + else: + iprot.skip(ftype) + elif fid == 7: + if ftype == TType.I64: + self.txnId = iprot.readI64() + else: + iprot.skip(ftype) + elif fid == 8: + if ftype == TType.BOOL: + self.deleteManagedDir = iprot.readBool() + else: + iprot.skip(ftype) + else: + iprot.skip(ftype) + iprot.readFieldEnd() + iprot.readStructEnd() + + def write(self, oprot): + if oprot._fast_encode is not None and self.thrift_spec is not None: + oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec])) + return + oprot.writeStructBegin("DropDatabaseRequest") + if self.name is not None: + oprot.writeFieldBegin("name", TType.STRING, 1) + oprot.writeString(self.name.encode("utf-8") if sys.version_info[0] == 2 else self.name) + oprot.writeFieldEnd() + if self.catalogName is not None: + oprot.writeFieldBegin("catalogName", TType.STRING, 2) + oprot.writeString(self.catalogName.encode("utf-8") if sys.version_info[0] == 2 else self.catalogName) + oprot.writeFieldEnd() + if self.ignoreUnknownDb is not None: + oprot.writeFieldBegin("ignoreUnknownDb", TType.BOOL, 3) + oprot.writeBool(self.ignoreUnknownDb) + oprot.writeFieldEnd() + if self.deleteData is not None: + oprot.writeFieldBegin("deleteData", TType.BOOL, 4) + oprot.writeBool(self.deleteData) + oprot.writeFieldEnd() + if self.cascade is not None: + oprot.writeFieldBegin("cascade", TType.BOOL, 5) + oprot.writeBool(self.cascade) + oprot.writeFieldEnd() + if self.softDelete is not None: + oprot.writeFieldBegin("softDelete", TType.BOOL, 6) + oprot.writeBool(self.softDelete) + oprot.writeFieldEnd() + if self.txnId is not None: + oprot.writeFieldBegin("txnId", TType.I64, 7) + oprot.writeI64(self.txnId) + oprot.writeFieldEnd() + if self.deleteManagedDir is not None: + oprot.writeFieldBegin("deleteManagedDir", TType.BOOL, 8) + oprot.writeBool(self.deleteManagedDir) + oprot.writeFieldEnd() + oprot.writeFieldStop() + oprot.writeStructEnd() + + def validate(self): + if self.name is None: + raise TProtocolException(message="Required field name is unset!") + if self.ignoreUnknownDb is None: + raise TProtocolException(message="Required field ignoreUnknownDb is unset!") + if self.deleteData is None: + raise TProtocolException(message="Required field deleteData is unset!") + if self.cascade is None: + raise TProtocolException(message="Required field cascade is unset!") + return + + def __repr__(self): + L = ["{}={!r}".format(key, value) for key, value in self.__dict__.items()] + return "{}({})".format(self.__class__.__name__, ", ".join(L)) + + def __eq__(self, other): + return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ + + def __ne__(self, other): + return not (self == other) + + +class CmRecycleRequest: + """ + Attributes: + - dataPath + - purge + + """ + + def __init__( + self, + dataPath=None, + purge=None, + ): + self.dataPath = dataPath + self.purge = purge + + def read(self, iprot): + if ( + iprot._fast_decode is not None + and isinstance(iprot.trans, TTransport.CReadableTransport) + and self.thrift_spec is not None + ): + iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec]) + return + iprot.readStructBegin() + while True: + (fname, ftype, fid) = iprot.readFieldBegin() + if ftype == TType.STOP: + break + if fid == 1: + if ftype == TType.STRING: + self.dataPath = ( + iprot.readString().decode("utf-8", errors="replace") if sys.version_info[0] == 2 else iprot.readString() + ) + else: + iprot.skip(ftype) + elif fid == 2: + if ftype == TType.BOOL: + self.purge = iprot.readBool() + else: + iprot.skip(ftype) + else: + iprot.skip(ftype) + iprot.readFieldEnd() + iprot.readStructEnd() + + def write(self, oprot): + if oprot._fast_encode is not None and self.thrift_spec is not None: + oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec])) + return + oprot.writeStructBegin("CmRecycleRequest") + if self.dataPath is not None: + oprot.writeFieldBegin("dataPath", TType.STRING, 1) + oprot.writeString(self.dataPath.encode("utf-8") if sys.version_info[0] == 2 else self.dataPath) + oprot.writeFieldEnd() + if self.purge is not None: + oprot.writeFieldBegin("purge", TType.BOOL, 2) + oprot.writeBool(self.purge) + oprot.writeFieldEnd() + oprot.writeFieldStop() + oprot.writeStructEnd() + + def validate(self): + if self.dataPath is None: + raise TProtocolException(message="Required field dataPath is unset!") + if self.purge is None: + raise TProtocolException(message="Required field purge is unset!") + return + + def __repr__(self): + L = ["{}={!r}".format(key, value) for key, value in self.__dict__.items()] + return "{}({})".format(self.__class__.__name__, ", ".join(L)) + + def __eq__(self, other): + return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ + + def __ne__(self, other): + return not (self == other) + + +class CmRecycleResponse: + def read(self, iprot): + if ( + iprot._fast_decode is not None + and isinstance(iprot.trans, TTransport.CReadableTransport) + and self.thrift_spec is not None + ): + iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec]) + return + iprot.readStructBegin() + while True: + (fname, ftype, fid) = iprot.readFieldBegin() + if ftype == TType.STOP: + break + else: + iprot.skip(ftype) + iprot.readFieldEnd() + iprot.readStructEnd() + + def write(self, oprot): + if oprot._fast_encode is not None and self.thrift_spec is not None: + oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec])) + return + oprot.writeStructBegin("CmRecycleResponse") + oprot.writeFieldStop() + oprot.writeStructEnd() + + def validate(self): + return + + def __repr__(self): + L = ["{}={!r}".format(key, value) for key, value in self.__dict__.items()] + return "{}({})".format(self.__class__.__name__, ", ".join(L)) + + def __eq__(self, other): + return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ + + def __ne__(self, other): + return not (self == other) + + +class TableMeta: + """ + Attributes: + - dbName + - tableName + - tableType + - comments + - catName + + """ + + def __init__( + self, + dbName=None, + tableName=None, + tableType=None, + comments=None, + catName=None, + ): + self.dbName = dbName + self.tableName = tableName + self.tableType = tableType + self.comments = comments + self.catName = catName + + def read(self, iprot): + if ( + iprot._fast_decode is not None + and isinstance(iprot.trans, TTransport.CReadableTransport) + and self.thrift_spec is not None + ): + iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec]) + return + iprot.readStructBegin() + while True: + (fname, ftype, fid) = iprot.readFieldBegin() + if ftype == TType.STOP: + break + if fid == 1: + if ftype == TType.STRING: + self.dbName = ( + iprot.readString().decode("utf-8", errors="replace") if sys.version_info[0] == 2 else iprot.readString() + ) + else: + iprot.skip(ftype) + elif fid == 2: + if ftype == TType.STRING: + self.tableName = ( + iprot.readString().decode("utf-8", errors="replace") if sys.version_info[0] == 2 else iprot.readString() + ) + else: + iprot.skip(ftype) + elif fid == 3: + if ftype == TType.STRING: + self.tableType = ( + iprot.readString().decode("utf-8", errors="replace") if sys.version_info[0] == 2 else iprot.readString() + ) + else: + iprot.skip(ftype) + elif fid == 4: + if ftype == TType.STRING: + self.comments = ( + iprot.readString().decode("utf-8", errors="replace") if sys.version_info[0] == 2 else iprot.readString() + ) + else: + iprot.skip(ftype) + elif fid == 5: + if ftype == TType.STRING: + self.catName = ( + iprot.readString().decode("utf-8", errors="replace") if sys.version_info[0] == 2 else iprot.readString() + ) + else: + iprot.skip(ftype) + else: + iprot.skip(ftype) + iprot.readFieldEnd() + iprot.readStructEnd() + + def write(self, oprot): + if oprot._fast_encode is not None and self.thrift_spec is not None: + oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec])) + return + oprot.writeStructBegin("TableMeta") + if self.dbName is not None: + oprot.writeFieldBegin("dbName", TType.STRING, 1) + oprot.writeString(self.dbName.encode("utf-8") if sys.version_info[0] == 2 else self.dbName) + oprot.writeFieldEnd() + if self.tableName is not None: + oprot.writeFieldBegin("tableName", TType.STRING, 2) + oprot.writeString(self.tableName.encode("utf-8") if sys.version_info[0] == 2 else self.tableName) + oprot.writeFieldEnd() + if self.tableType is not None: + oprot.writeFieldBegin("tableType", TType.STRING, 3) + oprot.writeString(self.tableType.encode("utf-8") if sys.version_info[0] == 2 else self.tableType) + oprot.writeFieldEnd() + if self.comments is not None: + oprot.writeFieldBegin("comments", TType.STRING, 4) + oprot.writeString(self.comments.encode("utf-8") if sys.version_info[0] == 2 else self.comments) + oprot.writeFieldEnd() + if self.catName is not None: + oprot.writeFieldBegin("catName", TType.STRING, 5) + oprot.writeString(self.catName.encode("utf-8") if sys.version_info[0] == 2 else self.catName) + oprot.writeFieldEnd() + oprot.writeFieldStop() + oprot.writeStructEnd() + + def validate(self): + if self.dbName is None: + raise TProtocolException(message="Required field dbName is unset!") + if self.tableName is None: + raise TProtocolException(message="Required field tableName is unset!") + if self.tableType is None: + raise TProtocolException(message="Required field tableType is unset!") + return + + def __repr__(self): + L = ["{}={!r}".format(key, value) for key, value in self.__dict__.items()] + return "{}({})".format(self.__class__.__name__, ", ".join(L)) + + def __eq__(self, other): + return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ + + def __ne__(self, other): + return not (self == other) + + +class Materialization: + """ + Attributes: + - sourceTablesUpdateDeleteModified + - sourceTablesCompacted + + """ + + def __init__( + self, + sourceTablesUpdateDeleteModified=None, + sourceTablesCompacted=None, + ): + self.sourceTablesUpdateDeleteModified = sourceTablesUpdateDeleteModified + self.sourceTablesCompacted = sourceTablesCompacted + + def read(self, iprot): + if ( + iprot._fast_decode is not None + and isinstance(iprot.trans, TTransport.CReadableTransport) + and self.thrift_spec is not None + ): + iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec]) + return + iprot.readStructBegin() + while True: + (fname, ftype, fid) = iprot.readFieldBegin() + if ftype == TType.STOP: + break + if fid == 1: + if ftype == TType.BOOL: + self.sourceTablesUpdateDeleteModified = iprot.readBool() + else: + iprot.skip(ftype) + elif fid == 2: + if ftype == TType.BOOL: + self.sourceTablesCompacted = iprot.readBool() + else: + iprot.skip(ftype) + else: + iprot.skip(ftype) + iprot.readFieldEnd() + iprot.readStructEnd() + + def write(self, oprot): + if oprot._fast_encode is not None and self.thrift_spec is not None: + oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec])) + return + oprot.writeStructBegin("Materialization") + if self.sourceTablesUpdateDeleteModified is not None: + oprot.writeFieldBegin("sourceTablesUpdateDeleteModified", TType.BOOL, 1) + oprot.writeBool(self.sourceTablesUpdateDeleteModified) + oprot.writeFieldEnd() + if self.sourceTablesCompacted is not None: + oprot.writeFieldBegin("sourceTablesCompacted", TType.BOOL, 2) + oprot.writeBool(self.sourceTablesCompacted) + oprot.writeFieldEnd() + oprot.writeFieldStop() + oprot.writeStructEnd() + + def validate(self): + if self.sourceTablesUpdateDeleteModified is None: + raise TProtocolException(message="Required field sourceTablesUpdateDeleteModified is unset!") + if self.sourceTablesCompacted is None: + raise TProtocolException(message="Required field sourceTablesCompacted is unset!") + return + + def __repr__(self): + L = ["{}={!r}".format(key, value) for key, value in self.__dict__.items()] + return "{}({})".format(self.__class__.__name__, ", ".join(L)) + + def __eq__(self, other): + return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ + + def __ne__(self, other): + return not (self == other) + + +class WMResourcePlan: + """ + Attributes: + - name + - status + - queryParallelism + - defaultPoolPath + - ns + + """ + + def __init__( + self, + name=None, + status=None, + queryParallelism=None, + defaultPoolPath=None, + ns=None, + ): + self.name = name + self.status = status + self.queryParallelism = queryParallelism + self.defaultPoolPath = defaultPoolPath + self.ns = ns + + def read(self, iprot): + if ( + iprot._fast_decode is not None + and isinstance(iprot.trans, TTransport.CReadableTransport) + and self.thrift_spec is not None + ): + iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec]) + return + iprot.readStructBegin() + while True: + (fname, ftype, fid) = iprot.readFieldBegin() + if ftype == TType.STOP: + break + if fid == 1: + if ftype == TType.STRING: + self.name = ( + iprot.readString().decode("utf-8", errors="replace") if sys.version_info[0] == 2 else iprot.readString() + ) + else: + iprot.skip(ftype) + elif fid == 2: + if ftype == TType.I32: + self.status = iprot.readI32() + else: + iprot.skip(ftype) + elif fid == 3: + if ftype == TType.I32: + self.queryParallelism = iprot.readI32() + else: + iprot.skip(ftype) + elif fid == 4: + if ftype == TType.STRING: + self.defaultPoolPath = ( + iprot.readString().decode("utf-8", errors="replace") if sys.version_info[0] == 2 else iprot.readString() + ) + else: + iprot.skip(ftype) + elif fid == 5: + if ftype == TType.STRING: + self.ns = ( + iprot.readString().decode("utf-8", errors="replace") if sys.version_info[0] == 2 else iprot.readString() + ) + else: + iprot.skip(ftype) + else: + iprot.skip(ftype) + iprot.readFieldEnd() + iprot.readStructEnd() + + def write(self, oprot): + if oprot._fast_encode is not None and self.thrift_spec is not None: + oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec])) + return + oprot.writeStructBegin("WMResourcePlan") + if self.name is not None: + oprot.writeFieldBegin("name", TType.STRING, 1) + oprot.writeString(self.name.encode("utf-8") if sys.version_info[0] == 2 else self.name) + oprot.writeFieldEnd() + if self.status is not None: + oprot.writeFieldBegin("status", TType.I32, 2) + oprot.writeI32(self.status) + oprot.writeFieldEnd() + if self.queryParallelism is not None: + oprot.writeFieldBegin("queryParallelism", TType.I32, 3) + oprot.writeI32(self.queryParallelism) + oprot.writeFieldEnd() + if self.defaultPoolPath is not None: + oprot.writeFieldBegin("defaultPoolPath", TType.STRING, 4) + oprot.writeString(self.defaultPoolPath.encode("utf-8") if sys.version_info[0] == 2 else self.defaultPoolPath) + oprot.writeFieldEnd() + if self.ns is not None: + oprot.writeFieldBegin("ns", TType.STRING, 5) + oprot.writeString(self.ns.encode("utf-8") if sys.version_info[0] == 2 else self.ns) + oprot.writeFieldEnd() + oprot.writeFieldStop() + oprot.writeStructEnd() + + def validate(self): + if self.name is None: + raise TProtocolException(message="Required field name is unset!") + return + + def __repr__(self): + L = ["{}={!r}".format(key, value) for key, value in self.__dict__.items()] + return "{}({})".format(self.__class__.__name__, ", ".join(L)) + + def __eq__(self, other): + return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ + + def __ne__(self, other): + return not (self == other) + + +class WMNullableResourcePlan: + """ + Attributes: + - name + - status + - queryParallelism + - isSetQueryParallelism + - defaultPoolPath + - isSetDefaultPoolPath + - ns + + """ + + def __init__( + self, + name=None, + status=None, + queryParallelism=None, + isSetQueryParallelism=None, + defaultPoolPath=None, + isSetDefaultPoolPath=None, + ns=None, + ): + self.name = name + self.status = status + self.queryParallelism = queryParallelism + self.isSetQueryParallelism = isSetQueryParallelism + self.defaultPoolPath = defaultPoolPath + self.isSetDefaultPoolPath = isSetDefaultPoolPath + self.ns = ns + + def read(self, iprot): + if ( + iprot._fast_decode is not None + and isinstance(iprot.trans, TTransport.CReadableTransport) + and self.thrift_spec is not None + ): + iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec]) + return + iprot.readStructBegin() + while True: + (fname, ftype, fid) = iprot.readFieldBegin() + if ftype == TType.STOP: + break + if fid == 1: + if ftype == TType.STRING: + self.name = ( + iprot.readString().decode("utf-8", errors="replace") if sys.version_info[0] == 2 else iprot.readString() + ) + else: + iprot.skip(ftype) + elif fid == 2: + if ftype == TType.I32: + self.status = iprot.readI32() + else: + iprot.skip(ftype) + elif fid == 4: + if ftype == TType.I32: + self.queryParallelism = iprot.readI32() + else: + iprot.skip(ftype) + elif fid == 5: + if ftype == TType.BOOL: + self.isSetQueryParallelism = iprot.readBool() + else: + iprot.skip(ftype) + elif fid == 6: + if ftype == TType.STRING: + self.defaultPoolPath = ( + iprot.readString().decode("utf-8", errors="replace") if sys.version_info[0] == 2 else iprot.readString() + ) + else: + iprot.skip(ftype) + elif fid == 7: + if ftype == TType.BOOL: + self.isSetDefaultPoolPath = iprot.readBool() + else: + iprot.skip(ftype) + elif fid == 8: + if ftype == TType.STRING: + self.ns = ( + iprot.readString().decode("utf-8", errors="replace") if sys.version_info[0] == 2 else iprot.readString() + ) + else: + iprot.skip(ftype) + else: + iprot.skip(ftype) + iprot.readFieldEnd() + iprot.readStructEnd() + + def write(self, oprot): + if oprot._fast_encode is not None and self.thrift_spec is not None: + oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec])) + return + oprot.writeStructBegin("WMNullableResourcePlan") + if self.name is not None: + oprot.writeFieldBegin("name", TType.STRING, 1) + oprot.writeString(self.name.encode("utf-8") if sys.version_info[0] == 2 else self.name) + oprot.writeFieldEnd() + if self.status is not None: + oprot.writeFieldBegin("status", TType.I32, 2) + oprot.writeI32(self.status) + oprot.writeFieldEnd() + if self.queryParallelism is not None: + oprot.writeFieldBegin("queryParallelism", TType.I32, 4) + oprot.writeI32(self.queryParallelism) + oprot.writeFieldEnd() + if self.isSetQueryParallelism is not None: + oprot.writeFieldBegin("isSetQueryParallelism", TType.BOOL, 5) + oprot.writeBool(self.isSetQueryParallelism) + oprot.writeFieldEnd() + if self.defaultPoolPath is not None: + oprot.writeFieldBegin("defaultPoolPath", TType.STRING, 6) + oprot.writeString(self.defaultPoolPath.encode("utf-8") if sys.version_info[0] == 2 else self.defaultPoolPath) + oprot.writeFieldEnd() + if self.isSetDefaultPoolPath is not None: + oprot.writeFieldBegin("isSetDefaultPoolPath", TType.BOOL, 7) + oprot.writeBool(self.isSetDefaultPoolPath) + oprot.writeFieldEnd() + if self.ns is not None: + oprot.writeFieldBegin("ns", TType.STRING, 8) + oprot.writeString(self.ns.encode("utf-8") if sys.version_info[0] == 2 else self.ns) + oprot.writeFieldEnd() + oprot.writeFieldStop() + oprot.writeStructEnd() + + def validate(self): + return + + def __repr__(self): + L = ["{}={!r}".format(key, value) for key, value in self.__dict__.items()] + return "{}({})".format(self.__class__.__name__, ", ".join(L)) + + def __eq__(self, other): + return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ + + def __ne__(self, other): + return not (self == other) + + +class WMPool: + """ + Attributes: + - resourcePlanName + - poolPath + - allocFraction + - queryParallelism + - schedulingPolicy + - ns + + """ + + def __init__( + self, + resourcePlanName=None, + poolPath=None, + allocFraction=None, + queryParallelism=None, + schedulingPolicy=None, + ns=None, + ): + self.resourcePlanName = resourcePlanName + self.poolPath = poolPath + self.allocFraction = allocFraction + self.queryParallelism = queryParallelism + self.schedulingPolicy = schedulingPolicy + self.ns = ns + + def read(self, iprot): + if ( + iprot._fast_decode is not None + and isinstance(iprot.trans, TTransport.CReadableTransport) + and self.thrift_spec is not None + ): + iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec]) + return + iprot.readStructBegin() + while True: + (fname, ftype, fid) = iprot.readFieldBegin() + if ftype == TType.STOP: + break + if fid == 1: + if ftype == TType.STRING: + self.resourcePlanName = ( + iprot.readString().decode("utf-8", errors="replace") if sys.version_info[0] == 2 else iprot.readString() + ) + else: + iprot.skip(ftype) + elif fid == 2: + if ftype == TType.STRING: + self.poolPath = ( + iprot.readString().decode("utf-8", errors="replace") if sys.version_info[0] == 2 else iprot.readString() + ) + else: + iprot.skip(ftype) + elif fid == 3: + if ftype == TType.DOUBLE: + self.allocFraction = iprot.readDouble() + else: + iprot.skip(ftype) + elif fid == 4: + if ftype == TType.I32: + self.queryParallelism = iprot.readI32() + else: + iprot.skip(ftype) + elif fid == 5: + if ftype == TType.STRING: + self.schedulingPolicy = ( + iprot.readString().decode("utf-8", errors="replace") if sys.version_info[0] == 2 else iprot.readString() + ) + else: + iprot.skip(ftype) + elif fid == 6: + if ftype == TType.STRING: + self.ns = ( + iprot.readString().decode("utf-8", errors="replace") if sys.version_info[0] == 2 else iprot.readString() + ) + else: + iprot.skip(ftype) + else: + iprot.skip(ftype) + iprot.readFieldEnd() + iprot.readStructEnd() + + def write(self, oprot): + if oprot._fast_encode is not None and self.thrift_spec is not None: + oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec])) + return + oprot.writeStructBegin("WMPool") + if self.resourcePlanName is not None: + oprot.writeFieldBegin("resourcePlanName", TType.STRING, 1) + oprot.writeString(self.resourcePlanName.encode("utf-8") if sys.version_info[0] == 2 else self.resourcePlanName) + oprot.writeFieldEnd() + if self.poolPath is not None: + oprot.writeFieldBegin("poolPath", TType.STRING, 2) + oprot.writeString(self.poolPath.encode("utf-8") if sys.version_info[0] == 2 else self.poolPath) + oprot.writeFieldEnd() + if self.allocFraction is not None: + oprot.writeFieldBegin("allocFraction", TType.DOUBLE, 3) + oprot.writeDouble(self.allocFraction) + oprot.writeFieldEnd() + if self.queryParallelism is not None: + oprot.writeFieldBegin("queryParallelism", TType.I32, 4) + oprot.writeI32(self.queryParallelism) + oprot.writeFieldEnd() + if self.schedulingPolicy is not None: + oprot.writeFieldBegin("schedulingPolicy", TType.STRING, 5) + oprot.writeString(self.schedulingPolicy.encode("utf-8") if sys.version_info[0] == 2 else self.schedulingPolicy) + oprot.writeFieldEnd() + if self.ns is not None: + oprot.writeFieldBegin("ns", TType.STRING, 6) + oprot.writeString(self.ns.encode("utf-8") if sys.version_info[0] == 2 else self.ns) + oprot.writeFieldEnd() + oprot.writeFieldStop() + oprot.writeStructEnd() + + def validate(self): + if self.resourcePlanName is None: + raise TProtocolException(message="Required field resourcePlanName is unset!") + if self.poolPath is None: + raise TProtocolException(message="Required field poolPath is unset!") + return + + def __repr__(self): + L = ["{}={!r}".format(key, value) for key, value in self.__dict__.items()] + return "{}({})".format(self.__class__.__name__, ", ".join(L)) + + def __eq__(self, other): + return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ + + def __ne__(self, other): + return not (self == other) + + +class WMNullablePool: + """ + Attributes: + - resourcePlanName + - poolPath + - allocFraction + - queryParallelism + - schedulingPolicy + - isSetSchedulingPolicy + - ns + + """ + + def __init__( + self, + resourcePlanName=None, + poolPath=None, + allocFraction=None, + queryParallelism=None, + schedulingPolicy=None, + isSetSchedulingPolicy=None, + ns=None, + ): + self.resourcePlanName = resourcePlanName + self.poolPath = poolPath + self.allocFraction = allocFraction + self.queryParallelism = queryParallelism + self.schedulingPolicy = schedulingPolicy + self.isSetSchedulingPolicy = isSetSchedulingPolicy + self.ns = ns + + def read(self, iprot): + if ( + iprot._fast_decode is not None + and isinstance(iprot.trans, TTransport.CReadableTransport) + and self.thrift_spec is not None + ): + iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec]) + return + iprot.readStructBegin() + while True: + (fname, ftype, fid) = iprot.readFieldBegin() + if ftype == TType.STOP: + break + if fid == 1: + if ftype == TType.STRING: + self.resourcePlanName = ( + iprot.readString().decode("utf-8", errors="replace") if sys.version_info[0] == 2 else iprot.readString() + ) + else: + iprot.skip(ftype) + elif fid == 2: + if ftype == TType.STRING: + self.poolPath = ( + iprot.readString().decode("utf-8", errors="replace") if sys.version_info[0] == 2 else iprot.readString() + ) + else: + iprot.skip(ftype) + elif fid == 3: + if ftype == TType.DOUBLE: + self.allocFraction = iprot.readDouble() + else: + iprot.skip(ftype) + elif fid == 4: + if ftype == TType.I32: + self.queryParallelism = iprot.readI32() + else: + iprot.skip(ftype) + elif fid == 5: + if ftype == TType.STRING: + self.schedulingPolicy = ( + iprot.readString().decode("utf-8", errors="replace") if sys.version_info[0] == 2 else iprot.readString() + ) + else: + iprot.skip(ftype) + elif fid == 6: + if ftype == TType.BOOL: + self.isSetSchedulingPolicy = iprot.readBool() + else: + iprot.skip(ftype) + elif fid == 7: + if ftype == TType.STRING: + self.ns = ( + iprot.readString().decode("utf-8", errors="replace") if sys.version_info[0] == 2 else iprot.readString() + ) + else: + iprot.skip(ftype) + else: + iprot.skip(ftype) + iprot.readFieldEnd() + iprot.readStructEnd() + + def write(self, oprot): + if oprot._fast_encode is not None and self.thrift_spec is not None: + oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec])) + return + oprot.writeStructBegin("WMNullablePool") + if self.resourcePlanName is not None: + oprot.writeFieldBegin("resourcePlanName", TType.STRING, 1) + oprot.writeString(self.resourcePlanName.encode("utf-8") if sys.version_info[0] == 2 else self.resourcePlanName) + oprot.writeFieldEnd() + if self.poolPath is not None: + oprot.writeFieldBegin("poolPath", TType.STRING, 2) + oprot.writeString(self.poolPath.encode("utf-8") if sys.version_info[0] == 2 else self.poolPath) + oprot.writeFieldEnd() + if self.allocFraction is not None: + oprot.writeFieldBegin("allocFraction", TType.DOUBLE, 3) + oprot.writeDouble(self.allocFraction) + oprot.writeFieldEnd() + if self.queryParallelism is not None: + oprot.writeFieldBegin("queryParallelism", TType.I32, 4) + oprot.writeI32(self.queryParallelism) + oprot.writeFieldEnd() + if self.schedulingPolicy is not None: + oprot.writeFieldBegin("schedulingPolicy", TType.STRING, 5) + oprot.writeString(self.schedulingPolicy.encode("utf-8") if sys.version_info[0] == 2 else self.schedulingPolicy) + oprot.writeFieldEnd() + if self.isSetSchedulingPolicy is not None: + oprot.writeFieldBegin("isSetSchedulingPolicy", TType.BOOL, 6) + oprot.writeBool(self.isSetSchedulingPolicy) + oprot.writeFieldEnd() + if self.ns is not None: + oprot.writeFieldBegin("ns", TType.STRING, 7) + oprot.writeString(self.ns.encode("utf-8") if sys.version_info[0] == 2 else self.ns) + oprot.writeFieldEnd() + oprot.writeFieldStop() + oprot.writeStructEnd() + + def validate(self): + if self.resourcePlanName is None: + raise TProtocolException(message="Required field resourcePlanName is unset!") + if self.poolPath is None: + raise TProtocolException(message="Required field poolPath is unset!") + return + + def __repr__(self): + L = ["{}={!r}".format(key, value) for key, value in self.__dict__.items()] + return "{}({})".format(self.__class__.__name__, ", ".join(L)) + + def __eq__(self, other): + return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ + + def __ne__(self, other): + return not (self == other) + + +class WMTrigger: + """ + Attributes: + - resourcePlanName + - triggerName + - triggerExpression + - actionExpression + - isInUnmanaged + - ns + + """ + + def __init__( + self, + resourcePlanName=None, + triggerName=None, + triggerExpression=None, + actionExpression=None, + isInUnmanaged=None, + ns=None, + ): + self.resourcePlanName = resourcePlanName + self.triggerName = triggerName + self.triggerExpression = triggerExpression + self.actionExpression = actionExpression + self.isInUnmanaged = isInUnmanaged + self.ns = ns + + def read(self, iprot): + if ( + iprot._fast_decode is not None + and isinstance(iprot.trans, TTransport.CReadableTransport) + and self.thrift_spec is not None + ): + iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec]) + return + iprot.readStructBegin() + while True: + (fname, ftype, fid) = iprot.readFieldBegin() + if ftype == TType.STOP: + break + if fid == 1: + if ftype == TType.STRING: + self.resourcePlanName = ( + iprot.readString().decode("utf-8", errors="replace") if sys.version_info[0] == 2 else iprot.readString() + ) + else: + iprot.skip(ftype) + elif fid == 2: + if ftype == TType.STRING: + self.triggerName = ( + iprot.readString().decode("utf-8", errors="replace") if sys.version_info[0] == 2 else iprot.readString() + ) + else: + iprot.skip(ftype) + elif fid == 3: + if ftype == TType.STRING: + self.triggerExpression = ( + iprot.readString().decode("utf-8", errors="replace") if sys.version_info[0] == 2 else iprot.readString() + ) + else: + iprot.skip(ftype) + elif fid == 4: + if ftype == TType.STRING: + self.actionExpression = ( + iprot.readString().decode("utf-8", errors="replace") if sys.version_info[0] == 2 else iprot.readString() + ) + else: + iprot.skip(ftype) + elif fid == 5: + if ftype == TType.BOOL: + self.isInUnmanaged = iprot.readBool() + else: + iprot.skip(ftype) + elif fid == 6: + if ftype == TType.STRING: + self.ns = ( + iprot.readString().decode("utf-8", errors="replace") if sys.version_info[0] == 2 else iprot.readString() + ) + else: + iprot.skip(ftype) + else: + iprot.skip(ftype) + iprot.readFieldEnd() + iprot.readStructEnd() + + def write(self, oprot): + if oprot._fast_encode is not None and self.thrift_spec is not None: + oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec])) + return + oprot.writeStructBegin("WMTrigger") + if self.resourcePlanName is not None: + oprot.writeFieldBegin("resourcePlanName", TType.STRING, 1) + oprot.writeString(self.resourcePlanName.encode("utf-8") if sys.version_info[0] == 2 else self.resourcePlanName) + oprot.writeFieldEnd() + if self.triggerName is not None: + oprot.writeFieldBegin("triggerName", TType.STRING, 2) + oprot.writeString(self.triggerName.encode("utf-8") if sys.version_info[0] == 2 else self.triggerName) + oprot.writeFieldEnd() + if self.triggerExpression is not None: + oprot.writeFieldBegin("triggerExpression", TType.STRING, 3) + oprot.writeString(self.triggerExpression.encode("utf-8") if sys.version_info[0] == 2 else self.triggerExpression) + oprot.writeFieldEnd() + if self.actionExpression is not None: + oprot.writeFieldBegin("actionExpression", TType.STRING, 4) + oprot.writeString(self.actionExpression.encode("utf-8") if sys.version_info[0] == 2 else self.actionExpression) + oprot.writeFieldEnd() + if self.isInUnmanaged is not None: + oprot.writeFieldBegin("isInUnmanaged", TType.BOOL, 5) + oprot.writeBool(self.isInUnmanaged) + oprot.writeFieldEnd() + if self.ns is not None: + oprot.writeFieldBegin("ns", TType.STRING, 6) + oprot.writeString(self.ns.encode("utf-8") if sys.version_info[0] == 2 else self.ns) + oprot.writeFieldEnd() + oprot.writeFieldStop() + oprot.writeStructEnd() + + def validate(self): + if self.resourcePlanName is None: + raise TProtocolException(message="Required field resourcePlanName is unset!") + if self.triggerName is None: + raise TProtocolException(message="Required field triggerName is unset!") + return + + def __repr__(self): + L = ["{}={!r}".format(key, value) for key, value in self.__dict__.items()] + return "{}({})".format(self.__class__.__name__, ", ".join(L)) + + def __eq__(self, other): + return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ + + def __ne__(self, other): + return not (self == other) + + +class WMMapping: + """ + Attributes: + - resourcePlanName + - entityType + - entityName + - poolPath + - ordering + - ns + + """ + + def __init__( + self, + resourcePlanName=None, + entityType=None, + entityName=None, + poolPath=None, + ordering=None, + ns=None, + ): + self.resourcePlanName = resourcePlanName + self.entityType = entityType + self.entityName = entityName + self.poolPath = poolPath + self.ordering = ordering + self.ns = ns + + def read(self, iprot): + if ( + iprot._fast_decode is not None + and isinstance(iprot.trans, TTransport.CReadableTransport) + and self.thrift_spec is not None + ): + iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec]) + return + iprot.readStructBegin() + while True: + (fname, ftype, fid) = iprot.readFieldBegin() + if ftype == TType.STOP: + break + if fid == 1: + if ftype == TType.STRING: + self.resourcePlanName = ( + iprot.readString().decode("utf-8", errors="replace") if sys.version_info[0] == 2 else iprot.readString() + ) + else: + iprot.skip(ftype) + elif fid == 2: + if ftype == TType.STRING: + self.entityType = ( + iprot.readString().decode("utf-8", errors="replace") if sys.version_info[0] == 2 else iprot.readString() + ) + else: + iprot.skip(ftype) + elif fid == 3: + if ftype == TType.STRING: + self.entityName = ( + iprot.readString().decode("utf-8", errors="replace") if sys.version_info[0] == 2 else iprot.readString() + ) + else: + iprot.skip(ftype) + elif fid == 4: + if ftype == TType.STRING: + self.poolPath = ( + iprot.readString().decode("utf-8", errors="replace") if sys.version_info[0] == 2 else iprot.readString() + ) + else: + iprot.skip(ftype) + elif fid == 5: + if ftype == TType.I32: + self.ordering = iprot.readI32() + else: + iprot.skip(ftype) + elif fid == 6: + if ftype == TType.STRING: + self.ns = ( + iprot.readString().decode("utf-8", errors="replace") if sys.version_info[0] == 2 else iprot.readString() + ) + else: + iprot.skip(ftype) + else: + iprot.skip(ftype) + iprot.readFieldEnd() + iprot.readStructEnd() + + def write(self, oprot): + if oprot._fast_encode is not None and self.thrift_spec is not None: + oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec])) + return + oprot.writeStructBegin("WMMapping") + if self.resourcePlanName is not None: + oprot.writeFieldBegin("resourcePlanName", TType.STRING, 1) + oprot.writeString(self.resourcePlanName.encode("utf-8") if sys.version_info[0] == 2 else self.resourcePlanName) + oprot.writeFieldEnd() + if self.entityType is not None: + oprot.writeFieldBegin("entityType", TType.STRING, 2) + oprot.writeString(self.entityType.encode("utf-8") if sys.version_info[0] == 2 else self.entityType) + oprot.writeFieldEnd() + if self.entityName is not None: + oprot.writeFieldBegin("entityName", TType.STRING, 3) + oprot.writeString(self.entityName.encode("utf-8") if sys.version_info[0] == 2 else self.entityName) + oprot.writeFieldEnd() + if self.poolPath is not None: + oprot.writeFieldBegin("poolPath", TType.STRING, 4) + oprot.writeString(self.poolPath.encode("utf-8") if sys.version_info[0] == 2 else self.poolPath) + oprot.writeFieldEnd() + if self.ordering is not None: + oprot.writeFieldBegin("ordering", TType.I32, 5) + oprot.writeI32(self.ordering) + oprot.writeFieldEnd() + if self.ns is not None: + oprot.writeFieldBegin("ns", TType.STRING, 6) + oprot.writeString(self.ns.encode("utf-8") if sys.version_info[0] == 2 else self.ns) + oprot.writeFieldEnd() + oprot.writeFieldStop() + oprot.writeStructEnd() + + def validate(self): + if self.resourcePlanName is None: + raise TProtocolException(message="Required field resourcePlanName is unset!") + if self.entityType is None: + raise TProtocolException(message="Required field entityType is unset!") + if self.entityName is None: + raise TProtocolException(message="Required field entityName is unset!") + return + + def __repr__(self): + L = ["{}={!r}".format(key, value) for key, value in self.__dict__.items()] + return "{}({})".format(self.__class__.__name__, ", ".join(L)) + + def __eq__(self, other): + return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ + + def __ne__(self, other): + return not (self == other) + + +class WMPoolTrigger: + """ + Attributes: + - pool + - trigger + - ns + + """ + + def __init__( + self, + pool=None, + trigger=None, + ns=None, + ): + self.pool = pool + self.trigger = trigger + self.ns = ns + + def read(self, iprot): + if ( + iprot._fast_decode is not None + and isinstance(iprot.trans, TTransport.CReadableTransport) + and self.thrift_spec is not None + ): + iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec]) + return + iprot.readStructBegin() + while True: + (fname, ftype, fid) = iprot.readFieldBegin() + if ftype == TType.STOP: + break + if fid == 1: + if ftype == TType.STRING: + self.pool = ( + iprot.readString().decode("utf-8", errors="replace") if sys.version_info[0] == 2 else iprot.readString() + ) + else: + iprot.skip(ftype) + elif fid == 2: + if ftype == TType.STRING: + self.trigger = ( + iprot.readString().decode("utf-8", errors="replace") if sys.version_info[0] == 2 else iprot.readString() + ) + else: + iprot.skip(ftype) + elif fid == 3: + if ftype == TType.STRING: + self.ns = ( + iprot.readString().decode("utf-8", errors="replace") if sys.version_info[0] == 2 else iprot.readString() + ) + else: + iprot.skip(ftype) + else: + iprot.skip(ftype) + iprot.readFieldEnd() + iprot.readStructEnd() + + def write(self, oprot): + if oprot._fast_encode is not None and self.thrift_spec is not None: + oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec])) + return + oprot.writeStructBegin("WMPoolTrigger") + if self.pool is not None: + oprot.writeFieldBegin("pool", TType.STRING, 1) + oprot.writeString(self.pool.encode("utf-8") if sys.version_info[0] == 2 else self.pool) + oprot.writeFieldEnd() + if self.trigger is not None: + oprot.writeFieldBegin("trigger", TType.STRING, 2) + oprot.writeString(self.trigger.encode("utf-8") if sys.version_info[0] == 2 else self.trigger) + oprot.writeFieldEnd() + if self.ns is not None: + oprot.writeFieldBegin("ns", TType.STRING, 3) + oprot.writeString(self.ns.encode("utf-8") if sys.version_info[0] == 2 else self.ns) + oprot.writeFieldEnd() + oprot.writeFieldStop() + oprot.writeStructEnd() + + def validate(self): + if self.pool is None: + raise TProtocolException(message="Required field pool is unset!") + if self.trigger is None: + raise TProtocolException(message="Required field trigger is unset!") + return + + def __repr__(self): + L = ["{}={!r}".format(key, value) for key, value in self.__dict__.items()] + return "{}({})".format(self.__class__.__name__, ", ".join(L)) + + def __eq__(self, other): + return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ + + def __ne__(self, other): + return not (self == other) + + +class WMFullResourcePlan: + """ + Attributes: + - plan + - pools + - mappings + - triggers + - poolTriggers + + """ + + def __init__( + self, + plan=None, + pools=None, + mappings=None, + triggers=None, + poolTriggers=None, + ): + self.plan = plan + self.pools = pools + self.mappings = mappings + self.triggers = triggers + self.poolTriggers = poolTriggers + + def read(self, iprot): + if ( + iprot._fast_decode is not None + and isinstance(iprot.trans, TTransport.CReadableTransport) + and self.thrift_spec is not None + ): + iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec]) + return + iprot.readStructBegin() + while True: + (fname, ftype, fid) = iprot.readFieldBegin() + if ftype == TType.STOP: + break + if fid == 1: + if ftype == TType.STRUCT: + self.plan = WMResourcePlan() + self.plan.read(iprot) + else: + iprot.skip(ftype) + elif fid == 2: + if ftype == TType.LIST: + self.pools = [] + (_etype998, _size995) = iprot.readListBegin() + for _i999 in range(_size995): + _elem1000 = WMPool() + _elem1000.read(iprot) + self.pools.append(_elem1000) + iprot.readListEnd() + else: + iprot.skip(ftype) + elif fid == 3: + if ftype == TType.LIST: + self.mappings = [] + (_etype1004, _size1001) = iprot.readListBegin() + for _i1005 in range(_size1001): + _elem1006 = WMMapping() + _elem1006.read(iprot) + self.mappings.append(_elem1006) + iprot.readListEnd() + else: + iprot.skip(ftype) + elif fid == 4: + if ftype == TType.LIST: + self.triggers = [] + (_etype1010, _size1007) = iprot.readListBegin() + for _i1011 in range(_size1007): + _elem1012 = WMTrigger() + _elem1012.read(iprot) + self.triggers.append(_elem1012) + iprot.readListEnd() + else: + iprot.skip(ftype) + elif fid == 5: + if ftype == TType.LIST: + self.poolTriggers = [] + (_etype1016, _size1013) = iprot.readListBegin() + for _i1017 in range(_size1013): + _elem1018 = WMPoolTrigger() + _elem1018.read(iprot) + self.poolTriggers.append(_elem1018) + iprot.readListEnd() + else: + iprot.skip(ftype) + else: + iprot.skip(ftype) + iprot.readFieldEnd() + iprot.readStructEnd() + + def write(self, oprot): + if oprot._fast_encode is not None and self.thrift_spec is not None: + oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec])) + return + oprot.writeStructBegin("WMFullResourcePlan") + if self.plan is not None: + oprot.writeFieldBegin("plan", TType.STRUCT, 1) + self.plan.write(oprot) + oprot.writeFieldEnd() + if self.pools is not None: + oprot.writeFieldBegin("pools", TType.LIST, 2) + oprot.writeListBegin(TType.STRUCT, len(self.pools)) + for iter1019 in self.pools: + iter1019.write(oprot) + oprot.writeListEnd() + oprot.writeFieldEnd() + if self.mappings is not None: + oprot.writeFieldBegin("mappings", TType.LIST, 3) + oprot.writeListBegin(TType.STRUCT, len(self.mappings)) + for iter1020 in self.mappings: + iter1020.write(oprot) + oprot.writeListEnd() + oprot.writeFieldEnd() + if self.triggers is not None: + oprot.writeFieldBegin("triggers", TType.LIST, 4) + oprot.writeListBegin(TType.STRUCT, len(self.triggers)) + for iter1021 in self.triggers: + iter1021.write(oprot) + oprot.writeListEnd() + oprot.writeFieldEnd() + if self.poolTriggers is not None: + oprot.writeFieldBegin("poolTriggers", TType.LIST, 5) + oprot.writeListBegin(TType.STRUCT, len(self.poolTriggers)) + for iter1022 in self.poolTriggers: + iter1022.write(oprot) + oprot.writeListEnd() + oprot.writeFieldEnd() + oprot.writeFieldStop() + oprot.writeStructEnd() + + def validate(self): + if self.plan is None: + raise TProtocolException(message="Required field plan is unset!") + if self.pools is None: + raise TProtocolException(message="Required field pools is unset!") + return + + def __repr__(self): + L = ["{}={!r}".format(key, value) for key, value in self.__dict__.items()] + return "{}({})".format(self.__class__.__name__, ", ".join(L)) + + def __eq__(self, other): + return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ + + def __ne__(self, other): + return not (self == other) + + +class WMCreateResourcePlanRequest: + """ + Attributes: + - resourcePlan + - copyFrom + + """ + + def __init__( + self, + resourcePlan=None, + copyFrom=None, + ): + self.resourcePlan = resourcePlan + self.copyFrom = copyFrom + + def read(self, iprot): + if ( + iprot._fast_decode is not None + and isinstance(iprot.trans, TTransport.CReadableTransport) + and self.thrift_spec is not None + ): + iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec]) + return + iprot.readStructBegin() + while True: + (fname, ftype, fid) = iprot.readFieldBegin() + if ftype == TType.STOP: + break + if fid == 1: + if ftype == TType.STRUCT: + self.resourcePlan = WMResourcePlan() + self.resourcePlan.read(iprot) + else: + iprot.skip(ftype) + elif fid == 2: + if ftype == TType.STRING: + self.copyFrom = ( + iprot.readString().decode("utf-8", errors="replace") if sys.version_info[0] == 2 else iprot.readString() + ) + else: + iprot.skip(ftype) + else: + iprot.skip(ftype) + iprot.readFieldEnd() + iprot.readStructEnd() + + def write(self, oprot): + if oprot._fast_encode is not None and self.thrift_spec is not None: + oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec])) + return + oprot.writeStructBegin("WMCreateResourcePlanRequest") + if self.resourcePlan is not None: + oprot.writeFieldBegin("resourcePlan", TType.STRUCT, 1) + self.resourcePlan.write(oprot) + oprot.writeFieldEnd() + if self.copyFrom is not None: + oprot.writeFieldBegin("copyFrom", TType.STRING, 2) + oprot.writeString(self.copyFrom.encode("utf-8") if sys.version_info[0] == 2 else self.copyFrom) + oprot.writeFieldEnd() + oprot.writeFieldStop() + oprot.writeStructEnd() + + def validate(self): + return + + def __repr__(self): + L = ["{}={!r}".format(key, value) for key, value in self.__dict__.items()] + return "{}({})".format(self.__class__.__name__, ", ".join(L)) + + def __eq__(self, other): + return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ + + def __ne__(self, other): + return not (self == other) + + +class WMCreateResourcePlanResponse: + def read(self, iprot): + if ( + iprot._fast_decode is not None + and isinstance(iprot.trans, TTransport.CReadableTransport) + and self.thrift_spec is not None + ): + iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec]) + return + iprot.readStructBegin() + while True: + (fname, ftype, fid) = iprot.readFieldBegin() + if ftype == TType.STOP: + break + else: + iprot.skip(ftype) + iprot.readFieldEnd() + iprot.readStructEnd() + + def write(self, oprot): + if oprot._fast_encode is not None and self.thrift_spec is not None: + oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec])) + return + oprot.writeStructBegin("WMCreateResourcePlanResponse") + oprot.writeFieldStop() + oprot.writeStructEnd() + + def validate(self): + return + + def __repr__(self): + L = ["{}={!r}".format(key, value) for key, value in self.__dict__.items()] + return "{}({})".format(self.__class__.__name__, ", ".join(L)) + + def __eq__(self, other): + return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ + + def __ne__(self, other): + return not (self == other) + + +class WMGetActiveResourcePlanRequest: + """ + Attributes: + - ns + + """ + + def __init__( + self, + ns=None, + ): + self.ns = ns + + def read(self, iprot): + if ( + iprot._fast_decode is not None + and isinstance(iprot.trans, TTransport.CReadableTransport) + and self.thrift_spec is not None + ): + iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec]) + return + iprot.readStructBegin() + while True: + (fname, ftype, fid) = iprot.readFieldBegin() + if ftype == TType.STOP: + break + if fid == 1: + if ftype == TType.STRING: + self.ns = ( + iprot.readString().decode("utf-8", errors="replace") if sys.version_info[0] == 2 else iprot.readString() + ) + else: + iprot.skip(ftype) + else: + iprot.skip(ftype) + iprot.readFieldEnd() + iprot.readStructEnd() + + def write(self, oprot): + if oprot._fast_encode is not None and self.thrift_spec is not None: + oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec])) + return + oprot.writeStructBegin("WMGetActiveResourcePlanRequest") + if self.ns is not None: + oprot.writeFieldBegin("ns", TType.STRING, 1) + oprot.writeString(self.ns.encode("utf-8") if sys.version_info[0] == 2 else self.ns) + oprot.writeFieldEnd() + oprot.writeFieldStop() + oprot.writeStructEnd() + + def validate(self): + return + + def __repr__(self): + L = ["{}={!r}".format(key, value) for key, value in self.__dict__.items()] + return "{}({})".format(self.__class__.__name__, ", ".join(L)) + + def __eq__(self, other): + return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ + + def __ne__(self, other): + return not (self == other) + + +class WMGetActiveResourcePlanResponse: + """ + Attributes: + - resourcePlan + + """ + + def __init__( + self, + resourcePlan=None, + ): + self.resourcePlan = resourcePlan + + def read(self, iprot): + if ( + iprot._fast_decode is not None + and isinstance(iprot.trans, TTransport.CReadableTransport) + and self.thrift_spec is not None + ): + iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec]) + return + iprot.readStructBegin() + while True: + (fname, ftype, fid) = iprot.readFieldBegin() + if ftype == TType.STOP: + break + if fid == 1: + if ftype == TType.STRUCT: + self.resourcePlan = WMFullResourcePlan() + self.resourcePlan.read(iprot) + else: + iprot.skip(ftype) + else: + iprot.skip(ftype) + iprot.readFieldEnd() + iprot.readStructEnd() + + def write(self, oprot): + if oprot._fast_encode is not None and self.thrift_spec is not None: + oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec])) + return + oprot.writeStructBegin("WMGetActiveResourcePlanResponse") + if self.resourcePlan is not None: + oprot.writeFieldBegin("resourcePlan", TType.STRUCT, 1) + self.resourcePlan.write(oprot) + oprot.writeFieldEnd() + oprot.writeFieldStop() + oprot.writeStructEnd() + + def validate(self): + return + + def __repr__(self): + L = ["{}={!r}".format(key, value) for key, value in self.__dict__.items()] + return "{}({})".format(self.__class__.__name__, ", ".join(L)) + + def __eq__(self, other): + return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ + + def __ne__(self, other): + return not (self == other) + + +class WMGetResourcePlanRequest: + """ + Attributes: + - resourcePlanName + - ns + + """ + + def __init__( + self, + resourcePlanName=None, + ns=None, + ): + self.resourcePlanName = resourcePlanName + self.ns = ns + + def read(self, iprot): + if ( + iprot._fast_decode is not None + and isinstance(iprot.trans, TTransport.CReadableTransport) + and self.thrift_spec is not None + ): + iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec]) + return + iprot.readStructBegin() + while True: + (fname, ftype, fid) = iprot.readFieldBegin() + if ftype == TType.STOP: + break + if fid == 1: + if ftype == TType.STRING: + self.resourcePlanName = ( + iprot.readString().decode("utf-8", errors="replace") if sys.version_info[0] == 2 else iprot.readString() + ) + else: + iprot.skip(ftype) + elif fid == 2: + if ftype == TType.STRING: + self.ns = ( + iprot.readString().decode("utf-8", errors="replace") if sys.version_info[0] == 2 else iprot.readString() + ) + else: + iprot.skip(ftype) + else: + iprot.skip(ftype) + iprot.readFieldEnd() + iprot.readStructEnd() + + def write(self, oprot): + if oprot._fast_encode is not None and self.thrift_spec is not None: + oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec])) + return + oprot.writeStructBegin("WMGetResourcePlanRequest") + if self.resourcePlanName is not None: + oprot.writeFieldBegin("resourcePlanName", TType.STRING, 1) + oprot.writeString(self.resourcePlanName.encode("utf-8") if sys.version_info[0] == 2 else self.resourcePlanName) + oprot.writeFieldEnd() + if self.ns is not None: + oprot.writeFieldBegin("ns", TType.STRING, 2) + oprot.writeString(self.ns.encode("utf-8") if sys.version_info[0] == 2 else self.ns) + oprot.writeFieldEnd() + oprot.writeFieldStop() + oprot.writeStructEnd() + + def validate(self): + return + + def __repr__(self): + L = ["{}={!r}".format(key, value) for key, value in self.__dict__.items()] + return "{}({})".format(self.__class__.__name__, ", ".join(L)) + + def __eq__(self, other): + return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ + + def __ne__(self, other): + return not (self == other) + + +class WMGetResourcePlanResponse: + """ + Attributes: + - resourcePlan + + """ + + def __init__( + self, + resourcePlan=None, + ): + self.resourcePlan = resourcePlan + + def read(self, iprot): + if ( + iprot._fast_decode is not None + and isinstance(iprot.trans, TTransport.CReadableTransport) + and self.thrift_spec is not None + ): + iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec]) + return + iprot.readStructBegin() + while True: + (fname, ftype, fid) = iprot.readFieldBegin() + if ftype == TType.STOP: + break + if fid == 1: + if ftype == TType.STRUCT: + self.resourcePlan = WMFullResourcePlan() + self.resourcePlan.read(iprot) + else: + iprot.skip(ftype) + else: + iprot.skip(ftype) + iprot.readFieldEnd() + iprot.readStructEnd() + + def write(self, oprot): + if oprot._fast_encode is not None and self.thrift_spec is not None: + oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec])) + return + oprot.writeStructBegin("WMGetResourcePlanResponse") + if self.resourcePlan is not None: + oprot.writeFieldBegin("resourcePlan", TType.STRUCT, 1) + self.resourcePlan.write(oprot) + oprot.writeFieldEnd() + oprot.writeFieldStop() + oprot.writeStructEnd() + + def validate(self): + return + + def __repr__(self): + L = ["{}={!r}".format(key, value) for key, value in self.__dict__.items()] + return "{}({})".format(self.__class__.__name__, ", ".join(L)) + + def __eq__(self, other): + return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ + + def __ne__(self, other): + return not (self == other) + + +class WMGetAllResourcePlanRequest: + """ + Attributes: + - ns + + """ + + def __init__( + self, + ns=None, + ): + self.ns = ns + + def read(self, iprot): + if ( + iprot._fast_decode is not None + and isinstance(iprot.trans, TTransport.CReadableTransport) + and self.thrift_spec is not None + ): + iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec]) + return + iprot.readStructBegin() + while True: + (fname, ftype, fid) = iprot.readFieldBegin() + if ftype == TType.STOP: + break + if fid == 1: + if ftype == TType.STRING: + self.ns = ( + iprot.readString().decode("utf-8", errors="replace") if sys.version_info[0] == 2 else iprot.readString() + ) + else: + iprot.skip(ftype) + else: + iprot.skip(ftype) + iprot.readFieldEnd() + iprot.readStructEnd() + + def write(self, oprot): + if oprot._fast_encode is not None and self.thrift_spec is not None: + oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec])) + return + oprot.writeStructBegin("WMGetAllResourcePlanRequest") + if self.ns is not None: + oprot.writeFieldBegin("ns", TType.STRING, 1) + oprot.writeString(self.ns.encode("utf-8") if sys.version_info[0] == 2 else self.ns) + oprot.writeFieldEnd() + oprot.writeFieldStop() + oprot.writeStructEnd() + + def validate(self): + return + + def __repr__(self): + L = ["{}={!r}".format(key, value) for key, value in self.__dict__.items()] + return "{}({})".format(self.__class__.__name__, ", ".join(L)) + + def __eq__(self, other): + return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ + + def __ne__(self, other): + return not (self == other) + + +class WMGetAllResourcePlanResponse: + """ + Attributes: + - resourcePlans + + """ + + def __init__( + self, + resourcePlans=None, + ): + self.resourcePlans = resourcePlans + + def read(self, iprot): + if ( + iprot._fast_decode is not None + and isinstance(iprot.trans, TTransport.CReadableTransport) + and self.thrift_spec is not None + ): + iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec]) + return + iprot.readStructBegin() + while True: + (fname, ftype, fid) = iprot.readFieldBegin() + if ftype == TType.STOP: + break + if fid == 1: + if ftype == TType.LIST: + self.resourcePlans = [] + (_etype1026, _size1023) = iprot.readListBegin() + for _i1027 in range(_size1023): + _elem1028 = WMResourcePlan() + _elem1028.read(iprot) + self.resourcePlans.append(_elem1028) + iprot.readListEnd() + else: + iprot.skip(ftype) + else: + iprot.skip(ftype) + iprot.readFieldEnd() + iprot.readStructEnd() + + def write(self, oprot): + if oprot._fast_encode is not None and self.thrift_spec is not None: + oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec])) + return + oprot.writeStructBegin("WMGetAllResourcePlanResponse") + if self.resourcePlans is not None: + oprot.writeFieldBegin("resourcePlans", TType.LIST, 1) + oprot.writeListBegin(TType.STRUCT, len(self.resourcePlans)) + for iter1029 in self.resourcePlans: + iter1029.write(oprot) + oprot.writeListEnd() + oprot.writeFieldEnd() + oprot.writeFieldStop() + oprot.writeStructEnd() + + def validate(self): + return + + def __repr__(self): + L = ["{}={!r}".format(key, value) for key, value in self.__dict__.items()] + return "{}({})".format(self.__class__.__name__, ", ".join(L)) + + def __eq__(self, other): + return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ + + def __ne__(self, other): + return not (self == other) + + +class WMAlterResourcePlanRequest: + """ + Attributes: + - resourcePlanName + - resourcePlan + - isEnableAndActivate + - isForceDeactivate + - isReplace + - ns + + """ + + def __init__( + self, + resourcePlanName=None, + resourcePlan=None, + isEnableAndActivate=None, + isForceDeactivate=None, + isReplace=None, + ns=None, + ): + self.resourcePlanName = resourcePlanName + self.resourcePlan = resourcePlan + self.isEnableAndActivate = isEnableAndActivate + self.isForceDeactivate = isForceDeactivate + self.isReplace = isReplace + self.ns = ns + + def read(self, iprot): + if ( + iprot._fast_decode is not None + and isinstance(iprot.trans, TTransport.CReadableTransport) + and self.thrift_spec is not None + ): + iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec]) + return + iprot.readStructBegin() + while True: + (fname, ftype, fid) = iprot.readFieldBegin() + if ftype == TType.STOP: + break + if fid == 1: + if ftype == TType.STRING: + self.resourcePlanName = ( + iprot.readString().decode("utf-8", errors="replace") if sys.version_info[0] == 2 else iprot.readString() + ) + else: + iprot.skip(ftype) + elif fid == 2: + if ftype == TType.STRUCT: + self.resourcePlan = WMNullableResourcePlan() + self.resourcePlan.read(iprot) + else: + iprot.skip(ftype) + elif fid == 3: + if ftype == TType.BOOL: + self.isEnableAndActivate = iprot.readBool() + else: + iprot.skip(ftype) + elif fid == 4: + if ftype == TType.BOOL: + self.isForceDeactivate = iprot.readBool() + else: + iprot.skip(ftype) + elif fid == 5: + if ftype == TType.BOOL: + self.isReplace = iprot.readBool() + else: + iprot.skip(ftype) + elif fid == 6: + if ftype == TType.STRING: + self.ns = ( + iprot.readString().decode("utf-8", errors="replace") if sys.version_info[0] == 2 else iprot.readString() + ) + else: + iprot.skip(ftype) + else: + iprot.skip(ftype) + iprot.readFieldEnd() + iprot.readStructEnd() + + def write(self, oprot): + if oprot._fast_encode is not None and self.thrift_spec is not None: + oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec])) + return + oprot.writeStructBegin("WMAlterResourcePlanRequest") + if self.resourcePlanName is not None: + oprot.writeFieldBegin("resourcePlanName", TType.STRING, 1) + oprot.writeString(self.resourcePlanName.encode("utf-8") if sys.version_info[0] == 2 else self.resourcePlanName) + oprot.writeFieldEnd() + if self.resourcePlan is not None: + oprot.writeFieldBegin("resourcePlan", TType.STRUCT, 2) + self.resourcePlan.write(oprot) + oprot.writeFieldEnd() + if self.isEnableAndActivate is not None: + oprot.writeFieldBegin("isEnableAndActivate", TType.BOOL, 3) + oprot.writeBool(self.isEnableAndActivate) + oprot.writeFieldEnd() + if self.isForceDeactivate is not None: + oprot.writeFieldBegin("isForceDeactivate", TType.BOOL, 4) + oprot.writeBool(self.isForceDeactivate) + oprot.writeFieldEnd() + if self.isReplace is not None: + oprot.writeFieldBegin("isReplace", TType.BOOL, 5) + oprot.writeBool(self.isReplace) + oprot.writeFieldEnd() + if self.ns is not None: + oprot.writeFieldBegin("ns", TType.STRING, 6) + oprot.writeString(self.ns.encode("utf-8") if sys.version_info[0] == 2 else self.ns) + oprot.writeFieldEnd() + oprot.writeFieldStop() + oprot.writeStructEnd() + + def validate(self): + return + + def __repr__(self): + L = ["{}={!r}".format(key, value) for key, value in self.__dict__.items()] + return "{}({})".format(self.__class__.__name__, ", ".join(L)) + + def __eq__(self, other): + return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ + + def __ne__(self, other): + return not (self == other) + + +class WMAlterResourcePlanResponse: + """ + Attributes: + - fullResourcePlan + + """ + + def __init__( + self, + fullResourcePlan=None, + ): + self.fullResourcePlan = fullResourcePlan + + def read(self, iprot): + if ( + iprot._fast_decode is not None + and isinstance(iprot.trans, TTransport.CReadableTransport) + and self.thrift_spec is not None + ): + iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec]) + return + iprot.readStructBegin() + while True: + (fname, ftype, fid) = iprot.readFieldBegin() + if ftype == TType.STOP: + break + if fid == 1: + if ftype == TType.STRUCT: + self.fullResourcePlan = WMFullResourcePlan() + self.fullResourcePlan.read(iprot) + else: + iprot.skip(ftype) + else: + iprot.skip(ftype) + iprot.readFieldEnd() + iprot.readStructEnd() + + def write(self, oprot): + if oprot._fast_encode is not None and self.thrift_spec is not None: + oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec])) + return + oprot.writeStructBegin("WMAlterResourcePlanResponse") + if self.fullResourcePlan is not None: + oprot.writeFieldBegin("fullResourcePlan", TType.STRUCT, 1) + self.fullResourcePlan.write(oprot) + oprot.writeFieldEnd() + oprot.writeFieldStop() + oprot.writeStructEnd() + + def validate(self): + return + + def __repr__(self): + L = ["{}={!r}".format(key, value) for key, value in self.__dict__.items()] + return "{}({})".format(self.__class__.__name__, ", ".join(L)) + + def __eq__(self, other): + return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ + + def __ne__(self, other): + return not (self == other) + + +class WMValidateResourcePlanRequest: + """ + Attributes: + - resourcePlanName + - ns + + """ + + def __init__( + self, + resourcePlanName=None, + ns=None, + ): + self.resourcePlanName = resourcePlanName + self.ns = ns + + def read(self, iprot): + if ( + iprot._fast_decode is not None + and isinstance(iprot.trans, TTransport.CReadableTransport) + and self.thrift_spec is not None + ): + iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec]) + return + iprot.readStructBegin() + while True: + (fname, ftype, fid) = iprot.readFieldBegin() + if ftype == TType.STOP: + break + if fid == 1: + if ftype == TType.STRING: + self.resourcePlanName = ( + iprot.readString().decode("utf-8", errors="replace") if sys.version_info[0] == 2 else iprot.readString() + ) + else: + iprot.skip(ftype) + elif fid == 2: + if ftype == TType.STRING: + self.ns = ( + iprot.readString().decode("utf-8", errors="replace") if sys.version_info[0] == 2 else iprot.readString() + ) + else: + iprot.skip(ftype) + else: + iprot.skip(ftype) + iprot.readFieldEnd() + iprot.readStructEnd() + + def write(self, oprot): + if oprot._fast_encode is not None and self.thrift_spec is not None: + oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec])) + return + oprot.writeStructBegin("WMValidateResourcePlanRequest") + if self.resourcePlanName is not None: + oprot.writeFieldBegin("resourcePlanName", TType.STRING, 1) + oprot.writeString(self.resourcePlanName.encode("utf-8") if sys.version_info[0] == 2 else self.resourcePlanName) + oprot.writeFieldEnd() + if self.ns is not None: + oprot.writeFieldBegin("ns", TType.STRING, 2) + oprot.writeString(self.ns.encode("utf-8") if sys.version_info[0] == 2 else self.ns) + oprot.writeFieldEnd() + oprot.writeFieldStop() + oprot.writeStructEnd() + + def validate(self): + return + + def __repr__(self): + L = ["{}={!r}".format(key, value) for key, value in self.__dict__.items()] + return "{}({})".format(self.__class__.__name__, ", ".join(L)) + + def __eq__(self, other): + return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ + + def __ne__(self, other): + return not (self == other) + + +class WMValidateResourcePlanResponse: + """ + Attributes: + - errors + - warnings + + """ + + def __init__( + self, + errors=None, + warnings=None, + ): + self.errors = errors + self.warnings = warnings + + def read(self, iprot): + if ( + iprot._fast_decode is not None + and isinstance(iprot.trans, TTransport.CReadableTransport) + and self.thrift_spec is not None + ): + iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec]) + return + iprot.readStructBegin() + while True: + (fname, ftype, fid) = iprot.readFieldBegin() + if ftype == TType.STOP: + break + if fid == 1: + if ftype == TType.LIST: + self.errors = [] + (_etype1033, _size1030) = iprot.readListBegin() + for _i1034 in range(_size1030): + _elem1035 = ( + iprot.readString().decode("utf-8", errors="replace") + if sys.version_info[0] == 2 + else iprot.readString() + ) + self.errors.append(_elem1035) + iprot.readListEnd() + else: + iprot.skip(ftype) + elif fid == 2: + if ftype == TType.LIST: + self.warnings = [] + (_etype1039, _size1036) = iprot.readListBegin() + for _i1040 in range(_size1036): + _elem1041 = ( + iprot.readString().decode("utf-8", errors="replace") + if sys.version_info[0] == 2 + else iprot.readString() + ) + self.warnings.append(_elem1041) + iprot.readListEnd() + else: + iprot.skip(ftype) + else: + iprot.skip(ftype) + iprot.readFieldEnd() + iprot.readStructEnd() + + def write(self, oprot): + if oprot._fast_encode is not None and self.thrift_spec is not None: + oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec])) + return + oprot.writeStructBegin("WMValidateResourcePlanResponse") + if self.errors is not None: + oprot.writeFieldBegin("errors", TType.LIST, 1) + oprot.writeListBegin(TType.STRING, len(self.errors)) + for iter1042 in self.errors: + oprot.writeString(iter1042.encode("utf-8") if sys.version_info[0] == 2 else iter1042) + oprot.writeListEnd() + oprot.writeFieldEnd() + if self.warnings is not None: + oprot.writeFieldBegin("warnings", TType.LIST, 2) + oprot.writeListBegin(TType.STRING, len(self.warnings)) + for iter1043 in self.warnings: + oprot.writeString(iter1043.encode("utf-8") if sys.version_info[0] == 2 else iter1043) + oprot.writeListEnd() + oprot.writeFieldEnd() + oprot.writeFieldStop() + oprot.writeStructEnd() + + def validate(self): + return + + def __repr__(self): + L = ["{}={!r}".format(key, value) for key, value in self.__dict__.items()] + return "{}({})".format(self.__class__.__name__, ", ".join(L)) + + def __eq__(self, other): + return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ + + def __ne__(self, other): + return not (self == other) + + +class WMDropResourcePlanRequest: + """ + Attributes: + - resourcePlanName + - ns + + """ + + def __init__( + self, + resourcePlanName=None, + ns=None, + ): + self.resourcePlanName = resourcePlanName + self.ns = ns + + def read(self, iprot): + if ( + iprot._fast_decode is not None + and isinstance(iprot.trans, TTransport.CReadableTransport) + and self.thrift_spec is not None + ): + iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec]) + return + iprot.readStructBegin() + while True: + (fname, ftype, fid) = iprot.readFieldBegin() + if ftype == TType.STOP: + break + if fid == 1: + if ftype == TType.STRING: + self.resourcePlanName = ( + iprot.readString().decode("utf-8", errors="replace") if sys.version_info[0] == 2 else iprot.readString() + ) + else: + iprot.skip(ftype) + elif fid == 2: + if ftype == TType.STRING: + self.ns = ( + iprot.readString().decode("utf-8", errors="replace") if sys.version_info[0] == 2 else iprot.readString() + ) + else: + iprot.skip(ftype) + else: + iprot.skip(ftype) + iprot.readFieldEnd() + iprot.readStructEnd() + + def write(self, oprot): + if oprot._fast_encode is not None and self.thrift_spec is not None: + oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec])) + return + oprot.writeStructBegin("WMDropResourcePlanRequest") + if self.resourcePlanName is not None: + oprot.writeFieldBegin("resourcePlanName", TType.STRING, 1) + oprot.writeString(self.resourcePlanName.encode("utf-8") if sys.version_info[0] == 2 else self.resourcePlanName) + oprot.writeFieldEnd() + if self.ns is not None: + oprot.writeFieldBegin("ns", TType.STRING, 2) + oprot.writeString(self.ns.encode("utf-8") if sys.version_info[0] == 2 else self.ns) + oprot.writeFieldEnd() + oprot.writeFieldStop() + oprot.writeStructEnd() + + def validate(self): + return + + def __repr__(self): + L = ["{}={!r}".format(key, value) for key, value in self.__dict__.items()] + return "{}({})".format(self.__class__.__name__, ", ".join(L)) + + def __eq__(self, other): + return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ + + def __ne__(self, other): + return not (self == other) + + +class WMDropResourcePlanResponse: + def read(self, iprot): + if ( + iprot._fast_decode is not None + and isinstance(iprot.trans, TTransport.CReadableTransport) + and self.thrift_spec is not None + ): + iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec]) + return + iprot.readStructBegin() + while True: + (fname, ftype, fid) = iprot.readFieldBegin() + if ftype == TType.STOP: + break + else: + iprot.skip(ftype) + iprot.readFieldEnd() + iprot.readStructEnd() + + def write(self, oprot): + if oprot._fast_encode is not None and self.thrift_spec is not None: + oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec])) + return + oprot.writeStructBegin("WMDropResourcePlanResponse") + oprot.writeFieldStop() + oprot.writeStructEnd() + + def validate(self): + return + + def __repr__(self): + L = ["{}={!r}".format(key, value) for key, value in self.__dict__.items()] + return "{}({})".format(self.__class__.__name__, ", ".join(L)) + + def __eq__(self, other): + return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ + + def __ne__(self, other): + return not (self == other) + + +class WMCreateTriggerRequest: + """ + Attributes: + - trigger + + """ + + def __init__( + self, + trigger=None, + ): + self.trigger = trigger + + def read(self, iprot): + if ( + iprot._fast_decode is not None + and isinstance(iprot.trans, TTransport.CReadableTransport) + and self.thrift_spec is not None + ): + iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec]) + return + iprot.readStructBegin() + while True: + (fname, ftype, fid) = iprot.readFieldBegin() + if ftype == TType.STOP: + break + if fid == 1: + if ftype == TType.STRUCT: + self.trigger = WMTrigger() + self.trigger.read(iprot) + else: + iprot.skip(ftype) + else: + iprot.skip(ftype) + iprot.readFieldEnd() + iprot.readStructEnd() + + def write(self, oprot): + if oprot._fast_encode is not None and self.thrift_spec is not None: + oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec])) + return + oprot.writeStructBegin("WMCreateTriggerRequest") + if self.trigger is not None: + oprot.writeFieldBegin("trigger", TType.STRUCT, 1) + self.trigger.write(oprot) + oprot.writeFieldEnd() + oprot.writeFieldStop() + oprot.writeStructEnd() + + def validate(self): + return + + def __repr__(self): + L = ["{}={!r}".format(key, value) for key, value in self.__dict__.items()] + return "{}({})".format(self.__class__.__name__, ", ".join(L)) + + def __eq__(self, other): + return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ + + def __ne__(self, other): + return not (self == other) + + +class WMCreateTriggerResponse: + def read(self, iprot): + if ( + iprot._fast_decode is not None + and isinstance(iprot.trans, TTransport.CReadableTransport) + and self.thrift_spec is not None + ): + iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec]) + return + iprot.readStructBegin() + while True: + (fname, ftype, fid) = iprot.readFieldBegin() + if ftype == TType.STOP: + break + else: + iprot.skip(ftype) + iprot.readFieldEnd() + iprot.readStructEnd() + + def write(self, oprot): + if oprot._fast_encode is not None and self.thrift_spec is not None: + oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec])) + return + oprot.writeStructBegin("WMCreateTriggerResponse") + oprot.writeFieldStop() + oprot.writeStructEnd() + + def validate(self): + return + + def __repr__(self): + L = ["{}={!r}".format(key, value) for key, value in self.__dict__.items()] + return "{}({})".format(self.__class__.__name__, ", ".join(L)) + + def __eq__(self, other): + return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ + + def __ne__(self, other): + return not (self == other) + + +class WMAlterTriggerRequest: + """ + Attributes: + - trigger + + """ + + def __init__( + self, + trigger=None, + ): + self.trigger = trigger + + def read(self, iprot): + if ( + iprot._fast_decode is not None + and isinstance(iprot.trans, TTransport.CReadableTransport) + and self.thrift_spec is not None + ): + iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec]) + return + iprot.readStructBegin() + while True: + (fname, ftype, fid) = iprot.readFieldBegin() + if ftype == TType.STOP: + break + if fid == 1: + if ftype == TType.STRUCT: + self.trigger = WMTrigger() + self.trigger.read(iprot) + else: + iprot.skip(ftype) + else: + iprot.skip(ftype) + iprot.readFieldEnd() + iprot.readStructEnd() + + def write(self, oprot): + if oprot._fast_encode is not None and self.thrift_spec is not None: + oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec])) + return + oprot.writeStructBegin("WMAlterTriggerRequest") + if self.trigger is not None: + oprot.writeFieldBegin("trigger", TType.STRUCT, 1) + self.trigger.write(oprot) + oprot.writeFieldEnd() + oprot.writeFieldStop() + oprot.writeStructEnd() + + def validate(self): + return + + def __repr__(self): + L = ["{}={!r}".format(key, value) for key, value in self.__dict__.items()] + return "{}({})".format(self.__class__.__name__, ", ".join(L)) + + def __eq__(self, other): + return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ + + def __ne__(self, other): + return not (self == other) + + +class WMAlterTriggerResponse: + def read(self, iprot): + if ( + iprot._fast_decode is not None + and isinstance(iprot.trans, TTransport.CReadableTransport) + and self.thrift_spec is not None + ): + iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec]) + return + iprot.readStructBegin() + while True: + (fname, ftype, fid) = iprot.readFieldBegin() + if ftype == TType.STOP: + break + else: + iprot.skip(ftype) + iprot.readFieldEnd() + iprot.readStructEnd() + + def write(self, oprot): + if oprot._fast_encode is not None and self.thrift_spec is not None: + oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec])) + return + oprot.writeStructBegin("WMAlterTriggerResponse") + oprot.writeFieldStop() + oprot.writeStructEnd() + + def validate(self): + return + + def __repr__(self): + L = ["{}={!r}".format(key, value) for key, value in self.__dict__.items()] + return "{}({})".format(self.__class__.__name__, ", ".join(L)) + + def __eq__(self, other): + return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ + + def __ne__(self, other): + return not (self == other) + + +class WMDropTriggerRequest: + """ + Attributes: + - resourcePlanName + - triggerName + - ns + + """ + + def __init__( + self, + resourcePlanName=None, + triggerName=None, + ns=None, + ): + self.resourcePlanName = resourcePlanName + self.triggerName = triggerName + self.ns = ns + + def read(self, iprot): + if ( + iprot._fast_decode is not None + and isinstance(iprot.trans, TTransport.CReadableTransport) + and self.thrift_spec is not None + ): + iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec]) + return + iprot.readStructBegin() + while True: + (fname, ftype, fid) = iprot.readFieldBegin() + if ftype == TType.STOP: + break + if fid == 1: + if ftype == TType.STRING: + self.resourcePlanName = ( + iprot.readString().decode("utf-8", errors="replace") if sys.version_info[0] == 2 else iprot.readString() + ) + else: + iprot.skip(ftype) + elif fid == 2: + if ftype == TType.STRING: + self.triggerName = ( + iprot.readString().decode("utf-8", errors="replace") if sys.version_info[0] == 2 else iprot.readString() + ) + else: + iprot.skip(ftype) + elif fid == 3: + if ftype == TType.STRING: + self.ns = ( + iprot.readString().decode("utf-8", errors="replace") if sys.version_info[0] == 2 else iprot.readString() + ) + else: + iprot.skip(ftype) + else: + iprot.skip(ftype) + iprot.readFieldEnd() + iprot.readStructEnd() + + def write(self, oprot): + if oprot._fast_encode is not None and self.thrift_spec is not None: + oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec])) + return + oprot.writeStructBegin("WMDropTriggerRequest") + if self.resourcePlanName is not None: + oprot.writeFieldBegin("resourcePlanName", TType.STRING, 1) + oprot.writeString(self.resourcePlanName.encode("utf-8") if sys.version_info[0] == 2 else self.resourcePlanName) + oprot.writeFieldEnd() + if self.triggerName is not None: + oprot.writeFieldBegin("triggerName", TType.STRING, 2) + oprot.writeString(self.triggerName.encode("utf-8") if sys.version_info[0] == 2 else self.triggerName) + oprot.writeFieldEnd() + if self.ns is not None: + oprot.writeFieldBegin("ns", TType.STRING, 3) + oprot.writeString(self.ns.encode("utf-8") if sys.version_info[0] == 2 else self.ns) + oprot.writeFieldEnd() + oprot.writeFieldStop() + oprot.writeStructEnd() + + def validate(self): + return + + def __repr__(self): + L = ["{}={!r}".format(key, value) for key, value in self.__dict__.items()] + return "{}({})".format(self.__class__.__name__, ", ".join(L)) + + def __eq__(self, other): + return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ + + def __ne__(self, other): + return not (self == other) + + +class WMDropTriggerResponse: + def read(self, iprot): + if ( + iprot._fast_decode is not None + and isinstance(iprot.trans, TTransport.CReadableTransport) + and self.thrift_spec is not None + ): + iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec]) + return + iprot.readStructBegin() + while True: + (fname, ftype, fid) = iprot.readFieldBegin() + if ftype == TType.STOP: + break + else: + iprot.skip(ftype) + iprot.readFieldEnd() + iprot.readStructEnd() + + def write(self, oprot): + if oprot._fast_encode is not None and self.thrift_spec is not None: + oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec])) + return + oprot.writeStructBegin("WMDropTriggerResponse") + oprot.writeFieldStop() + oprot.writeStructEnd() + + def validate(self): + return + + def __repr__(self): + L = ["{}={!r}".format(key, value) for key, value in self.__dict__.items()] + return "{}({})".format(self.__class__.__name__, ", ".join(L)) + + def __eq__(self, other): + return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ + + def __ne__(self, other): + return not (self == other) + + +class WMGetTriggersForResourePlanRequest: + """ + Attributes: + - resourcePlanName + - ns + + """ + + def __init__( + self, + resourcePlanName=None, + ns=None, + ): + self.resourcePlanName = resourcePlanName + self.ns = ns + + def read(self, iprot): + if ( + iprot._fast_decode is not None + and isinstance(iprot.trans, TTransport.CReadableTransport) + and self.thrift_spec is not None + ): + iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec]) + return + iprot.readStructBegin() + while True: + (fname, ftype, fid) = iprot.readFieldBegin() + if ftype == TType.STOP: + break + if fid == 1: + if ftype == TType.STRING: + self.resourcePlanName = ( + iprot.readString().decode("utf-8", errors="replace") if sys.version_info[0] == 2 else iprot.readString() + ) + else: + iprot.skip(ftype) + elif fid == 2: + if ftype == TType.STRING: + self.ns = ( + iprot.readString().decode("utf-8", errors="replace") if sys.version_info[0] == 2 else iprot.readString() + ) + else: + iprot.skip(ftype) + else: + iprot.skip(ftype) + iprot.readFieldEnd() + iprot.readStructEnd() + + def write(self, oprot): + if oprot._fast_encode is not None and self.thrift_spec is not None: + oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec])) + return + oprot.writeStructBegin("WMGetTriggersForResourePlanRequest") + if self.resourcePlanName is not None: + oprot.writeFieldBegin("resourcePlanName", TType.STRING, 1) + oprot.writeString(self.resourcePlanName.encode("utf-8") if sys.version_info[0] == 2 else self.resourcePlanName) + oprot.writeFieldEnd() + if self.ns is not None: + oprot.writeFieldBegin("ns", TType.STRING, 2) + oprot.writeString(self.ns.encode("utf-8") if sys.version_info[0] == 2 else self.ns) + oprot.writeFieldEnd() + oprot.writeFieldStop() + oprot.writeStructEnd() + + def validate(self): + return + + def __repr__(self): + L = ["{}={!r}".format(key, value) for key, value in self.__dict__.items()] + return "{}({})".format(self.__class__.__name__, ", ".join(L)) + + def __eq__(self, other): + return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ + + def __ne__(self, other): + return not (self == other) + + +class WMGetTriggersForResourePlanResponse: + """ + Attributes: + - triggers + + """ + + def __init__( + self, + triggers=None, + ): + self.triggers = triggers + + def read(self, iprot): + if ( + iprot._fast_decode is not None + and isinstance(iprot.trans, TTransport.CReadableTransport) + and self.thrift_spec is not None + ): + iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec]) + return + iprot.readStructBegin() + while True: + (fname, ftype, fid) = iprot.readFieldBegin() + if ftype == TType.STOP: + break + if fid == 1: + if ftype == TType.LIST: + self.triggers = [] + (_etype1047, _size1044) = iprot.readListBegin() + for _i1048 in range(_size1044): + _elem1049 = WMTrigger() + _elem1049.read(iprot) + self.triggers.append(_elem1049) + iprot.readListEnd() + else: + iprot.skip(ftype) + else: + iprot.skip(ftype) + iprot.readFieldEnd() + iprot.readStructEnd() + + def write(self, oprot): + if oprot._fast_encode is not None and self.thrift_spec is not None: + oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec])) + return + oprot.writeStructBegin("WMGetTriggersForResourePlanResponse") + if self.triggers is not None: + oprot.writeFieldBegin("triggers", TType.LIST, 1) + oprot.writeListBegin(TType.STRUCT, len(self.triggers)) + for iter1050 in self.triggers: + iter1050.write(oprot) + oprot.writeListEnd() + oprot.writeFieldEnd() + oprot.writeFieldStop() + oprot.writeStructEnd() + + def validate(self): + return + + def __repr__(self): + L = ["{}={!r}".format(key, value) for key, value in self.__dict__.items()] + return "{}({})".format(self.__class__.__name__, ", ".join(L)) + + def __eq__(self, other): + return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ + + def __ne__(self, other): + return not (self == other) + + +class WMCreatePoolRequest: + """ + Attributes: + - pool + + """ + + def __init__( + self, + pool=None, + ): + self.pool = pool + + def read(self, iprot): + if ( + iprot._fast_decode is not None + and isinstance(iprot.trans, TTransport.CReadableTransport) + and self.thrift_spec is not None + ): + iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec]) + return + iprot.readStructBegin() + while True: + (fname, ftype, fid) = iprot.readFieldBegin() + if ftype == TType.STOP: + break + if fid == 1: + if ftype == TType.STRUCT: + self.pool = WMPool() + self.pool.read(iprot) + else: + iprot.skip(ftype) + else: + iprot.skip(ftype) + iprot.readFieldEnd() + iprot.readStructEnd() + + def write(self, oprot): + if oprot._fast_encode is not None and self.thrift_spec is not None: + oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec])) + return + oprot.writeStructBegin("WMCreatePoolRequest") + if self.pool is not None: + oprot.writeFieldBegin("pool", TType.STRUCT, 1) + self.pool.write(oprot) + oprot.writeFieldEnd() + oprot.writeFieldStop() + oprot.writeStructEnd() + + def validate(self): + return + + def __repr__(self): + L = ["{}={!r}".format(key, value) for key, value in self.__dict__.items()] + return "{}({})".format(self.__class__.__name__, ", ".join(L)) + + def __eq__(self, other): + return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ + + def __ne__(self, other): + return not (self == other) + + +class WMCreatePoolResponse: + def read(self, iprot): + if ( + iprot._fast_decode is not None + and isinstance(iprot.trans, TTransport.CReadableTransport) + and self.thrift_spec is not None + ): + iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec]) + return + iprot.readStructBegin() + while True: + (fname, ftype, fid) = iprot.readFieldBegin() + if ftype == TType.STOP: + break + else: + iprot.skip(ftype) + iprot.readFieldEnd() + iprot.readStructEnd() + + def write(self, oprot): + if oprot._fast_encode is not None and self.thrift_spec is not None: + oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec])) + return + oprot.writeStructBegin("WMCreatePoolResponse") + oprot.writeFieldStop() + oprot.writeStructEnd() + + def validate(self): + return + + def __repr__(self): + L = ["{}={!r}".format(key, value) for key, value in self.__dict__.items()] + return "{}({})".format(self.__class__.__name__, ", ".join(L)) + + def __eq__(self, other): + return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ + + def __ne__(self, other): + return not (self == other) + + +class WMAlterPoolRequest: + """ + Attributes: + - pool + - poolPath + + """ + + def __init__( + self, + pool=None, + poolPath=None, + ): + self.pool = pool + self.poolPath = poolPath + + def read(self, iprot): + if ( + iprot._fast_decode is not None + and isinstance(iprot.trans, TTransport.CReadableTransport) + and self.thrift_spec is not None + ): + iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec]) + return + iprot.readStructBegin() + while True: + (fname, ftype, fid) = iprot.readFieldBegin() + if ftype == TType.STOP: + break + if fid == 1: + if ftype == TType.STRUCT: + self.pool = WMNullablePool() + self.pool.read(iprot) + else: + iprot.skip(ftype) + elif fid == 2: + if ftype == TType.STRING: + self.poolPath = ( + iprot.readString().decode("utf-8", errors="replace") if sys.version_info[0] == 2 else iprot.readString() + ) + else: + iprot.skip(ftype) + else: + iprot.skip(ftype) + iprot.readFieldEnd() + iprot.readStructEnd() + + def write(self, oprot): + if oprot._fast_encode is not None and self.thrift_spec is not None: + oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec])) + return + oprot.writeStructBegin("WMAlterPoolRequest") + if self.pool is not None: + oprot.writeFieldBegin("pool", TType.STRUCT, 1) + self.pool.write(oprot) + oprot.writeFieldEnd() + if self.poolPath is not None: + oprot.writeFieldBegin("poolPath", TType.STRING, 2) + oprot.writeString(self.poolPath.encode("utf-8") if sys.version_info[0] == 2 else self.poolPath) + oprot.writeFieldEnd() + oprot.writeFieldStop() + oprot.writeStructEnd() + + def validate(self): + return + + def __repr__(self): + L = ["{}={!r}".format(key, value) for key, value in self.__dict__.items()] + return "{}({})".format(self.__class__.__name__, ", ".join(L)) + + def __eq__(self, other): + return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ + + def __ne__(self, other): + return not (self == other) + + +class WMAlterPoolResponse: + def read(self, iprot): + if ( + iprot._fast_decode is not None + and isinstance(iprot.trans, TTransport.CReadableTransport) + and self.thrift_spec is not None + ): + iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec]) + return + iprot.readStructBegin() + while True: + (fname, ftype, fid) = iprot.readFieldBegin() + if ftype == TType.STOP: + break + else: + iprot.skip(ftype) + iprot.readFieldEnd() + iprot.readStructEnd() + + def write(self, oprot): + if oprot._fast_encode is not None and self.thrift_spec is not None: + oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec])) + return + oprot.writeStructBegin("WMAlterPoolResponse") + oprot.writeFieldStop() + oprot.writeStructEnd() + + def validate(self): + return + + def __repr__(self): + L = ["{}={!r}".format(key, value) for key, value in self.__dict__.items()] + return "{}({})".format(self.__class__.__name__, ", ".join(L)) + + def __eq__(self, other): + return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ + + def __ne__(self, other): + return not (self == other) + + +class WMDropPoolRequest: + """ + Attributes: + - resourcePlanName + - poolPath + - ns + + """ + + def __init__( + self, + resourcePlanName=None, + poolPath=None, + ns=None, + ): + self.resourcePlanName = resourcePlanName + self.poolPath = poolPath + self.ns = ns + + def read(self, iprot): + if ( + iprot._fast_decode is not None + and isinstance(iprot.trans, TTransport.CReadableTransport) + and self.thrift_spec is not None + ): + iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec]) + return + iprot.readStructBegin() + while True: + (fname, ftype, fid) = iprot.readFieldBegin() + if ftype == TType.STOP: + break + if fid == 1: + if ftype == TType.STRING: + self.resourcePlanName = ( + iprot.readString().decode("utf-8", errors="replace") if sys.version_info[0] == 2 else iprot.readString() + ) + else: + iprot.skip(ftype) + elif fid == 2: + if ftype == TType.STRING: + self.poolPath = ( + iprot.readString().decode("utf-8", errors="replace") if sys.version_info[0] == 2 else iprot.readString() + ) + else: + iprot.skip(ftype) + elif fid == 3: + if ftype == TType.STRING: + self.ns = ( + iprot.readString().decode("utf-8", errors="replace") if sys.version_info[0] == 2 else iprot.readString() + ) + else: + iprot.skip(ftype) + else: + iprot.skip(ftype) + iprot.readFieldEnd() + iprot.readStructEnd() + + def write(self, oprot): + if oprot._fast_encode is not None and self.thrift_spec is not None: + oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec])) + return + oprot.writeStructBegin("WMDropPoolRequest") + if self.resourcePlanName is not None: + oprot.writeFieldBegin("resourcePlanName", TType.STRING, 1) + oprot.writeString(self.resourcePlanName.encode("utf-8") if sys.version_info[0] == 2 else self.resourcePlanName) + oprot.writeFieldEnd() + if self.poolPath is not None: + oprot.writeFieldBegin("poolPath", TType.STRING, 2) + oprot.writeString(self.poolPath.encode("utf-8") if sys.version_info[0] == 2 else self.poolPath) + oprot.writeFieldEnd() + if self.ns is not None: + oprot.writeFieldBegin("ns", TType.STRING, 3) + oprot.writeString(self.ns.encode("utf-8") if sys.version_info[0] == 2 else self.ns) + oprot.writeFieldEnd() + oprot.writeFieldStop() + oprot.writeStructEnd() + + def validate(self): + return + + def __repr__(self): + L = ["{}={!r}".format(key, value) for key, value in self.__dict__.items()] + return "{}({})".format(self.__class__.__name__, ", ".join(L)) + + def __eq__(self, other): + return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ + + def __ne__(self, other): + return not (self == other) + + +class WMDropPoolResponse: + def read(self, iprot): + if ( + iprot._fast_decode is not None + and isinstance(iprot.trans, TTransport.CReadableTransport) + and self.thrift_spec is not None + ): + iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec]) + return + iprot.readStructBegin() + while True: + (fname, ftype, fid) = iprot.readFieldBegin() + if ftype == TType.STOP: + break + else: + iprot.skip(ftype) + iprot.readFieldEnd() + iprot.readStructEnd() + + def write(self, oprot): + if oprot._fast_encode is not None and self.thrift_spec is not None: + oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec])) + return + oprot.writeStructBegin("WMDropPoolResponse") + oprot.writeFieldStop() + oprot.writeStructEnd() + + def validate(self): + return + + def __repr__(self): + L = ["{}={!r}".format(key, value) for key, value in self.__dict__.items()] + return "{}({})".format(self.__class__.__name__, ", ".join(L)) + + def __eq__(self, other): + return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ + + def __ne__(self, other): + return not (self == other) + + +class WMCreateOrUpdateMappingRequest: + """ + Attributes: + - mapping + - update + + """ + + def __init__( + self, + mapping=None, + update=None, + ): + self.mapping = mapping + self.update = update + + def read(self, iprot): + if ( + iprot._fast_decode is not None + and isinstance(iprot.trans, TTransport.CReadableTransport) + and self.thrift_spec is not None + ): + iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec]) + return + iprot.readStructBegin() + while True: + (fname, ftype, fid) = iprot.readFieldBegin() + if ftype == TType.STOP: + break + if fid == 1: + if ftype == TType.STRUCT: + self.mapping = WMMapping() + self.mapping.read(iprot) + else: + iprot.skip(ftype) + elif fid == 2: + if ftype == TType.BOOL: + self.update = iprot.readBool() + else: + iprot.skip(ftype) + else: + iprot.skip(ftype) + iprot.readFieldEnd() + iprot.readStructEnd() + + def write(self, oprot): + if oprot._fast_encode is not None and self.thrift_spec is not None: + oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec])) + return + oprot.writeStructBegin("WMCreateOrUpdateMappingRequest") + if self.mapping is not None: + oprot.writeFieldBegin("mapping", TType.STRUCT, 1) + self.mapping.write(oprot) + oprot.writeFieldEnd() + if self.update is not None: + oprot.writeFieldBegin("update", TType.BOOL, 2) + oprot.writeBool(self.update) + oprot.writeFieldEnd() + oprot.writeFieldStop() + oprot.writeStructEnd() + + def validate(self): + return + + def __repr__(self): + L = ["{}={!r}".format(key, value) for key, value in self.__dict__.items()] + return "{}({})".format(self.__class__.__name__, ", ".join(L)) + + def __eq__(self, other): + return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ + + def __ne__(self, other): + return not (self == other) + + +class WMCreateOrUpdateMappingResponse: + def read(self, iprot): + if ( + iprot._fast_decode is not None + and isinstance(iprot.trans, TTransport.CReadableTransport) + and self.thrift_spec is not None + ): + iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec]) + return + iprot.readStructBegin() + while True: + (fname, ftype, fid) = iprot.readFieldBegin() + if ftype == TType.STOP: + break + else: + iprot.skip(ftype) + iprot.readFieldEnd() + iprot.readStructEnd() + + def write(self, oprot): + if oprot._fast_encode is not None and self.thrift_spec is not None: + oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec])) + return + oprot.writeStructBegin("WMCreateOrUpdateMappingResponse") + oprot.writeFieldStop() + oprot.writeStructEnd() + + def validate(self): + return + + def __repr__(self): + L = ["{}={!r}".format(key, value) for key, value in self.__dict__.items()] + return "{}({})".format(self.__class__.__name__, ", ".join(L)) + + def __eq__(self, other): + return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ + + def __ne__(self, other): + return not (self == other) + + +class WMDropMappingRequest: + """ + Attributes: + - mapping + + """ + + def __init__( + self, + mapping=None, + ): + self.mapping = mapping + + def read(self, iprot): + if ( + iprot._fast_decode is not None + and isinstance(iprot.trans, TTransport.CReadableTransport) + and self.thrift_spec is not None + ): + iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec]) + return + iprot.readStructBegin() + while True: + (fname, ftype, fid) = iprot.readFieldBegin() + if ftype == TType.STOP: + break + if fid == 1: + if ftype == TType.STRUCT: + self.mapping = WMMapping() + self.mapping.read(iprot) + else: + iprot.skip(ftype) + else: + iprot.skip(ftype) + iprot.readFieldEnd() + iprot.readStructEnd() + + def write(self, oprot): + if oprot._fast_encode is not None and self.thrift_spec is not None: + oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec])) + return + oprot.writeStructBegin("WMDropMappingRequest") + if self.mapping is not None: + oprot.writeFieldBegin("mapping", TType.STRUCT, 1) + self.mapping.write(oprot) + oprot.writeFieldEnd() + oprot.writeFieldStop() + oprot.writeStructEnd() + + def validate(self): + return + + def __repr__(self): + L = ["{}={!r}".format(key, value) for key, value in self.__dict__.items()] + return "{}({})".format(self.__class__.__name__, ", ".join(L)) + + def __eq__(self, other): + return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ + + def __ne__(self, other): + return not (self == other) + + +class WMDropMappingResponse: + def read(self, iprot): + if ( + iprot._fast_decode is not None + and isinstance(iprot.trans, TTransport.CReadableTransport) + and self.thrift_spec is not None + ): + iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec]) + return + iprot.readStructBegin() + while True: + (fname, ftype, fid) = iprot.readFieldBegin() + if ftype == TType.STOP: + break + else: + iprot.skip(ftype) + iprot.readFieldEnd() + iprot.readStructEnd() + + def write(self, oprot): + if oprot._fast_encode is not None and self.thrift_spec is not None: + oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec])) + return + oprot.writeStructBegin("WMDropMappingResponse") + oprot.writeFieldStop() + oprot.writeStructEnd() + + def validate(self): + return + + def __repr__(self): + L = ["{}={!r}".format(key, value) for key, value in self.__dict__.items()] + return "{}({})".format(self.__class__.__name__, ", ".join(L)) + + def __eq__(self, other): + return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ + + def __ne__(self, other): + return not (self == other) + + +class WMCreateOrDropTriggerToPoolMappingRequest: + """ + Attributes: + - resourcePlanName + - triggerName + - poolPath + - drop + - ns + + """ + + def __init__( + self, + resourcePlanName=None, + triggerName=None, + poolPath=None, + drop=None, + ns=None, + ): + self.resourcePlanName = resourcePlanName + self.triggerName = triggerName + self.poolPath = poolPath + self.drop = drop + self.ns = ns + + def read(self, iprot): + if ( + iprot._fast_decode is not None + and isinstance(iprot.trans, TTransport.CReadableTransport) + and self.thrift_spec is not None + ): + iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec]) + return + iprot.readStructBegin() + while True: + (fname, ftype, fid) = iprot.readFieldBegin() + if ftype == TType.STOP: + break + if fid == 1: + if ftype == TType.STRING: + self.resourcePlanName = ( + iprot.readString().decode("utf-8", errors="replace") if sys.version_info[0] == 2 else iprot.readString() + ) + else: + iprot.skip(ftype) + elif fid == 2: + if ftype == TType.STRING: + self.triggerName = ( + iprot.readString().decode("utf-8", errors="replace") if sys.version_info[0] == 2 else iprot.readString() + ) + else: + iprot.skip(ftype) + elif fid == 3: + if ftype == TType.STRING: + self.poolPath = ( + iprot.readString().decode("utf-8", errors="replace") if sys.version_info[0] == 2 else iprot.readString() + ) + else: + iprot.skip(ftype) + elif fid == 4: + if ftype == TType.BOOL: + self.drop = iprot.readBool() + else: + iprot.skip(ftype) + elif fid == 5: + if ftype == TType.STRING: + self.ns = ( + iprot.readString().decode("utf-8", errors="replace") if sys.version_info[0] == 2 else iprot.readString() + ) + else: + iprot.skip(ftype) + else: + iprot.skip(ftype) + iprot.readFieldEnd() + iprot.readStructEnd() + + def write(self, oprot): + if oprot._fast_encode is not None and self.thrift_spec is not None: + oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec])) + return + oprot.writeStructBegin("WMCreateOrDropTriggerToPoolMappingRequest") + if self.resourcePlanName is not None: + oprot.writeFieldBegin("resourcePlanName", TType.STRING, 1) + oprot.writeString(self.resourcePlanName.encode("utf-8") if sys.version_info[0] == 2 else self.resourcePlanName) + oprot.writeFieldEnd() + if self.triggerName is not None: + oprot.writeFieldBegin("triggerName", TType.STRING, 2) + oprot.writeString(self.triggerName.encode("utf-8") if sys.version_info[0] == 2 else self.triggerName) + oprot.writeFieldEnd() + if self.poolPath is not None: + oprot.writeFieldBegin("poolPath", TType.STRING, 3) + oprot.writeString(self.poolPath.encode("utf-8") if sys.version_info[0] == 2 else self.poolPath) + oprot.writeFieldEnd() + if self.drop is not None: + oprot.writeFieldBegin("drop", TType.BOOL, 4) + oprot.writeBool(self.drop) + oprot.writeFieldEnd() + if self.ns is not None: + oprot.writeFieldBegin("ns", TType.STRING, 5) + oprot.writeString(self.ns.encode("utf-8") if sys.version_info[0] == 2 else self.ns) + oprot.writeFieldEnd() + oprot.writeFieldStop() + oprot.writeStructEnd() + + def validate(self): + return + + def __repr__(self): + L = ["{}={!r}".format(key, value) for key, value in self.__dict__.items()] + return "{}({})".format(self.__class__.__name__, ", ".join(L)) + + def __eq__(self, other): + return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ + + def __ne__(self, other): + return not (self == other) + + +class WMCreateOrDropTriggerToPoolMappingResponse: + def read(self, iprot): + if ( + iprot._fast_decode is not None + and isinstance(iprot.trans, TTransport.CReadableTransport) + and self.thrift_spec is not None + ): + iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec]) + return + iprot.readStructBegin() + while True: + (fname, ftype, fid) = iprot.readFieldBegin() + if ftype == TType.STOP: + break + else: + iprot.skip(ftype) + iprot.readFieldEnd() + iprot.readStructEnd() + + def write(self, oprot): + if oprot._fast_encode is not None and self.thrift_spec is not None: + oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec])) + return + oprot.writeStructBegin("WMCreateOrDropTriggerToPoolMappingResponse") + oprot.writeFieldStop() + oprot.writeStructEnd() + + def validate(self): + return + + def __repr__(self): + L = ["{}={!r}".format(key, value) for key, value in self.__dict__.items()] + return "{}({})".format(self.__class__.__name__, ", ".join(L)) + + def __eq__(self, other): + return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ + + def __ne__(self, other): + return not (self == other) + + +class ISchema: + """ + Attributes: + - schemaType + - name + - catName + - dbName + - compatibility + - validationLevel + - canEvolve + - schemaGroup + - description + + """ + + def __init__( + self, + schemaType=None, + name=None, + catName=None, + dbName=None, + compatibility=None, + validationLevel=None, + canEvolve=None, + schemaGroup=None, + description=None, + ): + self.schemaType = schemaType + self.name = name + self.catName = catName + self.dbName = dbName + self.compatibility = compatibility + self.validationLevel = validationLevel + self.canEvolve = canEvolve + self.schemaGroup = schemaGroup + self.description = description + + def read(self, iprot): + if ( + iprot._fast_decode is not None + and isinstance(iprot.trans, TTransport.CReadableTransport) + and self.thrift_spec is not None + ): + iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec]) + return + iprot.readStructBegin() + while True: + (fname, ftype, fid) = iprot.readFieldBegin() + if ftype == TType.STOP: + break + if fid == 1: + if ftype == TType.I32: + self.schemaType = iprot.readI32() + else: + iprot.skip(ftype) + elif fid == 2: + if ftype == TType.STRING: + self.name = ( + iprot.readString().decode("utf-8", errors="replace") if sys.version_info[0] == 2 else iprot.readString() + ) + else: + iprot.skip(ftype) + elif fid == 3: + if ftype == TType.STRING: + self.catName = ( + iprot.readString().decode("utf-8", errors="replace") if sys.version_info[0] == 2 else iprot.readString() + ) + else: + iprot.skip(ftype) + elif fid == 4: + if ftype == TType.STRING: + self.dbName = ( + iprot.readString().decode("utf-8", errors="replace") if sys.version_info[0] == 2 else iprot.readString() + ) + else: + iprot.skip(ftype) + elif fid == 5: + if ftype == TType.I32: + self.compatibility = iprot.readI32() + else: + iprot.skip(ftype) + elif fid == 6: + if ftype == TType.I32: + self.validationLevel = iprot.readI32() + else: + iprot.skip(ftype) + elif fid == 7: + if ftype == TType.BOOL: + self.canEvolve = iprot.readBool() + else: + iprot.skip(ftype) + elif fid == 8: + if ftype == TType.STRING: + self.schemaGroup = ( + iprot.readString().decode("utf-8", errors="replace") if sys.version_info[0] == 2 else iprot.readString() + ) + else: + iprot.skip(ftype) + elif fid == 9: + if ftype == TType.STRING: + self.description = ( + iprot.readString().decode("utf-8", errors="replace") if sys.version_info[0] == 2 else iprot.readString() + ) + else: + iprot.skip(ftype) + else: + iprot.skip(ftype) + iprot.readFieldEnd() + iprot.readStructEnd() + + def write(self, oprot): + if oprot._fast_encode is not None and self.thrift_spec is not None: + oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec])) + return + oprot.writeStructBegin("ISchema") + if self.schemaType is not None: + oprot.writeFieldBegin("schemaType", TType.I32, 1) + oprot.writeI32(self.schemaType) + oprot.writeFieldEnd() + if self.name is not None: + oprot.writeFieldBegin("name", TType.STRING, 2) + oprot.writeString(self.name.encode("utf-8") if sys.version_info[0] == 2 else self.name) + oprot.writeFieldEnd() + if self.catName is not None: + oprot.writeFieldBegin("catName", TType.STRING, 3) + oprot.writeString(self.catName.encode("utf-8") if sys.version_info[0] == 2 else self.catName) + oprot.writeFieldEnd() + if self.dbName is not None: + oprot.writeFieldBegin("dbName", TType.STRING, 4) + oprot.writeString(self.dbName.encode("utf-8") if sys.version_info[0] == 2 else self.dbName) + oprot.writeFieldEnd() + if self.compatibility is not None: + oprot.writeFieldBegin("compatibility", TType.I32, 5) + oprot.writeI32(self.compatibility) + oprot.writeFieldEnd() + if self.validationLevel is not None: + oprot.writeFieldBegin("validationLevel", TType.I32, 6) + oprot.writeI32(self.validationLevel) + oprot.writeFieldEnd() + if self.canEvolve is not None: + oprot.writeFieldBegin("canEvolve", TType.BOOL, 7) + oprot.writeBool(self.canEvolve) + oprot.writeFieldEnd() + if self.schemaGroup is not None: + oprot.writeFieldBegin("schemaGroup", TType.STRING, 8) + oprot.writeString(self.schemaGroup.encode("utf-8") if sys.version_info[0] == 2 else self.schemaGroup) + oprot.writeFieldEnd() + if self.description is not None: + oprot.writeFieldBegin("description", TType.STRING, 9) + oprot.writeString(self.description.encode("utf-8") if sys.version_info[0] == 2 else self.description) + oprot.writeFieldEnd() + oprot.writeFieldStop() + oprot.writeStructEnd() + + def validate(self): + return + + def __repr__(self): + L = ["{}={!r}".format(key, value) for key, value in self.__dict__.items()] + return "{}({})".format(self.__class__.__name__, ", ".join(L)) + + def __eq__(self, other): + return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ + + def __ne__(self, other): + return not (self == other) + + +class ISchemaName: + """ + Attributes: + - catName + - dbName + - schemaName + + """ + + def __init__( + self, + catName=None, + dbName=None, + schemaName=None, + ): + self.catName = catName + self.dbName = dbName + self.schemaName = schemaName + + def read(self, iprot): + if ( + iprot._fast_decode is not None + and isinstance(iprot.trans, TTransport.CReadableTransport) + and self.thrift_spec is not None + ): + iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec]) + return + iprot.readStructBegin() + while True: + (fname, ftype, fid) = iprot.readFieldBegin() + if ftype == TType.STOP: + break + if fid == 1: + if ftype == TType.STRING: + self.catName = ( + iprot.readString().decode("utf-8", errors="replace") if sys.version_info[0] == 2 else iprot.readString() + ) + else: + iprot.skip(ftype) + elif fid == 2: + if ftype == TType.STRING: + self.dbName = ( + iprot.readString().decode("utf-8", errors="replace") if sys.version_info[0] == 2 else iprot.readString() + ) + else: + iprot.skip(ftype) + elif fid == 3: + if ftype == TType.STRING: + self.schemaName = ( + iprot.readString().decode("utf-8", errors="replace") if sys.version_info[0] == 2 else iprot.readString() + ) + else: + iprot.skip(ftype) + else: + iprot.skip(ftype) + iprot.readFieldEnd() + iprot.readStructEnd() + + def write(self, oprot): + if oprot._fast_encode is not None and self.thrift_spec is not None: + oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec])) + return + oprot.writeStructBegin("ISchemaName") + if self.catName is not None: + oprot.writeFieldBegin("catName", TType.STRING, 1) + oprot.writeString(self.catName.encode("utf-8") if sys.version_info[0] == 2 else self.catName) + oprot.writeFieldEnd() + if self.dbName is not None: + oprot.writeFieldBegin("dbName", TType.STRING, 2) + oprot.writeString(self.dbName.encode("utf-8") if sys.version_info[0] == 2 else self.dbName) + oprot.writeFieldEnd() + if self.schemaName is not None: + oprot.writeFieldBegin("schemaName", TType.STRING, 3) + oprot.writeString(self.schemaName.encode("utf-8") if sys.version_info[0] == 2 else self.schemaName) + oprot.writeFieldEnd() + oprot.writeFieldStop() + oprot.writeStructEnd() + + def validate(self): + return + + def __repr__(self): + L = ["{}={!r}".format(key, value) for key, value in self.__dict__.items()] + return "{}({})".format(self.__class__.__name__, ", ".join(L)) + + def __eq__(self, other): + return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ + + def __ne__(self, other): + return not (self == other) + + +class AlterISchemaRequest: + """ + Attributes: + - name + - newSchema + + """ + + def __init__( + self, + name=None, + newSchema=None, + ): + self.name = name + self.newSchema = newSchema + + def read(self, iprot): + if ( + iprot._fast_decode is not None + and isinstance(iprot.trans, TTransport.CReadableTransport) + and self.thrift_spec is not None + ): + iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec]) + return + iprot.readStructBegin() + while True: + (fname, ftype, fid) = iprot.readFieldBegin() + if ftype == TType.STOP: + break + if fid == 1: + if ftype == TType.STRUCT: + self.name = ISchemaName() + self.name.read(iprot) + else: + iprot.skip(ftype) + elif fid == 3: + if ftype == TType.STRUCT: + self.newSchema = ISchema() + self.newSchema.read(iprot) + else: + iprot.skip(ftype) + else: + iprot.skip(ftype) + iprot.readFieldEnd() + iprot.readStructEnd() + + def write(self, oprot): + if oprot._fast_encode is not None and self.thrift_spec is not None: + oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec])) + return + oprot.writeStructBegin("AlterISchemaRequest") + if self.name is not None: + oprot.writeFieldBegin("name", TType.STRUCT, 1) + self.name.write(oprot) + oprot.writeFieldEnd() + if self.newSchema is not None: + oprot.writeFieldBegin("newSchema", TType.STRUCT, 3) + self.newSchema.write(oprot) + oprot.writeFieldEnd() + oprot.writeFieldStop() + oprot.writeStructEnd() + + def validate(self): + return + + def __repr__(self): + L = ["{}={!r}".format(key, value) for key, value in self.__dict__.items()] + return "{}({})".format(self.__class__.__name__, ", ".join(L)) + + def __eq__(self, other): + return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ + + def __ne__(self, other): + return not (self == other) + + +class SchemaVersion: + """ + Attributes: + - schema + - version + - createdAt + - cols + - state + - description + - schemaText + - fingerprint + - name + - serDe + + """ + + def __init__( + self, + schema=None, + version=None, + createdAt=None, + cols=None, + state=None, + description=None, + schemaText=None, + fingerprint=None, + name=None, + serDe=None, + ): + self.schema = schema + self.version = version + self.createdAt = createdAt + self.cols = cols + self.state = state + self.description = description + self.schemaText = schemaText + self.fingerprint = fingerprint + self.name = name + self.serDe = serDe + + def read(self, iprot): + if ( + iprot._fast_decode is not None + and isinstance(iprot.trans, TTransport.CReadableTransport) + and self.thrift_spec is not None + ): + iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec]) + return + iprot.readStructBegin() + while True: + (fname, ftype, fid) = iprot.readFieldBegin() + if ftype == TType.STOP: + break + if fid == 1: + if ftype == TType.STRUCT: + self.schema = ISchemaName() + self.schema.read(iprot) + else: + iprot.skip(ftype) + elif fid == 2: + if ftype == TType.I32: + self.version = iprot.readI32() + else: + iprot.skip(ftype) + elif fid == 3: + if ftype == TType.I64: + self.createdAt = iprot.readI64() + else: + iprot.skip(ftype) + elif fid == 4: + if ftype == TType.LIST: + self.cols = [] + (_etype1054, _size1051) = iprot.readListBegin() + for _i1055 in range(_size1051): + _elem1056 = FieldSchema() + _elem1056.read(iprot) + self.cols.append(_elem1056) + iprot.readListEnd() + else: + iprot.skip(ftype) + elif fid == 5: + if ftype == TType.I32: + self.state = iprot.readI32() + else: + iprot.skip(ftype) + elif fid == 6: + if ftype == TType.STRING: + self.description = ( + iprot.readString().decode("utf-8", errors="replace") if sys.version_info[0] == 2 else iprot.readString() + ) + else: + iprot.skip(ftype) + elif fid == 7: + if ftype == TType.STRING: + self.schemaText = ( + iprot.readString().decode("utf-8", errors="replace") if sys.version_info[0] == 2 else iprot.readString() + ) + else: + iprot.skip(ftype) + elif fid == 8: + if ftype == TType.STRING: + self.fingerprint = ( + iprot.readString().decode("utf-8", errors="replace") if sys.version_info[0] == 2 else iprot.readString() + ) + else: + iprot.skip(ftype) + elif fid == 9: + if ftype == TType.STRING: + self.name = ( + iprot.readString().decode("utf-8", errors="replace") if sys.version_info[0] == 2 else iprot.readString() + ) + else: + iprot.skip(ftype) + elif fid == 10: + if ftype == TType.STRUCT: + self.serDe = SerDeInfo() + self.serDe.read(iprot) + else: + iprot.skip(ftype) + else: + iprot.skip(ftype) + iprot.readFieldEnd() + iprot.readStructEnd() + + def write(self, oprot): + if oprot._fast_encode is not None and self.thrift_spec is not None: + oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec])) + return + oprot.writeStructBegin("SchemaVersion") + if self.schema is not None: + oprot.writeFieldBegin("schema", TType.STRUCT, 1) + self.schema.write(oprot) + oprot.writeFieldEnd() + if self.version is not None: + oprot.writeFieldBegin("version", TType.I32, 2) + oprot.writeI32(self.version) + oprot.writeFieldEnd() + if self.createdAt is not None: + oprot.writeFieldBegin("createdAt", TType.I64, 3) + oprot.writeI64(self.createdAt) + oprot.writeFieldEnd() + if self.cols is not None: + oprot.writeFieldBegin("cols", TType.LIST, 4) + oprot.writeListBegin(TType.STRUCT, len(self.cols)) + for iter1057 in self.cols: + iter1057.write(oprot) + oprot.writeListEnd() + oprot.writeFieldEnd() + if self.state is not None: + oprot.writeFieldBegin("state", TType.I32, 5) + oprot.writeI32(self.state) + oprot.writeFieldEnd() + if self.description is not None: + oprot.writeFieldBegin("description", TType.STRING, 6) + oprot.writeString(self.description.encode("utf-8") if sys.version_info[0] == 2 else self.description) + oprot.writeFieldEnd() + if self.schemaText is not None: + oprot.writeFieldBegin("schemaText", TType.STRING, 7) + oprot.writeString(self.schemaText.encode("utf-8") if sys.version_info[0] == 2 else self.schemaText) + oprot.writeFieldEnd() + if self.fingerprint is not None: + oprot.writeFieldBegin("fingerprint", TType.STRING, 8) + oprot.writeString(self.fingerprint.encode("utf-8") if sys.version_info[0] == 2 else self.fingerprint) + oprot.writeFieldEnd() + if self.name is not None: + oprot.writeFieldBegin("name", TType.STRING, 9) + oprot.writeString(self.name.encode("utf-8") if sys.version_info[0] == 2 else self.name) + oprot.writeFieldEnd() + if self.serDe is not None: + oprot.writeFieldBegin("serDe", TType.STRUCT, 10) + self.serDe.write(oprot) + oprot.writeFieldEnd() + oprot.writeFieldStop() + oprot.writeStructEnd() + + def validate(self): + return + + def __repr__(self): + L = ["{}={!r}".format(key, value) for key, value in self.__dict__.items()] + return "{}({})".format(self.__class__.__name__, ", ".join(L)) + + def __eq__(self, other): + return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ + + def __ne__(self, other): + return not (self == other) + + +class SchemaVersionDescriptor: + """ + Attributes: + - schema + - version + + """ + + def __init__( + self, + schema=None, + version=None, + ): + self.schema = schema + self.version = version + + def read(self, iprot): + if ( + iprot._fast_decode is not None + and isinstance(iprot.trans, TTransport.CReadableTransport) + and self.thrift_spec is not None + ): + iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec]) + return + iprot.readStructBegin() + while True: + (fname, ftype, fid) = iprot.readFieldBegin() + if ftype == TType.STOP: + break + if fid == 1: + if ftype == TType.STRUCT: + self.schema = ISchemaName() + self.schema.read(iprot) + else: + iprot.skip(ftype) + elif fid == 2: + if ftype == TType.I32: + self.version = iprot.readI32() + else: + iprot.skip(ftype) + else: + iprot.skip(ftype) + iprot.readFieldEnd() + iprot.readStructEnd() + + def write(self, oprot): + if oprot._fast_encode is not None and self.thrift_spec is not None: + oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec])) + return + oprot.writeStructBegin("SchemaVersionDescriptor") + if self.schema is not None: + oprot.writeFieldBegin("schema", TType.STRUCT, 1) + self.schema.write(oprot) + oprot.writeFieldEnd() + if self.version is not None: + oprot.writeFieldBegin("version", TType.I32, 2) + oprot.writeI32(self.version) + oprot.writeFieldEnd() + oprot.writeFieldStop() + oprot.writeStructEnd() + + def validate(self): + return + + def __repr__(self): + L = ["{}={!r}".format(key, value) for key, value in self.__dict__.items()] + return "{}({})".format(self.__class__.__name__, ", ".join(L)) + + def __eq__(self, other): + return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ + + def __ne__(self, other): + return not (self == other) + + +class FindSchemasByColsRqst: + """ + Attributes: + - colName + - colNamespace + - type + + """ + + def __init__( + self, + colName=None, + colNamespace=None, + type=None, + ): + self.colName = colName + self.colNamespace = colNamespace + self.type = type + + def read(self, iprot): + if ( + iprot._fast_decode is not None + and isinstance(iprot.trans, TTransport.CReadableTransport) + and self.thrift_spec is not None + ): + iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec]) + return + iprot.readStructBegin() + while True: + (fname, ftype, fid) = iprot.readFieldBegin() + if ftype == TType.STOP: + break + if fid == 1: + if ftype == TType.STRING: + self.colName = ( + iprot.readString().decode("utf-8", errors="replace") if sys.version_info[0] == 2 else iprot.readString() + ) + else: + iprot.skip(ftype) + elif fid == 2: + if ftype == TType.STRING: + self.colNamespace = ( + iprot.readString().decode("utf-8", errors="replace") if sys.version_info[0] == 2 else iprot.readString() + ) + else: + iprot.skip(ftype) + elif fid == 3: + if ftype == TType.STRING: + self.type = ( + iprot.readString().decode("utf-8", errors="replace") if sys.version_info[0] == 2 else iprot.readString() + ) + else: + iprot.skip(ftype) + else: + iprot.skip(ftype) + iprot.readFieldEnd() + iprot.readStructEnd() + + def write(self, oprot): + if oprot._fast_encode is not None and self.thrift_spec is not None: + oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec])) + return + oprot.writeStructBegin("FindSchemasByColsRqst") + if self.colName is not None: + oprot.writeFieldBegin("colName", TType.STRING, 1) + oprot.writeString(self.colName.encode("utf-8") if sys.version_info[0] == 2 else self.colName) + oprot.writeFieldEnd() + if self.colNamespace is not None: + oprot.writeFieldBegin("colNamespace", TType.STRING, 2) + oprot.writeString(self.colNamespace.encode("utf-8") if sys.version_info[0] == 2 else self.colNamespace) + oprot.writeFieldEnd() + if self.type is not None: + oprot.writeFieldBegin("type", TType.STRING, 3) + oprot.writeString(self.type.encode("utf-8") if sys.version_info[0] == 2 else self.type) + oprot.writeFieldEnd() + oprot.writeFieldStop() + oprot.writeStructEnd() + + def validate(self): + return + + def __repr__(self): + L = ["{}={!r}".format(key, value) for key, value in self.__dict__.items()] + return "{}({})".format(self.__class__.__name__, ", ".join(L)) + + def __eq__(self, other): + return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ + + def __ne__(self, other): + return not (self == other) + + +class FindSchemasByColsResp: + """ + Attributes: + - schemaVersions + + """ + + def __init__( + self, + schemaVersions=None, + ): + self.schemaVersions = schemaVersions + + def read(self, iprot): + if ( + iprot._fast_decode is not None + and isinstance(iprot.trans, TTransport.CReadableTransport) + and self.thrift_spec is not None + ): + iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec]) + return + iprot.readStructBegin() + while True: + (fname, ftype, fid) = iprot.readFieldBegin() + if ftype == TType.STOP: + break + if fid == 1: + if ftype == TType.LIST: + self.schemaVersions = [] + (_etype1061, _size1058) = iprot.readListBegin() + for _i1062 in range(_size1058): + _elem1063 = SchemaVersionDescriptor() + _elem1063.read(iprot) + self.schemaVersions.append(_elem1063) + iprot.readListEnd() + else: + iprot.skip(ftype) + else: + iprot.skip(ftype) + iprot.readFieldEnd() + iprot.readStructEnd() + + def write(self, oprot): + if oprot._fast_encode is not None and self.thrift_spec is not None: + oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec])) + return + oprot.writeStructBegin("FindSchemasByColsResp") + if self.schemaVersions is not None: + oprot.writeFieldBegin("schemaVersions", TType.LIST, 1) + oprot.writeListBegin(TType.STRUCT, len(self.schemaVersions)) + for iter1064 in self.schemaVersions: + iter1064.write(oprot) + oprot.writeListEnd() + oprot.writeFieldEnd() + oprot.writeFieldStop() + oprot.writeStructEnd() + + def validate(self): + return + + def __repr__(self): + L = ["{}={!r}".format(key, value) for key, value in self.__dict__.items()] + return "{}({})".format(self.__class__.__name__, ", ".join(L)) + + def __eq__(self, other): + return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ + + def __ne__(self, other): + return not (self == other) + + +class MapSchemaVersionToSerdeRequest: + """ + Attributes: + - schemaVersion + - serdeName + + """ + + def __init__( + self, + schemaVersion=None, + serdeName=None, + ): + self.schemaVersion = schemaVersion + self.serdeName = serdeName + + def read(self, iprot): + if ( + iprot._fast_decode is not None + and isinstance(iprot.trans, TTransport.CReadableTransport) + and self.thrift_spec is not None + ): + iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec]) + return + iprot.readStructBegin() + while True: + (fname, ftype, fid) = iprot.readFieldBegin() + if ftype == TType.STOP: + break + if fid == 1: + if ftype == TType.STRUCT: + self.schemaVersion = SchemaVersionDescriptor() + self.schemaVersion.read(iprot) + else: + iprot.skip(ftype) + elif fid == 2: + if ftype == TType.STRING: + self.serdeName = ( + iprot.readString().decode("utf-8", errors="replace") if sys.version_info[0] == 2 else iprot.readString() + ) + else: + iprot.skip(ftype) + else: + iprot.skip(ftype) + iprot.readFieldEnd() + iprot.readStructEnd() + + def write(self, oprot): + if oprot._fast_encode is not None and self.thrift_spec is not None: + oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec])) + return + oprot.writeStructBegin("MapSchemaVersionToSerdeRequest") + if self.schemaVersion is not None: + oprot.writeFieldBegin("schemaVersion", TType.STRUCT, 1) + self.schemaVersion.write(oprot) + oprot.writeFieldEnd() + if self.serdeName is not None: + oprot.writeFieldBegin("serdeName", TType.STRING, 2) + oprot.writeString(self.serdeName.encode("utf-8") if sys.version_info[0] == 2 else self.serdeName) + oprot.writeFieldEnd() + oprot.writeFieldStop() + oprot.writeStructEnd() + + def validate(self): + return + + def __repr__(self): + L = ["{}={!r}".format(key, value) for key, value in self.__dict__.items()] + return "{}({})".format(self.__class__.__name__, ", ".join(L)) + + def __eq__(self, other): + return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ + + def __ne__(self, other): + return not (self == other) + + +class SetSchemaVersionStateRequest: + """ + Attributes: + - schemaVersion + - state + + """ + + def __init__( + self, + schemaVersion=None, + state=None, + ): + self.schemaVersion = schemaVersion + self.state = state + + def read(self, iprot): + if ( + iprot._fast_decode is not None + and isinstance(iprot.trans, TTransport.CReadableTransport) + and self.thrift_spec is not None + ): + iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec]) + return + iprot.readStructBegin() + while True: + (fname, ftype, fid) = iprot.readFieldBegin() + if ftype == TType.STOP: + break + if fid == 1: + if ftype == TType.STRUCT: + self.schemaVersion = SchemaVersionDescriptor() + self.schemaVersion.read(iprot) + else: + iprot.skip(ftype) + elif fid == 2: + if ftype == TType.I32: + self.state = iprot.readI32() + else: + iprot.skip(ftype) + else: + iprot.skip(ftype) + iprot.readFieldEnd() + iprot.readStructEnd() + + def write(self, oprot): + if oprot._fast_encode is not None and self.thrift_spec is not None: + oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec])) + return + oprot.writeStructBegin("SetSchemaVersionStateRequest") + if self.schemaVersion is not None: + oprot.writeFieldBegin("schemaVersion", TType.STRUCT, 1) + self.schemaVersion.write(oprot) + oprot.writeFieldEnd() + if self.state is not None: + oprot.writeFieldBegin("state", TType.I32, 2) + oprot.writeI32(self.state) + oprot.writeFieldEnd() + oprot.writeFieldStop() + oprot.writeStructEnd() + + def validate(self): + return + + def __repr__(self): + L = ["{}={!r}".format(key, value) for key, value in self.__dict__.items()] + return "{}({})".format(self.__class__.__name__, ", ".join(L)) + + def __eq__(self, other): + return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ + + def __ne__(self, other): + return not (self == other) + + +class GetSerdeRequest: + """ + Attributes: + - serdeName + + """ + + def __init__( + self, + serdeName=None, + ): + self.serdeName = serdeName + + def read(self, iprot): + if ( + iprot._fast_decode is not None + and isinstance(iprot.trans, TTransport.CReadableTransport) + and self.thrift_spec is not None + ): + iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec]) + return + iprot.readStructBegin() + while True: + (fname, ftype, fid) = iprot.readFieldBegin() + if ftype == TType.STOP: + break + if fid == 1: + if ftype == TType.STRING: + self.serdeName = ( + iprot.readString().decode("utf-8", errors="replace") if sys.version_info[0] == 2 else iprot.readString() + ) + else: + iprot.skip(ftype) + else: + iprot.skip(ftype) + iprot.readFieldEnd() + iprot.readStructEnd() + + def write(self, oprot): + if oprot._fast_encode is not None and self.thrift_spec is not None: + oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec])) + return + oprot.writeStructBegin("GetSerdeRequest") + if self.serdeName is not None: + oprot.writeFieldBegin("serdeName", TType.STRING, 1) + oprot.writeString(self.serdeName.encode("utf-8") if sys.version_info[0] == 2 else self.serdeName) + oprot.writeFieldEnd() + oprot.writeFieldStop() + oprot.writeStructEnd() + + def validate(self): + return + + def __repr__(self): + L = ["{}={!r}".format(key, value) for key, value in self.__dict__.items()] + return "{}({})".format(self.__class__.__name__, ", ".join(L)) + + def __eq__(self, other): + return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ + + def __ne__(self, other): + return not (self == other) + + +class RuntimeStat: + """ + Attributes: + - createTime + - weight + - payload + + """ + + def __init__( + self, + createTime=None, + weight=None, + payload=None, + ): + self.createTime = createTime + self.weight = weight + self.payload = payload + + def read(self, iprot): + if ( + iprot._fast_decode is not None + and isinstance(iprot.trans, TTransport.CReadableTransport) + and self.thrift_spec is not None + ): + iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec]) + return + iprot.readStructBegin() + while True: + (fname, ftype, fid) = iprot.readFieldBegin() + if ftype == TType.STOP: + break + if fid == 1: + if ftype == TType.I32: + self.createTime = iprot.readI32() + else: + iprot.skip(ftype) + elif fid == 2: + if ftype == TType.I32: + self.weight = iprot.readI32() + else: + iprot.skip(ftype) + elif fid == 3: + if ftype == TType.STRING: + self.payload = iprot.readBinary() + else: + iprot.skip(ftype) + else: + iprot.skip(ftype) + iprot.readFieldEnd() + iprot.readStructEnd() + + def write(self, oprot): + if oprot._fast_encode is not None and self.thrift_spec is not None: + oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec])) + return + oprot.writeStructBegin("RuntimeStat") + if self.createTime is not None: + oprot.writeFieldBegin("createTime", TType.I32, 1) + oprot.writeI32(self.createTime) + oprot.writeFieldEnd() + if self.weight is not None: + oprot.writeFieldBegin("weight", TType.I32, 2) + oprot.writeI32(self.weight) + oprot.writeFieldEnd() + if self.payload is not None: + oprot.writeFieldBegin("payload", TType.STRING, 3) + oprot.writeBinary(self.payload) + oprot.writeFieldEnd() + oprot.writeFieldStop() + oprot.writeStructEnd() + + def validate(self): + if self.weight is None: + raise TProtocolException(message="Required field weight is unset!") + if self.payload is None: + raise TProtocolException(message="Required field payload is unset!") + return + + def __repr__(self): + L = ["{}={!r}".format(key, value) for key, value in self.__dict__.items()] + return "{}({})".format(self.__class__.__name__, ", ".join(L)) + + def __eq__(self, other): + return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ + + def __ne__(self, other): + return not (self == other) + + +class GetRuntimeStatsRequest: + """ + Attributes: + - maxWeight + - maxCreateTime + + """ + + def __init__( + self, + maxWeight=None, + maxCreateTime=None, + ): + self.maxWeight = maxWeight + self.maxCreateTime = maxCreateTime + + def read(self, iprot): + if ( + iprot._fast_decode is not None + and isinstance(iprot.trans, TTransport.CReadableTransport) + and self.thrift_spec is not None + ): + iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec]) + return + iprot.readStructBegin() + while True: + (fname, ftype, fid) = iprot.readFieldBegin() + if ftype == TType.STOP: + break + if fid == 1: + if ftype == TType.I32: + self.maxWeight = iprot.readI32() + else: + iprot.skip(ftype) + elif fid == 2: + if ftype == TType.I32: + self.maxCreateTime = iprot.readI32() + else: + iprot.skip(ftype) + else: + iprot.skip(ftype) + iprot.readFieldEnd() + iprot.readStructEnd() + + def write(self, oprot): + if oprot._fast_encode is not None and self.thrift_spec is not None: + oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec])) + return + oprot.writeStructBegin("GetRuntimeStatsRequest") + if self.maxWeight is not None: + oprot.writeFieldBegin("maxWeight", TType.I32, 1) + oprot.writeI32(self.maxWeight) + oprot.writeFieldEnd() + if self.maxCreateTime is not None: + oprot.writeFieldBegin("maxCreateTime", TType.I32, 2) + oprot.writeI32(self.maxCreateTime) + oprot.writeFieldEnd() + oprot.writeFieldStop() + oprot.writeStructEnd() + + def validate(self): + if self.maxWeight is None: + raise TProtocolException(message="Required field maxWeight is unset!") + if self.maxCreateTime is None: + raise TProtocolException(message="Required field maxCreateTime is unset!") + return + + def __repr__(self): + L = ["{}={!r}".format(key, value) for key, value in self.__dict__.items()] + return "{}({})".format(self.__class__.__name__, ", ".join(L)) + + def __eq__(self, other): + return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ + + def __ne__(self, other): + return not (self == other) + + +class CreateTableRequest: + """ + Attributes: + - table + - envContext + - primaryKeys + - foreignKeys + - uniqueConstraints + - notNullConstraints + - defaultConstraints + - checkConstraints + - processorCapabilities + - processorIdentifier + + """ + + def __init__( + self, + table=None, + envContext=None, + primaryKeys=None, + foreignKeys=None, + uniqueConstraints=None, + notNullConstraints=None, + defaultConstraints=None, + checkConstraints=None, + processorCapabilities=None, + processorIdentifier=None, + ): + self.table = table + self.envContext = envContext + self.primaryKeys = primaryKeys + self.foreignKeys = foreignKeys + self.uniqueConstraints = uniqueConstraints + self.notNullConstraints = notNullConstraints + self.defaultConstraints = defaultConstraints + self.checkConstraints = checkConstraints + self.processorCapabilities = processorCapabilities + self.processorIdentifier = processorIdentifier + + def read(self, iprot): + if ( + iprot._fast_decode is not None + and isinstance(iprot.trans, TTransport.CReadableTransport) + and self.thrift_spec is not None + ): + iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec]) + return + iprot.readStructBegin() + while True: + (fname, ftype, fid) = iprot.readFieldBegin() + if ftype == TType.STOP: + break + if fid == 1: + if ftype == TType.STRUCT: + self.table = Table() + self.table.read(iprot) + else: + iprot.skip(ftype) + elif fid == 2: + if ftype == TType.STRUCT: + self.envContext = EnvironmentContext() + self.envContext.read(iprot) + else: + iprot.skip(ftype) + elif fid == 3: + if ftype == TType.LIST: + self.primaryKeys = [] + (_etype1068, _size1065) = iprot.readListBegin() + for _i1069 in range(_size1065): + _elem1070 = SQLPrimaryKey() + _elem1070.read(iprot) + self.primaryKeys.append(_elem1070) + iprot.readListEnd() + else: + iprot.skip(ftype) + elif fid == 4: + if ftype == TType.LIST: + self.foreignKeys = [] + (_etype1074, _size1071) = iprot.readListBegin() + for _i1075 in range(_size1071): + _elem1076 = SQLForeignKey() + _elem1076.read(iprot) + self.foreignKeys.append(_elem1076) + iprot.readListEnd() + else: + iprot.skip(ftype) + elif fid == 5: + if ftype == TType.LIST: + self.uniqueConstraints = [] + (_etype1080, _size1077) = iprot.readListBegin() + for _i1081 in range(_size1077): + _elem1082 = SQLUniqueConstraint() + _elem1082.read(iprot) + self.uniqueConstraints.append(_elem1082) + iprot.readListEnd() + else: + iprot.skip(ftype) + elif fid == 6: + if ftype == TType.LIST: + self.notNullConstraints = [] + (_etype1086, _size1083) = iprot.readListBegin() + for _i1087 in range(_size1083): + _elem1088 = SQLNotNullConstraint() + _elem1088.read(iprot) + self.notNullConstraints.append(_elem1088) + iprot.readListEnd() + else: + iprot.skip(ftype) + elif fid == 7: + if ftype == TType.LIST: + self.defaultConstraints = [] + (_etype1092, _size1089) = iprot.readListBegin() + for _i1093 in range(_size1089): + _elem1094 = SQLDefaultConstraint() + _elem1094.read(iprot) + self.defaultConstraints.append(_elem1094) + iprot.readListEnd() + else: + iprot.skip(ftype) + elif fid == 8: + if ftype == TType.LIST: + self.checkConstraints = [] + (_etype1098, _size1095) = iprot.readListBegin() + for _i1099 in range(_size1095): + _elem1100 = SQLCheckConstraint() + _elem1100.read(iprot) + self.checkConstraints.append(_elem1100) + iprot.readListEnd() + else: + iprot.skip(ftype) + elif fid == 9: + if ftype == TType.LIST: + self.processorCapabilities = [] + (_etype1104, _size1101) = iprot.readListBegin() + for _i1105 in range(_size1101): + _elem1106 = ( + iprot.readString().decode("utf-8", errors="replace") + if sys.version_info[0] == 2 + else iprot.readString() + ) + self.processorCapabilities.append(_elem1106) + iprot.readListEnd() + else: + iprot.skip(ftype) + elif fid == 10: + if ftype == TType.STRING: + self.processorIdentifier = ( + iprot.readString().decode("utf-8", errors="replace") if sys.version_info[0] == 2 else iprot.readString() + ) + else: + iprot.skip(ftype) + else: + iprot.skip(ftype) + iprot.readFieldEnd() + iprot.readStructEnd() + + def write(self, oprot): + if oprot._fast_encode is not None and self.thrift_spec is not None: + oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec])) + return + oprot.writeStructBegin("CreateTableRequest") + if self.table is not None: + oprot.writeFieldBegin("table", TType.STRUCT, 1) + self.table.write(oprot) + oprot.writeFieldEnd() + if self.envContext is not None: + oprot.writeFieldBegin("envContext", TType.STRUCT, 2) + self.envContext.write(oprot) + oprot.writeFieldEnd() + if self.primaryKeys is not None: + oprot.writeFieldBegin("primaryKeys", TType.LIST, 3) + oprot.writeListBegin(TType.STRUCT, len(self.primaryKeys)) + for iter1107 in self.primaryKeys: + iter1107.write(oprot) + oprot.writeListEnd() + oprot.writeFieldEnd() + if self.foreignKeys is not None: + oprot.writeFieldBegin("foreignKeys", TType.LIST, 4) + oprot.writeListBegin(TType.STRUCT, len(self.foreignKeys)) + for iter1108 in self.foreignKeys: + iter1108.write(oprot) + oprot.writeListEnd() + oprot.writeFieldEnd() + if self.uniqueConstraints is not None: + oprot.writeFieldBegin("uniqueConstraints", TType.LIST, 5) + oprot.writeListBegin(TType.STRUCT, len(self.uniqueConstraints)) + for iter1109 in self.uniqueConstraints: + iter1109.write(oprot) + oprot.writeListEnd() + oprot.writeFieldEnd() + if self.notNullConstraints is not None: + oprot.writeFieldBegin("notNullConstraints", TType.LIST, 6) + oprot.writeListBegin(TType.STRUCT, len(self.notNullConstraints)) + for iter1110 in self.notNullConstraints: + iter1110.write(oprot) + oprot.writeListEnd() + oprot.writeFieldEnd() + if self.defaultConstraints is not None: + oprot.writeFieldBegin("defaultConstraints", TType.LIST, 7) + oprot.writeListBegin(TType.STRUCT, len(self.defaultConstraints)) + for iter1111 in self.defaultConstraints: + iter1111.write(oprot) + oprot.writeListEnd() + oprot.writeFieldEnd() + if self.checkConstraints is not None: + oprot.writeFieldBegin("checkConstraints", TType.LIST, 8) + oprot.writeListBegin(TType.STRUCT, len(self.checkConstraints)) + for iter1112 in self.checkConstraints: + iter1112.write(oprot) + oprot.writeListEnd() + oprot.writeFieldEnd() + if self.processorCapabilities is not None: + oprot.writeFieldBegin("processorCapabilities", TType.LIST, 9) + oprot.writeListBegin(TType.STRING, len(self.processorCapabilities)) + for iter1113 in self.processorCapabilities: + oprot.writeString(iter1113.encode("utf-8") if sys.version_info[0] == 2 else iter1113) + oprot.writeListEnd() + oprot.writeFieldEnd() + if self.processorIdentifier is not None: + oprot.writeFieldBegin("processorIdentifier", TType.STRING, 10) + oprot.writeString(self.processorIdentifier.encode("utf-8") if sys.version_info[0] == 2 else self.processorIdentifier) + oprot.writeFieldEnd() + oprot.writeFieldStop() + oprot.writeStructEnd() + + def validate(self): + if self.table is None: + raise TProtocolException(message="Required field table is unset!") + return + + def __repr__(self): + L = ["{}={!r}".format(key, value) for key, value in self.__dict__.items()] + return "{}({})".format(self.__class__.__name__, ", ".join(L)) + + def __eq__(self, other): + return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ + + def __ne__(self, other): + return not (self == other) + + +class CreateDatabaseRequest: + """ + Attributes: + - databaseName + - description + - locationUri + - parameters + - privileges + - ownerName + - ownerType + - catalogName + - createTime + - managedLocationUri + - type + - dataConnectorName + + """ + + def __init__( + self, + databaseName=None, + description=None, + locationUri=None, + parameters=None, + privileges=None, + ownerName=None, + ownerType=None, + catalogName=None, + createTime=None, + managedLocationUri=None, + type=None, + dataConnectorName=None, + ): + self.databaseName = databaseName + self.description = description + self.locationUri = locationUri + self.parameters = parameters + self.privileges = privileges + self.ownerName = ownerName + self.ownerType = ownerType + self.catalogName = catalogName + self.createTime = createTime + self.managedLocationUri = managedLocationUri + self.type = type + self.dataConnectorName = dataConnectorName + + def read(self, iprot): + if ( + iprot._fast_decode is not None + and isinstance(iprot.trans, TTransport.CReadableTransport) + and self.thrift_spec is not None + ): + iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec]) + return + iprot.readStructBegin() + while True: + (fname, ftype, fid) = iprot.readFieldBegin() + if ftype == TType.STOP: + break + if fid == 1: + if ftype == TType.STRING: + self.databaseName = ( + iprot.readString().decode("utf-8", errors="replace") if sys.version_info[0] == 2 else iprot.readString() + ) + else: + iprot.skip(ftype) + elif fid == 2: + if ftype == TType.STRING: + self.description = ( + iprot.readString().decode("utf-8", errors="replace") if sys.version_info[0] == 2 else iprot.readString() + ) + else: + iprot.skip(ftype) + elif fid == 3: + if ftype == TType.STRING: + self.locationUri = ( + iprot.readString().decode("utf-8", errors="replace") if sys.version_info[0] == 2 else iprot.readString() + ) + else: + iprot.skip(ftype) + elif fid == 4: + if ftype == TType.MAP: + self.parameters = {} + (_ktype1115, _vtype1116, _size1114) = iprot.readMapBegin() + for _i1118 in range(_size1114): + _key1119 = ( + iprot.readString().decode("utf-8", errors="replace") + if sys.version_info[0] == 2 + else iprot.readString() + ) + _val1120 = ( + iprot.readString().decode("utf-8", errors="replace") + if sys.version_info[0] == 2 + else iprot.readString() + ) + self.parameters[_key1119] = _val1120 + iprot.readMapEnd() + else: + iprot.skip(ftype) + elif fid == 5: + if ftype == TType.STRUCT: + self.privileges = PrincipalPrivilegeSet() + self.privileges.read(iprot) + else: + iprot.skip(ftype) + elif fid == 6: + if ftype == TType.STRING: + self.ownerName = ( + iprot.readString().decode("utf-8", errors="replace") if sys.version_info[0] == 2 else iprot.readString() + ) + else: + iprot.skip(ftype) + elif fid == 7: + if ftype == TType.I32: + self.ownerType = iprot.readI32() + else: + iprot.skip(ftype) + elif fid == 8: + if ftype == TType.STRING: + self.catalogName = ( + iprot.readString().decode("utf-8", errors="replace") if sys.version_info[0] == 2 else iprot.readString() + ) + else: + iprot.skip(ftype) + elif fid == 9: + if ftype == TType.I32: + self.createTime = iprot.readI32() + else: + iprot.skip(ftype) + elif fid == 10: + if ftype == TType.STRING: + self.managedLocationUri = ( + iprot.readString().decode("utf-8", errors="replace") if sys.version_info[0] == 2 else iprot.readString() + ) + else: + iprot.skip(ftype) + elif fid == 11: + if ftype == TType.STRING: + self.type = ( + iprot.readString().decode("utf-8", errors="replace") if sys.version_info[0] == 2 else iprot.readString() + ) + else: + iprot.skip(ftype) + elif fid == 12: + if ftype == TType.STRING: + self.dataConnectorName = ( + iprot.readString().decode("utf-8", errors="replace") if sys.version_info[0] == 2 else iprot.readString() + ) + else: + iprot.skip(ftype) + else: + iprot.skip(ftype) + iprot.readFieldEnd() + iprot.readStructEnd() + + def write(self, oprot): + if oprot._fast_encode is not None and self.thrift_spec is not None: + oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec])) + return + oprot.writeStructBegin("CreateDatabaseRequest") + if self.databaseName is not None: + oprot.writeFieldBegin("databaseName", TType.STRING, 1) + oprot.writeString(self.databaseName.encode("utf-8") if sys.version_info[0] == 2 else self.databaseName) + oprot.writeFieldEnd() + if self.description is not None: + oprot.writeFieldBegin("description", TType.STRING, 2) + oprot.writeString(self.description.encode("utf-8") if sys.version_info[0] == 2 else self.description) + oprot.writeFieldEnd() + if self.locationUri is not None: + oprot.writeFieldBegin("locationUri", TType.STRING, 3) + oprot.writeString(self.locationUri.encode("utf-8") if sys.version_info[0] == 2 else self.locationUri) + oprot.writeFieldEnd() + if self.parameters is not None: + oprot.writeFieldBegin("parameters", TType.MAP, 4) + oprot.writeMapBegin(TType.STRING, TType.STRING, len(self.parameters)) + for kiter1121, viter1122 in self.parameters.items(): + oprot.writeString(kiter1121.encode("utf-8") if sys.version_info[0] == 2 else kiter1121) + oprot.writeString(viter1122.encode("utf-8") if sys.version_info[0] == 2 else viter1122) + oprot.writeMapEnd() + oprot.writeFieldEnd() + if self.privileges is not None: + oprot.writeFieldBegin("privileges", TType.STRUCT, 5) + self.privileges.write(oprot) + oprot.writeFieldEnd() + if self.ownerName is not None: + oprot.writeFieldBegin("ownerName", TType.STRING, 6) + oprot.writeString(self.ownerName.encode("utf-8") if sys.version_info[0] == 2 else self.ownerName) + oprot.writeFieldEnd() + if self.ownerType is not None: + oprot.writeFieldBegin("ownerType", TType.I32, 7) + oprot.writeI32(self.ownerType) + oprot.writeFieldEnd() + if self.catalogName is not None: + oprot.writeFieldBegin("catalogName", TType.STRING, 8) + oprot.writeString(self.catalogName.encode("utf-8") if sys.version_info[0] == 2 else self.catalogName) + oprot.writeFieldEnd() + if self.createTime is not None: + oprot.writeFieldBegin("createTime", TType.I32, 9) + oprot.writeI32(self.createTime) + oprot.writeFieldEnd() + if self.managedLocationUri is not None: + oprot.writeFieldBegin("managedLocationUri", TType.STRING, 10) + oprot.writeString(self.managedLocationUri.encode("utf-8") if sys.version_info[0] == 2 else self.managedLocationUri) + oprot.writeFieldEnd() + if self.type is not None: + oprot.writeFieldBegin("type", TType.STRING, 11) + oprot.writeString(self.type.encode("utf-8") if sys.version_info[0] == 2 else self.type) + oprot.writeFieldEnd() + if self.dataConnectorName is not None: + oprot.writeFieldBegin("dataConnectorName", TType.STRING, 12) + oprot.writeString(self.dataConnectorName.encode("utf-8") if sys.version_info[0] == 2 else self.dataConnectorName) + oprot.writeFieldEnd() + oprot.writeFieldStop() + oprot.writeStructEnd() + + def validate(self): + if self.databaseName is None: + raise TProtocolException(message="Required field databaseName is unset!") + return + + def __repr__(self): + L = ["{}={!r}".format(key, value) for key, value in self.__dict__.items()] + return "{}({})".format(self.__class__.__name__, ", ".join(L)) + + def __eq__(self, other): + return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ + + def __ne__(self, other): + return not (self == other) + + +class CreateDataConnectorRequest: + """ + Attributes: + - connector + + """ + + def __init__( + self, + connector=None, + ): + self.connector = connector + + def read(self, iprot): + if ( + iprot._fast_decode is not None + and isinstance(iprot.trans, TTransport.CReadableTransport) + and self.thrift_spec is not None + ): + iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec]) + return + iprot.readStructBegin() + while True: + (fname, ftype, fid) = iprot.readFieldBegin() + if ftype == TType.STOP: + break + if fid == 1: + if ftype == TType.STRUCT: + self.connector = DataConnector() + self.connector.read(iprot) + else: + iprot.skip(ftype) + else: + iprot.skip(ftype) + iprot.readFieldEnd() + iprot.readStructEnd() + + def write(self, oprot): + if oprot._fast_encode is not None and self.thrift_spec is not None: + oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec])) + return + oprot.writeStructBegin("CreateDataConnectorRequest") + if self.connector is not None: + oprot.writeFieldBegin("connector", TType.STRUCT, 1) + self.connector.write(oprot) + oprot.writeFieldEnd() + oprot.writeFieldStop() + oprot.writeStructEnd() + + def validate(self): + return + + def __repr__(self): + L = ["{}={!r}".format(key, value) for key, value in self.__dict__.items()] + return "{}({})".format(self.__class__.__name__, ", ".join(L)) + + def __eq__(self, other): + return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ + + def __ne__(self, other): + return not (self == other) + + +class GetDataConnectorRequest: + """ + Attributes: + - connectorName + + """ + + def __init__( + self, + connectorName=None, + ): + self.connectorName = connectorName + + def read(self, iprot): + if ( + iprot._fast_decode is not None + and isinstance(iprot.trans, TTransport.CReadableTransport) + and self.thrift_spec is not None + ): + iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec]) + return + iprot.readStructBegin() + while True: + (fname, ftype, fid) = iprot.readFieldBegin() + if ftype == TType.STOP: + break + if fid == 1: + if ftype == TType.STRING: + self.connectorName = ( + iprot.readString().decode("utf-8", errors="replace") if sys.version_info[0] == 2 else iprot.readString() + ) + else: + iprot.skip(ftype) + else: + iprot.skip(ftype) + iprot.readFieldEnd() + iprot.readStructEnd() + + def write(self, oprot): + if oprot._fast_encode is not None and self.thrift_spec is not None: + oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec])) + return + oprot.writeStructBegin("GetDataConnectorRequest") + if self.connectorName is not None: + oprot.writeFieldBegin("connectorName", TType.STRING, 1) + oprot.writeString(self.connectorName.encode("utf-8") if sys.version_info[0] == 2 else self.connectorName) + oprot.writeFieldEnd() + oprot.writeFieldStop() + oprot.writeStructEnd() + + def validate(self): + if self.connectorName is None: + raise TProtocolException(message="Required field connectorName is unset!") + return + + def __repr__(self): + L = ["{}={!r}".format(key, value) for key, value in self.__dict__.items()] + return "{}({})".format(self.__class__.__name__, ", ".join(L)) + + def __eq__(self, other): + return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ + + def __ne__(self, other): + return not (self == other) + + +class ScheduledQueryPollRequest: + """ + Attributes: + - clusterNamespace + + """ + + def __init__( + self, + clusterNamespace=None, + ): + self.clusterNamespace = clusterNamespace + + def read(self, iprot): + if ( + iprot._fast_decode is not None + and isinstance(iprot.trans, TTransport.CReadableTransport) + and self.thrift_spec is not None + ): + iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec]) + return + iprot.readStructBegin() + while True: + (fname, ftype, fid) = iprot.readFieldBegin() + if ftype == TType.STOP: + break + if fid == 1: + if ftype == TType.STRING: + self.clusterNamespace = ( + iprot.readString().decode("utf-8", errors="replace") if sys.version_info[0] == 2 else iprot.readString() + ) + else: + iprot.skip(ftype) + else: + iprot.skip(ftype) + iprot.readFieldEnd() + iprot.readStructEnd() + + def write(self, oprot): + if oprot._fast_encode is not None and self.thrift_spec is not None: + oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec])) + return + oprot.writeStructBegin("ScheduledQueryPollRequest") + if self.clusterNamespace is not None: + oprot.writeFieldBegin("clusterNamespace", TType.STRING, 1) + oprot.writeString(self.clusterNamespace.encode("utf-8") if sys.version_info[0] == 2 else self.clusterNamespace) + oprot.writeFieldEnd() + oprot.writeFieldStop() + oprot.writeStructEnd() + + def validate(self): + if self.clusterNamespace is None: + raise TProtocolException(message="Required field clusterNamespace is unset!") + return + + def __repr__(self): + L = ["{}={!r}".format(key, value) for key, value in self.__dict__.items()] + return "{}({})".format(self.__class__.__name__, ", ".join(L)) + + def __eq__(self, other): + return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ + + def __ne__(self, other): + return not (self == other) + + +class ScheduledQueryKey: + """ + Attributes: + - scheduleName + - clusterNamespace + + """ + + def __init__( + self, + scheduleName=None, + clusterNamespace=None, + ): + self.scheduleName = scheduleName + self.clusterNamespace = clusterNamespace + + def read(self, iprot): + if ( + iprot._fast_decode is not None + and isinstance(iprot.trans, TTransport.CReadableTransport) + and self.thrift_spec is not None + ): + iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec]) + return + iprot.readStructBegin() + while True: + (fname, ftype, fid) = iprot.readFieldBegin() + if ftype == TType.STOP: + break + if fid == 1: + if ftype == TType.STRING: + self.scheduleName = ( + iprot.readString().decode("utf-8", errors="replace") if sys.version_info[0] == 2 else iprot.readString() + ) + else: + iprot.skip(ftype) + elif fid == 2: + if ftype == TType.STRING: + self.clusterNamespace = ( + iprot.readString().decode("utf-8", errors="replace") if sys.version_info[0] == 2 else iprot.readString() + ) + else: + iprot.skip(ftype) + else: + iprot.skip(ftype) + iprot.readFieldEnd() + iprot.readStructEnd() + + def write(self, oprot): + if oprot._fast_encode is not None and self.thrift_spec is not None: + oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec])) + return + oprot.writeStructBegin("ScheduledQueryKey") + if self.scheduleName is not None: + oprot.writeFieldBegin("scheduleName", TType.STRING, 1) + oprot.writeString(self.scheduleName.encode("utf-8") if sys.version_info[0] == 2 else self.scheduleName) + oprot.writeFieldEnd() + if self.clusterNamespace is not None: + oprot.writeFieldBegin("clusterNamespace", TType.STRING, 2) + oprot.writeString(self.clusterNamespace.encode("utf-8") if sys.version_info[0] == 2 else self.clusterNamespace) + oprot.writeFieldEnd() + oprot.writeFieldStop() + oprot.writeStructEnd() + + def validate(self): + if self.scheduleName is None: + raise TProtocolException(message="Required field scheduleName is unset!") + if self.clusterNamespace is None: + raise TProtocolException(message="Required field clusterNamespace is unset!") + return + + def __repr__(self): + L = ["{}={!r}".format(key, value) for key, value in self.__dict__.items()] + return "{}({})".format(self.__class__.__name__, ", ".join(L)) + + def __eq__(self, other): + return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ + + def __ne__(self, other): + return not (self == other) + + +class ScheduledQueryPollResponse: + """ + Attributes: + - scheduleKey + - executionId + - query + - user + + """ + + def __init__( + self, + scheduleKey=None, + executionId=None, + query=None, + user=None, + ): + self.scheduleKey = scheduleKey + self.executionId = executionId + self.query = query + self.user = user + + def read(self, iprot): + if ( + iprot._fast_decode is not None + and isinstance(iprot.trans, TTransport.CReadableTransport) + and self.thrift_spec is not None + ): + iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec]) + return + iprot.readStructBegin() + while True: + (fname, ftype, fid) = iprot.readFieldBegin() + if ftype == TType.STOP: + break + if fid == 1: + if ftype == TType.STRUCT: + self.scheduleKey = ScheduledQueryKey() + self.scheduleKey.read(iprot) + else: + iprot.skip(ftype) + elif fid == 2: + if ftype == TType.I64: + self.executionId = iprot.readI64() + else: + iprot.skip(ftype) + elif fid == 3: + if ftype == TType.STRING: + self.query = ( + iprot.readString().decode("utf-8", errors="replace") if sys.version_info[0] == 2 else iprot.readString() + ) + else: + iprot.skip(ftype) + elif fid == 4: + if ftype == TType.STRING: + self.user = ( + iprot.readString().decode("utf-8", errors="replace") if sys.version_info[0] == 2 else iprot.readString() + ) + else: + iprot.skip(ftype) + else: + iprot.skip(ftype) + iprot.readFieldEnd() + iprot.readStructEnd() + + def write(self, oprot): + if oprot._fast_encode is not None and self.thrift_spec is not None: + oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec])) + return + oprot.writeStructBegin("ScheduledQueryPollResponse") + if self.scheduleKey is not None: + oprot.writeFieldBegin("scheduleKey", TType.STRUCT, 1) + self.scheduleKey.write(oprot) + oprot.writeFieldEnd() + if self.executionId is not None: + oprot.writeFieldBegin("executionId", TType.I64, 2) + oprot.writeI64(self.executionId) + oprot.writeFieldEnd() + if self.query is not None: + oprot.writeFieldBegin("query", TType.STRING, 3) + oprot.writeString(self.query.encode("utf-8") if sys.version_info[0] == 2 else self.query) + oprot.writeFieldEnd() + if self.user is not None: + oprot.writeFieldBegin("user", TType.STRING, 4) + oprot.writeString(self.user.encode("utf-8") if sys.version_info[0] == 2 else self.user) + oprot.writeFieldEnd() + oprot.writeFieldStop() + oprot.writeStructEnd() + + def validate(self): + return + + def __repr__(self): + L = ["{}={!r}".format(key, value) for key, value in self.__dict__.items()] + return "{}({})".format(self.__class__.__name__, ", ".join(L)) + + def __eq__(self, other): + return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ + + def __ne__(self, other): + return not (self == other) + + +class ScheduledQuery: + """ + Attributes: + - scheduleKey + - enabled + - schedule + - user + - query + - nextExecution + + """ + + def __init__( + self, + scheduleKey=None, + enabled=None, + schedule=None, + user=None, + query=None, + nextExecution=None, + ): + self.scheduleKey = scheduleKey + self.enabled = enabled + self.schedule = schedule + self.user = user + self.query = query + self.nextExecution = nextExecution + + def read(self, iprot): + if ( + iprot._fast_decode is not None + and isinstance(iprot.trans, TTransport.CReadableTransport) + and self.thrift_spec is not None + ): + iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec]) + return + iprot.readStructBegin() + while True: + (fname, ftype, fid) = iprot.readFieldBegin() + if ftype == TType.STOP: + break + if fid == 1: + if ftype == TType.STRUCT: + self.scheduleKey = ScheduledQueryKey() + self.scheduleKey.read(iprot) + else: + iprot.skip(ftype) + elif fid == 2: + if ftype == TType.BOOL: + self.enabled = iprot.readBool() + else: + iprot.skip(ftype) + elif fid == 4: + if ftype == TType.STRING: + self.schedule = ( + iprot.readString().decode("utf-8", errors="replace") if sys.version_info[0] == 2 else iprot.readString() + ) + else: + iprot.skip(ftype) + elif fid == 5: + if ftype == TType.STRING: + self.user = ( + iprot.readString().decode("utf-8", errors="replace") if sys.version_info[0] == 2 else iprot.readString() + ) + else: + iprot.skip(ftype) + elif fid == 6: + if ftype == TType.STRING: + self.query = ( + iprot.readString().decode("utf-8", errors="replace") if sys.version_info[0] == 2 else iprot.readString() + ) + else: + iprot.skip(ftype) + elif fid == 7: + if ftype == TType.I32: + self.nextExecution = iprot.readI32() + else: + iprot.skip(ftype) + else: + iprot.skip(ftype) + iprot.readFieldEnd() + iprot.readStructEnd() + + def write(self, oprot): + if oprot._fast_encode is not None and self.thrift_spec is not None: + oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec])) + return + oprot.writeStructBegin("ScheduledQuery") + if self.scheduleKey is not None: + oprot.writeFieldBegin("scheduleKey", TType.STRUCT, 1) + self.scheduleKey.write(oprot) + oprot.writeFieldEnd() + if self.enabled is not None: + oprot.writeFieldBegin("enabled", TType.BOOL, 2) + oprot.writeBool(self.enabled) + oprot.writeFieldEnd() + if self.schedule is not None: + oprot.writeFieldBegin("schedule", TType.STRING, 4) + oprot.writeString(self.schedule.encode("utf-8") if sys.version_info[0] == 2 else self.schedule) + oprot.writeFieldEnd() + if self.user is not None: + oprot.writeFieldBegin("user", TType.STRING, 5) + oprot.writeString(self.user.encode("utf-8") if sys.version_info[0] == 2 else self.user) + oprot.writeFieldEnd() + if self.query is not None: + oprot.writeFieldBegin("query", TType.STRING, 6) + oprot.writeString(self.query.encode("utf-8") if sys.version_info[0] == 2 else self.query) + oprot.writeFieldEnd() + if self.nextExecution is not None: + oprot.writeFieldBegin("nextExecution", TType.I32, 7) + oprot.writeI32(self.nextExecution) + oprot.writeFieldEnd() + oprot.writeFieldStop() + oprot.writeStructEnd() + + def validate(self): + if self.scheduleKey is None: + raise TProtocolException(message="Required field scheduleKey is unset!") + return + + def __repr__(self): + L = ["{}={!r}".format(key, value) for key, value in self.__dict__.items()] + return "{}({})".format(self.__class__.__name__, ", ".join(L)) + + def __eq__(self, other): + return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ + + def __ne__(self, other): + return not (self == other) + + +class ScheduledQueryMaintenanceRequest: + """ + Attributes: + - type + - scheduledQuery + + """ + + def __init__( + self, + type=None, + scheduledQuery=None, + ): + self.type = type + self.scheduledQuery = scheduledQuery + + def read(self, iprot): + if ( + iprot._fast_decode is not None + and isinstance(iprot.trans, TTransport.CReadableTransport) + and self.thrift_spec is not None + ): + iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec]) + return + iprot.readStructBegin() + while True: + (fname, ftype, fid) = iprot.readFieldBegin() + if ftype == TType.STOP: + break + if fid == 1: + if ftype == TType.I32: + self.type = iprot.readI32() + else: + iprot.skip(ftype) + elif fid == 2: + if ftype == TType.STRUCT: + self.scheduledQuery = ScheduledQuery() + self.scheduledQuery.read(iprot) + else: + iprot.skip(ftype) + else: + iprot.skip(ftype) + iprot.readFieldEnd() + iprot.readStructEnd() + + def write(self, oprot): + if oprot._fast_encode is not None and self.thrift_spec is not None: + oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec])) + return + oprot.writeStructBegin("ScheduledQueryMaintenanceRequest") + if self.type is not None: + oprot.writeFieldBegin("type", TType.I32, 1) + oprot.writeI32(self.type) + oprot.writeFieldEnd() + if self.scheduledQuery is not None: + oprot.writeFieldBegin("scheduledQuery", TType.STRUCT, 2) + self.scheduledQuery.write(oprot) + oprot.writeFieldEnd() + oprot.writeFieldStop() + oprot.writeStructEnd() + + def validate(self): + if self.type is None: + raise TProtocolException(message="Required field type is unset!") + if self.scheduledQuery is None: + raise TProtocolException(message="Required field scheduledQuery is unset!") + return + + def __repr__(self): + L = ["{}={!r}".format(key, value) for key, value in self.__dict__.items()] + return "{}({})".format(self.__class__.__name__, ", ".join(L)) + + def __eq__(self, other): + return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ + + def __ne__(self, other): + return not (self == other) + + +class ScheduledQueryProgressInfo: + """ + Attributes: + - scheduledExecutionId + - state + - executorQueryId + - errorMessage + + """ + + def __init__( + self, + scheduledExecutionId=None, + state=None, + executorQueryId=None, + errorMessage=None, + ): + self.scheduledExecutionId = scheduledExecutionId + self.state = state + self.executorQueryId = executorQueryId + self.errorMessage = errorMessage + + def read(self, iprot): + if ( + iprot._fast_decode is not None + and isinstance(iprot.trans, TTransport.CReadableTransport) + and self.thrift_spec is not None + ): + iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec]) + return + iprot.readStructBegin() + while True: + (fname, ftype, fid) = iprot.readFieldBegin() + if ftype == TType.STOP: + break + if fid == 1: + if ftype == TType.I64: + self.scheduledExecutionId = iprot.readI64() + else: + iprot.skip(ftype) + elif fid == 2: + if ftype == TType.I32: + self.state = iprot.readI32() + else: + iprot.skip(ftype) + elif fid == 3: + if ftype == TType.STRING: + self.executorQueryId = ( + iprot.readString().decode("utf-8", errors="replace") if sys.version_info[0] == 2 else iprot.readString() + ) + else: + iprot.skip(ftype) + elif fid == 4: + if ftype == TType.STRING: + self.errorMessage = ( + iprot.readString().decode("utf-8", errors="replace") if sys.version_info[0] == 2 else iprot.readString() + ) + else: + iprot.skip(ftype) + else: + iprot.skip(ftype) + iprot.readFieldEnd() + iprot.readStructEnd() + + def write(self, oprot): + if oprot._fast_encode is not None and self.thrift_spec is not None: + oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec])) + return + oprot.writeStructBegin("ScheduledQueryProgressInfo") + if self.scheduledExecutionId is not None: + oprot.writeFieldBegin("scheduledExecutionId", TType.I64, 1) + oprot.writeI64(self.scheduledExecutionId) + oprot.writeFieldEnd() + if self.state is not None: + oprot.writeFieldBegin("state", TType.I32, 2) + oprot.writeI32(self.state) + oprot.writeFieldEnd() + if self.executorQueryId is not None: + oprot.writeFieldBegin("executorQueryId", TType.STRING, 3) + oprot.writeString(self.executorQueryId.encode("utf-8") if sys.version_info[0] == 2 else self.executorQueryId) + oprot.writeFieldEnd() + if self.errorMessage is not None: + oprot.writeFieldBegin("errorMessage", TType.STRING, 4) + oprot.writeString(self.errorMessage.encode("utf-8") if sys.version_info[0] == 2 else self.errorMessage) + oprot.writeFieldEnd() + oprot.writeFieldStop() + oprot.writeStructEnd() + + def validate(self): + if self.scheduledExecutionId is None: + raise TProtocolException(message="Required field scheduledExecutionId is unset!") + if self.state is None: + raise TProtocolException(message="Required field state is unset!") + if self.executorQueryId is None: + raise TProtocolException(message="Required field executorQueryId is unset!") + return + + def __repr__(self): + L = ["{}={!r}".format(key, value) for key, value in self.__dict__.items()] + return "{}({})".format(self.__class__.__name__, ", ".join(L)) + + def __eq__(self, other): + return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ + + def __ne__(self, other): + return not (self == other) + + +class AlterPartitionsRequest: + """ + Attributes: + - catName + - dbName + - tableName + - partitions + - environmentContext + - writeId + - validWriteIdList + + """ + + def __init__( + self, + catName=None, + dbName=None, + tableName=None, + partitions=None, + environmentContext=None, + writeId=-1, + validWriteIdList=None, + ): + self.catName = catName + self.dbName = dbName + self.tableName = tableName + self.partitions = partitions + self.environmentContext = environmentContext + self.writeId = writeId + self.validWriteIdList = validWriteIdList + + def read(self, iprot): + if ( + iprot._fast_decode is not None + and isinstance(iprot.trans, TTransport.CReadableTransport) + and self.thrift_spec is not None + ): + iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec]) + return + iprot.readStructBegin() + while True: + (fname, ftype, fid) = iprot.readFieldBegin() + if ftype == TType.STOP: + break + if fid == 1: + if ftype == TType.STRING: + self.catName = ( + iprot.readString().decode("utf-8", errors="replace") if sys.version_info[0] == 2 else iprot.readString() + ) + else: + iprot.skip(ftype) + elif fid == 2: + if ftype == TType.STRING: + self.dbName = ( + iprot.readString().decode("utf-8", errors="replace") if sys.version_info[0] == 2 else iprot.readString() + ) + else: + iprot.skip(ftype) + elif fid == 3: + if ftype == TType.STRING: + self.tableName = ( + iprot.readString().decode("utf-8", errors="replace") if sys.version_info[0] == 2 else iprot.readString() + ) + else: + iprot.skip(ftype) + elif fid == 4: + if ftype == TType.LIST: + self.partitions = [] + (_etype1126, _size1123) = iprot.readListBegin() + for _i1127 in range(_size1123): + _elem1128 = Partition() + _elem1128.read(iprot) + self.partitions.append(_elem1128) + iprot.readListEnd() + else: + iprot.skip(ftype) + elif fid == 5: + if ftype == TType.STRUCT: + self.environmentContext = EnvironmentContext() + self.environmentContext.read(iprot) + else: + iprot.skip(ftype) + elif fid == 6: + if ftype == TType.I64: + self.writeId = iprot.readI64() + else: + iprot.skip(ftype) + elif fid == 7: + if ftype == TType.STRING: + self.validWriteIdList = ( + iprot.readString().decode("utf-8", errors="replace") if sys.version_info[0] == 2 else iprot.readString() + ) + else: + iprot.skip(ftype) + else: + iprot.skip(ftype) + iprot.readFieldEnd() + iprot.readStructEnd() + + def write(self, oprot): + if oprot._fast_encode is not None and self.thrift_spec is not None: + oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec])) + return + oprot.writeStructBegin("AlterPartitionsRequest") + if self.catName is not None: + oprot.writeFieldBegin("catName", TType.STRING, 1) + oprot.writeString(self.catName.encode("utf-8") if sys.version_info[0] == 2 else self.catName) + oprot.writeFieldEnd() + if self.dbName is not None: + oprot.writeFieldBegin("dbName", TType.STRING, 2) + oprot.writeString(self.dbName.encode("utf-8") if sys.version_info[0] == 2 else self.dbName) + oprot.writeFieldEnd() + if self.tableName is not None: + oprot.writeFieldBegin("tableName", TType.STRING, 3) + oprot.writeString(self.tableName.encode("utf-8") if sys.version_info[0] == 2 else self.tableName) + oprot.writeFieldEnd() + if self.partitions is not None: + oprot.writeFieldBegin("partitions", TType.LIST, 4) + oprot.writeListBegin(TType.STRUCT, len(self.partitions)) + for iter1129 in self.partitions: + iter1129.write(oprot) + oprot.writeListEnd() + oprot.writeFieldEnd() + if self.environmentContext is not None: + oprot.writeFieldBegin("environmentContext", TType.STRUCT, 5) + self.environmentContext.write(oprot) + oprot.writeFieldEnd() + if self.writeId is not None: + oprot.writeFieldBegin("writeId", TType.I64, 6) + oprot.writeI64(self.writeId) + oprot.writeFieldEnd() + if self.validWriteIdList is not None: + oprot.writeFieldBegin("validWriteIdList", TType.STRING, 7) + oprot.writeString(self.validWriteIdList.encode("utf-8") if sys.version_info[0] == 2 else self.validWriteIdList) + oprot.writeFieldEnd() + oprot.writeFieldStop() + oprot.writeStructEnd() + + def validate(self): + if self.dbName is None: + raise TProtocolException(message="Required field dbName is unset!") + if self.tableName is None: + raise TProtocolException(message="Required field tableName is unset!") + if self.partitions is None: + raise TProtocolException(message="Required field partitions is unset!") + return + + def __repr__(self): + L = ["{}={!r}".format(key, value) for key, value in self.__dict__.items()] + return "{}({})".format(self.__class__.__name__, ", ".join(L)) + + def __eq__(self, other): + return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ + + def __ne__(self, other): + return not (self == other) + + +class AlterPartitionsResponse: + def read(self, iprot): + if ( + iprot._fast_decode is not None + and isinstance(iprot.trans, TTransport.CReadableTransport) + and self.thrift_spec is not None + ): + iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec]) + return + iprot.readStructBegin() + while True: + (fname, ftype, fid) = iprot.readFieldBegin() + if ftype == TType.STOP: + break + else: + iprot.skip(ftype) + iprot.readFieldEnd() + iprot.readStructEnd() + + def write(self, oprot): + if oprot._fast_encode is not None and self.thrift_spec is not None: + oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec])) + return + oprot.writeStructBegin("AlterPartitionsResponse") + oprot.writeFieldStop() + oprot.writeStructEnd() + + def validate(self): + return + + def __repr__(self): + L = ["{}={!r}".format(key, value) for key, value in self.__dict__.items()] + return "{}({})".format(self.__class__.__name__, ", ".join(L)) + + def __eq__(self, other): + return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ + + def __ne__(self, other): + return not (self == other) + + +class RenamePartitionRequest: + """ + Attributes: + - catName + - dbName + - tableName + - partVals + - newPart + - validWriteIdList + - txnId + - clonePart + + """ + + def __init__( + self, + catName=None, + dbName=None, + tableName=None, + partVals=None, + newPart=None, + validWriteIdList=None, + txnId=None, + clonePart=None, + ): + self.catName = catName + self.dbName = dbName + self.tableName = tableName + self.partVals = partVals + self.newPart = newPart + self.validWriteIdList = validWriteIdList + self.txnId = txnId + self.clonePart = clonePart + + def read(self, iprot): + if ( + iprot._fast_decode is not None + and isinstance(iprot.trans, TTransport.CReadableTransport) + and self.thrift_spec is not None + ): + iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec]) + return + iprot.readStructBegin() + while True: + (fname, ftype, fid) = iprot.readFieldBegin() + if ftype == TType.STOP: + break + if fid == 1: + if ftype == TType.STRING: + self.catName = ( + iprot.readString().decode("utf-8", errors="replace") if sys.version_info[0] == 2 else iprot.readString() + ) + else: + iprot.skip(ftype) + elif fid == 2: + if ftype == TType.STRING: + self.dbName = ( + iprot.readString().decode("utf-8", errors="replace") if sys.version_info[0] == 2 else iprot.readString() + ) + else: + iprot.skip(ftype) + elif fid == 3: + if ftype == TType.STRING: + self.tableName = ( + iprot.readString().decode("utf-8", errors="replace") if sys.version_info[0] == 2 else iprot.readString() + ) + else: + iprot.skip(ftype) + elif fid == 4: + if ftype == TType.LIST: + self.partVals = [] + (_etype1133, _size1130) = iprot.readListBegin() + for _i1134 in range(_size1130): + _elem1135 = ( + iprot.readString().decode("utf-8", errors="replace") + if sys.version_info[0] == 2 + else iprot.readString() + ) + self.partVals.append(_elem1135) + iprot.readListEnd() + else: + iprot.skip(ftype) + elif fid == 5: + if ftype == TType.STRUCT: + self.newPart = Partition() + self.newPart.read(iprot) + else: + iprot.skip(ftype) + elif fid == 6: + if ftype == TType.STRING: + self.validWriteIdList = ( + iprot.readString().decode("utf-8", errors="replace") if sys.version_info[0] == 2 else iprot.readString() + ) + else: + iprot.skip(ftype) + elif fid == 7: + if ftype == TType.I64: + self.txnId = iprot.readI64() + else: + iprot.skip(ftype) + elif fid == 8: + if ftype == TType.BOOL: + self.clonePart = iprot.readBool() + else: + iprot.skip(ftype) + else: + iprot.skip(ftype) + iprot.readFieldEnd() + iprot.readStructEnd() + + def write(self, oprot): + if oprot._fast_encode is not None and self.thrift_spec is not None: + oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec])) + return + oprot.writeStructBegin("RenamePartitionRequest") + if self.catName is not None: + oprot.writeFieldBegin("catName", TType.STRING, 1) + oprot.writeString(self.catName.encode("utf-8") if sys.version_info[0] == 2 else self.catName) + oprot.writeFieldEnd() + if self.dbName is not None: + oprot.writeFieldBegin("dbName", TType.STRING, 2) + oprot.writeString(self.dbName.encode("utf-8") if sys.version_info[0] == 2 else self.dbName) + oprot.writeFieldEnd() + if self.tableName is not None: + oprot.writeFieldBegin("tableName", TType.STRING, 3) + oprot.writeString(self.tableName.encode("utf-8") if sys.version_info[0] == 2 else self.tableName) + oprot.writeFieldEnd() + if self.partVals is not None: + oprot.writeFieldBegin("partVals", TType.LIST, 4) + oprot.writeListBegin(TType.STRING, len(self.partVals)) + for iter1136 in self.partVals: + oprot.writeString(iter1136.encode("utf-8") if sys.version_info[0] == 2 else iter1136) + oprot.writeListEnd() + oprot.writeFieldEnd() + if self.newPart is not None: + oprot.writeFieldBegin("newPart", TType.STRUCT, 5) + self.newPart.write(oprot) + oprot.writeFieldEnd() + if self.validWriteIdList is not None: + oprot.writeFieldBegin("validWriteIdList", TType.STRING, 6) + oprot.writeString(self.validWriteIdList.encode("utf-8") if sys.version_info[0] == 2 else self.validWriteIdList) + oprot.writeFieldEnd() + if self.txnId is not None: + oprot.writeFieldBegin("txnId", TType.I64, 7) + oprot.writeI64(self.txnId) + oprot.writeFieldEnd() + if self.clonePart is not None: + oprot.writeFieldBegin("clonePart", TType.BOOL, 8) + oprot.writeBool(self.clonePart) + oprot.writeFieldEnd() + oprot.writeFieldStop() + oprot.writeStructEnd() + + def validate(self): + if self.dbName is None: + raise TProtocolException(message="Required field dbName is unset!") + if self.tableName is None: + raise TProtocolException(message="Required field tableName is unset!") + if self.partVals is None: + raise TProtocolException(message="Required field partVals is unset!") + if self.newPart is None: + raise TProtocolException(message="Required field newPart is unset!") + return + + def __repr__(self): + L = ["{}={!r}".format(key, value) for key, value in self.__dict__.items()] + return "{}({})".format(self.__class__.__name__, ", ".join(L)) + + def __eq__(self, other): + return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ + + def __ne__(self, other): + return not (self == other) + + +class RenamePartitionResponse: + def read(self, iprot): + if ( + iprot._fast_decode is not None + and isinstance(iprot.trans, TTransport.CReadableTransport) + and self.thrift_spec is not None + ): + iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec]) + return + iprot.readStructBegin() + while True: + (fname, ftype, fid) = iprot.readFieldBegin() + if ftype == TType.STOP: + break + else: + iprot.skip(ftype) + iprot.readFieldEnd() + iprot.readStructEnd() + + def write(self, oprot): + if oprot._fast_encode is not None and self.thrift_spec is not None: + oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec])) + return + oprot.writeStructBegin("RenamePartitionResponse") + oprot.writeFieldStop() + oprot.writeStructEnd() + + def validate(self): + return + + def __repr__(self): + L = ["{}={!r}".format(key, value) for key, value in self.__dict__.items()] + return "{}({})".format(self.__class__.__name__, ", ".join(L)) + + def __eq__(self, other): + return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ + + def __ne__(self, other): + return not (self == other) + + +class AlterTableRequest: + """ + Attributes: + - catName + - dbName + - tableName + - table + - environmentContext + - writeId + - validWriteIdList + - processorCapabilities + - processorIdentifier + + """ + + def __init__( + self, + catName=None, + dbName=None, + tableName=None, + table=None, + environmentContext=None, + writeId=-1, + validWriteIdList=None, + processorCapabilities=None, + processorIdentifier=None, + ): + self.catName = catName + self.dbName = dbName + self.tableName = tableName + self.table = table + self.environmentContext = environmentContext + self.writeId = writeId + self.validWriteIdList = validWriteIdList + self.processorCapabilities = processorCapabilities + self.processorIdentifier = processorIdentifier + + def read(self, iprot): + if ( + iprot._fast_decode is not None + and isinstance(iprot.trans, TTransport.CReadableTransport) + and self.thrift_spec is not None + ): + iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec]) + return + iprot.readStructBegin() + while True: + (fname, ftype, fid) = iprot.readFieldBegin() + if ftype == TType.STOP: + break + if fid == 1: + if ftype == TType.STRING: + self.catName = ( + iprot.readString().decode("utf-8", errors="replace") if sys.version_info[0] == 2 else iprot.readString() + ) + else: + iprot.skip(ftype) + elif fid == 2: + if ftype == TType.STRING: + self.dbName = ( + iprot.readString().decode("utf-8", errors="replace") if sys.version_info[0] == 2 else iprot.readString() + ) + else: + iprot.skip(ftype) + elif fid == 3: + if ftype == TType.STRING: + self.tableName = ( + iprot.readString().decode("utf-8", errors="replace") if sys.version_info[0] == 2 else iprot.readString() + ) + else: + iprot.skip(ftype) + elif fid == 4: + if ftype == TType.STRUCT: + self.table = Table() + self.table.read(iprot) + else: + iprot.skip(ftype) + elif fid == 5: + if ftype == TType.STRUCT: + self.environmentContext = EnvironmentContext() + self.environmentContext.read(iprot) + else: + iprot.skip(ftype) + elif fid == 6: + if ftype == TType.I64: + self.writeId = iprot.readI64() + else: + iprot.skip(ftype) + elif fid == 7: + if ftype == TType.STRING: + self.validWriteIdList = ( + iprot.readString().decode("utf-8", errors="replace") if sys.version_info[0] == 2 else iprot.readString() + ) + else: + iprot.skip(ftype) + elif fid == 8: + if ftype == TType.LIST: + self.processorCapabilities = [] + (_etype1140, _size1137) = iprot.readListBegin() + for _i1141 in range(_size1137): + _elem1142 = ( + iprot.readString().decode("utf-8", errors="replace") + if sys.version_info[0] == 2 + else iprot.readString() + ) + self.processorCapabilities.append(_elem1142) + iprot.readListEnd() + else: + iprot.skip(ftype) + elif fid == 9: + if ftype == TType.STRING: + self.processorIdentifier = ( + iprot.readString().decode("utf-8", errors="replace") if sys.version_info[0] == 2 else iprot.readString() + ) + else: + iprot.skip(ftype) + else: + iprot.skip(ftype) + iprot.readFieldEnd() + iprot.readStructEnd() + + def write(self, oprot): + if oprot._fast_encode is not None and self.thrift_spec is not None: + oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec])) + return + oprot.writeStructBegin("AlterTableRequest") + if self.catName is not None: + oprot.writeFieldBegin("catName", TType.STRING, 1) + oprot.writeString(self.catName.encode("utf-8") if sys.version_info[0] == 2 else self.catName) + oprot.writeFieldEnd() + if self.dbName is not None: + oprot.writeFieldBegin("dbName", TType.STRING, 2) + oprot.writeString(self.dbName.encode("utf-8") if sys.version_info[0] == 2 else self.dbName) + oprot.writeFieldEnd() + if self.tableName is not None: + oprot.writeFieldBegin("tableName", TType.STRING, 3) + oprot.writeString(self.tableName.encode("utf-8") if sys.version_info[0] == 2 else self.tableName) + oprot.writeFieldEnd() + if self.table is not None: + oprot.writeFieldBegin("table", TType.STRUCT, 4) + self.table.write(oprot) + oprot.writeFieldEnd() + if self.environmentContext is not None: + oprot.writeFieldBegin("environmentContext", TType.STRUCT, 5) + self.environmentContext.write(oprot) + oprot.writeFieldEnd() + if self.writeId is not None: + oprot.writeFieldBegin("writeId", TType.I64, 6) + oprot.writeI64(self.writeId) + oprot.writeFieldEnd() + if self.validWriteIdList is not None: + oprot.writeFieldBegin("validWriteIdList", TType.STRING, 7) + oprot.writeString(self.validWriteIdList.encode("utf-8") if sys.version_info[0] == 2 else self.validWriteIdList) + oprot.writeFieldEnd() + if self.processorCapabilities is not None: + oprot.writeFieldBegin("processorCapabilities", TType.LIST, 8) + oprot.writeListBegin(TType.STRING, len(self.processorCapabilities)) + for iter1143 in self.processorCapabilities: + oprot.writeString(iter1143.encode("utf-8") if sys.version_info[0] == 2 else iter1143) + oprot.writeListEnd() + oprot.writeFieldEnd() + if self.processorIdentifier is not None: + oprot.writeFieldBegin("processorIdentifier", TType.STRING, 9) + oprot.writeString(self.processorIdentifier.encode("utf-8") if sys.version_info[0] == 2 else self.processorIdentifier) + oprot.writeFieldEnd() + oprot.writeFieldStop() + oprot.writeStructEnd() + + def validate(self): + if self.dbName is None: + raise TProtocolException(message="Required field dbName is unset!") + if self.tableName is None: + raise TProtocolException(message="Required field tableName is unset!") + if self.table is None: + raise TProtocolException(message="Required field table is unset!") + return + + def __repr__(self): + L = ["{}={!r}".format(key, value) for key, value in self.__dict__.items()] + return "{}({})".format(self.__class__.__name__, ", ".join(L)) + + def __eq__(self, other): + return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ + + def __ne__(self, other): + return not (self == other) + + +class AlterTableResponse: + def read(self, iprot): + if ( + iprot._fast_decode is not None + and isinstance(iprot.trans, TTransport.CReadableTransport) + and self.thrift_spec is not None + ): + iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec]) + return + iprot.readStructBegin() + while True: + (fname, ftype, fid) = iprot.readFieldBegin() + if ftype == TType.STOP: + break + else: + iprot.skip(ftype) + iprot.readFieldEnd() + iprot.readStructEnd() + + def write(self, oprot): + if oprot._fast_encode is not None and self.thrift_spec is not None: + oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec])) + return + oprot.writeStructBegin("AlterTableResponse") + oprot.writeFieldStop() + oprot.writeStructEnd() + + def validate(self): + return + + def __repr__(self): + L = ["{}={!r}".format(key, value) for key, value in self.__dict__.items()] + return "{}({})".format(self.__class__.__name__, ", ".join(L)) + + def __eq__(self, other): + return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ + + def __ne__(self, other): + return not (self == other) + + +class GetPartitionsFilterSpec: + """ + Attributes: + - filterMode + - filters + + """ + + def __init__( + self, + filterMode=None, + filters=None, + ): + self.filterMode = filterMode + self.filters = filters + + def read(self, iprot): + if ( + iprot._fast_decode is not None + and isinstance(iprot.trans, TTransport.CReadableTransport) + and self.thrift_spec is not None + ): + iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec]) + return + iprot.readStructBegin() + while True: + (fname, ftype, fid) = iprot.readFieldBegin() + if ftype == TType.STOP: + break + if fid == 7: + if ftype == TType.I32: + self.filterMode = iprot.readI32() + else: + iprot.skip(ftype) + elif fid == 8: + if ftype == TType.LIST: + self.filters = [] + (_etype1147, _size1144) = iprot.readListBegin() + for _i1148 in range(_size1144): + _elem1149 = ( + iprot.readString().decode("utf-8", errors="replace") + if sys.version_info[0] == 2 + else iprot.readString() + ) + self.filters.append(_elem1149) + iprot.readListEnd() + else: + iprot.skip(ftype) + else: + iprot.skip(ftype) + iprot.readFieldEnd() + iprot.readStructEnd() + + def write(self, oprot): + if oprot._fast_encode is not None and self.thrift_spec is not None: + oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec])) + return + oprot.writeStructBegin("GetPartitionsFilterSpec") + if self.filterMode is not None: + oprot.writeFieldBegin("filterMode", TType.I32, 7) + oprot.writeI32(self.filterMode) + oprot.writeFieldEnd() + if self.filters is not None: + oprot.writeFieldBegin("filters", TType.LIST, 8) + oprot.writeListBegin(TType.STRING, len(self.filters)) + for iter1150 in self.filters: + oprot.writeString(iter1150.encode("utf-8") if sys.version_info[0] == 2 else iter1150) + oprot.writeListEnd() + oprot.writeFieldEnd() + oprot.writeFieldStop() + oprot.writeStructEnd() + + def validate(self): + return + + def __repr__(self): + L = ["{}={!r}".format(key, value) for key, value in self.__dict__.items()] + return "{}({})".format(self.__class__.__name__, ", ".join(L)) + + def __eq__(self, other): + return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ + + def __ne__(self, other): + return not (self == other) + + +class GetPartitionsResponse: + """ + Attributes: + - partitionSpec + + """ + + def __init__( + self, + partitionSpec=None, + ): + self.partitionSpec = partitionSpec + + def read(self, iprot): + if ( + iprot._fast_decode is not None + and isinstance(iprot.trans, TTransport.CReadableTransport) + and self.thrift_spec is not None + ): + iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec]) + return + iprot.readStructBegin() + while True: + (fname, ftype, fid) = iprot.readFieldBegin() + if ftype == TType.STOP: + break + if fid == 1: + if ftype == TType.LIST: + self.partitionSpec = [] + (_etype1154, _size1151) = iprot.readListBegin() + for _i1155 in range(_size1151): + _elem1156 = PartitionSpec() + _elem1156.read(iprot) + self.partitionSpec.append(_elem1156) + iprot.readListEnd() + else: + iprot.skip(ftype) + else: + iprot.skip(ftype) + iprot.readFieldEnd() + iprot.readStructEnd() + + def write(self, oprot): + if oprot._fast_encode is not None and self.thrift_spec is not None: + oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec])) + return + oprot.writeStructBegin("GetPartitionsResponse") + if self.partitionSpec is not None: + oprot.writeFieldBegin("partitionSpec", TType.LIST, 1) + oprot.writeListBegin(TType.STRUCT, len(self.partitionSpec)) + for iter1157 in self.partitionSpec: + iter1157.write(oprot) + oprot.writeListEnd() + oprot.writeFieldEnd() + oprot.writeFieldStop() + oprot.writeStructEnd() + + def validate(self): + return + + def __repr__(self): + L = ["{}={!r}".format(key, value) for key, value in self.__dict__.items()] + return "{}({})".format(self.__class__.__name__, ", ".join(L)) + + def __eq__(self, other): + return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ + + def __ne__(self, other): + return not (self == other) + + +class GetPartitionsRequest: + """ + Attributes: + - catName + - dbName + - tblName + - withAuth + - user + - groupNames + - projectionSpec + - filterSpec + - processorCapabilities + - processorIdentifier + - validWriteIdList + + """ + + def __init__( + self, + catName=None, + dbName=None, + tblName=None, + withAuth=None, + user=None, + groupNames=None, + projectionSpec=None, + filterSpec=None, + processorCapabilities=None, + processorIdentifier=None, + validWriteIdList=None, + ): + self.catName = catName + self.dbName = dbName + self.tblName = tblName + self.withAuth = withAuth + self.user = user + self.groupNames = groupNames + self.projectionSpec = projectionSpec + self.filterSpec = filterSpec + self.processorCapabilities = processorCapabilities + self.processorIdentifier = processorIdentifier + self.validWriteIdList = validWriteIdList + + def read(self, iprot): + if ( + iprot._fast_decode is not None + and isinstance(iprot.trans, TTransport.CReadableTransport) + and self.thrift_spec is not None + ): + iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec]) + return + iprot.readStructBegin() + while True: + (fname, ftype, fid) = iprot.readFieldBegin() + if ftype == TType.STOP: + break + if fid == 1: + if ftype == TType.STRING: + self.catName = ( + iprot.readString().decode("utf-8", errors="replace") if sys.version_info[0] == 2 else iprot.readString() + ) + else: + iprot.skip(ftype) + elif fid == 2: + if ftype == TType.STRING: + self.dbName = ( + iprot.readString().decode("utf-8", errors="replace") if sys.version_info[0] == 2 else iprot.readString() + ) + else: + iprot.skip(ftype) + elif fid == 3: + if ftype == TType.STRING: + self.tblName = ( + iprot.readString().decode("utf-8", errors="replace") if sys.version_info[0] == 2 else iprot.readString() + ) + else: + iprot.skip(ftype) + elif fid == 4: + if ftype == TType.BOOL: + self.withAuth = iprot.readBool() + else: + iprot.skip(ftype) + elif fid == 5: + if ftype == TType.STRING: + self.user = ( + iprot.readString().decode("utf-8", errors="replace") if sys.version_info[0] == 2 else iprot.readString() + ) + else: + iprot.skip(ftype) + elif fid == 6: + if ftype == TType.LIST: + self.groupNames = [] + (_etype1161, _size1158) = iprot.readListBegin() + for _i1162 in range(_size1158): + _elem1163 = ( + iprot.readString().decode("utf-8", errors="replace") + if sys.version_info[0] == 2 + else iprot.readString() + ) + self.groupNames.append(_elem1163) + iprot.readListEnd() + else: + iprot.skip(ftype) + elif fid == 7: + if ftype == TType.STRUCT: + self.projectionSpec = GetProjectionsSpec() + self.projectionSpec.read(iprot) + else: + iprot.skip(ftype) + elif fid == 8: + if ftype == TType.STRUCT: + self.filterSpec = GetPartitionsFilterSpec() + self.filterSpec.read(iprot) + else: + iprot.skip(ftype) + elif fid == 9: + if ftype == TType.LIST: + self.processorCapabilities = [] + (_etype1167, _size1164) = iprot.readListBegin() + for _i1168 in range(_size1164): + _elem1169 = ( + iprot.readString().decode("utf-8", errors="replace") + if sys.version_info[0] == 2 + else iprot.readString() + ) + self.processorCapabilities.append(_elem1169) + iprot.readListEnd() + else: + iprot.skip(ftype) + elif fid == 10: + if ftype == TType.STRING: + self.processorIdentifier = ( + iprot.readString().decode("utf-8", errors="replace") if sys.version_info[0] == 2 else iprot.readString() + ) + else: + iprot.skip(ftype) + elif fid == 11: + if ftype == TType.STRING: + self.validWriteIdList = ( + iprot.readString().decode("utf-8", errors="replace") if sys.version_info[0] == 2 else iprot.readString() + ) + else: + iprot.skip(ftype) + else: + iprot.skip(ftype) + iprot.readFieldEnd() + iprot.readStructEnd() + + def write(self, oprot): + if oprot._fast_encode is not None and self.thrift_spec is not None: + oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec])) + return + oprot.writeStructBegin("GetPartitionsRequest") + if self.catName is not None: + oprot.writeFieldBegin("catName", TType.STRING, 1) + oprot.writeString(self.catName.encode("utf-8") if sys.version_info[0] == 2 else self.catName) + oprot.writeFieldEnd() + if self.dbName is not None: + oprot.writeFieldBegin("dbName", TType.STRING, 2) + oprot.writeString(self.dbName.encode("utf-8") if sys.version_info[0] == 2 else self.dbName) + oprot.writeFieldEnd() + if self.tblName is not None: + oprot.writeFieldBegin("tblName", TType.STRING, 3) + oprot.writeString(self.tblName.encode("utf-8") if sys.version_info[0] == 2 else self.tblName) + oprot.writeFieldEnd() + if self.withAuth is not None: + oprot.writeFieldBegin("withAuth", TType.BOOL, 4) + oprot.writeBool(self.withAuth) + oprot.writeFieldEnd() + if self.user is not None: + oprot.writeFieldBegin("user", TType.STRING, 5) + oprot.writeString(self.user.encode("utf-8") if sys.version_info[0] == 2 else self.user) + oprot.writeFieldEnd() + if self.groupNames is not None: + oprot.writeFieldBegin("groupNames", TType.LIST, 6) + oprot.writeListBegin(TType.STRING, len(self.groupNames)) + for iter1170 in self.groupNames: + oprot.writeString(iter1170.encode("utf-8") if sys.version_info[0] == 2 else iter1170) + oprot.writeListEnd() + oprot.writeFieldEnd() + if self.projectionSpec is not None: + oprot.writeFieldBegin("projectionSpec", TType.STRUCT, 7) + self.projectionSpec.write(oprot) + oprot.writeFieldEnd() + if self.filterSpec is not None: + oprot.writeFieldBegin("filterSpec", TType.STRUCT, 8) + self.filterSpec.write(oprot) + oprot.writeFieldEnd() + if self.processorCapabilities is not None: + oprot.writeFieldBegin("processorCapabilities", TType.LIST, 9) + oprot.writeListBegin(TType.STRING, len(self.processorCapabilities)) + for iter1171 in self.processorCapabilities: + oprot.writeString(iter1171.encode("utf-8") if sys.version_info[0] == 2 else iter1171) + oprot.writeListEnd() + oprot.writeFieldEnd() + if self.processorIdentifier is not None: + oprot.writeFieldBegin("processorIdentifier", TType.STRING, 10) + oprot.writeString(self.processorIdentifier.encode("utf-8") if sys.version_info[0] == 2 else self.processorIdentifier) + oprot.writeFieldEnd() + if self.validWriteIdList is not None: + oprot.writeFieldBegin("validWriteIdList", TType.STRING, 11) + oprot.writeString(self.validWriteIdList.encode("utf-8") if sys.version_info[0] == 2 else self.validWriteIdList) + oprot.writeFieldEnd() + oprot.writeFieldStop() + oprot.writeStructEnd() + + def validate(self): + return + + def __repr__(self): + L = ["{}={!r}".format(key, value) for key, value in self.__dict__.items()] + return "{}({})".format(self.__class__.__name__, ", ".join(L)) + + def __eq__(self, other): + return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ + + def __ne__(self, other): + return not (self == other) + + +class GetFieldsRequest: + """ + Attributes: + - catName + - dbName + - tblName + - envContext + - validWriteIdList + - id + + """ + + def __init__( + self, + catName=None, + dbName=None, + tblName=None, + envContext=None, + validWriteIdList=None, + id=-1, + ): + self.catName = catName + self.dbName = dbName + self.tblName = tblName + self.envContext = envContext + self.validWriteIdList = validWriteIdList + self.id = id + + def read(self, iprot): + if ( + iprot._fast_decode is not None + and isinstance(iprot.trans, TTransport.CReadableTransport) + and self.thrift_spec is not None + ): + iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec]) + return + iprot.readStructBegin() + while True: + (fname, ftype, fid) = iprot.readFieldBegin() + if ftype == TType.STOP: + break + if fid == 1: + if ftype == TType.STRING: + self.catName = ( + iprot.readString().decode("utf-8", errors="replace") if sys.version_info[0] == 2 else iprot.readString() + ) + else: + iprot.skip(ftype) + elif fid == 2: + if ftype == TType.STRING: + self.dbName = ( + iprot.readString().decode("utf-8", errors="replace") if sys.version_info[0] == 2 else iprot.readString() + ) + else: + iprot.skip(ftype) + elif fid == 3: + if ftype == TType.STRING: + self.tblName = ( + iprot.readString().decode("utf-8", errors="replace") if sys.version_info[0] == 2 else iprot.readString() + ) + else: + iprot.skip(ftype) + elif fid == 4: + if ftype == TType.STRUCT: + self.envContext = EnvironmentContext() + self.envContext.read(iprot) + else: + iprot.skip(ftype) + elif fid == 5: + if ftype == TType.STRING: + self.validWriteIdList = ( + iprot.readString().decode("utf-8", errors="replace") if sys.version_info[0] == 2 else iprot.readString() + ) + else: + iprot.skip(ftype) + elif fid == 6: + if ftype == TType.I64: + self.id = iprot.readI64() + else: + iprot.skip(ftype) + else: + iprot.skip(ftype) + iprot.readFieldEnd() + iprot.readStructEnd() + + def write(self, oprot): + if oprot._fast_encode is not None and self.thrift_spec is not None: + oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec])) + return + oprot.writeStructBegin("GetFieldsRequest") + if self.catName is not None: + oprot.writeFieldBegin("catName", TType.STRING, 1) + oprot.writeString(self.catName.encode("utf-8") if sys.version_info[0] == 2 else self.catName) + oprot.writeFieldEnd() + if self.dbName is not None: + oprot.writeFieldBegin("dbName", TType.STRING, 2) + oprot.writeString(self.dbName.encode("utf-8") if sys.version_info[0] == 2 else self.dbName) + oprot.writeFieldEnd() + if self.tblName is not None: + oprot.writeFieldBegin("tblName", TType.STRING, 3) + oprot.writeString(self.tblName.encode("utf-8") if sys.version_info[0] == 2 else self.tblName) + oprot.writeFieldEnd() + if self.envContext is not None: + oprot.writeFieldBegin("envContext", TType.STRUCT, 4) + self.envContext.write(oprot) + oprot.writeFieldEnd() + if self.validWriteIdList is not None: + oprot.writeFieldBegin("validWriteIdList", TType.STRING, 5) + oprot.writeString(self.validWriteIdList.encode("utf-8") if sys.version_info[0] == 2 else self.validWriteIdList) + oprot.writeFieldEnd() + if self.id is not None: + oprot.writeFieldBegin("id", TType.I64, 6) + oprot.writeI64(self.id) + oprot.writeFieldEnd() + oprot.writeFieldStop() + oprot.writeStructEnd() + + def validate(self): + if self.dbName is None: + raise TProtocolException(message="Required field dbName is unset!") + if self.tblName is None: + raise TProtocolException(message="Required field tblName is unset!") + return + + def __repr__(self): + L = ["{}={!r}".format(key, value) for key, value in self.__dict__.items()] + return "{}({})".format(self.__class__.__name__, ", ".join(L)) + + def __eq__(self, other): + return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ + + def __ne__(self, other): + return not (self == other) + + +class GetFieldsResponse: + """ + Attributes: + - fields + + """ + + def __init__( + self, + fields=None, + ): + self.fields = fields + + def read(self, iprot): + if ( + iprot._fast_decode is not None + and isinstance(iprot.trans, TTransport.CReadableTransport) + and self.thrift_spec is not None + ): + iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec]) + return + iprot.readStructBegin() + while True: + (fname, ftype, fid) = iprot.readFieldBegin() + if ftype == TType.STOP: + break + if fid == 1: + if ftype == TType.LIST: + self.fields = [] + (_etype1175, _size1172) = iprot.readListBegin() + for _i1176 in range(_size1172): + _elem1177 = FieldSchema() + _elem1177.read(iprot) + self.fields.append(_elem1177) + iprot.readListEnd() + else: + iprot.skip(ftype) + else: + iprot.skip(ftype) + iprot.readFieldEnd() + iprot.readStructEnd() + + def write(self, oprot): + if oprot._fast_encode is not None and self.thrift_spec is not None: + oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec])) + return + oprot.writeStructBegin("GetFieldsResponse") + if self.fields is not None: + oprot.writeFieldBegin("fields", TType.LIST, 1) + oprot.writeListBegin(TType.STRUCT, len(self.fields)) + for iter1178 in self.fields: + iter1178.write(oprot) + oprot.writeListEnd() + oprot.writeFieldEnd() + oprot.writeFieldStop() + oprot.writeStructEnd() + + def validate(self): + if self.fields is None: + raise TProtocolException(message="Required field fields is unset!") + return + + def __repr__(self): + L = ["{}={!r}".format(key, value) for key, value in self.__dict__.items()] + return "{}({})".format(self.__class__.__name__, ", ".join(L)) + + def __eq__(self, other): + return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ + + def __ne__(self, other): + return not (self == other) + + +class GetSchemaRequest: + """ + Attributes: + - catName + - dbName + - tblName + - envContext + - validWriteIdList + - id + + """ + + def __init__( + self, + catName=None, + dbName=None, + tblName=None, + envContext=None, + validWriteIdList=None, + id=-1, + ): + self.catName = catName + self.dbName = dbName + self.tblName = tblName + self.envContext = envContext + self.validWriteIdList = validWriteIdList + self.id = id + + def read(self, iprot): + if ( + iprot._fast_decode is not None + and isinstance(iprot.trans, TTransport.CReadableTransport) + and self.thrift_spec is not None + ): + iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec]) + return + iprot.readStructBegin() + while True: + (fname, ftype, fid) = iprot.readFieldBegin() + if ftype == TType.STOP: + break + if fid == 1: + if ftype == TType.STRING: + self.catName = ( + iprot.readString().decode("utf-8", errors="replace") if sys.version_info[0] == 2 else iprot.readString() + ) + else: + iprot.skip(ftype) + elif fid == 2: + if ftype == TType.STRING: + self.dbName = ( + iprot.readString().decode("utf-8", errors="replace") if sys.version_info[0] == 2 else iprot.readString() + ) + else: + iprot.skip(ftype) + elif fid == 3: + if ftype == TType.STRING: + self.tblName = ( + iprot.readString().decode("utf-8", errors="replace") if sys.version_info[0] == 2 else iprot.readString() + ) + else: + iprot.skip(ftype) + elif fid == 4: + if ftype == TType.STRUCT: + self.envContext = EnvironmentContext() + self.envContext.read(iprot) + else: + iprot.skip(ftype) + elif fid == 5: + if ftype == TType.STRING: + self.validWriteIdList = ( + iprot.readString().decode("utf-8", errors="replace") if sys.version_info[0] == 2 else iprot.readString() + ) + else: + iprot.skip(ftype) + elif fid == 6: + if ftype == TType.I64: + self.id = iprot.readI64() + else: + iprot.skip(ftype) + else: + iprot.skip(ftype) + iprot.readFieldEnd() + iprot.readStructEnd() + + def write(self, oprot): + if oprot._fast_encode is not None and self.thrift_spec is not None: + oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec])) + return + oprot.writeStructBegin("GetSchemaRequest") + if self.catName is not None: + oprot.writeFieldBegin("catName", TType.STRING, 1) + oprot.writeString(self.catName.encode("utf-8") if sys.version_info[0] == 2 else self.catName) + oprot.writeFieldEnd() + if self.dbName is not None: + oprot.writeFieldBegin("dbName", TType.STRING, 2) + oprot.writeString(self.dbName.encode("utf-8") if sys.version_info[0] == 2 else self.dbName) + oprot.writeFieldEnd() + if self.tblName is not None: + oprot.writeFieldBegin("tblName", TType.STRING, 3) + oprot.writeString(self.tblName.encode("utf-8") if sys.version_info[0] == 2 else self.tblName) + oprot.writeFieldEnd() + if self.envContext is not None: + oprot.writeFieldBegin("envContext", TType.STRUCT, 4) + self.envContext.write(oprot) + oprot.writeFieldEnd() + if self.validWriteIdList is not None: + oprot.writeFieldBegin("validWriteIdList", TType.STRING, 5) + oprot.writeString(self.validWriteIdList.encode("utf-8") if sys.version_info[0] == 2 else self.validWriteIdList) + oprot.writeFieldEnd() + if self.id is not None: + oprot.writeFieldBegin("id", TType.I64, 6) + oprot.writeI64(self.id) + oprot.writeFieldEnd() + oprot.writeFieldStop() + oprot.writeStructEnd() + + def validate(self): + if self.dbName is None: + raise TProtocolException(message="Required field dbName is unset!") + if self.tblName is None: + raise TProtocolException(message="Required field tblName is unset!") + return + + def __repr__(self): + L = ["{}={!r}".format(key, value) for key, value in self.__dict__.items()] + return "{}({})".format(self.__class__.__name__, ", ".join(L)) + + def __eq__(self, other): + return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ + + def __ne__(self, other): + return not (self == other) + + +class GetSchemaResponse: + """ + Attributes: + - fields + + """ + + def __init__( + self, + fields=None, + ): + self.fields = fields + + def read(self, iprot): + if ( + iprot._fast_decode is not None + and isinstance(iprot.trans, TTransport.CReadableTransport) + and self.thrift_spec is not None + ): + iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec]) + return + iprot.readStructBegin() + while True: + (fname, ftype, fid) = iprot.readFieldBegin() + if ftype == TType.STOP: + break + if fid == 1: + if ftype == TType.LIST: + self.fields = [] + (_etype1182, _size1179) = iprot.readListBegin() + for _i1183 in range(_size1179): + _elem1184 = FieldSchema() + _elem1184.read(iprot) + self.fields.append(_elem1184) + iprot.readListEnd() + else: + iprot.skip(ftype) + else: + iprot.skip(ftype) + iprot.readFieldEnd() + iprot.readStructEnd() + + def write(self, oprot): + if oprot._fast_encode is not None and self.thrift_spec is not None: + oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec])) + return + oprot.writeStructBegin("GetSchemaResponse") + if self.fields is not None: + oprot.writeFieldBegin("fields", TType.LIST, 1) + oprot.writeListBegin(TType.STRUCT, len(self.fields)) + for iter1185 in self.fields: + iter1185.write(oprot) + oprot.writeListEnd() + oprot.writeFieldEnd() + oprot.writeFieldStop() + oprot.writeStructEnd() + + def validate(self): + if self.fields is None: + raise TProtocolException(message="Required field fields is unset!") + return + + def __repr__(self): + L = ["{}={!r}".format(key, value) for key, value in self.__dict__.items()] + return "{}({})".format(self.__class__.__name__, ", ".join(L)) + + def __eq__(self, other): + return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ + + def __ne__(self, other): + return not (self == other) + + +class GetPartitionRequest: + """ + Attributes: + - catName + - dbName + - tblName + - partVals + - validWriteIdList + - id + + """ + + def __init__( + self, + catName=None, + dbName=None, + tblName=None, + partVals=None, + validWriteIdList=None, + id=-1, + ): + self.catName = catName + self.dbName = dbName + self.tblName = tblName + self.partVals = partVals + self.validWriteIdList = validWriteIdList + self.id = id + + def read(self, iprot): + if ( + iprot._fast_decode is not None + and isinstance(iprot.trans, TTransport.CReadableTransport) + and self.thrift_spec is not None + ): + iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec]) + return + iprot.readStructBegin() + while True: + (fname, ftype, fid) = iprot.readFieldBegin() + if ftype == TType.STOP: + break + if fid == 1: + if ftype == TType.STRING: + self.catName = ( + iprot.readString().decode("utf-8", errors="replace") if sys.version_info[0] == 2 else iprot.readString() + ) + else: + iprot.skip(ftype) + elif fid == 2: + if ftype == TType.STRING: + self.dbName = ( + iprot.readString().decode("utf-8", errors="replace") if sys.version_info[0] == 2 else iprot.readString() + ) + else: + iprot.skip(ftype) + elif fid == 3: + if ftype == TType.STRING: + self.tblName = ( + iprot.readString().decode("utf-8", errors="replace") if sys.version_info[0] == 2 else iprot.readString() + ) + else: + iprot.skip(ftype) + elif fid == 4: + if ftype == TType.LIST: + self.partVals = [] + (_etype1189, _size1186) = iprot.readListBegin() + for _i1190 in range(_size1186): + _elem1191 = ( + iprot.readString().decode("utf-8", errors="replace") + if sys.version_info[0] == 2 + else iprot.readString() + ) + self.partVals.append(_elem1191) + iprot.readListEnd() + else: + iprot.skip(ftype) + elif fid == 5: + if ftype == TType.STRING: + self.validWriteIdList = ( + iprot.readString().decode("utf-8", errors="replace") if sys.version_info[0] == 2 else iprot.readString() + ) + else: + iprot.skip(ftype) + elif fid == 6: + if ftype == TType.I64: + self.id = iprot.readI64() + else: + iprot.skip(ftype) + else: + iprot.skip(ftype) + iprot.readFieldEnd() + iprot.readStructEnd() + + def write(self, oprot): + if oprot._fast_encode is not None and self.thrift_spec is not None: + oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec])) + return + oprot.writeStructBegin("GetPartitionRequest") + if self.catName is not None: + oprot.writeFieldBegin("catName", TType.STRING, 1) + oprot.writeString(self.catName.encode("utf-8") if sys.version_info[0] == 2 else self.catName) + oprot.writeFieldEnd() + if self.dbName is not None: + oprot.writeFieldBegin("dbName", TType.STRING, 2) + oprot.writeString(self.dbName.encode("utf-8") if sys.version_info[0] == 2 else self.dbName) + oprot.writeFieldEnd() + if self.tblName is not None: + oprot.writeFieldBegin("tblName", TType.STRING, 3) + oprot.writeString(self.tblName.encode("utf-8") if sys.version_info[0] == 2 else self.tblName) + oprot.writeFieldEnd() + if self.partVals is not None: + oprot.writeFieldBegin("partVals", TType.LIST, 4) + oprot.writeListBegin(TType.STRING, len(self.partVals)) + for iter1192 in self.partVals: + oprot.writeString(iter1192.encode("utf-8") if sys.version_info[0] == 2 else iter1192) + oprot.writeListEnd() + oprot.writeFieldEnd() + if self.validWriteIdList is not None: + oprot.writeFieldBegin("validWriteIdList", TType.STRING, 5) + oprot.writeString(self.validWriteIdList.encode("utf-8") if sys.version_info[0] == 2 else self.validWriteIdList) + oprot.writeFieldEnd() + if self.id is not None: + oprot.writeFieldBegin("id", TType.I64, 6) + oprot.writeI64(self.id) + oprot.writeFieldEnd() + oprot.writeFieldStop() + oprot.writeStructEnd() + + def validate(self): + if self.dbName is None: + raise TProtocolException(message="Required field dbName is unset!") + if self.tblName is None: + raise TProtocolException(message="Required field tblName is unset!") + if self.partVals is None: + raise TProtocolException(message="Required field partVals is unset!") + return + + def __repr__(self): + L = ["{}={!r}".format(key, value) for key, value in self.__dict__.items()] + return "{}({})".format(self.__class__.__name__, ", ".join(L)) + + def __eq__(self, other): + return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ + + def __ne__(self, other): + return not (self == other) + + +class GetPartitionResponse: + """ + Attributes: + - partition + + """ + + def __init__( + self, + partition=None, + ): + self.partition = partition + + def read(self, iprot): + if ( + iprot._fast_decode is not None + and isinstance(iprot.trans, TTransport.CReadableTransport) + and self.thrift_spec is not None + ): + iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec]) + return + iprot.readStructBegin() + while True: + (fname, ftype, fid) = iprot.readFieldBegin() + if ftype == TType.STOP: + break + if fid == 1: + if ftype == TType.STRUCT: + self.partition = Partition() + self.partition.read(iprot) + else: + iprot.skip(ftype) + else: + iprot.skip(ftype) + iprot.readFieldEnd() + iprot.readStructEnd() + + def write(self, oprot): + if oprot._fast_encode is not None and self.thrift_spec is not None: + oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec])) + return + oprot.writeStructBegin("GetPartitionResponse") + if self.partition is not None: + oprot.writeFieldBegin("partition", TType.STRUCT, 1) + self.partition.write(oprot) + oprot.writeFieldEnd() + oprot.writeFieldStop() + oprot.writeStructEnd() + + def validate(self): + if self.partition is None: + raise TProtocolException(message="Required field partition is unset!") + return + + def __repr__(self): + L = ["{}={!r}".format(key, value) for key, value in self.__dict__.items()] + return "{}({})".format(self.__class__.__name__, ", ".join(L)) + + def __eq__(self, other): + return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ + + def __ne__(self, other): + return not (self == other) + + +class PartitionsRequest: + """ + Attributes: + - catName + - dbName + - tblName + - maxParts + - validWriteIdList + - id + + """ + + def __init__( + self, + catName=None, + dbName=None, + tblName=None, + maxParts=-1, + validWriteIdList=None, + id=-1, + ): + self.catName = catName + self.dbName = dbName + self.tblName = tblName + self.maxParts = maxParts + self.validWriteIdList = validWriteIdList + self.id = id + + def read(self, iprot): + if ( + iprot._fast_decode is not None + and isinstance(iprot.trans, TTransport.CReadableTransport) + and self.thrift_spec is not None + ): + iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec]) + return + iprot.readStructBegin() + while True: + (fname, ftype, fid) = iprot.readFieldBegin() + if ftype == TType.STOP: + break + if fid == 1: + if ftype == TType.STRING: + self.catName = ( + iprot.readString().decode("utf-8", errors="replace") if sys.version_info[0] == 2 else iprot.readString() + ) + else: + iprot.skip(ftype) + elif fid == 2: + if ftype == TType.STRING: + self.dbName = ( + iprot.readString().decode("utf-8", errors="replace") if sys.version_info[0] == 2 else iprot.readString() + ) + else: + iprot.skip(ftype) + elif fid == 3: + if ftype == TType.STRING: + self.tblName = ( + iprot.readString().decode("utf-8", errors="replace") if sys.version_info[0] == 2 else iprot.readString() + ) + else: + iprot.skip(ftype) + elif fid == 4: + if ftype == TType.I16: + self.maxParts = iprot.readI16() + else: + iprot.skip(ftype) + elif fid == 5: + if ftype == TType.STRING: + self.validWriteIdList = ( + iprot.readString().decode("utf-8", errors="replace") if sys.version_info[0] == 2 else iprot.readString() + ) + else: + iprot.skip(ftype) + elif fid == 6: + if ftype == TType.I64: + self.id = iprot.readI64() + else: + iprot.skip(ftype) + else: + iprot.skip(ftype) + iprot.readFieldEnd() + iprot.readStructEnd() + + def write(self, oprot): + if oprot._fast_encode is not None and self.thrift_spec is not None: + oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec])) + return + oprot.writeStructBegin("PartitionsRequest") + if self.catName is not None: + oprot.writeFieldBegin("catName", TType.STRING, 1) + oprot.writeString(self.catName.encode("utf-8") if sys.version_info[0] == 2 else self.catName) + oprot.writeFieldEnd() + if self.dbName is not None: + oprot.writeFieldBegin("dbName", TType.STRING, 2) + oprot.writeString(self.dbName.encode("utf-8") if sys.version_info[0] == 2 else self.dbName) + oprot.writeFieldEnd() + if self.tblName is not None: + oprot.writeFieldBegin("tblName", TType.STRING, 3) + oprot.writeString(self.tblName.encode("utf-8") if sys.version_info[0] == 2 else self.tblName) + oprot.writeFieldEnd() + if self.maxParts is not None: + oprot.writeFieldBegin("maxParts", TType.I16, 4) + oprot.writeI16(self.maxParts) + oprot.writeFieldEnd() + if self.validWriteIdList is not None: + oprot.writeFieldBegin("validWriteIdList", TType.STRING, 5) + oprot.writeString(self.validWriteIdList.encode("utf-8") if sys.version_info[0] == 2 else self.validWriteIdList) + oprot.writeFieldEnd() + if self.id is not None: + oprot.writeFieldBegin("id", TType.I64, 6) + oprot.writeI64(self.id) + oprot.writeFieldEnd() + oprot.writeFieldStop() + oprot.writeStructEnd() + + def validate(self): + if self.dbName is None: + raise TProtocolException(message="Required field dbName is unset!") + if self.tblName is None: + raise TProtocolException(message="Required field tblName is unset!") + return + + def __repr__(self): + L = ["{}={!r}".format(key, value) for key, value in self.__dict__.items()] + return "{}({})".format(self.__class__.__name__, ", ".join(L)) + + def __eq__(self, other): + return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ + + def __ne__(self, other): + return not (self == other) + + +class PartitionsResponse: + """ + Attributes: + - partitions + + """ + + def __init__( + self, + partitions=None, + ): + self.partitions = partitions + + def read(self, iprot): + if ( + iprot._fast_decode is not None + and isinstance(iprot.trans, TTransport.CReadableTransport) + and self.thrift_spec is not None + ): + iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec]) + return + iprot.readStructBegin() + while True: + (fname, ftype, fid) = iprot.readFieldBegin() + if ftype == TType.STOP: + break + if fid == 1: + if ftype == TType.LIST: + self.partitions = [] + (_etype1196, _size1193) = iprot.readListBegin() + for _i1197 in range(_size1193): + _elem1198 = Partition() + _elem1198.read(iprot) + self.partitions.append(_elem1198) + iprot.readListEnd() + else: + iprot.skip(ftype) + else: + iprot.skip(ftype) + iprot.readFieldEnd() + iprot.readStructEnd() + + def write(self, oprot): + if oprot._fast_encode is not None and self.thrift_spec is not None: + oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec])) + return + oprot.writeStructBegin("PartitionsResponse") + if self.partitions is not None: + oprot.writeFieldBegin("partitions", TType.LIST, 1) + oprot.writeListBegin(TType.STRUCT, len(self.partitions)) + for iter1199 in self.partitions: + iter1199.write(oprot) + oprot.writeListEnd() + oprot.writeFieldEnd() + oprot.writeFieldStop() + oprot.writeStructEnd() + + def validate(self): + if self.partitions is None: + raise TProtocolException(message="Required field partitions is unset!") + return + + def __repr__(self): + L = ["{}={!r}".format(key, value) for key, value in self.__dict__.items()] + return "{}({})".format(self.__class__.__name__, ", ".join(L)) + + def __eq__(self, other): + return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ + + def __ne__(self, other): + return not (self == other) + + +class GetPartitionNamesPsRequest: + """ + Attributes: + - catName + - dbName + - tblName + - partValues + - maxParts + - validWriteIdList + - id + + """ + + def __init__( + self, + catName=None, + dbName=None, + tblName=None, + partValues=None, + maxParts=-1, + validWriteIdList=None, + id=-1, + ): + self.catName = catName + self.dbName = dbName + self.tblName = tblName + self.partValues = partValues + self.maxParts = maxParts + self.validWriteIdList = validWriteIdList + self.id = id + + def read(self, iprot): + if ( + iprot._fast_decode is not None + and isinstance(iprot.trans, TTransport.CReadableTransport) + and self.thrift_spec is not None + ): + iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec]) + return + iprot.readStructBegin() + while True: + (fname, ftype, fid) = iprot.readFieldBegin() + if ftype == TType.STOP: + break + if fid == 1: + if ftype == TType.STRING: + self.catName = ( + iprot.readString().decode("utf-8", errors="replace") if sys.version_info[0] == 2 else iprot.readString() + ) + else: + iprot.skip(ftype) + elif fid == 2: + if ftype == TType.STRING: + self.dbName = ( + iprot.readString().decode("utf-8", errors="replace") if sys.version_info[0] == 2 else iprot.readString() + ) + else: + iprot.skip(ftype) + elif fid == 3: + if ftype == TType.STRING: + self.tblName = ( + iprot.readString().decode("utf-8", errors="replace") if sys.version_info[0] == 2 else iprot.readString() + ) + else: + iprot.skip(ftype) + elif fid == 4: + if ftype == TType.LIST: + self.partValues = [] + (_etype1203, _size1200) = iprot.readListBegin() + for _i1204 in range(_size1200): + _elem1205 = ( + iprot.readString().decode("utf-8", errors="replace") + if sys.version_info[0] == 2 + else iprot.readString() + ) + self.partValues.append(_elem1205) + iprot.readListEnd() + else: + iprot.skip(ftype) + elif fid == 5: + if ftype == TType.I16: + self.maxParts = iprot.readI16() + else: + iprot.skip(ftype) + elif fid == 6: + if ftype == TType.STRING: + self.validWriteIdList = ( + iprot.readString().decode("utf-8", errors="replace") if sys.version_info[0] == 2 else iprot.readString() + ) + else: + iprot.skip(ftype) + elif fid == 7: + if ftype == TType.I64: + self.id = iprot.readI64() + else: + iprot.skip(ftype) + else: + iprot.skip(ftype) + iprot.readFieldEnd() + iprot.readStructEnd() + + def write(self, oprot): + if oprot._fast_encode is not None and self.thrift_spec is not None: + oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec])) + return + oprot.writeStructBegin("GetPartitionNamesPsRequest") + if self.catName is not None: + oprot.writeFieldBegin("catName", TType.STRING, 1) + oprot.writeString(self.catName.encode("utf-8") if sys.version_info[0] == 2 else self.catName) + oprot.writeFieldEnd() + if self.dbName is not None: + oprot.writeFieldBegin("dbName", TType.STRING, 2) + oprot.writeString(self.dbName.encode("utf-8") if sys.version_info[0] == 2 else self.dbName) + oprot.writeFieldEnd() + if self.tblName is not None: + oprot.writeFieldBegin("tblName", TType.STRING, 3) + oprot.writeString(self.tblName.encode("utf-8") if sys.version_info[0] == 2 else self.tblName) + oprot.writeFieldEnd() + if self.partValues is not None: + oprot.writeFieldBegin("partValues", TType.LIST, 4) + oprot.writeListBegin(TType.STRING, len(self.partValues)) + for iter1206 in self.partValues: + oprot.writeString(iter1206.encode("utf-8") if sys.version_info[0] == 2 else iter1206) + oprot.writeListEnd() + oprot.writeFieldEnd() + if self.maxParts is not None: + oprot.writeFieldBegin("maxParts", TType.I16, 5) + oprot.writeI16(self.maxParts) + oprot.writeFieldEnd() + if self.validWriteIdList is not None: + oprot.writeFieldBegin("validWriteIdList", TType.STRING, 6) + oprot.writeString(self.validWriteIdList.encode("utf-8") if sys.version_info[0] == 2 else self.validWriteIdList) + oprot.writeFieldEnd() + if self.id is not None: + oprot.writeFieldBegin("id", TType.I64, 7) + oprot.writeI64(self.id) + oprot.writeFieldEnd() + oprot.writeFieldStop() + oprot.writeStructEnd() + + def validate(self): + if self.dbName is None: + raise TProtocolException(message="Required field dbName is unset!") + if self.tblName is None: + raise TProtocolException(message="Required field tblName is unset!") + return + + def __repr__(self): + L = ["{}={!r}".format(key, value) for key, value in self.__dict__.items()] + return "{}({})".format(self.__class__.__name__, ", ".join(L)) + + def __eq__(self, other): + return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ + + def __ne__(self, other): + return not (self == other) + + +class GetPartitionNamesPsResponse: + """ + Attributes: + - names + + """ + + def __init__( + self, + names=None, + ): + self.names = names + + def read(self, iprot): + if ( + iprot._fast_decode is not None + and isinstance(iprot.trans, TTransport.CReadableTransport) + and self.thrift_spec is not None + ): + iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec]) + return + iprot.readStructBegin() + while True: + (fname, ftype, fid) = iprot.readFieldBegin() + if ftype == TType.STOP: + break + if fid == 1: + if ftype == TType.LIST: + self.names = [] + (_etype1210, _size1207) = iprot.readListBegin() + for _i1211 in range(_size1207): + _elem1212 = ( + iprot.readString().decode("utf-8", errors="replace") + if sys.version_info[0] == 2 + else iprot.readString() + ) + self.names.append(_elem1212) + iprot.readListEnd() + else: + iprot.skip(ftype) + else: + iprot.skip(ftype) + iprot.readFieldEnd() + iprot.readStructEnd() + + def write(self, oprot): + if oprot._fast_encode is not None and self.thrift_spec is not None: + oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec])) + return + oprot.writeStructBegin("GetPartitionNamesPsResponse") + if self.names is not None: + oprot.writeFieldBegin("names", TType.LIST, 1) + oprot.writeListBegin(TType.STRING, len(self.names)) + for iter1213 in self.names: + oprot.writeString(iter1213.encode("utf-8") if sys.version_info[0] == 2 else iter1213) + oprot.writeListEnd() + oprot.writeFieldEnd() + oprot.writeFieldStop() + oprot.writeStructEnd() + + def validate(self): + if self.names is None: + raise TProtocolException(message="Required field names is unset!") + return + + def __repr__(self): + L = ["{}={!r}".format(key, value) for key, value in self.__dict__.items()] + return "{}({})".format(self.__class__.__name__, ", ".join(L)) + + def __eq__(self, other): + return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ + + def __ne__(self, other): + return not (self == other) + + +class GetPartitionsPsWithAuthRequest: + """ + Attributes: + - catName + - dbName + - tblName + - partVals + - maxParts + - userName + - groupNames + - validWriteIdList + - id + + """ + + def __init__( + self, + catName=None, + dbName=None, + tblName=None, + partVals=None, + maxParts=-1, + userName=None, + groupNames=None, + validWriteIdList=None, + id=-1, + ): + self.catName = catName + self.dbName = dbName + self.tblName = tblName + self.partVals = partVals + self.maxParts = maxParts + self.userName = userName + self.groupNames = groupNames + self.validWriteIdList = validWriteIdList + self.id = id + + def read(self, iprot): + if ( + iprot._fast_decode is not None + and isinstance(iprot.trans, TTransport.CReadableTransport) + and self.thrift_spec is not None + ): + iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec]) + return + iprot.readStructBegin() + while True: + (fname, ftype, fid) = iprot.readFieldBegin() + if ftype == TType.STOP: + break + if fid == 1: + if ftype == TType.STRING: + self.catName = ( + iprot.readString().decode("utf-8", errors="replace") if sys.version_info[0] == 2 else iprot.readString() + ) + else: + iprot.skip(ftype) + elif fid == 2: + if ftype == TType.STRING: + self.dbName = ( + iprot.readString().decode("utf-8", errors="replace") if sys.version_info[0] == 2 else iprot.readString() + ) + else: + iprot.skip(ftype) + elif fid == 3: + if ftype == TType.STRING: + self.tblName = ( + iprot.readString().decode("utf-8", errors="replace") if sys.version_info[0] == 2 else iprot.readString() + ) + else: + iprot.skip(ftype) + elif fid == 4: + if ftype == TType.LIST: + self.partVals = [] + (_etype1217, _size1214) = iprot.readListBegin() + for _i1218 in range(_size1214): + _elem1219 = ( + iprot.readString().decode("utf-8", errors="replace") + if sys.version_info[0] == 2 + else iprot.readString() + ) + self.partVals.append(_elem1219) + iprot.readListEnd() + else: + iprot.skip(ftype) + elif fid == 5: + if ftype == TType.I16: + self.maxParts = iprot.readI16() + else: + iprot.skip(ftype) + elif fid == 6: + if ftype == TType.STRING: + self.userName = ( + iprot.readString().decode("utf-8", errors="replace") if sys.version_info[0] == 2 else iprot.readString() + ) + else: + iprot.skip(ftype) + elif fid == 7: + if ftype == TType.LIST: + self.groupNames = [] + (_etype1223, _size1220) = iprot.readListBegin() + for _i1224 in range(_size1220): + _elem1225 = ( + iprot.readString().decode("utf-8", errors="replace") + if sys.version_info[0] == 2 + else iprot.readString() + ) + self.groupNames.append(_elem1225) + iprot.readListEnd() + else: + iprot.skip(ftype) + elif fid == 8: + if ftype == TType.STRING: + self.validWriteIdList = ( + iprot.readString().decode("utf-8", errors="replace") if sys.version_info[0] == 2 else iprot.readString() + ) + else: + iprot.skip(ftype) + elif fid == 9: + if ftype == TType.I64: + self.id = iprot.readI64() + else: + iprot.skip(ftype) + else: + iprot.skip(ftype) + iprot.readFieldEnd() + iprot.readStructEnd() + + def write(self, oprot): + if oprot._fast_encode is not None and self.thrift_spec is not None: + oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec])) + return + oprot.writeStructBegin("GetPartitionsPsWithAuthRequest") + if self.catName is not None: + oprot.writeFieldBegin("catName", TType.STRING, 1) + oprot.writeString(self.catName.encode("utf-8") if sys.version_info[0] == 2 else self.catName) + oprot.writeFieldEnd() + if self.dbName is not None: + oprot.writeFieldBegin("dbName", TType.STRING, 2) + oprot.writeString(self.dbName.encode("utf-8") if sys.version_info[0] == 2 else self.dbName) + oprot.writeFieldEnd() + if self.tblName is not None: + oprot.writeFieldBegin("tblName", TType.STRING, 3) + oprot.writeString(self.tblName.encode("utf-8") if sys.version_info[0] == 2 else self.tblName) + oprot.writeFieldEnd() + if self.partVals is not None: + oprot.writeFieldBegin("partVals", TType.LIST, 4) + oprot.writeListBegin(TType.STRING, len(self.partVals)) + for iter1226 in self.partVals: + oprot.writeString(iter1226.encode("utf-8") if sys.version_info[0] == 2 else iter1226) + oprot.writeListEnd() + oprot.writeFieldEnd() + if self.maxParts is not None: + oprot.writeFieldBegin("maxParts", TType.I16, 5) + oprot.writeI16(self.maxParts) + oprot.writeFieldEnd() + if self.userName is not None: + oprot.writeFieldBegin("userName", TType.STRING, 6) + oprot.writeString(self.userName.encode("utf-8") if sys.version_info[0] == 2 else self.userName) + oprot.writeFieldEnd() + if self.groupNames is not None: + oprot.writeFieldBegin("groupNames", TType.LIST, 7) + oprot.writeListBegin(TType.STRING, len(self.groupNames)) + for iter1227 in self.groupNames: + oprot.writeString(iter1227.encode("utf-8") if sys.version_info[0] == 2 else iter1227) + oprot.writeListEnd() + oprot.writeFieldEnd() + if self.validWriteIdList is not None: + oprot.writeFieldBegin("validWriteIdList", TType.STRING, 8) + oprot.writeString(self.validWriteIdList.encode("utf-8") if sys.version_info[0] == 2 else self.validWriteIdList) + oprot.writeFieldEnd() + if self.id is not None: + oprot.writeFieldBegin("id", TType.I64, 9) + oprot.writeI64(self.id) + oprot.writeFieldEnd() + oprot.writeFieldStop() + oprot.writeStructEnd() + + def validate(self): + if self.dbName is None: + raise TProtocolException(message="Required field dbName is unset!") + if self.tblName is None: + raise TProtocolException(message="Required field tblName is unset!") + return + + def __repr__(self): + L = ["{}={!r}".format(key, value) for key, value in self.__dict__.items()] + return "{}({})".format(self.__class__.__name__, ", ".join(L)) + + def __eq__(self, other): + return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ + + def __ne__(self, other): + return not (self == other) + + +class GetPartitionsPsWithAuthResponse: + """ + Attributes: + - partitions + + """ + + def __init__( + self, + partitions=None, + ): + self.partitions = partitions + + def read(self, iprot): + if ( + iprot._fast_decode is not None + and isinstance(iprot.trans, TTransport.CReadableTransport) + and self.thrift_spec is not None + ): + iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec]) + return + iprot.readStructBegin() + while True: + (fname, ftype, fid) = iprot.readFieldBegin() + if ftype == TType.STOP: + break + if fid == 1: + if ftype == TType.LIST: + self.partitions = [] + (_etype1231, _size1228) = iprot.readListBegin() + for _i1232 in range(_size1228): + _elem1233 = Partition() + _elem1233.read(iprot) + self.partitions.append(_elem1233) + iprot.readListEnd() + else: + iprot.skip(ftype) + else: + iprot.skip(ftype) + iprot.readFieldEnd() + iprot.readStructEnd() + + def write(self, oprot): + if oprot._fast_encode is not None and self.thrift_spec is not None: + oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec])) + return + oprot.writeStructBegin("GetPartitionsPsWithAuthResponse") + if self.partitions is not None: + oprot.writeFieldBegin("partitions", TType.LIST, 1) + oprot.writeListBegin(TType.STRUCT, len(self.partitions)) + for iter1234 in self.partitions: + iter1234.write(oprot) + oprot.writeListEnd() + oprot.writeFieldEnd() + oprot.writeFieldStop() + oprot.writeStructEnd() + + def validate(self): + if self.partitions is None: + raise TProtocolException(message="Required field partitions is unset!") + return + + def __repr__(self): + L = ["{}={!r}".format(key, value) for key, value in self.__dict__.items()] + return "{}({})".format(self.__class__.__name__, ", ".join(L)) + + def __eq__(self, other): + return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ + + def __ne__(self, other): + return not (self == other) + + +class ReplicationMetrics: + """ + Attributes: + - scheduledExecutionId + - policy + - dumpExecutionId + - metadata + - progress + - messageFormat + + """ + + def __init__( + self, + scheduledExecutionId=None, + policy=None, + dumpExecutionId=None, + metadata=None, + progress=None, + messageFormat=None, + ): + self.scheduledExecutionId = scheduledExecutionId + self.policy = policy + self.dumpExecutionId = dumpExecutionId + self.metadata = metadata + self.progress = progress + self.messageFormat = messageFormat + + def read(self, iprot): + if ( + iprot._fast_decode is not None + and isinstance(iprot.trans, TTransport.CReadableTransport) + and self.thrift_spec is not None + ): + iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec]) + return + iprot.readStructBegin() + while True: + (fname, ftype, fid) = iprot.readFieldBegin() + if ftype == TType.STOP: + break + if fid == 1: + if ftype == TType.I64: + self.scheduledExecutionId = iprot.readI64() + else: + iprot.skip(ftype) + elif fid == 2: + if ftype == TType.STRING: + self.policy = ( + iprot.readString().decode("utf-8", errors="replace") if sys.version_info[0] == 2 else iprot.readString() + ) + else: + iprot.skip(ftype) + elif fid == 3: + if ftype == TType.I64: + self.dumpExecutionId = iprot.readI64() + else: + iprot.skip(ftype) + elif fid == 4: + if ftype == TType.STRING: + self.metadata = ( + iprot.readString().decode("utf-8", errors="replace") if sys.version_info[0] == 2 else iprot.readString() + ) + else: + iprot.skip(ftype) + elif fid == 5: + if ftype == TType.STRING: + self.progress = ( + iprot.readString().decode("utf-8", errors="replace") if sys.version_info[0] == 2 else iprot.readString() + ) + else: + iprot.skip(ftype) + elif fid == 6: + if ftype == TType.STRING: + self.messageFormat = ( + iprot.readString().decode("utf-8", errors="replace") if sys.version_info[0] == 2 else iprot.readString() + ) + else: + iprot.skip(ftype) + else: + iprot.skip(ftype) + iprot.readFieldEnd() + iprot.readStructEnd() + + def write(self, oprot): + if oprot._fast_encode is not None and self.thrift_spec is not None: + oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec])) + return + oprot.writeStructBegin("ReplicationMetrics") + if self.scheduledExecutionId is not None: + oprot.writeFieldBegin("scheduledExecutionId", TType.I64, 1) + oprot.writeI64(self.scheduledExecutionId) + oprot.writeFieldEnd() + if self.policy is not None: + oprot.writeFieldBegin("policy", TType.STRING, 2) + oprot.writeString(self.policy.encode("utf-8") if sys.version_info[0] == 2 else self.policy) + oprot.writeFieldEnd() + if self.dumpExecutionId is not None: + oprot.writeFieldBegin("dumpExecutionId", TType.I64, 3) + oprot.writeI64(self.dumpExecutionId) + oprot.writeFieldEnd() + if self.metadata is not None: + oprot.writeFieldBegin("metadata", TType.STRING, 4) + oprot.writeString(self.metadata.encode("utf-8") if sys.version_info[0] == 2 else self.metadata) + oprot.writeFieldEnd() + if self.progress is not None: + oprot.writeFieldBegin("progress", TType.STRING, 5) + oprot.writeString(self.progress.encode("utf-8") if sys.version_info[0] == 2 else self.progress) + oprot.writeFieldEnd() + if self.messageFormat is not None: + oprot.writeFieldBegin("messageFormat", TType.STRING, 6) + oprot.writeString(self.messageFormat.encode("utf-8") if sys.version_info[0] == 2 else self.messageFormat) + oprot.writeFieldEnd() + oprot.writeFieldStop() + oprot.writeStructEnd() + + def validate(self): + if self.scheduledExecutionId is None: + raise TProtocolException(message="Required field scheduledExecutionId is unset!") + if self.policy is None: + raise TProtocolException(message="Required field policy is unset!") + if self.dumpExecutionId is None: + raise TProtocolException(message="Required field dumpExecutionId is unset!") + return + + def __repr__(self): + L = ["{}={!r}".format(key, value) for key, value in self.__dict__.items()] + return "{}({})".format(self.__class__.__name__, ", ".join(L)) + + def __eq__(self, other): + return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ + + def __ne__(self, other): + return not (self == other) + + +class ReplicationMetricList: + """ + Attributes: + - replicationMetricList + + """ + + def __init__( + self, + replicationMetricList=None, + ): + self.replicationMetricList = replicationMetricList + + def read(self, iprot): + if ( + iprot._fast_decode is not None + and isinstance(iprot.trans, TTransport.CReadableTransport) + and self.thrift_spec is not None + ): + iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec]) + return + iprot.readStructBegin() + while True: + (fname, ftype, fid) = iprot.readFieldBegin() + if ftype == TType.STOP: + break + if fid == 1: + if ftype == TType.LIST: + self.replicationMetricList = [] + (_etype1238, _size1235) = iprot.readListBegin() + for _i1239 in range(_size1235): + _elem1240 = ReplicationMetrics() + _elem1240.read(iprot) + self.replicationMetricList.append(_elem1240) + iprot.readListEnd() + else: + iprot.skip(ftype) + else: + iprot.skip(ftype) + iprot.readFieldEnd() + iprot.readStructEnd() + + def write(self, oprot): + if oprot._fast_encode is not None and self.thrift_spec is not None: + oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec])) + return + oprot.writeStructBegin("ReplicationMetricList") + if self.replicationMetricList is not None: + oprot.writeFieldBegin("replicationMetricList", TType.LIST, 1) + oprot.writeListBegin(TType.STRUCT, len(self.replicationMetricList)) + for iter1241 in self.replicationMetricList: + iter1241.write(oprot) + oprot.writeListEnd() + oprot.writeFieldEnd() + oprot.writeFieldStop() + oprot.writeStructEnd() + + def validate(self): + if self.replicationMetricList is None: + raise TProtocolException(message="Required field replicationMetricList is unset!") + return + + def __repr__(self): + L = ["{}={!r}".format(key, value) for key, value in self.__dict__.items()] + return "{}({})".format(self.__class__.__name__, ", ".join(L)) + + def __eq__(self, other): + return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ + + def __ne__(self, other): + return not (self == other) + + +class GetReplicationMetricsRequest: + """ + Attributes: + - scheduledExecutionId + - policy + - dumpExecutionId + + """ + + def __init__( + self, + scheduledExecutionId=None, + policy=None, + dumpExecutionId=None, + ): + self.scheduledExecutionId = scheduledExecutionId + self.policy = policy + self.dumpExecutionId = dumpExecutionId + + def read(self, iprot): + if ( + iprot._fast_decode is not None + and isinstance(iprot.trans, TTransport.CReadableTransport) + and self.thrift_spec is not None + ): + iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec]) + return + iprot.readStructBegin() + while True: + (fname, ftype, fid) = iprot.readFieldBegin() + if ftype == TType.STOP: + break + if fid == 1: + if ftype == TType.I64: + self.scheduledExecutionId = iprot.readI64() + else: + iprot.skip(ftype) + elif fid == 2: + if ftype == TType.STRING: + self.policy = ( + iprot.readString().decode("utf-8", errors="replace") if sys.version_info[0] == 2 else iprot.readString() + ) + else: + iprot.skip(ftype) + elif fid == 3: + if ftype == TType.I64: + self.dumpExecutionId = iprot.readI64() + else: + iprot.skip(ftype) + else: + iprot.skip(ftype) + iprot.readFieldEnd() + iprot.readStructEnd() + + def write(self, oprot): + if oprot._fast_encode is not None and self.thrift_spec is not None: + oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec])) + return + oprot.writeStructBegin("GetReplicationMetricsRequest") + if self.scheduledExecutionId is not None: + oprot.writeFieldBegin("scheduledExecutionId", TType.I64, 1) + oprot.writeI64(self.scheduledExecutionId) + oprot.writeFieldEnd() + if self.policy is not None: + oprot.writeFieldBegin("policy", TType.STRING, 2) + oprot.writeString(self.policy.encode("utf-8") if sys.version_info[0] == 2 else self.policy) + oprot.writeFieldEnd() + if self.dumpExecutionId is not None: + oprot.writeFieldBegin("dumpExecutionId", TType.I64, 3) + oprot.writeI64(self.dumpExecutionId) + oprot.writeFieldEnd() + oprot.writeFieldStop() + oprot.writeStructEnd() + + def validate(self): + return + + def __repr__(self): + L = ["{}={!r}".format(key, value) for key, value in self.__dict__.items()] + return "{}({})".format(self.__class__.__name__, ", ".join(L)) + + def __eq__(self, other): + return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ + + def __ne__(self, other): + return not (self == other) + + +class GetOpenTxnsRequest: + """ + Attributes: + - excludeTxnTypes + + """ + + def __init__( + self, + excludeTxnTypes=None, + ): + self.excludeTxnTypes = excludeTxnTypes + + def read(self, iprot): + if ( + iprot._fast_decode is not None + and isinstance(iprot.trans, TTransport.CReadableTransport) + and self.thrift_spec is not None + ): + iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec]) + return + iprot.readStructBegin() + while True: + (fname, ftype, fid) = iprot.readFieldBegin() + if ftype == TType.STOP: + break + if fid == 1: + if ftype == TType.LIST: + self.excludeTxnTypes = [] + (_etype1245, _size1242) = iprot.readListBegin() + for _i1246 in range(_size1242): + _elem1247 = iprot.readI32() + self.excludeTxnTypes.append(_elem1247) + iprot.readListEnd() + else: + iprot.skip(ftype) + else: + iprot.skip(ftype) + iprot.readFieldEnd() + iprot.readStructEnd() + + def write(self, oprot): + if oprot._fast_encode is not None and self.thrift_spec is not None: + oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec])) + return + oprot.writeStructBegin("GetOpenTxnsRequest") + if self.excludeTxnTypes is not None: + oprot.writeFieldBegin("excludeTxnTypes", TType.LIST, 1) + oprot.writeListBegin(TType.I32, len(self.excludeTxnTypes)) + for iter1248 in self.excludeTxnTypes: + oprot.writeI32(iter1248) + oprot.writeListEnd() + oprot.writeFieldEnd() + oprot.writeFieldStop() + oprot.writeStructEnd() + + def validate(self): + return + + def __repr__(self): + L = ["{}={!r}".format(key, value) for key, value in self.__dict__.items()] + return "{}({})".format(self.__class__.__name__, ", ".join(L)) + + def __eq__(self, other): + return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ + + def __ne__(self, other): + return not (self == other) + + +class StoredProcedureRequest: + """ + Attributes: + - catName + - dbName + - procName + + """ + + def __init__( + self, + catName=None, + dbName=None, + procName=None, + ): + self.catName = catName + self.dbName = dbName + self.procName = procName + + def read(self, iprot): + if ( + iprot._fast_decode is not None + and isinstance(iprot.trans, TTransport.CReadableTransport) + and self.thrift_spec is not None + ): + iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec]) + return + iprot.readStructBegin() + while True: + (fname, ftype, fid) = iprot.readFieldBegin() + if ftype == TType.STOP: + break + if fid == 1: + if ftype == TType.STRING: + self.catName = ( + iprot.readString().decode("utf-8", errors="replace") if sys.version_info[0] == 2 else iprot.readString() + ) + else: + iprot.skip(ftype) + elif fid == 2: + if ftype == TType.STRING: + self.dbName = ( + iprot.readString().decode("utf-8", errors="replace") if sys.version_info[0] == 2 else iprot.readString() + ) + else: + iprot.skip(ftype) + elif fid == 3: + if ftype == TType.STRING: + self.procName = ( + iprot.readString().decode("utf-8", errors="replace") if sys.version_info[0] == 2 else iprot.readString() + ) + else: + iprot.skip(ftype) + else: + iprot.skip(ftype) + iprot.readFieldEnd() + iprot.readStructEnd() + + def write(self, oprot): + if oprot._fast_encode is not None and self.thrift_spec is not None: + oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec])) + return + oprot.writeStructBegin("StoredProcedureRequest") + if self.catName is not None: + oprot.writeFieldBegin("catName", TType.STRING, 1) + oprot.writeString(self.catName.encode("utf-8") if sys.version_info[0] == 2 else self.catName) + oprot.writeFieldEnd() + if self.dbName is not None: + oprot.writeFieldBegin("dbName", TType.STRING, 2) + oprot.writeString(self.dbName.encode("utf-8") if sys.version_info[0] == 2 else self.dbName) + oprot.writeFieldEnd() + if self.procName is not None: + oprot.writeFieldBegin("procName", TType.STRING, 3) + oprot.writeString(self.procName.encode("utf-8") if sys.version_info[0] == 2 else self.procName) + oprot.writeFieldEnd() + oprot.writeFieldStop() + oprot.writeStructEnd() + + def validate(self): + if self.catName is None: + raise TProtocolException(message="Required field catName is unset!") + if self.dbName is None: + raise TProtocolException(message="Required field dbName is unset!") + if self.procName is None: + raise TProtocolException(message="Required field procName is unset!") + return + + def __repr__(self): + L = ["{}={!r}".format(key, value) for key, value in self.__dict__.items()] + return "{}({})".format(self.__class__.__name__, ", ".join(L)) + + def __eq__(self, other): + return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ + + def __ne__(self, other): + return not (self == other) + + +class ListStoredProcedureRequest: + """ + Attributes: + - catName + - dbName + + """ + + def __init__( + self, + catName=None, + dbName=None, + ): + self.catName = catName + self.dbName = dbName + + def read(self, iprot): + if ( + iprot._fast_decode is not None + and isinstance(iprot.trans, TTransport.CReadableTransport) + and self.thrift_spec is not None + ): + iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec]) + return + iprot.readStructBegin() + while True: + (fname, ftype, fid) = iprot.readFieldBegin() + if ftype == TType.STOP: + break + if fid == 1: + if ftype == TType.STRING: + self.catName = ( + iprot.readString().decode("utf-8", errors="replace") if sys.version_info[0] == 2 else iprot.readString() + ) + else: + iprot.skip(ftype) + elif fid == 2: + if ftype == TType.STRING: + self.dbName = ( + iprot.readString().decode("utf-8", errors="replace") if sys.version_info[0] == 2 else iprot.readString() + ) + else: + iprot.skip(ftype) + else: + iprot.skip(ftype) + iprot.readFieldEnd() + iprot.readStructEnd() + + def write(self, oprot): + if oprot._fast_encode is not None and self.thrift_spec is not None: + oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec])) + return + oprot.writeStructBegin("ListStoredProcedureRequest") + if self.catName is not None: + oprot.writeFieldBegin("catName", TType.STRING, 1) + oprot.writeString(self.catName.encode("utf-8") if sys.version_info[0] == 2 else self.catName) + oprot.writeFieldEnd() + if self.dbName is not None: + oprot.writeFieldBegin("dbName", TType.STRING, 2) + oprot.writeString(self.dbName.encode("utf-8") if sys.version_info[0] == 2 else self.dbName) + oprot.writeFieldEnd() + oprot.writeFieldStop() + oprot.writeStructEnd() + + def validate(self): + if self.catName is None: + raise TProtocolException(message="Required field catName is unset!") + return + + def __repr__(self): + L = ["{}={!r}".format(key, value) for key, value in self.__dict__.items()] + return "{}({})".format(self.__class__.__name__, ", ".join(L)) + + def __eq__(self, other): + return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ + + def __ne__(self, other): + return not (self == other) + + +class StoredProcedure: + """ + Attributes: + - name + - dbName + - catName + - ownerName + - source + + """ + + def __init__( + self, + name=None, + dbName=None, + catName=None, + ownerName=None, + source=None, + ): + self.name = name + self.dbName = dbName + self.catName = catName + self.ownerName = ownerName + self.source = source + + def read(self, iprot): + if ( + iprot._fast_decode is not None + and isinstance(iprot.trans, TTransport.CReadableTransport) + and self.thrift_spec is not None + ): + iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec]) + return + iprot.readStructBegin() + while True: + (fname, ftype, fid) = iprot.readFieldBegin() + if ftype == TType.STOP: + break + if fid == 1: + if ftype == TType.STRING: + self.name = ( + iprot.readString().decode("utf-8", errors="replace") if sys.version_info[0] == 2 else iprot.readString() + ) + else: + iprot.skip(ftype) + elif fid == 2: + if ftype == TType.STRING: + self.dbName = ( + iprot.readString().decode("utf-8", errors="replace") if sys.version_info[0] == 2 else iprot.readString() + ) + else: + iprot.skip(ftype) + elif fid == 3: + if ftype == TType.STRING: + self.catName = ( + iprot.readString().decode("utf-8", errors="replace") if sys.version_info[0] == 2 else iprot.readString() + ) + else: + iprot.skip(ftype) + elif fid == 4: + if ftype == TType.STRING: + self.ownerName = ( + iprot.readString().decode("utf-8", errors="replace") if sys.version_info[0] == 2 else iprot.readString() + ) + else: + iprot.skip(ftype) + elif fid == 5: + if ftype == TType.STRING: + self.source = ( + iprot.readString().decode("utf-8", errors="replace") if sys.version_info[0] == 2 else iprot.readString() + ) + else: + iprot.skip(ftype) + else: + iprot.skip(ftype) + iprot.readFieldEnd() + iprot.readStructEnd() + + def write(self, oprot): + if oprot._fast_encode is not None and self.thrift_spec is not None: + oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec])) + return + oprot.writeStructBegin("StoredProcedure") + if self.name is not None: + oprot.writeFieldBegin("name", TType.STRING, 1) + oprot.writeString(self.name.encode("utf-8") if sys.version_info[0] == 2 else self.name) + oprot.writeFieldEnd() + if self.dbName is not None: + oprot.writeFieldBegin("dbName", TType.STRING, 2) + oprot.writeString(self.dbName.encode("utf-8") if sys.version_info[0] == 2 else self.dbName) + oprot.writeFieldEnd() + if self.catName is not None: + oprot.writeFieldBegin("catName", TType.STRING, 3) + oprot.writeString(self.catName.encode("utf-8") if sys.version_info[0] == 2 else self.catName) + oprot.writeFieldEnd() + if self.ownerName is not None: + oprot.writeFieldBegin("ownerName", TType.STRING, 4) + oprot.writeString(self.ownerName.encode("utf-8") if sys.version_info[0] == 2 else self.ownerName) + oprot.writeFieldEnd() + if self.source is not None: + oprot.writeFieldBegin("source", TType.STRING, 5) + oprot.writeString(self.source.encode("utf-8") if sys.version_info[0] == 2 else self.source) + oprot.writeFieldEnd() + oprot.writeFieldStop() + oprot.writeStructEnd() + + def validate(self): + return + + def __repr__(self): + L = ["{}={!r}".format(key, value) for key, value in self.__dict__.items()] + return "{}({})".format(self.__class__.__name__, ", ".join(L)) + + def __eq__(self, other): + return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ + + def __ne__(self, other): + return not (self == other) + + +class AddPackageRequest: + """ + Attributes: + - catName + - dbName + - packageName + - ownerName + - header + - body + + """ + + def __init__( + self, + catName=None, + dbName=None, + packageName=None, + ownerName=None, + header=None, + body=None, + ): + self.catName = catName + self.dbName = dbName + self.packageName = packageName + self.ownerName = ownerName + self.header = header + self.body = body + + def read(self, iprot): + if ( + iprot._fast_decode is not None + and isinstance(iprot.trans, TTransport.CReadableTransport) + and self.thrift_spec is not None + ): + iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec]) + return + iprot.readStructBegin() + while True: + (fname, ftype, fid) = iprot.readFieldBegin() + if ftype == TType.STOP: + break + if fid == 1: + if ftype == TType.STRING: + self.catName = ( + iprot.readString().decode("utf-8", errors="replace") if sys.version_info[0] == 2 else iprot.readString() + ) + else: + iprot.skip(ftype) + elif fid == 2: + if ftype == TType.STRING: + self.dbName = ( + iprot.readString().decode("utf-8", errors="replace") if sys.version_info[0] == 2 else iprot.readString() + ) + else: + iprot.skip(ftype) + elif fid == 3: + if ftype == TType.STRING: + self.packageName = ( + iprot.readString().decode("utf-8", errors="replace") if sys.version_info[0] == 2 else iprot.readString() + ) + else: + iprot.skip(ftype) + elif fid == 4: + if ftype == TType.STRING: + self.ownerName = ( + iprot.readString().decode("utf-8", errors="replace") if sys.version_info[0] == 2 else iprot.readString() + ) + else: + iprot.skip(ftype) + elif fid == 5: + if ftype == TType.STRING: + self.header = ( + iprot.readString().decode("utf-8", errors="replace") if sys.version_info[0] == 2 else iprot.readString() + ) + else: + iprot.skip(ftype) + elif fid == 6: + if ftype == TType.STRING: + self.body = ( + iprot.readString().decode("utf-8", errors="replace") if sys.version_info[0] == 2 else iprot.readString() + ) + else: + iprot.skip(ftype) + else: + iprot.skip(ftype) + iprot.readFieldEnd() + iprot.readStructEnd() + + def write(self, oprot): + if oprot._fast_encode is not None and self.thrift_spec is not None: + oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec])) + return + oprot.writeStructBegin("AddPackageRequest") + if self.catName is not None: + oprot.writeFieldBegin("catName", TType.STRING, 1) + oprot.writeString(self.catName.encode("utf-8") if sys.version_info[0] == 2 else self.catName) + oprot.writeFieldEnd() + if self.dbName is not None: + oprot.writeFieldBegin("dbName", TType.STRING, 2) + oprot.writeString(self.dbName.encode("utf-8") if sys.version_info[0] == 2 else self.dbName) + oprot.writeFieldEnd() + if self.packageName is not None: + oprot.writeFieldBegin("packageName", TType.STRING, 3) + oprot.writeString(self.packageName.encode("utf-8") if sys.version_info[0] == 2 else self.packageName) + oprot.writeFieldEnd() + if self.ownerName is not None: + oprot.writeFieldBegin("ownerName", TType.STRING, 4) + oprot.writeString(self.ownerName.encode("utf-8") if sys.version_info[0] == 2 else self.ownerName) + oprot.writeFieldEnd() + if self.header is not None: + oprot.writeFieldBegin("header", TType.STRING, 5) + oprot.writeString(self.header.encode("utf-8") if sys.version_info[0] == 2 else self.header) + oprot.writeFieldEnd() + if self.body is not None: + oprot.writeFieldBegin("body", TType.STRING, 6) + oprot.writeString(self.body.encode("utf-8") if sys.version_info[0] == 2 else self.body) + oprot.writeFieldEnd() + oprot.writeFieldStop() + oprot.writeStructEnd() + + def validate(self): + return + + def __repr__(self): + L = ["{}={!r}".format(key, value) for key, value in self.__dict__.items()] + return "{}({})".format(self.__class__.__name__, ", ".join(L)) + + def __eq__(self, other): + return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ + + def __ne__(self, other): + return not (self == other) + + +class GetPackageRequest: + """ + Attributes: + - catName + - dbName + - packageName + + """ + + def __init__( + self, + catName=None, + dbName=None, + packageName=None, + ): + self.catName = catName + self.dbName = dbName + self.packageName = packageName + + def read(self, iprot): + if ( + iprot._fast_decode is not None + and isinstance(iprot.trans, TTransport.CReadableTransport) + and self.thrift_spec is not None + ): + iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec]) + return + iprot.readStructBegin() + while True: + (fname, ftype, fid) = iprot.readFieldBegin() + if ftype == TType.STOP: + break + if fid == 1: + if ftype == TType.STRING: + self.catName = ( + iprot.readString().decode("utf-8", errors="replace") if sys.version_info[0] == 2 else iprot.readString() + ) + else: + iprot.skip(ftype) + elif fid == 2: + if ftype == TType.STRING: + self.dbName = ( + iprot.readString().decode("utf-8", errors="replace") if sys.version_info[0] == 2 else iprot.readString() + ) + else: + iprot.skip(ftype) + elif fid == 3: + if ftype == TType.STRING: + self.packageName = ( + iprot.readString().decode("utf-8", errors="replace") if sys.version_info[0] == 2 else iprot.readString() + ) + else: + iprot.skip(ftype) + else: + iprot.skip(ftype) + iprot.readFieldEnd() + iprot.readStructEnd() + + def write(self, oprot): + if oprot._fast_encode is not None and self.thrift_spec is not None: + oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec])) + return + oprot.writeStructBegin("GetPackageRequest") + if self.catName is not None: + oprot.writeFieldBegin("catName", TType.STRING, 1) + oprot.writeString(self.catName.encode("utf-8") if sys.version_info[0] == 2 else self.catName) + oprot.writeFieldEnd() + if self.dbName is not None: + oprot.writeFieldBegin("dbName", TType.STRING, 2) + oprot.writeString(self.dbName.encode("utf-8") if sys.version_info[0] == 2 else self.dbName) + oprot.writeFieldEnd() + if self.packageName is not None: + oprot.writeFieldBegin("packageName", TType.STRING, 3) + oprot.writeString(self.packageName.encode("utf-8") if sys.version_info[0] == 2 else self.packageName) + oprot.writeFieldEnd() + oprot.writeFieldStop() + oprot.writeStructEnd() + + def validate(self): + if self.catName is None: + raise TProtocolException(message="Required field catName is unset!") + if self.dbName is None: + raise TProtocolException(message="Required field dbName is unset!") + if self.packageName is None: + raise TProtocolException(message="Required field packageName is unset!") + return + + def __repr__(self): + L = ["{}={!r}".format(key, value) for key, value in self.__dict__.items()] + return "{}({})".format(self.__class__.__name__, ", ".join(L)) + + def __eq__(self, other): + return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ + + def __ne__(self, other): + return not (self == other) + + +class DropPackageRequest: + """ + Attributes: + - catName + - dbName + - packageName + + """ + + def __init__( + self, + catName=None, + dbName=None, + packageName=None, + ): + self.catName = catName + self.dbName = dbName + self.packageName = packageName + + def read(self, iprot): + if ( + iprot._fast_decode is not None + and isinstance(iprot.trans, TTransport.CReadableTransport) + and self.thrift_spec is not None + ): + iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec]) + return + iprot.readStructBegin() + while True: + (fname, ftype, fid) = iprot.readFieldBegin() + if ftype == TType.STOP: + break + if fid == 1: + if ftype == TType.STRING: + self.catName = ( + iprot.readString().decode("utf-8", errors="replace") if sys.version_info[0] == 2 else iprot.readString() + ) + else: + iprot.skip(ftype) + elif fid == 2: + if ftype == TType.STRING: + self.dbName = ( + iprot.readString().decode("utf-8", errors="replace") if sys.version_info[0] == 2 else iprot.readString() + ) + else: + iprot.skip(ftype) + elif fid == 3: + if ftype == TType.STRING: + self.packageName = ( + iprot.readString().decode("utf-8", errors="replace") if sys.version_info[0] == 2 else iprot.readString() + ) + else: + iprot.skip(ftype) + else: + iprot.skip(ftype) + iprot.readFieldEnd() + iprot.readStructEnd() + + def write(self, oprot): + if oprot._fast_encode is not None and self.thrift_spec is not None: + oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec])) + return + oprot.writeStructBegin("DropPackageRequest") + if self.catName is not None: + oprot.writeFieldBegin("catName", TType.STRING, 1) + oprot.writeString(self.catName.encode("utf-8") if sys.version_info[0] == 2 else self.catName) + oprot.writeFieldEnd() + if self.dbName is not None: + oprot.writeFieldBegin("dbName", TType.STRING, 2) + oprot.writeString(self.dbName.encode("utf-8") if sys.version_info[0] == 2 else self.dbName) + oprot.writeFieldEnd() + if self.packageName is not None: + oprot.writeFieldBegin("packageName", TType.STRING, 3) + oprot.writeString(self.packageName.encode("utf-8") if sys.version_info[0] == 2 else self.packageName) + oprot.writeFieldEnd() + oprot.writeFieldStop() + oprot.writeStructEnd() + + def validate(self): + if self.catName is None: + raise TProtocolException(message="Required field catName is unset!") + if self.dbName is None: + raise TProtocolException(message="Required field dbName is unset!") + if self.packageName is None: + raise TProtocolException(message="Required field packageName is unset!") + return + + def __repr__(self): + L = ["{}={!r}".format(key, value) for key, value in self.__dict__.items()] + return "{}({})".format(self.__class__.__name__, ", ".join(L)) + + def __eq__(self, other): + return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ + + def __ne__(self, other): + return not (self == other) + + +class ListPackageRequest: + """ + Attributes: + - catName + - dbName + + """ + + def __init__( + self, + catName=None, + dbName=None, + ): + self.catName = catName + self.dbName = dbName + + def read(self, iprot): + if ( + iprot._fast_decode is not None + and isinstance(iprot.trans, TTransport.CReadableTransport) + and self.thrift_spec is not None + ): + iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec]) + return + iprot.readStructBegin() + while True: + (fname, ftype, fid) = iprot.readFieldBegin() + if ftype == TType.STOP: + break + if fid == 1: + if ftype == TType.STRING: + self.catName = ( + iprot.readString().decode("utf-8", errors="replace") if sys.version_info[0] == 2 else iprot.readString() + ) + else: + iprot.skip(ftype) + elif fid == 2: + if ftype == TType.STRING: + self.dbName = ( + iprot.readString().decode("utf-8", errors="replace") if sys.version_info[0] == 2 else iprot.readString() + ) + else: + iprot.skip(ftype) + else: + iprot.skip(ftype) + iprot.readFieldEnd() + iprot.readStructEnd() + + def write(self, oprot): + if oprot._fast_encode is not None and self.thrift_spec is not None: + oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec])) + return + oprot.writeStructBegin("ListPackageRequest") + if self.catName is not None: + oprot.writeFieldBegin("catName", TType.STRING, 1) + oprot.writeString(self.catName.encode("utf-8") if sys.version_info[0] == 2 else self.catName) + oprot.writeFieldEnd() + if self.dbName is not None: + oprot.writeFieldBegin("dbName", TType.STRING, 2) + oprot.writeString(self.dbName.encode("utf-8") if sys.version_info[0] == 2 else self.dbName) + oprot.writeFieldEnd() + oprot.writeFieldStop() + oprot.writeStructEnd() + + def validate(self): + if self.catName is None: + raise TProtocolException(message="Required field catName is unset!") + return + + def __repr__(self): + L = ["{}={!r}".format(key, value) for key, value in self.__dict__.items()] + return "{}({})".format(self.__class__.__name__, ", ".join(L)) + + def __eq__(self, other): + return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ + + def __ne__(self, other): + return not (self == other) + + +class Package: + """ + Attributes: + - catName + - dbName + - packageName + - ownerName + - header + - body + + """ + + def __init__( + self, + catName=None, + dbName=None, + packageName=None, + ownerName=None, + header=None, + body=None, + ): + self.catName = catName + self.dbName = dbName + self.packageName = packageName + self.ownerName = ownerName + self.header = header + self.body = body + + def read(self, iprot): + if ( + iprot._fast_decode is not None + and isinstance(iprot.trans, TTransport.CReadableTransport) + and self.thrift_spec is not None + ): + iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec]) + return + iprot.readStructBegin() + while True: + (fname, ftype, fid) = iprot.readFieldBegin() + if ftype == TType.STOP: + break + if fid == 1: + if ftype == TType.STRING: + self.catName = ( + iprot.readString().decode("utf-8", errors="replace") if sys.version_info[0] == 2 else iprot.readString() + ) + else: + iprot.skip(ftype) + elif fid == 2: + if ftype == TType.STRING: + self.dbName = ( + iprot.readString().decode("utf-8", errors="replace") if sys.version_info[0] == 2 else iprot.readString() + ) + else: + iprot.skip(ftype) + elif fid == 3: + if ftype == TType.STRING: + self.packageName = ( + iprot.readString().decode("utf-8", errors="replace") if sys.version_info[0] == 2 else iprot.readString() + ) + else: + iprot.skip(ftype) + elif fid == 4: + if ftype == TType.STRING: + self.ownerName = ( + iprot.readString().decode("utf-8", errors="replace") if sys.version_info[0] == 2 else iprot.readString() + ) + else: + iprot.skip(ftype) + elif fid == 5: + if ftype == TType.STRING: + self.header = ( + iprot.readString().decode("utf-8", errors="replace") if sys.version_info[0] == 2 else iprot.readString() + ) + else: + iprot.skip(ftype) + elif fid == 6: + if ftype == TType.STRING: + self.body = ( + iprot.readString().decode("utf-8", errors="replace") if sys.version_info[0] == 2 else iprot.readString() + ) + else: + iprot.skip(ftype) + else: + iprot.skip(ftype) + iprot.readFieldEnd() + iprot.readStructEnd() + + def write(self, oprot): + if oprot._fast_encode is not None and self.thrift_spec is not None: + oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec])) + return + oprot.writeStructBegin("Package") + if self.catName is not None: + oprot.writeFieldBegin("catName", TType.STRING, 1) + oprot.writeString(self.catName.encode("utf-8") if sys.version_info[0] == 2 else self.catName) + oprot.writeFieldEnd() + if self.dbName is not None: + oprot.writeFieldBegin("dbName", TType.STRING, 2) + oprot.writeString(self.dbName.encode("utf-8") if sys.version_info[0] == 2 else self.dbName) + oprot.writeFieldEnd() + if self.packageName is not None: + oprot.writeFieldBegin("packageName", TType.STRING, 3) + oprot.writeString(self.packageName.encode("utf-8") if sys.version_info[0] == 2 else self.packageName) + oprot.writeFieldEnd() + if self.ownerName is not None: + oprot.writeFieldBegin("ownerName", TType.STRING, 4) + oprot.writeString(self.ownerName.encode("utf-8") if sys.version_info[0] == 2 else self.ownerName) + oprot.writeFieldEnd() + if self.header is not None: + oprot.writeFieldBegin("header", TType.STRING, 5) + oprot.writeString(self.header.encode("utf-8") if sys.version_info[0] == 2 else self.header) + oprot.writeFieldEnd() + if self.body is not None: + oprot.writeFieldBegin("body", TType.STRING, 6) + oprot.writeString(self.body.encode("utf-8") if sys.version_info[0] == 2 else self.body) + oprot.writeFieldEnd() + oprot.writeFieldStop() + oprot.writeStructEnd() + + def validate(self): + return + + def __repr__(self): + L = ["{}={!r}".format(key, value) for key, value in self.__dict__.items()] + return "{}({})".format(self.__class__.__name__, ", ".join(L)) + + def __eq__(self, other): + return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ + + def __ne__(self, other): + return not (self == other) + + +class GetAllWriteEventInfoRequest: + """ + Attributes: + - txnId + - dbName + - tableName + + """ + + def __init__( + self, + txnId=None, + dbName=None, + tableName=None, + ): + self.txnId = txnId + self.dbName = dbName + self.tableName = tableName + + def read(self, iprot): + if ( + iprot._fast_decode is not None + and isinstance(iprot.trans, TTransport.CReadableTransport) + and self.thrift_spec is not None + ): + iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec]) + return + iprot.readStructBegin() + while True: + (fname, ftype, fid) = iprot.readFieldBegin() + if ftype == TType.STOP: + break + if fid == 1: + if ftype == TType.I64: + self.txnId = iprot.readI64() + else: + iprot.skip(ftype) + elif fid == 2: + if ftype == TType.STRING: + self.dbName = ( + iprot.readString().decode("utf-8", errors="replace") if sys.version_info[0] == 2 else iprot.readString() + ) + else: + iprot.skip(ftype) + elif fid == 3: + if ftype == TType.STRING: + self.tableName = ( + iprot.readString().decode("utf-8", errors="replace") if sys.version_info[0] == 2 else iprot.readString() + ) + else: + iprot.skip(ftype) + else: + iprot.skip(ftype) + iprot.readFieldEnd() + iprot.readStructEnd() + + def write(self, oprot): + if oprot._fast_encode is not None and self.thrift_spec is not None: + oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec])) + return + oprot.writeStructBegin("GetAllWriteEventInfoRequest") + if self.txnId is not None: + oprot.writeFieldBegin("txnId", TType.I64, 1) + oprot.writeI64(self.txnId) + oprot.writeFieldEnd() + if self.dbName is not None: + oprot.writeFieldBegin("dbName", TType.STRING, 2) + oprot.writeString(self.dbName.encode("utf-8") if sys.version_info[0] == 2 else self.dbName) + oprot.writeFieldEnd() + if self.tableName is not None: + oprot.writeFieldBegin("tableName", TType.STRING, 3) + oprot.writeString(self.tableName.encode("utf-8") if sys.version_info[0] == 2 else self.tableName) + oprot.writeFieldEnd() + oprot.writeFieldStop() + oprot.writeStructEnd() + + def validate(self): + if self.txnId is None: + raise TProtocolException(message="Required field txnId is unset!") + return + + def __repr__(self): + L = ["{}={!r}".format(key, value) for key, value in self.__dict__.items()] + return "{}({})".format(self.__class__.__name__, ", ".join(L)) + + def __eq__(self, other): + return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ + + def __ne__(self, other): + return not (self == other) + + +class MetaException(TException): + """ + Attributes: + - message + + """ + + def __init__( + self, + message=None, + ): + super().__setattr__("message", message) + + def __setattr__(self, *args): + raise TypeError("can't modify immutable instance") + + def __delattr__(self, *args): + raise TypeError("can't modify immutable instance") + + def __hash__(self): + return hash(self.__class__) ^ hash((self.message,)) + + @classmethod + def read(cls, iprot): + if ( + iprot._fast_decode is not None + and isinstance(iprot.trans, TTransport.CReadableTransport) + and cls.thrift_spec is not None + ): + return iprot._fast_decode(None, iprot, [cls, cls.thrift_spec]) + iprot.readStructBegin() + message = None + while True: + (fname, ftype, fid) = iprot.readFieldBegin() + if ftype == TType.STOP: + break + if fid == 1: + if ftype == TType.STRING: + message = ( + iprot.readString().decode("utf-8", errors="replace") if sys.version_info[0] == 2 else iprot.readString() + ) + else: + iprot.skip(ftype) + else: + iprot.skip(ftype) + iprot.readFieldEnd() + iprot.readStructEnd() + return cls( + message=message, + ) + + def write(self, oprot): + if oprot._fast_encode is not None and self.thrift_spec is not None: + oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec])) + return + oprot.writeStructBegin("MetaException") + if self.message is not None: + oprot.writeFieldBegin("message", TType.STRING, 1) + oprot.writeString(self.message.encode("utf-8") if sys.version_info[0] == 2 else self.message) + oprot.writeFieldEnd() + oprot.writeFieldStop() + oprot.writeStructEnd() + + def validate(self): + return + + def __str__(self): + return repr(self) + + def __repr__(self): + L = ["{}={!r}".format(key, value) for key, value in self.__dict__.items()] + return "{}({})".format(self.__class__.__name__, ", ".join(L)) + + def __eq__(self, other): + return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ + + def __ne__(self, other): + return not (self == other) + + +class UnknownTableException(TException): + """ + Attributes: + - message + + """ + + def __init__( + self, + message=None, + ): + super().__setattr__("message", message) + + def __setattr__(self, *args): + raise TypeError("can't modify immutable instance") + + def __delattr__(self, *args): + raise TypeError("can't modify immutable instance") + + def __hash__(self): + return hash(self.__class__) ^ hash((self.message,)) + + @classmethod + def read(cls, iprot): + if ( + iprot._fast_decode is not None + and isinstance(iprot.trans, TTransport.CReadableTransport) + and cls.thrift_spec is not None + ): + return iprot._fast_decode(None, iprot, [cls, cls.thrift_spec]) + iprot.readStructBegin() + message = None + while True: + (fname, ftype, fid) = iprot.readFieldBegin() + if ftype == TType.STOP: + break + if fid == 1: + if ftype == TType.STRING: + message = ( + iprot.readString().decode("utf-8", errors="replace") if sys.version_info[0] == 2 else iprot.readString() + ) + else: + iprot.skip(ftype) + else: + iprot.skip(ftype) + iprot.readFieldEnd() + iprot.readStructEnd() + return cls( + message=message, + ) + + def write(self, oprot): + if oprot._fast_encode is not None and self.thrift_spec is not None: + oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec])) + return + oprot.writeStructBegin("UnknownTableException") + if self.message is not None: + oprot.writeFieldBegin("message", TType.STRING, 1) + oprot.writeString(self.message.encode("utf-8") if sys.version_info[0] == 2 else self.message) + oprot.writeFieldEnd() + oprot.writeFieldStop() + oprot.writeStructEnd() + + def validate(self): + return + + def __str__(self): + return repr(self) + + def __repr__(self): + L = ["{}={!r}".format(key, value) for key, value in self.__dict__.items()] + return "{}({})".format(self.__class__.__name__, ", ".join(L)) + + def __eq__(self, other): + return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ + + def __ne__(self, other): + return not (self == other) + + +class UnknownDBException(TException): + """ + Attributes: + - message + + """ + + def __init__( + self, + message=None, + ): + super().__setattr__("message", message) + + def __setattr__(self, *args): + raise TypeError("can't modify immutable instance") + + def __delattr__(self, *args): + raise TypeError("can't modify immutable instance") + + def __hash__(self): + return hash(self.__class__) ^ hash((self.message,)) + + @classmethod + def read(cls, iprot): + if ( + iprot._fast_decode is not None + and isinstance(iprot.trans, TTransport.CReadableTransport) + and cls.thrift_spec is not None + ): + return iprot._fast_decode(None, iprot, [cls, cls.thrift_spec]) + iprot.readStructBegin() + message = None + while True: + (fname, ftype, fid) = iprot.readFieldBegin() + if ftype == TType.STOP: + break + if fid == 1: + if ftype == TType.STRING: + message = ( + iprot.readString().decode("utf-8", errors="replace") if sys.version_info[0] == 2 else iprot.readString() + ) + else: + iprot.skip(ftype) + else: + iprot.skip(ftype) + iprot.readFieldEnd() + iprot.readStructEnd() + return cls( + message=message, + ) + + def write(self, oprot): + if oprot._fast_encode is not None and self.thrift_spec is not None: + oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec])) + return + oprot.writeStructBegin("UnknownDBException") + if self.message is not None: + oprot.writeFieldBegin("message", TType.STRING, 1) + oprot.writeString(self.message.encode("utf-8") if sys.version_info[0] == 2 else self.message) + oprot.writeFieldEnd() + oprot.writeFieldStop() + oprot.writeStructEnd() + + def validate(self): + return + + def __str__(self): + return repr(self) + + def __repr__(self): + L = ["{}={!r}".format(key, value) for key, value in self.__dict__.items()] + return "{}({})".format(self.__class__.__name__, ", ".join(L)) + + def __eq__(self, other): + return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ + + def __ne__(self, other): + return not (self == other) + + +class AlreadyExistsException(TException): + """ + Attributes: + - message + + """ + + def __init__( + self, + message=None, + ): + super().__setattr__("message", message) + + def __setattr__(self, *args): + raise TypeError("can't modify immutable instance") + + def __delattr__(self, *args): + raise TypeError("can't modify immutable instance") + + def __hash__(self): + return hash(self.__class__) ^ hash((self.message,)) + + @classmethod + def read(cls, iprot): + if ( + iprot._fast_decode is not None + and isinstance(iprot.trans, TTransport.CReadableTransport) + and cls.thrift_spec is not None + ): + return iprot._fast_decode(None, iprot, [cls, cls.thrift_spec]) + iprot.readStructBegin() + message = None + while True: + (fname, ftype, fid) = iprot.readFieldBegin() + if ftype == TType.STOP: + break + if fid == 1: + if ftype == TType.STRING: + message = ( + iprot.readString().decode("utf-8", errors="replace") if sys.version_info[0] == 2 else iprot.readString() + ) + else: + iprot.skip(ftype) + else: + iprot.skip(ftype) + iprot.readFieldEnd() + iprot.readStructEnd() + return cls( + message=message, + ) + + def write(self, oprot): + if oprot._fast_encode is not None and self.thrift_spec is not None: + oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec])) + return + oprot.writeStructBegin("AlreadyExistsException") + if self.message is not None: + oprot.writeFieldBegin("message", TType.STRING, 1) + oprot.writeString(self.message.encode("utf-8") if sys.version_info[0] == 2 else self.message) + oprot.writeFieldEnd() + oprot.writeFieldStop() + oprot.writeStructEnd() + + def validate(self): + return + + def __str__(self): + return repr(self) + + def __repr__(self): + L = ["{}={!r}".format(key, value) for key, value in self.__dict__.items()] + return "{}({})".format(self.__class__.__name__, ", ".join(L)) + + def __eq__(self, other): + return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ + + def __ne__(self, other): + return not (self == other) + + +class InvalidPartitionException(TException): + """ + Attributes: + - message + + """ + + def __init__( + self, + message=None, + ): + super().__setattr__("message", message) + + def __setattr__(self, *args): + raise TypeError("can't modify immutable instance") + + def __delattr__(self, *args): + raise TypeError("can't modify immutable instance") + + def __hash__(self): + return hash(self.__class__) ^ hash((self.message,)) + + @classmethod + def read(cls, iprot): + if ( + iprot._fast_decode is not None + and isinstance(iprot.trans, TTransport.CReadableTransport) + and cls.thrift_spec is not None + ): + return iprot._fast_decode(None, iprot, [cls, cls.thrift_spec]) + iprot.readStructBegin() + message = None + while True: + (fname, ftype, fid) = iprot.readFieldBegin() + if ftype == TType.STOP: + break + if fid == 1: + if ftype == TType.STRING: + message = ( + iprot.readString().decode("utf-8", errors="replace") if sys.version_info[0] == 2 else iprot.readString() + ) + else: + iprot.skip(ftype) + else: + iprot.skip(ftype) + iprot.readFieldEnd() + iprot.readStructEnd() + return cls( + message=message, + ) + + def write(self, oprot): + if oprot._fast_encode is not None and self.thrift_spec is not None: + oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec])) + return + oprot.writeStructBegin("InvalidPartitionException") + if self.message is not None: + oprot.writeFieldBegin("message", TType.STRING, 1) + oprot.writeString(self.message.encode("utf-8") if sys.version_info[0] == 2 else self.message) + oprot.writeFieldEnd() + oprot.writeFieldStop() + oprot.writeStructEnd() + + def validate(self): + return + + def __str__(self): + return repr(self) + + def __repr__(self): + L = ["{}={!r}".format(key, value) for key, value in self.__dict__.items()] + return "{}({})".format(self.__class__.__name__, ", ".join(L)) + + def __eq__(self, other): + return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ + + def __ne__(self, other): + return not (self == other) + + +class UnknownPartitionException(TException): + """ + Attributes: + - message + + """ + + def __init__( + self, + message=None, + ): + super().__setattr__("message", message) + + def __setattr__(self, *args): + raise TypeError("can't modify immutable instance") + + def __delattr__(self, *args): + raise TypeError("can't modify immutable instance") + + def __hash__(self): + return hash(self.__class__) ^ hash((self.message,)) + + @classmethod + def read(cls, iprot): + if ( + iprot._fast_decode is not None + and isinstance(iprot.trans, TTransport.CReadableTransport) + and cls.thrift_spec is not None + ): + return iprot._fast_decode(None, iprot, [cls, cls.thrift_spec]) + iprot.readStructBegin() + message = None + while True: + (fname, ftype, fid) = iprot.readFieldBegin() + if ftype == TType.STOP: + break + if fid == 1: + if ftype == TType.STRING: + message = ( + iprot.readString().decode("utf-8", errors="replace") if sys.version_info[0] == 2 else iprot.readString() + ) + else: + iprot.skip(ftype) + else: + iprot.skip(ftype) + iprot.readFieldEnd() + iprot.readStructEnd() + return cls( + message=message, + ) + + def write(self, oprot): + if oprot._fast_encode is not None and self.thrift_spec is not None: + oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec])) + return + oprot.writeStructBegin("UnknownPartitionException") + if self.message is not None: + oprot.writeFieldBegin("message", TType.STRING, 1) + oprot.writeString(self.message.encode("utf-8") if sys.version_info[0] == 2 else self.message) + oprot.writeFieldEnd() + oprot.writeFieldStop() + oprot.writeStructEnd() + + def validate(self): + return + + def __str__(self): + return repr(self) + + def __repr__(self): + L = ["{}={!r}".format(key, value) for key, value in self.__dict__.items()] + return "{}({})".format(self.__class__.__name__, ", ".join(L)) + + def __eq__(self, other): + return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ + + def __ne__(self, other): + return not (self == other) + + +class InvalidObjectException(TException): + """ + Attributes: + - message + + """ + + def __init__( + self, + message=None, + ): + super().__setattr__("message", message) + + def __setattr__(self, *args): + raise TypeError("can't modify immutable instance") + + def __delattr__(self, *args): + raise TypeError("can't modify immutable instance") + + def __hash__(self): + return hash(self.__class__) ^ hash((self.message,)) + + @classmethod + def read(cls, iprot): + if ( + iprot._fast_decode is not None + and isinstance(iprot.trans, TTransport.CReadableTransport) + and cls.thrift_spec is not None + ): + return iprot._fast_decode(None, iprot, [cls, cls.thrift_spec]) + iprot.readStructBegin() + message = None + while True: + (fname, ftype, fid) = iprot.readFieldBegin() + if ftype == TType.STOP: + break + if fid == 1: + if ftype == TType.STRING: + message = ( + iprot.readString().decode("utf-8", errors="replace") if sys.version_info[0] == 2 else iprot.readString() + ) + else: + iprot.skip(ftype) + else: + iprot.skip(ftype) + iprot.readFieldEnd() + iprot.readStructEnd() + return cls( + message=message, + ) + + def write(self, oprot): + if oprot._fast_encode is not None and self.thrift_spec is not None: + oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec])) + return + oprot.writeStructBegin("InvalidObjectException") + if self.message is not None: + oprot.writeFieldBegin("message", TType.STRING, 1) + oprot.writeString(self.message.encode("utf-8") if sys.version_info[0] == 2 else self.message) + oprot.writeFieldEnd() + oprot.writeFieldStop() + oprot.writeStructEnd() + + def validate(self): + return + + def __str__(self): + return repr(self) + + def __repr__(self): + L = ["{}={!r}".format(key, value) for key, value in self.__dict__.items()] + return "{}({})".format(self.__class__.__name__, ", ".join(L)) + + def __eq__(self, other): + return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ + + def __ne__(self, other): + return not (self == other) + + +class NoSuchObjectException(TException): + """ + Attributes: + - message + + """ + + def __init__( + self, + message=None, + ): + super().__setattr__("message", message) + + def __setattr__(self, *args): + raise TypeError("can't modify immutable instance") + + def __delattr__(self, *args): + raise TypeError("can't modify immutable instance") + + def __hash__(self): + return hash(self.__class__) ^ hash((self.message,)) + + @classmethod + def read(cls, iprot): + if ( + iprot._fast_decode is not None + and isinstance(iprot.trans, TTransport.CReadableTransport) + and cls.thrift_spec is not None + ): + return iprot._fast_decode(None, iprot, [cls, cls.thrift_spec]) + iprot.readStructBegin() + message = None + while True: + (fname, ftype, fid) = iprot.readFieldBegin() + if ftype == TType.STOP: + break + if fid == 1: + if ftype == TType.STRING: + message = ( + iprot.readString().decode("utf-8", errors="replace") if sys.version_info[0] == 2 else iprot.readString() + ) + else: + iprot.skip(ftype) + else: + iprot.skip(ftype) + iprot.readFieldEnd() + iprot.readStructEnd() + return cls( + message=message, + ) + + def write(self, oprot): + if oprot._fast_encode is not None and self.thrift_spec is not None: + oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec])) + return + oprot.writeStructBegin("NoSuchObjectException") + if self.message is not None: + oprot.writeFieldBegin("message", TType.STRING, 1) + oprot.writeString(self.message.encode("utf-8") if sys.version_info[0] == 2 else self.message) + oprot.writeFieldEnd() + oprot.writeFieldStop() + oprot.writeStructEnd() + + def validate(self): + return + + def __str__(self): + return repr(self) + + def __repr__(self): + L = ["{}={!r}".format(key, value) for key, value in self.__dict__.items()] + return "{}({})".format(self.__class__.__name__, ", ".join(L)) + + def __eq__(self, other): + return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ + + def __ne__(self, other): + return not (self == other) + + +class InvalidOperationException(TException): + """ + Attributes: + - message + + """ + + def __init__( + self, + message=None, + ): + super().__setattr__("message", message) + + def __setattr__(self, *args): + raise TypeError("can't modify immutable instance") + + def __delattr__(self, *args): + raise TypeError("can't modify immutable instance") + + def __hash__(self): + return hash(self.__class__) ^ hash((self.message,)) + + @classmethod + def read(cls, iprot): + if ( + iprot._fast_decode is not None + and isinstance(iprot.trans, TTransport.CReadableTransport) + and cls.thrift_spec is not None + ): + return iprot._fast_decode(None, iprot, [cls, cls.thrift_spec]) + iprot.readStructBegin() + message = None + while True: + (fname, ftype, fid) = iprot.readFieldBegin() + if ftype == TType.STOP: + break + if fid == 1: + if ftype == TType.STRING: + message = ( + iprot.readString().decode("utf-8", errors="replace") if sys.version_info[0] == 2 else iprot.readString() + ) + else: + iprot.skip(ftype) + else: + iprot.skip(ftype) + iprot.readFieldEnd() + iprot.readStructEnd() + return cls( + message=message, + ) + + def write(self, oprot): + if oprot._fast_encode is not None and self.thrift_spec is not None: + oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec])) + return + oprot.writeStructBegin("InvalidOperationException") + if self.message is not None: + oprot.writeFieldBegin("message", TType.STRING, 1) + oprot.writeString(self.message.encode("utf-8") if sys.version_info[0] == 2 else self.message) + oprot.writeFieldEnd() + oprot.writeFieldStop() + oprot.writeStructEnd() + + def validate(self): + return + + def __str__(self): + return repr(self) + + def __repr__(self): + L = ["{}={!r}".format(key, value) for key, value in self.__dict__.items()] + return "{}({})".format(self.__class__.__name__, ", ".join(L)) + + def __eq__(self, other): + return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ + + def __ne__(self, other): + return not (self == other) + + +class ConfigValSecurityException(TException): + """ + Attributes: + - message + + """ + + def __init__( + self, + message=None, + ): + super().__setattr__("message", message) + + def __setattr__(self, *args): + raise TypeError("can't modify immutable instance") + + def __delattr__(self, *args): + raise TypeError("can't modify immutable instance") + + def __hash__(self): + return hash(self.__class__) ^ hash((self.message,)) + + @classmethod + def read(cls, iprot): + if ( + iprot._fast_decode is not None + and isinstance(iprot.trans, TTransport.CReadableTransport) + and cls.thrift_spec is not None + ): + return iprot._fast_decode(None, iprot, [cls, cls.thrift_spec]) + iprot.readStructBegin() + message = None + while True: + (fname, ftype, fid) = iprot.readFieldBegin() + if ftype == TType.STOP: + break + if fid == 1: + if ftype == TType.STRING: + message = ( + iprot.readString().decode("utf-8", errors="replace") if sys.version_info[0] == 2 else iprot.readString() + ) + else: + iprot.skip(ftype) + else: + iprot.skip(ftype) + iprot.readFieldEnd() + iprot.readStructEnd() + return cls( + message=message, + ) + + def write(self, oprot): + if oprot._fast_encode is not None and self.thrift_spec is not None: + oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec])) + return + oprot.writeStructBegin("ConfigValSecurityException") + if self.message is not None: + oprot.writeFieldBegin("message", TType.STRING, 1) + oprot.writeString(self.message.encode("utf-8") if sys.version_info[0] == 2 else self.message) + oprot.writeFieldEnd() + oprot.writeFieldStop() + oprot.writeStructEnd() + + def validate(self): + return + + def __str__(self): + return repr(self) + + def __repr__(self): + L = ["{}={!r}".format(key, value) for key, value in self.__dict__.items()] + return "{}({})".format(self.__class__.__name__, ", ".join(L)) + + def __eq__(self, other): + return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ + + def __ne__(self, other): + return not (self == other) + + +class InvalidInputException(TException): + """ + Attributes: + - message + + """ + + def __init__( + self, + message=None, + ): + super().__setattr__("message", message) + + def __setattr__(self, *args): + raise TypeError("can't modify immutable instance") + + def __delattr__(self, *args): + raise TypeError("can't modify immutable instance") + + def __hash__(self): + return hash(self.__class__) ^ hash((self.message,)) + + @classmethod + def read(cls, iprot): + if ( + iprot._fast_decode is not None + and isinstance(iprot.trans, TTransport.CReadableTransport) + and cls.thrift_spec is not None + ): + return iprot._fast_decode(None, iprot, [cls, cls.thrift_spec]) + iprot.readStructBegin() + message = None + while True: + (fname, ftype, fid) = iprot.readFieldBegin() + if ftype == TType.STOP: + break + if fid == 1: + if ftype == TType.STRING: + message = ( + iprot.readString().decode("utf-8", errors="replace") if sys.version_info[0] == 2 else iprot.readString() + ) + else: + iprot.skip(ftype) + else: + iprot.skip(ftype) + iprot.readFieldEnd() + iprot.readStructEnd() + return cls( + message=message, + ) + + def write(self, oprot): + if oprot._fast_encode is not None and self.thrift_spec is not None: + oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec])) + return + oprot.writeStructBegin("InvalidInputException") + if self.message is not None: + oprot.writeFieldBegin("message", TType.STRING, 1) + oprot.writeString(self.message.encode("utf-8") if sys.version_info[0] == 2 else self.message) + oprot.writeFieldEnd() + oprot.writeFieldStop() + oprot.writeStructEnd() + + def validate(self): + return + + def __str__(self): + return repr(self) + + def __repr__(self): + L = ["{}={!r}".format(key, value) for key, value in self.__dict__.items()] + return "{}({})".format(self.__class__.__name__, ", ".join(L)) + + def __eq__(self, other): + return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ + + def __ne__(self, other): + return not (self == other) + + +class NoSuchTxnException(TException): + """ + Attributes: + - message + + """ + + def __init__( + self, + message=None, + ): + super().__setattr__("message", message) + + def __setattr__(self, *args): + raise TypeError("can't modify immutable instance") + + def __delattr__(self, *args): + raise TypeError("can't modify immutable instance") + + def __hash__(self): + return hash(self.__class__) ^ hash((self.message,)) + + @classmethod + def read(cls, iprot): + if ( + iprot._fast_decode is not None + and isinstance(iprot.trans, TTransport.CReadableTransport) + and cls.thrift_spec is not None + ): + return iprot._fast_decode(None, iprot, [cls, cls.thrift_spec]) + iprot.readStructBegin() + message = None + while True: + (fname, ftype, fid) = iprot.readFieldBegin() + if ftype == TType.STOP: + break + if fid == 1: + if ftype == TType.STRING: + message = ( + iprot.readString().decode("utf-8", errors="replace") if sys.version_info[0] == 2 else iprot.readString() + ) + else: + iprot.skip(ftype) + else: + iprot.skip(ftype) + iprot.readFieldEnd() + iprot.readStructEnd() + return cls( + message=message, + ) + + def write(self, oprot): + if oprot._fast_encode is not None and self.thrift_spec is not None: + oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec])) + return + oprot.writeStructBegin("NoSuchTxnException") + if self.message is not None: + oprot.writeFieldBegin("message", TType.STRING, 1) + oprot.writeString(self.message.encode("utf-8") if sys.version_info[0] == 2 else self.message) + oprot.writeFieldEnd() + oprot.writeFieldStop() + oprot.writeStructEnd() + + def validate(self): + return + + def __str__(self): + return repr(self) + + def __repr__(self): + L = ["{}={!r}".format(key, value) for key, value in self.__dict__.items()] + return "{}({})".format(self.__class__.__name__, ", ".join(L)) + + def __eq__(self, other): + return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ + + def __ne__(self, other): + return not (self == other) + + +class TxnAbortedException(TException): + """ + Attributes: + - message + + """ + + def __init__( + self, + message=None, + ): + super().__setattr__("message", message) + + def __setattr__(self, *args): + raise TypeError("can't modify immutable instance") + + def __delattr__(self, *args): + raise TypeError("can't modify immutable instance") + + def __hash__(self): + return hash(self.__class__) ^ hash((self.message,)) + + @classmethod + def read(cls, iprot): + if ( + iprot._fast_decode is not None + and isinstance(iprot.trans, TTransport.CReadableTransport) + and cls.thrift_spec is not None + ): + return iprot._fast_decode(None, iprot, [cls, cls.thrift_spec]) + iprot.readStructBegin() + message = None + while True: + (fname, ftype, fid) = iprot.readFieldBegin() + if ftype == TType.STOP: + break + if fid == 1: + if ftype == TType.STRING: + message = ( + iprot.readString().decode("utf-8", errors="replace") if sys.version_info[0] == 2 else iprot.readString() + ) + else: + iprot.skip(ftype) + else: + iprot.skip(ftype) + iprot.readFieldEnd() + iprot.readStructEnd() + return cls( + message=message, + ) + + def write(self, oprot): + if oprot._fast_encode is not None and self.thrift_spec is not None: + oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec])) + return + oprot.writeStructBegin("TxnAbortedException") + if self.message is not None: + oprot.writeFieldBegin("message", TType.STRING, 1) + oprot.writeString(self.message.encode("utf-8") if sys.version_info[0] == 2 else self.message) + oprot.writeFieldEnd() + oprot.writeFieldStop() + oprot.writeStructEnd() + + def validate(self): + return + + def __str__(self): + return repr(self) + + def __repr__(self): + L = ["{}={!r}".format(key, value) for key, value in self.__dict__.items()] + return "{}({})".format(self.__class__.__name__, ", ".join(L)) + + def __eq__(self, other): + return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ + + def __ne__(self, other): + return not (self == other) + + +class TxnOpenException(TException): + """ + Attributes: + - message + + """ + + def __init__( + self, + message=None, + ): + super().__setattr__("message", message) + + def __setattr__(self, *args): + raise TypeError("can't modify immutable instance") + + def __delattr__(self, *args): + raise TypeError("can't modify immutable instance") + + def __hash__(self): + return hash(self.__class__) ^ hash((self.message,)) + + @classmethod + def read(cls, iprot): + if ( + iprot._fast_decode is not None + and isinstance(iprot.trans, TTransport.CReadableTransport) + and cls.thrift_spec is not None + ): + return iprot._fast_decode(None, iprot, [cls, cls.thrift_spec]) + iprot.readStructBegin() + message = None + while True: + (fname, ftype, fid) = iprot.readFieldBegin() + if ftype == TType.STOP: + break + if fid == 1: + if ftype == TType.STRING: + message = ( + iprot.readString().decode("utf-8", errors="replace") if sys.version_info[0] == 2 else iprot.readString() + ) + else: + iprot.skip(ftype) + else: + iprot.skip(ftype) + iprot.readFieldEnd() + iprot.readStructEnd() + return cls( + message=message, + ) + + def write(self, oprot): + if oprot._fast_encode is not None and self.thrift_spec is not None: + oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec])) + return + oprot.writeStructBegin("TxnOpenException") + if self.message is not None: + oprot.writeFieldBegin("message", TType.STRING, 1) + oprot.writeString(self.message.encode("utf-8") if sys.version_info[0] == 2 else self.message) + oprot.writeFieldEnd() + oprot.writeFieldStop() + oprot.writeStructEnd() + + def validate(self): + return + + def __str__(self): + return repr(self) + + def __repr__(self): + L = ["{}={!r}".format(key, value) for key, value in self.__dict__.items()] + return "{}({})".format(self.__class__.__name__, ", ".join(L)) + + def __eq__(self, other): + return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ + + def __ne__(self, other): + return not (self == other) + + +class NoSuchLockException(TException): + """ + Attributes: + - message + + """ + + def __init__( + self, + message=None, + ): + super().__setattr__("message", message) + + def __setattr__(self, *args): + raise TypeError("can't modify immutable instance") + + def __delattr__(self, *args): + raise TypeError("can't modify immutable instance") + + def __hash__(self): + return hash(self.__class__) ^ hash((self.message,)) + + @classmethod + def read(cls, iprot): + if ( + iprot._fast_decode is not None + and isinstance(iprot.trans, TTransport.CReadableTransport) + and cls.thrift_spec is not None + ): + return iprot._fast_decode(None, iprot, [cls, cls.thrift_spec]) + iprot.readStructBegin() + message = None + while True: + (fname, ftype, fid) = iprot.readFieldBegin() + if ftype == TType.STOP: + break + if fid == 1: + if ftype == TType.STRING: + message = ( + iprot.readString().decode("utf-8", errors="replace") if sys.version_info[0] == 2 else iprot.readString() + ) + else: + iprot.skip(ftype) + else: + iprot.skip(ftype) + iprot.readFieldEnd() + iprot.readStructEnd() + return cls( + message=message, + ) + + def write(self, oprot): + if oprot._fast_encode is not None and self.thrift_spec is not None: + oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec])) + return + oprot.writeStructBegin("NoSuchLockException") + if self.message is not None: + oprot.writeFieldBegin("message", TType.STRING, 1) + oprot.writeString(self.message.encode("utf-8") if sys.version_info[0] == 2 else self.message) + oprot.writeFieldEnd() + oprot.writeFieldStop() + oprot.writeStructEnd() + + def validate(self): + return + + def __str__(self): + return repr(self) + + def __repr__(self): + L = ["{}={!r}".format(key, value) for key, value in self.__dict__.items()] + return "{}({})".format(self.__class__.__name__, ", ".join(L)) + + def __eq__(self, other): + return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ + + def __ne__(self, other): + return not (self == other) + + +all_structs.append(Version) +Version.thrift_spec = ( + None, # 0 + ( + 1, + TType.STRING, + "version", + "UTF8", + None, + ), # 1 + ( + 2, + TType.STRING, + "comments", + "UTF8", + None, + ), # 2 +) +all_structs.append(FieldSchema) +FieldSchema.thrift_spec = ( + None, # 0 + ( + 1, + TType.STRING, + "name", + "UTF8", + None, + ), # 1 + ( + 2, + TType.STRING, + "type", + "UTF8", + None, + ), # 2 + ( + 3, + TType.STRING, + "comment", + "UTF8", + None, + ), # 3 +) +all_structs.append(EnvironmentContext) +EnvironmentContext.thrift_spec = ( + None, # 0 + ( + 1, + TType.MAP, + "properties", + (TType.STRING, "UTF8", TType.STRING, "UTF8", False), + None, + ), # 1 +) +all_structs.append(SQLPrimaryKey) +SQLPrimaryKey.thrift_spec = ( + None, # 0 + ( + 1, + TType.STRING, + "table_db", + "UTF8", + None, + ), # 1 + ( + 2, + TType.STRING, + "table_name", + "UTF8", + None, + ), # 2 + ( + 3, + TType.STRING, + "column_name", + "UTF8", + None, + ), # 3 + ( + 4, + TType.I32, + "key_seq", + None, + None, + ), # 4 + ( + 5, + TType.STRING, + "pk_name", + "UTF8", + None, + ), # 5 + ( + 6, + TType.BOOL, + "enable_cstr", + None, + None, + ), # 6 + ( + 7, + TType.BOOL, + "validate_cstr", + None, + None, + ), # 7 + ( + 8, + TType.BOOL, + "rely_cstr", + None, + None, + ), # 8 + ( + 9, + TType.STRING, + "catName", + "UTF8", + None, + ), # 9 +) +all_structs.append(SQLForeignKey) +SQLForeignKey.thrift_spec = ( + None, # 0 + ( + 1, + TType.STRING, + "pktable_db", + "UTF8", + None, + ), # 1 + ( + 2, + TType.STRING, + "pktable_name", + "UTF8", + None, + ), # 2 + ( + 3, + TType.STRING, + "pkcolumn_name", + "UTF8", + None, + ), # 3 + ( + 4, + TType.STRING, + "fktable_db", + "UTF8", + None, + ), # 4 + ( + 5, + TType.STRING, + "fktable_name", + "UTF8", + None, + ), # 5 + ( + 6, + TType.STRING, + "fkcolumn_name", + "UTF8", + None, + ), # 6 + ( + 7, + TType.I32, + "key_seq", + None, + None, + ), # 7 + ( + 8, + TType.I32, + "update_rule", + None, + None, + ), # 8 + ( + 9, + TType.I32, + "delete_rule", + None, + None, + ), # 9 + ( + 10, + TType.STRING, + "fk_name", + "UTF8", + None, + ), # 10 + ( + 11, + TType.STRING, + "pk_name", + "UTF8", + None, + ), # 11 + ( + 12, + TType.BOOL, + "enable_cstr", + None, + None, + ), # 12 + ( + 13, + TType.BOOL, + "validate_cstr", + None, + None, + ), # 13 + ( + 14, + TType.BOOL, + "rely_cstr", + None, + None, + ), # 14 + ( + 15, + TType.STRING, + "catName", + "UTF8", + None, + ), # 15 +) +all_structs.append(SQLUniqueConstraint) +SQLUniqueConstraint.thrift_spec = ( + None, # 0 + ( + 1, + TType.STRING, + "catName", + "UTF8", + None, + ), # 1 + ( + 2, + TType.STRING, + "table_db", + "UTF8", + None, + ), # 2 + ( + 3, + TType.STRING, + "table_name", + "UTF8", + None, + ), # 3 + ( + 4, + TType.STRING, + "column_name", + "UTF8", + None, + ), # 4 + ( + 5, + TType.I32, + "key_seq", + None, + None, + ), # 5 + ( + 6, + TType.STRING, + "uk_name", + "UTF8", + None, + ), # 6 + ( + 7, + TType.BOOL, + "enable_cstr", + None, + None, + ), # 7 + ( + 8, + TType.BOOL, + "validate_cstr", + None, + None, + ), # 8 + ( + 9, + TType.BOOL, + "rely_cstr", + None, + None, + ), # 9 +) +all_structs.append(SQLNotNullConstraint) +SQLNotNullConstraint.thrift_spec = ( + None, # 0 + ( + 1, + TType.STRING, + "catName", + "UTF8", + None, + ), # 1 + ( + 2, + TType.STRING, + "table_db", + "UTF8", + None, + ), # 2 + ( + 3, + TType.STRING, + "table_name", + "UTF8", + None, + ), # 3 + ( + 4, + TType.STRING, + "column_name", + "UTF8", + None, + ), # 4 + ( + 5, + TType.STRING, + "nn_name", + "UTF8", + None, + ), # 5 + ( + 6, + TType.BOOL, + "enable_cstr", + None, + None, + ), # 6 + ( + 7, + TType.BOOL, + "validate_cstr", + None, + None, + ), # 7 + ( + 8, + TType.BOOL, + "rely_cstr", + None, + None, + ), # 8 +) +all_structs.append(SQLDefaultConstraint) +SQLDefaultConstraint.thrift_spec = ( + None, # 0 + ( + 1, + TType.STRING, + "catName", + "UTF8", + None, + ), # 1 + ( + 2, + TType.STRING, + "table_db", + "UTF8", + None, + ), # 2 + ( + 3, + TType.STRING, + "table_name", + "UTF8", + None, + ), # 3 + ( + 4, + TType.STRING, + "column_name", + "UTF8", + None, + ), # 4 + ( + 5, + TType.STRING, + "default_value", + "UTF8", + None, + ), # 5 + ( + 6, + TType.STRING, + "dc_name", + "UTF8", + None, + ), # 6 + ( + 7, + TType.BOOL, + "enable_cstr", + None, + None, + ), # 7 + ( + 8, + TType.BOOL, + "validate_cstr", + None, + None, + ), # 8 + ( + 9, + TType.BOOL, + "rely_cstr", + None, + None, + ), # 9 +) +all_structs.append(SQLCheckConstraint) +SQLCheckConstraint.thrift_spec = ( + None, # 0 + ( + 1, + TType.STRING, + "catName", + "UTF8", + None, + ), # 1 + ( + 2, + TType.STRING, + "table_db", + "UTF8", + None, + ), # 2 + ( + 3, + TType.STRING, + "table_name", + "UTF8", + None, + ), # 3 + ( + 4, + TType.STRING, + "column_name", + "UTF8", + None, + ), # 4 + ( + 5, + TType.STRING, + "check_expression", + "UTF8", + None, + ), # 5 + ( + 6, + TType.STRING, + "dc_name", + "UTF8", + None, + ), # 6 + ( + 7, + TType.BOOL, + "enable_cstr", + None, + None, + ), # 7 + ( + 8, + TType.BOOL, + "validate_cstr", + None, + None, + ), # 8 + ( + 9, + TType.BOOL, + "rely_cstr", + None, + None, + ), # 9 +) +all_structs.append(SQLAllTableConstraints) +SQLAllTableConstraints.thrift_spec = ( + None, # 0 + ( + 1, + TType.LIST, + "primaryKeys", + (TType.STRUCT, [SQLPrimaryKey, None], False), + None, + ), # 1 + ( + 2, + TType.LIST, + "foreignKeys", + (TType.STRUCT, [SQLForeignKey, None], False), + None, + ), # 2 + ( + 3, + TType.LIST, + "uniqueConstraints", + (TType.STRUCT, [SQLUniqueConstraint, None], False), + None, + ), # 3 + ( + 4, + TType.LIST, + "notNullConstraints", + (TType.STRUCT, [SQLNotNullConstraint, None], False), + None, + ), # 4 + ( + 5, + TType.LIST, + "defaultConstraints", + (TType.STRUCT, [SQLDefaultConstraint, None], False), + None, + ), # 5 + ( + 6, + TType.LIST, + "checkConstraints", + (TType.STRUCT, [SQLCheckConstraint, None], False), + None, + ), # 6 +) +all_structs.append(Type) +Type.thrift_spec = ( + None, # 0 + ( + 1, + TType.STRING, + "name", + "UTF8", + None, + ), # 1 + ( + 2, + TType.STRING, + "type1", + "UTF8", + None, + ), # 2 + ( + 3, + TType.STRING, + "type2", + "UTF8", + None, + ), # 3 + ( + 4, + TType.LIST, + "fields", + (TType.STRUCT, [FieldSchema, None], False), + None, + ), # 4 +) +all_structs.append(HiveObjectRef) +HiveObjectRef.thrift_spec = ( + None, # 0 + ( + 1, + TType.I32, + "objectType", + None, + None, + ), # 1 + ( + 2, + TType.STRING, + "dbName", + "UTF8", + None, + ), # 2 + ( + 3, + TType.STRING, + "objectName", + "UTF8", + None, + ), # 3 + ( + 4, + TType.LIST, + "partValues", + (TType.STRING, "UTF8", False), + None, + ), # 4 + ( + 5, + TType.STRING, + "columnName", + "UTF8", + None, + ), # 5 + ( + 6, + TType.STRING, + "catName", + "UTF8", + None, + ), # 6 +) +all_structs.append(PrivilegeGrantInfo) +PrivilegeGrantInfo.thrift_spec = ( + None, # 0 + ( + 1, + TType.STRING, + "privilege", + "UTF8", + None, + ), # 1 + ( + 2, + TType.I32, + "createTime", + None, + None, + ), # 2 + ( + 3, + TType.STRING, + "grantor", + "UTF8", + None, + ), # 3 + ( + 4, + TType.I32, + "grantorType", + None, + None, + ), # 4 + ( + 5, + TType.BOOL, + "grantOption", + None, + None, + ), # 5 +) +all_structs.append(HiveObjectPrivilege) +HiveObjectPrivilege.thrift_spec = ( + None, # 0 + ( + 1, + TType.STRUCT, + "hiveObject", + [HiveObjectRef, None], + None, + ), # 1 + ( + 2, + TType.STRING, + "principalName", + "UTF8", + None, + ), # 2 + ( + 3, + TType.I32, + "principalType", + None, + None, + ), # 3 + ( + 4, + TType.STRUCT, + "grantInfo", + [PrivilegeGrantInfo, None], + None, + ), # 4 + ( + 5, + TType.STRING, + "authorizer", + "UTF8", + None, + ), # 5 +) +all_structs.append(PrivilegeBag) +PrivilegeBag.thrift_spec = ( + None, # 0 + ( + 1, + TType.LIST, + "privileges", + (TType.STRUCT, [HiveObjectPrivilege, None], False), + None, + ), # 1 +) +all_structs.append(PrincipalPrivilegeSet) +PrincipalPrivilegeSet.thrift_spec = ( + None, # 0 + ( + 1, + TType.MAP, + "userPrivileges", + (TType.STRING, "UTF8", TType.LIST, (TType.STRUCT, [PrivilegeGrantInfo, None], False), False), + None, + ), # 1 + ( + 2, + TType.MAP, + "groupPrivileges", + (TType.STRING, "UTF8", TType.LIST, (TType.STRUCT, [PrivilegeGrantInfo, None], False), False), + None, + ), # 2 + ( + 3, + TType.MAP, + "rolePrivileges", + (TType.STRING, "UTF8", TType.LIST, (TType.STRUCT, [PrivilegeGrantInfo, None], False), False), + None, + ), # 3 +) +all_structs.append(GrantRevokePrivilegeRequest) +GrantRevokePrivilegeRequest.thrift_spec = ( + None, # 0 + ( + 1, + TType.I32, + "requestType", + None, + None, + ), # 1 + ( + 2, + TType.STRUCT, + "privileges", + [PrivilegeBag, None], + None, + ), # 2 + ( + 3, + TType.BOOL, + "revokeGrantOption", + None, + None, + ), # 3 +) +all_structs.append(GrantRevokePrivilegeResponse) +GrantRevokePrivilegeResponse.thrift_spec = ( + None, # 0 + ( + 1, + TType.BOOL, + "success", + None, + None, + ), # 1 +) +all_structs.append(TruncateTableRequest) +TruncateTableRequest.thrift_spec = ( + None, # 0 + ( + 1, + TType.STRING, + "dbName", + "UTF8", + None, + ), # 1 + ( + 2, + TType.STRING, + "tableName", + "UTF8", + None, + ), # 2 + ( + 3, + TType.LIST, + "partNames", + (TType.STRING, "UTF8", False), + None, + ), # 3 + ( + 4, + TType.I64, + "writeId", + None, + -1, + ), # 4 + ( + 5, + TType.STRING, + "validWriteIdList", + "UTF8", + None, + ), # 5 + ( + 6, + TType.STRUCT, + "environmentContext", + [EnvironmentContext, None], + None, + ), # 6 +) +all_structs.append(TruncateTableResponse) +TruncateTableResponse.thrift_spec = () +all_structs.append(Role) +Role.thrift_spec = ( + None, # 0 + ( + 1, + TType.STRING, + "roleName", + "UTF8", + None, + ), # 1 + ( + 2, + TType.I32, + "createTime", + None, + None, + ), # 2 + ( + 3, + TType.STRING, + "ownerName", + "UTF8", + None, + ), # 3 +) +all_structs.append(RolePrincipalGrant) +RolePrincipalGrant.thrift_spec = ( + None, # 0 + ( + 1, + TType.STRING, + "roleName", + "UTF8", + None, + ), # 1 + ( + 2, + TType.STRING, + "principalName", + "UTF8", + None, + ), # 2 + ( + 3, + TType.I32, + "principalType", + None, + None, + ), # 3 + ( + 4, + TType.BOOL, + "grantOption", + None, + None, + ), # 4 + ( + 5, + TType.I32, + "grantTime", + None, + None, + ), # 5 + ( + 6, + TType.STRING, + "grantorName", + "UTF8", + None, + ), # 6 + ( + 7, + TType.I32, + "grantorPrincipalType", + None, + None, + ), # 7 +) +all_structs.append(GetRoleGrantsForPrincipalRequest) +GetRoleGrantsForPrincipalRequest.thrift_spec = ( + None, # 0 + ( + 1, + TType.STRING, + "principal_name", + "UTF8", + None, + ), # 1 + ( + 2, + TType.I32, + "principal_type", + None, + None, + ), # 2 +) +all_structs.append(GetRoleGrantsForPrincipalResponse) +GetRoleGrantsForPrincipalResponse.thrift_spec = ( + None, # 0 + ( + 1, + TType.LIST, + "principalGrants", + (TType.STRUCT, [RolePrincipalGrant, None], False), + None, + ), # 1 +) +all_structs.append(GetPrincipalsInRoleRequest) +GetPrincipalsInRoleRequest.thrift_spec = ( + None, # 0 + ( + 1, + TType.STRING, + "roleName", + "UTF8", + None, + ), # 1 +) +all_structs.append(GetPrincipalsInRoleResponse) +GetPrincipalsInRoleResponse.thrift_spec = ( + None, # 0 + ( + 1, + TType.LIST, + "principalGrants", + (TType.STRUCT, [RolePrincipalGrant, None], False), + None, + ), # 1 +) +all_structs.append(GrantRevokeRoleRequest) +GrantRevokeRoleRequest.thrift_spec = ( + None, # 0 + ( + 1, + TType.I32, + "requestType", + None, + None, + ), # 1 + ( + 2, + TType.STRING, + "roleName", + "UTF8", + None, + ), # 2 + ( + 3, + TType.STRING, + "principalName", + "UTF8", + None, + ), # 3 + ( + 4, + TType.I32, + "principalType", + None, + None, + ), # 4 + ( + 5, + TType.STRING, + "grantor", + "UTF8", + None, + ), # 5 + ( + 6, + TType.I32, + "grantorType", + None, + None, + ), # 6 + ( + 7, + TType.BOOL, + "grantOption", + None, + None, + ), # 7 +) +all_structs.append(GrantRevokeRoleResponse) +GrantRevokeRoleResponse.thrift_spec = ( + None, # 0 + ( + 1, + TType.BOOL, + "success", + None, + None, + ), # 1 +) +all_structs.append(Catalog) +Catalog.thrift_spec = ( + None, # 0 + ( + 1, + TType.STRING, + "name", + "UTF8", + None, + ), # 1 + ( + 2, + TType.STRING, + "description", + "UTF8", + None, + ), # 2 + ( + 3, + TType.STRING, + "locationUri", + "UTF8", + None, + ), # 3 + ( + 4, + TType.I32, + "createTime", + None, + None, + ), # 4 +) +all_structs.append(CreateCatalogRequest) +CreateCatalogRequest.thrift_spec = ( + None, # 0 + ( + 1, + TType.STRUCT, + "catalog", + [Catalog, None], + None, + ), # 1 +) +all_structs.append(AlterCatalogRequest) +AlterCatalogRequest.thrift_spec = ( + None, # 0 + ( + 1, + TType.STRING, + "name", + "UTF8", + None, + ), # 1 + ( + 2, + TType.STRUCT, + "newCat", + [Catalog, None], + None, + ), # 2 +) +all_structs.append(GetCatalogRequest) +GetCatalogRequest.thrift_spec = ( + None, # 0 + ( + 1, + TType.STRING, + "name", + "UTF8", + None, + ), # 1 +) +all_structs.append(GetCatalogResponse) +GetCatalogResponse.thrift_spec = ( + None, # 0 + ( + 1, + TType.STRUCT, + "catalog", + [Catalog, None], + None, + ), # 1 +) +all_structs.append(GetCatalogsResponse) +GetCatalogsResponse.thrift_spec = ( + None, # 0 + ( + 1, + TType.LIST, + "names", + (TType.STRING, "UTF8", False), + None, + ), # 1 +) +all_structs.append(DropCatalogRequest) +DropCatalogRequest.thrift_spec = ( + None, # 0 + ( + 1, + TType.STRING, + "name", + "UTF8", + None, + ), # 1 +) +all_structs.append(Database) +Database.thrift_spec = ( + None, # 0 + ( + 1, + TType.STRING, + "name", + "UTF8", + None, + ), # 1 + ( + 2, + TType.STRING, + "description", + "UTF8", + None, + ), # 2 + ( + 3, + TType.STRING, + "locationUri", + "UTF8", + None, + ), # 3 + ( + 4, + TType.MAP, + "parameters", + (TType.STRING, "UTF8", TType.STRING, "UTF8", False), + None, + ), # 4 + ( + 5, + TType.STRUCT, + "privileges", + [PrincipalPrivilegeSet, None], + None, + ), # 5 + ( + 6, + TType.STRING, + "ownerName", + "UTF8", + None, + ), # 6 + ( + 7, + TType.I32, + "ownerType", + None, + None, + ), # 7 + ( + 8, + TType.STRING, + "catalogName", + "UTF8", + None, + ), # 8 + ( + 9, + TType.I32, + "createTime", + None, + None, + ), # 9 + ( + 10, + TType.STRING, + "managedLocationUri", + "UTF8", + None, + ), # 10 + ( + 11, + TType.I32, + "type", + None, + None, + ), # 11 + ( + 12, + TType.STRING, + "connector_name", + "UTF8", + None, + ), # 12 + ( + 13, + TType.STRING, + "remote_dbname", + "UTF8", + None, + ), # 13 +) +all_structs.append(SerDeInfo) +SerDeInfo.thrift_spec = ( + None, # 0 + ( + 1, + TType.STRING, + "name", + "UTF8", + None, + ), # 1 + ( + 2, + TType.STRING, + "serializationLib", + "UTF8", + None, + ), # 2 + ( + 3, + TType.MAP, + "parameters", + (TType.STRING, "UTF8", TType.STRING, "UTF8", False), + None, + ), # 3 + ( + 4, + TType.STRING, + "description", + "UTF8", + None, + ), # 4 + ( + 5, + TType.STRING, + "serializerClass", + "UTF8", + None, + ), # 5 + ( + 6, + TType.STRING, + "deserializerClass", + "UTF8", + None, + ), # 6 + ( + 7, + TType.I32, + "serdeType", + None, + None, + ), # 7 +) +all_structs.append(Order) +Order.thrift_spec = ( + None, # 0 + ( + 1, + TType.STRING, + "col", + "UTF8", + None, + ), # 1 + ( + 2, + TType.I32, + "order", + None, + None, + ), # 2 +) +all_structs.append(SkewedInfo) +SkewedInfo.thrift_spec = ( + None, # 0 + ( + 1, + TType.LIST, + "skewedColNames", + (TType.STRING, "UTF8", False), + None, + ), # 1 + ( + 2, + TType.LIST, + "skewedColValues", + (TType.LIST, (TType.STRING, "UTF8", False), False), + None, + ), # 2 + ( + 3, + TType.MAP, + "skewedColValueLocationMaps", + (TType.LIST, (TType.STRING, "UTF8", False), TType.STRING, "UTF8", False), + None, + ), # 3 +) +all_structs.append(StorageDescriptor) +StorageDescriptor.thrift_spec = ( + None, # 0 + ( + 1, + TType.LIST, + "cols", + (TType.STRUCT, [FieldSchema, None], False), + None, + ), # 1 + ( + 2, + TType.STRING, + "location", + "UTF8", + None, + ), # 2 + ( + 3, + TType.STRING, + "inputFormat", + "UTF8", + None, + ), # 3 + ( + 4, + TType.STRING, + "outputFormat", + "UTF8", + None, + ), # 4 + ( + 5, + TType.BOOL, + "compressed", + None, + None, + ), # 5 + ( + 6, + TType.I32, + "numBuckets", + None, + None, + ), # 6 + ( + 7, + TType.STRUCT, + "serdeInfo", + [SerDeInfo, None], + None, + ), # 7 + ( + 8, + TType.LIST, + "bucketCols", + (TType.STRING, "UTF8", False), + None, + ), # 8 + ( + 9, + TType.LIST, + "sortCols", + (TType.STRUCT, [Order, None], False), + None, + ), # 9 + ( + 10, + TType.MAP, + "parameters", + (TType.STRING, "UTF8", TType.STRING, "UTF8", False), + None, + ), # 10 + ( + 11, + TType.STRUCT, + "skewedInfo", + [SkewedInfo, None], + None, + ), # 11 + ( + 12, + TType.BOOL, + "storedAsSubDirectories", + None, + None, + ), # 12 +) +all_structs.append(CreationMetadata) +CreationMetadata.thrift_spec = ( + None, # 0 + ( + 1, + TType.STRING, + "catName", + "UTF8", + None, + ), # 1 + ( + 2, + TType.STRING, + "dbName", + "UTF8", + None, + ), # 2 + ( + 3, + TType.STRING, + "tblName", + "UTF8", + None, + ), # 3 + ( + 4, + TType.SET, + "tablesUsed", + (TType.STRING, "UTF8", False), + None, + ), # 4 + ( + 5, + TType.STRING, + "validTxnList", + "UTF8", + None, + ), # 5 + ( + 6, + TType.I64, + "materializationTime", + None, + None, + ), # 6 + ( + 7, + TType.LIST, + "sourceTables", + (TType.STRUCT, [SourceTable, None], False), + None, + ), # 7 +) +all_structs.append(BooleanColumnStatsData) +BooleanColumnStatsData.thrift_spec = ( + None, # 0 + ( + 1, + TType.I64, + "numTrues", + None, + None, + ), # 1 + ( + 2, + TType.I64, + "numFalses", + None, + None, + ), # 2 + ( + 3, + TType.I64, + "numNulls", + None, + None, + ), # 3 + ( + 4, + TType.STRING, + "bitVectors", + "BINARY", + None, + ), # 4 +) +all_structs.append(DoubleColumnStatsData) +DoubleColumnStatsData.thrift_spec = ( + None, # 0 + ( + 1, + TType.DOUBLE, + "lowValue", + None, + None, + ), # 1 + ( + 2, + TType.DOUBLE, + "highValue", + None, + None, + ), # 2 + ( + 3, + TType.I64, + "numNulls", + None, + None, + ), # 3 + ( + 4, + TType.I64, + "numDVs", + None, + None, + ), # 4 + ( + 5, + TType.STRING, + "bitVectors", + "BINARY", + None, + ), # 5 +) +all_structs.append(LongColumnStatsData) +LongColumnStatsData.thrift_spec = ( + None, # 0 + ( + 1, + TType.I64, + "lowValue", + None, + None, + ), # 1 + ( + 2, + TType.I64, + "highValue", + None, + None, + ), # 2 + ( + 3, + TType.I64, + "numNulls", + None, + None, + ), # 3 + ( + 4, + TType.I64, + "numDVs", + None, + None, + ), # 4 + ( + 5, + TType.STRING, + "bitVectors", + "BINARY", + None, + ), # 5 +) +all_structs.append(StringColumnStatsData) +StringColumnStatsData.thrift_spec = ( + None, # 0 + ( + 1, + TType.I64, + "maxColLen", + None, + None, + ), # 1 + ( + 2, + TType.DOUBLE, + "avgColLen", + None, + None, + ), # 2 + ( + 3, + TType.I64, + "numNulls", + None, + None, + ), # 3 + ( + 4, + TType.I64, + "numDVs", + None, + None, + ), # 4 + ( + 5, + TType.STRING, + "bitVectors", + "BINARY", + None, + ), # 5 +) +all_structs.append(BinaryColumnStatsData) +BinaryColumnStatsData.thrift_spec = ( + None, # 0 + ( + 1, + TType.I64, + "maxColLen", + None, + None, + ), # 1 + ( + 2, + TType.DOUBLE, + "avgColLen", + None, + None, + ), # 2 + ( + 3, + TType.I64, + "numNulls", + None, + None, + ), # 3 + ( + 4, + TType.STRING, + "bitVectors", + "BINARY", + None, + ), # 4 +) +all_structs.append(Decimal) +Decimal.thrift_spec = ( + None, # 0 + ( + 1, + TType.STRING, + "unscaled", + "BINARY", + None, + ), # 1 + None, # 2 + ( + 3, + TType.I16, + "scale", + None, + None, + ), # 3 +) +all_structs.append(DecimalColumnStatsData) +DecimalColumnStatsData.thrift_spec = ( + None, # 0 + ( + 1, + TType.STRUCT, + "lowValue", + [Decimal, None], + None, + ), # 1 + ( + 2, + TType.STRUCT, + "highValue", + [Decimal, None], + None, + ), # 2 + ( + 3, + TType.I64, + "numNulls", + None, + None, + ), # 3 + ( + 4, + TType.I64, + "numDVs", + None, + None, + ), # 4 + ( + 5, + TType.STRING, + "bitVectors", + "BINARY", + None, + ), # 5 +) +all_structs.append(Date) +Date.thrift_spec = ( + None, # 0 + ( + 1, + TType.I64, + "daysSinceEpoch", + None, + None, + ), # 1 +) +all_structs.append(DateColumnStatsData) +DateColumnStatsData.thrift_spec = ( + None, # 0 + ( + 1, + TType.STRUCT, + "lowValue", + [Date, None], + None, + ), # 1 + ( + 2, + TType.STRUCT, + "highValue", + [Date, None], + None, + ), # 2 + ( + 3, + TType.I64, + "numNulls", + None, + None, + ), # 3 + ( + 4, + TType.I64, + "numDVs", + None, + None, + ), # 4 + ( + 5, + TType.STRING, + "bitVectors", + "BINARY", + None, + ), # 5 +) +all_structs.append(Timestamp) +Timestamp.thrift_spec = ( + None, # 0 + ( + 1, + TType.I64, + "secondsSinceEpoch", + None, + None, + ), # 1 +) +all_structs.append(TimestampColumnStatsData) +TimestampColumnStatsData.thrift_spec = ( + None, # 0 + ( + 1, + TType.STRUCT, + "lowValue", + [Timestamp, None], + None, + ), # 1 + ( + 2, + TType.STRUCT, + "highValue", + [Timestamp, None], + None, + ), # 2 + ( + 3, + TType.I64, + "numNulls", + None, + None, + ), # 3 + ( + 4, + TType.I64, + "numDVs", + None, + None, + ), # 4 + ( + 5, + TType.STRING, + "bitVectors", + "BINARY", + None, + ), # 5 +) +all_structs.append(ColumnStatisticsData) +ColumnStatisticsData.thrift_spec = ( + None, # 0 + ( + 1, + TType.STRUCT, + "booleanStats", + [BooleanColumnStatsData, None], + None, + ), # 1 + ( + 2, + TType.STRUCT, + "longStats", + [LongColumnStatsData, None], + None, + ), # 2 + ( + 3, + TType.STRUCT, + "doubleStats", + [DoubleColumnStatsData, None], + None, + ), # 3 + ( + 4, + TType.STRUCT, + "stringStats", + [StringColumnStatsData, None], + None, + ), # 4 + ( + 5, + TType.STRUCT, + "binaryStats", + [BinaryColumnStatsData, None], + None, + ), # 5 + ( + 6, + TType.STRUCT, + "decimalStats", + [DecimalColumnStatsData, None], + None, + ), # 6 + ( + 7, + TType.STRUCT, + "dateStats", + [DateColumnStatsData, None], + None, + ), # 7 + ( + 8, + TType.STRUCT, + "timestampStats", + [TimestampColumnStatsData, None], + None, + ), # 8 +) +all_structs.append(ColumnStatisticsObj) +ColumnStatisticsObj.thrift_spec = ( + None, # 0 + ( + 1, + TType.STRING, + "colName", + "UTF8", + None, + ), # 1 + ( + 2, + TType.STRING, + "colType", + "UTF8", + None, + ), # 2 + ( + 3, + TType.STRUCT, + "statsData", + [ColumnStatisticsData, None], + None, + ), # 3 +) +all_structs.append(ColumnStatisticsDesc) +ColumnStatisticsDesc.thrift_spec = ( + None, # 0 + ( + 1, + TType.BOOL, + "isTblLevel", + None, + None, + ), # 1 + ( + 2, + TType.STRING, + "dbName", + "UTF8", + None, + ), # 2 + ( + 3, + TType.STRING, + "tableName", + "UTF8", + None, + ), # 3 + ( + 4, + TType.STRING, + "partName", + "UTF8", + None, + ), # 4 + ( + 5, + TType.I64, + "lastAnalyzed", + None, + None, + ), # 5 + ( + 6, + TType.STRING, + "catName", + "UTF8", + None, + ), # 6 +) +all_structs.append(ColumnStatistics) +ColumnStatistics.thrift_spec = ( + None, # 0 + ( + 1, + TType.STRUCT, + "statsDesc", + [ColumnStatisticsDesc, None], + None, + ), # 1 + ( + 2, + TType.LIST, + "statsObj", + (TType.STRUCT, [ColumnStatisticsObj, None], False), + None, + ), # 2 + ( + 3, + TType.BOOL, + "isStatsCompliant", + None, + None, + ), # 3 + ( + 4, + TType.STRING, + "engine", + "UTF8", + None, + ), # 4 +) +all_structs.append(FileMetadata) +FileMetadata.thrift_spec = ( + None, # 0 + ( + 1, + TType.BYTE, + "type", + None, + 1, + ), # 1 + ( + 2, + TType.BYTE, + "version", + None, + 1, + ), # 2 + ( + 3, + TType.LIST, + "data", + (TType.STRING, "BINARY", False), + None, + ), # 3 +) +all_structs.append(ObjectDictionary) +ObjectDictionary.thrift_spec = ( + None, # 0 + ( + 1, + TType.MAP, + "values", + (TType.STRING, "UTF8", TType.LIST, (TType.STRING, "BINARY", False), False), + None, + ), # 1 +) +all_structs.append(Table) +Table.thrift_spec = ( + None, # 0 + ( + 1, + TType.STRING, + "tableName", + "UTF8", + None, + ), # 1 + ( + 2, + TType.STRING, + "dbName", + "UTF8", + None, + ), # 2 + ( + 3, + TType.STRING, + "owner", + "UTF8", + None, + ), # 3 + ( + 4, + TType.I32, + "createTime", + None, + None, + ), # 4 + ( + 5, + TType.I32, + "lastAccessTime", + None, + None, + ), # 5 + ( + 6, + TType.I32, + "retention", + None, + None, + ), # 6 + ( + 7, + TType.STRUCT, + "sd", + [StorageDescriptor, None], + None, + ), # 7 + ( + 8, + TType.LIST, + "partitionKeys", + (TType.STRUCT, [FieldSchema, None], False), + None, + ), # 8 + ( + 9, + TType.MAP, + "parameters", + (TType.STRING, "UTF8", TType.STRING, "UTF8", False), + None, + ), # 9 + ( + 10, + TType.STRING, + "viewOriginalText", + "UTF8", + None, + ), # 10 + ( + 11, + TType.STRING, + "viewExpandedText", + "UTF8", + None, + ), # 11 + ( + 12, + TType.STRING, + "tableType", + "UTF8", + None, + ), # 12 + ( + 13, + TType.STRUCT, + "privileges", + [PrincipalPrivilegeSet, None], + None, + ), # 13 + ( + 14, + TType.BOOL, + "temporary", + None, + False, + ), # 14 + ( + 15, + TType.BOOL, + "rewriteEnabled", + None, + None, + ), # 15 + ( + 16, + TType.STRUCT, + "creationMetadata", + [CreationMetadata, None], + None, + ), # 16 + ( + 17, + TType.STRING, + "catName", + "UTF8", + None, + ), # 17 + ( + 18, + TType.I32, + "ownerType", + None, + 1, + ), # 18 + ( + 19, + TType.I64, + "writeId", + None, + -1, + ), # 19 + ( + 20, + TType.BOOL, + "isStatsCompliant", + None, + None, + ), # 20 + ( + 21, + TType.STRUCT, + "colStats", + [ColumnStatistics, None], + None, + ), # 21 + ( + 22, + TType.BYTE, + "accessType", + None, + None, + ), # 22 + ( + 23, + TType.LIST, + "requiredReadCapabilities", + (TType.STRING, "UTF8", False), + None, + ), # 23 + ( + 24, + TType.LIST, + "requiredWriteCapabilities", + (TType.STRING, "UTF8", False), + None, + ), # 24 + ( + 25, + TType.I64, + "id", + None, + None, + ), # 25 + ( + 26, + TType.STRUCT, + "fileMetadata", + [FileMetadata, None], + None, + ), # 26 + ( + 27, + TType.STRUCT, + "dictionary", + [ObjectDictionary, None], + None, + ), # 27 + ( + 28, + TType.I64, + "txnId", + None, + None, + ), # 28 +) +all_structs.append(SourceTable) +SourceTable.thrift_spec = ( + None, # 0 + ( + 1, + TType.STRUCT, + "table", + [Table, None], + None, + ), # 1 + ( + 2, + TType.I64, + "insertedCount", + None, + None, + ), # 2 + ( + 3, + TType.I64, + "updatedCount", + None, + None, + ), # 3 + ( + 4, + TType.I64, + "deletedCount", + None, + None, + ), # 4 +) +all_structs.append(Partition) +Partition.thrift_spec = ( + None, # 0 + ( + 1, + TType.LIST, + "values", + (TType.STRING, "UTF8", False), + None, + ), # 1 + ( + 2, + TType.STRING, + "dbName", + "UTF8", + None, + ), # 2 + ( + 3, + TType.STRING, + "tableName", + "UTF8", + None, + ), # 3 + ( + 4, + TType.I32, + "createTime", + None, + None, + ), # 4 + ( + 5, + TType.I32, + "lastAccessTime", + None, + None, + ), # 5 + ( + 6, + TType.STRUCT, + "sd", + [StorageDescriptor, None], + None, + ), # 6 + ( + 7, + TType.MAP, + "parameters", + (TType.STRING, "UTF8", TType.STRING, "UTF8", False), + None, + ), # 7 + ( + 8, + TType.STRUCT, + "privileges", + [PrincipalPrivilegeSet, None], + None, + ), # 8 + ( + 9, + TType.STRING, + "catName", + "UTF8", + None, + ), # 9 + ( + 10, + TType.I64, + "writeId", + None, + -1, + ), # 10 + ( + 11, + TType.BOOL, + "isStatsCompliant", + None, + None, + ), # 11 + ( + 12, + TType.STRUCT, + "colStats", + [ColumnStatistics, None], + None, + ), # 12 + ( + 13, + TType.STRUCT, + "fileMetadata", + [FileMetadata, None], + None, + ), # 13 +) +all_structs.append(PartitionWithoutSD) +PartitionWithoutSD.thrift_spec = ( + None, # 0 + ( + 1, + TType.LIST, + "values", + (TType.STRING, "UTF8", False), + None, + ), # 1 + ( + 2, + TType.I32, + "createTime", + None, + None, + ), # 2 + ( + 3, + TType.I32, + "lastAccessTime", + None, + None, + ), # 3 + ( + 4, + TType.STRING, + "relativePath", + "UTF8", + None, + ), # 4 + ( + 5, + TType.MAP, + "parameters", + (TType.STRING, "UTF8", TType.STRING, "UTF8", False), + None, + ), # 5 + ( + 6, + TType.STRUCT, + "privileges", + [PrincipalPrivilegeSet, None], + None, + ), # 6 +) +all_structs.append(PartitionSpecWithSharedSD) +PartitionSpecWithSharedSD.thrift_spec = ( + None, # 0 + ( + 1, + TType.LIST, + "partitions", + (TType.STRUCT, [PartitionWithoutSD, None], False), + None, + ), # 1 + ( + 2, + TType.STRUCT, + "sd", + [StorageDescriptor, None], + None, + ), # 2 +) +all_structs.append(PartitionListComposingSpec) +PartitionListComposingSpec.thrift_spec = ( + None, # 0 + ( + 1, + TType.LIST, + "partitions", + (TType.STRUCT, [Partition, None], False), + None, + ), # 1 +) +all_structs.append(PartitionSpec) +PartitionSpec.thrift_spec = ( + None, # 0 + ( + 1, + TType.STRING, + "dbName", + "UTF8", + None, + ), # 1 + ( + 2, + TType.STRING, + "tableName", + "UTF8", + None, + ), # 2 + ( + 3, + TType.STRING, + "rootPath", + "UTF8", + None, + ), # 3 + ( + 4, + TType.STRUCT, + "sharedSDPartitionSpec", + [PartitionSpecWithSharedSD, None], + None, + ), # 4 + ( + 5, + TType.STRUCT, + "partitionList", + [PartitionListComposingSpec, None], + None, + ), # 5 + ( + 6, + TType.STRING, + "catName", + "UTF8", + None, + ), # 6 + ( + 7, + TType.I64, + "writeId", + None, + -1, + ), # 7 + ( + 8, + TType.BOOL, + "isStatsCompliant", + None, + None, + ), # 8 +) +all_structs.append(AggrStats) +AggrStats.thrift_spec = ( + None, # 0 + ( + 1, + TType.LIST, + "colStats", + (TType.STRUCT, [ColumnStatisticsObj, None], False), + None, + ), # 1 + ( + 2, + TType.I64, + "partsFound", + None, + None, + ), # 2 + ( + 3, + TType.BOOL, + "isStatsCompliant", + None, + None, + ), # 3 +) +all_structs.append(SetPartitionsStatsRequest) +SetPartitionsStatsRequest.thrift_spec = ( + None, # 0 + ( + 1, + TType.LIST, + "colStats", + (TType.STRUCT, [ColumnStatistics, None], False), + None, + ), # 1 + ( + 2, + TType.BOOL, + "needMerge", + None, + None, + ), # 2 + ( + 3, + TType.I64, + "writeId", + None, + -1, + ), # 3 + ( + 4, + TType.STRING, + "validWriteIdList", + "UTF8", + None, + ), # 4 + ( + 5, + TType.STRING, + "engine", + "UTF8", + None, + ), # 5 +) +all_structs.append(SetPartitionsStatsResponse) +SetPartitionsStatsResponse.thrift_spec = ( + None, # 0 + ( + 1, + TType.BOOL, + "result", + None, + None, + ), # 1 +) +all_structs.append(Schema) +Schema.thrift_spec = ( + None, # 0 + ( + 1, + TType.LIST, + "fieldSchemas", + (TType.STRUCT, [FieldSchema, None], False), + None, + ), # 1 + ( + 2, + TType.MAP, + "properties", + (TType.STRING, "UTF8", TType.STRING, "UTF8", False), + None, + ), # 2 +) +all_structs.append(PrimaryKeysRequest) +PrimaryKeysRequest.thrift_spec = ( + None, # 0 + ( + 1, + TType.STRING, + "db_name", + "UTF8", + None, + ), # 1 + ( + 2, + TType.STRING, + "tbl_name", + "UTF8", + None, + ), # 2 + ( + 3, + TType.STRING, + "catName", + "UTF8", + None, + ), # 3 + ( + 4, + TType.STRING, + "validWriteIdList", + "UTF8", + None, + ), # 4 + ( + 5, + TType.I64, + "tableId", + None, + -1, + ), # 5 +) +all_structs.append(PrimaryKeysResponse) +PrimaryKeysResponse.thrift_spec = ( + None, # 0 + ( + 1, + TType.LIST, + "primaryKeys", + (TType.STRUCT, [SQLPrimaryKey, None], False), + None, + ), # 1 +) +all_structs.append(ForeignKeysRequest) +ForeignKeysRequest.thrift_spec = ( + None, # 0 + ( + 1, + TType.STRING, + "parent_db_name", + "UTF8", + None, + ), # 1 + ( + 2, + TType.STRING, + "parent_tbl_name", + "UTF8", + None, + ), # 2 + ( + 3, + TType.STRING, + "foreign_db_name", + "UTF8", + None, + ), # 3 + ( + 4, + TType.STRING, + "foreign_tbl_name", + "UTF8", + None, + ), # 4 + ( + 5, + TType.STRING, + "catName", + "UTF8", + None, + ), # 5 + ( + 6, + TType.STRING, + "validWriteIdList", + "UTF8", + None, + ), # 6 + ( + 7, + TType.I64, + "tableId", + None, + -1, + ), # 7 +) +all_structs.append(ForeignKeysResponse) +ForeignKeysResponse.thrift_spec = ( + None, # 0 + ( + 1, + TType.LIST, + "foreignKeys", + (TType.STRUCT, [SQLForeignKey, None], False), + None, + ), # 1 +) +all_structs.append(UniqueConstraintsRequest) +UniqueConstraintsRequest.thrift_spec = ( + None, # 0 + ( + 1, + TType.STRING, + "catName", + "UTF8", + None, + ), # 1 + ( + 2, + TType.STRING, + "db_name", + "UTF8", + None, + ), # 2 + ( + 3, + TType.STRING, + "tbl_name", + "UTF8", + None, + ), # 3 + ( + 4, + TType.STRING, + "validWriteIdList", + "UTF8", + None, + ), # 4 + ( + 5, + TType.I64, + "tableId", + None, + -1, + ), # 5 +) +all_structs.append(UniqueConstraintsResponse) +UniqueConstraintsResponse.thrift_spec = ( + None, # 0 + ( + 1, + TType.LIST, + "uniqueConstraints", + (TType.STRUCT, [SQLUniqueConstraint, None], False), + None, + ), # 1 +) +all_structs.append(NotNullConstraintsRequest) +NotNullConstraintsRequest.thrift_spec = ( + None, # 0 + ( + 1, + TType.STRING, + "catName", + "UTF8", + None, + ), # 1 + ( + 2, + TType.STRING, + "db_name", + "UTF8", + None, + ), # 2 + ( + 3, + TType.STRING, + "tbl_name", + "UTF8", + None, + ), # 3 + ( + 4, + TType.STRING, + "validWriteIdList", + "UTF8", + None, + ), # 4 + ( + 5, + TType.I64, + "tableId", + None, + -1, + ), # 5 +) +all_structs.append(NotNullConstraintsResponse) +NotNullConstraintsResponse.thrift_spec = ( + None, # 0 + ( + 1, + TType.LIST, + "notNullConstraints", + (TType.STRUCT, [SQLNotNullConstraint, None], False), + None, + ), # 1 +) +all_structs.append(DefaultConstraintsRequest) +DefaultConstraintsRequest.thrift_spec = ( + None, # 0 + ( + 1, + TType.STRING, + "catName", + "UTF8", + None, + ), # 1 + ( + 2, + TType.STRING, + "db_name", + "UTF8", + None, + ), # 2 + ( + 3, + TType.STRING, + "tbl_name", + "UTF8", + None, + ), # 3 + ( + 4, + TType.STRING, + "validWriteIdList", + "UTF8", + None, + ), # 4 + ( + 5, + TType.I64, + "tableId", + None, + -1, + ), # 5 +) +all_structs.append(DefaultConstraintsResponse) +DefaultConstraintsResponse.thrift_spec = ( + None, # 0 + ( + 1, + TType.LIST, + "defaultConstraints", + (TType.STRUCT, [SQLDefaultConstraint, None], False), + None, + ), # 1 +) +all_structs.append(CheckConstraintsRequest) +CheckConstraintsRequest.thrift_spec = ( + None, # 0 + ( + 1, + TType.STRING, + "catName", + "UTF8", + None, + ), # 1 + ( + 2, + TType.STRING, + "db_name", + "UTF8", + None, + ), # 2 + ( + 3, + TType.STRING, + "tbl_name", + "UTF8", + None, + ), # 3 + ( + 4, + TType.STRING, + "validWriteIdList", + "UTF8", + None, + ), # 4 + ( + 5, + TType.I64, + "tableId", + None, + -1, + ), # 5 +) +all_structs.append(CheckConstraintsResponse) +CheckConstraintsResponse.thrift_spec = ( + None, # 0 + ( + 1, + TType.LIST, + "checkConstraints", + (TType.STRUCT, [SQLCheckConstraint, None], False), + None, + ), # 1 +) +all_structs.append(AllTableConstraintsRequest) +AllTableConstraintsRequest.thrift_spec = ( + None, # 0 + ( + 1, + TType.STRING, + "dbName", + "UTF8", + None, + ), # 1 + ( + 2, + TType.STRING, + "tblName", + "UTF8", + None, + ), # 2 + ( + 3, + TType.STRING, + "catName", + "UTF8", + None, + ), # 3 + ( + 4, + TType.STRING, + "validWriteIdList", + "UTF8", + None, + ), # 4 + ( + 5, + TType.I64, + "tableId", + None, + -1, + ), # 5 +) +all_structs.append(AllTableConstraintsResponse) +AllTableConstraintsResponse.thrift_spec = ( + None, # 0 + ( + 1, + TType.STRUCT, + "allTableConstraints", + [SQLAllTableConstraints, None], + None, + ), # 1 +) +all_structs.append(DropConstraintRequest) +DropConstraintRequest.thrift_spec = ( + None, # 0 + ( + 1, + TType.STRING, + "dbname", + "UTF8", + None, + ), # 1 + ( + 2, + TType.STRING, + "tablename", + "UTF8", + None, + ), # 2 + ( + 3, + TType.STRING, + "constraintname", + "UTF8", + None, + ), # 3 + ( + 4, + TType.STRING, + "catName", + "UTF8", + None, + ), # 4 +) +all_structs.append(AddPrimaryKeyRequest) +AddPrimaryKeyRequest.thrift_spec = ( + None, # 0 + ( + 1, + TType.LIST, + "primaryKeyCols", + (TType.STRUCT, [SQLPrimaryKey, None], False), + None, + ), # 1 +) +all_structs.append(AddForeignKeyRequest) +AddForeignKeyRequest.thrift_spec = ( + None, # 0 + ( + 1, + TType.LIST, + "foreignKeyCols", + (TType.STRUCT, [SQLForeignKey, None], False), + None, + ), # 1 +) +all_structs.append(AddUniqueConstraintRequest) +AddUniqueConstraintRequest.thrift_spec = ( + None, # 0 + ( + 1, + TType.LIST, + "uniqueConstraintCols", + (TType.STRUCT, [SQLUniqueConstraint, None], False), + None, + ), # 1 +) +all_structs.append(AddNotNullConstraintRequest) +AddNotNullConstraintRequest.thrift_spec = ( + None, # 0 + ( + 1, + TType.LIST, + "notNullConstraintCols", + (TType.STRUCT, [SQLNotNullConstraint, None], False), + None, + ), # 1 +) +all_structs.append(AddDefaultConstraintRequest) +AddDefaultConstraintRequest.thrift_spec = ( + None, # 0 + ( + 1, + TType.LIST, + "defaultConstraintCols", + (TType.STRUCT, [SQLDefaultConstraint, None], False), + None, + ), # 1 +) +all_structs.append(AddCheckConstraintRequest) +AddCheckConstraintRequest.thrift_spec = ( + None, # 0 + ( + 1, + TType.LIST, + "checkConstraintCols", + (TType.STRUCT, [SQLCheckConstraint, None], False), + None, + ), # 1 +) +all_structs.append(PartitionsByExprResult) +PartitionsByExprResult.thrift_spec = ( + None, # 0 + ( + 1, + TType.LIST, + "partitions", + (TType.STRUCT, [Partition, None], False), + None, + ), # 1 + ( + 2, + TType.BOOL, + "hasUnknownPartitions", + None, + None, + ), # 2 +) +all_structs.append(PartitionsSpecByExprResult) +PartitionsSpecByExprResult.thrift_spec = ( + None, # 0 + ( + 1, + TType.LIST, + "partitionsSpec", + (TType.STRUCT, [PartitionSpec, None], False), + None, + ), # 1 + ( + 2, + TType.BOOL, + "hasUnknownPartitions", + None, + None, + ), # 2 +) +all_structs.append(PartitionsByExprRequest) +PartitionsByExprRequest.thrift_spec = ( + None, # 0 + ( + 1, + TType.STRING, + "dbName", + "UTF8", + None, + ), # 1 + ( + 2, + TType.STRING, + "tblName", + "UTF8", + None, + ), # 2 + ( + 3, + TType.STRING, + "expr", + "BINARY", + None, + ), # 3 + ( + 4, + TType.STRING, + "defaultPartitionName", + "UTF8", + None, + ), # 4 + ( + 5, + TType.I16, + "maxParts", + None, + -1, + ), # 5 + ( + 6, + TType.STRING, + "catName", + "UTF8", + None, + ), # 6 + ( + 7, + TType.STRING, + "order", + "UTF8", + None, + ), # 7 + ( + 8, + TType.STRING, + "validWriteIdList", + "UTF8", + None, + ), # 8 + ( + 9, + TType.I64, + "id", + None, + -1, + ), # 9 +) +all_structs.append(TableStatsResult) +TableStatsResult.thrift_spec = ( + None, # 0 + ( + 1, + TType.LIST, + "tableStats", + (TType.STRUCT, [ColumnStatisticsObj, None], False), + None, + ), # 1 + ( + 2, + TType.BOOL, + "isStatsCompliant", + None, + None, + ), # 2 +) +all_structs.append(PartitionsStatsResult) +PartitionsStatsResult.thrift_spec = ( + None, # 0 + ( + 1, + TType.MAP, + "partStats", + (TType.STRING, "UTF8", TType.LIST, (TType.STRUCT, [ColumnStatisticsObj, None], False), False), + None, + ), # 1 + ( + 2, + TType.BOOL, + "isStatsCompliant", + None, + None, + ), # 2 +) +all_structs.append(TableStatsRequest) +TableStatsRequest.thrift_spec = ( + None, # 0 + ( + 1, + TType.STRING, + "dbName", + "UTF8", + None, + ), # 1 + ( + 2, + TType.STRING, + "tblName", + "UTF8", + None, + ), # 2 + ( + 3, + TType.LIST, + "colNames", + (TType.STRING, "UTF8", False), + None, + ), # 3 + ( + 4, + TType.STRING, + "catName", + "UTF8", + None, + ), # 4 + ( + 5, + TType.STRING, + "validWriteIdList", + "UTF8", + None, + ), # 5 + ( + 6, + TType.STRING, + "engine", + "UTF8", + None, + ), # 6 + ( + 7, + TType.I64, + "id", + None, + -1, + ), # 7 +) +all_structs.append(PartitionsStatsRequest) +PartitionsStatsRequest.thrift_spec = ( + None, # 0 + ( + 1, + TType.STRING, + "dbName", + "UTF8", + None, + ), # 1 + ( + 2, + TType.STRING, + "tblName", + "UTF8", + None, + ), # 2 + ( + 3, + TType.LIST, + "colNames", + (TType.STRING, "UTF8", False), + None, + ), # 3 + ( + 4, + TType.LIST, + "partNames", + (TType.STRING, "UTF8", False), + None, + ), # 4 + ( + 5, + TType.STRING, + "catName", + "UTF8", + None, + ), # 5 + ( + 6, + TType.STRING, + "validWriteIdList", + "UTF8", + None, + ), # 6 + ( + 7, + TType.STRING, + "engine", + "UTF8", + None, + ), # 7 +) +all_structs.append(AddPartitionsResult) +AddPartitionsResult.thrift_spec = ( + None, # 0 + ( + 1, + TType.LIST, + "partitions", + (TType.STRUCT, [Partition, None], False), + None, + ), # 1 + ( + 2, + TType.BOOL, + "isStatsCompliant", + None, + None, + ), # 2 +) +all_structs.append(AddPartitionsRequest) +AddPartitionsRequest.thrift_spec = ( + None, # 0 + ( + 1, + TType.STRING, + "dbName", + "UTF8", + None, + ), # 1 + ( + 2, + TType.STRING, + "tblName", + "UTF8", + None, + ), # 2 + ( + 3, + TType.LIST, + "parts", + (TType.STRUCT, [Partition, None], False), + None, + ), # 3 + ( + 4, + TType.BOOL, + "ifNotExists", + None, + None, + ), # 4 + ( + 5, + TType.BOOL, + "needResult", + None, + True, + ), # 5 + ( + 6, + TType.STRING, + "catName", + "UTF8", + None, + ), # 6 + ( + 7, + TType.STRING, + "validWriteIdList", + "UTF8", + None, + ), # 7 +) +all_structs.append(DropPartitionsResult) +DropPartitionsResult.thrift_spec = ( + None, # 0 + ( + 1, + TType.LIST, + "partitions", + (TType.STRUCT, [Partition, None], False), + None, + ), # 1 +) +all_structs.append(DropPartitionsExpr) +DropPartitionsExpr.thrift_spec = ( + None, # 0 + ( + 1, + TType.STRING, + "expr", + "BINARY", + None, + ), # 1 + ( + 2, + TType.I32, + "partArchiveLevel", + None, + None, + ), # 2 +) +all_structs.append(RequestPartsSpec) +RequestPartsSpec.thrift_spec = ( + None, # 0 + ( + 1, + TType.LIST, + "names", + (TType.STRING, "UTF8", False), + None, + ), # 1 + ( + 2, + TType.LIST, + "exprs", + (TType.STRUCT, [DropPartitionsExpr, None], False), + None, + ), # 2 +) +all_structs.append(DropPartitionsRequest) +DropPartitionsRequest.thrift_spec = ( + None, # 0 + ( + 1, + TType.STRING, + "dbName", + "UTF8", + None, + ), # 1 + ( + 2, + TType.STRING, + "tblName", + "UTF8", + None, + ), # 2 + ( + 3, + TType.STRUCT, + "parts", + [RequestPartsSpec, None], + None, + ), # 3 + ( + 4, + TType.BOOL, + "deleteData", + None, + None, + ), # 4 + ( + 5, + TType.BOOL, + "ifExists", + None, + True, + ), # 5 + ( + 6, + TType.BOOL, + "ignoreProtection", + None, + None, + ), # 6 + ( + 7, + TType.STRUCT, + "environmentContext", + [EnvironmentContext, None], + None, + ), # 7 + ( + 8, + TType.BOOL, + "needResult", + None, + True, + ), # 8 + ( + 9, + TType.STRING, + "catName", + "UTF8", + None, + ), # 9 +) +all_structs.append(PartitionValuesRequest) +PartitionValuesRequest.thrift_spec = ( + None, # 0 + ( + 1, + TType.STRING, + "dbName", + "UTF8", + None, + ), # 1 + ( + 2, + TType.STRING, + "tblName", + "UTF8", + None, + ), # 2 + ( + 3, + TType.LIST, + "partitionKeys", + (TType.STRUCT, [FieldSchema, None], False), + None, + ), # 3 + ( + 4, + TType.BOOL, + "applyDistinct", + None, + True, + ), # 4 + ( + 5, + TType.STRING, + "filter", + "UTF8", + None, + ), # 5 + ( + 6, + TType.LIST, + "partitionOrder", + (TType.STRUCT, [FieldSchema, None], False), + None, + ), # 6 + ( + 7, + TType.BOOL, + "ascending", + None, + True, + ), # 7 + ( + 8, + TType.I64, + "maxParts", + None, + -1, + ), # 8 + ( + 9, + TType.STRING, + "catName", + "UTF8", + None, + ), # 9 + ( + 10, + TType.STRING, + "validWriteIdList", + "UTF8", + None, + ), # 10 +) +all_structs.append(PartitionValuesRow) +PartitionValuesRow.thrift_spec = ( + None, # 0 + ( + 1, + TType.LIST, + "row", + (TType.STRING, "UTF8", False), + None, + ), # 1 +) +all_structs.append(PartitionValuesResponse) +PartitionValuesResponse.thrift_spec = ( + None, # 0 + ( + 1, + TType.LIST, + "partitionValues", + (TType.STRUCT, [PartitionValuesRow, None], False), + None, + ), # 1 +) +all_structs.append(GetPartitionsByNamesRequest) +GetPartitionsByNamesRequest.thrift_spec = ( + None, # 0 + ( + 1, + TType.STRING, + "db_name", + "UTF8", + None, + ), # 1 + ( + 2, + TType.STRING, + "tbl_name", + "UTF8", + None, + ), # 2 + ( + 3, + TType.LIST, + "names", + (TType.STRING, "UTF8", False), + None, + ), # 3 + ( + 4, + TType.BOOL, + "get_col_stats", + None, + None, + ), # 4 + ( + 5, + TType.LIST, + "processorCapabilities", + (TType.STRING, "UTF8", False), + None, + ), # 5 + ( + 6, + TType.STRING, + "processorIdentifier", + "UTF8", + None, + ), # 6 + ( + 7, + TType.STRING, + "engine", + "UTF8", + None, + ), # 7 + ( + 8, + TType.STRING, + "validWriteIdList", + "UTF8", + None, + ), # 8 + ( + 9, + TType.BOOL, + "getFileMetadata", + None, + None, + ), # 9 + ( + 10, + TType.I64, + "id", + None, + -1, + ), # 10 +) +all_structs.append(GetPartitionsByNamesResult) +GetPartitionsByNamesResult.thrift_spec = ( + None, # 0 + ( + 1, + TType.LIST, + "partitions", + (TType.STRUCT, [Partition, None], False), + None, + ), # 1 + ( + 2, + TType.STRUCT, + "dictionary", + [ObjectDictionary, None], + None, + ), # 2 +) +all_structs.append(DataConnector) +DataConnector.thrift_spec = ( + None, # 0 + ( + 1, + TType.STRING, + "name", + "UTF8", + None, + ), # 1 + ( + 2, + TType.STRING, + "type", + "UTF8", + None, + ), # 2 + ( + 3, + TType.STRING, + "url", + "UTF8", + None, + ), # 3 + ( + 4, + TType.STRING, + "description", + "UTF8", + None, + ), # 4 + ( + 5, + TType.MAP, + "parameters", + (TType.STRING, "UTF8", TType.STRING, "UTF8", False), + None, + ), # 5 + ( + 6, + TType.STRING, + "ownerName", + "UTF8", + None, + ), # 6 + ( + 7, + TType.I32, + "ownerType", + None, + None, + ), # 7 + ( + 8, + TType.I32, + "createTime", + None, + None, + ), # 8 +) +all_structs.append(ResourceUri) +ResourceUri.thrift_spec = ( + None, # 0 + ( + 1, + TType.I32, + "resourceType", + None, + None, + ), # 1 + ( + 2, + TType.STRING, + "uri", + "UTF8", + None, + ), # 2 +) +all_structs.append(Function) +Function.thrift_spec = ( + None, # 0 + ( + 1, + TType.STRING, + "functionName", + "UTF8", + None, + ), # 1 + ( + 2, + TType.STRING, + "dbName", + "UTF8", + None, + ), # 2 + ( + 3, + TType.STRING, + "className", + "UTF8", + None, + ), # 3 + ( + 4, + TType.STRING, + "ownerName", + "UTF8", + None, + ), # 4 + ( + 5, + TType.I32, + "ownerType", + None, + None, + ), # 5 + ( + 6, + TType.I32, + "createTime", + None, + None, + ), # 6 + ( + 7, + TType.I32, + "functionType", + None, + None, + ), # 7 + ( + 8, + TType.LIST, + "resourceUris", + (TType.STRUCT, [ResourceUri, None], False), + None, + ), # 8 + ( + 9, + TType.STRING, + "catName", + "UTF8", + None, + ), # 9 +) +all_structs.append(TxnInfo) +TxnInfo.thrift_spec = ( + None, # 0 + ( + 1, + TType.I64, + "id", + None, + None, + ), # 1 + ( + 2, + TType.I32, + "state", + None, + None, + ), # 2 + ( + 3, + TType.STRING, + "user", + "UTF8", + None, + ), # 3 + ( + 4, + TType.STRING, + "hostname", + "UTF8", + None, + ), # 4 + ( + 5, + TType.STRING, + "agentInfo", + "UTF8", + "Unknown", + ), # 5 + ( + 6, + TType.I32, + "heartbeatCount", + None, + 0, + ), # 6 + ( + 7, + TType.STRING, + "metaInfo", + "UTF8", + None, + ), # 7 + ( + 8, + TType.I64, + "startedTime", + None, + None, + ), # 8 + ( + 9, + TType.I64, + "lastHeartbeatTime", + None, + None, + ), # 9 +) +all_structs.append(GetOpenTxnsInfoResponse) +GetOpenTxnsInfoResponse.thrift_spec = ( + None, # 0 + ( + 1, + TType.I64, + "txn_high_water_mark", + None, + None, + ), # 1 + ( + 2, + TType.LIST, + "open_txns", + (TType.STRUCT, [TxnInfo, None], False), + None, + ), # 2 +) +all_structs.append(GetOpenTxnsResponse) +GetOpenTxnsResponse.thrift_spec = ( + None, # 0 + ( + 1, + TType.I64, + "txn_high_water_mark", + None, + None, + ), # 1 + ( + 2, + TType.LIST, + "open_txns", + (TType.I64, None, False), + None, + ), # 2 + ( + 3, + TType.I64, + "min_open_txn", + None, + None, + ), # 3 + ( + 4, + TType.STRING, + "abortedBits", + "BINARY", + None, + ), # 4 +) +all_structs.append(OpenTxnRequest) +OpenTxnRequest.thrift_spec = ( + None, # 0 + ( + 1, + TType.I32, + "num_txns", + None, + None, + ), # 1 + ( + 2, + TType.STRING, + "user", + "UTF8", + None, + ), # 2 + ( + 3, + TType.STRING, + "hostname", + "UTF8", + None, + ), # 3 + ( + 4, + TType.STRING, + "agentInfo", + "UTF8", + "Unknown", + ), # 4 + ( + 5, + TType.STRING, + "replPolicy", + "UTF8", + None, + ), # 5 + ( + 6, + TType.LIST, + "replSrcTxnIds", + (TType.I64, None, False), + None, + ), # 6 + ( + 7, + TType.I32, + "txn_type", + None, + 0, + ), # 7 +) +all_structs.append(OpenTxnsResponse) +OpenTxnsResponse.thrift_spec = ( + None, # 0 + ( + 1, + TType.LIST, + "txn_ids", + (TType.I64, None, False), + None, + ), # 1 +) +all_structs.append(AbortTxnRequest) +AbortTxnRequest.thrift_spec = ( + None, # 0 + ( + 1, + TType.I64, + "txnid", + None, + None, + ), # 1 + ( + 2, + TType.STRING, + "replPolicy", + "UTF8", + None, + ), # 2 + ( + 3, + TType.I32, + "txn_type", + None, + None, + ), # 3 +) +all_structs.append(AbortTxnsRequest) +AbortTxnsRequest.thrift_spec = ( + None, # 0 + ( + 1, + TType.LIST, + "txn_ids", + (TType.I64, None, False), + None, + ), # 1 +) +all_structs.append(CommitTxnKeyValue) +CommitTxnKeyValue.thrift_spec = ( + None, # 0 + ( + 1, + TType.I64, + "tableId", + None, + None, + ), # 1 + ( + 2, + TType.STRING, + "key", + "UTF8", + None, + ), # 2 + ( + 3, + TType.STRING, + "value", + "UTF8", + None, + ), # 3 +) +all_structs.append(WriteEventInfo) +WriteEventInfo.thrift_spec = ( + None, # 0 + ( + 1, + TType.I64, + "writeId", + None, + None, + ), # 1 + ( + 2, + TType.STRING, + "database", + "UTF8", + None, + ), # 2 + ( + 3, + TType.STRING, + "table", + "UTF8", + None, + ), # 3 + ( + 4, + TType.STRING, + "files", + "UTF8", + None, + ), # 4 + ( + 5, + TType.STRING, + "partition", + "UTF8", + None, + ), # 5 + ( + 6, + TType.STRING, + "tableObj", + "UTF8", + None, + ), # 6 + ( + 7, + TType.STRING, + "partitionObj", + "UTF8", + None, + ), # 7 +) +all_structs.append(ReplLastIdInfo) +ReplLastIdInfo.thrift_spec = ( + None, # 0 + ( + 1, + TType.STRING, + "database", + "UTF8", + None, + ), # 1 + ( + 2, + TType.I64, + "lastReplId", + None, + None, + ), # 2 + ( + 3, + TType.STRING, + "table", + "UTF8", + None, + ), # 3 + ( + 4, + TType.STRING, + "catalog", + "UTF8", + None, + ), # 4 + ( + 5, + TType.LIST, + "partitionList", + (TType.STRING, "UTF8", False), + None, + ), # 5 +) +all_structs.append(UpdateTransactionalStatsRequest) +UpdateTransactionalStatsRequest.thrift_spec = ( + None, # 0 + ( + 1, + TType.I64, + "tableId", + None, + None, + ), # 1 + ( + 2, + TType.I64, + "insertCount", + None, + None, + ), # 2 + ( + 3, + TType.I64, + "updatedCount", + None, + None, + ), # 3 + ( + 4, + TType.I64, + "deletedCount", + None, + None, + ), # 4 +) +all_structs.append(CommitTxnRequest) +CommitTxnRequest.thrift_spec = ( + None, # 0 + ( + 1, + TType.I64, + "txnid", + None, + None, + ), # 1 + ( + 2, + TType.STRING, + "replPolicy", + "UTF8", + None, + ), # 2 + ( + 3, + TType.LIST, + "writeEventInfos", + (TType.STRUCT, [WriteEventInfo, None], False), + None, + ), # 3 + ( + 4, + TType.STRUCT, + "replLastIdInfo", + [ReplLastIdInfo, None], + None, + ), # 4 + ( + 5, + TType.STRUCT, + "keyValue", + [CommitTxnKeyValue, None], + None, + ), # 5 + ( + 6, + TType.BOOL, + "exclWriteEnabled", + None, + True, + ), # 6 + ( + 7, + TType.I32, + "txn_type", + None, + None, + ), # 7 +) +all_structs.append(ReplTblWriteIdStateRequest) +ReplTblWriteIdStateRequest.thrift_spec = ( + None, # 0 + ( + 1, + TType.STRING, + "validWriteIdlist", + "UTF8", + None, + ), # 1 + ( + 2, + TType.STRING, + "user", + "UTF8", + None, + ), # 2 + ( + 3, + TType.STRING, + "hostName", + "UTF8", + None, + ), # 3 + ( + 4, + TType.STRING, + "dbName", + "UTF8", + None, + ), # 4 + ( + 5, + TType.STRING, + "tableName", + "UTF8", + None, + ), # 5 + ( + 6, + TType.LIST, + "partNames", + (TType.STRING, "UTF8", False), + None, + ), # 6 +) +all_structs.append(GetValidWriteIdsRequest) +GetValidWriteIdsRequest.thrift_spec = ( + None, # 0 + ( + 1, + TType.LIST, + "fullTableNames", + (TType.STRING, "UTF8", False), + None, + ), # 1 + ( + 2, + TType.STRING, + "validTxnList", + "UTF8", + None, + ), # 2 + ( + 3, + TType.I64, + "writeId", + None, + None, + ), # 3 +) +all_structs.append(TableValidWriteIds) +TableValidWriteIds.thrift_spec = ( + None, # 0 + ( + 1, + TType.STRING, + "fullTableName", + "UTF8", + None, + ), # 1 + ( + 2, + TType.I64, + "writeIdHighWaterMark", + None, + None, + ), # 2 + ( + 3, + TType.LIST, + "invalidWriteIds", + (TType.I64, None, False), + None, + ), # 3 + ( + 4, + TType.I64, + "minOpenWriteId", + None, + None, + ), # 4 + ( + 5, + TType.STRING, + "abortedBits", + "BINARY", + None, + ), # 5 +) +all_structs.append(GetValidWriteIdsResponse) +GetValidWriteIdsResponse.thrift_spec = ( + None, # 0 + ( + 1, + TType.LIST, + "tblValidWriteIds", + (TType.STRUCT, [TableValidWriteIds, None], False), + None, + ), # 1 +) +all_structs.append(TxnToWriteId) +TxnToWriteId.thrift_spec = ( + None, # 0 + ( + 1, + TType.I64, + "txnId", + None, + None, + ), # 1 + ( + 2, + TType.I64, + "writeId", + None, + None, + ), # 2 +) +all_structs.append(AllocateTableWriteIdsRequest) +AllocateTableWriteIdsRequest.thrift_spec = ( + None, # 0 + ( + 1, + TType.STRING, + "dbName", + "UTF8", + None, + ), # 1 + ( + 2, + TType.STRING, + "tableName", + "UTF8", + None, + ), # 2 + ( + 3, + TType.LIST, + "txnIds", + (TType.I64, None, False), + None, + ), # 3 + ( + 4, + TType.STRING, + "replPolicy", + "UTF8", + None, + ), # 4 + ( + 5, + TType.LIST, + "srcTxnToWriteIdList", + (TType.STRUCT, [TxnToWriteId, None], False), + None, + ), # 5 +) +all_structs.append(AllocateTableWriteIdsResponse) +AllocateTableWriteIdsResponse.thrift_spec = ( + None, # 0 + ( + 1, + TType.LIST, + "txnToWriteIds", + (TType.STRUCT, [TxnToWriteId, None], False), + None, + ), # 1 +) +all_structs.append(MaxAllocatedTableWriteIdRequest) +MaxAllocatedTableWriteIdRequest.thrift_spec = ( + None, # 0 + ( + 1, + TType.STRING, + "dbName", + "UTF8", + None, + ), # 1 + ( + 2, + TType.STRING, + "tableName", + "UTF8", + None, + ), # 2 +) +all_structs.append(MaxAllocatedTableWriteIdResponse) +MaxAllocatedTableWriteIdResponse.thrift_spec = ( + None, # 0 + ( + 1, + TType.I64, + "maxWriteId", + None, + None, + ), # 1 +) +all_structs.append(SeedTableWriteIdsRequest) +SeedTableWriteIdsRequest.thrift_spec = ( + None, # 0 + ( + 1, + TType.STRING, + "dbName", + "UTF8", + None, + ), # 1 + ( + 2, + TType.STRING, + "tableName", + "UTF8", + None, + ), # 2 + ( + 3, + TType.I64, + "seedWriteId", + None, + None, + ), # 3 +) +all_structs.append(SeedTxnIdRequest) +SeedTxnIdRequest.thrift_spec = ( + None, # 0 + ( + 1, + TType.I64, + "seedTxnId", + None, + None, + ), # 1 +) +all_structs.append(LockComponent) +LockComponent.thrift_spec = ( + None, # 0 + ( + 1, + TType.I32, + "type", + None, + None, + ), # 1 + ( + 2, + TType.I32, + "level", + None, + None, + ), # 2 + ( + 3, + TType.STRING, + "dbname", + "UTF8", + None, + ), # 3 + ( + 4, + TType.STRING, + "tablename", + "UTF8", + None, + ), # 4 + ( + 5, + TType.STRING, + "partitionname", + "UTF8", + None, + ), # 5 + ( + 6, + TType.I32, + "operationType", + None, + 5, + ), # 6 + ( + 7, + TType.BOOL, + "isTransactional", + None, + False, + ), # 7 + ( + 8, + TType.BOOL, + "isDynamicPartitionWrite", + None, + False, + ), # 8 +) +all_structs.append(LockRequest) +LockRequest.thrift_spec = ( + None, # 0 + ( + 1, + TType.LIST, + "component", + (TType.STRUCT, [LockComponent, None], False), + None, + ), # 1 + ( + 2, + TType.I64, + "txnid", + None, + None, + ), # 2 + ( + 3, + TType.STRING, + "user", + "UTF8", + None, + ), # 3 + ( + 4, + TType.STRING, + "hostname", + "UTF8", + None, + ), # 4 + ( + 5, + TType.STRING, + "agentInfo", + "UTF8", + "Unknown", + ), # 5 + ( + 6, + TType.BOOL, + "zeroWaitReadEnabled", + None, + False, + ), # 6 + ( + 7, + TType.BOOL, + "exclusiveCTAS", + None, + False, + ), # 7 +) +all_structs.append(LockResponse) +LockResponse.thrift_spec = ( + None, # 0 + ( + 1, + TType.I64, + "lockid", + None, + None, + ), # 1 + ( + 2, + TType.I32, + "state", + None, + None, + ), # 2 + ( + 3, + TType.STRING, + "errorMessage", + "UTF8", + None, + ), # 3 +) +all_structs.append(CheckLockRequest) +CheckLockRequest.thrift_spec = ( + None, # 0 + ( + 1, + TType.I64, + "lockid", + None, + None, + ), # 1 + ( + 2, + TType.I64, + "txnid", + None, + None, + ), # 2 + ( + 3, + TType.I64, + "elapsed_ms", + None, + None, + ), # 3 +) +all_structs.append(UnlockRequest) +UnlockRequest.thrift_spec = ( + None, # 0 + ( + 1, + TType.I64, + "lockid", + None, + None, + ), # 1 +) +all_structs.append(ShowLocksRequest) +ShowLocksRequest.thrift_spec = ( + None, # 0 + ( + 1, + TType.STRING, + "dbname", + "UTF8", + None, + ), # 1 + ( + 2, + TType.STRING, + "tablename", + "UTF8", + None, + ), # 2 + ( + 3, + TType.STRING, + "partname", + "UTF8", + None, + ), # 3 + ( + 4, + TType.BOOL, + "isExtended", + None, + False, + ), # 4 + ( + 5, + TType.I64, + "txnid", + None, + None, + ), # 5 +) +all_structs.append(ShowLocksResponseElement) +ShowLocksResponseElement.thrift_spec = ( + None, # 0 + ( + 1, + TType.I64, + "lockid", + None, + None, + ), # 1 + ( + 2, + TType.STRING, + "dbname", + "UTF8", + None, + ), # 2 + ( + 3, + TType.STRING, + "tablename", + "UTF8", + None, + ), # 3 + ( + 4, + TType.STRING, + "partname", + "UTF8", + None, + ), # 4 + ( + 5, + TType.I32, + "state", + None, + None, + ), # 5 + ( + 6, + TType.I32, + "type", + None, + None, + ), # 6 + ( + 7, + TType.I64, + "txnid", + None, + None, + ), # 7 + ( + 8, + TType.I64, + "lastheartbeat", + None, + None, + ), # 8 + ( + 9, + TType.I64, + "acquiredat", + None, + None, + ), # 9 + ( + 10, + TType.STRING, + "user", + "UTF8", + None, + ), # 10 + ( + 11, + TType.STRING, + "hostname", + "UTF8", + None, + ), # 11 + ( + 12, + TType.I32, + "heartbeatCount", + None, + 0, + ), # 12 + ( + 13, + TType.STRING, + "agentInfo", + "UTF8", + None, + ), # 13 + ( + 14, + TType.I64, + "blockedByExtId", + None, + None, + ), # 14 + ( + 15, + TType.I64, + "blockedByIntId", + None, + None, + ), # 15 + ( + 16, + TType.I64, + "lockIdInternal", + None, + None, + ), # 16 +) +all_structs.append(ShowLocksResponse) +ShowLocksResponse.thrift_spec = ( + None, # 0 + ( + 1, + TType.LIST, + "locks", + (TType.STRUCT, [ShowLocksResponseElement, None], False), + None, + ), # 1 +) +all_structs.append(HeartbeatRequest) +HeartbeatRequest.thrift_spec = ( + None, # 0 + ( + 1, + TType.I64, + "lockid", + None, + None, + ), # 1 + ( + 2, + TType.I64, + "txnid", + None, + None, + ), # 2 +) +all_structs.append(HeartbeatTxnRangeRequest) +HeartbeatTxnRangeRequest.thrift_spec = ( + None, # 0 + ( + 1, + TType.I64, + "min", + None, + None, + ), # 1 + ( + 2, + TType.I64, + "max", + None, + None, + ), # 2 +) +all_structs.append(HeartbeatTxnRangeResponse) +HeartbeatTxnRangeResponse.thrift_spec = ( + None, # 0 + ( + 1, + TType.SET, + "aborted", + (TType.I64, None, False), + None, + ), # 1 + ( + 2, + TType.SET, + "nosuch", + (TType.I64, None, False), + None, + ), # 2 +) +all_structs.append(CompactionRequest) +CompactionRequest.thrift_spec = ( + None, # 0 + ( + 1, + TType.STRING, + "dbname", + "UTF8", + None, + ), # 1 + ( + 2, + TType.STRING, + "tablename", + "UTF8", + None, + ), # 2 + ( + 3, + TType.STRING, + "partitionname", + "UTF8", + None, + ), # 3 + ( + 4, + TType.I32, + "type", + None, + None, + ), # 4 + ( + 5, + TType.STRING, + "runas", + "UTF8", + None, + ), # 5 + ( + 6, + TType.MAP, + "properties", + (TType.STRING, "UTF8", TType.STRING, "UTF8", False), + None, + ), # 6 + ( + 7, + TType.STRING, + "initiatorId", + "UTF8", + None, + ), # 7 + ( + 8, + TType.STRING, + "initiatorVersion", + "UTF8", + None, + ), # 8 +) +all_structs.append(CompactionInfoStruct) +CompactionInfoStruct.thrift_spec = ( + None, # 0 + ( + 1, + TType.I64, + "id", + None, + None, + ), # 1 + ( + 2, + TType.STRING, + "dbname", + "UTF8", + None, + ), # 2 + ( + 3, + TType.STRING, + "tablename", + "UTF8", + None, + ), # 3 + ( + 4, + TType.STRING, + "partitionname", + "UTF8", + None, + ), # 4 + ( + 5, + TType.I32, + "type", + None, + None, + ), # 5 + ( + 6, + TType.STRING, + "runas", + "UTF8", + None, + ), # 6 + ( + 7, + TType.STRING, + "properties", + "UTF8", + None, + ), # 7 + ( + 8, + TType.BOOL, + "toomanyaborts", + None, + None, + ), # 8 + ( + 9, + TType.STRING, + "state", + "UTF8", + None, + ), # 9 + ( + 10, + TType.STRING, + "workerId", + "UTF8", + None, + ), # 10 + ( + 11, + TType.I64, + "start", + None, + None, + ), # 11 + ( + 12, + TType.I64, + "highestWriteId", + None, + None, + ), # 12 + ( + 13, + TType.STRING, + "errorMessage", + "UTF8", + None, + ), # 13 + ( + 14, + TType.BOOL, + "hasoldabort", + None, + None, + ), # 14 + ( + 15, + TType.I64, + "enqueueTime", + None, + None, + ), # 15 + ( + 16, + TType.I64, + "retryRetention", + None, + None, + ), # 16 +) +all_structs.append(OptionalCompactionInfoStruct) +OptionalCompactionInfoStruct.thrift_spec = ( + None, # 0 + ( + 1, + TType.STRUCT, + "ci", + [CompactionInfoStruct, None], + None, + ), # 1 +) +all_structs.append(CompactionMetricsDataStruct) +CompactionMetricsDataStruct.thrift_spec = ( + None, # 0 + ( + 1, + TType.STRING, + "dbname", + "UTF8", + None, + ), # 1 + ( + 2, + TType.STRING, + "tblname", + "UTF8", + None, + ), # 2 + ( + 3, + TType.STRING, + "partitionname", + "UTF8", + None, + ), # 3 + ( + 4, + TType.I32, + "type", + None, + None, + ), # 4 + ( + 5, + TType.I32, + "metricvalue", + None, + None, + ), # 5 + ( + 6, + TType.I32, + "version", + None, + None, + ), # 6 + ( + 7, + TType.I32, + "threshold", + None, + None, + ), # 7 +) +all_structs.append(CompactionMetricsDataResponse) +CompactionMetricsDataResponse.thrift_spec = ( + None, # 0 + ( + 1, + TType.STRUCT, + "data", + [CompactionMetricsDataStruct, None], + None, + ), # 1 +) +all_structs.append(CompactionMetricsDataRequest) +CompactionMetricsDataRequest.thrift_spec = ( + None, # 0 + ( + 1, + TType.STRING, + "dbName", + "UTF8", + None, + ), # 1 + ( + 2, + TType.STRING, + "tblName", + "UTF8", + None, + ), # 2 + ( + 3, + TType.STRING, + "partitionName", + "UTF8", + None, + ), # 3 + ( + 4, + TType.I32, + "type", + None, + None, + ), # 4 +) +all_structs.append(CompactionResponse) +CompactionResponse.thrift_spec = ( + None, # 0 + ( + 1, + TType.I64, + "id", + None, + None, + ), # 1 + ( + 2, + TType.STRING, + "state", + "UTF8", + None, + ), # 2 + ( + 3, + TType.BOOL, + "accepted", + None, + None, + ), # 3 + ( + 4, + TType.STRING, + "errormessage", + "UTF8", + None, + ), # 4 +) +all_structs.append(ShowCompactRequest) +ShowCompactRequest.thrift_spec = () +all_structs.append(ShowCompactResponseElement) +ShowCompactResponseElement.thrift_spec = ( + None, # 0 + ( + 1, + TType.STRING, + "dbname", + "UTF8", + None, + ), # 1 + ( + 2, + TType.STRING, + "tablename", + "UTF8", + None, + ), # 2 + ( + 3, + TType.STRING, + "partitionname", + "UTF8", + None, + ), # 3 + ( + 4, + TType.I32, + "type", + None, + None, + ), # 4 + ( + 5, + TType.STRING, + "state", + "UTF8", + None, + ), # 5 + ( + 6, + TType.STRING, + "workerid", + "UTF8", + None, + ), # 6 + ( + 7, + TType.I64, + "start", + None, + None, + ), # 7 + ( + 8, + TType.STRING, + "runAs", + "UTF8", + None, + ), # 8 + ( + 9, + TType.I64, + "hightestTxnId", + None, + None, + ), # 9 + ( + 10, + TType.STRING, + "metaInfo", + "UTF8", + None, + ), # 10 + ( + 11, + TType.I64, + "endTime", + None, + None, + ), # 11 + ( + 12, + TType.STRING, + "hadoopJobId", + "UTF8", + "None", + ), # 12 + ( + 13, + TType.I64, + "id", + None, + None, + ), # 13 + ( + 14, + TType.STRING, + "errorMessage", + "UTF8", + None, + ), # 14 + ( + 15, + TType.I64, + "enqueueTime", + None, + None, + ), # 15 + ( + 16, + TType.STRING, + "workerVersion", + "UTF8", + None, + ), # 16 + ( + 17, + TType.STRING, + "initiatorId", + "UTF8", + None, + ), # 17 + ( + 18, + TType.STRING, + "initiatorVersion", + "UTF8", + None, + ), # 18 + ( + 19, + TType.I64, + "cleanerStart", + None, + None, + ), # 19 +) +all_structs.append(ShowCompactResponse) +ShowCompactResponse.thrift_spec = ( + None, # 0 + ( + 1, + TType.LIST, + "compacts", + (TType.STRUCT, [ShowCompactResponseElement, None], False), + None, + ), # 1 +) +all_structs.append(GetLatestCommittedCompactionInfoRequest) +GetLatestCommittedCompactionInfoRequest.thrift_spec = ( + None, # 0 + ( + 1, + TType.STRING, + "dbname", + "UTF8", + None, + ), # 1 + ( + 2, + TType.STRING, + "tablename", + "UTF8", + None, + ), # 2 + ( + 3, + TType.LIST, + "partitionnames", + (TType.STRING, "UTF8", False), + None, + ), # 3 + ( + 4, + TType.I64, + "lastCompactionId", + None, + None, + ), # 4 +) +all_structs.append(GetLatestCommittedCompactionInfoResponse) +GetLatestCommittedCompactionInfoResponse.thrift_spec = ( + None, # 0 + ( + 1, + TType.LIST, + "compactions", + (TType.STRUCT, [CompactionInfoStruct, None], False), + None, + ), # 1 +) +all_structs.append(FindNextCompactRequest) +FindNextCompactRequest.thrift_spec = ( + None, # 0 + ( + 1, + TType.STRING, + "workerId", + "UTF8", + None, + ), # 1 + ( + 2, + TType.STRING, + "workerVersion", + "UTF8", + None, + ), # 2 +) +all_structs.append(AddDynamicPartitions) +AddDynamicPartitions.thrift_spec = ( + None, # 0 + ( + 1, + TType.I64, + "txnid", + None, + None, + ), # 1 + ( + 2, + TType.I64, + "writeid", + None, + None, + ), # 2 + ( + 3, + TType.STRING, + "dbname", + "UTF8", + None, + ), # 3 + ( + 4, + TType.STRING, + "tablename", + "UTF8", + None, + ), # 4 + ( + 5, + TType.LIST, + "partitionnames", + (TType.STRING, "UTF8", False), + None, + ), # 5 + ( + 6, + TType.I32, + "operationType", + None, + 5, + ), # 6 +) +all_structs.append(BasicTxnInfo) +BasicTxnInfo.thrift_spec = ( + None, # 0 + ( + 1, + TType.BOOL, + "isnull", + None, + None, + ), # 1 + ( + 2, + TType.I64, + "time", + None, + None, + ), # 2 + ( + 3, + TType.I64, + "txnid", + None, + None, + ), # 3 + ( + 4, + TType.STRING, + "dbname", + "UTF8", + None, + ), # 4 + ( + 5, + TType.STRING, + "tablename", + "UTF8", + None, + ), # 5 + ( + 6, + TType.STRING, + "partitionname", + "UTF8", + None, + ), # 6 +) +all_structs.append(NotificationEventRequest) +NotificationEventRequest.thrift_spec = ( + None, # 0 + ( + 1, + TType.I64, + "lastEvent", + None, + None, + ), # 1 + ( + 2, + TType.I32, + "maxEvents", + None, + None, + ), # 2 + ( + 3, + TType.LIST, + "eventTypeSkipList", + (TType.STRING, "UTF8", False), + None, + ), # 3 +) +all_structs.append(NotificationEvent) +NotificationEvent.thrift_spec = ( + None, # 0 + ( + 1, + TType.I64, + "eventId", + None, + None, + ), # 1 + ( + 2, + TType.I32, + "eventTime", + None, + None, + ), # 2 + ( + 3, + TType.STRING, + "eventType", + "UTF8", + None, + ), # 3 + ( + 4, + TType.STRING, + "dbName", + "UTF8", + None, + ), # 4 + ( + 5, + TType.STRING, + "tableName", + "UTF8", + None, + ), # 5 + ( + 6, + TType.STRING, + "message", + "UTF8", + None, + ), # 6 + ( + 7, + TType.STRING, + "messageFormat", + "UTF8", + None, + ), # 7 + ( + 8, + TType.STRING, + "catName", + "UTF8", + None, + ), # 8 +) +all_structs.append(NotificationEventResponse) +NotificationEventResponse.thrift_spec = ( + None, # 0 + ( + 1, + TType.LIST, + "events", + (TType.STRUCT, [NotificationEvent, None], False), + None, + ), # 1 +) +all_structs.append(CurrentNotificationEventId) +CurrentNotificationEventId.thrift_spec = ( + None, # 0 + ( + 1, + TType.I64, + "eventId", + None, + None, + ), # 1 +) +all_structs.append(NotificationEventsCountRequest) +NotificationEventsCountRequest.thrift_spec = ( + None, # 0 + ( + 1, + TType.I64, + "fromEventId", + None, + None, + ), # 1 + ( + 2, + TType.STRING, + "dbName", + "UTF8", + None, + ), # 2 + ( + 3, + TType.STRING, + "catName", + "UTF8", + None, + ), # 3 + ( + 4, + TType.I64, + "toEventId", + None, + None, + ), # 4 + ( + 5, + TType.I64, + "limit", + None, + None, + ), # 5 +) +all_structs.append(NotificationEventsCountResponse) +NotificationEventsCountResponse.thrift_spec = ( + None, # 0 + ( + 1, + TType.I64, + "eventsCount", + None, + None, + ), # 1 +) +all_structs.append(InsertEventRequestData) +InsertEventRequestData.thrift_spec = ( + None, # 0 + ( + 1, + TType.BOOL, + "replace", + None, + None, + ), # 1 + ( + 2, + TType.LIST, + "filesAdded", + (TType.STRING, "UTF8", False), + None, + ), # 2 + ( + 3, + TType.LIST, + "filesAddedChecksum", + (TType.STRING, "UTF8", False), + None, + ), # 3 + ( + 4, + TType.LIST, + "subDirectoryList", + (TType.STRING, "UTF8", False), + None, + ), # 4 + ( + 5, + TType.LIST, + "partitionVal", + (TType.STRING, "UTF8", False), + None, + ), # 5 +) +all_structs.append(FireEventRequestData) +FireEventRequestData.thrift_spec = ( + None, # 0 + ( + 1, + TType.STRUCT, + "insertData", + [InsertEventRequestData, None], + None, + ), # 1 + ( + 2, + TType.LIST, + "insertDatas", + (TType.STRUCT, [InsertEventRequestData, None], False), + None, + ), # 2 +) +all_structs.append(FireEventRequest) +FireEventRequest.thrift_spec = ( + None, # 0 + ( + 1, + TType.BOOL, + "successful", + None, + None, + ), # 1 + ( + 2, + TType.STRUCT, + "data", + [FireEventRequestData, None], + None, + ), # 2 + ( + 3, + TType.STRING, + "dbName", + "UTF8", + None, + ), # 3 + ( + 4, + TType.STRING, + "tableName", + "UTF8", + None, + ), # 4 + ( + 5, + TType.LIST, + "partitionVals", + (TType.STRING, "UTF8", False), + None, + ), # 5 + ( + 6, + TType.STRING, + "catName", + "UTF8", + None, + ), # 6 +) +all_structs.append(FireEventResponse) +FireEventResponse.thrift_spec = ( + None, # 0 + ( + 1, + TType.LIST, + "eventIds", + (TType.I64, None, False), + None, + ), # 1 +) +all_structs.append(WriteNotificationLogRequest) +WriteNotificationLogRequest.thrift_spec = ( + None, # 0 + ( + 1, + TType.I64, + "txnId", + None, + None, + ), # 1 + ( + 2, + TType.I64, + "writeId", + None, + None, + ), # 2 + ( + 3, + TType.STRING, + "db", + "UTF8", + None, + ), # 3 + ( + 4, + TType.STRING, + "table", + "UTF8", + None, + ), # 4 + ( + 5, + TType.STRUCT, + "fileInfo", + [InsertEventRequestData, None], + None, + ), # 5 + ( + 6, + TType.LIST, + "partitionVals", + (TType.STRING, "UTF8", False), + None, + ), # 6 +) +all_structs.append(WriteNotificationLogResponse) +WriteNotificationLogResponse.thrift_spec = () +all_structs.append(WriteNotificationLogBatchRequest) +WriteNotificationLogBatchRequest.thrift_spec = ( + None, # 0 + ( + 1, + TType.STRING, + "catalog", + "UTF8", + None, + ), # 1 + ( + 2, + TType.STRING, + "db", + "UTF8", + None, + ), # 2 + ( + 3, + TType.STRING, + "table", + "UTF8", + None, + ), # 3 + ( + 4, + TType.LIST, + "requestList", + (TType.STRUCT, [WriteNotificationLogRequest, None], False), + None, + ), # 4 +) +all_structs.append(WriteNotificationLogBatchResponse) +WriteNotificationLogBatchResponse.thrift_spec = () +all_structs.append(MetadataPpdResult) +MetadataPpdResult.thrift_spec = ( + None, # 0 + ( + 1, + TType.STRING, + "metadata", + "BINARY", + None, + ), # 1 + ( + 2, + TType.STRING, + "includeBitset", + "BINARY", + None, + ), # 2 +) +all_structs.append(GetFileMetadataByExprResult) +GetFileMetadataByExprResult.thrift_spec = ( + None, # 0 + ( + 1, + TType.MAP, + "metadata", + (TType.I64, None, TType.STRUCT, [MetadataPpdResult, None], False), + None, + ), # 1 + ( + 2, + TType.BOOL, + "isSupported", + None, + None, + ), # 2 +) +all_structs.append(GetFileMetadataByExprRequest) +GetFileMetadataByExprRequest.thrift_spec = ( + None, # 0 + ( + 1, + TType.LIST, + "fileIds", + (TType.I64, None, False), + None, + ), # 1 + ( + 2, + TType.STRING, + "expr", + "BINARY", + None, + ), # 2 + ( + 3, + TType.BOOL, + "doGetFooters", + None, + None, + ), # 3 + ( + 4, + TType.I32, + "type", + None, + None, + ), # 4 +) +all_structs.append(GetFileMetadataResult) +GetFileMetadataResult.thrift_spec = ( + None, # 0 + ( + 1, + TType.MAP, + "metadata", + (TType.I64, None, TType.STRING, "BINARY", False), + None, + ), # 1 + ( + 2, + TType.BOOL, + "isSupported", + None, + None, + ), # 2 +) +all_structs.append(GetFileMetadataRequest) +GetFileMetadataRequest.thrift_spec = ( + None, # 0 + ( + 1, + TType.LIST, + "fileIds", + (TType.I64, None, False), + None, + ), # 1 +) +all_structs.append(PutFileMetadataResult) +PutFileMetadataResult.thrift_spec = () +all_structs.append(PutFileMetadataRequest) +PutFileMetadataRequest.thrift_spec = ( + None, # 0 + ( + 1, + TType.LIST, + "fileIds", + (TType.I64, None, False), + None, + ), # 1 + ( + 2, + TType.LIST, + "metadata", + (TType.STRING, "BINARY", False), + None, + ), # 2 + ( + 3, + TType.I32, + "type", + None, + None, + ), # 3 +) +all_structs.append(ClearFileMetadataResult) +ClearFileMetadataResult.thrift_spec = () +all_structs.append(ClearFileMetadataRequest) +ClearFileMetadataRequest.thrift_spec = ( + None, # 0 + ( + 1, + TType.LIST, + "fileIds", + (TType.I64, None, False), + None, + ), # 1 +) +all_structs.append(CacheFileMetadataResult) +CacheFileMetadataResult.thrift_spec = ( + None, # 0 + ( + 1, + TType.BOOL, + "isSupported", + None, + None, + ), # 1 +) +all_structs.append(CacheFileMetadataRequest) +CacheFileMetadataRequest.thrift_spec = ( + None, # 0 + ( + 1, + TType.STRING, + "dbName", + "UTF8", + None, + ), # 1 + ( + 2, + TType.STRING, + "tblName", + "UTF8", + None, + ), # 2 + ( + 3, + TType.STRING, + "partName", + "UTF8", + None, + ), # 3 + ( + 4, + TType.BOOL, + "isAllParts", + None, + None, + ), # 4 +) +all_structs.append(GetAllFunctionsResponse) +GetAllFunctionsResponse.thrift_spec = ( + None, # 0 + ( + 1, + TType.LIST, + "functions", + (TType.STRUCT, [Function, None], False), + None, + ), # 1 +) +all_structs.append(ClientCapabilities) +ClientCapabilities.thrift_spec = ( + None, # 0 + ( + 1, + TType.LIST, + "values", + (TType.I32, None, False), + None, + ), # 1 +) +all_structs.append(GetProjectionsSpec) +GetProjectionsSpec.thrift_spec = ( + None, # 0 + ( + 1, + TType.LIST, + "fieldList", + (TType.STRING, "UTF8", False), + None, + ), # 1 + ( + 2, + TType.STRING, + "includeParamKeyPattern", + "UTF8", + None, + ), # 2 + ( + 3, + TType.STRING, + "excludeParamKeyPattern", + "UTF8", + None, + ), # 3 +) +all_structs.append(GetTableRequest) +GetTableRequest.thrift_spec = ( + None, # 0 + ( + 1, + TType.STRING, + "dbName", + "UTF8", + None, + ), # 1 + ( + 2, + TType.STRING, + "tblName", + "UTF8", + None, + ), # 2 + ( + 3, + TType.STRUCT, + "capabilities", + [ClientCapabilities, None], + None, + ), # 3 + ( + 4, + TType.STRING, + "catName", + "UTF8", + None, + ), # 4 + None, # 5 + ( + 6, + TType.STRING, + "validWriteIdList", + "UTF8", + None, + ), # 6 + ( + 7, + TType.BOOL, + "getColumnStats", + None, + None, + ), # 7 + ( + 8, + TType.LIST, + "processorCapabilities", + (TType.STRING, "UTF8", False), + None, + ), # 8 + ( + 9, + TType.STRING, + "processorIdentifier", + "UTF8", + None, + ), # 9 + ( + 10, + TType.STRING, + "engine", + "UTF8", + None, + ), # 10 + ( + 11, + TType.I64, + "id", + None, + -1, + ), # 11 +) +all_structs.append(GetTableResult) +GetTableResult.thrift_spec = ( + None, # 0 + ( + 1, + TType.STRUCT, + "table", + [Table, None], + None, + ), # 1 + ( + 2, + TType.BOOL, + "isStatsCompliant", + None, + None, + ), # 2 +) +all_structs.append(GetTablesRequest) +GetTablesRequest.thrift_spec = ( + None, # 0 + ( + 1, + TType.STRING, + "dbName", + "UTF8", + None, + ), # 1 + ( + 2, + TType.LIST, + "tblNames", + (TType.STRING, "UTF8", False), + None, + ), # 2 + ( + 3, + TType.STRUCT, + "capabilities", + [ClientCapabilities, None], + None, + ), # 3 + ( + 4, + TType.STRING, + "catName", + "UTF8", + None, + ), # 4 + ( + 5, + TType.LIST, + "processorCapabilities", + (TType.STRING, "UTF8", False), + None, + ), # 5 + ( + 6, + TType.STRING, + "processorIdentifier", + "UTF8", + None, + ), # 6 + ( + 7, + TType.STRUCT, + "projectionSpec", + [GetProjectionsSpec, None], + None, + ), # 7 + ( + 8, + TType.STRING, + "tablesPattern", + "UTF8", + None, + ), # 8 +) +all_structs.append(GetTablesResult) +GetTablesResult.thrift_spec = ( + None, # 0 + ( + 1, + TType.LIST, + "tables", + (TType.STRUCT, [Table, None], False), + None, + ), # 1 +) +all_structs.append(GetTablesExtRequest) +GetTablesExtRequest.thrift_spec = ( + None, # 0 + ( + 1, + TType.STRING, + "catalog", + "UTF8", + None, + ), # 1 + ( + 2, + TType.STRING, + "database", + "UTF8", + None, + ), # 2 + ( + 3, + TType.STRING, + "tableNamePattern", + "UTF8", + None, + ), # 3 + ( + 4, + TType.I32, + "requestedFields", + None, + None, + ), # 4 + ( + 5, + TType.I32, + "limit", + None, + None, + ), # 5 + ( + 6, + TType.LIST, + "processorCapabilities", + (TType.STRING, "UTF8", False), + None, + ), # 6 + ( + 7, + TType.STRING, + "processorIdentifier", + "UTF8", + None, + ), # 7 +) +all_structs.append(ExtendedTableInfo) +ExtendedTableInfo.thrift_spec = ( + None, # 0 + ( + 1, + TType.STRING, + "tblName", + "UTF8", + None, + ), # 1 + ( + 2, + TType.I32, + "accessType", + None, + None, + ), # 2 + ( + 3, + TType.LIST, + "requiredReadCapabilities", + (TType.STRING, "UTF8", False), + None, + ), # 3 + ( + 4, + TType.LIST, + "requiredWriteCapabilities", + (TType.STRING, "UTF8", False), + None, + ), # 4 +) +all_structs.append(GetDatabaseRequest) +GetDatabaseRequest.thrift_spec = ( + None, # 0 + ( + 1, + TType.STRING, + "name", + "UTF8", + None, + ), # 1 + ( + 2, + TType.STRING, + "catalogName", + "UTF8", + None, + ), # 2 + ( + 3, + TType.LIST, + "processorCapabilities", + (TType.STRING, "UTF8", False), + None, + ), # 3 + ( + 4, + TType.STRING, + "processorIdentifier", + "UTF8", + None, + ), # 4 +) +all_structs.append(DropDatabaseRequest) +DropDatabaseRequest.thrift_spec = ( + None, # 0 + ( + 1, + TType.STRING, + "name", + "UTF8", + None, + ), # 1 + ( + 2, + TType.STRING, + "catalogName", + "UTF8", + None, + ), # 2 + ( + 3, + TType.BOOL, + "ignoreUnknownDb", + None, + None, + ), # 3 + ( + 4, + TType.BOOL, + "deleteData", + None, + None, + ), # 4 + ( + 5, + TType.BOOL, + "cascade", + None, + None, + ), # 5 + ( + 6, + TType.BOOL, + "softDelete", + None, + False, + ), # 6 + ( + 7, + TType.I64, + "txnId", + None, + 0, + ), # 7 + ( + 8, + TType.BOOL, + "deleteManagedDir", + None, + True, + ), # 8 +) +all_structs.append(CmRecycleRequest) +CmRecycleRequest.thrift_spec = ( + None, # 0 + ( + 1, + TType.STRING, + "dataPath", + "UTF8", + None, + ), # 1 + ( + 2, + TType.BOOL, + "purge", + None, + None, + ), # 2 +) +all_structs.append(CmRecycleResponse) +CmRecycleResponse.thrift_spec = () +all_structs.append(TableMeta) +TableMeta.thrift_spec = ( + None, # 0 + ( + 1, + TType.STRING, + "dbName", + "UTF8", + None, + ), # 1 + ( + 2, + TType.STRING, + "tableName", + "UTF8", + None, + ), # 2 + ( + 3, + TType.STRING, + "tableType", + "UTF8", + None, + ), # 3 + ( + 4, + TType.STRING, + "comments", + "UTF8", + None, + ), # 4 + ( + 5, + TType.STRING, + "catName", + "UTF8", + None, + ), # 5 +) +all_structs.append(Materialization) +Materialization.thrift_spec = ( + None, # 0 + ( + 1, + TType.BOOL, + "sourceTablesUpdateDeleteModified", + None, + None, + ), # 1 + ( + 2, + TType.BOOL, + "sourceTablesCompacted", + None, + None, + ), # 2 +) +all_structs.append(WMResourcePlan) +WMResourcePlan.thrift_spec = ( + None, # 0 + ( + 1, + TType.STRING, + "name", + "UTF8", + None, + ), # 1 + ( + 2, + TType.I32, + "status", + None, + None, + ), # 2 + ( + 3, + TType.I32, + "queryParallelism", + None, + None, + ), # 3 + ( + 4, + TType.STRING, + "defaultPoolPath", + "UTF8", + None, + ), # 4 + ( + 5, + TType.STRING, + "ns", + "UTF8", + None, + ), # 5 +) +all_structs.append(WMNullableResourcePlan) +WMNullableResourcePlan.thrift_spec = ( + None, # 0 + ( + 1, + TType.STRING, + "name", + "UTF8", + None, + ), # 1 + ( + 2, + TType.I32, + "status", + None, + None, + ), # 2 + None, # 3 + ( + 4, + TType.I32, + "queryParallelism", + None, + None, + ), # 4 + ( + 5, + TType.BOOL, + "isSetQueryParallelism", + None, + None, + ), # 5 + ( + 6, + TType.STRING, + "defaultPoolPath", + "UTF8", + None, + ), # 6 + ( + 7, + TType.BOOL, + "isSetDefaultPoolPath", + None, + None, + ), # 7 + ( + 8, + TType.STRING, + "ns", + "UTF8", + None, + ), # 8 +) +all_structs.append(WMPool) +WMPool.thrift_spec = ( + None, # 0 + ( + 1, + TType.STRING, + "resourcePlanName", + "UTF8", + None, + ), # 1 + ( + 2, + TType.STRING, + "poolPath", + "UTF8", + None, + ), # 2 + ( + 3, + TType.DOUBLE, + "allocFraction", + None, + None, + ), # 3 + ( + 4, + TType.I32, + "queryParallelism", + None, + None, + ), # 4 + ( + 5, + TType.STRING, + "schedulingPolicy", + "UTF8", + None, + ), # 5 + ( + 6, + TType.STRING, + "ns", + "UTF8", + None, + ), # 6 +) +all_structs.append(WMNullablePool) +WMNullablePool.thrift_spec = ( + None, # 0 + ( + 1, + TType.STRING, + "resourcePlanName", + "UTF8", + None, + ), # 1 + ( + 2, + TType.STRING, + "poolPath", + "UTF8", + None, + ), # 2 + ( + 3, + TType.DOUBLE, + "allocFraction", + None, + None, + ), # 3 + ( + 4, + TType.I32, + "queryParallelism", + None, + None, + ), # 4 + ( + 5, + TType.STRING, + "schedulingPolicy", + "UTF8", + None, + ), # 5 + ( + 6, + TType.BOOL, + "isSetSchedulingPolicy", + None, + None, + ), # 6 + ( + 7, + TType.STRING, + "ns", + "UTF8", + None, + ), # 7 +) +all_structs.append(WMTrigger) +WMTrigger.thrift_spec = ( + None, # 0 + ( + 1, + TType.STRING, + "resourcePlanName", + "UTF8", + None, + ), # 1 + ( + 2, + TType.STRING, + "triggerName", + "UTF8", + None, + ), # 2 + ( + 3, + TType.STRING, + "triggerExpression", + "UTF8", + None, + ), # 3 + ( + 4, + TType.STRING, + "actionExpression", + "UTF8", + None, + ), # 4 + ( + 5, + TType.BOOL, + "isInUnmanaged", + None, + None, + ), # 5 + ( + 6, + TType.STRING, + "ns", + "UTF8", + None, + ), # 6 +) +all_structs.append(WMMapping) +WMMapping.thrift_spec = ( + None, # 0 + ( + 1, + TType.STRING, + "resourcePlanName", + "UTF8", + None, + ), # 1 + ( + 2, + TType.STRING, + "entityType", + "UTF8", + None, + ), # 2 + ( + 3, + TType.STRING, + "entityName", + "UTF8", + None, + ), # 3 + ( + 4, + TType.STRING, + "poolPath", + "UTF8", + None, + ), # 4 + ( + 5, + TType.I32, + "ordering", + None, + None, + ), # 5 + ( + 6, + TType.STRING, + "ns", + "UTF8", + None, + ), # 6 +) +all_structs.append(WMPoolTrigger) +WMPoolTrigger.thrift_spec = ( + None, # 0 + ( + 1, + TType.STRING, + "pool", + "UTF8", + None, + ), # 1 + ( + 2, + TType.STRING, + "trigger", + "UTF8", + None, + ), # 2 + ( + 3, + TType.STRING, + "ns", + "UTF8", + None, + ), # 3 +) +all_structs.append(WMFullResourcePlan) +WMFullResourcePlan.thrift_spec = ( + None, # 0 + ( + 1, + TType.STRUCT, + "plan", + [WMResourcePlan, None], + None, + ), # 1 + ( + 2, + TType.LIST, + "pools", + (TType.STRUCT, [WMPool, None], False), + None, + ), # 2 + ( + 3, + TType.LIST, + "mappings", + (TType.STRUCT, [WMMapping, None], False), + None, + ), # 3 + ( + 4, + TType.LIST, + "triggers", + (TType.STRUCT, [WMTrigger, None], False), + None, + ), # 4 + ( + 5, + TType.LIST, + "poolTriggers", + (TType.STRUCT, [WMPoolTrigger, None], False), + None, + ), # 5 +) +all_structs.append(WMCreateResourcePlanRequest) +WMCreateResourcePlanRequest.thrift_spec = ( + None, # 0 + ( + 1, + TType.STRUCT, + "resourcePlan", + [WMResourcePlan, None], + None, + ), # 1 + ( + 2, + TType.STRING, + "copyFrom", + "UTF8", + None, + ), # 2 +) +all_structs.append(WMCreateResourcePlanResponse) +WMCreateResourcePlanResponse.thrift_spec = () +all_structs.append(WMGetActiveResourcePlanRequest) +WMGetActiveResourcePlanRequest.thrift_spec = ( + None, # 0 + ( + 1, + TType.STRING, + "ns", + "UTF8", + None, + ), # 1 +) +all_structs.append(WMGetActiveResourcePlanResponse) +WMGetActiveResourcePlanResponse.thrift_spec = ( + None, # 0 + ( + 1, + TType.STRUCT, + "resourcePlan", + [WMFullResourcePlan, None], + None, + ), # 1 +) +all_structs.append(WMGetResourcePlanRequest) +WMGetResourcePlanRequest.thrift_spec = ( + None, # 0 + ( + 1, + TType.STRING, + "resourcePlanName", + "UTF8", + None, + ), # 1 + ( + 2, + TType.STRING, + "ns", + "UTF8", + None, + ), # 2 +) +all_structs.append(WMGetResourcePlanResponse) +WMGetResourcePlanResponse.thrift_spec = ( + None, # 0 + ( + 1, + TType.STRUCT, + "resourcePlan", + [WMFullResourcePlan, None], + None, + ), # 1 +) +all_structs.append(WMGetAllResourcePlanRequest) +WMGetAllResourcePlanRequest.thrift_spec = ( + None, # 0 + ( + 1, + TType.STRING, + "ns", + "UTF8", + None, + ), # 1 +) +all_structs.append(WMGetAllResourcePlanResponse) +WMGetAllResourcePlanResponse.thrift_spec = ( + None, # 0 + ( + 1, + TType.LIST, + "resourcePlans", + (TType.STRUCT, [WMResourcePlan, None], False), + None, + ), # 1 +) +all_structs.append(WMAlterResourcePlanRequest) +WMAlterResourcePlanRequest.thrift_spec = ( + None, # 0 + ( + 1, + TType.STRING, + "resourcePlanName", + "UTF8", + None, + ), # 1 + ( + 2, + TType.STRUCT, + "resourcePlan", + [WMNullableResourcePlan, None], + None, + ), # 2 + ( + 3, + TType.BOOL, + "isEnableAndActivate", + None, + None, + ), # 3 + ( + 4, + TType.BOOL, + "isForceDeactivate", + None, + None, + ), # 4 + ( + 5, + TType.BOOL, + "isReplace", + None, + None, + ), # 5 + ( + 6, + TType.STRING, + "ns", + "UTF8", + None, + ), # 6 +) +all_structs.append(WMAlterResourcePlanResponse) +WMAlterResourcePlanResponse.thrift_spec = ( + None, # 0 + ( + 1, + TType.STRUCT, + "fullResourcePlan", + [WMFullResourcePlan, None], + None, + ), # 1 +) +all_structs.append(WMValidateResourcePlanRequest) +WMValidateResourcePlanRequest.thrift_spec = ( + None, # 0 + ( + 1, + TType.STRING, + "resourcePlanName", + "UTF8", + None, + ), # 1 + ( + 2, + TType.STRING, + "ns", + "UTF8", + None, + ), # 2 +) +all_structs.append(WMValidateResourcePlanResponse) +WMValidateResourcePlanResponse.thrift_spec = ( + None, # 0 + ( + 1, + TType.LIST, + "errors", + (TType.STRING, "UTF8", False), + None, + ), # 1 + ( + 2, + TType.LIST, + "warnings", + (TType.STRING, "UTF8", False), + None, + ), # 2 +) +all_structs.append(WMDropResourcePlanRequest) +WMDropResourcePlanRequest.thrift_spec = ( + None, # 0 + ( + 1, + TType.STRING, + "resourcePlanName", + "UTF8", + None, + ), # 1 + ( + 2, + TType.STRING, + "ns", + "UTF8", + None, + ), # 2 +) +all_structs.append(WMDropResourcePlanResponse) +WMDropResourcePlanResponse.thrift_spec = () +all_structs.append(WMCreateTriggerRequest) +WMCreateTriggerRequest.thrift_spec = ( + None, # 0 + ( + 1, + TType.STRUCT, + "trigger", + [WMTrigger, None], + None, + ), # 1 +) +all_structs.append(WMCreateTriggerResponse) +WMCreateTriggerResponse.thrift_spec = () +all_structs.append(WMAlterTriggerRequest) +WMAlterTriggerRequest.thrift_spec = ( + None, # 0 + ( + 1, + TType.STRUCT, + "trigger", + [WMTrigger, None], + None, + ), # 1 +) +all_structs.append(WMAlterTriggerResponse) +WMAlterTriggerResponse.thrift_spec = () +all_structs.append(WMDropTriggerRequest) +WMDropTriggerRequest.thrift_spec = ( + None, # 0 + ( + 1, + TType.STRING, + "resourcePlanName", + "UTF8", + None, + ), # 1 + ( + 2, + TType.STRING, + "triggerName", + "UTF8", + None, + ), # 2 + ( + 3, + TType.STRING, + "ns", + "UTF8", + None, + ), # 3 +) +all_structs.append(WMDropTriggerResponse) +WMDropTriggerResponse.thrift_spec = () +all_structs.append(WMGetTriggersForResourePlanRequest) +WMGetTriggersForResourePlanRequest.thrift_spec = ( + None, # 0 + ( + 1, + TType.STRING, + "resourcePlanName", + "UTF8", + None, + ), # 1 + ( + 2, + TType.STRING, + "ns", + "UTF8", + None, + ), # 2 +) +all_structs.append(WMGetTriggersForResourePlanResponse) +WMGetTriggersForResourePlanResponse.thrift_spec = ( + None, # 0 + ( + 1, + TType.LIST, + "triggers", + (TType.STRUCT, [WMTrigger, None], False), + None, + ), # 1 +) +all_structs.append(WMCreatePoolRequest) +WMCreatePoolRequest.thrift_spec = ( + None, # 0 + ( + 1, + TType.STRUCT, + "pool", + [WMPool, None], + None, + ), # 1 +) +all_structs.append(WMCreatePoolResponse) +WMCreatePoolResponse.thrift_spec = () +all_structs.append(WMAlterPoolRequest) +WMAlterPoolRequest.thrift_spec = ( + None, # 0 + ( + 1, + TType.STRUCT, + "pool", + [WMNullablePool, None], + None, + ), # 1 + ( + 2, + TType.STRING, + "poolPath", + "UTF8", + None, + ), # 2 +) +all_structs.append(WMAlterPoolResponse) +WMAlterPoolResponse.thrift_spec = () +all_structs.append(WMDropPoolRequest) +WMDropPoolRequest.thrift_spec = ( + None, # 0 + ( + 1, + TType.STRING, + "resourcePlanName", + "UTF8", + None, + ), # 1 + ( + 2, + TType.STRING, + "poolPath", + "UTF8", + None, + ), # 2 + ( + 3, + TType.STRING, + "ns", + "UTF8", + None, + ), # 3 +) +all_structs.append(WMDropPoolResponse) +WMDropPoolResponse.thrift_spec = () +all_structs.append(WMCreateOrUpdateMappingRequest) +WMCreateOrUpdateMappingRequest.thrift_spec = ( + None, # 0 + ( + 1, + TType.STRUCT, + "mapping", + [WMMapping, None], + None, + ), # 1 + ( + 2, + TType.BOOL, + "update", + None, + None, + ), # 2 +) +all_structs.append(WMCreateOrUpdateMappingResponse) +WMCreateOrUpdateMappingResponse.thrift_spec = () +all_structs.append(WMDropMappingRequest) +WMDropMappingRequest.thrift_spec = ( + None, # 0 + ( + 1, + TType.STRUCT, + "mapping", + [WMMapping, None], + None, + ), # 1 +) +all_structs.append(WMDropMappingResponse) +WMDropMappingResponse.thrift_spec = () +all_structs.append(WMCreateOrDropTriggerToPoolMappingRequest) +WMCreateOrDropTriggerToPoolMappingRequest.thrift_spec = ( + None, # 0 + ( + 1, + TType.STRING, + "resourcePlanName", + "UTF8", + None, + ), # 1 + ( + 2, + TType.STRING, + "triggerName", + "UTF8", + None, + ), # 2 + ( + 3, + TType.STRING, + "poolPath", + "UTF8", + None, + ), # 3 + ( + 4, + TType.BOOL, + "drop", + None, + None, + ), # 4 + ( + 5, + TType.STRING, + "ns", + "UTF8", + None, + ), # 5 +) +all_structs.append(WMCreateOrDropTriggerToPoolMappingResponse) +WMCreateOrDropTriggerToPoolMappingResponse.thrift_spec = () +all_structs.append(ISchema) +ISchema.thrift_spec = ( + None, # 0 + ( + 1, + TType.I32, + "schemaType", + None, + None, + ), # 1 + ( + 2, + TType.STRING, + "name", + "UTF8", + None, + ), # 2 + ( + 3, + TType.STRING, + "catName", + "UTF8", + None, + ), # 3 + ( + 4, + TType.STRING, + "dbName", + "UTF8", + None, + ), # 4 + ( + 5, + TType.I32, + "compatibility", + None, + None, + ), # 5 + ( + 6, + TType.I32, + "validationLevel", + None, + None, + ), # 6 + ( + 7, + TType.BOOL, + "canEvolve", + None, + None, + ), # 7 + ( + 8, + TType.STRING, + "schemaGroup", + "UTF8", + None, + ), # 8 + ( + 9, + TType.STRING, + "description", + "UTF8", + None, + ), # 9 +) +all_structs.append(ISchemaName) +ISchemaName.thrift_spec = ( + None, # 0 + ( + 1, + TType.STRING, + "catName", + "UTF8", + None, + ), # 1 + ( + 2, + TType.STRING, + "dbName", + "UTF8", + None, + ), # 2 + ( + 3, + TType.STRING, + "schemaName", + "UTF8", + None, + ), # 3 +) +all_structs.append(AlterISchemaRequest) +AlterISchemaRequest.thrift_spec = ( + None, # 0 + ( + 1, + TType.STRUCT, + "name", + [ISchemaName, None], + None, + ), # 1 + None, # 2 + ( + 3, + TType.STRUCT, + "newSchema", + [ISchema, None], + None, + ), # 3 +) +all_structs.append(SchemaVersion) +SchemaVersion.thrift_spec = ( + None, # 0 + ( + 1, + TType.STRUCT, + "schema", + [ISchemaName, None], + None, + ), # 1 + ( + 2, + TType.I32, + "version", + None, + None, + ), # 2 + ( + 3, + TType.I64, + "createdAt", + None, + None, + ), # 3 + ( + 4, + TType.LIST, + "cols", + (TType.STRUCT, [FieldSchema, None], False), + None, + ), # 4 + ( + 5, + TType.I32, + "state", + None, + None, + ), # 5 + ( + 6, + TType.STRING, + "description", + "UTF8", + None, + ), # 6 + ( + 7, + TType.STRING, + "schemaText", + "UTF8", + None, + ), # 7 + ( + 8, + TType.STRING, + "fingerprint", + "UTF8", + None, + ), # 8 + ( + 9, + TType.STRING, + "name", + "UTF8", + None, + ), # 9 + ( + 10, + TType.STRUCT, + "serDe", + [SerDeInfo, None], + None, + ), # 10 +) +all_structs.append(SchemaVersionDescriptor) +SchemaVersionDescriptor.thrift_spec = ( + None, # 0 + ( + 1, + TType.STRUCT, + "schema", + [ISchemaName, None], + None, + ), # 1 + ( + 2, + TType.I32, + "version", + None, + None, + ), # 2 +) +all_structs.append(FindSchemasByColsRqst) +FindSchemasByColsRqst.thrift_spec = ( + None, # 0 + ( + 1, + TType.STRING, + "colName", + "UTF8", + None, + ), # 1 + ( + 2, + TType.STRING, + "colNamespace", + "UTF8", + None, + ), # 2 + ( + 3, + TType.STRING, + "type", + "UTF8", + None, + ), # 3 +) +all_structs.append(FindSchemasByColsResp) +FindSchemasByColsResp.thrift_spec = ( + None, # 0 + ( + 1, + TType.LIST, + "schemaVersions", + (TType.STRUCT, [SchemaVersionDescriptor, None], False), + None, + ), # 1 +) +all_structs.append(MapSchemaVersionToSerdeRequest) +MapSchemaVersionToSerdeRequest.thrift_spec = ( + None, # 0 + ( + 1, + TType.STRUCT, + "schemaVersion", + [SchemaVersionDescriptor, None], + None, + ), # 1 + ( + 2, + TType.STRING, + "serdeName", + "UTF8", + None, + ), # 2 +) +all_structs.append(SetSchemaVersionStateRequest) +SetSchemaVersionStateRequest.thrift_spec = ( + None, # 0 + ( + 1, + TType.STRUCT, + "schemaVersion", + [SchemaVersionDescriptor, None], + None, + ), # 1 + ( + 2, + TType.I32, + "state", + None, + None, + ), # 2 +) +all_structs.append(GetSerdeRequest) +GetSerdeRequest.thrift_spec = ( + None, # 0 + ( + 1, + TType.STRING, + "serdeName", + "UTF8", + None, + ), # 1 +) +all_structs.append(RuntimeStat) +RuntimeStat.thrift_spec = ( + None, # 0 + ( + 1, + TType.I32, + "createTime", + None, + None, + ), # 1 + ( + 2, + TType.I32, + "weight", + None, + None, + ), # 2 + ( + 3, + TType.STRING, + "payload", + "BINARY", + None, + ), # 3 +) +all_structs.append(GetRuntimeStatsRequest) +GetRuntimeStatsRequest.thrift_spec = ( + None, # 0 + ( + 1, + TType.I32, + "maxWeight", + None, + None, + ), # 1 + ( + 2, + TType.I32, + "maxCreateTime", + None, + None, + ), # 2 +) +all_structs.append(CreateTableRequest) +CreateTableRequest.thrift_spec = ( + None, # 0 + ( + 1, + TType.STRUCT, + "table", + [Table, None], + None, + ), # 1 + ( + 2, + TType.STRUCT, + "envContext", + [EnvironmentContext, None], + None, + ), # 2 + ( + 3, + TType.LIST, + "primaryKeys", + (TType.STRUCT, [SQLPrimaryKey, None], False), + None, + ), # 3 + ( + 4, + TType.LIST, + "foreignKeys", + (TType.STRUCT, [SQLForeignKey, None], False), + None, + ), # 4 + ( + 5, + TType.LIST, + "uniqueConstraints", + (TType.STRUCT, [SQLUniqueConstraint, None], False), + None, + ), # 5 + ( + 6, + TType.LIST, + "notNullConstraints", + (TType.STRUCT, [SQLNotNullConstraint, None], False), + None, + ), # 6 + ( + 7, + TType.LIST, + "defaultConstraints", + (TType.STRUCT, [SQLDefaultConstraint, None], False), + None, + ), # 7 + ( + 8, + TType.LIST, + "checkConstraints", + (TType.STRUCT, [SQLCheckConstraint, None], False), + None, + ), # 8 + ( + 9, + TType.LIST, + "processorCapabilities", + (TType.STRING, "UTF8", False), + None, + ), # 9 + ( + 10, + TType.STRING, + "processorIdentifier", + "UTF8", + None, + ), # 10 +) +all_structs.append(CreateDatabaseRequest) +CreateDatabaseRequest.thrift_spec = ( + None, # 0 + ( + 1, + TType.STRING, + "databaseName", + "UTF8", + None, + ), # 1 + ( + 2, + TType.STRING, + "description", + "UTF8", + None, + ), # 2 + ( + 3, + TType.STRING, + "locationUri", + "UTF8", + None, + ), # 3 + ( + 4, + TType.MAP, + "parameters", + (TType.STRING, "UTF8", TType.STRING, "UTF8", False), + None, + ), # 4 + ( + 5, + TType.STRUCT, + "privileges", + [PrincipalPrivilegeSet, None], + None, + ), # 5 + ( + 6, + TType.STRING, + "ownerName", + "UTF8", + None, + ), # 6 + ( + 7, + TType.I32, + "ownerType", + None, + None, + ), # 7 + ( + 8, + TType.STRING, + "catalogName", + "UTF8", + None, + ), # 8 + ( + 9, + TType.I32, + "createTime", + None, + None, + ), # 9 + ( + 10, + TType.STRING, + "managedLocationUri", + "UTF8", + None, + ), # 10 + ( + 11, + TType.STRING, + "type", + "UTF8", + None, + ), # 11 + ( + 12, + TType.STRING, + "dataConnectorName", + "UTF8", + None, + ), # 12 +) +all_structs.append(CreateDataConnectorRequest) +CreateDataConnectorRequest.thrift_spec = ( + None, # 0 + ( + 1, + TType.STRUCT, + "connector", + [DataConnector, None], + None, + ), # 1 +) +all_structs.append(GetDataConnectorRequest) +GetDataConnectorRequest.thrift_spec = ( + None, # 0 + ( + 1, + TType.STRING, + "connectorName", + "UTF8", + None, + ), # 1 +) +all_structs.append(ScheduledQueryPollRequest) +ScheduledQueryPollRequest.thrift_spec = ( + None, # 0 + ( + 1, + TType.STRING, + "clusterNamespace", + "UTF8", + None, + ), # 1 +) +all_structs.append(ScheduledQueryKey) +ScheduledQueryKey.thrift_spec = ( + None, # 0 + ( + 1, + TType.STRING, + "scheduleName", + "UTF8", + None, + ), # 1 + ( + 2, + TType.STRING, + "clusterNamespace", + "UTF8", + None, + ), # 2 +) +all_structs.append(ScheduledQueryPollResponse) +ScheduledQueryPollResponse.thrift_spec = ( + None, # 0 + ( + 1, + TType.STRUCT, + "scheduleKey", + [ScheduledQueryKey, None], + None, + ), # 1 + ( + 2, + TType.I64, + "executionId", + None, + None, + ), # 2 + ( + 3, + TType.STRING, + "query", + "UTF8", + None, + ), # 3 + ( + 4, + TType.STRING, + "user", + "UTF8", + None, + ), # 4 +) +all_structs.append(ScheduledQuery) +ScheduledQuery.thrift_spec = ( + None, # 0 + ( + 1, + TType.STRUCT, + "scheduleKey", + [ScheduledQueryKey, None], + None, + ), # 1 + ( + 2, + TType.BOOL, + "enabled", + None, + None, + ), # 2 + None, # 3 + ( + 4, + TType.STRING, + "schedule", + "UTF8", + None, + ), # 4 + ( + 5, + TType.STRING, + "user", + "UTF8", + None, + ), # 5 + ( + 6, + TType.STRING, + "query", + "UTF8", + None, + ), # 6 + ( + 7, + TType.I32, + "nextExecution", + None, + None, + ), # 7 +) +all_structs.append(ScheduledQueryMaintenanceRequest) +ScheduledQueryMaintenanceRequest.thrift_spec = ( + None, # 0 + ( + 1, + TType.I32, + "type", + None, + None, + ), # 1 + ( + 2, + TType.STRUCT, + "scheduledQuery", + [ScheduledQuery, None], + None, + ), # 2 +) +all_structs.append(ScheduledQueryProgressInfo) +ScheduledQueryProgressInfo.thrift_spec = ( + None, # 0 + ( + 1, + TType.I64, + "scheduledExecutionId", + None, + None, + ), # 1 + ( + 2, + TType.I32, + "state", + None, + None, + ), # 2 + ( + 3, + TType.STRING, + "executorQueryId", + "UTF8", + None, + ), # 3 + ( + 4, + TType.STRING, + "errorMessage", + "UTF8", + None, + ), # 4 +) +all_structs.append(AlterPartitionsRequest) +AlterPartitionsRequest.thrift_spec = ( + None, # 0 + ( + 1, + TType.STRING, + "catName", + "UTF8", + None, + ), # 1 + ( + 2, + TType.STRING, + "dbName", + "UTF8", + None, + ), # 2 + ( + 3, + TType.STRING, + "tableName", + "UTF8", + None, + ), # 3 + ( + 4, + TType.LIST, + "partitions", + (TType.STRUCT, [Partition, None], False), + None, + ), # 4 + ( + 5, + TType.STRUCT, + "environmentContext", + [EnvironmentContext, None], + None, + ), # 5 + ( + 6, + TType.I64, + "writeId", + None, + -1, + ), # 6 + ( + 7, + TType.STRING, + "validWriteIdList", + "UTF8", + None, + ), # 7 +) +all_structs.append(AlterPartitionsResponse) +AlterPartitionsResponse.thrift_spec = () +all_structs.append(RenamePartitionRequest) +RenamePartitionRequest.thrift_spec = ( + None, # 0 + ( + 1, + TType.STRING, + "catName", + "UTF8", + None, + ), # 1 + ( + 2, + TType.STRING, + "dbName", + "UTF8", + None, + ), # 2 + ( + 3, + TType.STRING, + "tableName", + "UTF8", + None, + ), # 3 + ( + 4, + TType.LIST, + "partVals", + (TType.STRING, "UTF8", False), + None, + ), # 4 + ( + 5, + TType.STRUCT, + "newPart", + [Partition, None], + None, + ), # 5 + ( + 6, + TType.STRING, + "validWriteIdList", + "UTF8", + None, + ), # 6 + ( + 7, + TType.I64, + "txnId", + None, + None, + ), # 7 + ( + 8, + TType.BOOL, + "clonePart", + None, + None, + ), # 8 +) +all_structs.append(RenamePartitionResponse) +RenamePartitionResponse.thrift_spec = () +all_structs.append(AlterTableRequest) +AlterTableRequest.thrift_spec = ( + None, # 0 + ( + 1, + TType.STRING, + "catName", + "UTF8", + None, + ), # 1 + ( + 2, + TType.STRING, + "dbName", + "UTF8", + None, + ), # 2 + ( + 3, + TType.STRING, + "tableName", + "UTF8", + None, + ), # 3 + ( + 4, + TType.STRUCT, + "table", + [Table, None], + None, + ), # 4 + ( + 5, + TType.STRUCT, + "environmentContext", + [EnvironmentContext, None], + None, + ), # 5 + ( + 6, + TType.I64, + "writeId", + None, + -1, + ), # 6 + ( + 7, + TType.STRING, + "validWriteIdList", + "UTF8", + None, + ), # 7 + ( + 8, + TType.LIST, + "processorCapabilities", + (TType.STRING, "UTF8", False), + None, + ), # 8 + ( + 9, + TType.STRING, + "processorIdentifier", + "UTF8", + None, + ), # 9 +) +all_structs.append(AlterTableResponse) +AlterTableResponse.thrift_spec = () +all_structs.append(GetPartitionsFilterSpec) +GetPartitionsFilterSpec.thrift_spec = ( + None, # 0 + None, # 1 + None, # 2 + None, # 3 + None, # 4 + None, # 5 + None, # 6 + ( + 7, + TType.I32, + "filterMode", + None, + None, + ), # 7 + ( + 8, + TType.LIST, + "filters", + (TType.STRING, "UTF8", False), + None, + ), # 8 +) +all_structs.append(GetPartitionsResponse) +GetPartitionsResponse.thrift_spec = ( + None, # 0 + ( + 1, + TType.LIST, + "partitionSpec", + (TType.STRUCT, [PartitionSpec, None], False), + None, + ), # 1 +) +all_structs.append(GetPartitionsRequest) +GetPartitionsRequest.thrift_spec = ( + None, # 0 + ( + 1, + TType.STRING, + "catName", + "UTF8", + None, + ), # 1 + ( + 2, + TType.STRING, + "dbName", + "UTF8", + None, + ), # 2 + ( + 3, + TType.STRING, + "tblName", + "UTF8", + None, + ), # 3 + ( + 4, + TType.BOOL, + "withAuth", + None, + None, + ), # 4 + ( + 5, + TType.STRING, + "user", + "UTF8", + None, + ), # 5 + ( + 6, + TType.LIST, + "groupNames", + (TType.STRING, "UTF8", False), + None, + ), # 6 + ( + 7, + TType.STRUCT, + "projectionSpec", + [GetProjectionsSpec, None], + None, + ), # 7 + ( + 8, + TType.STRUCT, + "filterSpec", + [GetPartitionsFilterSpec, None], + None, + ), # 8 + ( + 9, + TType.LIST, + "processorCapabilities", + (TType.STRING, "UTF8", False), + None, + ), # 9 + ( + 10, + TType.STRING, + "processorIdentifier", + "UTF8", + None, + ), # 10 + ( + 11, + TType.STRING, + "validWriteIdList", + "UTF8", + None, + ), # 11 +) +all_structs.append(GetFieldsRequest) +GetFieldsRequest.thrift_spec = ( + None, # 0 + ( + 1, + TType.STRING, + "catName", + "UTF8", + None, + ), # 1 + ( + 2, + TType.STRING, + "dbName", + "UTF8", + None, + ), # 2 + ( + 3, + TType.STRING, + "tblName", + "UTF8", + None, + ), # 3 + ( + 4, + TType.STRUCT, + "envContext", + [EnvironmentContext, None], + None, + ), # 4 + ( + 5, + TType.STRING, + "validWriteIdList", + "UTF8", + None, + ), # 5 + ( + 6, + TType.I64, + "id", + None, + -1, + ), # 6 +) +all_structs.append(GetFieldsResponse) +GetFieldsResponse.thrift_spec = ( + None, # 0 + ( + 1, + TType.LIST, + "fields", + (TType.STRUCT, [FieldSchema, None], False), + None, + ), # 1 +) +all_structs.append(GetSchemaRequest) +GetSchemaRequest.thrift_spec = ( + None, # 0 + ( + 1, + TType.STRING, + "catName", + "UTF8", + None, + ), # 1 + ( + 2, + TType.STRING, + "dbName", + "UTF8", + None, + ), # 2 + ( + 3, + TType.STRING, + "tblName", + "UTF8", + None, + ), # 3 + ( + 4, + TType.STRUCT, + "envContext", + [EnvironmentContext, None], + None, + ), # 4 + ( + 5, + TType.STRING, + "validWriteIdList", + "UTF8", + None, + ), # 5 + ( + 6, + TType.I64, + "id", + None, + -1, + ), # 6 +) +all_structs.append(GetSchemaResponse) +GetSchemaResponse.thrift_spec = ( + None, # 0 + ( + 1, + TType.LIST, + "fields", + (TType.STRUCT, [FieldSchema, None], False), + None, + ), # 1 +) +all_structs.append(GetPartitionRequest) +GetPartitionRequest.thrift_spec = ( + None, # 0 + ( + 1, + TType.STRING, + "catName", + "UTF8", + None, + ), # 1 + ( + 2, + TType.STRING, + "dbName", + "UTF8", + None, + ), # 2 + ( + 3, + TType.STRING, + "tblName", + "UTF8", + None, + ), # 3 + ( + 4, + TType.LIST, + "partVals", + (TType.STRING, "UTF8", False), + None, + ), # 4 + ( + 5, + TType.STRING, + "validWriteIdList", + "UTF8", + None, + ), # 5 + ( + 6, + TType.I64, + "id", + None, + -1, + ), # 6 +) +all_structs.append(GetPartitionResponse) +GetPartitionResponse.thrift_spec = ( + None, # 0 + ( + 1, + TType.STRUCT, + "partition", + [Partition, None], + None, + ), # 1 +) +all_structs.append(PartitionsRequest) +PartitionsRequest.thrift_spec = ( + None, # 0 + ( + 1, + TType.STRING, + "catName", + "UTF8", + None, + ), # 1 + ( + 2, + TType.STRING, + "dbName", + "UTF8", + None, + ), # 2 + ( + 3, + TType.STRING, + "tblName", + "UTF8", + None, + ), # 3 + ( + 4, + TType.I16, + "maxParts", + None, + -1, + ), # 4 + ( + 5, + TType.STRING, + "validWriteIdList", + "UTF8", + None, + ), # 5 + ( + 6, + TType.I64, + "id", + None, + -1, + ), # 6 +) +all_structs.append(PartitionsResponse) +PartitionsResponse.thrift_spec = ( + None, # 0 + ( + 1, + TType.LIST, + "partitions", + (TType.STRUCT, [Partition, None], False), + None, + ), # 1 +) +all_structs.append(GetPartitionNamesPsRequest) +GetPartitionNamesPsRequest.thrift_spec = ( + None, # 0 + ( + 1, + TType.STRING, + "catName", + "UTF8", + None, + ), # 1 + ( + 2, + TType.STRING, + "dbName", + "UTF8", + None, + ), # 2 + ( + 3, + TType.STRING, + "tblName", + "UTF8", + None, + ), # 3 + ( + 4, + TType.LIST, + "partValues", + (TType.STRING, "UTF8", False), + None, + ), # 4 + ( + 5, + TType.I16, + "maxParts", + None, + -1, + ), # 5 + ( + 6, + TType.STRING, + "validWriteIdList", + "UTF8", + None, + ), # 6 + ( + 7, + TType.I64, + "id", + None, + -1, + ), # 7 +) +all_structs.append(GetPartitionNamesPsResponse) +GetPartitionNamesPsResponse.thrift_spec = ( + None, # 0 + ( + 1, + TType.LIST, + "names", + (TType.STRING, "UTF8", False), + None, + ), # 1 +) +all_structs.append(GetPartitionsPsWithAuthRequest) +GetPartitionsPsWithAuthRequest.thrift_spec = ( + None, # 0 + ( + 1, + TType.STRING, + "catName", + "UTF8", + None, + ), # 1 + ( + 2, + TType.STRING, + "dbName", + "UTF8", + None, + ), # 2 + ( + 3, + TType.STRING, + "tblName", + "UTF8", + None, + ), # 3 + ( + 4, + TType.LIST, + "partVals", + (TType.STRING, "UTF8", False), + None, + ), # 4 + ( + 5, + TType.I16, + "maxParts", + None, + -1, + ), # 5 + ( + 6, + TType.STRING, + "userName", + "UTF8", + None, + ), # 6 + ( + 7, + TType.LIST, + "groupNames", + (TType.STRING, "UTF8", False), + None, + ), # 7 + ( + 8, + TType.STRING, + "validWriteIdList", + "UTF8", + None, + ), # 8 + ( + 9, + TType.I64, + "id", + None, + -1, + ), # 9 +) +all_structs.append(GetPartitionsPsWithAuthResponse) +GetPartitionsPsWithAuthResponse.thrift_spec = ( + None, # 0 + ( + 1, + TType.LIST, + "partitions", + (TType.STRUCT, [Partition, None], False), + None, + ), # 1 +) +all_structs.append(ReplicationMetrics) +ReplicationMetrics.thrift_spec = ( + None, # 0 + ( + 1, + TType.I64, + "scheduledExecutionId", + None, + None, + ), # 1 + ( + 2, + TType.STRING, + "policy", + "UTF8", + None, + ), # 2 + ( + 3, + TType.I64, + "dumpExecutionId", + None, + None, + ), # 3 + ( + 4, + TType.STRING, + "metadata", + "UTF8", + None, + ), # 4 + ( + 5, + TType.STRING, + "progress", + "UTF8", + None, + ), # 5 + ( + 6, + TType.STRING, + "messageFormat", + "UTF8", + None, + ), # 6 +) +all_structs.append(ReplicationMetricList) +ReplicationMetricList.thrift_spec = ( + None, # 0 + ( + 1, + TType.LIST, + "replicationMetricList", + (TType.STRUCT, [ReplicationMetrics, None], False), + None, + ), # 1 +) +all_structs.append(GetReplicationMetricsRequest) +GetReplicationMetricsRequest.thrift_spec = ( + None, # 0 + ( + 1, + TType.I64, + "scheduledExecutionId", + None, + None, + ), # 1 + ( + 2, + TType.STRING, + "policy", + "UTF8", + None, + ), # 2 + ( + 3, + TType.I64, + "dumpExecutionId", + None, + None, + ), # 3 +) +all_structs.append(GetOpenTxnsRequest) +GetOpenTxnsRequest.thrift_spec = ( + None, # 0 + ( + 1, + TType.LIST, + "excludeTxnTypes", + (TType.I32, None, False), + None, + ), # 1 +) +all_structs.append(StoredProcedureRequest) +StoredProcedureRequest.thrift_spec = ( + None, # 0 + ( + 1, + TType.STRING, + "catName", + "UTF8", + None, + ), # 1 + ( + 2, + TType.STRING, + "dbName", + "UTF8", + None, + ), # 2 + ( + 3, + TType.STRING, + "procName", + "UTF8", + None, + ), # 3 +) +all_structs.append(ListStoredProcedureRequest) +ListStoredProcedureRequest.thrift_spec = ( + None, # 0 + ( + 1, + TType.STRING, + "catName", + "UTF8", + None, + ), # 1 + ( + 2, + TType.STRING, + "dbName", + "UTF8", + None, + ), # 2 +) +all_structs.append(StoredProcedure) +StoredProcedure.thrift_spec = ( + None, # 0 + ( + 1, + TType.STRING, + "name", + "UTF8", + None, + ), # 1 + ( + 2, + TType.STRING, + "dbName", + "UTF8", + None, + ), # 2 + ( + 3, + TType.STRING, + "catName", + "UTF8", + None, + ), # 3 + ( + 4, + TType.STRING, + "ownerName", + "UTF8", + None, + ), # 4 + ( + 5, + TType.STRING, + "source", + "UTF8", + None, + ), # 5 +) +all_structs.append(AddPackageRequest) +AddPackageRequest.thrift_spec = ( + None, # 0 + ( + 1, + TType.STRING, + "catName", + "UTF8", + None, + ), # 1 + ( + 2, + TType.STRING, + "dbName", + "UTF8", + None, + ), # 2 + ( + 3, + TType.STRING, + "packageName", + "UTF8", + None, + ), # 3 + ( + 4, + TType.STRING, + "ownerName", + "UTF8", + None, + ), # 4 + ( + 5, + TType.STRING, + "header", + "UTF8", + None, + ), # 5 + ( + 6, + TType.STRING, + "body", + "UTF8", + None, + ), # 6 +) +all_structs.append(GetPackageRequest) +GetPackageRequest.thrift_spec = ( + None, # 0 + ( + 1, + TType.STRING, + "catName", + "UTF8", + None, + ), # 1 + ( + 2, + TType.STRING, + "dbName", + "UTF8", + None, + ), # 2 + ( + 3, + TType.STRING, + "packageName", + "UTF8", + None, + ), # 3 +) +all_structs.append(DropPackageRequest) +DropPackageRequest.thrift_spec = ( + None, # 0 + ( + 1, + TType.STRING, + "catName", + "UTF8", + None, + ), # 1 + ( + 2, + TType.STRING, + "dbName", + "UTF8", + None, + ), # 2 + ( + 3, + TType.STRING, + "packageName", + "UTF8", + None, + ), # 3 +) +all_structs.append(ListPackageRequest) +ListPackageRequest.thrift_spec = ( + None, # 0 + ( + 1, + TType.STRING, + "catName", + "UTF8", + None, + ), # 1 + ( + 2, + TType.STRING, + "dbName", + "UTF8", + None, + ), # 2 +) +all_structs.append(Package) +Package.thrift_spec = ( + None, # 0 + ( + 1, + TType.STRING, + "catName", + "UTF8", + None, + ), # 1 + ( + 2, + TType.STRING, + "dbName", + "UTF8", + None, + ), # 2 + ( + 3, + TType.STRING, + "packageName", + "UTF8", + None, + ), # 3 + ( + 4, + TType.STRING, + "ownerName", + "UTF8", + None, + ), # 4 + ( + 5, + TType.STRING, + "header", + "UTF8", + None, + ), # 5 + ( + 6, + TType.STRING, + "body", + "UTF8", + None, + ), # 6 +) +all_structs.append(GetAllWriteEventInfoRequest) +GetAllWriteEventInfoRequest.thrift_spec = ( + None, # 0 + ( + 1, + TType.I64, + "txnId", + None, + None, + ), # 1 + ( + 2, + TType.STRING, + "dbName", + "UTF8", + None, + ), # 2 + ( + 3, + TType.STRING, + "tableName", + "UTF8", + None, + ), # 3 +) +all_structs.append(MetaException) +MetaException.thrift_spec = ( + None, # 0 + ( + 1, + TType.STRING, + "message", + "UTF8", + None, + ), # 1 +) +all_structs.append(UnknownTableException) +UnknownTableException.thrift_spec = ( + None, # 0 + ( + 1, + TType.STRING, + "message", + "UTF8", + None, + ), # 1 +) +all_structs.append(UnknownDBException) +UnknownDBException.thrift_spec = ( + None, # 0 + ( + 1, + TType.STRING, + "message", + "UTF8", + None, + ), # 1 +) +all_structs.append(AlreadyExistsException) +AlreadyExistsException.thrift_spec = ( + None, # 0 + ( + 1, + TType.STRING, + "message", + "UTF8", + None, + ), # 1 +) +all_structs.append(InvalidPartitionException) +InvalidPartitionException.thrift_spec = ( + None, # 0 + ( + 1, + TType.STRING, + "message", + "UTF8", + None, + ), # 1 +) +all_structs.append(UnknownPartitionException) +UnknownPartitionException.thrift_spec = ( + None, # 0 + ( + 1, + TType.STRING, + "message", + "UTF8", + None, + ), # 1 +) +all_structs.append(InvalidObjectException) +InvalidObjectException.thrift_spec = ( + None, # 0 + ( + 1, + TType.STRING, + "message", + "UTF8", + None, + ), # 1 +) +all_structs.append(NoSuchObjectException) +NoSuchObjectException.thrift_spec = ( + None, # 0 + ( + 1, + TType.STRING, + "message", + "UTF8", + None, + ), # 1 +) +all_structs.append(InvalidOperationException) +InvalidOperationException.thrift_spec = ( + None, # 0 + ( + 1, + TType.STRING, + "message", + "UTF8", + None, + ), # 1 +) +all_structs.append(ConfigValSecurityException) +ConfigValSecurityException.thrift_spec = ( + None, # 0 + ( + 1, + TType.STRING, + "message", + "UTF8", + None, + ), # 1 +) +all_structs.append(InvalidInputException) +InvalidInputException.thrift_spec = ( + None, # 0 + ( + 1, + TType.STRING, + "message", + "UTF8", + None, + ), # 1 +) +all_structs.append(NoSuchTxnException) +NoSuchTxnException.thrift_spec = ( + None, # 0 + ( + 1, + TType.STRING, + "message", + "UTF8", + None, + ), # 1 +) +all_structs.append(TxnAbortedException) +TxnAbortedException.thrift_spec = ( + None, # 0 + ( + 1, + TType.STRING, + "message", + "UTF8", + None, + ), # 1 +) +all_structs.append(TxnOpenException) +TxnOpenException.thrift_spec = ( + None, # 0 + ( + 1, + TType.STRING, + "message", + "UTF8", + None, + ), # 1 +) +all_structs.append(NoSuchLockException) +NoSuchLockException.thrift_spec = ( + None, # 0 + ( + 1, + TType.STRING, + "message", + "UTF8", + None, + ), # 1 +) +fix_spec(all_structs) +del all_structs From ac3e7116af0cd1f27870a4d48fb28a1f8dea7a6d Mon Sep 17 00:00:00 2001 From: Fokko Driesprong Date: Wed, 3 Aug 2022 18:56:42 +0200 Subject: [PATCH 152/642] Python: Handle OAuthErrorResponse properly (#5416) --- pyiceberg/catalog/rest.py | 63 ++++++++++++++++++++++++-------------- pyiceberg/exceptions.py | 4 +++ tests/catalog/test_rest.py | 13 ++++++++ 3 files changed, 57 insertions(+), 23 deletions(-) diff --git a/pyiceberg/catalog/rest.py b/pyiceberg/catalog/rest.py index bcedbc5cd4..afac2dd6c7 100644 --- a/pyiceberg/catalog/rest.py +++ b/pyiceberg/catalog/rest.py @@ -18,6 +18,7 @@ from typing import ( Dict, List, + Literal, Optional, Set, Tuple, @@ -40,6 +41,7 @@ NamespaceAlreadyExistsError, NoSuchNamespaceError, NoSuchTableError, + OAuthError, RESTError, ServerError, ServiceUnavailableError, @@ -148,6 +150,14 @@ class ErrorResponse(IcebergBaseModel): error: ErrorResponseMessage = Field() +class OAuthErrorResponse(IcebergBaseModel): + error: Literal[ + "invalid_request", "invalid_client", "invalid_grant", "unauthorized_client", "unsupported_grant_type", "invalid_scope" + ] + error_description: Optional[str] + error_uri: Optional[str] + + class RestCatalog(Catalog): token: str config: Properties @@ -233,7 +243,7 @@ def _fetch_access_token(self, credentials: str) -> str: try: response.raise_for_status() except HTTPError as exc: - self._handle_non_200_response(exc, {401: BadCredentialsError}) + self._handle_non_200_response(exc, {400: OAuthError, 401: BadCredentialsError}) return TokenResponse(**response.json()).access_token @@ -255,39 +265,46 @@ def _split_identifier_for_json(self, identifier: Union[str, Identifier]) -> Dict return {"namespace": identifier[:-1], "name": identifier[-1]} def _handle_non_200_response(self, exc: HTTPError, error_handler: Dict[int, Type[Exception]]): - try: - response = ErrorResponse(**exc.response.json()) - except JSONDecodeError: - # In the case we don't have a proper response - response = ErrorResponse( - error=ErrorResponseMessage( - message=f"Could not decode json payload: {exc.response.text}", - type="RESTError", - code=exc.response.status_code, - ) - ) - + exception: Type[Exception] code = exc.response.status_code if code in error_handler: - raise error_handler[code](response.error.message) from exc + exception = error_handler[code] elif code == 400: - raise BadRequestError(response.error.message) from exc + exception = BadRequestError elif code == 401: - raise UnauthorizedError(response.error.message) from exc + exception = UnauthorizedError elif code == 403: - raise ForbiddenError(response.error.message) from exc + exception = ForbiddenError elif code == 422: - raise RESTError(response.error.message) from exc + exception = RESTError elif code == 419: - raise AuthorizationExpiredError(response.error.message) + exception = AuthorizationExpiredError elif code == 501: - raise NotImplementedError(response.error.message) + exception = NotImplementedError elif code == 503: - raise ServiceUnavailableError(response.error.message) from exc + exception = ServiceUnavailableError elif 500 <= code < 600: - raise ServerError(response.error.message) from exc + exception = ServerError else: - raise RESTError(response.error.message) from exc + exception = RESTError + + try: + if exception == OAuthError: + # The OAuthErrorResponse has a different format + error = OAuthErrorResponse(**exc.response.json()) + response = str(error.error) + if description := error.error_description: + response += f": {description}" + if uri := error.error_uri: + response += f" ({uri})" + else: + error = ErrorResponse(**exc.response.json()).error + response = f"{error.type}: {error.message}" + except JSONDecodeError: + # In the case we don't have a proper response + response = f"RESTError: Could not decode json payload: {exc.response.text}" + + raise exception(response) from exc def create_table( self, diff --git a/pyiceberg/exceptions.py b/pyiceberg/exceptions.py index 432c25675f..b5db6a0e1b 100644 --- a/pyiceberg/exceptions.py +++ b/pyiceberg/exceptions.py @@ -70,3 +70,7 @@ class ForbiddenError(RESTError): class AuthorizationExpiredError(RESTError): """When the credentials are expired when performing an action on the REST catalog""" + + +class OAuthError(RESTError): + """Raises when there is an error with the OAuth call""" diff --git a/tests/catalog/test_rest.py b/tests/catalog/test_rest.py index b3c7b661f2..11b1885d54 100644 --- a/tests/catalog/test_rest.py +++ b/tests/catalog/test_rest.py @@ -27,6 +27,7 @@ NamespaceAlreadyExistsError, NoSuchNamespaceError, NoSuchTableError, + OAuthError, TableAlreadyExistsError, ) from pyiceberg.schema import Schema @@ -76,6 +77,18 @@ def test_token_200(rest_mock: Mocker): assert RestCatalog("rest", {}, TEST_URI, TEST_CREDENTIALS).token == TEST_TOKEN +def test_token_400(rest_mock: Mocker): + rest_mock.post( + f"{TEST_URI}v1/oauth/tokens", + json={"error": "invalid_client", "error_description": "Credentials for key invalid_key do not match"}, + status_code=400, + ) + + with pytest.raises(OAuthError) as e: + RestCatalog("rest", {}, TEST_URI, credentials=TEST_CREDENTIALS) + assert str(e.value) == "invalid_client: Credentials for key invalid_key do not match" + + def test_token_401(rest_mock: Mocker): message = "Invalid client ID: abc" rest_mock.post( From 8502e844734d2e2300e733d680c64cdd16af66ce Mon Sep 17 00:00:00 2001 From: Fokko Driesprong Date: Wed, 3 Aug 2022 18:57:34 +0200 Subject: [PATCH 153/642] Python: Add and cleanup tests (#5422) --- pyiceberg/catalog/rest.py | 14 ------------- tests/avro/test_resolver.py | 40 +++++++++++++++++++++++++++++++++++++ 2 files changed, 40 insertions(+), 14 deletions(-) diff --git a/pyiceberg/catalog/rest.py b/pyiceberg/catalog/rest.py index afac2dd6c7..c68f4468b5 100644 --- a/pyiceberg/catalog/rest.py +++ b/pyiceberg/catalog/rest.py @@ -21,7 +21,6 @@ Literal, Optional, Set, - Tuple, Type, Union, ) @@ -191,19 +190,6 @@ def __init__( self.config = self._fetch_config(properties) super().__init__(name, properties) - @staticmethod - def _split_credential(token: str) -> Tuple[str, str]: - """Splits the token in a client id and secret - - Args: - token: The token with a semicolon as a separator - - Returns: - The client id and secret - """ - client, secret = token.split(":") - return client, secret - @property def headers(self) -> Properties: headers = { diff --git a/tests/avro/test_resolver.py b/tests/avro/test_resolver.py index 3b51a4ab48..8c1309d620 100644 --- a/tests/avro/test_resolver.py +++ b/tests/avro/test_resolver.py @@ -177,6 +177,46 @@ def test_promote_decimal_to_decimal(): assert promote(DecimalType(19, 25), DecimalType(22, 25)) == DecimalReader(22, 25) +def test_struct_not_aligned(): + with pytest.raises(ResolveException): + assert promote(StructType(), StringType()) + + +def test_map_not_aligned(): + with pytest.raises(ResolveException): + assert promote(MapType(1, StringType(), 2, IntegerType()), StringType()) + + +def test_primitive_not_aligned(): + with pytest.raises(ResolveException): + assert promote(IntegerType(), MapType(1, StringType(), 2, IntegerType())) + + +def test_integer_not_aligned(): + with pytest.raises(ResolveException): + assert promote(IntegerType(), StringType()) + + +def test_float_not_aligned(): + with pytest.raises(ResolveException): + assert promote(FloatType(), StringType()) + + +def test_string_not_aligned(): + with pytest.raises(ResolveException): + assert promote(StringType(), FloatType()) + + +def test_binary_not_aligned(): + with pytest.raises(ResolveException): + assert promote(BinaryType(), FloatType()) + + +def test_decimal_not_aligned(): + with pytest.raises(ResolveException): + assert promote(DecimalType(22, 19), StringType()) + + def test_promote_decimal_to_decimal_reduce_precision(): # DecimalType(P, S) to DecimalType(P2, S) where P2 > P with pytest.raises(ResolveException) as exc_info: From 1623586bc9e19cd84eaf398880f66052c568e922 Mon Sep 17 00:00:00 2001 From: Fokko Driesprong Date: Fri, 5 Aug 2022 21:03:29 +0200 Subject: [PATCH 154/642] Python: Add http status code to RESTError (#5449) If it isn't caught, we still want to know what the error code was --- pyiceberg/catalog/rest.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyiceberg/catalog/rest.py b/pyiceberg/catalog/rest.py index c68f4468b5..bd6ea36b34 100644 --- a/pyiceberg/catalog/rest.py +++ b/pyiceberg/catalog/rest.py @@ -288,7 +288,7 @@ def _handle_non_200_response(self, exc: HTTPError, error_handler: Dict[int, Type response = f"{error.type}: {error.message}" except JSONDecodeError: # In the case we don't have a proper response - response = f"RESTError: Could not decode json payload: {exc.response.text}" + response = f"RESTError {exc.response.status_code}: Could not decode json payload: {exc.response.text}" raise exception(response) from exc From 32cae832572bfcf929b04787143d857123f06635 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Sun, 7 Aug 2022 09:14:07 -0700 Subject: [PATCH 155/642] Bump coverage from 6.4.2 to 6.4.3 in /python (#5454) Bumps [coverage](https://github.com/nedbat/coveragepy) from 6.4.2 to 6.4.3. - [Release notes](https://github.com/nedbat/coveragepy/releases) - [Changelog](https://github.com/nedbat/coveragepy/blob/master/CHANGES.rst) - [Commits](https://github.com/nedbat/coveragepy/compare/6.4.2...6.4.3) Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- poetry.lock | 197 ++++++++++++++++++++++++++++++++++++++++++++----- pyproject.toml | 2 +- 2 files changed, 179 insertions(+), 20 deletions(-) diff --git a/poetry.lock b/poetry.lock index b7ecd0bd04..4b64cf6d16 100644 --- a/poetry.lock +++ b/poetry.lock @@ -68,7 +68,7 @@ python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*" [[package]] name = "coverage" -version = "6.4.2" +version = "6.4.3" description = "Code coverage measurement for Python" category = "dev" optional = false @@ -232,8 +232,8 @@ optional = false python-versions = ">=3.6" [package.extras] -dev = ["pre-commit", "tox"] -testing = ["pytest", "pytest-benchmark"] +testing = ["pytest-benchmark", "pytest"] +dev = ["tox", "pre-commit"] [[package]] name = "pre-commit" @@ -505,11 +505,16 @@ snappy = ["python-snappy"] [metadata] lock-version = "1.1" python-versions = "^3.8" -content-hash = "fa4637615911260d5d031479bd2e2481d952e1ecb6feaddfe6be6c358132465c" +content-hash = "5f3903f6d4a01345a4c7ae00a6e4fa74fe08bd062ab61394976ccd36674c890d" [metadata.files] -atomicwrites = [] -attrs = [] +atomicwrites = [ + {file = "atomicwrites-1.4.1.tar.gz", hash = "sha256:81b2c9071a49367a7f770170e5eec8cb66567cfbbc8c73d20ce5ca4a8d71cf11"}, +] +attrs = [ + {file = "attrs-22.1.0-py2.py3-none-any.whl", hash = "sha256:86efa402f67bf2df34f51a335487cf46b1ec130d02b8d39fd248abfd30da551c"}, + {file = "attrs-22.1.0.tar.gz", hash = "sha256:29adc2665447e5191d0e7c568fde78b21f9672d344281d0c6e1ab085429b22b6"}, +] certifi = [ {file = "certifi-2022.6.15-py3-none-any.whl", hash = "sha256:fe86415d55e84719d75f8b69414f6438ac3547d2078ab91b67e779ef69378412"}, {file = "certifi-2022.6.15.tar.gz", hash = "sha256:84c85a9078b11105f04f3036a9482ae10e4621616db313fe045dd24743a0820d"}, @@ -592,18 +597,84 @@ colorama = [ {file = "colorama-0.4.5-py2.py3-none-any.whl", hash = "sha256:854bf444933e37f5824ae7bfc1e98d5bce2ebe4160d46b5edf346a89358e99da"}, {file = "colorama-0.4.5.tar.gz", hash = "sha256:e6c6b4334fc50988a639d9b98aa429a0b57da6e17b9a44f0451f930b6967b7a4"}, ] -coverage = [] -distlib = [] +coverage = [ + {file = "coverage-6.4.3-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:f50d3a822947572496ea922ee7825becd8e3ae6fbd2400cd8236b7d64b17f285"}, + {file = "coverage-6.4.3-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:d5191d53afbe5b6059895fa7f58223d3751c42b8101fb3ce767e1a0b1a1d8f87"}, + {file = "coverage-6.4.3-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:04010af3c06ce2bfeb3b1e4e05d136f88d88c25f76cd4faff5d1fd84d11581ea"}, + {file = "coverage-6.4.3-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:6630d8d943644ea62132789940ca97d05fac83f73186eaf0930ffa715fbdab6b"}, + {file = "coverage-6.4.3-cp310-cp310-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:05de0762c1caed4a162b3e305f36cf20a548ff4da0be6766ad5c870704be3660"}, + {file = "coverage-6.4.3-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:0e3a41aad5919613483aad9ebd53336905cab1bd6788afd3995c2a972d89d795"}, + {file = "coverage-6.4.3-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:a2738ba1ee544d6f294278cfb6de2dc1f9a737a780469b5366e662a218f806c3"}, + {file = "coverage-6.4.3-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:a0d2df4227f645a879010461df2cea6b7e3fb5a97d7eafa210f7fb60345af9e8"}, + {file = "coverage-6.4.3-cp310-cp310-win32.whl", hash = "sha256:73a10939dc345460ca0655356a470dd3de9759919186a82383c87b6eb315faf2"}, + {file = "coverage-6.4.3-cp310-cp310-win_amd64.whl", hash = "sha256:53c8edd3b83a4ddba3d8c506f1359401e7770b30f2188f15c17a338adf5a14db"}, + {file = "coverage-6.4.3-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:f1eda5cae434282712e40b42aaf590b773382afc3642786ac3ed39053973f61f"}, + {file = "coverage-6.4.3-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:59fc88bc13e30f25167e807b8cad3c41b7218ef4473a20c86fd98a7968733083"}, + {file = "coverage-6.4.3-cp37-cp37m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:d75314b00825d70e1e34b07396e23f47ed1d4feedc0122748f9f6bd31a544840"}, + {file = "coverage-6.4.3-cp37-cp37m-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:52f8b9fcf3c5e427d51bbab1fb92b575a9a9235d516f175b24712bcd4b5be917"}, + {file = "coverage-6.4.3-cp37-cp37m-musllinux_1_1_aarch64.whl", hash = "sha256:5a559aab40c716de80c7212295d0dc96bc1b6c719371c20dd18c5187c3155518"}, + {file = "coverage-6.4.3-cp37-cp37m-musllinux_1_1_i686.whl", hash = "sha256:306788fd019bb90e9cbb83d3f3c6becad1c048dd432af24f8320cf38ac085684"}, + {file = "coverage-6.4.3-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:920a734fe3d311ca01883b4a19aa386c97b82b69fbc023458899cff0a0d621b9"}, + {file = "coverage-6.4.3-cp37-cp37m-win32.whl", hash = "sha256:ab9ef0187d6c62b09dec83a84a3b94f71f9690784c84fd762fb3cf2d2b44c914"}, + {file = "coverage-6.4.3-cp37-cp37m-win_amd64.whl", hash = "sha256:39ebd8e120cb77a06ee3d5fc26f9732670d1c397d7cd3acf02f6f62693b89b80"}, + {file = "coverage-6.4.3-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:bc698580216050b5f4a34d2cdd2838b429c53314f1c4835fab7338200a8396f2"}, + {file = "coverage-6.4.3-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:877ee5478fd78e100362aed56db47ccc5f23f6e7bb035a8896855f4c3e49bc9b"}, + {file = "coverage-6.4.3-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:555a498999c44f5287cc95500486cd0d4f021af9162982cbe504d4cb388f73b5"}, + {file = "coverage-6.4.3-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:eff095a5aac7011fdb51a2c82a8fae9ec5211577f4b764e1e59cfa27ceeb1b59"}, + {file = "coverage-6.4.3-cp38-cp38-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:5de1e9335e2569974e20df0ce31493d315a830d7987e71a24a2a335a8d8459d3"}, + {file = "coverage-6.4.3-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:7856ea39059d75f822ff0df3a51ea6d76307c897048bdec3aad1377e4e9dca20"}, + {file = "coverage-6.4.3-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:411fdd9f4203afd93b056c0868c8f9e5e16813e765de962f27e4e5798356a052"}, + {file = "coverage-6.4.3-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:cdf7b83f04a313a21afb1f8730fe4dd09577fefc53bbdfececf78b2006f4268e"}, + {file = "coverage-6.4.3-cp38-cp38-win32.whl", hash = "sha256:ab2b1a89d2bc7647622e9eaf06128a5b5451dccf7c242deaa31420b055716481"}, + {file = "coverage-6.4.3-cp38-cp38-win_amd64.whl", hash = "sha256:0e34247274bde982bbc613894d33f9e36358179db2ed231dd101c48dd298e7b0"}, + {file = "coverage-6.4.3-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:b104b6b1827d6a22483c469e3983a204bcf9c6bf7544bf90362c4654ebc2edf3"}, + {file = "coverage-6.4.3-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:adf1a0d272633b21d645dd6e02e3293429c1141c7d65a58e4cbcd592d53b8e01"}, + {file = "coverage-6.4.3-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:ff9832434a9193fbd716fbe05f9276484e18d26cc4cf850853594bb322807ac3"}, + {file = "coverage-6.4.3-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:923f9084d7e1d31b5f74c92396b05b18921ed01ee5350402b561a79dce3ea48d"}, + {file = "coverage-6.4.3-cp39-cp39-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:e4d64304acf79766e650f7acb81d263a3ea6e2d0d04c5172b7189180ff2c023c"}, + {file = "coverage-6.4.3-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:fc294de50941d3da66a09dca06e206297709332050973eca17040278cb0918ff"}, + {file = "coverage-6.4.3-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:a42eaaae772f14a5194f181740a67bfd48e8806394b8c67aa4399e09d0d6b5db"}, + {file = "coverage-6.4.3-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:4822327b35cb032ff16af3bec27f73985448f08e874146b5b101e0e558b613dd"}, + {file = "coverage-6.4.3-cp39-cp39-win32.whl", hash = "sha256:f217850ac0e046ede611312703423767ca032a7b952b5257efac963942c055de"}, + {file = "coverage-6.4.3-cp39-cp39-win_amd64.whl", hash = "sha256:0a84376e4fd13cebce2c0ef8c2f037929c8307fb94af1e5dbe50272a1c651b5d"}, + {file = "coverage-6.4.3-pp36.pp37.pp38-none-any.whl", hash = "sha256:068d6f2a893af838291b8809c876973d885543411ea460f3e6886ac0ee941732"}, + {file = "coverage-6.4.3.tar.gz", hash = "sha256:ec2ae1f398e5aca655b7084392d23e80efb31f7a660d2eecf569fb9f79b3fb94"}, +] +distlib = [ + {file = "distlib-0.3.5-py2.py3-none-any.whl", hash = "sha256:b710088c59f06338ca514800ad795a132da19fda270e3ce4affc74abf955a26c"}, + {file = "distlib-0.3.5.tar.gz", hash = "sha256:a7f75737c70be3b25e2bee06288cec4e4c221de18455b2dd037fe2a795cab2fe"}, +] docutils = [ {file = "docutils-0.19-py3-none-any.whl", hash = "sha256:5e1de4d849fee02c63b040a4a3fd567f4ab104defd8a5511fbbc24a8a017efbc"}, {file = "docutils-0.19.tar.gz", hash = "sha256:33995a6753c30b7f577febfc2c50411fec6aac7f7ffeb7c4cfe5991072dcf9e6"}, ] -fastavro = [] +fastavro = [ + {file = "fastavro-1.5.4-cp310-cp310-macosx_10_14_x86_64.whl", hash = "sha256:d316cc476b2b24ef06402b8bfa047f8f72a9d6df2de777bb30d9ededda7e3a02"}, + {file = "fastavro-1.5.4-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:8459faec46e34f2dfeb9b70ee8c36e935e626cff8608d675724718987a5f9ce5"}, + {file = "fastavro-1.5.4-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:cd44636d7ff8365a57b88707b747371fffb676c8c1f68c0d423ec36623888668"}, + {file = "fastavro-1.5.4-cp310-cp310-win_amd64.whl", hash = "sha256:2402428b26d3c08a58acfa723833e19fb75077872bcb2475a4c81195cdae6a5d"}, + {file = "fastavro-1.5.4-cp37-cp37m-macosx_10_15_x86_64.whl", hash = "sha256:5afc14398f4191d1a807aa59d2fba5ed869b31343679ec43dbc289db0a8e35c5"}, + {file = "fastavro-1.5.4-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:5217e9713a3ea03205532394fba4d743749155b04b10b12a12fc26d225b89792"}, + {file = "fastavro-1.5.4-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:2e93a5eecb28cc35d670c9c4df70223fa9bcd6d9ca21b38b1b7ae13ece60c7fb"}, + {file = "fastavro-1.5.4-cp37-cp37m-win_amd64.whl", hash = "sha256:1a2f2465efd0e7de557c4034e8d4d88a132750cfa51e1582362a1b3a1a9fa911"}, + {file = "fastavro-1.5.4-cp38-cp38-macosx_10_15_x86_64.whl", hash = "sha256:f7d5bc76c03c692d9acea0e5d5baceec19e1c059b26cb8ae9f4481e842be95a5"}, + {file = "fastavro-1.5.4-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:80fe920229ab1f40eccb1b4918481cdd8a20e5e7dce19308ab38b23732da8a70"}, + {file = "fastavro-1.5.4-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:a3d190aee86ab73caa1aa550eba850be2ca5dd29d814b38720f4e300184e01d5"}, + {file = "fastavro-1.5.4-cp38-cp38-win_amd64.whl", hash = "sha256:b6c30299a49b11f42251cb81c8e15db67750642eac7ba5c194a5ee95c83ebb11"}, + {file = "fastavro-1.5.4-cp39-cp39-macosx_10_15_x86_64.whl", hash = "sha256:1f7685f3a3c38352abab432bad2f9f2229a0e5f5f8548831e887c30f8396f2e9"}, + {file = "fastavro-1.5.4-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:bd021ec850fd30020b7c4fa868466fb7f95450f1f06eac92bd2204cbd8e45fb8"}, + {file = "fastavro-1.5.4-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:06a7b5602dfa032c92f20ca90b8bde88251573773e501bedf5e8b76b9feb14a3"}, + {file = "fastavro-1.5.4-cp39-cp39-win_amd64.whl", hash = "sha256:18250aa2ab0f7a095b1865565cf9976ea4605c201129636e6defe24ec3ef112c"}, + {file = "fastavro-1.5.4.tar.gz", hash = "sha256:d86f72c966713fb699570a18f7960cf4110b069c70681d7538be8d671c9db7c8"}, +] filelock = [ {file = "filelock-3.7.1-py3-none-any.whl", hash = "sha256:37def7b658813cda163b56fc564cdc75e86d338246458c4c28ae84cabefa2404"}, {file = "filelock-3.7.1.tar.gz", hash = "sha256:3a0fd85166ad9dbab54c9aec96737b744106dc5f15c0b09a6744a445299fcf04"}, ] -identify = [] +identify = [ + {file = "identify-2.5.2-py2.py3-none-any.whl", hash = "sha256:feaa9db2dc0ce333b453ce171c0cf1247bbfde2c55fc6bb785022d411a1b78b5"}, + {file = "identify-2.5.2.tar.gz", hash = "sha256:a3d4c096b384d50d5e6dc5bc8b9bc44f1f61cefebd750a7b3e9f939b53fb214d"}, +] idna = [ {file = "idna-3.3-py3-none-any.whl", hash = "sha256:84d9dd047ffa80596e0f246e2eab0b391788b0503584e8945f2368256d2735ff"}, {file = "idna-3.3.tar.gz", hash = "sha256:9d643ff0a55b762d5cdb124b8eaa99c66322e2157b69160bc32796e824360e6d"}, @@ -648,12 +719,38 @@ nodeenv = [ {file = "nodeenv-1.7.0-py2.py3-none-any.whl", hash = "sha256:27083a7b96a25f2f5e1d8cb4b6317ee8aeda3bdd121394e5ac54e498028a042e"}, {file = "nodeenv-1.7.0.tar.gz", hash = "sha256:e0e7f7dfb85fc5394c6fe1e8fa98131a2473e04311a45afb6508f7cf1836fa2b"}, ] -numpy = [] +numpy = [ + {file = "numpy-1.23.1-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:b15c3f1ed08df4980e02cc79ee058b788a3d0bef2fb3c9ca90bb8cbd5b8a3a04"}, + {file = "numpy-1.23.1-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:9ce242162015b7e88092dccd0e854548c0926b75c7924a3495e02c6067aba1f5"}, + {file = "numpy-1.23.1-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:e0d7447679ae9a7124385ccf0ea990bb85bb869cef217e2ea6c844b6a6855073"}, + {file = "numpy-1.23.1-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:3119daed207e9410eaf57dcf9591fdc68045f60483d94956bee0bfdcba790953"}, + {file = "numpy-1.23.1-cp310-cp310-win32.whl", hash = "sha256:3ab67966c8d45d55a2bdf40701536af6443763907086c0a6d1232688e27e5447"}, + {file = "numpy-1.23.1-cp310-cp310-win_amd64.whl", hash = "sha256:1865fdf51446839ca3fffaab172461f2b781163f6f395f1aed256b1ddc253622"}, + {file = "numpy-1.23.1-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:aeba539285dcf0a1ba755945865ec61240ede5432df41d6e29fab305f4384db2"}, + {file = "numpy-1.23.1-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:7e8229f3687cdadba2c4faef39204feb51ef7c1a9b669247d49a24f3e2e1617c"}, + {file = "numpy-1.23.1-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:68b69f52e6545af010b76516f5daaef6173e73353e3295c5cb9f96c35d755641"}, + {file = "numpy-1.23.1-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:1408c3527a74a0209c781ac82bde2182b0f0bf54dea6e6a363fe0cc4488a7ce7"}, + {file = "numpy-1.23.1-cp38-cp38-win32.whl", hash = "sha256:47f10ab202fe4d8495ff484b5561c65dd59177949ca07975663f4494f7269e3e"}, + {file = "numpy-1.23.1-cp38-cp38-win_amd64.whl", hash = "sha256:37e5ebebb0eb54c5b4a9b04e6f3018e16b8ef257d26c8945925ba8105008e645"}, + {file = "numpy-1.23.1-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:173f28921b15d341afadf6c3898a34f20a0569e4ad5435297ba262ee8941e77b"}, + {file = "numpy-1.23.1-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:876f60de09734fbcb4e27a97c9a286b51284df1326b1ac5f1bf0ad3678236b22"}, + {file = "numpy-1.23.1-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:35590b9c33c0f1c9732b3231bb6a72d1e4f77872390c47d50a615686ae7ed3fd"}, + {file = "numpy-1.23.1-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:a35c4e64dfca659fe4d0f1421fc0f05b8ed1ca8c46fb73d9e5a7f175f85696bb"}, + {file = "numpy-1.23.1-cp39-cp39-win32.whl", hash = "sha256:c2f91f88230042a130ceb1b496932aa717dcbd665350beb821534c5c7e15881c"}, + {file = "numpy-1.23.1-cp39-cp39-win_amd64.whl", hash = "sha256:37ece2bd095e9781a7156852e43d18044fd0d742934833335599c583618181b9"}, + {file = "numpy-1.23.1-pp38-pypy38_pp73-macosx_10_9_x86_64.whl", hash = "sha256:8002574a6b46ac3b5739a003b5233376aeac5163e5dcd43dd7ad062f3e186129"}, + {file = "numpy-1.23.1-pp38-pypy38_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:5d732d17b8a9061540a10fda5bfeabca5785700ab5469a5e9b93aca5e2d3a5fb"}, + {file = "numpy-1.23.1-pp38-pypy38_pp73-win_amd64.whl", hash = "sha256:55df0f7483b822855af67e38fb3a526e787adf189383b4934305565d71c4b148"}, + {file = "numpy-1.23.1.tar.gz", hash = "sha256:d748ef349bfef2e1194b59da37ed5a29c19ea8d7e6342019921ba2ba4fd8b624"}, +] packaging = [ {file = "packaging-21.3-py3-none-any.whl", hash = "sha256:ef103e05f519cdc783ae24ea4e2e0f508a9c99b2d4969652eed6a2e1ea5bd522"}, {file = "packaging-21.3.tar.gz", hash = "sha256:dd47c42927d89ab911e606518907cc2d3a1f38bbd026385970643f9c5b8ecfeb"}, ] -pep517 = [] +pep517 = [ + {file = "pep517-0.13.0-py3-none-any.whl", hash = "sha256:4ba4446d80aed5b5eac6509ade100bff3e7943a8489de249654a5ae9b33ee35b"}, + {file = "pep517-0.13.0.tar.gz", hash = "sha256:ae69927c5c172be1add9203726d4b84cf3ebad1edcd5f71fcdc746e66e829f59"}, +] platformdirs = [ {file = "platformdirs-2.5.2-py3-none-any.whl", hash = "sha256:027d8e83a2d7de06bbac4e5ef7e023c02b863d7ea5d079477e722bb41ab25788"}, {file = "platformdirs-2.5.2.tar.gz", hash = "sha256:58c8abb07dcb441e6ee4b11d8df0ac856038f944ab98b7be6b27b2a3c7feef19"}, @@ -662,7 +759,10 @@ pluggy = [ {file = "pluggy-1.0.0-py2.py3-none-any.whl", hash = "sha256:74134bbf457f031a36d68416e1509f34bd5ccc019f0bcc952c7b909d06b37bd3"}, {file = "pluggy-1.0.0.tar.gz", hash = "sha256:4224373bacce55f955a878bf9cfa763c1e360858e330072059e10bad68531159"}, ] -pre-commit = [] +pre-commit = [ + {file = "pre_commit-2.20.0-py2.py3-none-any.whl", hash = "sha256:51a5ba7c480ae8072ecdb6933df22d2f812dc897d5fe848778116129a681aac7"}, + {file = "pre_commit-2.20.0.tar.gz", hash = "sha256:a978dac7bc9ec0bcee55c18a277d553b0f419d259dadb4b9418ff2d00eb43959"}, +] py = [ {file = "py-1.11.0-py2.py3-none-any.whl", hash = "sha256:607c53218732647dff4acdfcd50cb62615cedf612e72d1724fb1a0cc6405b378"}, {file = "py-1.11.0.tar.gz", hash = "sha256:51c75c4126074b472f746a24399ad32f6053d1b34b68d2fa41e558e6f4a98719"}, @@ -841,12 +941,17 @@ requests = [ {file = "requests-2.28.1-py3-none-any.whl", hash = "sha256:8fefa2a1a1365bf5520aac41836fbee479da67864514bdb821f31ce07ce65349"}, {file = "requests-2.28.1.tar.gz", hash = "sha256:7c5599b102feddaa661c826c56ab4fee28bfd17f5abca1ebbe3e7f19d7c97983"}, ] -requests-mock = [] +requests-mock = [ + {file = "requests-mock-1.9.3.tar.gz", hash = "sha256:8d72abe54546c1fc9696fa1516672f1031d72a55a1d66c85184f972a24ba0eba"}, + {file = "requests_mock-1.9.3-py2.py3-none-any.whl", hash = "sha256:0a2d38a117c08bb78939ec163522976ad59a6b7fdd82b709e23bb98004a44970"}, +] six = [ {file = "six-1.16.0-py2.py3-none-any.whl", hash = "sha256:8abb2f1d86890a2dfb989f9a77cfcfd3e47c2a354b01111771326f8aa26e0254"}, {file = "six-1.16.0.tar.gz", hash = "sha256:1e61c37477a1626458e36f7b1d82aa5c9b094fa4802892072e49de9c60c4c926"}, ] -thrift = [] +thrift = [ + {file = "thrift-0.16.0.tar.gz", hash = "sha256:2b5b6488fcded21f9d312aa23c9ff6a0195d0f6ae26ddbd5ad9e3e25dfc14408"}, +] toml = [ {file = "toml-0.10.2-py2.py3-none-any.whl", hash = "sha256:806143ae5bfb6a3c6e736a764057db0e6a0e05e338b5630894a5f779cabb4f9b"}, {file = "toml-0.10.2.tar.gz", hash = "sha256:b3bda1d108d5dd99f4a20d24d9c348e91c4db7ab1b749200bded2f839ccbe68f"}, @@ -859,7 +964,61 @@ typing-extensions = [ {file = "typing_extensions-4.3.0-py3-none-any.whl", hash = "sha256:25642c956049920a5aa49edcdd6ab1e06d7e5d467fc00e0506c44ac86fbfca02"}, {file = "typing_extensions-4.3.0.tar.gz", hash = "sha256:e6d2677a32f47fc7eb2795db1dd15c1f34eff616bcaf2cfb5e997f854fa1c4a6"}, ] -urllib3 = [] -virtualenv = [] -zipp = [] -zstandard = [] +urllib3 = [ + {file = "urllib3-1.26.11-py2.py3-none-any.whl", hash = "sha256:c33ccba33c819596124764c23a97d25f32b28433ba0dedeb77d873a38722c9bc"}, + {file = "urllib3-1.26.11.tar.gz", hash = "sha256:ea6e8fb210b19d950fab93b60c9009226c63a28808bc8386e05301e25883ac0a"}, +] +virtualenv = [ + {file = "virtualenv-20.16.2-py2.py3-none-any.whl", hash = "sha256:635b272a8e2f77cb051946f46c60a54ace3cb5e25568228bd6b57fc70eca9ff3"}, + {file = "virtualenv-20.16.2.tar.gz", hash = "sha256:0ef5be6d07181946891f5abc8047fda8bc2f0b4b9bf222c64e6e8963baee76db"}, +] +zipp = [ + {file = "zipp-3.8.1-py3-none-any.whl", hash = "sha256:47c40d7fe183a6f21403a199b3e4192cca5774656965b0a4988ad2f8feb5f009"}, + {file = "zipp-3.8.1.tar.gz", hash = "sha256:05b45f1ee8f807d0cc928485ca40a07cb491cf092ff587c0df9cb1fd154848d2"}, +] +zstandard = [ + {file = "zstandard-0.18.0-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:ef7e8a200e4c8ac9102ed3c90ed2aa379f6b880f63032200909c1be21951f556"}, + {file = "zstandard-0.18.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:2dc466207016564805e56d28375f4f533b525ff50d6776946980dff5465566ac"}, + {file = "zstandard-0.18.0-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:4a2ee1d4f98447f3e5183ecfce5626f983504a4a0c005fbe92e60fa8e5d547ec"}, + {file = "zstandard-0.18.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:d956e2f03c7200d7e61345e0880c292783ec26618d0d921dcad470cb195bbce2"}, + {file = "zstandard-0.18.0-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_12_i686.manylinux2010_i686.whl", hash = "sha256:ce6f59cba9854fd14da5bfe34217a1501143057313966637b7291d1b0267bd1e"}, + {file = "zstandard-0.18.0-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:a7fa67cba473623848b6e88acf8d799b1906178fd883fb3a1da24561c779593b"}, + {file = "zstandard-0.18.0-cp310-cp310-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:cdb44d7284c8c5dd1b66dfb86dda7f4560fa94bfbbc1d2da749ba44831335e32"}, + {file = "zstandard-0.18.0-cp310-cp310-win32.whl", hash = "sha256:63694a376cde0aa8b1971d06ca28e8f8b5f492779cb6ee1cc46bbc3f019a42a5"}, + {file = "zstandard-0.18.0-cp310-cp310-win_amd64.whl", hash = "sha256:702a8324cd90c74d9c8780d02bf55e79da3193c870c9665ad3a11647e3ad1435"}, + {file = "zstandard-0.18.0-cp36-cp36m-macosx_10_9_x86_64.whl", hash = "sha256:46f679bc5dfd938db4fb058218d9dc4db1336ffaf1ea774ff152ecadabd40805"}, + {file = "zstandard-0.18.0-cp36-cp36m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:dc2a4de9f363b3247d472362a65041fe4c0f59e01a2846b15d13046be866a885"}, + {file = "zstandard-0.18.0-cp36-cp36m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:bd3220d7627fd4d26397211cb3b560ec7cc4a94b75cfce89e847e8ce7fabe32d"}, + {file = "zstandard-0.18.0-cp36-cp36m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_12_i686.manylinux2010_i686.whl", hash = "sha256:39e98cf4773234bd9cebf9f9db730e451dfcfe435e220f8921242afda8321887"}, + {file = "zstandard-0.18.0-cp36-cp36m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:5228e596eb1554598c872a337bbe4e5afe41cd1f8b1b15f2e35b50d061e35244"}, + {file = "zstandard-0.18.0-cp36-cp36m-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:d4a8fd45746a6c31e729f35196e80b8f1e9987c59f5ccb8859d7c6a6fbeb9c63"}, + {file = "zstandard-0.18.0-cp36-cp36m-win32.whl", hash = "sha256:4cbb85f29a990c2fdbf7bc63246567061a362ddca886d7fae6f780267c0a9e67"}, + {file = "zstandard-0.18.0-cp36-cp36m-win_amd64.whl", hash = "sha256:bfa6c8549fa18e6497a738b7033c49f94a8e2e30c5fbe2d14d0b5aa8bbc1695d"}, + {file = "zstandard-0.18.0-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:e02043297c1832f2666cd2204f381bef43b10d56929e13c42c10c732c6e3b4ed"}, + {file = "zstandard-0.18.0-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:7231543d38d2b7e02ef7cc78ef7ffd86419437e1114ff08709fe25a160e24bd6"}, + {file = "zstandard-0.18.0-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:c86befac87445927488f5c8f205d11566f64c11519db223e9d282b945fa60dab"}, + {file = "zstandard-0.18.0-cp37-cp37m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_12_i686.manylinux2010_i686.whl", hash = "sha256:999a4e1768f219826ba3fa2064fab1c86dd72fdd47a42536235478c3bb3ca3e2"}, + {file = "zstandard-0.18.0-cp37-cp37m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:9df59cd1cf3c62075ee2a4da767089d19d874ac3ad42b04a71a167e91b384722"}, + {file = "zstandard-0.18.0-cp37-cp37m-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:1be31e9e3f7607ee0cdd60915410a5968b205d3e7aa83b7fcf3dd76dbbdb39e0"}, + {file = "zstandard-0.18.0-cp37-cp37m-win32.whl", hash = "sha256:490d11b705b8ae9dc845431bacc8dd1cef2408aede176620a5cd0cd411027936"}, + {file = "zstandard-0.18.0-cp37-cp37m-win_amd64.whl", hash = "sha256:266aba27fa9cc5e9091d3d325ebab1fa260f64e83e42516d5e73947c70216a5b"}, + {file = "zstandard-0.18.0-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:8b2260c4e07dd0723eadb586de7718b61acca4083a490dda69c5719d79bc715c"}, + {file = "zstandard-0.18.0-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:3af8c2383d02feb6650e9255491ec7d0824f6e6dd2bbe3e521c469c985f31fb1"}, + {file = "zstandard-0.18.0-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:28723a1d2e4df778573b76b321ebe9f3469ac98988104c2af116dd344802c3f8"}, + {file = "zstandard-0.18.0-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:19cac7108ff2c342317fad6dc97604b47a41f403c8f19d0bfc396dfadc3638b8"}, + {file = "zstandard-0.18.0-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_12_i686.manylinux2010_i686.whl", hash = "sha256:76725d1ee83a8915100a310bbad5d9c1fc6397410259c94033b8318d548d9990"}, + {file = "zstandard-0.18.0-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:d716a7694ce1fa60b20bc10f35c4a22be446ef7f514c8dbc8f858b61976de2fb"}, + {file = "zstandard-0.18.0-cp38-cp38-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:49685bf9a55d1ab34bd8423ea22db836ba43a181ac6b045ac4272093d5cb874e"}, + {file = "zstandard-0.18.0-cp38-cp38-win32.whl", hash = "sha256:1af1268a7dc870eb27515fb8db1f3e6c5a555d2b7bcc476fc3bab8886c7265ab"}, + {file = "zstandard-0.18.0-cp38-cp38-win_amd64.whl", hash = "sha256:1dc2d3809e763055a1a6c1a73f2b677320cc9a5aa1a7c6cfb35aee59bddc42d9"}, + {file = "zstandard-0.18.0-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:eea18c1e7442f2aa9aff1bb84550dbb6a1f711faf6e48e7319de8f2b2e923c2a"}, + {file = "zstandard-0.18.0-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:8677ffc6a6096cccbd892e558471c901fd821aba12b7fbc63833c7346f549224"}, + {file = "zstandard-0.18.0-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:083dc08abf03807af9beeb2b6a91c23ad78add2499f828176a3c7b742c44df02"}, + {file = "zstandard-0.18.0-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:c990063664c08169c84474acecc9251ee035871589025cac47c060ff4ec4bc1a"}, + {file = "zstandard-0.18.0-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_12_i686.manylinux2010_i686.whl", hash = "sha256:533db8a6fac6248b2cb2c935e7b92f994efbdeb72e1ffa0b354432e087bb5a3e"}, + {file = "zstandard-0.18.0-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:dbb3cb8a082d62b8a73af42291569d266b05605e017a3d8a06a0e5c30b5f10f0"}, + {file = "zstandard-0.18.0-cp39-cp39-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:d6c85ca5162049ede475b7ec98e87f9390501d44a3d6776ddd504e872464ec25"}, + {file = "zstandard-0.18.0-cp39-cp39-win32.whl", hash = "sha256:75479e7c2b3eebf402c59fbe57d21bc400cefa145ca356ee053b0a08908c5784"}, + {file = "zstandard-0.18.0-cp39-cp39-win_amd64.whl", hash = "sha256:d85bfabad444812133a92fc6fbe463e1d07581dba72f041f07a360e63808b23c"}, + {file = "zstandard-0.18.0.tar.gz", hash = "sha256:0ac0357a0d985b4ff31a854744040d7b5754385d1f98f7145c30e02c6865cb6f"}, +] diff --git a/pyproject.toml b/pyproject.toml index 438de24049..e729235eaa 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -60,7 +60,7 @@ pytest = "^7.0.0" pytest-checkdocs = "^2.0.0" pre-commit = "^2.0.0" fastavro = "^1.5.4" -coverage = { version = "^6.0.0", extras = ["toml"] } +coverage = { version = "^6.4.3", extras = ["toml"] } requests-mock = "^1.9.3" [build-system] From ceb799000db5c78f1fa34fc6caa78fe39726b21f Mon Sep 17 00:00:00 2001 From: Fokko Driesprong Date: Sun, 7 Aug 2022 18:53:46 +0200 Subject: [PATCH 156/642] Python: Ensure that the streams use context managers (#5436) --- pyiceberg/io/base.py | 16 ++++++++++++++++ pyiceberg/io/memory.py | 6 ++++++ pyiceberg/serializers.py | 8 ++++---- tests/avro/test_decoder.py | 6 ++++++ 4 files changed, 32 insertions(+), 4 deletions(-) diff --git a/pyiceberg/io/base.py b/pyiceberg/io/base.py index 3098102b99..6169dcdfd5 100644 --- a/pyiceberg/io/base.py +++ b/pyiceberg/io/base.py @@ -51,6 +51,14 @@ def tell(self) -> int: def close(self) -> None: ... + @abstractmethod + def __enter__(self): + ... + + @abstractmethod + def __exit__(self, exc_type, exc_val, exc_tb): + ... + @runtime_checkable class OutputStream(Protocol): # pragma: no cover @@ -68,6 +76,14 @@ def write(self, b: bytes) -> int: def close(self) -> None: ... + @abstractmethod + def __enter__(self): + ... + + @abstractmethod + def __exit__(self, exc_type, exc_val, exc_tb): + ... + class InputFile(ABC): """A base class for InputFile implementations diff --git a/pyiceberg/io/memory.py b/pyiceberg/io/memory.py index 73f0d5125d..28db68c1f0 100644 --- a/pyiceberg/io/memory.py +++ b/pyiceberg/io/memory.py @@ -70,3 +70,9 @@ def tell(self) -> int: def close(self) -> None: del self.buffer self.pos = 0 + + def __enter__(self): + pass + + def __exit__(self, exc_type, exc_val, exc_tb): + self.close() diff --git a/pyiceberg/serializers.py b/pyiceberg/serializers.py index eee0c435ac..d922cf7944 100644 --- a/pyiceberg/serializers.py +++ b/pyiceberg/serializers.py @@ -54,7 +54,8 @@ def table_metadata(input_file: InputFile, encoding: str = "utf-8") -> TableMetad TableMetadata: A table metadata instance """ - return FromByteStream.table_metadata(byte_stream=input_file.open(), encoding=encoding) + with input_file.open() as input_stream: + return FromByteStream.table_metadata(byte_stream=input_stream, encoding=encoding) class ToOutputFile: @@ -70,6 +71,5 @@ def table_metadata( output_file (OutputFile): A custom implementation of the iceberg.io.file.OutputFile abstract base class overwrite (bool): Where to overwrite the file if it already exists. Defaults to `False`. """ - f = output_file.create(overwrite=overwrite) - f.write(metadata.json().encode("utf-8")) - f.close() + with output_file.create(overwrite=overwrite) as output_stream: + output_stream.write(metadata.json().encode("utf-8")) diff --git a/tests/avro/test_decoder.py b/tests/avro/test_decoder.py index 0616d49314..15b3029ec1 100644 --- a/tests/avro/test_decoder.py +++ b/tests/avro/test_decoder.py @@ -119,6 +119,12 @@ def closed(self) -> bool: def close(self) -> None: pass + def __enter__(self): + pass + + def __exit__(self, exc_type, exc_val, exc_tb): + self.close() + def test_read_single_byte_at_the_time(): decoder = BinaryDecoder(OneByteAtATimeInputStream()) From c80a821d09f88eefa0f958c5d1667ebaf9b83b0d Mon Sep 17 00:00:00 2001 From: Samuel Redai <43911210+samredai@users.noreply.github.com> Date: Sun, 7 Aug 2022 16:08:46 -0400 Subject: [PATCH 157/642] Python: Move io package base.py classes to __init__.py (#5456) --- pyiceberg/avro/decoder.py | 2 +- pyiceberg/avro/file.py | 2 +- pyiceberg/io/__init__.py | 200 +++++++++++++++++++++ pyiceberg/io/base.py | 216 ----------------------- pyiceberg/io/memory.py | 2 +- pyiceberg/io/pyarrow.py | 2 +- pyiceberg/manifest.py | 2 +- pyiceberg/serializers.py | 2 +- tests/avro/test_decoder.py | 2 +- tests/avro/test_reader.py | 2 +- tests/conftest.py | 4 +- tests/io/{test_io_base.py => test_io.py} | 2 +- tests/io/test_pyarrow.py | 2 +- tests/utils/test_manifest.py | 2 +- 14 files changed, 213 insertions(+), 229 deletions(-) delete mode 100644 pyiceberg/io/base.py rename tests/io/{test_io_base.py => test_io.py} (99%) diff --git a/pyiceberg/avro/decoder.py b/pyiceberg/avro/decoder.py index 3a05cfeec3..776c8bae31 100644 --- a/pyiceberg/avro/decoder.py +++ b/pyiceberg/avro/decoder.py @@ -19,7 +19,7 @@ from datetime import date, datetime, time from io import SEEK_CUR -from pyiceberg.io.base import InputStream +from pyiceberg.io import InputStream from pyiceberg.utils.datetime import ( days_to_date, micros_to_time, diff --git a/pyiceberg/avro/file.py b/pyiceberg/avro/file.py index 3b518a5184..cce705382f 100644 --- a/pyiceberg/avro/file.py +++ b/pyiceberg/avro/file.py @@ -28,7 +28,7 @@ from pyiceberg.avro.decoder import BinaryDecoder from pyiceberg.avro.reader import AvroStruct, ConstructReader, StructReader from pyiceberg.avro.resolver import resolve -from pyiceberg.io.base import InputFile, InputStream +from pyiceberg.io import InputFile, InputStream from pyiceberg.io.memory import MemoryInputStream from pyiceberg.schema import Schema, visit from pyiceberg.types import ( diff --git a/pyiceberg/io/__init__.py b/pyiceberg/io/__init__.py index 13a83393a9..6169dcdfd5 100644 --- a/pyiceberg/io/__init__.py +++ b/pyiceberg/io/__init__.py @@ -14,3 +14,203 @@ # KIND, either express or implied. See the License for the # specific language governing permissions and limitations # under the License. +"""Base FileIO classes for implementing reading and writing table files + +The FileIO abstraction includes a subset of full filesystem implementations. Specifically, +Iceberg needs to read or write a file at a given location (as a seekable stream), as well +as check if a file exists. An implementation of the FileIO abstract base class is responsible +for returning an InputFile instance, an OutputFile instance, and deleting a file given +its location. +""" +from abc import ABC, abstractmethod +from io import SEEK_SET +from typing import Protocol, Union, runtime_checkable + + +@runtime_checkable +class InputStream(Protocol): + """A protocol for the file-like object returned by InputFile.open(...) + + This outlines the minimally required methods for a seekable input stream returned from an InputFile + implementation's `open(...)` method. These methods are a subset of IOBase/RawIOBase. + """ + + @abstractmethod + def read(self, size: int = 0) -> bytes: + ... + + @abstractmethod + def seek(self, offset: int, whence: int = SEEK_SET) -> int: + ... + + @abstractmethod + def tell(self) -> int: + ... + + @abstractmethod + def close(self) -> None: + ... + + @abstractmethod + def __enter__(self): + ... + + @abstractmethod + def __exit__(self, exc_type, exc_val, exc_tb): + ... + + +@runtime_checkable +class OutputStream(Protocol): # pragma: no cover + """A protocol for the file-like object returned by OutputFile.create(...) + + This outlines the minimally required methods for a writable output stream returned from an OutputFile + implementation's `create(...)` method. These methods are a subset of IOBase/RawIOBase. + """ + + @abstractmethod + def write(self, b: bytes) -> int: + ... + + @abstractmethod + def close(self) -> None: + ... + + @abstractmethod + def __enter__(self): + ... + + @abstractmethod + def __exit__(self, exc_type, exc_val, exc_tb): + ... + + +class InputFile(ABC): + """A base class for InputFile implementations + + Args: + location(str): A URI or a path to a local file + + Attributes: + location(str): The URI or path to a local file for an InputFile instance + exists(bool): Whether the file exists or not + """ + + def __init__(self, location: str): + self._location = location + + @abstractmethod + def __len__(self) -> int: + """Returns the total length of the file, in bytes""" + + @property + def location(self) -> str: + """The fully-qualified location of the input file""" + return self._location + + @abstractmethod + def exists(self) -> bool: + """Checks whether the location exists + + + Raises: + PermissionError: If the file at self.location cannot be accessed due to a permission error + """ + + @abstractmethod + def open(self) -> InputStream: + """This method should return an object that matches the InputStream protocol + + Returns: + InputStream: An object that matches the InputStream protocol + + Raises: + PermissionError: If the file at self.location cannot be accessed due to a permission error + FileNotFoundError: If the file at self.location does not exist + """ + + +class OutputFile(ABC): + """A base class for OutputFile implementations + + Args: + location(str): A URI or a path to a local file + + Attributes: + location(str): The URI or path to a local file for an OutputFile instance + exists(bool): Whether the file exists or not + """ + + def __init__(self, location: str): + self._location = location + + @abstractmethod + def __len__(self) -> int: + """Returns the total length of the file, in bytes""" + + @property + def location(self) -> str: + """The fully-qualified location of the output file""" + return self._location + + @abstractmethod + def exists(self) -> bool: + """Checks whether the location exists + + + Raises: + PermissionError: If the file at self.location cannot be accessed due to a permission error + """ + + @abstractmethod + def to_input_file(self) -> InputFile: + """Returns an InputFile for the location of this output file""" + + @abstractmethod + def create(self, overwrite: bool = False) -> OutputStream: + """This method should return an object that matches the OutputStream protocol. + + Args: + overwrite(bool): If the file already exists at `self.location` + and `overwrite` is False a FileExistsError should be raised + + Returns: + OutputStream: An object that matches the OutputStream protocol + + Raises: + PermissionError: If the file at self.location cannot be accessed due to a permission error + FileExistsError: If the file at self.location already exists and `overwrite=False` + """ + + +class FileIO(ABC): + """A base class for FileIO implementations""" + + @abstractmethod + def new_input(self, location: str) -> InputFile: + """Get an InputFile instance to read bytes from the file at the given location + + Args: + location(str): A URI or a path to a local file + """ + + @abstractmethod + def new_output(self, location: str) -> OutputFile: + """Get an OutputFile instance to write bytes to the file at the given location + + Args: + location(str): A URI or a path to a local file + """ + + @abstractmethod + def delete(self, location: Union[str, InputFile, OutputFile]) -> None: + """Delete the file at the given path + + Args: + location(str, InputFile, OutputFile): A URI or a path to a local file--if an InputFile instance or + an OutputFile instance is provided, the location attribute for that instance is used as the URI to delete + + Raises: + PermissionError: If the file at location cannot be accessed due to a permission error + FileNotFoundError: When the file at the provided location does not exist + """ diff --git a/pyiceberg/io/base.py b/pyiceberg/io/base.py deleted file mode 100644 index 6169dcdfd5..0000000000 --- a/pyiceberg/io/base.py +++ /dev/null @@ -1,216 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. -"""Base FileIO classes for implementing reading and writing table files - -The FileIO abstraction includes a subset of full filesystem implementations. Specifically, -Iceberg needs to read or write a file at a given location (as a seekable stream), as well -as check if a file exists. An implementation of the FileIO abstract base class is responsible -for returning an InputFile instance, an OutputFile instance, and deleting a file given -its location. -""" -from abc import ABC, abstractmethod -from io import SEEK_SET -from typing import Protocol, Union, runtime_checkable - - -@runtime_checkable -class InputStream(Protocol): - """A protocol for the file-like object returned by InputFile.open(...) - - This outlines the minimally required methods for a seekable input stream returned from an InputFile - implementation's `open(...)` method. These methods are a subset of IOBase/RawIOBase. - """ - - @abstractmethod - def read(self, size: int = 0) -> bytes: - ... - - @abstractmethod - def seek(self, offset: int, whence: int = SEEK_SET) -> int: - ... - - @abstractmethod - def tell(self) -> int: - ... - - @abstractmethod - def close(self) -> None: - ... - - @abstractmethod - def __enter__(self): - ... - - @abstractmethod - def __exit__(self, exc_type, exc_val, exc_tb): - ... - - -@runtime_checkable -class OutputStream(Protocol): # pragma: no cover - """A protocol for the file-like object returned by OutputFile.create(...) - - This outlines the minimally required methods for a writable output stream returned from an OutputFile - implementation's `create(...)` method. These methods are a subset of IOBase/RawIOBase. - """ - - @abstractmethod - def write(self, b: bytes) -> int: - ... - - @abstractmethod - def close(self) -> None: - ... - - @abstractmethod - def __enter__(self): - ... - - @abstractmethod - def __exit__(self, exc_type, exc_val, exc_tb): - ... - - -class InputFile(ABC): - """A base class for InputFile implementations - - Args: - location(str): A URI or a path to a local file - - Attributes: - location(str): The URI or path to a local file for an InputFile instance - exists(bool): Whether the file exists or not - """ - - def __init__(self, location: str): - self._location = location - - @abstractmethod - def __len__(self) -> int: - """Returns the total length of the file, in bytes""" - - @property - def location(self) -> str: - """The fully-qualified location of the input file""" - return self._location - - @abstractmethod - def exists(self) -> bool: - """Checks whether the location exists - - - Raises: - PermissionError: If the file at self.location cannot be accessed due to a permission error - """ - - @abstractmethod - def open(self) -> InputStream: - """This method should return an object that matches the InputStream protocol - - Returns: - InputStream: An object that matches the InputStream protocol - - Raises: - PermissionError: If the file at self.location cannot be accessed due to a permission error - FileNotFoundError: If the file at self.location does not exist - """ - - -class OutputFile(ABC): - """A base class for OutputFile implementations - - Args: - location(str): A URI or a path to a local file - - Attributes: - location(str): The URI or path to a local file for an OutputFile instance - exists(bool): Whether the file exists or not - """ - - def __init__(self, location: str): - self._location = location - - @abstractmethod - def __len__(self) -> int: - """Returns the total length of the file, in bytes""" - - @property - def location(self) -> str: - """The fully-qualified location of the output file""" - return self._location - - @abstractmethod - def exists(self) -> bool: - """Checks whether the location exists - - - Raises: - PermissionError: If the file at self.location cannot be accessed due to a permission error - """ - - @abstractmethod - def to_input_file(self) -> InputFile: - """Returns an InputFile for the location of this output file""" - - @abstractmethod - def create(self, overwrite: bool = False) -> OutputStream: - """This method should return an object that matches the OutputStream protocol. - - Args: - overwrite(bool): If the file already exists at `self.location` - and `overwrite` is False a FileExistsError should be raised - - Returns: - OutputStream: An object that matches the OutputStream protocol - - Raises: - PermissionError: If the file at self.location cannot be accessed due to a permission error - FileExistsError: If the file at self.location already exists and `overwrite=False` - """ - - -class FileIO(ABC): - """A base class for FileIO implementations""" - - @abstractmethod - def new_input(self, location: str) -> InputFile: - """Get an InputFile instance to read bytes from the file at the given location - - Args: - location(str): A URI or a path to a local file - """ - - @abstractmethod - def new_output(self, location: str) -> OutputFile: - """Get an OutputFile instance to write bytes to the file at the given location - - Args: - location(str): A URI or a path to a local file - """ - - @abstractmethod - def delete(self, location: Union[str, InputFile, OutputFile]) -> None: - """Delete the file at the given path - - Args: - location(str, InputFile, OutputFile): A URI or a path to a local file--if an InputFile instance or - an OutputFile instance is provided, the location attribute for that instance is used as the URI to delete - - Raises: - PermissionError: If the file at location cannot be accessed due to a permission error - FileNotFoundError: When the file at the provided location does not exist - """ diff --git a/pyiceberg/io/memory.py b/pyiceberg/io/memory.py index 28db68c1f0..1ea17feac1 100644 --- a/pyiceberg/io/memory.py +++ b/pyiceberg/io/memory.py @@ -17,7 +17,7 @@ from io import SEEK_CUR, SEEK_END, SEEK_SET -from pyiceberg.io.base import InputStream +from pyiceberg.io import InputStream class MemoryInputStream(InputStream): diff --git a/pyiceberg/io/pyarrow.py b/pyiceberg/io/pyarrow.py index 8268cdc7f0..c07a1118aa 100644 --- a/pyiceberg/io/pyarrow.py +++ b/pyiceberg/io/pyarrow.py @@ -28,7 +28,7 @@ from pyarrow.fs import FileInfo, FileSystem, FileType -from pyiceberg.io.base import ( +from pyiceberg.io import ( FileIO, InputFile, InputStream, diff --git a/pyiceberg/manifest.py b/pyiceberg/manifest.py index b1349390d9..ffb1c3b910 100644 --- a/pyiceberg/manifest.py +++ b/pyiceberg/manifest.py @@ -29,7 +29,7 @@ from pyiceberg.avro.file import AvroFile from pyiceberg.avro.reader import AvroStruct -from pyiceberg.io.base import InputFile +from pyiceberg.io import InputFile from pyiceberg.schema import Schema from pyiceberg.types import ( IcebergType, diff --git a/pyiceberg/serializers.py b/pyiceberg/serializers.py index d922cf7944..5906cb435d 100644 --- a/pyiceberg/serializers.py +++ b/pyiceberg/serializers.py @@ -19,7 +19,7 @@ import json from typing import Union -from pyiceberg.io.base import InputFile, InputStream, OutputFile +from pyiceberg.io import InputFile, InputStream, OutputFile from pyiceberg.table.metadata import TableMetadata, TableMetadataV1, TableMetadataV2 diff --git a/tests/avro/test_decoder.py b/tests/avro/test_decoder.py index 15b3029ec1..73aba1a1be 100644 --- a/tests/avro/test_decoder.py +++ b/tests/avro/test_decoder.py @@ -22,7 +22,7 @@ from pyiceberg.avro.decoder import BinaryDecoder from pyiceberg.avro.resolver import promote -from pyiceberg.io.base import InputStream +from pyiceberg.io import InputStream from pyiceberg.io.memory import MemoryInputStream from pyiceberg.types import DoubleType, FloatType diff --git a/tests/avro/test_reader.py b/tests/avro/test_reader.py index 9790590fd4..e4b0cb665e 100644 --- a/tests/avro/test_reader.py +++ b/tests/avro/test_reader.py @@ -53,7 +53,7 @@ TimestamptzType, TimeType, ) -from tests.io.test_io_base import LocalInputFile +from tests.io.test_io import LocalInputFile def test_read_header(generated_manifest_entry_file: str, iceberg_manifest_entry_schema: Schema): diff --git a/tests/conftest.py b/tests/conftest.py index 7aa9290390..e2072790f1 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -32,7 +32,7 @@ import pytest from pyiceberg import schema -from pyiceberg.io.base import ( +from pyiceberg.io import ( FileIO, InputFile, OutputFile, @@ -53,7 +53,7 @@ StructType, ) from tests.catalog.test_base import InMemoryCatalog -from tests.io.test_io_base import LocalInputFile +from tests.io.test_io import LocalInputFile class FooStruct: diff --git a/tests/io/test_io_base.py b/tests/io/test_io.py similarity index 99% rename from tests/io/test_io_base.py rename to tests/io/test_io.py index 3165c4099a..c9bc60cd22 100644 --- a/tests/io/test_io_base.py +++ b/tests/io/test_io.py @@ -22,7 +22,7 @@ import pytest -from pyiceberg.io.base import ( +from pyiceberg.io import ( FileIO, InputFile, InputStream, diff --git a/tests/io/test_pyarrow.py b/tests/io/test_pyarrow.py index 67bdee4427..43dcd22a8d 100644 --- a/tests/io/test_pyarrow.py +++ b/tests/io/test_pyarrow.py @@ -23,7 +23,7 @@ import pytest from pyarrow.fs import FileType -from pyiceberg.io.base import InputStream, OutputStream +from pyiceberg.io import InputStream, OutputStream from pyiceberg.io.pyarrow import PyArrowFile, PyArrowFileIO diff --git a/tests/utils/test_manifest.py b/tests/utils/test_manifest.py index 289989bed3..66b70a2dc0 100644 --- a/tests/utils/test_manifest.py +++ b/tests/utils/test_manifest.py @@ -23,7 +23,7 @@ read_manifest_entry, read_manifest_list, ) -from tests.io.test_io_base import LocalInputFile +from tests.io.test_io import LocalInputFile def test_read_manifest_entry(generated_manifest_entry_file: str): From b39daa3df45acf815d87981fdbeedf5154d20d39 Mon Sep 17 00:00:00 2001 From: Samuel Redai <43911210+samredai@users.noreply.github.com> Date: Sun, 7 Aug 2022 16:24:29 -0400 Subject: [PATCH 158/642] Python: Move catalog package base.py classes to __init__.py (#5457) --- pyiceberg/catalog/__init__.py | 247 +++++++++++++++++++++++++++++++- pyiceberg/catalog/base.py | 261 ---------------------------------- pyiceberg/catalog/hive.py | 8 +- pyiceberg/catalog/rest.py | 8 +- pyiceberg/table/base.py | 2 +- pyiceberg/typedef.py | 20 +++ tests/catalog/test_base.py | 8 +- tests/catalog/test_hive.py | 2 +- tests/catalog/test_rest.py | 2 +- 9 files changed, 285 insertions(+), 273 deletions(-) delete mode 100644 pyiceberg/catalog/base.py create mode 100644 pyiceberg/typedef.py diff --git a/pyiceberg/catalog/__init__.py b/pyiceberg/catalog/__init__.py index d78cfc86a8..25063189a9 100644 --- a/pyiceberg/catalog/__init__.py +++ b/pyiceberg/catalog/__init__.py @@ -14,7 +14,248 @@ # KIND, either express or implied. See the License for the # specific language governing permissions and limitations # under the License. -from typing import Dict, Tuple -Identifier = Tuple[str, ...] -Properties = Dict[str, str] +from __future__ import annotations + +from abc import ABC, abstractmethod +from dataclasses import dataclass + +from pyiceberg.schema import Schema +from pyiceberg.table.base import Table +from pyiceberg.table.partitioning import UNPARTITIONED_PARTITION_SPEC, PartitionSpec +from pyiceberg.table.sorting import UNSORTED_SORT_ORDER, SortOrder +from pyiceberg.typedef import Identifier, Properties + + +@dataclass +class PropertiesUpdateSummary: + removed: list[str] + updated: list[str] + missing: list[str] + + +class Catalog(ABC): + """Base Catalog for table operations like - create, drop, load, list and others. + + The catalog table APIs accept a table identifier, which is fully classified table name. The identifier can be a string or + tuple of strings. If the identifier is a string, it is split into a tuple on '.'. If it is a tuple, it is used as-is. + + The catalog namespace APIs follow a similar convention wherein they also accept a namespace identifier that can be a string + or tuple of strings. + + Attributes: + name (str): Name of the catalog + properties (Properties): Catalog properties + """ + + def __init__(self, name: str, properties: Properties): + self._name = name + self._properties = properties + + @property + def name(self) -> str: + return self._name + + @property + def properties(self) -> Properties: + return self._properties + + @abstractmethod + def create_table( + self, + identifier: str | Identifier, + schema: Schema, + location: str | None = None, + partition_spec: PartitionSpec = UNPARTITIONED_PARTITION_SPEC, + sort_order: SortOrder = UNSORTED_SORT_ORDER, + properties: Properties | None = None, + ) -> Table: + """Create a table + + Args: + identifier (str | Identifier): Table identifier. + schema (Schema): Table's schema. + location (str): Location for the table. Optional Argument. + partition_spec (PartitionSpec): PartitionSpec for the table. + sort_order (SortOrder): SortOrder for the table. + properties (Properties | None): Table properties that can be a string based dictionary. Optional Argument. + + Returns: + Table: the created table instance + + Raises: + TableAlreadyExistsError: If a table with the name already exists + """ + + @abstractmethod + def load_table(self, identifier: str | Identifier) -> Table: + """Loads the table's metadata and returns the table instance. + + You can also use this method to check for table existence using 'try catalog.table() except NoSuchTableError' + Note: This method doesn't scan data stored in the table. + + Args: + identifier (str | Identifier): Table identifier. + + Returns: + Table: the table instance with its metadata + + Raises: + NoSuchTableError: If a table with the name does not exist + """ + + @abstractmethod + def drop_table(self, identifier: str | Identifier) -> None: + """Drop a table. + + Args: + identifier (str | Identifier): Table identifier. + + Raises: + NoSuchTableError: If a table with the name does not exist + """ + + @abstractmethod + def purge_table(self, identifier: str | Identifier) -> None: + """Drop a table and purge all data and metadata files. + + Args: + identifier (str | Identifier): Table identifier. + + Raises: + NoSuchTableError: If a table with the name does not exist + """ + + @abstractmethod + def rename_table(self, from_identifier: str | Identifier, to_identifier: str | Identifier) -> Table: + """Rename a fully classified table name + + Args: + from_identifier (str | Identifier): Existing table identifier. + to_identifier (str | Identifier): New table identifier. + + Returns: + Table: the updated table instance with its metadata + + Raises: + NoSuchTableError: If a table with the name does not exist + """ + + @abstractmethod + def create_namespace(self, namespace: str | Identifier, properties: Properties | None = None) -> None: + """Create a namespace in the catalog. + + Args: + namespace (str | Identifier): Namespace identifier + properties (Properties | None): A string dictionary of properties for the given namespace + + Raises: + NamespaceAlreadyExistsError: If a namespace with the given name already exists + """ + + @abstractmethod + def drop_namespace(self, namespace: str | Identifier) -> None: + """Drop a namespace. + + Args: + namespace (str | Identifier): Namespace identifier + + Raises: + NoSuchNamespaceError: If a namespace with the given name does not exist + NamespaceNotEmptyError: If the namespace is not empty + """ + + @abstractmethod + def list_tables(self, namespace: str | Identifier) -> list[Identifier]: + """List tables under the given namespace in the catalog. + + If namespace not provided, will list all tables in the catalog. + + Args: + namespace (str | Identifier | None): Namespace identifier to search. + + Returns: + List[Identifier]: list of table identifiers. + + Raises: + NoSuchNamespaceError: If a namespace with the given name does not exist + """ + + @abstractmethod + def list_namespaces(self) -> list[Identifier]: + """List namespaces from the given namespace. If not given, list top-level namespaces from the catalog. + + Returns: + List[Identifier]: a List of namespace identifiers + + Raises: + NoSuchNamespaceError: If a namespace with the given name does not exist + """ + + @abstractmethod + def load_namespace_properties(self, namespace: str | Identifier) -> Properties: + """Get properties for a namespace. + + Args: + namespace (str | Identifier): Namespace identifier + + Returns: + Properties: Properties for the given namespace + + Raises: + NoSuchNamespaceError: If a namespace with the given name does not exist + """ + + @abstractmethod + def update_namespace_properties( + self, namespace: str | Identifier, removals: set[str] | None = None, updates: Properties | None = None + ) -> PropertiesUpdateSummary: + """Removes provided property keys and updates properties for a namespace. + + Args: + namespace (str | Identifier): Namespace identifier + removals (Set[str]): Set of property keys that need to be removed. Optional Argument. + updates (Properties | None): Properties to be updated for the given namespace. Optional Argument. + + Raises: + NoSuchNamespaceError: If a namespace with the given name does not exist + ValueError: If removals and updates have overlapping keys. + """ + + @staticmethod + def identifier_to_tuple(identifier: str | Identifier) -> Identifier: + """Parses an identifier to a tuple. + + If the identifier is a string, it is split into a tuple on '.'. If it is a tuple, it is used as-is. + + Args: + identifier (str | Identifier: an identifier, either a string or tuple of strings + + Returns: + Identifier: a tuple of strings + """ + return identifier if isinstance(identifier, tuple) else tuple(str.split(identifier, ".")) + + @staticmethod + def table_name_from(identifier: str | Identifier) -> str: + """Extracts table name from a table identifier + + Args: + identifier (str | Identifier: a table identifier + + Returns: + str: Table name + """ + return Catalog.identifier_to_tuple(identifier)[-1] + + @staticmethod + def namespace_from(identifier: str | Identifier) -> Identifier: + """Extracts table namespace from a table identifier + + Args: + identifier (str | Identifier: a table identifier + + Returns: + Identifier: Namespace identifier + """ + return Catalog.identifier_to_tuple(identifier)[:-1] diff --git a/pyiceberg/catalog/base.py b/pyiceberg/catalog/base.py deleted file mode 100644 index 49eca489bb..0000000000 --- a/pyiceberg/catalog/base.py +++ /dev/null @@ -1,261 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -from __future__ import annotations - -from abc import ABC, abstractmethod -from dataclasses import dataclass - -from pyiceberg.catalog import Identifier, Properties -from pyiceberg.schema import Schema -from pyiceberg.table.base import Table -from pyiceberg.table.partitioning import UNPARTITIONED_PARTITION_SPEC, PartitionSpec -from pyiceberg.table.sorting import UNSORTED_SORT_ORDER, SortOrder - - -@dataclass -class PropertiesUpdateSummary: - removed: list[str] - updated: list[str] - missing: list[str] - - -class Catalog(ABC): - """Base Catalog for table operations like - create, drop, load, list and others. - - The catalog table APIs accept a table identifier, which is fully classified table name. The identifier can be a string or - tuple of strings. If the identifier is a string, it is split into a tuple on '.'. If it is a tuple, it is used as-is. - - The catalog namespace APIs follow a similar convention wherein they also accept a namespace identifier that can be a string - or tuple of strings. - - Attributes: - name (str): Name of the catalog - properties (Properties): Catalog properties - """ - - def __init__(self, name: str, properties: Properties): - self._name = name - self._properties = properties - - @property - def name(self) -> str: - return self._name - - @property - def properties(self) -> Properties: - return self._properties - - @abstractmethod - def create_table( - self, - identifier: str | Identifier, - schema: Schema, - location: str | None = None, - partition_spec: PartitionSpec = UNPARTITIONED_PARTITION_SPEC, - sort_order: SortOrder = UNSORTED_SORT_ORDER, - properties: Properties | None = None, - ) -> Table: - """Create a table - - Args: - identifier (str | Identifier): Table identifier. - schema (Schema): Table's schema. - location (str): Location for the table. Optional Argument. - partition_spec (PartitionSpec): PartitionSpec for the table. - sort_order (SortOrder): SortOrder for the table. - properties (Properties | None): Table properties that can be a string based dictionary. Optional Argument. - - Returns: - Table: the created table instance - - Raises: - TableAlreadyExistsError: If a table with the name already exists - """ - - @abstractmethod - def load_table(self, identifier: str | Identifier) -> Table: - """Loads the table's metadata and returns the table instance. - - You can also use this method to check for table existence using 'try catalog.table() except NoSuchTableError' - Note: This method doesn't scan data stored in the table. - - Args: - identifier (str | Identifier): Table identifier. - - Returns: - Table: the table instance with its metadata - - Raises: - NoSuchTableError: If a table with the name does not exist - """ - - @abstractmethod - def drop_table(self, identifier: str | Identifier) -> None: - """Drop a table. - - Args: - identifier (str | Identifier): Table identifier. - - Raises: - NoSuchTableError: If a table with the name does not exist - """ - - @abstractmethod - def purge_table(self, identifier: str | Identifier) -> None: - """Drop a table and purge all data and metadata files. - - Args: - identifier (str | Identifier): Table identifier. - - Raises: - NoSuchTableError: If a table with the name does not exist - """ - - @abstractmethod - def rename_table(self, from_identifier: str | Identifier, to_identifier: str | Identifier) -> Table: - """Rename a fully classified table name - - Args: - from_identifier (str | Identifier): Existing table identifier. - to_identifier (str | Identifier): New table identifier. - - Returns: - Table: the updated table instance with its metadata - - Raises: - NoSuchTableError: If a table with the name does not exist - """ - - @abstractmethod - def create_namespace(self, namespace: str | Identifier, properties: Properties | None = None) -> None: - """Create a namespace in the catalog. - - Args: - namespace (str | Identifier): Namespace identifier - properties (Properties | None): A string dictionary of properties for the given namespace - - Raises: - NamespaceAlreadyExistsError: If a namespace with the given name already exists - """ - - @abstractmethod - def drop_namespace(self, namespace: str | Identifier) -> None: - """Drop a namespace. - - Args: - namespace (str | Identifier): Namespace identifier - - Raises: - NoSuchNamespaceError: If a namespace with the given name does not exist - NamespaceNotEmptyError: If the namespace is not empty - """ - - @abstractmethod - def list_tables(self, namespace: str | Identifier) -> list[Identifier]: - """List tables under the given namespace in the catalog. - - If namespace not provided, will list all tables in the catalog. - - Args: - namespace (str | Identifier | None): Namespace identifier to search. - - Returns: - List[Identifier]: list of table identifiers. - - Raises: - NoSuchNamespaceError: If a namespace with the given name does not exist - """ - - @abstractmethod - def list_namespaces(self) -> list[Identifier]: - """List namespaces from the given namespace. If not given, list top-level namespaces from the catalog. - - Returns: - List[Identifier]: a List of namespace identifiers - - Raises: - NoSuchNamespaceError: If a namespace with the given name does not exist - """ - - @abstractmethod - def load_namespace_properties(self, namespace: str | Identifier) -> Properties: - """Get properties for a namespace. - - Args: - namespace (str | Identifier): Namespace identifier - - Returns: - Properties: Properties for the given namespace - - Raises: - NoSuchNamespaceError: If a namespace with the given name does not exist - """ - - @abstractmethod - def update_namespace_properties( - self, namespace: str | Identifier, removals: set[str] | None = None, updates: Properties | None = None - ) -> PropertiesUpdateSummary: - """Removes provided property keys and updates properties for a namespace. - - Args: - namespace (str | Identifier): Namespace identifier - removals (Set[str]): Set of property keys that need to be removed. Optional Argument. - updates (Properties | None): Properties to be updated for the given namespace. Optional Argument. - - Raises: - NoSuchNamespaceError: If a namespace with the given name does not exist - ValueError: If removals and updates have overlapping keys. - """ - - @staticmethod - def identifier_to_tuple(identifier: str | Identifier) -> Identifier: - """Parses an identifier to a tuple. - - If the identifier is a string, it is split into a tuple on '.'. If it is a tuple, it is used as-is. - - Args: - identifier (str | Identifier: an identifier, either a string or tuple of strings - - Returns: - Identifier: a tuple of strings - """ - return identifier if isinstance(identifier, tuple) else tuple(str.split(identifier, ".")) - - @staticmethod - def table_name_from(identifier: str | Identifier) -> str: - """Extracts table name from a table identifier - - Args: - identifier (str | Identifier: a table identifier - - Returns: - str: Table name - """ - return Catalog.identifier_to_tuple(identifier)[-1] - - @staticmethod - def namespace_from(identifier: str | Identifier) -> Identifier: - """Extracts table namespace from a table identifier - - Args: - identifier (str | Identifier: a table identifier - - Returns: - Identifier: Namespace identifier - """ - return Catalog.identifier_to_tuple(identifier)[:-1] diff --git a/pyiceberg/catalog/hive.py b/pyiceberg/catalog/hive.py index f04fc22338..991f800130 100644 --- a/pyiceberg/catalog/hive.py +++ b/pyiceberg/catalog/hive.py @@ -43,8 +43,12 @@ from thrift.protocol import TBinaryProtocol from thrift.transport import TSocket, TTransport -from pyiceberg.catalog import Identifier, Properties -from pyiceberg.catalog.base import Catalog, PropertiesUpdateSummary +from pyiceberg.catalog import ( + Catalog, + Identifier, + Properties, + PropertiesUpdateSummary, +) from pyiceberg.exceptions import ( NamespaceAlreadyExistsError, NamespaceNotEmptyError, diff --git a/pyiceberg/catalog/rest.py b/pyiceberg/catalog/rest.py index bd6ea36b34..073996931d 100644 --- a/pyiceberg/catalog/rest.py +++ b/pyiceberg/catalog/rest.py @@ -30,8 +30,12 @@ from requests import HTTPError from pyiceberg import __version__ -from pyiceberg.catalog import Identifier, Properties -from pyiceberg.catalog.base import Catalog, PropertiesUpdateSummary +from pyiceberg.catalog import ( + Catalog, + Identifier, + Properties, + PropertiesUpdateSummary, +) from pyiceberg.exceptions import ( AuthorizationExpiredError, BadCredentialsError, diff --git a/pyiceberg/table/base.py b/pyiceberg/table/base.py index 5ade9d7251..ad31e75cdd 100644 --- a/pyiceberg/table/base.py +++ b/pyiceberg/table/base.py @@ -19,8 +19,8 @@ from pydantic import Field -from pyiceberg.catalog.base import Identifier from pyiceberg.table.metadata import TableMetadataV1, TableMetadataV2 +from pyiceberg.typedef import Identifier from pyiceberg.utils.iceberg_base_model import IcebergBaseModel diff --git a/pyiceberg/typedef.py b/pyiceberg/typedef.py new file mode 100644 index 0000000000..d78cfc86a8 --- /dev/null +++ b/pyiceberg/typedef.py @@ -0,0 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +from typing import Dict, Tuple + +Identifier = Tuple[str, ...] +Properties = Dict[str, str] diff --git a/tests/catalog/test_base.py b/tests/catalog/test_base.py index 3ca41da083..7b29aeb5da 100644 --- a/tests/catalog/test_base.py +++ b/tests/catalog/test_base.py @@ -25,8 +25,12 @@ import pytest -from pyiceberg.catalog import Identifier, Properties -from pyiceberg.catalog.base import Catalog, PropertiesUpdateSummary +from pyiceberg.catalog import ( + Catalog, + Identifier, + Properties, + PropertiesUpdateSummary, +) from pyiceberg.exceptions import ( NamespaceAlreadyExistsError, NamespaceNotEmptyError, diff --git a/tests/catalog/test_hive.py b/tests/catalog/test_hive.py index 77e0de02e7..63443ee720 100644 --- a/tests/catalog/test_hive.py +++ b/tests/catalog/test_hive.py @@ -31,7 +31,7 @@ ) from hive_metastore.ttypes import Table as HiveTable -from pyiceberg.catalog.base import PropertiesUpdateSummary +from pyiceberg.catalog import PropertiesUpdateSummary from pyiceberg.catalog.hive import HiveCatalog from pyiceberg.exceptions import ( NamespaceAlreadyExistsError, diff --git a/tests/catalog/test_rest.py b/tests/catalog/test_rest.py index 11b1885d54..5829bd1448 100644 --- a/tests/catalog/test_rest.py +++ b/tests/catalog/test_rest.py @@ -20,7 +20,7 @@ import pytest from requests_mock import Mocker -from pyiceberg.catalog.base import PropertiesUpdateSummary, Table +from pyiceberg.catalog import PropertiesUpdateSummary, Table from pyiceberg.catalog.rest import RestCatalog from pyiceberg.exceptions import ( BadCredentialsError, From 7a6191945f1353087947f9781a4f2cdd3bdc3588 Mon Sep 17 00:00:00 2001 From: Samuel Redai <43911210+samredai@users.noreply.github.com> Date: Sun, 7 Aug 2022 17:54:09 -0400 Subject: [PATCH 159/642] Python: Move table module base.py classes to __init__.py (#5458) --- pyiceberg/catalog/__init__.py | 2 +- pyiceberg/catalog/hive.py | 2 +- pyiceberg/catalog/rest.py | 2 +- pyiceberg/table/__init__.py | 14 ++++++++++++++ pyiceberg/table/base.py | 30 ------------------------------ tests/catalog/test_base.py | 2 +- 6 files changed, 18 insertions(+), 34 deletions(-) delete mode 100644 pyiceberg/table/base.py diff --git a/pyiceberg/catalog/__init__.py b/pyiceberg/catalog/__init__.py index 25063189a9..8c40d886bc 100644 --- a/pyiceberg/catalog/__init__.py +++ b/pyiceberg/catalog/__init__.py @@ -21,7 +21,7 @@ from dataclasses import dataclass from pyiceberg.schema import Schema -from pyiceberg.table.base import Table +from pyiceberg.table import Table from pyiceberg.table.partitioning import UNPARTITIONED_PARTITION_SPEC, PartitionSpec from pyiceberg.table.sorting import UNSORTED_SORT_ORDER, SortOrder from pyiceberg.typedef import Identifier, Properties diff --git a/pyiceberg/catalog/hive.py b/pyiceberg/catalog/hive.py index 991f800130..a34b69e970 100644 --- a/pyiceberg/catalog/hive.py +++ b/pyiceberg/catalog/hive.py @@ -57,7 +57,7 @@ TableAlreadyExistsError, ) from pyiceberg.schema import Schema -from pyiceberg.table.base import Table +from pyiceberg.table import Table from pyiceberg.table.partitioning import PartitionSpec from pyiceberg.table.sorting import UNSORTED_SORT_ORDER, SortOrder from pyiceberg.types import ( diff --git a/pyiceberg/catalog/rest.py b/pyiceberg/catalog/rest.py index 073996931d..5e11933fff 100644 --- a/pyiceberg/catalog/rest.py +++ b/pyiceberg/catalog/rest.py @@ -52,7 +52,7 @@ UnauthorizedError, ) from pyiceberg.schema import Schema -from pyiceberg.table.base import Table +from pyiceberg.table import Table from pyiceberg.table.metadata import TableMetadataV1, TableMetadataV2 from pyiceberg.table.partitioning import UNPARTITIONED_PARTITION_SPEC, PartitionSpec from pyiceberg.table.sorting import UNSORTED_SORT_ORDER, SortOrder diff --git a/pyiceberg/table/__init__.py b/pyiceberg/table/__init__.py index 13a83393a9..585ef6db04 100644 --- a/pyiceberg/table/__init__.py +++ b/pyiceberg/table/__init__.py @@ -14,3 +14,17 @@ # KIND, either express or implied. See the License for the # specific language governing permissions and limitations # under the License. + +from typing import Optional, Union + +from pydantic import Field + +from pyiceberg.table.metadata import TableMetadataV1, TableMetadataV2 +from pyiceberg.typedef import Identifier +from pyiceberg.utils.iceberg_base_model import IcebergBaseModel + + +class Table(IcebergBaseModel): + identifier: Identifier = Field() + metadata_location: Optional[str] = Field() + metadata: Optional[Union[TableMetadataV1, TableMetadataV2]] = Field() diff --git a/pyiceberg/table/base.py b/pyiceberg/table/base.py deleted file mode 100644 index ad31e75cdd..0000000000 --- a/pyiceberg/table/base.py +++ /dev/null @@ -1,30 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -from typing import Optional, Union - -from pydantic import Field - -from pyiceberg.table.metadata import TableMetadataV1, TableMetadataV2 -from pyiceberg.typedef import Identifier -from pyiceberg.utils.iceberg_base_model import IcebergBaseModel - - -class Table(IcebergBaseModel): - identifier: Identifier = Field() - metadata_location: Optional[str] = Field() - metadata: Optional[Union[TableMetadataV1, TableMetadataV2]] = Field() diff --git a/tests/catalog/test_base.py b/tests/catalog/test_base.py index 7b29aeb5da..9ac86c1f26 100644 --- a/tests/catalog/test_base.py +++ b/tests/catalog/test_base.py @@ -39,7 +39,7 @@ TableAlreadyExistsError, ) from pyiceberg.schema import Schema -from pyiceberg.table.base import Table +from pyiceberg.table import Table from pyiceberg.table.metadata import INITIAL_SPEC_ID from pyiceberg.table.partitioning import UNPARTITIONED_PARTITION_SPEC, PartitionSpec from pyiceberg.table.sorting import UNSORTED_SORT_ORDER, SortOrder From b7d529fe0df7d979014a255f5cbf3ab6f2b49ec3 Mon Sep 17 00:00:00 2001 From: Fokko Driesprong Date: Tue, 9 Aug 2022 00:43:36 +0200 Subject: [PATCH 160/642] Python: Initialize the properties with an empty dict (#5465) * Python: Initialize the properties with a frozendict Instead of having an optional property. I think this is a nicer API and it also removes the awkward `properties or {}` logic. We can't use a {} in the constructor since that will always reference the same object, and therefore is a source of weird bugs. https://florimond.dev/en/posts/2018/08/python-mutable-defaults-are-the-source-of-all-evil/ Next to that, we can also use the frozendict in the future when we want to return immutable data: https://pypi.org/project/frozendict/ The frozendict is sometimes faster than the default impl: https://github.com/Marco-Sulla/python-frozendict#benchmarks Next to that, it is also hashable, which is also a nice property. The package itself doesn't depend on anything, so we pull in very little additional requirements. I've added it into `typedef.py` which will also be introduced in: https://github.com/apache/iceberg/pull/5360 I didn't want to create a new file for it, and also didn't find any places where it could fit in. * Remove package * Implement FrozenDict ourselves --- pyiceberg/catalog/__init__.py | 16 ++++++++-------- pyiceberg/catalog/hive.py | 9 +++++---- pyiceberg/catalog/rest.py | 10 +++++----- pyiceberg/table/metadata.py | 2 +- pyiceberg/typedef.py | 14 +++++++++++++- tests/catalog/test_base.py | 7 ++++--- tests/test_schema.py | 7 ++++--- tests/test_typedef.py | 31 +++++++++++++++++++++++++++++++ 8 files changed, 71 insertions(+), 25 deletions(-) create mode 100644 tests/test_typedef.py diff --git a/pyiceberg/catalog/__init__.py b/pyiceberg/catalog/__init__.py index 8c40d886bc..3282c1bbfb 100644 --- a/pyiceberg/catalog/__init__.py +++ b/pyiceberg/catalog/__init__.py @@ -24,7 +24,7 @@ from pyiceberg.table import Table from pyiceberg.table.partitioning import UNPARTITIONED_PARTITION_SPEC, PartitionSpec from pyiceberg.table.sorting import UNSORTED_SORT_ORDER, SortOrder -from pyiceberg.typedef import Identifier, Properties +from pyiceberg.typedef import EMPTY_DICT, Identifier, Properties @dataclass @@ -68,7 +68,7 @@ def create_table( location: str | None = None, partition_spec: PartitionSpec = UNPARTITIONED_PARTITION_SPEC, sort_order: SortOrder = UNSORTED_SORT_ORDER, - properties: Properties | None = None, + properties: Properties = EMPTY_DICT, ) -> Table: """Create a table @@ -78,7 +78,7 @@ def create_table( location (str): Location for the table. Optional Argument. partition_spec (PartitionSpec): PartitionSpec for the table. sort_order (SortOrder): SortOrder for the table. - properties (Properties | None): Table properties that can be a string based dictionary. Optional Argument. + properties (Properties): Table properties that can be a string based dictionary. Optional Argument. Returns: Table: the created table instance @@ -142,12 +142,12 @@ def rename_table(self, from_identifier: str | Identifier, to_identifier: str | I """ @abstractmethod - def create_namespace(self, namespace: str | Identifier, properties: Properties | None = None) -> None: + def create_namespace(self, namespace: str | Identifier, properties: Properties = EMPTY_DICT) -> None: """Create a namespace in the catalog. Args: namespace (str | Identifier): Namespace identifier - properties (Properties | None): A string dictionary of properties for the given namespace + properties (Properties): A string dictionary of properties for the given namespace Raises: NamespaceAlreadyExistsError: If a namespace with the given name already exists @@ -172,7 +172,7 @@ def list_tables(self, namespace: str | Identifier) -> list[Identifier]: If namespace not provided, will list all tables in the catalog. Args: - namespace (str | Identifier | None): Namespace identifier to search. + namespace (str | Identifier): Namespace identifier to search. Returns: List[Identifier]: list of table identifiers. @@ -208,14 +208,14 @@ def load_namespace_properties(self, namespace: str | Identifier) -> Properties: @abstractmethod def update_namespace_properties( - self, namespace: str | Identifier, removals: set[str] | None = None, updates: Properties | None = None + self, namespace: str | Identifier, removals: set[str] | None = None, updates: Properties = EMPTY_DICT ) -> PropertiesUpdateSummary: """Removes provided property keys and updates properties for a namespace. Args: namespace (str | Identifier): Namespace identifier removals (Set[str]): Set of property keys that need to be removed. Optional Argument. - updates (Properties | None): Properties to be updated for the given namespace. Optional Argument. + updates (Properties): Properties to be updated for the given namespace. Optional Argument. Raises: NoSuchNamespaceError: If a namespace with the given name does not exist diff --git a/pyiceberg/catalog/hive.py b/pyiceberg/catalog/hive.py index a34b69e970..737938208b 100644 --- a/pyiceberg/catalog/hive.py +++ b/pyiceberg/catalog/hive.py @@ -60,6 +60,7 @@ from pyiceberg.table import Table from pyiceberg.table.partitioning import PartitionSpec from pyiceberg.table.sorting import UNSORTED_SORT_ORDER, SortOrder +from pyiceberg.typedef import EMPTY_DICT from pyiceberg.types import ( BinaryType, BooleanType, @@ -213,7 +214,7 @@ def create_table( location: Optional[str] = None, partition_spec: Optional[PartitionSpec] = None, sort_order: SortOrder = UNSORTED_SORT_ORDER, - properties: Optional[Properties] = None, + properties: Properties = EMPTY_DICT, ) -> Table: """Create a table @@ -326,7 +327,7 @@ def rename_table(self, from_identifier: Union[str, Identifier], to_identifier: U raise NoSuchNamespaceError(f"Database does not exists: {to_database_name}") from e return Table() - def create_namespace(self, namespace: Union[str, Identifier], properties: Optional[Properties] = None) -> None: + def create_namespace(self, namespace: Union[str, Identifier], properties: Properties = EMPTY_DICT) -> None: """Create a namespace in the catalog. Args: @@ -342,7 +343,7 @@ def create_namespace(self, namespace: Union[str, Identifier], properties: Option try: with self._client as open_client: - open_client.create_database(_annotate_namespace(hive_database, properties or {})) + open_client.create_database(_annotate_namespace(hive_database, properties)) except AlreadyExistsException as e: raise NamespaceAlreadyExistsError(f"Database {database_name} already exists") from e @@ -417,7 +418,7 @@ def load_namespace_properties(self, namespace: Union[str, Identifier]) -> Proper raise NoSuchNamespaceError(f"Database does not exists: {database_name}") from e def update_namespace_properties( - self, namespace: Union[str, Identifier], removals: Optional[Set[str]] = None, updates: Optional[Properties] = None + self, namespace: Union[str, Identifier], removals: Optional[Set[str]] = None, updates: Properties = EMPTY_DICT ) -> PropertiesUpdateSummary: """Removes provided property keys and updates properties for a namespace. diff --git a/pyiceberg/catalog/rest.py b/pyiceberg/catalog/rest.py index 5e11933fff..d8055a3fd9 100644 --- a/pyiceberg/catalog/rest.py +++ b/pyiceberg/catalog/rest.py @@ -56,6 +56,7 @@ from pyiceberg.table.metadata import TableMetadataV1, TableMetadataV2 from pyiceberg.table.partitioning import UNPARTITIONED_PARTITION_SPEC, PartitionSpec from pyiceberg.table.sorting import UNSORTED_SORT_ORDER, SortOrder +from pyiceberg.typedef import EMPTY_DICT from pyiceberg.utils.iceberg_base_model import IcebergBaseModel @@ -303,10 +304,9 @@ def create_table( location: Optional[str] = None, partition_spec: PartitionSpec = UNPARTITIONED_PARTITION_SPEC, sort_order: SortOrder = UNSORTED_SORT_ORDER, - properties: Optional[Properties] = None, + properties: Properties = EMPTY_DICT, ) -> Table: namespace_and_table = self._split_identifier_for_path(identifier) - properties = properties or {} request = CreateTableRequest( name=namespace_and_table["table"], location=location, @@ -387,8 +387,8 @@ def rename_table(self, from_identifier: Union[str, Identifier], to_identifier: U except HTTPError as exc: self._handle_non_200_response(exc, {404: NoSuchTableError, 409: TableAlreadyExistsError}) - def create_namespace(self, namespace: Union[str, Identifier], properties: Optional[Properties] = None) -> None: - payload = {"namespace": self.identifier_to_tuple(namespace), "properties": properties or {}} + def create_namespace(self, namespace: Union[str, Identifier], properties: Properties = EMPTY_DICT) -> None: + payload = {"namespace": self.identifier_to_tuple(namespace), "properties": properties} response = requests.post(self.url(Endpoints.create_namespace), json=payload, headers=self.headers) try: response.raise_for_status() @@ -424,7 +424,7 @@ def load_namespace_properties(self, namespace: Union[str, Identifier]) -> Proper return NamespaceResponse(**response.json()).properties def update_namespace_properties( - self, namespace: Union[str, Identifier], removals: Optional[Set[str]] = None, updates: Optional[Properties] = None + self, namespace: Union[str, Identifier], removals: Optional[Set[str]] = None, updates: Properties = EMPTY_DICT ) -> PropertiesUpdateSummary: namespace = NAMESPACE_SEPARATOR.join(self.identifier_to_tuple(namespace)) payload = {"removals": list(removals or []), "updates": updates} diff --git a/pyiceberg/table/metadata.py b/pyiceberg/table/metadata.py index 9608bf138b..9f48863d4d 100644 --- a/pyiceberg/table/metadata.py +++ b/pyiceberg/table/metadata.py @@ -137,7 +137,7 @@ def construct_refs(cls, data: Dict[str, Any]): are always assigned an unused ID when evolving specs.""" properties: Dict[str, str] = Field(default_factory=dict) - """ A string to string map of table properties. This is used to + """A string to string map of table properties. This is used to control settings that affect reading and writing and is not intended to be used for arbitrary metadata. For example, commit.retry.num-retries is used to control the number of commit retries.""" diff --git a/pyiceberg/typedef.py b/pyiceberg/typedef.py index d78cfc86a8..52e7df9e72 100644 --- a/pyiceberg/typedef.py +++ b/pyiceberg/typedef.py @@ -14,7 +14,19 @@ # KIND, either express or implied. See the License for the # specific language governing permissions and limitations # under the License. -from typing import Dict, Tuple + +from typing import Any, Dict, Tuple + + +class FrozenDict(Dict): + def __setitem__(self, instance, value): + raise AttributeError("FrozenDict does not support assignment") + + def update(self, *args: Any, **kwargs: Any) -> None: # type: ignore + raise AttributeError("FrozenDict does not support .update()") + + +EMPTY_DICT = FrozenDict() Identifier = Tuple[str, ...] Properties = Dict[str, str] diff --git a/tests/catalog/test_base.py b/tests/catalog/test_base.py index 9ac86c1f26..22f0b85f7f 100644 --- a/tests/catalog/test_base.py +++ b/tests/catalog/test_base.py @@ -43,6 +43,7 @@ from pyiceberg.table.metadata import INITIAL_SPEC_ID from pyiceberg.table.partitioning import UNPARTITIONED_PARTITION_SPEC, PartitionSpec from pyiceberg.table.sorting import UNSORTED_SORT_ORDER, SortOrder +from pyiceberg.typedef import EMPTY_DICT from tests.table.test_metadata import EXAMPLE_TABLE_METADATA_V1 @@ -64,7 +65,7 @@ def create_table( location: Optional[str] = None, partition_spec: PartitionSpec = UNPARTITIONED_PARTITION_SPEC, sort_order: SortOrder = UNSORTED_SORT_ORDER, - properties: Optional[Properties] = None, + properties: Properties = EMPTY_DICT, ) -> Table: identifier = Catalog.identifier_to_tuple(identifier) @@ -112,7 +113,7 @@ def rename_table(self, from_identifier: Union[str, Identifier], to_identifier: U self.__tables[to_identifier] = Table(identifier=to_identifier, metadata=table.metadata) return self.__tables[to_identifier] - def create_namespace(self, namespace: Union[str, Identifier], properties: Optional[Properties] = None) -> None: + def create_namespace(self, namespace: Union[str, Identifier], properties: Properties = EMPTY_DICT) -> None: namespace = Catalog.identifier_to_tuple(namespace) if namespace in self.__namespaces: raise NamespaceAlreadyExistsError(f"Namespace already exists: {namespace}") @@ -148,7 +149,7 @@ def load_namespace_properties(self, namespace: Union[str, Identifier]) -> Proper raise NoSuchNamespaceError(f"Namespace does not exist: {namespace}") from error def update_namespace_properties( - self, namespace: Union[str, Identifier], removals: Optional[Set[str]] = None, updates: Optional[Properties] = None + self, namespace: Union[str, Identifier], removals: Optional[Set[str]] = None, updates: Properties = EMPTY_DICT ) -> PropertiesUpdateSummary: removed: Set[str] = set() updated: Set[str] = set() diff --git a/tests/test_schema.py b/tests/test_schema.py index ad094ab934..f99b315c69 100644 --- a/tests/test_schema.py +++ b/tests/test_schema.py @@ -16,7 +16,7 @@ # under the License. from textwrap import dedent -from typing import Any, Dict, Optional +from typing import Any, Dict import pytest @@ -24,6 +24,7 @@ from pyiceberg.expressions.base import Accessor from pyiceberg.files import StructProtocol from pyiceberg.schema import Schema, build_position_accessors +from pyiceberg.typedef import EMPTY_DICT from pyiceberg.types import ( BooleanType, FloatType, @@ -388,8 +389,8 @@ def test_build_position_accessors(table_schema_nested): def test_build_position_accessors_with_struct(table_schema_nested: Schema): class TestStruct(StructProtocol): - def __init__(self, pos: Optional[Dict[int, Any]] = None): - self._pos: Dict[int, Any] = pos or {} + def __init__(self, pos: Dict[int, Any] = EMPTY_DICT): + self._pos: Dict[int, Any] = pos def set(self, pos: int, value) -> None: pass diff --git a/tests/test_typedef.py b/tests/test_typedef.py new file mode 100644 index 0000000000..24a98dfeb8 --- /dev/null +++ b/tests/test_typedef.py @@ -0,0 +1,31 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +import pytest + +from pyiceberg.typedef import FrozenDict + + +def test_setitem_frozendict(): + d = FrozenDict(foo=1, bar=2) + with pytest.raises(AttributeError): + d["foo"] = 3 + + +def test_update_frozendict(): + d = FrozenDict(foo=1, bar=2) + with pytest.raises(AttributeError): + d.update({"yes": 2}) From 5a7280abcc39b8255cd3323245e16cfb4d8929d8 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 9 Aug 2022 09:16:07 -0700 Subject: [PATCH 161/642] Python: Bump pyarrow from 8.0.0 to 9.0.0 in /python (#5455) Bumps [pyarrow](https://github.com/apache/arrow) from 8.0.0 to 9.0.0. - [Release notes](https://github.com/apache/arrow/releases) - [Commits](https://github.com/apache/arrow/compare/go/v8.0.0...go/v9.0.0) Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- poetry.lock | 60 +++++++++++++++++++++++--------------------------- pyproject.toml | 2 +- 2 files changed, 29 insertions(+), 33 deletions(-) diff --git a/poetry.lock b/poetry.lock index 4b64cf6d16..65eb536563 100644 --- a/poetry.lock +++ b/poetry.lock @@ -261,7 +261,7 @@ python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*" [[package]] name = "pyarrow" -version = "8.0.0" +version = "9.0.0" description = "Python library for Apache Arrow" category = "main" optional = true @@ -505,7 +505,7 @@ snappy = ["python-snappy"] [metadata] lock-version = "1.1" python-versions = "^3.8" -content-hash = "5f3903f6d4a01345a4c7ae00a6e4fa74fe08bd062ab61394976ccd36674c890d" +content-hash = "8278a78ca5397fddd33ce52b4ff7dda2d57c221a1ee2c70a3bcf154dedfc72f5" [metadata.files] atomicwrites = [ @@ -768,36 +768,32 @@ py = [ {file = "py-1.11.0.tar.gz", hash = "sha256:51c75c4126074b472f746a24399ad32f6053d1b34b68d2fa41e558e6f4a98719"}, ] pyarrow = [ - {file = "pyarrow-8.0.0-cp310-cp310-macosx_10_13_universal2.whl", hash = "sha256:d5ef4372559b191cafe7db8932801eee252bfc35e983304e7d60b6954576a071"}, - {file = "pyarrow-8.0.0-cp310-cp310-macosx_10_13_x86_64.whl", hash = "sha256:863be6bad6c53797129610930794a3e797cb7d41c0a30e6794a2ac0e42ce41b8"}, - {file = "pyarrow-8.0.0-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:69b043a3fce064ebd9fbae6abc30e885680296e5bd5e6f7353e6a87966cf2ad7"}, - {file = "pyarrow-8.0.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:51e58778fcb8829fca37fbfaea7f208d5ce7ea89ea133dd13d8ce745278ee6f0"}, - {file = "pyarrow-8.0.0-cp310-cp310-manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:15511ce2f50343f3fd5e9f7c30e4d004da9134e9597e93e9c96c3985928cbe82"}, - {file = "pyarrow-8.0.0-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:ea132067ec712d1b1116a841db1c95861508862b21eddbcafefbce8e4b96b867"}, - {file = "pyarrow-8.0.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:deb400df8f19a90b662babceb6dd12daddda6bb357c216e558b207c0770c7654"}, - {file = "pyarrow-8.0.0-cp310-cp310-win_amd64.whl", hash = "sha256:3bd201af6e01f475f02be88cf1f6ee9856ab98c11d8bbb6f58347c58cd07be00"}, - {file = "pyarrow-8.0.0-cp37-cp37m-macosx_10_13_x86_64.whl", hash = "sha256:78a6ac39cd793582998dac88ab5c1c1dd1e6503df6672f064f33a21937ec1d8d"}, - {file = "pyarrow-8.0.0-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:d6f1e1040413651819074ef5b500835c6c42e6c446532a1ddef8bc5054e8dba5"}, - {file = "pyarrow-8.0.0-cp37-cp37m-manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:98c13b2e28a91b0fbf24b483df54a8d7814c074c2623ecef40dce1fa52f6539b"}, - {file = "pyarrow-8.0.0-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:c9c97c8e288847e091dfbcdf8ce51160e638346f51919a9e74fe038b2e8aee62"}, - {file = "pyarrow-8.0.0-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:edad25522ad509e534400d6ab98cf1872d30c31bc5e947712bfd57def7af15bb"}, - {file = "pyarrow-8.0.0-cp37-cp37m-win_amd64.whl", hash = "sha256:ece333706a94c1221ced8b299042f85fd88b5db802d71be70024433ddf3aecab"}, - {file = "pyarrow-8.0.0-cp38-cp38-macosx_10_13_x86_64.whl", hash = "sha256:95c7822eb37663e073da9892f3499fe28e84f3464711a3e555e0c5463fd53a19"}, - {file = "pyarrow-8.0.0-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:25a5f7c7f36df520b0b7363ba9f51c3070799d4b05d587c60c0adaba57763479"}, - {file = "pyarrow-8.0.0-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:ce64bc1da3109ef5ab9e4c60316945a7239c798098a631358e9ab39f6e5529e9"}, - {file = "pyarrow-8.0.0-cp38-cp38-manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:541e7845ce5f27a861eb5b88ee165d931943347eec17b9ff1e308663531c9647"}, - {file = "pyarrow-8.0.0-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:8cd86e04a899bef43e25184f4b934584861d787cf7519851a8c031803d45c6d8"}, - {file = "pyarrow-8.0.0-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:ba2b7aa7efb59156b87987a06f5241932914e4d5bbb74a465306b00a6c808849"}, - {file = "pyarrow-8.0.0-cp38-cp38-win_amd64.whl", hash = "sha256:42b7982301a9ccd06e1dd4fabd2e8e5df74b93ce4c6b87b81eb9e2d86dc79871"}, - {file = "pyarrow-8.0.0-cp39-cp39-macosx_10_13_universal2.whl", hash = "sha256:1dd482ccb07c96188947ad94d7536ab696afde23ad172df8e18944ec79f55055"}, - {file = "pyarrow-8.0.0-cp39-cp39-macosx_10_13_x86_64.whl", hash = "sha256:81b87b782a1366279411f7b235deab07c8c016e13f9af9f7c7b0ee564fedcc8f"}, - {file = "pyarrow-8.0.0-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:03a10daad957970e914920b793f6a49416699e791f4c827927fd4e4d892a5d16"}, - {file = "pyarrow-8.0.0-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:65c7f4cc2be195e3db09296d31a654bb6d8786deebcab00f0e2455fd109d7456"}, - {file = "pyarrow-8.0.0-cp39-cp39-manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:3fee786259d986f8c046100ced54d63b0c8c9f7cdb7d1bbe07dc69e0f928141c"}, - {file = "pyarrow-8.0.0-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:6ea2c54e6b5ecd64e8299d2abb40770fe83a718f5ddc3825ddd5cd28e352cce1"}, - {file = "pyarrow-8.0.0-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:8392b9a1e837230090fe916415ed4c3433b2ddb1a798e3f6438303c70fbabcfc"}, - {file = "pyarrow-8.0.0-cp39-cp39-win_amd64.whl", hash = "sha256:cb06cacc19f3b426681f2f6803cc06ff481e7fe5b3a533b406bc5b2138843d4f"}, - {file = "pyarrow-8.0.0.tar.gz", hash = "sha256:4a18a211ed888f1ac0b0ebcb99e2d9a3e913a481120ee9b1fe33d3fedb945d4e"}, + {file = "pyarrow-9.0.0-cp310-cp310-macosx_10_13_universal2.whl", hash = "sha256:767cafb14278165ad539a2918c14c1b73cf20689747c21375c38e3fe62884902"}, + {file = "pyarrow-9.0.0-cp310-cp310-macosx_10_13_x86_64.whl", hash = "sha256:0238998dc692efcb4e41ae74738d7c1234723271ccf520bd8312dca07d49ef8d"}, + {file = "pyarrow-9.0.0-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:55328348b9139c2b47450d512d716c2248fd58e2f04e2fc23a65e18726666d42"}, + {file = "pyarrow-9.0.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:fc856628acd8d281652c15b6268ec7f27ebcb015abbe99d9baad17f02adc51f1"}, + {file = "pyarrow-9.0.0-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:29eb3e086e2b26202f3a4678316b93cfb15d0e2ba20f3ec12db8fd9cc07cde63"}, + {file = "pyarrow-9.0.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:2e753f8fcf07d8e3a0efa0c8bd51fef5c90281ffd4c5637c08ce42cd0ac297de"}, + {file = "pyarrow-9.0.0-cp310-cp310-win_amd64.whl", hash = "sha256:3eef8a981f45d89de403e81fb83b8119c20824caddf1404274e41a5d66c73806"}, + {file = "pyarrow-9.0.0-cp37-cp37m-macosx_10_13_x86_64.whl", hash = "sha256:7fa56cbd415cef912677270b8e41baad70cde04c6d8a8336eeb2aba85aa93706"}, + {file = "pyarrow-9.0.0-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:f8c46bde1030d704e2796182286d1c56846552c50a39ad5bf5a20c0d8159fc35"}, + {file = "pyarrow-9.0.0-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:8ad430cee28ebc4d6661fc7315747c7a18ae2a74e67498dcb039e1c762a2fb67"}, + {file = "pyarrow-9.0.0-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:81a60bb291a964f63b2717fb1b28f6615ffab7e8585322bfb8a6738e6b321282"}, + {file = "pyarrow-9.0.0-cp37-cp37m-win_amd64.whl", hash = "sha256:9cef618159567d5f62040f2b79b1c7b38e3885f4ffad0ec97cd2d86f88b67cef"}, + {file = "pyarrow-9.0.0-cp38-cp38-macosx_10_13_x86_64.whl", hash = "sha256:5526a3bfb404ff6d31d62ea582cf2466c7378a474a99ee04d1a9b05de5264541"}, + {file = "pyarrow-9.0.0-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:da3e0f319509a5881867effd7024099fb06950a0768dad0d6873668bb88cfaba"}, + {file = "pyarrow-9.0.0-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:2c715eca2092273dcccf6f08437371e04d112f9354245ba2fbe6c801879450b7"}, + {file = "pyarrow-9.0.0-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:f11a645a41ee531c3a5edda45dea07c42267f52571f818d388971d33fc7e2d4a"}, + {file = "pyarrow-9.0.0-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:a5b390bdcfb8c5b900ef543f911cdfec63e88524fafbcc15f83767202a4a2491"}, + {file = "pyarrow-9.0.0-cp38-cp38-win_amd64.whl", hash = "sha256:d9eb04db626fa24fdfb83c00f76679ca0d98728cdbaa0481b6402bf793a290c0"}, + {file = "pyarrow-9.0.0-cp39-cp39-macosx_10_13_universal2.whl", hash = "sha256:4eebdab05afa23d5d5274b24c1cbeb1ba017d67c280f7d39fd8a8f18cbad2ec9"}, + {file = "pyarrow-9.0.0-cp39-cp39-macosx_10_13_x86_64.whl", hash = "sha256:02b820ecd1da02012092c180447de449fc688d0c3f9ff8526ca301cdd60dacd0"}, + {file = "pyarrow-9.0.0-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:92f3977e901db1ef5cba30d6cc1d7942b8d94b910c60f89013e8f7bb86a86eef"}, + {file = "pyarrow-9.0.0-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:f241bd488c2705df930eedfe304ada71191dcf67d6b98ceda0cc934fd2a8388e"}, + {file = "pyarrow-9.0.0-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:1c5a073a930c632058461547e0bc572da1e724b17b6b9eb31a97da13f50cb6e0"}, + {file = "pyarrow-9.0.0-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:f59bcd5217a3ae1e17870792f82b2ff92df9f3862996e2c78e156c13e56ff62e"}, + {file = "pyarrow-9.0.0-cp39-cp39-win_amd64.whl", hash = "sha256:fe2ce795fa1d95e4e940fe5661c3c58aee7181c730f65ac5dd8794a77228de59"}, + {file = "pyarrow-9.0.0.tar.gz", hash = "sha256:7fb02bebc13ab55573d1ae9bb5002a6d20ba767bf8569b52fce5301d42495ab7"}, ] pycparser = [ {file = "pycparser-2.21-py2.py3-none-any.whl", hash = "sha256:8ee45429555515e1f6b185e78100aea234072576aa43ab53aefcae078162fca9"}, diff --git a/pyproject.toml b/pyproject.toml index e729235eaa..eb81644983 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -47,7 +47,7 @@ requests = "^2.28.1" pydantic = "^1.9.1" -pyarrow = { version = "^8.0.0", optional = true } +pyarrow = { version = "^9.0.0", optional = true } zstandard = { version = "^0.18.0", optional = true } From 806bf70cb58b3dd37b4a64a9ba777d06114100da Mon Sep 17 00:00:00 2001 From: Fokko Driesprong Date: Wed, 10 Aug 2022 18:35:37 +0200 Subject: [PATCH 162/642] Python: Validate identifiers in REST catalog (#5485) --- pyiceberg/catalog/rest.py | 41 +++++++++++++++++++------ tests/catalog/test_rest.py | 62 +++++++++++++++++++++++++++++++++++++- 2 files changed, 92 insertions(+), 11 deletions(-) diff --git a/pyiceberg/catalog/rest.py b/pyiceberg/catalog/rest.py index d8055a3fd9..46057158ad 100644 --- a/pyiceberg/catalog/rest.py +++ b/pyiceberg/catalog/rest.py @@ -195,6 +195,13 @@ def __init__( self.config = self._fetch_config(properties) super().__init__(name, properties) + def _check_valid_namespace_identifier(self, identifier: Union[str, Identifier]) -> Identifier: + """The identifier should have at least one element""" + identifier_tuple = Catalog.identifier_to_tuple(identifier) + if len(identifier_tuple) < 1: + raise NoSuchNamespaceError(f"Empty namespace identifier: {identifier}") + return identifier_tuple + @property def headers(self) -> Properties: headers = { @@ -248,12 +255,16 @@ def _fetch_config(self, properties: Properties) -> Properties: return config def _split_identifier_for_path(self, identifier: Union[str, Identifier]) -> Properties: - identifier = self.identifier_to_tuple(identifier) - return {"namespace": NAMESPACE_SEPARATOR.join(identifier[:-1]), "table": identifier[-1]} + identifier_tuple = self.identifier_to_tuple(identifier) + if len(identifier_tuple) <= 1: + raise NoSuchTableError(f"Missing namespace or invalid identifier: {'.'.join(identifier_tuple)}") + return {"namespace": NAMESPACE_SEPARATOR.join(identifier_tuple[:-1]), "table": identifier_tuple[-1]} def _split_identifier_for_json(self, identifier: Union[str, Identifier]) -> Dict[str, Union[Identifier, str]]: - identifier = self.identifier_to_tuple(identifier) - return {"namespace": identifier[:-1], "name": identifier[-1]} + identifier_tuple = self.identifier_to_tuple(identifier) + if len(identifier_tuple) <= 1: + raise NoSuchTableError(f"Missing namespace or invalid identifier: {identifier_tuple}") + return {"namespace": identifier_tuple[:-1], "name": identifier_tuple[-1]} def _handle_non_200_response(self, exc: HTTPError, error_handler: Dict[int, Type[Exception]]): exception: Type[Exception] @@ -335,7 +346,8 @@ def create_table( ) def list_tables(self, namespace: Union[str, Identifier]) -> List[Identifier]: - namespace_concat = NAMESPACE_SEPARATOR.join(self.identifier_to_tuple(namespace)) + namespace_tuple = self._check_valid_namespace_identifier(namespace) + namespace_concat = NAMESPACE_SEPARATOR.join(namespace_tuple) response = requests.get( self.url(Endpoints.list_tables, namespace=namespace_concat), headers=self.headers, @@ -347,6 +359,11 @@ def list_tables(self, namespace: Union[str, Identifier]) -> List[Identifier]: return [(*table.namespace, table.name) for table in ListTablesResponse(**response.json()).identifiers] def load_table(self, identifier: Union[str, Identifier]) -> Table: + identifier_tuple = self.identifier_to_tuple(identifier) + + if len(identifier_tuple) <= 1: + raise NoSuchTableError(f"Missing namespace or invalid identifier: {identifier}") + response = requests.get( self.url(Endpoints.load_table, prefixed=True, **self._split_identifier_for_path(identifier)), headers=self.headers ) @@ -358,7 +375,7 @@ def load_table(self, identifier: Union[str, Identifier]) -> Table: table_response = TableResponse(**response.json()) return Table( - identifier=(self.name,) + self.identifier_to_tuple(identifier), + identifier=(self.name,) + identifier_tuple, metadata_location=table_response.metadata_location, metadata=table_response.metadata, ) @@ -388,7 +405,8 @@ def rename_table(self, from_identifier: Union[str, Identifier], to_identifier: U self._handle_non_200_response(exc, {404: NoSuchTableError, 409: TableAlreadyExistsError}) def create_namespace(self, namespace: Union[str, Identifier], properties: Properties = EMPTY_DICT) -> None: - payload = {"namespace": self.identifier_to_tuple(namespace), "properties": properties} + namespace_tuple = self._check_valid_namespace_identifier(namespace) + payload = {"namespace": namespace_tuple, "properties": properties} response = requests.post(self.url(Endpoints.create_namespace), json=payload, headers=self.headers) try: response.raise_for_status() @@ -396,7 +414,8 @@ def create_namespace(self, namespace: Union[str, Identifier], properties: Proper self._handle_non_200_response(exc, {404: NoSuchNamespaceError, 409: NamespaceAlreadyExistsError}) def drop_namespace(self, namespace: Union[str, Identifier]) -> None: - namespace = NAMESPACE_SEPARATOR.join(self.identifier_to_tuple(namespace)) + namespace_tuple = self._check_valid_namespace_identifier(namespace) + namespace = NAMESPACE_SEPARATOR.join(namespace_tuple) response = requests.delete(self.url(Endpoints.drop_namespace, namespace=namespace), headers=self.headers) try: response.raise_for_status() @@ -414,7 +433,8 @@ def list_namespaces(self) -> List[Identifier]: return namespaces.namespaces def load_namespace_properties(self, namespace: Union[str, Identifier]) -> Properties: - namespace = NAMESPACE_SEPARATOR.join(self.identifier_to_tuple(namespace)) + namespace_tuple = self._check_valid_namespace_identifier(namespace) + namespace = NAMESPACE_SEPARATOR.join(namespace_tuple) response = requests.get(self.url(Endpoints.load_namespace_metadata, namespace=namespace), headers=self.headers) try: response.raise_for_status() @@ -426,7 +446,8 @@ def load_namespace_properties(self, namespace: Union[str, Identifier]) -> Proper def update_namespace_properties( self, namespace: Union[str, Identifier], removals: Optional[Set[str]] = None, updates: Properties = EMPTY_DICT ) -> PropertiesUpdateSummary: - namespace = NAMESPACE_SEPARATOR.join(self.identifier_to_tuple(namespace)) + namespace_tuple = self._check_valid_namespace_identifier(namespace) + namespace = NAMESPACE_SEPARATOR.join(namespace_tuple) payload = {"removals": list(removals or []), "updates": updates} response = requests.post(self.url(Endpoints.update_properties, namespace=namespace), json=payload, headers=self.headers) try: diff --git a/tests/catalog/test_rest.py b/tests/catalog/test_rest.py index 5829bd1448..121428b2b2 100644 --- a/tests/catalog/test_rest.py +++ b/tests/catalog/test_rest.py @@ -14,7 +14,7 @@ # KIND, either express or implied. See the License for the # specific language governing permissions and limitations # under the License. -# pylint: disable=redefined-outer-name +# pylint: disable=redefined-outer-name,unused-argument from uuid import UUID import pytest @@ -622,3 +622,63 @@ def test_delete_table_404(rest_mock: Mocker): with pytest.raises(NoSuchTableError) as e: RestCatalog("rest", {}, TEST_URI, token=TEST_TOKEN).drop_table(("example", "fokko")) assert "Table does not exist" in str(e.value) + + +def test_create_table_missing_namespace(rest_mock: Mocker, table_schema_simple: Schema): + table = "table" + with pytest.raises(NoSuchTableError) as e: + # Missing namespace + RestCatalog("rest", {}, TEST_URI, token=TEST_TOKEN).create_table(table, table_schema_simple) + assert f"Missing namespace or invalid identifier: {table}" in str(e.value) + + +def test_load_table_invalid_namespace(rest_mock: Mocker): + table = "table" + with pytest.raises(NoSuchTableError) as e: + # Missing namespace + RestCatalog("rest", {}, TEST_URI, token=TEST_TOKEN).load_table(table) + assert f"Missing namespace or invalid identifier: {table}" in str(e.value) + + +def test_drop_table_invalid_namespace(rest_mock: Mocker): + table = "table" + with pytest.raises(NoSuchTableError) as e: + # Missing namespace + RestCatalog("rest", {}, TEST_URI, token=TEST_TOKEN).drop_table(table) + assert f"Missing namespace or invalid identifier: {table}" in str(e.value) + + +def test_purge_table_invalid_namespace(rest_mock: Mocker): + table = "table" + with pytest.raises(NoSuchTableError) as e: + # Missing namespace + RestCatalog("rest", {}, TEST_URI, token=TEST_TOKEN).purge_table(table) + assert f"Missing namespace or invalid identifier: {table}" in str(e.value) + + +def test_create_namespace_invalid_namespace(rest_mock: Mocker): + with pytest.raises(NoSuchNamespaceError) as e: + # Missing namespace + RestCatalog("rest", {}, TEST_URI, token=TEST_TOKEN).create_namespace(()) + assert "Empty namespace identifier" in str(e.value) + + +def test_drop_namespace_invalid_namespace(rest_mock: Mocker): + with pytest.raises(NoSuchNamespaceError) as e: + # Missing namespace + RestCatalog("rest", {}, TEST_URI, token=TEST_TOKEN).drop_namespace(()) + assert "Empty namespace identifier" in str(e.value) + + +def test_load_namespace_properties_invalid_namespace(rest_mock: Mocker): + with pytest.raises(NoSuchNamespaceError) as e: + # Missing namespace + RestCatalog("rest", {}, TEST_URI, token=TEST_TOKEN).load_namespace_properties(()) + assert "Empty namespace identifier" in str(e.value) + + +def test_update_namespace_properties_invalid_namespace(rest_mock: Mocker): + with pytest.raises(NoSuchNamespaceError) as e: + # Missing namespace + RestCatalog("rest", {}, TEST_URI, token=TEST_TOKEN).update_namespace_properties(()) + assert "Empty namespace identifier" in str(e.value) From 233a0a1ce0fa4ab2ae2f143bcb8b38f168f4a986 Mon Sep 17 00:00:00 2001 From: Fokko Driesprong Date: Wed, 10 Aug 2022 20:10:14 +0200 Subject: [PATCH 163/642] Python: Fix docstring (#5489) Follow up of https://github.com/apache/iceberg/pull/5465 --- pyiceberg/catalog/__init__.py | 6 +++--- pyiceberg/catalog/hive.py | 4 ++-- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/pyiceberg/catalog/__init__.py b/pyiceberg/catalog/__init__.py index 3282c1bbfb..53c8b0b3fa 100644 --- a/pyiceberg/catalog/__init__.py +++ b/pyiceberg/catalog/__init__.py @@ -75,10 +75,10 @@ def create_table( Args: identifier (str | Identifier): Table identifier. schema (Schema): Table's schema. - location (str): Location for the table. Optional Argument. + location (str | None): Location for the table. Optional Argument. partition_spec (PartitionSpec): PartitionSpec for the table. sort_order (SortOrder): SortOrder for the table. - properties (Properties): Table properties that can be a string based dictionary. Optional Argument. + properties (Properties): Table properties that can be a string based dictionary. Returns: Table: the created table instance @@ -215,7 +215,7 @@ def update_namespace_properties( Args: namespace (str | Identifier): Namespace identifier removals (Set[str]): Set of property keys that need to be removed. Optional Argument. - updates (Properties): Properties to be updated for the given namespace. Optional Argument. + updates (Properties): Properties to be updated for the given namespace. Raises: NoSuchNamespaceError: If a namespace with the given name does not exist diff --git a/pyiceberg/catalog/hive.py b/pyiceberg/catalog/hive.py index 737938208b..f8ad209fbd 100644 --- a/pyiceberg/catalog/hive.py +++ b/pyiceberg/catalog/hive.py @@ -224,7 +224,7 @@ def create_table( location: Location for the table. Optional Argument. partition_spec: PartitionSpec for the table. sort_order: SortOrder for the table. - properties: Table properties that can be a string based dictionary. Optional Argument. + properties: Table properties that can be a string based dictionary. Returns: Table: the created table instance @@ -425,7 +425,7 @@ def update_namespace_properties( Args: namespace: Namespace identifier removals: Set of property keys that need to be removed. Optional Argument. - updates: Properties to be updated for the given namespace. Optional Argument. + updates: Properties to be updated for the given namespace. Raises: NoSuchNamespaceError: If a namespace with the given name does not exist From 071d1a63b0eff212b09ffd9d90ed18d6ffc00abc Mon Sep 17 00:00:00 2001 From: Fokko Driesprong Date: Fri, 12 Aug 2022 17:32:02 +0200 Subject: [PATCH 164/642] Python: Add __str__ implementation to SortOrder and SortField (#5490) --- pyiceberg/table/sorting.py | 28 ++++++++++++++++++++- tests/table/test_sorting.py | 49 ++++++++++++++++++++++++++----------- 2 files changed, 62 insertions(+), 15 deletions(-) diff --git a/pyiceberg/table/sorting.py b/pyiceberg/table/sorting.py index e2a72fd24c..043496823a 100644 --- a/pyiceberg/table/sorting.py +++ b/pyiceberg/table/sorting.py @@ -27,7 +27,7 @@ from pydantic import Field, root_validator -from pyiceberg.transforms import Transform +from pyiceberg.transforms import IdentityTransform, Transform from pyiceberg.types import IcebergType from pyiceberg.utils.iceberg_base_model import IcebergBaseModel @@ -36,11 +36,23 @@ class SortDirection(Enum): ASC = "asc" DESC = "desc" + def __str__(self) -> str: + return self.name + + def __repr__(self) -> str: + return f"SortDirection.{self.name}" + class NullOrder(Enum): NULLS_FIRST = "nulls-first" NULLS_LAST = "nulls-last" + def __str__(self) -> str: + return self.name.replace("_", " ") + + def __repr__(self) -> str: + return f"NullOrder.{self.name}" + class SortField(IcebergBaseModel): """Sort order field @@ -83,6 +95,13 @@ def set_null_order(cls, values: Dict[str, Any]) -> Dict[str, Any]: direction: SortDirection = Field() null_order: NullOrder = Field(alias="null-order") + def __str__(self): + if type(self.transform) == IdentityTransform: + # In the case of an identity transform, we can omit the transform + return f"{self.source_id} {self.direction} {self.null_order}" + else: + return f"{self.transform}({self.source_id}) {self.direction} {self.null_order}" + class SortOrder(IcebergBaseModel): """Describes how the data is sorted within the table @@ -106,6 +125,13 @@ def __init__(self, order_id: Optional[int] = None, *fields: SortField, **data: A order_id: Optional[int] = Field(alias="order-id") fields: List[SortField] = Field(default_factory=list) + def __str__(self) -> str: + result_str = "[" + if self.fields: + result_str += "\n " + "\n ".join([str(field) for field in self.fields]) + "\n" + result_str += "]" + return result_str + UNSORTED_SORT_ORDER_ID = 0 UNSORTED_SORT_ORDER = SortOrder(order_id=UNSORTED_SORT_ORDER_ID) diff --git a/tests/table/test_sorting.py b/tests/table/test_sorting.py index 1cdf4d0c06..300b26f1ce 100644 --- a/tests/table/test_sorting.py +++ b/tests/table/test_sorting.py @@ -14,6 +14,10 @@ # KIND, either express or implied. See the License for the # specific language governing permissions and limitations # under the License. +# pylint:disable=redefined-outer-name,eval-used + +import pytest + from pyiceberg.table.metadata import TableMetadata from pyiceberg.table.sorting import ( UNSORTED_SORT_ORDER, @@ -26,31 +30,29 @@ from tests.table.test_metadata import EXAMPLE_TABLE_METADATA_V2 -def test_serialize_sort_order_unsorted(): - assert UNSORTED_SORT_ORDER.json() == '{"order-id": 0, "fields": []}' - - -def test_serialize_sort_order(): - sort_order = SortOrder( +@pytest.fixture +def sort_order() -> SortOrder: + return SortOrder( 22, SortField(source_id=19, transform=IdentityTransform(), null_order=NullOrder.NULLS_FIRST), SortField(source_id=25, transform=BucketTransform(4), direction=SortDirection.DESC), SortField(source_id=22, transform=VoidTransform(), direction=SortDirection.ASC), ) + + +def test_serialize_sort_order_unsorted(): + assert UNSORTED_SORT_ORDER.json() == '{"order-id": 0, "fields": []}' + + +def test_serialize_sort_order(sort_order: SortOrder): expected = '{"order-id": 22, "fields": [{"source-id": 19, "transform": "identity", "direction": "asc", "null-order": "nulls-first"}, {"source-id": 25, "transform": "bucket[4]", "direction": "desc", "null-order": "nulls-last"}, {"source-id": 22, "transform": "void", "direction": "asc", "null-order": "nulls-first"}]}' assert sort_order.json() == expected -def test_deserialize_sort_order(): - expected = SortOrder( - 22, - SortField(source_id=19, transform=IdentityTransform(), null_order=NullOrder.NULLS_FIRST), - SortField(source_id=25, transform=BucketTransform(4), direction=SortDirection.DESC), - SortField(source_id=22, transform=VoidTransform(), direction=SortDirection.ASC), - ) +def test_deserialize_sort_order(sort_order: SortOrder): payload = '{"order-id": 22, "fields": [{"source-id": 19, "transform": "identity", "direction": "asc", "null-order": "nulls-first"}, {"source-id": 25, "transform": "bucket[4]", "direction": "desc", "null-order": "nulls-last"}, {"source-id": 22, "transform": "void", "direction": "asc", "null-order": "nulls-first"}]}' - assert SortOrder.parse_raw(payload) == expected + assert SortOrder.parse_raw(payload) == sort_order def test_sorting_schema(): @@ -68,3 +70,22 @@ def test_sorting_schema(): ), ) ] + + +def test_sorting_to_string(sort_order: SortOrder): + expected = """[ + 19 ASC NULLS FIRST + bucket[4](25) DESC NULLS LAST + void(22) ASC NULLS FIRST +]""" + assert str(sort_order) == expected + + +def test_sorting_to_repr(sort_order: SortOrder): + expected = """SortOrder(order_id=22, fields=[SortField(source_id=19, transform=IdentityTransform(), direction=SortDirection.ASC, null_order=NullOrder.NULLS_FIRST), SortField(source_id=25, transform=BucketTransform(num_buckets=4), direction=SortDirection.DESC, null_order=NullOrder.NULLS_LAST), SortField(source_id=22, transform=VoidTransform(), direction=SortDirection.ASC, null_order=NullOrder.NULLS_FIRST)])""" + assert repr(sort_order) == expected + + +def test_sorting_repr(sort_order: SortOrder): + """To make sure that the repr converts back to the original object""" + assert sort_order == eval(repr(sort_order)) From 4bd0e5ecec11524f5074c99e41553bc683b7c669 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Sun, 14 Aug 2022 10:12:12 -0700 Subject: [PATCH 165/642] Build: Bump pydantic from 1.9.1 to 1.9.2 in /python (#5522) Bumps [pydantic](https://github.com/samuelcolvin/pydantic) from 1.9.1 to 1.9.2. - [Release notes](https://github.com/samuelcolvin/pydantic/releases) - [Changelog](https://github.com/pydantic/pydantic/blob/master/HISTORY.md) - [Commits](https://github.com/samuelcolvin/pydantic/compare/v1.9.1...v1.9.2) Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- poetry.lock | 120 ++++++++++++++++++++++++------------------------- pyproject.toml | 2 +- 2 files changed, 61 insertions(+), 61 deletions(-) diff --git a/poetry.lock b/poetry.lock index 65eb536563..01370863de 100644 --- a/poetry.lock +++ b/poetry.lock @@ -15,10 +15,10 @@ optional = false python-versions = ">=3.5" [package.extras] -dev = ["coverage[toml] (>=5.0.2)", "hypothesis", "pympler", "pytest (>=4.3.0)", "mypy (>=0.900,!=0.940)", "pytest-mypy-plugins", "zope.interface", "furo", "sphinx", "sphinx-notfound-page", "pre-commit", "cloudpickle"] -docs = ["furo", "sphinx", "zope.interface", "sphinx-notfound-page"] -tests = ["coverage[toml] (>=5.0.2)", "hypothesis", "pympler", "pytest (>=4.3.0)", "mypy (>=0.900,!=0.940)", "pytest-mypy-plugins", "zope.interface", "cloudpickle"] -tests_no_zope = ["coverage[toml] (>=5.0.2)", "hypothesis", "pympler", "pytest (>=4.3.0)", "mypy (>=0.900,!=0.940)", "pytest-mypy-plugins", "cloudpickle"] +tests_no_zope = ["cloudpickle", "pytest-mypy-plugins", "mypy (>=0.900,!=0.940)", "pytest (>=4.3.0)", "pympler", "hypothesis", "coverage[toml] (>=5.0.2)"] +tests = ["cloudpickle", "zope.interface", "pytest-mypy-plugins", "mypy (>=0.900,!=0.940)", "pytest (>=4.3.0)", "pympler", "hypothesis", "coverage[toml] (>=5.0.2)"] +docs = ["sphinx-notfound-page", "zope.interface", "sphinx", "furo"] +dev = ["cloudpickle", "pre-commit", "sphinx-notfound-page", "sphinx", "furo", "zope.interface", "pytest-mypy-plugins", "mypy (>=0.900,!=0.940)", "pytest (>=4.3.0)", "pympler", "hypothesis", "coverage[toml] (>=5.0.2)"] [[package]] name = "certifi" @@ -119,8 +119,8 @@ optional = false python-versions = ">=3.7" [package.extras] -docs = ["furo (>=2021.8.17b43)", "sphinx (>=4.1)", "sphinx-autodoc-typehints (>=1.12)"] -testing = ["covdefaults (>=1.2.0)", "coverage (>=4)", "pytest (>=4)", "pytest-cov", "pytest-timeout (>=1.4.2)"] +testing = ["pytest-timeout (>=1.4.2)", "pytest-cov", "pytest (>=4)", "coverage (>=4)", "covdefaults (>=1.2.0)"] +docs = ["sphinx-autodoc-typehints (>=1.12)", "sphinx (>=4.1)", "furo (>=2021.8.17b43)"] [[package]] name = "identify" @@ -153,9 +153,9 @@ python-versions = ">=3.7" zipp = ">=0.5" [package.extras] -docs = ["sphinx", "jaraco.packaging (>=9)", "rst.linker (>=1.9)"] +testing = ["importlib-resources (>=1.3)", "pytest-mypy (>=0.9.1)", "pytest-black (>=0.3.7)", "pytest-perf (>=0.9.2)", "flufl.flake8", "pyfakefs", "packaging", "pytest-enabler (>=1.3)", "pytest-cov", "pytest-flake8", "pytest-checkdocs (>=2.4)", "pytest (>=6)"] perf = ["ipython"] -testing = ["pytest (>=6)", "pytest-checkdocs (>=2.4)", "pytest-flake8", "pytest-cov", "pytest-enabler (>=1.3)", "packaging", "pyfakefs", "flufl.flake8", "pytest-perf (>=0.9.2)", "pytest-black (>=0.3.7)", "pytest-mypy (>=0.9.1)", "importlib-resources (>=1.3)"] +docs = ["rst.linker (>=1.9)", "jaraco.packaging (>=9)", "sphinx"] [[package]] name = "iniconfig" @@ -220,8 +220,8 @@ optional = false python-versions = ">=3.7" [package.extras] -docs = ["furo (>=2021.7.5b38)", "proselint (>=0.10.2)", "sphinx-autodoc-typehints (>=1.12)", "sphinx (>=4)"] -test = ["appdirs (==1.4.4)", "pytest-cov (>=2.7)", "pytest-mock (>=3.6)", "pytest (>=6)"] +test = ["pytest (>=6)", "pytest-mock (>=3.6)", "pytest-cov (>=2.7)", "appdirs (==1.4.4)"] +docs = ["sphinx (>=4)", "sphinx-autodoc-typehints (>=1.12)", "proselint (>=0.10.2)", "furo (>=2021.7.5b38)"] [[package]] name = "pluggy" @@ -280,7 +280,7 @@ python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*" [[package]] name = "pydantic" -version = "1.9.1" +version = "1.9.2" description = "Data validation and settings management using python type hints" category = "main" optional = false @@ -290,8 +290,8 @@ python-versions = ">=3.6.1" typing-extensions = ">=3.7.4.3" [package.extras] -dotenv = ["python-dotenv (>=0.10.4)"] email = ["email-validator (>=1.0.3)"] +dotenv = ["python-dotenv (>=0.10.4)"] [[package]] name = "pyparsing" @@ -302,7 +302,7 @@ optional = false python-versions = ">=3.6.8" [package.extras] -diagrams = ["railroad-diagrams", "jinja2"] +diagrams = ["jinja2", "railroad-diagrams"] [[package]] name = "pytest" @@ -323,7 +323,7 @@ py = ">=1.8.2" tomli = ">=1.0.0" [package.extras] -testing = ["argcomplete", "hypothesis (>=3.56)", "mock", "nose", "pygments (>=2.7.2)", "requests", "xmlschema"] +testing = ["xmlschema", "requests", "pygments (>=2.7.2)", "nose", "mock", "hypothesis (>=3.56)", "argcomplete"] [[package]] name = "pytest-checkdocs" @@ -339,8 +339,8 @@ importlib-metadata = ">=4" pep517 = "*" [package.extras] -docs = ["sphinx", "jaraco.packaging (>=8.2)", "rst.linker (>=1.9)"] -testing = ["pytest (>=4.6)", "pytest-checkdocs (>=2.4)", "pytest-flake8", "pytest-cov", "pytest-enabler (>=1.0.1)", "types-docutils", "pytest-black (>=0.3.7)", "pytest-mypy"] +testing = ["pytest-mypy", "pytest-black (>=0.3.7)", "types-docutils", "pytest-enabler (>=1.0.1)", "pytest-cov", "pytest-flake8", "pytest-checkdocs (>=2.4)", "pytest (>=4.6)"] +docs = ["rst.linker (>=1.9)", "jaraco.packaging (>=8.2)", "sphinx"] [[package]] name = "python-snappy" @@ -373,8 +373,8 @@ idna = ">=2.5,<4" urllib3 = ">=1.21.1,<1.27" [package.extras] -socks = ["PySocks (>=1.5.6,!=1.5.7)"] use_chardet_on_py3 = ["chardet (>=3.0.2,<6)"] +socks = ["PySocks (>=1.5.6,!=1.5.7)"] [[package]] name = "requests-mock" @@ -389,8 +389,8 @@ requests = ">=2.3,<3" six = "*" [package.extras] +test = ["testtools", "testrepository (>=0.0.18)", "sphinx", "pytest", "purl", "mock", "fixtures"] fixture = ["fixtures"] -test = ["fixtures", "mock", "purl", "pytest", "sphinx", "testrepository (>=0.0.18)", "testtools"] [[package]] name = "six" @@ -449,9 +449,9 @@ optional = false python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*, !=3.5.*, <4" [package.extras] -brotli = ["brotlicffi (>=0.8.0)", "brotli (>=1.0.9)", "brotlipy (>=0.6.0)"] -secure = ["pyOpenSSL (>=0.14)", "cryptography (>=1.3.4)", "idna (>=2.0.0)", "certifi", "ipaddress"] socks = ["PySocks (>=1.5.6,!=1.5.7,<2.0)"] +secure = ["ipaddress", "certifi", "idna (>=2.0.0)", "cryptography (>=1.3.4)", "pyOpenSSL (>=0.14)"] +brotli = ["brotlipy (>=0.6.0)", "brotli (>=1.0.9)", "brotlicffi (>=0.8.0)"] [[package]] name = "virtualenv" @@ -467,8 +467,8 @@ filelock = ">=3.2,<4" platformdirs = ">=2,<3" [package.extras] -docs = ["proselint (>=0.10.2)", "sphinx (>=3)", "sphinx-argparse (>=0.2.5)", "sphinx-rtd-theme (>=0.4.3)", "towncrier (>=21.3)"] -testing = ["coverage (>=4)", "coverage-enable-subprocess (>=1)", "flaky (>=3)", "packaging (>=20.0)", "pytest (>=4)", "pytest-env (>=0.6.2)", "pytest-freezegun (>=0.4.1)", "pytest-mock (>=2)", "pytest-randomly (>=1)", "pytest-timeout (>=1)"] +testing = ["pytest-timeout (>=1)", "pytest-randomly (>=1)", "pytest-mock (>=2)", "pytest-freezegun (>=0.4.1)", "pytest-env (>=0.6.2)", "pytest (>=4)", "packaging (>=20.0)", "flaky (>=3)", "coverage-enable-subprocess (>=1)", "coverage (>=4)"] +docs = ["towncrier (>=21.3)", "sphinx-rtd-theme (>=0.4.3)", "sphinx-argparse (>=0.2.5)", "sphinx (>=3)", "proselint (>=0.10.2)"] [[package]] name = "zipp" @@ -479,8 +479,8 @@ optional = false python-versions = ">=3.7" [package.extras] -docs = ["sphinx", "jaraco.packaging (>=9)", "rst.linker (>=1.9)", "jaraco.tidelift (>=1.4)"] -testing = ["pytest (>=6)", "pytest-checkdocs (>=2.4)", "pytest-flake8", "pytest-cov", "pytest-enabler (>=1.3)", "jaraco.itertools", "func-timeout", "pytest-black (>=0.3.7)", "pytest-mypy (>=0.9.1)"] +testing = ["pytest-mypy (>=0.9.1)", "pytest-black (>=0.3.7)", "func-timeout", "jaraco.itertools", "pytest-enabler (>=1.3)", "pytest-cov", "pytest-flake8", "pytest-checkdocs (>=2.4)", "pytest (>=6)"] +docs = ["jaraco.tidelift (>=1.4)", "rst.linker (>=1.9)", "jaraco.packaging (>=9)", "sphinx"] [[package]] name = "zstandard" @@ -505,7 +505,7 @@ snappy = ["python-snappy"] [metadata] lock-version = "1.1" python-versions = "^3.8" -content-hash = "8278a78ca5397fddd33ce52b4ff7dda2d57c221a1ee2c70a3bcf154dedfc72f5" +content-hash = "7573b12450ca9f900e286cf215db1d582b0ba4186045af0a088b7d7673caa73c" [metadata.files] atomicwrites = [ @@ -800,41 +800,41 @@ pycparser = [ {file = "pycparser-2.21.tar.gz", hash = "sha256:e644fdec12f7872f86c58ff790da456218b10f863970249516d60a5eaca77206"}, ] pydantic = [ - {file = "pydantic-1.9.1-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:c8098a724c2784bf03e8070993f6d46aa2eeca031f8d8a048dff277703e6e193"}, - {file = "pydantic-1.9.1-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:c320c64dd876e45254bdd350f0179da737463eea41c43bacbee9d8c9d1021f11"}, - {file = "pydantic-1.9.1-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:18f3e912f9ad1bdec27fb06b8198a2ccc32f201e24174cec1b3424dda605a310"}, - {file = "pydantic-1.9.1-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:c11951b404e08b01b151222a1cb1a9f0a860a8153ce8334149ab9199cd198131"}, - {file = "pydantic-1.9.1-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:8bc541a405423ce0e51c19f637050acdbdf8feca34150e0d17f675e72d119580"}, - {file = "pydantic-1.9.1-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:e565a785233c2d03724c4dc55464559639b1ba9ecf091288dd47ad9c629433bd"}, - {file = "pydantic-1.9.1-cp310-cp310-win_amd64.whl", hash = "sha256:a4a88dcd6ff8fd47c18b3a3709a89adb39a6373f4482e04c1b765045c7e282fd"}, - {file = "pydantic-1.9.1-cp36-cp36m-macosx_10_9_x86_64.whl", hash = "sha256:447d5521575f18e18240906beadc58551e97ec98142266e521c34968c76c8761"}, - {file = "pydantic-1.9.1-cp36-cp36m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:985ceb5d0a86fcaa61e45781e567a59baa0da292d5ed2e490d612d0de5796918"}, - {file = "pydantic-1.9.1-cp36-cp36m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:059b6c1795170809103a1538255883e1983e5b831faea6558ef873d4955b4a74"}, - {file = "pydantic-1.9.1-cp36-cp36m-musllinux_1_1_i686.whl", hash = "sha256:d12f96b5b64bec3f43c8e82b4aab7599d0157f11c798c9f9c528a72b9e0b339a"}, - {file = "pydantic-1.9.1-cp36-cp36m-musllinux_1_1_x86_64.whl", hash = "sha256:ae72f8098acb368d877b210ebe02ba12585e77bd0db78ac04a1ee9b9f5dd2166"}, - {file = "pydantic-1.9.1-cp36-cp36m-win_amd64.whl", hash = "sha256:79b485767c13788ee314669008d01f9ef3bc05db9ea3298f6a50d3ef596a154b"}, - {file = "pydantic-1.9.1-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:494f7c8537f0c02b740c229af4cb47c0d39840b829ecdcfc93d91dcbb0779892"}, - {file = "pydantic-1.9.1-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:f0f047e11febe5c3198ed346b507e1d010330d56ad615a7e0a89fae604065a0e"}, - {file = "pydantic-1.9.1-cp37-cp37m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:969dd06110cb780da01336b281f53e2e7eb3a482831df441fb65dd30403f4608"}, - {file = "pydantic-1.9.1-cp37-cp37m-musllinux_1_1_i686.whl", hash = "sha256:177071dfc0df6248fd22b43036f936cfe2508077a72af0933d0c1fa269b18537"}, - {file = "pydantic-1.9.1-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:9bcf8b6e011be08fb729d110f3e22e654a50f8a826b0575c7196616780683380"}, - {file = "pydantic-1.9.1-cp37-cp37m-win_amd64.whl", hash = "sha256:a955260d47f03df08acf45689bd163ed9df82c0e0124beb4251b1290fa7ae728"}, - {file = "pydantic-1.9.1-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:9ce157d979f742a915b75f792dbd6aa63b8eccaf46a1005ba03aa8a986bde34a"}, - {file = "pydantic-1.9.1-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:0bf07cab5b279859c253d26a9194a8906e6f4a210063b84b433cf90a569de0c1"}, - {file = "pydantic-1.9.1-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:5d93d4e95eacd313d2c765ebe40d49ca9dd2ed90e5b37d0d421c597af830c195"}, - {file = "pydantic-1.9.1-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:1542636a39c4892c4f4fa6270696902acb186a9aaeac6f6cf92ce6ae2e88564b"}, - {file = "pydantic-1.9.1-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:a9af62e9b5b9bc67b2a195ebc2c2662fdf498a822d62f902bf27cccb52dbbf49"}, - {file = "pydantic-1.9.1-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:fe4670cb32ea98ffbf5a1262f14c3e102cccd92b1869df3bb09538158ba90fe6"}, - {file = "pydantic-1.9.1-cp38-cp38-win_amd64.whl", hash = "sha256:9f659a5ee95c8baa2436d392267988fd0f43eb774e5eb8739252e5a7e9cf07e0"}, - {file = "pydantic-1.9.1-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:b83ba3825bc91dfa989d4eed76865e71aea3a6ca1388b59fc801ee04c4d8d0d6"}, - {file = "pydantic-1.9.1-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:1dd8fecbad028cd89d04a46688d2fcc14423e8a196d5b0a5c65105664901f810"}, - {file = "pydantic-1.9.1-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:02eefd7087268b711a3ff4db528e9916ac9aa18616da7bca69c1871d0b7a091f"}, - {file = "pydantic-1.9.1-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:7eb57ba90929bac0b6cc2af2373893d80ac559adda6933e562dcfb375029acee"}, - {file = "pydantic-1.9.1-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:4ce9ae9e91f46c344bec3b03d6ee9612802682c1551aaf627ad24045ce090761"}, - {file = "pydantic-1.9.1-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:72ccb318bf0c9ab97fc04c10c37683d9eea952ed526707fabf9ac5ae59b701fd"}, - {file = "pydantic-1.9.1-cp39-cp39-win_amd64.whl", hash = "sha256:61b6760b08b7c395975d893e0b814a11cf011ebb24f7d869e7118f5a339a82e1"}, - {file = "pydantic-1.9.1-py3-none-any.whl", hash = "sha256:4988c0f13c42bfa9ddd2fe2f569c9d54646ce84adc5de84228cfe83396f3bd58"}, - {file = "pydantic-1.9.1.tar.gz", hash = "sha256:1ed987c3ff29fff7fd8c3ea3a3ea877ad310aae2ef9889a119e22d3f2db0691a"}, + {file = "pydantic-1.9.2-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:9c9e04a6cdb7a363d7cb3ccf0efea51e0abb48e180c0d31dca8d247967d85c6e"}, + {file = "pydantic-1.9.2-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:fafe841be1103f340a24977f61dee76172e4ae5f647ab9e7fd1e1fca51524f08"}, + {file = "pydantic-1.9.2-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:afacf6d2a41ed91fc631bade88b1d319c51ab5418870802cedb590b709c5ae3c"}, + {file = "pydantic-1.9.2-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:3ee0d69b2a5b341fc7927e92cae7ddcfd95e624dfc4870b32a85568bd65e6131"}, + {file = "pydantic-1.9.2-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:ff68fc85355532ea77559ede81f35fff79a6a5543477e168ab3a381887caea76"}, + {file = "pydantic-1.9.2-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:c0f5e142ef8217019e3eef6ae1b6b55f09a7a15972958d44fbd228214cede567"}, + {file = "pydantic-1.9.2-cp310-cp310-win_amd64.whl", hash = "sha256:615661bfc37e82ac677543704437ff737418e4ea04bef9cf11c6d27346606044"}, + {file = "pydantic-1.9.2-cp36-cp36m-macosx_10_9_x86_64.whl", hash = "sha256:328558c9f2eed77bd8fffad3cef39dbbe3edc7044517f4625a769d45d4cf7555"}, + {file = "pydantic-1.9.2-cp36-cp36m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:2bd446bdb7755c3a94e56d7bdfd3ee92396070efa8ef3a34fab9579fe6aa1d84"}, + {file = "pydantic-1.9.2-cp36-cp36m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:e0b214e57623a535936005797567231a12d0da0c29711eb3514bc2b3cd008d0f"}, + {file = "pydantic-1.9.2-cp36-cp36m-musllinux_1_1_i686.whl", hash = "sha256:d8ce3fb0841763a89322ea0432f1f59a2d3feae07a63ea2c958b2315e1ae8adb"}, + {file = "pydantic-1.9.2-cp36-cp36m-musllinux_1_1_x86_64.whl", hash = "sha256:b34ba24f3e2d0b39b43f0ca62008f7ba962cff51efa56e64ee25c4af6eed987b"}, + {file = "pydantic-1.9.2-cp36-cp36m-win_amd64.whl", hash = "sha256:84d76ecc908d917f4684b354a39fd885d69dd0491be175f3465fe4b59811c001"}, + {file = "pydantic-1.9.2-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:4de71c718c9756d679420c69f216776c2e977459f77e8f679a4a961dc7304a56"}, + {file = "pydantic-1.9.2-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:5803ad846cdd1ed0d97eb00292b870c29c1f03732a010e66908ff48a762f20e4"}, + {file = "pydantic-1.9.2-cp37-cp37m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:a8c5360a0297a713b4123608a7909e6869e1b56d0e96eb0d792c27585d40757f"}, + {file = "pydantic-1.9.2-cp37-cp37m-musllinux_1_1_i686.whl", hash = "sha256:cdb4272678db803ddf94caa4f94f8672e9a46bae4a44f167095e4d06fec12979"}, + {file = "pydantic-1.9.2-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:19b5686387ea0d1ea52ecc4cffb71abb21702c5e5b2ac626fd4dbaa0834aa49d"}, + {file = "pydantic-1.9.2-cp37-cp37m-win_amd64.whl", hash = "sha256:32e0b4fb13ad4db4058a7c3c80e2569adbd810c25e6ca3bbd8b2a9cc2cc871d7"}, + {file = "pydantic-1.9.2-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:91089b2e281713f3893cd01d8e576771cd5bfdfbff5d0ed95969f47ef6d676c3"}, + {file = "pydantic-1.9.2-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:e631c70c9280e3129f071635b81207cad85e6c08e253539467e4ead0e5b219aa"}, + {file = "pydantic-1.9.2-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:4b3946f87e5cef3ba2e7bd3a4eb5a20385fe36521d6cc1ebf3c08a6697c6cfb3"}, + {file = "pydantic-1.9.2-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:5565a49effe38d51882cb7bac18bda013cdb34d80ac336428e8908f0b72499b0"}, + {file = "pydantic-1.9.2-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:bd67cb2c2d9602ad159389c29e4ca964b86fa2f35c2faef54c3eb28b4efd36c8"}, + {file = "pydantic-1.9.2-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:4aafd4e55e8ad5bd1b19572ea2df546ccace7945853832bb99422a79c70ce9b8"}, + {file = "pydantic-1.9.2-cp38-cp38-win_amd64.whl", hash = "sha256:d70916235d478404a3fa8c997b003b5f33aeac4686ac1baa767234a0f8ac2326"}, + {file = "pydantic-1.9.2-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:f0ca86b525264daa5f6b192f216a0d1e860b7383e3da1c65a1908f9c02f42801"}, + {file = "pydantic-1.9.2-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:1061c6ee6204f4f5a27133126854948e3b3d51fcc16ead2e5d04378c199b2f44"}, + {file = "pydantic-1.9.2-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:e78578f0c7481c850d1c969aca9a65405887003484d24f6110458fb02cca7747"}, + {file = "pydantic-1.9.2-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:5da164119602212a3fe7e3bc08911a89db4710ae51444b4224c2382fd09ad453"}, + {file = "pydantic-1.9.2-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:7ead3cd020d526f75b4188e0a8d71c0dbbe1b4b6b5dc0ea775a93aca16256aeb"}, + {file = "pydantic-1.9.2-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:7d0f183b305629765910eaad707800d2f47c6ac5bcfb8c6397abdc30b69eeb15"}, + {file = "pydantic-1.9.2-cp39-cp39-win_amd64.whl", hash = "sha256:f1a68f4f65a9ee64b6ccccb5bf7e17db07caebd2730109cb8a95863cfa9c4e55"}, + {file = "pydantic-1.9.2-py3-none-any.whl", hash = "sha256:78a4d6bdfd116a559aeec9a4cfe77dda62acc6233f8b56a716edad2651023e5e"}, + {file = "pydantic-1.9.2.tar.gz", hash = "sha256:8cb0bc509bfb71305d7a59d00163d5f9fc4530f0881ea32c74ff4f74c85f3d3d"}, ] pyparsing = [ {file = "pyparsing-3.0.9-py3-none-any.whl", hash = "sha256:5026bae9a10eeaefb61dab2f09052b9f4307d44aee4eda64b309723d8d206bbc"}, diff --git a/pyproject.toml b/pyproject.toml index eb81644983..e70dca7f03 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -45,7 +45,7 @@ python = "^3.8" mmh3 = "^3.0.0" requests = "^2.28.1" -pydantic = "^1.9.1" +pydantic = "^1.9.2" pyarrow = { version = "^9.0.0", optional = true } From 920b3e6541babb5f853d912c5962db5825335d2c Mon Sep 17 00:00:00 2001 From: Fokko Driesprong Date: Mon, 15 Aug 2022 22:40:17 +0200 Subject: [PATCH 166/642] Python: Implement Hive create and load table (#5447) --- pyiceberg/catalog/hive.py | 152 +++++++++++++++---- pyiceberg/io/__init__.py | 12 ++ pyiceberg/schema.py | 26 ++++ pyiceberg/table/metadata.py | 5 +- tests/catalog/test_hive.py | 276 ++++++++++++++++++++++++++++++++--- tests/conftest.py | 64 ++++++++ tests/table/test_metadata.py | 101 +++---------- tests/table/test_sorting.py | 7 +- 8 files changed, 509 insertions(+), 134 deletions(-) diff --git a/pyiceberg/catalog/hive.py b/pyiceberg/catalog/hive.py index f8ad209fbd..63ba43956a 100644 --- a/pyiceberg/catalog/hive.py +++ b/pyiceberg/catalog/hive.py @@ -16,6 +16,7 @@ # under the License. import getpass import time +import uuid from typing import ( Any, Dict, @@ -56,9 +57,12 @@ NoSuchTableError, TableAlreadyExistsError, ) -from pyiceberg.schema import Schema +from pyiceberg.io import FileIO, load_file_io +from pyiceberg.schema import Schema, SchemaVisitor, visit +from pyiceberg.serializers import FromInputFile, ToOutputFile from pyiceberg.table import Table -from pyiceberg.table.partitioning import PartitionSpec +from pyiceberg.table.metadata import DEFAULT_LAST_PARTITION_ID, TableMetadataV1, TableMetadataV2 +from pyiceberg.table.partitioning import UNPARTITIONED_PARTITION_SPEC, PartitionSpec from pyiceberg.table.sorting import UNSORTED_SORT_ORDER, SortOrder from pyiceberg.typedef import EMPTY_DICT from pyiceberg.types import ( @@ -69,11 +73,12 @@ DoubleType, FixedType, FloatType, - IcebergType, IntegerType, ListType, LongType, MapType, + NestedField, + PrimitiveType, StringType, StructType, TimestampType, @@ -95,13 +100,15 @@ UUIDType: "string", BinaryType: "binary", FixedType: "binary", - DecimalType: None, - StructType: None, - ListType: None, - MapType: None, } +COMMENT = "comment" OWNER = "owner" +TABLE_TYPE = "table_type" +METADATA_LOCATION = "metadata_location" +ICEBERG = "iceberg" +LOCATION = "location" +WAREHOUSE = "warehouse" class _HiveClient: @@ -129,7 +136,7 @@ def __exit__(self, exc_type, exc_val, exc_tb): def _construct_hive_storage_descriptor(schema: Schema, location: Optional[str]) -> StorageDescriptor: ser_de_info = SerDeInfo(serializationLib="org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe") return StorageDescriptor( - _convert_schema_to_columns(schema), + [FieldSchema(field.name, visit(field.field_type, SchemaToHiveConverter()), field.doc) for field in schema.fields], location, "org.apache.hadoop.mapred.FileInputFormat", "org.apache.hadoop.mapred.FileOutputFormat", @@ -137,16 +144,6 @@ def _construct_hive_storage_descriptor(schema: Schema, location: Optional[str]) ) -def _convert_schema_to_columns(schema: Schema) -> List[FieldSchema]: - return [FieldSchema(field.name, _iceberg_type_to_hive_types(field.field_type), field.doc) for field in schema.fields] - - -def _iceberg_type_to_hive_types(col_type: IcebergType) -> str: - if hive_type := hive_types.get(type(col_type)): - return hive_type - raise NotImplementedError(f"Not yet implemented column type {col_type}") - - PROP_EXTERNAL = "EXTERNAL" PROP_TABLE_TYPE = "table_type" PROP_METADATA_LOCATION = "metadata_location" @@ -164,9 +161,9 @@ def _construct_parameters(metadata_location: str, previous_metadata_location: Op def _annotate_namespace(database: HiveDatabase, properties: Properties) -> HiveDatabase: params = {} for key, value in properties.items(): - if key == "comment": + if key == COMMENT: database.description = value - elif key == "location": + elif key == LOCATION: database.locationUri = value else: params[key] = value @@ -174,6 +171,46 @@ def _annotate_namespace(database: HiveDatabase, properties: Properties) -> HiveD return database +HIVE_PRIMITIVE_TYPES = { + BooleanType: "boolean", + IntegerType: "int", + LongType: "bigint", + FloatType: "float", + DoubleType: "double", + DateType: "date", + TimeType: "string", + TimestampType: "timestamp", + StringType: "string", + UUIDType: "string", + BinaryType: "binary", + FixedType: "binary", +} + + +class SchemaToHiveConverter(SchemaVisitor[str]): + def schema(self, schema: Schema, struct_result: str) -> str: + return struct_result + + def struct(self, struct: StructType, field_results: List[str]) -> str: + return f"struct<{','.join(field_results)}>" + + def field(self, field: NestedField, field_result: str) -> str: + return f"{field.name}:{field_result}" + + def list(self, list_type: ListType, element_result: str) -> str: + return f"array<{element_result}>" + + def map(self, map_type: MapType, key_result: str, value_result: str) -> str: + # Key has to be primitive for Hive + return f"map<{key_result},{value_result}>" + + def primitive(self, primitive: PrimitiveType) -> str: + if isinstance(primitive, DecimalType): + return f"decimal({primitive.precision},{primitive.scale})" + else: + return HIVE_PRIMITIVE_TYPES[type(primitive)] + + class HiveCatalog(Catalog): _client: _HiveClient @@ -202,17 +239,46 @@ def __init__(self, name: str, properties: Properties, uri: str): super().__init__(name, properties) self._client = _HiveClient(uri) - def _convert_hive_into_iceberg(self, table: HiveTable) -> Table: - # Requires reading the manifest, will implement this in another PR - # Check the table type - return Table(identifier=(table.dbName, table.tableName)) + def _convert_hive_into_iceberg(self, table: HiveTable, io: FileIO) -> Table: + properties: Dict[str, str] = table.parameters + if TABLE_TYPE not in properties: + raise NoSuchTableError(f"Property table_type missing, could not determine type: {table.dbName}.{table.tableName}") + + table_type = properties[TABLE_TYPE] + if table_type.lower() != ICEBERG: + raise NoSuchTableError(f"Property table_type is {table_type}, expected {ICEBERG}: {table.dbName}.{table.tableName}") + + if prop_metadata_location := properties.get(METADATA_LOCATION): + metadata_location = prop_metadata_location + else: + raise NoSuchTableError(f"Table property {METADATA_LOCATION} is missing") + + file = io.new_input(metadata_location) + metadata = FromInputFile.table_metadata(file) + return Table(identifier=(table.dbName, table.tableName), metadata=metadata, metadata_location=metadata_location) + + def _write_metadata(self, metadata: Union[TableMetadataV1, TableMetadataV2], io: FileIO, metadata_path: str): + ToOutputFile.table_metadata(metadata, io.new_output(metadata_path)) + + def _resolve_table_location(self, location: Optional[str], database_name: str, table_name: str): + if not location: + database_properties = self.load_namespace_properties(database_name) + if database_location := database_properties.get(LOCATION): + database_location = database_location.rstrip("/") + return f"{database_location}/{table_name}" + + if warehouse_location := self.properties.get(WAREHOUSE): + warehouse_location = warehouse_location.rstrip("/") + return f"{warehouse_location}/{database_name}/{table_name}" + raise ValueError("Cannot determine location from warehouse, please provide an explicit location") + return location def create_table( self, identifier: Union[str, Identifier], schema: Schema, location: Optional[str] = None, - partition_spec: Optional[PartitionSpec] = None, + partition_spec: PartitionSpec = UNPARTITIONED_PARTITION_SPEC, sort_order: SortOrder = UNSORTED_SORT_ORDER, properties: Properties = EMPTY_DICT, ) -> Table: @@ -234,7 +300,29 @@ def create_table( ValueError: If the identifier is invalid """ database_name, table_name = self.identifier_to_database_and_table(identifier) - current_time_millis = int(time.time()) + current_time_millis = int(time.time() * 1000) + + location = self._resolve_table_location(location, database_name, table_name) + + metadata_location = f"{location}/metadata/00000-{uuid.uuid4()}.metadata.json" + metadata = TableMetadataV2( + location=location, + schemas=[schema], + current_schema_id=schema.schema_id, + partition_specs=[partition_spec], + default_spec_id=partition_spec.spec_id, + sort_orders=[sort_order], + default_sort_order_id=sort_order.order_id, + properties=properties or {}, + last_updated_ms=current_time_millis, + last_column_id=schema.highest_field_id, + last_partition_id=max(field.field_id for field in partition_spec.fields) + if partition_spec.fields + else DEFAULT_LAST_PARTITION_ID, + ) + io = load_file_io({**self.properties, **properties}) + self._write_metadata(metadata, io, metadata_location) + tbl = HiveTable( dbName=database_name, tableName=table_name, @@ -243,7 +331,7 @@ def create_table( lastAccessTime=current_time_millis // 1000, sd=_construct_hive_storage_descriptor(schema, location), tableType="EXTERNAL_TABLE", - parameters=_construct_parameters("s3://"), + parameters=_construct_parameters(metadata_location), ) try: with self._client as open_client: @@ -251,7 +339,8 @@ def create_table( hive_table = open_client.get_table(dbname=database_name, tbl_name=table_name) except AlreadyExistsException as e: raise TableAlreadyExistsError(f"Table {database_name}.{table_name} already exists") from e - return self._convert_hive_into_iceberg(hive_table) + + return self._convert_hive_into_iceberg(hive_table, io) def load_table(self, identifier: Union[str, Identifier]) -> Table: """Loads the table's metadata and returns the table instance. @@ -275,7 +364,8 @@ def load_table(self, identifier: Union[str, Identifier]) -> Table: except NoSuchObjectException as e: raise NoSuchTableError(f"Table does not exists: {table_name}") from e - return self._convert_hive_into_iceberg(hive_table) + io = load_file_io({**self.properties, **hive_table.parameters}) + return self._convert_hive_into_iceberg(hive_table, io) def drop_table(self, identifier: Union[str, Identifier]) -> None: """Drop a table. @@ -410,9 +500,9 @@ def load_namespace_properties(self, namespace: Union[str, Identifier]) -> Proper with self._client as open_client: database = open_client.get_database(name=database_name) properties = database.parameters - properties["location"] = database.locationUri + properties[LOCATION] = database.locationUri if comment := database.description: - properties["comment"] = comment + properties[COMMENT] = comment return properties except NoSuchObjectException as e: raise NoSuchNamespaceError(f"Database does not exists: {database_name}") from e diff --git a/pyiceberg/io/__init__.py b/pyiceberg/io/__init__.py index 6169dcdfd5..5a317c21e6 100644 --- a/pyiceberg/io/__init__.py +++ b/pyiceberg/io/__init__.py @@ -26,6 +26,8 @@ from io import SEEK_SET from typing import Protocol, Union, runtime_checkable +from pyiceberg.typedef import Properties + @runtime_checkable class InputStream(Protocol): @@ -214,3 +216,13 @@ def delete(self, location: Union[str, InputFile, OutputFile]) -> None: PermissionError: If the file at location cannot be accessed due to a permission error FileNotFoundError: When the file at the provided location does not exist """ + + +def load_file_io(_: Properties) -> FileIO: + # To be implemented in a different PR. + # - If py-file-io is present, load the right Python class + # - When the property is missing, map from Java's filo-io to an appropriate FileIO + # - Extend the FileIO structure with a initialize that pass in properties (could also be the constructor?) + from pyiceberg.io.pyarrow import PyArrowFileIO + + return PyArrowFileIO() diff --git a/pyiceberg/schema.py b/pyiceberg/schema.py index 5e5d4af56d..657f8fbb07 100644 --- a/pyiceberg/schema.py +++ b/pyiceberg/schema.py @@ -169,6 +169,10 @@ def find_type(self, name_or_id: Union[str, int], case_sensitive: bool = True) -> raise ValueError(f"Could not find field with name or id {name_or_id}, case_sensitive={case_sensitive}") return field.field_type + @property + def highest_field_id(self): + return visit(self.as_struct(), _FindLastFieldId()) + def find_column_name(self, column_id: int) -> Optional[str]: """Find a column name given a column ID @@ -612,3 +616,25 @@ def build_position_accessors(schema_or_type: Union[Schema, IcebergType]) -> Dict Dict[int, Accessor]: An index of field IDs to accessors """ return visit(schema_or_type, _BuildPositionAccessors()) + + +class _FindLastFieldId(SchemaVisitor[int]): + """Traverses the schema to get the highest field-id""" + + def schema(self, schema: Schema, struct_result: int) -> int: + return struct_result + + def struct(self, struct: StructType, field_results: List[int]) -> int: + return max(field_results) + + def field(self, field: NestedField, field_result: int) -> int: + return max(field.field_id, field_result) + + def list(self, list_type: ListType, element_result: int) -> int: + return element_result + + def map(self, map_type: MapType, key_result: int, value_result: int) -> int: + return max(key_result, value_result) + + def primitive(self, primitive: PrimitiveType) -> int: + return 0 diff --git a/pyiceberg/table/metadata.py b/pyiceberg/table/metadata.py index 9f48863d4d..33abd4c8df 100644 --- a/pyiceberg/table/metadata.py +++ b/pyiceberg/table/metadata.py @@ -14,6 +14,7 @@ # KIND, either express or implied. See the License for the # specific language governing permissions and limitations # under the License. +import uuid from copy import copy from typing import ( Any, @@ -323,12 +324,12 @@ def check_partition_specs(cls, values: Dict[str, Any]): def check_sort_orders(cls, values: Dict[str, Any]): return check_sort_orders(values) - format_version: Literal[2] = Field(alias="format-version") + format_version: Literal[2] = Field(alias="format-version", default=2) """An integer version number for the format. Currently, this can be 1 or 2 based on the spec. Implementations must throw an exception if a table’s version is higher than the supported version.""" - table_uuid: UUID = Field(alias="table-uuid") + table_uuid: UUID = Field(alias="table-uuid", default_factory=uuid.uuid4) """A UUID that identifies the table, generated when the table is created. Implementations must throw an exception if a table’s UUID does not match the expected UUID after refreshing metadata.""" diff --git a/tests/catalog/test_hive.py b/tests/catalog/test_hive.py index 63443ee720..5227c6b8fe 100644 --- a/tests/catalog/test_hive.py +++ b/tests/catalog/test_hive.py @@ -15,6 +15,9 @@ # specific language governing permissions and limitations # under the License. # pylint: disable=protected-access,redefined-outer-name +import json +import uuid +from typing import Any, Dict from unittest.mock import MagicMock, patch import pytest @@ -32,7 +35,7 @@ from hive_metastore.ttypes import Table as HiveTable from pyiceberg.catalog import PropertiesUpdateSummary -from pyiceberg.catalog.hive import HiveCatalog +from pyiceberg.catalog.hive import WAREHOUSE, HiveCatalog, _construct_hive_storage_descriptor from pyiceberg.exceptions import ( NamespaceAlreadyExistsError, NamespaceNotEmptyError, @@ -40,13 +43,36 @@ NoSuchTableError, ) from pyiceberg.schema import Schema +from pyiceberg.serializers import ToOutputFile +from pyiceberg.table.metadata import TableMetadata, TableMetadataV2 +from pyiceberg.table.partitioning import PartitionField, PartitionSpec +from pyiceberg.table.refs import SnapshotRef, SnapshotRefType +from pyiceberg.table.snapshots import Operation, Snapshot, Summary +from pyiceberg.table.sorting import ( + NullOrder, + SortDirection, + SortField, + SortOrder, +) +from pyiceberg.transforms import BucketTransform, IdentityTransform +from pyiceberg.types import ( + BooleanType, + IntegerType, + LongType, + NestedField, + StringType, +) +from tests.conftest import LocalFileIO HIVE_CATALOG_NAME = "hive" HIVE_METASTORE_FAKE_URL = "thrift://unknown:9083" @pytest.fixture -def hive_table() -> HiveTable: +def hive_table(tmp_path_factory, example_table_metadata_v2: Dict[str, Any]) -> HiveTable: + metadata_path = str(tmp_path_factory.mktemp("metadata") / f"{uuid.uuid4()}.metadata.json") + ToOutputFile.table_metadata(TableMetadataV2(**example_table_metadata_v2), LocalFileIO().new_output(str(metadata_path)), True) + return HiveTable( tableName="new_tabl2e", dbName="default", @@ -81,7 +107,12 @@ def hive_table() -> HiveTable: storedAsSubDirectories=False, ), partitionKeys=[], - parameters={"EXTERNAL": "TRUE", "transient_lastDdlTime": "1659092339"}, + parameters={ + "EXTERNAL": "TRUE", + "transient_lastDdlTime": "1659092339", + "table_type": "ICEBERG", + "metadata_location": metadata_path, + }, viewOriginalText=None, viewExpandedText=None, tableType="EXTERNAL_TABLE", @@ -104,12 +135,17 @@ def hive_table() -> HiveTable: ) -@pytest.fixture -def hive_database() -> HiveDatabase: +@pytest.fixture(scope="session") +def hive_database(tmp_path_factory) -> HiveDatabase: + # Pre-create the directory, this has to be done because + # of a local FS. Not needed with an actual object store. + database_path = tmp_path_factory.mktemp("database") + manifest_path = database_path / "database" / "table" / "metadata" + manifest_path.mkdir(parents=True) return HiveDatabase( name="default", description=None, - locationUri="file:/tmp/default2.db", + locationUri=str(database_path / "database"), parameters={"test": "property"}, privileges=None, ownerName=None, @@ -139,16 +175,23 @@ def test_check_number_of_namespaces(table_schema_simple: Schema): catalog.create_table("table", schema=table_schema_simple) -@patch("time.time", MagicMock(return_value=12345000)) -def test_create_table(table_schema_simple: Schema, hive_table: HiveTable): +@patch("time.time", MagicMock(return_value=12345)) +@patch("uuid.uuid4", MagicMock(return_value="01234567-0123-0123-0123-0123456789ab")) +def test_create_table(table_schema_simple: Schema, hive_database: HiveDatabase, hive_table: HiveTable): catalog = HiveCatalog(HIVE_CATALOG_NAME, {}, uri=HIVE_METASTORE_FAKE_URL) catalog._client = MagicMock() - catalog._client.__enter__().create_table.return_value = hive_table + catalog._client.__enter__().create_table.return_value = None catalog._client.__enter__().get_table.return_value = hive_table - + catalog._client.__enter__().get_database.return_value = hive_database catalog.create_table(("default", "table"), schema=table_schema_simple, properties={"owner": "javaberg"}) + called_hive_table: HiveTable = catalog._client.__enter__().create_table.call_args[0][0] + # This one is generated within the function itself, so we need to extract + # it to construct the assert_called_with + metadata_location: str = called_hive_table.parameters["metadata_location"] + assert metadata_location.endswith(".metadata.json") + assert "/database/table/metadata/" in metadata_location catalog._client.__enter__().create_table.assert_called_with( HiveTable( tableName="table", @@ -163,7 +206,7 @@ def test_create_table(table_schema_simple: Schema, hive_table: HiveTable): FieldSchema(name="bar", type="int", comment=None), FieldSchema(name="baz", type="boolean", comment=None), ], - location=None, + location=f"{hive_database.locationUri}/table", inputFormat="org.apache.hadoop.mapred.FileInputFormat", outputFormat="org.apache.hadoop.mapred.FileOutputFormat", compressed=None, @@ -184,7 +227,7 @@ def test_create_table(table_schema_simple: Schema, hive_table: HiveTable): storedAsSubDirectories=None, ), partitionKeys=None, - parameters={"EXTERNAL": "TRUE", "table_type": "ICEBERG", "metadata_location": "s3://"}, + parameters={"EXTERNAL": "TRUE", "table_type": "ICEBERG", "metadata_location": metadata_location}, viewOriginalText=None, viewExpandedText=None, tableType="EXTERNAL_TABLE", @@ -207,15 +250,137 @@ def test_create_table(table_schema_simple: Schema, hive_table: HiveTable): ) ) + with open(metadata_location, encoding="utf-8") as f: + payload = json.load(f) + + metadata = TableMetadata.parse_obj(payload) + + assert "database/table" in metadata.location + + assert metadata + assert metadata == TableMetadataV2( + # The following two ones are dynamic + location=metadata.location, + table_uuid=metadata.table_uuid, + last_updated_ms=12345000, + last_column_id=3, + schemas=[ + Schema( + NestedField(field_id=1, name="foo", field_type=StringType(), required=False), + NestedField(field_id=2, name="bar", field_type=IntegerType(), required=True), + NestedField(field_id=3, name="baz", field_type=BooleanType(), required=False), + schema_id=1, + identifier_field_ids=[2], + ) + ], + current_schema_id=1, + partition_specs=[PartitionSpec(spec_id=0, fields=())], + default_spec_id=0, + last_partition_id=1000, + properties={"owner": "javaberg"}, + current_snapshot_id=None, + snapshots=[], + snapshot_log=[], + metadata_log=[], + sort_orders=[SortOrder(order_id=0)], + default_sort_order_id=0, + refs={}, + format_version=2, + last_sequence_number=0, + ) + def test_load_table(hive_table: HiveTable): catalog = HiveCatalog(HIVE_CATALOG_NAME, {}, uri=HIVE_METASTORE_FAKE_URL) catalog._client = MagicMock() catalog._client.__enter__().get_table.return_value = hive_table - catalog.load_table(("default", "table")) - - catalog._client.__enter__().get_table.assert_called_with(dbname="default", tbl_name="table") + table = catalog.load_table(("default", "new_tabl2e")) + + catalog._client.__enter__().get_table.assert_called_with(dbname="default", tbl_name="new_tabl2e") + + assert table.identifier == ("default", "new_tabl2e") + assert table.metadata == TableMetadataV2( + location="s3://bucket/test/location", + table_uuid=uuid.UUID("9c12d441-03fe-4693-9a96-a0705ddf69c1"), + last_updated_ms=1602638573590, + last_column_id=3, + schemas=[ + Schema( + NestedField(field_id=1, name="x", field_type=LongType(), required=True), + schema_id=0, + identifier_field_ids=[], + ), + Schema( + NestedField(field_id=1, name="x", field_type=LongType(), required=True), + NestedField(field_id=2, name="y", field_type=LongType(), required=True, doc="comment"), + NestedField(field_id=3, name="z", field_type=LongType(), required=True), + schema_id=1, + identifier_field_ids=[1, 2], + ), + ], + current_schema_id=1, + partition_specs=[ + PartitionSpec( + spec_id=0, fields=(PartitionField(source_id=1, field_id=1000, transform=IdentityTransform(), name="x"),) + ) + ], + default_spec_id=0, + last_partition_id=1000, + properties={"read.split.target.size": "134217728"}, + current_snapshot_id=3055729675574597004, + snapshots=[ + Snapshot( + snapshot_id=3051729675574597004, + parent_snapshot_id=None, + sequence_number=0, + timestamp_ms=1515100955770, + manifest_list="s3://a/b/1.avro", + summary=Summary(Operation.APPEND), + schema_id=None, + ), + Snapshot( + snapshot_id=3055729675574597004, + parent_snapshot_id=3051729675574597004, + sequence_number=1, + timestamp_ms=1555100955770, + manifest_list="s3://a/b/2.avro", + summary=Summary(Operation.APPEND), + schema_id=1, + ), + ], + snapshot_log=[ + {"snapshot-id": 3051729675574597004, "timestamp-ms": 1515100955770}, + {"snapshot-id": 3055729675574597004, "timestamp-ms": 1555100955770}, + ], + metadata_log=[], + sort_orders=[ + SortOrder( + 3, + SortField( + source_id=2, transform=IdentityTransform(), direction=SortDirection.ASC, null_order=NullOrder.NULLS_FIRST + ), + SortField( + source_id=3, + transform=BucketTransform(num_buckets=4), + direction=SortDirection.DESC, + null_order=NullOrder.NULLS_LAST, + ), + ) + ], + default_sort_order_id=3, + refs={ + "main": SnapshotRef( + snapshot_id=3055729675574597004, + snapshot_ref_type=SnapshotRefType.BRANCH, + min_snapshots_to_keep=None, + max_snapshot_age_ms=None, + max_ref_age_ms=None, + ) + }, + format_version=2, + last_sequence_number=34, + ) def test_rename_table_from_does_not_exists(): @@ -377,7 +542,7 @@ def test_load_namespace_properties(hive_database: HiveDatabase): catalog._client = MagicMock() catalog._client.__enter__().get_database.return_value = hive_database - assert catalog.load_namespace_properties("default2") == {"location": "file:/tmp/default2.db", "test": "property"} + assert catalog.load_namespace_properties("default2") == {"location": hive_database.locationUri, "test": "property"} catalog._client.__enter__().get_database.assert_called_with(name="default2") @@ -410,7 +575,7 @@ def test_update_namespace_properties(hive_database: HiveDatabase): HiveDatabase( name="default", description=None, - locationUri="file:/tmp/default2.db", + locationUri=hive_database.locationUri, parameters={"test": None, "label": "core"}, privileges=None, ownerName=None, @@ -444,3 +609,80 @@ def test_update_namespace_properties_overlap(): catalog.update_namespace_properties(("table",), removals=set("a"), updates={"a": "b"}) assert "Updates and deletes have an overlap: {'a'}" in str(exc_info.value) + + +def test_construct_hive_storage_descriptor_simple(table_schema_simple: Schema): + descriptor = _construct_hive_storage_descriptor(table_schema_simple, "s3://") + assert descriptor == StorageDescriptor( + cols=[ + FieldSchema(name="foo", type="string", comment=None), + FieldSchema(name="bar", type="int", comment=None), + FieldSchema(name="baz", type="boolean", comment=None), + ], + location="s3://", + inputFormat="org.apache.hadoop.mapred.FileInputFormat", + outputFormat="org.apache.hadoop.mapred.FileOutputFormat", + compressed=None, + numBuckets=None, + serdeInfo=SerDeInfo( + name=None, + serializationLib="org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe", + parameters=None, + description=None, + serializerClass=None, + deserializerClass=None, + serdeType=None, + ), + bucketCols=None, + sortCols=None, + parameters=None, + skewedInfo=None, + storedAsSubDirectories=None, + ) + + +def test_construct_hive_storage_descriptor_nested(table_schema_nested: Schema): + descriptor = _construct_hive_storage_descriptor(table_schema_nested, "s3://") + assert descriptor == StorageDescriptor( + cols=[ + FieldSchema(name="foo", type="string", comment=None), + FieldSchema(name="bar", type="int", comment=None), + FieldSchema(name="baz", type="boolean", comment=None), + FieldSchema(name="qux", type="array", comment=None), + FieldSchema(name="quux", type="map>", comment=None), + FieldSchema(name="location", type="array>", comment=None), + FieldSchema(name="person", type="struct", comment=None), + ], + location="s3://", + inputFormat="org.apache.hadoop.mapred.FileInputFormat", + outputFormat="org.apache.hadoop.mapred.FileOutputFormat", + compressed=None, + numBuckets=None, + serdeInfo=SerDeInfo( + name=None, + serializationLib="org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe", + parameters=None, + description=None, + serializerClass=None, + deserializerClass=None, + serdeType=None, + ), + bucketCols=None, + sortCols=None, + parameters=None, + skewedInfo=None, + storedAsSubDirectories=None, + ) + + +def test_resolve_table_location_warehouse(hive_database: HiveDatabase): + catalog = HiveCatalog(HIVE_CATALOG_NAME, {WAREHOUSE: "/tmp/warehouse/"}, uri=HIVE_METASTORE_FAKE_URL) + + # Set this one to None, so we'll fall back to the properties + hive_database.locationUri = None + + catalog._client = MagicMock() + catalog._client.__enter__().get_database.return_value = hive_database + + location = catalog._resolve_table_location(None, "database", "table") + assert location == "/tmp/warehouse/database/table" diff --git a/tests/conftest.py b/tests/conftest.py index e2072790f1..079cc1ac4d 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -219,6 +219,70 @@ def all_avro_types() -> Dict[str, Any]: } +@pytest.fixture +def example_table_metadata_v2() -> Dict[str, Any]: + return { + "format-version": 2, + "table-uuid": "9c12d441-03fe-4693-9a96-a0705ddf69c1", + "location": "s3://bucket/test/location", + "last-sequence-number": 34, + "last-updated-ms": 1602638573590, + "last-column-id": 3, + "current-schema-id": 1, + "schemas": [ + {"type": "struct", "schema-id": 0, "fields": [{"id": 1, "name": "x", "required": True, "type": "long"}]}, + { + "type": "struct", + "schema-id": 1, + "identifier-field-ids": [1, 2], + "fields": [ + {"id": 1, "name": "x", "required": True, "type": "long"}, + {"id": 2, "name": "y", "required": True, "type": "long", "doc": "comment"}, + {"id": 3, "name": "z", "required": True, "type": "long"}, + ], + }, + ], + "default-spec-id": 0, + "partition-specs": [{"spec-id": 0, "fields": [{"name": "x", "transform": "identity", "source-id": 1, "field-id": 1000}]}], + "last-partition-id": 1000, + "default-sort-order-id": 3, + "sort-orders": [ + { + "order-id": 3, + "fields": [ + {"transform": "identity", "source-id": 2, "direction": "asc", "null-order": "nulls-first"}, + {"transform": "bucket[4]", "source-id": 3, "direction": "desc", "null-order": "nulls-last"}, + ], + } + ], + "properties": {"read.split.target.size": 134217728}, + "current-snapshot-id": 3055729675574597004, + "snapshots": [ + { + "snapshot-id": 3051729675574597004, + "timestamp-ms": 1515100955770, + "sequence-number": 0, + "summary": {"operation": "append"}, + "manifest-list": "s3://a/b/1.avro", + }, + { + "snapshot-id": 3055729675574597004, + "parent-snapshot-id": 3051729675574597004, + "timestamp-ms": 1555100955770, + "sequence-number": 1, + "summary": {"operation": "append"}, + "manifest-list": "s3://a/b/2.avro", + "schema-id": 1, + }, + ], + "snapshot-log": [ + {"snapshot-id": 3051729675574597004, "timestamp-ms": 1515100955770}, + {"snapshot-id": 3055729675574597004, "timestamp-ms": 1555100955770}, + ], + "metadata-log": [], + } + + @pytest.fixture def catalog() -> InMemoryCatalog: return InMemoryCatalog("test.in.memory.catalog", {"test.key": "test.value"}) diff --git a/tests/table/test_metadata.py b/tests/table/test_metadata.py index 9c7352d511..352e7574f5 100644 --- a/tests/table/test_metadata.py +++ b/tests/table/test_metadata.py @@ -17,6 +17,7 @@ import io import json +from typing import Any, Dict from uuid import UUID import pytest @@ -49,90 +50,28 @@ "current-snapshot-id": -1, "snapshots": [{"snapshot-id": 1925, "timestamp-ms": 1602638573822}], } -EXAMPLE_TABLE_METADATA_V2 = { - "format-version": 2, - "table-uuid": "9c12d441-03fe-4693-9a96-a0705ddf69c1", - "location": "s3://bucket/test/location", - "last-sequence-number": 34, - "last-updated-ms": 1602638573590, - "last-column-id": 3, - "current-schema-id": 1, - "schemas": [ - {"type": "struct", "schema-id": 0, "fields": [{"id": 1, "name": "x", "required": True, "type": "long"}]}, - { - "type": "struct", - "schema-id": 1, - "identifier-field-ids": [1, 2], - "fields": [ - {"id": 1, "name": "x", "required": True, "type": "long"}, - {"id": 2, "name": "y", "required": True, "type": "long", "doc": "comment"}, - {"id": 3, "name": "z", "required": True, "type": "long"}, - ], - }, - ], - "default-spec-id": 0, - "partition-specs": [{"spec-id": 0, "fields": [{"name": "x", "transform": "identity", "source-id": 1, "field-id": 1000}]}], - "last-partition-id": 1000, - "default-sort-order-id": 3, - "sort-orders": [ - { - "order-id": 3, - "fields": [ - {"transform": "identity", "source-id": 2, "direction": "asc", "null-order": "nulls-first"}, - {"transform": "bucket[4]", "source-id": 3, "direction": "desc", "null-order": "nulls-last"}, - ], - } - ], - "properties": {"read.split.target.size": 134217728}, - "current-snapshot-id": 3055729675574597004, - "snapshots": [ - { - "snapshot-id": 3051729675574597004, - "timestamp-ms": 1515100955770, - "sequence-number": 0, - "summary": {"operation": "append"}, - "manifest-list": "s3://a/b/1.avro", - }, - { - "snapshot-id": 3055729675574597004, - "parent-snapshot-id": 3051729675574597004, - "timestamp-ms": 1555100955770, - "sequence-number": 1, - "summary": {"operation": "append"}, - "manifest-list": "s3://a/b/2.avro", - "schema-id": 1, - }, - ], - "snapshot-log": [ - {"snapshot-id": 3051729675574597004, "timestamp-ms": 1515100955770}, - {"snapshot-id": 3055729675574597004, "timestamp-ms": 1555100955770}, - ], - "metadata-log": [], -} -@pytest.mark.parametrize( - "metadata", - [ - EXAMPLE_TABLE_METADATA_V1, - EXAMPLE_TABLE_METADATA_V2, - ], -) -def test_from_dict(metadata: dict): +def test_from_dict_v1(): + """Test initialization of a TableMetadata instance from a dictionary""" + TableMetadata.parse_obj(EXAMPLE_TABLE_METADATA_V1) + + +def test_from_dict_v2(example_table_metadata_v2: Dict[str, Any]): """Test initialization of a TableMetadata instance from a dictionary""" - TableMetadata.parse_obj(metadata) + TableMetadata.parse_obj(example_table_metadata_v2) -def test_from_byte_stream(): +def test_from_byte_stream(example_table_metadata_v2: Dict[str, Any]): """Test generating a TableMetadata instance from a file-like byte stream""" - data = bytes(json.dumps(EXAMPLE_TABLE_METADATA_V2), encoding="utf-8") + data = bytes(json.dumps(example_table_metadata_v2), encoding="utf-8") byte_stream = io.BytesIO(data) FromByteStream.table_metadata(byte_stream=byte_stream) -def test_v2_metadata_parsing(): +def test_v2_metadata_parsing(example_table_metadata_v2: Dict[str, Any]): """Test retrieving values from a TableMetadata instance of version 2""" - table_metadata = TableMetadata.parse_obj(EXAMPLE_TABLE_METADATA_V2) + table_metadata = TableMetadata.parse_obj(example_table_metadata_v2) assert table_metadata.format_version == 2 assert table_metadata.table_uuid == UUID("9c12d441-03fe-4693-9a96-a0705ddf69c1") @@ -173,17 +112,17 @@ def test_v1_metadata_parsing_directly(): assert table_metadata.default_sort_order_id == 0 -def test_parsing_correct_types(): - table_metadata = TableMetadataV2(**EXAMPLE_TABLE_METADATA_V2) +def test_parsing_correct_types(example_table_metadata_v2: Dict[str, Any]): + table_metadata = TableMetadataV2(**example_table_metadata_v2) assert isinstance(table_metadata.schemas[0], Schema) assert isinstance(table_metadata.schemas[0].fields[0], NestedField) assert isinstance(table_metadata.schemas[0].fields[0].field_type, LongType) -def test_updating_metadata(): +def test_updating_metadata(example_table_metadata_v2: Dict[str, Any]): """Test creating a new TableMetadata instance that's an updated version of an existing TableMetadata instance""" - table_metadata = TableMetadataV2(**EXAMPLE_TABLE_METADATA_V2) + table_metadata = TableMetadataV2(**example_table_metadata_v2) new_schema = { "type": "struct", @@ -211,8 +150,8 @@ def test_serialize_v1(): assert table_metadata == expected -def test_serialize_v2(): - table_metadata = TableMetadataV2(**EXAMPLE_TABLE_METADATA_V2).json() +def test_serialize_v2(example_table_metadata_v2: Dict[str, Any]): + table_metadata = TableMetadataV2(**example_table_metadata_v2).json() expected = """{"location": "s3://bucket/test/location", "table-uuid": "9c12d441-03fe-4693-9a96-a0705ddf69c1", "last-updated-ms": 1602638573590, "last-column-id": 3, "schemas": [{"type": "struct", "fields": [{"id": 1, "name": "x", "type": "long", "required": true}], "schema-id": 0, "identifier-field-ids": []}, {"type": "struct", "fields": [{"id": 1, "name": "x", "type": "long", "required": true}, {"id": 2, "name": "y", "type": "long", "required": true, "doc": "comment"}, {"id": 3, "name": "z", "type": "long", "required": true}], "schema-id": 1, "identifier-field-ids": [1, 2]}], "current-schema-id": 1, "partition-specs": [{"spec-id": 0, "fields": [{"source-id": 1, "field-id": 1000, "transform": "identity", "name": "x"}]}], "default-spec-id": 0, "last-partition-id": 1000, "properties": {"read.split.target.size": "134217728"}, "current-snapshot-id": 3055729675574597004, "snapshots": [{"snapshot-id": 3051729675574597004, "sequence-number": 0, "timestamp-ms": 1515100955770, "manifest-list": "s3://a/b/1.avro", "summary": {"operation": "append"}}, {"snapshot-id": 3055729675574597004, "parent-snapshot-id": 3051729675574597004, "sequence-number": 1, "timestamp-ms": 1555100955770, "manifest-list": "s3://a/b/2.avro", "summary": {"operation": "append"}, "schema-id": 1}], "snapshot-log": [{"snapshot-id": 3051729675574597004, "timestamp-ms": 1515100955770}, {"snapshot-id": 3055729675574597004, "timestamp-ms": 1555100955770}], "metadata-log": [], "sort-orders": [{"order-id": 3, "fields": [{"source-id": 2, "transform": "identity", "direction": "asc", "null-order": "nulls-first"}, {"source-id": 3, "transform": "bucket[4]", "direction": "desc", "null-order": "nulls-last"}]}], "default-sort-order-id": 3, "refs": {"main": {"snapshot-id": 3055729675574597004, "type": "branch"}}, "format-version": 2, "last-sequence-number": 34}""" assert table_metadata == expected @@ -523,8 +462,8 @@ def test_v1_write_metadata_for_v2(): assert "partition-spec" not in metadata_v2 -def test_v2_ref_creation(): - table_metadata = TableMetadataV2(**EXAMPLE_TABLE_METADATA_V2) +def test_v2_ref_creation(example_table_metadata_v2: Dict[str, Any]): + table_metadata = TableMetadataV2(**example_table_metadata_v2) assert table_metadata.refs == {"main": SnapshotRef(snapshot_id=3055729675574597004, snapshot_ref_type=SnapshotRefType.BRANCH)} diff --git a/tests/table/test_sorting.py b/tests/table/test_sorting.py index 300b26f1ce..e5012dead2 100644 --- a/tests/table/test_sorting.py +++ b/tests/table/test_sorting.py @@ -16,6 +16,8 @@ # under the License. # pylint:disable=redefined-outer-name,eval-used +from typing import Any, Dict + import pytest from pyiceberg.table.metadata import TableMetadata @@ -27,7 +29,6 @@ SortOrder, ) from pyiceberg.transforms import BucketTransform, IdentityTransform, VoidTransform -from tests.table.test_metadata import EXAMPLE_TABLE_METADATA_V2 @pytest.fixture @@ -55,8 +56,8 @@ def test_deserialize_sort_order(sort_order: SortOrder): assert SortOrder.parse_raw(payload) == sort_order -def test_sorting_schema(): - table_metadata = TableMetadata.parse_obj(EXAMPLE_TABLE_METADATA_V2) +def test_sorting_schema(example_table_metadata_v2: Dict[str, Any]): + table_metadata = TableMetadata.parse_obj(example_table_metadata_v2) assert table_metadata.sort_orders == [ SortOrder( From 76db3387c546d9f787ce8d38e5ac30d5b8d571bd Mon Sep 17 00:00:00 2001 From: Fokko Driesprong Date: Mon, 15 Aug 2022 23:23:53 +0200 Subject: [PATCH 167/642] Python: Add support for hierarchical namespaces (#5467) --- pyiceberg/catalog/__init__.py | 5 ++++- pyiceberg/catalog/hive.py | 6 +++++- pyiceberg/catalog/rest.py | 15 ++++++++++++--- tests/catalog/test_base.py | 2 +- tests/catalog/test_hive.py | 7 +++++++ tests/catalog/test_rest.py | 11 +++++++++++ 6 files changed, 40 insertions(+), 6 deletions(-) diff --git a/pyiceberg/catalog/__init__.py b/pyiceberg/catalog/__init__.py index 53c8b0b3fa..ad7db96723 100644 --- a/pyiceberg/catalog/__init__.py +++ b/pyiceberg/catalog/__init__.py @@ -182,9 +182,12 @@ def list_tables(self, namespace: str | Identifier) -> list[Identifier]: """ @abstractmethod - def list_namespaces(self) -> list[Identifier]: + def list_namespaces(self, namespace: str | Identifier = ()) -> list[Identifier]: """List namespaces from the given namespace. If not given, list top-level namespaces from the catalog. + Args: + namespace (str | Identifier): Namespace identifier to search. + Returns: List[Identifier]: a List of namespace identifiers diff --git a/pyiceberg/catalog/hive.py b/pyiceberg/catalog/hive.py index 63ba43956a..c2d8b6b84f 100644 --- a/pyiceberg/catalog/hive.py +++ b/pyiceberg/catalog/hive.py @@ -474,12 +474,16 @@ def list_tables(self, namespace: Union[str, Identifier]) -> List[Identifier]: with self._client as open_client: return [(database_name, table_name) for table_name in open_client.get_all_tables(db_name=database_name)] - def list_namespaces(self) -> List[Identifier]: + def list_namespaces(self, namespace: Union[str, Identifier] = ()) -> List[Identifier]: """List namespaces from the given namespace. If not given, list top-level namespaces from the catalog. Returns: List[Identifier]: a List of namespace identifiers """ + # Hive does not support hierarchical namespaces, therefore return an empty list + if namespace: + return [] + with self._client as open_client: return list(map(self.identifier_to_tuple, open_client.get_all_databases())) diff --git a/pyiceberg/catalog/rest.py b/pyiceberg/catalog/rest.py index 46057158ad..0457bca0e8 100644 --- a/pyiceberg/catalog/rest.py +++ b/pyiceberg/catalog/rest.py @@ -422,15 +422,24 @@ def drop_namespace(self, namespace: Union[str, Identifier]) -> None: except HTTPError as exc: self._handle_non_200_response(exc, {404: NoSuchNamespaceError}) - def list_namespaces(self) -> List[Identifier]: - response = requests.get(self.url(Endpoints.list_namespaces), headers=self.headers) + def list_namespaces(self, namespace: Union[str, Identifier] = ()) -> List[Identifier]: + namespace_tuple = self.identifier_to_tuple(namespace) + response = requests.get( + self.url( + f"{Endpoints.list_namespaces}?parent={NAMESPACE_SEPARATOR.join(namespace_tuple)}" + if namespace_tuple + else Endpoints.list_namespaces + ), + headers=self.headers, + ) response.raise_for_status() namespaces = ListNamespaceResponse(**response.json()) try: response.raise_for_status() except HTTPError as exc: self._handle_non_200_response(exc, {}) - return namespaces.namespaces + + return [namespace_tuple + child_namespace for child_namespace in namespaces.namespaces] def load_namespace_properties(self, namespace: Union[str, Identifier]) -> Properties: namespace_tuple = self._check_valid_namespace_identifier(namespace) diff --git a/tests/catalog/test_base.py b/tests/catalog/test_base.py index 22f0b85f7f..365ba98e0c 100644 --- a/tests/catalog/test_base.py +++ b/tests/catalog/test_base.py @@ -138,7 +138,7 @@ def list_tables(self, namespace: Optional[Union[str, Identifier]] = None) -> Lis return list_tables - def list_namespaces(self) -> List[Identifier]: + def list_namespaces(self, namespace: Union[str, Identifier] = ()) -> List[Identifier]: return list(self.__namespaces.keys()) def load_namespace_properties(self, namespace: Union[str, Identifier]) -> Properties: diff --git a/tests/catalog/test_hive.py b/tests/catalog/test_hive.py index 5227c6b8fe..6a0ba1fa4c 100644 --- a/tests/catalog/test_hive.py +++ b/tests/catalog/test_hive.py @@ -467,6 +467,13 @@ def test_list_namespaces(): catalog._client.__enter__().get_all_databases.assert_called() +def test_list_namespaces_with_parent(): + catalog = HiveCatalog(HIVE_CATALOG_NAME, {}, uri=HIVE_METASTORE_FAKE_URL) + + # Hive does not support hierarchical namespaces + assert catalog.list_namespaces(("accounting",)) == [] + + def test_drop_table(): catalog = HiveCatalog(HIVE_CATALOG_NAME, {}, uri=HIVE_METASTORE_FAKE_URL) diff --git a/tests/catalog/test_rest.py b/tests/catalog/test_rest.py index 121428b2b2..b13022f085 100644 --- a/tests/catalog/test_rest.py +++ b/tests/catalog/test_rest.py @@ -151,6 +151,17 @@ def test_list_namespaces_200(rest_mock: Mocker): ] +def test_list_namespace_with_parent_200(rest_mock: Mocker): + rest_mock.get( + f"{TEST_URI}v1/namespaces?parent=accounting", + json={"namespaces": [["tax"]]}, + status_code=200, + ) + assert RestCatalog("rest", {}, TEST_URI, token=TEST_TOKEN).list_namespaces(("accounting",)) == [ + ("accounting", "tax"), + ] + + def test_create_namespace_200(rest_mock: Mocker): namespace = "leden" rest_mock.post( From fadb3140ea208d864b9353ef467aad830c1e38dc Mon Sep 17 00:00:00 2001 From: Fokko Driesprong Date: Tue, 16 Aug 2022 00:25:40 +0200 Subject: [PATCH 168/642] Python: Add a CLI to interact with catalogs (#5417) --- poetry.lock | 382 +++++----------- pyiceberg/catalog/__init__.py | 17 +- pyiceberg/catalog/hive.py | 4 +- pyiceberg/catalog/rest.py | 29 +- pyiceberg/cli/__init__.py | 16 + pyiceberg/cli/console.py | 404 +++++++++++++++++ pyiceberg/cli/output.py | 163 +++++++ pyiceberg/exceptions.py | 4 + pyiceberg/table/metadata.py | 6 + pyproject.toml | 14 + tests/catalog/test_base.py | 2 +- tests/catalog/test_hive.py | 42 +- tests/catalog/test_rest.py | 60 ++- tests/cli/__init__.py | 16 + tests/cli/test_console.py | 811 ++++++++++++++++++++++++++++++++++ tests/cli/test_output.py | 16 + tests/conftest.py | 123 +++--- 17 files changed, 1693 insertions(+), 416 deletions(-) create mode 100644 pyiceberg/cli/__init__.py create mode 100644 pyiceberg/cli/console.py create mode 100644 pyiceberg/cli/output.py create mode 100644 tests/cli/__init__.py create mode 100644 tests/cli/test_console.py create mode 100644 tests/cli/test_output.py diff --git a/poetry.lock b/poetry.lock index 01370863de..4fa41d6c67 100644 --- a/poetry.lock +++ b/poetry.lock @@ -15,10 +15,10 @@ optional = false python-versions = ">=3.5" [package.extras] -tests_no_zope = ["cloudpickle", "pytest-mypy-plugins", "mypy (>=0.900,!=0.940)", "pytest (>=4.3.0)", "pympler", "hypothesis", "coverage[toml] (>=5.0.2)"] -tests = ["cloudpickle", "zope.interface", "pytest-mypy-plugins", "mypy (>=0.900,!=0.940)", "pytest (>=4.3.0)", "pympler", "hypothesis", "coverage[toml] (>=5.0.2)"] -docs = ["sphinx-notfound-page", "zope.interface", "sphinx", "furo"] -dev = ["cloudpickle", "pre-commit", "sphinx-notfound-page", "sphinx", "furo", "zope.interface", "pytest-mypy-plugins", "mypy (>=0.900,!=0.940)", "pytest (>=4.3.0)", "pympler", "hypothesis", "coverage[toml] (>=5.0.2)"] +dev = ["coverage[toml] (>=5.0.2)", "hypothesis", "pympler", "pytest (>=4.3.0)", "mypy (>=0.900,!=0.940)", "pytest-mypy-plugins", "zope.interface", "furo", "sphinx", "sphinx-notfound-page", "pre-commit", "cloudpickle"] +docs = ["furo", "sphinx", "zope.interface", "sphinx-notfound-page"] +tests = ["coverage[toml] (>=5.0.2)", "hypothesis", "pympler", "pytest (>=4.3.0)", "mypy (>=0.900,!=0.940)", "pytest-mypy-plugins", "zope.interface", "cloudpickle"] +tests_no_zope = ["coverage[toml] (>=5.0.2)", "hypothesis", "pympler", "pytest (>=4.3.0)", "mypy (>=0.900,!=0.940)", "pytest-mypy-plugins", "cloudpickle"] [[package]] name = "certifi" @@ -58,14 +58,36 @@ python-versions = ">=3.6.0" [package.extras] unicode_backport = ["unicodedata2"] +[[package]] +name = "click" +version = "8.1.3" +description = "Composable command line interface toolkit" +category = "main" +optional = false +python-versions = ">=3.7" + +[package.dependencies] +colorama = {version = "*", markers = "platform_system == \"Windows\""} + [[package]] name = "colorama" version = "0.4.5" description = "Cross-platform colored terminal text." -category = "dev" +category = "main" optional = false python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*" +[[package]] +name = "commonmark" +version = "0.9.1" +description = "Python parser for the CommonMark Markdown spec" +category = "main" +optional = false +python-versions = "*" + +[package.extras] +test = ["hypothesis (==3.55.3)", "flake8 (==3.7.8)"] + [[package]] name = "coverage" version = "6.4.3" @@ -98,7 +120,7 @@ python-versions = ">=3.7" [[package]] name = "fastavro" -version = "1.5.4" +version = "1.6.0" description = "Fast read/write of AVRO files" category = "dev" optional = false @@ -112,19 +134,19 @@ zstandard = ["zstandard"] [[package]] name = "filelock" -version = "3.7.1" +version = "3.8.0" description = "A platform independent file lock." category = "dev" optional = false python-versions = ">=3.7" [package.extras] -testing = ["pytest-timeout (>=1.4.2)", "pytest-cov", "pytest (>=4)", "coverage (>=4)", "covdefaults (>=1.2.0)"] -docs = ["sphinx-autodoc-typehints (>=1.12)", "sphinx (>=4.1)", "furo (>=2021.8.17b43)"] +testing = ["pytest-timeout (>=2.1)", "pytest-cov (>=3)", "pytest (>=7.1.2)", "coverage (>=6.4.2)", "covdefaults (>=2.2)"] +docs = ["sphinx-autodoc-typehints (>=1.19.1)", "sphinx (>=5.1.1)", "furo (>=2022.6.21)"] [[package]] name = "identify" -version = "2.5.2" +version = "2.5.3" description = "File identification library for Python" category = "dev" optional = false @@ -153,9 +175,9 @@ python-versions = ">=3.7" zipp = ">=0.5" [package.extras] -testing = ["importlib-resources (>=1.3)", "pytest-mypy (>=0.9.1)", "pytest-black (>=0.3.7)", "pytest-perf (>=0.9.2)", "flufl.flake8", "pyfakefs", "packaging", "pytest-enabler (>=1.3)", "pytest-cov", "pytest-flake8", "pytest-checkdocs (>=2.4)", "pytest (>=6)"] +docs = ["sphinx", "jaraco.packaging (>=9)", "rst.linker (>=1.9)"] perf = ["ipython"] -docs = ["rst.linker (>=1.9)", "jaraco.packaging (>=9)", "sphinx"] +testing = ["pytest (>=6)", "pytest-checkdocs (>=2.4)", "pytest-flake8", "pytest-cov", "pytest-enabler (>=1.3)", "packaging", "pyfakefs", "flufl.flake8", "pytest-perf (>=0.9.2)", "pytest-black (>=0.3.7)", "pytest-mypy (>=0.9.1)", "importlib-resources (>=1.3)"] [[package]] name = "iniconfig" @@ -183,7 +205,7 @@ python-versions = ">=2.7,!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,!=3.4.*,!=3.5.*,!=3.6.* [[package]] name = "numpy" -version = "1.23.1" +version = "1.23.2" description = "NumPy is the fundamental package for array computing with Python." category = "main" optional = true @@ -220,8 +242,8 @@ optional = false python-versions = ">=3.7" [package.extras] -test = ["pytest (>=6)", "pytest-mock (>=3.6)", "pytest-cov (>=2.7)", "appdirs (==1.4.4)"] -docs = ["sphinx (>=4)", "sphinx-autodoc-typehints (>=1.12)", "proselint (>=0.10.2)", "furo (>=2021.7.5b38)"] +docs = ["furo (>=2021.7.5b38)", "proselint (>=0.10.2)", "sphinx-autodoc-typehints (>=1.12)", "sphinx (>=4)"] +test = ["appdirs (==1.4.4)", "pytest-cov (>=2.7)", "pytest-mock (>=3.6)", "pytest (>=6)"] [[package]] name = "pluggy" @@ -232,8 +254,8 @@ optional = false python-versions = ">=3.6" [package.extras] -testing = ["pytest-benchmark", "pytest"] -dev = ["tox", "pre-commit"] +dev = ["pre-commit", "tox"] +testing = ["pytest", "pytest-benchmark"] [[package]] name = "pre-commit" @@ -293,6 +315,17 @@ typing-extensions = ">=3.7.4.3" email = ["email-validator (>=1.0.3)"] dotenv = ["python-dotenv (>=0.10.4)"] +[[package]] +name = "pygments" +version = "2.13.0" +description = "Pygments is a syntax highlighting package written in Python." +category = "main" +optional = false +python-versions = ">=3.6" + +[package.extras] +plugins = ["importlib-metadata"] + [[package]] name = "pyparsing" version = "3.0.9" @@ -302,7 +335,7 @@ optional = false python-versions = ">=3.6.8" [package.extras] -diagrams = ["jinja2", "railroad-diagrams"] +diagrams = ["railroad-diagrams", "jinja2"] [[package]] name = "pytest" @@ -323,7 +356,7 @@ py = ">=1.8.2" tomli = ">=1.0.0" [package.extras] -testing = ["xmlschema", "requests", "pygments (>=2.7.2)", "nose", "mock", "hypothesis (>=3.56)", "argcomplete"] +testing = ["argcomplete", "hypothesis (>=3.56)", "mock", "nose", "pygments (>=2.7.2)", "requests", "xmlschema"] [[package]] name = "pytest-checkdocs" @@ -339,8 +372,8 @@ importlib-metadata = ">=4" pep517 = "*" [package.extras] -testing = ["pytest-mypy", "pytest-black (>=0.3.7)", "types-docutils", "pytest-enabler (>=1.0.1)", "pytest-cov", "pytest-flake8", "pytest-checkdocs (>=2.4)", "pytest (>=4.6)"] -docs = ["rst.linker (>=1.9)", "jaraco.packaging (>=8.2)", "sphinx"] +docs = ["sphinx", "jaraco.packaging (>=8.2)", "rst.linker (>=1.9)"] +testing = ["pytest (>=4.6)", "pytest-checkdocs (>=2.4)", "pytest-flake8", "pytest-cov", "pytest-enabler (>=1.0.1)", "types-docutils", "pytest-black (>=0.3.7)", "pytest-mypy"] [[package]] name = "python-snappy" @@ -373,8 +406,8 @@ idna = ">=2.5,<4" urllib3 = ">=1.21.1,<1.27" [package.extras] -use_chardet_on_py3 = ["chardet (>=3.0.2,<6)"] socks = ["PySocks (>=1.5.6,!=1.5.7)"] +use_chardet_on_py3 = ["chardet (>=3.0.2,<6)"] [[package]] name = "requests-mock" @@ -389,8 +422,24 @@ requests = ">=2.3,<3" six = "*" [package.extras] -test = ["testtools", "testrepository (>=0.0.18)", "sphinx", "pytest", "purl", "mock", "fixtures"] fixture = ["fixtures"] +test = ["fixtures", "mock", "purl", "pytest", "sphinx", "testrepository (>=0.0.18)", "testtools"] + +[[package]] +name = "rich" +version = "12.5.1" +description = "Render rich text, tables, progress bars, syntax highlighting, markdown and more to the terminal" +category = "main" +optional = false +python-versions = ">=3.6.3,<4.0.0" + +[package.dependencies] +commonmark = ">=0.9.0,<0.10.0" +pygments = ">=2.6.0,<3.0.0" +typing-extensions = {version = ">=4.0.0,<5.0", markers = "python_version < \"3.9\""} + +[package.extras] +jupyter = ["ipywidgets (>=7.5.1,<8.0.0)"] [[package]] name = "six" @@ -449,26 +498,26 @@ optional = false python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*, !=3.5.*, <4" [package.extras] +brotli = ["brotlicffi (>=0.8.0)", "brotli (>=1.0.9)", "brotlipy (>=0.6.0)"] +secure = ["pyOpenSSL (>=0.14)", "cryptography (>=1.3.4)", "idna (>=2.0.0)", "certifi", "ipaddress"] socks = ["PySocks (>=1.5.6,!=1.5.7,<2.0)"] -secure = ["ipaddress", "certifi", "idna (>=2.0.0)", "cryptography (>=1.3.4)", "pyOpenSSL (>=0.14)"] -brotli = ["brotlipy (>=0.6.0)", "brotli (>=1.0.9)", "brotlicffi (>=0.8.0)"] [[package]] name = "virtualenv" -version = "20.16.2" +version = "20.16.3" description = "Virtual Python Environment builder" category = "dev" optional = false python-versions = ">=3.6" [package.dependencies] -distlib = ">=0.3.1,<1" -filelock = ">=3.2,<4" -platformdirs = ">=2,<3" +distlib = ">=0.3.5,<1" +filelock = ">=3.4.1,<4" +platformdirs = ">=2.4,<3" [package.extras] -testing = ["pytest-timeout (>=1)", "pytest-randomly (>=1)", "pytest-mock (>=2)", "pytest-freezegun (>=0.4.1)", "pytest-env (>=0.6.2)", "pytest (>=4)", "packaging (>=20.0)", "flaky (>=3)", "coverage-enable-subprocess (>=1)", "coverage (>=4)"] -docs = ["towncrier (>=21.3)", "sphinx-rtd-theme (>=0.4.3)", "sphinx-argparse (>=0.2.5)", "sphinx (>=3)", "proselint (>=0.10.2)"] +docs = ["proselint (>=0.13)", "sphinx (>=5.1.1)", "sphinx-argparse (>=0.3.1)", "sphinx-rtd-theme (>=1)", "towncrier (>=21.9)"] +testing = ["coverage (>=6.2)", "coverage-enable-subprocess (>=1)", "flaky (>=3.7)", "packaging (>=21.3)", "pytest (>=7.0.1)", "pytest-env (>=0.6.2)", "pytest-freezegun (>=0.4.2)", "pytest-mock (>=3.6.1)", "pytest-randomly (>=3.10.3)", "pytest-timeout (>=2.1)"] [[package]] name = "zipp" @@ -479,8 +528,8 @@ optional = false python-versions = ">=3.7" [package.extras] -testing = ["pytest-mypy (>=0.9.1)", "pytest-black (>=0.3.7)", "func-timeout", "jaraco.itertools", "pytest-enabler (>=1.3)", "pytest-cov", "pytest-flake8", "pytest-checkdocs (>=2.4)", "pytest (>=6)"] -docs = ["jaraco.tidelift (>=1.4)", "rst.linker (>=1.9)", "jaraco.packaging (>=9)", "sphinx"] +docs = ["sphinx", "jaraco.packaging (>=9)", "rst.linker (>=1.9)", "jaraco.tidelift (>=1.4)"] +testing = ["pytest (>=6)", "pytest-checkdocs (>=2.4)", "pytest-flake8", "pytest-cov", "pytest-enabler (>=1.3)", "jaraco.itertools", "func-timeout", "pytest-black (>=0.3.7)", "pytest-mypy (>=0.9.1)"] [[package]] name = "zstandard" @@ -505,16 +554,11 @@ snappy = ["python-snappy"] [metadata] lock-version = "1.1" python-versions = "^3.8" -content-hash = "7573b12450ca9f900e286cf215db1d582b0ba4186045af0a088b7d7673caa73c" +content-hash = "b104866552b491be8fdfb214b2ed3d51a65b7bd6a2ecace8eedec95a633772cb" [metadata.files] -atomicwrites = [ - {file = "atomicwrites-1.4.1.tar.gz", hash = "sha256:81b2c9071a49367a7f770170e5eec8cb66567cfbbc8c73d20ce5ca4a8d71cf11"}, -] -attrs = [ - {file = "attrs-22.1.0-py2.py3-none-any.whl", hash = "sha256:86efa402f67bf2df34f51a335487cf46b1ec130d02b8d39fd248abfd30da551c"}, - {file = "attrs-22.1.0.tar.gz", hash = "sha256:29adc2665447e5191d0e7c568fde78b21f9672d344281d0c6e1ab085429b22b6"}, -] +atomicwrites = [] +attrs = [] certifi = [ {file = "certifi-2022.6.15-py3-none-any.whl", hash = "sha256:fe86415d55e84719d75f8b69414f6438ac3547d2078ab91b67e779ef69378412"}, {file = "certifi-2022.6.15.tar.gz", hash = "sha256:84c85a9078b11105f04f3036a9482ae10e4621616db313fe045dd24743a0820d"}, @@ -593,88 +637,21 @@ charset-normalizer = [ {file = "charset-normalizer-2.1.0.tar.gz", hash = "sha256:575e708016ff3a5e3681541cb9d79312c416835686d054a23accb873b254f413"}, {file = "charset_normalizer-2.1.0-py3-none-any.whl", hash = "sha256:5189b6f22b01957427f35b6a08d9a0bc45b46d3788ef5a92e978433c7a35f8a5"}, ] +click = [] colorama = [ {file = "colorama-0.4.5-py2.py3-none-any.whl", hash = "sha256:854bf444933e37f5824ae7bfc1e98d5bce2ebe4160d46b5edf346a89358e99da"}, {file = "colorama-0.4.5.tar.gz", hash = "sha256:e6c6b4334fc50988a639d9b98aa429a0b57da6e17b9a44f0451f930b6967b7a4"}, ] -coverage = [ - {file = "coverage-6.4.3-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:f50d3a822947572496ea922ee7825becd8e3ae6fbd2400cd8236b7d64b17f285"}, - {file = "coverage-6.4.3-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:d5191d53afbe5b6059895fa7f58223d3751c42b8101fb3ce767e1a0b1a1d8f87"}, - {file = "coverage-6.4.3-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:04010af3c06ce2bfeb3b1e4e05d136f88d88c25f76cd4faff5d1fd84d11581ea"}, - {file = "coverage-6.4.3-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:6630d8d943644ea62132789940ca97d05fac83f73186eaf0930ffa715fbdab6b"}, - {file = "coverage-6.4.3-cp310-cp310-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:05de0762c1caed4a162b3e305f36cf20a548ff4da0be6766ad5c870704be3660"}, - {file = "coverage-6.4.3-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:0e3a41aad5919613483aad9ebd53336905cab1bd6788afd3995c2a972d89d795"}, - {file = "coverage-6.4.3-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:a2738ba1ee544d6f294278cfb6de2dc1f9a737a780469b5366e662a218f806c3"}, - {file = "coverage-6.4.3-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:a0d2df4227f645a879010461df2cea6b7e3fb5a97d7eafa210f7fb60345af9e8"}, - {file = "coverage-6.4.3-cp310-cp310-win32.whl", hash = "sha256:73a10939dc345460ca0655356a470dd3de9759919186a82383c87b6eb315faf2"}, - {file = "coverage-6.4.3-cp310-cp310-win_amd64.whl", hash = "sha256:53c8edd3b83a4ddba3d8c506f1359401e7770b30f2188f15c17a338adf5a14db"}, - {file = "coverage-6.4.3-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:f1eda5cae434282712e40b42aaf590b773382afc3642786ac3ed39053973f61f"}, - {file = "coverage-6.4.3-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:59fc88bc13e30f25167e807b8cad3c41b7218ef4473a20c86fd98a7968733083"}, - {file = "coverage-6.4.3-cp37-cp37m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:d75314b00825d70e1e34b07396e23f47ed1d4feedc0122748f9f6bd31a544840"}, - {file = "coverage-6.4.3-cp37-cp37m-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:52f8b9fcf3c5e427d51bbab1fb92b575a9a9235d516f175b24712bcd4b5be917"}, - {file = "coverage-6.4.3-cp37-cp37m-musllinux_1_1_aarch64.whl", hash = "sha256:5a559aab40c716de80c7212295d0dc96bc1b6c719371c20dd18c5187c3155518"}, - {file = "coverage-6.4.3-cp37-cp37m-musllinux_1_1_i686.whl", hash = "sha256:306788fd019bb90e9cbb83d3f3c6becad1c048dd432af24f8320cf38ac085684"}, - {file = "coverage-6.4.3-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:920a734fe3d311ca01883b4a19aa386c97b82b69fbc023458899cff0a0d621b9"}, - {file = "coverage-6.4.3-cp37-cp37m-win32.whl", hash = "sha256:ab9ef0187d6c62b09dec83a84a3b94f71f9690784c84fd762fb3cf2d2b44c914"}, - {file = "coverage-6.4.3-cp37-cp37m-win_amd64.whl", hash = "sha256:39ebd8e120cb77a06ee3d5fc26f9732670d1c397d7cd3acf02f6f62693b89b80"}, - {file = "coverage-6.4.3-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:bc698580216050b5f4a34d2cdd2838b429c53314f1c4835fab7338200a8396f2"}, - {file = "coverage-6.4.3-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:877ee5478fd78e100362aed56db47ccc5f23f6e7bb035a8896855f4c3e49bc9b"}, - {file = "coverage-6.4.3-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:555a498999c44f5287cc95500486cd0d4f021af9162982cbe504d4cb388f73b5"}, - {file = "coverage-6.4.3-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:eff095a5aac7011fdb51a2c82a8fae9ec5211577f4b764e1e59cfa27ceeb1b59"}, - {file = "coverage-6.4.3-cp38-cp38-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:5de1e9335e2569974e20df0ce31493d315a830d7987e71a24a2a335a8d8459d3"}, - {file = "coverage-6.4.3-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:7856ea39059d75f822ff0df3a51ea6d76307c897048bdec3aad1377e4e9dca20"}, - {file = "coverage-6.4.3-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:411fdd9f4203afd93b056c0868c8f9e5e16813e765de962f27e4e5798356a052"}, - {file = "coverage-6.4.3-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:cdf7b83f04a313a21afb1f8730fe4dd09577fefc53bbdfececf78b2006f4268e"}, - {file = "coverage-6.4.3-cp38-cp38-win32.whl", hash = "sha256:ab2b1a89d2bc7647622e9eaf06128a5b5451dccf7c242deaa31420b055716481"}, - {file = "coverage-6.4.3-cp38-cp38-win_amd64.whl", hash = "sha256:0e34247274bde982bbc613894d33f9e36358179db2ed231dd101c48dd298e7b0"}, - {file = "coverage-6.4.3-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:b104b6b1827d6a22483c469e3983a204bcf9c6bf7544bf90362c4654ebc2edf3"}, - {file = "coverage-6.4.3-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:adf1a0d272633b21d645dd6e02e3293429c1141c7d65a58e4cbcd592d53b8e01"}, - {file = "coverage-6.4.3-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:ff9832434a9193fbd716fbe05f9276484e18d26cc4cf850853594bb322807ac3"}, - {file = "coverage-6.4.3-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:923f9084d7e1d31b5f74c92396b05b18921ed01ee5350402b561a79dce3ea48d"}, - {file = "coverage-6.4.3-cp39-cp39-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:e4d64304acf79766e650f7acb81d263a3ea6e2d0d04c5172b7189180ff2c023c"}, - {file = "coverage-6.4.3-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:fc294de50941d3da66a09dca06e206297709332050973eca17040278cb0918ff"}, - {file = "coverage-6.4.3-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:a42eaaae772f14a5194f181740a67bfd48e8806394b8c67aa4399e09d0d6b5db"}, - {file = "coverage-6.4.3-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:4822327b35cb032ff16af3bec27f73985448f08e874146b5b101e0e558b613dd"}, - {file = "coverage-6.4.3-cp39-cp39-win32.whl", hash = "sha256:f217850ac0e046ede611312703423767ca032a7b952b5257efac963942c055de"}, - {file = "coverage-6.4.3-cp39-cp39-win_amd64.whl", hash = "sha256:0a84376e4fd13cebce2c0ef8c2f037929c8307fb94af1e5dbe50272a1c651b5d"}, - {file = "coverage-6.4.3-pp36.pp37.pp38-none-any.whl", hash = "sha256:068d6f2a893af838291b8809c876973d885543411ea460f3e6886ac0ee941732"}, - {file = "coverage-6.4.3.tar.gz", hash = "sha256:ec2ae1f398e5aca655b7084392d23e80efb31f7a660d2eecf569fb9f79b3fb94"}, -] -distlib = [ - {file = "distlib-0.3.5-py2.py3-none-any.whl", hash = "sha256:b710088c59f06338ca514800ad795a132da19fda270e3ce4affc74abf955a26c"}, - {file = "distlib-0.3.5.tar.gz", hash = "sha256:a7f75737c70be3b25e2bee06288cec4e4c221de18455b2dd037fe2a795cab2fe"}, -] +commonmark = [] +coverage = [] +distlib = [] docutils = [ {file = "docutils-0.19-py3-none-any.whl", hash = "sha256:5e1de4d849fee02c63b040a4a3fd567f4ab104defd8a5511fbbc24a8a017efbc"}, {file = "docutils-0.19.tar.gz", hash = "sha256:33995a6753c30b7f577febfc2c50411fec6aac7f7ffeb7c4cfe5991072dcf9e6"}, ] -fastavro = [ - {file = "fastavro-1.5.4-cp310-cp310-macosx_10_14_x86_64.whl", hash = "sha256:d316cc476b2b24ef06402b8bfa047f8f72a9d6df2de777bb30d9ededda7e3a02"}, - {file = "fastavro-1.5.4-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:8459faec46e34f2dfeb9b70ee8c36e935e626cff8608d675724718987a5f9ce5"}, - {file = "fastavro-1.5.4-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:cd44636d7ff8365a57b88707b747371fffb676c8c1f68c0d423ec36623888668"}, - {file = "fastavro-1.5.4-cp310-cp310-win_amd64.whl", hash = "sha256:2402428b26d3c08a58acfa723833e19fb75077872bcb2475a4c81195cdae6a5d"}, - {file = "fastavro-1.5.4-cp37-cp37m-macosx_10_15_x86_64.whl", hash = "sha256:5afc14398f4191d1a807aa59d2fba5ed869b31343679ec43dbc289db0a8e35c5"}, - {file = "fastavro-1.5.4-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:5217e9713a3ea03205532394fba4d743749155b04b10b12a12fc26d225b89792"}, - {file = "fastavro-1.5.4-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:2e93a5eecb28cc35d670c9c4df70223fa9bcd6d9ca21b38b1b7ae13ece60c7fb"}, - {file = "fastavro-1.5.4-cp37-cp37m-win_amd64.whl", hash = "sha256:1a2f2465efd0e7de557c4034e8d4d88a132750cfa51e1582362a1b3a1a9fa911"}, - {file = "fastavro-1.5.4-cp38-cp38-macosx_10_15_x86_64.whl", hash = "sha256:f7d5bc76c03c692d9acea0e5d5baceec19e1c059b26cb8ae9f4481e842be95a5"}, - {file = "fastavro-1.5.4-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:80fe920229ab1f40eccb1b4918481cdd8a20e5e7dce19308ab38b23732da8a70"}, - {file = "fastavro-1.5.4-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:a3d190aee86ab73caa1aa550eba850be2ca5dd29d814b38720f4e300184e01d5"}, - {file = "fastavro-1.5.4-cp38-cp38-win_amd64.whl", hash = "sha256:b6c30299a49b11f42251cb81c8e15db67750642eac7ba5c194a5ee95c83ebb11"}, - {file = "fastavro-1.5.4-cp39-cp39-macosx_10_15_x86_64.whl", hash = "sha256:1f7685f3a3c38352abab432bad2f9f2229a0e5f5f8548831e887c30f8396f2e9"}, - {file = "fastavro-1.5.4-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:bd021ec850fd30020b7c4fa868466fb7f95450f1f06eac92bd2204cbd8e45fb8"}, - {file = "fastavro-1.5.4-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:06a7b5602dfa032c92f20ca90b8bde88251573773e501bedf5e8b76b9feb14a3"}, - {file = "fastavro-1.5.4-cp39-cp39-win_amd64.whl", hash = "sha256:18250aa2ab0f7a095b1865565cf9976ea4605c201129636e6defe24ec3ef112c"}, - {file = "fastavro-1.5.4.tar.gz", hash = "sha256:d86f72c966713fb699570a18f7960cf4110b069c70681d7538be8d671c9db7c8"}, -] -filelock = [ - {file = "filelock-3.7.1-py3-none-any.whl", hash = "sha256:37def7b658813cda163b56fc564cdc75e86d338246458c4c28ae84cabefa2404"}, - {file = "filelock-3.7.1.tar.gz", hash = "sha256:3a0fd85166ad9dbab54c9aec96737b744106dc5f15c0b09a6744a445299fcf04"}, -] -identify = [ - {file = "identify-2.5.2-py2.py3-none-any.whl", hash = "sha256:feaa9db2dc0ce333b453ce171c0cf1247bbfde2c55fc6bb785022d411a1b78b5"}, - {file = "identify-2.5.2.tar.gz", hash = "sha256:a3d4c096b384d50d5e6dc5bc8b9bc44f1f61cefebd750a7b3e9f939b53fb214d"}, -] +fastavro = [] +filelock = [] +identify = [] idna = [ {file = "idna-3.3-py3-none-any.whl", hash = "sha256:84d9dd047ffa80596e0f246e2eab0b391788b0503584e8945f2368256d2735ff"}, {file = "idna-3.3.tar.gz", hash = "sha256:9d643ff0a55b762d5cdb124b8eaa99c66322e2157b69160bc32796e824360e6d"}, @@ -719,38 +696,12 @@ nodeenv = [ {file = "nodeenv-1.7.0-py2.py3-none-any.whl", hash = "sha256:27083a7b96a25f2f5e1d8cb4b6317ee8aeda3bdd121394e5ac54e498028a042e"}, {file = "nodeenv-1.7.0.tar.gz", hash = "sha256:e0e7f7dfb85fc5394c6fe1e8fa98131a2473e04311a45afb6508f7cf1836fa2b"}, ] -numpy = [ - {file = "numpy-1.23.1-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:b15c3f1ed08df4980e02cc79ee058b788a3d0bef2fb3c9ca90bb8cbd5b8a3a04"}, - {file = "numpy-1.23.1-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:9ce242162015b7e88092dccd0e854548c0926b75c7924a3495e02c6067aba1f5"}, - {file = "numpy-1.23.1-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:e0d7447679ae9a7124385ccf0ea990bb85bb869cef217e2ea6c844b6a6855073"}, - {file = "numpy-1.23.1-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:3119daed207e9410eaf57dcf9591fdc68045f60483d94956bee0bfdcba790953"}, - {file = "numpy-1.23.1-cp310-cp310-win32.whl", hash = "sha256:3ab67966c8d45d55a2bdf40701536af6443763907086c0a6d1232688e27e5447"}, - {file = "numpy-1.23.1-cp310-cp310-win_amd64.whl", hash = "sha256:1865fdf51446839ca3fffaab172461f2b781163f6f395f1aed256b1ddc253622"}, - {file = "numpy-1.23.1-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:aeba539285dcf0a1ba755945865ec61240ede5432df41d6e29fab305f4384db2"}, - {file = "numpy-1.23.1-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:7e8229f3687cdadba2c4faef39204feb51ef7c1a9b669247d49a24f3e2e1617c"}, - {file = "numpy-1.23.1-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:68b69f52e6545af010b76516f5daaef6173e73353e3295c5cb9f96c35d755641"}, - {file = "numpy-1.23.1-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:1408c3527a74a0209c781ac82bde2182b0f0bf54dea6e6a363fe0cc4488a7ce7"}, - {file = "numpy-1.23.1-cp38-cp38-win32.whl", hash = "sha256:47f10ab202fe4d8495ff484b5561c65dd59177949ca07975663f4494f7269e3e"}, - {file = "numpy-1.23.1-cp38-cp38-win_amd64.whl", hash = "sha256:37e5ebebb0eb54c5b4a9b04e6f3018e16b8ef257d26c8945925ba8105008e645"}, - {file = "numpy-1.23.1-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:173f28921b15d341afadf6c3898a34f20a0569e4ad5435297ba262ee8941e77b"}, - {file = "numpy-1.23.1-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:876f60de09734fbcb4e27a97c9a286b51284df1326b1ac5f1bf0ad3678236b22"}, - {file = "numpy-1.23.1-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:35590b9c33c0f1c9732b3231bb6a72d1e4f77872390c47d50a615686ae7ed3fd"}, - {file = "numpy-1.23.1-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:a35c4e64dfca659fe4d0f1421fc0f05b8ed1ca8c46fb73d9e5a7f175f85696bb"}, - {file = "numpy-1.23.1-cp39-cp39-win32.whl", hash = "sha256:c2f91f88230042a130ceb1b496932aa717dcbd665350beb821534c5c7e15881c"}, - {file = "numpy-1.23.1-cp39-cp39-win_amd64.whl", hash = "sha256:37ece2bd095e9781a7156852e43d18044fd0d742934833335599c583618181b9"}, - {file = "numpy-1.23.1-pp38-pypy38_pp73-macosx_10_9_x86_64.whl", hash = "sha256:8002574a6b46ac3b5739a003b5233376aeac5163e5dcd43dd7ad062f3e186129"}, - {file = "numpy-1.23.1-pp38-pypy38_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:5d732d17b8a9061540a10fda5bfeabca5785700ab5469a5e9b93aca5e2d3a5fb"}, - {file = "numpy-1.23.1-pp38-pypy38_pp73-win_amd64.whl", hash = "sha256:55df0f7483b822855af67e38fb3a526e787adf189383b4934305565d71c4b148"}, - {file = "numpy-1.23.1.tar.gz", hash = "sha256:d748ef349bfef2e1194b59da37ed5a29c19ea8d7e6342019921ba2ba4fd8b624"}, -] +numpy = [] packaging = [ {file = "packaging-21.3-py3-none-any.whl", hash = "sha256:ef103e05f519cdc783ae24ea4e2e0f508a9c99b2d4969652eed6a2e1ea5bd522"}, {file = "packaging-21.3.tar.gz", hash = "sha256:dd47c42927d89ab911e606518907cc2d3a1f38bbd026385970643f9c5b8ecfeb"}, ] -pep517 = [ - {file = "pep517-0.13.0-py3-none-any.whl", hash = "sha256:4ba4446d80aed5b5eac6509ade100bff3e7943a8489de249654a5ae9b33ee35b"}, - {file = "pep517-0.13.0.tar.gz", hash = "sha256:ae69927c5c172be1add9203726d4b84cf3ebad1edcd5f71fcdc746e66e829f59"}, -] +pep517 = [] platformdirs = [ {file = "platformdirs-2.5.2-py3-none-any.whl", hash = "sha256:027d8e83a2d7de06bbac4e5ef7e023c02b863d7ea5d079477e722bb41ab25788"}, {file = "platformdirs-2.5.2.tar.gz", hash = "sha256:58c8abb07dcb441e6ee4b11d8df0ac856038f944ab98b7be6b27b2a3c7feef19"}, @@ -759,83 +710,18 @@ pluggy = [ {file = "pluggy-1.0.0-py2.py3-none-any.whl", hash = "sha256:74134bbf457f031a36d68416e1509f34bd5ccc019f0bcc952c7b909d06b37bd3"}, {file = "pluggy-1.0.0.tar.gz", hash = "sha256:4224373bacce55f955a878bf9cfa763c1e360858e330072059e10bad68531159"}, ] -pre-commit = [ - {file = "pre_commit-2.20.0-py2.py3-none-any.whl", hash = "sha256:51a5ba7c480ae8072ecdb6933df22d2f812dc897d5fe848778116129a681aac7"}, - {file = "pre_commit-2.20.0.tar.gz", hash = "sha256:a978dac7bc9ec0bcee55c18a277d553b0f419d259dadb4b9418ff2d00eb43959"}, -] +pre-commit = [] py = [ {file = "py-1.11.0-py2.py3-none-any.whl", hash = "sha256:607c53218732647dff4acdfcd50cb62615cedf612e72d1724fb1a0cc6405b378"}, {file = "py-1.11.0.tar.gz", hash = "sha256:51c75c4126074b472f746a24399ad32f6053d1b34b68d2fa41e558e6f4a98719"}, ] -pyarrow = [ - {file = "pyarrow-9.0.0-cp310-cp310-macosx_10_13_universal2.whl", hash = "sha256:767cafb14278165ad539a2918c14c1b73cf20689747c21375c38e3fe62884902"}, - {file = "pyarrow-9.0.0-cp310-cp310-macosx_10_13_x86_64.whl", hash = "sha256:0238998dc692efcb4e41ae74738d7c1234723271ccf520bd8312dca07d49ef8d"}, - {file = "pyarrow-9.0.0-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:55328348b9139c2b47450d512d716c2248fd58e2f04e2fc23a65e18726666d42"}, - {file = "pyarrow-9.0.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:fc856628acd8d281652c15b6268ec7f27ebcb015abbe99d9baad17f02adc51f1"}, - {file = "pyarrow-9.0.0-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:29eb3e086e2b26202f3a4678316b93cfb15d0e2ba20f3ec12db8fd9cc07cde63"}, - {file = "pyarrow-9.0.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:2e753f8fcf07d8e3a0efa0c8bd51fef5c90281ffd4c5637c08ce42cd0ac297de"}, - {file = "pyarrow-9.0.0-cp310-cp310-win_amd64.whl", hash = "sha256:3eef8a981f45d89de403e81fb83b8119c20824caddf1404274e41a5d66c73806"}, - {file = "pyarrow-9.0.0-cp37-cp37m-macosx_10_13_x86_64.whl", hash = "sha256:7fa56cbd415cef912677270b8e41baad70cde04c6d8a8336eeb2aba85aa93706"}, - {file = "pyarrow-9.0.0-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:f8c46bde1030d704e2796182286d1c56846552c50a39ad5bf5a20c0d8159fc35"}, - {file = "pyarrow-9.0.0-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:8ad430cee28ebc4d6661fc7315747c7a18ae2a74e67498dcb039e1c762a2fb67"}, - {file = "pyarrow-9.0.0-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:81a60bb291a964f63b2717fb1b28f6615ffab7e8585322bfb8a6738e6b321282"}, - {file = "pyarrow-9.0.0-cp37-cp37m-win_amd64.whl", hash = "sha256:9cef618159567d5f62040f2b79b1c7b38e3885f4ffad0ec97cd2d86f88b67cef"}, - {file = "pyarrow-9.0.0-cp38-cp38-macosx_10_13_x86_64.whl", hash = "sha256:5526a3bfb404ff6d31d62ea582cf2466c7378a474a99ee04d1a9b05de5264541"}, - {file = "pyarrow-9.0.0-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:da3e0f319509a5881867effd7024099fb06950a0768dad0d6873668bb88cfaba"}, - {file = "pyarrow-9.0.0-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:2c715eca2092273dcccf6f08437371e04d112f9354245ba2fbe6c801879450b7"}, - {file = "pyarrow-9.0.0-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:f11a645a41ee531c3a5edda45dea07c42267f52571f818d388971d33fc7e2d4a"}, - {file = "pyarrow-9.0.0-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:a5b390bdcfb8c5b900ef543f911cdfec63e88524fafbcc15f83767202a4a2491"}, - {file = "pyarrow-9.0.0-cp38-cp38-win_amd64.whl", hash = "sha256:d9eb04db626fa24fdfb83c00f76679ca0d98728cdbaa0481b6402bf793a290c0"}, - {file = "pyarrow-9.0.0-cp39-cp39-macosx_10_13_universal2.whl", hash = "sha256:4eebdab05afa23d5d5274b24c1cbeb1ba017d67c280f7d39fd8a8f18cbad2ec9"}, - {file = "pyarrow-9.0.0-cp39-cp39-macosx_10_13_x86_64.whl", hash = "sha256:02b820ecd1da02012092c180447de449fc688d0c3f9ff8526ca301cdd60dacd0"}, - {file = "pyarrow-9.0.0-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:92f3977e901db1ef5cba30d6cc1d7942b8d94b910c60f89013e8f7bb86a86eef"}, - {file = "pyarrow-9.0.0-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:f241bd488c2705df930eedfe304ada71191dcf67d6b98ceda0cc934fd2a8388e"}, - {file = "pyarrow-9.0.0-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:1c5a073a930c632058461547e0bc572da1e724b17b6b9eb31a97da13f50cb6e0"}, - {file = "pyarrow-9.0.0-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:f59bcd5217a3ae1e17870792f82b2ff92df9f3862996e2c78e156c13e56ff62e"}, - {file = "pyarrow-9.0.0-cp39-cp39-win_amd64.whl", hash = "sha256:fe2ce795fa1d95e4e940fe5661c3c58aee7181c730f65ac5dd8794a77228de59"}, - {file = "pyarrow-9.0.0.tar.gz", hash = "sha256:7fb02bebc13ab55573d1ae9bb5002a6d20ba767bf8569b52fce5301d42495ab7"}, -] +pyarrow = [] pycparser = [ {file = "pycparser-2.21-py2.py3-none-any.whl", hash = "sha256:8ee45429555515e1f6b185e78100aea234072576aa43ab53aefcae078162fca9"}, {file = "pycparser-2.21.tar.gz", hash = "sha256:e644fdec12f7872f86c58ff790da456218b10f863970249516d60a5eaca77206"}, ] -pydantic = [ - {file = "pydantic-1.9.2-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:9c9e04a6cdb7a363d7cb3ccf0efea51e0abb48e180c0d31dca8d247967d85c6e"}, - {file = "pydantic-1.9.2-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:fafe841be1103f340a24977f61dee76172e4ae5f647ab9e7fd1e1fca51524f08"}, - {file = "pydantic-1.9.2-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:afacf6d2a41ed91fc631bade88b1d319c51ab5418870802cedb590b709c5ae3c"}, - {file = "pydantic-1.9.2-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:3ee0d69b2a5b341fc7927e92cae7ddcfd95e624dfc4870b32a85568bd65e6131"}, - {file = "pydantic-1.9.2-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:ff68fc85355532ea77559ede81f35fff79a6a5543477e168ab3a381887caea76"}, - {file = "pydantic-1.9.2-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:c0f5e142ef8217019e3eef6ae1b6b55f09a7a15972958d44fbd228214cede567"}, - {file = "pydantic-1.9.2-cp310-cp310-win_amd64.whl", hash = "sha256:615661bfc37e82ac677543704437ff737418e4ea04bef9cf11c6d27346606044"}, - {file = "pydantic-1.9.2-cp36-cp36m-macosx_10_9_x86_64.whl", hash = "sha256:328558c9f2eed77bd8fffad3cef39dbbe3edc7044517f4625a769d45d4cf7555"}, - {file = "pydantic-1.9.2-cp36-cp36m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:2bd446bdb7755c3a94e56d7bdfd3ee92396070efa8ef3a34fab9579fe6aa1d84"}, - {file = "pydantic-1.9.2-cp36-cp36m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:e0b214e57623a535936005797567231a12d0da0c29711eb3514bc2b3cd008d0f"}, - {file = "pydantic-1.9.2-cp36-cp36m-musllinux_1_1_i686.whl", hash = "sha256:d8ce3fb0841763a89322ea0432f1f59a2d3feae07a63ea2c958b2315e1ae8adb"}, - {file = "pydantic-1.9.2-cp36-cp36m-musllinux_1_1_x86_64.whl", hash = "sha256:b34ba24f3e2d0b39b43f0ca62008f7ba962cff51efa56e64ee25c4af6eed987b"}, - {file = "pydantic-1.9.2-cp36-cp36m-win_amd64.whl", hash = "sha256:84d76ecc908d917f4684b354a39fd885d69dd0491be175f3465fe4b59811c001"}, - {file = "pydantic-1.9.2-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:4de71c718c9756d679420c69f216776c2e977459f77e8f679a4a961dc7304a56"}, - {file = "pydantic-1.9.2-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:5803ad846cdd1ed0d97eb00292b870c29c1f03732a010e66908ff48a762f20e4"}, - {file = "pydantic-1.9.2-cp37-cp37m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:a8c5360a0297a713b4123608a7909e6869e1b56d0e96eb0d792c27585d40757f"}, - {file = "pydantic-1.9.2-cp37-cp37m-musllinux_1_1_i686.whl", hash = "sha256:cdb4272678db803ddf94caa4f94f8672e9a46bae4a44f167095e4d06fec12979"}, - {file = "pydantic-1.9.2-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:19b5686387ea0d1ea52ecc4cffb71abb21702c5e5b2ac626fd4dbaa0834aa49d"}, - {file = "pydantic-1.9.2-cp37-cp37m-win_amd64.whl", hash = "sha256:32e0b4fb13ad4db4058a7c3c80e2569adbd810c25e6ca3bbd8b2a9cc2cc871d7"}, - {file = "pydantic-1.9.2-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:91089b2e281713f3893cd01d8e576771cd5bfdfbff5d0ed95969f47ef6d676c3"}, - {file = "pydantic-1.9.2-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:e631c70c9280e3129f071635b81207cad85e6c08e253539467e4ead0e5b219aa"}, - {file = "pydantic-1.9.2-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:4b3946f87e5cef3ba2e7bd3a4eb5a20385fe36521d6cc1ebf3c08a6697c6cfb3"}, - {file = "pydantic-1.9.2-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:5565a49effe38d51882cb7bac18bda013cdb34d80ac336428e8908f0b72499b0"}, - {file = "pydantic-1.9.2-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:bd67cb2c2d9602ad159389c29e4ca964b86fa2f35c2faef54c3eb28b4efd36c8"}, - {file = "pydantic-1.9.2-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:4aafd4e55e8ad5bd1b19572ea2df546ccace7945853832bb99422a79c70ce9b8"}, - {file = "pydantic-1.9.2-cp38-cp38-win_amd64.whl", hash = "sha256:d70916235d478404a3fa8c997b003b5f33aeac4686ac1baa767234a0f8ac2326"}, - {file = "pydantic-1.9.2-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:f0ca86b525264daa5f6b192f216a0d1e860b7383e3da1c65a1908f9c02f42801"}, - {file = "pydantic-1.9.2-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:1061c6ee6204f4f5a27133126854948e3b3d51fcc16ead2e5d04378c199b2f44"}, - {file = "pydantic-1.9.2-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:e78578f0c7481c850d1c969aca9a65405887003484d24f6110458fb02cca7747"}, - {file = "pydantic-1.9.2-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:5da164119602212a3fe7e3bc08911a89db4710ae51444b4224c2382fd09ad453"}, - {file = "pydantic-1.9.2-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:7ead3cd020d526f75b4188e0a8d71c0dbbe1b4b6b5dc0ea775a93aca16256aeb"}, - {file = "pydantic-1.9.2-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:7d0f183b305629765910eaad707800d2f47c6ac5bcfb8c6397abdc30b69eeb15"}, - {file = "pydantic-1.9.2-cp39-cp39-win_amd64.whl", hash = "sha256:f1a68f4f65a9ee64b6ccccb5bf7e17db07caebd2730109cb8a95863cfa9c4e55"}, - {file = "pydantic-1.9.2-py3-none-any.whl", hash = "sha256:78a4d6bdfd116a559aeec9a4cfe77dda62acc6233f8b56a716edad2651023e5e"}, - {file = "pydantic-1.9.2.tar.gz", hash = "sha256:8cb0bc509bfb71305d7a59d00163d5f9fc4530f0881ea32c74ff4f74c85f3d3d"}, -] +pydantic = [] +pygments = [] pyparsing = [ {file = "pyparsing-3.0.9-py3-none-any.whl", hash = "sha256:5026bae9a10eeaefb61dab2f09052b9f4307d44aee4eda64b309723d8d206bbc"}, {file = "pyparsing-3.0.9.tar.gz", hash = "sha256:2b020ecf7d21b687f219b71ecad3631f644a47f01403fa1d1036b0c6416d70fb"}, @@ -937,17 +823,13 @@ requests = [ {file = "requests-2.28.1-py3-none-any.whl", hash = "sha256:8fefa2a1a1365bf5520aac41836fbee479da67864514bdb821f31ce07ce65349"}, {file = "requests-2.28.1.tar.gz", hash = "sha256:7c5599b102feddaa661c826c56ab4fee28bfd17f5abca1ebbe3e7f19d7c97983"}, ] -requests-mock = [ - {file = "requests-mock-1.9.3.tar.gz", hash = "sha256:8d72abe54546c1fc9696fa1516672f1031d72a55a1d66c85184f972a24ba0eba"}, - {file = "requests_mock-1.9.3-py2.py3-none-any.whl", hash = "sha256:0a2d38a117c08bb78939ec163522976ad59a6b7fdd82b709e23bb98004a44970"}, -] +requests-mock = [] +rich = [] six = [ {file = "six-1.16.0-py2.py3-none-any.whl", hash = "sha256:8abb2f1d86890a2dfb989f9a77cfcfd3e47c2a354b01111771326f8aa26e0254"}, {file = "six-1.16.0.tar.gz", hash = "sha256:1e61c37477a1626458e36f7b1d82aa5c9b094fa4802892072e49de9c60c4c926"}, ] -thrift = [ - {file = "thrift-0.16.0.tar.gz", hash = "sha256:2b5b6488fcded21f9d312aa23c9ff6a0195d0f6ae26ddbd5ad9e3e25dfc14408"}, -] +thrift = [] toml = [ {file = "toml-0.10.2-py2.py3-none-any.whl", hash = "sha256:806143ae5bfb6a3c6e736a764057db0e6a0e05e338b5630894a5f779cabb4f9b"}, {file = "toml-0.10.2.tar.gz", hash = "sha256:b3bda1d108d5dd99f4a20d24d9c348e91c4db7ab1b749200bded2f839ccbe68f"}, @@ -960,61 +842,7 @@ typing-extensions = [ {file = "typing_extensions-4.3.0-py3-none-any.whl", hash = "sha256:25642c956049920a5aa49edcdd6ab1e06d7e5d467fc00e0506c44ac86fbfca02"}, {file = "typing_extensions-4.3.0.tar.gz", hash = "sha256:e6d2677a32f47fc7eb2795db1dd15c1f34eff616bcaf2cfb5e997f854fa1c4a6"}, ] -urllib3 = [ - {file = "urllib3-1.26.11-py2.py3-none-any.whl", hash = "sha256:c33ccba33c819596124764c23a97d25f32b28433ba0dedeb77d873a38722c9bc"}, - {file = "urllib3-1.26.11.tar.gz", hash = "sha256:ea6e8fb210b19d950fab93b60c9009226c63a28808bc8386e05301e25883ac0a"}, -] -virtualenv = [ - {file = "virtualenv-20.16.2-py2.py3-none-any.whl", hash = "sha256:635b272a8e2f77cb051946f46c60a54ace3cb5e25568228bd6b57fc70eca9ff3"}, - {file = "virtualenv-20.16.2.tar.gz", hash = "sha256:0ef5be6d07181946891f5abc8047fda8bc2f0b4b9bf222c64e6e8963baee76db"}, -] -zipp = [ - {file = "zipp-3.8.1-py3-none-any.whl", hash = "sha256:47c40d7fe183a6f21403a199b3e4192cca5774656965b0a4988ad2f8feb5f009"}, - {file = "zipp-3.8.1.tar.gz", hash = "sha256:05b45f1ee8f807d0cc928485ca40a07cb491cf092ff587c0df9cb1fd154848d2"}, -] -zstandard = [ - {file = "zstandard-0.18.0-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:ef7e8a200e4c8ac9102ed3c90ed2aa379f6b880f63032200909c1be21951f556"}, - {file = "zstandard-0.18.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:2dc466207016564805e56d28375f4f533b525ff50d6776946980dff5465566ac"}, - {file = "zstandard-0.18.0-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:4a2ee1d4f98447f3e5183ecfce5626f983504a4a0c005fbe92e60fa8e5d547ec"}, - {file = "zstandard-0.18.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:d956e2f03c7200d7e61345e0880c292783ec26618d0d921dcad470cb195bbce2"}, - {file = "zstandard-0.18.0-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_12_i686.manylinux2010_i686.whl", hash = "sha256:ce6f59cba9854fd14da5bfe34217a1501143057313966637b7291d1b0267bd1e"}, - {file = "zstandard-0.18.0-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:a7fa67cba473623848b6e88acf8d799b1906178fd883fb3a1da24561c779593b"}, - {file = "zstandard-0.18.0-cp310-cp310-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:cdb44d7284c8c5dd1b66dfb86dda7f4560fa94bfbbc1d2da749ba44831335e32"}, - {file = "zstandard-0.18.0-cp310-cp310-win32.whl", hash = "sha256:63694a376cde0aa8b1971d06ca28e8f8b5f492779cb6ee1cc46bbc3f019a42a5"}, - {file = "zstandard-0.18.0-cp310-cp310-win_amd64.whl", hash = "sha256:702a8324cd90c74d9c8780d02bf55e79da3193c870c9665ad3a11647e3ad1435"}, - {file = "zstandard-0.18.0-cp36-cp36m-macosx_10_9_x86_64.whl", hash = "sha256:46f679bc5dfd938db4fb058218d9dc4db1336ffaf1ea774ff152ecadabd40805"}, - {file = "zstandard-0.18.0-cp36-cp36m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:dc2a4de9f363b3247d472362a65041fe4c0f59e01a2846b15d13046be866a885"}, - {file = "zstandard-0.18.0-cp36-cp36m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:bd3220d7627fd4d26397211cb3b560ec7cc4a94b75cfce89e847e8ce7fabe32d"}, - {file = "zstandard-0.18.0-cp36-cp36m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_12_i686.manylinux2010_i686.whl", hash = "sha256:39e98cf4773234bd9cebf9f9db730e451dfcfe435e220f8921242afda8321887"}, - {file = "zstandard-0.18.0-cp36-cp36m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:5228e596eb1554598c872a337bbe4e5afe41cd1f8b1b15f2e35b50d061e35244"}, - {file = "zstandard-0.18.0-cp36-cp36m-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:d4a8fd45746a6c31e729f35196e80b8f1e9987c59f5ccb8859d7c6a6fbeb9c63"}, - {file = "zstandard-0.18.0-cp36-cp36m-win32.whl", hash = "sha256:4cbb85f29a990c2fdbf7bc63246567061a362ddca886d7fae6f780267c0a9e67"}, - {file = "zstandard-0.18.0-cp36-cp36m-win_amd64.whl", hash = "sha256:bfa6c8549fa18e6497a738b7033c49f94a8e2e30c5fbe2d14d0b5aa8bbc1695d"}, - {file = "zstandard-0.18.0-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:e02043297c1832f2666cd2204f381bef43b10d56929e13c42c10c732c6e3b4ed"}, - {file = "zstandard-0.18.0-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:7231543d38d2b7e02ef7cc78ef7ffd86419437e1114ff08709fe25a160e24bd6"}, - {file = "zstandard-0.18.0-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:c86befac87445927488f5c8f205d11566f64c11519db223e9d282b945fa60dab"}, - {file = "zstandard-0.18.0-cp37-cp37m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_12_i686.manylinux2010_i686.whl", hash = "sha256:999a4e1768f219826ba3fa2064fab1c86dd72fdd47a42536235478c3bb3ca3e2"}, - {file = "zstandard-0.18.0-cp37-cp37m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:9df59cd1cf3c62075ee2a4da767089d19d874ac3ad42b04a71a167e91b384722"}, - {file = "zstandard-0.18.0-cp37-cp37m-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:1be31e9e3f7607ee0cdd60915410a5968b205d3e7aa83b7fcf3dd76dbbdb39e0"}, - {file = "zstandard-0.18.0-cp37-cp37m-win32.whl", hash = "sha256:490d11b705b8ae9dc845431bacc8dd1cef2408aede176620a5cd0cd411027936"}, - {file = "zstandard-0.18.0-cp37-cp37m-win_amd64.whl", hash = "sha256:266aba27fa9cc5e9091d3d325ebab1fa260f64e83e42516d5e73947c70216a5b"}, - {file = "zstandard-0.18.0-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:8b2260c4e07dd0723eadb586de7718b61acca4083a490dda69c5719d79bc715c"}, - {file = "zstandard-0.18.0-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:3af8c2383d02feb6650e9255491ec7d0824f6e6dd2bbe3e521c469c985f31fb1"}, - {file = "zstandard-0.18.0-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:28723a1d2e4df778573b76b321ebe9f3469ac98988104c2af116dd344802c3f8"}, - {file = "zstandard-0.18.0-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:19cac7108ff2c342317fad6dc97604b47a41f403c8f19d0bfc396dfadc3638b8"}, - {file = "zstandard-0.18.0-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_12_i686.manylinux2010_i686.whl", hash = "sha256:76725d1ee83a8915100a310bbad5d9c1fc6397410259c94033b8318d548d9990"}, - {file = "zstandard-0.18.0-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:d716a7694ce1fa60b20bc10f35c4a22be446ef7f514c8dbc8f858b61976de2fb"}, - {file = "zstandard-0.18.0-cp38-cp38-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:49685bf9a55d1ab34bd8423ea22db836ba43a181ac6b045ac4272093d5cb874e"}, - {file = "zstandard-0.18.0-cp38-cp38-win32.whl", hash = "sha256:1af1268a7dc870eb27515fb8db1f3e6c5a555d2b7bcc476fc3bab8886c7265ab"}, - {file = "zstandard-0.18.0-cp38-cp38-win_amd64.whl", hash = "sha256:1dc2d3809e763055a1a6c1a73f2b677320cc9a5aa1a7c6cfb35aee59bddc42d9"}, - {file = "zstandard-0.18.0-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:eea18c1e7442f2aa9aff1bb84550dbb6a1f711faf6e48e7319de8f2b2e923c2a"}, - {file = "zstandard-0.18.0-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:8677ffc6a6096cccbd892e558471c901fd821aba12b7fbc63833c7346f549224"}, - {file = "zstandard-0.18.0-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:083dc08abf03807af9beeb2b6a91c23ad78add2499f828176a3c7b742c44df02"}, - {file = "zstandard-0.18.0-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:c990063664c08169c84474acecc9251ee035871589025cac47c060ff4ec4bc1a"}, - {file = "zstandard-0.18.0-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_12_i686.manylinux2010_i686.whl", hash = "sha256:533db8a6fac6248b2cb2c935e7b92f994efbdeb72e1ffa0b354432e087bb5a3e"}, - {file = "zstandard-0.18.0-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:dbb3cb8a082d62b8a73af42291569d266b05605e017a3d8a06a0e5c30b5f10f0"}, - {file = "zstandard-0.18.0-cp39-cp39-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:d6c85ca5162049ede475b7ec98e87f9390501d44a3d6776ddd504e872464ec25"}, - {file = "zstandard-0.18.0-cp39-cp39-win32.whl", hash = "sha256:75479e7c2b3eebf402c59fbe57d21bc400cefa145ca356ee053b0a08908c5784"}, - {file = "zstandard-0.18.0-cp39-cp39-win_amd64.whl", hash = "sha256:d85bfabad444812133a92fc6fbe463e1d07581dba72f041f07a360e63808b23c"}, - {file = "zstandard-0.18.0.tar.gz", hash = "sha256:0ac0357a0d985b4ff31a854744040d7b5754385d1f98f7145c30e02c6865cb6f"}, -] +urllib3 = [] +virtualenv = [] +zipp = [] +zstandard = [] diff --git a/pyiceberg/catalog/__init__.py b/pyiceberg/catalog/__init__.py index ad7db96723..871db36265 100644 --- a/pyiceberg/catalog/__init__.py +++ b/pyiceberg/catalog/__init__.py @@ -44,21 +44,16 @@ class Catalog(ABC): or tuple of strings. Attributes: - name (str): Name of the catalog + name (str | None): Name of the catalog properties (Properties): Catalog properties """ - def __init__(self, name: str, properties: Properties): - self._name = name - self._properties = properties + name: str | None + properties: Properties - @property - def name(self) -> str: - return self._name - - @property - def properties(self) -> Properties: - return self._properties + def __init__(self, name: str | None, **properties: str): + self.name = name + self.properties = properties @abstractmethod def create_table( diff --git a/pyiceberg/catalog/hive.py b/pyiceberg/catalog/hive.py index c2d8b6b84f..e218051fd2 100644 --- a/pyiceberg/catalog/hive.py +++ b/pyiceberg/catalog/hive.py @@ -235,8 +235,8 @@ def identifier_to_database_and_table( return tuple_identifier[0], tuple_identifier[1] - def __init__(self, name: str, properties: Properties, uri: str): - super().__init__(name, properties) + def __init__(self, name: str, uri: str, **properties: str): + super().__init__(name, **properties) self._client = _HiveClient(uri) def _convert_hive_into_iceberg(self, table: HiveTable, io: FileIO) -> Table: diff --git a/pyiceberg/catalog/rest.py b/pyiceberg/catalog/rest.py index 0457bca0e8..9d46c211f4 100644 --- a/pyiceberg/catalog/rest.py +++ b/pyiceberg/catalog/rest.py @@ -163,18 +163,18 @@ class OAuthErrorResponse(IcebergBaseModel): class RestCatalog(Catalog): - token: str + token: Optional[str] config: Properties uri: str def __init__( self, - name: str, - properties: Properties, + name: Optional[str], uri: str, - credentials: Optional[str] = None, + credential: Optional[str] = None, token: Optional[str] = None, + **properties: str, ): """Rest Catalog @@ -182,18 +182,20 @@ def __init__( Args: name: Name to identify the catalog - properties: Properties that are passed along to the configuration uri: The base-url of the REST Catalog endpoint - credentials: The credentials for authentication against the client + credential: The credentials for authentication against the client token: The bearer token + properties: Properties that are passed along to the configuration """ self.uri = uri - if credentials: - self.token = self._fetch_access_token(credentials) + if credential: + self.token = self._fetch_access_token(credential) elif token: self.token = token + else: + self.token = None self.config = self._fetch_config(properties) - super().__init__(name, properties) + super().__init__(name, **properties) def _check_valid_namespace_identifier(self, identifier: Union[str, Identifier]) -> Identifier: """The identifier should have at least one element""" @@ -232,8 +234,8 @@ def url(self, endpoint: str, prefixed: bool = True, **kwargs) -> str: return url + endpoint.format(**kwargs) - def _fetch_access_token(self, credentials: str) -> str: - client_id, client_secret = credentials.split(":") + def _fetch_access_token(self, credential: str) -> str: + client_id, client_secret = credential.split(":") data = {GRANT_TYPE: CLIENT_CREDENTIALS, CLIENT_ID: client_id, CLIENT_SECRET: client_secret, SCOPE: CATALOG_SCOPE} url = self.url(Endpoints.get_token, prefixed=False) # Uses application/x-www-form-urlencoded by default @@ -373,9 +375,8 @@ def load_table(self, identifier: Union[str, Identifier]) -> Table: self._handle_non_200_response(exc, {404: NoSuchTableError}) table_response = TableResponse(**response.json()) - return Table( - identifier=(self.name,) + identifier_tuple, + identifier=(self.name,) + identifier_tuple if self.name else identifier_tuple, metadata_location=table_response.metadata_location, metadata=table_response.metadata, ) @@ -458,7 +459,9 @@ def update_namespace_properties( namespace_tuple = self._check_valid_namespace_identifier(namespace) namespace = NAMESPACE_SEPARATOR.join(namespace_tuple) payload = {"removals": list(removals or []), "updates": updates} + print(f"{payload}") response = requests.post(self.url(Endpoints.update_properties, namespace=namespace), json=payload, headers=self.headers) + print(f"{response.json()}") try: response.raise_for_status() except HTTPError as exc: diff --git a/pyiceberg/cli/__init__.py b/pyiceberg/cli/__init__.py new file mode 100644 index 0000000000..13a83393a9 --- /dev/null +++ b/pyiceberg/cli/__init__.py @@ -0,0 +1,16 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. diff --git a/pyiceberg/cli/console.py b/pyiceberg/cli/console.py new file mode 100644 index 0000000000..c95508d2b9 --- /dev/null +++ b/pyiceberg/cli/console.py @@ -0,0 +1,404 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# pylint: disable=broad-except,redefined-builtin,redefined-outer-name +import os +from typing import ( + Dict, + Literal, + Optional, + Tuple, + Type, +) + +import click +from click import Context + +from pyiceberg.catalog import Catalog +from pyiceberg.catalog.hive import HiveCatalog +from pyiceberg.catalog.rest import RestCatalog +from pyiceberg.cli.output import ConsoleOutput, JsonOutput, Output +from pyiceberg.exceptions import NoSuchNamespaceError, NoSuchPropertyException, NoSuchTableError + +SUPPORTED_CATALOGS: Dict[str, Type[Catalog]] = {"thrift": HiveCatalog, "http": RestCatalog} + + +@click.group() +@click.option("--catalog", default=None) +@click.option("--output", type=click.Choice(["text", "json"]), default="text") +@click.option("--uri") +@click.option("--credential") +@click.pass_context +def run(ctx: Context, catalog: Optional[str], output: str, uri: Optional[str], credential: Optional[str]): + uri_env_var = "PYICEBERG_URI" + credential_env_var = "PYICEBERG_CREDENTIAL" + + if not uri: + uri = os.environ.get(uri_env_var) + if not credential: + credential = os.environ.get(credential_env_var) + + ctx.ensure_object(dict) + if output == "text": + ctx.obj["output"] = ConsoleOutput() + else: + ctx.obj["output"] = JsonOutput() + + if not uri: + ctx.obj["output"].exception( + ValueError(f"Missing uri. Please provide using --uri or using environment variable {uri_env_var}") + ) + ctx.exit(1) + + assert uri # for mypy + + for scheme, catalog_type in SUPPORTED_CATALOGS.items(): + if uri.startswith(scheme): + ctx.obj["catalog"] = catalog_type(catalog, uri=uri, credential=credential) # type: ignore + break + + if not isinstance(ctx.obj["catalog"], Catalog): + + ctx.obj["output"].exception( + ValueError("Could not determine catalog type from uri. REST (http/https) and Hive (thrift) is supported") + ) + ctx.exit(1) + + +def _catalog_and_output(ctx: Context) -> Tuple[Catalog, Output]: + """ + Small helper to set the types + """ + return ctx.obj["catalog"], ctx.obj["output"] + + +@run.command() +@click.pass_context +@click.argument("parent", required=False) +def list(ctx: Context, parent: Optional[str]): # pylint: disable=redefined-builtin + """Lists tables or namespaces""" + catalog, output = _catalog_and_output(ctx) + + # still wip, will become more beautiful + # https://github.com/apache/iceberg/pull/5467/ + # has been merged + try: + if parent: + identifiers = catalog.list_tables(parent) + else: + identifiers = catalog.list_namespaces() + output.identifiers(identifiers) + except Exception as exc: + output.exception(exc) + ctx.exit(1) + + +@run.command() +@click.option("--entity", type=click.Choice(["any", "namespace", "table"]), default="any") +@click.argument("identifier") +@click.pass_context +def describe(ctx: Context, entity: Literal["name", "namespace", "table"], identifier: str): + """Describes a namespace xor table""" + catalog, output = _catalog_and_output(ctx) + identifier_tuple = Catalog.identifier_to_tuple(identifier) + + if entity in {"namespace", "any"} and len(identifier_tuple) > 0: + try: + namespace_properties = catalog.load_namespace_properties(identifier_tuple) + output.describe_properties(namespace_properties) + ctx.exit(0) + except NoSuchNamespaceError as exc: + if entity != "any": + output.exception(exc) + ctx.exit(1) + + if entity in {"table", "any"} and len(identifier_tuple) > 1: + try: + catalog_table = catalog.load_table(identifier) + output.describe_table(catalog_table) + ctx.exit(0) + except NoSuchTableError as exc: + output.exception(exc) + ctx.exit(1) + + output.exception(NoSuchTableError(f"Table or namespace does not exist: {identifier}")) + ctx.exit(1) + + +@run.command() +@click.argument("identifier") +@click.pass_context +def schema(ctx: Context, identifier: str): + """Gets the schema of the table""" + catalog, output = _catalog_and_output(ctx) + + try: + metadata = catalog.load_table(identifier).metadata + assert metadata + output.schema(metadata.current_schema()) + except Exception as exc: + output.exception(exc) + ctx.exit(1) + + +@run.command() +@click.argument("identifier") +@click.pass_context +def spec(ctx: Context, identifier: str): + """Returns the partition spec of the table""" + catalog, output = _catalog_and_output(ctx) + try: + metadata = catalog.load_table(identifier).metadata + assert metadata + output.spec(metadata.current_partition_spec()) + except Exception as exc: + output.exception(exc) + ctx.exit(1) + + +@run.command() +@click.argument("identifier") +@click.pass_context +def uuid(ctx: Context, identifier: str): + """Returns the UUID of the table""" + catalog, output = _catalog_and_output(ctx) + try: + metadata = catalog.load_table(identifier).metadata + assert metadata + output.uuid(metadata.table_uuid) + except Exception as exc: + output.exception(exc) + ctx.exit(1) + + +@run.command() +@click.argument("identifier") +@click.pass_context +def location(ctx: Context, identifier: str): + """Returns the location of the table""" + catalog, output = _catalog_and_output(ctx) + try: + metadata = catalog.load_table(identifier).metadata + assert metadata + output.text(metadata.location) + except Exception as exc: + output.exception(exc) + ctx.exit(1) + + +@run.group() +def drop(): + """Operations to drop a namespace or table""" + + +@drop.command() +@click.argument("identifier") +@click.pass_context +def table(ctx: Context, identifier: str): # noqa: F811 + """Drop table""" + catalog, output = _catalog_and_output(ctx) + + try: + catalog.drop_table(identifier) + output.text(f"Dropped table: {identifier}") + except Exception as exc: + output.exception(exc) + ctx.exit(1) + + +@drop.command() +@click.argument("identifier") +@click.pass_context +def namespace(ctx, identifier: str): + """Drop namespace""" + catalog, output = _catalog_and_output(ctx) + + try: + catalog.drop_namespace(identifier) + output.text(f"Dropped namespace: {identifier}") + except Exception as exc: + output.exception(exc) + ctx.exit(1) + + +@run.command() +@click.argument("from_identifier") +@click.argument("to_identifier") +@click.pass_context +def rename(ctx, from_identifier: str, to_identifier: str): + """Renames a table""" + catalog, output = _catalog_and_output(ctx) + + try: + catalog.rename_table(from_identifier, to_identifier) + output.text(f"Renamed table from {from_identifier} to {to_identifier}") + except Exception as exc: + output.exception(exc) + ctx.exit(1) + + +@run.group() +def properties(): + """Properties on tables/namespaces""" + + +@properties.command() +@click.option("--entity", type=click.Choice(["any", "namespace", "table"]), default="any") +@click.argument("identifier") +@click.argument("property_name", required=False) +@click.pass_context +def get(ctx: Context, entity: Literal["name", "namespace", "table"], identifier: str, property_name: str): + """Fetches a property of a namespace or table""" + catalog, output = _catalog_and_output(ctx) + identifier_tuple = Catalog.identifier_to_tuple(identifier) + + try: + namespace_properties = catalog.load_namespace_properties(identifier_tuple) + + if property_name: + if property_value := namespace_properties.get(property_name): + output.text(property_value) + ctx.exit(0) + elif entity != "any": + output.exception(NoSuchPropertyException(f"Could not find property {property_name} on {entity} {identifier}")) + ctx.exit(1) + else: + output.describe_properties(namespace_properties) + ctx.exit(0) + except NoSuchNamespaceError as exc: + if entity != "any": + output.exception(exc) + ctx.exit(1) + + if len(identifier_tuple) > 1: + try: + metadata = catalog.load_table(identifier_tuple).metadata + assert metadata + + if property_name: + if property_value := metadata.properties.get(property_name): + output.text(property_value) + ctx.exit(0) + elif entity != "any": + output.exception(NoSuchPropertyException(f"Could not find property {property_name} on {entity} {identifier}")) + ctx.exit(1) + else: + output.describe_properties(metadata.properties) + ctx.exit(0) + except NoSuchTableError as exc: + if entity != "any": + output.exception(exc) + ctx.exit(1) + + property_err = "" + if property_name: + property_err = f" with property {property_name}" + + output.exception(NoSuchNamespaceError(f"Table or namespace does not exist: {identifier}{property_err}")) + ctx.exit(1) + + +@properties.group() +def set(): + """Removes properties on tables/namespaces""" + + +@set.command() # type: ignore +@click.argument("identifier") +@click.argument("property_name") +@click.argument("property_value") +@click.pass_context +def namespace(ctx: Context, identifier: str, property_name: str, property_value: str): # noqa: F811 + """Sets a property of a namespace or table""" + catalog, output = _catalog_and_output(ctx) + try: + catalog.update_namespace_properties(identifier, updates={property_name: property_value}) + output.text(f"Updated {property_name} on {identifier}") + except NoSuchNamespaceError as exc: + output.exception(exc) + ctx.exit(1) + + +@set.command() # type: ignore +@click.argument("identifier") +@click.argument("property_name") +@click.argument("property_value") +@click.pass_context +def table(ctx: Context, identifier: str, property_name: str, property_value: str): # noqa: F811 + """Sets a property on a table""" + catalog, output = _catalog_and_output(ctx) + identifier_tuple = Catalog.identifier_to_tuple(identifier) + + try: + _ = catalog.load_table(identifier_tuple) + output.text(f"Setting {property_name}={property_value} on {identifier}") + output.exception(NotImplementedError("Writing is WIP")) + ctx.exit(1) + + except NoSuchTableError as exc: + output.exception(exc) + ctx.exit(1) + + output.exception(NoSuchNamespaceError(f"Could not find table/namespace {identifier} with property {property}")) + ctx.exit(1) + + +@properties.group() +def remove(): + """Removes properties on tables/namespaces""" + + +@remove.command() # type: ignore +@click.argument("identifier") +@click.argument("property_name") +@click.pass_context +def namespace(ctx: Context, identifier: str, property_name: str): # noqa: F811 + """Removes a property from a namespace""" + catalog, output = _catalog_and_output(ctx) + try: + result = catalog.update_namespace_properties(identifier, removals={property_name}) + + if result.removed == [property_name]: + output.text(f"Property {property_name} removed from {identifier}") + else: + raise NoSuchPropertyException(f"Property {property_name} does not exists on {identifier}") + except Exception as exc: + output.exception(exc) + ctx.exit(1) + + +@remove.command() # type: ignore +@click.argument("identifier") +@click.argument("property_name") +@click.pass_context +def table(ctx: Context, identifier: str, property_name: str): # noqa: F811 + """Removes a property from a table""" + catalog, output = _catalog_and_output(ctx) + try: + table = catalog.load_table(identifier) + assert table.metadata + if property_name in table.metadata.properties: + # We should think of the process here + # Do we want something similar as in Java: + # https://github.com/apache/iceberg/blob/master/api/src/main/java/org/apache/iceberg/Table.java#L178 + del table.metadata.properties + output.exception(NotImplementedError("Writing is WIP")) + ctx.exit(1) + else: + raise NoSuchPropertyException(f"Property {property_name} does not exists on {identifier}") + except Exception as exc: + output.exception(exc) + ctx.exit(1) diff --git a/pyiceberg/cli/output.py b/pyiceberg/cli/output.py new file mode 100644 index 0000000000..fc7c53cfc0 --- /dev/null +++ b/pyiceberg/cli/output.py @@ -0,0 +1,163 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +import json +from abc import ABC, abstractmethod +from typing import Any, List, Optional +from uuid import UUID + +from rich.console import Console +from rich.table import Table as RichTable +from rich.tree import Tree + +from pyiceberg.schema import Schema +from pyiceberg.table import Table +from pyiceberg.table.partitioning import PartitionSpec +from pyiceberg.typedef import Identifier, Properties + + +class Output(ABC): + """Output interface for exporting""" + + @abstractmethod + def exception(self, ex: Exception): + ... + + @abstractmethod + def identifiers(self, identifiers: List[Identifier]): + ... + + @abstractmethod + def describe_table(self, table): + ... + + @abstractmethod + def describe_properties(self, properties: Properties): + ... + + @abstractmethod + def text(self, response: str): + ... + + @abstractmethod + def schema(self, schema: Schema): + ... + + @abstractmethod + def spec(self, spec: PartitionSpec): + ... + + @abstractmethod + def uuid(self, uuid: Optional[UUID]): + ... + + +class ConsoleOutput(Output): + """Writes to the console""" + + @property + def _table(self) -> RichTable: + return RichTable.grid(padding=(0, 2)) + + def exception(self, ex: Exception): + Console(stderr=True).print(ex) + + def identifiers(self, identifiers: List[Identifier]): + table = self._table + for identifier in identifiers: + table.add_row(".".join(identifier)) + + Console().print(table) + + def describe_table(self, table: Table): + assert table.metadata + metadata = table.metadata + table_properties = self._table + + for key, value in metadata.properties.items(): + table_properties.add_row(key, value) + + schema_tree = Tree("Schema") + for field in metadata.current_schema().fields: + schema_tree.add(str(field)) + + snapshot_tree = Tree("Snapshots") + for snapshot in metadata.snapshots: + snapshot_tree.add(f"Snapshot {snapshot.schema_id}: {snapshot.manifest_list}") + + output_table = self._table + output_table.add_row("Table format version", str(metadata.format_version)) + output_table.add_row("Metadata location", table.metadata_location) + output_table.add_row("Table UUID", str(table.metadata.table_uuid)) + output_table.add_row("Last Updated", str(metadata.last_updated_ms)) + output_table.add_row("Partition spec", str(metadata.current_partition_spec())) + output_table.add_row("Sort order", str(metadata.current_sort_order())) + output_table.add_row("Schema", schema_tree) + output_table.add_row("Snapshots", snapshot_tree) + output_table.add_row("Properties", table_properties) + Console().print(output_table) + + def describe_properties(self, properties: Properties): + output_table = self._table + for k, v in properties.items(): + output_table.add_row(k, v) + Console().print(output_table) + + def text(self, response: str): + Console().print(response) + + def schema(self, schema: Schema): + output_table = self._table + for field in schema.fields: + output_table.add_row(field.name, str(field.field_type)) + Console().print(output_table) + + def spec(self, spec: PartitionSpec): + Console().print(str(spec)) + + def uuid(self, uuid: Optional[UUID]): + Console().print(str(uuid) if uuid else "missing") + + +class JsonOutput(Output): + """Writes json to stdout""" + + def _out(self, d: Any) -> None: + print(json.dumps(d)) + + def exception(self, ex: Exception): + self._out({"type": ex.__class__.__name__, "message": str(ex)}) + + def identifiers(self, identifiers: List[Identifier]): + self._out([".".join(identifier) for identifier in identifiers]) + + def describe_table(self, table: Table): + print(table.json()) + + def describe_properties(self, properties: Properties): + self._out(properties) + + def text(self, response: str): + print(json.dumps(response)) + + def schema(self, schema: Schema): + print(schema.json()) + + def spec(self, spec: PartitionSpec): + print(spec.json()) + + def uuid(self, uuid: Optional[UUID]): + self._out({"uuid": str(uuid) if uuid else "missing"}) diff --git a/pyiceberg/exceptions.py b/pyiceberg/exceptions.py index b5db6a0e1b..ea2d542e90 100644 --- a/pyiceberg/exceptions.py +++ b/pyiceberg/exceptions.py @@ -74,3 +74,7 @@ class AuthorizationExpiredError(RESTError): class OAuthError(RESTError): """Raises when there is an error with the OAuth call""" + + +class NoSuchPropertyException(Exception): + """When a property is missing""" diff --git a/pyiceberg/table/metadata.py b/pyiceberg/table/metadata.py index 33abd4c8df..5f48a44d33 100644 --- a/pyiceberg/table/metadata.py +++ b/pyiceberg/table/metadata.py @@ -86,6 +86,12 @@ class TableMetadataCommonFields(IcebergBaseModel): def current_schema(self) -> Schema: return next(schema for schema in self.schemas if schema.schema_id == self.current_schema_id) + def current_sort_order(self) -> SortOrder: + return next(sort_order for sort_order in self.sort_orders if sort_order.order_id == self.default_sort_order_id) + + def current_partition_spec(self) -> PartitionSpec: + return next(spec for spec in self.partition_specs if spec.spec_id == self.default_spec_id) + @root_validator(pre=True) def cleanup_snapshot_id(cls, data: Dict[str, Any]): if data.get("current-snapshot-id") == -1: diff --git a/pyproject.toml b/pyproject.toml index e70dca7f03..a4785bf9c3 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -44,6 +44,8 @@ packages = [ python = "^3.8" mmh3 = "^3.0.0" requests = "^2.28.1" +click = "^8.1.3" +rich = "^12.5.1" pydantic = "^1.9.2" @@ -63,6 +65,10 @@ fastavro = "^1.5.4" coverage = { version = "^6.4.3", extras = ["toml"] } requests-mock = "^1.9.3" +[tool.poetry.scripts] +pyiceberg = "pyiceberg.cli.console:run" + + [build-system] requires = ["poetry-core>=1.0.0"] build-backend = "poetry.core.masonry.api" @@ -132,5 +138,13 @@ ignore_missing_imports = true module = "requests_mock.*" ignore_missing_imports = true +[[tool.mypy.overrides]] +module = "click.*" +ignore_missing_imports = true + +[[tool.mypy.overrides]] +module = "rich.*" +ignore_missing_imports = true + [tool.coverage.run] source = ['pyiceberg/'] diff --git a/tests/catalog/test_base.py b/tests/catalog/test_base.py index 365ba98e0c..ff84e64a03 100644 --- a/tests/catalog/test_base.py +++ b/tests/catalog/test_base.py @@ -54,7 +54,7 @@ class InMemoryCatalog(Catalog): __namespaces: Dict[Identifier, Properties] def __init__(self, name: str, properties: Properties): - super().__init__(name, properties) + super().__init__(name, **properties) self.__tables = {} self.__namespaces = {} diff --git a/tests/catalog/test_hive.py b/tests/catalog/test_hive.py index 6a0ba1fa4c..32dce64e31 100644 --- a/tests/catalog/test_hive.py +++ b/tests/catalog/test_hive.py @@ -35,7 +35,7 @@ from hive_metastore.ttypes import Table as HiveTable from pyiceberg.catalog import PropertiesUpdateSummary -from pyiceberg.catalog.hive import WAREHOUSE, HiveCatalog, _construct_hive_storage_descriptor +from pyiceberg.catalog.hive import HiveCatalog, _construct_hive_storage_descriptor from pyiceberg.exceptions import ( NamespaceAlreadyExistsError, NamespaceNotEmptyError, @@ -160,7 +160,7 @@ def hive_database(tmp_path_factory) -> HiveDatabase: def test_check_number_of_namespaces(table_schema_simple: Schema): - catalog = HiveCatalog(HIVE_CATALOG_NAME, {}, uri=HIVE_METASTORE_FAKE_URL) + catalog = HiveCatalog(HIVE_CATALOG_NAME, uri=HIVE_METASTORE_FAKE_URL) with pytest.raises(ValueError): catalog.create_table(("default", "namespace", "table"), schema=table_schema_simple) @@ -178,7 +178,7 @@ def test_check_number_of_namespaces(table_schema_simple: Schema): @patch("time.time", MagicMock(return_value=12345)) @patch("uuid.uuid4", MagicMock(return_value="01234567-0123-0123-0123-0123456789ab")) def test_create_table(table_schema_simple: Schema, hive_database: HiveDatabase, hive_table: HiveTable): - catalog = HiveCatalog(HIVE_CATALOG_NAME, {}, uri=HIVE_METASTORE_FAKE_URL) + catalog = HiveCatalog(HIVE_CATALOG_NAME, uri=HIVE_METASTORE_FAKE_URL) catalog._client = MagicMock() catalog._client.__enter__().create_table.return_value = None @@ -291,7 +291,7 @@ def test_create_table(table_schema_simple: Schema, hive_database: HiveDatabase, def test_load_table(hive_table: HiveTable): - catalog = HiveCatalog(HIVE_CATALOG_NAME, {}, uri=HIVE_METASTORE_FAKE_URL) + catalog = HiveCatalog(HIVE_CATALOG_NAME, uri=HIVE_METASTORE_FAKE_URL) catalog._client = MagicMock() catalog._client.__enter__().get_table.return_value = hive_table @@ -384,7 +384,7 @@ def test_load_table(hive_table: HiveTable): def test_rename_table_from_does_not_exists(): - catalog = HiveCatalog(HIVE_CATALOG_NAME, {}, uri=HIVE_METASTORE_FAKE_URL) + catalog = HiveCatalog(HIVE_CATALOG_NAME, uri=HIVE_METASTORE_FAKE_URL) catalog._client = MagicMock() catalog._client.__enter__().alter_table.side_effect = NoSuchObjectException( @@ -398,7 +398,7 @@ def test_rename_table_from_does_not_exists(): def test_rename_table_to_namespace_does_not_exists(): - catalog = HiveCatalog(HIVE_CATALOG_NAME, {}, uri=HIVE_METASTORE_FAKE_URL) + catalog = HiveCatalog(HIVE_CATALOG_NAME, uri=HIVE_METASTORE_FAKE_URL) catalog._client = MagicMock() catalog._client.__enter__().alter_table.side_effect = InvalidOperationException( @@ -412,7 +412,7 @@ def test_rename_table_to_namespace_does_not_exists(): def test_drop_database_does_not_empty(): - catalog = HiveCatalog(HIVE_CATALOG_NAME, {}, uri=HIVE_METASTORE_FAKE_URL) + catalog = HiveCatalog(HIVE_CATALOG_NAME, uri=HIVE_METASTORE_FAKE_URL) catalog._client = MagicMock() catalog._client.__enter__().drop_database.side_effect = InvalidOperationException( @@ -426,7 +426,7 @@ def test_drop_database_does_not_empty(): def test_drop_database_does_not_exists(): - catalog = HiveCatalog(HIVE_CATALOG_NAME, {}, uri=HIVE_METASTORE_FAKE_URL) + catalog = HiveCatalog(HIVE_CATALOG_NAME, uri=HIVE_METASTORE_FAKE_URL) catalog._client = MagicMock() catalog._client.__enter__().drop_database.side_effect = MetaException(message="java.lang.NullPointerException") @@ -438,7 +438,7 @@ def test_drop_database_does_not_exists(): def test_list_tables(): - catalog = HiveCatalog(HIVE_CATALOG_NAME, {}, uri=HIVE_METASTORE_FAKE_URL) + catalog = HiveCatalog(HIVE_CATALOG_NAME, uri=HIVE_METASTORE_FAKE_URL) catalog._client = MagicMock() catalog._client.__enter__().get_all_tables.return_value = ["table1", "table2"] @@ -457,7 +457,7 @@ def test_list_tables(): def test_list_namespaces(): - catalog = HiveCatalog(HIVE_CATALOG_NAME, {}, uri=HIVE_METASTORE_FAKE_URL) + catalog = HiveCatalog(HIVE_CATALOG_NAME, uri=HIVE_METASTORE_FAKE_URL) catalog._client = MagicMock() catalog._client.__enter__().get_all_databases.return_value = ["namespace1", "namespace2"] @@ -475,7 +475,7 @@ def test_list_namespaces_with_parent(): def test_drop_table(): - catalog = HiveCatalog(HIVE_CATALOG_NAME, {}, uri=HIVE_METASTORE_FAKE_URL) + catalog = HiveCatalog(HIVE_CATALOG_NAME, uri=HIVE_METASTORE_FAKE_URL) catalog._client = MagicMock() catalog._client.__enter__().get_all_databases.return_value = ["namespace1", "namespace2"] @@ -486,7 +486,7 @@ def test_drop_table(): def test_drop_table_does_not_exists(): - catalog = HiveCatalog(HIVE_CATALOG_NAME, {}, uri=HIVE_METASTORE_FAKE_URL) + catalog = HiveCatalog(HIVE_CATALOG_NAME, uri=HIVE_METASTORE_FAKE_URL) catalog._client = MagicMock() catalog._client.__enter__().drop_table.side_effect = NoSuchObjectException(message="does_not_exists") @@ -498,14 +498,14 @@ def test_drop_table_does_not_exists(): def test_purge_table(): - catalog = HiveCatalog(HIVE_CATALOG_NAME, {}, uri=HIVE_METASTORE_FAKE_URL) + catalog = HiveCatalog(HIVE_CATALOG_NAME, uri=HIVE_METASTORE_FAKE_URL) with pytest.raises(NotImplementedError): catalog.purge_table(("default", "does_not_exists")) def test_create_database(): - catalog = HiveCatalog(HIVE_CATALOG_NAME, {}, uri=HIVE_METASTORE_FAKE_URL) + catalog = HiveCatalog(HIVE_CATALOG_NAME, uri=HIVE_METASTORE_FAKE_URL) catalog._client = MagicMock() catalog._client.__enter__().create_database.return_value = None @@ -532,7 +532,7 @@ def test_create_database(): def test_create_database_already_exists(): - catalog = HiveCatalog(HIVE_CATALOG_NAME, {}, uri=HIVE_METASTORE_FAKE_URL) + catalog = HiveCatalog(HIVE_CATALOG_NAME, uri=HIVE_METASTORE_FAKE_URL) catalog._client = MagicMock() catalog._client.__enter__().create_database.side_effect = AlreadyExistsException(message="Database default already exists") @@ -544,7 +544,7 @@ def test_create_database_already_exists(): def test_load_namespace_properties(hive_database: HiveDatabase): - catalog = HiveCatalog(HIVE_CATALOG_NAME, {}, uri=HIVE_METASTORE_FAKE_URL) + catalog = HiveCatalog(HIVE_CATALOG_NAME, uri=HIVE_METASTORE_FAKE_URL) catalog._client = MagicMock() catalog._client.__enter__().get_database.return_value = hive_database @@ -555,7 +555,7 @@ def test_load_namespace_properties(hive_database: HiveDatabase): def test_load_namespace_properties_does_not_exists(): - catalog = HiveCatalog(HIVE_CATALOG_NAME, {}, uri=HIVE_METASTORE_FAKE_URL) + catalog = HiveCatalog(HIVE_CATALOG_NAME, uri=HIVE_METASTORE_FAKE_URL) catalog._client = MagicMock() catalog._client.__enter__().get_database.side_effect = NoSuchObjectException(message="does_not_exists") @@ -567,7 +567,7 @@ def test_load_namespace_properties_does_not_exists(): def test_update_namespace_properties(hive_database: HiveDatabase): - catalog = HiveCatalog(HIVE_CATALOG_NAME, {}, uri=HIVE_METASTORE_FAKE_URL) + catalog = HiveCatalog(HIVE_CATALOG_NAME, uri=HIVE_METASTORE_FAKE_URL) catalog._client = MagicMock() catalog._client.__enter__().get_database.return_value = hive_database @@ -598,7 +598,7 @@ def test_update_namespace_properties(hive_database: HiveDatabase): def test_update_namespace_properties_namespace_does_not_exists(): - catalog = HiveCatalog(HIVE_CATALOG_NAME, {}, uri=HIVE_METASTORE_FAKE_URL) + catalog = HiveCatalog(HIVE_CATALOG_NAME, uri=HIVE_METASTORE_FAKE_URL) catalog._client = MagicMock() catalog._client.__enter__().get_database.side_effect = NoSuchObjectException(message="does_not_exists") @@ -610,7 +610,7 @@ def test_update_namespace_properties_namespace_does_not_exists(): def test_update_namespace_properties_overlap(): - catalog = HiveCatalog(HIVE_CATALOG_NAME, {}, uri=HIVE_METASTORE_FAKE_URL) + catalog = HiveCatalog(HIVE_CATALOG_NAME, uri=HIVE_METASTORE_FAKE_URL) with pytest.raises(ValueError) as exc_info: catalog.update_namespace_properties(("table",), removals=set("a"), updates={"a": "b"}) @@ -683,7 +683,7 @@ def test_construct_hive_storage_descriptor_nested(table_schema_nested: Schema): def test_resolve_table_location_warehouse(hive_database: HiveDatabase): - catalog = HiveCatalog(HIVE_CATALOG_NAME, {WAREHOUSE: "/tmp/warehouse/"}, uri=HIVE_METASTORE_FAKE_URL) + catalog = HiveCatalog(HIVE_CATALOG_NAME, warehouse="/tmp/warehouse/", uri=HIVE_METASTORE_FAKE_URL) # Set this one to None, so we'll fall back to the properties hive_database.locationUri = None diff --git a/tests/catalog/test_rest.py b/tests/catalog/test_rest.py index b13022f085..ae45c0ac05 100644 --- a/tests/catalog/test_rest.py +++ b/tests/catalog/test_rest.py @@ -74,7 +74,7 @@ def test_token_200(rest_mock: Mocker): }, status_code=200, ) - assert RestCatalog("rest", {}, TEST_URI, TEST_CREDENTIALS).token == TEST_TOKEN + assert RestCatalog("rest", TEST_URI, TEST_CREDENTIALS).token == TEST_TOKEN def test_token_400(rest_mock: Mocker): @@ -85,7 +85,7 @@ def test_token_400(rest_mock: Mocker): ) with pytest.raises(OAuthError) as e: - RestCatalog("rest", {}, TEST_URI, credentials=TEST_CREDENTIALS) + RestCatalog("rest", TEST_URI, credential=TEST_CREDENTIALS) assert str(e.value) == "invalid_client: Credentials for key invalid_key do not match" @@ -104,7 +104,7 @@ def test_token_401(rest_mock: Mocker): ) with pytest.raises(BadCredentialsError) as e: - RestCatalog("rest", {}, TEST_URI, credentials=TEST_CREDENTIALS) + RestCatalog("rest", TEST_URI, credential=TEST_CREDENTIALS) assert message in str(e.value) @@ -116,7 +116,7 @@ def test_list_tables_200(rest_mock: Mocker): status_code=200, ) - assert RestCatalog("rest", {}, TEST_URI, token=TEST_TOKEN).list_tables(namespace) == [("examples", "fooshare")] + assert RestCatalog("rest", TEST_URI, token=TEST_TOKEN).list_tables(namespace) == [("examples", "fooshare")] def test_list_tables_404(rest_mock: Mocker): @@ -133,7 +133,7 @@ def test_list_tables_404(rest_mock: Mocker): status_code=404, ) with pytest.raises(NoSuchNamespaceError) as e: - RestCatalog("rest", {}, TEST_URI, token=TEST_TOKEN).list_tables(namespace) + RestCatalog("rest", TEST_URI, token=TEST_TOKEN).list_tables(namespace) assert "Namespace does not exist" in str(e.value) @@ -143,7 +143,7 @@ def test_list_namespaces_200(rest_mock: Mocker): json={"namespaces": [["default"], ["examples"], ["fokko"], ["system"]]}, status_code=200, ) - assert RestCatalog("rest", {}, TEST_URI, token=TEST_TOKEN).list_namespaces() == [ + assert RestCatalog("rest", TEST_URI, token=TEST_TOKEN).list_namespaces() == [ ("default",), ("examples",), ("fokko",), @@ -169,7 +169,7 @@ def test_create_namespace_200(rest_mock: Mocker): json={"namespace": [namespace], "properties": {}}, status_code=200, ) - RestCatalog("rest", {}, TEST_URI, token=TEST_TOKEN).create_namespace(namespace) + RestCatalog("rest", TEST_URI, token=TEST_TOKEN).create_namespace(namespace) def test_create_namespace_409(rest_mock: Mocker): @@ -186,7 +186,7 @@ def test_create_namespace_409(rest_mock: Mocker): status_code=409, ) with pytest.raises(NamespaceAlreadyExistsError) as e: - RestCatalog("rest", {}, TEST_URI, token=TEST_TOKEN).create_namespace(namespace) + RestCatalog("rest", TEST_URI, token=TEST_TOKEN).create_namespace(namespace) assert "Namespace already exists" in str(e.value) @@ -204,7 +204,7 @@ def test_drop_namespace_404(rest_mock: Mocker): status_code=404, ) with pytest.raises(NoSuchNamespaceError) as e: - RestCatalog("rest", {}, TEST_URI, token=TEST_TOKEN).drop_namespace(namespace) + RestCatalog("rest", TEST_URI, token=TEST_TOKEN).drop_namespace(namespace) assert "Namespace does not exist" in str(e.value) @@ -215,7 +215,7 @@ def test_load_namespace_properties_200(rest_mock: Mocker): json={"namespace": ["fokko"], "properties": {"prop": "yes"}}, status_code=204, ) - assert RestCatalog("rest", {}, TEST_URI, token=TEST_TOKEN).load_namespace_properties(namespace) == {"prop": "yes"} + assert RestCatalog("rest", TEST_URI, token=TEST_TOKEN).load_namespace_properties(namespace) == {"prop": "yes"} def test_load_namespace_properties_404(rest_mock: Mocker): @@ -232,7 +232,7 @@ def test_load_namespace_properties_404(rest_mock: Mocker): status_code=404, ) with pytest.raises(NoSuchNamespaceError) as e: - RestCatalog("rest", {}, TEST_URI, token=TEST_TOKEN).load_namespace_properties(namespace) + RestCatalog("rest", TEST_URI, token=TEST_TOKEN).load_namespace_properties(namespace) assert "Namespace does not exist" in str(e.value) @@ -242,9 +242,7 @@ def test_update_namespace_properties_200(rest_mock: Mocker): json={"removed": [], "updated": ["prop"], "missing": ["abc"]}, status_code=200, ) - response = RestCatalog("rest", {}, TEST_URI, token=TEST_TOKEN).update_namespace_properties( - ("fokko",), {"abc"}, {"prop": "yes"} - ) + response = RestCatalog("rest", TEST_URI, token=TEST_TOKEN).update_namespace_properties(("fokko",), {"abc"}, {"prop": "yes"}) assert response == PropertiesUpdateSummary(removed=[], updated=["prop"], missing=["abc"]) @@ -262,7 +260,7 @@ def test_update_namespace_properties_404(rest_mock: Mocker): status_code=404, ) with pytest.raises(NoSuchNamespaceError) as e: - RestCatalog("rest", {}, TEST_URI, token=TEST_TOKEN).update_namespace_properties(("fokko",), {"abc"}, {"prop": "yes"}) + RestCatalog("rest", TEST_URI, token=TEST_TOKEN).update_namespace_properties(("fokko",), {"abc"}, {"prop": "yes"}) assert "Namespace does not exist" in str(e.value) @@ -339,7 +337,7 @@ def test_load_table_200(rest_mock: Mocker): }, status_code=200, ) - table = RestCatalog("rest", {}, TEST_URI, token=TEST_TOKEN).load_table(("fokko", "table")) + table = RestCatalog("rest", TEST_URI, token=TEST_TOKEN).load_table(("fokko", "table")) assert table == Table( identifier=("rest", "fokko", "table"), metadata_location=None, @@ -433,7 +431,7 @@ def test_load_table_404(rest_mock: Mocker): ) with pytest.raises(NoSuchTableError) as e: - RestCatalog("rest", {}, TEST_URI, token=TEST_TOKEN).load_table(("fokko", "does_not_exists")) + RestCatalog("rest", TEST_URI, token=TEST_TOKEN).load_table(("fokko", "does_not_exists")) assert "Table does not exist" in str(e.value) @@ -451,7 +449,7 @@ def test_drop_table_404(rest_mock: Mocker): ) with pytest.raises(NoSuchTableError) as e: - RestCatalog("rest", {}, TEST_URI, token=TEST_TOKEN).drop_table(("fokko", "does_not_exists")) + RestCatalog("rest", TEST_URI, token=TEST_TOKEN).drop_table(("fokko", "does_not_exists")) assert "Table does not exist" in str(e.value) @@ -514,7 +512,7 @@ def test_create_table_200(rest_mock: Mocker, table_schema_simple: Schema): }, status_code=200, ) - table = RestCatalog("rest", {}, TEST_URI, token=TEST_TOKEN).create_table( + table = RestCatalog("rest", TEST_URI, token=TEST_TOKEN).create_table( identifier=("fokko", "fokko2"), schema=table_schema_simple, location=None, @@ -585,7 +583,7 @@ def test_create_table_409(rest_mock, table_schema_simple: Schema): ) with pytest.raises(TableAlreadyExistsError) as e: - RestCatalog("rest", {}, TEST_URI, token=TEST_TOKEN).create_table( + RestCatalog("rest", TEST_URI, token=TEST_TOKEN).create_table( identifier=("fokko", "fokko2"), schema=table_schema_simple, location=None, @@ -606,7 +604,7 @@ def test_delete_namespace_204(rest_mock: Mocker): json={}, status_code=204, ) - RestCatalog("rest", {}, TEST_URI, token=TEST_TOKEN).drop_namespace(namespace) + RestCatalog("rest", TEST_URI, token=TEST_TOKEN).drop_namespace(namespace) def test_delete_table_204(rest_mock: Mocker): @@ -615,7 +613,7 @@ def test_delete_table_204(rest_mock: Mocker): json={}, status_code=204, ) - RestCatalog("rest", {}, TEST_URI, token=TEST_TOKEN).drop_table(("example", "fokko")) + RestCatalog("rest", TEST_URI, token=TEST_TOKEN).drop_table(("example", "fokko")) def test_delete_table_404(rest_mock: Mocker): @@ -631,7 +629,7 @@ def test_delete_table_404(rest_mock: Mocker): status_code=404, ) with pytest.raises(NoSuchTableError) as e: - RestCatalog("rest", {}, TEST_URI, token=TEST_TOKEN).drop_table(("example", "fokko")) + RestCatalog("rest", TEST_URI, token=TEST_TOKEN).drop_table(("example", "fokko")) assert "Table does not exist" in str(e.value) @@ -639,7 +637,7 @@ def test_create_table_missing_namespace(rest_mock: Mocker, table_schema_simple: table = "table" with pytest.raises(NoSuchTableError) as e: # Missing namespace - RestCatalog("rest", {}, TEST_URI, token=TEST_TOKEN).create_table(table, table_schema_simple) + RestCatalog("rest", TEST_URI, token=TEST_TOKEN).create_table(table, table_schema_simple) assert f"Missing namespace or invalid identifier: {table}" in str(e.value) @@ -647,7 +645,7 @@ def test_load_table_invalid_namespace(rest_mock: Mocker): table = "table" with pytest.raises(NoSuchTableError) as e: # Missing namespace - RestCatalog("rest", {}, TEST_URI, token=TEST_TOKEN).load_table(table) + RestCatalog("rest", TEST_URI, token=TEST_TOKEN).load_table(table) assert f"Missing namespace or invalid identifier: {table}" in str(e.value) @@ -655,7 +653,7 @@ def test_drop_table_invalid_namespace(rest_mock: Mocker): table = "table" with pytest.raises(NoSuchTableError) as e: # Missing namespace - RestCatalog("rest", {}, TEST_URI, token=TEST_TOKEN).drop_table(table) + RestCatalog("rest", TEST_URI, token=TEST_TOKEN).drop_table(table) assert f"Missing namespace or invalid identifier: {table}" in str(e.value) @@ -663,33 +661,33 @@ def test_purge_table_invalid_namespace(rest_mock: Mocker): table = "table" with pytest.raises(NoSuchTableError) as e: # Missing namespace - RestCatalog("rest", {}, TEST_URI, token=TEST_TOKEN).purge_table(table) + RestCatalog("rest", TEST_URI, token=TEST_TOKEN).purge_table(table) assert f"Missing namespace or invalid identifier: {table}" in str(e.value) def test_create_namespace_invalid_namespace(rest_mock: Mocker): with pytest.raises(NoSuchNamespaceError) as e: # Missing namespace - RestCatalog("rest", {}, TEST_URI, token=TEST_TOKEN).create_namespace(()) + RestCatalog("rest", TEST_URI, token=TEST_TOKEN).create_namespace(()) assert "Empty namespace identifier" in str(e.value) def test_drop_namespace_invalid_namespace(rest_mock: Mocker): with pytest.raises(NoSuchNamespaceError) as e: # Missing namespace - RestCatalog("rest", {}, TEST_URI, token=TEST_TOKEN).drop_namespace(()) + RestCatalog("rest", TEST_URI, token=TEST_TOKEN).drop_namespace(()) assert "Empty namespace identifier" in str(e.value) def test_load_namespace_properties_invalid_namespace(rest_mock: Mocker): with pytest.raises(NoSuchNamespaceError) as e: # Missing namespace - RestCatalog("rest", {}, TEST_URI, token=TEST_TOKEN).load_namespace_properties(()) + RestCatalog("rest", TEST_URI, token=TEST_TOKEN).load_namespace_properties(()) assert "Empty namespace identifier" in str(e.value) def test_update_namespace_properties_invalid_namespace(rest_mock: Mocker): with pytest.raises(NoSuchNamespaceError) as e: # Missing namespace - RestCatalog("rest", {}, TEST_URI, token=TEST_TOKEN).update_namespace_properties(()) + RestCatalog("rest", TEST_URI, token=TEST_TOKEN).update_namespace_properties(()) assert "Empty namespace identifier" in str(e.value) diff --git a/tests/cli/__init__.py b/tests/cli/__init__.py new file mode 100644 index 0000000000..a67d5ea255 --- /dev/null +++ b/tests/cli/__init__.py @@ -0,0 +1,16 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. diff --git a/tests/cli/test_console.py b/tests/cli/test_console.py new file mode 100644 index 0000000000..a22e9d5c42 --- /dev/null +++ b/tests/cli/test_console.py @@ -0,0 +1,811 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +import os +from typing import ( + List, + Optional, + Set, + Union, +) +from unittest import mock + +from click.testing import CliRunner + +from pyiceberg.catalog import Catalog, PropertiesUpdateSummary +from pyiceberg.cli.console import run +from pyiceberg.exceptions import NoSuchNamespaceError, NoSuchTableError +from pyiceberg.schema import Schema +from pyiceberg.table import Table, TableMetadataV2 +from pyiceberg.table.partitioning import UNPARTITIONED_PARTITION_SPEC, PartitionSpec +from pyiceberg.table.sorting import UNSORTED_SORT_ORDER, SortOrder +from pyiceberg.typedef import EMPTY_DICT, Identifier, Properties +from tests.conftest import EXAMPLE_TABLE_METADATA_V2 + + +class MockCatalog(Catalog): + def create_table( + self, + identifier: Union[str, Identifier], + schema: Schema, + location: Optional[str] = None, + partition_spec: PartitionSpec = UNPARTITIONED_PARTITION_SPEC, + sort_order: SortOrder = UNSORTED_SORT_ORDER, + properties: Properties = EMPTY_DICT, + ) -> Table: + return Table(identifier=identifier, metadata_location="s3://tmp/", metadata=TableMetadataV2(**EXAMPLE_TABLE_METADATA_V2)) + + def load_table(self, identifier: Union[str, Identifier]) -> Table: + tuple_identifier = Catalog.identifier_to_tuple(identifier) + if tuple_identifier == ("default", "foo"): + return Table( + identifier=tuple_identifier, + metadata_location="s3://tmp/", + metadata=TableMetadataV2(**EXAMPLE_TABLE_METADATA_V2), + ) + else: + raise NoSuchTableError(f"Table does not exist: {'.'.join(tuple_identifier)}") + + def drop_table(self, identifier: Union[str, Identifier]) -> None: + tuple_identifier = Catalog.identifier_to_tuple(identifier) + if tuple_identifier == ("default", "foo"): + return None + else: + raise NoSuchTableError(f"Table does not exist: {identifier}") + + def purge_table(self, identifier: Union[str, Identifier]) -> None: + self.drop_table(identifier) + + def rename_table(self, from_identifier: Union[str, Identifier], to_identifier: Union[str, Identifier]) -> Table: + tuple_identifier = Catalog.identifier_to_tuple(from_identifier) + if tuple_identifier == ("default", "foo"): + return Table( + identifier=tuple_identifier, metadata_location="s3://tmp/", metadata=TableMetadataV2(**EXAMPLE_TABLE_METADATA_V2) + ) + else: + raise NoSuchTableError(f"Table does not exist: {from_identifier}") + + def create_namespace(self, namespace: Union[str, Identifier], properties: Properties = EMPTY_DICT) -> None: + return None + + def drop_namespace(self, namespace: Union[str, Identifier]) -> None: + tuple_identifier = Catalog.identifier_to_tuple(namespace) + if tuple_identifier == ("default",): + return None + else: + raise NoSuchNamespaceError(f"Namespace does not exist: {namespace}") + + def list_tables(self, namespace: Union[str, Identifier]) -> List[Identifier]: + return [ + ("default", "foo"), + ("default", "bar"), + ] + + def list_namespaces(self) -> List[Identifier]: + return [("default",), ("personal",)] + + def load_namespace_properties(self, namespace: Union[str, Identifier]) -> Properties: + identifier = Catalog.identifier_to_tuple(namespace) + if identifier == ("default",): + return {"location": "s3://warehouse/database/location"} + else: + raise NoSuchNamespaceError(f"Namespace does not exists: {namespace}") + + def update_namespace_properties( + self, namespace: Union[str, Identifier], removals: Optional[Set[str]] = None, updates: Properties = EMPTY_DICT + ) -> PropertiesUpdateSummary: + identifier = Catalog.identifier_to_tuple(namespace) + + if identifier == ("default",): + return PropertiesUpdateSummary(removed=["location"], updated=[], missing=[]) + else: + raise NoSuchNamespaceError(f"Namespace does not exists: {namespace}") + + +MOCK_ENVIRONMENT = {"PYICEBERG_URI": "test://doesnotexist"} +MOCK_CATALOGS = {"test": MockCatalog} + + +def test_missing_uri(): + runner = CliRunner() + result = runner.invoke(run, ["list"]) + assert result.exit_code == 1 + assert result.output == "Missing uri. Please provide using --uri or using environment variable \nPYICEBERG_URI\n" + + +@mock.patch.dict(os.environ, MOCK_ENVIRONMENT) +@mock.patch("pyiceberg.cli.console.SUPPORTED_CATALOGS", MOCK_CATALOGS) +def test_list_root(): + runner = CliRunner() + result = runner.invoke(run, ["list"]) + assert result.exit_code == 0 + assert result.output == "default \npersonal\n" + + +@mock.patch.dict(os.environ, MOCK_ENVIRONMENT) +@mock.patch("pyiceberg.cli.console.SUPPORTED_CATALOGS", MOCK_CATALOGS) +def test_list_namespace(): + runner = CliRunner() + result = runner.invoke(run, ["list", "default"]) + assert result.exit_code == 0 + assert result.output == "default.foo\ndefault.bar\n" + + +@mock.patch.dict(os.environ, MOCK_ENVIRONMENT) +@mock.patch("pyiceberg.cli.console.SUPPORTED_CATALOGS", MOCK_CATALOGS) +def test_describe_namespace(): + runner = CliRunner() + result = runner.invoke(run, ["describe", "default"]) + assert result.exit_code == 0 + assert result.output == "location s3://warehouse/database/location\n" + + +@mock.patch.dict(os.environ, MOCK_ENVIRONMENT) +@mock.patch("pyiceberg.cli.console.SUPPORTED_CATALOGS", MOCK_CATALOGS) +def test_describe_namespace_does_not_exists(): + runner = CliRunner() + result = runner.invoke(run, ["describe", "doesnotexists"]) + assert result.exit_code == 1 + assert result.output == "Table or namespace does not exist: doesnotexists\n" + + +@mock.patch.dict(os.environ, MOCK_ENVIRONMENT) +@mock.patch("pyiceberg.cli.console.SUPPORTED_CATALOGS", MOCK_CATALOGS) +def test_describe_table(): + runner = CliRunner() + result = runner.invoke(run, ["describe", "default.foo"]) + assert result.exit_code == 0 + assert ( + # Strip the whitespace on the end + "\n".join([line.rstrip() for line in result.output.split("\n")]) + == """Table format version 2 +Metadata location s3://tmp/ +Table UUID 9c12d441-03fe-4693-9a96-a0705ddf69c1 +Last Updated 1602638573590 +Partition spec [ + 1000: x: identity(1) + ] +Sort order [ + 2 ASC NULLS FIRST + bucket[4](3) DESC NULLS LAST + ] +Schema Schema + ├── 1: x: required long + ├── 2: y: required long (comment) + └── 3: z: required long +Snapshots Snapshots + ├── Snapshot None: s3://a/b/1.avro + └── Snapshot 1: s3://a/b/2.avro +Properties read.split.target.size 134217728 +""" + ) + + +@mock.patch.dict(os.environ, MOCK_ENVIRONMENT) +@mock.patch("pyiceberg.cli.console.SUPPORTED_CATALOGS", MOCK_CATALOGS) +def test_describe_table_does_not_exists(): + runner = CliRunner() + result = runner.invoke(run, ["describe", "default.doesnotexit"]) + assert result.exit_code == 1 + assert result.output == "Table does not exist: default.doesnotexit\n" + + +@mock.patch.dict(os.environ, MOCK_ENVIRONMENT) +@mock.patch("pyiceberg.cli.console.SUPPORTED_CATALOGS", MOCK_CATALOGS) +def test_schema(): + runner = CliRunner() + result = runner.invoke(run, ["schema", "default.foo"]) + assert result.exit_code == 0 + assert ( + result.output + == """x long +y long +z long +""" + ) + + +@mock.patch.dict(os.environ, MOCK_ENVIRONMENT) +@mock.patch("pyiceberg.cli.console.SUPPORTED_CATALOGS", MOCK_CATALOGS) +def test_schema_does_not_exists(): + runner = CliRunner() + result = runner.invoke(run, ["describe", "default.doesnotexit"]) + assert result.exit_code == 1 + assert result.output == "Table does not exist: default.doesnotexit\n" + + +@mock.patch.dict(os.environ, MOCK_ENVIRONMENT) +@mock.patch("pyiceberg.cli.console.SUPPORTED_CATALOGS", MOCK_CATALOGS) +def test_spec(): + runner = CliRunner() + result = runner.invoke(run, ["spec", "default.foo"]) + assert result.exit_code == 0 + assert ( + result.output + == """[ + 1000: x: identity(1) +] +""" + ) + + +@mock.patch.dict(os.environ, MOCK_ENVIRONMENT) +@mock.patch("pyiceberg.cli.console.SUPPORTED_CATALOGS", MOCK_CATALOGS) +def test_spec_does_not_exists(): + runner = CliRunner() + result = runner.invoke(run, ["spec", "default.doesnotexit"]) + assert result.exit_code == 1 + assert result.output == "Table does not exist: default.doesnotexit\n" + + +@mock.patch.dict(os.environ, MOCK_ENVIRONMENT) +@mock.patch("pyiceberg.cli.console.SUPPORTED_CATALOGS", MOCK_CATALOGS) +def test_uuid(): + runner = CliRunner() + result = runner.invoke(run, ["uuid", "default.foo"]) + assert result.exit_code == 0 + assert result.output == """9c12d441-03fe-4693-9a96-a0705ddf69c1\n""" + + +@mock.patch.dict(os.environ, MOCK_ENVIRONMENT) +@mock.patch("pyiceberg.cli.console.SUPPORTED_CATALOGS", MOCK_CATALOGS) +def test_uuid_does_not_exists(): + runner = CliRunner() + result = runner.invoke(run, ["uuid", "default.doesnotexit"]) + assert result.exit_code == 1 + assert result.output == "Table does not exist: default.doesnotexit\n" + + +@mock.patch.dict(os.environ, MOCK_ENVIRONMENT) +@mock.patch("pyiceberg.cli.console.SUPPORTED_CATALOGS", MOCK_CATALOGS) +def test_location(): + runner = CliRunner() + result = runner.invoke(run, ["location", "default.foo"]) + assert result.exit_code == 0 + assert result.output == """s3://bucket/test/location\n""" + + +@mock.patch.dict(os.environ, MOCK_ENVIRONMENT) +@mock.patch("pyiceberg.cli.console.SUPPORTED_CATALOGS", MOCK_CATALOGS) +def test_location_does_not_exists(): + runner = CliRunner() + result = runner.invoke(run, ["location", "default.doesnotexit"]) + assert result.exit_code == 1 + assert result.output == "Table does not exist: default.doesnotexit\n" + + +@mock.patch.dict(os.environ, MOCK_ENVIRONMENT) +@mock.patch("pyiceberg.cli.console.SUPPORTED_CATALOGS", MOCK_CATALOGS) +def test_drop_table(): + runner = CliRunner() + result = runner.invoke(run, ["drop", "table", "default.foo"]) + assert result.exit_code == 0 + assert result.output == """Dropped table: default.foo\n""" + + +@mock.patch.dict(os.environ, MOCK_ENVIRONMENT) +@mock.patch("pyiceberg.cli.console.SUPPORTED_CATALOGS", MOCK_CATALOGS) +def test_drop_table_does_not_exists(): + runner = CliRunner() + result = runner.invoke(run, ["drop", "table", "default.doesnotexit"]) + assert result.exit_code == 1 + assert result.output == "Table does not exist: default.doesnotexit\n" + + +@mock.patch.dict(os.environ, MOCK_ENVIRONMENT) +@mock.patch("pyiceberg.cli.console.SUPPORTED_CATALOGS", MOCK_CATALOGS) +def test_drop_namespace(): + runner = CliRunner() + result = runner.invoke(run, ["drop", "namespace", "default"]) + assert result.exit_code == 0 + assert result.output == """Dropped namespace: default\n""" + + +@mock.patch.dict(os.environ, MOCK_ENVIRONMENT) +@mock.patch("pyiceberg.cli.console.SUPPORTED_CATALOGS", MOCK_CATALOGS) +def test_drop_namespace_does_not_exists(): + runner = CliRunner() + result = runner.invoke(run, ["drop", "namespace", "doesnotexit"]) + assert result.exit_code == 1 + assert result.output == "Namespace does not exist: doesnotexit\n" + + +@mock.patch.dict(os.environ, MOCK_ENVIRONMENT) +@mock.patch("pyiceberg.cli.console.SUPPORTED_CATALOGS", MOCK_CATALOGS) +def test_rename_table(): + runner = CliRunner() + result = runner.invoke(run, ["rename", "default.foo", "default.bar"]) + assert result.exit_code == 0 + assert result.output == """Renamed table from default.foo to default.bar\n""" + + +@mock.patch.dict(os.environ, MOCK_ENVIRONMENT) +@mock.patch("pyiceberg.cli.console.SUPPORTED_CATALOGS", MOCK_CATALOGS) +def test_rename_table_does_not_exists(): + runner = CliRunner() + result = runner.invoke(run, ["rename", "default.doesnotexit", "default.bar"]) + assert result.exit_code == 1 + assert result.output == "Table does not exist: default.doesnotexit\n" + + +@mock.patch.dict(os.environ, MOCK_ENVIRONMENT) +@mock.patch("pyiceberg.cli.console.SUPPORTED_CATALOGS", MOCK_CATALOGS) +def test_properties_get_table(): + runner = CliRunner() + result = runner.invoke(run, ["properties", "get", "default.foo"]) + assert result.exit_code == 0 + assert result.output == "read.split.target.size 134217728\n" + + +@mock.patch.dict(os.environ, MOCK_ENVIRONMENT) +@mock.patch("pyiceberg.cli.console.SUPPORTED_CATALOGS", MOCK_CATALOGS) +def test_properties_get_table_specific_property(): + runner = CliRunner() + result = runner.invoke(run, ["properties", "get", "default.foo", "read.split.target.size"]) + assert result.exit_code == 0 + assert result.output == "134217728\n" + + +@mock.patch.dict(os.environ, MOCK_ENVIRONMENT) +@mock.patch("pyiceberg.cli.console.SUPPORTED_CATALOGS", MOCK_CATALOGS) +def test_properties_get_table_specific_property_that_doesnt_exist(): + runner = CliRunner() + result = runner.invoke(run, ["properties", "get", "default.fokko", "doesnotexist"]) + assert result.exit_code == 1 + assert result.output == "Table or namespace does not exist: default.fokko with property doesnotexist\n" + + +@mock.patch.dict(os.environ, MOCK_ENVIRONMENT) +@mock.patch("pyiceberg.cli.console.SUPPORTED_CATALOGS", MOCK_CATALOGS) +def test_properties_get_table_does_not_exist(): + runner = CliRunner() + result = runner.invoke(run, ["properties", "get", "doesnotexist"]) + assert result.exit_code == 1 + assert result.output == "Table or namespace does not exist: doesnotexist\n" + + +@mock.patch.dict(os.environ, MOCK_ENVIRONMENT) +@mock.patch("pyiceberg.cli.console.SUPPORTED_CATALOGS", MOCK_CATALOGS) +def test_properties_get_namespace(): + runner = CliRunner() + result = runner.invoke(run, ["properties", "get", "default"]) + assert result.exit_code == 0 + assert result.output == "location s3://warehouse/database/location\n" + + +@mock.patch.dict(os.environ, MOCK_ENVIRONMENT) +@mock.patch("pyiceberg.cli.console.SUPPORTED_CATALOGS", MOCK_CATALOGS) +def test_properties_get_namespace_specific_property(): + runner = CliRunner() + result = runner.invoke(run, ["properties", "get", "default", "location"]) + assert result.exit_code == 0 + assert result.output == "s3://warehouse/database/location\n" + + +@mock.patch.dict(os.environ, MOCK_ENVIRONMENT) +@mock.patch("pyiceberg.cli.console.SUPPORTED_CATALOGS", MOCK_CATALOGS) +def test_properties_set_namespace(): + runner = CliRunner() + result = runner.invoke(run, ["properties", "set", "namespace", "default", "location", "s3://new_location"]) + assert result.exit_code == 0 + assert result.output == "Updated location on default\n" + + +@mock.patch.dict(os.environ, MOCK_ENVIRONMENT) +@mock.patch("pyiceberg.cli.console.SUPPORTED_CATALOGS", MOCK_CATALOGS) +def test_properties_set_namespace_that_doesnt_exist(): + runner = CliRunner() + result = runner.invoke(run, ["properties", "set", "namespace", "doesnotexists", "location", "s3://new_location"]) + assert result.exit_code == 1 + assert result.output == "Namespace does not exists: doesnotexists\n" + + +@mock.patch.dict(os.environ, MOCK_ENVIRONMENT) +@mock.patch("pyiceberg.cli.console.SUPPORTED_CATALOGS", MOCK_CATALOGS) +def test_properties_set_table(): + runner = CliRunner() + result = runner.invoke(run, ["properties", "set", "table", "default.foo", "location", "s3://new_location"]) + assert result.exit_code == 1 + assert "Writing is WIP" in result.output + + +@mock.patch.dict(os.environ, MOCK_ENVIRONMENT) +@mock.patch("pyiceberg.cli.console.SUPPORTED_CATALOGS", MOCK_CATALOGS) +def test_properties_set_table_does_not_exist(): + runner = CliRunner() + result = runner.invoke(run, ["properties", "set", "table", "default.doesnotexist", "location", "s3://new_location"]) + assert result.exit_code == 1 + assert result.output == "Table does not exist: default.doesnotexist\n" + + +@mock.patch.dict(os.environ, MOCK_ENVIRONMENT) +@mock.patch("pyiceberg.cli.console.SUPPORTED_CATALOGS", MOCK_CATALOGS) +def test_properties_remove_namespace(): + runner = CliRunner() + result = runner.invoke(run, ["properties", "remove", "namespace", "default", "location"]) + assert result.exit_code == 0 + assert result.output == "Property location removed from default\n" + + +@mock.patch.dict(os.environ, MOCK_ENVIRONMENT) +@mock.patch("pyiceberg.cli.console.SUPPORTED_CATALOGS", MOCK_CATALOGS) +def test_properties_remove_namespace_that_doesnt_exist(): + runner = CliRunner() + result = runner.invoke(run, ["properties", "remove", "namespace", "doesnotexists", "location"]) + assert result.exit_code == 1 + assert result.output == "Namespace does not exists: doesnotexists\n" + + +@mock.patch.dict(os.environ, MOCK_ENVIRONMENT) +@mock.patch("pyiceberg.cli.console.SUPPORTED_CATALOGS", MOCK_CATALOGS) +def test_properties_remove_table(): + runner = CliRunner() + result = runner.invoke(run, ["properties", "remove", "table", "default.foo", "read.split.target.size"]) + assert result.exit_code == 1 + assert result.output == "Writing is WIP\n1\n" + + +@mock.patch.dict(os.environ, MOCK_ENVIRONMENT) +@mock.patch("pyiceberg.cli.console.SUPPORTED_CATALOGS", MOCK_CATALOGS) +def test_properties_remove_table_property_does_not_exists(): + runner = CliRunner() + result = runner.invoke(run, ["properties", "remove", "table", "default.foo", "doesnotexist"]) + assert result.exit_code == 1 + assert result.output == "Property doesnotexist does not exists on default.foo\n" + + +@mock.patch.dict(os.environ, MOCK_ENVIRONMENT) +@mock.patch("pyiceberg.cli.console.SUPPORTED_CATALOGS", MOCK_CATALOGS) +def test_properties_remove_table_does_not_exist(): + runner = CliRunner() + result = runner.invoke(run, ["properties", "remove", "table", "default.doesnotexist", "location"]) + assert result.exit_code == 1 + assert result.output == "Table does not exist: default.doesnotexist\n" + + +@mock.patch.dict(os.environ, MOCK_ENVIRONMENT) +@mock.patch("pyiceberg.cli.console.SUPPORTED_CATALOGS", MOCK_CATALOGS) +def test_json_list_root(): + runner = CliRunner() + result = runner.invoke(run, ["--output=json", "list"]) + assert result.exit_code == 0 + assert result.output == """["default", "personal"]\n""" + + +@mock.patch.dict(os.environ, MOCK_ENVIRONMENT) +@mock.patch("pyiceberg.cli.console.SUPPORTED_CATALOGS", MOCK_CATALOGS) +def test_json_list_namespace(): + runner = CliRunner() + result = runner.invoke(run, ["--output=json", "list", "default"]) + assert result.exit_code == 0 + assert result.output == """["default.foo", "default.bar"]\n""" + + +@mock.patch.dict(os.environ, MOCK_ENVIRONMENT) +@mock.patch("pyiceberg.cli.console.SUPPORTED_CATALOGS", MOCK_CATALOGS) +def test_json_describe_namespace(): + runner = CliRunner() + result = runner.invoke(run, ["--output=json", "describe", "default"]) + assert result.exit_code == 0 + assert result.output == """{"location": "s3://warehouse/database/location"}\n""" + + +@mock.patch.dict(os.environ, MOCK_ENVIRONMENT) +@mock.patch("pyiceberg.cli.console.SUPPORTED_CATALOGS", MOCK_CATALOGS) +def test_json_describe_namespace_does_not_exists(): + runner = CliRunner() + result = runner.invoke(run, ["--output=json", "describe", "doesnotexists"]) + assert result.exit_code == 1 + assert result.output == """{"type": "NoSuchTableError", "message": "Table or namespace does not exist: doesnotexists"}\n""" + + +@mock.patch.dict(os.environ, MOCK_ENVIRONMENT) +@mock.patch("pyiceberg.cli.console.SUPPORTED_CATALOGS", MOCK_CATALOGS) +def test_json_describe_table(): + runner = CliRunner() + result = runner.invoke(run, ["--output=json", "describe", "default.foo"]) + assert result.exit_code == 0 + assert ( + result.output + == """{"identifier": ["default", "foo"], "metadata_location": "s3://tmp/", "metadata": {"location": "s3://bucket/test/location", "table-uuid": "9c12d441-03fe-4693-9a96-a0705ddf69c1", "last-updated-ms": 1602638573590, "last-column-id": 3, "schemas": [{"type": "struct", "fields": [{"id": 1, "name": "x", "type": "long", "required": true}], "schema-id": 0, "identifier-field-ids": []}, {"type": "struct", "fields": [{"id": 1, "name": "x", "type": "long", "required": true}, {"id": 2, "name": "y", "type": "long", "required": true, "doc": "comment"}, {"id": 3, "name": "z", "type": "long", "required": true}], "schema-id": 1, "identifier-field-ids": [1, 2]}], "current-schema-id": 1, "partition-specs": [{"spec-id": 0, "fields": [{"source-id": 1, "field-id": 1000, "transform": "identity", "name": "x"}]}], "default-spec-id": 0, "last-partition-id": 1000, "properties": {"read.split.target.size": "134217728"}, "current-snapshot-id": 3055729675574597004, "snapshots": [{"snapshot-id": 3051729675574597004, "sequence-number": 0, "timestamp-ms": 1515100955770, "manifest-list": "s3://a/b/1.avro", "summary": {"operation": "append"}}, {"snapshot-id": 3055729675574597004, "parent-snapshot-id": 3051729675574597004, "sequence-number": 1, "timestamp-ms": 1555100955770, "manifest-list": "s3://a/b/2.avro", "summary": {"operation": "append"}, "schema-id": 1}], "snapshot-log": [{"snapshot-id": 3051729675574597004, "timestamp-ms": 1515100955770}, {"snapshot-id": 3055729675574597004, "timestamp-ms": 1555100955770}], "metadata-log": [], "sort-orders": [{"order-id": 3, "fields": [{"source-id": 2, "transform": "identity", "direction": "asc", "null-order": "nulls-first"}, {"source-id": 3, "transform": "bucket[4]", "direction": "desc", "null-order": "nulls-last"}]}], "default-sort-order-id": 3, "refs": {"main": {"snapshot-id": 3055729675574597004, "type": "branch"}}, "format-version": 2, "last-sequence-number": 34}}\n""" + ) + + +@mock.patch.dict(os.environ, MOCK_ENVIRONMENT) +@mock.patch("pyiceberg.cli.console.SUPPORTED_CATALOGS", MOCK_CATALOGS) +def test_json_describe_table_does_not_exists(): + runner = CliRunner() + result = runner.invoke(run, ["--output=json", "describe", "default.doesnotexit"]) + assert result.exit_code == 1 + assert result.output == """{"type": "NoSuchTableError", "message": "Table does not exist: default.doesnotexit"}\n""" + + +@mock.patch.dict(os.environ, MOCK_ENVIRONMENT) +@mock.patch("pyiceberg.cli.console.SUPPORTED_CATALOGS", MOCK_CATALOGS) +def test_json_schema(): + runner = CliRunner() + result = runner.invoke(run, ["--output=json", "schema", "default.foo"]) + assert result.exit_code == 0 + assert ( + result.output + == """{"type": "struct", "fields": [{"id": 1, "name": "x", "type": "long", "required": true}, {"id": 2, "name": "y", "type": "long", "required": true, "doc": "comment"}, {"id": 3, "name": "z", "type": "long", "required": true}], "schema-id": 1, "identifier-field-ids": [1, 2]}\n""" + ) + + +@mock.patch.dict(os.environ, MOCK_ENVIRONMENT) +@mock.patch("pyiceberg.cli.console.SUPPORTED_CATALOGS", MOCK_CATALOGS) +def test_json_schema_does_not_exists(): + runner = CliRunner() + result = runner.invoke(run, ["--output=json", "describe", "default.doesnotexit"]) + assert result.exit_code == 1 + assert result.output == """{"type": "NoSuchTableError", "message": "Table does not exist: default.doesnotexit"}\n""" + + +@mock.patch.dict(os.environ, MOCK_ENVIRONMENT) +@mock.patch("pyiceberg.cli.console.SUPPORTED_CATALOGS", MOCK_CATALOGS) +def test_json_spec(): + runner = CliRunner() + result = runner.invoke(run, ["--output=json", "spec", "default.foo"]) + assert result.exit_code == 0 + assert ( + result.output + == """{"spec-id": 0, "fields": [{"source-id": 1, "field-id": 1000, "transform": "identity", "name": "x"}]}\n""" + ) + + +@mock.patch.dict(os.environ, MOCK_ENVIRONMENT) +@mock.patch("pyiceberg.cli.console.SUPPORTED_CATALOGS", MOCK_CATALOGS) +def test_json_spec_does_not_exists(): + runner = CliRunner() + result = runner.invoke(run, ["--output=json", "spec", "default.doesnotexit"]) + assert result.exit_code == 1 + assert result.output == """{"type": "NoSuchTableError", "message": "Table does not exist: default.doesnotexit"}\n""" + + +@mock.patch.dict(os.environ, MOCK_ENVIRONMENT) +@mock.patch("pyiceberg.cli.console.SUPPORTED_CATALOGS", MOCK_CATALOGS) +def test_json_uuid(): + runner = CliRunner() + result = runner.invoke(run, ["--output=json", "uuid", "default.foo"]) + assert result.exit_code == 0 + assert result.output == """{"uuid": "9c12d441-03fe-4693-9a96-a0705ddf69c1"}\n""" + + +@mock.patch.dict(os.environ, MOCK_ENVIRONMENT) +@mock.patch("pyiceberg.cli.console.SUPPORTED_CATALOGS", MOCK_CATALOGS) +def test_json_uuid_does_not_exists(): + runner = CliRunner() + result = runner.invoke(run, ["--output=json", "uuid", "default.doesnotexit"]) + assert result.exit_code == 1 + assert result.output == """{"type": "NoSuchTableError", "message": "Table does not exist: default.doesnotexit"}\n""" + + +@mock.patch.dict(os.environ, MOCK_ENVIRONMENT) +@mock.patch("pyiceberg.cli.console.SUPPORTED_CATALOGS", MOCK_CATALOGS) +def test_json_location(): + runner = CliRunner() + result = runner.invoke(run, ["--output=json", "location", "default.foo"]) + assert result.exit_code == 0 + assert result.output == """"s3://bucket/test/location"\n""" + + +@mock.patch.dict(os.environ, MOCK_ENVIRONMENT) +@mock.patch("pyiceberg.cli.console.SUPPORTED_CATALOGS", MOCK_CATALOGS) +def test_json_location_does_not_exists(): + runner = CliRunner() + result = runner.invoke(run, ["--output=json", "location", "default.doesnotexist"]) + assert result.exit_code == 1 + assert result.output == """{"type": "NoSuchTableError", "message": "Table does not exist: default.doesnotexist"}\n""" + + +@mock.patch.dict(os.environ, MOCK_ENVIRONMENT) +@mock.patch("pyiceberg.cli.console.SUPPORTED_CATALOGS", MOCK_CATALOGS) +def test_json_drop_table(): + runner = CliRunner() + result = runner.invoke(run, ["--output=json", "drop", "table", "default.foo"]) + assert result.exit_code == 0 + assert result.output == """"Dropped table: default.foo"\n""" + + +@mock.patch.dict(os.environ, MOCK_ENVIRONMENT) +@mock.patch("pyiceberg.cli.console.SUPPORTED_CATALOGS", MOCK_CATALOGS) +def test_json_drop_table_does_not_exists(): + runner = CliRunner() + result = runner.invoke(run, ["--output=json", "drop", "table", "default.doesnotexist"]) + assert result.exit_code == 1 + assert result.output == """{"type": "NoSuchTableError", "message": "Table does not exist: default.doesnotexist"}\n""" + + +@mock.patch.dict(os.environ, MOCK_ENVIRONMENT) +@mock.patch("pyiceberg.cli.console.SUPPORTED_CATALOGS", MOCK_CATALOGS) +def test_json_drop_namespace(): + runner = CliRunner() + result = runner.invoke(run, ["--output=json", "drop", "namespace", "default"]) + assert result.exit_code == 0 + assert result.output == """"Dropped namespace: default"\n""" + + +@mock.patch.dict(os.environ, MOCK_ENVIRONMENT) +@mock.patch("pyiceberg.cli.console.SUPPORTED_CATALOGS", MOCK_CATALOGS) +def test_json_drop_namespace_does_not_exists(): + runner = CliRunner() + result = runner.invoke(run, ["--output=json", "drop", "namespace", "doesnotexist"]) + assert result.exit_code == 1 + assert result.output == """{"type": "NoSuchNamespaceError", "message": "Namespace does not exist: doesnotexist"}\n""" + + +@mock.patch.dict(os.environ, MOCK_ENVIRONMENT) +@mock.patch("pyiceberg.cli.console.SUPPORTED_CATALOGS", MOCK_CATALOGS) +def test_json_rename_table(): + runner = CliRunner() + result = runner.invoke(run, ["--output=json", "rename", "default.foo", "default.bar"]) + assert result.exit_code == 0 + assert result.output == """"Renamed table from default.foo to default.bar"\n""" + + +@mock.patch.dict(os.environ, MOCK_ENVIRONMENT) +@mock.patch("pyiceberg.cli.console.SUPPORTED_CATALOGS", MOCK_CATALOGS) +def test_json_rename_table_does_not_exists(): + runner = CliRunner() + result = runner.invoke(run, ["--output=json", "rename", "default.doesnotexit", "default.bar"]) + assert result.exit_code == 1 + assert result.output == """{"type": "NoSuchTableError", "message": "Table does not exist: default.doesnotexit"}\n""" + + +@mock.patch.dict(os.environ, MOCK_ENVIRONMENT) +@mock.patch("pyiceberg.cli.console.SUPPORTED_CATALOGS", MOCK_CATALOGS) +def test_json_properties_get_table(): + runner = CliRunner() + result = runner.invoke(run, ["--output=json", "properties", "get", "default.foo"]) + assert result.exit_code == 0 + assert result.output == """{"read.split.target.size": "134217728"}\n""" + + +@mock.patch.dict(os.environ, MOCK_ENVIRONMENT) +@mock.patch("pyiceberg.cli.console.SUPPORTED_CATALOGS", MOCK_CATALOGS) +def test_json_properties_get_table_specific_property(): + runner = CliRunner() + result = runner.invoke(run, ["--output=json", "properties", "get", "default.foo", "read.split.target.size"]) + assert result.exit_code == 0 + assert result.output == """"134217728"\n""" + + +@mock.patch.dict(os.environ, MOCK_ENVIRONMENT) +@mock.patch("pyiceberg.cli.console.SUPPORTED_CATALOGS", MOCK_CATALOGS) +def test_json_properties_get_table_specific_property_that_doesnt_exist(): + runner = CliRunner() + result = runner.invoke(run, ["--output=json", "properties", "get", "default.fokko", "doesnotexist"]) + assert result.exit_code == 1 + assert ( + result.output + == """{"type": "NoSuchNamespaceError", "message": "Table or namespace does not exist: default.fokko with property doesnotexist"}\n""" + ) + + +@mock.patch.dict(os.environ, MOCK_ENVIRONMENT) +@mock.patch("pyiceberg.cli.console.SUPPORTED_CATALOGS", MOCK_CATALOGS) +def test_json_properties_get_table_does_not_exist(): + runner = CliRunner() + result = runner.invoke(run, ["--output=json", "properties", "get", "doesnotexist"]) + assert result.exit_code == 1 + assert result.output == """{"type": "NoSuchNamespaceError", "message": "Table or namespace does not exist: doesnotexist"}\n""" + + +@mock.patch.dict(os.environ, MOCK_ENVIRONMENT) +@mock.patch("pyiceberg.cli.console.SUPPORTED_CATALOGS", MOCK_CATALOGS) +def test_json_properties_get_namespace(): + runner = CliRunner() + result = runner.invoke(run, ["--output=json", "properties", "get", "default"]) + assert result.exit_code == 0 + assert result.output == """{"location": "s3://warehouse/database/location"}\n""" + + +@mock.patch.dict(os.environ, MOCK_ENVIRONMENT) +@mock.patch("pyiceberg.cli.console.SUPPORTED_CATALOGS", MOCK_CATALOGS) +def test_json_properties_get_namespace_specific_property(): + runner = CliRunner() + result = runner.invoke(run, ["--output=json", "properties", "get", "default", "location"]) + assert result.exit_code == 0 + assert result.output == """"s3://warehouse/database/location"\n""" + + +@mock.patch.dict(os.environ, MOCK_ENVIRONMENT) +@mock.patch("pyiceberg.cli.console.SUPPORTED_CATALOGS", MOCK_CATALOGS) +def test_json_properties_set_namespace(): + runner = CliRunner() + result = runner.invoke(run, ["--output=json", "properties", "set", "namespace", "default", "location", "s3://new_location"]) + assert result.exit_code == 0 + assert result.output == """"Updated location on default"\n""" + + +@mock.patch.dict(os.environ, MOCK_ENVIRONMENT) +@mock.patch("pyiceberg.cli.console.SUPPORTED_CATALOGS", MOCK_CATALOGS) +def test_json_properties_set_namespace_that_doesnt_exist(): + runner = CliRunner() + result = runner.invoke( + run, ["--output=json", "properties", "set", "namespace", "doesnotexists", "location", "s3://new_location"] + ) + assert result.exit_code == 1 + assert result.output == """{"type": "NoSuchNamespaceError", "message": "Namespace does not exists: doesnotexists"}\n""" + + +@mock.patch.dict(os.environ, MOCK_ENVIRONMENT) +@mock.patch("pyiceberg.cli.console.SUPPORTED_CATALOGS", MOCK_CATALOGS) +def test_json_properties_set_table(): + runner = CliRunner() + result = runner.invoke(run, ["--output=json", "properties", "set", "table", "default.foo", "location", "s3://new_location"]) + assert result.exit_code == 1 + assert "Writing is WIP" in result.output + + +@mock.patch.dict(os.environ, MOCK_ENVIRONMENT) +@mock.patch("pyiceberg.cli.console.SUPPORTED_CATALOGS", MOCK_CATALOGS) +def test_json_properties_set_table_does_not_exist(): + runner = CliRunner() + result = runner.invoke( + run, ["--output=json", "properties", "set", "table", "default.doesnotexist", "location", "s3://new_location"] + ) + assert result.exit_code == 1 + assert result.output == """{"type": "NoSuchTableError", "message": "Table does not exist: default.doesnotexist"}\n""" + + +@mock.patch.dict(os.environ, MOCK_ENVIRONMENT) +@mock.patch("pyiceberg.cli.console.SUPPORTED_CATALOGS", MOCK_CATALOGS) +def test_json_properties_remove_namespace(): + runner = CliRunner() + result = runner.invoke(run, ["--output=json", "properties", "remove", "namespace", "default", "location"]) + assert result.exit_code == 0 + assert result.output == """"Property location removed from default"\n""" + + +@mock.patch.dict(os.environ, MOCK_ENVIRONMENT) +@mock.patch("pyiceberg.cli.console.SUPPORTED_CATALOGS", MOCK_CATALOGS) +def test_json_properties_remove_namespace_that_doesnt_exist(): + runner = CliRunner() + result = runner.invoke(run, ["--output=json", "properties", "remove", "namespace", "doesnotexist", "location"]) + assert result.exit_code == 1 + assert result.output == """{"type": "NoSuchNamespaceError", "message": "Namespace does not exists: doesnotexist"}\n""" + + +@mock.patch.dict(os.environ, MOCK_ENVIRONMENT) +@mock.patch("pyiceberg.cli.console.SUPPORTED_CATALOGS", MOCK_CATALOGS) +def test_json_properties_remove_table(): + runner = CliRunner() + result = runner.invoke(run, ["--output=json", "properties", "remove", "table", "default.foo", "read.split.target.size"]) + assert result.exit_code == 1 + assert "Writing is WIP" in result.output + + +@mock.patch.dict(os.environ, MOCK_ENVIRONMENT) +@mock.patch("pyiceberg.cli.console.SUPPORTED_CATALOGS", MOCK_CATALOGS) +def test_json_properties_remove_table_property_does_not_exists(): + runner = CliRunner() + result = runner.invoke(run, ["--output=json", "properties", "remove", "table", "default.foo", "doesnotexist"]) + assert result.exit_code == 1 + assert ( + result.output + == """{"type": "NoSuchPropertyException", "message": "Property doesnotexist does not exists on default.foo"}\n""" + ) + + +@mock.patch.dict(os.environ, MOCK_ENVIRONMENT) +@mock.patch("pyiceberg.cli.console.SUPPORTED_CATALOGS", MOCK_CATALOGS) +def test_json_properties_remove_table_does_not_exist(): + runner = CliRunner() + result = runner.invoke(run, ["--output=json", "properties", "remove", "table", "default.doesnotexist", "location"]) + print(result) + assert result.exit_code == 1 + assert result.output == """{"type": "NoSuchTableError", "message": "Table does not exist: default.doesnotexist"}\n""" diff --git a/tests/cli/test_output.py b/tests/cli/test_output.py new file mode 100644 index 0000000000..a67d5ea255 --- /dev/null +++ b/tests/cli/test_output.py @@ -0,0 +1,16 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. diff --git a/tests/conftest.py b/tests/conftest.py index 079cc1ac4d..55db81e887 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -219,68 +219,71 @@ def all_avro_types() -> Dict[str, Any]: } +EXAMPLE_TABLE_METADATA_V2 = { + "format-version": 2, + "table-uuid": "9c12d441-03fe-4693-9a96-a0705ddf69c1", + "location": "s3://bucket/test/location", + "last-sequence-number": 34, + "last-updated-ms": 1602638573590, + "last-column-id": 3, + "current-schema-id": 1, + "schemas": [ + {"type": "struct", "schema-id": 0, "fields": [{"id": 1, "name": "x", "required": True, "type": "long"}]}, + { + "type": "struct", + "schema-id": 1, + "identifier-field-ids": [1, 2], + "fields": [ + {"id": 1, "name": "x", "required": True, "type": "long"}, + {"id": 2, "name": "y", "required": True, "type": "long", "doc": "comment"}, + {"id": 3, "name": "z", "required": True, "type": "long"}, + ], + }, + ], + "default-spec-id": 0, + "partition-specs": [{"spec-id": 0, "fields": [{"name": "x", "transform": "identity", "source-id": 1, "field-id": 1000}]}], + "last-partition-id": 1000, + "default-sort-order-id": 3, + "sort-orders": [ + { + "order-id": 3, + "fields": [ + {"transform": "identity", "source-id": 2, "direction": "asc", "null-order": "nulls-first"}, + {"transform": "bucket[4]", "source-id": 3, "direction": "desc", "null-order": "nulls-last"}, + ], + } + ], + "properties": {"read.split.target.size": 134217728}, + "current-snapshot-id": 3055729675574597004, + "snapshots": [ + { + "snapshot-id": 3051729675574597004, + "timestamp-ms": 1515100955770, + "sequence-number": 0, + "summary": {"operation": "append"}, + "manifest-list": "s3://a/b/1.avro", + }, + { + "snapshot-id": 3055729675574597004, + "parent-snapshot-id": 3051729675574597004, + "timestamp-ms": 1555100955770, + "sequence-number": 1, + "summary": {"operation": "append"}, + "manifest-list": "s3://a/b/2.avro", + "schema-id": 1, + }, + ], + "snapshot-log": [ + {"snapshot-id": 3051729675574597004, "timestamp-ms": 1515100955770}, + {"snapshot-id": 3055729675574597004, "timestamp-ms": 1555100955770}, + ], + "metadata-log": [], +} + + @pytest.fixture def example_table_metadata_v2() -> Dict[str, Any]: - return { - "format-version": 2, - "table-uuid": "9c12d441-03fe-4693-9a96-a0705ddf69c1", - "location": "s3://bucket/test/location", - "last-sequence-number": 34, - "last-updated-ms": 1602638573590, - "last-column-id": 3, - "current-schema-id": 1, - "schemas": [ - {"type": "struct", "schema-id": 0, "fields": [{"id": 1, "name": "x", "required": True, "type": "long"}]}, - { - "type": "struct", - "schema-id": 1, - "identifier-field-ids": [1, 2], - "fields": [ - {"id": 1, "name": "x", "required": True, "type": "long"}, - {"id": 2, "name": "y", "required": True, "type": "long", "doc": "comment"}, - {"id": 3, "name": "z", "required": True, "type": "long"}, - ], - }, - ], - "default-spec-id": 0, - "partition-specs": [{"spec-id": 0, "fields": [{"name": "x", "transform": "identity", "source-id": 1, "field-id": 1000}]}], - "last-partition-id": 1000, - "default-sort-order-id": 3, - "sort-orders": [ - { - "order-id": 3, - "fields": [ - {"transform": "identity", "source-id": 2, "direction": "asc", "null-order": "nulls-first"}, - {"transform": "bucket[4]", "source-id": 3, "direction": "desc", "null-order": "nulls-last"}, - ], - } - ], - "properties": {"read.split.target.size": 134217728}, - "current-snapshot-id": 3055729675574597004, - "snapshots": [ - { - "snapshot-id": 3051729675574597004, - "timestamp-ms": 1515100955770, - "sequence-number": 0, - "summary": {"operation": "append"}, - "manifest-list": "s3://a/b/1.avro", - }, - { - "snapshot-id": 3055729675574597004, - "parent-snapshot-id": 3051729675574597004, - "timestamp-ms": 1555100955770, - "sequence-number": 1, - "summary": {"operation": "append"}, - "manifest-list": "s3://a/b/2.avro", - "schema-id": 1, - }, - ], - "snapshot-log": [ - {"snapshot-id": 3051729675574597004, "timestamp-ms": 1515100955770}, - {"snapshot-id": 3055729675574597004, "timestamp-ms": 1555100955770}, - ], - "metadata-log": [], - } + return EXAMPLE_TABLE_METADATA_V2 @pytest.fixture From c719c198906b31e400581fcfd77e7beb68550705 Mon Sep 17 00:00:00 2001 From: Fokko Driesprong Date: Tue, 16 Aug 2022 01:06:16 +0200 Subject: [PATCH 169/642] Python: Fix typo metadataLocation -> metadata-location (#5448) --- pyiceberg/catalog/rest.py | 2 +- pyiceberg/table/__init__.py | 2 +- tests/catalog/test_base.py | 10 ++++++++-- tests/catalog/test_rest.py | 24 ++++++++++++------------ 4 files changed, 22 insertions(+), 16 deletions(-) diff --git a/pyiceberg/catalog/rest.py b/pyiceberg/catalog/rest.py index 9d46c211f4..d3500beb29 100644 --- a/pyiceberg/catalog/rest.py +++ b/pyiceberg/catalog/rest.py @@ -93,7 +93,7 @@ class Endpoints: class TableResponse(IcebergBaseModel): - metadata_location: Optional[str] = Field(alias="metadata-location", default=None) + metadata_location: str = Field(alias="metadata-location") metadata: Union[TableMetadataV1, TableMetadataV2] = Field() config: Properties = Field(default_factory=dict) diff --git a/pyiceberg/table/__init__.py b/pyiceberg/table/__init__.py index 585ef6db04..b9ccb1f4df 100644 --- a/pyiceberg/table/__init__.py +++ b/pyiceberg/table/__init__.py @@ -26,5 +26,5 @@ class Table(IcebergBaseModel): identifier: Identifier = Field() - metadata_location: Optional[str] = Field() + metadata_location: str = Field() metadata: Optional[Union[TableMetadataV1, TableMetadataV2]] = Field() diff --git a/tests/catalog/test_base.py b/tests/catalog/test_base.py index ff84e64a03..9ac65ce02e 100644 --- a/tests/catalog/test_base.py +++ b/tests/catalog/test_base.py @@ -77,7 +77,11 @@ def create_table( if namespace not in self.__namespaces: self.__namespaces[namespace] = {} - table = Table(identifier=identifier, metadata=EXAMPLE_TABLE_METADATA_V1) + table = Table( + identifier=identifier, + metadata=EXAMPLE_TABLE_METADATA_V1, + metadata_location=f's3://warehouse/{"/".join(identifier)}/metadata/metadata.json', + ) self.__tables[identifier] = table return table @@ -110,7 +114,9 @@ def rename_table(self, from_identifier: Union[str, Identifier], to_identifier: U if to_namespace not in self.__namespaces: self.__namespaces[to_namespace] = {} - self.__tables[to_identifier] = Table(identifier=to_identifier, metadata=table.metadata) + self.__tables[to_identifier] = Table( + identifier=to_identifier, metadata=table.metadata, metadata_location=table.metadata_location + ) return self.__tables[to_identifier] def create_namespace(self, namespace: Union[str, Identifier], properties: Properties = EMPTY_DICT) -> None: diff --git a/tests/catalog/test_rest.py b/tests/catalog/test_rest.py index ae45c0ac05..785c350b6c 100644 --- a/tests/catalog/test_rest.py +++ b/tests/catalog/test_rest.py @@ -268,11 +268,11 @@ def test_load_table_200(rest_mock: Mocker): rest_mock.get( f"{TEST_URI}v1/namespaces/fokko/tables/table", json={ - "metadataLocation": "s3://tabular-public-us-west-2-dev/bb30733e-8769-4dab-aa1b-e76245bb2bd4/b55d9dda-6561-423a-8bfc-787980ce421f/metadata/00001-5f2f8166-244c-4eae-ac36-384ecdec81fc.gz.metadata.json", + "metadata-location": "s3://warehouse/database/table/metadata/00001-5f2f8166-244c-4eae-ac36-384ecdec81fc.gz.metadata.json", "metadata": { "format-version": 1, "table-uuid": "b55d9dda-6561-423a-8bfc-787980ce421f", - "location": "s3://tabular-public-us-west-2-dev/bb30733e-8769-4dab-aa1b-e76245bb2bd4/b55d9dda-6561-423a-8bfc-787980ce421f", + "location": "s3://warehouse/database/table", "last-updated-ms": 1646787054459, "last-column-id": 2, "schema": { @@ -321,7 +321,7 @@ def test_load_table_200(rest_mock: Mocker): "total-position-deletes": "0", "total-equality-deletes": "0", }, - "manifest-list": "s3://tabular-public-us-west-2-dev/bb30733e-8769-4dab-aa1b-e76245bb2bd4/b55d9dda-6561-423a-8bfc-787980ce421f/metadata/snap-3497810964824022504-1-c4f68204-666b-4e50-a9df-b10c34bf6b82.avro", + "manifest-list": "s3://warehouse/database/table/metadata/snap-3497810964824022504-1-c4f68204-666b-4e50-a9df-b10c34bf6b82.avro", "schema-id": 0, } ], @@ -329,7 +329,7 @@ def test_load_table_200(rest_mock: Mocker): "metadata-log": [ { "timestamp-ms": 1646787031514, - "metadata-file": "s3://tabular-public-us-west-2-dev/bb30733e-8769-4dab-aa1b-e76245bb2bd4/b55d9dda-6561-423a-8bfc-787980ce421f/metadata/00000-88484a1c-00e5-4a07-a787-c0e7aeffa805.gz.metadata.json", + "metadata-file": "s3://warehouse/database/table/metadata/00000-88484a1c-00e5-4a07-a787-c0e7aeffa805.gz.metadata.json", } ], }, @@ -340,9 +340,9 @@ def test_load_table_200(rest_mock: Mocker): table = RestCatalog("rest", TEST_URI, token=TEST_TOKEN).load_table(("fokko", "table")) assert table == Table( identifier=("rest", "fokko", "table"), - metadata_location=None, + metadata_location="s3://warehouse/database/table/metadata/00001-5f2f8166-244c-4eae-ac36-384ecdec81fc.gz.metadata.json", metadata=TableMetadataV1( - location="s3://tabular-public-us-west-2-dev/bb30733e-8769-4dab-aa1b-e76245bb2bd4/b55d9dda-6561-423a-8bfc-787980ce421f", + location="s3://warehouse/database/table", table_uuid=UUID("b55d9dda-6561-423a-8bfc-787980ce421f"), last_updated_ms=1646787054459, last_column_id=2, @@ -366,7 +366,7 @@ def test_load_table_200(rest_mock: Mocker): parent_snapshot_id=None, sequence_number=None, timestamp_ms=1646787054459, - manifest_list="s3://tabular-public-us-west-2-dev/bb30733e-8769-4dab-aa1b-e76245bb2bd4/b55d9dda-6561-423a-8bfc-787980ce421f/metadata/snap-3497810964824022504-1-c4f68204-666b-4e50-a9df-b10c34bf6b82.avro", + manifest_list="s3://warehouse/database/table/metadata/snap-3497810964824022504-1-c4f68204-666b-4e50-a9df-b10c34bf6b82.avro", summary=Summary( operation=Operation.APPEND, **{ # type: ignore @@ -390,7 +390,7 @@ def test_load_table_200(rest_mock: Mocker): metadata_log=[ { "timestamp-ms": 1646787031514, - "metadata-file": "s3://tabular-public-us-west-2-dev/bb30733e-8769-4dab-aa1b-e76245bb2bd4/b55d9dda-6561-423a-8bfc-787980ce421f/metadata/00000-88484a1c-00e5-4a07-a787-c0e7aeffa805.gz.metadata.json", + "metadata-file": "s3://warehouse/database/table/metadata/00000-88484a1c-00e5-4a07-a787-c0e7aeffa805.gz.metadata.json", } ], sort_orders=[SortOrder(order_id=0)], @@ -457,11 +457,11 @@ def test_create_table_200(rest_mock: Mocker, table_schema_simple: Schema): rest_mock.post( f"{TEST_URI}v1/namespaces/fokko/tables", json={ - "metadataLocation": None, + "metadata-location": "s3://warehouse/database/table/metadata.json", "metadata": { "format-version": 1, "table-uuid": "bf289591-dcc0-4234-ad4f-5c3eed811a29", - "location": "s3://tabular-wh-us-west-2-dev/8bcb0838-50fc-472d-9ddb-8feb89ef5f1e/bf289591-dcc0-4234-ad4f-5c3eed811a29", + "location": "s3://warehouse/database/table", "last-updated-ms": 1657810967051, "last-column-id": 3, "schema": { @@ -524,9 +524,9 @@ def test_create_table_200(rest_mock: Mocker, table_schema_simple: Schema): ) assert table == Table( identifier=("rest", "fokko", "fokko2"), - metadata_location=None, + metadata_location="s3://warehouse/database/table/metadata.json", metadata=TableMetadataV1( - location="s3://tabular-wh-us-west-2-dev/8bcb0838-50fc-472d-9ddb-8feb89ef5f1e/bf289591-dcc0-4234-ad4f-5c3eed811a29", + location="s3://warehouse/database/table", table_uuid=UUID("bf289591-dcc0-4234-ad4f-5c3eed811a29"), last_updated_ms=1657810967051, last_column_id=3, From d132b1d60507bc313f6706b2066ab1ffb12c6761 Mon Sep 17 00:00:00 2001 From: Samuel Redai <43911210+samredai@users.noreply.github.com> Date: Tue, 16 Aug 2022 08:22:55 -0700 Subject: [PATCH 170/642] Fix linter and test failures (#5542) * Fix linter and test failures * Replace pipe with Union * Update python/tests/catalog/test_hive.py Co-authored-by: Fokko Driesprong * Update python/pyiceberg/catalog/rest.py Co-authored-by: Fokko Driesprong * Update python/pyiceberg/cli/console.py Co-authored-by: Fokko Driesprong * Update python/tests/catalog/test_rest.py Co-authored-by: Fokko Driesprong Co-authored-by: Fokko Driesprong --- tests/catalog/test_hive.py | 2 +- tests/catalog/test_rest.py | 2 +- tests/cli/test_console.py | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/tests/catalog/test_hive.py b/tests/catalog/test_hive.py index 32dce64e31..e3baec7b84 100644 --- a/tests/catalog/test_hive.py +++ b/tests/catalog/test_hive.py @@ -468,7 +468,7 @@ def test_list_namespaces(): def test_list_namespaces_with_parent(): - catalog = HiveCatalog(HIVE_CATALOG_NAME, {}, uri=HIVE_METASTORE_FAKE_URL) + catalog = HiveCatalog(HIVE_CATALOG_NAME, uri=HIVE_METASTORE_FAKE_URL) # Hive does not support hierarchical namespaces assert catalog.list_namespaces(("accounting",)) == [] diff --git a/tests/catalog/test_rest.py b/tests/catalog/test_rest.py index 785c350b6c..cff7563ab5 100644 --- a/tests/catalog/test_rest.py +++ b/tests/catalog/test_rest.py @@ -157,7 +157,7 @@ def test_list_namespace_with_parent_200(rest_mock: Mocker): json={"namespaces": [["tax"]]}, status_code=200, ) - assert RestCatalog("rest", {}, TEST_URI, token=TEST_TOKEN).list_namespaces(("accounting",)) == [ + assert RestCatalog("rest", uri=TEST_URI, token=TEST_TOKEN).list_namespaces(("accounting",)) == [ ("accounting", "tax"), ] diff --git a/tests/cli/test_console.py b/tests/cli/test_console.py index a22e9d5c42..659ace6c23 100644 --- a/tests/cli/test_console.py +++ b/tests/cli/test_console.py @@ -94,7 +94,7 @@ def list_tables(self, namespace: Union[str, Identifier]) -> List[Identifier]: ("default", "bar"), ] - def list_namespaces(self) -> List[Identifier]: + def list_namespaces(self, namespace: Union[str, Identifier] = ()) -> List[Identifier]: return [("default",), ("personal",)] def load_namespace_properties(self, namespace: Union[str, Identifier]) -> Properties: From 6091d01750968595ca7097800a70dc6950d4c6a7 Mon Sep 17 00:00:00 2001 From: Fokko Driesprong Date: Tue, 16 Aug 2022 17:52:00 +0200 Subject: [PATCH 171/642] Python: Remove unused `# type: ignore` (#5534) --- pyiceberg/serializers.py | 4 ++-- pyiceberg/typedef.py | 2 +- pyproject.toml | 1 + 3 files changed, 4 insertions(+), 3 deletions(-) diff --git a/pyiceberg/serializers.py b/pyiceberg/serializers.py index 5906cb435d..fde592e5fc 100644 --- a/pyiceberg/serializers.py +++ b/pyiceberg/serializers.py @@ -35,8 +35,8 @@ def table_metadata(byte_stream: InputStream, encoding: str = "utf-8") -> TableMe encoding (default "utf-8"): The byte encoder to use for the reader """ reader = codecs.getreader(encoding) - metadata = json.load(reader(byte_stream)) # type: ignore - return TableMetadata.parse_obj(metadata) # type: ignore + metadata = json.load(reader(byte_stream)) + return TableMetadata.parse_obj(metadata) class FromInputFile: diff --git a/pyiceberg/typedef.py b/pyiceberg/typedef.py index 52e7df9e72..f9fa9c96f6 100644 --- a/pyiceberg/typedef.py +++ b/pyiceberg/typedef.py @@ -22,7 +22,7 @@ class FrozenDict(Dict): def __setitem__(self, instance, value): raise AttributeError("FrozenDict does not support assignment") - def update(self, *args: Any, **kwargs: Any) -> None: # type: ignore + def update(self, *args: Any, **kwargs: Any) -> None: raise AttributeError("FrozenDict does not support .update()") diff --git a/pyproject.toml b/pyproject.toml index a4785bf9c3..ef6d36e9f2 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -97,6 +97,7 @@ all = true no_implicit_optional = true warn_redundant_casts = true warn_unreachable = true +warn_unused_ignores = true [[tool.mypy.overrides]] module = "pyarrow.*" From 0dc51c4f25b65423cfae01821794c5f61e185570 Mon Sep 17 00:00:00 2001 From: Fokko Driesprong Date: Tue, 16 Aug 2022 17:52:30 +0200 Subject: [PATCH 172/642] Python: Bump pre-commit to the latest version (#5466) Periodic maintenance --- .pre-commit-config.yaml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 7c9875a468..cce5d2ba0b 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -43,12 +43,12 @@ repos: - id: mypy args: [ --install-types, --non-interactive, --config=python/pyproject.toml] - repo: https://github.com/hadialqattan/pycln - rev: v2.0.4 + rev: v2.1.1 hooks: - id: pycln args: [--config=python/pyproject.toml] - repo: https://github.com/asottile/pyupgrade - rev: v2.37.2 + rev: v2.37.3 hooks: - id: pyupgrade args: [--py38-plus] @@ -58,7 +58,7 @@ repos: - id: pylint args: [ --rcfile=python/pylintrc ] - repo: https://github.com/pycqa/flake8 - rev: '4.0.1' + rev: '5.0.4' hooks: - id: flake8 args: [ "--ignore=E501,W503,E203" ] From c34ca11a0ce62aedf6baa9285e39ec6d0f219a4a Mon Sep 17 00:00:00 2001 From: Fokko Driesprong Date: Tue, 16 Aug 2022 22:54:10 +0200 Subject: [PATCH 173/642] Python: Write the column docs as well (#5547) --- pyiceberg/cli/output.py | 2 +- tests/cli/test_console.py | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/pyiceberg/cli/output.py b/pyiceberg/cli/output.py index fc7c53cfc0..9b635e1741 100644 --- a/pyiceberg/cli/output.py +++ b/pyiceberg/cli/output.py @@ -122,7 +122,7 @@ def text(self, response: str): def schema(self, schema: Schema): output_table = self._table for field in schema.fields: - output_table.add_row(field.name, str(field.field_type)) + output_table.add_row(field.name, str(field.field_type), field.doc or "") Console().print(output_table) def spec(self, spec: PartitionSpec): diff --git a/tests/cli/test_console.py b/tests/cli/test_console.py index 659ace6c23..32d6b80d53 100644 --- a/tests/cli/test_console.py +++ b/tests/cli/test_console.py @@ -210,9 +210,9 @@ def test_schema(): result = runner.invoke(run, ["schema", "default.foo"]) assert result.exit_code == 0 assert ( - result.output + "\n".join([line.rstrip() for line in result.output.split("\n")]) == """x long -y long +y long comment z long """ ) From 7e5768f5df617d6ad3dcb47e40a9392ef96d8475 Mon Sep 17 00:00:00 2001 From: Fokko Driesprong Date: Thu, 18 Aug 2022 18:42:25 +0200 Subject: [PATCH 174/642] Python: Implement __repr__ for SnapshotRefType (#5564) --- pyiceberg/table/refs.py | 3 +++ tests/table/test_refs.py | 34 ++++++++++++++++++++++++++++++++++ 2 files changed, 37 insertions(+) create mode 100644 tests/table/test_refs.py diff --git a/pyiceberg/table/refs.py b/pyiceberg/table/refs.py index ab8c03fc37..58f6121ed9 100644 --- a/pyiceberg/table/refs.py +++ b/pyiceberg/table/refs.py @@ -28,6 +28,9 @@ class SnapshotRefType(str, Enum): BRANCH = "branch" TAG = "tag" + def __repr__(self) -> str: + return f"SnapshotRefType.{self.name}" + class SnapshotRef(IcebergBaseModel): snapshot_id: int = Field(alias="snapshot-id") diff --git a/tests/table/test_refs.py b/tests/table/test_refs.py new file mode 100644 index 0000000000..abd71ee110 --- /dev/null +++ b/tests/table/test_refs.py @@ -0,0 +1,34 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# pylint:disable=eval-used +from pyiceberg.table.refs import SnapshotRef, SnapshotRefType + + +def test_snapshot_with_properties_repr(): + snapshot_ref = SnapshotRef( + snapshot_id=3051729675574597004, + snapshot_ref_type=SnapshotRefType.TAG, + min_snapshots_to_keep=None, + max_snapshot_age_ms=None, + max_ref_age_ms=10000000, + ) + + assert ( + repr(snapshot_ref) + == """SnapshotRef(snapshot_id=3051729675574597004, snapshot_ref_type=SnapshotRefType.TAG, min_snapshots_to_keep=None, max_snapshot_age_ms=None, max_ref_age_ms=10000000)""" + ) + assert snapshot_ref == eval(repr(snapshot_ref)) From 25bf71439a4d92830778c02685a2ed0ffb8b0775 Mon Sep 17 00:00:00 2001 From: Fokko Driesprong Date: Thu, 18 Aug 2022 18:43:11 +0200 Subject: [PATCH 175/642] Python: Implement __repr__ for Summary and Operation (#5563) --- pyiceberg/table/snapshots.py | 7 +++ tests/table/test_snapshots.py | 94 +++++++++++++++++++---------------- 2 files changed, 58 insertions(+), 43 deletions(-) diff --git a/pyiceberg/table/snapshots.py b/pyiceberg/table/snapshots.py index e6e613e179..042691891e 100644 --- a/pyiceberg/table/snapshots.py +++ b/pyiceberg/table/snapshots.py @@ -39,6 +39,9 @@ class Operation(Enum): OVERWRITE = "overwrite" DELETE = "delete" + def __repr__(self) -> str: + return f"Operation.{self.name}" + class Summary(IcebergBaseModel): """ @@ -79,6 +82,10 @@ def operation(self) -> Operation: def additional_properties(self) -> Dict[str, str]: return self._additional_properties + def __repr__(self) -> str: + repr_properties = f", **{repr(self._additional_properties)}" if self._additional_properties else "" + return f"Summary({repr(self.operation)}{repr_properties})" + class Snapshot(IcebergBaseModel): snapshot_id: int = Field(alias="snapshot-id") diff --git a/tests/table/test_snapshots.py b/tests/table/test_snapshots.py index afce7209de..5dc48807ba 100644 --- a/tests/table/test_snapshots.py +++ b/tests/table/test_snapshots.py @@ -14,28 +14,47 @@ # KIND, either express or implied. See the License for the # specific language governing permissions and limitations # under the License. -from pyiceberg.table.snapshots import Operation, Snapshot, Summary - +# pylint:disable=redefined-outer-name,eval-used +import pytest -def test_serialize_summary(): - assert Summary(Operation.APPEND).json() == """{"operation": "append"}""" +from pyiceberg.table.snapshots import Operation, Snapshot, Summary -def test_serialize_summary_with_properties(): - assert Summary(Operation.APPEND, property="yes").json() == """{"operation": "append", "property": "yes"}""" +@pytest.fixture +def snapshot() -> Snapshot: + return Snapshot( + snapshot_id=25, + parent_snapshot_id=19, + sequence_number=200, + timestamp_ms=1602638573590, + manifest_list="s3:/a/b/c.avro", + summary=Summary(Operation.APPEND), + schema_id=3, + ) -def test_serialize_snapshot(): - snapshot = Snapshot( +@pytest.fixture +def snapshot_with_properties() -> Snapshot: + return Snapshot( snapshot_id=25, parent_snapshot_id=19, sequence_number=200, timestamp_ms=1602638573590, manifest_list="s3:/a/b/c.avro", - summary=Summary(Operation.APPEND), + summary=Summary(Operation.APPEND, foo="bar"), schema_id=3, ) + +def test_serialize_summary(): + assert Summary(Operation.APPEND).json() == """{"operation": "append"}""" + + +def test_serialize_summary_with_properties(): + assert Summary(Operation.APPEND, property="yes").json() == """{"operation": "append", "property": "yes"}""" + + +def test_serialize_snapshot(snapshot: Snapshot): assert ( snapshot.json() == """{"snapshot-id": 25, "parent-snapshot-id": 19, "sequence-number": 200, "timestamp-ms": 1602638573590, "manifest-list": "s3:/a/b/c.avro", "summary": {"operation": "append"}, "schema-id": 3}""" @@ -51,26 +70,15 @@ def test_serialize_snapshot_without_sequence_number(): summary=Summary(Operation.APPEND), schema_id=3, ) - actual = snapshot.json() expected = """{"snapshot-id": 25, "parent-snapshot-id": 19, "timestamp-ms": 1602638573590, "manifest-list": "s3:/a/b/c.avro", "summary": {"operation": "append"}, "schema-id": 3}""" assert actual == expected -def test_serialize_snapshot_with_properties(): - snapshot = Snapshot( - snapshot_id=25, - parent_snapshot_id=19, - sequence_number=200, - timestamp_ms=1602638573590, - manifest_list="s3:/a/b/c.avro", - summary=Summary(Operation.APPEND, property="yes"), - schema_id=3, - ) - +def test_serialize_snapshot_with_properties(snapshot_with_properties: Snapshot): assert ( - snapshot.json() - == """{"snapshot-id": 25, "parent-snapshot-id": 19, "sequence-number": 200, "timestamp-ms": 1602638573590, "manifest-list": "s3:/a/b/c.avro", "summary": {"operation": "append", "property": "yes"}, "schema-id": 3}""" + snapshot_with_properties.json() + == """{"snapshot-id": 25, "parent-snapshot-id": 19, "sequence-number": 200, "timestamp-ms": 1602638573590, "manifest-list": "s3:/a/b/c.avro", "summary": {"operation": "append", "foo": "bar"}, "schema-id": 3}""" ) @@ -85,29 +93,29 @@ def test_deserialize_summary_with_properties(): assert summary.additional_properties == {"property": "yes"} -def test_deserialize_snapshot(): +def test_deserialize_snapshot(snapshot: Snapshot): payload = """{"snapshot-id": 25, "parent-snapshot-id": 19, "sequence-number": 200, "timestamp-ms": 1602638573590, "manifest-list": "s3:/a/b/c.avro", "summary": {"operation": "append"}, "schema-id": 3}""" + actual = Snapshot.parse_raw(payload) + assert actual == snapshot + + +def test_deserialize_snapshot_with_properties(snapshot_with_properties: Snapshot): + payload = """{"snapshot-id": 25, "parent-snapshot-id": 19, "sequence-number": 200, "timestamp-ms": 1602638573590, "manifest-list": "s3:/a/b/c.avro", "summary": {"operation": "append", "foo": "bar"}, "schema-id": 3}""" snapshot = Snapshot.parse_raw(payload) - assert snapshot == Snapshot( - snapshot_id=25, - parent_snapshot_id=19, - sequence_number=200, - timestamp_ms=1602638573590, - manifest_list="s3:/a/b/c.avro", - summary=Summary(Operation.APPEND), - schema_id=3, + assert snapshot == snapshot_with_properties + + +def test_snapshot_repr(snapshot: Snapshot): + assert ( + repr(snapshot) + == """Snapshot(snapshot_id=25, parent_snapshot_id=19, sequence_number=200, timestamp_ms=1602638573590, manifest_list='s3:/a/b/c.avro', summary=Summary(Operation.APPEND), schema_id=3)""" ) + assert snapshot == eval(repr(snapshot)) -def test_deserialize_snapshot_with_properties(): - payload = """{"snapshot-id": 25, "parent-snapshot-id": 19, "sequence-number": 200, "timestamp-ms": 1602638573590, "manifest-list": "s3:/a/b/c.avro", "summary": {"operation": "append", "property": "yes"}, "schema-id": 3}""" - snapshot = Snapshot.parse_raw(payload) - assert snapshot == Snapshot( - snapshot_id=25, - parent_snapshot_id=19, - sequence_number=200, - timestamp_ms=1602638573590, - manifest_list="s3:/a/b/c.avro", - summary=Summary(Operation.APPEND, property="yes"), - schema_id=3, +def test_snapshot_with_properties_repr(snapshot_with_properties: Snapshot): + assert ( + repr(snapshot_with_properties) + == """Snapshot(snapshot_id=25, parent_snapshot_id=19, sequence_number=200, timestamp_ms=1602638573590, manifest_list='s3:/a/b/c.avro', summary=Summary(Operation.APPEND, **{'foo': 'bar'}), schema_id=3)""" ) + assert snapshot_with_properties == eval(repr(snapshot_with_properties)) From 27e3046b8d615eddefd7d8b12e4111cbb8f98c4d Mon Sep 17 00:00:00 2001 From: Fokko Driesprong Date: Fri, 19 Aug 2022 17:38:53 +0200 Subject: [PATCH 176/642] Python: Update copy_on_model_validation (#5583) --- pyiceberg/utils/iceberg_base_model.py | 1 - 1 file changed, 1 deletion(-) diff --git a/pyiceberg/utils/iceberg_base_model.py b/pyiceberg/utils/iceberg_base_model.py index e3ad2eab0f..e218f9f89b 100644 --- a/pyiceberg/utils/iceberg_base_model.py +++ b/pyiceberg/utils/iceberg_base_model.py @@ -37,7 +37,6 @@ class IcebergBaseModel(BaseModel): class Config: keep_untouched = (cached_property,) allow_population_by_field_name = True - copy_on_model_validation = False frozen = True def dict(self, exclude_none: bool = True, **kwargs): From 682b68374d3b4a348bebea4dd78dec774fa27b68 Mon Sep 17 00:00:00 2001 From: Fokko Driesprong Date: Fri, 19 Aug 2022 18:51:04 +0200 Subject: [PATCH 177/642] Python: Add Table API methods (#5562) --- pyiceberg/cli/console.py | 17 ++-- pyiceberg/cli/output.py | 7 +- pyiceberg/table/__init__.py | 71 +++++++++++++- pyiceberg/table/metadata.py | 18 +--- pyiceberg/table/snapshots.py | 10 ++ pyiceberg/table/sorting.py | 2 +- tests/catalog/test_hive.py | 27 ++++-- tests/cli/test_console.py | 2 +- tests/conftest.py | 3 +- tests/table/test_init.py | 180 +++++++++++++++++++++++++++++++++++ tests/table/test_metadata.py | 21 +++- 11 files changed, 315 insertions(+), 43 deletions(-) create mode 100644 tests/table/test_init.py diff --git a/pyiceberg/cli/console.py b/pyiceberg/cli/console.py index c95508d2b9..b4e0c15d95 100644 --- a/pyiceberg/cli/console.py +++ b/pyiceberg/cli/console.py @@ -146,9 +146,8 @@ def schema(ctx: Context, identifier: str): catalog, output = _catalog_and_output(ctx) try: - metadata = catalog.load_table(identifier).metadata - assert metadata - output.schema(metadata.current_schema()) + table = catalog.load_table(identifier) + output.schema(table.schema()) except Exception as exc: output.exception(exc) ctx.exit(1) @@ -161,9 +160,8 @@ def spec(ctx: Context, identifier: str): """Returns the partition spec of the table""" catalog, output = _catalog_and_output(ctx) try: - metadata = catalog.load_table(identifier).metadata - assert metadata - output.spec(metadata.current_partition_spec()) + table = catalog.load_table(identifier) + output.spec(table.spec()) except Exception as exc: output.exception(exc) ctx.exit(1) @@ -177,7 +175,6 @@ def uuid(ctx: Context, identifier: str): catalog, output = _catalog_and_output(ctx) try: metadata = catalog.load_table(identifier).metadata - assert metadata output.uuid(metadata.table_uuid) except Exception as exc: output.exception(exc) @@ -191,9 +188,8 @@ def location(ctx: Context, identifier: str): """Returns the location of the table""" catalog, output = _catalog_and_output(ctx) try: - metadata = catalog.load_table(identifier).metadata - assert metadata - output.text(metadata.location) + table = catalog.load_table(identifier) + output.text(table.location()) except Exception as exc: output.exception(exc) ctx.exit(1) @@ -389,7 +385,6 @@ def table(ctx: Context, identifier: str, property_name: str): # noqa: F811 catalog, output = _catalog_and_output(ctx) try: table = catalog.load_table(identifier) - assert table.metadata if property_name in table.metadata.properties: # We should think of the process here # Do we want something similar as in Java: diff --git a/pyiceberg/cli/output.py b/pyiceberg/cli/output.py index 9b635e1741..f3dbc45163 100644 --- a/pyiceberg/cli/output.py +++ b/pyiceberg/cli/output.py @@ -83,7 +83,6 @@ def identifiers(self, identifiers: List[Identifier]): Console().print(table) def describe_table(self, table: Table): - assert table.metadata metadata = table.metadata table_properties = self._table @@ -91,7 +90,7 @@ def describe_table(self, table: Table): table_properties.add_row(key, value) schema_tree = Tree("Schema") - for field in metadata.current_schema().fields: + for field in table.schema().fields: schema_tree.add(str(field)) snapshot_tree = Tree("Snapshots") @@ -103,8 +102,8 @@ def describe_table(self, table: Table): output_table.add_row("Metadata location", table.metadata_location) output_table.add_row("Table UUID", str(table.metadata.table_uuid)) output_table.add_row("Last Updated", str(metadata.last_updated_ms)) - output_table.add_row("Partition spec", str(metadata.current_partition_spec())) - output_table.add_row("Sort order", str(metadata.current_sort_order())) + output_table.add_row("Partition spec", str(table.spec())) + output_table.add_row("Sort order", str(table.sort_order())) output_table.add_row("Schema", schema_tree) output_table.add_row("Snapshots", snapshot_tree) output_table.add_row("Properties", table_properties) diff --git a/pyiceberg/table/__init__.py b/pyiceberg/table/__init__.py index b9ccb1f4df..bf9f128a32 100644 --- a/pyiceberg/table/__init__.py +++ b/pyiceberg/table/__init__.py @@ -15,11 +15,21 @@ # specific language governing permissions and limitations # under the License. -from typing import Optional, Union + +from typing import ( + Dict, + List, + Optional, + Union, +) from pydantic import Field +from pyiceberg.schema import Schema from pyiceberg.table.metadata import TableMetadataV1, TableMetadataV2 +from pyiceberg.table.partitioning import PartitionSpec +from pyiceberg.table.snapshots import Snapshot, SnapshotLogEntry +from pyiceberg.table.sorting import SortOrder from pyiceberg.typedef import Identifier from pyiceberg.utils.iceberg_base_model import IcebergBaseModel @@ -27,4 +37,61 @@ class Table(IcebergBaseModel): identifier: Identifier = Field() metadata_location: str = Field() - metadata: Optional[Union[TableMetadataV1, TableMetadataV2]] = Field() + metadata: Union[TableMetadataV1, TableMetadataV2] = Field() + + def refresh(self): + """Refresh the current table metadata""" + raise NotImplementedError("To be implemented") + + def schema(self) -> Schema: + """Return the schema for this table""" + return next(schema for schema in self.metadata.schemas if schema.schema_id == self.metadata.current_schema_id) + + def schemas(self) -> Dict[int, Schema]: + """Return a dict of the schema of this table""" + return {schema.schema_id: schema for schema in self.metadata.schemas} + + def spec(self) -> PartitionSpec: + """Return the partition spec of this table""" + return next(spec for spec in self.metadata.partition_specs if spec.spec_id == self.metadata.default_spec_id) + + def specs(self) -> Dict[int, PartitionSpec]: + """Return a dict the partition specs this table""" + return {spec.spec_id: spec for spec in self.metadata.partition_specs} + + def sort_order(self) -> SortOrder: + """Return the sort order of this table""" + return next( + sort_order for sort_order in self.metadata.sort_orders if sort_order.order_id == self.metadata.default_sort_order_id + ) + + def sort_orders(self) -> Dict[int, SortOrder]: + """Return a dict of the sort orders of this table""" + return {sort_order.order_id: sort_order for sort_order in self.metadata.sort_orders} + + def location(self) -> str: + """Return the table's base location.""" + return self.metadata.location + + def current_snapshot(self) -> Optional[Snapshot]: + """Get the current snapshot for this table, or None if there is no current snapshot.""" + if snapshot_id := self.metadata.current_snapshot_id: + return self.snapshot_by_id(snapshot_id) + return None + + def snapshot_by_id(self, snapshot_id: int) -> Optional[Snapshot]: + """Get the snapshot of this table with the given id, or None if there is no matching snapshot.""" + try: + return next(snapshot for snapshot in self.metadata.snapshots if snapshot.snapshot_id == snapshot_id) + except StopIteration: + return None + + def snapshot_by_name(self, name: str) -> Optional[Snapshot]: + """Returns the snapshot referenced by the given name or null if no such reference exists.""" + if ref := self.metadata.refs.get(name): + return self.snapshot_by_id(ref.snapshot_id) + return None + + def history(self) -> List[SnapshotLogEntry]: + """Get the snapshot history of this table.""" + return self.metadata.snapshot_log diff --git a/pyiceberg/table/metadata.py b/pyiceberg/table/metadata.py index 5f48a44d33..d5514a5bec 100644 --- a/pyiceberg/table/metadata.py +++ b/pyiceberg/table/metadata.py @@ -32,7 +32,7 @@ from pyiceberg.schema import Schema from pyiceberg.table.partitioning import PartitionSpec from pyiceberg.table.refs import MAIN_BRANCH, SnapshotRef, SnapshotRefType -from pyiceberg.table.snapshots import Snapshot +from pyiceberg.table.snapshots import MetadataLogEntry, Snapshot, SnapshotLogEntry from pyiceberg.table.sorting import UNSORTED_SORT_ORDER, UNSORTED_SORT_ORDER_ID, SortOrder from pyiceberg.utils.iceberg_base_model import IcebergBaseModel @@ -83,15 +83,6 @@ class TableMetadataCommonFields(IcebergBaseModel): """Metadata for an Iceberg table as specified in the Apache Iceberg spec (https://iceberg.apache.org/spec/#iceberg-table-spec)""" - def current_schema(self) -> Schema: - return next(schema for schema in self.schemas if schema.schema_id == self.current_schema_id) - - def current_sort_order(self) -> SortOrder: - return next(sort_order for sort_order in self.sort_orders if sort_order.order_id == self.default_sort_order_id) - - def current_partition_spec(self) -> PartitionSpec: - return next(spec for spec in self.partition_specs if spec.spec_id == self.default_spec_id) - @root_validator(pre=True) def cleanup_snapshot_id(cls, data: Dict[str, Any]): if data.get("current-snapshot-id") == -1: @@ -104,7 +95,8 @@ def cleanup_snapshot_id(cls, data: Dict[str, Any]): def construct_refs(cls, data: Dict[str, Any]): # This is going to be much nicer as soon as refs is an actual pydantic object if current_snapshot_id := data.get("current_snapshot_id"): - data["refs"] = {MAIN_BRANCH: SnapshotRef(snapshot_id=current_snapshot_id, snapshot_ref_type=SnapshotRefType.BRANCH)} + if MAIN_BRANCH not in data["refs"]: + data["refs"][MAIN_BRANCH] = SnapshotRef(snapshot_id=current_snapshot_id, snapshot_ref_type=SnapshotRefType.BRANCH) return data location: str = Field() @@ -158,7 +150,7 @@ def construct_refs(cls, data: Dict[str, Any]): deleted from the file system until the last snapshot in which it was listed is garbage collected.""" - snapshot_log: List[Dict[str, Any]] = Field(alias="snapshot-log", default_factory=list) + snapshot_log: List[SnapshotLogEntry] = Field(alias="snapshot-log", default_factory=list) """A list (optional) of timestamp and snapshot ID pairs that encodes changes to the current snapshot for the table. Each time the current-snapshot-id is changed, a new entry should be added with the @@ -166,7 +158,7 @@ def construct_refs(cls, data: Dict[str, Any]): expired from the list of valid snapshots, all entries before a snapshot that has expired should be removed.""" - metadata_log: List[Dict[str, Any]] = Field(alias="metadata-log", default_factory=list) + metadata_log: List[MetadataLogEntry] = Field(alias="metadata-log", default_factory=list) """A list (optional) of timestamp and metadata file location pairs that encodes changes to the previous metadata files for the table. Each time a new metadata file is created, a new entry of the previous metadata diff --git a/pyiceberg/table/snapshots.py b/pyiceberg/table/snapshots.py index 042691891e..8f061777f7 100644 --- a/pyiceberg/table/snapshots.py +++ b/pyiceberg/table/snapshots.py @@ -95,3 +95,13 @@ class Snapshot(IcebergBaseModel): manifest_list: Optional[str] = Field(alias="manifest-list", description="Location of the snapshot's manifest list file") summary: Optional[Summary] = Field() schema_id: Optional[int] = Field(alias="schema-id", default=None) + + +class MetadataLogEntry(IcebergBaseModel): + metadata_file: str = Field(alias="metadata-file") + timestamp_ms: int = Field(alias="timestamp-ms") + + +class SnapshotLogEntry(IcebergBaseModel): + snapshot_id: str = Field(alias="snapshot-id") + timestamp_ms: int = Field(alias="timestamp-ms") diff --git a/pyiceberg/table/sorting.py b/pyiceberg/table/sorting.py index 043496823a..8199caff7b 100644 --- a/pyiceberg/table/sorting.py +++ b/pyiceberg/table/sorting.py @@ -122,7 +122,7 @@ def __init__(self, order_id: Optional[int] = None, *fields: SortField, **data: A data["fields"] = fields super().__init__(**data) - order_id: Optional[int] = Field(alias="order-id") + order_id: int = Field(alias="order-id") fields: List[SortField] = Field(default_factory=list) def __str__(self) -> str: diff --git a/tests/catalog/test_hive.py b/tests/catalog/test_hive.py index e3baec7b84..e6b06e762d 100644 --- a/tests/catalog/test_hive.py +++ b/tests/catalog/test_hive.py @@ -47,7 +47,13 @@ from pyiceberg.table.metadata import TableMetadata, TableMetadataV2 from pyiceberg.table.partitioning import PartitionField, PartitionSpec from pyiceberg.table.refs import SnapshotRef, SnapshotRefType -from pyiceberg.table.snapshots import Operation, Snapshot, Summary +from pyiceberg.table.snapshots import ( + MetadataLogEntry, + Operation, + Snapshot, + SnapshotLogEntry, + Summary, +) from pyiceberg.table.sorting import ( NullOrder, SortDirection, @@ -336,7 +342,7 @@ def test_load_table(hive_table: HiveTable): sequence_number=0, timestamp_ms=1515100955770, manifest_list="s3://a/b/1.avro", - summary=Summary(Operation.APPEND), + summary=Summary(operation=Operation.APPEND), schema_id=None, ), Snapshot( @@ -345,15 +351,15 @@ def test_load_table(hive_table: HiveTable): sequence_number=1, timestamp_ms=1555100955770, manifest_list="s3://a/b/2.avro", - summary=Summary(Operation.APPEND), + summary=Summary(operation=Operation.APPEND), schema_id=1, ), ], snapshot_log=[ - {"snapshot-id": 3051729675574597004, "timestamp-ms": 1515100955770}, - {"snapshot-id": 3055729675574597004, "timestamp-ms": 1555100955770}, + SnapshotLogEntry(snapshot_id="3051729675574597004", timestamp_ms=1515100955770), + SnapshotLogEntry(snapshot_id="3055729675574597004", timestamp_ms=1555100955770), ], - metadata_log=[], + metadata_log=[MetadataLogEntry(metadata_file="s3://bucket/.../v1.json", timestamp_ms=1515100)], sort_orders=[ SortOrder( 3, @@ -370,13 +376,20 @@ def test_load_table(hive_table: HiveTable): ], default_sort_order_id=3, refs={ + "test": SnapshotRef( + snapshot_id=3051729675574597004, + snapshot_ref_type=SnapshotRefType.TAG, + min_snapshots_to_keep=None, + max_snapshot_age_ms=None, + max_ref_age_ms=10000000, + ), "main": SnapshotRef( snapshot_id=3055729675574597004, snapshot_ref_type=SnapshotRefType.BRANCH, min_snapshots_to_keep=None, max_snapshot_age_ms=None, max_ref_age_ms=None, - ) + ), }, format_version=2, last_sequence_number=34, diff --git a/tests/cli/test_console.py b/tests/cli/test_console.py index 32d6b80d53..2a001d14af 100644 --- a/tests/cli/test_console.py +++ b/tests/cli/test_console.py @@ -520,7 +520,7 @@ def test_json_describe_table(): assert result.exit_code == 0 assert ( result.output - == """{"identifier": ["default", "foo"], "metadata_location": "s3://tmp/", "metadata": {"location": "s3://bucket/test/location", "table-uuid": "9c12d441-03fe-4693-9a96-a0705ddf69c1", "last-updated-ms": 1602638573590, "last-column-id": 3, "schemas": [{"type": "struct", "fields": [{"id": 1, "name": "x", "type": "long", "required": true}], "schema-id": 0, "identifier-field-ids": []}, {"type": "struct", "fields": [{"id": 1, "name": "x", "type": "long", "required": true}, {"id": 2, "name": "y", "type": "long", "required": true, "doc": "comment"}, {"id": 3, "name": "z", "type": "long", "required": true}], "schema-id": 1, "identifier-field-ids": [1, 2]}], "current-schema-id": 1, "partition-specs": [{"spec-id": 0, "fields": [{"source-id": 1, "field-id": 1000, "transform": "identity", "name": "x"}]}], "default-spec-id": 0, "last-partition-id": 1000, "properties": {"read.split.target.size": "134217728"}, "current-snapshot-id": 3055729675574597004, "snapshots": [{"snapshot-id": 3051729675574597004, "sequence-number": 0, "timestamp-ms": 1515100955770, "manifest-list": "s3://a/b/1.avro", "summary": {"operation": "append"}}, {"snapshot-id": 3055729675574597004, "parent-snapshot-id": 3051729675574597004, "sequence-number": 1, "timestamp-ms": 1555100955770, "manifest-list": "s3://a/b/2.avro", "summary": {"operation": "append"}, "schema-id": 1}], "snapshot-log": [{"snapshot-id": 3051729675574597004, "timestamp-ms": 1515100955770}, {"snapshot-id": 3055729675574597004, "timestamp-ms": 1555100955770}], "metadata-log": [], "sort-orders": [{"order-id": 3, "fields": [{"source-id": 2, "transform": "identity", "direction": "asc", "null-order": "nulls-first"}, {"source-id": 3, "transform": "bucket[4]", "direction": "desc", "null-order": "nulls-last"}]}], "default-sort-order-id": 3, "refs": {"main": {"snapshot-id": 3055729675574597004, "type": "branch"}}, "format-version": 2, "last-sequence-number": 34}}\n""" + == """{"identifier": ["default", "foo"], "metadata_location": "s3://tmp/", "metadata": {"location": "s3://bucket/test/location", "table-uuid": "9c12d441-03fe-4693-9a96-a0705ddf69c1", "last-updated-ms": 1602638573590, "last-column-id": 3, "schemas": [{"type": "struct", "fields": [{"id": 1, "name": "x", "type": "long", "required": true}], "schema-id": 0, "identifier-field-ids": []}, {"type": "struct", "fields": [{"id": 1, "name": "x", "type": "long", "required": true}, {"id": 2, "name": "y", "type": "long", "required": true, "doc": "comment"}, {"id": 3, "name": "z", "type": "long", "required": true}], "schema-id": 1, "identifier-field-ids": [1, 2]}], "current-schema-id": 1, "partition-specs": [{"spec-id": 0, "fields": [{"source-id": 1, "field-id": 1000, "transform": "identity", "name": "x"}]}], "default-spec-id": 0, "last-partition-id": 1000, "properties": {"read.split.target.size": "134217728"}, "current-snapshot-id": 3055729675574597004, "snapshots": [{"snapshot-id": 3051729675574597004, "sequence-number": 0, "timestamp-ms": 1515100955770, "manifest-list": "s3://a/b/1.avro", "summary": {"operation": "append"}}, {"snapshot-id": 3055729675574597004, "parent-snapshot-id": 3051729675574597004, "sequence-number": 1, "timestamp-ms": 1555100955770, "manifest-list": "s3://a/b/2.avro", "summary": {"operation": "append"}, "schema-id": 1}], "snapshot-log": [{"snapshot-id": "3051729675574597004", "timestamp-ms": 1515100955770}, {"snapshot-id": "3055729675574597004", "timestamp-ms": 1555100955770}], "metadata-log": [{"metadata-file": "s3://bucket/.../v1.json", "timestamp-ms": 1515100}], "sort-orders": [{"order-id": 3, "fields": [{"source-id": 2, "transform": "identity", "direction": "asc", "null-order": "nulls-first"}, {"source-id": 3, "transform": "bucket[4]", "direction": "desc", "null-order": "nulls-last"}]}], "default-sort-order-id": 3, "refs": {"test": {"snapshot-id": 3051729675574597004, "type": "tag", "max-ref-age-ms": 10000000}, "main": {"snapshot-id": 3055729675574597004, "type": "branch"}}, "format-version": 2, "last-sequence-number": 34}}\n""" ) diff --git a/tests/conftest.py b/tests/conftest.py index 55db81e887..089443cead 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -277,7 +277,8 @@ def all_avro_types() -> Dict[str, Any]: {"snapshot-id": 3051729675574597004, "timestamp-ms": 1515100955770}, {"snapshot-id": 3055729675574597004, "timestamp-ms": 1555100955770}, ], - "metadata-log": [], + "metadata-log": [{"metadata-file": "s3://bucket/.../v1.json", "timestamp-ms": 1515100}], + "refs": {"test": {"snapshot-id": 3051729675574597004, "type": "tag", "max-ref-age-ms": 10000000}}, } diff --git a/tests/table/test_init.py b/tests/table/test_init.py new file mode 100644 index 0000000000..b06421fcfb --- /dev/null +++ b/tests/table/test_init.py @@ -0,0 +1,180 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# pylint:disable=redefined-outer-name +from typing import Any, Dict + +import pytest + +from pyiceberg.schema import Schema +from pyiceberg.table import PartitionSpec, Table, TableMetadataV2 +from pyiceberg.table.partitioning import PartitionField +from pyiceberg.table.snapshots import ( + Operation, + Snapshot, + SnapshotLogEntry, + Summary, +) +from pyiceberg.table.sorting import ( + NullOrder, + SortDirection, + SortField, + SortOrder, +) +from pyiceberg.transforms import BucketTransform, IdentityTransform +from pyiceberg.types import LongType, NestedField + + +@pytest.fixture +def table(example_table_metadata_v2: Dict[str, Any]) -> Table: + table_metadata = TableMetadataV2(**example_table_metadata_v2) + return Table( + identifier=("database", "table"), + metadata=table_metadata, + metadata_location=f"{table_metadata.location}/uuid.metadata.json", + ) + + +def test_schema(table): + assert table.schema() == Schema( + fields=( + NestedField(field_id=1, name="x", field_type=LongType(), required=True), + NestedField(field_id=2, name="y", field_type=LongType(), required=True, doc="comment"), + NestedField(field_id=3, name="z", field_type=LongType(), required=True), + ), + schema_id=1, + identifier_field_ids=[1, 2], + ) + + +def test_schemas(table): + assert table.schemas() == { + 0: Schema( + fields=(NestedField(field_id=1, name="x", field_type=LongType(), required=True),), + schema_id=0, + identifier_field_ids=[], + ), + 1: Schema( + fields=( + NestedField(field_id=1, name="x", field_type=LongType(), required=True), + NestedField(field_id=2, name="y", field_type=LongType(), required=True, doc="comment"), + NestedField(field_id=3, name="z", field_type=LongType(), required=True), + ), + schema_id=1, + identifier_field_ids=[1, 2], + ), + } + + +def test_spec(table): + assert table.spec() == PartitionSpec( + spec_id=0, fields=(PartitionField(source_id=1, field_id=1000, transform=IdentityTransform(), name="x"),) + ) + + +def test_specs(table): + assert table.specs() == { + 0: PartitionSpec(spec_id=0, fields=(PartitionField(source_id=1, field_id=1000, transform=IdentityTransform(), name="x"),)) + } + + +def test_sort_order(table): + assert table.sort_order() == SortOrder( + order_id=3, + fields=[ + SortField(source_id=2, transform=IdentityTransform(), direction=SortDirection.ASC, null_order=NullOrder.NULLS_FIRST), + SortField( + source_id=3, + transform=BucketTransform(num_buckets=4), + direction=SortDirection.DESC, + null_order=NullOrder.NULLS_LAST, + ), + ], + ) + + +def test_sort_orders(table): + assert table.sort_orders() == { + 3: SortOrder( + order_id=3, + fields=[ + SortField( + source_id=2, transform=IdentityTransform(), direction=SortDirection.ASC, null_order=NullOrder.NULLS_FIRST + ), + SortField( + source_id=3, + transform=BucketTransform(num_buckets=4), + direction=SortDirection.DESC, + null_order=NullOrder.NULLS_LAST, + ), + ], + ) + } + + +def test_location(table): + assert table.location() == "s3://bucket/test/location" + + +def test_current_snapshot(table): + assert table.current_snapshot() == Snapshot( + snapshot_id=3055729675574597004, + parent_snapshot_id=3051729675574597004, + sequence_number=1, + timestamp_ms=1555100955770, + manifest_list="s3://a/b/2.avro", + summary=Summary(operation=Operation.APPEND), + schema_id=1, + ) + + +def test_snapshot_by_id(table): + assert table.snapshot_by_id(3055729675574597004) == Snapshot( + snapshot_id=3055729675574597004, + parent_snapshot_id=3051729675574597004, + sequence_number=1, + timestamp_ms=1555100955770, + manifest_list="s3://a/b/2.avro", + summary=Summary(operation=Operation.APPEND), + schema_id=1, + ) + + +def test_snapshot_by_id_does_not_exist(table): + assert table.snapshot_by_id(-1) is None + + +def test_snapshot_by_name(table): + assert table.snapshot_by_name("test") == Snapshot( + snapshot_id=3051729675574597004, + parent_snapshot_id=None, + sequence_number=0, + timestamp_ms=1515100955770, + manifest_list="s3://a/b/1.avro", + summary=Summary(operation=Operation.APPEND), + schema_id=None, + ) + + +def test_snapshot_by_name_does_not_exist(table): + assert table.snapshot_by_name("doesnotexist") is None + + +def test_history(table): + assert table.history() == [ + SnapshotLogEntry(snapshot_id="3051729675574597004", timestamp_ms=1515100955770), + SnapshotLogEntry(snapshot_id="3055729675574597004", timestamp_ms=1555100955770), + ] diff --git a/tests/table/test_metadata.py b/tests/table/test_metadata.py index 352e7574f5..c357b9d825 100644 --- a/tests/table/test_metadata.py +++ b/tests/table/test_metadata.py @@ -87,7 +87,7 @@ def test_v2_metadata_parsing(example_table_metadata_v2: Dict[str, Any]): assert table_metadata.properties["read.split.target.size"] == "134217728" assert table_metadata.current_snapshot_id == 3055729675574597004 assert table_metadata.snapshots[0].snapshot_id == 3051729675574597004 - assert table_metadata.snapshot_log[0]["timestamp-ms"] == 1515100955770 + assert table_metadata.snapshot_log[0].timestamp_ms == 1515100955770 assert table_metadata.sort_orders[0].order_id == 3 assert table_metadata.default_sort_order_id == 3 @@ -152,7 +152,7 @@ def test_serialize_v1(): def test_serialize_v2(example_table_metadata_v2: Dict[str, Any]): table_metadata = TableMetadataV2(**example_table_metadata_v2).json() - expected = """{"location": "s3://bucket/test/location", "table-uuid": "9c12d441-03fe-4693-9a96-a0705ddf69c1", "last-updated-ms": 1602638573590, "last-column-id": 3, "schemas": [{"type": "struct", "fields": [{"id": 1, "name": "x", "type": "long", "required": true}], "schema-id": 0, "identifier-field-ids": []}, {"type": "struct", "fields": [{"id": 1, "name": "x", "type": "long", "required": true}, {"id": 2, "name": "y", "type": "long", "required": true, "doc": "comment"}, {"id": 3, "name": "z", "type": "long", "required": true}], "schema-id": 1, "identifier-field-ids": [1, 2]}], "current-schema-id": 1, "partition-specs": [{"spec-id": 0, "fields": [{"source-id": 1, "field-id": 1000, "transform": "identity", "name": "x"}]}], "default-spec-id": 0, "last-partition-id": 1000, "properties": {"read.split.target.size": "134217728"}, "current-snapshot-id": 3055729675574597004, "snapshots": [{"snapshot-id": 3051729675574597004, "sequence-number": 0, "timestamp-ms": 1515100955770, "manifest-list": "s3://a/b/1.avro", "summary": {"operation": "append"}}, {"snapshot-id": 3055729675574597004, "parent-snapshot-id": 3051729675574597004, "sequence-number": 1, "timestamp-ms": 1555100955770, "manifest-list": "s3://a/b/2.avro", "summary": {"operation": "append"}, "schema-id": 1}], "snapshot-log": [{"snapshot-id": 3051729675574597004, "timestamp-ms": 1515100955770}, {"snapshot-id": 3055729675574597004, "timestamp-ms": 1555100955770}], "metadata-log": [], "sort-orders": [{"order-id": 3, "fields": [{"source-id": 2, "transform": "identity", "direction": "asc", "null-order": "nulls-first"}, {"source-id": 3, "transform": "bucket[4]", "direction": "desc", "null-order": "nulls-last"}]}], "default-sort-order-id": 3, "refs": {"main": {"snapshot-id": 3055729675574597004, "type": "branch"}}, "format-version": 2, "last-sequence-number": 34}""" + expected = """{"location": "s3://bucket/test/location", "table-uuid": "9c12d441-03fe-4693-9a96-a0705ddf69c1", "last-updated-ms": 1602638573590, "last-column-id": 3, "schemas": [{"type": "struct", "fields": [{"id": 1, "name": "x", "type": "long", "required": true}], "schema-id": 0, "identifier-field-ids": []}, {"type": "struct", "fields": [{"id": 1, "name": "x", "type": "long", "required": true}, {"id": 2, "name": "y", "type": "long", "required": true, "doc": "comment"}, {"id": 3, "name": "z", "type": "long", "required": true}], "schema-id": 1, "identifier-field-ids": [1, 2]}], "current-schema-id": 1, "partition-specs": [{"spec-id": 0, "fields": [{"source-id": 1, "field-id": 1000, "transform": "identity", "name": "x"}]}], "default-spec-id": 0, "last-partition-id": 1000, "properties": {"read.split.target.size": "134217728"}, "current-snapshot-id": 3055729675574597004, "snapshots": [{"snapshot-id": 3051729675574597004, "sequence-number": 0, "timestamp-ms": 1515100955770, "manifest-list": "s3://a/b/1.avro", "summary": {"operation": "append"}}, {"snapshot-id": 3055729675574597004, "parent-snapshot-id": 3051729675574597004, "sequence-number": 1, "timestamp-ms": 1555100955770, "manifest-list": "s3://a/b/2.avro", "summary": {"operation": "append"}, "schema-id": 1}], "snapshot-log": [{"snapshot-id": "3051729675574597004", "timestamp-ms": 1515100955770}, {"snapshot-id": "3055729675574597004", "timestamp-ms": 1555100955770}], "metadata-log": [{"metadata-file": "s3://bucket/.../v1.json", "timestamp-ms": 1515100}], "sort-orders": [{"order-id": 3, "fields": [{"source-id": 2, "transform": "identity", "direction": "asc", "null-order": "nulls-first"}, {"source-id": 3, "transform": "bucket[4]", "direction": "desc", "null-order": "nulls-last"}]}], "default-sort-order-id": 3, "refs": {"test": {"snapshot-id": 3051729675574597004, "type": "tag", "max-ref-age-ms": 10000000}, "main": {"snapshot-id": 3055729675574597004, "type": "branch"}}, "format-version": 2, "last-sequence-number": 34}""" assert table_metadata == expected @@ -464,7 +464,22 @@ def test_v1_write_metadata_for_v2(): def test_v2_ref_creation(example_table_metadata_v2: Dict[str, Any]): table_metadata = TableMetadataV2(**example_table_metadata_v2) - assert table_metadata.refs == {"main": SnapshotRef(snapshot_id=3055729675574597004, snapshot_ref_type=SnapshotRefType.BRANCH)} + assert table_metadata.refs == { + "main": SnapshotRef( + snapshot_id=3055729675574597004, + snapshot_ref_type=SnapshotRefType.BRANCH, + min_snapshots_to_keep=None, + max_snapshot_age_ms=None, + max_ref_age_ms=None, + ), + "test": SnapshotRef( + snapshot_id=3051729675574597004, + snapshot_ref_type=SnapshotRefType.TAG, + min_snapshots_to_keep=None, + max_snapshot_age_ms=None, + max_ref_age_ms=10000000, + ), + } def test_metadata_v1(): From e2892d16e6b6bca99ae9b8417f9767a8aead0c20 Mon Sep 17 00:00:00 2001 From: Fokko Driesprong Date: Sat, 20 Aug 2022 00:20:02 +0200 Subject: [PATCH 178/642] Python: Add wrapper to handle CLI exit code (#5546) --- pyiceberg/cli/console.py | 253 +++++++++++++++++--------------------- tests/cli/test_console.py | 98 ++++++++------- 2 files changed, 164 insertions(+), 187 deletions(-) diff --git a/pyiceberg/cli/console.py b/pyiceberg/cli/console.py index b4e0c15d95..5ce6cf4c04 100644 --- a/pyiceberg/cli/console.py +++ b/pyiceberg/cli/console.py @@ -16,6 +16,7 @@ # under the License. # pylint: disable=broad-except,redefined-builtin,redefined-outer-name import os +from functools import wraps from typing import ( Dict, Literal, @@ -36,6 +37,23 @@ SUPPORTED_CATALOGS: Dict[str, Type[Catalog]] = {"thrift": HiveCatalog, "http": RestCatalog} +def catch_exception(): + def decorator(func): + @wraps(func) + def wrapper(*args, **kwargs): + try: + return func(*args, **kwargs) + except Exception as e: + ctx: Context = click.get_current_context(silent=True) + _, output = _catalog_and_output(ctx) + output.exception(e) + ctx.exit(1) + + return wrapper + + return decorator + + @click.group() @click.option("--catalog", default=None) @click.option("--output", type=click.Choice(["text", "json"]), default="text") @@ -88,111 +106,93 @@ def _catalog_and_output(ctx: Context) -> Tuple[Catalog, Output]: @run.command() @click.pass_context @click.argument("parent", required=False) +@catch_exception() def list(ctx: Context, parent: Optional[str]): # pylint: disable=redefined-builtin """Lists tables or namespaces""" catalog, output = _catalog_and_output(ctx) - # still wip, will become more beautiful - # https://github.com/apache/iceberg/pull/5467/ - # has been merged - try: - if parent: - identifiers = catalog.list_tables(parent) - else: - identifiers = catalog.list_namespaces() - output.identifiers(identifiers) - except Exception as exc: - output.exception(exc) - ctx.exit(1) + identifiers = catalog.list_namespaces(parent or ()) + if not identifiers and parent: + identifiers = catalog.list_tables(parent) + output.identifiers(identifiers) @run.command() @click.option("--entity", type=click.Choice(["any", "namespace", "table"]), default="any") @click.argument("identifier") @click.pass_context +@catch_exception() def describe(ctx: Context, entity: Literal["name", "namespace", "table"], identifier: str): """Describes a namespace xor table""" catalog, output = _catalog_and_output(ctx) identifier_tuple = Catalog.identifier_to_tuple(identifier) + is_namespace = False if entity in {"namespace", "any"} and len(identifier_tuple) > 0: try: namespace_properties = catalog.load_namespace_properties(identifier_tuple) output.describe_properties(namespace_properties) - ctx.exit(0) + is_namespace = True except NoSuchNamespaceError as exc: - if entity != "any": - output.exception(exc) - ctx.exit(1) + if entity != "any" or len(identifier_tuple) == 1: # type: ignore + raise exc + is_table = False if entity in {"table", "any"} and len(identifier_tuple) > 1: try: catalog_table = catalog.load_table(identifier) output.describe_table(catalog_table) - ctx.exit(0) + is_table = True except NoSuchTableError as exc: - output.exception(exc) - ctx.exit(1) + if entity != "any": + raise exc - output.exception(NoSuchTableError(f"Table or namespace does not exist: {identifier}")) - ctx.exit(1) + if is_namespace is False and is_table is False: + raise NoSuchTableError(f"Table or namespace does not exist: {identifier}") @run.command() @click.argument("identifier") @click.pass_context +@catch_exception() def schema(ctx: Context, identifier: str): """Gets the schema of the table""" catalog, output = _catalog_and_output(ctx) - - try: - table = catalog.load_table(identifier) - output.schema(table.schema()) - except Exception as exc: - output.exception(exc) - ctx.exit(1) + table = catalog.load_table(identifier) + output.schema(table.schema()) @run.command() @click.argument("identifier") @click.pass_context +@catch_exception() def spec(ctx: Context, identifier: str): """Returns the partition spec of the table""" catalog, output = _catalog_and_output(ctx) - try: - table = catalog.load_table(identifier) - output.spec(table.spec()) - except Exception as exc: - output.exception(exc) - ctx.exit(1) + table = catalog.load_table(identifier) + output.spec(table.spec()) @run.command() @click.argument("identifier") @click.pass_context +@catch_exception() def uuid(ctx: Context, identifier: str): """Returns the UUID of the table""" catalog, output = _catalog_and_output(ctx) - try: - metadata = catalog.load_table(identifier).metadata - output.uuid(metadata.table_uuid) - except Exception as exc: - output.exception(exc) - ctx.exit(1) + metadata = catalog.load_table(identifier).metadata + output.uuid(metadata.table_uuid) @run.command() @click.argument("identifier") @click.pass_context +@catch_exception() def location(ctx: Context, identifier: str): """Returns the location of the table""" catalog, output = _catalog_and_output(ctx) - try: - table = catalog.load_table(identifier) - output.text(table.location()) - except Exception as exc: - output.exception(exc) - ctx.exit(1) + table = catalog.load_table(identifier) + output.text(table.location()) @run.group() @@ -203,47 +203,38 @@ def drop(): @drop.command() @click.argument("identifier") @click.pass_context +@catch_exception() def table(ctx: Context, identifier: str): # noqa: F811 """Drop table""" catalog, output = _catalog_and_output(ctx) - try: - catalog.drop_table(identifier) - output.text(f"Dropped table: {identifier}") - except Exception as exc: - output.exception(exc) - ctx.exit(1) + catalog.drop_table(identifier) + output.text(f"Dropped table: {identifier}") @drop.command() @click.argument("identifier") @click.pass_context +@catch_exception() def namespace(ctx, identifier: str): """Drop namespace""" catalog, output = _catalog_and_output(ctx) - try: - catalog.drop_namespace(identifier) - output.text(f"Dropped namespace: {identifier}") - except Exception as exc: - output.exception(exc) - ctx.exit(1) + catalog.drop_namespace(identifier) + output.text(f"Dropped namespace: {identifier}") @run.command() @click.argument("from_identifier") @click.argument("to_identifier") @click.pass_context +@catch_exception() def rename(ctx, from_identifier: str, to_identifier: str): """Renames a table""" catalog, output = _catalog_and_output(ctx) - try: - catalog.rename_table(from_identifier, to_identifier) - output.text(f"Renamed table from {from_identifier} to {to_identifier}") - except Exception as exc: - output.exception(exc) - ctx.exit(1) + catalog.rename_table(from_identifier, to_identifier) + output.text(f"Renamed table from {from_identifier} to {to_identifier}") @run.group() @@ -256,55 +247,50 @@ def properties(): @click.argument("identifier") @click.argument("property_name", required=False) @click.pass_context +@catch_exception() def get(ctx: Context, entity: Literal["name", "namespace", "table"], identifier: str, property_name: str): """Fetches a property of a namespace or table""" catalog, output = _catalog_and_output(ctx) identifier_tuple = Catalog.identifier_to_tuple(identifier) - try: - namespace_properties = catalog.load_namespace_properties(identifier_tuple) - - if property_name: - if property_value := namespace_properties.get(property_name): - output.text(property_value) - ctx.exit(0) - elif entity != "any": - output.exception(NoSuchPropertyException(f"Could not find property {property_name} on {entity} {identifier}")) - ctx.exit(1) - else: - output.describe_properties(namespace_properties) - ctx.exit(0) - except NoSuchNamespaceError as exc: - if entity != "any": - output.exception(exc) - ctx.exit(1) - - if len(identifier_tuple) > 1: + is_namespace = False + if entity in {"namespace", "any"}: try: - metadata = catalog.load_table(identifier_tuple).metadata - assert metadata + namespace_properties = catalog.load_namespace_properties(identifier_tuple) if property_name: - if property_value := metadata.properties.get(property_name): + if property_value := namespace_properties.get(property_name): output.text(property_value) - ctx.exit(0) - elif entity != "any": - output.exception(NoSuchPropertyException(f"Could not find property {property_name} on {entity} {identifier}")) - ctx.exit(1) + is_namespace = True + else: + raise NoSuchPropertyException(f"Could not find property {property_name} on namespace {identifier}") else: - output.describe_properties(metadata.properties) - ctx.exit(0) - except NoSuchTableError as exc: - if entity != "any": - output.exception(exc) - ctx.exit(1) + output.describe_properties(namespace_properties) + is_namespace = True + except NoSuchNamespaceError as exc: + if entity != "any" or len(identifier_tuple) <= 1: # type: ignore + raise exc + is_table = False + if is_namespace is False and len(identifier_tuple) > 1 and entity in {"table", "any"}: + metadata = catalog.load_table(identifier_tuple).metadata + assert metadata - property_err = "" - if property_name: - property_err = f" with property {property_name}" + if property_name: + if property_value := metadata.properties.get(property_name): + output.text(property_value) + is_table = True + else: + raise NoSuchPropertyException(f"Could not find property {property_name} on table {identifier}") + else: + output.describe_properties(metadata.properties) + is_table = True + + if is_namespace is False and is_table is False: + property_err = "" + if property_name: + property_err = f" with property {property_name}" - output.exception(NoSuchNamespaceError(f"Table or namespace does not exist: {identifier}{property_err}")) - ctx.exit(1) + raise NoSuchNamespaceError(f"Table or namespace does not exist: {identifier}{property_err}") @properties.group() @@ -317,15 +303,13 @@ def set(): @click.argument("property_name") @click.argument("property_value") @click.pass_context +@catch_exception() def namespace(ctx: Context, identifier: str, property_name: str, property_value: str): # noqa: F811 """Sets a property of a namespace or table""" catalog, output = _catalog_and_output(ctx) - try: - catalog.update_namespace_properties(identifier, updates={property_name: property_value}) - output.text(f"Updated {property_name} on {identifier}") - except NoSuchNamespaceError as exc: - output.exception(exc) - ctx.exit(1) + + catalog.update_namespace_properties(identifier, updates={property_name: property_value}) + output.text(f"Updated {property_name} on {identifier}") @set.command() # type: ignore @@ -333,23 +317,15 @@ def namespace(ctx: Context, identifier: str, property_name: str, property_value: @click.argument("property_name") @click.argument("property_value") @click.pass_context +@catch_exception() def table(ctx: Context, identifier: str, property_name: str, property_value: str): # noqa: F811 """Sets a property on a table""" catalog, output = _catalog_and_output(ctx) identifier_tuple = Catalog.identifier_to_tuple(identifier) - try: - _ = catalog.load_table(identifier_tuple) - output.text(f"Setting {property_name}={property_value} on {identifier}") - output.exception(NotImplementedError("Writing is WIP")) - ctx.exit(1) - - except NoSuchTableError as exc: - output.exception(exc) - ctx.exit(1) - - output.exception(NoSuchNamespaceError(f"Could not find table/namespace {identifier} with property {property}")) - ctx.exit(1) + _ = catalog.load_table(identifier_tuple) + output.text(f"Setting {property_name}={property_value} on {identifier}") + raise NotImplementedError("Writing is WIP") @properties.group() @@ -361,39 +337,34 @@ def remove(): @click.argument("identifier") @click.argument("property_name") @click.pass_context +@catch_exception() def namespace(ctx: Context, identifier: str, property_name: str): # noqa: F811 """Removes a property from a namespace""" catalog, output = _catalog_and_output(ctx) - try: - result = catalog.update_namespace_properties(identifier, removals={property_name}) - if result.removed == [property_name]: - output.text(f"Property {property_name} removed from {identifier}") - else: - raise NoSuchPropertyException(f"Property {property_name} does not exists on {identifier}") - except Exception as exc: - output.exception(exc) - ctx.exit(1) + result = catalog.update_namespace_properties(identifier, removals={property_name}) + + if result.removed == [property_name]: + output.text(f"Property {property_name} removed from {identifier}") + else: + raise NoSuchPropertyException(f"Property {property_name} does not exist on {identifier}") @remove.command() # type: ignore @click.argument("identifier") @click.argument("property_name") @click.pass_context +@catch_exception() def table(ctx: Context, identifier: str, property_name: str): # noqa: F811 """Removes a property from a table""" catalog, output = _catalog_and_output(ctx) - try: - table = catalog.load_table(identifier) - if property_name in table.metadata.properties: - # We should think of the process here - # Do we want something similar as in Java: - # https://github.com/apache/iceberg/blob/master/api/src/main/java/org/apache/iceberg/Table.java#L178 - del table.metadata.properties - output.exception(NotImplementedError("Writing is WIP")) - ctx.exit(1) - else: - raise NoSuchPropertyException(f"Property {property_name} does not exists on {identifier}") - except Exception as exc: - output.exception(exc) + table = catalog.load_table(identifier) + if property_name in table.metadata.properties: + # We should think of the process here + # Do we want something similar as in Java: + # https://github.com/apache/iceberg/blob/master/api/src/main/java/org/apache/iceberg/Table.java#L178 + del table.metadata.properties + output.exception(NotImplementedError("Writing is WIP")) ctx.exit(1) + else: + raise NoSuchPropertyException(f"Property {property_name} does not exist on {identifier}") diff --git a/tests/cli/test_console.py b/tests/cli/test_console.py index 2a001d14af..704e6729f5 100644 --- a/tests/cli/test_console.py +++ b/tests/cli/test_console.py @@ -95,14 +95,18 @@ def list_tables(self, namespace: Union[str, Identifier]) -> List[Identifier]: ] def list_namespaces(self, namespace: Union[str, Identifier] = ()) -> List[Identifier]: - return [("default",), ("personal",)] + # No hierarchical namespaces for now + if namespace == (): + return [("default",), ("personal",)] + else: + return [] def load_namespace_properties(self, namespace: Union[str, Identifier]) -> Properties: identifier = Catalog.identifier_to_tuple(namespace) if identifier == ("default",): return {"location": "s3://warehouse/database/location"} else: - raise NoSuchNamespaceError(f"Namespace does not exists: {namespace}") + raise NoSuchNamespaceError(f"Namespace does not exist: {'.'.join(namespace)}") def update_namespace_properties( self, namespace: Union[str, Identifier], removals: Optional[Set[str]] = None, updates: Properties = EMPTY_DICT @@ -112,7 +116,7 @@ def update_namespace_properties( if identifier == ("default",): return PropertiesUpdateSummary(removed=["location"], updated=[], missing=[]) else: - raise NoSuchNamespaceError(f"Namespace does not exists: {namespace}") + raise NoSuchNamespaceError(f"Namespace does not exist: {namespace}") MOCK_ENVIRONMENT = {"PYICEBERG_URI": "test://doesnotexist"} @@ -155,11 +159,11 @@ def test_describe_namespace(): @mock.patch.dict(os.environ, MOCK_ENVIRONMENT) @mock.patch("pyiceberg.cli.console.SUPPORTED_CATALOGS", MOCK_CATALOGS) -def test_describe_namespace_does_not_exists(): +def test_describe_namespace_does_not_exist(): runner = CliRunner() - result = runner.invoke(run, ["describe", "doesnotexists"]) + result = runner.invoke(run, ["describe", "doesnotexist"]) assert result.exit_code == 1 - assert result.output == "Table or namespace does not exist: doesnotexists\n" + assert result.output == "Namespace does not exist: doesnotexist\n" @mock.patch.dict(os.environ, MOCK_ENVIRONMENT) @@ -196,11 +200,11 @@ def test_describe_table(): @mock.patch.dict(os.environ, MOCK_ENVIRONMENT) @mock.patch("pyiceberg.cli.console.SUPPORTED_CATALOGS", MOCK_CATALOGS) -def test_describe_table_does_not_exists(): +def test_describe_table_does_not_exist(): runner = CliRunner() result = runner.invoke(run, ["describe", "default.doesnotexit"]) assert result.exit_code == 1 - assert result.output == "Table does not exist: default.doesnotexit\n" + assert result.output == "Table or namespace does not exist: default.doesnotexit\n" @mock.patch.dict(os.environ, MOCK_ENVIRONMENT) @@ -220,9 +224,9 @@ def test_schema(): @mock.patch.dict(os.environ, MOCK_ENVIRONMENT) @mock.patch("pyiceberg.cli.console.SUPPORTED_CATALOGS", MOCK_CATALOGS) -def test_schema_does_not_exists(): +def test_schema_does_not_exist(): runner = CliRunner() - result = runner.invoke(run, ["describe", "default.doesnotexit"]) + result = runner.invoke(run, ["schema", "default.doesnotexit"]) assert result.exit_code == 1 assert result.output == "Table does not exist: default.doesnotexit\n" @@ -244,7 +248,7 @@ def test_spec(): @mock.patch.dict(os.environ, MOCK_ENVIRONMENT) @mock.patch("pyiceberg.cli.console.SUPPORTED_CATALOGS", MOCK_CATALOGS) -def test_spec_does_not_exists(): +def test_spec_does_not_exist(): runner = CliRunner() result = runner.invoke(run, ["spec", "default.doesnotexit"]) assert result.exit_code == 1 @@ -262,7 +266,7 @@ def test_uuid(): @mock.patch.dict(os.environ, MOCK_ENVIRONMENT) @mock.patch("pyiceberg.cli.console.SUPPORTED_CATALOGS", MOCK_CATALOGS) -def test_uuid_does_not_exists(): +def test_uuid_does_not_exist(): runner = CliRunner() result = runner.invoke(run, ["uuid", "default.doesnotexit"]) assert result.exit_code == 1 @@ -280,7 +284,7 @@ def test_location(): @mock.patch.dict(os.environ, MOCK_ENVIRONMENT) @mock.patch("pyiceberg.cli.console.SUPPORTED_CATALOGS", MOCK_CATALOGS) -def test_location_does_not_exists(): +def test_location_does_not_exist(): runner = CliRunner() result = runner.invoke(run, ["location", "default.doesnotexit"]) assert result.exit_code == 1 @@ -298,7 +302,7 @@ def test_drop_table(): @mock.patch.dict(os.environ, MOCK_ENVIRONMENT) @mock.patch("pyiceberg.cli.console.SUPPORTED_CATALOGS", MOCK_CATALOGS) -def test_drop_table_does_not_exists(): +def test_drop_table_does_not_exist(): runner = CliRunner() result = runner.invoke(run, ["drop", "table", "default.doesnotexit"]) assert result.exit_code == 1 @@ -316,7 +320,7 @@ def test_drop_namespace(): @mock.patch.dict(os.environ, MOCK_ENVIRONMENT) @mock.patch("pyiceberg.cli.console.SUPPORTED_CATALOGS", MOCK_CATALOGS) -def test_drop_namespace_does_not_exists(): +def test_drop_namespace_does_not_exist(): runner = CliRunner() result = runner.invoke(run, ["drop", "namespace", "doesnotexit"]) assert result.exit_code == 1 @@ -334,7 +338,7 @@ def test_rename_table(): @mock.patch.dict(os.environ, MOCK_ENVIRONMENT) @mock.patch("pyiceberg.cli.console.SUPPORTED_CATALOGS", MOCK_CATALOGS) -def test_rename_table_does_not_exists(): +def test_rename_table_does_not_exist(): runner = CliRunner() result = runner.invoke(run, ["rename", "default.doesnotexit", "default.bar"]) assert result.exit_code == 1 @@ -363,9 +367,9 @@ def test_properties_get_table_specific_property(): @mock.patch("pyiceberg.cli.console.SUPPORTED_CATALOGS", MOCK_CATALOGS) def test_properties_get_table_specific_property_that_doesnt_exist(): runner = CliRunner() - result = runner.invoke(run, ["properties", "get", "default.fokko", "doesnotexist"]) + result = runner.invoke(run, ["properties", "get", "default.foo", "doesnotexist"]) assert result.exit_code == 1 - assert result.output == "Table or namespace does not exist: default.fokko with property doesnotexist\n" + assert result.output == "Could not find property doesnotexist on table default.foo\n" @mock.patch.dict(os.environ, MOCK_ENVIRONMENT) @@ -374,7 +378,7 @@ def test_properties_get_table_does_not_exist(): runner = CliRunner() result = runner.invoke(run, ["properties", "get", "doesnotexist"]) assert result.exit_code == 1 - assert result.output == "Table or namespace does not exist: doesnotexist\n" + assert result.output == "Namespace does not exist: doesnotexist\n" @mock.patch.dict(os.environ, MOCK_ENVIRONMENT) @@ -408,9 +412,9 @@ def test_properties_set_namespace(): @mock.patch("pyiceberg.cli.console.SUPPORTED_CATALOGS", MOCK_CATALOGS) def test_properties_set_namespace_that_doesnt_exist(): runner = CliRunner() - result = runner.invoke(run, ["properties", "set", "namespace", "doesnotexists", "location", "s3://new_location"]) + result = runner.invoke(run, ["properties", "set", "namespace", "doesnotexist", "location", "s3://new_location"]) assert result.exit_code == 1 - assert result.output == "Namespace does not exists: doesnotexists\n" + assert result.output == "Namespace does not exist: doesnotexist\n" @mock.patch.dict(os.environ, MOCK_ENVIRONMENT) @@ -444,9 +448,9 @@ def test_properties_remove_namespace(): @mock.patch("pyiceberg.cli.console.SUPPORTED_CATALOGS", MOCK_CATALOGS) def test_properties_remove_namespace_that_doesnt_exist(): runner = CliRunner() - result = runner.invoke(run, ["properties", "remove", "namespace", "doesnotexists", "location"]) + result = runner.invoke(run, ["properties", "remove", "namespace", "doesnotexist", "location"]) assert result.exit_code == 1 - assert result.output == "Namespace does not exists: doesnotexists\n" + assert result.output == "Namespace does not exist: doesnotexist\n" @mock.patch.dict(os.environ, MOCK_ENVIRONMENT) @@ -460,11 +464,11 @@ def test_properties_remove_table(): @mock.patch.dict(os.environ, MOCK_ENVIRONMENT) @mock.patch("pyiceberg.cli.console.SUPPORTED_CATALOGS", MOCK_CATALOGS) -def test_properties_remove_table_property_does_not_exists(): +def test_properties_remove_table_property_does_not_exist(): runner = CliRunner() result = runner.invoke(run, ["properties", "remove", "table", "default.foo", "doesnotexist"]) assert result.exit_code == 1 - assert result.output == "Property doesnotexist does not exists on default.foo\n" + assert result.output == "Property doesnotexist does not exist on default.foo\n" @mock.patch.dict(os.environ, MOCK_ENVIRONMENT) @@ -505,11 +509,11 @@ def test_json_describe_namespace(): @mock.patch.dict(os.environ, MOCK_ENVIRONMENT) @mock.patch("pyiceberg.cli.console.SUPPORTED_CATALOGS", MOCK_CATALOGS) -def test_json_describe_namespace_does_not_exists(): +def test_json_describe_namespace_does_not_exist(): runner = CliRunner() - result = runner.invoke(run, ["--output=json", "describe", "doesnotexists"]) + result = runner.invoke(run, ["--output=json", "describe", "doesnotexist"]) assert result.exit_code == 1 - assert result.output == """{"type": "NoSuchTableError", "message": "Table or namespace does not exist: doesnotexists"}\n""" + assert result.output == """{"type": "NoSuchNamespaceError", "message": "Namespace does not exist: doesnotexist"}\n""" @mock.patch.dict(os.environ, MOCK_ENVIRONMENT) @@ -526,11 +530,13 @@ def test_json_describe_table(): @mock.patch.dict(os.environ, MOCK_ENVIRONMENT) @mock.patch("pyiceberg.cli.console.SUPPORTED_CATALOGS", MOCK_CATALOGS) -def test_json_describe_table_does_not_exists(): +def test_json_describe_table_does_not_exist(): runner = CliRunner() result = runner.invoke(run, ["--output=json", "describe", "default.doesnotexit"]) assert result.exit_code == 1 - assert result.output == """{"type": "NoSuchTableError", "message": "Table does not exist: default.doesnotexit"}\n""" + assert ( + result.output == """{"type": "NoSuchTableError", "message": "Table or namespace does not exist: default.doesnotexit"}\n""" + ) @mock.patch.dict(os.environ, MOCK_ENVIRONMENT) @@ -547,9 +553,9 @@ def test_json_schema(): @mock.patch.dict(os.environ, MOCK_ENVIRONMENT) @mock.patch("pyiceberg.cli.console.SUPPORTED_CATALOGS", MOCK_CATALOGS) -def test_json_schema_does_not_exists(): +def test_json_schema_does_not_exist(): runner = CliRunner() - result = runner.invoke(run, ["--output=json", "describe", "default.doesnotexit"]) + result = runner.invoke(run, ["--output=json", "schema", "default.doesnotexit"]) assert result.exit_code == 1 assert result.output == """{"type": "NoSuchTableError", "message": "Table does not exist: default.doesnotexit"}\n""" @@ -568,7 +574,7 @@ def test_json_spec(): @mock.patch.dict(os.environ, MOCK_ENVIRONMENT) @mock.patch("pyiceberg.cli.console.SUPPORTED_CATALOGS", MOCK_CATALOGS) -def test_json_spec_does_not_exists(): +def test_json_spec_does_not_exist(): runner = CliRunner() result = runner.invoke(run, ["--output=json", "spec", "default.doesnotexit"]) assert result.exit_code == 1 @@ -586,7 +592,7 @@ def test_json_uuid(): @mock.patch.dict(os.environ, MOCK_ENVIRONMENT) @mock.patch("pyiceberg.cli.console.SUPPORTED_CATALOGS", MOCK_CATALOGS) -def test_json_uuid_does_not_exists(): +def test_json_uuid_does_not_exist(): runner = CliRunner() result = runner.invoke(run, ["--output=json", "uuid", "default.doesnotexit"]) assert result.exit_code == 1 @@ -604,7 +610,7 @@ def test_json_location(): @mock.patch.dict(os.environ, MOCK_ENVIRONMENT) @mock.patch("pyiceberg.cli.console.SUPPORTED_CATALOGS", MOCK_CATALOGS) -def test_json_location_does_not_exists(): +def test_json_location_does_not_exist(): runner = CliRunner() result = runner.invoke(run, ["--output=json", "location", "default.doesnotexist"]) assert result.exit_code == 1 @@ -622,7 +628,7 @@ def test_json_drop_table(): @mock.patch.dict(os.environ, MOCK_ENVIRONMENT) @mock.patch("pyiceberg.cli.console.SUPPORTED_CATALOGS", MOCK_CATALOGS) -def test_json_drop_table_does_not_exists(): +def test_json_drop_table_does_not_exist(): runner = CliRunner() result = runner.invoke(run, ["--output=json", "drop", "table", "default.doesnotexist"]) assert result.exit_code == 1 @@ -640,7 +646,7 @@ def test_json_drop_namespace(): @mock.patch.dict(os.environ, MOCK_ENVIRONMENT) @mock.patch("pyiceberg.cli.console.SUPPORTED_CATALOGS", MOCK_CATALOGS) -def test_json_drop_namespace_does_not_exists(): +def test_json_drop_namespace_does_not_exist(): runner = CliRunner() result = runner.invoke(run, ["--output=json", "drop", "namespace", "doesnotexist"]) assert result.exit_code == 1 @@ -658,7 +664,7 @@ def test_json_rename_table(): @mock.patch.dict(os.environ, MOCK_ENVIRONMENT) @mock.patch("pyiceberg.cli.console.SUPPORTED_CATALOGS", MOCK_CATALOGS) -def test_json_rename_table_does_not_exists(): +def test_json_rename_table_does_not_exist(): runner = CliRunner() result = runner.invoke(run, ["--output=json", "rename", "default.doesnotexit", "default.bar"]) assert result.exit_code == 1 @@ -687,11 +693,11 @@ def test_json_properties_get_table_specific_property(): @mock.patch("pyiceberg.cli.console.SUPPORTED_CATALOGS", MOCK_CATALOGS) def test_json_properties_get_table_specific_property_that_doesnt_exist(): runner = CliRunner() - result = runner.invoke(run, ["--output=json", "properties", "get", "default.fokko", "doesnotexist"]) + result = runner.invoke(run, ["--output=json", "properties", "get", "default.foo", "doesnotexist"]) assert result.exit_code == 1 assert ( result.output - == """{"type": "NoSuchNamespaceError", "message": "Table or namespace does not exist: default.fokko with property doesnotexist"}\n""" + == """{"type": "NoSuchPropertyException", "message": "Could not find property doesnotexist on table default.foo"}\n""" ) @@ -701,7 +707,7 @@ def test_json_properties_get_table_does_not_exist(): runner = CliRunner() result = runner.invoke(run, ["--output=json", "properties", "get", "doesnotexist"]) assert result.exit_code == 1 - assert result.output == """{"type": "NoSuchNamespaceError", "message": "Table or namespace does not exist: doesnotexist"}\n""" + assert result.output == """{"type": "NoSuchNamespaceError", "message": "Namespace does not exist: doesnotexist"}\n""" @mock.patch.dict(os.environ, MOCK_ENVIRONMENT) @@ -736,10 +742,10 @@ def test_json_properties_set_namespace(): def test_json_properties_set_namespace_that_doesnt_exist(): runner = CliRunner() result = runner.invoke( - run, ["--output=json", "properties", "set", "namespace", "doesnotexists", "location", "s3://new_location"] + run, ["--output=json", "properties", "set", "namespace", "doesnotexist", "location", "s3://new_location"] ) assert result.exit_code == 1 - assert result.output == """{"type": "NoSuchNamespaceError", "message": "Namespace does not exists: doesnotexists"}\n""" + assert result.output == """{"type": "NoSuchNamespaceError", "message": "Namespace does not exist: doesnotexist"}\n""" @mock.patch.dict(os.environ, MOCK_ENVIRONMENT) @@ -777,7 +783,7 @@ def test_json_properties_remove_namespace_that_doesnt_exist(): runner = CliRunner() result = runner.invoke(run, ["--output=json", "properties", "remove", "namespace", "doesnotexist", "location"]) assert result.exit_code == 1 - assert result.output == """{"type": "NoSuchNamespaceError", "message": "Namespace does not exists: doesnotexist"}\n""" + assert result.output == """{"type": "NoSuchNamespaceError", "message": "Namespace does not exist: doesnotexist"}\n""" @mock.patch.dict(os.environ, MOCK_ENVIRONMENT) @@ -791,13 +797,13 @@ def test_json_properties_remove_table(): @mock.patch.dict(os.environ, MOCK_ENVIRONMENT) @mock.patch("pyiceberg.cli.console.SUPPORTED_CATALOGS", MOCK_CATALOGS) -def test_json_properties_remove_table_property_does_not_exists(): +def test_json_properties_remove_table_property_does_not_exist(): runner = CliRunner() result = runner.invoke(run, ["--output=json", "properties", "remove", "table", "default.foo", "doesnotexist"]) assert result.exit_code == 1 assert ( result.output - == """{"type": "NoSuchPropertyException", "message": "Property doesnotexist does not exists on default.foo"}\n""" + == """{"type": "NoSuchPropertyException", "message": "Property doesnotexist does not exist on default.foo"}\n""" ) From 4cad6b4f567c6bea499b4a28dc28baf2d692ed14 Mon Sep 17 00:00:00 2001 From: Fokko Driesprong Date: Sat, 20 Aug 2022 00:44:27 +0200 Subject: [PATCH 179/642] Python: Add configuration (#5488) --- pyiceberg/catalog/__init__.py | 101 +++++++++++- pyiceberg/catalog/hive.py | 4 +- pyiceberg/catalog/rest.py | 29 ++-- pyiceberg/cli/console.py | 40 ++--- pyiceberg/exceptions.py | 4 + pyiceberg/typedef.py | 8 +- pyiceberg/utils/config.py | 146 +++++++++++++++++ tests/catalog/test_base.py | 2 +- tests/catalog/test_hive.py | 17 +- tests/catalog/test_rest.py | 70 ++++---- tests/cli/test_console.py | 289 +++++++++++++++++----------------- tests/conftest.py | 2 +- tests/utils/test_config.py | 54 +++++++ 13 files changed, 536 insertions(+), 230 deletions(-) create mode 100644 pyiceberg/utils/config.py create mode 100644 tests/utils/test_config.py diff --git a/pyiceberg/catalog/__init__.py b/pyiceberg/catalog/__init__.py index 871db36265..c9ce2aab1f 100644 --- a/pyiceberg/catalog/__init__.py +++ b/pyiceberg/catalog/__init__.py @@ -17,14 +17,92 @@ from __future__ import annotations +import logging from abc import ABC, abstractmethod from dataclasses import dataclass +from enum import Enum +from typing import Callable +from pyiceberg.exceptions import NotInstalledError from pyiceberg.schema import Schema from pyiceberg.table import Table from pyiceberg.table.partitioning import UNPARTITIONED_PARTITION_SPEC, PartitionSpec from pyiceberg.table.sorting import UNSORTED_SORT_ORDER, SortOrder -from pyiceberg.typedef import EMPTY_DICT, Identifier, Properties +from pyiceberg.typedef import ( + EMPTY_DICT, + Identifier, + Properties, + RecursiveDict, +) +from pyiceberg.utils.config import Config, merge_config + +logger = logging.getLogger(__name__) + +_ENV_CONFIG = Config() + +TYPE = "type" + + +class CatalogType(Enum): + REST = "rest" + HIVE = "hive" + + +def load_rest(name: str, conf: Properties) -> Catalog: + from pyiceberg.catalog.rest import RestCatalog + + return RestCatalog(name, **conf) + + +def load_hive(name: str, conf: Properties) -> Catalog: + try: + from pyiceberg.catalog.hive import HiveCatalog + + return HiveCatalog(name, **conf) + except ImportError as exc: + raise NotInstalledError("Apache Hive support not installed: pip install 'pyiceberg[hive]'") from exc + + +AVAILABLE_CATALOGS: dict[CatalogType, Callable[[str, Properties], Catalog]] = { + CatalogType.REST: load_rest, + CatalogType.HIVE: load_hive, +} + + +def infer_catalog_type(catalog_properties: RecursiveDict) -> CatalogType | None: + """Tries to infer the type based on the dict + + Args: + catalog_properties: Catalog properties + + Returns: + The inferred type based on the provided properties + """ + if uri := catalog_properties.get("uri"): + if isinstance(uri, str): + if uri.startswith("http"): + return CatalogType.REST + elif uri.startswith("thrift"): + return CatalogType.HIVE + return None + + +def load_catalog(name: str, **properties: str | None) -> Catalog: + env = _ENV_CONFIG.get_catalog_config(name) + conf = merge_config(env or {}, properties) + + if provided_catalog_type := conf.get(TYPE): + catalog_type = CatalogType[provided_catalog_type.upper()] + else: + if inferred_catalog_type := infer_catalog_type(conf): + catalog_type = inferred_catalog_type + else: + raise ValueError(f"Invalid configuration. Could not determine the catalog type: {properties}") + + if catalog_type: + return AVAILABLE_CATALOGS[catalog_type](name, conf) + + raise ValueError(f"Could not initialize catalog with the following properties: {properties}") @dataclass @@ -48,13 +126,30 @@ class Catalog(ABC): properties (Properties): Catalog properties """ - name: str | None + name: str properties: Properties - def __init__(self, name: str | None, **properties: str): + def __init__(self, name: str, **properties: str): self.name = name self.properties = properties + def property(self, key: str) -> str: + """Returns a property from the properties variable. If it doesn't exist, it will raise an error. + + Args: + key: The key of the property + + Returns: The value of the property + + Raises: + ValueError: When the property cannot be found, with a pointer on how to set the property. + """ + if key not in self.properties: + raise ValueError( + f"{type(self).__name__} expects an {key} property. Please set in config or using environment variable PYICEBERG_CATALOG__{self.name.upper()}__{key.upper()}" + ) + return self.properties[key] + @abstractmethod def create_table( self, diff --git a/pyiceberg/catalog/hive.py b/pyiceberg/catalog/hive.py index e218051fd2..0d4e852aa4 100644 --- a/pyiceberg/catalog/hive.py +++ b/pyiceberg/catalog/hive.py @@ -235,9 +235,9 @@ def identifier_to_database_and_table( return tuple_identifier[0], tuple_identifier[1] - def __init__(self, name: str, uri: str, **properties: str): + def __init__(self, name: str, **properties: str): super().__init__(name, **properties) - self._client = _HiveClient(uri) + self._client = _HiveClient(self.property("uri")) def _convert_hive_into_iceberg(self, table: HiveTable, io: FileIO) -> Table: properties: Dict[str, str] = table.parameters diff --git a/pyiceberg/catalog/rest.py b/pyiceberg/catalog/rest.py index d3500beb29..02509aad72 100644 --- a/pyiceberg/catalog/rest.py +++ b/pyiceberg/catalog/rest.py @@ -164,16 +164,11 @@ class OAuthErrorResponse(IcebergBaseModel): class RestCatalog(Catalog): token: Optional[str] - config: Properties - uri: str def __init__( self, - name: Optional[str], - uri: str, - credential: Optional[str] = None, - token: Optional[str] = None, + name: str, **properties: str, ): """Rest Catalog @@ -182,21 +177,15 @@ def __init__( Args: name: Name to identify the catalog - uri: The base-url of the REST Catalog endpoint - credential: The credentials for authentication against the client - token: The bearer token properties: Properties that are passed along to the configuration """ - self.uri = uri - if credential: - self.token = self._fetch_access_token(credential) - elif token: - self.token = token - else: - self.token = None - self.config = self._fetch_config(properties) super().__init__(name, **properties) + self.uri = self.property("uri") + if credential := self.properties.get("credential"): + properties["token"] = self._fetch_access_token(credential) + super().__init__(name, **self._fetch_config(properties)) + def _check_valid_namespace_identifier(self, identifier: Union[str, Identifier]) -> Identifier: """The identifier should have at least one element""" identifier_tuple = Catalog.identifier_to_tuple(identifier) @@ -210,8 +199,8 @@ def headers(self) -> Properties: "Content-type": "application/json", "X-Client-Version": __version__, } - if self.token: - headers[AUTHORIZATION_HEADER] = f"{BEARER_PREFIX} {self.token}" + if token := self.properties.get("token"): + headers[AUTHORIZATION_HEADER] = f"{BEARER_PREFIX} {token}" return headers def url(self, endpoint: str, prefixed: bool = True, **kwargs) -> str: @@ -229,7 +218,7 @@ def url(self, endpoint: str, prefixed: bool = True, **kwargs) -> str: url = url + "v1/" if url.endswith("/") else url + "/v1/" if prefixed: - url += self.config.get(PREFIX, "") + url += self.properties.get(PREFIX, "") url = url if url.endswith("/") else url + "/" return url + endpoint.format(**kwargs) diff --git a/pyiceberg/cli/console.py b/pyiceberg/cli/console.py index 5ce6cf4c04..3f9647654a 100644 --- a/pyiceberg/cli/console.py +++ b/pyiceberg/cli/console.py @@ -15,7 +15,6 @@ # specific language governing permissions and limitations # under the License. # pylint: disable=broad-except,redefined-builtin,redefined-outer-name -import os from functools import wraps from typing import ( Dict, @@ -28,7 +27,7 @@ import click from click import Context -from pyiceberg.catalog import Catalog +from pyiceberg.catalog import Catalog, load_catalog from pyiceberg.catalog.hive import HiveCatalog from pyiceberg.catalog.rest import RestCatalog from pyiceberg.cli.output import ConsoleOutput, JsonOutput, Output @@ -55,19 +54,17 @@ def wrapper(*args, **kwargs): @click.group() -@click.option("--catalog", default=None) +@click.option("--catalog", default="default") @click.option("--output", type=click.Choice(["text", "json"]), default="text") @click.option("--uri") @click.option("--credential") @click.pass_context -def run(ctx: Context, catalog: Optional[str], output: str, uri: Optional[str], credential: Optional[str]): - uri_env_var = "PYICEBERG_URI" - credential_env_var = "PYICEBERG_CREDENTIAL" - - if not uri: - uri = os.environ.get(uri_env_var) - if not credential: - credential = os.environ.get(credential_env_var) +def run(ctx: Context, catalog: str, output: str, uri: Optional[str], credential: Optional[str]): + properties = {} + if uri: + properties["uri"] = uri + if credential: + properties["credential"] = credential ctx.ensure_object(dict) if output == "text": @@ -75,21 +72,18 @@ def run(ctx: Context, catalog: Optional[str], output: str, uri: Optional[str], c else: ctx.obj["output"] = JsonOutput() - if not uri: - ctx.obj["output"].exception( - ValueError(f"Missing uri. Please provide using --uri or using environment variable {uri_env_var}") - ) + try: + try: + ctx.obj["catalog"] = load_catalog(catalog, **properties) + except ValueError as exc: + raise ValueError( + f"URI missing, please provide using --uri, the config or environment variable PYICEBERG_CATALOG__{catalog.upper()}__URI" + ) from exc + except Exception as e: + ctx.obj["output"].exception(e) ctx.exit(1) - assert uri # for mypy - - for scheme, catalog_type in SUPPORTED_CATALOGS.items(): - if uri.startswith(scheme): - ctx.obj["catalog"] = catalog_type(catalog, uri=uri, credential=credential) # type: ignore - break - if not isinstance(ctx.obj["catalog"], Catalog): - ctx.obj["output"].exception( ValueError("Could not determine catalog type from uri. REST (http/https) and Hive (thrift) is supported") ) diff --git a/pyiceberg/exceptions.py b/pyiceberg/exceptions.py index ea2d542e90..86e8a8102f 100644 --- a/pyiceberg/exceptions.py +++ b/pyiceberg/exceptions.py @@ -78,3 +78,7 @@ class OAuthError(RESTError): class NoSuchPropertyException(Exception): """When a property is missing""" + + +class NotInstalledError(Exception): + """When an optional dependency is not installed""" diff --git a/pyiceberg/typedef.py b/pyiceberg/typedef.py index f9fa9c96f6..dc95702813 100644 --- a/pyiceberg/typedef.py +++ b/pyiceberg/typedef.py @@ -15,7 +15,12 @@ # specific language governing permissions and limitations # under the License. -from typing import Any, Dict, Tuple +from typing import ( + Any, + Dict, + Tuple, + Union, +) class FrozenDict(Dict): @@ -30,3 +35,4 @@ def update(self, *args: Any, **kwargs: Any) -> None: Identifier = Tuple[str, ...] Properties = Dict[str, str] +RecursiveDict = Dict[str, Union[str, "RecursiveDict"]] # type: ignore diff --git a/pyiceberg/utils/config.py b/pyiceberg/utils/config.py new file mode 100644 index 0000000000..c13ec13272 --- /dev/null +++ b/pyiceberg/utils/config.py @@ -0,0 +1,146 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +import logging +import os +from typing import Any, List, Optional + +import yaml + +from pyiceberg.typedef import FrozenDict, RecursiveDict + +PYICEBERG = "pyiceberg_" +CATALOG = "catalog" +HOME = "HOME" +PYICEBERG_HOME = "PYICEBERG_HOME" +PYICEBERG_YML = ".pyiceberg.yaml" + +logger = logging.getLogger(__name__) + + +def _coalesce(lhs: Optional[Any], rhs: Optional[Any]) -> Optional[Any]: + return lhs or rhs + + +def merge_config(lhs: RecursiveDict, rhs: RecursiveDict) -> RecursiveDict: + """merges right-hand side into the left-hand side""" + new_config = lhs.copy() + for rhs_key, rhs_value in rhs.items(): + if rhs_key in new_config: + lhs_value = new_config[rhs_key] + if isinstance(lhs_value, dict) and isinstance(rhs_value, dict): + # If they are both dicts, then we have to go deeper + new_config[rhs_key] = merge_config(lhs_value, rhs_value) + else: + # Take the non-null value, with precedence on rhs + new_config[rhs_key] = _coalesce(rhs_value, lhs_value) + else: + # New key + new_config[rhs_key] = rhs_value + + return new_config + + +def _lowercase_dictionary_keys(input_dict: RecursiveDict) -> RecursiveDict: + """Lowers all the keys of a dictionary in a recursive manner, to make the lookup case-insensitive""" + return {k.lower(): _lowercase_dictionary_keys(v) if isinstance(v, dict) else v for k, v in input_dict.items()} + + +class Config: + config: RecursiveDict + + def __init__(self): + config = self._from_configuration_files() or {} + config = merge_config(config, self._from_environment_variables(config)) + self.config = FrozenDict(**config) + + @staticmethod + def _from_configuration_files() -> Optional[RecursiveDict]: + """Loads the first configuration file that its finds + + Will first look in the PYICEBERG_HOME env variable, + and then in the home directory. + """ + + def _load_yaml(directory: Optional[str]) -> Optional[RecursiveDict]: + if directory: + path = f"{directory.rstrip('/')}/{PYICEBERG_YML}" + if os.path.isfile(path): + with open(path, encoding="utf-8") as f: + file_config = yaml.safe_load(f) + file_config_lowercase = _lowercase_dictionary_keys(file_config) + return file_config_lowercase + return None + + # Give priority to the PYICEBERG_HOME directory + if pyiceberg_home_config := _load_yaml(os.environ.get(PYICEBERG_HOME)): + return pyiceberg_home_config + # Look into the home directory + if pyiceberg_home_config := _load_yaml(os.environ.get(HOME)): + return pyiceberg_home_config + # Didn't find a config + return None + + @staticmethod + def _from_environment_variables(config: RecursiveDict) -> RecursiveDict: + """Reads the environment variables, to check if there are any prepended by PYICEBERG_ + + Args: + config: Existing configuration that's being amended with configuration from environment variables + + Returns: + Amended configuration + """ + + def set_property(_config: RecursiveDict, path: List[str], config_value: str): + while len(path) > 0: + element = path.pop(0) + if len(path) == 0: + # We're at the end + _config[element] = config_value + else: + # We have to go deeper + if element not in _config: + _config[element] = {} + if isinstance(_config[element], dict): + _config = _config[element] # type: ignore + else: + raise ValueError( + f"Incompatible configurations, merging dict with a value: {'.'.join(path)}, value: {config_value}" + ) + + for env_var, config_value in os.environ.items(): + # Make it lowercase to make it case-insensitive + env_var_lower = env_var.lower() + if env_var_lower.startswith(PYICEBERG.lower()): + key = env_var_lower[len(PYICEBERG) :] + parts = key.split("__") + parts_normalized = [part.replace("_", "-") for part in parts] + set_property(config, parts_normalized, config_value) + + return config + + def get_catalog_config(self, catalog_name: str) -> Optional[RecursiveDict]: + if CATALOG in self.config: + catalog_name_lower = catalog_name.lower() + catalogs = self.config[CATALOG] + if not isinstance(catalogs, dict): + raise ValueError(f"Catalog configurations needs to be an object: {catalog_name}") + if catalog_name_lower in catalogs: + catalog_conf = catalogs[catalog_name_lower] + assert isinstance(catalog_conf, dict), f"Configuration path catalogs.{catalog_name_lower} needs to be an object" + return catalog_conf + return None diff --git a/tests/catalog/test_base.py b/tests/catalog/test_base.py index 9ac65ce02e..a01b3f5788 100644 --- a/tests/catalog/test_base.py +++ b/tests/catalog/test_base.py @@ -53,7 +53,7 @@ class InMemoryCatalog(Catalog): __tables: Dict[Identifier, Table] __namespaces: Dict[Identifier, Properties] - def __init__(self, name: str, properties: Properties): + def __init__(self, name: str, **properties: str): super().__init__(name, **properties) self.__tables = {} self.__namespaces = {} diff --git a/tests/catalog/test_hive.py b/tests/catalog/test_hive.py index e6b06e762d..ea7a193c1c 100644 --- a/tests/catalog/test_hive.py +++ b/tests/catalog/test_hive.py @@ -165,6 +165,16 @@ def hive_database(tmp_path_factory) -> HiveDatabase: ) +def test_no_uri_supplied(): + with pytest.raises(ValueError) as exc_info: + HiveCatalog("production") + + assert ( + "HiveCatalog expects an uri property. Please set in config or using environment variable PYICEBERG_CATALOG__PRODUCTION__URI" + in str(exc_info.value) + ) + + def test_check_number_of_namespaces(table_schema_simple: Schema): catalog = HiveCatalog(HIVE_CATALOG_NAME, uri=HIVE_METASTORE_FAKE_URL) @@ -480,13 +490,6 @@ def test_list_namespaces(): catalog._client.__enter__().get_all_databases.assert_called() -def test_list_namespaces_with_parent(): - catalog = HiveCatalog(HIVE_CATALOG_NAME, uri=HIVE_METASTORE_FAKE_URL) - - # Hive does not support hierarchical namespaces - assert catalog.list_namespaces(("accounting",)) == [] - - def test_drop_table(): catalog = HiveCatalog(HIVE_CATALOG_NAME, uri=HIVE_METASTORE_FAKE_URL) diff --git a/tests/catalog/test_rest.py b/tests/catalog/test_rest.py index cff7563ab5..2f112a6075 100644 --- a/tests/catalog/test_rest.py +++ b/tests/catalog/test_rest.py @@ -63,6 +63,16 @@ def rest_mock(requests_mock: Mocker): return requests_mock +def test_no_uri_supplied(): + with pytest.raises(ValueError) as exc_info: + RestCatalog("production") + + assert ( + "RestCatalog expects an uri property. Please set in config or using environment variable PYICEBERG_CATALOG__PRODUCTION__URI" + in str(exc_info.value) + ) + + def test_token_200(rest_mock: Mocker): rest_mock.post( f"{TEST_URI}v1/oauth/tokens", @@ -74,7 +84,7 @@ def test_token_200(rest_mock: Mocker): }, status_code=200, ) - assert RestCatalog("rest", TEST_URI, TEST_CREDENTIALS).token == TEST_TOKEN + assert RestCatalog("rest", uri=TEST_URI, credential=TEST_CREDENTIALS).property("token") == TEST_TOKEN def test_token_400(rest_mock: Mocker): @@ -85,7 +95,7 @@ def test_token_400(rest_mock: Mocker): ) with pytest.raises(OAuthError) as e: - RestCatalog("rest", TEST_URI, credential=TEST_CREDENTIALS) + RestCatalog("rest", uri=TEST_URI, credential=TEST_CREDENTIALS) assert str(e.value) == "invalid_client: Credentials for key invalid_key do not match" @@ -104,7 +114,7 @@ def test_token_401(rest_mock: Mocker): ) with pytest.raises(BadCredentialsError) as e: - RestCatalog("rest", TEST_URI, credential=TEST_CREDENTIALS) + RestCatalog("rest", uri=TEST_URI, credential=TEST_CREDENTIALS) assert message in str(e.value) @@ -116,7 +126,7 @@ def test_list_tables_200(rest_mock: Mocker): status_code=200, ) - assert RestCatalog("rest", TEST_URI, token=TEST_TOKEN).list_tables(namespace) == [("examples", "fooshare")] + assert RestCatalog("rest", uri=TEST_URI, token=TEST_TOKEN).list_tables(namespace) == [("examples", "fooshare")] def test_list_tables_404(rest_mock: Mocker): @@ -133,7 +143,7 @@ def test_list_tables_404(rest_mock: Mocker): status_code=404, ) with pytest.raises(NoSuchNamespaceError) as e: - RestCatalog("rest", TEST_URI, token=TEST_TOKEN).list_tables(namespace) + RestCatalog("rest", uri=TEST_URI, token=TEST_TOKEN).list_tables(namespace) assert "Namespace does not exist" in str(e.value) @@ -143,7 +153,7 @@ def test_list_namespaces_200(rest_mock: Mocker): json={"namespaces": [["default"], ["examples"], ["fokko"], ["system"]]}, status_code=200, ) - assert RestCatalog("rest", TEST_URI, token=TEST_TOKEN).list_namespaces() == [ + assert RestCatalog("rest", uri=TEST_URI, token=TEST_TOKEN).list_namespaces() == [ ("default",), ("examples",), ("fokko",), @@ -169,7 +179,7 @@ def test_create_namespace_200(rest_mock: Mocker): json={"namespace": [namespace], "properties": {}}, status_code=200, ) - RestCatalog("rest", TEST_URI, token=TEST_TOKEN).create_namespace(namespace) + RestCatalog("rest", uri=TEST_URI, token=TEST_TOKEN).create_namespace(namespace) def test_create_namespace_409(rest_mock: Mocker): @@ -186,7 +196,7 @@ def test_create_namespace_409(rest_mock: Mocker): status_code=409, ) with pytest.raises(NamespaceAlreadyExistsError) as e: - RestCatalog("rest", TEST_URI, token=TEST_TOKEN).create_namespace(namespace) + RestCatalog("rest", uri=TEST_URI, token=TEST_TOKEN).create_namespace(namespace) assert "Namespace already exists" in str(e.value) @@ -204,7 +214,7 @@ def test_drop_namespace_404(rest_mock: Mocker): status_code=404, ) with pytest.raises(NoSuchNamespaceError) as e: - RestCatalog("rest", TEST_URI, token=TEST_TOKEN).drop_namespace(namespace) + RestCatalog("rest", uri=TEST_URI, token=TEST_TOKEN).drop_namespace(namespace) assert "Namespace does not exist" in str(e.value) @@ -215,7 +225,7 @@ def test_load_namespace_properties_200(rest_mock: Mocker): json={"namespace": ["fokko"], "properties": {"prop": "yes"}}, status_code=204, ) - assert RestCatalog("rest", TEST_URI, token=TEST_TOKEN).load_namespace_properties(namespace) == {"prop": "yes"} + assert RestCatalog("rest", uri=TEST_URI, token=TEST_TOKEN).load_namespace_properties(namespace) == {"prop": "yes"} def test_load_namespace_properties_404(rest_mock: Mocker): @@ -232,7 +242,7 @@ def test_load_namespace_properties_404(rest_mock: Mocker): status_code=404, ) with pytest.raises(NoSuchNamespaceError) as e: - RestCatalog("rest", TEST_URI, token=TEST_TOKEN).load_namespace_properties(namespace) + RestCatalog("rest", uri=TEST_URI, token=TEST_TOKEN).load_namespace_properties(namespace) assert "Namespace does not exist" in str(e.value) @@ -242,7 +252,9 @@ def test_update_namespace_properties_200(rest_mock: Mocker): json={"removed": [], "updated": ["prop"], "missing": ["abc"]}, status_code=200, ) - response = RestCatalog("rest", TEST_URI, token=TEST_TOKEN).update_namespace_properties(("fokko",), {"abc"}, {"prop": "yes"}) + response = RestCatalog("rest", uri=TEST_URI, token=TEST_TOKEN).update_namespace_properties( + ("fokko",), {"abc"}, {"prop": "yes"} + ) assert response == PropertiesUpdateSummary(removed=[], updated=["prop"], missing=["abc"]) @@ -260,7 +272,7 @@ def test_update_namespace_properties_404(rest_mock: Mocker): status_code=404, ) with pytest.raises(NoSuchNamespaceError) as e: - RestCatalog("rest", TEST_URI, token=TEST_TOKEN).update_namespace_properties(("fokko",), {"abc"}, {"prop": "yes"}) + RestCatalog("rest", uri=TEST_URI, token=TEST_TOKEN).update_namespace_properties(("fokko",), {"abc"}, {"prop": "yes"}) assert "Namespace does not exist" in str(e.value) @@ -337,7 +349,7 @@ def test_load_table_200(rest_mock: Mocker): }, status_code=200, ) - table = RestCatalog("rest", TEST_URI, token=TEST_TOKEN).load_table(("fokko", "table")) + table = RestCatalog("rest", uri=TEST_URI, token=TEST_TOKEN).load_table(("fokko", "table")) assert table == Table( identifier=("rest", "fokko", "table"), metadata_location="s3://warehouse/database/table/metadata/00001-5f2f8166-244c-4eae-ac36-384ecdec81fc.gz.metadata.json", @@ -431,7 +443,7 @@ def test_load_table_404(rest_mock: Mocker): ) with pytest.raises(NoSuchTableError) as e: - RestCatalog("rest", TEST_URI, token=TEST_TOKEN).load_table(("fokko", "does_not_exists")) + RestCatalog("rest", uri=TEST_URI, token=TEST_TOKEN).load_table(("fokko", "does_not_exists")) assert "Table does not exist" in str(e.value) @@ -449,7 +461,7 @@ def test_drop_table_404(rest_mock: Mocker): ) with pytest.raises(NoSuchTableError) as e: - RestCatalog("rest", TEST_URI, token=TEST_TOKEN).drop_table(("fokko", "does_not_exists")) + RestCatalog("rest", uri=TEST_URI, token=TEST_TOKEN).drop_table(("fokko", "does_not_exists")) assert "Table does not exist" in str(e.value) @@ -512,7 +524,7 @@ def test_create_table_200(rest_mock: Mocker, table_schema_simple: Schema): }, status_code=200, ) - table = RestCatalog("rest", TEST_URI, token=TEST_TOKEN).create_table( + table = RestCatalog("rest", uri=TEST_URI, token=TEST_TOKEN).create_table( identifier=("fokko", "fokko2"), schema=table_schema_simple, location=None, @@ -583,7 +595,7 @@ def test_create_table_409(rest_mock, table_schema_simple: Schema): ) with pytest.raises(TableAlreadyExistsError) as e: - RestCatalog("rest", TEST_URI, token=TEST_TOKEN).create_table( + RestCatalog("rest", uri=TEST_URI, token=TEST_TOKEN).create_table( identifier=("fokko", "fokko2"), schema=table_schema_simple, location=None, @@ -604,7 +616,7 @@ def test_delete_namespace_204(rest_mock: Mocker): json={}, status_code=204, ) - RestCatalog("rest", TEST_URI, token=TEST_TOKEN).drop_namespace(namespace) + RestCatalog("rest", uri=TEST_URI, token=TEST_TOKEN).drop_namespace(namespace) def test_delete_table_204(rest_mock: Mocker): @@ -613,7 +625,7 @@ def test_delete_table_204(rest_mock: Mocker): json={}, status_code=204, ) - RestCatalog("rest", TEST_URI, token=TEST_TOKEN).drop_table(("example", "fokko")) + RestCatalog("rest", uri=TEST_URI, token=TEST_TOKEN).drop_table(("example", "fokko")) def test_delete_table_404(rest_mock: Mocker): @@ -629,7 +641,7 @@ def test_delete_table_404(rest_mock: Mocker): status_code=404, ) with pytest.raises(NoSuchTableError) as e: - RestCatalog("rest", TEST_URI, token=TEST_TOKEN).drop_table(("example", "fokko")) + RestCatalog("rest", uri=TEST_URI, token=TEST_TOKEN).drop_table(("example", "fokko")) assert "Table does not exist" in str(e.value) @@ -637,7 +649,7 @@ def test_create_table_missing_namespace(rest_mock: Mocker, table_schema_simple: table = "table" with pytest.raises(NoSuchTableError) as e: # Missing namespace - RestCatalog("rest", TEST_URI, token=TEST_TOKEN).create_table(table, table_schema_simple) + RestCatalog("rest", uri=TEST_URI, token=TEST_TOKEN).create_table(table, table_schema_simple) assert f"Missing namespace or invalid identifier: {table}" in str(e.value) @@ -645,7 +657,7 @@ def test_load_table_invalid_namespace(rest_mock: Mocker): table = "table" with pytest.raises(NoSuchTableError) as e: # Missing namespace - RestCatalog("rest", TEST_URI, token=TEST_TOKEN).load_table(table) + RestCatalog("rest", uri=TEST_URI, token=TEST_TOKEN).load_table(table) assert f"Missing namespace or invalid identifier: {table}" in str(e.value) @@ -653,7 +665,7 @@ def test_drop_table_invalid_namespace(rest_mock: Mocker): table = "table" with pytest.raises(NoSuchTableError) as e: # Missing namespace - RestCatalog("rest", TEST_URI, token=TEST_TOKEN).drop_table(table) + RestCatalog("rest", uri=TEST_URI, token=TEST_TOKEN).drop_table(table) assert f"Missing namespace or invalid identifier: {table}" in str(e.value) @@ -661,33 +673,33 @@ def test_purge_table_invalid_namespace(rest_mock: Mocker): table = "table" with pytest.raises(NoSuchTableError) as e: # Missing namespace - RestCatalog("rest", TEST_URI, token=TEST_TOKEN).purge_table(table) + RestCatalog("rest", uri=TEST_URI, token=TEST_TOKEN).purge_table(table) assert f"Missing namespace or invalid identifier: {table}" in str(e.value) def test_create_namespace_invalid_namespace(rest_mock: Mocker): with pytest.raises(NoSuchNamespaceError) as e: # Missing namespace - RestCatalog("rest", TEST_URI, token=TEST_TOKEN).create_namespace(()) + RestCatalog("rest", uri=TEST_URI, token=TEST_TOKEN).create_namespace(()) assert "Empty namespace identifier" in str(e.value) def test_drop_namespace_invalid_namespace(rest_mock: Mocker): with pytest.raises(NoSuchNamespaceError) as e: # Missing namespace - RestCatalog("rest", TEST_URI, token=TEST_TOKEN).drop_namespace(()) + RestCatalog("rest", uri=TEST_URI, token=TEST_TOKEN).drop_namespace(()) assert "Empty namespace identifier" in str(e.value) def test_load_namespace_properties_invalid_namespace(rest_mock: Mocker): with pytest.raises(NoSuchNamespaceError) as e: # Missing namespace - RestCatalog("rest", TEST_URI, token=TEST_TOKEN).load_namespace_properties(()) + RestCatalog("rest", uri=TEST_URI, token=TEST_TOKEN).load_namespace_properties(()) assert "Empty namespace identifier" in str(e.value) def test_update_namespace_properties_invalid_namespace(rest_mock: Mocker): with pytest.raises(NoSuchNamespaceError) as e: # Missing namespace - RestCatalog("rest", TEST_URI, token=TEST_TOKEN).update_namespace_properties(()) + RestCatalog("rest", uri=TEST_URI, token=TEST_TOKEN).update_namespace_properties(()) assert "Empty namespace identifier" in str(e.value) diff --git a/tests/cli/test_console.py b/tests/cli/test_console.py index 704e6729f5..9f8ac2245a 100644 --- a/tests/cli/test_console.py +++ b/tests/cli/test_console.py @@ -119,20 +119,23 @@ def update_namespace_properties( raise NoSuchNamespaceError(f"Namespace does not exist: {namespace}") -MOCK_ENVIRONMENT = {"PYICEBERG_URI": "test://doesnotexist"} -MOCK_CATALOGS = {"test": MockCatalog} +MOCK_CATALOG = MockCatalog("production", uri="http://somewhere") +MOCK_ENVIRONMENT = {"PYICEBERG_CATALOG__PRODUCTION__URI": "test://doesnotexist"} def test_missing_uri(): runner = CliRunner() result = runner.invoke(run, ["list"]) assert result.exit_code == 1 - assert result.output == "Missing uri. Please provide using --uri or using environment variable \nPYICEBERG_URI\n" + assert ( + result.output + == "URI missing, please provide using --uri, the config or environment variable \nPYICEBERG_CATALOG__DEFAULT__URI\n" + ) @mock.patch.dict(os.environ, MOCK_ENVIRONMENT) -@mock.patch("pyiceberg.cli.console.SUPPORTED_CATALOGS", MOCK_CATALOGS) -def test_list_root(): +@mock.patch("pyiceberg.cli.console.load_catalog", return_value=MOCK_CATALOG) +def test_list_root(_): runner = CliRunner() result = runner.invoke(run, ["list"]) assert result.exit_code == 0 @@ -140,8 +143,8 @@ def test_list_root(): @mock.patch.dict(os.environ, MOCK_ENVIRONMENT) -@mock.patch("pyiceberg.cli.console.SUPPORTED_CATALOGS", MOCK_CATALOGS) -def test_list_namespace(): +@mock.patch("pyiceberg.cli.console.load_catalog", return_value=MOCK_CATALOG) +def test_list_namespace(_): runner = CliRunner() result = runner.invoke(run, ["list", "default"]) assert result.exit_code == 0 @@ -149,8 +152,8 @@ def test_list_namespace(): @mock.patch.dict(os.environ, MOCK_ENVIRONMENT) -@mock.patch("pyiceberg.cli.console.SUPPORTED_CATALOGS", MOCK_CATALOGS) -def test_describe_namespace(): +@mock.patch("pyiceberg.cli.console.load_catalog", return_value=MOCK_CATALOG) +def test_describe_namespace(_): runner = CliRunner() result = runner.invoke(run, ["describe", "default"]) assert result.exit_code == 0 @@ -158,8 +161,8 @@ def test_describe_namespace(): @mock.patch.dict(os.environ, MOCK_ENVIRONMENT) -@mock.patch("pyiceberg.cli.console.SUPPORTED_CATALOGS", MOCK_CATALOGS) -def test_describe_namespace_does_not_exist(): +@mock.patch("pyiceberg.cli.console.load_catalog", return_value=MOCK_CATALOG) +def test_describe_namespace_does_not_exists(_): runner = CliRunner() result = runner.invoke(run, ["describe", "doesnotexist"]) assert result.exit_code == 1 @@ -167,8 +170,8 @@ def test_describe_namespace_does_not_exist(): @mock.patch.dict(os.environ, MOCK_ENVIRONMENT) -@mock.patch("pyiceberg.cli.console.SUPPORTED_CATALOGS", MOCK_CATALOGS) -def test_describe_table(): +@mock.patch("pyiceberg.cli.console.load_catalog", return_value=MOCK_CATALOG) +def test_describe_table(_): runner = CliRunner() result = runner.invoke(run, ["describe", "default.foo"]) assert result.exit_code == 0 @@ -199,8 +202,8 @@ def test_describe_table(): @mock.patch.dict(os.environ, MOCK_ENVIRONMENT) -@mock.patch("pyiceberg.cli.console.SUPPORTED_CATALOGS", MOCK_CATALOGS) -def test_describe_table_does_not_exist(): +@mock.patch("pyiceberg.cli.console.load_catalog", return_value=MOCK_CATALOG) +def test_describe_table_does_not_exists(_): runner = CliRunner() result = runner.invoke(run, ["describe", "default.doesnotexit"]) assert result.exit_code == 1 @@ -208,8 +211,8 @@ def test_describe_table_does_not_exist(): @mock.patch.dict(os.environ, MOCK_ENVIRONMENT) -@mock.patch("pyiceberg.cli.console.SUPPORTED_CATALOGS", MOCK_CATALOGS) -def test_schema(): +@mock.patch("pyiceberg.cli.console.load_catalog", return_value=MOCK_CATALOG) +def test_schema(_): runner = CliRunner() result = runner.invoke(run, ["schema", "default.foo"]) assert result.exit_code == 0 @@ -223,8 +226,8 @@ def test_schema(): @mock.patch.dict(os.environ, MOCK_ENVIRONMENT) -@mock.patch("pyiceberg.cli.console.SUPPORTED_CATALOGS", MOCK_CATALOGS) -def test_schema_does_not_exist(): +@mock.patch("pyiceberg.cli.console.load_catalog", return_value=MOCK_CATALOG) +def test_schema_does_not_exists(_): runner = CliRunner() result = runner.invoke(run, ["schema", "default.doesnotexit"]) assert result.exit_code == 1 @@ -232,8 +235,8 @@ def test_schema_does_not_exist(): @mock.patch.dict(os.environ, MOCK_ENVIRONMENT) -@mock.patch("pyiceberg.cli.console.SUPPORTED_CATALOGS", MOCK_CATALOGS) -def test_spec(): +@mock.patch("pyiceberg.cli.console.load_catalog", return_value=MOCK_CATALOG) +def test_spec(_): runner = CliRunner() result = runner.invoke(run, ["spec", "default.foo"]) assert result.exit_code == 0 @@ -247,8 +250,8 @@ def test_spec(): @mock.patch.dict(os.environ, MOCK_ENVIRONMENT) -@mock.patch("pyiceberg.cli.console.SUPPORTED_CATALOGS", MOCK_CATALOGS) -def test_spec_does_not_exist(): +@mock.patch("pyiceberg.cli.console.load_catalog", return_value=MOCK_CATALOG) +def test_spec_does_not_exists(_): runner = CliRunner() result = runner.invoke(run, ["spec", "default.doesnotexit"]) assert result.exit_code == 1 @@ -256,8 +259,8 @@ def test_spec_does_not_exist(): @mock.patch.dict(os.environ, MOCK_ENVIRONMENT) -@mock.patch("pyiceberg.cli.console.SUPPORTED_CATALOGS", MOCK_CATALOGS) -def test_uuid(): +@mock.patch("pyiceberg.cli.console.load_catalog", return_value=MOCK_CATALOG) +def test_uuid(_): runner = CliRunner() result = runner.invoke(run, ["uuid", "default.foo"]) assert result.exit_code == 0 @@ -265,8 +268,8 @@ def test_uuid(): @mock.patch.dict(os.environ, MOCK_ENVIRONMENT) -@mock.patch("pyiceberg.cli.console.SUPPORTED_CATALOGS", MOCK_CATALOGS) -def test_uuid_does_not_exist(): +@mock.patch("pyiceberg.cli.console.load_catalog", return_value=MOCK_CATALOG) +def test_uuid_does_not_exists(_): runner = CliRunner() result = runner.invoke(run, ["uuid", "default.doesnotexit"]) assert result.exit_code == 1 @@ -274,8 +277,8 @@ def test_uuid_does_not_exist(): @mock.patch.dict(os.environ, MOCK_ENVIRONMENT) -@mock.patch("pyiceberg.cli.console.SUPPORTED_CATALOGS", MOCK_CATALOGS) -def test_location(): +@mock.patch("pyiceberg.cli.console.load_catalog", return_value=MOCK_CATALOG) +def test_location(_): runner = CliRunner() result = runner.invoke(run, ["location", "default.foo"]) assert result.exit_code == 0 @@ -283,8 +286,8 @@ def test_location(): @mock.patch.dict(os.environ, MOCK_ENVIRONMENT) -@mock.patch("pyiceberg.cli.console.SUPPORTED_CATALOGS", MOCK_CATALOGS) -def test_location_does_not_exist(): +@mock.patch("pyiceberg.cli.console.load_catalog", return_value=MOCK_CATALOG) +def test_location_does_not_exists(_): runner = CliRunner() result = runner.invoke(run, ["location", "default.doesnotexit"]) assert result.exit_code == 1 @@ -292,8 +295,8 @@ def test_location_does_not_exist(): @mock.patch.dict(os.environ, MOCK_ENVIRONMENT) -@mock.patch("pyiceberg.cli.console.SUPPORTED_CATALOGS", MOCK_CATALOGS) -def test_drop_table(): +@mock.patch("pyiceberg.cli.console.load_catalog", return_value=MOCK_CATALOG) +def test_drop_table(_): runner = CliRunner() result = runner.invoke(run, ["drop", "table", "default.foo"]) assert result.exit_code == 0 @@ -301,8 +304,8 @@ def test_drop_table(): @mock.patch.dict(os.environ, MOCK_ENVIRONMENT) -@mock.patch("pyiceberg.cli.console.SUPPORTED_CATALOGS", MOCK_CATALOGS) -def test_drop_table_does_not_exist(): +@mock.patch("pyiceberg.cli.console.load_catalog", return_value=MOCK_CATALOG) +def test_drop_table_does_not_exists(_): runner = CliRunner() result = runner.invoke(run, ["drop", "table", "default.doesnotexit"]) assert result.exit_code == 1 @@ -310,8 +313,8 @@ def test_drop_table_does_not_exist(): @mock.patch.dict(os.environ, MOCK_ENVIRONMENT) -@mock.patch("pyiceberg.cli.console.SUPPORTED_CATALOGS", MOCK_CATALOGS) -def test_drop_namespace(): +@mock.patch("pyiceberg.cli.console.load_catalog", return_value=MOCK_CATALOG) +def test_drop_namespace(_): runner = CliRunner() result = runner.invoke(run, ["drop", "namespace", "default"]) assert result.exit_code == 0 @@ -319,8 +322,8 @@ def test_drop_namespace(): @mock.patch.dict(os.environ, MOCK_ENVIRONMENT) -@mock.patch("pyiceberg.cli.console.SUPPORTED_CATALOGS", MOCK_CATALOGS) -def test_drop_namespace_does_not_exist(): +@mock.patch("pyiceberg.cli.console.load_catalog", return_value=MOCK_CATALOG) +def test_drop_namespace_does_not_exists(_): runner = CliRunner() result = runner.invoke(run, ["drop", "namespace", "doesnotexit"]) assert result.exit_code == 1 @@ -328,8 +331,8 @@ def test_drop_namespace_does_not_exist(): @mock.patch.dict(os.environ, MOCK_ENVIRONMENT) -@mock.patch("pyiceberg.cli.console.SUPPORTED_CATALOGS", MOCK_CATALOGS) -def test_rename_table(): +@mock.patch("pyiceberg.cli.console.load_catalog", return_value=MOCK_CATALOG) +def test_rename_table(_): runner = CliRunner() result = runner.invoke(run, ["rename", "default.foo", "default.bar"]) assert result.exit_code == 0 @@ -337,8 +340,8 @@ def test_rename_table(): @mock.patch.dict(os.environ, MOCK_ENVIRONMENT) -@mock.patch("pyiceberg.cli.console.SUPPORTED_CATALOGS", MOCK_CATALOGS) -def test_rename_table_does_not_exist(): +@mock.patch("pyiceberg.cli.console.load_catalog", return_value=MOCK_CATALOG) +def test_rename_table_does_not_exists(_): runner = CliRunner() result = runner.invoke(run, ["rename", "default.doesnotexit", "default.bar"]) assert result.exit_code == 1 @@ -346,8 +349,8 @@ def test_rename_table_does_not_exist(): @mock.patch.dict(os.environ, MOCK_ENVIRONMENT) -@mock.patch("pyiceberg.cli.console.SUPPORTED_CATALOGS", MOCK_CATALOGS) -def test_properties_get_table(): +@mock.patch("pyiceberg.cli.console.load_catalog", return_value=MOCK_CATALOG) +def test_properties_get_table(_): runner = CliRunner() result = runner.invoke(run, ["properties", "get", "default.foo"]) assert result.exit_code == 0 @@ -355,8 +358,8 @@ def test_properties_get_table(): @mock.patch.dict(os.environ, MOCK_ENVIRONMENT) -@mock.patch("pyiceberg.cli.console.SUPPORTED_CATALOGS", MOCK_CATALOGS) -def test_properties_get_table_specific_property(): +@mock.patch("pyiceberg.cli.console.load_catalog", return_value=MOCK_CATALOG) +def test_properties_get_table_specific_property(_): runner = CliRunner() result = runner.invoke(run, ["properties", "get", "default.foo", "read.split.target.size"]) assert result.exit_code == 0 @@ -364,8 +367,8 @@ def test_properties_get_table_specific_property(): @mock.patch.dict(os.environ, MOCK_ENVIRONMENT) -@mock.patch("pyiceberg.cli.console.SUPPORTED_CATALOGS", MOCK_CATALOGS) -def test_properties_get_table_specific_property_that_doesnt_exist(): +@mock.patch("pyiceberg.cli.console.load_catalog", return_value=MOCK_CATALOG) +def test_properties_get_table_specific_property_that_doesnt_exist(_): runner = CliRunner() result = runner.invoke(run, ["properties", "get", "default.foo", "doesnotexist"]) assert result.exit_code == 1 @@ -373,8 +376,8 @@ def test_properties_get_table_specific_property_that_doesnt_exist(): @mock.patch.dict(os.environ, MOCK_ENVIRONMENT) -@mock.patch("pyiceberg.cli.console.SUPPORTED_CATALOGS", MOCK_CATALOGS) -def test_properties_get_table_does_not_exist(): +@mock.patch("pyiceberg.cli.console.load_catalog", return_value=MOCK_CATALOG) +def test_properties_get_table_does_not_exist(_): runner = CliRunner() result = runner.invoke(run, ["properties", "get", "doesnotexist"]) assert result.exit_code == 1 @@ -382,8 +385,8 @@ def test_properties_get_table_does_not_exist(): @mock.patch.dict(os.environ, MOCK_ENVIRONMENT) -@mock.patch("pyiceberg.cli.console.SUPPORTED_CATALOGS", MOCK_CATALOGS) -def test_properties_get_namespace(): +@mock.patch("pyiceberg.cli.console.load_catalog", return_value=MOCK_CATALOG) +def test_properties_get_namespace(_): runner = CliRunner() result = runner.invoke(run, ["properties", "get", "default"]) assert result.exit_code == 0 @@ -391,8 +394,8 @@ def test_properties_get_namespace(): @mock.patch.dict(os.environ, MOCK_ENVIRONMENT) -@mock.patch("pyiceberg.cli.console.SUPPORTED_CATALOGS", MOCK_CATALOGS) -def test_properties_get_namespace_specific_property(): +@mock.patch("pyiceberg.cli.console.load_catalog", return_value=MOCK_CATALOG) +def test_properties_get_namespace_specific_property(_): runner = CliRunner() result = runner.invoke(run, ["properties", "get", "default", "location"]) assert result.exit_code == 0 @@ -400,8 +403,8 @@ def test_properties_get_namespace_specific_property(): @mock.patch.dict(os.environ, MOCK_ENVIRONMENT) -@mock.patch("pyiceberg.cli.console.SUPPORTED_CATALOGS", MOCK_CATALOGS) -def test_properties_set_namespace(): +@mock.patch("pyiceberg.cli.console.load_catalog", return_value=MOCK_CATALOG) +def test_properties_set_namespace(_): runner = CliRunner() result = runner.invoke(run, ["properties", "set", "namespace", "default", "location", "s3://new_location"]) assert result.exit_code == 0 @@ -409,8 +412,8 @@ def test_properties_set_namespace(): @mock.patch.dict(os.environ, MOCK_ENVIRONMENT) -@mock.patch("pyiceberg.cli.console.SUPPORTED_CATALOGS", MOCK_CATALOGS) -def test_properties_set_namespace_that_doesnt_exist(): +@mock.patch("pyiceberg.cli.console.load_catalog", return_value=MOCK_CATALOG) +def test_properties_set_namespace_that_doesnt_exist(_): runner = CliRunner() result = runner.invoke(run, ["properties", "set", "namespace", "doesnotexist", "location", "s3://new_location"]) assert result.exit_code == 1 @@ -418,8 +421,8 @@ def test_properties_set_namespace_that_doesnt_exist(): @mock.patch.dict(os.environ, MOCK_ENVIRONMENT) -@mock.patch("pyiceberg.cli.console.SUPPORTED_CATALOGS", MOCK_CATALOGS) -def test_properties_set_table(): +@mock.patch("pyiceberg.cli.console.load_catalog", return_value=MOCK_CATALOG) +def test_properties_set_table(_): runner = CliRunner() result = runner.invoke(run, ["properties", "set", "table", "default.foo", "location", "s3://new_location"]) assert result.exit_code == 1 @@ -427,8 +430,8 @@ def test_properties_set_table(): @mock.patch.dict(os.environ, MOCK_ENVIRONMENT) -@mock.patch("pyiceberg.cli.console.SUPPORTED_CATALOGS", MOCK_CATALOGS) -def test_properties_set_table_does_not_exist(): +@mock.patch("pyiceberg.cli.console.load_catalog", return_value=MOCK_CATALOG) +def test_properties_set_table_does_not_exist(_): runner = CliRunner() result = runner.invoke(run, ["properties", "set", "table", "default.doesnotexist", "location", "s3://new_location"]) assert result.exit_code == 1 @@ -436,8 +439,8 @@ def test_properties_set_table_does_not_exist(): @mock.patch.dict(os.environ, MOCK_ENVIRONMENT) -@mock.patch("pyiceberg.cli.console.SUPPORTED_CATALOGS", MOCK_CATALOGS) -def test_properties_remove_namespace(): +@mock.patch("pyiceberg.cli.console.load_catalog", return_value=MOCK_CATALOG) +def test_properties_remove_namespace(_): runner = CliRunner() result = runner.invoke(run, ["properties", "remove", "namespace", "default", "location"]) assert result.exit_code == 0 @@ -445,8 +448,8 @@ def test_properties_remove_namespace(): @mock.patch.dict(os.environ, MOCK_ENVIRONMENT) -@mock.patch("pyiceberg.cli.console.SUPPORTED_CATALOGS", MOCK_CATALOGS) -def test_properties_remove_namespace_that_doesnt_exist(): +@mock.patch("pyiceberg.cli.console.load_catalog", return_value=MOCK_CATALOG) +def test_properties_remove_namespace_that_doesnt_exist(_): runner = CliRunner() result = runner.invoke(run, ["properties", "remove", "namespace", "doesnotexist", "location"]) assert result.exit_code == 1 @@ -454,8 +457,8 @@ def test_properties_remove_namespace_that_doesnt_exist(): @mock.patch.dict(os.environ, MOCK_ENVIRONMENT) -@mock.patch("pyiceberg.cli.console.SUPPORTED_CATALOGS", MOCK_CATALOGS) -def test_properties_remove_table(): +@mock.patch("pyiceberg.cli.console.load_catalog", return_value=MOCK_CATALOG) +def test_properties_remove_table(_): runner = CliRunner() result = runner.invoke(run, ["properties", "remove", "table", "default.foo", "read.split.target.size"]) assert result.exit_code == 1 @@ -463,8 +466,8 @@ def test_properties_remove_table(): @mock.patch.dict(os.environ, MOCK_ENVIRONMENT) -@mock.patch("pyiceberg.cli.console.SUPPORTED_CATALOGS", MOCK_CATALOGS) -def test_properties_remove_table_property_does_not_exist(): +@mock.patch("pyiceberg.cli.console.load_catalog", return_value=MOCK_CATALOG) +def test_properties_remove_table_property_does_not_exists(_): runner = CliRunner() result = runner.invoke(run, ["properties", "remove", "table", "default.foo", "doesnotexist"]) assert result.exit_code == 1 @@ -472,8 +475,8 @@ def test_properties_remove_table_property_does_not_exist(): @mock.patch.dict(os.environ, MOCK_ENVIRONMENT) -@mock.patch("pyiceberg.cli.console.SUPPORTED_CATALOGS", MOCK_CATALOGS) -def test_properties_remove_table_does_not_exist(): +@mock.patch("pyiceberg.cli.console.load_catalog", return_value=MOCK_CATALOG) +def test_properties_remove_table_does_not_exist(_): runner = CliRunner() result = runner.invoke(run, ["properties", "remove", "table", "default.doesnotexist", "location"]) assert result.exit_code == 1 @@ -481,8 +484,8 @@ def test_properties_remove_table_does_not_exist(): @mock.patch.dict(os.environ, MOCK_ENVIRONMENT) -@mock.patch("pyiceberg.cli.console.SUPPORTED_CATALOGS", MOCK_CATALOGS) -def test_json_list_root(): +@mock.patch("pyiceberg.cli.console.load_catalog", return_value=MOCK_CATALOG) +def test_json_list_root(_): runner = CliRunner() result = runner.invoke(run, ["--output=json", "list"]) assert result.exit_code == 0 @@ -490,8 +493,8 @@ def test_json_list_root(): @mock.patch.dict(os.environ, MOCK_ENVIRONMENT) -@mock.patch("pyiceberg.cli.console.SUPPORTED_CATALOGS", MOCK_CATALOGS) -def test_json_list_namespace(): +@mock.patch("pyiceberg.cli.console.load_catalog", return_value=MOCK_CATALOG) +def test_json_list_namespace(_): runner = CliRunner() result = runner.invoke(run, ["--output=json", "list", "default"]) assert result.exit_code == 0 @@ -499,8 +502,8 @@ def test_json_list_namespace(): @mock.patch.dict(os.environ, MOCK_ENVIRONMENT) -@mock.patch("pyiceberg.cli.console.SUPPORTED_CATALOGS", MOCK_CATALOGS) -def test_json_describe_namespace(): +@mock.patch("pyiceberg.cli.console.load_catalog", return_value=MOCK_CATALOG) +def test_json_describe_namespace(_): runner = CliRunner() result = runner.invoke(run, ["--output=json", "describe", "default"]) assert result.exit_code == 0 @@ -508,8 +511,8 @@ def test_json_describe_namespace(): @mock.patch.dict(os.environ, MOCK_ENVIRONMENT) -@mock.patch("pyiceberg.cli.console.SUPPORTED_CATALOGS", MOCK_CATALOGS) -def test_json_describe_namespace_does_not_exist(): +@mock.patch("pyiceberg.cli.console.load_catalog", return_value=MOCK_CATALOG) +def test_json_describe_namespace_does_not_exists(_): runner = CliRunner() result = runner.invoke(run, ["--output=json", "describe", "doesnotexist"]) assert result.exit_code == 1 @@ -517,8 +520,8 @@ def test_json_describe_namespace_does_not_exist(): @mock.patch.dict(os.environ, MOCK_ENVIRONMENT) -@mock.patch("pyiceberg.cli.console.SUPPORTED_CATALOGS", MOCK_CATALOGS) -def test_json_describe_table(): +@mock.patch("pyiceberg.cli.console.load_catalog", return_value=MOCK_CATALOG) +def test_json_describe_table(_): runner = CliRunner() result = runner.invoke(run, ["--output=json", "describe", "default.foo"]) assert result.exit_code == 0 @@ -529,8 +532,8 @@ def test_json_describe_table(): @mock.patch.dict(os.environ, MOCK_ENVIRONMENT) -@mock.patch("pyiceberg.cli.console.SUPPORTED_CATALOGS", MOCK_CATALOGS) -def test_json_describe_table_does_not_exist(): +@mock.patch("pyiceberg.cli.console.load_catalog", return_value=MOCK_CATALOG) +def test_json_describe_table_does_not_exists(_): runner = CliRunner() result = runner.invoke(run, ["--output=json", "describe", "default.doesnotexit"]) assert result.exit_code == 1 @@ -540,8 +543,8 @@ def test_json_describe_table_does_not_exist(): @mock.patch.dict(os.environ, MOCK_ENVIRONMENT) -@mock.patch("pyiceberg.cli.console.SUPPORTED_CATALOGS", MOCK_CATALOGS) -def test_json_schema(): +@mock.patch("pyiceberg.cli.console.load_catalog", return_value=MOCK_CATALOG) +def test_json_schema(_): runner = CliRunner() result = runner.invoke(run, ["--output=json", "schema", "default.foo"]) assert result.exit_code == 0 @@ -552,8 +555,8 @@ def test_json_schema(): @mock.patch.dict(os.environ, MOCK_ENVIRONMENT) -@mock.patch("pyiceberg.cli.console.SUPPORTED_CATALOGS", MOCK_CATALOGS) -def test_json_schema_does_not_exist(): +@mock.patch("pyiceberg.cli.console.load_catalog", return_value=MOCK_CATALOG) +def test_json_schema_does_not_exists(_): runner = CliRunner() result = runner.invoke(run, ["--output=json", "schema", "default.doesnotexit"]) assert result.exit_code == 1 @@ -561,8 +564,8 @@ def test_json_schema_does_not_exist(): @mock.patch.dict(os.environ, MOCK_ENVIRONMENT) -@mock.patch("pyiceberg.cli.console.SUPPORTED_CATALOGS", MOCK_CATALOGS) -def test_json_spec(): +@mock.patch("pyiceberg.cli.console.load_catalog", return_value=MOCK_CATALOG) +def test_json_spec(_): runner = CliRunner() result = runner.invoke(run, ["--output=json", "spec", "default.foo"]) assert result.exit_code == 0 @@ -573,8 +576,8 @@ def test_json_spec(): @mock.patch.dict(os.environ, MOCK_ENVIRONMENT) -@mock.patch("pyiceberg.cli.console.SUPPORTED_CATALOGS", MOCK_CATALOGS) -def test_json_spec_does_not_exist(): +@mock.patch("pyiceberg.cli.console.load_catalog", return_value=MOCK_CATALOG) +def test_json_spec_does_not_exists(_): runner = CliRunner() result = runner.invoke(run, ["--output=json", "spec", "default.doesnotexit"]) assert result.exit_code == 1 @@ -582,8 +585,8 @@ def test_json_spec_does_not_exist(): @mock.patch.dict(os.environ, MOCK_ENVIRONMENT) -@mock.patch("pyiceberg.cli.console.SUPPORTED_CATALOGS", MOCK_CATALOGS) -def test_json_uuid(): +@mock.patch("pyiceberg.cli.console.load_catalog", return_value=MOCK_CATALOG) +def test_json_uuid(_): runner = CliRunner() result = runner.invoke(run, ["--output=json", "uuid", "default.foo"]) assert result.exit_code == 0 @@ -591,8 +594,8 @@ def test_json_uuid(): @mock.patch.dict(os.environ, MOCK_ENVIRONMENT) -@mock.patch("pyiceberg.cli.console.SUPPORTED_CATALOGS", MOCK_CATALOGS) -def test_json_uuid_does_not_exist(): +@mock.patch("pyiceberg.cli.console.load_catalog", return_value=MOCK_CATALOG) +def test_json_uuid_does_not_exists(_): runner = CliRunner() result = runner.invoke(run, ["--output=json", "uuid", "default.doesnotexit"]) assert result.exit_code == 1 @@ -600,8 +603,8 @@ def test_json_uuid_does_not_exist(): @mock.patch.dict(os.environ, MOCK_ENVIRONMENT) -@mock.patch("pyiceberg.cli.console.SUPPORTED_CATALOGS", MOCK_CATALOGS) -def test_json_location(): +@mock.patch("pyiceberg.cli.console.load_catalog", return_value=MOCK_CATALOG) +def test_json_location(_): runner = CliRunner() result = runner.invoke(run, ["--output=json", "location", "default.foo"]) assert result.exit_code == 0 @@ -609,8 +612,8 @@ def test_json_location(): @mock.patch.dict(os.environ, MOCK_ENVIRONMENT) -@mock.patch("pyiceberg.cli.console.SUPPORTED_CATALOGS", MOCK_CATALOGS) -def test_json_location_does_not_exist(): +@mock.patch("pyiceberg.cli.console.load_catalog", return_value=MOCK_CATALOG) +def test_json_location_does_not_exists(_): runner = CliRunner() result = runner.invoke(run, ["--output=json", "location", "default.doesnotexist"]) assert result.exit_code == 1 @@ -618,8 +621,8 @@ def test_json_location_does_not_exist(): @mock.patch.dict(os.environ, MOCK_ENVIRONMENT) -@mock.patch("pyiceberg.cli.console.SUPPORTED_CATALOGS", MOCK_CATALOGS) -def test_json_drop_table(): +@mock.patch("pyiceberg.cli.console.load_catalog", return_value=MOCK_CATALOG) +def test_json_drop_table(_): runner = CliRunner() result = runner.invoke(run, ["--output=json", "drop", "table", "default.foo"]) assert result.exit_code == 0 @@ -627,8 +630,8 @@ def test_json_drop_table(): @mock.patch.dict(os.environ, MOCK_ENVIRONMENT) -@mock.patch("pyiceberg.cli.console.SUPPORTED_CATALOGS", MOCK_CATALOGS) -def test_json_drop_table_does_not_exist(): +@mock.patch("pyiceberg.cli.console.load_catalog", return_value=MOCK_CATALOG) +def test_json_drop_table_does_not_exists(_): runner = CliRunner() result = runner.invoke(run, ["--output=json", "drop", "table", "default.doesnotexist"]) assert result.exit_code == 1 @@ -636,8 +639,8 @@ def test_json_drop_table_does_not_exist(): @mock.patch.dict(os.environ, MOCK_ENVIRONMENT) -@mock.patch("pyiceberg.cli.console.SUPPORTED_CATALOGS", MOCK_CATALOGS) -def test_json_drop_namespace(): +@mock.patch("pyiceberg.cli.console.load_catalog", return_value=MOCK_CATALOG) +def test_json_drop_namespace(_): runner = CliRunner() result = runner.invoke(run, ["--output=json", "drop", "namespace", "default"]) assert result.exit_code == 0 @@ -645,8 +648,8 @@ def test_json_drop_namespace(): @mock.patch.dict(os.environ, MOCK_ENVIRONMENT) -@mock.patch("pyiceberg.cli.console.SUPPORTED_CATALOGS", MOCK_CATALOGS) -def test_json_drop_namespace_does_not_exist(): +@mock.patch("pyiceberg.cli.console.load_catalog", return_value=MOCK_CATALOG) +def test_json_drop_namespace_does_not_exists(_): runner = CliRunner() result = runner.invoke(run, ["--output=json", "drop", "namespace", "doesnotexist"]) assert result.exit_code == 1 @@ -654,8 +657,8 @@ def test_json_drop_namespace_does_not_exist(): @mock.patch.dict(os.environ, MOCK_ENVIRONMENT) -@mock.patch("pyiceberg.cli.console.SUPPORTED_CATALOGS", MOCK_CATALOGS) -def test_json_rename_table(): +@mock.patch("pyiceberg.cli.console.load_catalog", return_value=MOCK_CATALOG) +def test_json_rename_table(_): runner = CliRunner() result = runner.invoke(run, ["--output=json", "rename", "default.foo", "default.bar"]) assert result.exit_code == 0 @@ -663,8 +666,8 @@ def test_json_rename_table(): @mock.patch.dict(os.environ, MOCK_ENVIRONMENT) -@mock.patch("pyiceberg.cli.console.SUPPORTED_CATALOGS", MOCK_CATALOGS) -def test_json_rename_table_does_not_exist(): +@mock.patch("pyiceberg.cli.console.load_catalog", return_value=MOCK_CATALOG) +def test_json_rename_table_does_not_exists(_): runner = CliRunner() result = runner.invoke(run, ["--output=json", "rename", "default.doesnotexit", "default.bar"]) assert result.exit_code == 1 @@ -672,8 +675,8 @@ def test_json_rename_table_does_not_exist(): @mock.patch.dict(os.environ, MOCK_ENVIRONMENT) -@mock.patch("pyiceberg.cli.console.SUPPORTED_CATALOGS", MOCK_CATALOGS) -def test_json_properties_get_table(): +@mock.patch("pyiceberg.cli.console.load_catalog", return_value=MOCK_CATALOG) +def test_json_properties_get_table(_): runner = CliRunner() result = runner.invoke(run, ["--output=json", "properties", "get", "default.foo"]) assert result.exit_code == 0 @@ -681,8 +684,8 @@ def test_json_properties_get_table(): @mock.patch.dict(os.environ, MOCK_ENVIRONMENT) -@mock.patch("pyiceberg.cli.console.SUPPORTED_CATALOGS", MOCK_CATALOGS) -def test_json_properties_get_table_specific_property(): +@mock.patch("pyiceberg.cli.console.load_catalog", return_value=MOCK_CATALOG) +def test_json_properties_get_table_specific_property(_): runner = CliRunner() result = runner.invoke(run, ["--output=json", "properties", "get", "default.foo", "read.split.target.size"]) assert result.exit_code == 0 @@ -690,8 +693,8 @@ def test_json_properties_get_table_specific_property(): @mock.patch.dict(os.environ, MOCK_ENVIRONMENT) -@mock.patch("pyiceberg.cli.console.SUPPORTED_CATALOGS", MOCK_CATALOGS) -def test_json_properties_get_table_specific_property_that_doesnt_exist(): +@mock.patch("pyiceberg.cli.console.load_catalog", return_value=MOCK_CATALOG) +def test_json_properties_get_table_specific_property_that_doesnt_exist(_): runner = CliRunner() result = runner.invoke(run, ["--output=json", "properties", "get", "default.foo", "doesnotexist"]) assert result.exit_code == 1 @@ -702,8 +705,8 @@ def test_json_properties_get_table_specific_property_that_doesnt_exist(): @mock.patch.dict(os.environ, MOCK_ENVIRONMENT) -@mock.patch("pyiceberg.cli.console.SUPPORTED_CATALOGS", MOCK_CATALOGS) -def test_json_properties_get_table_does_not_exist(): +@mock.patch("pyiceberg.cli.console.load_catalog", return_value=MOCK_CATALOG) +def test_json_properties_get_table_does_not_exist(_): runner = CliRunner() result = runner.invoke(run, ["--output=json", "properties", "get", "doesnotexist"]) assert result.exit_code == 1 @@ -711,8 +714,8 @@ def test_json_properties_get_table_does_not_exist(): @mock.patch.dict(os.environ, MOCK_ENVIRONMENT) -@mock.patch("pyiceberg.cli.console.SUPPORTED_CATALOGS", MOCK_CATALOGS) -def test_json_properties_get_namespace(): +@mock.patch("pyiceberg.cli.console.load_catalog", return_value=MOCK_CATALOG) +def test_json_properties_get_namespace(_): runner = CliRunner() result = runner.invoke(run, ["--output=json", "properties", "get", "default"]) assert result.exit_code == 0 @@ -720,8 +723,8 @@ def test_json_properties_get_namespace(): @mock.patch.dict(os.environ, MOCK_ENVIRONMENT) -@mock.patch("pyiceberg.cli.console.SUPPORTED_CATALOGS", MOCK_CATALOGS) -def test_json_properties_get_namespace_specific_property(): +@mock.patch("pyiceberg.cli.console.load_catalog", return_value=MOCK_CATALOG) +def test_json_properties_get_namespace_specific_property(_): runner = CliRunner() result = runner.invoke(run, ["--output=json", "properties", "get", "default", "location"]) assert result.exit_code == 0 @@ -729,8 +732,8 @@ def test_json_properties_get_namespace_specific_property(): @mock.patch.dict(os.environ, MOCK_ENVIRONMENT) -@mock.patch("pyiceberg.cli.console.SUPPORTED_CATALOGS", MOCK_CATALOGS) -def test_json_properties_set_namespace(): +@mock.patch("pyiceberg.cli.console.load_catalog", return_value=MOCK_CATALOG) +def test_json_properties_set_namespace(_): runner = CliRunner() result = runner.invoke(run, ["--output=json", "properties", "set", "namespace", "default", "location", "s3://new_location"]) assert result.exit_code == 0 @@ -738,8 +741,8 @@ def test_json_properties_set_namespace(): @mock.patch.dict(os.environ, MOCK_ENVIRONMENT) -@mock.patch("pyiceberg.cli.console.SUPPORTED_CATALOGS", MOCK_CATALOGS) -def test_json_properties_set_namespace_that_doesnt_exist(): +@mock.patch("pyiceberg.cli.console.load_catalog", return_value=MOCK_CATALOG) +def test_json_properties_set_namespace_that_doesnt_exist(_): runner = CliRunner() result = runner.invoke( run, ["--output=json", "properties", "set", "namespace", "doesnotexist", "location", "s3://new_location"] @@ -749,8 +752,8 @@ def test_json_properties_set_namespace_that_doesnt_exist(): @mock.patch.dict(os.environ, MOCK_ENVIRONMENT) -@mock.patch("pyiceberg.cli.console.SUPPORTED_CATALOGS", MOCK_CATALOGS) -def test_json_properties_set_table(): +@mock.patch("pyiceberg.cli.console.load_catalog", return_value=MOCK_CATALOG) +def test_json_properties_set_table(_): runner = CliRunner() result = runner.invoke(run, ["--output=json", "properties", "set", "table", "default.foo", "location", "s3://new_location"]) assert result.exit_code == 1 @@ -758,8 +761,8 @@ def test_json_properties_set_table(): @mock.patch.dict(os.environ, MOCK_ENVIRONMENT) -@mock.patch("pyiceberg.cli.console.SUPPORTED_CATALOGS", MOCK_CATALOGS) -def test_json_properties_set_table_does_not_exist(): +@mock.patch("pyiceberg.cli.console.load_catalog", return_value=MOCK_CATALOG) +def test_json_properties_set_table_does_not_exist(_): runner = CliRunner() result = runner.invoke( run, ["--output=json", "properties", "set", "table", "default.doesnotexist", "location", "s3://new_location"] @@ -769,8 +772,8 @@ def test_json_properties_set_table_does_not_exist(): @mock.patch.dict(os.environ, MOCK_ENVIRONMENT) -@mock.patch("pyiceberg.cli.console.SUPPORTED_CATALOGS", MOCK_CATALOGS) -def test_json_properties_remove_namespace(): +@mock.patch("pyiceberg.cli.console.load_catalog", return_value=MOCK_CATALOG) +def test_json_properties_remove_namespace(_): runner = CliRunner() result = runner.invoke(run, ["--output=json", "properties", "remove", "namespace", "default", "location"]) assert result.exit_code == 0 @@ -778,8 +781,8 @@ def test_json_properties_remove_namespace(): @mock.patch.dict(os.environ, MOCK_ENVIRONMENT) -@mock.patch("pyiceberg.cli.console.SUPPORTED_CATALOGS", MOCK_CATALOGS) -def test_json_properties_remove_namespace_that_doesnt_exist(): +@mock.patch("pyiceberg.cli.console.load_catalog", return_value=MOCK_CATALOG) +def test_json_properties_remove_namespace_that_doesnt_exist(_): runner = CliRunner() result = runner.invoke(run, ["--output=json", "properties", "remove", "namespace", "doesnotexist", "location"]) assert result.exit_code == 1 @@ -787,8 +790,8 @@ def test_json_properties_remove_namespace_that_doesnt_exist(): @mock.patch.dict(os.environ, MOCK_ENVIRONMENT) -@mock.patch("pyiceberg.cli.console.SUPPORTED_CATALOGS", MOCK_CATALOGS) -def test_json_properties_remove_table(): +@mock.patch("pyiceberg.cli.console.load_catalog", return_value=MOCK_CATALOG) +def test_json_properties_remove_table(_): runner = CliRunner() result = runner.invoke(run, ["--output=json", "properties", "remove", "table", "default.foo", "read.split.target.size"]) assert result.exit_code == 1 @@ -796,8 +799,8 @@ def test_json_properties_remove_table(): @mock.patch.dict(os.environ, MOCK_ENVIRONMENT) -@mock.patch("pyiceberg.cli.console.SUPPORTED_CATALOGS", MOCK_CATALOGS) -def test_json_properties_remove_table_property_does_not_exist(): +@mock.patch("pyiceberg.cli.console.load_catalog", return_value=MOCK_CATALOG) +def test_json_properties_remove_table_property_does_not_exists(_): runner = CliRunner() result = runner.invoke(run, ["--output=json", "properties", "remove", "table", "default.foo", "doesnotexist"]) assert result.exit_code == 1 @@ -808,8 +811,8 @@ def test_json_properties_remove_table_property_does_not_exist(): @mock.patch.dict(os.environ, MOCK_ENVIRONMENT) -@mock.patch("pyiceberg.cli.console.SUPPORTED_CATALOGS", MOCK_CATALOGS) -def test_json_properties_remove_table_does_not_exist(): +@mock.patch("pyiceberg.cli.console.load_catalog", return_value=MOCK_CATALOG) +def test_json_properties_remove_table_does_not_exist(_): runner = CliRunner() result = runner.invoke(run, ["--output=json", "properties", "remove", "table", "default.doesnotexist", "location"]) print(result) diff --git a/tests/conftest.py b/tests/conftest.py index 089443cead..b4d2e005fe 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -289,7 +289,7 @@ def example_table_metadata_v2() -> Dict[str, Any]: @pytest.fixture def catalog() -> InMemoryCatalog: - return InMemoryCatalog("test.in.memory.catalog", {"test.key": "test.value"}) + return InMemoryCatalog("test.in.memory.catalog", **{"test.key": "test.value"}) manifest_entry_records = [ diff --git a/tests/utils/test_config.py b/tests/utils/test_config.py new file mode 100644 index 0000000000..e4a8970876 --- /dev/null +++ b/tests/utils/test_config.py @@ -0,0 +1,54 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +import os +from unittest import mock + +import yaml + +from pyiceberg.utils.config import Config, _lowercase_dictionary_keys + +EXAMPLE_ENV = {"PYICEBERG_CATALOG__PRODUCTION__URI": "https://service.io/api"} + + +def test_config(): + """To check if all the file lookups go well without any mocking""" + assert Config() + + +@mock.patch.dict(os.environ, EXAMPLE_ENV) +def test_from_environment_variables(): + assert Config().get_catalog_config("production") == {"uri": "https://service.io/api"} + + +@mock.patch.dict(os.environ, EXAMPLE_ENV) +def test_from_environment_variables_uppercase(): + assert Config().get_catalog_config("PRODUCTION") == {"uri": "https://service.io/api"} + + +def test_from_configuration_files(tmp_path_factory): + config_path = str(tmp_path_factory.mktemp("config")) + with open(f"{config_path}/.pyiceberg.yaml", "w", encoding="utf-8") as file: + yaml.dump({"catalog": {"production": {"uri": "https://service.io/api"}}}, file) + + os.environ["PYICEBERG_HOME"] = config_path + assert Config().get_catalog_config("production") == {"uri": "https://service.io/api"} + + +def test_lowercase_dictionary_keys(): + uppercase_keys = {"UPPER": {"NESTED_UPPER": {"YES"}}} + expected = {"upper": {"nested_upper": {"YES"}}} + assert _lowercase_dictionary_keys(uppercase_keys) == expected From 211a6f2c5dd1ea62d4cab8af6969c70d1d5964aa Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Sun, 21 Aug 2022 12:18:21 -0700 Subject: [PATCH 180/642] Build: Bump coverage from 6.4.3 to 6.4.4 in /python (#5599) Bumps [coverage](https://github.com/nedbat/coveragepy) from 6.4.3 to 6.4.4. - [Release notes](https://github.com/nedbat/coveragepy/releases) - [Changelog](https://github.com/nedbat/coveragepy/blob/master/CHANGES.rst) - [Commits](https://github.com/nedbat/coveragepy/compare/6.4.3...6.4.4) Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- poetry.lock | 344 ++++++++++++++++++++++++++++++++++++++++++------- pyproject.toml | 2 +- 2 files changed, 299 insertions(+), 47 deletions(-) diff --git a/poetry.lock b/poetry.lock index 4fa41d6c67..bb3eaa2415 100644 --- a/poetry.lock +++ b/poetry.lock @@ -15,10 +15,10 @@ optional = false python-versions = ">=3.5" [package.extras] -dev = ["coverage[toml] (>=5.0.2)", "hypothesis", "pympler", "pytest (>=4.3.0)", "mypy (>=0.900,!=0.940)", "pytest-mypy-plugins", "zope.interface", "furo", "sphinx", "sphinx-notfound-page", "pre-commit", "cloudpickle"] -docs = ["furo", "sphinx", "zope.interface", "sphinx-notfound-page"] -tests = ["coverage[toml] (>=5.0.2)", "hypothesis", "pympler", "pytest (>=4.3.0)", "mypy (>=0.900,!=0.940)", "pytest-mypy-plugins", "zope.interface", "cloudpickle"] -tests_no_zope = ["coverage[toml] (>=5.0.2)", "hypothesis", "pympler", "pytest (>=4.3.0)", "mypy (>=0.900,!=0.940)", "pytest-mypy-plugins", "cloudpickle"] +tests_no_zope = ["cloudpickle", "pytest-mypy-plugins", "mypy (>=0.900,!=0.940)", "pytest (>=4.3.0)", "pympler", "hypothesis", "coverage[toml] (>=5.0.2)"] +tests = ["cloudpickle", "zope.interface", "pytest-mypy-plugins", "mypy (>=0.900,!=0.940)", "pytest (>=4.3.0)", "pympler", "hypothesis", "coverage[toml] (>=5.0.2)"] +docs = ["sphinx-notfound-page", "zope.interface", "sphinx", "furo"] +dev = ["cloudpickle", "pre-commit", "sphinx-notfound-page", "sphinx", "furo", "zope.interface", "pytest-mypy-plugins", "mypy (>=0.900,!=0.940)", "pytest (>=4.3.0)", "pympler", "hypothesis", "coverage[toml] (>=5.0.2)"] [[package]] name = "certifi" @@ -90,7 +90,7 @@ test = ["hypothesis (==3.55.3)", "flake8 (==3.7.8)"] [[package]] name = "coverage" -version = "6.4.3" +version = "6.4.4" description = "Code coverage measurement for Python" category = "dev" optional = false @@ -175,9 +175,9 @@ python-versions = ">=3.7" zipp = ">=0.5" [package.extras] -docs = ["sphinx", "jaraco.packaging (>=9)", "rst.linker (>=1.9)"] +testing = ["importlib-resources (>=1.3)", "pytest-mypy (>=0.9.1)", "pytest-black (>=0.3.7)", "pytest-perf (>=0.9.2)", "flufl.flake8", "pyfakefs", "packaging", "pytest-enabler (>=1.3)", "pytest-cov", "pytest-flake8", "pytest-checkdocs (>=2.4)", "pytest (>=6)"] perf = ["ipython"] -testing = ["pytest (>=6)", "pytest-checkdocs (>=2.4)", "pytest-flake8", "pytest-cov", "pytest-enabler (>=1.3)", "packaging", "pyfakefs", "flufl.flake8", "pytest-perf (>=0.9.2)", "pytest-black (>=0.3.7)", "pytest-mypy (>=0.9.1)", "importlib-resources (>=1.3)"] +docs = ["rst.linker (>=1.9)", "jaraco.packaging (>=9)", "sphinx"] [[package]] name = "iniconfig" @@ -242,8 +242,8 @@ optional = false python-versions = ">=3.7" [package.extras] -docs = ["furo (>=2021.7.5b38)", "proselint (>=0.10.2)", "sphinx-autodoc-typehints (>=1.12)", "sphinx (>=4)"] -test = ["appdirs (==1.4.4)", "pytest-cov (>=2.7)", "pytest-mock (>=3.6)", "pytest (>=6)"] +test = ["pytest (>=6)", "pytest-mock (>=3.6)", "pytest-cov (>=2.7)", "appdirs (==1.4.4)"] +docs = ["sphinx (>=4)", "sphinx-autodoc-typehints (>=1.12)", "proselint (>=0.10.2)", "furo (>=2021.7.5b38)"] [[package]] name = "pluggy" @@ -254,8 +254,8 @@ optional = false python-versions = ">=3.6" [package.extras] -dev = ["pre-commit", "tox"] -testing = ["pytest", "pytest-benchmark"] +testing = ["pytest-benchmark", "pytest"] +dev = ["tox", "pre-commit"] [[package]] name = "pre-commit" @@ -335,7 +335,7 @@ optional = false python-versions = ">=3.6.8" [package.extras] -diagrams = ["railroad-diagrams", "jinja2"] +diagrams = ["jinja2", "railroad-diagrams"] [[package]] name = "pytest" @@ -356,7 +356,7 @@ py = ">=1.8.2" tomli = ">=1.0.0" [package.extras] -testing = ["argcomplete", "hypothesis (>=3.56)", "mock", "nose", "pygments (>=2.7.2)", "requests", "xmlschema"] +testing = ["xmlschema", "requests", "pygments (>=2.7.2)", "nose", "mock", "hypothesis (>=3.56)", "argcomplete"] [[package]] name = "pytest-checkdocs" @@ -372,8 +372,8 @@ importlib-metadata = ">=4" pep517 = "*" [package.extras] -docs = ["sphinx", "jaraco.packaging (>=8.2)", "rst.linker (>=1.9)"] -testing = ["pytest (>=4.6)", "pytest-checkdocs (>=2.4)", "pytest-flake8", "pytest-cov", "pytest-enabler (>=1.0.1)", "types-docutils", "pytest-black (>=0.3.7)", "pytest-mypy"] +testing = ["pytest-mypy", "pytest-black (>=0.3.7)", "types-docutils", "pytest-enabler (>=1.0.1)", "pytest-cov", "pytest-flake8", "pytest-checkdocs (>=2.4)", "pytest (>=4.6)"] +docs = ["rst.linker (>=1.9)", "jaraco.packaging (>=8.2)", "sphinx"] [[package]] name = "python-snappy" @@ -406,8 +406,8 @@ idna = ">=2.5,<4" urllib3 = ">=1.21.1,<1.27" [package.extras] -socks = ["PySocks (>=1.5.6,!=1.5.7)"] use_chardet_on_py3 = ["chardet (>=3.0.2,<6)"] +socks = ["PySocks (>=1.5.6,!=1.5.7)"] [[package]] name = "requests-mock" @@ -422,8 +422,8 @@ requests = ">=2.3,<3" six = "*" [package.extras] +test = ["testtools", "testrepository (>=0.0.18)", "sphinx", "pytest", "purl", "mock", "fixtures"] fixture = ["fixtures"] -test = ["fixtures", "mock", "purl", "pytest", "sphinx", "testrepository (>=0.0.18)", "testtools"] [[package]] name = "rich" @@ -498,9 +498,9 @@ optional = false python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*, !=3.5.*, <4" [package.extras] -brotli = ["brotlicffi (>=0.8.0)", "brotli (>=1.0.9)", "brotlipy (>=0.6.0)"] -secure = ["pyOpenSSL (>=0.14)", "cryptography (>=1.3.4)", "idna (>=2.0.0)", "certifi", "ipaddress"] socks = ["PySocks (>=1.5.6,!=1.5.7,<2.0)"] +secure = ["ipaddress", "certifi", "idna (>=2.0.0)", "cryptography (>=1.3.4)", "pyOpenSSL (>=0.14)"] +brotli = ["brotlipy (>=0.6.0)", "brotli (>=1.0.9)", "brotlicffi (>=0.8.0)"] [[package]] name = "virtualenv" @@ -516,8 +516,8 @@ filelock = ">=3.4.1,<4" platformdirs = ">=2.4,<3" [package.extras] -docs = ["proselint (>=0.13)", "sphinx (>=5.1.1)", "sphinx-argparse (>=0.3.1)", "sphinx-rtd-theme (>=1)", "towncrier (>=21.9)"] -testing = ["coverage (>=6.2)", "coverage-enable-subprocess (>=1)", "flaky (>=3.7)", "packaging (>=21.3)", "pytest (>=7.0.1)", "pytest-env (>=0.6.2)", "pytest-freezegun (>=0.4.2)", "pytest-mock (>=3.6.1)", "pytest-randomly (>=3.10.3)", "pytest-timeout (>=2.1)"] +testing = ["pytest-timeout (>=2.1)", "pytest-randomly (>=3.10.3)", "pytest-mock (>=3.6.1)", "pytest-freezegun (>=0.4.2)", "pytest-env (>=0.6.2)", "pytest (>=7.0.1)", "packaging (>=21.3)", "flaky (>=3.7)", "coverage-enable-subprocess (>=1)", "coverage (>=6.2)"] +docs = ["towncrier (>=21.9)", "sphinx-rtd-theme (>=1)", "sphinx-argparse (>=0.3.1)", "sphinx (>=5.1.1)", "proselint (>=0.13)"] [[package]] name = "zipp" @@ -528,8 +528,8 @@ optional = false python-versions = ">=3.7" [package.extras] -docs = ["sphinx", "jaraco.packaging (>=9)", "rst.linker (>=1.9)", "jaraco.tidelift (>=1.4)"] -testing = ["pytest (>=6)", "pytest-checkdocs (>=2.4)", "pytest-flake8", "pytest-cov", "pytest-enabler (>=1.3)", "jaraco.itertools", "func-timeout", "pytest-black (>=0.3.7)", "pytest-mypy (>=0.9.1)"] +testing = ["pytest-mypy (>=0.9.1)", "pytest-black (>=0.3.7)", "func-timeout", "jaraco.itertools", "pytest-enabler (>=1.3)", "pytest-cov", "pytest-flake8", "pytest-checkdocs (>=2.4)", "pytest (>=6)"] +docs = ["jaraco.tidelift (>=1.4)", "rst.linker (>=1.9)", "jaraco.packaging (>=9)", "sphinx"] [[package]] name = "zstandard" @@ -554,11 +554,16 @@ snappy = ["python-snappy"] [metadata] lock-version = "1.1" python-versions = "^3.8" -content-hash = "b104866552b491be8fdfb214b2ed3d51a65b7bd6a2ecace8eedec95a633772cb" +content-hash = "3f89f0c9e9d0ad7886a68b9fe8b393e90b629b828fa757ea7e7205b51cde1843" [metadata.files] -atomicwrites = [] -attrs = [] +atomicwrites = [ + {file = "atomicwrites-1.4.1.tar.gz", hash = "sha256:81b2c9071a49367a7f770170e5eec8cb66567cfbbc8c73d20ce5ca4a8d71cf11"}, +] +attrs = [ + {file = "attrs-22.1.0-py2.py3-none-any.whl", hash = "sha256:86efa402f67bf2df34f51a335487cf46b1ec130d02b8d39fd248abfd30da551c"}, + {file = "attrs-22.1.0.tar.gz", hash = "sha256:29adc2665447e5191d0e7c568fde78b21f9672d344281d0c6e1ab085429b22b6"}, +] certifi = [ {file = "certifi-2022.6.15-py3-none-any.whl", hash = "sha256:fe86415d55e84719d75f8b69414f6438ac3547d2078ab91b67e779ef69378412"}, {file = "certifi-2022.6.15.tar.gz", hash = "sha256:84c85a9078b11105f04f3036a9482ae10e4621616db313fe045dd24743a0820d"}, @@ -637,21 +642,105 @@ charset-normalizer = [ {file = "charset-normalizer-2.1.0.tar.gz", hash = "sha256:575e708016ff3a5e3681541cb9d79312c416835686d054a23accb873b254f413"}, {file = "charset_normalizer-2.1.0-py3-none-any.whl", hash = "sha256:5189b6f22b01957427f35b6a08d9a0bc45b46d3788ef5a92e978433c7a35f8a5"}, ] -click = [] +click = [ + {file = "click-8.1.3-py3-none-any.whl", hash = "sha256:bb4d8133cb15a609f44e8213d9b391b0809795062913b383c62be0ee95b1db48"}, + {file = "click-8.1.3.tar.gz", hash = "sha256:7682dc8afb30297001674575ea00d1814d808d6a36af415a82bd481d37ba7b8e"}, +] colorama = [ {file = "colorama-0.4.5-py2.py3-none-any.whl", hash = "sha256:854bf444933e37f5824ae7bfc1e98d5bce2ebe4160d46b5edf346a89358e99da"}, {file = "colorama-0.4.5.tar.gz", hash = "sha256:e6c6b4334fc50988a639d9b98aa429a0b57da6e17b9a44f0451f930b6967b7a4"}, ] -commonmark = [] -coverage = [] -distlib = [] +commonmark = [ + {file = "commonmark-0.9.1-py2.py3-none-any.whl", hash = "sha256:da2f38c92590f83de410ba1a3cbceafbc74fee9def35f9251ba9a971d6d66fd9"}, + {file = "commonmark-0.9.1.tar.gz", hash = "sha256:452f9dc859be7f06631ddcb328b6919c67984aca654e5fefb3914d54691aed60"}, +] +coverage = [ + {file = "coverage-6.4.4-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:e7b4da9bafad21ea45a714d3ea6f3e1679099e420c8741c74905b92ee9bfa7cc"}, + {file = "coverage-6.4.4-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:fde17bc42e0716c94bf19d92e4c9f5a00c5feb401f5bc01101fdf2a8b7cacf60"}, + {file = "coverage-6.4.4-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:cdbb0d89923c80dbd435b9cf8bba0ff55585a3cdb28cbec65f376c041472c60d"}, + {file = "coverage-6.4.4-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:67f9346aeebea54e845d29b487eb38ec95f2ecf3558a3cffb26ee3f0dcc3e760"}, + {file = "coverage-6.4.4-cp310-cp310-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:42c499c14efd858b98c4e03595bf914089b98400d30789511577aa44607a1b74"}, + {file = "coverage-6.4.4-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:c35cca192ba700979d20ac43024a82b9b32a60da2f983bec6c0f5b84aead635c"}, + {file = "coverage-6.4.4-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:9cc4f107009bca5a81caef2fca843dbec4215c05e917a59dec0c8db5cff1d2aa"}, + {file = "coverage-6.4.4-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:5f444627b3664b80d078c05fe6a850dd711beeb90d26731f11d492dcbadb6973"}, + {file = "coverage-6.4.4-cp310-cp310-win32.whl", hash = "sha256:66e6df3ac4659a435677d8cd40e8eb1ac7219345d27c41145991ee9bf4b806a0"}, + {file = "coverage-6.4.4-cp310-cp310-win_amd64.whl", hash = "sha256:35ef1f8d8a7a275aa7410d2f2c60fa6443f4a64fae9be671ec0696a68525b875"}, + {file = "coverage-6.4.4-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:c1328d0c2f194ffda30a45f11058c02410e679456276bfa0bbe0b0ee87225fac"}, + {file = "coverage-6.4.4-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:61b993f3998ee384935ee423c3d40894e93277f12482f6e777642a0141f55782"}, + {file = "coverage-6.4.4-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:d5dd4b8e9cd0deb60e6fcc7b0647cbc1da6c33b9e786f9c79721fd303994832f"}, + {file = "coverage-6.4.4-cp311-cp311-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:7026f5afe0d1a933685d8f2169d7c2d2e624f6255fb584ca99ccca8c0e966fd7"}, + {file = "coverage-6.4.4-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:9c7b9b498eb0c0d48b4c2abc0e10c2d78912203f972e0e63e3c9dc21f15abdaa"}, + {file = "coverage-6.4.4-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:ee2b2fb6eb4ace35805f434e0f6409444e1466a47f620d1d5763a22600f0f892"}, + {file = "coverage-6.4.4-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:ab066f5ab67059d1f1000b5e1aa8bbd75b6ed1fc0014559aea41a9eb66fc2ce0"}, + {file = "coverage-6.4.4-cp311-cp311-win32.whl", hash = "sha256:9d6e1f3185cbfd3d91ac77ea065d85d5215d3dfa45b191d14ddfcd952fa53796"}, + {file = "coverage-6.4.4-cp311-cp311-win_amd64.whl", hash = "sha256:e3d3c4cc38b2882f9a15bafd30aec079582b819bec1b8afdbde8f7797008108a"}, + {file = "coverage-6.4.4-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:a095aa0a996ea08b10580908e88fbaf81ecf798e923bbe64fb98d1807db3d68a"}, + {file = "coverage-6.4.4-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:ef6f44409ab02e202b31a05dd6666797f9de2aa2b4b3534e9d450e42dea5e817"}, + {file = "coverage-6.4.4-cp37-cp37m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:4b7101938584d67e6f45f0015b60e24a95bf8dea19836b1709a80342e01b472f"}, + {file = "coverage-6.4.4-cp37-cp37m-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:14a32ec68d721c3d714d9b105c7acf8e0f8a4f4734c811eda75ff3718570b5e3"}, + {file = "coverage-6.4.4-cp37-cp37m-musllinux_1_1_aarch64.whl", hash = "sha256:6a864733b22d3081749450466ac80698fe39c91cb6849b2ef8752fd7482011f3"}, + {file = "coverage-6.4.4-cp37-cp37m-musllinux_1_1_i686.whl", hash = "sha256:08002f9251f51afdcc5e3adf5d5d66bb490ae893d9e21359b085f0e03390a820"}, + {file = "coverage-6.4.4-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:a3b2752de32c455f2521a51bd3ffb53c5b3ae92736afde67ce83477f5c1dd928"}, + {file = "coverage-6.4.4-cp37-cp37m-win32.whl", hash = "sha256:f855b39e4f75abd0dfbcf74a82e84ae3fc260d523fcb3532786bcbbcb158322c"}, + {file = "coverage-6.4.4-cp37-cp37m-win_amd64.whl", hash = "sha256:ee6ae6bbcac0786807295e9687169fba80cb0617852b2fa118a99667e8e6815d"}, + {file = "coverage-6.4.4-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:564cd0f5b5470094df06fab676c6d77547abfdcb09b6c29c8a97c41ad03b103c"}, + {file = "coverage-6.4.4-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:cbbb0e4cd8ddcd5ef47641cfac97d8473ab6b132dd9a46bacb18872828031685"}, + {file = "coverage-6.4.4-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:6113e4df2fa73b80f77663445be6d567913fb3b82a86ceb64e44ae0e4b695de1"}, + {file = "coverage-6.4.4-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:8d032bfc562a52318ae05047a6eb801ff31ccee172dc0d2504614e911d8fa83e"}, + {file = "coverage-6.4.4-cp38-cp38-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:e431e305a1f3126477abe9a184624a85308da8edf8486a863601d58419d26ffa"}, + {file = "coverage-6.4.4-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:cf2afe83a53f77aec067033199797832617890e15bed42f4a1a93ea24794ae3e"}, + {file = "coverage-6.4.4-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:783bc7c4ee524039ca13b6d9b4186a67f8e63d91342c713e88c1865a38d0892a"}, + {file = "coverage-6.4.4-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:ff934ced84054b9018665ca3967fc48e1ac99e811f6cc99ea65978e1d384454b"}, + {file = "coverage-6.4.4-cp38-cp38-win32.whl", hash = "sha256:e1fabd473566fce2cf18ea41171d92814e4ef1495e04471786cbc943b89a3781"}, + {file = "coverage-6.4.4-cp38-cp38-win_amd64.whl", hash = "sha256:4179502f210ebed3ccfe2f78bf8e2d59e50b297b598b100d6c6e3341053066a2"}, + {file = "coverage-6.4.4-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:98c0b9e9b572893cdb0a00e66cf961a238f8d870d4e1dc8e679eb8bdc2eb1b86"}, + {file = "coverage-6.4.4-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:fc600f6ec19b273da1d85817eda339fb46ce9eef3e89f220055d8696e0a06908"}, + {file = "coverage-6.4.4-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:7a98d6bf6d4ca5c07a600c7b4e0c5350cd483c85c736c522b786be90ea5bac4f"}, + {file = "coverage-6.4.4-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:01778769097dbd705a24e221f42be885c544bb91251747a8a3efdec6eb4788f2"}, + {file = "coverage-6.4.4-cp39-cp39-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:dfa0b97eb904255e2ab24166071b27408f1f69c8fbda58e9c0972804851e0558"}, + {file = "coverage-6.4.4-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:fcbe3d9a53e013f8ab88734d7e517eb2cd06b7e689bedf22c0eb68db5e4a0a19"}, + {file = "coverage-6.4.4-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:15e38d853ee224e92ccc9a851457fb1e1f12d7a5df5ae44544ce7863691c7a0d"}, + {file = "coverage-6.4.4-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:6913dddee2deff8ab2512639c5168c3e80b3ebb0f818fed22048ee46f735351a"}, + {file = "coverage-6.4.4-cp39-cp39-win32.whl", hash = "sha256:354df19fefd03b9a13132fa6643527ef7905712109d9c1c1903f2133d3a4e145"}, + {file = "coverage-6.4.4-cp39-cp39-win_amd64.whl", hash = "sha256:1238b08f3576201ebf41f7c20bf59baa0d05da941b123c6656e42cdb668e9827"}, + {file = "coverage-6.4.4-pp36.pp37.pp38-none-any.whl", hash = "sha256:f67cf9f406cf0d2f08a3515ce2db5b82625a7257f88aad87904674def6ddaec1"}, + {file = "coverage-6.4.4.tar.gz", hash = "sha256:e16c45b726acb780e1e6f88b286d3c10b3914ab03438f32117c4aa52d7f30d58"}, +] +distlib = [ + {file = "distlib-0.3.5-py2.py3-none-any.whl", hash = "sha256:b710088c59f06338ca514800ad795a132da19fda270e3ce4affc74abf955a26c"}, + {file = "distlib-0.3.5.tar.gz", hash = "sha256:a7f75737c70be3b25e2bee06288cec4e4c221de18455b2dd037fe2a795cab2fe"}, +] docutils = [ {file = "docutils-0.19-py3-none-any.whl", hash = "sha256:5e1de4d849fee02c63b040a4a3fd567f4ab104defd8a5511fbbc24a8a017efbc"}, {file = "docutils-0.19.tar.gz", hash = "sha256:33995a6753c30b7f577febfc2c50411fec6aac7f7ffeb7c4cfe5991072dcf9e6"}, ] -fastavro = [] -filelock = [] -identify = [] +fastavro = [ + {file = "fastavro-1.6.0-cp310-cp310-macosx_10_14_x86_64.whl", hash = "sha256:cbaa7e5db05694c70985c7ddc8afd9f7c9aef3e66e78cb18d05c70db221eaa1a"}, + {file = "fastavro-1.6.0-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:8d18d3540aab5f34bb88ad58d2f45b2c64f10bd04ad126bb79024a1e07d6db2b"}, + {file = "fastavro-1.6.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:d1271f11304951d4b61415480ec53c2d44c297eb00fd2d95098171805a6350d7"}, + {file = "fastavro-1.6.0-cp310-cp310-win_amd64.whl", hash = "sha256:a80eec11dfc879299f4e20e6c5b19307dc50ec20a6f431fcfe2f29c5ae63c575"}, + {file = "fastavro-1.6.0-cp37-cp37m-macosx_10_15_x86_64.whl", hash = "sha256:77354c17dc36cd21a3127004fd83854e799899b26f42fbe4a8bce3424c493551"}, + {file = "fastavro-1.6.0-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:209afeb9dcdf624b10f223c17d3469fefd9669bdaceaf5a3fb47cb74a107e443"}, + {file = "fastavro-1.6.0-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:b78ce36f45cd582c8d8b5c8ade764b378d0f3581c1e62fba895945984dec107a"}, + {file = "fastavro-1.6.0-cp37-cp37m-win_amd64.whl", hash = "sha256:6ac6db0c8229e5c3f633208e9d7e9e016ee725cbc53a9e5993b4c070c189f506"}, + {file = "fastavro-1.6.0-cp38-cp38-macosx_10_15_x86_64.whl", hash = "sha256:7511fbf7e1251ae34965894cf59ad982612268a5043fbfeaf309db8f75c25489"}, + {file = "fastavro-1.6.0-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:c039f259c61e75d920b2ce236d676a7dbf122ee136df92133542a99174491392"}, + {file = "fastavro-1.6.0-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:970599975812c8df4aba06ac4e4e558211fc645f3ca44a8fab12b0f2152fd959"}, + {file = "fastavro-1.6.0-cp38-cp38-win_amd64.whl", hash = "sha256:ca4a3b4c39d2b280523111fb4ea09dd114ab78e7ee8adb166804624ce4b7ad98"}, + {file = "fastavro-1.6.0-cp39-cp39-macosx_10_15_x86_64.whl", hash = "sha256:878b06ac7f6e411c849ceafd2cc7dd9e6eb3f7d4a5268aa6f02eb8460a069082"}, + {file = "fastavro-1.6.0-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:8b788093cf45763e69cd8ffa2464a518edd13340a48e13c4211f85f25f91c972"}, + {file = "fastavro-1.6.0-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:a69d35e66812b3aed7204d796c2a0f132d2d5a115d94cc1fb3f5da3577a74d97"}, + {file = "fastavro-1.6.0-cp39-cp39-win_amd64.whl", hash = "sha256:4eee1bfa7a5d35d252a58c96b727c4645a0e0db3f4c44b94de444d3b086bbc48"}, + {file = "fastavro-1.6.0.tar.gz", hash = "sha256:64759b161bbacb5fdcbd54bd5cee05d8510836e09290e958937334dbf1bb587f"}, +] +filelock = [ + {file = "filelock-3.8.0-py3-none-any.whl", hash = "sha256:617eb4e5eedc82fc5f47b6d61e4d11cb837c56cb4544e39081099fa17ad109d4"}, + {file = "filelock-3.8.0.tar.gz", hash = "sha256:55447caa666f2198c5b6b13a26d2084d26fa5b115c00d065664b2124680c4edc"}, +] +identify = [ + {file = "identify-2.5.3-py2.py3-none-any.whl", hash = "sha256:25851c8c1370effb22aaa3c987b30449e9ff0cece408f810ae6ce408fdd20893"}, + {file = "identify-2.5.3.tar.gz", hash = "sha256:887e7b91a1be152b0d46bbf072130235a8117392b9f1828446079a816a05ef44"}, +] idna = [ {file = "idna-3.3-py3-none-any.whl", hash = "sha256:84d9dd047ffa80596e0f246e2eab0b391788b0503584e8945f2368256d2735ff"}, {file = "idna-3.3.tar.gz", hash = "sha256:9d643ff0a55b762d5cdb124b8eaa99c66322e2157b69160bc32796e824360e6d"}, @@ -696,12 +785,44 @@ nodeenv = [ {file = "nodeenv-1.7.0-py2.py3-none-any.whl", hash = "sha256:27083a7b96a25f2f5e1d8cb4b6317ee8aeda3bdd121394e5ac54e498028a042e"}, {file = "nodeenv-1.7.0.tar.gz", hash = "sha256:e0e7f7dfb85fc5394c6fe1e8fa98131a2473e04311a45afb6508f7cf1836fa2b"}, ] -numpy = [] +numpy = [ + {file = "numpy-1.23.2-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:e603ca1fb47b913942f3e660a15e55a9ebca906857edfea476ae5f0fe9b457d5"}, + {file = "numpy-1.23.2-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:633679a472934b1c20a12ed0c9a6c9eb167fbb4cb89031939bfd03dd9dbc62b8"}, + {file = "numpy-1.23.2-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:17e5226674f6ea79e14e3b91bfbc153fdf3ac13f5cc54ee7bc8fdbe820a32da0"}, + {file = "numpy-1.23.2-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:bdc02c0235b261925102b1bd586579b7158e9d0d07ecb61148a1799214a4afd5"}, + {file = "numpy-1.23.2-cp310-cp310-win32.whl", hash = "sha256:df28dda02c9328e122661f399f7655cdcbcf22ea42daa3650a26bce08a187450"}, + {file = "numpy-1.23.2-cp310-cp310-win_amd64.whl", hash = "sha256:8ebf7e194b89bc66b78475bd3624d92980fca4e5bb86dda08d677d786fefc414"}, + {file = "numpy-1.23.2-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:dc76bca1ca98f4b122114435f83f1fcf3c0fe48e4e6f660e07996abf2f53903c"}, + {file = "numpy-1.23.2-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:ecfdd68d334a6b97472ed032b5b37a30d8217c097acfff15e8452c710e775524"}, + {file = "numpy-1.23.2-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:5593f67e66dea4e237f5af998d31a43e447786b2154ba1ad833676c788f37cde"}, + {file = "numpy-1.23.2-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:ac987b35df8c2a2eab495ee206658117e9ce867acf3ccb376a19e83070e69418"}, + {file = "numpy-1.23.2-cp311-cp311-win32.whl", hash = "sha256:d98addfd3c8728ee8b2c49126f3c44c703e2b005d4a95998e2167af176a9e722"}, + {file = "numpy-1.23.2-cp311-cp311-win_amd64.whl", hash = "sha256:8ecb818231afe5f0f568c81f12ce50f2b828ff2b27487520d85eb44c71313b9e"}, + {file = "numpy-1.23.2-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:909c56c4d4341ec8315291a105169d8aae732cfb4c250fbc375a1efb7a844f8f"}, + {file = "numpy-1.23.2-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:8247f01c4721479e482cc2f9f7d973f3f47810cbc8c65e38fd1bbd3141cc9842"}, + {file = "numpy-1.23.2-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:b8b97a8a87cadcd3f94659b4ef6ec056261fa1e1c3317f4193ac231d4df70215"}, + {file = "numpy-1.23.2-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:bd5b7ccae24e3d8501ee5563e82febc1771e73bd268eef82a1e8d2b4d556ae66"}, + {file = "numpy-1.23.2-cp38-cp38-win32.whl", hash = "sha256:9b83d48e464f393d46e8dd8171687394d39bc5abfe2978896b77dc2604e8635d"}, + {file = "numpy-1.23.2-cp38-cp38-win_amd64.whl", hash = "sha256:dec198619b7dbd6db58603cd256e092bcadef22a796f778bf87f8592b468441d"}, + {file = "numpy-1.23.2-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:4f41f5bf20d9a521f8cab3a34557cd77b6f205ab2116651f12959714494268b0"}, + {file = "numpy-1.23.2-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:806cc25d5c43e240db709875e947076b2826f47c2c340a5a2f36da5bb10c58d6"}, + {file = "numpy-1.23.2-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:8f9d84a24889ebb4c641a9b99e54adb8cab50972f0166a3abc14c3b93163f074"}, + {file = "numpy-1.23.2-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:c403c81bb8ffb1c993d0165a11493fd4bf1353d258f6997b3ee288b0a48fce77"}, + {file = "numpy-1.23.2-cp39-cp39-win32.whl", hash = "sha256:cf8c6aed12a935abf2e290860af8e77b26a042eb7f2582ff83dc7ed5f963340c"}, + {file = "numpy-1.23.2-cp39-cp39-win_amd64.whl", hash = "sha256:5e28cd64624dc2354a349152599e55308eb6ca95a13ce6a7d5679ebff2962913"}, + {file = "numpy-1.23.2-pp38-pypy38_pp73-macosx_10_9_x86_64.whl", hash = "sha256:806970e69106556d1dd200e26647e9bee5e2b3f1814f9da104a943e8d548ca38"}, + {file = "numpy-1.23.2-pp38-pypy38_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:2bd879d3ca4b6f39b7770829f73278b7c5e248c91d538aab1e506c628353e47f"}, + {file = "numpy-1.23.2-pp38-pypy38_pp73-win_amd64.whl", hash = "sha256:be6b350dfbc7f708d9d853663772a9310783ea58f6035eec649fb9c4371b5389"}, + {file = "numpy-1.23.2.tar.gz", hash = "sha256:b78d00e48261fbbd04aa0d7427cf78d18401ee0abd89c7559bbf422e5b1c7d01"}, +] packaging = [ {file = "packaging-21.3-py3-none-any.whl", hash = "sha256:ef103e05f519cdc783ae24ea4e2e0f508a9c99b2d4969652eed6a2e1ea5bd522"}, {file = "packaging-21.3.tar.gz", hash = "sha256:dd47c42927d89ab911e606518907cc2d3a1f38bbd026385970643f9c5b8ecfeb"}, ] -pep517 = [] +pep517 = [ + {file = "pep517-0.13.0-py3-none-any.whl", hash = "sha256:4ba4446d80aed5b5eac6509ade100bff3e7943a8489de249654a5ae9b33ee35b"}, + {file = "pep517-0.13.0.tar.gz", hash = "sha256:ae69927c5c172be1add9203726d4b84cf3ebad1edcd5f71fcdc746e66e829f59"}, +] platformdirs = [ {file = "platformdirs-2.5.2-py3-none-any.whl", hash = "sha256:027d8e83a2d7de06bbac4e5ef7e023c02b863d7ea5d079477e722bb41ab25788"}, {file = "platformdirs-2.5.2.tar.gz", hash = "sha256:58c8abb07dcb441e6ee4b11d8df0ac856038f944ab98b7be6b27b2a3c7feef19"}, @@ -710,18 +831,87 @@ pluggy = [ {file = "pluggy-1.0.0-py2.py3-none-any.whl", hash = "sha256:74134bbf457f031a36d68416e1509f34bd5ccc019f0bcc952c7b909d06b37bd3"}, {file = "pluggy-1.0.0.tar.gz", hash = "sha256:4224373bacce55f955a878bf9cfa763c1e360858e330072059e10bad68531159"}, ] -pre-commit = [] +pre-commit = [ + {file = "pre_commit-2.20.0-py2.py3-none-any.whl", hash = "sha256:51a5ba7c480ae8072ecdb6933df22d2f812dc897d5fe848778116129a681aac7"}, + {file = "pre_commit-2.20.0.tar.gz", hash = "sha256:a978dac7bc9ec0bcee55c18a277d553b0f419d259dadb4b9418ff2d00eb43959"}, +] py = [ {file = "py-1.11.0-py2.py3-none-any.whl", hash = "sha256:607c53218732647dff4acdfcd50cb62615cedf612e72d1724fb1a0cc6405b378"}, {file = "py-1.11.0.tar.gz", hash = "sha256:51c75c4126074b472f746a24399ad32f6053d1b34b68d2fa41e558e6f4a98719"}, ] -pyarrow = [] +pyarrow = [ + {file = "pyarrow-9.0.0-cp310-cp310-macosx_10_13_universal2.whl", hash = "sha256:767cafb14278165ad539a2918c14c1b73cf20689747c21375c38e3fe62884902"}, + {file = "pyarrow-9.0.0-cp310-cp310-macosx_10_13_x86_64.whl", hash = "sha256:0238998dc692efcb4e41ae74738d7c1234723271ccf520bd8312dca07d49ef8d"}, + {file = "pyarrow-9.0.0-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:55328348b9139c2b47450d512d716c2248fd58e2f04e2fc23a65e18726666d42"}, + {file = "pyarrow-9.0.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:fc856628acd8d281652c15b6268ec7f27ebcb015abbe99d9baad17f02adc51f1"}, + {file = "pyarrow-9.0.0-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:29eb3e086e2b26202f3a4678316b93cfb15d0e2ba20f3ec12db8fd9cc07cde63"}, + {file = "pyarrow-9.0.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:2e753f8fcf07d8e3a0efa0c8bd51fef5c90281ffd4c5637c08ce42cd0ac297de"}, + {file = "pyarrow-9.0.0-cp310-cp310-win_amd64.whl", hash = "sha256:3eef8a981f45d89de403e81fb83b8119c20824caddf1404274e41a5d66c73806"}, + {file = "pyarrow-9.0.0-cp37-cp37m-macosx_10_13_x86_64.whl", hash = "sha256:7fa56cbd415cef912677270b8e41baad70cde04c6d8a8336eeb2aba85aa93706"}, + {file = "pyarrow-9.0.0-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:f8c46bde1030d704e2796182286d1c56846552c50a39ad5bf5a20c0d8159fc35"}, + {file = "pyarrow-9.0.0-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:8ad430cee28ebc4d6661fc7315747c7a18ae2a74e67498dcb039e1c762a2fb67"}, + {file = "pyarrow-9.0.0-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:81a60bb291a964f63b2717fb1b28f6615ffab7e8585322bfb8a6738e6b321282"}, + {file = "pyarrow-9.0.0-cp37-cp37m-win_amd64.whl", hash = "sha256:9cef618159567d5f62040f2b79b1c7b38e3885f4ffad0ec97cd2d86f88b67cef"}, + {file = "pyarrow-9.0.0-cp38-cp38-macosx_10_13_x86_64.whl", hash = "sha256:5526a3bfb404ff6d31d62ea582cf2466c7378a474a99ee04d1a9b05de5264541"}, + {file = "pyarrow-9.0.0-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:da3e0f319509a5881867effd7024099fb06950a0768dad0d6873668bb88cfaba"}, + {file = "pyarrow-9.0.0-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:2c715eca2092273dcccf6f08437371e04d112f9354245ba2fbe6c801879450b7"}, + {file = "pyarrow-9.0.0-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:f11a645a41ee531c3a5edda45dea07c42267f52571f818d388971d33fc7e2d4a"}, + {file = "pyarrow-9.0.0-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:a5b390bdcfb8c5b900ef543f911cdfec63e88524fafbcc15f83767202a4a2491"}, + {file = "pyarrow-9.0.0-cp38-cp38-win_amd64.whl", hash = "sha256:d9eb04db626fa24fdfb83c00f76679ca0d98728cdbaa0481b6402bf793a290c0"}, + {file = "pyarrow-9.0.0-cp39-cp39-macosx_10_13_universal2.whl", hash = "sha256:4eebdab05afa23d5d5274b24c1cbeb1ba017d67c280f7d39fd8a8f18cbad2ec9"}, + {file = "pyarrow-9.0.0-cp39-cp39-macosx_10_13_x86_64.whl", hash = "sha256:02b820ecd1da02012092c180447de449fc688d0c3f9ff8526ca301cdd60dacd0"}, + {file = "pyarrow-9.0.0-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:92f3977e901db1ef5cba30d6cc1d7942b8d94b910c60f89013e8f7bb86a86eef"}, + {file = "pyarrow-9.0.0-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:f241bd488c2705df930eedfe304ada71191dcf67d6b98ceda0cc934fd2a8388e"}, + {file = "pyarrow-9.0.0-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:1c5a073a930c632058461547e0bc572da1e724b17b6b9eb31a97da13f50cb6e0"}, + {file = "pyarrow-9.0.0-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:f59bcd5217a3ae1e17870792f82b2ff92df9f3862996e2c78e156c13e56ff62e"}, + {file = "pyarrow-9.0.0-cp39-cp39-win_amd64.whl", hash = "sha256:fe2ce795fa1d95e4e940fe5661c3c58aee7181c730f65ac5dd8794a77228de59"}, + {file = "pyarrow-9.0.0.tar.gz", hash = "sha256:7fb02bebc13ab55573d1ae9bb5002a6d20ba767bf8569b52fce5301d42495ab7"}, +] pycparser = [ {file = "pycparser-2.21-py2.py3-none-any.whl", hash = "sha256:8ee45429555515e1f6b185e78100aea234072576aa43ab53aefcae078162fca9"}, {file = "pycparser-2.21.tar.gz", hash = "sha256:e644fdec12f7872f86c58ff790da456218b10f863970249516d60a5eaca77206"}, ] -pydantic = [] -pygments = [] +pydantic = [ + {file = "pydantic-1.9.2-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:9c9e04a6cdb7a363d7cb3ccf0efea51e0abb48e180c0d31dca8d247967d85c6e"}, + {file = "pydantic-1.9.2-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:fafe841be1103f340a24977f61dee76172e4ae5f647ab9e7fd1e1fca51524f08"}, + {file = "pydantic-1.9.2-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:afacf6d2a41ed91fc631bade88b1d319c51ab5418870802cedb590b709c5ae3c"}, + {file = "pydantic-1.9.2-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:3ee0d69b2a5b341fc7927e92cae7ddcfd95e624dfc4870b32a85568bd65e6131"}, + {file = "pydantic-1.9.2-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:ff68fc85355532ea77559ede81f35fff79a6a5543477e168ab3a381887caea76"}, + {file = "pydantic-1.9.2-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:c0f5e142ef8217019e3eef6ae1b6b55f09a7a15972958d44fbd228214cede567"}, + {file = "pydantic-1.9.2-cp310-cp310-win_amd64.whl", hash = "sha256:615661bfc37e82ac677543704437ff737418e4ea04bef9cf11c6d27346606044"}, + {file = "pydantic-1.9.2-cp36-cp36m-macosx_10_9_x86_64.whl", hash = "sha256:328558c9f2eed77bd8fffad3cef39dbbe3edc7044517f4625a769d45d4cf7555"}, + {file = "pydantic-1.9.2-cp36-cp36m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:2bd446bdb7755c3a94e56d7bdfd3ee92396070efa8ef3a34fab9579fe6aa1d84"}, + {file = "pydantic-1.9.2-cp36-cp36m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:e0b214e57623a535936005797567231a12d0da0c29711eb3514bc2b3cd008d0f"}, + {file = "pydantic-1.9.2-cp36-cp36m-musllinux_1_1_i686.whl", hash = "sha256:d8ce3fb0841763a89322ea0432f1f59a2d3feae07a63ea2c958b2315e1ae8adb"}, + {file = "pydantic-1.9.2-cp36-cp36m-musllinux_1_1_x86_64.whl", hash = "sha256:b34ba24f3e2d0b39b43f0ca62008f7ba962cff51efa56e64ee25c4af6eed987b"}, + {file = "pydantic-1.9.2-cp36-cp36m-win_amd64.whl", hash = "sha256:84d76ecc908d917f4684b354a39fd885d69dd0491be175f3465fe4b59811c001"}, + {file = "pydantic-1.9.2-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:4de71c718c9756d679420c69f216776c2e977459f77e8f679a4a961dc7304a56"}, + {file = "pydantic-1.9.2-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:5803ad846cdd1ed0d97eb00292b870c29c1f03732a010e66908ff48a762f20e4"}, + {file = "pydantic-1.9.2-cp37-cp37m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:a8c5360a0297a713b4123608a7909e6869e1b56d0e96eb0d792c27585d40757f"}, + {file = "pydantic-1.9.2-cp37-cp37m-musllinux_1_1_i686.whl", hash = "sha256:cdb4272678db803ddf94caa4f94f8672e9a46bae4a44f167095e4d06fec12979"}, + {file = "pydantic-1.9.2-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:19b5686387ea0d1ea52ecc4cffb71abb21702c5e5b2ac626fd4dbaa0834aa49d"}, + {file = "pydantic-1.9.2-cp37-cp37m-win_amd64.whl", hash = "sha256:32e0b4fb13ad4db4058a7c3c80e2569adbd810c25e6ca3bbd8b2a9cc2cc871d7"}, + {file = "pydantic-1.9.2-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:91089b2e281713f3893cd01d8e576771cd5bfdfbff5d0ed95969f47ef6d676c3"}, + {file = "pydantic-1.9.2-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:e631c70c9280e3129f071635b81207cad85e6c08e253539467e4ead0e5b219aa"}, + {file = "pydantic-1.9.2-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:4b3946f87e5cef3ba2e7bd3a4eb5a20385fe36521d6cc1ebf3c08a6697c6cfb3"}, + {file = "pydantic-1.9.2-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:5565a49effe38d51882cb7bac18bda013cdb34d80ac336428e8908f0b72499b0"}, + {file = "pydantic-1.9.2-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:bd67cb2c2d9602ad159389c29e4ca964b86fa2f35c2faef54c3eb28b4efd36c8"}, + {file = "pydantic-1.9.2-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:4aafd4e55e8ad5bd1b19572ea2df546ccace7945853832bb99422a79c70ce9b8"}, + {file = "pydantic-1.9.2-cp38-cp38-win_amd64.whl", hash = "sha256:d70916235d478404a3fa8c997b003b5f33aeac4686ac1baa767234a0f8ac2326"}, + {file = "pydantic-1.9.2-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:f0ca86b525264daa5f6b192f216a0d1e860b7383e3da1c65a1908f9c02f42801"}, + {file = "pydantic-1.9.2-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:1061c6ee6204f4f5a27133126854948e3b3d51fcc16ead2e5d04378c199b2f44"}, + {file = "pydantic-1.9.2-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:e78578f0c7481c850d1c969aca9a65405887003484d24f6110458fb02cca7747"}, + {file = "pydantic-1.9.2-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:5da164119602212a3fe7e3bc08911a89db4710ae51444b4224c2382fd09ad453"}, + {file = "pydantic-1.9.2-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:7ead3cd020d526f75b4188e0a8d71c0dbbe1b4b6b5dc0ea775a93aca16256aeb"}, + {file = "pydantic-1.9.2-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:7d0f183b305629765910eaad707800d2f47c6ac5bcfb8c6397abdc30b69eeb15"}, + {file = "pydantic-1.9.2-cp39-cp39-win_amd64.whl", hash = "sha256:f1a68f4f65a9ee64b6ccccb5bf7e17db07caebd2730109cb8a95863cfa9c4e55"}, + {file = "pydantic-1.9.2-py3-none-any.whl", hash = "sha256:78a4d6bdfd116a559aeec9a4cfe77dda62acc6233f8b56a716edad2651023e5e"}, + {file = "pydantic-1.9.2.tar.gz", hash = "sha256:8cb0bc509bfb71305d7a59d00163d5f9fc4530f0881ea32c74ff4f74c85f3d3d"}, +] +pygments = [ + {file = "Pygments-2.13.0-py3-none-any.whl", hash = "sha256:f643f331ab57ba3c9d89212ee4a2dabc6e94f117cf4eefde99a0574720d14c42"}, + {file = "Pygments-2.13.0.tar.gz", hash = "sha256:56a8508ae95f98e2b9bdf93a6be5ae3f7d8af858b43e02c5a2ff083726be40c1"}, +] pyparsing = [ {file = "pyparsing-3.0.9-py3-none-any.whl", hash = "sha256:5026bae9a10eeaefb61dab2f09052b9f4307d44aee4eda64b309723d8d206bbc"}, {file = "pyparsing-3.0.9.tar.gz", hash = "sha256:2b020ecf7d21b687f219b71ecad3631f644a47f01403fa1d1036b0c6416d70fb"}, @@ -823,13 +1013,21 @@ requests = [ {file = "requests-2.28.1-py3-none-any.whl", hash = "sha256:8fefa2a1a1365bf5520aac41836fbee479da67864514bdb821f31ce07ce65349"}, {file = "requests-2.28.1.tar.gz", hash = "sha256:7c5599b102feddaa661c826c56ab4fee28bfd17f5abca1ebbe3e7f19d7c97983"}, ] -requests-mock = [] -rich = [] +requests-mock = [ + {file = "requests-mock-1.9.3.tar.gz", hash = "sha256:8d72abe54546c1fc9696fa1516672f1031d72a55a1d66c85184f972a24ba0eba"}, + {file = "requests_mock-1.9.3-py2.py3-none-any.whl", hash = "sha256:0a2d38a117c08bb78939ec163522976ad59a6b7fdd82b709e23bb98004a44970"}, +] +rich = [ + {file = "rich-12.5.1-py3-none-any.whl", hash = "sha256:2eb4e6894cde1e017976d2975ac210ef515d7548bc595ba20e195fb9628acdeb"}, + {file = "rich-12.5.1.tar.gz", hash = "sha256:63a5c5ce3673d3d5fbbf23cd87e11ab84b6b451436f1b7f19ec54b6bc36ed7ca"}, +] six = [ {file = "six-1.16.0-py2.py3-none-any.whl", hash = "sha256:8abb2f1d86890a2dfb989f9a77cfcfd3e47c2a354b01111771326f8aa26e0254"}, {file = "six-1.16.0.tar.gz", hash = "sha256:1e61c37477a1626458e36f7b1d82aa5c9b094fa4802892072e49de9c60c4c926"}, ] -thrift = [] +thrift = [ + {file = "thrift-0.16.0.tar.gz", hash = "sha256:2b5b6488fcded21f9d312aa23c9ff6a0195d0f6ae26ddbd5ad9e3e25dfc14408"}, +] toml = [ {file = "toml-0.10.2-py2.py3-none-any.whl", hash = "sha256:806143ae5bfb6a3c6e736a764057db0e6a0e05e338b5630894a5f779cabb4f9b"}, {file = "toml-0.10.2.tar.gz", hash = "sha256:b3bda1d108d5dd99f4a20d24d9c348e91c4db7ab1b749200bded2f839ccbe68f"}, @@ -842,7 +1040,61 @@ typing-extensions = [ {file = "typing_extensions-4.3.0-py3-none-any.whl", hash = "sha256:25642c956049920a5aa49edcdd6ab1e06d7e5d467fc00e0506c44ac86fbfca02"}, {file = "typing_extensions-4.3.0.tar.gz", hash = "sha256:e6d2677a32f47fc7eb2795db1dd15c1f34eff616bcaf2cfb5e997f854fa1c4a6"}, ] -urllib3 = [] -virtualenv = [] -zipp = [] -zstandard = [] +urllib3 = [ + {file = "urllib3-1.26.11-py2.py3-none-any.whl", hash = "sha256:c33ccba33c819596124764c23a97d25f32b28433ba0dedeb77d873a38722c9bc"}, + {file = "urllib3-1.26.11.tar.gz", hash = "sha256:ea6e8fb210b19d950fab93b60c9009226c63a28808bc8386e05301e25883ac0a"}, +] +virtualenv = [ + {file = "virtualenv-20.16.3-py2.py3-none-any.whl", hash = "sha256:4193b7bc8a6cd23e4eb251ac64f29b4398ab2c233531e66e40b19a6b7b0d30c1"}, + {file = "virtualenv-20.16.3.tar.gz", hash = "sha256:d86ea0bb50e06252d79e6c241507cb904fcd66090c3271381372d6221a3970f9"}, +] +zipp = [ + {file = "zipp-3.8.1-py3-none-any.whl", hash = "sha256:47c40d7fe183a6f21403a199b3e4192cca5774656965b0a4988ad2f8feb5f009"}, + {file = "zipp-3.8.1.tar.gz", hash = "sha256:05b45f1ee8f807d0cc928485ca40a07cb491cf092ff587c0df9cb1fd154848d2"}, +] +zstandard = [ + {file = "zstandard-0.18.0-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:ef7e8a200e4c8ac9102ed3c90ed2aa379f6b880f63032200909c1be21951f556"}, + {file = "zstandard-0.18.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:2dc466207016564805e56d28375f4f533b525ff50d6776946980dff5465566ac"}, + {file = "zstandard-0.18.0-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:4a2ee1d4f98447f3e5183ecfce5626f983504a4a0c005fbe92e60fa8e5d547ec"}, + {file = "zstandard-0.18.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:d956e2f03c7200d7e61345e0880c292783ec26618d0d921dcad470cb195bbce2"}, + {file = "zstandard-0.18.0-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_12_i686.manylinux2010_i686.whl", hash = "sha256:ce6f59cba9854fd14da5bfe34217a1501143057313966637b7291d1b0267bd1e"}, + {file = "zstandard-0.18.0-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:a7fa67cba473623848b6e88acf8d799b1906178fd883fb3a1da24561c779593b"}, + {file = "zstandard-0.18.0-cp310-cp310-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:cdb44d7284c8c5dd1b66dfb86dda7f4560fa94bfbbc1d2da749ba44831335e32"}, + {file = "zstandard-0.18.0-cp310-cp310-win32.whl", hash = "sha256:63694a376cde0aa8b1971d06ca28e8f8b5f492779cb6ee1cc46bbc3f019a42a5"}, + {file = "zstandard-0.18.0-cp310-cp310-win_amd64.whl", hash = "sha256:702a8324cd90c74d9c8780d02bf55e79da3193c870c9665ad3a11647e3ad1435"}, + {file = "zstandard-0.18.0-cp36-cp36m-macosx_10_9_x86_64.whl", hash = "sha256:46f679bc5dfd938db4fb058218d9dc4db1336ffaf1ea774ff152ecadabd40805"}, + {file = "zstandard-0.18.0-cp36-cp36m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:dc2a4de9f363b3247d472362a65041fe4c0f59e01a2846b15d13046be866a885"}, + {file = "zstandard-0.18.0-cp36-cp36m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:bd3220d7627fd4d26397211cb3b560ec7cc4a94b75cfce89e847e8ce7fabe32d"}, + {file = "zstandard-0.18.0-cp36-cp36m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_12_i686.manylinux2010_i686.whl", hash = "sha256:39e98cf4773234bd9cebf9f9db730e451dfcfe435e220f8921242afda8321887"}, + {file = "zstandard-0.18.0-cp36-cp36m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:5228e596eb1554598c872a337bbe4e5afe41cd1f8b1b15f2e35b50d061e35244"}, + {file = "zstandard-0.18.0-cp36-cp36m-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:d4a8fd45746a6c31e729f35196e80b8f1e9987c59f5ccb8859d7c6a6fbeb9c63"}, + {file = "zstandard-0.18.0-cp36-cp36m-win32.whl", hash = "sha256:4cbb85f29a990c2fdbf7bc63246567061a362ddca886d7fae6f780267c0a9e67"}, + {file = "zstandard-0.18.0-cp36-cp36m-win_amd64.whl", hash = "sha256:bfa6c8549fa18e6497a738b7033c49f94a8e2e30c5fbe2d14d0b5aa8bbc1695d"}, + {file = "zstandard-0.18.0-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:e02043297c1832f2666cd2204f381bef43b10d56929e13c42c10c732c6e3b4ed"}, + {file = "zstandard-0.18.0-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:7231543d38d2b7e02ef7cc78ef7ffd86419437e1114ff08709fe25a160e24bd6"}, + {file = "zstandard-0.18.0-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:c86befac87445927488f5c8f205d11566f64c11519db223e9d282b945fa60dab"}, + {file = "zstandard-0.18.0-cp37-cp37m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_12_i686.manylinux2010_i686.whl", hash = "sha256:999a4e1768f219826ba3fa2064fab1c86dd72fdd47a42536235478c3bb3ca3e2"}, + {file = "zstandard-0.18.0-cp37-cp37m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:9df59cd1cf3c62075ee2a4da767089d19d874ac3ad42b04a71a167e91b384722"}, + {file = "zstandard-0.18.0-cp37-cp37m-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:1be31e9e3f7607ee0cdd60915410a5968b205d3e7aa83b7fcf3dd76dbbdb39e0"}, + {file = "zstandard-0.18.0-cp37-cp37m-win32.whl", hash = "sha256:490d11b705b8ae9dc845431bacc8dd1cef2408aede176620a5cd0cd411027936"}, + {file = "zstandard-0.18.0-cp37-cp37m-win_amd64.whl", hash = "sha256:266aba27fa9cc5e9091d3d325ebab1fa260f64e83e42516d5e73947c70216a5b"}, + {file = "zstandard-0.18.0-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:8b2260c4e07dd0723eadb586de7718b61acca4083a490dda69c5719d79bc715c"}, + {file = "zstandard-0.18.0-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:3af8c2383d02feb6650e9255491ec7d0824f6e6dd2bbe3e521c469c985f31fb1"}, + {file = "zstandard-0.18.0-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:28723a1d2e4df778573b76b321ebe9f3469ac98988104c2af116dd344802c3f8"}, + {file = "zstandard-0.18.0-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:19cac7108ff2c342317fad6dc97604b47a41f403c8f19d0bfc396dfadc3638b8"}, + {file = "zstandard-0.18.0-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_12_i686.manylinux2010_i686.whl", hash = "sha256:76725d1ee83a8915100a310bbad5d9c1fc6397410259c94033b8318d548d9990"}, + {file = "zstandard-0.18.0-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:d716a7694ce1fa60b20bc10f35c4a22be446ef7f514c8dbc8f858b61976de2fb"}, + {file = "zstandard-0.18.0-cp38-cp38-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:49685bf9a55d1ab34bd8423ea22db836ba43a181ac6b045ac4272093d5cb874e"}, + {file = "zstandard-0.18.0-cp38-cp38-win32.whl", hash = "sha256:1af1268a7dc870eb27515fb8db1f3e6c5a555d2b7bcc476fc3bab8886c7265ab"}, + {file = "zstandard-0.18.0-cp38-cp38-win_amd64.whl", hash = "sha256:1dc2d3809e763055a1a6c1a73f2b677320cc9a5aa1a7c6cfb35aee59bddc42d9"}, + {file = "zstandard-0.18.0-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:eea18c1e7442f2aa9aff1bb84550dbb6a1f711faf6e48e7319de8f2b2e923c2a"}, + {file = "zstandard-0.18.0-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:8677ffc6a6096cccbd892e558471c901fd821aba12b7fbc63833c7346f549224"}, + {file = "zstandard-0.18.0-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:083dc08abf03807af9beeb2b6a91c23ad78add2499f828176a3c7b742c44df02"}, + {file = "zstandard-0.18.0-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:c990063664c08169c84474acecc9251ee035871589025cac47c060ff4ec4bc1a"}, + {file = "zstandard-0.18.0-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_12_i686.manylinux2010_i686.whl", hash = "sha256:533db8a6fac6248b2cb2c935e7b92f994efbdeb72e1ffa0b354432e087bb5a3e"}, + {file = "zstandard-0.18.0-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:dbb3cb8a082d62b8a73af42291569d266b05605e017a3d8a06a0e5c30b5f10f0"}, + {file = "zstandard-0.18.0-cp39-cp39-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:d6c85ca5162049ede475b7ec98e87f9390501d44a3d6776ddd504e872464ec25"}, + {file = "zstandard-0.18.0-cp39-cp39-win32.whl", hash = "sha256:75479e7c2b3eebf402c59fbe57d21bc400cefa145ca356ee053b0a08908c5784"}, + {file = "zstandard-0.18.0-cp39-cp39-win_amd64.whl", hash = "sha256:d85bfabad444812133a92fc6fbe463e1d07581dba72f041f07a360e63808b23c"}, + {file = "zstandard-0.18.0.tar.gz", hash = "sha256:0ac0357a0d985b4ff31a854744040d7b5754385d1f98f7145c30e02c6865cb6f"}, +] diff --git a/pyproject.toml b/pyproject.toml index ef6d36e9f2..86cd1995af 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -62,7 +62,7 @@ pytest = "^7.0.0" pytest-checkdocs = "^2.0.0" pre-commit = "^2.0.0" fastavro = "^1.5.4" -coverage = { version = "^6.4.3", extras = ["toml"] } +coverage = { version = "^6.4.4", extras = ["toml"] } requests-mock = "^1.9.3" [tool.poetry.scripts] From d500503d504fc699edff8163e9d65730383c2e66 Mon Sep 17 00:00:00 2001 From: Ryan Blue Date: Sun, 21 Aug 2022 13:10:18 -0700 Subject: [PATCH 181/642] Python: Fix CLI after properties refactor (#5594) --- pyiceberg/catalog/__init__.py | 17 ----------------- pyiceberg/catalog/hive.py | 2 +- pyiceberg/catalog/rest.py | 17 ++++++++--------- pyiceberg/cli/console.py | 7 ++++--- pyiceberg/cli/output.py | 15 ++++++++++++++- tests/catalog/test_hive.py | 7 +------ tests/catalog/test_rest.py | 9 ++------- 7 files changed, 30 insertions(+), 44 deletions(-) diff --git a/pyiceberg/catalog/__init__.py b/pyiceberg/catalog/__init__.py index c9ce2aab1f..ee8372cdc9 100644 --- a/pyiceberg/catalog/__init__.py +++ b/pyiceberg/catalog/__init__.py @@ -133,23 +133,6 @@ def __init__(self, name: str, **properties: str): self.name = name self.properties = properties - def property(self, key: str) -> str: - """Returns a property from the properties variable. If it doesn't exist, it will raise an error. - - Args: - key: The key of the property - - Returns: The value of the property - - Raises: - ValueError: When the property cannot be found, with a pointer on how to set the property. - """ - if key not in self.properties: - raise ValueError( - f"{type(self).__name__} expects an {key} property. Please set in config or using environment variable PYICEBERG_CATALOG__{self.name.upper()}__{key.upper()}" - ) - return self.properties[key] - @abstractmethod def create_table( self, diff --git a/pyiceberg/catalog/hive.py b/pyiceberg/catalog/hive.py index 0d4e852aa4..0783b14ec8 100644 --- a/pyiceberg/catalog/hive.py +++ b/pyiceberg/catalog/hive.py @@ -237,7 +237,7 @@ def identifier_to_database_and_table( def __init__(self, name: str, **properties: str): super().__init__(name, **properties) - self._client = _HiveClient(self.property("uri")) + self._client = _HiveClient(properties["uri"]) def _convert_hive_into_iceberg(self, table: HiveTable, io: FileIO) -> Table: properties: Dict[str, str] = table.parameters diff --git a/pyiceberg/catalog/rest.py b/pyiceberg/catalog/rest.py index 02509aad72..29b5d55d9b 100644 --- a/pyiceberg/catalog/rest.py +++ b/pyiceberg/catalog/rest.py @@ -179,10 +179,9 @@ def __init__( name: Name to identify the catalog properties: Properties that are passed along to the configuration """ - super().__init__(name, **properties) - - self.uri = self.property("uri") - if credential := self.properties.get("credential"): + self.properties = properties + self.uri = properties["uri"] + if credential := properties.get("credential"): properties["token"] = self._fetch_access_token(credential) super().__init__(name, **self._fetch_config(properties)) @@ -238,7 +237,10 @@ def _fetch_access_token(self, credential: str) -> str: def _fetch_config(self, properties: Properties) -> Properties: response = requests.get(self.url(Endpoints.get_config, prefixed=False), headers=self.headers) - response.raise_for_status() + try: + response.raise_for_status() + except HTTPError as exc: + self._handle_non_200_response(exc, {}) config_response = ConfigResponse(**response.json()) config = config_response.defaults config.update(properties) @@ -422,13 +424,12 @@ def list_namespaces(self, namespace: Union[str, Identifier] = ()) -> List[Identi ), headers=self.headers, ) - response.raise_for_status() - namespaces = ListNamespaceResponse(**response.json()) try: response.raise_for_status() except HTTPError as exc: self._handle_non_200_response(exc, {}) + namespaces = ListNamespaceResponse(**response.json()) return [namespace_tuple + child_namespace for child_namespace in namespaces.namespaces] def load_namespace_properties(self, namespace: Union[str, Identifier]) -> Properties: @@ -448,9 +449,7 @@ def update_namespace_properties( namespace_tuple = self._check_valid_namespace_identifier(namespace) namespace = NAMESPACE_SEPARATOR.join(namespace_tuple) payload = {"removals": list(removals or []), "updates": updates} - print(f"{payload}") response = requests.post(self.url(Endpoints.update_properties, namespace=namespace), json=payload, headers=self.headers) - print(f"{response.json()}") try: response.raise_for_status() except HTTPError as exc: diff --git a/pyiceberg/cli/console.py b/pyiceberg/cli/console.py index 3f9647654a..b3ed16273c 100644 --- a/pyiceberg/cli/console.py +++ b/pyiceberg/cli/console.py @@ -55,11 +55,12 @@ def wrapper(*args, **kwargs): @click.group() @click.option("--catalog", default="default") +@click.option("--verbose", type=click.BOOL) @click.option("--output", type=click.Choice(["text", "json"]), default="text") @click.option("--uri") @click.option("--credential") @click.pass_context -def run(ctx: Context, catalog: str, output: str, uri: Optional[str], credential: Optional[str]): +def run(ctx: Context, catalog: str, verbose: bool, output: str, uri: Optional[str], credential: Optional[str]): properties = {} if uri: properties["uri"] = uri @@ -68,9 +69,9 @@ def run(ctx: Context, catalog: str, output: str, uri: Optional[str], credential: ctx.ensure_object(dict) if output == "text": - ctx.obj["output"] = ConsoleOutput() + ctx.obj["output"] = ConsoleOutput(verbose=verbose) else: - ctx.obj["output"] = JsonOutput() + ctx.obj["output"] = JsonOutput(verbose=verbose) try: try: diff --git a/pyiceberg/cli/output.py b/pyiceberg/cli/output.py index f3dbc45163..20cc8b4890 100644 --- a/pyiceberg/cli/output.py +++ b/pyiceberg/cli/output.py @@ -68,12 +68,20 @@ def uuid(self, uuid: Optional[UUID]): class ConsoleOutput(Output): """Writes to the console""" + verbose: bool + + def __init__(self, **properties: Any): + self.verbose = properties.get("verbose", False) + @property def _table(self) -> RichTable: return RichTable.grid(padding=(0, 2)) def exception(self, ex: Exception): - Console(stderr=True).print(ex) + if self.verbose: + Console(stderr=True).print_exception() + else: + Console(stderr=True).print(ex) def identifiers(self, identifiers: List[Identifier]): table = self._table @@ -134,6 +142,11 @@ def uuid(self, uuid: Optional[UUID]): class JsonOutput(Output): """Writes json to stdout""" + verbose: bool + + def __init__(self, **properties: Any): + self.verbose = properties.get("verbose", False) + def _out(self, d: Any) -> None: print(json.dumps(d)) diff --git a/tests/catalog/test_hive.py b/tests/catalog/test_hive.py index ea7a193c1c..04d269a992 100644 --- a/tests/catalog/test_hive.py +++ b/tests/catalog/test_hive.py @@ -166,14 +166,9 @@ def hive_database(tmp_path_factory) -> HiveDatabase: def test_no_uri_supplied(): - with pytest.raises(ValueError) as exc_info: + with pytest.raises(KeyError): HiveCatalog("production") - assert ( - "HiveCatalog expects an uri property. Please set in config or using environment variable PYICEBERG_CATALOG__PRODUCTION__URI" - in str(exc_info.value) - ) - def test_check_number_of_namespaces(table_schema_simple: Schema): catalog = HiveCatalog(HIVE_CATALOG_NAME, uri=HIVE_METASTORE_FAKE_URL) diff --git a/tests/catalog/test_rest.py b/tests/catalog/test_rest.py index 2f112a6075..5b60cd8f7c 100644 --- a/tests/catalog/test_rest.py +++ b/tests/catalog/test_rest.py @@ -64,14 +64,9 @@ def rest_mock(requests_mock: Mocker): def test_no_uri_supplied(): - with pytest.raises(ValueError) as exc_info: + with pytest.raises(KeyError): RestCatalog("production") - assert ( - "RestCatalog expects an uri property. Please set in config or using environment variable PYICEBERG_CATALOG__PRODUCTION__URI" - in str(exc_info.value) - ) - def test_token_200(rest_mock: Mocker): rest_mock.post( @@ -84,7 +79,7 @@ def test_token_200(rest_mock: Mocker): }, status_code=200, ) - assert RestCatalog("rest", uri=TEST_URI, credential=TEST_CREDENTIALS).property("token") == TEST_TOKEN + assert RestCatalog("rest", uri=TEST_URI, credential=TEST_CREDENTIALS).properties["token"] == TEST_TOKEN def test_token_400(rest_mock: Mocker): From 8235c3c07e83adac4abeb0dff0b3790cb673fec7 Mon Sep 17 00:00:00 2001 From: Fokko Driesprong Date: Sun, 21 Aug 2022 22:21:20 +0200 Subject: [PATCH 182/642] Python: Add logic for loading custom FileIO (#5588) --- pyiceberg/catalog/hive.py | 2 +- pyiceberg/io/__init__.py | 90 +++++++++++++++++++++++++++++++++++---- tests/io/test_io.py | 68 +++++++++++++++++++++++++++++ 3 files changed, 151 insertions(+), 9 deletions(-) diff --git a/pyiceberg/catalog/hive.py b/pyiceberg/catalog/hive.py index 0783b14ec8..efa578b3a1 100644 --- a/pyiceberg/catalog/hive.py +++ b/pyiceberg/catalog/hive.py @@ -320,7 +320,7 @@ def create_table( if partition_spec.fields else DEFAULT_LAST_PARTITION_ID, ) - io = load_file_io({**self.properties, **properties}) + io = load_file_io({**self.properties, **properties}, location=location) self._write_metadata(metadata, io, metadata_location) tbl = HiveTable( diff --git a/pyiceberg/io/__init__.py b/pyiceberg/io/__init__.py index 5a317c21e6..bf21f8c3d1 100644 --- a/pyiceberg/io/__init__.py +++ b/pyiceberg/io/__init__.py @@ -22,11 +22,23 @@ for returning an InputFile instance, an OutputFile instance, and deleting a file given its location. """ +import importlib +import logging from abc import ABC, abstractmethod from io import SEEK_SET -from typing import Protocol, Union, runtime_checkable +from typing import ( + Dict, + List, + Optional, + Protocol, + Union, + runtime_checkable, +) +from urllib.parse import urlparse -from pyiceberg.typedef import Properties +from pyiceberg.typedef import EMPTY_DICT, Properties + +logger = logging.getLogger(__name__) @runtime_checkable @@ -188,6 +200,11 @@ def create(self, overwrite: bool = False) -> OutputStream: class FileIO(ABC): """A base class for FileIO implementations""" + properties: Properties + + def __init__(self, properties: Properties = EMPTY_DICT): + self.properties = properties + @abstractmethod def new_input(self, location: str) -> InputFile: """Get an InputFile instance to read bytes from the file at the given location @@ -218,11 +235,68 @@ def delete(self, location: Union[str, InputFile, OutputFile]) -> None: """ -def load_file_io(_: Properties) -> FileIO: - # To be implemented in a different PR. - # - If py-file-io is present, load the right Python class - # - When the property is missing, map from Java's filo-io to an appropriate FileIO - # - Extend the FileIO structure with a initialize that pass in properties (could also be the constructor?) +LOCATION = "location" +WAREHOUSE = "warehouse" + +ARROW_FILE_IO = "pyiceberg.io.pyarrow.PyArrowFileIO" + +# Mappings from the Java FileIO impl to a Python one. The list is ordered by preference. +# If an implementation isn't installed, it will fall back to the next one. +SCHEMA_TO_FILE_IO: Dict[str, List[str]] = { + "s3": [ARROW_FILE_IO], + "gcs": [ARROW_FILE_IO], + "file": [ARROW_FILE_IO], + "hdfs": [ARROW_FILE_IO], +} + + +def _import_file_io(io_impl: str, properties: Properties) -> Optional[FileIO]: + try: + path_parts = io_impl.split(".") + if len(path_parts) < 2: + raise ValueError(f"py-io-impl should be full path (module.CustomFileIO), got: {io_impl}") + module_name, class_name = ".".join(path_parts[:-1]), path_parts[-1] + module = importlib.import_module(module_name) + class_ = getattr(module, class_name) + return class_(properties) + except ImportError: + logger.exception("Could not initialize FileIO: %s", io_impl) + return None + + +PY_IO_IMPL = "py-io-impl" + + +def _infer_file_io_from_schema(path: str, properties: Properties) -> Optional[FileIO]: + parsed_url = urlparse(path) + if file_ios := SCHEMA_TO_FILE_IO.get(parsed_url.scheme): + for file_io_path in file_ios: + if file_io := _import_file_io(file_io_path, properties): + return file_io + else: + logger.warning("No preferred file implementation for schema: %s", parsed_url.scheme) + return None + + +def load_file_io(properties: Properties, location: Optional[str] = None) -> FileIO: + # First look for the py-io-impl property to directly load the class + if io_impl := properties.get(PY_IO_IMPL): + if file_io := _import_file_io(io_impl, properties): + return file_io + else: + raise ValueError(f"Could not initialize FileIO: {io_impl}") + + # Check the table location + if location: + if file_io := _infer_file_io_from_schema(location, properties): + return file_io + + # Look at the schema of the warehouse + if warehouse_location := properties.get(WAREHOUSE): + if file_io := _infer_file_io_from_schema(warehouse_location, properties): + return file_io + + # Default to PyArrow from pyiceberg.io.pyarrow import PyArrowFileIO - return PyArrowFileIO() + return PyArrowFileIO(properties) diff --git a/tests/io/test_io.py b/tests/io/test_io.py index c9bc60cd22..0048861aee 100644 --- a/tests/io/test_io.py +++ b/tests/io/test_io.py @@ -18,16 +18,21 @@ import os import tempfile from typing import Union +from unittest.mock import patch from urllib.parse import ParseResult, urlparse import pytest from pyiceberg.io import ( + ARROW_FILE_IO, + PY_IO_IMPL, FileIO, InputFile, InputStream, OutputFile, OutputStream, + _import_file_io, + load_file_io, ) from pyiceberg.io.pyarrow import PyArrowFile, PyArrowFileIO @@ -376,3 +381,66 @@ def test_deleting_local_file_using_file_io_output_file(CustomFileIO, CustomOutpu # Confirm that the file no longer exists assert not os.path.exists(file_location) + + +class MockFileIO(FileIO): + def new_input(self, location: str): + raise NotImplementedError() + + def new_output(self, location: str): + raise NotImplementedError() + + def delete(self, location: Union[str, InputFile, OutputFile]) -> None: + raise NotImplementedError() + + +def test_import_file_io(): + assert isinstance(_import_file_io(ARROW_FILE_IO, {}), PyArrowFileIO) + + +def test_import_file_io_does_not_exist(): + assert _import_file_io("pyiceberg.does.not.exist.FileIO", {}) is None + + +def test_load_file(): + assert isinstance(load_file_io({PY_IO_IMPL: ARROW_FILE_IO}), PyArrowFileIO) + + +def test_load_file_io_no_arguments(): + assert isinstance(load_file_io({}), PyArrowFileIO) + + +def test_load_file_io_does_not_exist(): + with pytest.raises(ValueError) as exc_info: + load_file_io({PY_IO_IMPL: "pyiceberg.does.not.exist.FileIO"}) + + assert "Could not initialize FileIO: pyiceberg.does.not.exist.FileIO" in str(exc_info.value) + + +def test_load_file_io_warehouse(): + assert isinstance(load_file_io({"warehouse": "s3://some-path/"}), PyArrowFileIO) + + +def test_load_file_io_location(): + assert isinstance(load_file_io({"location": "s3://some-path/"}), PyArrowFileIO) + + +def test_load_file_io_location_no_schema(): + assert isinstance(load_file_io({"location": "/no-schema/"}), PyArrowFileIO) + + +@patch.dict("pyiceberg.io.SCHEMA_TO_FILE_IO", {"test": ["tests.io.test_io.MockFileIO"]}) +def test_mock_warehouse_location_file_io(): + # For testing the selection logic + assert isinstance(load_file_io({"warehouse": "test://some-path/"}), MockFileIO) + + +@patch.dict("pyiceberg.io.SCHEMA_TO_FILE_IO", {"test": ["tests.io.test_io.MockFileIO"]}) +def test_mock_table_location_file_io(): + # For testing the selection logic + assert isinstance(load_file_io({}, "test://some-path/"), MockFileIO) + + +def test_gibberish_table_location_file_io(): + # For testing the selection logic + assert isinstance(load_file_io({}, "gibberish"), PyArrowFileIO) From 50b1e77954e49200c64c5258cab116d71c28d04a Mon Sep 17 00:00:00 2001 From: Fokko Driesprong Date: Wed, 24 Aug 2022 21:19:16 +0200 Subject: [PATCH 183/642] Python: Add additional information to the describe command (#5609) --- pyiceberg/cli/output.py | 8 +++++--- pyiceberg/manifest.py | 12 ++++++++++++ pyiceberg/table/__init__.py | 2 -- pyiceberg/table/snapshots.py | 7 +++++++ tests/cli/test_console.py | 10 +++++++--- 5 files changed, 31 insertions(+), 8 deletions(-) diff --git a/pyiceberg/cli/output.py b/pyiceberg/cli/output.py index 20cc8b4890..270567930e 100644 --- a/pyiceberg/cli/output.py +++ b/pyiceberg/cli/output.py @@ -97,13 +97,14 @@ def describe_table(self, table: Table): for key, value in metadata.properties.items(): table_properties.add_row(key, value) - schema_tree = Tree("Schema") + schema_tree = Tree(f"Schema, id={table.metadata.current_schema_id}") for field in table.schema().fields: schema_tree.add(str(field)) snapshot_tree = Tree("Snapshots") for snapshot in metadata.snapshots: - snapshot_tree.add(f"Snapshot {snapshot.schema_id}: {snapshot.manifest_list}") + manifest_list_str = f": {snapshot.manifest_list}" if snapshot.manifest_list else "" + snapshot_tree.add(f"Snapshot {snapshot.snapshot_id}, schema {snapshot.schema_id}{manifest_list_str}") output_table = self._table output_table.add_row("Table format version", str(metadata.format_version)) @@ -112,7 +113,8 @@ def describe_table(self, table: Table): output_table.add_row("Last Updated", str(metadata.last_updated_ms)) output_table.add_row("Partition spec", str(table.spec())) output_table.add_row("Sort order", str(table.sort_order())) - output_table.add_row("Schema", schema_tree) + output_table.add_row("Current schema", schema_tree) + output_table.add_row("Current snapshot", str(table.current_snapshot())) output_table.add_row("Snapshots", snapshot_tree) output_table.add_row("Properties", table_properties) Console().print(output_table) diff --git a/pyiceberg/manifest.py b/pyiceberg/manifest.py index ffb1c3b910..6079fcc336 100644 --- a/pyiceberg/manifest.py +++ b/pyiceberg/manifest.py @@ -46,23 +46,35 @@ class DataFileContent(int, Enum): POSITION_DELETES = 1 EQUALITY_DELETES = 2 + def __repr__(self) -> str: + return f"DataFileContent.{self.name}" + class ManifestContent(int, Enum): DATA = 0 DELETES = 1 + def __repr__(self) -> str: + return f"ManifestContent.{self.name}" + class ManifestEntryStatus(int, Enum): EXISTING = 0 ADDED = 1 DELETED = 2 + def __repr__(self) -> str: + return f"ManifestEntryStatus.{self.name}" + class FileFormat(str, Enum): AVRO = "AVRO" PARQUET = "PARQUET" ORC = "ORC" + def __repr__(self) -> str: + return f"FileFormat.{self.name}" + class DataFile(IcebergBaseModel): content: DataFileContent = Field(default=DataFileContent.DATA) diff --git a/pyiceberg/table/__init__.py b/pyiceberg/table/__init__.py index bf9f128a32..edcd1a3230 100644 --- a/pyiceberg/table/__init__.py +++ b/pyiceberg/table/__init__.py @@ -14,8 +14,6 @@ # KIND, either express or implied. See the License for the # specific language governing permissions and limitations # under the License. - - from typing import ( Dict, List, diff --git a/pyiceberg/table/snapshots.py b/pyiceberg/table/snapshots.py index 8f061777f7..849b2f3c06 100644 --- a/pyiceberg/table/snapshots.py +++ b/pyiceberg/table/snapshots.py @@ -96,6 +96,13 @@ class Snapshot(IcebergBaseModel): summary: Optional[Summary] = Field() schema_id: Optional[int] = Field(alias="schema-id", default=None) + def __str__(self) -> str: + operation = f"{self.summary.operation}: " if self.summary else "" + parent_id = f", parent_id={self.parent_snapshot_id}" if self.parent_snapshot_id else "" + schema_id = f", schema_id={self.schema_id}" if self.schema_id is not None else "" + result_str = f"{operation}id={self.snapshot_id}{parent_id}{schema_id}" + return result_str + class MetadataLogEntry(IcebergBaseModel): metadata_file: str = Field(alias="metadata-file") diff --git a/tests/cli/test_console.py b/tests/cli/test_console.py index 9f8ac2245a..c0a2022e03 100644 --- a/tests/cli/test_console.py +++ b/tests/cli/test_console.py @@ -189,13 +189,17 @@ def test_describe_table(_): 2 ASC NULLS FIRST bucket[4](3) DESC NULLS LAST ] -Schema Schema +Current schema Schema, id=1 ├── 1: x: required long ├── 2: y: required long (comment) └── 3: z: required long +Current snapshot Operation.APPEND: id=3055729675574597004, + parent_id=3051729675574597004, schema_id=1 Snapshots Snapshots - ├── Snapshot None: s3://a/b/1.avro - └── Snapshot 1: s3://a/b/2.avro + ├── Snapshot 3051729675574597004, schema None: + │ s3://a/b/1.avro + └── Snapshot 3055729675574597004, schema 1: + s3://a/b/2.avro Properties read.split.target.size 134217728 """ ) From bd07213d24e62f4adcf06139cba8b4f953151324 Mon Sep 17 00:00:00 2001 From: Fokko Driesprong Date: Thu, 25 Aug 2022 10:03:52 +0200 Subject: [PATCH 184/642] Python: Add types to bin packing (#5617) * Python: Add types to bin packing * Change int into T --- pyiceberg/utils/bin_packing.py | 69 +++++++++++++++++++++------------ tests/utils/test_bin_packing.py | 13 ++++--- 2 files changed, 53 insertions(+), 29 deletions(-) diff --git a/pyiceberg/utils/bin_packing.py b/pyiceberg/utils/bin_packing.py index ad27089686..7cf09ff6d1 100644 --- a/pyiceberg/utils/bin_packing.py +++ b/pyiceberg/utils/bin_packing.py @@ -14,10 +14,47 @@ # KIND, either express or implied. See the License for the # specific language governing permissions and limitations # under the License. +from typing import ( + Callable, + Generic, + Iterable, + List, + Optional, + TypeVar, +) + +T = TypeVar("T") + + +class Bin(Generic[T]): + def __init__(self, target_weight: int) -> None: + self.bin_weight = 0 + self.target_weight = target_weight + self.items: List[T] = [] + + def weight(self) -> int: + return self.bin_weight + + def can_add(self, weight: int) -> bool: + return self.bin_weight + weight <= self.target_weight + + def add(self, item: T, weight: int) -> None: + self.bin_weight += weight + self.items.append(item) class PackingIterator: - def __init__(self, items, target_weight, lookback, weight_func, largest_bin_first=False): + + bins: List[Bin] + + def __init__( + self, + items: Iterable[T], + target_weight: int, + lookback: int, + weight_func: Callable[[T], int], + largest_bin_first: bool = False, + ) -> None: self.items = iter(items) self.target_weight = target_weight self.lookback = lookback @@ -25,10 +62,10 @@ def __init__(self, items, target_weight, lookback, weight_func, largest_bin_firs self.largest_bin_first = largest_bin_first self.bins = [] - def __iter__(self): + def __iter__(self) -> "PackingIterator": return self - def __next__(self): + def __next__(self) -> List[T]: while True: try: item = next(self.items) @@ -37,46 +74,30 @@ def __next__(self): if bin_ is not None: bin_.add(item, weight) else: - bin_ = self.Bin(self.target_weight) + bin_ = Bin(self.target_weight) bin_.add(item, weight) self.bins.append(bin_) if len(self.bins) > self.lookback: - return list(self.remove_bin().items) + return self.remove_bin().items except StopIteration: break if len(self.bins) == 0: raise StopIteration() - return list(self.remove_bin().items) + return self.remove_bin().items - def find_bin(self, weight): + def find_bin(self, weight: int) -> Optional[Bin]: for bin_ in self.bins: if bin_.can_add(weight): return bin_ return None - def remove_bin(self): + def remove_bin(self) -> Bin: if self.largest_bin_first: bin_ = max(self.bins, key=lambda b: b.weight()) self.bins.remove(bin_) return bin_ else: return self.bins.pop(0) - - class Bin: - def __init__(self, target_weight: int): - self.bin_weight = 0 - self.target_weight = target_weight - self.items: list = [] - - def weight(self) -> int: - return self.bin_weight - - def can_add(self, weight) -> bool: - return self.bin_weight + weight <= self.target_weight - - def add(self, item, weight): - self.bin_weight += weight - self.items.append(item) diff --git a/tests/utils/test_bin_packing.py b/tests/utils/test_bin_packing.py index 5447dfe928..2c1cda5bdd 100644 --- a/tests/utils/test_bin_packing.py +++ b/tests/utils/test_bin_packing.py @@ -16,6 +16,7 @@ # under the License. import random +from typing import List import pytest @@ -35,11 +36,11 @@ ), # sparse ], ) -def test_bin_packing(splits, lookback, split_size, open_cost): - def weight_func(x): +def test_bin_packing(splits, lookback, split_size, open_cost) -> None: + def weight_func(x: int) -> int: return max(x, open_cost) - item_list_sums = [sum(item) for item in PackingIterator(splits, split_size, lookback, weight_func)] + item_list_sums: List[int] = [sum(item) for item in PackingIterator(splits, split_size, lookback, weight_func)] assert all([split_size >= item_sum >= 0 for item_sum in item_list_sums]) @@ -76,8 +77,10 @@ def weight_func(x): ), ], ) -def test_bin_packing_lookback(splits, target_weight, lookback, largest_bin_first, expected_lists): - def weight_func(x): +def test_bin_packing_lookback( + splits: List[int], target_weight: int, lookback: int, largest_bin_first: bool, expected_lists: List[List[int]] +): + def weight_func(x: int) -> int: return x assert list(PackingIterator(splits, target_weight, lookback, weight_func, largest_bin_first)) == expected_lists From f5e9f66c03a1b672578e5fafa8cbfe28a19b7e46 Mon Sep 17 00:00:00 2001 From: Fokko Driesprong Date: Mon, 29 Aug 2022 00:16:15 +0200 Subject: [PATCH 185/642] Python: Add annotations typedef (#5628) --- pyiceberg/typedef.py | 4 ++-- tests/test_typedef.py | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/pyiceberg/typedef.py b/pyiceberg/typedef.py index dc95702813..ad62fbb4ad 100644 --- a/pyiceberg/typedef.py +++ b/pyiceberg/typedef.py @@ -23,8 +23,8 @@ ) -class FrozenDict(Dict): - def __setitem__(self, instance, value): +class FrozenDict(Dict[Any, Any]): + def __setitem__(self, instance: Any, value: Any) -> None: raise AttributeError("FrozenDict does not support assignment") def update(self, *args: Any, **kwargs: Any) -> None: diff --git a/tests/test_typedef.py b/tests/test_typedef.py index 24a98dfeb8..01eed9cde0 100644 --- a/tests/test_typedef.py +++ b/tests/test_typedef.py @@ -19,13 +19,13 @@ from pyiceberg.typedef import FrozenDict -def test_setitem_frozendict(): +def test_setitem_frozendict() -> None: d = FrozenDict(foo=1, bar=2) with pytest.raises(AttributeError): d["foo"] = 3 -def test_update_frozendict(): +def test_update_frozendict() -> None: d = FrozenDict(foo=1, bar=2) with pytest.raises(AttributeError): d.update({"yes": 2}) From d35dcbd87f533b8d8ef6c61d09235eff938936c3 Mon Sep 17 00:00:00 2001 From: jun-he Date: Mon, 29 Aug 2022 06:31:15 -0700 Subject: [PATCH 186/642] Python: Add year, month, day, and hour transforms (#5462) * add time (year, month, day, hour) transforms * address the comments * update the PR to address the comments --- pyiceberg/transforms.py | 227 +++++++++++++++++++++++++++++++++++- pyiceberg/utils/datetime.py | 42 +++++++ tests/test_transforms.py | 185 +++++++++++++++++++++++++++++ 3 files changed, 451 insertions(+), 3 deletions(-) diff --git a/pyiceberg/transforms.py b/pyiceberg/transforms.py index fdb4978d60..270918e67b 100644 --- a/pyiceberg/transforms.py +++ b/pyiceberg/transforms.py @@ -18,6 +18,7 @@ import base64 import struct from abc import ABC, abstractmethod +from enum import IntEnum from functools import singledispatch from typing import ( Any, @@ -58,6 +59,10 @@ VOID = "void" BUCKET = "bucket" TRUNCATE = "truncate" +YEAR = "year" +MONTH = "month" +DAY = "day" +HOUR = "hour" BUCKET_PARSER = ParseNumberFromBrackets(BUCKET) TRUNCATE_PARSER = ParseNumberFromBrackets(TRUNCATE) @@ -92,6 +97,14 @@ def validate(cls, v: Any): return BucketTransform(num_buckets=BUCKET_PARSER.match(v)) elif v.startswith(TRUNCATE): return TruncateTransform(width=TRUNCATE_PARSER.match(v)) + elif v == YEAR: + return YearTransform() + elif v == MONTH: + return MonthTransform() + elif v == DAY: + return DayTransform() + elif v == HOUR: + return HourTransform() else: return UnknownTransform(transform=v) return v @@ -141,7 +154,6 @@ class BucketTransform(Transform[S, int]): num_buckets (int): The number of buckets. """ - _source_type: IcebergType = PrivateAttr() _num_buckets: PositiveInt = PrivateAttr() def __init__(self, num_buckets: int, **data: Any): @@ -215,6 +227,217 @@ def __repr__(self) -> str: return f"BucketTransform(num_buckets={self._num_buckets})" +class TimeResolution(IntEnum): + YEAR = 6 + MONTH = 5 + WEEK = 4 + DAY = 3 + HOUR = 2 + MINUTE = 1 + SECOND = 0 + + +class TimeTransform(Transform[S, int], Singleton): + @property + @abstractmethod + def granularity(self) -> TimeResolution: + ... + + def satisfies_order_of(self, other: Transform) -> bool: + return self.granularity <= other.granularity if hasattr(other, "granularity") else False + + def result_type(self, source: IcebergType) -> IcebergType: + return IntegerType() + + @property + def dedup_name(self) -> str: + return "time" + + @property + def preserves_order(self) -> bool: + return True + + +class YearTransform(TimeTransform): + """Transforms a datetime value into a year value. + + Example: + >>> transform = YearTransform() + >>> transform.transform(TimestampType())(1512151975038194) + 47 + """ + + __root__: Literal["year"] = Field(default="year") + + def transform(self, source: IcebergType) -> Callable[[Optional[S]], Optional[int]]: + source_type = type(source) + if source_type == DateType: + + def year_func(v): + return datetime.days_to_years(v) + + elif source_type in {TimestampType, TimestamptzType}: + + def year_func(v): + return datetime.micros_to_years(v) + + else: + raise ValueError(f"Cannot apply year transform for type: {source}") + + return lambda v: year_func(v) if v is not None else None + + def can_transform(self, source: IcebergType) -> bool: + return type(source) in { + DateType, + TimestampType, + TimestamptzType, + } + + @property + def granularity(self) -> TimeResolution: + return TimeResolution.YEAR + + def to_human_string(self, _: IcebergType, value: Optional[S]) -> str: + return datetime.to_human_year(value) if isinstance(value, int) else "null" + + def __repr__(self) -> str: + return "YearTransform()" + + +class MonthTransform(TimeTransform): + """Transforms a datetime value into a month value. + + Example: + >>> transform = MonthTransform() + >>> transform.transform(DateType())(17501) + 575 + """ + + __root__: Literal["month"] = Field(default="month") + + def transform(self, source: IcebergType) -> Callable[[Optional[S]], Optional[int]]: + source_type = type(source) + if source_type == DateType: + + def month_func(v): + return datetime.days_to_months(v) + + elif source_type in {TimestampType, TimestamptzType}: + + def month_func(v): + return datetime.micros_to_months(v) + + else: + raise ValueError(f"Cannot apply month transform for type: {source}") + + return lambda v: month_func(v) if v else None + + def can_transform(self, source: IcebergType) -> bool: + return type(source) in { + DateType, + TimestampType, + TimestamptzType, + } + + @property + def granularity(self) -> TimeResolution: + return TimeResolution.MONTH + + def to_human_string(self, _: IcebergType, value: Optional[S]) -> str: + return datetime.to_human_month(value) if isinstance(value, int) else "null" + + def __repr__(self) -> str: + return "MonthTransform()" + + +class DayTransform(TimeTransform): + """Transforms a datetime value into a day value. + + Example: + >>> transform = MonthTransform() + >>> transform.transform(DateType())(17501) + 17501 + """ + + __root__: Literal["day"] = Field(default="day") + + def transform(self, source: IcebergType) -> Callable[[Optional[S]], Optional[int]]: + source_type = type(source) + if source_type == DateType: + + def day_func(v): + return v + + elif source_type in {TimestampType, TimestamptzType}: + + def day_func(v): + return datetime.micros_to_days(v) + + else: + raise ValueError(f"Cannot apply day transform for type: {source}") + + return lambda v: day_func(v) if v else None + + def can_transform(self, source: IcebergType) -> bool: + return type(source) in { + DateType, + TimestampType, + TimestamptzType, + } + + def result_type(self, source: IcebergType) -> IcebergType: + return DateType() + + @property + def granularity(self) -> TimeResolution: + return TimeResolution.DAY + + def to_human_string(self, _: IcebergType, value: Optional[S]) -> str: + return datetime.to_human_day(value) if isinstance(value, int) else "null" + + def __repr__(self) -> str: + return "DayTransform()" + + +class HourTransform(TimeTransform): + """Transforms a datetime value into a hour value. + + Example: + >>> transform = HourTransform() + >>> transform.transform(TimestampType())(1512151975038194) + 420042 + """ + + __root__: Literal["hour"] = Field(default="hour") + + def transform(self, source: IcebergType) -> Callable[[Optional[S]], Optional[int]]: + if type(source) in {TimestampType, TimestamptzType}: + + def hour_func(v): + return datetime.micros_to_hours(v) + + else: + raise ValueError(f"Cannot apply hour transform for type: {source}") + + return lambda v: hour_func(v) if v else None + + def can_transform(self, source: IcebergType) -> bool: + return type(source) in { + TimestampType, + TimestamptzType, + } + + @property + def granularity(self) -> TimeResolution: + return TimeResolution.HOUR + + def to_human_string(self, _: IcebergType, value: Optional[S]) -> str: + return datetime.to_human_hour(value) if isinstance(value, int) else "null" + + def __repr__(self) -> str: + return "HourTransform()" + + def _base64encode(buffer: bytes) -> str: """Converts bytes to base64 string""" return base64.b64encode(buffer).decode("ISO-8859-1") @@ -230,7 +453,6 @@ class IdentityTransform(Transform[S, S]): """ __root__: Literal["identity"] = Field(default="identity") - _source_type: IcebergType = PrivateAttr() def transform(self, source: IcebergType) -> Callable[[Optional[S]], Optional[S]]: return lambda v: v @@ -389,7 +611,6 @@ class UnknownTransform(Transform): """ __root__: Literal["unknown"] = Field(default="unknown") - _source_type: IcebergType = PrivateAttr() _transform: str = PrivateAttr() def __init__(self, transform: str, **data: Any): diff --git a/pyiceberg/utils/datetime.py b/pyiceberg/utils/datetime.py index 9e491ce9ce..5bfcf772fe 100644 --- a/pyiceberg/utils/datetime.py +++ b/pyiceberg/utils/datetime.py @@ -98,11 +98,26 @@ def micros_to_timestamptz(micros: int): return EPOCH_TIMESTAMPTZ + dt +def to_human_year(year_ordinal: int) -> str: + """Converts a DateType value to human string""" + return f"{EPOCH_TIMESTAMP.year + year_ordinal:0=4d}" + + +def to_human_month(month_ordinal: int) -> str: + """Converts a DateType value to human string""" + return f"{EPOCH_TIMESTAMP.year + month_ordinal // 12:0=4d}-{1 + month_ordinal % 12:0=2d}" + + def to_human_day(day_ordinal: int) -> str: """Converts a DateType value to human string""" return (EPOCH_DATE + timedelta(days=day_ordinal)).isoformat() +def to_human_hour(hour_ordinal: int) -> str: + """Converts a DateType value to human string""" + return (EPOCH_TIMESTAMP + timedelta(hours=hour_ordinal)).isoformat("-", "hours") + + def to_human_time(micros_from_midnight: int) -> str: """Converts a TimeType value to human string""" return micros_to_time(micros_from_midnight).isoformat() @@ -116,3 +131,30 @@ def to_human_timestamptz(timestamp_micros: int) -> str: def to_human_timestamp(timestamp_micros: int) -> str: """Converts a TimestampType value to human string""" return (EPOCH_TIMESTAMP + timedelta(microseconds=timestamp_micros)).isoformat() + + +def micros_to_hours(timestamp: int) -> int: + """Converts a timestamp in microseconds to a date in hours""" + return int((datetime.utcfromtimestamp(timestamp // 1_000_000) - EPOCH_TIMESTAMP).total_seconds() / 3600) + + +def days_to_months(days: int) -> int: + """Creates a date from the number of days from 1970-01-01""" + d = days_to_date(days) + return (d.year - EPOCH_DATE.year) * 12 + (d.month - EPOCH_DATE.month) + + +def micros_to_months(timestamp: int) -> int: + dt = micros_to_timestamp(timestamp) + return (dt.year - EPOCH_TIMESTAMP.year) * 12 + (dt.month - EPOCH_TIMESTAMP.month) - (1 if dt.day < EPOCH_TIMESTAMP.day else 0) + + +def days_to_years(days: int) -> int: + return days_to_date(days).year - EPOCH_DATE.year + + +def micros_to_years(timestamp: int) -> int: + dt = micros_to_timestamp(timestamp) + return (dt.year - EPOCH_TIMESTAMP.year) - ( + 1 if dt.month < EPOCH_TIMESTAMP.month or (dt.month == EPOCH_TIMESTAMP.month and dt.day < EPOCH_TIMESTAMP.day) else 0 + ) diff --git a/tests/test_transforms.py b/tests/test_transforms.py index bdebae94e4..eec3e4aebe 100644 --- a/tests/test_transforms.py +++ b/tests/test_transforms.py @@ -24,11 +24,15 @@ from pyiceberg import transforms from pyiceberg.transforms import ( BucketTransform, + DayTransform, + HourTransform, IdentityTransform, + MonthTransform, Transform, TruncateTransform, UnknownTransform, VoidTransform, + YearTransform, ) from pyiceberg.types import ( BinaryType, @@ -139,6 +143,125 @@ def test_string_with_surrogate_pair(): assert bucket_transform(string_with_surrogate_pair) == mmh3.hash(as_bytes) +@pytest.mark.parametrize( + "date,date_transform,expected", + [ + (47, YearTransform(), "2017"), + (575, MonthTransform(), "2017-12"), + (17501, DayTransform(), "2017-12-01"), + ], +) +def test_date_to_human_string(date, date_transform, expected): + assert date_transform.to_human_string(DateType(), date) == expected + + +@pytest.mark.parametrize( + "date_transform", + [ + YearTransform(), + MonthTransform(), + DayTransform(), + ], +) +def test_none_date_to_human_string(date_transform): + assert date_transform.to_human_string(DateType(), None) == "null" + + +def test_hour_to_human_string(): + assert HourTransform().to_human_string(TimestampType(), None) == "null" + assert HourTransform().to_human_string(TimestampType(), 420042) == "2017-12-01-18" + + +@pytest.mark.parametrize( + "negative_value,time_transform,expected", + [ + (-1, YearTransform(), "1969"), + (-1, MonthTransform(), "1969-12"), + (-1, DayTransform(), "1969-12-31"), + (-1, HourTransform(), "1969-12-31-23"), + ], +) +def test_negative_value_to_human_string(negative_value, time_transform, expected): + assert time_transform.to_human_string(TimestampType(), negative_value) == expected + + +@pytest.mark.parametrize( + "type_var", + [ + DateType(), + TimestampType(), + TimestamptzType(), + ], +) +def test_time_methods(type_var): + assert YearTransform().can_transform(type_var) + assert MonthTransform().can_transform(type_var) + assert DayTransform().can_transform(type_var) + assert YearTransform().preserves_order + assert MonthTransform().preserves_order + assert DayTransform().preserves_order + assert YearTransform().result_type(type_var) == IntegerType() + assert MonthTransform().result_type(type_var) == IntegerType() + assert DayTransform().result_type(type_var) == DateType() + assert YearTransform().dedup_name == "time" + assert MonthTransform().dedup_name == "time" + assert DayTransform().dedup_name == "time" + + +@pytest.mark.parametrize( + "transform,type_var,value,expected", + [ + (DayTransform(), DateType(), 17501, 17501), + (DayTransform(), DateType(), -1, -1), + (MonthTransform(), DateType(), 17501, 575), + (MonthTransform(), DateType(), -1, -1), + (YearTransform(), DateType(), 17501, 47), + (YearTransform(), DateType(), -1, -1), + (YearTransform(), TimestampType(), 1512151975038194, 47), + (YearTransform(), TimestampType(), -1, -1), + (MonthTransform(), TimestamptzType(), 1512151975038194, 575), + (MonthTransform(), TimestamptzType(), -1, -1), + (DayTransform(), TimestampType(), 1512151975038194, 17501), + (DayTransform(), TimestampType(), -1, -1), + ], +) +def test_time_apply_method(transform, type_var, value, expected): + assert transform.transform(type_var)(value) == expected + + +@pytest.mark.parametrize( + "type_var", + [ + TimestampType(), + TimestamptzType(), + ], +) +def test_hour_method(type_var): + assert HourTransform().can_transform(type_var) + assert HourTransform().result_type(type_var) == IntegerType() + assert HourTransform().transform(type_var)(1512151975038194) == 420042 + assert HourTransform().dedup_name == "time" + + +@pytest.mark.parametrize( + "transform,other_transform", + [ + (YearTransform(), MonthTransform()), + (YearTransform(), DayTransform()), + (YearTransform(), HourTransform()), + (MonthTransform(), DayTransform()), + (MonthTransform(), HourTransform()), + (DayTransform(), HourTransform()), + ], +) +def test_satisfies_order_of_method(transform, other_transform): + assert transform.satisfies_order_of(transform) + assert other_transform.satisfies_order_of(transform) + assert not transform.satisfies_order_of(other_transform) + assert not transform.satisfies_order_of(VoidTransform()) + assert not other_transform.satisfies_order_of(IdentityTransform()) + + @pytest.mark.parametrize( "type_var,value,expected", [ @@ -318,3 +441,65 @@ def test_void_transform_str(): def test_void_transform_repr(): assert repr(VoidTransform()) == "VoidTransform()" + + +def test_year_transform_serialize(): + assert YearTransform().json() == '"year"' + + +def test_year_transform_deserialize(): + transform = TestType.parse_raw('"year"').__root__ + assert transform == YearTransform() + + +def test_month_transform_serialize(): + assert MonthTransform().json() == '"month"' + + +def test_month_transform_deserialize(): + transform = TestType.parse_raw('"month"').__root__ + assert transform == MonthTransform() + + +def test_day_transform_serialize(): + assert DayTransform().json() == '"day"' + + +def test_day_transform_deserialize(): + transform = TestType.parse_raw('"day"').__root__ + assert transform == DayTransform() + + +def test_hour_transform_serialize(): + assert HourTransform().json() == '"hour"' + + +def test_hour_transform_deserialize(): + transform = TestType.parse_raw('"hour"').__root__ + assert transform == HourTransform() + + +@pytest.mark.parametrize( + "transform,transform_str", + [ + (YearTransform(), "year"), + (MonthTransform(), "month"), + (DayTransform(), "day"), + (HourTransform(), "hour"), + ], +) +def test_datetime_transform_str(transform, transform_str): + assert str(transform) == transform_str + + +@pytest.mark.parametrize( + "transform,transform_repr", + [ + (YearTransform(), "YearTransform()"), + (MonthTransform(), "MonthTransform()"), + (DayTransform(), "DayTransform()"), + (HourTransform(), "HourTransform()"), + ], +) +def test_datetime_transform_repr(transform, transform_repr): + assert repr(transform) == transform_repr From 72d483f8f4284ce8b1e10ac6c541cd8fbb82b5cd Mon Sep 17 00:00:00 2001 From: Ryan Blue Date: Mon, 29 Aug 2022 13:37:20 -0700 Subject: [PATCH 187/642] Python: Fix date/time transforms. (#5667) --- pyiceberg/utils/datetime.py | 22 +++++++++------------- 1 file changed, 9 insertions(+), 13 deletions(-) diff --git a/pyiceberg/utils/datetime.py b/pyiceberg/utils/datetime.py index 5bfcf772fe..76a5e9d484 100644 --- a/pyiceberg/utils/datetime.py +++ b/pyiceberg/utils/datetime.py @@ -35,7 +35,7 @@ def micros_to_days(timestamp: int) -> int: """Converts a timestamp in microseconds to a date in days""" - return (datetime.fromtimestamp(timestamp / 1_000_000) - EPOCH_TIMESTAMP).days + return timedelta(microseconds=timestamp).days def micros_to_time(micros: int) -> time: @@ -133,28 +133,24 @@ def to_human_timestamp(timestamp_micros: int) -> str: return (EPOCH_TIMESTAMP + timedelta(microseconds=timestamp_micros)).isoformat() -def micros_to_hours(timestamp: int) -> int: - """Converts a timestamp in microseconds to a date in hours""" - return int((datetime.utcfromtimestamp(timestamp // 1_000_000) - EPOCH_TIMESTAMP).total_seconds() / 3600) +def micros_to_hours(micros: int) -> int: + """Converts a timestamp in microseconds to hours from 1970-01-01T00:00""" + return micros // 3_600_000_000 def days_to_months(days: int) -> int: - """Creates a date from the number of days from 1970-01-01""" d = days_to_date(days) return (d.year - EPOCH_DATE.year) * 12 + (d.month - EPOCH_DATE.month) -def micros_to_months(timestamp: int) -> int: - dt = micros_to_timestamp(timestamp) - return (dt.year - EPOCH_TIMESTAMP.year) * 12 + (dt.month - EPOCH_TIMESTAMP.month) - (1 if dt.day < EPOCH_TIMESTAMP.day else 0) +def micros_to_months(micros: int) -> int: + dt = micros_to_timestamp(micros) + return (dt.year - EPOCH_TIMESTAMP.year) * 12 + (dt.month - EPOCH_TIMESTAMP.month) def days_to_years(days: int) -> int: return days_to_date(days).year - EPOCH_DATE.year -def micros_to_years(timestamp: int) -> int: - dt = micros_to_timestamp(timestamp) - return (dt.year - EPOCH_TIMESTAMP.year) - ( - 1 if dt.month < EPOCH_TIMESTAMP.month or (dt.month == EPOCH_TIMESTAMP.month and dt.day < EPOCH_TIMESTAMP.day) else 0 - ) +def micros_to_years(micros: int) -> int: + return micros_to_timestamp(micros).year - EPOCH_TIMESTAMP.year From fcd178db02b8dc7258fd18e432d7ff2bd870cad2 Mon Sep 17 00:00:00 2001 From: Fokko Driesprong Date: Tue, 30 Aug 2022 19:01:19 +0200 Subject: [PATCH 188/642] Python: Include PyYaml as a dependency (#5674) Currently it is missing: ``` root@88de3a02961f:/# pip install "git+https://github.com/apache/iceberg.git#subdirectory=python[pyarrow]"^C root@88de3a02961f:/# pyiceberg Traceback (most recent call last): File "/usr/local/bin/pyiceberg", line 5, in from pyiceberg.cli.console import run File "/usr/local/lib/python3.9/site-packages/pyiceberg/cli/console.py", line 30, in from pyiceberg.catalog import Catalog, load_catalog File "/usr/local/lib/python3.9/site-packages/pyiceberg/catalog/__init__.py", line 37, in from pyiceberg.utils.config import Config, merge_config File "/usr/local/lib/python3.9/site-packages/pyiceberg/utils/config.py", line 21, in import yaml ModuleNotFoundError: No module named 'yaml' ``` --- poetry.lock | 360 ++++++++----------------------------------------- pyproject.toml | 1 + 2 files changed, 55 insertions(+), 306 deletions(-) diff --git a/poetry.lock b/poetry.lock index bb3eaa2415..0aeb687a58 100644 --- a/poetry.lock +++ b/poetry.lock @@ -15,10 +15,10 @@ optional = false python-versions = ">=3.5" [package.extras] -tests_no_zope = ["cloudpickle", "pytest-mypy-plugins", "mypy (>=0.900,!=0.940)", "pytest (>=4.3.0)", "pympler", "hypothesis", "coverage[toml] (>=5.0.2)"] -tests = ["cloudpickle", "zope.interface", "pytest-mypy-plugins", "mypy (>=0.900,!=0.940)", "pytest (>=4.3.0)", "pympler", "hypothesis", "coverage[toml] (>=5.0.2)"] -docs = ["sphinx-notfound-page", "zope.interface", "sphinx", "furo"] -dev = ["cloudpickle", "pre-commit", "sphinx-notfound-page", "sphinx", "furo", "zope.interface", "pytest-mypy-plugins", "mypy (>=0.900,!=0.940)", "pytest (>=4.3.0)", "pympler", "hypothesis", "coverage[toml] (>=5.0.2)"] +dev = ["cloudpickle", "coverage[toml] (>=5.0.2)", "furo", "hypothesis", "mypy (>=0.900,!=0.940)", "pre-commit", "pympler", "pytest (>=4.3.0)", "pytest-mypy-plugins", "sphinx", "sphinx-notfound-page", "zope.interface"] +docs = ["furo", "sphinx", "sphinx-notfound-page", "zope.interface"] +tests = ["cloudpickle", "coverage[toml] (>=5.0.2)", "hypothesis", "mypy (>=0.900,!=0.940)", "pympler", "pytest (>=4.3.0)", "pytest-mypy-plugins", "zope.interface"] +tests_no_zope = ["cloudpickle", "coverage[toml] (>=5.0.2)", "hypothesis", "mypy (>=0.900,!=0.940)", "pympler", "pytest (>=4.3.0)", "pytest-mypy-plugins"] [[package]] name = "certifi" @@ -49,7 +49,7 @@ python-versions = ">=3.6.1" [[package]] name = "charset-normalizer" -version = "2.1.0" +version = "2.1.1" description = "The Real First Universal Charset Detector. Open, modern and actively maintained alternative to Chardet." category = "main" optional = false @@ -86,7 +86,7 @@ optional = false python-versions = "*" [package.extras] -test = ["hypothesis (==3.55.3)", "flake8 (==3.7.8)"] +test = ["flake8 (==3.7.8)", "hypothesis (==3.55.3)"] [[package]] name = "coverage" @@ -104,7 +104,7 @@ toml = ["tomli"] [[package]] name = "distlib" -version = "0.3.5" +version = "0.3.6" description = "Distribution utilities" category = "dev" optional = false @@ -127,7 +127,7 @@ optional = false python-versions = ">=3.7" [package.extras] -codecs = ["python-snappy", "zstandard", "lz4"] +codecs = ["lz4", "python-snappy", "zstandard"] lz4 = ["lz4"] snappy = ["python-snappy"] zstandard = ["zstandard"] @@ -141,8 +141,8 @@ optional = false python-versions = ">=3.7" [package.extras] -testing = ["pytest-timeout (>=2.1)", "pytest-cov (>=3)", "pytest (>=7.1.2)", "coverage (>=6.4.2)", "covdefaults (>=2.2)"] -docs = ["sphinx-autodoc-typehints (>=1.19.1)", "sphinx (>=5.1.1)", "furo (>=2022.6.21)"] +docs = ["furo (>=2022.6.21)", "sphinx (>=5.1.1)", "sphinx-autodoc-typehints (>=1.19.1)"] +testing = ["covdefaults (>=2.2)", "coverage (>=6.4.2)", "pytest (>=7.1.2)", "pytest-cov (>=3)", "pytest-timeout (>=2.1)"] [[package]] name = "identify" @@ -175,9 +175,9 @@ python-versions = ">=3.7" zipp = ">=0.5" [package.extras] -testing = ["importlib-resources (>=1.3)", "pytest-mypy (>=0.9.1)", "pytest-black (>=0.3.7)", "pytest-perf (>=0.9.2)", "flufl.flake8", "pyfakefs", "packaging", "pytest-enabler (>=1.3)", "pytest-cov", "pytest-flake8", "pytest-checkdocs (>=2.4)", "pytest (>=6)"] +docs = ["jaraco.packaging (>=9)", "rst.linker (>=1.9)", "sphinx"] perf = ["ipython"] -docs = ["rst.linker (>=1.9)", "jaraco.packaging (>=9)", "sphinx"] +testing = ["flufl.flake8", "importlib-resources (>=1.3)", "packaging", "pyfakefs", "pytest (>=6)", "pytest-black (>=0.3.7)", "pytest-checkdocs (>=2.4)", "pytest-cov", "pytest-enabler (>=1.3)", "pytest-flake8", "pytest-mypy (>=0.9.1)", "pytest-perf (>=0.9.2)"] [[package]] name = "iniconfig" @@ -242,8 +242,8 @@ optional = false python-versions = ">=3.7" [package.extras] -test = ["pytest (>=6)", "pytest-mock (>=3.6)", "pytest-cov (>=2.7)", "appdirs (==1.4.4)"] -docs = ["sphinx (>=4)", "sphinx-autodoc-typehints (>=1.12)", "proselint (>=0.10.2)", "furo (>=2021.7.5b38)"] +docs = ["furo (>=2021.7.5b38)", "proselint (>=0.10.2)", "sphinx (>=4)", "sphinx-autodoc-typehints (>=1.12)"] +test = ["appdirs (==1.4.4)", "pytest (>=6)", "pytest-cov (>=2.7)", "pytest-mock (>=3.6)"] [[package]] name = "pluggy" @@ -254,8 +254,8 @@ optional = false python-versions = ">=3.6" [package.extras] -testing = ["pytest-benchmark", "pytest"] -dev = ["tox", "pre-commit"] +dev = ["pre-commit", "tox"] +testing = ["pytest", "pytest-benchmark"] [[package]] name = "pre-commit" @@ -312,8 +312,8 @@ python-versions = ">=3.6.1" typing-extensions = ">=3.7.4.3" [package.extras] -email = ["email-validator (>=1.0.3)"] dotenv = ["python-dotenv (>=0.10.4)"] +email = ["email-validator (>=1.0.3)"] [[package]] name = "pygments" @@ -356,7 +356,7 @@ py = ">=1.8.2" tomli = ">=1.0.0" [package.extras] -testing = ["xmlschema", "requests", "pygments (>=2.7.2)", "nose", "mock", "hypothesis (>=3.56)", "argcomplete"] +testing = ["argcomplete", "hypothesis (>=3.56)", "mock", "nose", "pygments (>=2.7.2)", "requests", "xmlschema"] [[package]] name = "pytest-checkdocs" @@ -372,8 +372,8 @@ importlib-metadata = ">=4" pep517 = "*" [package.extras] -testing = ["pytest-mypy", "pytest-black (>=0.3.7)", "types-docutils", "pytest-enabler (>=1.0.1)", "pytest-cov", "pytest-flake8", "pytest-checkdocs (>=2.4)", "pytest (>=4.6)"] -docs = ["rst.linker (>=1.9)", "jaraco.packaging (>=8.2)", "sphinx"] +docs = ["jaraco.packaging (>=8.2)", "rst.linker (>=1.9)", "sphinx"] +testing = ["pytest (>=4.6)", "pytest-black (>=0.3.7)", "pytest-checkdocs (>=2.4)", "pytest-cov", "pytest-enabler (>=1.0.1)", "pytest-flake8", "pytest-mypy", "types-docutils"] [[package]] name = "python-snappy" @@ -406,8 +406,8 @@ idna = ">=2.5,<4" urllib3 = ">=1.21.1,<1.27" [package.extras] -use_chardet_on_py3 = ["chardet (>=3.0.2,<6)"] socks = ["PySocks (>=1.5.6,!=1.5.7)"] +use_chardet_on_py3 = ["chardet (>=3.0.2,<6)"] [[package]] name = "requests-mock" @@ -422,8 +422,8 @@ requests = ">=2.3,<3" six = "*" [package.extras] -test = ["testtools", "testrepository (>=0.0.18)", "sphinx", "pytest", "purl", "mock", "fixtures"] fixture = ["fixtures"] +test = ["fixtures", "mock", "purl", "pytest", "sphinx", "testrepository (>=0.0.18)", "testtools"] [[package]] name = "rich" @@ -491,16 +491,16 @@ python-versions = ">=3.7" [[package]] name = "urllib3" -version = "1.26.11" +version = "1.26.12" description = "HTTP library with thread-safe connection pooling, file post, and more." category = "main" optional = false python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*, !=3.5.*, <4" [package.extras] +brotli = ["brotli (>=1.0.9)", "brotlicffi (>=0.8.0)", "brotlipy (>=0.6.0)"] +secure = ["certifi", "cryptography (>=1.3.4)", "idna (>=2.0.0)", "ipaddress", "pyOpenSSL (>=0.14)", "urllib3-secure-extra"] socks = ["PySocks (>=1.5.6,!=1.5.7,<2.0)"] -secure = ["ipaddress", "certifi", "idna (>=2.0.0)", "cryptography (>=1.3.4)", "pyOpenSSL (>=0.14)"] -brotli = ["brotlipy (>=0.6.0)", "brotli (>=1.0.9)", "brotlicffi (>=0.8.0)"] [[package]] name = "virtualenv" @@ -516,8 +516,8 @@ filelock = ">=3.4.1,<4" platformdirs = ">=2.4,<3" [package.extras] -testing = ["pytest-timeout (>=2.1)", "pytest-randomly (>=3.10.3)", "pytest-mock (>=3.6.1)", "pytest-freezegun (>=0.4.2)", "pytest-env (>=0.6.2)", "pytest (>=7.0.1)", "packaging (>=21.3)", "flaky (>=3.7)", "coverage-enable-subprocess (>=1)", "coverage (>=6.2)"] -docs = ["towncrier (>=21.9)", "sphinx-rtd-theme (>=1)", "sphinx-argparse (>=0.3.1)", "sphinx (>=5.1.1)", "proselint (>=0.13)"] +docs = ["proselint (>=0.13)", "sphinx (>=5.1.1)", "sphinx-argparse (>=0.3.1)", "sphinx-rtd-theme (>=1)", "towncrier (>=21.9)"] +testing = ["coverage (>=6.2)", "coverage-enable-subprocess (>=1)", "flaky (>=3.7)", "packaging (>=21.3)", "pytest (>=7.0.1)", "pytest-env (>=0.6.2)", "pytest-freezegun (>=0.4.2)", "pytest-mock (>=3.6.1)", "pytest-randomly (>=3.10.3)", "pytest-timeout (>=2.1)"] [[package]] name = "zipp" @@ -528,8 +528,8 @@ optional = false python-versions = ">=3.7" [package.extras] -testing = ["pytest-mypy (>=0.9.1)", "pytest-black (>=0.3.7)", "func-timeout", "jaraco.itertools", "pytest-enabler (>=1.3)", "pytest-cov", "pytest-flake8", "pytest-checkdocs (>=2.4)", "pytest (>=6)"] -docs = ["jaraco.tidelift (>=1.4)", "rst.linker (>=1.9)", "jaraco.packaging (>=9)", "sphinx"] +docs = ["jaraco.packaging (>=9)", "jaraco.tidelift (>=1.4)", "rst.linker (>=1.9)", "sphinx"] +testing = ["func-timeout", "jaraco.itertools", "pytest (>=6)", "pytest-black (>=0.3.7)", "pytest-checkdocs (>=2.4)", "pytest-cov", "pytest-enabler (>=1.3)", "pytest-flake8", "pytest-mypy (>=0.9.1)"] [[package]] name = "zstandard" @@ -554,16 +554,11 @@ snappy = ["python-snappy"] [metadata] lock-version = "1.1" python-versions = "^3.8" -content-hash = "3f89f0c9e9d0ad7886a68b9fe8b393e90b629b828fa757ea7e7205b51cde1843" +content-hash = "279aca560c214b2c2f4583a9c13d586b1e58c29d121f8b97798efe9e0612ab98" [metadata.files] -atomicwrites = [ - {file = "atomicwrites-1.4.1.tar.gz", hash = "sha256:81b2c9071a49367a7f770170e5eec8cb66567cfbbc8c73d20ce5ca4a8d71cf11"}, -] -attrs = [ - {file = "attrs-22.1.0-py2.py3-none-any.whl", hash = "sha256:86efa402f67bf2df34f51a335487cf46b1ec130d02b8d39fd248abfd30da551c"}, - {file = "attrs-22.1.0.tar.gz", hash = "sha256:29adc2665447e5191d0e7c568fde78b21f9672d344281d0c6e1ab085429b22b6"}, -] +atomicwrites = [] +attrs = [] certifi = [ {file = "certifi-2022.6.15-py3-none-any.whl", hash = "sha256:fe86415d55e84719d75f8b69414f6438ac3547d2078ab91b67e779ef69378412"}, {file = "certifi-2022.6.15.tar.gz", hash = "sha256:84c85a9078b11105f04f3036a9482ae10e4621616db313fe045dd24743a0820d"}, @@ -638,109 +633,25 @@ cfgv = [ {file = "cfgv-3.3.1-py2.py3-none-any.whl", hash = "sha256:c6a0883f3917a037485059700b9e75da2464e6c27051014ad85ba6aaa5884426"}, {file = "cfgv-3.3.1.tar.gz", hash = "sha256:f5a830efb9ce7a445376bb66ec94c638a9787422f96264c98edc6bdeed8ab736"}, ] -charset-normalizer = [ - {file = "charset-normalizer-2.1.0.tar.gz", hash = "sha256:575e708016ff3a5e3681541cb9d79312c416835686d054a23accb873b254f413"}, - {file = "charset_normalizer-2.1.0-py3-none-any.whl", hash = "sha256:5189b6f22b01957427f35b6a08d9a0bc45b46d3788ef5a92e978433c7a35f8a5"}, -] -click = [ - {file = "click-8.1.3-py3-none-any.whl", hash = "sha256:bb4d8133cb15a609f44e8213d9b391b0809795062913b383c62be0ee95b1db48"}, - {file = "click-8.1.3.tar.gz", hash = "sha256:7682dc8afb30297001674575ea00d1814d808d6a36af415a82bd481d37ba7b8e"}, -] +charset-normalizer = [] +click = [] colorama = [ {file = "colorama-0.4.5-py2.py3-none-any.whl", hash = "sha256:854bf444933e37f5824ae7bfc1e98d5bce2ebe4160d46b5edf346a89358e99da"}, {file = "colorama-0.4.5.tar.gz", hash = "sha256:e6c6b4334fc50988a639d9b98aa429a0b57da6e17b9a44f0451f930b6967b7a4"}, ] -commonmark = [ - {file = "commonmark-0.9.1-py2.py3-none-any.whl", hash = "sha256:da2f38c92590f83de410ba1a3cbceafbc74fee9def35f9251ba9a971d6d66fd9"}, - {file = "commonmark-0.9.1.tar.gz", hash = "sha256:452f9dc859be7f06631ddcb328b6919c67984aca654e5fefb3914d54691aed60"}, -] -coverage = [ - {file = "coverage-6.4.4-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:e7b4da9bafad21ea45a714d3ea6f3e1679099e420c8741c74905b92ee9bfa7cc"}, - {file = "coverage-6.4.4-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:fde17bc42e0716c94bf19d92e4c9f5a00c5feb401f5bc01101fdf2a8b7cacf60"}, - {file = "coverage-6.4.4-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:cdbb0d89923c80dbd435b9cf8bba0ff55585a3cdb28cbec65f376c041472c60d"}, - {file = "coverage-6.4.4-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:67f9346aeebea54e845d29b487eb38ec95f2ecf3558a3cffb26ee3f0dcc3e760"}, - {file = "coverage-6.4.4-cp310-cp310-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:42c499c14efd858b98c4e03595bf914089b98400d30789511577aa44607a1b74"}, - {file = "coverage-6.4.4-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:c35cca192ba700979d20ac43024a82b9b32a60da2f983bec6c0f5b84aead635c"}, - {file = "coverage-6.4.4-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:9cc4f107009bca5a81caef2fca843dbec4215c05e917a59dec0c8db5cff1d2aa"}, - {file = "coverage-6.4.4-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:5f444627b3664b80d078c05fe6a850dd711beeb90d26731f11d492dcbadb6973"}, - {file = "coverage-6.4.4-cp310-cp310-win32.whl", hash = "sha256:66e6df3ac4659a435677d8cd40e8eb1ac7219345d27c41145991ee9bf4b806a0"}, - {file = "coverage-6.4.4-cp310-cp310-win_amd64.whl", hash = "sha256:35ef1f8d8a7a275aa7410d2f2c60fa6443f4a64fae9be671ec0696a68525b875"}, - {file = "coverage-6.4.4-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:c1328d0c2f194ffda30a45f11058c02410e679456276bfa0bbe0b0ee87225fac"}, - {file = "coverage-6.4.4-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:61b993f3998ee384935ee423c3d40894e93277f12482f6e777642a0141f55782"}, - {file = "coverage-6.4.4-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:d5dd4b8e9cd0deb60e6fcc7b0647cbc1da6c33b9e786f9c79721fd303994832f"}, - {file = "coverage-6.4.4-cp311-cp311-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:7026f5afe0d1a933685d8f2169d7c2d2e624f6255fb584ca99ccca8c0e966fd7"}, - {file = "coverage-6.4.4-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:9c7b9b498eb0c0d48b4c2abc0e10c2d78912203f972e0e63e3c9dc21f15abdaa"}, - {file = "coverage-6.4.4-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:ee2b2fb6eb4ace35805f434e0f6409444e1466a47f620d1d5763a22600f0f892"}, - {file = "coverage-6.4.4-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:ab066f5ab67059d1f1000b5e1aa8bbd75b6ed1fc0014559aea41a9eb66fc2ce0"}, - {file = "coverage-6.4.4-cp311-cp311-win32.whl", hash = "sha256:9d6e1f3185cbfd3d91ac77ea065d85d5215d3dfa45b191d14ddfcd952fa53796"}, - {file = "coverage-6.4.4-cp311-cp311-win_amd64.whl", hash = "sha256:e3d3c4cc38b2882f9a15bafd30aec079582b819bec1b8afdbde8f7797008108a"}, - {file = "coverage-6.4.4-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:a095aa0a996ea08b10580908e88fbaf81ecf798e923bbe64fb98d1807db3d68a"}, - {file = "coverage-6.4.4-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:ef6f44409ab02e202b31a05dd6666797f9de2aa2b4b3534e9d450e42dea5e817"}, - {file = "coverage-6.4.4-cp37-cp37m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:4b7101938584d67e6f45f0015b60e24a95bf8dea19836b1709a80342e01b472f"}, - {file = "coverage-6.4.4-cp37-cp37m-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:14a32ec68d721c3d714d9b105c7acf8e0f8a4f4734c811eda75ff3718570b5e3"}, - {file = "coverage-6.4.4-cp37-cp37m-musllinux_1_1_aarch64.whl", hash = "sha256:6a864733b22d3081749450466ac80698fe39c91cb6849b2ef8752fd7482011f3"}, - {file = "coverage-6.4.4-cp37-cp37m-musllinux_1_1_i686.whl", hash = "sha256:08002f9251f51afdcc5e3adf5d5d66bb490ae893d9e21359b085f0e03390a820"}, - {file = "coverage-6.4.4-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:a3b2752de32c455f2521a51bd3ffb53c5b3ae92736afde67ce83477f5c1dd928"}, - {file = "coverage-6.4.4-cp37-cp37m-win32.whl", hash = "sha256:f855b39e4f75abd0dfbcf74a82e84ae3fc260d523fcb3532786bcbbcb158322c"}, - {file = "coverage-6.4.4-cp37-cp37m-win_amd64.whl", hash = "sha256:ee6ae6bbcac0786807295e9687169fba80cb0617852b2fa118a99667e8e6815d"}, - {file = "coverage-6.4.4-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:564cd0f5b5470094df06fab676c6d77547abfdcb09b6c29c8a97c41ad03b103c"}, - {file = "coverage-6.4.4-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:cbbb0e4cd8ddcd5ef47641cfac97d8473ab6b132dd9a46bacb18872828031685"}, - {file = "coverage-6.4.4-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:6113e4df2fa73b80f77663445be6d567913fb3b82a86ceb64e44ae0e4b695de1"}, - {file = "coverage-6.4.4-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:8d032bfc562a52318ae05047a6eb801ff31ccee172dc0d2504614e911d8fa83e"}, - {file = "coverage-6.4.4-cp38-cp38-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:e431e305a1f3126477abe9a184624a85308da8edf8486a863601d58419d26ffa"}, - {file = "coverage-6.4.4-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:cf2afe83a53f77aec067033199797832617890e15bed42f4a1a93ea24794ae3e"}, - {file = "coverage-6.4.4-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:783bc7c4ee524039ca13b6d9b4186a67f8e63d91342c713e88c1865a38d0892a"}, - {file = "coverage-6.4.4-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:ff934ced84054b9018665ca3967fc48e1ac99e811f6cc99ea65978e1d384454b"}, - {file = "coverage-6.4.4-cp38-cp38-win32.whl", hash = "sha256:e1fabd473566fce2cf18ea41171d92814e4ef1495e04471786cbc943b89a3781"}, - {file = "coverage-6.4.4-cp38-cp38-win_amd64.whl", hash = "sha256:4179502f210ebed3ccfe2f78bf8e2d59e50b297b598b100d6c6e3341053066a2"}, - {file = "coverage-6.4.4-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:98c0b9e9b572893cdb0a00e66cf961a238f8d870d4e1dc8e679eb8bdc2eb1b86"}, - {file = "coverage-6.4.4-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:fc600f6ec19b273da1d85817eda339fb46ce9eef3e89f220055d8696e0a06908"}, - {file = "coverage-6.4.4-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:7a98d6bf6d4ca5c07a600c7b4e0c5350cd483c85c736c522b786be90ea5bac4f"}, - {file = "coverage-6.4.4-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:01778769097dbd705a24e221f42be885c544bb91251747a8a3efdec6eb4788f2"}, - {file = "coverage-6.4.4-cp39-cp39-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:dfa0b97eb904255e2ab24166071b27408f1f69c8fbda58e9c0972804851e0558"}, - {file = "coverage-6.4.4-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:fcbe3d9a53e013f8ab88734d7e517eb2cd06b7e689bedf22c0eb68db5e4a0a19"}, - {file = "coverage-6.4.4-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:15e38d853ee224e92ccc9a851457fb1e1f12d7a5df5ae44544ce7863691c7a0d"}, - {file = "coverage-6.4.4-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:6913dddee2deff8ab2512639c5168c3e80b3ebb0f818fed22048ee46f735351a"}, - {file = "coverage-6.4.4-cp39-cp39-win32.whl", hash = "sha256:354df19fefd03b9a13132fa6643527ef7905712109d9c1c1903f2133d3a4e145"}, - {file = "coverage-6.4.4-cp39-cp39-win_amd64.whl", hash = "sha256:1238b08f3576201ebf41f7c20bf59baa0d05da941b123c6656e42cdb668e9827"}, - {file = "coverage-6.4.4-pp36.pp37.pp38-none-any.whl", hash = "sha256:f67cf9f406cf0d2f08a3515ce2db5b82625a7257f88aad87904674def6ddaec1"}, - {file = "coverage-6.4.4.tar.gz", hash = "sha256:e16c45b726acb780e1e6f88b286d3c10b3914ab03438f32117c4aa52d7f30d58"}, -] +commonmark = [] +coverage = [] distlib = [ - {file = "distlib-0.3.5-py2.py3-none-any.whl", hash = "sha256:b710088c59f06338ca514800ad795a132da19fda270e3ce4affc74abf955a26c"}, - {file = "distlib-0.3.5.tar.gz", hash = "sha256:a7f75737c70be3b25e2bee06288cec4e4c221de18455b2dd037fe2a795cab2fe"}, + {file = "distlib-0.3.6-py2.py3-none-any.whl", hash = "sha256:f35c4b692542ca110de7ef0bea44d73981caeb34ca0b9b6b2e6d7790dda8f80e"}, + {file = "distlib-0.3.6.tar.gz", hash = "sha256:14bad2d9b04d3a36127ac97f30b12a19268f211063d8f8ee4f47108896e11b46"}, ] docutils = [ {file = "docutils-0.19-py3-none-any.whl", hash = "sha256:5e1de4d849fee02c63b040a4a3fd567f4ab104defd8a5511fbbc24a8a017efbc"}, {file = "docutils-0.19.tar.gz", hash = "sha256:33995a6753c30b7f577febfc2c50411fec6aac7f7ffeb7c4cfe5991072dcf9e6"}, ] -fastavro = [ - {file = "fastavro-1.6.0-cp310-cp310-macosx_10_14_x86_64.whl", hash = "sha256:cbaa7e5db05694c70985c7ddc8afd9f7c9aef3e66e78cb18d05c70db221eaa1a"}, - {file = "fastavro-1.6.0-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:8d18d3540aab5f34bb88ad58d2f45b2c64f10bd04ad126bb79024a1e07d6db2b"}, - {file = "fastavro-1.6.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:d1271f11304951d4b61415480ec53c2d44c297eb00fd2d95098171805a6350d7"}, - {file = "fastavro-1.6.0-cp310-cp310-win_amd64.whl", hash = "sha256:a80eec11dfc879299f4e20e6c5b19307dc50ec20a6f431fcfe2f29c5ae63c575"}, - {file = "fastavro-1.6.0-cp37-cp37m-macosx_10_15_x86_64.whl", hash = "sha256:77354c17dc36cd21a3127004fd83854e799899b26f42fbe4a8bce3424c493551"}, - {file = "fastavro-1.6.0-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:209afeb9dcdf624b10f223c17d3469fefd9669bdaceaf5a3fb47cb74a107e443"}, - {file = "fastavro-1.6.0-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:b78ce36f45cd582c8d8b5c8ade764b378d0f3581c1e62fba895945984dec107a"}, - {file = "fastavro-1.6.0-cp37-cp37m-win_amd64.whl", hash = "sha256:6ac6db0c8229e5c3f633208e9d7e9e016ee725cbc53a9e5993b4c070c189f506"}, - {file = "fastavro-1.6.0-cp38-cp38-macosx_10_15_x86_64.whl", hash = "sha256:7511fbf7e1251ae34965894cf59ad982612268a5043fbfeaf309db8f75c25489"}, - {file = "fastavro-1.6.0-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:c039f259c61e75d920b2ce236d676a7dbf122ee136df92133542a99174491392"}, - {file = "fastavro-1.6.0-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:970599975812c8df4aba06ac4e4e558211fc645f3ca44a8fab12b0f2152fd959"}, - {file = "fastavro-1.6.0-cp38-cp38-win_amd64.whl", hash = "sha256:ca4a3b4c39d2b280523111fb4ea09dd114ab78e7ee8adb166804624ce4b7ad98"}, - {file = "fastavro-1.6.0-cp39-cp39-macosx_10_15_x86_64.whl", hash = "sha256:878b06ac7f6e411c849ceafd2cc7dd9e6eb3f7d4a5268aa6f02eb8460a069082"}, - {file = "fastavro-1.6.0-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:8b788093cf45763e69cd8ffa2464a518edd13340a48e13c4211f85f25f91c972"}, - {file = "fastavro-1.6.0-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:a69d35e66812b3aed7204d796c2a0f132d2d5a115d94cc1fb3f5da3577a74d97"}, - {file = "fastavro-1.6.0-cp39-cp39-win_amd64.whl", hash = "sha256:4eee1bfa7a5d35d252a58c96b727c4645a0e0db3f4c44b94de444d3b086bbc48"}, - {file = "fastavro-1.6.0.tar.gz", hash = "sha256:64759b161bbacb5fdcbd54bd5cee05d8510836e09290e958937334dbf1bb587f"}, -] -filelock = [ - {file = "filelock-3.8.0-py3-none-any.whl", hash = "sha256:617eb4e5eedc82fc5f47b6d61e4d11cb837c56cb4544e39081099fa17ad109d4"}, - {file = "filelock-3.8.0.tar.gz", hash = "sha256:55447caa666f2198c5b6b13a26d2084d26fa5b115c00d065664b2124680c4edc"}, -] -identify = [ - {file = "identify-2.5.3-py2.py3-none-any.whl", hash = "sha256:25851c8c1370effb22aaa3c987b30449e9ff0cece408f810ae6ce408fdd20893"}, - {file = "identify-2.5.3.tar.gz", hash = "sha256:887e7b91a1be152b0d46bbf072130235a8117392b9f1828446079a816a05ef44"}, -] +fastavro = [] +filelock = [] +identify = [] idna = [ {file = "idna-3.3-py3-none-any.whl", hash = "sha256:84d9dd047ffa80596e0f246e2eab0b391788b0503584e8945f2368256d2735ff"}, {file = "idna-3.3.tar.gz", hash = "sha256:9d643ff0a55b762d5cdb124b8eaa99c66322e2157b69160bc32796e824360e6d"}, @@ -785,44 +696,12 @@ nodeenv = [ {file = "nodeenv-1.7.0-py2.py3-none-any.whl", hash = "sha256:27083a7b96a25f2f5e1d8cb4b6317ee8aeda3bdd121394e5ac54e498028a042e"}, {file = "nodeenv-1.7.0.tar.gz", hash = "sha256:e0e7f7dfb85fc5394c6fe1e8fa98131a2473e04311a45afb6508f7cf1836fa2b"}, ] -numpy = [ - {file = "numpy-1.23.2-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:e603ca1fb47b913942f3e660a15e55a9ebca906857edfea476ae5f0fe9b457d5"}, - {file = "numpy-1.23.2-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:633679a472934b1c20a12ed0c9a6c9eb167fbb4cb89031939bfd03dd9dbc62b8"}, - {file = "numpy-1.23.2-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:17e5226674f6ea79e14e3b91bfbc153fdf3ac13f5cc54ee7bc8fdbe820a32da0"}, - {file = "numpy-1.23.2-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:bdc02c0235b261925102b1bd586579b7158e9d0d07ecb61148a1799214a4afd5"}, - {file = "numpy-1.23.2-cp310-cp310-win32.whl", hash = "sha256:df28dda02c9328e122661f399f7655cdcbcf22ea42daa3650a26bce08a187450"}, - {file = "numpy-1.23.2-cp310-cp310-win_amd64.whl", hash = "sha256:8ebf7e194b89bc66b78475bd3624d92980fca4e5bb86dda08d677d786fefc414"}, - {file = "numpy-1.23.2-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:dc76bca1ca98f4b122114435f83f1fcf3c0fe48e4e6f660e07996abf2f53903c"}, - {file = "numpy-1.23.2-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:ecfdd68d334a6b97472ed032b5b37a30d8217c097acfff15e8452c710e775524"}, - {file = "numpy-1.23.2-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:5593f67e66dea4e237f5af998d31a43e447786b2154ba1ad833676c788f37cde"}, - {file = "numpy-1.23.2-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:ac987b35df8c2a2eab495ee206658117e9ce867acf3ccb376a19e83070e69418"}, - {file = "numpy-1.23.2-cp311-cp311-win32.whl", hash = "sha256:d98addfd3c8728ee8b2c49126f3c44c703e2b005d4a95998e2167af176a9e722"}, - {file = "numpy-1.23.2-cp311-cp311-win_amd64.whl", hash = "sha256:8ecb818231afe5f0f568c81f12ce50f2b828ff2b27487520d85eb44c71313b9e"}, - {file = "numpy-1.23.2-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:909c56c4d4341ec8315291a105169d8aae732cfb4c250fbc375a1efb7a844f8f"}, - {file = "numpy-1.23.2-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:8247f01c4721479e482cc2f9f7d973f3f47810cbc8c65e38fd1bbd3141cc9842"}, - {file = "numpy-1.23.2-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:b8b97a8a87cadcd3f94659b4ef6ec056261fa1e1c3317f4193ac231d4df70215"}, - {file = "numpy-1.23.2-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:bd5b7ccae24e3d8501ee5563e82febc1771e73bd268eef82a1e8d2b4d556ae66"}, - {file = "numpy-1.23.2-cp38-cp38-win32.whl", hash = "sha256:9b83d48e464f393d46e8dd8171687394d39bc5abfe2978896b77dc2604e8635d"}, - {file = "numpy-1.23.2-cp38-cp38-win_amd64.whl", hash = "sha256:dec198619b7dbd6db58603cd256e092bcadef22a796f778bf87f8592b468441d"}, - {file = "numpy-1.23.2-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:4f41f5bf20d9a521f8cab3a34557cd77b6f205ab2116651f12959714494268b0"}, - {file = "numpy-1.23.2-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:806cc25d5c43e240db709875e947076b2826f47c2c340a5a2f36da5bb10c58d6"}, - {file = "numpy-1.23.2-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:8f9d84a24889ebb4c641a9b99e54adb8cab50972f0166a3abc14c3b93163f074"}, - {file = "numpy-1.23.2-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:c403c81bb8ffb1c993d0165a11493fd4bf1353d258f6997b3ee288b0a48fce77"}, - {file = "numpy-1.23.2-cp39-cp39-win32.whl", hash = "sha256:cf8c6aed12a935abf2e290860af8e77b26a042eb7f2582ff83dc7ed5f963340c"}, - {file = "numpy-1.23.2-cp39-cp39-win_amd64.whl", hash = "sha256:5e28cd64624dc2354a349152599e55308eb6ca95a13ce6a7d5679ebff2962913"}, - {file = "numpy-1.23.2-pp38-pypy38_pp73-macosx_10_9_x86_64.whl", hash = "sha256:806970e69106556d1dd200e26647e9bee5e2b3f1814f9da104a943e8d548ca38"}, - {file = "numpy-1.23.2-pp38-pypy38_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:2bd879d3ca4b6f39b7770829f73278b7c5e248c91d538aab1e506c628353e47f"}, - {file = "numpy-1.23.2-pp38-pypy38_pp73-win_amd64.whl", hash = "sha256:be6b350dfbc7f708d9d853663772a9310783ea58f6035eec649fb9c4371b5389"}, - {file = "numpy-1.23.2.tar.gz", hash = "sha256:b78d00e48261fbbd04aa0d7427cf78d18401ee0abd89c7559bbf422e5b1c7d01"}, -] +numpy = [] packaging = [ {file = "packaging-21.3-py3-none-any.whl", hash = "sha256:ef103e05f519cdc783ae24ea4e2e0f508a9c99b2d4969652eed6a2e1ea5bd522"}, {file = "packaging-21.3.tar.gz", hash = "sha256:dd47c42927d89ab911e606518907cc2d3a1f38bbd026385970643f9c5b8ecfeb"}, ] -pep517 = [ - {file = "pep517-0.13.0-py3-none-any.whl", hash = "sha256:4ba4446d80aed5b5eac6509ade100bff3e7943a8489de249654a5ae9b33ee35b"}, - {file = "pep517-0.13.0.tar.gz", hash = "sha256:ae69927c5c172be1add9203726d4b84cf3ebad1edcd5f71fcdc746e66e829f59"}, -] +pep517 = [] platformdirs = [ {file = "platformdirs-2.5.2-py3-none-any.whl", hash = "sha256:027d8e83a2d7de06bbac4e5ef7e023c02b863d7ea5d079477e722bb41ab25788"}, {file = "platformdirs-2.5.2.tar.gz", hash = "sha256:58c8abb07dcb441e6ee4b11d8df0ac856038f944ab98b7be6b27b2a3c7feef19"}, @@ -831,87 +710,18 @@ pluggy = [ {file = "pluggy-1.0.0-py2.py3-none-any.whl", hash = "sha256:74134bbf457f031a36d68416e1509f34bd5ccc019f0bcc952c7b909d06b37bd3"}, {file = "pluggy-1.0.0.tar.gz", hash = "sha256:4224373bacce55f955a878bf9cfa763c1e360858e330072059e10bad68531159"}, ] -pre-commit = [ - {file = "pre_commit-2.20.0-py2.py3-none-any.whl", hash = "sha256:51a5ba7c480ae8072ecdb6933df22d2f812dc897d5fe848778116129a681aac7"}, - {file = "pre_commit-2.20.0.tar.gz", hash = "sha256:a978dac7bc9ec0bcee55c18a277d553b0f419d259dadb4b9418ff2d00eb43959"}, -] +pre-commit = [] py = [ {file = "py-1.11.0-py2.py3-none-any.whl", hash = "sha256:607c53218732647dff4acdfcd50cb62615cedf612e72d1724fb1a0cc6405b378"}, {file = "py-1.11.0.tar.gz", hash = "sha256:51c75c4126074b472f746a24399ad32f6053d1b34b68d2fa41e558e6f4a98719"}, ] -pyarrow = [ - {file = "pyarrow-9.0.0-cp310-cp310-macosx_10_13_universal2.whl", hash = "sha256:767cafb14278165ad539a2918c14c1b73cf20689747c21375c38e3fe62884902"}, - {file = "pyarrow-9.0.0-cp310-cp310-macosx_10_13_x86_64.whl", hash = "sha256:0238998dc692efcb4e41ae74738d7c1234723271ccf520bd8312dca07d49ef8d"}, - {file = "pyarrow-9.0.0-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:55328348b9139c2b47450d512d716c2248fd58e2f04e2fc23a65e18726666d42"}, - {file = "pyarrow-9.0.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:fc856628acd8d281652c15b6268ec7f27ebcb015abbe99d9baad17f02adc51f1"}, - {file = "pyarrow-9.0.0-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:29eb3e086e2b26202f3a4678316b93cfb15d0e2ba20f3ec12db8fd9cc07cde63"}, - {file = "pyarrow-9.0.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:2e753f8fcf07d8e3a0efa0c8bd51fef5c90281ffd4c5637c08ce42cd0ac297de"}, - {file = "pyarrow-9.0.0-cp310-cp310-win_amd64.whl", hash = "sha256:3eef8a981f45d89de403e81fb83b8119c20824caddf1404274e41a5d66c73806"}, - {file = "pyarrow-9.0.0-cp37-cp37m-macosx_10_13_x86_64.whl", hash = "sha256:7fa56cbd415cef912677270b8e41baad70cde04c6d8a8336eeb2aba85aa93706"}, - {file = "pyarrow-9.0.0-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:f8c46bde1030d704e2796182286d1c56846552c50a39ad5bf5a20c0d8159fc35"}, - {file = "pyarrow-9.0.0-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:8ad430cee28ebc4d6661fc7315747c7a18ae2a74e67498dcb039e1c762a2fb67"}, - {file = "pyarrow-9.0.0-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:81a60bb291a964f63b2717fb1b28f6615ffab7e8585322bfb8a6738e6b321282"}, - {file = "pyarrow-9.0.0-cp37-cp37m-win_amd64.whl", hash = "sha256:9cef618159567d5f62040f2b79b1c7b38e3885f4ffad0ec97cd2d86f88b67cef"}, - {file = "pyarrow-9.0.0-cp38-cp38-macosx_10_13_x86_64.whl", hash = "sha256:5526a3bfb404ff6d31d62ea582cf2466c7378a474a99ee04d1a9b05de5264541"}, - {file = "pyarrow-9.0.0-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:da3e0f319509a5881867effd7024099fb06950a0768dad0d6873668bb88cfaba"}, - {file = "pyarrow-9.0.0-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:2c715eca2092273dcccf6f08437371e04d112f9354245ba2fbe6c801879450b7"}, - {file = "pyarrow-9.0.0-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:f11a645a41ee531c3a5edda45dea07c42267f52571f818d388971d33fc7e2d4a"}, - {file = "pyarrow-9.0.0-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:a5b390bdcfb8c5b900ef543f911cdfec63e88524fafbcc15f83767202a4a2491"}, - {file = "pyarrow-9.0.0-cp38-cp38-win_amd64.whl", hash = "sha256:d9eb04db626fa24fdfb83c00f76679ca0d98728cdbaa0481b6402bf793a290c0"}, - {file = "pyarrow-9.0.0-cp39-cp39-macosx_10_13_universal2.whl", hash = "sha256:4eebdab05afa23d5d5274b24c1cbeb1ba017d67c280f7d39fd8a8f18cbad2ec9"}, - {file = "pyarrow-9.0.0-cp39-cp39-macosx_10_13_x86_64.whl", hash = "sha256:02b820ecd1da02012092c180447de449fc688d0c3f9ff8526ca301cdd60dacd0"}, - {file = "pyarrow-9.0.0-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:92f3977e901db1ef5cba30d6cc1d7942b8d94b910c60f89013e8f7bb86a86eef"}, - {file = "pyarrow-9.0.0-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:f241bd488c2705df930eedfe304ada71191dcf67d6b98ceda0cc934fd2a8388e"}, - {file = "pyarrow-9.0.0-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:1c5a073a930c632058461547e0bc572da1e724b17b6b9eb31a97da13f50cb6e0"}, - {file = "pyarrow-9.0.0-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:f59bcd5217a3ae1e17870792f82b2ff92df9f3862996e2c78e156c13e56ff62e"}, - {file = "pyarrow-9.0.0-cp39-cp39-win_amd64.whl", hash = "sha256:fe2ce795fa1d95e4e940fe5661c3c58aee7181c730f65ac5dd8794a77228de59"}, - {file = "pyarrow-9.0.0.tar.gz", hash = "sha256:7fb02bebc13ab55573d1ae9bb5002a6d20ba767bf8569b52fce5301d42495ab7"}, -] +pyarrow = [] pycparser = [ {file = "pycparser-2.21-py2.py3-none-any.whl", hash = "sha256:8ee45429555515e1f6b185e78100aea234072576aa43ab53aefcae078162fca9"}, {file = "pycparser-2.21.tar.gz", hash = "sha256:e644fdec12f7872f86c58ff790da456218b10f863970249516d60a5eaca77206"}, ] -pydantic = [ - {file = "pydantic-1.9.2-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:9c9e04a6cdb7a363d7cb3ccf0efea51e0abb48e180c0d31dca8d247967d85c6e"}, - {file = "pydantic-1.9.2-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:fafe841be1103f340a24977f61dee76172e4ae5f647ab9e7fd1e1fca51524f08"}, - {file = "pydantic-1.9.2-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:afacf6d2a41ed91fc631bade88b1d319c51ab5418870802cedb590b709c5ae3c"}, - {file = "pydantic-1.9.2-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:3ee0d69b2a5b341fc7927e92cae7ddcfd95e624dfc4870b32a85568bd65e6131"}, - {file = "pydantic-1.9.2-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:ff68fc85355532ea77559ede81f35fff79a6a5543477e168ab3a381887caea76"}, - {file = "pydantic-1.9.2-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:c0f5e142ef8217019e3eef6ae1b6b55f09a7a15972958d44fbd228214cede567"}, - {file = "pydantic-1.9.2-cp310-cp310-win_amd64.whl", hash = "sha256:615661bfc37e82ac677543704437ff737418e4ea04bef9cf11c6d27346606044"}, - {file = "pydantic-1.9.2-cp36-cp36m-macosx_10_9_x86_64.whl", hash = "sha256:328558c9f2eed77bd8fffad3cef39dbbe3edc7044517f4625a769d45d4cf7555"}, - {file = "pydantic-1.9.2-cp36-cp36m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:2bd446bdb7755c3a94e56d7bdfd3ee92396070efa8ef3a34fab9579fe6aa1d84"}, - {file = "pydantic-1.9.2-cp36-cp36m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:e0b214e57623a535936005797567231a12d0da0c29711eb3514bc2b3cd008d0f"}, - {file = "pydantic-1.9.2-cp36-cp36m-musllinux_1_1_i686.whl", hash = "sha256:d8ce3fb0841763a89322ea0432f1f59a2d3feae07a63ea2c958b2315e1ae8adb"}, - {file = "pydantic-1.9.2-cp36-cp36m-musllinux_1_1_x86_64.whl", hash = "sha256:b34ba24f3e2d0b39b43f0ca62008f7ba962cff51efa56e64ee25c4af6eed987b"}, - {file = "pydantic-1.9.2-cp36-cp36m-win_amd64.whl", hash = "sha256:84d76ecc908d917f4684b354a39fd885d69dd0491be175f3465fe4b59811c001"}, - {file = "pydantic-1.9.2-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:4de71c718c9756d679420c69f216776c2e977459f77e8f679a4a961dc7304a56"}, - {file = "pydantic-1.9.2-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:5803ad846cdd1ed0d97eb00292b870c29c1f03732a010e66908ff48a762f20e4"}, - {file = "pydantic-1.9.2-cp37-cp37m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:a8c5360a0297a713b4123608a7909e6869e1b56d0e96eb0d792c27585d40757f"}, - {file = "pydantic-1.9.2-cp37-cp37m-musllinux_1_1_i686.whl", hash = "sha256:cdb4272678db803ddf94caa4f94f8672e9a46bae4a44f167095e4d06fec12979"}, - {file = "pydantic-1.9.2-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:19b5686387ea0d1ea52ecc4cffb71abb21702c5e5b2ac626fd4dbaa0834aa49d"}, - {file = "pydantic-1.9.2-cp37-cp37m-win_amd64.whl", hash = "sha256:32e0b4fb13ad4db4058a7c3c80e2569adbd810c25e6ca3bbd8b2a9cc2cc871d7"}, - {file = "pydantic-1.9.2-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:91089b2e281713f3893cd01d8e576771cd5bfdfbff5d0ed95969f47ef6d676c3"}, - {file = "pydantic-1.9.2-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:e631c70c9280e3129f071635b81207cad85e6c08e253539467e4ead0e5b219aa"}, - {file = "pydantic-1.9.2-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:4b3946f87e5cef3ba2e7bd3a4eb5a20385fe36521d6cc1ebf3c08a6697c6cfb3"}, - {file = "pydantic-1.9.2-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:5565a49effe38d51882cb7bac18bda013cdb34d80ac336428e8908f0b72499b0"}, - {file = "pydantic-1.9.2-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:bd67cb2c2d9602ad159389c29e4ca964b86fa2f35c2faef54c3eb28b4efd36c8"}, - {file = "pydantic-1.9.2-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:4aafd4e55e8ad5bd1b19572ea2df546ccace7945853832bb99422a79c70ce9b8"}, - {file = "pydantic-1.9.2-cp38-cp38-win_amd64.whl", hash = "sha256:d70916235d478404a3fa8c997b003b5f33aeac4686ac1baa767234a0f8ac2326"}, - {file = "pydantic-1.9.2-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:f0ca86b525264daa5f6b192f216a0d1e860b7383e3da1c65a1908f9c02f42801"}, - {file = "pydantic-1.9.2-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:1061c6ee6204f4f5a27133126854948e3b3d51fcc16ead2e5d04378c199b2f44"}, - {file = "pydantic-1.9.2-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:e78578f0c7481c850d1c969aca9a65405887003484d24f6110458fb02cca7747"}, - {file = "pydantic-1.9.2-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:5da164119602212a3fe7e3bc08911a89db4710ae51444b4224c2382fd09ad453"}, - {file = "pydantic-1.9.2-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:7ead3cd020d526f75b4188e0a8d71c0dbbe1b4b6b5dc0ea775a93aca16256aeb"}, - {file = "pydantic-1.9.2-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:7d0f183b305629765910eaad707800d2f47c6ac5bcfb8c6397abdc30b69eeb15"}, - {file = "pydantic-1.9.2-cp39-cp39-win_amd64.whl", hash = "sha256:f1a68f4f65a9ee64b6ccccb5bf7e17db07caebd2730109cb8a95863cfa9c4e55"}, - {file = "pydantic-1.9.2-py3-none-any.whl", hash = "sha256:78a4d6bdfd116a559aeec9a4cfe77dda62acc6233f8b56a716edad2651023e5e"}, - {file = "pydantic-1.9.2.tar.gz", hash = "sha256:8cb0bc509bfb71305d7a59d00163d5f9fc4530f0881ea32c74ff4f74c85f3d3d"}, -] -pygments = [ - {file = "Pygments-2.13.0-py3-none-any.whl", hash = "sha256:f643f331ab57ba3c9d89212ee4a2dabc6e94f117cf4eefde99a0574720d14c42"}, - {file = "Pygments-2.13.0.tar.gz", hash = "sha256:56a8508ae95f98e2b9bdf93a6be5ae3f7d8af858b43e02c5a2ff083726be40c1"}, -] +pydantic = [] +pygments = [] pyparsing = [ {file = "pyparsing-3.0.9-py3-none-any.whl", hash = "sha256:5026bae9a10eeaefb61dab2f09052b9f4307d44aee4eda64b309723d8d206bbc"}, {file = "pyparsing-3.0.9.tar.gz", hash = "sha256:2b020ecf7d21b687f219b71ecad3631f644a47f01403fa1d1036b0c6416d70fb"}, @@ -1013,21 +823,13 @@ requests = [ {file = "requests-2.28.1-py3-none-any.whl", hash = "sha256:8fefa2a1a1365bf5520aac41836fbee479da67864514bdb821f31ce07ce65349"}, {file = "requests-2.28.1.tar.gz", hash = "sha256:7c5599b102feddaa661c826c56ab4fee28bfd17f5abca1ebbe3e7f19d7c97983"}, ] -requests-mock = [ - {file = "requests-mock-1.9.3.tar.gz", hash = "sha256:8d72abe54546c1fc9696fa1516672f1031d72a55a1d66c85184f972a24ba0eba"}, - {file = "requests_mock-1.9.3-py2.py3-none-any.whl", hash = "sha256:0a2d38a117c08bb78939ec163522976ad59a6b7fdd82b709e23bb98004a44970"}, -] -rich = [ - {file = "rich-12.5.1-py3-none-any.whl", hash = "sha256:2eb4e6894cde1e017976d2975ac210ef515d7548bc595ba20e195fb9628acdeb"}, - {file = "rich-12.5.1.tar.gz", hash = "sha256:63a5c5ce3673d3d5fbbf23cd87e11ab84b6b451436f1b7f19ec54b6bc36ed7ca"}, -] +requests-mock = [] +rich = [] six = [ {file = "six-1.16.0-py2.py3-none-any.whl", hash = "sha256:8abb2f1d86890a2dfb989f9a77cfcfd3e47c2a354b01111771326f8aa26e0254"}, {file = "six-1.16.0.tar.gz", hash = "sha256:1e61c37477a1626458e36f7b1d82aa5c9b094fa4802892072e49de9c60c4c926"}, ] -thrift = [ - {file = "thrift-0.16.0.tar.gz", hash = "sha256:2b5b6488fcded21f9d312aa23c9ff6a0195d0f6ae26ddbd5ad9e3e25dfc14408"}, -] +thrift = [] toml = [ {file = "toml-0.10.2-py2.py3-none-any.whl", hash = "sha256:806143ae5bfb6a3c6e736a764057db0e6a0e05e338b5630894a5f779cabb4f9b"}, {file = "toml-0.10.2.tar.gz", hash = "sha256:b3bda1d108d5dd99f4a20d24d9c348e91c4db7ab1b749200bded2f839ccbe68f"}, @@ -1040,61 +842,7 @@ typing-extensions = [ {file = "typing_extensions-4.3.0-py3-none-any.whl", hash = "sha256:25642c956049920a5aa49edcdd6ab1e06d7e5d467fc00e0506c44ac86fbfca02"}, {file = "typing_extensions-4.3.0.tar.gz", hash = "sha256:e6d2677a32f47fc7eb2795db1dd15c1f34eff616bcaf2cfb5e997f854fa1c4a6"}, ] -urllib3 = [ - {file = "urllib3-1.26.11-py2.py3-none-any.whl", hash = "sha256:c33ccba33c819596124764c23a97d25f32b28433ba0dedeb77d873a38722c9bc"}, - {file = "urllib3-1.26.11.tar.gz", hash = "sha256:ea6e8fb210b19d950fab93b60c9009226c63a28808bc8386e05301e25883ac0a"}, -] -virtualenv = [ - {file = "virtualenv-20.16.3-py2.py3-none-any.whl", hash = "sha256:4193b7bc8a6cd23e4eb251ac64f29b4398ab2c233531e66e40b19a6b7b0d30c1"}, - {file = "virtualenv-20.16.3.tar.gz", hash = "sha256:d86ea0bb50e06252d79e6c241507cb904fcd66090c3271381372d6221a3970f9"}, -] -zipp = [ - {file = "zipp-3.8.1-py3-none-any.whl", hash = "sha256:47c40d7fe183a6f21403a199b3e4192cca5774656965b0a4988ad2f8feb5f009"}, - {file = "zipp-3.8.1.tar.gz", hash = "sha256:05b45f1ee8f807d0cc928485ca40a07cb491cf092ff587c0df9cb1fd154848d2"}, -] -zstandard = [ - {file = "zstandard-0.18.0-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:ef7e8a200e4c8ac9102ed3c90ed2aa379f6b880f63032200909c1be21951f556"}, - {file = "zstandard-0.18.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:2dc466207016564805e56d28375f4f533b525ff50d6776946980dff5465566ac"}, - {file = "zstandard-0.18.0-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:4a2ee1d4f98447f3e5183ecfce5626f983504a4a0c005fbe92e60fa8e5d547ec"}, - {file = "zstandard-0.18.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:d956e2f03c7200d7e61345e0880c292783ec26618d0d921dcad470cb195bbce2"}, - {file = "zstandard-0.18.0-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_12_i686.manylinux2010_i686.whl", hash = "sha256:ce6f59cba9854fd14da5bfe34217a1501143057313966637b7291d1b0267bd1e"}, - {file = "zstandard-0.18.0-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:a7fa67cba473623848b6e88acf8d799b1906178fd883fb3a1da24561c779593b"}, - {file = "zstandard-0.18.0-cp310-cp310-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:cdb44d7284c8c5dd1b66dfb86dda7f4560fa94bfbbc1d2da749ba44831335e32"}, - {file = "zstandard-0.18.0-cp310-cp310-win32.whl", hash = "sha256:63694a376cde0aa8b1971d06ca28e8f8b5f492779cb6ee1cc46bbc3f019a42a5"}, - {file = "zstandard-0.18.0-cp310-cp310-win_amd64.whl", hash = "sha256:702a8324cd90c74d9c8780d02bf55e79da3193c870c9665ad3a11647e3ad1435"}, - {file = "zstandard-0.18.0-cp36-cp36m-macosx_10_9_x86_64.whl", hash = "sha256:46f679bc5dfd938db4fb058218d9dc4db1336ffaf1ea774ff152ecadabd40805"}, - {file = "zstandard-0.18.0-cp36-cp36m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:dc2a4de9f363b3247d472362a65041fe4c0f59e01a2846b15d13046be866a885"}, - {file = "zstandard-0.18.0-cp36-cp36m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:bd3220d7627fd4d26397211cb3b560ec7cc4a94b75cfce89e847e8ce7fabe32d"}, - {file = "zstandard-0.18.0-cp36-cp36m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_12_i686.manylinux2010_i686.whl", hash = "sha256:39e98cf4773234bd9cebf9f9db730e451dfcfe435e220f8921242afda8321887"}, - {file = "zstandard-0.18.0-cp36-cp36m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:5228e596eb1554598c872a337bbe4e5afe41cd1f8b1b15f2e35b50d061e35244"}, - {file = "zstandard-0.18.0-cp36-cp36m-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:d4a8fd45746a6c31e729f35196e80b8f1e9987c59f5ccb8859d7c6a6fbeb9c63"}, - {file = "zstandard-0.18.0-cp36-cp36m-win32.whl", hash = "sha256:4cbb85f29a990c2fdbf7bc63246567061a362ddca886d7fae6f780267c0a9e67"}, - {file = "zstandard-0.18.0-cp36-cp36m-win_amd64.whl", hash = "sha256:bfa6c8549fa18e6497a738b7033c49f94a8e2e30c5fbe2d14d0b5aa8bbc1695d"}, - {file = "zstandard-0.18.0-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:e02043297c1832f2666cd2204f381bef43b10d56929e13c42c10c732c6e3b4ed"}, - {file = "zstandard-0.18.0-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:7231543d38d2b7e02ef7cc78ef7ffd86419437e1114ff08709fe25a160e24bd6"}, - {file = "zstandard-0.18.0-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:c86befac87445927488f5c8f205d11566f64c11519db223e9d282b945fa60dab"}, - {file = "zstandard-0.18.0-cp37-cp37m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_12_i686.manylinux2010_i686.whl", hash = "sha256:999a4e1768f219826ba3fa2064fab1c86dd72fdd47a42536235478c3bb3ca3e2"}, - {file = "zstandard-0.18.0-cp37-cp37m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:9df59cd1cf3c62075ee2a4da767089d19d874ac3ad42b04a71a167e91b384722"}, - {file = "zstandard-0.18.0-cp37-cp37m-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:1be31e9e3f7607ee0cdd60915410a5968b205d3e7aa83b7fcf3dd76dbbdb39e0"}, - {file = "zstandard-0.18.0-cp37-cp37m-win32.whl", hash = "sha256:490d11b705b8ae9dc845431bacc8dd1cef2408aede176620a5cd0cd411027936"}, - {file = "zstandard-0.18.0-cp37-cp37m-win_amd64.whl", hash = "sha256:266aba27fa9cc5e9091d3d325ebab1fa260f64e83e42516d5e73947c70216a5b"}, - {file = "zstandard-0.18.0-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:8b2260c4e07dd0723eadb586de7718b61acca4083a490dda69c5719d79bc715c"}, - {file = "zstandard-0.18.0-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:3af8c2383d02feb6650e9255491ec7d0824f6e6dd2bbe3e521c469c985f31fb1"}, - {file = "zstandard-0.18.0-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:28723a1d2e4df778573b76b321ebe9f3469ac98988104c2af116dd344802c3f8"}, - {file = "zstandard-0.18.0-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:19cac7108ff2c342317fad6dc97604b47a41f403c8f19d0bfc396dfadc3638b8"}, - {file = "zstandard-0.18.0-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_12_i686.manylinux2010_i686.whl", hash = "sha256:76725d1ee83a8915100a310bbad5d9c1fc6397410259c94033b8318d548d9990"}, - {file = "zstandard-0.18.0-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:d716a7694ce1fa60b20bc10f35c4a22be446ef7f514c8dbc8f858b61976de2fb"}, - {file = "zstandard-0.18.0-cp38-cp38-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:49685bf9a55d1ab34bd8423ea22db836ba43a181ac6b045ac4272093d5cb874e"}, - {file = "zstandard-0.18.0-cp38-cp38-win32.whl", hash = "sha256:1af1268a7dc870eb27515fb8db1f3e6c5a555d2b7bcc476fc3bab8886c7265ab"}, - {file = "zstandard-0.18.0-cp38-cp38-win_amd64.whl", hash = "sha256:1dc2d3809e763055a1a6c1a73f2b677320cc9a5aa1a7c6cfb35aee59bddc42d9"}, - {file = "zstandard-0.18.0-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:eea18c1e7442f2aa9aff1bb84550dbb6a1f711faf6e48e7319de8f2b2e923c2a"}, - {file = "zstandard-0.18.0-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:8677ffc6a6096cccbd892e558471c901fd821aba12b7fbc63833c7346f549224"}, - {file = "zstandard-0.18.0-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:083dc08abf03807af9beeb2b6a91c23ad78add2499f828176a3c7b742c44df02"}, - {file = "zstandard-0.18.0-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:c990063664c08169c84474acecc9251ee035871589025cac47c060ff4ec4bc1a"}, - {file = "zstandard-0.18.0-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_12_i686.manylinux2010_i686.whl", hash = "sha256:533db8a6fac6248b2cb2c935e7b92f994efbdeb72e1ffa0b354432e087bb5a3e"}, - {file = "zstandard-0.18.0-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:dbb3cb8a082d62b8a73af42291569d266b05605e017a3d8a06a0e5c30b5f10f0"}, - {file = "zstandard-0.18.0-cp39-cp39-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:d6c85ca5162049ede475b7ec98e87f9390501d44a3d6776ddd504e872464ec25"}, - {file = "zstandard-0.18.0-cp39-cp39-win32.whl", hash = "sha256:75479e7c2b3eebf402c59fbe57d21bc400cefa145ca356ee053b0a08908c5784"}, - {file = "zstandard-0.18.0-cp39-cp39-win_amd64.whl", hash = "sha256:d85bfabad444812133a92fc6fbe463e1d07581dba72f041f07a360e63808b23c"}, - {file = "zstandard-0.18.0.tar.gz", hash = "sha256:0ac0357a0d985b4ff31a854744040d7b5754385d1f98f7145c30e02c6865cb6f"}, -] +urllib3 = [] +virtualenv = [] +zipp = [] +zstandard = [] diff --git a/pyproject.toml b/pyproject.toml index 86cd1995af..b55a6ce5c0 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -64,6 +64,7 @@ pre-commit = "^2.0.0" fastavro = "^1.5.4" coverage = { version = "^6.4.4", extras = ["toml"] } requests-mock = "^1.9.3" +PyYAML = "^6.0.0" [tool.poetry.scripts] pyiceberg = "pyiceberg.cli.console:run" From d619f48c61120235a45f915bcdd3f8f9a5643ba5 Mon Sep 17 00:00:00 2001 From: Fokko Driesprong Date: Wed, 31 Aug 2022 19:28:25 +0200 Subject: [PATCH 189/642] Python: Reassign schema/partition-spec/sort-order IDs (#5627) * Python: Reassign schema/partition-spec/sort-order ids When creating a new schema Resolves #5468 * Convert into pre-order * Small docstring improvements * Fix order of the structs/lists/maps --- poetry.lock | 61 ++++++-- pyiceberg/catalog/hive.py | 22 +-- pyiceberg/catalog/rest.py | 5 +- pyiceberg/schema.py | 154 ++++++++++++++++++- pyiceberg/serializers.py | 9 +- pyiceberg/table/__init__.py | 13 +- pyiceberg/table/metadata.py | 56 +++++-- pyiceberg/table/partitioning.py | 22 +++ pyiceberg/table/sorting.py | 33 ++++ pyiceberg/types.py | 1 + pyiceberg/utils/iceberg_base_model.py | 19 ++- tests/catalog/test_hive.py | 15 +- tests/cli/test_console.py | 3 +- tests/table/test_init.py | 3 +- tests/table/test_metadata.py | 208 ++++++++++++++++++++++++-- tests/table/test_sorting.py | 4 +- 16 files changed, 533 insertions(+), 95 deletions(-) diff --git a/poetry.lock b/poetry.lock index 0aeb687a58..45f58cf672 100644 --- a/poetry.lock +++ b/poetry.lock @@ -302,14 +302,14 @@ python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*" [[package]] name = "pydantic" -version = "1.9.2" +version = "1.10.0" description = "Data validation and settings management using python type hints" category = "main" optional = false -python-versions = ">=3.6.1" +python-versions = ">=3.7" [package.dependencies] -typing-extensions = ">=3.7.4.3" +typing-extensions = ">=4.1.0" [package.extras] dotenv = ["python-dotenv (>=0.10.4)"] @@ -411,7 +411,7 @@ use_chardet_on_py3 = ["chardet (>=3.0.2,<6)"] [[package]] name = "requests-mock" -version = "1.9.3" +version = "1.10.0" description = "Mock out responses from the requests package" category = "dev" optional = false @@ -423,7 +423,7 @@ six = "*" [package.extras] fixture = ["fixtures"] -test = ["fixtures", "mock", "purl", "pytest", "sphinx", "testrepository (>=0.0.18)", "testtools"] +test = ["fixtures", "mock", "purl", "pytest", "requests-futures", "sphinx", "testrepository (>=0.0.18)", "testtools"] [[package]] name = "rich" @@ -504,7 +504,7 @@ socks = ["PySocks (>=1.5.6,!=1.5.7,<2.0)"] [[package]] name = "virtualenv" -version = "20.16.3" +version = "20.16.4" description = "Virtual Python Environment builder" category = "dev" optional = false @@ -720,7 +720,44 @@ pycparser = [ {file = "pycparser-2.21-py2.py3-none-any.whl", hash = "sha256:8ee45429555515e1f6b185e78100aea234072576aa43ab53aefcae078162fca9"}, {file = "pycparser-2.21.tar.gz", hash = "sha256:e644fdec12f7872f86c58ff790da456218b10f863970249516d60a5eaca77206"}, ] -pydantic = [] +pydantic = [ + {file = "pydantic-1.10.0-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:7e34e46dd08dafd4c75b8378efe3eae7d8e5212950fcd894d86c1df2dcfb80fe"}, + {file = "pydantic-1.10.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:4af55f33ae5be6cccecd4fa462630daffef1f161f60c3f194b24eca705d50748"}, + {file = "pydantic-1.10.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:1856bc6640aced42886f7ee48f5ed1fa5adf35e34064b5f9532b52d5a3b8a0d3"}, + {file = "pydantic-1.10.0-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:d73ae7e210929a1b7d288034835dd787e5b0597192d58ab7342bacbeec0f33df"}, + {file = "pydantic-1.10.0-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:a1192c17667d21652ab93b5eecd1a776cd0a4e384ea8c331bb830c9d130293af"}, + {file = "pydantic-1.10.0-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:026427be4e251f876e7519a63af37ae5ebb8b593ca8b02180bdc6becd1ea4ef4"}, + {file = "pydantic-1.10.0-cp310-cp310-win_amd64.whl", hash = "sha256:d1dffae1f219d06a997ec78d1d2daafdbfecf243ad8eb36bfbcbc73e30e17385"}, + {file = "pydantic-1.10.0-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:b549eebe8de4e50fc3b4f8c1f9cc2f731d91787fc3f7d031561668377b8679bc"}, + {file = "pydantic-1.10.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:a0ba8710bfdaddb7424c05ad2dc1da04796003751eac6ad30c218ac1d68a174e"}, + {file = "pydantic-1.10.0-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:f0985ba95af937389c9ce8d747138417303569cb736bd12469646ef53cd66e1c"}, + {file = "pydantic-1.10.0-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:d484fbbe6267b6c936a6d005d5170ab553f3f4367348c7e88d3e17f0a7179981"}, + {file = "pydantic-1.10.0-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:9500586151cd56a20bacb8f1082df1b4489000120d1c7ddc44c8b20870e8adbd"}, + {file = "pydantic-1.10.0-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:1b5212604aaf5954e9a7cea8f0c60d6dbef996aa7b41edefd329e6b5011ce8cf"}, + {file = "pydantic-1.10.0-cp311-cp311-win_amd64.whl", hash = "sha256:39212b3853eea165a3cda11075d5b7d09d4291fcbc3c0ecefd23797ee21b29e9"}, + {file = "pydantic-1.10.0-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:b3e3aed33fbd9518cf508d5415a58af683743d53dc5e58953973d73605774f34"}, + {file = "pydantic-1.10.0-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:ed4e5c18cac70fadd4cf339f444c4f1795f0876dfd5b70cf0a841890b52f0001"}, + {file = "pydantic-1.10.0-cp37-cp37m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:45a6d0a9fdaad2a27ea69aec4659705ed8f60a5664e892c73e2b977d8f5166cc"}, + {file = "pydantic-1.10.0-cp37-cp37m-musllinux_1_1_i686.whl", hash = "sha256:158f1479367da20914961b5406ac3b29dfe1d858ae2af96c444f73543defcf0c"}, + {file = "pydantic-1.10.0-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:172aaeeaff8fc3ac326fb8a2934a063ca0938586c5fe8848285052de83a240f7"}, + {file = "pydantic-1.10.0-cp37-cp37m-win_amd64.whl", hash = "sha256:231b19c010288bfbfdcd3f79df38b5ff893c6547cd8c7d006203435790b22815"}, + {file = "pydantic-1.10.0-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:22206c152f9b86c0ee169928f9c24e1c0c566edb2462600b298ccb04860961aa"}, + {file = "pydantic-1.10.0-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:d8ef840ef803ef17a7bd52480eb85faca0eed728d70233fd560f7d1066330247"}, + {file = "pydantic-1.10.0-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:1f99b4de6936a0f9fe255d1c7fdc447700ddd027c9ad38a612d453ed5fc7d6d0"}, + {file = "pydantic-1.10.0-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:134b4fd805737496ce4efd24ce2f8da0e08c66dcfc054fee1a19673eec780f2c"}, + {file = "pydantic-1.10.0-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:c4c76af6ad47bc46cf16bd0e4a5e536a7a2bec0dec14ea08b712daa6645bf293"}, + {file = "pydantic-1.10.0-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:e03402b0a6b23a2d0b9ee31e45d80612c95562b5af8b5c900171b9d9015ddc5f"}, + {file = "pydantic-1.10.0-cp38-cp38-win_amd64.whl", hash = "sha256:3a3a60fcb5ce08cab593b7978d02db67b8d153e9d582adab7c0b69d7200d78be"}, + {file = "pydantic-1.10.0-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:d8e5c5a50821c55b76dcf422610225cb7e44685cdd81832d0d504fa8c9343f35"}, + {file = "pydantic-1.10.0-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:645b83297a9428a675c98c1f69a7237a381900e34f23245c0ea73d74e454bf68"}, + {file = "pydantic-1.10.0-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:95ab3f31f35dc4f8fc85b04d13569e5fdc9de2d3050ae64c1fdc3430dfe7d92d"}, + {file = "pydantic-1.10.0-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:e290915a0ed53d3c59d6071fc7d2c843ed04c33affcd752dd1f3daa859b44a76"}, + {file = "pydantic-1.10.0-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:af669da39ede365069dbc5de56564b011e3353f801acdbdd7145002a78abc3d9"}, + {file = "pydantic-1.10.0-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:8e796f915762dec4678fafc89b1f0441ab9209517a8a682ddb3f988f7ffe0827"}, + {file = "pydantic-1.10.0-cp39-cp39-win_amd64.whl", hash = "sha256:652727f9e1d3ae30bd8a4dfbebcafd50df45277b97f3deabbbfedcf731f94aa5"}, + {file = "pydantic-1.10.0-py3-none-any.whl", hash = "sha256:4d2b9258f5bd2d129bd4cf2d31f9d40094b9ed6ef64896e2f7a70729b2d599ea"}, + {file = "pydantic-1.10.0.tar.gz", hash = "sha256:e13788fcad1baf5eb3236856b2a9a74f7dac6b3ea7ca1f60a4ad8bad4239cf4c"}, +] pygments = [] pyparsing = [ {file = "pyparsing-3.0.9-py3-none-any.whl", hash = "sha256:5026bae9a10eeaefb61dab2f09052b9f4307d44aee4eda64b309723d8d206bbc"}, @@ -823,7 +860,10 @@ requests = [ {file = "requests-2.28.1-py3-none-any.whl", hash = "sha256:8fefa2a1a1365bf5520aac41836fbee479da67864514bdb821f31ce07ce65349"}, {file = "requests-2.28.1.tar.gz", hash = "sha256:7c5599b102feddaa661c826c56ab4fee28bfd17f5abca1ebbe3e7f19d7c97983"}, ] -requests-mock = [] +requests-mock = [ + {file = "requests-mock-1.10.0.tar.gz", hash = "sha256:59c9c32419a9fb1ae83ec242d98e889c45bd7d7a65d48375cc243ec08441658b"}, + {file = "requests_mock-1.10.0-py2.py3-none-any.whl", hash = "sha256:2fdbb637ad17ee15c06f33d31169e71bf9fe2bdb7bc9da26185be0dd8d842699"}, +] rich = [] six = [ {file = "six-1.16.0-py2.py3-none-any.whl", hash = "sha256:8abb2f1d86890a2dfb989f9a77cfcfd3e47c2a354b01111771326f8aa26e0254"}, @@ -843,6 +883,9 @@ typing-extensions = [ {file = "typing_extensions-4.3.0.tar.gz", hash = "sha256:e6d2677a32f47fc7eb2795db1dd15c1f34eff616bcaf2cfb5e997f854fa1c4a6"}, ] urllib3 = [] -virtualenv = [] +virtualenv = [ + {file = "virtualenv-20.16.4-py3-none-any.whl", hash = "sha256:035ed57acce4ac35c82c9d8802202b0e71adac011a511ff650cbcf9635006a22"}, + {file = "virtualenv-20.16.4.tar.gz", hash = "sha256:014f766e4134d0008dcaa1f95bafa0fb0f575795d07cae50b1bee514185d6782"}, +] zipp = [] zstandard = [] diff --git a/pyiceberg/catalog/hive.py b/pyiceberg/catalog/hive.py index efa578b3a1..2ffbe66a99 100644 --- a/pyiceberg/catalog/hive.py +++ b/pyiceberg/catalog/hive.py @@ -61,7 +61,7 @@ from pyiceberg.schema import Schema, SchemaVisitor, visit from pyiceberg.serializers import FromInputFile, ToOutputFile from pyiceberg.table import Table -from pyiceberg.table.metadata import DEFAULT_LAST_PARTITION_ID, TableMetadataV1, TableMetadataV2 +from pyiceberg.table.metadata import TableMetadata, new_table_metadata from pyiceberg.table.partitioning import UNPARTITIONED_PARTITION_SPEC, PartitionSpec from pyiceberg.table.sorting import UNSORTED_SORT_ORDER, SortOrder from pyiceberg.typedef import EMPTY_DICT @@ -257,10 +257,10 @@ def _convert_hive_into_iceberg(self, table: HiveTable, io: FileIO) -> Table: metadata = FromInputFile.table_metadata(file) return Table(identifier=(table.dbName, table.tableName), metadata=metadata, metadata_location=metadata_location) - def _write_metadata(self, metadata: Union[TableMetadataV1, TableMetadataV2], io: FileIO, metadata_path: str): + def _write_metadata(self, metadata: TableMetadata, io: FileIO, metadata_path: str): ToOutputFile.table_metadata(metadata, io.new_output(metadata_path)) - def _resolve_table_location(self, location: Optional[str], database_name: str, table_name: str): + def _resolve_table_location(self, location: Optional[str], database_name: str, table_name: str) -> str: if not location: database_properties = self.load_namespace_properties(database_name) if database_location := database_properties.get(LOCATION): @@ -305,20 +305,8 @@ def create_table( location = self._resolve_table_location(location, database_name, table_name) metadata_location = f"{location}/metadata/00000-{uuid.uuid4()}.metadata.json" - metadata = TableMetadataV2( - location=location, - schemas=[schema], - current_schema_id=schema.schema_id, - partition_specs=[partition_spec], - default_spec_id=partition_spec.spec_id, - sort_orders=[sort_order], - default_sort_order_id=sort_order.order_id, - properties=properties or {}, - last_updated_ms=current_time_millis, - last_column_id=schema.highest_field_id, - last_partition_id=max(field.field_id for field in partition_spec.fields) - if partition_spec.fields - else DEFAULT_LAST_PARTITION_ID, + metadata = new_table_metadata( + location=location, schema=schema, partition_spec=partition_spec, sort_order=sort_order, properties=properties ) io = load_file_io({**self.properties, **properties}, location=location) self._write_metadata(metadata, io, metadata_location) diff --git a/pyiceberg/catalog/rest.py b/pyiceberg/catalog/rest.py index 29b5d55d9b..4931e5671b 100644 --- a/pyiceberg/catalog/rest.py +++ b/pyiceberg/catalog/rest.py @@ -52,8 +52,7 @@ UnauthorizedError, ) from pyiceberg.schema import Schema -from pyiceberg.table import Table -from pyiceberg.table.metadata import TableMetadataV1, TableMetadataV2 +from pyiceberg.table import Table, TableMetadata from pyiceberg.table.partitioning import UNPARTITIONED_PARTITION_SPEC, PartitionSpec from pyiceberg.table.sorting import UNSORTED_SORT_ORDER, SortOrder from pyiceberg.typedef import EMPTY_DICT @@ -94,7 +93,7 @@ class Endpoints: class TableResponse(IcebergBaseModel): metadata_location: str = Field(alias="metadata-location") - metadata: Union[TableMetadataV1, TableMetadataV2] = Field() + metadata: TableMetadata = Field() config: Properties = Field(default_factory=dict) diff --git a/pyiceberg/schema.py b/pyiceberg/schema.py index 657f8fbb07..6620759f86 100644 --- a/pyiceberg/schema.py +++ b/pyiceberg/schema.py @@ -15,12 +15,13 @@ # specific language governing permissions and limitations # under the License. # pylint: disable=W0511 - +import itertools from abc import ABC, abstractmethod from dataclasses import dataclass -from functools import cached_property, singledispatch +from functools import cached_property, partial, singledispatch from typing import ( Any, + Callable, Dict, Generic, List, @@ -46,6 +47,8 @@ T = TypeVar("T") +INITIAL_SCHEMA_ID = 0 + class Schema(IcebergBaseModel): """A table Schema @@ -57,7 +60,7 @@ class Schema(IcebergBaseModel): type: Literal["struct"] = "struct" fields: Tuple[NestedField, ...] = Field(default_factory=tuple) - schema_id: int = Field(alias="schema-id") + schema_id: int = Field(alias="schema-id", default=INITIAL_SCHEMA_ID) identifier_field_ids: List[int] = Field(alias="identifier-field-ids", default_factory=list) _name_to_id: Dict[str, int] = PrivateAttr() @@ -149,7 +152,7 @@ def find_field(self, name_or_id: Union[str, int], case_sensitive: bool = True) - else: field_id = self._lazy_name_to_id_lower.get(name_or_id.lower()) - if not field_id: + if field_id is None: raise ValueError(f"Could not find field with name or id {name_or_id}, case_sensitive={case_sensitive}") return self._lazy_id_to_field.get(field_id) @@ -276,6 +279,32 @@ def primitive(self, primitive: PrimitiveType) -> T: """Visit a PrimitiveType""" +class PreOrderSchemaVisitor(Generic[T], ABC): + @abstractmethod + def schema(self, schema: Schema, struct_result: Callable[[], T]) -> T: + """Visit a Schema""" + + @abstractmethod + def struct(self, struct: StructType, field_results: List[Callable[[], T]]) -> T: + """Visit a StructType""" + + @abstractmethod + def field(self, field: NestedField, field_result: Callable[[], T]) -> T: + """Visit a NestedField""" + + @abstractmethod + def list(self, list_type: ListType, element_result: Callable[[], T]) -> T: + """Visit a ListType""" + + @abstractmethod + def map(self, map_type: MapType, key_result: Callable[[], T], value_result: Callable[[], T]) -> T: + """Visit a MapType""" + + @abstractmethod + def primitive(self, primitive: PrimitiveType) -> T: + """Visit a PrimitiveType""" + + @dataclass(init=True, eq=True, frozen=True) class Accessor: """An accessor for a specific position in a container that implements the StructProtocol""" @@ -375,6 +404,63 @@ def _(obj: PrimitiveType, visitor: SchemaVisitor[T]) -> T: return visitor.primitive(obj) +@singledispatch +def pre_order_visit(obj, visitor: PreOrderSchemaVisitor[T]) -> T: + """A generic function for applying a schema visitor to any point within a schema + + The function traverses the schema in pre-order fashion. This is a slimmed down version + compared to the post-order traversal (missing before and after methods), mostly + because we don't use the pre-order traversal much. + + Args: + obj(Schema | IcebergType): An instance of a Schema or an IcebergType + visitor (PreOrderSchemaVisitor[T]): An instance of an implementation of the generic PreOrderSchemaVisitor base class + + Raises: + NotImplementedError: If attempting to visit an unrecognized object type + """ + raise NotImplementedError("Cannot visit non-type: %s" % obj) + + +@pre_order_visit.register(Schema) +def _(obj: Schema, visitor: PreOrderSchemaVisitor[T]) -> T: + """Visit a Schema with a concrete PreOrderSchemaVisitor""" + return visitor.schema(obj, lambda: pre_order_visit(obj.as_struct(), visitor)) + + +@pre_order_visit.register(StructType) +def _(obj: StructType, visitor: PreOrderSchemaVisitor[T]) -> T: + """Visit a StructType with a concrete PreOrderSchemaVisitor""" + return visitor.struct( + obj, + [ + partial( + lambda field: visitor.field(field, partial(lambda field: pre_order_visit(field.field_type, visitor), field)), + field, + ) + for field in obj.fields + ], + ) + + +@pre_order_visit.register(ListType) +def _(obj: ListType, visitor: PreOrderSchemaVisitor[T]) -> T: + """Visit a ListType with a concrete PreOrderSchemaVisitor""" + return visitor.list(obj, lambda: pre_order_visit(obj.element_type, visitor)) + + +@pre_order_visit.register(MapType) +def _(obj: MapType, visitor: PreOrderSchemaVisitor[T]) -> T: + """Visit a MapType with a concrete PreOrderSchemaVisitor""" + return visitor.map(obj, lambda: pre_order_visit(obj.key_type, visitor), lambda: pre_order_visit(obj.value_type, visitor)) + + +@pre_order_visit.register(PrimitiveType) +def _(obj: PrimitiveType, visitor: PreOrderSchemaVisitor[T]) -> T: + """Visit a PrimitiveType with a concrete PreOrderSchemaVisitor""" + return visitor.primitive(obj) + + class _IndexById(SchemaVisitor[Dict[int, NestedField]]): """A schema visitor for generating a field ID to NestedField index""" @@ -638,3 +724,63 @@ def map(self, map_type: MapType, key_result: int, value_result: int) -> int: def primitive(self, primitive: PrimitiveType) -> int: return 0 + + +def assign_fresh_schema_ids(schema: Schema) -> Schema: + """Traverses the schema, and sets new IDs""" + return pre_order_visit(schema, _SetFreshIDs()) + + +class _SetFreshIDs(PreOrderSchemaVisitor[IcebergType]): + """Traverses the schema and assigns monotonically increasing ids""" + + counter: itertools.count + reserved_ids: Dict[int, int] + + def __init__(self, start: int = 1) -> None: + self.counter = itertools.count(start) + self.reserved_ids = {} + + def _get_and_increment(self) -> int: + return next(self.counter) + + def schema(self, schema: Schema, struct_result: Callable[[], StructType]) -> Schema: + # First we keep the original identifier_field_ids here, we remap afterwards + fields = struct_result().fields + return Schema(*fields, identifier_field_ids=[self.reserved_ids[field_id] for field_id in schema.identifier_field_ids]) + + def struct(self, struct: StructType, field_results: List[Callable[[], IcebergType]]) -> StructType: + # assign IDs for this struct's fields first + self.reserved_ids.update({field.field_id: self._get_and_increment() for field in struct.fields}) + return StructType(*[field() for field in field_results]) + + def field(self, field: NestedField, field_result: Callable[[], IcebergType]) -> IcebergType: + return NestedField( + field_id=self.reserved_ids[field.field_id], + name=field.name, + field_type=field_result(), + required=field.required, + doc=field.doc, + ) + + def list(self, list_type: ListType, element_result: Callable[[], IcebergType]) -> ListType: + self.reserved_ids[list_type.element_id] = self._get_and_increment() + return ListType( + element_id=self.reserved_ids[list_type.element_id], + element=element_result(), + element_required=list_type.element_required, + ) + + def map(self, map_type: MapType, key_result: Callable[[], IcebergType], value_result: Callable[[], IcebergType]) -> MapType: + self.reserved_ids[map_type.key_id] = self._get_and_increment() + self.reserved_ids[map_type.value_id] = self._get_and_increment() + return MapType( + key_id=self.reserved_ids[map_type.key_id], + key_type=key_result(), + value_id=self.reserved_ids[map_type.value_id], + value_type=value_result(), + value_required=map_type.value_required, + ) + + def primitive(self, primitive: PrimitiveType) -> PrimitiveType: + return primitive diff --git a/pyiceberg/serializers.py b/pyiceberg/serializers.py index fde592e5fc..7f4c884214 100644 --- a/pyiceberg/serializers.py +++ b/pyiceberg/serializers.py @@ -17,10 +17,9 @@ import codecs import json -from typing import Union from pyiceberg.io import InputFile, InputStream, OutputFile -from pyiceberg.table.metadata import TableMetadata, TableMetadataV1, TableMetadataV2 +from pyiceberg.table.metadata import TableMetadata, TableMetadataUtil class FromByteStream: @@ -36,7 +35,7 @@ def table_metadata(byte_stream: InputStream, encoding: str = "utf-8") -> TableMe """ reader = codecs.getreader(encoding) metadata = json.load(reader(byte_stream)) - return TableMetadata.parse_obj(metadata) + return TableMetadataUtil.parse_obj(metadata) class FromInputFile: @@ -62,9 +61,7 @@ class ToOutputFile: """A collection of methods that serialize Iceberg objects into files given an OutputFile instance""" @staticmethod - def table_metadata( - metadata: Union[TableMetadataV1, TableMetadataV2], output_file: OutputFile, overwrite: bool = False - ) -> None: + def table_metadata(metadata: TableMetadata, output_file: OutputFile, overwrite: bool = False) -> None: """Write a TableMetadata instance to an output file Args: diff --git a/pyiceberg/table/__init__.py b/pyiceberg/table/__init__.py index edcd1a3230..73a2bcc9b5 100644 --- a/pyiceberg/table/__init__.py +++ b/pyiceberg/table/__init__.py @@ -14,17 +14,14 @@ # KIND, either express or implied. See the License for the # specific language governing permissions and limitations # under the License. -from typing import ( - Dict, - List, - Optional, - Union, -) + + +from typing import Dict, List, Optional from pydantic import Field from pyiceberg.schema import Schema -from pyiceberg.table.metadata import TableMetadataV1, TableMetadataV2 +from pyiceberg.table.metadata import TableMetadata from pyiceberg.table.partitioning import PartitionSpec from pyiceberg.table.snapshots import Snapshot, SnapshotLogEntry from pyiceberg.table.sorting import SortOrder @@ -35,7 +32,7 @@ class Table(IcebergBaseModel): identifier: Identifier = Field() metadata_location: str = Field() - metadata: Union[TableMetadataV1, TableMetadataV2] = Field() + metadata: TableMetadata = Field() def refresh(self): """Refresh the current table metadata""" diff --git a/pyiceberg/table/metadata.py b/pyiceberg/table/metadata.py index d5514a5bec..b0818fc1e8 100644 --- a/pyiceberg/table/metadata.py +++ b/pyiceberg/table/metadata.py @@ -14,6 +14,7 @@ # KIND, either express or implied. See the License for the # specific language governing permissions and limitations # under the License. +import datetime import uuid from copy import copy from typing import ( @@ -24,16 +25,22 @@ Optional, Union, ) -from uuid import UUID, uuid4 from pydantic import Field, root_validator from pyiceberg.exceptions import ValidationError -from pyiceberg.schema import Schema -from pyiceberg.table.partitioning import PartitionSpec +from pyiceberg.schema import Schema, assign_fresh_schema_ids +from pyiceberg.table.partitioning import PartitionSpec, assign_fresh_partition_spec_ids from pyiceberg.table.refs import MAIN_BRANCH, SnapshotRef, SnapshotRefType from pyiceberg.table.snapshots import MetadataLogEntry, Snapshot, SnapshotLogEntry -from pyiceberg.table.sorting import UNSORTED_SORT_ORDER, UNSORTED_SORT_ORDER_ID, SortOrder +from pyiceberg.table.sorting import ( + UNSORTED_SORT_ORDER, + UNSORTED_SORT_ORDER_ID, + SortOrder, + assign_fresh_sort_order_ids, +) +from pyiceberg.typedef import EMPTY_DICT, Properties +from pyiceberg.utils.datetime import datetime_to_micros from pyiceberg.utils.iceberg_base_model import IcebergBaseModel INITIAL_SEQUENCE_NUMBER = 0 @@ -103,12 +110,12 @@ def construct_refs(cls, data: Dict[str, Any]): """The table’s base location. This is used by writers to determine where to store data files, manifest files, and table metadata files.""" - table_uuid: Optional[UUID] = Field(alias="table-uuid", default_factory=uuid4) + table_uuid: uuid.UUID = Field(alias="table-uuid", default_factory=uuid.uuid4) """A UUID that identifies the table, generated when the table is created. Implementations must throw an exception if a table’s UUID does not match the expected UUID after refreshing metadata.""" - last_updated_ms: int = Field(alias="last-updated-ms") + last_updated_ms: int = Field(alias="last-updated-ms", default_factory=lambda: datetime_to_micros(datetime.datetime.now())) """Timestamp in milliseconds from the unix epoch when the table was last updated. Each table metadata file should update this field just before writing.""" @@ -141,7 +148,7 @@ def construct_refs(cls, data: Dict[str, Any]): to be used for arbitrary metadata. For example, commit.retry.num-retries is used to control the number of commit retries.""" - current_snapshot_id: Optional[int] = Field(alias="current-snapshot-id") + current_snapshot_id: Optional[int] = Field(alias="current-snapshot-id", default=None) """ID of the current table snapshot.""" snapshots: List[Snapshot] = Field(default_factory=list) @@ -327,24 +334,43 @@ def check_sort_orders(cls, values: Dict[str, Any]): based on the spec. Implementations must throw an exception if a table’s version is higher than the supported version.""" - table_uuid: UUID = Field(alias="table-uuid", default_factory=uuid.uuid4) - """A UUID that identifies the table, generated when the table is created. - Implementations must throw an exception if a table’s UUID does not match - the expected UUID after refreshing metadata.""" - last_sequence_number: int = Field(alias="last-sequence-number", default=INITIAL_SEQUENCE_NUMBER) """The table’s highest assigned sequence number, a monotonically increasing long that tracks the order of snapshots in a table.""" -class TableMetadata: +TableMetadata = Union[TableMetadataV1, TableMetadataV2] + + +def new_table_metadata( + schema: Schema, partition_spec: PartitionSpec, sort_order: SortOrder, location: str, properties: Properties = EMPTY_DICT +) -> TableMetadata: + fresh_schema = assign_fresh_schema_ids(schema) + fresh_partition_spec = assign_fresh_partition_spec_ids(partition_spec, schema, fresh_schema) + fresh_sort_order = assign_fresh_sort_order_ids(sort_order, schema, fresh_schema) + + return TableMetadataV2( + location=location, + schemas=[fresh_schema], + last_column_id=fresh_schema.highest_field_id, + current_schema_id=fresh_schema.schema_id, + partition_specs=[fresh_partition_spec], + default_spec_id=fresh_partition_spec.spec_id, + sort_orders=[fresh_sort_order], + default_sort_order_id=fresh_sort_order.order_id, + properties=properties, + last_partition_id=fresh_partition_spec.last_assigned_field_id, + ) + + +class TableMetadataUtil: """Helper class for parsing TableMetadata""" # Once this has been resolved, we can simplify this: https://github.com/samuelcolvin/pydantic/issues/3846 - # TableMetadata = Annotated[Union[TableMetadataV1, TableMetadataV2], Field(alias="format-version", discriminator="format-version")] + # TableMetadata = Annotated[TableMetadata, Field(alias="format-version", discriminator="format-version")] @staticmethod - def parse_obj(data: dict) -> Union[TableMetadataV1, TableMetadataV2]: + def parse_obj(data: dict) -> TableMetadata: if "format-version" not in data: raise ValidationError(f"Missing format-version in TableMetadata: {data}") diff --git a/pyiceberg/table/partitioning.py b/pyiceberg/table/partitioning.py index f5222ad595..30a9d63415 100644 --- a/pyiceberg/table/partitioning.py +++ b/pyiceberg/table/partitioning.py @@ -25,9 +25,11 @@ from pydantic import Field +from pyiceberg.schema import Schema from pyiceberg.transforms import Transform from pyiceberg.utils.iceberg_base_model import IcebergBaseModel +INITIAL_SPEC_ID = 0 _PARTITION_DATA_ID_START: int = 1000 @@ -157,3 +159,23 @@ def compatible_with(self, other: "PartitionSpec") -> bool: UNPARTITIONED_PARTITION_SPEC = PartitionSpec(spec_id=0) + + +def assign_fresh_partition_spec_ids(spec: PartitionSpec, old_schema: Schema, fresh_schema: Schema) -> PartitionSpec: + partition_fields = [] + for pos, field in enumerate(spec.fields): + original_column_name = old_schema.find_column_name(field.source_id) + if original_column_name is None: + raise ValueError(f"Could not find in old schema: {field}") + fresh_field = fresh_schema.find_field(original_column_name) + if fresh_field is None: + raise ValueError(f"Could not find field in fresh schema: {original_column_name}") + partition_fields.append( + PartitionField( + name=field.name, + source_id=fresh_field.field_id, + field_id=_PARTITION_DATA_ID_START + pos, + transform=field.transform, + ) + ) + return PartitionSpec(INITIAL_SPEC_ID, fields=tuple(partition_fields)) diff --git a/pyiceberg/table/sorting.py b/pyiceberg/table/sorting.py index 8199caff7b..013470fa57 100644 --- a/pyiceberg/table/sorting.py +++ b/pyiceberg/table/sorting.py @@ -27,6 +27,7 @@ from pydantic import Field, root_validator +from pyiceberg.schema import Schema from pyiceberg.transforms import IdentityTransform, Transform from pyiceberg.types import IcebergType from pyiceberg.utils.iceberg_base_model import IcebergBaseModel @@ -125,6 +126,10 @@ def __init__(self, order_id: Optional[int] = None, *fields: SortField, **data: A order_id: int = Field(alias="order-id") fields: List[SortField] = Field(default_factory=list) + @property + def is_unsorted(self) -> bool: + return len(self.fields) == 0 + def __str__(self) -> str: result_str = "[" if self.fields: @@ -135,3 +140,31 @@ def __str__(self) -> str: UNSORTED_SORT_ORDER_ID = 0 UNSORTED_SORT_ORDER = SortOrder(order_id=UNSORTED_SORT_ORDER_ID) +INITIAL_SORT_ORDER_ID = 1 + + +def assign_fresh_sort_order_ids(sort_order: SortOrder, old_schema: Schema, fresh_schema: Schema) -> SortOrder: + if sort_order.is_unsorted: + return UNSORTED_SORT_ORDER + + fresh_fields = [] + for field in sort_order.fields: + original_field = old_schema.find_column_name(field.source_id) + if original_field is None: + raise ValueError(f"Could not find in old schema: {field}") + fresh_field = fresh_schema.find_field(original_field) + if fresh_field is None: + raise ValueError(f"Could not find field in fresh schema: {original_field}") + fresh_fields.append( + SortField( + source_id=fresh_field.field_id, + transform=field.transform, + direction=field.direction, + null_order=field.null_order, + ) + ) + + return SortOrder( + INITIAL_SORT_ORDER_ID, + *fresh_fields, + ) diff --git a/pyiceberg/types.py b/pyiceberg/types.py index 94c7345726..2f4da429b7 100644 --- a/pyiceberg/types.py +++ b/pyiceberg/types.py @@ -168,6 +168,7 @@ def __init__(self, precision: int, scale: int): super().__init__( __root__=f"decimal({precision}, {scale})", ) + # assert precision < scale, "precision should be smaller than scale" self._precision = precision self._scale = scale diff --git a/pyiceberg/utils/iceberg_base_model.py b/pyiceberg/utils/iceberg_base_model.py index e218f9f89b..e558e4a444 100644 --- a/pyiceberg/utils/iceberg_base_model.py +++ b/pyiceberg/utils/iceberg_base_model.py @@ -15,6 +15,7 @@ # specific language governing permissions and limitations # under the License. from functools import cached_property +from typing import Optional, Set from pydantic import BaseModel @@ -39,14 +40,18 @@ class Config: allow_population_by_field_name = True frozen = True - def dict(self, exclude_none: bool = True, **kwargs): - return super().dict(exclude_none=exclude_none, **kwargs) - - def json(self, exclude_none: bool = True, by_alias: bool = True, **kwargs): + def _exclude_private_properties(self, exclude: Optional[Set[str]] = None) -> Set[str]: # A small trick to exclude private properties. Properties are serialized by pydantic, # regardless if they start with an underscore. # This will look at the dict, and find the fields and exclude them - exclude = set.union( - {field for field in self.__dict__ if field.startswith("_") and not field == "__root__"}, kwargs.get("exclude", set()) + return set.union( + {field for field in self.__dict__ if field.startswith("_") and not field == "__root__"}, exclude or set() + ) + + def dict(self, exclude_none: bool = True, exclude: Optional[Set[str]] = None, **kwargs): + return super().dict(exclude_none=exclude_none, exclude=self._exclude_private_properties(exclude), **kwargs) + + def json(self, exclude_none: bool = True, exclude: Optional[Set[str]] = None, by_alias: bool = True, **kwargs): + return super().json( + exclude_none=exclude_none, exclude=self._exclude_private_properties(exclude), by_alias=by_alias, **kwargs ) - return super().json(exclude_none=exclude_none, exclude=exclude, by_alias=by_alias, **kwargs) diff --git a/tests/catalog/test_hive.py b/tests/catalog/test_hive.py index 04d269a992..663fa33fe4 100644 --- a/tests/catalog/test_hive.py +++ b/tests/catalog/test_hive.py @@ -44,7 +44,7 @@ ) from pyiceberg.schema import Schema from pyiceberg.serializers import ToOutputFile -from pyiceberg.table.metadata import TableMetadata, TableMetadataV2 +from pyiceberg.table.metadata import TableMetadataUtil, TableMetadataV2 from pyiceberg.table.partitioning import PartitionField, PartitionSpec from pyiceberg.table.refs import SnapshotRef, SnapshotRefType from pyiceberg.table.snapshots import ( @@ -187,7 +187,6 @@ def test_check_number_of_namespaces(table_schema_simple: Schema): @patch("time.time", MagicMock(return_value=12345)) -@patch("uuid.uuid4", MagicMock(return_value="01234567-0123-0123-0123-0123456789ab")) def test_create_table(table_schema_simple: Schema, hive_database: HiveDatabase, hive_table: HiveTable): catalog = HiveCatalog(HIVE_CATALOG_NAME, uri=HIVE_METASTORE_FAKE_URL) @@ -264,28 +263,26 @@ def test_create_table(table_schema_simple: Schema, hive_database: HiveDatabase, with open(metadata_location, encoding="utf-8") as f: payload = json.load(f) - metadata = TableMetadata.parse_obj(payload) + metadata = TableMetadataUtil.parse_obj(payload) assert "database/table" in metadata.location - assert metadata assert metadata == TableMetadataV2( - # The following two ones are dynamic location=metadata.location, table_uuid=metadata.table_uuid, - last_updated_ms=12345000, + last_updated_ms=metadata.last_updated_ms, last_column_id=3, schemas=[ Schema( NestedField(field_id=1, name="foo", field_type=StringType(), required=False), NestedField(field_id=2, name="bar", field_type=IntegerType(), required=True), NestedField(field_id=3, name="baz", field_type=BooleanType(), required=False), - schema_id=1, + schema_id=0, identifier_field_ids=[2], ) ], - current_schema_id=1, - partition_specs=[PartitionSpec(spec_id=0, fields=())], + current_schema_id=0, + partition_specs=[PartitionSpec(spec_id=0)], default_spec_id=0, last_partition_id=1000, properties={"owner": "javaberg"}, diff --git a/tests/cli/test_console.py b/tests/cli/test_console.py index c0a2022e03..7c57e7b26e 100644 --- a/tests/cli/test_console.py +++ b/tests/cli/test_console.py @@ -29,7 +29,8 @@ from pyiceberg.cli.console import run from pyiceberg.exceptions import NoSuchNamespaceError, NoSuchTableError from pyiceberg.schema import Schema -from pyiceberg.table import Table, TableMetadataV2 +from pyiceberg.table import Table +from pyiceberg.table.metadata import TableMetadataV2 from pyiceberg.table.partitioning import UNPARTITIONED_PARTITION_SPEC, PartitionSpec from pyiceberg.table.sorting import UNSORTED_SORT_ORDER, SortOrder from pyiceberg.typedef import EMPTY_DICT, Identifier, Properties diff --git a/tests/table/test_init.py b/tests/table/test_init.py index b06421fcfb..567e1f07a5 100644 --- a/tests/table/test_init.py +++ b/tests/table/test_init.py @@ -20,7 +20,8 @@ import pytest from pyiceberg.schema import Schema -from pyiceberg.table import PartitionSpec, Table, TableMetadataV2 +from pyiceberg.table import PartitionSpec, Table +from pyiceberg.table.metadata import TableMetadataV2 from pyiceberg.table.partitioning import PartitionField from pyiceberg.table.snapshots import ( Operation, diff --git a/tests/table/test_metadata.py b/tests/table/test_metadata.py index c357b9d825..4cb6795a4a 100644 --- a/tests/table/test_metadata.py +++ b/tests/table/test_metadata.py @@ -18,6 +18,7 @@ import io import json from typing import Any, Dict +from unittest.mock import MagicMock, patch from uuid import UUID import pytest @@ -25,11 +26,28 @@ from pyiceberg.exceptions import ValidationError from pyiceberg.schema import Schema from pyiceberg.serializers import FromByteStream -from pyiceberg.table.metadata import TableMetadata, TableMetadataV1, TableMetadataV2 +from pyiceberg.table import SortOrder +from pyiceberg.table.metadata import ( + TableMetadataUtil, + TableMetadataV1, + TableMetadataV2, + new_table_metadata, +) from pyiceberg.table.partitioning import PartitionField, PartitionSpec from pyiceberg.table.refs import SnapshotRef, SnapshotRefType +from pyiceberg.table.sorting import NullOrder, SortDirection, SortField from pyiceberg.transforms import IdentityTransform -from pyiceberg.types import LongType, NestedField +from pyiceberg.types import ( + BooleanType, + FloatType, + IntegerType, + ListType, + LongType, + MapType, + NestedField, + StringType, + StructType, +) EXAMPLE_TABLE_METADATA_V1 = { "format-version": 1, @@ -54,12 +72,12 @@ def test_from_dict_v1(): """Test initialization of a TableMetadata instance from a dictionary""" - TableMetadata.parse_obj(EXAMPLE_TABLE_METADATA_V1) + TableMetadataUtil.parse_obj(EXAMPLE_TABLE_METADATA_V1) def test_from_dict_v2(example_table_metadata_v2: Dict[str, Any]): """Test initialization of a TableMetadata instance from a dictionary""" - TableMetadata.parse_obj(example_table_metadata_v2) + TableMetadataUtil.parse_obj(example_table_metadata_v2) def test_from_byte_stream(example_table_metadata_v2: Dict[str, Any]): @@ -71,7 +89,7 @@ def test_from_byte_stream(example_table_metadata_v2: Dict[str, Any]): def test_v2_metadata_parsing(example_table_metadata_v2: Dict[str, Any]): """Test retrieving values from a TableMetadata instance of version 2""" - table_metadata = TableMetadata.parse_obj(example_table_metadata_v2) + table_metadata = TableMetadataUtil.parse_obj(example_table_metadata_v2) assert table_metadata.format_version == 2 assert table_metadata.table_uuid == UUID("9c12d441-03fe-4693-9a96-a0705ddf69c1") @@ -138,10 +156,10 @@ def test_updating_metadata(example_table_metadata_v2: Dict[str, Any]): mutable_table_metadata["schemas"].append(new_schema) mutable_table_metadata["current-schema-id"] = 1 - new_table_metadata = TableMetadataV2(**mutable_table_metadata) + table_metadata = TableMetadataV2(**mutable_table_metadata) - assert new_table_metadata.current_schema_id == 1 - assert new_table_metadata.schemas[-1] == Schema(**new_schema) + assert table_metadata.current_schema_id == 1 + assert table_metadata.schemas[-1] == Schema(**new_schema) def test_serialize_v1(): @@ -202,7 +220,7 @@ def test_invalid_format_version(): } with pytest.raises(ValidationError) as exc_info: - TableMetadata.parse_obj(table_metadata_invalid_format_version) + TableMetadataUtil.parse_obj(table_metadata_invalid_format_version) assert "Unknown format version: -1" in str(exc_info.value) @@ -240,7 +258,7 @@ def test_current_schema_not_found(): } with pytest.raises(ValidationError) as exc_info: - TableMetadata.parse_obj(table_metadata_schema_not_found) + TableMetadataUtil.parse_obj(table_metadata_schema_not_found) assert "current-schema-id 2 can't be found in the schemas" in str(exc_info.value) @@ -286,7 +304,7 @@ def test_sort_order_not_found(): } with pytest.raises(ValidationError) as exc_info: - TableMetadata.parse_obj(table_metadata_schema_not_found) + TableMetadataUtil.parse_obj(table_metadata_schema_not_found) assert "default-sort-order-id 4 can't be found" in str(exc_info.value) @@ -323,7 +341,7 @@ def test_sort_order_unsorted(): "snapshots": [], } - table_metadata = TableMetadata.parse_obj(table_metadata_schema_not_found) + table_metadata = TableMetadataUtil.parse_obj(table_metadata_schema_not_found) # Most important here is that we correctly handle sort-order-id 0 assert len(table_metadata.sort_orders) == 0 @@ -358,7 +376,7 @@ def test_invalid_partition_spec(): "last-partition-id": 1000, } with pytest.raises(ValidationError) as exc_info: - TableMetadata.parse_obj(table_metadata_spec_not_found) + TableMetadataUtil.parse_obj(table_metadata_spec_not_found) assert "default-spec-id 1 can't be found" in str(exc_info.value) @@ -531,3 +549,167 @@ def test_metadata_v1(): "metadata-log": [], } TableMetadataV1(**valid_v1) + + +@patch("time.time", MagicMock(return_value=12345)) +def test_make_metadata_fresh(): + schema = Schema( + NestedField(field_id=10, name="foo", field_type=StringType(), required=False), + NestedField(field_id=22, name="bar", field_type=IntegerType(), required=True), + NestedField(field_id=33, name="baz", field_type=BooleanType(), required=False), + NestedField( + field_id=41, + name="qux", + field_type=ListType(element_id=56, element_type=StringType(), element_required=True), + required=True, + ), + NestedField( + field_id=6, + name="quux", + field_type=MapType( + key_id=77, + key_type=StringType(), + value_id=88, + value_type=MapType(key_id=91, key_type=StringType(), value_id=102, value_type=IntegerType(), value_required=True), + value_required=True, + ), + required=True, + ), + NestedField( + field_id=113, + name="location", + field_type=ListType( + element_id=124, + element_type=StructType( + NestedField(field_id=132, name="latitude", field_type=FloatType(), required=False), + NestedField(field_id=143, name="longitude", field_type=FloatType(), required=False), + ), + element_required=True, + ), + required=True, + ), + NestedField( + field_id=155, + name="person", + field_type=StructType( + NestedField(field_id=169, name="name", field_type=StringType(), required=False), + NestedField(field_id=178, name="age", field_type=IntegerType(), required=True), + ), + required=False, + ), + schema_id=10, + identifier_field_ids=[22], + ) + + partition_spec = PartitionSpec( + spec_id=10, fields=(PartitionField(source_id=22, field_id=1022, transform=IdentityTransform(), name="bar"),) + ) + + sort_order = SortOrder( + order_id=10, + fields=[ + SortField(source_id=10, transform=IdentityTransform(), direction=SortDirection.ASC, null_order=NullOrder.NULLS_LAST) + ], + ) + + actual = new_table_metadata( + schema=schema, partition_spec=partition_spec, sort_order=sort_order, location="s3://", properties={} + ) + + expected = TableMetadataV2( + location="s3://", + table_uuid=actual.table_uuid, + last_updated_ms=actual.last_updated_ms, + last_column_id=17, + schemas=[ + Schema( + NestedField(field_id=1, name="foo", field_type=StringType(), required=False), + NestedField(field_id=2, name="bar", field_type=IntegerType(), required=True), + NestedField(field_id=3, name="baz", field_type=BooleanType(), required=False), + NestedField( + field_id=4, + name="qux", + field_type=ListType(type="list", element_id=8, element_type=StringType(), element_required=True), + required=True, + ), + NestedField( + field_id=5, + name="quux", + field_type=MapType( + type="map", + key_id=9, + key_type=StringType(), + value_id=10, + value_type=MapType( + type="map", + key_id=11, + key_type=StringType(), + value_id=12, + value_type=IntegerType(), + value_required=True, + ), + value_required=True, + ), + required=True, + ), + NestedField( + field_id=6, + name="location", + field_type=ListType( + type="list", + element_id=13, + element_type=StructType( + fields=( + NestedField(field_id=14, name="latitude", field_type=FloatType(), required=False), + NestedField(field_id=15, name="longitude", field_type=FloatType(), required=False), + ) + ), + element_required=True, + ), + required=True, + ), + NestedField( + field_id=7, + name="person", + field_type=StructType( + fields=( + NestedField(field_id=16, name="name", field_type=StringType(), required=False), + NestedField(field_id=17, name="age", field_type=IntegerType(), required=True), + ) + ), + required=False, + ), + schema_id=0, + identifier_field_ids=[2], + ) + ], + current_schema_id=0, + partition_specs=[ + PartitionSpec( + spec_id=0, fields=(PartitionField(source_id=2, field_id=1000, transform=IdentityTransform(), name="bar"),) + ) + ], + default_spec_id=0, + last_partition_id=1000, + properties={}, + current_snapshot_id=None, + snapshots=[], + snapshot_log=[], + metadata_log=[], + sort_orders=[ + SortOrder( + order_id=1, + fields=[ + SortField( + source_id=1, transform=IdentityTransform(), direction=SortDirection.ASC, null_order=NullOrder.NULLS_LAST + ) + ], + ) + ], + default_sort_order_id=1, + refs={}, + format_version=2, + last_sequence_number=0, + ) + + assert actual.dict() == expected.dict() diff --git a/tests/table/test_sorting.py b/tests/table/test_sorting.py index e5012dead2..384a636e09 100644 --- a/tests/table/test_sorting.py +++ b/tests/table/test_sorting.py @@ -20,7 +20,7 @@ import pytest -from pyiceberg.table.metadata import TableMetadata +from pyiceberg.table.metadata import TableMetadataUtil from pyiceberg.table.sorting import ( UNSORTED_SORT_ORDER, NullOrder, @@ -57,7 +57,7 @@ def test_deserialize_sort_order(sort_order: SortOrder): def test_sorting_schema(example_table_metadata_v2: Dict[str, Any]): - table_metadata = TableMetadata.parse_obj(example_table_metadata_v2) + table_metadata = TableMetadataUtil.parse_obj(example_table_metadata_v2) assert table_metadata.sort_orders == [ SortOrder( From b2026c5d2ac06ada369d1955a25d6aa16864a56f Mon Sep 17 00:00:00 2001 From: Samuel Redai <43911210+samredai@users.noreply.github.com> Date: Thu, 1 Sep 2022 05:34:12 -0400 Subject: [PATCH 190/642] [Python] FsspecFileIO that wraps any fsspec filesystem (#5332) --- Makefile | 4 +- poetry.lock | 345 +++++++++++++++++++++++++++++++++++----- pyiceberg/io/fsspec.py | 196 +++++++++++++++++++++++ pyproject.toml | 17 ++ tests/conftest.py | 21 +++ tests/io/test_fsspec.py | 200 +++++++++++++++++++++++ 6 files changed, 742 insertions(+), 41 deletions(-) create mode 100644 pyiceberg/io/fsspec.py create mode 100644 tests/io/test_fsspec.py diff --git a/Makefile b/Makefile index aef783db33..0a67cc7817 100644 --- a/Makefile +++ b/Makefile @@ -17,13 +17,13 @@ install: pip install poetry - poetry install -E pyarrow -E hive + poetry install -E pyarrow -E hive -E s3fs lint: poetry run pre-commit run --all-files test: - poetry run coverage run --source=pyiceberg/ -m pytest tests/ ${PYTEST_ARGS} + poetry run coverage run --source=pyiceberg/ -m pytest tests/ -m "not s3" ${PYTEST_ARGS} poetry run coverage report -m --fail-under=90 poetry run coverage html poetry run coverage xml diff --git a/poetry.lock b/poetry.lock index 45f58cf672..78aac49821 100644 --- a/poetry.lock +++ b/poetry.lock @@ -1,3 +1,71 @@ +[[package]] +name = "aiobotocore" +version = "2.3.4" +description = "Async client for aws services using botocore and aiohttp" +category = "main" +optional = true +python-versions = ">=3.6" + +[package.dependencies] +aiohttp = ">=3.3.1" +aioitertools = ">=0.5.1" +botocore = ">=1.24.21,<1.24.22" +wrapt = ">=1.10.10" + +[package.extras] +awscli = ["awscli (>=1.22.76,<1.22.77)"] +boto3 = ["boto3 (>=1.21.21,<1.21.22)"] + +[[package]] +name = "aiohttp" +version = "3.8.1" +description = "Async http client/server framework (asyncio)" +category = "main" +optional = true +python-versions = ">=3.6" + +[package.dependencies] +aiosignal = ">=1.1.2" +async-timeout = ">=4.0.0a3,<5.0" +attrs = ">=17.3.0" +charset-normalizer = ">=2.0,<3.0" +frozenlist = ">=1.1.1" +multidict = ">=4.5,<7.0" +yarl = ">=1.0,<2.0" + +[package.extras] +speedups = ["aiodns", "brotli", "cchardet"] + +[[package]] +name = "aioitertools" +version = "0.10.0" +description = "itertools and builtins for AsyncIO and mixed iterables" +category = "main" +optional = true +python-versions = ">=3.6" + +[package.dependencies] +typing_extensions = {version = ">=4.0", markers = "python_version < \"3.10\""} + +[[package]] +name = "aiosignal" +version = "1.2.0" +description = "aiosignal: a list of registered asynchronous callbacks" +category = "main" +optional = true +python-versions = ">=3.6" + +[package.dependencies] +frozenlist = ">=1.1.0" + +[[package]] +name = "async-timeout" +version = "4.0.2" +description = "Timeout context manager for asyncio programs" +category = "main" +optional = true +python-versions = ">=3.6" + [[package]] name = "atomicwrites" version = "1.4.1" @@ -10,7 +78,7 @@ python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*" name = "attrs" version = "22.1.0" description = "Classes Without Boilerplate" -category = "dev" +category = "main" optional = false python-versions = ">=3.5" @@ -20,6 +88,22 @@ docs = ["furo", "sphinx", "sphinx-notfound-page", "zope.interface"] tests = ["cloudpickle", "coverage[toml] (>=5.0.2)", "hypothesis", "mypy (>=0.900,!=0.940)", "pympler", "pytest (>=4.3.0)", "pytest-mypy-plugins", "zope.interface"] tests_no_zope = ["cloudpickle", "coverage[toml] (>=5.0.2)", "hypothesis", "mypy (>=0.900,!=0.940)", "pympler", "pytest (>=4.3.0)", "pytest-mypy-plugins"] +[[package]] +name = "botocore" +version = "1.24.21" +description = "Low-level, data-driven core of boto 3." +category = "main" +optional = true +python-versions = ">= 3.6" + +[package.dependencies] +jmespath = ">=0.7.1,<2.0.0" +python-dateutil = ">=2.1,<3.0.0" +urllib3 = ">=1.25.4,<1.27" + +[package.extras] +crt = ["awscrt (==0.13.5)"] + [[package]] name = "certifi" version = "2022.6.15" @@ -144,6 +228,45 @@ python-versions = ">=3.7" docs = ["furo (>=2022.6.21)", "sphinx (>=5.1.1)", "sphinx-autodoc-typehints (>=1.19.1)"] testing = ["covdefaults (>=2.2)", "coverage (>=6.4.2)", "pytest (>=7.1.2)", "pytest-cov (>=3)", "pytest-timeout (>=2.1)"] +[[package]] +name = "frozenlist" +version = "1.3.1" +description = "A list-like structure which implements collections.abc.MutableSequence" +category = "main" +optional = true +python-versions = ">=3.7" + +[[package]] +name = "fsspec" +version = "2022.5.0" +description = "File-system specification" +category = "main" +optional = false +python-versions = ">=3.7" + +[package.extras] +abfs = ["adlfs"] +adl = ["adlfs"] +arrow = ["pyarrow (>=1)"] +dask = ["dask", "distributed"] +dropbox = ["dropbox", "dropboxdrivefs", "requests"] +entrypoints = ["importlib-metadata"] +fuse = ["fusepy"] +gcs = ["gcsfs"] +git = ["pygit2"] +github = ["requests"] +gs = ["gcsfs"] +gui = ["panel"] +hdfs = ["pyarrow (>=1)"] +http = ["aiohttp", "requests"] +libarchive = ["libarchive-c"] +oci = ["ocifs"] +s3 = ["s3fs"] +sftp = ["paramiko"] +smb = ["smbprotocol"] +ssh = ["paramiko"] +tqdm = ["tqdm"] + [[package]] name = "identify" version = "2.5.3" @@ -187,6 +310,14 @@ category = "dev" optional = false python-versions = "*" +[[package]] +name = "jmespath" +version = "1.0.1" +description = "JSON Matching Expressions" +category = "main" +optional = true +python-versions = ">=3.7" + [[package]] name = "mmh3" version = "3.0.0" @@ -195,6 +326,14 @@ category = "main" optional = false python-versions = "*" +[[package]] +name = "multidict" +version = "6.0.2" +description = "multidict implementation" +category = "main" +optional = true +python-versions = ">=3.7" + [[package]] name = "nodeenv" version = "1.7.0" @@ -302,7 +441,7 @@ python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*" [[package]] name = "pydantic" -version = "1.10.0" +version = "1.10.1" description = "Data validation and settings management using python type hints" category = "main" optional = false @@ -375,6 +514,17 @@ pep517 = "*" docs = ["jaraco.packaging (>=8.2)", "rst.linker (>=1.9)", "sphinx"] testing = ["pytest (>=4.6)", "pytest-black (>=0.3.7)", "pytest-checkdocs (>=2.4)", "pytest-cov", "pytest-enabler (>=1.0.1)", "pytest-flake8", "pytest-mypy", "types-docutils"] +[[package]] +name = "python-dateutil" +version = "2.8.2" +description = "Extensions to the standard Python datetime module" +category = "main" +optional = true +python-versions = "!=3.0.*,!=3.1.*,!=3.2.*,>=2.7" + +[package.dependencies] +six = ">=1.5" + [[package]] name = "python-snappy" version = "0.6.1" @@ -441,6 +591,23 @@ typing-extensions = {version = ">=4.0.0,<5.0", markers = "python_version < \"3.9 [package.extras] jupyter = ["ipywidgets (>=7.5.1,<8.0.0)"] +[[package]] +name = "s3fs" +version = "2022.5.0" +description = "Convenient Filesystem interface over S3" +category = "main" +optional = true +python-versions = ">= 3.7" + +[package.dependencies] +aiobotocore = ">=2.3.0,<2.4.0" +aiohttp = "<=4" +fsspec = "2022.5.0" + +[package.extras] +awscli = ["aiobotocore[awscli] (>=2.3.0,<2.4.0)"] +boto3 = ["aiobotocore[boto3] (>=2.3.0,<2.4.0)"] + [[package]] name = "six" version = "1.16.0" @@ -519,6 +686,26 @@ platformdirs = ">=2.4,<3" docs = ["proselint (>=0.13)", "sphinx (>=5.1.1)", "sphinx-argparse (>=0.3.1)", "sphinx-rtd-theme (>=1)", "towncrier (>=21.9)"] testing = ["coverage (>=6.2)", "coverage-enable-subprocess (>=1)", "flaky (>=3.7)", "packaging (>=21.3)", "pytest (>=7.0.1)", "pytest-env (>=0.6.2)", "pytest-freezegun (>=0.4.2)", "pytest-mock (>=3.6.1)", "pytest-randomly (>=3.10.3)", "pytest-timeout (>=2.1)"] +[[package]] +name = "wrapt" +version = "1.14.1" +description = "Module for decorators, wrappers and monkey patching." +category = "main" +optional = true +python-versions = "!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,!=3.4.*,>=2.7" + +[[package]] +name = "yarl" +version = "1.8.1" +description = "Yet another URL library" +category = "main" +optional = true +python-versions = ">=3.7" + +[package.dependencies] +idna = ">=2.0" +multidict = ">=4.0" + [[package]] name = "zipp" version = "3.8.1" @@ -549,16 +736,23 @@ cffi = ["cffi (>=1.11)"] hive = ["thrift"] pyarrow = ["pyarrow"] python-snappy = ["zstandard"] +s3fs = ["s3fs"] snappy = ["python-snappy"] [metadata] lock-version = "1.1" python-versions = "^3.8" -content-hash = "279aca560c214b2c2f4583a9c13d586b1e58c29d121f8b97798efe9e0612ab98" +content-hash = "7c6acb4d397c3920d6d6a47ce29593e9cb4799cf8e676ddb4d8dea2d2e9e43e5" [metadata.files] +aiobotocore = [] +aiohttp = [] +aioitertools = [] +aiosignal = [] +async-timeout = [] atomicwrites = [] attrs = [] +botocore = [] certifi = [ {file = "certifi-2022.6.15-py3-none-any.whl", hash = "sha256:fe86415d55e84719d75f8b69414f6438ac3547d2078ab91b67e779ef69378412"}, {file = "certifi-2022.6.15.tar.gz", hash = "sha256:84c85a9078b11105f04f3036a9482ae10e4621616db313fe045dd24743a0820d"}, @@ -651,6 +845,8 @@ docutils = [ ] fastavro = [] filelock = [] +frozenlist = [] +fsspec = [] identify = [] idna = [ {file = "idna-3.3-py3-none-any.whl", hash = "sha256:84d9dd047ffa80596e0f246e2eab0b391788b0503584e8945f2368256d2735ff"}, @@ -664,6 +860,7 @@ iniconfig = [ {file = "iniconfig-1.1.1-py2.py3-none-any.whl", hash = "sha256:011e24c64b7f47f6ebd835bb12a743f2fbe9a26d4cecaa7f53bc4f35ee9da8b3"}, {file = "iniconfig-1.1.1.tar.gz", hash = "sha256:bc3af051d7d14b2ee5ef9969666def0cd1a000e121eaea580d4a313df4b37f32"}, ] +jmespath = [] mmh3 = [ {file = "mmh3-3.0.0-cp36-cp36m-macosx_10_9_x86_64.whl", hash = "sha256:23912dde2ad4f701926948dd8e79a0e42b000f73962806f153931f52985e1e07"}, {file = "mmh3-3.0.0-cp36-cp36m-manylinux1_x86_64.whl", hash = "sha256:07f1308a410dc406d6a3c282a685728d00a87f3ed684f012671b96d6cc6a41c3"}, @@ -692,6 +889,7 @@ mmh3 = [ {file = "mmh3-3.0.0-cp39-cp39-win_amd64.whl", hash = "sha256:bd870aedd9189eff1cf4e1687869b56c7e9461ee869789139c3e704009e5c227"}, {file = "mmh3-3.0.0.tar.gz", hash = "sha256:d1ec578c09a07d3518ec9be540b87546397fa3455de73c166fcce51eaa5c41c5"}, ] +multidict = [] nodeenv = [ {file = "nodeenv-1.7.0-py2.py3-none-any.whl", hash = "sha256:27083a7b96a25f2f5e1d8cb4b6317ee8aeda3bdd121394e5ac54e498028a042e"}, {file = "nodeenv-1.7.0.tar.gz", hash = "sha256:e0e7f7dfb85fc5394c6fe1e8fa98131a2473e04311a45afb6508f7cf1836fa2b"}, @@ -721,42 +919,42 @@ pycparser = [ {file = "pycparser-2.21.tar.gz", hash = "sha256:e644fdec12f7872f86c58ff790da456218b10f863970249516d60a5eaca77206"}, ] pydantic = [ - {file = "pydantic-1.10.0-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:7e34e46dd08dafd4c75b8378efe3eae7d8e5212950fcd894d86c1df2dcfb80fe"}, - {file = "pydantic-1.10.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:4af55f33ae5be6cccecd4fa462630daffef1f161f60c3f194b24eca705d50748"}, - {file = "pydantic-1.10.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:1856bc6640aced42886f7ee48f5ed1fa5adf35e34064b5f9532b52d5a3b8a0d3"}, - {file = "pydantic-1.10.0-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:d73ae7e210929a1b7d288034835dd787e5b0597192d58ab7342bacbeec0f33df"}, - {file = "pydantic-1.10.0-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:a1192c17667d21652ab93b5eecd1a776cd0a4e384ea8c331bb830c9d130293af"}, - {file = "pydantic-1.10.0-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:026427be4e251f876e7519a63af37ae5ebb8b593ca8b02180bdc6becd1ea4ef4"}, - {file = "pydantic-1.10.0-cp310-cp310-win_amd64.whl", hash = "sha256:d1dffae1f219d06a997ec78d1d2daafdbfecf243ad8eb36bfbcbc73e30e17385"}, - {file = "pydantic-1.10.0-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:b549eebe8de4e50fc3b4f8c1f9cc2f731d91787fc3f7d031561668377b8679bc"}, - {file = "pydantic-1.10.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:a0ba8710bfdaddb7424c05ad2dc1da04796003751eac6ad30c218ac1d68a174e"}, - {file = "pydantic-1.10.0-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:f0985ba95af937389c9ce8d747138417303569cb736bd12469646ef53cd66e1c"}, - {file = "pydantic-1.10.0-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:d484fbbe6267b6c936a6d005d5170ab553f3f4367348c7e88d3e17f0a7179981"}, - {file = "pydantic-1.10.0-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:9500586151cd56a20bacb8f1082df1b4489000120d1c7ddc44c8b20870e8adbd"}, - {file = "pydantic-1.10.0-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:1b5212604aaf5954e9a7cea8f0c60d6dbef996aa7b41edefd329e6b5011ce8cf"}, - {file = "pydantic-1.10.0-cp311-cp311-win_amd64.whl", hash = "sha256:39212b3853eea165a3cda11075d5b7d09d4291fcbc3c0ecefd23797ee21b29e9"}, - {file = "pydantic-1.10.0-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:b3e3aed33fbd9518cf508d5415a58af683743d53dc5e58953973d73605774f34"}, - {file = "pydantic-1.10.0-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:ed4e5c18cac70fadd4cf339f444c4f1795f0876dfd5b70cf0a841890b52f0001"}, - {file = "pydantic-1.10.0-cp37-cp37m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:45a6d0a9fdaad2a27ea69aec4659705ed8f60a5664e892c73e2b977d8f5166cc"}, - {file = "pydantic-1.10.0-cp37-cp37m-musllinux_1_1_i686.whl", hash = "sha256:158f1479367da20914961b5406ac3b29dfe1d858ae2af96c444f73543defcf0c"}, - {file = "pydantic-1.10.0-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:172aaeeaff8fc3ac326fb8a2934a063ca0938586c5fe8848285052de83a240f7"}, - {file = "pydantic-1.10.0-cp37-cp37m-win_amd64.whl", hash = "sha256:231b19c010288bfbfdcd3f79df38b5ff893c6547cd8c7d006203435790b22815"}, - {file = "pydantic-1.10.0-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:22206c152f9b86c0ee169928f9c24e1c0c566edb2462600b298ccb04860961aa"}, - {file = "pydantic-1.10.0-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:d8ef840ef803ef17a7bd52480eb85faca0eed728d70233fd560f7d1066330247"}, - {file = "pydantic-1.10.0-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:1f99b4de6936a0f9fe255d1c7fdc447700ddd027c9ad38a612d453ed5fc7d6d0"}, - {file = "pydantic-1.10.0-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:134b4fd805737496ce4efd24ce2f8da0e08c66dcfc054fee1a19673eec780f2c"}, - {file = "pydantic-1.10.0-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:c4c76af6ad47bc46cf16bd0e4a5e536a7a2bec0dec14ea08b712daa6645bf293"}, - {file = "pydantic-1.10.0-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:e03402b0a6b23a2d0b9ee31e45d80612c95562b5af8b5c900171b9d9015ddc5f"}, - {file = "pydantic-1.10.0-cp38-cp38-win_amd64.whl", hash = "sha256:3a3a60fcb5ce08cab593b7978d02db67b8d153e9d582adab7c0b69d7200d78be"}, - {file = "pydantic-1.10.0-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:d8e5c5a50821c55b76dcf422610225cb7e44685cdd81832d0d504fa8c9343f35"}, - {file = "pydantic-1.10.0-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:645b83297a9428a675c98c1f69a7237a381900e34f23245c0ea73d74e454bf68"}, - {file = "pydantic-1.10.0-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:95ab3f31f35dc4f8fc85b04d13569e5fdc9de2d3050ae64c1fdc3430dfe7d92d"}, - {file = "pydantic-1.10.0-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:e290915a0ed53d3c59d6071fc7d2c843ed04c33affcd752dd1f3daa859b44a76"}, - {file = "pydantic-1.10.0-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:af669da39ede365069dbc5de56564b011e3353f801acdbdd7145002a78abc3d9"}, - {file = "pydantic-1.10.0-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:8e796f915762dec4678fafc89b1f0441ab9209517a8a682ddb3f988f7ffe0827"}, - {file = "pydantic-1.10.0-cp39-cp39-win_amd64.whl", hash = "sha256:652727f9e1d3ae30bd8a4dfbebcafd50df45277b97f3deabbbfedcf731f94aa5"}, - {file = "pydantic-1.10.0-py3-none-any.whl", hash = "sha256:4d2b9258f5bd2d129bd4cf2d31f9d40094b9ed6ef64896e2f7a70729b2d599ea"}, - {file = "pydantic-1.10.0.tar.gz", hash = "sha256:e13788fcad1baf5eb3236856b2a9a74f7dac6b3ea7ca1f60a4ad8bad4239cf4c"}, + {file = "pydantic-1.10.1-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:221166d99726238f71adc4fa9f3e94063a10787574b966f86a774559e709ac5a"}, + {file = "pydantic-1.10.1-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:a90e85d95fd968cd7cae122e0d3e0e1f6613bc88c1ff3fe838ac9785ea4b1c4c"}, + {file = "pydantic-1.10.1-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:f2157aaf5718c648eaec9e654a34179ae42ffc363dc3ad058538a4f3ecbd9341"}, + {file = "pydantic-1.10.1-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:6142246fc9adb51cadaeb84fb52a86f3adad4c6a7b0938a5dd0b1356b0088217"}, + {file = "pydantic-1.10.1-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:60dad97a09b6f44690c05467a4f397b62bfc2c839ac39102819d6979abc2be0d"}, + {file = "pydantic-1.10.1-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:d6f5bcb59d33ec46621dae76e714c53035087666cac80c81c9047a84f3ff93d0"}, + {file = "pydantic-1.10.1-cp310-cp310-win_amd64.whl", hash = "sha256:522906820cd60e63c7960ba83078bf2d2ad2dd0870bf68248039bcb1ec3eb0a4"}, + {file = "pydantic-1.10.1-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:d545c89d88bdd5559db17aeb5a61a26799903e4bd76114779b3bf1456690f6ce"}, + {file = "pydantic-1.10.1-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:ad2374b5b3b771dcc6e2f6e0d56632ab63b90e9808b7a73ad865397fcdb4b2cd"}, + {file = "pydantic-1.10.1-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:90e02f61b7354ed330f294a437d0bffac9e21a5d46cb4cc3c89d220e497db7ac"}, + {file = "pydantic-1.10.1-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:cc5ffe7bd0b4778fa5b7a5f825c52d6cfea3ae2d9b52b05b9b1d97e36dee23a8"}, + {file = "pydantic-1.10.1-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:7acb7b66ffd2bc046eaff0063df84c83fc3826722d5272adaeadf6252e17f691"}, + {file = "pydantic-1.10.1-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:7e6786ed5faa559dea5a77f6d2de9a08d18130de9344533535d945f34bdcd42e"}, + {file = "pydantic-1.10.1-cp311-cp311-win_amd64.whl", hash = "sha256:c7bf8ff1d18186eb0cbe42bd9bfb4cbf7fde1fd01b8608925458990c21f202f0"}, + {file = "pydantic-1.10.1-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:14a5babda137a294df7ad5f220986d79bbb87fdeb332c6ded61ce19da7f5f3bf"}, + {file = "pydantic-1.10.1-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:5659cb9c6b3d27fc0067025c4f5a205f5e838232a4a929b412781117c2343d44"}, + {file = "pydantic-1.10.1-cp37-cp37m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:c8d70fb91b03c32d2e857b071a22a5225e6b625ca82bd2cc8dd729d88e0bd200"}, + {file = "pydantic-1.10.1-cp37-cp37m-musllinux_1_1_i686.whl", hash = "sha256:9a93be313e40f12c6f2cb84533b226bbe23d0774872e38d83415e6890215e3a6"}, + {file = "pydantic-1.10.1-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:d55aeb01bb7bd7c7e1bd904668a4a2ffcbb1c248e7ae9eb40a272fd7e67dd98b"}, + {file = "pydantic-1.10.1-cp37-cp37m-win_amd64.whl", hash = "sha256:43d41b6f13706488e854729955ba8f740e6ec375cd16b72b81dc24b9d84f0d15"}, + {file = "pydantic-1.10.1-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:f31ffe0e38805a0e6410330f78147bb89193b136d7a5f79cae60d3e849b520a6"}, + {file = "pydantic-1.10.1-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:8eee69eda7674977b079a21e7bf825b59d8bf15145300e8034ed3eb239ac444f"}, + {file = "pydantic-1.10.1-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:6f927bff6c319fc92e0a2cbeb2609b5c1cd562862f4b54ec905e353282b7c8b1"}, + {file = "pydantic-1.10.1-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:eb1bc3f8fef6ba36977108505e90558911e7fbccb4e930805d5dd90891b56ff4"}, + {file = "pydantic-1.10.1-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:96ab6ce1346d14c6e581a69c333bdd1b492df9cf85ad31ad77a8aa42180b7e09"}, + {file = "pydantic-1.10.1-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:444cf220a12134da1cd42fe4f45edff622139e10177ce3d8ef2b4f41db1291b2"}, + {file = "pydantic-1.10.1-cp38-cp38-win_amd64.whl", hash = "sha256:dbfbff83565b4514dd8cebc8b8c81a12247e89427ff997ad0a9da7b2b1065c12"}, + {file = "pydantic-1.10.1-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:5327406f4bfd5aee784e7ad2a6a5fdd7171c19905bf34cb1994a1ba73a87c468"}, + {file = "pydantic-1.10.1-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:1072eae28bf034a311764c130784e8065201a90edbca10f495c906737b3bd642"}, + {file = "pydantic-1.10.1-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:ce901335667a68dfbc10dd2ee6c0d676b89210d754441c2469fbc37baf7ee2ed"}, + {file = "pydantic-1.10.1-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:54d6465cd2112441305faf5143a491b40de07a203116b5755a2108e36b25308d"}, + {file = "pydantic-1.10.1-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:2b5e5e7a0ec96704099e271911a1049321ba1afda92920df0769898a7e9a1298"}, + {file = "pydantic-1.10.1-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:ae43704358304da45c1c3dd7056f173c618b252f91594bcb6d6f6b4c6c284dee"}, + {file = "pydantic-1.10.1-cp39-cp39-win_amd64.whl", hash = "sha256:2d7da49229ffb1049779a5a6c1c50a26da164bd053cf8ee9042197dc08a98259"}, + {file = "pydantic-1.10.1-py3-none-any.whl", hash = "sha256:f8b10e59c035ff3dcc9791619d6e6c5141e0fa5cbe264e19e267b8d523b210bf"}, + {file = "pydantic-1.10.1.tar.gz", hash = "sha256:d41bb80347a8a2d51fbd6f1748b42aca14541315878447ba159617544712f770"}, ] pygments = [] pyparsing = [ @@ -771,6 +969,7 @@ pytest-checkdocs = [ {file = "pytest-checkdocs-2.7.1.tar.gz", hash = "sha256:2b33b85eddfe5846a69bea4a759303e2d5a3be11d03bc7149f5ba1ef47e6c1ae"}, {file = "pytest_checkdocs-2.7.1-py3-none-any.whl", hash = "sha256:294898c64c9ce1a178edc6660e48da23c7543bfd5a1cea7f0ca4c167745d8461"}, ] +python-dateutil = [] python-snappy = [ {file = "python-snappy-0.6.1.tar.gz", hash = "sha256:b6a107ab06206acc5359d4c5632bd9b22d448702a79b3169b0c62e0fb808bb2a"}, {file = "python_snappy-0.6.1-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:b7f920eaf46ebf41bd26f9df51c160d40f9e00b7b48471c3438cb8d027f7fb9b"}, @@ -865,6 +1064,7 @@ requests-mock = [ {file = "requests_mock-1.10.0-py2.py3-none-any.whl", hash = "sha256:2fdbb637ad17ee15c06f33d31169e71bf9fe2bdb7bc9da26185be0dd8d842699"}, ] rich = [] +s3fs = [] six = [ {file = "six-1.16.0-py2.py3-none-any.whl", hash = "sha256:8abb2f1d86890a2dfb989f9a77cfcfd3e47c2a354b01111771326f8aa26e0254"}, {file = "six-1.16.0.tar.gz", hash = "sha256:1e61c37477a1626458e36f7b1d82aa5c9b094fa4802892072e49de9c60c4c926"}, @@ -887,5 +1087,72 @@ virtualenv = [ {file = "virtualenv-20.16.4-py3-none-any.whl", hash = "sha256:035ed57acce4ac35c82c9d8802202b0e71adac011a511ff650cbcf9635006a22"}, {file = "virtualenv-20.16.4.tar.gz", hash = "sha256:014f766e4134d0008dcaa1f95bafa0fb0f575795d07cae50b1bee514185d6782"}, ] +wrapt = [ + {file = "wrapt-1.14.1-cp27-cp27m-macosx_10_9_x86_64.whl", hash = "sha256:1b376b3f4896e7930f1f772ac4b064ac12598d1c38d04907e696cc4d794b43d3"}, + {file = "wrapt-1.14.1-cp27-cp27m-manylinux1_i686.whl", hash = "sha256:903500616422a40a98a5a3c4ff4ed9d0066f3b4c951fa286018ecdf0750194ef"}, + {file = "wrapt-1.14.1-cp27-cp27m-manylinux1_x86_64.whl", hash = "sha256:5a9a0d155deafd9448baff28c08e150d9b24ff010e899311ddd63c45c2445e28"}, + {file = "wrapt-1.14.1-cp27-cp27m-manylinux2010_i686.whl", hash = "sha256:ddaea91abf8b0d13443f6dac52e89051a5063c7d014710dcb4d4abb2ff811a59"}, + {file = "wrapt-1.14.1-cp27-cp27m-manylinux2010_x86_64.whl", hash = "sha256:36f582d0c6bc99d5f39cd3ac2a9062e57f3cf606ade29a0a0d6b323462f4dd87"}, + {file = "wrapt-1.14.1-cp27-cp27mu-manylinux1_i686.whl", hash = "sha256:7ef58fb89674095bfc57c4069e95d7a31cfdc0939e2a579882ac7d55aadfd2a1"}, + {file = "wrapt-1.14.1-cp27-cp27mu-manylinux1_x86_64.whl", hash = "sha256:e2f83e18fe2f4c9e7db597e988f72712c0c3676d337d8b101f6758107c42425b"}, + {file = "wrapt-1.14.1-cp27-cp27mu-manylinux2010_i686.whl", hash = "sha256:ee2b1b1769f6707a8a445162ea16dddf74285c3964f605877a20e38545c3c462"}, + {file = "wrapt-1.14.1-cp27-cp27mu-manylinux2010_x86_64.whl", hash = "sha256:833b58d5d0b7e5b9832869f039203389ac7cbf01765639c7309fd50ef619e0b1"}, + {file = "wrapt-1.14.1-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:80bb5c256f1415f747011dc3604b59bc1f91c6e7150bd7db03b19170ee06b320"}, + {file = "wrapt-1.14.1-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:07f7a7d0f388028b2df1d916e94bbb40624c59b48ecc6cbc232546706fac74c2"}, + {file = "wrapt-1.14.1-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:02b41b633c6261feff8ddd8d11c711df6842aba629fdd3da10249a53211a72c4"}, + {file = "wrapt-1.14.1-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:2fe803deacd09a233e4762a1adcea5db5d31e6be577a43352936179d14d90069"}, + {file = "wrapt-1.14.1-cp310-cp310-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:257fd78c513e0fb5cdbe058c27a0624c9884e735bbd131935fd49e9fe719d310"}, + {file = "wrapt-1.14.1-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:4fcc4649dc762cddacd193e6b55bc02edca674067f5f98166d7713b193932b7f"}, + {file = "wrapt-1.14.1-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:11871514607b15cfeb87c547a49bca19fde402f32e2b1c24a632506c0a756656"}, + {file = "wrapt-1.14.1-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:8ad85f7f4e20964db4daadcab70b47ab05c7c1cf2a7c1e51087bfaa83831854c"}, + {file = "wrapt-1.14.1-cp310-cp310-win32.whl", hash = "sha256:a9a52172be0b5aae932bef82a79ec0a0ce87288c7d132946d645eba03f0ad8a8"}, + {file = "wrapt-1.14.1-cp310-cp310-win_amd64.whl", hash = "sha256:6d323e1554b3d22cfc03cd3243b5bb815a51f5249fdcbb86fda4bf62bab9e164"}, + {file = "wrapt-1.14.1-cp35-cp35m-manylinux1_i686.whl", hash = "sha256:43ca3bbbe97af00f49efb06e352eae40434ca9d915906f77def219b88e85d907"}, + {file = "wrapt-1.14.1-cp35-cp35m-manylinux1_x86_64.whl", hash = "sha256:6b1a564e6cb69922c7fe3a678b9f9a3c54e72b469875aa8018f18b4d1dd1adf3"}, + {file = "wrapt-1.14.1-cp35-cp35m-manylinux2010_i686.whl", hash = "sha256:00b6d4ea20a906c0ca56d84f93065b398ab74b927a7a3dbd470f6fc503f95dc3"}, + {file = "wrapt-1.14.1-cp35-cp35m-manylinux2010_x86_64.whl", hash = "sha256:a85d2b46be66a71bedde836d9e41859879cc54a2a04fad1191eb50c2066f6e9d"}, + {file = "wrapt-1.14.1-cp35-cp35m-win32.whl", hash = "sha256:dbcda74c67263139358f4d188ae5faae95c30929281bc6866d00573783c422b7"}, + {file = "wrapt-1.14.1-cp35-cp35m-win_amd64.whl", hash = "sha256:b21bb4c09ffabfa0e85e3a6b623e19b80e7acd709b9f91452b8297ace2a8ab00"}, + {file = "wrapt-1.14.1-cp36-cp36m-macosx_10_9_x86_64.whl", hash = "sha256:9e0fd32e0148dd5dea6af5fee42beb949098564cc23211a88d799e434255a1f4"}, + {file = "wrapt-1.14.1-cp36-cp36m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:9736af4641846491aedb3c3f56b9bc5568d92b0692303b5a305301a95dfd38b1"}, + {file = "wrapt-1.14.1-cp36-cp36m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:5b02d65b9ccf0ef6c34cba6cf5bf2aab1bb2f49c6090bafeecc9cd81ad4ea1c1"}, + {file = "wrapt-1.14.1-cp36-cp36m-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:21ac0156c4b089b330b7666db40feee30a5d52634cc4560e1905d6529a3897ff"}, + {file = "wrapt-1.14.1-cp36-cp36m-musllinux_1_1_aarch64.whl", hash = "sha256:9f3e6f9e05148ff90002b884fbc2a86bd303ae847e472f44ecc06c2cd2fcdb2d"}, + {file = "wrapt-1.14.1-cp36-cp36m-musllinux_1_1_i686.whl", hash = "sha256:6e743de5e9c3d1b7185870f480587b75b1cb604832e380d64f9504a0535912d1"}, + {file = "wrapt-1.14.1-cp36-cp36m-musllinux_1_1_x86_64.whl", hash = "sha256:d79d7d5dc8a32b7093e81e97dad755127ff77bcc899e845f41bf71747af0c569"}, + {file = "wrapt-1.14.1-cp36-cp36m-win32.whl", hash = "sha256:81b19725065dcb43df02b37e03278c011a09e49757287dca60c5aecdd5a0b8ed"}, + {file = "wrapt-1.14.1-cp36-cp36m-win_amd64.whl", hash = "sha256:b014c23646a467558be7da3d6b9fa409b2c567d2110599b7cf9a0c5992b3b471"}, + {file = "wrapt-1.14.1-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:88bd7b6bd70a5b6803c1abf6bca012f7ed963e58c68d76ee20b9d751c74a3248"}, + {file = "wrapt-1.14.1-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:b5901a312f4d14c59918c221323068fad0540e34324925c8475263841dbdfe68"}, + {file = "wrapt-1.14.1-cp37-cp37m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:d77c85fedff92cf788face9bfa3ebaa364448ebb1d765302e9af11bf449ca36d"}, + {file = "wrapt-1.14.1-cp37-cp37m-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:8d649d616e5c6a678b26d15ece345354f7c2286acd6db868e65fcc5ff7c24a77"}, + {file = "wrapt-1.14.1-cp37-cp37m-musllinux_1_1_aarch64.whl", hash = "sha256:7d2872609603cb35ca513d7404a94d6d608fc13211563571117046c9d2bcc3d7"}, + {file = "wrapt-1.14.1-cp37-cp37m-musllinux_1_1_i686.whl", hash = "sha256:ee6acae74a2b91865910eef5e7de37dc6895ad96fa23603d1d27ea69df545015"}, + {file = "wrapt-1.14.1-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:2b39d38039a1fdad98c87279b48bc5dce2c0ca0d73483b12cb72aa9609278e8a"}, + {file = "wrapt-1.14.1-cp37-cp37m-win32.whl", hash = "sha256:60db23fa423575eeb65ea430cee741acb7c26a1365d103f7b0f6ec412b893853"}, + {file = "wrapt-1.14.1-cp37-cp37m-win_amd64.whl", hash = "sha256:709fe01086a55cf79d20f741f39325018f4df051ef39fe921b1ebe780a66184c"}, + {file = "wrapt-1.14.1-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:8c0ce1e99116d5ab21355d8ebe53d9460366704ea38ae4d9f6933188f327b456"}, + {file = "wrapt-1.14.1-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:e3fb1677c720409d5f671e39bac6c9e0e422584e5f518bfd50aa4cbbea02433f"}, + {file = "wrapt-1.14.1-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:642c2e7a804fcf18c222e1060df25fc210b9c58db7c91416fb055897fc27e8cc"}, + {file = "wrapt-1.14.1-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:7b7c050ae976e286906dd3f26009e117eb000fb2cf3533398c5ad9ccc86867b1"}, + {file = "wrapt-1.14.1-cp38-cp38-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:ef3f72c9666bba2bab70d2a8b79f2c6d2c1a42a7f7e2b0ec83bb2f9e383950af"}, + {file = "wrapt-1.14.1-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:01c205616a89d09827986bc4e859bcabd64f5a0662a7fe95e0d359424e0e071b"}, + {file = "wrapt-1.14.1-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:5a0f54ce2c092aaf439813735584b9537cad479575a09892b8352fea5e988dc0"}, + {file = "wrapt-1.14.1-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:2cf71233a0ed05ccdabe209c606fe0bac7379fdcf687f39b944420d2a09fdb57"}, + {file = "wrapt-1.14.1-cp38-cp38-win32.whl", hash = "sha256:aa31fdcc33fef9eb2552cbcbfee7773d5a6792c137b359e82879c101e98584c5"}, + {file = "wrapt-1.14.1-cp38-cp38-win_amd64.whl", hash = "sha256:d1967f46ea8f2db647c786e78d8cc7e4313dbd1b0aca360592d8027b8508e24d"}, + {file = "wrapt-1.14.1-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:3232822c7d98d23895ccc443bbdf57c7412c5a65996c30442ebe6ed3df335383"}, + {file = "wrapt-1.14.1-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:988635d122aaf2bdcef9e795435662bcd65b02f4f4c1ae37fbee7401c440b3a7"}, + {file = "wrapt-1.14.1-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:9cca3c2cdadb362116235fdbd411735de4328c61425b0aa9f872fd76d02c4e86"}, + {file = "wrapt-1.14.1-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:d52a25136894c63de15a35bc0bdc5adb4b0e173b9c0d07a2be9d3ca64a332735"}, + {file = "wrapt-1.14.1-cp39-cp39-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:40e7bc81c9e2b2734ea4bc1aceb8a8f0ceaac7c5299bc5d69e37c44d9081d43b"}, + {file = "wrapt-1.14.1-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:b9b7a708dd92306328117d8c4b62e2194d00c365f18eff11a9b53c6f923b01e3"}, + {file = "wrapt-1.14.1-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:6a9a25751acb379b466ff6be78a315e2b439d4c94c1e99cb7266d40a537995d3"}, + {file = "wrapt-1.14.1-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:34aa51c45f28ba7f12accd624225e2b1e5a3a45206aa191f6f9aac931d9d56fe"}, + {file = "wrapt-1.14.1-cp39-cp39-win32.whl", hash = "sha256:dee0ce50c6a2dd9056c20db781e9c1cfd33e77d2d569f5d1d9321c641bb903d5"}, + {file = "wrapt-1.14.1-cp39-cp39-win_amd64.whl", hash = "sha256:dee60e1de1898bde3b238f18340eec6148986da0455d8ba7848d50470a7a32fb"}, + {file = "wrapt-1.14.1.tar.gz", hash = "sha256:380a85cf89e0e69b7cfbe2ea9f765f004ff419f34194018a6827ac0e3edfed4d"}, +] +yarl = [] zipp = [] zstandard = [] diff --git a/pyiceberg/io/fsspec.py b/pyiceberg/io/fsspec.py new file mode 100644 index 0000000000..769c309a84 --- /dev/null +++ b/pyiceberg/io/fsspec.py @@ -0,0 +1,196 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +"""FileIO implementation for reading and writing table files that uses fsspec compatible filesystems""" + +from functools import lru_cache +from typing import Callable, Union +from urllib.parse import urlparse + +from fsspec import AbstractFileSystem + +from pyiceberg.io import FileIO, InputFile, OutputFile +from pyiceberg.typedef import Properties + + +def _s3(properties: Properties, **fs_properties) -> AbstractFileSystem: + from s3fs import S3FileSystem + + client_kwargs = { + "endpoint_url": properties.get("s3.endpoint"), + "aws_access_key_id": properties.get("s3.access-key-id"), + "aws_secret_access_key": properties.get("s3.secret-access-key"), + } + + config_kwargs = {"signature_version": properties.get("s3.signer")} + + return S3FileSystem(client_kwargs=client_kwargs, config_kwargs=config_kwargs, **fs_properties) + + +SCHEME_TO_FS = { + "s3": _s3, + "s3a": _s3, + "s3n": _s3, +} + + +class FsspecInputFile(InputFile): + """An input file implementation for the FsspecFileIO + + Args: + location(str): A URI to a file location + fs(AbstractFileSystem): An fsspec filesystem instance + """ + + def __init__(self, location: str, fs: AbstractFileSystem): + self._fs = fs + super().__init__(location=location) + + def __len__(self) -> int: + """Returns the total length of the file, in bytes""" + object_info = self._fs.info(self.location) + if size := object_info.get("Size"): + return size + elif size := object_info.get("size"): + return size + raise RuntimeError(f"Cannot retrieve object info: {self.location}") + + def exists(self) -> bool: + """Checks whether the location exists""" + return self._fs.lexists(self.location) + + def open(self): + """Create an input stream for reading the contents of the file + + Returns: + OpenFile: An fsspec compliant file-like object + """ + return self._fs.open(self.location, "rb") + + +class FsspecOutputFile(OutputFile): + """An output file implementation for the FsspecFileIO + + Args: + location(str): A URI to a file location + fs(AbstractFileSystem): An fsspec filesystem instance + """ + + def __init__(self, location: str, fs: AbstractFileSystem): + self._fs = fs + super().__init__(location=location) + + def __len__(self) -> int: + """Returns the total length of the file, in bytes""" + object_info = self._fs.info(self.location) + if size := object_info.get("Size"): + return size + elif size := object_info.get("size"): + return size + raise RuntimeError(f"Cannot retrieve object info: {self.location}") + + def exists(self) -> bool: + """Checks whether the location exists""" + return self._fs.lexists(self.location) + + def create(self, overwrite: bool = False): + """Create an output stream for reading the contents of the file + + Args: + overwrite(bool): Whether to overwrite the file if it already exists + + Returns: + OpenFile: An fsspec compliant file-like object + + Raises: + FileExistsError: If the file already exists at the location and overwrite is set to False + + Note: + If overwrite is set to False, a check is first performed to verify that the file does not exist. + This is not thread-safe and a possibility does exist that the file can be created by a concurrent + process after the existence check yet before the output stream is created. In such a case, the default + behavior will truncate the contents of the existing file when opening the output stream. + """ + if not overwrite and self.exists(): + raise FileExistsError(f"Cannot create file, file already exists: {self.location}") + return self._fs.open(self.location, "wb") + + def to_input_file(self) -> FsspecInputFile: + """Returns a new FsspecInputFile for the location at `self.location`""" + return FsspecInputFile(location=self.location, fs=self._fs) + + +class FsspecFileIO(FileIO): + """A FileIO implementation that uses fsspec""" + + def __init__(self, properties: Properties): + self._scheme_to_fs = {} + self._scheme_to_fs.update(SCHEME_TO_FS) + self.get_fs: Callable = lru_cache(self._get_fs) + super().__init__(properties=properties) + + def new_input(self, location: str) -> FsspecInputFile: + """Get an FsspecInputFile instance to read bytes from the file at the given location + + Args: + location(str): A URI or a path to a local file + + Returns: + FsspecInputFile: An FsspecInputFile instance for the given location + """ + uri = urlparse(location) + fs = self.get_fs(uri.scheme) + return FsspecInputFile(location=location, fs=fs) + + def new_output(self, location: str) -> FsspecOutputFile: + """Get an FsspecOutputFile instance to write bytes to the file at the given location + + Args: + location(str): A URI or a path to a local file + + Returns: + FsspecOutputFile: An FsspecOutputFile instance for the given location + """ + uri = urlparse(location) + fs = self.get_fs(uri.scheme) + return FsspecOutputFile(location=location, fs=fs) + + def delete(self, location: Union[str, InputFile, OutputFile]) -> None: + """Delete the file at the given location + + Args: + location(str, InputFile, OutputFile): The URI to the file--if an InputFile instance or an + OutputFile instance is provided, the location attribute for that instance is used as the location + to delete + """ + if isinstance(location, (InputFile, OutputFile)): + str_location = location.location # Use InputFile or OutputFile location + else: + str_location = location + + uri = urlparse(str_location) + fs = self.get_fs(uri.scheme) + fs.rm(str_location) + + def _get_fs(self, scheme: str) -> AbstractFileSystem: + """Get a filesystem for a specific scheme""" + if scheme not in self._scheme_to_fs: + raise ValueError(f"No registered filesystem for scheme: {scheme}") + return self._scheme_to_fs[scheme](self.properties, **self._fs_properties()) + + def _fs_properties(self): + """Get fs properties from the file-io property map""" + return {k[3:]: v for k, v in self.properties.items() if k.startswith("fs_")} diff --git a/pyproject.toml b/pyproject.toml index b55a6ce5c0..3d6d9d25df 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -48,6 +48,7 @@ click = "^8.1.3" rich = "^12.5.1" pydantic = "^1.9.2" +fsspec = "2022.5.0" pyarrow = { version = "^9.0.0", optional = true } @@ -57,6 +58,8 @@ python-snappy = { version = "^0.6.1", optional = true } thrift = { version = "^0.16.0", optional = true } +s3fs = { version = "2022.5.0", optional = true} + [tool.poetry.dev-dependencies] pytest = "^7.0.0" pytest-checkdocs = "^2.0.0" @@ -79,6 +82,12 @@ pyarrow = ["pyarrow"] snappy = ["python-snappy"] python-snappy = ["zstandard"] hive = ["thrift"] +s3fs = ["s3fs"] + +[tool.pytest.ini_options] +markers = [ + "s3: marks a test as requiring access to s3 compliant storage (use with --aws-access-key-id, ----aws-secret-access-key, and --endpoint-url args)" +] [tool.black] line-length = 130 @@ -148,5 +157,13 @@ ignore_missing_imports = true module = "rich.*" ignore_missing_imports = true +[[tool.mypy.overrides]] +module = "fsspec.*" +ignore_missing_imports = true + +[[tool.mypy.overrides]] +module = "s3fs.*" +ignore_missing_imports = true + [tool.coverage.run] source = ['pyiceberg/'] diff --git a/tests/conftest.py b/tests/conftest.py index b4d2e005fe..4030fbe9f1 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -37,6 +37,7 @@ InputFile, OutputFile, OutputStream, + fsspec, ) from pyiceberg.schema import Schema from pyiceberg.types import ( @@ -56,6 +57,16 @@ from tests.io.test_io import LocalInputFile +def pytest_addoption(parser): + parser.addoption( + "--s3.endpoint", action="store", default="http://localhost:9000", help="The S3 endpoint URL for tests marked as s3" + ) + parser.addoption("--s3.access-key-id", action="store", default="admin", help="The AWS access key ID for tests marked as s3") + parser.addoption( + "--s3.secret-access-key", action="store", default="password", help="The AWS secret access key ID for tests marked as s3" + ) + + class FooStruct: """An example of an object that abides by StructProtocol""" @@ -1111,3 +1122,13 @@ def iceberg_manifest_entry_schema() -> Schema: schema_id=1, identifier_field_ids=[], ) + + +@pytest.fixture +def fsspec_fileio(request): + properties = { + "s3.endpoint": request.config.getoption("--s3.endpoint"), + "s3.access-key-id": request.config.getoption("--s3.access-key-id"), + "s3.secret-access-key": request.config.getoption("--s3.secret-access-key"), + } + return fsspec.FsspecFileIO(properties=properties) diff --git a/tests/io/test_fsspec.py b/tests/io/test_fsspec.py new file mode 100644 index 0000000000..2c05740fc3 --- /dev/null +++ b/tests/io/test_fsspec.py @@ -0,0 +1,200 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +import uuid + +import pytest + +from pyiceberg.io import fsspec +from tests.io.test_io import LocalInputFile + + +@pytest.mark.s3 +def test_fsspec_new_input_file(fsspec_fileio): + """Test creating a new input file from an fsspec file-io""" + filename = str(uuid.uuid4()) + + input_file = fsspec_fileio.new_input(f"s3://warehouse/{filename}") + + assert isinstance(input_file, fsspec.FsspecInputFile) + assert input_file.location == f"s3://warehouse/{filename}" + + +@pytest.mark.s3 +def test_fsspec_new_s3_output_file(fsspec_fileio): + """Test creating a new output file from an fsspec file-io""" + filename = str(uuid.uuid4()) + + output_file = fsspec_fileio.new_output(f"s3://warehouse/{filename}") + + assert isinstance(output_file, fsspec.FsspecOutputFile) + assert output_file.location == f"s3://warehouse/{filename}" + + +@pytest.mark.s3 +def test_fsspec_write_and_read_file(fsspec_fileio): + """Test writing and reading a file using FsspecInputFile and FsspecOutputFile""" + filename = str(uuid.uuid4()) + output_file = fsspec_fileio.new_output(location=f"s3://warehouse/{filename}") + with output_file.create() as f: + f.write(b"foo") + + input_file = fsspec_fileio.new_input(f"s3://warehouse/{filename}") + assert input_file.open().read() == b"foo" + + fsspec_fileio.delete(input_file) + + +@pytest.mark.s3 +def test_fsspec_getting_length_of_file(fsspec_fileio): + """Test getting the length of an FsspecInputFile and FsspecOutputFile""" + filename = str(uuid.uuid4()) + + output_file = fsspec_fileio.new_output(location=f"s3://warehouse/{filename}") + with output_file.create() as f: + f.write(b"foobar") + + assert len(output_file) == 6 + + input_file = fsspec_fileio.new_input(location=f"s3://warehouse/{filename}") + assert len(input_file) == 6 + + fsspec_fileio.delete(output_file) + + +@pytest.mark.s3 +def test_fsspec_file_tell(fsspec_fileio): + """Test finding cursor position for an fsspec file-io file""" + + filename = str(uuid.uuid4()) + + output_file = fsspec_fileio.new_output(location=f"s3://warehouse/{filename}") + with output_file.create() as f: + f.write(b"foobar") + + input_file = fsspec_fileio.new_input(location=f"s3://warehouse/{filename}") + f = input_file.open() + + f.seek(0) + assert f.tell() == 0 + f.seek(1) + assert f.tell() == 1 + f.seek(3) + assert f.tell() == 3 + f.seek(0) + assert f.tell() == 0 + + +@pytest.mark.s3 +def test_fsspec_read_specified_bytes_for_file(fsspec_fileio): + """Test reading a specified number of bytes from an fsspec file-io file""" + + filename = str(uuid.uuid4()) + output_file = fsspec_fileio.new_output(location=f"s3://warehouse/{filename}") + with output_file.create() as f: + f.write(b"foo") + + input_file = fsspec_fileio.new_input(location=f"s3://warehouse/{filename}") + f = input_file.open() + + f.seek(0) + assert b"f" == f.read(1) + f.seek(0) + assert b"fo" == f.read(2) + f.seek(1) + assert b"o" == f.read(1) + f.seek(1) + assert b"oo" == f.read(2) + f.seek(0) + assert b"foo" == f.read(999) # test reading amount larger than entire content length + + fsspec_fileio.delete(input_file) + + +@pytest.mark.s3 +def test_fsspec_raise_on_opening_file_not_found(fsspec_fileio): + """Test that an fsppec input file raises appropriately when the s3 file is not found""" + + filename = str(uuid.uuid4()) + input_file = fsspec_fileio.new_input(location=f"s3://warehouse/{filename}") + with pytest.raises(FileNotFoundError) as exc_info: + input_file.open().read() + + assert filename in str(exc_info.value) + + +@pytest.mark.s3 +def test_checking_if_a_file_exists(fsspec_fileio): + """Test checking if a file exists""" + + non_existent_file = fsspec_fileio.new_input(location="s3://warehouse/does-not-exist.txt") + assert not non_existent_file.exists() + + filename = str(uuid.uuid4()) + output_file = fsspec_fileio.new_output(location=f"s3://warehouse/{filename}") + assert not output_file.exists() + with output_file.create() as f: + f.write(b"foo") + + existing_input_file = fsspec_fileio.new_input(location=f"s3://warehouse/{filename}") + assert existing_input_file.exists() + + existing_output_file = fsspec_fileio.new_output(location=f"s3://warehouse/{filename}") + assert existing_output_file.exists() + + fsspec_fileio.delete(existing_output_file) + + +@pytest.mark.s3 +def test_closing_a_file(fsspec_fileio): + """Test closing an output file and input file""" + filename = str(uuid.uuid4()) + output_file = fsspec_fileio.new_output(location=f"s3://warehouse/{filename}") + with output_file.create() as f: + f.write(b"foo") + assert not f.closed + assert f.closed + + input_file = fsspec_fileio.new_input(location=f"s3://warehouse/{filename}") + f = input_file.open() + assert not f.closed + f.close() + assert f.closed + + fsspec_fileio.delete(f"s3://warehouse/{filename}") + + +@pytest.mark.s3 +def test_fsspec_converting_an_outputfile_to_an_inputfile(fsspec_fileio): + """Test converting an output file to an input file""" + filename = str(uuid.uuid4()) + output_file = fsspec_fileio.new_output(location=f"s3://warehouse/{filename}") + input_file = output_file.to_input_file() + assert input_file.location == output_file.location + + +@pytest.mark.s3 +def test_writing_avro_file(generated_manifest_entry_file, fsspec_fileio): + """Test that bytes match when reading a local avro file, writing it using fsspec file-io, and then reading it again""" + filename = str(uuid.uuid4()) + with LocalInputFile(generated_manifest_entry_file).open() as f: + b1 = f.read() + with fsspec_fileio.new_output(location=f"s3://warehouse/{filename}").create() as out_f: + out_f.write(b1) + with fsspec_fileio.new_input(location=f"s3://warehouse/{filename}").open() as in_f: + b2 = in_f.read() + assert b1 == b2 # Check that bytes of read from local avro file match bytes written to s3 From 480f496a5c02c31cde0e225c9485459ecffa2f0f Mon Sep 17 00:00:00 2001 From: Fokko Driesprong Date: Fri, 2 Sep 2022 17:26:57 +0200 Subject: [PATCH 191/642] Python: Fix issues with optional dependencies (#5687) --- poetry.lock | 4 ++-- pyiceberg/cli/console.py | 12 +----------- pyiceberg/io/__init__.py | 18 +++++++++++++----- pyproject.toml | 2 +- tests/io/test_io.py | 3 ++- 5 files changed, 19 insertions(+), 20 deletions(-) diff --git a/poetry.lock b/poetry.lock index 78aac49821..dc3720d2c9 100644 --- a/poetry.lock +++ b/poetry.lock @@ -537,7 +537,7 @@ python-versions = "*" name = "pyyaml" version = "6.0" description = "YAML parser and emitter for Python" -category = "dev" +category = "main" optional = false python-versions = ">=3.6" @@ -742,7 +742,7 @@ snappy = ["python-snappy"] [metadata] lock-version = "1.1" python-versions = "^3.8" -content-hash = "7c6acb4d397c3920d6d6a47ce29593e9cb4799cf8e676ddb4d8dea2d2e9e43e5" +content-hash = "3c949093a3f568c641fdbc96f7f1f32640cf5c19572b3cacdcc5a7d6c3135855" [metadata.files] aiobotocore = [] diff --git a/pyiceberg/cli/console.py b/pyiceberg/cli/console.py index b3ed16273c..404a882d9e 100644 --- a/pyiceberg/cli/console.py +++ b/pyiceberg/cli/console.py @@ -16,25 +16,15 @@ # under the License. # pylint: disable=broad-except,redefined-builtin,redefined-outer-name from functools import wraps -from typing import ( - Dict, - Literal, - Optional, - Tuple, - Type, -) +from typing import Literal, Optional, Tuple import click from click import Context from pyiceberg.catalog import Catalog, load_catalog -from pyiceberg.catalog.hive import HiveCatalog -from pyiceberg.catalog.rest import RestCatalog from pyiceberg.cli.output import ConsoleOutput, JsonOutput, Output from pyiceberg.exceptions import NoSuchNamespaceError, NoSuchPropertyException, NoSuchTableError -SUPPORTED_CATALOGS: Dict[str, Type[Catalog]] = {"thrift": HiveCatalog, "http": RestCatalog} - def catch_exception(): def decorator(func): diff --git a/pyiceberg/io/__init__.py b/pyiceberg/io/__init__.py index bf21f8c3d1..d40d486b06 100644 --- a/pyiceberg/io/__init__.py +++ b/pyiceberg/io/__init__.py @@ -239,11 +239,14 @@ def delete(self, location: Union[str, InputFile, OutputFile]) -> None: WAREHOUSE = "warehouse" ARROW_FILE_IO = "pyiceberg.io.pyarrow.PyArrowFileIO" +FSSPEC_FILE_IO = "pyiceberg.io.fsspec.FsspecFileIO" # Mappings from the Java FileIO impl to a Python one. The list is ordered by preference. # If an implementation isn't installed, it will fall back to the next one. SCHEMA_TO_FILE_IO: Dict[str, List[str]] = { - "s3": [ARROW_FILE_IO], + "s3": [FSSPEC_FILE_IO, ARROW_FILE_IO], + "s3a": [FSSPEC_FILE_IO, ARROW_FILE_IO], + "s3n": [FSSPEC_FILE_IO, ARROW_FILE_IO], "gcs": [ARROW_FILE_IO], "file": [ARROW_FILE_IO], "hdfs": [ARROW_FILE_IO], @@ -296,7 +299,12 @@ def load_file_io(properties: Properties, location: Optional[str] = None) -> File if file_io := _infer_file_io_from_schema(warehouse_location, properties): return file_io - # Default to PyArrow - from pyiceberg.io.pyarrow import PyArrowFileIO - - return PyArrowFileIO(properties) + try: + # Default to PyArrow + from pyiceberg.io.pyarrow import PyArrowFileIO + + return PyArrowFileIO(properties) + except ModuleNotFoundError as e: + raise ModuleNotFoundError( + 'Could not load a FileIO, please consider installing one: pip3 install "pyiceberg[s3fs]", for more options refer to the docs.' + ) from e diff --git a/pyproject.toml b/pyproject.toml index 3d6d9d25df..8039547e13 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -46,6 +46,7 @@ mmh3 = "^3.0.0" requests = "^2.28.1" click = "^8.1.3" rich = "^12.5.1" +pyyaml = "^6.0.0" pydantic = "^1.9.2" fsspec = "2022.5.0" @@ -67,7 +68,6 @@ pre-commit = "^2.0.0" fastavro = "^1.5.4" coverage = { version = "^6.4.4", extras = ["toml"] } requests-mock = "^1.9.3" -PyYAML = "^6.0.0" [tool.poetry.scripts] pyiceberg = "pyiceberg.cli.console:run" diff --git a/tests/io/test_io.py b/tests/io/test_io.py index 0048861aee..0e67870b54 100644 --- a/tests/io/test_io.py +++ b/tests/io/test_io.py @@ -34,6 +34,7 @@ _import_file_io, load_file_io, ) +from pyiceberg.io.fsspec import FsspecFileIO from pyiceberg.io.pyarrow import PyArrowFile, PyArrowFileIO @@ -418,7 +419,7 @@ def test_load_file_io_does_not_exist(): def test_load_file_io_warehouse(): - assert isinstance(load_file_io({"warehouse": "s3://some-path/"}), PyArrowFileIO) + assert isinstance(load_file_io({"warehouse": "s3://some-path/"}), FsspecFileIO) def test_load_file_io_location(): From 76f517591dd40421029d547350ebba3912954ff9 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 5 Sep 2022 09:32:32 -0700 Subject: [PATCH 192/642] Build: Bump pytest from 7.1.2 to 7.1.3 in /python (#5703) Bumps [pytest](https://github.com/pytest-dev/pytest) from 7.1.2 to 7.1.3. - [Release notes](https://github.com/pytest-dev/pytest/releases) - [Changelog](https://github.com/pytest-dev/pytest/blob/main/CHANGELOG.rst) - [Commits](https://github.com/pytest-dev/pytest/compare/7.1.2...7.1.3) Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- poetry.lock | 590 +++++++++++++++++++++++++++++++++++++++++++++---- pyproject.toml | 2 +- 2 files changed, 545 insertions(+), 47 deletions(-) diff --git a/poetry.lock b/poetry.lock index dc3720d2c9..f57261c441 100644 --- a/poetry.lock +++ b/poetry.lock @@ -34,7 +34,7 @@ multidict = ">=4.5,<7.0" yarl = ">=1.0,<2.0" [package.extras] -speedups = ["aiodns", "brotli", "cchardet"] +speedups = ["Brotli", "aiodns", "cchardet"] [[package]] name = "aioitertools" @@ -66,14 +66,6 @@ category = "main" optional = true python-versions = ">=3.6" -[[package]] -name = "atomicwrites" -version = "1.4.1" -description = "Atomic file writes." -category = "dev" -optional = false -python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*" - [[package]] name = "attrs" version = "22.1.0" @@ -342,6 +334,9 @@ category = "dev" optional = false python-versions = ">=2.7,!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,!=3.4.*,!=3.5.*,!=3.6.*" +[package.dependencies] +setuptools = "*" + [[package]] name = "numpy" version = "1.23.2" @@ -478,14 +473,13 @@ diagrams = ["jinja2", "railroad-diagrams"] [[package]] name = "pytest" -version = "7.1.2" +version = "7.1.3" description = "pytest: simple powerful testing with Python" category = "dev" optional = false python-versions = ">=3.7" [package.dependencies] -atomicwrites = {version = ">=1.0", markers = "sys_platform == \"win32\""} attrs = ">=19.2.0" colorama = {version = "*", markers = "sys_platform == \"win32\""} iniconfig = "*" @@ -608,6 +602,19 @@ fsspec = "2022.5.0" awscli = ["aiobotocore[awscli] (>=2.3.0,<2.4.0)"] boto3 = ["aiobotocore[boto3] (>=2.3.0,<2.4.0)"] +[[package]] +name = "setuptools" +version = "65.3.0" +description = "Easily download, build, install, upgrade, and uninstall Python packages" +category = "dev" +optional = false +python-versions = ">=3.7" + +[package.extras] +docs = ["furo", "jaraco.packaging (>=9)", "jaraco.tidelift (>=1.4)", "pygments-github-lexers (==0.0.5)", "rst.linker (>=1.9)", "sphinx", "sphinx-favicon", "sphinx-hoverxref (<2)", "sphinx-inline-tabs", "sphinx-notfound-page (==0.8.3)", "sphinx-reredirects", "sphinxcontrib-towncrier"] +testing = ["build[virtualenv]", "filelock (>=3.4.0)", "flake8 (<5)", "flake8-2020", "ini2toml[lite] (>=0.9)", "jaraco.envs (>=2.2)", "jaraco.path (>=3.2.0)", "mock", "pip (>=19.1)", "pip-run (>=8.8)", "pytest (>=6)", "pytest-black (>=0.3.7)", "pytest-checkdocs (>=2.4)", "pytest-cov", "pytest-enabler (>=1.3)", "pytest-flake8", "pytest-mypy (>=0.9.1)", "pytest-perf", "pytest-xdist", "tomli-w (>=1.0.0)", "virtualenv (>=13.0.0)", "wheel"] +testing-integration = ["build[virtualenv]", "filelock (>=3.4.0)", "jaraco.envs (>=2.2)", "jaraco.path (>=3.2.0)", "pytest", "pytest-enabler", "pytest-xdist", "tomli", "virtualenv (>=13.0.0)", "wheel"] + [[package]] name = "six" version = "1.16.0" @@ -742,17 +749,107 @@ snappy = ["python-snappy"] [metadata] lock-version = "1.1" python-versions = "^3.8" -content-hash = "3c949093a3f568c641fdbc96f7f1f32640cf5c19572b3cacdcc5a7d6c3135855" +content-hash = "8aadbca4c1e2def4875c5acc9fba8e9bc049e86d734d6301d2a9451e537eb21d" [metadata.files] -aiobotocore = [] -aiohttp = [] -aioitertools = [] -aiosignal = [] -async-timeout = [] -atomicwrites = [] -attrs = [] -botocore = [] +aiobotocore = [ + {file = "aiobotocore-2.3.4-py3-none-any.whl", hash = "sha256:eae059eb51726cee4de2027cfc72bfccc76cf0c229d6b2b08f640e53a568f657"}, + {file = "aiobotocore-2.3.4.tar.gz", hash = "sha256:6554ebea5764f66f4be544a4fcaa0953ee80e600dd7bd818ba4893d72bf12bfb"}, +] +aiohttp = [ + {file = "aiohttp-3.8.1-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:1ed0b6477896559f17b9eaeb6d38e07f7f9ffe40b9f0f9627ae8b9926ae260a8"}, + {file = "aiohttp-3.8.1-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:7dadf3c307b31e0e61689cbf9e06be7a867c563d5a63ce9dca578f956609abf8"}, + {file = "aiohttp-3.8.1-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:a79004bb58748f31ae1cbe9fa891054baaa46fb106c2dc7af9f8e3304dc30316"}, + {file = "aiohttp-3.8.1-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:12de6add4038df8f72fac606dff775791a60f113a725c960f2bab01d8b8e6b15"}, + {file = "aiohttp-3.8.1-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:6f0d5f33feb5f69ddd57a4a4bd3d56c719a141080b445cbf18f238973c5c9923"}, + {file = "aiohttp-3.8.1-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:eaba923151d9deea315be1f3e2b31cc39a6d1d2f682f942905951f4e40200922"}, + {file = "aiohttp-3.8.1-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_12_i686.manylinux2010_i686.whl", hash = "sha256:099ebd2c37ac74cce10a3527d2b49af80243e2a4fa39e7bce41617fbc35fa3c1"}, + {file = "aiohttp-3.8.1-cp310-cp310-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:2e5d962cf7e1d426aa0e528a7e198658cdc8aa4fe87f781d039ad75dcd52c516"}, + {file = "aiohttp-3.8.1-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:fa0ffcace9b3aa34d205d8130f7873fcfefcb6a4dd3dd705b0dab69af6712642"}, + {file = "aiohttp-3.8.1-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:61bfc23df345d8c9716d03717c2ed5e27374e0fe6f659ea64edcd27b4b044cf7"}, + {file = "aiohttp-3.8.1-cp310-cp310-musllinux_1_1_ppc64le.whl", hash = "sha256:31560d268ff62143e92423ef183680b9829b1b482c011713ae941997921eebc8"}, + {file = "aiohttp-3.8.1-cp310-cp310-musllinux_1_1_s390x.whl", hash = "sha256:01d7bdb774a9acc838e6b8f1d114f45303841b89b95984cbb7d80ea41172a9e3"}, + {file = "aiohttp-3.8.1-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:97ef77eb6b044134c0b3a96e16abcb05ecce892965a2124c566af0fd60f717e2"}, + {file = "aiohttp-3.8.1-cp310-cp310-win32.whl", hash = "sha256:c2aef4703f1f2ddc6df17519885dbfa3514929149d3ff900b73f45998f2532fa"}, + {file = "aiohttp-3.8.1-cp310-cp310-win_amd64.whl", hash = "sha256:713ac174a629d39b7c6a3aa757b337599798da4c1157114a314e4e391cd28e32"}, + {file = "aiohttp-3.8.1-cp36-cp36m-macosx_10_9_x86_64.whl", hash = "sha256:473d93d4450880fe278696549f2e7aed8cd23708c3c1997981464475f32137db"}, + {file = "aiohttp-3.8.1-cp36-cp36m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:99b5eeae8e019e7aad8af8bb314fb908dd2e028b3cdaad87ec05095394cce632"}, + {file = "aiohttp-3.8.1-cp36-cp36m-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:3af642b43ce56c24d063325dd2cf20ee012d2b9ba4c3c008755a301aaea720ad"}, + {file = "aiohttp-3.8.1-cp36-cp36m-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:c3630c3ef435c0a7c549ba170a0633a56e92629aeed0e707fec832dee313fb7a"}, + {file = "aiohttp-3.8.1-cp36-cp36m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_12_i686.manylinux2010_i686.whl", hash = "sha256:4a4a4e30bf1edcad13fb0804300557aedd07a92cabc74382fdd0ba6ca2661091"}, + {file = "aiohttp-3.8.1-cp36-cp36m-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:6f8b01295e26c68b3a1b90efb7a89029110d3a4139270b24fda961893216c440"}, + {file = "aiohttp-3.8.1-cp36-cp36m-musllinux_1_1_aarch64.whl", hash = "sha256:a25fa703a527158aaf10dafd956f7d42ac6d30ec80e9a70846253dd13e2f067b"}, + {file = "aiohttp-3.8.1-cp36-cp36m-musllinux_1_1_i686.whl", hash = "sha256:5bfde62d1d2641a1f5173b8c8c2d96ceb4854f54a44c23102e2ccc7e02f003ec"}, + {file = "aiohttp-3.8.1-cp36-cp36m-musllinux_1_1_ppc64le.whl", hash = "sha256:51467000f3647d519272392f484126aa716f747859794ac9924a7aafa86cd411"}, + {file = "aiohttp-3.8.1-cp36-cp36m-musllinux_1_1_s390x.whl", hash = "sha256:03a6d5349c9ee8f79ab3ff3694d6ce1cfc3ced1c9d36200cb8f08ba06bd3b782"}, + {file = "aiohttp-3.8.1-cp36-cp36m-musllinux_1_1_x86_64.whl", hash = "sha256:102e487eeb82afac440581e5d7f8f44560b36cf0bdd11abc51a46c1cd88914d4"}, + {file = "aiohttp-3.8.1-cp36-cp36m-win32.whl", hash = "sha256:4aed991a28ea3ce320dc8ce655875e1e00a11bdd29fe9444dd4f88c30d558602"}, + {file = "aiohttp-3.8.1-cp36-cp36m-win_amd64.whl", hash = "sha256:b0e20cddbd676ab8a64c774fefa0ad787cc506afd844de95da56060348021e96"}, + {file = "aiohttp-3.8.1-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:37951ad2f4a6df6506750a23f7cbabad24c73c65f23f72e95897bb2cecbae676"}, + {file = "aiohttp-3.8.1-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:5c23b1ad869653bc818e972b7a3a79852d0e494e9ab7e1a701a3decc49c20d51"}, + {file = "aiohttp-3.8.1-cp37-cp37m-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:15b09b06dae900777833fe7fc4b4aa426556ce95847a3e8d7548e2d19e34edb8"}, + {file = "aiohttp-3.8.1-cp37-cp37m-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:477c3ea0ba410b2b56b7efb072c36fa91b1e6fc331761798fa3f28bb224830dd"}, + {file = "aiohttp-3.8.1-cp37-cp37m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_12_i686.manylinux2010_i686.whl", hash = "sha256:2f2f69dca064926e79997f45b2f34e202b320fd3782f17a91941f7eb85502ee2"}, + {file = "aiohttp-3.8.1-cp37-cp37m-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:ef9612483cb35171d51d9173647eed5d0069eaa2ee812793a75373447d487aa4"}, + {file = "aiohttp-3.8.1-cp37-cp37m-musllinux_1_1_aarch64.whl", hash = "sha256:6d69f36d445c45cda7b3b26afef2fc34ef5ac0cdc75584a87ef307ee3c8c6d00"}, + {file = "aiohttp-3.8.1-cp37-cp37m-musllinux_1_1_i686.whl", hash = "sha256:55c3d1072704d27401c92339144d199d9de7b52627f724a949fc7d5fc56d8b93"}, + {file = "aiohttp-3.8.1-cp37-cp37m-musllinux_1_1_ppc64le.whl", hash = "sha256:b9d00268fcb9f66fbcc7cd9fe423741d90c75ee029a1d15c09b22d23253c0a44"}, + {file = "aiohttp-3.8.1-cp37-cp37m-musllinux_1_1_s390x.whl", hash = "sha256:07b05cd3305e8a73112103c834e91cd27ce5b4bd07850c4b4dbd1877d3f45be7"}, + {file = "aiohttp-3.8.1-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:c34dc4958b232ef6188c4318cb7b2c2d80521c9a56c52449f8f93ab7bc2a8a1c"}, + {file = "aiohttp-3.8.1-cp37-cp37m-win32.whl", hash = "sha256:d2f9b69293c33aaa53d923032fe227feac867f81682f002ce33ffae978f0a9a9"}, + {file = "aiohttp-3.8.1-cp37-cp37m-win_amd64.whl", hash = "sha256:6ae828d3a003f03ae31915c31fa684b9890ea44c9c989056fea96e3d12a9fa17"}, + {file = "aiohttp-3.8.1-cp38-cp38-macosx_10_9_universal2.whl", hash = "sha256:0c7ebbbde809ff4e970824b2b6cb7e4222be6b95a296e46c03cf050878fc1785"}, + {file = "aiohttp-3.8.1-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:8b7ef7cbd4fec9a1e811a5de813311ed4f7ac7d93e0fda233c9b3e1428f7dd7b"}, + {file = "aiohttp-3.8.1-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:c3d6a4d0619e09dcd61021debf7059955c2004fa29f48788a3dfaf9c9901a7cd"}, + {file = "aiohttp-3.8.1-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:718626a174e7e467f0558954f94af117b7d4695d48eb980146016afa4b580b2e"}, + {file = "aiohttp-3.8.1-cp38-cp38-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:589c72667a5febd36f1315aa6e5f56dd4aa4862df295cb51c769d16142ddd7cd"}, + {file = "aiohttp-3.8.1-cp38-cp38-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:2ed076098b171573161eb146afcb9129b5ff63308960aeca4b676d9d3c35e700"}, + {file = "aiohttp-3.8.1-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_12_i686.manylinux2010_i686.whl", hash = "sha256:086f92daf51a032d062ec5f58af5ca6a44d082c35299c96376a41cbb33034675"}, + {file = "aiohttp-3.8.1-cp38-cp38-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:11691cf4dc5b94236ccc609b70fec991234e7ef8d4c02dd0c9668d1e486f5abf"}, + {file = "aiohttp-3.8.1-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:31d1e1c0dbf19ebccbfd62eff461518dcb1e307b195e93bba60c965a4dcf1ba0"}, + {file = "aiohttp-3.8.1-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:11a67c0d562e07067c4e86bffc1553f2cf5b664d6111c894671b2b8712f3aba5"}, + {file = "aiohttp-3.8.1-cp38-cp38-musllinux_1_1_ppc64le.whl", hash = "sha256:bb01ba6b0d3f6c68b89fce7305080145d4877ad3acaed424bae4d4ee75faa950"}, + {file = "aiohttp-3.8.1-cp38-cp38-musllinux_1_1_s390x.whl", hash = "sha256:44db35a9e15d6fe5c40d74952e803b1d96e964f683b5a78c3cc64eb177878155"}, + {file = "aiohttp-3.8.1-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:844a9b460871ee0a0b0b68a64890dae9c415e513db0f4a7e3cab41a0f2fedf33"}, + {file = "aiohttp-3.8.1-cp38-cp38-win32.whl", hash = "sha256:7d08744e9bae2ca9c382581f7dce1273fe3c9bae94ff572c3626e8da5b193c6a"}, + {file = "aiohttp-3.8.1-cp38-cp38-win_amd64.whl", hash = "sha256:04d48b8ce6ab3cf2097b1855e1505181bdd05586ca275f2505514a6e274e8e75"}, + {file = "aiohttp-3.8.1-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:f5315a2eb0239185af1bddb1abf472d877fede3cc8d143c6cddad37678293237"}, + {file = "aiohttp-3.8.1-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:a996d01ca39b8dfe77440f3cd600825d05841088fd6bc0144cc6c2ec14cc5f74"}, + {file = "aiohttp-3.8.1-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:13487abd2f761d4be7c8ff9080de2671e53fff69711d46de703c310c4c9317ca"}, + {file = "aiohttp-3.8.1-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:ea302f34477fda3f85560a06d9ebdc7fa41e82420e892fc50b577e35fc6a50b2"}, + {file = "aiohttp-3.8.1-cp39-cp39-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:a2f635ce61a89c5732537a7896b6319a8fcfa23ba09bec36e1b1ac0ab31270d2"}, + {file = "aiohttp-3.8.1-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:e999f2d0e12eea01caeecb17b653f3713d758f6dcc770417cf29ef08d3931421"}, + {file = "aiohttp-3.8.1-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_12_i686.manylinux2010_i686.whl", hash = "sha256:0770e2806a30e744b4e21c9d73b7bee18a1cfa3c47991ee2e5a65b887c49d5cf"}, + {file = "aiohttp-3.8.1-cp39-cp39-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:d15367ce87c8e9e09b0f989bfd72dc641bcd04ba091c68cd305312d00962addd"}, + {file = "aiohttp-3.8.1-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:6c7cefb4b0640703eb1069835c02486669312bf2f12b48a748e0a7756d0de33d"}, + {file = "aiohttp-3.8.1-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:71927042ed6365a09a98a6377501af5c9f0a4d38083652bcd2281a06a5976724"}, + {file = "aiohttp-3.8.1-cp39-cp39-musllinux_1_1_ppc64le.whl", hash = "sha256:28d490af82bc6b7ce53ff31337a18a10498303fe66f701ab65ef27e143c3b0ef"}, + {file = "aiohttp-3.8.1-cp39-cp39-musllinux_1_1_s390x.whl", hash = "sha256:b6613280ccedf24354406caf785db748bebbddcf31408b20c0b48cb86af76866"}, + {file = "aiohttp-3.8.1-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:81e3d8c34c623ca4e36c46524a3530e99c0bc95ed068fd6e9b55cb721d408fb2"}, + {file = "aiohttp-3.8.1-cp39-cp39-win32.whl", hash = "sha256:7187a76598bdb895af0adbd2fb7474d7f6025d170bc0a1130242da817ce9e7d1"}, + {file = "aiohttp-3.8.1-cp39-cp39-win_amd64.whl", hash = "sha256:1c182cb873bc91b411e184dab7a2b664d4fea2743df0e4d57402f7f3fa644bac"}, + {file = "aiohttp-3.8.1.tar.gz", hash = "sha256:fc5471e1a54de15ef71c1bc6ebe80d4dc681ea600e68bfd1cbce40427f0b7578"}, +] +aioitertools = [ + {file = "aioitertools-0.10.0-py3-none-any.whl", hash = "sha256:a2ea2a39ebf272a2fbb58bfdb73e1daeeb6686edbbc8082215dfc8b8ffffa6e8"}, + {file = "aioitertools-0.10.0.tar.gz", hash = "sha256:7d1d1d4a03d462c5a0840787d3df098f125847e0d38b833b30f8f8cbc45a1420"}, +] +aiosignal = [ + {file = "aiosignal-1.2.0-py3-none-any.whl", hash = "sha256:26e62109036cd181df6e6ad646f91f0dcfd05fe16d0cb924138ff2ab75d64e3a"}, + {file = "aiosignal-1.2.0.tar.gz", hash = "sha256:78ed67db6c7b7ced4f98e495e572106d5c432a93e1ddd1bf475e1dc05f5b7df2"}, +] +async-timeout = [ + {file = "async-timeout-4.0.2.tar.gz", hash = "sha256:2163e1640ddb52b7a8c80d0a67a08587e5d245cc9c553a74a847056bc2976b15"}, + {file = "async_timeout-4.0.2-py3-none-any.whl", hash = "sha256:8ca1e4fcf50d07413d66d1a5e416e42cfdf5851c981d679a09851a6853383b3c"}, +] +attrs = [ + {file = "attrs-22.1.0-py2.py3-none-any.whl", hash = "sha256:86efa402f67bf2df34f51a335487cf46b1ec130d02b8d39fd248abfd30da551c"}, + {file = "attrs-22.1.0.tar.gz", hash = "sha256:29adc2665447e5191d0e7c568fde78b21f9672d344281d0c6e1ab085429b22b6"}, +] +botocore = [ + {file = "botocore-1.24.21-py3-none-any.whl", hash = "sha256:92daca8775e738a9db9b465d533019285f09d541e903233261299fd87c2f842c"}, + {file = "botocore-1.24.21.tar.gz", hash = "sha256:7e976cfd0a61601e74624ef8f5246b40a01f2cce73a011ef29cf80a6e371d0fa"}, +] certifi = [ {file = "certifi-2022.6.15-py3-none-any.whl", hash = "sha256:fe86415d55e84719d75f8b69414f6438ac3547d2078ab91b67e779ef69378412"}, {file = "certifi-2022.6.15.tar.gz", hash = "sha256:84c85a9078b11105f04f3036a9482ae10e4621616db313fe045dd24743a0820d"}, @@ -827,14 +924,74 @@ cfgv = [ {file = "cfgv-3.3.1-py2.py3-none-any.whl", hash = "sha256:c6a0883f3917a037485059700b9e75da2464e6c27051014ad85ba6aaa5884426"}, {file = "cfgv-3.3.1.tar.gz", hash = "sha256:f5a830efb9ce7a445376bb66ec94c638a9787422f96264c98edc6bdeed8ab736"}, ] -charset-normalizer = [] -click = [] +charset-normalizer = [ + {file = "charset-normalizer-2.1.1.tar.gz", hash = "sha256:5a3d016c7c547f69d6f81fb0db9449ce888b418b5b9952cc5e6e66843e9dd845"}, + {file = "charset_normalizer-2.1.1-py3-none-any.whl", hash = "sha256:83e9a75d1911279afd89352c68b45348559d1fc0506b054b346651b5e7fee29f"}, +] +click = [ + {file = "click-8.1.3-py3-none-any.whl", hash = "sha256:bb4d8133cb15a609f44e8213d9b391b0809795062913b383c62be0ee95b1db48"}, + {file = "click-8.1.3.tar.gz", hash = "sha256:7682dc8afb30297001674575ea00d1814d808d6a36af415a82bd481d37ba7b8e"}, +] colorama = [ {file = "colorama-0.4.5-py2.py3-none-any.whl", hash = "sha256:854bf444933e37f5824ae7bfc1e98d5bce2ebe4160d46b5edf346a89358e99da"}, {file = "colorama-0.4.5.tar.gz", hash = "sha256:e6c6b4334fc50988a639d9b98aa429a0b57da6e17b9a44f0451f930b6967b7a4"}, ] -commonmark = [] -coverage = [] +commonmark = [ + {file = "commonmark-0.9.1-py2.py3-none-any.whl", hash = "sha256:da2f38c92590f83de410ba1a3cbceafbc74fee9def35f9251ba9a971d6d66fd9"}, + {file = "commonmark-0.9.1.tar.gz", hash = "sha256:452f9dc859be7f06631ddcb328b6919c67984aca654e5fefb3914d54691aed60"}, +] +coverage = [ + {file = "coverage-6.4.4-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:e7b4da9bafad21ea45a714d3ea6f3e1679099e420c8741c74905b92ee9bfa7cc"}, + {file = "coverage-6.4.4-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:fde17bc42e0716c94bf19d92e4c9f5a00c5feb401f5bc01101fdf2a8b7cacf60"}, + {file = "coverage-6.4.4-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:cdbb0d89923c80dbd435b9cf8bba0ff55585a3cdb28cbec65f376c041472c60d"}, + {file = "coverage-6.4.4-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:67f9346aeebea54e845d29b487eb38ec95f2ecf3558a3cffb26ee3f0dcc3e760"}, + {file = "coverage-6.4.4-cp310-cp310-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:42c499c14efd858b98c4e03595bf914089b98400d30789511577aa44607a1b74"}, + {file = "coverage-6.4.4-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:c35cca192ba700979d20ac43024a82b9b32a60da2f983bec6c0f5b84aead635c"}, + {file = "coverage-6.4.4-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:9cc4f107009bca5a81caef2fca843dbec4215c05e917a59dec0c8db5cff1d2aa"}, + {file = "coverage-6.4.4-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:5f444627b3664b80d078c05fe6a850dd711beeb90d26731f11d492dcbadb6973"}, + {file = "coverage-6.4.4-cp310-cp310-win32.whl", hash = "sha256:66e6df3ac4659a435677d8cd40e8eb1ac7219345d27c41145991ee9bf4b806a0"}, + {file = "coverage-6.4.4-cp310-cp310-win_amd64.whl", hash = "sha256:35ef1f8d8a7a275aa7410d2f2c60fa6443f4a64fae9be671ec0696a68525b875"}, + {file = "coverage-6.4.4-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:c1328d0c2f194ffda30a45f11058c02410e679456276bfa0bbe0b0ee87225fac"}, + {file = "coverage-6.4.4-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:61b993f3998ee384935ee423c3d40894e93277f12482f6e777642a0141f55782"}, + {file = "coverage-6.4.4-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:d5dd4b8e9cd0deb60e6fcc7b0647cbc1da6c33b9e786f9c79721fd303994832f"}, + {file = "coverage-6.4.4-cp311-cp311-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:7026f5afe0d1a933685d8f2169d7c2d2e624f6255fb584ca99ccca8c0e966fd7"}, + {file = "coverage-6.4.4-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:9c7b9b498eb0c0d48b4c2abc0e10c2d78912203f972e0e63e3c9dc21f15abdaa"}, + {file = "coverage-6.4.4-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:ee2b2fb6eb4ace35805f434e0f6409444e1466a47f620d1d5763a22600f0f892"}, + {file = "coverage-6.4.4-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:ab066f5ab67059d1f1000b5e1aa8bbd75b6ed1fc0014559aea41a9eb66fc2ce0"}, + {file = "coverage-6.4.4-cp311-cp311-win32.whl", hash = "sha256:9d6e1f3185cbfd3d91ac77ea065d85d5215d3dfa45b191d14ddfcd952fa53796"}, + {file = "coverage-6.4.4-cp311-cp311-win_amd64.whl", hash = "sha256:e3d3c4cc38b2882f9a15bafd30aec079582b819bec1b8afdbde8f7797008108a"}, + {file = "coverage-6.4.4-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:a095aa0a996ea08b10580908e88fbaf81ecf798e923bbe64fb98d1807db3d68a"}, + {file = "coverage-6.4.4-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:ef6f44409ab02e202b31a05dd6666797f9de2aa2b4b3534e9d450e42dea5e817"}, + {file = "coverage-6.4.4-cp37-cp37m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:4b7101938584d67e6f45f0015b60e24a95bf8dea19836b1709a80342e01b472f"}, + {file = "coverage-6.4.4-cp37-cp37m-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:14a32ec68d721c3d714d9b105c7acf8e0f8a4f4734c811eda75ff3718570b5e3"}, + {file = "coverage-6.4.4-cp37-cp37m-musllinux_1_1_aarch64.whl", hash = "sha256:6a864733b22d3081749450466ac80698fe39c91cb6849b2ef8752fd7482011f3"}, + {file = "coverage-6.4.4-cp37-cp37m-musllinux_1_1_i686.whl", hash = "sha256:08002f9251f51afdcc5e3adf5d5d66bb490ae893d9e21359b085f0e03390a820"}, + {file = "coverage-6.4.4-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:a3b2752de32c455f2521a51bd3ffb53c5b3ae92736afde67ce83477f5c1dd928"}, + {file = "coverage-6.4.4-cp37-cp37m-win32.whl", hash = "sha256:f855b39e4f75abd0dfbcf74a82e84ae3fc260d523fcb3532786bcbbcb158322c"}, + {file = "coverage-6.4.4-cp37-cp37m-win_amd64.whl", hash = "sha256:ee6ae6bbcac0786807295e9687169fba80cb0617852b2fa118a99667e8e6815d"}, + {file = "coverage-6.4.4-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:564cd0f5b5470094df06fab676c6d77547abfdcb09b6c29c8a97c41ad03b103c"}, + {file = "coverage-6.4.4-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:cbbb0e4cd8ddcd5ef47641cfac97d8473ab6b132dd9a46bacb18872828031685"}, + {file = "coverage-6.4.4-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:6113e4df2fa73b80f77663445be6d567913fb3b82a86ceb64e44ae0e4b695de1"}, + {file = "coverage-6.4.4-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:8d032bfc562a52318ae05047a6eb801ff31ccee172dc0d2504614e911d8fa83e"}, + {file = "coverage-6.4.4-cp38-cp38-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:e431e305a1f3126477abe9a184624a85308da8edf8486a863601d58419d26ffa"}, + {file = "coverage-6.4.4-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:cf2afe83a53f77aec067033199797832617890e15bed42f4a1a93ea24794ae3e"}, + {file = "coverage-6.4.4-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:783bc7c4ee524039ca13b6d9b4186a67f8e63d91342c713e88c1865a38d0892a"}, + {file = "coverage-6.4.4-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:ff934ced84054b9018665ca3967fc48e1ac99e811f6cc99ea65978e1d384454b"}, + {file = "coverage-6.4.4-cp38-cp38-win32.whl", hash = "sha256:e1fabd473566fce2cf18ea41171d92814e4ef1495e04471786cbc943b89a3781"}, + {file = "coverage-6.4.4-cp38-cp38-win_amd64.whl", hash = "sha256:4179502f210ebed3ccfe2f78bf8e2d59e50b297b598b100d6c6e3341053066a2"}, + {file = "coverage-6.4.4-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:98c0b9e9b572893cdb0a00e66cf961a238f8d870d4e1dc8e679eb8bdc2eb1b86"}, + {file = "coverage-6.4.4-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:fc600f6ec19b273da1d85817eda339fb46ce9eef3e89f220055d8696e0a06908"}, + {file = "coverage-6.4.4-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:7a98d6bf6d4ca5c07a600c7b4e0c5350cd483c85c736c522b786be90ea5bac4f"}, + {file = "coverage-6.4.4-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:01778769097dbd705a24e221f42be885c544bb91251747a8a3efdec6eb4788f2"}, + {file = "coverage-6.4.4-cp39-cp39-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:dfa0b97eb904255e2ab24166071b27408f1f69c8fbda58e9c0972804851e0558"}, + {file = "coverage-6.4.4-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:fcbe3d9a53e013f8ab88734d7e517eb2cd06b7e689bedf22c0eb68db5e4a0a19"}, + {file = "coverage-6.4.4-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:15e38d853ee224e92ccc9a851457fb1e1f12d7a5df5ae44544ce7863691c7a0d"}, + {file = "coverage-6.4.4-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:6913dddee2deff8ab2512639c5168c3e80b3ebb0f818fed22048ee46f735351a"}, + {file = "coverage-6.4.4-cp39-cp39-win32.whl", hash = "sha256:354df19fefd03b9a13132fa6643527ef7905712109d9c1c1903f2133d3a4e145"}, + {file = "coverage-6.4.4-cp39-cp39-win_amd64.whl", hash = "sha256:1238b08f3576201ebf41f7c20bf59baa0d05da941b123c6656e42cdb668e9827"}, + {file = "coverage-6.4.4-pp36.pp37.pp38-none-any.whl", hash = "sha256:f67cf9f406cf0d2f08a3515ce2db5b82625a7257f88aad87904674def6ddaec1"}, + {file = "coverage-6.4.4.tar.gz", hash = "sha256:e16c45b726acb780e1e6f88b286d3c10b3914ab03438f32117c4aa52d7f30d58"}, +] distlib = [ {file = "distlib-0.3.6-py2.py3-none-any.whl", hash = "sha256:f35c4b692542ca110de7ef0bea44d73981caeb34ca0b9b6b2e6d7790dda8f80e"}, {file = "distlib-0.3.6.tar.gz", hash = "sha256:14bad2d9b04d3a36127ac97f30b12a19268f211063d8f8ee4f47108896e11b46"}, @@ -843,11 +1000,98 @@ docutils = [ {file = "docutils-0.19-py3-none-any.whl", hash = "sha256:5e1de4d849fee02c63b040a4a3fd567f4ab104defd8a5511fbbc24a8a017efbc"}, {file = "docutils-0.19.tar.gz", hash = "sha256:33995a6753c30b7f577febfc2c50411fec6aac7f7ffeb7c4cfe5991072dcf9e6"}, ] -fastavro = [] -filelock = [] -frozenlist = [] -fsspec = [] -identify = [] +fastavro = [ + {file = "fastavro-1.6.0-cp310-cp310-macosx_10_14_x86_64.whl", hash = "sha256:cbaa7e5db05694c70985c7ddc8afd9f7c9aef3e66e78cb18d05c70db221eaa1a"}, + {file = "fastavro-1.6.0-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:8d18d3540aab5f34bb88ad58d2f45b2c64f10bd04ad126bb79024a1e07d6db2b"}, + {file = "fastavro-1.6.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:d1271f11304951d4b61415480ec53c2d44c297eb00fd2d95098171805a6350d7"}, + {file = "fastavro-1.6.0-cp310-cp310-win_amd64.whl", hash = "sha256:a80eec11dfc879299f4e20e6c5b19307dc50ec20a6f431fcfe2f29c5ae63c575"}, + {file = "fastavro-1.6.0-cp37-cp37m-macosx_10_15_x86_64.whl", hash = "sha256:77354c17dc36cd21a3127004fd83854e799899b26f42fbe4a8bce3424c493551"}, + {file = "fastavro-1.6.0-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:209afeb9dcdf624b10f223c17d3469fefd9669bdaceaf5a3fb47cb74a107e443"}, + {file = "fastavro-1.6.0-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:b78ce36f45cd582c8d8b5c8ade764b378d0f3581c1e62fba895945984dec107a"}, + {file = "fastavro-1.6.0-cp37-cp37m-win_amd64.whl", hash = "sha256:6ac6db0c8229e5c3f633208e9d7e9e016ee725cbc53a9e5993b4c070c189f506"}, + {file = "fastavro-1.6.0-cp38-cp38-macosx_10_15_x86_64.whl", hash = "sha256:7511fbf7e1251ae34965894cf59ad982612268a5043fbfeaf309db8f75c25489"}, + {file = "fastavro-1.6.0-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:c039f259c61e75d920b2ce236d676a7dbf122ee136df92133542a99174491392"}, + {file = "fastavro-1.6.0-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:970599975812c8df4aba06ac4e4e558211fc645f3ca44a8fab12b0f2152fd959"}, + {file = "fastavro-1.6.0-cp38-cp38-win_amd64.whl", hash = "sha256:ca4a3b4c39d2b280523111fb4ea09dd114ab78e7ee8adb166804624ce4b7ad98"}, + {file = "fastavro-1.6.0-cp39-cp39-macosx_10_15_x86_64.whl", hash = "sha256:878b06ac7f6e411c849ceafd2cc7dd9e6eb3f7d4a5268aa6f02eb8460a069082"}, + {file = "fastavro-1.6.0-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:8b788093cf45763e69cd8ffa2464a518edd13340a48e13c4211f85f25f91c972"}, + {file = "fastavro-1.6.0-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:a69d35e66812b3aed7204d796c2a0f132d2d5a115d94cc1fb3f5da3577a74d97"}, + {file = "fastavro-1.6.0-cp39-cp39-win_amd64.whl", hash = "sha256:4eee1bfa7a5d35d252a58c96b727c4645a0e0db3f4c44b94de444d3b086bbc48"}, + {file = "fastavro-1.6.0.tar.gz", hash = "sha256:64759b161bbacb5fdcbd54bd5cee05d8510836e09290e958937334dbf1bb587f"}, +] +filelock = [ + {file = "filelock-3.8.0-py3-none-any.whl", hash = "sha256:617eb4e5eedc82fc5f47b6d61e4d11cb837c56cb4544e39081099fa17ad109d4"}, + {file = "filelock-3.8.0.tar.gz", hash = "sha256:55447caa666f2198c5b6b13a26d2084d26fa5b115c00d065664b2124680c4edc"}, +] +frozenlist = [ + {file = "frozenlist-1.3.1-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:5f271c93f001748fc26ddea409241312a75e13466b06c94798d1a341cf0e6989"}, + {file = "frozenlist-1.3.1-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:9c6ef8014b842f01f5d2b55315f1af5cbfde284eb184075c189fd657c2fd8204"}, + {file = "frozenlist-1.3.1-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:219a9676e2eae91cb5cc695a78b4cb43d8123e4160441d2b6ce8d2c70c60e2f3"}, + {file = "frozenlist-1.3.1-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:b47d64cdd973aede3dd71a9364742c542587db214e63b7529fbb487ed67cddd9"}, + {file = "frozenlist-1.3.1-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:2af6f7a4e93f5d08ee3f9152bce41a6015b5cf87546cb63872cc19b45476e98a"}, + {file = "frozenlist-1.3.1-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:a718b427ff781c4f4e975525edb092ee2cdef6a9e7bc49e15063b088961806f8"}, + {file = "frozenlist-1.3.1-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:c56c299602c70bc1bb5d1e75f7d8c007ca40c9d7aebaf6e4ba52925d88ef826d"}, + {file = "frozenlist-1.3.1-cp310-cp310-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:717470bfafbb9d9be624da7780c4296aa7935294bd43a075139c3d55659038ca"}, + {file = "frozenlist-1.3.1-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:31b44f1feb3630146cffe56344704b730c33e042ffc78d21f2125a6a91168131"}, + {file = "frozenlist-1.3.1-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:c3b31180b82c519b8926e629bf9f19952c743e089c41380ddca5db556817b221"}, + {file = "frozenlist-1.3.1-cp310-cp310-musllinux_1_1_ppc64le.whl", hash = "sha256:d82bed73544e91fb081ab93e3725e45dd8515c675c0e9926b4e1f420a93a6ab9"}, + {file = "frozenlist-1.3.1-cp310-cp310-musllinux_1_1_s390x.whl", hash = "sha256:49459f193324fbd6413e8e03bd65789e5198a9fa3095e03f3620dee2f2dabff2"}, + {file = "frozenlist-1.3.1-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:94e680aeedc7fd3b892b6fa8395b7b7cc4b344046c065ed4e7a1e390084e8cb5"}, + {file = "frozenlist-1.3.1-cp310-cp310-win32.whl", hash = "sha256:fabb953ab913dadc1ff9dcc3a7a7d3dc6a92efab3a0373989b8063347f8705be"}, + {file = "frozenlist-1.3.1-cp310-cp310-win_amd64.whl", hash = "sha256:eee0c5ecb58296580fc495ac99b003f64f82a74f9576a244d04978a7e97166db"}, + {file = "frozenlist-1.3.1-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:0bc75692fb3770cf2b5856a6c2c9de967ca744863c5e89595df64e252e4b3944"}, + {file = "frozenlist-1.3.1-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:086ca1ac0a40e722d6833d4ce74f5bf1aba2c77cbfdc0cd83722ffea6da52a04"}, + {file = "frozenlist-1.3.1-cp37-cp37m-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:1b51eb355e7f813bcda00276b0114c4172872dc5fb30e3fea059b9367c18fbcb"}, + {file = "frozenlist-1.3.1-cp37-cp37m-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:74140933d45271c1a1283f708c35187f94e1256079b3c43f0c2267f9db5845ff"}, + {file = "frozenlist-1.3.1-cp37-cp37m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:ee4c5120ddf7d4dd1eaf079af3af7102b56d919fa13ad55600a4e0ebe532779b"}, + {file = "frozenlist-1.3.1-cp37-cp37m-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:97d9e00f3ac7c18e685320601f91468ec06c58acc185d18bb8e511f196c8d4b2"}, + {file = "frozenlist-1.3.1-cp37-cp37m-musllinux_1_1_aarch64.whl", hash = "sha256:6e19add867cebfb249b4e7beac382d33215d6d54476bb6be46b01f8cafb4878b"}, + {file = "frozenlist-1.3.1-cp37-cp37m-musllinux_1_1_i686.whl", hash = "sha256:a027f8f723d07c3f21963caa7d585dcc9b089335565dabe9c814b5f70c52705a"}, + {file = "frozenlist-1.3.1-cp37-cp37m-musllinux_1_1_ppc64le.whl", hash = "sha256:61d7857950a3139bce035ad0b0945f839532987dfb4c06cfe160254f4d19df03"}, + {file = "frozenlist-1.3.1-cp37-cp37m-musllinux_1_1_s390x.whl", hash = "sha256:53b2b45052e7149ee8b96067793db8ecc1ae1111f2f96fe1f88ea5ad5fd92d10"}, + {file = "frozenlist-1.3.1-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:bbb1a71b1784e68870800b1bc9f3313918edc63dbb8f29fbd2e767ce5821696c"}, + {file = "frozenlist-1.3.1-cp37-cp37m-win32.whl", hash = "sha256:ab6fa8c7871877810e1b4e9392c187a60611fbf0226a9e0b11b7b92f5ac72792"}, + {file = "frozenlist-1.3.1-cp37-cp37m-win_amd64.whl", hash = "sha256:f89139662cc4e65a4813f4babb9ca9544e42bddb823d2ec434e18dad582543bc"}, + {file = "frozenlist-1.3.1-cp38-cp38-macosx_10_9_universal2.whl", hash = "sha256:4c0c99e31491a1d92cde8648f2e7ccad0e9abb181f6ac3ddb9fc48b63301808e"}, + {file = "frozenlist-1.3.1-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:61e8cb51fba9f1f33887e22488bad1e28dd8325b72425f04517a4d285a04c519"}, + {file = "frozenlist-1.3.1-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:cc2f3e368ee5242a2cbe28323a866656006382872c40869b49b265add546703f"}, + {file = "frozenlist-1.3.1-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:58fb94a01414cddcdc6839807db77ae8057d02ddafc94a42faee6004e46c9ba8"}, + {file = "frozenlist-1.3.1-cp38-cp38-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:022178b277cb9277d7d3b3f2762d294f15e85cd2534047e68a118c2bb0058f3e"}, + {file = "frozenlist-1.3.1-cp38-cp38-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:572ce381e9fe027ad5e055f143763637dcbac2542cfe27f1d688846baeef5170"}, + {file = "frozenlist-1.3.1-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:19127f8dcbc157ccb14c30e6f00392f372ddb64a6ffa7106b26ff2196477ee9f"}, + {file = "frozenlist-1.3.1-cp38-cp38-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:42719a8bd3792744c9b523674b752091a7962d0d2d117f0b417a3eba97d1164b"}, + {file = "frozenlist-1.3.1-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:2743bb63095ef306041c8f8ea22bd6e4d91adabf41887b1ad7886c4c1eb43d5f"}, + {file = "frozenlist-1.3.1-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:fa47319a10e0a076709644a0efbcaab9e91902c8bd8ef74c6adb19d320f69b83"}, + {file = "frozenlist-1.3.1-cp38-cp38-musllinux_1_1_ppc64le.whl", hash = "sha256:52137f0aea43e1993264a5180c467a08a3e372ca9d378244c2d86133f948b26b"}, + {file = "frozenlist-1.3.1-cp38-cp38-musllinux_1_1_s390x.whl", hash = "sha256:f5abc8b4d0c5b556ed8cd41490b606fe99293175a82b98e652c3f2711b452988"}, + {file = "frozenlist-1.3.1-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:1e1cf7bc8cbbe6ce3881863671bac258b7d6bfc3706c600008925fb799a256e2"}, + {file = "frozenlist-1.3.1-cp38-cp38-win32.whl", hash = "sha256:0dde791b9b97f189874d654c55c24bf7b6782343e14909c84beebd28b7217845"}, + {file = "frozenlist-1.3.1-cp38-cp38-win_amd64.whl", hash = "sha256:9494122bf39da6422b0972c4579e248867b6b1b50c9b05df7e04a3f30b9a413d"}, + {file = "frozenlist-1.3.1-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:31bf9539284f39ff9398deabf5561c2b0da5bb475590b4e13dd8b268d7a3c5c1"}, + {file = "frozenlist-1.3.1-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:e0c8c803f2f8db7217898d11657cb6042b9b0553a997c4a0601f48a691480fab"}, + {file = "frozenlist-1.3.1-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:da5ba7b59d954f1f214d352308d1d86994d713b13edd4b24a556bcc43d2ddbc3"}, + {file = "frozenlist-1.3.1-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:74e6b2b456f21fc93ce1aff2b9728049f1464428ee2c9752a4b4f61e98c4db96"}, + {file = "frozenlist-1.3.1-cp39-cp39-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:526d5f20e954d103b1d47232e3839f3453c02077b74203e43407b962ab131e7b"}, + {file = "frozenlist-1.3.1-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:b499c6abe62a7a8d023e2c4b2834fce78a6115856ae95522f2f974139814538c"}, + {file = "frozenlist-1.3.1-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:ab386503f53bbbc64d1ad4b6865bf001414930841a870fc97f1546d4d133f141"}, + {file = "frozenlist-1.3.1-cp39-cp39-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:5f63c308f82a7954bf8263a6e6de0adc67c48a8b484fab18ff87f349af356efd"}, + {file = "frozenlist-1.3.1-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:12607804084d2244a7bd4685c9d0dca5df17a6a926d4f1967aa7978b1028f89f"}, + {file = "frozenlist-1.3.1-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:da1cdfa96425cbe51f8afa43e392366ed0b36ce398f08b60de6b97e3ed4affef"}, + {file = "frozenlist-1.3.1-cp39-cp39-musllinux_1_1_ppc64le.whl", hash = "sha256:f810e764617b0748b49a731ffaa525d9bb36ff38332411704c2400125af859a6"}, + {file = "frozenlist-1.3.1-cp39-cp39-musllinux_1_1_s390x.whl", hash = "sha256:35c3d79b81908579beb1fb4e7fcd802b7b4921f1b66055af2578ff7734711cfa"}, + {file = "frozenlist-1.3.1-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:c92deb5d9acce226a501b77307b3b60b264ca21862bd7d3e0c1f3594022f01bc"}, + {file = "frozenlist-1.3.1-cp39-cp39-win32.whl", hash = "sha256:5e77a8bd41e54b05e4fb2708dc6ce28ee70325f8c6f50f3df86a44ecb1d7a19b"}, + {file = "frozenlist-1.3.1-cp39-cp39-win_amd64.whl", hash = "sha256:625d8472c67f2d96f9a4302a947f92a7adbc1e20bedb6aff8dbc8ff039ca6189"}, + {file = "frozenlist-1.3.1.tar.gz", hash = "sha256:3a735e4211a04ccfa3f4833547acdf5d2f863bfeb01cfd3edaffbc251f15cec8"}, +] +fsspec = [ + {file = "fsspec-2022.5.0-py3-none-any.whl", hash = "sha256:2c198c50eb541a80bbd03540b07602c4a957366f3fb416a1f270d34bd4ff0926"}, + {file = "fsspec-2022.5.0.tar.gz", hash = "sha256:7a5459c75c44e760fbe6a3ccb1f37e81e023cde7da8ba20401258d877ec483b4"}, +] +identify = [ + {file = "identify-2.5.3-py2.py3-none-any.whl", hash = "sha256:25851c8c1370effb22aaa3c987b30449e9ff0cece408f810ae6ce408fdd20893"}, + {file = "identify-2.5.3.tar.gz", hash = "sha256:887e7b91a1be152b0d46bbf072130235a8117392b9f1828446079a816a05ef44"}, +] idna = [ {file = "idna-3.3-py3-none-any.whl", hash = "sha256:84d9dd047ffa80596e0f246e2eab0b391788b0503584e8945f2368256d2735ff"}, {file = "idna-3.3.tar.gz", hash = "sha256:9d643ff0a55b762d5cdb124b8eaa99c66322e2157b69160bc32796e824360e6d"}, @@ -860,7 +1104,10 @@ iniconfig = [ {file = "iniconfig-1.1.1-py2.py3-none-any.whl", hash = "sha256:011e24c64b7f47f6ebd835bb12a743f2fbe9a26d4cecaa7f53bc4f35ee9da8b3"}, {file = "iniconfig-1.1.1.tar.gz", hash = "sha256:bc3af051d7d14b2ee5ef9969666def0cd1a000e121eaea580d4a313df4b37f32"}, ] -jmespath = [] +jmespath = [ + {file = "jmespath-1.0.1-py3-none-any.whl", hash = "sha256:02e2e4cc71b5bcab88332eebf907519190dd9e6e82107fa7f83b1003a6252980"}, + {file = "jmespath-1.0.1.tar.gz", hash = "sha256:90261b206d6defd58fdd5e85f478bf633a2901798906be2ad389150c5c60edbe"}, +] mmh3 = [ {file = "mmh3-3.0.0-cp36-cp36m-macosx_10_9_x86_64.whl", hash = "sha256:23912dde2ad4f701926948dd8e79a0e42b000f73962806f153931f52985e1e07"}, {file = "mmh3-3.0.0-cp36-cp36m-manylinux1_x86_64.whl", hash = "sha256:07f1308a410dc406d6a3c282a685728d00a87f3ed684f012671b96d6cc6a41c3"}, @@ -889,17 +1136,109 @@ mmh3 = [ {file = "mmh3-3.0.0-cp39-cp39-win_amd64.whl", hash = "sha256:bd870aedd9189eff1cf4e1687869b56c7e9461ee869789139c3e704009e5c227"}, {file = "mmh3-3.0.0.tar.gz", hash = "sha256:d1ec578c09a07d3518ec9be540b87546397fa3455de73c166fcce51eaa5c41c5"}, ] -multidict = [] +multidict = [ + {file = "multidict-6.0.2-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:0b9e95a740109c6047602f4db4da9949e6c5945cefbad34a1299775ddc9a62e2"}, + {file = "multidict-6.0.2-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:ac0e27844758d7177989ce406acc6a83c16ed4524ebc363c1f748cba184d89d3"}, + {file = "multidict-6.0.2-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:041b81a5f6b38244b34dc18c7b6aba91f9cdaf854d9a39e5ff0b58e2b5773b9c"}, + {file = "multidict-6.0.2-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:5fdda29a3c7e76a064f2477c9aab1ba96fd94e02e386f1e665bca1807fc5386f"}, + {file = "multidict-6.0.2-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:3368bf2398b0e0fcbf46d85795adc4c259299fec50c1416d0f77c0a843a3eed9"}, + {file = "multidict-6.0.2-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:f4f052ee022928d34fe1f4d2bc743f32609fb79ed9c49a1710a5ad6b2198db20"}, + {file = "multidict-6.0.2-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:225383a6603c086e6cef0f2f05564acb4f4d5f019a4e3e983f572b8530f70c88"}, + {file = "multidict-6.0.2-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:50bd442726e288e884f7be9071016c15a8742eb689a593a0cac49ea093eef0a7"}, + {file = "multidict-6.0.2-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:47e6a7e923e9cada7c139531feac59448f1f47727a79076c0b1ee80274cd8eee"}, + {file = "multidict-6.0.2-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:0556a1d4ea2d949efe5fd76a09b4a82e3a4a30700553a6725535098d8d9fb672"}, + {file = "multidict-6.0.2-cp310-cp310-musllinux_1_1_ppc64le.whl", hash = "sha256:626fe10ac87851f4cffecee161fc6f8f9853f0f6f1035b59337a51d29ff3b4f9"}, + {file = "multidict-6.0.2-cp310-cp310-musllinux_1_1_s390x.whl", hash = "sha256:8064b7c6f0af936a741ea1efd18690bacfbae4078c0c385d7c3f611d11f0cf87"}, + {file = "multidict-6.0.2-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:2d36e929d7f6a16d4eb11b250719c39560dd70545356365b494249e2186bc389"}, + {file = "multidict-6.0.2-cp310-cp310-win32.whl", hash = "sha256:fcb91630817aa8b9bc4a74023e4198480587269c272c58b3279875ed7235c293"}, + {file = "multidict-6.0.2-cp310-cp310-win_amd64.whl", hash = "sha256:8cbf0132f3de7cc6c6ce00147cc78e6439ea736cee6bca4f068bcf892b0fd658"}, + {file = "multidict-6.0.2-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:05f6949d6169878a03e607a21e3b862eaf8e356590e8bdae4227eedadacf6e51"}, + {file = "multidict-6.0.2-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:e2c2e459f7050aeb7c1b1276763364884595d47000c1cddb51764c0d8976e608"}, + {file = "multidict-6.0.2-cp37-cp37m-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:d0509e469d48940147e1235d994cd849a8f8195e0bca65f8f5439c56e17872a3"}, + {file = "multidict-6.0.2-cp37-cp37m-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:514fe2b8d750d6cdb4712346a2c5084a80220821a3e91f3f71eec11cf8d28fd4"}, + {file = "multidict-6.0.2-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:19adcfc2a7197cdc3987044e3f415168fc5dc1f720c932eb1ef4f71a2067e08b"}, + {file = "multidict-6.0.2-cp37-cp37m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:b9d153e7f1f9ba0b23ad1568b3b9e17301e23b042c23870f9ee0522dc5cc79e8"}, + {file = "multidict-6.0.2-cp37-cp37m-musllinux_1_1_aarch64.whl", hash = "sha256:aef9cc3d9c7d63d924adac329c33835e0243b5052a6dfcbf7732a921c6e918ba"}, + {file = "multidict-6.0.2-cp37-cp37m-musllinux_1_1_i686.whl", hash = "sha256:4571f1beddff25f3e925eea34268422622963cd8dc395bb8778eb28418248e43"}, + {file = "multidict-6.0.2-cp37-cp37m-musllinux_1_1_ppc64le.whl", hash = "sha256:d48b8ee1d4068561ce8033d2c344cf5232cb29ee1a0206a7b828c79cbc5982b8"}, + {file = "multidict-6.0.2-cp37-cp37m-musllinux_1_1_s390x.whl", hash = "sha256:45183c96ddf61bf96d2684d9fbaf6f3564d86b34cb125761f9a0ef9e36c1d55b"}, + {file = "multidict-6.0.2-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:75bdf08716edde767b09e76829db8c1e5ca9d8bb0a8d4bd94ae1eafe3dac5e15"}, + {file = "multidict-6.0.2-cp37-cp37m-win32.whl", hash = "sha256:a45e1135cb07086833ce969555df39149680e5471c04dfd6a915abd2fc3f6dbc"}, + {file = "multidict-6.0.2-cp37-cp37m-win_amd64.whl", hash = "sha256:6f3cdef8a247d1eafa649085812f8a310e728bdf3900ff6c434eafb2d443b23a"}, + {file = "multidict-6.0.2-cp38-cp38-macosx_10_9_universal2.whl", hash = "sha256:0327292e745a880459ef71be14e709aaea2f783f3537588fb4ed09b6c01bca60"}, + {file = "multidict-6.0.2-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:e875b6086e325bab7e680e4316d667fc0e5e174bb5611eb16b3ea121c8951b86"}, + {file = "multidict-6.0.2-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:feea820722e69451743a3d56ad74948b68bf456984d63c1a92e8347b7b88452d"}, + {file = "multidict-6.0.2-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:9cc57c68cb9139c7cd6fc39f211b02198e69fb90ce4bc4a094cf5fe0d20fd8b0"}, + {file = "multidict-6.0.2-cp38-cp38-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:497988d6b6ec6ed6f87030ec03280b696ca47dbf0648045e4e1d28b80346560d"}, + {file = "multidict-6.0.2-cp38-cp38-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:89171b2c769e03a953d5969b2f272efa931426355b6c0cb508022976a17fd376"}, + {file = "multidict-6.0.2-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:684133b1e1fe91eda8fa7447f137c9490a064c6b7f392aa857bba83a28cfb693"}, + {file = "multidict-6.0.2-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:fd9fc9c4849a07f3635ccffa895d57abce554b467d611a5009ba4f39b78a8849"}, + {file = "multidict-6.0.2-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:e07c8e79d6e6fd37b42f3250dba122053fddb319e84b55dd3a8d6446e1a7ee49"}, + {file = "multidict-6.0.2-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:4070613ea2227da2bfb2c35a6041e4371b0af6b0be57f424fe2318b42a748516"}, + {file = "multidict-6.0.2-cp38-cp38-musllinux_1_1_ppc64le.whl", hash = "sha256:47fbeedbf94bed6547d3aa632075d804867a352d86688c04e606971595460227"}, + {file = "multidict-6.0.2-cp38-cp38-musllinux_1_1_s390x.whl", hash = "sha256:5774d9218d77befa7b70d836004a768fb9aa4fdb53c97498f4d8d3f67bb9cfa9"}, + {file = "multidict-6.0.2-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:2957489cba47c2539a8eb7ab32ff49101439ccf78eab724c828c1a54ff3ff98d"}, + {file = "multidict-6.0.2-cp38-cp38-win32.whl", hash = "sha256:e5b20e9599ba74391ca0cfbd7b328fcc20976823ba19bc573983a25b32e92b57"}, + {file = "multidict-6.0.2-cp38-cp38-win_amd64.whl", hash = "sha256:8004dca28e15b86d1b1372515f32eb6f814bdf6f00952699bdeb541691091f96"}, + {file = "multidict-6.0.2-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:2e4a0785b84fb59e43c18a015ffc575ba93f7d1dbd272b4cdad9f5134b8a006c"}, + {file = "multidict-6.0.2-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:6701bf8a5d03a43375909ac91b6980aea74b0f5402fbe9428fc3f6edf5d9677e"}, + {file = "multidict-6.0.2-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:a007b1638e148c3cfb6bf0bdc4f82776cef0ac487191d093cdc316905e504071"}, + {file = "multidict-6.0.2-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:07a017cfa00c9890011628eab2503bee5872f27144936a52eaab449be5eaf032"}, + {file = "multidict-6.0.2-cp39-cp39-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:c207fff63adcdf5a485969131dc70e4b194327666b7e8a87a97fbc4fd80a53b2"}, + {file = "multidict-6.0.2-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:373ba9d1d061c76462d74e7de1c0c8e267e9791ee8cfefcf6b0b2495762c370c"}, + {file = "multidict-6.0.2-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:bfba7c6d5d7c9099ba21f84662b037a0ffd4a5e6b26ac07d19e423e6fdf965a9"}, + {file = "multidict-6.0.2-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:19d9bad105dfb34eb539c97b132057a4e709919ec4dd883ece5838bcbf262b80"}, + {file = "multidict-6.0.2-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:de989b195c3d636ba000ee4281cd03bb1234635b124bf4cd89eeee9ca8fcb09d"}, + {file = "multidict-6.0.2-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:7c40b7bbece294ae3a87c1bc2abff0ff9beef41d14188cda94ada7bcea99b0fb"}, + {file = "multidict-6.0.2-cp39-cp39-musllinux_1_1_ppc64le.whl", hash = "sha256:d16cce709ebfadc91278a1c005e3c17dd5f71f5098bfae1035149785ea6e9c68"}, + {file = "multidict-6.0.2-cp39-cp39-musllinux_1_1_s390x.whl", hash = "sha256:a2c34a93e1d2aa35fbf1485e5010337c72c6791407d03aa5f4eed920343dd360"}, + {file = "multidict-6.0.2-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:feba80698173761cddd814fa22e88b0661e98cb810f9f986c54aa34d281e4937"}, + {file = "multidict-6.0.2-cp39-cp39-win32.whl", hash = "sha256:23b616fdc3c74c9fe01d76ce0d1ce872d2d396d8fa8e4899398ad64fb5aa214a"}, + {file = "multidict-6.0.2-cp39-cp39-win_amd64.whl", hash = "sha256:4bae31803d708f6f15fd98be6a6ac0b6958fcf68fda3c77a048a4f9073704aae"}, + {file = "multidict-6.0.2.tar.gz", hash = "sha256:5ff3bd75f38e4c43f1f470f2df7a4d430b821c4ce22be384e1459cb57d6bb013"}, +] nodeenv = [ {file = "nodeenv-1.7.0-py2.py3-none-any.whl", hash = "sha256:27083a7b96a25f2f5e1d8cb4b6317ee8aeda3bdd121394e5ac54e498028a042e"}, {file = "nodeenv-1.7.0.tar.gz", hash = "sha256:e0e7f7dfb85fc5394c6fe1e8fa98131a2473e04311a45afb6508f7cf1836fa2b"}, ] -numpy = [] +numpy = [ + {file = "numpy-1.23.2-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:e603ca1fb47b913942f3e660a15e55a9ebca906857edfea476ae5f0fe9b457d5"}, + {file = "numpy-1.23.2-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:633679a472934b1c20a12ed0c9a6c9eb167fbb4cb89031939bfd03dd9dbc62b8"}, + {file = "numpy-1.23.2-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:17e5226674f6ea79e14e3b91bfbc153fdf3ac13f5cc54ee7bc8fdbe820a32da0"}, + {file = "numpy-1.23.2-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:bdc02c0235b261925102b1bd586579b7158e9d0d07ecb61148a1799214a4afd5"}, + {file = "numpy-1.23.2-cp310-cp310-win32.whl", hash = "sha256:df28dda02c9328e122661f399f7655cdcbcf22ea42daa3650a26bce08a187450"}, + {file = "numpy-1.23.2-cp310-cp310-win_amd64.whl", hash = "sha256:8ebf7e194b89bc66b78475bd3624d92980fca4e5bb86dda08d677d786fefc414"}, + {file = "numpy-1.23.2-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:dc76bca1ca98f4b122114435f83f1fcf3c0fe48e4e6f660e07996abf2f53903c"}, + {file = "numpy-1.23.2-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:ecfdd68d334a6b97472ed032b5b37a30d8217c097acfff15e8452c710e775524"}, + {file = "numpy-1.23.2-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:5593f67e66dea4e237f5af998d31a43e447786b2154ba1ad833676c788f37cde"}, + {file = "numpy-1.23.2-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:ac987b35df8c2a2eab495ee206658117e9ce867acf3ccb376a19e83070e69418"}, + {file = "numpy-1.23.2-cp311-cp311-win32.whl", hash = "sha256:d98addfd3c8728ee8b2c49126f3c44c703e2b005d4a95998e2167af176a9e722"}, + {file = "numpy-1.23.2-cp311-cp311-win_amd64.whl", hash = "sha256:8ecb818231afe5f0f568c81f12ce50f2b828ff2b27487520d85eb44c71313b9e"}, + {file = "numpy-1.23.2-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:909c56c4d4341ec8315291a105169d8aae732cfb4c250fbc375a1efb7a844f8f"}, + {file = "numpy-1.23.2-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:8247f01c4721479e482cc2f9f7d973f3f47810cbc8c65e38fd1bbd3141cc9842"}, + {file = "numpy-1.23.2-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:b8b97a8a87cadcd3f94659b4ef6ec056261fa1e1c3317f4193ac231d4df70215"}, + {file = "numpy-1.23.2-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:bd5b7ccae24e3d8501ee5563e82febc1771e73bd268eef82a1e8d2b4d556ae66"}, + {file = "numpy-1.23.2-cp38-cp38-win32.whl", hash = "sha256:9b83d48e464f393d46e8dd8171687394d39bc5abfe2978896b77dc2604e8635d"}, + {file = "numpy-1.23.2-cp38-cp38-win_amd64.whl", hash = "sha256:dec198619b7dbd6db58603cd256e092bcadef22a796f778bf87f8592b468441d"}, + {file = "numpy-1.23.2-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:4f41f5bf20d9a521f8cab3a34557cd77b6f205ab2116651f12959714494268b0"}, + {file = "numpy-1.23.2-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:806cc25d5c43e240db709875e947076b2826f47c2c340a5a2f36da5bb10c58d6"}, + {file = "numpy-1.23.2-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:8f9d84a24889ebb4c641a9b99e54adb8cab50972f0166a3abc14c3b93163f074"}, + {file = "numpy-1.23.2-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:c403c81bb8ffb1c993d0165a11493fd4bf1353d258f6997b3ee288b0a48fce77"}, + {file = "numpy-1.23.2-cp39-cp39-win32.whl", hash = "sha256:cf8c6aed12a935abf2e290860af8e77b26a042eb7f2582ff83dc7ed5f963340c"}, + {file = "numpy-1.23.2-cp39-cp39-win_amd64.whl", hash = "sha256:5e28cd64624dc2354a349152599e55308eb6ca95a13ce6a7d5679ebff2962913"}, + {file = "numpy-1.23.2-pp38-pypy38_pp73-macosx_10_9_x86_64.whl", hash = "sha256:806970e69106556d1dd200e26647e9bee5e2b3f1814f9da104a943e8d548ca38"}, + {file = "numpy-1.23.2-pp38-pypy38_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:2bd879d3ca4b6f39b7770829f73278b7c5e248c91d538aab1e506c628353e47f"}, + {file = "numpy-1.23.2-pp38-pypy38_pp73-win_amd64.whl", hash = "sha256:be6b350dfbc7f708d9d853663772a9310783ea58f6035eec649fb9c4371b5389"}, + {file = "numpy-1.23.2.tar.gz", hash = "sha256:b78d00e48261fbbd04aa0d7427cf78d18401ee0abd89c7559bbf422e5b1c7d01"}, +] packaging = [ {file = "packaging-21.3-py3-none-any.whl", hash = "sha256:ef103e05f519cdc783ae24ea4e2e0f508a9c99b2d4969652eed6a2e1ea5bd522"}, {file = "packaging-21.3.tar.gz", hash = "sha256:dd47c42927d89ab911e606518907cc2d3a1f38bbd026385970643f9c5b8ecfeb"}, ] -pep517 = [] +pep517 = [ + {file = "pep517-0.13.0-py3-none-any.whl", hash = "sha256:4ba4446d80aed5b5eac6509ade100bff3e7943a8489de249654a5ae9b33ee35b"}, + {file = "pep517-0.13.0.tar.gz", hash = "sha256:ae69927c5c172be1add9203726d4b84cf3ebad1edcd5f71fcdc746e66e829f59"}, +] platformdirs = [ {file = "platformdirs-2.5.2-py3-none-any.whl", hash = "sha256:027d8e83a2d7de06bbac4e5ef7e023c02b863d7ea5d079477e722bb41ab25788"}, {file = "platformdirs-2.5.2.tar.gz", hash = "sha256:58c8abb07dcb441e6ee4b11d8df0ac856038f944ab98b7be6b27b2a3c7feef19"}, @@ -908,12 +1247,42 @@ pluggy = [ {file = "pluggy-1.0.0-py2.py3-none-any.whl", hash = "sha256:74134bbf457f031a36d68416e1509f34bd5ccc019f0bcc952c7b909d06b37bd3"}, {file = "pluggy-1.0.0.tar.gz", hash = "sha256:4224373bacce55f955a878bf9cfa763c1e360858e330072059e10bad68531159"}, ] -pre-commit = [] +pre-commit = [ + {file = "pre_commit-2.20.0-py2.py3-none-any.whl", hash = "sha256:51a5ba7c480ae8072ecdb6933df22d2f812dc897d5fe848778116129a681aac7"}, + {file = "pre_commit-2.20.0.tar.gz", hash = "sha256:a978dac7bc9ec0bcee55c18a277d553b0f419d259dadb4b9418ff2d00eb43959"}, +] py = [ {file = "py-1.11.0-py2.py3-none-any.whl", hash = "sha256:607c53218732647dff4acdfcd50cb62615cedf612e72d1724fb1a0cc6405b378"}, {file = "py-1.11.0.tar.gz", hash = "sha256:51c75c4126074b472f746a24399ad32f6053d1b34b68d2fa41e558e6f4a98719"}, ] -pyarrow = [] +pyarrow = [ + {file = "pyarrow-9.0.0-cp310-cp310-macosx_10_13_universal2.whl", hash = "sha256:767cafb14278165ad539a2918c14c1b73cf20689747c21375c38e3fe62884902"}, + {file = "pyarrow-9.0.0-cp310-cp310-macosx_10_13_x86_64.whl", hash = "sha256:0238998dc692efcb4e41ae74738d7c1234723271ccf520bd8312dca07d49ef8d"}, + {file = "pyarrow-9.0.0-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:55328348b9139c2b47450d512d716c2248fd58e2f04e2fc23a65e18726666d42"}, + {file = "pyarrow-9.0.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:fc856628acd8d281652c15b6268ec7f27ebcb015abbe99d9baad17f02adc51f1"}, + {file = "pyarrow-9.0.0-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:29eb3e086e2b26202f3a4678316b93cfb15d0e2ba20f3ec12db8fd9cc07cde63"}, + {file = "pyarrow-9.0.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:2e753f8fcf07d8e3a0efa0c8bd51fef5c90281ffd4c5637c08ce42cd0ac297de"}, + {file = "pyarrow-9.0.0-cp310-cp310-win_amd64.whl", hash = "sha256:3eef8a981f45d89de403e81fb83b8119c20824caddf1404274e41a5d66c73806"}, + {file = "pyarrow-9.0.0-cp37-cp37m-macosx_10_13_x86_64.whl", hash = "sha256:7fa56cbd415cef912677270b8e41baad70cde04c6d8a8336eeb2aba85aa93706"}, + {file = "pyarrow-9.0.0-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:f8c46bde1030d704e2796182286d1c56846552c50a39ad5bf5a20c0d8159fc35"}, + {file = "pyarrow-9.0.0-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:8ad430cee28ebc4d6661fc7315747c7a18ae2a74e67498dcb039e1c762a2fb67"}, + {file = "pyarrow-9.0.0-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:81a60bb291a964f63b2717fb1b28f6615ffab7e8585322bfb8a6738e6b321282"}, + {file = "pyarrow-9.0.0-cp37-cp37m-win_amd64.whl", hash = "sha256:9cef618159567d5f62040f2b79b1c7b38e3885f4ffad0ec97cd2d86f88b67cef"}, + {file = "pyarrow-9.0.0-cp38-cp38-macosx_10_13_x86_64.whl", hash = "sha256:5526a3bfb404ff6d31d62ea582cf2466c7378a474a99ee04d1a9b05de5264541"}, + {file = "pyarrow-9.0.0-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:da3e0f319509a5881867effd7024099fb06950a0768dad0d6873668bb88cfaba"}, + {file = "pyarrow-9.0.0-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:2c715eca2092273dcccf6f08437371e04d112f9354245ba2fbe6c801879450b7"}, + {file = "pyarrow-9.0.0-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:f11a645a41ee531c3a5edda45dea07c42267f52571f818d388971d33fc7e2d4a"}, + {file = "pyarrow-9.0.0-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:a5b390bdcfb8c5b900ef543f911cdfec63e88524fafbcc15f83767202a4a2491"}, + {file = "pyarrow-9.0.0-cp38-cp38-win_amd64.whl", hash = "sha256:d9eb04db626fa24fdfb83c00f76679ca0d98728cdbaa0481b6402bf793a290c0"}, + {file = "pyarrow-9.0.0-cp39-cp39-macosx_10_13_universal2.whl", hash = "sha256:4eebdab05afa23d5d5274b24c1cbeb1ba017d67c280f7d39fd8a8f18cbad2ec9"}, + {file = "pyarrow-9.0.0-cp39-cp39-macosx_10_13_x86_64.whl", hash = "sha256:02b820ecd1da02012092c180447de449fc688d0c3f9ff8526ca301cdd60dacd0"}, + {file = "pyarrow-9.0.0-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:92f3977e901db1ef5cba30d6cc1d7942b8d94b910c60f89013e8f7bb86a86eef"}, + {file = "pyarrow-9.0.0-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:f241bd488c2705df930eedfe304ada71191dcf67d6b98ceda0cc934fd2a8388e"}, + {file = "pyarrow-9.0.0-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:1c5a073a930c632058461547e0bc572da1e724b17b6b9eb31a97da13f50cb6e0"}, + {file = "pyarrow-9.0.0-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:f59bcd5217a3ae1e17870792f82b2ff92df9f3862996e2c78e156c13e56ff62e"}, + {file = "pyarrow-9.0.0-cp39-cp39-win_amd64.whl", hash = "sha256:fe2ce795fa1d95e4e940fe5661c3c58aee7181c730f65ac5dd8794a77228de59"}, + {file = "pyarrow-9.0.0.tar.gz", hash = "sha256:7fb02bebc13ab55573d1ae9bb5002a6d20ba767bf8569b52fce5301d42495ab7"}, +] pycparser = [ {file = "pycparser-2.21-py2.py3-none-any.whl", hash = "sha256:8ee45429555515e1f6b185e78100aea234072576aa43ab53aefcae078162fca9"}, {file = "pycparser-2.21.tar.gz", hash = "sha256:e644fdec12f7872f86c58ff790da456218b10f863970249516d60a5eaca77206"}, @@ -956,20 +1325,26 @@ pydantic = [ {file = "pydantic-1.10.1-py3-none-any.whl", hash = "sha256:f8b10e59c035ff3dcc9791619d6e6c5141e0fa5cbe264e19e267b8d523b210bf"}, {file = "pydantic-1.10.1.tar.gz", hash = "sha256:d41bb80347a8a2d51fbd6f1748b42aca14541315878447ba159617544712f770"}, ] -pygments = [] +pygments = [ + {file = "Pygments-2.13.0-py3-none-any.whl", hash = "sha256:f643f331ab57ba3c9d89212ee4a2dabc6e94f117cf4eefde99a0574720d14c42"}, + {file = "Pygments-2.13.0.tar.gz", hash = "sha256:56a8508ae95f98e2b9bdf93a6be5ae3f7d8af858b43e02c5a2ff083726be40c1"}, +] pyparsing = [ {file = "pyparsing-3.0.9-py3-none-any.whl", hash = "sha256:5026bae9a10eeaefb61dab2f09052b9f4307d44aee4eda64b309723d8d206bbc"}, {file = "pyparsing-3.0.9.tar.gz", hash = "sha256:2b020ecf7d21b687f219b71ecad3631f644a47f01403fa1d1036b0c6416d70fb"}, ] pytest = [ - {file = "pytest-7.1.2-py3-none-any.whl", hash = "sha256:13d0e3ccfc2b6e26be000cb6568c832ba67ba32e719443bfe725814d3c42433c"}, - {file = "pytest-7.1.2.tar.gz", hash = "sha256:a06a0425453864a270bc45e71f783330a7428defb4230fb5e6a731fde06ecd45"}, + {file = "pytest-7.1.3-py3-none-any.whl", hash = "sha256:1377bda3466d70b55e3f5cecfa55bb7cfcf219c7964629b967c37cf0bda818b7"}, + {file = "pytest-7.1.3.tar.gz", hash = "sha256:4f365fec2dff9c1162f834d9f18af1ba13062db0c708bf7b946f8a5c76180c39"}, ] pytest-checkdocs = [ {file = "pytest-checkdocs-2.7.1.tar.gz", hash = "sha256:2b33b85eddfe5846a69bea4a759303e2d5a3be11d03bc7149f5ba1ef47e6c1ae"}, {file = "pytest_checkdocs-2.7.1-py3-none-any.whl", hash = "sha256:294898c64c9ce1a178edc6660e48da23c7543bfd5a1cea7f0ca4c167745d8461"}, ] -python-dateutil = [] +python-dateutil = [ + {file = "python-dateutil-2.8.2.tar.gz", hash = "sha256:0123cacc1627ae19ddf3c27a5de5bd67ee4586fbdd6440d9748f8abb483d3e86"}, + {file = "python_dateutil-2.8.2-py2.py3-none-any.whl", hash = "sha256:961d03dc3453ebbc59dbdea9e4e11c5651520a876d0f4db161e8674aae935da9"}, +] python-snappy = [ {file = "python-snappy-0.6.1.tar.gz", hash = "sha256:b6a107ab06206acc5359d4c5632bd9b22d448702a79b3169b0c62e0fb808bb2a"}, {file = "python_snappy-0.6.1-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:b7f920eaf46ebf41bd26f9df51c160d40f9e00b7b48471c3438cb8d027f7fb9b"}, @@ -1063,13 +1438,25 @@ requests-mock = [ {file = "requests-mock-1.10.0.tar.gz", hash = "sha256:59c9c32419a9fb1ae83ec242d98e889c45bd7d7a65d48375cc243ec08441658b"}, {file = "requests_mock-1.10.0-py2.py3-none-any.whl", hash = "sha256:2fdbb637ad17ee15c06f33d31169e71bf9fe2bdb7bc9da26185be0dd8d842699"}, ] -rich = [] -s3fs = [] +rich = [ + {file = "rich-12.5.1-py3-none-any.whl", hash = "sha256:2eb4e6894cde1e017976d2975ac210ef515d7548bc595ba20e195fb9628acdeb"}, + {file = "rich-12.5.1.tar.gz", hash = "sha256:63a5c5ce3673d3d5fbbf23cd87e11ab84b6b451436f1b7f19ec54b6bc36ed7ca"}, +] +s3fs = [ + {file = "s3fs-2022.5.0-py3-none-any.whl", hash = "sha256:c7bb327303e931cf641f3cac9b6412a8f1e2908cebd3de30900cb872a4a9f5d2"}, + {file = "s3fs-2022.5.0.tar.gz", hash = "sha256:b40a3cbfaf80cbabaf0e332331117ccac69290efdac347184fb3ab6003f70465"}, +] +setuptools = [ + {file = "setuptools-65.3.0-py3-none-any.whl", hash = "sha256:2e24e0bec025f035a2e72cdd1961119f557d78ad331bb00ff82efb2ab8da8e82"}, + {file = "setuptools-65.3.0.tar.gz", hash = "sha256:7732871f4f7fa58fb6bdcaeadb0161b2bd046c85905dbaa066bdcbcc81953b57"}, +] six = [ {file = "six-1.16.0-py2.py3-none-any.whl", hash = "sha256:8abb2f1d86890a2dfb989f9a77cfcfd3e47c2a354b01111771326f8aa26e0254"}, {file = "six-1.16.0.tar.gz", hash = "sha256:1e61c37477a1626458e36f7b1d82aa5c9b094fa4802892072e49de9c60c4c926"}, ] -thrift = [] +thrift = [ + {file = "thrift-0.16.0.tar.gz", hash = "sha256:2b5b6488fcded21f9d312aa23c9ff6a0195d0f6ae26ddbd5ad9e3e25dfc14408"}, +] toml = [ {file = "toml-0.10.2-py2.py3-none-any.whl", hash = "sha256:806143ae5bfb6a3c6e736a764057db0e6a0e05e338b5630894a5f779cabb4f9b"}, {file = "toml-0.10.2.tar.gz", hash = "sha256:b3bda1d108d5dd99f4a20d24d9c348e91c4db7ab1b749200bded2f839ccbe68f"}, @@ -1082,7 +1469,10 @@ typing-extensions = [ {file = "typing_extensions-4.3.0-py3-none-any.whl", hash = "sha256:25642c956049920a5aa49edcdd6ab1e06d7e5d467fc00e0506c44ac86fbfca02"}, {file = "typing_extensions-4.3.0.tar.gz", hash = "sha256:e6d2677a32f47fc7eb2795db1dd15c1f34eff616bcaf2cfb5e997f854fa1c4a6"}, ] -urllib3 = [] +urllib3 = [ + {file = "urllib3-1.26.12-py2.py3-none-any.whl", hash = "sha256:b930dd878d5a8afb066a637fbb35144fe7901e3b209d1cd4f524bd0e9deee997"}, + {file = "urllib3-1.26.12.tar.gz", hash = "sha256:3fa96cf423e6987997fc326ae8df396db2a8b7c667747d47ddd8ecba91f4a74e"}, +] virtualenv = [ {file = "virtualenv-20.16.4-py3-none-any.whl", hash = "sha256:035ed57acce4ac35c82c9d8802202b0e71adac011a511ff650cbcf9635006a22"}, {file = "virtualenv-20.16.4.tar.gz", hash = "sha256:014f766e4134d0008dcaa1f95bafa0fb0f575795d07cae50b1bee514185d6782"}, @@ -1153,6 +1543,114 @@ wrapt = [ {file = "wrapt-1.14.1-cp39-cp39-win_amd64.whl", hash = "sha256:dee60e1de1898bde3b238f18340eec6148986da0455d8ba7848d50470a7a32fb"}, {file = "wrapt-1.14.1.tar.gz", hash = "sha256:380a85cf89e0e69b7cfbe2ea9f765f004ff419f34194018a6827ac0e3edfed4d"}, ] -yarl = [] -zipp = [] -zstandard = [] +yarl = [ + {file = "yarl-1.8.1-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:abc06b97407868ef38f3d172762f4069323de52f2b70d133d096a48d72215d28"}, + {file = "yarl-1.8.1-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:07b21e274de4c637f3e3b7104694e53260b5fc10d51fb3ec5fed1da8e0f754e3"}, + {file = "yarl-1.8.1-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:9de955d98e02fab288c7718662afb33aab64212ecb368c5dc866d9a57bf48880"}, + {file = "yarl-1.8.1-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:7ec362167e2c9fd178f82f252b6d97669d7245695dc057ee182118042026da40"}, + {file = "yarl-1.8.1-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:20df6ff4089bc86e4a66e3b1380460f864df3dd9dccaf88d6b3385d24405893b"}, + {file = "yarl-1.8.1-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:5999c4662631cb798496535afbd837a102859568adc67d75d2045e31ec3ac497"}, + {file = "yarl-1.8.1-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:ed19b74e81b10b592084a5ad1e70f845f0aacb57577018d31de064e71ffa267a"}, + {file = "yarl-1.8.1-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:1e4808f996ca39a6463f45182e2af2fae55e2560be586d447ce8016f389f626f"}, + {file = "yarl-1.8.1-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:2d800b9c2eaf0684c08be5f50e52bfa2aa920e7163c2ea43f4f431e829b4f0fd"}, + {file = "yarl-1.8.1-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:6628d750041550c5d9da50bb40b5cf28a2e63b9388bac10fedd4f19236ef4957"}, + {file = "yarl-1.8.1-cp310-cp310-musllinux_1_1_ppc64le.whl", hash = "sha256:f5af52738e225fcc526ae64071b7e5342abe03f42e0e8918227b38c9aa711e28"}, + {file = "yarl-1.8.1-cp310-cp310-musllinux_1_1_s390x.whl", hash = "sha256:76577f13333b4fe345c3704811ac7509b31499132ff0181f25ee26619de2c843"}, + {file = "yarl-1.8.1-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:0c03f456522d1ec815893d85fccb5def01ffaa74c1b16ff30f8aaa03eb21e453"}, + {file = "yarl-1.8.1-cp310-cp310-win32.whl", hash = "sha256:ea30a42dc94d42f2ba4d0f7c0ffb4f4f9baa1b23045910c0c32df9c9902cb272"}, + {file = "yarl-1.8.1-cp310-cp310-win_amd64.whl", hash = "sha256:9130ddf1ae9978abe63808b6b60a897e41fccb834408cde79522feb37fb72fb0"}, + {file = "yarl-1.8.1-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:0ab5a138211c1c366404d912824bdcf5545ccba5b3ff52c42c4af4cbdc2c5035"}, + {file = "yarl-1.8.1-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a0fb2cb4204ddb456a8e32381f9a90000429489a25f64e817e6ff94879d432fc"}, + {file = "yarl-1.8.1-cp37-cp37m-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:85cba594433915d5c9a0d14b24cfba0339f57a2fff203a5d4fd070e593307d0b"}, + {file = "yarl-1.8.1-cp37-cp37m-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:1ca7e596c55bd675432b11320b4eacc62310c2145d6801a1f8e9ad160685a231"}, + {file = "yarl-1.8.1-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:d0f77539733e0ec2475ddcd4e26777d08996f8cd55d2aef82ec4d3896687abda"}, + {file = "yarl-1.8.1-cp37-cp37m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:29e256649f42771829974e742061c3501cc50cf16e63f91ed8d1bf98242e5507"}, + {file = "yarl-1.8.1-cp37-cp37m-musllinux_1_1_aarch64.whl", hash = "sha256:7fce6cbc6c170ede0221cc8c91b285f7f3c8b9fe28283b51885ff621bbe0f8ee"}, + {file = "yarl-1.8.1-cp37-cp37m-musllinux_1_1_i686.whl", hash = "sha256:59ddd85a1214862ce7c7c66457f05543b6a275b70a65de366030d56159a979f0"}, + {file = "yarl-1.8.1-cp37-cp37m-musllinux_1_1_ppc64le.whl", hash = "sha256:12768232751689c1a89b0376a96a32bc7633c08da45ad985d0c49ede691f5c0d"}, + {file = "yarl-1.8.1-cp37-cp37m-musllinux_1_1_s390x.whl", hash = "sha256:b19255dde4b4f4c32e012038f2c169bb72e7f081552bea4641cab4d88bc409dd"}, + {file = "yarl-1.8.1-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:6c8148e0b52bf9535c40c48faebb00cb294ee577ca069d21bd5c48d302a83780"}, + {file = "yarl-1.8.1-cp37-cp37m-win32.whl", hash = "sha256:de839c3a1826a909fdbfe05f6fe2167c4ab033f1133757b5936efe2f84904c07"}, + {file = "yarl-1.8.1-cp37-cp37m-win_amd64.whl", hash = "sha256:dd032e8422a52e5a4860e062eb84ac94ea08861d334a4bcaf142a63ce8ad4802"}, + {file = "yarl-1.8.1-cp38-cp38-macosx_10_9_universal2.whl", hash = "sha256:19cd801d6f983918a3f3a39f3a45b553c015c5aac92ccd1fac619bd74beece4a"}, + {file = "yarl-1.8.1-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:6347f1a58e658b97b0a0d1ff7658a03cb79bdbda0331603bed24dd7054a6dea1"}, + {file = "yarl-1.8.1-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:7c0da7e44d0c9108d8b98469338705e07f4bb7dab96dbd8fa4e91b337db42548"}, + {file = "yarl-1.8.1-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:5587bba41399854703212b87071c6d8638fa6e61656385875f8c6dff92b2e461"}, + {file = "yarl-1.8.1-cp38-cp38-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:31a9a04ecccd6b03e2b0e12e82131f1488dea5555a13a4d32f064e22a6003cfe"}, + {file = "yarl-1.8.1-cp38-cp38-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:205904cffd69ae972a1707a1bd3ea7cded594b1d773a0ce66714edf17833cdae"}, + {file = "yarl-1.8.1-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:ea513a25976d21733bff523e0ca836ef1679630ef4ad22d46987d04b372d57fc"}, + {file = "yarl-1.8.1-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:d0b51530877d3ad7a8d47b2fff0c8df3b8f3b8deddf057379ba50b13df2a5eae"}, + {file = "yarl-1.8.1-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:d2b8f245dad9e331540c350285910b20dd913dc86d4ee410c11d48523c4fd546"}, + {file = "yarl-1.8.1-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:ab2a60d57ca88e1d4ca34a10e9fb4ab2ac5ad315543351de3a612bbb0560bead"}, + {file = "yarl-1.8.1-cp38-cp38-musllinux_1_1_ppc64le.whl", hash = "sha256:449c957ffc6bc2309e1fbe67ab7d2c1efca89d3f4912baeb8ead207bb3cc1cd4"}, + {file = "yarl-1.8.1-cp38-cp38-musllinux_1_1_s390x.whl", hash = "sha256:a165442348c211b5dea67c0206fc61366212d7082ba8118c8c5c1c853ea4d82e"}, + {file = "yarl-1.8.1-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:b3ded839a5c5608eec8b6f9ae9a62cb22cd037ea97c627f38ae0841a48f09eae"}, + {file = "yarl-1.8.1-cp38-cp38-win32.whl", hash = "sha256:c1445a0c562ed561d06d8cbc5c8916c6008a31c60bc3655cdd2de1d3bf5174a0"}, + {file = "yarl-1.8.1-cp38-cp38-win_amd64.whl", hash = "sha256:56c11efb0a89700987d05597b08a1efcd78d74c52febe530126785e1b1a285f4"}, + {file = "yarl-1.8.1-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:e80ed5a9939ceb6fda42811542f31c8602be336b1fb977bccb012e83da7e4936"}, + {file = "yarl-1.8.1-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:6afb336e23a793cd3b6476c30f030a0d4c7539cd81649683b5e0c1b0ab0bf350"}, + {file = "yarl-1.8.1-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:4c322cbaa4ed78a8aac89b2174a6df398faf50e5fc12c4c191c40c59d5e28357"}, + {file = "yarl-1.8.1-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:fae37373155f5ef9b403ab48af5136ae9851151f7aacd9926251ab26b953118b"}, + {file = "yarl-1.8.1-cp39-cp39-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:5395da939ffa959974577eff2cbfc24b004a2fb6c346918f39966a5786874e54"}, + {file = "yarl-1.8.1-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:076eede537ab978b605f41db79a56cad2e7efeea2aa6e0fa8f05a26c24a034fb"}, + {file = "yarl-1.8.1-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:3d1a50e461615747dd93c099f297c1994d472b0f4d2db8a64e55b1edf704ec1c"}, + {file = "yarl-1.8.1-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:7de89c8456525650ffa2bb56a3eee6af891e98f498babd43ae307bd42dca98f6"}, + {file = "yarl-1.8.1-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:4a88510731cd8d4befaba5fbd734a7dd914de5ab8132a5b3dde0bbd6c9476c64"}, + {file = "yarl-1.8.1-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:2d93a049d29df172f48bcb09acf9226318e712ce67374f893b460b42cc1380ae"}, + {file = "yarl-1.8.1-cp39-cp39-musllinux_1_1_ppc64le.whl", hash = "sha256:21ac44b763e0eec15746a3d440f5e09ad2ecc8b5f6dcd3ea8cb4773d6d4703e3"}, + {file = "yarl-1.8.1-cp39-cp39-musllinux_1_1_s390x.whl", hash = "sha256:d0272228fabe78ce00a3365ffffd6f643f57a91043e119c289aaba202f4095b0"}, + {file = "yarl-1.8.1-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:99449cd5366fe4608e7226c6cae80873296dfa0cde45d9b498fefa1de315a09e"}, + {file = "yarl-1.8.1-cp39-cp39-win32.whl", hash = "sha256:8b0af1cf36b93cee99a31a545fe91d08223e64390c5ecc5e94c39511832a4bb6"}, + {file = "yarl-1.8.1-cp39-cp39-win_amd64.whl", hash = "sha256:de49d77e968de6626ba7ef4472323f9d2e5a56c1d85b7c0e2a190b2173d3b9be"}, + {file = "yarl-1.8.1.tar.gz", hash = "sha256:af887845b8c2e060eb5605ff72b6f2dd2aab7a761379373fd89d314f4752abbf"}, +] +zipp = [ + {file = "zipp-3.8.1-py3-none-any.whl", hash = "sha256:47c40d7fe183a6f21403a199b3e4192cca5774656965b0a4988ad2f8feb5f009"}, + {file = "zipp-3.8.1.tar.gz", hash = "sha256:05b45f1ee8f807d0cc928485ca40a07cb491cf092ff587c0df9cb1fd154848d2"}, +] +zstandard = [ + {file = "zstandard-0.18.0-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:ef7e8a200e4c8ac9102ed3c90ed2aa379f6b880f63032200909c1be21951f556"}, + {file = "zstandard-0.18.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:2dc466207016564805e56d28375f4f533b525ff50d6776946980dff5465566ac"}, + {file = "zstandard-0.18.0-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:4a2ee1d4f98447f3e5183ecfce5626f983504a4a0c005fbe92e60fa8e5d547ec"}, + {file = "zstandard-0.18.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:d956e2f03c7200d7e61345e0880c292783ec26618d0d921dcad470cb195bbce2"}, + {file = "zstandard-0.18.0-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_12_i686.manylinux2010_i686.whl", hash = "sha256:ce6f59cba9854fd14da5bfe34217a1501143057313966637b7291d1b0267bd1e"}, + {file = "zstandard-0.18.0-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:a7fa67cba473623848b6e88acf8d799b1906178fd883fb3a1da24561c779593b"}, + {file = "zstandard-0.18.0-cp310-cp310-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:cdb44d7284c8c5dd1b66dfb86dda7f4560fa94bfbbc1d2da749ba44831335e32"}, + {file = "zstandard-0.18.0-cp310-cp310-win32.whl", hash = "sha256:63694a376cde0aa8b1971d06ca28e8f8b5f492779cb6ee1cc46bbc3f019a42a5"}, + {file = "zstandard-0.18.0-cp310-cp310-win_amd64.whl", hash = "sha256:702a8324cd90c74d9c8780d02bf55e79da3193c870c9665ad3a11647e3ad1435"}, + {file = "zstandard-0.18.0-cp36-cp36m-macosx_10_9_x86_64.whl", hash = "sha256:46f679bc5dfd938db4fb058218d9dc4db1336ffaf1ea774ff152ecadabd40805"}, + {file = "zstandard-0.18.0-cp36-cp36m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:dc2a4de9f363b3247d472362a65041fe4c0f59e01a2846b15d13046be866a885"}, + {file = "zstandard-0.18.0-cp36-cp36m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:bd3220d7627fd4d26397211cb3b560ec7cc4a94b75cfce89e847e8ce7fabe32d"}, + {file = "zstandard-0.18.0-cp36-cp36m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_12_i686.manylinux2010_i686.whl", hash = "sha256:39e98cf4773234bd9cebf9f9db730e451dfcfe435e220f8921242afda8321887"}, + {file = "zstandard-0.18.0-cp36-cp36m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:5228e596eb1554598c872a337bbe4e5afe41cd1f8b1b15f2e35b50d061e35244"}, + {file = "zstandard-0.18.0-cp36-cp36m-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:d4a8fd45746a6c31e729f35196e80b8f1e9987c59f5ccb8859d7c6a6fbeb9c63"}, + {file = "zstandard-0.18.0-cp36-cp36m-win32.whl", hash = "sha256:4cbb85f29a990c2fdbf7bc63246567061a362ddca886d7fae6f780267c0a9e67"}, + {file = "zstandard-0.18.0-cp36-cp36m-win_amd64.whl", hash = "sha256:bfa6c8549fa18e6497a738b7033c49f94a8e2e30c5fbe2d14d0b5aa8bbc1695d"}, + {file = "zstandard-0.18.0-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:e02043297c1832f2666cd2204f381bef43b10d56929e13c42c10c732c6e3b4ed"}, + {file = "zstandard-0.18.0-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:7231543d38d2b7e02ef7cc78ef7ffd86419437e1114ff08709fe25a160e24bd6"}, + {file = "zstandard-0.18.0-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:c86befac87445927488f5c8f205d11566f64c11519db223e9d282b945fa60dab"}, + {file = "zstandard-0.18.0-cp37-cp37m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_12_i686.manylinux2010_i686.whl", hash = "sha256:999a4e1768f219826ba3fa2064fab1c86dd72fdd47a42536235478c3bb3ca3e2"}, + {file = "zstandard-0.18.0-cp37-cp37m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:9df59cd1cf3c62075ee2a4da767089d19d874ac3ad42b04a71a167e91b384722"}, + {file = "zstandard-0.18.0-cp37-cp37m-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:1be31e9e3f7607ee0cdd60915410a5968b205d3e7aa83b7fcf3dd76dbbdb39e0"}, + {file = "zstandard-0.18.0-cp37-cp37m-win32.whl", hash = "sha256:490d11b705b8ae9dc845431bacc8dd1cef2408aede176620a5cd0cd411027936"}, + {file = "zstandard-0.18.0-cp37-cp37m-win_amd64.whl", hash = "sha256:266aba27fa9cc5e9091d3d325ebab1fa260f64e83e42516d5e73947c70216a5b"}, + {file = "zstandard-0.18.0-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:8b2260c4e07dd0723eadb586de7718b61acca4083a490dda69c5719d79bc715c"}, + {file = "zstandard-0.18.0-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:3af8c2383d02feb6650e9255491ec7d0824f6e6dd2bbe3e521c469c985f31fb1"}, + {file = "zstandard-0.18.0-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:28723a1d2e4df778573b76b321ebe9f3469ac98988104c2af116dd344802c3f8"}, + {file = "zstandard-0.18.0-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:19cac7108ff2c342317fad6dc97604b47a41f403c8f19d0bfc396dfadc3638b8"}, + {file = "zstandard-0.18.0-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_12_i686.manylinux2010_i686.whl", hash = "sha256:76725d1ee83a8915100a310bbad5d9c1fc6397410259c94033b8318d548d9990"}, + {file = "zstandard-0.18.0-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:d716a7694ce1fa60b20bc10f35c4a22be446ef7f514c8dbc8f858b61976de2fb"}, + {file = "zstandard-0.18.0-cp38-cp38-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:49685bf9a55d1ab34bd8423ea22db836ba43a181ac6b045ac4272093d5cb874e"}, + {file = "zstandard-0.18.0-cp38-cp38-win32.whl", hash = "sha256:1af1268a7dc870eb27515fb8db1f3e6c5a555d2b7bcc476fc3bab8886c7265ab"}, + {file = "zstandard-0.18.0-cp38-cp38-win_amd64.whl", hash = "sha256:1dc2d3809e763055a1a6c1a73f2b677320cc9a5aa1a7c6cfb35aee59bddc42d9"}, + {file = "zstandard-0.18.0-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:eea18c1e7442f2aa9aff1bb84550dbb6a1f711faf6e48e7319de8f2b2e923c2a"}, + {file = "zstandard-0.18.0-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:8677ffc6a6096cccbd892e558471c901fd821aba12b7fbc63833c7346f549224"}, + {file = "zstandard-0.18.0-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:083dc08abf03807af9beeb2b6a91c23ad78add2499f828176a3c7b742c44df02"}, + {file = "zstandard-0.18.0-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:c990063664c08169c84474acecc9251ee035871589025cac47c060ff4ec4bc1a"}, + {file = "zstandard-0.18.0-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_12_i686.manylinux2010_i686.whl", hash = "sha256:533db8a6fac6248b2cb2c935e7b92f994efbdeb72e1ffa0b354432e087bb5a3e"}, + {file = "zstandard-0.18.0-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:dbb3cb8a082d62b8a73af42291569d266b05605e017a3d8a06a0e5c30b5f10f0"}, + {file = "zstandard-0.18.0-cp39-cp39-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:d6c85ca5162049ede475b7ec98e87f9390501d44a3d6776ddd504e872464ec25"}, + {file = "zstandard-0.18.0-cp39-cp39-win32.whl", hash = "sha256:75479e7c2b3eebf402c59fbe57d21bc400cefa145ca356ee053b0a08908c5784"}, + {file = "zstandard-0.18.0-cp39-cp39-win_amd64.whl", hash = "sha256:d85bfabad444812133a92fc6fbe463e1d07581dba72f041f07a360e63808b23c"}, + {file = "zstandard-0.18.0.tar.gz", hash = "sha256:0ac0357a0d985b4ff31a854744040d7b5754385d1f98f7145c30e02c6865cb6f"}, +] diff --git a/pyproject.toml b/pyproject.toml index 8039547e13..7be9f64afb 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -62,7 +62,7 @@ thrift = { version = "^0.16.0", optional = true } s3fs = { version = "2022.5.0", optional = true} [tool.poetry.dev-dependencies] -pytest = "^7.0.0" +pytest = "^7.1.3" pytest-checkdocs = "^2.0.0" pre-commit = "^2.0.0" fastavro = "^1.5.4" From cd80a8f946967026f65648725cd82e82cc37887e Mon Sep 17 00:00:00 2001 From: Samuel Redai <43911210+samredai@users.noreply.github.com> Date: Mon, 5 Sep 2022 12:37:21 -0400 Subject: [PATCH 193/642] Python: Remove fs_properties pass-through (#5689) --- pyiceberg/io/fsspec.py | 10 +++------- 1 file changed, 3 insertions(+), 7 deletions(-) diff --git a/pyiceberg/io/fsspec.py b/pyiceberg/io/fsspec.py index 769c309a84..d9bb21b64e 100644 --- a/pyiceberg/io/fsspec.py +++ b/pyiceberg/io/fsspec.py @@ -26,7 +26,7 @@ from pyiceberg.typedef import Properties -def _s3(properties: Properties, **fs_properties) -> AbstractFileSystem: +def _s3(properties: Properties) -> AbstractFileSystem: from s3fs import S3FileSystem client_kwargs = { @@ -37,7 +37,7 @@ def _s3(properties: Properties, **fs_properties) -> AbstractFileSystem: config_kwargs = {"signature_version": properties.get("s3.signer")} - return S3FileSystem(client_kwargs=client_kwargs, config_kwargs=config_kwargs, **fs_properties) + return S3FileSystem(client_kwargs=client_kwargs, config_kwargs=config_kwargs) SCHEME_TO_FS = { @@ -189,8 +189,4 @@ def _get_fs(self, scheme: str) -> AbstractFileSystem: """Get a filesystem for a specific scheme""" if scheme not in self._scheme_to_fs: raise ValueError(f"No registered filesystem for scheme: {scheme}") - return self._scheme_to_fs[scheme](self.properties, **self._fs_properties()) - - def _fs_properties(self): - """Get fs properties from the file-io property map""" - return {k[3:]: v for k, v in self.properties.items() if k.startswith("fs_")} + return self._scheme_to_fs[scheme](self.properties) From 96f5bb72384baf053c7ef3b8e443af9c810436e7 Mon Sep 17 00:00:00 2001 From: Fokko Driesprong Date: Tue, 6 Sep 2022 16:00:28 +0200 Subject: [PATCH 194/642] Python: Pass through the location for the FileIO (#5709) This way we can determine to load the correct FileIO --- pyiceberg/catalog/hive.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyiceberg/catalog/hive.py b/pyiceberg/catalog/hive.py index 2ffbe66a99..1edba2d76e 100644 --- a/pyiceberg/catalog/hive.py +++ b/pyiceberg/catalog/hive.py @@ -352,7 +352,7 @@ def load_table(self, identifier: Union[str, Identifier]) -> Table: except NoSuchObjectException as e: raise NoSuchTableError(f"Table does not exists: {table_name}") from e - io = load_file_io({**self.properties, **hive_table.parameters}) + io = load_file_io({**self.properties, **hive_table.parameters}, hive_table.sd.location) return self._convert_hive_into_iceberg(hive_table, io) def drop_table(self, identifier: Union[str, Identifier]) -> None: From ddc6f1ca411eeca615fec6b54df2bf81b77ce105 Mon Sep 17 00:00:00 2001 From: Joshua Robinson <3334099+joshuarobinson@users.noreply.github.com> Date: Wed, 7 Sep 2022 12:27:03 +0200 Subject: [PATCH 195/642] Python:Fix FileIO fallback to pyarrow when s3fs not present. (#5717) --- pyiceberg/io/__init__.py | 4 ++-- pyiceberg/io/fsspec.py | 3 +-- 2 files changed, 3 insertions(+), 4 deletions(-) diff --git a/pyiceberg/io/__init__.py b/pyiceberg/io/__init__.py index d40d486b06..16955c2b23 100644 --- a/pyiceberg/io/__init__.py +++ b/pyiceberg/io/__init__.py @@ -262,8 +262,8 @@ def _import_file_io(io_impl: str, properties: Properties) -> Optional[FileIO]: module = importlib.import_module(module_name) class_ = getattr(module, class_name) return class_(properties) - except ImportError: - logger.exception("Could not initialize FileIO: %s", io_impl) + except ModuleNotFoundError: + logger.warning("Could not initialize FileIO: %s", io_impl) return None diff --git a/pyiceberg/io/fsspec.py b/pyiceberg/io/fsspec.py index d9bb21b64e..20d972b967 100644 --- a/pyiceberg/io/fsspec.py +++ b/pyiceberg/io/fsspec.py @@ -21,14 +21,13 @@ from urllib.parse import urlparse from fsspec import AbstractFileSystem +from s3fs import S3FileSystem from pyiceberg.io import FileIO, InputFile, OutputFile from pyiceberg.typedef import Properties def _s3(properties: Properties) -> AbstractFileSystem: - from s3fs import S3FileSystem - client_kwargs = { "endpoint_url": properties.get("s3.endpoint"), "aws_access_key_id": properties.get("s3.access-key-id"), From f16a015abb9adb397439ee9ac5f9f6dd5444a234 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 13 Sep 2022 09:00:46 -0700 Subject: [PATCH 196/642] Build: Bump fastavro from 1.6.0 to 1.6.1 in /python (#5745) Bumps [fastavro](https://github.com/fastavro/fastavro) from 1.6.0 to 1.6.1. - [Release notes](https://github.com/fastavro/fastavro/releases) - [Changelog](https://github.com/fastavro/fastavro/blob/master/ChangeLog) - [Commits](https://github.com/fastavro/fastavro/compare/1.6.0...1.6.1) --- updated-dependencies: - dependency-name: fastavro dependency-type: direct:development update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- poetry.lock | 38 +++++++++++++++++++------------------- pyproject.toml | 2 +- 2 files changed, 20 insertions(+), 20 deletions(-) diff --git a/poetry.lock b/poetry.lock index f57261c441..dd33353496 100644 --- a/poetry.lock +++ b/poetry.lock @@ -196,7 +196,7 @@ python-versions = ">=3.7" [[package]] name = "fastavro" -version = "1.6.0" +version = "1.6.1" description = "Fast read/write of AVRO files" category = "dev" optional = false @@ -749,7 +749,7 @@ snappy = ["python-snappy"] [metadata] lock-version = "1.1" python-versions = "^3.8" -content-hash = "8aadbca4c1e2def4875c5acc9fba8e9bc049e86d734d6301d2a9451e537eb21d" +content-hash = "450282a4da16945da7dfc2a3c44b5e2343fd11f9d440826a4fd8af8b94ff7531" [metadata.files] aiobotocore = [ @@ -1001,23 +1001,23 @@ docutils = [ {file = "docutils-0.19.tar.gz", hash = "sha256:33995a6753c30b7f577febfc2c50411fec6aac7f7ffeb7c4cfe5991072dcf9e6"}, ] fastavro = [ - {file = "fastavro-1.6.0-cp310-cp310-macosx_10_14_x86_64.whl", hash = "sha256:cbaa7e5db05694c70985c7ddc8afd9f7c9aef3e66e78cb18d05c70db221eaa1a"}, - {file = "fastavro-1.6.0-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:8d18d3540aab5f34bb88ad58d2f45b2c64f10bd04ad126bb79024a1e07d6db2b"}, - {file = "fastavro-1.6.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:d1271f11304951d4b61415480ec53c2d44c297eb00fd2d95098171805a6350d7"}, - {file = "fastavro-1.6.0-cp310-cp310-win_amd64.whl", hash = "sha256:a80eec11dfc879299f4e20e6c5b19307dc50ec20a6f431fcfe2f29c5ae63c575"}, - {file = "fastavro-1.6.0-cp37-cp37m-macosx_10_15_x86_64.whl", hash = "sha256:77354c17dc36cd21a3127004fd83854e799899b26f42fbe4a8bce3424c493551"}, - {file = "fastavro-1.6.0-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:209afeb9dcdf624b10f223c17d3469fefd9669bdaceaf5a3fb47cb74a107e443"}, - {file = "fastavro-1.6.0-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:b78ce36f45cd582c8d8b5c8ade764b378d0f3581c1e62fba895945984dec107a"}, - {file = "fastavro-1.6.0-cp37-cp37m-win_amd64.whl", hash = "sha256:6ac6db0c8229e5c3f633208e9d7e9e016ee725cbc53a9e5993b4c070c189f506"}, - {file = "fastavro-1.6.0-cp38-cp38-macosx_10_15_x86_64.whl", hash = "sha256:7511fbf7e1251ae34965894cf59ad982612268a5043fbfeaf309db8f75c25489"}, - {file = "fastavro-1.6.0-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:c039f259c61e75d920b2ce236d676a7dbf122ee136df92133542a99174491392"}, - {file = "fastavro-1.6.0-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:970599975812c8df4aba06ac4e4e558211fc645f3ca44a8fab12b0f2152fd959"}, - {file = "fastavro-1.6.0-cp38-cp38-win_amd64.whl", hash = "sha256:ca4a3b4c39d2b280523111fb4ea09dd114ab78e7ee8adb166804624ce4b7ad98"}, - {file = "fastavro-1.6.0-cp39-cp39-macosx_10_15_x86_64.whl", hash = "sha256:878b06ac7f6e411c849ceafd2cc7dd9e6eb3f7d4a5268aa6f02eb8460a069082"}, - {file = "fastavro-1.6.0-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:8b788093cf45763e69cd8ffa2464a518edd13340a48e13c4211f85f25f91c972"}, - {file = "fastavro-1.6.0-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:a69d35e66812b3aed7204d796c2a0f132d2d5a115d94cc1fb3f5da3577a74d97"}, - {file = "fastavro-1.6.0-cp39-cp39-win_amd64.whl", hash = "sha256:4eee1bfa7a5d35d252a58c96b727c4645a0e0db3f4c44b94de444d3b086bbc48"}, - {file = "fastavro-1.6.0.tar.gz", hash = "sha256:64759b161bbacb5fdcbd54bd5cee05d8510836e09290e958937334dbf1bb587f"}, + {file = "fastavro-1.6.1-cp310-cp310-macosx_10_14_x86_64.whl", hash = "sha256:de244146a3d7bc4e60c7be11f317054ee0e57ca807d85923bd8407981488ff18"}, + {file = "fastavro-1.6.1-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:1ed84fb27672e54b0bafecae1f128f6e2950828f451e59f93bb74eeccb31b496"}, + {file = "fastavro-1.6.1-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:98df91eff0c061ffeb722732ba61d4ea9c7afd0f5403534991a984e75e3b2272"}, + {file = "fastavro-1.6.1-cp310-cp310-win_amd64.whl", hash = "sha256:fd60999204f60d5bcc4e9a4461ce867efb23d271e955684b36c04970750e2ccb"}, + {file = "fastavro-1.6.1-cp37-cp37m-macosx_10_15_x86_64.whl", hash = "sha256:19d4e83bf952fc2893d11d4c82cf04c3938c921a7c5bf277256cda1005e59f16"}, + {file = "fastavro-1.6.1-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:945d5f6e7a3076758b52fe1cebecfd86b66e6925a3a38732e93f3e1eab955256"}, + {file = "fastavro-1.6.1-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:315084c45339845273193545a50f3d616787eb3231b0b1f3df229196b22f192e"}, + {file = "fastavro-1.6.1-cp37-cp37m-win_amd64.whl", hash = "sha256:3ef81602dc6ecac3d646d64fe13aeb11252f75b76141b5b3d755bbf548313718"}, + {file = "fastavro-1.6.1-cp38-cp38-macosx_10_15_x86_64.whl", hash = "sha256:462c3177aa231f8b023d4af0a73f41517df8160399564fc750a711faa0889b59"}, + {file = "fastavro-1.6.1-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:3dd8c98d410fbb46a475242cea1b86061d2568db89919eb9079a9a183f04d444"}, + {file = "fastavro-1.6.1-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:b8b77e8f33e55d8f63d65f58a2b2c99248093e2da2e8a86f58320f5715fc3f54"}, + {file = "fastavro-1.6.1-cp38-cp38-win_amd64.whl", hash = "sha256:ac31f490e3030a6c608fffa6e337bd3f1b00f227bcb098df88a6e3e38982dfa5"}, + {file = "fastavro-1.6.1-cp39-cp39-macosx_10_15_x86_64.whl", hash = "sha256:3e479b9e4b5e6184b57daab57f1992e649590e902d26b7e2ef50a605626e9053"}, + {file = "fastavro-1.6.1-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:f2ec2fa5f100e0f1010f550de6de8b1019a683c5034613772b4444d7c85bdb57"}, + {file = "fastavro-1.6.1-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:1b34c4d33e22ada2fa19566a43976b3f5ca937ab6e054e65e1467bf67353f352"}, + {file = "fastavro-1.6.1-cp39-cp39-win_amd64.whl", hash = "sha256:18f0ddf3cc4f09839c08eec67ed3be3d391894874cb87f856a0362540d18df17"}, + {file = "fastavro-1.6.1.tar.gz", hash = "sha256:bc37a6edbde7a04a9df28ab838b94df7527e5eea648d61633233abba59205146"}, ] filelock = [ {file = "filelock-3.8.0-py3-none-any.whl", hash = "sha256:617eb4e5eedc82fc5f47b6d61e4d11cb837c56cb4544e39081099fa17ad109d4"}, diff --git a/pyproject.toml b/pyproject.toml index 7be9f64afb..055281b0fa 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -65,7 +65,7 @@ s3fs = { version = "2022.5.0", optional = true} pytest = "^7.1.3" pytest-checkdocs = "^2.0.0" pre-commit = "^2.0.0" -fastavro = "^1.5.4" +fastavro = "^1.6.1" coverage = { version = "^6.4.4", extras = ["toml"] } requests-mock = "^1.9.3" From 1a403da3718a06b507b2b5a4c791d64726767bc6 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 13 Sep 2022 09:53:04 -0700 Subject: [PATCH 197/642] Python: Bump pydantic from 1.10.1 to 1.10.2 (#5744) Bumps [pydantic](https://github.com/pydantic/pydantic) from 1.10.1 to 1.10.2. - [Release notes](https://github.com/pydantic/pydantic/releases) - [Changelog](https://github.com/pydantic/pydantic/blob/main/HISTORY.md) - [Commits](https://github.com/pydantic/pydantic/compare/v1.10.1...v1.10.2) --- updated-dependencies: - dependency-name: pydantic dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- poetry.lock | 76 +++++++++++++++++++++++++------------------------- pyproject.toml | 2 +- 2 files changed, 39 insertions(+), 39 deletions(-) diff --git a/poetry.lock b/poetry.lock index dd33353496..c512e41026 100644 --- a/poetry.lock +++ b/poetry.lock @@ -436,7 +436,7 @@ python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*" [[package]] name = "pydantic" -version = "1.10.1" +version = "1.10.2" description = "Data validation and settings management using python type hints" category = "main" optional = false @@ -749,7 +749,7 @@ snappy = ["python-snappy"] [metadata] lock-version = "1.1" python-versions = "^3.8" -content-hash = "450282a4da16945da7dfc2a3c44b5e2343fd11f9d440826a4fd8af8b94ff7531" +content-hash = "b77f3ba52efd281107434ef1d70c6d4767ab0b339bd826a66966e040394b719e" [metadata.files] aiobotocore = [ @@ -1288,42 +1288,42 @@ pycparser = [ {file = "pycparser-2.21.tar.gz", hash = "sha256:e644fdec12f7872f86c58ff790da456218b10f863970249516d60a5eaca77206"}, ] pydantic = [ - {file = "pydantic-1.10.1-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:221166d99726238f71adc4fa9f3e94063a10787574b966f86a774559e709ac5a"}, - {file = "pydantic-1.10.1-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:a90e85d95fd968cd7cae122e0d3e0e1f6613bc88c1ff3fe838ac9785ea4b1c4c"}, - {file = "pydantic-1.10.1-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:f2157aaf5718c648eaec9e654a34179ae42ffc363dc3ad058538a4f3ecbd9341"}, - {file = "pydantic-1.10.1-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:6142246fc9adb51cadaeb84fb52a86f3adad4c6a7b0938a5dd0b1356b0088217"}, - {file = "pydantic-1.10.1-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:60dad97a09b6f44690c05467a4f397b62bfc2c839ac39102819d6979abc2be0d"}, - {file = "pydantic-1.10.1-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:d6f5bcb59d33ec46621dae76e714c53035087666cac80c81c9047a84f3ff93d0"}, - {file = "pydantic-1.10.1-cp310-cp310-win_amd64.whl", hash = "sha256:522906820cd60e63c7960ba83078bf2d2ad2dd0870bf68248039bcb1ec3eb0a4"}, - {file = "pydantic-1.10.1-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:d545c89d88bdd5559db17aeb5a61a26799903e4bd76114779b3bf1456690f6ce"}, - {file = "pydantic-1.10.1-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:ad2374b5b3b771dcc6e2f6e0d56632ab63b90e9808b7a73ad865397fcdb4b2cd"}, - {file = "pydantic-1.10.1-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:90e02f61b7354ed330f294a437d0bffac9e21a5d46cb4cc3c89d220e497db7ac"}, - {file = "pydantic-1.10.1-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:cc5ffe7bd0b4778fa5b7a5f825c52d6cfea3ae2d9b52b05b9b1d97e36dee23a8"}, - {file = "pydantic-1.10.1-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:7acb7b66ffd2bc046eaff0063df84c83fc3826722d5272adaeadf6252e17f691"}, - {file = "pydantic-1.10.1-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:7e6786ed5faa559dea5a77f6d2de9a08d18130de9344533535d945f34bdcd42e"}, - {file = "pydantic-1.10.1-cp311-cp311-win_amd64.whl", hash = "sha256:c7bf8ff1d18186eb0cbe42bd9bfb4cbf7fde1fd01b8608925458990c21f202f0"}, - {file = "pydantic-1.10.1-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:14a5babda137a294df7ad5f220986d79bbb87fdeb332c6ded61ce19da7f5f3bf"}, - {file = "pydantic-1.10.1-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:5659cb9c6b3d27fc0067025c4f5a205f5e838232a4a929b412781117c2343d44"}, - {file = "pydantic-1.10.1-cp37-cp37m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:c8d70fb91b03c32d2e857b071a22a5225e6b625ca82bd2cc8dd729d88e0bd200"}, - {file = "pydantic-1.10.1-cp37-cp37m-musllinux_1_1_i686.whl", hash = "sha256:9a93be313e40f12c6f2cb84533b226bbe23d0774872e38d83415e6890215e3a6"}, - {file = "pydantic-1.10.1-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:d55aeb01bb7bd7c7e1bd904668a4a2ffcbb1c248e7ae9eb40a272fd7e67dd98b"}, - {file = "pydantic-1.10.1-cp37-cp37m-win_amd64.whl", hash = "sha256:43d41b6f13706488e854729955ba8f740e6ec375cd16b72b81dc24b9d84f0d15"}, - {file = "pydantic-1.10.1-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:f31ffe0e38805a0e6410330f78147bb89193b136d7a5f79cae60d3e849b520a6"}, - {file = "pydantic-1.10.1-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:8eee69eda7674977b079a21e7bf825b59d8bf15145300e8034ed3eb239ac444f"}, - {file = "pydantic-1.10.1-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:6f927bff6c319fc92e0a2cbeb2609b5c1cd562862f4b54ec905e353282b7c8b1"}, - {file = "pydantic-1.10.1-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:eb1bc3f8fef6ba36977108505e90558911e7fbccb4e930805d5dd90891b56ff4"}, - {file = "pydantic-1.10.1-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:96ab6ce1346d14c6e581a69c333bdd1b492df9cf85ad31ad77a8aa42180b7e09"}, - {file = "pydantic-1.10.1-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:444cf220a12134da1cd42fe4f45edff622139e10177ce3d8ef2b4f41db1291b2"}, - {file = "pydantic-1.10.1-cp38-cp38-win_amd64.whl", hash = "sha256:dbfbff83565b4514dd8cebc8b8c81a12247e89427ff997ad0a9da7b2b1065c12"}, - {file = "pydantic-1.10.1-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:5327406f4bfd5aee784e7ad2a6a5fdd7171c19905bf34cb1994a1ba73a87c468"}, - {file = "pydantic-1.10.1-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:1072eae28bf034a311764c130784e8065201a90edbca10f495c906737b3bd642"}, - {file = "pydantic-1.10.1-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:ce901335667a68dfbc10dd2ee6c0d676b89210d754441c2469fbc37baf7ee2ed"}, - {file = "pydantic-1.10.1-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:54d6465cd2112441305faf5143a491b40de07a203116b5755a2108e36b25308d"}, - {file = "pydantic-1.10.1-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:2b5e5e7a0ec96704099e271911a1049321ba1afda92920df0769898a7e9a1298"}, - {file = "pydantic-1.10.1-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:ae43704358304da45c1c3dd7056f173c618b252f91594bcb6d6f6b4c6c284dee"}, - {file = "pydantic-1.10.1-cp39-cp39-win_amd64.whl", hash = "sha256:2d7da49229ffb1049779a5a6c1c50a26da164bd053cf8ee9042197dc08a98259"}, - {file = "pydantic-1.10.1-py3-none-any.whl", hash = "sha256:f8b10e59c035ff3dcc9791619d6e6c5141e0fa5cbe264e19e267b8d523b210bf"}, - {file = "pydantic-1.10.1.tar.gz", hash = "sha256:d41bb80347a8a2d51fbd6f1748b42aca14541315878447ba159617544712f770"}, + {file = "pydantic-1.10.2-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:bb6ad4489af1bac6955d38ebcb95079a836af31e4c4f74aba1ca05bb9f6027bd"}, + {file = "pydantic-1.10.2-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:a1f5a63a6dfe19d719b1b6e6106561869d2efaca6167f84f5ab9347887d78b98"}, + {file = "pydantic-1.10.2-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:352aedb1d71b8b0736c6d56ad2bd34c6982720644b0624462059ab29bd6e5912"}, + {file = "pydantic-1.10.2-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:19b3b9ccf97af2b7519c42032441a891a5e05c68368f40865a90eb88833c2559"}, + {file = "pydantic-1.10.2-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:e9069e1b01525a96e6ff49e25876d90d5a563bc31c658289a8772ae186552236"}, + {file = "pydantic-1.10.2-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:355639d9afc76bcb9b0c3000ddcd08472ae75318a6eb67a15866b87e2efa168c"}, + {file = "pydantic-1.10.2-cp310-cp310-win_amd64.whl", hash = "sha256:ae544c47bec47a86bc7d350f965d8b15540e27e5aa4f55170ac6a75e5f73b644"}, + {file = "pydantic-1.10.2-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:a4c805731c33a8db4b6ace45ce440c4ef5336e712508b4d9e1aafa617dc9907f"}, + {file = "pydantic-1.10.2-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:d49f3db871575e0426b12e2f32fdb25e579dea16486a26e5a0474af87cb1ab0a"}, + {file = "pydantic-1.10.2-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:37c90345ec7dd2f1bcef82ce49b6235b40f282b94d3eec47e801baf864d15525"}, + {file = "pydantic-1.10.2-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:7b5ba54d026c2bd2cb769d3468885f23f43710f651688e91f5fb1edcf0ee9283"}, + {file = "pydantic-1.10.2-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:05e00dbebbe810b33c7a7362f231893183bcc4251f3f2ff991c31d5c08240c42"}, + {file = "pydantic-1.10.2-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:2d0567e60eb01bccda3a4df01df677adf6b437958d35c12a3ac3e0f078b0ee52"}, + {file = "pydantic-1.10.2-cp311-cp311-win_amd64.whl", hash = "sha256:c6f981882aea41e021f72779ce2a4e87267458cc4d39ea990729e21ef18f0f8c"}, + {file = "pydantic-1.10.2-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:c4aac8e7103bf598373208f6299fa9a5cfd1fc571f2d40bf1dd1955a63d6eeb5"}, + {file = "pydantic-1.10.2-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:81a7b66c3f499108b448f3f004801fcd7d7165fb4200acb03f1c2402da73ce4c"}, + {file = "pydantic-1.10.2-cp37-cp37m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:bedf309630209e78582ffacda64a21f96f3ed2e51fbf3962d4d488e503420254"}, + {file = "pydantic-1.10.2-cp37-cp37m-musllinux_1_1_i686.whl", hash = "sha256:9300fcbebf85f6339a02c6994b2eb3ff1b9c8c14f502058b5bf349d42447dcf5"}, + {file = "pydantic-1.10.2-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:216f3bcbf19c726b1cc22b099dd409aa371f55c08800bcea4c44c8f74b73478d"}, + {file = "pydantic-1.10.2-cp37-cp37m-win_amd64.whl", hash = "sha256:dd3f9a40c16daf323cf913593083698caee97df2804aa36c4b3175d5ac1b92a2"}, + {file = "pydantic-1.10.2-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:b97890e56a694486f772d36efd2ba31612739bc6f3caeee50e9e7e3ebd2fdd13"}, + {file = "pydantic-1.10.2-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:9cabf4a7f05a776e7793e72793cd92cc865ea0e83a819f9ae4ecccb1b8aa6116"}, + {file = "pydantic-1.10.2-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:06094d18dd5e6f2bbf93efa54991c3240964bb663b87729ac340eb5014310624"}, + {file = "pydantic-1.10.2-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:cc78cc83110d2f275ec1970e7a831f4e371ee92405332ebfe9860a715f8336e1"}, + {file = "pydantic-1.10.2-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:1ee433e274268a4b0c8fde7ad9d58ecba12b069a033ecc4645bb6303c062d2e9"}, + {file = "pydantic-1.10.2-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:7c2abc4393dea97a4ccbb4ec7d8658d4e22c4765b7b9b9445588f16c71ad9965"}, + {file = "pydantic-1.10.2-cp38-cp38-win_amd64.whl", hash = "sha256:0b959f4d8211fc964772b595ebb25f7652da3f22322c007b6fed26846a40685e"}, + {file = "pydantic-1.10.2-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:c33602f93bfb67779f9c507e4d69451664524389546bacfe1bee13cae6dc7488"}, + {file = "pydantic-1.10.2-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:5760e164b807a48a8f25f8aa1a6d857e6ce62e7ec83ea5d5c5a802eac81bad41"}, + {file = "pydantic-1.10.2-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:6eb843dcc411b6a2237a694f5e1d649fc66c6064d02b204a7e9d194dff81eb4b"}, + {file = "pydantic-1.10.2-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:4b8795290deaae348c4eba0cebb196e1c6b98bdbe7f50b2d0d9a4a99716342fe"}, + {file = "pydantic-1.10.2-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:e0bedafe4bc165ad0a56ac0bd7695df25c50f76961da29c050712596cf092d6d"}, + {file = "pydantic-1.10.2-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:2e05aed07fa02231dbf03d0adb1be1d79cabb09025dd45aa094aa8b4e7b9dcda"}, + {file = "pydantic-1.10.2-cp39-cp39-win_amd64.whl", hash = "sha256:c1ba1afb396148bbc70e9eaa8c06c1716fdddabaf86e7027c5988bae2a829ab6"}, + {file = "pydantic-1.10.2-py3-none-any.whl", hash = "sha256:1b6ee725bd6e83ec78b1aa32c5b1fa67a3a65badddde3976bca5fe4568f27709"}, + {file = "pydantic-1.10.2.tar.gz", hash = "sha256:91b8e218852ef6007c2b98cd861601c6a09f1aa32bbbb74fab5b1c33d4a1e410"}, ] pygments = [ {file = "Pygments-2.13.0-py3-none-any.whl", hash = "sha256:f643f331ab57ba3c9d89212ee4a2dabc6e94f117cf4eefde99a0574720d14c42"}, diff --git a/pyproject.toml b/pyproject.toml index 055281b0fa..b57f20d607 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -48,7 +48,7 @@ click = "^8.1.3" rich = "^12.5.1" pyyaml = "^6.0.0" -pydantic = "^1.9.2" +pydantic = "^1.10.2" fsspec = "2022.5.0" pyarrow = { version = "^9.0.0", optional = true } From c79600c72c1a707d1f8ce4659394abc85f90a47c Mon Sep 17 00:00:00 2001 From: Dhruv Pratap Date: Wed, 14 Sep 2022 11:08:37 -0400 Subject: [PATCH 198/642] Python: Make Get Properties CLI options consistent. (#5736) Consistent with Set and Remove CLI options * Python: Make Get Properties CLI options consistent with Set and Remove CLI Options * Python: Explicitly specify command names in the annotations and not infer it from the method name to fix F811 lint issues of duplicate method declaration. --- pyiceberg/cli/console.py | 74 +++++++++++++++++++-------------------- tests/cli/test_console.py | 46 ++++++++++++++++-------- 2 files changed, 68 insertions(+), 52 deletions(-) diff --git a/pyiceberg/cli/console.py b/pyiceberg/cli/console.py index 404a882d9e..3a040159d8 100644 --- a/pyiceberg/cli/console.py +++ b/pyiceberg/cli/console.py @@ -227,55 +227,53 @@ def properties(): """Properties on tables/namespaces""" -@properties.command() -@click.option("--entity", type=click.Choice(["any", "namespace", "table"]), default="any") +@properties.group() +def get(): + """Fetch properties on tables/namespaces""" + + +@get.command("namespace") @click.argument("identifier") @click.argument("property_name", required=False) @click.pass_context @catch_exception() -def get(ctx: Context, entity: Literal["name", "namespace", "table"], identifier: str, property_name: str): - """Fetches a property of a namespace or table""" +def get_namespace(ctx: Context, identifier: str, property_name: str): + """Fetch properties on a namespace""" catalog, output = _catalog_and_output(ctx) identifier_tuple = Catalog.identifier_to_tuple(identifier) - is_namespace = False - if entity in {"namespace", "any"}: - try: - namespace_properties = catalog.load_namespace_properties(identifier_tuple) + namespace_properties = catalog.load_namespace_properties(identifier_tuple) + assert namespace_properties - if property_name: - if property_value := namespace_properties.get(property_name): - output.text(property_value) - is_namespace = True - else: - raise NoSuchPropertyException(f"Could not find property {property_name} on namespace {identifier}") - else: - output.describe_properties(namespace_properties) - is_namespace = True - except NoSuchNamespaceError as exc: - if entity != "any" or len(identifier_tuple) <= 1: # type: ignore - raise exc - is_table = False - if is_namespace is False and len(identifier_tuple) > 1 and entity in {"table", "any"}: - metadata = catalog.load_table(identifier_tuple).metadata - assert metadata - - if property_name: - if property_value := metadata.properties.get(property_name): - output.text(property_value) - is_table = True - else: - raise NoSuchPropertyException(f"Could not find property {property_name} on table {identifier}") + if property_name: + if property_value := namespace_properties.get(property_name): + output.text(property_value) else: - output.describe_properties(metadata.properties) - is_table = True + raise NoSuchPropertyException(f"Could not find property {property_name} on namespace {identifier}") + else: + output.describe_properties(namespace_properties) - if is_namespace is False and is_table is False: - property_err = "" - if property_name: - property_err = f" with property {property_name}" - raise NoSuchNamespaceError(f"Table or namespace does not exist: {identifier}{property_err}") +@get.command("table") +@click.argument("identifier") +@click.argument("property_name", required=False) +@click.pass_context +@catch_exception() +def get_table(ctx: Context, identifier: str, property_name: str): + """Fetch properties on a table""" + catalog, output = _catalog_and_output(ctx) + identifier_tuple = Catalog.identifier_to_tuple(identifier) + + metadata = catalog.load_table(identifier_tuple).metadata + assert metadata + + if property_name: + if property_value := metadata.properties.get(property_name): + output.text(property_value) + else: + raise NoSuchPropertyException(f"Could not find property {property_name} on table {identifier}") + else: + output.describe_properties(metadata.properties) @properties.group() diff --git a/tests/cli/test_console.py b/tests/cli/test_console.py index 7c57e7b26e..38382cd1bb 100644 --- a/tests/cli/test_console.py +++ b/tests/cli/test_console.py @@ -357,7 +357,7 @@ def test_rename_table_does_not_exists(_): @mock.patch("pyiceberg.cli.console.load_catalog", return_value=MOCK_CATALOG) def test_properties_get_table(_): runner = CliRunner() - result = runner.invoke(run, ["properties", "get", "default.foo"]) + result = runner.invoke(run, ["properties", "get", "table", "default.foo"]) assert result.exit_code == 0 assert result.output == "read.split.target.size 134217728\n" @@ -366,7 +366,7 @@ def test_properties_get_table(_): @mock.patch("pyiceberg.cli.console.load_catalog", return_value=MOCK_CATALOG) def test_properties_get_table_specific_property(_): runner = CliRunner() - result = runner.invoke(run, ["properties", "get", "default.foo", "read.split.target.size"]) + result = runner.invoke(run, ["properties", "get", "table", "default.foo", "read.split.target.size"]) assert result.exit_code == 0 assert result.output == "134217728\n" @@ -375,7 +375,7 @@ def test_properties_get_table_specific_property(_): @mock.patch("pyiceberg.cli.console.load_catalog", return_value=MOCK_CATALOG) def test_properties_get_table_specific_property_that_doesnt_exist(_): runner = CliRunner() - result = runner.invoke(run, ["properties", "get", "default.foo", "doesnotexist"]) + result = runner.invoke(run, ["properties", "get", "table", "default.foo", "doesnotexist"]) assert result.exit_code == 1 assert result.output == "Could not find property doesnotexist on table default.foo\n" @@ -384,16 +384,16 @@ def test_properties_get_table_specific_property_that_doesnt_exist(_): @mock.patch("pyiceberg.cli.console.load_catalog", return_value=MOCK_CATALOG) def test_properties_get_table_does_not_exist(_): runner = CliRunner() - result = runner.invoke(run, ["properties", "get", "doesnotexist"]) + result = runner.invoke(run, ["properties", "get", "table", "doesnotexist"]) assert result.exit_code == 1 - assert result.output == "Namespace does not exist: doesnotexist\n" + assert result.output == "Table does not exist: doesnotexist\n" @mock.patch.dict(os.environ, MOCK_ENVIRONMENT) @mock.patch("pyiceberg.cli.console.load_catalog", return_value=MOCK_CATALOG) def test_properties_get_namespace(_): runner = CliRunner() - result = runner.invoke(run, ["properties", "get", "default"]) + result = runner.invoke(run, ["properties", "get", "namespace", "default"]) assert result.exit_code == 0 assert result.output == "location s3://warehouse/database/location\n" @@ -402,11 +402,20 @@ def test_properties_get_namespace(_): @mock.patch("pyiceberg.cli.console.load_catalog", return_value=MOCK_CATALOG) def test_properties_get_namespace_specific_property(_): runner = CliRunner() - result = runner.invoke(run, ["properties", "get", "default", "location"]) + result = runner.invoke(run, ["properties", "get", "namespace", "default", "location"]) assert result.exit_code == 0 assert result.output == "s3://warehouse/database/location\n" +@mock.patch.dict(os.environ, MOCK_ENVIRONMENT) +@mock.patch("pyiceberg.cli.console.load_catalog", return_value=MOCK_CATALOG) +def test_properties_get_namespace_does_not_exist(_): + runner = CliRunner() + result = runner.invoke(run, ["properties", "get", "namespace", "doesnotexist"]) + assert result.exit_code == 1 + assert result.output == "Namespace does not exist: doesnotexist\n" + + @mock.patch.dict(os.environ, MOCK_ENVIRONMENT) @mock.patch("pyiceberg.cli.console.load_catalog", return_value=MOCK_CATALOG) def test_properties_set_namespace(_): @@ -683,7 +692,7 @@ def test_json_rename_table_does_not_exists(_): @mock.patch("pyiceberg.cli.console.load_catalog", return_value=MOCK_CATALOG) def test_json_properties_get_table(_): runner = CliRunner() - result = runner.invoke(run, ["--output=json", "properties", "get", "default.foo"]) + result = runner.invoke(run, ["--output=json", "properties", "get", "table", "default.foo"]) assert result.exit_code == 0 assert result.output == """{"read.split.target.size": "134217728"}\n""" @@ -692,7 +701,7 @@ def test_json_properties_get_table(_): @mock.patch("pyiceberg.cli.console.load_catalog", return_value=MOCK_CATALOG) def test_json_properties_get_table_specific_property(_): runner = CliRunner() - result = runner.invoke(run, ["--output=json", "properties", "get", "default.foo", "read.split.target.size"]) + result = runner.invoke(run, ["--output=json", "properties", "get", "table", "default.foo", "read.split.target.size"]) assert result.exit_code == 0 assert result.output == """"134217728"\n""" @@ -701,7 +710,7 @@ def test_json_properties_get_table_specific_property(_): @mock.patch("pyiceberg.cli.console.load_catalog", return_value=MOCK_CATALOG) def test_json_properties_get_table_specific_property_that_doesnt_exist(_): runner = CliRunner() - result = runner.invoke(run, ["--output=json", "properties", "get", "default.foo", "doesnotexist"]) + result = runner.invoke(run, ["--output=json", "properties", "get", "table", "default.foo", "doesnotexist"]) assert result.exit_code == 1 assert ( result.output @@ -713,16 +722,16 @@ def test_json_properties_get_table_specific_property_that_doesnt_exist(_): @mock.patch("pyiceberg.cli.console.load_catalog", return_value=MOCK_CATALOG) def test_json_properties_get_table_does_not_exist(_): runner = CliRunner() - result = runner.invoke(run, ["--output=json", "properties", "get", "doesnotexist"]) + result = runner.invoke(run, ["--output=json", "properties", "get", "table", "doesnotexist"]) assert result.exit_code == 1 - assert result.output == """{"type": "NoSuchNamespaceError", "message": "Namespace does not exist: doesnotexist"}\n""" + assert result.output == """{"type": "NoSuchTableError", "message": "Table does not exist: doesnotexist"}\n""" @mock.patch.dict(os.environ, MOCK_ENVIRONMENT) @mock.patch("pyiceberg.cli.console.load_catalog", return_value=MOCK_CATALOG) def test_json_properties_get_namespace(_): runner = CliRunner() - result = runner.invoke(run, ["--output=json", "properties", "get", "default"]) + result = runner.invoke(run, ["--output=json", "properties", "get", "namespace", "default"]) assert result.exit_code == 0 assert result.output == """{"location": "s3://warehouse/database/location"}\n""" @@ -731,11 +740,20 @@ def test_json_properties_get_namespace(_): @mock.patch("pyiceberg.cli.console.load_catalog", return_value=MOCK_CATALOG) def test_json_properties_get_namespace_specific_property(_): runner = CliRunner() - result = runner.invoke(run, ["--output=json", "properties", "get", "default", "location"]) + result = runner.invoke(run, ["--output=json", "properties", "get", "namespace", "default", "location"]) assert result.exit_code == 0 assert result.output == """"s3://warehouse/database/location"\n""" +@mock.patch.dict(os.environ, MOCK_ENVIRONMENT) +@mock.patch("pyiceberg.cli.console.load_catalog", return_value=MOCK_CATALOG) +def test_json_properties_get_namespace_does_not_exist(_): + runner = CliRunner() + result = runner.invoke(run, ["--output=json", "properties", "get", "namespace", "doesnotexist"]) + assert result.exit_code == 1 + assert result.output == """{"type": "NoSuchNamespaceError", "message": "Namespace does not exist: doesnotexist"}\n""" + + @mock.patch.dict(os.environ, MOCK_ENVIRONMENT) @mock.patch("pyiceberg.cli.console.load_catalog", return_value=MOCK_CATALOG) def test_json_properties_set_namespace(_): From b60d1f2ba8fbec836e171f1e467b6ca2773b4570 Mon Sep 17 00:00:00 2001 From: Fokko Driesprong Date: Thu, 15 Sep 2022 20:21:22 -0700 Subject: [PATCH 199/642] Python: Add CLI command to list files (#5690) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This makes it easy to check the FileIO: ``` > pyiceberg files nyc.taxis Snapshots: nyc.taxis └── Snapshot 5937117119577207079, schema 0: file:/Users/fokkodriesprong/Desktop/docker-spark-iceberg/wh/nyc.db/taxis/metadata/snap-5937117119577207079-1-94656c4f-4c66-4600-a4ca-f30377300527.avro └── Manifest: file:/Users/fokkodriesprong/Desktop/docker-spark-iceberg/wh/nyc.db/taxis/metadata/94656c4f-4c66-4600-a4ca-f30377300527-m0.avro └── Datafile: file:/Users/fokkodriesprong/Desktop/docker-spark-iceberg/wh/nyc.db/taxis/data/00003-4-a245d9ee-8462-4a08-8cbc-26b8b33b9377-00001.parquet ``` --- pyiceberg/cli/console.py | 15 ++ pyiceberg/cli/output.py | 82 +++++++--- pyiceberg/manifest.py | 6 +- pyiceberg/table/snapshots.py | 15 +- tests/avro/test_reader.py | 6 +- tests/conftest.py | 16 +- tests/table/test_snapshots.py | 39 +++++ tests/utils/test_manifest.py | 293 +++++++++++++++++++++++++++++++++- 8 files changed, 437 insertions(+), 35 deletions(-) diff --git a/pyiceberg/cli/console.py b/pyiceberg/cli/console.py index 3a040159d8..e8234d7c7e 100644 --- a/pyiceberg/cli/console.py +++ b/pyiceberg/cli/console.py @@ -24,6 +24,7 @@ from pyiceberg.catalog import Catalog, load_catalog from pyiceberg.cli.output import ConsoleOutput, JsonOutput, Output from pyiceberg.exceptions import NoSuchNamespaceError, NoSuchPropertyException, NoSuchTableError +from pyiceberg.io import load_file_io def catch_exception(): @@ -136,6 +137,20 @@ def describe(ctx: Context, entity: Literal["name", "namespace", "table"], identi raise NoSuchTableError(f"Table or namespace does not exist: {identifier}") +@run.command() +@click.argument("identifier") +@click.option("--history", is_flag=True) +@click.pass_context +@catch_exception() +def files(ctx: Context, identifier: str, history: bool): + """Lists all the files of the table""" + catalog, output = _catalog_and_output(ctx) + + catalog_table = catalog.load_table(identifier) + io = load_file_io({**catalog.properties, **catalog_table.metadata.properties}) + output.files(catalog_table, io, history) + + @run.command() @click.argument("identifier") @click.pass_context diff --git a/pyiceberg/cli/output.py b/pyiceberg/cli/output.py index 270567930e..c7907c7e5a 100644 --- a/pyiceberg/cli/output.py +++ b/pyiceberg/cli/output.py @@ -23,6 +23,7 @@ from rich.table import Table as RichTable from rich.tree import Tree +from pyiceberg.io import FileIO from pyiceberg.schema import Schema from pyiceberg.table import Table from pyiceberg.table.partitioning import PartitionSpec @@ -33,35 +34,39 @@ class Output(ABC): """Output interface for exporting""" @abstractmethod - def exception(self, ex: Exception): + def exception(self, ex: Exception) -> None: ... @abstractmethod - def identifiers(self, identifiers: List[Identifier]): + def identifiers(self, identifiers: List[Identifier]) -> None: ... @abstractmethod - def describe_table(self, table): + def describe_table(self, table: Table) -> None: ... @abstractmethod - def describe_properties(self, properties: Properties): + def files(self, table: Table, io: FileIO, history: bool) -> None: ... @abstractmethod - def text(self, response: str): + def describe_properties(self, properties: Properties) -> None: ... @abstractmethod - def schema(self, schema: Schema): + def text(self, response: str) -> None: ... @abstractmethod - def spec(self, spec: PartitionSpec): + def schema(self, schema: Schema) -> None: ... @abstractmethod - def uuid(self, uuid: Optional[UUID]): + def spec(self, spec: PartitionSpec) -> None: + ... + + @abstractmethod + def uuid(self, uuid: Optional[UUID]) -> None: ... @@ -70,27 +75,27 @@ class ConsoleOutput(Output): verbose: bool - def __init__(self, **properties: Any): + def __init__(self, **properties: Any) -> None: self.verbose = properties.get("verbose", False) @property def _table(self) -> RichTable: return RichTable.grid(padding=(0, 2)) - def exception(self, ex: Exception): + def exception(self, ex: Exception) -> None: if self.verbose: Console(stderr=True).print_exception() else: Console(stderr=True).print(ex) - def identifiers(self, identifiers: List[Identifier]): + def identifiers(self, identifiers: List[Identifier]) -> None: table = self._table for identifier in identifiers: table.add_row(".".join(identifier)) Console().print(table) - def describe_table(self, table: Table): + def describe_table(self, table: Table) -> None: metadata = table.metadata table_properties = self._table @@ -119,25 +124,47 @@ def describe_table(self, table: Table): output_table.add_row("Properties", table_properties) Console().print(output_table) - def describe_properties(self, properties: Properties): + def files(self, table: Table, io: FileIO, history: bool) -> None: + if history: + snapshots = table.metadata.snapshots + else: + if snapshot := table.current_snapshot(): + snapshots = [snapshot] + else: + snapshots = [] + + snapshot_tree = Tree(f"Snapshots: {'.'.join(table.identifier)}") + + for snapshot in snapshots: + manifest_list_str = f": {snapshot.manifest_list}" if snapshot.manifest_list else "" + list_tree = snapshot_tree.add(f"Snapshot {snapshot.snapshot_id}, schema {snapshot.schema_id}{manifest_list_str}") + + manifest_list = snapshot.fetch_manifest_list(io) + for manifest in manifest_list: + manifest_tree = list_tree.add(f"Manifest: {manifest.manifest_path}") + for manifest_entry in manifest.fetch_manifest_entry(io): + manifest_tree.add(f"Datafile: {manifest_entry.data_file.file_path}") + Console().print(snapshot_tree) + + def describe_properties(self, properties: Properties) -> None: output_table = self._table for k, v in properties.items(): output_table.add_row(k, v) Console().print(output_table) - def text(self, response: str): + def text(self, response: str) -> None: Console().print(response) - def schema(self, schema: Schema): + def schema(self, schema: Schema) -> None: output_table = self._table for field in schema.fields: output_table.add_row(field.name, str(field.field_type), field.doc or "") Console().print(output_table) - def spec(self, spec: PartitionSpec): + def spec(self, spec: PartitionSpec) -> None: Console().print(str(spec)) - def uuid(self, uuid: Optional[UUID]): + def uuid(self, uuid: Optional[UUID]) -> None: Console().print(str(uuid) if uuid else "missing") @@ -146,32 +173,35 @@ class JsonOutput(Output): verbose: bool - def __init__(self, **properties: Any): + def __init__(self, **properties: Any) -> None: self.verbose = properties.get("verbose", False) def _out(self, d: Any) -> None: print(json.dumps(d)) - def exception(self, ex: Exception): + def exception(self, ex: Exception) -> None: self._out({"type": ex.__class__.__name__, "message": str(ex)}) - def identifiers(self, identifiers: List[Identifier]): + def identifiers(self, identifiers: List[Identifier]) -> None: self._out([".".join(identifier) for identifier in identifiers]) - def describe_table(self, table: Table): + def describe_table(self, table: Table) -> None: print(table.json()) - def describe_properties(self, properties: Properties): + def describe_properties(self, properties: Properties) -> None: self._out(properties) - def text(self, response: str): + def text(self, response: str) -> None: print(json.dumps(response)) - def schema(self, schema: Schema): + def schema(self, schema: Schema) -> None: print(schema.json()) - def spec(self, spec: PartitionSpec): + def files(self, table: Table, io: FileIO, history: bool) -> None: + pass + + def spec(self, spec: PartitionSpec) -> None: print(spec.json()) - def uuid(self, uuid: Optional[UUID]): + def uuid(self, uuid: Optional[UUID]) -> None: self._out({"uuid": str(uuid) if uuid else "missing"}) diff --git a/pyiceberg/manifest.py b/pyiceberg/manifest.py index 6079fcc336..751dfbef2a 100644 --- a/pyiceberg/manifest.py +++ b/pyiceberg/manifest.py @@ -29,7 +29,7 @@ from pyiceberg.avro.file import AvroFile from pyiceberg.avro.reader import AvroStruct -from pyiceberg.io import InputFile +from pyiceberg.io import FileIO, InputFile from pyiceberg.schema import Schema from pyiceberg.types import ( IcebergType, @@ -128,6 +128,10 @@ class ManifestFile(IcebergBaseModel): partitions: Optional[List[FieldSummary]] = Field() key_metadata: Optional[bytes] = Field() + def fetch_manifest_entry(self, io: FileIO) -> List[ManifestEntry]: + file = io.new_input(self.manifest_path) + return list(read_manifest_entry(file)) + def read_manifest_entry(input_file: InputFile) -> Iterator[ManifestEntry]: with AvroFile(input_file) as reader: diff --git a/pyiceberg/table/snapshots.py b/pyiceberg/table/snapshots.py index 849b2f3c06..35dd3c87e7 100644 --- a/pyiceberg/table/snapshots.py +++ b/pyiceberg/table/snapshots.py @@ -15,10 +15,17 @@ # specific language governing permissions and limitations # under the License. from enum import Enum -from typing import Dict, Optional, Union +from typing import ( + Dict, + List, + Optional, + Union, +) from pydantic import Field, PrivateAttr, root_validator +from pyiceberg.io import FileIO +from pyiceberg.manifest import ManifestFile, read_manifest_list from pyiceberg.utils.iceberg_base_model import IcebergBaseModel OPERATION = "operation" @@ -103,6 +110,12 @@ def __str__(self) -> str: result_str = f"{operation}id={self.snapshot_id}{parent_id}{schema_id}" return result_str + def fetch_manifest_list(self, io: FileIO) -> List[ManifestFile]: + if self.manifest_list is not None: + file = io.new_input(self.manifest_list) + return list(read_manifest_list(file)) + return [] + class MetadataLogEntry(IcebergBaseModel): metadata_file: str = Field(alias="metadata-file") diff --git a/tests/avro/test_reader.py b/tests/avro/test_reader.py index e4b0cb665e..44505a6174 100644 --- a/tests/avro/test_reader.py +++ b/tests/avro/test_reader.py @@ -376,9 +376,10 @@ def test_read_manifest_file_file(generated_manifest_file_file: str): records = list(reader) assert len(records) == 1, f"Expected 1 records, got {len(records)}" - assert records[0] == AvroStruct( + actual = records[0] + expected = AvroStruct( _data=[ - "/home/iceberg/warehouse/nyc/taxis_partitioned/metadata/0125c686-8aa6-4502-bdcc-b6d17ca41a3b-m0.avro", + actual.get(0), 7989, 0, 9182715666859759686, @@ -391,6 +392,7 @@ def test_read_manifest_file_file(generated_manifest_file_file: str): 0, ] ) + assert actual == expected def test_fixed_reader(): diff --git a/tests/conftest.py b/tests/conftest.py index 4030fbe9f1..be51c90357 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -26,7 +26,12 @@ """ import os from tempfile import TemporaryDirectory -from typing import Any, Dict, Union +from typing import ( + Any, + Dict, + Generator, + Union, +) from urllib.parse import urlparse import pytest @@ -932,7 +937,7 @@ def LocalFileIOFixture(): @pytest.fixture(scope="session") -def generated_manifest_entry_file(avro_schema_manifest_entry): +def generated_manifest_entry_file(avro_schema_manifest_entry: Dict[str, Any]) -> Generator[str, None, None]: from fastavro import parse_schema, writer parsed_schema = parse_schema(avro_schema_manifest_entry) @@ -945,11 +950,16 @@ def generated_manifest_entry_file(avro_schema_manifest_entry): @pytest.fixture(scope="session") -def generated_manifest_file_file(avro_schema_manifest_file): +def generated_manifest_file_file( + avro_schema_manifest_file: Dict[str, Any], generated_manifest_entry_file: str +) -> Generator[str, None, None]: from fastavro import parse_schema, writer parsed_schema = parse_schema(avro_schema_manifest_file) + # Make sure that a valid manifest_path is set + manifest_file_records[0]["manifest_path"] = generated_manifest_entry_file + with TemporaryDirectory() as tmpdir: tmp_avro_file = tmpdir + "/manifest.avro" with open(tmp_avro_file, "wb") as out: diff --git a/tests/table/test_snapshots.py b/tests/table/test_snapshots.py index 5dc48807ba..60d3bfa226 100644 --- a/tests/table/test_snapshots.py +++ b/tests/table/test_snapshots.py @@ -17,6 +17,8 @@ # pylint:disable=redefined-outer-name,eval-used import pytest +from pyiceberg.io.pyarrow import PyArrowFileIO +from pyiceberg.manifest import FieldSummary, ManifestContent, ManifestFile from pyiceberg.table.snapshots import Operation, Snapshot, Summary @@ -119,3 +121,40 @@ def test_snapshot_with_properties_repr(snapshot_with_properties: Snapshot): == """Snapshot(snapshot_id=25, parent_snapshot_id=19, sequence_number=200, timestamp_ms=1602638573590, manifest_list='s3:/a/b/c.avro', summary=Summary(Operation.APPEND, **{'foo': 'bar'}), schema_id=3)""" ) assert snapshot_with_properties == eval(repr(snapshot_with_properties)) + + +def test_fetch_manifest_list(generated_manifest_file_file: str): + snapshot = Snapshot( + snapshot_id=25, + parent_snapshot_id=19, + sequence_number=200, + timestamp_ms=1602638573590, + manifest_list=generated_manifest_file_file, + summary=Summary(Operation.APPEND), + schema_id=3, + ) + io = PyArrowFileIO() + actual = snapshot.fetch_manifest_list(io) + assert actual == [ + ManifestFile( + manifest_path=actual[0].manifest_path, # Is a temp path that changes every time + manifest_length=7989, + partition_spec_id=0, + content=ManifestContent.DATA, + sequence_number=0, + min_sequence_number=0, + added_snapshot_id=9182715666859759686, + added_data_files_count=3, + existing_data_files_count=0, + deleted_data_files_count=0, + added_rows_count=237993, + existing_rows_counts=None, + deleted_rows_count=0, + partitions=[ + FieldSummary( + contains_null=True, contains_nan=False, lower_bound=b"\x01\x00\x00\x00", upper_bound=b"\x02\x00\x00\x00" + ) + ], + key_metadata=None, + ) + ] diff --git a/tests/utils/test_manifest.py b/tests/utils/test_manifest.py index 66b70a2dc0..ae19c5d242 100644 --- a/tests/utils/test_manifest.py +++ b/tests/utils/test_manifest.py @@ -14,15 +14,22 @@ # KIND, either express or implied. See the License for the # specific language governing permissions and limitations # under the License. + +from pyiceberg.io import load_file_io from pyiceberg.manifest import ( DataFile, + DataFileContent, FieldSummary, FileFormat, + ManifestContent, ManifestEntry, + ManifestEntryStatus, ManifestFile, read_manifest_entry, read_manifest_list, ) +from pyiceberg.table import Snapshot +from pyiceberg.table.snapshots import Operation, Summary from tests.io.test_io import LocalInputFile @@ -262,9 +269,10 @@ def test_read_manifest_entry(generated_manifest_entry_file: str): def test_read_manifest_list(generated_manifest_file_file: str): input_file = LocalInputFile(generated_manifest_file_file) - assert list(read_manifest_list(input_file)) == [ + actual = list(read_manifest_list(input_file)) + expected = [ ManifestFile( - manifest_path="/home/iceberg/warehouse/nyc/taxis_partitioned/metadata/0125c686-8aa6-4502-bdcc-b6d17ca41a3b-m0.avro", + manifest_path=actual[0].manifest_path, manifest_length=7989, partition_spec_id=0, added_snapshot_id=9182715666859759686, @@ -281,3 +289,284 @@ def test_read_manifest_list(generated_manifest_file_file: str): deleted_rows_count=0, ) ] + assert actual == expected + + +def test_read_manifest(generated_manifest_file_file: str, generated_manifest_entry_file: str): + io = load_file_io({}) + + snapshot = Snapshot( + snapshot_id=25, + parent_snapshot_id=19, + timestamp_ms=1602638573590, + manifest_list=generated_manifest_file_file, + summary=Summary(Operation.APPEND), + schema_id=3, + ) + manifest_list = snapshot.fetch_manifest_list(io) + + assert manifest_list == [ + ManifestFile( + manifest_path=generated_manifest_entry_file, + manifest_length=7989, + partition_spec_id=0, + content=ManifestContent.DATA, + sequence_number=0, + min_sequence_number=0, + added_snapshot_id=9182715666859759686, + added_data_files_count=3, + existing_data_files_count=0, + deleted_data_files_count=0, + added_rows_count=237993, + existing_rows_counts=None, + deleted_rows_count=0, + partitions=[ + FieldSummary( + contains_null=True, contains_nan=False, lower_bound=b"\x01\x00\x00\x00", upper_bound=b"\x02\x00\x00\x00" + ) + ], + key_metadata=None, + ) + ] + + actual = manifest_list[0].fetch_manifest_entry(io) + expected = [ + ManifestEntry( + status=ManifestEntryStatus.ADDED, + snapshot_id=8744736658442914487, + sequence_number=None, + data_file=DataFile( + content=DataFileContent.DATA, + file_path="/home/iceberg/warehouse/nyc/taxis_partitioned/data/VendorID=null/00000-633-d8a4223e-dc97-45a1-86e1-adaba6e8abd7-00001.parquet", + file_format=FileFormat.PARQUET, + partition={"VendorID": None}, + record_count=19513, + file_size_in_bytes=388872, + block_size_in_bytes=67108864, + column_sizes={ + 1: 53, + 2: 98153, + 3: 98693, + 4: 53, + 5: 53, + 6: 53, + 7: 17425, + 8: 18528, + 9: 53, + 10: 44788, + 11: 35571, + 12: 53, + 13: 1243, + 14: 2355, + 15: 12750, + 16: 4029, + 17: 110, + 18: 47194, + 19: 2948, + }, + value_counts={ + 1: 19513, + 2: 19513, + 3: 19513, + 4: 19513, + 5: 19513, + 6: 19513, + 7: 19513, + 8: 19513, + 9: 19513, + 10: 19513, + 11: 19513, + 12: 19513, + 13: 19513, + 14: 19513, + 15: 19513, + 16: 19513, + 17: 19513, + 18: 19513, + 19: 19513, + }, + null_value_counts={ + 1: 19513, + 2: 0, + 3: 0, + 4: 19513, + 5: 19513, + 6: 19513, + 7: 0, + 8: 0, + 9: 19513, + 10: 0, + 11: 0, + 12: 19513, + 13: 0, + 14: 0, + 15: 0, + 16: 0, + 17: 0, + 18: 0, + 19: 0, + }, + nan_value_counts={16: 0, 17: 0, 18: 0, 19: 0, 10: 0, 11: 0, 12: 0, 13: 0, 14: 0, 15: 0}, + distinct_counts=None, + lower_bounds={ + 2: b"2020-04-01 00:00", + 3: b"2020-04-01 00:12", + 7: b"\x03\x00\x00\x00", + 8: b"\x01\x00\x00\x00", + 10: b"\xf6(\\\x8f\xc2\x05S\xc0", + 11: b"\x00\x00\x00\x00\x00\x00\x00\x00", + 13: b"\x00\x00\x00\x00\x00\x00\x00\x00", + 14: b"\x00\x00\x00\x00\x00\x00\xe0\xbf", + 15: b")\\\x8f\xc2\xf5(\x08\xc0", + 16: b"\x00\x00\x00\x00\x00\x00\x00\x00", + 17: b"\x00\x00\x00\x00\x00\x00\x00\x00", + 18: b"\xf6(\\\x8f\xc2\xc5S\xc0", + 19: b"\x00\x00\x00\x00\x00\x00\x04\xc0", + }, + upper_bounds={ + 2: b"2020-04-30 23:5:", + 3: b"2020-05-01 00:41", + 7: b"\t\x01\x00\x00", + 8: b"\t\x01\x00\x00", + 10: b"\xcd\xcc\xcc\xcc\xcc,_@", + 11: b"\x1f\x85\xebQ\\\xe2\xfe@", + 13: b"\x00\x00\x00\x00\x00\x00\x12@", + 14: b"\x00\x00\x00\x00\x00\x00\xe0?", + 15: b"q=\n\xd7\xa3\xf01@", + 16: b"\x00\x00\x00\x00\x00`B@", + 17: b"333333\xd3?", + 18: b"\x00\x00\x00\x00\x00\x18b@", + 19: b"\x00\x00\x00\x00\x00\x00\x04@", + }, + key_metadata=None, + split_offsets=[4], + equality_ids=None, + sort_order_id=0, + ), + ), + ManifestEntry( + status=ManifestEntryStatus.ADDED, + snapshot_id=8744736658442914487, + sequence_number=None, + data_file=DataFile( + content=DataFileContent.DATA, + file_path="/home/iceberg/warehouse/nyc/taxis_partitioned/data/VendorID=1/00000-633-d8a4223e-dc97-45a1-86e1-adaba6e8abd7-00002.parquet", + file_format=FileFormat.PARQUET, + partition={"VendorID": 1}, + record_count=95050, + file_size_in_bytes=1265950, + block_size_in_bytes=67108864, + column_sizes={ + 1: 318, + 2: 329806, + 3: 331632, + 4: 15343, + 5: 2351, + 6: 3389, + 7: 71269, + 8: 76429, + 9: 16383, + 10: 86992, + 11: 89608, + 12: 265, + 13: 19377, + 14: 1692, + 15: 76162, + 16: 4354, + 17: 759, + 18: 120650, + 19: 11804, + }, + value_counts={ + 1: 95050, + 2: 95050, + 3: 95050, + 4: 95050, + 5: 95050, + 6: 95050, + 7: 95050, + 8: 95050, + 9: 95050, + 10: 95050, + 11: 95050, + 12: 95050, + 13: 95050, + 14: 95050, + 15: 95050, + 16: 95050, + 17: 95050, + 18: 95050, + 19: 95050, + }, + null_value_counts={ + 1: 0, + 2: 0, + 3: 0, + 4: 0, + 5: 0, + 6: 0, + 7: 0, + 8: 0, + 9: 0, + 10: 0, + 11: 0, + 12: 95050, + 13: 0, + 14: 0, + 15: 0, + 16: 0, + 17: 0, + 18: 0, + 19: 0, + }, + nan_value_counts={16: 0, 17: 0, 18: 0, 19: 0, 10: 0, 11: 0, 12: 0, 13: 0, 14: 0, 15: 0}, + distinct_counts=None, + lower_bounds={ + 1: b"\x01\x00\x00\x00", + 2: b"2020-04-01 00:00", + 3: b"2020-04-01 00:03", + 4: b"\x00\x00\x00\x00", + 5: b"\x01\x00\x00\x00", + 6: b"N", + 7: b"\x01\x00\x00\x00", + 8: b"\x01\x00\x00\x00", + 9: b"\x01\x00\x00\x00", + 10: b"\x00\x00\x00\x00\x00\x00\x00\x00", + 11: b"\x00\x00\x00\x00\x00\x00\x00\x00", + 13: b"\x00\x00\x00\x00\x00\x00\x00\x00", + 14: b"\x00\x00\x00\x00\x00\x00\x00\x00", + 15: b"\x00\x00\x00\x00\x00\x00\x00\x00", + 16: b"\x00\x00\x00\x00\x00\x00\x00\x00", + 17: b"\x00\x00\x00\x00\x00\x00\x00\x00", + 18: b"\x00\x00\x00\x00\x00\x00\x00\x00", + 19: b"\x00\x00\x00\x00\x00\x00\x00\x00", + }, + upper_bounds={ + 1: b"\x01\x00\x00\x00", + 2: b"2020-04-30 23:5:", + 3: b"2020-05-01 00:1:", + 4: b"\x06\x00\x00\x00", + 5: b"c\x00\x00\x00", + 6: b"Y", + 7: b"\t\x01\x00\x00", + 8: b"\t\x01\x00\x00", + 9: b"\x04\x00\x00\x00", + 10: b"\\\x8f\xc2\xf5(8\x8c@", + 11: b"\xcd\xcc\xcc\xcc\xcc,f@", + 13: b"\x00\x00\x00\x00\x00\x00\x1c@", + 14: b"\x9a\x99\x99\x99\x99\x99\xf1?", + 15: b"\x00\x00\x00\x00\x00\x00Y@", + 16: b"\x00\x00\x00\x00\x00\xb0X@", + 17: b"333333\xd3?", + 18: b"\xc3\xf5(\\\x8f:\x8c@", + 19: b"\x00\x00\x00\x00\x00\x00\x04@", + }, + key_metadata=None, + split_offsets=[4], + equality_ids=None, + sort_order_id=0, + ), + ), + ] + + assert actual == expected From 518ba94c3506b294de6620ecce4266c0a4423a78 Mon Sep 17 00:00:00 2001 From: Fokko Driesprong Date: Fri, 16 Sep 2022 10:09:25 -0700 Subject: [PATCH 200/642] Python: Bump fsspec and s3fs to 2022.8.2 (#5757) --- poetry.lock | 628 +++++++------------------------------------------ pyproject.toml | 4 +- 2 files changed, 93 insertions(+), 539 deletions(-) diff --git a/poetry.lock b/poetry.lock index c512e41026..1ee6a3fa0d 100644 --- a/poetry.lock +++ b/poetry.lock @@ -1,20 +1,20 @@ [[package]] name = "aiobotocore" -version = "2.3.4" +version = "2.4.0" description = "Async client for aws services using botocore and aiohttp" category = "main" optional = true -python-versions = ">=3.6" +python-versions = ">=3.7" [package.dependencies] aiohttp = ">=3.3.1" aioitertools = ">=0.5.1" -botocore = ">=1.24.21,<1.24.22" +botocore = ">=1.27.59,<1.27.60" wrapt = ">=1.10.10" [package.extras] -awscli = ["awscli (>=1.22.76,<1.22.77)"] -boto3 = ["boto3 (>=1.21.21,<1.21.22)"] +awscli = ["awscli (>=1.25.60,<1.25.61)"] +boto3 = ["boto3 (>=1.24.59,<1.24.60)"] [[package]] name = "aiohttp" @@ -34,7 +34,7 @@ multidict = ">=4.5,<7.0" yarl = ">=1.0,<2.0" [package.extras] -speedups = ["Brotli", "aiodns", "cchardet"] +speedups = ["aiodns", "brotli", "cchardet"] [[package]] name = "aioitertools" @@ -82,11 +82,11 @@ tests_no_zope = ["cloudpickle", "coverage[toml] (>=5.0.2)", "hypothesis", "mypy [[package]] name = "botocore" -version = "1.24.21" +version = "1.27.59" description = "Low-level, data-driven core of boto 3." category = "main" optional = true -python-versions = ">= 3.6" +python-versions = ">= 3.7" [package.dependencies] jmespath = ">=0.7.1,<2.0.0" @@ -94,11 +94,11 @@ python-dateutil = ">=2.1,<3.0.0" urllib3 = ">=1.25.4,<1.27" [package.extras] -crt = ["awscrt (==0.13.5)"] +crt = ["awscrt (==0.14.0)"] [[package]] name = "certifi" -version = "2022.6.15" +version = "2022.6.15.2" description = "Python package for providing Mozilla's CA Bundle." category = "main" optional = false @@ -230,7 +230,7 @@ python-versions = ">=3.7" [[package]] name = "fsspec" -version = "2022.5.0" +version = "2022.8.2" description = "File-system specification" category = "main" optional = false @@ -250,7 +250,7 @@ github = ["requests"] gs = ["gcsfs"] gui = ["panel"] hdfs = ["pyarrow (>=1)"] -http = ["aiohttp", "requests"] +http = ["aiohttp (!=4.0.0a0,!=4.0.0a1)", "requests"] libarchive = ["libarchive-c"] oci = ["ocifs"] s3 = ["s3fs"] @@ -261,7 +261,7 @@ tqdm = ["tqdm"] [[package]] name = "identify" -version = "2.5.3" +version = "2.5.5" description = "File identification library for Python" category = "dev" optional = false @@ -272,7 +272,7 @@ license = ["ukkonen"] [[package]] name = "idna" -version = "3.3" +version = "3.4" description = "Internationalized Domain Names in Applications (IDNA)" category = "main" optional = false @@ -334,12 +334,9 @@ category = "dev" optional = false python-versions = ">=2.7,!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,!=3.4.*,!=3.5.*,!=3.6.*" -[package.dependencies] -setuptools = "*" - [[package]] name = "numpy" -version = "1.23.2" +version = "1.23.3" description = "NumPy is the fundamental package for array computing with Python." category = "main" optional = true @@ -587,33 +584,20 @@ jupyter = ["ipywidgets (>=7.5.1,<8.0.0)"] [[package]] name = "s3fs" -version = "2022.5.0" +version = "2022.8.2" description = "Convenient Filesystem interface over S3" category = "main" optional = true python-versions = ">= 3.7" [package.dependencies] -aiobotocore = ">=2.3.0,<2.4.0" -aiohttp = "<=4" -fsspec = "2022.5.0" +aiobotocore = ">=2.4.0,<2.5.0" +aiohttp = "<4.0.0a0 || >4.0.0a0,<4.0.0a1 || >4.0.0a1" +fsspec = "2022.8.2" [package.extras] -awscli = ["aiobotocore[awscli] (>=2.3.0,<2.4.0)"] -boto3 = ["aiobotocore[boto3] (>=2.3.0,<2.4.0)"] - -[[package]] -name = "setuptools" -version = "65.3.0" -description = "Easily download, build, install, upgrade, and uninstall Python packages" -category = "dev" -optional = false -python-versions = ">=3.7" - -[package.extras] -docs = ["furo", "jaraco.packaging (>=9)", "jaraco.tidelift (>=1.4)", "pygments-github-lexers (==0.0.5)", "rst.linker (>=1.9)", "sphinx", "sphinx-favicon", "sphinx-hoverxref (<2)", "sphinx-inline-tabs", "sphinx-notfound-page (==0.8.3)", "sphinx-reredirects", "sphinxcontrib-towncrier"] -testing = ["build[virtualenv]", "filelock (>=3.4.0)", "flake8 (<5)", "flake8-2020", "ini2toml[lite] (>=0.9)", "jaraco.envs (>=2.2)", "jaraco.path (>=3.2.0)", "mock", "pip (>=19.1)", "pip-run (>=8.8)", "pytest (>=6)", "pytest-black (>=0.3.7)", "pytest-checkdocs (>=2.4)", "pytest-cov", "pytest-enabler (>=1.3)", "pytest-flake8", "pytest-mypy (>=0.9.1)", "pytest-perf", "pytest-xdist", "tomli-w (>=1.0.0)", "virtualenv (>=13.0.0)", "wheel"] -testing-integration = ["build[virtualenv]", "filelock (>=3.4.0)", "jaraco.envs (>=2.2)", "jaraco.path (>=3.2.0)", "pytest", "pytest-enabler", "pytest-xdist", "tomli", "virtualenv (>=13.0.0)", "wheel"] +awscli = ["aiobotocore[awscli] (>=2.4.0,<2.5.0)"] +boto3 = ["aiobotocore[boto3] (>=2.4.0,<2.5.0)"] [[package]] name = "six" @@ -678,7 +662,7 @@ socks = ["PySocks (>=1.5.6,!=1.5.7,<2.0)"] [[package]] name = "virtualenv" -version = "20.16.4" +version = "20.16.5" description = "Virtual Python Environment builder" category = "dev" optional = false @@ -749,110 +733,25 @@ snappy = ["python-snappy"] [metadata] lock-version = "1.1" python-versions = "^3.8" -content-hash = "b77f3ba52efd281107434ef1d70c6d4767ab0b339bd826a66966e040394b719e" +content-hash = "338797a2b74b191ab928e59a772d455a1e454df2c70d1ccae51088d43b06d4b4" [metadata.files] aiobotocore = [ - {file = "aiobotocore-2.3.4-py3-none-any.whl", hash = "sha256:eae059eb51726cee4de2027cfc72bfccc76cf0c229d6b2b08f640e53a568f657"}, - {file = "aiobotocore-2.3.4.tar.gz", hash = "sha256:6554ebea5764f66f4be544a4fcaa0953ee80e600dd7bd818ba4893d72bf12bfb"}, -] -aiohttp = [ - {file = "aiohttp-3.8.1-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:1ed0b6477896559f17b9eaeb6d38e07f7f9ffe40b9f0f9627ae8b9926ae260a8"}, - {file = "aiohttp-3.8.1-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:7dadf3c307b31e0e61689cbf9e06be7a867c563d5a63ce9dca578f956609abf8"}, - {file = "aiohttp-3.8.1-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:a79004bb58748f31ae1cbe9fa891054baaa46fb106c2dc7af9f8e3304dc30316"}, - {file = "aiohttp-3.8.1-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:12de6add4038df8f72fac606dff775791a60f113a725c960f2bab01d8b8e6b15"}, - {file = "aiohttp-3.8.1-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:6f0d5f33feb5f69ddd57a4a4bd3d56c719a141080b445cbf18f238973c5c9923"}, - {file = "aiohttp-3.8.1-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:eaba923151d9deea315be1f3e2b31cc39a6d1d2f682f942905951f4e40200922"}, - {file = "aiohttp-3.8.1-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_12_i686.manylinux2010_i686.whl", hash = "sha256:099ebd2c37ac74cce10a3527d2b49af80243e2a4fa39e7bce41617fbc35fa3c1"}, - {file = "aiohttp-3.8.1-cp310-cp310-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:2e5d962cf7e1d426aa0e528a7e198658cdc8aa4fe87f781d039ad75dcd52c516"}, - {file = "aiohttp-3.8.1-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:fa0ffcace9b3aa34d205d8130f7873fcfefcb6a4dd3dd705b0dab69af6712642"}, - {file = "aiohttp-3.8.1-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:61bfc23df345d8c9716d03717c2ed5e27374e0fe6f659ea64edcd27b4b044cf7"}, - {file = "aiohttp-3.8.1-cp310-cp310-musllinux_1_1_ppc64le.whl", hash = "sha256:31560d268ff62143e92423ef183680b9829b1b482c011713ae941997921eebc8"}, - {file = "aiohttp-3.8.1-cp310-cp310-musllinux_1_1_s390x.whl", hash = "sha256:01d7bdb774a9acc838e6b8f1d114f45303841b89b95984cbb7d80ea41172a9e3"}, - {file = "aiohttp-3.8.1-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:97ef77eb6b044134c0b3a96e16abcb05ecce892965a2124c566af0fd60f717e2"}, - {file = "aiohttp-3.8.1-cp310-cp310-win32.whl", hash = "sha256:c2aef4703f1f2ddc6df17519885dbfa3514929149d3ff900b73f45998f2532fa"}, - {file = "aiohttp-3.8.1-cp310-cp310-win_amd64.whl", hash = "sha256:713ac174a629d39b7c6a3aa757b337599798da4c1157114a314e4e391cd28e32"}, - {file = "aiohttp-3.8.1-cp36-cp36m-macosx_10_9_x86_64.whl", hash = "sha256:473d93d4450880fe278696549f2e7aed8cd23708c3c1997981464475f32137db"}, - {file = "aiohttp-3.8.1-cp36-cp36m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:99b5eeae8e019e7aad8af8bb314fb908dd2e028b3cdaad87ec05095394cce632"}, - {file = "aiohttp-3.8.1-cp36-cp36m-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:3af642b43ce56c24d063325dd2cf20ee012d2b9ba4c3c008755a301aaea720ad"}, - {file = "aiohttp-3.8.1-cp36-cp36m-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:c3630c3ef435c0a7c549ba170a0633a56e92629aeed0e707fec832dee313fb7a"}, - {file = "aiohttp-3.8.1-cp36-cp36m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_12_i686.manylinux2010_i686.whl", hash = "sha256:4a4a4e30bf1edcad13fb0804300557aedd07a92cabc74382fdd0ba6ca2661091"}, - {file = "aiohttp-3.8.1-cp36-cp36m-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:6f8b01295e26c68b3a1b90efb7a89029110d3a4139270b24fda961893216c440"}, - {file = "aiohttp-3.8.1-cp36-cp36m-musllinux_1_1_aarch64.whl", hash = "sha256:a25fa703a527158aaf10dafd956f7d42ac6d30ec80e9a70846253dd13e2f067b"}, - {file = "aiohttp-3.8.1-cp36-cp36m-musllinux_1_1_i686.whl", hash = "sha256:5bfde62d1d2641a1f5173b8c8c2d96ceb4854f54a44c23102e2ccc7e02f003ec"}, - {file = "aiohttp-3.8.1-cp36-cp36m-musllinux_1_1_ppc64le.whl", hash = "sha256:51467000f3647d519272392f484126aa716f747859794ac9924a7aafa86cd411"}, - {file = "aiohttp-3.8.1-cp36-cp36m-musllinux_1_1_s390x.whl", hash = "sha256:03a6d5349c9ee8f79ab3ff3694d6ce1cfc3ced1c9d36200cb8f08ba06bd3b782"}, - {file = "aiohttp-3.8.1-cp36-cp36m-musllinux_1_1_x86_64.whl", hash = "sha256:102e487eeb82afac440581e5d7f8f44560b36cf0bdd11abc51a46c1cd88914d4"}, - {file = "aiohttp-3.8.1-cp36-cp36m-win32.whl", hash = "sha256:4aed991a28ea3ce320dc8ce655875e1e00a11bdd29fe9444dd4f88c30d558602"}, - {file = "aiohttp-3.8.1-cp36-cp36m-win_amd64.whl", hash = "sha256:b0e20cddbd676ab8a64c774fefa0ad787cc506afd844de95da56060348021e96"}, - {file = "aiohttp-3.8.1-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:37951ad2f4a6df6506750a23f7cbabad24c73c65f23f72e95897bb2cecbae676"}, - {file = "aiohttp-3.8.1-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:5c23b1ad869653bc818e972b7a3a79852d0e494e9ab7e1a701a3decc49c20d51"}, - {file = "aiohttp-3.8.1-cp37-cp37m-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:15b09b06dae900777833fe7fc4b4aa426556ce95847a3e8d7548e2d19e34edb8"}, - {file = "aiohttp-3.8.1-cp37-cp37m-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:477c3ea0ba410b2b56b7efb072c36fa91b1e6fc331761798fa3f28bb224830dd"}, - {file = "aiohttp-3.8.1-cp37-cp37m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_12_i686.manylinux2010_i686.whl", hash = "sha256:2f2f69dca064926e79997f45b2f34e202b320fd3782f17a91941f7eb85502ee2"}, - {file = "aiohttp-3.8.1-cp37-cp37m-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:ef9612483cb35171d51d9173647eed5d0069eaa2ee812793a75373447d487aa4"}, - {file = "aiohttp-3.8.1-cp37-cp37m-musllinux_1_1_aarch64.whl", hash = "sha256:6d69f36d445c45cda7b3b26afef2fc34ef5ac0cdc75584a87ef307ee3c8c6d00"}, - {file = "aiohttp-3.8.1-cp37-cp37m-musllinux_1_1_i686.whl", hash = "sha256:55c3d1072704d27401c92339144d199d9de7b52627f724a949fc7d5fc56d8b93"}, - {file = "aiohttp-3.8.1-cp37-cp37m-musllinux_1_1_ppc64le.whl", hash = "sha256:b9d00268fcb9f66fbcc7cd9fe423741d90c75ee029a1d15c09b22d23253c0a44"}, - {file = "aiohttp-3.8.1-cp37-cp37m-musllinux_1_1_s390x.whl", hash = "sha256:07b05cd3305e8a73112103c834e91cd27ce5b4bd07850c4b4dbd1877d3f45be7"}, - {file = "aiohttp-3.8.1-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:c34dc4958b232ef6188c4318cb7b2c2d80521c9a56c52449f8f93ab7bc2a8a1c"}, - {file = "aiohttp-3.8.1-cp37-cp37m-win32.whl", hash = "sha256:d2f9b69293c33aaa53d923032fe227feac867f81682f002ce33ffae978f0a9a9"}, - {file = "aiohttp-3.8.1-cp37-cp37m-win_amd64.whl", hash = "sha256:6ae828d3a003f03ae31915c31fa684b9890ea44c9c989056fea96e3d12a9fa17"}, - {file = "aiohttp-3.8.1-cp38-cp38-macosx_10_9_universal2.whl", hash = "sha256:0c7ebbbde809ff4e970824b2b6cb7e4222be6b95a296e46c03cf050878fc1785"}, - {file = "aiohttp-3.8.1-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:8b7ef7cbd4fec9a1e811a5de813311ed4f7ac7d93e0fda233c9b3e1428f7dd7b"}, - {file = "aiohttp-3.8.1-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:c3d6a4d0619e09dcd61021debf7059955c2004fa29f48788a3dfaf9c9901a7cd"}, - {file = "aiohttp-3.8.1-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:718626a174e7e467f0558954f94af117b7d4695d48eb980146016afa4b580b2e"}, - {file = "aiohttp-3.8.1-cp38-cp38-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:589c72667a5febd36f1315aa6e5f56dd4aa4862df295cb51c769d16142ddd7cd"}, - {file = "aiohttp-3.8.1-cp38-cp38-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:2ed076098b171573161eb146afcb9129b5ff63308960aeca4b676d9d3c35e700"}, - {file = "aiohttp-3.8.1-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_12_i686.manylinux2010_i686.whl", hash = "sha256:086f92daf51a032d062ec5f58af5ca6a44d082c35299c96376a41cbb33034675"}, - {file = "aiohttp-3.8.1-cp38-cp38-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:11691cf4dc5b94236ccc609b70fec991234e7ef8d4c02dd0c9668d1e486f5abf"}, - {file = "aiohttp-3.8.1-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:31d1e1c0dbf19ebccbfd62eff461518dcb1e307b195e93bba60c965a4dcf1ba0"}, - {file = "aiohttp-3.8.1-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:11a67c0d562e07067c4e86bffc1553f2cf5b664d6111c894671b2b8712f3aba5"}, - {file = "aiohttp-3.8.1-cp38-cp38-musllinux_1_1_ppc64le.whl", hash = "sha256:bb01ba6b0d3f6c68b89fce7305080145d4877ad3acaed424bae4d4ee75faa950"}, - {file = "aiohttp-3.8.1-cp38-cp38-musllinux_1_1_s390x.whl", hash = "sha256:44db35a9e15d6fe5c40d74952e803b1d96e964f683b5a78c3cc64eb177878155"}, - {file = "aiohttp-3.8.1-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:844a9b460871ee0a0b0b68a64890dae9c415e513db0f4a7e3cab41a0f2fedf33"}, - {file = "aiohttp-3.8.1-cp38-cp38-win32.whl", hash = "sha256:7d08744e9bae2ca9c382581f7dce1273fe3c9bae94ff572c3626e8da5b193c6a"}, - {file = "aiohttp-3.8.1-cp38-cp38-win_amd64.whl", hash = "sha256:04d48b8ce6ab3cf2097b1855e1505181bdd05586ca275f2505514a6e274e8e75"}, - {file = "aiohttp-3.8.1-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:f5315a2eb0239185af1bddb1abf472d877fede3cc8d143c6cddad37678293237"}, - {file = "aiohttp-3.8.1-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:a996d01ca39b8dfe77440f3cd600825d05841088fd6bc0144cc6c2ec14cc5f74"}, - {file = "aiohttp-3.8.1-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:13487abd2f761d4be7c8ff9080de2671e53fff69711d46de703c310c4c9317ca"}, - {file = "aiohttp-3.8.1-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:ea302f34477fda3f85560a06d9ebdc7fa41e82420e892fc50b577e35fc6a50b2"}, - {file = "aiohttp-3.8.1-cp39-cp39-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:a2f635ce61a89c5732537a7896b6319a8fcfa23ba09bec36e1b1ac0ab31270d2"}, - {file = "aiohttp-3.8.1-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:e999f2d0e12eea01caeecb17b653f3713d758f6dcc770417cf29ef08d3931421"}, - {file = "aiohttp-3.8.1-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_12_i686.manylinux2010_i686.whl", hash = "sha256:0770e2806a30e744b4e21c9d73b7bee18a1cfa3c47991ee2e5a65b887c49d5cf"}, - {file = "aiohttp-3.8.1-cp39-cp39-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:d15367ce87c8e9e09b0f989bfd72dc641bcd04ba091c68cd305312d00962addd"}, - {file = "aiohttp-3.8.1-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:6c7cefb4b0640703eb1069835c02486669312bf2f12b48a748e0a7756d0de33d"}, - {file = "aiohttp-3.8.1-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:71927042ed6365a09a98a6377501af5c9f0a4d38083652bcd2281a06a5976724"}, - {file = "aiohttp-3.8.1-cp39-cp39-musllinux_1_1_ppc64le.whl", hash = "sha256:28d490af82bc6b7ce53ff31337a18a10498303fe66f701ab65ef27e143c3b0ef"}, - {file = "aiohttp-3.8.1-cp39-cp39-musllinux_1_1_s390x.whl", hash = "sha256:b6613280ccedf24354406caf785db748bebbddcf31408b20c0b48cb86af76866"}, - {file = "aiohttp-3.8.1-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:81e3d8c34c623ca4e36c46524a3530e99c0bc95ed068fd6e9b55cb721d408fb2"}, - {file = "aiohttp-3.8.1-cp39-cp39-win32.whl", hash = "sha256:7187a76598bdb895af0adbd2fb7474d7f6025d170bc0a1130242da817ce9e7d1"}, - {file = "aiohttp-3.8.1-cp39-cp39-win_amd64.whl", hash = "sha256:1c182cb873bc91b411e184dab7a2b664d4fea2743df0e4d57402f7f3fa644bac"}, - {file = "aiohttp-3.8.1.tar.gz", hash = "sha256:fc5471e1a54de15ef71c1bc6ebe80d4dc681ea600e68bfd1cbce40427f0b7578"}, -] -aioitertools = [ - {file = "aioitertools-0.10.0-py3-none-any.whl", hash = "sha256:a2ea2a39ebf272a2fbb58bfdb73e1daeeb6686edbbc8082215dfc8b8ffffa6e8"}, - {file = "aioitertools-0.10.0.tar.gz", hash = "sha256:7d1d1d4a03d462c5a0840787d3df098f125847e0d38b833b30f8f8cbc45a1420"}, -] -aiosignal = [ - {file = "aiosignal-1.2.0-py3-none-any.whl", hash = "sha256:26e62109036cd181df6e6ad646f91f0dcfd05fe16d0cb924138ff2ab75d64e3a"}, - {file = "aiosignal-1.2.0.tar.gz", hash = "sha256:78ed67db6c7b7ced4f98e495e572106d5c432a93e1ddd1bf475e1dc05f5b7df2"}, -] -async-timeout = [ - {file = "async-timeout-4.0.2.tar.gz", hash = "sha256:2163e1640ddb52b7a8c80d0a67a08587e5d245cc9c553a74a847056bc2976b15"}, - {file = "async_timeout-4.0.2-py3-none-any.whl", hash = "sha256:8ca1e4fcf50d07413d66d1a5e416e42cfdf5851c981d679a09851a6853383b3c"}, -] -attrs = [ - {file = "attrs-22.1.0-py2.py3-none-any.whl", hash = "sha256:86efa402f67bf2df34f51a335487cf46b1ec130d02b8d39fd248abfd30da551c"}, - {file = "attrs-22.1.0.tar.gz", hash = "sha256:29adc2665447e5191d0e7c568fde78b21f9672d344281d0c6e1ab085429b22b6"}, + {file = "aiobotocore-2.4.0-py3-none-any.whl", hash = "sha256:6c25381d31b712652bc2f6008683949351c240c56d24b1d8ae252c1034a50f63"}, + {file = "aiobotocore-2.4.0.tar.gz", hash = "sha256:f9fe0698cc497861bdb54cd16161c804031f758ada9480c35540f20c0c078385"}, ] +aiohttp = [] +aioitertools = [] +aiosignal = [] +async-timeout = [] +attrs = [] botocore = [ - {file = "botocore-1.24.21-py3-none-any.whl", hash = "sha256:92daca8775e738a9db9b465d533019285f09d541e903233261299fd87c2f842c"}, - {file = "botocore-1.24.21.tar.gz", hash = "sha256:7e976cfd0a61601e74624ef8f5246b40a01f2cce73a011ef29cf80a6e371d0fa"}, + {file = "botocore-1.27.59-py3-none-any.whl", hash = "sha256:69d756791fc024bda54f6c53f71ae34e695ee41bbbc1743d9179c4837a4929da"}, + {file = "botocore-1.27.59.tar.gz", hash = "sha256:eda4aed6ee719a745d1288eaf1beb12f6f6448ad1fa12f159405db14ba9c92cf"}, ] certifi = [ - {file = "certifi-2022.6.15-py3-none-any.whl", hash = "sha256:fe86415d55e84719d75f8b69414f6438ac3547d2078ab91b67e779ef69378412"}, - {file = "certifi-2022.6.15.tar.gz", hash = "sha256:84c85a9078b11105f04f3036a9482ae10e4621616db313fe045dd24743a0820d"}, + {file = "certifi-2022.6.15.2-py3-none-any.whl", hash = "sha256:0aa1a42fbd57645fabeb6290a7687c21755b0344ecaeaa05f4e9f6207ae2e9a8"}, + {file = "certifi-2022.6.15.2.tar.gz", hash = "sha256:aa08c101214127b9b0472ca6338315113c9487d45376fd3e669201b477c71003"}, ] cffi = [ {file = "cffi-1.15.1-cp27-cp27m-macosx_10_9_x86_64.whl", hash = "sha256:a66d3508133af6e8548451b25058d5812812ec3798c886bf38ed24a98216fab2"}, @@ -924,74 +823,14 @@ cfgv = [ {file = "cfgv-3.3.1-py2.py3-none-any.whl", hash = "sha256:c6a0883f3917a037485059700b9e75da2464e6c27051014ad85ba6aaa5884426"}, {file = "cfgv-3.3.1.tar.gz", hash = "sha256:f5a830efb9ce7a445376bb66ec94c638a9787422f96264c98edc6bdeed8ab736"}, ] -charset-normalizer = [ - {file = "charset-normalizer-2.1.1.tar.gz", hash = "sha256:5a3d016c7c547f69d6f81fb0db9449ce888b418b5b9952cc5e6e66843e9dd845"}, - {file = "charset_normalizer-2.1.1-py3-none-any.whl", hash = "sha256:83e9a75d1911279afd89352c68b45348559d1fc0506b054b346651b5e7fee29f"}, -] -click = [ - {file = "click-8.1.3-py3-none-any.whl", hash = "sha256:bb4d8133cb15a609f44e8213d9b391b0809795062913b383c62be0ee95b1db48"}, - {file = "click-8.1.3.tar.gz", hash = "sha256:7682dc8afb30297001674575ea00d1814d808d6a36af415a82bd481d37ba7b8e"}, -] +charset-normalizer = [] +click = [] colorama = [ {file = "colorama-0.4.5-py2.py3-none-any.whl", hash = "sha256:854bf444933e37f5824ae7bfc1e98d5bce2ebe4160d46b5edf346a89358e99da"}, {file = "colorama-0.4.5.tar.gz", hash = "sha256:e6c6b4334fc50988a639d9b98aa429a0b57da6e17b9a44f0451f930b6967b7a4"}, ] -commonmark = [ - {file = "commonmark-0.9.1-py2.py3-none-any.whl", hash = "sha256:da2f38c92590f83de410ba1a3cbceafbc74fee9def35f9251ba9a971d6d66fd9"}, - {file = "commonmark-0.9.1.tar.gz", hash = "sha256:452f9dc859be7f06631ddcb328b6919c67984aca654e5fefb3914d54691aed60"}, -] -coverage = [ - {file = "coverage-6.4.4-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:e7b4da9bafad21ea45a714d3ea6f3e1679099e420c8741c74905b92ee9bfa7cc"}, - {file = "coverage-6.4.4-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:fde17bc42e0716c94bf19d92e4c9f5a00c5feb401f5bc01101fdf2a8b7cacf60"}, - {file = "coverage-6.4.4-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:cdbb0d89923c80dbd435b9cf8bba0ff55585a3cdb28cbec65f376c041472c60d"}, - {file = "coverage-6.4.4-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:67f9346aeebea54e845d29b487eb38ec95f2ecf3558a3cffb26ee3f0dcc3e760"}, - {file = "coverage-6.4.4-cp310-cp310-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:42c499c14efd858b98c4e03595bf914089b98400d30789511577aa44607a1b74"}, - {file = "coverage-6.4.4-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:c35cca192ba700979d20ac43024a82b9b32a60da2f983bec6c0f5b84aead635c"}, - {file = "coverage-6.4.4-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:9cc4f107009bca5a81caef2fca843dbec4215c05e917a59dec0c8db5cff1d2aa"}, - {file = "coverage-6.4.4-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:5f444627b3664b80d078c05fe6a850dd711beeb90d26731f11d492dcbadb6973"}, - {file = "coverage-6.4.4-cp310-cp310-win32.whl", hash = "sha256:66e6df3ac4659a435677d8cd40e8eb1ac7219345d27c41145991ee9bf4b806a0"}, - {file = "coverage-6.4.4-cp310-cp310-win_amd64.whl", hash = "sha256:35ef1f8d8a7a275aa7410d2f2c60fa6443f4a64fae9be671ec0696a68525b875"}, - {file = "coverage-6.4.4-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:c1328d0c2f194ffda30a45f11058c02410e679456276bfa0bbe0b0ee87225fac"}, - {file = "coverage-6.4.4-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:61b993f3998ee384935ee423c3d40894e93277f12482f6e777642a0141f55782"}, - {file = "coverage-6.4.4-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:d5dd4b8e9cd0deb60e6fcc7b0647cbc1da6c33b9e786f9c79721fd303994832f"}, - {file = "coverage-6.4.4-cp311-cp311-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:7026f5afe0d1a933685d8f2169d7c2d2e624f6255fb584ca99ccca8c0e966fd7"}, - {file = "coverage-6.4.4-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:9c7b9b498eb0c0d48b4c2abc0e10c2d78912203f972e0e63e3c9dc21f15abdaa"}, - {file = "coverage-6.4.4-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:ee2b2fb6eb4ace35805f434e0f6409444e1466a47f620d1d5763a22600f0f892"}, - {file = "coverage-6.4.4-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:ab066f5ab67059d1f1000b5e1aa8bbd75b6ed1fc0014559aea41a9eb66fc2ce0"}, - {file = "coverage-6.4.4-cp311-cp311-win32.whl", hash = "sha256:9d6e1f3185cbfd3d91ac77ea065d85d5215d3dfa45b191d14ddfcd952fa53796"}, - {file = "coverage-6.4.4-cp311-cp311-win_amd64.whl", hash = "sha256:e3d3c4cc38b2882f9a15bafd30aec079582b819bec1b8afdbde8f7797008108a"}, - {file = "coverage-6.4.4-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:a095aa0a996ea08b10580908e88fbaf81ecf798e923bbe64fb98d1807db3d68a"}, - {file = "coverage-6.4.4-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:ef6f44409ab02e202b31a05dd6666797f9de2aa2b4b3534e9d450e42dea5e817"}, - {file = "coverage-6.4.4-cp37-cp37m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:4b7101938584d67e6f45f0015b60e24a95bf8dea19836b1709a80342e01b472f"}, - {file = "coverage-6.4.4-cp37-cp37m-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:14a32ec68d721c3d714d9b105c7acf8e0f8a4f4734c811eda75ff3718570b5e3"}, - {file = "coverage-6.4.4-cp37-cp37m-musllinux_1_1_aarch64.whl", hash = "sha256:6a864733b22d3081749450466ac80698fe39c91cb6849b2ef8752fd7482011f3"}, - {file = "coverage-6.4.4-cp37-cp37m-musllinux_1_1_i686.whl", hash = "sha256:08002f9251f51afdcc5e3adf5d5d66bb490ae893d9e21359b085f0e03390a820"}, - {file = "coverage-6.4.4-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:a3b2752de32c455f2521a51bd3ffb53c5b3ae92736afde67ce83477f5c1dd928"}, - {file = "coverage-6.4.4-cp37-cp37m-win32.whl", hash = "sha256:f855b39e4f75abd0dfbcf74a82e84ae3fc260d523fcb3532786bcbbcb158322c"}, - {file = "coverage-6.4.4-cp37-cp37m-win_amd64.whl", hash = "sha256:ee6ae6bbcac0786807295e9687169fba80cb0617852b2fa118a99667e8e6815d"}, - {file = "coverage-6.4.4-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:564cd0f5b5470094df06fab676c6d77547abfdcb09b6c29c8a97c41ad03b103c"}, - {file = "coverage-6.4.4-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:cbbb0e4cd8ddcd5ef47641cfac97d8473ab6b132dd9a46bacb18872828031685"}, - {file = "coverage-6.4.4-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:6113e4df2fa73b80f77663445be6d567913fb3b82a86ceb64e44ae0e4b695de1"}, - {file = "coverage-6.4.4-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:8d032bfc562a52318ae05047a6eb801ff31ccee172dc0d2504614e911d8fa83e"}, - {file = "coverage-6.4.4-cp38-cp38-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:e431e305a1f3126477abe9a184624a85308da8edf8486a863601d58419d26ffa"}, - {file = "coverage-6.4.4-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:cf2afe83a53f77aec067033199797832617890e15bed42f4a1a93ea24794ae3e"}, - {file = "coverage-6.4.4-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:783bc7c4ee524039ca13b6d9b4186a67f8e63d91342c713e88c1865a38d0892a"}, - {file = "coverage-6.4.4-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:ff934ced84054b9018665ca3967fc48e1ac99e811f6cc99ea65978e1d384454b"}, - {file = "coverage-6.4.4-cp38-cp38-win32.whl", hash = "sha256:e1fabd473566fce2cf18ea41171d92814e4ef1495e04471786cbc943b89a3781"}, - {file = "coverage-6.4.4-cp38-cp38-win_amd64.whl", hash = "sha256:4179502f210ebed3ccfe2f78bf8e2d59e50b297b598b100d6c6e3341053066a2"}, - {file = "coverage-6.4.4-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:98c0b9e9b572893cdb0a00e66cf961a238f8d870d4e1dc8e679eb8bdc2eb1b86"}, - {file = "coverage-6.4.4-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:fc600f6ec19b273da1d85817eda339fb46ce9eef3e89f220055d8696e0a06908"}, - {file = "coverage-6.4.4-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:7a98d6bf6d4ca5c07a600c7b4e0c5350cd483c85c736c522b786be90ea5bac4f"}, - {file = "coverage-6.4.4-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:01778769097dbd705a24e221f42be885c544bb91251747a8a3efdec6eb4788f2"}, - {file = "coverage-6.4.4-cp39-cp39-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:dfa0b97eb904255e2ab24166071b27408f1f69c8fbda58e9c0972804851e0558"}, - {file = "coverage-6.4.4-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:fcbe3d9a53e013f8ab88734d7e517eb2cd06b7e689bedf22c0eb68db5e4a0a19"}, - {file = "coverage-6.4.4-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:15e38d853ee224e92ccc9a851457fb1e1f12d7a5df5ae44544ce7863691c7a0d"}, - {file = "coverage-6.4.4-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:6913dddee2deff8ab2512639c5168c3e80b3ebb0f818fed22048ee46f735351a"}, - {file = "coverage-6.4.4-cp39-cp39-win32.whl", hash = "sha256:354df19fefd03b9a13132fa6643527ef7905712109d9c1c1903f2133d3a4e145"}, - {file = "coverage-6.4.4-cp39-cp39-win_amd64.whl", hash = "sha256:1238b08f3576201ebf41f7c20bf59baa0d05da941b123c6656e42cdb668e9827"}, - {file = "coverage-6.4.4-pp36.pp37.pp38-none-any.whl", hash = "sha256:f67cf9f406cf0d2f08a3515ce2db5b82625a7257f88aad87904674def6ddaec1"}, - {file = "coverage-6.4.4.tar.gz", hash = "sha256:e16c45b726acb780e1e6f88b286d3c10b3914ab03438f32117c4aa52d7f30d58"}, -] +commonmark = [] +coverage = [] distlib = [ {file = "distlib-0.3.6-py2.py3-none-any.whl", hash = "sha256:f35c4b692542ca110de7ef0bea44d73981caeb34ca0b9b6b2e6d7790dda8f80e"}, {file = "distlib-0.3.6.tar.gz", hash = "sha256:14bad2d9b04d3a36127ac97f30b12a19268f211063d8f8ee4f47108896e11b46"}, @@ -1019,82 +858,19 @@ fastavro = [ {file = "fastavro-1.6.1-cp39-cp39-win_amd64.whl", hash = "sha256:18f0ddf3cc4f09839c08eec67ed3be3d391894874cb87f856a0362540d18df17"}, {file = "fastavro-1.6.1.tar.gz", hash = "sha256:bc37a6edbde7a04a9df28ab838b94df7527e5eea648d61633233abba59205146"}, ] -filelock = [ - {file = "filelock-3.8.0-py3-none-any.whl", hash = "sha256:617eb4e5eedc82fc5f47b6d61e4d11cb837c56cb4544e39081099fa17ad109d4"}, - {file = "filelock-3.8.0.tar.gz", hash = "sha256:55447caa666f2198c5b6b13a26d2084d26fa5b115c00d065664b2124680c4edc"}, -] -frozenlist = [ - {file = "frozenlist-1.3.1-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:5f271c93f001748fc26ddea409241312a75e13466b06c94798d1a341cf0e6989"}, - {file = "frozenlist-1.3.1-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:9c6ef8014b842f01f5d2b55315f1af5cbfde284eb184075c189fd657c2fd8204"}, - {file = "frozenlist-1.3.1-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:219a9676e2eae91cb5cc695a78b4cb43d8123e4160441d2b6ce8d2c70c60e2f3"}, - {file = "frozenlist-1.3.1-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:b47d64cdd973aede3dd71a9364742c542587db214e63b7529fbb487ed67cddd9"}, - {file = "frozenlist-1.3.1-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:2af6f7a4e93f5d08ee3f9152bce41a6015b5cf87546cb63872cc19b45476e98a"}, - {file = "frozenlist-1.3.1-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:a718b427ff781c4f4e975525edb092ee2cdef6a9e7bc49e15063b088961806f8"}, - {file = "frozenlist-1.3.1-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:c56c299602c70bc1bb5d1e75f7d8c007ca40c9d7aebaf6e4ba52925d88ef826d"}, - {file = "frozenlist-1.3.1-cp310-cp310-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:717470bfafbb9d9be624da7780c4296aa7935294bd43a075139c3d55659038ca"}, - {file = "frozenlist-1.3.1-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:31b44f1feb3630146cffe56344704b730c33e042ffc78d21f2125a6a91168131"}, - {file = "frozenlist-1.3.1-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:c3b31180b82c519b8926e629bf9f19952c743e089c41380ddca5db556817b221"}, - {file = "frozenlist-1.3.1-cp310-cp310-musllinux_1_1_ppc64le.whl", hash = "sha256:d82bed73544e91fb081ab93e3725e45dd8515c675c0e9926b4e1f420a93a6ab9"}, - {file = "frozenlist-1.3.1-cp310-cp310-musllinux_1_1_s390x.whl", hash = "sha256:49459f193324fbd6413e8e03bd65789e5198a9fa3095e03f3620dee2f2dabff2"}, - {file = "frozenlist-1.3.1-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:94e680aeedc7fd3b892b6fa8395b7b7cc4b344046c065ed4e7a1e390084e8cb5"}, - {file = "frozenlist-1.3.1-cp310-cp310-win32.whl", hash = "sha256:fabb953ab913dadc1ff9dcc3a7a7d3dc6a92efab3a0373989b8063347f8705be"}, - {file = "frozenlist-1.3.1-cp310-cp310-win_amd64.whl", hash = "sha256:eee0c5ecb58296580fc495ac99b003f64f82a74f9576a244d04978a7e97166db"}, - {file = "frozenlist-1.3.1-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:0bc75692fb3770cf2b5856a6c2c9de967ca744863c5e89595df64e252e4b3944"}, - {file = "frozenlist-1.3.1-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:086ca1ac0a40e722d6833d4ce74f5bf1aba2c77cbfdc0cd83722ffea6da52a04"}, - {file = "frozenlist-1.3.1-cp37-cp37m-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:1b51eb355e7f813bcda00276b0114c4172872dc5fb30e3fea059b9367c18fbcb"}, - {file = "frozenlist-1.3.1-cp37-cp37m-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:74140933d45271c1a1283f708c35187f94e1256079b3c43f0c2267f9db5845ff"}, - {file = "frozenlist-1.3.1-cp37-cp37m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:ee4c5120ddf7d4dd1eaf079af3af7102b56d919fa13ad55600a4e0ebe532779b"}, - {file = "frozenlist-1.3.1-cp37-cp37m-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:97d9e00f3ac7c18e685320601f91468ec06c58acc185d18bb8e511f196c8d4b2"}, - {file = "frozenlist-1.3.1-cp37-cp37m-musllinux_1_1_aarch64.whl", hash = "sha256:6e19add867cebfb249b4e7beac382d33215d6d54476bb6be46b01f8cafb4878b"}, - {file = "frozenlist-1.3.1-cp37-cp37m-musllinux_1_1_i686.whl", hash = "sha256:a027f8f723d07c3f21963caa7d585dcc9b089335565dabe9c814b5f70c52705a"}, - {file = "frozenlist-1.3.1-cp37-cp37m-musllinux_1_1_ppc64le.whl", hash = "sha256:61d7857950a3139bce035ad0b0945f839532987dfb4c06cfe160254f4d19df03"}, - {file = "frozenlist-1.3.1-cp37-cp37m-musllinux_1_1_s390x.whl", hash = "sha256:53b2b45052e7149ee8b96067793db8ecc1ae1111f2f96fe1f88ea5ad5fd92d10"}, - {file = "frozenlist-1.3.1-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:bbb1a71b1784e68870800b1bc9f3313918edc63dbb8f29fbd2e767ce5821696c"}, - {file = "frozenlist-1.3.1-cp37-cp37m-win32.whl", hash = "sha256:ab6fa8c7871877810e1b4e9392c187a60611fbf0226a9e0b11b7b92f5ac72792"}, - {file = "frozenlist-1.3.1-cp37-cp37m-win_amd64.whl", hash = "sha256:f89139662cc4e65a4813f4babb9ca9544e42bddb823d2ec434e18dad582543bc"}, - {file = "frozenlist-1.3.1-cp38-cp38-macosx_10_9_universal2.whl", hash = "sha256:4c0c99e31491a1d92cde8648f2e7ccad0e9abb181f6ac3ddb9fc48b63301808e"}, - {file = "frozenlist-1.3.1-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:61e8cb51fba9f1f33887e22488bad1e28dd8325b72425f04517a4d285a04c519"}, - {file = "frozenlist-1.3.1-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:cc2f3e368ee5242a2cbe28323a866656006382872c40869b49b265add546703f"}, - {file = "frozenlist-1.3.1-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:58fb94a01414cddcdc6839807db77ae8057d02ddafc94a42faee6004e46c9ba8"}, - {file = "frozenlist-1.3.1-cp38-cp38-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:022178b277cb9277d7d3b3f2762d294f15e85cd2534047e68a118c2bb0058f3e"}, - {file = "frozenlist-1.3.1-cp38-cp38-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:572ce381e9fe027ad5e055f143763637dcbac2542cfe27f1d688846baeef5170"}, - {file = "frozenlist-1.3.1-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:19127f8dcbc157ccb14c30e6f00392f372ddb64a6ffa7106b26ff2196477ee9f"}, - {file = "frozenlist-1.3.1-cp38-cp38-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:42719a8bd3792744c9b523674b752091a7962d0d2d117f0b417a3eba97d1164b"}, - {file = "frozenlist-1.3.1-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:2743bb63095ef306041c8f8ea22bd6e4d91adabf41887b1ad7886c4c1eb43d5f"}, - {file = "frozenlist-1.3.1-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:fa47319a10e0a076709644a0efbcaab9e91902c8bd8ef74c6adb19d320f69b83"}, - {file = "frozenlist-1.3.1-cp38-cp38-musllinux_1_1_ppc64le.whl", hash = "sha256:52137f0aea43e1993264a5180c467a08a3e372ca9d378244c2d86133f948b26b"}, - {file = "frozenlist-1.3.1-cp38-cp38-musllinux_1_1_s390x.whl", hash = "sha256:f5abc8b4d0c5b556ed8cd41490b606fe99293175a82b98e652c3f2711b452988"}, - {file = "frozenlist-1.3.1-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:1e1cf7bc8cbbe6ce3881863671bac258b7d6bfc3706c600008925fb799a256e2"}, - {file = "frozenlist-1.3.1-cp38-cp38-win32.whl", hash = "sha256:0dde791b9b97f189874d654c55c24bf7b6782343e14909c84beebd28b7217845"}, - {file = "frozenlist-1.3.1-cp38-cp38-win_amd64.whl", hash = "sha256:9494122bf39da6422b0972c4579e248867b6b1b50c9b05df7e04a3f30b9a413d"}, - {file = "frozenlist-1.3.1-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:31bf9539284f39ff9398deabf5561c2b0da5bb475590b4e13dd8b268d7a3c5c1"}, - {file = "frozenlist-1.3.1-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:e0c8c803f2f8db7217898d11657cb6042b9b0553a997c4a0601f48a691480fab"}, - {file = "frozenlist-1.3.1-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:da5ba7b59d954f1f214d352308d1d86994d713b13edd4b24a556bcc43d2ddbc3"}, - {file = "frozenlist-1.3.1-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:74e6b2b456f21fc93ce1aff2b9728049f1464428ee2c9752a4b4f61e98c4db96"}, - {file = "frozenlist-1.3.1-cp39-cp39-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:526d5f20e954d103b1d47232e3839f3453c02077b74203e43407b962ab131e7b"}, - {file = "frozenlist-1.3.1-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:b499c6abe62a7a8d023e2c4b2834fce78a6115856ae95522f2f974139814538c"}, - {file = "frozenlist-1.3.1-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:ab386503f53bbbc64d1ad4b6865bf001414930841a870fc97f1546d4d133f141"}, - {file = "frozenlist-1.3.1-cp39-cp39-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:5f63c308f82a7954bf8263a6e6de0adc67c48a8b484fab18ff87f349af356efd"}, - {file = "frozenlist-1.3.1-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:12607804084d2244a7bd4685c9d0dca5df17a6a926d4f1967aa7978b1028f89f"}, - {file = "frozenlist-1.3.1-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:da1cdfa96425cbe51f8afa43e392366ed0b36ce398f08b60de6b97e3ed4affef"}, - {file = "frozenlist-1.3.1-cp39-cp39-musllinux_1_1_ppc64le.whl", hash = "sha256:f810e764617b0748b49a731ffaa525d9bb36ff38332411704c2400125af859a6"}, - {file = "frozenlist-1.3.1-cp39-cp39-musllinux_1_1_s390x.whl", hash = "sha256:35c3d79b81908579beb1fb4e7fcd802b7b4921f1b66055af2578ff7734711cfa"}, - {file = "frozenlist-1.3.1-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:c92deb5d9acce226a501b77307b3b60b264ca21862bd7d3e0c1f3594022f01bc"}, - {file = "frozenlist-1.3.1-cp39-cp39-win32.whl", hash = "sha256:5e77a8bd41e54b05e4fb2708dc6ce28ee70325f8c6f50f3df86a44ecb1d7a19b"}, - {file = "frozenlist-1.3.1-cp39-cp39-win_amd64.whl", hash = "sha256:625d8472c67f2d96f9a4302a947f92a7adbc1e20bedb6aff8dbc8ff039ca6189"}, - {file = "frozenlist-1.3.1.tar.gz", hash = "sha256:3a735e4211a04ccfa3f4833547acdf5d2f863bfeb01cfd3edaffbc251f15cec8"}, -] +filelock = [] +frozenlist = [] fsspec = [ - {file = "fsspec-2022.5.0-py3-none-any.whl", hash = "sha256:2c198c50eb541a80bbd03540b07602c4a957366f3fb416a1f270d34bd4ff0926"}, - {file = "fsspec-2022.5.0.tar.gz", hash = "sha256:7a5459c75c44e760fbe6a3ccb1f37e81e023cde7da8ba20401258d877ec483b4"}, + {file = "fsspec-2022.8.2-py3-none-any.whl", hash = "sha256:6374804a2c0d24f225a67d009ee1eabb4046ad00c793c3f6df97e426c890a1d9"}, + {file = "fsspec-2022.8.2.tar.gz", hash = "sha256:7f12b90964a98a7e921d27fb36be536ea036b73bf3b724ac0b0bd7b8e39c7c18"}, ] identify = [ - {file = "identify-2.5.3-py2.py3-none-any.whl", hash = "sha256:25851c8c1370effb22aaa3c987b30449e9ff0cece408f810ae6ce408fdd20893"}, - {file = "identify-2.5.3.tar.gz", hash = "sha256:887e7b91a1be152b0d46bbf072130235a8117392b9f1828446079a816a05ef44"}, + {file = "identify-2.5.5-py2.py3-none-any.whl", hash = "sha256:ef78c0d96098a3b5fe7720be4a97e73f439af7cf088ebf47b620aeaa10fadf97"}, + {file = "identify-2.5.5.tar.gz", hash = "sha256:322a5699daecf7c6fd60e68852f36f2ecbb6a36ff6e6e973e0d2bb6fca203ee6"}, ] idna = [ - {file = "idna-3.3-py3-none-any.whl", hash = "sha256:84d9dd047ffa80596e0f246e2eab0b391788b0503584e8945f2368256d2735ff"}, - {file = "idna-3.3.tar.gz", hash = "sha256:9d643ff0a55b762d5cdb124b8eaa99c66322e2157b69160bc32796e824360e6d"}, + {file = "idna-3.4-py3-none-any.whl", hash = "sha256:90b77e79eaa3eba6de819a0c442c0b4ceefc341a7a2ab77d7562bf49f425c5c2"}, + {file = "idna-3.4.tar.gz", hash = "sha256:814f528e8dead7d329833b91c5faa87d60bf71824cd12a7530b5526063d02cb4"}, ] importlib-metadata = [ {file = "importlib_metadata-4.12.0-py3-none-any.whl", hash = "sha256:7401a975809ea1fdc658c3aa4f78cc2195a0e019c5cbc4c06122884e9ae80c23"}, @@ -1104,10 +880,7 @@ iniconfig = [ {file = "iniconfig-1.1.1-py2.py3-none-any.whl", hash = "sha256:011e24c64b7f47f6ebd835bb12a743f2fbe9a26d4cecaa7f53bc4f35ee9da8b3"}, {file = "iniconfig-1.1.1.tar.gz", hash = "sha256:bc3af051d7d14b2ee5ef9969666def0cd1a000e121eaea580d4a313df4b37f32"}, ] -jmespath = [ - {file = "jmespath-1.0.1-py3-none-any.whl", hash = "sha256:02e2e4cc71b5bcab88332eebf907519190dd9e6e82107fa7f83b1003a6252980"}, - {file = "jmespath-1.0.1.tar.gz", hash = "sha256:90261b206d6defd58fdd5e85f478bf633a2901798906be2ad389150c5c60edbe"}, -] +jmespath = [] mmh3 = [ {file = "mmh3-3.0.0-cp36-cp36m-macosx_10_9_x86_64.whl", hash = "sha256:23912dde2ad4f701926948dd8e79a0e42b000f73962806f153931f52985e1e07"}, {file = "mmh3-3.0.0-cp36-cp36m-manylinux1_x86_64.whl", hash = "sha256:07f1308a410dc406d6a3c282a685728d00a87f3ed684f012671b96d6cc6a41c3"}, @@ -1136,109 +909,46 @@ mmh3 = [ {file = "mmh3-3.0.0-cp39-cp39-win_amd64.whl", hash = "sha256:bd870aedd9189eff1cf4e1687869b56c7e9461ee869789139c3e704009e5c227"}, {file = "mmh3-3.0.0.tar.gz", hash = "sha256:d1ec578c09a07d3518ec9be540b87546397fa3455de73c166fcce51eaa5c41c5"}, ] -multidict = [ - {file = "multidict-6.0.2-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:0b9e95a740109c6047602f4db4da9949e6c5945cefbad34a1299775ddc9a62e2"}, - {file = "multidict-6.0.2-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:ac0e27844758d7177989ce406acc6a83c16ed4524ebc363c1f748cba184d89d3"}, - {file = "multidict-6.0.2-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:041b81a5f6b38244b34dc18c7b6aba91f9cdaf854d9a39e5ff0b58e2b5773b9c"}, - {file = "multidict-6.0.2-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:5fdda29a3c7e76a064f2477c9aab1ba96fd94e02e386f1e665bca1807fc5386f"}, - {file = "multidict-6.0.2-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:3368bf2398b0e0fcbf46d85795adc4c259299fec50c1416d0f77c0a843a3eed9"}, - {file = "multidict-6.0.2-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:f4f052ee022928d34fe1f4d2bc743f32609fb79ed9c49a1710a5ad6b2198db20"}, - {file = "multidict-6.0.2-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:225383a6603c086e6cef0f2f05564acb4f4d5f019a4e3e983f572b8530f70c88"}, - {file = "multidict-6.0.2-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:50bd442726e288e884f7be9071016c15a8742eb689a593a0cac49ea093eef0a7"}, - {file = "multidict-6.0.2-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:47e6a7e923e9cada7c139531feac59448f1f47727a79076c0b1ee80274cd8eee"}, - {file = "multidict-6.0.2-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:0556a1d4ea2d949efe5fd76a09b4a82e3a4a30700553a6725535098d8d9fb672"}, - {file = "multidict-6.0.2-cp310-cp310-musllinux_1_1_ppc64le.whl", hash = "sha256:626fe10ac87851f4cffecee161fc6f8f9853f0f6f1035b59337a51d29ff3b4f9"}, - {file = "multidict-6.0.2-cp310-cp310-musllinux_1_1_s390x.whl", hash = "sha256:8064b7c6f0af936a741ea1efd18690bacfbae4078c0c385d7c3f611d11f0cf87"}, - {file = "multidict-6.0.2-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:2d36e929d7f6a16d4eb11b250719c39560dd70545356365b494249e2186bc389"}, - {file = "multidict-6.0.2-cp310-cp310-win32.whl", hash = "sha256:fcb91630817aa8b9bc4a74023e4198480587269c272c58b3279875ed7235c293"}, - {file = "multidict-6.0.2-cp310-cp310-win_amd64.whl", hash = "sha256:8cbf0132f3de7cc6c6ce00147cc78e6439ea736cee6bca4f068bcf892b0fd658"}, - {file = "multidict-6.0.2-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:05f6949d6169878a03e607a21e3b862eaf8e356590e8bdae4227eedadacf6e51"}, - {file = "multidict-6.0.2-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:e2c2e459f7050aeb7c1b1276763364884595d47000c1cddb51764c0d8976e608"}, - {file = "multidict-6.0.2-cp37-cp37m-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:d0509e469d48940147e1235d994cd849a8f8195e0bca65f8f5439c56e17872a3"}, - {file = "multidict-6.0.2-cp37-cp37m-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:514fe2b8d750d6cdb4712346a2c5084a80220821a3e91f3f71eec11cf8d28fd4"}, - {file = "multidict-6.0.2-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:19adcfc2a7197cdc3987044e3f415168fc5dc1f720c932eb1ef4f71a2067e08b"}, - {file = "multidict-6.0.2-cp37-cp37m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:b9d153e7f1f9ba0b23ad1568b3b9e17301e23b042c23870f9ee0522dc5cc79e8"}, - {file = "multidict-6.0.2-cp37-cp37m-musllinux_1_1_aarch64.whl", hash = "sha256:aef9cc3d9c7d63d924adac329c33835e0243b5052a6dfcbf7732a921c6e918ba"}, - {file = "multidict-6.0.2-cp37-cp37m-musllinux_1_1_i686.whl", hash = "sha256:4571f1beddff25f3e925eea34268422622963cd8dc395bb8778eb28418248e43"}, - {file = "multidict-6.0.2-cp37-cp37m-musllinux_1_1_ppc64le.whl", hash = "sha256:d48b8ee1d4068561ce8033d2c344cf5232cb29ee1a0206a7b828c79cbc5982b8"}, - {file = "multidict-6.0.2-cp37-cp37m-musllinux_1_1_s390x.whl", hash = "sha256:45183c96ddf61bf96d2684d9fbaf6f3564d86b34cb125761f9a0ef9e36c1d55b"}, - {file = "multidict-6.0.2-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:75bdf08716edde767b09e76829db8c1e5ca9d8bb0a8d4bd94ae1eafe3dac5e15"}, - {file = "multidict-6.0.2-cp37-cp37m-win32.whl", hash = "sha256:a45e1135cb07086833ce969555df39149680e5471c04dfd6a915abd2fc3f6dbc"}, - {file = "multidict-6.0.2-cp37-cp37m-win_amd64.whl", hash = "sha256:6f3cdef8a247d1eafa649085812f8a310e728bdf3900ff6c434eafb2d443b23a"}, - {file = "multidict-6.0.2-cp38-cp38-macosx_10_9_universal2.whl", hash = "sha256:0327292e745a880459ef71be14e709aaea2f783f3537588fb4ed09b6c01bca60"}, - {file = "multidict-6.0.2-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:e875b6086e325bab7e680e4316d667fc0e5e174bb5611eb16b3ea121c8951b86"}, - {file = "multidict-6.0.2-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:feea820722e69451743a3d56ad74948b68bf456984d63c1a92e8347b7b88452d"}, - {file = "multidict-6.0.2-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:9cc57c68cb9139c7cd6fc39f211b02198e69fb90ce4bc4a094cf5fe0d20fd8b0"}, - {file = "multidict-6.0.2-cp38-cp38-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:497988d6b6ec6ed6f87030ec03280b696ca47dbf0648045e4e1d28b80346560d"}, - {file = "multidict-6.0.2-cp38-cp38-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:89171b2c769e03a953d5969b2f272efa931426355b6c0cb508022976a17fd376"}, - {file = "multidict-6.0.2-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:684133b1e1fe91eda8fa7447f137c9490a064c6b7f392aa857bba83a28cfb693"}, - {file = "multidict-6.0.2-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:fd9fc9c4849a07f3635ccffa895d57abce554b467d611a5009ba4f39b78a8849"}, - {file = "multidict-6.0.2-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:e07c8e79d6e6fd37b42f3250dba122053fddb319e84b55dd3a8d6446e1a7ee49"}, - {file = "multidict-6.0.2-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:4070613ea2227da2bfb2c35a6041e4371b0af6b0be57f424fe2318b42a748516"}, - {file = "multidict-6.0.2-cp38-cp38-musllinux_1_1_ppc64le.whl", hash = "sha256:47fbeedbf94bed6547d3aa632075d804867a352d86688c04e606971595460227"}, - {file = "multidict-6.0.2-cp38-cp38-musllinux_1_1_s390x.whl", hash = "sha256:5774d9218d77befa7b70d836004a768fb9aa4fdb53c97498f4d8d3f67bb9cfa9"}, - {file = "multidict-6.0.2-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:2957489cba47c2539a8eb7ab32ff49101439ccf78eab724c828c1a54ff3ff98d"}, - {file = "multidict-6.0.2-cp38-cp38-win32.whl", hash = "sha256:e5b20e9599ba74391ca0cfbd7b328fcc20976823ba19bc573983a25b32e92b57"}, - {file = "multidict-6.0.2-cp38-cp38-win_amd64.whl", hash = "sha256:8004dca28e15b86d1b1372515f32eb6f814bdf6f00952699bdeb541691091f96"}, - {file = "multidict-6.0.2-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:2e4a0785b84fb59e43c18a015ffc575ba93f7d1dbd272b4cdad9f5134b8a006c"}, - {file = "multidict-6.0.2-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:6701bf8a5d03a43375909ac91b6980aea74b0f5402fbe9428fc3f6edf5d9677e"}, - {file = "multidict-6.0.2-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:a007b1638e148c3cfb6bf0bdc4f82776cef0ac487191d093cdc316905e504071"}, - {file = "multidict-6.0.2-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:07a017cfa00c9890011628eab2503bee5872f27144936a52eaab449be5eaf032"}, - {file = "multidict-6.0.2-cp39-cp39-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:c207fff63adcdf5a485969131dc70e4b194327666b7e8a87a97fbc4fd80a53b2"}, - {file = "multidict-6.0.2-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:373ba9d1d061c76462d74e7de1c0c8e267e9791ee8cfefcf6b0b2495762c370c"}, - {file = "multidict-6.0.2-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:bfba7c6d5d7c9099ba21f84662b037a0ffd4a5e6b26ac07d19e423e6fdf965a9"}, - {file = "multidict-6.0.2-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:19d9bad105dfb34eb539c97b132057a4e709919ec4dd883ece5838bcbf262b80"}, - {file = "multidict-6.0.2-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:de989b195c3d636ba000ee4281cd03bb1234635b124bf4cd89eeee9ca8fcb09d"}, - {file = "multidict-6.0.2-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:7c40b7bbece294ae3a87c1bc2abff0ff9beef41d14188cda94ada7bcea99b0fb"}, - {file = "multidict-6.0.2-cp39-cp39-musllinux_1_1_ppc64le.whl", hash = "sha256:d16cce709ebfadc91278a1c005e3c17dd5f71f5098bfae1035149785ea6e9c68"}, - {file = "multidict-6.0.2-cp39-cp39-musllinux_1_1_s390x.whl", hash = "sha256:a2c34a93e1d2aa35fbf1485e5010337c72c6791407d03aa5f4eed920343dd360"}, - {file = "multidict-6.0.2-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:feba80698173761cddd814fa22e88b0661e98cb810f9f986c54aa34d281e4937"}, - {file = "multidict-6.0.2-cp39-cp39-win32.whl", hash = "sha256:23b616fdc3c74c9fe01d76ce0d1ce872d2d396d8fa8e4899398ad64fb5aa214a"}, - {file = "multidict-6.0.2-cp39-cp39-win_amd64.whl", hash = "sha256:4bae31803d708f6f15fd98be6a6ac0b6958fcf68fda3c77a048a4f9073704aae"}, - {file = "multidict-6.0.2.tar.gz", hash = "sha256:5ff3bd75f38e4c43f1f470f2df7a4d430b821c4ce22be384e1459cb57d6bb013"}, -] +multidict = [] nodeenv = [ {file = "nodeenv-1.7.0-py2.py3-none-any.whl", hash = "sha256:27083a7b96a25f2f5e1d8cb4b6317ee8aeda3bdd121394e5ac54e498028a042e"}, {file = "nodeenv-1.7.0.tar.gz", hash = "sha256:e0e7f7dfb85fc5394c6fe1e8fa98131a2473e04311a45afb6508f7cf1836fa2b"}, ] numpy = [ - {file = "numpy-1.23.2-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:e603ca1fb47b913942f3e660a15e55a9ebca906857edfea476ae5f0fe9b457d5"}, - {file = "numpy-1.23.2-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:633679a472934b1c20a12ed0c9a6c9eb167fbb4cb89031939bfd03dd9dbc62b8"}, - {file = "numpy-1.23.2-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:17e5226674f6ea79e14e3b91bfbc153fdf3ac13f5cc54ee7bc8fdbe820a32da0"}, - {file = "numpy-1.23.2-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:bdc02c0235b261925102b1bd586579b7158e9d0d07ecb61148a1799214a4afd5"}, - {file = "numpy-1.23.2-cp310-cp310-win32.whl", hash = "sha256:df28dda02c9328e122661f399f7655cdcbcf22ea42daa3650a26bce08a187450"}, - {file = "numpy-1.23.2-cp310-cp310-win_amd64.whl", hash = "sha256:8ebf7e194b89bc66b78475bd3624d92980fca4e5bb86dda08d677d786fefc414"}, - {file = "numpy-1.23.2-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:dc76bca1ca98f4b122114435f83f1fcf3c0fe48e4e6f660e07996abf2f53903c"}, - {file = "numpy-1.23.2-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:ecfdd68d334a6b97472ed032b5b37a30d8217c097acfff15e8452c710e775524"}, - {file = "numpy-1.23.2-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:5593f67e66dea4e237f5af998d31a43e447786b2154ba1ad833676c788f37cde"}, - {file = "numpy-1.23.2-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:ac987b35df8c2a2eab495ee206658117e9ce867acf3ccb376a19e83070e69418"}, - {file = "numpy-1.23.2-cp311-cp311-win32.whl", hash = "sha256:d98addfd3c8728ee8b2c49126f3c44c703e2b005d4a95998e2167af176a9e722"}, - {file = "numpy-1.23.2-cp311-cp311-win_amd64.whl", hash = "sha256:8ecb818231afe5f0f568c81f12ce50f2b828ff2b27487520d85eb44c71313b9e"}, - {file = "numpy-1.23.2-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:909c56c4d4341ec8315291a105169d8aae732cfb4c250fbc375a1efb7a844f8f"}, - {file = "numpy-1.23.2-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:8247f01c4721479e482cc2f9f7d973f3f47810cbc8c65e38fd1bbd3141cc9842"}, - {file = "numpy-1.23.2-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:b8b97a8a87cadcd3f94659b4ef6ec056261fa1e1c3317f4193ac231d4df70215"}, - {file = "numpy-1.23.2-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:bd5b7ccae24e3d8501ee5563e82febc1771e73bd268eef82a1e8d2b4d556ae66"}, - {file = "numpy-1.23.2-cp38-cp38-win32.whl", hash = "sha256:9b83d48e464f393d46e8dd8171687394d39bc5abfe2978896b77dc2604e8635d"}, - {file = "numpy-1.23.2-cp38-cp38-win_amd64.whl", hash = "sha256:dec198619b7dbd6db58603cd256e092bcadef22a796f778bf87f8592b468441d"}, - {file = "numpy-1.23.2-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:4f41f5bf20d9a521f8cab3a34557cd77b6f205ab2116651f12959714494268b0"}, - {file = "numpy-1.23.2-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:806cc25d5c43e240db709875e947076b2826f47c2c340a5a2f36da5bb10c58d6"}, - {file = "numpy-1.23.2-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:8f9d84a24889ebb4c641a9b99e54adb8cab50972f0166a3abc14c3b93163f074"}, - {file = "numpy-1.23.2-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:c403c81bb8ffb1c993d0165a11493fd4bf1353d258f6997b3ee288b0a48fce77"}, - {file = "numpy-1.23.2-cp39-cp39-win32.whl", hash = "sha256:cf8c6aed12a935abf2e290860af8e77b26a042eb7f2582ff83dc7ed5f963340c"}, - {file = "numpy-1.23.2-cp39-cp39-win_amd64.whl", hash = "sha256:5e28cd64624dc2354a349152599e55308eb6ca95a13ce6a7d5679ebff2962913"}, - {file = "numpy-1.23.2-pp38-pypy38_pp73-macosx_10_9_x86_64.whl", hash = "sha256:806970e69106556d1dd200e26647e9bee5e2b3f1814f9da104a943e8d548ca38"}, - {file = "numpy-1.23.2-pp38-pypy38_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:2bd879d3ca4b6f39b7770829f73278b7c5e248c91d538aab1e506c628353e47f"}, - {file = "numpy-1.23.2-pp38-pypy38_pp73-win_amd64.whl", hash = "sha256:be6b350dfbc7f708d9d853663772a9310783ea58f6035eec649fb9c4371b5389"}, - {file = "numpy-1.23.2.tar.gz", hash = "sha256:b78d00e48261fbbd04aa0d7427cf78d18401ee0abd89c7559bbf422e5b1c7d01"}, + {file = "numpy-1.23.3-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:c9f707b5bb73bf277d812ded9896f9512a43edff72712f31667d0a8c2f8e71ee"}, + {file = "numpy-1.23.3-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:ffcf105ecdd9396e05a8e58e81faaaf34d3f9875f137c7372450baa5d77c9a54"}, + {file = "numpy-1.23.3-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:0ea3f98a0ffce3f8f57675eb9119f3f4edb81888b6874bc1953f91e0b1d4f440"}, + {file = "numpy-1.23.3-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:004f0efcb2fe1c0bd6ae1fcfc69cc8b6bf2407e0f18be308612007a0762b4089"}, + {file = "numpy-1.23.3-cp310-cp310-win32.whl", hash = "sha256:98dcbc02e39b1658dc4b4508442a560fe3ca5ca0d989f0df062534e5ca3a5c1a"}, + {file = "numpy-1.23.3-cp310-cp310-win_amd64.whl", hash = "sha256:39a664e3d26ea854211867d20ebcc8023257c1800ae89773cbba9f9e97bae036"}, + {file = "numpy-1.23.3-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:1f27b5322ac4067e67c8f9378b41c746d8feac8bdd0e0ffede5324667b8a075c"}, + {file = "numpy-1.23.3-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:2ad3ec9a748a8943e6eb4358201f7e1c12ede35f510b1a2221b70af4bb64295c"}, + {file = "numpy-1.23.3-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:bdc9febce3e68b697d931941b263c59e0c74e8f18861f4064c1f712562903411"}, + {file = "numpy-1.23.3-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:301c00cf5e60e08e04d842fc47df641d4a181e651c7135c50dc2762ffe293dbd"}, + {file = "numpy-1.23.3-cp311-cp311-win32.whl", hash = "sha256:7cd1328e5bdf0dee621912f5833648e2daca72e3839ec1d6695e91089625f0b4"}, + {file = "numpy-1.23.3-cp311-cp311-win_amd64.whl", hash = "sha256:8355fc10fd33a5a70981a5b8a0de51d10af3688d7a9e4a34fcc8fa0d7467bb7f"}, + {file = "numpy-1.23.3-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:bc6e8da415f359b578b00bcfb1d08411c96e9a97f9e6c7adada554a0812a6cc6"}, + {file = "numpy-1.23.3-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:22d43376ee0acd547f3149b9ec12eec2f0ca4a6ab2f61753c5b29bb3e795ac4d"}, + {file = "numpy-1.23.3-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a64403f634e5ffdcd85e0b12c08f04b3080d3e840aef118721021f9b48fc1460"}, + {file = "numpy-1.23.3-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:efd9d3abe5774404becdb0748178b48a218f1d8c44e0375475732211ea47c67e"}, + {file = "numpy-1.23.3-cp38-cp38-win32.whl", hash = "sha256:f8c02ec3c4c4fcb718fdf89a6c6f709b14949408e8cf2a2be5bfa9c49548fd85"}, + {file = "numpy-1.23.3-cp38-cp38-win_amd64.whl", hash = "sha256:e868b0389c5ccfc092031a861d4e158ea164d8b7fdbb10e3b5689b4fc6498df6"}, + {file = "numpy-1.23.3-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:09f6b7bdffe57fc61d869a22f506049825d707b288039d30f26a0d0d8ea05164"}, + {file = "numpy-1.23.3-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:8c79d7cf86d049d0c5089231a5bcd31edb03555bd93d81a16870aa98c6cfb79d"}, + {file = "numpy-1.23.3-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:e5d5420053bbb3dd64c30e58f9363d7a9c27444c3648e61460c1237f9ec3fa14"}, + {file = "numpy-1.23.3-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:d5422d6a1ea9b15577a9432e26608c73a78faf0b9039437b075cf322c92e98e7"}, + {file = "numpy-1.23.3-cp39-cp39-win32.whl", hash = "sha256:c1ba66c48b19cc9c2975c0d354f24058888cdc674bebadceb3cdc9ec403fb5d1"}, + {file = "numpy-1.23.3-cp39-cp39-win_amd64.whl", hash = "sha256:78a63d2df1d947bd9d1b11d35564c2f9e4b57898aae4626638056ec1a231c40c"}, + {file = "numpy-1.23.3-pp38-pypy38_pp73-macosx_10_9_x86_64.whl", hash = "sha256:17c0e467ade9bda685d5ac7f5fa729d8d3e76b23195471adae2d6a6941bd2c18"}, + {file = "numpy-1.23.3-pp38-pypy38_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:91b8d6768a75247026e951dce3b2aac79dc7e78622fc148329135ba189813584"}, + {file = "numpy-1.23.3-pp38-pypy38_pp73-win_amd64.whl", hash = "sha256:94c15ca4e52671a59219146ff584488907b1f9b3fc232622b47e2cf832e94fb8"}, + {file = "numpy-1.23.3.tar.gz", hash = "sha256:51bf49c0cd1d52be0a240aa66f3458afc4b95d8993d2d04f0d91fa60c10af6cd"}, ] packaging = [ {file = "packaging-21.3-py3-none-any.whl", hash = "sha256:ef103e05f519cdc783ae24ea4e2e0f508a9c99b2d4969652eed6a2e1ea5bd522"}, {file = "packaging-21.3.tar.gz", hash = "sha256:dd47c42927d89ab911e606518907cc2d3a1f38bbd026385970643f9c5b8ecfeb"}, ] -pep517 = [ - {file = "pep517-0.13.0-py3-none-any.whl", hash = "sha256:4ba4446d80aed5b5eac6509ade100bff3e7943a8489de249654a5ae9b33ee35b"}, - {file = "pep517-0.13.0.tar.gz", hash = "sha256:ae69927c5c172be1add9203726d4b84cf3ebad1edcd5f71fcdc746e66e829f59"}, -] +pep517 = [] platformdirs = [ {file = "platformdirs-2.5.2-py3-none-any.whl", hash = "sha256:027d8e83a2d7de06bbac4e5ef7e023c02b863d7ea5d079477e722bb41ab25788"}, {file = "platformdirs-2.5.2.tar.gz", hash = "sha256:58c8abb07dcb441e6ee4b11d8df0ac856038f944ab98b7be6b27b2a3c7feef19"}, @@ -1247,42 +957,12 @@ pluggy = [ {file = "pluggy-1.0.0-py2.py3-none-any.whl", hash = "sha256:74134bbf457f031a36d68416e1509f34bd5ccc019f0bcc952c7b909d06b37bd3"}, {file = "pluggy-1.0.0.tar.gz", hash = "sha256:4224373bacce55f955a878bf9cfa763c1e360858e330072059e10bad68531159"}, ] -pre-commit = [ - {file = "pre_commit-2.20.0-py2.py3-none-any.whl", hash = "sha256:51a5ba7c480ae8072ecdb6933df22d2f812dc897d5fe848778116129a681aac7"}, - {file = "pre_commit-2.20.0.tar.gz", hash = "sha256:a978dac7bc9ec0bcee55c18a277d553b0f419d259dadb4b9418ff2d00eb43959"}, -] +pre-commit = [] py = [ {file = "py-1.11.0-py2.py3-none-any.whl", hash = "sha256:607c53218732647dff4acdfcd50cb62615cedf612e72d1724fb1a0cc6405b378"}, {file = "py-1.11.0.tar.gz", hash = "sha256:51c75c4126074b472f746a24399ad32f6053d1b34b68d2fa41e558e6f4a98719"}, ] -pyarrow = [ - {file = "pyarrow-9.0.0-cp310-cp310-macosx_10_13_universal2.whl", hash = "sha256:767cafb14278165ad539a2918c14c1b73cf20689747c21375c38e3fe62884902"}, - {file = "pyarrow-9.0.0-cp310-cp310-macosx_10_13_x86_64.whl", hash = "sha256:0238998dc692efcb4e41ae74738d7c1234723271ccf520bd8312dca07d49ef8d"}, - {file = "pyarrow-9.0.0-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:55328348b9139c2b47450d512d716c2248fd58e2f04e2fc23a65e18726666d42"}, - {file = "pyarrow-9.0.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:fc856628acd8d281652c15b6268ec7f27ebcb015abbe99d9baad17f02adc51f1"}, - {file = "pyarrow-9.0.0-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:29eb3e086e2b26202f3a4678316b93cfb15d0e2ba20f3ec12db8fd9cc07cde63"}, - {file = "pyarrow-9.0.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:2e753f8fcf07d8e3a0efa0c8bd51fef5c90281ffd4c5637c08ce42cd0ac297de"}, - {file = "pyarrow-9.0.0-cp310-cp310-win_amd64.whl", hash = "sha256:3eef8a981f45d89de403e81fb83b8119c20824caddf1404274e41a5d66c73806"}, - {file = "pyarrow-9.0.0-cp37-cp37m-macosx_10_13_x86_64.whl", hash = "sha256:7fa56cbd415cef912677270b8e41baad70cde04c6d8a8336eeb2aba85aa93706"}, - {file = "pyarrow-9.0.0-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:f8c46bde1030d704e2796182286d1c56846552c50a39ad5bf5a20c0d8159fc35"}, - {file = "pyarrow-9.0.0-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:8ad430cee28ebc4d6661fc7315747c7a18ae2a74e67498dcb039e1c762a2fb67"}, - {file = "pyarrow-9.0.0-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:81a60bb291a964f63b2717fb1b28f6615ffab7e8585322bfb8a6738e6b321282"}, - {file = "pyarrow-9.0.0-cp37-cp37m-win_amd64.whl", hash = "sha256:9cef618159567d5f62040f2b79b1c7b38e3885f4ffad0ec97cd2d86f88b67cef"}, - {file = "pyarrow-9.0.0-cp38-cp38-macosx_10_13_x86_64.whl", hash = "sha256:5526a3bfb404ff6d31d62ea582cf2466c7378a474a99ee04d1a9b05de5264541"}, - {file = "pyarrow-9.0.0-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:da3e0f319509a5881867effd7024099fb06950a0768dad0d6873668bb88cfaba"}, - {file = "pyarrow-9.0.0-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:2c715eca2092273dcccf6f08437371e04d112f9354245ba2fbe6c801879450b7"}, - {file = "pyarrow-9.0.0-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:f11a645a41ee531c3a5edda45dea07c42267f52571f818d388971d33fc7e2d4a"}, - {file = "pyarrow-9.0.0-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:a5b390bdcfb8c5b900ef543f911cdfec63e88524fafbcc15f83767202a4a2491"}, - {file = "pyarrow-9.0.0-cp38-cp38-win_amd64.whl", hash = "sha256:d9eb04db626fa24fdfb83c00f76679ca0d98728cdbaa0481b6402bf793a290c0"}, - {file = "pyarrow-9.0.0-cp39-cp39-macosx_10_13_universal2.whl", hash = "sha256:4eebdab05afa23d5d5274b24c1cbeb1ba017d67c280f7d39fd8a8f18cbad2ec9"}, - {file = "pyarrow-9.0.0-cp39-cp39-macosx_10_13_x86_64.whl", hash = "sha256:02b820ecd1da02012092c180447de449fc688d0c3f9ff8526ca301cdd60dacd0"}, - {file = "pyarrow-9.0.0-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:92f3977e901db1ef5cba30d6cc1d7942b8d94b910c60f89013e8f7bb86a86eef"}, - {file = "pyarrow-9.0.0-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:f241bd488c2705df930eedfe304ada71191dcf67d6b98ceda0cc934fd2a8388e"}, - {file = "pyarrow-9.0.0-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:1c5a073a930c632058461547e0bc572da1e724b17b6b9eb31a97da13f50cb6e0"}, - {file = "pyarrow-9.0.0-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:f59bcd5217a3ae1e17870792f82b2ff92df9f3862996e2c78e156c13e56ff62e"}, - {file = "pyarrow-9.0.0-cp39-cp39-win_amd64.whl", hash = "sha256:fe2ce795fa1d95e4e940fe5661c3c58aee7181c730f65ac5dd8794a77228de59"}, - {file = "pyarrow-9.0.0.tar.gz", hash = "sha256:7fb02bebc13ab55573d1ae9bb5002a6d20ba767bf8569b52fce5301d42495ab7"}, -] +pyarrow = [] pycparser = [ {file = "pycparser-2.21-py2.py3-none-any.whl", hash = "sha256:8ee45429555515e1f6b185e78100aea234072576aa43ab53aefcae078162fca9"}, {file = "pycparser-2.21.tar.gz", hash = "sha256:e644fdec12f7872f86c58ff790da456218b10f863970249516d60a5eaca77206"}, @@ -1325,10 +1005,7 @@ pydantic = [ {file = "pydantic-1.10.2-py3-none-any.whl", hash = "sha256:1b6ee725bd6e83ec78b1aa32c5b1fa67a3a65badddde3976bca5fe4568f27709"}, {file = "pydantic-1.10.2.tar.gz", hash = "sha256:91b8e218852ef6007c2b98cd861601c6a09f1aa32bbbb74fab5b1c33d4a1e410"}, ] -pygments = [ - {file = "Pygments-2.13.0-py3-none-any.whl", hash = "sha256:f643f331ab57ba3c9d89212ee4a2dabc6e94f117cf4eefde99a0574720d14c42"}, - {file = "Pygments-2.13.0.tar.gz", hash = "sha256:56a8508ae95f98e2b9bdf93a6be5ae3f7d8af858b43e02c5a2ff083726be40c1"}, -] +pygments = [] pyparsing = [ {file = "pyparsing-3.0.9-py3-none-any.whl", hash = "sha256:5026bae9a10eeaefb61dab2f09052b9f4307d44aee4eda64b309723d8d206bbc"}, {file = "pyparsing-3.0.9.tar.gz", hash = "sha256:2b020ecf7d21b687f219b71ecad3631f644a47f01403fa1d1036b0c6416d70fb"}, @@ -1341,10 +1018,7 @@ pytest-checkdocs = [ {file = "pytest-checkdocs-2.7.1.tar.gz", hash = "sha256:2b33b85eddfe5846a69bea4a759303e2d5a3be11d03bc7149f5ba1ef47e6c1ae"}, {file = "pytest_checkdocs-2.7.1-py3-none-any.whl", hash = "sha256:294898c64c9ce1a178edc6660e48da23c7543bfd5a1cea7f0ca4c167745d8461"}, ] -python-dateutil = [ - {file = "python-dateutil-2.8.2.tar.gz", hash = "sha256:0123cacc1627ae19ddf3c27a5de5bd67ee4586fbdd6440d9748f8abb483d3e86"}, - {file = "python_dateutil-2.8.2-py2.py3-none-any.whl", hash = "sha256:961d03dc3453ebbc59dbdea9e4e11c5651520a876d0f4db161e8674aae935da9"}, -] +python-dateutil = [] python-snappy = [ {file = "python-snappy-0.6.1.tar.gz", hash = "sha256:b6a107ab06206acc5359d4c5632bd9b22d448702a79b3169b0c62e0fb808bb2a"}, {file = "python_snappy-0.6.1-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:b7f920eaf46ebf41bd26f9df51c160d40f9e00b7b48471c3438cb8d027f7fb9b"}, @@ -1438,25 +1112,16 @@ requests-mock = [ {file = "requests-mock-1.10.0.tar.gz", hash = "sha256:59c9c32419a9fb1ae83ec242d98e889c45bd7d7a65d48375cc243ec08441658b"}, {file = "requests_mock-1.10.0-py2.py3-none-any.whl", hash = "sha256:2fdbb637ad17ee15c06f33d31169e71bf9fe2bdb7bc9da26185be0dd8d842699"}, ] -rich = [ - {file = "rich-12.5.1-py3-none-any.whl", hash = "sha256:2eb4e6894cde1e017976d2975ac210ef515d7548bc595ba20e195fb9628acdeb"}, - {file = "rich-12.5.1.tar.gz", hash = "sha256:63a5c5ce3673d3d5fbbf23cd87e11ab84b6b451436f1b7f19ec54b6bc36ed7ca"}, -] +rich = [] s3fs = [ - {file = "s3fs-2022.5.0-py3-none-any.whl", hash = "sha256:c7bb327303e931cf641f3cac9b6412a8f1e2908cebd3de30900cb872a4a9f5d2"}, - {file = "s3fs-2022.5.0.tar.gz", hash = "sha256:b40a3cbfaf80cbabaf0e332331117ccac69290efdac347184fb3ab6003f70465"}, -] -setuptools = [ - {file = "setuptools-65.3.0-py3-none-any.whl", hash = "sha256:2e24e0bec025f035a2e72cdd1961119f557d78ad331bb00ff82efb2ab8da8e82"}, - {file = "setuptools-65.3.0.tar.gz", hash = "sha256:7732871f4f7fa58fb6bdcaeadb0161b2bd046c85905dbaa066bdcbcc81953b57"}, + {file = "s3fs-2022.8.2-py3-none-any.whl", hash = "sha256:63369eb9b14687080fc5ac72912f51d5c99d58a24d488199c19e664d0f872fd2"}, + {file = "s3fs-2022.8.2.tar.gz", hash = "sha256:3ca0701a89a9e125a28de90829d19f96f41ddb1d8b4379076c29ed82c4af86cd"}, ] six = [ {file = "six-1.16.0-py2.py3-none-any.whl", hash = "sha256:8abb2f1d86890a2dfb989f9a77cfcfd3e47c2a354b01111771326f8aa26e0254"}, {file = "six-1.16.0.tar.gz", hash = "sha256:1e61c37477a1626458e36f7b1d82aa5c9b094fa4802892072e49de9c60c4c926"}, ] -thrift = [ - {file = "thrift-0.16.0.tar.gz", hash = "sha256:2b5b6488fcded21f9d312aa23c9ff6a0195d0f6ae26ddbd5ad9e3e25dfc14408"}, -] +thrift = [] toml = [ {file = "toml-0.10.2-py2.py3-none-any.whl", hash = "sha256:806143ae5bfb6a3c6e736a764057db0e6a0e05e338b5630894a5f779cabb4f9b"}, {file = "toml-0.10.2.tar.gz", hash = "sha256:b3bda1d108d5dd99f4a20d24d9c348e91c4db7ab1b749200bded2f839ccbe68f"}, @@ -1469,13 +1134,10 @@ typing-extensions = [ {file = "typing_extensions-4.3.0-py3-none-any.whl", hash = "sha256:25642c956049920a5aa49edcdd6ab1e06d7e5d467fc00e0506c44ac86fbfca02"}, {file = "typing_extensions-4.3.0.tar.gz", hash = "sha256:e6d2677a32f47fc7eb2795db1dd15c1f34eff616bcaf2cfb5e997f854fa1c4a6"}, ] -urllib3 = [ - {file = "urllib3-1.26.12-py2.py3-none-any.whl", hash = "sha256:b930dd878d5a8afb066a637fbb35144fe7901e3b209d1cd4f524bd0e9deee997"}, - {file = "urllib3-1.26.12.tar.gz", hash = "sha256:3fa96cf423e6987997fc326ae8df396db2a8b7c667747d47ddd8ecba91f4a74e"}, -] +urllib3 = [] virtualenv = [ - {file = "virtualenv-20.16.4-py3-none-any.whl", hash = "sha256:035ed57acce4ac35c82c9d8802202b0e71adac011a511ff650cbcf9635006a22"}, - {file = "virtualenv-20.16.4.tar.gz", hash = "sha256:014f766e4134d0008dcaa1f95bafa0fb0f575795d07cae50b1bee514185d6782"}, + {file = "virtualenv-20.16.5-py3-none-any.whl", hash = "sha256:d07dfc5df5e4e0dbc92862350ad87a36ed505b978f6c39609dc489eadd5b0d27"}, + {file = "virtualenv-20.16.5.tar.gz", hash = "sha256:227ea1b9994fdc5ea31977ba3383ef296d7472ea85be9d6732e42a91c04e80da"}, ] wrapt = [ {file = "wrapt-1.14.1-cp27-cp27m-macosx_10_9_x86_64.whl", hash = "sha256:1b376b3f4896e7930f1f772ac4b064ac12598d1c38d04907e696cc4d794b43d3"}, @@ -1543,114 +1205,6 @@ wrapt = [ {file = "wrapt-1.14.1-cp39-cp39-win_amd64.whl", hash = "sha256:dee60e1de1898bde3b238f18340eec6148986da0455d8ba7848d50470a7a32fb"}, {file = "wrapt-1.14.1.tar.gz", hash = "sha256:380a85cf89e0e69b7cfbe2ea9f765f004ff419f34194018a6827ac0e3edfed4d"}, ] -yarl = [ - {file = "yarl-1.8.1-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:abc06b97407868ef38f3d172762f4069323de52f2b70d133d096a48d72215d28"}, - {file = "yarl-1.8.1-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:07b21e274de4c637f3e3b7104694e53260b5fc10d51fb3ec5fed1da8e0f754e3"}, - {file = "yarl-1.8.1-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:9de955d98e02fab288c7718662afb33aab64212ecb368c5dc866d9a57bf48880"}, - {file = "yarl-1.8.1-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:7ec362167e2c9fd178f82f252b6d97669d7245695dc057ee182118042026da40"}, - {file = "yarl-1.8.1-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:20df6ff4089bc86e4a66e3b1380460f864df3dd9dccaf88d6b3385d24405893b"}, - {file = "yarl-1.8.1-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:5999c4662631cb798496535afbd837a102859568adc67d75d2045e31ec3ac497"}, - {file = "yarl-1.8.1-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:ed19b74e81b10b592084a5ad1e70f845f0aacb57577018d31de064e71ffa267a"}, - {file = "yarl-1.8.1-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:1e4808f996ca39a6463f45182e2af2fae55e2560be586d447ce8016f389f626f"}, - {file = "yarl-1.8.1-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:2d800b9c2eaf0684c08be5f50e52bfa2aa920e7163c2ea43f4f431e829b4f0fd"}, - {file = "yarl-1.8.1-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:6628d750041550c5d9da50bb40b5cf28a2e63b9388bac10fedd4f19236ef4957"}, - {file = "yarl-1.8.1-cp310-cp310-musllinux_1_1_ppc64le.whl", hash = "sha256:f5af52738e225fcc526ae64071b7e5342abe03f42e0e8918227b38c9aa711e28"}, - {file = "yarl-1.8.1-cp310-cp310-musllinux_1_1_s390x.whl", hash = "sha256:76577f13333b4fe345c3704811ac7509b31499132ff0181f25ee26619de2c843"}, - {file = "yarl-1.8.1-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:0c03f456522d1ec815893d85fccb5def01ffaa74c1b16ff30f8aaa03eb21e453"}, - {file = "yarl-1.8.1-cp310-cp310-win32.whl", hash = "sha256:ea30a42dc94d42f2ba4d0f7c0ffb4f4f9baa1b23045910c0c32df9c9902cb272"}, - {file = "yarl-1.8.1-cp310-cp310-win_amd64.whl", hash = "sha256:9130ddf1ae9978abe63808b6b60a897e41fccb834408cde79522feb37fb72fb0"}, - {file = "yarl-1.8.1-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:0ab5a138211c1c366404d912824bdcf5545ccba5b3ff52c42c4af4cbdc2c5035"}, - {file = "yarl-1.8.1-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a0fb2cb4204ddb456a8e32381f9a90000429489a25f64e817e6ff94879d432fc"}, - {file = "yarl-1.8.1-cp37-cp37m-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:85cba594433915d5c9a0d14b24cfba0339f57a2fff203a5d4fd070e593307d0b"}, - {file = "yarl-1.8.1-cp37-cp37m-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:1ca7e596c55bd675432b11320b4eacc62310c2145d6801a1f8e9ad160685a231"}, - {file = "yarl-1.8.1-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:d0f77539733e0ec2475ddcd4e26777d08996f8cd55d2aef82ec4d3896687abda"}, - {file = "yarl-1.8.1-cp37-cp37m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:29e256649f42771829974e742061c3501cc50cf16e63f91ed8d1bf98242e5507"}, - {file = "yarl-1.8.1-cp37-cp37m-musllinux_1_1_aarch64.whl", hash = "sha256:7fce6cbc6c170ede0221cc8c91b285f7f3c8b9fe28283b51885ff621bbe0f8ee"}, - {file = "yarl-1.8.1-cp37-cp37m-musllinux_1_1_i686.whl", hash = "sha256:59ddd85a1214862ce7c7c66457f05543b6a275b70a65de366030d56159a979f0"}, - {file = "yarl-1.8.1-cp37-cp37m-musllinux_1_1_ppc64le.whl", hash = "sha256:12768232751689c1a89b0376a96a32bc7633c08da45ad985d0c49ede691f5c0d"}, - {file = "yarl-1.8.1-cp37-cp37m-musllinux_1_1_s390x.whl", hash = "sha256:b19255dde4b4f4c32e012038f2c169bb72e7f081552bea4641cab4d88bc409dd"}, - {file = "yarl-1.8.1-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:6c8148e0b52bf9535c40c48faebb00cb294ee577ca069d21bd5c48d302a83780"}, - {file = "yarl-1.8.1-cp37-cp37m-win32.whl", hash = "sha256:de839c3a1826a909fdbfe05f6fe2167c4ab033f1133757b5936efe2f84904c07"}, - {file = "yarl-1.8.1-cp37-cp37m-win_amd64.whl", hash = "sha256:dd032e8422a52e5a4860e062eb84ac94ea08861d334a4bcaf142a63ce8ad4802"}, - {file = "yarl-1.8.1-cp38-cp38-macosx_10_9_universal2.whl", hash = "sha256:19cd801d6f983918a3f3a39f3a45b553c015c5aac92ccd1fac619bd74beece4a"}, - {file = "yarl-1.8.1-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:6347f1a58e658b97b0a0d1ff7658a03cb79bdbda0331603bed24dd7054a6dea1"}, - {file = "yarl-1.8.1-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:7c0da7e44d0c9108d8b98469338705e07f4bb7dab96dbd8fa4e91b337db42548"}, - {file = "yarl-1.8.1-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:5587bba41399854703212b87071c6d8638fa6e61656385875f8c6dff92b2e461"}, - {file = "yarl-1.8.1-cp38-cp38-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:31a9a04ecccd6b03e2b0e12e82131f1488dea5555a13a4d32f064e22a6003cfe"}, - {file = "yarl-1.8.1-cp38-cp38-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:205904cffd69ae972a1707a1bd3ea7cded594b1d773a0ce66714edf17833cdae"}, - {file = "yarl-1.8.1-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:ea513a25976d21733bff523e0ca836ef1679630ef4ad22d46987d04b372d57fc"}, - {file = "yarl-1.8.1-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:d0b51530877d3ad7a8d47b2fff0c8df3b8f3b8deddf057379ba50b13df2a5eae"}, - {file = "yarl-1.8.1-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:d2b8f245dad9e331540c350285910b20dd913dc86d4ee410c11d48523c4fd546"}, - {file = "yarl-1.8.1-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:ab2a60d57ca88e1d4ca34a10e9fb4ab2ac5ad315543351de3a612bbb0560bead"}, - {file = "yarl-1.8.1-cp38-cp38-musllinux_1_1_ppc64le.whl", hash = "sha256:449c957ffc6bc2309e1fbe67ab7d2c1efca89d3f4912baeb8ead207bb3cc1cd4"}, - {file = "yarl-1.8.1-cp38-cp38-musllinux_1_1_s390x.whl", hash = "sha256:a165442348c211b5dea67c0206fc61366212d7082ba8118c8c5c1c853ea4d82e"}, - {file = "yarl-1.8.1-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:b3ded839a5c5608eec8b6f9ae9a62cb22cd037ea97c627f38ae0841a48f09eae"}, - {file = "yarl-1.8.1-cp38-cp38-win32.whl", hash = "sha256:c1445a0c562ed561d06d8cbc5c8916c6008a31c60bc3655cdd2de1d3bf5174a0"}, - {file = "yarl-1.8.1-cp38-cp38-win_amd64.whl", hash = "sha256:56c11efb0a89700987d05597b08a1efcd78d74c52febe530126785e1b1a285f4"}, - {file = "yarl-1.8.1-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:e80ed5a9939ceb6fda42811542f31c8602be336b1fb977bccb012e83da7e4936"}, - {file = "yarl-1.8.1-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:6afb336e23a793cd3b6476c30f030a0d4c7539cd81649683b5e0c1b0ab0bf350"}, - {file = "yarl-1.8.1-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:4c322cbaa4ed78a8aac89b2174a6df398faf50e5fc12c4c191c40c59d5e28357"}, - {file = "yarl-1.8.1-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:fae37373155f5ef9b403ab48af5136ae9851151f7aacd9926251ab26b953118b"}, - {file = "yarl-1.8.1-cp39-cp39-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:5395da939ffa959974577eff2cbfc24b004a2fb6c346918f39966a5786874e54"}, - {file = "yarl-1.8.1-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:076eede537ab978b605f41db79a56cad2e7efeea2aa6e0fa8f05a26c24a034fb"}, - {file = "yarl-1.8.1-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:3d1a50e461615747dd93c099f297c1994d472b0f4d2db8a64e55b1edf704ec1c"}, - {file = "yarl-1.8.1-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:7de89c8456525650ffa2bb56a3eee6af891e98f498babd43ae307bd42dca98f6"}, - {file = "yarl-1.8.1-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:4a88510731cd8d4befaba5fbd734a7dd914de5ab8132a5b3dde0bbd6c9476c64"}, - {file = "yarl-1.8.1-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:2d93a049d29df172f48bcb09acf9226318e712ce67374f893b460b42cc1380ae"}, - {file = "yarl-1.8.1-cp39-cp39-musllinux_1_1_ppc64le.whl", hash = "sha256:21ac44b763e0eec15746a3d440f5e09ad2ecc8b5f6dcd3ea8cb4773d6d4703e3"}, - {file = "yarl-1.8.1-cp39-cp39-musllinux_1_1_s390x.whl", hash = "sha256:d0272228fabe78ce00a3365ffffd6f643f57a91043e119c289aaba202f4095b0"}, - {file = "yarl-1.8.1-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:99449cd5366fe4608e7226c6cae80873296dfa0cde45d9b498fefa1de315a09e"}, - {file = "yarl-1.8.1-cp39-cp39-win32.whl", hash = "sha256:8b0af1cf36b93cee99a31a545fe91d08223e64390c5ecc5e94c39511832a4bb6"}, - {file = "yarl-1.8.1-cp39-cp39-win_amd64.whl", hash = "sha256:de49d77e968de6626ba7ef4472323f9d2e5a56c1d85b7c0e2a190b2173d3b9be"}, - {file = "yarl-1.8.1.tar.gz", hash = "sha256:af887845b8c2e060eb5605ff72b6f2dd2aab7a761379373fd89d314f4752abbf"}, -] -zipp = [ - {file = "zipp-3.8.1-py3-none-any.whl", hash = "sha256:47c40d7fe183a6f21403a199b3e4192cca5774656965b0a4988ad2f8feb5f009"}, - {file = "zipp-3.8.1.tar.gz", hash = "sha256:05b45f1ee8f807d0cc928485ca40a07cb491cf092ff587c0df9cb1fd154848d2"}, -] -zstandard = [ - {file = "zstandard-0.18.0-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:ef7e8a200e4c8ac9102ed3c90ed2aa379f6b880f63032200909c1be21951f556"}, - {file = "zstandard-0.18.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:2dc466207016564805e56d28375f4f533b525ff50d6776946980dff5465566ac"}, - {file = "zstandard-0.18.0-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:4a2ee1d4f98447f3e5183ecfce5626f983504a4a0c005fbe92e60fa8e5d547ec"}, - {file = "zstandard-0.18.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:d956e2f03c7200d7e61345e0880c292783ec26618d0d921dcad470cb195bbce2"}, - {file = "zstandard-0.18.0-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_12_i686.manylinux2010_i686.whl", hash = "sha256:ce6f59cba9854fd14da5bfe34217a1501143057313966637b7291d1b0267bd1e"}, - {file = "zstandard-0.18.0-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:a7fa67cba473623848b6e88acf8d799b1906178fd883fb3a1da24561c779593b"}, - {file = "zstandard-0.18.0-cp310-cp310-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:cdb44d7284c8c5dd1b66dfb86dda7f4560fa94bfbbc1d2da749ba44831335e32"}, - {file = "zstandard-0.18.0-cp310-cp310-win32.whl", hash = "sha256:63694a376cde0aa8b1971d06ca28e8f8b5f492779cb6ee1cc46bbc3f019a42a5"}, - {file = "zstandard-0.18.0-cp310-cp310-win_amd64.whl", hash = "sha256:702a8324cd90c74d9c8780d02bf55e79da3193c870c9665ad3a11647e3ad1435"}, - {file = "zstandard-0.18.0-cp36-cp36m-macosx_10_9_x86_64.whl", hash = "sha256:46f679bc5dfd938db4fb058218d9dc4db1336ffaf1ea774ff152ecadabd40805"}, - {file = "zstandard-0.18.0-cp36-cp36m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:dc2a4de9f363b3247d472362a65041fe4c0f59e01a2846b15d13046be866a885"}, - {file = "zstandard-0.18.0-cp36-cp36m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:bd3220d7627fd4d26397211cb3b560ec7cc4a94b75cfce89e847e8ce7fabe32d"}, - {file = "zstandard-0.18.0-cp36-cp36m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_12_i686.manylinux2010_i686.whl", hash = "sha256:39e98cf4773234bd9cebf9f9db730e451dfcfe435e220f8921242afda8321887"}, - {file = "zstandard-0.18.0-cp36-cp36m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:5228e596eb1554598c872a337bbe4e5afe41cd1f8b1b15f2e35b50d061e35244"}, - {file = "zstandard-0.18.0-cp36-cp36m-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:d4a8fd45746a6c31e729f35196e80b8f1e9987c59f5ccb8859d7c6a6fbeb9c63"}, - {file = "zstandard-0.18.0-cp36-cp36m-win32.whl", hash = "sha256:4cbb85f29a990c2fdbf7bc63246567061a362ddca886d7fae6f780267c0a9e67"}, - {file = "zstandard-0.18.0-cp36-cp36m-win_amd64.whl", hash = "sha256:bfa6c8549fa18e6497a738b7033c49f94a8e2e30c5fbe2d14d0b5aa8bbc1695d"}, - {file = "zstandard-0.18.0-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:e02043297c1832f2666cd2204f381bef43b10d56929e13c42c10c732c6e3b4ed"}, - {file = "zstandard-0.18.0-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:7231543d38d2b7e02ef7cc78ef7ffd86419437e1114ff08709fe25a160e24bd6"}, - {file = "zstandard-0.18.0-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:c86befac87445927488f5c8f205d11566f64c11519db223e9d282b945fa60dab"}, - {file = "zstandard-0.18.0-cp37-cp37m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_12_i686.manylinux2010_i686.whl", hash = "sha256:999a4e1768f219826ba3fa2064fab1c86dd72fdd47a42536235478c3bb3ca3e2"}, - {file = "zstandard-0.18.0-cp37-cp37m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:9df59cd1cf3c62075ee2a4da767089d19d874ac3ad42b04a71a167e91b384722"}, - {file = "zstandard-0.18.0-cp37-cp37m-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:1be31e9e3f7607ee0cdd60915410a5968b205d3e7aa83b7fcf3dd76dbbdb39e0"}, - {file = "zstandard-0.18.0-cp37-cp37m-win32.whl", hash = "sha256:490d11b705b8ae9dc845431bacc8dd1cef2408aede176620a5cd0cd411027936"}, - {file = "zstandard-0.18.0-cp37-cp37m-win_amd64.whl", hash = "sha256:266aba27fa9cc5e9091d3d325ebab1fa260f64e83e42516d5e73947c70216a5b"}, - {file = "zstandard-0.18.0-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:8b2260c4e07dd0723eadb586de7718b61acca4083a490dda69c5719d79bc715c"}, - {file = "zstandard-0.18.0-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:3af8c2383d02feb6650e9255491ec7d0824f6e6dd2bbe3e521c469c985f31fb1"}, - {file = "zstandard-0.18.0-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:28723a1d2e4df778573b76b321ebe9f3469ac98988104c2af116dd344802c3f8"}, - {file = "zstandard-0.18.0-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:19cac7108ff2c342317fad6dc97604b47a41f403c8f19d0bfc396dfadc3638b8"}, - {file = "zstandard-0.18.0-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_12_i686.manylinux2010_i686.whl", hash = "sha256:76725d1ee83a8915100a310bbad5d9c1fc6397410259c94033b8318d548d9990"}, - {file = "zstandard-0.18.0-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:d716a7694ce1fa60b20bc10f35c4a22be446ef7f514c8dbc8f858b61976de2fb"}, - {file = "zstandard-0.18.0-cp38-cp38-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:49685bf9a55d1ab34bd8423ea22db836ba43a181ac6b045ac4272093d5cb874e"}, - {file = "zstandard-0.18.0-cp38-cp38-win32.whl", hash = "sha256:1af1268a7dc870eb27515fb8db1f3e6c5a555d2b7bcc476fc3bab8886c7265ab"}, - {file = "zstandard-0.18.0-cp38-cp38-win_amd64.whl", hash = "sha256:1dc2d3809e763055a1a6c1a73f2b677320cc9a5aa1a7c6cfb35aee59bddc42d9"}, - {file = "zstandard-0.18.0-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:eea18c1e7442f2aa9aff1bb84550dbb6a1f711faf6e48e7319de8f2b2e923c2a"}, - {file = "zstandard-0.18.0-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:8677ffc6a6096cccbd892e558471c901fd821aba12b7fbc63833c7346f549224"}, - {file = "zstandard-0.18.0-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:083dc08abf03807af9beeb2b6a91c23ad78add2499f828176a3c7b742c44df02"}, - {file = "zstandard-0.18.0-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:c990063664c08169c84474acecc9251ee035871589025cac47c060ff4ec4bc1a"}, - {file = "zstandard-0.18.0-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_12_i686.manylinux2010_i686.whl", hash = "sha256:533db8a6fac6248b2cb2c935e7b92f994efbdeb72e1ffa0b354432e087bb5a3e"}, - {file = "zstandard-0.18.0-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:dbb3cb8a082d62b8a73af42291569d266b05605e017a3d8a06a0e5c30b5f10f0"}, - {file = "zstandard-0.18.0-cp39-cp39-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:d6c85ca5162049ede475b7ec98e87f9390501d44a3d6776ddd504e872464ec25"}, - {file = "zstandard-0.18.0-cp39-cp39-win32.whl", hash = "sha256:75479e7c2b3eebf402c59fbe57d21bc400cefa145ca356ee053b0a08908c5784"}, - {file = "zstandard-0.18.0-cp39-cp39-win_amd64.whl", hash = "sha256:d85bfabad444812133a92fc6fbe463e1d07581dba72f041f07a360e63808b23c"}, - {file = "zstandard-0.18.0.tar.gz", hash = "sha256:0ac0357a0d985b4ff31a854744040d7b5754385d1f98f7145c30e02c6865cb6f"}, -] +yarl = [] +zipp = [] +zstandard = [] diff --git a/pyproject.toml b/pyproject.toml index b57f20d607..aa5ee6cd6c 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -49,7 +49,7 @@ rich = "^12.5.1" pyyaml = "^6.0.0" pydantic = "^1.10.2" -fsspec = "2022.5.0" +fsspec = "2022.8.2" pyarrow = { version = "^9.0.0", optional = true } @@ -59,7 +59,7 @@ python-snappy = { version = "^0.6.1", optional = true } thrift = { version = "^0.16.0", optional = true } -s3fs = { version = "2022.5.0", optional = true} +s3fs = { version = "2022.8.2", optional = true } [tool.poetry.dev-dependencies] pytest = "^7.1.3" From e7147186a79d1439f1d04b0ffe5f8b6d26f09211 Mon Sep 17 00:00:00 2001 From: Fokko Driesprong Date: Mon, 19 Sep 2022 08:37:20 +0200 Subject: [PATCH 201/642] Python: Set the Iceberg Spec version (#5766) We pass in the version of the response that we expect for a certain version. If we change anything in the future in the spec, we can maintain backward compatibility until the version is being bumped. --- pyiceberg/catalog/rest.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/pyiceberg/catalog/rest.py b/pyiceberg/catalog/rest.py index 4931e5671b..23d0ccf942 100644 --- a/pyiceberg/catalog/rest.py +++ b/pyiceberg/catalog/rest.py @@ -58,6 +58,8 @@ from pyiceberg.typedef import EMPTY_DICT from pyiceberg.utils.iceberg_base_model import IcebergBaseModel +ICEBERG_REST_SPEC_VERSION = "0.14.1" + class Endpoints: get_config: str = "config" @@ -195,7 +197,8 @@ def _check_valid_namespace_identifier(self, identifier: Union[str, Identifier]) def headers(self) -> Properties: headers = { "Content-type": "application/json", - "X-Client-Version": __version__, + "X-Client-Version": ICEBERG_REST_SPEC_VERSION, + "User-Agent": f"PyIceberg/{__version__}", } if token := self.properties.get("token"): headers[AUTHORIZATION_HEADER] = f"{BEARER_PREFIX} {token}" From 1f8614f0887d8e5b0e6e516f0b425238d4fa8957 Mon Sep 17 00:00:00 2001 From: Fokko Driesprong Date: Mon, 19 Sep 2022 08:38:19 +0200 Subject: [PATCH 202/642] Python: Add docker-compose for s3 tests (#5750) This PR adds a docker-compose.yml with the right configuration to run the s3 tests --- CONTRIBUTING.md | 8 +++++++- Makefile | 7 +++++++ dev/docker-compose.yml | 46 ++++++++++++++++++++++++++++++++++++++++++ dev/run-minio.sh | 33 ++++++++++++++++++++++++++++++ 4 files changed, 93 insertions(+), 1 deletion(-) create mode 100644 dev/docker-compose.yml create mode 100755 dev/run-minio.sh diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 3a258189f7..91ae8215b4 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -37,7 +37,7 @@ If you want to install the library on the host, you can simply run `pip3 install To set up IDEA with Poetry ([also on Loom](https://www.loom.com/share/6d36464d45f244729d91003e7f671fd2)): - Open up the Python project in IntelliJ -- Make sure that you're on a lastest master (that includes Poetry) +- Make sure that you're on latest master (that includes Poetry) - Go to File -> Project Structure (⌘;) - Go to Platform Settings -> SDKs - Click the + sign -> Add Python SDK @@ -72,6 +72,12 @@ For Python, we use pytest in combination with coverage to maintain 90% code cove make test ``` +By default we ignore the s3 tests that require minio to be running. To run this suite, we can run: + +```bash +make test-s3 +``` + To pass additional arguments to pytest, you can use `PYTEST_ARGS`. *Run pytest in verbose mode* diff --git a/Makefile b/Makefile index 0a67cc7817..14a03c3bdc 100644 --- a/Makefile +++ b/Makefile @@ -27,3 +27,10 @@ test: poetry run coverage report -m --fail-under=90 poetry run coverage html poetry run coverage xml + +test-s3: + sh ./dev/run-minio.sh + poetry run coverage run --source=pyiceberg/ -m pytest tests/ ${PYTEST_ARGS} + poetry run coverage report -m --fail-under=90 + poetry run coverage html + poetry run coverage xml diff --git a/dev/docker-compose.yml b/dev/docker-compose.yml new file mode 100644 index 0000000000..f7fbd6deaa --- /dev/null +++ b/dev/docker-compose.yml @@ -0,0 +1,46 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +version: "3" + +services: + minio: + image: minio/minio + container_name: pyiceberg-minio + environment: + - MINIO_ROOT_USER=admin + - MINIO_ROOT_PASSWORD=password + ports: + - 9001:9001 + - 9000:9000 + command: [ "server", "/data", "--console-address", ":9001" ] + mc: + depends_on: + - minio + image: minio/mc + container_name: pyiceberg-mc + environment: + - AWS_ACCESS_KEY_ID=admin + - AWS_SECRET_ACCESS_KEY=password + - AWS_REGION=us-east-1 + entrypoint: > + /bin/sh -c " + until (/usr/bin/mc config host add minio http://minio:9000 admin password) do echo '...waiting...' && sleep 1; done; + /usr/bin/mc rm -r --force minio/warehouse; + /usr/bin/mc mb minio/warehouse; + /usr/bin/mc policy set public minio/warehouse; + exit 0; + " diff --git a/dev/run-minio.sh b/dev/run-minio.sh new file mode 100755 index 0000000000..0e40be4e9d --- /dev/null +++ b/dev/run-minio.sh @@ -0,0 +1,33 @@ +#!/bin/bash +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# + +set -ex + +if [[ $(docker ps -q --filter "name=pyiceberg-minio" --filter "status=running" ) ]]; then + echo "Minio backend running" +else + docker-compose -f dev/docker-compose.yml kill + docker-compose -f dev/docker-compose.yml up -d + while [[ -z $(docker ps -q --filter "name=pyiceberg-minio" --filter "status=running" ) ]] + do + echo "Waiting for Minio" + sleep 1 + done +fi From 90b95f64f82d92266fdd5954fdc4dfc31a79b65b Mon Sep 17 00:00:00 2001 From: Fokko Driesprong Date: Mon, 19 Sep 2022 08:39:22 +0200 Subject: [PATCH 203/642] Python: Remove the pre-validators (#5686) I would like to remove the pre-validators because they are confusing. Mostly because in the pre-validators the defaults and aliases aren't applied, do we have to check all the cases. Removing those requires setting defaults, and checking them afterward. --- pyiceberg/table/metadata.py | 57 +++++++++++++++++++++---------------- 1 file changed, 32 insertions(+), 25 deletions(-) diff --git a/pyiceberg/table/metadata.py b/pyiceberg/table/metadata.py index b0818fc1e8..d3d1fbbfa5 100644 --- a/pyiceberg/table/metadata.py +++ b/pyiceberg/table/metadata.py @@ -43,17 +43,23 @@ from pyiceberg.utils.datetime import datetime_to_micros from pyiceberg.utils.iceberg_base_model import IcebergBaseModel +CURRENT_SNAPSHOT_ID = "current_snapshot_id" +CURRENT_SCHEMA_ID = "current_schema_id" +SCHEMAS = "schemas" +PARTITION_SPECS = "partition_specs" +SORT_ORDERS = "sort_orders" +REFS = "refs" + INITIAL_SEQUENCE_NUMBER = 0 INITIAL_SPEC_ID = 0 DEFAULT_SCHEMA_ID = 0 -DEFAULT_LAST_PARTITION_ID = 1000 def check_schemas(values: Dict[str, Any]) -> Dict[str, Any]: """Validator to check if the current-schema-id is actually present in schemas""" - current_schema_id = values["current_schema_id"] + current_schema_id = values[CURRENT_SCHEMA_ID] - for schema in values["schemas"]: + for schema in values[SCHEMAS]: if schema.schema_id == current_schema_id: return values @@ -64,7 +70,7 @@ def check_partition_specs(values: Dict[str, Any]) -> Dict[str, Any]: """Validator to check if the default-spec-id is present in partition-specs""" default_spec_id = values["default_spec_id"] - partition_specs: List[PartitionSpec] = values["partition_specs"] + partition_specs: List[PartitionSpec] = values[PARTITION_SPECS] for spec in partition_specs: if spec.spec_id == default_spec_id: return values @@ -77,7 +83,7 @@ def check_sort_orders(values: Dict[str, Any]) -> Dict[str, Any]: default_sort_order_id: int = values["default_sort_order_id"] if default_sort_order_id != UNSORTED_SORT_ORDER_ID: - sort_orders: List[SortOrder] = values["sort_orders"] + sort_orders: List[SortOrder] = values[SORT_ORDERS] for sort_order in sort_orders: if sort_order.order_id == default_sort_order_id: return values @@ -90,20 +96,20 @@ class TableMetadataCommonFields(IcebergBaseModel): """Metadata for an Iceberg table as specified in the Apache Iceberg spec (https://iceberg.apache.org/spec/#iceberg-table-spec)""" - @root_validator(pre=True) + @root_validator(skip_on_failure=True) def cleanup_snapshot_id(cls, data: Dict[str, Any]): - if data.get("current-snapshot-id") == -1: + if data[CURRENT_SNAPSHOT_ID] == -1: # We treat -1 and None the same, by cleaning this up # in a pre-validator, we can simplify the logic later on - data["current-snapshot-id"] = None + data[CURRENT_SNAPSHOT_ID] = None return data @root_validator(skip_on_failure=True) def construct_refs(cls, data: Dict[str, Any]): # This is going to be much nicer as soon as refs is an actual pydantic object - if current_snapshot_id := data.get("current_snapshot_id"): - if MAIN_BRANCH not in data["refs"]: - data["refs"][MAIN_BRANCH] = SnapshotRef(snapshot_id=current_snapshot_id, snapshot_ref_type=SnapshotRefType.BRANCH) + if current_snapshot_id := data.get(CURRENT_SNAPSHOT_ID): + if MAIN_BRANCH not in data[REFS]: + data[REFS][MAIN_BRANCH] = SnapshotRef(snapshot_id=current_snapshot_id, snapshot_ref_type=SnapshotRefType.BRANCH) return data location: str = Field() @@ -137,7 +143,7 @@ def construct_refs(cls, data: Dict[str, Any]): default_spec_id: int = Field(alias="default-spec-id", default=INITIAL_SPEC_ID) """ID of the “current” spec that writers should use by default.""" - last_partition_id: int = Field(alias="last-partition-id") + last_partition_id: Optional[int] = Field(alias="last-partition-id") """An integer; the highest assigned partition field ID across all partition specs for the table. This is used to ensure partition fields are always assigned an unused ID when evolving specs.""" @@ -203,24 +209,20 @@ class TableMetadataV1(TableMetadataCommonFields, IcebergBaseModel): # because bumping the version should be an explicit operation that is up # to the owner of the table. - @root_validator(pre=True) + @root_validator def set_v2_compatible_defaults(cls, data: Dict[str, Any]) -> Dict[str, Any]: """Sets default values to be compatible with the format v2 - Set some sensible defaults for V1, so we comply with the schema - this is in pre=True, meaning that this will be done before validation. - We don't want to make the fields optional, since they are required for V2 - Args: data: The raw arguments when initializing a V1 TableMetadata Returns: The TableMetadata with the defaults applied """ - if data.get("schema") and "schema-id" not in data["schema"]: - data["schema"]["schema-id"] = DEFAULT_SCHEMA_ID - if data.get("partition-spec") and "last-partition-id" not in data: - data["last-partition-id"] = max(spec["field-id"] for spec in data["partition-spec"]) + # When the schema doesn't have an ID + if data.get("schema") and "schema_id" not in data["schema"]: + data["schema"]["schema_id"] = DEFAULT_SCHEMA_ID + return data @root_validator(skip_on_failure=True) @@ -258,11 +260,16 @@ def construct_partition_specs(cls, data: Dict[str, Any]) -> Dict[str, Any]: Returns: The TableMetadata with the partition_specs set, if not provided """ - if not data.get("partition_specs"): + if not data.get(PARTITION_SPECS): fields = data["partition_spec"] - data["partition_specs"] = [PartitionSpec(spec_id=INITIAL_SPEC_ID, fields=fields)] + data[PARTITION_SPECS] = [PartitionSpec(spec_id=INITIAL_SPEC_ID, fields=fields)] else: check_partition_specs(data) + + if "last_partition_id" not in data or data.get("last_partition_id") is None: + if partition_specs := data.get(PARTITION_SPECS): + data["last_partition_id"] = max(spec.last_assigned_field_id for spec in partition_specs) + return data @root_validator(skip_on_failure=True) @@ -278,8 +285,8 @@ def set_sort_orders(cls, data: Dict[str, Any]): Returns: The TableMetadata with the sort_orders set, if not provided """ - if not data.get("sort_orders"): - data["sort_orders"] = [UNSORTED_SORT_ORDER] + if not data.get(SORT_ORDERS): + data[SORT_ORDERS] = [UNSORTED_SORT_ORDER] else: check_sort_orders(data) return data From 7f394c51a0299945f44ba1c6fc38a65cc65ef67d Mon Sep 17 00:00:00 2001 From: Joshua Robinson <3334099+joshuarobinson@users.noreply.github.com> Date: Mon, 19 Sep 2022 15:36:54 +0200 Subject: [PATCH 204/642] Python: PyArrow support for S3/S3A with properties (#5747) --- pyiceberg/io/pyarrow.py | 49 ++++++++++++++++++++------- tests/io/test_io.py | 56 +++++++++++++++++-------------- tests/io/test_pyarrow.py | 71 +++++++++++++++------------------------- 3 files changed, 96 insertions(+), 80 deletions(-) diff --git a/pyiceberg/io/pyarrow.py b/pyiceberg/io/pyarrow.py index c07a1118aa..dd82bbfb77 100644 --- a/pyiceberg/io/pyarrow.py +++ b/pyiceberg/io/pyarrow.py @@ -23,10 +23,16 @@ """ import os -from typing import Union +from functools import lru_cache +from typing import Callable, Tuple, Union from urllib.parse import urlparse -from pyarrow.fs import FileInfo, FileSystem, FileType +from pyarrow.fs import ( + FileInfo, + FileSystem, + FileType, + S3FileSystem, +) from pyiceberg.io import ( FileIO, @@ -35,6 +41,7 @@ OutputFile, OutputStream, ) +from pyiceberg.typedef import EMPTY_DICT, Properties class PyArrowFile(InputFile, OutputFile): @@ -59,12 +66,9 @@ class PyArrowFile(InputFile, OutputFile): >>> # output_file.create().write(b'foobytes') """ - def __init__(self, location: str): - parsed_location = urlparse(location) # Create a ParseResult from the URI - if not parsed_location.scheme: # If no scheme, assume the path is to a local file - self._filesystem, self._path = FileSystem.from_uri(os.path.abspath(location)) - else: - self._filesystem, self._path = FileSystem.from_uri(location) # Infer the proper filesystem + def __init__(self, location: str, path: str, fs: FileSystem): + self._filesystem = fs + self._path = path super().__init__(location=location) def _file_info(self) -> FileInfo: @@ -165,6 +169,24 @@ def to_input_file(self) -> "PyArrowFile": class PyArrowFileIO(FileIO): + def __init__(self, properties: Properties = EMPTY_DICT): + self.get_fs_and_path: Callable = lru_cache(self._get_fs_and_path) + super().__init__(properties=properties) + + def _get_fs_and_path(self, location: str) -> Tuple[FileSystem, str]: + uri = urlparse(location) # Create a ParseResult from the URI + if not uri.scheme: # If no scheme, assume the path is to a local file + return FileSystem.from_uri(os.path.abspath(location)) + elif uri.scheme in {"s3", "s3a", "s3n"}: + client_kwargs = { + "endpoint_override": self.properties.get("s3.endpoint"), + "access_key": self.properties.get("s3.access-key-id"), + "secret_key": self.properties.get("s3.secret-access-key"), + } + return (S3FileSystem(**client_kwargs), uri.netloc + uri.path) + else: + return FileSystem.from_uri(location) # Infer the proper filesystem + def new_input(self, location: str) -> PyArrowFile: """Get a PyArrowFile instance to read bytes from the file at the given location @@ -174,7 +196,8 @@ def new_input(self, location: str) -> PyArrowFile: Returns: PyArrowFile: A PyArrowFile instance for the given location """ - return PyArrowFile(location) + fs, path = self.get_fs_and_path(location) + return PyArrowFile(fs=fs, location=location, path=path) def new_output(self, location: str) -> PyArrowFile: """Get a PyArrowFile instance to write bytes to the file at the given location @@ -185,7 +208,8 @@ def new_output(self, location: str) -> PyArrowFile: Returns: PyArrowFile: A PyArrowFile instance for the given location """ - return PyArrowFile(location) + fs, path = self.get_fs_and_path(location) + return PyArrowFile(fs=fs, location=location, path=path) def delete(self, location: Union[str, InputFile, OutputFile]) -> None: """Delete the file at the given location @@ -201,9 +225,10 @@ def delete(self, location: Union[str, InputFile, OutputFile]) -> None: an AWS error code 15 """ str_path = location.location if isinstance(location, (InputFile, OutputFile)) else location - filesystem, path = FileSystem.from_uri(str_path) # Infer the proper filesystem + fs, path = self.get_fs_and_path(str_path) + try: - filesystem.delete_file(path) + fs.delete_file(path) except FileNotFoundError: raise except PermissionError: diff --git a/tests/io/test_io.py b/tests/io/test_io.py index 0e67870b54..fe04b76f7e 100644 --- a/tests/io/test_io.py +++ b/tests/io/test_io.py @@ -35,7 +35,7 @@ load_file_io, ) from pyiceberg.io.fsspec import FsspecFileIO -from pyiceberg.io.pyarrow import PyArrowFile, PyArrowFileIO +from pyiceberg.io.pyarrow import PyArrowFileIO class LocalInputFile(InputFile): @@ -133,8 +133,8 @@ def delete(self, location: Union[str, InputFile, OutputFile]) -> None: raise FileNotFoundError(f"Cannot delete file, does not exist: {parsed_location.path}") from e -@pytest.mark.parametrize("CustomInputFile", [LocalInputFile, PyArrowFile]) -def test_custom_local_input_file(CustomInputFile): +@pytest.mark.parametrize("CustomFileIO", [LocalFileIO, PyArrowFileIO]) +def test_custom_local_input_file(CustomFileIO): """Test initializing an InputFile implementation to read a local file""" with tempfile.TemporaryDirectory() as tmpdirname: file_location = os.path.join(tmpdirname, "foo.txt") @@ -146,7 +146,7 @@ def test_custom_local_input_file(CustomInputFile): # Instantiate the input file absolute_file_location = os.path.abspath(file_location) - input_file = CustomInputFile(location=f"{absolute_file_location}") + input_file = CustomFileIO().new_input(location=f"{absolute_file_location}") # Test opening and reading the file f = input_file.open() @@ -155,15 +155,15 @@ def test_custom_local_input_file(CustomInputFile): assert len(input_file) == 3 -@pytest.mark.parametrize("CustomOutputFile", [LocalOutputFile, PyArrowFile]) -def test_custom_local_output_file(CustomOutputFile): +@pytest.mark.parametrize("CustomFileIO", [LocalFileIO, PyArrowFileIO]) +def test_custom_local_output_file(CustomFileIO): """Test initializing an OutputFile implementation to write to a local file""" with tempfile.TemporaryDirectory() as tmpdirname: file_location = os.path.join(tmpdirname, "foo.txt") # Instantiate the output file absolute_file_location = os.path.abspath(file_location) - output_file = CustomOutputFile(location=f"{absolute_file_location}") + output_file = CustomFileIO().new_output(location=f"{absolute_file_location}") # Create the output file and write to it f = output_file.create() @@ -176,8 +176,8 @@ def test_custom_local_output_file(CustomOutputFile): assert len(output_file) == 3 -@pytest.mark.parametrize("CustomOutputFile", [LocalOutputFile, PyArrowFile]) -def test_custom_local_output_file_with_overwrite(CustomOutputFile): +@pytest.mark.parametrize("CustomFileIO", [LocalFileIO, PyArrowFileIO]) +def test_custom_local_output_file_with_overwrite(CustomFileIO): """Test initializing an OutputFile implementation to overwrite a local file""" with tempfile.TemporaryDirectory() as tmpdirname: output_file_location = os.path.join(tmpdirname, "foo.txt") @@ -187,7 +187,7 @@ def test_custom_local_output_file_with_overwrite(CustomOutputFile): f.write(b"foo") # Instantiate an output file - output_file = CustomOutputFile(location=f"{output_file_location}") + output_file = CustomFileIO().new_output(location=f"{output_file_location}") # Confirm that a FileExistsError is raised when overwrite=False with pytest.raises(FileExistsError): @@ -201,8 +201,8 @@ def test_custom_local_output_file_with_overwrite(CustomOutputFile): assert f.read() == b"bar" -@pytest.mark.parametrize("CustomFile", [LocalInputFile, LocalOutputFile, PyArrowFile, PyArrowFile]) -def test_custom_file_exists(CustomFile): +@pytest.mark.parametrize("CustomFileIO", [LocalFileIO, PyArrowFileIO]) +def test_custom_file_exists(CustomFileIO): """Test that the exists property returns the proper value for existing and non-existing files""" with tempfile.TemporaryDirectory() as tmpdirname: file_location = os.path.join(tmpdirname, "foo.txt") @@ -218,23 +218,31 @@ def test_custom_file_exists(CustomFile): absolute_file_location = os.path.abspath(file_location) non_existent_absolute_file_location = os.path.abspath(nonexistent_file_location) - # Create File instances - file = CustomFile(location=f"{absolute_file_location}") - non_existent_file = CustomFile(location=f"{non_existent_absolute_file_location}") + # Create InputFile instances + file = CustomFileIO().new_input(location=f"{absolute_file_location}") + non_existent_file = CustomFileIO().new_input(location=f"{non_existent_absolute_file_location}") + + # Test opening and reading the file + assert file.exists() + assert not non_existent_file.exists() + + # Create OutputFile instances + file = CustomFileIO().new_output(location=f"{absolute_file_location}") + non_existent_file = CustomFileIO().new_output(location=f"{non_existent_absolute_file_location}") # Test opening and reading the file assert file.exists() assert not non_existent_file.exists() -@pytest.mark.parametrize("CustomOutputFile", [LocalOutputFile, PyArrowFile]) -def test_output_file_to_input_file(CustomOutputFile): +@pytest.mark.parametrize("CustomFileIO", [LocalFileIO, PyArrowFileIO]) +def test_output_file_to_input_file(CustomFileIO): """Test initializing an InputFile using the `to_input_file()` method on an OutputFile instance""" with tempfile.TemporaryDirectory() as tmpdirname: output_file_location = os.path.join(tmpdirname, "foo.txt") # Create an output file instance - output_file = CustomOutputFile(location=f"{output_file_location}") + output_file = CustomFileIO().new_output(location=f"{output_file_location}") # Create the output file and write to it f = output_file.create() @@ -334,8 +342,8 @@ def test_raise_file_not_found_error_for_fileio_delete(CustomFileIO): assert not os.path.exists(output_file_location) -@pytest.mark.parametrize("CustomFileIO, CustomInputFile", [(LocalFileIO, LocalInputFile), (PyArrowFileIO, PyArrowFile)]) -def test_deleting_local_file_using_file_io_input_file(CustomFileIO, CustomInputFile): +@pytest.mark.parametrize("CustomFileIO", [LocalFileIO, PyArrowFileIO]) +def test_deleting_local_file_using_file_io_input_file(CustomFileIO): """Test deleting a local file by passing an InputFile instance to FileIO.delete(...)""" with tempfile.TemporaryDirectory() as tmpdirname: # Write to the temporary file @@ -350,7 +358,7 @@ def test_deleting_local_file_using_file_io_input_file(CustomFileIO, CustomInputF assert os.path.exists(file_location) # Instantiate the custom InputFile - input_file = CustomInputFile(location=f"{file_location}") + input_file = CustomFileIO().new_input(location=f"{file_location}") # Delete the file using the file-io implementations delete method file_io.delete(input_file) @@ -359,8 +367,8 @@ def test_deleting_local_file_using_file_io_input_file(CustomFileIO, CustomInputF assert not os.path.exists(file_location) -@pytest.mark.parametrize("CustomFileIO, CustomOutputFile", [(LocalFileIO, LocalOutputFile), (PyArrowFileIO, PyArrowFile)]) -def test_deleting_local_file_using_file_io_output_file(CustomFileIO, CustomOutputFile): +@pytest.mark.parametrize("CustomFileIO", [LocalFileIO, PyArrowFileIO]) +def test_deleting_local_file_using_file_io_output_file(CustomFileIO): """Test deleting a local file by passing an OutputFile instance to FileIO.delete(...)""" with tempfile.TemporaryDirectory() as tmpdirname: # Write to the temporary file @@ -375,7 +383,7 @@ def test_deleting_local_file_using_file_io_output_file(CustomFileIO, CustomOutpu assert os.path.exists(file_location) # Instantiate the custom OutputFile - output_file = CustomOutputFile(location=f"{file_location}") + output_file = CustomFileIO().new_output(location=f"{file_location}") # Delete the file using the file-io implementations delete method file_io.delete(output_file) diff --git a/tests/io/test_pyarrow.py b/tests/io/test_pyarrow.py index 43dcd22a8d..1b0b07f671 100644 --- a/tests/io/test_pyarrow.py +++ b/tests/io/test_pyarrow.py @@ -40,7 +40,7 @@ def test_pyarrow_input_file(): # Instantiate the input file absolute_file_location = os.path.abspath(file_location) - input_file = PyArrowFile(location=f"{absolute_file_location}") + input_file = PyArrowFileIO().new_input(location=f"{absolute_file_location}") # Test opening and reading the file f = input_file.open() @@ -58,7 +58,7 @@ def test_pyarrow_output_file(): # Instantiate the output file absolute_file_location = os.path.abspath(file_location) - output_file = PyArrowFile(location=f"{absolute_file_location}") + output_file = PyArrowFileIO().new_output(location=f"{absolute_file_location}") # Create the output file and write to it f = output_file.create() @@ -76,12 +76,12 @@ def test_pyarrow_invalid_scheme(): """Test that a ValueError is raised if a location is provided with an invalid scheme""" with pytest.raises(ValueError) as exc_info: - PyArrowFile("foo://bar/baz.txt") + PyArrowFileIO().new_input("foo://bar/baz.txt") assert ("Unrecognized filesystem type in URI") in str(exc_info.value) with pytest.raises(ValueError) as exc_info: - PyArrowFile("foo://bar/baz.txt") + PyArrowFileIO().new_output("foo://bar/baz.txt") assert ("Unrecognized filesystem type in URI") in str(exc_info.value) @@ -96,8 +96,7 @@ def test_pyarrow_violating_input_stream_protocol(): filesystem_mock = MagicMock() filesystem_mock.open_input_file.return_value = input_file_mock - input_file = PyArrowFile("foo.txt") - input_file._filesystem = filesystem_mock + input_file = PyArrowFile("foo.txt", path="foo.txt", fs=filesystem_mock) f = input_file.open() assert not isinstance(f, InputStream) @@ -118,8 +117,7 @@ def test_pyarrow_violating_output_stream_protocol(): filesystem_mock.open_output_stream.return_value = output_file_mock filesystem_mock.get_file_info.return_value = file_info_mock - output_file = PyArrowFile("foo.txt") - output_file._filesystem = filesystem_mock + output_file = PyArrowFile("foo.txt", path="foo.txt", fs=filesystem_mock) f = output_file.create() @@ -131,7 +129,7 @@ def test_raise_on_opening_a_local_file_not_found(): with tempfile.TemporaryDirectory() as tmpdirname: file_location = os.path.join(tmpdirname, "foo.txt") - f = PyArrowFile(file_location) + f = PyArrowFileIO().new_input(file_location) with pytest.raises(FileNotFoundError) as exc_info: f.open() @@ -145,7 +143,7 @@ def test_raise_on_opening_a_local_file_no_permission(): with tempfile.TemporaryDirectory() as tmpdirname: os.chmod(tmpdirname, 0o600) file_location = os.path.join(tmpdirname, "foo.txt") - f = PyArrowFile(file_location) + f = PyArrowFileIO().new_input(file_location) with pytest.raises(PermissionError) as exc_info: f.open() @@ -159,7 +157,7 @@ def test_raise_on_checking_if_local_file_exists_no_permission(): with tempfile.TemporaryDirectory() as tmpdirname: os.chmod(tmpdirname, 0o600) file_location = os.path.join(tmpdirname, "foo.txt") - f = PyArrowFile(file_location) + f = PyArrowFileIO().new_input(file_location) with pytest.raises(PermissionError) as exc_info: f.create() @@ -173,7 +171,7 @@ def test_raise_on_creating_a_local_file_no_permission(): with tempfile.TemporaryDirectory() as tmpdirname: os.chmod(tmpdirname, 0o600) file_location = os.path.join(tmpdirname, "foo.txt") - f = PyArrowFile(file_location) + f = PyArrowFileIO().new_input(file_location) with pytest.raises(PermissionError) as exc_info: f.create() @@ -195,17 +193,13 @@ def test_raise_on_delete_file_with_no_permission(): assert "Cannot delete file" in str(exc_info.value) -@patch("pyiceberg.io.pyarrow.PyArrowFile.exists", return_value=False) -@patch("pyiceberg.io.pyarrow.FileSystem") -def test_raise_on_opening_an_s3_file_no_permission(filesystem_mock, exists_mock): +def test_raise_on_opening_an_s3_file_no_permission(): """Test that opening a PyArrowFile raises a PermissionError when the pyarrow error includes 'AWS Error [code 15]'""" s3fs_mock = MagicMock() s3fs_mock.open_input_file.side_effect = OSError("AWS Error [code 15]") - filesystem_mock.from_uri.return_value = (s3fs_mock, "foo.txt") - - f = PyArrowFile("s3://foo/bar.txt") + f = PyArrowFile("s3://foo/bar.txt", path="foo/bar.txt", fs=s3fs_mock) with pytest.raises(PermissionError) as exc_info: f.open() @@ -213,17 +207,13 @@ def test_raise_on_opening_an_s3_file_no_permission(filesystem_mock, exists_mock) assert "Cannot open file, access denied:" in str(exc_info.value) -@patch("pyiceberg.io.pyarrow.PyArrowFile.exists", return_value=False) -@patch("pyiceberg.io.pyarrow.FileSystem") -def test_raise_on_opening_an_s3_file_not_found(filesystem_mock, exists_mock): +def test_raise_on_opening_an_s3_file_not_found(): """Test that a PyArrowFile raises a FileNotFoundError when the pyarrow error includes 'Path does not exist'""" s3fs_mock = MagicMock() s3fs_mock.open_input_file.side_effect = OSError("Path does not exist") - filesystem_mock.from_uri.return_value = (s3fs_mock, "foo.txt") - - f = PyArrowFile("s3://foo/bar.txt") + f = PyArrowFile("s3://foo/bar.txt", path="foo/bar.txt", fs=s3fs_mock) with pytest.raises(FileNotFoundError) as exc_info: f.open() @@ -232,16 +222,13 @@ def test_raise_on_opening_an_s3_file_not_found(filesystem_mock, exists_mock): @patch("pyiceberg.io.pyarrow.PyArrowFile.exists", return_value=False) -@patch("pyiceberg.io.pyarrow.FileSystem") -def test_raise_on_creating_an_s3_file_no_permission(filesystem_mock, exists_mock): +def test_raise_on_creating_an_s3_file_no_permission(_): """Test that creating a PyArrowFile raises a PermissionError when the pyarrow error includes 'AWS Error [code 15]'""" s3fs_mock = MagicMock() s3fs_mock.open_output_stream.side_effect = OSError("AWS Error [code 15]") - filesystem_mock.from_uri.return_value = (s3fs_mock, "foo.txt") - - f = PyArrowFile("s3://foo/bar.txt") + f = PyArrowFile("s3://foo/bar.txt", path="foo/bar.txt", fs=s3fs_mock) with pytest.raises(PermissionError) as exc_info: f.create() @@ -249,35 +236,31 @@ def test_raise_on_creating_an_s3_file_no_permission(filesystem_mock, exists_mock assert "Cannot create file, access denied:" in str(exc_info.value) -@patch("pyiceberg.io.pyarrow.FileSystem") -def test_deleting_s3_file_no_permission(filesystem_mock): +def test_deleting_s3_file_no_permission(): """Test that a PyArrowFile raises a PermissionError when the pyarrow OSError includes 'AWS Error [code 15]'""" s3fs_mock = MagicMock() s3fs_mock.delete_file.side_effect = OSError("AWS Error [code 15]") - filesystem_mock.from_uri.return_value = (s3fs_mock, "foo.txt") - - file_io = PyArrowFileIO() + with patch.object(PyArrowFileIO, "_get_fs_and_path") as submocked: + submocked.return_value = (s3fs_mock, "bar/foo.txt") - with pytest.raises(PermissionError) as exc_info: - file_io.delete("s3://foo/bar.txt") + with pytest.raises(PermissionError) as exc_info: + PyArrowFileIO().delete("s3://foo/bar.txt") assert "Cannot delete file, access denied:" in str(exc_info.value) -@patch("pyiceberg.io.pyarrow.FileSystem") -def test_deleting_s3_file_not_found(filesystem_mock): +def test_deleting_s3_file_not_found(): """Test that a PyArrowFile raises a PermissionError when the pyarrow error includes 'AWS Error [code 15]'""" s3fs_mock = MagicMock() s3fs_mock.delete_file.side_effect = OSError("Path does not exist") - filesystem_mock.from_uri.return_value = (s3fs_mock, "foo.txt") - - file_io = PyArrowFileIO() + with patch.object(PyArrowFileIO, "_get_fs_and_path") as submocked: + submocked.return_value = (s3fs_mock, "bar/foo.txt") - with pytest.raises(FileNotFoundError) as exc_info: - file_io.delete("s3://foo/bar.txt") + with pytest.raises(FileNotFoundError) as exc_info: + PyArrowFileIO().delete("s3://foo/bar.txt") - assert "Cannot delete file, does not exist:" in str(exc_info.value) + assert "Cannot delete file, does not exist:" in str(exc_info.value) From 7c0983d0f5170b4a63dd187ec49109156583719b Mon Sep 17 00:00:00 2001 From: Fokko Driesprong Date: Tue, 20 Sep 2022 04:11:08 +0200 Subject: [PATCH 205/642] Python: Fine-tune the API (#5672) --- pyiceberg/schema.py | 4 +--- pyiceberg/table/metadata.py | 8 ++++++-- pyiceberg/table/partitioning.py | 21 ++++++++++--------- pyiceberg/table/sorting.py | 25 ++++++++++++----------- pyproject.toml | 5 ++--- tests/catalog/test_base.py | 8 ++++---- tests/catalog/test_hive.py | 23 ++++++++++++--------- tests/catalog/test_rest.py | 17 +++++++--------- tests/table/test_init.py | 4 ++-- tests/table/test_metadata.py | 23 ++++++++++++++------- tests/table/test_partitioning.py | 35 ++++++++++++++++---------------- tests/table/test_sorting.py | 11 +++++++--- tests/test_schema.py | 28 ++++++++++++------------- 13 files changed, 114 insertions(+), 98 deletions(-) diff --git a/pyiceberg/schema.py b/pyiceberg/schema.py index 6620759f86..b0fc41b0e1 100644 --- a/pyiceberg/schema.py +++ b/pyiceberg/schema.py @@ -75,9 +75,7 @@ def __str__(self): return "table {\n" + "\n".join([" " + str(field) for field in self.columns]) + "\n}" def __repr__(self): - return ( - f"Schema(fields={repr(self.columns)}, schema_id={self.schema_id}, identifier_field_ids={self.identifier_field_ids})" - ) + return f"Schema({', '.join(repr(column) for column in self.columns)}, schema_id={self.schema_id}, identifier_field_ids={self.identifier_field_ids})" def __eq__(self, other) -> bool: if not other: diff --git a/pyiceberg/table/metadata.py b/pyiceberg/table/metadata.py index d3d1fbbfa5..49cd323e5f 100644 --- a/pyiceberg/table/metadata.py +++ b/pyiceberg/table/metadata.py @@ -46,6 +46,8 @@ CURRENT_SNAPSHOT_ID = "current_snapshot_id" CURRENT_SCHEMA_ID = "current_schema_id" SCHEMAS = "schemas" +DEFAULT_SPEC_ID = "default_spec_id" +PARTITION_SPEC = "partition_spec" PARTITION_SPECS = "partition_specs" SORT_ORDERS = "sort_orders" REFS = "refs" @@ -261,8 +263,10 @@ def construct_partition_specs(cls, data: Dict[str, Any]) -> Dict[str, Any]: The TableMetadata with the partition_specs set, if not provided """ if not data.get(PARTITION_SPECS): - fields = data["partition_spec"] - data[PARTITION_SPECS] = [PartitionSpec(spec_id=INITIAL_SPEC_ID, fields=fields)] + fields = data[PARTITION_SPEC] + migrated_spec = PartitionSpec(*fields) + data[PARTITION_SPECS] = [migrated_spec] + data[DEFAULT_SPEC_ID] = migrated_spec.spec_id else: check_partition_specs(data) diff --git a/pyiceberg/table/partitioning.py b/pyiceberg/table/partitioning.py index 30a9d63415..8dcbf68454 100644 --- a/pyiceberg/table/partitioning.py +++ b/pyiceberg/table/partitioning.py @@ -29,7 +29,7 @@ from pyiceberg.transforms import Transform from pyiceberg.utils.iceberg_base_model import IcebergBaseModel -INITIAL_SPEC_ID = 0 +INITIAL_PARTITION_SPEC_ID = 0 _PARTITION_DATA_ID_START: int = 1000 @@ -82,19 +82,16 @@ class PartitionSpec(IcebergBaseModel): fields(List[PartitionField): list of partition fields to produce partition values """ - spec_id: int = Field(alias="spec-id") - fields: Tuple[PartitionField, ...] = Field(default_factory=tuple) + spec_id: int = Field(alias="spec-id", default=INITIAL_PARTITION_SPEC_ID) + fields: Tuple[PartitionField, ...] = Field(alias="fields", default_factory=tuple) def __init__( self, - spec_id: Optional[int] = None, - fields: Optional[Tuple[PartitionField, ...]] = None, + *fields: PartitionField, **data: Any, ): - if spec_id is not None: - data["spec-id"] = spec_id - if fields is not None: - data["fields"] = fields + if fields: + data["fields"] = tuple(fields) super().__init__(**data) def __eq__(self, other: Any) -> bool: @@ -121,6 +118,10 @@ def __str__(self): result_str += "]" return result_str + def __repr__(self) -> str: + fields = f"{', '.join(repr(column) for column in self.fields)}, " if self.fields else "" + return f"PartitionSpec({fields}spec_id={self.spec_id})" + def is_unpartitioned(self) -> bool: return not self.fields @@ -178,4 +179,4 @@ def assign_fresh_partition_spec_ids(spec: PartitionSpec, old_schema: Schema, fre transform=field.transform, ) ) - return PartitionSpec(INITIAL_SPEC_ID, fields=tuple(partition_fields)) + return PartitionSpec(*partition_fields, spec_id=INITIAL_PARTITION_SPEC_ID) diff --git a/pyiceberg/table/sorting.py b/pyiceberg/table/sorting.py index 013470fa57..4fafcd3b1f 100644 --- a/pyiceberg/table/sorting.py +++ b/pyiceberg/table/sorting.py @@ -104,6 +104,9 @@ def __str__(self): return f"{self.transform}({self.source_id}) {self.direction} {self.null_order}" +INITIAL_SORT_ORDER_ID = 1 + + class SortOrder(IcebergBaseModel): """Describes how the data is sorted within the table @@ -112,20 +115,18 @@ class SortOrder(IcebergBaseModel): The order of the sort fields within the list defines the order in which the sort is applied to the data. Args: - order_id (int): The id of the sort-order. To keep track of historical sorting + order_id (int): An unique id of the sort-order of a table. fields (List[SortField]): The fields how the table is sorted """ - def __init__(self, order_id: Optional[int] = None, *fields: SortField, **data: Any): - if order_id is not None: - data["order-id"] = order_id + order_id: int = Field(alias="order-id", default=INITIAL_SORT_ORDER_ID) + fields: List[SortField] = Field(default_factory=list) + + def __init__(self, *fields: SortField, **data: Any): if fields: data["fields"] = fields super().__init__(**data) - order_id: int = Field(alias="order-id") - fields: List[SortField] = Field(default_factory=list) - @property def is_unsorted(self) -> bool: return len(self.fields) == 0 @@ -137,10 +138,13 @@ def __str__(self) -> str: result_str += "]" return result_str + def __repr__(self): + fields = f"{', '.join(repr(column) for column in self.fields)}, " if self.fields else "" + return f"SortOrder({fields}order_id={self.order_id})" + UNSORTED_SORT_ORDER_ID = 0 UNSORTED_SORT_ORDER = SortOrder(order_id=UNSORTED_SORT_ORDER_ID) -INITIAL_SORT_ORDER_ID = 1 def assign_fresh_sort_order_ids(sort_order: SortOrder, old_schema: Schema, fresh_schema: Schema) -> SortOrder: @@ -164,7 +168,4 @@ def assign_fresh_sort_order_ids(sort_order: SortOrder, old_schema: Schema, fresh ) ) - return SortOrder( - INITIAL_SORT_ORDER_ID, - *fresh_fields, - ) + return SortOrder(*fresh_fields, order_id=INITIAL_SORT_ORDER_ID) diff --git a/pyproject.toml b/pyproject.toml index aa5ee6cd6c..69881b8018 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -51,9 +51,9 @@ pyyaml = "^6.0.0" pydantic = "^1.10.2" fsspec = "2022.8.2" -pyarrow = { version = "^9.0.0", optional = true } +zstandard = "^0.18.0" -zstandard = { version = "^0.18.0", optional = true } +pyarrow = { version = "^9.0.0", optional = true } python-snappy = { version = "^0.6.1", optional = true } @@ -80,7 +80,6 @@ build-backend = "poetry.core.masonry.api" [tool.poetry.extras] pyarrow = ["pyarrow"] snappy = ["python-snappy"] -python-snappy = ["zstandard"] hive = ["thrift"] s3fs = ["s3fs"] diff --git a/tests/catalog/test_base.py b/tests/catalog/test_base.py index a01b3f5788..5aea87c918 100644 --- a/tests/catalog/test_base.py +++ b/tests/catalog/test_base.py @@ -40,9 +40,9 @@ ) from pyiceberg.schema import Schema from pyiceberg.table import Table -from pyiceberg.table.metadata import INITIAL_SPEC_ID -from pyiceberg.table.partitioning import UNPARTITIONED_PARTITION_SPEC, PartitionSpec +from pyiceberg.table.partitioning import UNPARTITIONED_PARTITION_SPEC, PartitionField, PartitionSpec from pyiceberg.table.sorting import UNSORTED_SORT_ORDER, SortOrder +from pyiceberg.transforms import IdentityTransform from pyiceberg.typedef import EMPTY_DICT from tests.table.test_metadata import EXAMPLE_TABLE_METADATA_V1 @@ -186,7 +186,7 @@ def update_namespace_properties( TEST_TABLE_NAME = "my_table" TEST_TABLE_SCHEMA = Schema(schema_id=1) TEST_TABLE_LOCATION = "protocol://some/location" -TEST_TABLE_PARTITION_SPEC = PartitionSpec(spec_id=INITIAL_SPEC_ID, fields=()) +TEST_TABLE_PARTITION_SPEC = PartitionSpec(PartitionField(name="x", transform=IdentityTransform(), source_id=1, field_id=1000)) TEST_TABLE_PROPERTIES = {"key1": "value1", "key2": "value2"} NO_SUCH_TABLE_ERROR = "Table does not exist: \\('com', 'organization', 'department', 'my_table'\\)" TABLE_ALREADY_EXISTS_ERROR = "Table already exists: \\('com', 'organization', 'department', 'my_table'\\)" @@ -200,7 +200,7 @@ def given_catalog_has_a_table(catalog: InMemoryCatalog) -> Table: identifier=TEST_TABLE_IDENTIFIER, schema=TEST_TABLE_SCHEMA, location=TEST_TABLE_LOCATION, - partition_spec=TEST_TABLE_PARTITION_SPEC, + partition_spec=UNPARTITIONED_PARTITION_SPEC, properties=TEST_TABLE_PROPERTIES, ) diff --git a/tests/catalog/test_hive.py b/tests/catalog/test_hive.py index 663fa33fe4..bc45bb457e 100644 --- a/tests/catalog/test_hive.py +++ b/tests/catalog/test_hive.py @@ -77,7 +77,8 @@ @pytest.fixture def hive_table(tmp_path_factory, example_table_metadata_v2: Dict[str, Any]) -> HiveTable: metadata_path = str(tmp_path_factory.mktemp("metadata") / f"{uuid.uuid4()}.metadata.json") - ToOutputFile.table_metadata(TableMetadataV2(**example_table_metadata_v2), LocalFileIO().new_output(str(metadata_path)), True) + metadata = TableMetadataV2(**example_table_metadata_v2) + ToOutputFile.table_metadata(metadata, LocalFileIO().new_output(str(metadata_path)), True) return HiveTable( tableName="new_tabl2e", @@ -267,7 +268,7 @@ def test_create_table(table_schema_simple: Schema, hive_database: HiveDatabase, assert "database/table" in metadata.location - assert metadata == TableMetadataV2( + expected = TableMetadataV2( location=metadata.location, table_uuid=metadata.table_uuid, last_updated_ms=metadata.last_updated_ms, @@ -282,10 +283,10 @@ def test_create_table(table_schema_simple: Schema, hive_database: HiveDatabase, ) ], current_schema_id=0, - partition_specs=[PartitionSpec(spec_id=0)], - default_spec_id=0, last_partition_id=1000, properties={"owner": "javaberg"}, + partition_specs=[PartitionSpec()], + default_spec_id=0, current_snapshot_id=None, snapshots=[], snapshot_log=[], @@ -297,6 +298,8 @@ def test_create_table(table_schema_simple: Schema, hive_database: HiveDatabase, last_sequence_number=0, ) + assert metadata.dict() == expected.dict() + def test_load_table(hive_table: HiveTable): catalog = HiveCatalog(HIVE_CATALOG_NAME, uri=HIVE_METASTORE_FAKE_URL) @@ -307,8 +310,7 @@ def test_load_table(hive_table: HiveTable): catalog._client.__enter__().get_table.assert_called_with(dbname="default", tbl_name="new_tabl2e") - assert table.identifier == ("default", "new_tabl2e") - assert table.metadata == TableMetadataV2( + expected = TableMetadataV2( location="s3://bucket/test/location", table_uuid=uuid.UUID("9c12d441-03fe-4693-9a96-a0705ddf69c1"), last_updated_ms=1602638573590, @@ -329,9 +331,7 @@ def test_load_table(hive_table: HiveTable): ], current_schema_id=1, partition_specs=[ - PartitionSpec( - spec_id=0, fields=(PartitionField(source_id=1, field_id=1000, transform=IdentityTransform(), name="x"),) - ) + PartitionSpec(PartitionField(source_id=1, field_id=1000, transform=IdentityTransform(), name="x"), spec_id=0) ], default_spec_id=0, last_partition_id=1000, @@ -364,7 +364,6 @@ def test_load_table(hive_table: HiveTable): metadata_log=[MetadataLogEntry(metadata_file="s3://bucket/.../v1.json", timestamp_ms=1515100)], sort_orders=[ SortOrder( - 3, SortField( source_id=2, transform=IdentityTransform(), direction=SortDirection.ASC, null_order=NullOrder.NULLS_FIRST ), @@ -374,6 +373,7 @@ def test_load_table(hive_table: HiveTable): direction=SortDirection.DESC, null_order=NullOrder.NULLS_LAST, ), + order_id=3, ) ], default_sort_order_id=3, @@ -397,6 +397,9 @@ def test_load_table(hive_table: HiveTable): last_sequence_number=34, ) + assert table.identifier == ("default", "new_tabl2e") + assert expected == table.metadata + def test_rename_table_from_does_not_exists(): catalog = HiveCatalog(HIVE_CATALOG_NAME, uri=HIVE_METASTORE_FAKE_URL) diff --git a/tests/catalog/test_rest.py b/tests/catalog/test_rest.py index 5b60cd8f7c..e97f8ad50f 100644 --- a/tests/catalog/test_rest.py +++ b/tests/catalog/test_rest.py @@ -344,8 +344,8 @@ def test_load_table_200(rest_mock: Mocker): }, status_code=200, ) - table = RestCatalog("rest", uri=TEST_URI, token=TEST_TOKEN).load_table(("fokko", "table")) - assert table == Table( + actual = RestCatalog("rest", uri=TEST_URI, token=TEST_TOKEN).load_table(("fokko", "table")) + expected = Table( identifier=("rest", "fokko", "table"), metadata_location="s3://warehouse/database/table/metadata/00001-5f2f8166-244c-4eae-ac36-384ecdec81fc.gz.metadata.json", metadata=TableMetadataV1( @@ -362,7 +362,6 @@ def test_load_table_200(rest_mock: Mocker): ) ], current_schema_id=0, - partition_specs=[PartitionSpec(spec_id=0, fields=())], default_spec_id=0, last_partition_id=999, properties={"owner": "bryan", "write.metadata.compression-codec": "gzip"}, @@ -422,6 +421,7 @@ def test_load_table_200(rest_mock: Mocker): ), config={"client.factory": "io.tabular.iceberg.catalog.TabularAwsClientFactory", "region": "us-west-2"}, ) + assert actual == expected def test_load_table_404(rest_mock: Mocker): @@ -496,7 +496,6 @@ def test_create_table_200(rest_mock: Mocker, table_schema_simple: Schema): ], "partition-spec": [], "default-spec-id": 0, - "partition-specs": [{"spec-id": 0, "fields": []}], "last-partition-id": 999, "default-sort-order-id": 0, "sort-orders": [{"order-id": 0, "fields": []}], @@ -524,9 +523,9 @@ def test_create_table_200(rest_mock: Mocker, table_schema_simple: Schema): schema=table_schema_simple, location=None, partition_spec=PartitionSpec( - spec_id=1, fields=(PartitionField(source_id=1, field_id=1000, transform=TruncateTransform(width=3), name="id"),) + PartitionField(source_id=1, field_id=1000, transform=TruncateTransform(width=3), name="id"), spec_id=1 ), - sort_order=SortOrder(1, SortField(source_id=2, transform=IdentityTransform())), + sort_order=SortOrder(SortField(source_id=2, transform=IdentityTransform())), properties={"owner": "fokko"}, ) assert table == Table( @@ -547,7 +546,6 @@ def test_create_table_200(rest_mock: Mocker, table_schema_simple: Schema): ) ], current_schema_id=0, - partition_specs=[PartitionSpec(spec_id=0, fields=())], default_spec_id=0, last_partition_id=999, properties={ @@ -595,10 +593,9 @@ def test_create_table_409(rest_mock, table_schema_simple: Schema): schema=table_schema_simple, location=None, partition_spec=PartitionSpec( - spec_id=1, - fields=(PartitionField(source_id=1, field_id=1000, transform=TruncateTransform(width=3), name="id"),), + PartitionField(source_id=1, field_id=1000, transform=TruncateTransform(width=3), name="id") ), - sort_order=SortOrder(1, SortField(source_id=2, transform=IdentityTransform())), + sort_order=SortOrder(SortField(source_id=2, transform=IdentityTransform())), properties={"owner": "fokko"}, ) assert "Table already exists" in str(e.value) diff --git a/tests/table/test_init.py b/tests/table/test_init.py index 567e1f07a5..d0117c4df1 100644 --- a/tests/table/test_init.py +++ b/tests/table/test_init.py @@ -82,13 +82,13 @@ def test_schemas(table): def test_spec(table): assert table.spec() == PartitionSpec( - spec_id=0, fields=(PartitionField(source_id=1, field_id=1000, transform=IdentityTransform(), name="x"),) + PartitionField(source_id=1, field_id=1000, transform=IdentityTransform(), name="x"), spec_id=0 ) def test_specs(table): assert table.specs() == { - 0: PartitionSpec(spec_id=0, fields=(PartitionField(source_id=1, field_id=1000, transform=IdentityTransform(), name="x"),)) + 0: PartitionSpec(PartitionField(source_id=1, field_id=1000, transform=IdentityTransform(), name="x"), spec_id=0) } diff --git a/tests/table/test_metadata.py b/tests/table/test_metadata.py index 4cb6795a4a..c88db6a2da 100644 --- a/tests/table/test_metadata.py +++ b/tests/table/test_metadata.py @@ -122,8 +122,20 @@ def test_v1_metadata_parsing_directly(): assert table_metadata.location == "s3://bucket/test/location" assert table_metadata.last_updated_ms == 1602638573874 assert table_metadata.last_column_id == 3 + assert table_metadata.schemas == [ + Schema( + NestedField(field_id=1, name="x", field_type=LongType(), required=True), + NestedField(field_id=2, name="y", field_type=LongType(), required=True, doc="comment"), + NestedField(field_id=3, name="z", field_type=LongType(), required=True), + schema_id=0, + identifier_field_ids=[], + ) + ] assert table_metadata.schemas[0].schema_id == 0 assert table_metadata.current_schema_id == 0 + assert table_metadata.partition_specs == [ + PartitionSpec(PartitionField(source_id=1, field_id=1000, transform=IdentityTransform(), name="x"), spec_id=0) + ] assert table_metadata.default_spec_id == 0 assert table_metadata.last_partition_id == 1000 assert table_metadata.current_snapshot_id is None @@ -163,9 +175,10 @@ def test_updating_metadata(example_table_metadata_v2: Dict[str, Any]): def test_serialize_v1(): - table_metadata = TableMetadataV1(**EXAMPLE_TABLE_METADATA_V1).json() + table_metadata = TableMetadataV1(**EXAMPLE_TABLE_METADATA_V1) + table_metadata_json = table_metadata.json() expected = """{"location": "s3://bucket/test/location", "table-uuid": "d20125c8-7284-442c-9aea-15fee620737c", "last-updated-ms": 1602638573874, "last-column-id": 3, "schemas": [{"type": "struct", "fields": [{"id": 1, "name": "x", "type": "long", "required": true}, {"id": 2, "name": "y", "type": "long", "required": true, "doc": "comment"}, {"id": 3, "name": "z", "type": "long", "required": true}], "schema-id": 0, "identifier-field-ids": []}], "current-schema-id": 0, "partition-specs": [{"spec-id": 0, "fields": [{"source-id": 1, "field-id": 1000, "transform": "identity", "name": "x"}]}], "default-spec-id": 0, "last-partition-id": 1000, "properties": {}, "snapshots": [{"snapshot-id": 1925, "timestamp-ms": 1602638573822}], "snapshot-log": [], "metadata-log": [], "sort-orders": [{"order-id": 0, "fields": []}], "default-sort-order-id": 0, "refs": {}, "format-version": 1, "schema": {"type": "struct", "fields": [{"id": 1, "name": "x", "type": "long", "required": true}, {"id": 2, "name": "y", "type": "long", "required": true, "doc": "comment"}, {"id": 3, "name": "z", "type": "long", "required": true}], "schema-id": 0, "identifier-field-ids": []}, "partition-spec": [{"name": "x", "transform": "identity", "source-id": 1, "field-id": 1000}]}""" - assert table_metadata == expected + assert table_metadata_json == expected def test_serialize_v2(example_table_metadata_v2: Dict[str, Any]): @@ -189,11 +202,7 @@ def test_migrate_v1_partition_specs(): assert len(table_metadata.partition_specs) == 1 # Spec ID gets added automatically assert table_metadata.partition_specs == [ - PartitionSpec( - spec_id=0, - fields=(PartitionField(source_id=1, field_id=1000, transform=IdentityTransform(), name="x"),), - last_assigned_field_id=1000, - ), + PartitionSpec(PartitionField(source_id=1, field_id=1000, transform=IdentityTransform(), name="x")), ] diff --git a/tests/table/test_partitioning.py b/tests/table/test_partitioning.py index 1fe9dfeb01..9aa0f73891 100644 --- a/tests/table/test_partitioning.py +++ b/tests/table/test_partitioning.py @@ -14,7 +14,7 @@ # KIND, either express or implied. See the License for the # specific language governing permissions and limitations # under the License. -from pyiceberg.table.partitioning import PartitionField, PartitionSpec +from pyiceberg.table.partitioning import UNPARTITIONED_PARTITION_SPEC, PartitionField, PartitionSpec from pyiceberg.transforms import BucketTransform, TruncateTransform @@ -34,11 +34,15 @@ def test_partition_field_init(): ) +def test_unpartitioned_partition_spec_repr(): + assert repr(PartitionSpec()) == "PartitionSpec(spec_id=0)" + + def test_partition_spec_init(): bucket_transform: BucketTransform = BucketTransform(4) id_field1 = PartitionField(3, 1001, bucket_transform, "id") - partition_spec1 = PartitionSpec(0, (id_field1,)) + partition_spec1 = PartitionSpec(id_field1) assert partition_spec1.spec_id == 0 assert partition_spec1 == partition_spec1 @@ -47,7 +51,7 @@ def test_partition_spec_init(): assert not partition_spec1.is_unpartitioned() # only differ by PartitionField field_id id_field2 = PartitionField(3, 1002, bucket_transform, "id") - partition_spec2 = PartitionSpec(0, (id_field2,)) + partition_spec2 = PartitionSpec(id_field2) assert partition_spec1 != partition_spec2 assert partition_spec1.compatible_with(partition_spec2) assert partition_spec1.fields_by_source_id(3) == [id_field1] @@ -57,31 +61,28 @@ def test_partition_compatible_with(): bucket_transform: BucketTransform = BucketTransform(4) field1 = PartitionField(3, 100, bucket_transform, "id") field2 = PartitionField(3, 102, bucket_transform, "id") - lhs = PartitionSpec(0, (field1,)) - rhs = PartitionSpec(0, (field1, field2)) + lhs = PartitionSpec( + field1, + ) + rhs = PartitionSpec(field1, field2) assert not lhs.compatible_with(rhs) def test_unpartitioned(): - unpartitioned = PartitionSpec(1, ()) - - assert not unpartitioned.fields - assert unpartitioned.is_unpartitioned() - assert str(unpartitioned) == "[]" + assert len(UNPARTITIONED_PARTITION_SPEC.fields) == 0 + assert UNPARTITIONED_PARTITION_SPEC.is_unpartitioned() + assert str(UNPARTITIONED_PARTITION_SPEC) == "[]" -def test_serialize_unpartition_spec(): - unpartitioned = PartitionSpec(1, ()) - assert unpartitioned.json() == """{"spec-id": 1, "fields": []}""" +def test_serialize_unpartitioned_spec(): + assert UNPARTITIONED_PARTITION_SPEC.json() == """{"spec-id": 0, "fields": []}""" def test_serialize_partition_spec(): partitioned = PartitionSpec( + PartitionField(source_id=1, field_id=1000, transform=TruncateTransform(width=19), name="str_truncate"), + PartitionField(source_id=2, field_id=1001, transform=BucketTransform(num_buckets=25), name="int_bucket"), spec_id=3, - fields=( - PartitionField(source_id=1, field_id=1000, transform=TruncateTransform(width=19), name="str_truncate"), - PartitionField(source_id=2, field_id=1001, transform=BucketTransform(num_buckets=25), name="int_bucket"), - ), ) assert ( partitioned.json() diff --git a/tests/table/test_sorting.py b/tests/table/test_sorting.py index 384a636e09..00ad34ba80 100644 --- a/tests/table/test_sorting.py +++ b/tests/table/test_sorting.py @@ -34,10 +34,10 @@ @pytest.fixture def sort_order() -> SortOrder: return SortOrder( - 22, SortField(source_id=19, transform=IdentityTransform(), null_order=NullOrder.NULLS_FIRST), SortField(source_id=25, transform=BucketTransform(4), direction=SortDirection.DESC), SortField(source_id=22, transform=VoidTransform(), direction=SortDirection.ASC), + order_id=22, ) @@ -61,7 +61,6 @@ def test_sorting_schema(example_table_metadata_v2: Dict[str, Any]): assert table_metadata.sort_orders == [ SortOrder( - 3, SortField(2, IdentityTransform(), SortDirection.ASC, null_order=NullOrder.NULLS_FIRST), SortField( 3, @@ -69,6 +68,7 @@ def test_sorting_schema(example_table_metadata_v2: Dict[str, Any]): direction=SortDirection.DESC, null_order=NullOrder.NULLS_LAST, ), + order_id=3, ) ] @@ -83,10 +83,15 @@ def test_sorting_to_string(sort_order: SortOrder): def test_sorting_to_repr(sort_order: SortOrder): - expected = """SortOrder(order_id=22, fields=[SortField(source_id=19, transform=IdentityTransform(), direction=SortDirection.ASC, null_order=NullOrder.NULLS_FIRST), SortField(source_id=25, transform=BucketTransform(num_buckets=4), direction=SortDirection.DESC, null_order=NullOrder.NULLS_LAST), SortField(source_id=22, transform=VoidTransform(), direction=SortDirection.ASC, null_order=NullOrder.NULLS_FIRST)])""" + expected = """SortOrder(SortField(source_id=19, transform=IdentityTransform(), direction=SortDirection.ASC, null_order=NullOrder.NULLS_FIRST), SortField(source_id=25, transform=BucketTransform(num_buckets=4), direction=SortDirection.DESC, null_order=NullOrder.NULLS_LAST), SortField(source_id=22, transform=VoidTransform(), direction=SortDirection.ASC, null_order=NullOrder.NULLS_FIRST), order_id=22)""" assert repr(sort_order) == expected +def test_unsorting_to_repr(): + expected = """SortOrder(order_id=0)""" + assert repr(UNSORTED_SORT_ORDER) == expected + + def test_sorting_repr(sort_order: SortOrder): """To make sure that the repr converts back to the original object""" assert sort_order == eval(repr(sort_order)) diff --git a/tests/test_schema.py b/tests/test_schema.py index f99b315c69..9fbebb0902 100644 --- a/tests/test_schema.py +++ b/tests/test_schema.py @@ -49,22 +49,20 @@ def test_schema_str(table_schema_simple: Schema): ) -@pytest.mark.parametrize( - "schema_repr, expected_repr", - [ - ( - schema.Schema(NestedField(1, "foo", StringType()), schema_id=1), - "Schema(fields=(NestedField(field_id=1, name='foo', field_type=StringType(), required=True),), schema_id=1, identifier_field_ids=[])", - ), - ( - schema.Schema(NestedField(1, "foo", StringType()), NestedField(2, "bar", IntegerType(), required=False), schema_id=1), - "Schema(fields=(NestedField(field_id=1, name='foo', field_type=StringType(), required=True), NestedField(field_id=2, name='bar', field_type=IntegerType(), required=False)), schema_id=1, identifier_field_ids=[])", - ), - ], -) -def test_schema_repr(schema_repr: Schema, expected_repr: str): +def test_schema_repr_single_field(): """Test schema representation""" - assert repr(schema_repr) == expected_repr + actual = repr(schema.Schema(NestedField(1, "foo", StringType()), schema_id=1)) + expected = "Schema(NestedField(field_id=1, name='foo', field_type=StringType(), required=True), schema_id=1, identifier_field_ids=[])" + assert expected == actual + + +def test_schema_repr_two_fields(): + """Test schema representation""" + actual = repr( + schema.Schema(NestedField(1, "foo", StringType()), NestedField(2, "bar", IntegerType(), required=False), schema_id=1) + ) + expected = "Schema(NestedField(field_id=1, name='foo', field_type=StringType(), required=True), NestedField(field_id=2, name='bar', field_type=IntegerType(), required=False), schema_id=1, identifier_field_ids=[])" + assert expected == actual def test_schema_raise_on_duplicate_names(): From b732e143ea28af786182ee1aa25b4a6af735eed4 Mon Sep 17 00:00:00 2001 From: Joshua Robinson <3334099+joshuarobinson@users.noreply.github.com> Date: Tue, 20 Sep 2022 20:07:36 +0200 Subject: [PATCH 206/642] Python: Handle optional Avro fields in conversion. (#5796) Found by processing fields in manifestentry with empty split_offsets field. For pos_to_dict, check if values is None before processing as list, struct, or dict. Added unit tests to verify. Thanks to @fokko for the fix. --- pyiceberg/manifest.py | 20 ++++++++++++----- tests/avro/test_reader.py | 46 +++++++++++++++++++++++++++++++++++++++ 2 files changed, 60 insertions(+), 6 deletions(-) diff --git a/pyiceberg/manifest.py b/pyiceberg/manifest.py index 751dfbef2a..8619d18abf 100644 --- a/pyiceberg/manifest.py +++ b/pyiceberg/manifest.py @@ -173,22 +173,30 @@ def _(schema: Schema, struct: AvroStruct) -> Dict[str, Any]: @_convert_pos_to_dict.register def _(struct_type: StructType, values: AvroStruct) -> Dict[str, Any]: """Iterates over all the fields in the dict, and gets the data from the struct""" - return {field.name: _convert_pos_to_dict(field.field_type, values.get(pos)) for pos, field in enumerate(struct_type.fields)} + return ( + {field.name: _convert_pos_to_dict(field.field_type, values.get(pos)) for pos, field in enumerate(struct_type.fields)} + if values is not None + else None + ) @_convert_pos_to_dict.register def _(list_type: ListType, values: List[Any]) -> Any: """In the case of a list, we'll go over the elements in the list to handle complex types""" - return [_convert_pos_to_dict(list_type.element_type, value) for value in values] + return [_convert_pos_to_dict(list_type.element_type, value) for value in values] if values is not None else None @_convert_pos_to_dict.register def _(map_type: MapType, values: Dict) -> Dict: """In the case of a map, we both traverse over the key and value to handle complex types""" - return { - _convert_pos_to_dict(map_type.key_type, key): _convert_pos_to_dict(map_type.value_type, value) - for key, value in values.items() - } + return ( + { + _convert_pos_to_dict(map_type.key_type, key): _convert_pos_to_dict(map_type.value_type, value) + for key, value in values.items() + } + if values is not None + else None + ) @_convert_pos_to_dict.register diff --git a/tests/avro/test_reader.py b/tests/avro/test_reader.py index 44505a6174..77e186b4f7 100644 --- a/tests/avro/test_reader.py +++ b/tests/avro/test_reader.py @@ -36,6 +36,7 @@ TimestamptzReader, primitive_reader, ) +from pyiceberg.manifest import _convert_pos_to_dict from pyiceberg.schema import Schema from pyiceberg.types import ( BinaryType, @@ -46,9 +47,13 @@ FixedType, FloatType, IntegerType, + ListType, LongType, + MapType, + NestedField, PrimitiveType, StringType, + StructType, TimestampType, TimestamptzType, TimeType, @@ -395,6 +400,47 @@ def test_read_manifest_file_file(generated_manifest_file_file: str): assert actual == expected +def test_null_list_convert_pos_to_dict(): + data = _convert_pos_to_dict( + Schema( + NestedField(name="field", field_id=1, field_type=ListType(element_id=2, element=StringType(), element_required=False)) + ), + AvroStruct([None]), + ) + assert data["field"] is None + + +def test_null_dict_convert_pos_to_dict(): + data = _convert_pos_to_dict( + Schema( + NestedField( + name="field", + field_id=1, + field_type=MapType(key_id=2, key_type=StringType(), value_id=3, value_type=StringType(), value_required=False), + ) + ), + AvroStruct([None]), + ) + assert data["field"] is None + + +def test_null_struct_convert_pos_to_dict(): + data = _convert_pos_to_dict( + Schema( + NestedField( + name="field", + field_id=1, + field_type=StructType( + NestedField(2, "required_field", StringType(), True), NestedField(3, "optional_field", IntegerType()) + ), + required=False, + ) + ), + AvroStruct([None]), + ) + assert data["field"] is None + + def test_fixed_reader(): assert primitive_reader(FixedType(22)) == FixedReader(22) From 830cd71b2676cc062f8c1a63b5acc5202011e877 Mon Sep 17 00:00:00 2001 From: Fokko Driesprong Date: Wed, 21 Sep 2022 01:54:39 +0200 Subject: [PATCH 207/642] Python: Remove duplicate dispatch types (#5808) --- pyiceberg/expressions/base.py | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/pyiceberg/expressions/base.py b/pyiceberg/expressions/base.py index f4584fdb4c..afc1cf47a2 100644 --- a/pyiceberg/expressions/base.py +++ b/pyiceberg/expressions/base.py @@ -599,26 +599,26 @@ def visit(obj, visitor: BooleanExpressionVisitor[T]) -> T: raise NotImplementedError(f"Cannot visit unsupported expression: {obj}") -@visit.register(AlwaysTrue) -def _(obj: AlwaysTrue, visitor: BooleanExpressionVisitor[T]) -> T: +@visit.register +def _(_: AlwaysTrue, visitor: BooleanExpressionVisitor[T]) -> T: """Visit an AlwaysTrue boolean expression with a concrete BooleanExpressionVisitor""" return visitor.visit_true() -@visit.register(AlwaysFalse) -def _(obj: AlwaysFalse, visitor: BooleanExpressionVisitor[T]) -> T: +@visit.register +def _(_: AlwaysFalse, visitor: BooleanExpressionVisitor[T]) -> T: """Visit an AlwaysFalse boolean expression with a concrete BooleanExpressionVisitor""" return visitor.visit_false() -@visit.register(Not) +@visit.register def _(obj: Not, visitor: BooleanExpressionVisitor[T]) -> T: """Visit a Not boolean expression with a concrete BooleanExpressionVisitor""" child_result: T = visit(obj.child, visitor=visitor) return visitor.visit_not(child_result=child_result) -@visit.register(And) +@visit.register def _(obj: And, visitor: BooleanExpressionVisitor[T]) -> T: """Visit an And boolean expression with a concrete BooleanExpressionVisitor""" left_result: T = visit(obj.left, visitor=visitor) @@ -626,13 +626,13 @@ def _(obj: And, visitor: BooleanExpressionVisitor[T]) -> T: return visitor.visit_and(left_result=left_result, right_result=right_result) -@visit.register(UnboundPredicate) +@visit.register def _(obj: UnboundPredicate, visitor: BooleanExpressionVisitor[T]) -> T: """Visit an In boolean expression with a concrete BooleanExpressionVisitor""" return visitor.visit_unbound_predicate(predicate=obj) -@visit.register(Or) +@visit.register def _(obj: Or, visitor: BooleanExpressionVisitor[T]) -> T: """Visit an Or boolean expression with a concrete BooleanExpressionVisitor""" left_result: T = visit(obj.left, visitor=visitor) From ef6d9d2ea57e9f641a4585ebdbb5049db2f0492c Mon Sep 17 00:00:00 2001 From: Fokko Driesprong Date: Wed, 21 Sep 2022 01:58:24 +0200 Subject: [PATCH 208/642] Python: Fix docstring (#5804) --- pyiceberg/expressions/base.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyiceberg/expressions/base.py b/pyiceberg/expressions/base.py index afc1cf47a2..ec70f450b1 100644 --- a/pyiceberg/expressions/base.py +++ b/pyiceberg/expressions/base.py @@ -545,7 +545,7 @@ def visit_not(self, child_result: T) -> T: """Visit method for a Not boolean expression Args: - result (T): The result of visiting the child of the Not boolean expression + child_result (T): The result of visiting the child of the Not boolean expression """ @abstractmethod From e4f93c626e4e5170e32bc6f6aade4f9bba082c69 Mon Sep 17 00:00:00 2001 From: Fokko Driesprong Date: Wed, 21 Sep 2022 21:30:52 +0200 Subject: [PATCH 209/642] Python: Split Python docs (#5727) * Python: Split Python docs This PR will split the Python docs in a separate site. The main reason for this is that the docs are part of the Java release, which is not in sync with the Python release cylce. Meaning that there is a high probability that the docs does not match with current version of the code. This will publish the docs to Github pages, by pushing this to the `gh-pages` branch. We can set up an alias from Apache, and point pyiceberg.apache.org to the github pages endpoint. I also tried readthedocs, but I found that not straightforward. Mostly because they have a build process on their end that will pull the code, and build the docs. This involves another pipeline that we have to monitor, and we have to set up webhooks. I am a simple man, and I like simple things, therefore I went for mkdocs. This can push the docs to github pages in a single command: https://www.mkdocs.org/user-guide/deploying-your-docs/#project-pages Considerations: - Decided to keep it to a single page for now, we can break it out into different pages later on. Let me know what you think of this. - We build the docs now when we push to master, probably we'll change this later to trigger on tags. - I've removed the Python docs from the other docs to avoid confusion and make sure that we have a single source of truth. An example is shown here: https://fokko.github.io/incubator-iceberg/ (Once this is merged, I'll remove that one) Closes #363 Closes #3283 * Comments --- .pre-commit-config.yaml | 8 + CONTRIBUTING.md | 3 + README.md | 10 +- mkdocs/README.md | 28 ++ mkdocs/docs/index.md | 548 ++++++++++++++++++++++++++++++++++++++++ mkdocs/mkdocs.yml | 18 ++ mkdocs/requirements.txt | 19 ++ 7 files changed, 628 insertions(+), 6 deletions(-) create mode 100644 mkdocs/README.md create mode 100644 mkdocs/docs/index.md create mode 100644 mkdocs/mkdocs.yml create mode 100644 mkdocs/requirements.txt diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index cce5d2ba0b..54c644b32c 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -63,3 +63,11 @@ repos: - id: flake8 args: [ "--ignore=E501,W503,E203" ] additional_dependencies: [ flake8-bugbear==22.7.1, flake8-comprehensions==3.10.0 ] + - repo: https://github.com/executablebooks/mdformat + rev: 0.7.16 + hooks: + - id: mdformat + additional_dependencies: + - mdformat-black + - mdformat-config + - mdformat-beautysh diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 91ae8215b4..1bde5ce7db 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -26,6 +26,7 @@ pip install poetry ``` If you have an older version of pip and virtualenv you need to update these: + ```bash pip install --upgrade virtualenv pip ``` @@ -81,11 +82,13 @@ make test-s3 To pass additional arguments to pytest, you can use `PYTEST_ARGS`. *Run pytest in verbose mode* + ```sh make test PYTEST_ARGS="-v" ``` *Run pytest with pdb enabled* + ```sh make test PYTEST_ARGS="--pdb" ``` diff --git a/README.md b/README.md index 292fb7b9ee..6dfefbce6a 100644 --- a/README.md +++ b/README.md @@ -17,12 +17,11 @@ # Iceberg Python -py-iceberg is a python library for programmatic access to iceberg table metadata as well as to table data in iceberg format. -It is an implementation of [iceberg table spec](https://iceberg.apache.org/spec/) in Python. +pyiceberg is a python library for programmatic access to iceberg table metadata as well as to table data in iceberg format. It is a Python implementation of [iceberg table spec](https://iceberg.apache.org/spec/). Documentation is available at [https://pyiceberg.apache.org/](https://pyiceberg.apache.org/). ## Getting Started -py-iceberg is currently in development, for development and testing purposes the best way to install the library is to perform the following steps: +pyiceberg is currently in development, for development and testing purposes the best way to install the library is to perform the following steps: ``` git clone https://github.com/apache/iceberg.git @@ -30,11 +29,9 @@ cd iceberg/python pip install -e . ``` -Development is made easy using [Poetry](https://python-poetry.org/docs/#installation). - ## Development -Poetry provides virtual environments for development: +Development is made easy using [Poetry](https://python-poetry.org/docs/#installation). Poetry provides virtual environments for development: ```bash poetry shell @@ -54,4 +51,5 @@ poetry run pytest ``` ## Get in Touch + - [Iceberg community](https://iceberg.apache.org/community/) diff --git a/mkdocs/README.md b/mkdocs/README.md new file mode 100644 index 0000000000..e9e0462bee --- /dev/null +++ b/mkdocs/README.md @@ -0,0 +1,28 @@ + + +# Docs + +The pyiceberg docs are stored in `docs/`. + +## Running docs locally + +```sh +pip3 install -r requirements.txt +mkdocs serve +open http://localhost:8000/ +``` diff --git a/mkdocs/docs/index.md b/mkdocs/docs/index.md new file mode 100644 index 0000000000..35351ef0c8 --- /dev/null +++ b/mkdocs/docs/index.md @@ -0,0 +1,548 @@ + + +# PyIceberg + +Much of the python api conforms to the Java API. You can get more info about the java api [here](https://iceberg.apache.org/docs/latest/java-api-quickstart/). + +## Installing + +You can install the latest release version from pypi: + +```sh +pip3 install "pyiceberg[s3fs,hive]" +``` + +Or install the latest development version locally: + +```sh +git clone https://github.com/apache/iceberg.git +cd iceberg/python +pip3 install -e ".[s3fs,hive]" +``` + +You can mix and match optional dependencies: + +| Key | Description: | +|-----------|----------------------------------------------------------------------| +| hive | Support for the Hive metastore | +| pyarrow | PyArrow as a FileIO implementation to interact with the object store | +| s3fs | S3FS as a FileIO implementation to interact with the object store | +| snappy | Support for snappy Avro compression | + +# Python CLI Quickstart + +Pyiceberg comes with a CLI that's available after installing the `pyiceberg` package. + +```sh +➜ pyiceberg --help +Usage: pyiceberg [OPTIONS] COMMAND [ARGS]... + +Options: +--catalog TEXT +--verbose BOOLEAN +--output [text|json] +--uri TEXT +--credential TEXT +--help Show this message and exit. + +Commands: +describe Describes a namespace xor table +drop Operations to drop a namespace or table +list Lists tables or namespaces +location Returns the location of the table +properties Properties on tables/namespaces +rename Renames a table +schema Gets the schema of the table +spec Returns the partition spec of the table +uuid Returns the UUID of the table +``` + +# Configuration + +There are three ways of setting the configuration. + +For the CLI you can pass it in using `--uri` and `--credential` and it will automatically detect the type based on the scheme (`http(s)` for rest, `thrift` for Hive). + +Secondly, YAML based configuration is supported `cat ~/.pyiceberg.yaml`: + +```yaml +catalog: + default: + uri: thrift://localhost:9083 + s3.endpoint: http://localhost:9000 + s3.access-key-id: admin + s3.secret-access-key: password + + rest: + uri: http://rest-catalog/ws/ + credential: t-1234:secret +``` + +Lastly, you can also set it using environment variables: + +```sh +export PYICEBERG_CATALOG__DEFAULT__URI=thrift://localhost:9083 + +export PYICEBERG_CATALOG__REST__URI=http://rest-catalog/ws/ +export PYICEBERG_CATALOG__REST__CREDENTIAL=t-1234:secret +``` + +Where the structure is equivalent to the YAML. The levels are separated using a double underscore (`__`). + +## FileIO configuration + +For the FileIO there are several configuration options available: + +| Key | Example | Description | +|----------------------|---------------------|-----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| +| s3.endpoint | https://10.0.19.25/ | Configure an alternative endpoint of the S3 service for the FileIO to access. This could be used to use S3FileIO with any s3-compatible object storage service that has a different endpoint, or access a private S3 endpoint in a virtual private cloud. | +| s3.access-key-id | admin | Configure the static secret access key used to access the FileIO. | +| s3.secret-access-key | password | Configure the static session token used to access the FileIO. | +| s3.signer | bearer | Configure the signature version of the FileIO. | + +# CLI Quickstart + +This example assumes that you have a default catalog set. If you want to load another catalog, for example, the rest example above. Then you need to set `--catalog rest`. + +```sh +➜ pyiceberg list +default +nyc +``` + +```sh +➜ pyiceberg list nyc +nyc.taxis +``` + +```sh +pyiceberg describe nyc.taxis +Table format version 1 +Metadata location file:/.../nyc.db/taxis/metadata/00000-aa3a3eac-ea08-4255-b890-383a64a94e42.metadata.json +Table UUID 6cdfda33-bfa3-48a7-a09e-7abb462e3460 +Last Updated 1661783158061 +Partition spec [] +Sort order [] +Current schema Schema, id=0 +├── 1: VendorID: optional long +├── 2: tpep_pickup_datetime: optional timestamptz +├── 3: tpep_dropoff_datetime: optional timestamptz +├── 4: passenger_count: optional double +├── 5: trip_distance: optional double +├── 6: RatecodeID: optional double +├── 7: store_and_fwd_flag: optional string +├── 8: PULocationID: optional long +├── 9: DOLocationID: optional long +├── 10: payment_type: optional long +├── 11: fare_amount: optional double +├── 12: extra: optional double +├── 13: mta_tax: optional double +├── 14: tip_amount: optional double +├── 15: tolls_amount: optional double +├── 16: improvement_surcharge: optional double +├── 17: total_amount: optional double +├── 18: congestion_surcharge: optional double +└── 19: airport_fee: optional double +Current snapshot Operation.APPEND: id=5937117119577207079, schema_id=0 +Snapshots Snapshots +└── Snapshot 5937117119577207079, schema 0: file:/.../nyc.db/taxis/metadata/snap-5937117119577207079-1-94656c4f-4c66-4600-a4ca-f30377300527.avro +Properties owner root +write.format.default parquet +``` + +Or output in JSON for automation: + +```sh +pyiceberg --output json describe nyc.taxis | jq +{ + "identifier": [ + "nyc", + "taxis" + ], + "metadata_location": "file:/.../nyc.db/taxis/metadata/00000-aa3a3eac-ea08-4255-b890-383a64a94e42.metadata.json", + "metadata": { + "location": "file:/.../nyc.db/taxis", + "table-uuid": "6cdfda33-bfa3-48a7-a09e-7abb462e3460", + "last-updated-ms": 1661783158061, + "last-column-id": 19, + "schemas": [ + { + "type": "struct", + "fields": [ + { + "id": 1, + "name": "VendorID", + "type": "long", + "required": false + }, +... + { + "id": 19, + "name": "airport_fee", + "type": "double", + "required": false + } + ], + "schema-id": 0, + "identifier-field-ids": [] + } + ], + "current-schema-id": 0, + "partition-specs": [ + { + "spec-id": 0, + "fields": [] + } + ], + "default-spec-id": 0, + "last-partition-id": 999, + "properties": { + "owner": "root", + "write.format.default": "parquet" + }, + "current-snapshot-id": 5937117119577207000, + "snapshots": [ + { + "snapshot-id": 5937117119577207000, + "timestamp-ms": 1661783158061, + "manifest-list": "file:/.../nyc.db/taxis/metadata/snap-5937117119577207079-1-94656c4f-4c66-4600-a4ca-f30377300527.avro", + "summary": { + "operation": "append", + "spark.app.id": "local-1661783139151", + "added-data-files": "1", + "added-records": "2979431", + "added-files-size": "46600777", + "changed-partition-count": "1", + "total-records": "2979431", + "total-files-size": "46600777", + "total-data-files": "1", + "total-delete-files": "0", + "total-position-deletes": "0", + "total-equality-deletes": "0" + }, + "schema-id": 0 + } + ], + "snapshot-log": [ + { + "snapshot-id": "5937117119577207079", + "timestamp-ms": 1661783158061 + } + ], + "metadata-log": [], + "sort-orders": [ + { + "order-id": 0, + "fields": [] + } + ], + "default-sort-order-id": 0, + "refs": { + "main": { + "snapshot-id": 5937117119577207000, + "type": "branch" + } + }, + "format-version": 1, + "schema": { + "type": "struct", + "fields": [ + { + "id": 1, + "name": "VendorID", + "type": "long", + "required": false + }, +... + { + "id": 19, + "name": "airport_fee", + "type": "double", + "required": false + } + ], + "schema-id": 0, + "identifier-field-ids": [] + }, + "partition-spec": [] + } +} +``` + +# Python API + +To instantiate a catalog: + +```python +from pyiceberg.catalog import load_catalog + +catalog = load_catalog("prod") + +catalog.list_namespaces() +``` + +Returns: + +``` +[('default',), ('nyc',)] +``` + +Listing the tables in the `nyc` namespace: + +```python +catalog.list_tables("nyc") +``` + +Returns: + +``` +[('nyc', 'taxis')] +``` + +Loading the `taxis` table: + +```python +catalog.load_table(("nyc", "taxis")) +``` + +``` +Table( + identifier=('nyc', 'taxis'), + metadata_location='s3a://warehouse/wh/nyc.db/taxis/metadata/00002-6ea51ce3-62aa-4197-9cf8-43d07c3440ca.metadata.json', + metadata=TableMetadataV2( + location='s3a://warehouse/wh/nyc.db/taxis', + table_uuid=UUID('ebd5d172-2162-453d-b586-1cdce52c1116'), + last_updated_ms=1662633437826, + last_column_id=19, + schemas=[Schema( + NestedField(field_id=1, name='VendorID', field_type=LongType(), required=False), + NestedField(field_id=2, name='tpep_pickup_datetime', field_type=TimestamptzType(), required=False), + NestedField(field_id=3, name='tpep_dropoff_datetime', field_type=TimestamptzType(), required=False), + NestedField(field_id=4, name='passenger_count', field_type=DoubleType(), required=False), + NestedField(field_id=5, name='trip_distance', field_type=DoubleType(), required=False), + NestedField(field_id=6, name='RatecodeID', field_type=DoubleType(), required=False), + NestedField(field_id=7, name='store_and_fwd_flag', field_type=StringType(), required=False), + NestedField(field_id=8, name='PULocationID', field_type=LongType(), required=False), + NestedField(field_id=9, name='DOLocationID', field_type=LongType(), required=False), + NestedField(field_id=10, name='payment_type', field_type=LongType(), required=False), + NestedField(field_id=11, name='fare_amount', field_type=DoubleType(), required=False), + NestedField(field_id=12, name='extra', field_type=DoubleType(), required=False), + NestedField(field_id=13, name='mta_tax', field_type=DoubleType(), required=False), + NestedField(field_id=14, name='tip_amount', field_type=DoubleType(), required=False), + NestedField(field_id=15, name='tolls_amount', field_type=DoubleType(), required=False), + NestedField(field_id=16, name='improvement_surcharge', field_type=DoubleType(), required=False), + NestedField(field_id=17, name='total_amount', field_type=DoubleType(), required=False), + NestedField(field_id=18, name='congestion_surcharge', field_type=DoubleType(), required=False), + NestedField(field_id=19, name='airport_fee', field_type=DoubleType(), required=False) + ), + schema_id=0, + identifier_field_ids=[] + )], + current_schema_id=0, + partition_specs=[PartitionSpec(spec_id=0)], + default_spec_id=0, + last_partition_id=999, + properties={ + 'owner': 'root', + 'write.format.default': 'parquet' + }, + current_snapshot_id=8334458494559715805, + snapshots=[ + Snapshot( + snapshot_id=7910949481055846233, + parent_snapshot_id=None, + sequence_number=None, + timestamp_ms=1662489306555, + manifest_list='s3a://warehouse/wh/nyc.db/taxis/metadata/snap-7910949481055846233-1-3eb7a2e1-5b7a-4e76-a29a-3e29c176eea4.avro', + summary=Summary( + Operation.APPEND, + **{ + 'spark.app.id': 'local-1662489289173', + 'added-data-files': '1', + 'added-records': '2979431', + 'added-files-size': '46600777', + 'changed-partition-count': '1', + 'total-records': '2979431', + 'total-files-size': '46600777', + 'total-data-files': '1', + 'total-delete-files': '0', + 'total-position-deletes': '0', + 'total-equality-deletes': '0' + } + ), + schema_id=0 + ), + ], + snapshot_log=[ + SnapshotLogEntry( + snapshot_id='7910949481055846233', + timestamp_ms=1662489306555 + ) + ], + metadata_log=[ + MetadataLogEntry( + metadata_file='s3a://warehouse/wh/nyc.db/taxis/metadata/00000-b58341ba-6a63-4eea-9b2f-e85e47c7d09f.metadata.json', + timestamp_ms=1662489306555 + ) + ], + sort_orders=[SortOrder(order_id=0)], + default_sort_order_id=0, + refs={ + 'main': SnapshotRef( + snapshot_id=8334458494559715805, + snapshot_ref_type=SnapshotRefType.BRANCH, + min_snapshots_to_keep=None, + max_snapshot_age_ms=None, + max_ref_age_ms=None + ) + }, + format_version=2, + last_sequence_number=1 + ) +) +``` + +And to create a table from a catalog: + +```python +from pyiceberg.schema import Schema +from pyiceberg.types import TimestampType, DoubleType, StringType, NestedField + +schema = Schema( + NestedField( + field_id=1, name="datetime", field_type=TimestampType(), required=False + ), + NestedField(field_id=2, name="bid", field_type=DoubleType(), required=False), + NestedField(field_id=3, name="ask", field_type=DoubleType(), required=False), + NestedField(field_id=4, name="symbol", field_type=StringType(), required=False), +) + +from pyiceberg.table.partitioning import PartitionSpec, PartitionField +from pyiceberg.transforms import DayTransform + +partition_spec = PartitionSpec( + PartitionField( + source_id=1, field_id=1000, transform=DayTransform(), name="datetime_day" + ) +) + +from pyiceberg.table.sorting import SortOrder, SortField +from pyiceberg.transforms import IdentityTransform + +sort_order = SortOrder(SortField(source_id=4, transform=IdentityTransform())) + +from pyiceberg.catalog.hive import HiveCatalog + +catalog = HiveCatalog(name="prod", uri="thrift://localhost:9083/") + +catalog.create_table( + identifier="default.bids", + location="/Users/fokkodriesprong/Desktop/docker-spark-iceberg/wh/bids/", + schema=schema, + partition_spec=partition_spec, + sort_order=sort_order, +) +``` + +Which returns a newly created table: + +``` +Table( + identifier=('default', 'bids'), + metadata_location='/Users/fokkodriesprong/Desktop/docker-spark-iceberg/wh/bids//metadata/00000-c8cd93ab-f784-474d-a167-b1a86b05195f.metadata.json', + metadata=TableMetadataV2( + location='/Users/fokkodriesprong/Desktop/docker-spark-iceberg/wh/bids/', + table_uuid=UUID('38d4cb39-4945-4bf2-b374-984b5c4984d2'), + last_updated_ms=1661847562069, + last_column_id=4, + schemas=[ + Schema( + NestedField(field_id=1, name='datetime', field_type=TimestampType(), required=False), + NestedField(field_id=2, name='bid', field_type=DoubleType(), required=False), + NestedField(field_id=3, name='ask', field_type=DoubleType(), required=False), + NestedField(field_id=4, name='symbol', field_type=StringType(), required=False)), + schema_id=1, + identifier_field_ids=[]) + ], + current_schema_id=1, + partition_specs=[ + PartitionSpec( + PartitionField(source_id=1, field_id=1000, transform=DayTransform(), name='datetime_day'),)) + ], + default_spec_id=0, + last_partition_id=1000, + properties={}, + current_snapshot_id=None, + snapshots=[], + snapshot_log=[], + metadata_log=[], + sort_orders=[ + SortOrder(order_id=1, fields=[SortField(source_id=4, transform=IdentityTransform(), direction=SortDirection.ASC, null_order=NullOrder.NULLS_FIRST)]) + ], + default_sort_order_id=1, + refs={}, + format_version=2, + last_sequence_number=0 + ) +) +``` + +# Feature Support + +The goal is that the python library will provide a functional, performant subset of the Java library. The initial focus has been on reading table metadata and provide a convenient CLI to go through the catalog. + +## Metadata + +| Operation | Java | Python | +|:------------------------|:-----:|:------:| +| Get Schema | X | X | +| Get Snapshots | X | X | +| Plan Scan | X | | +| Plan Scan for Snapshot | X | | +| Update Current Snapshot | X | | +| Set Table Properties | X | X | +| Create Table | X | X | +| Drop Table | X | X | +| Alter Table | X | | + +## Types + +The types are kept in `pyiceberg.types`. + +Primitive types: + +- `BooleanType` +- `StringType` +- `IntegerType` +- `LongType` +- `FloatType` +- `DoubleType` +- `DateType` +- `TimeType` +- `TimestampType` +- `TimestamptzType` +- `BinaryType` +- `UUIDType` + +Complex types: + +- `StructType` +- `ListType` +- `MapType` +- `FixedType(16)` +- `DecimalType(8, 3)` diff --git a/mkdocs/mkdocs.yml b/mkdocs/mkdocs.yml new file mode 100644 index 0000000000..c84a2de465 --- /dev/null +++ b/mkdocs/mkdocs.yml @@ -0,0 +1,18 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +--- +site_name: PyIceberg diff --git a/mkdocs/requirements.txt b/mkdocs/requirements.txt new file mode 100644 index 0000000000..642a688ebc --- /dev/null +++ b/mkdocs/requirements.txt @@ -0,0 +1,19 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +mkdocs==1.3.1 +jinja2==3.0.3 From a1b2a88cecb7e114f9b89c55eb55bd6702b8a76a Mon Sep 17 00:00:00 2001 From: Fokko Driesprong Date: Thu, 22 Sep 2022 17:09:43 +0200 Subject: [PATCH 210/642] Python: Include the tests in source distribution (#5829) Adds the tests to the source distribution so we can run them when validating a release. --- pyproject.toml | 1 + 1 file changed, 1 insertion(+) diff --git a/pyproject.toml b/pyproject.toml index 69881b8018..087110fb94 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -37,6 +37,7 @@ packages = [ { include = "pyiceberg" }, { from = "vendor", include = "fb303" }, { from = "vendor", include = "hive_metastore" }, + { include = "tests", format = "sdist" }, ] From 8af9366ec285760713935770b82dfc07f16585ca Mon Sep 17 00:00:00 2001 From: Samuel Redai <43911210+samredai@users.noreply.github.com> Date: Thu, 22 Sep 2022 19:33:30 -0400 Subject: [PATCH 211/642] Python: Add BoundBooleanExpressionVisitor for bound expressions (#5303) Co-authored-by: Fokko Driesprong --- pyiceberg/expressions/base.py | 167 ++++++++++- tests/expressions/test_expressions_base.py | 328 ++++++++++++++++++++- 2 files changed, 484 insertions(+), 11 deletions(-) diff --git a/pyiceberg/expressions/base.py b/pyiceberg/expressions/base.py index ec70f450b1..82b7a414cf 100644 --- a/pyiceberg/expressions/base.py +++ b/pyiceberg/expressions/base.py @@ -19,7 +19,12 @@ from abc import ABC, abstractmethod from dataclasses import dataclass from functools import reduce, singledispatch -from typing import ClassVar, Generic, TypeVar +from typing import ( + Any, + ClassVar, + Generic, + TypeVar, +) from pyiceberg.expressions.literals import Literal from pyiceberg.files import StructProtocol @@ -628,10 +633,16 @@ def _(obj: And, visitor: BooleanExpressionVisitor[T]) -> T: @visit.register def _(obj: UnboundPredicate, visitor: BooleanExpressionVisitor[T]) -> T: - """Visit an In boolean expression with a concrete BooleanExpressionVisitor""" + """Visit an unbound boolean expression with a concrete BooleanExpressionVisitor""" return visitor.visit_unbound_predicate(predicate=obj) +@visit.register +def _(obj: BoundPredicate, visitor: BooleanExpressionVisitor[T]) -> T: + """Visit a bound boolean expression with a concrete BooleanExpressionVisitor""" + return visitor.visit_bound_predicate(predicate=obj) + + @visit.register def _(obj: Or, visitor: BooleanExpressionVisitor[T]) -> T: """Visit an Or boolean expression with a concrete BooleanExpressionVisitor""" @@ -672,3 +683,155 @@ def visit_unbound_predicate(self, predicate) -> BooleanExpression: def visit_bound_predicate(self, predicate) -> BooleanExpression: raise TypeError(f"Found already bound predicate: {predicate}") + + +class BoundBooleanExpressionVisitor(BooleanExpressionVisitor[T], ABC): + @abstractmethod + def visit_in(self, term: BoundTerm[T], literals: set[Literal[Any]]) -> T: + """Visit a bound In predicate""" + + @abstractmethod + def visit_not_in(self, term: BoundTerm[T], literals: set[Literal[Any]]) -> T: + """Visit a bound NotIn predicate""" + + @abstractmethod + def visit_is_nan(self, term: BoundTerm[T]) -> T: + """Visit a bound IsNan predicate""" + + @abstractmethod + def visit_not_nan(self, term: BoundTerm[T]) -> T: + """Visit a bound NotNan predicate""" + + @abstractmethod + def visit_is_null(self, term: BoundTerm[T]) -> T: + """Visit a bound IsNull predicate""" + + @abstractmethod + def visit_not_null(self, term: BoundTerm[T]) -> T: + """Visit a bound NotNull predicate""" + + @abstractmethod + def visit_equal(self, term: BoundTerm[T], literal: Literal[Any]) -> T: + """Visit a bound Equal predicate""" + + @abstractmethod + def visit_not_equal(self, term: BoundTerm[T], literal: Literal[Any]) -> T: + """Visit a bound NotEqual predicate""" + + @abstractmethod + def visit_greater_than_or_equal(self, term: BoundTerm[T], literal: Literal[Any]) -> T: + """Visit a bound GreaterThanOrEqual predicate""" + + @abstractmethod + def visit_greater_than(self, term: BoundTerm[T], literal: Literal[Any]) -> T: + """Visit a bound GreaterThan predicate""" + + @abstractmethod + def visit_less_than(self, term: BoundTerm[T], literal: Literal[Any]) -> T: + """Visit a bound LessThan predicate""" + + @abstractmethod + def visit_less_than_or_equal(self, term: BoundTerm[T], literal: Literal[Any]) -> T: + """Visit a bound LessThanOrEqual predicate""" + + @abstractmethod + def visit_true(self) -> T: + """Visit a bound True predicate""" + + @abstractmethod + def visit_false(self) -> T: + """Visit a bound False predicate""" + + @abstractmethod + def visit_not(self, child_result: T) -> T: + """Visit a bound Not predicate""" + + @abstractmethod + def visit_and(self, left_result: T, right_result: T) -> T: + """Visit a bound And predicate""" + + @abstractmethod + def visit_or(self, left_result: T, right_result: T) -> T: + """Visit a bound Or predicate""" + + def visit_unbound_predicate(self, predicate: UnboundPredicate[T]): + """Visit an unbound predicate + Args: + predicate (UnboundPredicate[T]): An unbound predicate + Raises: + TypeError: This always raises since an unbound predicate is not expected in a bound boolean expression + """ + raise TypeError(f"Not a bound predicate: {predicate}") + + def visit_bound_predicate(self, predicate: BoundPredicate[T]) -> T: + """Visit a bound predicate + Args: + predicate (BoundPredicate[T]): A bound predicate + """ + return visit_bound_predicate(predicate, self) + + +@singledispatch +def visit_bound_predicate(expr, visitor: BooleanExpressionVisitor[T]) -> T: # pylint: disable=unused-argument + raise TypeError(f"Unknown predicate: {expr}") + + +@visit_bound_predicate.register(BoundIn) +def _(expr: BoundIn, visitor: BoundBooleanExpressionVisitor[T]) -> T: + return visitor.visit_in(term=expr.term, literals=expr.literals) + + +@visit_bound_predicate.register(BoundNotIn) +def _(expr: BoundNotIn, visitor: BoundBooleanExpressionVisitor[T]) -> T: + return visitor.visit_not_in(term=expr.term, literals=expr.literals) + + +@visit_bound_predicate.register(BoundIsNaN) +def _(expr: BoundIsNaN, visitor: BoundBooleanExpressionVisitor[T]) -> T: + return visitor.visit_is_nan(term=expr.term) + + +@visit_bound_predicate.register(BoundNotNaN) +def _(expr: BoundNotNaN, visitor: BoundBooleanExpressionVisitor[T]) -> T: + return visitor.visit_not_nan(term=expr.term) + + +@visit_bound_predicate.register(BoundIsNull) +def _(expr: BoundIsNull, visitor: BoundBooleanExpressionVisitor[T]) -> T: + return visitor.visit_is_null(term=expr.term) + + +@visit_bound_predicate.register(BoundNotNull) +def _(expr: BoundNotNull, visitor: BoundBooleanExpressionVisitor[T]) -> T: + return visitor.visit_not_null(term=expr.term) + + +@visit_bound_predicate.register(BoundEqualTo) +def _(expr: BoundEqualTo, visitor: BoundBooleanExpressionVisitor[T]) -> T: + return visitor.visit_equal(term=expr.term, literal=expr.literal) + + +@visit_bound_predicate.register(BoundNotEqualTo) +def _(expr: BoundNotEqualTo, visitor: BoundBooleanExpressionVisitor[T]) -> T: + return visitor.visit_not_equal(term=expr.term, literal=expr.literal) + + +@visit_bound_predicate.register(BoundGreaterThanOrEqual) +def _(expr: BoundGreaterThanOrEqual, visitor: BoundBooleanExpressionVisitor[T]) -> T: + """Visit a bound GreaterThanOrEqual predicate""" + return visitor.visit_greater_than_or_equal(term=expr.term, literal=expr.literal) + + +@visit_bound_predicate.register(BoundGreaterThan) +def _(expr: BoundGreaterThan, visitor: BoundBooleanExpressionVisitor[T]) -> T: + return visitor.visit_greater_than(term=expr.term, literal=expr.literal) + + +@visit_bound_predicate.register(BoundLessThan) +def _(expr: BoundLessThan, visitor: BoundBooleanExpressionVisitor[T]) -> T: + return visitor.visit_less_than(term=expr.term, literal=expr.literal) + + +@visit_bound_predicate.register(BoundLessThanOrEqual) +def _(expr: BoundLessThanOrEqual, visitor: BoundBooleanExpressionVisitor[T]) -> T: + return visitor.visit_less_than_or_equal(term=expr.term, literal=expr.literal) diff --git a/tests/expressions/test_expressions_base.py b/tests/expressions/test_expressions_base.py index cf74298296..94bf94b72a 100644 --- a/tests/expressions/test_expressions_base.py +++ b/tests/expressions/test_expressions_base.py @@ -17,12 +17,17 @@ import uuid from decimal import Decimal -from typing import List +from typing import List, Set import pytest from pyiceberg.expressions import base -from pyiceberg.expressions.literals import LongLiteral, StringLiteral, literal +from pyiceberg.expressions.literals import ( + Literal, + LongLiteral, + StringLiteral, + literal, +) from pyiceberg.schema import Accessor, Schema from pyiceberg.types import ( DoubleType, @@ -56,10 +61,10 @@ def __str__(self): return "testexprb" -class BooleanExpressionVisitor(base.BooleanExpressionVisitor[List]): - """A test implementation of a BooleanExpressionVisit +class ExampleVisitor(base.BooleanExpressionVisitor[List]): + """A test implementation of a BooleanExpressionVisitor - As this visitor visits each node, it appends an element to a `visit_histor` list. This enables testing that a given expression is + As this visitor visits each node, it appends an element to a `visit_history` list. This enables testing that a given expression is visited in an expected order by the `visit` method. """ @@ -104,17 +109,95 @@ def visit_test_expression_b(self) -> List: @base.visit.register(ExpressionA) -def _(obj: ExpressionA, visitor: BooleanExpressionVisitor) -> List: +def _(obj: ExpressionA, visitor: ExampleVisitor) -> List: """Visit a ExpressionA with a BooleanExpressionVisitor""" return visitor.visit_test_expression_a() @base.visit.register(ExpressionB) -def _(obj: ExpressionB, visitor: BooleanExpressionVisitor) -> List: +def _(obj: ExpressionB, visitor: ExampleVisitor) -> List: """Visit a ExpressionB with a BooleanExpressionVisitor""" return visitor.visit_test_expression_b() +class FooBoundBooleanExpressionVisitor(base.BoundBooleanExpressionVisitor[List]): + """A test implementation of a BoundBooleanExpressionVisitor + As this visitor visits each node, it appends an element to a `visit_history` list. This enables testing that a given bound expression is + visited in an expected order by the `visit` method. + """ + + def __init__(self): + self.visit_history: List = [] + + def visit_in(self, term: base.BoundTerm, literals: Set) -> List: + self.visit_history.append("IN") + return self.visit_history + + def visit_not_in(self, term: base.BoundTerm, literals: Set) -> List: + self.visit_history.append("NOT_IN") + return self.visit_history + + def visit_is_nan(self, term: base.BoundTerm) -> List: + self.visit_history.append("IS_NAN") + return self.visit_history + + def visit_not_nan(self, term: base.BoundTerm) -> List: + self.visit_history.append("NOT_NAN") + return self.visit_history + + def visit_is_null(self, term: base.BoundTerm) -> List: + self.visit_history.append("IS_NULL") + return self.visit_history + + def visit_not_null(self, term: base.BoundTerm) -> List: + self.visit_history.append("NOT_NULL") + return self.visit_history + + def visit_equal(self, term: base.BoundTerm, literal: Literal) -> List: # pylint: disable=redefined-outer-name + self.visit_history.append("EQUAL") + return self.visit_history + + def visit_not_equal(self, term: base.BoundTerm, literal: Literal) -> List: # pylint: disable=redefined-outer-name + self.visit_history.append("NOT_EQUAL") + return self.visit_history + + def visit_greater_than_or_equal(self, term: base.BoundTerm, literal: Literal) -> List: # pylint: disable=redefined-outer-name + self.visit_history.append("GREATER_THAN_OR_EQUAL") + return self.visit_history + + def visit_greater_than(self, term: base.BoundTerm, literal: Literal) -> List: # pylint: disable=redefined-outer-name + self.visit_history.append("GREATER_THAN") + return self.visit_history + + def visit_less_than(self, term: base.BoundTerm, literal: Literal) -> List: # pylint: disable=redefined-outer-name + self.visit_history.append("LESS_THAN") + return self.visit_history + + def visit_less_than_or_equal(self, term: base.BoundTerm, literal: Literal) -> List: # pylint: disable=redefined-outer-name + self.visit_history.append("LESS_THAN_OR_EQUAL") + return self.visit_history + + def visit_true(self) -> List: + self.visit_history.append("TRUE") + return self.visit_history + + def visit_false(self) -> List: + self.visit_history.append("FALSE") + return self.visit_history + + def visit_not(self, child_result: List) -> List: + self.visit_history.append("NOT") + return self.visit_history + + def visit_and(self, left_result: List, right_result: List) -> List: + self.visit_history.append("AND") + return self.visit_history + + def visit_or(self, left_result: List, right_result: List) -> List: + self.visit_history.append("OR") + return self.visit_history + + @pytest.mark.parametrize( "op, rep", [ @@ -530,7 +613,7 @@ def test_boolean_expression_visitor(): base.Not(ExpressionA()), ExpressionB(), ) - visitor = BooleanExpressionVisitor() + visitor = ExampleVisitor() result = base.visit(expr, visitor=visitor) assert result == [ "ExpressionA", @@ -552,7 +635,7 @@ def test_boolean_expression_visitor(): def test_boolean_expression_visit_raise_not_implemented_error(): """Test raise NotImplementedError when visiting an unsupported object type""" - visitor = BooleanExpressionVisitor() + visitor = ExampleVisitor() with pytest.raises(NotImplementedError) as exc_info: base.visit("foo", visitor=visitor) @@ -841,3 +924,230 @@ def test_not_expression_binding(unbound_not_expression, expected_bound_expressio """Test that visiting an unbound NOT expression with a bind-visitor returns the expected bound expression""" bound_expression = base.visit(unbound_not_expression, visitor=base.BindVisitor(schema=table_schema_simple)) assert bound_expression == expected_bound_expression + + +def test_bound_boolean_expression_visitor_and_in(): + """Test visiting an And and In expression with a bound boolean expression visitor""" + bound_expression = base.And( + base.BoundIn[str]( + term=base.BoundReference( + field=NestedField(field_id=1, name="foo", field_type=StringType(), required=False), + accessor=Accessor(position=0, inner=None), + ), + literals=(StringLiteral("foo"), StringLiteral("bar")), + ), + base.BoundIn[str]( + term=base.BoundReference( + field=NestedField(field_id=2, name="bar", field_type=StringType(), required=False), + accessor=Accessor(position=1, inner=None), + ), + literals=(StringLiteral("baz"), StringLiteral("qux")), + ), + ) + visitor = FooBoundBooleanExpressionVisitor() + result = base.visit(bound_expression, visitor=visitor) + assert result == ["IN", "IN", "AND"] + + +def test_bound_boolean_expression_visitor_or(): + """Test visiting an Or expression with a bound boolean expression visitor""" + bound_expression = base.Or( + base.Not( + base.BoundIn[str]( + base.BoundReference[str]( + field=NestedField(field_id=1, name="foo", field_type=StringType(), required=False), + accessor=Accessor(position=0, inner=None), + ), + {StringLiteral("foo"), StringLiteral("bar")}, + ) + ), + base.Not( + base.BoundIn[str]( + base.BoundReference[str]( + field=NestedField(field_id=2, name="bar", field_type=StringType(), required=False), + accessor=Accessor(position=1, inner=None), + ), + {StringLiteral("baz"), StringLiteral("qux")}, + ) + ), + ) + visitor = FooBoundBooleanExpressionVisitor() + result = base.visit(bound_expression, visitor=visitor) + assert result == ["IN", "NOT", "IN", "NOT", "OR"] + + +def test_bound_boolean_expression_visitor_equal(): + bound_expression = base.BoundEqualTo[str]( + term=base.BoundReference( + field=NestedField(field_id=2, name="bar", field_type=StringType(), required=False), + accessor=Accessor(position=1, inner=None), + ), + literal=StringLiteral("foo"), + ) + visitor = FooBoundBooleanExpressionVisitor() + result = base.visit(bound_expression, visitor=visitor) + assert result == ["EQUAL"] + + +def test_bound_boolean_expression_visitor_not_equal(): + bound_expression = base.BoundNotEqualTo[str]( + term=base.BoundReference( + field=NestedField(field_id=1, name="foo", field_type=StringType(), required=False), + accessor=Accessor(position=0, inner=None), + ), + literal=StringLiteral("foo"), + ) + visitor = FooBoundBooleanExpressionVisitor() + result = base.visit(bound_expression, visitor=visitor) + assert result == ["NOT_EQUAL"] + + +def test_bound_boolean_expression_visitor_always_true(): + bound_expression = base.AlwaysTrue() + visitor = FooBoundBooleanExpressionVisitor() + result = base.visit(bound_expression, visitor=visitor) + assert result == ["TRUE"] + + +def test_bound_boolean_expression_visitor_always_false(): + bound_expression = base.AlwaysFalse() + visitor = FooBoundBooleanExpressionVisitor() + result = base.visit(bound_expression, visitor=visitor) + assert result == ["FALSE"] + + +def test_bound_boolean_expression_visitor_in(): + bound_expression = base.BoundIn[str]( + term=base.BoundReference( + field=NestedField(field_id=1, name="foo", field_type=StringType(), required=False), + accessor=Accessor(position=0, inner=None), + ), + literals=(StringLiteral("foo"), StringLiteral("bar")), + ) + visitor = FooBoundBooleanExpressionVisitor() + result = base.visit(bound_expression, visitor=visitor) + assert result == ["IN"] + + +def test_bound_boolean_expression_visitor_not_in(): + bound_expression = base.BoundNotIn[str]( + term=base.BoundReference( + field=NestedField(field_id=1, name="foo", field_type=StringType(), required=False), + accessor=Accessor(position=0, inner=None), + ), + literals=(StringLiteral("foo"), StringLiteral("bar")), + ) + visitor = FooBoundBooleanExpressionVisitor() + result = base.visit(bound_expression, visitor=visitor) + assert result == ["NOT_IN"] + + +def test_bound_boolean_expression_visitor_is_nan(): + bound_expression = base.BoundIsNaN[str]( + term=base.BoundReference( + field=NestedField(field_id=3, name="baz", field_type=FloatType(), required=False), + accessor=Accessor(position=0, inner=None), + ), + ) + visitor = FooBoundBooleanExpressionVisitor() + result = base.visit(bound_expression, visitor=visitor) + assert result == ["IS_NAN"] + + +def test_bound_boolean_expression_visitor_not_nan(): + bound_expression = base.BoundNotNaN[str]( + term=base.BoundReference( + field=NestedField(field_id=3, name="baz", field_type=FloatType(), required=False), + accessor=Accessor(position=0, inner=None), + ), + ) + visitor = FooBoundBooleanExpressionVisitor() + result = base.visit(bound_expression, visitor=visitor) + assert result == ["NOT_NAN"] + + +def test_bound_boolean_expression_visitor_is_null(): + bound_expression = base.BoundIsNull[str]( + term=base.BoundReference( + field=NestedField(field_id=1, name="foo", field_type=StringType(), required=False), + accessor=Accessor(position=0, inner=None), + ), + ) + visitor = FooBoundBooleanExpressionVisitor() + result = base.visit(bound_expression, visitor=visitor) + assert result == ["IS_NULL"] + + +def test_bound_boolean_expression_visitor_not_null(): + bound_expression = base.BoundNotNull[str]( + term=base.BoundReference( + field=NestedField(field_id=1, name="foo", field_type=StringType(), required=False), + accessor=Accessor(position=0, inner=None), + ), + ) + visitor = FooBoundBooleanExpressionVisitor() + result = base.visit(bound_expression, visitor=visitor) + assert result == ["NOT_NULL"] + + +def test_bound_boolean_expression_visitor_greater_than(): + bound_expression = base.BoundGreaterThan[str]( + term=base.BoundReference( + field=NestedField(field_id=1, name="foo", field_type=StringType(), required=False), + accessor=Accessor(position=0, inner=None), + ), + literal=StringLiteral("foo"), + ) + visitor = FooBoundBooleanExpressionVisitor() + result = base.visit(bound_expression, visitor=visitor) + assert result == ["GREATER_THAN"] + + +def test_bound_boolean_expression_visitor_greater_than_or_equal(): + bound_expression = base.BoundGreaterThanOrEqual[str]( + term=base.BoundReference( + field=NestedField(field_id=1, name="foo", field_type=StringType(), required=False), + accessor=Accessor(position=0, inner=None), + ), + literal=StringLiteral("foo"), + ) + visitor = FooBoundBooleanExpressionVisitor() + result = base.visit(bound_expression, visitor=visitor) + assert result == ["GREATER_THAN_OR_EQUAL"] + + +def test_bound_boolean_expression_visitor_less_than(): + bound_expression = base.BoundLessThan[str]( + term=base.BoundReference( + field=NestedField(field_id=1, name="foo", field_type=StringType(), required=False), + accessor=Accessor(position=0, inner=None), + ), + literal=StringLiteral("foo"), + ) + visitor = FooBoundBooleanExpressionVisitor() + result = base.visit(bound_expression, visitor=visitor) + assert result == ["LESS_THAN"] + + +def test_bound_boolean_expression_visitor_less_than_or_equal(): + bound_expression = base.BoundLessThanOrEqual[str]( + term=base.BoundReference( + field=NestedField(field_id=1, name="foo", field_type=StringType(), required=False), + accessor=Accessor(position=0, inner=None), + ), + literal=StringLiteral("foo"), + ) + visitor = FooBoundBooleanExpressionVisitor() + result = base.visit(bound_expression, visitor=visitor) + assert result == ["LESS_THAN_OR_EQUAL"] + + +def test_bound_boolean_expression_visitor_raise_on_unbound_predicate(): + bound_expression = base.LessThanOrEqual[str]( + term=base.Reference("foo"), + literal=StringLiteral("foo"), + ) + visitor = FooBoundBooleanExpressionVisitor() + with pytest.raises(TypeError) as exc_info: + base.visit(bound_expression, visitor=visitor) + assert "Not a bound predicate" in str(exc_info.value) From d894d10089778f54e97da4aa55e800af2b9dc606 Mon Sep 17 00:00:00 2001 From: Fokko Driesprong Date: Fri, 23 Sep 2022 01:35:15 +0200 Subject: [PATCH 212/642] Python: Test if version is PEP440 compliant (#5834) --- pyproject.toml | 5 +++++ tests/test_version.py | 12 ++++-------- 2 files changed, 9 insertions(+), 8 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 087110fb94..a9f906017d 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -69,6 +69,7 @@ pre-commit = "^2.0.0" fastavro = "^1.6.1" coverage = { version = "^6.4.4", extras = ["toml"] } requests-mock = "^1.9.3" +packaging = "^21.3" [tool.poetry.scripts] pyiceberg = "pyiceberg.cli.console:run" @@ -165,5 +166,9 @@ ignore_missing_imports = true module = "s3fs.*" ignore_missing_imports = true +[[tool.mypy.overrides]] +module = "packaging.*" +ignore_missing_imports = true + [tool.coverage.run] source = ['pyiceberg/'] diff --git a/tests/test_version.py b/tests/test_version.py index 12a058e3f3..dffd702871 100644 --- a/tests/test_version.py +++ b/tests/test_version.py @@ -14,16 +14,12 @@ # KIND, either express or implied. See the License for the # specific language governing permissions and limitations # under the License. -import re -from pyiceberg import __version__ +from packaging.version import Version -VERSION_REGEX = re.compile(r"^\d+.\d+.\d+(.(dev|rc)\d+)?$") +from pyiceberg import __version__ def test_version_format(): - # should be in the format of 0.14.0 or 0.14.0.dev0 - assert VERSION_REGEX.search(__version__) - - # RCs should work as well - assert VERSION_REGEX.search("0.1.0.rc0") + # We should be able to parse the version + assert Version(__version__) From 4e9a00dc0e831a2c5ae740c9a149234575fee1d0 Mon Sep 17 00:00:00 2001 From: Fokko Driesprong Date: Fri, 23 Sep 2022 17:55:41 +0200 Subject: [PATCH 213/642] Python: Add the Makefile to the source distribution (#5838) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This makes it easier for testing. ``` ➜ python git:(fd-add-makefile-to-sdist) ✗ poetry build Building pyiceberg (0.1.0.dev0) - Building sdist - Built pyiceberg-0.1.0.dev0.tar.gz - Building wheel - Built pyiceberg-0.1.0.dev0-py3-none-any.whl ➜ python git:(fd-add-makefile-to-sdist) ✗ cd dist ➜ dist git:(fd-add-makefile-to-sdist) ✗ tar -xf pyiceberg-0.1.0.dev0.tar.gz ➜ dist git:(fd-add-makefile-to-sdist) ✗ cd pyiceberg-0.1.0.dev0 ➜ pyiceberg-0.1.0.dev0 git:(fd-add-makefile-to-sdist) ✗ find . | grep Makefile ./Makefile ➜ pyiceberg-0.1.0.dev0 git:(fd-add-makefile-to-sdist) ✗ head Makefile ``` --- pyproject.toml | 1 + 1 file changed, 1 insertion(+) diff --git a/pyproject.toml b/pyproject.toml index a9f906017d..3a51fca34e 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -38,6 +38,7 @@ packages = [ { from = "vendor", include = "fb303" }, { from = "vendor", include = "hive_metastore" }, { include = "tests", format = "sdist" }, + { include = "Makefile", format = "sdist" } ] From 60dd908e82b36c05fc3ef694a86d30aedf6e88f3 Mon Sep 17 00:00:00 2001 From: Fokko Driesprong Date: Fri, 23 Sep 2022 20:30:28 +0200 Subject: [PATCH 214/642] Python: Add NOTICE to source dist and wheel (#5843) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ``` ➜ python git:(master) ✗ poetry build Building pyiceberg (0.1.0.dev0) - Building sdist - Built pyiceberg-0.1.0.dev0.tar.gz - Building wheel - Built pyiceberg-0.1.0.dev0-py3-none-any.whl ➜ python git:(master) ✗ cd dist ➜ dist git:(master) ✗ tar -xf pyiceberg-0.1.0.tar.gz tar: Error opening archive: Failed to open 'pyiceberg-0.1.0.tar.gz' ➜ dist git:(master) ✗ tar -xf pyiceberg-0.1.0.dev0.tar.gz ➜ dist git:(master) ✗ cd pyiceberg-0.1.0.dev0 ➜ pyiceberg-0.1.0.dev0 git:(master) ✗ ls -lah | grep -i notice -rw-r--r-- 1 fokkodriesprong staff 251B Jun 14 21:52 NOTICE ``` --- pyproject.toml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index 3a51fca34e..dc672d9671 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -38,7 +38,8 @@ packages = [ { from = "vendor", include = "fb303" }, { from = "vendor", include = "hive_metastore" }, { include = "tests", format = "sdist" }, - { include = "Makefile", format = "sdist" } + { include = "Makefile", format = "sdist" }, + { include = "NOTICE", format = ["sdist", "wheel"] } ] From c8787d7da100f8313f27211c2529c389b7a0d6c2 Mon Sep 17 00:00:00 2001 From: Fokko Driesprong Date: Fri, 23 Sep 2022 23:49:46 +0200 Subject: [PATCH 215/642] Python: Add license checker to the source distribution (#5840) * Python: Add license checker to the source distribution * Missing comma --- Makefile | 3 ++ dev/check-license | 76 +++++++++++++++++++++++++++++++++++++++++++++++ pyproject.toml | 1 + 3 files changed, 80 insertions(+) create mode 100755 dev/check-license diff --git a/Makefile b/Makefile index 14a03c3bdc..c934a563d0 100644 --- a/Makefile +++ b/Makefile @@ -19,6 +19,9 @@ install: pip install poetry poetry install -E pyarrow -E hive -E s3fs +check-license: + ./dev/check-license + lint: poetry run pre-commit run --all-files diff --git a/dev/check-license b/dev/check-license new file mode 100755 index 0000000000..8c9ddb1f1f --- /dev/null +++ b/dev/check-license @@ -0,0 +1,76 @@ +#!/usr/bin/env bash + +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + + +acquire_rat_jar () { + + URL="https://repo.maven.apache.org/maven2/org/apache/rat/apache-rat/${RAT_VERSION}/apache-rat-${RAT_VERSION}.jar" + + JAR="$rat_jar" + + # Download rat launch jar if it hasn't been downloaded yet + if [ ! -f "$JAR" ]; then + # Download + printf "Attempting to fetch rat\n" + JAR_DL="${JAR}.part" + if [ $(command -v curl) ]; then + curl -L --silent "${URL}" > "$JAR_DL" && mv "$JAR_DL" "$JAR" + elif [ $(command -v wget) ]; then + wget --quiet ${URL} -O "$JAR_DL" && mv "$JAR_DL" "$JAR" + else + printf "You do not have curl or wget installed, please install rat manually.\n" + exit -1 + fi + fi + + unzip -tq "$JAR" &> /dev/null + if [ $? -ne 0 ]; then + # We failed to download + rm "$JAR" + printf "Our attempt to download rat locally to ${JAR} failed. Please install rat manually.\n" + exit -1 + fi +} + +# Go to the Spark project root directory +FWDIR="$(cd "`dirname "$0"`"/..; pwd)" +cd "$FWDIR" + +if test -x "$JAVA_HOME/bin/java"; then + declare java_cmd="$JAVA_HOME/bin/java" +else + declare java_cmd=java +fi + +export RAT_VERSION=0.12 +export rat_jar="$FWDIR"/lib/apache-rat-${RAT_VERSION}.jar +mkdir -p "$FWDIR"/lib + +[[ -f "$rat_jar" ]] || acquire_rat_jar || { + echo "Download failed. Obtain the rat jar manually and place it at $rat_jar" + exit 1 +} + +mkdir -p build +$java_cmd -jar "$rat_jar" -d "$FWDIR" + +if [ $? -ne 0 ]; then + echo "RAT exited abnormally" + exit 1 +fi diff --git a/pyproject.toml b/pyproject.toml index dc672d9671..b5f72bd975 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -38,6 +38,7 @@ packages = [ { from = "vendor", include = "fb303" }, { from = "vendor", include = "hive_metastore" }, { include = "tests", format = "sdist" }, + { include = "dev/check-license", format = "sdist" }, { include = "Makefile", format = "sdist" }, { include = "NOTICE", format = ["sdist", "wheel"] } ] From 0e116640073164bc220948428ef7732798e2933d Mon Sep 17 00:00:00 2001 From: Fokko Driesprong Date: Sat, 24 Sep 2022 02:14:33 +0200 Subject: [PATCH 216/642] Python: Bump pre-commit (#5842) --- .pre-commit-config.yaml | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 54c644b32c..7fbabf315e 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -29,7 +29,7 @@ repos: - id: check-yaml - id: check-ast - repo: https://github.com/ambv/black - rev: 22.6.0 + rev: 22.8.0 hooks: - id: black - repo: https://github.com/pre-commit/mirrors-isort @@ -48,12 +48,12 @@ repos: - id: pycln args: [--config=python/pyproject.toml] - repo: https://github.com/asottile/pyupgrade - rev: v2.37.3 + rev: v2.38.0 hooks: - id: pyupgrade args: [--py38-plus] - repo: https://github.com/pycqa/pylint - rev: v2.14.5 + rev: v2.15.3 hooks: - id: pylint args: [ --rcfile=python/pylintrc ] @@ -61,8 +61,8 @@ repos: rev: '5.0.4' hooks: - id: flake8 - args: [ "--ignore=E501,W503,E203" ] - additional_dependencies: [ flake8-bugbear==22.7.1, flake8-comprehensions==3.10.0 ] + args: [ "--ignore=E501,W503,E203,B024" ] + additional_dependencies: [ flake8-bugbear==22.9.11, flake8-comprehensions==3.10.0 ] - repo: https://github.com/executablebooks/mdformat rev: 0.7.16 hooks: From baee77f89978a95e90a98e543bdf4e7d64d262de Mon Sep 17 00:00:00 2001 From: Fokko Driesprong Date: Sat, 24 Sep 2022 07:45:15 +0200 Subject: [PATCH 217/642] Python: Remove version modifier (#5835) --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index b5f72bd975..665ef46a77 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -17,7 +17,7 @@ [tool.poetry] name = "pyiceberg" -version = "0.1.0.dev0" +version = "0.1.0" readme = "README.md" homepage = "https://iceberg.apache.org/" repository = "https://github.com/apache/iceberg/" From 7b9d6c31004e2c695f17c897a2fe951deca8512c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Eduard=20Tudenh=C3=B6fner?= Date: Fri, 30 Sep 2022 00:01:11 +0200 Subject: [PATCH 218/642] Python: Don't throw missing URI error when --uri was provided (#5885) --- pyiceberg/cli/console.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/pyiceberg/cli/console.py b/pyiceberg/cli/console.py index e8234d7c7e..bac4ae6bc2 100644 --- a/pyiceberg/cli/console.py +++ b/pyiceberg/cli/console.py @@ -68,9 +68,11 @@ def run(ctx: Context, catalog: str, verbose: bool, output: str, uri: Optional[st try: ctx.obj["catalog"] = load_catalog(catalog, **properties) except ValueError as exc: - raise ValueError( - f"URI missing, please provide using --uri, the config or environment variable PYICEBERG_CATALOG__{catalog.upper()}__URI" - ) from exc + if not uri: + raise ValueError( + f"URI missing, please provide using --uri, the config or environment variable PYICEBERG_CATALOG__{catalog.upper()}__URI" + ) from exc + raise exc except Exception as e: ctx.obj["output"].exception(e) ctx.exit(1) From 90bbe3d3992a07ec86d221445406b865872af1b4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Eduard=20Tudenh=C3=B6fner?= Date: Fri, 30 Sep 2022 00:11:29 +0200 Subject: [PATCH 219/642] Python: Update README test instructions (#5869) --- README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 6dfefbce6a..7e1d06b24c 100644 --- a/README.md +++ b/README.md @@ -35,8 +35,8 @@ Development is made easy using [Poetry](https://python-poetry.org/docs/#installa ```bash poetry shell -poetry install -E pyarrow -pytest +make install +make test ``` For more information, please refer to the [Manage environments](https://python-poetry.org/docs/managing-environments/) section of Poetry. From c9cdf36bae5f4ab7ac7ad0b3e9417eb8535f5ecd Mon Sep 17 00:00:00 2001 From: Samuel Redai <43911210+samredai@users.noreply.github.com> Date: Thu, 29 Sep 2022 18:12:12 -0400 Subject: [PATCH 220/642] Docs: Fix formatting in Python CLI (#5868) --- mkdocs/docs/index.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/mkdocs/docs/index.md b/mkdocs/docs/index.md index 35351ef0c8..35ffee1875 100644 --- a/mkdocs/docs/index.md +++ b/mkdocs/docs/index.md @@ -131,7 +131,7 @@ nyc.taxis ``` ```sh -pyiceberg describe nyc.taxis +➜ pyiceberg describe nyc.taxis Table format version 1 Metadata location file:/.../nyc.db/taxis/metadata/00000-aa3a3eac-ea08-4255-b890-383a64a94e42.metadata.json Table UUID 6cdfda33-bfa3-48a7-a09e-7abb462e3460 @@ -168,7 +168,7 @@ write.format.default parquet Or output in JSON for automation: ```sh -pyiceberg --output json describe nyc.taxis | jq +➜ pyiceberg --output json describe nyc.taxis | jq { "identifier": [ "nyc", From 0016f2728ef2750c01012fee0981a821ad0de05a Mon Sep 17 00:00:00 2001 From: Fokko Driesprong Date: Fri, 30 Sep 2022 15:03:45 +0200 Subject: [PATCH 221/642] Python: Add code conventions for Python (#5879) * Python: Add code conventions for Python * Comments --- CONTRIBUTING.md | 33 +++++++++++++++++++++++++ pyiceberg/utils/deprecated.py | 45 +++++++++++++++++++++++++++++++++++ 2 files changed, 78 insertions(+) create mode 100644 pyiceberg/utils/deprecated.py diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 1bde5ce7db..d673b43920 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -94,3 +94,36 @@ make test PYTEST_ARGS="--pdb" ``` To see all available pytest arguments, run `make test PYTEST_ARGS="--help"`. + +# Code standards + +Below are the formalized conventions that we adhere to in the PyIceberg project. The goal of this is to have a common agreement on how to evolve the codebase, but also using it as guidelines for newcomers to the project. + +## API Compatibility + +We try to keep the Python public API compatible across versions. The Python official [PEP-8](https://peps.python.org/pep-0008/) defines Public methods as: _Public attributes should have no leading underscores_. This means not removing any methods without any notice, or removing or renaming any existing parameters. Adding new optional parameters is okay. + +If you want to remove a method, please add a deprecation notice by annotating the function using `@deprecated`: + +```python +from pyiceberg.utils.deprecated import deprecated + + +@deprecated( + deprecated_in="0.1.0", + removed_in="0.2.0", + help_message="Please use load_something_else() instead", +) +def load_something(): + pass +``` + +Which will warn: + +``` +Call to load_something, deprecated in 0.1.0, will be removed in 0.2.0. Please use load_something_else() instead. +``` + +## Third party libraries + +Since we expect PyIceberg to be integrated into the Python ecosystem, we want to be hesitant with the use of third party packages. Adding a lot of packages makes the library heavyweight, and causes incompatibilities with other projects if they use a different version of the library. Also, big libraries such as `s3fs`, `pyarrow`, `thrift` should be optional to avoid downloading everything, while not being sure if is actually being used. diff --git a/pyiceberg/utils/deprecated.py b/pyiceberg/utils/deprecated.py new file mode 100644 index 0000000000..318849ac95 --- /dev/null +++ b/pyiceberg/utils/deprecated.py @@ -0,0 +1,45 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +import functools +import warnings +from typing import Callable, Optional + + +def deprecated(deprecated_in: str, removed_in: str, help_message: Optional[str] = None): + """This is a decorator which can be used to mark functions + as deprecated. It will result in a warning being emitted + when the function is used.""" + + if help_message is not None: + help_message = f" {help_message}." + + def decorator(func: Callable): + @functools.wraps(func) + def new_func(*args, **kwargs): + warnings.simplefilter("always", DeprecationWarning) # turn off filter + + warnings.warn( + f"Call to {func.__name__}, deprecated in {deprecated_in}, will be removed in {removed_in}.{help_message}", + category=DeprecationWarning, + stacklevel=2, + ) + warnings.simplefilter("default", DeprecationWarning) # reset filter + return func(*args, **kwargs) + + return new_func + + return decorator From c49252450a09b6726fa014a6c94bc25b14c6fd2b Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Sat, 1 Oct 2022 11:53:04 +0200 Subject: [PATCH 222/642] Build: Bump jinja2 from 3.0.3 to 3.1.2 in /python (#5849) Bumps [jinja2](https://github.com/pallets/jinja) from 3.0.3 to 3.1.2. - [Release notes](https://github.com/pallets/jinja/releases) - [Changelog](https://github.com/pallets/jinja/blob/main/CHANGES.rst) - [Commits](https://github.com/pallets/jinja/compare/3.0.3...3.1.2) --- updated-dependencies: - dependency-name: jinja2 dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- mkdocs/requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mkdocs/requirements.txt b/mkdocs/requirements.txt index 642a688ebc..b21284bb84 100644 --- a/mkdocs/requirements.txt +++ b/mkdocs/requirements.txt @@ -16,4 +16,4 @@ # under the License. mkdocs==1.3.1 -jinja2==3.0.3 +jinja2==3.1.2 From 606e94b363435a4917a108351c1c42da5929fd6c Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Sun, 2 Oct 2022 20:59:50 +0200 Subject: [PATCH 223/642] Build: Bump coverage from 6.4.4 to 6.5.0 in /python (#5904) Bumps [coverage](https://github.com/nedbat/coveragepy) from 6.4.4 to 6.5.0. - [Release notes](https://github.com/nedbat/coveragepy/releases) - [Changelog](https://github.com/nedbat/coveragepy/blob/master/CHANGES.rst) - [Commits](https://github.com/nedbat/coveragepy/compare/6.4.4...6.5.0) --- updated-dependencies: - dependency-name: coverage dependency-type: direct:development update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- poetry.lock | 514 ++++++++++++++++++++++++++++++++++++++++++++++--- pyproject.toml | 2 +- 2 files changed, 484 insertions(+), 32 deletions(-) diff --git a/poetry.lock b/poetry.lock index 1ee6a3fa0d..db7c36bd63 100644 --- a/poetry.lock +++ b/poetry.lock @@ -34,7 +34,7 @@ multidict = ">=4.5,<7.0" yarl = ">=1.0,<2.0" [package.extras] -speedups = ["aiodns", "brotli", "cchardet"] +speedups = ["Brotli", "aiodns", "cchardet"] [[package]] name = "aioitertools" @@ -109,7 +109,7 @@ name = "cffi" version = "1.15.1" description = "Foreign Function Interface for Python calling C code." category = "main" -optional = true +optional = false python-versions = "*" [package.dependencies] @@ -166,7 +166,7 @@ test = ["flake8 (==3.7.8)", "hypothesis (==3.55.3)"] [[package]] name = "coverage" -version = "6.4.4" +version = "6.5.0" description = "Code coverage measurement for Python" category = "dev" optional = false @@ -334,6 +334,9 @@ category = "dev" optional = false python-versions = ">=2.7,!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,!=3.4.*,!=3.5.*,!=3.6.*" +[package.dependencies] +setuptools = "*" + [[package]] name = "numpy" version = "1.23.3" @@ -428,7 +431,7 @@ name = "pycparser" version = "2.21" description = "C parser in Python" category = "main" -optional = true +optional = false python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*" [[package]] @@ -599,6 +602,19 @@ fsspec = "2022.8.2" awscli = ["aiobotocore[awscli] (>=2.4.0,<2.5.0)"] boto3 = ["aiobotocore[boto3] (>=2.4.0,<2.5.0)"] +[[package]] +name = "setuptools" +version = "65.4.1" +description = "Easily download, build, install, upgrade, and uninstall Python packages" +category = "dev" +optional = false +python-versions = ">=3.7" + +[package.extras] +docs = ["furo", "jaraco.packaging (>=9)", "jaraco.tidelift (>=1.4)", "pygments-github-lexers (==0.0.5)", "rst.linker (>=1.9)", "sphinx (>=3.5)", "sphinx-favicon", "sphinx-hoverxref (<2)", "sphinx-inline-tabs", "sphinx-notfound-page (==0.8.3)", "sphinx-reredirects", "sphinxcontrib-towncrier"] +testing = ["build[virtualenv]", "filelock (>=3.4.0)", "flake8 (<5)", "flake8-2020", "ini2toml[lite] (>=0.9)", "jaraco.envs (>=2.2)", "jaraco.path (>=3.2.0)", "mock", "pip (>=19.1)", "pip-run (>=8.8)", "pytest (>=6)", "pytest-black (>=0.3.7)", "pytest-checkdocs (>=2.4)", "pytest-cov", "pytest-enabler (>=1.3)", "pytest-flake8", "pytest-mypy (>=0.9.1)", "pytest-perf", "pytest-xdist", "tomli-w (>=1.0.0)", "virtualenv (>=13.0.0)", "wheel"] +testing-integration = ["build[virtualenv]", "filelock (>=3.4.0)", "jaraco.envs (>=2.2)", "jaraco.path (>=3.2.0)", "pytest", "pytest-enabler", "pytest-xdist", "tomli", "virtualenv (>=13.0.0)", "wheel"] + [[package]] name = "six" version = "1.16.0" @@ -714,7 +730,7 @@ name = "zstandard" version = "0.18.0" description = "Zstandard bindings for Python" category = "main" -optional = true +optional = false python-versions = ">=3.6" [package.dependencies] @@ -726,25 +742,109 @@ cffi = ["cffi (>=1.11)"] [extras] hive = ["thrift"] pyarrow = ["pyarrow"] -python-snappy = ["zstandard"] s3fs = ["s3fs"] snappy = ["python-snappy"] [metadata] lock-version = "1.1" python-versions = "^3.8" -content-hash = "338797a2b74b191ab928e59a772d455a1e454df2c70d1ccae51088d43b06d4b4" +content-hash = "5fc9986c3483dd8965a5f676d7f621bed361f0fdfbe7646875aa77f9c920af88" [metadata.files] aiobotocore = [ {file = "aiobotocore-2.4.0-py3-none-any.whl", hash = "sha256:6c25381d31b712652bc2f6008683949351c240c56d24b1d8ae252c1034a50f63"}, {file = "aiobotocore-2.4.0.tar.gz", hash = "sha256:f9fe0698cc497861bdb54cd16161c804031f758ada9480c35540f20c0c078385"}, ] -aiohttp = [] -aioitertools = [] -aiosignal = [] -async-timeout = [] -attrs = [] +aiohttp = [ + {file = "aiohttp-3.8.1-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:1ed0b6477896559f17b9eaeb6d38e07f7f9ffe40b9f0f9627ae8b9926ae260a8"}, + {file = "aiohttp-3.8.1-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:7dadf3c307b31e0e61689cbf9e06be7a867c563d5a63ce9dca578f956609abf8"}, + {file = "aiohttp-3.8.1-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:a79004bb58748f31ae1cbe9fa891054baaa46fb106c2dc7af9f8e3304dc30316"}, + {file = "aiohttp-3.8.1-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:12de6add4038df8f72fac606dff775791a60f113a725c960f2bab01d8b8e6b15"}, + {file = "aiohttp-3.8.1-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:6f0d5f33feb5f69ddd57a4a4bd3d56c719a141080b445cbf18f238973c5c9923"}, + {file = "aiohttp-3.8.1-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:eaba923151d9deea315be1f3e2b31cc39a6d1d2f682f942905951f4e40200922"}, + {file = "aiohttp-3.8.1-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_12_i686.manylinux2010_i686.whl", hash = "sha256:099ebd2c37ac74cce10a3527d2b49af80243e2a4fa39e7bce41617fbc35fa3c1"}, + {file = "aiohttp-3.8.1-cp310-cp310-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:2e5d962cf7e1d426aa0e528a7e198658cdc8aa4fe87f781d039ad75dcd52c516"}, + {file = "aiohttp-3.8.1-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:fa0ffcace9b3aa34d205d8130f7873fcfefcb6a4dd3dd705b0dab69af6712642"}, + {file = "aiohttp-3.8.1-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:61bfc23df345d8c9716d03717c2ed5e27374e0fe6f659ea64edcd27b4b044cf7"}, + {file = "aiohttp-3.8.1-cp310-cp310-musllinux_1_1_ppc64le.whl", hash = "sha256:31560d268ff62143e92423ef183680b9829b1b482c011713ae941997921eebc8"}, + {file = "aiohttp-3.8.1-cp310-cp310-musllinux_1_1_s390x.whl", hash = "sha256:01d7bdb774a9acc838e6b8f1d114f45303841b89b95984cbb7d80ea41172a9e3"}, + {file = "aiohttp-3.8.1-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:97ef77eb6b044134c0b3a96e16abcb05ecce892965a2124c566af0fd60f717e2"}, + {file = "aiohttp-3.8.1-cp310-cp310-win32.whl", hash = "sha256:c2aef4703f1f2ddc6df17519885dbfa3514929149d3ff900b73f45998f2532fa"}, + {file = "aiohttp-3.8.1-cp310-cp310-win_amd64.whl", hash = "sha256:713ac174a629d39b7c6a3aa757b337599798da4c1157114a314e4e391cd28e32"}, + {file = "aiohttp-3.8.1-cp36-cp36m-macosx_10_9_x86_64.whl", hash = "sha256:473d93d4450880fe278696549f2e7aed8cd23708c3c1997981464475f32137db"}, + {file = "aiohttp-3.8.1-cp36-cp36m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:99b5eeae8e019e7aad8af8bb314fb908dd2e028b3cdaad87ec05095394cce632"}, + {file = "aiohttp-3.8.1-cp36-cp36m-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:3af642b43ce56c24d063325dd2cf20ee012d2b9ba4c3c008755a301aaea720ad"}, + {file = "aiohttp-3.8.1-cp36-cp36m-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:c3630c3ef435c0a7c549ba170a0633a56e92629aeed0e707fec832dee313fb7a"}, + {file = "aiohttp-3.8.1-cp36-cp36m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_12_i686.manylinux2010_i686.whl", hash = "sha256:4a4a4e30bf1edcad13fb0804300557aedd07a92cabc74382fdd0ba6ca2661091"}, + {file = "aiohttp-3.8.1-cp36-cp36m-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:6f8b01295e26c68b3a1b90efb7a89029110d3a4139270b24fda961893216c440"}, + {file = "aiohttp-3.8.1-cp36-cp36m-musllinux_1_1_aarch64.whl", hash = "sha256:a25fa703a527158aaf10dafd956f7d42ac6d30ec80e9a70846253dd13e2f067b"}, + {file = "aiohttp-3.8.1-cp36-cp36m-musllinux_1_1_i686.whl", hash = "sha256:5bfde62d1d2641a1f5173b8c8c2d96ceb4854f54a44c23102e2ccc7e02f003ec"}, + {file = "aiohttp-3.8.1-cp36-cp36m-musllinux_1_1_ppc64le.whl", hash = "sha256:51467000f3647d519272392f484126aa716f747859794ac9924a7aafa86cd411"}, + {file = "aiohttp-3.8.1-cp36-cp36m-musllinux_1_1_s390x.whl", hash = "sha256:03a6d5349c9ee8f79ab3ff3694d6ce1cfc3ced1c9d36200cb8f08ba06bd3b782"}, + {file = "aiohttp-3.8.1-cp36-cp36m-musllinux_1_1_x86_64.whl", hash = "sha256:102e487eeb82afac440581e5d7f8f44560b36cf0bdd11abc51a46c1cd88914d4"}, + {file = "aiohttp-3.8.1-cp36-cp36m-win32.whl", hash = "sha256:4aed991a28ea3ce320dc8ce655875e1e00a11bdd29fe9444dd4f88c30d558602"}, + {file = "aiohttp-3.8.1-cp36-cp36m-win_amd64.whl", hash = "sha256:b0e20cddbd676ab8a64c774fefa0ad787cc506afd844de95da56060348021e96"}, + {file = "aiohttp-3.8.1-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:37951ad2f4a6df6506750a23f7cbabad24c73c65f23f72e95897bb2cecbae676"}, + {file = "aiohttp-3.8.1-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:5c23b1ad869653bc818e972b7a3a79852d0e494e9ab7e1a701a3decc49c20d51"}, + {file = "aiohttp-3.8.1-cp37-cp37m-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:15b09b06dae900777833fe7fc4b4aa426556ce95847a3e8d7548e2d19e34edb8"}, + {file = "aiohttp-3.8.1-cp37-cp37m-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:477c3ea0ba410b2b56b7efb072c36fa91b1e6fc331761798fa3f28bb224830dd"}, + {file = "aiohttp-3.8.1-cp37-cp37m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_12_i686.manylinux2010_i686.whl", hash = "sha256:2f2f69dca064926e79997f45b2f34e202b320fd3782f17a91941f7eb85502ee2"}, + {file = "aiohttp-3.8.1-cp37-cp37m-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:ef9612483cb35171d51d9173647eed5d0069eaa2ee812793a75373447d487aa4"}, + {file = "aiohttp-3.8.1-cp37-cp37m-musllinux_1_1_aarch64.whl", hash = "sha256:6d69f36d445c45cda7b3b26afef2fc34ef5ac0cdc75584a87ef307ee3c8c6d00"}, + {file = "aiohttp-3.8.1-cp37-cp37m-musllinux_1_1_i686.whl", hash = "sha256:55c3d1072704d27401c92339144d199d9de7b52627f724a949fc7d5fc56d8b93"}, + {file = "aiohttp-3.8.1-cp37-cp37m-musllinux_1_1_ppc64le.whl", hash = "sha256:b9d00268fcb9f66fbcc7cd9fe423741d90c75ee029a1d15c09b22d23253c0a44"}, + {file = "aiohttp-3.8.1-cp37-cp37m-musllinux_1_1_s390x.whl", hash = "sha256:07b05cd3305e8a73112103c834e91cd27ce5b4bd07850c4b4dbd1877d3f45be7"}, + {file = "aiohttp-3.8.1-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:c34dc4958b232ef6188c4318cb7b2c2d80521c9a56c52449f8f93ab7bc2a8a1c"}, + {file = "aiohttp-3.8.1-cp37-cp37m-win32.whl", hash = "sha256:d2f9b69293c33aaa53d923032fe227feac867f81682f002ce33ffae978f0a9a9"}, + {file = "aiohttp-3.8.1-cp37-cp37m-win_amd64.whl", hash = "sha256:6ae828d3a003f03ae31915c31fa684b9890ea44c9c989056fea96e3d12a9fa17"}, + {file = "aiohttp-3.8.1-cp38-cp38-macosx_10_9_universal2.whl", hash = "sha256:0c7ebbbde809ff4e970824b2b6cb7e4222be6b95a296e46c03cf050878fc1785"}, + {file = "aiohttp-3.8.1-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:8b7ef7cbd4fec9a1e811a5de813311ed4f7ac7d93e0fda233c9b3e1428f7dd7b"}, + {file = "aiohttp-3.8.1-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:c3d6a4d0619e09dcd61021debf7059955c2004fa29f48788a3dfaf9c9901a7cd"}, + {file = "aiohttp-3.8.1-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:718626a174e7e467f0558954f94af117b7d4695d48eb980146016afa4b580b2e"}, + {file = "aiohttp-3.8.1-cp38-cp38-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:589c72667a5febd36f1315aa6e5f56dd4aa4862df295cb51c769d16142ddd7cd"}, + {file = "aiohttp-3.8.1-cp38-cp38-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:2ed076098b171573161eb146afcb9129b5ff63308960aeca4b676d9d3c35e700"}, + {file = "aiohttp-3.8.1-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_12_i686.manylinux2010_i686.whl", hash = "sha256:086f92daf51a032d062ec5f58af5ca6a44d082c35299c96376a41cbb33034675"}, + {file = "aiohttp-3.8.1-cp38-cp38-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:11691cf4dc5b94236ccc609b70fec991234e7ef8d4c02dd0c9668d1e486f5abf"}, + {file = "aiohttp-3.8.1-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:31d1e1c0dbf19ebccbfd62eff461518dcb1e307b195e93bba60c965a4dcf1ba0"}, + {file = "aiohttp-3.8.1-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:11a67c0d562e07067c4e86bffc1553f2cf5b664d6111c894671b2b8712f3aba5"}, + {file = "aiohttp-3.8.1-cp38-cp38-musllinux_1_1_ppc64le.whl", hash = "sha256:bb01ba6b0d3f6c68b89fce7305080145d4877ad3acaed424bae4d4ee75faa950"}, + {file = "aiohttp-3.8.1-cp38-cp38-musllinux_1_1_s390x.whl", hash = "sha256:44db35a9e15d6fe5c40d74952e803b1d96e964f683b5a78c3cc64eb177878155"}, + {file = "aiohttp-3.8.1-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:844a9b460871ee0a0b0b68a64890dae9c415e513db0f4a7e3cab41a0f2fedf33"}, + {file = "aiohttp-3.8.1-cp38-cp38-win32.whl", hash = "sha256:7d08744e9bae2ca9c382581f7dce1273fe3c9bae94ff572c3626e8da5b193c6a"}, + {file = "aiohttp-3.8.1-cp38-cp38-win_amd64.whl", hash = "sha256:04d48b8ce6ab3cf2097b1855e1505181bdd05586ca275f2505514a6e274e8e75"}, + {file = "aiohttp-3.8.1-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:f5315a2eb0239185af1bddb1abf472d877fede3cc8d143c6cddad37678293237"}, + {file = "aiohttp-3.8.1-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:a996d01ca39b8dfe77440f3cd600825d05841088fd6bc0144cc6c2ec14cc5f74"}, + {file = "aiohttp-3.8.1-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:13487abd2f761d4be7c8ff9080de2671e53fff69711d46de703c310c4c9317ca"}, + {file = "aiohttp-3.8.1-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:ea302f34477fda3f85560a06d9ebdc7fa41e82420e892fc50b577e35fc6a50b2"}, + {file = "aiohttp-3.8.1-cp39-cp39-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:a2f635ce61a89c5732537a7896b6319a8fcfa23ba09bec36e1b1ac0ab31270d2"}, + {file = "aiohttp-3.8.1-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:e999f2d0e12eea01caeecb17b653f3713d758f6dcc770417cf29ef08d3931421"}, + {file = "aiohttp-3.8.1-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_12_i686.manylinux2010_i686.whl", hash = "sha256:0770e2806a30e744b4e21c9d73b7bee18a1cfa3c47991ee2e5a65b887c49d5cf"}, + {file = "aiohttp-3.8.1-cp39-cp39-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:d15367ce87c8e9e09b0f989bfd72dc641bcd04ba091c68cd305312d00962addd"}, + {file = "aiohttp-3.8.1-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:6c7cefb4b0640703eb1069835c02486669312bf2f12b48a748e0a7756d0de33d"}, + {file = "aiohttp-3.8.1-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:71927042ed6365a09a98a6377501af5c9f0a4d38083652bcd2281a06a5976724"}, + {file = "aiohttp-3.8.1-cp39-cp39-musllinux_1_1_ppc64le.whl", hash = "sha256:28d490af82bc6b7ce53ff31337a18a10498303fe66f701ab65ef27e143c3b0ef"}, + {file = "aiohttp-3.8.1-cp39-cp39-musllinux_1_1_s390x.whl", hash = "sha256:b6613280ccedf24354406caf785db748bebbddcf31408b20c0b48cb86af76866"}, + {file = "aiohttp-3.8.1-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:81e3d8c34c623ca4e36c46524a3530e99c0bc95ed068fd6e9b55cb721d408fb2"}, + {file = "aiohttp-3.8.1-cp39-cp39-win32.whl", hash = "sha256:7187a76598bdb895af0adbd2fb7474d7f6025d170bc0a1130242da817ce9e7d1"}, + {file = "aiohttp-3.8.1-cp39-cp39-win_amd64.whl", hash = "sha256:1c182cb873bc91b411e184dab7a2b664d4fea2743df0e4d57402f7f3fa644bac"}, + {file = "aiohttp-3.8.1.tar.gz", hash = "sha256:fc5471e1a54de15ef71c1bc6ebe80d4dc681ea600e68bfd1cbce40427f0b7578"}, +] +aioitertools = [ + {file = "aioitertools-0.10.0-py3-none-any.whl", hash = "sha256:a2ea2a39ebf272a2fbb58bfdb73e1daeeb6686edbbc8082215dfc8b8ffffa6e8"}, + {file = "aioitertools-0.10.0.tar.gz", hash = "sha256:7d1d1d4a03d462c5a0840787d3df098f125847e0d38b833b30f8f8cbc45a1420"}, +] +aiosignal = [ + {file = "aiosignal-1.2.0-py3-none-any.whl", hash = "sha256:26e62109036cd181df6e6ad646f91f0dcfd05fe16d0cb924138ff2ab75d64e3a"}, + {file = "aiosignal-1.2.0.tar.gz", hash = "sha256:78ed67db6c7b7ced4f98e495e572106d5c432a93e1ddd1bf475e1dc05f5b7df2"}, +] +async-timeout = [ + {file = "async-timeout-4.0.2.tar.gz", hash = "sha256:2163e1640ddb52b7a8c80d0a67a08587e5d245cc9c553a74a847056bc2976b15"}, + {file = "async_timeout-4.0.2-py3-none-any.whl", hash = "sha256:8ca1e4fcf50d07413d66d1a5e416e42cfdf5851c981d679a09851a6853383b3c"}, +] +attrs = [ + {file = "attrs-22.1.0-py2.py3-none-any.whl", hash = "sha256:86efa402f67bf2df34f51a335487cf46b1ec130d02b8d39fd248abfd30da551c"}, + {file = "attrs-22.1.0.tar.gz", hash = "sha256:29adc2665447e5191d0e7c568fde78b21f9672d344281d0c6e1ab085429b22b6"}, +] botocore = [ {file = "botocore-1.27.59-py3-none-any.whl", hash = "sha256:69d756791fc024bda54f6c53f71ae34e695ee41bbbc1743d9179c4837a4929da"}, {file = "botocore-1.27.59.tar.gz", hash = "sha256:eda4aed6ee719a745d1288eaf1beb12f6f6448ad1fa12f159405db14ba9c92cf"}, @@ -823,14 +923,74 @@ cfgv = [ {file = "cfgv-3.3.1-py2.py3-none-any.whl", hash = "sha256:c6a0883f3917a037485059700b9e75da2464e6c27051014ad85ba6aaa5884426"}, {file = "cfgv-3.3.1.tar.gz", hash = "sha256:f5a830efb9ce7a445376bb66ec94c638a9787422f96264c98edc6bdeed8ab736"}, ] -charset-normalizer = [] -click = [] +charset-normalizer = [ + {file = "charset-normalizer-2.1.1.tar.gz", hash = "sha256:5a3d016c7c547f69d6f81fb0db9449ce888b418b5b9952cc5e6e66843e9dd845"}, + {file = "charset_normalizer-2.1.1-py3-none-any.whl", hash = "sha256:83e9a75d1911279afd89352c68b45348559d1fc0506b054b346651b5e7fee29f"}, +] +click = [ + {file = "click-8.1.3-py3-none-any.whl", hash = "sha256:bb4d8133cb15a609f44e8213d9b391b0809795062913b383c62be0ee95b1db48"}, + {file = "click-8.1.3.tar.gz", hash = "sha256:7682dc8afb30297001674575ea00d1814d808d6a36af415a82bd481d37ba7b8e"}, +] colorama = [ {file = "colorama-0.4.5-py2.py3-none-any.whl", hash = "sha256:854bf444933e37f5824ae7bfc1e98d5bce2ebe4160d46b5edf346a89358e99da"}, {file = "colorama-0.4.5.tar.gz", hash = "sha256:e6c6b4334fc50988a639d9b98aa429a0b57da6e17b9a44f0451f930b6967b7a4"}, ] -commonmark = [] -coverage = [] +commonmark = [ + {file = "commonmark-0.9.1-py2.py3-none-any.whl", hash = "sha256:da2f38c92590f83de410ba1a3cbceafbc74fee9def35f9251ba9a971d6d66fd9"}, + {file = "commonmark-0.9.1.tar.gz", hash = "sha256:452f9dc859be7f06631ddcb328b6919c67984aca654e5fefb3914d54691aed60"}, +] +coverage = [ + {file = "coverage-6.5.0-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:ef8674b0ee8cc11e2d574e3e2998aea5df5ab242e012286824ea3c6970580e53"}, + {file = "coverage-6.5.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:784f53ebc9f3fd0e2a3f6a78b2be1bd1f5575d7863e10c6e12504f240fd06660"}, + {file = "coverage-6.5.0-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:b4a5be1748d538a710f87542f22c2cad22f80545a847ad91ce45e77417293eb4"}, + {file = "coverage-6.5.0-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:83516205e254a0cb77d2d7bb3632ee019d93d9f4005de31dca0a8c3667d5bc04"}, + {file = "coverage-6.5.0-cp310-cp310-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:af4fffaffc4067232253715065e30c5a7ec6faac36f8fc8d6f64263b15f74db0"}, + {file = "coverage-6.5.0-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:97117225cdd992a9c2a5515db1f66b59db634f59d0679ca1fa3fe8da32749cae"}, + {file = "coverage-6.5.0-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:a1170fa54185845505fbfa672f1c1ab175446c887cce8212c44149581cf2d466"}, + {file = "coverage-6.5.0-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:11b990d520ea75e7ee8dcab5bc908072aaada194a794db9f6d7d5cfd19661e5a"}, + {file = "coverage-6.5.0-cp310-cp310-win32.whl", hash = "sha256:5dbec3b9095749390c09ab7c89d314727f18800060d8d24e87f01fb9cfb40b32"}, + {file = "coverage-6.5.0-cp310-cp310-win_amd64.whl", hash = "sha256:59f53f1dc5b656cafb1badd0feb428c1e7bc19b867479ff72f7a9dd9b479f10e"}, + {file = "coverage-6.5.0-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:4a5375e28c5191ac38cca59b38edd33ef4cc914732c916f2929029b4bfb50795"}, + {file = "coverage-6.5.0-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:c4ed2820d919351f4167e52425e096af41bfabacb1857186c1ea32ff9983ed75"}, + {file = "coverage-6.5.0-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:33a7da4376d5977fbf0a8ed91c4dffaaa8dbf0ddbf4c8eea500a2486d8bc4d7b"}, + {file = "coverage-6.5.0-cp311-cp311-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:a8fb6cf131ac4070c9c5a3e21de0f7dc5a0fbe8bc77c9456ced896c12fcdad91"}, + {file = "coverage-6.5.0-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:a6b7d95969b8845250586f269e81e5dfdd8ff828ddeb8567a4a2eaa7313460c4"}, + {file = "coverage-6.5.0-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:1ef221513e6f68b69ee9e159506d583d31aa3567e0ae84eaad9d6ec1107dddaa"}, + {file = "coverage-6.5.0-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:cca4435eebea7962a52bdb216dec27215d0df64cf27fc1dd538415f5d2b9da6b"}, + {file = "coverage-6.5.0-cp311-cp311-win32.whl", hash = "sha256:98e8a10b7a314f454d9eff4216a9a94d143a7ee65018dd12442e898ee2310578"}, + {file = "coverage-6.5.0-cp311-cp311-win_amd64.whl", hash = "sha256:bc8ef5e043a2af066fa8cbfc6e708d58017024dc4345a1f9757b329a249f041b"}, + {file = "coverage-6.5.0-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:4433b90fae13f86fafff0b326453dd42fc9a639a0d9e4eec4d366436d1a41b6d"}, + {file = "coverage-6.5.0-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:f4f05d88d9a80ad3cac6244d36dd89a3c00abc16371769f1340101d3cb899fc3"}, + {file = "coverage-6.5.0-cp37-cp37m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:94e2565443291bd778421856bc975d351738963071e9b8839ca1fc08b42d4bef"}, + {file = "coverage-6.5.0-cp37-cp37m-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:027018943386e7b942fa832372ebc120155fd970837489896099f5cfa2890f79"}, + {file = "coverage-6.5.0-cp37-cp37m-musllinux_1_1_aarch64.whl", hash = "sha256:255758a1e3b61db372ec2736c8e2a1fdfaf563977eedbdf131de003ca5779b7d"}, + {file = "coverage-6.5.0-cp37-cp37m-musllinux_1_1_i686.whl", hash = "sha256:851cf4ff24062c6aec510a454b2584f6e998cada52d4cb58c5e233d07172e50c"}, + {file = "coverage-6.5.0-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:12adf310e4aafddc58afdb04d686795f33f4d7a6fa67a7a9d4ce7d6ae24d949f"}, + {file = "coverage-6.5.0-cp37-cp37m-win32.whl", hash = "sha256:b5604380f3415ba69de87a289a2b56687faa4fe04dbee0754bfcae433489316b"}, + {file = "coverage-6.5.0-cp37-cp37m-win_amd64.whl", hash = "sha256:4a8dbc1f0fbb2ae3de73eb0bdbb914180c7abfbf258e90b311dcd4f585d44bd2"}, + {file = "coverage-6.5.0-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:d900bb429fdfd7f511f868cedd03a6bbb142f3f9118c09b99ef8dc9bf9643c3c"}, + {file = "coverage-6.5.0-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:2198ea6fc548de52adc826f62cb18554caedfb1d26548c1b7c88d8f7faa8f6ba"}, + {file = "coverage-6.5.0-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:6c4459b3de97b75e3bd6b7d4b7f0db13f17f504f3d13e2a7c623786289dd670e"}, + {file = "coverage-6.5.0-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:20c8ac5386253717e5ccc827caad43ed66fea0efe255727b1053a8154d952398"}, + {file = "coverage-6.5.0-cp38-cp38-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:6b07130585d54fe8dff3d97b93b0e20290de974dc8177c320aeaf23459219c0b"}, + {file = "coverage-6.5.0-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:dbdb91cd8c048c2b09eb17713b0c12a54fbd587d79adcebad543bc0cd9a3410b"}, + {file = "coverage-6.5.0-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:de3001a203182842a4630e7b8d1a2c7c07ec1b45d3084a83d5d227a3806f530f"}, + {file = "coverage-6.5.0-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:e07f4a4a9b41583d6eabec04f8b68076ab3cd44c20bd29332c6572dda36f372e"}, + {file = "coverage-6.5.0-cp38-cp38-win32.whl", hash = "sha256:6d4817234349a80dbf03640cec6109cd90cba068330703fa65ddf56b60223a6d"}, + {file = "coverage-6.5.0-cp38-cp38-win_amd64.whl", hash = "sha256:7ccf362abd726b0410bf8911c31fbf97f09f8f1061f8c1cf03dfc4b6372848f6"}, + {file = "coverage-6.5.0-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:633713d70ad6bfc49b34ead4060531658dc6dfc9b3eb7d8a716d5873377ab745"}, + {file = "coverage-6.5.0-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:95203854f974e07af96358c0b261f1048d8e1083f2de9b1c565e1be4a3a48cfc"}, + {file = "coverage-6.5.0-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:b9023e237f4c02ff739581ef35969c3739445fb059b060ca51771e69101efffe"}, + {file = "coverage-6.5.0-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:265de0fa6778d07de30bcf4d9dc471c3dc4314a23a3c6603d356a3c9abc2dfcf"}, + {file = "coverage-6.5.0-cp39-cp39-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:8f830ed581b45b82451a40faabb89c84e1a998124ee4212d440e9c6cf70083e5"}, + {file = "coverage-6.5.0-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:7b6be138d61e458e18d8e6ddcddd36dd96215edfe5f1168de0b1b32635839b62"}, + {file = "coverage-6.5.0-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:42eafe6778551cf006a7c43153af1211c3aaab658d4d66fa5fcc021613d02518"}, + {file = "coverage-6.5.0-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:723e8130d4ecc8f56e9a611e73b31219595baa3bb252d539206f7bbbab6ffc1f"}, + {file = "coverage-6.5.0-cp39-cp39-win32.whl", hash = "sha256:d9ecf0829c6a62b9b573c7bb6d4dcd6ba8b6f80be9ba4fc7ed50bf4ac9aecd72"}, + {file = "coverage-6.5.0-cp39-cp39-win_amd64.whl", hash = "sha256:fc2af30ed0d5ae0b1abdb4ebdce598eafd5b35397d4d75deb341a614d333d987"}, + {file = "coverage-6.5.0-pp36.pp37.pp38-none-any.whl", hash = "sha256:1431986dac3923c5945271f169f59c45b8802a114c8f548d611f2015133df77a"}, + {file = "coverage-6.5.0.tar.gz", hash = "sha256:f642e90754ee3e06b0e7e51bce3379590e76b7f76b708e1a71ff043f87025c84"}, +] distlib = [ {file = "distlib-0.3.6-py2.py3-none-any.whl", hash = "sha256:f35c4b692542ca110de7ef0bea44d73981caeb34ca0b9b6b2e6d7790dda8f80e"}, {file = "distlib-0.3.6.tar.gz", hash = "sha256:14bad2d9b04d3a36127ac97f30b12a19268f211063d8f8ee4f47108896e11b46"}, @@ -858,8 +1018,71 @@ fastavro = [ {file = "fastavro-1.6.1-cp39-cp39-win_amd64.whl", hash = "sha256:18f0ddf3cc4f09839c08eec67ed3be3d391894874cb87f856a0362540d18df17"}, {file = "fastavro-1.6.1.tar.gz", hash = "sha256:bc37a6edbde7a04a9df28ab838b94df7527e5eea648d61633233abba59205146"}, ] -filelock = [] -frozenlist = [] +filelock = [ + {file = "filelock-3.8.0-py3-none-any.whl", hash = "sha256:617eb4e5eedc82fc5f47b6d61e4d11cb837c56cb4544e39081099fa17ad109d4"}, + {file = "filelock-3.8.0.tar.gz", hash = "sha256:55447caa666f2198c5b6b13a26d2084d26fa5b115c00d065664b2124680c4edc"}, +] +frozenlist = [ + {file = "frozenlist-1.3.1-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:5f271c93f001748fc26ddea409241312a75e13466b06c94798d1a341cf0e6989"}, + {file = "frozenlist-1.3.1-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:9c6ef8014b842f01f5d2b55315f1af5cbfde284eb184075c189fd657c2fd8204"}, + {file = "frozenlist-1.3.1-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:219a9676e2eae91cb5cc695a78b4cb43d8123e4160441d2b6ce8d2c70c60e2f3"}, + {file = "frozenlist-1.3.1-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:b47d64cdd973aede3dd71a9364742c542587db214e63b7529fbb487ed67cddd9"}, + {file = "frozenlist-1.3.1-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:2af6f7a4e93f5d08ee3f9152bce41a6015b5cf87546cb63872cc19b45476e98a"}, + {file = "frozenlist-1.3.1-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:a718b427ff781c4f4e975525edb092ee2cdef6a9e7bc49e15063b088961806f8"}, + {file = "frozenlist-1.3.1-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:c56c299602c70bc1bb5d1e75f7d8c007ca40c9d7aebaf6e4ba52925d88ef826d"}, + {file = "frozenlist-1.3.1-cp310-cp310-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:717470bfafbb9d9be624da7780c4296aa7935294bd43a075139c3d55659038ca"}, + {file = "frozenlist-1.3.1-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:31b44f1feb3630146cffe56344704b730c33e042ffc78d21f2125a6a91168131"}, + {file = "frozenlist-1.3.1-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:c3b31180b82c519b8926e629bf9f19952c743e089c41380ddca5db556817b221"}, + {file = "frozenlist-1.3.1-cp310-cp310-musllinux_1_1_ppc64le.whl", hash = "sha256:d82bed73544e91fb081ab93e3725e45dd8515c675c0e9926b4e1f420a93a6ab9"}, + {file = "frozenlist-1.3.1-cp310-cp310-musllinux_1_1_s390x.whl", hash = "sha256:49459f193324fbd6413e8e03bd65789e5198a9fa3095e03f3620dee2f2dabff2"}, + {file = "frozenlist-1.3.1-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:94e680aeedc7fd3b892b6fa8395b7b7cc4b344046c065ed4e7a1e390084e8cb5"}, + {file = "frozenlist-1.3.1-cp310-cp310-win32.whl", hash = "sha256:fabb953ab913dadc1ff9dcc3a7a7d3dc6a92efab3a0373989b8063347f8705be"}, + {file = "frozenlist-1.3.1-cp310-cp310-win_amd64.whl", hash = "sha256:eee0c5ecb58296580fc495ac99b003f64f82a74f9576a244d04978a7e97166db"}, + {file = "frozenlist-1.3.1-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:0bc75692fb3770cf2b5856a6c2c9de967ca744863c5e89595df64e252e4b3944"}, + {file = "frozenlist-1.3.1-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:086ca1ac0a40e722d6833d4ce74f5bf1aba2c77cbfdc0cd83722ffea6da52a04"}, + {file = "frozenlist-1.3.1-cp37-cp37m-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:1b51eb355e7f813bcda00276b0114c4172872dc5fb30e3fea059b9367c18fbcb"}, + {file = "frozenlist-1.3.1-cp37-cp37m-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:74140933d45271c1a1283f708c35187f94e1256079b3c43f0c2267f9db5845ff"}, + {file = "frozenlist-1.3.1-cp37-cp37m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:ee4c5120ddf7d4dd1eaf079af3af7102b56d919fa13ad55600a4e0ebe532779b"}, + {file = "frozenlist-1.3.1-cp37-cp37m-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:97d9e00f3ac7c18e685320601f91468ec06c58acc185d18bb8e511f196c8d4b2"}, + {file = "frozenlist-1.3.1-cp37-cp37m-musllinux_1_1_aarch64.whl", hash = "sha256:6e19add867cebfb249b4e7beac382d33215d6d54476bb6be46b01f8cafb4878b"}, + {file = "frozenlist-1.3.1-cp37-cp37m-musllinux_1_1_i686.whl", hash = "sha256:a027f8f723d07c3f21963caa7d585dcc9b089335565dabe9c814b5f70c52705a"}, + {file = "frozenlist-1.3.1-cp37-cp37m-musllinux_1_1_ppc64le.whl", hash = "sha256:61d7857950a3139bce035ad0b0945f839532987dfb4c06cfe160254f4d19df03"}, + {file = "frozenlist-1.3.1-cp37-cp37m-musllinux_1_1_s390x.whl", hash = "sha256:53b2b45052e7149ee8b96067793db8ecc1ae1111f2f96fe1f88ea5ad5fd92d10"}, + {file = "frozenlist-1.3.1-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:bbb1a71b1784e68870800b1bc9f3313918edc63dbb8f29fbd2e767ce5821696c"}, + {file = "frozenlist-1.3.1-cp37-cp37m-win32.whl", hash = "sha256:ab6fa8c7871877810e1b4e9392c187a60611fbf0226a9e0b11b7b92f5ac72792"}, + {file = "frozenlist-1.3.1-cp37-cp37m-win_amd64.whl", hash = "sha256:f89139662cc4e65a4813f4babb9ca9544e42bddb823d2ec434e18dad582543bc"}, + {file = "frozenlist-1.3.1-cp38-cp38-macosx_10_9_universal2.whl", hash = "sha256:4c0c99e31491a1d92cde8648f2e7ccad0e9abb181f6ac3ddb9fc48b63301808e"}, + {file = "frozenlist-1.3.1-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:61e8cb51fba9f1f33887e22488bad1e28dd8325b72425f04517a4d285a04c519"}, + {file = "frozenlist-1.3.1-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:cc2f3e368ee5242a2cbe28323a866656006382872c40869b49b265add546703f"}, + {file = "frozenlist-1.3.1-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:58fb94a01414cddcdc6839807db77ae8057d02ddafc94a42faee6004e46c9ba8"}, + {file = "frozenlist-1.3.1-cp38-cp38-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:022178b277cb9277d7d3b3f2762d294f15e85cd2534047e68a118c2bb0058f3e"}, + {file = "frozenlist-1.3.1-cp38-cp38-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:572ce381e9fe027ad5e055f143763637dcbac2542cfe27f1d688846baeef5170"}, + {file = "frozenlist-1.3.1-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:19127f8dcbc157ccb14c30e6f00392f372ddb64a6ffa7106b26ff2196477ee9f"}, + {file = "frozenlist-1.3.1-cp38-cp38-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:42719a8bd3792744c9b523674b752091a7962d0d2d117f0b417a3eba97d1164b"}, + {file = "frozenlist-1.3.1-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:2743bb63095ef306041c8f8ea22bd6e4d91adabf41887b1ad7886c4c1eb43d5f"}, + {file = "frozenlist-1.3.1-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:fa47319a10e0a076709644a0efbcaab9e91902c8bd8ef74c6adb19d320f69b83"}, + {file = "frozenlist-1.3.1-cp38-cp38-musllinux_1_1_ppc64le.whl", hash = "sha256:52137f0aea43e1993264a5180c467a08a3e372ca9d378244c2d86133f948b26b"}, + {file = "frozenlist-1.3.1-cp38-cp38-musllinux_1_1_s390x.whl", hash = "sha256:f5abc8b4d0c5b556ed8cd41490b606fe99293175a82b98e652c3f2711b452988"}, + {file = "frozenlist-1.3.1-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:1e1cf7bc8cbbe6ce3881863671bac258b7d6bfc3706c600008925fb799a256e2"}, + {file = "frozenlist-1.3.1-cp38-cp38-win32.whl", hash = "sha256:0dde791b9b97f189874d654c55c24bf7b6782343e14909c84beebd28b7217845"}, + {file = "frozenlist-1.3.1-cp38-cp38-win_amd64.whl", hash = "sha256:9494122bf39da6422b0972c4579e248867b6b1b50c9b05df7e04a3f30b9a413d"}, + {file = "frozenlist-1.3.1-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:31bf9539284f39ff9398deabf5561c2b0da5bb475590b4e13dd8b268d7a3c5c1"}, + {file = "frozenlist-1.3.1-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:e0c8c803f2f8db7217898d11657cb6042b9b0553a997c4a0601f48a691480fab"}, + {file = "frozenlist-1.3.1-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:da5ba7b59d954f1f214d352308d1d86994d713b13edd4b24a556bcc43d2ddbc3"}, + {file = "frozenlist-1.3.1-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:74e6b2b456f21fc93ce1aff2b9728049f1464428ee2c9752a4b4f61e98c4db96"}, + {file = "frozenlist-1.3.1-cp39-cp39-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:526d5f20e954d103b1d47232e3839f3453c02077b74203e43407b962ab131e7b"}, + {file = "frozenlist-1.3.1-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:b499c6abe62a7a8d023e2c4b2834fce78a6115856ae95522f2f974139814538c"}, + {file = "frozenlist-1.3.1-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:ab386503f53bbbc64d1ad4b6865bf001414930841a870fc97f1546d4d133f141"}, + {file = "frozenlist-1.3.1-cp39-cp39-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:5f63c308f82a7954bf8263a6e6de0adc67c48a8b484fab18ff87f349af356efd"}, + {file = "frozenlist-1.3.1-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:12607804084d2244a7bd4685c9d0dca5df17a6a926d4f1967aa7978b1028f89f"}, + {file = "frozenlist-1.3.1-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:da1cdfa96425cbe51f8afa43e392366ed0b36ce398f08b60de6b97e3ed4affef"}, + {file = "frozenlist-1.3.1-cp39-cp39-musllinux_1_1_ppc64le.whl", hash = "sha256:f810e764617b0748b49a731ffaa525d9bb36ff38332411704c2400125af859a6"}, + {file = "frozenlist-1.3.1-cp39-cp39-musllinux_1_1_s390x.whl", hash = "sha256:35c3d79b81908579beb1fb4e7fcd802b7b4921f1b66055af2578ff7734711cfa"}, + {file = "frozenlist-1.3.1-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:c92deb5d9acce226a501b77307b3b60b264ca21862bd7d3e0c1f3594022f01bc"}, + {file = "frozenlist-1.3.1-cp39-cp39-win32.whl", hash = "sha256:5e77a8bd41e54b05e4fb2708dc6ce28ee70325f8c6f50f3df86a44ecb1d7a19b"}, + {file = "frozenlist-1.3.1-cp39-cp39-win_amd64.whl", hash = "sha256:625d8472c67f2d96f9a4302a947f92a7adbc1e20bedb6aff8dbc8ff039ca6189"}, + {file = "frozenlist-1.3.1.tar.gz", hash = "sha256:3a735e4211a04ccfa3f4833547acdf5d2f863bfeb01cfd3edaffbc251f15cec8"}, +] fsspec = [ {file = "fsspec-2022.8.2-py3-none-any.whl", hash = "sha256:6374804a2c0d24f225a67d009ee1eabb4046ad00c793c3f6df97e426c890a1d9"}, {file = "fsspec-2022.8.2.tar.gz", hash = "sha256:7f12b90964a98a7e921d27fb36be536ea036b73bf3b724ac0b0bd7b8e39c7c18"}, @@ -880,7 +1103,10 @@ iniconfig = [ {file = "iniconfig-1.1.1-py2.py3-none-any.whl", hash = "sha256:011e24c64b7f47f6ebd835bb12a743f2fbe9a26d4cecaa7f53bc4f35ee9da8b3"}, {file = "iniconfig-1.1.1.tar.gz", hash = "sha256:bc3af051d7d14b2ee5ef9969666def0cd1a000e121eaea580d4a313df4b37f32"}, ] -jmespath = [] +jmespath = [ + {file = "jmespath-1.0.1-py3-none-any.whl", hash = "sha256:02e2e4cc71b5bcab88332eebf907519190dd9e6e82107fa7f83b1003a6252980"}, + {file = "jmespath-1.0.1.tar.gz", hash = "sha256:90261b206d6defd58fdd5e85f478bf633a2901798906be2ad389150c5c60edbe"}, +] mmh3 = [ {file = "mmh3-3.0.0-cp36-cp36m-macosx_10_9_x86_64.whl", hash = "sha256:23912dde2ad4f701926948dd8e79a0e42b000f73962806f153931f52985e1e07"}, {file = "mmh3-3.0.0-cp36-cp36m-manylinux1_x86_64.whl", hash = "sha256:07f1308a410dc406d6a3c282a685728d00a87f3ed684f012671b96d6cc6a41c3"}, @@ -909,7 +1135,67 @@ mmh3 = [ {file = "mmh3-3.0.0-cp39-cp39-win_amd64.whl", hash = "sha256:bd870aedd9189eff1cf4e1687869b56c7e9461ee869789139c3e704009e5c227"}, {file = "mmh3-3.0.0.tar.gz", hash = "sha256:d1ec578c09a07d3518ec9be540b87546397fa3455de73c166fcce51eaa5c41c5"}, ] -multidict = [] +multidict = [ + {file = "multidict-6.0.2-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:0b9e95a740109c6047602f4db4da9949e6c5945cefbad34a1299775ddc9a62e2"}, + {file = "multidict-6.0.2-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:ac0e27844758d7177989ce406acc6a83c16ed4524ebc363c1f748cba184d89d3"}, + {file = "multidict-6.0.2-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:041b81a5f6b38244b34dc18c7b6aba91f9cdaf854d9a39e5ff0b58e2b5773b9c"}, + {file = "multidict-6.0.2-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:5fdda29a3c7e76a064f2477c9aab1ba96fd94e02e386f1e665bca1807fc5386f"}, + {file = "multidict-6.0.2-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:3368bf2398b0e0fcbf46d85795adc4c259299fec50c1416d0f77c0a843a3eed9"}, + {file = "multidict-6.0.2-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:f4f052ee022928d34fe1f4d2bc743f32609fb79ed9c49a1710a5ad6b2198db20"}, + {file = "multidict-6.0.2-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:225383a6603c086e6cef0f2f05564acb4f4d5f019a4e3e983f572b8530f70c88"}, + {file = "multidict-6.0.2-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:50bd442726e288e884f7be9071016c15a8742eb689a593a0cac49ea093eef0a7"}, + {file = "multidict-6.0.2-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:47e6a7e923e9cada7c139531feac59448f1f47727a79076c0b1ee80274cd8eee"}, + {file = "multidict-6.0.2-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:0556a1d4ea2d949efe5fd76a09b4a82e3a4a30700553a6725535098d8d9fb672"}, + {file = "multidict-6.0.2-cp310-cp310-musllinux_1_1_ppc64le.whl", hash = "sha256:626fe10ac87851f4cffecee161fc6f8f9853f0f6f1035b59337a51d29ff3b4f9"}, + {file = "multidict-6.0.2-cp310-cp310-musllinux_1_1_s390x.whl", hash = "sha256:8064b7c6f0af936a741ea1efd18690bacfbae4078c0c385d7c3f611d11f0cf87"}, + {file = "multidict-6.0.2-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:2d36e929d7f6a16d4eb11b250719c39560dd70545356365b494249e2186bc389"}, + {file = "multidict-6.0.2-cp310-cp310-win32.whl", hash = "sha256:fcb91630817aa8b9bc4a74023e4198480587269c272c58b3279875ed7235c293"}, + {file = "multidict-6.0.2-cp310-cp310-win_amd64.whl", hash = "sha256:8cbf0132f3de7cc6c6ce00147cc78e6439ea736cee6bca4f068bcf892b0fd658"}, + {file = "multidict-6.0.2-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:05f6949d6169878a03e607a21e3b862eaf8e356590e8bdae4227eedadacf6e51"}, + {file = "multidict-6.0.2-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:e2c2e459f7050aeb7c1b1276763364884595d47000c1cddb51764c0d8976e608"}, + {file = "multidict-6.0.2-cp37-cp37m-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:d0509e469d48940147e1235d994cd849a8f8195e0bca65f8f5439c56e17872a3"}, + {file = "multidict-6.0.2-cp37-cp37m-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:514fe2b8d750d6cdb4712346a2c5084a80220821a3e91f3f71eec11cf8d28fd4"}, + {file = "multidict-6.0.2-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:19adcfc2a7197cdc3987044e3f415168fc5dc1f720c932eb1ef4f71a2067e08b"}, + {file = "multidict-6.0.2-cp37-cp37m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:b9d153e7f1f9ba0b23ad1568b3b9e17301e23b042c23870f9ee0522dc5cc79e8"}, + {file = "multidict-6.0.2-cp37-cp37m-musllinux_1_1_aarch64.whl", hash = "sha256:aef9cc3d9c7d63d924adac329c33835e0243b5052a6dfcbf7732a921c6e918ba"}, + {file = "multidict-6.0.2-cp37-cp37m-musllinux_1_1_i686.whl", hash = "sha256:4571f1beddff25f3e925eea34268422622963cd8dc395bb8778eb28418248e43"}, + {file = "multidict-6.0.2-cp37-cp37m-musllinux_1_1_ppc64le.whl", hash = "sha256:d48b8ee1d4068561ce8033d2c344cf5232cb29ee1a0206a7b828c79cbc5982b8"}, + {file = "multidict-6.0.2-cp37-cp37m-musllinux_1_1_s390x.whl", hash = "sha256:45183c96ddf61bf96d2684d9fbaf6f3564d86b34cb125761f9a0ef9e36c1d55b"}, + {file = "multidict-6.0.2-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:75bdf08716edde767b09e76829db8c1e5ca9d8bb0a8d4bd94ae1eafe3dac5e15"}, + {file = "multidict-6.0.2-cp37-cp37m-win32.whl", hash = "sha256:a45e1135cb07086833ce969555df39149680e5471c04dfd6a915abd2fc3f6dbc"}, + {file = "multidict-6.0.2-cp37-cp37m-win_amd64.whl", hash = "sha256:6f3cdef8a247d1eafa649085812f8a310e728bdf3900ff6c434eafb2d443b23a"}, + {file = "multidict-6.0.2-cp38-cp38-macosx_10_9_universal2.whl", hash = "sha256:0327292e745a880459ef71be14e709aaea2f783f3537588fb4ed09b6c01bca60"}, + {file = "multidict-6.0.2-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:e875b6086e325bab7e680e4316d667fc0e5e174bb5611eb16b3ea121c8951b86"}, + {file = "multidict-6.0.2-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:feea820722e69451743a3d56ad74948b68bf456984d63c1a92e8347b7b88452d"}, + {file = "multidict-6.0.2-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:9cc57c68cb9139c7cd6fc39f211b02198e69fb90ce4bc4a094cf5fe0d20fd8b0"}, + {file = "multidict-6.0.2-cp38-cp38-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:497988d6b6ec6ed6f87030ec03280b696ca47dbf0648045e4e1d28b80346560d"}, + {file = "multidict-6.0.2-cp38-cp38-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:89171b2c769e03a953d5969b2f272efa931426355b6c0cb508022976a17fd376"}, + {file = "multidict-6.0.2-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:684133b1e1fe91eda8fa7447f137c9490a064c6b7f392aa857bba83a28cfb693"}, + {file = "multidict-6.0.2-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:fd9fc9c4849a07f3635ccffa895d57abce554b467d611a5009ba4f39b78a8849"}, + {file = "multidict-6.0.2-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:e07c8e79d6e6fd37b42f3250dba122053fddb319e84b55dd3a8d6446e1a7ee49"}, + {file = "multidict-6.0.2-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:4070613ea2227da2bfb2c35a6041e4371b0af6b0be57f424fe2318b42a748516"}, + {file = "multidict-6.0.2-cp38-cp38-musllinux_1_1_ppc64le.whl", hash = "sha256:47fbeedbf94bed6547d3aa632075d804867a352d86688c04e606971595460227"}, + {file = "multidict-6.0.2-cp38-cp38-musllinux_1_1_s390x.whl", hash = "sha256:5774d9218d77befa7b70d836004a768fb9aa4fdb53c97498f4d8d3f67bb9cfa9"}, + {file = "multidict-6.0.2-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:2957489cba47c2539a8eb7ab32ff49101439ccf78eab724c828c1a54ff3ff98d"}, + {file = "multidict-6.0.2-cp38-cp38-win32.whl", hash = "sha256:e5b20e9599ba74391ca0cfbd7b328fcc20976823ba19bc573983a25b32e92b57"}, + {file = "multidict-6.0.2-cp38-cp38-win_amd64.whl", hash = "sha256:8004dca28e15b86d1b1372515f32eb6f814bdf6f00952699bdeb541691091f96"}, + {file = "multidict-6.0.2-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:2e4a0785b84fb59e43c18a015ffc575ba93f7d1dbd272b4cdad9f5134b8a006c"}, + {file = "multidict-6.0.2-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:6701bf8a5d03a43375909ac91b6980aea74b0f5402fbe9428fc3f6edf5d9677e"}, + {file = "multidict-6.0.2-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:a007b1638e148c3cfb6bf0bdc4f82776cef0ac487191d093cdc316905e504071"}, + {file = "multidict-6.0.2-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:07a017cfa00c9890011628eab2503bee5872f27144936a52eaab449be5eaf032"}, + {file = "multidict-6.0.2-cp39-cp39-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:c207fff63adcdf5a485969131dc70e4b194327666b7e8a87a97fbc4fd80a53b2"}, + {file = "multidict-6.0.2-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:373ba9d1d061c76462d74e7de1c0c8e267e9791ee8cfefcf6b0b2495762c370c"}, + {file = "multidict-6.0.2-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:bfba7c6d5d7c9099ba21f84662b037a0ffd4a5e6b26ac07d19e423e6fdf965a9"}, + {file = "multidict-6.0.2-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:19d9bad105dfb34eb539c97b132057a4e709919ec4dd883ece5838bcbf262b80"}, + {file = "multidict-6.0.2-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:de989b195c3d636ba000ee4281cd03bb1234635b124bf4cd89eeee9ca8fcb09d"}, + {file = "multidict-6.0.2-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:7c40b7bbece294ae3a87c1bc2abff0ff9beef41d14188cda94ada7bcea99b0fb"}, + {file = "multidict-6.0.2-cp39-cp39-musllinux_1_1_ppc64le.whl", hash = "sha256:d16cce709ebfadc91278a1c005e3c17dd5f71f5098bfae1035149785ea6e9c68"}, + {file = "multidict-6.0.2-cp39-cp39-musllinux_1_1_s390x.whl", hash = "sha256:a2c34a93e1d2aa35fbf1485e5010337c72c6791407d03aa5f4eed920343dd360"}, + {file = "multidict-6.0.2-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:feba80698173761cddd814fa22e88b0661e98cb810f9f986c54aa34d281e4937"}, + {file = "multidict-6.0.2-cp39-cp39-win32.whl", hash = "sha256:23b616fdc3c74c9fe01d76ce0d1ce872d2d396d8fa8e4899398ad64fb5aa214a"}, + {file = "multidict-6.0.2-cp39-cp39-win_amd64.whl", hash = "sha256:4bae31803d708f6f15fd98be6a6ac0b6958fcf68fda3c77a048a4f9073704aae"}, + {file = "multidict-6.0.2.tar.gz", hash = "sha256:5ff3bd75f38e4c43f1f470f2df7a4d430b821c4ce22be384e1459cb57d6bb013"}, +] nodeenv = [ {file = "nodeenv-1.7.0-py2.py3-none-any.whl", hash = "sha256:27083a7b96a25f2f5e1d8cb4b6317ee8aeda3bdd121394e5ac54e498028a042e"}, {file = "nodeenv-1.7.0.tar.gz", hash = "sha256:e0e7f7dfb85fc5394c6fe1e8fa98131a2473e04311a45afb6508f7cf1836fa2b"}, @@ -948,7 +1234,10 @@ packaging = [ {file = "packaging-21.3-py3-none-any.whl", hash = "sha256:ef103e05f519cdc783ae24ea4e2e0f508a9c99b2d4969652eed6a2e1ea5bd522"}, {file = "packaging-21.3.tar.gz", hash = "sha256:dd47c42927d89ab911e606518907cc2d3a1f38bbd026385970643f9c5b8ecfeb"}, ] -pep517 = [] +pep517 = [ + {file = "pep517-0.13.0-py3-none-any.whl", hash = "sha256:4ba4446d80aed5b5eac6509ade100bff3e7943a8489de249654a5ae9b33ee35b"}, + {file = "pep517-0.13.0.tar.gz", hash = "sha256:ae69927c5c172be1add9203726d4b84cf3ebad1edcd5f71fcdc746e66e829f59"}, +] platformdirs = [ {file = "platformdirs-2.5.2-py3-none-any.whl", hash = "sha256:027d8e83a2d7de06bbac4e5ef7e023c02b863d7ea5d079477e722bb41ab25788"}, {file = "platformdirs-2.5.2.tar.gz", hash = "sha256:58c8abb07dcb441e6ee4b11d8df0ac856038f944ab98b7be6b27b2a3c7feef19"}, @@ -957,12 +1246,42 @@ pluggy = [ {file = "pluggy-1.0.0-py2.py3-none-any.whl", hash = "sha256:74134bbf457f031a36d68416e1509f34bd5ccc019f0bcc952c7b909d06b37bd3"}, {file = "pluggy-1.0.0.tar.gz", hash = "sha256:4224373bacce55f955a878bf9cfa763c1e360858e330072059e10bad68531159"}, ] -pre-commit = [] +pre-commit = [ + {file = "pre_commit-2.20.0-py2.py3-none-any.whl", hash = "sha256:51a5ba7c480ae8072ecdb6933df22d2f812dc897d5fe848778116129a681aac7"}, + {file = "pre_commit-2.20.0.tar.gz", hash = "sha256:a978dac7bc9ec0bcee55c18a277d553b0f419d259dadb4b9418ff2d00eb43959"}, +] py = [ {file = "py-1.11.0-py2.py3-none-any.whl", hash = "sha256:607c53218732647dff4acdfcd50cb62615cedf612e72d1724fb1a0cc6405b378"}, {file = "py-1.11.0.tar.gz", hash = "sha256:51c75c4126074b472f746a24399ad32f6053d1b34b68d2fa41e558e6f4a98719"}, ] -pyarrow = [] +pyarrow = [ + {file = "pyarrow-9.0.0-cp310-cp310-macosx_10_13_universal2.whl", hash = "sha256:767cafb14278165ad539a2918c14c1b73cf20689747c21375c38e3fe62884902"}, + {file = "pyarrow-9.0.0-cp310-cp310-macosx_10_13_x86_64.whl", hash = "sha256:0238998dc692efcb4e41ae74738d7c1234723271ccf520bd8312dca07d49ef8d"}, + {file = "pyarrow-9.0.0-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:55328348b9139c2b47450d512d716c2248fd58e2f04e2fc23a65e18726666d42"}, + {file = "pyarrow-9.0.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:fc856628acd8d281652c15b6268ec7f27ebcb015abbe99d9baad17f02adc51f1"}, + {file = "pyarrow-9.0.0-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:29eb3e086e2b26202f3a4678316b93cfb15d0e2ba20f3ec12db8fd9cc07cde63"}, + {file = "pyarrow-9.0.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:2e753f8fcf07d8e3a0efa0c8bd51fef5c90281ffd4c5637c08ce42cd0ac297de"}, + {file = "pyarrow-9.0.0-cp310-cp310-win_amd64.whl", hash = "sha256:3eef8a981f45d89de403e81fb83b8119c20824caddf1404274e41a5d66c73806"}, + {file = "pyarrow-9.0.0-cp37-cp37m-macosx_10_13_x86_64.whl", hash = "sha256:7fa56cbd415cef912677270b8e41baad70cde04c6d8a8336eeb2aba85aa93706"}, + {file = "pyarrow-9.0.0-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:f8c46bde1030d704e2796182286d1c56846552c50a39ad5bf5a20c0d8159fc35"}, + {file = "pyarrow-9.0.0-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:8ad430cee28ebc4d6661fc7315747c7a18ae2a74e67498dcb039e1c762a2fb67"}, + {file = "pyarrow-9.0.0-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:81a60bb291a964f63b2717fb1b28f6615ffab7e8585322bfb8a6738e6b321282"}, + {file = "pyarrow-9.0.0-cp37-cp37m-win_amd64.whl", hash = "sha256:9cef618159567d5f62040f2b79b1c7b38e3885f4ffad0ec97cd2d86f88b67cef"}, + {file = "pyarrow-9.0.0-cp38-cp38-macosx_10_13_x86_64.whl", hash = "sha256:5526a3bfb404ff6d31d62ea582cf2466c7378a474a99ee04d1a9b05de5264541"}, + {file = "pyarrow-9.0.0-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:da3e0f319509a5881867effd7024099fb06950a0768dad0d6873668bb88cfaba"}, + {file = "pyarrow-9.0.0-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:2c715eca2092273dcccf6f08437371e04d112f9354245ba2fbe6c801879450b7"}, + {file = "pyarrow-9.0.0-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:f11a645a41ee531c3a5edda45dea07c42267f52571f818d388971d33fc7e2d4a"}, + {file = "pyarrow-9.0.0-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:a5b390bdcfb8c5b900ef543f911cdfec63e88524fafbcc15f83767202a4a2491"}, + {file = "pyarrow-9.0.0-cp38-cp38-win_amd64.whl", hash = "sha256:d9eb04db626fa24fdfb83c00f76679ca0d98728cdbaa0481b6402bf793a290c0"}, + {file = "pyarrow-9.0.0-cp39-cp39-macosx_10_13_universal2.whl", hash = "sha256:4eebdab05afa23d5d5274b24c1cbeb1ba017d67c280f7d39fd8a8f18cbad2ec9"}, + {file = "pyarrow-9.0.0-cp39-cp39-macosx_10_13_x86_64.whl", hash = "sha256:02b820ecd1da02012092c180447de449fc688d0c3f9ff8526ca301cdd60dacd0"}, + {file = "pyarrow-9.0.0-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:92f3977e901db1ef5cba30d6cc1d7942b8d94b910c60f89013e8f7bb86a86eef"}, + {file = "pyarrow-9.0.0-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:f241bd488c2705df930eedfe304ada71191dcf67d6b98ceda0cc934fd2a8388e"}, + {file = "pyarrow-9.0.0-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:1c5a073a930c632058461547e0bc572da1e724b17b6b9eb31a97da13f50cb6e0"}, + {file = "pyarrow-9.0.0-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:f59bcd5217a3ae1e17870792f82b2ff92df9f3862996e2c78e156c13e56ff62e"}, + {file = "pyarrow-9.0.0-cp39-cp39-win_amd64.whl", hash = "sha256:fe2ce795fa1d95e4e940fe5661c3c58aee7181c730f65ac5dd8794a77228de59"}, + {file = "pyarrow-9.0.0.tar.gz", hash = "sha256:7fb02bebc13ab55573d1ae9bb5002a6d20ba767bf8569b52fce5301d42495ab7"}, +] pycparser = [ {file = "pycparser-2.21-py2.py3-none-any.whl", hash = "sha256:8ee45429555515e1f6b185e78100aea234072576aa43ab53aefcae078162fca9"}, {file = "pycparser-2.21.tar.gz", hash = "sha256:e644fdec12f7872f86c58ff790da456218b10f863970249516d60a5eaca77206"}, @@ -1005,7 +1324,10 @@ pydantic = [ {file = "pydantic-1.10.2-py3-none-any.whl", hash = "sha256:1b6ee725bd6e83ec78b1aa32c5b1fa67a3a65badddde3976bca5fe4568f27709"}, {file = "pydantic-1.10.2.tar.gz", hash = "sha256:91b8e218852ef6007c2b98cd861601c6a09f1aa32bbbb74fab5b1c33d4a1e410"}, ] -pygments = [] +pygments = [ + {file = "Pygments-2.13.0-py3-none-any.whl", hash = "sha256:f643f331ab57ba3c9d89212ee4a2dabc6e94f117cf4eefde99a0574720d14c42"}, + {file = "Pygments-2.13.0.tar.gz", hash = "sha256:56a8508ae95f98e2b9bdf93a6be5ae3f7d8af858b43e02c5a2ff083726be40c1"}, +] pyparsing = [ {file = "pyparsing-3.0.9-py3-none-any.whl", hash = "sha256:5026bae9a10eeaefb61dab2f09052b9f4307d44aee4eda64b309723d8d206bbc"}, {file = "pyparsing-3.0.9.tar.gz", hash = "sha256:2b020ecf7d21b687f219b71ecad3631f644a47f01403fa1d1036b0c6416d70fb"}, @@ -1018,7 +1340,10 @@ pytest-checkdocs = [ {file = "pytest-checkdocs-2.7.1.tar.gz", hash = "sha256:2b33b85eddfe5846a69bea4a759303e2d5a3be11d03bc7149f5ba1ef47e6c1ae"}, {file = "pytest_checkdocs-2.7.1-py3-none-any.whl", hash = "sha256:294898c64c9ce1a178edc6660e48da23c7543bfd5a1cea7f0ca4c167745d8461"}, ] -python-dateutil = [] +python-dateutil = [ + {file = "python-dateutil-2.8.2.tar.gz", hash = "sha256:0123cacc1627ae19ddf3c27a5de5bd67ee4586fbdd6440d9748f8abb483d3e86"}, + {file = "python_dateutil-2.8.2-py2.py3-none-any.whl", hash = "sha256:961d03dc3453ebbc59dbdea9e4e11c5651520a876d0f4db161e8674aae935da9"}, +] python-snappy = [ {file = "python-snappy-0.6.1.tar.gz", hash = "sha256:b6a107ab06206acc5359d4c5632bd9b22d448702a79b3169b0c62e0fb808bb2a"}, {file = "python_snappy-0.6.1-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:b7f920eaf46ebf41bd26f9df51c160d40f9e00b7b48471c3438cb8d027f7fb9b"}, @@ -1077,6 +1402,13 @@ pyyaml = [ {file = "PyYAML-6.0-cp310-cp310-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:f84fbc98b019fef2ee9a1cb3ce93e3187a6df0b2538a651bfb890254ba9f90b5"}, {file = "PyYAML-6.0-cp310-cp310-win32.whl", hash = "sha256:2cd5df3de48857ed0544b34e2d40e9fac445930039f3cfe4bcc592a1f836d513"}, {file = "PyYAML-6.0-cp310-cp310-win_amd64.whl", hash = "sha256:daf496c58a8c52083df09b80c860005194014c3698698d1a57cbcfa182142a3a"}, + {file = "PyYAML-6.0-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:d4b0ba9512519522b118090257be113b9468d804b19d63c71dbcf4a48fa32358"}, + {file = "PyYAML-6.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:81957921f441d50af23654aa6c5e5eaf9b06aba7f0a19c18a538dc7ef291c5a1"}, + {file = "PyYAML-6.0-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:afa17f5bc4d1b10afd4466fd3a44dc0e245382deca5b3c353d8b757f9e3ecb8d"}, + {file = "PyYAML-6.0-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:dbad0e9d368bb989f4515da330b88a057617d16b6a8245084f1b05400f24609f"}, + {file = "PyYAML-6.0-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:432557aa2c09802be39460360ddffd48156e30721f5e8d917f01d31694216782"}, + {file = "PyYAML-6.0-cp311-cp311-win32.whl", hash = "sha256:bfaef573a63ba8923503d27530362590ff4f576c626d86a9fed95822a8255fd7"}, + {file = "PyYAML-6.0-cp311-cp311-win_amd64.whl", hash = "sha256:01b45c0191e6d66c470b6cf1b9531a771a83c1c4208272ead47a3ae4f2f603bf"}, {file = "PyYAML-6.0-cp36-cp36m-macosx_10_9_x86_64.whl", hash = "sha256:897b80890765f037df3403d22bab41627ca8811ae55e9a722fd0392850ec4d86"}, {file = "PyYAML-6.0-cp36-cp36m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:50602afada6d6cbfad699b0c7bb50d5ccffa7e46a3d738092afddc1f9758427f"}, {file = "PyYAML-6.0-cp36-cp36m-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:48c346915c114f5fdb3ead70312bd042a953a8ce5c7106d5bfb1a5254e47da92"}, @@ -1112,16 +1444,25 @@ requests-mock = [ {file = "requests-mock-1.10.0.tar.gz", hash = "sha256:59c9c32419a9fb1ae83ec242d98e889c45bd7d7a65d48375cc243ec08441658b"}, {file = "requests_mock-1.10.0-py2.py3-none-any.whl", hash = "sha256:2fdbb637ad17ee15c06f33d31169e71bf9fe2bdb7bc9da26185be0dd8d842699"}, ] -rich = [] +rich = [ + {file = "rich-12.5.1-py3-none-any.whl", hash = "sha256:2eb4e6894cde1e017976d2975ac210ef515d7548bc595ba20e195fb9628acdeb"}, + {file = "rich-12.5.1.tar.gz", hash = "sha256:63a5c5ce3673d3d5fbbf23cd87e11ab84b6b451436f1b7f19ec54b6bc36ed7ca"}, +] s3fs = [ {file = "s3fs-2022.8.2-py3-none-any.whl", hash = "sha256:63369eb9b14687080fc5ac72912f51d5c99d58a24d488199c19e664d0f872fd2"}, {file = "s3fs-2022.8.2.tar.gz", hash = "sha256:3ca0701a89a9e125a28de90829d19f96f41ddb1d8b4379076c29ed82c4af86cd"}, ] +setuptools = [ + {file = "setuptools-65.4.1-py3-none-any.whl", hash = "sha256:1b6bdc6161661409c5f21508763dc63ab20a9ac2f8ba20029aaaa7fdb9118012"}, + {file = "setuptools-65.4.1.tar.gz", hash = "sha256:3050e338e5871e70c72983072fe34f6032ae1cdeeeb67338199c2f74e083a80e"}, +] six = [ {file = "six-1.16.0-py2.py3-none-any.whl", hash = "sha256:8abb2f1d86890a2dfb989f9a77cfcfd3e47c2a354b01111771326f8aa26e0254"}, {file = "six-1.16.0.tar.gz", hash = "sha256:1e61c37477a1626458e36f7b1d82aa5c9b094fa4802892072e49de9c60c4c926"}, ] -thrift = [] +thrift = [ + {file = "thrift-0.16.0.tar.gz", hash = "sha256:2b5b6488fcded21f9d312aa23c9ff6a0195d0f6ae26ddbd5ad9e3e25dfc14408"}, +] toml = [ {file = "toml-0.10.2-py2.py3-none-any.whl", hash = "sha256:806143ae5bfb6a3c6e736a764057db0e6a0e05e338b5630894a5f779cabb4f9b"}, {file = "toml-0.10.2.tar.gz", hash = "sha256:b3bda1d108d5dd99f4a20d24d9c348e91c4db7ab1b749200bded2f839ccbe68f"}, @@ -1134,7 +1475,10 @@ typing-extensions = [ {file = "typing_extensions-4.3.0-py3-none-any.whl", hash = "sha256:25642c956049920a5aa49edcdd6ab1e06d7e5d467fc00e0506c44ac86fbfca02"}, {file = "typing_extensions-4.3.0.tar.gz", hash = "sha256:e6d2677a32f47fc7eb2795db1dd15c1f34eff616bcaf2cfb5e997f854fa1c4a6"}, ] -urllib3 = [] +urllib3 = [ + {file = "urllib3-1.26.12-py2.py3-none-any.whl", hash = "sha256:b930dd878d5a8afb066a637fbb35144fe7901e3b209d1cd4f524bd0e9deee997"}, + {file = "urllib3-1.26.12.tar.gz", hash = "sha256:3fa96cf423e6987997fc326ae8df396db2a8b7c667747d47ddd8ecba91f4a74e"}, +] virtualenv = [ {file = "virtualenv-20.16.5-py3-none-any.whl", hash = "sha256:d07dfc5df5e4e0dbc92862350ad87a36ed505b978f6c39609dc489eadd5b0d27"}, {file = "virtualenv-20.16.5.tar.gz", hash = "sha256:227ea1b9994fdc5ea31977ba3383ef296d7472ea85be9d6732e42a91c04e80da"}, @@ -1205,6 +1549,114 @@ wrapt = [ {file = "wrapt-1.14.1-cp39-cp39-win_amd64.whl", hash = "sha256:dee60e1de1898bde3b238f18340eec6148986da0455d8ba7848d50470a7a32fb"}, {file = "wrapt-1.14.1.tar.gz", hash = "sha256:380a85cf89e0e69b7cfbe2ea9f765f004ff419f34194018a6827ac0e3edfed4d"}, ] -yarl = [] -zipp = [] -zstandard = [] +yarl = [ + {file = "yarl-1.8.1-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:abc06b97407868ef38f3d172762f4069323de52f2b70d133d096a48d72215d28"}, + {file = "yarl-1.8.1-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:07b21e274de4c637f3e3b7104694e53260b5fc10d51fb3ec5fed1da8e0f754e3"}, + {file = "yarl-1.8.1-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:9de955d98e02fab288c7718662afb33aab64212ecb368c5dc866d9a57bf48880"}, + {file = "yarl-1.8.1-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:7ec362167e2c9fd178f82f252b6d97669d7245695dc057ee182118042026da40"}, + {file = "yarl-1.8.1-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:20df6ff4089bc86e4a66e3b1380460f864df3dd9dccaf88d6b3385d24405893b"}, + {file = "yarl-1.8.1-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:5999c4662631cb798496535afbd837a102859568adc67d75d2045e31ec3ac497"}, + {file = "yarl-1.8.1-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:ed19b74e81b10b592084a5ad1e70f845f0aacb57577018d31de064e71ffa267a"}, + {file = "yarl-1.8.1-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:1e4808f996ca39a6463f45182e2af2fae55e2560be586d447ce8016f389f626f"}, + {file = "yarl-1.8.1-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:2d800b9c2eaf0684c08be5f50e52bfa2aa920e7163c2ea43f4f431e829b4f0fd"}, + {file = "yarl-1.8.1-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:6628d750041550c5d9da50bb40b5cf28a2e63b9388bac10fedd4f19236ef4957"}, + {file = "yarl-1.8.1-cp310-cp310-musllinux_1_1_ppc64le.whl", hash = "sha256:f5af52738e225fcc526ae64071b7e5342abe03f42e0e8918227b38c9aa711e28"}, + {file = "yarl-1.8.1-cp310-cp310-musllinux_1_1_s390x.whl", hash = "sha256:76577f13333b4fe345c3704811ac7509b31499132ff0181f25ee26619de2c843"}, + {file = "yarl-1.8.1-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:0c03f456522d1ec815893d85fccb5def01ffaa74c1b16ff30f8aaa03eb21e453"}, + {file = "yarl-1.8.1-cp310-cp310-win32.whl", hash = "sha256:ea30a42dc94d42f2ba4d0f7c0ffb4f4f9baa1b23045910c0c32df9c9902cb272"}, + {file = "yarl-1.8.1-cp310-cp310-win_amd64.whl", hash = "sha256:9130ddf1ae9978abe63808b6b60a897e41fccb834408cde79522feb37fb72fb0"}, + {file = "yarl-1.8.1-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:0ab5a138211c1c366404d912824bdcf5545ccba5b3ff52c42c4af4cbdc2c5035"}, + {file = "yarl-1.8.1-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a0fb2cb4204ddb456a8e32381f9a90000429489a25f64e817e6ff94879d432fc"}, + {file = "yarl-1.8.1-cp37-cp37m-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:85cba594433915d5c9a0d14b24cfba0339f57a2fff203a5d4fd070e593307d0b"}, + {file = "yarl-1.8.1-cp37-cp37m-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:1ca7e596c55bd675432b11320b4eacc62310c2145d6801a1f8e9ad160685a231"}, + {file = "yarl-1.8.1-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:d0f77539733e0ec2475ddcd4e26777d08996f8cd55d2aef82ec4d3896687abda"}, + {file = "yarl-1.8.1-cp37-cp37m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:29e256649f42771829974e742061c3501cc50cf16e63f91ed8d1bf98242e5507"}, + {file = "yarl-1.8.1-cp37-cp37m-musllinux_1_1_aarch64.whl", hash = "sha256:7fce6cbc6c170ede0221cc8c91b285f7f3c8b9fe28283b51885ff621bbe0f8ee"}, + {file = "yarl-1.8.1-cp37-cp37m-musllinux_1_1_i686.whl", hash = "sha256:59ddd85a1214862ce7c7c66457f05543b6a275b70a65de366030d56159a979f0"}, + {file = "yarl-1.8.1-cp37-cp37m-musllinux_1_1_ppc64le.whl", hash = "sha256:12768232751689c1a89b0376a96a32bc7633c08da45ad985d0c49ede691f5c0d"}, + {file = "yarl-1.8.1-cp37-cp37m-musllinux_1_1_s390x.whl", hash = "sha256:b19255dde4b4f4c32e012038f2c169bb72e7f081552bea4641cab4d88bc409dd"}, + {file = "yarl-1.8.1-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:6c8148e0b52bf9535c40c48faebb00cb294ee577ca069d21bd5c48d302a83780"}, + {file = "yarl-1.8.1-cp37-cp37m-win32.whl", hash = "sha256:de839c3a1826a909fdbfe05f6fe2167c4ab033f1133757b5936efe2f84904c07"}, + {file = "yarl-1.8.1-cp37-cp37m-win_amd64.whl", hash = "sha256:dd032e8422a52e5a4860e062eb84ac94ea08861d334a4bcaf142a63ce8ad4802"}, + {file = "yarl-1.8.1-cp38-cp38-macosx_10_9_universal2.whl", hash = "sha256:19cd801d6f983918a3f3a39f3a45b553c015c5aac92ccd1fac619bd74beece4a"}, + {file = "yarl-1.8.1-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:6347f1a58e658b97b0a0d1ff7658a03cb79bdbda0331603bed24dd7054a6dea1"}, + {file = "yarl-1.8.1-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:7c0da7e44d0c9108d8b98469338705e07f4bb7dab96dbd8fa4e91b337db42548"}, + {file = "yarl-1.8.1-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:5587bba41399854703212b87071c6d8638fa6e61656385875f8c6dff92b2e461"}, + {file = "yarl-1.8.1-cp38-cp38-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:31a9a04ecccd6b03e2b0e12e82131f1488dea5555a13a4d32f064e22a6003cfe"}, + {file = "yarl-1.8.1-cp38-cp38-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:205904cffd69ae972a1707a1bd3ea7cded594b1d773a0ce66714edf17833cdae"}, + {file = "yarl-1.8.1-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:ea513a25976d21733bff523e0ca836ef1679630ef4ad22d46987d04b372d57fc"}, + {file = "yarl-1.8.1-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:d0b51530877d3ad7a8d47b2fff0c8df3b8f3b8deddf057379ba50b13df2a5eae"}, + {file = "yarl-1.8.1-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:d2b8f245dad9e331540c350285910b20dd913dc86d4ee410c11d48523c4fd546"}, + {file = "yarl-1.8.1-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:ab2a60d57ca88e1d4ca34a10e9fb4ab2ac5ad315543351de3a612bbb0560bead"}, + {file = "yarl-1.8.1-cp38-cp38-musllinux_1_1_ppc64le.whl", hash = "sha256:449c957ffc6bc2309e1fbe67ab7d2c1efca89d3f4912baeb8ead207bb3cc1cd4"}, + {file = "yarl-1.8.1-cp38-cp38-musllinux_1_1_s390x.whl", hash = "sha256:a165442348c211b5dea67c0206fc61366212d7082ba8118c8c5c1c853ea4d82e"}, + {file = "yarl-1.8.1-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:b3ded839a5c5608eec8b6f9ae9a62cb22cd037ea97c627f38ae0841a48f09eae"}, + {file = "yarl-1.8.1-cp38-cp38-win32.whl", hash = "sha256:c1445a0c562ed561d06d8cbc5c8916c6008a31c60bc3655cdd2de1d3bf5174a0"}, + {file = "yarl-1.8.1-cp38-cp38-win_amd64.whl", hash = "sha256:56c11efb0a89700987d05597b08a1efcd78d74c52febe530126785e1b1a285f4"}, + {file = "yarl-1.8.1-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:e80ed5a9939ceb6fda42811542f31c8602be336b1fb977bccb012e83da7e4936"}, + {file = "yarl-1.8.1-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:6afb336e23a793cd3b6476c30f030a0d4c7539cd81649683b5e0c1b0ab0bf350"}, + {file = "yarl-1.8.1-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:4c322cbaa4ed78a8aac89b2174a6df398faf50e5fc12c4c191c40c59d5e28357"}, + {file = "yarl-1.8.1-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:fae37373155f5ef9b403ab48af5136ae9851151f7aacd9926251ab26b953118b"}, + {file = "yarl-1.8.1-cp39-cp39-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:5395da939ffa959974577eff2cbfc24b004a2fb6c346918f39966a5786874e54"}, + {file = "yarl-1.8.1-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:076eede537ab978b605f41db79a56cad2e7efeea2aa6e0fa8f05a26c24a034fb"}, + {file = "yarl-1.8.1-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:3d1a50e461615747dd93c099f297c1994d472b0f4d2db8a64e55b1edf704ec1c"}, + {file = "yarl-1.8.1-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:7de89c8456525650ffa2bb56a3eee6af891e98f498babd43ae307bd42dca98f6"}, + {file = "yarl-1.8.1-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:4a88510731cd8d4befaba5fbd734a7dd914de5ab8132a5b3dde0bbd6c9476c64"}, + {file = "yarl-1.8.1-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:2d93a049d29df172f48bcb09acf9226318e712ce67374f893b460b42cc1380ae"}, + {file = "yarl-1.8.1-cp39-cp39-musllinux_1_1_ppc64le.whl", hash = "sha256:21ac44b763e0eec15746a3d440f5e09ad2ecc8b5f6dcd3ea8cb4773d6d4703e3"}, + {file = "yarl-1.8.1-cp39-cp39-musllinux_1_1_s390x.whl", hash = "sha256:d0272228fabe78ce00a3365ffffd6f643f57a91043e119c289aaba202f4095b0"}, + {file = "yarl-1.8.1-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:99449cd5366fe4608e7226c6cae80873296dfa0cde45d9b498fefa1de315a09e"}, + {file = "yarl-1.8.1-cp39-cp39-win32.whl", hash = "sha256:8b0af1cf36b93cee99a31a545fe91d08223e64390c5ecc5e94c39511832a4bb6"}, + {file = "yarl-1.8.1-cp39-cp39-win_amd64.whl", hash = "sha256:de49d77e968de6626ba7ef4472323f9d2e5a56c1d85b7c0e2a190b2173d3b9be"}, + {file = "yarl-1.8.1.tar.gz", hash = "sha256:af887845b8c2e060eb5605ff72b6f2dd2aab7a761379373fd89d314f4752abbf"}, +] +zipp = [ + {file = "zipp-3.8.1-py3-none-any.whl", hash = "sha256:47c40d7fe183a6f21403a199b3e4192cca5774656965b0a4988ad2f8feb5f009"}, + {file = "zipp-3.8.1.tar.gz", hash = "sha256:05b45f1ee8f807d0cc928485ca40a07cb491cf092ff587c0df9cb1fd154848d2"}, +] +zstandard = [ + {file = "zstandard-0.18.0-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:ef7e8a200e4c8ac9102ed3c90ed2aa379f6b880f63032200909c1be21951f556"}, + {file = "zstandard-0.18.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:2dc466207016564805e56d28375f4f533b525ff50d6776946980dff5465566ac"}, + {file = "zstandard-0.18.0-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:4a2ee1d4f98447f3e5183ecfce5626f983504a4a0c005fbe92e60fa8e5d547ec"}, + {file = "zstandard-0.18.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:d956e2f03c7200d7e61345e0880c292783ec26618d0d921dcad470cb195bbce2"}, + {file = "zstandard-0.18.0-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_12_i686.manylinux2010_i686.whl", hash = "sha256:ce6f59cba9854fd14da5bfe34217a1501143057313966637b7291d1b0267bd1e"}, + {file = "zstandard-0.18.0-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:a7fa67cba473623848b6e88acf8d799b1906178fd883fb3a1da24561c779593b"}, + {file = "zstandard-0.18.0-cp310-cp310-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:cdb44d7284c8c5dd1b66dfb86dda7f4560fa94bfbbc1d2da749ba44831335e32"}, + {file = "zstandard-0.18.0-cp310-cp310-win32.whl", hash = "sha256:63694a376cde0aa8b1971d06ca28e8f8b5f492779cb6ee1cc46bbc3f019a42a5"}, + {file = "zstandard-0.18.0-cp310-cp310-win_amd64.whl", hash = "sha256:702a8324cd90c74d9c8780d02bf55e79da3193c870c9665ad3a11647e3ad1435"}, + {file = "zstandard-0.18.0-cp36-cp36m-macosx_10_9_x86_64.whl", hash = "sha256:46f679bc5dfd938db4fb058218d9dc4db1336ffaf1ea774ff152ecadabd40805"}, + {file = "zstandard-0.18.0-cp36-cp36m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:dc2a4de9f363b3247d472362a65041fe4c0f59e01a2846b15d13046be866a885"}, + {file = "zstandard-0.18.0-cp36-cp36m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:bd3220d7627fd4d26397211cb3b560ec7cc4a94b75cfce89e847e8ce7fabe32d"}, + {file = "zstandard-0.18.0-cp36-cp36m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_12_i686.manylinux2010_i686.whl", hash = "sha256:39e98cf4773234bd9cebf9f9db730e451dfcfe435e220f8921242afda8321887"}, + {file = "zstandard-0.18.0-cp36-cp36m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:5228e596eb1554598c872a337bbe4e5afe41cd1f8b1b15f2e35b50d061e35244"}, + {file = "zstandard-0.18.0-cp36-cp36m-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:d4a8fd45746a6c31e729f35196e80b8f1e9987c59f5ccb8859d7c6a6fbeb9c63"}, + {file = "zstandard-0.18.0-cp36-cp36m-win32.whl", hash = "sha256:4cbb85f29a990c2fdbf7bc63246567061a362ddca886d7fae6f780267c0a9e67"}, + {file = "zstandard-0.18.0-cp36-cp36m-win_amd64.whl", hash = "sha256:bfa6c8549fa18e6497a738b7033c49f94a8e2e30c5fbe2d14d0b5aa8bbc1695d"}, + {file = "zstandard-0.18.0-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:e02043297c1832f2666cd2204f381bef43b10d56929e13c42c10c732c6e3b4ed"}, + {file = "zstandard-0.18.0-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:7231543d38d2b7e02ef7cc78ef7ffd86419437e1114ff08709fe25a160e24bd6"}, + {file = "zstandard-0.18.0-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:c86befac87445927488f5c8f205d11566f64c11519db223e9d282b945fa60dab"}, + {file = "zstandard-0.18.0-cp37-cp37m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_12_i686.manylinux2010_i686.whl", hash = "sha256:999a4e1768f219826ba3fa2064fab1c86dd72fdd47a42536235478c3bb3ca3e2"}, + {file = "zstandard-0.18.0-cp37-cp37m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:9df59cd1cf3c62075ee2a4da767089d19d874ac3ad42b04a71a167e91b384722"}, + {file = "zstandard-0.18.0-cp37-cp37m-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:1be31e9e3f7607ee0cdd60915410a5968b205d3e7aa83b7fcf3dd76dbbdb39e0"}, + {file = "zstandard-0.18.0-cp37-cp37m-win32.whl", hash = "sha256:490d11b705b8ae9dc845431bacc8dd1cef2408aede176620a5cd0cd411027936"}, + {file = "zstandard-0.18.0-cp37-cp37m-win_amd64.whl", hash = "sha256:266aba27fa9cc5e9091d3d325ebab1fa260f64e83e42516d5e73947c70216a5b"}, + {file = "zstandard-0.18.0-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:8b2260c4e07dd0723eadb586de7718b61acca4083a490dda69c5719d79bc715c"}, + {file = "zstandard-0.18.0-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:3af8c2383d02feb6650e9255491ec7d0824f6e6dd2bbe3e521c469c985f31fb1"}, + {file = "zstandard-0.18.0-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:28723a1d2e4df778573b76b321ebe9f3469ac98988104c2af116dd344802c3f8"}, + {file = "zstandard-0.18.0-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:19cac7108ff2c342317fad6dc97604b47a41f403c8f19d0bfc396dfadc3638b8"}, + {file = "zstandard-0.18.0-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_12_i686.manylinux2010_i686.whl", hash = "sha256:76725d1ee83a8915100a310bbad5d9c1fc6397410259c94033b8318d548d9990"}, + {file = "zstandard-0.18.0-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:d716a7694ce1fa60b20bc10f35c4a22be446ef7f514c8dbc8f858b61976de2fb"}, + {file = "zstandard-0.18.0-cp38-cp38-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:49685bf9a55d1ab34bd8423ea22db836ba43a181ac6b045ac4272093d5cb874e"}, + {file = "zstandard-0.18.0-cp38-cp38-win32.whl", hash = "sha256:1af1268a7dc870eb27515fb8db1f3e6c5a555d2b7bcc476fc3bab8886c7265ab"}, + {file = "zstandard-0.18.0-cp38-cp38-win_amd64.whl", hash = "sha256:1dc2d3809e763055a1a6c1a73f2b677320cc9a5aa1a7c6cfb35aee59bddc42d9"}, + {file = "zstandard-0.18.0-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:eea18c1e7442f2aa9aff1bb84550dbb6a1f711faf6e48e7319de8f2b2e923c2a"}, + {file = "zstandard-0.18.0-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:8677ffc6a6096cccbd892e558471c901fd821aba12b7fbc63833c7346f549224"}, + {file = "zstandard-0.18.0-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:083dc08abf03807af9beeb2b6a91c23ad78add2499f828176a3c7b742c44df02"}, + {file = "zstandard-0.18.0-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:c990063664c08169c84474acecc9251ee035871589025cac47c060ff4ec4bc1a"}, + {file = "zstandard-0.18.0-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_12_i686.manylinux2010_i686.whl", hash = "sha256:533db8a6fac6248b2cb2c935e7b92f994efbdeb72e1ffa0b354432e087bb5a3e"}, + {file = "zstandard-0.18.0-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:dbb3cb8a082d62b8a73af42291569d266b05605e017a3d8a06a0e5c30b5f10f0"}, + {file = "zstandard-0.18.0-cp39-cp39-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:d6c85ca5162049ede475b7ec98e87f9390501d44a3d6776ddd504e872464ec25"}, + {file = "zstandard-0.18.0-cp39-cp39-win32.whl", hash = "sha256:75479e7c2b3eebf402c59fbe57d21bc400cefa145ca356ee053b0a08908c5784"}, + {file = "zstandard-0.18.0-cp39-cp39-win_amd64.whl", hash = "sha256:d85bfabad444812133a92fc6fbe463e1d07581dba72f041f07a360e63808b23c"}, + {file = "zstandard-0.18.0.tar.gz", hash = "sha256:0ac0357a0d985b4ff31a854744040d7b5754385d1f98f7145c30e02c6865cb6f"}, +] diff --git a/pyproject.toml b/pyproject.toml index 665ef46a77..556ac6fb70 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -70,7 +70,7 @@ pytest = "^7.1.3" pytest-checkdocs = "^2.0.0" pre-commit = "^2.0.0" fastavro = "^1.6.1" -coverage = { version = "^6.4.4", extras = ["toml"] } +coverage = { version = "^6.5.0", extras = ["toml"] } requests-mock = "^1.9.3" packaging = "^21.3" From ae76d719a4f0421d004dc66e3b14c90de0057c2f Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Sun, 2 Oct 2022 22:00:13 +0200 Subject: [PATCH 224/642] Build: Bump rich from 12.5.1 to 12.6.0 in /python (#5905) Bumps [rich](https://github.com/willmcgugan/rich) from 12.5.1 to 12.6.0. - [Release notes](https://github.com/willmcgugan/rich/releases) - [Changelog](https://github.com/Textualize/rich/blob/master/CHANGELOG.md) - [Commits](https://github.com/willmcgugan/rich/compare/v12.5.1...v12.6.0) --- updated-dependencies: - dependency-name: rich dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- poetry.lock | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/poetry.lock b/poetry.lock index db7c36bd63..000744f25a 100644 --- a/poetry.lock +++ b/poetry.lock @@ -571,7 +571,7 @@ test = ["fixtures", "mock", "purl", "pytest", "requests-futures", "sphinx", "tes [[package]] name = "rich" -version = "12.5.1" +version = "12.6.0" description = "Render rich text, tables, progress bars, syntax highlighting, markdown and more to the terminal" category = "main" optional = false @@ -1445,8 +1445,8 @@ requests-mock = [ {file = "requests_mock-1.10.0-py2.py3-none-any.whl", hash = "sha256:2fdbb637ad17ee15c06f33d31169e71bf9fe2bdb7bc9da26185be0dd8d842699"}, ] rich = [ - {file = "rich-12.5.1-py3-none-any.whl", hash = "sha256:2eb4e6894cde1e017976d2975ac210ef515d7548bc595ba20e195fb9628acdeb"}, - {file = "rich-12.5.1.tar.gz", hash = "sha256:63a5c5ce3673d3d5fbbf23cd87e11ab84b6b451436f1b7f19ec54b6bc36ed7ca"}, + {file = "rich-12.6.0-py3-none-any.whl", hash = "sha256:a4eb26484f2c82589bd9a17c73d32a010b1e29d89f1604cd9bf3a2097b81bb5e"}, + {file = "rich-12.6.0.tar.gz", hash = "sha256:ba3a3775974105c221d31141f2c116f4fd65c5ceb0698657a11e9f295ec93fd0"}, ] s3fs = [ {file = "s3fs-2022.8.2-py3-none-any.whl", hash = "sha256:63369eb9b14687080fc5ac72912f51d5c99d58a24d488199c19e664d0f872fd2"}, From 8573e1a317eddd372ff0b389142db2dec1ae325a Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Sun, 2 Oct 2022 22:44:56 +0200 Subject: [PATCH 225/642] Build: Bump pytest-checkdocs from 2.7.1 to 2.8.1 in /python (#5903) Bumps [pytest-checkdocs](https://github.com/jaraco/pytest-checkdocs) from 2.7.1 to 2.8.1. - [Release notes](https://github.com/jaraco/pytest-checkdocs/releases) - [Changelog](https://github.com/jaraco/pytest-checkdocs/blob/main/CHANGES.rst) - [Commits](https://github.com/jaraco/pytest-checkdocs/compare/v2.7.1...v2.8.1) --- updated-dependencies: - dependency-name: pytest-checkdocs dependency-type: direct:development update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- poetry.lock | 40 ++++++++++++++++++++++++++++++++-------- pyproject.toml | 2 +- 2 files changed, 33 insertions(+), 9 deletions(-) diff --git a/poetry.lock b/poetry.lock index 000744f25a..371dc67212 100644 --- a/poetry.lock +++ b/poetry.lock @@ -96,6 +96,26 @@ urllib3 = ">=1.25.4,<1.27" [package.extras] crt = ["awscrt (==0.14.0)"] +[[package]] +name = "build" +version = "0.8.0" +description = "A simple, correct PEP 517 build frontend" +category = "dev" +optional = false +python-versions = ">=3.6" + +[package.dependencies] +colorama = {version = "*", markers = "os_name == \"nt\""} +packaging = ">=19.0" +pep517 = ">=0.9.1" +tomli = {version = ">=1.0.0", markers = "python_version < \"3.11\""} + +[package.extras] +docs = ["furo (>=2021.08.31)", "sphinx (>=4.0,<5.0)", "sphinx-argparse-cli (>=1.5)", "sphinx-autodoc-typehints (>=1.10)"] +test = ["filelock (>=3)", "pytest (>=6.2.4)", "pytest-cov (>=2.12)", "pytest-mock (>=2)", "pytest-rerunfailures (>=9.1)", "pytest-xdist (>=1.34)", "setuptools (>=42.0.0)", "setuptools (>=56.0.0)", "toml (>=0.10.0)", "wheel (>=0.36.0)"] +typing = ["importlib-metadata (>=4.6.4)", "mypy (==0.950)", "typing-extensions (>=3.7.4.3)"] +virtualenv = ["virtualenv (>=20.0.35)"] + [[package]] name = "certifi" version = "2022.6.15.2" @@ -493,20 +513,20 @@ testing = ["argcomplete", "hypothesis (>=3.56)", "mock", "nose", "pygments (>=2. [[package]] name = "pytest-checkdocs" -version = "2.7.1" +version = "2.8.1" description = "check the README when running tests" category = "dev" optional = false -python-versions = ">=3.6" +python-versions = ">=3.7" [package.dependencies] +build = "*" docutils = ">=0.15" importlib-metadata = ">=4" -pep517 = "*" [package.extras] -docs = ["jaraco.packaging (>=8.2)", "rst.linker (>=1.9)", "sphinx"] -testing = ["pytest (>=4.6)", "pytest-black (>=0.3.7)", "pytest-checkdocs (>=2.4)", "pytest-cov", "pytest-enabler (>=1.0.1)", "pytest-flake8", "pytest-mypy", "types-docutils"] +docs = ["jaraco.packaging (>=9)", "rst.linker (>=1.9)", "sphinx (>=3.5)"] +testing = ["flake8 (<5)", "pytest (>=6)", "pytest-black (>=0.3.7)", "pytest-checkdocs (>=2.4)", "pytest-cov", "pytest-enabler (>=1.3)", "pytest-flake8", "pytest-mypy (>=0.9.1)", "types-docutils"] [[package]] name = "python-dateutil" @@ -748,7 +768,7 @@ snappy = ["python-snappy"] [metadata] lock-version = "1.1" python-versions = "^3.8" -content-hash = "5fc9986c3483dd8965a5f676d7f621bed361f0fdfbe7646875aa77f9c920af88" +content-hash = "3c89241579eabf944551a3ee85b80e3bf1bfd661914c1e217a6c382f872c1cdf" [metadata.files] aiobotocore = [ @@ -849,6 +869,10 @@ botocore = [ {file = "botocore-1.27.59-py3-none-any.whl", hash = "sha256:69d756791fc024bda54f6c53f71ae34e695ee41bbbc1743d9179c4837a4929da"}, {file = "botocore-1.27.59.tar.gz", hash = "sha256:eda4aed6ee719a745d1288eaf1beb12f6f6448ad1fa12f159405db14ba9c92cf"}, ] +build = [ + {file = "build-0.8.0-py3-none-any.whl", hash = "sha256:19b0ed489f92ace6947698c3ca8436cb0556a66e2aa2d34cd70e2a5d27cd0437"}, + {file = "build-0.8.0.tar.gz", hash = "sha256:887a6d471c901b1a6e6574ebaeeebb45e5269a79d095fe9a8f88d6614ed2e5f0"}, +] certifi = [ {file = "certifi-2022.6.15.2-py3-none-any.whl", hash = "sha256:0aa1a42fbd57645fabeb6290a7687c21755b0344ecaeaa05f4e9f6207ae2e9a8"}, {file = "certifi-2022.6.15.2.tar.gz", hash = "sha256:aa08c101214127b9b0472ca6338315113c9487d45376fd3e669201b477c71003"}, @@ -1337,8 +1361,8 @@ pytest = [ {file = "pytest-7.1.3.tar.gz", hash = "sha256:4f365fec2dff9c1162f834d9f18af1ba13062db0c708bf7b946f8a5c76180c39"}, ] pytest-checkdocs = [ - {file = "pytest-checkdocs-2.7.1.tar.gz", hash = "sha256:2b33b85eddfe5846a69bea4a759303e2d5a3be11d03bc7149f5ba1ef47e6c1ae"}, - {file = "pytest_checkdocs-2.7.1-py3-none-any.whl", hash = "sha256:294898c64c9ce1a178edc6660e48da23c7543bfd5a1cea7f0ca4c167745d8461"}, + {file = "pytest-checkdocs-2.8.1.tar.gz", hash = "sha256:9fdf57c96213d3cdddd86b09b4a4fc31ce295821091cf1f37d92ecbcfbbe5276"}, + {file = "pytest_checkdocs-2.8.1-py3-none-any.whl", hash = "sha256:2600428c88487e685394c795d5478c422fbbdf05272cf9393839f06fda1aeb05"}, ] python-dateutil = [ {file = "python-dateutil-2.8.2.tar.gz", hash = "sha256:0123cacc1627ae19ddf3c27a5de5bd67ee4586fbdd6440d9748f8abb483d3e86"}, diff --git a/pyproject.toml b/pyproject.toml index 556ac6fb70..88494df851 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -67,7 +67,7 @@ s3fs = { version = "2022.8.2", optional = true } [tool.poetry.dev-dependencies] pytest = "^7.1.3" -pytest-checkdocs = "^2.0.0" +pytest-checkdocs = "^2.8.1" pre-commit = "^2.0.0" fastavro = "^1.6.1" coverage = { version = "^6.5.0", extras = ["toml"] } From 85eb309caa91256be0088a3055922a896b472d6c Mon Sep 17 00:00:00 2001 From: Fokko Driesprong Date: Tue, 4 Oct 2022 19:04:57 +0200 Subject: [PATCH 226/642] Python: Fix the link in the README (#5898) --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 7e1d06b24c..59cb29f47a 100644 --- a/README.md +++ b/README.md @@ -17,7 +17,7 @@ # Iceberg Python -pyiceberg is a python library for programmatic access to iceberg table metadata as well as to table data in iceberg format. It is a Python implementation of [iceberg table spec](https://iceberg.apache.org/spec/). Documentation is available at [https://pyiceberg.apache.org/](https://pyiceberg.apache.org/). +pyiceberg is a python library for programmatic access to iceberg table metadata as well as to table data in iceberg format. It is a Python implementation of [iceberg table spec](https://iceberg.apache.org/spec/). Documentation is available at [https://py.iceberg.apache.org/](https://py.iceberg.apache.org/). ## Getting Started From 7ab431fb3b26dd667612bc94a3efc8c85939c3df Mon Sep 17 00:00:00 2001 From: Fokko Driesprong Date: Wed, 5 Oct 2022 14:33:57 +0200 Subject: [PATCH 227/642] Python: Bump version to 0.2.0.dev0 (#5906) * Python: Bump version to 0.2.0 In preparation of the next release * Add dev modifier --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index 88494df851..4eda320a05 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -17,7 +17,7 @@ [tool.poetry] name = "pyiceberg" -version = "0.1.0" +version = "0.2.0.dev0" readme = "README.md" homepage = "https://iceberg.apache.org/" repository = "https://github.com/apache/iceberg/" From 6bb6ad39dd57fcad50aa0dd2228a366c0308bbf5 Mon Sep 17 00:00:00 2001 From: Fokko Driesprong Date: Wed, 5 Oct 2022 19:11:09 +0200 Subject: [PATCH 228/642] Python: Pin the version in the codebase (#5854) --- pyiceberg/__init__.py | 3 +-- pyproject.toml | 1 - tests/test_version.py | 11 +++++++---- 3 files changed, 8 insertions(+), 7 deletions(-) diff --git a/pyiceberg/__init__.py b/pyiceberg/__init__.py index 0c4a31db7a..ba28617441 100644 --- a/pyiceberg/__init__.py +++ b/pyiceberg/__init__.py @@ -14,6 +14,5 @@ # KIND, either express or implied. See the License for the # specific language governing permissions and limitations # under the License. -from importlib import metadata -__version__ = metadata.version(__package__) +__version__ = "0.2.0.dev0" diff --git a/pyproject.toml b/pyproject.toml index 4eda320a05..6a0f895511 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -72,7 +72,6 @@ pre-commit = "^2.0.0" fastavro = "^1.6.1" coverage = { version = "^6.5.0", extras = ["toml"] } requests-mock = "^1.9.3" -packaging = "^21.3" [tool.poetry.scripts] pyiceberg = "pyiceberg.cli.console:run" diff --git a/tests/test_version.py b/tests/test_version.py index dffd702871..a1f2191e16 100644 --- a/tests/test_version.py +++ b/tests/test_version.py @@ -15,11 +15,14 @@ # specific language governing permissions and limitations # under the License. -from packaging.version import Version - from pyiceberg import __version__ def test_version_format(): - # We should be able to parse the version - assert Version(__version__) + from importlib import metadata + + installed_version = metadata.version("pyiceberg") + + assert ( + __version__ == installed_version + ), f"{__version__} <> {installed_version}, the installed version does not match with the current codebase" From 763d3f2b535b11f32779bbfae5cedbfb6c2d6855 Mon Sep 17 00:00:00 2001 From: Fokko Driesprong Date: Wed, 5 Oct 2022 22:33:18 +0200 Subject: [PATCH 229/642] Python: Add rewriteNot (#5925) --- pyiceberg/expressions/base.py | 29 ++++++++++ tests/expressions/test_expressions_base.py | 65 ++++++++++++++++++++++ 2 files changed, 94 insertions(+) diff --git a/pyiceberg/expressions/base.py b/pyiceberg/expressions/base.py index 82b7a414cf..ea0873ede0 100644 --- a/pyiceberg/expressions/base.py +++ b/pyiceberg/expressions/base.py @@ -835,3 +835,32 @@ def _(expr: BoundLessThan, visitor: BoundBooleanExpressionVisitor[T]) -> T: @visit_bound_predicate.register(BoundLessThanOrEqual) def _(expr: BoundLessThanOrEqual, visitor: BoundBooleanExpressionVisitor[T]) -> T: return visitor.visit_less_than_or_equal(term=expr.term, literal=expr.literal) + + +def rewrite_not(expr: BooleanExpression) -> BooleanExpression: + return visit(expr, _RewriteNotVisitor()) + + +class _RewriteNotVisitor(BooleanExpressionVisitor[BooleanExpression]): + """Inverts the negations""" + + def visit_true(self) -> BooleanExpression: + return AlwaysTrue() + + def visit_false(self) -> BooleanExpression: + return AlwaysFalse() + + def visit_not(self, child_result: BooleanExpression) -> BooleanExpression: + return ~child_result + + def visit_and(self, left_result: BooleanExpression, right_result: BooleanExpression) -> BooleanExpression: + return And(left=left_result, right=right_result) + + def visit_or(self, left_result: BooleanExpression, right_result: BooleanExpression) -> BooleanExpression: + return Or(left=left_result, right=right_result) + + def visit_unbound_predicate(self, predicate) -> BooleanExpression: + return predicate + + def visit_bound_predicate(self, predicate) -> BooleanExpression: + return predicate diff --git a/tests/expressions/test_expressions_base.py b/tests/expressions/test_expressions_base.py index 94bf94b72a..02997eaa18 100644 --- a/tests/expressions/test_expressions_base.py +++ b/tests/expressions/test_expressions_base.py @@ -22,6 +22,7 @@ import pytest from pyiceberg.expressions import base +from pyiceberg.expressions.base import rewrite_not from pyiceberg.expressions.literals import ( Literal, LongLiteral, @@ -1151,3 +1152,67 @@ def test_bound_boolean_expression_visitor_raise_on_unbound_predicate(): with pytest.raises(TypeError) as exc_info: base.visit(bound_expression, visitor=visitor) assert "Not a bound predicate" in str(exc_info.value) + + +def test_rewrite_not_equal_to(): + assert rewrite_not(base.Not(base.EqualTo(base.Reference("x"), literal(34.56)))) == base.NotEqualTo( + base.Reference("x"), literal(34.56) + ) + + +def test_rewrite_not_not_equal_to(): + assert rewrite_not(base.Not(base.NotEqualTo(base.Reference("x"), literal(34.56)))) == base.EqualTo( + base.Reference("x"), literal(34.56) + ) + + +def test_rewrite_not_in(): + assert rewrite_not(base.Not(base.In(base.Reference("x"), (literal(34.56),)))) == base.NotIn( + base.Reference("x"), (literal(34.56),) + ) + + +def test_rewrite_and(): + assert rewrite_not( + base.Not( + base.And( + base.EqualTo(base.Reference("x"), literal(34.56)), + base.EqualTo(base.Reference("y"), literal(34.56)), + ) + ) + ) == base.Or( + base.NotEqualTo(term=base.Reference(name="x"), literal=literal(34.56)), + base.NotEqualTo(term=base.Reference(name="y"), literal=literal(34.56)), + ) + + +def test_rewrite_or(): + assert rewrite_not( + base.Not( + base.Or( + base.EqualTo(base.Reference("x"), literal(34.56)), + base.EqualTo(base.Reference("y"), literal(34.56)), + ) + ) + ) == base.And( + base.NotEqualTo(term=base.Reference(name="x"), literal=literal(34.56)), + base.NotEqualTo(term=base.Reference(name="y"), literal=literal(34.56)), + ) + + +def test_rewrite_always_false(): + assert rewrite_not(base.Not(base.AlwaysFalse())) == base.AlwaysTrue() + + +def test_rewrite_always_true(): + assert rewrite_not(base.Not(base.AlwaysTrue())) == base.AlwaysFalse() + + +def test_rewrite_bound(): + schema = Schema(NestedField(2, "a", IntegerType(), required=False), schema_id=1) + assert rewrite_not(base.IsNull(base.Reference("a")).bind(schema)) == base.BoundIsNull( + term=base.BoundReference( + field=NestedField(field_id=2, name="a", field_type=IntegerType(), required=False), + accessor=Accessor(position=0, inner=None), + ) + ) From 35072ea2fc05cc17371f8ef8c8e61717d939bf47 Mon Sep 17 00:00:00 2001 From: Fokko Driesprong Date: Wed, 5 Oct 2022 22:45:49 +0200 Subject: [PATCH 230/642] Python: Add more expression tests (#5923) --- pyiceberg/expressions/base.py | 25 +- pyiceberg/schema.py | 25 +- tests/expressions/test_expressions_base.py | 477 ++++++++++++++++----- 3 files changed, 412 insertions(+), 115 deletions(-) diff --git a/pyiceberg/expressions/base.py b/pyiceberg/expressions/base.py index ea0873ede0..5272145758 100644 --- a/pyiceberg/expressions/base.py +++ b/pyiceberg/expressions/base.py @@ -65,11 +65,11 @@ class BoundTerm(Term[T], Bound, ABC): @abstractmethod def ref(self) -> BoundReference[T]: - ... + """Returns the bound reference""" @abstractmethod - def eval(self, struct: StructProtocol): # pylint: disable=W0613 - ... # pragma: no cover + def eval(self, struct: StructProtocol) -> T: # pylint: disable=W0613 + """Returns the value at the referenced field's position in an object that abides by the StructProtocol""" @dataclass(frozen=True) @@ -129,13 +129,7 @@ def bind(self, schema: Schema, case_sensitive: bool = True) -> BoundReference[T] BoundReference: A reference bound to the specific field in the Iceberg schema """ field = schema.find_field(name_or_id=self.name, case_sensitive=case_sensitive) - if not field: - raise ValueError(f"Cannot find field '{self.name}' in schema: {schema}") - accessor = schema.accessor_for_field(field.field_id) - if not accessor: - raise ValueError(f"Cannot find accessor for field '{self.name}' in schema: {schema}") - return BoundReference(field=field, accessor=accessor) @@ -233,6 +227,7 @@ class BoundPredicate(Generic[T], Bound, BooleanExpression): term: BoundTerm[T] def __invert__(self) -> BoundPredicate[T]: + """Inverts the predicate""" raise NotImplementedError @@ -242,9 +237,11 @@ class UnboundPredicate(Generic[T], Unbound[BooleanExpression], BooleanExpression term: UnboundTerm[T] def __invert__(self) -> UnboundPredicate[T]: + """Inverts the predicate""" raise NotImplementedError def bind(self, schema: Schema, case_sensitive: bool = True) -> BooleanExpression: + """Binds the predicate to a schema""" raise NotImplementedError @@ -255,12 +252,14 @@ def bind(self, schema: Schema, case_sensitive: bool = True) -> BooleanExpression return self.as_bound(bound_term) def __invert__(self) -> UnaryPredicate[T]: + """Inverts the unary predicate""" raise NotImplementedError @dataclass(frozen=True) class BoundUnaryPredicate(BoundPredicate[T]): def __invert__(self) -> BoundUnaryPredicate[T]: + """Inverts the unary predicate""" raise NotImplementedError @@ -347,6 +346,7 @@ class SetPredicate(UnboundPredicate[T]): literals: tuple[Literal[T], ...] def __invert__(self) -> SetPredicate[T]: + """Inverted expression of the SetPredicate""" raise NotImplementedError def bind(self, schema: Schema, case_sensitive: bool = True) -> BooleanExpression: @@ -359,6 +359,7 @@ class BoundSetPredicate(BoundPredicate[T]): literals: set[Literal[T]] def __invert__(self) -> BoundSetPredicate[T]: + """Inverted expression of the SetPredicate""" raise NotImplementedError @@ -435,6 +436,7 @@ def bind(self, schema: Schema, case_sensitive: bool = True) -> BooleanExpression return self.as_bound(bound_term, self.literal.to(bound_term.ref().field.field_type)) def __invert__(self) -> LiteralPredicate[T]: + """Inverts the literal predicate""" raise NotImplementedError @@ -443,6 +445,7 @@ class BoundLiteralPredicate(BoundPredicate[T]): literal: Literal[T] def __invert__(self) -> BoundLiteralPredicate[T]: + """Inverts the bound literal predicate""" raise NotImplementedError @@ -678,10 +681,10 @@ def visit_and(self, left_result: BooleanExpression, right_result: BooleanExpress def visit_or(self, left_result: BooleanExpression, right_result: BooleanExpression) -> BooleanExpression: return Or(left=left_result, right=right_result) - def visit_unbound_predicate(self, predicate) -> BooleanExpression: + def visit_unbound_predicate(self, predicate: UnboundPredicate) -> BooleanExpression: return predicate.bind(self._schema, case_sensitive=self._case_sensitive) - def visit_bound_predicate(self, predicate) -> BooleanExpression: + def visit_bound_predicate(self, predicate: BoundPredicate) -> BooleanExpression: raise TypeError(f"Found already bound predicate: {predicate}") diff --git a/pyiceberg/schema.py b/pyiceberg/schema.py index b0fc41b0e1..7f2c3166ff 100644 --- a/pyiceberg/schema.py +++ b/pyiceberg/schema.py @@ -133,27 +133,33 @@ def as_struct(self) -> StructType: """Returns the schema as a struct""" return StructType(*self.fields) - def find_field(self, name_or_id: Union[str, int], case_sensitive: bool = True) -> Optional[NestedField]: + def find_field(self, name_or_id: Union[str, int], case_sensitive: bool = True) -> NestedField: """Find a field using a field name or field ID Args: name_or_id (str | int): Either a field name or a field ID case_sensitive (bool, optional): Whether to perform a case-sensitive lookup using a field name. Defaults to True. + Raises: + ValueError: When the value cannot be found + Returns: NestedField: The matched NestedField """ if isinstance(name_or_id, int): - return self._lazy_id_to_field.get(name_or_id) + if name_or_id not in self._lazy_id_to_field: + raise ValueError(f"Could not find field with id: {name_or_id}") + return self._lazy_id_to_field[name_or_id] + if case_sensitive: field_id = self._name_to_id.get(name_or_id) else: field_id = self._lazy_name_to_id_lower.get(name_or_id.lower()) if field_id is None: - raise ValueError(f"Could not find field with name or id {name_or_id}, case_sensitive={case_sensitive}") + raise ValueError(f"Could not find field with name {name_or_id}, case_sensitive={case_sensitive}") - return self._lazy_id_to_field.get(field_id) + return self._lazy_id_to_field[field_id] def find_type(self, name_or_id: Union[str, int], case_sensitive: bool = True) -> IcebergType: """Find a field type using a field name or field ID @@ -185,16 +191,23 @@ def find_column_name(self, column_id: int) -> Optional[str]: """ return self._lazy_id_to_name.get(column_id) - def accessor_for_field(self, field_id: int) -> Optional["Accessor"]: + def accessor_for_field(self, field_id: int) -> "Accessor": """Find a schema position accessor given a field ID Args: field_id (int): The ID of the field + Raises: + ValueError: When the value cannot be found + Returns: Accessor: An accessor for the given field ID """ - return self._lazy_id_to_accessor.get(field_id) + + if field_id not in self._lazy_id_to_accessor: + raise ValueError(f"Could not find accessor for field with id: {field_id}") + + return self._lazy_id_to_accessor[field_id] def select(self, names: List[str], case_sensitive: bool = True) -> "Schema": """Return a new schema instance pruned to a subset of columns diff --git a/tests/expressions/test_expressions_base.py b/tests/expressions/test_expressions_base.py index 02997eaa18..30b6688c75 100644 --- a/tests/expressions/test_expressions_base.py +++ b/tests/expressions/test_expressions_base.py @@ -22,7 +22,7 @@ import pytest from pyiceberg.expressions import base -from pyiceberg.expressions.base import rewrite_not +from pyiceberg.expressions.base import rewrite_not, visit_bound_predicate from pyiceberg.expressions.literals import ( Literal, LongLiteral, @@ -213,7 +213,7 @@ def visit_or(self, left_result: List, right_result: List) -> List: (base.Not(ExpressionA()), "Not(child=ExpressionA())"), ], ) -def test_reprs(op, rep): +def test_reprs(op: base.BooleanExpression, rep: str): assert repr(op) == rep @@ -227,6 +227,42 @@ def test_isnull_bind(): assert base.IsNull(base.Reference("a")).bind(schema) == bound +def test_invert_is_null_bind(): + schema = Schema(NestedField(2, "a", IntegerType(), required=False), schema_id=1) + assert ~base.IsNull(base.Reference("a")).bind(schema) == base.NotNull(base.Reference("a")).bind(schema) + + +def test_invert_not_null_bind(): + schema = Schema(NestedField(2, "a", IntegerType(), required=False), schema_id=1) + assert ~base.NotNull(base.Reference("a")).bind(schema) == base.IsNull(base.Reference("a")).bind(schema) + + +def test_invert_is_nan_bind(): + schema = Schema(NestedField(2, "a", DoubleType(), required=False), schema_id=1) + assert ~base.IsNaN(base.Reference("a")).bind(schema) == base.NotNaN(base.Reference("a")).bind(schema) + + +def test_invert_not_nan_bind(): + schema = Schema(NestedField(2, "a", DoubleType(), required=False), schema_id=1) + assert ~base.NotNaN(base.Reference("a")).bind(schema) == base.IsNaN(base.Reference("a")).bind(schema) + + +def test_bind_expr_does_not_exists(): + schema = Schema(NestedField(2, "a", IntegerType()), schema_id=1) + with pytest.raises(ValueError) as exc_info: + base.IsNull(base.Reference("b")).bind(schema) + + assert str(exc_info.value) == "Could not find field with name b, case_sensitive=True" + + +def test_bind_does_not_exists(): + schema = Schema(NestedField(2, "a", IntegerType()), schema_id=1) + with pytest.raises(ValueError) as exc_info: + base.Reference("b").bind(schema) + + assert str(exc_info.value) == "Could not find field with name b, case_sensitive=True" + + def test_isnull_bind_required(): schema = Schema(NestedField(2, "a", IntegerType(), required=True), schema_id=1) assert base.IsNull(base.Reference("a")).bind(schema) == base.AlwaysFalse() @@ -301,138 +337,289 @@ def test_strs(op, string): assert str(op) == string -def test_ref_binding_case_sensitive(request): - schema = request.getfixturevalue("table_schema_simple") - ref = base.Reference("foo") - bound = base.BoundReference(schema.find_field(1), schema.accessor_for_field(1)) - assert ref.bind(schema, case_sensitive=True) == bound +def test_ref_binding_case_sensitive(table_schema_simple: Schema): + ref = base.Reference[str]("foo") + bound = base.BoundReference[str](table_schema_simple.find_field(1), table_schema_simple.accessor_for_field(1)) + assert ref.bind(table_schema_simple, case_sensitive=True) == bound -def test_ref_binding_case_sensitive_failure(request): - schema = request.getfixturevalue("table_schema_simple") - ref = base.Reference("Foo") +def test_ref_binding_case_sensitive_failure(table_schema_simple: Schema): + ref = base.Reference[str]("Foo") with pytest.raises(ValueError): - ref.bind(schema, case_sensitive=True) + ref.bind(table_schema_simple, case_sensitive=True) -def test_ref_binding_case_insensitive(request): - schema = request.getfixturevalue("table_schema_simple") - ref = base.Reference("Foo") - bound = base.BoundReference(schema.find_field(1), schema.accessor_for_field(1)) - assert ref.bind(schema, case_sensitive=False) == bound +def test_ref_binding_case_insensitive(table_schema_simple: Schema): + ref = base.Reference[str]("Foo") + bound = base.BoundReference[str](table_schema_simple.find_field(1), table_schema_simple.accessor_for_field(1)) + assert ref.bind(table_schema_simple, case_sensitive=False) == bound -def test_ref_binding_case_insensitive_failure(request): - schema = request.getfixturevalue("table_schema_simple") - ref = base.Reference("Foot") +def test_ref_binding_case_insensitive_failure(table_schema_simple: Schema): + ref = base.Reference[str]("Foot") with pytest.raises(ValueError): - ref.bind(schema, case_sensitive=False) + ref.bind(table_schema_simple, case_sensitive=False) def test_in_to_eq(): assert base.In(base.Reference("x"), (literal(34.56),)) == base.EqualTo(base.Reference("x"), literal(34.56)) -def test_bind_in(request): - schema = request.getfixturevalue("table_schema_simple") +def test_empty_bind_in(table_schema_simple: Schema): + bound = base.BoundIn[str]( + base.BoundReference(table_schema_simple.find_field(1), table_schema_simple.accessor_for_field(1)), set() + ) + assert bound == base.AlwaysFalse() + + +def test_empty_bind_not_in(table_schema_simple: Schema): + bound = base.BoundNotIn( + base.BoundReference[str](table_schema_simple.find_field(1), table_schema_simple.accessor_for_field(1)), set() + ) + assert bound == base.AlwaysTrue() + + +def test_bind_not_in_equal_term(table_schema_simple: Schema): + bound = base.BoundNotIn( + base.BoundReference(table_schema_simple.find_field(1), table_schema_simple.accessor_for_field(1)), {literal("hello")} + ) + assert ( + base.BoundNotEqualTo[str]( + term=base.BoundReference( + field=NestedField(field_id=1, name="foo", field_type=StringType(), required=False), + accessor=Accessor(position=0, inner=None), + ), + literal=StringLiteral("hello"), + ) + == bound + ) + + +def test_in_empty(): + assert base.In(base.Reference("foo"), ()) == base.AlwaysFalse() + + +def test_not_in_empty(): + assert base.NotIn(base.Reference("foo"), ()) == base.AlwaysTrue() + + +def test_not_in_equal(): + assert base.NotIn(base.Reference("foo"), (literal("hello"),)) == base.NotEqualTo( + term=base.Reference(name="foo"), literal=StringLiteral("hello") + ) + + +def test_bind_in(table_schema_simple: Schema): bound = base.BoundIn( - base.BoundReference(schema.find_field(1), schema.accessor_for_field(1)), {literal("hello"), literal("world")} + base.BoundReference(table_schema_simple.find_field(1), table_schema_simple.accessor_for_field(1)), + {literal("hello"), literal("world")}, ) - assert base.In(base.Reference("foo"), (literal("hello"), literal("world"))).bind(schema) == bound + assert base.In(base.Reference("foo"), (literal("hello"), literal("world"))).bind(table_schema_simple) == bound -def test_bind_dedup(request): - schema = request.getfixturevalue("table_schema_simple") +def test_bind_in_invert(table_schema_simple: Schema): bound = base.BoundIn( - base.BoundReference(schema.find_field(1), schema.accessor_for_field(1)), {literal("hello"), literal("world")} + base.BoundReference(table_schema_simple.find_field(1), table_schema_simple.accessor_for_field(1)), + {literal("hello"), literal("world")}, + ) + assert ~bound == base.BoundNotIn( + base.BoundReference(table_schema_simple.find_field(1), table_schema_simple.accessor_for_field(1)), + {literal("hello"), literal("world")}, ) - assert base.In(base.Reference("foo"), (literal("hello"), literal("world"), literal("world"))).bind(schema) == bound -def test_bind_dedup_to_eq(request): - schema = request.getfixturevalue("table_schema_simple") - bound = base.BoundEqualTo(base.BoundReference(schema.find_field(1), schema.accessor_for_field(1)), literal("hello")) - assert base.In(base.Reference("foo"), (literal("hello"), literal("hello"))).bind(schema) == bound +def test_bind_not_in_invert(table_schema_simple: Schema): + bound = base.BoundNotIn( + base.BoundReference(table_schema_simple.find_field(1), table_schema_simple.accessor_for_field(1)), + {literal("hello"), literal("world")}, + ) + assert ~bound == base.BoundIn( + base.BoundReference(table_schema_simple.find_field(1), table_schema_simple.accessor_for_field(1)), + {literal("hello"), literal("world")}, + ) -@pytest.mark.parametrize( - "a, schema", - [ - ( - base.NotIn(base.Reference("foo"), (literal("hello"), literal("world"))), - "table_schema_simple", - ), - ( - base.NotEqualTo(base.Reference("foo"), literal("hello")), - "table_schema_simple", - ), - ( - base.EqualTo(base.Reference("foo"), literal("hello")), - "table_schema_simple", - ), - ( - base.GreaterThan(base.Reference("foo"), literal("hello")), - "table_schema_simple", +def test_bind_dedup(table_schema_simple: Schema): + bound = base.BoundIn( + base.BoundReference(table_schema_simple.find_field(1), table_schema_simple.accessor_for_field(1)), + {literal("hello"), literal("world")}, + ) + assert ( + base.In(base.Reference("foo"), (literal("hello"), literal("world"), literal("world"))).bind(table_schema_simple) == bound + ) + + +def test_bind_dedup_to_eq(table_schema_simple: Schema): + bound = base.BoundEqualTo( + base.BoundReference(table_schema_simple.find_field(1), table_schema_simple.accessor_for_field(1)), literal("hello") + ) + assert base.In(base.Reference("foo"), (literal("hello"), literal("hello"))).bind(table_schema_simple) == bound + + +def test_bound_equal_to_invert(table_schema_simple: Schema): + bound = base.BoundEqualTo( + base.BoundReference(table_schema_simple.find_field(1), table_schema_simple.accessor_for_field(1)), literal("hello") + ) + assert ~bound == base.BoundNotEqualTo( + term=base.BoundReference[str]( + field=NestedField(field_id=1, name="foo", field_type=StringType(), required=False), + accessor=Accessor(position=0, inner=None), ), - ( - base.LessThan(base.Reference("foo"), literal("hello")), - "table_schema_simple", + literal=StringLiteral("hello"), + ) + + +def test_bound_not_equal_to_invert(table_schema_simple: Schema): + bound = base.BoundNotEqualTo( + base.BoundReference(table_schema_simple.find_field(1), table_schema_simple.accessor_for_field(1)), literal("hello") + ) + assert ~bound == base.BoundEqualTo( + term=base.BoundReference[str]( + field=NestedField(field_id=1, name="foo", field_type=StringType(), required=False), + accessor=Accessor(position=0, inner=None), ), - ( - base.GreaterThanOrEqual(base.Reference("foo"), literal("hello")), - "table_schema_simple", + literal=StringLiteral("hello"), + ) + + +def test_bound_greater_than_or_equal_invert(table_schema_simple: Schema): + bound = base.BoundGreaterThanOrEqual( + base.BoundReference(table_schema_simple.find_field(1), table_schema_simple.accessor_for_field(1)), literal("hello") + ) + assert ~bound == base.BoundLessThan( + term=base.BoundReference[str]( + field=NestedField(field_id=1, name="foo", field_type=StringType(), required=False), + accessor=Accessor(position=0, inner=None), ), - ( - base.LessThanOrEqual(base.Reference("foo"), literal("hello")), - "table_schema_simple", + literal=StringLiteral("hello"), + ) + + +def test_bound_greater_than_invert(table_schema_simple: Schema): + bound = base.BoundGreaterThan( + base.BoundReference(table_schema_simple.find_field(1), table_schema_simple.accessor_for_field(1)), literal("hello") + ) + assert ~bound == base.BoundLessThanOrEqual( + term=base.BoundReference[str]( + field=NestedField(field_id=1, name="foo", field_type=StringType(), required=False), + accessor=Accessor(position=0, inner=None), ), - ], -) -def test_bind(a, schema, request): - schema = request.getfixturevalue(schema) - assert a.bind(schema, case_sensitive=True).term.field == schema.find_field(a.term.name, case_sensitive=True) + literal=StringLiteral("hello"), + ) -@pytest.mark.parametrize( - "a, schema", - [ - ( - base.In(base.Reference("Bar"), (literal(5), literal(2))), - "table_schema_simple", +def test_bound_less_than_invert(table_schema_simple: Schema): + bound = base.BoundLessThan( + base.BoundReference(table_schema_simple.find_field(1), table_schema_simple.accessor_for_field(1)), literal("hello") + ) + assert ~bound == base.BoundGreaterThanOrEqual( + term=base.BoundReference[str]( + field=NestedField(field_id=1, name="foo", field_type=StringType(), required=False), + accessor=Accessor(position=0, inner=None), ), - ( - base.NotIn(base.Reference("Bar"), (literal(5), literal(2))), - "table_schema_simple", + literal=StringLiteral("hello"), + ) + + +def test_bound_less_than_or_equal_invert(table_schema_simple: Schema): + bound = base.BoundLessThanOrEqual( + base.BoundReference(table_schema_simple.find_field(1), table_schema_simple.accessor_for_field(1)), literal("hello") + ) + assert ~bound == base.BoundGreaterThan( + term=base.BoundReference[str]( + field=NestedField(field_id=1, name="foo", field_type=StringType(), required=False), + accessor=Accessor(position=0, inner=None), ), - ( - base.NotEqualTo(base.Reference("Bar"), literal(5)), - "table_schema_simple", + literal=StringLiteral("hello"), + ) + + +def test_not_equal_to_invert(): + bound = base.NotEqualTo( + term=base.BoundReference( + field=NestedField(field_id=1, name="foo", field_type=StringType(), required=False), + accessor=Accessor(position=0, inner=None), ), - ( - base.EqualTo(base.Reference("Bar"), literal(5)), - "table_schema_simple", + literal=StringLiteral("hello"), + ) + assert ~bound == base.EqualTo( + term=base.BoundReference( + field=NestedField(field_id=1, name="foo", field_type=StringType(), required=False), + accessor=Accessor(position=0, inner=None), ), - ( - base.GreaterThan(base.Reference("Bar"), literal(5)), - "table_schema_simple", + literal=StringLiteral("hello"), + ) + + +def test_greater_than_or_equal_invert(): + bound = base.GreaterThanOrEqual( + term=base.BoundReference( + field=NestedField(field_id=1, name="foo", field_type=StringType(), required=False), + accessor=Accessor(position=0, inner=None), ), - ( - base.LessThan(base.Reference("Bar"), literal(5)), - "table_schema_simple", + literal=StringLiteral("hello"), + ) + assert ~bound == base.LessThan( + term=base.BoundReference( + field=NestedField(field_id=1, name="foo", field_type=StringType(), required=False), + accessor=Accessor(position=0, inner=None), ), - ( - base.GreaterThanOrEqual(base.Reference("Bar"), literal(5)), - "table_schema_simple", + literal=StringLiteral("hello"), + ) + + +def test_less_than_or_equal_invert(): + bound = base.LessThanOrEqual( + term=base.BoundReference( + field=NestedField(field_id=1, name="foo", field_type=StringType(), required=False), + accessor=Accessor(position=0, inner=None), ), - ( - base.LessThanOrEqual(base.Reference("Bar"), literal(5)), - "table_schema_simple", + literal=StringLiteral("hello"), + ) + assert ~bound == base.GreaterThan( + term=base.BoundReference( + field=NestedField(field_id=1, name="foo", field_type=StringType(), required=False), + accessor=Accessor(position=0, inner=None), ), + literal=StringLiteral("hello"), + ) + + +@pytest.mark.parametrize( + "pred", + [ + base.NotIn(base.Reference("foo"), (literal("hello"), literal("world"))), + base.NotEqualTo(base.Reference("foo"), literal("hello")), + base.EqualTo(base.Reference("foo"), literal("hello")), + base.GreaterThan(base.Reference("foo"), literal("hello")), + base.LessThan(base.Reference("foo"), literal("hello")), + base.GreaterThanOrEqual(base.Reference("foo"), literal("hello")), + base.LessThanOrEqual(base.Reference("foo"), literal("hello")), ], ) -def test_bind_case_insensitive(a, schema, request): - schema = request.getfixturevalue(schema) - assert a.bind(schema, case_sensitive=False).term.field == schema.find_field(a.term.name, case_sensitive=False) +def test_bind(pred, table_schema_simple: Schema): + assert pred.bind(table_schema_simple, case_sensitive=True).term.field == table_schema_simple.find_field( + pred.term.name, case_sensitive=True + ) + + +@pytest.mark.parametrize( + "pred", + [ + base.In(base.Reference("Bar"), (literal(5), literal(2))), + base.NotIn(base.Reference("Bar"), (literal(5), literal(2))), + base.NotEqualTo(base.Reference("Bar"), literal(5)), + base.EqualTo(base.Reference("Bar"), literal(5)), + base.GreaterThan(base.Reference("Bar"), literal(5)), + base.LessThan(base.Reference("Bar"), literal(5)), + base.GreaterThanOrEqual(base.Reference("Bar"), literal(5)), + base.LessThanOrEqual(base.Reference("Bar"), literal(5)), + ], +) +def test_bind_case_insensitive(pred, table_schema_simple: Schema): + assert pred.bind(table_schema_simple, case_sensitive=False).term.field == table_schema_simple.find_field( + pred.term.name, case_sensitive=False + ) @pytest.mark.parametrize( @@ -526,15 +713,24 @@ def test_reduce(lhs, rhs): [ (base.And(base.AlwaysTrue(), ExpressionB()), ExpressionB()), (base.And(base.AlwaysFalse(), ExpressionB()), base.AlwaysFalse()), + (base.And(ExpressionB(), base.AlwaysTrue()), ExpressionB()), (base.Or(base.AlwaysTrue(), ExpressionB()), base.AlwaysTrue()), (base.Or(base.AlwaysFalse(), ExpressionB()), ExpressionB()), + (base.Or(ExpressionA(), base.AlwaysFalse()), ExpressionA()), (base.Not(base.Not(ExpressionA())), ExpressionA()), + (base.Not(base.AlwaysTrue()), base.AlwaysFalse()), + (base.Not(base.AlwaysFalse()), base.AlwaysTrue()), ], ) def test_base_AlwaysTrue_base_AlwaysFalse(lhs, rhs): assert lhs == rhs +def test_invert_always(): + assert ~base.AlwaysFalse() == base.AlwaysTrue() + assert ~base.AlwaysTrue() == base.AlwaysFalse() + + def test_accessor_base_class(foo_struct): """Test retrieving a value at a position of a container using an accessor""" @@ -643,28 +839,46 @@ def test_boolean_expression_visit_raise_not_implemented_error(): assert str(exc_info.value) == "Cannot visit unsupported expression: foo" -def test_always_true_expression_binding(table_schema_simple): +def test_bind_visitor_already_bound(table_schema_simple: Schema): + bound = base.BoundEqualTo( + base.BoundReference(table_schema_simple.find_field(1), table_schema_simple.accessor_for_field(1)), literal("hello") + ) + with pytest.raises(TypeError) as exc_info: + base.BindVisitor(base.visit(bound, visitor=base.BindVisitor(schema=table_schema_simple))) # type: ignore + assert ( + "Found already bound predicate: BoundEqualTo(term=BoundReference(field=NestedField(field_id=1, name='foo', field_type=StringType(), required=False), accessor=Accessor(position=0,inner=None)), literal=StringLiteral(hello))" + == str(exc_info.value) + ) + + +def test_visit_bound_visitor_unknown_predicate(): + with pytest.raises(TypeError) as exc_info: + visit_bound_predicate({"something"}, FooBoundBooleanExpressionVisitor()) + assert "Unknown predicate: {'something'}" == str(exc_info.value) + + +def test_always_true_expression_binding(table_schema_simple: Schema): """Test that visiting an always-true expression returns always-true""" unbound_expression = base.AlwaysTrue() bound_expression = base.visit(unbound_expression, visitor=base.BindVisitor(schema=table_schema_simple)) assert bound_expression == base.AlwaysTrue() -def test_always_false_expression_binding(table_schema_simple): +def test_always_false_expression_binding(table_schema_simple: Schema): """Test that visiting an always-false expression returns always-false""" unbound_expression = base.AlwaysFalse() bound_expression = base.visit(unbound_expression, visitor=base.BindVisitor(schema=table_schema_simple)) assert bound_expression == base.AlwaysFalse() -def test_always_false_and_always_true_expression_binding(table_schema_simple): +def test_always_false_and_always_true_expression_binding(table_schema_simple: Schema): """Test that visiting both an always-true AND always-false expression returns always-false""" unbound_expression = base.And(base.AlwaysTrue(), base.AlwaysFalse()) bound_expression = base.visit(unbound_expression, visitor=base.BindVisitor(schema=table_schema_simple)) assert bound_expression == base.AlwaysFalse() -def test_always_false_or_always_true_expression_binding(table_schema_simple): +def test_always_false_or_always_true_expression_binding(table_schema_simple: Schema): """Test that visiting always-true OR always-false expression returns always-true""" unbound_expression = base.Or(base.AlwaysTrue(), base.AlwaysFalse()) bound_expression = base.visit(unbound_expression, visitor=base.BindVisitor(schema=table_schema_simple)) @@ -1154,6 +1368,73 @@ def test_bound_boolean_expression_visitor_raise_on_unbound_predicate(): assert "Not a bound predicate" in str(exc_info.value) +def test_bound_predicate_invert(): + with pytest.raises(NotImplementedError): + _ = ~base.BoundPredicate( + term=base.BoundReference( + field=NestedField(field_id=1, name="foo", field_type=StringType(), required=False), + accessor=Accessor(position=0, inner=None), + ) + ) + + +def test_bound_unary_predicate_invert(): + with pytest.raises(NotImplementedError): + _ = ~base.BoundUnaryPredicate( + term=base.BoundReference( + field=NestedField(field_id=1, name="foo", field_type=StringType(), required=False), + accessor=Accessor(position=0, inner=None), + ) + ) + + +def test_bound_set_predicate_invert(): + with pytest.raises(NotImplementedError): + _ = ~base.BoundSetPredicate( + term=base.BoundReference( + field=NestedField(field_id=1, name="foo", field_type=StringType(), required=False), + accessor=Accessor(position=0, inner=None), + ), + literals={literal("hello"), literal("world")}, + ) + + +def test_bound_literal_predicate_invert(): + with pytest.raises(NotImplementedError): + _ = ~base.BoundLiteralPredicate( + term=base.BoundReference( + field=NestedField(field_id=1, name="foo", field_type=StringType(), required=False), + accessor=Accessor(position=0, inner=None), + ), + literal=literal("world"), + ) + + +def test_unbound_predicate_invert(): + with pytest.raises(NotImplementedError): + _ = ~base.UnboundPredicate(term=base.Reference("a")) + + +def test_unbound_predicate_bind(table_schema_simple: Schema): + with pytest.raises(NotImplementedError): + _ = base.UnboundPredicate(term=base.Reference("a")).bind(table_schema_simple) + + +def test_unbound_unary_predicate_invert(): + with pytest.raises(NotImplementedError): + _ = ~base.UnaryPredicate(term=base.Reference("a")) + + +def test_unbound_set_predicate_invert(): + with pytest.raises(NotImplementedError): + _ = ~base.SetPredicate(term=base.Reference("a"), literals=(literal("hello"), literal("world"))) + + +def test_unbound_literal_predicate_invert(): + with pytest.raises(NotImplementedError): + _ = ~base.LiteralPredicate(term=base.Reference("a"), literal=literal("hello")) + + def test_rewrite_not_equal_to(): assert rewrite_not(base.Not(base.EqualTo(base.Reference("x"), literal(34.56)))) == base.NotEqualTo( base.Reference("x"), literal(34.56) From 392d7e479566ae428c67e3e169acf93f98d481ff Mon Sep 17 00:00:00 2001 From: Fokko Driesprong Date: Wed, 5 Oct 2022 22:48:49 +0200 Subject: [PATCH 231/642] Python: Remove the __init__ from the tests (#5919) --- pyproject.toml | 4 ++++ tests/__init__.py | 16 ---------------- tests/avro/__init__.py | 16 ---------------- tests/catalog/__init__.py | 16 ---------------- tests/cli/__init__.py | 16 ---------------- tests/expressions/__init__.py | 11 ----------- tests/io/__init__.py | 16 ---------------- tests/io/test_io.py | 21 ++++++--------------- tests/table/__init__.py | 16 ---------------- tests/utils/__init__.py | 16 ---------------- 10 files changed, 10 insertions(+), 138 deletions(-) delete mode 100644 tests/__init__.py delete mode 100644 tests/avro/__init__.py delete mode 100644 tests/catalog/__init__.py delete mode 100644 tests/cli/__init__.py delete mode 100644 tests/expressions/__init__.py delete mode 100644 tests/io/__init__.py delete mode 100644 tests/table/__init__.py delete mode 100644 tests/utils/__init__.py diff --git a/pyproject.toml b/pyproject.toml index 6a0f895511..3c12aea47f 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -172,5 +172,9 @@ ignore_missing_imports = true module = "packaging.*" ignore_missing_imports = true +[[tool.mypy.overrides]] +module = "tests.*" +ignore_missing_imports = true + [tool.coverage.run] source = ['pyiceberg/'] diff --git a/tests/__init__.py b/tests/__init__.py deleted file mode 100644 index 13a83393a9..0000000000 --- a/tests/__init__.py +++ /dev/null @@ -1,16 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. diff --git a/tests/avro/__init__.py b/tests/avro/__init__.py deleted file mode 100644 index a67d5ea255..0000000000 --- a/tests/avro/__init__.py +++ /dev/null @@ -1,16 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. diff --git a/tests/catalog/__init__.py b/tests/catalog/__init__.py deleted file mode 100644 index a67d5ea255..0000000000 --- a/tests/catalog/__init__.py +++ /dev/null @@ -1,16 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. diff --git a/tests/cli/__init__.py b/tests/cli/__init__.py deleted file mode 100644 index a67d5ea255..0000000000 --- a/tests/cli/__init__.py +++ /dev/null @@ -1,16 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. diff --git a/tests/expressions/__init__.py b/tests/expressions/__init__.py deleted file mode 100644 index 00eaa2ffe4..0000000000 --- a/tests/expressions/__init__.py +++ /dev/null @@ -1,11 +0,0 @@ -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. diff --git a/tests/io/__init__.py b/tests/io/__init__.py deleted file mode 100644 index 13a83393a9..0000000000 --- a/tests/io/__init__.py +++ /dev/null @@ -1,16 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. diff --git a/tests/io/test_io.py b/tests/io/test_io.py index fe04b76f7e..24e061dc8b 100644 --- a/tests/io/test_io.py +++ b/tests/io/test_io.py @@ -392,17 +392,6 @@ def test_deleting_local_file_using_file_io_output_file(CustomFileIO): assert not os.path.exists(file_location) -class MockFileIO(FileIO): - def new_input(self, location: str): - raise NotImplementedError() - - def new_output(self, location: str): - raise NotImplementedError() - - def delete(self, location: Union[str, InputFile, OutputFile]) -> None: - raise NotImplementedError() - - def test_import_file_io(): assert isinstance(_import_file_io(ARROW_FILE_IO, {}), PyArrowFileIO) @@ -438,16 +427,18 @@ def test_load_file_io_location_no_schema(): assert isinstance(load_file_io({"location": "/no-schema/"}), PyArrowFileIO) -@patch.dict("pyiceberg.io.SCHEMA_TO_FILE_IO", {"test": ["tests.io.test_io.MockFileIO"]}) +@patch.dict("pyiceberg.io.SCHEMA_TO_FILE_IO", {"test": ["tests.io.test_io.LocalFileIO"]}) def test_mock_warehouse_location_file_io(): # For testing the selection logic - assert isinstance(load_file_io({"warehouse": "test://some-path/"}), MockFileIO) + io = load_file_io({"warehouse": "test://some-path/"}) + assert io.properties["warehouse"] == "test://some-path/" -@patch.dict("pyiceberg.io.SCHEMA_TO_FILE_IO", {"test": ["tests.io.test_io.MockFileIO"]}) +@patch.dict("pyiceberg.io.SCHEMA_TO_FILE_IO", {"test": ["tests.io.test_io.LocalFileIO"]}) def test_mock_table_location_file_io(): # For testing the selection logic - assert isinstance(load_file_io({}, "test://some-path/"), MockFileIO) + io = load_file_io({}, "test://some-path/") + assert io.properties == {} def test_gibberish_table_location_file_io(): diff --git a/tests/table/__init__.py b/tests/table/__init__.py deleted file mode 100644 index 13a83393a9..0000000000 --- a/tests/table/__init__.py +++ /dev/null @@ -1,16 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. diff --git a/tests/utils/__init__.py b/tests/utils/__init__.py deleted file mode 100644 index 13a83393a9..0000000000 --- a/tests/utils/__init__.py +++ /dev/null @@ -1,16 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. From 8bb7cf86fc547f0b2f5353c2ce470ffc3a251ca9 Mon Sep 17 00:00:00 2001 From: Fokko Driesprong Date: Fri, 7 Oct 2022 00:55:15 +0200 Subject: [PATCH 232/642] Python: Update release instructions (#5856) --- dev/RELEASE.md | 84 ++++++++++++++++++++++---------------------------- 1 file changed, 37 insertions(+), 47 deletions(-) diff --git a/dev/RELEASE.md b/dev/RELEASE.md index 03ec767808..b0513f341c 100644 --- a/dev/RELEASE.md +++ b/dev/RELEASE.md @@ -19,76 +19,56 @@ # How to release -The guide to release the Python package. +The guide to release PyIceberg. -First we're going to release a release candidate (RC) and publish it to the public to test. Once the vote has passed on the RC, we can release the new version. +First we're going to release a release candidate (RC) and publish it to the public for testing and validation. Once the vote has passed on the RC, we can release the new version. ## Running a release candidate Make sure that you're on the version that you want to release. ```bash -export RC=rc0 -export VERSION=0.0.1.${RC} +export RC=rc1 +export VERSION=0.1.0${RC} export VERSION_WITHOUT_RC=${VERSION/rc?/} export VERSION_BRANCH=${VERSION_WITHOUT_RC//./-} +export GIT_TAG=pyiceberg-${VERSION} -git checkout -b apache-iceberg-python-${VERSION_BRANCH} +git tag -s ${GIT_TAG} -m "PyIceberg ${VERSION}" +git push apache ${GIT_TAG} -git tag -s ${VERSION} -m "Apache Iceberg Python ${VERSION}" - -export GIT_TAG=$(git show-ref ${VERSION}) -export GIT_TAG_HASH=${GIT_TAG:0:40} -export LAST_COMMIT_ID=$(git rev-list ${VERSION} 2> /dev/null | head -n 1) +export GIT_TAG_REF=$(git show-ref ${GIT_TAG}) +export GIT_TAG_HASH=${GIT_TAG_REF:0:40} +export LAST_COMMIT_ID=$(git rev-list ${GIT_TAG} 2> /dev/null | head -n 1) ``` The `-s` option will sign the commit. If you don't have a key yet, you can find the instructions [here](http://www.apache.org/dev/openpgp.html#key-gen-generate-key). To install gpg on a M1 based Mac, a couple of additional steps are required: https://gist.github.com/phortuin/cf24b1cca3258720c71ad42977e1ba57 -Next we'll create a source distribution (`sdist`) which will generate a `.tar.gz` with all the source files. - -```bash -# Update the version -poetry version ${VERSION} +Next we'll create a source distribution (`sdist`) which will generate a `.tar.gz` with all the source files. So we can upload the files to the Apache SVN. -git diff pyiceberg/__init__.py -git add pyiceberg/__init__.py -git commit -s -m "Set to version ${VERSION}" ``` - -Now we can stage the version in pypi and upload the files to the Apache SVN. - -Next we're going to build the artifacts: - -```bash -rm -rf dist/ poetry build ``` This will create two artifacts: ``` -Building apache-iceberg (0.1.0) +Building pyiceberg (0.1.0) - Building sdist - - Built apache-iceberg-0.1.0.tar.gz + - Built pyiceberg-0.1.0.tar.gz - Building wheel - Built apache_iceberg-0.1.0-py3-none-any.whl ``` The `sdist` contains the source which can be used for checking licenses, and the wheel is a compiled version for quick installation. -Next, we can upload them to pypi. Please keep in mind that this **won't** bump the version for everyone that hasn't pinned their version, since it is RC [pre-release and those are ignored](https://packaging.python.org/en/latest/guides/distributing-packages-using-setuptools/#pre-release-versioning). - -``` -twine upload dist/* -``` - -Before committing the files to the Apache SVN artifact distribution SVN, we need to generate hashes, and we need t o sign them using gpg: +Before committing the files to the Apache SVN artifact distribution SVN, we need to generate hashes, and we need to sign them using gpg: ```bash -for name in "apache_iceberg-${VERSION}-py3-none-any.whl" "apache-iceberg-${VERSION}.tar.gz" +for name in "pyiceberg-${VERSION_WITHOUT_RC}-py3-none-any.whl" "pyiceberg-${VERSION_WITHOUT_RC}.tar.gz" do gpg --yes --armor --local-user fokko@apache.org --output "dist/${name}.asc" --detach-sig "dist/${name}" - shasum -a 512 "dist/${name}" > "dist/${name}.sha512" + (cd dist/ && shasum -a 512 "${name}" > "${name}.sha512") done ``` @@ -98,11 +78,20 @@ Next, we'll clone the Apache SVN, copy and commit the files: export SVN_TMP_DIR=/tmp/iceberg-${VERSION_BRANCH}/ svn checkout https://dist.apache.org/repos/dist/dev/iceberg $SVN_TMP_DIR -export SVN_TMP_DIR_VERSIONED=${SVN_TMP_DIR}apache-iceberg-$VERSION/ +export SVN_TMP_DIR_VERSIONED=${SVN_TMP_DIR}pyiceberg-$VERSION/ mkdir -p $SVN_TMP_DIR_VERSIONED cp dist/* $SVN_TMP_DIR_VERSIONED svn add $SVN_TMP_DIR_VERSIONED -svn ci -m "Apache Iceberg ${VERSION}" ${SVN_TMP_DIR_VERSIONED} +svn ci -m "PyIceberg ${VERSION}" ${SVN_TMP_DIR_VERSIONED} +``` + +Next, we can upload them to pypi. Please keep in mind that this **won't** bump the version for everyone that hasn't pinned their version, we set it to a RC [pre-release and those are ignored](https://packaging.python.org/en/latest/guides/distributing-packages-using-setuptools/#pre-release-versioning). + +``` +poetry version ${VERSION} +rm -rf dist/ +poetry build +twine upload -s dist/* ``` Finally, we can generate the email what we'll send to the mail list: @@ -110,20 +99,20 @@ Finally, we can generate the email what we'll send to the mail list: ```bash cat << EOF > release-announcement-email.txt To: dev@iceberg.apache.org -Subject: [VOTE] Release Apache Iceberg Python Client $VERSION +Subject: [VOTE] Release Apache PyIceberg $VERSION_WITHOUT_RC Hi Everyone, -I propose that we release the following RC as the official Apache Iceberg Python Client $VERSION release. +I propose that we release the following RC as the official PyIceberg $VERSION_WITHOUT_RC release. The commit ID is $LAST_COMMIT_ID -* This corresponds to the tag: $GIT_TAG_HASH -* https://github.com/apache/iceberg/commits/apache-iceberg-python-$VERSION_BRANCH -* https://github.com/apache/iceberg/tree/$GIT_TAG_HASH +* This corresponds to the tag: $GIT_TAG ($GIT_TAG_HASH) +* https://github.com/apache/iceberg/releases/tag/$GIT_TAG +* https://github.com/apache/iceberg/tree/$LAST_COMMIT_ID The release tarball, signature, and checksums are here: -* https://dist.apache.org/repos/dist/dev/iceberg/apache-iceberg-python-$VERSION/ +* https://dist.apache.org/repos/dist/dev/iceberg/pyiceberg-$VERSION/ You can find the KEYS file here: @@ -131,13 +120,14 @@ You can find the KEYS file here: Convenience binary artifacts are staged on pypi: -https://pypi.org/project/apache-iceberg/$VERSION/ +https://pypi.org/project/pyiceberg/$VERSION/ -And can be installed using: pip install apache-iceberg==$VERSION +And can be installed using: pip3 install pyiceberg==$VERSION Please download, verify, and test. + Please vote in the next 72 hours. -[ ] +1 Release this as Apache Iceberg Python Client $VERSION +[ ] +1 Release this as PyIceberg $VERSION [ ] +0 [ ] -1 Do not release this because... EOF From f89556d505d4b81a1af8336923cb3d69a64a1579 Mon Sep 17 00:00:00 2001 From: Fokko Driesprong Date: Fri, 7 Oct 2022 22:53:43 +0200 Subject: [PATCH 233/642] Python: Add partition_type to partition_spec (#5929) --- pyiceberg/table/partitioning.py | 27 +++++++++++++++++++++++++-- tests/table/test_partitioning.py | 20 ++++++++++++++++++++ 2 files changed, 45 insertions(+), 2 deletions(-) diff --git a/pyiceberg/table/partitioning.py b/pyiceberg/table/partitioning.py index 8dcbf68454..1445a3f327 100644 --- a/pyiceberg/table/partitioning.py +++ b/pyiceberg/table/partitioning.py @@ -27,6 +27,7 @@ from pyiceberg.schema import Schema from pyiceberg.transforms import Transform +from pyiceberg.types import NestedField, StructType from pyiceberg.utils.iceberg_base_model import IcebergBaseModel INITIAL_PARTITION_SPEC_ID = 0 @@ -77,9 +78,8 @@ class PartitionSpec(IcebergBaseModel): PartitionSpec captures the transformation from table data to partition values Attributes: - schema(Schema): the schema of data table spec_id(int): any change to PartitionSpec will produce a new specId - fields(List[PartitionField): list of partition fields to produce partition values + fields(Tuple[PartitionField): list of partition fields to produce partition values """ spec_id: int = Field(alias="spec-id", default=INITIAL_PARTITION_SPEC_ID) @@ -158,6 +158,29 @@ def compatible_with(self, other: "PartitionSpec") -> bool: for this_field, that_field in zip(self.fields, other.fields) ) + def partition_type(self, schema: Schema) -> StructType: + """Produces a struct of the PartitionSpec + + The partition fields should be optional: + + - All partition transforms are required to produce null if the input value is null, so it can + happen when the source column is optional + - Partition fields may be added later, in which case not all files would have the result field, + and it may be null. + + There is a case where we can guarantee that a partition field in the first and only partition spec + that uses a required source column will never be null, but it doesn't seem worth tracking this case. + + :param schema: The schema to bind to + :return: A StructType that represents the PartitionSpec, with a NestedField for each PartitionField + """ + nested_fields = [] + for field in self.fields: + source_type = schema.find_type(field.source_id) + result_type = field.transform.result_type(source_type) + nested_fields.append(NestedField(field.field_id, field.name, result_type, required=False)) + return StructType(*nested_fields) + UNPARTITIONED_PARTITION_SPEC = PartitionSpec(spec_id=0) diff --git a/tests/table/test_partitioning.py b/tests/table/test_partitioning.py index 9aa0f73891..b3cbf30f8f 100644 --- a/tests/table/test_partitioning.py +++ b/tests/table/test_partitioning.py @@ -14,8 +14,15 @@ # KIND, either express or implied. See the License for the # specific language governing permissions and limitations # under the License. +from pyiceberg.schema import Schema from pyiceberg.table.partitioning import UNPARTITIONED_PARTITION_SPEC, PartitionField, PartitionSpec from pyiceberg.transforms import BucketTransform, TruncateTransform +from pyiceberg.types import ( + IntegerType, + NestedField, + StringType, + StructType, +) def test_partition_field_init(): @@ -102,3 +109,16 @@ def test_deserialize_partition_spec(): PartitionField(source_id=2, field_id=1001, transform=BucketTransform(num_buckets=25), name="int_bucket"), ), ) + + +def test_partition_type(table_schema_simple: Schema) -> None: + spec = PartitionSpec( + PartitionField(source_id=1, field_id=1000, transform=TruncateTransform(width=19), name="str_truncate"), + PartitionField(source_id=2, field_id=1001, transform=BucketTransform(num_buckets=25), name="int_bucket"), + spec_id=3, + ) + + assert spec.partition_type(table_schema_simple) == StructType( + NestedField(field_id=1000, name="str_truncate", field_type=StringType(), required=False), + NestedField(field_id=1001, name="int_bucket", field_type=IntegerType(), required=False), + ) From c3544bab0a6343d73edf6ff84bb6dfc7f6d1b806 Mon Sep 17 00:00:00 2001 From: Fokko Driesprong Date: Sun, 9 Oct 2022 22:18:54 +0200 Subject: [PATCH 234/642] Python: Ability to Prune Columns (#5931) This is required if we want to implement `.select(*columns: str)` --- pyiceberg/schema.py | 140 +++++++++++++++++++++++ pyiceberg/types.py | 4 + tests/test_schema.py | 258 ++++++++++++++++++++++++++++++++++++++++++- 3 files changed, 401 insertions(+), 1 deletion(-) diff --git a/pyiceberg/schema.py b/pyiceberg/schema.py index 7f2c3166ff..7f7d096ace 100644 --- a/pyiceberg/schema.py +++ b/pyiceberg/schema.py @@ -27,6 +27,7 @@ List, Literal, Optional, + Set, Tuple, TypeVar, Union, @@ -795,3 +796,142 @@ def map(self, map_type: MapType, key_result: Callable[[], IcebergType], value_re def primitive(self, primitive: PrimitiveType) -> PrimitiveType: return primitive + + +def prune_columns(schema: Schema, selected: Set[int], select_full_types: bool = True) -> Schema: + result = visit(schema.as_struct(), _PruneColumnsVisitor(selected, select_full_types)) + return Schema(*(result or StructType()).fields, schema_id=schema.schema_id, identifier_field_ids=schema.identifier_field_ids) + + +class _PruneColumnsVisitor(SchemaVisitor[Optional[IcebergType]]): + selected: Set[int] + select_full_types: bool + + def __init__(self, selected: Set[int], select_full_types: bool): + self.selected = selected + self.select_full_types = select_full_types + + def schema(self, schema: Schema, struct_result: Optional[IcebergType]) -> Optional[IcebergType]: + return struct_result + + def struct(self, struct: StructType, field_results: List[Optional[IcebergType]]) -> Optional[IcebergType]: + fields = struct.fields + selected_fields = [] + same_type = True + + for idx, projected_type in enumerate(field_results): + field = fields[idx] + if field.field_type == projected_type: + selected_fields.append(field) + elif projected_type is not None: + same_type = False + # Type has changed, create a new field with the projected type + selected_fields.append( + NestedField( + field_id=field.field_id, + name=field.name, + field_type=projected_type, + doc=field.doc, + required=field.required, + ) + ) + + if selected_fields: + if len(selected_fields) == len(fields) and same_type is True: + # Nothing has changed, and we can return the original struct + return struct + else: + return StructType(*selected_fields) + return None + + def field(self, field: NestedField, field_result: Optional[IcebergType]) -> Optional[IcebergType]: + if field.field_id in self.selected: + if self.select_full_types: + return field.field_type + elif field.field_type.is_struct: + return self._project_selected_struct(field_result) + else: + if not field.field_type.is_primitive: + raise ValueError( + f"Cannot explicitly project List or Map types, {field.field_id}:{field.name} of type {field.field_type} was selected" + ) + # Selected non-struct field + return field.field_type + elif field_result is not None: + # This field wasn't selected but a subfield was so include that + return field_result + else: + return None + + def list(self, list_type: ListType, element_result: Optional[IcebergType]) -> Optional[IcebergType]: + if list_type.element_id in self.selected: + if self.select_full_types: + return list_type + elif list_type.element_type and list_type.element_type.is_struct: + projected_struct = self._project_selected_struct(element_result) + return self._project_list(list_type, projected_struct) + else: + if not list_type.element_type.is_primitive: + raise ValueError( + f"Cannot explicitly project List or Map types, {list_type.element_id} of type {list_type.element_type} was selected" + ) + return list_type + elif element_result is not None: + return self._project_list(list_type, element_result) + else: + return None + + def map( + self, map_type: MapType, key_result: Optional[IcebergType], value_result: Optional[IcebergType] + ) -> Optional[IcebergType]: + if map_type.value_id in self.selected: + if self.select_full_types: + return map_type + elif map_type.value_type and map_type.value_type.is_struct: + projected_struct = self._project_selected_struct(value_result) + return self._project_map(map_type, projected_struct) + if not map_type.value_type.is_primitive: + raise ValueError( + f"Cannot explicitly project List or Map types, Map value {map_type.value_id} of type {map_type.value_type} was selected" + ) + return map_type + elif value_result is not None: + return self._project_map(map_type, value_result) + elif map_type.key_id in self.selected: + return map_type + return None + + def primitive(self, primitive: PrimitiveType) -> Optional[IcebergType]: + return None + + @staticmethod + def _project_selected_struct(projected_field: Optional[IcebergType]) -> StructType: + if projected_field and not isinstance(projected_field, StructType): + raise ValueError("Expected a struct") + + if projected_field is None: + return StructType() + else: + return projected_field + + @staticmethod + def _project_list(list_type: ListType, element_result: IcebergType): + if list_type.element_type == element_result: + return list_type + else: + return ListType( + element_id=list_type.element_id, element_type=element_result, element_required=list_type.element_required + ) + + @staticmethod + def _project_map(map_type: MapType, value_result: IcebergType): + if map_type.value_type == value_result: + return map_type + else: + return MapType( + key_id=map_type.key_id, + value_id=map_type.value_id, + key_type=map_type.key_type, + value_type=value_result, + value_required=map_type.value_required, + ) diff --git a/pyiceberg/types.py b/pyiceberg/types.py index 2f4da429b7..86a234a4e5 100644 --- a/pyiceberg/types.py +++ b/pyiceberg/types.py @@ -97,6 +97,10 @@ def validate(cls, v: Any) -> "IcebergType": def is_primitive(self) -> bool: return isinstance(self, PrimitiveType) + @property + def is_struct(self) -> bool: + return isinstance(self, StructType) + class PrimitiveType(IcebergType): """Base class for all Iceberg Primitive Types""" diff --git a/tests/test_schema.py b/tests/test_schema.py index 9fbebb0902..0017760ed2 100644 --- a/tests/test_schema.py +++ b/tests/test_schema.py @@ -23,7 +23,7 @@ from pyiceberg import schema from pyiceberg.expressions.base import Accessor from pyiceberg.files import StructProtocol -from pyiceberg.schema import Schema, build_position_accessors +from pyiceberg.schema import Schema, build_position_accessors, prune_columns from pyiceberg.typedef import EMPTY_DICT from pyiceberg.types import ( BooleanType, @@ -415,3 +415,259 @@ def test_deserialize_schema(table_schema_simple: Schema): ) expected = table_schema_simple assert actual == expected + + +def test_prune_columns_string(table_schema_nested: Schema): + assert prune_columns(table_schema_nested, {1}, False) == Schema( + NestedField(field_id=1, name="foo", field_type=StringType(), required=False), schema_id=1, identifier_field_ids=[1] + ) + + +def test_prune_columns_string_full(table_schema_nested: Schema): + assert prune_columns(table_schema_nested, {1}, True) == Schema( + NestedField(field_id=1, name="foo", field_type=StringType(), required=False), schema_id=1, identifier_field_ids=[1] + ) + + +def test_prune_columns_list(table_schema_nested: Schema): + assert prune_columns(table_schema_nested, {5}, False) == Schema( + NestedField( + field_id=4, + name="qux", + field_type=ListType(type="list", element_id=5, element_type=StringType(), element_required=True), + required=True, + ), + schema_id=1, + identifier_field_ids=[1], + ) + + +def test_prune_columns_list_itself(table_schema_nested: Schema): + with pytest.raises(ValueError) as exc_info: + assert prune_columns(table_schema_nested, {4}, False) + assert "Cannot explicitly project List or Map types, 4:qux of type list was selected" in str(exc_info.value) + + +def test_prune_columns_list_full(table_schema_nested: Schema): + assert prune_columns(table_schema_nested, {5}, True) == Schema( + NestedField( + field_id=4, + name="qux", + field_type=ListType(type="list", element_id=5, element_type=StringType(), element_required=True), + required=True, + ), + schema_id=1, + identifier_field_ids=[1], + ) + + +def test_prune_columns_map(table_schema_nested: Schema): + assert prune_columns(table_schema_nested, {9}, False) == Schema( + NestedField( + field_id=6, + name="quux", + field_type=MapType( + type="map", + key_id=7, + key_type=StringType(), + value_id=8, + value_type=MapType( + type="map", key_id=9, key_type=StringType(), value_id=10, value_type=IntegerType(), value_required=True + ), + value_required=True, + ), + required=True, + ), + schema_id=1, + identifier_field_ids=[1], + ) + + +def test_prune_columns_map_itself(table_schema_nested: Schema): + with pytest.raises(ValueError) as exc_info: + assert prune_columns(table_schema_nested, {6}, False) + assert "Cannot explicitly project List or Map types, 6:quux of type map> was selected" in str( + exc_info.value + ) + + +def test_prune_columns_map_full(table_schema_nested: Schema): + assert prune_columns(table_schema_nested, {9}, True) == Schema( + NestedField( + field_id=6, + name="quux", + field_type=MapType( + type="map", + key_id=7, + key_type=StringType(), + value_id=8, + value_type=MapType( + type="map", key_id=9, key_type=StringType(), value_id=10, value_type=IntegerType(), value_required=True + ), + value_required=True, + ), + required=True, + ), + schema_id=1, + identifier_field_ids=[1], + ) + + +def test_prune_columns_map_key(table_schema_nested: Schema): + assert prune_columns(table_schema_nested, {10}, False) == Schema( + NestedField( + field_id=6, + name="quux", + field_type=MapType( + type="map", + key_id=7, + key_type=StringType(), + value_id=8, + value_type=MapType( + type="map", key_id=9, key_type=StringType(), value_id=10, value_type=IntegerType(), value_required=True + ), + value_required=True, + ), + required=True, + ), + schema_id=1, + identifier_field_ids=[1], + ) + + +def test_prune_columns_struct(table_schema_nested: Schema): + assert prune_columns(table_schema_nested, {16}, False) == Schema( + NestedField( + field_id=15, + name="person", + field_type=StructType(NestedField(field_id=16, name="name", field_type=StringType(), required=False)), + required=False, + ), + schema_id=1, + identifier_field_ids=[1], + ) + + +def test_prune_columns_struct_full(table_schema_nested: Schema): + actual = prune_columns(table_schema_nested, {16}, True) + assert actual == Schema( + NestedField( + field_id=15, + name="person", + field_type=StructType(NestedField(field_id=16, name="name", field_type=StringType(), required=False)), + required=False, + ), + schema_id=1, + identifier_field_ids=[1], + ) + + +def test_prune_columns_empty_struct(): + schema_empty_struct = Schema( + NestedField( + field_id=15, + name="person", + field_type=StructType(), + required=False, + ) + ) + assert prune_columns(schema_empty_struct, {15}, False) == Schema( + NestedField(field_id=15, name="person", field_type=StructType(), required=False), schema_id=0, identifier_field_ids=[] + ) + + +def test_prune_columns_empty_struct_full(): + schema_empty_struct = Schema( + NestedField( + field_id=15, + name="person", + field_type=StructType(), + required=False, + ) + ) + assert prune_columns(schema_empty_struct, {15}, True) == Schema( + NestedField(field_id=15, name="person", field_type=StructType(), required=False), schema_id=0, identifier_field_ids=[] + ) + + +def test_prune_columns_struct_in_map(): + table_schema_nested = Schema( + NestedField( + field_id=6, + name="id_to_person", + field_type=MapType( + key_id=7, + key_type=IntegerType(), + value_id=8, + value_type=StructType( + NestedField(field_id=10, name="name", field_type=StringType(), required=False), + NestedField(field_id=11, name="age", field_type=IntegerType(), required=True), + ), + value_required=True, + ), + required=True, + ), + schema_id=1, + identifier_field_ids=[1], + ) + assert prune_columns(table_schema_nested, {11}, False) == Schema( + NestedField( + field_id=6, + name="id_to_person", + field_type=MapType( + type="map", + key_id=7, + key_type=IntegerType(), + value_id=8, + value_type=StructType(NestedField(field_id=11, name="age", field_type=IntegerType(), required=True)), + value_required=True, + ), + required=True, + ), + schema_id=1, + identifier_field_ids=[1], + ) + + +def test_prune_columns_struct_in_map_full(): + table_schema_nested = Schema( + NestedField( + field_id=6, + name="id_to_person", + field_type=MapType( + key_id=7, + key_type=IntegerType(), + value_id=8, + value_type=StructType( + NestedField(field_id=10, name="name", field_type=StringType(), required=False), + NestedField(field_id=11, name="age", field_type=IntegerType(), required=True), + ), + value_required=True, + ), + required=True, + ), + schema_id=1, + identifier_field_ids=[1], + ) + assert prune_columns(table_schema_nested, {11}, True) == Schema( + NestedField( + field_id=6, + name="id_to_person", + field_type=MapType( + type="map", + key_id=7, + key_type=IntegerType(), + value_id=8, + value_type=StructType(NestedField(field_id=11, name="age", field_type=IntegerType(), required=True)), + value_required=True, + ), + required=True, + ), + schema_id=1, + identifier_field_ids=[1], + ) + + +def test_prune_columns_select_original_schema(table_schema_nested: Schema): + ids = set(range(table_schema_nested.highest_field_id)) + assert prune_columns(table_schema_nested, ids, True) == table_schema_nested From 12f6cc6333fe8275ca1bd6ecf34463f875359d82 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 10 Oct 2022 09:27:31 +0200 Subject: [PATCH 235/642] Build: Bump pytest-checkdocs from 2.8.1 to 2.9.0 in /python (#5941) Bumps [pytest-checkdocs](https://github.com/jaraco/pytest-checkdocs) from 2.8.1 to 2.9.0. - [Release notes](https://github.com/jaraco/pytest-checkdocs/releases) - [Changelog](https://github.com/jaraco/pytest-checkdocs/blob/main/CHANGES.rst) - [Commits](https://github.com/jaraco/pytest-checkdocs/compare/v2.8.1...v2.9.0) --- updated-dependencies: - dependency-name: pytest-checkdocs dependency-type: direct:development update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- poetry.lock | 12 ++++++------ pyproject.toml | 2 +- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/poetry.lock b/poetry.lock index 371dc67212..0b55c50615 100644 --- a/poetry.lock +++ b/poetry.lock @@ -513,7 +513,7 @@ testing = ["argcomplete", "hypothesis (>=3.56)", "mock", "nose", "pygments (>=2. [[package]] name = "pytest-checkdocs" -version = "2.8.1" +version = "2.9.0" description = "check the README when running tests" category = "dev" optional = false @@ -522,10 +522,10 @@ python-versions = ">=3.7" [package.dependencies] build = "*" docutils = ">=0.15" -importlib-metadata = ">=4" +importlib-metadata = {version = ">=4", markers = "python_version < \"3.10\""} [package.extras] -docs = ["jaraco.packaging (>=9)", "rst.linker (>=1.9)", "sphinx (>=3.5)"] +docs = ["furo", "jaraco.packaging (>=9)", "rst.linker (>=1.9)", "sphinx (>=3.5)"] testing = ["flake8 (<5)", "pytest (>=6)", "pytest-black (>=0.3.7)", "pytest-checkdocs (>=2.4)", "pytest-cov", "pytest-enabler (>=1.3)", "pytest-flake8", "pytest-mypy (>=0.9.1)", "types-docutils"] [[package]] @@ -768,7 +768,7 @@ snappy = ["python-snappy"] [metadata] lock-version = "1.1" python-versions = "^3.8" -content-hash = "3c89241579eabf944551a3ee85b80e3bf1bfd661914c1e217a6c382f872c1cdf" +content-hash = "c5636e98f0e6735e1ba219ba5ab1070eaa640da77dc8013d86b3cfc9f15eede5" [metadata.files] aiobotocore = [ @@ -1361,8 +1361,8 @@ pytest = [ {file = "pytest-7.1.3.tar.gz", hash = "sha256:4f365fec2dff9c1162f834d9f18af1ba13062db0c708bf7b946f8a5c76180c39"}, ] pytest-checkdocs = [ - {file = "pytest-checkdocs-2.8.1.tar.gz", hash = "sha256:9fdf57c96213d3cdddd86b09b4a4fc31ce295821091cf1f37d92ecbcfbbe5276"}, - {file = "pytest_checkdocs-2.8.1-py3-none-any.whl", hash = "sha256:2600428c88487e685394c795d5478c422fbbdf05272cf9393839f06fda1aeb05"}, + {file = "pytest-checkdocs-2.9.0.tar.gz", hash = "sha256:54b3d1357a289e3f6336d31d4ceb72e99afe0fa4f552990f1b1f015f1925543b"}, + {file = "pytest_checkdocs-2.9.0-py3-none-any.whl", hash = "sha256:4231e5426915a02385e386b5eb4a70490b143d97cede4c7c77691bb320b5a73f"}, ] python-dateutil = [ {file = "python-dateutil-2.8.2.tar.gz", hash = "sha256:0123cacc1627ae19ddf3c27a5de5bd67ee4586fbdd6440d9748f8abb483d3e86"}, diff --git a/pyproject.toml b/pyproject.toml index 3c12aea47f..06f77dc24e 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -67,7 +67,7 @@ s3fs = { version = "2022.8.2", optional = true } [tool.poetry.dev-dependencies] pytest = "^7.1.3" -pytest-checkdocs = "^2.8.1" +pytest-checkdocs = "^2.9.0" pre-commit = "^2.0.0" fastavro = "^1.6.1" coverage = { version = "^6.5.0", extras = ["toml"] } From 194714f9e160890a9a5461364a80c885c1e68701 Mon Sep 17 00:00:00 2001 From: Fokko Driesprong Date: Mon, 10 Oct 2022 11:36:41 +0200 Subject: [PATCH 236/642] Python: Catch ValidationError on invalid REST response (#5897) * Python: Catch ValidationError on invalid REST response If an implementation of the rest catalog returns something different than we expect, we just propagate the ValidationError, which subclasses the ValueError. When using the CLI, it will tell you that the URI is not set, but actually it is, it is just invalid. This will return the json to the user, and also tell what's wrong with the response. * Fix the formatting --- pyiceberg/catalog/rest.py | 8 +++++++- tests/catalog/test_rest.py | 15 +++++++++++++++ 2 files changed, 22 insertions(+), 1 deletion(-) diff --git a/pyiceberg/catalog/rest.py b/pyiceberg/catalog/rest.py index 23d0ccf942..09f5793f3c 100644 --- a/pyiceberg/catalog/rest.py +++ b/pyiceberg/catalog/rest.py @@ -26,7 +26,7 @@ ) import requests -from pydantic import Field +from pydantic import Field, ValidationError from requests import HTTPError from pyiceberg import __version__ @@ -300,6 +300,12 @@ def _handle_non_200_response(self, exc: HTTPError, error_handler: Dict[int, Type except JSONDecodeError: # In the case we don't have a proper response response = f"RESTError {exc.response.status_code}: Could not decode json payload: {exc.response.text}" + except ValidationError as e: + # In the case we don't have a proper response + errs = ", ".join(err["msg"] for err in e.errors()) + response = ( + f"RESTError {exc.response.status_code}: Received unexpected JSON Payload: {exc.response.text}, errors: {errs}" + ) raise exception(response) from exc diff --git a/tests/catalog/test_rest.py b/tests/catalog/test_rest.py index e97f8ad50f..270bad85f1 100644 --- a/tests/catalog/test_rest.py +++ b/tests/catalog/test_rest.py @@ -28,6 +28,7 @@ NoSuchNamespaceError, NoSuchTableError, OAuthError, + RESTError, TableAlreadyExistsError, ) from pyiceberg.schema import Schema @@ -113,6 +114,20 @@ def test_token_401(rest_mock: Mocker): assert message in str(e.value) +def test_token_401_oauth_error(rest_mock: Mocker): + """This test returns a OAuth error instead of an OpenAPI error""" + message = """RESTError 401: Received unexpected JSON Payload: {"error": "invalid_client", "error_description": "Invalid credentials"}, errors: value is not a valid dict""" + rest_mock.post( + f"{TEST_URI}v1/oauth/tokens", + json={"error": "invalid_client", "error_description": "Invalid credentials"}, + status_code=401, + ) + + with pytest.raises(RESTError) as e: + RestCatalog("rest", uri=TEST_URI, credential=TEST_CREDENTIALS) + assert message in str(e.value) + + def test_list_tables_200(rest_mock: Mocker): namespace = "examples" rest_mock.get( From 160763560fb3a63abf9c5b42677272d6d05e1ba5 Mon Sep 17 00:00:00 2001 From: Fokko Driesprong Date: Thu, 13 Oct 2022 01:31:55 +0200 Subject: [PATCH 237/642] Python: Manifest evaluator (#5845) --- pyiceberg/expressions/base.py | 241 +++++- pyiceberg/manifest.py | 4 +- tests/expressions/test_expressions_base.py | 859 ++++++++++++++++++++- tests/table/test_snapshots.py | 4 +- tests/utils/test_manifest.py | 6 +- 5 files changed, 1100 insertions(+), 14 deletions(-) diff --git a/pyiceberg/expressions/base.py b/pyiceberg/expressions/base.py index 5272145758..aabfb0a226 100644 --- a/pyiceberg/expressions/base.py +++ b/pyiceberg/expressions/base.py @@ -21,15 +21,25 @@ from functools import reduce, singledispatch from typing import ( Any, + Callable, ClassVar, Generic, TypeVar, ) +from pyiceberg.conversions import from_bytes from pyiceberg.expressions.literals import Literal from pyiceberg.files import StructProtocol +from pyiceberg.manifest import ManifestFile, PartitionFieldSummary from pyiceberg.schema import Accessor, Schema -from pyiceberg.types import DoubleType, FloatType, NestedField +from pyiceberg.table import PartitionSpec +from pyiceberg.types import ( + DoubleType, + FloatType, + IcebergType, + NestedField, + PrimitiveType, +) from pyiceberg.utils.singleton import Singleton T = TypeVar("T") @@ -654,17 +664,37 @@ def _(obj: Or, visitor: BooleanExpressionVisitor[T]) -> T: return visitor.visit_or(left_result=left_result, right_result=right_result) +def bind(schema: Schema, expression: BooleanExpression, case_sensitive: bool) -> BooleanExpression: + """Travers over an expression to bind the predicates to the schema + + Args: + schema (Schema): A schema to use when binding the expression + expression (BooleanExpression): An expression containing UnboundPredicates that can be bound + case_sensitive (bool): Whether to consider case when binding a reference to a field in a schema, defaults to True + + Raises: + TypeError: In the case a predicate is already bound + """ + return visit(expression, BindVisitor(schema, case_sensitive)) + + class BindVisitor(BooleanExpressionVisitor[BooleanExpression]): """Rewrites a boolean expression by replacing unbound references with references to fields in a struct schema Args: schema (Schema): A schema to use when binding the expression case_sensitive (bool): Whether to consider case when binding a reference to a field in a schema, defaults to True + + Raises: + TypeError: In the case a predicate is already bound """ + schema: Schema + case_sensitive: bool + def __init__(self, schema: Schema, case_sensitive: bool = True) -> None: - self._schema = schema - self._case_sensitive = case_sensitive + self.schema = schema + self.case_sensitive = case_sensitive def visit_true(self) -> BooleanExpression: return AlwaysTrue() @@ -682,7 +712,7 @@ def visit_or(self, left_result: BooleanExpression, right_result: BooleanExpressi return Or(left=left_result, right=right_result) def visit_unbound_predicate(self, predicate: UnboundPredicate) -> BooleanExpression: - return predicate.bind(self._schema, case_sensitive=self._case_sensitive) + return predicate.bind(self.schema, case_sensitive=self.case_sensitive) def visit_bound_predicate(self, predicate: BoundPredicate) -> BooleanExpression: raise TypeError(f"Found already bound predicate: {predicate}") @@ -775,7 +805,7 @@ def visit_bound_predicate(self, predicate: BoundPredicate[T]) -> T: @singledispatch -def visit_bound_predicate(expr, visitor: BooleanExpressionVisitor[T]) -> T: # pylint: disable=unused-argument +def visit_bound_predicate(expr, _: BooleanExpressionVisitor[T]) -> T: raise TypeError(f"Unknown predicate: {expr}") @@ -867,3 +897,204 @@ def visit_unbound_predicate(self, predicate) -> BooleanExpression: def visit_bound_predicate(self, predicate) -> BooleanExpression: return predicate + + +ROWS_MIGHT_MATCH = True +ROWS_CANNOT_MATCH = False +IN_PREDICATE_LIMIT = 200 + + +def _from_byte_buffer(field_type: IcebergType, val: bytes): + if not isinstance(field_type, PrimitiveType): + raise ValueError(f"Expected a PrimitiveType, got: {type(field_type)}") + return from_bytes(field_type, val) + + +class _ManifestEvalVisitor(BoundBooleanExpressionVisitor[bool]): + partition_fields: list[PartitionFieldSummary] + partition_filter: BooleanExpression + + def __init__(self, partition_struct_schema: Schema, partition_filter: BooleanExpression, case_sensitive: bool = True): + self.partition_filter = bind(partition_struct_schema, rewrite_not(partition_filter), case_sensitive) + + def eval(self, manifest: ManifestFile) -> bool: + if partitions := manifest.partitions: + self.partition_fields = partitions + return visit(self.partition_filter, self) + + # No partition information + return ROWS_MIGHT_MATCH + + def visit_in(self, term: BoundTerm, literals: set[Literal[Any]]) -> bool: + pos = term.ref().accessor.position + field = self.partition_fields[pos] + + if field.lower_bound is None: + return ROWS_CANNOT_MATCH + + if len(literals) > IN_PREDICATE_LIMIT: + return ROWS_MIGHT_MATCH + + lower = _from_byte_buffer(term.ref().field.field_type, field.lower_bound) + + if all(lower > val.value for val in literals): + return ROWS_CANNOT_MATCH + + if field.upper_bound is not None: + upper = _from_byte_buffer(term.ref().field.field_type, field.upper_bound) + if all(upper < val.value for val in literals): + return ROWS_CANNOT_MATCH + + return ROWS_MIGHT_MATCH + + def visit_not_in(self, term: BoundTerm, literals: set[Literal[Any]]) -> bool: + # because the bounds are not necessarily a min or max value, this cannot be answered using + # them. notIn(col, {X, ...}) with (X, Y) doesn't guarantee that X is a value in col. + return ROWS_MIGHT_MATCH + + def visit_is_nan(self, term: BoundTerm) -> bool: + pos = term.ref().accessor.position + field = self.partition_fields[pos] + + if field.contains_nan is False: + return ROWS_CANNOT_MATCH + + return ROWS_MIGHT_MATCH + + def visit_not_nan(self, term: BoundTerm) -> bool: + pos = term.ref().accessor.position + field = self.partition_fields[pos] + + if field.contains_nan is True and field.contains_null is False and field.lower_bound is None: + return ROWS_CANNOT_MATCH + + return ROWS_MIGHT_MATCH + + def visit_is_null(self, term: BoundTerm) -> bool: + pos = term.ref().accessor.position + + if self.partition_fields[pos].contains_null is False: + return ROWS_CANNOT_MATCH + + return ROWS_MIGHT_MATCH + + def visit_not_null(self, term: BoundTerm) -> bool: + pos = term.ref().accessor.position + + # contains_null encodes whether at least one partition value is null, + # lowerBound is null if all partition values are null + all_null = self.partition_fields[pos].contains_null is True and self.partition_fields[pos].lower_bound is None + + if all_null and type(term.ref().field.field_type) in {DoubleType, FloatType}: + # floating point types may include NaN values, which we check separately. + # In case bounds don't include NaN value, contains_nan needs to be checked against. + all_null = self.partition_fields[pos].contains_nan is False + + if all_null: + return ROWS_CANNOT_MATCH + + return ROWS_MIGHT_MATCH + + def visit_equal(self, term: BoundTerm, literal: Literal[Any]) -> bool: + pos = term.ref().accessor.position + field = self.partition_fields[pos] + + if field.lower_bound is None: + # values are all null and literal cannot contain null + return ROWS_CANNOT_MATCH + + lower = _from_byte_buffer(term.ref().field.field_type, field.lower_bound) + + if lower > literal.value: + return ROWS_CANNOT_MATCH + + upper = _from_byte_buffer(term.ref().field.field_type, field.lower_bound) + + if literal.value > upper: + return ROWS_CANNOT_MATCH + + return ROWS_MIGHT_MATCH + + def visit_not_equal(self, term: BoundTerm, literal: Literal[Any]) -> bool: + # because the bounds are not necessarily a min or max value, this cannot be answered using + # them. notEq(col, X) with (X, Y) doesn't guarantee that X is a value in col. + return ROWS_MIGHT_MATCH + + def visit_greater_than_or_equal(self, term: BoundTerm, literal: Literal[Any]) -> bool: + pos = term.ref().accessor.position + field = self.partition_fields[pos] + + if field.upper_bound is None: + return ROWS_CANNOT_MATCH + + upper = _from_byte_buffer(term.ref().field.field_type, field.upper_bound) + + if literal.value > upper: + return ROWS_CANNOT_MATCH + + return ROWS_MIGHT_MATCH + + def visit_greater_than(self, term: BoundTerm, literal: Literal[Any]) -> bool: + pos = term.ref().accessor.position + field = self.partition_fields[pos] + + if field.upper_bound is None: + return ROWS_CANNOT_MATCH + + upper = _from_byte_buffer(term.ref().field.field_type, field.upper_bound) + + if literal.value >= upper: + return ROWS_CANNOT_MATCH + + return ROWS_MIGHT_MATCH + + def visit_less_than(self, term: BoundTerm, literal: Literal[Any]) -> bool: + pos = term.ref().accessor.position + field = self.partition_fields[pos] + + if field.lower_bound is None: + return ROWS_CANNOT_MATCH + + lower = _from_byte_buffer(term.ref().field.field_type, field.lower_bound) + + if literal.value <= lower: + return ROWS_CANNOT_MATCH + + return ROWS_MIGHT_MATCH + + def visit_less_than_or_equal(self, term: BoundTerm, literal: Literal[Any]) -> bool: + pos = term.ref().accessor.position + field = self.partition_fields[pos] + + if field.lower_bound is None: + return ROWS_CANNOT_MATCH + + lower = _from_byte_buffer(term.ref().field.field_type, field.lower_bound) + + if literal.value < lower: + return ROWS_CANNOT_MATCH + + return ROWS_MIGHT_MATCH + + def visit_true(self) -> bool: + return ROWS_MIGHT_MATCH + + def visit_false(self) -> bool: + return ROWS_CANNOT_MATCH + + def visit_not(self, child_result: bool) -> bool: + return not child_result + + def visit_and(self, left_result: bool, right_result: bool) -> bool: + return left_result and right_result + + def visit_or(self, left_result: bool, right_result: bool) -> bool: + return left_result or right_result + + +def manifest_evaluator( + partition_spec: PartitionSpec, schema: Schema, partition_filter: BooleanExpression, case_sensitive: bool = True +) -> Callable[[ManifestFile], bool]: + partition_schema = Schema(*partition_spec.partition_type(schema)) + evaluator = _ManifestEvalVisitor(partition_schema, partition_filter, case_sensitive) + return evaluator.eval diff --git a/pyiceberg/manifest.py b/pyiceberg/manifest.py index 8619d18abf..855e5fc32b 100644 --- a/pyiceberg/manifest.py +++ b/pyiceberg/manifest.py @@ -104,7 +104,7 @@ class ManifestEntry(IcebergBaseModel): data_file: DataFile = Field() -class FieldSummary(IcebergBaseModel): +class PartitionFieldSummary(IcebergBaseModel): contains_null: bool = Field() contains_nan: Optional[bool] = Field() lower_bound: Optional[bytes] = Field() @@ -125,7 +125,7 @@ class ManifestFile(IcebergBaseModel): added_rows_count: Optional[int] = Field() existing_rows_counts: Optional[int] = Field() deleted_rows_count: Optional[int] = Field() - partitions: Optional[List[FieldSummary]] = Field() + partitions: Optional[List[PartitionFieldSummary]] = Field() key_metadata: Optional[bytes] = Field() def fetch_manifest_entry(self, io: FileIO) -> List[ManifestEntry]: diff --git a/tests/expressions/test_expressions_base.py b/tests/expressions/test_expressions_base.py index 30b6688c75..6a254c53cc 100644 --- a/tests/expressions/test_expressions_base.py +++ b/tests/expressions/test_expressions_base.py @@ -17,24 +17,37 @@ import uuid from decimal import Decimal -from typing import List, Set +from typing import Any, List, Set import pytest +from pyiceberg.conversions import to_bytes from pyiceberg.expressions import base -from pyiceberg.expressions.base import rewrite_not, visit_bound_predicate +from pyiceberg.expressions.base import ( + IN_PREDICATE_LIMIT, + BoundPredicate, + _from_byte_buffer, + _ManifestEvalVisitor, + rewrite_not, + visit_bound_predicate, +) from pyiceberg.expressions.literals import ( Literal, LongLiteral, StringLiteral, literal, ) +from pyiceberg.manifest import ManifestFile, PartitionFieldSummary from pyiceberg.schema import Accessor, Schema from pyiceberg.types import ( DoubleType, FloatType, + IcebergType, IntegerType, + ListType, + LongType, NestedField, + PrimitiveType, StringType, ) from pyiceberg.utils.singleton import Singleton @@ -1368,6 +1381,841 @@ def test_bound_boolean_expression_visitor_raise_on_unbound_predicate(): assert "Not a bound predicate" in str(exc_info.value) +def _to_byte_buffer(field_type: IcebergType, val: Any): + if not isinstance(field_type, PrimitiveType): + raise ValueError(f"Expected a PrimitiveType, got: {type(field_type)}") + return to_bytes(field_type, val) + + +def _to_manifest_file(*partitions: PartitionFieldSummary) -> ManifestFile: + return ManifestFile( + manifest_path="", + manifest_length=0, + partition_spec_id=0, + partitions=partitions, + ) + + +def _create_manifest_evaluator(bound_expr: BoundPredicate) -> _ManifestEvalVisitor: + """For testing. Creates a bogus evaluator, and then replaces the expression""" + evaluator = _ManifestEvalVisitor( + Schema(NestedField(1, "id", LongType())), base.EqualTo(term=base.Reference("id"), literal=literal("foo")) + ) + evaluator.partition_filter = bound_expr + return evaluator + + +def test_manifest_evaluator_less_than_no_overlap(): + expr = base.BoundLessThan( + term=base.BoundReference( + field=NestedField(field_id=1, name="foo", field_type=StringType(), required=False), + accessor=Accessor(position=0, inner=None), + ), + literal=StringLiteral("c"), + ) + + manifest = _to_manifest_file( + PartitionFieldSummary( + contains_null=False, + contains_nan=False, + lower_bound=_to_byte_buffer(StringType(), "a"), + upper_bound=_to_byte_buffer(StringType(), "b"), + ) + ) + + assert _create_manifest_evaluator(expr).eval(manifest) + + +def test_manifest_evaluator_less_than_overlap(): + expr = base.BoundLessThan( + term=base.BoundReference( + field=NestedField(field_id=1, name="foo", field_type=StringType(), required=False), + accessor=Accessor(position=0, inner=None), + ), + literal=StringLiteral("a"), + ) + + manifest = _to_manifest_file( + PartitionFieldSummary( + contains_null=False, + contains_nan=False, + lower_bound=_to_byte_buffer(StringType(), "a"), + upper_bound=_to_byte_buffer(StringType(), "b"), + ) + ) + + assert not _create_manifest_evaluator(expr).eval(manifest) + + +def test_manifest_evaluator_less_than_all_null(): + expr = base.BoundLessThan( + term=base.BoundReference( + field=NestedField(field_id=1, name="foo", field_type=StringType(), required=False), + accessor=Accessor(position=0, inner=None), + ), + literal=StringLiteral("a"), + ) + + manifest = _to_manifest_file( + PartitionFieldSummary(contains_null=False, contains_nan=False, lower_bound=None, upper_bound=None) + ) + + # All null + assert not _create_manifest_evaluator(expr).eval(manifest) + + +def test_manifest_evaluator_less_than_no_match(): + expr = base.BoundLessThan( + term=base.BoundReference( + field=NestedField(field_id=1, name="foo", field_type=StringType(), required=False), + accessor=Accessor(position=0, inner=None), + ), + literal=StringLiteral("a"), + ) + + manifest = _to_manifest_file( + PartitionFieldSummary( + contains_null=False, + contains_nan=False, + lower_bound=_to_byte_buffer(StringType(), "a"), + upper_bound=_to_byte_buffer(StringType(), "b"), + ) + ) + + assert not _create_manifest_evaluator(expr).eval(manifest) + + +def test_manifest_evaluator_less_than_or_equal_no_overlap(): + expr = base.BoundLessThanOrEqual( + term=base.BoundReference( + field=NestedField(field_id=1, name="foo", field_type=StringType(), required=False), + accessor=Accessor(position=0, inner=None), + ), + literal=StringLiteral("c"), + ) + + manifest = _to_manifest_file( + PartitionFieldSummary( + contains_null=False, + contains_nan=False, + lower_bound=_to_byte_buffer(StringType(), "a"), + upper_bound=_to_byte_buffer(StringType(), "b"), + ) + ) + + assert _create_manifest_evaluator(expr).eval(manifest) + + +def test_manifest_evaluator_less_than_or_equal_overlap(): + expr = base.BoundLessThanOrEqual( + term=base.BoundReference( + field=NestedField(field_id=1, name="foo", field_type=StringType(), required=False), + accessor=Accessor(position=0, inner=None), + ), + literal=StringLiteral("a"), + ) + + manifest = _to_manifest_file( + PartitionFieldSummary( + contains_null=False, + contains_nan=False, + lower_bound=_to_byte_buffer(StringType(), "a"), + upper_bound=_to_byte_buffer(StringType(), "b"), + ) + ) + + assert _create_manifest_evaluator(expr).eval(manifest) + + +def test_manifest_evaluator_less_than_or_equal_all_null(): + expr = base.BoundLessThanOrEqual( + term=base.BoundReference( + field=NestedField(field_id=1, name="foo", field_type=StringType(), required=False), + accessor=Accessor(position=0, inner=None), + ), + literal=StringLiteral("a"), + ) + + manifest = _to_manifest_file( + PartitionFieldSummary(contains_null=False, contains_nan=False, lower_bound=None, upper_bound=None) + ) + + # All null + assert not _create_manifest_evaluator(expr).eval(manifest) + + +def test_manifest_evaluator_less_than_or_equal_no_match(): + expr = base.BoundLessThanOrEqual( + term=base.BoundReference( + field=NestedField(field_id=1, name="foo", field_type=StringType(), required=False), + accessor=Accessor(position=0, inner=None), + ), + literal=StringLiteral("a"), + ) + + manifest = _to_manifest_file( + PartitionFieldSummary( + contains_null=False, + contains_nan=False, + lower_bound=_to_byte_buffer(StringType(), "b"), + upper_bound=_to_byte_buffer(StringType(), "c"), + ) + ) + + assert not _create_manifest_evaluator(expr).eval(manifest) + + +def test_manifest_evaluator_equal_no_overlap(): + expr = base.BoundEqualTo( + term=base.BoundReference( + field=NestedField(field_id=1, name="foo", field_type=StringType(), required=False), + accessor=Accessor(position=0, inner=None), + ), + literal=StringLiteral("c"), + ) + + manifest = _to_manifest_file( + PartitionFieldSummary( + contains_null=False, + contains_nan=False, + lower_bound=_to_byte_buffer(StringType(), "a"), + upper_bound=_to_byte_buffer(StringType(), "b"), + ) + ) + + assert not _create_manifest_evaluator(expr).eval(manifest) + + +def test_manifest_evaluator_equal_overlap(): + expr = base.BoundEqualTo( + term=base.BoundReference( + field=NestedField(field_id=1, name="foo", field_type=StringType(), required=False), + accessor=Accessor(position=0, inner=None), + ), + literal=StringLiteral("a"), + ) + + manifest = ManifestFile( + manifest_path="", + manifest_length=0, + partition_spec_id=0, + partitions=[ + PartitionFieldSummary( + contains_null=False, + contains_nan=False, + lower_bound=_to_byte_buffer(StringType(), "a"), + upper_bound=_to_byte_buffer(StringType(), "b"), + ) + ], + ) + + assert _create_manifest_evaluator(expr).eval(manifest) + + +def test_manifest_evaluator_equal_all_null(): + expr = base.BoundEqualTo( + term=base.BoundReference( + field=NestedField(field_id=1, name="foo", field_type=StringType(), required=False), + accessor=Accessor(position=0, inner=None), + ), + literal=StringLiteral("a"), + ) + + manifest = _to_manifest_file( + PartitionFieldSummary(contains_null=False, contains_nan=False, lower_bound=None, upper_bound=None) + ) + + # All null + assert not _create_manifest_evaluator(expr).eval(manifest) + + +def test_manifest_evaluator_equal_no_match(): + expr = base.BoundEqualTo( + term=base.BoundReference( + field=NestedField(field_id=1, name="foo", field_type=StringType(), required=False), + accessor=Accessor(position=0, inner=None), + ), + literal=StringLiteral("a"), + ) + + manifest = _to_manifest_file( + PartitionFieldSummary( + contains_null=False, + contains_nan=False, + lower_bound=_to_byte_buffer(StringType(), "b"), + upper_bound=_to_byte_buffer(StringType(), "c"), + ) + ) + + assert not _create_manifest_evaluator(expr).eval(manifest) + + +def test_manifest_evaluator_greater_than_no_overlap(): + expr = base.BoundGreaterThan( + term=base.BoundReference( + field=NestedField(field_id=1, name="foo", field_type=StringType(), required=False), + accessor=Accessor(position=0, inner=None), + ), + literal=StringLiteral("a"), + ) + + manifest = _to_manifest_file( + PartitionFieldSummary( + contains_null=False, + contains_nan=False, + lower_bound=_to_byte_buffer(StringType(), "b"), + upper_bound=_to_byte_buffer(StringType(), "c"), + ) + ) + + assert _create_manifest_evaluator(expr).eval(manifest) + + +def test_manifest_evaluator_greater_than_overlap(): + expr = base.BoundGreaterThan( + term=base.BoundReference( + field=NestedField(field_id=1, name="foo", field_type=StringType(), required=False), + accessor=Accessor(position=0, inner=None), + ), + literal=StringLiteral("c"), + ) + + manifest = _to_manifest_file( + PartitionFieldSummary( + contains_null=False, + contains_nan=False, + lower_bound=_to_byte_buffer(StringType(), "b"), + upper_bound=_to_byte_buffer(StringType(), "c"), + ) + ) + + assert not _create_manifest_evaluator(expr).eval(manifest) + + +def test_manifest_evaluator_greater_than_all_null(): + expr = base.BoundGreaterThan( + term=base.BoundReference( + field=NestedField(field_id=1, name="foo", field_type=StringType(), required=False), + accessor=Accessor(position=0, inner=None), + ), + literal=StringLiteral("a"), + ) + + manifest = _to_manifest_file( + PartitionFieldSummary(contains_null=False, contains_nan=False, lower_bound=None, upper_bound=None) + ) + + # All null + assert not _create_manifest_evaluator(expr).eval(manifest) + + +def test_manifest_evaluator_greater_than_no_match(): + expr = base.BoundGreaterThan( + term=base.BoundReference( + field=NestedField(field_id=1, name="foo", field_type=StringType(), required=False), + accessor=Accessor(position=0, inner=None), + ), + literal=StringLiteral("d"), + ) + + manifest = _to_manifest_file( + PartitionFieldSummary( + contains_null=False, + contains_nan=False, + lower_bound=_to_byte_buffer(StringType(), "b"), + upper_bound=_to_byte_buffer(StringType(), "c"), + ) + ) + + assert not _create_manifest_evaluator(expr).eval(manifest) + + +def test_manifest_evaluator_greater_than_or_equal_no_overlap(): + expr = base.BoundGreaterThanOrEqual( + term=base.BoundReference( + field=NestedField(field_id=1, name="foo", field_type=StringType(), required=False), + accessor=Accessor(position=0, inner=None), + ), + literal=StringLiteral("a"), + ) + + manifest = _to_manifest_file( + PartitionFieldSummary( + contains_null=False, + contains_nan=False, + lower_bound=_to_byte_buffer(StringType(), "b"), + upper_bound=_to_byte_buffer(StringType(), "c"), + ) + ) + + assert _create_manifest_evaluator(expr).eval(manifest) + + +def test_manifest_evaluator_greater_than_or_equal_overlap(): + expr = base.BoundGreaterThanOrEqual( + term=base.BoundReference( + field=NestedField(field_id=1, name="foo", field_type=StringType(), required=False), + accessor=Accessor(position=0, inner=None), + ), + literal=StringLiteral("b"), + ) + + manifest = _to_manifest_file( + PartitionFieldSummary( + contains_null=False, + contains_nan=False, + lower_bound=_to_byte_buffer(StringType(), "a"), + upper_bound=_to_byte_buffer(StringType(), "b"), + ) + ) + + assert _create_manifest_evaluator(expr).eval(manifest) + + +def test_manifest_evaluator_greater_than_or_equal_all_null(): + expr = base.BoundGreaterThanOrEqual( + term=base.BoundReference( + field=NestedField(field_id=1, name="foo", field_type=StringType(), required=False), + accessor=Accessor(position=0, inner=None), + ), + literal=StringLiteral("a"), + ) + + manifest = _to_manifest_file( + PartitionFieldSummary(contains_null=False, contains_nan=False, lower_bound=None, upper_bound=None) + ) + + # All null + assert not _create_manifest_evaluator(expr).eval(manifest) + + +def test_manifest_evaluator_greater_than_or_equal_no_match(): + expr = base.BoundGreaterThanOrEqual( + term=base.BoundReference( + field=NestedField(field_id=1, name="foo", field_type=StringType(), required=False), + accessor=Accessor(position=0, inner=None), + ), + literal=StringLiteral("d"), + ) + + manifest = _to_manifest_file( + PartitionFieldSummary( + contains_null=False, + contains_nan=False, + lower_bound=_to_byte_buffer(StringType(), "b"), + upper_bound=_to_byte_buffer(StringType(), "c"), + ) + ) + + assert not _create_manifest_evaluator(expr).eval(manifest) + + +def test_manifest_evaluator_is_nan(): + expr = base.BoundIsNaN( + term=base.BoundReference( + field=NestedField(field_id=1, name="foo", field_type=DoubleType(), required=False), + accessor=Accessor(position=0, inner=None), + ) + ) + + manifest = _to_manifest_file( + PartitionFieldSummary(contains_null=False, contains_nan=True, lower_bound=None, upper_bound=None) + ) + + assert _create_manifest_evaluator(expr).eval(manifest) + + +def test_manifest_evaluator_is_nan_inverse(): + expr = base.BoundIsNaN( + term=base.BoundReference( + field=NestedField(field_id=1, name="foo", field_type=DoubleType(), required=False), + accessor=Accessor(position=0, inner=None), + ) + ) + + manifest = _to_manifest_file( + PartitionFieldSummary( + contains_null=True, + contains_nan=False, + lower_bound=_to_byte_buffer(DoubleType(), 18.15), + upper_bound=_to_byte_buffer(DoubleType(), 19.25), + ) + ) + + assert not _create_manifest_evaluator(expr).eval(manifest) + + +def test_manifest_evaluator_not_nan(): + expr = base.BoundNotNaN( + term=base.BoundReference( + field=NestedField(field_id=1, name="foo", field_type=DoubleType(), required=False), + accessor=Accessor(position=0, inner=None), + ) + ) + + manifest = _to_manifest_file( + PartitionFieldSummary( + contains_null=False, + contains_nan=False, + lower_bound=_to_byte_buffer(DoubleType(), 18.15), + upper_bound=_to_byte_buffer(DoubleType(), 19.25), + ) + ) + + assert _create_manifest_evaluator(expr).eval(manifest) + + +def test_manifest_evaluator_not_nan_inverse(): + expr = base.BoundNotNaN( + term=base.BoundReference( + field=NestedField(field_id=1, name="foo", field_type=DoubleType(), required=False), + accessor=Accessor(position=0, inner=None), + ) + ) + + manifest = _to_manifest_file( + PartitionFieldSummary(contains_null=False, contains_nan=True, lower_bound=None, upper_bound=None) + ) + + assert not _create_manifest_evaluator(expr).eval(manifest) + + +def test_manifest_evaluator_is_null(): + expr = base.BoundIsNull( + term=base.BoundReference( + field=NestedField(field_id=1, name="foo", field_type=StringType(), required=False), + accessor=Accessor(position=0, inner=None), + ), + ) + + manifest = _to_manifest_file( + PartitionFieldSummary( + contains_null=True, + contains_nan=False, + lower_bound=_to_byte_buffer(StringType(), "a"), + upper_bound=_to_byte_buffer(StringType(), "b"), + ) + ) + + assert _create_manifest_evaluator(expr).eval(manifest) + + +def test_manifest_evaluator_is_null_inverse(): + expr = base.BoundIsNull( + term=base.BoundReference( + field=NestedField(field_id=1, name="foo", field_type=StringType(), required=False), + accessor=Accessor(position=0, inner=None), + ), + ) + + manifest = _to_manifest_file( + PartitionFieldSummary(contains_null=False, contains_nan=True, lower_bound=None, upper_bound=None) + ) + + assert not _create_manifest_evaluator(expr).eval(manifest) + + +def test_manifest_evaluator_not_null(): + expr = base.BoundNotNull( + term=base.BoundReference( + field=NestedField(field_id=1, name="foo", field_type=StringType(), required=False), + accessor=Accessor(position=0, inner=None), + ), + ) + + manifest = _to_manifest_file( + PartitionFieldSummary(contains_null=False, contains_nan=False, lower_bound=None, upper_bound=None) + ) + + assert _create_manifest_evaluator(expr).eval(manifest) + + +def test_manifest_evaluator_not_null_nan(): + expr = base.BoundNotNull( + term=base.BoundReference( + field=NestedField(field_id=1, name="foo", field_type=DoubleType(), required=False), + accessor=Accessor(position=0, inner=None), + ), + ) + + manifest = _to_manifest_file( + PartitionFieldSummary(contains_null=True, contains_nan=False, lower_bound=None, upper_bound=None) + ) + + assert not _create_manifest_evaluator(expr).eval(manifest) + + +def test_manifest_evaluator_not_null_inverse(): + expr = base.BoundNotNull( + term=base.BoundReference( + field=NestedField(field_id=1, name="foo", field_type=StringType(), required=False), + accessor=Accessor(position=0, inner=None), + ), + ) + + manifest = _to_manifest_file(PartitionFieldSummary(contains_null=True, contains_nan=True, lower_bound=None, upper_bound=None)) + + assert not _create_manifest_evaluator(expr).eval(manifest) + + +def test_manifest_evaluator_not_equal_to(): + expr = base.BoundNotEqualTo( + term=base.BoundReference( + field=NestedField(field_id=1, name="foo", field_type=StringType(), required=False), + accessor=Accessor(position=0, inner=None), + ), + literal=literal("this"), + ) + + manifest = _to_manifest_file(PartitionFieldSummary(contains_null=True, contains_nan=True, lower_bound=None, upper_bound=None)) + + assert _create_manifest_evaluator(expr).eval(manifest) + + +def test_manifest_evaluator_in(): + expr = base.BoundIn( + term=base.BoundReference( + field=NestedField(field_id=1, name="foo", field_type=LongType(), required=False), + accessor=Accessor(position=0, inner=None), + ), + literals={LongLiteral(i) for i in range(22)}, + ) + + manifest = _to_manifest_file( + PartitionFieldSummary( + contains_null=True, + contains_nan=True, + lower_bound=_to_byte_buffer(LongType(), 5), + upper_bound=_to_byte_buffer(LongType(), 10), + ) + ) + + assert _create_manifest_evaluator(expr).eval(manifest) + + +def test_manifest_evaluator_not_in(): + expr = base.BoundNotIn( + term=base.BoundReference( + field=NestedField(field_id=1, name="foo", field_type=LongType(), required=False), + accessor=Accessor(position=0, inner=None), + ), + literals={LongLiteral(i) for i in range(22)}, + ) + + manifest = _to_manifest_file( + PartitionFieldSummary( + contains_null=True, + contains_nan=True, + lower_bound=False, + upper_bound=False, + ) + ) + + assert _create_manifest_evaluator(expr).eval(manifest) + + +def test_manifest_evaluator_in_null(): + expr = base.BoundIn( + term=base.BoundReference( + field=NestedField(field_id=1, name="foo", field_type=LongType(), required=False), + accessor=Accessor(position=0, inner=None), + ), + literals={LongLiteral(i) for i in range(22)}, + ) + + manifest = _to_manifest_file( + PartitionFieldSummary( + contains_null=True, + contains_nan=True, + lower_bound=None, + upper_bound=_to_byte_buffer(LongType(), 10), + ) + ) + + assert not _create_manifest_evaluator(expr).eval(manifest) + + +def test_manifest_evaluator_in_inverse(): + expr = base.BoundIn( + term=base.BoundReference( + field=NestedField(field_id=1, name="foo", field_type=LongType(), required=False), + accessor=Accessor(position=0, inner=None), + ), + literals={LongLiteral(i) for i in range(22)}, + ) + + manifest = _to_manifest_file( + PartitionFieldSummary( + contains_null=True, + contains_nan=True, + lower_bound=_to_byte_buffer(LongType(), 1815), + upper_bound=_to_byte_buffer(LongType(), 1925), + ) + ) + + assert not _create_manifest_evaluator(expr).eval(manifest) + + +def test_manifest_evaluator_in_overflow(): + expr = base.BoundIn( + term=base.BoundReference( + field=NestedField(field_id=1, name="foo", field_type=LongType(), required=False), + accessor=Accessor(position=0, inner=None), + ), + literals={LongLiteral(i) for i in range(IN_PREDICATE_LIMIT + 1)}, + ) + + manifest = _to_manifest_file( + PartitionFieldSummary( + contains_null=True, + contains_nan=True, + lower_bound=_to_byte_buffer(LongType(), 1815), + upper_bound=_to_byte_buffer(LongType(), 1925), + ) + ) + + assert _create_manifest_evaluator(expr).eval(manifest) + + +def test_manifest_evaluator_less_than_lower_bound(): + expr = base.BoundIn( + term=base.BoundReference( + field=NestedField(field_id=1, name="foo", field_type=LongType(), required=False), + accessor=Accessor(position=0, inner=None), + ), + literals={LongLiteral(i) for i in range(2)}, + ) + + manifest = _to_manifest_file( + PartitionFieldSummary( + contains_null=True, + contains_nan=True, + lower_bound=_to_byte_buffer(LongType(), 5), + upper_bound=_to_byte_buffer(LongType(), 10), + ) + ) + + assert not _create_manifest_evaluator(expr).eval(manifest) + + +def test_manifest_evaluator_greater_than_upper_bound(): + expr = base.BoundIn( + term=base.BoundReference( + field=NestedField(field_id=1, name="foo", field_type=LongType(), required=False), + accessor=Accessor(position=0, inner=None), + ), + literals={LongLiteral(i) for i in range(20, 22)}, + ) + + manifest = _to_manifest_file( + PartitionFieldSummary( + contains_null=True, + contains_nan=True, + lower_bound=_to_byte_buffer(LongType(), 5), + upper_bound=_to_byte_buffer(LongType(), 10), + ) + ) + + assert not _create_manifest_evaluator(expr).eval(manifest) + + +def test_manifest_evaluator_true(): + expr = base.AlwaysTrue() + + manifest = _to_manifest_file(PartitionFieldSummary(contains_null=True, contains_nan=True, lower_bound=None, upper_bound=None)) + + assert _create_manifest_evaluator(expr).eval(manifest) + + +def test_manifest_evaluator_false(): + expr = base.AlwaysFalse() + + manifest = _to_manifest_file(PartitionFieldSummary(contains_null=True, contains_nan=True, lower_bound=None, upper_bound=None)) + + assert not _create_manifest_evaluator(expr).eval(manifest) + + +def test_manifest_evaluator_not(): + expr = base.Not( + base.BoundIn( + term=base.BoundReference( + field=NestedField(field_id=1, name="foo", field_type=LongType(), required=False), + accessor=Accessor(position=0, inner=None), + ), + literals={LongLiteral(i) for i in range(22)}, + ) + ) + + manifest = _to_manifest_file( + PartitionFieldSummary( + contains_null=True, + contains_nan=True, + lower_bound=_to_byte_buffer(LongType(), 5), + upper_bound=_to_byte_buffer(LongType(), 10), + ) + ) + assert not _create_manifest_evaluator(expr).eval(manifest) + + +def test_manifest_evaluator_and(): + expr = base.And( + base.BoundIn( + term=base.BoundReference( + field=NestedField(field_id=1, name="foo", field_type=LongType(), required=False), + accessor=Accessor(position=0, inner=None), + ), + literals={LongLiteral(i) for i in range(22)}, + ), + base.BoundIn( + term=base.BoundReference( + field=NestedField(field_id=1, name="foo", field_type=LongType(), required=False), + accessor=Accessor(position=0, inner=None), + ), + literals={LongLiteral(i) for i in range(22)}, + ), + ) + + manifest = _to_manifest_file( + PartitionFieldSummary( + contains_null=True, + contains_nan=True, + lower_bound=_to_byte_buffer(LongType(), 5), + upper_bound=_to_byte_buffer(LongType(), 10), + ) + ) + assert _create_manifest_evaluator(expr).eval(manifest) + + +def test_manifest_evaluator_or(): + expr = base.Or( + base.BoundIn( + term=base.BoundReference( + field=NestedField(field_id=1, name="foo", field_type=LongType(), required=False), + accessor=Accessor(position=0, inner=None), + ), + literals={LongLiteral(i) for i in range(22)}, + ), + base.BoundIn( + term=base.BoundReference( + field=NestedField(field_id=1, name="foo", field_type=LongType(), required=False), + accessor=Accessor(position=0, inner=None), + ), + literals={LongLiteral(i) for i in range(22)}, + ), + ) + + manifest = _to_manifest_file( + PartitionFieldSummary( + contains_null=True, + contains_nan=True, + lower_bound=_to_byte_buffer(LongType(), 5), + upper_bound=_to_byte_buffer(LongType(), 10), + ) + ) + + assert _create_manifest_evaluator(expr).eval(manifest) + + def test_bound_predicate_invert(): with pytest.raises(NotImplementedError): _ = ~base.BoundPredicate( @@ -1410,6 +2258,13 @@ def test_bound_literal_predicate_invert(): ) +def test_non_primitive_from_byte_buffer(): + with pytest.raises(ValueError) as exc_info: + _ = _from_byte_buffer(ListType(element_id=1, element_type=StringType()), b"\0x00") + + assert str(exc_info.value) == "Expected a PrimitiveType, got: " + + def test_unbound_predicate_invert(): with pytest.raises(NotImplementedError): _ = ~base.UnboundPredicate(term=base.Reference("a")) diff --git a/tests/table/test_snapshots.py b/tests/table/test_snapshots.py index 60d3bfa226..6460ba4daf 100644 --- a/tests/table/test_snapshots.py +++ b/tests/table/test_snapshots.py @@ -18,7 +18,7 @@ import pytest from pyiceberg.io.pyarrow import PyArrowFileIO -from pyiceberg.manifest import FieldSummary, ManifestContent, ManifestFile +from pyiceberg.manifest import ManifestContent, ManifestFile, PartitionFieldSummary from pyiceberg.table.snapshots import Operation, Snapshot, Summary @@ -151,7 +151,7 @@ def test_fetch_manifest_list(generated_manifest_file_file: str): existing_rows_counts=None, deleted_rows_count=0, partitions=[ - FieldSummary( + PartitionFieldSummary( contains_null=True, contains_nan=False, lower_bound=b"\x01\x00\x00\x00", upper_bound=b"\x02\x00\x00\x00" ) ], diff --git a/tests/utils/test_manifest.py b/tests/utils/test_manifest.py index ae19c5d242..7b1d391e25 100644 --- a/tests/utils/test_manifest.py +++ b/tests/utils/test_manifest.py @@ -19,12 +19,12 @@ from pyiceberg.manifest import ( DataFile, DataFileContent, - FieldSummary, FileFormat, ManifestContent, ManifestEntry, ManifestEntryStatus, ManifestFile, + PartitionFieldSummary, read_manifest_entry, read_manifest_list, ) @@ -280,7 +280,7 @@ def test_read_manifest_list(generated_manifest_file_file: str): existing_data_files_count=0, deleted_data_files_count=0, partitions=[ - FieldSummary( + PartitionFieldSummary( contains_null=True, contains_nan=False, lower_bound=b"\x01\x00\x00\x00", upper_bound=b"\x02\x00\x00\x00" ) ], @@ -321,7 +321,7 @@ def test_read_manifest(generated_manifest_file_file: str, generated_manifest_ent existing_rows_counts=None, deleted_rows_count=0, partitions=[ - FieldSummary( + PartitionFieldSummary( contains_null=True, contains_nan=False, lower_bound=b"\x01\x00\x00\x00", upper_bound=b"\x02\x00\x00\x00" ) ], From 00c321dca7b023da21d4371302239fce86106866 Mon Sep 17 00:00:00 2001 From: Fokko Driesprong Date: Thu, 13 Oct 2022 20:23:59 +0200 Subject: [PATCH 238/642] Python: Bump pre-commit versions (#5943) --- .pre-commit-config.yaml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 7fbabf315e..62b1881950 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -29,7 +29,7 @@ repos: - id: check-yaml - id: check-ast - repo: https://github.com/ambv/black - rev: 22.8.0 + rev: 22.10.0 hooks: - id: black - repo: https://github.com/pre-commit/mirrors-isort @@ -38,7 +38,7 @@ repos: - id: isort args: [ --settings-path=python/pyproject.toml ] - repo: https://github.com/pre-commit/mirrors-mypy - rev: v0.971 + rev: v0.982 hooks: - id: mypy args: [ --install-types, --non-interactive, --config=python/pyproject.toml] @@ -48,7 +48,7 @@ repos: - id: pycln args: [--config=python/pyproject.toml] - repo: https://github.com/asottile/pyupgrade - rev: v2.38.0 + rev: v3.0.0 hooks: - id: pyupgrade args: [--py38-plus] From a82e865499b2917f80df110c58fc96d5d2542754 Mon Sep 17 00:00:00 2001 From: Fokko Driesprong Date: Thu, 13 Oct 2022 20:25:29 +0200 Subject: [PATCH 239/642] Python: Fix incorrect annotation, StringType to BinaryType (#5950) --- pyiceberg/avro/reader.py | 28 ++++++++++++++-------------- 1 file changed, 14 insertions(+), 14 deletions(-) diff --git a/pyiceberg/avro/reader.py b/pyiceberg/avro/reader.py index 247cf1c3ad..3de1105575 100644 --- a/pyiceberg/avro/reader.py +++ b/pyiceberg/avro/reader.py @@ -355,68 +355,68 @@ def primitive_reader(primitive: PrimitiveType) -> Reader: raise ValueError(f"Unknown type: {primitive}") -@primitive_reader.register(FixedType) +@primitive_reader.register def _(primitive: FixedType) -> Reader: return FixedReader(primitive.length) -@primitive_reader.register(DecimalType) +@primitive_reader.register def _(primitive: DecimalType) -> Reader: return DecimalReader(primitive.precision, primitive.scale) -@primitive_reader.register(BooleanType) +@primitive_reader.register def _(_: BooleanType) -> Reader: return BooleanReader() -@primitive_reader.register(IntegerType) +@primitive_reader.register def _(_: IntegerType) -> Reader: return IntegerReader() -@primitive_reader.register(LongType) +@primitive_reader.register def _(_: LongType) -> Reader: # Ints and longs are encoded the same way in Python and # also binary compatible in Avro return IntegerReader() -@primitive_reader.register(FloatType) +@primitive_reader.register def _(_: FloatType) -> Reader: return FloatReader() -@primitive_reader.register(DoubleType) +@primitive_reader.register def _(_: DoubleType) -> Reader: return DoubleReader() -@primitive_reader.register(DateType) +@primitive_reader.register def _(_: DateType) -> Reader: return DateReader() -@primitive_reader.register(TimeType) +@primitive_reader.register def _(_: TimeType) -> Reader: return TimeReader() -@primitive_reader.register(TimestampType) +@primitive_reader.register def _(_: TimestampType) -> Reader: return TimestampReader() -@primitive_reader.register(TimestamptzType) +@primitive_reader.register def _(_: TimestamptzType) -> Reader: return TimestamptzReader() -@primitive_reader.register(StringType) +@primitive_reader.register def _(_: StringType) -> Reader: return StringReader() -@primitive_reader.register(BinaryType) -def _(_: StringType) -> Reader: +@primitive_reader.register +def _(_: BinaryType) -> Reader: return BinaryReader() From d4272864d0d8f2b42067c24138e021419996df5f Mon Sep 17 00:00:00 2001 From: Fokko Driesprong Date: Mon, 17 Oct 2022 13:11:51 +0200 Subject: [PATCH 240/642] Python: Cleanup inconsistencies around OAuth responses (#5957) * Python: Cleanup inconsistency in OAuth responses 401 also returns oauth token reponse https://github.com/apache/iceberg/blob/master/open-api/rest-catalog-open-api.yaml#L170-L178 * Comments * Switch them around --- pyiceberg/catalog/__init__.py | 35 +++++++++++++++++++++++++++++------ pyiceberg/catalog/rest.py | 18 ++++++++++++------ pyiceberg/cli/console.py | 9 +-------- pyiceberg/exceptions.py | 4 ---- tests/catalog/test_rest.py | 28 +++------------------------- 5 files changed, 45 insertions(+), 49 deletions(-) diff --git a/pyiceberg/catalog/__init__.py b/pyiceberg/catalog/__init__.py index ee8372cdc9..d4e3163c29 100644 --- a/pyiceberg/catalog/__init__.py +++ b/pyiceberg/catalog/__init__.py @@ -41,6 +41,7 @@ _ENV_CONFIG = Config() TYPE = "type" +URI = "uri" class CatalogType(Enum): @@ -69,7 +70,7 @@ def load_hive(name: str, conf: Properties) -> Catalog: } -def infer_catalog_type(catalog_properties: RecursiveDict) -> CatalogType | None: +def infer_catalog_type(name: str, catalog_properties: RecursiveDict) -> CatalogType | None: """Tries to infer the type based on the dict Args: @@ -77,6 +78,9 @@ def infer_catalog_type(catalog_properties: RecursiveDict) -> CatalogType | None: Returns: The inferred type based on the provided properties + + Raises: + ValueError: Raises a ValueError in case properties are missing, or the wrong type """ if uri := catalog_properties.get("uri"): if isinstance(uri, str): @@ -84,20 +88,39 @@ def infer_catalog_type(catalog_properties: RecursiveDict) -> CatalogType | None: return CatalogType.REST elif uri.startswith("thrift"): return CatalogType.HIVE - return None + else: + raise ValueError(f"Could not infer the catalog type from the uri: {uri}") + else: + raise ValueError(f"Expects the URI to be a string, got: {type(uri)}") + raise ValueError( + f"URI missing, please provide using --uri, the config or environment variable PYICEBERG_CATALOG__{name.upper()}__URI" + ) def load_catalog(name: str, **properties: str | None) -> Catalog: + """Load the catalog based on the properties + + Will look up the properties from the config, based on the name + + Args: + name: The name of the catalog + properties: The properties that are used next to the configuration + + Returns: + An initialized Catalog + + Raises: + ValueError: Raises a ValueError in case properties are missing or malformed, + or if it could not determine the catalog based on the properties + """ env = _ENV_CONFIG.get_catalog_config(name) conf = merge_config(env or {}, properties) + catalog_type: CatalogType | None if provided_catalog_type := conf.get(TYPE): catalog_type = CatalogType[provided_catalog_type.upper()] else: - if inferred_catalog_type := infer_catalog_type(conf): - catalog_type = inferred_catalog_type - else: - raise ValueError(f"Invalid configuration. Could not determine the catalog type: {properties}") + catalog_type = infer_catalog_type(name, conf) if catalog_type: return AVAILABLE_CATALOGS[catalog_type](name, conf) diff --git a/pyiceberg/catalog/rest.py b/pyiceberg/catalog/rest.py index 09f5793f3c..90db6691f9 100644 --- a/pyiceberg/catalog/rest.py +++ b/pyiceberg/catalog/rest.py @@ -31,6 +31,7 @@ from pyiceberg import __version__ from pyiceberg.catalog import ( + URI, Catalog, Identifier, Properties, @@ -38,7 +39,6 @@ ) from pyiceberg.exceptions import ( AuthorizationExpiredError, - BadCredentialsError, BadRequestError, ForbiddenError, NamespaceAlreadyExistsError, @@ -88,7 +88,9 @@ class Endpoints: CREDENTIAL = "credential" GRANT_TYPE = "grant_type" SCOPE = "scope" +TOKEN = "token" TOKEN_EXCHANGE = "urn:ietf:params:oauth:grant-type:token-exchange" +SEMICOLON = ":" NAMESPACE_SEPARATOR = b"\x1F".decode("UTF-8") @@ -181,9 +183,10 @@ def __init__( properties: Properties that are passed along to the configuration """ self.properties = properties - self.uri = properties["uri"] - if credential := properties.get("credential"): - properties["token"] = self._fetch_access_token(credential) + self.uri = properties[URI] + + if credential := properties.get(CREDENTIAL): + properties[TOKEN] = self._fetch_access_token(credential) super().__init__(name, **self._fetch_config(properties)) def _check_valid_namespace_identifier(self, identifier: Union[str, Identifier]) -> Identifier: @@ -225,7 +228,10 @@ def url(self, endpoint: str, prefixed: bool = True, **kwargs) -> str: return url + endpoint.format(**kwargs) def _fetch_access_token(self, credential: str) -> str: - client_id, client_secret = credential.split(":") + if SEMICOLON in credential: + client_id, client_secret = credential.split(SEMICOLON) + else: + client_id, client_secret = None, credential data = {GRANT_TYPE: CLIENT_CREDENTIALS, CLIENT_ID: client_id, CLIENT_SECRET: client_secret, SCOPE: CATALOG_SCOPE} url = self.url(Endpoints.get_token, prefixed=False) # Uses application/x-www-form-urlencoded by default @@ -233,7 +239,7 @@ def _fetch_access_token(self, credential: str) -> str: try: response.raise_for_status() except HTTPError as exc: - self._handle_non_200_response(exc, {400: OAuthError, 401: BadCredentialsError}) + self._handle_non_200_response(exc, {400: OAuthError, 401: OAuthError}) return TokenResponse(**response.json()).access_token diff --git a/pyiceberg/cli/console.py b/pyiceberg/cli/console.py index bac4ae6bc2..e783a4312f 100644 --- a/pyiceberg/cli/console.py +++ b/pyiceberg/cli/console.py @@ -65,14 +65,7 @@ def run(ctx: Context, catalog: str, verbose: bool, output: str, uri: Optional[st ctx.obj["output"] = JsonOutput(verbose=verbose) try: - try: - ctx.obj["catalog"] = load_catalog(catalog, **properties) - except ValueError as exc: - if not uri: - raise ValueError( - f"URI missing, please provide using --uri, the config or environment variable PYICEBERG_CATALOG__{catalog.upper()}__URI" - ) from exc - raise exc + ctx.obj["catalog"] = load_catalog(catalog, **properties) except Exception as e: ctx.obj["output"].exception(e) ctx.exit(1) diff --git a/pyiceberg/exceptions.py b/pyiceberg/exceptions.py index 86e8a8102f..e44125cd9c 100644 --- a/pyiceberg/exceptions.py +++ b/pyiceberg/exceptions.py @@ -44,10 +44,6 @@ class RESTError(Exception): """Raises when there is an unknown response from the REST Catalog""" -class BadCredentialsError(RESTError): - """Raises when providing invalid credentials""" - - class BadRequestError(RESTError): """Raises when an invalid request is being made""" diff --git a/tests/catalog/test_rest.py b/tests/catalog/test_rest.py index 270bad85f1..e91d8e3674 100644 --- a/tests/catalog/test_rest.py +++ b/tests/catalog/test_rest.py @@ -23,12 +23,10 @@ from pyiceberg.catalog import PropertiesUpdateSummary, Table from pyiceberg.catalog.rest import RestCatalog from pyiceberg.exceptions import ( - BadCredentialsError, NamespaceAlreadyExistsError, NoSuchNamespaceError, NoSuchTableError, OAuthError, - RESTError, TableAlreadyExistsError, ) from pyiceberg.schema import Schema @@ -96,34 +94,14 @@ def test_token_400(rest_mock: Mocker): def test_token_401(rest_mock: Mocker): - message = "Invalid client ID: abc" + message = "invalid_client" rest_mock.post( f"{TEST_URI}v1/oauth/tokens", - json={ - "error": { - "message": message, - "type": "BadCredentialsException", - "code": 401, - } - }, - status_code=401, - ) - - with pytest.raises(BadCredentialsError) as e: - RestCatalog("rest", uri=TEST_URI, credential=TEST_CREDENTIALS) - assert message in str(e.value) - - -def test_token_401_oauth_error(rest_mock: Mocker): - """This test returns a OAuth error instead of an OpenAPI error""" - message = """RESTError 401: Received unexpected JSON Payload: {"error": "invalid_client", "error_description": "Invalid credentials"}, errors: value is not a valid dict""" - rest_mock.post( - f"{TEST_URI}v1/oauth/tokens", - json={"error": "invalid_client", "error_description": "Invalid credentials"}, + json={"error": "invalid_client", "error_description": "Unknown or invalid client"}, status_code=401, ) - with pytest.raises(RESTError) as e: + with pytest.raises(OAuthError) as e: RestCatalog("rest", uri=TEST_URI, credential=TEST_CREDENTIALS) assert message in str(e.value) From 6f1333e1cdb52eb6879747025fcbcc55d9b34b6e Mon Sep 17 00:00:00 2001 From: Dhruv Pratap Date: Thu, 20 Oct 2022 14:16:49 -0400 Subject: [PATCH 241/642] Python: Add SSL support for REST Catalog (#6019) * Python: Add support for providing SSL config for REST Catalog client. * Python: Add documentation on mTLS REST Catalog config. Encapsulate headers as well as it no longer needs to be a property. * Update python/pyiceberg/catalog/rest.py Co-authored-by: Fokko Driesprong * Update python/pyiceberg/catalog/rest.py Co-authored-by: Fokko Driesprong * Python: Refactor to initialize requests session in a single method. * Python: Set the HTTP headers after fetching the OAuth token as the Oauth token call expects the Content-Type application/x-www-form-urlencoded. Enforce the content-type check for oauth calls in unit tests as well. Co-authored-by: Fokko Driesprong --- mkdocs/docs/index.md | 8 ++++ pyiceberg/catalog/rest.py | 84 +++++++++++++++++++++----------------- tests/catalog/test_rest.py | 69 ++++++++++++++++++++++++++++++- 3 files changed, 123 insertions(+), 38 deletions(-) diff --git a/mkdocs/docs/index.md b/mkdocs/docs/index.md index 35ffee1875..b7597d0f00 100644 --- a/mkdocs/docs/index.md +++ b/mkdocs/docs/index.md @@ -91,6 +91,14 @@ catalog: rest: uri: http://rest-catalog/ws/ credential: t-1234:secret + + mtls-secured-catalog: + uri: https://rest-catalog/ws/ + ssl: + client: + cert: /absolute/path/to/client.crt + key: /absolute/path/to/client.key + cabundle: /absolute/path/to/cabundle.pem ``` Lastly, you can also set it using environment variables: diff --git a/pyiceberg/catalog/rest.py b/pyiceberg/catalog/rest.py index 90db6691f9..3caf091660 100644 --- a/pyiceberg/catalog/rest.py +++ b/pyiceberg/catalog/rest.py @@ -25,9 +25,8 @@ Union, ) -import requests from pydantic import Field, ValidationError -from requests import HTTPError +from requests import HTTPError, Session from pyiceberg import __version__ from pyiceberg.catalog import ( @@ -91,6 +90,11 @@ class Endpoints: TOKEN = "token" TOKEN_EXCHANGE = "urn:ietf:params:oauth:grant-type:token-exchange" SEMICOLON = ":" +KEY = "key" +CERT = "cert" +CLIENT = "client" +CA_BUNDLE = "cabundle" +SSL = "ssl" NAMESPACE_SEPARATOR = b"\x1F".decode("UTF-8") @@ -166,8 +170,9 @@ class OAuthErrorResponse(IcebergBaseModel): class RestCatalog(Catalog): - token: Optional[str] uri: str + session: Session + properties: dict def __init__( self, @@ -184,11 +189,35 @@ def __init__( """ self.properties = properties self.uri = properties[URI] - - if credential := properties.get(CREDENTIAL): - properties[TOKEN] = self._fetch_access_token(credential) + self._create_session() super().__init__(name, **self._fetch_config(properties)) + def _create_session(self) -> None: + """Creates a request session with provided catalog configuration""" + + self.session = Session() + # Sets the client side and server side SSL cert verification, if provided as properties. + if ssl_config := self.properties.get(SSL): + if ssl_ca_bundle := ssl_config.get(CA_BUNDLE): + self.session.verify = ssl_ca_bundle + if ssl_client := ssl_config.get(CLIENT): + if all(k in ssl_client for k in (CERT, KEY)): + self.session.cert = (ssl_client[CERT], ssl_client[KEY]) + elif ssl_client_cert := ssl_client.get(CERT): + self.session.cert = ssl_client_cert + + # Set Auth token for subsequent calls in the session + if token := self.properties.get(TOKEN): + self.session.headers[AUTHORIZATION_HEADER] = f"{BEARER_PREFIX} {token}" + elif credential := self.properties.get(CREDENTIAL): + token = self._fetch_access_token(credential) + self.session.headers[AUTHORIZATION_HEADER] = f"{BEARER_PREFIX} {token}" + + # Set HTTP headers + self.session.headers["Content-type"] = "application/json" + self.session.headers["X-Client-Version"] = ICEBERG_REST_SPEC_VERSION + self.session.headers["User-Agent"] = f"PyIceberg/{__version__}" + def _check_valid_namespace_identifier(self, identifier: Union[str, Identifier]) -> Identifier: """The identifier should have at least one element""" identifier_tuple = Catalog.identifier_to_tuple(identifier) @@ -196,17 +225,6 @@ def _check_valid_namespace_identifier(self, identifier: Union[str, Identifier]) raise NoSuchNamespaceError(f"Empty namespace identifier: {identifier}") return identifier_tuple - @property - def headers(self) -> Properties: - headers = { - "Content-type": "application/json", - "X-Client-Version": ICEBERG_REST_SPEC_VERSION, - "User-Agent": f"PyIceberg/{__version__}", - } - if token := self.properties.get("token"): - headers[AUTHORIZATION_HEADER] = f"{BEARER_PREFIX} {token}" - return headers - def url(self, endpoint: str, prefixed: bool = True, **kwargs) -> str: """Constructs the endpoint @@ -235,7 +253,7 @@ def _fetch_access_token(self, credential: str) -> str: data = {GRANT_TYPE: CLIENT_CREDENTIALS, CLIENT_ID: client_id, CLIENT_SECRET: client_secret, SCOPE: CATALOG_SCOPE} url = self.url(Endpoints.get_token, prefixed=False) # Uses application/x-www-form-urlencoded by default - response = requests.post(url=url, data=data) + response = self.session.post(url=url, data=data) try: response.raise_for_status() except HTTPError as exc: @@ -244,7 +262,7 @@ def _fetch_access_token(self, credential: str) -> str: return TokenResponse(**response.json()).access_token def _fetch_config(self, properties: Properties) -> Properties: - response = requests.get(self.url(Endpoints.get_config, prefixed=False), headers=self.headers) + response = self.session.get(self.url(Endpoints.get_config, prefixed=False)) try: response.raise_for_status() except HTTPError as exc: @@ -334,10 +352,9 @@ def create_table( properties=properties, ) serialized_json = request.json() - response = requests.post( + response = self.session.post( self.url(Endpoints.create_table, namespace=namespace_and_table["namespace"]), data=serialized_json, - headers=self.headers, ) try: response.raise_for_status() @@ -355,10 +372,7 @@ def create_table( def list_tables(self, namespace: Union[str, Identifier]) -> List[Identifier]: namespace_tuple = self._check_valid_namespace_identifier(namespace) namespace_concat = NAMESPACE_SEPARATOR.join(namespace_tuple) - response = requests.get( - self.url(Endpoints.list_tables, namespace=namespace_concat), - headers=self.headers, - ) + response = self.session.get(self.url(Endpoints.list_tables, namespace=namespace_concat)) try: response.raise_for_status() except HTTPError as exc: @@ -371,9 +385,7 @@ def load_table(self, identifier: Union[str, Identifier]) -> Table: if len(identifier_tuple) <= 1: raise NoSuchTableError(f"Missing namespace or invalid identifier: {identifier}") - response = requests.get( - self.url(Endpoints.load_table, prefixed=True, **self._split_identifier_for_path(identifier)), headers=self.headers - ) + response = self.session.get(self.url(Endpoints.load_table, prefixed=True, **self._split_identifier_for_path(identifier))) try: response.raise_for_status() except HTTPError as exc: @@ -387,9 +399,8 @@ def load_table(self, identifier: Union[str, Identifier]) -> Table: ) def drop_table(self, identifier: Union[str, Identifier], purge_requested: bool = False) -> None: - response = requests.delete( + response = self.session.delete( self.url(Endpoints.drop_table, prefixed=True, purge=purge_requested, **self._split_identifier_for_path(identifier)), - headers=self.headers, ) try: response.raise_for_status() @@ -404,7 +415,7 @@ def rename_table(self, from_identifier: Union[str, Identifier], to_identifier: U "source": self._split_identifier_for_json(from_identifier), "destination": self._split_identifier_for_json(to_identifier), } - response = requests.post(self.url(Endpoints.rename_table), json=payload, headers=self.headers) + response = self.session.post(self.url(Endpoints.rename_table), json=payload) try: response.raise_for_status() except HTTPError as exc: @@ -413,7 +424,7 @@ def rename_table(self, from_identifier: Union[str, Identifier], to_identifier: U def create_namespace(self, namespace: Union[str, Identifier], properties: Properties = EMPTY_DICT) -> None: namespace_tuple = self._check_valid_namespace_identifier(namespace) payload = {"namespace": namespace_tuple, "properties": properties} - response = requests.post(self.url(Endpoints.create_namespace), json=payload, headers=self.headers) + response = self.session.post(self.url(Endpoints.create_namespace), json=payload) try: response.raise_for_status() except HTTPError as exc: @@ -422,7 +433,7 @@ def create_namespace(self, namespace: Union[str, Identifier], properties: Proper def drop_namespace(self, namespace: Union[str, Identifier]) -> None: namespace_tuple = self._check_valid_namespace_identifier(namespace) namespace = NAMESPACE_SEPARATOR.join(namespace_tuple) - response = requests.delete(self.url(Endpoints.drop_namespace, namespace=namespace), headers=self.headers) + response = self.session.delete(self.url(Endpoints.drop_namespace, namespace=namespace)) try: response.raise_for_status() except HTTPError as exc: @@ -430,13 +441,12 @@ def drop_namespace(self, namespace: Union[str, Identifier]) -> None: def list_namespaces(self, namespace: Union[str, Identifier] = ()) -> List[Identifier]: namespace_tuple = self.identifier_to_tuple(namespace) - response = requests.get( + response = self.session.get( self.url( f"{Endpoints.list_namespaces}?parent={NAMESPACE_SEPARATOR.join(namespace_tuple)}" if namespace_tuple else Endpoints.list_namespaces ), - headers=self.headers, ) try: response.raise_for_status() @@ -449,7 +459,7 @@ def list_namespaces(self, namespace: Union[str, Identifier] = ()) -> List[Identi def load_namespace_properties(self, namespace: Union[str, Identifier]) -> Properties: namespace_tuple = self._check_valid_namespace_identifier(namespace) namespace = NAMESPACE_SEPARATOR.join(namespace_tuple) - response = requests.get(self.url(Endpoints.load_namespace_metadata, namespace=namespace), headers=self.headers) + response = self.session.get(self.url(Endpoints.load_namespace_metadata, namespace=namespace)) try: response.raise_for_status() except HTTPError as exc: @@ -463,7 +473,7 @@ def update_namespace_properties( namespace_tuple = self._check_valid_namespace_identifier(namespace) namespace = NAMESPACE_SEPARATOR.join(namespace_tuple) payload = {"removals": list(removals or []), "updates": updates} - response = requests.post(self.url(Endpoints.update_properties, namespace=namespace), json=payload, headers=self.headers) + response = self.session.post(self.url(Endpoints.update_properties, namespace=namespace), json=payload) try: response.raise_for_status() except HTTPError as exc: diff --git a/tests/catalog/test_rest.py b/tests/catalog/test_rest.py index e91d8e3674..8779f3b947 100644 --- a/tests/catalog/test_rest.py +++ b/tests/catalog/test_rest.py @@ -20,6 +20,7 @@ import pytest from requests_mock import Mocker +import pyiceberg from pyiceberg.catalog import PropertiesUpdateSummary, Table from pyiceberg.catalog.rest import RestCatalog from pyiceberg.exceptions import ( @@ -46,6 +47,15 @@ TEST_URI = "https://iceberg-test-catalog/" TEST_CREDENTIALS = "client:secret" TEST_TOKEN = "some_jwt_token" +TEST_HEADERS = { + "Content-type": "application/json", + "X-Client-Version": "0.14.1", + "User-Agent": f"PyIceberg/{pyiceberg.__version__}", + "Authorization": f"Bearer {TEST_TOKEN}", +} +OAUTH_TEST_HEADERS = { + "Content-type": "application/x-www-form-urlencoded", +} @pytest.fixture @@ -77,8 +87,11 @@ def test_token_200(rest_mock: Mocker): "issued_token_type": "urn:ietf:params:oauth:token-type:access_token", }, status_code=200, + request_headers=OAUTH_TEST_HEADERS, + ) + assert ( + RestCatalog("rest", uri=TEST_URI, credential=TEST_CREDENTIALS).session.headers["Authorization"] == f"Bearer {TEST_TOKEN}" ) - assert RestCatalog("rest", uri=TEST_URI, credential=TEST_CREDENTIALS).properties["token"] == TEST_TOKEN def test_token_400(rest_mock: Mocker): @@ -86,6 +99,7 @@ def test_token_400(rest_mock: Mocker): f"{TEST_URI}v1/oauth/tokens", json={"error": "invalid_client", "error_description": "Credentials for key invalid_key do not match"}, status_code=400, + request_headers=OAUTH_TEST_HEADERS, ) with pytest.raises(OAuthError) as e: @@ -99,6 +113,7 @@ def test_token_401(rest_mock: Mocker): f"{TEST_URI}v1/oauth/tokens", json={"error": "invalid_client", "error_description": "Unknown or invalid client"}, status_code=401, + request_headers=OAUTH_TEST_HEADERS, ) with pytest.raises(OAuthError) as e: @@ -112,6 +127,7 @@ def test_list_tables_200(rest_mock: Mocker): f"{TEST_URI}v1/namespaces/{namespace}/tables", json={"identifiers": [{"namespace": ["examples"], "name": "fooshare"}]}, status_code=200, + request_headers=TEST_HEADERS, ) assert RestCatalog("rest", uri=TEST_URI, token=TEST_TOKEN).list_tables(namespace) == [("examples", "fooshare")] @@ -129,6 +145,7 @@ def test_list_tables_404(rest_mock: Mocker): } }, status_code=404, + request_headers=TEST_HEADERS, ) with pytest.raises(NoSuchNamespaceError) as e: RestCatalog("rest", uri=TEST_URI, token=TEST_TOKEN).list_tables(namespace) @@ -140,6 +157,7 @@ def test_list_namespaces_200(rest_mock: Mocker): f"{TEST_URI}v1/namespaces", json={"namespaces": [["default"], ["examples"], ["fokko"], ["system"]]}, status_code=200, + request_headers=TEST_HEADERS, ) assert RestCatalog("rest", uri=TEST_URI, token=TEST_TOKEN).list_namespaces() == [ ("default",), @@ -154,6 +172,7 @@ def test_list_namespace_with_parent_200(rest_mock: Mocker): f"{TEST_URI}v1/namespaces?parent=accounting", json={"namespaces": [["tax"]]}, status_code=200, + request_headers=TEST_HEADERS, ) assert RestCatalog("rest", uri=TEST_URI, token=TEST_TOKEN).list_namespaces(("accounting",)) == [ ("accounting", "tax"), @@ -166,6 +185,7 @@ def test_create_namespace_200(rest_mock: Mocker): f"{TEST_URI}v1/namespaces", json={"namespace": [namespace], "properties": {}}, status_code=200, + request_headers=TEST_HEADERS, ) RestCatalog("rest", uri=TEST_URI, token=TEST_TOKEN).create_namespace(namespace) @@ -182,6 +202,7 @@ def test_create_namespace_409(rest_mock: Mocker): } }, status_code=409, + request_headers=TEST_HEADERS, ) with pytest.raises(NamespaceAlreadyExistsError) as e: RestCatalog("rest", uri=TEST_URI, token=TEST_TOKEN).create_namespace(namespace) @@ -200,6 +221,7 @@ def test_drop_namespace_404(rest_mock: Mocker): } }, status_code=404, + request_headers=TEST_HEADERS, ) with pytest.raises(NoSuchNamespaceError) as e: RestCatalog("rest", uri=TEST_URI, token=TEST_TOKEN).drop_namespace(namespace) @@ -212,6 +234,7 @@ def test_load_namespace_properties_200(rest_mock: Mocker): f"{TEST_URI}v1/namespaces/{namespace}", json={"namespace": ["fokko"], "properties": {"prop": "yes"}}, status_code=204, + request_headers=TEST_HEADERS, ) assert RestCatalog("rest", uri=TEST_URI, token=TEST_TOKEN).load_namespace_properties(namespace) == {"prop": "yes"} @@ -228,6 +251,7 @@ def test_load_namespace_properties_404(rest_mock: Mocker): } }, status_code=404, + request_headers=TEST_HEADERS, ) with pytest.raises(NoSuchNamespaceError) as e: RestCatalog("rest", uri=TEST_URI, token=TEST_TOKEN).load_namespace_properties(namespace) @@ -239,6 +263,7 @@ def test_update_namespace_properties_200(rest_mock: Mocker): f"{TEST_URI}v1/namespaces/fokko/properties", json={"removed": [], "updated": ["prop"], "missing": ["abc"]}, status_code=200, + request_headers=TEST_HEADERS, ) response = RestCatalog("rest", uri=TEST_URI, token=TEST_TOKEN).update_namespace_properties( ("fokko",), {"abc"}, {"prop": "yes"} @@ -258,6 +283,7 @@ def test_update_namespace_properties_404(rest_mock: Mocker): } }, status_code=404, + request_headers=TEST_HEADERS, ) with pytest.raises(NoSuchNamespaceError) as e: RestCatalog("rest", uri=TEST_URI, token=TEST_TOKEN).update_namespace_properties(("fokko",), {"abc"}, {"prop": "yes"}) @@ -336,6 +362,7 @@ def test_load_table_200(rest_mock: Mocker): "config": {"client.factory": "io.tabular.iceberg.catalog.TabularAwsClientFactory", "region": "us-west-2"}, }, status_code=200, + request_headers=TEST_HEADERS, ) actual = RestCatalog("rest", uri=TEST_URI, token=TEST_TOKEN).load_table(("fokko", "table")) expected = Table( @@ -428,6 +455,7 @@ def test_load_table_404(rest_mock: Mocker): } }, status_code=404, + request_headers=TEST_HEADERS, ) with pytest.raises(NoSuchTableError) as e: @@ -446,6 +474,7 @@ def test_drop_table_404(rest_mock: Mocker): } }, status_code=404, + request_headers=TEST_HEADERS, ) with pytest.raises(NoSuchTableError) as e: @@ -510,6 +539,7 @@ def test_create_table_200(rest_mock: Mocker, table_schema_simple: Schema): }, }, status_code=200, + request_headers=TEST_HEADERS, ) table = RestCatalog("rest", uri=TEST_URI, token=TEST_TOKEN).create_table( identifier=("fokko", "fokko2"), @@ -578,6 +608,7 @@ def test_create_table_409(rest_mock, table_schema_simple: Schema): } }, status_code=409, + request_headers=TEST_HEADERS, ) with pytest.raises(TableAlreadyExistsError) as e: @@ -600,6 +631,7 @@ def test_delete_namespace_204(rest_mock: Mocker): f"{TEST_URI}v1/namespaces/{namespace}", json={}, status_code=204, + request_headers=TEST_HEADERS, ) RestCatalog("rest", uri=TEST_URI, token=TEST_TOKEN).drop_namespace(namespace) @@ -609,6 +641,7 @@ def test_delete_table_204(rest_mock: Mocker): f"{TEST_URI}v1/namespaces/example/tables/fokko", json={}, status_code=204, + request_headers=TEST_HEADERS, ) RestCatalog("rest", uri=TEST_URI, token=TEST_TOKEN).drop_table(("example", "fokko")) @@ -624,6 +657,7 @@ def test_delete_table_404(rest_mock: Mocker): } }, status_code=404, + request_headers=TEST_HEADERS, ) with pytest.raises(NoSuchTableError) as e: RestCatalog("rest", uri=TEST_URI, token=TEST_TOKEN).drop_table(("example", "fokko")) @@ -688,3 +722,36 @@ def test_update_namespace_properties_invalid_namespace(rest_mock: Mocker): # Missing namespace RestCatalog("rest", uri=TEST_URI, token=TEST_TOKEN).update_namespace_properties(()) assert "Empty namespace identifier" in str(e.value) + + +def test_request_session_with_ssl_ca_bundle(): + # Given + catalog_properties = { + "uri": TEST_URI, + "token": TEST_TOKEN, + "ssl": { + "cabundle": "path_to_ca_bundle", + }, + } + with pytest.raises(OSError) as e: + # Missing namespace + RestCatalog("rest", **catalog_properties) + assert "Could not find a suitable TLS CA certificate bundle, invalid path: path_to_ca_bundle" in str(e.value) + + +def test_request_session_with_ssl_client_cert(): + # Given + catalog_properties = { + "uri": TEST_URI, + "token": TEST_TOKEN, + "ssl": { + "client": { + "cert": "path_to_client_cert", + "key": "path_to_client_key", + } + }, + } + with pytest.raises(OSError) as e: + # Missing namespace + RestCatalog("rest", **catalog_properties) + assert "Could not find the TLS certificate file, invalid path: path_to_client_cert" in str(e.value) From 8b3d48e073ee9cf37dc3dfd61c049aa2edc32b06 Mon Sep 17 00:00:00 2001 From: Fokko Driesprong Date: Fri, 21 Oct 2022 02:10:36 +0200 Subject: [PATCH 242/642] Python: Split expressions base (#5987) I also played around with a bound.py, but it is so intertwined with the unbound, that I think it makes more sense to have it into a single file. --- pyiceberg/expressions/__init__.py | 513 +++++++ .../expressions/{base.py => visitors.py} | 548 +------ tests/expressions/test_expressions.py | 746 ++++++++++ ...t_expressions_base.py => test_visitors.py} | 1276 +++++------------ tests/test_schema.py | 2 +- 5 files changed, 1633 insertions(+), 1452 deletions(-) rename pyiceberg/expressions/{base.py => visitors.py} (58%) create mode 100644 tests/expressions/test_expressions.py rename tests/expressions/{test_expressions_base.py => test_visitors.py} (55%) diff --git a/pyiceberg/expressions/__init__.py b/pyiceberg/expressions/__init__.py index 13a83393a9..39d4884a51 100644 --- a/pyiceberg/expressions/__init__.py +++ b/pyiceberg/expressions/__init__.py @@ -14,3 +14,516 @@ # KIND, either express or implied. See the License for the # specific language governing permissions and limitations # under the License. + +from __future__ import annotations + +from abc import ABC, abstractmethod +from dataclasses import dataclass +from functools import reduce +from typing import ClassVar, Generic, TypeVar + +from pyiceberg.expressions.literals import Literal +from pyiceberg.files import StructProtocol +from pyiceberg.schema import Accessor, Schema +from pyiceberg.types import DoubleType, FloatType, NestedField +from pyiceberg.utils.singleton import Singleton + +T = TypeVar("T") +B = TypeVar("B") + + +class BooleanExpression(ABC): + """An expression that evaluates to a boolean""" + + @abstractmethod + def __invert__(self) -> BooleanExpression: + """Transform the Expression into its negated version.""" + + +class Term(Generic[T], ABC): + """A simple expression that evaluates to a value""" + + +class Bound(ABC): + """Represents a bound value expression""" + + +class Unbound(Generic[B], ABC): + """Represents an unbound value expression""" + + @abstractmethod + def bind(self, schema: Schema, case_sensitive: bool = True) -> B: + ... # pragma: no cover + + +class BoundTerm(Term[T], Bound, ABC): + """Represents a bound term""" + + @abstractmethod + def ref(self) -> BoundReference[T]: + """Returns the bound reference""" + + @abstractmethod + def eval(self, struct: StructProtocol) -> T: # pylint: disable=W0613 + """Returns the value at the referenced field's position in an object that abides by the StructProtocol""" + + +@dataclass(frozen=True) +class BoundReference(BoundTerm[T]): + """A reference bound to a field in a schema + + Args: + field (NestedField): A referenced field in an Iceberg schema + accessor (Accessor): An Accessor object to access the value at the field's position + """ + + field: NestedField + accessor: Accessor + + def eval(self, struct: StructProtocol) -> T: + """Returns the value at the referenced field's position in an object that abides by the StructProtocol + + Args: + struct (StructProtocol): A row object that abides by the StructProtocol and returns values given a position + Returns: + Any: The value at the referenced field's position in `struct` + """ + return self.accessor.get(struct) + + def ref(self) -> BoundReference[T]: + return self + + +class UnboundTerm(Term[T], Unbound[BoundTerm[T]], ABC): + """Represents an unbound term.""" + + +@dataclass(frozen=True) +class Reference(UnboundTerm[T]): + """A reference not yet bound to a field in a schema + + Args: + name (str): The name of the field + + Note: + An unbound reference is sometimes referred to as a "named" reference + """ + + name: str + + def bind(self, schema: Schema, case_sensitive: bool = True) -> BoundReference[T]: + """Bind the reference to an Iceberg schema + + Args: + schema (Schema): An Iceberg schema + case_sensitive (bool): Whether to consider case when binding the reference to the field + + Raises: + ValueError: If an empty name is provided + + Returns: + BoundReference: A reference bound to the specific field in the Iceberg schema + """ + field = schema.find_field(name_or_id=self.name, case_sensitive=case_sensitive) + accessor = schema.accessor_for_field(field.field_id) + return BoundReference(field=field, accessor=accessor) + + +@dataclass(frozen=True, init=False) +class And(BooleanExpression): + """AND operation expression - logical conjunction""" + + left: BooleanExpression + right: BooleanExpression + + def __new__(cls, left: BooleanExpression, right: BooleanExpression, *rest: BooleanExpression): + if rest: + return reduce(And, (left, right, *rest)) + if left is AlwaysFalse() or right is AlwaysFalse(): + return AlwaysFalse() + elif left is AlwaysTrue(): + return right + elif right is AlwaysTrue(): + return left + else: + result = super().__new__(cls) + object.__setattr__(result, "left", left) + object.__setattr__(result, "right", right) + return result + + def __invert__(self) -> Or: + return Or(~self.left, ~self.right) + + +@dataclass(frozen=True, init=False) +class Or(BooleanExpression): + """OR operation expression - logical disjunction""" + + left: BooleanExpression + right: BooleanExpression + + def __new__(cls, left: BooleanExpression, right: BooleanExpression, *rest: BooleanExpression): + if rest: + return reduce(Or, (left, right, *rest)) + if left is AlwaysTrue() or right is AlwaysTrue(): + return AlwaysTrue() + elif left is AlwaysFalse(): + return right + elif right is AlwaysFalse(): + return left + else: + result = super().__new__(cls) + object.__setattr__(result, "left", left) + object.__setattr__(result, "right", right) + return result + + def __invert__(self) -> And: + return And(~self.left, ~self.right) + + +@dataclass(frozen=True, init=False) +class Not(BooleanExpression): + """NOT operation expression - logical negation""" + + child: BooleanExpression + + def __new__(cls, child: BooleanExpression): + if child is AlwaysTrue(): + return AlwaysFalse() + elif child is AlwaysFalse(): + return AlwaysTrue() + elif isinstance(child, Not): + return child.child + result = super().__new__(cls) + object.__setattr__(result, "child", child) + return result + + def __invert__(self) -> BooleanExpression: + return self.child + + +@dataclass(frozen=True) +class AlwaysTrue(BooleanExpression, Singleton): + """TRUE expression""" + + def __invert__(self) -> AlwaysFalse: + return AlwaysFalse() + + +@dataclass(frozen=True) +class AlwaysFalse(BooleanExpression, Singleton): + """FALSE expression""" + + def __invert__(self) -> AlwaysTrue: + return AlwaysTrue() + + +@dataclass(frozen=True) +class BoundPredicate(Generic[T], Bound, BooleanExpression): + term: BoundTerm[T] + + def __invert__(self) -> BoundPredicate[T]: + """Inverts the predicate""" + raise NotImplementedError + + +@dataclass(frozen=True) +class UnboundPredicate(Generic[T], Unbound[BooleanExpression], BooleanExpression): + as_bound: ClassVar[type] + term: UnboundTerm[T] + + def __invert__(self) -> UnboundPredicate[T]: + """Inverts the predicate""" + raise NotImplementedError + + def bind(self, schema: Schema, case_sensitive: bool = True) -> BooleanExpression: + """Binds the predicate to a schema""" + raise NotImplementedError + + +@dataclass(frozen=True) +class UnaryPredicate(UnboundPredicate[T]): + def bind(self, schema: Schema, case_sensitive: bool = True) -> BooleanExpression: + bound_term = self.term.bind(schema, case_sensitive) + return self.as_bound(bound_term) + + def __invert__(self) -> UnaryPredicate[T]: + """Inverts the unary predicate""" + raise NotImplementedError + + +@dataclass(frozen=True) +class BoundUnaryPredicate(BoundPredicate[T]): + def __invert__(self) -> BoundUnaryPredicate[T]: + """Inverts the unary predicate""" + raise NotImplementedError + + +@dataclass(frozen=True) +class BoundIsNull(BoundUnaryPredicate[T]): + def __new__(cls, term: BoundTerm[T]): # pylint: disable=W0221 + if term.ref().field.required: + return AlwaysFalse() + return super().__new__(cls) + + def __invert__(self) -> BoundNotNull[T]: + return BoundNotNull(self.term) + + +@dataclass(frozen=True) +class BoundNotNull(BoundUnaryPredicate[T]): + def __new__(cls, term: BoundTerm[T]): # pylint: disable=W0221 + if term.ref().field.required: + return AlwaysTrue() + return super().__new__(cls) + + def __invert__(self) -> BoundIsNull: + return BoundIsNull(self.term) + + +@dataclass(frozen=True) +class IsNull(UnaryPredicate[T]): + as_bound = BoundIsNull + + def __invert__(self) -> NotNull[T]: + return NotNull(self.term) + + +@dataclass(frozen=True) +class NotNull(UnaryPredicate[T]): + as_bound = BoundNotNull + + def __invert__(self) -> IsNull[T]: + return IsNull(self.term) + + +@dataclass(frozen=True) +class BoundIsNaN(BoundUnaryPredicate[T]): + def __new__(cls, term: BoundTerm[T]): # pylint: disable=W0221 + bound_type = term.ref().field.field_type + if type(bound_type) in {FloatType, DoubleType}: + return super().__new__(cls) + return AlwaysFalse() + + def __invert__(self) -> BoundNotNaN[T]: + return BoundNotNaN(self.term) + + +@dataclass(frozen=True) +class BoundNotNaN(BoundUnaryPredicate[T]): + def __new__(cls, term: BoundTerm[T]): # pylint: disable=W0221 + bound_type = term.ref().field.field_type + if type(bound_type) in {FloatType, DoubleType}: + return super().__new__(cls) + return AlwaysTrue() + + def __invert__(self) -> BoundIsNaN[T]: + return BoundIsNaN(self.term) + + +@dataclass(frozen=True) +class IsNaN(UnaryPredicate[T]): + as_bound = BoundIsNaN + + def __invert__(self) -> NotNaN[T]: + return NotNaN(self.term) + + +@dataclass(frozen=True) +class NotNaN(UnaryPredicate[T]): + as_bound = BoundNotNaN + + def __invert__(self) -> IsNaN[T]: + return IsNaN(self.term) + + +@dataclass(frozen=True) +class SetPredicate(UnboundPredicate[T]): + literals: tuple[Literal[T], ...] + + def __invert__(self) -> SetPredicate[T]: + """Inverted expression of the SetPredicate""" + raise NotImplementedError + + def bind(self, schema: Schema, case_sensitive: bool = True) -> BooleanExpression: + bound_term = self.term.bind(schema, case_sensitive) + return self.as_bound(bound_term, {lit.to(bound_term.ref().field.field_type) for lit in self.literals}) + + +@dataclass(frozen=True) +class BoundSetPredicate(BoundPredicate[T]): + literals: set[Literal[T]] + + def __invert__(self) -> BoundSetPredicate[T]: + """Inverted expression of the SetPredicate""" + raise NotImplementedError + + +@dataclass(frozen=True) +class BoundIn(BoundSetPredicate[T]): + def __new__(cls, term: BoundTerm[T], literals: set[Literal[T]]): # pylint: disable=W0221 + count = len(literals) + if count == 0: + return AlwaysFalse() + elif count == 1: + return BoundEqualTo(term, next(iter(literals))) + else: + return super().__new__(cls) + + def __invert__(self) -> BoundNotIn[T]: + return BoundNotIn(self.term, self.literals) + + +@dataclass(frozen=True) +class BoundNotIn(BoundSetPredicate[T]): + def __new__(cls, term: BoundTerm[T], literals: set[Literal[T]]): # pylint: disable=W0221 + count = len(literals) + if count == 0: + return AlwaysTrue() + elif count == 1: + return BoundNotEqualTo(term, next(iter(literals))) + else: + return super().__new__(cls) + + def __invert__(self) -> BoundIn[T]: + return BoundIn(self.term, self.literals) + + +@dataclass(frozen=True) +class In(SetPredicate[T]): + as_bound = BoundIn + + def __new__(cls, term: UnboundTerm[T], literals: tuple[Literal[T], ...]): # pylint: disable=W0221 + count = len(literals) + if count == 0: + return AlwaysFalse() + elif count == 1: + return EqualTo(term, literals[0]) + else: + return super().__new__(cls) + + def __invert__(self) -> NotIn[T]: + return NotIn(self.term, self.literals) + + +@dataclass(frozen=True) +class NotIn(SetPredicate[T]): + as_bound = BoundNotIn + + def __new__(cls, term: UnboundTerm[T], literals: tuple[Literal[T], ...]): # pylint: disable=W0221 + count = len(literals) + if count == 0: + return AlwaysTrue() + elif count == 1: + return NotEqualTo(term, literals[0]) + else: + return super().__new__(cls) + + def __invert__(self) -> In[T]: + return In(self.term, self.literals) + + +@dataclass(frozen=True) +class LiteralPredicate(UnboundPredicate[T]): + literal: Literal[T] + + def bind(self, schema: Schema, case_sensitive: bool = True) -> BooleanExpression: + bound_term = self.term.bind(schema, case_sensitive) + return self.as_bound(bound_term, self.literal.to(bound_term.ref().field.field_type)) + + def __invert__(self) -> LiteralPredicate[T]: + """Inverts the literal predicate""" + raise NotImplementedError + + +@dataclass(frozen=True) +class BoundLiteralPredicate(BoundPredicate[T]): + literal: Literal[T] + + def __invert__(self) -> BoundLiteralPredicate[T]: + """Inverts the bound literal predicate""" + raise NotImplementedError + + +@dataclass(frozen=True) +class BoundEqualTo(BoundLiteralPredicate[T]): + def __invert__(self) -> BoundNotEqualTo[T]: + return BoundNotEqualTo(self.term, self.literal) + + +@dataclass(frozen=True) +class BoundNotEqualTo(BoundLiteralPredicate[T]): + def __invert__(self) -> BoundEqualTo[T]: + return BoundEqualTo(self.term, self.literal) + + +@dataclass(frozen=True) +class BoundGreaterThanOrEqual(BoundLiteralPredicate[T]): + def __invert__(self) -> BoundLessThan[T]: + return BoundLessThan(self.term, self.literal) + + +@dataclass(frozen=True) +class BoundGreaterThan(BoundLiteralPredicate[T]): + def __invert__(self) -> BoundLessThanOrEqual[T]: + return BoundLessThanOrEqual(self.term, self.literal) + + +@dataclass(frozen=True) +class BoundLessThan(BoundLiteralPredicate[T]): + def __invert__(self) -> BoundGreaterThanOrEqual[T]: + return BoundGreaterThanOrEqual(self.term, self.literal) + + +@dataclass(frozen=True) +class BoundLessThanOrEqual(BoundLiteralPredicate[T]): + def __invert__(self) -> BoundGreaterThan[T]: + return BoundGreaterThan(self.term, self.literal) + + +@dataclass(frozen=True) +class EqualTo(LiteralPredicate[T]): + as_bound = BoundEqualTo + + def __invert__(self) -> NotEqualTo[T]: + return NotEqualTo(self.term, self.literal) + + +@dataclass(frozen=True) +class NotEqualTo(LiteralPredicate[T]): + as_bound = BoundNotEqualTo + + def __invert__(self) -> EqualTo[T]: + return EqualTo(self.term, self.literal) + + +@dataclass(frozen=True) +class LessThan(LiteralPredicate[T]): + as_bound = BoundLessThan + + def __invert__(self) -> GreaterThanOrEqual[T]: + return GreaterThanOrEqual(self.term, self.literal) + + +@dataclass(frozen=True) +class GreaterThanOrEqual(LiteralPredicate[T]): + as_bound = BoundGreaterThanOrEqual + + def __invert__(self) -> LessThan[T]: + return LessThan(self.term, self.literal) + + +@dataclass(frozen=True) +class GreaterThan(LiteralPredicate[T]): + as_bound = BoundGreaterThan + + def __invert__(self) -> LessThanOrEqual[T]: + return LessThanOrEqual(self.term, self.literal) + + +@dataclass(frozen=True) +class LessThanOrEqual(LiteralPredicate[T]): + as_bound = BoundLessThanOrEqual + + def __invert__(self) -> GreaterThan[T]: + return GreaterThan(self.term, self.literal) diff --git a/pyiceberg/expressions/base.py b/pyiceberg/expressions/visitors.py similarity index 58% rename from pyiceberg/expressions/base.py rename to pyiceberg/expressions/visitors.py index aabfb0a226..4853490315 100644 --- a/pyiceberg/expressions/base.py +++ b/pyiceberg/expressions/visitors.py @@ -14,533 +14,51 @@ # KIND, either express or implied. See the License for the # specific language governing permissions and limitations # under the License. -from __future__ import annotations - from abc import ABC, abstractmethod -from dataclasses import dataclass -from functools import reduce, singledispatch +from functools import singledispatch from typing import ( Any, Callable, - ClassVar, Generic, - TypeVar, + List, + Set, ) from pyiceberg.conversions import from_bytes +from pyiceberg.expressions import ( + AlwaysFalse, + AlwaysTrue, + And, + BooleanExpression, + BoundEqualTo, + BoundGreaterThan, + BoundGreaterThanOrEqual, + BoundIn, + BoundIsNaN, + BoundIsNull, + BoundLessThan, + BoundLessThanOrEqual, + BoundNotEqualTo, + BoundNotIn, + BoundNotNaN, + BoundNotNull, + BoundPredicate, + BoundTerm, + Not, + Or, + T, + UnboundPredicate, +) from pyiceberg.expressions.literals import Literal -from pyiceberg.files import StructProtocol from pyiceberg.manifest import ManifestFile, PartitionFieldSummary -from pyiceberg.schema import Accessor, Schema +from pyiceberg.schema import Schema from pyiceberg.table import PartitionSpec from pyiceberg.types import ( DoubleType, FloatType, IcebergType, - NestedField, PrimitiveType, ) -from pyiceberg.utils.singleton import Singleton - -T = TypeVar("T") -B = TypeVar("B") - - -class BooleanExpression(ABC): - """An expression that evaluates to a boolean""" - - @abstractmethod - def __invert__(self) -> BooleanExpression: - """Transform the Expression into its negated version.""" - - -class Term(Generic[T], ABC): - """A simple expression that evaluates to a value""" - - -class Bound(ABC): - """Represents a bound value expression""" - - -class Unbound(Generic[B], ABC): - """Represents an unbound value expression""" - - @abstractmethod - def bind(self, schema: Schema, case_sensitive: bool = True) -> B: - ... # pragma: no cover - - -class BoundTerm(Term[T], Bound, ABC): - """Represents a bound term""" - - @abstractmethod - def ref(self) -> BoundReference[T]: - """Returns the bound reference""" - - @abstractmethod - def eval(self, struct: StructProtocol) -> T: # pylint: disable=W0613 - """Returns the value at the referenced field's position in an object that abides by the StructProtocol""" - - -@dataclass(frozen=True) -class BoundReference(BoundTerm[T]): - """A reference bound to a field in a schema - - Args: - field (NestedField): A referenced field in an Iceberg schema - accessor (Accessor): An Accessor object to access the value at the field's position - """ - - field: NestedField - accessor: Accessor - - def eval(self, struct: StructProtocol) -> T: - """Returns the value at the referenced field's position in an object that abides by the StructProtocol - - Args: - struct (StructProtocol): A row object that abides by the StructProtocol and returns values given a position - Returns: - Any: The value at the referenced field's position in `struct` - """ - return self.accessor.get(struct) - - def ref(self) -> BoundReference[T]: - return self - - -class UnboundTerm(Term[T], Unbound[BoundTerm[T]], ABC): - """Represents an unbound term.""" - - -@dataclass(frozen=True) -class Reference(UnboundTerm[T]): - """A reference not yet bound to a field in a schema - - Args: - name (str): The name of the field - - Note: - An unbound reference is sometimes referred to as a "named" reference - """ - - name: str - - def bind(self, schema: Schema, case_sensitive: bool = True) -> BoundReference[T]: - """Bind the reference to an Iceberg schema - - Args: - schema (Schema): An Iceberg schema - case_sensitive (bool): Whether to consider case when binding the reference to the field - - Raises: - ValueError: If an empty name is provided - - Returns: - BoundReference: A reference bound to the specific field in the Iceberg schema - """ - field = schema.find_field(name_or_id=self.name, case_sensitive=case_sensitive) - accessor = schema.accessor_for_field(field.field_id) - return BoundReference(field=field, accessor=accessor) - - -@dataclass(frozen=True, init=False) -class And(BooleanExpression): - """AND operation expression - logical conjunction""" - - left: BooleanExpression - right: BooleanExpression - - def __new__(cls, left: BooleanExpression, right: BooleanExpression, *rest: BooleanExpression): - if rest: - return reduce(And, (left, right, *rest)) - if left is AlwaysFalse() or right is AlwaysFalse(): - return AlwaysFalse() - elif left is AlwaysTrue(): - return right - elif right is AlwaysTrue(): - return left - else: - result = super().__new__(cls) - object.__setattr__(result, "left", left) - object.__setattr__(result, "right", right) - return result - - def __invert__(self) -> Or: - return Or(~self.left, ~self.right) - - -@dataclass(frozen=True, init=False) -class Or(BooleanExpression): - """OR operation expression - logical disjunction""" - - left: BooleanExpression - right: BooleanExpression - - def __new__(cls, left: BooleanExpression, right: BooleanExpression, *rest: BooleanExpression): - if rest: - return reduce(Or, (left, right, *rest)) - if left is AlwaysTrue() or right is AlwaysTrue(): - return AlwaysTrue() - elif left is AlwaysFalse(): - return right - elif right is AlwaysFalse(): - return left - else: - result = super().__new__(cls) - object.__setattr__(result, "left", left) - object.__setattr__(result, "right", right) - return result - - def __invert__(self) -> And: - return And(~self.left, ~self.right) - - -@dataclass(frozen=True, init=False) -class Not(BooleanExpression): - """NOT operation expression - logical negation""" - - child: BooleanExpression - - def __new__(cls, child: BooleanExpression): - if child is AlwaysTrue(): - return AlwaysFalse() - elif child is AlwaysFalse(): - return AlwaysTrue() - elif isinstance(child, Not): - return child.child - result = super().__new__(cls) - object.__setattr__(result, "child", child) - return result - - def __invert__(self) -> BooleanExpression: - return self.child - - -@dataclass(frozen=True) -class AlwaysTrue(BooleanExpression, Singleton): - """TRUE expression""" - - def __invert__(self) -> AlwaysFalse: - return AlwaysFalse() - - -@dataclass(frozen=True) -class AlwaysFalse(BooleanExpression, Singleton): - """FALSE expression""" - - def __invert__(self) -> AlwaysTrue: - return AlwaysTrue() - - -@dataclass(frozen=True) -class BoundPredicate(Generic[T], Bound, BooleanExpression): - term: BoundTerm[T] - - def __invert__(self) -> BoundPredicate[T]: - """Inverts the predicate""" - raise NotImplementedError - - -@dataclass(frozen=True) -class UnboundPredicate(Generic[T], Unbound[BooleanExpression], BooleanExpression): - as_bound: ClassVar[type] - term: UnboundTerm[T] - - def __invert__(self) -> UnboundPredicate[T]: - """Inverts the predicate""" - raise NotImplementedError - - def bind(self, schema: Schema, case_sensitive: bool = True) -> BooleanExpression: - """Binds the predicate to a schema""" - raise NotImplementedError - - -@dataclass(frozen=True) -class UnaryPredicate(UnboundPredicate[T]): - def bind(self, schema: Schema, case_sensitive: bool = True) -> BooleanExpression: - bound_term = self.term.bind(schema, case_sensitive) - return self.as_bound(bound_term) - - def __invert__(self) -> UnaryPredicate[T]: - """Inverts the unary predicate""" - raise NotImplementedError - - -@dataclass(frozen=True) -class BoundUnaryPredicate(BoundPredicate[T]): - def __invert__(self) -> BoundUnaryPredicate[T]: - """Inverts the unary predicate""" - raise NotImplementedError - - -@dataclass(frozen=True) -class BoundIsNull(BoundUnaryPredicate[T]): - def __new__(cls, term: BoundTerm[T]): # pylint: disable=W0221 - if term.ref().field.required: - return AlwaysFalse() - return super().__new__(cls) - - def __invert__(self) -> BoundNotNull[T]: - return BoundNotNull(self.term) - - -@dataclass(frozen=True) -class BoundNotNull(BoundUnaryPredicate[T]): - def __new__(cls, term: BoundTerm[T]): # pylint: disable=W0221 - if term.ref().field.required: - return AlwaysTrue() - return super().__new__(cls) - - def __invert__(self) -> BoundIsNull: - return BoundIsNull(self.term) - - -@dataclass(frozen=True) -class IsNull(UnaryPredicate[T]): - as_bound = BoundIsNull - - def __invert__(self) -> NotNull[T]: - return NotNull(self.term) - - -@dataclass(frozen=True) -class NotNull(UnaryPredicate[T]): - as_bound = BoundNotNull - - def __invert__(self) -> IsNull[T]: - return IsNull(self.term) - - -@dataclass(frozen=True) -class BoundIsNaN(BoundUnaryPredicate[T]): - def __new__(cls, term: BoundTerm[T]): # pylint: disable=W0221 - bound_type = term.ref().field.field_type - if type(bound_type) in {FloatType, DoubleType}: - return super().__new__(cls) - return AlwaysFalse() - - def __invert__(self) -> BoundNotNaN[T]: - return BoundNotNaN(self.term) - - -@dataclass(frozen=True) -class BoundNotNaN(BoundUnaryPredicate[T]): - def __new__(cls, term: BoundTerm[T]): # pylint: disable=W0221 - bound_type = term.ref().field.field_type - if type(bound_type) in {FloatType, DoubleType}: - return super().__new__(cls) - return AlwaysTrue() - - def __invert__(self) -> BoundIsNaN[T]: - return BoundIsNaN(self.term) - - -@dataclass(frozen=True) -class IsNaN(UnaryPredicate[T]): - as_bound = BoundIsNaN - - def __invert__(self) -> NotNaN[T]: - return NotNaN(self.term) - - -@dataclass(frozen=True) -class NotNaN(UnaryPredicate[T]): - as_bound = BoundNotNaN - - def __invert__(self) -> IsNaN[T]: - return IsNaN(self.term) - - -@dataclass(frozen=True) -class SetPredicate(UnboundPredicate[T]): - literals: tuple[Literal[T], ...] - - def __invert__(self) -> SetPredicate[T]: - """Inverted expression of the SetPredicate""" - raise NotImplementedError - - def bind(self, schema: Schema, case_sensitive: bool = True) -> BooleanExpression: - bound_term = self.term.bind(schema, case_sensitive) - return self.as_bound(bound_term, {lit.to(bound_term.ref().field.field_type) for lit in self.literals}) - - -@dataclass(frozen=True) -class BoundSetPredicate(BoundPredicate[T]): - literals: set[Literal[T]] - - def __invert__(self) -> BoundSetPredicate[T]: - """Inverted expression of the SetPredicate""" - raise NotImplementedError - - -@dataclass(frozen=True) -class BoundIn(BoundSetPredicate[T]): - def __new__(cls, term: BoundTerm[T], literals: set[Literal[T]]): # pylint: disable=W0221 - count = len(literals) - if count == 0: - return AlwaysFalse() - elif count == 1: - return BoundEqualTo(term, next(iter(literals))) - else: - return super().__new__(cls) - - def __invert__(self) -> BoundNotIn[T]: - return BoundNotIn(self.term, self.literals) - - -@dataclass(frozen=True) -class BoundNotIn(BoundSetPredicate[T]): - def __new__(cls, term: BoundTerm[T], literals: set[Literal[T]]): # pylint: disable=W0221 - count = len(literals) - if count == 0: - return AlwaysTrue() - elif count == 1: - return BoundNotEqualTo(term, next(iter(literals))) - else: - return super().__new__(cls) - - def __invert__(self) -> BoundIn[T]: - return BoundIn(self.term, self.literals) - - -@dataclass(frozen=True) -class In(SetPredicate[T]): - as_bound = BoundIn - - def __new__(cls, term: UnboundTerm[T], literals: tuple[Literal[T], ...]): # pylint: disable=W0221 - count = len(literals) - if count == 0: - return AlwaysFalse() - elif count == 1: - return EqualTo(term, literals[0]) - else: - return super().__new__(cls) - - def __invert__(self) -> NotIn[T]: - return NotIn(self.term, self.literals) - - -@dataclass(frozen=True) -class NotIn(SetPredicate[T]): - as_bound = BoundNotIn - - def __new__(cls, term: UnboundTerm[T], literals: tuple[Literal[T], ...]): # pylint: disable=W0221 - count = len(literals) - if count == 0: - return AlwaysTrue() - elif count == 1: - return NotEqualTo(term, literals[0]) - else: - return super().__new__(cls) - - def __invert__(self) -> In[T]: - return In(self.term, self.literals) - - -@dataclass(frozen=True) -class LiteralPredicate(UnboundPredicate[T]): - literal: Literal[T] - - def bind(self, schema: Schema, case_sensitive: bool = True) -> BooleanExpression: - bound_term = self.term.bind(schema, case_sensitive) - return self.as_bound(bound_term, self.literal.to(bound_term.ref().field.field_type)) - - def __invert__(self) -> LiteralPredicate[T]: - """Inverts the literal predicate""" - raise NotImplementedError - - -@dataclass(frozen=True) -class BoundLiteralPredicate(BoundPredicate[T]): - literal: Literal[T] - - def __invert__(self) -> BoundLiteralPredicate[T]: - """Inverts the bound literal predicate""" - raise NotImplementedError - - -@dataclass(frozen=True) -class BoundEqualTo(BoundLiteralPredicate[T]): - def __invert__(self) -> BoundNotEqualTo[T]: - return BoundNotEqualTo(self.term, self.literal) - - -@dataclass(frozen=True) -class BoundNotEqualTo(BoundLiteralPredicate[T]): - def __invert__(self) -> BoundEqualTo[T]: - return BoundEqualTo(self.term, self.literal) - - -@dataclass(frozen=True) -class BoundGreaterThanOrEqual(BoundLiteralPredicate[T]): - def __invert__(self) -> BoundLessThan[T]: - return BoundLessThan(self.term, self.literal) - - -@dataclass(frozen=True) -class BoundGreaterThan(BoundLiteralPredicate[T]): - def __invert__(self) -> BoundLessThanOrEqual[T]: - return BoundLessThanOrEqual(self.term, self.literal) - - -@dataclass(frozen=True) -class BoundLessThan(BoundLiteralPredicate[T]): - def __invert__(self) -> BoundGreaterThanOrEqual[T]: - return BoundGreaterThanOrEqual(self.term, self.literal) - - -@dataclass(frozen=True) -class BoundLessThanOrEqual(BoundLiteralPredicate[T]): - def __invert__(self) -> BoundGreaterThan[T]: - return BoundGreaterThan(self.term, self.literal) - - -@dataclass(frozen=True) -class EqualTo(LiteralPredicate[T]): - as_bound = BoundEqualTo - - def __invert__(self) -> NotEqualTo[T]: - return NotEqualTo(self.term, self.literal) - - -@dataclass(frozen=True) -class NotEqualTo(LiteralPredicate[T]): - as_bound = BoundNotEqualTo - - def __invert__(self) -> EqualTo[T]: - return EqualTo(self.term, self.literal) - - -@dataclass(frozen=True) -class LessThan(LiteralPredicate[T]): - as_bound = BoundLessThan - - def __invert__(self) -> GreaterThanOrEqual[T]: - return GreaterThanOrEqual(self.term, self.literal) - - -@dataclass(frozen=True) -class GreaterThanOrEqual(LiteralPredicate[T]): - as_bound = BoundGreaterThanOrEqual - - def __invert__(self) -> LessThan[T]: - return LessThan(self.term, self.literal) - - -@dataclass(frozen=True) -class GreaterThan(LiteralPredicate[T]): - as_bound = BoundGreaterThan - - def __invert__(self) -> LessThanOrEqual[T]: - return LessThanOrEqual(self.term, self.literal) - - -@dataclass(frozen=True) -class LessThanOrEqual(LiteralPredicate[T]): - as_bound = BoundLessThanOrEqual - - def __invert__(self) -> GreaterThan[T]: - return GreaterThan(self.term, self.literal) class BooleanExpressionVisitor(Generic[T], ABC): @@ -720,11 +238,11 @@ def visit_bound_predicate(self, predicate: BoundPredicate) -> BooleanExpression: class BoundBooleanExpressionVisitor(BooleanExpressionVisitor[T], ABC): @abstractmethod - def visit_in(self, term: BoundTerm[T], literals: set[Literal[Any]]) -> T: + def visit_in(self, term: BoundTerm[T], literals: Set[Literal[Any]]) -> T: """Visit a bound In predicate""" @abstractmethod - def visit_not_in(self, term: BoundTerm[T], literals: set[Literal[Any]]) -> T: + def visit_not_in(self, term: BoundTerm[T], literals: Set[Literal[Any]]) -> T: """Visit a bound NotIn predicate""" @abstractmethod @@ -911,7 +429,7 @@ def _from_byte_buffer(field_type: IcebergType, val: bytes): class _ManifestEvalVisitor(BoundBooleanExpressionVisitor[bool]): - partition_fields: list[PartitionFieldSummary] + partition_fields: List[PartitionFieldSummary] partition_filter: BooleanExpression def __init__(self, partition_struct_schema: Schema, partition_filter: BooleanExpression, case_sensitive: bool = True): @@ -925,7 +443,7 @@ def eval(self, manifest: ManifestFile) -> bool: # No partition information return ROWS_MIGHT_MATCH - def visit_in(self, term: BoundTerm, literals: set[Literal[Any]]) -> bool: + def visit_in(self, term: BoundTerm, literals: Set[Literal[Any]]) -> bool: pos = term.ref().accessor.position field = self.partition_fields[pos] @@ -947,7 +465,7 @@ def visit_in(self, term: BoundTerm, literals: set[Literal[Any]]) -> bool: return ROWS_MIGHT_MATCH - def visit_not_in(self, term: BoundTerm, literals: set[Literal[Any]]) -> bool: + def visit_not_in(self, term: BoundTerm, literals: Set[Literal[Any]]) -> bool: # because the bounds are not necessarily a min or max value, this cannot be answered using # them. notIn(col, {X, ...}) with (X, Y) doesn't guarantee that X is a value in col. return ROWS_MIGHT_MATCH diff --git a/tests/expressions/test_expressions.py b/tests/expressions/test_expressions.py new file mode 100644 index 0000000000..6ccada73a4 --- /dev/null +++ b/tests/expressions/test_expressions.py @@ -0,0 +1,746 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +import uuid +from decimal import Decimal + +import pytest + +from pyiceberg.expressions import ( + AlwaysFalse, + AlwaysTrue, + And, + BooleanExpression, + BoundEqualTo, + BoundGreaterThan, + BoundGreaterThanOrEqual, + BoundIn, + BoundIsNaN, + BoundIsNull, + BoundLessThan, + BoundLessThanOrEqual, + BoundLiteralPredicate, + BoundNotEqualTo, + BoundNotIn, + BoundNotNaN, + BoundNotNull, + BoundPredicate, + BoundReference, + BoundSetPredicate, + BoundUnaryPredicate, + EqualTo, + GreaterThan, + GreaterThanOrEqual, + In, + IsNaN, + IsNull, + LessThan, + LessThanOrEqual, + LiteralPredicate, + Not, + NotEqualTo, + NotIn, + NotNaN, + NotNull, + Or, + Reference, + SetPredicate, + UnaryPredicate, + UnboundPredicate, +) +from pyiceberg.expressions.literals import StringLiteral, literal +from pyiceberg.expressions.visitors import _from_byte_buffer +from pyiceberg.schema import Accessor, Schema +from pyiceberg.types import ( + DoubleType, + FloatType, + IntegerType, + ListType, + NestedField, + StringType, +) +from tests.expressions.test_visitors import ExpressionA, ExpressionB + + +@pytest.mark.parametrize( + "op, rep", + [ + ( + And(ExpressionA(), ExpressionB()), + "And(left=ExpressionA(), right=ExpressionB())", + ), + ( + Or(ExpressionA(), ExpressionB()), + "Or(left=ExpressionA(), right=ExpressionB())", + ), + (Not(ExpressionA()), "Not(child=ExpressionA())"), + ], +) +def test_reprs(op: BooleanExpression, rep: str): + assert repr(op) == rep + + +def test_isnull_inverse(): + assert ~IsNull(Reference("a")) == NotNull(Reference("a")) + + +def test_isnull_bind(): + schema = Schema(NestedField(2, "a", IntegerType()), schema_id=1) + bound = BoundIsNull(BoundReference(schema.find_field(2), schema.accessor_for_field(2))) + assert IsNull(Reference("a")).bind(schema) == bound + + +def test_invert_is_null_bind(): + schema = Schema(NestedField(2, "a", IntegerType(), required=False), schema_id=1) + assert ~IsNull(Reference("a")).bind(schema) == NotNull(Reference("a")).bind(schema) + + +def test_invert_not_null_bind(): + schema = Schema(NestedField(2, "a", IntegerType(), required=False), schema_id=1) + assert ~NotNull(Reference("a")).bind(schema) == IsNull(Reference("a")).bind(schema) + + +def test_invert_is_nan_bind(): + schema = Schema(NestedField(2, "a", DoubleType(), required=False), schema_id=1) + assert ~IsNaN(Reference("a")).bind(schema) == NotNaN(Reference("a")).bind(schema) + + +def test_invert_not_nan_bind(): + schema = Schema(NestedField(2, "a", DoubleType(), required=False), schema_id=1) + assert ~NotNaN(Reference("a")).bind(schema) == IsNaN(Reference("a")).bind(schema) + + +def test_bind_expr_does_not_exists(): + schema = Schema(NestedField(2, "a", IntegerType()), schema_id=1) + with pytest.raises(ValueError) as exc_info: + IsNull(Reference("b")).bind(schema) + + assert str(exc_info.value) == "Could not find field with name b, case_sensitive=True" + + +def test_bind_does_not_exists(): + schema = Schema(NestedField(2, "a", IntegerType()), schema_id=1) + with pytest.raises(ValueError) as exc_info: + Reference("b").bind(schema) + + assert str(exc_info.value) == "Could not find field with name b, case_sensitive=True" + + +def test_isnull_bind_required(): + schema = Schema(NestedField(2, "a", IntegerType(), required=True), schema_id=1) + assert IsNull(Reference("a")).bind(schema) == AlwaysFalse() + + +def test_notnull_inverse(): + assert ~NotNull(Reference("a")) == IsNull(Reference("a")) + + +def test_notnull_bind(): + schema = Schema(NestedField(2, "a", IntegerType()), schema_id=1) + bound = BoundNotNull(BoundReference(schema.find_field(2), schema.accessor_for_field(2))) + assert NotNull(Reference("a")).bind(schema) == bound + + +def test_notnull_bind_required(): + schema = Schema(NestedField(2, "a", IntegerType(), required=True), schema_id=1) + assert NotNull(Reference("a")).bind(schema) == AlwaysTrue() + + +def test_isnan_inverse(): + assert ~IsNaN(Reference("f")) == NotNaN(Reference("f")) + + +def test_isnan_bind_float(): + schema = Schema(NestedField(2, "f", FloatType()), schema_id=1) + bound = BoundIsNaN(BoundReference(schema.find_field(2), schema.accessor_for_field(2))) + assert IsNaN(Reference("f")).bind(schema) == bound + + +def test_isnan_bind_double(): + schema = Schema(NestedField(2, "d", DoubleType()), schema_id=1) + bound = BoundIsNaN(BoundReference(schema.find_field(2), schema.accessor_for_field(2))) + assert IsNaN(Reference("d")).bind(schema) == bound + + +def test_isnan_bind_nonfloat(): + schema = Schema(NestedField(2, "i", IntegerType()), schema_id=1) + assert IsNaN(Reference("i")).bind(schema) == AlwaysFalse() + + +def test_notnan_inverse(): + assert ~NotNaN(Reference("f")) == IsNaN(Reference("f")) + + +def test_notnan_bind_float(): + schema = Schema(NestedField(2, "f", FloatType()), schema_id=1) + bound = BoundNotNaN(BoundReference(schema.find_field(2), schema.accessor_for_field(2))) + assert NotNaN(Reference("f")).bind(schema) == bound + + +def test_notnan_bind_double(): + schema = Schema(NestedField(2, "d", DoubleType()), schema_id=1) + bound = BoundNotNaN(BoundReference(schema.find_field(2), schema.accessor_for_field(2))) + assert NotNaN(Reference("d")).bind(schema) == bound + + +def test_notnan_bind_nonfloat(): + schema = Schema(NestedField(2, "i", IntegerType()), schema_id=1) + assert NotNaN(Reference("i")).bind(schema) == AlwaysTrue() + + +@pytest.mark.parametrize( + "op, string", + [ + (And(ExpressionA(), ExpressionB()), "And(left=ExpressionA(), right=ExpressionB())"), + (Or(ExpressionA(), ExpressionB()), "Or(left=ExpressionA(), right=ExpressionB())"), + (Not(ExpressionA()), "Not(child=ExpressionA())"), + ], +) +def test_strs(op, string): + assert str(op) == string + + +def test_ref_binding_case_sensitive(table_schema_simple: Schema): + ref = Reference[str]("foo") + bound = BoundReference[str](table_schema_simple.find_field(1), table_schema_simple.accessor_for_field(1)) + assert ref.bind(table_schema_simple, case_sensitive=True) == bound + + +def test_ref_binding_case_sensitive_failure(table_schema_simple: Schema): + ref = Reference[str]("Foo") + with pytest.raises(ValueError): + ref.bind(table_schema_simple, case_sensitive=True) + + +def test_ref_binding_case_insensitive(table_schema_simple: Schema): + ref = Reference[str]("Foo") + bound = BoundReference[str](table_schema_simple.find_field(1), table_schema_simple.accessor_for_field(1)) + assert ref.bind(table_schema_simple, case_sensitive=False) == bound + + +def test_ref_binding_case_insensitive_failure(table_schema_simple: Schema): + ref = Reference[str]("Foot") + with pytest.raises(ValueError): + ref.bind(table_schema_simple, case_sensitive=False) + + +def test_in_to_eq(): + assert In(Reference("x"), (literal(34.56),)) == EqualTo(Reference("x"), literal(34.56)) + + +def test_empty_bind_in(table_schema_simple: Schema): + bound = BoundIn[str](BoundReference(table_schema_simple.find_field(1), table_schema_simple.accessor_for_field(1)), set()) + assert bound == AlwaysFalse() + + +def test_empty_bind_not_in(table_schema_simple: Schema): + bound = BoundNotIn(BoundReference[str](table_schema_simple.find_field(1), table_schema_simple.accessor_for_field(1)), set()) + assert bound == AlwaysTrue() + + +def test_bind_not_in_equal_term(table_schema_simple: Schema): + bound = BoundNotIn( + BoundReference(table_schema_simple.find_field(1), table_schema_simple.accessor_for_field(1)), {literal("hello")} + ) + assert ( + BoundNotEqualTo[str]( + term=BoundReference( + field=NestedField(field_id=1, name="foo", field_type=StringType(), required=False), + accessor=Accessor(position=0, inner=None), + ), + literal=StringLiteral("hello"), + ) + == bound + ) + + +def test_in_empty(): + assert In(Reference("foo"), ()) == AlwaysFalse() + + +def test_not_in_empty(): + assert NotIn(Reference("foo"), ()) == AlwaysTrue() + + +def test_not_in_equal(): + assert NotIn(Reference("foo"), (literal("hello"),)) == NotEqualTo(term=Reference(name="foo"), literal=StringLiteral("hello")) + + +def test_bind_in(table_schema_simple: Schema): + bound = BoundIn( + BoundReference(table_schema_simple.find_field(1), table_schema_simple.accessor_for_field(1)), + {literal("hello"), literal("world")}, + ) + assert In(Reference("foo"), (literal("hello"), literal("world"))).bind(table_schema_simple) == bound + + +def test_bind_in_invert(table_schema_simple: Schema): + bound = BoundIn( + BoundReference(table_schema_simple.find_field(1), table_schema_simple.accessor_for_field(1)), + {literal("hello"), literal("world")}, + ) + assert ~bound == BoundNotIn( + BoundReference(table_schema_simple.find_field(1), table_schema_simple.accessor_for_field(1)), + {literal("hello"), literal("world")}, + ) + + +def test_bind_not_in_invert(table_schema_simple: Schema): + bound = BoundNotIn( + BoundReference(table_schema_simple.find_field(1), table_schema_simple.accessor_for_field(1)), + {literal("hello"), literal("world")}, + ) + assert ~bound == BoundIn( + BoundReference(table_schema_simple.find_field(1), table_schema_simple.accessor_for_field(1)), + {literal("hello"), literal("world")}, + ) + + +def test_bind_dedup(table_schema_simple: Schema): + bound = BoundIn( + BoundReference(table_schema_simple.find_field(1), table_schema_simple.accessor_for_field(1)), + {literal("hello"), literal("world")}, + ) + assert In(Reference("foo"), (literal("hello"), literal("world"), literal("world"))).bind(table_schema_simple) == bound + + +def test_bind_dedup_to_eq(table_schema_simple: Schema): + bound = BoundEqualTo( + BoundReference(table_schema_simple.find_field(1), table_schema_simple.accessor_for_field(1)), literal("hello") + ) + assert In(Reference("foo"), (literal("hello"), literal("hello"))).bind(table_schema_simple) == bound + + +def test_bound_equal_to_invert(table_schema_simple: Schema): + bound = BoundEqualTo( + BoundReference(table_schema_simple.find_field(1), table_schema_simple.accessor_for_field(1)), literal("hello") + ) + assert ~bound == BoundNotEqualTo( + term=BoundReference[str]( + field=NestedField(field_id=1, name="foo", field_type=StringType(), required=False), + accessor=Accessor(position=0, inner=None), + ), + literal=StringLiteral("hello"), + ) + + +def test_bound_not_equal_to_invert(table_schema_simple: Schema): + bound = BoundNotEqualTo( + BoundReference(table_schema_simple.find_field(1), table_schema_simple.accessor_for_field(1)), literal("hello") + ) + assert ~bound == BoundEqualTo( + term=BoundReference[str]( + field=NestedField(field_id=1, name="foo", field_type=StringType(), required=False), + accessor=Accessor(position=0, inner=None), + ), + literal=StringLiteral("hello"), + ) + + +def test_bound_greater_than_or_equal_invert(table_schema_simple: Schema): + bound = BoundGreaterThanOrEqual( + BoundReference(table_schema_simple.find_field(1), table_schema_simple.accessor_for_field(1)), literal("hello") + ) + assert ~bound == BoundLessThan( + term=BoundReference[str]( + field=NestedField(field_id=1, name="foo", field_type=StringType(), required=False), + accessor=Accessor(position=0, inner=None), + ), + literal=StringLiteral("hello"), + ) + + +def test_bound_greater_than_invert(table_schema_simple: Schema): + bound = BoundGreaterThan( + BoundReference(table_schema_simple.find_field(1), table_schema_simple.accessor_for_field(1)), literal("hello") + ) + assert ~bound == BoundLessThanOrEqual( + term=BoundReference[str]( + field=NestedField(field_id=1, name="foo", field_type=StringType(), required=False), + accessor=Accessor(position=0, inner=None), + ), + literal=StringLiteral("hello"), + ) + + +def test_bound_less_than_invert(table_schema_simple: Schema): + bound = BoundLessThan( + BoundReference(table_schema_simple.find_field(1), table_schema_simple.accessor_for_field(1)), literal("hello") + ) + assert ~bound == BoundGreaterThanOrEqual( + term=BoundReference[str]( + field=NestedField(field_id=1, name="foo", field_type=StringType(), required=False), + accessor=Accessor(position=0, inner=None), + ), + literal=StringLiteral("hello"), + ) + + +def test_bound_less_than_or_equal_invert(table_schema_simple: Schema): + bound = BoundLessThanOrEqual( + BoundReference(table_schema_simple.find_field(1), table_schema_simple.accessor_for_field(1)), literal("hello") + ) + assert ~bound == BoundGreaterThan( + term=BoundReference[str]( + field=NestedField(field_id=1, name="foo", field_type=StringType(), required=False), + accessor=Accessor(position=0, inner=None), + ), + literal=StringLiteral("hello"), + ) + + +def test_not_equal_to_invert(): + bound = NotEqualTo( + term=BoundReference( + field=NestedField(field_id=1, name="foo", field_type=StringType(), required=False), + accessor=Accessor(position=0, inner=None), + ), + literal=StringLiteral("hello"), + ) + assert ~bound == EqualTo( + term=BoundReference( + field=NestedField(field_id=1, name="foo", field_type=StringType(), required=False), + accessor=Accessor(position=0, inner=None), + ), + literal=StringLiteral("hello"), + ) + + +def test_greater_than_or_equal_invert(): + bound = GreaterThanOrEqual( + term=BoundReference( + field=NestedField(field_id=1, name="foo", field_type=StringType(), required=False), + accessor=Accessor(position=0, inner=None), + ), + literal=StringLiteral("hello"), + ) + assert ~bound == LessThan( + term=BoundReference( + field=NestedField(field_id=1, name="foo", field_type=StringType(), required=False), + accessor=Accessor(position=0, inner=None), + ), + literal=StringLiteral("hello"), + ) + + +def test_less_than_or_equal_invert(): + bound = LessThanOrEqual( + term=BoundReference( + field=NestedField(field_id=1, name="foo", field_type=StringType(), required=False), + accessor=Accessor(position=0, inner=None), + ), + literal=StringLiteral("hello"), + ) + assert ~bound == GreaterThan( + term=BoundReference( + field=NestedField(field_id=1, name="foo", field_type=StringType(), required=False), + accessor=Accessor(position=0, inner=None), + ), + literal=StringLiteral("hello"), + ) + + +@pytest.mark.parametrize( + "pred", + [ + NotIn(Reference("foo"), (literal("hello"), literal("world"))), + NotEqualTo(Reference("foo"), literal("hello")), + EqualTo(Reference("foo"), literal("hello")), + GreaterThan(Reference("foo"), literal("hello")), + LessThan(Reference("foo"), literal("hello")), + GreaterThanOrEqual(Reference("foo"), literal("hello")), + LessThanOrEqual(Reference("foo"), literal("hello")), + ], +) +def test_bind(pred, table_schema_simple: Schema): + assert pred.bind(table_schema_simple, case_sensitive=True).term.field == table_schema_simple.find_field( + pred.term.name, case_sensitive=True + ) + + +@pytest.mark.parametrize( + "pred", + [ + In(Reference("Bar"), (literal(5), literal(2))), + NotIn(Reference("Bar"), (literal(5), literal(2))), + NotEqualTo(Reference("Bar"), literal(5)), + EqualTo(Reference("Bar"), literal(5)), + GreaterThan(Reference("Bar"), literal(5)), + LessThan(Reference("Bar"), literal(5)), + GreaterThanOrEqual(Reference("Bar"), literal(5)), + LessThanOrEqual(Reference("Bar"), literal(5)), + ], +) +def test_bind_case_insensitive(pred, table_schema_simple: Schema): + assert pred.bind(table_schema_simple, case_sensitive=False).term.field == table_schema_simple.find_field( + pred.term.name, case_sensitive=False + ) + + +@pytest.mark.parametrize( + "exp, testexpra, testexprb", + [ + ( + And(ExpressionA(), ExpressionB()), + And(ExpressionA(), ExpressionB()), + Or(ExpressionA(), ExpressionB()), + ), + ( + Or(ExpressionA(), ExpressionB()), + Or(ExpressionA(), ExpressionB()), + And(ExpressionA(), ExpressionB()), + ), + (Not(ExpressionA()), Not(ExpressionA()), ExpressionB()), + (ExpressionA(), ExpressionA(), ExpressionB()), + (ExpressionB(), ExpressionB(), ExpressionA()), + ( + In(Reference("foo"), (literal("hello"), literal("world"))), + In(Reference("foo"), (literal("hello"), literal("world"))), + In(Reference("not_foo"), (literal("hello"), literal("world"))), + ), + ( + In(Reference("foo"), (literal("hello"), literal("world"))), + In(Reference("foo"), (literal("hello"), literal("world"))), + In(Reference("foo"), (literal("goodbye"), literal("world"))), + ), + ], +) +def test_eq(exp, testexpra, testexprb): + assert exp == testexpra and exp != testexprb + + +@pytest.mark.parametrize( + "lhs, rhs", + [ + ( + And(ExpressionA(), ExpressionB()), + Or(ExpressionB(), ExpressionA()), + ), + ( + Or(ExpressionA(), ExpressionB()), + And(ExpressionB(), ExpressionA()), + ), + ( + Not(ExpressionA()), + ExpressionA(), + ), + ( + In(Reference("foo"), (literal("hello"), literal("world"))), + NotIn(Reference("foo"), (literal("hello"), literal("world"))), + ), + ( + NotIn(Reference("foo"), (literal("hello"), literal("world"))), + In(Reference("foo"), (literal("hello"), literal("world"))), + ), + (GreaterThan(Reference("foo"), literal(5)), LessThanOrEqual(Reference("foo"), literal(5))), + (LessThan(Reference("foo"), literal(5)), GreaterThanOrEqual(Reference("foo"), literal(5))), + (EqualTo(Reference("foo"), literal(5)), NotEqualTo(Reference("foo"), literal(5))), + ( + ExpressionA(), + ExpressionB(), + ), + ], +) +def test_negate(lhs, rhs): + assert ~lhs == rhs + + +@pytest.mark.parametrize( + "lhs, rhs", + [ + ( + And(ExpressionA(), ExpressionB(), ExpressionA()), + And(And(ExpressionA(), ExpressionB()), ExpressionA()), + ), + ( + Or(ExpressionA(), ExpressionB(), ExpressionA()), + Or(Or(ExpressionA(), ExpressionB()), ExpressionA()), + ), + (Not(Not(ExpressionA())), ExpressionA()), + ], +) +def test_reduce(lhs, rhs): + assert lhs == rhs + + +@pytest.mark.parametrize( + "lhs, rhs", + [ + (And(AlwaysTrue(), ExpressionB()), ExpressionB()), + (And(AlwaysFalse(), ExpressionB()), AlwaysFalse()), + (And(ExpressionB(), AlwaysTrue()), ExpressionB()), + (Or(AlwaysTrue(), ExpressionB()), AlwaysTrue()), + (Or(AlwaysFalse(), ExpressionB()), ExpressionB()), + (Or(ExpressionA(), AlwaysFalse()), ExpressionA()), + (Not(Not(ExpressionA())), ExpressionA()), + (Not(AlwaysTrue()), AlwaysFalse()), + (Not(AlwaysFalse()), AlwaysTrue()), + ], +) +def test_base_AlwaysTrue_base_AlwaysFalse(lhs, rhs): + assert lhs == rhs + + +def test_invert_always(): + assert ~AlwaysFalse() == AlwaysTrue() + assert ~AlwaysTrue() == AlwaysFalse() + + +def test_accessor_base_class(foo_struct): + """Test retrieving a value at a position of a container using an accessor""" + + uuid_value = uuid.uuid4() + + foo_struct.set(0, "foo") + foo_struct.set(1, "bar") + foo_struct.set(2, "baz") + foo_struct.set(3, 1) + foo_struct.set(4, 2) + foo_struct.set(5, 3) + foo_struct.set(6, 1.234) + foo_struct.set(7, Decimal("1.234")) + foo_struct.set(8, uuid_value) + foo_struct.set(9, True) + foo_struct.set(10, False) + foo_struct.set(11, b"\x19\x04\x9e?") + + assert Accessor(position=0).get(foo_struct) == "foo" + assert Accessor(position=1).get(foo_struct) == "bar" + assert Accessor(position=2).get(foo_struct) == "baz" + assert Accessor(position=3).get(foo_struct) == 1 + assert Accessor(position=4).get(foo_struct) == 2 + assert Accessor(position=5).get(foo_struct) == 3 + assert Accessor(position=6).get(foo_struct) == 1.234 + assert Accessor(position=7).get(foo_struct) == Decimal("1.234") + assert Accessor(position=8).get(foo_struct) == uuid_value + assert Accessor(position=9).get(foo_struct) is True + assert Accessor(position=10).get(foo_struct) is False + assert Accessor(position=11).get(foo_struct) == b"\x19\x04\x9e?" + + +def test_bound_reference_str_and_repr(): + """Test str and repr of BoundReference""" + field = NestedField(field_id=1, name="foo", field_type=StringType(), required=False) + position1_accessor = Accessor(position=1) + bound_ref = BoundReference(field=field, accessor=position1_accessor) + assert str(bound_ref) == f"BoundReference(field={repr(field)}, accessor={repr(position1_accessor)})" + assert repr(bound_ref) == f"BoundReference(field={repr(field)}, accessor={repr(position1_accessor)})" + + +def test_bound_reference_field_property(): + """Test str and repr of BoundReference""" + field = NestedField(field_id=1, name="foo", field_type=StringType(), required=False) + position1_accessor = Accessor(position=1) + bound_ref = BoundReference(field=field, accessor=position1_accessor) + assert bound_ref.field == NestedField(field_id=1, name="foo", field_type=StringType(), required=False) + + +def test_bound_reference(table_schema_simple, foo_struct): + """Test creating a BoundReference and evaluating it on a StructProtocol""" + foo_struct.set(pos=1, value="foovalue") + foo_struct.set(pos=2, value=123) + foo_struct.set(pos=3, value=True) + + position1_accessor = Accessor(position=1) + position2_accessor = Accessor(position=2) + position3_accessor = Accessor(position=3) + + field1 = table_schema_simple.find_field(1) + field2 = table_schema_simple.find_field(2) + field3 = table_schema_simple.find_field(3) + + bound_ref1 = BoundReference(field=field1, accessor=position1_accessor) + bound_ref2 = BoundReference(field=field2, accessor=position2_accessor) + bound_ref3 = BoundReference(field=field3, accessor=position3_accessor) + + assert bound_ref1.eval(foo_struct) == "foovalue" + assert bound_ref2.eval(foo_struct) == 123 + assert bound_ref3.eval(foo_struct) is True + + +def test_bound_predicate_invert(): + with pytest.raises(NotImplementedError): + _ = ~BoundPredicate( + term=BoundReference( + field=NestedField(field_id=1, name="foo", field_type=StringType(), required=False), + accessor=Accessor(position=0, inner=None), + ) + ) + + +def test_bound_unary_predicate_invert(): + with pytest.raises(NotImplementedError): + _ = ~BoundUnaryPredicate( + term=BoundReference( + field=NestedField(field_id=1, name="foo", field_type=StringType(), required=False), + accessor=Accessor(position=0, inner=None), + ) + ) + + +def test_bound_set_predicate_invert(): + with pytest.raises(NotImplementedError): + _ = ~BoundSetPredicate( + term=BoundReference( + field=NestedField(field_id=1, name="foo", field_type=StringType(), required=False), + accessor=Accessor(position=0, inner=None), + ), + literals={literal("hello"), literal("world")}, + ) + + +def test_bound_literal_predicate_invert(): + with pytest.raises(NotImplementedError): + _ = ~BoundLiteralPredicate( + term=BoundReference( + field=NestedField(field_id=1, name="foo", field_type=StringType(), required=False), + accessor=Accessor(position=0, inner=None), + ), + literal=literal("world"), + ) + + +def test_non_primitive_from_byte_buffer(): + with pytest.raises(ValueError) as exc_info: + _ = _from_byte_buffer(ListType(element_id=1, element_type=StringType()), b"\0x00") + + assert str(exc_info.value) == "Expected a PrimitiveType, got: " + + +def test_unbound_predicate_invert(): + with pytest.raises(NotImplementedError): + _ = ~UnboundPredicate(term=Reference("a")) + + +def test_unbound_predicate_bind(table_schema_simple: Schema): + with pytest.raises(NotImplementedError): + _ = UnboundPredicate(term=Reference("a")).bind(table_schema_simple) + + +def test_unbound_unary_predicate_invert(): + with pytest.raises(NotImplementedError): + _ = ~UnaryPredicate(term=Reference("a")) + + +def test_unbound_set_predicate_invert(): + with pytest.raises(NotImplementedError): + _ = ~SetPredicate(term=Reference("a"), literals=(literal("hello"), literal("world"))) + + +def test_unbound_literal_predicate_invert(): + with pytest.raises(NotImplementedError): + _ = ~LiteralPredicate(term=Reference("a"), literal=literal("hello")) diff --git a/tests/expressions/test_expressions_base.py b/tests/expressions/test_visitors.py similarity index 55% rename from tests/expressions/test_expressions_base.py rename to tests/expressions/test_visitors.py index 6a254c53cc..078dd88747 100644 --- a/tests/expressions/test_expressions_base.py +++ b/tests/expressions/test_visitors.py @@ -15,21 +15,47 @@ # specific language governing permissions and limitations # under the License. -import uuid -from decimal import Decimal from typing import Any, List, Set import pytest from pyiceberg.conversions import to_bytes -from pyiceberg.expressions import base -from pyiceberg.expressions.base import ( - IN_PREDICATE_LIMIT, +from pyiceberg.expressions import ( + AlwaysFalse, + AlwaysTrue, + And, + BooleanExpression, + BoundEqualTo, + BoundGreaterThan, + BoundGreaterThanOrEqual, + BoundIn, + BoundIsNaN, + BoundIsNull, + BoundLessThan, + BoundLessThanOrEqual, + BoundLiteralPredicate, + BoundNotEqualTo, + BoundNotIn, + BoundNotNaN, + BoundNotNull, BoundPredicate, - _from_byte_buffer, - _ManifestEvalVisitor, - rewrite_not, - visit_bound_predicate, + BoundReference, + BoundSetPredicate, + BoundTerm, + BoundUnaryPredicate, + EqualTo, + In, + IsNull, + LessThanOrEqual, + LiteralPredicate, + Not, + NotEqualTo, + NotIn, + Or, + Reference, + SetPredicate, + UnaryPredicate, + UnboundPredicate, ) from pyiceberg.expressions.literals import ( Literal, @@ -37,6 +63,16 @@ StringLiteral, literal, ) +from pyiceberg.expressions.visitors import ( + IN_PREDICATE_LIMIT, + BindVisitor, + BooleanExpressionVisitor, + BoundBooleanExpressionVisitor, + _ManifestEvalVisitor, + rewrite_not, + visit, + visit_bound_predicate, +) from pyiceberg.manifest import ManifestFile, PartitionFieldSummary from pyiceberg.schema import Accessor, Schema from pyiceberg.types import ( @@ -44,7 +80,6 @@ FloatType, IcebergType, IntegerType, - ListType, LongType, NestedField, PrimitiveType, @@ -53,7 +88,7 @@ from pyiceberg.utils.singleton import Singleton -class ExpressionA(base.BooleanExpression, Singleton): +class ExpressionA(BooleanExpression, Singleton): def __invert__(self): return ExpressionB() @@ -64,7 +99,7 @@ def __str__(self): return "testexpra" -class ExpressionB(base.BooleanExpression, Singleton): +class ExpressionB(BooleanExpression, Singleton): def __invert__(self): return ExpressionA() @@ -75,7 +110,7 @@ def __str__(self): return "testexprb" -class ExampleVisitor(base.BooleanExpressionVisitor[List]): +class ExampleVisitor(BooleanExpressionVisitor[List]): """A test implementation of a BooleanExpressionVisitor As this visitor visits each node, it appends an element to a `visit_history` list. This enables testing that a given expression is @@ -122,19 +157,19 @@ def visit_test_expression_b(self) -> List: return self.visit_history -@base.visit.register(ExpressionA) +@visit.register(ExpressionA) def _(obj: ExpressionA, visitor: ExampleVisitor) -> List: """Visit a ExpressionA with a BooleanExpressionVisitor""" return visitor.visit_test_expression_a() -@base.visit.register(ExpressionB) +@visit.register(ExpressionB) def _(obj: ExpressionB, visitor: ExampleVisitor) -> List: """Visit a ExpressionB with a BooleanExpressionVisitor""" return visitor.visit_test_expression_b() -class FooBoundBooleanExpressionVisitor(base.BoundBooleanExpressionVisitor[List]): +class FooBoundBooleanExpressionVisitor(BoundBooleanExpressionVisitor[List]): """A test implementation of a BoundBooleanExpressionVisitor As this visitor visits each node, it appends an element to a `visit_history` list. This enables testing that a given bound expression is visited in an expected order by the `visit` method. @@ -143,51 +178,51 @@ class FooBoundBooleanExpressionVisitor(base.BoundBooleanExpressionVisitor[List]) def __init__(self): self.visit_history: List = [] - def visit_in(self, term: base.BoundTerm, literals: Set) -> List: + def visit_in(self, term: BoundTerm, literals: Set) -> List: self.visit_history.append("IN") return self.visit_history - def visit_not_in(self, term: base.BoundTerm, literals: Set) -> List: + def visit_not_in(self, term: BoundTerm, literals: Set) -> List: self.visit_history.append("NOT_IN") return self.visit_history - def visit_is_nan(self, term: base.BoundTerm) -> List: + def visit_is_nan(self, term: BoundTerm) -> List: self.visit_history.append("IS_NAN") return self.visit_history - def visit_not_nan(self, term: base.BoundTerm) -> List: + def visit_not_nan(self, term: BoundTerm) -> List: self.visit_history.append("NOT_NAN") return self.visit_history - def visit_is_null(self, term: base.BoundTerm) -> List: + def visit_is_null(self, term: BoundTerm) -> List: self.visit_history.append("IS_NULL") return self.visit_history - def visit_not_null(self, term: base.BoundTerm) -> List: + def visit_not_null(self, term: BoundTerm) -> List: self.visit_history.append("NOT_NULL") return self.visit_history - def visit_equal(self, term: base.BoundTerm, literal: Literal) -> List: # pylint: disable=redefined-outer-name + def visit_equal(self, term: BoundTerm, literal: Literal) -> List: # pylint: disable=redefined-outer-name self.visit_history.append("EQUAL") return self.visit_history - def visit_not_equal(self, term: base.BoundTerm, literal: Literal) -> List: # pylint: disable=redefined-outer-name + def visit_not_equal(self, term: BoundTerm, literal: Literal) -> List: # pylint: disable=redefined-outer-name self.visit_history.append("NOT_EQUAL") return self.visit_history - def visit_greater_than_or_equal(self, term: base.BoundTerm, literal: Literal) -> List: # pylint: disable=redefined-outer-name + def visit_greater_than_or_equal(self, term: BoundTerm, literal: Literal) -> List: # pylint: disable=redefined-outer-name self.visit_history.append("GREATER_THAN_OR_EQUAL") return self.visit_history - def visit_greater_than(self, term: base.BoundTerm, literal: Literal) -> List: # pylint: disable=redefined-outer-name + def visit_greater_than(self, term: BoundTerm, literal: Literal) -> List: # pylint: disable=redefined-outer-name self.visit_history.append("GREATER_THAN") return self.visit_history - def visit_less_than(self, term: base.BoundTerm, literal: Literal) -> List: # pylint: disable=redefined-outer-name + def visit_less_than(self, term: BoundTerm, literal: Literal) -> List: # pylint: disable=redefined-outer-name self.visit_history.append("LESS_THAN") return self.visit_history - def visit_less_than_or_equal(self, term: base.BoundTerm, literal: Literal) -> List: # pylint: disable=redefined-outer-name + def visit_less_than_or_equal(self, term: BoundTerm, literal: Literal) -> List: # pylint: disable=redefined-outer-name self.visit_history.append("LESS_THAN_OR_EQUAL") return self.visit_history @@ -212,619 +247,15 @@ def visit_or(self, left_result: List, right_result: List) -> List: return self.visit_history -@pytest.mark.parametrize( - "op, rep", - [ - ( - base.And(ExpressionA(), ExpressionB()), - "And(left=ExpressionA(), right=ExpressionB())", - ), - ( - base.Or(ExpressionA(), ExpressionB()), - "Or(left=ExpressionA(), right=ExpressionB())", - ), - (base.Not(ExpressionA()), "Not(child=ExpressionA())"), - ], -) -def test_reprs(op: base.BooleanExpression, rep: str): - assert repr(op) == rep - - -def test_isnull_inverse(): - assert ~base.IsNull(base.Reference("a")) == base.NotNull(base.Reference("a")) - - -def test_isnull_bind(): - schema = Schema(NestedField(2, "a", IntegerType()), schema_id=1) - bound = base.BoundIsNull(base.BoundReference(schema.find_field(2), schema.accessor_for_field(2))) - assert base.IsNull(base.Reference("a")).bind(schema) == bound - - -def test_invert_is_null_bind(): - schema = Schema(NestedField(2, "a", IntegerType(), required=False), schema_id=1) - assert ~base.IsNull(base.Reference("a")).bind(schema) == base.NotNull(base.Reference("a")).bind(schema) - - -def test_invert_not_null_bind(): - schema = Schema(NestedField(2, "a", IntegerType(), required=False), schema_id=1) - assert ~base.NotNull(base.Reference("a")).bind(schema) == base.IsNull(base.Reference("a")).bind(schema) - - -def test_invert_is_nan_bind(): - schema = Schema(NestedField(2, "a", DoubleType(), required=False), schema_id=1) - assert ~base.IsNaN(base.Reference("a")).bind(schema) == base.NotNaN(base.Reference("a")).bind(schema) - - -def test_invert_not_nan_bind(): - schema = Schema(NestedField(2, "a", DoubleType(), required=False), schema_id=1) - assert ~base.NotNaN(base.Reference("a")).bind(schema) == base.IsNaN(base.Reference("a")).bind(schema) - - -def test_bind_expr_does_not_exists(): - schema = Schema(NestedField(2, "a", IntegerType()), schema_id=1) - with pytest.raises(ValueError) as exc_info: - base.IsNull(base.Reference("b")).bind(schema) - - assert str(exc_info.value) == "Could not find field with name b, case_sensitive=True" - - -def test_bind_does_not_exists(): - schema = Schema(NestedField(2, "a", IntegerType()), schema_id=1) - with pytest.raises(ValueError) as exc_info: - base.Reference("b").bind(schema) - - assert str(exc_info.value) == "Could not find field with name b, case_sensitive=True" - - -def test_isnull_bind_required(): - schema = Schema(NestedField(2, "a", IntegerType(), required=True), schema_id=1) - assert base.IsNull(base.Reference("a")).bind(schema) == base.AlwaysFalse() - - -def test_notnull_inverse(): - assert ~base.NotNull(base.Reference("a")) == base.IsNull(base.Reference("a")) - - -def test_notnull_bind(): - schema = Schema(NestedField(2, "a", IntegerType()), schema_id=1) - bound = base.BoundNotNull(base.BoundReference(schema.find_field(2), schema.accessor_for_field(2))) - assert base.NotNull(base.Reference("a")).bind(schema) == bound - - -def test_notnull_bind_required(): - schema = Schema(NestedField(2, "a", IntegerType(), required=True), schema_id=1) - assert base.NotNull(base.Reference("a")).bind(schema) == base.AlwaysTrue() - - -def test_isnan_inverse(): - assert ~base.IsNaN(base.Reference("f")) == base.NotNaN(base.Reference("f")) - - -def test_isnan_bind_float(): - schema = Schema(NestedField(2, "f", FloatType()), schema_id=1) - bound = base.BoundIsNaN(base.BoundReference(schema.find_field(2), schema.accessor_for_field(2))) - assert base.IsNaN(base.Reference("f")).bind(schema) == bound - - -def test_isnan_bind_double(): - schema = Schema(NestedField(2, "d", DoubleType()), schema_id=1) - bound = base.BoundIsNaN(base.BoundReference(schema.find_field(2), schema.accessor_for_field(2))) - assert base.IsNaN(base.Reference("d")).bind(schema) == bound - - -def test_isnan_bind_nonfloat(): - schema = Schema(NestedField(2, "i", IntegerType()), schema_id=1) - assert base.IsNaN(base.Reference("i")).bind(schema) == base.AlwaysFalse() - - -def test_notnan_inverse(): - assert ~base.NotNaN(base.Reference("f")) == base.IsNaN(base.Reference("f")) - - -def test_notnan_bind_float(): - schema = Schema(NestedField(2, "f", FloatType()), schema_id=1) - bound = base.BoundNotNaN(base.BoundReference(schema.find_field(2), schema.accessor_for_field(2))) - assert base.NotNaN(base.Reference("f")).bind(schema) == bound - - -def test_notnan_bind_double(): - schema = Schema(NestedField(2, "d", DoubleType()), schema_id=1) - bound = base.BoundNotNaN(base.BoundReference(schema.find_field(2), schema.accessor_for_field(2))) - assert base.NotNaN(base.Reference("d")).bind(schema) == bound - - -def test_notnan_bind_nonfloat(): - schema = Schema(NestedField(2, "i", IntegerType()), schema_id=1) - assert base.NotNaN(base.Reference("i")).bind(schema) == base.AlwaysTrue() - - -@pytest.mark.parametrize( - "op, string", - [ - (base.And(ExpressionA(), ExpressionB()), "And(left=ExpressionA(), right=ExpressionB())"), - (base.Or(ExpressionA(), ExpressionB()), "Or(left=ExpressionA(), right=ExpressionB())"), - (base.Not(ExpressionA()), "Not(child=ExpressionA())"), - ], -) -def test_strs(op, string): - assert str(op) == string - - -def test_ref_binding_case_sensitive(table_schema_simple: Schema): - ref = base.Reference[str]("foo") - bound = base.BoundReference[str](table_schema_simple.find_field(1), table_schema_simple.accessor_for_field(1)) - assert ref.bind(table_schema_simple, case_sensitive=True) == bound - - -def test_ref_binding_case_sensitive_failure(table_schema_simple: Schema): - ref = base.Reference[str]("Foo") - with pytest.raises(ValueError): - ref.bind(table_schema_simple, case_sensitive=True) - - -def test_ref_binding_case_insensitive(table_schema_simple: Schema): - ref = base.Reference[str]("Foo") - bound = base.BoundReference[str](table_schema_simple.find_field(1), table_schema_simple.accessor_for_field(1)) - assert ref.bind(table_schema_simple, case_sensitive=False) == bound - - -def test_ref_binding_case_insensitive_failure(table_schema_simple: Schema): - ref = base.Reference[str]("Foot") - with pytest.raises(ValueError): - ref.bind(table_schema_simple, case_sensitive=False) - - -def test_in_to_eq(): - assert base.In(base.Reference("x"), (literal(34.56),)) == base.EqualTo(base.Reference("x"), literal(34.56)) - - -def test_empty_bind_in(table_schema_simple: Schema): - bound = base.BoundIn[str]( - base.BoundReference(table_schema_simple.find_field(1), table_schema_simple.accessor_for_field(1)), set() - ) - assert bound == base.AlwaysFalse() - - -def test_empty_bind_not_in(table_schema_simple: Schema): - bound = base.BoundNotIn( - base.BoundReference[str](table_schema_simple.find_field(1), table_schema_simple.accessor_for_field(1)), set() - ) - assert bound == base.AlwaysTrue() - - -def test_bind_not_in_equal_term(table_schema_simple: Schema): - bound = base.BoundNotIn( - base.BoundReference(table_schema_simple.find_field(1), table_schema_simple.accessor_for_field(1)), {literal("hello")} - ) - assert ( - base.BoundNotEqualTo[str]( - term=base.BoundReference( - field=NestedField(field_id=1, name="foo", field_type=StringType(), required=False), - accessor=Accessor(position=0, inner=None), - ), - literal=StringLiteral("hello"), - ) - == bound - ) - - -def test_in_empty(): - assert base.In(base.Reference("foo"), ()) == base.AlwaysFalse() - - -def test_not_in_empty(): - assert base.NotIn(base.Reference("foo"), ()) == base.AlwaysTrue() - - -def test_not_in_equal(): - assert base.NotIn(base.Reference("foo"), (literal("hello"),)) == base.NotEqualTo( - term=base.Reference(name="foo"), literal=StringLiteral("hello") - ) - - -def test_bind_in(table_schema_simple: Schema): - bound = base.BoundIn( - base.BoundReference(table_schema_simple.find_field(1), table_schema_simple.accessor_for_field(1)), - {literal("hello"), literal("world")}, - ) - assert base.In(base.Reference("foo"), (literal("hello"), literal("world"))).bind(table_schema_simple) == bound - - -def test_bind_in_invert(table_schema_simple: Schema): - bound = base.BoundIn( - base.BoundReference(table_schema_simple.find_field(1), table_schema_simple.accessor_for_field(1)), - {literal("hello"), literal("world")}, - ) - assert ~bound == base.BoundNotIn( - base.BoundReference(table_schema_simple.find_field(1), table_schema_simple.accessor_for_field(1)), - {literal("hello"), literal("world")}, - ) - - -def test_bind_not_in_invert(table_schema_simple: Schema): - bound = base.BoundNotIn( - base.BoundReference(table_schema_simple.find_field(1), table_schema_simple.accessor_for_field(1)), - {literal("hello"), literal("world")}, - ) - assert ~bound == base.BoundIn( - base.BoundReference(table_schema_simple.find_field(1), table_schema_simple.accessor_for_field(1)), - {literal("hello"), literal("world")}, - ) - - -def test_bind_dedup(table_schema_simple: Schema): - bound = base.BoundIn( - base.BoundReference(table_schema_simple.find_field(1), table_schema_simple.accessor_for_field(1)), - {literal("hello"), literal("world")}, - ) - assert ( - base.In(base.Reference("foo"), (literal("hello"), literal("world"), literal("world"))).bind(table_schema_simple) == bound - ) - - -def test_bind_dedup_to_eq(table_schema_simple: Schema): - bound = base.BoundEqualTo( - base.BoundReference(table_schema_simple.find_field(1), table_schema_simple.accessor_for_field(1)), literal("hello") - ) - assert base.In(base.Reference("foo"), (literal("hello"), literal("hello"))).bind(table_schema_simple) == bound - - -def test_bound_equal_to_invert(table_schema_simple: Schema): - bound = base.BoundEqualTo( - base.BoundReference(table_schema_simple.find_field(1), table_schema_simple.accessor_for_field(1)), literal("hello") - ) - assert ~bound == base.BoundNotEqualTo( - term=base.BoundReference[str]( - field=NestedField(field_id=1, name="foo", field_type=StringType(), required=False), - accessor=Accessor(position=0, inner=None), - ), - literal=StringLiteral("hello"), - ) - - -def test_bound_not_equal_to_invert(table_schema_simple: Schema): - bound = base.BoundNotEqualTo( - base.BoundReference(table_schema_simple.find_field(1), table_schema_simple.accessor_for_field(1)), literal("hello") - ) - assert ~bound == base.BoundEqualTo( - term=base.BoundReference[str]( - field=NestedField(field_id=1, name="foo", field_type=StringType(), required=False), - accessor=Accessor(position=0, inner=None), - ), - literal=StringLiteral("hello"), - ) - - -def test_bound_greater_than_or_equal_invert(table_schema_simple: Schema): - bound = base.BoundGreaterThanOrEqual( - base.BoundReference(table_schema_simple.find_field(1), table_schema_simple.accessor_for_field(1)), literal("hello") - ) - assert ~bound == base.BoundLessThan( - term=base.BoundReference[str]( - field=NestedField(field_id=1, name="foo", field_type=StringType(), required=False), - accessor=Accessor(position=0, inner=None), - ), - literal=StringLiteral("hello"), - ) - - -def test_bound_greater_than_invert(table_schema_simple: Schema): - bound = base.BoundGreaterThan( - base.BoundReference(table_schema_simple.find_field(1), table_schema_simple.accessor_for_field(1)), literal("hello") - ) - assert ~bound == base.BoundLessThanOrEqual( - term=base.BoundReference[str]( - field=NestedField(field_id=1, name="foo", field_type=StringType(), required=False), - accessor=Accessor(position=0, inner=None), - ), - literal=StringLiteral("hello"), - ) - - -def test_bound_less_than_invert(table_schema_simple: Schema): - bound = base.BoundLessThan( - base.BoundReference(table_schema_simple.find_field(1), table_schema_simple.accessor_for_field(1)), literal("hello") - ) - assert ~bound == base.BoundGreaterThanOrEqual( - term=base.BoundReference[str]( - field=NestedField(field_id=1, name="foo", field_type=StringType(), required=False), - accessor=Accessor(position=0, inner=None), - ), - literal=StringLiteral("hello"), - ) - - -def test_bound_less_than_or_equal_invert(table_schema_simple: Schema): - bound = base.BoundLessThanOrEqual( - base.BoundReference(table_schema_simple.find_field(1), table_schema_simple.accessor_for_field(1)), literal("hello") - ) - assert ~bound == base.BoundGreaterThan( - term=base.BoundReference[str]( - field=NestedField(field_id=1, name="foo", field_type=StringType(), required=False), - accessor=Accessor(position=0, inner=None), - ), - literal=StringLiteral("hello"), - ) - - -def test_not_equal_to_invert(): - bound = base.NotEqualTo( - term=base.BoundReference( - field=NestedField(field_id=1, name="foo", field_type=StringType(), required=False), - accessor=Accessor(position=0, inner=None), - ), - literal=StringLiteral("hello"), - ) - assert ~bound == base.EqualTo( - term=base.BoundReference( - field=NestedField(field_id=1, name="foo", field_type=StringType(), required=False), - accessor=Accessor(position=0, inner=None), - ), - literal=StringLiteral("hello"), - ) - - -def test_greater_than_or_equal_invert(): - bound = base.GreaterThanOrEqual( - term=base.BoundReference( - field=NestedField(field_id=1, name="foo", field_type=StringType(), required=False), - accessor=Accessor(position=0, inner=None), - ), - literal=StringLiteral("hello"), - ) - assert ~bound == base.LessThan( - term=base.BoundReference( - field=NestedField(field_id=1, name="foo", field_type=StringType(), required=False), - accessor=Accessor(position=0, inner=None), - ), - literal=StringLiteral("hello"), - ) - - -def test_less_than_or_equal_invert(): - bound = base.LessThanOrEqual( - term=base.BoundReference( - field=NestedField(field_id=1, name="foo", field_type=StringType(), required=False), - accessor=Accessor(position=0, inner=None), - ), - literal=StringLiteral("hello"), - ) - assert ~bound == base.GreaterThan( - term=base.BoundReference( - field=NestedField(field_id=1, name="foo", field_type=StringType(), required=False), - accessor=Accessor(position=0, inner=None), - ), - literal=StringLiteral("hello"), - ) - - -@pytest.mark.parametrize( - "pred", - [ - base.NotIn(base.Reference("foo"), (literal("hello"), literal("world"))), - base.NotEqualTo(base.Reference("foo"), literal("hello")), - base.EqualTo(base.Reference("foo"), literal("hello")), - base.GreaterThan(base.Reference("foo"), literal("hello")), - base.LessThan(base.Reference("foo"), literal("hello")), - base.GreaterThanOrEqual(base.Reference("foo"), literal("hello")), - base.LessThanOrEqual(base.Reference("foo"), literal("hello")), - ], -) -def test_bind(pred, table_schema_simple: Schema): - assert pred.bind(table_schema_simple, case_sensitive=True).term.field == table_schema_simple.find_field( - pred.term.name, case_sensitive=True - ) - - -@pytest.mark.parametrize( - "pred", - [ - base.In(base.Reference("Bar"), (literal(5), literal(2))), - base.NotIn(base.Reference("Bar"), (literal(5), literal(2))), - base.NotEqualTo(base.Reference("Bar"), literal(5)), - base.EqualTo(base.Reference("Bar"), literal(5)), - base.GreaterThan(base.Reference("Bar"), literal(5)), - base.LessThan(base.Reference("Bar"), literal(5)), - base.GreaterThanOrEqual(base.Reference("Bar"), literal(5)), - base.LessThanOrEqual(base.Reference("Bar"), literal(5)), - ], -) -def test_bind_case_insensitive(pred, table_schema_simple: Schema): - assert pred.bind(table_schema_simple, case_sensitive=False).term.field == table_schema_simple.find_field( - pred.term.name, case_sensitive=False - ) - - -@pytest.mark.parametrize( - "exp, testexpra, testexprb", - [ - ( - base.And(ExpressionA(), ExpressionB()), - base.And(ExpressionA(), ExpressionB()), - base.Or(ExpressionA(), ExpressionB()), - ), - ( - base.Or(ExpressionA(), ExpressionB()), - base.Or(ExpressionA(), ExpressionB()), - base.And(ExpressionA(), ExpressionB()), - ), - (base.Not(ExpressionA()), base.Not(ExpressionA()), ExpressionB()), - (ExpressionA(), ExpressionA(), ExpressionB()), - (ExpressionB(), ExpressionB(), ExpressionA()), - ( - base.In(base.Reference("foo"), (literal("hello"), literal("world"))), - base.In(base.Reference("foo"), (literal("hello"), literal("world"))), - base.In(base.Reference("not_foo"), (literal("hello"), literal("world"))), - ), - ( - base.In(base.Reference("foo"), (literal("hello"), literal("world"))), - base.In(base.Reference("foo"), (literal("hello"), literal("world"))), - base.In(base.Reference("foo"), (literal("goodbye"), literal("world"))), - ), - ], -) -def test_eq(exp, testexpra, testexprb): - assert exp == testexpra and exp != testexprb - - -@pytest.mark.parametrize( - "lhs, rhs", - [ - ( - base.And(ExpressionA(), ExpressionB()), - base.Or(ExpressionB(), ExpressionA()), - ), - ( - base.Or(ExpressionA(), ExpressionB()), - base.And(ExpressionB(), ExpressionA()), - ), - ( - base.Not(ExpressionA()), - ExpressionA(), - ), - ( - base.In(base.Reference("foo"), (literal("hello"), literal("world"))), - base.NotIn(base.Reference("foo"), (literal("hello"), literal("world"))), - ), - ( - base.NotIn(base.Reference("foo"), (literal("hello"), literal("world"))), - base.In(base.Reference("foo"), (literal("hello"), literal("world"))), - ), - (base.GreaterThan(base.Reference("foo"), literal(5)), base.LessThanOrEqual(base.Reference("foo"), literal(5))), - (base.LessThan(base.Reference("foo"), literal(5)), base.GreaterThanOrEqual(base.Reference("foo"), literal(5))), - (base.EqualTo(base.Reference("foo"), literal(5)), base.NotEqualTo(base.Reference("foo"), literal(5))), - ( - ExpressionA(), - ExpressionB(), - ), - ], -) -def test_negate(lhs, rhs): - assert ~lhs == rhs - - -@pytest.mark.parametrize( - "lhs, rhs", - [ - ( - base.And(ExpressionA(), ExpressionB(), ExpressionA()), - base.And(base.And(ExpressionA(), ExpressionB()), ExpressionA()), - ), - ( - base.Or(ExpressionA(), ExpressionB(), ExpressionA()), - base.Or(base.Or(ExpressionA(), ExpressionB()), ExpressionA()), - ), - (base.Not(base.Not(ExpressionA())), ExpressionA()), - ], -) -def test_reduce(lhs, rhs): - assert lhs == rhs - - -@pytest.mark.parametrize( - "lhs, rhs", - [ - (base.And(base.AlwaysTrue(), ExpressionB()), ExpressionB()), - (base.And(base.AlwaysFalse(), ExpressionB()), base.AlwaysFalse()), - (base.And(ExpressionB(), base.AlwaysTrue()), ExpressionB()), - (base.Or(base.AlwaysTrue(), ExpressionB()), base.AlwaysTrue()), - (base.Or(base.AlwaysFalse(), ExpressionB()), ExpressionB()), - (base.Or(ExpressionA(), base.AlwaysFalse()), ExpressionA()), - (base.Not(base.Not(ExpressionA())), ExpressionA()), - (base.Not(base.AlwaysTrue()), base.AlwaysFalse()), - (base.Not(base.AlwaysFalse()), base.AlwaysTrue()), - ], -) -def test_base_AlwaysTrue_base_AlwaysFalse(lhs, rhs): - assert lhs == rhs - - -def test_invert_always(): - assert ~base.AlwaysFalse() == base.AlwaysTrue() - assert ~base.AlwaysTrue() == base.AlwaysFalse() - - -def test_accessor_base_class(foo_struct): - """Test retrieving a value at a position of a container using an accessor""" - - uuid_value = uuid.uuid4() - - foo_struct.set(0, "foo") - foo_struct.set(1, "bar") - foo_struct.set(2, "baz") - foo_struct.set(3, 1) - foo_struct.set(4, 2) - foo_struct.set(5, 3) - foo_struct.set(6, 1.234) - foo_struct.set(7, Decimal("1.234")) - foo_struct.set(8, uuid_value) - foo_struct.set(9, True) - foo_struct.set(10, False) - foo_struct.set(11, b"\x19\x04\x9e?") - - assert base.Accessor(position=0).get(foo_struct) == "foo" - assert base.Accessor(position=1).get(foo_struct) == "bar" - assert base.Accessor(position=2).get(foo_struct) == "baz" - assert base.Accessor(position=3).get(foo_struct) == 1 - assert base.Accessor(position=4).get(foo_struct) == 2 - assert base.Accessor(position=5).get(foo_struct) == 3 - assert base.Accessor(position=6).get(foo_struct) == 1.234 - assert base.Accessor(position=7).get(foo_struct) == Decimal("1.234") - assert base.Accessor(position=8).get(foo_struct) == uuid_value - assert base.Accessor(position=9).get(foo_struct) is True - assert base.Accessor(position=10).get(foo_struct) is False - assert base.Accessor(position=11).get(foo_struct) == b"\x19\x04\x9e?" - - -def test_bound_reference_str_and_repr(): - """Test str and repr of BoundReference""" - field = NestedField(field_id=1, name="foo", field_type=StringType(), required=False) - position1_accessor = base.Accessor(position=1) - bound_ref = base.BoundReference(field=field, accessor=position1_accessor) - assert str(bound_ref) == f"BoundReference(field={repr(field)}, accessor={repr(position1_accessor)})" - assert repr(bound_ref) == f"BoundReference(field={repr(field)}, accessor={repr(position1_accessor)})" - - -def test_bound_reference_field_property(): - """Test str and repr of BoundReference""" - field = NestedField(field_id=1, name="foo", field_type=StringType(), required=False) - position1_accessor = base.Accessor(position=1) - bound_ref = base.BoundReference(field=field, accessor=position1_accessor) - assert bound_ref.field == NestedField(field_id=1, name="foo", field_type=StringType(), required=False) - - -def test_bound_reference(table_schema_simple, foo_struct): - """Test creating a BoundReference and evaluating it on a StructProtocol""" - foo_struct.set(pos=1, value="foovalue") - foo_struct.set(pos=2, value=123) - foo_struct.set(pos=3, value=True) - - position1_accessor = base.Accessor(position=1) - position2_accessor = base.Accessor(position=2) - position3_accessor = base.Accessor(position=3) - - field1 = table_schema_simple.find_field(1) - field2 = table_schema_simple.find_field(2) - field3 = table_schema_simple.find_field(3) - - bound_ref1 = base.BoundReference(field=field1, accessor=position1_accessor) - bound_ref2 = base.BoundReference(field=field2, accessor=position2_accessor) - bound_ref3 = base.BoundReference(field=field3, accessor=position3_accessor) - - assert bound_ref1.eval(foo_struct) == "foovalue" - assert bound_ref2.eval(foo_struct) == 123 - assert bound_ref3.eval(foo_struct) is True - - def test_boolean_expression_visitor(): """Test post-order traversal of boolean expression visit method""" - expr = base.And( - base.Or(base.Not(ExpressionA()), base.Not(ExpressionB()), ExpressionA(), ExpressionB()), - base.Not(ExpressionA()), + expr = And( + Or(Not(ExpressionA()), Not(ExpressionB()), ExpressionA(), ExpressionB()), + Not(ExpressionA()), ExpressionB(), ) visitor = ExampleVisitor() - result = base.visit(expr, visitor=visitor) + result = visit(expr, visitor=visitor) assert result == [ "ExpressionA", "NOT", @@ -847,17 +278,17 @@ def test_boolean_expression_visit_raise_not_implemented_error(): """Test raise NotImplementedError when visiting an unsupported object type""" visitor = ExampleVisitor() with pytest.raises(NotImplementedError) as exc_info: - base.visit("foo", visitor=visitor) + visit("foo", visitor=visitor) assert str(exc_info.value) == "Cannot visit unsupported expression: foo" def test_bind_visitor_already_bound(table_schema_simple: Schema): - bound = base.BoundEqualTo( - base.BoundReference(table_schema_simple.find_field(1), table_schema_simple.accessor_for_field(1)), literal("hello") + bound = BoundEqualTo( + BoundReference(table_schema_simple.find_field(1), table_schema_simple.accessor_for_field(1)), literal("hello") ) with pytest.raises(TypeError) as exc_info: - base.BindVisitor(base.visit(bound, visitor=base.BindVisitor(schema=table_schema_simple))) # type: ignore + BindVisitor(visit(bound, visitor=BindVisitor(schema=table_schema_simple))) # type: ignore assert ( "Found already bound predicate: BoundEqualTo(term=BoundReference(field=NestedField(field_id=1, name='foo', field_type=StringType(), required=False), accessor=Accessor(position=0,inner=None)), literal=StringLiteral(hello))" == str(exc_info.value) @@ -872,50 +303,50 @@ def test_visit_bound_visitor_unknown_predicate(): def test_always_true_expression_binding(table_schema_simple: Schema): """Test that visiting an always-true expression returns always-true""" - unbound_expression = base.AlwaysTrue() - bound_expression = base.visit(unbound_expression, visitor=base.BindVisitor(schema=table_schema_simple)) - assert bound_expression == base.AlwaysTrue() + unbound_expression = AlwaysTrue() + bound_expression = visit(unbound_expression, visitor=BindVisitor(schema=table_schema_simple)) + assert bound_expression == AlwaysTrue() def test_always_false_expression_binding(table_schema_simple: Schema): """Test that visiting an always-false expression returns always-false""" - unbound_expression = base.AlwaysFalse() - bound_expression = base.visit(unbound_expression, visitor=base.BindVisitor(schema=table_schema_simple)) - assert bound_expression == base.AlwaysFalse() + unbound_expression = AlwaysFalse() + bound_expression = visit(unbound_expression, visitor=BindVisitor(schema=table_schema_simple)) + assert bound_expression == AlwaysFalse() def test_always_false_and_always_true_expression_binding(table_schema_simple: Schema): """Test that visiting both an always-true AND always-false expression returns always-false""" - unbound_expression = base.And(base.AlwaysTrue(), base.AlwaysFalse()) - bound_expression = base.visit(unbound_expression, visitor=base.BindVisitor(schema=table_schema_simple)) - assert bound_expression == base.AlwaysFalse() + unbound_expression = And(AlwaysTrue(), AlwaysFalse()) + bound_expression = visit(unbound_expression, visitor=BindVisitor(schema=table_schema_simple)) + assert bound_expression == AlwaysFalse() def test_always_false_or_always_true_expression_binding(table_schema_simple: Schema): """Test that visiting always-true OR always-false expression returns always-true""" - unbound_expression = base.Or(base.AlwaysTrue(), base.AlwaysFalse()) - bound_expression = base.visit(unbound_expression, visitor=base.BindVisitor(schema=table_schema_simple)) - assert bound_expression == base.AlwaysTrue() + unbound_expression = Or(AlwaysTrue(), AlwaysFalse()) + bound_expression = visit(unbound_expression, visitor=BindVisitor(schema=table_schema_simple)) + assert bound_expression == AlwaysTrue() @pytest.mark.parametrize( "unbound_and_expression,expected_bound_expression", [ ( - base.And( - base.In(base.Reference("foo"), (literal("foo"), literal("bar"))), - base.In(base.Reference("bar"), (literal(1), literal(2), literal(3))), + And( + In(Reference("foo"), (literal("foo"), literal("bar"))), + In(Reference("bar"), (literal(1), literal(2), literal(3))), ), - base.And( - base.BoundIn[str]( - base.BoundReference( + And( + BoundIn[str]( + BoundReference( field=NestedField(field_id=1, name="foo", field_type=StringType(), required=False), accessor=Accessor(position=0, inner=None), ), {StringLiteral("foo"), StringLiteral("bar")}, ), - base.BoundIn[int]( - base.BoundReference( + BoundIn[int]( + BoundReference( field=NestedField(field_id=2, name="bar", field_type=IntegerType(), required=True), accessor=Accessor(position=1, inner=None), ), @@ -924,36 +355,36 @@ def test_always_false_or_always_true_expression_binding(table_schema_simple: Sch ), ), ( - base.And( - base.In(base.Reference("foo"), (literal("bar"), literal("baz"))), - base.In( - base.Reference("bar"), + And( + In(Reference("foo"), (literal("bar"), literal("baz"))), + In( + Reference("bar"), (literal(1),), ), - base.In( - base.Reference("foo"), + In( + Reference("foo"), (literal("baz"),), ), ), - base.And( - base.And( - base.BoundIn[str]( - base.BoundReference( + And( + And( + BoundIn[str]( + BoundReference( field=NestedField(field_id=1, name="foo", field_type=StringType(), required=False), accessor=Accessor(position=0, inner=None), ), {StringLiteral("bar"), StringLiteral("baz")}, ), - base.BoundEqualTo[int]( - base.BoundReference( + BoundEqualTo[int]( + BoundReference( field=NestedField(field_id=2, name="bar", field_type=IntegerType(), required=True), accessor=Accessor(position=1, inner=None), ), LongLiteral(1), ), ), - base.BoundEqualTo[str]( - base.BoundReference( + BoundEqualTo[str]( + BoundReference( field=NestedField(field_id=1, name="foo", field_type=StringType(), required=False), accessor=Accessor(position=0, inner=None), ), @@ -965,7 +396,7 @@ def test_always_false_or_always_true_expression_binding(table_schema_simple: Sch ) def test_and_expression_binding(unbound_and_expression, expected_bound_expression, table_schema_simple): """Test that visiting an unbound AND expression with a bind-visitor returns the expected bound expression""" - bound_expression = base.visit(unbound_and_expression, visitor=base.BindVisitor(schema=table_schema_simple)) + bound_expression = visit(unbound_and_expression, visitor=BindVisitor(schema=table_schema_simple)) assert bound_expression == expected_bound_expression @@ -973,20 +404,20 @@ def test_and_expression_binding(unbound_and_expression, expected_bound_expressio "unbound_or_expression,expected_bound_expression", [ ( - base.Or( - base.In(base.Reference("foo"), (literal("foo"), literal("bar"))), - base.In(base.Reference("bar"), (literal(1), literal(2), literal(3))), + Or( + In(Reference("foo"), (literal("foo"), literal("bar"))), + In(Reference("bar"), (literal(1), literal(2), literal(3))), ), - base.Or( - base.BoundIn[str]( - base.BoundReference( + Or( + BoundIn[str]( + BoundReference( field=NestedField(field_id=1, name="foo", field_type=StringType(), required=False), accessor=Accessor(position=0, inner=None), ), {StringLiteral("foo"), StringLiteral("bar")}, ), - base.BoundIn[int]( - base.BoundReference( + BoundIn[int]( + BoundReference( field=NestedField(field_id=2, name="bar", field_type=IntegerType(), required=True), accessor=Accessor(position=1, inner=None), ), @@ -995,36 +426,36 @@ def test_and_expression_binding(unbound_and_expression, expected_bound_expressio ), ), ( - base.Or( - base.In(base.Reference("foo"), (literal("bar"), literal("baz"))), - base.In( - base.Reference("foo"), + Or( + In(Reference("foo"), (literal("bar"), literal("baz"))), + In( + Reference("foo"), (literal("bar"),), ), - base.In( - base.Reference("foo"), + In( + Reference("foo"), (literal("baz"),), ), ), - base.Or( - base.Or( - base.BoundIn[str]( - base.BoundReference( + Or( + Or( + BoundIn[str]( + BoundReference( field=NestedField(field_id=1, name="foo", field_type=StringType(), required=False), accessor=Accessor(position=0, inner=None), ), {StringLiteral("bar"), StringLiteral("baz")}, ), - base.BoundIn[str]( - base.BoundReference( + BoundIn[str]( + BoundReference( field=NestedField(field_id=1, name="foo", field_type=StringType(), required=False), accessor=Accessor(position=0, inner=None), ), {StringLiteral("bar")}, ), ), - base.BoundIn[str]( - base.BoundReference( + BoundIn[str]( + BoundReference( field=NestedField(field_id=1, name="foo", field_type=StringType(), required=False), accessor=Accessor(position=0, inner=None), ), @@ -1033,31 +464,31 @@ def test_and_expression_binding(unbound_and_expression, expected_bound_expressio ), ), ( - base.Or( - base.AlwaysTrue(), - base.AlwaysFalse(), + Or( + AlwaysTrue(), + AlwaysFalse(), ), - base.AlwaysTrue(), + AlwaysTrue(), ), ( - base.Or( - base.AlwaysTrue(), - base.AlwaysTrue(), + Or( + AlwaysTrue(), + AlwaysTrue(), ), - base.AlwaysTrue(), + AlwaysTrue(), ), ( - base.Or( - base.AlwaysFalse(), - base.AlwaysFalse(), + Or( + AlwaysFalse(), + AlwaysFalse(), ), - base.AlwaysFalse(), + AlwaysFalse(), ), ], ) def test_or_expression_binding(unbound_or_expression, expected_bound_expression, table_schema_simple): """Test that visiting an unbound OR expression with a bind-visitor returns the expected bound expression""" - bound_expression = base.visit(unbound_or_expression, visitor=base.BindVisitor(schema=table_schema_simple)) + bound_expression = visit(unbound_or_expression, visitor=BindVisitor(schema=table_schema_simple)) assert bound_expression == expected_bound_expression @@ -1065,9 +496,9 @@ def test_or_expression_binding(unbound_or_expression, expected_bound_expression, "unbound_in_expression,expected_bound_expression", [ ( - base.In(base.Reference("foo"), (literal("foo"), literal("bar"))), - base.BoundIn[str]( - base.BoundReference[str]( + In(Reference("foo"), (literal("foo"), literal("bar"))), + BoundIn[str]( + BoundReference[str]( field=NestedField(field_id=1, name="foo", field_type=StringType(), required=False), accessor=Accessor(position=0, inner=None), ), @@ -1075,9 +506,9 @@ def test_or_expression_binding(unbound_or_expression, expected_bound_expression, ), ), ( - base.In(base.Reference("foo"), (literal("bar"), literal("baz"))), - base.BoundIn[str]( - base.BoundReference[str]( + In(Reference("foo"), (literal("bar"), literal("baz"))), + BoundIn[str]( + BoundReference[str]( field=NestedField(field_id=1, name="foo", field_type=StringType(), required=False), accessor=Accessor(position=0, inner=None), ), @@ -1085,12 +516,12 @@ def test_or_expression_binding(unbound_or_expression, expected_bound_expression, ), ), ( - base.In( - base.Reference("foo"), + In( + Reference("foo"), (literal("bar"),), ), - base.BoundEqualTo( - base.BoundReference[str]( + BoundEqualTo( + BoundReference[str]( field=NestedField(field_id=1, name="foo", field_type=StringType(), required=False), accessor=Accessor(position=0, inner=None), ), @@ -1101,7 +532,7 @@ def test_or_expression_binding(unbound_or_expression, expected_bound_expression, ) def test_in_expression_binding(unbound_in_expression, expected_bound_expression, table_schema_simple): """Test that visiting an unbound IN expression with a bind-visitor returns the expected bound expression""" - bound_expression = base.visit(unbound_in_expression, visitor=base.BindVisitor(schema=table_schema_simple)) + bound_expression = visit(unbound_in_expression, visitor=BindVisitor(schema=table_schema_simple)) assert bound_expression == expected_bound_expression @@ -1109,10 +540,10 @@ def test_in_expression_binding(unbound_in_expression, expected_bound_expression, "unbound_not_expression,expected_bound_expression", [ ( - base.Not(base.In(base.Reference("foo"), (literal("foo"), literal("bar")))), - base.Not( - base.BoundIn[str]( - base.BoundReference[str]( + Not(In(Reference("foo"), (literal("foo"), literal("bar")))), + Not( + BoundIn[str]( + BoundReference[str]( field=NestedField(field_id=1, name="foo", field_type=StringType(), required=False), accessor=Accessor(position=0, inner=None), ), @@ -1121,23 +552,23 @@ def test_in_expression_binding(unbound_in_expression, expected_bound_expression, ), ), ( - base.Not( - base.Or( - base.In(base.Reference("foo"), (literal("foo"), literal("bar"))), - base.In(base.Reference("foo"), (literal("foo"), literal("bar"), literal("baz"))), + Not( + Or( + In(Reference("foo"), (literal("foo"), literal("bar"))), + In(Reference("foo"), (literal("foo"), literal("bar"), literal("baz"))), ) ), - base.Not( - base.Or( - base.BoundIn[str]( - base.BoundReference( + Not( + Or( + BoundIn[str]( + BoundReference( field=NestedField(field_id=1, name="foo", field_type=StringType(), required=False), accessor=Accessor(position=0, inner=None), ), {StringLiteral("foo"), StringLiteral("bar")}, ), - base.BoundIn[str]( - base.BoundReference( + BoundIn[str]( + BoundReference( field=NestedField(field_id=1, name="foo", field_type=StringType(), required=False), accessor=Accessor(position=0, inner=None), ), @@ -1150,22 +581,22 @@ def test_in_expression_binding(unbound_in_expression, expected_bound_expression, ) def test_not_expression_binding(unbound_not_expression, expected_bound_expression, table_schema_simple): """Test that visiting an unbound NOT expression with a bind-visitor returns the expected bound expression""" - bound_expression = base.visit(unbound_not_expression, visitor=base.BindVisitor(schema=table_schema_simple)) + bound_expression = visit(unbound_not_expression, visitor=BindVisitor(schema=table_schema_simple)) assert bound_expression == expected_bound_expression def test_bound_boolean_expression_visitor_and_in(): """Test visiting an And and In expression with a bound boolean expression visitor""" - bound_expression = base.And( - base.BoundIn[str]( - term=base.BoundReference( + bound_expression = And( + BoundIn[str]( + term=BoundReference( field=NestedField(field_id=1, name="foo", field_type=StringType(), required=False), accessor=Accessor(position=0, inner=None), ), literals=(StringLiteral("foo"), StringLiteral("bar")), ), - base.BoundIn[str]( - term=base.BoundReference( + BoundIn[str]( + term=BoundReference( field=NestedField(field_id=2, name="bar", field_type=StringType(), required=False), accessor=Accessor(position=1, inner=None), ), @@ -1173,25 +604,25 @@ def test_bound_boolean_expression_visitor_and_in(): ), ) visitor = FooBoundBooleanExpressionVisitor() - result = base.visit(bound_expression, visitor=visitor) + result = visit(bound_expression, visitor=visitor) assert result == ["IN", "IN", "AND"] def test_bound_boolean_expression_visitor_or(): """Test visiting an Or expression with a bound boolean expression visitor""" - bound_expression = base.Or( - base.Not( - base.BoundIn[str]( - base.BoundReference[str]( + bound_expression = Or( + Not( + BoundIn[str]( + BoundReference[str]( field=NestedField(field_id=1, name="foo", field_type=StringType(), required=False), accessor=Accessor(position=0, inner=None), ), {StringLiteral("foo"), StringLiteral("bar")}, ) ), - base.Not( - base.BoundIn[str]( - base.BoundReference[str]( + Not( + BoundIn[str]( + BoundReference[str]( field=NestedField(field_id=2, name="bar", field_type=StringType(), required=False), accessor=Accessor(position=1, inner=None), ), @@ -1200,184 +631,184 @@ def test_bound_boolean_expression_visitor_or(): ), ) visitor = FooBoundBooleanExpressionVisitor() - result = base.visit(bound_expression, visitor=visitor) + result = visit(bound_expression, visitor=visitor) assert result == ["IN", "NOT", "IN", "NOT", "OR"] def test_bound_boolean_expression_visitor_equal(): - bound_expression = base.BoundEqualTo[str]( - term=base.BoundReference( + bound_expression = BoundEqualTo[str]( + term=BoundReference( field=NestedField(field_id=2, name="bar", field_type=StringType(), required=False), accessor=Accessor(position=1, inner=None), ), literal=StringLiteral("foo"), ) visitor = FooBoundBooleanExpressionVisitor() - result = base.visit(bound_expression, visitor=visitor) + result = visit(bound_expression, visitor=visitor) assert result == ["EQUAL"] def test_bound_boolean_expression_visitor_not_equal(): - bound_expression = base.BoundNotEqualTo[str]( - term=base.BoundReference( + bound_expression = BoundNotEqualTo[str]( + term=BoundReference( field=NestedField(field_id=1, name="foo", field_type=StringType(), required=False), accessor=Accessor(position=0, inner=None), ), literal=StringLiteral("foo"), ) visitor = FooBoundBooleanExpressionVisitor() - result = base.visit(bound_expression, visitor=visitor) + result = visit(bound_expression, visitor=visitor) assert result == ["NOT_EQUAL"] def test_bound_boolean_expression_visitor_always_true(): - bound_expression = base.AlwaysTrue() + bound_expression = AlwaysTrue() visitor = FooBoundBooleanExpressionVisitor() - result = base.visit(bound_expression, visitor=visitor) + result = visit(bound_expression, visitor=visitor) assert result == ["TRUE"] def test_bound_boolean_expression_visitor_always_false(): - bound_expression = base.AlwaysFalse() + bound_expression = AlwaysFalse() visitor = FooBoundBooleanExpressionVisitor() - result = base.visit(bound_expression, visitor=visitor) + result = visit(bound_expression, visitor=visitor) assert result == ["FALSE"] def test_bound_boolean_expression_visitor_in(): - bound_expression = base.BoundIn[str]( - term=base.BoundReference( + bound_expression = BoundIn[str]( + term=BoundReference( field=NestedField(field_id=1, name="foo", field_type=StringType(), required=False), accessor=Accessor(position=0, inner=None), ), literals=(StringLiteral("foo"), StringLiteral("bar")), ) visitor = FooBoundBooleanExpressionVisitor() - result = base.visit(bound_expression, visitor=visitor) + result = visit(bound_expression, visitor=visitor) assert result == ["IN"] def test_bound_boolean_expression_visitor_not_in(): - bound_expression = base.BoundNotIn[str]( - term=base.BoundReference( + bound_expression = BoundNotIn[str]( + term=BoundReference( field=NestedField(field_id=1, name="foo", field_type=StringType(), required=False), accessor=Accessor(position=0, inner=None), ), literals=(StringLiteral("foo"), StringLiteral("bar")), ) visitor = FooBoundBooleanExpressionVisitor() - result = base.visit(bound_expression, visitor=visitor) + result = visit(bound_expression, visitor=visitor) assert result == ["NOT_IN"] def test_bound_boolean_expression_visitor_is_nan(): - bound_expression = base.BoundIsNaN[str]( - term=base.BoundReference( + bound_expression = BoundIsNaN[str]( + term=BoundReference( field=NestedField(field_id=3, name="baz", field_type=FloatType(), required=False), accessor=Accessor(position=0, inner=None), ), ) visitor = FooBoundBooleanExpressionVisitor() - result = base.visit(bound_expression, visitor=visitor) + result = visit(bound_expression, visitor=visitor) assert result == ["IS_NAN"] def test_bound_boolean_expression_visitor_not_nan(): - bound_expression = base.BoundNotNaN[str]( - term=base.BoundReference( + bound_expression = BoundNotNaN[str]( + term=BoundReference( field=NestedField(field_id=3, name="baz", field_type=FloatType(), required=False), accessor=Accessor(position=0, inner=None), ), ) visitor = FooBoundBooleanExpressionVisitor() - result = base.visit(bound_expression, visitor=visitor) + result = visit(bound_expression, visitor=visitor) assert result == ["NOT_NAN"] def test_bound_boolean_expression_visitor_is_null(): - bound_expression = base.BoundIsNull[str]( - term=base.BoundReference( + bound_expression = BoundIsNull[str]( + term=BoundReference( field=NestedField(field_id=1, name="foo", field_type=StringType(), required=False), accessor=Accessor(position=0, inner=None), ), ) visitor = FooBoundBooleanExpressionVisitor() - result = base.visit(bound_expression, visitor=visitor) + result = visit(bound_expression, visitor=visitor) assert result == ["IS_NULL"] def test_bound_boolean_expression_visitor_not_null(): - bound_expression = base.BoundNotNull[str]( - term=base.BoundReference( + bound_expression = BoundNotNull[str]( + term=BoundReference( field=NestedField(field_id=1, name="foo", field_type=StringType(), required=False), accessor=Accessor(position=0, inner=None), ), ) visitor = FooBoundBooleanExpressionVisitor() - result = base.visit(bound_expression, visitor=visitor) + result = visit(bound_expression, visitor=visitor) assert result == ["NOT_NULL"] def test_bound_boolean_expression_visitor_greater_than(): - bound_expression = base.BoundGreaterThan[str]( - term=base.BoundReference( + bound_expression = BoundGreaterThan[str]( + term=BoundReference( field=NestedField(field_id=1, name="foo", field_type=StringType(), required=False), accessor=Accessor(position=0, inner=None), ), literal=StringLiteral("foo"), ) visitor = FooBoundBooleanExpressionVisitor() - result = base.visit(bound_expression, visitor=visitor) + result = visit(bound_expression, visitor=visitor) assert result == ["GREATER_THAN"] def test_bound_boolean_expression_visitor_greater_than_or_equal(): - bound_expression = base.BoundGreaterThanOrEqual[str]( - term=base.BoundReference( + bound_expression = BoundGreaterThanOrEqual[str]( + term=BoundReference( field=NestedField(field_id=1, name="foo", field_type=StringType(), required=False), accessor=Accessor(position=0, inner=None), ), literal=StringLiteral("foo"), ) visitor = FooBoundBooleanExpressionVisitor() - result = base.visit(bound_expression, visitor=visitor) + result = visit(bound_expression, visitor=visitor) assert result == ["GREATER_THAN_OR_EQUAL"] def test_bound_boolean_expression_visitor_less_than(): - bound_expression = base.BoundLessThan[str]( - term=base.BoundReference( + bound_expression = BoundLessThan[str]( + term=BoundReference( field=NestedField(field_id=1, name="foo", field_type=StringType(), required=False), accessor=Accessor(position=0, inner=None), ), literal=StringLiteral("foo"), ) visitor = FooBoundBooleanExpressionVisitor() - result = base.visit(bound_expression, visitor=visitor) + result = visit(bound_expression, visitor=visitor) assert result == ["LESS_THAN"] def test_bound_boolean_expression_visitor_less_than_or_equal(): - bound_expression = base.BoundLessThanOrEqual[str]( - term=base.BoundReference( + bound_expression = BoundLessThanOrEqual[str]( + term=BoundReference( field=NestedField(field_id=1, name="foo", field_type=StringType(), required=False), accessor=Accessor(position=0, inner=None), ), literal=StringLiteral("foo"), ) visitor = FooBoundBooleanExpressionVisitor() - result = base.visit(bound_expression, visitor=visitor) + result = visit(bound_expression, visitor=visitor) assert result == ["LESS_THAN_OR_EQUAL"] def test_bound_boolean_expression_visitor_raise_on_unbound_predicate(): - bound_expression = base.LessThanOrEqual[str]( - term=base.Reference("foo"), + bound_expression = LessThanOrEqual[str]( + term=Reference("foo"), literal=StringLiteral("foo"), ) visitor = FooBoundBooleanExpressionVisitor() with pytest.raises(TypeError) as exc_info: - base.visit(bound_expression, visitor=visitor) + visit(bound_expression, visitor=visitor) assert "Not a bound predicate" in str(exc_info.value) @@ -1399,15 +830,15 @@ def _to_manifest_file(*partitions: PartitionFieldSummary) -> ManifestFile: def _create_manifest_evaluator(bound_expr: BoundPredicate) -> _ManifestEvalVisitor: """For testing. Creates a bogus evaluator, and then replaces the expression""" evaluator = _ManifestEvalVisitor( - Schema(NestedField(1, "id", LongType())), base.EqualTo(term=base.Reference("id"), literal=literal("foo")) + Schema(NestedField(1, "id", LongType())), EqualTo(term=Reference("id"), literal=literal("foo")) ) evaluator.partition_filter = bound_expr return evaluator def test_manifest_evaluator_less_than_no_overlap(): - expr = base.BoundLessThan( - term=base.BoundReference( + expr = BoundLessThan( + term=BoundReference( field=NestedField(field_id=1, name="foo", field_type=StringType(), required=False), accessor=Accessor(position=0, inner=None), ), @@ -1427,8 +858,8 @@ def test_manifest_evaluator_less_than_no_overlap(): def test_manifest_evaluator_less_than_overlap(): - expr = base.BoundLessThan( - term=base.BoundReference( + expr = BoundLessThan( + term=BoundReference( field=NestedField(field_id=1, name="foo", field_type=StringType(), required=False), accessor=Accessor(position=0, inner=None), ), @@ -1448,8 +879,8 @@ def test_manifest_evaluator_less_than_overlap(): def test_manifest_evaluator_less_than_all_null(): - expr = base.BoundLessThan( - term=base.BoundReference( + expr = BoundLessThan( + term=BoundReference( field=NestedField(field_id=1, name="foo", field_type=StringType(), required=False), accessor=Accessor(position=0, inner=None), ), @@ -1465,8 +896,8 @@ def test_manifest_evaluator_less_than_all_null(): def test_manifest_evaluator_less_than_no_match(): - expr = base.BoundLessThan( - term=base.BoundReference( + expr = BoundLessThan( + term=BoundReference( field=NestedField(field_id=1, name="foo", field_type=StringType(), required=False), accessor=Accessor(position=0, inner=None), ), @@ -1486,8 +917,8 @@ def test_manifest_evaluator_less_than_no_match(): def test_manifest_evaluator_less_than_or_equal_no_overlap(): - expr = base.BoundLessThanOrEqual( - term=base.BoundReference( + expr = BoundLessThanOrEqual( + term=BoundReference( field=NestedField(field_id=1, name="foo", field_type=StringType(), required=False), accessor=Accessor(position=0, inner=None), ), @@ -1507,8 +938,8 @@ def test_manifest_evaluator_less_than_or_equal_no_overlap(): def test_manifest_evaluator_less_than_or_equal_overlap(): - expr = base.BoundLessThanOrEqual( - term=base.BoundReference( + expr = BoundLessThanOrEqual( + term=BoundReference( field=NestedField(field_id=1, name="foo", field_type=StringType(), required=False), accessor=Accessor(position=0, inner=None), ), @@ -1528,8 +959,8 @@ def test_manifest_evaluator_less_than_or_equal_overlap(): def test_manifest_evaluator_less_than_or_equal_all_null(): - expr = base.BoundLessThanOrEqual( - term=base.BoundReference( + expr = BoundLessThanOrEqual( + term=BoundReference( field=NestedField(field_id=1, name="foo", field_type=StringType(), required=False), accessor=Accessor(position=0, inner=None), ), @@ -1545,8 +976,8 @@ def test_manifest_evaluator_less_than_or_equal_all_null(): def test_manifest_evaluator_less_than_or_equal_no_match(): - expr = base.BoundLessThanOrEqual( - term=base.BoundReference( + expr = BoundLessThanOrEqual( + term=BoundReference( field=NestedField(field_id=1, name="foo", field_type=StringType(), required=False), accessor=Accessor(position=0, inner=None), ), @@ -1566,8 +997,8 @@ def test_manifest_evaluator_less_than_or_equal_no_match(): def test_manifest_evaluator_equal_no_overlap(): - expr = base.BoundEqualTo( - term=base.BoundReference( + expr = BoundEqualTo( + term=BoundReference( field=NestedField(field_id=1, name="foo", field_type=StringType(), required=False), accessor=Accessor(position=0, inner=None), ), @@ -1587,8 +1018,8 @@ def test_manifest_evaluator_equal_no_overlap(): def test_manifest_evaluator_equal_overlap(): - expr = base.BoundEqualTo( - term=base.BoundReference( + expr = BoundEqualTo( + term=BoundReference( field=NestedField(field_id=1, name="foo", field_type=StringType(), required=False), accessor=Accessor(position=0, inner=None), ), @@ -1613,8 +1044,8 @@ def test_manifest_evaluator_equal_overlap(): def test_manifest_evaluator_equal_all_null(): - expr = base.BoundEqualTo( - term=base.BoundReference( + expr = BoundEqualTo( + term=BoundReference( field=NestedField(field_id=1, name="foo", field_type=StringType(), required=False), accessor=Accessor(position=0, inner=None), ), @@ -1630,8 +1061,8 @@ def test_manifest_evaluator_equal_all_null(): def test_manifest_evaluator_equal_no_match(): - expr = base.BoundEqualTo( - term=base.BoundReference( + expr = BoundEqualTo( + term=BoundReference( field=NestedField(field_id=1, name="foo", field_type=StringType(), required=False), accessor=Accessor(position=0, inner=None), ), @@ -1651,8 +1082,8 @@ def test_manifest_evaluator_equal_no_match(): def test_manifest_evaluator_greater_than_no_overlap(): - expr = base.BoundGreaterThan( - term=base.BoundReference( + expr = BoundGreaterThan( + term=BoundReference( field=NestedField(field_id=1, name="foo", field_type=StringType(), required=False), accessor=Accessor(position=0, inner=None), ), @@ -1672,8 +1103,8 @@ def test_manifest_evaluator_greater_than_no_overlap(): def test_manifest_evaluator_greater_than_overlap(): - expr = base.BoundGreaterThan( - term=base.BoundReference( + expr = BoundGreaterThan( + term=BoundReference( field=NestedField(field_id=1, name="foo", field_type=StringType(), required=False), accessor=Accessor(position=0, inner=None), ), @@ -1693,8 +1124,8 @@ def test_manifest_evaluator_greater_than_overlap(): def test_manifest_evaluator_greater_than_all_null(): - expr = base.BoundGreaterThan( - term=base.BoundReference( + expr = BoundGreaterThan( + term=BoundReference( field=NestedField(field_id=1, name="foo", field_type=StringType(), required=False), accessor=Accessor(position=0, inner=None), ), @@ -1710,8 +1141,8 @@ def test_manifest_evaluator_greater_than_all_null(): def test_manifest_evaluator_greater_than_no_match(): - expr = base.BoundGreaterThan( - term=base.BoundReference( + expr = BoundGreaterThan( + term=BoundReference( field=NestedField(field_id=1, name="foo", field_type=StringType(), required=False), accessor=Accessor(position=0, inner=None), ), @@ -1731,8 +1162,8 @@ def test_manifest_evaluator_greater_than_no_match(): def test_manifest_evaluator_greater_than_or_equal_no_overlap(): - expr = base.BoundGreaterThanOrEqual( - term=base.BoundReference( + expr = BoundGreaterThanOrEqual( + term=BoundReference( field=NestedField(field_id=1, name="foo", field_type=StringType(), required=False), accessor=Accessor(position=0, inner=None), ), @@ -1752,8 +1183,8 @@ def test_manifest_evaluator_greater_than_or_equal_no_overlap(): def test_manifest_evaluator_greater_than_or_equal_overlap(): - expr = base.BoundGreaterThanOrEqual( - term=base.BoundReference( + expr = BoundGreaterThanOrEqual( + term=BoundReference( field=NestedField(field_id=1, name="foo", field_type=StringType(), required=False), accessor=Accessor(position=0, inner=None), ), @@ -1773,8 +1204,8 @@ def test_manifest_evaluator_greater_than_or_equal_overlap(): def test_manifest_evaluator_greater_than_or_equal_all_null(): - expr = base.BoundGreaterThanOrEqual( - term=base.BoundReference( + expr = BoundGreaterThanOrEqual( + term=BoundReference( field=NestedField(field_id=1, name="foo", field_type=StringType(), required=False), accessor=Accessor(position=0, inner=None), ), @@ -1790,8 +1221,8 @@ def test_manifest_evaluator_greater_than_or_equal_all_null(): def test_manifest_evaluator_greater_than_or_equal_no_match(): - expr = base.BoundGreaterThanOrEqual( - term=base.BoundReference( + expr = BoundGreaterThanOrEqual( + term=BoundReference( field=NestedField(field_id=1, name="foo", field_type=StringType(), required=False), accessor=Accessor(position=0, inner=None), ), @@ -1811,8 +1242,8 @@ def test_manifest_evaluator_greater_than_or_equal_no_match(): def test_manifest_evaluator_is_nan(): - expr = base.BoundIsNaN( - term=base.BoundReference( + expr = BoundIsNaN( + term=BoundReference( field=NestedField(field_id=1, name="foo", field_type=DoubleType(), required=False), accessor=Accessor(position=0, inner=None), ) @@ -1826,8 +1257,8 @@ def test_manifest_evaluator_is_nan(): def test_manifest_evaluator_is_nan_inverse(): - expr = base.BoundIsNaN( - term=base.BoundReference( + expr = BoundIsNaN( + term=BoundReference( field=NestedField(field_id=1, name="foo", field_type=DoubleType(), required=False), accessor=Accessor(position=0, inner=None), ) @@ -1846,8 +1277,8 @@ def test_manifest_evaluator_is_nan_inverse(): def test_manifest_evaluator_not_nan(): - expr = base.BoundNotNaN( - term=base.BoundReference( + expr = BoundNotNaN( + term=BoundReference( field=NestedField(field_id=1, name="foo", field_type=DoubleType(), required=False), accessor=Accessor(position=0, inner=None), ) @@ -1866,8 +1297,8 @@ def test_manifest_evaluator_not_nan(): def test_manifest_evaluator_not_nan_inverse(): - expr = base.BoundNotNaN( - term=base.BoundReference( + expr = BoundNotNaN( + term=BoundReference( field=NestedField(field_id=1, name="foo", field_type=DoubleType(), required=False), accessor=Accessor(position=0, inner=None), ) @@ -1881,8 +1312,8 @@ def test_manifest_evaluator_not_nan_inverse(): def test_manifest_evaluator_is_null(): - expr = base.BoundIsNull( - term=base.BoundReference( + expr = BoundIsNull( + term=BoundReference( field=NestedField(field_id=1, name="foo", field_type=StringType(), required=False), accessor=Accessor(position=0, inner=None), ), @@ -1901,8 +1332,8 @@ def test_manifest_evaluator_is_null(): def test_manifest_evaluator_is_null_inverse(): - expr = base.BoundIsNull( - term=base.BoundReference( + expr = BoundIsNull( + term=BoundReference( field=NestedField(field_id=1, name="foo", field_type=StringType(), required=False), accessor=Accessor(position=0, inner=None), ), @@ -1916,8 +1347,8 @@ def test_manifest_evaluator_is_null_inverse(): def test_manifest_evaluator_not_null(): - expr = base.BoundNotNull( - term=base.BoundReference( + expr = BoundNotNull( + term=BoundReference( field=NestedField(field_id=1, name="foo", field_type=StringType(), required=False), accessor=Accessor(position=0, inner=None), ), @@ -1931,8 +1362,8 @@ def test_manifest_evaluator_not_null(): def test_manifest_evaluator_not_null_nan(): - expr = base.BoundNotNull( - term=base.BoundReference( + expr = BoundNotNull( + term=BoundReference( field=NestedField(field_id=1, name="foo", field_type=DoubleType(), required=False), accessor=Accessor(position=0, inner=None), ), @@ -1946,8 +1377,8 @@ def test_manifest_evaluator_not_null_nan(): def test_manifest_evaluator_not_null_inverse(): - expr = base.BoundNotNull( - term=base.BoundReference( + expr = BoundNotNull( + term=BoundReference( field=NestedField(field_id=1, name="foo", field_type=StringType(), required=False), accessor=Accessor(position=0, inner=None), ), @@ -1959,8 +1390,8 @@ def test_manifest_evaluator_not_null_inverse(): def test_manifest_evaluator_not_equal_to(): - expr = base.BoundNotEqualTo( - term=base.BoundReference( + expr = BoundNotEqualTo( + term=BoundReference( field=NestedField(field_id=1, name="foo", field_type=StringType(), required=False), accessor=Accessor(position=0, inner=None), ), @@ -1973,8 +1404,8 @@ def test_manifest_evaluator_not_equal_to(): def test_manifest_evaluator_in(): - expr = base.BoundIn( - term=base.BoundReference( + expr = BoundIn( + term=BoundReference( field=NestedField(field_id=1, name="foo", field_type=LongType(), required=False), accessor=Accessor(position=0, inner=None), ), @@ -1994,8 +1425,8 @@ def test_manifest_evaluator_in(): def test_manifest_evaluator_not_in(): - expr = base.BoundNotIn( - term=base.BoundReference( + expr = BoundNotIn( + term=BoundReference( field=NestedField(field_id=1, name="foo", field_type=LongType(), required=False), accessor=Accessor(position=0, inner=None), ), @@ -2015,8 +1446,8 @@ def test_manifest_evaluator_not_in(): def test_manifest_evaluator_in_null(): - expr = base.BoundIn( - term=base.BoundReference( + expr = BoundIn( + term=BoundReference( field=NestedField(field_id=1, name="foo", field_type=LongType(), required=False), accessor=Accessor(position=0, inner=None), ), @@ -2036,8 +1467,8 @@ def test_manifest_evaluator_in_null(): def test_manifest_evaluator_in_inverse(): - expr = base.BoundIn( - term=base.BoundReference( + expr = BoundIn( + term=BoundReference( field=NestedField(field_id=1, name="foo", field_type=LongType(), required=False), accessor=Accessor(position=0, inner=None), ), @@ -2057,8 +1488,8 @@ def test_manifest_evaluator_in_inverse(): def test_manifest_evaluator_in_overflow(): - expr = base.BoundIn( - term=base.BoundReference( + expr = BoundIn( + term=BoundReference( field=NestedField(field_id=1, name="foo", field_type=LongType(), required=False), accessor=Accessor(position=0, inner=None), ), @@ -2078,8 +1509,8 @@ def test_manifest_evaluator_in_overflow(): def test_manifest_evaluator_less_than_lower_bound(): - expr = base.BoundIn( - term=base.BoundReference( + expr = BoundIn( + term=BoundReference( field=NestedField(field_id=1, name="foo", field_type=LongType(), required=False), accessor=Accessor(position=0, inner=None), ), @@ -2099,8 +1530,8 @@ def test_manifest_evaluator_less_than_lower_bound(): def test_manifest_evaluator_greater_than_upper_bound(): - expr = base.BoundIn( - term=base.BoundReference( + expr = BoundIn( + term=BoundReference( field=NestedField(field_id=1, name="foo", field_type=LongType(), required=False), accessor=Accessor(position=0, inner=None), ), @@ -2120,7 +1551,7 @@ def test_manifest_evaluator_greater_than_upper_bound(): def test_manifest_evaluator_true(): - expr = base.AlwaysTrue() + expr = AlwaysTrue() manifest = _to_manifest_file(PartitionFieldSummary(contains_null=True, contains_nan=True, lower_bound=None, upper_bound=None)) @@ -2128,7 +1559,7 @@ def test_manifest_evaluator_true(): def test_manifest_evaluator_false(): - expr = base.AlwaysFalse() + expr = AlwaysFalse() manifest = _to_manifest_file(PartitionFieldSummary(contains_null=True, contains_nan=True, lower_bound=None, upper_bound=None)) @@ -2136,9 +1567,9 @@ def test_manifest_evaluator_false(): def test_manifest_evaluator_not(): - expr = base.Not( - base.BoundIn( - term=base.BoundReference( + expr = Not( + BoundIn( + term=BoundReference( field=NestedField(field_id=1, name="foo", field_type=LongType(), required=False), accessor=Accessor(position=0, inner=None), ), @@ -2158,16 +1589,16 @@ def test_manifest_evaluator_not(): def test_manifest_evaluator_and(): - expr = base.And( - base.BoundIn( - term=base.BoundReference( + expr = And( + BoundIn( + term=BoundReference( field=NestedField(field_id=1, name="foo", field_type=LongType(), required=False), accessor=Accessor(position=0, inner=None), ), literals={LongLiteral(i) for i in range(22)}, ), - base.BoundIn( - term=base.BoundReference( + BoundIn( + term=BoundReference( field=NestedField(field_id=1, name="foo", field_type=LongType(), required=False), accessor=Accessor(position=0, inner=None), ), @@ -2187,16 +1618,16 @@ def test_manifest_evaluator_and(): def test_manifest_evaluator_or(): - expr = base.Or( - base.BoundIn( - term=base.BoundReference( + expr = Or( + BoundIn( + term=BoundReference( field=NestedField(field_id=1, name="foo", field_type=LongType(), required=False), accessor=Accessor(position=0, inner=None), ), literals={LongLiteral(i) for i in range(22)}, ), - base.BoundIn( - term=base.BoundReference( + BoundIn( + term=BoundReference( field=NestedField(field_id=1, name="foo", field_type=LongType(), required=False), accessor=Accessor(position=0, inner=None), ), @@ -2218,8 +1649,8 @@ def test_manifest_evaluator_or(): def test_bound_predicate_invert(): with pytest.raises(NotImplementedError): - _ = ~base.BoundPredicate( - term=base.BoundReference( + _ = ~BoundPredicate( + term=BoundReference( field=NestedField(field_id=1, name="foo", field_type=StringType(), required=False), accessor=Accessor(position=0, inner=None), ) @@ -2228,8 +1659,8 @@ def test_bound_predicate_invert(): def test_bound_unary_predicate_invert(): with pytest.raises(NotImplementedError): - _ = ~base.BoundUnaryPredicate( - term=base.BoundReference( + _ = ~BoundUnaryPredicate( + term=BoundReference( field=NestedField(field_id=1, name="foo", field_type=StringType(), required=False), accessor=Accessor(position=0, inner=None), ) @@ -2238,8 +1669,8 @@ def test_bound_unary_predicate_invert(): def test_bound_set_predicate_invert(): with pytest.raises(NotImplementedError): - _ = ~base.BoundSetPredicate( - term=base.BoundReference( + _ = ~BoundSetPredicate( + term=BoundReference( field=NestedField(field_id=1, name="foo", field_type=StringType(), required=False), accessor=Accessor(position=0, inner=None), ), @@ -2249,8 +1680,8 @@ def test_bound_set_predicate_invert(): def test_bound_literal_predicate_invert(): with pytest.raises(NotImplementedError): - _ = ~base.BoundLiteralPredicate( - term=base.BoundReference( + _ = ~BoundLiteralPredicate( + term=BoundReference( field=NestedField(field_id=1, name="foo", field_type=StringType(), required=False), accessor=Accessor(position=0, inner=None), ), @@ -2258,96 +1689,69 @@ def test_bound_literal_predicate_invert(): ) -def test_non_primitive_from_byte_buffer(): - with pytest.raises(ValueError) as exc_info: - _ = _from_byte_buffer(ListType(element_id=1, element_type=StringType()), b"\0x00") - - assert str(exc_info.value) == "Expected a PrimitiveType, got: " - - def test_unbound_predicate_invert(): with pytest.raises(NotImplementedError): - _ = ~base.UnboundPredicate(term=base.Reference("a")) + _ = ~UnboundPredicate(term=Reference("a")) def test_unbound_predicate_bind(table_schema_simple: Schema): with pytest.raises(NotImplementedError): - _ = base.UnboundPredicate(term=base.Reference("a")).bind(table_schema_simple) + _ = UnboundPredicate(term=Reference("a")).bind(table_schema_simple) def test_unbound_unary_predicate_invert(): with pytest.raises(NotImplementedError): - _ = ~base.UnaryPredicate(term=base.Reference("a")) + _ = ~UnaryPredicate(term=Reference("a")) def test_unbound_set_predicate_invert(): with pytest.raises(NotImplementedError): - _ = ~base.SetPredicate(term=base.Reference("a"), literals=(literal("hello"), literal("world"))) + _ = ~SetPredicate(term=Reference("a"), literals=(literal("hello"), literal("world"))) def test_unbound_literal_predicate_invert(): with pytest.raises(NotImplementedError): - _ = ~base.LiteralPredicate(term=base.Reference("a"), literal=literal("hello")) + _ = ~LiteralPredicate(term=Reference("a"), literal=literal("hello")) def test_rewrite_not_equal_to(): - assert rewrite_not(base.Not(base.EqualTo(base.Reference("x"), literal(34.56)))) == base.NotEqualTo( - base.Reference("x"), literal(34.56) - ) + assert rewrite_not(Not(EqualTo(Reference("x"), literal(34.56)))) == NotEqualTo(Reference("x"), literal(34.56)) def test_rewrite_not_not_equal_to(): - assert rewrite_not(base.Not(base.NotEqualTo(base.Reference("x"), literal(34.56)))) == base.EqualTo( - base.Reference("x"), literal(34.56) - ) + assert rewrite_not(Not(NotEqualTo(Reference("x"), literal(34.56)))) == EqualTo(Reference("x"), literal(34.56)) def test_rewrite_not_in(): - assert rewrite_not(base.Not(base.In(base.Reference("x"), (literal(34.56),)))) == base.NotIn( - base.Reference("x"), (literal(34.56),) - ) + assert rewrite_not(Not(In(Reference("x"), (literal(34.56),)))) == NotIn(Reference("x"), (literal(34.56),)) def test_rewrite_and(): - assert rewrite_not( - base.Not( - base.And( - base.EqualTo(base.Reference("x"), literal(34.56)), - base.EqualTo(base.Reference("y"), literal(34.56)), - ) - ) - ) == base.Or( - base.NotEqualTo(term=base.Reference(name="x"), literal=literal(34.56)), - base.NotEqualTo(term=base.Reference(name="y"), literal=literal(34.56)), + assert rewrite_not(Not(And(EqualTo(Reference("x"), literal(34.56)), EqualTo(Reference("y"), literal(34.56)),))) == Or( + NotEqualTo(term=Reference(name="x"), literal=literal(34.56)), + NotEqualTo(term=Reference(name="y"), literal=literal(34.56)), ) def test_rewrite_or(): - assert rewrite_not( - base.Not( - base.Or( - base.EqualTo(base.Reference("x"), literal(34.56)), - base.EqualTo(base.Reference("y"), literal(34.56)), - ) - ) - ) == base.And( - base.NotEqualTo(term=base.Reference(name="x"), literal=literal(34.56)), - base.NotEqualTo(term=base.Reference(name="y"), literal=literal(34.56)), + assert rewrite_not(Not(Or(EqualTo(Reference("x"), literal(34.56)), EqualTo(Reference("y"), literal(34.56)),))) == And( + NotEqualTo(term=Reference(name="x"), literal=literal(34.56)), + NotEqualTo(term=Reference(name="y"), literal=literal(34.56)), ) def test_rewrite_always_false(): - assert rewrite_not(base.Not(base.AlwaysFalse())) == base.AlwaysTrue() + assert rewrite_not(Not(AlwaysFalse())) == AlwaysTrue() def test_rewrite_always_true(): - assert rewrite_not(base.Not(base.AlwaysTrue())) == base.AlwaysFalse() + assert rewrite_not(Not(AlwaysTrue())) == AlwaysFalse() def test_rewrite_bound(): schema = Schema(NestedField(2, "a", IntegerType(), required=False), schema_id=1) - assert rewrite_not(base.IsNull(base.Reference("a")).bind(schema)) == base.BoundIsNull( - term=base.BoundReference( + assert rewrite_not(IsNull(Reference("a")).bind(schema)) == BoundIsNull( + term=BoundReference( field=NestedField(field_id=2, name="a", field_type=IntegerType(), required=False), accessor=Accessor(position=0, inner=None), ) diff --git a/tests/test_schema.py b/tests/test_schema.py index 0017760ed2..5b64d95d6b 100644 --- a/tests/test_schema.py +++ b/tests/test_schema.py @@ -21,7 +21,7 @@ import pytest from pyiceberg import schema -from pyiceberg.expressions.base import Accessor +from pyiceberg.expressions import Accessor from pyiceberg.files import StructProtocol from pyiceberg.schema import Schema, build_position_accessors, prune_columns from pyiceberg.typedef import EMPTY_DICT From 9ae5dbdf8fa28f06344afa1dc7d65e78df4b5346 Mon Sep 17 00:00:00 2001 From: Fokko Driesprong Date: Fri, 21 Oct 2022 22:06:25 +0200 Subject: [PATCH 243/642] Python: Visitor to convert Iceberg to PyArrow schema (#5949) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This is required for manually specifying datasets: https://arrow.apache.org/docs/python/dataset.html#manual-specification-of-the-dataset From PyArrow: The dataset() function allows easy creation of a Dataset viewing a directory, crawling all subdirectories for files and partitioning information. However sometimes discovery is not required and the dataset’s files and partitions are already known (for example, when this information is stored in metadata). In this case it is possible to create a Dataset explicitly without any automatic discovery or inference. --- pyiceberg/io/pyarrow.py | 134 ++++++++++++++++++++++++++++++++++- tests/io/test_pyarrow.py | 149 ++++++++++++++++++++++++++++++++++++++- 2 files changed, 280 insertions(+), 3 deletions(-) diff --git a/pyiceberg/io/pyarrow.py b/pyiceberg/io/pyarrow.py index dd82bbfb77..248f3603a3 100644 --- a/pyiceberg/io/pyarrow.py +++ b/pyiceberg/io/pyarrow.py @@ -23,10 +23,16 @@ """ import os -from functools import lru_cache -from typing import Callable, Tuple, Union +from functools import lru_cache, singledispatch +from typing import ( + Callable, + List, + Tuple, + Union, +) from urllib.parse import urlparse +import pyarrow as pa from pyarrow.fs import ( FileInfo, FileSystem, @@ -41,7 +47,28 @@ OutputFile, OutputStream, ) +from pyiceberg.schema import Schema, SchemaVisitor, visit from pyiceberg.typedef import EMPTY_DICT, Properties +from pyiceberg.types import ( + BinaryType, + BooleanType, + DateType, + DecimalType, + DoubleType, + FixedType, + FloatType, + IntegerType, + ListType, + LongType, + MapType, + NestedField, + PrimitiveType, + StringType, + StructType, + TimestampType, + TimestamptzType, + TimeType, +) class PyArrowFile(InputFile, OutputFile): @@ -239,3 +266,106 @@ def delete(self, location: Union[str, InputFile, OutputFile]) -> None: elif e.errno == 13 or "AWS Error [code 15]" in str(e): raise PermissionError(f"Cannot delete file, access denied: {location}") from e raise # pragma: no cover - If some other kind of OSError, raise the raw error + + +def schema_to_pyarrow(schema: Schema) -> pa.schema: + return visit(schema, _ConvertToArrowSchema()) + + +class _ConvertToArrowSchema(SchemaVisitor[pa.DataType]): + def schema(self, _: Schema, struct_result: pa.StructType) -> pa.schema: + return pa.schema(list(struct_result)) + + def struct(self, _: StructType, field_results: List[pa.DataType]) -> pa.DataType: + return pa.struct(field_results) + + def field(self, field: NestedField, field_result: pa.DataType) -> pa.Field: + return pa.field( + name=field.name, + type=field_result, + nullable=not field.required, + metadata={"doc": field.doc, "id": str(field.field_id)} if field.doc else {}, + ) + + def list(self, _: ListType, element_result: pa.DataType) -> pa.DataType: + return pa.list_(value_type=element_result) + + def map(self, _: MapType, key_result: pa.DataType, value_result: pa.DataType) -> pa.DataType: + return pa.map_(key_type=key_result, item_type=value_result) + + def primitive(self, primitive: PrimitiveType) -> pa.DataType: + return _iceberg_to_pyarrow_type(primitive) + + +@singledispatch +def _iceberg_to_pyarrow_type(primitive: PrimitiveType) -> pa.DataType: + raise ValueError(f"Unknown type: {primitive}") + + +@_iceberg_to_pyarrow_type.register +def _(primitive: FixedType) -> pa.DataType: + return pa.binary(primitive.length) + + +@_iceberg_to_pyarrow_type.register +def _(primitive: DecimalType) -> pa.DataType: + return pa.decimal128(primitive.precision, primitive.scale) + + +@_iceberg_to_pyarrow_type.register +def _(_: BooleanType) -> pa.DataType: + return pa.bool_() + + +@_iceberg_to_pyarrow_type.register +def _(_: IntegerType) -> pa.DataType: + return pa.int32() + + +@_iceberg_to_pyarrow_type.register +def _(_: LongType) -> pa.DataType: + return pa.int64() + + +@_iceberg_to_pyarrow_type.register +def _(_: FloatType) -> pa.DataType: + # 32-bit IEEE 754 floating point + return pa.float32() + + +@_iceberg_to_pyarrow_type.register +def _(_: DoubleType) -> pa.DataType: + # 64-bit IEEE 754 floating point + return pa.float64() + + +@_iceberg_to_pyarrow_type.register +def _(_: DateType) -> pa.DataType: + # Date encoded as an int + return pa.date32() + + +@_iceberg_to_pyarrow_type.register +def _(_: TimeType) -> pa.DataType: + return pa.time64("us") + + +@_iceberg_to_pyarrow_type.register +def _(_: TimestampType) -> pa.DataType: + return pa.timestamp(unit="ms") + + +@_iceberg_to_pyarrow_type.register +def _(_: TimestamptzType) -> pa.DataType: + return pa.timestamp(unit="ms", tz="+00:00") + + +@_iceberg_to_pyarrow_type.register +def _(_: StringType) -> pa.DataType: + return pa.string() + + +@_iceberg_to_pyarrow_type.register +def _(_: BinaryType) -> pa.DataType: + # Variable length by default + return pa.binary() diff --git a/tests/io/test_pyarrow.py b/tests/io/test_pyarrow.py index 1b0b07f671..c199c711d1 100644 --- a/tests/io/test_pyarrow.py +++ b/tests/io/test_pyarrow.py @@ -20,11 +20,35 @@ import tempfile from unittest.mock import MagicMock, patch +import pyarrow as pa import pytest from pyarrow.fs import FileType from pyiceberg.io import InputStream, OutputStream -from pyiceberg.io.pyarrow import PyArrowFile, PyArrowFileIO +from pyiceberg.io.pyarrow import ( + PyArrowFile, + PyArrowFileIO, + _ConvertToArrowSchema, + schema_to_pyarrow, +) +from pyiceberg.schema import Schema, visit +from pyiceberg.types import ( + BinaryType, + BooleanType, + DateType, + DecimalType, + DoubleType, + FixedType, + FloatType, + IntegerType, + ListType, + LongType, + MapType, + StringType, + TimestampType, + TimestamptzType, + TimeType, +) def test_pyarrow_input_file(): @@ -264,3 +288,126 @@ def test_deleting_s3_file_not_found(): PyArrowFileIO().delete("s3://foo/bar.txt") assert "Cannot delete file, does not exist:" in str(exc_info.value) + + +def test_schema_to_pyarrow_schema(table_schema_nested: Schema): + actual = schema_to_pyarrow(table_schema_nested) + expected = """foo: string +bar: int32 not null +baz: bool +qux: list not null + child 0, item: string +quux: map> not null + child 0, entries: struct> not null + child 0, key: string not null + child 1, value: map + child 0, entries: struct not null + child 0, key: string not null + child 1, value: int32 +location: list> not null + child 0, item: struct + child 0, latitude: float + child 1, longitude: float +person: struct + child 0, name: string + child 1, age: int32 not null""" + assert repr(actual) == expected + + +def test_fixed_type_to_pyarrow(): + length = 22 + iceberg_type = FixedType(length) + assert visit(iceberg_type, _ConvertToArrowSchema()) == pa.binary(length) + + +def test_decimal_type_to_pyarrow(): + precision = 25 + scale = 19 + iceberg_type = DecimalType(precision, scale) + assert visit(iceberg_type, _ConvertToArrowSchema()) == pa.decimal128(precision, scale) + + +def test_boolean_type_to_pyarrow(): + iceberg_type = BooleanType() + assert visit(iceberg_type, _ConvertToArrowSchema()) == pa.bool_() + + +def test_integer_type_to_pyarrow(): + iceberg_type = IntegerType() + assert visit(iceberg_type, _ConvertToArrowSchema()) == pa.int32() + + +def test_long_type_to_pyarrow(): + iceberg_type = LongType() + assert visit(iceberg_type, _ConvertToArrowSchema()) == pa.int64() + + +def test_float_type_to_pyarrow(): + iceberg_type = FloatType() + assert visit(iceberg_type, _ConvertToArrowSchema()) == pa.float32() + + +def test_double_type_to_pyarrow(): + iceberg_type = DoubleType() + assert visit(iceberg_type, _ConvertToArrowSchema()) == pa.float64() + + +def test_date_type_to_pyarrow(): + iceberg_type = DateType() + assert visit(iceberg_type, _ConvertToArrowSchema()) == pa.date32() + + +def test_time_type_to_pyarrow(): + iceberg_type = TimeType() + assert visit(iceberg_type, _ConvertToArrowSchema()) == pa.time64("us") + + +def test_timestamp_type_to_pyarrow(): + iceberg_type = TimestampType() + assert visit(iceberg_type, _ConvertToArrowSchema()) == pa.timestamp(unit="ms") + + +def test_timestamptz_type_to_pyarrow(): + iceberg_type = TimestamptzType() + assert visit(iceberg_type, _ConvertToArrowSchema()) == pa.timestamp(unit="ms", tz="+00:00") + + +def test_string_type_to_pyarrow(): + iceberg_type = StringType() + assert visit(iceberg_type, _ConvertToArrowSchema()) == pa.string() + + +def test_binary_type_to_pyarrow(): + iceberg_type = BinaryType() + assert visit(iceberg_type, _ConvertToArrowSchema()) == pa.binary() + + +def test_struct_type_to_pyarrow(table_schema_simple: Schema): + expected = pa.struct( + [ + pa.field("foo", pa.string(), nullable=True, metadata={"id": "1"}), + pa.field("bar", pa.int32(), nullable=False, metadata={"id": "2"}), + pa.field("baz", pa.bool_(), nullable=True, metadata={"id": "3"}), + ] + ) + assert visit(table_schema_simple.as_struct(), _ConvertToArrowSchema()) == expected + + +def test_map_type_to_pyarrow(): + iceberg_map = MapType( + key_id=1, + key_type=IntegerType(), + value_id=2, + value_type=StringType(), + value_required=True, + ) + assert visit(iceberg_map, _ConvertToArrowSchema()) == pa.map_(pa.int32(), pa.string()) + + +def test_list_type_to_pyarrow(): + iceberg_map = ListType( + element_id=1, + element_type=IntegerType(), + element_required=True, + ) + assert visit(iceberg_map, _ConvertToArrowSchema()) == pa.list_(pa.int32()) From c0cb876f17688c485129b48da73d114d92a40cbf Mon Sep 17 00:00:00 2001 From: Fokko Driesprong Date: Fri, 21 Oct 2022 22:15:32 +0200 Subject: [PATCH 244/642] Python: Implement Schema.select (#5966) --- pyiceberg/schema.py | 30 +++++++++++++++++------------- tests/test_schema.py | 41 +++++++++++++++++++++++++++++++---------- 2 files changed, 48 insertions(+), 23 deletions(-) diff --git a/pyiceberg/schema.py b/pyiceberg/schema.py index 7f7d096ace..82a9533392 100644 --- a/pyiceberg/schema.py +++ b/pyiceberg/schema.py @@ -210,7 +210,7 @@ def accessor_for_field(self, field_id: int) -> "Accessor": return self._lazy_id_to_accessor[field_id] - def select(self, names: List[str], case_sensitive: bool = True) -> "Schema": + def select(self, *names: str, case_sensitive: bool = True) -> "Schema": """Return a new schema instance pruned to a subset of columns Args: @@ -219,20 +219,20 @@ def select(self, names: List[str], case_sensitive: bool = True) -> "Schema": Returns: Schema: A new schema with pruned columns + + Raises: + ValueError: If a column is selected that doesn't exist """ - if case_sensitive: - return self._case_sensitive_select(schema=self, names=names) - return self._case_insensitive_select(schema=self, names=names) - @classmethod - def _case_sensitive_select(cls, schema: "Schema", names: List[str]): - # TODO: Add a PruneColumns schema visitor and use it here - raise NotImplementedError() + try: + if case_sensitive: + ids = {self._name_to_id[name] for name in names} + else: + ids = {self._lazy_name_to_id_lower[name.lower()] for name in names} + except KeyError as e: + raise ValueError(f"Could not find column: {e}") from e - @classmethod - def _case_insensitive_select(cls, schema: "Schema", names: List[str]): - # TODO: Add a PruneColumns schema visitor and use it here - raise NotImplementedError() + return prune_columns(self, ids) class SchemaVisitor(Generic[T], ABC): @@ -800,7 +800,11 @@ def primitive(self, primitive: PrimitiveType) -> PrimitiveType: def prune_columns(schema: Schema, selected: Set[int], select_full_types: bool = True) -> Schema: result = visit(schema.as_struct(), _PruneColumnsVisitor(selected, select_full_types)) - return Schema(*(result or StructType()).fields, schema_id=schema.schema_id, identifier_field_ids=schema.identifier_field_ids) + return Schema( + *(result or StructType()).fields, + schema_id=schema.schema_id, + identifier_field_ids=list(selected.intersection(schema.identifier_field_ids)), + ) class _PruneColumnsVisitor(SchemaVisitor[Optional[IcebergType]]): diff --git a/tests/test_schema.py b/tests/test_schema.py index 5b64d95d6b..b2a2790438 100644 --- a/tests/test_schema.py +++ b/tests/test_schema.py @@ -438,7 +438,7 @@ def test_prune_columns_list(table_schema_nested: Schema): required=True, ), schema_id=1, - identifier_field_ids=[1], + identifier_field_ids=[], ) @@ -457,7 +457,7 @@ def test_prune_columns_list_full(table_schema_nested: Schema): required=True, ), schema_id=1, - identifier_field_ids=[1], + identifier_field_ids=[], ) @@ -479,7 +479,7 @@ def test_prune_columns_map(table_schema_nested: Schema): required=True, ), schema_id=1, - identifier_field_ids=[1], + identifier_field_ids=[], ) @@ -509,7 +509,7 @@ def test_prune_columns_map_full(table_schema_nested: Schema): required=True, ), schema_id=1, - identifier_field_ids=[1], + identifier_field_ids=[], ) @@ -531,7 +531,7 @@ def test_prune_columns_map_key(table_schema_nested: Schema): required=True, ), schema_id=1, - identifier_field_ids=[1], + identifier_field_ids=[], ) @@ -544,7 +544,7 @@ def test_prune_columns_struct(table_schema_nested: Schema): required=False, ), schema_id=1, - identifier_field_ids=[1], + identifier_field_ids=[], ) @@ -558,7 +558,7 @@ def test_prune_columns_struct_full(table_schema_nested: Schema): required=False, ), schema_id=1, - identifier_field_ids=[1], + identifier_field_ids=[], ) @@ -625,7 +625,7 @@ def test_prune_columns_struct_in_map(): required=True, ), schema_id=1, - identifier_field_ids=[1], + identifier_field_ids=[], ) @@ -647,7 +647,7 @@ def test_prune_columns_struct_in_map_full(): required=True, ), schema_id=1, - identifier_field_ids=[1], + identifier_field_ids=[], ) assert prune_columns(table_schema_nested, {11}, True) == Schema( NestedField( @@ -664,10 +664,31 @@ def test_prune_columns_struct_in_map_full(): required=True, ), schema_id=1, - identifier_field_ids=[1], + identifier_field_ids=[], ) def test_prune_columns_select_original_schema(table_schema_nested: Schema): ids = set(range(table_schema_nested.highest_field_id)) assert prune_columns(table_schema_nested, ids, True) == table_schema_nested + + +def test_schema_select(table_schema_nested: Schema): + assert table_schema_nested.select("bar", "baz") == Schema( + NestedField(field_id=2, name="bar", field_type=IntegerType(), required=True), + NestedField(field_id=3, name="baz", field_type=BooleanType(), required=False), + schema_id=1, + identifier_field_ids=[], + ) + + +def test_schema_select_case_insensitive(table_schema_nested: Schema): + assert table_schema_nested.select("BAZ", case_sensitive=False) == Schema( + NestedField(field_id=3, name="baz", field_type=BooleanType(), required=False), schema_id=1, identifier_field_ids=[] + ) + + +def test_schema_select_cant_be_found(table_schema_nested: Schema): + with pytest.raises(ValueError) as exc_info: + table_schema_nested.select("BAZ", case_sensitive=True) + assert "Could not find column: 'BAZ'" in str(exc_info.value) From 93b7cbed9f383dca240a2b6456b1565f5355c097 Mon Sep 17 00:00:00 2001 From: Fokko Driesprong Date: Fri, 21 Oct 2022 23:15:29 +0200 Subject: [PATCH 245/642] Python: Implement S3V4RestSigner (#5969) --- pyiceberg/exceptions.py | 4 ++ pyiceberg/io/__init__.py | 2 + pyiceberg/io/fsspec.py | 60 ++++++++++++++++++++++--- pyproject.toml | 4 ++ tests/io/test_fsspec.py | 96 ++++++++++++++++++++++++++++++++++++++++ 5 files changed, 161 insertions(+), 5 deletions(-) diff --git a/pyiceberg/exceptions.py b/pyiceberg/exceptions.py index e44125cd9c..b3cc2778e2 100644 --- a/pyiceberg/exceptions.py +++ b/pyiceberg/exceptions.py @@ -78,3 +78,7 @@ class NoSuchPropertyException(Exception): class NotInstalledError(Exception): """When an optional dependency is not installed""" + + +class SignError(Exception): + """Raises when unable to sign a S3 request""" diff --git a/pyiceberg/io/__init__.py b/pyiceberg/io/__init__.py index 16955c2b23..12a1fd0005 100644 --- a/pyiceberg/io/__init__.py +++ b/pyiceberg/io/__init__.py @@ -285,6 +285,7 @@ def load_file_io(properties: Properties, location: Optional[str] = None) -> File # First look for the py-io-impl property to directly load the class if io_impl := properties.get(PY_IO_IMPL): if file_io := _import_file_io(io_impl, properties): + logger.info("Loaded FileIO: %s", io_impl) return file_io else: raise ValueError(f"Could not initialize FileIO: {io_impl}") @@ -301,6 +302,7 @@ def load_file_io(properties: Properties, location: Optional[str] = None) -> File try: # Default to PyArrow + logger.info("Defaulting to PyArrow FileIO") from pyiceberg.io.pyarrow import PyArrowFileIO return PyArrowFileIO(properties) diff --git a/pyiceberg/io/fsspec.py b/pyiceberg/io/fsspec.py index 20d972b967..f47616a75e 100644 --- a/pyiceberg/io/fsspec.py +++ b/pyiceberg/io/fsspec.py @@ -15,17 +15,51 @@ # specific language governing permissions and limitations # under the License. """FileIO implementation for reading and writing table files that uses fsspec compatible filesystems""" - -from functools import lru_cache -from typing import Callable, Union +import logging +from functools import lru_cache, partial +from typing import Callable, Dict, Union from urllib.parse import urlparse +import requests +from botocore import UNSIGNED +from botocore.awsrequest import AWSRequest from fsspec import AbstractFileSystem +from requests import HTTPError from s3fs import S3FileSystem +from pyiceberg.exceptions import SignError from pyiceberg.io import FileIO, InputFile, OutputFile from pyiceberg.typedef import Properties +logger = logging.getLogger(__name__) + + +def s3v4_rest_signer(properties: Properties, request: AWSRequest, **_) -> AWSRequest: + signer_url = properties["uri"].rstrip("/") + signer_headers = {"Authorization": f"Bearer {properties['token']}"} + signer_body = { + "method": request.method, + "region": request.context["client_region"], + "uri": request.url, + "headers": {key: [val] for key, val in request.headers.items()}, + } + try: + response = requests.post(f"{signer_url}/v1/aws/s3/sign", headers=signer_headers, json=signer_body) + response.raise_for_status() + response_json = response.json() + except HTTPError as e: + raise SignError(f"Failed to sign request {response.status_code}: {signer_body}") from e + + for key, value in response_json["headers"].items(): + request.headers.add_header(key, ", ".join(value)) + + request.url = response_json["uri"] + + return request + + +SIGNERS: Dict[str, Callable[[Properties, AWSRequest], AWSRequest]] = {"S3V4RestSigner": s3v4_rest_signer} + def _s3(properties: Properties) -> AbstractFileSystem: client_kwargs = { @@ -33,10 +67,26 @@ def _s3(properties: Properties) -> AbstractFileSystem: "aws_access_key_id": properties.get("s3.access-key-id"), "aws_secret_access_key": properties.get("s3.secret-access-key"), } + config_kwargs = {} + register_events: Dict[str, Callable] = {} + + if signer := properties.get("s3.signer"): + logger.info("Loading signer %s", signer) + if singer_func := SIGNERS.get(signer): + singer_func_with_properties = partial(singer_func, properties) + register_events["before-sign.s3"] = singer_func_with_properties + + # Disable the AWS Signer + config_kwargs["signature_version"] = UNSIGNED + else: + raise ValueError(f"Signer not available: {signer}") + + fs = S3FileSystem(client_kwargs=client_kwargs, config_kwargs=config_kwargs) - config_kwargs = {"signature_version": properties.get("s3.signer")} + for event_name, event_function in register_events.items(): + fs.s3.meta.events.register_last(event_name, event_function, unique_id=1925) - return S3FileSystem(client_kwargs=client_kwargs, config_kwargs=config_kwargs) + return fs SCHEME_TO_FS = { diff --git a/pyproject.toml b/pyproject.toml index 06f77dc24e..aef7fa68d0 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -176,5 +176,9 @@ ignore_missing_imports = true module = "tests.*" ignore_missing_imports = true +[[tool.mypy.overrides]] +module = "botocore.*" +ignore_missing_imports = true + [tool.coverage.run] source = ['pyiceberg/'] diff --git a/tests/io/test_fsspec.py b/tests/io/test_fsspec.py index 2c05740fc3..e1ac9f91e3 100644 --- a/tests/io/test_fsspec.py +++ b/tests/io/test_fsspec.py @@ -18,8 +18,12 @@ import uuid import pytest +from botocore.awsrequest import AWSRequest +from requests_mock import Mocker +from pyiceberg.exceptions import SignError from pyiceberg.io import fsspec +from pyiceberg.io.fsspec import s3v4_rest_signer from tests.io.test_io import LocalInputFile @@ -198,3 +202,95 @@ def test_writing_avro_file(generated_manifest_entry_file, fsspec_fileio): with fsspec_fileio.new_input(location=f"s3://warehouse/{filename}").open() as in_f: b2 = in_f.read() assert b1 == b2 # Check that bytes of read from local avro file match bytes written to s3 + + +TEST_URI = "https://iceberg-test-signer" + + +def test_s3v4_rest_signer(requests_mock: Mocker): + new_uri = "https://other-bucket/metadata/snap-8048355899640248710-1-a5c8ea2d-aa1f-48e8-89f4-1fa69db8c742.avro" + requests_mock.post( + f"{TEST_URI}/v1/aws/s3/sign", + json={ + "uri": new_uri, + "headers": { + "Authorization": [ + "AWS4-HMAC-SHA256 Credential=ASIAQPRZZYGHUT57DL3I/20221017/us-west-2/s3/aws4_request, SignedHeaders=host;x-amz-content-sha256;x-amz-date;x-amz-security-token, Signature=430582a17d61ab02c272896fa59195f277af4bdf2121c441685e589f044bbe02" + ], + "Host": ["bucket.s3.us-west-2.amazonaws.com"], + "User-Agent": ["Botocore/1.27.59 Python/3.10.7 Darwin/21.5.0"], + "x-amz-content-sha256": ["UNSIGNED-PAYLOAD"], + "X-Amz-Date": ["20221017T102940Z"], + "X-Amz-Security-Token": [ + "YQoJb3JpZ2luX2VjEDoaCXVzLXdlc3QtMiJGMEQCID/fFxZP5oaEgQmcwP6XhZa0xSq9lmLSx8ffaWbySfUPAiAesa7sjd/WV4uwRTO0S03y/MWVtgpH+/NyZQ4bZgLVriqrAggTEAEaDDAzMzQwNzIyMjE1OSIMOeFOWhZIurMmAqjsKogCxMCqxX8ZjK0gacAkcDqBCyA7qTSLhdfKQIH/w7WpLBU1km+cRUWWCudan6gZsAq867DBaKEP7qI05DAWr9MChAkgUgyI8/G3Z23ET0gAedf3GsJbakB0F1kklx8jPmj4BPCht9RcTiXiJ5DxTS/cRCcalIQXmPFbaJSqpBusVG2EkWnm1v7VQrNPE2Os2b2P293vpbhwkyCEQiGRVva4Sw9D1sKvqSsK10QCRG+os6dFEOu1kARaXi6pStvR4OVmj7OYeAYjzaFchn7nz2CSae0M4IluiYQ01eQAywbfRo9DpKSmDM/DnPZWJnD/woLhaaaCrCxSSEaFsvGOHFhLd3Rknw1v0jADMILUtJoGOp4BpqKqyMz0CY3kpKL0jfR3ykTf/ge9wWVE0Alr7wRIkGCIURkhslGHqSyFRGoTqIXaxU+oPbwlw/0w/nYO7qQ6bTANOWye/wgw4h/NmJ6vU7wnZTXwREf1r6MF72++bE/fMk19LfVb8jN/qrUqAUXTc8gBAUxL5pgy8+oT/JnI2BkVrrLS4ilxEXP9Ahm+6GDUYXV4fBpqpZwdkzQ/5Gw=" + ], + }, + "extensions": {}, + }, + status_code=200, + ) + + request = AWSRequest( + method="HEAD", + url="https://bucket/metadata/snap-8048355899640248710-1-a5c8ea2d-aa1f-48e8-89f4-1fa69db8c742.avro", + headers={"User-Agent": "Botocore/1.27.59 Python/3.10.7 Darwin/21.5.0"}, + data=b"", + params={}, + auth_path="/metadata/snap-8048355899640248710-1-a5c8ea2d-aa1f-48e8-89f4-1fa69db8c742.avro", + ) + request.context = { + "client_region": "us-west-2", + "has_streaming_input": False, + "auth_type": None, + "signing": {"bucket": "bucket"}, + "retries": {"attempt": 1, "invocation-id": "75d143fb-0219-439b-872c-18213d1c8d54"}, + } + + signed_request = s3v4_rest_signer({"token": "abc", "uri": TEST_URI}, request) + + assert signed_request.url == new_uri + assert dict(signed_request.headers) == { + "Authorization": "AWS4-HMAC-SHA256 Credential=ASIAQPRZZYGHUT57DL3I/20221017/us-west-2/s3/aws4_request, SignedHeaders=host;x-amz-content-sha256;x-amz-date;x-amz-security-token, Signature=430582a17d61ab02c272896fa59195f277af4bdf2121c441685e589f044bbe02", + "Host": "bucket.s3.us-west-2.amazonaws.com", + "User-Agent": "Botocore/1.27.59 Python/3.10.7 Darwin/21.5.0", + "X-Amz-Date": "20221017T102940Z", + "X-Amz-Security-Token": "YQoJb3JpZ2luX2VjEDoaCXVzLXdlc3QtMiJGMEQCID/fFxZP5oaEgQmcwP6XhZa0xSq9lmLSx8ffaWbySfUPAiAesa7sjd/WV4uwRTO0S03y/MWVtgpH+/NyZQ4bZgLVriqrAggTEAEaDDAzMzQwNzIyMjE1OSIMOeFOWhZIurMmAqjsKogCxMCqxX8ZjK0gacAkcDqBCyA7qTSLhdfKQIH/w7WpLBU1km+cRUWWCudan6gZsAq867DBaKEP7qI05DAWr9MChAkgUgyI8/G3Z23ET0gAedf3GsJbakB0F1kklx8jPmj4BPCht9RcTiXiJ5DxTS/cRCcalIQXmPFbaJSqpBusVG2EkWnm1v7VQrNPE2Os2b2P293vpbhwkyCEQiGRVva4Sw9D1sKvqSsK10QCRG+os6dFEOu1kARaXi6pStvR4OVmj7OYeAYjzaFchn7nz2CSae0M4IluiYQ01eQAywbfRo9DpKSmDM/DnPZWJnD/woLhaaaCrCxSSEaFsvGOHFhLd3Rknw1v0jADMILUtJoGOp4BpqKqyMz0CY3kpKL0jfR3ykTf/ge9wWVE0Alr7wRIkGCIURkhslGHqSyFRGoTqIXaxU+oPbwlw/0w/nYO7qQ6bTANOWye/wgw4h/NmJ6vU7wnZTXwREf1r6MF72++bE/fMk19LfVb8jN/qrUqAUXTc8gBAUxL5pgy8+oT/JnI2BkVrrLS4ilxEXP9Ahm+6GDUYXV4fBpqpZwdkzQ/5Gw=", + "x-amz-content-sha256": "UNSIGNED-PAYLOAD", + } + + +def test_s3v4_rest_signer_forbidden(requests_mock: Mocker): + requests_mock.post( + f"{TEST_URI}/v1/aws/s3/sign", + json={ + "method": "HEAD", + "region": "us-west-2", + "uri": "https://bucket/metadata/snap-8048355899640248710-1-a5c8ea2d-aa1f-48e8-89f4-1fa69db8c742.avro", + "headers": {"User-Agent": ["Botocore/1.27.59 Python/3.10.7 Darwin/21.5.0"]}, + }, + status_code=401, + ) + + request = AWSRequest( + method="HEAD", + url="https://bucket/metadata/snap-8048355899640248710-1-a5c8ea2d-aa1f-48e8-89f4-1fa69db8c742.avro", + headers={"User-Agent": "Botocore/1.27.59 Python/3.10.7 Darwin/21.5.0"}, + data=b"", + params={}, + auth_path="/metadata/snap-8048355899640248710-1-a5c8ea2d-aa1f-48e8-89f4-1fa69db8c742.avro", + ) + request.context = { + "client_region": "us-west-2", + "has_streaming_input": False, + "auth_type": None, + "signing": {"bucket": "bucket"}, + "retries": {"attempt": 1, "invocation-id": "75d143fb-0219-439b-872c-18213d1c8d54"}, + } + + with pytest.raises(SignError) as exc_info: + _ = s3v4_rest_signer({"token": "abc", "uri": TEST_URI}, request) + + assert ( + """Failed to sign request 401: {'method': 'HEAD', 'region': 'us-west-2', 'uri': 'https://bucket/metadata/snap-8048355899640248710-1-a5c8ea2d-aa1f-48e8-89f4-1fa69db8c742.avro', 'headers': {'User-Agent': ['Botocore/1.27.59 Python/3.10.7 Darwin/21.5.0']}}""" + in str(exc_info.value) + ) From 529889c4ed3688e3e7ddd14392ece1d9495cb3ae Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 24 Oct 2022 10:15:56 +0200 Subject: [PATCH 246/642] Build: Bump mkdocs from 1.3.1 to 1.4.1 in /python (#6033) Bumps [mkdocs](https://github.com/mkdocs/mkdocs) from 1.3.1 to 1.4.1. - [Release notes](https://github.com/mkdocs/mkdocs/releases) - [Commits](https://github.com/mkdocs/mkdocs/compare/1.3.1...1.4.1) --- updated-dependencies: - dependency-name: mkdocs dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- mkdocs/requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mkdocs/requirements.txt b/mkdocs/requirements.txt index b21284bb84..a71574e759 100644 --- a/mkdocs/requirements.txt +++ b/mkdocs/requirements.txt @@ -15,5 +15,5 @@ # specific language governing permissions and limitations # under the License. -mkdocs==1.3.1 +mkdocs==1.4.1 jinja2==3.1.2 From 4016230d8d05b3ef332b8119f25ae0880b5e05de Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Sun, 30 Oct 2022 21:56:42 +0100 Subject: [PATCH 247/642] Build: Bump pytest from 7.1.3 to 7.2.0 in /python (#6080) Bumps [pytest](https://github.com/pytest-dev/pytest) from 7.1.3 to 7.2.0. - [Release notes](https://github.com/pytest-dev/pytest/releases) - [Changelog](https://github.com/pytest-dev/pytest/blob/main/CHANGELOG.rst) - [Commits](https://github.com/pytest-dev/pytest/compare/7.1.3...7.2.0) --- updated-dependencies: - dependency-name: pytest dependency-type: direct:development update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- poetry.lock | 45 ++++++++++++++++++++++++--------------------- pyproject.toml | 2 +- 2 files changed, 25 insertions(+), 22 deletions(-) diff --git a/poetry.lock b/poetry.lock index 0b55c50615..0a79cdd11b 100644 --- a/poetry.lock +++ b/poetry.lock @@ -78,7 +78,7 @@ python-versions = ">=3.5" dev = ["cloudpickle", "coverage[toml] (>=5.0.2)", "furo", "hypothesis", "mypy (>=0.900,!=0.940)", "pre-commit", "pympler", "pytest (>=4.3.0)", "pytest-mypy-plugins", "sphinx", "sphinx-notfound-page", "zope.interface"] docs = ["furo", "sphinx", "sphinx-notfound-page", "zope.interface"] tests = ["cloudpickle", "coverage[toml] (>=5.0.2)", "hypothesis", "mypy (>=0.900,!=0.940)", "pympler", "pytest (>=4.3.0)", "pytest-mypy-plugins", "zope.interface"] -tests_no_zope = ["cloudpickle", "coverage[toml] (>=5.0.2)", "hypothesis", "mypy (>=0.900,!=0.940)", "pympler", "pytest (>=4.3.0)", "pytest-mypy-plugins"] +tests-no-zope = ["cloudpickle", "coverage[toml] (>=5.0.2)", "hypothesis", "mypy (>=0.900,!=0.940)", "pympler", "pytest (>=4.3.0)", "pytest-mypy-plugins"] [[package]] name = "botocore" @@ -152,7 +152,7 @@ optional = false python-versions = ">=3.6.0" [package.extras] -unicode_backport = ["unicodedata2"] +unicode-backport = ["unicodedata2"] [[package]] name = "click" @@ -214,6 +214,17 @@ category = "dev" optional = false python-versions = ">=3.7" +[[package]] +name = "exceptiongroup" +version = "1.0.0" +description = "Backport of PEP 654 (exception groups)" +category = "dev" +optional = false +python-versions = ">=3.7" + +[package.extras] +test = ["pytest (>=6)"] + [[package]] name = "fastavro" version = "1.6.1" @@ -427,14 +438,6 @@ pyyaml = ">=5.1" toml = "*" virtualenv = ">=20.0.8" -[[package]] -name = "py" -version = "1.11.0" -description = "library with cross-python path, ini-parsing, io, code, log facilities" -category = "dev" -optional = false -python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*" - [[package]] name = "pyarrow" version = "9.0.0" @@ -493,7 +496,7 @@ diagrams = ["jinja2", "railroad-diagrams"] [[package]] name = "pytest" -version = "7.1.3" +version = "7.2.0" description = "pytest: simple powerful testing with Python" category = "dev" optional = false @@ -502,11 +505,11 @@ python-versions = ">=3.7" [package.dependencies] attrs = ">=19.2.0" colorama = {version = "*", markers = "sys_platform == \"win32\""} +exceptiongroup = {version = ">=1.0.0rc8", markers = "python_version < \"3.11\""} iniconfig = "*" packaging = "*" pluggy = ">=0.12,<2.0" -py = ">=1.8.2" -tomli = ">=1.0.0" +tomli = {version = ">=1.0.0", markers = "python_version < \"3.11\""} [package.extras] testing = ["argcomplete", "hypothesis (>=3.56)", "mock", "nose", "pygments (>=2.7.2)", "requests", "xmlschema"] @@ -571,7 +574,7 @@ urllib3 = ">=1.21.1,<1.27" [package.extras] socks = ["PySocks (>=1.5.6,!=1.5.7)"] -use_chardet_on_py3 = ["chardet (>=3.0.2,<6)"] +use-chardet-on-py3 = ["chardet (>=3.0.2,<6)"] [[package]] name = "requests-mock" @@ -768,7 +771,7 @@ snappy = ["python-snappy"] [metadata] lock-version = "1.1" python-versions = "^3.8" -content-hash = "c5636e98f0e6735e1ba219ba5ab1070eaa640da77dc8013d86b3cfc9f15eede5" +content-hash = "8c92da03c5df159ccc83d8e7f64268a3eb87bf3b37ceeeee4c264bcf3cadf85f" [metadata.files] aiobotocore = [ @@ -1023,6 +1026,10 @@ docutils = [ {file = "docutils-0.19-py3-none-any.whl", hash = "sha256:5e1de4d849fee02c63b040a4a3fd567f4ab104defd8a5511fbbc24a8a017efbc"}, {file = "docutils-0.19.tar.gz", hash = "sha256:33995a6753c30b7f577febfc2c50411fec6aac7f7ffeb7c4cfe5991072dcf9e6"}, ] +exceptiongroup = [ + {file = "exceptiongroup-1.0.0-py3-none-any.whl", hash = "sha256:2ac84b496be68464a2da60da518af3785fff8b7ec0d090a581604bc870bdee41"}, + {file = "exceptiongroup-1.0.0.tar.gz", hash = "sha256:affbabf13fb6e98988c38d9c5650e701569fe3c1de3233cfb61c5f33774690ad"}, +] fastavro = [ {file = "fastavro-1.6.1-cp310-cp310-macosx_10_14_x86_64.whl", hash = "sha256:de244146a3d7bc4e60c7be11f317054ee0e57ca807d85923bd8407981488ff18"}, {file = "fastavro-1.6.1-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:1ed84fb27672e54b0bafecae1f128f6e2950828f451e59f93bb74eeccb31b496"}, @@ -1274,10 +1281,6 @@ pre-commit = [ {file = "pre_commit-2.20.0-py2.py3-none-any.whl", hash = "sha256:51a5ba7c480ae8072ecdb6933df22d2f812dc897d5fe848778116129a681aac7"}, {file = "pre_commit-2.20.0.tar.gz", hash = "sha256:a978dac7bc9ec0bcee55c18a277d553b0f419d259dadb4b9418ff2d00eb43959"}, ] -py = [ - {file = "py-1.11.0-py2.py3-none-any.whl", hash = "sha256:607c53218732647dff4acdfcd50cb62615cedf612e72d1724fb1a0cc6405b378"}, - {file = "py-1.11.0.tar.gz", hash = "sha256:51c75c4126074b472f746a24399ad32f6053d1b34b68d2fa41e558e6f4a98719"}, -] pyarrow = [ {file = "pyarrow-9.0.0-cp310-cp310-macosx_10_13_universal2.whl", hash = "sha256:767cafb14278165ad539a2918c14c1b73cf20689747c21375c38e3fe62884902"}, {file = "pyarrow-9.0.0-cp310-cp310-macosx_10_13_x86_64.whl", hash = "sha256:0238998dc692efcb4e41ae74738d7c1234723271ccf520bd8312dca07d49ef8d"}, @@ -1357,8 +1360,8 @@ pyparsing = [ {file = "pyparsing-3.0.9.tar.gz", hash = "sha256:2b020ecf7d21b687f219b71ecad3631f644a47f01403fa1d1036b0c6416d70fb"}, ] pytest = [ - {file = "pytest-7.1.3-py3-none-any.whl", hash = "sha256:1377bda3466d70b55e3f5cecfa55bb7cfcf219c7964629b967c37cf0bda818b7"}, - {file = "pytest-7.1.3.tar.gz", hash = "sha256:4f365fec2dff9c1162f834d9f18af1ba13062db0c708bf7b946f8a5c76180c39"}, + {file = "pytest-7.2.0-py3-none-any.whl", hash = "sha256:892f933d339f068883b6fd5a459f03d85bfcb355e4981e146d2c7616c21fef71"}, + {file = "pytest-7.2.0.tar.gz", hash = "sha256:c4014eb40e10f11f355ad4e3c2fb2c6c6d1919c73f3b5a433de4708202cade59"}, ] pytest-checkdocs = [ {file = "pytest-checkdocs-2.9.0.tar.gz", hash = "sha256:54b3d1357a289e3f6336d31d4ceb72e99afe0fa4f552990f1b1f015f1925543b"}, diff --git a/pyproject.toml b/pyproject.toml index aef7fa68d0..0682e8e2b2 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -66,7 +66,7 @@ thrift = { version = "^0.16.0", optional = true } s3fs = { version = "2022.8.2", optional = true } [tool.poetry.dev-dependencies] -pytest = "^7.1.3" +pytest = "^7.2.0" pytest-checkdocs = "^2.9.0" pre-commit = "^2.0.0" fastavro = "^1.6.1" From 77a63ed569485ce0f73f7fd3102aaddb74d8105e Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Sun, 30 Oct 2022 22:09:30 +0100 Subject: [PATCH 248/642] Build: Bump pyarrow from 9.0.0 to 10.0.0 in /python (#6081) Bumps [pyarrow](https://github.com/apache/arrow) from 9.0.0 to 10.0.0. - [Release notes](https://github.com/apache/arrow/releases) - [Commits](https://github.com/apache/arrow/compare/go/v9.0.0...go/v10.0.0) --- updated-dependencies: - dependency-name: pyarrow dependency-type: direct:production update-type: version-update:semver-major ... Signed-off-by: dependabot[bot] Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- poetry.lock | 52 +++++++++++++++++++++++--------------------------- pyproject.toml | 2 +- 2 files changed, 25 insertions(+), 29 deletions(-) diff --git a/poetry.lock b/poetry.lock index 0a79cdd11b..a5d0072790 100644 --- a/poetry.lock +++ b/poetry.lock @@ -440,7 +440,7 @@ virtualenv = ">=20.0.8" [[package]] name = "pyarrow" -version = "9.0.0" +version = "10.0.0" description = "Python library for Apache Arrow" category = "main" optional = true @@ -771,7 +771,7 @@ snappy = ["python-snappy"] [metadata] lock-version = "1.1" python-versions = "^3.8" -content-hash = "8c92da03c5df159ccc83d8e7f64268a3eb87bf3b37ceeeee4c264bcf3cadf85f" +content-hash = "7110b63fc7d0031d380af65190f19efcab1a03c8288b0c60d67c661e073e0051" [metadata.files] aiobotocore = [ @@ -1282,32 +1282,28 @@ pre-commit = [ {file = "pre_commit-2.20.0.tar.gz", hash = "sha256:a978dac7bc9ec0bcee55c18a277d553b0f419d259dadb4b9418ff2d00eb43959"}, ] pyarrow = [ - {file = "pyarrow-9.0.0-cp310-cp310-macosx_10_13_universal2.whl", hash = "sha256:767cafb14278165ad539a2918c14c1b73cf20689747c21375c38e3fe62884902"}, - {file = "pyarrow-9.0.0-cp310-cp310-macosx_10_13_x86_64.whl", hash = "sha256:0238998dc692efcb4e41ae74738d7c1234723271ccf520bd8312dca07d49ef8d"}, - {file = "pyarrow-9.0.0-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:55328348b9139c2b47450d512d716c2248fd58e2f04e2fc23a65e18726666d42"}, - {file = "pyarrow-9.0.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:fc856628acd8d281652c15b6268ec7f27ebcb015abbe99d9baad17f02adc51f1"}, - {file = "pyarrow-9.0.0-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:29eb3e086e2b26202f3a4678316b93cfb15d0e2ba20f3ec12db8fd9cc07cde63"}, - {file = "pyarrow-9.0.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:2e753f8fcf07d8e3a0efa0c8bd51fef5c90281ffd4c5637c08ce42cd0ac297de"}, - {file = "pyarrow-9.0.0-cp310-cp310-win_amd64.whl", hash = "sha256:3eef8a981f45d89de403e81fb83b8119c20824caddf1404274e41a5d66c73806"}, - {file = "pyarrow-9.0.0-cp37-cp37m-macosx_10_13_x86_64.whl", hash = "sha256:7fa56cbd415cef912677270b8e41baad70cde04c6d8a8336eeb2aba85aa93706"}, - {file = "pyarrow-9.0.0-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:f8c46bde1030d704e2796182286d1c56846552c50a39ad5bf5a20c0d8159fc35"}, - {file = "pyarrow-9.0.0-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:8ad430cee28ebc4d6661fc7315747c7a18ae2a74e67498dcb039e1c762a2fb67"}, - {file = "pyarrow-9.0.0-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:81a60bb291a964f63b2717fb1b28f6615ffab7e8585322bfb8a6738e6b321282"}, - {file = "pyarrow-9.0.0-cp37-cp37m-win_amd64.whl", hash = "sha256:9cef618159567d5f62040f2b79b1c7b38e3885f4ffad0ec97cd2d86f88b67cef"}, - {file = "pyarrow-9.0.0-cp38-cp38-macosx_10_13_x86_64.whl", hash = "sha256:5526a3bfb404ff6d31d62ea582cf2466c7378a474a99ee04d1a9b05de5264541"}, - {file = "pyarrow-9.0.0-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:da3e0f319509a5881867effd7024099fb06950a0768dad0d6873668bb88cfaba"}, - {file = "pyarrow-9.0.0-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:2c715eca2092273dcccf6f08437371e04d112f9354245ba2fbe6c801879450b7"}, - {file = "pyarrow-9.0.0-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:f11a645a41ee531c3a5edda45dea07c42267f52571f818d388971d33fc7e2d4a"}, - {file = "pyarrow-9.0.0-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:a5b390bdcfb8c5b900ef543f911cdfec63e88524fafbcc15f83767202a4a2491"}, - {file = "pyarrow-9.0.0-cp38-cp38-win_amd64.whl", hash = "sha256:d9eb04db626fa24fdfb83c00f76679ca0d98728cdbaa0481b6402bf793a290c0"}, - {file = "pyarrow-9.0.0-cp39-cp39-macosx_10_13_universal2.whl", hash = "sha256:4eebdab05afa23d5d5274b24c1cbeb1ba017d67c280f7d39fd8a8f18cbad2ec9"}, - {file = "pyarrow-9.0.0-cp39-cp39-macosx_10_13_x86_64.whl", hash = "sha256:02b820ecd1da02012092c180447de449fc688d0c3f9ff8526ca301cdd60dacd0"}, - {file = "pyarrow-9.0.0-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:92f3977e901db1ef5cba30d6cc1d7942b8d94b910c60f89013e8f7bb86a86eef"}, - {file = "pyarrow-9.0.0-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:f241bd488c2705df930eedfe304ada71191dcf67d6b98ceda0cc934fd2a8388e"}, - {file = "pyarrow-9.0.0-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:1c5a073a930c632058461547e0bc572da1e724b17b6b9eb31a97da13f50cb6e0"}, - {file = "pyarrow-9.0.0-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:f59bcd5217a3ae1e17870792f82b2ff92df9f3862996e2c78e156c13e56ff62e"}, - {file = "pyarrow-9.0.0-cp39-cp39-win_amd64.whl", hash = "sha256:fe2ce795fa1d95e4e940fe5661c3c58aee7181c730f65ac5dd8794a77228de59"}, - {file = "pyarrow-9.0.0.tar.gz", hash = "sha256:7fb02bebc13ab55573d1ae9bb5002a6d20ba767bf8569b52fce5301d42495ab7"}, + {file = "pyarrow-10.0.0-cp310-cp310-macosx_10_14_universal2.whl", hash = "sha256:10e031794d019425d34406edffe7e32157359e9455f9edb97a1732f8dabf802f"}, + {file = "pyarrow-10.0.0-cp310-cp310-macosx_10_14_x86_64.whl", hash = "sha256:e4c6da9f9e1ff96781ee1478f7cc0860e66c23584887b8e297c4b9905c3c9066"}, + {file = "pyarrow-10.0.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:4051664d354b14939b5da35cfa77821ade594bc1cf56dd2032b3068c96697d74"}, + {file = "pyarrow-10.0.0-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:65d4a312f3ced318423704355acaccc7f7bdfe242472e59bdd54aa0f8837adf8"}, + {file = "pyarrow-10.0.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:758284e1ebd3f2a9abb30544bfec28d151a398bb7c0f2578cbca5ee5b000364a"}, + {file = "pyarrow-10.0.0-cp310-cp310-win_amd64.whl", hash = "sha256:f329951d56b3b943c353f7b27c894e02367a7efbb9fef7979c6b24e02dbfcf55"}, + {file = "pyarrow-10.0.0-cp37-cp37m-macosx_10_14_x86_64.whl", hash = "sha256:511735040b83f2993f78d7fb615e7b88253d75f41500e87e587c40156ff88120"}, + {file = "pyarrow-10.0.0-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:3d2694f08c8d4482d14e3798ff036dbd81ae6b1c47948f52515e1aa90fbec3f0"}, + {file = "pyarrow-10.0.0-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:c79300e1a3e23f2bf4defcf0d70ff5ea25ef6ebf6f121d8670ee14bb662bb7ca"}, + {file = "pyarrow-10.0.0-cp37-cp37m-win_amd64.whl", hash = "sha256:f76157d9579571c865860e5fd004537c03e21139db76692d96fd8a186adab1f2"}, + {file = "pyarrow-10.0.0-cp38-cp38-macosx_10_14_x86_64.whl", hash = "sha256:69b8a1fd99201178799b02f18498633847109b701856ec762f314352a431b7d0"}, + {file = "pyarrow-10.0.0-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:68ccb82c04c0f7abf7a95541d5e9d9d94290fc66a2d36d3f6ea0777f40c15654"}, + {file = "pyarrow-10.0.0-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:b45f969ed924282e9d4ede38f3430630d809c36dbff65452cabce03141943d28"}, + {file = "pyarrow-10.0.0-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:b9f63ceb8346aac0bcb487fafe9faca642ad448ca649fcf66a027c6e120cbc12"}, + {file = "pyarrow-10.0.0-cp38-cp38-win_amd64.whl", hash = "sha256:7ce026274cd5d9934cd3694e89edecde4e036018bbc6cb735fd33b9e967e7d47"}, + {file = "pyarrow-10.0.0-cp39-cp39-macosx_10_14_universal2.whl", hash = "sha256:7e6b837cc44cd62a0e280c8fc4de94ebce503d6d1190e6e94157ab49a8bea67b"}, + {file = "pyarrow-10.0.0-cp39-cp39-macosx_10_14_x86_64.whl", hash = "sha256:7be7f42f713068293308c989a4a3a2de03b70199bdbe753901c6595ff8640c64"}, + {file = "pyarrow-10.0.0-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:b3e3148468d3eed3779d68241f1d13ed4ee7cca4c6dbc7c07e5062b93ad4da33"}, + {file = "pyarrow-10.0.0-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:2d326a9d47ac237d81b8c4337e9d30a0b361835b536fc7ea53991455ce761fbd"}, + {file = "pyarrow-10.0.0-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:25f51dca780fc22cfd7ac30f6bdfe70eb99145aee9acfda987f2c49955d66ed9"}, + {file = "pyarrow-10.0.0-cp39-cp39-win_amd64.whl", hash = "sha256:d45a59e2f47826544c0ca70bc0f7ed8ffa5ad23f93b0458230c7e983bcad1acf"}, + {file = "pyarrow-10.0.0.tar.gz", hash = "sha256:b153b05765393557716e3729cf988442b3ae4f5567364ded40d58c07feed27c2"}, ] pycparser = [ {file = "pycparser-2.21-py2.py3-none-any.whl", hash = "sha256:8ee45429555515e1f6b185e78100aea234072576aa43ab53aefcae078162fca9"}, diff --git a/pyproject.toml b/pyproject.toml index 0682e8e2b2..b3e52c6f37 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -57,7 +57,7 @@ fsspec = "2022.8.2" zstandard = "^0.18.0" -pyarrow = { version = "^9.0.0", optional = true } +pyarrow = { version = ">=9,<11", optional = true } python-snappy = { version = "^0.6.1", optional = true } From 67c5f988da0f0bacd325611b22045de3bfbea170 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Sun, 30 Oct 2022 22:53:26 +0100 Subject: [PATCH 249/642] Build: Bump zstandard from 0.18.0 to 0.19.0 in /python (#6082) --- poetry.lock | 99 +++++++++++++++++++++++++++----------------------- pyproject.toml | 2 +- 2 files changed, 54 insertions(+), 47 deletions(-) diff --git a/poetry.lock b/poetry.lock index a5d0072790..3c674a695b 100644 --- a/poetry.lock +++ b/poetry.lock @@ -750,7 +750,7 @@ testing = ["func-timeout", "jaraco.itertools", "pytest (>=6)", "pytest-black (>= [[package]] name = "zstandard" -version = "0.18.0" +version = "0.19.0" description = "Zstandard bindings for Python" category = "main" optional = false @@ -771,7 +771,7 @@ snappy = ["python-snappy"] [metadata] lock-version = "1.1" python-versions = "^3.8" -content-hash = "7110b63fc7d0031d380af65190f19efcab1a03c8288b0c60d67c661e073e0051" +content-hash = "7e931ee336b58a21ee512568504e6899a9c1476f0e9d17b09c43c8e88c2e313c" [metadata.files] aiobotocore = [ @@ -1638,48 +1638,55 @@ zipp = [ {file = "zipp-3.8.1.tar.gz", hash = "sha256:05b45f1ee8f807d0cc928485ca40a07cb491cf092ff587c0df9cb1fd154848d2"}, ] zstandard = [ - {file = "zstandard-0.18.0-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:ef7e8a200e4c8ac9102ed3c90ed2aa379f6b880f63032200909c1be21951f556"}, - {file = "zstandard-0.18.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:2dc466207016564805e56d28375f4f533b525ff50d6776946980dff5465566ac"}, - {file = "zstandard-0.18.0-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:4a2ee1d4f98447f3e5183ecfce5626f983504a4a0c005fbe92e60fa8e5d547ec"}, - {file = "zstandard-0.18.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:d956e2f03c7200d7e61345e0880c292783ec26618d0d921dcad470cb195bbce2"}, - {file = "zstandard-0.18.0-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_12_i686.manylinux2010_i686.whl", hash = "sha256:ce6f59cba9854fd14da5bfe34217a1501143057313966637b7291d1b0267bd1e"}, - {file = "zstandard-0.18.0-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:a7fa67cba473623848b6e88acf8d799b1906178fd883fb3a1da24561c779593b"}, - {file = "zstandard-0.18.0-cp310-cp310-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:cdb44d7284c8c5dd1b66dfb86dda7f4560fa94bfbbc1d2da749ba44831335e32"}, - {file = "zstandard-0.18.0-cp310-cp310-win32.whl", hash = "sha256:63694a376cde0aa8b1971d06ca28e8f8b5f492779cb6ee1cc46bbc3f019a42a5"}, - {file = "zstandard-0.18.0-cp310-cp310-win_amd64.whl", hash = "sha256:702a8324cd90c74d9c8780d02bf55e79da3193c870c9665ad3a11647e3ad1435"}, - {file = "zstandard-0.18.0-cp36-cp36m-macosx_10_9_x86_64.whl", hash = "sha256:46f679bc5dfd938db4fb058218d9dc4db1336ffaf1ea774ff152ecadabd40805"}, - {file = "zstandard-0.18.0-cp36-cp36m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:dc2a4de9f363b3247d472362a65041fe4c0f59e01a2846b15d13046be866a885"}, - {file = "zstandard-0.18.0-cp36-cp36m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:bd3220d7627fd4d26397211cb3b560ec7cc4a94b75cfce89e847e8ce7fabe32d"}, - {file = "zstandard-0.18.0-cp36-cp36m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_12_i686.manylinux2010_i686.whl", hash = "sha256:39e98cf4773234bd9cebf9f9db730e451dfcfe435e220f8921242afda8321887"}, - {file = "zstandard-0.18.0-cp36-cp36m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:5228e596eb1554598c872a337bbe4e5afe41cd1f8b1b15f2e35b50d061e35244"}, - {file = "zstandard-0.18.0-cp36-cp36m-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:d4a8fd45746a6c31e729f35196e80b8f1e9987c59f5ccb8859d7c6a6fbeb9c63"}, - {file = "zstandard-0.18.0-cp36-cp36m-win32.whl", hash = "sha256:4cbb85f29a990c2fdbf7bc63246567061a362ddca886d7fae6f780267c0a9e67"}, - {file = "zstandard-0.18.0-cp36-cp36m-win_amd64.whl", hash = "sha256:bfa6c8549fa18e6497a738b7033c49f94a8e2e30c5fbe2d14d0b5aa8bbc1695d"}, - {file = "zstandard-0.18.0-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:e02043297c1832f2666cd2204f381bef43b10d56929e13c42c10c732c6e3b4ed"}, - {file = "zstandard-0.18.0-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:7231543d38d2b7e02ef7cc78ef7ffd86419437e1114ff08709fe25a160e24bd6"}, - {file = "zstandard-0.18.0-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:c86befac87445927488f5c8f205d11566f64c11519db223e9d282b945fa60dab"}, - {file = "zstandard-0.18.0-cp37-cp37m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_12_i686.manylinux2010_i686.whl", hash = "sha256:999a4e1768f219826ba3fa2064fab1c86dd72fdd47a42536235478c3bb3ca3e2"}, - {file = "zstandard-0.18.0-cp37-cp37m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:9df59cd1cf3c62075ee2a4da767089d19d874ac3ad42b04a71a167e91b384722"}, - {file = "zstandard-0.18.0-cp37-cp37m-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:1be31e9e3f7607ee0cdd60915410a5968b205d3e7aa83b7fcf3dd76dbbdb39e0"}, - {file = "zstandard-0.18.0-cp37-cp37m-win32.whl", hash = "sha256:490d11b705b8ae9dc845431bacc8dd1cef2408aede176620a5cd0cd411027936"}, - {file = "zstandard-0.18.0-cp37-cp37m-win_amd64.whl", hash = "sha256:266aba27fa9cc5e9091d3d325ebab1fa260f64e83e42516d5e73947c70216a5b"}, - {file = "zstandard-0.18.0-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:8b2260c4e07dd0723eadb586de7718b61acca4083a490dda69c5719d79bc715c"}, - {file = "zstandard-0.18.0-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:3af8c2383d02feb6650e9255491ec7d0824f6e6dd2bbe3e521c469c985f31fb1"}, - {file = "zstandard-0.18.0-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:28723a1d2e4df778573b76b321ebe9f3469ac98988104c2af116dd344802c3f8"}, - {file = "zstandard-0.18.0-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:19cac7108ff2c342317fad6dc97604b47a41f403c8f19d0bfc396dfadc3638b8"}, - {file = "zstandard-0.18.0-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_12_i686.manylinux2010_i686.whl", hash = "sha256:76725d1ee83a8915100a310bbad5d9c1fc6397410259c94033b8318d548d9990"}, - {file = "zstandard-0.18.0-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:d716a7694ce1fa60b20bc10f35c4a22be446ef7f514c8dbc8f858b61976de2fb"}, - {file = "zstandard-0.18.0-cp38-cp38-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:49685bf9a55d1ab34bd8423ea22db836ba43a181ac6b045ac4272093d5cb874e"}, - {file = "zstandard-0.18.0-cp38-cp38-win32.whl", hash = "sha256:1af1268a7dc870eb27515fb8db1f3e6c5a555d2b7bcc476fc3bab8886c7265ab"}, - {file = "zstandard-0.18.0-cp38-cp38-win_amd64.whl", hash = "sha256:1dc2d3809e763055a1a6c1a73f2b677320cc9a5aa1a7c6cfb35aee59bddc42d9"}, - {file = "zstandard-0.18.0-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:eea18c1e7442f2aa9aff1bb84550dbb6a1f711faf6e48e7319de8f2b2e923c2a"}, - {file = "zstandard-0.18.0-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:8677ffc6a6096cccbd892e558471c901fd821aba12b7fbc63833c7346f549224"}, - {file = "zstandard-0.18.0-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:083dc08abf03807af9beeb2b6a91c23ad78add2499f828176a3c7b742c44df02"}, - {file = "zstandard-0.18.0-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:c990063664c08169c84474acecc9251ee035871589025cac47c060ff4ec4bc1a"}, - {file = "zstandard-0.18.0-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_12_i686.manylinux2010_i686.whl", hash = "sha256:533db8a6fac6248b2cb2c935e7b92f994efbdeb72e1ffa0b354432e087bb5a3e"}, - {file = "zstandard-0.18.0-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:dbb3cb8a082d62b8a73af42291569d266b05605e017a3d8a06a0e5c30b5f10f0"}, - {file = "zstandard-0.18.0-cp39-cp39-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:d6c85ca5162049ede475b7ec98e87f9390501d44a3d6776ddd504e872464ec25"}, - {file = "zstandard-0.18.0-cp39-cp39-win32.whl", hash = "sha256:75479e7c2b3eebf402c59fbe57d21bc400cefa145ca356ee053b0a08908c5784"}, - {file = "zstandard-0.18.0-cp39-cp39-win_amd64.whl", hash = "sha256:d85bfabad444812133a92fc6fbe463e1d07581dba72f041f07a360e63808b23c"}, - {file = "zstandard-0.18.0.tar.gz", hash = "sha256:0ac0357a0d985b4ff31a854744040d7b5754385d1f98f7145c30e02c6865cb6f"}, + {file = "zstandard-0.19.0-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:a65e0119ad39e855427520f7829618f78eb2824aa05e63ff19b466080cd99210"}, + {file = "zstandard-0.19.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:4fa496d2d674c6e9cffc561639d17009d29adee84a27cf1e12d3c9be14aa8feb"}, + {file = "zstandard-0.19.0-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:8f7c68de4f362c1b2f426395fe4e05028c56d0782b2ec3ae18a5416eaf775576"}, + {file = "zstandard-0.19.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:d1a7a716bb04b1c3c4a707e38e2dee46ac544fff931e66d7ae944f3019fc55b8"}, + {file = "zstandard-0.19.0-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_12_i686.manylinux2010_i686.whl", hash = "sha256:72758c9f785831d9d744af282d54c3e0f9db34f7eae521c33798695464993da2"}, + {file = "zstandard-0.19.0-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:04c298d381a3b6274b0a8001f0da0ec7819d052ad9c3b0863fe8c7f154061f76"}, + {file = "zstandard-0.19.0-cp310-cp310-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:aef0889417eda2db000d791f9739f5cecb9ccdd45c98f82c6be531bdc67ff0f2"}, + {file = "zstandard-0.19.0-cp310-cp310-win32.whl", hash = "sha256:9d97c713433087ba5cee61a3e8edb54029753d45a4288ad61a176fa4718033ce"}, + {file = "zstandard-0.19.0-cp310-cp310-win_amd64.whl", hash = "sha256:81ab21d03e3b0351847a86a0b298b297fde1e152752614138021d6d16a476ea6"}, + {file = "zstandard-0.19.0-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:593f96718ad906e24d6534187fdade28b611f8ed06e27ba972ba48aecec45fc6"}, + {file = "zstandard-0.19.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:5e21032efe673b887464667d09406bab6e16d96b09ad87e80859e3a20b6745b6"}, + {file = "zstandard-0.19.0-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:876567136b0359f6581ecd892bdb4ca03a0eead0265db73206c78cff03bcdb0f"}, + {file = "zstandard-0.19.0-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:aa9087571729c968cd853d54b3f6e9d0ec61e45cd2c31e0eb8a0d4bdbbe6da2f"}, + {file = "zstandard-0.19.0-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:8371217dff635cfc0220db2720fc3ce728cd47e72bb7572cca035332823dbdfc"}, + {file = "zstandard-0.19.0-cp311-cp311-win32.whl", hash = "sha256:126aa8433773efad0871f624339c7984a9c43913952f77d5abeee7f95a0c0860"}, + {file = "zstandard-0.19.0-cp311-cp311-win_amd64.whl", hash = "sha256:0fde1c56ec118940974e726c2a27e5b54e71e16c6f81d0b4722112b91d2d9009"}, + {file = "zstandard-0.19.0-cp36-cp36m-macosx_10_9_x86_64.whl", hash = "sha256:898500957ae5e7f31b7271ace4e6f3625b38c0ac84e8cedde8de3a77a7fdae5e"}, + {file = "zstandard-0.19.0-cp36-cp36m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:660b91eca10ee1b44c47843894abe3e6cfd80e50c90dee3123befbf7ca486bd3"}, + {file = "zstandard-0.19.0-cp36-cp36m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:55b3187e0bed004533149882ef8c24e954321f3be81f8a9ceffe35099b82a0d0"}, + {file = "zstandard-0.19.0-cp36-cp36m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_12_i686.manylinux2010_i686.whl", hash = "sha256:6d2182e648e79213b3881998b30225b3f4b1f3e681f1c1eaf4cacf19bde1040d"}, + {file = "zstandard-0.19.0-cp36-cp36m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:8ec2c146e10b59c376b6bc0369929647fcd95404a503a7aa0990f21c16462248"}, + {file = "zstandard-0.19.0-cp36-cp36m-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:67710d220af405f5ce22712fa741d85e8b3ada7a457ea419b038469ba379837c"}, + {file = "zstandard-0.19.0-cp36-cp36m-win32.whl", hash = "sha256:f097dda5d4f9b9b01b3c9fa2069f9c02929365f48f341feddf3d6b32510a2f93"}, + {file = "zstandard-0.19.0-cp36-cp36m-win_amd64.whl", hash = "sha256:f4ebfe03cbae821ef994b2e58e4df6a087470cc522aca502614e82a143365d45"}, + {file = "zstandard-0.19.0-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:b80f6f6478f9d4ca26daee6c61584499493bf97950cfaa1a02b16bb5c2c17e70"}, + {file = "zstandard-0.19.0-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:909bdd4e19ea437eb9b45d6695d722f6f0fd9d8f493e837d70f92062b9f39faf"}, + {file = "zstandard-0.19.0-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:e9c90a44470f2999779057aeaf33461cbd8bb59d8f15e983150d10bb260e16e0"}, + {file = "zstandard-0.19.0-cp37-cp37m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_12_i686.manylinux2010_i686.whl", hash = "sha256:401508efe02341ae681752a87e8ac9ef76df85ef1a238a7a21786a489d2c983d"}, + {file = "zstandard-0.19.0-cp37-cp37m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:47dfa52bed3097c705451bafd56dac26535545a987b6759fa39da1602349d7ba"}, + {file = "zstandard-0.19.0-cp37-cp37m-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:1a4fb8b4ac6772e4d656103ccaf2e43e45bd16b5da324b963d58ef360d09eb73"}, + {file = "zstandard-0.19.0-cp37-cp37m-win32.whl", hash = "sha256:d63b04e16df8ea21dfcedbf5a60e11cbba9d835d44cb3cbff233cfd037a916d5"}, + {file = "zstandard-0.19.0-cp37-cp37m-win_amd64.whl", hash = "sha256:74c2637d12eaacb503b0b06efdf55199a11b1d7c580bd3dd9dfe84cac97ef2f6"}, + {file = "zstandard-0.19.0-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:2e4812720582d0803e84aefa2ac48ce1e1e6e200ca3ce1ae2be6d410c1d637ae"}, + {file = "zstandard-0.19.0-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:4514b19abe6dbd36d6c5d75c54faca24b1ceb3999193c5b1f4b685abeabde3d0"}, + {file = "zstandard-0.19.0-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:6caed86cd47ae93915d9031dc04be5283c275e1a2af2ceff33932071f3eeff4d"}, + {file = "zstandard-0.19.0-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:7ccc4727300f223184520a6064c161a90b5d0283accd72d1455bcd85ec44dd0d"}, + {file = "zstandard-0.19.0-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_12_i686.manylinux2010_i686.whl", hash = "sha256:879411d04068bd489db57dcf6b82ffad3c5fb2a1fdd30817c566d8b7bedee442"}, + {file = "zstandard-0.19.0-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:8c9ca56345b0c5574db47560603de9d05f63cce5dfeb3a456eb60f3fec737ff2"}, + {file = "zstandard-0.19.0-cp38-cp38-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:d777d239036815e9b3a093fa9208ad314c040c26d7246617e70e23025b60083a"}, + {file = "zstandard-0.19.0-cp38-cp38-win32.whl", hash = "sha256:be6329b5ba18ec5d32dc26181e0148e423347ed936dda48bf49fb243895d1566"}, + {file = "zstandard-0.19.0-cp38-cp38-win_amd64.whl", hash = "sha256:3d5bb598963ac1f1f5b72dd006adb46ca6203e4fb7269a5b6e1f99e85b07ad38"}, + {file = "zstandard-0.19.0-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:619f9bf37cdb4c3dc9d4120d2a1003f5db9446f3618a323219f408f6a9df6725"}, + {file = "zstandard-0.19.0-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:b253d0c53c8ee12c3e53d181fb9ef6ce2cd9c41cbca1c56a535e4fc8ec41e241"}, + {file = "zstandard-0.19.0-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:3c927b6aa682c6d96225e1c797f4a5d0b9f777b327dea912b23471aaf5385376"}, + {file = "zstandard-0.19.0-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:2f01b27d0b453f07cbcff01405cdd007e71f5d6410eb01303a16ba19213e58e4"}, + {file = "zstandard-0.19.0-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_12_i686.manylinux2010_i686.whl", hash = "sha256:c7560f622e3849cc8f3e999791a915addd08fafe80b47fcf3ffbda5b5151047c"}, + {file = "zstandard-0.19.0-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:e892d3177380ec080550b56a7ffeab680af25575d291766bdd875147ba246a91"}, + {file = "zstandard-0.19.0-cp39-cp39-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:60a86b7b2b1c300779167cf595e019e61afcc0e20c4838692983a921db9006ac"}, + {file = "zstandard-0.19.0-cp39-cp39-win32.whl", hash = "sha256:755020d5aeb1b10bffd93d119e7709a2a7475b6ad79c8d5226cea3f76d152ce0"}, + {file = "zstandard-0.19.0-cp39-cp39-win_amd64.whl", hash = "sha256:55a513ec67e85abd8b8b83af8813368036f03e2d29a50fc94033504918273980"}, + {file = "zstandard-0.19.0.tar.gz", hash = "sha256:31d12fcd942dd8dbf52ca5f6b1bbe287f44e5d551a081a983ff3ea2082867863"}, ] diff --git a/pyproject.toml b/pyproject.toml index b3e52c6f37..1c0253bfea 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -55,7 +55,7 @@ pyyaml = "^6.0.0" pydantic = "^1.10.2" fsspec = "2022.8.2" -zstandard = "^0.18.0" +zstandard = ">=0.18,<0.20" pyarrow = { version = ">=9,<11", optional = true } From 1f4377f54db9634ec04c68f6dae01dc73cc13885 Mon Sep 17 00:00:00 2001 From: Joshua Robinson <3334099+joshuarobinson@users.noreply.github.com> Date: Mon, 31 Oct 2022 17:44:00 +0100 Subject: [PATCH 250/642] Python: PyArrow timestamp in microseconds (#6070) The Iceberg spec keeps timestamp in microsecond format so we should convert to a PyArrow type that doesn't lose precision. ms = millisecond us = microsecond --- pyiceberg/io/pyarrow.py | 4 ++-- tests/io/test_pyarrow.py | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/pyiceberg/io/pyarrow.py b/pyiceberg/io/pyarrow.py index 248f3603a3..e70cec6349 100644 --- a/pyiceberg/io/pyarrow.py +++ b/pyiceberg/io/pyarrow.py @@ -352,12 +352,12 @@ def _(_: TimeType) -> pa.DataType: @_iceberg_to_pyarrow_type.register def _(_: TimestampType) -> pa.DataType: - return pa.timestamp(unit="ms") + return pa.timestamp(unit="us") @_iceberg_to_pyarrow_type.register def _(_: TimestamptzType) -> pa.DataType: - return pa.timestamp(unit="ms", tz="+00:00") + return pa.timestamp(unit="us", tz="+00:00") @_iceberg_to_pyarrow_type.register diff --git a/tests/io/test_pyarrow.py b/tests/io/test_pyarrow.py index c199c711d1..5a08011c03 100644 --- a/tests/io/test_pyarrow.py +++ b/tests/io/test_pyarrow.py @@ -364,12 +364,12 @@ def test_time_type_to_pyarrow(): def test_timestamp_type_to_pyarrow(): iceberg_type = TimestampType() - assert visit(iceberg_type, _ConvertToArrowSchema()) == pa.timestamp(unit="ms") + assert visit(iceberg_type, _ConvertToArrowSchema()) == pa.timestamp(unit="us") def test_timestamptz_type_to_pyarrow(): iceberg_type = TimestamptzType() - assert visit(iceberg_type, _ConvertToArrowSchema()) == pa.timestamp(unit="ms", tz="+00:00") + assert visit(iceberg_type, _ConvertToArrowSchema()) == pa.timestamp(unit="us", tz="+00:00") def test_string_type_to_pyarrow(): From 80360d965da5ea768a57b55348f049ee226f9508 Mon Sep 17 00:00:00 2001 From: Fokko Driesprong Date: Thu, 3 Nov 2022 18:39:12 +0100 Subject: [PATCH 251/642] Python: Use Types from Typing (#6114) --- .pre-commit-config.yaml | 2 +- CONTRIBUTING.md | 6 ++++ pyiceberg/avro/codecs/__init__.py | 4 ++- pyiceberg/avro/file.py | 9 ++--- pyiceberg/avro/reader.py | 19 +++++++---- pyiceberg/catalog/__init__.py | 49 +++++++++++++++------------- pyiceberg/expressions/literals.py | 29 +++++++++------- pyiceberg/utils/schema_conversion.py | 14 +++++--- 8 files changed, 82 insertions(+), 50 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 62b1881950..91452557d1 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -51,7 +51,7 @@ repos: rev: v3.0.0 hooks: - id: pyupgrade - args: [--py38-plus] + args: [ --py38-plus, --keep-runtime-typing ] - repo: https://github.com/pycqa/pylint rev: v2.15.3 hooks: diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index d673b43920..ee2d23a82b 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -124,6 +124,12 @@ Which will warn: Call to load_something, deprecated in 0.1.0, will be removed in 0.2.0. Please use load_something_else() instead. ``` +## Type annotations + +For the type annotation we currently rely on the `Typing` package that comes with Python. + +Since we're supporting from Python 3.8 onwards, we can't use the [type hints from the standard collections](https://peps.python.org/pep-0585/). + ## Third party libraries Since we expect PyIceberg to be integrated into the Python ecosystem, we want to be hesitant with the use of third party packages. Adding a lot of packages makes the library heavyweight, and causes incompatibilities with other projects if they use a different version of the library. Also, big libraries such as `s3fs`, `pyarrow`, `thrift` should be optional to avoid downloading everything, while not being sure if is actually being used. diff --git a/pyiceberg/avro/codecs/__init__.py b/pyiceberg/avro/codecs/__init__.py index 0512766280..8010b9397e 100644 --- a/pyiceberg/avro/codecs/__init__.py +++ b/pyiceberg/avro/codecs/__init__.py @@ -25,13 +25,15 @@ """ from __future__ import annotations +from typing import Dict, Optional, Type + from pyiceberg.avro.codecs.bzip2 import BZip2Codec from pyiceberg.avro.codecs.codec import Codec from pyiceberg.avro.codecs.deflate import DeflateCodec from pyiceberg.avro.codecs.snappy_codec import SnappyCodec from pyiceberg.avro.codecs.zstandard_codec import ZStandardCodec -KNOWN_CODECS: dict[str, type[Codec] | None] = { +KNOWN_CODECS: Dict[str, Optional[Type[Codec]]] = { "null": None, "bzip2": BZip2Codec, "snappy": SnappyCodec, diff --git a/pyiceberg/avro/file.py b/pyiceberg/avro/file.py index cce705382f..f17056640f 100644 --- a/pyiceberg/avro/file.py +++ b/pyiceberg/avro/file.py @@ -23,6 +23,7 @@ import json from dataclasses import dataclass from io import SEEK_SET +from typing import Optional, Type from pyiceberg.avro.codecs import KNOWN_CODECS, Codec from pyiceberg.avro.decoder import BinaryDecoder @@ -65,7 +66,7 @@ class AvroFileHeader: meta: dict[str, str] sync: bytes - def compression_codec(self) -> type[Codec] | None: + def compression_codec(self) -> Optional[Type[Codec]]: """Get the file's compression codec algorithm from the file's metadata. In the case of a null codec, we return a None indicating that we @@ -108,7 +109,7 @@ def __next__(self) -> AvroStruct: class AvroFile: input_file: InputFile - read_schema: Schema | None + read_schema: Optional[Schema] input_stream: InputStream header: AvroFileHeader schema: Schema @@ -116,9 +117,9 @@ class AvroFile: reader: StructReader decoder: BinaryDecoder - block: Block | None = None + block: Optional[Block] = None - def __init__(self, input_file: InputFile, read_schema: Schema | None = None) -> None: + def __init__(self, input_file: InputFile, read_schema: Optional[Schema] = None) -> None: self.input_file = input_file self.read_schema = read_schema diff --git a/pyiceberg/avro/reader.py b/pyiceberg/avro/reader.py index 3de1105575..6b93b67937 100644 --- a/pyiceberg/avro/reader.py +++ b/pyiceberg/avro/reader.py @@ -31,7 +31,14 @@ from datetime import date, datetime, time from decimal import Decimal from functools import singledispatch -from typing import Any, Callable +from typing import ( + Any, + Callable, + List, + Optional, + Tuple, + Union, +) from uuid import UUID from pyiceberg.avro.decoder import BinaryDecoder @@ -98,7 +105,7 @@ def _skip_map_array(decoder: BinaryDecoder, skip_entry: Callable) -> None: @dataclass(frozen=True) class AvroStruct(StructProtocol): - _data: list[Any | StructProtocol] = dataclassfield() + _data: List[Union[Any, StructProtocol]] = dataclassfield() def set(self, pos: int, value: Any) -> None: self._data[pos] = value @@ -242,7 +249,7 @@ def skip(self, decoder: BinaryDecoder) -> None: class OptionReader(Reader): option: Reader = dataclassfield() - def read(self, decoder: BinaryDecoder) -> Any | None: + def read(self, decoder: BinaryDecoder) -> Optional[Any]: # For the Iceberg spec it is required to set the default value to null # From https://iceberg.apache.org/spec/#avro # Optional fields must always set the Avro field default value to null. @@ -263,10 +270,10 @@ def skip(self, decoder: BinaryDecoder) -> None: @dataclass(frozen=True) class StructReader(Reader): - fields: tuple[tuple[int | None, Reader], ...] = dataclassfield() + fields: Tuple[Tuple[Optional[int], Reader], ...] = dataclassfield() def read(self, decoder: BinaryDecoder) -> AvroStruct: - result: list[Any | StructProtocol] = [None] * len(self.fields) + result: List[Union[Any, StructProtocol]] = [None] * len(self.fields) for (pos, field) in self.fields: if pos is not None: result[pos] = field.read(decoder) @@ -332,7 +339,7 @@ class ConstructReader(SchemaVisitor[Reader]): def schema(self, schema: Schema, struct_result: Reader) -> Reader: return struct_result - def struct(self, struct: StructType, field_results: list[Reader]) -> Reader: + def struct(self, struct: StructType, field_results: List[Reader]) -> Reader: return StructReader(tuple(enumerate(field_results))) def field(self, field: NestedField, field_result: Reader) -> Reader: diff --git a/pyiceberg/catalog/__init__.py b/pyiceberg/catalog/__init__.py index d4e3163c29..bb5cf417bf 100644 --- a/pyiceberg/catalog/__init__.py +++ b/pyiceberg/catalog/__init__.py @@ -21,7 +21,12 @@ from abc import ABC, abstractmethod from dataclasses import dataclass from enum import Enum -from typing import Callable +from typing import ( + Callable, + List, + Optional, + Union, +) from pyiceberg.exceptions import NotInstalledError from pyiceberg.schema import Schema @@ -70,7 +75,7 @@ def load_hive(name: str, conf: Properties) -> Catalog: } -def infer_catalog_type(name: str, catalog_properties: RecursiveDict) -> CatalogType | None: +def infer_catalog_type(name: str, catalog_properties: RecursiveDict) -> Optional[CatalogType]: """Tries to infer the type based on the dict Args: @@ -97,7 +102,7 @@ def infer_catalog_type(name: str, catalog_properties: RecursiveDict) -> CatalogT ) -def load_catalog(name: str, **properties: str | None) -> Catalog: +def load_catalog(name: str, **properties: Optional[str]) -> Catalog: """Load the catalog based on the properties Will look up the properties from the config, based on the name @@ -116,7 +121,7 @@ def load_catalog(name: str, **properties: str | None) -> Catalog: env = _ENV_CONFIG.get_catalog_config(name) conf = merge_config(env or {}, properties) - catalog_type: CatalogType | None + catalog_type: Optional[CatalogType] if provided_catalog_type := conf.get(TYPE): catalog_type = CatalogType[provided_catalog_type.upper()] else: @@ -130,9 +135,9 @@ def load_catalog(name: str, **properties: str | None) -> Catalog: @dataclass class PropertiesUpdateSummary: - removed: list[str] - updated: list[str] - missing: list[str] + removed: List[str] + updated: List[str] + missing: List[str] class Catalog(ABC): @@ -159,9 +164,9 @@ def __init__(self, name: str, **properties: str): @abstractmethod def create_table( self, - identifier: str | Identifier, + identifier: Union[str, Identifier], schema: Schema, - location: str | None = None, + location: Optional[str] = None, partition_spec: PartitionSpec = UNPARTITIONED_PARTITION_SPEC, sort_order: SortOrder = UNSORTED_SORT_ORDER, properties: Properties = EMPTY_DICT, @@ -184,7 +189,7 @@ def create_table( """ @abstractmethod - def load_table(self, identifier: str | Identifier) -> Table: + def load_table(self, identifier: Union[str, Identifier]) -> Table: """Loads the table's metadata and returns the table instance. You can also use this method to check for table existence using 'try catalog.table() except NoSuchTableError' @@ -201,7 +206,7 @@ def load_table(self, identifier: str | Identifier) -> Table: """ @abstractmethod - def drop_table(self, identifier: str | Identifier) -> None: + def drop_table(self, identifier: Union[str, Identifier]) -> None: """Drop a table. Args: @@ -212,7 +217,7 @@ def drop_table(self, identifier: str | Identifier) -> None: """ @abstractmethod - def purge_table(self, identifier: str | Identifier) -> None: + def purge_table(self, identifier: Union[str, Identifier]) -> None: """Drop a table and purge all data and metadata files. Args: @@ -223,7 +228,7 @@ def purge_table(self, identifier: str | Identifier) -> None: """ @abstractmethod - def rename_table(self, from_identifier: str | Identifier, to_identifier: str | Identifier) -> Table: + def rename_table(self, from_identifier: Union[str, Identifier], to_identifier: Union[str, Identifier]) -> Table: """Rename a fully classified table name Args: @@ -238,7 +243,7 @@ def rename_table(self, from_identifier: str | Identifier, to_identifier: str | I """ @abstractmethod - def create_namespace(self, namespace: str | Identifier, properties: Properties = EMPTY_DICT) -> None: + def create_namespace(self, namespace: Union[str, Identifier], properties: Properties = EMPTY_DICT) -> None: """Create a namespace in the catalog. Args: @@ -250,7 +255,7 @@ def create_namespace(self, namespace: str | Identifier, properties: Properties = """ @abstractmethod - def drop_namespace(self, namespace: str | Identifier) -> None: + def drop_namespace(self, namespace: Union[str, Identifier]) -> None: """Drop a namespace. Args: @@ -262,7 +267,7 @@ def drop_namespace(self, namespace: str | Identifier) -> None: """ @abstractmethod - def list_tables(self, namespace: str | Identifier) -> list[Identifier]: + def list_tables(self, namespace: Union[str, Identifier]) -> List[Identifier]: """List tables under the given namespace in the catalog. If namespace not provided, will list all tables in the catalog. @@ -278,7 +283,7 @@ def list_tables(self, namespace: str | Identifier) -> list[Identifier]: """ @abstractmethod - def list_namespaces(self, namespace: str | Identifier = ()) -> list[Identifier]: + def list_namespaces(self, namespace: Union[str, Identifier] = ()) -> List[Identifier]: """List namespaces from the given namespace. If not given, list top-level namespaces from the catalog. Args: @@ -292,7 +297,7 @@ def list_namespaces(self, namespace: str | Identifier = ()) -> list[Identifier]: """ @abstractmethod - def load_namespace_properties(self, namespace: str | Identifier) -> Properties: + def load_namespace_properties(self, namespace: Union[str, Identifier]) -> Properties: """Get properties for a namespace. Args: @@ -307,7 +312,7 @@ def load_namespace_properties(self, namespace: str | Identifier) -> Properties: @abstractmethod def update_namespace_properties( - self, namespace: str | Identifier, removals: set[str] | None = None, updates: Properties = EMPTY_DICT + self, namespace: Union[str, Identifier], removals: set[str] | None = None, updates: Properties = EMPTY_DICT ) -> PropertiesUpdateSummary: """Removes provided property keys and updates properties for a namespace. @@ -322,7 +327,7 @@ def update_namespace_properties( """ @staticmethod - def identifier_to_tuple(identifier: str | Identifier) -> Identifier: + def identifier_to_tuple(identifier: Union[str, Identifier]) -> Identifier: """Parses an identifier to a tuple. If the identifier is a string, it is split into a tuple on '.'. If it is a tuple, it is used as-is. @@ -336,7 +341,7 @@ def identifier_to_tuple(identifier: str | Identifier) -> Identifier: return identifier if isinstance(identifier, tuple) else tuple(str.split(identifier, ".")) @staticmethod - def table_name_from(identifier: str | Identifier) -> str: + def table_name_from(identifier: Union[str, Identifier]) -> str: """Extracts table name from a table identifier Args: @@ -348,7 +353,7 @@ def table_name_from(identifier: str | Identifier) -> str: return Catalog.identifier_to_tuple(identifier)[-1] @staticmethod - def namespace_from(identifier: str | Identifier) -> Identifier: + def namespace_from(identifier: Union[str, Identifier]) -> Identifier: """Extracts table namespace from a table identifier Args: diff --git a/pyiceberg/expressions/literals.py b/pyiceberg/expressions/literals.py index f49ee61805..458cbda41d 100644 --- a/pyiceberg/expressions/literals.py +++ b/pyiceberg/expressions/literals.py @@ -25,7 +25,12 @@ from abc import ABC, abstractmethod from decimal import ROUND_HALF_UP, Decimal from functools import singledispatch, singledispatchmethod -from typing import Generic, TypeVar +from typing import ( + Generic, + Optional, + TypeVar, + Union, +) from uuid import UUID from pyiceberg.types import ( @@ -217,7 +222,7 @@ def _(self, type_var: LongType) -> Literal[int]: return self @to.register(IntegerType) - def _(self, type_var: IntegerType) -> AboveMax | BelowMin | Literal[int]: + def _(self, _: IntegerType) -> Union[AboveMax, BelowMin, Literal[int]]: if IntegerType.max < self.value: return AboveMax() elif IntegerType.min > self.value: @@ -305,7 +310,7 @@ def _(self, type_var: DoubleType) -> Literal[float]: return self @to.register(FloatType) - def _(self, type_var: FloatType) -> AboveMax | BelowMin | Literal[float]: + def _(self, _: FloatType) -> Union[AboveMax, BelowMin, Literal[float]]: if FloatType.max < self.value: return AboveMax() elif FloatType.min > self.value: @@ -369,7 +374,7 @@ def to(self, type_var): return None @to.register(DecimalType) - def _(self, type_var: DecimalType) -> Literal[Decimal] | None: + def _(self, type_var: DecimalType) -> Optional[Literal[Decimal]]: if type_var.scale == abs(self.value.as_tuple().exponent): return self return None @@ -388,28 +393,28 @@ def _(self, type_var: StringType) -> Literal[str]: return self @to.register(DateType) - def _(self, type_var: DateType) -> Literal[int] | None: + def _(self, type_var: DateType) -> Optional[Literal[int]]: try: return DateLiteral(date_to_days(self.value)) except (TypeError, ValueError): return None @to.register(TimeType) - def _(self, type_var: TimeType) -> Literal[int] | None: + def _(self, type_var: TimeType) -> Optional[Literal[int]]: try: return TimeLiteral(time_to_micros(self.value)) except (TypeError, ValueError): return None @to.register(TimestampType) - def _(self, type_var: TimestampType) -> Literal[int] | None: + def _(self, type_var: TimestampType) -> Optional[Literal[int]]: try: return TimestampLiteral(timestamp_to_micros(self.value)) except (TypeError, ValueError): return None @to.register(TimestamptzType) - def _(self, type_var: TimestamptzType) -> Literal[int] | None: + def _(self, type_var: TimestamptzType) -> Optional[Literal[int]]: try: return TimestampLiteral(timestamptz_to_micros(self.value)) except (TypeError, ValueError): @@ -420,7 +425,7 @@ def _(self, type_var: UUIDType) -> Literal[UUID]: return UUIDLiteral(UUID(self.value)) @to.register(DecimalType) - def _(self, type_var: DecimalType) -> Literal[Decimal] | None: + def _(self, type_var: DecimalType) -> Optional[Literal[Decimal]]: dec = Decimal(self.value) if type_var.scale == abs(dec.as_tuple().exponent): return DecimalLiteral(dec) @@ -450,7 +455,7 @@ def to(self, type_var): return None @to.register(FixedType) - def _(self, type_var: FixedType) -> Literal[bytes] | None: + def _(self, type_var: FixedType) -> Optional[Literal[bytes]]: if len(self.value) == type_var.length: return self else: @@ -470,11 +475,11 @@ def to(self, type_var): return None @to.register(BinaryType) - def _(self, type_var: BinaryType) -> Literal[bytes]: + def _(self, _: BinaryType) -> Literal[bytes]: return self @to.register(FixedType) - def _(self, type_var: FixedType) -> Literal[bytes] | None: + def _(self, type_var: FixedType) -> Optional[Literal[bytes]]: if type_var.length == len(self.value): return FixedLiteral(self.value) else: diff --git a/pyiceberg/utils/schema_conversion.py b/pyiceberg/utils/schema_conversion.py index d6c32e0121..e3a9a437d1 100644 --- a/pyiceberg/utils/schema_conversion.py +++ b/pyiceberg/utils/schema_conversion.py @@ -18,7 +18,13 @@ from __future__ import annotations import logging -from typing import Any +from typing import ( + Any, + Dict, + List, + Tuple, + Union, +) from pyiceberg.schema import Schema from pyiceberg.types import ( @@ -112,7 +118,7 @@ def avro_to_iceberg(self, avro_schema: dict[str, Any]) -> Schema: """ return Schema(*[self._convert_field(field) for field in avro_schema["fields"]], schema_id=1) - def _resolve_union(self, type_union: dict | list | str) -> tuple[str | dict[str, Any], bool]: + def _resolve_union(self, type_union: Union[Dict, List, str]) -> Tuple[Union[str, Dict[str, Any]], bool]: """ Converts Unions into their type and resolves if the field is required @@ -135,7 +141,7 @@ def _resolve_union(self, type_union: dict | list | str) -> tuple[str | dict[str, Raises: TypeError: In the case non-optional union types are encountered """ - avro_types: dict | list + avro_types: Union[Dict, List] if isinstance(type_union, str): # It is a primitive and required return type_union, True @@ -161,7 +167,7 @@ def _resolve_union(self, type_union: dict | list | str) -> tuple[str | dict[str, # Filter the null value and return the type return list(filter(lambda t: t != "null", avro_types))[0], False - def _convert_schema(self, avro_type: str | dict[str, Any]) -> IcebergType: + def _convert_schema(self, avro_type: Union[str, Dict[str, Any]]) -> IcebergType: """ Resolves the Avro type From 48f2a5efd79eec0c239c6d67eab18e73fb0cdfc5 Mon Sep 17 00:00:00 2001 From: Fokko Driesprong Date: Thu, 3 Nov 2022 19:14:53 +0100 Subject: [PATCH 252/642] Python: Add the REST token to the properties (#6086) --- pyiceberg/catalog/__init__.py | 1 + pyiceberg/catalog/rest.py | 9 +++++---- pyiceberg/io/fsspec.py | 9 +++++++-- 3 files changed, 13 insertions(+), 6 deletions(-) diff --git a/pyiceberg/catalog/__init__.py b/pyiceberg/catalog/__init__.py index bb5cf417bf..f7c8499aff 100644 --- a/pyiceberg/catalog/__init__.py +++ b/pyiceberg/catalog/__init__.py @@ -45,6 +45,7 @@ _ENV_CONFIG = Config() +TOKEN = "token" TYPE = "type" URI = "uri" diff --git a/pyiceberg/catalog/rest.py b/pyiceberg/catalog/rest.py index 3caf091660..8822172101 100644 --- a/pyiceberg/catalog/rest.py +++ b/pyiceberg/catalog/rest.py @@ -30,6 +30,7 @@ from pyiceberg import __version__ from pyiceberg.catalog import ( + TOKEN, URI, Catalog, Identifier, @@ -87,7 +88,6 @@ class Endpoints: CREDENTIAL = "credential" GRANT_TYPE = "grant_type" SCOPE = "scope" -TOKEN = "token" TOKEN_EXCHANGE = "urn:ietf:params:oauth:grant-type:token-exchange" SEMICOLON = ":" KEY = "key" @@ -206,12 +206,13 @@ def _create_session(self) -> None: elif ssl_client_cert := ssl_client.get(CERT): self.session.cert = ssl_client_cert + # If we have credentials, but not a token, we want to fetch a token + if TOKEN not in self.properties and CREDENTIAL in self.properties: + self.properties[TOKEN] = self._fetch_access_token(self.properties[CREDENTIAL]) + # Set Auth token for subsequent calls in the session if token := self.properties.get(TOKEN): self.session.headers[AUTHORIZATION_HEADER] = f"{BEARER_PREFIX} {token}" - elif credential := self.properties.get(CREDENTIAL): - token = self._fetch_access_token(credential) - self.session.headers[AUTHORIZATION_HEADER] = f"{BEARER_PREFIX} {token}" # Set HTTP headers self.session.headers["Content-type"] = "application/json" diff --git a/pyiceberg/io/fsspec.py b/pyiceberg/io/fsspec.py index f47616a75e..641187721a 100644 --- a/pyiceberg/io/fsspec.py +++ b/pyiceberg/io/fsspec.py @@ -27,6 +27,7 @@ from requests import HTTPError from s3fs import S3FileSystem +from pyiceberg.catalog import TOKEN from pyiceberg.exceptions import SignError from pyiceberg.io import FileIO, InputFile, OutputFile from pyiceberg.typedef import Properties @@ -35,16 +36,20 @@ def s3v4_rest_signer(properties: Properties, request: AWSRequest, **_) -> AWSRequest: + if TOKEN not in properties: + raise SignError("Signer set, but token is not available") + signer_url = properties["uri"].rstrip("/") - signer_headers = {"Authorization": f"Bearer {properties['token']}"} + signer_headers = {"Authorization": f"Bearer {properties[TOKEN]}"} signer_body = { "method": request.method, "region": request.context["client_region"], "uri": request.url, "headers": {key: [val] for key, val in request.headers.items()}, } + + response = requests.post(f"{signer_url}/v1/aws/s3/sign", headers=signer_headers, json=signer_body) try: - response = requests.post(f"{signer_url}/v1/aws/s3/sign", headers=signer_headers, json=signer_body) response.raise_for_status() response_json = response.json() except HTTPError as e: From c2dea0bf993d16c465266be6ce38384ac42de860 Mon Sep 17 00:00:00 2001 From: Fokko Driesprong Date: Thu, 3 Nov 2022 19:18:34 +0100 Subject: [PATCH 253/642] Python: Pin versions explicitly (#6078) PR apache#6076 fails because the poetry update command also bumped the version of FastAvro. I don't think this is wanted behavior, therefore I suggest to pin the versions, instead of providing a lower bound. Dependabot will take care of the bumping of the versions every week. --- poetry.lock | 309 ++++++++++++++++++++++++++----------------------- pyproject.toml | 34 +++--- 2 files changed, 179 insertions(+), 164 deletions(-) diff --git a/poetry.lock b/poetry.lock index 3c674a695b..4f48e959d0 100644 --- a/poetry.lock +++ b/poetry.lock @@ -18,7 +18,7 @@ boto3 = ["boto3 (>=1.24.59,<1.24.60)"] [[package]] name = "aiohttp" -version = "3.8.1" +version = "3.8.3" description = "Async http client/server framework (asyncio)" category = "main" optional = true @@ -38,7 +38,7 @@ speedups = ["Brotli", "aiodns", "cchardet"] [[package]] name = "aioitertools" -version = "0.10.0" +version = "0.11.0" description = "itertools and builtins for AsyncIO and mixed iterables" category = "main" optional = true @@ -98,7 +98,7 @@ crt = ["awscrt (==0.14.0)"] [[package]] name = "build" -version = "0.8.0" +version = "0.9.0" description = "A simple, correct PEP 517 build frontend" category = "dev" optional = false @@ -118,7 +118,7 @@ virtualenv = ["virtualenv (>=20.0.35)"] [[package]] name = "certifi" -version = "2022.6.15.2" +version = "2022.9.24" description = "Python package for providing Mozilla's CA Bundle." category = "main" optional = false @@ -167,11 +167,11 @@ colorama = {version = "*", markers = "platform_system == \"Windows\""} [[package]] name = "colorama" -version = "0.4.5" +version = "0.4.6" description = "Cross-platform colored terminal text." category = "main" optional = false -python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*" +python-versions = "!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,!=3.4.*,!=3.5.*,!=3.6.*,>=2.7" [[package]] name = "commonmark" @@ -261,7 +261,7 @@ python-versions = ">=3.7" [[package]] name = "fsspec" -version = "2022.8.2" +version = "2022.10.0" description = "File-system specification" category = "main" optional = false @@ -292,7 +292,7 @@ tqdm = ["tqdm"] [[package]] name = "identify" -version = "2.5.5" +version = "2.5.8" description = "File identification library for Python" category = "dev" optional = false @@ -311,7 +311,7 @@ python-versions = ">=3.5" [[package]] name = "importlib-metadata" -version = "4.12.0" +version = "5.0.0" description = "Read metadata from Python packages" category = "dev" optional = false @@ -321,9 +321,9 @@ python-versions = ">=3.7" zipp = ">=0.5" [package.extras] -docs = ["jaraco.packaging (>=9)", "rst.linker (>=1.9)", "sphinx"] +docs = ["furo", "jaraco.packaging (>=9)", "jaraco.tidelift (>=1.4)", "rst.linker (>=1.9)", "sphinx (>=3.5)"] perf = ["ipython"] -testing = ["flufl.flake8", "importlib-resources (>=1.3)", "packaging", "pyfakefs", "pytest (>=6)", "pytest-black (>=0.3.7)", "pytest-checkdocs (>=2.4)", "pytest-cov", "pytest-enabler (>=1.3)", "pytest-flake8", "pytest-mypy (>=0.9.1)", "pytest-perf (>=0.9.2)"] +testing = ["flake8 (<5)", "flufl.flake8", "importlib-resources (>=1.3)", "packaging", "pyfakefs", "pytest (>=6)", "pytest-black (>=0.3.7)", "pytest-checkdocs (>=2.4)", "pytest-cov", "pytest-enabler (>=1.3)", "pytest-flake8", "pytest-mypy (>=0.9.1)", "pytest-perf (>=0.9.2)"] [[package]] name = "iniconfig" @@ -370,7 +370,7 @@ setuptools = "*" [[package]] name = "numpy" -version = "1.23.3" +version = "1.23.4" description = "NumPy is the fundamental package for array computing with Python." category = "main" optional = true @@ -610,7 +610,7 @@ jupyter = ["ipywidgets (>=7.5.1,<8.0.0)"] [[package]] name = "s3fs" -version = "2022.8.2" +version = "2022.10.0" description = "Convenient Filesystem interface over S3" category = "main" optional = true @@ -619,7 +619,7 @@ python-versions = ">= 3.7" [package.dependencies] aiobotocore = ">=2.4.0,<2.5.0" aiohttp = "<4.0.0a0 || >4.0.0a0,<4.0.0a1 || >4.0.0a1" -fsspec = "2022.8.2" +fsspec = "2022.10.0" [package.extras] awscli = ["aiobotocore[awscli] (>=2.4.0,<2.5.0)"] @@ -627,7 +627,7 @@ boto3 = ["aiobotocore[boto3] (>=2.4.0,<2.5.0)"] [[package]] name = "setuptools" -version = "65.4.1" +version = "65.5.0" description = "Easily download, build, install, upgrade, and uninstall Python packages" category = "dev" optional = false @@ -680,7 +680,7 @@ python-versions = ">=3.7" [[package]] name = "typing-extensions" -version = "4.3.0" +version = "4.4.0" description = "Backported and Experimental Type Hints for Python 3.7+" category = "main" optional = false @@ -701,19 +701,19 @@ socks = ["PySocks (>=1.5.6,!=1.5.7,<2.0)"] [[package]] name = "virtualenv" -version = "20.16.5" +version = "20.16.6" description = "Virtual Python Environment builder" category = "dev" optional = false python-versions = ">=3.6" [package.dependencies] -distlib = ">=0.3.5,<1" +distlib = ">=0.3.6,<1" filelock = ">=3.4.1,<4" platformdirs = ">=2.4,<3" [package.extras] -docs = ["proselint (>=0.13)", "sphinx (>=5.1.1)", "sphinx-argparse (>=0.3.1)", "sphinx-rtd-theme (>=1)", "towncrier (>=21.9)"] +docs = ["proselint (>=0.13)", "sphinx (>=5.3)", "sphinx-argparse (>=0.3.2)", "sphinx-rtd-theme (>=1)", "towncrier (>=22.8)"] testing = ["coverage (>=6.2)", "coverage-enable-subprocess (>=1)", "flaky (>=3.7)", "packaging (>=21.3)", "pytest (>=7.0.1)", "pytest-env (>=0.6.2)", "pytest-freezegun (>=0.4.2)", "pytest-mock (>=3.6.1)", "pytest-randomly (>=3.10.3)", "pytest-timeout (>=2.1)"] [[package]] @@ -738,15 +738,15 @@ multidict = ">=4.0" [[package]] name = "zipp" -version = "3.8.1" +version = "3.10.0" description = "Backport of pathlib-compatible object wrapper for zip files" category = "dev" optional = false python-versions = ">=3.7" [package.extras] -docs = ["jaraco.packaging (>=9)", "jaraco.tidelift (>=1.4)", "rst.linker (>=1.9)", "sphinx"] -testing = ["func-timeout", "jaraco.itertools", "pytest (>=6)", "pytest-black (>=0.3.7)", "pytest-checkdocs (>=2.4)", "pytest-cov", "pytest-enabler (>=1.3)", "pytest-flake8", "pytest-mypy (>=0.9.1)"] +docs = ["furo", "jaraco.packaging (>=9)", "jaraco.tidelift (>=1.4)", "rst.linker (>=1.9)", "sphinx (>=3.5)"] +testing = ["flake8 (<5)", "func-timeout", "jaraco.functools", "jaraco.itertools", "more-itertools", "pytest (>=6)", "pytest-black (>=0.3.7)", "pytest-checkdocs (>=2.4)", "pytest-cov", "pytest-enabler (>=1.3)", "pytest-flake8", "pytest-mypy (>=0.9.1)"] [[package]] name = "zstandard" @@ -771,7 +771,7 @@ snappy = ["python-snappy"] [metadata] lock-version = "1.1" python-versions = "^3.8" -content-hash = "7e931ee336b58a21ee512568504e6899a9c1476f0e9d17b09c43c8e88c2e313c" +content-hash = "18498bdad449ca86cb9f7151243a5e16c651dee0d3c399274d802eaff91c4451" [metadata.files] aiobotocore = [ @@ -779,82 +779,97 @@ aiobotocore = [ {file = "aiobotocore-2.4.0.tar.gz", hash = "sha256:f9fe0698cc497861bdb54cd16161c804031f758ada9480c35540f20c0c078385"}, ] aiohttp = [ - {file = "aiohttp-3.8.1-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:1ed0b6477896559f17b9eaeb6d38e07f7f9ffe40b9f0f9627ae8b9926ae260a8"}, - {file = "aiohttp-3.8.1-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:7dadf3c307b31e0e61689cbf9e06be7a867c563d5a63ce9dca578f956609abf8"}, - {file = "aiohttp-3.8.1-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:a79004bb58748f31ae1cbe9fa891054baaa46fb106c2dc7af9f8e3304dc30316"}, - {file = "aiohttp-3.8.1-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:12de6add4038df8f72fac606dff775791a60f113a725c960f2bab01d8b8e6b15"}, - {file = "aiohttp-3.8.1-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:6f0d5f33feb5f69ddd57a4a4bd3d56c719a141080b445cbf18f238973c5c9923"}, - {file = "aiohttp-3.8.1-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:eaba923151d9deea315be1f3e2b31cc39a6d1d2f682f942905951f4e40200922"}, - {file = "aiohttp-3.8.1-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_12_i686.manylinux2010_i686.whl", hash = "sha256:099ebd2c37ac74cce10a3527d2b49af80243e2a4fa39e7bce41617fbc35fa3c1"}, - {file = "aiohttp-3.8.1-cp310-cp310-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:2e5d962cf7e1d426aa0e528a7e198658cdc8aa4fe87f781d039ad75dcd52c516"}, - {file = "aiohttp-3.8.1-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:fa0ffcace9b3aa34d205d8130f7873fcfefcb6a4dd3dd705b0dab69af6712642"}, - {file = "aiohttp-3.8.1-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:61bfc23df345d8c9716d03717c2ed5e27374e0fe6f659ea64edcd27b4b044cf7"}, - {file = "aiohttp-3.8.1-cp310-cp310-musllinux_1_1_ppc64le.whl", hash = "sha256:31560d268ff62143e92423ef183680b9829b1b482c011713ae941997921eebc8"}, - {file = "aiohttp-3.8.1-cp310-cp310-musllinux_1_1_s390x.whl", hash = "sha256:01d7bdb774a9acc838e6b8f1d114f45303841b89b95984cbb7d80ea41172a9e3"}, - {file = "aiohttp-3.8.1-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:97ef77eb6b044134c0b3a96e16abcb05ecce892965a2124c566af0fd60f717e2"}, - {file = "aiohttp-3.8.1-cp310-cp310-win32.whl", hash = "sha256:c2aef4703f1f2ddc6df17519885dbfa3514929149d3ff900b73f45998f2532fa"}, - {file = "aiohttp-3.8.1-cp310-cp310-win_amd64.whl", hash = "sha256:713ac174a629d39b7c6a3aa757b337599798da4c1157114a314e4e391cd28e32"}, - {file = "aiohttp-3.8.1-cp36-cp36m-macosx_10_9_x86_64.whl", hash = "sha256:473d93d4450880fe278696549f2e7aed8cd23708c3c1997981464475f32137db"}, - {file = "aiohttp-3.8.1-cp36-cp36m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:99b5eeae8e019e7aad8af8bb314fb908dd2e028b3cdaad87ec05095394cce632"}, - {file = "aiohttp-3.8.1-cp36-cp36m-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:3af642b43ce56c24d063325dd2cf20ee012d2b9ba4c3c008755a301aaea720ad"}, - {file = "aiohttp-3.8.1-cp36-cp36m-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:c3630c3ef435c0a7c549ba170a0633a56e92629aeed0e707fec832dee313fb7a"}, - {file = "aiohttp-3.8.1-cp36-cp36m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_12_i686.manylinux2010_i686.whl", hash = "sha256:4a4a4e30bf1edcad13fb0804300557aedd07a92cabc74382fdd0ba6ca2661091"}, - {file = "aiohttp-3.8.1-cp36-cp36m-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:6f8b01295e26c68b3a1b90efb7a89029110d3a4139270b24fda961893216c440"}, - {file = "aiohttp-3.8.1-cp36-cp36m-musllinux_1_1_aarch64.whl", hash = "sha256:a25fa703a527158aaf10dafd956f7d42ac6d30ec80e9a70846253dd13e2f067b"}, - {file = "aiohttp-3.8.1-cp36-cp36m-musllinux_1_1_i686.whl", hash = "sha256:5bfde62d1d2641a1f5173b8c8c2d96ceb4854f54a44c23102e2ccc7e02f003ec"}, - {file = "aiohttp-3.8.1-cp36-cp36m-musllinux_1_1_ppc64le.whl", hash = "sha256:51467000f3647d519272392f484126aa716f747859794ac9924a7aafa86cd411"}, - {file = "aiohttp-3.8.1-cp36-cp36m-musllinux_1_1_s390x.whl", hash = "sha256:03a6d5349c9ee8f79ab3ff3694d6ce1cfc3ced1c9d36200cb8f08ba06bd3b782"}, - {file = "aiohttp-3.8.1-cp36-cp36m-musllinux_1_1_x86_64.whl", hash = "sha256:102e487eeb82afac440581e5d7f8f44560b36cf0bdd11abc51a46c1cd88914d4"}, - {file = "aiohttp-3.8.1-cp36-cp36m-win32.whl", hash = "sha256:4aed991a28ea3ce320dc8ce655875e1e00a11bdd29fe9444dd4f88c30d558602"}, - {file = "aiohttp-3.8.1-cp36-cp36m-win_amd64.whl", hash = "sha256:b0e20cddbd676ab8a64c774fefa0ad787cc506afd844de95da56060348021e96"}, - {file = "aiohttp-3.8.1-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:37951ad2f4a6df6506750a23f7cbabad24c73c65f23f72e95897bb2cecbae676"}, - {file = "aiohttp-3.8.1-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:5c23b1ad869653bc818e972b7a3a79852d0e494e9ab7e1a701a3decc49c20d51"}, - {file = "aiohttp-3.8.1-cp37-cp37m-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:15b09b06dae900777833fe7fc4b4aa426556ce95847a3e8d7548e2d19e34edb8"}, - {file = "aiohttp-3.8.1-cp37-cp37m-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:477c3ea0ba410b2b56b7efb072c36fa91b1e6fc331761798fa3f28bb224830dd"}, - {file = "aiohttp-3.8.1-cp37-cp37m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_12_i686.manylinux2010_i686.whl", hash = "sha256:2f2f69dca064926e79997f45b2f34e202b320fd3782f17a91941f7eb85502ee2"}, - {file = "aiohttp-3.8.1-cp37-cp37m-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:ef9612483cb35171d51d9173647eed5d0069eaa2ee812793a75373447d487aa4"}, - {file = "aiohttp-3.8.1-cp37-cp37m-musllinux_1_1_aarch64.whl", hash = "sha256:6d69f36d445c45cda7b3b26afef2fc34ef5ac0cdc75584a87ef307ee3c8c6d00"}, - {file = "aiohttp-3.8.1-cp37-cp37m-musllinux_1_1_i686.whl", hash = "sha256:55c3d1072704d27401c92339144d199d9de7b52627f724a949fc7d5fc56d8b93"}, - {file = "aiohttp-3.8.1-cp37-cp37m-musllinux_1_1_ppc64le.whl", hash = "sha256:b9d00268fcb9f66fbcc7cd9fe423741d90c75ee029a1d15c09b22d23253c0a44"}, - {file = "aiohttp-3.8.1-cp37-cp37m-musllinux_1_1_s390x.whl", hash = "sha256:07b05cd3305e8a73112103c834e91cd27ce5b4bd07850c4b4dbd1877d3f45be7"}, - {file = "aiohttp-3.8.1-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:c34dc4958b232ef6188c4318cb7b2c2d80521c9a56c52449f8f93ab7bc2a8a1c"}, - {file = "aiohttp-3.8.1-cp37-cp37m-win32.whl", hash = "sha256:d2f9b69293c33aaa53d923032fe227feac867f81682f002ce33ffae978f0a9a9"}, - {file = "aiohttp-3.8.1-cp37-cp37m-win_amd64.whl", hash = "sha256:6ae828d3a003f03ae31915c31fa684b9890ea44c9c989056fea96e3d12a9fa17"}, - {file = "aiohttp-3.8.1-cp38-cp38-macosx_10_9_universal2.whl", hash = "sha256:0c7ebbbde809ff4e970824b2b6cb7e4222be6b95a296e46c03cf050878fc1785"}, - {file = "aiohttp-3.8.1-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:8b7ef7cbd4fec9a1e811a5de813311ed4f7ac7d93e0fda233c9b3e1428f7dd7b"}, - {file = "aiohttp-3.8.1-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:c3d6a4d0619e09dcd61021debf7059955c2004fa29f48788a3dfaf9c9901a7cd"}, - {file = "aiohttp-3.8.1-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:718626a174e7e467f0558954f94af117b7d4695d48eb980146016afa4b580b2e"}, - {file = "aiohttp-3.8.1-cp38-cp38-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:589c72667a5febd36f1315aa6e5f56dd4aa4862df295cb51c769d16142ddd7cd"}, - {file = "aiohttp-3.8.1-cp38-cp38-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:2ed076098b171573161eb146afcb9129b5ff63308960aeca4b676d9d3c35e700"}, - {file = "aiohttp-3.8.1-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_12_i686.manylinux2010_i686.whl", hash = "sha256:086f92daf51a032d062ec5f58af5ca6a44d082c35299c96376a41cbb33034675"}, - {file = "aiohttp-3.8.1-cp38-cp38-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:11691cf4dc5b94236ccc609b70fec991234e7ef8d4c02dd0c9668d1e486f5abf"}, - {file = "aiohttp-3.8.1-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:31d1e1c0dbf19ebccbfd62eff461518dcb1e307b195e93bba60c965a4dcf1ba0"}, - {file = "aiohttp-3.8.1-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:11a67c0d562e07067c4e86bffc1553f2cf5b664d6111c894671b2b8712f3aba5"}, - {file = "aiohttp-3.8.1-cp38-cp38-musllinux_1_1_ppc64le.whl", hash = "sha256:bb01ba6b0d3f6c68b89fce7305080145d4877ad3acaed424bae4d4ee75faa950"}, - {file = "aiohttp-3.8.1-cp38-cp38-musllinux_1_1_s390x.whl", hash = "sha256:44db35a9e15d6fe5c40d74952e803b1d96e964f683b5a78c3cc64eb177878155"}, - {file = "aiohttp-3.8.1-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:844a9b460871ee0a0b0b68a64890dae9c415e513db0f4a7e3cab41a0f2fedf33"}, - {file = "aiohttp-3.8.1-cp38-cp38-win32.whl", hash = "sha256:7d08744e9bae2ca9c382581f7dce1273fe3c9bae94ff572c3626e8da5b193c6a"}, - {file = "aiohttp-3.8.1-cp38-cp38-win_amd64.whl", hash = "sha256:04d48b8ce6ab3cf2097b1855e1505181bdd05586ca275f2505514a6e274e8e75"}, - {file = "aiohttp-3.8.1-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:f5315a2eb0239185af1bddb1abf472d877fede3cc8d143c6cddad37678293237"}, - {file = "aiohttp-3.8.1-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:a996d01ca39b8dfe77440f3cd600825d05841088fd6bc0144cc6c2ec14cc5f74"}, - {file = "aiohttp-3.8.1-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:13487abd2f761d4be7c8ff9080de2671e53fff69711d46de703c310c4c9317ca"}, - {file = "aiohttp-3.8.1-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:ea302f34477fda3f85560a06d9ebdc7fa41e82420e892fc50b577e35fc6a50b2"}, - {file = "aiohttp-3.8.1-cp39-cp39-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:a2f635ce61a89c5732537a7896b6319a8fcfa23ba09bec36e1b1ac0ab31270d2"}, - {file = "aiohttp-3.8.1-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:e999f2d0e12eea01caeecb17b653f3713d758f6dcc770417cf29ef08d3931421"}, - {file = "aiohttp-3.8.1-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_12_i686.manylinux2010_i686.whl", hash = "sha256:0770e2806a30e744b4e21c9d73b7bee18a1cfa3c47991ee2e5a65b887c49d5cf"}, - {file = "aiohttp-3.8.1-cp39-cp39-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:d15367ce87c8e9e09b0f989bfd72dc641bcd04ba091c68cd305312d00962addd"}, - {file = "aiohttp-3.8.1-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:6c7cefb4b0640703eb1069835c02486669312bf2f12b48a748e0a7756d0de33d"}, - {file = "aiohttp-3.8.1-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:71927042ed6365a09a98a6377501af5c9f0a4d38083652bcd2281a06a5976724"}, - {file = "aiohttp-3.8.1-cp39-cp39-musllinux_1_1_ppc64le.whl", hash = "sha256:28d490af82bc6b7ce53ff31337a18a10498303fe66f701ab65ef27e143c3b0ef"}, - {file = "aiohttp-3.8.1-cp39-cp39-musllinux_1_1_s390x.whl", hash = "sha256:b6613280ccedf24354406caf785db748bebbddcf31408b20c0b48cb86af76866"}, - {file = "aiohttp-3.8.1-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:81e3d8c34c623ca4e36c46524a3530e99c0bc95ed068fd6e9b55cb721d408fb2"}, - {file = "aiohttp-3.8.1-cp39-cp39-win32.whl", hash = "sha256:7187a76598bdb895af0adbd2fb7474d7f6025d170bc0a1130242da817ce9e7d1"}, - {file = "aiohttp-3.8.1-cp39-cp39-win_amd64.whl", hash = "sha256:1c182cb873bc91b411e184dab7a2b664d4fea2743df0e4d57402f7f3fa644bac"}, - {file = "aiohttp-3.8.1.tar.gz", hash = "sha256:fc5471e1a54de15ef71c1bc6ebe80d4dc681ea600e68bfd1cbce40427f0b7578"}, + {file = "aiohttp-3.8.3-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:ba71c9b4dcbb16212f334126cc3d8beb6af377f6703d9dc2d9fb3874fd667ee9"}, + {file = "aiohttp-3.8.3-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:d24b8bb40d5c61ef2d9b6a8f4528c2f17f1c5d2d31fed62ec860f6006142e83e"}, + {file = "aiohttp-3.8.3-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:f88df3a83cf9df566f171adba39d5bd52814ac0b94778d2448652fc77f9eb491"}, + {file = "aiohttp-3.8.3-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:b97decbb3372d4b69e4d4c8117f44632551c692bb1361b356a02b97b69e18a62"}, + {file = "aiohttp-3.8.3-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:309aa21c1d54b8ef0723181d430347d7452daaff93e8e2363db8e75c72c2fb2d"}, + {file = "aiohttp-3.8.3-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:ad5383a67514e8e76906a06741febd9126fc7c7ff0f599d6fcce3e82b80d026f"}, + {file = "aiohttp-3.8.3-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:20acae4f268317bb975671e375493dbdbc67cddb5f6c71eebdb85b34444ac46b"}, + {file = "aiohttp-3.8.3-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:05a3c31c6d7cd08c149e50dc7aa2568317f5844acd745621983380597f027a18"}, + {file = "aiohttp-3.8.3-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:d6f76310355e9fae637c3162936e9504b4767d5c52ca268331e2756e54fd4ca5"}, + {file = "aiohttp-3.8.3-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:256deb4b29fe5e47893fa32e1de2d73c3afe7407738bd3c63829874661d4822d"}, + {file = "aiohttp-3.8.3-cp310-cp310-musllinux_1_1_ppc64le.whl", hash = "sha256:5c59fcd80b9049b49acd29bd3598cada4afc8d8d69bd4160cd613246912535d7"}, + {file = "aiohttp-3.8.3-cp310-cp310-musllinux_1_1_s390x.whl", hash = "sha256:059a91e88f2c00fe40aed9031b3606c3f311414f86a90d696dd982e7aec48142"}, + {file = "aiohttp-3.8.3-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:2feebbb6074cdbd1ac276dbd737b40e890a1361b3cc30b74ac2f5e24aab41f7b"}, + {file = "aiohttp-3.8.3-cp310-cp310-win32.whl", hash = "sha256:5bf651afd22d5f0c4be16cf39d0482ea494f5c88f03e75e5fef3a85177fecdeb"}, + {file = "aiohttp-3.8.3-cp310-cp310-win_amd64.whl", hash = "sha256:653acc3880459f82a65e27bd6526e47ddf19e643457d36a2250b85b41a564715"}, + {file = "aiohttp-3.8.3-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:86fc24e58ecb32aee09f864cb11bb91bc4c1086615001647dbfc4dc8c32f4008"}, + {file = "aiohttp-3.8.3-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:75e14eac916f024305db517e00a9252714fce0abcb10ad327fb6dcdc0d060f1d"}, + {file = "aiohttp-3.8.3-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:d1fde0f44029e02d02d3993ad55ce93ead9bb9b15c6b7ccd580f90bd7e3de476"}, + {file = "aiohttp-3.8.3-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:4ab94426ddb1ecc6a0b601d832d5d9d421820989b8caa929114811369673235c"}, + {file = "aiohttp-3.8.3-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:89d2e02167fa95172c017732ed7725bc8523c598757f08d13c5acca308e1a061"}, + {file = "aiohttp-3.8.3-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:02f9a2c72fc95d59b881cf38a4b2be9381b9527f9d328771e90f72ac76f31ad8"}, + {file = "aiohttp-3.8.3-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:9c7149272fb5834fc186328e2c1fa01dda3e1fa940ce18fded6d412e8f2cf76d"}, + {file = "aiohttp-3.8.3-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:512bd5ab136b8dc0ffe3fdf2dfb0c4b4f49c8577f6cae55dca862cd37a4564e2"}, + {file = "aiohttp-3.8.3-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:7018ecc5fe97027214556afbc7c502fbd718d0740e87eb1217b17efd05b3d276"}, + {file = "aiohttp-3.8.3-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:88c70ed9da9963d5496d38320160e8eb7e5f1886f9290475a881db12f351ab5d"}, + {file = "aiohttp-3.8.3-cp311-cp311-musllinux_1_1_ppc64le.whl", hash = "sha256:da22885266bbfb3f78218dc40205fed2671909fbd0720aedba39b4515c038091"}, + {file = "aiohttp-3.8.3-cp311-cp311-musllinux_1_1_s390x.whl", hash = "sha256:e65bc19919c910127c06759a63747ebe14f386cda573d95bcc62b427ca1afc73"}, + {file = "aiohttp-3.8.3-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:08c78317e950e0762c2983f4dd58dc5e6c9ff75c8a0efeae299d363d439c8e34"}, + {file = "aiohttp-3.8.3-cp311-cp311-win32.whl", hash = "sha256:45d88b016c849d74ebc6f2b6e8bc17cabf26e7e40c0661ddd8fae4c00f015697"}, + {file = "aiohttp-3.8.3-cp311-cp311-win_amd64.whl", hash = "sha256:96372fc29471646b9b106ee918c8eeb4cca423fcbf9a34daa1b93767a88a2290"}, + {file = "aiohttp-3.8.3-cp36-cp36m-macosx_10_9_x86_64.whl", hash = "sha256:c971bf3786b5fad82ce5ad570dc6ee420f5b12527157929e830f51c55dc8af77"}, + {file = "aiohttp-3.8.3-cp36-cp36m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:ff25f48fc8e623d95eca0670b8cc1469a83783c924a602e0fbd47363bb54aaca"}, + {file = "aiohttp-3.8.3-cp36-cp36m-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:e381581b37db1db7597b62a2e6b8b57c3deec95d93b6d6407c5b61ddc98aca6d"}, + {file = "aiohttp-3.8.3-cp36-cp36m-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:db19d60d846283ee275d0416e2a23493f4e6b6028825b51290ac05afc87a6f97"}, + {file = "aiohttp-3.8.3-cp36-cp36m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:25892c92bee6d9449ffac82c2fe257f3a6f297792cdb18ad784737d61e7a9a85"}, + {file = "aiohttp-3.8.3-cp36-cp36m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:398701865e7a9565d49189f6c90868efaca21be65c725fc87fc305906be915da"}, + {file = "aiohttp-3.8.3-cp36-cp36m-musllinux_1_1_aarch64.whl", hash = "sha256:4a4fbc769ea9b6bd97f4ad0b430a6807f92f0e5eb020f1e42ece59f3ecfc4585"}, + {file = "aiohttp-3.8.3-cp36-cp36m-musllinux_1_1_i686.whl", hash = "sha256:b29bfd650ed8e148f9c515474a6ef0ba1090b7a8faeee26b74a8ff3b33617502"}, + {file = "aiohttp-3.8.3-cp36-cp36m-musllinux_1_1_ppc64le.whl", hash = "sha256:1e56b9cafcd6531bab5d9b2e890bb4937f4165109fe98e2b98ef0dcfcb06ee9d"}, + {file = "aiohttp-3.8.3-cp36-cp36m-musllinux_1_1_s390x.whl", hash = "sha256:ec40170327d4a404b0d91855d41bfe1fe4b699222b2b93e3d833a27330a87a6d"}, + {file = "aiohttp-3.8.3-cp36-cp36m-musllinux_1_1_x86_64.whl", hash = "sha256:2df5f139233060578d8c2c975128fb231a89ca0a462b35d4b5fcf7c501ebdbe1"}, + {file = "aiohttp-3.8.3-cp36-cp36m-win32.whl", hash = "sha256:f973157ffeab5459eefe7b97a804987876dd0a55570b8fa56b4e1954bf11329b"}, + {file = "aiohttp-3.8.3-cp36-cp36m-win_amd64.whl", hash = "sha256:437399385f2abcd634865705bdc180c8314124b98299d54fe1d4c8990f2f9494"}, + {file = "aiohttp-3.8.3-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:09e28f572b21642128ef31f4e8372adb6888846f32fecb288c8b0457597ba61a"}, + {file = "aiohttp-3.8.3-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:6f3553510abdbec67c043ca85727396ceed1272eef029b050677046d3387be8d"}, + {file = "aiohttp-3.8.3-cp37-cp37m-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:e168a7560b7c61342ae0412997b069753f27ac4862ec7867eff74f0fe4ea2ad9"}, + {file = "aiohttp-3.8.3-cp37-cp37m-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:db4c979b0b3e0fa7e9e69ecd11b2b3174c6963cebadeecfb7ad24532ffcdd11a"}, + {file = "aiohttp-3.8.3-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:e164e0a98e92d06da343d17d4e9c4da4654f4a4588a20d6c73548a29f176abe2"}, + {file = "aiohttp-3.8.3-cp37-cp37m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:e8a78079d9a39ca9ca99a8b0ac2fdc0c4d25fc80c8a8a82e5c8211509c523363"}, + {file = "aiohttp-3.8.3-cp37-cp37m-musllinux_1_1_aarch64.whl", hash = "sha256:21b30885a63c3f4ff5b77a5d6caf008b037cb521a5f33eab445dc566f6d092cc"}, + {file = "aiohttp-3.8.3-cp37-cp37m-musllinux_1_1_i686.whl", hash = "sha256:4b0f30372cef3fdc262f33d06e7b411cd59058ce9174ef159ad938c4a34a89da"}, + {file = "aiohttp-3.8.3-cp37-cp37m-musllinux_1_1_ppc64le.whl", hash = "sha256:8135fa153a20d82ffb64f70a1b5c2738684afa197839b34cc3e3c72fa88d302c"}, + {file = "aiohttp-3.8.3-cp37-cp37m-musllinux_1_1_s390x.whl", hash = "sha256:ad61a9639792fd790523ba072c0555cd6be5a0baf03a49a5dd8cfcf20d56df48"}, + {file = "aiohttp-3.8.3-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:978b046ca728073070e9abc074b6299ebf3501e8dee5e26efacb13cec2b2dea0"}, + {file = "aiohttp-3.8.3-cp37-cp37m-win32.whl", hash = "sha256:0d2c6d8c6872df4a6ec37d2ede71eff62395b9e337b4e18efd2177de883a5033"}, + {file = "aiohttp-3.8.3-cp37-cp37m-win_amd64.whl", hash = "sha256:21d69797eb951f155026651f7e9362877334508d39c2fc37bd04ff55b2007091"}, + {file = "aiohttp-3.8.3-cp38-cp38-macosx_10_9_universal2.whl", hash = "sha256:2ca9af5f8f5812d475c5259393f52d712f6d5f0d7fdad9acdb1107dd9e3cb7eb"}, + {file = "aiohttp-3.8.3-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:1d90043c1882067f1bd26196d5d2db9aa6d268def3293ed5fb317e13c9413ea4"}, + {file = "aiohttp-3.8.3-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:d737fc67b9a970f3234754974531dc9afeea11c70791dcb7db53b0cf81b79784"}, + {file = "aiohttp-3.8.3-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:ebf909ea0a3fc9596e40d55d8000702a85e27fd578ff41a5500f68f20fd32e6c"}, + {file = "aiohttp-3.8.3-cp38-cp38-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:5835f258ca9f7c455493a57ee707b76d2d9634d84d5d7f62e77be984ea80b849"}, + {file = "aiohttp-3.8.3-cp38-cp38-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:da37dcfbf4b7f45d80ee386a5f81122501ec75672f475da34784196690762f4b"}, + {file = "aiohttp-3.8.3-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:87f44875f2804bc0511a69ce44a9595d5944837a62caecc8490bbdb0e18b1342"}, + {file = "aiohttp-3.8.3-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:527b3b87b24844ea7865284aabfab08eb0faf599b385b03c2aa91fc6edd6e4b6"}, + {file = "aiohttp-3.8.3-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:d5ba88df9aa5e2f806650fcbeedbe4f6e8736e92fc0e73b0400538fd25a4dd96"}, + {file = "aiohttp-3.8.3-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:e7b8813be97cab8cb52b1375f41f8e6804f6507fe4660152e8ca5c48f0436017"}, + {file = "aiohttp-3.8.3-cp38-cp38-musllinux_1_1_ppc64le.whl", hash = "sha256:2dea10edfa1a54098703cb7acaa665c07b4e7568472a47f4e64e6319d3821ccf"}, + {file = "aiohttp-3.8.3-cp38-cp38-musllinux_1_1_s390x.whl", hash = "sha256:713d22cd9643ba9025d33c4af43943c7a1eb8547729228de18d3e02e278472b6"}, + {file = "aiohttp-3.8.3-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:2d252771fc85e0cf8da0b823157962d70639e63cb9b578b1dec9868dd1f4f937"}, + {file = "aiohttp-3.8.3-cp38-cp38-win32.whl", hash = "sha256:66bd5f950344fb2b3dbdd421aaa4e84f4411a1a13fca3aeb2bcbe667f80c9f76"}, + {file = "aiohttp-3.8.3-cp38-cp38-win_amd64.whl", hash = "sha256:84b14f36e85295fe69c6b9789b51a0903b774046d5f7df538176516c3e422446"}, + {file = "aiohttp-3.8.3-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:16c121ba0b1ec2b44b73e3a8a171c4f999b33929cd2397124a8c7fcfc8cd9e06"}, + {file = "aiohttp-3.8.3-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:8d6aaa4e7155afaf994d7924eb290abbe81a6905b303d8cb61310a2aba1c68ba"}, + {file = "aiohttp-3.8.3-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:43046a319664a04b146f81b40e1545d4c8ac7b7dd04c47e40bf09f65f2437346"}, + {file = "aiohttp-3.8.3-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:599418aaaf88a6d02a8c515e656f6faf3d10618d3dd95866eb4436520096c84b"}, + {file = "aiohttp-3.8.3-cp39-cp39-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:92a2964319d359f494f16011e23434f6f8ef0434acd3cf154a6b7bec511e2fb7"}, + {file = "aiohttp-3.8.3-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:73a4131962e6d91109bca6536416aa067cf6c4efb871975df734f8d2fd821b37"}, + {file = "aiohttp-3.8.3-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:598adde339d2cf7d67beaccda3f2ce7c57b3b412702f29c946708f69cf8222aa"}, + {file = "aiohttp-3.8.3-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:75880ed07be39beff1881d81e4a907cafb802f306efd6d2d15f2b3c69935f6fb"}, + {file = "aiohttp-3.8.3-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:a0239da9fbafd9ff82fd67c16704a7d1bccf0d107a300e790587ad05547681c8"}, + {file = "aiohttp-3.8.3-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:4e3a23ec214e95c9fe85a58470b660efe6534b83e6cbe38b3ed52b053d7cb6ad"}, + {file = "aiohttp-3.8.3-cp39-cp39-musllinux_1_1_ppc64le.whl", hash = "sha256:47841407cc89a4b80b0c52276f3cc8138bbbfba4b179ee3acbd7d77ae33f7ac4"}, + {file = "aiohttp-3.8.3-cp39-cp39-musllinux_1_1_s390x.whl", hash = "sha256:54d107c89a3ebcd13228278d68f1436d3f33f2dd2af5415e3feaeb1156e1a62c"}, + {file = "aiohttp-3.8.3-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:c37c5cce780349d4d51739ae682dec63573847a2a8dcb44381b174c3d9c8d403"}, + {file = "aiohttp-3.8.3-cp39-cp39-win32.whl", hash = "sha256:f178d2aadf0166be4df834c4953da2d7eef24719e8aec9a65289483eeea9d618"}, + {file = "aiohttp-3.8.3-cp39-cp39-win_amd64.whl", hash = "sha256:88e5be56c231981428f4f506c68b6a46fa25c4123a2e86d156c58a8369d31ab7"}, + {file = "aiohttp-3.8.3.tar.gz", hash = "sha256:3828fb41b7203176b82fe5d699e0d845435f2374750a44b480ea6b930f6be269"}, ] aioitertools = [ - {file = "aioitertools-0.10.0-py3-none-any.whl", hash = "sha256:a2ea2a39ebf272a2fbb58bfdb73e1daeeb6686edbbc8082215dfc8b8ffffa6e8"}, - {file = "aioitertools-0.10.0.tar.gz", hash = "sha256:7d1d1d4a03d462c5a0840787d3df098f125847e0d38b833b30f8f8cbc45a1420"}, + {file = "aioitertools-0.11.0-py3-none-any.whl", hash = "sha256:04b95e3dab25b449def24d7df809411c10e62aab0cbe31a50ca4e68748c43394"}, + {file = "aioitertools-0.11.0.tar.gz", hash = "sha256:42c68b8dd3a69c2bf7f2233bf7df4bb58b557bca5252ac02ed5187bbc67d6831"}, ] aiosignal = [ {file = "aiosignal-1.2.0-py3-none-any.whl", hash = "sha256:26e62109036cd181df6e6ad646f91f0dcfd05fe16d0cb924138ff2ab75d64e3a"}, @@ -873,12 +888,12 @@ botocore = [ {file = "botocore-1.27.59.tar.gz", hash = "sha256:eda4aed6ee719a745d1288eaf1beb12f6f6448ad1fa12f159405db14ba9c92cf"}, ] build = [ - {file = "build-0.8.0-py3-none-any.whl", hash = "sha256:19b0ed489f92ace6947698c3ca8436cb0556a66e2aa2d34cd70e2a5d27cd0437"}, - {file = "build-0.8.0.tar.gz", hash = "sha256:887a6d471c901b1a6e6574ebaeeebb45e5269a79d095fe9a8f88d6614ed2e5f0"}, + {file = "build-0.9.0-py3-none-any.whl", hash = "sha256:38a7a2b7a0bdc61a42a0a67509d88c71ecfc37b393baba770fae34e20929ff69"}, + {file = "build-0.9.0.tar.gz", hash = "sha256:1a07724e891cbd898923145eb7752ee7653674c511378eb9c7691aab1612bc3c"}, ] certifi = [ - {file = "certifi-2022.6.15.2-py3-none-any.whl", hash = "sha256:0aa1a42fbd57645fabeb6290a7687c21755b0344ecaeaa05f4e9f6207ae2e9a8"}, - {file = "certifi-2022.6.15.2.tar.gz", hash = "sha256:aa08c101214127b9b0472ca6338315113c9487d45376fd3e669201b477c71003"}, + {file = "certifi-2022.9.24-py3-none-any.whl", hash = "sha256:90c1a32f1d68f940488354e36370f6cca89f0f106db09518524c88d6ed83f382"}, + {file = "certifi-2022.9.24.tar.gz", hash = "sha256:0d9c601124e5a6ba9712dbc60d9c53c21e34f5f641fe83002317394311bdce14"}, ] cffi = [ {file = "cffi-1.15.1-cp27-cp27m-macosx_10_9_x86_64.whl", hash = "sha256:a66d3508133af6e8548451b25058d5812812ec3798c886bf38ed24a98216fab2"}, @@ -959,8 +974,8 @@ click = [ {file = "click-8.1.3.tar.gz", hash = "sha256:7682dc8afb30297001674575ea00d1814d808d6a36af415a82bd481d37ba7b8e"}, ] colorama = [ - {file = "colorama-0.4.5-py2.py3-none-any.whl", hash = "sha256:854bf444933e37f5824ae7bfc1e98d5bce2ebe4160d46b5edf346a89358e99da"}, - {file = "colorama-0.4.5.tar.gz", hash = "sha256:e6c6b4334fc50988a639d9b98aa429a0b57da6e17b9a44f0451f930b6967b7a4"}, + {file = "colorama-0.4.6-py2.py3-none-any.whl", hash = "sha256:4f1d9991f5acc0ca119f9d443620b77f9d6b33703e51011c16baf57afb285fc6"}, + {file = "colorama-0.4.6.tar.gz", hash = "sha256:08695f5cb7ed6e0531a20572697297273c47b8cae5a63ffc6d6ed5c201be6e44"}, ] commonmark = [ {file = "commonmark-0.9.1-py2.py3-none-any.whl", hash = "sha256:da2f38c92590f83de410ba1a3cbceafbc74fee9def35f9251ba9a971d6d66fd9"}, @@ -1115,20 +1130,20 @@ frozenlist = [ {file = "frozenlist-1.3.1.tar.gz", hash = "sha256:3a735e4211a04ccfa3f4833547acdf5d2f863bfeb01cfd3edaffbc251f15cec8"}, ] fsspec = [ - {file = "fsspec-2022.8.2-py3-none-any.whl", hash = "sha256:6374804a2c0d24f225a67d009ee1eabb4046ad00c793c3f6df97e426c890a1d9"}, - {file = "fsspec-2022.8.2.tar.gz", hash = "sha256:7f12b90964a98a7e921d27fb36be536ea036b73bf3b724ac0b0bd7b8e39c7c18"}, + {file = "fsspec-2022.10.0-py3-none-any.whl", hash = "sha256:6b7c6ab3b476cdf17efcfeccde7fca28ef5a48f73a71010aaceec5fc15bf9ebf"}, + {file = "fsspec-2022.10.0.tar.gz", hash = "sha256:cb6092474e90487a51de768170f3afa50ca8982c26150a59072b16433879ff1d"}, ] identify = [ - {file = "identify-2.5.5-py2.py3-none-any.whl", hash = "sha256:ef78c0d96098a3b5fe7720be4a97e73f439af7cf088ebf47b620aeaa10fadf97"}, - {file = "identify-2.5.5.tar.gz", hash = "sha256:322a5699daecf7c6fd60e68852f36f2ecbb6a36ff6e6e973e0d2bb6fca203ee6"}, + {file = "identify-2.5.8-py2.py3-none-any.whl", hash = "sha256:48b7925fe122720088aeb7a6c34f17b27e706b72c61070f27fe3789094233440"}, + {file = "identify-2.5.8.tar.gz", hash = "sha256:7a214a10313b9489a0d61467db2856ae8d0b8306fc923e03a9effa53d8aedc58"}, ] idna = [ {file = "idna-3.4-py3-none-any.whl", hash = "sha256:90b77e79eaa3eba6de819a0c442c0b4ceefc341a7a2ab77d7562bf49f425c5c2"}, {file = "idna-3.4.tar.gz", hash = "sha256:814f528e8dead7d329833b91c5faa87d60bf71824cd12a7530b5526063d02cb4"}, ] importlib-metadata = [ - {file = "importlib_metadata-4.12.0-py3-none-any.whl", hash = "sha256:7401a975809ea1fdc658c3aa4f78cc2195a0e019c5cbc4c06122884e9ae80c23"}, - {file = "importlib_metadata-4.12.0.tar.gz", hash = "sha256:637245b8bab2b6502fcbc752cc4b7a6f6243bb02b31c5c26156ad103d3d45670"}, + {file = "importlib_metadata-5.0.0-py3-none-any.whl", hash = "sha256:ddb0e35065e8938f867ed4928d0ae5bf2a53b7773871bfe6bcc7e4fcdc7dea43"}, + {file = "importlib_metadata-5.0.0.tar.gz", hash = "sha256:da31db32b304314d044d3c12c79bd59e307889b287ad12ff387b3500835fc2ab"}, ] iniconfig = [ {file = "iniconfig-1.1.1-py2.py3-none-any.whl", hash = "sha256:011e24c64b7f47f6ebd835bb12a743f2fbe9a26d4cecaa7f53bc4f35ee9da8b3"}, @@ -1232,34 +1247,34 @@ nodeenv = [ {file = "nodeenv-1.7.0.tar.gz", hash = "sha256:e0e7f7dfb85fc5394c6fe1e8fa98131a2473e04311a45afb6508f7cf1836fa2b"}, ] numpy = [ - {file = "numpy-1.23.3-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:c9f707b5bb73bf277d812ded9896f9512a43edff72712f31667d0a8c2f8e71ee"}, - {file = "numpy-1.23.3-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:ffcf105ecdd9396e05a8e58e81faaaf34d3f9875f137c7372450baa5d77c9a54"}, - {file = "numpy-1.23.3-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:0ea3f98a0ffce3f8f57675eb9119f3f4edb81888b6874bc1953f91e0b1d4f440"}, - {file = "numpy-1.23.3-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:004f0efcb2fe1c0bd6ae1fcfc69cc8b6bf2407e0f18be308612007a0762b4089"}, - {file = "numpy-1.23.3-cp310-cp310-win32.whl", hash = "sha256:98dcbc02e39b1658dc4b4508442a560fe3ca5ca0d989f0df062534e5ca3a5c1a"}, - {file = "numpy-1.23.3-cp310-cp310-win_amd64.whl", hash = "sha256:39a664e3d26ea854211867d20ebcc8023257c1800ae89773cbba9f9e97bae036"}, - {file = "numpy-1.23.3-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:1f27b5322ac4067e67c8f9378b41c746d8feac8bdd0e0ffede5324667b8a075c"}, - {file = "numpy-1.23.3-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:2ad3ec9a748a8943e6eb4358201f7e1c12ede35f510b1a2221b70af4bb64295c"}, - {file = "numpy-1.23.3-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:bdc9febce3e68b697d931941b263c59e0c74e8f18861f4064c1f712562903411"}, - {file = "numpy-1.23.3-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:301c00cf5e60e08e04d842fc47df641d4a181e651c7135c50dc2762ffe293dbd"}, - {file = "numpy-1.23.3-cp311-cp311-win32.whl", hash = "sha256:7cd1328e5bdf0dee621912f5833648e2daca72e3839ec1d6695e91089625f0b4"}, - {file = "numpy-1.23.3-cp311-cp311-win_amd64.whl", hash = "sha256:8355fc10fd33a5a70981a5b8a0de51d10af3688d7a9e4a34fcc8fa0d7467bb7f"}, - {file = "numpy-1.23.3-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:bc6e8da415f359b578b00bcfb1d08411c96e9a97f9e6c7adada554a0812a6cc6"}, - {file = "numpy-1.23.3-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:22d43376ee0acd547f3149b9ec12eec2f0ca4a6ab2f61753c5b29bb3e795ac4d"}, - {file = "numpy-1.23.3-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a64403f634e5ffdcd85e0b12c08f04b3080d3e840aef118721021f9b48fc1460"}, - {file = "numpy-1.23.3-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:efd9d3abe5774404becdb0748178b48a218f1d8c44e0375475732211ea47c67e"}, - {file = "numpy-1.23.3-cp38-cp38-win32.whl", hash = "sha256:f8c02ec3c4c4fcb718fdf89a6c6f709b14949408e8cf2a2be5bfa9c49548fd85"}, - {file = "numpy-1.23.3-cp38-cp38-win_amd64.whl", hash = "sha256:e868b0389c5ccfc092031a861d4e158ea164d8b7fdbb10e3b5689b4fc6498df6"}, - {file = "numpy-1.23.3-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:09f6b7bdffe57fc61d869a22f506049825d707b288039d30f26a0d0d8ea05164"}, - {file = "numpy-1.23.3-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:8c79d7cf86d049d0c5089231a5bcd31edb03555bd93d81a16870aa98c6cfb79d"}, - {file = "numpy-1.23.3-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:e5d5420053bbb3dd64c30e58f9363d7a9c27444c3648e61460c1237f9ec3fa14"}, - {file = "numpy-1.23.3-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:d5422d6a1ea9b15577a9432e26608c73a78faf0b9039437b075cf322c92e98e7"}, - {file = "numpy-1.23.3-cp39-cp39-win32.whl", hash = "sha256:c1ba66c48b19cc9c2975c0d354f24058888cdc674bebadceb3cdc9ec403fb5d1"}, - {file = "numpy-1.23.3-cp39-cp39-win_amd64.whl", hash = "sha256:78a63d2df1d947bd9d1b11d35564c2f9e4b57898aae4626638056ec1a231c40c"}, - {file = "numpy-1.23.3-pp38-pypy38_pp73-macosx_10_9_x86_64.whl", hash = "sha256:17c0e467ade9bda685d5ac7f5fa729d8d3e76b23195471adae2d6a6941bd2c18"}, - {file = "numpy-1.23.3-pp38-pypy38_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:91b8d6768a75247026e951dce3b2aac79dc7e78622fc148329135ba189813584"}, - {file = "numpy-1.23.3-pp38-pypy38_pp73-win_amd64.whl", hash = "sha256:94c15ca4e52671a59219146ff584488907b1f9b3fc232622b47e2cf832e94fb8"}, - {file = "numpy-1.23.3.tar.gz", hash = "sha256:51bf49c0cd1d52be0a240aa66f3458afc4b95d8993d2d04f0d91fa60c10af6cd"}, + {file = "numpy-1.23.4-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:95d79ada05005f6f4f337d3bb9de8a7774f259341c70bc88047a1f7b96a4bcb2"}, + {file = "numpy-1.23.4-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:926db372bc4ac1edf81cfb6c59e2a881606b409ddc0d0920b988174b2e2a767f"}, + {file = "numpy-1.23.4-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:c237129f0e732885c9a6076a537e974160482eab8f10db6292e92154d4c67d71"}, + {file = "numpy-1.23.4-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:a8365b942f9c1a7d0f0dc974747d99dd0a0cdfc5949a33119caf05cb314682d3"}, + {file = "numpy-1.23.4-cp310-cp310-win32.whl", hash = "sha256:2341f4ab6dba0834b685cce16dad5f9b6606ea8a00e6da154f5dbded70fdc4dd"}, + {file = "numpy-1.23.4-cp310-cp310-win_amd64.whl", hash = "sha256:d331afac87c92373826af83d2b2b435f57b17a5c74e6268b79355b970626e329"}, + {file = "numpy-1.23.4-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:488a66cb667359534bc70028d653ba1cf307bae88eab5929cd707c761ff037db"}, + {file = "numpy-1.23.4-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:ce03305dd694c4873b9429274fd41fc7eb4e0e4dea07e0af97a933b079a5814f"}, + {file = "numpy-1.23.4-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:8981d9b5619569899666170c7c9748920f4a5005bf79c72c07d08c8a035757b0"}, + {file = "numpy-1.23.4-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:7a70a7d3ce4c0e9284e92285cba91a4a3f5214d87ee0e95928f3614a256a1488"}, + {file = "numpy-1.23.4-cp311-cp311-win32.whl", hash = "sha256:5e13030f8793e9ee42f9c7d5777465a560eb78fa7e11b1c053427f2ccab90c79"}, + {file = "numpy-1.23.4-cp311-cp311-win_amd64.whl", hash = "sha256:7607b598217745cc40f751da38ffd03512d33ec06f3523fb0b5f82e09f6f676d"}, + {file = "numpy-1.23.4-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:7ab46e4e7ec63c8a5e6dbf5c1b9e1c92ba23a7ebecc86c336cb7bf3bd2fb10e5"}, + {file = "numpy-1.23.4-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:a8aae2fb3180940011b4862b2dd3756616841c53db9734b27bb93813cd79fce6"}, + {file = "numpy-1.23.4-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:8c053d7557a8f022ec823196d242464b6955a7e7e5015b719e76003f63f82d0f"}, + {file = "numpy-1.23.4-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:a0882323e0ca4245eb0a3d0a74f88ce581cc33aedcfa396e415e5bba7bf05f68"}, + {file = "numpy-1.23.4-cp38-cp38-win32.whl", hash = "sha256:dada341ebb79619fe00a291185bba370c9803b1e1d7051610e01ed809ef3a4ba"}, + {file = "numpy-1.23.4-cp38-cp38-win_amd64.whl", hash = "sha256:0fe563fc8ed9dc4474cbf70742673fc4391d70f4363f917599a7fa99f042d5a8"}, + {file = "numpy-1.23.4-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:c67b833dbccefe97cdd3f52798d430b9d3430396af7cdb2a0c32954c3ef73894"}, + {file = "numpy-1.23.4-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:f76025acc8e2114bb664294a07ede0727aa75d63a06d2fae96bf29a81747e4a7"}, + {file = "numpy-1.23.4-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:12ac457b63ec8ded85d85c1e17d85efd3c2b0967ca39560b307a35a6703a4735"}, + {file = "numpy-1.23.4-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:95de7dc7dc47a312f6feddd3da2500826defdccbc41608d0031276a24181a2c0"}, + {file = "numpy-1.23.4-cp39-cp39-win32.whl", hash = "sha256:f2f390aa4da44454db40a1f0201401f9036e8d578a25f01a6e237cea238337ef"}, + {file = "numpy-1.23.4-cp39-cp39-win_amd64.whl", hash = "sha256:f260da502d7441a45695199b4e7fd8ca87db659ba1c78f2bbf31f934fe76ae0e"}, + {file = "numpy-1.23.4-pp38-pypy38_pp73-macosx_10_9_x86_64.whl", hash = "sha256:61be02e3bf810b60ab74e81d6d0d36246dbfb644a462458bb53b595791251911"}, + {file = "numpy-1.23.4-pp38-pypy38_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:296d17aed51161dbad3c67ed6d164e51fcd18dbcd5dd4f9d0a9c6055dce30810"}, + {file = "numpy-1.23.4-pp38-pypy38_pp73-win_amd64.whl", hash = "sha256:4d52914c88b4930dafb6c48ba5115a96cbab40f45740239d9f4159c4ba779962"}, + {file = "numpy-1.23.4.tar.gz", hash = "sha256:ed2cc92af0efad20198638c69bb0fc2870a58dabfba6eb722c933b48556c686c"}, ] packaging = [ {file = "packaging-21.3-py3-none-any.whl", hash = "sha256:ef103e05f519cdc783ae24ea4e2e0f508a9c99b2d4969652eed6a2e1ea5bd522"}, @@ -1472,12 +1487,12 @@ rich = [ {file = "rich-12.6.0.tar.gz", hash = "sha256:ba3a3775974105c221d31141f2c116f4fd65c5ceb0698657a11e9f295ec93fd0"}, ] s3fs = [ - {file = "s3fs-2022.8.2-py3-none-any.whl", hash = "sha256:63369eb9b14687080fc5ac72912f51d5c99d58a24d488199c19e664d0f872fd2"}, - {file = "s3fs-2022.8.2.tar.gz", hash = "sha256:3ca0701a89a9e125a28de90829d19f96f41ddb1d8b4379076c29ed82c4af86cd"}, + {file = "s3fs-2022.10.0-py3-none-any.whl", hash = "sha256:1e134c3577171699feb7c1a0c4713260d5b48296e1708737ff940baef6e2c153"}, + {file = "s3fs-2022.10.0.tar.gz", hash = "sha256:e8deb80f20bd0b2059141b874fdb9d6aeb8cce35312ea5f2c02b225a78a00406"}, ] setuptools = [ - {file = "setuptools-65.4.1-py3-none-any.whl", hash = "sha256:1b6bdc6161661409c5f21508763dc63ab20a9ac2f8ba20029aaaa7fdb9118012"}, - {file = "setuptools-65.4.1.tar.gz", hash = "sha256:3050e338e5871e70c72983072fe34f6032ae1cdeeeb67338199c2f74e083a80e"}, + {file = "setuptools-65.5.0-py3-none-any.whl", hash = "sha256:f62ea9da9ed6289bfe868cd6845968a2c854d1427f8548d52cae02a42b4f0356"}, + {file = "setuptools-65.5.0.tar.gz", hash = "sha256:512e5536220e38146176efb833d4a62aa726b7bbff82cfbc8ba9eaa3996e0b17"}, ] six = [ {file = "six-1.16.0-py2.py3-none-any.whl", hash = "sha256:8abb2f1d86890a2dfb989f9a77cfcfd3e47c2a354b01111771326f8aa26e0254"}, @@ -1495,16 +1510,16 @@ tomli = [ {file = "tomli-2.0.1.tar.gz", hash = "sha256:de526c12914f0c550d15924c62d72abc48d6fe7364aa87328337a31007fe8a4f"}, ] typing-extensions = [ - {file = "typing_extensions-4.3.0-py3-none-any.whl", hash = "sha256:25642c956049920a5aa49edcdd6ab1e06d7e5d467fc00e0506c44ac86fbfca02"}, - {file = "typing_extensions-4.3.0.tar.gz", hash = "sha256:e6d2677a32f47fc7eb2795db1dd15c1f34eff616bcaf2cfb5e997f854fa1c4a6"}, + {file = "typing_extensions-4.4.0-py3-none-any.whl", hash = "sha256:16fa4864408f655d35ec496218b85f79b3437c829e93320c7c9215ccfd92489e"}, + {file = "typing_extensions-4.4.0.tar.gz", hash = "sha256:1511434bb92bf8dd198c12b1cc812e800d4181cfcb867674e0f8279cc93087aa"}, ] urllib3 = [ {file = "urllib3-1.26.12-py2.py3-none-any.whl", hash = "sha256:b930dd878d5a8afb066a637fbb35144fe7901e3b209d1cd4f524bd0e9deee997"}, {file = "urllib3-1.26.12.tar.gz", hash = "sha256:3fa96cf423e6987997fc326ae8df396db2a8b7c667747d47ddd8ecba91f4a74e"}, ] virtualenv = [ - {file = "virtualenv-20.16.5-py3-none-any.whl", hash = "sha256:d07dfc5df5e4e0dbc92862350ad87a36ed505b978f6c39609dc489eadd5b0d27"}, - {file = "virtualenv-20.16.5.tar.gz", hash = "sha256:227ea1b9994fdc5ea31977ba3383ef296d7472ea85be9d6732e42a91c04e80da"}, + {file = "virtualenv-20.16.6-py3-none-any.whl", hash = "sha256:186ca84254abcbde98180fd17092f9628c5fe742273c02724972a1d8a2035108"}, + {file = "virtualenv-20.16.6.tar.gz", hash = "sha256:530b850b523c6449406dfba859d6345e48ef19b8439606c5d74d7d3c9e14d76e"}, ] wrapt = [ {file = "wrapt-1.14.1-cp27-cp27m-macosx_10_9_x86_64.whl", hash = "sha256:1b376b3f4896e7930f1f772ac4b064ac12598d1c38d04907e696cc4d794b43d3"}, @@ -1634,8 +1649,8 @@ yarl = [ {file = "yarl-1.8.1.tar.gz", hash = "sha256:af887845b8c2e060eb5605ff72b6f2dd2aab7a761379373fd89d314f4752abbf"}, ] zipp = [ - {file = "zipp-3.8.1-py3-none-any.whl", hash = "sha256:47c40d7fe183a6f21403a199b3e4192cca5774656965b0a4988ad2f8feb5f009"}, - {file = "zipp-3.8.1.tar.gz", hash = "sha256:05b45f1ee8f807d0cc928485ca40a07cb491cf092ff587c0df9cb1fd154848d2"}, + {file = "zipp-3.10.0-py3-none-any.whl", hash = "sha256:4fcb6f278987a6605757302a6e40e896257570d11c51628968ccb2a47e80c6c1"}, + {file = "zipp-3.10.0.tar.gz", hash = "sha256:7a7262fd930bd3e36c50b9a64897aec3fafff3dfdeec9623ae22b40e93f99bb8"}, ] zstandard = [ {file = "zstandard-0.19.0-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:a65e0119ad39e855427520f7829618f78eb2824aa05e63ff19b466080cd99210"}, diff --git a/pyproject.toml b/pyproject.toml index 1c0253bfea..3e2f95f7db 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -46,32 +46,32 @@ packages = [ [tool.poetry.dependencies] python = "^3.8" -mmh3 = "^3.0.0" -requests = "^2.28.1" -click = "^8.1.3" -rich = "^12.5.1" -pyyaml = "^6.0.0" +mmh3 = "3.0.0" +requests = "2.28.1" +click = "8.1.3" +rich = "12.6.0" +pyyaml = "6.0.0" -pydantic = "^1.10.2" -fsspec = "2022.8.2" +pydantic = "1.10.2" +fsspec = "2022.10.0" -zstandard = ">=0.18,<0.20" +zstandard = "0.19.0" -pyarrow = { version = ">=9,<11", optional = true } +pyarrow = { version = "10.0.0", optional = true } -python-snappy = { version = "^0.6.1", optional = true } +python-snappy = { version = "0.6.1", optional = true } -thrift = { version = "^0.16.0", optional = true } +thrift = { version = "0.16.0", optional = true } -s3fs = { version = "2022.8.2", optional = true } +s3fs = { version = "2022.10.0", optional = true } [tool.poetry.dev-dependencies] -pytest = "^7.2.0" -pytest-checkdocs = "^2.9.0" -pre-commit = "^2.0.0" -fastavro = "^1.6.1" +pytest = "7.2.0" +pytest-checkdocs = "2.9.0" +pre-commit = "2.20.0" +fastavro = "1.6.1" coverage = { version = "^6.5.0", extras = ["toml"] } -requests-mock = "^1.9.3" +requests-mock = "1.10.0" [tool.poetry.scripts] pyiceberg = "pyiceberg.cli.console:run" From 6cc62da00480792a482b4f02e52740af523c2005 Mon Sep 17 00:00:00 2001 From: Douglas Drinka Date: Fri, 4 Nov 2022 00:55:55 -0500 Subject: [PATCH 254/642] Python: Struct fields should be provided to Schema constructor (#6115) --- pyiceberg/expressions/visitors.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/pyiceberg/expressions/visitors.py b/pyiceberg/expressions/visitors.py index 4853490315..6bcc6bd2ef 100644 --- a/pyiceberg/expressions/visitors.py +++ b/pyiceberg/expressions/visitors.py @@ -613,6 +613,7 @@ def visit_or(self, left_result: bool, right_result: bool) -> bool: def manifest_evaluator( partition_spec: PartitionSpec, schema: Schema, partition_filter: BooleanExpression, case_sensitive: bool = True ) -> Callable[[ManifestFile], bool]: - partition_schema = Schema(*partition_spec.partition_type(schema)) + partition_type = partition_spec.partition_type(schema) + partition_schema = Schema(*partition_type.fields) evaluator = _ManifestEvalVisitor(partition_schema, partition_filter, case_sensitive) return evaluator.eval From 184224fc9bdab68510c79a8a160992e35e07a1b6 Mon Sep 17 00:00:00 2001 From: Fokko Driesprong Date: Fri, 4 Nov 2022 21:10:01 +0100 Subject: [PATCH 255/642] Python: Fix the caching (#6010) --- pyiceberg/io/pyarrow.py | 33 +++++++++++++++++++++------------ tests/io/test_pyarrow.py | 8 ++++---- 2 files changed, 25 insertions(+), 16 deletions(-) diff --git a/pyiceberg/io/pyarrow.py b/pyiceberg/io/pyarrow.py index e70cec6349..bcc6dc94bf 100644 --- a/pyiceberg/io/pyarrow.py +++ b/pyiceberg/io/pyarrow.py @@ -37,6 +37,7 @@ FileInfo, FileSystem, FileType, + LocalFileSystem, S3FileSystem, ) @@ -197,22 +198,27 @@ def to_input_file(self) -> "PyArrowFile": class PyArrowFileIO(FileIO): def __init__(self, properties: Properties = EMPTY_DICT): - self.get_fs_and_path: Callable = lru_cache(self._get_fs_and_path) + self.get_fs: Callable = lru_cache(self._get_fs) super().__init__(properties=properties) - def _get_fs_and_path(self, location: str) -> Tuple[FileSystem, str]: - uri = urlparse(location) # Create a ParseResult from the URI - if not uri.scheme: # If no scheme, assume the path is to a local file - return FileSystem.from_uri(os.path.abspath(location)) - elif uri.scheme in {"s3", "s3a", "s3n"}: + @staticmethod + def parse_location(location: str) -> Tuple[str, str]: + """Returns the path without the scheme""" + uri = urlparse(location) + return uri.scheme or "file", os.path.abspath(location) if not uri.scheme else f"{uri.netloc}{uri.path}" + + def _get_fs(self, scheme: str) -> FileSystem: + if scheme in {"s3", "s3a", "s3n"}: client_kwargs = { "endpoint_override": self.properties.get("s3.endpoint"), "access_key": self.properties.get("s3.access-key-id"), "secret_key": self.properties.get("s3.secret-access-key"), } - return (S3FileSystem(**client_kwargs), uri.netloc + uri.path) + return S3FileSystem(**client_kwargs) + elif scheme == "file": + return LocalFileSystem() else: - return FileSystem.from_uri(location) # Infer the proper filesystem + raise ValueError(f"Unrecognized filesystem type in URI: {scheme}") def new_input(self, location: str) -> PyArrowFile: """Get a PyArrowFile instance to read bytes from the file at the given location @@ -223,7 +229,8 @@ def new_input(self, location: str) -> PyArrowFile: Returns: PyArrowFile: A PyArrowFile instance for the given location """ - fs, path = self.get_fs_and_path(location) + scheme, path = self.parse_location(location) + fs = self._get_fs(scheme) return PyArrowFile(fs=fs, location=location, path=path) def new_output(self, location: str) -> PyArrowFile: @@ -235,7 +242,8 @@ def new_output(self, location: str) -> PyArrowFile: Returns: PyArrowFile: A PyArrowFile instance for the given location """ - fs, path = self.get_fs_and_path(location) + scheme, path = self.parse_location(location) + fs = self._get_fs(scheme) return PyArrowFile(fs=fs, location=location, path=path) def delete(self, location: Union[str, InputFile, OutputFile]) -> None: @@ -251,8 +259,9 @@ def delete(self, location: Union[str, InputFile, OutputFile]) -> None: PermissionError: If the file at the provided location cannot be accessed due to a permission error such as an AWS error code 15 """ - str_path = location.location if isinstance(location, (InputFile, OutputFile)) else location - fs, path = self.get_fs_and_path(str_path) + str_location = location.location if isinstance(location, (InputFile, OutputFile)) else location + scheme, path = self.parse_location(str_location) + fs = self._get_fs(scheme) try: fs.delete_file(path) diff --git a/tests/io/test_pyarrow.py b/tests/io/test_pyarrow.py index 5a08011c03..db25c0d134 100644 --- a/tests/io/test_pyarrow.py +++ b/tests/io/test_pyarrow.py @@ -266,8 +266,8 @@ def test_deleting_s3_file_no_permission(): s3fs_mock = MagicMock() s3fs_mock.delete_file.side_effect = OSError("AWS Error [code 15]") - with patch.object(PyArrowFileIO, "_get_fs_and_path") as submocked: - submocked.return_value = (s3fs_mock, "bar/foo.txt") + with patch.object(PyArrowFileIO, "_get_fs") as submocked: + submocked.return_value = s3fs_mock with pytest.raises(PermissionError) as exc_info: PyArrowFileIO().delete("s3://foo/bar.txt") @@ -281,8 +281,8 @@ def test_deleting_s3_file_not_found(): s3fs_mock = MagicMock() s3fs_mock.delete_file.side_effect = OSError("Path does not exist") - with patch.object(PyArrowFileIO, "_get_fs_and_path") as submocked: - submocked.return_value = (s3fs_mock, "bar/foo.txt") + with patch.object(PyArrowFileIO, "_get_fs") as submocked: + submocked.return_value = s3fs_mock with pytest.raises(FileNotFoundError) as exc_info: PyArrowFileIO().delete("s3://foo/bar.txt") From 67c6c0a6e1cf1850181023dc9a2e8e95d50cfaa2 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Sun, 6 Nov 2022 09:39:51 +0100 Subject: [PATCH 256/642] Build: Bump mkdocs from 1.4.1 to 1.4.2 in /python (#6130) --- mkdocs/requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mkdocs/requirements.txt b/mkdocs/requirements.txt index a71574e759..143c4cacfb 100644 --- a/mkdocs/requirements.txt +++ b/mkdocs/requirements.txt @@ -15,5 +15,5 @@ # specific language governing permissions and limitations # under the License. -mkdocs==1.4.1 +mkdocs==1.4.2 jinja2==3.1.2 From 643f022c6eafa46a1ff9fd529f570060fb0c6110 Mon Sep 17 00:00:00 2001 From: Fokko Driesprong Date: Sun, 6 Nov 2022 22:10:34 +0100 Subject: [PATCH 257/642] Python: Replace mmh3 with mmhash3 (#6076) The mmh3 project is abandoned, and not being maintained. Solves #5901 --- poetry.lock | 68 ++++++++++++++++++++++++++++---------------------- pyproject.toml | 2 +- 2 files changed, 39 insertions(+), 31 deletions(-) diff --git a/poetry.lock b/poetry.lock index 4f48e959d0..2cb2689f57 100644 --- a/poetry.lock +++ b/poetry.lock @@ -342,8 +342,8 @@ optional = true python-versions = ">=3.7" [[package]] -name = "mmh3" -version = "3.0.0" +name = "mmhash3" +version = "3.0.1" description = "Python wrapper for MurmurHash (MurmurHash3), a set of fast and robust hash functions." category = "main" optional = false @@ -771,7 +771,7 @@ snappy = ["python-snappy"] [metadata] lock-version = "1.1" python-versions = "^3.8" -content-hash = "18498bdad449ca86cb9f7151243a5e16c651dee0d3c399274d802eaff91c4451" +content-hash = "4f90d01e509c4918b7e4ce6da629f5e009165a499cd2f43c25f3ef73ef58af86" [metadata.files] aiobotocore = [ @@ -1153,33 +1153,41 @@ jmespath = [ {file = "jmespath-1.0.1-py3-none-any.whl", hash = "sha256:02e2e4cc71b5bcab88332eebf907519190dd9e6e82107fa7f83b1003a6252980"}, {file = "jmespath-1.0.1.tar.gz", hash = "sha256:90261b206d6defd58fdd5e85f478bf633a2901798906be2ad389150c5c60edbe"}, ] -mmh3 = [ - {file = "mmh3-3.0.0-cp36-cp36m-macosx_10_9_x86_64.whl", hash = "sha256:23912dde2ad4f701926948dd8e79a0e42b000f73962806f153931f52985e1e07"}, - {file = "mmh3-3.0.0-cp36-cp36m-manylinux1_x86_64.whl", hash = "sha256:07f1308a410dc406d6a3c282a685728d00a87f3ed684f012671b96d6cc6a41c3"}, - {file = "mmh3-3.0.0-cp36-cp36m-manylinux2010_x86_64.whl", hash = "sha256:167cbc2b5ae27f3bccd797a2e8a9e7561791bee4cc2885f2c140eedc5df000ef"}, - {file = "mmh3-3.0.0-cp36-cp36m-manylinux2014_aarch64.whl", hash = "sha256:8fb833c2942917eff54f984b067d93e5a3c54dbb00720323460cdfed9292835f"}, - {file = "mmh3-3.0.0-cp36-cp36m-win32.whl", hash = "sha256:b7d26d0243ed9a5b8bf7aa8c53697cb79dff1e1d207f42396b7a7cb2a62298b7"}, - {file = "mmh3-3.0.0-cp36-cp36m-win_amd64.whl", hash = "sha256:2b6c79fc314b34b911245b460a79b601fff39bb807521fb7ed7c15cacf0394ac"}, - {file = "mmh3-3.0.0-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:6d0b3e9def1fdfe4eadd35ee26bf72bd715ba97711f7101302d54c9d2e70ba27"}, - {file = "mmh3-3.0.0-cp37-cp37m-manylinux1_x86_64.whl", hash = "sha256:8803d28c17cf898f5f00c0433e8b13d51fa3bb4ebecf59872ba1eaa20d94128a"}, - {file = "mmh3-3.0.0-cp37-cp37m-manylinux2010_x86_64.whl", hash = "sha256:01e456edf9cc381298a590923aadd1c0bf9934d93433099a5001d656112437c2"}, - {file = "mmh3-3.0.0-cp37-cp37m-manylinux2014_aarch64.whl", hash = "sha256:ff69ddc2d46e3e42720840b6b4f7bfb032fd1e677fac347fdfff6e4d9fd01212"}, - {file = "mmh3-3.0.0-cp37-cp37m-win32.whl", hash = "sha256:e08a5d81a2ff53625953290187bed4ae96a6972e2b5cd5984a6ebc5a9aab256c"}, - {file = "mmh3-3.0.0-cp37-cp37m-win_amd64.whl", hash = "sha256:12484ac80373db77d8a6beb7615e7dac8b6c3fb118905311a51450b4fc4a24d1"}, - {file = "mmh3-3.0.0-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:93c96e657e9bf9e9ef12ddaeae9f109c0b3134146e2eff2cbddde5a34190920e"}, - {file = "mmh3-3.0.0-cp38-cp38-manylinux1_x86_64.whl", hash = "sha256:9097be65aa95460bc68b6108601da8894757532450daf74034e4eaecd536acca"}, - {file = "mmh3-3.0.0-cp38-cp38-manylinux2010_x86_64.whl", hash = "sha256:19874e12acb4119ef1ef83062ef4ac953c3343dd07a67ede8fa096d0393f34be"}, - {file = "mmh3-3.0.0-cp38-cp38-manylinux2014_aarch64.whl", hash = "sha256:4589adcb609d1547aac7c1ac1064eb27cdd44b65b7e8a114e2971cd3b7110306"}, - {file = "mmh3-3.0.0-cp38-cp38-win32.whl", hash = "sha256:7a311efd4ecf122f21392ec6bf447c620cc783d20bdb9aec60bb469a54318419"}, - {file = "mmh3-3.0.0-cp38-cp38-win_amd64.whl", hash = "sha256:3566d1455fa4a09f8fb1aa5b37f68914949674f9aa2bd630e9fdf344207f55b5"}, - {file = "mmh3-3.0.0-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:92fdffd63edb67c30dbaba18a7448d762209c0e678b0c9d577d17b30362b59a3"}, - {file = "mmh3-3.0.0-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:3e52b869572c09db0c1a483f6e9cedbccfae8a282d95e552d3d4bd0712ab3196"}, - {file = "mmh3-3.0.0-cp39-cp39-manylinux1_x86_64.whl", hash = "sha256:f1cce018cc82a8a6287e6aeb139e441129837b810f2ddf372e3ff7f0fefb0947"}, - {file = "mmh3-3.0.0-cp39-cp39-manylinux2010_x86_64.whl", hash = "sha256:0fd09c4b61fcddbcf0a87d5463b4e6d2919896736a67efc5248d5c74c1c9c742"}, - {file = "mmh3-3.0.0-cp39-cp39-manylinux2014_aarch64.whl", hash = "sha256:c17fe2e276edd37ad8a6aff3b1663d3479c2c5c5993539c1050422a1dae33033"}, - {file = "mmh3-3.0.0-cp39-cp39-win32.whl", hash = "sha256:150439b906b4deaf6d796b2c2d11fb6159f08d02330d97723071ab3bf43b51df"}, - {file = "mmh3-3.0.0-cp39-cp39-win_amd64.whl", hash = "sha256:bd870aedd9189eff1cf4e1687869b56c7e9461ee869789139c3e704009e5c227"}, - {file = "mmh3-3.0.0.tar.gz", hash = "sha256:d1ec578c09a07d3518ec9be540b87546397fa3455de73c166fcce51eaa5c41c5"}, +mmhash3 = [ + {file = "mmhash3-3.0.1-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:47deea30cd8d3d5cd52dc740902a4c70383bfe8248eac29d0877fe63e03c2713"}, + {file = "mmhash3-3.0.1-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:ecdaf4d1de617818bf05cd526ca558db6010beeba7ea9e19f695f2bdcac0e0a4"}, + {file = "mmhash3-3.0.1-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:4675585617584e9e1aafa3a90ac0ac9a437257c507748d97de8b21977e9d6745"}, + {file = "mmhash3-3.0.1-cp310-cp310-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:ebfd0c2af09b41f0fe1dd260799bda90a0fc7eba4477ccaeb3951527700cd58f"}, + {file = "mmhash3-3.0.1-cp310-cp310-win32.whl", hash = "sha256:68587dec7b8acdb7528fd511e295d8b5ccfe26022923a69867e1822f0fdb4c44"}, + {file = "mmhash3-3.0.1-cp310-cp310-win_amd64.whl", hash = "sha256:54954ebe3a614f19ab4cfe24fa66ea093c493d9fac0533d002dd64c2540a0c99"}, + {file = "mmhash3-3.0.1-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:b172f3bd3220b0bf56fd7cc760fd8a9033def47103611d63fe867003079a1256"}, + {file = "mmhash3-3.0.1-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:de7895eafabc32f7c0c09a81a624921f536768de6e438e3de69e3e954a4d7072"}, + {file = "mmhash3-3.0.1-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:b4b0914effe4ddd8d33149e3508564c17719465b0cc81691c4fa50d5e0e14f80"}, + {file = "mmhash3-3.0.1-cp311-cp311-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:c0575050ac691475938df1ad03d8738c5bd1eadef62093e76157ebb7f2df0946"}, + {file = "mmhash3-3.0.1-cp311-cp311-win32.whl", hash = "sha256:22f92f0f88f28b215357acd346362fa9f7c9fffb436beb42cc4b442b676dbaa3"}, + {file = "mmhash3-3.0.1-cp311-cp311-win_amd64.whl", hash = "sha256:538240ab7936bf71b18304e5a7e7fd3c4c2fab103330ea99584bb4f777299a2b"}, + {file = "mmhash3-3.0.1-cp36-cp36m-macosx_10_9_x86_64.whl", hash = "sha256:ca791bfb311e36ce13998e4632262ed4b95da9d3461941e18b6690760171a045"}, + {file = "mmhash3-3.0.1-cp36-cp36m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:b41708f72c6aa2a49ada1f0b61e85c05cd8389ad31d463fd5bca29999a4d5f9c"}, + {file = "mmhash3-3.0.1-cp36-cp36m-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:a3ce9b4533ddc0a88ba045a27309714c3b127bd05e57fd252d1d5a71d4247ea7"}, + {file = "mmhash3-3.0.1-cp36-cp36m-win32.whl", hash = "sha256:bfafeb96fdeb10db8767d06e1f07b6fdcddba4aaa0dd15058561a49f7ae45345"}, + {file = "mmhash3-3.0.1-cp36-cp36m-win_amd64.whl", hash = "sha256:97fe077b24c948709ed2afc749bf6285d407bc54ff12c63d2dc86678c38a0b8e"}, + {file = "mmhash3-3.0.1-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:0cfd91ccd5fca1ba7ee925297131a15dfb94c714bfe6ba0fb3b1ca78b12bbfec"}, + {file = "mmhash3-3.0.1-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:6d51b1005233141ce7394531af40a3f0fc1f274467bf8dff44dcf7987924fe58"}, + {file = "mmhash3-3.0.1-cp37-cp37m-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:855c67b100e37df166acf79cdff58fa8f9f6c48be0d1e1b6e9ad0fa34a9661ef"}, + {file = "mmhash3-3.0.1-cp37-cp37m-win32.whl", hash = "sha256:bb3030df1334fd665427f8be8e8ce4f04aeab7f6010ce4f2c128f0099bdab96f"}, + {file = "mmhash3-3.0.1-cp37-cp37m-win_amd64.whl", hash = "sha256:1545e1177294afe4912d5a5e401c7fa9b799dd109b30289e7af74d5b07e7c474"}, + {file = "mmhash3-3.0.1-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:2479899e7dda834a671991a1098a691ab1c2eaa20c3e939d691ca4a19361cfe0"}, + {file = "mmhash3-3.0.1-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:c9056196d5e3d3d844433a63d806a683f710ab3aaf1c910550c7746464bc43ae"}, + {file = "mmhash3-3.0.1-cp38-cp38-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:a0d4c307af0bf70207305f70f131898be071d1b19a89f462b13487f5c25e8d4e"}, + {file = "mmhash3-3.0.1-cp38-cp38-win32.whl", hash = "sha256:5f885f65e329fd14bc38debac4a79eacf381e856965d9c65c4d1c6946ea190d0"}, + {file = "mmhash3-3.0.1-cp38-cp38-win_amd64.whl", hash = "sha256:3b42d0bda5e1cd22c18b16887b0521060fb59d0aaaaf033feacbc0a2492d20fe"}, + {file = "mmhash3-3.0.1-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:d3f333286bb87aa9dc6bd8e7960a55a27b011a401f24b889a50e6d219f65e7c9"}, + {file = "mmhash3-3.0.1-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:6b7ef2eb95a18bcd02ce0d3e047adde3a025afd96c1d266a8a0d44574f44a307"}, + {file = "mmhash3-3.0.1-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:d6ac8a5f511c60f341bf9cae462bb4941abb149d98464ba5f4f4548875b601c6"}, + {file = "mmhash3-3.0.1-cp39-cp39-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:efef9e632e6e248e46f52d108a5ebd1f28edaf427b7fd47ebf97dbff4b2cab81"}, + {file = "mmhash3-3.0.1-cp39-cp39-win32.whl", hash = "sha256:bdac06d72e448c67afb12e758b203d875e29d4097bb279a38a5649d44b518ba7"}, + {file = "mmhash3-3.0.1-cp39-cp39-win_amd64.whl", hash = "sha256:0baeaa20cac5f75ed14f28056826bd9d9c8b2354b382073f3fd5190708992a0d"}, + {file = "mmhash3-3.0.1.tar.gz", hash = "sha256:a00d68f4a1cc434b9501513c8a29e18ed1ddad383677d72b41d71d0d862348af"}, ] multidict = [ {file = "multidict-6.0.2-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:0b9e95a740109c6047602f4db4da9949e6c5945cefbad34a1299775ddc9a62e2"}, diff --git a/pyproject.toml b/pyproject.toml index 3e2f95f7db..7ce563403d 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -46,7 +46,7 @@ packages = [ [tool.poetry.dependencies] python = "^3.8" -mmh3 = "3.0.0" +mmhash3 = "3.0.1" requests = "2.28.1" click = "8.1.3" rich = "12.6.0" From cdef0b55d9de0389602e9814288ad2b8f944154a Mon Sep 17 00:00:00 2001 From: Douglas Drinka Date: Sun, 6 Nov 2022 17:28:51 -0700 Subject: [PATCH 258/642] Python: Support creating a DateLiteral from a date (#6123) Closes #6120. --- pyiceberg/expressions/literals.py | 9 ++++++++- pyiceberg/utils/datetime.py | 7 ++++++- tests/expressions/test_literals.py | 12 ++++++++++++ tests/test_transforms.py | 11 +++++++---- 4 files changed, 33 insertions(+), 6 deletions(-) diff --git a/pyiceberg/expressions/literals.py b/pyiceberg/expressions/literals.py index 458cbda41d..ac4c0711a9 100644 --- a/pyiceberg/expressions/literals.py +++ b/pyiceberg/expressions/literals.py @@ -23,6 +23,7 @@ import struct from abc import ABC, abstractmethod +from datetime import date from decimal import ROUND_HALF_UP, Decimal from functools import singledispatch, singledispatchmethod from typing import ( @@ -50,6 +51,7 @@ UUIDType, ) from pyiceberg.utils.datetime import ( + date_str_to_days, date_to_days, micros_to_days, time_to_micros, @@ -164,6 +166,11 @@ def _(value: Decimal) -> Literal[Decimal]: return DecimalLiteral(value) +@literal.register(date) +def _(value: date) -> Literal[int]: + return DateLiteral(date_to_days(value)) + + class AboveMax(Singleton): @property def value(self): @@ -395,7 +402,7 @@ def _(self, type_var: StringType) -> Literal[str]: @to.register(DateType) def _(self, type_var: DateType) -> Optional[Literal[int]]: try: - return DateLiteral(date_to_days(self.value)) + return DateLiteral(date_str_to_days(self.value)) except (TypeError, ValueError): return None diff --git a/pyiceberg/utils/datetime.py b/pyiceberg/utils/datetime.py index 76a5e9d484..31a8e6193d 100644 --- a/pyiceberg/utils/datetime.py +++ b/pyiceberg/utils/datetime.py @@ -47,11 +47,16 @@ def micros_to_time(micros: int) -> time: return time(hour=hours, minute=minutes, second=seconds, microsecond=microseconds) -def date_to_days(date_str: str) -> int: +def date_str_to_days(date_str: str) -> int: """Converts an ISO-8601 formatted date to days from 1970-01-01""" return (date.fromisoformat(date_str) - EPOCH_DATE).days +def date_to_days(date_val: date) -> int: + """Converts a Python date object to days from 1970-01-01""" + return (date_val - EPOCH_DATE).days + + def days_to_date(days: int) -> date: """Creates a date from the number of days from 1970-01-01""" return EPOCH_DATE + timedelta(days) diff --git a/tests/expressions/test_literals.py b/tests/expressions/test_literals.py index 019c925bfc..f34c8ae683 100644 --- a/tests/expressions/test_literals.py +++ b/tests/expressions/test_literals.py @@ -339,6 +339,17 @@ def test_string_to_decimal_literal(): # MISC +def test_python_date_conversion(): + one_day_str = "2022-03-28" + one_day_date = datetime.date(2022, 3, 28) + + from_str_lit = literal(one_day_str).to(DateType()) + from_date_lit = literal(one_day_date) + + assert isinstance(from_date_lit, DateLiteral) + assert from_str_lit == from_date_lit + + @pytest.mark.parametrize( "lit, primitive_type", [ @@ -348,6 +359,7 @@ def test_string_to_decimal_literal(): (literal(34.11), FloatType()), (literal(3.5028235e38), DoubleType()), (literal(Decimal(34.55).quantize(Decimal("0.01"))), DecimalType(9, 2)), + (literal(datetime.date(2017, 8, 18)), DateType()), (literal("2017-08-18"), DateType()), (literal("14:21:01.919"), TimeType()), (literal("2017-08-18T14:21:01.919"), TimestampType()), diff --git a/tests/test_transforms.py b/tests/test_transforms.py index eec3e4aebe..3e4e63494a 100644 --- a/tests/test_transforms.py +++ b/tests/test_transforms.py @@ -15,6 +15,7 @@ # specific language governing permissions and limitations # under the License. # pylint: disable=eval-used,protected-access +from datetime import date from decimal import Decimal from uuid import UUID @@ -51,6 +52,7 @@ UUIDType, ) from pyiceberg.utils.datetime import ( + date_str_to_days, date_to_days, time_to_micros, timestamp_to_micros, @@ -65,7 +67,8 @@ (1, IntegerType(), 1392991556), (34, IntegerType(), 2017239379), (34, LongType(), 2017239379), - (date_to_days("2017-11-16"), DateType(), -653330422), + (date_to_days(date(2017, 11, 16)), DateType(), -653330422), + (date_str_to_days("2017-11-16"), DateType(), -653330422), (time_to_micros("22:31:08"), TimeType(), -662762989), ( timestamp_to_micros("2017-11-16T22:31:08"), @@ -144,15 +147,15 @@ def test_string_with_surrogate_pair(): @pytest.mark.parametrize( - "date,date_transform,expected", + "date_val,date_transform,expected", [ (47, YearTransform(), "2017"), (575, MonthTransform(), "2017-12"), (17501, DayTransform(), "2017-12-01"), ], ) -def test_date_to_human_string(date, date_transform, expected): - assert date_transform.to_human_string(DateType(), date) == expected +def test_date_to_human_string(date_val, date_transform, expected): + assert date_transform.to_human_string(DateType(), date_val) == expected @pytest.mark.parametrize( From fcc331e7307aaf21a84e783c8f1a328b5bc533e2 Mon Sep 17 00:00:00 2001 From: Douglas Drinka Date: Mon, 7 Nov 2022 02:16:01 -0700 Subject: [PATCH 259/642] Python: Fix typo in `_ManifestEvalVisitor.visit_equal` (#6117) * Fix typo in `_ManifestEvalVisitor.visit_equal` * Fix linting Co-authored-by: Fokko Driesprong --- pyiceberg/expressions/visitors.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pyiceberg/expressions/visitors.py b/pyiceberg/expressions/visitors.py index 6bcc6bd2ef..4c6df0b079 100644 --- a/pyiceberg/expressions/visitors.py +++ b/pyiceberg/expressions/visitors.py @@ -517,7 +517,7 @@ def visit_equal(self, term: BoundTerm, literal: Literal[Any]) -> bool: pos = term.ref().accessor.position field = self.partition_fields[pos] - if field.lower_bound is None: + if field.lower_bound is None or field.upper_bound is None: # values are all null and literal cannot contain null return ROWS_CANNOT_MATCH @@ -526,7 +526,7 @@ def visit_equal(self, term: BoundTerm, literal: Literal[Any]) -> bool: if lower > literal.value: return ROWS_CANNOT_MATCH - upper = _from_byte_buffer(term.ref().field.field_type, field.lower_bound) + upper = _from_byte_buffer(term.ref().field.field_type, field.upper_bound) if literal.value > upper: return ROWS_CANNOT_MATCH From c04286755dec93abe872bdf9933678b2f4e5038b Mon Sep 17 00:00:00 2001 From: Hendrik Makait Date: Mon, 7 Nov 2022 15:31:28 +0100 Subject: [PATCH 260/642] Python: Implement pythonic `__len__` (#6135) --- pyiceberg/avro/reader.py | 11 +++++++---- pyiceberg/expressions/literals.py | 4 ++-- pyiceberg/io/pyarrow.py | 2 +- pyiceberg/types.py | 11 +++++------ tests/test_types.py | 4 ++-- 5 files changed, 17 insertions(+), 15 deletions(-) diff --git a/pyiceberg/avro/reader.py b/pyiceberg/avro/reader.py index 6b93b67937..ef8b416d1c 100644 --- a/pyiceberg/avro/reader.py +++ b/pyiceberg/avro/reader.py @@ -216,13 +216,16 @@ def skip(self, decoder: BinaryDecoder) -> None: @dataclass(frozen=True) class FixedReader(Reader): - length: int = dataclassfield() + _len: int = dataclassfield() def read(self, decoder: BinaryDecoder) -> bytes: - return decoder.read(self.length) + return decoder.read(len(self)) def skip(self, decoder: BinaryDecoder) -> None: - decoder.skip(self.length) + decoder.skip(len(self)) + + def __len__(self): + return self._len class BinaryReader(Reader): @@ -364,7 +367,7 @@ def primitive_reader(primitive: PrimitiveType) -> Reader: @primitive_reader.register def _(primitive: FixedType) -> Reader: - return FixedReader(primitive.length) + return FixedReader(len(primitive)) @primitive_reader.register diff --git a/pyiceberg/expressions/literals.py b/pyiceberg/expressions/literals.py index ac4c0711a9..d38cd5f741 100644 --- a/pyiceberg/expressions/literals.py +++ b/pyiceberg/expressions/literals.py @@ -463,7 +463,7 @@ def to(self, type_var): @to.register(FixedType) def _(self, type_var: FixedType) -> Optional[Literal[bytes]]: - if len(self.value) == type_var.length: + if len(self.value) == len(type_var): return self else: return None @@ -487,7 +487,7 @@ def _(self, _: BinaryType) -> Literal[bytes]: @to.register(FixedType) def _(self, type_var: FixedType) -> Optional[Literal[bytes]]: - if type_var.length == len(self.value): + if len(type_var) == len(self.value): return FixedLiteral(self.value) else: return None diff --git a/pyiceberg/io/pyarrow.py b/pyiceberg/io/pyarrow.py index bcc6dc94bf..d2492a1da2 100644 --- a/pyiceberg/io/pyarrow.py +++ b/pyiceberg/io/pyarrow.py @@ -313,7 +313,7 @@ def _iceberg_to_pyarrow_type(primitive: PrimitiveType) -> pa.DataType: @_iceberg_to_pyarrow_type.register def _(primitive: FixedType) -> pa.DataType: - return pa.binary(primitive.length) + return pa.binary(len(primitive)) @_iceberg_to_pyarrow_type.register diff --git a/pyiceberg/types.py b/pyiceberg/types.py index 86a234a4e5..c9c8847435 100644 --- a/pyiceberg/types.py +++ b/pyiceberg/types.py @@ -126,7 +126,7 @@ class FixedType(PrimitiveType): """ __root__: str = Field() - _length: int = PrivateAttr() + _len: int = PrivateAttr() @staticmethod def parse(str_repr: str) -> "FixedType": @@ -134,14 +134,13 @@ def parse(str_repr: str) -> "FixedType": def __init__(self, length: int): super().__init__(__root__=f"fixed[{length}]") - self._length = length + self._len = length - @property - def length(self) -> int: - return self._length + def __len__(self) -> int: + return self._len def __repr__(self) -> str: - return f"FixedType(length={self._length})" + return f"FixedType(length={self._len})" class DecimalType(PrimitiveType): diff --git a/tests/test_types.py b/tests/test_types.py index 56ed291fad..3bbe1926d6 100644 --- a/tests/test_types.py +++ b/tests/test_types.py @@ -100,7 +100,7 @@ def test_is_primitive(input_type, result): def test_fixed_type(): type_var = FixedType(length=5) - assert type_var.length == 5 + assert len(type_var) == 5 assert str(type_var) == "fixed[5]" assert repr(type_var) == "FixedType(length=5)" assert str(type_var) == str(eval(repr(type_var))) @@ -401,7 +401,7 @@ def test_deserialization_fixed(): inner = fixed.__root__ assert isinstance(inner, FixedType) - assert inner.length == 22 + assert len(inner) == 22 def test_str_fixed(): From a6543f871414f0adac6d09c9030f40e80d1145bf Mon Sep 17 00:00:00 2001 From: Fokko Driesprong Date: Wed, 9 Nov 2022 22:15:06 +0100 Subject: [PATCH 261/642] Python: Fix Evaluator tests (#6140) This uses the same tests as the JVM implementation. --- tests/expressions/test_visitors.py | 1094 +++++++++++----------------- 1 file changed, 420 insertions(+), 674 deletions(-) diff --git a/tests/expressions/test_visitors.py b/tests/expressions/test_visitors.py index 078dd88747..fd91e8a219 100644 --- a/tests/expressions/test_visitors.py +++ b/tests/expressions/test_visitors.py @@ -14,6 +14,7 @@ # KIND, either express or implied. See the License for the # specific language governing permissions and limitations # under the License. +# pylint:disable=redefined-outer-name from typing import Any, List, Set @@ -44,13 +45,19 @@ BoundTerm, BoundUnaryPredicate, EqualTo, + GreaterThan, + GreaterThanOrEqual, In, + IsNaN, IsNull, + LessThan, LessThanOrEqual, LiteralPredicate, Not, NotEqualTo, NotIn, + NotNaN, + NotNull, Or, Reference, SetPredicate, @@ -64,7 +71,6 @@ literal, ) from pyiceberg.expressions.visitors import ( - IN_PREDICATE_LIMIT, BindVisitor, BooleanExpressionVisitor, BoundBooleanExpressionVisitor, @@ -80,7 +86,6 @@ FloatType, IcebergType, IntegerType, - LongType, NestedField, PrimitiveType, StringType, @@ -827,824 +832,565 @@ def _to_manifest_file(*partitions: PartitionFieldSummary) -> ManifestFile: ) -def _create_manifest_evaluator(bound_expr: BoundPredicate) -> _ManifestEvalVisitor: - """For testing. Creates a bogus evaluator, and then replaces the expression""" - evaluator = _ManifestEvalVisitor( - Schema(NestedField(1, "id", LongType())), EqualTo(term=Reference("id"), literal=literal("foo")) - ) - evaluator.partition_filter = bound_expr - return evaluator +INT_MIN_VALUE = 30 +INT_MAX_VALUE = 79 +INT_MIN = _to_byte_buffer(IntegerType(), INT_MIN_VALUE) +INT_MAX = _to_byte_buffer(IntegerType(), INT_MAX_VALUE) -def test_manifest_evaluator_less_than_no_overlap(): - expr = BoundLessThan( - term=BoundReference( - field=NestedField(field_id=1, name="foo", field_type=StringType(), required=False), - accessor=Accessor(position=0, inner=None), - ), - literal=StringLiteral("c"), - ) +STRING_MIN = _to_byte_buffer(StringType(), "a") +STRING_MAX = _to_byte_buffer(StringType(), "z") - manifest = _to_manifest_file( - PartitionFieldSummary( - contains_null=False, - contains_nan=False, - lower_bound=_to_byte_buffer(StringType(), "a"), - upper_bound=_to_byte_buffer(StringType(), "b"), - ) + +@pytest.fixture +def schema() -> Schema: + return Schema( + NestedField(1, "id", IntegerType(), required=True), + NestedField(2, "all_nulls_missing_nan", StringType(), required=False), + NestedField(3, "some_nulls", StringType(), required=False), + NestedField(4, "no_nulls", StringType(), required=False), + NestedField(5, "float", FloatType(), required=False), + NestedField(6, "all_nulls_double", DoubleType(), required=False), + NestedField(7, "all_nulls_no_nans", FloatType(), required=False), + NestedField(8, "all_nans", DoubleType(), required=False), + NestedField(9, "both_nan_and_null", FloatType(), required=False), + NestedField(10, "no_nan_or_null", DoubleType(), required=False), + NestedField(11, "all_nulls_missing_nan_float", FloatType(), required=False), + NestedField(12, "all_same_value_or_null", StringType(), required=False), + NestedField(13, "no_nulls_same_value_a", StringType(), required=False), ) - assert _create_manifest_evaluator(expr).eval(manifest) +@pytest.fixture +def manifest_no_stats() -> ManifestFile: + return _to_manifest_file() -def test_manifest_evaluator_less_than_overlap(): - expr = BoundLessThan( - term=BoundReference( - field=NestedField(field_id=1, name="foo", field_type=StringType(), required=False), - accessor=Accessor(position=0, inner=None), - ), - literal=StringLiteral("a"), - ) - manifest = _to_manifest_file( +@pytest.fixture +def manifest() -> ManifestFile: + return _to_manifest_file( + # id PartitionFieldSummary( contains_null=False, - contains_nan=False, - lower_bound=_to_byte_buffer(StringType(), "a"), - upper_bound=_to_byte_buffer(StringType(), "b"), - ) - ) - - assert not _create_manifest_evaluator(expr).eval(manifest) - - -def test_manifest_evaluator_less_than_all_null(): - expr = BoundLessThan( - term=BoundReference( - field=NestedField(field_id=1, name="foo", field_type=StringType(), required=False), - accessor=Accessor(position=0, inner=None), + contains_nan=None, + lower_bound=INT_MIN, + upper_bound=INT_MAX, ), - literal=StringLiteral("a"), - ) - - manifest = _to_manifest_file( - PartitionFieldSummary(contains_null=False, contains_nan=False, lower_bound=None, upper_bound=None) - ) - - # All null - assert not _create_manifest_evaluator(expr).eval(manifest) - - -def test_manifest_evaluator_less_than_no_match(): - expr = BoundLessThan( - term=BoundReference( - field=NestedField(field_id=1, name="foo", field_type=StringType(), required=False), - accessor=Accessor(position=0, inner=None), - ), - literal=StringLiteral("a"), - ) - - manifest = _to_manifest_file( + # all_nulls_missing_nan PartitionFieldSummary( - contains_null=False, - contains_nan=False, - lower_bound=_to_byte_buffer(StringType(), "a"), - upper_bound=_to_byte_buffer(StringType(), "b"), - ) - ) - - assert not _create_manifest_evaluator(expr).eval(manifest) - - -def test_manifest_evaluator_less_than_or_equal_no_overlap(): - expr = BoundLessThanOrEqual( - term=BoundReference( - field=NestedField(field_id=1, name="foo", field_type=StringType(), required=False), - accessor=Accessor(position=0, inner=None), + contains_null=True, + contains_nan=None, + lower_bound=None, + upper_bound=None, ), - literal=StringLiteral("c"), - ) - - manifest = _to_manifest_file( + # some_nulls PartitionFieldSummary( - contains_null=False, - contains_nan=False, - lower_bound=_to_byte_buffer(StringType(), "a"), - upper_bound=_to_byte_buffer(StringType(), "b"), - ) - ) - - assert _create_manifest_evaluator(expr).eval(manifest) - - -def test_manifest_evaluator_less_than_or_equal_overlap(): - expr = BoundLessThanOrEqual( - term=BoundReference( - field=NestedField(field_id=1, name="foo", field_type=StringType(), required=False), - accessor=Accessor(position=0, inner=None), + contains_null=True, + contains_nan=None, + lower_bound=STRING_MIN, + upper_bound=STRING_MAX, ), - literal=StringLiteral("a"), - ) - - manifest = _to_manifest_file( + # no_nulls PartitionFieldSummary( contains_null=False, - contains_nan=False, - lower_bound=_to_byte_buffer(StringType(), "a"), - upper_bound=_to_byte_buffer(StringType(), "b"), - ) - ) - - assert _create_manifest_evaluator(expr).eval(manifest) - - -def test_manifest_evaluator_less_than_or_equal_all_null(): - expr = BoundLessThanOrEqual( - term=BoundReference( - field=NestedField(field_id=1, name="foo", field_type=StringType(), required=False), - accessor=Accessor(position=0, inner=None), + contains_nan=None, + lower_bound=STRING_MIN, + upper_bound=STRING_MAX, ), - literal=StringLiteral("a"), - ) - - manifest = _to_manifest_file( - PartitionFieldSummary(contains_null=False, contains_nan=False, lower_bound=None, upper_bound=None) - ) - - # All null - assert not _create_manifest_evaluator(expr).eval(manifest) - - -def test_manifest_evaluator_less_than_or_equal_no_match(): - expr = BoundLessThanOrEqual( - term=BoundReference( - field=NestedField(field_id=1, name="foo", field_type=StringType(), required=False), - accessor=Accessor(position=0, inner=None), + # float + PartitionFieldSummary( + contains_null=True, + contains_nan=None, + lower_bound=_to_byte_buffer(FloatType(), 0.0), + upper_bound=_to_byte_buffer(FloatType(), 20.0), ), - literal=StringLiteral("a"), - ) - - manifest = _to_manifest_file( + # all_nulls_double + PartitionFieldSummary(contains_null=True, contains_nan=None, lower_bound=None, upper_bound=None), + # all_nulls_no_nans PartitionFieldSummary( - contains_null=False, + contains_null=True, contains_nan=False, - lower_bound=_to_byte_buffer(StringType(), "b"), - upper_bound=_to_byte_buffer(StringType(), "c"), - ) - ) - - assert not _create_manifest_evaluator(expr).eval(manifest) - - -def test_manifest_evaluator_equal_no_overlap(): - expr = BoundEqualTo( - term=BoundReference( - field=NestedField(field_id=1, name="foo", field_type=StringType(), required=False), - accessor=Accessor(position=0, inner=None), + lower_bound=None, + upper_bound=None, ), - literal=StringLiteral("c"), - ) - - manifest = _to_manifest_file( + # all_nans PartitionFieldSummary( contains_null=False, - contains_nan=False, - lower_bound=_to_byte_buffer(StringType(), "a"), - upper_bound=_to_byte_buffer(StringType(), "b"), - ) - ) - - assert not _create_manifest_evaluator(expr).eval(manifest) - - -def test_manifest_evaluator_equal_overlap(): - expr = BoundEqualTo( - term=BoundReference( - field=NestedField(field_id=1, name="foo", field_type=StringType(), required=False), - accessor=Accessor(position=0, inner=None), - ), - literal=StringLiteral("a"), - ) - - manifest = ManifestFile( - manifest_path="", - manifest_length=0, - partition_spec_id=0, - partitions=[ - PartitionFieldSummary( - contains_null=False, - contains_nan=False, - lower_bound=_to_byte_buffer(StringType(), "a"), - upper_bound=_to_byte_buffer(StringType(), "b"), - ) - ], - ) - - assert _create_manifest_evaluator(expr).eval(manifest) - - -def test_manifest_evaluator_equal_all_null(): - expr = BoundEqualTo( - term=BoundReference( - field=NestedField(field_id=1, name="foo", field_type=StringType(), required=False), - accessor=Accessor(position=0, inner=None), + contains_nan=True, + lower_bound=None, + upper_bound=None, ), - literal=StringLiteral("a"), - ) - - manifest = _to_manifest_file( - PartitionFieldSummary(contains_null=False, contains_nan=False, lower_bound=None, upper_bound=None) - ) - - # All null - assert not _create_manifest_evaluator(expr).eval(manifest) - - -def test_manifest_evaluator_equal_no_match(): - expr = BoundEqualTo( - term=BoundReference( - field=NestedField(field_id=1, name="foo", field_type=StringType(), required=False), - accessor=Accessor(position=0, inner=None), + # both_nan_and_null + PartitionFieldSummary( + contains_null=True, + contains_nan=True, + lower_bound=None, + upper_bound=None, ), - literal=StringLiteral("a"), - ) - - manifest = _to_manifest_file( + # no_nan_or_null PartitionFieldSummary( contains_null=False, contains_nan=False, - lower_bound=_to_byte_buffer(StringType(), "b"), - upper_bound=_to_byte_buffer(StringType(), "c"), - ) - ) - - assert not _create_manifest_evaluator(expr).eval(manifest) - - -def test_manifest_evaluator_greater_than_no_overlap(): - expr = BoundGreaterThan( - term=BoundReference( - field=NestedField(field_id=1, name="foo", field_type=StringType(), required=False), - accessor=Accessor(position=0, inner=None), + lower_bound=_to_byte_buffer(FloatType(), 0.0), + upper_bound=_to_byte_buffer(FloatType(), 20.0), ), - literal=StringLiteral("a"), - ) - - manifest = _to_manifest_file( + # all_nulls_missing_nan_float + PartitionFieldSummary(contains_null=True, contains_nan=None, lower_bound=None, upper_bound=None), + # all_same_value_or_null PartitionFieldSummary( - contains_null=False, - contains_nan=False, - lower_bound=_to_byte_buffer(StringType(), "b"), - upper_bound=_to_byte_buffer(StringType(), "c"), - ) - ) - - assert _create_manifest_evaluator(expr).eval(manifest) - - -def test_manifest_evaluator_greater_than_overlap(): - expr = BoundGreaterThan( - term=BoundReference( - field=NestedField(field_id=1, name="foo", field_type=StringType(), required=False), - accessor=Accessor(position=0, inner=None), + contains_null=True, + contains_nan=None, + lower_bound=STRING_MIN, + upper_bound=STRING_MIN, ), - literal=StringLiteral("c"), - ) - - manifest = _to_manifest_file( + # no_nulls_same_value_a PartitionFieldSummary( contains_null=False, - contains_nan=False, - lower_bound=_to_byte_buffer(StringType(), "b"), - upper_bound=_to_byte_buffer(StringType(), "c"), - ) + contains_nan=None, + lower_bound=STRING_MIN, + upper_bound=STRING_MIN, + ), ) - assert not _create_manifest_evaluator(expr).eval(manifest) +def test_all_nulls(schema: Schema, manifest: ManifestFile) -> None: + assert not _ManifestEvalVisitor(schema, NotNull(Reference("all_nulls_missing_nan"))).eval( + manifest + ), "Should skip: all nulls column with non-floating type contains all null" -def test_manifest_evaluator_greater_than_all_null(): - expr = BoundGreaterThan( - term=BoundReference( - field=NestedField(field_id=1, name="foo", field_type=StringType(), required=False), - accessor=Accessor(position=0, inner=None), - ), - literal=StringLiteral("a"), - ) + assert _ManifestEvalVisitor(schema, NotNull(Reference("all_nulls_missing_nan_float"))).eval( + manifest + ), "Should read: no NaN information may indicate presence of NaN value" - manifest = _to_manifest_file( - PartitionFieldSummary(contains_null=False, contains_nan=False, lower_bound=None, upper_bound=None) - ) + assert _ManifestEvalVisitor(schema, NotNull(Reference("some_nulls"))).eval( + manifest + ), "Should read: column with some nulls contains a non-null value" - # All null - assert not _create_manifest_evaluator(expr).eval(manifest) + assert _ManifestEvalVisitor(schema, NotNull(Reference("no_nulls"))).eval( + manifest + ), "Should read: non-null column contains a non-null value" -def test_manifest_evaluator_greater_than_no_match(): - expr = BoundGreaterThan( - term=BoundReference( - field=NestedField(field_id=1, name="foo", field_type=StringType(), required=False), - accessor=Accessor(position=0, inner=None), - ), - literal=StringLiteral("d"), - ) +def test_no_nulls(schema: Schema, manifest: ManifestFile) -> None: + assert _ManifestEvalVisitor(schema, IsNull(Reference("all_nulls_missing_nan"))).eval( + manifest + ), "Should read: at least one null value in all null column" - manifest = _to_manifest_file( - PartitionFieldSummary( - contains_null=False, - contains_nan=False, - lower_bound=_to_byte_buffer(StringType(), "b"), - upper_bound=_to_byte_buffer(StringType(), "c"), - ) - ) + assert _ManifestEvalVisitor(schema, IsNull(Reference("some_nulls"))).eval( + manifest + ), "Should read: column with some nulls contains a null value" - assert not _create_manifest_evaluator(expr).eval(manifest) + assert not _ManifestEvalVisitor(schema, IsNull(Reference("no_nulls"))).eval( + manifest + ), "Should skip: non-null column contains no null values" + assert _ManifestEvalVisitor(schema, IsNull(Reference("both_nan_and_null"))).eval( + manifest + ), "Should read: both_nan_and_null column contains no null values" -def test_manifest_evaluator_greater_than_or_equal_no_overlap(): - expr = BoundGreaterThanOrEqual( - term=BoundReference( - field=NestedField(field_id=1, name="foo", field_type=StringType(), required=False), - accessor=Accessor(position=0, inner=None), - ), - literal=StringLiteral("a"), - ) - manifest = _to_manifest_file( - PartitionFieldSummary( - contains_null=False, - contains_nan=False, - lower_bound=_to_byte_buffer(StringType(), "b"), - upper_bound=_to_byte_buffer(StringType(), "c"), - ) - ) +def test_is_nan(schema: Schema, manifest: ManifestFile) -> None: + assert _ManifestEvalVisitor(schema, IsNaN(Reference("float"))).eval( + manifest + ), "Should read: no information on if there are nan value in float column" - assert _create_manifest_evaluator(expr).eval(manifest) + assert _ManifestEvalVisitor(schema, IsNaN(Reference("all_nulls_double"))).eval( + manifest + ), "Should read: no NaN information may indicate presence of NaN value" + assert _ManifestEvalVisitor(schema, IsNaN(Reference("all_nulls_missing_nan_float"))).eval( + manifest + ), "Should read: no NaN information may indicate presence of NaN value" -def test_manifest_evaluator_greater_than_or_equal_overlap(): - expr = BoundGreaterThanOrEqual( - term=BoundReference( - field=NestedField(field_id=1, name="foo", field_type=StringType(), required=False), - accessor=Accessor(position=0, inner=None), - ), - literal=StringLiteral("b"), - ) + assert not _ManifestEvalVisitor(schema, IsNaN(Reference("all_nulls_no_nans"))).eval( + manifest + ), "Should skip: no nan column doesn't contain nan value" - manifest = _to_manifest_file( - PartitionFieldSummary( - contains_null=False, - contains_nan=False, - lower_bound=_to_byte_buffer(StringType(), "a"), - upper_bound=_to_byte_buffer(StringType(), "b"), - ) - ) + assert _ManifestEvalVisitor(schema, IsNaN(Reference("all_nans"))).eval( + manifest + ), "Should read: all_nans column contains nan value" - assert _create_manifest_evaluator(expr).eval(manifest) + assert _ManifestEvalVisitor(schema, IsNaN(Reference("both_nan_and_null"))).eval( + manifest + ), "Should read: both_nan_and_null column contains nan value" + assert not _ManifestEvalVisitor(schema, IsNaN(Reference("no_nan_or_null"))).eval( + manifest + ), "Should skip: no_nan_or_null column doesn't contain nan value" -def test_manifest_evaluator_greater_than_or_equal_all_null(): - expr = BoundGreaterThanOrEqual( - term=BoundReference( - field=NestedField(field_id=1, name="foo", field_type=StringType(), required=False), - accessor=Accessor(position=0, inner=None), - ), - literal=StringLiteral("a"), - ) - manifest = _to_manifest_file( - PartitionFieldSummary(contains_null=False, contains_nan=False, lower_bound=None, upper_bound=None) - ) +def test_not_nan(schema: Schema, manifest: ManifestFile) -> None: + assert _ManifestEvalVisitor(schema, NotNaN(Reference("float"))).eval( + manifest + ), "Should read: no information on if there are nan value in float column" - # All null - assert not _create_manifest_evaluator(expr).eval(manifest) + assert _ManifestEvalVisitor(schema, NotNaN(Reference("all_nulls_double"))).eval( + manifest + ), "Should read: all null column contains non nan value" + assert _ManifestEvalVisitor(schema, NotNaN(Reference("all_nulls_no_nans"))).eval( + manifest + ), "Should read: no_nans column contains non nan value" -def test_manifest_evaluator_greater_than_or_equal_no_match(): - expr = BoundGreaterThanOrEqual( - term=BoundReference( - field=NestedField(field_id=1, name="foo", field_type=StringType(), required=False), - accessor=Accessor(position=0, inner=None), - ), - literal=StringLiteral("d"), - ) + assert not _ManifestEvalVisitor(schema, NotNaN(Reference("all_nans"))).eval( + manifest + ), "Should skip: all nans column doesn't contain non nan value" - manifest = _to_manifest_file( - PartitionFieldSummary( - contains_null=False, - contains_nan=False, - lower_bound=_to_byte_buffer(StringType(), "b"), - upper_bound=_to_byte_buffer(StringType(), "c"), - ) - ) + assert _ManifestEvalVisitor(schema, NotNaN(Reference("both_nan_and_null"))).eval( + manifest + ), "Should read: both_nan_and_null nans column contains non nan value" - assert not _create_manifest_evaluator(expr).eval(manifest) + assert _ManifestEvalVisitor(schema, NotNaN(Reference("no_nan_or_null"))).eval( + manifest + ), "Should read: no_nan_or_null column contains non nan value" -def test_manifest_evaluator_is_nan(): - expr = BoundIsNaN( - term=BoundReference( - field=NestedField(field_id=1, name="foo", field_type=DoubleType(), required=False), - accessor=Accessor(position=0, inner=None), - ) - ) +def test_missing_stats(schema: Schema, manifest_no_stats: ManifestFile): + expressions: List[BooleanExpression] = [ + LessThan(Reference[int]("id"), LongLiteral(5)), + LessThanOrEqual(Reference[int]("id"), LongLiteral(30)), + EqualTo(Reference[int]("id"), LongLiteral(70)), + GreaterThan(Reference[int]("id"), LongLiteral(78)), + GreaterThanOrEqual(Reference[int]("id"), LongLiteral(90)), + NotEqualTo(Reference[int]("id"), LongLiteral(101)), + IsNull(Reference[int]("id")), + NotNull(Reference[int]("id")), + IsNaN(Reference[float]("float")), + NotNaN(Reference[float]("float")), + ] - manifest = _to_manifest_file( - PartitionFieldSummary(contains_null=False, contains_nan=True, lower_bound=None, upper_bound=None) - ) + for expr in expressions: + assert _ManifestEvalVisitor(schema, expr).eval(manifest_no_stats), f"Should read when missing stats for expr: {expr}" - assert _create_manifest_evaluator(expr).eval(manifest) +def test_not(schema: Schema, manifest: ManifestFile): + assert _ManifestEvalVisitor(schema, Not(LessThan(Reference[int]("id"), LongLiteral(INT_MIN_VALUE - 25)))).eval( + manifest + ), "Should read: not(false)" -def test_manifest_evaluator_is_nan_inverse(): - expr = BoundIsNaN( - term=BoundReference( - field=NestedField(field_id=1, name="foo", field_type=DoubleType(), required=False), - accessor=Accessor(position=0, inner=None), - ) - ) + assert not _ManifestEvalVisitor(schema, Not(GreaterThan(Reference[int]("id"), LongLiteral(INT_MIN_VALUE - 25)))).eval( + manifest + ), "Should skip: not(true)" - manifest = _to_manifest_file( - PartitionFieldSummary( - contains_null=True, - contains_nan=False, - lower_bound=_to_byte_buffer(DoubleType(), 18.15), - upper_bound=_to_byte_buffer(DoubleType(), 19.25), - ) - ) - assert not _create_manifest_evaluator(expr).eval(manifest) +def test_and(schema: Schema, manifest: ManifestFile): + assert not _ManifestEvalVisitor( + schema, + And( + LessThan(Reference[int]("id"), LongLiteral(INT_MIN_VALUE - 25)), + GreaterThanOrEqual(Reference[int]("id"), LongLiteral(INT_MIN_VALUE - 30)), + ), + ).eval(manifest), "Should skip: and(false, true)" + assert not _ManifestEvalVisitor( + schema, + And( + LessThan(Reference[int]("id"), LongLiteral(INT_MIN_VALUE - 25)), + GreaterThanOrEqual(Reference[int]("id"), LongLiteral(INT_MAX_VALUE + 1)), + ), + ).eval(manifest), "Should skip: and(false, false)" -def test_manifest_evaluator_not_nan(): - expr = BoundNotNaN( - term=BoundReference( - field=NestedField(field_id=1, name="foo", field_type=DoubleType(), required=False), - accessor=Accessor(position=0, inner=None), - ) - ) + assert _ManifestEvalVisitor( + schema, + And( + GreaterThan(Reference[int]("id"), LongLiteral(INT_MIN_VALUE - 25)), + LessThanOrEqual(Reference[int]("id"), LongLiteral(INT_MIN_VALUE)), + ), + ).eval(manifest), "Should read: and(true, true)" - manifest = _to_manifest_file( - PartitionFieldSummary( - contains_null=False, - contains_nan=False, - lower_bound=_to_byte_buffer(DoubleType(), 18.15), - upper_bound=_to_byte_buffer(DoubleType(), 19.25), - ) - ) - assert _create_manifest_evaluator(expr).eval(manifest) +def test_or(schema: Schema, manifest: ManifestFile): + assert not _ManifestEvalVisitor( + schema, + Or( + LessThan(Reference[int]("id"), LongLiteral(INT_MIN_VALUE - 25)), + GreaterThanOrEqual(Reference[int]("id"), LongLiteral(INT_MAX_VALUE + 1)), + ), + ).eval(manifest), "Should skip: or(false, false)" + assert _ManifestEvalVisitor( + schema, + Or( + LessThan(Reference[int]("id"), LongLiteral(INT_MIN_VALUE - 25)), + GreaterThanOrEqual(Reference[int]("id"), LongLiteral(INT_MAX_VALUE - 19)), + ), + ).eval(manifest), "Should read: or(false, true)" -def test_manifest_evaluator_not_nan_inverse(): - expr = BoundNotNaN( - term=BoundReference( - field=NestedField(field_id=1, name="foo", field_type=DoubleType(), required=False), - accessor=Accessor(position=0, inner=None), - ) - ) - manifest = _to_manifest_file( - PartitionFieldSummary(contains_null=False, contains_nan=True, lower_bound=None, upper_bound=None) - ) +def test_integer_lt(schema: Schema, manifest: ManifestFile): + assert not _ManifestEvalVisitor(schema, LessThan(Reference[int]("id"), LongLiteral(INT_MIN_VALUE - 25))).eval( + manifest + ), "Should not read: id range below lower bound (5 < 30)" - assert not _create_manifest_evaluator(expr).eval(manifest) + assert not _ManifestEvalVisitor(schema, LessThan(Reference[int]("id"), LongLiteral(INT_MIN_VALUE))).eval( + manifest + ), "Should not read: id range below lower bound (30 is not < 30)" + assert _ManifestEvalVisitor(schema, LessThan(Reference[int]("id"), LongLiteral(INT_MIN_VALUE + 1))).eval( + manifest + ), "Should read: one possible id" -def test_manifest_evaluator_is_null(): - expr = BoundIsNull( - term=BoundReference( - field=NestedField(field_id=1, name="foo", field_type=StringType(), required=False), - accessor=Accessor(position=0, inner=None), - ), - ) + assert _ManifestEvalVisitor(schema, LessThan(Reference[int]("id"), LongLiteral(INT_MAX_VALUE))).eval( + manifest + ), "Should read: may possible ids" - manifest = _to_manifest_file( - PartitionFieldSummary( - contains_null=True, - contains_nan=False, - lower_bound=_to_byte_buffer(StringType(), "a"), - upper_bound=_to_byte_buffer(StringType(), "b"), - ) - ) - assert _create_manifest_evaluator(expr).eval(manifest) +def test_integer_lt_eq(schema: Schema, manifest: ManifestFile): + assert not _ManifestEvalVisitor(schema, LessThanOrEqual(Reference[int]("id"), LongLiteral(INT_MIN_VALUE - 25))).eval( + manifest + ), "Should not read: id range below lower bound (5 < 30)" + assert not _ManifestEvalVisitor(schema, LessThanOrEqual(Reference[int]("id"), LongLiteral(INT_MIN_VALUE - 1))).eval( + manifest + ), "Should not read: id range below lower bound (29 < 30)" -def test_manifest_evaluator_is_null_inverse(): - expr = BoundIsNull( - term=BoundReference( - field=NestedField(field_id=1, name="foo", field_type=StringType(), required=False), - accessor=Accessor(position=0, inner=None), - ), - ) + assert _ManifestEvalVisitor(schema, LessThanOrEqual(Reference[int]("id"), LongLiteral(INT_MIN_VALUE))).eval( + manifest + ), "Should read: one possible id" - manifest = _to_manifest_file( - PartitionFieldSummary(contains_null=False, contains_nan=True, lower_bound=None, upper_bound=None) - ) + assert _ManifestEvalVisitor(schema, LessThanOrEqual(Reference[int]("id"), LongLiteral(INT_MAX_VALUE))).eval( + manifest + ), "Should read: many possible ids" - assert not _create_manifest_evaluator(expr).eval(manifest) +def test_integer_gt(schema: Schema, manifest: ManifestFile): + assert not _ManifestEvalVisitor(schema, GreaterThan(Reference[int]("id"), LongLiteral(INT_MAX_VALUE + 6))).eval( + manifest + ), "Should not read: id range above upper bound (85 < 79)" -def test_manifest_evaluator_not_null(): - expr = BoundNotNull( - term=BoundReference( - field=NestedField(field_id=1, name="foo", field_type=StringType(), required=False), - accessor=Accessor(position=0, inner=None), - ), - ) + assert not _ManifestEvalVisitor(schema, GreaterThan(Reference[int]("id"), LongLiteral(INT_MAX_VALUE))).eval( + manifest + ), "Should not read: id range above upper bound (79 is not > 79)" - manifest = _to_manifest_file( - PartitionFieldSummary(contains_null=False, contains_nan=False, lower_bound=None, upper_bound=None) - ) + assert _ManifestEvalVisitor(schema, GreaterThan(Reference[int]("id"), LongLiteral(INT_MAX_VALUE - 1))).eval( + manifest + ), "Should read: one possible id" - assert _create_manifest_evaluator(expr).eval(manifest) + assert _ManifestEvalVisitor(schema, GreaterThan(Reference[int]("id"), LongLiteral(INT_MAX_VALUE - 4))).eval( + manifest + ), "Should read: may possible ids" -def test_manifest_evaluator_not_null_nan(): - expr = BoundNotNull( - term=BoundReference( - field=NestedField(field_id=1, name="foo", field_type=DoubleType(), required=False), - accessor=Accessor(position=0, inner=None), - ), - ) +def test_integer_gt_eq(schema: Schema, manifest: ManifestFile): + assert not _ManifestEvalVisitor(schema, GreaterThanOrEqual(Reference[int]("id"), LongLiteral(INT_MAX_VALUE + 6))).eval( + manifest + ), "Should not read: id range above upper bound (85 < 79)" - manifest = _to_manifest_file( - PartitionFieldSummary(contains_null=True, contains_nan=False, lower_bound=None, upper_bound=None) - ) + assert not _ManifestEvalVisitor(schema, GreaterThanOrEqual(Reference[int]("id"), LongLiteral(INT_MAX_VALUE + 1))).eval( + manifest + ), "Should not read: id range above upper bound (80 > 79)" - assert not _create_manifest_evaluator(expr).eval(manifest) + assert _ManifestEvalVisitor(schema, GreaterThanOrEqual(Reference[int]("id"), LongLiteral(INT_MAX_VALUE))).eval( + manifest + ), "Should read: one possible id" + assert _ManifestEvalVisitor(schema, GreaterThanOrEqual(Reference[int]("id"), LongLiteral(INT_MAX_VALUE))).eval( + manifest + ), "Should read: may possible ids" -def test_manifest_evaluator_not_null_inverse(): - expr = BoundNotNull( - term=BoundReference( - field=NestedField(field_id=1, name="foo", field_type=StringType(), required=False), - accessor=Accessor(position=0, inner=None), - ), - ) - manifest = _to_manifest_file(PartitionFieldSummary(contains_null=True, contains_nan=True, lower_bound=None, upper_bound=None)) +def test_integer_eq(schema: Schema, manifest: ManifestFile): + assert not _ManifestEvalVisitor(schema, EqualTo(Reference[int]("id"), LongLiteral(INT_MIN_VALUE - 25))).eval( + manifest + ), "Should not read: id below lower bound" - assert not _create_manifest_evaluator(expr).eval(manifest) + assert not _ManifestEvalVisitor(schema, EqualTo(Reference[int]("id"), LongLiteral(INT_MIN_VALUE - 1))).eval( + manifest + ), "Should not read: id below lower bound" + assert _ManifestEvalVisitor(schema, EqualTo(Reference[int]("id"), LongLiteral(INT_MIN_VALUE))).eval( + manifest + ), "Should read: id equal to lower bound" -def test_manifest_evaluator_not_equal_to(): - expr = BoundNotEqualTo( - term=BoundReference( - field=NestedField(field_id=1, name="foo", field_type=StringType(), required=False), - accessor=Accessor(position=0, inner=None), - ), - literal=literal("this"), - ) + assert _ManifestEvalVisitor(schema, EqualTo(Reference[int]("id"), LongLiteral(INT_MAX_VALUE - 4))).eval( + manifest + ), "Should read: id between lower and upper bounds" - manifest = _to_manifest_file(PartitionFieldSummary(contains_null=True, contains_nan=True, lower_bound=None, upper_bound=None)) + assert _ManifestEvalVisitor(schema, EqualTo(Reference[int]("id"), LongLiteral(INT_MAX_VALUE))).eval( + manifest + ), "Should read: id equal to upper bound" - assert _create_manifest_evaluator(expr).eval(manifest) + assert not _ManifestEvalVisitor(schema, EqualTo(Reference[int]("id"), LongLiteral(INT_MAX_VALUE + 1))).eval( + manifest + ), "Should not read: id above upper bound" + assert not _ManifestEvalVisitor(schema, EqualTo(Reference[int]("id"), LongLiteral(INT_MAX_VALUE + 6))).eval( + manifest + ), "Should not read: id above upper bound" -def test_manifest_evaluator_in(): - expr = BoundIn( - term=BoundReference( - field=NestedField(field_id=1, name="foo", field_type=LongType(), required=False), - accessor=Accessor(position=0, inner=None), - ), - literals={LongLiteral(i) for i in range(22)}, - ) - manifest = _to_manifest_file( - PartitionFieldSummary( - contains_null=True, - contains_nan=True, - lower_bound=_to_byte_buffer(LongType(), 5), - upper_bound=_to_byte_buffer(LongType(), 10), - ) - ) +def test_integer_not_eq(schema: Schema, manifest: ManifestFile): + assert _ManifestEvalVisitor(schema, NotEqualTo(Reference[int]("id"), LongLiteral(INT_MIN_VALUE - 25))).eval( + manifest + ), "Should read: id below lower bound" - assert _create_manifest_evaluator(expr).eval(manifest) + assert _ManifestEvalVisitor(schema, NotEqualTo(Reference[int]("id"), LongLiteral(INT_MIN_VALUE - 1))).eval( + manifest + ), "Should read: id below lower bound" + assert _ManifestEvalVisitor(schema, NotEqualTo(Reference[int]("id"), LongLiteral(INT_MIN_VALUE))).eval( + manifest + ), "Should read: id equal to lower bound" -def test_manifest_evaluator_not_in(): - expr = BoundNotIn( - term=BoundReference( - field=NestedField(field_id=1, name="foo", field_type=LongType(), required=False), - accessor=Accessor(position=0, inner=None), - ), - literals={LongLiteral(i) for i in range(22)}, - ) + assert _ManifestEvalVisitor(schema, NotEqualTo(Reference[int]("id"), LongLiteral(INT_MAX_VALUE - 4))).eval( + manifest + ), "Should read: id between lower and upper bounds" - manifest = _to_manifest_file( - PartitionFieldSummary( - contains_null=True, - contains_nan=True, - lower_bound=False, - upper_bound=False, - ) - ) + assert _ManifestEvalVisitor(schema, NotEqualTo(Reference[int]("id"), LongLiteral(INT_MAX_VALUE))).eval( + manifest + ), "Should read: id equal to upper bound" - assert _create_manifest_evaluator(expr).eval(manifest) + assert _ManifestEvalVisitor(schema, NotEqualTo(Reference[int]("id"), LongLiteral(INT_MAX_VALUE + 1))).eval( + manifest + ), "Should read: id above upper bound" + assert _ManifestEvalVisitor(schema, NotEqualTo(Reference[int]("id"), LongLiteral(INT_MAX_VALUE + 6))).eval( + manifest + ), "Should read: id above upper bound" -def test_manifest_evaluator_in_null(): - expr = BoundIn( - term=BoundReference( - field=NestedField(field_id=1, name="foo", field_type=LongType(), required=False), - accessor=Accessor(position=0, inner=None), - ), - literals={LongLiteral(i) for i in range(22)}, - ) - manifest = _to_manifest_file( - PartitionFieldSummary( - contains_null=True, - contains_nan=True, - lower_bound=None, - upper_bound=_to_byte_buffer(LongType(), 10), - ) - ) +def test_integer_not_eq_rewritten(schema: Schema, manifest: ManifestFile): + assert _ManifestEvalVisitor(schema, Not(EqualTo(Reference[int]("id"), LongLiteral(INT_MIN_VALUE - 25)))).eval( + manifest + ), "Should read: id below lower bound" - assert not _create_manifest_evaluator(expr).eval(manifest) + assert _ManifestEvalVisitor(schema, Not(EqualTo(Reference[int]("id"), LongLiteral(INT_MIN_VALUE - 1)))).eval( + manifest + ), "Should read: id below lower bound" + assert _ManifestEvalVisitor(schema, Not(EqualTo(Reference[int]("id"), LongLiteral(INT_MIN_VALUE)))).eval( + manifest + ), "Should read: id equal to lower bound" -def test_manifest_evaluator_in_inverse(): - expr = BoundIn( - term=BoundReference( - field=NestedField(field_id=1, name="foo", field_type=LongType(), required=False), - accessor=Accessor(position=0, inner=None), - ), - literals={LongLiteral(i) for i in range(22)}, - ) + assert _ManifestEvalVisitor(schema, Not(EqualTo(Reference[int]("id"), LongLiteral(INT_MAX_VALUE - 4)))).eval( + manifest + ), "Should read: id between lower and upper bounds" - manifest = _to_manifest_file( - PartitionFieldSummary( - contains_null=True, - contains_nan=True, - lower_bound=_to_byte_buffer(LongType(), 1815), - upper_bound=_to_byte_buffer(LongType(), 1925), - ) - ) + assert _ManifestEvalVisitor(schema, Not(EqualTo(Reference[int]("id"), LongLiteral(INT_MAX_VALUE)))).eval( + manifest + ), "Should read: id equal to upper bound" - assert not _create_manifest_evaluator(expr).eval(manifest) + assert _ManifestEvalVisitor(schema, Not(EqualTo(Reference[int]("id"), LongLiteral(INT_MAX_VALUE + 1)))).eval( + manifest + ), "Should read: id above upper bound" + assert _ManifestEvalVisitor(schema, Not(EqualTo(Reference[int]("id"), LongLiteral(INT_MAX_VALUE + 6)))).eval( + manifest + ), "Should read: id above upper bound" -def test_manifest_evaluator_in_overflow(): - expr = BoundIn( - term=BoundReference( - field=NestedField(field_id=1, name="foo", field_type=LongType(), required=False), - accessor=Accessor(position=0, inner=None), - ), - literals={LongLiteral(i) for i in range(IN_PREDICATE_LIMIT + 1)}, - ) - manifest = _to_manifest_file( - PartitionFieldSummary( - contains_null=True, - contains_nan=True, - lower_bound=_to_byte_buffer(LongType(), 1815), - upper_bound=_to_byte_buffer(LongType(), 1925), - ) - ) +def test_integer_not_eq_rewritten_case_insensitive(schema: Schema, manifest: ManifestFile): + assert _ManifestEvalVisitor( + schema, Not(EqualTo(Reference[int]("ID"), LongLiteral(INT_MIN_VALUE - 25))), case_sensitive=False + ).eval(manifest), "Should read: id below lower bound" - assert _create_manifest_evaluator(expr).eval(manifest) + assert _ManifestEvalVisitor( + schema, Not(EqualTo(Reference[int]("ID"), LongLiteral(INT_MIN_VALUE - 1))), case_sensitive=False + ).eval(manifest), "Should read: id below lower bound" + assert _ManifestEvalVisitor( + schema, Not(EqualTo(Reference[int]("ID"), LongLiteral(INT_MIN_VALUE))), case_sensitive=False + ).eval(manifest), "Should read: id equal to lower bound" -def test_manifest_evaluator_less_than_lower_bound(): - expr = BoundIn( - term=BoundReference( - field=NestedField(field_id=1, name="foo", field_type=LongType(), required=False), - accessor=Accessor(position=0, inner=None), - ), - literals={LongLiteral(i) for i in range(2)}, - ) + assert _ManifestEvalVisitor( + schema, Not(EqualTo(Reference[int]("ID"), LongLiteral(INT_MAX_VALUE - 4))), case_sensitive=False + ).eval(manifest), "Should read: id between lower and upper bounds" - manifest = _to_manifest_file( - PartitionFieldSummary( - contains_null=True, - contains_nan=True, - lower_bound=_to_byte_buffer(LongType(), 5), - upper_bound=_to_byte_buffer(LongType(), 10), - ) - ) + assert _ManifestEvalVisitor( + schema, Not(EqualTo(Reference[int]("ID"), LongLiteral(INT_MAX_VALUE))), case_sensitive=False + ).eval(manifest), "Should read: id equal to upper bound" - assert not _create_manifest_evaluator(expr).eval(manifest) + assert _ManifestEvalVisitor( + schema, Not(EqualTo(Reference[int]("ID"), LongLiteral(INT_MAX_VALUE + 1))), case_sensitive=False + ).eval(manifest), "Should read: id above upper bound" + assert _ManifestEvalVisitor( + schema, Not(EqualTo(Reference[int]("ID"), LongLiteral(INT_MAX_VALUE + 6))), case_sensitive=False + ).eval(manifest), "Should read: id above upper bound" -def test_manifest_evaluator_greater_than_upper_bound(): - expr = BoundIn( - term=BoundReference( - field=NestedField(field_id=1, name="foo", field_type=LongType(), required=False), - accessor=Accessor(position=0, inner=None), - ), - literals={LongLiteral(i) for i in range(20, 22)}, - ) - manifest = _to_manifest_file( - PartitionFieldSummary( - contains_null=True, - contains_nan=True, - lower_bound=_to_byte_buffer(LongType(), 5), - upper_bound=_to_byte_buffer(LongType(), 10), - ) - ) +def test_integer_in(schema: Schema, manifest: ManifestFile): + assert not _ManifestEvalVisitor( + schema, In(Reference[int]("id"), (LongLiteral(INT_MIN_VALUE - 25), LongLiteral(INT_MIN_VALUE - 24))) + ).eval(manifest), "Should not read: id below lower bound (5 < 30, 6 < 30)" - assert not _create_manifest_evaluator(expr).eval(manifest) + assert not _ManifestEvalVisitor( + schema, In(Reference[int]("id"), (LongLiteral(INT_MIN_VALUE - 2), LongLiteral(INT_MIN_VALUE - 1))) + ).eval(manifest), "Should not read: id below lower bound (28 < 30, 29 < 30)" + assert _ManifestEvalVisitor( + schema, In(Reference[int]("id"), (LongLiteral(INT_MIN_VALUE - 1), LongLiteral(INT_MIN_VALUE))) + ).eval(manifest), "Should read: id equal to lower bound (30 == 30)" -def test_manifest_evaluator_true(): - expr = AlwaysTrue() + assert _ManifestEvalVisitor( + schema, In(Reference[int]("id"), (LongLiteral(INT_MAX_VALUE - 4), LongLiteral(INT_MAX_VALUE - 3))) + ).eval(manifest), "Should read: id between lower and upper bounds (30 < 75 < 79, 30 < 76 < 79)" - manifest = _to_manifest_file(PartitionFieldSummary(contains_null=True, contains_nan=True, lower_bound=None, upper_bound=None)) + assert _ManifestEvalVisitor( + schema, In(Reference[int]("id"), (LongLiteral(INT_MAX_VALUE), LongLiteral(INT_MAX_VALUE + 1))) + ).eval(manifest), "Should read: id equal to upper bound (79 == 79)" - assert _create_manifest_evaluator(expr).eval(manifest) + assert not _ManifestEvalVisitor( + schema, In(Reference[int]("id"), (LongLiteral(INT_MAX_VALUE + 1), LongLiteral(INT_MAX_VALUE + 2))) + ).eval(manifest), "Should not read: id above upper bound (80 > 79, 81 > 79)" + assert not _ManifestEvalVisitor( + schema, In(Reference[int]("id"), (LongLiteral(INT_MAX_VALUE + 6), LongLiteral(INT_MAX_VALUE + 7))) + ).eval(manifest), "Should not read: id above upper bound (85 > 79, 86 > 79)" -def test_manifest_evaluator_false(): - expr = AlwaysFalse() + assert not _ManifestEvalVisitor( + schema, In(Reference[str]("all_nulls_missing_nan"), (StringLiteral("abc"), StringLiteral("def"))) + ).eval(manifest), "Should skip: in on all nulls column" - manifest = _to_manifest_file(PartitionFieldSummary(contains_null=True, contains_nan=True, lower_bound=None, upper_bound=None)) + assert _ManifestEvalVisitor(schema, In(Reference[str]("some_nulls"), (StringLiteral("abc"), StringLiteral("def")))).eval( + manifest + ), "Should read: in on some nulls column" - assert not _create_manifest_evaluator(expr).eval(manifest) + assert _ManifestEvalVisitor(schema, In(Reference[str]("no_nulls"), (StringLiteral("abc"), StringLiteral("def")))).eval( + manifest + ), "Should read: in on no nulls column" -def test_manifest_evaluator_not(): - expr = Not( - BoundIn( - term=BoundReference( - field=NestedField(field_id=1, name="foo", field_type=LongType(), required=False), - accessor=Accessor(position=0, inner=None), - ), - literals={LongLiteral(i) for i in range(22)}, - ) - ) +def test_integer_not_in(schema: Schema, manifest: ManifestFile): + assert _ManifestEvalVisitor( + schema, NotIn(Reference[int]("id"), (LongLiteral(INT_MIN_VALUE - 25), LongLiteral(INT_MIN_VALUE - 24))) + ).eval(manifest), "Should read: id below lower bound (5 < 30, 6 < 30)" - manifest = _to_manifest_file( - PartitionFieldSummary( - contains_null=True, - contains_nan=True, - lower_bound=_to_byte_buffer(LongType(), 5), - upper_bound=_to_byte_buffer(LongType(), 10), - ) - ) - assert not _create_manifest_evaluator(expr).eval(manifest) + assert _ManifestEvalVisitor( + schema, NotIn(Reference[int]("id"), (LongLiteral(INT_MIN_VALUE - 2), LongLiteral(INT_MIN_VALUE - 1))) + ).eval(manifest), "Should read: id below lower bound (28 < 30, 29 < 30)" + assert _ManifestEvalVisitor( + schema, NotIn(Reference[int]("id"), (LongLiteral(INT_MIN_VALUE - 1), LongLiteral(INT_MIN_VALUE))) + ).eval(manifest), "Should read: id equal to lower bound (30 == 30)" -def test_manifest_evaluator_and(): - expr = And( - BoundIn( - term=BoundReference( - field=NestedField(field_id=1, name="foo", field_type=LongType(), required=False), - accessor=Accessor(position=0, inner=None), - ), - literals={LongLiteral(i) for i in range(22)}, - ), - BoundIn( - term=BoundReference( - field=NestedField(field_id=1, name="foo", field_type=LongType(), required=False), - accessor=Accessor(position=0, inner=None), - ), - literals={LongLiteral(i) for i in range(22)}, - ), - ) + assert _ManifestEvalVisitor( + schema, NotIn(Reference[int]("id"), (LongLiteral(INT_MAX_VALUE - 4), LongLiteral(INT_MAX_VALUE - 3))) + ).eval(manifest), "Should read: id between lower and upper bounds (30 < 75 < 79, 30 < 76 < 79)" - manifest = _to_manifest_file( - PartitionFieldSummary( - contains_null=True, - contains_nan=True, - lower_bound=_to_byte_buffer(LongType(), 5), - upper_bound=_to_byte_buffer(LongType(), 10), - ) - ) - assert _create_manifest_evaluator(expr).eval(manifest) + assert _ManifestEvalVisitor( + schema, NotIn(Reference[int]("id"), (LongLiteral(INT_MAX_VALUE), LongLiteral(INT_MAX_VALUE + 1))) + ).eval(manifest), "Should read: id equal to upper bound (79 == 79)" + assert _ManifestEvalVisitor( + schema, NotIn(Reference[int]("id"), (LongLiteral(INT_MAX_VALUE + 1), LongLiteral(INT_MAX_VALUE + 2))) + ).eval(manifest), "Should read: id above upper bound (80 > 79, 81 > 79)" -def test_manifest_evaluator_or(): - expr = Or( - BoundIn( - term=BoundReference( - field=NestedField(field_id=1, name="foo", field_type=LongType(), required=False), - accessor=Accessor(position=0, inner=None), - ), - literals={LongLiteral(i) for i in range(22)}, - ), - BoundIn( - term=BoundReference( - field=NestedField(field_id=1, name="foo", field_type=LongType(), required=False), - accessor=Accessor(position=0, inner=None), - ), - literals={LongLiteral(i) for i in range(22)}, - ), - ) + assert _ManifestEvalVisitor( + schema, NotIn(Reference[int]("id"), (LongLiteral(INT_MAX_VALUE + 6), LongLiteral(INT_MAX_VALUE + 7))) + ).eval(manifest), "Should read: id above upper bound (85 > 79, 86 > 79)" - manifest = _to_manifest_file( - PartitionFieldSummary( - contains_null=True, - contains_nan=True, - lower_bound=_to_byte_buffer(LongType(), 5), - upper_bound=_to_byte_buffer(LongType(), 10), - ) - ) + assert _ManifestEvalVisitor( + schema, NotIn(Reference[str]("all_nulls_missing_nan"), (StringLiteral("abc"), StringLiteral("def"))) + ).eval(manifest), "Should read: notIn on no nulls column" + + assert _ManifestEvalVisitor(schema, NotIn(Reference[str]("some_nulls"), (StringLiteral("abc"), StringLiteral("def")))).eval( + manifest + ), "Should read: in on some nulls column" - assert _create_manifest_evaluator(expr).eval(manifest) + assert _ManifestEvalVisitor(schema, NotIn(Reference[str]("no_nulls"), (StringLiteral("abc"), StringLiteral("def")))).eval( + manifest + ), "Should read: in on no nulls column" def test_bound_predicate_invert(): From 178327d075f4006bea5613f15dd10c157c740f06 Mon Sep 17 00:00:00 2001 From: Fokko Driesprong Date: Thu, 10 Nov 2022 01:45:52 +0100 Subject: [PATCH 262/642] Python: Use FileIO from Table and forward S3 credentials (#6161) --- pyiceberg/catalog/hive.py | 9 +++++-- pyiceberg/catalog/rest.py | 4 ++++ pyiceberg/cli/console.py | 4 +--- pyiceberg/cli/output.py | 20 +++++++++++----- pyiceberg/io/fsspec.py | 1 + pyiceberg/io/pyarrow.py | 1 + pyiceberg/table/__init__.py | 47 +++++++++++++++++++++++++++++++------ tests/cli/test_console.py | 7 ++++-- 8 files changed, 73 insertions(+), 20 deletions(-) diff --git a/pyiceberg/catalog/hive.py b/pyiceberg/catalog/hive.py index 1edba2d76e..d002d165ac 100644 --- a/pyiceberg/catalog/hive.py +++ b/pyiceberg/catalog/hive.py @@ -255,7 +255,12 @@ def _convert_hive_into_iceberg(self, table: HiveTable, io: FileIO) -> Table: file = io.new_input(metadata_location) metadata = FromInputFile.table_metadata(file) - return Table(identifier=(table.dbName, table.tableName), metadata=metadata, metadata_location=metadata_location) + return Table( + identifier=(table.dbName, table.tableName), + metadata=metadata, + metadata_location=metadata_location, + catalog_properties=self.properties, + ) def _write_metadata(self, metadata: TableMetadata, io: FileIO, metadata_path: str): ToOutputFile.table_metadata(metadata, io.new_output(metadata_path)) @@ -403,7 +408,7 @@ def rename_table(self, from_identifier: Union[str, Identifier], to_identifier: U raise NoSuchTableError(f"Table does not exist: {from_table_name}") from e except InvalidOperationException as e: raise NoSuchNamespaceError(f"Database does not exists: {to_database_name}") from e - return Table() + return self.load_table(to_identifier) def create_namespace(self, namespace: Union[str, Identifier], properties: Properties = EMPTY_DICT) -> None: """Create a namespace in the catalog. diff --git a/pyiceberg/catalog/rest.py b/pyiceberg/catalog/rest.py index 8822172101..2ac057f5c8 100644 --- a/pyiceberg/catalog/rest.py +++ b/pyiceberg/catalog/rest.py @@ -368,6 +368,8 @@ def create_table( identifier=(self.name,) + self.identifier_to_tuple(identifier), metadata_location=table_response.metadata_location, metadata=table_response.metadata, + catalog_properties=self.properties, + config=table_response.config, ) def list_tables(self, namespace: Union[str, Identifier]) -> List[Identifier]: @@ -397,6 +399,8 @@ def load_table(self, identifier: Union[str, Identifier]) -> Table: identifier=(self.name,) + identifier_tuple if self.name else identifier_tuple, metadata_location=table_response.metadata_location, metadata=table_response.metadata, + catalog_properties=self.properties, + config=table_response.config, ) def drop_table(self, identifier: Union[str, Identifier], purge_requested: bool = False) -> None: diff --git a/pyiceberg/cli/console.py b/pyiceberg/cli/console.py index e783a4312f..f1e7802953 100644 --- a/pyiceberg/cli/console.py +++ b/pyiceberg/cli/console.py @@ -24,7 +24,6 @@ from pyiceberg.catalog import Catalog, load_catalog from pyiceberg.cli.output import ConsoleOutput, JsonOutput, Output from pyiceberg.exceptions import NoSuchNamespaceError, NoSuchPropertyException, NoSuchTableError -from pyiceberg.io import load_file_io def catch_exception(): @@ -142,8 +141,7 @@ def files(ctx: Context, identifier: str, history: bool): catalog, output = _catalog_and_output(ctx) catalog_table = catalog.load_table(identifier) - io = load_file_io({**catalog.properties, **catalog_table.metadata.properties}) - output.files(catalog_table, io, history) + output.files(catalog_table, history) @run.command() diff --git a/pyiceberg/cli/output.py b/pyiceberg/cli/output.py index c7907c7e5a..2f83298cfc 100644 --- a/pyiceberg/cli/output.py +++ b/pyiceberg/cli/output.py @@ -23,11 +23,11 @@ from rich.table import Table as RichTable from rich.tree import Tree -from pyiceberg.io import FileIO from pyiceberg.schema import Schema -from pyiceberg.table import Table +from pyiceberg.table import Table, TableMetadata from pyiceberg.table.partitioning import PartitionSpec from pyiceberg.typedef import Identifier, Properties +from pyiceberg.utils.iceberg_base_model import IcebergBaseModel class Output(ABC): @@ -46,7 +46,7 @@ def describe_table(self, table: Table) -> None: ... @abstractmethod - def files(self, table: Table, io: FileIO, history: bool) -> None: + def files(self, table: Table, history: bool) -> None: ... @abstractmethod @@ -124,7 +124,7 @@ def describe_table(self, table: Table) -> None: output_table.add_row("Properties", table_properties) Console().print(output_table) - def files(self, table: Table, io: FileIO, history: bool) -> None: + def files(self, table: Table, history: bool) -> None: if history: snapshots = table.metadata.snapshots else: @@ -134,6 +134,7 @@ def files(self, table: Table, io: FileIO, history: bool) -> None: snapshots = [] snapshot_tree = Tree(f"Snapshots: {'.'.join(table.identifier)}") + io = table.io for snapshot in snapshots: manifest_list_str = f": {snapshot.manifest_list}" if snapshot.manifest_list else "" @@ -186,7 +187,14 @@ def identifiers(self, identifiers: List[Identifier]) -> None: self._out([".".join(identifier) for identifier in identifiers]) def describe_table(self, table: Table) -> None: - print(table.json()) + class FauxTable(IcebergBaseModel): + """Just to encode it using Pydantic""" + + identifier: Identifier + metadata_location: str + metadata: TableMetadata + + print(FauxTable(identifier=table.identifier, metadata=table.metadata, metadata_location=table.metadata_location).json()) def describe_properties(self, properties: Properties) -> None: self._out(properties) @@ -197,7 +205,7 @@ def text(self, response: str) -> None: def schema(self, schema: Schema) -> None: print(schema.json()) - def files(self, table: Table, io: FileIO, history: bool) -> None: + def files(self, table: Table, history: bool) -> None: pass def spec(self, spec: PartitionSpec) -> None: diff --git a/pyiceberg/io/fsspec.py b/pyiceberg/io/fsspec.py index 641187721a..42ebee5d4a 100644 --- a/pyiceberg/io/fsspec.py +++ b/pyiceberg/io/fsspec.py @@ -71,6 +71,7 @@ def _s3(properties: Properties) -> AbstractFileSystem: "endpoint_url": properties.get("s3.endpoint"), "aws_access_key_id": properties.get("s3.access-key-id"), "aws_secret_access_key": properties.get("s3.secret-access-key"), + "aws_session_token": properties.get("s3.session-token"), } config_kwargs = {} register_events: Dict[str, Callable] = {} diff --git a/pyiceberg/io/pyarrow.py b/pyiceberg/io/pyarrow.py index d2492a1da2..7be778f1ff 100644 --- a/pyiceberg/io/pyarrow.py +++ b/pyiceberg/io/pyarrow.py @@ -213,6 +213,7 @@ def _get_fs(self, scheme: str) -> FileSystem: "endpoint_override": self.properties.get("s3.endpoint"), "access_key": self.properties.get("s3.access-key-id"), "secret_key": self.properties.get("s3.secret-access-key"), + "session_token": self.properties.get("s3.session-token"), } return S3FileSystem(**client_kwargs) elif scheme == "file": diff --git a/pyiceberg/table/__init__.py b/pyiceberg/table/__init__.py index 73a2bcc9b5..7ec46dabba 100644 --- a/pyiceberg/table/__init__.py +++ b/pyiceberg/table/__init__.py @@ -14,25 +14,45 @@ # KIND, either express or implied. See the License for the # specific language governing permissions and limitations # under the License. - - -from typing import Dict, List, Optional +from functools import cached_property +from typing import ( + Any, + Dict, + List, + Optional, +) from pydantic import Field +from pyiceberg.io import FileIO, load_file_io from pyiceberg.schema import Schema from pyiceberg.table.metadata import TableMetadata from pyiceberg.table.partitioning import PartitionSpec from pyiceberg.table.snapshots import Snapshot, SnapshotLogEntry from pyiceberg.table.sorting import SortOrder -from pyiceberg.typedef import Identifier -from pyiceberg.utils.iceberg_base_model import IcebergBaseModel +from pyiceberg.typedef import EMPTY_DICT, Identifier, Properties -class Table(IcebergBaseModel): +class Table: identifier: Identifier = Field() - metadata_location: str = Field() metadata: TableMetadata = Field() + metadata_location: str = Field() + catalog_properties: Properties + config: Properties + + def __init__( + self, + identifier: Identifier, + metadata: TableMetadata, + metadata_location: str, + catalog_properties: Properties = EMPTY_DICT, + config: Properties = EMPTY_DICT, + ): + self.identifier = identifier + self.metadata = metadata + self.metadata_location = metadata_location + self.catalog_properties = catalog_properties + self.config = config def refresh(self): """Refresh the current table metadata""" @@ -90,3 +110,16 @@ def snapshot_by_name(self, name: str) -> Optional[Snapshot]: def history(self) -> List[SnapshotLogEntry]: """Get the snapshot history of this table.""" return self.metadata.snapshot_log + + @cached_property + def io(self) -> FileIO: + return load_file_io({**self.catalog_properties, **self.metadata.properties, **self.config}) + + def __eq__(self, other: Any) -> bool: + return ( + self.identifier == other.identifier + and self.metadata == other.metadata + and self.metadata_location == other.metadata_location + if isinstance(other, Table) + else False + ) diff --git a/tests/cli/test_console.py b/tests/cli/test_console.py index 38382cd1bb..ebb9ab37d8 100644 --- a/tests/cli/test_console.py +++ b/tests/cli/test_console.py @@ -47,7 +47,11 @@ def create_table( sort_order: SortOrder = UNSORTED_SORT_ORDER, properties: Properties = EMPTY_DICT, ) -> Table: - return Table(identifier=identifier, metadata_location="s3://tmp/", metadata=TableMetadataV2(**EXAMPLE_TABLE_METADATA_V2)) + return Table( + identifier=Catalog.identifier_to_tuple(identifier), + metadata_location="s3://tmp/", + metadata=TableMetadataV2(**EXAMPLE_TABLE_METADATA_V2), + ) def load_table(self, identifier: Union[str, Identifier]) -> Table: tuple_identifier = Catalog.identifier_to_tuple(identifier) @@ -838,6 +842,5 @@ def test_json_properties_remove_table_property_does_not_exists(_): def test_json_properties_remove_table_does_not_exist(_): runner = CliRunner() result = runner.invoke(run, ["--output=json", "properties", "remove", "table", "default.doesnotexist", "location"]) - print(result) assert result.exit_code == 1 assert result.output == """{"type": "NoSuchTableError", "message": "Table does not exist: default.doesnotexist"}\n""" From 0d437ebf157484d090ae9895c035774d47d14b23 Mon Sep 17 00:00:00 2001 From: Fokko Driesprong Date: Thu, 10 Nov 2022 18:14:37 +0100 Subject: [PATCH 263/642] Python: Add initial TableScan implementation (#6145) --- pyiceberg/table/__init__.py | 103 ++++++++++++++++++++++++++++++++++++ tests/table/test_init.py | 83 +++++++++++++++++++++++++++++ 2 files changed, 186 insertions(+) diff --git a/pyiceberg/table/__init__.py b/pyiceberg/table/__init__.py index 7ec46dabba..44bed4c357 100644 --- a/pyiceberg/table/__init__.py +++ b/pyiceberg/table/__init__.py @@ -14,16 +14,20 @@ # KIND, either express or implied. See the License for the # specific language governing permissions and limitations # under the License. +from __future__ import annotations + from functools import cached_property from typing import ( Any, Dict, List, Optional, + Tuple, ) from pydantic import Field +from pyiceberg.expressions import AlwaysTrue, And, BooleanExpression from pyiceberg.io import FileIO, load_file_io from pyiceberg.schema import Schema from pyiceberg.table.metadata import TableMetadata @@ -58,6 +62,29 @@ def refresh(self): """Refresh the current table metadata""" raise NotImplementedError("To be implemented") + def name(self) -> Identifier: + """Return the identifier of this table""" + return self.identifier + + def scan( + self, + row_filter: Optional[BooleanExpression] = None, + partition_filter: Optional[BooleanExpression] = None, + selected_fields: Tuple[str] = ("*",), + case_sensitive: bool = True, + snapshot_id: Optional[int] = None, + options: Properties = EMPTY_DICT, + ) -> TableScan: + return TableScan( + table=self, + row_filter=row_filter or AlwaysTrue(), + partition_filter=partition_filter or AlwaysTrue(), + selected_fields=selected_fields, + case_sensitive=case_sensitive, + snapshot_id=snapshot_id, + options=options, + ) + def schema(self) -> Schema: """Return the schema for this table""" return next(schema for schema in self.metadata.schemas if schema.schema_id == self.metadata.current_schema_id) @@ -123,3 +150,79 @@ def __eq__(self, other: Any) -> bool: if isinstance(other, Table) else False ) + + +class TableScan: + table: Table + row_filter: BooleanExpression + partition_filter: BooleanExpression + selected_fields: Tuple[str] + case_sensitive: bool + snapshot_id: Optional[int] + options: Properties + + def __init__( + self, + table: Table, + row_filter: Optional[BooleanExpression] = None, + partition_filter: Optional[BooleanExpression] = None, + selected_fields: Tuple[str] = ("*",), + case_sensitive: bool = True, + snapshot_id: Optional[int] = None, + options: Properties = EMPTY_DICT, + ): + self.table = table + self.row_filter = row_filter or AlwaysTrue() + self.partition_filter = partition_filter or AlwaysTrue() + self.selected_fields = selected_fields + self.case_sensitive = case_sensitive + self.snapshot_id = snapshot_id + self.options = options + + def snapshot(self) -> Optional[Snapshot]: + if self.snapshot_id: + return self.table.snapshot_by_id(self.snapshot_id) + return self.table.current_snapshot() + + def projection(self) -> Schema: + snapshot_schema = self.table.schema() + if snapshot := self.snapshot(): + if snapshot_schema_id := snapshot.schema_id: + snapshot_schema = self.table.schemas()[snapshot_schema_id] + + if "*" in self.selected_fields: + return snapshot_schema + + return snapshot_schema.select(*self.selected_fields, case_sensitive=self.case_sensitive) + + def plan_files(self): + raise NotImplementedError("Not yet implemented") + + def to_arrow(self): + raise NotImplementedError("Not yet implemented") + + def update(self, **overrides) -> TableScan: + """Creates a copy of this table scan with updated fields.""" + return TableScan(**{**self.__dict__, **overrides}) + + def use_ref(self, name: str): + if self.snapshot_id: + raise ValueError(f"Cannot override ref, already set snapshot id={self.snapshot_id}") + if snapshot := self.table.snapshot_by_name(name): + return self.update(snapshot_id=snapshot.snapshot_id) + + raise ValueError(f"Cannot scan unknown ref={name}") + + def select(self, *field_names: str) -> TableScan: + if "*" in self.selected_fields: + return self.update(selected_fields=field_names) + return self.update(selected_fields=tuple(set(self.selected_fields).intersection(set(field_names)))) + + def filter_rows(self, new_row_filter: BooleanExpression) -> TableScan: + return self.update(row_filter=And(self.row_filter, new_row_filter)) + + def filter_partitions(self, new_partition_filter: BooleanExpression) -> TableScan: + return self.update(partition_filter=And(self.partition_filter, new_partition_filter)) + + def with_case_sensitive(self, case_sensitive: bool = True) -> TableScan: + return self.update(case_sensitive=case_sensitive) diff --git a/tests/table/test_init.py b/tests/table/test_init.py index d0117c4df1..e42c0f2d81 100644 --- a/tests/table/test_init.py +++ b/tests/table/test_init.py @@ -19,6 +19,12 @@ import pytest +from pyiceberg.expressions import ( + AlwaysTrue, + And, + EqualTo, + In, +) from pyiceberg.schema import Schema from pyiceberg.table import PartitionSpec, Table from pyiceberg.table.metadata import TableMetadataV2 @@ -179,3 +185,80 @@ def test_history(table): SnapshotLogEntry(snapshot_id="3051729675574597004", timestamp_ms=1515100955770), SnapshotLogEntry(snapshot_id="3055729675574597004", timestamp_ms=1555100955770), ] + + +def test_table_scan_select(table: Table): + scan = table.scan() + assert scan.selected_fields == ("*",) + assert scan.select("a", "b").selected_fields == ("a", "b") + assert scan.select("a", "c").select("a").selected_fields == ("a",) + + +def test_table_scan_row_filter(table: Table): + scan = table.scan() + assert scan.row_filter == AlwaysTrue() + assert scan.filter_rows(EqualTo("x", 10)).row_filter == EqualTo("x", 10) # type: ignore + assert scan.filter_rows(EqualTo("x", 10)).filter_rows(In("y", (10, 11))).row_filter == And( # type: ignore + EqualTo("x", 10), In("y", (10, 11)) # type: ignore + ) + + +def test_table_scan_partition_filter(table: Table): + scan = table.scan() + assert scan.row_filter == AlwaysTrue() + assert scan.filter_partitions(EqualTo("x", 10)).partition_filter == EqualTo("x", 10) # type: ignore + assert scan.filter_partitions(EqualTo("x", 10)).filter_partitions(In("y", (10, 11))).partition_filter == And( # type: ignore + EqualTo("x", 10), In("y", (10, 11)) # type: ignore + ) + + +def test_table_scan_ref(table: Table): + scan = table.scan() + assert scan.use_ref("test").snapshot_id == 3051729675574597004 + + +def test_table_scan_ref_does_not_exists(table: Table): + scan = table.scan() + + with pytest.raises(ValueError) as exc_info: + _ = scan.use_ref("boom") + + assert "Cannot scan unknown ref=boom" in str(exc_info.value) + + +def test_table_scan_projection_full_schema(table: Table): + scan = table.scan() + assert scan.select("x", "y", "z").projection() == Schema( + NestedField(field_id=1, name="x", field_type=LongType(), required=True), + NestedField(field_id=2, name="y", field_type=LongType(), required=True, doc="comment"), + NestedField(field_id=3, name="z", field_type=LongType(), required=True), + schema_id=1, + identifier_field_ids=[1, 2], + ) + + +def test_table_scan_projection_single_column(table: Table): + scan = table.scan() + assert scan.select("y").projection() == Schema( + NestedField(field_id=2, name="y", field_type=LongType(), required=True, doc="comment"), + schema_id=1, + identifier_field_ids=[2], + ) + + +def test_table_scan_projection_single_column_case_sensitive(table: Table): + scan = table.scan() + assert scan.with_case_sensitive(False).select("Y").projection() == Schema( + NestedField(field_id=2, name="y", field_type=LongType(), required=True, doc="comment"), + schema_id=1, + identifier_field_ids=[2], + ) + + +def test_table_scan_projection_unknown_column(table: Table): + scan = table.scan() + + with pytest.raises(ValueError) as exc_info: + _ = scan.select("a").projection() + + assert "Could not find column: 'a'" in str(exc_info.value) From b19eb6e415584fc8a9994077f47411db992bef50 Mon Sep 17 00:00:00 2001 From: Fokko Driesprong Date: Thu, 10 Nov 2022 19:32:22 +0100 Subject: [PATCH 264/642] Python: Move FileIO initialization to the catalog (#6170) --- pyiceberg/catalog/__init__.py | 4 ++++ pyiceberg/catalog/hive.py | 2 +- pyiceberg/catalog/rest.py | 6 ++---- pyiceberg/io/__init__.py | 2 +- pyiceberg/table/__init__.py | 22 ++++------------------ tests/catalog/test_base.py | 7 ++++++- tests/catalog/test_rest.py | 4 +++- tests/cli/test_console.py | 8 +++++++- tests/table/test_init.py | 2 ++ 9 files changed, 30 insertions(+), 27 deletions(-) diff --git a/pyiceberg/catalog/__init__.py b/pyiceberg/catalog/__init__.py index f7c8499aff..7b6c5f229e 100644 --- a/pyiceberg/catalog/__init__.py +++ b/pyiceberg/catalog/__init__.py @@ -29,6 +29,7 @@ ) from pyiceberg.exceptions import NotInstalledError +from pyiceberg.io import FileIO, load_file_io from pyiceberg.schema import Schema from pyiceberg.table import Table from pyiceberg.table.partitioning import UNPARTITIONED_PARTITION_SPEC, PartitionSpec @@ -162,6 +163,9 @@ def __init__(self, name: str, **properties: str): self.name = name self.properties = properties + def _load_file_io(self, properties: Properties = EMPTY_DICT) -> FileIO: + return load_file_io({**self.properties, **properties}) + @abstractmethod def create_table( self, diff --git a/pyiceberg/catalog/hive.py b/pyiceberg/catalog/hive.py index d002d165ac..779b135d53 100644 --- a/pyiceberg/catalog/hive.py +++ b/pyiceberg/catalog/hive.py @@ -259,7 +259,7 @@ def _convert_hive_into_iceberg(self, table: HiveTable, io: FileIO) -> Table: identifier=(table.dbName, table.tableName), metadata=metadata, metadata_location=metadata_location, - catalog_properties=self.properties, + io=self._load_file_io(metadata.properties), ) def _write_metadata(self, metadata: TableMetadata, io: FileIO, metadata_path: str): diff --git a/pyiceberg/catalog/rest.py b/pyiceberg/catalog/rest.py index 2ac057f5c8..46c2e04315 100644 --- a/pyiceberg/catalog/rest.py +++ b/pyiceberg/catalog/rest.py @@ -368,8 +368,7 @@ def create_table( identifier=(self.name,) + self.identifier_to_tuple(identifier), metadata_location=table_response.metadata_location, metadata=table_response.metadata, - catalog_properties=self.properties, - config=table_response.config, + io=self._load_file_io({**table_response.metadata.properties, **table_response.config}), ) def list_tables(self, namespace: Union[str, Identifier]) -> List[Identifier]: @@ -399,8 +398,7 @@ def load_table(self, identifier: Union[str, Identifier]) -> Table: identifier=(self.name,) + identifier_tuple if self.name else identifier_tuple, metadata_location=table_response.metadata_location, metadata=table_response.metadata, - catalog_properties=self.properties, - config=table_response.config, + io=self._load_file_io({**table_response.metadata.properties, **table_response.config}), ) def drop_table(self, identifier: Union[str, Identifier], purge_requested: bool = False) -> None: diff --git a/pyiceberg/io/__init__.py b/pyiceberg/io/__init__.py index 12a1fd0005..6d734316ef 100644 --- a/pyiceberg/io/__init__.py +++ b/pyiceberg/io/__init__.py @@ -281,7 +281,7 @@ def _infer_file_io_from_schema(path: str, properties: Properties) -> Optional[Fi return None -def load_file_io(properties: Properties, location: Optional[str] = None) -> FileIO: +def load_file_io(properties: Properties = EMPTY_DICT, location: Optional[str] = None) -> FileIO: # First look for the py-io-impl property to directly load the class if io_impl := properties.get(PY_IO_IMPL): if file_io := _import_file_io(io_impl, properties): diff --git a/pyiceberg/table/__init__.py b/pyiceberg/table/__init__.py index 44bed4c357..3af1c13e66 100644 --- a/pyiceberg/table/__init__.py +++ b/pyiceberg/table/__init__.py @@ -16,7 +16,6 @@ # under the License. from __future__ import annotations -from functools import cached_property from typing import ( Any, Dict, @@ -28,7 +27,7 @@ from pydantic import Field from pyiceberg.expressions import AlwaysTrue, And, BooleanExpression -from pyiceberg.io import FileIO, load_file_io +from pyiceberg.io import FileIO from pyiceberg.schema import Schema from pyiceberg.table.metadata import TableMetadata from pyiceberg.table.partitioning import PartitionSpec @@ -41,22 +40,13 @@ class Table: identifier: Identifier = Field() metadata: TableMetadata = Field() metadata_location: str = Field() - catalog_properties: Properties - config: Properties + io: FileIO - def __init__( - self, - identifier: Identifier, - metadata: TableMetadata, - metadata_location: str, - catalog_properties: Properties = EMPTY_DICT, - config: Properties = EMPTY_DICT, - ): + def __init__(self, identifier: Identifier, metadata: TableMetadata, metadata_location: str, io: FileIO): self.identifier = identifier self.metadata = metadata self.metadata_location = metadata_location - self.catalog_properties = catalog_properties - self.config = config + self.io = io def refresh(self): """Refresh the current table metadata""" @@ -138,10 +128,6 @@ def history(self) -> List[SnapshotLogEntry]: """Get the snapshot history of this table.""" return self.metadata.snapshot_log - @cached_property - def io(self) -> FileIO: - return load_file_io({**self.catalog_properties, **self.metadata.properties, **self.config}) - def __eq__(self, other: Any) -> bool: return ( self.identifier == other.identifier diff --git a/tests/catalog/test_base.py b/tests/catalog/test_base.py index 5aea87c918..5b8886ceea 100644 --- a/tests/catalog/test_base.py +++ b/tests/catalog/test_base.py @@ -38,6 +38,7 @@ NoSuchTableError, TableAlreadyExistsError, ) +from pyiceberg.io import load_file_io from pyiceberg.schema import Schema from pyiceberg.table import Table from pyiceberg.table.partitioning import UNPARTITIONED_PARTITION_SPEC, PartitionField, PartitionSpec @@ -81,6 +82,7 @@ def create_table( identifier=identifier, metadata=EXAMPLE_TABLE_METADATA_V1, metadata_location=f's3://warehouse/{"/".join(identifier)}/metadata/metadata.json', + io=load_file_io(), ) self.__tables[identifier] = table return table @@ -115,7 +117,10 @@ def rename_table(self, from_identifier: Union[str, Identifier], to_identifier: U self.__namespaces[to_namespace] = {} self.__tables[to_identifier] = Table( - identifier=to_identifier, metadata=table.metadata, metadata_location=table.metadata_location + identifier=to_identifier, + metadata=table.metadata, + metadata_location=table.metadata_location, + io=load_file_io(), ) return self.__tables[to_identifier] diff --git a/tests/catalog/test_rest.py b/tests/catalog/test_rest.py index 8779f3b947..c84c24ae5e 100644 --- a/tests/catalog/test_rest.py +++ b/tests/catalog/test_rest.py @@ -30,6 +30,7 @@ OAuthError, TableAlreadyExistsError, ) +from pyiceberg.io import load_file_io from pyiceberg.schema import Schema from pyiceberg.table.metadata import TableMetadataV1 from pyiceberg.table.partitioning import PartitionField, PartitionSpec @@ -439,7 +440,7 @@ def test_load_table_200(rest_mock: Mocker): ), partition_spec=[], ), - config={"client.factory": "io.tabular.iceberg.catalog.TabularAwsClientFactory", "region": "us-west-2"}, + io=load_file_io(), ) assert actual == expected @@ -594,6 +595,7 @@ def test_create_table_200(rest_mock: Mocker, table_schema_simple: Schema): ), partition_spec=[], ), + io=load_file_io(), ) diff --git a/tests/cli/test_console.py b/tests/cli/test_console.py index ebb9ab37d8..1c5728831b 100644 --- a/tests/cli/test_console.py +++ b/tests/cli/test_console.py @@ -28,6 +28,7 @@ from pyiceberg.catalog import Catalog, PropertiesUpdateSummary from pyiceberg.cli.console import run from pyiceberg.exceptions import NoSuchNamespaceError, NoSuchTableError +from pyiceberg.io import load_file_io from pyiceberg.schema import Schema from pyiceberg.table import Table from pyiceberg.table.metadata import TableMetadataV2 @@ -51,6 +52,7 @@ def create_table( identifier=Catalog.identifier_to_tuple(identifier), metadata_location="s3://tmp/", metadata=TableMetadataV2(**EXAMPLE_TABLE_METADATA_V2), + io=load_file_io(), ) def load_table(self, identifier: Union[str, Identifier]) -> Table: @@ -60,6 +62,7 @@ def load_table(self, identifier: Union[str, Identifier]) -> Table: identifier=tuple_identifier, metadata_location="s3://tmp/", metadata=TableMetadataV2(**EXAMPLE_TABLE_METADATA_V2), + io=load_file_io(), ) else: raise NoSuchTableError(f"Table does not exist: {'.'.join(tuple_identifier)}") @@ -78,7 +81,10 @@ def rename_table(self, from_identifier: Union[str, Identifier], to_identifier: U tuple_identifier = Catalog.identifier_to_tuple(from_identifier) if tuple_identifier == ("default", "foo"): return Table( - identifier=tuple_identifier, metadata_location="s3://tmp/", metadata=TableMetadataV2(**EXAMPLE_TABLE_METADATA_V2) + identifier=tuple_identifier, + metadata_location="s3://tmp/", + metadata=TableMetadataV2(**EXAMPLE_TABLE_METADATA_V2), + io=load_file_io(), ) else: raise NoSuchTableError(f"Table does not exist: {from_identifier}") diff --git a/tests/table/test_init.py b/tests/table/test_init.py index e42c0f2d81..e20320701c 100644 --- a/tests/table/test_init.py +++ b/tests/table/test_init.py @@ -25,6 +25,7 @@ EqualTo, In, ) +from pyiceberg.io import load_file_io from pyiceberg.schema import Schema from pyiceberg.table import PartitionSpec, Table from pyiceberg.table.metadata import TableMetadataV2 @@ -52,6 +53,7 @@ def table(example_table_metadata_v2: Dict[str, Any]) -> Table: identifier=("database", "table"), metadata=table_metadata, metadata_location=f"{table_metadata.location}/uuid.metadata.json", + io=load_file_io(), ) From 69996a53acc841f5927d787323259c81bb784cf0 Mon Sep 17 00:00:00 2001 From: Alec Heifetz Date: Fri, 11 Nov 2022 15:53:29 -0500 Subject: [PATCH 265/642] Python: Fix previous metadata location property Co-authored-by: Alec Heifetz --- pyiceberg/catalog/hive.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyiceberg/catalog/hive.py b/pyiceberg/catalog/hive.py index 779b135d53..65041697d6 100644 --- a/pyiceberg/catalog/hive.py +++ b/pyiceberg/catalog/hive.py @@ -153,7 +153,7 @@ def _construct_hive_storage_descriptor(schema: Schema, location: Optional[str]) def _construct_parameters(metadata_location: str, previous_metadata_location: Optional[str] = None) -> Dict[str, Any]: properties = {PROP_EXTERNAL: "TRUE", PROP_TABLE_TYPE: "ICEBERG", PROP_METADATA_LOCATION: metadata_location} if previous_metadata_location: - properties[previous_metadata_location] = previous_metadata_location + properties[PROP_PREVIOUS_METADATA_LOCATION] = previous_metadata_location return properties From 8c3ea4d9916554c85e7eef5ec80b27f26c694aaa Mon Sep 17 00:00:00 2001 From: Fokko Driesprong Date: Mon, 14 Nov 2022 23:00:58 +0100 Subject: [PATCH 266/642] Python: Make invalid Literal conversions explicit (#6141) Currently we silently turn Literals into None if we can't convert them, instead I prefer to raise an exception. This can cause silent bugs like: EqualTo(Reference("id"), StringLiteral("123a")) will turn into a IsNull predicate since 123a cannot be casted to an Long (assuming that the `id` column is a Long). --- pyiceberg/expressions/literals.py | 264 +++++++++++++++++------------ tests/expressions/test_literals.py | 134 ++++++++++----- 2 files changed, 248 insertions(+), 150 deletions(-) diff --git a/pyiceberg/expressions/literals.py b/pyiceberg/expressions/literals.py index d38cd5f741..81d92a4448 100644 --- a/pyiceberg/expressions/literals.py +++ b/pyiceberg/expressions/literals.py @@ -27,8 +27,9 @@ from decimal import ROUND_HALF_UP, Decimal from functools import singledispatch, singledispatchmethod from typing import ( + Any, Generic, - Optional, + Type, TypeVar, Union, ) @@ -42,6 +43,7 @@ DoubleType, FixedType, FloatType, + IcebergType, IntegerType, LongType, StringType, @@ -66,49 +68,50 @@ class Literal(Generic[T], ABC): """Literal which has a value and can be converted between types""" - def __init__(self, value: T, value_type: type): + def __init__(self, value: T, value_type: Type): if value is None or not isinstance(value, value_type): raise TypeError(f"Invalid literal value: {value} (not a {value_type})") self._value = value @property def value(self) -> T: - return self._value # type: ignore + return self._value + @singledispatchmethod @abstractmethod - def to(self, type_var) -> Literal: + def to(self, type_var: IcebergType) -> Literal: ... # pragma: no cover - def __repr__(self): + def __repr__(self) -> str: return f"{type(self).__name__}({self.value})" - def __str__(self): + def __str__(self) -> str: return str(self.value) - def __hash__(self): + def __hash__(self) -> int: return hash(self.value) - def __eq__(self, other): + def __eq__(self, other) -> bool: return self.value == other.value - def __ne__(self, other): + def __ne__(self, other) -> bool: return not self.__eq__(other) - def __lt__(self, other): + def __lt__(self, other) -> bool: return self.value < other.value - def __gt__(self, other): + def __gt__(self, other) -> bool: return self.value > other.value - def __le__(self, other): + def __le__(self, other) -> bool: return self.value <= other.value - def __ge__(self, other): + def __ge__(self, other) -> bool: return self.value >= other.value @singledispatch -def literal(value) -> Literal: +def literal(value: Any) -> Literal: """ A generic Literal factory to construct an iceberg Literal based on python primitive data type using dynamic overloading @@ -171,36 +174,60 @@ def _(value: date) -> Literal[int]: return DateLiteral(date_to_days(value)) -class AboveMax(Singleton): - @property - def value(self): - raise ValueError("AboveMax has no value") +class FloatAboveMax(Literal[float], Singleton): + def __init__(self): + super().__init__(FloatType.max, float) + + def to(self, type_var: IcebergType) -> Literal: # type: ignore + raise TypeError("Cannot change the type of FloatAboveMax") + + def __repr__(self) -> str: + return "FloatAboveMax()" - def to(self, type_var): - raise TypeError("Cannot change the type of AboveMax") + def __str__(self) -> str: + return "FloatAboveMax" - def __repr__(self): - return "AboveMax()" - def __str__(self): - return "AboveMax" +class FloatBelowMin(Literal[float], Singleton): + def __init__(self): + super().__init__(FloatType.min, float) + + def to(self, type_var: IcebergType) -> Literal: # type: ignore + raise TypeError("Cannot change the type of FloatBelowMin") + + def __repr__(self) -> str: + return "FloatBelowMin()" + def __str__(self) -> str: + return "FloatBelowMin" -class BelowMin(Singleton): + +class IntAboveMax(Literal[int]): def __init__(self): - pass + super().__init__(IntegerType.max, int) - def value(self): - raise ValueError("BelowMin has no value") + def to(self, type_var: IcebergType) -> Literal: # type: ignore + raise TypeError("Cannot change the type of IntAboveMax") - def to(self, type_var): - raise TypeError("Cannot change the type of BelowMin") + def __repr__(self) -> str: + return "IntAboveMax()" - def __repr__(self): - return "BelowMin()" + def __str__(self) -> str: + return "IntAboveMax" - def __str__(self): - return "BelowMin" + +class IntBelowMin(Literal[int]): + def __init__(self): + super().__init__(IntegerType.min, int) + + def to(self, type_var: IcebergType) -> Literal: # type: ignore + raise TypeError("Cannot change the type of IntBelowMin") + + def __repr__(self) -> str: + return "IntBelowMin()" + + def __str__(self) -> str: + return "IntBelowMin" class BooleanLiteral(Literal[bool]): @@ -208,11 +235,11 @@ def __init__(self, value: bool): super().__init__(value, bool) @singledispatchmethod - def to(self, type_var): - return None + def to(self, type_var: IcebergType) -> Literal: + raise TypeError(f"Cannot convert BooleanLiteral into {type_var}") @to.register(BooleanType) - def _(self, type_var): + def _(self, _: BooleanType) -> Literal[bool]: return self @@ -221,39 +248,39 @@ def __init__(self, value: int): super().__init__(value, int) @singledispatchmethod - def to(self, type_var): - return None + def to(self, type_var: IcebergType) -> Literal: + raise TypeError(f"Cannot convert LongLiteral into {type_var}") @to.register(LongType) - def _(self, type_var: LongType) -> Literal[int]: + def _(self, _: LongType) -> Literal[int]: return self @to.register(IntegerType) - def _(self, _: IntegerType) -> Union[AboveMax, BelowMin, Literal[int]]: + def _(self, _: IntegerType) -> Literal[int]: if IntegerType.max < self.value: - return AboveMax() + return IntAboveMax() elif IntegerType.min > self.value: - return BelowMin() + return IntBelowMin() return self @to.register(FloatType) - def _(self, type_var: FloatType) -> Literal[float]: + def _(self, _: FloatType) -> Literal[float]: return FloatLiteral(float(self.value)) @to.register(DoubleType) - def _(self, type_var: DoubleType) -> Literal[float]: + def _(self, _: DoubleType) -> Literal[float]: return DoubleLiteral(float(self.value)) @to.register(DateType) - def _(self, type_var: DateType) -> Literal[int]: + def _(self, _: DateType) -> Literal[int]: return DateLiteral(self.value) @to.register(TimeType) - def _(self, type_var: TimeType) -> Literal[int]: + def _(self, _: TimeType) -> Literal[int]: return TimeLiteral(self.value) @to.register(TimestampType) - def _(self, type_var: TimestampType) -> Literal[int]: + def _(self, _: TimestampType) -> Literal[int]: return TimestampLiteral(self.value) @to.register(DecimalType) @@ -272,31 +299,31 @@ def __init__(self, value: float): super().__init__(value, float) self._value32 = struct.unpack(" bool: return self._value32 == other - def __lt__(self, other): + def __lt__(self, other) -> bool: return self._value32 < other - def __gt__(self, other): + def __gt__(self, other) -> bool: return self._value32 > other - def __le__(self, other): + def __le__(self, other) -> bool: return self._value32 <= other - def __ge__(self, other): + def __ge__(self, other) -> bool: return self._value32 >= other @singledispatchmethod - def to(self, type_var): - return None + def to(self, type_var: IcebergType) -> Literal: + raise TypeError(f"Cannot convert FloatLiteral into {type_var}") @to.register(FloatType) - def _(self, type_var: FloatType) -> Literal[float]: + def _(self, _: FloatType) -> Literal[float]: return self @to.register(DoubleType) - def _(self, type_var: DoubleType) -> Literal[float]: + def _(self, _: DoubleType) -> Literal[float]: return DoubleLiteral(self.value) @to.register(DecimalType) @@ -309,19 +336,19 @@ def __init__(self, value: float): super().__init__(value, float) @singledispatchmethod - def to(self, type_var): - return None + def to(self, type_var: IcebergType) -> Literal: + raise TypeError(f"Cannot convert DoubleLiteral into {type_var}") @to.register(DoubleType) - def _(self, type_var: DoubleType) -> Literal[float]: + def _(self, _: DoubleType) -> Literal[float]: return self @to.register(FloatType) - def _(self, _: FloatType) -> Union[AboveMax, BelowMin, Literal[float]]: + def _(self, _: FloatType) -> Union[FloatAboveMax, FloatBelowMin, FloatLiteral]: if FloatType.max < self.value: - return AboveMax() + return FloatAboveMax() elif FloatType.min > self.value: - return BelowMin() + return FloatBelowMin() return FloatLiteral(self.value) @to.register(DecimalType) @@ -334,11 +361,11 @@ def __init__(self, value: int): super().__init__(value, int) @singledispatchmethod - def to(self, type_var): - return None + def to(self, type_var: IcebergType) -> Literal: + raise TypeError(f"Cannot convert DateLiteral into {type_var}") @to.register(DateType) - def _(self, type_var: DateType) -> Literal[int]: + def _(self, _: DateType) -> Literal[int]: return self @@ -347,11 +374,11 @@ def __init__(self, value: int): super().__init__(value, int) @singledispatchmethod - def to(self, type_var): - return None + def to(self, type_var: IcebergType) -> Literal: + raise TypeError(f"Cannot convert TimeLiteral into {type_var}") @to.register(TimeType) - def _(self, type_var: TimeType) -> Literal[int]: + def _(self, _: TimeType) -> Literal[int]: return self @@ -360,15 +387,15 @@ def __init__(self, value: int): super().__init__(value, int) @singledispatchmethod - def to(self, type_var): - return None + def to(self, type_var: IcebergType) -> Literal: + raise TypeError(f"Cannot convert TimestampLiteral into {type_var}") @to.register(TimestampType) - def _(self, type_var: TimestampType) -> Literal[int]: + def _(self, _: TimestampType) -> Literal[int]: return self @to.register(DateType) - def _(self, type_var: DateType) -> Literal[int]: + def _(self, _: DateType) -> Literal[int]: return DateLiteral(micros_to_days(self.value)) @@ -377,14 +404,14 @@ def __init__(self, value: Decimal): super().__init__(value, Decimal) @singledispatchmethod - def to(self, type_var): - return None + def to(self, type_var: IcebergType) -> Literal: + raise TypeError(f"Cannot convert DecimalLiteral into {type_var}") @to.register(DecimalType) - def _(self, type_var: DecimalType) -> Optional[Literal[Decimal]]: + def _(self, type_var: DecimalType) -> Literal[Decimal]: if type_var.scale == abs(self.value.as_tuple().exponent): return self - return None + raise ValueError(f"Could not convert {self.value} into a {type_var}") class StringLiteral(Literal[str]): @@ -392,52 +419,69 @@ def __init__(self, value: str): super().__init__(value, str) @singledispatchmethod - def to(self, type_var): - return None + def to(self, type_var: IcebergType) -> Literal: + raise TypeError(f"Cannot convert StringLiteral into {type_var}") @to.register(StringType) - def _(self, type_var: StringType) -> Literal[str]: + def _(self, _: StringType) -> Literal[str]: return self + @to.register(IntegerType) + def _(self, type_var: IntegerType) -> Union[IntAboveMax, IntBelowMin, LongLiteral]: + try: + number = int(float(self.value)) + + if IntegerType.max < number: + return IntAboveMax() + elif IntegerType.min > number: + return IntBelowMin() + return LongLiteral(number) + except ValueError as e: + raise ValueError(f"Could not convert {self.value} into a {type_var}") from e + + @to.register(LongType) + def _(self, type_var: LongType) -> Literal[int]: + try: + return LongLiteral(int(float(self.value))) + except (TypeError, ValueError) as e: + raise ValueError(f"Could not convert {self.value} into a {type_var}") from e + @to.register(DateType) - def _(self, type_var: DateType) -> Optional[Literal[int]]: + def _(self, type_var: DateType) -> Literal[int]: try: return DateLiteral(date_str_to_days(self.value)) - except (TypeError, ValueError): - return None + except (TypeError, ValueError) as e: + raise ValueError(f"Could not convert {self.value} into a {type_var}") from e @to.register(TimeType) - def _(self, type_var: TimeType) -> Optional[Literal[int]]: + def _(self, type_var: TimeType) -> Literal[int]: try: return TimeLiteral(time_to_micros(self.value)) - except (TypeError, ValueError): - return None + except (TypeError, ValueError) as e: + raise ValueError(f"Could not convert {self.value} into a {type_var}") from e @to.register(TimestampType) - def _(self, type_var: TimestampType) -> Optional[Literal[int]]: + def _(self, type_var: TimestampType) -> Literal[int]: try: return TimestampLiteral(timestamp_to_micros(self.value)) - except (TypeError, ValueError): - return None + except (TypeError, ValueError) as e: + raise ValueError(f"Could not convert {self.value} into a {type_var}") from e @to.register(TimestamptzType) - def _(self, type_var: TimestamptzType) -> Optional[Literal[int]]: - try: - return TimestampLiteral(timestamptz_to_micros(self.value)) - except (TypeError, ValueError): - return None + def _(self, _: TimestamptzType) -> Literal[int]: + return TimestampLiteral(timestamptz_to_micros(self.value)) @to.register(UUIDType) - def _(self, type_var: UUIDType) -> Literal[UUID]: + def _(self, _: UUIDType) -> Literal[UUID]: return UUIDLiteral(UUID(self.value)) @to.register(DecimalType) - def _(self, type_var: DecimalType) -> Optional[Literal[Decimal]]: + def _(self, type_var: DecimalType) -> Literal[Decimal]: dec = Decimal(self.value) if type_var.scale == abs(dec.as_tuple().exponent): return DecimalLiteral(dec) else: - return None + raise ValueError(f"Could not convert {self.value} into a {type_var}") class UUIDLiteral(Literal[UUID]): @@ -445,11 +489,11 @@ def __init__(self, value: UUID): super().__init__(value, UUID) @singledispatchmethod - def to(self, type_var): - return None + def to(self, type_var: IcebergType) -> Literal: + raise TypeError(f"Cannot convert UUIDLiteral into {type_var}") @to.register(UUIDType) - def _(self, type_var: UUIDType) -> Literal[UUID]: + def _(self, _: UUIDType) -> Literal[UUID]: return self @@ -458,18 +502,18 @@ def __init__(self, value: bytes): super().__init__(value, bytes) @singledispatchmethod - def to(self, type_var): - return None + def to(self, type_var: IcebergType) -> Literal: + raise TypeError(f"Cannot convert FixedLiteral into {type_var}") @to.register(FixedType) - def _(self, type_var: FixedType) -> Optional[Literal[bytes]]: + def _(self, type_var: FixedType) -> Literal[bytes]: if len(self.value) == len(type_var): return self else: - return None + raise ValueError(f"Could not convert {self.value!r} into a {type_var}") @to.register(BinaryType) - def _(self, type_var: BinaryType) -> Literal[bytes]: + def _(self, _: BinaryType) -> Literal[bytes]: return BinaryLiteral(self.value) @@ -478,16 +522,18 @@ def __init__(self, value: bytes): super().__init__(value, bytes) @singledispatchmethod - def to(self, type_var): - return None + def to(self, type_var: IcebergType) -> Literal: + raise TypeError(f"Cannot convert BinaryLiteral into {type_var}") @to.register(BinaryType) def _(self, _: BinaryType) -> Literal[bytes]: return self @to.register(FixedType) - def _(self, type_var: FixedType) -> Optional[Literal[bytes]]: + def _(self, type_var: FixedType) -> Literal[bytes]: if len(type_var) == len(self.value): return FixedLiteral(self.value) else: - return None + raise TypeError( + f"Cannot convert BinaryLiteral into {type_var}, different length: {len(type_var)} <> {len(self.value)}" + ) diff --git a/tests/expressions/test_literals.py b/tests/expressions/test_literals.py index f34c8ae683..dee19359e6 100644 --- a/tests/expressions/test_literals.py +++ b/tests/expressions/test_literals.py @@ -14,6 +14,8 @@ # KIND, either express or implied. See the License for the # specific language governing permissions and limitations # under the License. +# pylint:disable=eval-used + import datetime import uuid from decimal import Decimal @@ -21,15 +23,17 @@ import pytest from pyiceberg.expressions.literals import ( - AboveMax, - BelowMin, BinaryLiteral, BooleanLiteral, DateLiteral, DecimalLiteral, DoubleLiteral, FixedLiteral, + FloatAboveMax, + FloatBelowMin, FloatLiteral, + IntAboveMax, + IntBelowMin, LongLiteral, StringLiteral, TimeLiteral, @@ -53,8 +57,6 @@ UUIDType, ) -# Base - def test_literal_from_none_error(): with pytest.raises(TypeError) as e: @@ -147,11 +149,11 @@ def test_long_to_integer_within_bound(): def test_long_to_integer_outside_bound(): big_lit = literal(IntegerType.max + 1).to(LongType()) above_max_lit = big_lit.to(IntegerType()) - assert above_max_lit == AboveMax() + assert above_max_lit == IntAboveMax() small_lit = literal(IntegerType.min - 1).to(LongType()) below_min_lit = small_lit.to(IntegerType()) - assert below_min_lit == BelowMin() + assert below_min_lit == IntBelowMin() def test_long_to_float_conversion(): @@ -218,11 +220,11 @@ def test_double_to_float_within_bound(): def test_double_to_float_outside_bound(): big_lit = literal(FloatType.max + 1.0e37).to(DoubleType()) above_max_lit = big_lit.to(FloatType()) - assert above_max_lit == AboveMax() + assert above_max_lit == FloatAboveMax() small_lit = literal(FloatType.min - 1.0e37).to(DoubleType()) below_min_lit = small_lit.to(FloatType()) - assert below_min_lit == BelowMin() + assert below_min_lit == FloatBelowMin() @pytest.mark.parametrize( @@ -239,9 +241,15 @@ def test_decimal_to_decimal_conversion(): assert lit.value.as_tuple() == lit.to(DecimalType(9, 2)).value.as_tuple() assert lit.value.as_tuple() == lit.to(DecimalType(11, 2)).value.as_tuple() - assert lit.to(DecimalType(9, 0)) is None - assert lit.to(DecimalType(9, 1)) is None - assert lit.to(DecimalType(9, 3)) is None + with pytest.raises(ValueError) as e: + _ = lit.to(DecimalType(9, 0)) + assert "Could not convert 34.11 into a decimal(9, 0)" in str(e.value) + with pytest.raises(ValueError) as e: + _ = lit.to(DecimalType(9, 1)) + assert "Could not convert 34.11 into a decimal(9, 1)" in str(e.value) + with pytest.raises(ValueError) as e: + _ = lit.to(DecimalType(9, 3)) + assert "Could not convert 34.11 into a decimal(9, 3)" in str(e.value) def test_timestamp_to_date(): @@ -251,15 +259,15 @@ def test_timestamp_to_date(): assert date_lit.value == 0 -# STRING - - def test_string_literal(): sqrt2 = literal("1.414").to(StringType()) pi = literal("3.141").to(StringType()) pi_string_lit = StringLiteral("3.141") pi_double_lit = literal(3.141).to(DoubleType()) + assert literal("3.141").to(IntegerType()) == literal(3) + assert literal("3.141").to(LongType()) == literal(3) + assert sqrt2 != pi assert pi != pi_double_lit assert pi == pi_string_lit @@ -312,12 +320,16 @@ def test_string_to_timestamp_literal(): def test_timestamp_with_zone_without_zone_in_literal(): timestamp_str = literal("2017-08-18T14:21:01.919234") - assert timestamp_str.to(TimestamptzType()) is None + with pytest.raises(ValueError) as e: + _ = timestamp_str.to(timestamp_str.to(TimestamptzType())) + assert "Invalid timestamp with zone: 2017-08-18T14:21:01.919234 (must be ISO-8601)" in str(e.value) def test_timestamp_without_zone_with_zone_in_literal(): timestamp_str = literal("2017-08-18T14:21:01.919234+07:00") - assert timestamp_str.to(TimestampType()) is None + with pytest.raises(ValueError) as e: + _ = timestamp_str.to(TimestampType()) + assert "Could not convert 2017-08-18T14:21:01.919234+07:00 into a timestamp" in str(e.value) def test_string_to_uuid_literal(): @@ -431,12 +443,18 @@ def test_binary_to_fixed(): fixed_lit = lit.to(FixedType(3)) assert fixed_lit is not None assert lit.value == fixed_lit.value - assert lit.to(FixedType(4)) is None + + with pytest.raises(TypeError) as e: + _ = lit.to(FixedType(4)) + assert "Cannot convert BinaryLiteral into fixed[4], different length: 4 <> 3" in str(e.value) def test_binary_to_smaller_fixed_none(): lit = literal(bytearray([0x00, 0x01, 0x02])) - assert lit.to(FixedType(2)) is None + + with pytest.raises(TypeError) as e: + _ = lit.to(FixedType(2)) + assert "Cannot convert BinaryLiteral into fixed[2], different length: 2 <> 3" in str(e.value) def test_fixed_to_binary(): @@ -448,35 +466,61 @@ def test_fixed_to_binary(): def test_fixed_to_smaller_fixed_none(): lit = literal(bytearray([0x00, 0x01, 0x02])).to(FixedType(3)) - assert lit.to(FixedType(2)) is None + with pytest.raises(ValueError) as e: + lit.to(lit.to(FixedType(2))) + assert "Could not convert b'\\x00\\x01\\x02' into a fixed[2]" in str(e.value) -def test_above_max(): - a = AboveMax() +def test_above_max_float(): + a = FloatAboveMax() # singleton - assert a == AboveMax() - assert str(a) == "AboveMax" - assert repr(a) == "AboveMax()" - with pytest.raises(ValueError) as e: - a.value() - assert "AboveMax has no value" in str(e.value) + assert a == FloatAboveMax() + assert str(a) == "FloatAboveMax" + assert repr(a) == "FloatAboveMax()" + assert a.value == FloatType.max + assert a == eval(repr(a)) with pytest.raises(TypeError) as e: a.to(IntegerType()) - assert "Cannot change the type of AboveMax" in str(e.value) + assert "Cannot change the type of FloatAboveMax" in str(e.value) -def test_below_min(): - b = BelowMin() +def test_below_min_float(): + b = FloatBelowMin() # singleton - assert b == BelowMin() - assert str(b) == "BelowMin" - assert repr(b) == "BelowMin()" - with pytest.raises(ValueError) as e: - b.value() - assert "BelowMin has no value" in str(e.value) + assert b == FloatBelowMin() + assert str(b) == "FloatBelowMin" + assert repr(b) == "FloatBelowMin()" + assert b == eval(repr(b)) + assert b.value == FloatType.min with pytest.raises(TypeError) as e: b.to(IntegerType()) - assert "Cannot change the type of BelowMin" in str(e.value) + assert "Cannot change the type of FloatBelowMin" in str(e.value) + + +def test_above_max_int(): + a = IntAboveMax() + # singleton + assert a == IntAboveMax() + assert str(a) == "IntAboveMax" + assert repr(a) == "IntAboveMax()" + assert a.value == IntegerType.max + assert a == eval(repr(a)) + with pytest.raises(TypeError) as e: + a.to(IntegerType()) + assert "Cannot change the type of IntAboveMax" in str(e.value) + + +def test_below_min_int(): + b = IntBelowMin() + # singleton + assert b == IntBelowMin() + assert str(b) == "IntBelowMin" + assert repr(b) == "IntBelowMin()" + assert b == eval(repr(b)) + assert b.value == IntegerType.min + with pytest.raises(TypeError) as e: + b.to(IntegerType()) + assert "Cannot change the type of IntBelowMin" in str(e.value) def test_invalid_boolean_conversions(): @@ -531,7 +575,8 @@ def test_invalid_long_conversions(): ], ) def test_invalid_float_conversions(lit, test_type): - assert lit.to(test_type) is None + with pytest.raises(TypeError): + _ = lit.to(test_type) @pytest.mark.parametrize("lit", [literal("2017-08-18").to(DateType())]) @@ -597,6 +642,13 @@ def test_invalid_timestamp_conversions(): ) +def test_invalid_decimal_conversion_scale(): + lit = literal(Decimal("34.11")) + with pytest.raises(ValueError) as e: + lit.to(DecimalType(9, 4)) + assert "Could not convert 34.11 into a decimal(9, 4)" in str(e.value) + + def test_invalid_decimal_conversions(): assert_invalid_conversions( literal(Decimal("34.11")), @@ -610,7 +662,6 @@ def test_invalid_decimal_conversions(): TimeType(), TimestampType(), TimestamptzType(), - DecimalType(9, 4), StringType(), UUIDType(), FixedType(1), @@ -622,7 +673,7 @@ def test_invalid_decimal_conversions(): def test_invalid_string_conversions(): assert_invalid_conversions( literal("abc"), - [BooleanType(), IntegerType(), LongType(), FloatType(), DoubleType(), FixedType(1), BinaryType()], + [BooleanType(), FloatType(), DoubleType(), FixedType(1), BinaryType()], ) @@ -689,4 +740,5 @@ def test_invalid_binary_conversions(): def assert_invalid_conversions(lit, types=None): for type_var in types: - assert lit.to(type_var) is None + with pytest.raises(TypeError): + _ = lit.to(type_var) From 8860f3e05313e24b8a0669edf86d94010e9aa902 Mon Sep 17 00:00:00 2001 From: Fokko Driesprong Date: Wed, 16 Nov 2022 21:48:45 +0100 Subject: [PATCH 267/642] Python: Fix rough edges around literals (#6197) * Python: Fix rough edges around literals * timezone -> zone offset --- pyiceberg/expressions/literals.py | 20 ++++----- pyiceberg/utils/datetime.py | 6 +++ tests/expressions/test_literals.py | 66 +++++++++++++++++++++++++++++- 3 files changed, 80 insertions(+), 12 deletions(-) diff --git a/pyiceberg/expressions/literals.py b/pyiceberg/expressions/literals.py index 81d92a4448..a84cd9ae39 100644 --- a/pyiceberg/expressions/literals.py +++ b/pyiceberg/expressions/literals.py @@ -31,7 +31,6 @@ Generic, Type, TypeVar, - Union, ) from uuid import UUID @@ -344,7 +343,7 @@ def _(self, _: DoubleType) -> Literal[float]: return self @to.register(FloatType) - def _(self, _: FloatType) -> Union[FloatAboveMax, FloatBelowMin, FloatLiteral]: + def _(self, _: FloatType) -> Literal[float]: if FloatType.max < self.value: return FloatAboveMax() elif FloatType.min > self.value: @@ -427,7 +426,7 @@ def _(self, _: StringType) -> Literal[str]: return self @to.register(IntegerType) - def _(self, type_var: IntegerType) -> Union[IntAboveMax, IntBelowMin, LongLiteral]: + def _(self, type_var: IntegerType) -> Literal[int]: try: number = int(float(self.value)) @@ -461,11 +460,8 @@ def _(self, type_var: TimeType) -> Literal[int]: raise ValueError(f"Could not convert {self.value} into a {type_var}") from e @to.register(TimestampType) - def _(self, type_var: TimestampType) -> Literal[int]: - try: - return TimestampLiteral(timestamp_to_micros(self.value)) - except (TypeError, ValueError) as e: - raise ValueError(f"Could not convert {self.value} into a {type_var}") from e + def _(self, _: TimestampType) -> Literal[int]: + return TimestampLiteral(timestamp_to_micros(self.value)) @to.register(TimestamptzType) def _(self, _: TimestamptzType) -> Literal[int]: @@ -481,7 +477,9 @@ def _(self, type_var: DecimalType) -> Literal[Decimal]: if type_var.scale == abs(dec.as_tuple().exponent): return DecimalLiteral(dec) else: - raise ValueError(f"Could not convert {self.value} into a {type_var}") + raise ValueError( + f"Could not convert {self.value} into a {type_var}, scales differ {type_var.scale} <> {abs(dec.as_tuple().exponent)}" + ) class UUIDLiteral(Literal[UUID]): @@ -510,7 +508,9 @@ def _(self, type_var: FixedType) -> Literal[bytes]: if len(self.value) == len(type_var): return self else: - raise ValueError(f"Could not convert {self.value!r} into a {type_var}") + raise ValueError( + f"Could not convert {self.value!r} into a {type_var}, lengths differ {len(self.value)} <> {len(type_var)}" + ) @to.register(BinaryType) def _(self, _: BinaryType) -> Literal[bytes]: diff --git a/pyiceberg/utils/datetime.py b/pyiceberg/utils/datetime.py index 31a8e6193d..7353ebd4ba 100644 --- a/pyiceberg/utils/datetime.py +++ b/pyiceberg/utils/datetime.py @@ -81,6 +81,9 @@ def timestamp_to_micros(timestamp_str: str) -> int: """Converts an ISO-9601 formatted timestamp without zone to microseconds from 1970-01-01T00:00:00.000000""" if ISO_TIMESTAMP.fullmatch(timestamp_str): return datetime_to_micros(datetime.fromisoformat(timestamp_str)) + if ISO_TIMESTAMPTZ.fullmatch(timestamp_str): + # When we can match a timestamp without a zone, we can give a more specific error + raise ValueError(f"Zone offset provided, but not expected: {timestamp_str}") raise ValueError(f"Invalid timestamp without zone: {timestamp_str} (must be ISO-8601)") @@ -88,6 +91,9 @@ def timestamptz_to_micros(timestamptz_str: str) -> int: """Converts an ISO-8601 formatted timestamp with zone to microseconds from 1970-01-01T00:00:00.000000+00:00""" if ISO_TIMESTAMPTZ.fullmatch(timestamptz_str): return datetime_to_micros(datetime.fromisoformat(timestamptz_str)) + if ISO_TIMESTAMP.fullmatch(timestamptz_str): + # When we can match a timestamp without a zone, we can give a more specific error + raise ValueError(f"Missing zone offset: {timestamptz_str} (must be ISO-8601)") raise ValueError(f"Invalid timestamp with zone: {timestamptz_str} (must be ISO-8601)") diff --git a/tests/expressions/test_literals.py b/tests/expressions/test_literals.py index dee19359e6..96656ba825 100644 --- a/tests/expressions/test_literals.py +++ b/tests/expressions/test_literals.py @@ -322,14 +322,28 @@ def test_timestamp_with_zone_without_zone_in_literal(): timestamp_str = literal("2017-08-18T14:21:01.919234") with pytest.raises(ValueError) as e: _ = timestamp_str.to(timestamp_str.to(TimestamptzType())) - assert "Invalid timestamp with zone: 2017-08-18T14:21:01.919234 (must be ISO-8601)" in str(e.value) + assert "Missing zone offset: 2017-08-18T14:21:01.919234 (must be ISO-8601)" in str(e.value) + + +def test_invalid_timestamp_in_literal(): + timestamp_str = literal("abc") + with pytest.raises(ValueError) as e: + _ = timestamp_str.to(timestamp_str.to(TimestamptzType())) + assert "Invalid timestamp with zone: abc (must be ISO-8601)" in str(e.value) def test_timestamp_without_zone_with_zone_in_literal(): timestamp_str = literal("2017-08-18T14:21:01.919234+07:00") with pytest.raises(ValueError) as e: _ = timestamp_str.to(TimestampType()) - assert "Could not convert 2017-08-18T14:21:01.919234+07:00 into a timestamp" in str(e.value) + assert "Zone offset provided, but not expected: 2017-08-18T14:21:01.919234+07:00" in str(e.value) + + +def test_invalid_timestamp_with_zone_in_literal(): + timestamp_str = literal("abc") + with pytest.raises(ValueError) as e: + _ = timestamp_str.to(TimestampType()) + assert "Invalid timestamp without zone: abc (must be ISO-8601)" in str(e.value) def test_string_to_uuid_literal(): @@ -742,3 +756,51 @@ def assert_invalid_conversions(lit, types=None): for type_var in types: with pytest.raises(TypeError): _ = lit.to(type_var) + + +def test_compare_floats(): + lhs = literal(18.15).to(FloatType()) + rhs = literal(19.25).to(FloatType()) + assert lhs != rhs + assert lhs < rhs + assert lhs <= rhs + assert not lhs > rhs + assert not lhs >= rhs + + +def test_string_to_int_max_value(): + assert isinstance(literal(str(IntegerType.max + 1)).to(IntegerType()), IntAboveMax) + + +def test_string_to_int_min_value(): + assert isinstance(literal(str(IntegerType.min - 1)).to(IntegerType()), IntBelowMin) + + +def test_string_to_integer_type_invalid_value(): + with pytest.raises(ValueError) as e: + _ = literal("abc").to(IntegerType()) + assert "Could not convert abc into a int" in str(e.value) + + +def test_string_to_long_type_invalid_value(): + with pytest.raises(ValueError) as e: + _ = literal("abc").to(LongType()) + assert "Could not convert abc into a long" in str(e.value) + + +def test_string_to_date_type_invalid_value(): + with pytest.raises(ValueError) as e: + _ = literal("abc").to(DateType()) + assert "Could not convert abc into a date" in str(e.value) + + +def test_string_to_time_type_invalid_value(): + with pytest.raises(ValueError) as e: + _ = literal("abc").to(TimeType()) + assert "Could not convert abc into a time" in str(e.value) + + +def test_string_to_decimal_type_invalid_value(): + with pytest.raises(ValueError) as e: + _ = literal("18.15").to(DecimalType(10, 0)) + assert "Could not convert 18.15 into a decimal(10, 0), scales differ 0 <> 2" in str(e.value) From 726efd906017808144e36613a0c38ae7819f756b Mon Sep 17 00:00:00 2001 From: Luigi Cerone Date: Fri, 18 Nov 2022 17:06:38 +0100 Subject: [PATCH 268/642] Python: Update mypy to 0.991 (#6159) * Update mypy and fix mypy errors * Remove coalesce method --- .pre-commit-config.yaml | 2 +- pyiceberg/catalog/__init__.py | 11 +++++++---- pyiceberg/typedef.py | 2 +- pyiceberg/utils/config.py | 8 ++------ pyproject.toml | 1 + tests/avro/test_decoder.py | 5 +++-- 6 files changed, 15 insertions(+), 14 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 91452557d1..e6f71b94de 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -38,7 +38,7 @@ repos: - id: isort args: [ --settings-path=python/pyproject.toml ] - repo: https://github.com/pre-commit/mirrors-mypy - rev: v0.982 + rev: v0.991 hooks: - id: mypy args: [ --install-types, --non-interactive, --config=python/pyproject.toml] diff --git a/pyiceberg/catalog/__init__.py b/pyiceberg/catalog/__init__.py index 7b6c5f229e..01fa6f3cf6 100644 --- a/pyiceberg/catalog/__init__.py +++ b/pyiceberg/catalog/__init__.py @@ -26,6 +26,7 @@ List, Optional, Union, + cast, ) from pyiceberg.exceptions import NotInstalledError @@ -121,16 +122,18 @@ def load_catalog(name: str, **properties: Optional[str]) -> Catalog: or if it could not determine the catalog based on the properties """ env = _ENV_CONFIG.get_catalog_config(name) - conf = merge_config(env or {}, properties) + conf: RecursiveDict = merge_config(env or {}, cast(RecursiveDict, properties)) catalog_type: Optional[CatalogType] - if provided_catalog_type := conf.get(TYPE): + provided_catalog_type = conf.get(TYPE) + + if provided_catalog_type and isinstance(provided_catalog_type, str): catalog_type = CatalogType[provided_catalog_type.upper()] - else: + elif not provided_catalog_type: catalog_type = infer_catalog_type(name, conf) if catalog_type: - return AVAILABLE_CATALOGS[catalog_type](name, conf) + return AVAILABLE_CATALOGS[catalog_type](name, cast(dict[str, str], conf)) raise ValueError(f"Could not initialize catalog with the following properties: {properties}") diff --git a/pyiceberg/typedef.py b/pyiceberg/typedef.py index ad62fbb4ad..76add54bdd 100644 --- a/pyiceberg/typedef.py +++ b/pyiceberg/typedef.py @@ -35,4 +35,4 @@ def update(self, *args: Any, **kwargs: Any) -> None: Identifier = Tuple[str, ...] Properties = Dict[str, str] -RecursiveDict = Dict[str, Union[str, "RecursiveDict"]] # type: ignore +RecursiveDict = Dict[str, Union[str, "RecursiveDict"]] diff --git a/pyiceberg/utils/config.py b/pyiceberg/utils/config.py index c13ec13272..65aa81d359 100644 --- a/pyiceberg/utils/config.py +++ b/pyiceberg/utils/config.py @@ -16,7 +16,7 @@ # under the License. import logging import os -from typing import Any, List, Optional +from typing import List, Optional import yaml @@ -31,10 +31,6 @@ logger = logging.getLogger(__name__) -def _coalesce(lhs: Optional[Any], rhs: Optional[Any]) -> Optional[Any]: - return lhs or rhs - - def merge_config(lhs: RecursiveDict, rhs: RecursiveDict) -> RecursiveDict: """merges right-hand side into the left-hand side""" new_config = lhs.copy() @@ -46,7 +42,7 @@ def merge_config(lhs: RecursiveDict, rhs: RecursiveDict) -> RecursiveDict: new_config[rhs_key] = merge_config(lhs_value, rhs_value) else: # Take the non-null value, with precedence on rhs - new_config[rhs_key] = _coalesce(rhs_value, lhs_value) + new_config[rhs_key] = lhs_value or rhs_value else: # New key new_config[rhs_key] = rhs_value diff --git a/pyproject.toml b/pyproject.toml index 7ce563403d..b54f266e8b 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -108,6 +108,7 @@ all = true [tool.mypy] no_implicit_optional = true +namespace_packages = false warn_redundant_casts = true warn_unreachable = true warn_unused_ignores = true diff --git a/tests/avro/test_decoder.py b/tests/avro/test_decoder.py index 73aba1a1be..1d2a9d4c26 100644 --- a/tests/avro/test_decoder.py +++ b/tests/avro/test_decoder.py @@ -107,10 +107,11 @@ def read(self, size: int = 0) -> bytes: return int.to_bytes(1, self.pos, byteorder="little") def seek(self, offset: int, whence: int = SEEK_SET) -> int: - pass + self.pos = offset + return self.pos def tell(self) -> int: - pass + return self.pos @property def closed(self) -> bool: From 1b38491711a892005fa445db55bfb7b0e27cb324 Mon Sep 17 00:00:00 2001 From: Fokko Driesprong Date: Sat, 19 Nov 2022 23:21:28 +0100 Subject: [PATCH 269/642] Python: Remove dataclasses (#6139) --- poetry.lock | 177 +++++---- pyiceberg/expressions/__init__.py | 526 ++++++++++++++++--------- pyiceberg/expressions/literals.py | 99 ++--- pyiceberg/expressions/visitors.py | 126 +++--- pyiceberg/typedef.py | 7 +- pyproject.toml | 1 + tests/expressions/test_expressions.py | 522 +++++++++++++++++-------- tests/expressions/test_literals.py | 41 +- tests/expressions/test_visitors.py | 535 +++++++++++--------------- tests/table/test_init.py | 12 +- 10 files changed, 1144 insertions(+), 902 deletions(-) diff --git a/poetry.lock b/poetry.lock index 2cb2689f57..e5157bda8c 100644 --- a/poetry.lock +++ b/poetry.lock @@ -49,11 +49,11 @@ typing_extensions = {version = ">=4.0", markers = "python_version < \"3.10\""} [[package]] name = "aiosignal" -version = "1.2.0" +version = "1.3.1" description = "aiosignal: a list of registered asynchronous callbacks" category = "main" optional = true -python-versions = ">=3.6" +python-versions = ">=3.7" [package.dependencies] frozenlist = ">=1.1.0" @@ -216,7 +216,7 @@ python-versions = ">=3.7" [[package]] name = "exceptiongroup" -version = "1.0.0" +version = "1.0.4" description = "Backport of PEP 654 (exception groups)" category = "dev" optional = false @@ -253,7 +253,7 @@ testing = ["covdefaults (>=2.2)", "coverage (>=6.4.2)", "pytest (>=7.1.2)", "pyt [[package]] name = "frozenlist" -version = "1.3.1" +version = "1.3.3" description = "A list-like structure which implements collections.abc.MutableSequence" category = "main" optional = true @@ -400,15 +400,15 @@ tomli = {version = ">=1.1.0", markers = "python_version < \"3.11\""} [[package]] name = "platformdirs" -version = "2.5.2" -description = "A small Python module for determining appropriate platform-specific dirs, e.g. a \"user data dir\"." +version = "2.5.4" +description = "A small Python package for determining appropriate platform-specific dirs, e.g. a \"user data dir\"." category = "dev" optional = false python-versions = ">=3.7" [package.extras] -docs = ["furo (>=2021.7.5b38)", "proselint (>=0.10.2)", "sphinx (>=4)", "sphinx-autodoc-typehints (>=1.12)"] -test = ["appdirs (==1.4.4)", "pytest (>=6)", "pytest-cov (>=2.7)", "pytest-mock (>=3.6)"] +docs = ["furo (>=2022.9.29)", "proselint (>=0.13)", "sphinx (>=5.3)", "sphinx-autodoc-typehints (>=1.19.4)"] +test = ["appdirs (==1.4.4)", "pytest (>=7.2)", "pytest-cov (>=4)", "pytest-mock (>=3.10)"] [[package]] name = "pluggy" @@ -627,7 +627,7 @@ boto3 = ["aiobotocore[boto3] (>=2.4.0,<2.5.0)"] [[package]] name = "setuptools" -version = "65.5.0" +version = "65.5.1" description = "Easily download, build, install, upgrade, and uninstall Python packages" category = "dev" optional = false @@ -635,7 +635,7 @@ python-versions = ">=3.7" [package.extras] docs = ["furo", "jaraco.packaging (>=9)", "jaraco.tidelift (>=1.4)", "pygments-github-lexers (==0.0.5)", "rst.linker (>=1.9)", "sphinx (>=3.5)", "sphinx-favicon", "sphinx-hoverxref (<2)", "sphinx-inline-tabs", "sphinx-notfound-page (==0.8.3)", "sphinx-reredirects", "sphinxcontrib-towncrier"] -testing = ["build[virtualenv]", "filelock (>=3.4.0)", "flake8 (<5)", "flake8-2020", "ini2toml[lite] (>=0.9)", "jaraco.envs (>=2.2)", "jaraco.path (>=3.2.0)", "mock", "pip (>=19.1)", "pip-run (>=8.8)", "pytest (>=6)", "pytest-black (>=0.3.7)", "pytest-checkdocs (>=2.4)", "pytest-cov", "pytest-enabler (>=1.3)", "pytest-flake8", "pytest-mypy (>=0.9.1)", "pytest-perf", "pytest-xdist", "tomli-w (>=1.0.0)", "virtualenv (>=13.0.0)", "wheel"] +testing = ["build[virtualenv]", "filelock (>=3.4.0)", "flake8 (<5)", "flake8-2020", "ini2toml[lite] (>=0.9)", "jaraco.envs (>=2.2)", "jaraco.path (>=3.2.0)", "pip (>=19.1)", "pip-run (>=8.8)", "pytest (>=6)", "pytest-black (>=0.3.7)", "pytest-checkdocs (>=2.4)", "pytest-cov", "pytest-enabler (>=1.3)", "pytest-flake8", "pytest-mypy (>=0.9.1)", "pytest-perf", "pytest-timeout", "pytest-xdist", "tomli-w (>=1.0.0)", "virtualenv (>=13.0.0)", "wheel"] testing-integration = ["build[virtualenv]", "filelock (>=3.4.0)", "jaraco.envs (>=2.2)", "jaraco.path (>=3.2.0)", "pytest", "pytest-enabler", "pytest-xdist", "tomli", "virtualenv (>=13.0.0)", "wheel"] [[package]] @@ -701,7 +701,7 @@ socks = ["PySocks (>=1.5.6,!=1.5.7,<2.0)"] [[package]] name = "virtualenv" -version = "20.16.6" +version = "20.16.7" description = "Virtual Python Environment builder" category = "dev" optional = false @@ -771,7 +771,7 @@ snappy = ["python-snappy"] [metadata] lock-version = "1.1" python-versions = "^3.8" -content-hash = "4f90d01e509c4918b7e4ce6da629f5e009165a499cd2f43c25f3ef73ef58af86" +content-hash = "fda6f8ad5abce655e854325c9e0e944c6ad818487479c97078d1dce93496465e" [metadata.files] aiobotocore = [ @@ -872,8 +872,8 @@ aioitertools = [ {file = "aioitertools-0.11.0.tar.gz", hash = "sha256:42c68b8dd3a69c2bf7f2233bf7df4bb58b557bca5252ac02ed5187bbc67d6831"}, ] aiosignal = [ - {file = "aiosignal-1.2.0-py3-none-any.whl", hash = "sha256:26e62109036cd181df6e6ad646f91f0dcfd05fe16d0cb924138ff2ab75d64e3a"}, - {file = "aiosignal-1.2.0.tar.gz", hash = "sha256:78ed67db6c7b7ced4f98e495e572106d5c432a93e1ddd1bf475e1dc05f5b7df2"}, + {file = "aiosignal-1.3.1-py3-none-any.whl", hash = "sha256:f8376fb07dd1e86a584e4fcdec80b36b7f81aac666ebc724e2c090300dd83b17"}, + {file = "aiosignal-1.3.1.tar.gz", hash = "sha256:54cd96e15e1649b75d6c87526a6ff0b6c1b0dd3459f43d9ca11d48c339b68cfc"}, ] async-timeout = [ {file = "async-timeout-4.0.2.tar.gz", hash = "sha256:2163e1640ddb52b7a8c80d0a67a08587e5d245cc9c553a74a847056bc2976b15"}, @@ -1042,8 +1042,8 @@ docutils = [ {file = "docutils-0.19.tar.gz", hash = "sha256:33995a6753c30b7f577febfc2c50411fec6aac7f7ffeb7c4cfe5991072dcf9e6"}, ] exceptiongroup = [ - {file = "exceptiongroup-1.0.0-py3-none-any.whl", hash = "sha256:2ac84b496be68464a2da60da518af3785fff8b7ec0d090a581604bc870bdee41"}, - {file = "exceptiongroup-1.0.0.tar.gz", hash = "sha256:affbabf13fb6e98988c38d9c5650e701569fe3c1de3233cfb61c5f33774690ad"}, + {file = "exceptiongroup-1.0.4-py3-none-any.whl", hash = "sha256:542adf9dea4055530d6e1279602fa5cb11dab2395fa650b8674eaec35fc4a828"}, + {file = "exceptiongroup-1.0.4.tar.gz", hash = "sha256:bd14967b79cd9bdb54d97323216f8fdf533e278df937aa2a90089e7d6e06e5ec"}, ] fastavro = [ {file = "fastavro-1.6.1-cp310-cp310-macosx_10_14_x86_64.whl", hash = "sha256:de244146a3d7bc4e60c7be11f317054ee0e57ca807d85923bd8407981488ff18"}, @@ -1069,65 +1069,80 @@ filelock = [ {file = "filelock-3.8.0.tar.gz", hash = "sha256:55447caa666f2198c5b6b13a26d2084d26fa5b115c00d065664b2124680c4edc"}, ] frozenlist = [ - {file = "frozenlist-1.3.1-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:5f271c93f001748fc26ddea409241312a75e13466b06c94798d1a341cf0e6989"}, - {file = "frozenlist-1.3.1-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:9c6ef8014b842f01f5d2b55315f1af5cbfde284eb184075c189fd657c2fd8204"}, - {file = "frozenlist-1.3.1-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:219a9676e2eae91cb5cc695a78b4cb43d8123e4160441d2b6ce8d2c70c60e2f3"}, - {file = "frozenlist-1.3.1-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:b47d64cdd973aede3dd71a9364742c542587db214e63b7529fbb487ed67cddd9"}, - {file = "frozenlist-1.3.1-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:2af6f7a4e93f5d08ee3f9152bce41a6015b5cf87546cb63872cc19b45476e98a"}, - {file = "frozenlist-1.3.1-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:a718b427ff781c4f4e975525edb092ee2cdef6a9e7bc49e15063b088961806f8"}, - {file = "frozenlist-1.3.1-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:c56c299602c70bc1bb5d1e75f7d8c007ca40c9d7aebaf6e4ba52925d88ef826d"}, - {file = "frozenlist-1.3.1-cp310-cp310-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:717470bfafbb9d9be624da7780c4296aa7935294bd43a075139c3d55659038ca"}, - {file = "frozenlist-1.3.1-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:31b44f1feb3630146cffe56344704b730c33e042ffc78d21f2125a6a91168131"}, - {file = "frozenlist-1.3.1-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:c3b31180b82c519b8926e629bf9f19952c743e089c41380ddca5db556817b221"}, - {file = "frozenlist-1.3.1-cp310-cp310-musllinux_1_1_ppc64le.whl", hash = "sha256:d82bed73544e91fb081ab93e3725e45dd8515c675c0e9926b4e1f420a93a6ab9"}, - {file = "frozenlist-1.3.1-cp310-cp310-musllinux_1_1_s390x.whl", hash = "sha256:49459f193324fbd6413e8e03bd65789e5198a9fa3095e03f3620dee2f2dabff2"}, - {file = "frozenlist-1.3.1-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:94e680aeedc7fd3b892b6fa8395b7b7cc4b344046c065ed4e7a1e390084e8cb5"}, - {file = "frozenlist-1.3.1-cp310-cp310-win32.whl", hash = "sha256:fabb953ab913dadc1ff9dcc3a7a7d3dc6a92efab3a0373989b8063347f8705be"}, - {file = "frozenlist-1.3.1-cp310-cp310-win_amd64.whl", hash = "sha256:eee0c5ecb58296580fc495ac99b003f64f82a74f9576a244d04978a7e97166db"}, - {file = "frozenlist-1.3.1-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:0bc75692fb3770cf2b5856a6c2c9de967ca744863c5e89595df64e252e4b3944"}, - {file = "frozenlist-1.3.1-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:086ca1ac0a40e722d6833d4ce74f5bf1aba2c77cbfdc0cd83722ffea6da52a04"}, - {file = "frozenlist-1.3.1-cp37-cp37m-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:1b51eb355e7f813bcda00276b0114c4172872dc5fb30e3fea059b9367c18fbcb"}, - {file = "frozenlist-1.3.1-cp37-cp37m-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:74140933d45271c1a1283f708c35187f94e1256079b3c43f0c2267f9db5845ff"}, - {file = "frozenlist-1.3.1-cp37-cp37m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:ee4c5120ddf7d4dd1eaf079af3af7102b56d919fa13ad55600a4e0ebe532779b"}, - {file = "frozenlist-1.3.1-cp37-cp37m-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:97d9e00f3ac7c18e685320601f91468ec06c58acc185d18bb8e511f196c8d4b2"}, - {file = "frozenlist-1.3.1-cp37-cp37m-musllinux_1_1_aarch64.whl", hash = "sha256:6e19add867cebfb249b4e7beac382d33215d6d54476bb6be46b01f8cafb4878b"}, - {file = "frozenlist-1.3.1-cp37-cp37m-musllinux_1_1_i686.whl", hash = "sha256:a027f8f723d07c3f21963caa7d585dcc9b089335565dabe9c814b5f70c52705a"}, - {file = "frozenlist-1.3.1-cp37-cp37m-musllinux_1_1_ppc64le.whl", hash = "sha256:61d7857950a3139bce035ad0b0945f839532987dfb4c06cfe160254f4d19df03"}, - {file = "frozenlist-1.3.1-cp37-cp37m-musllinux_1_1_s390x.whl", hash = "sha256:53b2b45052e7149ee8b96067793db8ecc1ae1111f2f96fe1f88ea5ad5fd92d10"}, - {file = "frozenlist-1.3.1-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:bbb1a71b1784e68870800b1bc9f3313918edc63dbb8f29fbd2e767ce5821696c"}, - {file = "frozenlist-1.3.1-cp37-cp37m-win32.whl", hash = "sha256:ab6fa8c7871877810e1b4e9392c187a60611fbf0226a9e0b11b7b92f5ac72792"}, - {file = "frozenlist-1.3.1-cp37-cp37m-win_amd64.whl", hash = "sha256:f89139662cc4e65a4813f4babb9ca9544e42bddb823d2ec434e18dad582543bc"}, - {file = "frozenlist-1.3.1-cp38-cp38-macosx_10_9_universal2.whl", hash = "sha256:4c0c99e31491a1d92cde8648f2e7ccad0e9abb181f6ac3ddb9fc48b63301808e"}, - {file = "frozenlist-1.3.1-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:61e8cb51fba9f1f33887e22488bad1e28dd8325b72425f04517a4d285a04c519"}, - {file = "frozenlist-1.3.1-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:cc2f3e368ee5242a2cbe28323a866656006382872c40869b49b265add546703f"}, - {file = "frozenlist-1.3.1-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:58fb94a01414cddcdc6839807db77ae8057d02ddafc94a42faee6004e46c9ba8"}, - {file = "frozenlist-1.3.1-cp38-cp38-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:022178b277cb9277d7d3b3f2762d294f15e85cd2534047e68a118c2bb0058f3e"}, - {file = "frozenlist-1.3.1-cp38-cp38-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:572ce381e9fe027ad5e055f143763637dcbac2542cfe27f1d688846baeef5170"}, - {file = "frozenlist-1.3.1-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:19127f8dcbc157ccb14c30e6f00392f372ddb64a6ffa7106b26ff2196477ee9f"}, - {file = "frozenlist-1.3.1-cp38-cp38-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:42719a8bd3792744c9b523674b752091a7962d0d2d117f0b417a3eba97d1164b"}, - {file = "frozenlist-1.3.1-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:2743bb63095ef306041c8f8ea22bd6e4d91adabf41887b1ad7886c4c1eb43d5f"}, - {file = "frozenlist-1.3.1-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:fa47319a10e0a076709644a0efbcaab9e91902c8bd8ef74c6adb19d320f69b83"}, - {file = "frozenlist-1.3.1-cp38-cp38-musllinux_1_1_ppc64le.whl", hash = "sha256:52137f0aea43e1993264a5180c467a08a3e372ca9d378244c2d86133f948b26b"}, - {file = "frozenlist-1.3.1-cp38-cp38-musllinux_1_1_s390x.whl", hash = "sha256:f5abc8b4d0c5b556ed8cd41490b606fe99293175a82b98e652c3f2711b452988"}, - {file = "frozenlist-1.3.1-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:1e1cf7bc8cbbe6ce3881863671bac258b7d6bfc3706c600008925fb799a256e2"}, - {file = "frozenlist-1.3.1-cp38-cp38-win32.whl", hash = "sha256:0dde791b9b97f189874d654c55c24bf7b6782343e14909c84beebd28b7217845"}, - {file = "frozenlist-1.3.1-cp38-cp38-win_amd64.whl", hash = "sha256:9494122bf39da6422b0972c4579e248867b6b1b50c9b05df7e04a3f30b9a413d"}, - {file = "frozenlist-1.3.1-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:31bf9539284f39ff9398deabf5561c2b0da5bb475590b4e13dd8b268d7a3c5c1"}, - {file = "frozenlist-1.3.1-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:e0c8c803f2f8db7217898d11657cb6042b9b0553a997c4a0601f48a691480fab"}, - {file = "frozenlist-1.3.1-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:da5ba7b59d954f1f214d352308d1d86994d713b13edd4b24a556bcc43d2ddbc3"}, - {file = "frozenlist-1.3.1-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:74e6b2b456f21fc93ce1aff2b9728049f1464428ee2c9752a4b4f61e98c4db96"}, - {file = "frozenlist-1.3.1-cp39-cp39-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:526d5f20e954d103b1d47232e3839f3453c02077b74203e43407b962ab131e7b"}, - {file = "frozenlist-1.3.1-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:b499c6abe62a7a8d023e2c4b2834fce78a6115856ae95522f2f974139814538c"}, - {file = "frozenlist-1.3.1-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:ab386503f53bbbc64d1ad4b6865bf001414930841a870fc97f1546d4d133f141"}, - {file = "frozenlist-1.3.1-cp39-cp39-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:5f63c308f82a7954bf8263a6e6de0adc67c48a8b484fab18ff87f349af356efd"}, - {file = "frozenlist-1.3.1-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:12607804084d2244a7bd4685c9d0dca5df17a6a926d4f1967aa7978b1028f89f"}, - {file = "frozenlist-1.3.1-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:da1cdfa96425cbe51f8afa43e392366ed0b36ce398f08b60de6b97e3ed4affef"}, - {file = "frozenlist-1.3.1-cp39-cp39-musllinux_1_1_ppc64le.whl", hash = "sha256:f810e764617b0748b49a731ffaa525d9bb36ff38332411704c2400125af859a6"}, - {file = "frozenlist-1.3.1-cp39-cp39-musllinux_1_1_s390x.whl", hash = "sha256:35c3d79b81908579beb1fb4e7fcd802b7b4921f1b66055af2578ff7734711cfa"}, - {file = "frozenlist-1.3.1-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:c92deb5d9acce226a501b77307b3b60b264ca21862bd7d3e0c1f3594022f01bc"}, - {file = "frozenlist-1.3.1-cp39-cp39-win32.whl", hash = "sha256:5e77a8bd41e54b05e4fb2708dc6ce28ee70325f8c6f50f3df86a44ecb1d7a19b"}, - {file = "frozenlist-1.3.1-cp39-cp39-win_amd64.whl", hash = "sha256:625d8472c67f2d96f9a4302a947f92a7adbc1e20bedb6aff8dbc8ff039ca6189"}, - {file = "frozenlist-1.3.1.tar.gz", hash = "sha256:3a735e4211a04ccfa3f4833547acdf5d2f863bfeb01cfd3edaffbc251f15cec8"}, + {file = "frozenlist-1.3.3-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:ff8bf625fe85e119553b5383ba0fb6aa3d0ec2ae980295aaefa552374926b3f4"}, + {file = "frozenlist-1.3.3-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:dfbac4c2dfcc082fcf8d942d1e49b6aa0766c19d3358bd86e2000bf0fa4a9cf0"}, + {file = "frozenlist-1.3.3-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:b1c63e8d377d039ac769cd0926558bb7068a1f7abb0f003e3717ee003ad85530"}, + {file = "frozenlist-1.3.3-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:7fdfc24dcfce5b48109867c13b4cb15e4660e7bd7661741a391f821f23dfdca7"}, + {file = "frozenlist-1.3.3-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:2c926450857408e42f0bbc295e84395722ce74bae69a3b2aa2a65fe22cb14b99"}, + {file = "frozenlist-1.3.3-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:1841e200fdafc3d51f974d9d377c079a0694a8f06de2e67b48150328d66d5483"}, + {file = "frozenlist-1.3.3-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:f470c92737afa7d4c3aacc001e335062d582053d4dbe73cda126f2d7031068dd"}, + {file = "frozenlist-1.3.3-cp310-cp310-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:783263a4eaad7c49983fe4b2e7b53fa9770c136c270d2d4bbb6d2192bf4d9caf"}, + {file = "frozenlist-1.3.3-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:924620eef691990dfb56dc4709f280f40baee568c794b5c1885800c3ecc69816"}, + {file = "frozenlist-1.3.3-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:ae4dc05c465a08a866b7a1baf360747078b362e6a6dbeb0c57f234db0ef88ae0"}, + {file = "frozenlist-1.3.3-cp310-cp310-musllinux_1_1_ppc64le.whl", hash = "sha256:bed331fe18f58d844d39ceb398b77d6ac0b010d571cba8267c2e7165806b00ce"}, + {file = "frozenlist-1.3.3-cp310-cp310-musllinux_1_1_s390x.whl", hash = "sha256:02c9ac843e3390826a265e331105efeab489ffaf4dd86384595ee8ce6d35ae7f"}, + {file = "frozenlist-1.3.3-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:9545a33965d0d377b0bc823dcabf26980e77f1b6a7caa368a365a9497fb09420"}, + {file = "frozenlist-1.3.3-cp310-cp310-win32.whl", hash = "sha256:d5cd3ab21acbdb414bb6c31958d7b06b85eeb40f66463c264a9b343a4e238642"}, + {file = "frozenlist-1.3.3-cp310-cp310-win_amd64.whl", hash = "sha256:b756072364347cb6aa5b60f9bc18e94b2f79632de3b0190253ad770c5df17db1"}, + {file = "frozenlist-1.3.3-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:b4395e2f8d83fbe0c627b2b696acce67868793d7d9750e90e39592b3626691b7"}, + {file = "frozenlist-1.3.3-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:14143ae966a6229350021384870458e4777d1eae4c28d1a7aa47f24d030e6678"}, + {file = "frozenlist-1.3.3-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:5d8860749e813a6f65bad8285a0520607c9500caa23fea6ee407e63debcdbef6"}, + {file = "frozenlist-1.3.3-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:23d16d9f477bb55b6154654e0e74557040575d9d19fe78a161bd33d7d76808e8"}, + {file = "frozenlist-1.3.3-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:eb82dbba47a8318e75f679690190c10a5e1f447fbf9df41cbc4c3afd726d88cb"}, + {file = "frozenlist-1.3.3-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:9309869032abb23d196cb4e4db574232abe8b8be1339026f489eeb34a4acfd91"}, + {file = "frozenlist-1.3.3-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:a97b4fe50b5890d36300820abd305694cb865ddb7885049587a5678215782a6b"}, + {file = "frozenlist-1.3.3-cp311-cp311-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:c188512b43542b1e91cadc3c6c915a82a5eb95929134faf7fd109f14f9892ce4"}, + {file = "frozenlist-1.3.3-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:303e04d422e9b911a09ad499b0368dc551e8c3cd15293c99160c7f1f07b59a48"}, + {file = "frozenlist-1.3.3-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:0771aed7f596c7d73444c847a1c16288937ef988dc04fb9f7be4b2aa91db609d"}, + {file = "frozenlist-1.3.3-cp311-cp311-musllinux_1_1_ppc64le.whl", hash = "sha256:66080ec69883597e4d026f2f71a231a1ee9887835902dbe6b6467d5a89216cf6"}, + {file = "frozenlist-1.3.3-cp311-cp311-musllinux_1_1_s390x.whl", hash = "sha256:41fe21dc74ad3a779c3d73a2786bdf622ea81234bdd4faf90b8b03cad0c2c0b4"}, + {file = "frozenlist-1.3.3-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:f20380df709d91525e4bee04746ba612a4df0972c1b8f8e1e8af997e678c7b81"}, + {file = "frozenlist-1.3.3-cp311-cp311-win32.whl", hash = "sha256:f30f1928162e189091cf4d9da2eac617bfe78ef907a761614ff577ef4edfb3c8"}, + {file = "frozenlist-1.3.3-cp311-cp311-win_amd64.whl", hash = "sha256:a6394d7dadd3cfe3f4b3b186e54d5d8504d44f2d58dcc89d693698e8b7132b32"}, + {file = "frozenlist-1.3.3-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:8df3de3a9ab8325f94f646609a66cbeeede263910c5c0de0101079ad541af332"}, + {file = "frozenlist-1.3.3-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:0693c609e9742c66ba4870bcee1ad5ff35462d5ffec18710b4ac89337ff16e27"}, + {file = "frozenlist-1.3.3-cp37-cp37m-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:cd4210baef299717db0a600d7a3cac81d46ef0e007f88c9335db79f8979c0d3d"}, + {file = "frozenlist-1.3.3-cp37-cp37m-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:394c9c242113bfb4b9aa36e2b80a05ffa163a30691c7b5a29eba82e937895d5e"}, + {file = "frozenlist-1.3.3-cp37-cp37m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:6327eb8e419f7d9c38f333cde41b9ae348bec26d840927332f17e887a8dcb70d"}, + {file = "frozenlist-1.3.3-cp37-cp37m-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:2e24900aa13212e75e5b366cb9065e78bbf3893d4baab6052d1aca10d46d944c"}, + {file = "frozenlist-1.3.3-cp37-cp37m-musllinux_1_1_aarch64.whl", hash = "sha256:3843f84a6c465a36559161e6c59dce2f2ac10943040c2fd021cfb70d58c4ad56"}, + {file = "frozenlist-1.3.3-cp37-cp37m-musllinux_1_1_i686.whl", hash = "sha256:84610c1502b2461255b4c9b7d5e9c48052601a8957cd0aea6ec7a7a1e1fb9420"}, + {file = "frozenlist-1.3.3-cp37-cp37m-musllinux_1_1_ppc64le.whl", hash = "sha256:c21b9aa40e08e4f63a2f92ff3748e6b6c84d717d033c7b3438dd3123ee18f70e"}, + {file = "frozenlist-1.3.3-cp37-cp37m-musllinux_1_1_s390x.whl", hash = "sha256:efce6ae830831ab6a22b9b4091d411698145cb9b8fc869e1397ccf4b4b6455cb"}, + {file = "frozenlist-1.3.3-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:40de71985e9042ca00b7953c4f41eabc3dc514a2d1ff534027f091bc74416401"}, + {file = "frozenlist-1.3.3-cp37-cp37m-win32.whl", hash = "sha256:180c00c66bde6146a860cbb81b54ee0df350d2daf13ca85b275123bbf85de18a"}, + {file = "frozenlist-1.3.3-cp37-cp37m-win_amd64.whl", hash = "sha256:9bbbcedd75acdfecf2159663b87f1bb5cfc80e7cd99f7ddd9d66eb98b14a8411"}, + {file = "frozenlist-1.3.3-cp38-cp38-macosx_10_9_universal2.whl", hash = "sha256:034a5c08d36649591be1cbb10e09da9f531034acfe29275fc5454a3b101ce41a"}, + {file = "frozenlist-1.3.3-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:ba64dc2b3b7b158c6660d49cdb1d872d1d0bf4e42043ad8d5006099479a194e5"}, + {file = "frozenlist-1.3.3-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:47df36a9fe24054b950bbc2db630d508cca3aa27ed0566c0baf661225e52c18e"}, + {file = "frozenlist-1.3.3-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:008a054b75d77c995ea26629ab3a0c0d7281341f2fa7e1e85fa6153ae29ae99c"}, + {file = "frozenlist-1.3.3-cp38-cp38-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:841ea19b43d438a80b4de62ac6ab21cfe6827bb8a9dc62b896acc88eaf9cecba"}, + {file = "frozenlist-1.3.3-cp38-cp38-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:e235688f42b36be2b6b06fc37ac2126a73b75fb8d6bc66dd632aa35286238703"}, + {file = "frozenlist-1.3.3-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:ca713d4af15bae6e5d79b15c10c8522859a9a89d3b361a50b817c98c2fb402a2"}, + {file = "frozenlist-1.3.3-cp38-cp38-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:9ac5995f2b408017b0be26d4a1d7c61bce106ff3d9e3324374d66b5964325448"}, + {file = "frozenlist-1.3.3-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:a4ae8135b11652b08a8baf07631d3ebfe65a4c87909dbef5fa0cdde440444ee4"}, + {file = "frozenlist-1.3.3-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:4ea42116ceb6bb16dbb7d526e242cb6747b08b7710d9782aa3d6732bd8d27649"}, + {file = "frozenlist-1.3.3-cp38-cp38-musllinux_1_1_ppc64le.whl", hash = "sha256:810860bb4bdce7557bc0febb84bbd88198b9dbc2022d8eebe5b3590b2ad6c842"}, + {file = "frozenlist-1.3.3-cp38-cp38-musllinux_1_1_s390x.whl", hash = "sha256:ee78feb9d293c323b59a6f2dd441b63339a30edf35abcb51187d2fc26e696d13"}, + {file = "frozenlist-1.3.3-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:0af2e7c87d35b38732e810befb9d797a99279cbb85374d42ea61c1e9d23094b3"}, + {file = "frozenlist-1.3.3-cp38-cp38-win32.whl", hash = "sha256:899c5e1928eec13fd6f6d8dc51be23f0d09c5281e40d9cf4273d188d9feeaf9b"}, + {file = "frozenlist-1.3.3-cp38-cp38-win_amd64.whl", hash = "sha256:7f44e24fa70f6fbc74aeec3e971f60a14dde85da364aa87f15d1be94ae75aeef"}, + {file = "frozenlist-1.3.3-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:2b07ae0c1edaa0a36339ec6cce700f51b14a3fc6545fdd32930d2c83917332cf"}, + {file = "frozenlist-1.3.3-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:ebb86518203e12e96af765ee89034a1dbb0c3c65052d1b0c19bbbd6af8a145e1"}, + {file = "frozenlist-1.3.3-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:5cf820485f1b4c91e0417ea0afd41ce5cf5965011b3c22c400f6d144296ccbc0"}, + {file = "frozenlist-1.3.3-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:5c11e43016b9024240212d2a65043b70ed8dfd3b52678a1271972702d990ac6d"}, + {file = "frozenlist-1.3.3-cp39-cp39-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:8fa3c6e3305aa1146b59a09b32b2e04074945ffcfb2f0931836d103a2c38f936"}, + {file = "frozenlist-1.3.3-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:352bd4c8c72d508778cf05ab491f6ef36149f4d0cb3c56b1b4302852255d05d5"}, + {file = "frozenlist-1.3.3-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:65a5e4d3aa679610ac6e3569e865425b23b372277f89b5ef06cf2cdaf1ebf22b"}, + {file = "frozenlist-1.3.3-cp39-cp39-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:b1e2c1185858d7e10ff045c496bbf90ae752c28b365fef2c09cf0fa309291669"}, + {file = "frozenlist-1.3.3-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:f163d2fd041c630fed01bc48d28c3ed4a3b003c00acd396900e11ee5316b56bb"}, + {file = "frozenlist-1.3.3-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:05cdb16d09a0832eedf770cb7bd1fe57d8cf4eaf5aced29c4e41e3f20b30a784"}, + {file = "frozenlist-1.3.3-cp39-cp39-musllinux_1_1_ppc64le.whl", hash = "sha256:8bae29d60768bfa8fb92244b74502b18fae55a80eac13c88eb0b496d4268fd2d"}, + {file = "frozenlist-1.3.3-cp39-cp39-musllinux_1_1_s390x.whl", hash = "sha256:eedab4c310c0299961ac285591acd53dc6723a1ebd90a57207c71f6e0c2153ab"}, + {file = "frozenlist-1.3.3-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:3bbdf44855ed8f0fbcd102ef05ec3012d6a4fd7c7562403f76ce6a52aeffb2b1"}, + {file = "frozenlist-1.3.3-cp39-cp39-win32.whl", hash = "sha256:efa568b885bca461f7c7b9e032655c0c143d305bf01c30caf6db2854a4532b38"}, + {file = "frozenlist-1.3.3-cp39-cp39-win_amd64.whl", hash = "sha256:cfe33efc9cb900a4c46f91a5ceba26d6df370ffddd9ca386eb1d4f0ad97b9ea9"}, + {file = "frozenlist-1.3.3.tar.gz", hash = "sha256:58bcc55721e8a90b88332d6cd441261ebb22342e238296bb330968952fbb3a6a"}, ] fsspec = [ {file = "fsspec-2022.10.0-py3-none-any.whl", hash = "sha256:6b7c6ab3b476cdf17efcfeccde7fca28ef5a48f73a71010aaceec5fc15bf9ebf"}, @@ -1293,8 +1308,8 @@ pep517 = [ {file = "pep517-0.13.0.tar.gz", hash = "sha256:ae69927c5c172be1add9203726d4b84cf3ebad1edcd5f71fcdc746e66e829f59"}, ] platformdirs = [ - {file = "platformdirs-2.5.2-py3-none-any.whl", hash = "sha256:027d8e83a2d7de06bbac4e5ef7e023c02b863d7ea5d079477e722bb41ab25788"}, - {file = "platformdirs-2.5.2.tar.gz", hash = "sha256:58c8abb07dcb441e6ee4b11d8df0ac856038f944ab98b7be6b27b2a3c7feef19"}, + {file = "platformdirs-2.5.4-py3-none-any.whl", hash = "sha256:af0276409f9a02373d540bf8480021a048711d572745aef4b7842dad245eba10"}, + {file = "platformdirs-2.5.4.tar.gz", hash = "sha256:1006647646d80f16130f052404c6b901e80ee4ed6bef6792e1f238a8969106f7"}, ] pluggy = [ {file = "pluggy-1.0.0-py2.py3-none-any.whl", hash = "sha256:74134bbf457f031a36d68416e1509f34bd5ccc019f0bcc952c7b909d06b37bd3"}, @@ -1499,8 +1514,8 @@ s3fs = [ {file = "s3fs-2022.10.0.tar.gz", hash = "sha256:e8deb80f20bd0b2059141b874fdb9d6aeb8cce35312ea5f2c02b225a78a00406"}, ] setuptools = [ - {file = "setuptools-65.5.0-py3-none-any.whl", hash = "sha256:f62ea9da9ed6289bfe868cd6845968a2c854d1427f8548d52cae02a42b4f0356"}, - {file = "setuptools-65.5.0.tar.gz", hash = "sha256:512e5536220e38146176efb833d4a62aa726b7bbff82cfbc8ba9eaa3996e0b17"}, + {file = "setuptools-65.5.1-py3-none-any.whl", hash = "sha256:d0b9a8433464d5800cbe05094acf5c6d52a91bfac9b52bcfc4d41382be5d5d31"}, + {file = "setuptools-65.5.1.tar.gz", hash = "sha256:e197a19aa8ec9722928f2206f8de752def0e4c9fc6953527360d1c36d94ddb2f"}, ] six = [ {file = "six-1.16.0-py2.py3-none-any.whl", hash = "sha256:8abb2f1d86890a2dfb989f9a77cfcfd3e47c2a354b01111771326f8aa26e0254"}, @@ -1526,8 +1541,8 @@ urllib3 = [ {file = "urllib3-1.26.12.tar.gz", hash = "sha256:3fa96cf423e6987997fc326ae8df396db2a8b7c667747d47ddd8ecba91f4a74e"}, ] virtualenv = [ - {file = "virtualenv-20.16.6-py3-none-any.whl", hash = "sha256:186ca84254abcbde98180fd17092f9628c5fe742273c02724972a1d8a2035108"}, - {file = "virtualenv-20.16.6.tar.gz", hash = "sha256:530b850b523c6449406dfba859d6345e48ef19b8439606c5d74d7d3c9e14d76e"}, + {file = "virtualenv-20.16.7-py3-none-any.whl", hash = "sha256:efd66b00386fdb7dbe4822d172303f40cd05e50e01740b19ea42425cbe653e29"}, + {file = "virtualenv-20.16.7.tar.gz", hash = "sha256:8691e3ff9387f743e00f6bb20f70121f5e4f596cae754531f2b3b3a1b1ac696e"}, ] wrapt = [ {file = "wrapt-1.14.1-cp27-cp27m-macosx_10_9_x86_64.whl", hash = "sha256:1b376b3f4896e7930f1f772ac4b064ac12598d1c38d04907e696cc4d794b43d3"}, diff --git a/pyiceberg/expressions/__init__.py b/pyiceberg/expressions/__init__.py index 39d4884a51..d6ae395c17 100644 --- a/pyiceberg/expressions/__init__.py +++ b/pyiceberg/expressions/__init__.py @@ -18,18 +18,37 @@ from __future__ import annotations from abc import ABC, abstractmethod -from dataclasses import dataclass from functools import reduce -from typing import ClassVar, Generic, TypeVar - -from pyiceberg.expressions.literals import Literal +from typing import ( + Any, + Generic, + Iterable, + Set, + Type, + Union, +) + +from pyiceberg.expressions.literals import Literal, literal from pyiceberg.files import StructProtocol from pyiceberg.schema import Accessor, Schema +from pyiceberg.typedef import L from pyiceberg.types import DoubleType, FloatType, NestedField from pyiceberg.utils.singleton import Singleton -T = TypeVar("T") -B = TypeVar("B") + +def _to_unbound_term(term: Union[str, UnboundTerm]) -> UnboundTerm: + return Reference(term) if isinstance(term, str) else term + + +def _to_literal_set(values: Union[Iterable[L], Iterable[Literal[L]]]) -> Set[Literal[L]]: + return {_to_literal(v) for v in values} + + +def _to_literal(value: Union[L, Literal[L]]) -> Literal[L]: + if isinstance(value, Literal): + return value + else: + return literal(value) class BooleanExpression(ABC): @@ -40,7 +59,7 @@ def __invert__(self) -> BooleanExpression: """Transform the Expression into its negated version.""" -class Term(Generic[T], ABC): +class Term(Generic[L], ABC): """A simple expression that evaluates to a value""" @@ -48,28 +67,28 @@ class Bound(ABC): """Represents a bound value expression""" -class Unbound(Generic[B], ABC): +class Unbound(ABC): """Represents an unbound value expression""" + @property @abstractmethod - def bind(self, schema: Schema, case_sensitive: bool = True) -> B: - ... # pragma: no cover + def as_bound(self) -> Type[Bound]: + ... -class BoundTerm(Term[T], Bound, ABC): +class BoundTerm(Term[L], Bound, ABC): """Represents a bound term""" @abstractmethod - def ref(self) -> BoundReference[T]: + def ref(self) -> BoundReference[L]: """Returns the bound reference""" @abstractmethod - def eval(self, struct: StructProtocol) -> T: # pylint: disable=W0613 + def eval(self, struct: StructProtocol) -> L: # pylint: disable=W0613 """Returns the value at the referenced field's position in an object that abides by the StructProtocol""" -@dataclass(frozen=True) -class BoundReference(BoundTerm[T]): +class BoundReference(BoundTerm[L]): """A reference bound to a field in a schema Args: @@ -80,7 +99,11 @@ class BoundReference(BoundTerm[T]): field: NestedField accessor: Accessor - def eval(self, struct: StructProtocol) -> T: + def __init__(self, field: NestedField, accessor: Accessor): + self.field = field + self.accessor = accessor + + def eval(self, struct: StructProtocol) -> L: """Returns the value at the referenced field's position in an object that abides by the StructProtocol Args: @@ -90,16 +113,25 @@ def eval(self, struct: StructProtocol) -> T: """ return self.accessor.get(struct) - def ref(self) -> BoundReference[T]: + def __eq__(self, other): + return self.field == other.field if isinstance(other, BoundReference) else False + + def __repr__(self) -> str: + return f"BoundReference(field={repr(self.field)}, accessor={repr(self.accessor)})" + + def ref(self) -> BoundReference[L]: return self -class UnboundTerm(Term[T], Unbound[BoundTerm[T]], ABC): +class UnboundTerm(Term, Unbound, ABC): """Represents an unbound term.""" + @abstractmethod + def bind(self, schema: Schema, case_sensitive: bool = True) -> BoundTerm: + ... # pragma: no cover + -@dataclass(frozen=True) -class Reference(UnboundTerm[T]): +class Reference(UnboundTerm): """A reference not yet bound to a field in a schema Args: @@ -111,7 +143,16 @@ class Reference(UnboundTerm[T]): name: str - def bind(self, schema: Schema, case_sensitive: bool = True) -> BoundReference[T]: + def __init__(self, name: str): + self.name = name + + def __repr__(self): + return f"Reference(name={repr(self.name)})" + + def __eq__(self, other): + return self.name == other.name if isinstance(other, Reference) else False + + def bind(self, schema: Schema, case_sensitive: bool = True) -> BoundReference[L]: """Bind the reference to an Iceberg schema Args: @@ -126,10 +167,13 @@ def bind(self, schema: Schema, case_sensitive: bool = True) -> BoundReference[T] """ field = schema.find_field(name_or_id=self.name, case_sensitive=case_sensitive) accessor = schema.accessor_for_field(field.field_id) - return BoundReference(field=field, accessor=accessor) + return self.as_bound(field=field, accessor=accessor) + + @property + def as_bound(self) -> Type[BoundReference]: + return BoundReference[L] -@dataclass(frozen=True, init=False) class And(BooleanExpression): """AND operation expression - logical conjunction""" @@ -146,16 +190,24 @@ def __new__(cls, left: BooleanExpression, right: BooleanExpression, *rest: Boole elif right is AlwaysTrue(): return left else: - result = super().__new__(cls) - object.__setattr__(result, "left", left) - object.__setattr__(result, "right", right) - return result + obj = super().__new__(cls) + obj.left = left + obj.right = right + return obj + + def __eq__(self, other: Any) -> bool: + return self.left == other.left and self.right == other.right if isinstance(other, And) else False + + def __str__(self) -> str: + return f"And(left={str(self.left)}, right={str(self.right)})" + + def __repr__(self) -> str: + return f"And(left={repr(self.left)}, right={repr(self.right)})" def __invert__(self) -> Or: return Or(~self.left, ~self.right) -@dataclass(frozen=True, init=False) class Or(BooleanExpression): """OR operation expression - logical disjunction""" @@ -172,16 +224,21 @@ def __new__(cls, left: BooleanExpression, right: BooleanExpression, *rest: Boole elif right is AlwaysFalse(): return left else: - result = super().__new__(cls) - object.__setattr__(result, "left", left) - object.__setattr__(result, "right", right) - return result + obj = super().__new__(cls) + obj.left = left + obj.right = right + return obj + + def __eq__(self, other: Any) -> bool: + return self.left == other.left and self.right == other.right if isinstance(other, Or) else False + + def __repr__(self) -> str: + return f"Or(left={repr(self.left)}, right={repr(self.right)})" def __invert__(self) -> And: return And(~self.left, ~self.right) -@dataclass(frozen=True, init=False) class Not(BooleanExpression): """NOT operation expression - logical negation""" @@ -194,85 +251,108 @@ def __new__(cls, child: BooleanExpression): return AlwaysTrue() elif isinstance(child, Not): return child.child - result = super().__new__(cls) - object.__setattr__(result, "child", child) - return result + obj = super().__new__(cls) + obj.child = child + return obj + + def __repr__(self) -> str: + return f"Not(child={repr(self.child)})" + + def __eq__(self, other: Any) -> bool: + return self.child == other.child if isinstance(other, Not) else False def __invert__(self) -> BooleanExpression: return self.child -@dataclass(frozen=True) class AlwaysTrue(BooleanExpression, Singleton): """TRUE expression""" def __invert__(self) -> AlwaysFalse: return AlwaysFalse() + def __str__(self): + return "AlwaysTrue()" + + def __repr__(self): + return "AlwaysTrue()" + -@dataclass(frozen=True) class AlwaysFalse(BooleanExpression, Singleton): """FALSE expression""" def __invert__(self) -> AlwaysTrue: return AlwaysTrue() + def __str__(self): + return "AlwaysFalse()" + + def __repr__(self): + return "AlwaysFalse()" + -@dataclass(frozen=True) -class BoundPredicate(Generic[T], Bound, BooleanExpression): - term: BoundTerm[T] +class BoundPredicate(Generic[L], Bound, BooleanExpression, ABC): + term: BoundTerm[L] - def __invert__(self) -> BoundPredicate[T]: - """Inverts the predicate""" - raise NotImplementedError + def __init__(self, term: BoundTerm[L]): + self.term = term + def __eq__(self, other: Any) -> bool: + if isinstance(other, BoundPredicate): + return self.term == other.term + return False -@dataclass(frozen=True) -class UnboundPredicate(Generic[T], Unbound[BooleanExpression], BooleanExpression): - as_bound: ClassVar[type] - term: UnboundTerm[T] - def __invert__(self) -> UnboundPredicate[T]: - """Inverts the predicate""" - raise NotImplementedError +class UnboundPredicate(Generic[L], Unbound, BooleanExpression, ABC): + term: UnboundTerm - def bind(self, schema: Schema, case_sensitive: bool = True) -> BooleanExpression: - """Binds the predicate to a schema""" - raise NotImplementedError + def __init__(self, term: Union[str, UnboundTerm]): + self.term = _to_unbound_term(term) + def __eq__(self, other): + return self.term == other.term if isinstance(other, UnboundPredicate) else False -@dataclass(frozen=True) -class UnaryPredicate(UnboundPredicate[T]): - def bind(self, schema: Schema, case_sensitive: bool = True) -> BooleanExpression: + @abstractmethod + def bind(self, schema: Schema, case_sensitive: bool = True) -> BoundPredicate: + ... + + @property + @abstractmethod + def as_bound(self) -> Type[BoundPredicate]: + ... + + +class UnaryPredicate(UnboundPredicate[Any], ABC): + def bind(self, schema: Schema, case_sensitive: bool = True) -> BoundUnaryPredicate: bound_term = self.term.bind(schema, case_sensitive) return self.as_bound(bound_term) - def __invert__(self) -> UnaryPredicate[T]: - """Inverts the unary predicate""" - raise NotImplementedError + def __repr__(self) -> str: + return f"{str(self.__class__.__name__)}(term={repr(self.term)})" + + @property + @abstractmethod + def as_bound(self) -> Type[BoundUnaryPredicate[L]]: + ... -@dataclass(frozen=True) -class BoundUnaryPredicate(BoundPredicate[T]): - def __invert__(self) -> BoundUnaryPredicate[T]: - """Inverts the unary predicate""" - raise NotImplementedError +class BoundUnaryPredicate(BoundPredicate[L], ABC): + def __repr__(self) -> str: + return f"{str(self.__class__.__name__)}(term={repr(self.term)})" -@dataclass(frozen=True) -class BoundIsNull(BoundUnaryPredicate[T]): - def __new__(cls, term: BoundTerm[T]): # pylint: disable=W0221 +class BoundIsNull(BoundUnaryPredicate[L]): + def __new__(cls, term: BoundTerm[L]): # pylint: disable=W0221 if term.ref().field.required: return AlwaysFalse() return super().__new__(cls) - def __invert__(self) -> BoundNotNull[T]: + def __invert__(self) -> BoundNotNull[L]: return BoundNotNull(self.term) -@dataclass(frozen=True) -class BoundNotNull(BoundUnaryPredicate[T]): - def __new__(cls, term: BoundTerm[T]): # pylint: disable=W0221 +class BoundNotNull(BoundUnaryPredicate[L]): + def __new__(cls, term: BoundTerm[L]): # pylint: disable=W0221 if term.ref().field.required: return AlwaysTrue() return super().__new__(cls) @@ -281,87 +361,114 @@ def __invert__(self) -> BoundIsNull: return BoundIsNull(self.term) -@dataclass(frozen=True) -class IsNull(UnaryPredicate[T]): - as_bound = BoundIsNull - - def __invert__(self) -> NotNull[T]: +class IsNull(UnaryPredicate): + def __invert__(self) -> NotNull: return NotNull(self.term) + @property + def as_bound(self) -> Type[BoundIsNull[L]]: + return BoundIsNull[L] -@dataclass(frozen=True) -class NotNull(UnaryPredicate[T]): - as_bound = BoundNotNull - def __invert__(self) -> IsNull[T]: +class NotNull(UnaryPredicate): + def __invert__(self) -> IsNull: return IsNull(self.term) + @property + def as_bound(self) -> Type[BoundNotNull[L]]: + return BoundNotNull[L] + -@dataclass(frozen=True) -class BoundIsNaN(BoundUnaryPredicate[T]): - def __new__(cls, term: BoundTerm[T]): # pylint: disable=W0221 +class BoundIsNaN(BoundUnaryPredicate[L]): + def __new__(cls, term: BoundTerm[L]): # pylint: disable=W0221 bound_type = term.ref().field.field_type if type(bound_type) in {FloatType, DoubleType}: return super().__new__(cls) return AlwaysFalse() - def __invert__(self) -> BoundNotNaN[T]: + def __invert__(self) -> BoundNotNaN[L]: return BoundNotNaN(self.term) -@dataclass(frozen=True) -class BoundNotNaN(BoundUnaryPredicate[T]): - def __new__(cls, term: BoundTerm[T]): # pylint: disable=W0221 +class BoundNotNaN(BoundUnaryPredicate[L]): + def __new__(cls, term: BoundTerm[L]): # pylint: disable=W0221 bound_type = term.ref().field.field_type if type(bound_type) in {FloatType, DoubleType}: return super().__new__(cls) return AlwaysTrue() - def __invert__(self) -> BoundIsNaN[T]: + def __invert__(self) -> BoundIsNaN[L]: return BoundIsNaN(self.term) -@dataclass(frozen=True) -class IsNaN(UnaryPredicate[T]): - as_bound = BoundIsNaN - - def __invert__(self) -> NotNaN[T]: +class IsNaN(UnaryPredicate): + def __invert__(self) -> NotNaN: return NotNaN(self.term) + @property + def as_bound(self) -> Type[BoundIsNaN[L]]: + return BoundIsNaN[L] -@dataclass(frozen=True) -class NotNaN(UnaryPredicate[T]): - as_bound = BoundNotNaN - def __invert__(self) -> IsNaN[T]: +class NotNaN(UnaryPredicate): + def __invert__(self) -> IsNaN: return IsNaN(self.term) + @property + def as_bound(self) -> Type[BoundNotNaN[L]]: + return BoundNotNaN[L] + -@dataclass(frozen=True) -class SetPredicate(UnboundPredicate[T]): - literals: tuple[Literal[T], ...] +class SetPredicate(Generic[L], UnboundPredicate[L], ABC): + literals: Set[Literal[L]] - def __invert__(self) -> SetPredicate[T]: - """Inverted expression of the SetPredicate""" - raise NotImplementedError + def __init__(self, term: Union[str, UnboundTerm], literals: Union[Iterable[L], Iterable[Literal[L]]]): + super().__init__(term) + self.literals = _to_literal_set(literals) - def bind(self, schema: Schema, case_sensitive: bool = True) -> BooleanExpression: + def bind(self, schema: Schema, case_sensitive: bool = True) -> BoundSetPredicate[L]: bound_term = self.term.bind(schema, case_sensitive) return self.as_bound(bound_term, {lit.to(bound_term.ref().field.field_type) for lit in self.literals}) + def __str__(self): + # Sort to make it deterministic + return f"{str(self.__class__.__name__)}({str(self.term)}, {{{', '.join(sorted([str(literal) for literal in self.literals]))}}})" + + def __repr__(self) -> str: + # Sort to make it deterministic + return f"{str(self.__class__.__name__)}({repr(self.term)}, {{{', '.join(sorted([repr(literal) for literal in self.literals]))}}})" + + def __eq__(self, other: Any) -> bool: + return self.term == other.term and self.literals == other.literals if isinstance(other, SetPredicate) else False + + @property + @abstractmethod + def as_bound(self) -> Type[BoundSetPredicate[L]]: + return BoundSetPredicate[L] + + +class BoundSetPredicate(BoundPredicate[L], ABC): + literals: Set[Literal[L]] + + def __init__(self, term: BoundTerm[L], literals: Set[Literal[L]]): + # Since we don't know the type of BoundPredicate[L], we have to ignore this one + super().__init__(term) # type: ignore + self.literals = _to_literal_set(literals) # pylint: disable=W0621 -@dataclass(frozen=True) -class BoundSetPredicate(BoundPredicate[T]): - literals: set[Literal[T]] + def __str__(self): + # Sort to make it deterministic + return f"{str(self.__class__.__name__)}({str(self.term)}, {{{', '.join(sorted([str(literal) for literal in self.literals]))}}})" - def __invert__(self) -> BoundSetPredicate[T]: - """Inverted expression of the SetPredicate""" - raise NotImplementedError + def __repr__(self) -> str: + # Sort to make it deterministic + return f"{str(self.__class__.__name__)}({repr(self.term)}, {{{', '.join(sorted([repr(literal) for literal in self.literals]))}}})" + def __eq__(self, other: Any) -> bool: + return self.term == other.term and self.literals == other.literals if isinstance(other, BoundSetPredicate) else False -@dataclass(frozen=True) -class BoundIn(BoundSetPredicate[T]): - def __new__(cls, term: BoundTerm[T], literals: set[Literal[T]]): # pylint: disable=W0221 + +class BoundIn(BoundSetPredicate[L]): + def __new__(cls, term: BoundTerm[L], literals: Set[Literal[L]]): # pylint: disable=W0221 count = len(literals) if count == 0: return AlwaysFalse() @@ -370,13 +477,19 @@ def __new__(cls, term: BoundTerm[T], literals: set[Literal[T]]): # pylint: disa else: return super().__new__(cls) - def __invert__(self) -> BoundNotIn[T]: + def __invert__(self) -> BoundNotIn[L]: return BoundNotIn(self.term, self.literals) + def __eq__(self, other: Any) -> bool: + return self.term == other.term and self.literals == other.literals if isinstance(other, BoundIn) else False + -@dataclass(frozen=True) -class BoundNotIn(BoundSetPredicate[T]): - def __new__(cls, term: BoundTerm[T], literals: set[Literal[T]]): # pylint: disable=W0221 +class BoundNotIn(BoundSetPredicate[L]): + def __new__( # pylint: disable=W0221 + cls, + term: BoundTerm[L], + literals: Set[Literal[L]], + ): count = len(literals) if count == 0: return AlwaysTrue() @@ -385,145 +498,174 @@ def __new__(cls, term: BoundTerm[T], literals: set[Literal[T]]): # pylint: disa else: return super().__new__(cls) - def __invert__(self) -> BoundIn[T]: + def __invert__(self) -> BoundIn[L]: return BoundIn(self.term, self.literals) -@dataclass(frozen=True) -class In(SetPredicate[T]): - as_bound = BoundIn - - def __new__(cls, term: UnboundTerm[T], literals: tuple[Literal[T], ...]): # pylint: disable=W0221 - count = len(literals) +class In(SetPredicate[L]): + def __new__(cls, term: Union[str, UnboundTerm], literals: Union[Iterable[L], Iterable[Literal[L]]]): # pylint: disable=W0221 + literals_set: Set[Literal[L]] = _to_literal_set(literals) + count = len(literals_set) if count == 0: return AlwaysFalse() elif count == 1: - return EqualTo(term, literals[0]) + return EqualTo(term, next(iter(literals))) else: return super().__new__(cls) - def __invert__(self) -> NotIn[T]: - return NotIn(self.term, self.literals) + def __invert__(self) -> NotIn: + return NotIn[L](self.term, self.literals) + @property + def as_bound(self) -> Type[BoundIn[L]]: + return BoundIn[L] -@dataclass(frozen=True) -class NotIn(SetPredicate[T]): - as_bound = BoundNotIn - def __new__(cls, term: UnboundTerm[T], literals: tuple[Literal[T], ...]): # pylint: disable=W0221 - count = len(literals) +class NotIn(SetPredicate[L], ABC): + def __new__(cls, term: Union[str, UnboundTerm], literals: Union[Iterable[L], Iterable[Literal[L]]]): # pylint: disable=W0221 + literals_set: Set[Literal[L]] = _to_literal_set(literals) + count = len(literals_set) if count == 0: return AlwaysTrue() elif count == 1: - return NotEqualTo(term, literals[0]) + return NotEqualTo(term, next(iter(literals_set))) else: return super().__new__(cls) - def __invert__(self) -> In[T]: - return In(self.term, self.literals) + def __invert__(self) -> In: + return In[L](self.term, self.literals) + + def __eq__(self, other: Any) -> bool: + if isinstance(other, NotIn): + return self.term == other.term and self.literals == other.literals + return False + + @property + def as_bound(self) -> Type[BoundNotIn[L]]: + return BoundNotIn[L] + +class LiteralPredicate(Generic[L], UnboundPredicate[L], ABC): + literal: Literal[L] -@dataclass(frozen=True) -class LiteralPredicate(UnboundPredicate[T]): - literal: Literal[T] + def __init__(self, term: Union[str, UnboundTerm], literal: Union[L, Literal[L]]): # pylint: disable=W0621 + super().__init__(term) + self.literal = _to_literal(literal) # pylint: disable=W0621 - def bind(self, schema: Schema, case_sensitive: bool = True) -> BooleanExpression: + def bind(self, schema: Schema, case_sensitive: bool = True) -> BoundLiteralPredicate[L]: bound_term = self.term.bind(schema, case_sensitive) return self.as_bound(bound_term, self.literal.to(bound_term.ref().field.field_type)) - def __invert__(self) -> LiteralPredicate[T]: - """Inverts the literal predicate""" - raise NotImplementedError + def __eq__(self, other): + if isinstance(other, LiteralPredicate): + return self.term == other.term and self.literal == other.literal + return False + def __repr__(self) -> str: + return f"{str(self.__class__.__name__)}(term={repr(self.term)}, literal={repr(self.literal)})" -@dataclass(frozen=True) -class BoundLiteralPredicate(BoundPredicate[T]): - literal: Literal[T] + @property + @abstractmethod + def as_bound(self) -> Type[BoundLiteralPredicate[L]]: + ... + + +class BoundLiteralPredicate(BoundPredicate[L], ABC): + literal: Literal[L] + + def __init__(self, term: BoundTerm[L], literal: Literal[L]): # pylint: disable=W0621 + # Since we don't know the type of BoundPredicate[L], we have to ignore this one + super().__init__(term) # type: ignore + self.literal = literal # pylint: disable=W0621 - def __invert__(self) -> BoundLiteralPredicate[T]: - """Inverts the bound literal predicate""" - raise NotImplementedError + def __eq__(self, other): + if isinstance(other, BoundLiteralPredicate): + return self.term == other.term and self.literal == other.literal + return False + def __repr__(self) -> str: + return f"{str(self.__class__.__name__)}(term={repr(self.term)}, literal={repr(self.literal)})" -@dataclass(frozen=True) -class BoundEqualTo(BoundLiteralPredicate[T]): - def __invert__(self) -> BoundNotEqualTo[T]: + +class BoundEqualTo(BoundLiteralPredicate[L]): + def __invert__(self) -> BoundNotEqualTo[L]: return BoundNotEqualTo(self.term, self.literal) -@dataclass(frozen=True) -class BoundNotEqualTo(BoundLiteralPredicate[T]): - def __invert__(self) -> BoundEqualTo[T]: +class BoundNotEqualTo(BoundLiteralPredicate[L]): + def __invert__(self) -> BoundEqualTo[L]: return BoundEqualTo(self.term, self.literal) -@dataclass(frozen=True) -class BoundGreaterThanOrEqual(BoundLiteralPredicate[T]): - def __invert__(self) -> BoundLessThan[T]: +class BoundGreaterThanOrEqual(BoundLiteralPredicate[L]): + def __invert__(self) -> BoundLessThan[L]: return BoundLessThan(self.term, self.literal) -@dataclass(frozen=True) -class BoundGreaterThan(BoundLiteralPredicate[T]): - def __invert__(self) -> BoundLessThanOrEqual[T]: +class BoundGreaterThan(BoundLiteralPredicate[L]): + def __invert__(self) -> BoundLessThanOrEqual[L]: return BoundLessThanOrEqual(self.term, self.literal) -@dataclass(frozen=True) -class BoundLessThan(BoundLiteralPredicate[T]): - def __invert__(self) -> BoundGreaterThanOrEqual[T]: +class BoundLessThan(BoundLiteralPredicate[L]): + def __invert__(self) -> BoundGreaterThanOrEqual[L]: return BoundGreaterThanOrEqual(self.term, self.literal) -@dataclass(frozen=True) -class BoundLessThanOrEqual(BoundLiteralPredicate[T]): - def __invert__(self) -> BoundGreaterThan[T]: +class BoundLessThanOrEqual(BoundLiteralPredicate[L]): + def __invert__(self) -> BoundGreaterThan[L]: return BoundGreaterThan(self.term, self.literal) -@dataclass(frozen=True) -class EqualTo(LiteralPredicate[T]): - as_bound = BoundEqualTo - - def __invert__(self) -> NotEqualTo[T]: +class EqualTo(LiteralPredicate[L]): + def __invert__(self) -> NotEqualTo: return NotEqualTo(self.term, self.literal) + @property + def as_bound(self) -> Type[BoundEqualTo[L]]: + return BoundEqualTo[L] -@dataclass(frozen=True) -class NotEqualTo(LiteralPredicate[T]): - as_bound = BoundNotEqualTo - def __invert__(self) -> EqualTo[T]: +class NotEqualTo(LiteralPredicate[L]): + def __invert__(self) -> EqualTo: return EqualTo(self.term, self.literal) + @property + def as_bound(self) -> Type[BoundNotEqualTo[L]]: + return BoundNotEqualTo[L] -@dataclass(frozen=True) -class LessThan(LiteralPredicate[T]): - as_bound = BoundLessThan - def __invert__(self) -> GreaterThanOrEqual[T]: +class LessThan(LiteralPredicate): + def __invert__(self) -> GreaterThanOrEqual: return GreaterThanOrEqual(self.term, self.literal) + @property + def as_bound(self) -> Type[BoundLessThan[L]]: + return BoundLessThan[L] -@dataclass(frozen=True) -class GreaterThanOrEqual(LiteralPredicate[T]): - as_bound = BoundGreaterThanOrEqual - def __invert__(self) -> LessThan[T]: +class GreaterThanOrEqual(LiteralPredicate[L]): + def __invert__(self) -> LessThan: return LessThan(self.term, self.literal) + @property + def as_bound(self) -> Type[BoundGreaterThanOrEqual[L]]: + return BoundGreaterThanOrEqual[L] -@dataclass(frozen=True) -class GreaterThan(LiteralPredicate[T]): - as_bound = BoundGreaterThan - def __invert__(self) -> LessThanOrEqual[T]: +class GreaterThan(LiteralPredicate[L]): + def __invert__(self) -> LessThanOrEqual: return LessThanOrEqual(self.term, self.literal) + @property + def as_bound(self) -> Type[BoundGreaterThan[L]]: + return BoundGreaterThan[L] -@dataclass(frozen=True) -class LessThanOrEqual(LiteralPredicate[T]): - as_bound = BoundLessThanOrEqual - def __invert__(self) -> GreaterThan[T]: +class LessThanOrEqual(LiteralPredicate[L]): + def __invert__(self) -> GreaterThan: return GreaterThan(self.term, self.literal) + + @property + def as_bound(self) -> Type[BoundLessThanOrEqual[L]]: + return BoundLessThanOrEqual[L] diff --git a/pyiceberg/expressions/literals.py b/pyiceberg/expressions/literals.py index a84cd9ae39..ada6fcc8eb 100644 --- a/pyiceberg/expressions/literals.py +++ b/pyiceberg/expressions/literals.py @@ -23,17 +23,12 @@ import struct from abc import ABC, abstractmethod -from datetime import date from decimal import ROUND_HALF_UP, Decimal -from functools import singledispatch, singledispatchmethod -from typing import ( - Any, - Generic, - Type, - TypeVar, -) +from functools import singledispatchmethod +from typing import Any, Generic, Type from uuid import UUID +from pyiceberg.typedef import L from pyiceberg.types import ( BinaryType, BooleanType, @@ -53,7 +48,6 @@ ) from pyiceberg.utils.datetime import ( date_str_to_days, - date_to_days, micros_to_days, time_to_micros, timestamp_to_micros, @@ -61,19 +55,19 @@ ) from pyiceberg.utils.singleton import Singleton -T = TypeVar("T") - -class Literal(Generic[T], ABC): +class Literal(Generic[L], ABC): """Literal which has a value and can be converted between types""" - def __init__(self, value: T, value_type: Type): + _value: L + + def __init__(self, value: L, value_type: Type[L]): if value is None or not isinstance(value, value_type): - raise TypeError(f"Invalid literal value: {value} (not a {value_type})") + raise TypeError(f"Invalid literal value: {value!r} (not a {value_type})") self._value = value @property - def value(self) -> T: + def value(self) -> L: return self._value @singledispatchmethod @@ -82,7 +76,7 @@ def to(self, type_var: IcebergType) -> Literal: ... # pragma: no cover def __repr__(self) -> str: - return f"{type(self).__name__}({self.value})" + return f"{type(self).__name__}({self.value!r})" def __str__(self) -> str: return str(self.value) @@ -90,7 +84,7 @@ def __str__(self) -> str: def __hash__(self) -> int: return hash(self.value) - def __eq__(self, other) -> bool: + def __eq__(self, other: Any) -> bool: return self.value == other.value def __ne__(self, other) -> bool: @@ -109,11 +103,9 @@ def __ge__(self, other) -> bool: return self.value >= other.value -@singledispatch -def literal(value: Any) -> Literal: +def literal(value: L) -> Literal[L]: """ A generic Literal factory to construct an iceberg Literal based on python primitive data type - using dynamic overloading Args: value(python primitive type): the value to be associated with literal @@ -123,54 +115,22 @@ def literal(value: Any) -> Literal: >>> literal(123) LongLiteral(123) """ - raise TypeError(f"Invalid literal value: {repr(value)}") - - -@literal.register(bool) -def _(value: bool) -> Literal[bool]: - return BooleanLiteral(value) - - -@literal.register(int) -def _(value: int) -> Literal[int]: - return LongLiteral(value) - - -@literal.register(float) -def _(value: float) -> Literal[float]: - # expression binding can convert to FloatLiteral if needed - return DoubleLiteral(value) - - -@literal.register(str) -def _(value: str) -> Literal[str]: - return StringLiteral(value) - - -@literal.register(UUID) -def _(value: UUID) -> Literal[UUID]: - return UUIDLiteral(value) - - -@literal.register(bytes) -def _(value: bytes) -> Literal[bytes]: - # expression binding can convert to FixedLiteral if needed - return BinaryLiteral(value) - - -@literal.register(bytearray) -def _(value: bytearray) -> Literal[bytes]: - return BinaryLiteral(bytes(value)) - - -@literal.register(Decimal) -def _(value: Decimal) -> Literal[Decimal]: - return DecimalLiteral(value) - - -@literal.register(date) -def _(value: date) -> Literal[int]: - return DateLiteral(date_to_days(value)) + if isinstance(value, float): + return DoubleLiteral(value) + elif isinstance(value, bool): + return BooleanLiteral(value) # type: ignore + elif isinstance(value, int): + return LongLiteral(value) + elif isinstance(value, str): + return StringLiteral(value) + elif isinstance(value, UUID): + return UUIDLiteral(value) + elif isinstance(value, bytes): + return BinaryLiteral(value) + elif isinstance(value, Decimal): + return DecimalLiteral(value) + else: + raise TypeError(f"Invalid literal value: {repr(value)}") class FloatAboveMax(Literal[float], Singleton): @@ -481,6 +441,9 @@ def _(self, type_var: DecimalType) -> Literal[Decimal]: f"Could not convert {self.value} into a {type_var}, scales differ {type_var.scale} <> {abs(dec.as_tuple().exponent)}" ) + def __repr__(self) -> str: + return f"literal({repr(self.value)})" + class UUIDLiteral(Literal[UUID]): def __init__(self, value: UUID): diff --git a/pyiceberg/expressions/visitors.py b/pyiceberg/expressions/visitors.py index 4c6df0b079..283e7d2317 100644 --- a/pyiceberg/expressions/visitors.py +++ b/pyiceberg/expressions/visitors.py @@ -17,11 +17,11 @@ from abc import ABC, abstractmethod from functools import singledispatch from typing import ( - Any, Callable, Generic, List, Set, + TypeVar, ) from pyiceberg.conversions import from_bytes @@ -44,9 +44,9 @@ BoundNotNull, BoundPredicate, BoundTerm, + L, Not, Or, - T, UnboundPredicate, ) from pyiceberg.expressions.literals import Literal @@ -60,6 +60,8 @@ PrimitiveType, ) +T = TypeVar("T") + class BooleanExpressionVisitor(Generic[T], ABC): @abstractmethod @@ -89,8 +91,8 @@ def visit_and(self, left_result: T, right_result: T) -> T: """Visit method for an And boolean expression Args: - left_result (T): The result of visiting the left side of the expression - right_result (T): The result of visiting the right side of the expression + left_result (R): The result of visiting the left side of the expression + right_result (R): The result of visiting the right side of the expression """ @abstractmethod @@ -98,12 +100,12 @@ def visit_or(self, left_result: T, right_result: T) -> T: """Visit method for an Or boolean expression Args: - left_result (T): The result of visiting the left side of the expression - right_result (T): The result of visiting the right side of the expression + left_result (R): The result of visiting the left side of the expression + right_result (R): The result of visiting the right side of the expression """ @abstractmethod - def visit_unbound_predicate(self, predicate) -> T: + def visit_unbound_predicate(self, predicate: UnboundPredicate) -> T: """Visit method for an unbound predicate in an expression tree Args: @@ -111,11 +113,11 @@ def visit_unbound_predicate(self, predicate) -> T: """ @abstractmethod - def visit_bound_predicate(self, predicate) -> T: + def visit_bound_predicate(self, predicate: BoundPredicate[L]) -> T: """Visit method for a bound predicate in an expression tree Args: - predicate (BoundPredicate): An instance of a BoundPredicate + predicate (BoundPredicate[L]): An instance of a BoundPredicate """ @@ -127,7 +129,7 @@ def visit(obj, visitor: BooleanExpressionVisitor[T]) -> T: Args: obj(BooleanExpression): An instance of a BooleanExpression - visitor(BooleanExpressionVisitor[T]): An instance of an implementation of the generic BooleanExpressionVisitor base class + visitor(BooleanExpressionVisitor[R]): An instance of an implementation of the generic BooleanExpressionVisitor base class Raises: NotImplementedError: If attempting to visit an unsupported expression @@ -135,26 +137,26 @@ def visit(obj, visitor: BooleanExpressionVisitor[T]) -> T: raise NotImplementedError(f"Cannot visit unsupported expression: {obj}") -@visit.register +@visit.register(AlwaysTrue) def _(_: AlwaysTrue, visitor: BooleanExpressionVisitor[T]) -> T: """Visit an AlwaysTrue boolean expression with a concrete BooleanExpressionVisitor""" return visitor.visit_true() -@visit.register +@visit.register(AlwaysFalse) def _(_: AlwaysFalse, visitor: BooleanExpressionVisitor[T]) -> T: """Visit an AlwaysFalse boolean expression with a concrete BooleanExpressionVisitor""" return visitor.visit_false() -@visit.register +@visit.register(Not) def _(obj: Not, visitor: BooleanExpressionVisitor[T]) -> T: """Visit a Not boolean expression with a concrete BooleanExpressionVisitor""" child_result: T = visit(obj.child, visitor=visitor) return visitor.visit_not(child_result=child_result) -@visit.register +@visit.register(And) def _(obj: And, visitor: BooleanExpressionVisitor[T]) -> T: """Visit an And boolean expression with a concrete BooleanExpressionVisitor""" left_result: T = visit(obj.left, visitor=visitor) @@ -162,19 +164,19 @@ def _(obj: And, visitor: BooleanExpressionVisitor[T]) -> T: return visitor.visit_and(left_result=left_result, right_result=right_result) -@visit.register +@visit.register(UnboundPredicate) def _(obj: UnboundPredicate, visitor: BooleanExpressionVisitor[T]) -> T: """Visit an unbound boolean expression with a concrete BooleanExpressionVisitor""" return visitor.visit_unbound_predicate(predicate=obj) -@visit.register -def _(obj: BoundPredicate, visitor: BooleanExpressionVisitor[T]) -> T: +@visit.register(BoundPredicate) +def _(obj: BoundPredicate[L], visitor: BooleanExpressionVisitor[T]) -> T: """Visit a bound boolean expression with a concrete BooleanExpressionVisitor""" return visitor.visit_bound_predicate(predicate=obj) -@visit.register +@visit.register(Or) def _(obj: Or, visitor: BooleanExpressionVisitor[T]) -> T: """Visit an Or boolean expression with a concrete BooleanExpressionVisitor""" left_result: T = visit(obj.left, visitor=visitor) @@ -232,57 +234,57 @@ def visit_or(self, left_result: BooleanExpression, right_result: BooleanExpressi def visit_unbound_predicate(self, predicate: UnboundPredicate) -> BooleanExpression: return predicate.bind(self.schema, case_sensitive=self.case_sensitive) - def visit_bound_predicate(self, predicate: BoundPredicate) -> BooleanExpression: + def visit_bound_predicate(self, predicate: BoundPredicate[L]) -> BooleanExpression: raise TypeError(f"Found already bound predicate: {predicate}") class BoundBooleanExpressionVisitor(BooleanExpressionVisitor[T], ABC): @abstractmethod - def visit_in(self, term: BoundTerm[T], literals: Set[Literal[Any]]) -> T: + def visit_in(self, term: BoundTerm[L], literals: Set[Literal[L]]) -> T: """Visit a bound In predicate""" @abstractmethod - def visit_not_in(self, term: BoundTerm[T], literals: Set[Literal[Any]]) -> T: + def visit_not_in(self, term: BoundTerm[L], literals: Set[Literal[L]]) -> T: """Visit a bound NotIn predicate""" @abstractmethod - def visit_is_nan(self, term: BoundTerm[T]) -> T: + def visit_is_nan(self, term: BoundTerm[L]) -> T: """Visit a bound IsNan predicate""" @abstractmethod - def visit_not_nan(self, term: BoundTerm[T]) -> T: + def visit_not_nan(self, term: BoundTerm[L]) -> T: """Visit a bound NotNan predicate""" @abstractmethod - def visit_is_null(self, term: BoundTerm[T]) -> T: + def visit_is_null(self, term: BoundTerm[L]) -> T: """Visit a bound IsNull predicate""" @abstractmethod - def visit_not_null(self, term: BoundTerm[T]) -> T: + def visit_not_null(self, term: BoundTerm[L]) -> T: """Visit a bound NotNull predicate""" @abstractmethod - def visit_equal(self, term: BoundTerm[T], literal: Literal[Any]) -> T: + def visit_equal(self, term: BoundTerm[L], literal: Literal[L]) -> T: """Visit a bound Equal predicate""" @abstractmethod - def visit_not_equal(self, term: BoundTerm[T], literal: Literal[Any]) -> T: + def visit_not_equal(self, term: BoundTerm[L], literal: Literal[L]) -> T: """Visit a bound NotEqual predicate""" @abstractmethod - def visit_greater_than_or_equal(self, term: BoundTerm[T], literal: Literal[Any]) -> T: + def visit_greater_than_or_equal(self, term: BoundTerm[L], literal: Literal[L]) -> T: """Visit a bound GreaterThanOrEqual predicate""" @abstractmethod - def visit_greater_than(self, term: BoundTerm[T], literal: Literal[Any]) -> T: + def visit_greater_than(self, term: BoundTerm[L], literal: Literal[L]) -> T: """Visit a bound GreaterThan predicate""" @abstractmethod - def visit_less_than(self, term: BoundTerm[T], literal: Literal[Any]) -> T: + def visit_less_than(self, term: BoundTerm[L], literal: Literal[L]) -> T: """Visit a bound LessThan predicate""" @abstractmethod - def visit_less_than_or_equal(self, term: BoundTerm[T], literal: Literal[Any]) -> T: + def visit_less_than_or_equal(self, term: BoundTerm[L], literal: Literal[L]) -> T: """Visit a bound LessThanOrEqual predicate""" @abstractmethod @@ -305,86 +307,86 @@ def visit_and(self, left_result: T, right_result: T) -> T: def visit_or(self, left_result: T, right_result: T) -> T: """Visit a bound Or predicate""" - def visit_unbound_predicate(self, predicate: UnboundPredicate[T]): + def visit_unbound_predicate(self, predicate: UnboundPredicate): """Visit an unbound predicate Args: - predicate (UnboundPredicate[T]): An unbound predicate + predicate (UnboundPredicate): An unbound predicate Raises: TypeError: This always raises since an unbound predicate is not expected in a bound boolean expression """ raise TypeError(f"Not a bound predicate: {predicate}") - def visit_bound_predicate(self, predicate: BoundPredicate[T]) -> T: + def visit_bound_predicate(self, predicate: BoundPredicate[L]) -> T: """Visit a bound predicate Args: - predicate (BoundPredicate[T]): A bound predicate + predicate (BoundPredicate[L]): A bound predicate """ return visit_bound_predicate(predicate, self) @singledispatch -def visit_bound_predicate(expr, _: BooleanExpressionVisitor[T]) -> T: +def visit_bound_predicate(expr: BoundPredicate[L], _: BooleanExpressionVisitor[T]) -> T: raise TypeError(f"Unknown predicate: {expr}") @visit_bound_predicate.register(BoundIn) -def _(expr: BoundIn, visitor: BoundBooleanExpressionVisitor[T]) -> T: +def _(expr: BoundIn[L], visitor: BoundBooleanExpressionVisitor[T]) -> T: return visitor.visit_in(term=expr.term, literals=expr.literals) @visit_bound_predicate.register(BoundNotIn) -def _(expr: BoundNotIn, visitor: BoundBooleanExpressionVisitor[T]) -> T: +def _(expr: BoundNotIn[L], visitor: BoundBooleanExpressionVisitor[T]) -> T: return visitor.visit_not_in(term=expr.term, literals=expr.literals) @visit_bound_predicate.register(BoundIsNaN) -def _(expr: BoundIsNaN, visitor: BoundBooleanExpressionVisitor[T]) -> T: +def _(expr: BoundIsNaN[L], visitor: BoundBooleanExpressionVisitor[T]) -> T: return visitor.visit_is_nan(term=expr.term) @visit_bound_predicate.register(BoundNotNaN) -def _(expr: BoundNotNaN, visitor: BoundBooleanExpressionVisitor[T]) -> T: +def _(expr: BoundNotNaN[L], visitor: BoundBooleanExpressionVisitor[T]) -> T: return visitor.visit_not_nan(term=expr.term) @visit_bound_predicate.register(BoundIsNull) -def _(expr: BoundIsNull, visitor: BoundBooleanExpressionVisitor[T]) -> T: +def _(expr: BoundIsNull[L], visitor: BoundBooleanExpressionVisitor[T]) -> T: return visitor.visit_is_null(term=expr.term) @visit_bound_predicate.register(BoundNotNull) -def _(expr: BoundNotNull, visitor: BoundBooleanExpressionVisitor[T]) -> T: +def _(expr: BoundNotNull[L], visitor: BoundBooleanExpressionVisitor[T]) -> T: return visitor.visit_not_null(term=expr.term) @visit_bound_predicate.register(BoundEqualTo) -def _(expr: BoundEqualTo, visitor: BoundBooleanExpressionVisitor[T]) -> T: +def _(expr: BoundEqualTo[L], visitor: BoundBooleanExpressionVisitor[T]) -> T: return visitor.visit_equal(term=expr.term, literal=expr.literal) @visit_bound_predicate.register(BoundNotEqualTo) -def _(expr: BoundNotEqualTo, visitor: BoundBooleanExpressionVisitor[T]) -> T: +def _(expr: BoundNotEqualTo[L], visitor: BoundBooleanExpressionVisitor[T]) -> T: return visitor.visit_not_equal(term=expr.term, literal=expr.literal) @visit_bound_predicate.register(BoundGreaterThanOrEqual) -def _(expr: BoundGreaterThanOrEqual, visitor: BoundBooleanExpressionVisitor[T]) -> T: +def _(expr: BoundGreaterThanOrEqual[L], visitor: BoundBooleanExpressionVisitor[T]) -> T: """Visit a bound GreaterThanOrEqual predicate""" return visitor.visit_greater_than_or_equal(term=expr.term, literal=expr.literal) @visit_bound_predicate.register(BoundGreaterThan) -def _(expr: BoundGreaterThan, visitor: BoundBooleanExpressionVisitor[T]) -> T: +def _(expr: BoundGreaterThan[L], visitor: BoundBooleanExpressionVisitor[T]) -> T: return visitor.visit_greater_than(term=expr.term, literal=expr.literal) @visit_bound_predicate.register(BoundLessThan) -def _(expr: BoundLessThan, visitor: BoundBooleanExpressionVisitor[T]) -> T: +def _(expr: BoundLessThan[L], visitor: BoundBooleanExpressionVisitor[T]) -> T: return visitor.visit_less_than(term=expr.term, literal=expr.literal) @visit_bound_predicate.register(BoundLessThanOrEqual) -def _(expr: BoundLessThanOrEqual, visitor: BoundBooleanExpressionVisitor[T]) -> T: +def _(expr: BoundLessThanOrEqual[L], visitor: BoundBooleanExpressionVisitor[T]) -> T: return visitor.visit_less_than_or_equal(term=expr.term, literal=expr.literal) @@ -410,10 +412,10 @@ def visit_and(self, left_result: BooleanExpression, right_result: BooleanExpress def visit_or(self, left_result: BooleanExpression, right_result: BooleanExpression) -> BooleanExpression: return Or(left=left_result, right=right_result) - def visit_unbound_predicate(self, predicate) -> BooleanExpression: + def visit_unbound_predicate(self, predicate: UnboundPredicate) -> BooleanExpression: return predicate - def visit_bound_predicate(self, predicate) -> BooleanExpression: + def visit_bound_predicate(self, predicate: BoundPredicate[L]) -> BooleanExpression: return predicate @@ -443,7 +445,7 @@ def eval(self, manifest: ManifestFile) -> bool: # No partition information return ROWS_MIGHT_MATCH - def visit_in(self, term: BoundTerm, literals: Set[Literal[Any]]) -> bool: + def visit_in(self, term: BoundTerm[L], literals: Set[Literal[L]]) -> bool: pos = term.ref().accessor.position field = self.partition_fields[pos] @@ -465,12 +467,12 @@ def visit_in(self, term: BoundTerm, literals: Set[Literal[Any]]) -> bool: return ROWS_MIGHT_MATCH - def visit_not_in(self, term: BoundTerm, literals: Set[Literal[Any]]) -> bool: + def visit_not_in(self, term: BoundTerm[L], literals: Set[Literal[L]]) -> bool: # because the bounds are not necessarily a min or max value, this cannot be answered using # them. notIn(col, {X, ...}) with (X, Y) doesn't guarantee that X is a value in col. return ROWS_MIGHT_MATCH - def visit_is_nan(self, term: BoundTerm) -> bool: + def visit_is_nan(self, term: BoundTerm[L]) -> bool: pos = term.ref().accessor.position field = self.partition_fields[pos] @@ -479,7 +481,7 @@ def visit_is_nan(self, term: BoundTerm) -> bool: return ROWS_MIGHT_MATCH - def visit_not_nan(self, term: BoundTerm) -> bool: + def visit_not_nan(self, term: BoundTerm[L]) -> bool: pos = term.ref().accessor.position field = self.partition_fields[pos] @@ -488,7 +490,7 @@ def visit_not_nan(self, term: BoundTerm) -> bool: return ROWS_MIGHT_MATCH - def visit_is_null(self, term: BoundTerm) -> bool: + def visit_is_null(self, term: BoundTerm[L]) -> bool: pos = term.ref().accessor.position if self.partition_fields[pos].contains_null is False: @@ -496,7 +498,7 @@ def visit_is_null(self, term: BoundTerm) -> bool: return ROWS_MIGHT_MATCH - def visit_not_null(self, term: BoundTerm) -> bool: + def visit_not_null(self, term: BoundTerm[L]) -> bool: pos = term.ref().accessor.position # contains_null encodes whether at least one partition value is null, @@ -513,7 +515,7 @@ def visit_not_null(self, term: BoundTerm) -> bool: return ROWS_MIGHT_MATCH - def visit_equal(self, term: BoundTerm, literal: Literal[Any]) -> bool: + def visit_equal(self, term: BoundTerm[L], literal: Literal[L]) -> bool: pos = term.ref().accessor.position field = self.partition_fields[pos] @@ -533,12 +535,12 @@ def visit_equal(self, term: BoundTerm, literal: Literal[Any]) -> bool: return ROWS_MIGHT_MATCH - def visit_not_equal(self, term: BoundTerm, literal: Literal[Any]) -> bool: + def visit_not_equal(self, term: BoundTerm[L], literal: Literal[L]) -> bool: # because the bounds are not necessarily a min or max value, this cannot be answered using # them. notEq(col, X) with (X, Y) doesn't guarantee that X is a value in col. return ROWS_MIGHT_MATCH - def visit_greater_than_or_equal(self, term: BoundTerm, literal: Literal[Any]) -> bool: + def visit_greater_than_or_equal(self, term: BoundTerm[L], literal: Literal[L]) -> bool: pos = term.ref().accessor.position field = self.partition_fields[pos] @@ -552,7 +554,7 @@ def visit_greater_than_or_equal(self, term: BoundTerm, literal: Literal[Any]) -> return ROWS_MIGHT_MATCH - def visit_greater_than(self, term: BoundTerm, literal: Literal[Any]) -> bool: + def visit_greater_than(self, term: BoundTerm[L], literal: Literal[L]) -> bool: pos = term.ref().accessor.position field = self.partition_fields[pos] @@ -566,7 +568,7 @@ def visit_greater_than(self, term: BoundTerm, literal: Literal[Any]) -> bool: return ROWS_MIGHT_MATCH - def visit_less_than(self, term: BoundTerm, literal: Literal[Any]) -> bool: + def visit_less_than(self, term: BoundTerm[L], literal: Literal[L]) -> bool: pos = term.ref().accessor.position field = self.partition_fields[pos] @@ -580,7 +582,7 @@ def visit_less_than(self, term: BoundTerm, literal: Literal[Any]) -> bool: return ROWS_MIGHT_MATCH - def visit_less_than_or_equal(self, term: BoundTerm, literal: Literal[Any]) -> bool: + def visit_less_than_or_equal(self, term: BoundTerm[L], literal: Literal[L]) -> bool: pos = term.ref().accessor.position field = self.partition_fields[pos] diff --git a/pyiceberg/typedef.py b/pyiceberg/typedef.py index 76add54bdd..b038bfb17c 100644 --- a/pyiceberg/typedef.py +++ b/pyiceberg/typedef.py @@ -14,13 +14,15 @@ # KIND, either express or implied. See the License for the # specific language governing permissions and limitations # under the License. - +from decimal import Decimal from typing import ( Any, Dict, Tuple, + TypeVar, Union, ) +from uuid import UUID class FrozenDict(Dict[Any, Any]): @@ -36,3 +38,6 @@ def update(self, *args: Any, **kwargs: Any) -> None: Identifier = Tuple[str, ...] Properties = Dict[str, str] RecursiveDict = Dict[str, Union[str, "RecursiveDict"]] + +# Represents the literal value +L = TypeVar("L", str, bool, int, float, bytes, UUID, Decimal) diff --git a/pyproject.toml b/pyproject.toml index b54f266e8b..181093e469 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -72,6 +72,7 @@ pre-commit = "2.20.0" fastavro = "1.6.1" coverage = { version = "^6.5.0", extras = ["toml"] } requests-mock = "1.10.0" +typing-extensions = '4.4.0' [tool.poetry.scripts] pyiceberg = "pyiceberg.cli.console:run" diff --git a/tests/expressions/test_expressions.py b/tests/expressions/test_expressions.py index 6ccada73a4..2772ac2c34 100644 --- a/tests/expressions/test_expressions.py +++ b/tests/expressions/test_expressions.py @@ -14,17 +14,18 @@ # KIND, either express or implied. See the License for the # specific language governing permissions and limitations # under the License. +# pylint:disable=redefined-outer-name,eval-used import uuid from decimal import Decimal import pytest +from typing_extensions import assert_type from pyiceberg.expressions import ( AlwaysFalse, AlwaysTrue, And, - BooleanExpression, BoundEqualTo, BoundGreaterThan, BoundGreaterThanOrEqual, @@ -33,15 +34,11 @@ BoundIsNull, BoundLessThan, BoundLessThanOrEqual, - BoundLiteralPredicate, BoundNotEqualTo, BoundNotIn, BoundNotNaN, BoundNotNull, - BoundPredicate, BoundReference, - BoundSetPredicate, - BoundUnaryPredicate, EqualTo, GreaterThan, GreaterThanOrEqual, @@ -50,7 +47,6 @@ IsNull, LessThan, LessThanOrEqual, - LiteralPredicate, Not, NotEqualTo, NotIn, @@ -58,11 +54,8 @@ NotNull, Or, Reference, - SetPredicate, - UnaryPredicate, - UnboundPredicate, ) -from pyiceberg.expressions.literals import StringLiteral, literal +from pyiceberg.expressions.literals import literal from pyiceberg.expressions.visitors import _from_byte_buffer from pyiceberg.schema import Accessor, Schema from pyiceberg.types import ( @@ -73,27 +66,10 @@ NestedField, StringType, ) +from tests.conftest import FooStruct from tests.expressions.test_visitors import ExpressionA, ExpressionB -@pytest.mark.parametrize( - "op, rep", - [ - ( - And(ExpressionA(), ExpressionB()), - "And(left=ExpressionA(), right=ExpressionB())", - ), - ( - Or(ExpressionA(), ExpressionB()), - "Or(left=ExpressionA(), right=ExpressionB())", - ), - (Not(ExpressionA()), "Not(child=ExpressionA())"), - ], -) -def test_reprs(op: BooleanExpression, rep: str): - assert repr(op) == rep - - def test_isnull_inverse(): assert ~IsNull(Reference("a")) == NotNull(Reference("a")) @@ -202,48 +178,36 @@ def test_notnan_bind_nonfloat(): assert NotNaN(Reference("i")).bind(schema) == AlwaysTrue() -@pytest.mark.parametrize( - "op, string", - [ - (And(ExpressionA(), ExpressionB()), "And(left=ExpressionA(), right=ExpressionB())"), - (Or(ExpressionA(), ExpressionB()), "Or(left=ExpressionA(), right=ExpressionB())"), - (Not(ExpressionA()), "Not(child=ExpressionA())"), - ], -) -def test_strs(op, string): - assert str(op) == string - - def test_ref_binding_case_sensitive(table_schema_simple: Schema): - ref = Reference[str]("foo") + ref = Reference("foo") bound = BoundReference[str](table_schema_simple.find_field(1), table_schema_simple.accessor_for_field(1)) assert ref.bind(table_schema_simple, case_sensitive=True) == bound def test_ref_binding_case_sensitive_failure(table_schema_simple: Schema): - ref = Reference[str]("Foo") + ref = Reference("Foo") with pytest.raises(ValueError): ref.bind(table_schema_simple, case_sensitive=True) def test_ref_binding_case_insensitive(table_schema_simple: Schema): - ref = Reference[str]("Foo") + ref = Reference("Foo") bound = BoundReference[str](table_schema_simple.find_field(1), table_schema_simple.accessor_for_field(1)) assert ref.bind(table_schema_simple, case_sensitive=False) == bound def test_ref_binding_case_insensitive_failure(table_schema_simple: Schema): - ref = Reference[str]("Foot") + ref = Reference("Foot") with pytest.raises(ValueError): ref.bind(table_schema_simple, case_sensitive=False) def test_in_to_eq(): - assert In(Reference("x"), (literal(34.56),)) == EqualTo(Reference("x"), literal(34.56)) + assert In("x", (34.56,)) == EqualTo(Reference("x"), 34.56) def test_empty_bind_in(table_schema_simple: Schema): - bound = BoundIn[str](BoundReference(table_schema_simple.find_field(1), table_schema_simple.accessor_for_field(1)), set()) + bound = BoundIn(BoundReference[str](table_schema_simple.find_field(1), table_schema_simple.accessor_for_field(1)), set()) assert bound == AlwaysFalse() @@ -257,12 +221,12 @@ def test_bind_not_in_equal_term(table_schema_simple: Schema): BoundReference(table_schema_simple.find_field(1), table_schema_simple.accessor_for_field(1)), {literal("hello")} ) assert ( - BoundNotEqualTo[str]( + BoundNotEqualTo( term=BoundReference( field=NestedField(field_id=1, name="foo", field_type=StringType(), required=False), accessor=Accessor(position=0, inner=None), ), - literal=StringLiteral("hello"), + literal=literal("hello"), ) == bound ) @@ -272,57 +236,69 @@ def test_in_empty(): assert In(Reference("foo"), ()) == AlwaysFalse() +def test_in_set(): + assert In(Reference("foo"), {"a", "bc", "def"}).literals == {literal("a"), literal("bc"), literal("def")} + + +def test_in_tuple(): + assert In(Reference("foo"), ("a", "bc", "def")).literals == {literal("a"), literal("bc"), literal("def")} + + +def test_in_list(): + assert In(Reference("foo"), ["a", "bc", "def"]).literals == {literal("a"), literal("bc"), literal("def")} + + def test_not_in_empty(): assert NotIn(Reference("foo"), ()) == AlwaysTrue() def test_not_in_equal(): - assert NotIn(Reference("foo"), (literal("hello"),)) == NotEqualTo(term=Reference(name="foo"), literal=StringLiteral("hello")) + assert NotIn(Reference("foo"), ("hello",)) == NotEqualTo(term=Reference(name="foo"), literal="hello") def test_bind_in(table_schema_simple: Schema): - bound = BoundIn( + bound = BoundIn[str]( BoundReference(table_schema_simple.find_field(1), table_schema_simple.accessor_for_field(1)), {literal("hello"), literal("world")}, ) - assert In(Reference("foo"), (literal("hello"), literal("world"))).bind(table_schema_simple) == bound + assert In(Reference("foo"), ("hello", "world")).bind(table_schema_simple) == bound def test_bind_in_invert(table_schema_simple: Schema): - bound = BoundIn( + bound = BoundIn[str]( BoundReference(table_schema_simple.find_field(1), table_schema_simple.accessor_for_field(1)), {literal("hello"), literal("world")}, ) - assert ~bound == BoundNotIn( + assert ~bound == BoundNotIn[str]( BoundReference(table_schema_simple.find_field(1), table_schema_simple.accessor_for_field(1)), {literal("hello"), literal("world")}, ) def test_bind_not_in_invert(table_schema_simple: Schema): - bound = BoundNotIn( + bound = BoundNotIn[str]( BoundReference(table_schema_simple.find_field(1), table_schema_simple.accessor_for_field(1)), {literal("hello"), literal("world")}, ) - assert ~bound == BoundIn( + assert ~bound == BoundIn[str]( BoundReference(table_schema_simple.find_field(1), table_schema_simple.accessor_for_field(1)), {literal("hello"), literal("world")}, ) def test_bind_dedup(table_schema_simple: Schema): - bound = BoundIn( + bound = BoundIn[str]( BoundReference(table_schema_simple.find_field(1), table_schema_simple.accessor_for_field(1)), {literal("hello"), literal("world")}, ) - assert In(Reference("foo"), (literal("hello"), literal("world"), literal("world"))).bind(table_schema_simple) == bound + assert In(Reference("foo"), ("hello", "world", "world")).bind(table_schema_simple) == bound def test_bind_dedup_to_eq(table_schema_simple: Schema): bound = BoundEqualTo( BoundReference(table_schema_simple.find_field(1), table_schema_simple.accessor_for_field(1)), literal("hello") ) - assert In(Reference("foo"), (literal("hello"), literal("hello"))).bind(table_schema_simple) == bound + assert In(Reference("foo"), ("hello", "hello")).bind(table_schema_simple) == bound def test_bound_equal_to_invert(table_schema_simple: Schema): @@ -330,11 +306,11 @@ def test_bound_equal_to_invert(table_schema_simple: Schema): BoundReference(table_schema_simple.find_field(1), table_schema_simple.accessor_for_field(1)), literal("hello") ) assert ~bound == BoundNotEqualTo( - term=BoundReference[str]( + term=BoundReference( field=NestedField(field_id=1, name="foo", field_type=StringType(), required=False), accessor=Accessor(position=0, inner=None), ), - literal=StringLiteral("hello"), + literal=literal("hello"), ) @@ -343,11 +319,11 @@ def test_bound_not_equal_to_invert(table_schema_simple: Schema): BoundReference(table_schema_simple.find_field(1), table_schema_simple.accessor_for_field(1)), literal("hello") ) assert ~bound == BoundEqualTo( - term=BoundReference[str]( + term=BoundReference( field=NestedField(field_id=1, name="foo", field_type=StringType(), required=False), accessor=Accessor(position=0, inner=None), ), - literal=StringLiteral("hello"), + literal=literal("hello"), ) @@ -356,11 +332,11 @@ def test_bound_greater_than_or_equal_invert(table_schema_simple: Schema): BoundReference(table_schema_simple.find_field(1), table_schema_simple.accessor_for_field(1)), literal("hello") ) assert ~bound == BoundLessThan( - term=BoundReference[str]( + term=BoundReference( field=NestedField(field_id=1, name="foo", field_type=StringType(), required=False), accessor=Accessor(position=0, inner=None), ), - literal=StringLiteral("hello"), + literal=literal("hello"), ) @@ -369,11 +345,11 @@ def test_bound_greater_than_invert(table_schema_simple: Schema): BoundReference(table_schema_simple.find_field(1), table_schema_simple.accessor_for_field(1)), literal("hello") ) assert ~bound == BoundLessThanOrEqual( - term=BoundReference[str]( + term=BoundReference( field=NestedField(field_id=1, name="foo", field_type=StringType(), required=False), accessor=Accessor(position=0, inner=None), ), - literal=StringLiteral("hello"), + literal=literal("hello"), ) @@ -382,11 +358,11 @@ def test_bound_less_than_invert(table_schema_simple: Schema): BoundReference(table_schema_simple.find_field(1), table_schema_simple.accessor_for_field(1)), literal("hello") ) assert ~bound == BoundGreaterThanOrEqual( - term=BoundReference[str]( + term=BoundReference( field=NestedField(field_id=1, name="foo", field_type=StringType(), required=False), accessor=Accessor(position=0, inner=None), ), - literal=StringLiteral("hello"), + literal=literal("hello"), ) @@ -395,11 +371,11 @@ def test_bound_less_than_or_equal_invert(table_schema_simple: Schema): BoundReference(table_schema_simple.find_field(1), table_schema_simple.accessor_for_field(1)), literal("hello") ) assert ~bound == BoundGreaterThan( - term=BoundReference[str]( + term=BoundReference( field=NestedField(field_id=1, name="foo", field_type=StringType(), required=False), accessor=Accessor(position=0, inner=None), ), - literal=StringLiteral("hello"), + literal=literal("hello"), ) @@ -409,14 +385,14 @@ def test_not_equal_to_invert(): field=NestedField(field_id=1, name="foo", field_type=StringType(), required=False), accessor=Accessor(position=0, inner=None), ), - literal=StringLiteral("hello"), + literal="hello", ) assert ~bound == EqualTo( term=BoundReference( field=NestedField(field_id=1, name="foo", field_type=StringType(), required=False), accessor=Accessor(position=0, inner=None), ), - literal=StringLiteral("hello"), + literal="hello", ) @@ -426,14 +402,14 @@ def test_greater_than_or_equal_invert(): field=NestedField(field_id=1, name="foo", field_type=StringType(), required=False), accessor=Accessor(position=0, inner=None), ), - literal=StringLiteral("hello"), + literal="hello", ) assert ~bound == LessThan( term=BoundReference( field=NestedField(field_id=1, name="foo", field_type=StringType(), required=False), accessor=Accessor(position=0, inner=None), ), - literal=StringLiteral("hello"), + literal="hello", ) @@ -443,27 +419,27 @@ def test_less_than_or_equal_invert(): field=NestedField(field_id=1, name="foo", field_type=StringType(), required=False), accessor=Accessor(position=0, inner=None), ), - literal=StringLiteral("hello"), + literal="hello", ) assert ~bound == GreaterThan( term=BoundReference( field=NestedField(field_id=1, name="foo", field_type=StringType(), required=False), accessor=Accessor(position=0, inner=None), ), - literal=StringLiteral("hello"), + literal="hello", ) @pytest.mark.parametrize( "pred", [ - NotIn(Reference("foo"), (literal("hello"), literal("world"))), - NotEqualTo(Reference("foo"), literal("hello")), - EqualTo(Reference("foo"), literal("hello")), - GreaterThan(Reference("foo"), literal("hello")), - LessThan(Reference("foo"), literal("hello")), - GreaterThanOrEqual(Reference("foo"), literal("hello")), - LessThanOrEqual(Reference("foo"), literal("hello")), + NotIn(Reference("foo"), ("hello", "world")), + NotEqualTo(Reference("foo"), "hello"), + EqualTo(Reference("foo"), "hello"), + GreaterThan(Reference("foo"), "hello"), + LessThan(Reference("foo"), "hello"), + GreaterThanOrEqual(Reference("foo"), "hello"), + LessThanOrEqual(Reference("foo"), "hello"), ], ) def test_bind(pred, table_schema_simple: Schema): @@ -475,14 +451,14 @@ def test_bind(pred, table_schema_simple: Schema): @pytest.mark.parametrize( "pred", [ - In(Reference("Bar"), (literal(5), literal(2))), - NotIn(Reference("Bar"), (literal(5), literal(2))), - NotEqualTo(Reference("Bar"), literal(5)), - EqualTo(Reference("Bar"), literal(5)), - GreaterThan(Reference("Bar"), literal(5)), - LessThan(Reference("Bar"), literal(5)), - GreaterThanOrEqual(Reference("Bar"), literal(5)), - LessThanOrEqual(Reference("Bar"), literal(5)), + In(Reference("Bar"), (5, 2)), + NotIn(Reference("Bar"), (5, 2)), + NotEqualTo(Reference("Bar"), 5), + EqualTo(Reference("Bar"), 5), + GreaterThan(Reference("Bar"), 5), + LessThan(Reference("Bar"), 5), + GreaterThanOrEqual(Reference("Bar"), 5), + LessThanOrEqual(Reference("Bar"), 5), ], ) def test_bind_case_insensitive(pred, table_schema_simple: Schema): @@ -508,14 +484,14 @@ def test_bind_case_insensitive(pred, table_schema_simple: Schema): (ExpressionA(), ExpressionA(), ExpressionB()), (ExpressionB(), ExpressionB(), ExpressionA()), ( - In(Reference("foo"), (literal("hello"), literal("world"))), - In(Reference("foo"), (literal("hello"), literal("world"))), - In(Reference("not_foo"), (literal("hello"), literal("world"))), + In(Reference("foo"), ("hello", "world")), + In(Reference("foo"), ("hello", "world")), + In(Reference("not_foo"), ("hello", "world")), ), ( - In(Reference("foo"), (literal("hello"), literal("world"))), - In(Reference("foo"), (literal("hello"), literal("world"))), - In(Reference("foo"), (literal("goodbye"), literal("world"))), + In(Reference("foo"), ("hello", "world")), + In(Reference("foo"), ("hello", "world")), + In(Reference("foo"), ("goodbye", "world")), ), ], ) @@ -539,16 +515,16 @@ def test_eq(exp, testexpra, testexprb): ExpressionA(), ), ( - In(Reference("foo"), (literal("hello"), literal("world"))), - NotIn(Reference("foo"), (literal("hello"), literal("world"))), + In(Reference("foo"), ("hello", "world")), + NotIn(Reference("foo"), ("hello", "world")), ), ( - NotIn(Reference("foo"), (literal("hello"), literal("world"))), - In(Reference("foo"), (literal("hello"), literal("world"))), + NotIn(Reference("foo"), ("hello", "world")), + In(Reference("foo"), ("hello", "world")), ), - (GreaterThan(Reference("foo"), literal(5)), LessThanOrEqual(Reference("foo"), literal(5))), - (LessThan(Reference("foo"), literal(5)), GreaterThanOrEqual(Reference("foo"), literal(5))), - (EqualTo(Reference("foo"), literal(5)), NotEqualTo(Reference("foo"), literal(5))), + (GreaterThan(Reference("foo"), 5), LessThanOrEqual(Reference("foo"), 5)), + (LessThan(Reference("foo"), 5), GreaterThanOrEqual(Reference("foo"), 5)), + (EqualTo(Reference("foo"), 5), NotEqualTo(Reference("foo"), 5)), ( ExpressionA(), ExpressionB(), @@ -632,24 +608,271 @@ def test_accessor_base_class(foo_struct): assert Accessor(position=11).get(foo_struct) == b"\x19\x04\x9e?" -def test_bound_reference_str_and_repr(): - """Test str and repr of BoundReference""" - field = NestedField(field_id=1, name="foo", field_type=StringType(), required=False) - position1_accessor = Accessor(position=1) - bound_ref = BoundReference(field=field, accessor=position1_accessor) - assert str(bound_ref) == f"BoundReference(field={repr(field)}, accessor={repr(position1_accessor)})" - assert repr(bound_ref) == f"BoundReference(field={repr(field)}, accessor={repr(position1_accessor)})" +@pytest.fixture +def field() -> NestedField: + return NestedField(field_id=1, name="foo", field_type=StringType(), required=False) + + +@pytest.fixture +def accessor() -> Accessor: + return Accessor(position=1) + + +@pytest.fixture +def term(field: NestedField, accessor: Accessor) -> BoundReference: + return BoundReference( + field=field, + accessor=accessor, + ) + + +def test_bound_reference(field: NestedField, accessor: Accessor) -> None: + bound_ref = BoundReference(field=field, accessor=accessor) + assert str(bound_ref) == f"BoundReference(field={repr(field)}, accessor={repr(accessor)})" + assert repr(bound_ref) == f"BoundReference(field={repr(field)}, accessor={repr(accessor)})" + assert bound_ref == eval(repr(bound_ref)) + + +def test_reference() -> None: + abc = "abc" + ref = Reference(abc) + assert str(ref) == "Reference(name='abc')" + assert repr(ref) == "Reference(name='abc')" + assert ref == eval(repr(ref)) + + +def test_and() -> None: + null = IsNull(Reference("a")) + nan = IsNaN(Reference("b")) + and_ = And(null, nan) + assert str(and_) == f"And(left={str(null)}, right={str(nan)})" + assert repr(and_) == f"And(left={repr(null)}, right={repr(nan)})" + assert and_ == eval(repr(and_)) + + +def test_or() -> None: + null = IsNull(Reference("a")) + nan = IsNaN(Reference("b")) + or_ = Or(null, nan) + assert str(or_) == f"Or(left={str(null)}, right={str(nan)})" + assert repr(or_) == f"Or(left={repr(null)}, right={repr(nan)})" + assert or_ == eval(repr(or_)) + + +def test_not() -> None: + null = IsNull(Reference("a")) + or_ = Not(null) + assert str(or_) == f"Not(child={str(null)})" + assert repr(or_) == f"Not(child={repr(null)})" + assert or_ == eval(repr(or_)) + + +def test_always_true() -> None: + always_true = AlwaysTrue() + assert str(always_true) == "AlwaysTrue()" + assert repr(always_true) == "AlwaysTrue()" + assert always_true == eval(repr(always_true)) + + +def test_always_false() -> None: + always_false = AlwaysFalse() + assert str(always_false) == "AlwaysFalse()" + assert repr(always_false) == "AlwaysFalse()" + assert always_false == eval(repr(always_false)) def test_bound_reference_field_property(): - """Test str and repr of BoundReference""" field = NestedField(field_id=1, name="foo", field_type=StringType(), required=False) position1_accessor = Accessor(position=1) bound_ref = BoundReference(field=field, accessor=position1_accessor) assert bound_ref.field == NestedField(field_id=1, name="foo", field_type=StringType(), required=False) -def test_bound_reference(table_schema_simple, foo_struct): +def test_bound_is_null(term: BoundReference) -> None: + bound_is_null = BoundIsNull(term) + assert str(bound_is_null) == f"BoundIsNull(term={str(term)})" + assert repr(bound_is_null) == f"BoundIsNull(term={repr(term)})" + assert bound_is_null == eval(repr(bound_is_null)) + + +def test_bound_is_not_null(term: BoundReference) -> None: + bound_not_null = BoundNotNull(term) + assert str(bound_not_null) == f"BoundNotNull(term={str(term)})" + assert repr(bound_not_null) == f"BoundNotNull(term={repr(term)})" + assert bound_not_null == eval(repr(bound_not_null)) + + +def test_is_null() -> None: + ref = Reference("a") + is_null = IsNull(ref) + assert str(is_null) == f"IsNull(term={str(ref)})" + assert repr(is_null) == f"IsNull(term={repr(ref)})" + assert is_null == eval(repr(is_null)) + + +def test_not_null() -> None: + ref = Reference("a") + non_null = NotNull(ref) + assert str(non_null) == f"NotNull(term={str(ref)})" + assert repr(non_null) == f"NotNull(term={repr(ref)})" + assert non_null == eval(repr(non_null)) + + +def test_bound_is_nan(accessor: Accessor) -> None: + # We need a FloatType here + term = BoundReference[float]( + field=NestedField(field_id=1, name="foo", field_type=FloatType(), required=False), + accessor=accessor, + ) + bound_is_nan = BoundIsNaN(term) + assert str(bound_is_nan) == f"BoundIsNaN(term={str(term)})" + assert repr(bound_is_nan) == f"BoundIsNaN(term={repr(term)})" + assert bound_is_nan == eval(repr(bound_is_nan)) + + +def test_bound_is_not_nan(accessor: Accessor) -> None: + # We need a FloatType here + term = BoundReference[float]( + field=NestedField(field_id=1, name="foo", field_type=FloatType(), required=False), + accessor=accessor, + ) + bound_not_nan = BoundNotNaN(term) + assert str(bound_not_nan) == f"BoundNotNaN(term={str(term)})" + assert repr(bound_not_nan) == f"BoundNotNaN(term={repr(term)})" + assert bound_not_nan == eval(repr(bound_not_nan)) + + +def test_is_nan() -> None: + ref = Reference("a") + is_nan = IsNaN(ref) + assert str(is_nan) == f"IsNaN(term={str(ref)})" + assert repr(is_nan) == f"IsNaN(term={repr(ref)})" + assert is_nan == eval(repr(is_nan)) + + +def test_not_nan() -> None: + ref = Reference("a") + not_nan = NotNaN(ref) + assert str(not_nan) == f"NotNaN(term={str(ref)})" + assert repr(not_nan) == f"NotNaN(term={repr(ref)})" + assert not_nan == eval(repr(not_nan)) + + +def test_bound_in(term: BoundReference) -> None: + bound_in = BoundIn(term, {literal("a"), literal("b"), literal("c")}) + assert str(bound_in) == f"BoundIn({str(term)}, {{a, b, c}})" + assert repr(bound_in) == f"BoundIn({repr(term)}, {{literal('a'), literal('b'), literal('c')}})" + assert bound_in == eval(repr(bound_in)) + + +def test_bound_not_in(term: BoundReference) -> None: + bound_not_in = BoundNotIn(term, {literal("a"), literal("b"), literal("c")}) + assert str(bound_not_in) == f"BoundNotIn({str(term)}, {{a, b, c}})" + assert repr(bound_not_in) == f"BoundNotIn({repr(term)}, {{literal('a'), literal('b'), literal('c')}})" + assert bound_not_in == eval(repr(bound_not_in)) + + +def test_in() -> None: + ref = Reference("a") + unbound_in = In(ref, {"a", "b", "c"}) + assert str(unbound_in) == f"In({str(ref)}, {{a, b, c}})" + assert repr(unbound_in) == f"In({repr(ref)}, {{literal('a'), literal('b'), literal('c')}})" + assert unbound_in == eval(repr(unbound_in)) + + +def test_not_in() -> None: + ref = Reference("a") + not_in = NotIn(ref, {"a", "b", "c"}) + assert str(not_in) == f"NotIn({str(ref)}, {{a, b, c}})" + assert repr(not_in) == f"NotIn({repr(ref)}, {{literal('a'), literal('b'), literal('c')}})" + assert not_in == eval(repr(not_in)) + + +def test_bound_equal_to(term: BoundReference) -> None: + bound_equal_to = BoundEqualTo(term, literal("a")) + assert str(bound_equal_to) == f"BoundEqualTo(term={str(term)}, literal=literal('a'))" + assert repr(bound_equal_to) == f"BoundEqualTo(term={repr(term)}, literal=literal('a'))" + assert bound_equal_to == eval(repr(bound_equal_to)) + + +def test_bound_not_equal_to(term: BoundReference) -> None: + bound_not_equal_to = BoundNotEqualTo(term, literal("a")) + assert str(bound_not_equal_to) == f"BoundNotEqualTo(term={str(term)}, literal=literal('a'))" + assert repr(bound_not_equal_to) == f"BoundNotEqualTo(term={repr(term)}, literal=literal('a'))" + assert bound_not_equal_to == eval(repr(bound_not_equal_to)) + + +def test_bound_greater_than_or_equal_to(term: BoundReference) -> None: + bound_greater_than_or_equal_to = BoundGreaterThanOrEqual(term, literal("a")) + assert str(bound_greater_than_or_equal_to) == f"BoundGreaterThanOrEqual(term={str(term)}, literal=literal('a'))" + assert repr(bound_greater_than_or_equal_to) == f"BoundGreaterThanOrEqual(term={repr(term)}, literal=literal('a'))" + assert bound_greater_than_or_equal_to == eval(repr(bound_greater_than_or_equal_to)) + + +def test_bound_greater_than(term: BoundReference) -> None: + bound_greater_than = BoundGreaterThan(term, literal("a")) + assert str(bound_greater_than) == f"BoundGreaterThan(term={str(term)}, literal=literal('a'))" + assert repr(bound_greater_than) == f"BoundGreaterThan(term={repr(term)}, literal=literal('a'))" + assert bound_greater_than == eval(repr(bound_greater_than)) + + +def test_bound_less_than(term: BoundReference) -> None: + bound_less_than = BoundLessThan(term, literal("a")) + assert str(bound_less_than) == f"BoundLessThan(term={str(term)}, literal=literal('a'))" + assert repr(bound_less_than) == f"BoundLessThan(term={repr(term)}, literal=literal('a'))" + assert bound_less_than == eval(repr(bound_less_than)) + + +def test_bound_less_than_or_equal(term: BoundReference) -> None: + bound_less_than_or_equal = BoundLessThanOrEqual(term, literal("a")) + assert str(bound_less_than_or_equal) == f"BoundLessThanOrEqual(term={str(term)}, literal=literal('a'))" + assert repr(bound_less_than_or_equal) == f"BoundLessThanOrEqual(term={repr(term)}, literal=literal('a'))" + assert bound_less_than_or_equal == eval(repr(bound_less_than_or_equal)) + + +def test_equal_to() -> None: + equal_to = EqualTo(Reference("a"), literal("a")) + assert str(equal_to) == "EqualTo(term=Reference(name='a'), literal=literal('a'))" + assert repr(equal_to) == "EqualTo(term=Reference(name='a'), literal=literal('a'))" + assert equal_to == eval(repr(equal_to)) + + +def test_not_equal_to() -> None: + not_equal_to = NotEqualTo(Reference("a"), literal("a")) + assert str(not_equal_to) == "NotEqualTo(term=Reference(name='a'), literal=literal('a'))" + assert repr(not_equal_to) == "NotEqualTo(term=Reference(name='a'), literal=literal('a'))" + assert not_equal_to == eval(repr(not_equal_to)) + + +def test_greater_than_or_equal_to() -> None: + greater_than_or_equal_to = GreaterThanOrEqual(Reference("a"), literal("a")) + assert str(greater_than_or_equal_to) == "GreaterThanOrEqual(term=Reference(name='a'), literal=literal('a'))" + assert repr(greater_than_or_equal_to) == "GreaterThanOrEqual(term=Reference(name='a'), literal=literal('a'))" + assert greater_than_or_equal_to == eval(repr(greater_than_or_equal_to)) + + +def test_greater_than() -> None: + greater_than = GreaterThan(Reference("a"), literal("a")) + assert str(greater_than) == "GreaterThan(term=Reference(name='a'), literal=literal('a'))" + assert repr(greater_than) == "GreaterThan(term=Reference(name='a'), literal=literal('a'))" + assert greater_than == eval(repr(greater_than)) + + +def test_less_than() -> None: + less_than = LessThan(Reference("a"), literal("a")) + assert str(less_than) == "LessThan(term=Reference(name='a'), literal=literal('a'))" + assert repr(less_than) == "LessThan(term=Reference(name='a'), literal=literal('a'))" + assert less_than == eval(repr(less_than)) + + +def test_less_than_or_equal() -> None: + less_than_or_equal = LessThanOrEqual(Reference("a"), literal("a")) + assert str(less_than_or_equal) == "LessThanOrEqual(term=Reference(name='a'), literal=literal('a'))" + assert repr(less_than_or_equal) == "LessThanOrEqual(term=Reference(name='a'), literal=literal('a'))" + assert less_than_or_equal == eval(repr(less_than_or_equal)) + + +def test_bound_reference_eval(table_schema_simple: Schema, foo_struct: FooStruct) -> None: """Test creating a BoundReference and evaluating it on a StructProtocol""" foo_struct.set(pos=1, value="foovalue") foo_struct.set(pos=2, value=123) @@ -672,48 +895,6 @@ def test_bound_reference(table_schema_simple, foo_struct): assert bound_ref3.eval(foo_struct) is True -def test_bound_predicate_invert(): - with pytest.raises(NotImplementedError): - _ = ~BoundPredicate( - term=BoundReference( - field=NestedField(field_id=1, name="foo", field_type=StringType(), required=False), - accessor=Accessor(position=0, inner=None), - ) - ) - - -def test_bound_unary_predicate_invert(): - with pytest.raises(NotImplementedError): - _ = ~BoundUnaryPredicate( - term=BoundReference( - field=NestedField(field_id=1, name="foo", field_type=StringType(), required=False), - accessor=Accessor(position=0, inner=None), - ) - ) - - -def test_bound_set_predicate_invert(): - with pytest.raises(NotImplementedError): - _ = ~BoundSetPredicate( - term=BoundReference( - field=NestedField(field_id=1, name="foo", field_type=StringType(), required=False), - accessor=Accessor(position=0, inner=None), - ), - literals={literal("hello"), literal("world")}, - ) - - -def test_bound_literal_predicate_invert(): - with pytest.raises(NotImplementedError): - _ = ~BoundLiteralPredicate( - term=BoundReference( - field=NestedField(field_id=1, name="foo", field_type=StringType(), required=False), - accessor=Accessor(position=0, inner=None), - ), - literal=literal("world"), - ) - - def test_non_primitive_from_byte_buffer(): with pytest.raises(ValueError) as exc_info: _ = _from_byte_buffer(ListType(element_id=1, element_type=StringType()), b"\0x00") @@ -721,26 +902,25 @@ def test_non_primitive_from_byte_buffer(): assert str(exc_info.value) == "Expected a PrimitiveType, got: " -def test_unbound_predicate_invert(): - with pytest.raises(NotImplementedError): - _ = ~UnboundPredicate(term=Reference("a")) - +def test_string_argument_unbound_unary(): + assert IsNull("a") == IsNull(Reference("a")) -def test_unbound_predicate_bind(table_schema_simple: Schema): - with pytest.raises(NotImplementedError): - _ = UnboundPredicate(term=Reference("a")).bind(table_schema_simple) +def test_string_argument_unbound_literal(): + assert EqualTo("a", "b") == EqualTo(Reference("a"), "b") -def test_unbound_unary_predicate_invert(): - with pytest.raises(NotImplementedError): - _ = ~UnaryPredicate(term=Reference("a")) +def test_string_argument_unbound_set(): + assert In("a", {"b", "c"}) == In(Reference("a"), {"b", "c"}) -def test_unbound_set_predicate_invert(): - with pytest.raises(NotImplementedError): - _ = ~SetPredicate(term=Reference("a"), literals=(literal("hello"), literal("world"))) +# __ __ ___ +# | \/ |_ _| _ \_ _ +# | |\/| | || | _/ || | +# |_| |_|\_, |_| \_, | +# |__/ |__/ -def test_unbound_literal_predicate_invert(): - with pytest.raises(NotImplementedError): - _ = ~LiteralPredicate(term=Reference("a"), literal=literal("hello")) +assert_type(EqualTo("a", "b"), EqualTo[str]) +assert_type(In("a", ("a", "b", "c")), In[str]) +assert_type(In("a", (1, 2, 3)), In[int]) +assert_type(NotIn("a", ("a", "b", "c")), NotIn[str]) diff --git a/tests/expressions/test_literals.py b/tests/expressions/test_literals.py index 96656ba825..d203195930 100644 --- a/tests/expressions/test_literals.py +++ b/tests/expressions/test_literals.py @@ -19,8 +19,10 @@ import datetime import uuid from decimal import Decimal +from typing import Set import pytest +from typing_extensions import assert_type from pyiceberg.expressions.literals import ( BinaryLiteral, @@ -34,6 +36,7 @@ FloatLiteral, IntAboveMax, IntBelowMin, + Literal, LongLiteral, StringLiteral, TimeLiteral, @@ -367,13 +370,11 @@ def test_string_to_decimal_literal(): def test_python_date_conversion(): one_day_str = "2022-03-28" - one_day_date = datetime.date(2022, 3, 28) from_str_lit = literal(one_day_str).to(DateType()) - from_date_lit = literal(one_day_date) - assert isinstance(from_date_lit, DateLiteral) - assert from_str_lit == from_date_lit + assert isinstance(from_str_lit, DateLiteral) + assert from_str_lit.value == 19079 @pytest.mark.parametrize( @@ -385,14 +386,12 @@ def test_python_date_conversion(): (literal(34.11), FloatType()), (literal(3.5028235e38), DoubleType()), (literal(Decimal(34.55).quantize(Decimal("0.01"))), DecimalType(9, 2)), - (literal(datetime.date(2017, 8, 18)), DateType()), (literal("2017-08-18"), DateType()), (literal("14:21:01.919"), TimeType()), (literal("2017-08-18T14:21:01.919"), TimestampType()), (literal("abc"), StringType()), (literal(uuid.uuid4()), UUIDType()), (literal(bytes([0x01, 0x02, 0x03])), FixedType(3)), - (literal(bytearray([0x03, 0x04, 0x05, 0x06])), BinaryType()), ], ) def test_identity_conversions(lit, primitive_type): @@ -412,8 +411,8 @@ def test_fixed_literal(): def test_binary_literal(): - bin_lit012 = literal(bytearray([0x00, 0x01, 0x02])) - bin_lit013 = literal(bytearray([0x00, 0x01, 0x03])) + bin_lit012 = literal(bytes([0x00, 0x01, 0x02])) + bin_lit013 = literal(bytes([0x00, 0x01, 0x03])) assert bin_lit012 == bin_lit012 assert bin_lit012 != bin_lit013 assert bin_lit012 < bin_lit013 @@ -424,7 +423,7 @@ def test_binary_literal(): def test_raise_on_comparison_to_none(): - bin_lit012 = literal(bytearray([0x00, 0x01, 0x02])) + bin_lit012 = literal(bytes([0x00, 0x01, 0x02])) fixed_lit012 = literal(bytes([0x00, 0x01, 0x02])) with pytest.raises(AttributeError): @@ -453,7 +452,7 @@ def test_raise_on_comparison_to_none(): def test_binary_to_fixed(): - lit = literal(bytearray([0x00, 0x01, 0x02])) + lit = literal(bytes([0x00, 0x01, 0x02])) fixed_lit = lit.to(FixedType(3)) assert fixed_lit is not None assert lit.value == fixed_lit.value @@ -464,7 +463,7 @@ def test_binary_to_fixed(): def test_binary_to_smaller_fixed_none(): - lit = literal(bytearray([0x00, 0x01, 0x02])) + lit = literal(bytes([0x00, 0x01, 0x02])) with pytest.raises(TypeError) as e: _ = lit.to(FixedType(2)) @@ -479,7 +478,7 @@ def test_fixed_to_binary(): def test_fixed_to_smaller_fixed_none(): - lit = literal(bytearray([0x00, 0x01, 0x02])).to(FixedType(3)) + lit = literal(bytes([0x00, 0x01, 0x02])).to(FixedType(3)) with pytest.raises(ValueError) as e: lit.to(lit.to(FixedType(2))) assert "Could not convert b'\\x00\\x01\\x02' into a fixed[2]" in str(e.value) @@ -734,7 +733,7 @@ def test_invalid_fixed_conversions(): def test_invalid_binary_conversions(): assert_invalid_conversions( - literal(bytearray([0x00, 0x01, 0x02])), + literal(bytes([0x00, 0x01, 0x02])), [ BooleanType(), IntegerType(), @@ -804,3 +803,19 @@ def test_string_to_decimal_type_invalid_value(): with pytest.raises(ValueError) as e: _ = literal("18.15").to(DecimalType(10, 0)) assert "Could not convert 18.15 into a decimal(10, 0), scales differ 0 <> 2" in str(e.value) + + +# __ __ ___ +# | \/ |_ _| _ \_ _ +# | |\/| | || | _/ || | +# |_| |_|\_, |_| \_, | +# |__/ |__/ + +assert_type(literal("str"), Literal[str]) +assert_type(literal(True), Literal[bool]) +assert_type(literal(123), Literal[int]) +assert_type(literal(123.4), Literal[float]) +assert_type(literal(uuid.UUID("f79c3e09-677c-4bbd-a479-3f349cb785e7")), Literal[uuid.UUID]) +assert_type(literal(bytes([0x01, 0x02, 0x03])), Literal[bytes]) +assert_type(literal(Decimal("19.25")), Literal[Decimal]) +assert_type({literal(1), literal(2), literal(3)}, Set[Literal[int]]) diff --git a/tests/expressions/test_visitors.py b/tests/expressions/test_visitors.py index fd91e8a219..62407013fc 100644 --- a/tests/expressions/test_visitors.py +++ b/tests/expressions/test_visitors.py @@ -34,16 +34,12 @@ BoundIsNull, BoundLessThan, BoundLessThanOrEqual, - BoundLiteralPredicate, BoundNotEqualTo, BoundNotIn, BoundNotNaN, BoundNotNull, - BoundPredicate, BoundReference, - BoundSetPredicate, BoundTerm, - BoundUnaryPredicate, EqualTo, GreaterThan, GreaterThanOrEqual, @@ -52,7 +48,6 @@ IsNull, LessThan, LessThanOrEqual, - LiteralPredicate, Not, NotEqualTo, NotIn, @@ -60,16 +55,8 @@ NotNull, Or, Reference, - SetPredicate, - UnaryPredicate, - UnboundPredicate, -) -from pyiceberg.expressions.literals import ( - Literal, - LongLiteral, - StringLiteral, - literal, ) +from pyiceberg.expressions.literals import Literal, literal from pyiceberg.expressions.visitors import ( BindVisitor, BooleanExpressionVisitor, @@ -289,13 +276,14 @@ def test_boolean_expression_visit_raise_not_implemented_error(): def test_bind_visitor_already_bound(table_schema_simple: Schema): - bound = BoundEqualTo( - BoundReference(table_schema_simple.find_field(1), table_schema_simple.accessor_for_field(1)), literal("hello") + bound = BoundEqualTo[str]( + term=BoundReference(table_schema_simple.find_field(1), table_schema_simple.accessor_for_field(1)), + literal=literal("hello"), ) with pytest.raises(TypeError) as exc_info: - BindVisitor(visit(bound, visitor=BindVisitor(schema=table_schema_simple))) # type: ignore + visit(bound, visitor=BindVisitor(schema=table_schema_simple)) assert ( - "Found already bound predicate: BoundEqualTo(term=BoundReference(field=NestedField(field_id=1, name='foo', field_type=StringType(), required=False), accessor=Accessor(position=0,inner=None)), literal=StringLiteral(hello))" + "Found already bound predicate: BoundEqualTo(term=BoundReference(field=NestedField(field_id=1, name='foo', field_type=StringType(), required=False), accessor=Accessor(position=0,inner=None)), literal=literal('hello'))" == str(exc_info.value) ) @@ -339,61 +327,61 @@ def test_always_false_or_always_true_expression_binding(table_schema_simple: Sch [ ( And( - In(Reference("foo"), (literal("foo"), literal("bar"))), - In(Reference("bar"), (literal(1), literal(2), literal(3))), + In(Reference("foo"), {"foo", "bar"}), + In(Reference("bar"), {1, 2, 3}), ), And( - BoundIn[str]( + BoundIn( BoundReference( field=NestedField(field_id=1, name="foo", field_type=StringType(), required=False), accessor=Accessor(position=0, inner=None), ), - {StringLiteral("foo"), StringLiteral("bar")}, + {literal("foo"), literal("bar")}, ), BoundIn[int]( BoundReference( field=NestedField(field_id=2, name="bar", field_type=IntegerType(), required=True), accessor=Accessor(position=1, inner=None), ), - {LongLiteral(1), LongLiteral(2), LongLiteral(3)}, + {literal(1), literal(2), literal(3)}, ), ), ), ( And( - In(Reference("foo"), (literal("bar"), literal("baz"))), + In(Reference("foo"), ("bar", "baz")), In( Reference("bar"), - (literal(1),), + (1,), ), In( Reference("foo"), - (literal("baz"),), + ("baz",), ), ), And( And( - BoundIn[str]( + BoundIn( BoundReference( field=NestedField(field_id=1, name="foo", field_type=StringType(), required=False), accessor=Accessor(position=0, inner=None), ), - {StringLiteral("bar"), StringLiteral("baz")}, + {literal("bar"), literal("baz")}, ), BoundEqualTo[int]( BoundReference( field=NestedField(field_id=2, name="bar", field_type=IntegerType(), required=True), accessor=Accessor(position=1, inner=None), ), - LongLiteral(1), + literal(1), ), ), - BoundEqualTo[str]( + BoundEqualTo( BoundReference( field=NestedField(field_id=1, name="foo", field_type=StringType(), required=False), accessor=Accessor(position=0, inner=None), ), - StringLiteral("baz"), + literal("baz"), ), ), ), @@ -410,61 +398,61 @@ def test_and_expression_binding(unbound_and_expression, expected_bound_expressio [ ( Or( - In(Reference("foo"), (literal("foo"), literal("bar"))), - In(Reference("bar"), (literal(1), literal(2), literal(3))), + In(Reference("foo"), ("foo", "bar")), + In(Reference("bar"), (1, 2, 3)), ), Or( - BoundIn[str]( + BoundIn( BoundReference( field=NestedField(field_id=1, name="foo", field_type=StringType(), required=False), accessor=Accessor(position=0, inner=None), ), - {StringLiteral("foo"), StringLiteral("bar")}, + {literal("foo"), literal("bar")}, ), BoundIn[int]( BoundReference( field=NestedField(field_id=2, name="bar", field_type=IntegerType(), required=True), accessor=Accessor(position=1, inner=None), ), - {LongLiteral(1), LongLiteral(2), LongLiteral(3)}, + {literal(1), literal(2), literal(3)}, ), ), ), ( Or( - In(Reference("foo"), (literal("bar"), literal("baz"))), + In(Reference("foo"), ("bar", "baz")), In( Reference("foo"), - (literal("bar"),), + ("bar",), ), In( Reference("foo"), - (literal("baz"),), + ("baz",), ), ), Or( Or( - BoundIn[str]( + BoundIn( BoundReference( field=NestedField(field_id=1, name="foo", field_type=StringType(), required=False), accessor=Accessor(position=0, inner=None), ), - {StringLiteral("bar"), StringLiteral("baz")}, + {literal("bar"), literal("baz")}, ), - BoundIn[str]( + BoundIn( BoundReference( field=NestedField(field_id=1, name="foo", field_type=StringType(), required=False), accessor=Accessor(position=0, inner=None), ), - {StringLiteral("bar")}, + {literal("bar")}, ), ), - BoundIn[str]( + BoundIn( BoundReference( field=NestedField(field_id=1, name="foo", field_type=StringType(), required=False), accessor=Accessor(position=0, inner=None), ), - {StringLiteral("baz")}, + {literal("baz")}, ), ), ), @@ -501,36 +489,36 @@ def test_or_expression_binding(unbound_or_expression, expected_bound_expression, "unbound_in_expression,expected_bound_expression", [ ( - In(Reference("foo"), (literal("foo"), literal("bar"))), - BoundIn[str]( - BoundReference[str]( + In(Reference("foo"), ("foo", "bar")), + BoundIn( + BoundReference( field=NestedField(field_id=1, name="foo", field_type=StringType(), required=False), accessor=Accessor(position=0, inner=None), ), - {StringLiteral("foo"), StringLiteral("bar")}, + {literal("foo"), literal("bar")}, ), ), ( - In(Reference("foo"), (literal("bar"), literal("baz"))), - BoundIn[str]( - BoundReference[str]( + In(Reference("foo"), ("bar", "baz")), + BoundIn( + BoundReference( field=NestedField(field_id=1, name="foo", field_type=StringType(), required=False), accessor=Accessor(position=0, inner=None), ), - {StringLiteral("bar"), StringLiteral("baz")}, + {literal("bar"), literal("baz")}, ), ), ( In( Reference("foo"), - (literal("bar"),), + ("bar",), ), BoundEqualTo( - BoundReference[str]( + BoundReference( field=NestedField(field_id=1, name="foo", field_type=StringType(), required=False), accessor=Accessor(position=0, inner=None), ), - StringLiteral("bar"), + literal("bar"), ), ), ], @@ -545,39 +533,39 @@ def test_in_expression_binding(unbound_in_expression, expected_bound_expression, "unbound_not_expression,expected_bound_expression", [ ( - Not(In(Reference("foo"), (literal("foo"), literal("bar")))), + Not(In(Reference("foo"), ("foo", "bar"))), Not( - BoundIn[str]( - BoundReference[str]( + BoundIn( + BoundReference( field=NestedField(field_id=1, name="foo", field_type=StringType(), required=False), accessor=Accessor(position=0, inner=None), ), - {StringLiteral("foo"), StringLiteral("bar")}, + {literal("foo"), literal("bar")}, ) ), ), ( Not( Or( - In(Reference("foo"), (literal("foo"), literal("bar"))), - In(Reference("foo"), (literal("foo"), literal("bar"), literal("baz"))), + In(Reference("foo"), ("foo", "bar")), + In(Reference("foo"), ("foo", "bar", "baz")), ) ), Not( Or( - BoundIn[str]( + BoundIn( BoundReference( field=NestedField(field_id=1, name="foo", field_type=StringType(), required=False), accessor=Accessor(position=0, inner=None), ), - {StringLiteral("foo"), StringLiteral("bar")}, + {literal("foo"), literal("bar")}, ), - BoundIn[str]( + BoundIn( BoundReference( field=NestedField(field_id=1, name="foo", field_type=StringType(), required=False), accessor=Accessor(position=0, inner=None), ), - {StringLiteral("foo"), StringLiteral("bar"), StringLiteral("baz")}, + {literal("foo"), literal("bar"), literal("baz")}, ), ), ), @@ -593,19 +581,19 @@ def test_not_expression_binding(unbound_not_expression, expected_bound_expressio def test_bound_boolean_expression_visitor_and_in(): """Test visiting an And and In expression with a bound boolean expression visitor""" bound_expression = And( - BoundIn[str]( + BoundIn( term=BoundReference( field=NestedField(field_id=1, name="foo", field_type=StringType(), required=False), accessor=Accessor(position=0, inner=None), ), - literals=(StringLiteral("foo"), StringLiteral("bar")), + literals={literal("foo"), literal("bar")}, ), - BoundIn[str]( + BoundIn( term=BoundReference( field=NestedField(field_id=2, name="bar", field_type=StringType(), required=False), accessor=Accessor(position=1, inner=None), ), - literals=(StringLiteral("baz"), StringLiteral("qux")), + literals={literal("baz"), literal("qux")}, ), ) visitor = FooBoundBooleanExpressionVisitor() @@ -617,21 +605,21 @@ def test_bound_boolean_expression_visitor_or(): """Test visiting an Or expression with a bound boolean expression visitor""" bound_expression = Or( Not( - BoundIn[str]( - BoundReference[str]( + BoundIn( + BoundReference( field=NestedField(field_id=1, name="foo", field_type=StringType(), required=False), accessor=Accessor(position=0, inner=None), ), - {StringLiteral("foo"), StringLiteral("bar")}, + {literal("foo"), literal("bar")}, ) ), Not( - BoundIn[str]( - BoundReference[str]( + BoundIn( + BoundReference( field=NestedField(field_id=2, name="bar", field_type=StringType(), required=False), accessor=Accessor(position=1, inner=None), ), - {StringLiteral("baz"), StringLiteral("qux")}, + {literal("baz"), literal("qux")}, ) ), ) @@ -641,12 +629,12 @@ def test_bound_boolean_expression_visitor_or(): def test_bound_boolean_expression_visitor_equal(): - bound_expression = BoundEqualTo[str]( + bound_expression = BoundEqualTo( term=BoundReference( field=NestedField(field_id=2, name="bar", field_type=StringType(), required=False), accessor=Accessor(position=1, inner=None), ), - literal=StringLiteral("foo"), + literal=literal("foo"), ) visitor = FooBoundBooleanExpressionVisitor() result = visit(bound_expression, visitor=visitor) @@ -654,12 +642,12 @@ def test_bound_boolean_expression_visitor_equal(): def test_bound_boolean_expression_visitor_not_equal(): - bound_expression = BoundNotEqualTo[str]( + bound_expression = BoundNotEqualTo( term=BoundReference( field=NestedField(field_id=1, name="foo", field_type=StringType(), required=False), accessor=Accessor(position=0, inner=None), ), - literal=StringLiteral("foo"), + literal=literal("foo"), ) visitor = FooBoundBooleanExpressionVisitor() result = visit(bound_expression, visitor=visitor) @@ -681,12 +669,12 @@ def test_bound_boolean_expression_visitor_always_false(): def test_bound_boolean_expression_visitor_in(): - bound_expression = BoundIn[str]( + bound_expression = BoundIn( term=BoundReference( field=NestedField(field_id=1, name="foo", field_type=StringType(), required=False), accessor=Accessor(position=0, inner=None), ), - literals=(StringLiteral("foo"), StringLiteral("bar")), + literals={literal("foo"), literal("bar")}, ) visitor = FooBoundBooleanExpressionVisitor() result = visit(bound_expression, visitor=visitor) @@ -694,12 +682,12 @@ def test_bound_boolean_expression_visitor_in(): def test_bound_boolean_expression_visitor_not_in(): - bound_expression = BoundNotIn[str]( + bound_expression = BoundNotIn( term=BoundReference( field=NestedField(field_id=1, name="foo", field_type=StringType(), required=False), accessor=Accessor(position=0, inner=None), ), - literals=(StringLiteral("foo"), StringLiteral("bar")), + literals={literal("foo"), literal("bar")}, ) visitor = FooBoundBooleanExpressionVisitor() result = visit(bound_expression, visitor=visitor) @@ -707,7 +695,7 @@ def test_bound_boolean_expression_visitor_not_in(): def test_bound_boolean_expression_visitor_is_nan(): - bound_expression = BoundIsNaN[str]( + bound_expression = BoundIsNaN( term=BoundReference( field=NestedField(field_id=3, name="baz", field_type=FloatType(), required=False), accessor=Accessor(position=0, inner=None), @@ -719,7 +707,7 @@ def test_bound_boolean_expression_visitor_is_nan(): def test_bound_boolean_expression_visitor_not_nan(): - bound_expression = BoundNotNaN[str]( + bound_expression = BoundNotNaN( term=BoundReference( field=NestedField(field_id=3, name="baz", field_type=FloatType(), required=False), accessor=Accessor(position=0, inner=None), @@ -731,7 +719,7 @@ def test_bound_boolean_expression_visitor_not_nan(): def test_bound_boolean_expression_visitor_is_null(): - bound_expression = BoundIsNull[str]( + bound_expression = BoundIsNull( term=BoundReference( field=NestedField(field_id=1, name="foo", field_type=StringType(), required=False), accessor=Accessor(position=0, inner=None), @@ -743,7 +731,7 @@ def test_bound_boolean_expression_visitor_is_null(): def test_bound_boolean_expression_visitor_not_null(): - bound_expression = BoundNotNull[str]( + bound_expression = BoundNotNull( term=BoundReference( field=NestedField(field_id=1, name="foo", field_type=StringType(), required=False), accessor=Accessor(position=0, inner=None), @@ -755,12 +743,12 @@ def test_bound_boolean_expression_visitor_not_null(): def test_bound_boolean_expression_visitor_greater_than(): - bound_expression = BoundGreaterThan[str]( + bound_expression = BoundGreaterThan( term=BoundReference( field=NestedField(field_id=1, name="foo", field_type=StringType(), required=False), accessor=Accessor(position=0, inner=None), ), - literal=StringLiteral("foo"), + literal=literal("foo"), ) visitor = FooBoundBooleanExpressionVisitor() result = visit(bound_expression, visitor=visitor) @@ -768,12 +756,12 @@ def test_bound_boolean_expression_visitor_greater_than(): def test_bound_boolean_expression_visitor_greater_than_or_equal(): - bound_expression = BoundGreaterThanOrEqual[str]( + bound_expression = BoundGreaterThanOrEqual( term=BoundReference( field=NestedField(field_id=1, name="foo", field_type=StringType(), required=False), accessor=Accessor(position=0, inner=None), ), - literal=StringLiteral("foo"), + literal=literal("foo"), ) visitor = FooBoundBooleanExpressionVisitor() result = visit(bound_expression, visitor=visitor) @@ -781,12 +769,12 @@ def test_bound_boolean_expression_visitor_greater_than_or_equal(): def test_bound_boolean_expression_visitor_less_than(): - bound_expression = BoundLessThan[str]( + bound_expression = BoundLessThan( term=BoundReference( field=NestedField(field_id=1, name="foo", field_type=StringType(), required=False), accessor=Accessor(position=0, inner=None), ), - literal=StringLiteral("foo"), + literal=literal("foo"), ) visitor = FooBoundBooleanExpressionVisitor() result = visit(bound_expression, visitor=visitor) @@ -794,12 +782,12 @@ def test_bound_boolean_expression_visitor_less_than(): def test_bound_boolean_expression_visitor_less_than_or_equal(): - bound_expression = BoundLessThanOrEqual[str]( + bound_expression = BoundLessThanOrEqual( term=BoundReference( field=NestedField(field_id=1, name="foo", field_type=StringType(), required=False), accessor=Accessor(position=0, inner=None), ), - literal=StringLiteral("foo"), + literal=literal("foo"), ) visitor = FooBoundBooleanExpressionVisitor() result = visit(bound_expression, visitor=visitor) @@ -807,9 +795,9 @@ def test_bound_boolean_expression_visitor_less_than_or_equal(): def test_bound_boolean_expression_visitor_raise_on_unbound_predicate(): - bound_expression = LessThanOrEqual[str]( + bound_expression = LessThanOrEqual( term=Reference("foo"), - literal=StringLiteral("foo"), + literal="foo", ) visitor = FooBoundBooleanExpressionVisitor() with pytest.raises(TypeError) as exc_info: @@ -1047,16 +1035,16 @@ def test_not_nan(schema: Schema, manifest: ManifestFile) -> None: def test_missing_stats(schema: Schema, manifest_no_stats: ManifestFile): expressions: List[BooleanExpression] = [ - LessThan(Reference[int]("id"), LongLiteral(5)), - LessThanOrEqual(Reference[int]("id"), LongLiteral(30)), - EqualTo(Reference[int]("id"), LongLiteral(70)), - GreaterThan(Reference[int]("id"), LongLiteral(78)), - GreaterThanOrEqual(Reference[int]("id"), LongLiteral(90)), - NotEqualTo(Reference[int]("id"), LongLiteral(101)), - IsNull(Reference[int]("id")), - NotNull(Reference[int]("id")), - IsNaN(Reference[float]("float")), - NotNaN(Reference[float]("float")), + LessThan(Reference("id"), 5), + LessThanOrEqual(Reference("id"), 30), + EqualTo(Reference("id"), 70), + GreaterThan(Reference("id"), 78), + GreaterThanOrEqual(Reference("id"), 90), + NotEqualTo(Reference("id"), 101), + IsNull(Reference("id")), + NotNull(Reference("id")), + IsNaN(Reference("float")), + NotNaN(Reference("float")), ] for expr in expressions: @@ -1064,11 +1052,11 @@ def test_missing_stats(schema: Schema, manifest_no_stats: ManifestFile): def test_not(schema: Schema, manifest: ManifestFile): - assert _ManifestEvalVisitor(schema, Not(LessThan(Reference[int]("id"), LongLiteral(INT_MIN_VALUE - 25)))).eval( + assert _ManifestEvalVisitor(schema, Not(LessThan(Reference("id"), INT_MIN_VALUE - 25))).eval( manifest ), "Should read: not(false)" - assert not _ManifestEvalVisitor(schema, Not(GreaterThan(Reference[int]("id"), LongLiteral(INT_MIN_VALUE - 25)))).eval( + assert not _ManifestEvalVisitor(schema, Not(GreaterThan(Reference("id"), INT_MIN_VALUE - 25))).eval( manifest ), "Should skip: not(true)" @@ -1077,24 +1065,24 @@ def test_and(schema: Schema, manifest: ManifestFile): assert not _ManifestEvalVisitor( schema, And( - LessThan(Reference[int]("id"), LongLiteral(INT_MIN_VALUE - 25)), - GreaterThanOrEqual(Reference[int]("id"), LongLiteral(INT_MIN_VALUE - 30)), + LessThan(Reference("id"), INT_MIN_VALUE - 25), + GreaterThanOrEqual(Reference("id"), INT_MIN_VALUE - 30), ), ).eval(manifest), "Should skip: and(false, true)" assert not _ManifestEvalVisitor( schema, And( - LessThan(Reference[int]("id"), LongLiteral(INT_MIN_VALUE - 25)), - GreaterThanOrEqual(Reference[int]("id"), LongLiteral(INT_MAX_VALUE + 1)), + LessThan(Reference("id"), INT_MIN_VALUE - 25), + GreaterThanOrEqual(Reference("id"), INT_MAX_VALUE + 1), ), ).eval(manifest), "Should skip: and(false, false)" assert _ManifestEvalVisitor( schema, And( - GreaterThan(Reference[int]("id"), LongLiteral(INT_MIN_VALUE - 25)), - LessThanOrEqual(Reference[int]("id"), LongLiteral(INT_MIN_VALUE)), + GreaterThan(Reference("id"), INT_MIN_VALUE - 25), + LessThanOrEqual(Reference("id"), INT_MIN_VALUE), ), ).eval(manifest), "Should read: and(true, true)" @@ -1103,386 +1091,317 @@ def test_or(schema: Schema, manifest: ManifestFile): assert not _ManifestEvalVisitor( schema, Or( - LessThan(Reference[int]("id"), LongLiteral(INT_MIN_VALUE - 25)), - GreaterThanOrEqual(Reference[int]("id"), LongLiteral(INT_MAX_VALUE + 1)), + LessThan(Reference("id"), INT_MIN_VALUE - 25), + GreaterThanOrEqual(Reference("id"), INT_MAX_VALUE + 1), ), ).eval(manifest), "Should skip: or(false, false)" assert _ManifestEvalVisitor( schema, Or( - LessThan(Reference[int]("id"), LongLiteral(INT_MIN_VALUE - 25)), - GreaterThanOrEqual(Reference[int]("id"), LongLiteral(INT_MAX_VALUE - 19)), + LessThan(Reference("id"), INT_MIN_VALUE - 25), + GreaterThanOrEqual(Reference("id"), INT_MAX_VALUE - 19), ), ).eval(manifest), "Should read: or(false, true)" def test_integer_lt(schema: Schema, manifest: ManifestFile): - assert not _ManifestEvalVisitor(schema, LessThan(Reference[int]("id"), LongLiteral(INT_MIN_VALUE - 25))).eval( + assert not _ManifestEvalVisitor(schema, LessThan(Reference("id"), INT_MIN_VALUE - 25)).eval( manifest ), "Should not read: id range below lower bound (5 < 30)" - assert not _ManifestEvalVisitor(schema, LessThan(Reference[int]("id"), LongLiteral(INT_MIN_VALUE))).eval( + assert not _ManifestEvalVisitor(schema, LessThan(Reference("id"), INT_MIN_VALUE)).eval( manifest ), "Should not read: id range below lower bound (30 is not < 30)" - assert _ManifestEvalVisitor(schema, LessThan(Reference[int]("id"), LongLiteral(INT_MIN_VALUE + 1))).eval( + assert _ManifestEvalVisitor(schema, LessThan(Reference("id"), INT_MIN_VALUE + 1)).eval( manifest ), "Should read: one possible id" - assert _ManifestEvalVisitor(schema, LessThan(Reference[int]("id"), LongLiteral(INT_MAX_VALUE))).eval( - manifest - ), "Should read: may possible ids" + assert _ManifestEvalVisitor(schema, LessThan(Reference("id"), INT_MAX_VALUE)).eval(manifest), "Should read: may possible ids" def test_integer_lt_eq(schema: Schema, manifest: ManifestFile): - assert not _ManifestEvalVisitor(schema, LessThanOrEqual(Reference[int]("id"), LongLiteral(INT_MIN_VALUE - 25))).eval( + assert not _ManifestEvalVisitor(schema, LessThanOrEqual(Reference("id"), INT_MIN_VALUE - 25)).eval( manifest ), "Should not read: id range below lower bound (5 < 30)" - assert not _ManifestEvalVisitor(schema, LessThanOrEqual(Reference[int]("id"), LongLiteral(INT_MIN_VALUE - 1))).eval( + assert not _ManifestEvalVisitor(schema, LessThanOrEqual(Reference("id"), INT_MIN_VALUE - 1)).eval( manifest ), "Should not read: id range below lower bound (29 < 30)" - assert _ManifestEvalVisitor(schema, LessThanOrEqual(Reference[int]("id"), LongLiteral(INT_MIN_VALUE))).eval( + assert _ManifestEvalVisitor(schema, LessThanOrEqual(Reference("id"), INT_MIN_VALUE)).eval( manifest ), "Should read: one possible id" - assert _ManifestEvalVisitor(schema, LessThanOrEqual(Reference[int]("id"), LongLiteral(INT_MAX_VALUE))).eval( + assert _ManifestEvalVisitor(schema, LessThanOrEqual(Reference("id"), INT_MAX_VALUE)).eval( manifest ), "Should read: many possible ids" def test_integer_gt(schema: Schema, manifest: ManifestFile): - assert not _ManifestEvalVisitor(schema, GreaterThan(Reference[int]("id"), LongLiteral(INT_MAX_VALUE + 6))).eval( + assert not _ManifestEvalVisitor(schema, GreaterThan(Reference("id"), INT_MAX_VALUE + 6)).eval( manifest ), "Should not read: id range above upper bound (85 < 79)" - assert not _ManifestEvalVisitor(schema, GreaterThan(Reference[int]("id"), LongLiteral(INT_MAX_VALUE))).eval( + assert not _ManifestEvalVisitor(schema, GreaterThan(Reference("id"), INT_MAX_VALUE)).eval( manifest ), "Should not read: id range above upper bound (79 is not > 79)" - assert _ManifestEvalVisitor(schema, GreaterThan(Reference[int]("id"), LongLiteral(INT_MAX_VALUE - 1))).eval( + assert _ManifestEvalVisitor(schema, GreaterThan(Reference("id"), INT_MAX_VALUE - 1)).eval( manifest ), "Should read: one possible id" - assert _ManifestEvalVisitor(schema, GreaterThan(Reference[int]("id"), LongLiteral(INT_MAX_VALUE - 4))).eval( + assert _ManifestEvalVisitor(schema, GreaterThan(Reference("id"), INT_MAX_VALUE - 4)).eval( manifest ), "Should read: may possible ids" def test_integer_gt_eq(schema: Schema, manifest: ManifestFile): - assert not _ManifestEvalVisitor(schema, GreaterThanOrEqual(Reference[int]("id"), LongLiteral(INT_MAX_VALUE + 6))).eval( + assert not _ManifestEvalVisitor(schema, GreaterThanOrEqual(Reference("id"), INT_MAX_VALUE + 6)).eval( manifest ), "Should not read: id range above upper bound (85 < 79)" - assert not _ManifestEvalVisitor(schema, GreaterThanOrEqual(Reference[int]("id"), LongLiteral(INT_MAX_VALUE + 1))).eval( + assert not _ManifestEvalVisitor(schema, GreaterThanOrEqual(Reference("id"), INT_MAX_VALUE + 1)).eval( manifest ), "Should not read: id range above upper bound (80 > 79)" - assert _ManifestEvalVisitor(schema, GreaterThanOrEqual(Reference[int]("id"), LongLiteral(INT_MAX_VALUE))).eval( + assert _ManifestEvalVisitor(schema, GreaterThanOrEqual(Reference("id"), INT_MAX_VALUE)).eval( manifest ), "Should read: one possible id" - assert _ManifestEvalVisitor(schema, GreaterThanOrEqual(Reference[int]("id"), LongLiteral(INT_MAX_VALUE))).eval( + assert _ManifestEvalVisitor(schema, GreaterThanOrEqual(Reference("id"), INT_MAX_VALUE)).eval( manifest ), "Should read: may possible ids" def test_integer_eq(schema: Schema, manifest: ManifestFile): - assert not _ManifestEvalVisitor(schema, EqualTo(Reference[int]("id"), LongLiteral(INT_MIN_VALUE - 25))).eval( + assert not _ManifestEvalVisitor(schema, EqualTo(Reference("id"), INT_MIN_VALUE - 25)).eval( manifest ), "Should not read: id below lower bound" - assert not _ManifestEvalVisitor(schema, EqualTo(Reference[int]("id"), LongLiteral(INT_MIN_VALUE - 1))).eval( + assert not _ManifestEvalVisitor(schema, EqualTo(Reference("id"), INT_MIN_VALUE - 1)).eval( manifest ), "Should not read: id below lower bound" - assert _ManifestEvalVisitor(schema, EqualTo(Reference[int]("id"), LongLiteral(INT_MIN_VALUE))).eval( + assert _ManifestEvalVisitor(schema, EqualTo(Reference("id"), INT_MIN_VALUE)).eval( manifest ), "Should read: id equal to lower bound" - assert _ManifestEvalVisitor(schema, EqualTo(Reference[int]("id"), LongLiteral(INT_MAX_VALUE - 4))).eval( + assert _ManifestEvalVisitor(schema, EqualTo(Reference("id"), INT_MAX_VALUE - 4)).eval( manifest ), "Should read: id between lower and upper bounds" - assert _ManifestEvalVisitor(schema, EqualTo(Reference[int]("id"), LongLiteral(INT_MAX_VALUE))).eval( + assert _ManifestEvalVisitor(schema, EqualTo(Reference("id"), INT_MAX_VALUE)).eval( manifest ), "Should read: id equal to upper bound" - assert not _ManifestEvalVisitor(schema, EqualTo(Reference[int]("id"), LongLiteral(INT_MAX_VALUE + 1))).eval( + assert not _ManifestEvalVisitor(schema, EqualTo(Reference("id"), INT_MAX_VALUE + 1)).eval( manifest ), "Should not read: id above upper bound" - assert not _ManifestEvalVisitor(schema, EqualTo(Reference[int]("id"), LongLiteral(INT_MAX_VALUE + 6))).eval( + assert not _ManifestEvalVisitor(schema, EqualTo(Reference("id"), INT_MAX_VALUE + 6)).eval( manifest ), "Should not read: id above upper bound" def test_integer_not_eq(schema: Schema, manifest: ManifestFile): - assert _ManifestEvalVisitor(schema, NotEqualTo(Reference[int]("id"), LongLiteral(INT_MIN_VALUE - 25))).eval( + assert _ManifestEvalVisitor(schema, NotEqualTo(Reference("id"), INT_MIN_VALUE - 25)).eval( manifest ), "Should read: id below lower bound" - assert _ManifestEvalVisitor(schema, NotEqualTo(Reference[int]("id"), LongLiteral(INT_MIN_VALUE - 1))).eval( + assert _ManifestEvalVisitor(schema, NotEqualTo(Reference("id"), INT_MIN_VALUE - 1)).eval( manifest ), "Should read: id below lower bound" - assert _ManifestEvalVisitor(schema, NotEqualTo(Reference[int]("id"), LongLiteral(INT_MIN_VALUE))).eval( + assert _ManifestEvalVisitor(schema, NotEqualTo(Reference("id"), INT_MIN_VALUE)).eval( manifest ), "Should read: id equal to lower bound" - assert _ManifestEvalVisitor(schema, NotEqualTo(Reference[int]("id"), LongLiteral(INT_MAX_VALUE - 4))).eval( + assert _ManifestEvalVisitor(schema, NotEqualTo(Reference("id"), INT_MAX_VALUE - 4)).eval( manifest ), "Should read: id between lower and upper bounds" - assert _ManifestEvalVisitor(schema, NotEqualTo(Reference[int]("id"), LongLiteral(INT_MAX_VALUE))).eval( + assert _ManifestEvalVisitor(schema, NotEqualTo(Reference("id"), INT_MAX_VALUE)).eval( manifest ), "Should read: id equal to upper bound" - assert _ManifestEvalVisitor(schema, NotEqualTo(Reference[int]("id"), LongLiteral(INT_MAX_VALUE + 1))).eval( + assert _ManifestEvalVisitor(schema, NotEqualTo(Reference("id"), INT_MAX_VALUE + 1)).eval( manifest ), "Should read: id above upper bound" - assert _ManifestEvalVisitor(schema, NotEqualTo(Reference[int]("id"), LongLiteral(INT_MAX_VALUE + 6))).eval( + assert _ManifestEvalVisitor(schema, NotEqualTo(Reference("id"), INT_MAX_VALUE + 6)).eval( manifest ), "Should read: id above upper bound" def test_integer_not_eq_rewritten(schema: Schema, manifest: ManifestFile): - assert _ManifestEvalVisitor(schema, Not(EqualTo(Reference[int]("id"), LongLiteral(INT_MIN_VALUE - 25)))).eval( + assert _ManifestEvalVisitor(schema, Not(EqualTo(Reference("id"), INT_MIN_VALUE - 25))).eval( manifest ), "Should read: id below lower bound" - assert _ManifestEvalVisitor(schema, Not(EqualTo(Reference[int]("id"), LongLiteral(INT_MIN_VALUE - 1)))).eval( + assert _ManifestEvalVisitor(schema, Not(EqualTo(Reference("id"), INT_MIN_VALUE - 1))).eval( manifest ), "Should read: id below lower bound" - assert _ManifestEvalVisitor(schema, Not(EqualTo(Reference[int]("id"), LongLiteral(INT_MIN_VALUE)))).eval( + assert _ManifestEvalVisitor(schema, Not(EqualTo(Reference("id"), INT_MIN_VALUE))).eval( manifest ), "Should read: id equal to lower bound" - assert _ManifestEvalVisitor(schema, Not(EqualTo(Reference[int]("id"), LongLiteral(INT_MAX_VALUE - 4)))).eval( + assert _ManifestEvalVisitor(schema, Not(EqualTo(Reference("id"), INT_MAX_VALUE - 4))).eval( manifest ), "Should read: id between lower and upper bounds" - assert _ManifestEvalVisitor(schema, Not(EqualTo(Reference[int]("id"), LongLiteral(INT_MAX_VALUE)))).eval( + assert _ManifestEvalVisitor(schema, Not(EqualTo(Reference("id"), INT_MAX_VALUE))).eval( manifest ), "Should read: id equal to upper bound" - assert _ManifestEvalVisitor(schema, Not(EqualTo(Reference[int]("id"), LongLiteral(INT_MAX_VALUE + 1)))).eval( + assert _ManifestEvalVisitor(schema, Not(EqualTo(Reference("id"), INT_MAX_VALUE + 1))).eval( manifest ), "Should read: id above upper bound" - assert _ManifestEvalVisitor(schema, Not(EqualTo(Reference[int]("id"), LongLiteral(INT_MAX_VALUE + 6)))).eval( + assert _ManifestEvalVisitor(schema, Not(EqualTo(Reference("id"), INT_MAX_VALUE + 6))).eval( manifest ), "Should read: id above upper bound" def test_integer_not_eq_rewritten_case_insensitive(schema: Schema, manifest: ManifestFile): - assert _ManifestEvalVisitor( - schema, Not(EqualTo(Reference[int]("ID"), LongLiteral(INT_MIN_VALUE - 25))), case_sensitive=False - ).eval(manifest), "Should read: id below lower bound" + assert _ManifestEvalVisitor(schema, Not(EqualTo(Reference("ID"), INT_MIN_VALUE - 25)), case_sensitive=False).eval( + manifest + ), "Should read: id below lower bound" - assert _ManifestEvalVisitor( - schema, Not(EqualTo(Reference[int]("ID"), LongLiteral(INT_MIN_VALUE - 1))), case_sensitive=False - ).eval(manifest), "Should read: id below lower bound" + assert _ManifestEvalVisitor(schema, Not(EqualTo(Reference("ID"), INT_MIN_VALUE - 1)), case_sensitive=False).eval( + manifest + ), "Should read: id below lower bound" - assert _ManifestEvalVisitor( - schema, Not(EqualTo(Reference[int]("ID"), LongLiteral(INT_MIN_VALUE))), case_sensitive=False - ).eval(manifest), "Should read: id equal to lower bound" + assert _ManifestEvalVisitor(schema, Not(EqualTo(Reference("ID"), INT_MIN_VALUE)), case_sensitive=False).eval( + manifest + ), "Should read: id equal to lower bound" - assert _ManifestEvalVisitor( - schema, Not(EqualTo(Reference[int]("ID"), LongLiteral(INT_MAX_VALUE - 4))), case_sensitive=False - ).eval(manifest), "Should read: id between lower and upper bounds" + assert _ManifestEvalVisitor(schema, Not(EqualTo(Reference("ID"), INT_MAX_VALUE - 4)), case_sensitive=False).eval( + manifest + ), "Should read: id between lower and upper bounds" - assert _ManifestEvalVisitor( - schema, Not(EqualTo(Reference[int]("ID"), LongLiteral(INT_MAX_VALUE))), case_sensitive=False - ).eval(manifest), "Should read: id equal to upper bound" + assert _ManifestEvalVisitor(schema, Not(EqualTo(Reference("ID"), INT_MAX_VALUE)), case_sensitive=False).eval( + manifest + ), "Should read: id equal to upper bound" - assert _ManifestEvalVisitor( - schema, Not(EqualTo(Reference[int]("ID"), LongLiteral(INT_MAX_VALUE + 1))), case_sensitive=False - ).eval(manifest), "Should read: id above upper bound" + assert _ManifestEvalVisitor(schema, Not(EqualTo(Reference("ID"), INT_MAX_VALUE + 1)), case_sensitive=False).eval( + manifest + ), "Should read: id above upper bound" - assert _ManifestEvalVisitor( - schema, Not(EqualTo(Reference[int]("ID"), LongLiteral(INT_MAX_VALUE + 6))), case_sensitive=False - ).eval(manifest), "Should read: id above upper bound" + assert _ManifestEvalVisitor(schema, Not(EqualTo(Reference("ID"), INT_MAX_VALUE + 6)), case_sensitive=False).eval( + manifest + ), "Should read: id above upper bound" def test_integer_in(schema: Schema, manifest: ManifestFile): - assert not _ManifestEvalVisitor( - schema, In(Reference[int]("id"), (LongLiteral(INT_MIN_VALUE - 25), LongLiteral(INT_MIN_VALUE - 24))) - ).eval(manifest), "Should not read: id below lower bound (5 < 30, 6 < 30)" + assert not _ManifestEvalVisitor(schema, In(Reference("id"), (INT_MIN_VALUE - 25, INT_MIN_VALUE - 24))).eval( + manifest + ), "Should not read: id below lower bound (5 < 30, 6 < 30)" - assert not _ManifestEvalVisitor( - schema, In(Reference[int]("id"), (LongLiteral(INT_MIN_VALUE - 2), LongLiteral(INT_MIN_VALUE - 1))) - ).eval(manifest), "Should not read: id below lower bound (28 < 30, 29 < 30)" + assert not _ManifestEvalVisitor(schema, In(Reference("id"), (INT_MIN_VALUE - 2, INT_MIN_VALUE - 1))).eval( + manifest + ), "Should not read: id below lower bound (28 < 30, 29 < 30)" - assert _ManifestEvalVisitor( - schema, In(Reference[int]("id"), (LongLiteral(INT_MIN_VALUE - 1), LongLiteral(INT_MIN_VALUE))) - ).eval(manifest), "Should read: id equal to lower bound (30 == 30)" + assert _ManifestEvalVisitor(schema, In(Reference("id"), (INT_MIN_VALUE - 1, INT_MIN_VALUE))).eval( + manifest + ), "Should read: id equal to lower bound (30 == 30)" - assert _ManifestEvalVisitor( - schema, In(Reference[int]("id"), (LongLiteral(INT_MAX_VALUE - 4), LongLiteral(INT_MAX_VALUE - 3))) - ).eval(manifest), "Should read: id between lower and upper bounds (30 < 75 < 79, 30 < 76 < 79)" + assert _ManifestEvalVisitor(schema, In(Reference("id"), (INT_MAX_VALUE - 4, INT_MAX_VALUE - 3))).eval( + manifest + ), "Should read: id between lower and upper bounds (30 < 75 < 79, 30 < 76 < 79)" - assert _ManifestEvalVisitor( - schema, In(Reference[int]("id"), (LongLiteral(INT_MAX_VALUE), LongLiteral(INT_MAX_VALUE + 1))) - ).eval(manifest), "Should read: id equal to upper bound (79 == 79)" + assert _ManifestEvalVisitor(schema, In(Reference("id"), (INT_MAX_VALUE, INT_MAX_VALUE + 1))).eval( + manifest + ), "Should read: id equal to upper bound (79 == 79)" - assert not _ManifestEvalVisitor( - schema, In(Reference[int]("id"), (LongLiteral(INT_MAX_VALUE + 1), LongLiteral(INT_MAX_VALUE + 2))) - ).eval(manifest), "Should not read: id above upper bound (80 > 79, 81 > 79)" + assert not _ManifestEvalVisitor(schema, In(Reference("id"), (INT_MAX_VALUE + 1, INT_MAX_VALUE + 2))).eval( + manifest + ), "Should not read: id above upper bound (80 > 79, 81 > 79)" - assert not _ManifestEvalVisitor( - schema, In(Reference[int]("id"), (LongLiteral(INT_MAX_VALUE + 6), LongLiteral(INT_MAX_VALUE + 7))) - ).eval(manifest), "Should not read: id above upper bound (85 > 79, 86 > 79)" + assert not _ManifestEvalVisitor(schema, In(Reference("id"), (INT_MAX_VALUE + 6, INT_MAX_VALUE + 7))).eval( + manifest + ), "Should not read: id above upper bound (85 > 79, 86 > 79)" - assert not _ManifestEvalVisitor( - schema, In(Reference[str]("all_nulls_missing_nan"), (StringLiteral("abc"), StringLiteral("def"))) - ).eval(manifest), "Should skip: in on all nulls column" + assert not _ManifestEvalVisitor(schema, In(Reference("all_nulls_missing_nan"), ("abc", "def"))).eval( + manifest + ), "Should skip: in on all nulls column" - assert _ManifestEvalVisitor(schema, In(Reference[str]("some_nulls"), (StringLiteral("abc"), StringLiteral("def")))).eval( + assert _ManifestEvalVisitor(schema, In(Reference("some_nulls"), ("abc", "def"))).eval( manifest ), "Should read: in on some nulls column" - assert _ManifestEvalVisitor(schema, In(Reference[str]("no_nulls"), (StringLiteral("abc"), StringLiteral("def")))).eval( + assert _ManifestEvalVisitor(schema, In(Reference("no_nulls"), ("abc", "def"))).eval( manifest ), "Should read: in on no nulls column" def test_integer_not_in(schema: Schema, manifest: ManifestFile): - assert _ManifestEvalVisitor( - schema, NotIn(Reference[int]("id"), (LongLiteral(INT_MIN_VALUE - 25), LongLiteral(INT_MIN_VALUE - 24))) - ).eval(manifest), "Should read: id below lower bound (5 < 30, 6 < 30)" + assert _ManifestEvalVisitor(schema, NotIn(Reference("id"), (INT_MIN_VALUE - 25, INT_MIN_VALUE - 24))).eval( + manifest + ), "Should read: id below lower bound (5 < 30, 6 < 30)" - assert _ManifestEvalVisitor( - schema, NotIn(Reference[int]("id"), (LongLiteral(INT_MIN_VALUE - 2), LongLiteral(INT_MIN_VALUE - 1))) - ).eval(manifest), "Should read: id below lower bound (28 < 30, 29 < 30)" + assert _ManifestEvalVisitor(schema, NotIn(Reference("id"), (INT_MIN_VALUE - 2, INT_MIN_VALUE - 1))).eval( + manifest + ), "Should read: id below lower bound (28 < 30, 29 < 30)" - assert _ManifestEvalVisitor( - schema, NotIn(Reference[int]("id"), (LongLiteral(INT_MIN_VALUE - 1), LongLiteral(INT_MIN_VALUE))) - ).eval(manifest), "Should read: id equal to lower bound (30 == 30)" + assert _ManifestEvalVisitor(schema, NotIn(Reference("id"), (INT_MIN_VALUE - 1, INT_MIN_VALUE))).eval( + manifest + ), "Should read: id equal to lower bound (30 == 30)" - assert _ManifestEvalVisitor( - schema, NotIn(Reference[int]("id"), (LongLiteral(INT_MAX_VALUE - 4), LongLiteral(INT_MAX_VALUE - 3))) - ).eval(manifest), "Should read: id between lower and upper bounds (30 < 75 < 79, 30 < 76 < 79)" + assert _ManifestEvalVisitor(schema, NotIn(Reference("id"), (INT_MAX_VALUE - 4, INT_MAX_VALUE - 3))).eval( + manifest + ), "Should read: id between lower and upper bounds (30 < 75 < 79, 30 < 76 < 79)" - assert _ManifestEvalVisitor( - schema, NotIn(Reference[int]("id"), (LongLiteral(INT_MAX_VALUE), LongLiteral(INT_MAX_VALUE + 1))) - ).eval(manifest), "Should read: id equal to upper bound (79 == 79)" + assert _ManifestEvalVisitor(schema, NotIn(Reference("id"), (INT_MAX_VALUE, INT_MAX_VALUE + 1))).eval( + manifest + ), "Should read: id equal to upper bound (79 == 79)" - assert _ManifestEvalVisitor( - schema, NotIn(Reference[int]("id"), (LongLiteral(INT_MAX_VALUE + 1), LongLiteral(INT_MAX_VALUE + 2))) - ).eval(manifest), "Should read: id above upper bound (80 > 79, 81 > 79)" + assert _ManifestEvalVisitor(schema, NotIn(Reference("id"), (INT_MAX_VALUE + 1, INT_MAX_VALUE + 2))).eval( + manifest + ), "Should read: id above upper bound (80 > 79, 81 > 79)" - assert _ManifestEvalVisitor( - schema, NotIn(Reference[int]("id"), (LongLiteral(INT_MAX_VALUE + 6), LongLiteral(INT_MAX_VALUE + 7))) - ).eval(manifest), "Should read: id above upper bound (85 > 79, 86 > 79)" + assert _ManifestEvalVisitor(schema, NotIn(Reference("id"), (INT_MAX_VALUE + 6, INT_MAX_VALUE + 7))).eval( + manifest + ), "Should read: id above upper bound (85 > 79, 86 > 79)" - assert _ManifestEvalVisitor( - schema, NotIn(Reference[str]("all_nulls_missing_nan"), (StringLiteral("abc"), StringLiteral("def"))) - ).eval(manifest), "Should read: notIn on no nulls column" + assert _ManifestEvalVisitor(schema, NotIn(Reference("all_nulls_missing_nan"), ("abc", "def"))).eval( + manifest + ), "Should read: notIn on no nulls column" - assert _ManifestEvalVisitor(schema, NotIn(Reference[str]("some_nulls"), (StringLiteral("abc"), StringLiteral("def")))).eval( + assert _ManifestEvalVisitor(schema, NotIn(Reference("some_nulls"), ("abc", "def"))).eval( manifest ), "Should read: in on some nulls column" - assert _ManifestEvalVisitor(schema, NotIn(Reference[str]("no_nulls"), (StringLiteral("abc"), StringLiteral("def")))).eval( + assert _ManifestEvalVisitor(schema, NotIn(Reference("no_nulls"), ("abc", "def"))).eval( manifest ), "Should read: in on no nulls column" -def test_bound_predicate_invert(): - with pytest.raises(NotImplementedError): - _ = ~BoundPredicate( - term=BoundReference( - field=NestedField(field_id=1, name="foo", field_type=StringType(), required=False), - accessor=Accessor(position=0, inner=None), - ) - ) - - -def test_bound_unary_predicate_invert(): - with pytest.raises(NotImplementedError): - _ = ~BoundUnaryPredicate( - term=BoundReference( - field=NestedField(field_id=1, name="foo", field_type=StringType(), required=False), - accessor=Accessor(position=0, inner=None), - ) - ) - - -def test_bound_set_predicate_invert(): - with pytest.raises(NotImplementedError): - _ = ~BoundSetPredicate( - term=BoundReference( - field=NestedField(field_id=1, name="foo", field_type=StringType(), required=False), - accessor=Accessor(position=0, inner=None), - ), - literals={literal("hello"), literal("world")}, - ) - - -def test_bound_literal_predicate_invert(): - with pytest.raises(NotImplementedError): - _ = ~BoundLiteralPredicate( - term=BoundReference( - field=NestedField(field_id=1, name="foo", field_type=StringType(), required=False), - accessor=Accessor(position=0, inner=None), - ), - literal=literal("world"), - ) - - -def test_unbound_predicate_invert(): - with pytest.raises(NotImplementedError): - _ = ~UnboundPredicate(term=Reference("a")) - - -def test_unbound_predicate_bind(table_schema_simple: Schema): - with pytest.raises(NotImplementedError): - _ = UnboundPredicate(term=Reference("a")).bind(table_schema_simple) - - -def test_unbound_unary_predicate_invert(): - with pytest.raises(NotImplementedError): - _ = ~UnaryPredicate(term=Reference("a")) - - -def test_unbound_set_predicate_invert(): - with pytest.raises(NotImplementedError): - _ = ~SetPredicate(term=Reference("a"), literals=(literal("hello"), literal("world"))) - - -def test_unbound_literal_predicate_invert(): - with pytest.raises(NotImplementedError): - _ = ~LiteralPredicate(term=Reference("a"), literal=literal("hello")) - - def test_rewrite_not_equal_to(): - assert rewrite_not(Not(EqualTo(Reference("x"), literal(34.56)))) == NotEqualTo(Reference("x"), literal(34.56)) + assert rewrite_not(Not(EqualTo(Reference("x"), 34.56))) == NotEqualTo(Reference("x"), 34.56) def test_rewrite_not_not_equal_to(): - assert rewrite_not(Not(NotEqualTo(Reference("x"), literal(34.56)))) == EqualTo(Reference("x"), literal(34.56)) + assert rewrite_not(Not(NotEqualTo(Reference("x"), 34.56))) == EqualTo(Reference("x"), 34.56) def test_rewrite_not_in(): - assert rewrite_not(Not(In(Reference("x"), (literal(34.56),)))) == NotIn(Reference("x"), (literal(34.56),)) + assert rewrite_not(Not(In(Reference("x"), (34.56,)))) == NotIn(Reference("x"), (34.56,)) def test_rewrite_and(): - assert rewrite_not(Not(And(EqualTo(Reference("x"), literal(34.56)), EqualTo(Reference("y"), literal(34.56)),))) == Or( - NotEqualTo(term=Reference(name="x"), literal=literal(34.56)), - NotEqualTo(term=Reference(name="y"), literal=literal(34.56)), + assert rewrite_not(Not(And(EqualTo(Reference("x"), 34.56), EqualTo(Reference("y"), 34.56),))) == Or( + NotEqualTo(term=Reference(name="x"), literal=34.56), + NotEqualTo(term=Reference(name="y"), literal=34.56), ) def test_rewrite_or(): - assert rewrite_not(Not(Or(EqualTo(Reference("x"), literal(34.56)), EqualTo(Reference("y"), literal(34.56)),))) == And( - NotEqualTo(term=Reference(name="x"), literal=literal(34.56)), - NotEqualTo(term=Reference(name="y"), literal=literal(34.56)), + assert rewrite_not(Not(Or(EqualTo(Reference("x"), 34.56), EqualTo(Reference("y"), 34.56),))) == And( + NotEqualTo(term=Reference(name="x"), literal=34.56), + NotEqualTo(term=Reference(name="y"), literal=34.56), ) diff --git a/tests/table/test_init.py b/tests/table/test_init.py index e20320701c..8e4dc7820e 100644 --- a/tests/table/test_init.py +++ b/tests/table/test_init.py @@ -199,18 +199,18 @@ def test_table_scan_select(table: Table): def test_table_scan_row_filter(table: Table): scan = table.scan() assert scan.row_filter == AlwaysTrue() - assert scan.filter_rows(EqualTo("x", 10)).row_filter == EqualTo("x", 10) # type: ignore - assert scan.filter_rows(EqualTo("x", 10)).filter_rows(In("y", (10, 11))).row_filter == And( # type: ignore - EqualTo("x", 10), In("y", (10, 11)) # type: ignore + assert scan.filter_rows(EqualTo("x", 10)).row_filter == EqualTo("x", 10) + assert scan.filter_rows(EqualTo("x", 10)).filter_rows(In("y", (10, 11))).row_filter == And( + EqualTo("x", 10), In("y", (10, 11)) ) def test_table_scan_partition_filter(table: Table): scan = table.scan() assert scan.row_filter == AlwaysTrue() - assert scan.filter_partitions(EqualTo("x", 10)).partition_filter == EqualTo("x", 10) # type: ignore - assert scan.filter_partitions(EqualTo("x", 10)).filter_partitions(In("y", (10, 11))).partition_filter == And( # type: ignore - EqualTo("x", 10), In("y", (10, 11)) # type: ignore + assert scan.filter_partitions(EqualTo("x", 10)).partition_filter == EqualTo("x", 10) + assert scan.filter_partitions(EqualTo("x", 10)).filter_partitions(In("y", (10, 11))).partition_filter == And( + EqualTo("x", 10), In("y", (10, 11)) ) From 715ae43bae1380a5980db03046e0ba0fdb0c2619 Mon Sep 17 00:00:00 2001 From: Fokko Driesprong Date: Sat, 19 Nov 2022 23:34:37 +0100 Subject: [PATCH 270/642] Python: Fix type nits from expression refactor (#6225) --- pyiceberg/expressions/__init__.py | 44 +++++++++++++++------------ pyiceberg/expressions/literals.py | 2 +- pyiceberg/expressions/visitors.py | 18 +++++------ pyiceberg/typedef.py | 2 +- tests/expressions/test_expressions.py | 2 +- 5 files changed, 36 insertions(+), 32 deletions(-) diff --git a/pyiceberg/expressions/__init__.py b/pyiceberg/expressions/__init__.py index d6ae395c17..37ca9ba3d1 100644 --- a/pyiceberg/expressions/__init__.py +++ b/pyiceberg/expressions/__init__.py @@ -70,6 +70,10 @@ class Bound(ABC): class Unbound(ABC): """Represents an unbound value expression""" + @abstractmethod + def bind(self, schema: Schema, case_sensitive: bool = True) -> Bound: + ... + @property @abstractmethod def as_bound(self) -> Type[Bound]: @@ -128,7 +132,7 @@ class UnboundTerm(Term, Unbound, ABC): @abstractmethod def bind(self, schema: Schema, case_sensitive: bool = True) -> BoundTerm: - ... # pragma: no cover + ... class Reference(UnboundTerm): @@ -513,7 +517,7 @@ def __new__(cls, term: Union[str, UnboundTerm], literals: Union[Iterable[L], Ite else: return super().__new__(cls) - def __invert__(self) -> NotIn: + def __invert__(self) -> NotIn[L]: return NotIn[L](self.term, self.literals) @property @@ -532,7 +536,7 @@ def __new__(cls, term: Union[str, UnboundTerm], literals: Union[Iterable[L], Ite else: return super().__new__(cls) - def __invert__(self) -> In: + def __invert__(self) -> In[L]: return In[L](self.term, self.literals) def __eq__(self, other: Any) -> bool: @@ -589,17 +593,17 @@ def __repr__(self) -> str: class BoundEqualTo(BoundLiteralPredicate[L]): def __invert__(self) -> BoundNotEqualTo[L]: - return BoundNotEqualTo(self.term, self.literal) + return BoundNotEqualTo[L](self.term, self.literal) class BoundNotEqualTo(BoundLiteralPredicate[L]): def __invert__(self) -> BoundEqualTo[L]: - return BoundEqualTo(self.term, self.literal) + return BoundEqualTo[L](self.term, self.literal) class BoundGreaterThanOrEqual(BoundLiteralPredicate[L]): def __invert__(self) -> BoundLessThan[L]: - return BoundLessThan(self.term, self.literal) + return BoundLessThan[L](self.term, self.literal) class BoundGreaterThan(BoundLiteralPredicate[L]): @@ -609,17 +613,17 @@ def __invert__(self) -> BoundLessThanOrEqual[L]: class BoundLessThan(BoundLiteralPredicate[L]): def __invert__(self) -> BoundGreaterThanOrEqual[L]: - return BoundGreaterThanOrEqual(self.term, self.literal) + return BoundGreaterThanOrEqual[L](self.term, self.literal) class BoundLessThanOrEqual(BoundLiteralPredicate[L]): def __invert__(self) -> BoundGreaterThan[L]: - return BoundGreaterThan(self.term, self.literal) + return BoundGreaterThan[L](self.term, self.literal) class EqualTo(LiteralPredicate[L]): def __invert__(self) -> NotEqualTo: - return NotEqualTo(self.term, self.literal) + return NotEqualTo[L](self.term, self.literal) @property def as_bound(self) -> Type[BoundEqualTo[L]]: @@ -627,17 +631,17 @@ def as_bound(self) -> Type[BoundEqualTo[L]]: class NotEqualTo(LiteralPredicate[L]): - def __invert__(self) -> EqualTo: - return EqualTo(self.term, self.literal) + def __invert__(self) -> EqualTo[L]: + return EqualTo[L](self.term, self.literal) @property def as_bound(self) -> Type[BoundNotEqualTo[L]]: return BoundNotEqualTo[L] -class LessThan(LiteralPredicate): - def __invert__(self) -> GreaterThanOrEqual: - return GreaterThanOrEqual(self.term, self.literal) +class LessThan(LiteralPredicate[L]): + def __invert__(self) -> GreaterThanOrEqual[L]: + return GreaterThanOrEqual[L](self.term, self.literal) @property def as_bound(self) -> Type[BoundLessThan[L]]: @@ -645,8 +649,8 @@ def as_bound(self) -> Type[BoundLessThan[L]]: class GreaterThanOrEqual(LiteralPredicate[L]): - def __invert__(self) -> LessThan: - return LessThan(self.term, self.literal) + def __invert__(self) -> LessThan[L]: + return LessThan[L](self.term, self.literal) @property def as_bound(self) -> Type[BoundGreaterThanOrEqual[L]]: @@ -654,8 +658,8 @@ def as_bound(self) -> Type[BoundGreaterThanOrEqual[L]]: class GreaterThan(LiteralPredicate[L]): - def __invert__(self) -> LessThanOrEqual: - return LessThanOrEqual(self.term, self.literal) + def __invert__(self) -> LessThanOrEqual[L]: + return LessThanOrEqual[L](self.term, self.literal) @property def as_bound(self) -> Type[BoundGreaterThan[L]]: @@ -663,8 +667,8 @@ def as_bound(self) -> Type[BoundGreaterThan[L]]: class LessThanOrEqual(LiteralPredicate[L]): - def __invert__(self) -> GreaterThan: - return GreaterThan(self.term, self.literal) + def __invert__(self) -> GreaterThan[L]: + return GreaterThan[L](self.term, self.literal) @property def as_bound(self) -> Type[BoundLessThanOrEqual[L]]: diff --git a/pyiceberg/expressions/literals.py b/pyiceberg/expressions/literals.py index ada6fcc8eb..c0ce0188bb 100644 --- a/pyiceberg/expressions/literals.py +++ b/pyiceberg/expressions/literals.py @@ -118,7 +118,7 @@ def literal(value: L) -> Literal[L]: if isinstance(value, float): return DoubleLiteral(value) elif isinstance(value, bool): - return BooleanLiteral(value) # type: ignore + return BooleanLiteral(value) elif isinstance(value, int): return LongLiteral(value) elif isinstance(value, str): diff --git a/pyiceberg/expressions/visitors.py b/pyiceberg/expressions/visitors.py index 283e7d2317..f1c0e00288 100644 --- a/pyiceberg/expressions/visitors.py +++ b/pyiceberg/expressions/visitors.py @@ -91,8 +91,8 @@ def visit_and(self, left_result: T, right_result: T) -> T: """Visit method for an And boolean expression Args: - left_result (R): The result of visiting the left side of the expression - right_result (R): The result of visiting the right side of the expression + left_result (T): The result of visiting the left side of the expression + right_result (T): The result of visiting the right side of the expression """ @abstractmethod @@ -100,12 +100,12 @@ def visit_or(self, left_result: T, right_result: T) -> T: """Visit method for an Or boolean expression Args: - left_result (R): The result of visiting the left side of the expression - right_result (R): The result of visiting the right side of the expression + left_result (T): The result of visiting the left side of the expression + right_result (T): The result of visiting the right side of the expression """ @abstractmethod - def visit_unbound_predicate(self, predicate: UnboundPredicate) -> T: + def visit_unbound_predicate(self, predicate: UnboundPredicate[L]) -> T: """Visit method for an unbound predicate in an expression tree Args: @@ -231,7 +231,7 @@ def visit_and(self, left_result: BooleanExpression, right_result: BooleanExpress def visit_or(self, left_result: BooleanExpression, right_result: BooleanExpression) -> BooleanExpression: return Or(left=left_result, right=right_result) - def visit_unbound_predicate(self, predicate: UnboundPredicate) -> BooleanExpression: + def visit_unbound_predicate(self, predicate: UnboundPredicate[L]) -> BooleanExpression: return predicate.bind(self.schema, case_sensitive=self.case_sensitive) def visit_bound_predicate(self, predicate: BoundPredicate[L]) -> BooleanExpression: @@ -307,10 +307,10 @@ def visit_and(self, left_result: T, right_result: T) -> T: def visit_or(self, left_result: T, right_result: T) -> T: """Visit a bound Or predicate""" - def visit_unbound_predicate(self, predicate: UnboundPredicate): + def visit_unbound_predicate(self, predicate: UnboundPredicate[L]): """Visit an unbound predicate Args: - predicate (UnboundPredicate): An unbound predicate + predicate (UnboundPredicate[L]): An unbound predicate Raises: TypeError: This always raises since an unbound predicate is not expected in a bound boolean expression """ @@ -412,7 +412,7 @@ def visit_and(self, left_result: BooleanExpression, right_result: BooleanExpress def visit_or(self, left_result: BooleanExpression, right_result: BooleanExpression) -> BooleanExpression: return Or(left=left_result, right=right_result) - def visit_unbound_predicate(self, predicate: UnboundPredicate) -> BooleanExpression: + def visit_unbound_predicate(self, predicate: UnboundPredicate[L]) -> BooleanExpression: return predicate def visit_bound_predicate(self, predicate: BoundPredicate[L]) -> BooleanExpression: diff --git a/pyiceberg/typedef.py b/pyiceberg/typedef.py index b038bfb17c..d676d68fee 100644 --- a/pyiceberg/typedef.py +++ b/pyiceberg/typedef.py @@ -40,4 +40,4 @@ def update(self, *args: Any, **kwargs: Any) -> None: RecursiveDict = Dict[str, Union[str, "RecursiveDict"]] # Represents the literal value -L = TypeVar("L", str, bool, int, float, bytes, UUID, Decimal) +L = TypeVar("L", str, bool, int, float, bytes, UUID, Decimal, covariant=True) diff --git a/tests/expressions/test_expressions.py b/tests/expressions/test_expressions.py index 2772ac2c34..d0f910cd3d 100644 --- a/tests/expressions/test_expressions.py +++ b/tests/expressions/test_expressions.py @@ -203,7 +203,7 @@ def test_ref_binding_case_insensitive_failure(table_schema_simple: Schema): def test_in_to_eq(): - assert In("x", (34.56,)) == EqualTo(Reference("x"), 34.56) + assert In("x", (34.56,)) == EqualTo("x", 34.56) def test_empty_bind_in(table_schema_simple: Schema): From 5d8f0fc7ef17534e869937713c29d09fee91622a Mon Sep 17 00:00:00 2001 From: Ryan Blue Date: Sun, 20 Nov 2022 11:35:26 -0800 Subject: [PATCH 271/642] Python: Add expression evaluator (#6127) --- pyiceberg/avro/reader.py | 2 +- pyiceberg/expressions/__init__.py | 9 +- pyiceberg/expressions/visitors.py | 88 ++++++++++++-- pyiceberg/files.py | 15 --- pyiceberg/schema.py | 2 +- pyiceberg/typedef.py | 16 +++ tests/expressions/test_evaluator.py | 179 ++++++++++++++++++++++++++++ tests/test_schema.py | 3 +- 8 files changed, 284 insertions(+), 30 deletions(-) create mode 100644 tests/expressions/test_evaluator.py diff --git a/pyiceberg/avro/reader.py b/pyiceberg/avro/reader.py index ef8b416d1c..d1d0638a9d 100644 --- a/pyiceberg/avro/reader.py +++ b/pyiceberg/avro/reader.py @@ -42,8 +42,8 @@ from uuid import UUID from pyiceberg.avro.decoder import BinaryDecoder -from pyiceberg.files import StructProtocol from pyiceberg.schema import Schema, SchemaVisitor +from pyiceberg.typedef import StructProtocol from pyiceberg.types import ( BinaryType, BooleanType, diff --git a/pyiceberg/expressions/__init__.py b/pyiceberg/expressions/__init__.py index 37ca9ba3d1..735ff68617 100644 --- a/pyiceberg/expressions/__init__.py +++ b/pyiceberg/expressions/__init__.py @@ -18,7 +18,7 @@ from __future__ import annotations from abc import ABC, abstractmethod -from functools import reduce +from functools import cached_property, reduce from typing import ( Any, Generic, @@ -29,9 +29,8 @@ ) from pyiceberg.expressions.literals import Literal, literal -from pyiceberg.files import StructProtocol from pyiceberg.schema import Accessor, Schema -from pyiceberg.typedef import L +from pyiceberg.typedef import L, StructProtocol from pyiceberg.types import DoubleType, FloatType, NestedField from pyiceberg.utils.singleton import Singleton @@ -459,6 +458,10 @@ def __init__(self, term: BoundTerm[L], literals: Set[Literal[L]]): super().__init__(term) # type: ignore self.literals = _to_literal_set(literals) # pylint: disable=W0621 + @cached_property + def value_set(self) -> Set[L]: + return {lit.value for lit in self.literals} + def __str__(self): # Sort to make it deterministic return f"{str(self.__class__.__name__)}({str(self.term)}, {{{', '.join(sorted([str(literal) for literal in self.literals]))}}})" diff --git a/pyiceberg/expressions/visitors.py b/pyiceberg/expressions/visitors.py index f1c0e00288..84d7e12058 100644 --- a/pyiceberg/expressions/visitors.py +++ b/pyiceberg/expressions/visitors.py @@ -53,6 +53,7 @@ from pyiceberg.manifest import ManifestFile, PartitionFieldSummary from pyiceberg.schema import Schema from pyiceberg.table import PartitionSpec +from pyiceberg.typedef import StructProtocol from pyiceberg.types import ( DoubleType, FloatType, @@ -240,11 +241,11 @@ def visit_bound_predicate(self, predicate: BoundPredicate[L]) -> BooleanExpressi class BoundBooleanExpressionVisitor(BooleanExpressionVisitor[T], ABC): @abstractmethod - def visit_in(self, term: BoundTerm[L], literals: Set[Literal[L]]) -> T: + def visit_in(self, term: BoundTerm[L], literals: Set[L]) -> T: """Visit a bound In predicate""" @abstractmethod - def visit_not_in(self, term: BoundTerm[L], literals: Set[Literal[L]]) -> T: + def visit_not_in(self, term: BoundTerm[L], literals: Set[L]) -> T: """Visit a bound NotIn predicate""" @abstractmethod @@ -331,12 +332,12 @@ def visit_bound_predicate(expr: BoundPredicate[L], _: BooleanExpressionVisitor[T @visit_bound_predicate.register(BoundIn) def _(expr: BoundIn[L], visitor: BoundBooleanExpressionVisitor[T]) -> T: - return visitor.visit_in(term=expr.term, literals=expr.literals) + return visitor.visit_in(term=expr.term, literals=expr.value_set) @visit_bound_predicate.register(BoundNotIn) def _(expr: BoundNotIn[L], visitor: BoundBooleanExpressionVisitor[T]) -> T: - return visitor.visit_not_in(term=expr.term, literals=expr.literals) + return visitor.visit_not_in(term=expr.term, literals=expr.value_set) @visit_bound_predicate.register(BoundIsNaN) @@ -419,6 +420,77 @@ def visit_bound_predicate(self, predicate: BoundPredicate[L]) -> BooleanExpressi return predicate +def expression_evaluator( + schema: Schema, unbound: BooleanExpression, case_sensitive: bool = True +) -> Callable[[StructProtocol], bool]: + return _ExpressionEvaluator(schema, unbound, case_sensitive).eval + + +class _ExpressionEvaluator(BoundBooleanExpressionVisitor[bool]): + bound: BooleanExpression + struct: StructProtocol + + def __init__(self, schema: Schema, unbound: BooleanExpression, case_sensitive: bool = True): + self.bound = bind(schema, unbound, case_sensitive) + + def eval(self, struct: StructProtocol) -> bool: + self.struct = struct + return visit(self.bound, self) + + def visit_in(self, term: BoundTerm[L], literals: Set[L]) -> bool: + return term.eval(self.struct) in literals + + def visit_not_in(self, term: BoundTerm[L], literals: Set[L]) -> bool: + return term.eval(self.struct) not in literals + + def visit_is_nan(self, term: BoundTerm[L]) -> bool: + val = term.eval(self.struct) + return val != val + + def visit_not_nan(self, term: BoundTerm[L]) -> bool: + val = term.eval(self.struct) + return val == val + + def visit_is_null(self, term: BoundTerm[L]) -> bool: + return term.eval(self.struct) is None + + def visit_not_null(self, term: BoundTerm[L]) -> bool: + return term.eval(self.struct) is not None + + def visit_equal(self, term: BoundTerm[L], literal: Literal[L]) -> bool: + return term.eval(self.struct) == literal.value + + def visit_not_equal(self, term: BoundTerm[L], literal: Literal[L]) -> bool: + return term.eval(self.struct) != literal.value + + def visit_greater_than_or_equal(self, term: BoundTerm[L], literal: Literal[L]) -> bool: + return term.eval(self.struct) >= literal.value + + def visit_greater_than(self, term: BoundTerm[L], literal: Literal[L]) -> bool: + return term.eval(self.struct) > literal.value + + def visit_less_than(self, term: BoundTerm[L], literal: Literal[L]) -> bool: + return term.eval(self.struct) < literal.value + + def visit_less_than_or_equal(self, term: BoundTerm[L], literal: Literal[L]) -> bool: + return term.eval(self.struct) <= literal.value + + def visit_true(self) -> bool: + return True + + def visit_false(self) -> bool: + return False + + def visit_not(self, child_result: bool) -> bool: + return not child_result + + def visit_and(self, left_result: bool, right_result: bool) -> bool: + return left_result and right_result + + def visit_or(self, left_result: bool, right_result: bool) -> bool: + return left_result or right_result + + ROWS_MIGHT_MATCH = True ROWS_CANNOT_MATCH = False IN_PREDICATE_LIMIT = 200 @@ -445,7 +517,7 @@ def eval(self, manifest: ManifestFile) -> bool: # No partition information return ROWS_MIGHT_MATCH - def visit_in(self, term: BoundTerm[L], literals: Set[Literal[L]]) -> bool: + def visit_in(self, term: BoundTerm[L], literals: Set[L]) -> bool: pos = term.ref().accessor.position field = self.partition_fields[pos] @@ -457,17 +529,17 @@ def visit_in(self, term: BoundTerm[L], literals: Set[Literal[L]]) -> bool: lower = _from_byte_buffer(term.ref().field.field_type, field.lower_bound) - if all(lower > val.value for val in literals): + if all(lower > val for val in literals): return ROWS_CANNOT_MATCH if field.upper_bound is not None: upper = _from_byte_buffer(term.ref().field.field_type, field.upper_bound) - if all(upper < val.value for val in literals): + if all(upper < val for val in literals): return ROWS_CANNOT_MATCH return ROWS_MIGHT_MATCH - def visit_not_in(self, term: BoundTerm[L], literals: Set[Literal[L]]) -> bool: + def visit_not_in(self, term: BoundTerm[L], literals: Set[L]) -> bool: # because the bounds are not necessarily a min or max value, this cannot be answered using # them. notIn(col, {X, ...}) with (X, Y) doesn't guarantee that X is a value in col. return ROWS_MIGHT_MATCH diff --git a/pyiceberg/files.py b/pyiceberg/files.py index e560092357..b832bbbfe7 100644 --- a/pyiceberg/files.py +++ b/pyiceberg/files.py @@ -14,9 +14,7 @@ # KIND, either express or implied. See the License for the # specific language governing permissions and limitations # under the License. -from abc import abstractmethod from enum import Enum, auto -from typing import Any, Protocol, runtime_checkable class FileContentType(Enum): @@ -34,16 +32,3 @@ class FileFormat(Enum): PARQUET = auto() AVRO = auto() METADATA = auto() - - -@runtime_checkable -class StructProtocol(Protocol): # pragma: no cover - """A generic protocol used by accessors to get and set at positions of an object""" - - @abstractmethod - def get(self, pos: int) -> Any: - ... - - @abstractmethod - def set(self, pos: int, value: Any) -> None: - ... diff --git a/pyiceberg/schema.py b/pyiceberg/schema.py index 82a9533392..9472335f64 100644 --- a/pyiceberg/schema.py +++ b/pyiceberg/schema.py @@ -35,7 +35,7 @@ from pydantic import Field, PrivateAttr -from pyiceberg.files import StructProtocol +from pyiceberg.typedef import StructProtocol from pyiceberg.types import ( IcebergType, ListType, diff --git a/pyiceberg/typedef.py b/pyiceberg/typedef.py index d676d68fee..97e23e7888 100644 --- a/pyiceberg/typedef.py +++ b/pyiceberg/typedef.py @@ -14,13 +14,16 @@ # KIND, either express or implied. See the License for the # specific language governing permissions and limitations # under the License. +from abc import abstractmethod from decimal import Decimal from typing import ( Any, Dict, + Protocol, Tuple, TypeVar, Union, + runtime_checkable, ) from uuid import UUID @@ -41,3 +44,16 @@ def update(self, *args: Any, **kwargs: Any) -> None: # Represents the literal value L = TypeVar("L", str, bool, int, float, bytes, UUID, Decimal, covariant=True) + + +@runtime_checkable +class StructProtocol(Protocol): # pragma: no cover + """A generic protocol used by accessors to get and set at positions of an object""" + + @abstractmethod + def get(self, pos: int) -> Any: + ... + + @abstractmethod + def set(self, pos: int, value: Any) -> None: + ... diff --git a/tests/expressions/test_evaluator.py b/tests/expressions/test_evaluator.py new file mode 100644 index 0000000000..07a553947e --- /dev/null +++ b/tests/expressions/test_evaluator.py @@ -0,0 +1,179 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +from typing import Any, List + +from pyiceberg.expressions import ( + AlwaysFalse, + AlwaysTrue, + And, + EqualTo, + GreaterThan, + GreaterThanOrEqual, + In, + IsNaN, + IsNull, + LessThan, + LessThanOrEqual, + Not, + NotEqualTo, + NotIn, + NotNaN, + NotNull, + Or, +) +from pyiceberg.expressions.visitors import expression_evaluator +from pyiceberg.schema import Schema +from pyiceberg.typedef import StructProtocol +from pyiceberg.types import ( + DoubleType, + LongType, + NestedField, + StringType, +) + + +class Record(StructProtocol): + data: List[Any] + + def __init__(self, *values): + self.data = list(values) + + def get(self, pos: int) -> Any: + return self.data[pos] + + def set(self, pos: int, value: Any) -> None: + self.data[pos] = value + + +SIMPLE_SCHEMA = Schema( + NestedField(id=1, name="id", field_type=LongType()), NestedField(id=2, name="data", field_type=StringType(), required=False) +) + +FLOAT_SCHEMA = Schema( + NestedField(id=1, name="id", field_type=LongType()), NestedField(id=2, name="f", field_type=DoubleType(), required=False) +) + + +def test_true(): + evaluate = expression_evaluator(SIMPLE_SCHEMA, AlwaysTrue()) + assert evaluate(Record(1, "a")) + + +def test_false(): + evaluate = expression_evaluator(SIMPLE_SCHEMA, AlwaysFalse()) + assert not evaluate(Record(1, "a")) + + +def test_less_than(): + evaluate = expression_evaluator(SIMPLE_SCHEMA, LessThan("id", 3)) + assert evaluate(Record(2, "a")) + assert not evaluate(Record(3, "a")) + + +def test_less_than_or_equal(): + evaluate = expression_evaluator(SIMPLE_SCHEMA, LessThanOrEqual("id", 3)) + assert evaluate(Record(1, "a")) + assert evaluate(Record(3, "a")) + assert not evaluate(Record(4, "a")) + + +def test_greater_than(): + evaluate = expression_evaluator(SIMPLE_SCHEMA, GreaterThan("id", 3)) + assert not evaluate(Record(1, "a")) + assert not evaluate(Record(3, "a")) + assert evaluate(Record(4, "a")) + + +def test_greater_than_or_equal(): + evaluate = expression_evaluator(SIMPLE_SCHEMA, GreaterThanOrEqual("id", 3)) + assert not evaluate(Record(2, "a")) + assert evaluate(Record(3, "a")) + assert evaluate(Record(4, "a")) + + +def test_equal_to(): + evaluate = expression_evaluator(SIMPLE_SCHEMA, EqualTo("id", 3)) + assert not evaluate(Record(2, "a")) + assert evaluate(Record(3, "a")) + assert not evaluate(Record(4, "a")) + + +def test_not_equal_to(): + evaluate = expression_evaluator(SIMPLE_SCHEMA, NotEqualTo("id", 3)) + assert evaluate(Record(2, "a")) + assert not evaluate(Record(3, "a")) + assert evaluate(Record(4, "a")) + + +def test_in(): + evaluate = expression_evaluator(SIMPLE_SCHEMA, In("id", [1, 2, 3])) + assert evaluate(Record(2, "a")) + assert evaluate(Record(3, "a")) + assert not evaluate(Record(4, "a")) + + +def test_not_in(): + evaluate = expression_evaluator(SIMPLE_SCHEMA, NotIn("id", [1, 2, 3])) + assert not evaluate(Record(2, "a")) + assert not evaluate(Record(3, "a")) + assert evaluate(Record(4, "a")) + + +def test_is_null(): + evaluate = expression_evaluator(SIMPLE_SCHEMA, IsNull("data")) + assert not evaluate(Record(2, "a")) + assert evaluate(Record(3, None)) + + +def test_not_null(): + evaluate = expression_evaluator(SIMPLE_SCHEMA, NotNull("data")) + assert evaluate(Record(2, "a")) + assert not evaluate(Record(3, None)) + + +def test_is_nan(): + evaluate = expression_evaluator(FLOAT_SCHEMA, IsNaN("f")) + assert not evaluate(Record(2, 0.0)) + assert not evaluate(Record(3, float("infinity"))) + assert evaluate(Record(4, float("nan"))) + + +def test_not_nan(): + evaluate = expression_evaluator(FLOAT_SCHEMA, NotNaN("f")) + assert evaluate(Record(2, 0.0)) + assert evaluate(Record(3, float("infinity"))) + assert not evaluate(Record(4, float("nan"))) + + +def test_not(): + evaluate = expression_evaluator(SIMPLE_SCHEMA, Not(LessThan("id", 3))) + assert not evaluate(Record(2, "a")) + assert evaluate(Record(3, "a")) + + +def test_and(): + evaluate = expression_evaluator(SIMPLE_SCHEMA, And(LessThan("id", 3), GreaterThan("id", 1))) + assert not evaluate(Record(1, "a")) + assert evaluate(Record(2, "a")) + assert not evaluate(Record(3, "a")) + + +def test_or(): + evaluate = expression_evaluator(SIMPLE_SCHEMA, Or(LessThan("id", 2), GreaterThan("id", 2))) + assert evaluate(Record(1, "a")) + assert not evaluate(Record(2, "a")) + assert evaluate(Record(3, "a")) diff --git a/tests/test_schema.py b/tests/test_schema.py index b2a2790438..ee8513bee2 100644 --- a/tests/test_schema.py +++ b/tests/test_schema.py @@ -22,9 +22,8 @@ from pyiceberg import schema from pyiceberg.expressions import Accessor -from pyiceberg.files import StructProtocol from pyiceberg.schema import Schema, build_position_accessors, prune_columns -from pyiceberg.typedef import EMPTY_DICT +from pyiceberg.typedef import EMPTY_DICT, StructProtocol from pyiceberg.types import ( BooleanType, FloatType, From 21446fe870f319bbd941fe7b3d23b8e8ace200a8 Mon Sep 17 00:00:00 2001 From: Ryan Blue Date: Mon, 21 Nov 2022 04:29:48 -0800 Subject: [PATCH 272/642] Python: Minor fixes to expression types (#6228) * Python: Minor fixes to expression types. * Fix bind return types. --- pyiceberg/expressions/__init__.py | 26 +++++++++++++++----------- pyiceberg/expressions/visitors.py | 6 +++--- tests/expressions/test_expressions.py | 20 ++++++++++---------- 3 files changed, 28 insertions(+), 24 deletions(-) diff --git a/pyiceberg/expressions/__init__.py b/pyiceberg/expressions/__init__.py index 735ff68617..7aabd5181a 100644 --- a/pyiceberg/expressions/__init__.py +++ b/pyiceberg/expressions/__init__.py @@ -25,6 +25,7 @@ Iterable, Set, Type, + TypeVar, Union, ) @@ -66,11 +67,14 @@ class Bound(ABC): """Represents a bound value expression""" -class Unbound(ABC): +B = TypeVar("B") + + +class Unbound(Generic[B], ABC): """Represents an unbound value expression""" @abstractmethod - def bind(self, schema: Schema, case_sensitive: bool = True) -> Bound: + def bind(self, schema: Schema, case_sensitive: bool = True) -> B: ... @property @@ -126,11 +130,11 @@ def ref(self) -> BoundReference[L]: return self -class UnboundTerm(Term, Unbound, ABC): +class UnboundTerm(Term, Unbound[BoundTerm[L]], ABC): """Represents an unbound term.""" @abstractmethod - def bind(self, schema: Schema, case_sensitive: bool = True) -> BoundTerm: + def bind(self, schema: Schema, case_sensitive: bool = True) -> BoundTerm[L]: ... @@ -306,7 +310,7 @@ def __eq__(self, other: Any) -> bool: return False -class UnboundPredicate(Generic[L], Unbound, BooleanExpression, ABC): +class UnboundPredicate(Generic[L], Unbound[BooleanExpression], BooleanExpression, ABC): term: UnboundTerm def __init__(self, term: Union[str, UnboundTerm]): @@ -316,7 +320,7 @@ def __eq__(self, other): return self.term == other.term if isinstance(other, UnboundPredicate) else False @abstractmethod - def bind(self, schema: Schema, case_sensitive: bool = True) -> BoundPredicate: + def bind(self, schema: Schema, case_sensitive: bool = True) -> BooleanExpression: ... @property @@ -326,7 +330,7 @@ def as_bound(self) -> Type[BoundPredicate]: class UnaryPredicate(UnboundPredicate[Any], ABC): - def bind(self, schema: Schema, case_sensitive: bool = True) -> BoundUnaryPredicate: + def bind(self, schema: Schema, case_sensitive: bool = True) -> BoundUnaryPredicate[Any]: bound_term = self.term.bind(schema, case_sensitive) return self.as_bound(bound_term) @@ -335,7 +339,7 @@ def __repr__(self) -> str: @property @abstractmethod - def as_bound(self) -> Type[BoundUnaryPredicate[L]]: + def as_bound(self) -> Type[BoundUnaryPredicate[Any]]: ... @@ -422,7 +426,7 @@ def as_bound(self) -> Type[BoundNotNaN[L]]: return BoundNotNaN[L] -class SetPredicate(Generic[L], UnboundPredicate[L], ABC): +class SetPredicate(UnboundPredicate[L], ABC): literals: Set[Literal[L]] def __init__(self, term: Union[str, UnboundTerm], literals: Union[Iterable[L], Iterable[Literal[L]]]): @@ -552,7 +556,7 @@ def as_bound(self) -> Type[BoundNotIn[L]]: return BoundNotIn[L] -class LiteralPredicate(Generic[L], UnboundPredicate[L], ABC): +class LiteralPredicate(UnboundPredicate[L], ABC): literal: Literal[L] def __init__(self, term: Union[str, UnboundTerm], literal: Union[L, Literal[L]]): # pylint: disable=W0621 @@ -625,7 +629,7 @@ def __invert__(self) -> BoundGreaterThan[L]: class EqualTo(LiteralPredicate[L]): - def __invert__(self) -> NotEqualTo: + def __invert__(self) -> NotEqualTo[L]: return NotEqualTo[L](self.term, self.literal) @property diff --git a/pyiceberg/expressions/visitors.py b/pyiceberg/expressions/visitors.py index 84d7e12058..366d52d6fe 100644 --- a/pyiceberg/expressions/visitors.py +++ b/pyiceberg/expressions/visitors.py @@ -110,7 +110,7 @@ def visit_unbound_predicate(self, predicate: UnboundPredicate[L]) -> T: """Visit method for an unbound predicate in an expression tree Args: - predicate (UnboundPredicate): An instance of an UnboundPredicate + predicate (UnboundPredicate[L): An instance of an UnboundPredicate """ @abstractmethod @@ -130,7 +130,7 @@ def visit(obj, visitor: BooleanExpressionVisitor[T]) -> T: Args: obj(BooleanExpression): An instance of a BooleanExpression - visitor(BooleanExpressionVisitor[R]): An instance of an implementation of the generic BooleanExpressionVisitor base class + visitor(BooleanExpressionVisitor[T]): An instance of an implementation of the generic BooleanExpressionVisitor base class Raises: NotImplementedError: If attempting to visit an unsupported expression @@ -166,7 +166,7 @@ def _(obj: And, visitor: BooleanExpressionVisitor[T]) -> T: @visit.register(UnboundPredicate) -def _(obj: UnboundPredicate, visitor: BooleanExpressionVisitor[T]) -> T: +def _(obj: UnboundPredicate[L], visitor: BooleanExpressionVisitor[T]) -> T: """Visit an unbound boolean expression with a concrete BooleanExpressionVisitor""" return visitor.visit_unbound_predicate(predicate=obj) diff --git a/tests/expressions/test_expressions.py b/tests/expressions/test_expressions.py index d0f910cd3d..8115088ecd 100644 --- a/tests/expressions/test_expressions.py +++ b/tests/expressions/test_expressions.py @@ -180,7 +180,7 @@ def test_notnan_bind_nonfloat(): def test_ref_binding_case_sensitive(table_schema_simple: Schema): ref = Reference("foo") - bound = BoundReference[str](table_schema_simple.find_field(1), table_schema_simple.accessor_for_field(1)) + bound = BoundReference(table_schema_simple.find_field(1), table_schema_simple.accessor_for_field(1)) assert ref.bind(table_schema_simple, case_sensitive=True) == bound @@ -192,7 +192,7 @@ def test_ref_binding_case_sensitive_failure(table_schema_simple: Schema): def test_ref_binding_case_insensitive(table_schema_simple: Schema): ref = Reference("Foo") - bound = BoundReference[str](table_schema_simple.find_field(1), table_schema_simple.accessor_for_field(1)) + bound = BoundReference(table_schema_simple.find_field(1), table_schema_simple.accessor_for_field(1)) assert ref.bind(table_schema_simple, case_sensitive=False) == bound @@ -207,12 +207,12 @@ def test_in_to_eq(): def test_empty_bind_in(table_schema_simple: Schema): - bound = BoundIn(BoundReference[str](table_schema_simple.find_field(1), table_schema_simple.accessor_for_field(1)), set()) + bound = BoundIn(BoundReference(table_schema_simple.find_field(1), table_schema_simple.accessor_for_field(1)), set()) assert bound == AlwaysFalse() def test_empty_bind_not_in(table_schema_simple: Schema): - bound = BoundNotIn(BoundReference[str](table_schema_simple.find_field(1), table_schema_simple.accessor_for_field(1)), set()) + bound = BoundNotIn(BoundReference(table_schema_simple.find_field(1), table_schema_simple.accessor_for_field(1)), set()) assert bound == AlwaysTrue() @@ -257,7 +257,7 @@ def test_not_in_equal(): def test_bind_in(table_schema_simple: Schema): - bound = BoundIn[str]( + bound = BoundIn( BoundReference(table_schema_simple.find_field(1), table_schema_simple.accessor_for_field(1)), {literal("hello"), literal("world")}, ) @@ -265,29 +265,29 @@ def test_bind_in(table_schema_simple: Schema): def test_bind_in_invert(table_schema_simple: Schema): - bound = BoundIn[str]( + bound = BoundIn( BoundReference(table_schema_simple.find_field(1), table_schema_simple.accessor_for_field(1)), {literal("hello"), literal("world")}, ) - assert ~bound == BoundNotIn[str]( + assert ~bound == BoundNotIn( BoundReference(table_schema_simple.find_field(1), table_schema_simple.accessor_for_field(1)), {literal("hello"), literal("world")}, ) def test_bind_not_in_invert(table_schema_simple: Schema): - bound = BoundNotIn[str]( + bound = BoundNotIn( BoundReference(table_schema_simple.find_field(1), table_schema_simple.accessor_for_field(1)), {literal("hello"), literal("world")}, ) - assert ~bound == BoundIn[str]( + assert ~bound == BoundIn( BoundReference(table_schema_simple.find_field(1), table_schema_simple.accessor_for_field(1)), {literal("hello"), literal("world")}, ) def test_bind_dedup(table_schema_simple: Schema): - bound = BoundIn[str]( + bound = BoundIn( BoundReference(table_schema_simple.find_field(1), table_schema_simple.accessor_for_field(1)), {literal("hello"), literal("world")}, ) From 2ade9ecdc07230c33944c9de4857703bf2abd1ba Mon Sep 17 00:00:00 2001 From: Rushan Jiang Date: Mon, 21 Nov 2022 09:35:30 -0500 Subject: [PATCH 273/642] Python: Add support for GlueCatalog (#6034) * add glueCatalog bone * add boto3 * first draft based on previous PR * add personal test and implements load_table * test create table functionality * tested drop table functionality * reformat create_table and load_table and perform some test * add some todo item * reimplement create_table and load_table * update load_table's io initialization * suggestions 1 - 4 * remain: refactor constant name * refactor code structure and add exception caught * fix some typo and make constants defined above * format fix partially * update environment * fix style issue * let comment be more specific * add ignore missing boto3 to mypy config * skip test_glue temporarily * fix typo PROP_GLUE_TABLE_TYPE * change dict[str, str] to properties * fix all sanity issue * make boto3 optional * implement list namespaces, change aws account for test * fix keyerror in default warehouse location * fix keyerror in default warehouse location * update apache header * add unit test prototype * format unit test * fix the fixture scope issue * add comment for the gist * add apache head * move fixtures to conftest in utils * rollback changes to hive apache license header * update nit, error messages. solve case sensitive error. remove duplicate code in create_table * update format * put io instantiation to _convert_glue_to_iceberg and make it consistent * make boto3 and moto extra * update makeFile to include boto3 and moto * add NoSuchIcebergTableError * use NoSuchPropertyException * make global var consistent with hive * move common global variable to pyiceberg.catalog.base * complete rest of gluecatalog first_draft * add comments for class methods * add integration test and fix some small bugs * add next_token for list_tables * add next_token for list_namespaces * make create_table's description key consistent with that of create_namespace * add unit tests for table-related operations * add unit tests for namespace-related operations * formalize integration test * aggregate all GlueCatalog instance to a fixture * remove redundant print * optimize test for rename table * refactor duplicated code segment and details * fix some potential keyError issue * reimplement purge_table to correct logic * add load glue catalog and move shared variables to __init__.py * fix format issue * move delete_data_files, delete_files to __init__.py * rename and make constant strings variables * fix minor style issues * fix configuration after rebase * add doc for glue catalog * handle loading database created by AWS Glue Console * add comment to helper functions * add check to disable hierarchical namespace, add removing strips for database_location and warehouse path * refactor the rename_table logic to add check if the table is a valid iceberg table * revise comments * fix create table input key error * persist table description when renaming the table * update after rebase --- Makefile | 2 +- mkdocs/docs/index.md | 9 + poetry.lock | 286 ++++++++++++- pyiceberg/catalog/__init__.py | 59 +++ pyiceberg/catalog/glue.py | 549 +++++++++++++++++++++++++ pyiceberg/exceptions.py | 4 + pyproject.toml | 19 + tests/catalog/integration_test_glue.py | 324 +++++++++++++++ tests/catalog/test_glue.py | 471 +++++++++++++++++++++ tests/conftest.py | 113 +++++ 10 files changed, 1831 insertions(+), 5 deletions(-) create mode 100644 pyiceberg/catalog/glue.py create mode 100644 tests/catalog/integration_test_glue.py create mode 100644 tests/catalog/test_glue.py diff --git a/Makefile b/Makefile index c934a563d0..32cd895f53 100644 --- a/Makefile +++ b/Makefile @@ -17,7 +17,7 @@ install: pip install poetry - poetry install -E pyarrow -E hive -E s3fs + poetry install -E pyarrow -E hive -E s3fs -E glue check-license: ./dev/check-license diff --git a/mkdocs/docs/index.md b/mkdocs/docs/index.md index b7597d0f00..c05af4effa 100644 --- a/mkdocs/docs/index.md +++ b/mkdocs/docs/index.md @@ -40,6 +40,7 @@ You can mix and match optional dependencies: | Key | Description: | |-----------|----------------------------------------------------------------------| | hive | Support for the Hive metastore | +| glue | Support for AWS Glue | | pyarrow | PyArrow as a FileIO implementation to interact with the object store | | s3fs | S3FS as a FileIO implementation to interact with the object store | | snappy | Support for snappy Avro compression | @@ -99,6 +100,9 @@ catalog: cert: /absolute/path/to/client.crt key: /absolute/path/to/client.key cabundle: /absolute/path/to/cabundle.pem + + glue: + type: glue ``` Lastly, you can also set it using environment variables: @@ -108,10 +112,15 @@ export PYICEBERG_CATALOG__DEFAULT__URI=thrift://localhost:9083 export PYICEBERG_CATALOG__REST__URI=http://rest-catalog/ws/ export PYICEBERG_CATALOG__REST__CREDENTIAL=t-1234:secret + +export PYICEBERG_CATALOG__GLUE__TYPE=glue ``` Where the structure is equivalent to the YAML. The levels are separated using a double underscore (`__`). +If you want to use AWS Glue as the catalog, you can use the last two ways to configure the pyiceberg and refer +[How to configure AWS credentials](https://docs.aws.amazon.com/cli/latest/userguide/cli-chap-configure.html) to set your AWS account credentials locally. + ## FileIO configuration For the FileIO there are several configuration options available: diff --git a/poetry.lock b/poetry.lock index e5157bda8c..caccecda60 100644 --- a/poetry.lock +++ b/poetry.lock @@ -80,12 +80,28 @@ docs = ["furo", "sphinx", "sphinx-notfound-page", "zope.interface"] tests = ["cloudpickle", "coverage[toml] (>=5.0.2)", "hypothesis", "mypy (>=0.900,!=0.940)", "pympler", "pytest (>=4.3.0)", "pytest-mypy-plugins", "zope.interface"] tests-no-zope = ["cloudpickle", "coverage[toml] (>=5.0.2)", "hypothesis", "mypy (>=0.900,!=0.940)", "pympler", "pytest (>=4.3.0)", "pytest-mypy-plugins"] +[[package]] +name = "boto3" +version = "1.24.59" +description = "The AWS SDK for Python" +category = "main" +optional = false +python-versions = ">= 3.7" + +[package.dependencies] +botocore = ">=1.27.59,<1.28.0" +jmespath = ">=0.7.1,<2.0.0" +s3transfer = ">=0.6.0,<0.7.0" + +[package.extras] +crt = ["botocore[crt] (>=1.21.0,<2.0a0)"] + [[package]] name = "botocore" version = "1.27.59" description = "Low-level, data-driven core of boto 3." category = "main" -optional = true +optional = false python-versions = ">= 3.7" [package.dependencies] @@ -198,6 +214,25 @@ tomli = {version = "*", optional = true, markers = "python_full_version <= \"3.1 [package.extras] toml = ["tomli"] +[[package]] +name = "cryptography" +version = "38.0.3" +description = "cryptography is a package which provides cryptographic recipes and primitives to Python developers." +category = "dev" +optional = false +python-versions = ">=3.6" + +[package.dependencies] +cffi = ">=1.12" + +[package.extras] +docs = ["sphinx (>=1.6.5,!=1.8.0,!=3.1.0,!=3.1.1)", "sphinx-rtd-theme"] +docstest = ["pyenchant (>=1.6.11)", "sphinxcontrib-spelling (>=4.0.1)", "twine (>=1.12.0)"] +pep8test = ["black", "flake8", "flake8-import-order", "pep8-naming"] +sdist = ["setuptools-rust (>=0.11.4)"] +ssh = ["bcrypt (>=3.1.5)"] +test = ["hypothesis (>=1.11.4,!=3.79.2)", "iso8601", "pretend", "pytest (>=6.2.0)", "pytest-benchmark", "pytest-cov", "pytest-subtests", "pytest-xdist", "pytz"] + [[package]] name = "distlib" version = "0.3.6" @@ -333,12 +368,34 @@ category = "dev" optional = false python-versions = "*" +[[package]] +name = "jinja2" +version = "3.1.2" +description = "A very fast and expressive template engine." +category = "dev" +optional = false +python-versions = ">=3.7" + +[package.dependencies] +MarkupSafe = ">=2.0" + +[package.extras] +i18n = ["Babel (>=2.7)"] + [[package]] name = "jmespath" version = "1.0.1" description = "JSON Matching Expressions" category = "main" -optional = true +optional = false +python-versions = ">=3.7" + +[[package]] +name = "markupsafe" +version = "2.1.1" +description = "Safely add untrusted strings to HTML/XML markup." +category = "dev" +optional = false python-versions = ">=3.7" [[package]] @@ -349,6 +406,51 @@ category = "main" optional = false python-versions = "*" +[[package]] +name = "moto" +version = "4.0.9" +description = "A library that allows your python tests to easily mock out the boto library" +category = "dev" +optional = false +python-versions = ">=3.6" + +[package.dependencies] +boto3 = ">=1.9.201" +botocore = ">=1.12.201" +cryptography = ">=3.3.1" +Jinja2 = ">=2.10.1" +MarkupSafe = "!=2.0.0a1" +python-dateutil = ">=2.1,<3.0.0" +pytz = "*" +requests = ">=2.5" +responses = ">=0.13.0" +werkzeug = ">=0.5,<2.2.0 || >2.2.0,<2.2.1 || >2.2.1" +xmltodict = "*" + +[package.extras] +all = ["PyYAML (>=5.1)", "aws-xray-sdk (>=0.93,!=0.96)", "cfn-lint (>=0.4.0)", "docker (>=2.5.1)", "ecdsa (!=0.15)", "graphql-core", "idna (>=2.5,<4)", "jsondiff (>=1.1.2)", "openapi-spec-validator (>=0.2.8)", "pyparsing (>=3.0.7)", "python-jose[cryptography] (>=3.1.0,<4.0.0)", "setuptools", "sshpubkeys (>=3.1.0)"] +apigateway = ["PyYAML (>=5.1)", "ecdsa (!=0.15)", "openapi-spec-validator (>=0.2.8)", "python-jose[cryptography] (>=3.1.0,<4.0.0)"] +apigatewayv2 = ["PyYAML (>=5.1)"] +appsync = ["graphql-core"] +awslambda = ["docker (>=2.5.1)"] +batch = ["docker (>=2.5.1)"] +cloudformation = ["PyYAML (>=5.1)", "aws-xray-sdk (>=0.93,!=0.96)", "cfn-lint (>=0.4.0)", "docker (>=2.5.1)", "ecdsa (!=0.15)", "graphql-core", "idna (>=2.5,<4)", "jsondiff (>=1.1.2)", "openapi-spec-validator (>=0.2.8)", "pyparsing (>=3.0.7)", "python-jose[cryptography] (>=3.1.0,<4.0.0)", "setuptools", "sshpubkeys (>=3.1.0)"] +cognitoidp = ["ecdsa (!=0.15)", "python-jose[cryptography] (>=3.1.0,<4.0.0)"] +ds = ["sshpubkeys (>=3.1.0)"] +dynamodb = ["docker (>=2.5.1)"] +dynamodb2 = ["docker (>=2.5.1)"] +dynamodbstreams = ["docker (>=2.5.1)"] +ebs = ["sshpubkeys (>=3.1.0)"] +ec2 = ["sshpubkeys (>=3.1.0)"] +efs = ["sshpubkeys (>=3.1.0)"] +glue = ["pyparsing (>=3.0.7)"] +iotdata = ["jsondiff (>=1.1.2)"] +route53resolver = ["sshpubkeys (>=3.1.0)"] +s3 = ["PyYAML (>=5.1)"] +server = ["PyYAML (>=5.1)", "aws-xray-sdk (>=0.93,!=0.96)", "cfn-lint (>=0.4.0)", "docker (>=2.5.1)", "ecdsa (!=0.15)", "flask (!=2.2.0,!=2.2.1)", "flask-cors", "graphql-core", "idna (>=2.5,<4)", "jsondiff (>=1.1.2)", "openapi-spec-validator (>=0.2.8)", "pyparsing (>=3.0.7)", "python-jose[cryptography] (>=3.1.0,<4.0.0)", "setuptools", "sshpubkeys (>=3.1.0)"] +ssm = ["PyYAML (>=5.1)", "dataclasses"] +xray = ["aws-xray-sdk (>=0.93,!=0.96)", "setuptools"] + [[package]] name = "multidict" version = "6.0.2" @@ -536,7 +638,7 @@ name = "python-dateutil" version = "2.8.2" description = "Extensions to the standard Python datetime module" category = "main" -optional = true +optional = false python-versions = "!=3.0.*,!=3.1.*,!=3.2.*,>=2.7" [package.dependencies] @@ -550,6 +652,14 @@ category = "main" optional = true python-versions = "*" +[[package]] +name = "pytz" +version = "2022.6" +description = "World timezone definitions, modern and historical" +category = "dev" +optional = false +python-versions = "*" + [[package]] name = "pyyaml" version = "6.0" @@ -592,6 +702,23 @@ six = "*" fixture = ["fixtures"] test = ["fixtures", "mock", "purl", "pytest", "requests-futures", "sphinx", "testrepository (>=0.0.18)", "testtools"] +[[package]] +name = "responses" +version = "0.22.0" +description = "A utility library for mocking out the `requests` Python library." +category = "dev" +optional = false +python-versions = ">=3.7" + +[package.dependencies] +requests = ">=2.22.0,<3.0" +toml = "*" +types-toml = "*" +urllib3 = ">=1.25.10" + +[package.extras] +tests = ["coverage (>=6.0.0)", "flake8", "mypy", "pytest (>=7.0.0)", "pytest-asyncio", "pytest-cov", "pytest-httpserver", "types-requests"] + [[package]] name = "rich" version = "12.6.0" @@ -625,6 +752,20 @@ fsspec = "2022.10.0" awscli = ["aiobotocore[awscli] (>=2.4.0,<2.5.0)"] boto3 = ["aiobotocore[boto3] (>=2.4.0,<2.5.0)"] +[[package]] +name = "s3transfer" +version = "0.6.0" +description = "An Amazon S3 Transfer Manager" +category = "main" +optional = false +python-versions = ">= 3.7" + +[package.dependencies] +botocore = ">=1.12.36,<2.0a.0" + +[package.extras] +crt = ["botocore[crt] (>=1.20.29,<2.0a.0)"] + [[package]] name = "setuptools" version = "65.5.1" @@ -678,6 +819,14 @@ category = "dev" optional = false python-versions = ">=3.7" +[[package]] +name = "types-toml" +version = "0.10.8.1" +description = "Typing stubs for toml" +category = "dev" +optional = false +python-versions = "*" + [[package]] name = "typing-extensions" version = "4.4.0" @@ -716,6 +865,20 @@ platformdirs = ">=2.4,<3" docs = ["proselint (>=0.13)", "sphinx (>=5.3)", "sphinx-argparse (>=0.3.2)", "sphinx-rtd-theme (>=1)", "towncrier (>=22.8)"] testing = ["coverage (>=6.2)", "coverage-enable-subprocess (>=1)", "flaky (>=3.7)", "packaging (>=21.3)", "pytest (>=7.0.1)", "pytest-env (>=0.6.2)", "pytest-freezegun (>=0.4.2)", "pytest-mock (>=3.6.1)", "pytest-randomly (>=3.10.3)", "pytest-timeout (>=2.1)"] +[[package]] +name = "werkzeug" +version = "2.2.2" +description = "The comprehensive WSGI web application library." +category = "dev" +optional = false +python-versions = ">=3.7" + +[package.dependencies] +MarkupSafe = ">=2.1.1" + +[package.extras] +watchdog = ["watchdog"] + [[package]] name = "wrapt" version = "1.14.1" @@ -724,6 +887,14 @@ category = "main" optional = true python-versions = "!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,!=3.4.*,>=2.7" +[[package]] +name = "xmltodict" +version = "0.13.0" +description = "Makes working with XML feel like you are working with JSON" +category = "dev" +optional = false +python-versions = ">=3.4" + [[package]] name = "yarl" version = "1.8.1" @@ -763,6 +934,7 @@ cffi = {version = ">=1.11", markers = "platform_python_implementation == \"PyPy\ cffi = ["cffi (>=1.11)"] [extras] +glue = ["boto3"] hive = ["thrift"] pyarrow = ["pyarrow"] s3fs = ["s3fs"] @@ -771,7 +943,7 @@ snappy = ["python-snappy"] [metadata] lock-version = "1.1" python-versions = "^3.8" -content-hash = "fda6f8ad5abce655e854325c9e0e944c6ad818487479c97078d1dce93496465e" +content-hash = "588c954b75cf70e3c695dfe27b859be09c183323becda37ebfdde00caa5730fd" [metadata.files] aiobotocore = [ @@ -883,6 +1055,10 @@ attrs = [ {file = "attrs-22.1.0-py2.py3-none-any.whl", hash = "sha256:86efa402f67bf2df34f51a335487cf46b1ec130d02b8d39fd248abfd30da551c"}, {file = "attrs-22.1.0.tar.gz", hash = "sha256:29adc2665447e5191d0e7c568fde78b21f9672d344281d0c6e1ab085429b22b6"}, ] +boto3 = [ + {file = "boto3-1.24.59-py3-none-any.whl", hash = "sha256:34ab44146a2c4e7f4e72737f4b27e6eb5e0a7855c2f4599e3d9199b6a0a2d575"}, + {file = "boto3-1.24.59.tar.gz", hash = "sha256:a50b4323f9579cfe22fcf5531fbd40b567d4d74c1adce06aeb5c95fce2a6fb40"}, +] botocore = [ {file = "botocore-1.27.59-py3-none-any.whl", hash = "sha256:69d756791fc024bda54f6c53f71ae34e695ee41bbbc1743d9179c4837a4929da"}, {file = "botocore-1.27.59.tar.gz", hash = "sha256:eda4aed6ee719a745d1288eaf1beb12f6f6448ad1fa12f159405db14ba9c92cf"}, @@ -1033,6 +1209,34 @@ coverage = [ {file = "coverage-6.5.0-pp36.pp37.pp38-none-any.whl", hash = "sha256:1431986dac3923c5945271f169f59c45b8802a114c8f548d611f2015133df77a"}, {file = "coverage-6.5.0.tar.gz", hash = "sha256:f642e90754ee3e06b0e7e51bce3379590e76b7f76b708e1a71ff043f87025c84"}, ] +cryptography = [ + {file = "cryptography-38.0.3-cp36-abi3-macosx_10_10_universal2.whl", hash = "sha256:984fe150f350a3c91e84de405fe49e688aa6092b3525f407a18b9646f6612320"}, + {file = "cryptography-38.0.3-cp36-abi3-macosx_10_10_x86_64.whl", hash = "sha256:ed7b00096790213e09eb11c97cc6e2b757f15f3d2f85833cd2d3ec3fe37c1722"}, + {file = "cryptography-38.0.3-cp36-abi3-manylinux_2_17_aarch64.manylinux2014_aarch64.manylinux_2_24_aarch64.whl", hash = "sha256:bbf203f1a814007ce24bd4d51362991d5cb90ba0c177a9c08825f2cc304d871f"}, + {file = "cryptography-38.0.3-cp36-abi3-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:554bec92ee7d1e9d10ded2f7e92a5d70c1f74ba9524947c0ba0c850c7b011828"}, + {file = "cryptography-38.0.3-cp36-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:b1b52c9e5f8aa2b802d48bd693190341fae201ea51c7a167d69fc48b60e8a959"}, + {file = "cryptography-38.0.3-cp36-abi3-manylinux_2_24_x86_64.whl", hash = "sha256:728f2694fa743a996d7784a6194da430f197d5c58e2f4e278612b359f455e4a2"}, + {file = "cryptography-38.0.3-cp36-abi3-manylinux_2_28_aarch64.whl", hash = "sha256:dfb4f4dd568de1b6af9f4cda334adf7d72cf5bc052516e1b2608b683375dd95c"}, + {file = "cryptography-38.0.3-cp36-abi3-manylinux_2_28_x86_64.whl", hash = "sha256:5419a127426084933076132d317911e3c6eb77568a1ce23c3ac1e12d111e61e0"}, + {file = "cryptography-38.0.3-cp36-abi3-musllinux_1_1_aarch64.whl", hash = "sha256:9b24bcff7853ed18a63cfb0c2b008936a9554af24af2fb146e16d8e1aed75748"}, + {file = "cryptography-38.0.3-cp36-abi3-musllinux_1_1_x86_64.whl", hash = "sha256:25c1d1f19729fb09d42e06b4bf9895212292cb27bb50229f5aa64d039ab29146"}, + {file = "cryptography-38.0.3-cp36-abi3-win32.whl", hash = "sha256:7f836217000342d448e1c9a342e9163149e45d5b5eca76a30e84503a5a96cab0"}, + {file = "cryptography-38.0.3-cp36-abi3-win_amd64.whl", hash = "sha256:c46837ea467ed1efea562bbeb543994c2d1f6e800785bd5a2c98bc096f5cb220"}, + {file = "cryptography-38.0.3-pp37-pypy37_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:06fc3cc7b6f6cca87bd56ec80a580c88f1da5306f505876a71c8cfa7050257dd"}, + {file = "cryptography-38.0.3-pp37-pypy37_pp73-manylinux_2_24_x86_64.whl", hash = "sha256:65535bc550b70bd6271984d9863a37741352b4aad6fb1b3344a54e6950249b55"}, + {file = "cryptography-38.0.3-pp37-pypy37_pp73-manylinux_2_28_x86_64.whl", hash = "sha256:5e89468fbd2fcd733b5899333bc54d0d06c80e04cd23d8c6f3e0542358c6060b"}, + {file = "cryptography-38.0.3-pp38-pypy38_pp73-macosx_10_10_x86_64.whl", hash = "sha256:6ab9516b85bebe7aa83f309bacc5f44a61eeb90d0b4ec125d2d003ce41932d36"}, + {file = "cryptography-38.0.3-pp38-pypy38_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:068147f32fa662c81aebab95c74679b401b12b57494872886eb5c1139250ec5d"}, + {file = "cryptography-38.0.3-pp38-pypy38_pp73-manylinux_2_24_x86_64.whl", hash = "sha256:402852a0aea73833d982cabb6d0c3bb582c15483d29fb7085ef2c42bfa7e38d7"}, + {file = "cryptography-38.0.3-pp38-pypy38_pp73-manylinux_2_28_x86_64.whl", hash = "sha256:b1b35d9d3a65542ed2e9d90115dfd16bbc027b3f07ee3304fc83580f26e43249"}, + {file = "cryptography-38.0.3-pp38-pypy38_pp73-win_amd64.whl", hash = "sha256:6addc3b6d593cd980989261dc1cce38263c76954d758c3c94de51f1e010c9a50"}, + {file = "cryptography-38.0.3-pp39-pypy39_pp73-macosx_10_10_x86_64.whl", hash = "sha256:be243c7e2bfcf6cc4cb350c0d5cdf15ca6383bbcb2a8ef51d3c9411a9d4386f0"}, + {file = "cryptography-38.0.3-pp39-pypy39_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:78cf5eefac2b52c10398a42765bfa981ce2372cbc0457e6bf9658f41ec3c41d8"}, + {file = "cryptography-38.0.3-pp39-pypy39_pp73-manylinux_2_24_x86_64.whl", hash = "sha256:4e269dcd9b102c5a3d72be3c45d8ce20377b8076a43cbed6f660a1afe365e436"}, + {file = "cryptography-38.0.3-pp39-pypy39_pp73-manylinux_2_28_x86_64.whl", hash = "sha256:8d41a46251bf0634e21fac50ffd643216ccecfaf3701a063257fe0b2be1b6548"}, + {file = "cryptography-38.0.3-pp39-pypy39_pp73-win_amd64.whl", hash = "sha256:785e4056b5a8b28f05a533fab69febf5004458e20dad7e2e13a3120d8ecec75a"}, + {file = "cryptography-38.0.3.tar.gz", hash = "sha256:bfbe6ee19615b07a98b1d2287d6a6073f734735b49ee45b11324d85efc4d5cbd"}, +] distlib = [ {file = "distlib-0.3.6-py2.py3-none-any.whl", hash = "sha256:f35c4b692542ca110de7ef0bea44d73981caeb34ca0b9b6b2e6d7790dda8f80e"}, {file = "distlib-0.3.6.tar.gz", hash = "sha256:14bad2d9b04d3a36127ac97f30b12a19268f211063d8f8ee4f47108896e11b46"}, @@ -1164,10 +1368,56 @@ iniconfig = [ {file = "iniconfig-1.1.1-py2.py3-none-any.whl", hash = "sha256:011e24c64b7f47f6ebd835bb12a743f2fbe9a26d4cecaa7f53bc4f35ee9da8b3"}, {file = "iniconfig-1.1.1.tar.gz", hash = "sha256:bc3af051d7d14b2ee5ef9969666def0cd1a000e121eaea580d4a313df4b37f32"}, ] +jinja2 = [ + {file = "Jinja2-3.1.2-py3-none-any.whl", hash = "sha256:6088930bfe239f0e6710546ab9c19c9ef35e29792895fed6e6e31a023a182a61"}, + {file = "Jinja2-3.1.2.tar.gz", hash = "sha256:31351a702a408a9e7595a8fc6150fc3f43bb6bf7e319770cbc0db9df9437e852"}, +] jmespath = [ {file = "jmespath-1.0.1-py3-none-any.whl", hash = "sha256:02e2e4cc71b5bcab88332eebf907519190dd9e6e82107fa7f83b1003a6252980"}, {file = "jmespath-1.0.1.tar.gz", hash = "sha256:90261b206d6defd58fdd5e85f478bf633a2901798906be2ad389150c5c60edbe"}, ] +markupsafe = [ + {file = "MarkupSafe-2.1.1-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:86b1f75c4e7c2ac2ccdaec2b9022845dbb81880ca318bb7a0a01fbf7813e3812"}, + {file = "MarkupSafe-2.1.1-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:f121a1420d4e173a5d96e47e9a0c0dcff965afdf1626d28de1460815f7c4ee7a"}, + {file = "MarkupSafe-2.1.1-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a49907dd8420c5685cfa064a1335b6754b74541bbb3706c259c02ed65b644b3e"}, + {file = "MarkupSafe-2.1.1-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:10c1bfff05d95783da83491be968e8fe789263689c02724e0c691933c52994f5"}, + {file = "MarkupSafe-2.1.1-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:b7bd98b796e2b6553da7225aeb61f447f80a1ca64f41d83612e6139ca5213aa4"}, + {file = "MarkupSafe-2.1.1-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:b09bf97215625a311f669476f44b8b318b075847b49316d3e28c08e41a7a573f"}, + {file = "MarkupSafe-2.1.1-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:694deca8d702d5db21ec83983ce0bb4b26a578e71fbdbd4fdcd387daa90e4d5e"}, + {file = "MarkupSafe-2.1.1-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:efc1913fd2ca4f334418481c7e595c00aad186563bbc1ec76067848c7ca0a933"}, + {file = "MarkupSafe-2.1.1-cp310-cp310-win32.whl", hash = "sha256:4a33dea2b688b3190ee12bd7cfa29d39c9ed176bda40bfa11099a3ce5d3a7ac6"}, + {file = "MarkupSafe-2.1.1-cp310-cp310-win_amd64.whl", hash = "sha256:dda30ba7e87fbbb7eab1ec9f58678558fd9a6b8b853530e176eabd064da81417"}, + {file = "MarkupSafe-2.1.1-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:671cd1187ed5e62818414afe79ed29da836dde67166a9fac6d435873c44fdd02"}, + {file = "MarkupSafe-2.1.1-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:3799351e2336dc91ea70b034983ee71cf2f9533cdff7c14c90ea126bfd95d65a"}, + {file = "MarkupSafe-2.1.1-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:e72591e9ecd94d7feb70c1cbd7be7b3ebea3f548870aa91e2732960fa4d57a37"}, + {file = "MarkupSafe-2.1.1-cp37-cp37m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:6fbf47b5d3728c6aea2abb0589b5d30459e369baa772e0f37a0320185e87c980"}, + {file = "MarkupSafe-2.1.1-cp37-cp37m-musllinux_1_1_aarch64.whl", hash = "sha256:d5ee4f386140395a2c818d149221149c54849dfcfcb9f1debfe07a8b8bd63f9a"}, + {file = "MarkupSafe-2.1.1-cp37-cp37m-musllinux_1_1_i686.whl", hash = "sha256:bcb3ed405ed3222f9904899563d6fc492ff75cce56cba05e32eff40e6acbeaa3"}, + {file = "MarkupSafe-2.1.1-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:e1c0b87e09fa55a220f058d1d49d3fb8df88fbfab58558f1198e08c1e1de842a"}, + {file = "MarkupSafe-2.1.1-cp37-cp37m-win32.whl", hash = "sha256:8dc1c72a69aa7e082593c4a203dcf94ddb74bb5c8a731e4e1eb68d031e8498ff"}, + {file = "MarkupSafe-2.1.1-cp37-cp37m-win_amd64.whl", hash = "sha256:97a68e6ada378df82bc9f16b800ab77cbf4b2fada0081794318520138c088e4a"}, + {file = "MarkupSafe-2.1.1-cp38-cp38-macosx_10_9_universal2.whl", hash = "sha256:e8c843bbcda3a2f1e3c2ab25913c80a3c5376cd00c6e8c4a86a89a28c8dc5452"}, + {file = "MarkupSafe-2.1.1-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:0212a68688482dc52b2d45013df70d169f542b7394fc744c02a57374a4207003"}, + {file = "MarkupSafe-2.1.1-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:8e576a51ad59e4bfaac456023a78f6b5e6e7651dcd383bcc3e18d06f9b55d6d1"}, + {file = "MarkupSafe-2.1.1-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:4b9fe39a2ccc108a4accc2676e77da025ce383c108593d65cc909add5c3bd601"}, + {file = "MarkupSafe-2.1.1-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:96e37a3dc86e80bf81758c152fe66dbf60ed5eca3d26305edf01892257049925"}, + {file = "MarkupSafe-2.1.1-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:6d0072fea50feec76a4c418096652f2c3238eaa014b2f94aeb1d56a66b41403f"}, + {file = "MarkupSafe-2.1.1-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:089cf3dbf0cd6c100f02945abeb18484bd1ee57a079aefd52cffd17fba910b88"}, + {file = "MarkupSafe-2.1.1-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:6a074d34ee7a5ce3effbc526b7083ec9731bb3cbf921bbe1d3005d4d2bdb3a63"}, + {file = "MarkupSafe-2.1.1-cp38-cp38-win32.whl", hash = "sha256:421be9fbf0ffe9ffd7a378aafebbf6f4602d564d34be190fc19a193232fd12b1"}, + {file = "MarkupSafe-2.1.1-cp38-cp38-win_amd64.whl", hash = "sha256:fc7b548b17d238737688817ab67deebb30e8073c95749d55538ed473130ec0c7"}, + {file = "MarkupSafe-2.1.1-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:e04e26803c9c3851c931eac40c695602c6295b8d432cbe78609649ad9bd2da8a"}, + {file = "MarkupSafe-2.1.1-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:b87db4360013327109564f0e591bd2a3b318547bcef31b468a92ee504d07ae4f"}, + {file = "MarkupSafe-2.1.1-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:99a2a507ed3ac881b975a2976d59f38c19386d128e7a9a18b7df6fff1fd4c1d6"}, + {file = "MarkupSafe-2.1.1-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:56442863ed2b06d19c37f94d999035e15ee982988920e12a5b4ba29b62ad1f77"}, + {file = "MarkupSafe-2.1.1-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:3ce11ee3f23f79dbd06fb3d63e2f6af7b12db1d46932fe7bd8afa259a5996603"}, + {file = "MarkupSafe-2.1.1-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:33b74d289bd2f5e527beadcaa3f401e0df0a89927c1559c8566c066fa4248ab7"}, + {file = "MarkupSafe-2.1.1-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:43093fb83d8343aac0b1baa75516da6092f58f41200907ef92448ecab8825135"}, + {file = "MarkupSafe-2.1.1-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:8e3dcf21f367459434c18e71b2a9532d96547aef8a871872a5bd69a715c15f96"}, + {file = "MarkupSafe-2.1.1-cp39-cp39-win32.whl", hash = "sha256:d4306c36ca495956b6d568d276ac11fdd9c30a36f1b6eb928070dc5360b22e1c"}, + {file = "MarkupSafe-2.1.1-cp39-cp39-win_amd64.whl", hash = "sha256:46d00d6cfecdde84d40e572d63735ef81423ad31184100411e6e3388d405e247"}, + {file = "MarkupSafe-2.1.1.tar.gz", hash = "sha256:7f91197cc9e48f989d12e4e6fbc46495c446636dfc81b9ccf50bb0ec74b91d4b"}, +] mmhash3 = [ {file = "mmhash3-3.0.1-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:47deea30cd8d3d5cd52dc740902a4c70383bfe8248eac29d0877fe63e03c2713"}, {file = "mmhash3-3.0.1-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:ecdaf4d1de617818bf05cd526ca558db6010beeba7ea9e19f695f2bdcac0e0a4"}, @@ -1204,6 +1454,10 @@ mmhash3 = [ {file = "mmhash3-3.0.1-cp39-cp39-win_amd64.whl", hash = "sha256:0baeaa20cac5f75ed14f28056826bd9d9c8b2354b382073f3fd5190708992a0d"}, {file = "mmhash3-3.0.1.tar.gz", hash = "sha256:a00d68f4a1cc434b9501513c8a29e18ed1ddad383677d72b41d71d0d862348af"}, ] +moto = [ + {file = "moto-4.0.9-py3-none-any.whl", hash = "sha256:2fb909d2ea1b732f89604e4268e2c2207c253e590a635a410c3c2aaebb34e113"}, + {file = "moto-4.0.9.tar.gz", hash = "sha256:ba03b638cf3b1cec64cbe9ac0d184ca898b69020c8e3c5b9b4961c1670629010"}, +] multidict = [ {file = "multidict-6.0.2-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:0b9e95a740109c6047602f4db4da9949e6c5945cefbad34a1299775ddc9a62e2"}, {file = "multidict-6.0.2-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:ac0e27844758d7177989ce406acc6a83c16ed4524ebc363c1f748cba184d89d3"}, @@ -1455,6 +1709,10 @@ python-snappy = [ {file = "python_snappy-0.6.1-pp37-pypy37_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:0bdb6942180660bda7f7d01f4c0def3cfc72b1c6d99aad964801775a3e379aba"}, {file = "python_snappy-0.6.1-pp37-pypy37_pp73-win_amd64.whl", hash = "sha256:03bb511380fca2a13325b6f16fe8234c8e12da9660f0258cd45d9a02ffc916af"}, ] +pytz = [ + {file = "pytz-2022.6-py2.py3-none-any.whl", hash = "sha256:222439474e9c98fced559f1709d89e6c9cbf8d79c794ff3eb9f8800064291427"}, + {file = "pytz-2022.6.tar.gz", hash = "sha256:e89512406b793ca39f5971bc999cc538ce125c0e51c27941bef4568b460095e2"}, +] pyyaml = [ {file = "PyYAML-6.0-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:d4db7c7aef085872ef65a8fd7d6d09a14ae91f691dec3e87ee5ee0539d516f53"}, {file = "PyYAML-6.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:9df7ed3b3d2e0ecfe09e14741b857df43adb5a3ddadc919a2d94fbdf78fea53c"}, @@ -1505,6 +1763,10 @@ requests-mock = [ {file = "requests-mock-1.10.0.tar.gz", hash = "sha256:59c9c32419a9fb1ae83ec242d98e889c45bd7d7a65d48375cc243ec08441658b"}, {file = "requests_mock-1.10.0-py2.py3-none-any.whl", hash = "sha256:2fdbb637ad17ee15c06f33d31169e71bf9fe2bdb7bc9da26185be0dd8d842699"}, ] +responses = [ + {file = "responses-0.22.0-py3-none-any.whl", hash = "sha256:dcf294d204d14c436fddcc74caefdbc5764795a40ff4e6a7740ed8ddbf3294be"}, + {file = "responses-0.22.0.tar.gz", hash = "sha256:396acb2a13d25297789a5866b4881cf4e46ffd49cc26c43ab1117f40b973102e"}, +] rich = [ {file = "rich-12.6.0-py3-none-any.whl", hash = "sha256:a4eb26484f2c82589bd9a17c73d32a010b1e29d89f1604cd9bf3a2097b81bb5e"}, {file = "rich-12.6.0.tar.gz", hash = "sha256:ba3a3775974105c221d31141f2c116f4fd65c5ceb0698657a11e9f295ec93fd0"}, @@ -1513,6 +1775,10 @@ s3fs = [ {file = "s3fs-2022.10.0-py3-none-any.whl", hash = "sha256:1e134c3577171699feb7c1a0c4713260d5b48296e1708737ff940baef6e2c153"}, {file = "s3fs-2022.10.0.tar.gz", hash = "sha256:e8deb80f20bd0b2059141b874fdb9d6aeb8cce35312ea5f2c02b225a78a00406"}, ] +s3transfer = [ + {file = "s3transfer-0.6.0-py3-none-any.whl", hash = "sha256:06176b74f3a15f61f1b4f25a1fc29a4429040b7647133a463da8fa5bd28d5ecd"}, + {file = "s3transfer-0.6.0.tar.gz", hash = "sha256:2ed07d3866f523cc561bf4a00fc5535827981b117dd7876f036b0c1aca42c947"}, +] setuptools = [ {file = "setuptools-65.5.1-py3-none-any.whl", hash = "sha256:d0b9a8433464d5800cbe05094acf5c6d52a91bfac9b52bcfc4d41382be5d5d31"}, {file = "setuptools-65.5.1.tar.gz", hash = "sha256:e197a19aa8ec9722928f2206f8de752def0e4c9fc6953527360d1c36d94ddb2f"}, @@ -1532,6 +1798,10 @@ tomli = [ {file = "tomli-2.0.1-py3-none-any.whl", hash = "sha256:939de3e7a6161af0c887ef91b7d41a53e7c5a1ca976325f429cb46ea9bc30ecc"}, {file = "tomli-2.0.1.tar.gz", hash = "sha256:de526c12914f0c550d15924c62d72abc48d6fe7364aa87328337a31007fe8a4f"}, ] +types-toml = [ + {file = "types-toml-0.10.8.1.tar.gz", hash = "sha256:171bdb3163d79a520560f24ba916a9fc9bff81659c5448a9fea89240923722be"}, + {file = "types_toml-0.10.8.1-py3-none-any.whl", hash = "sha256:b7b5c4977f96ab7b5ac06d8a6590d17c0bf252a96efc03b109c2711fb3e0eafd"}, +] typing-extensions = [ {file = "typing_extensions-4.4.0-py3-none-any.whl", hash = "sha256:16fa4864408f655d35ec496218b85f79b3437c829e93320c7c9215ccfd92489e"}, {file = "typing_extensions-4.4.0.tar.gz", hash = "sha256:1511434bb92bf8dd198c12b1cc812e800d4181cfcb867674e0f8279cc93087aa"}, @@ -1544,6 +1814,10 @@ virtualenv = [ {file = "virtualenv-20.16.7-py3-none-any.whl", hash = "sha256:efd66b00386fdb7dbe4822d172303f40cd05e50e01740b19ea42425cbe653e29"}, {file = "virtualenv-20.16.7.tar.gz", hash = "sha256:8691e3ff9387f743e00f6bb20f70121f5e4f596cae754531f2b3b3a1b1ac696e"}, ] +werkzeug = [ + {file = "Werkzeug-2.2.2-py3-none-any.whl", hash = "sha256:f979ab81f58d7318e064e99c4506445d60135ac5cd2e177a2de0089bfd4c9bd5"}, + {file = "Werkzeug-2.2.2.tar.gz", hash = "sha256:7ea2d48322cc7c0f8b3a215ed73eabd7b5d75d0b50e31ab006286ccff9e00b8f"}, +] wrapt = [ {file = "wrapt-1.14.1-cp27-cp27m-macosx_10_9_x86_64.whl", hash = "sha256:1b376b3f4896e7930f1f772ac4b064ac12598d1c38d04907e696cc4d794b43d3"}, {file = "wrapt-1.14.1-cp27-cp27m-manylinux1_i686.whl", hash = "sha256:903500616422a40a98a5a3c4ff4ed9d0066f3b4c951fa286018ecdf0750194ef"}, @@ -1610,6 +1884,10 @@ wrapt = [ {file = "wrapt-1.14.1-cp39-cp39-win_amd64.whl", hash = "sha256:dee60e1de1898bde3b238f18340eec6148986da0455d8ba7848d50470a7a32fb"}, {file = "wrapt-1.14.1.tar.gz", hash = "sha256:380a85cf89e0e69b7cfbe2ea9f765f004ff419f34194018a6827ac0e3edfed4d"}, ] +xmltodict = [ + {file = "xmltodict-0.13.0-py2.py3-none-any.whl", hash = "sha256:aa89e8fd76320154a40d19a0df04a4695fb9dc5ba977cbb68ab3e4eb225e7852"}, + {file = "xmltodict-0.13.0.tar.gz", hash = "sha256:341595a488e3e01a85a9d8911d8912fd922ede5fecc4dce437eb4b6c8d037e56"}, +] yarl = [ {file = "yarl-1.8.1-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:abc06b97407868ef38f3d172762f4069323de52f2b70d133d096a48d72215d28"}, {file = "yarl-1.8.1-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:07b21e274de4c637f3e3b7104694e53260b5fc10d51fb3ec5fed1da8e0f754e3"}, diff --git a/pyiceberg/catalog/__init__.py b/pyiceberg/catalog/__init__.py index 01fa6f3cf6..9a88d59e25 100644 --- a/pyiceberg/catalog/__init__.py +++ b/pyiceberg/catalog/__init__.py @@ -25,12 +25,14 @@ Callable, List, Optional, + Set, Union, cast, ) from pyiceberg.exceptions import NotInstalledError from pyiceberg.io import FileIO, load_file_io +from pyiceberg.manifest import ManifestFile from pyiceberg.schema import Schema from pyiceberg.table import Table from pyiceberg.table.partitioning import UNPARTITIONED_PARTITION_SPEC, PartitionSpec @@ -49,12 +51,21 @@ TOKEN = "token" TYPE = "type" +ICEBERG = "iceberg" +TABLE_TYPE = "table_type" +WAREHOUSE = "warehouse" +METADATA_LOCATION = "metadata_location" +MANIFEST = "manifest" +MANIFEST_LIST = "manifest list" +PREVIOUS_METADATA = "previous metadata" +METADATA = "metadata" URI = "uri" class CatalogType(Enum): REST = "rest" HIVE = "hive" + GLUE = "glue" def load_rest(name: str, conf: Properties) -> Catalog: @@ -72,9 +83,19 @@ def load_hive(name: str, conf: Properties) -> Catalog: raise NotInstalledError("Apache Hive support not installed: pip install 'pyiceberg[hive]'") from exc +def load_glue(name: str, conf: Properties) -> Catalog: + try: + from pyiceberg.catalog.glue import GlueCatalog + + return GlueCatalog(name, **conf) + except ImportError as exc: + raise NotInstalledError("AWS glue support not installed: pip install 'pyiceberg[glue]'") from exc + + AVAILABLE_CATALOGS: dict[CatalogType, Callable[[str, Properties], Catalog]] = { CatalogType.REST: load_rest, CatalogType.HIVE: load_hive, + CatalogType.GLUE: load_glue, } @@ -138,6 +159,44 @@ def load_catalog(name: str, **properties: Optional[str]) -> Catalog: raise ValueError(f"Could not initialize catalog with the following properties: {properties}") +def delete_files(io: FileIO, files_to_delete: Set[str], file_type: str) -> None: + """Helper to delete files. + + Log warnings if failing to delete any file + + Args: + io: The FileIO used to delete the object + files_to_delete: A set of file paths to be deleted + file_type: The type of the file + """ + for file in files_to_delete: + try: + io.delete(file) + except OSError as exc: + logger.warning(msg=f"Failed to delete {file_type} file {file}", exc_info=exc) + + +def delete_data_files(io: FileIO, manifests_to_delete: List[ManifestFile]) -> None: + """Helper to delete data files linked to given manifests. + + Log warnings if failing to delete any file + + Args: + io: The FileIO used to delete the object + manifests_to_delete: A list of manifest contains paths of data files to be deleted + """ + deleted_files: dict[str, bool] = {} + for manifest_file in manifests_to_delete: + for entry in manifest_file.fetch_manifest_entry(io): + path = entry.data_file.file_path + if not deleted_files.get(path, False): + try: + io.delete(path) + except OSError as exc: + logger.warning(msg=f"Failed to delete data file {path}", exc_info=exc) + deleted_files[path] = True + + @dataclass class PropertiesUpdateSummary: removed: List[str] diff --git a/pyiceberg/catalog/glue.py b/pyiceberg/catalog/glue.py new file mode 100644 index 0000000000..45af6ef97f --- /dev/null +++ b/pyiceberg/catalog/glue.py @@ -0,0 +1,549 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + + +import uuid +from typing import ( + Any, + Dict, + List, + Optional, + Set, + Tuple, + Type, + Union, +) + +import boto3 + +from pyiceberg.catalog import ( + ICEBERG, + MANIFEST, + MANIFEST_LIST, + METADATA, + METADATA_LOCATION, + PREVIOUS_METADATA, + TABLE_TYPE, + WAREHOUSE, + Catalog, + Identifier, + Properties, + PropertiesUpdateSummary, + delete_data_files, + delete_files, +) +from pyiceberg.exceptions import ( + NamespaceAlreadyExistsError, + NamespaceNotEmptyError, + NoSuchIcebergTableError, + NoSuchNamespaceError, + NoSuchPropertyException, + NoSuchTableError, + TableAlreadyExistsError, +) +from pyiceberg.io import FileIO, load_file_io +from pyiceberg.schema import Schema +from pyiceberg.serializers import FromInputFile, ToOutputFile +from pyiceberg.table import Table +from pyiceberg.table.metadata import TableMetadata, new_table_metadata +from pyiceberg.table.partitioning import UNPARTITIONED_PARTITION_SPEC, PartitionSpec +from pyiceberg.table.sorting import UNSORTED_SORT_ORDER, SortOrder +from pyiceberg.typedef import EMPTY_DICT + +EXTERNAL_TABLE_TYPE = "EXTERNAL_TABLE" +GLUE_CLIENT = "glue" + +PROP_GLUE_TABLE = "Table" +PROP_GLUE_TABLE_TYPE = "TableType" +PROP_GLUE_TABLE_DESCRIPTION = "Description" +PROP_GLUE_TABLE_PARAMETERS = "Parameters" +PROP_GLUE_TABLE_DATABASE_NAME = "DatabaseName" +PROP_GLUE_TABLE_NAME = "Name" +PROP_GLUE_TABLE_OWNER = "Owner" +PROP_GLUE_TABLE_STORAGE_DESCRIPTOR = "StorageDescriptor" + +PROP_GLUE_TABLELIST = "TableList" + +PROP_GLUE_DATABASE = "Database" +PROP_GLUE_DATABASE_LIST = "DatabaseList" +PROP_GLUE_DATABASE_NAME = "Name" +PROP_GLUE_DATABASE_LOCATION = "LocationUri" +PROP_GLUE_DATABASE_DESCRIPTION = "Description" +PROP_GLUE_DATABASE_PARAMETERS = "Parameters" + +PROP_GLUE_NEXT_TOKEN = "NextToken" + +GLUE_DESCRIPTION_KEY = "comment" +GLUE_DATABASE_LOCATION_KEY = "location" + + +def _construct_parameters(metadata_location: str) -> Properties: + return {TABLE_TYPE: ICEBERG.upper(), METADATA_LOCATION: metadata_location} + + +def _construct_table_input(table_name: str, metadata_location: str, properties: Properties) -> Dict[str, Any]: + table_input = { + PROP_GLUE_TABLE_NAME: table_name, + PROP_GLUE_TABLE_TYPE: EXTERNAL_TABLE_TYPE, + PROP_GLUE_TABLE_PARAMETERS: _construct_parameters(metadata_location), + } + + if table_description := properties.get(GLUE_DESCRIPTION_KEY): + table_input[PROP_GLUE_TABLE_DESCRIPTION] = table_description + + return table_input + + +def _construct_database_input(database_name: str, properties: Properties) -> Dict[str, Any]: + database_input: Dict[str, Any] = {PROP_GLUE_DATABASE_NAME: database_name} + parameters = {} + for k, v in properties.items(): + if k == GLUE_DESCRIPTION_KEY: + database_input[PROP_GLUE_DATABASE_DESCRIPTION] = v + elif k == GLUE_DATABASE_LOCATION_KEY: + database_input[PROP_GLUE_DATABASE_LOCATION] = v + else: + parameters[k] = v + database_input[PROP_GLUE_DATABASE_PARAMETERS] = parameters + return database_input + + +def _write_metadata(metadata: TableMetadata, io: FileIO, metadate_path: str) -> None: + ToOutputFile.table_metadata(metadata, io.new_output(metadate_path)) + + +class GlueCatalog(Catalog): + @staticmethod + def identifier_to_database( + identifier: Union[str, Identifier], err: Union[Type[ValueError], Type[NoSuchNamespaceError]] = ValueError + ) -> str: + tuple_identifier = Catalog.identifier_to_tuple(identifier) + if len(tuple_identifier) != 1: + raise err(f"Invalid database, hierarchical namespaces are not supported: {identifier}") + + return tuple_identifier[0] + + @staticmethod + def identifier_to_database_and_table( + identifier: Union[str, Identifier], + err: Union[Type[ValueError], Type[NoSuchTableError], Type[NoSuchNamespaceError]] = ValueError, + ) -> Tuple[str, str]: + tuple_identifier = Catalog.identifier_to_tuple(identifier) + if len(tuple_identifier) != 2: + raise err(f"Invalid path, hierarchical namespaces are not supported: {identifier}") + + return tuple_identifier[0], tuple_identifier[1] + + def __init__(self, name: str, **properties: str): + super().__init__(name, **properties) + self.glue = boto3.client(GLUE_CLIENT) + + def _convert_glue_to_iceberg(self, glue_table: Dict[str, Any]) -> Table: + properties: Properties = glue_table.get(PROP_GLUE_TABLE_PARAMETERS, {}) + + if TABLE_TYPE not in properties: + raise NoSuchPropertyException( + f"Property {TABLE_TYPE} missing, could not determine type: " + f"{glue_table[PROP_GLUE_TABLE_DATABASE_NAME]}.{glue_table[PROP_GLUE_TABLE_NAME]}" + ) + glue_table_type = properties[TABLE_TYPE] + + if glue_table_type.lower() != ICEBERG: + raise NoSuchIcebergTableError( + f"Property table_type is {glue_table_type}, expected {ICEBERG}: " + f"{glue_table[PROP_GLUE_TABLE_DATABASE_NAME]}.{glue_table[PROP_GLUE_TABLE_NAME]}" + ) + + if METADATA_LOCATION not in properties: + raise NoSuchPropertyException( + f"Table property {METADATA_LOCATION} is missing, cannot find metadata for: " + f"{glue_table[PROP_GLUE_TABLE_DATABASE_NAME]}.{glue_table[PROP_GLUE_TABLE_NAME]}" + ) + metadata_location = properties[METADATA_LOCATION] + + io = load_file_io(properties=self.properties, location=metadata_location) + file = io.new_input(metadata_location) + metadata = FromInputFile.table_metadata(file) + return Table( + identifier=(glue_table[PROP_GLUE_TABLE_DATABASE_NAME], glue_table[PROP_GLUE_TABLE_NAME]), + metadata=metadata, + metadata_location=metadata_location, + io=self._load_file_io(metadata.properties), + ) + + def _default_warehouse_location(self, database_name: str, table_name: str) -> str: + database_properties = self.load_namespace_properties(database_name) + if database_location := database_properties.get(GLUE_DATABASE_LOCATION_KEY): + database_location = database_location.rstrip("/") + return f"{database_location}/{table_name}" + + if warehouse_path := self.properties.get(WAREHOUSE): + warehouse_path = warehouse_path.rstrip("/") + return f"{warehouse_path}/{database_name}.db/{table_name}" + + raise ValueError("No default path is set, please specify a location when creating a table") + + def _resolve_table_location(self, location: Optional[str], database_name: str, table_name: str) -> str: + if not location: + return self._default_warehouse_location(database_name, table_name) + return location + + def _create_glue_table(self, identifier: Union[str, Identifier], table_input: Dict[str, Any]) -> None: + database_name, table_name = self.identifier_to_database_and_table(identifier) + try: + self.glue.create_table(DatabaseName=database_name, TableInput=table_input) + except self.glue.exceptions.AlreadyExistsException as e: + raise TableAlreadyExistsError(f"Table {database_name}.{table_name} already exists") from e + except self.glue.exceptions.EntityNotFoundException as e: + raise NoSuchNamespaceError(f"Database {database_name} does not exist") from e + + def create_table( + self, + identifier: Union[str, Identifier], + schema: Schema, + location: Optional[str] = None, + partition_spec: PartitionSpec = UNPARTITIONED_PARTITION_SPEC, + sort_order: SortOrder = UNSORTED_SORT_ORDER, + properties: Properties = EMPTY_DICT, + ) -> Table: + """Create an Iceberg table in Glue catalog + + Args: + identifier: Table identifier. + schema: Table's schema. + location: Location for the table. Optional Argument. + partition_spec: PartitionSpec for the table. + sort_order: SortOrder for the table. + properties: Table properties that can be a string based dictionary. + + Returns: + Table: the created table instance + + Raises: + AlreadyExistsError: If a table with the name already exists + ValueError: If the identifier is invalid, or no path is given to store metadata + + """ + database_name, table_name = self.identifier_to_database_and_table(identifier) + + location = self._resolve_table_location(location, database_name, table_name) + metadata_location = f"{location}/metadata/00000-{uuid.uuid4()}.metadata.json" + metadata = new_table_metadata( + location=location, schema=schema, partition_spec=partition_spec, sort_order=sort_order, properties=properties + ) + io = load_file_io(properties=self.properties, location=metadata_location) + _write_metadata(metadata, io, metadata_location) + + self._create_glue_table( + identifier=identifier, table_input=_construct_table_input(table_name, metadata_location, properties) + ) + loaded_table = self.load_table(identifier=(database_name, table_name)) + return loaded_table + + def load_table(self, identifier: Union[str, Identifier]) -> Table: + """Loads the table's metadata and returns the table instance. + + You can also use this method to check for table existence using 'try catalog.table() except TableNotFoundError' + Note: This method doesn't scan data stored in the table. + + Args: + identifier: Table identifier. + + Returns: + Table: the table instance with its metadata + + Raises: + NoSuchTableError: If a table with the name does not exist, or the identifier is invalid + """ + database_name, table_name = self.identifier_to_database_and_table(identifier, NoSuchTableError) + try: + load_table_response = self.glue.get_table(DatabaseName=database_name, Name=table_name) + except self.glue.exceptions.EntityNotFoundException as e: + raise NoSuchTableError(f"Table does not exists: {database_name}.{table_name}") from e + + return self._convert_glue_to_iceberg(load_table_response.get(PROP_GLUE_TABLE, {})) + + def drop_table(self, identifier: Union[str, Identifier]) -> None: + """Drop a table. + + Args: + identifier: Table identifier. + + Raises: + NoSuchTableError: If a table with the name does not exist, or the identifier is invalid + """ + database_name, table_name = self.identifier_to_database_and_table(identifier, NoSuchTableError) + try: + self.glue.delete_table(DatabaseName=database_name, Name=table_name) + except self.glue.exceptions.EntityNotFoundException as e: + raise NoSuchTableError(f"Table does not exists: {database_name}.{table_name}") from e + + def purge_table(self, identifier: Union[str, Identifier]) -> None: + """Drop a table and purge all data and metadata files. + + Note: This method only logs warning rather than raise exception when encountering file deletion failure + + Args: + identifier (str | Identifier): Table identifier. + + Raises: + NoSuchTableError: If a table with the name does not exist, or the identifier is invalid + """ + table = self.load_table(identifier) + self.drop_table(identifier) + io = load_file_io(self.properties, table.metadata_location) + metadata = table.metadata + manifest_lists_to_delete = set() + manifests_to_delete = [] + for snapshot in metadata.snapshots: + manifests_to_delete += snapshot.fetch_manifest_list(io) + if snapshot.manifest_list is not None: + manifest_lists_to_delete.add(snapshot.manifest_list) + + manifest_paths_to_delete = {manifest.manifest_path for manifest in manifests_to_delete} + prev_metadata_files = {log.metadata_file for log in metadata.metadata_log} + + delete_data_files(io, manifests_to_delete) + delete_files(io, manifest_paths_to_delete, MANIFEST) + delete_files(io, manifest_lists_to_delete, MANIFEST_LIST) + delete_files(io, prev_metadata_files, PREVIOUS_METADATA) + delete_files(io, {table.metadata_location}, METADATA) + + def rename_table(self, from_identifier: Union[str, Identifier], to_identifier: Union[str, Identifier]) -> Table: + """Rename a fully classified table name + + This method can only rename Iceberg tables in AWS Glue + + Args: + from_identifier: Existing table identifier. + to_identifier: New table identifier. + + Returns: + Table: the updated table instance with its metadata + + Raises: + ValueError: When the from table identifier is invalid + NoSuchTableError: When a table with the name does not exist + NoSuchIcebergTableError: When the from table is not a valid iceberg table + NoSuchPropertyException: When the from table miss some required properties + NoSuchNamespaceError: When the destination namespace doesn't exist + """ + from_database_name, from_table_name = self.identifier_to_database_and_table(from_identifier, NoSuchTableError) + to_database_name, to_table_name = self.identifier_to_database_and_table(to_identifier) + try: + get_table_response = self.glue.get_table(DatabaseName=from_database_name, Name=from_table_name) + except self.glue.exceptions.EntityNotFoundException as e: + raise NoSuchTableError(f"Table does not exists: {from_database_name}.{from_table_name}") from e + + glue_table = get_table_response[PROP_GLUE_TABLE] + + try: + # verify that from_identifier is a valid iceberg table + self._convert_glue_to_iceberg(glue_table) + except NoSuchPropertyException as e: + raise NoSuchPropertyException( + f"Failed to rename table {from_database_name}.{from_table_name} since it miss required properties" + ) from e + except NoSuchIcebergTableError as e: + raise NoSuchIcebergTableError( + f"Failed to rename table {from_database_name}.{from_table_name} since it is not a valid iceberg table" + ) from e + + new_table_input = {PROP_GLUE_TABLE_NAME: to_table_name} + # use the same Glue info to create the new table, pointing to the old metadata + if table_type := glue_table.get(PROP_GLUE_TABLE_TYPE): + new_table_input[PROP_GLUE_TABLE_TYPE] = table_type + if table_parameters := glue_table.get(PROP_GLUE_TABLE_PARAMETERS): + new_table_input[PROP_GLUE_TABLE_PARAMETERS] = table_parameters + if table_owner := glue_table.get(PROP_GLUE_TABLE_OWNER): + new_table_input[PROP_GLUE_TABLE_OWNER] = table_owner + if table_storage_descriptor := glue_table.get(PROP_GLUE_TABLE_STORAGE_DESCRIPTOR): + new_table_input[PROP_GLUE_TABLE_STORAGE_DESCRIPTOR] = table_storage_descriptor + if table_description := glue_table.get(PROP_GLUE_TABLE_DESCRIPTION): + new_table_input[PROP_GLUE_TABLE_DESCRIPTION] = table_description + + self._create_glue_table(identifier=to_identifier, table_input=new_table_input) + try: + self.drop_table(from_identifier) + except Exception as e: + self.drop_table(to_identifier) + raise ValueError( + f"Fail to drop old table {from_database_name}.{from_table_name}, " + f"after renaming to {to_database_name}.{to_table_name} roll back to use the old one" + ) from e + return self.load_table(to_identifier) + + def create_namespace(self, namespace: Union[str, Identifier], properties: Properties = EMPTY_DICT) -> None: + """Create a namespace in the catalog. + + Args: + namespace: Namespace identifier + properties: A string dictionary of properties for the given namespace + + Raises: + ValueError: If the identifier is invalid + AlreadyExistsError: If a namespace with the given name already exists + """ + database_name = self.identifier_to_database(namespace) + try: + self.glue.create_database(DatabaseInput=_construct_database_input(database_name, properties)) + except self.glue.exceptions.AlreadyExistsException as e: + raise NamespaceAlreadyExistsError(f"Database {database_name} already exists") from e + + def drop_namespace(self, namespace: Union[str, Identifier]) -> None: + """Drop a namespace. + + A Glue namespace can only be dropped if it is empty + + Args: + namespace: Namespace identifier + + Raises: + NoSuchNamespaceError: If a namespace with the given name does not exist, or the identifier is invalid + NamespaceNotEmptyError: If the namespace is not empty + """ + database_name = self.identifier_to_database(namespace, NoSuchNamespaceError) + try: + table_list = self.list_tables(namespace=database_name) + except NoSuchNamespaceError as e: + raise NoSuchNamespaceError(f"Database does not exists: {database_name}") from e + + if len(table_list) > 0: + raise NamespaceNotEmptyError(f"Database {database_name} is not empty") + + self.glue.delete_database(Name=database_name) + + def list_tables(self, namespace: Union[str, Identifier]) -> List[Identifier]: + """List tables under the given namespace in the catalog (including non-Iceberg tables) + + Args: + namespace (str | Identifier): Namespace identifier to search. + + Returns: + List[Identifier]: list of table identifiers. + + Raises: + NoSuchNamespaceError: If a namespace with the given name does not exist, or the identifier is invalid + """ + + database_name = self.identifier_to_database(namespace, NoSuchNamespaceError) + table_list = [] + try: + table_list_response = self.glue.get_tables(DatabaseName=database_name) + next_token = table_list_response.get(PROP_GLUE_NEXT_TOKEN) + table_list += table_list_response.get(PROP_GLUE_TABLELIST, []) + while next_token: + table_list_response = self.glue.get_tables(DatabaseName=database_name, NextToken=next_token) + next_token = table_list_response.get(PROP_GLUE_NEXT_TOKEN) + table_list += table_list_response.get(PROP_GLUE_TABLELIST, []) + except self.glue.exceptions.EntityNotFoundException as e: + raise NoSuchNamespaceError(f"Database does not exists: {database_name}") from e + return [(database_name, table.get(PROP_GLUE_TABLE_NAME)) for table in table_list] + + def list_namespaces(self, namespace: Union[str, Identifier] = ()) -> List[Identifier]: + """List namespaces from the given namespace. If not given, list top-level namespaces from the catalog. + + Returns: + List[Identifier]: a List of namespace identifiers + """ + # Glue does not support hierarchical namespace, therefore return an empty list + if namespace: + return [] + database_list = [] + databases_response = self.glue.get_databases() + next_token = databases_response.get(PROP_GLUE_NEXT_TOKEN) + database_list += databases_response.get(PROP_GLUE_DATABASE_LIST, []) + while next_token: + databases_response = self.glue.get_databases(NextToken=next_token) + next_token = databases_response.get(PROP_GLUE_NEXT_TOKEN) + database_list += databases_response.get(PROP_GLUE_DATABASE_LIST, []) + return [self.identifier_to_tuple(database.get(PROP_GLUE_DATABASE_NAME)) for database in database_list] + + def load_namespace_properties(self, namespace: Union[str, Identifier]) -> Properties: + """Get properties for a namespace. + + Args: + namespace: Namespace identifier + + Returns: + Properties: Properties for the given namespace + + Raises: + NoSuchNamespaceError: If a namespace with the given name does not exist, or identifier is invalid + """ + database_name = self.identifier_to_database(namespace, NoSuchNamespaceError) + try: + database_response = self.glue.get_database(Name=database_name) + except self.glue.exceptions.EntityNotFoundException as e: + raise NoSuchNamespaceError(f"Database does not exists: {database_name}") from e + except self.glue.exceptions.InvalidInputException as e: + raise NoSuchNamespaceError(f"Invalid input for namespace {database_name}") from e + + database = database_response[PROP_GLUE_DATABASE] + if PROP_GLUE_DATABASE_PARAMETERS not in database: + return {} + + properties = dict(database[PROP_GLUE_DATABASE_PARAMETERS]) + if database_location := database.get(PROP_GLUE_DATABASE_LOCATION): + properties[GLUE_DATABASE_LOCATION_KEY] = database_location + if database_description := database.get(PROP_GLUE_DATABASE_DESCRIPTION): + properties[GLUE_DESCRIPTION_KEY] = database_description + + return properties + + def update_namespace_properties( + self, namespace: Union[str, Identifier], removals: Optional[Set[str]] = None, updates: Properties = EMPTY_DICT + ) -> PropertiesUpdateSummary: + """Removes provided property keys and updates properties for a namespace. + + Args: + namespace: Namespace identifier + removals: Set of property keys that need to be removed. Optional Argument. + updates: Properties to be updated for the given namespace. + + Raises: + NoSuchNamespaceError: If a namespace with the given name does not exist， or identifier is invalid + ValueError: If removals and updates have overlapping keys. + """ + removed: Set[str] = set() + updated: Set[str] = set() + + if updates and removals: + overlap = set(removals) & set(updates.keys()) + if overlap: + raise ValueError(f"Updates and deletes have an overlap: {overlap}") + database_name = self.identifier_to_database(namespace, NoSuchNamespaceError) + current_properties = self.load_namespace_properties(namespace=database_name) + new_properties = dict(current_properties) + + if removals: + for key in removals: + if key in new_properties: + new_properties.pop(key) + removed.add(key) + if updates: + for key, value in updates.items(): + new_properties[key] = value + updated.add(key) + + self.glue.update_database(Name=database_name, DatabaseInput=_construct_database_input(database_name, new_properties)) + + expected_to_change = (removals or set()).difference(removed) + + return PropertiesUpdateSummary( + removed=list(removed or []), updated=list(updates.keys() if updates else []), missing=list(expected_to_change) + ) diff --git a/pyiceberg/exceptions.py b/pyiceberg/exceptions.py index b3cc2778e2..0438a5322a 100644 --- a/pyiceberg/exceptions.py +++ b/pyiceberg/exceptions.py @@ -36,6 +36,10 @@ class NoSuchTableError(Exception): """Raises when the table can't be found in the REST catalog""" +class NoSuchIcebergTableError(NoSuchTableError): + """Raises when the table found in the REST catalog is not an iceberg table""" + + class NoSuchNamespaceError(Exception): """Raised when a referenced name-space is not found""" diff --git a/pyproject.toml b/pyproject.toml index 181093e469..514ad9e74f 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -64,6 +64,7 @@ python-snappy = { version = "0.6.1", optional = true } thrift = { version = "0.16.0", optional = true } s3fs = { version = "2022.10.0", optional = true } +boto3 = {version = "1.24.59", optional = true} [tool.poetry.dev-dependencies] pytest = "7.2.0" @@ -72,6 +73,7 @@ pre-commit = "2.20.0" fastavro = "1.6.1" coverage = { version = "^6.5.0", extras = ["toml"] } requests-mock = "1.10.0" +moto = "^4.0.6" typing-extensions = '4.4.0' [tool.poetry.scripts] @@ -87,6 +89,7 @@ pyarrow = ["pyarrow"] snappy = ["python-snappy"] hive = ["thrift"] s3fs = ["s3fs"] +glue = ["boto3"] [tool.pytest.ini_options] markers = [ @@ -178,9 +181,25 @@ ignore_missing_imports = true module = "tests.*" ignore_missing_imports = true +[[tool.mypy.overrides]] +module = "boto3" +ignore_missing_imports = true + [[tool.mypy.overrides]] module = "botocore.*" ignore_missing_imports = true +[[tool.mypy.overrides]] +module = "moto" +ignore_missing_imports = true + +[[tool.mypy.overrides]] +module = "aiobotocore.*" +ignore_missing_imports = true + +[[tool.mypy.overrides]] +module = "aiohttp.*" +ignore_missing_imports = true + [tool.coverage.run] source = ['pyiceberg/'] diff --git a/tests/catalog/integration_test_glue.py b/tests/catalog/integration_test_glue.py new file mode 100644 index 0000000000..70ac3a00a4 --- /dev/null +++ b/tests/catalog/integration_test_glue.py @@ -0,0 +1,324 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +import os + +import boto3 +import pytest +from botocore.exceptions import ClientError + +from pyiceberg.catalog.glue import GlueCatalog +from pyiceberg.exceptions import ( + NamespaceAlreadyExistsError, + NamespaceNotEmptyError, + NoSuchNamespaceError, + NoSuchTableError, + TableAlreadyExistsError, +) +from pyiceberg.schema import Schema +from tests.catalog.test_glue import ( + get_random_database_name, + get_random_databases, + get_random_table_name, + get_random_tables, +) + +# The number of random characters in generated table/database name +RANDOM_LENGTH = 20 +# The number of tables/databases used in list_table/namespace test +LIST_TEST_NUMBER = 2 + + +def get_bucket_name(): + """ + Set the environment variable AWS_TEST_BUCKET for a default bucket to test + """ + bucket_name = os.getenv("AWS_TEST_BUCKET") + if bucket_name is None: + raise ValueError("Please specify a bucket to run the test by setting environment variable AWS_TEST_BUCKET") + return bucket_name + + +def get_s3_path(bucket_name, database_name=None, table_name=None): + result_path = f"s3://{bucket_name}" + if database_name is not None: + result_path += f"/{database_name}.db" + + if table_name is not None: + result_path += f"/{table_name}" + return result_path + + +@pytest.fixture(name="s3", scope="module") +def fixture_s3_client(): + yield boto3.client("s3") + + +@pytest.fixture(name="glue", scope="module") +def fixture_glue_client(): + yield boto3.client("glue") + + +def clean_up(test_catalog): + """Clean all databases and tables created during the integration test""" + for database_name in test_catalog.list_namespaces(): + database_name = database_name[0] + if "my_iceberg_database-" in database_name: + for identifier in test_catalog.list_tables(database_name): + test_catalog.purge_table(identifier) + test_catalog.drop_namespace(database_name) + + +@pytest.fixture(name="test_catalog", scope="module") +def fixture_test_catalog(): + """The pre- and post-setting of aws integration test""" + test_catalog = GlueCatalog("glue", warehouse=get_s3_path(get_bucket_name())) + yield test_catalog + clean_up(test_catalog) + + +def test_create_table(test_catalog, s3, table_schema_nested: Schema): + table_name = get_random_table_name() + database_name = get_random_database_name() + identifier = (database_name, table_name) + test_catalog.create_namespace(database_name) + test_catalog.create_table(identifier, table_schema_nested, get_s3_path(get_bucket_name(), database_name, table_name)) + table = test_catalog.load_table(identifier) + assert table.identifier == identifier + metadata_location = table.metadata_location.split(get_bucket_name())[1][1:] + s3.head_object(Bucket=get_bucket_name(), Key=metadata_location) + + +def test_create_table_with_invalid_location(table_schema_nested: Schema): + table_name = get_random_table_name() + database_name = get_random_database_name() + identifier = (database_name, table_name) + test_catalog_no_warehouse = GlueCatalog("glue") + test_catalog_no_warehouse.create_namespace(database_name) + with pytest.raises(ValueError): + test_catalog_no_warehouse.create_table(identifier, table_schema_nested) + test_catalog_no_warehouse.drop_namespace(database_name) + + +def test_create_table_with_default_location(test_catalog, s3, table_schema_nested: Schema): + table_name = get_random_table_name() + database_name = get_random_database_name() + identifier = (database_name, table_name) + test_catalog.create_namespace(database_name) + test_catalog.create_table(identifier, table_schema_nested) + table = test_catalog.load_table(identifier) + assert table.identifier == identifier + metadata_location = table.metadata_location.split(get_bucket_name())[1][1:] + s3.head_object(Bucket=get_bucket_name(), Key=metadata_location) + + +def test_create_table_with_invalid_database(test_catalog, table_schema_nested: Schema): + table_name = get_random_table_name() + identifier = ("invalid", table_name) + with pytest.raises(NoSuchNamespaceError): + test_catalog.create_table(identifier, table_schema_nested) + + +def test_create_duplicated_table(test_catalog, table_schema_nested: Schema): + table_name = get_random_table_name() + database_name = get_random_database_name() + test_catalog.create_namespace(database_name) + test_catalog.create_table((database_name, table_name), table_schema_nested) + with pytest.raises(TableAlreadyExistsError): + test_catalog.create_table((database_name, table_name), table_schema_nested) + + +def test_load_table(test_catalog, table_schema_nested): + table_name = get_random_table_name() + database_name = get_random_database_name() + identifier = (database_name, table_name) + test_catalog.create_namespace(database_name) + table = test_catalog.create_table(identifier, table_schema_nested) + loaded_table = test_catalog.load_table(identifier) + assert table.identifier == loaded_table.identifier + assert table.metadata_location == loaded_table.metadata_location + assert table.metadata == loaded_table.metadata + + +def test_list_tables(test_catalog, table_schema_nested): + test_tables = get_random_tables(LIST_TEST_NUMBER) + database_name = get_random_database_name() + test_catalog.create_namespace(database_name) + for table_name in test_tables: + test_catalog.create_table((database_name, table_name), table_schema_nested) + identifier_list = test_catalog.list_tables(database_name) + assert len(identifier_list) == LIST_TEST_NUMBER + for table_name in test_tables: + assert (database_name, table_name) in identifier_list + + +def test_rename_table(test_catalog, s3, table_schema_nested): + table_name = get_random_table_name() + database_name = get_random_database_name() + new_database_name = get_random_database_name() + test_catalog.create_namespace(database_name) + test_catalog.create_namespace(new_database_name) + new_table_name = f"rename-{table_name}" + identifier = (database_name, table_name) + table = test_catalog.create_table(identifier, table_schema_nested) + assert table.identifier == identifier + new_identifier = (new_database_name, new_table_name) + test_catalog.rename_table(identifier, new_identifier) + new_table = test_catalog.load_table(new_identifier) + assert new_table.identifier == new_identifier + assert new_table.metadata_location == table.metadata_location + metadata_location = new_table.metadata_location.split(get_bucket_name())[1][1:] + s3.head_object(Bucket=get_bucket_name(), Key=metadata_location) + with pytest.raises(NoSuchTableError): + test_catalog.load_table(identifier) + + +def test_drop_table(test_catalog, table_schema_nested): + table_name = get_random_table_name() + database_name = get_random_database_name() + identifier = (database_name, table_name) + test_catalog.create_namespace(database_name) + table = test_catalog.create_table(identifier, table_schema_nested) + assert table.identifier == identifier + test_catalog.drop_table(identifier) + with pytest.raises(NoSuchTableError): + test_catalog.load_table(identifier) + + +def test_purge_table(test_catalog, s3, table_schema_nested): + table_name = get_random_table_name() + database_name = get_random_database_name() + identifier = (database_name, table_name) + test_catalog.create_namespace(database_name) + test_catalog.create_table(identifier, table_schema_nested) + table = test_catalog.load_table(identifier) + assert table.identifier == identifier + metadata_location = table.metadata_location.split(get_bucket_name())[1][1:] + s3.head_object(Bucket=get_bucket_name(), Key=metadata_location) + test_catalog.purge_table(identifier) + with pytest.raises(NoSuchTableError): + test_catalog.load_table(identifier) + with pytest.raises(ClientError): + s3.head_object(Bucket=get_bucket_name(), Key=metadata_location) + + +def test_create_namespace(test_catalog): + database_name = get_random_database_name() + test_catalog.create_namespace(database_name) + assert (database_name,) in test_catalog.list_namespaces() + + +def test_create_duplicate_namespace(test_catalog): + database_name = get_random_database_name() + test_catalog.create_namespace(database_name) + with pytest.raises(NamespaceAlreadyExistsError): + test_catalog.create_namespace(database_name) + + +def test_create_namespace_with_comment_and_location(test_catalog): + database_name = get_random_database_name() + test_location = get_s3_path(get_bucket_name(), database_name) + test_properties = { + "comment": "this is a test description", + "location": test_location, + } + test_catalog.create_namespace(namespace=database_name, properties=test_properties) + loaded_database_list = test_catalog.list_namespaces() + assert (database_name,) in loaded_database_list + properties = test_catalog.load_namespace_properties(database_name) + assert properties["comment"] == "this is a test description" + assert properties["location"] == test_location + + +def test_list_namespaces(test_catalog): + database_list = get_random_databases(LIST_TEST_NUMBER) + for database_name in database_list: + test_catalog.create_namespace(database_name) + db_list = test_catalog.list_namespaces() + for database_name in database_list: + assert (database_name,) in db_list + assert len(test_catalog.list_namespaces(list(database_list)[0])) == 0 + + +def test_drop_namespace(test_catalog, table_schema_nested: Schema): + table_name = get_random_table_name() + database_name = get_random_database_name() + test_catalog.create_namespace(database_name) + assert (database_name,) in test_catalog.list_namespaces() + test_catalog.create_table((database_name, table_name), table_schema_nested) + with pytest.raises(NamespaceNotEmptyError): + test_catalog.drop_namespace(database_name) + test_catalog.drop_table((database_name, table_name)) + test_catalog.drop_namespace(database_name) + assert (database_name,) not in test_catalog.list_namespaces() + + +def test_load_namespace_properties(test_catalog): + warehouse_location = get_s3_path(get_bucket_name()) + database_name = get_random_database_name() + test_properties = { + "comment": "this is a test description", + "location": f"{warehouse_location}/{database_name}.db", + "test_property1": "1", + "test_property2": "2", + "test_property3": "3", + } + + test_catalog.create_namespace(database_name, test_properties) + listed_properties = test_catalog.load_namespace_properties(database_name) + for k, v in listed_properties.items(): + assert k in test_properties + assert v == test_properties[k] + + +def test_load_empty_namespace_properties(test_catalog): + database_name = get_random_database_name() + test_catalog.create_namespace(database_name) + listed_properties = test_catalog.load_namespace_properties(database_name) + assert listed_properties == {} + + +def test_load_default_namespace_properties(test_catalog, glue): + database_name = get_random_database_name() + # simulate creating database with default settings through AWS Glue Web Console + glue.create_database(DatabaseInput={"Name": database_name}) + listed_properties = test_catalog.load_namespace_properties(database_name) + assert listed_properties == {} + + +def test_update_namespace_properties(test_catalog): + warehouse_location = get_s3_path(get_bucket_name()) + database_name = get_random_database_name() + test_properties = { + "comment": "this is a test description", + "location": f"{warehouse_location}/{database_name}.db", + "test_property1": "1", + "test_property2": "2", + "test_property3": "3", + } + removals = {"test_property1", "test_property2", "test_property3", "should_not_removed"} + updates = {"test_property4": "4", "test_property5": "5", "comment": "updated test description"} + test_catalog.create_namespace(database_name, test_properties) + update_report = test_catalog.update_namespace_properties(database_name, removals, updates) + for k in updates.keys(): + assert k in update_report.updated + for k in removals: + if k == "should_not_removed": + assert k in update_report.missing + else: + assert k in update_report.removed + assert "updated test description" == test_catalog.load_namespace_properties(database_name)["comment"] diff --git a/tests/catalog/test_glue.py b/tests/catalog/test_glue.py new file mode 100644 index 0000000000..ccdeaff033 --- /dev/null +++ b/tests/catalog/test_glue.py @@ -0,0 +1,471 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +import random +import re +import string + +import pytest +from moto import mock_glue + +from pyiceberg.catalog.glue import GlueCatalog +from pyiceberg.exceptions import ( + NamespaceAlreadyExistsError, + NamespaceNotEmptyError, + NoSuchIcebergTableError, + NoSuchNamespaceError, + NoSuchPropertyException, + NoSuchTableError, + TableAlreadyExistsError, +) + +BUCKET_NAME = "test_bucket" +RANDOM_LENGTH = 20 +LIST_TEST_NUMBER = 100 +table_metadata_location_regex = re.compile( + r"""s3://test_bucket/my_iceberg_database-[a-z]{20}.db/ + my_iceberg_table-[a-z]{20}/metadata/ + 00000-[a-f0-9]{8}-?[a-f0-9]{4}-?4[a-f0-9]{3}-?[89ab][a-f0-9]{3}-?[a-f0-9]{12}.metadata.json""", + re.X, +) + + +def get_random_table_name(): + prefix = "my_iceberg_table-" + random_tag = "".join(random.choice(string.ascii_letters) for _ in range(RANDOM_LENGTH)) + return (prefix + random_tag).lower() + + +def get_random_tables(n): + result = set() + for _ in range(n): + result.add(get_random_table_name()) + return result + + +def get_random_database_name(): + prefix = "my_iceberg_database-" + random_tag = "".join(random.choice(string.ascii_letters) for _ in range(RANDOM_LENGTH)) + return (prefix + random_tag).lower() + + +def get_random_databases(n): + result = set() + for _ in range(n): + result.add(get_random_database_name()) + return result + + +@pytest.fixture(name="_bucket_initialize") +def fixture_s3_bucket(_s3): + _s3.create_bucket(Bucket=BUCKET_NAME) + + +@mock_glue +def test_create_table_with_database_location(_bucket_initialize, _patch_aiobotocore, table_schema_nested): + database_name = get_random_database_name() + table_name = get_random_table_name() + identifier = (database_name, table_name) + test_catalog = GlueCatalog("glue") + test_catalog.create_namespace(namespace=database_name, properties={"location": f"s3://{BUCKET_NAME}/{database_name}.db"}) + table = test_catalog.create_table(identifier, table_schema_nested) + assert table.identifier == identifier + assert table_metadata_location_regex.match(table.metadata_location) + + +@mock_glue +def test_create_table_with_default_warehouse(_bucket_initialize, _patch_aiobotocore, table_schema_nested): + database_name = get_random_database_name() + table_name = get_random_table_name() + identifier = (database_name, table_name) + test_catalog = GlueCatalog("glue", warehouse=f"s3://{BUCKET_NAME}") + test_catalog.create_namespace(namespace=database_name) + table = test_catalog.create_table(identifier, table_schema_nested) + assert table.identifier == identifier + assert table_metadata_location_regex.match(table.metadata_location) + + +@mock_glue +def test_create_table_with_given_location(_bucket_initialize, _patch_aiobotocore, table_schema_nested): + database_name = get_random_database_name() + table_name = get_random_table_name() + identifier = (database_name, table_name) + test_catalog = GlueCatalog("glue") + test_catalog.create_namespace(namespace=database_name) + table = test_catalog.create_table( + identifier=identifier, schema=table_schema_nested, location=f"s3://{BUCKET_NAME}/{database_name}.db/{table_name}" + ) + assert table.identifier == identifier + assert table_metadata_location_regex.match(table.metadata_location) + + +@mock_glue +def test_create_table_with_no_location(_bucket_initialize, _patch_aiobotocore, table_schema_nested): + database_name = get_random_database_name() + table_name = get_random_table_name() + identifier = (database_name, table_name) + test_catalog = GlueCatalog("glue") + test_catalog.create_namespace(namespace=database_name) + with pytest.raises(ValueError): + test_catalog.create_table(identifier=identifier, schema=table_schema_nested) + + +@mock_glue +def test_create_table_with_strips(_bucket_initialize, _patch_aiobotocore, table_schema_nested): + database_name = get_random_database_name() + table_name = get_random_table_name() + identifier = (database_name, table_name) + test_catalog = GlueCatalog("glue") + test_catalog.create_namespace(namespace=database_name, properties={"location": f"s3://{BUCKET_NAME}/{database_name}.db/"}) + table = test_catalog.create_table(identifier, table_schema_nested) + assert table.identifier == identifier + assert table_metadata_location_regex.match(table.metadata_location) + database_name = get_random_database_name() + table_name = get_random_table_name() + identifier = (database_name, table_name) + test_catalog_strip = GlueCatalog("glue", warehouse=f"s3://{BUCKET_NAME}/") + test_catalog_strip.create_namespace(namespace=database_name) + table_strip = test_catalog_strip.create_table(identifier, table_schema_nested) + assert table_strip.identifier == identifier + assert table_metadata_location_regex.match(table_strip.metadata_location) + + +@mock_glue +def test_create_table_with_no_database(_bucket_initialize, _patch_aiobotocore, table_schema_nested): + database_name = get_random_database_name() + table_name = get_random_table_name() + identifier = (database_name, table_name) + test_catalog = GlueCatalog("glue") + with pytest.raises(NoSuchNamespaceError): + test_catalog.create_table(identifier=identifier, schema=table_schema_nested) + + +@mock_glue +def test_create_duplicated_table(_bucket_initialize, _patch_aiobotocore, table_schema_nested): + database_name = get_random_database_name() + table_name = get_random_table_name() + identifier = (database_name, table_name) + test_catalog = GlueCatalog("glue", warehouse=f"s3://{BUCKET_NAME}") + test_catalog.create_namespace(namespace=database_name) + test_catalog.create_table(identifier, table_schema_nested) + with pytest.raises(TableAlreadyExistsError): + test_catalog.create_table(identifier, table_schema_nested) + + +@mock_glue +def test_load_table(_bucket_initialize, _patch_aiobotocore, table_schema_nested): + database_name = get_random_database_name() + table_name = get_random_table_name() + identifier = (database_name, table_name) + test_catalog = GlueCatalog("glue", warehouse=f"s3://{BUCKET_NAME}") + test_catalog.create_namespace(namespace=database_name) + test_catalog.create_table(identifier, table_schema_nested) + table = test_catalog.load_table(identifier) + assert table.identifier == identifier + assert table_metadata_location_regex.match(table.metadata_location) + + +@mock_glue +def test_load_non_exist_table(_bucket_initialize, _patch_aiobotocore): + database_name = get_random_database_name() + table_name = get_random_table_name() + identifier = (database_name, table_name) + test_catalog = GlueCatalog("glue", warehouse=f"s3://{BUCKET_NAME}") + test_catalog.create_namespace(namespace=database_name) + with pytest.raises(NoSuchTableError): + test_catalog.load_table(identifier) + + +@mock_glue +def test_drop_table(_bucket_initialize, _patch_aiobotocore, table_schema_nested): + database_name = get_random_database_name() + table_name = get_random_table_name() + identifier = (database_name, table_name) + test_catalog = GlueCatalog("glue", warehouse=f"s3://{BUCKET_NAME}") + test_catalog.create_namespace(namespace=database_name) + test_catalog.create_table(identifier, table_schema_nested) + table = test_catalog.load_table(identifier) + assert table.identifier == identifier + assert table_metadata_location_regex.match(table.metadata_location) + test_catalog.drop_table(identifier) + with pytest.raises(NoSuchTableError): + test_catalog.load_table(identifier) + + +@mock_glue +def test_drop_non_exist_table(_bucket_initialize, _patch_aiobotocore): + database_name = get_random_database_name() + table_name = get_random_table_name() + identifier = (database_name, table_name) + test_catalog = GlueCatalog("glue", warehouse=f"s3://{BUCKET_NAME}") + with pytest.raises(NoSuchTableError): + test_catalog.drop_table(identifier) + + +@mock_glue +def test_rename_table(_bucket_initialize, _patch_aiobotocore, table_schema_nested): + database_name = get_random_database_name() + table_name = get_random_table_name() + new_table_name = get_random_table_name() + identifier = (database_name, table_name) + new_identifier = (database_name, new_table_name) + test_catalog = GlueCatalog("glue", warehouse=f"s3://{BUCKET_NAME}") + test_catalog.create_namespace(namespace=database_name) + table = test_catalog.create_table(identifier, table_schema_nested) + assert table.identifier == identifier + assert table_metadata_location_regex.match(table.metadata_location) + test_catalog.rename_table(identifier, new_identifier) + new_table = test_catalog.load_table(new_identifier) + assert new_table.identifier == new_identifier + # the metadata_location should not change + assert new_table.metadata_location == table.metadata_location + # old table should be dropped + with pytest.raises(NoSuchTableError): + test_catalog.load_table(identifier) + + +@mock_glue +def test_rename_table_no_params(_glue, _bucket_initialize, _patch_aiobotocore): + database_name = get_random_database_name() + new_database_name = get_random_database_name() + table_name = get_random_table_name() + new_table_name = get_random_table_name() + identifier = (database_name, table_name) + new_identifier = (new_database_name, new_table_name) + test_catalog = GlueCatalog("glue", warehouse=f"s3://{BUCKET_NAME}") + test_catalog.create_namespace(namespace=database_name) + test_catalog.create_namespace(namespace=new_database_name) + _glue.create_table( + DatabaseName=database_name, + TableInput={"Name": table_name, "TableType": "EXTERNAL_TABLE", "Parameters": {"table_type": "iceberg"}}, + ) + with pytest.raises(NoSuchPropertyException): + test_catalog.rename_table(identifier, new_identifier) + + +@mock_glue +def test_rename_non_iceberg_table(_glue, _bucket_initialize, _patch_aiobotocore): + database_name = get_random_database_name() + new_database_name = get_random_database_name() + table_name = get_random_table_name() + new_table_name = get_random_table_name() + identifier = (database_name, table_name) + new_identifier = (new_database_name, new_table_name) + test_catalog = GlueCatalog("glue", warehouse=f"s3://{BUCKET_NAME}") + test_catalog.create_namespace(namespace=database_name) + test_catalog.create_namespace(namespace=new_database_name) + _glue.create_table( + DatabaseName=database_name, + TableInput={ + "Name": table_name, + "TableType": "EXTERNAL_TABLE", + "Parameters": {"table_type": "noniceberg", "metadata_location": "test"}, + }, + ) + with pytest.raises(NoSuchIcebergTableError): + test_catalog.rename_table(identifier, new_identifier) + + +@mock_glue +def test_list_tables(_bucket_initialize, _patch_aiobotocore, table_schema_nested): + database_name = get_random_database_name() + table_list = get_random_tables(LIST_TEST_NUMBER) + test_catalog = GlueCatalog("glue", warehouse=f"s3://{BUCKET_NAME}") + test_catalog.create_namespace(namespace=database_name) + for table_name in table_list: + test_catalog.create_table((database_name, table_name), table_schema_nested) + loaded_table_list = test_catalog.list_tables(database_name) + for table_name in table_list: + assert (database_name, table_name) in loaded_table_list + + +@mock_glue +def test_list_namespaces(_bucket_initialize, _patch_aiobotocore): + database_list = get_random_databases(LIST_TEST_NUMBER) + test_catalog = GlueCatalog("glue") + for database_name in database_list: + test_catalog.create_namespace(namespace=database_name) + loaded_database_list = test_catalog.list_namespaces() + for database_name in database_list: + assert (database_name,) in loaded_database_list + + +@mock_glue +def test_create_namespace_no_properties(_bucket_initialize, _patch_aiobotocore): + database_name = get_random_database_name() + test_catalog = GlueCatalog("glue") + test_catalog.create_namespace(namespace=database_name) + loaded_database_list = test_catalog.list_namespaces() + assert len(loaded_database_list) == 1 + assert (database_name,) in loaded_database_list + properties = test_catalog.load_namespace_properties(database_name) + assert properties == {} + + +@mock_glue +def test_create_namespace_with_comment_and_location(_bucket_initialize, _patch_aiobotocore): + database_name = get_random_database_name() + test_location = f"s3://{BUCKET_NAME}/{database_name}.db" + test_properties = { + "comment": "this is a test description", + "location": test_location, + } + test_catalog = GlueCatalog("glue") + test_catalog.create_namespace(namespace=database_name, properties=test_properties) + loaded_database_list = test_catalog.list_namespaces() + assert len(loaded_database_list) == 1 + assert (database_name,) in loaded_database_list + properties = test_catalog.load_namespace_properties(database_name) + assert properties["comment"] == "this is a test description" + assert properties["location"] == test_location + + +@mock_glue +def test_create_duplicated_namespace(_bucket_initialize, _patch_aiobotocore): + database_name = get_random_database_name() + test_catalog = GlueCatalog("glue") + test_catalog.create_namespace(namespace=database_name) + loaded_database_list = test_catalog.list_namespaces() + assert len(loaded_database_list) == 1 + assert (database_name,) in loaded_database_list + with pytest.raises(NamespaceAlreadyExistsError): + test_catalog.create_namespace(namespace=database_name, properties={"test": "test"}) + + +@mock_glue +def test_drop_namespace(_bucket_initialize, _patch_aiobotocore): + database_name = get_random_database_name() + test_catalog = GlueCatalog("glue") + test_catalog.create_namespace(namespace=database_name) + loaded_database_list = test_catalog.list_namespaces() + assert len(loaded_database_list) == 1 + assert (database_name,) in loaded_database_list + test_catalog.drop_namespace(database_name) + loaded_database_list = test_catalog.list_namespaces() + assert len(loaded_database_list) == 0 + + +@mock_glue +def test_drop_non_empty_namespace(_bucket_initialize, _patch_aiobotocore, table_schema_nested): + database_name = get_random_database_name() + table_name = get_random_table_name() + identifier = (database_name, table_name) + test_catalog = GlueCatalog("glue", warehouse=f"s3://{BUCKET_NAME}") + test_catalog.create_namespace(namespace=database_name) + test_catalog.create_table(identifier, table_schema_nested) + assert len(test_catalog.list_tables(database_name)) == 1 + with pytest.raises(NamespaceNotEmptyError): + test_catalog.drop_namespace(database_name) + + +@mock_glue +def test_drop_non_exist_namespace(_bucket_initialize, _patch_aiobotocore): + database_name = get_random_database_name() + test_catalog = GlueCatalog("glue") + with pytest.raises(NoSuchNamespaceError): + test_catalog.drop_namespace(database_name) + + +@mock_glue +def test_load_namespace_properties(_bucket_initialize, _patch_aiobotocore): + database_name = get_random_database_name() + test_location = f"s3://{BUCKET_NAME}/{database_name}.db" + test_properties = { + "comment": "this is a test description", + "location": test_location, + "test_property1": "1", + "test_property2": "2", + "test_property3": "3", + } + test_catalog = GlueCatalog("glue") + test_catalog.create_namespace(database_name, test_properties) + listed_properties = test_catalog.load_namespace_properties(database_name) + for k, v in listed_properties.items(): + assert k in test_properties + assert v == test_properties[k] + + +@mock_glue +def test_load_non_exist_namespace_properties(_bucket_initialize, _patch_aiobotocore): + database_name = get_random_database_name() + test_catalog = GlueCatalog("glue") + with pytest.raises(NoSuchNamespaceError): + test_catalog.load_namespace_properties(database_name) + + +@mock_glue +def test_update_namespace_properties(_bucket_initialize, _patch_aiobotocore): + database_name = get_random_database_name() + test_properties = { + "comment": "this is a test description", + "location": f"s3://{BUCKET_NAME}/{database_name}.db", + "test_property1": "1", + "test_property2": "2", + "test_property3": "3", + } + removals = {"test_property1", "test_property2", "test_property3", "should_not_removed"} + updates = {"test_property4": "4", "test_property5": "5", "comment": "updated test description"} + test_catalog = GlueCatalog("glue") + test_catalog.create_namespace(database_name, test_properties) + update_report = test_catalog.update_namespace_properties(database_name, removals, updates) + for k in updates.keys(): + assert k in update_report.updated + for k in removals: + if k == "should_not_removed": + assert k in update_report.missing + else: + assert k in update_report.removed + assert "updated test description" == test_catalog.load_namespace_properties(database_name)["comment"] + test_catalog.drop_namespace(database_name) + + +@mock_glue +def test_load_empty_namespace_properties(_bucket_initialize, _patch_aiobotocore): + database_name = get_random_database_name() + test_catalog = GlueCatalog("glue") + test_catalog.create_namespace(database_name) + listed_properties = test_catalog.load_namespace_properties(database_name) + assert listed_properties == {} + + +@mock_glue +def test_load_default_namespace_properties(_glue, _bucket_initialize, _patch_aiobotocore): + database_name = get_random_database_name() + # simulate creating database with default settings through AWS Glue Web Console + _glue.create_database(DatabaseInput={"Name": database_name}) + test_catalog = GlueCatalog("glue") + listed_properties = test_catalog.load_namespace_properties(database_name) + assert listed_properties == {} + + +@mock_glue +def test_update_namespace_properties_overlap_update_removal(_bucket_initialize, _patch_aiobotocore): + database_name = get_random_database_name() + test_properties = { + "comment": "this is a test description", + "location": f"s3://{BUCKET_NAME}/{database_name}.db", + "test_property1": "1", + "test_property2": "2", + "test_property3": "3", + } + removals = {"test_property1", "test_property2", "test_property3", "should_not_removed"} + updates = {"test_property1": "4", "test_property5": "5", "comment": "updated test description"} + test_catalog = GlueCatalog("glue") + test_catalog.create_namespace(database_name, test_properties) + with pytest.raises(ValueError): + test_catalog.update_namespace_properties(database_name, removals, updates) + # should not modify the properties + assert test_catalog.load_namespace_properties(database_name) == test_properties diff --git a/tests/conftest.py b/tests/conftest.py index be51c90357..d11d155b47 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -28,13 +28,24 @@ from tempfile import TemporaryDirectory from typing import ( Any, + Callable, Dict, Generator, Union, ) +from unittest.mock import MagicMock from urllib.parse import urlparse +import aiobotocore.awsrequest +import aiobotocore.endpoint +import aiohttp +import aiohttp.client_reqrep +import aiohttp.typedefs +import boto3 +import botocore.awsrequest +import botocore.model import pytest +from moto import mock_glue, mock_s3 from pyiceberg import schema from pyiceberg.io import ( @@ -1142,3 +1153,105 @@ def fsspec_fileio(request): "s3.secret-access-key": request.config.getoption("--s3.secret-access-key"), } return fsspec.FsspecFileIO(properties=properties) + + +class MockAWSResponse(aiobotocore.awsrequest.AioAWSResponse): + """ + A mocked aws response implementation (for test use only) + See https://github.com/aio-libs/aiobotocore/issues/755 + """ + + def __init__(self, response: botocore.awsrequest.AWSResponse): + self._moto_response = response + self.status_code = response.status_code + self.raw = MockHttpClientResponse(response) + + # adapt async methods to use moto's response + async def _content_prop(self) -> bytes: + return self._moto_response.content + + async def _text_prop(self) -> str: + return self._moto_response.text + + +class MockHttpClientResponse(aiohttp.client_reqrep.ClientResponse): + """ + A mocked http client response implementation (for test use only) + See https://github.com/aio-libs/aiobotocore/issues/755 + """ + + def __init__(self, response: botocore.awsrequest.AWSResponse): + async def read(*_) -> bytes: + # streaming/range requests. used by s3fs + return response.content + + self.content = MagicMock(aiohttp.StreamReader) + self.content.read = read + self.response = response + + @property + def raw_headers(self) -> aiohttp.typedefs.RawHeaders: + # Return the headers encoded the way that aiobotocore expects them + return {k.encode("utf-8"): str(v).encode("utf-8") for k, v in self.response.headers.items()}.items() + + +def patch_aiobotocore(): + """ + Patch aiobotocore to work with moto + See https://github.com/aio-libs/aiobotocore/issues/755 + """ + + def factory(original: Callable) -> Callable: + def patched_convert_to_response_dict( + http_response: botocore.awsrequest.AWSResponse, operation_model: botocore.model.OperationModel + ): + return original(MockAWSResponse(http_response), operation_model) + + return patched_convert_to_response_dict + + aiobotocore.endpoint.convert_to_response_dict = factory(aiobotocore.endpoint.convert_to_response_dict) + + +@pytest.fixture(name="_patch_aiobotocore") +def fixture_aiobotocore(): + """ + Patch aiobotocore to work with moto + pending close of this issue: https://github.com/aio-libs/aiobotocore/issues/755 + """ + stored_method = aiobotocore.endpoint.convert_to_response_dict + yield patch_aiobotocore() + # restore the changed method after the fixture is destroyed + aiobotocore.endpoint.convert_to_response_dict = stored_method + + +def aws_credentials(): + os.environ["AWS_ACCESS_KEY_ID"] = "testing" + os.environ["AWS_SECRET_ACCESS_KEY"] = "testing" + os.environ["AWS_SECURITY_TOKEN"] = "testing" + os.environ["AWS_SESSION_TOKEN"] = "testing" + os.environ["AWS_DEFAULT_REGION"] = "us-east-1" + + +@pytest.fixture(name="_aws_credentials") +def fixture_aws_credentials(): + """Mocked AWS Credentials for moto.""" + yield aws_credentials() + os.environ.pop("AWS_ACCESS_KEY_ID") + os.environ.pop("AWS_SECRET_ACCESS_KEY") + os.environ.pop("AWS_SECURITY_TOKEN") + os.environ.pop("AWS_SESSION_TOKEN") + os.environ.pop("AWS_DEFAULT_REGION") + + +@pytest.fixture(name="_s3") +def fixture_s3(_aws_credentials): + """Mocked S3 client""" + with mock_s3(): + yield boto3.client("s3", region_name="us-east-1") + + +@pytest.fixture(name="_glue") +def fixture_glue(_aws_credentials): + """Mocked glue client""" + with mock_glue(): + yield boto3.client("glue", region_name="us-east-1") From 36810a03bdf21dd6768bab314b85d9631db5b1a7 Mon Sep 17 00:00:00 2001 From: Fokko Driesprong Date: Mon, 21 Nov 2022 21:25:42 +0100 Subject: [PATCH 274/642] Python: Disallow Any generics (#6232) Mypy allows us to warn us when a type is missing: `List`, should be `List[str]` This way we don't accidentially forget any type. --- pyiceberg/avro/reader.py | 7 +-- pyiceberg/catalog/rest.py | 6 +-- pyiceberg/expressions/__init__.py | 30 ++++++----- pyiceberg/expressions/literals.py | 26 +++++----- pyiceberg/io/fsspec.py | 4 +- pyiceberg/io/pyarrow.py | 2 +- pyiceberg/manifest.py | 4 +- pyiceberg/schema.py | 2 +- pyiceberg/table/metadata.py | 2 +- pyiceberg/table/partitioning.py | 4 +- pyiceberg/table/sorting.py | 4 +- pyiceberg/transforms.py | 18 +++---- pyiceberg/utils/bin_packing.py | 12 +++-- pyiceberg/utils/deprecated.py | 2 +- pyiceberg/utils/parsing.py | 2 +- pyiceberg/utils/schema_conversion.py | 6 ++- pyiceberg/utils/singleton.py | 2 +- pyproject.toml | 1 + tests/expressions/test_expressions.py | 23 +++++---- tests/expressions/test_visitors.py | 72 +++++++++++++++------------ tests/test_transforms.py | 3 +- 21 files changed, 126 insertions(+), 106 deletions(-) diff --git a/pyiceberg/avro/reader.py b/pyiceberg/avro/reader.py index d1d0638a9d..b193ec91b8 100644 --- a/pyiceberg/avro/reader.py +++ b/pyiceberg/avro/reader.py @@ -34,6 +34,7 @@ from typing import ( Any, Callable, + Dict, List, Optional, Tuple, @@ -67,7 +68,7 @@ from pyiceberg.utils.singleton import Singleton -def _skip_map_array(decoder: BinaryDecoder, skip_entry: Callable) -> None: +def _skip_map_array(decoder: BinaryDecoder, skip_entry: Callable[[], None]) -> None: """Skips over an array or map Both the array and map are encoded similar, and we can re-use @@ -294,7 +295,7 @@ def skip(self, decoder: BinaryDecoder) -> None: class ListReader(Reader): element: Reader - def read(self, decoder: BinaryDecoder) -> list: + def read(self, decoder: BinaryDecoder) -> List[Any]: read_items = [] block_count = decoder.read_int() while block_count != 0: @@ -315,7 +316,7 @@ class MapReader(Reader): key: Reader value: Reader - def read(self, decoder: BinaryDecoder) -> dict: + def read(self, decoder: BinaryDecoder) -> Dict[Any, Any]: read_items = {} block_count = decoder.read_int() while block_count != 0: diff --git a/pyiceberg/catalog/rest.py b/pyiceberg/catalog/rest.py index 46c2e04315..40c2abb588 100644 --- a/pyiceberg/catalog/rest.py +++ b/pyiceberg/catalog/rest.py @@ -172,7 +172,7 @@ class OAuthErrorResponse(IcebergBaseModel): class RestCatalog(Catalog): uri: str session: Session - properties: dict + properties: Properties def __init__( self, @@ -198,9 +198,9 @@ def _create_session(self) -> None: self.session = Session() # Sets the client side and server side SSL cert verification, if provided as properties. if ssl_config := self.properties.get(SSL): - if ssl_ca_bundle := ssl_config.get(CA_BUNDLE): + if ssl_ca_bundle := ssl_config.get(CA_BUNDLE): # type: ignore self.session.verify = ssl_ca_bundle - if ssl_client := ssl_config.get(CLIENT): + if ssl_client := ssl_config.get(CLIENT): # type: ignore if all(k in ssl_client for k in (CERT, KEY)): self.session.cert = (ssl_client[CERT], ssl_client[KEY]) elif ssl_client_cert := ssl_client.get(CERT): diff --git a/pyiceberg/expressions/__init__.py b/pyiceberg/expressions/__init__.py index 7aabd5181a..f39e15006b 100644 --- a/pyiceberg/expressions/__init__.py +++ b/pyiceberg/expressions/__init__.py @@ -36,7 +36,7 @@ from pyiceberg.utils.singleton import Singleton -def _to_unbound_term(term: Union[str, UnboundTerm]) -> UnboundTerm: +def _to_unbound_term(term: Union[str, UnboundTerm[Any]]) -> UnboundTerm[Any]: return Reference(term) if isinstance(term, str) else term @@ -130,7 +130,7 @@ def ref(self) -> BoundReference[L]: return self -class UnboundTerm(Term, Unbound[BoundTerm[L]], ABC): +class UnboundTerm(Term[Any], Unbound[BoundTerm[L]], ABC): """Represents an unbound term.""" @abstractmethod @@ -138,7 +138,7 @@ def bind(self, schema: Schema, case_sensitive: bool = True) -> BoundTerm[L]: ... -class Reference(UnboundTerm): +class Reference(UnboundTerm[Any]): """A reference not yet bound to a field in a schema Args: @@ -174,10 +174,10 @@ def bind(self, schema: Schema, case_sensitive: bool = True) -> BoundReference[L] """ field = schema.find_field(name_or_id=self.name, case_sensitive=case_sensitive) accessor = schema.accessor_for_field(field.field_id) - return self.as_bound(field=field, accessor=accessor) + return self.as_bound(field=field, accessor=accessor) # type: ignore @property - def as_bound(self) -> Type[BoundReference]: + def as_bound(self) -> Type[BoundReference[L]]: return BoundReference[L] @@ -311,9 +311,9 @@ def __eq__(self, other: Any) -> bool: class UnboundPredicate(Generic[L], Unbound[BooleanExpression], BooleanExpression, ABC): - term: UnboundTerm + term: UnboundTerm[Any] - def __init__(self, term: Union[str, UnboundTerm]): + def __init__(self, term: Union[str, UnboundTerm[Any]]): self.term = _to_unbound_term(term) def __eq__(self, other): @@ -325,7 +325,7 @@ def bind(self, schema: Schema, case_sensitive: bool = True) -> BooleanExpression @property @abstractmethod - def as_bound(self) -> Type[BoundPredicate]: + def as_bound(self) -> Type[BoundPredicate[L]]: ... @@ -364,7 +364,7 @@ def __new__(cls, term: BoundTerm[L]): # pylint: disable=W0221 return AlwaysTrue() return super().__new__(cls) - def __invert__(self) -> BoundIsNull: + def __invert__(self) -> BoundIsNull[L]: return BoundIsNull(self.term) @@ -429,7 +429,7 @@ def as_bound(self) -> Type[BoundNotNaN[L]]: class SetPredicate(UnboundPredicate[L], ABC): literals: Set[Literal[L]] - def __init__(self, term: Union[str, UnboundTerm], literals: Union[Iterable[L], Iterable[Literal[L]]]): + def __init__(self, term: Union[str, UnboundTerm[Any]], literals: Union[Iterable[L], Iterable[Literal[L]]]): super().__init__(term) self.literals = _to_literal_set(literals) @@ -514,7 +514,9 @@ def __invert__(self) -> BoundIn[L]: class In(SetPredicate[L]): - def __new__(cls, term: Union[str, UnboundTerm], literals: Union[Iterable[L], Iterable[Literal[L]]]): # pylint: disable=W0221 + def __new__( + cls, term: Union[str, UnboundTerm[Any]], literals: Union[Iterable[L], Iterable[Literal[L]]] + ): # pylint: disable=W0221 literals_set: Set[Literal[L]] = _to_literal_set(literals) count = len(literals_set) if count == 0: @@ -533,7 +535,9 @@ def as_bound(self) -> Type[BoundIn[L]]: class NotIn(SetPredicate[L], ABC): - def __new__(cls, term: Union[str, UnboundTerm], literals: Union[Iterable[L], Iterable[Literal[L]]]): # pylint: disable=W0221 + def __new__( + cls, term: Union[str, UnboundTerm[Any]], literals: Union[Iterable[L], Iterable[Literal[L]]] + ): # pylint: disable=W0221 literals_set: Set[Literal[L]] = _to_literal_set(literals) count = len(literals_set) if count == 0: @@ -559,7 +563,7 @@ def as_bound(self) -> Type[BoundNotIn[L]]: class LiteralPredicate(UnboundPredicate[L], ABC): literal: Literal[L] - def __init__(self, term: Union[str, UnboundTerm], literal: Union[L, Literal[L]]): # pylint: disable=W0621 + def __init__(self, term: Union[str, UnboundTerm[Any]], literal: Union[L, Literal[L]]): # pylint: disable=W0621 super().__init__(term) self.literal = _to_literal(literal) # pylint: disable=W0621 diff --git a/pyiceberg/expressions/literals.py b/pyiceberg/expressions/literals.py index c0ce0188bb..44ab9a15e6 100644 --- a/pyiceberg/expressions/literals.py +++ b/pyiceberg/expressions/literals.py @@ -72,7 +72,7 @@ def value(self) -> L: @singledispatchmethod @abstractmethod - def to(self, type_var: IcebergType) -> Literal: + def to(self, type_var: IcebergType) -> Literal[L]: ... # pragma: no cover def __repr__(self) -> str: @@ -194,7 +194,7 @@ def __init__(self, value: bool): super().__init__(value, bool) @singledispatchmethod - def to(self, type_var: IcebergType) -> Literal: + def to(self, type_var: IcebergType) -> Literal[bool]: # type: ignore raise TypeError(f"Cannot convert BooleanLiteral into {type_var}") @to.register(BooleanType) @@ -207,7 +207,7 @@ def __init__(self, value: int): super().__init__(value, int) @singledispatchmethod - def to(self, type_var: IcebergType) -> Literal: + def to(self, type_var: IcebergType) -> Literal: # type: ignore raise TypeError(f"Cannot convert LongLiteral into {type_var}") @to.register(LongType) @@ -274,7 +274,7 @@ def __ge__(self, other) -> bool: return self._value32 >= other @singledispatchmethod - def to(self, type_var: IcebergType) -> Literal: + def to(self, type_var: IcebergType) -> Literal: # type: ignore raise TypeError(f"Cannot convert FloatLiteral into {type_var}") @to.register(FloatType) @@ -295,7 +295,7 @@ def __init__(self, value: float): super().__init__(value, float) @singledispatchmethod - def to(self, type_var: IcebergType) -> Literal: + def to(self, type_var: IcebergType) -> Literal: # type: ignore raise TypeError(f"Cannot convert DoubleLiteral into {type_var}") @to.register(DoubleType) @@ -320,7 +320,7 @@ def __init__(self, value: int): super().__init__(value, int) @singledispatchmethod - def to(self, type_var: IcebergType) -> Literal: + def to(self, type_var: IcebergType) -> Literal: # type: ignore raise TypeError(f"Cannot convert DateLiteral into {type_var}") @to.register(DateType) @@ -333,7 +333,7 @@ def __init__(self, value: int): super().__init__(value, int) @singledispatchmethod - def to(self, type_var: IcebergType) -> Literal: + def to(self, type_var: IcebergType) -> Literal: # type: ignore raise TypeError(f"Cannot convert TimeLiteral into {type_var}") @to.register(TimeType) @@ -346,7 +346,7 @@ def __init__(self, value: int): super().__init__(value, int) @singledispatchmethod - def to(self, type_var: IcebergType) -> Literal: + def to(self, type_var: IcebergType) -> Literal: # type: ignore raise TypeError(f"Cannot convert TimestampLiteral into {type_var}") @to.register(TimestampType) @@ -363,7 +363,7 @@ def __init__(self, value: Decimal): super().__init__(value, Decimal) @singledispatchmethod - def to(self, type_var: IcebergType) -> Literal: + def to(self, type_var: IcebergType) -> Literal: # type: ignore raise TypeError(f"Cannot convert DecimalLiteral into {type_var}") @to.register(DecimalType) @@ -378,7 +378,7 @@ def __init__(self, value: str): super().__init__(value, str) @singledispatchmethod - def to(self, type_var: IcebergType) -> Literal: + def to(self, type_var: IcebergType) -> Literal: # type: ignore raise TypeError(f"Cannot convert StringLiteral into {type_var}") @to.register(StringType) @@ -450,7 +450,7 @@ def __init__(self, value: UUID): super().__init__(value, UUID) @singledispatchmethod - def to(self, type_var: IcebergType) -> Literal: + def to(self, type_var: IcebergType) -> Literal: # type: ignore raise TypeError(f"Cannot convert UUIDLiteral into {type_var}") @to.register(UUIDType) @@ -463,7 +463,7 @@ def __init__(self, value: bytes): super().__init__(value, bytes) @singledispatchmethod - def to(self, type_var: IcebergType) -> Literal: + def to(self, type_var: IcebergType) -> Literal: # type: ignore raise TypeError(f"Cannot convert FixedLiteral into {type_var}") @to.register(FixedType) @@ -485,7 +485,7 @@ def __init__(self, value: bytes): super().__init__(value, bytes) @singledispatchmethod - def to(self, type_var: IcebergType) -> Literal: + def to(self, type_var: IcebergType) -> Literal: # type: ignore raise TypeError(f"Cannot convert BinaryLiteral into {type_var}") @to.register(BinaryType) diff --git a/pyiceberg/io/fsspec.py b/pyiceberg/io/fsspec.py index 42ebee5d4a..1f5dad686c 100644 --- a/pyiceberg/io/fsspec.py +++ b/pyiceberg/io/fsspec.py @@ -74,7 +74,7 @@ def _s3(properties: Properties) -> AbstractFileSystem: "aws_session_token": properties.get("s3.session-token"), } config_kwargs = {} - register_events: Dict[str, Callable] = {} + register_events: Dict[str, Callable[[Properties], None]] = {} if signer := properties.get("s3.signer"): logger.info("Loading signer %s", signer) @@ -194,7 +194,7 @@ class FsspecFileIO(FileIO): def __init__(self, properties: Properties): self._scheme_to_fs = {} self._scheme_to_fs.update(SCHEME_TO_FS) - self.get_fs: Callable = lru_cache(self._get_fs) + self.get_fs: Callable[[str], AbstractFileSystem] = lru_cache(self._get_fs) super().__init__(properties=properties) def new_input(self, location: str) -> FsspecInputFile: diff --git a/pyiceberg/io/pyarrow.py b/pyiceberg/io/pyarrow.py index 7be778f1ff..d5e23e5c9d 100644 --- a/pyiceberg/io/pyarrow.py +++ b/pyiceberg/io/pyarrow.py @@ -198,7 +198,7 @@ def to_input_file(self) -> "PyArrowFile": class PyArrowFileIO(FileIO): def __init__(self, properties: Properties = EMPTY_DICT): - self.get_fs: Callable = lru_cache(self._get_fs) + self.get_fs: Callable[[str], FileSystem] = lru_cache(self._get_fs) super().__init__(properties=properties) @staticmethod diff --git a/pyiceberg/manifest.py b/pyiceberg/manifest.py index 855e5fc32b..34c6da924a 100644 --- a/pyiceberg/manifest.py +++ b/pyiceberg/manifest.py @@ -187,7 +187,7 @@ def _(list_type: ListType, values: List[Any]) -> Any: @_convert_pos_to_dict.register -def _(map_type: MapType, values: Dict) -> Dict: +def _(map_type: MapType, values: Dict[Any, Any]) -> Dict[Any, Any]: """In the case of a map, we both traverse over the key and value to handle complex types""" return ( { @@ -200,5 +200,5 @@ def _(map_type: MapType, values: Dict) -> Dict: @_convert_pos_to_dict.register -def _(primitive: PrimitiveType, value: Any) -> Any: # pylint: disable=unused-argument +def _(_: PrimitiveType, value: Any) -> Any: return value diff --git a/pyiceberg/schema.py b/pyiceberg/schema.py index 9472335f64..ca1d50828b 100644 --- a/pyiceberg/schema.py +++ b/pyiceberg/schema.py @@ -746,7 +746,7 @@ def assign_fresh_schema_ids(schema: Schema) -> Schema: class _SetFreshIDs(PreOrderSchemaVisitor[IcebergType]): """Traverses the schema and assigns monotonically increasing ids""" - counter: itertools.count + counter: itertools.count # type: ignore reserved_ids: Dict[int, int] def __init__(self, start: int = 1) -> None: diff --git a/pyiceberg/table/metadata.py b/pyiceberg/table/metadata.py index 49cd323e5f..df96036167 100644 --- a/pyiceberg/table/metadata.py +++ b/pyiceberg/table/metadata.py @@ -381,7 +381,7 @@ class TableMetadataUtil: # TableMetadata = Annotated[TableMetadata, Field(alias="format-version", discriminator="format-version")] @staticmethod - def parse_obj(data: dict) -> TableMetadata: + def parse_obj(data: Dict[str, Any]) -> TableMetadata: if "format-version" not in data: raise ValidationError(f"Missing format-version in TableMetadata: {data}") diff --git a/pyiceberg/table/partitioning.py b/pyiceberg/table/partitioning.py index 1445a3f327..7ec9e3089b 100644 --- a/pyiceberg/table/partitioning.py +++ b/pyiceberg/table/partitioning.py @@ -48,14 +48,14 @@ class PartitionField(IcebergBaseModel): source_id: int = Field(alias="source-id") field_id: int = Field(alias="field-id") - transform: Transform = Field() + transform: Transform[Any, Any] = Field() name: str = Field() def __init__( self, source_id: Optional[int] = None, field_id: Optional[int] = None, - transform: Optional[Transform] = None, + transform: Optional[Transform[Any, Any]] = None, name: Optional[str] = None, **data: Any, ): diff --git a/pyiceberg/table/sorting.py b/pyiceberg/table/sorting.py index 4fafcd3b1f..22acfac2ed 100644 --- a/pyiceberg/table/sorting.py +++ b/pyiceberg/table/sorting.py @@ -69,7 +69,7 @@ class SortField(IcebergBaseModel): def __init__( self, source_id: Optional[int] = None, - transform: Optional[Union[Transform, Callable[[IcebergType], Transform]]] = None, + transform: Optional[Union[Transform[Any, Any], Callable[[IcebergType], Transform[Any, Any]]]] = None, direction: Optional[SortDirection] = None, null_order: Optional[NullOrder] = None, **data: Any, @@ -92,7 +92,7 @@ def set_null_order(cls, values: Dict[str, Any]) -> Dict[str, Any]: return values source_id: int = Field(alias="source-id") - transform: Transform = Field() + transform: Transform[Any, Any] = Field() direction: SortDirection = Field() null_order: NullOrder = Field(alias="null-order") diff --git a/pyiceberg/transforms.py b/pyiceberg/transforms.py index 270918e67b..14d76fd8cb 100644 --- a/pyiceberg/transforms.py +++ b/pyiceberg/transforms.py @@ -243,7 +243,7 @@ class TimeTransform(Transform[S, int], Singleton): def granularity(self) -> TimeResolution: ... - def satisfies_order_of(self, other: Transform) -> bool: + def satisfies_order_of(self, other: Transform[S, T]) -> bool: return self.granularity <= other.granularity if hasattr(other, "granularity") else False def result_type(self, source: IcebergType) -> IcebergType: @@ -258,7 +258,7 @@ def preserves_order(self) -> bool: return True -class YearTransform(TimeTransform): +class YearTransform(TimeTransform[S]): """Transforms a datetime value into a year value. Example: @@ -304,7 +304,7 @@ def __repr__(self) -> str: return "YearTransform()" -class MonthTransform(TimeTransform): +class MonthTransform(TimeTransform[S]): """Transforms a datetime value into a month value. Example: @@ -350,7 +350,7 @@ def __repr__(self) -> str: return "MonthTransform()" -class DayTransform(TimeTransform): +class DayTransform(TimeTransform[S]): """Transforms a datetime value into a day value. Example: @@ -399,7 +399,7 @@ def __repr__(self) -> str: return "DayTransform()" -class HourTransform(TimeTransform): +class HourTransform(TimeTransform[S]): """Transforms a datetime value into a hour value. Example: @@ -467,7 +467,7 @@ def result_type(self, source: IcebergType) -> IcebergType: def preserves_order(self) -> bool: return True - def satisfies_order_of(self, other: Transform) -> bool: + def satisfies_order_of(self, other: Transform[S, T]) -> bool: """ordering by value is the same as long as the other preserves order""" return other.preserves_order @@ -537,7 +537,7 @@ def truncate_func(v): return lambda v: truncate_func(v) if v else None - def satisfies_order_of(self, other: Transform) -> bool: + def satisfies_order_of(self, other: Transform[S, T]) -> bool: if self == other: return True elif ( @@ -601,7 +601,7 @@ def _(_type: IcebergType, value: int) -> str: return datetime.to_human_timestamptz(value) -class UnknownTransform(Transform): +class UnknownTransform(Transform[S, T]): """A transform that represents when an unknown transform is provided Args: source_type (IcebergType): An Iceberg `Type` @@ -630,7 +630,7 @@ def __repr__(self) -> str: return f"UnknownTransform(transform={repr(self._transform)})" -class VoidTransform(Transform, Singleton): +class VoidTransform(Transform[S, None], Singleton): """A transform that always returns None""" __root__ = "void" diff --git a/pyiceberg/utils/bin_packing.py b/pyiceberg/utils/bin_packing.py index 7cf09ff6d1..23049c03e2 100644 --- a/pyiceberg/utils/bin_packing.py +++ b/pyiceberg/utils/bin_packing.py @@ -14,6 +14,8 @@ # KIND, either express or implied. See the License for the # specific language governing permissions and limitations # under the License. +from __future__ import annotations + from typing import ( Callable, Generic, @@ -43,9 +45,9 @@ def add(self, item: T, weight: int) -> None: self.items.append(item) -class PackingIterator: +class PackingIterator(Generic[T]): - bins: List[Bin] + bins: List[Bin[T]] def __init__( self, @@ -62,7 +64,7 @@ def __init__( self.largest_bin_first = largest_bin_first self.bins = [] - def __iter__(self) -> "PackingIterator": + def __iter__(self) -> PackingIterator[T]: return self def __next__(self) -> List[T]: @@ -88,13 +90,13 @@ def __next__(self) -> List[T]: return self.remove_bin().items - def find_bin(self, weight: int) -> Optional[Bin]: + def find_bin(self, weight: int) -> Optional[Bin[T]]: for bin_ in self.bins: if bin_.can_add(weight): return bin_ return None - def remove_bin(self) -> Bin: + def remove_bin(self) -> Bin[T]: if self.largest_bin_first: bin_ = max(self.bins, key=lambda b: b.weight()) self.bins.remove(bin_) diff --git a/pyiceberg/utils/deprecated.py b/pyiceberg/utils/deprecated.py index 318849ac95..4b96881b1b 100644 --- a/pyiceberg/utils/deprecated.py +++ b/pyiceberg/utils/deprecated.py @@ -27,7 +27,7 @@ def deprecated(deprecated_in: str, removed_in: str, help_message: Optional[str] if help_message is not None: help_message = f" {help_message}." - def decorator(func: Callable): + def decorator(func: Callable): # type: ignore @functools.wraps(func) def new_func(*args, **kwargs): warnings.simplefilter("always", DeprecationWarning) # turn off filter diff --git a/pyiceberg/utils/parsing.py b/pyiceberg/utils/parsing.py index 0566ed6c28..c2a254a2a9 100644 --- a/pyiceberg/utils/parsing.py +++ b/pyiceberg/utils/parsing.py @@ -21,7 +21,7 @@ class ParseNumberFromBrackets: """Extracts the size from a string in the form of prefix[22]""" - regex: Pattern + regex: Pattern # type: ignore prefix: str def __init__(self, prefix: str): diff --git a/pyiceberg/utils/schema_conversion.py b/pyiceberg/utils/schema_conversion.py index e3a9a437d1..c2bb5c93a5 100644 --- a/pyiceberg/utils/schema_conversion.py +++ b/pyiceberg/utils/schema_conversion.py @@ -118,7 +118,9 @@ def avro_to_iceberg(self, avro_schema: dict[str, Any]) -> Schema: """ return Schema(*[self._convert_field(field) for field in avro_schema["fields"]], schema_id=1) - def _resolve_union(self, type_union: Union[Dict, List, str]) -> Tuple[Union[str, Dict[str, Any]], bool]: + def _resolve_union( + self, type_union: Union[Dict[str, str], List[Union[str, Dict[str, str]]], str] + ) -> Tuple[Union[str, Dict[str, Any]], bool]: """ Converts Unions into their type and resolves if the field is required @@ -141,7 +143,7 @@ def _resolve_union(self, type_union: Union[Dict, List, str]) -> Tuple[Union[str, Raises: TypeError: In the case non-optional union types are encountered """ - avro_types: Union[Dict, List] + avro_types: Union[Dict[str, str], List[Union[Dict[str, str], str]]] if isinstance(type_union, str): # It is a primitive and required return type_union, True diff --git a/pyiceberg/utils/singleton.py b/pyiceberg/utils/singleton.py index 4dd48d5867..ec9eac4a13 100644 --- a/pyiceberg/utils/singleton.py +++ b/pyiceberg/utils/singleton.py @@ -39,7 +39,7 @@ def _convert_to_hashable_type(element: Any) -> Any: class Singleton: - _instances: ClassVar[Dict] = {} + _instances: ClassVar[Dict] = {} # type: ignore def __new__(cls, *args, **kwargs): key = (cls, tuple(args), _convert_to_hashable_type(kwargs)) diff --git a/pyproject.toml b/pyproject.toml index 514ad9e74f..fda44c0904 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -116,6 +116,7 @@ namespace_packages = false warn_redundant_casts = true warn_unreachable = true warn_unused_ignores = true +disallow_any_generics = true [[tool.mypy.overrides]] module = "pyarrow.*" diff --git a/tests/expressions/test_expressions.py b/tests/expressions/test_expressions.py index 8115088ecd..d1d0d74581 100644 --- a/tests/expressions/test_expressions.py +++ b/tests/expressions/test_expressions.py @@ -18,6 +18,7 @@ import uuid from decimal import Decimal +from typing import Any import pytest from typing_extensions import assert_type @@ -619,7 +620,7 @@ def accessor() -> Accessor: @pytest.fixture -def term(field: NestedField, accessor: Accessor) -> BoundReference: +def term(field: NestedField, accessor: Accessor) -> BoundReference[Any]: return BoundReference( field=field, accessor=accessor, @@ -688,14 +689,14 @@ def test_bound_reference_field_property(): assert bound_ref.field == NestedField(field_id=1, name="foo", field_type=StringType(), required=False) -def test_bound_is_null(term: BoundReference) -> None: +def test_bound_is_null(term: BoundReference[Any]) -> None: bound_is_null = BoundIsNull(term) assert str(bound_is_null) == f"BoundIsNull(term={str(term)})" assert repr(bound_is_null) == f"BoundIsNull(term={repr(term)})" assert bound_is_null == eval(repr(bound_is_null)) -def test_bound_is_not_null(term: BoundReference) -> None: +def test_bound_is_not_null(term: BoundReference[Any]) -> None: bound_not_null = BoundNotNull(term) assert str(bound_not_null) == f"BoundNotNull(term={str(term)})" assert repr(bound_not_null) == f"BoundNotNull(term={repr(term)})" @@ -758,14 +759,14 @@ def test_not_nan() -> None: assert not_nan == eval(repr(not_nan)) -def test_bound_in(term: BoundReference) -> None: +def test_bound_in(term: BoundReference[Any]) -> None: bound_in = BoundIn(term, {literal("a"), literal("b"), literal("c")}) assert str(bound_in) == f"BoundIn({str(term)}, {{a, b, c}})" assert repr(bound_in) == f"BoundIn({repr(term)}, {{literal('a'), literal('b'), literal('c')}})" assert bound_in == eval(repr(bound_in)) -def test_bound_not_in(term: BoundReference) -> None: +def test_bound_not_in(term: BoundReference[Any]) -> None: bound_not_in = BoundNotIn(term, {literal("a"), literal("b"), literal("c")}) assert str(bound_not_in) == f"BoundNotIn({str(term)}, {{a, b, c}})" assert repr(bound_not_in) == f"BoundNotIn({repr(term)}, {{literal('a'), literal('b'), literal('c')}})" @@ -788,42 +789,42 @@ def test_not_in() -> None: assert not_in == eval(repr(not_in)) -def test_bound_equal_to(term: BoundReference) -> None: +def test_bound_equal_to(term: BoundReference[Any]) -> None: bound_equal_to = BoundEqualTo(term, literal("a")) assert str(bound_equal_to) == f"BoundEqualTo(term={str(term)}, literal=literal('a'))" assert repr(bound_equal_to) == f"BoundEqualTo(term={repr(term)}, literal=literal('a'))" assert bound_equal_to == eval(repr(bound_equal_to)) -def test_bound_not_equal_to(term: BoundReference) -> None: +def test_bound_not_equal_to(term: BoundReference[Any]) -> None: bound_not_equal_to = BoundNotEqualTo(term, literal("a")) assert str(bound_not_equal_to) == f"BoundNotEqualTo(term={str(term)}, literal=literal('a'))" assert repr(bound_not_equal_to) == f"BoundNotEqualTo(term={repr(term)}, literal=literal('a'))" assert bound_not_equal_to == eval(repr(bound_not_equal_to)) -def test_bound_greater_than_or_equal_to(term: BoundReference) -> None: +def test_bound_greater_than_or_equal_to(term: BoundReference[Any]) -> None: bound_greater_than_or_equal_to = BoundGreaterThanOrEqual(term, literal("a")) assert str(bound_greater_than_or_equal_to) == f"BoundGreaterThanOrEqual(term={str(term)}, literal=literal('a'))" assert repr(bound_greater_than_or_equal_to) == f"BoundGreaterThanOrEqual(term={repr(term)}, literal=literal('a'))" assert bound_greater_than_or_equal_to == eval(repr(bound_greater_than_or_equal_to)) -def test_bound_greater_than(term: BoundReference) -> None: +def test_bound_greater_than(term: BoundReference[Any]) -> None: bound_greater_than = BoundGreaterThan(term, literal("a")) assert str(bound_greater_than) == f"BoundGreaterThan(term={str(term)}, literal=literal('a'))" assert repr(bound_greater_than) == f"BoundGreaterThan(term={repr(term)}, literal=literal('a'))" assert bound_greater_than == eval(repr(bound_greater_than)) -def test_bound_less_than(term: BoundReference) -> None: +def test_bound_less_than(term: BoundReference[Any]) -> None: bound_less_than = BoundLessThan(term, literal("a")) assert str(bound_less_than) == f"BoundLessThan(term={str(term)}, literal=literal('a'))" assert repr(bound_less_than) == f"BoundLessThan(term={repr(term)}, literal=literal('a'))" assert bound_less_than == eval(repr(bound_less_than)) -def test_bound_less_than_or_equal(term: BoundReference) -> None: +def test_bound_less_than_or_equal(term: BoundReference[Any]) -> None: bound_less_than_or_equal = BoundLessThanOrEqual(term, literal("a")) assert str(bound_less_than_or_equal) == f"BoundLessThanOrEqual(term={str(term)}, literal=literal('a'))" assert repr(bound_less_than_or_equal) == f"BoundLessThanOrEqual(term={repr(term)}, literal=literal('a'))" diff --git a/tests/expressions/test_visitors.py b/tests/expressions/test_visitors.py index 62407013fc..0f07156be4 100644 --- a/tests/expressions/test_visitors.py +++ b/tests/expressions/test_visitors.py @@ -38,6 +38,7 @@ BoundNotIn, BoundNotNaN, BoundNotNull, + BoundPredicate, BoundReference, BoundTerm, EqualTo, @@ -55,6 +56,7 @@ NotNull, Or, Reference, + UnboundPredicate, ) from pyiceberg.expressions.literals import Literal, literal from pyiceberg.expressions.visitors import ( @@ -102,7 +104,7 @@ def __str__(self): return "testexprb" -class ExampleVisitor(BooleanExpressionVisitor[List]): +class ExampleVisitor(BooleanExpressionVisitor[List[str]]): """A test implementation of a BooleanExpressionVisitor As this visitor visits each node, it appends an element to a `visit_history` list. This enables testing that a given expression is @@ -110,131 +112,137 @@ class ExampleVisitor(BooleanExpressionVisitor[List]): """ def __init__(self): - self.visit_history: List = [] + self.visit_history: List[str] = [] - def visit_true(self) -> List: + def visit_true(self) -> List[str]: self.visit_history.append("TRUE") return self.visit_history - def visit_false(self) -> List: + def visit_false(self) -> List[str]: self.visit_history.append("FALSE") return self.visit_history - def visit_not(self, child_result: List) -> List: + def visit_not(self, child_result: List[str]) -> List[str]: self.visit_history.append("NOT") return self.visit_history - def visit_and(self, left_result: List, right_result: List) -> List: + def visit_and(self, left_result: List[str], right_result: List[str]) -> List[str]: self.visit_history.append("AND") return self.visit_history - def visit_or(self, left_result: List, right_result: List) -> List: + def visit_or(self, left_result: List[str], right_result: List[str]) -> List[str]: self.visit_history.append("OR") return self.visit_history - def visit_unbound_predicate(self, predicate) -> List: + def visit_unbound_predicate(self, predicate: UnboundPredicate[Any]) -> List[str]: self.visit_history.append("UNBOUND PREDICATE") return self.visit_history - def visit_bound_predicate(self, predicate) -> List: + def visit_bound_predicate(self, predicate: BoundPredicate[Any]) -> List[str]: self.visit_history.append("BOUND PREDICATE") return self.visit_history - def visit_test_expression_a(self) -> List: + def visit_test_expression_a(self) -> List[str]: self.visit_history.append("ExpressionA") return self.visit_history - def visit_test_expression_b(self) -> List: + def visit_test_expression_b(self) -> List[str]: self.visit_history.append("ExpressionB") return self.visit_history @visit.register(ExpressionA) -def _(obj: ExpressionA, visitor: ExampleVisitor) -> List: +def _(obj: ExpressionA, visitor: ExampleVisitor) -> List[str]: """Visit a ExpressionA with a BooleanExpressionVisitor""" return visitor.visit_test_expression_a() @visit.register(ExpressionB) -def _(obj: ExpressionB, visitor: ExampleVisitor) -> List: +def _(obj: ExpressionB, visitor: ExampleVisitor) -> List[str]: """Visit a ExpressionB with a BooleanExpressionVisitor""" return visitor.visit_test_expression_b() -class FooBoundBooleanExpressionVisitor(BoundBooleanExpressionVisitor[List]): +class FooBoundBooleanExpressionVisitor(BoundBooleanExpressionVisitor[List[str]]): """A test implementation of a BoundBooleanExpressionVisitor As this visitor visits each node, it appends an element to a `visit_history` list. This enables testing that a given bound expression is visited in an expected order by the `visit` method. """ def __init__(self): - self.visit_history: List = [] + self.visit_history: List[str] = [] - def visit_in(self, term: BoundTerm, literals: Set) -> List: + def visit_in(self, term: BoundTerm[Any], literals: Set[Any]) -> List[str]: self.visit_history.append("IN") return self.visit_history - def visit_not_in(self, term: BoundTerm, literals: Set) -> List: + def visit_not_in(self, term: BoundTerm[Any], literals: Set[Any]) -> List[str]: self.visit_history.append("NOT_IN") return self.visit_history - def visit_is_nan(self, term: BoundTerm) -> List: + def visit_is_nan(self, term: BoundTerm[Any]) -> List[str]: self.visit_history.append("IS_NAN") return self.visit_history - def visit_not_nan(self, term: BoundTerm) -> List: + def visit_not_nan(self, term: BoundTerm[Any]) -> List[str]: self.visit_history.append("NOT_NAN") return self.visit_history - def visit_is_null(self, term: BoundTerm) -> List: + def visit_is_null(self, term: BoundTerm[Any]) -> List[str]: self.visit_history.append("IS_NULL") return self.visit_history - def visit_not_null(self, term: BoundTerm) -> List: + def visit_not_null(self, term: BoundTerm[Any]) -> List[str]: self.visit_history.append("NOT_NULL") return self.visit_history - def visit_equal(self, term: BoundTerm, literal: Literal) -> List: # pylint: disable=redefined-outer-name + def visit_equal(self, term: BoundTerm[Any], literal: Literal[Any]) -> List[str]: # pylint: disable=redefined-outer-name self.visit_history.append("EQUAL") return self.visit_history - def visit_not_equal(self, term: BoundTerm, literal: Literal) -> List: # pylint: disable=redefined-outer-name + def visit_not_equal(self, term: BoundTerm[Any], literal: Literal[Any]) -> List[str]: # pylint: disable=redefined-outer-name self.visit_history.append("NOT_EQUAL") return self.visit_history - def visit_greater_than_or_equal(self, term: BoundTerm, literal: Literal) -> List: # pylint: disable=redefined-outer-name + def visit_greater_than_or_equal( + self, term: BoundTerm[Any], literal: Literal[Any] + ) -> List[str]: # pylint: disable=redefined-outer-name self.visit_history.append("GREATER_THAN_OR_EQUAL") return self.visit_history - def visit_greater_than(self, term: BoundTerm, literal: Literal) -> List: # pylint: disable=redefined-outer-name + def visit_greater_than( + self, term: BoundTerm[Any], literal: Literal[Any] + ) -> List[str]: # pylint: disable=redefined-outer-name self.visit_history.append("GREATER_THAN") return self.visit_history - def visit_less_than(self, term: BoundTerm, literal: Literal) -> List: # pylint: disable=redefined-outer-name + def visit_less_than(self, term: BoundTerm[Any], literal: Literal[Any]) -> List[str]: # pylint: disable=redefined-outer-name self.visit_history.append("LESS_THAN") return self.visit_history - def visit_less_than_or_equal(self, term: BoundTerm, literal: Literal) -> List: # pylint: disable=redefined-outer-name + def visit_less_than_or_equal( + self, term: BoundTerm[Any], literal: Literal[Any] + ) -> List[str]: # pylint: disable=redefined-outer-name self.visit_history.append("LESS_THAN_OR_EQUAL") return self.visit_history - def visit_true(self) -> List: + def visit_true(self) -> List[str]: self.visit_history.append("TRUE") return self.visit_history - def visit_false(self) -> List: + def visit_false(self) -> List[str]: self.visit_history.append("FALSE") return self.visit_history - def visit_not(self, child_result: List) -> List: + def visit_not(self, child_result: List[str]) -> List[str]: self.visit_history.append("NOT") return self.visit_history - def visit_and(self, left_result: List, right_result: List) -> List: + def visit_and(self, left_result: List[str], right_result: List[str]) -> List[str]: self.visit_history.append("AND") return self.visit_history - def visit_or(self, left_result: List, right_result: List) -> List: + def visit_or(self, left_result: List[str], right_result: List[str]) -> List[str]: self.visit_history.append("OR") return self.visit_history diff --git a/tests/test_transforms.py b/tests/test_transforms.py index 3e4e63494a..f8bc42e2dc 100644 --- a/tests/test_transforms.py +++ b/tests/test_transforms.py @@ -17,6 +17,7 @@ # pylint: disable=eval-used,protected-access from datetime import date from decimal import Decimal +from typing import Any from uuid import UUID import mmh3 as mmh3 @@ -392,7 +393,7 @@ def test_void_transform(): class TestType(IcebergBaseModel): - __root__: Transform + __root__: Transform[Any, Any] def test_bucket_transform_serialize(): From 9c3b6a809fd3b18d1360743b08d7f0cbd177df57 Mon Sep 17 00:00:00 2001 From: Ryan Blue Date: Mon, 21 Nov 2022 14:26:02 -0800 Subject: [PATCH 275/642] Python: Implement DataScan.plan_files (#6233) --- mkdocs/docs/index.md | 2 +- poetry.lock | 215 +++++++++++++++----------- pyiceberg/catalog/__init__.py | 2 +- pyiceberg/catalog/glue.py | 4 +- pyiceberg/catalog/hive.py | 2 +- pyiceberg/catalog/rest.py | 2 +- pyiceberg/cli/output.py | 4 +- pyiceberg/expressions/visitors.py | 2 +- pyiceberg/manifest.py | 8 + pyiceberg/{table => }/partitioning.py | 0 pyiceberg/table/__init__.py | 180 +++++++++++++++++++-- pyiceberg/table/metadata.py | 2 +- pyiceberg/table/snapshots.py | 2 +- pyiceberg/typedef.py | 21 +++ pyproject.toml | 7 + tests/catalog/test_base.py | 2 +- tests/catalog/test_hive.py | 2 +- tests/catalog/test_rest.py | 2 +- tests/cli/test_console.py | 2 +- tests/table/test_init.py | 4 +- tests/table/test_metadata.py | 2 +- tests/table/test_partitioning.py | 2 +- tests/table/test_snapshots.py | 2 +- tests/utils/test_manifest.py | 2 +- 24 files changed, 351 insertions(+), 122 deletions(-) rename pyiceberg/{table => }/partitioning.py (100%) diff --git a/mkdocs/docs/index.md b/mkdocs/docs/index.md index c05af4effa..adbcc0e0c8 100644 --- a/mkdocs/docs/index.md +++ b/mkdocs/docs/index.md @@ -449,7 +449,7 @@ schema = Schema( NestedField(field_id=4, name="symbol", field_type=StringType(), required=False), ) -from pyiceberg.table.partitioning import PartitionSpec, PartitionField +from pyiceberg.partitioning import PartitionSpec, PartitionField from pyiceberg.transforms import DayTransform partition_spec = PartitionSpec( diff --git a/poetry.lock b/poetry.lock index caccecda60..0338992a40 100644 --- a/poetry.lock +++ b/poetry.lock @@ -34,7 +34,7 @@ multidict = ">=4.5,<7.0" yarl = ">=1.0,<2.0" [package.extras] -speedups = ["Brotli", "aiodns", "cchardet"] +speedups = ["aiodns", "brotli", "cchardet"] [[package]] name = "aioitertools" @@ -75,10 +75,10 @@ optional = false python-versions = ">=3.5" [package.extras] -dev = ["cloudpickle", "coverage[toml] (>=5.0.2)", "furo", "hypothesis", "mypy (>=0.900,!=0.940)", "pre-commit", "pympler", "pytest (>=4.3.0)", "pytest-mypy-plugins", "sphinx", "sphinx-notfound-page", "zope.interface"] -docs = ["furo", "sphinx", "sphinx-notfound-page", "zope.interface"] -tests = ["cloudpickle", "coverage[toml] (>=5.0.2)", "hypothesis", "mypy (>=0.900,!=0.940)", "pympler", "pytest (>=4.3.0)", "pytest-mypy-plugins", "zope.interface"] -tests-no-zope = ["cloudpickle", "coverage[toml] (>=5.0.2)", "hypothesis", "mypy (>=0.900,!=0.940)", "pympler", "pytest (>=4.3.0)", "pytest-mypy-plugins"] +dev = ["coverage[toml] (>=5.0.2)", "hypothesis", "pympler", "pytest (>=4.3.0)", "mypy (>=0.900,!=0.940)", "pytest-mypy-plugins", "zope.interface", "furo", "sphinx", "sphinx-notfound-page", "pre-commit", "cloudpickle"] +docs = ["furo", "sphinx", "zope.interface", "sphinx-notfound-page"] +tests = ["coverage[toml] (>=5.0.2)", "hypothesis", "pympler", "pytest (>=4.3.0)", "mypy (>=0.900,!=0.940)", "pytest-mypy-plugins", "zope.interface", "cloudpickle"] +tests_no_zope = ["coverage[toml] (>=5.0.2)", "hypothesis", "pympler", "pytest (>=4.3.0)", "mypy (>=0.900,!=0.940)", "pytest-mypy-plugins", "cloudpickle"] [[package]] name = "boto3" @@ -128,7 +128,7 @@ tomli = {version = ">=1.0.0", markers = "python_version < \"3.11\""} [package.extras] docs = ["furo (>=2021.08.31)", "sphinx (>=4.0,<5.0)", "sphinx-argparse-cli (>=1.5)", "sphinx-autodoc-typehints (>=1.10)"] -test = ["filelock (>=3)", "pytest (>=6.2.4)", "pytest-cov (>=2.12)", "pytest-mock (>=2)", "pytest-rerunfailures (>=9.1)", "pytest-xdist (>=1.34)", "setuptools (>=42.0.0)", "setuptools (>=56.0.0)", "toml (>=0.10.0)", "wheel (>=0.36.0)"] +test = ["filelock (>=3)", "pytest (>=6.2.4)", "pytest-cov (>=2.12)", "pytest-mock (>=2)", "pytest-rerunfailures (>=9.1)", "pytest-xdist (>=1.34)", "toml (>=0.10.0)", "wheel (>=0.36.0)", "setuptools (>=42.0.0)", "setuptools (>=56.0.0)"] typing = ["importlib-metadata (>=4.6.4)", "mypy (==0.950)", "typing-extensions (>=3.7.4.3)"] virtualenv = ["virtualenv (>=20.0.35)"] @@ -168,7 +168,7 @@ optional = false python-versions = ">=3.6.0" [package.extras] -unicode-backport = ["unicodedata2"] +unicode_backport = ["unicodedata2"] [[package]] name = "click" @@ -227,11 +227,11 @@ cffi = ">=1.12" [package.extras] docs = ["sphinx (>=1.6.5,!=1.8.0,!=3.1.0,!=3.1.1)", "sphinx-rtd-theme"] -docstest = ["pyenchant (>=1.6.11)", "sphinxcontrib-spelling (>=4.0.1)", "twine (>=1.12.0)"] +docstest = ["pyenchant (>=1.6.11)", "twine (>=1.12.0)", "sphinxcontrib-spelling (>=4.0.1)"] pep8test = ["black", "flake8", "flake8-import-order", "pep8-naming"] sdist = ["setuptools-rust (>=0.11.4)"] ssh = ["bcrypt (>=3.1.5)"] -test = ["hypothesis (>=1.11.4,!=3.79.2)", "iso8601", "pretend", "pytest (>=6.2.0)", "pytest-benchmark", "pytest-cov", "pytest-subtests", "pytest-xdist", "pytz"] +test = ["pytest (>=6.2.0)", "pytest-benchmark", "pytest-cov", "pytest-subtests", "pytest-xdist", "pretend", "iso8601", "pytz", "hypothesis (>=1.11.4,!=3.79.2)"] [[package]] name = "distlib" @@ -249,6 +249,17 @@ category = "dev" optional = false python-versions = ">=3.7" +[[package]] +name = "duckdb" +version = "0.6.0" +description = "DuckDB embedded database" +category = "main" +optional = true +python-versions = "*" + +[package.dependencies] +numpy = ">=1.14" + [[package]] name = "exceptiongroup" version = "1.0.4" @@ -269,7 +280,7 @@ optional = false python-versions = ">=3.7" [package.extras] -codecs = ["lz4", "python-snappy", "zstandard"] +codecs = ["python-snappy", "zstandard", "lz4"] lz4 = ["lz4"] snappy = ["python-snappy"] zstandard = ["zstandard"] @@ -307,7 +318,7 @@ abfs = ["adlfs"] adl = ["adlfs"] arrow = ["pyarrow (>=1)"] dask = ["dask", "distributed"] -dropbox = ["dropbox", "dropboxdrivefs", "requests"] +dropbox = ["dropboxdrivefs", "requests", "dropbox"] entrypoints = ["importlib-metadata"] fuse = ["fusepy"] gcs = ["gcsfs"] @@ -316,7 +327,7 @@ github = ["requests"] gs = ["gcsfs"] gui = ["panel"] hdfs = ["pyarrow (>=1)"] -http = ["aiohttp (!=4.0.0a0,!=4.0.0a1)", "requests"] +http = ["requests", "aiohttp (!=4.0.0a0,!=4.0.0a1)"] libarchive = ["libarchive-c"] oci = ["ocifs"] s3 = ["s3fs"] @@ -327,7 +338,7 @@ tqdm = ["tqdm"] [[package]] name = "identify" -version = "2.5.8" +version = "2.5.9" description = "File identification library for Python" category = "dev" optional = false @@ -356,9 +367,9 @@ python-versions = ">=3.7" zipp = ">=0.5" [package.extras] -docs = ["furo", "jaraco.packaging (>=9)", "jaraco.tidelift (>=1.4)", "rst.linker (>=1.9)", "sphinx (>=3.5)"] +docs = ["sphinx (>=3.5)", "jaraco.packaging (>=9)", "rst.linker (>=1.9)", "furo", "jaraco.tidelift (>=1.4)"] perf = ["ipython"] -testing = ["flake8 (<5)", "flufl.flake8", "importlib-resources (>=1.3)", "packaging", "pyfakefs", "pytest (>=6)", "pytest-black (>=0.3.7)", "pytest-checkdocs (>=2.4)", "pytest-cov", "pytest-enabler (>=1.3)", "pytest-flake8", "pytest-mypy (>=0.9.1)", "pytest-perf (>=0.9.2)"] +testing = ["pytest (>=6)", "pytest-checkdocs (>=2.4)", "pytest-flake8", "flake8 (<5)", "pytest-cov", "pytest-enabler (>=1.3)", "packaging", "pyfakefs", "flufl.flake8", "pytest-perf (>=0.9.2)", "pytest-black (>=0.3.7)", "pytest-mypy (>=0.9.1)", "importlib-resources (>=1.3)"] [[package]] name = "iniconfig" @@ -408,7 +419,7 @@ python-versions = "*" [[package]] name = "moto" -version = "4.0.9" +version = "4.0.10" description = "A library that allows your python tests to easily mock out the boto library" category = "dev" optional = false @@ -428,14 +439,14 @@ werkzeug = ">=0.5,<2.2.0 || >2.2.0,<2.2.1 || >2.2.1" xmltodict = "*" [package.extras] -all = ["PyYAML (>=5.1)", "aws-xray-sdk (>=0.93,!=0.96)", "cfn-lint (>=0.4.0)", "docker (>=2.5.1)", "ecdsa (!=0.15)", "graphql-core", "idna (>=2.5,<4)", "jsondiff (>=1.1.2)", "openapi-spec-validator (>=0.2.8)", "pyparsing (>=3.0.7)", "python-jose[cryptography] (>=3.1.0,<4.0.0)", "setuptools", "sshpubkeys (>=3.1.0)"] -apigateway = ["PyYAML (>=5.1)", "ecdsa (!=0.15)", "openapi-spec-validator (>=0.2.8)", "python-jose[cryptography] (>=3.1.0,<4.0.0)"] +all = ["PyYAML (>=5.1)", "python-jose[cryptography] (>=3.1.0,<4.0.0)", "ecdsa (!=0.15)", "docker (>=2.5.1)", "graphql-core", "jsondiff (>=1.1.2)", "aws-xray-sdk (>=0.93,!=0.96)", "idna (>=2.5,<4)", "cfn-lint (>=0.40.0)", "sshpubkeys (>=3.1.0)", "pyparsing (>=3.0.7)", "openapi-spec-validator (>=0.2.8)", "setuptools"] +apigateway = ["PyYAML (>=5.1)", "python-jose[cryptography] (>=3.1.0,<4.0.0)", "ecdsa (!=0.15)", "openapi-spec-validator (>=0.2.8)"] apigatewayv2 = ["PyYAML (>=5.1)"] appsync = ["graphql-core"] awslambda = ["docker (>=2.5.1)"] batch = ["docker (>=2.5.1)"] -cloudformation = ["PyYAML (>=5.1)", "aws-xray-sdk (>=0.93,!=0.96)", "cfn-lint (>=0.4.0)", "docker (>=2.5.1)", "ecdsa (!=0.15)", "graphql-core", "idna (>=2.5,<4)", "jsondiff (>=1.1.2)", "openapi-spec-validator (>=0.2.8)", "pyparsing (>=3.0.7)", "python-jose[cryptography] (>=3.1.0,<4.0.0)", "setuptools", "sshpubkeys (>=3.1.0)"] -cognitoidp = ["ecdsa (!=0.15)", "python-jose[cryptography] (>=3.1.0,<4.0.0)"] +cloudformation = ["PyYAML (>=5.1)", "python-jose[cryptography] (>=3.1.0,<4.0.0)", "ecdsa (!=0.15)", "docker (>=2.5.1)", "graphql-core", "jsondiff (>=1.1.2)", "aws-xray-sdk (>=0.93,!=0.96)", "idna (>=2.5,<4)", "cfn-lint (>=0.40.0)", "sshpubkeys (>=3.1.0)", "pyparsing (>=3.0.7)", "openapi-spec-validator (>=0.2.8)", "setuptools"] +cognitoidp = ["python-jose[cryptography] (>=3.1.0,<4.0.0)", "ecdsa (!=0.15)"] ds = ["sshpubkeys (>=3.1.0)"] dynamodb = ["docker (>=2.5.1)"] dynamodb2 = ["docker (>=2.5.1)"] @@ -447,7 +458,7 @@ glue = ["pyparsing (>=3.0.7)"] iotdata = ["jsondiff (>=1.1.2)"] route53resolver = ["sshpubkeys (>=3.1.0)"] s3 = ["PyYAML (>=5.1)"] -server = ["PyYAML (>=5.1)", "aws-xray-sdk (>=0.93,!=0.96)", "cfn-lint (>=0.4.0)", "docker (>=2.5.1)", "ecdsa (!=0.15)", "flask (!=2.2.0,!=2.2.1)", "flask-cors", "graphql-core", "idna (>=2.5,<4)", "jsondiff (>=1.1.2)", "openapi-spec-validator (>=0.2.8)", "pyparsing (>=3.0.7)", "python-jose[cryptography] (>=3.1.0,<4.0.0)", "setuptools", "sshpubkeys (>=3.1.0)"] +server = ["PyYAML (>=5.1)", "python-jose[cryptography] (>=3.1.0,<4.0.0)", "ecdsa (!=0.15)", "docker (>=2.5.1)", "graphql-core", "jsondiff (>=1.1.2)", "aws-xray-sdk (>=0.93,!=0.96)", "idna (>=2.5,<4)", "cfn-lint (>=0.40.0)", "sshpubkeys (>=3.1.0)", "pyparsing (>=3.0.7)", "openapi-spec-validator (>=0.2.8)", "setuptools", "flask (!=2.2.0,!=2.2.1)", "flask-cors"] ssm = ["PyYAML (>=5.1)", "dataclasses"] xray = ["aws-xray-sdk (>=0.93,!=0.96)", "setuptools"] @@ -467,12 +478,9 @@ category = "dev" optional = false python-versions = ">=2.7,!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,!=3.4.*,!=3.5.*,!=3.6.*" -[package.dependencies] -setuptools = "*" - [[package]] name = "numpy" -version = "1.23.4" +version = "1.23.5" description = "NumPy is the fundamental package for array computing with Python." category = "main" optional = true @@ -509,8 +517,8 @@ optional = false python-versions = ">=3.7" [package.extras] -docs = ["furo (>=2022.9.29)", "proselint (>=0.13)", "sphinx (>=5.3)", "sphinx-autodoc-typehints (>=1.19.4)"] -test = ["appdirs (==1.4.4)", "pytest (>=7.2)", "pytest-cov (>=4)", "pytest-mock (>=3.10)"] +docs = ["furo (>=2022.9.29)", "proselint (>=0.13)", "sphinx-autodoc-typehints (>=1.19.4)", "sphinx (>=5.3)"] +test = ["appdirs (==1.4.4)", "pytest-cov (>=4)", "pytest-mock (>=3.10)", "pytest (>=7.2)"] [[package]] name = "pluggy" @@ -594,7 +602,7 @@ optional = false python-versions = ">=3.6.8" [package.extras] -diagrams = ["jinja2", "railroad-diagrams"] +diagrams = ["railroad-diagrams", "jinja2"] [[package]] name = "pytest" @@ -630,8 +638,8 @@ docutils = ">=0.15" importlib-metadata = {version = ">=4", markers = "python_version < \"3.10\""} [package.extras] -docs = ["furo", "jaraco.packaging (>=9)", "rst.linker (>=1.9)", "sphinx (>=3.5)"] -testing = ["flake8 (<5)", "pytest (>=6)", "pytest-black (>=0.3.7)", "pytest-checkdocs (>=2.4)", "pytest-cov", "pytest-enabler (>=1.3)", "pytest-flake8", "pytest-mypy (>=0.9.1)", "types-docutils"] +docs = ["sphinx (>=3.5)", "jaraco.packaging (>=9)", "rst.linker (>=1.9)", "furo"] +testing = ["pytest (>=6)", "pytest-checkdocs (>=2.4)", "pytest-flake8", "flake8 (<5)", "pytest-cov", "pytest-enabler (>=1.3)", "types-docutils", "pytest-black (>=0.3.7)", "pytest-mypy (>=0.9.1)"] [[package]] name = "python-dateutil" @@ -684,7 +692,7 @@ urllib3 = ">=1.21.1,<1.27" [package.extras] socks = ["PySocks (>=1.5.6,!=1.5.7)"] -use-chardet-on-py3 = ["chardet (>=3.0.2,<6)"] +use_chardet_on_py3 = ["chardet (>=3.0.2,<6)"] [[package]] name = "requests-mock" @@ -700,7 +708,7 @@ six = "*" [package.extras] fixture = ["fixtures"] -test = ["fixtures", "mock", "purl", "pytest", "requests-futures", "sphinx", "testrepository (>=0.0.18)", "testtools"] +test = ["fixtures", "mock", "purl", "pytest", "sphinx", "testrepository (>=0.0.18)", "testtools", "requests-futures"] [[package]] name = "responses" @@ -717,7 +725,7 @@ types-toml = "*" urllib3 = ">=1.25.10" [package.extras] -tests = ["coverage (>=6.0.0)", "flake8", "mypy", "pytest (>=7.0.0)", "pytest-asyncio", "pytest-cov", "pytest-httpserver", "types-requests"] +tests = ["pytest (>=7.0.0)", "coverage (>=6.0.0)", "pytest-cov", "pytest-asyncio", "pytest-httpserver", "flake8", "types-requests", "mypy"] [[package]] name = "rich" @@ -766,19 +774,6 @@ botocore = ">=1.12.36,<2.0a.0" [package.extras] crt = ["botocore[crt] (>=1.20.29,<2.0a.0)"] -[[package]] -name = "setuptools" -version = "65.5.1" -description = "Easily download, build, install, upgrade, and uninstall Python packages" -category = "dev" -optional = false -python-versions = ">=3.7" - -[package.extras] -docs = ["furo", "jaraco.packaging (>=9)", "jaraco.tidelift (>=1.4)", "pygments-github-lexers (==0.0.5)", "rst.linker (>=1.9)", "sphinx (>=3.5)", "sphinx-favicon", "sphinx-hoverxref (<2)", "sphinx-inline-tabs", "sphinx-notfound-page (==0.8.3)", "sphinx-reredirects", "sphinxcontrib-towncrier"] -testing = ["build[virtualenv]", "filelock (>=3.4.0)", "flake8 (<5)", "flake8-2020", "ini2toml[lite] (>=0.9)", "jaraco.envs (>=2.2)", "jaraco.path (>=3.2.0)", "pip (>=19.1)", "pip-run (>=8.8)", "pytest (>=6)", "pytest-black (>=0.3.7)", "pytest-checkdocs (>=2.4)", "pytest-cov", "pytest-enabler (>=1.3)", "pytest-flake8", "pytest-mypy (>=0.9.1)", "pytest-perf", "pytest-timeout", "pytest-xdist", "tomli-w (>=1.0.0)", "virtualenv (>=13.0.0)", "wheel"] -testing-integration = ["build[virtualenv]", "filelock (>=3.4.0)", "jaraco.envs (>=2.2)", "jaraco.path (>=3.2.0)", "pytest", "pytest-enabler", "pytest-xdist", "tomli", "virtualenv (>=13.0.0)", "wheel"] - [[package]] name = "six" version = "1.16.0" @@ -844,8 +839,8 @@ optional = false python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*, !=3.5.*, <4" [package.extras] -brotli = ["brotli (>=1.0.9)", "brotlicffi (>=0.8.0)", "brotlipy (>=0.6.0)"] -secure = ["certifi", "cryptography (>=1.3.4)", "idna (>=2.0.0)", "ipaddress", "pyOpenSSL (>=0.14)", "urllib3-secure-extra"] +brotli = ["brotlicffi (>=0.8.0)", "brotli (>=1.0.9)", "brotlipy (>=0.6.0)"] +secure = ["pyOpenSSL (>=0.14)", "cryptography (>=1.3.4)", "idna (>=2.0.0)", "certifi", "urllib3-secure-extra", "ipaddress"] socks = ["PySocks (>=1.5.6,!=1.5.7,<2.0)"] [[package]] @@ -916,8 +911,8 @@ optional = false python-versions = ">=3.7" [package.extras] -docs = ["furo", "jaraco.packaging (>=9)", "jaraco.tidelift (>=1.4)", "rst.linker (>=1.9)", "sphinx (>=3.5)"] -testing = ["flake8 (<5)", "func-timeout", "jaraco.functools", "jaraco.itertools", "more-itertools", "pytest (>=6)", "pytest-black (>=0.3.7)", "pytest-checkdocs (>=2.4)", "pytest-cov", "pytest-enabler (>=1.3)", "pytest-flake8", "pytest-mypy (>=0.9.1)"] +docs = ["sphinx (>=3.5)", "jaraco.packaging (>=9)", "rst.linker (>=1.9)", "furo", "jaraco.tidelift (>=1.4)"] +testing = ["pytest (>=6)", "pytest-checkdocs (>=2.4)", "pytest-flake8", "flake8 (<5)", "pytest-cov", "pytest-enabler (>=1.3)", "jaraco.itertools", "func-timeout", "jaraco.functools", "more-itertools", "pytest-black (>=0.3.7)", "pytest-mypy (>=0.9.1)"] [[package]] name = "zstandard" @@ -934,6 +929,7 @@ cffi = {version = ">=1.11", markers = "platform_python_implementation == \"PyPy\ cffi = ["cffi (>=1.11)"] [extras] +duckdb = ["duckdb", "pyarrow"] glue = ["boto3"] hive = ["thrift"] pyarrow = ["pyarrow"] @@ -943,7 +939,7 @@ snappy = ["python-snappy"] [metadata] lock-version = "1.1" python-versions = "^3.8" -content-hash = "588c954b75cf70e3c695dfe27b859be09c183323becda37ebfdde00caa5730fd" +content-hash = "704760d1f2db5291b48b9fd04e48bdd0f3253b06d1d40a34403cc95d3f12a705" [metadata.files] aiobotocore = [ @@ -1245,6 +1241,55 @@ docutils = [ {file = "docutils-0.19-py3-none-any.whl", hash = "sha256:5e1de4d849fee02c63b040a4a3fd567f4ab104defd8a5511fbbc24a8a017efbc"}, {file = "docutils-0.19.tar.gz", hash = "sha256:33995a6753c30b7f577febfc2c50411fec6aac7f7ffeb7c4cfe5991072dcf9e6"}, ] +duckdb = [ + {file = "duckdb-0.6.0-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:0375f174f6ea9e65a5a1db20663d1cee0663ef4021b1591d515fe69822244871"}, + {file = "duckdb-0.6.0-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:6419820349c2c939d740dc1e045df3bc031afb1b86d36e876cec09e6ca84d71b"}, + {file = "duckdb-0.6.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:cfe007492d02ee2b76e530a4b52168d0a92819b5b38be50061665d7ebee7a3d2"}, + {file = "duckdb-0.6.0-cp310-cp310-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:1ae9ff3c8e1d510621888db313dcd808a3e52caedc85c8944100e512b29f6eb6"}, + {file = "duckdb-0.6.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:bcec180d90a61caa790cda3de69bd2ea7a62b898c243d045ea68bfe657a5e99a"}, + {file = "duckdb-0.6.0-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:fe3f54079db38cb7bd2101b6f96519c2bd24f66474ba1b20a987093d6bfa4b82"}, + {file = "duckdb-0.6.0-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:00c6f009ba84745e1afdfe03c7641c4601e6e8b4c3e3ee1f770eada4ae9e29d8"}, + {file = "duckdb-0.6.0-cp310-cp310-win32.whl", hash = "sha256:019096b210c921d01ae0c4ec17deb7a487f20c94ee2a811744bc9d7d23bcee98"}, + {file = "duckdb-0.6.0-cp310-cp310-win_amd64.whl", hash = "sha256:7d7ab4e963141246771d5f15c151dae84a1fd90a986312a77cdc999faa89eae4"}, + {file = "duckdb-0.6.0-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:9ba293125ce3acc6dcec148e22b37c49880e2319e415f322b65ffbcabf762afb"}, + {file = "duckdb-0.6.0-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:f23d5bef667e4631aa9aa57909e2b1eeeb583680ce350c008364894761d3ff55"}, + {file = "duckdb-0.6.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:14972984422d37113fb15a93437db9283b647029db8a7c6c0935977997fe1d7f"}, + {file = "duckdb-0.6.0-cp311-cp311-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:d096d1b623f0a166db621748988bcc0cab9ac8c99c6d5ecc8a72dca71a1a4a49"}, + {file = "duckdb-0.6.0-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:39efe9498309d9e1ce9b693ade5be4ec1d3528c0adc115936717710a396791b0"}, + {file = "duckdb-0.6.0-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:01b03d20f0218fcfbae25b14f06db763ce7951d912e1142a2684fc4613ca546e"}, + {file = "duckdb-0.6.0-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:a7f42d1259bff2d8d010ba064b41bce1ce3d8d79a322e2fe69d21f172f38fe9a"}, + {file = "duckdb-0.6.0-cp311-cp311-win32.whl", hash = "sha256:0df5fc44a3dc31ebc3a42b7f6da510d45e0d8494955a5e22baa497ee1dc5c3f6"}, + {file = "duckdb-0.6.0-cp311-cp311-win_amd64.whl", hash = "sha256:3f36de549f7117f0c30a17b58c2e12c2cf5054a2fe0aef7c04674f1602959c4a"}, + {file = "duckdb-0.6.0-cp36-cp36m-macosx_10_9_x86_64.whl", hash = "sha256:7b312cab0da395aea7bd7b84cb0a6a21dd5e03e5993b9ae2e6c5e9cfa2607b21"}, + {file = "duckdb-0.6.0-cp36-cp36m-win32.whl", hash = "sha256:238e4cc0bc715e688346ae7cd0eaacd9840eabf9ac1f8135e6ac208ce9f07235"}, + {file = "duckdb-0.6.0-cp36-cp36m-win_amd64.whl", hash = "sha256:c746a25d3bcb378c0bed65fd93f8086056529f216b063ac20dd7fe878f6c7438"}, + {file = "duckdb-0.6.0-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:36d45de5d64722768c579fc4fe4ac3c937c65f0ab34a05d1cf2eda449ce79a81"}, + {file = "duckdb-0.6.0-cp37-cp37m-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:902a3a0e712d9114f6b4067c7b53d1d64bddd825d96d9fd61578dc58c13f5524"}, + {file = "duckdb-0.6.0-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:03c13bf558febda114cdd2de9db6d202cbd5bfdbac66dbdc94faa432313b39dd"}, + {file = "duckdb-0.6.0-cp37-cp37m-musllinux_1_1_i686.whl", hash = "sha256:3875b9cc8f008c3a69138a872f2fb9d4e655f889414616755aba700f03a9b399"}, + {file = "duckdb-0.6.0-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:0447fc65e930baef6bbfe11807dd6badd6a24ebb5d739908e50fc0d9f68504f1"}, + {file = "duckdb-0.6.0-cp37-cp37m-win32.whl", hash = "sha256:9796c1359ef31808a5f2e83ab981ba4760da02e0bdbc66d4f46db7e9e2c0fe54"}, + {file = "duckdb-0.6.0-cp37-cp37m-win_amd64.whl", hash = "sha256:853474b71fccb804fc7c338eeca7c795f385dfe3b666cd6184fd5a9c6640958e"}, + {file = "duckdb-0.6.0-cp38-cp38-macosx_10_9_universal2.whl", hash = "sha256:358e2d444619f558563cf1f625680925d67344e62fec81a7a1bf2ed9f959a0b0"}, + {file = "duckdb-0.6.0-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:0c8590f99d63d5407e40e45a90b250ec92f7edaddc67c475af2d82f1a11b18c9"}, + {file = "duckdb-0.6.0-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:ed875f32c3139e84ceb42eeda3b12e56bd803de0015494a25a8176766169ff69"}, + {file = "duckdb-0.6.0-cp38-cp38-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:5f4f0b50f7f4f4c22fc7debd28d1b81705152e52d44425bf777395cdf541b9bb"}, + {file = "duckdb-0.6.0-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:860a62fcf67f8ae86cd946f0ca74d6b22f00ebd845c588fbdd761eca5923000e"}, + {file = "duckdb-0.6.0-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:54de408dcc95f0cd5fffba6d54044b3e97840db93b8e7c961853941d5ec59a30"}, + {file = "duckdb-0.6.0-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:afa8f0f391608c4139752d6f377c4119e2598df3e84087a321bf285348e319e2"}, + {file = "duckdb-0.6.0-cp38-cp38-win32.whl", hash = "sha256:2ddd6f73c42b78fd862ead4df6a730c3087589e842320ec10ad6ce0a4e170b0e"}, + {file = "duckdb-0.6.0-cp38-cp38-win_amd64.whl", hash = "sha256:7ae9d30de3cb881e07b7e95b4ff16b8c121a7714f4ad376c7ef583601a7c1bd9"}, + {file = "duckdb-0.6.0-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:e12b4087cdc48c5c2ed2cc0dbf648df357ace88e3f47dd4152958bd5c5646794"}, + {file = "duckdb-0.6.0-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:89b6c72cd82089c5f3b313bb78de1d8f96cfe87e80bff9b93ee837e29ddf55fe"}, + {file = "duckdb-0.6.0-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:3d859640d49ef856e4d68a086d7c3a17f38b380e9b10387a0419630c17c32b52"}, + {file = "duckdb-0.6.0-cp39-cp39-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:30a2382678a2fc9c0284fb976e3392f50af780dfa404fc18a5d34e443478864f"}, + {file = "duckdb-0.6.0-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:0a9c478f964a45e94338a922af36cd7413aae504d365bb94850270d53bc27182"}, + {file = "duckdb-0.6.0-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:5dc228ef3f0067f312c12a3d14e8ae1c8b4f2cbba637af917979bf73821d6ba0"}, + {file = "duckdb-0.6.0-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:c4851fd0869692f7c3be846bd3325a3f2f45f13821a6fc58dc4e2bd4fecf0b71"}, + {file = "duckdb-0.6.0-cp39-cp39-win32.whl", hash = "sha256:253c1c68635462811f1bef3d10fac36b5907461ee387ba441b7d5dc03844b31e"}, + {file = "duckdb-0.6.0-cp39-cp39-win_amd64.whl", hash = "sha256:5f11a9cd5f860db3ec66dae6f3c5b21df21b82fcbe1a6622acca14c16c0a0cc2"}, + {file = "duckdb-0.6.0.tar.gz", hash = "sha256:74e0e4cd1b77aaec9f76e3a0b4cf8535d80f2282f38c6248d4ec826a9606fe81"}, +] exceptiongroup = [ {file = "exceptiongroup-1.0.4-py3-none-any.whl", hash = "sha256:542adf9dea4055530d6e1279602fa5cb11dab2395fa650b8674eaec35fc4a828"}, {file = "exceptiongroup-1.0.4.tar.gz", hash = "sha256:bd14967b79cd9bdb54d97323216f8fdf533e278df937aa2a90089e7d6e06e5ec"}, @@ -1353,8 +1398,8 @@ fsspec = [ {file = "fsspec-2022.10.0.tar.gz", hash = "sha256:cb6092474e90487a51de768170f3afa50ca8982c26150a59072b16433879ff1d"}, ] identify = [ - {file = "identify-2.5.8-py2.py3-none-any.whl", hash = "sha256:48b7925fe122720088aeb7a6c34f17b27e706b72c61070f27fe3789094233440"}, - {file = "identify-2.5.8.tar.gz", hash = "sha256:7a214a10313b9489a0d61467db2856ae8d0b8306fc923e03a9effa53d8aedc58"}, + {file = "identify-2.5.9-py2.py3-none-any.whl", hash = "sha256:a390fb696e164dbddb047a0db26e57972ae52fbd037ae68797e5ae2f4492485d"}, + {file = "identify-2.5.9.tar.gz", hash = "sha256:906036344ca769539610436e40a684e170c3648b552194980bb7b617a8daeb9f"}, ] idna = [ {file = "idna-3.4-py3-none-any.whl", hash = "sha256:90b77e79eaa3eba6de819a0c442c0b4ceefc341a7a2ab77d7562bf49f425c5c2"}, @@ -1455,8 +1500,8 @@ mmhash3 = [ {file = "mmhash3-3.0.1.tar.gz", hash = "sha256:a00d68f4a1cc434b9501513c8a29e18ed1ddad383677d72b41d71d0d862348af"}, ] moto = [ - {file = "moto-4.0.9-py3-none-any.whl", hash = "sha256:2fb909d2ea1b732f89604e4268e2c2207c253e590a635a410c3c2aaebb34e113"}, - {file = "moto-4.0.9.tar.gz", hash = "sha256:ba03b638cf3b1cec64cbe9ac0d184ca898b69020c8e3c5b9b4961c1670629010"}, + {file = "moto-4.0.10-py3-none-any.whl", hash = "sha256:356bf792b439228891c910e2a0fafd4264334cf9000b508c732ff43d8694fb6a"}, + {file = "moto-4.0.10.tar.gz", hash = "sha256:9ba96d04a472d5682493cad7fee33337da34ebef18b397af1ea6dfb41efbe148"}, ] multidict = [ {file = "multidict-6.0.2-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:0b9e95a740109c6047602f4db4da9949e6c5945cefbad34a1299775ddc9a62e2"}, @@ -1524,34 +1569,34 @@ nodeenv = [ {file = "nodeenv-1.7.0.tar.gz", hash = "sha256:e0e7f7dfb85fc5394c6fe1e8fa98131a2473e04311a45afb6508f7cf1836fa2b"}, ] numpy = [ - {file = "numpy-1.23.4-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:95d79ada05005f6f4f337d3bb9de8a7774f259341c70bc88047a1f7b96a4bcb2"}, - {file = "numpy-1.23.4-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:926db372bc4ac1edf81cfb6c59e2a881606b409ddc0d0920b988174b2e2a767f"}, - {file = "numpy-1.23.4-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:c237129f0e732885c9a6076a537e974160482eab8f10db6292e92154d4c67d71"}, - {file = "numpy-1.23.4-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:a8365b942f9c1a7d0f0dc974747d99dd0a0cdfc5949a33119caf05cb314682d3"}, - {file = "numpy-1.23.4-cp310-cp310-win32.whl", hash = "sha256:2341f4ab6dba0834b685cce16dad5f9b6606ea8a00e6da154f5dbded70fdc4dd"}, - {file = "numpy-1.23.4-cp310-cp310-win_amd64.whl", hash = "sha256:d331afac87c92373826af83d2b2b435f57b17a5c74e6268b79355b970626e329"}, - {file = "numpy-1.23.4-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:488a66cb667359534bc70028d653ba1cf307bae88eab5929cd707c761ff037db"}, - {file = "numpy-1.23.4-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:ce03305dd694c4873b9429274fd41fc7eb4e0e4dea07e0af97a933b079a5814f"}, - {file = "numpy-1.23.4-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:8981d9b5619569899666170c7c9748920f4a5005bf79c72c07d08c8a035757b0"}, - {file = "numpy-1.23.4-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:7a70a7d3ce4c0e9284e92285cba91a4a3f5214d87ee0e95928f3614a256a1488"}, - {file = "numpy-1.23.4-cp311-cp311-win32.whl", hash = "sha256:5e13030f8793e9ee42f9c7d5777465a560eb78fa7e11b1c053427f2ccab90c79"}, - {file = "numpy-1.23.4-cp311-cp311-win_amd64.whl", hash = "sha256:7607b598217745cc40f751da38ffd03512d33ec06f3523fb0b5f82e09f6f676d"}, - {file = "numpy-1.23.4-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:7ab46e4e7ec63c8a5e6dbf5c1b9e1c92ba23a7ebecc86c336cb7bf3bd2fb10e5"}, - {file = "numpy-1.23.4-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:a8aae2fb3180940011b4862b2dd3756616841c53db9734b27bb93813cd79fce6"}, - {file = "numpy-1.23.4-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:8c053d7557a8f022ec823196d242464b6955a7e7e5015b719e76003f63f82d0f"}, - {file = "numpy-1.23.4-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:a0882323e0ca4245eb0a3d0a74f88ce581cc33aedcfa396e415e5bba7bf05f68"}, - {file = "numpy-1.23.4-cp38-cp38-win32.whl", hash = "sha256:dada341ebb79619fe00a291185bba370c9803b1e1d7051610e01ed809ef3a4ba"}, - {file = "numpy-1.23.4-cp38-cp38-win_amd64.whl", hash = "sha256:0fe563fc8ed9dc4474cbf70742673fc4391d70f4363f917599a7fa99f042d5a8"}, - {file = "numpy-1.23.4-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:c67b833dbccefe97cdd3f52798d430b9d3430396af7cdb2a0c32954c3ef73894"}, - {file = "numpy-1.23.4-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:f76025acc8e2114bb664294a07ede0727aa75d63a06d2fae96bf29a81747e4a7"}, - {file = "numpy-1.23.4-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:12ac457b63ec8ded85d85c1e17d85efd3c2b0967ca39560b307a35a6703a4735"}, - {file = "numpy-1.23.4-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:95de7dc7dc47a312f6feddd3da2500826defdccbc41608d0031276a24181a2c0"}, - {file = "numpy-1.23.4-cp39-cp39-win32.whl", hash = "sha256:f2f390aa4da44454db40a1f0201401f9036e8d578a25f01a6e237cea238337ef"}, - {file = "numpy-1.23.4-cp39-cp39-win_amd64.whl", hash = "sha256:f260da502d7441a45695199b4e7fd8ca87db659ba1c78f2bbf31f934fe76ae0e"}, - {file = "numpy-1.23.4-pp38-pypy38_pp73-macosx_10_9_x86_64.whl", hash = "sha256:61be02e3bf810b60ab74e81d6d0d36246dbfb644a462458bb53b595791251911"}, - {file = "numpy-1.23.4-pp38-pypy38_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:296d17aed51161dbad3c67ed6d164e51fcd18dbcd5dd4f9d0a9c6055dce30810"}, - {file = "numpy-1.23.4-pp38-pypy38_pp73-win_amd64.whl", hash = "sha256:4d52914c88b4930dafb6c48ba5115a96cbab40f45740239d9f4159c4ba779962"}, - {file = "numpy-1.23.4.tar.gz", hash = "sha256:ed2cc92af0efad20198638c69bb0fc2870a58dabfba6eb722c933b48556c686c"}, + {file = "numpy-1.23.5-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:9c88793f78fca17da0145455f0d7826bcb9f37da4764af27ac945488116efe63"}, + {file = "numpy-1.23.5-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:e9f4c4e51567b616be64e05d517c79a8a22f3606499941d97bb76f2ca59f982d"}, + {file = "numpy-1.23.5-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:7903ba8ab592b82014713c491f6c5d3a1cde5b4a3bf116404e08f5b52f6daf43"}, + {file = "numpy-1.23.5-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:5e05b1c973a9f858c74367553e236f287e749465f773328c8ef31abe18f691e1"}, + {file = "numpy-1.23.5-cp310-cp310-win32.whl", hash = "sha256:522e26bbf6377e4d76403826ed689c295b0b238f46c28a7251ab94716da0b280"}, + {file = "numpy-1.23.5-cp310-cp310-win_amd64.whl", hash = "sha256:dbee87b469018961d1ad79b1a5d50c0ae850000b639bcb1b694e9981083243b6"}, + {file = "numpy-1.23.5-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:ce571367b6dfe60af04e04a1834ca2dc5f46004ac1cc756fb95319f64c095a96"}, + {file = "numpy-1.23.5-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:56e454c7833e94ec9769fa0f86e6ff8e42ee38ce0ce1fa4cbb747ea7e06d56aa"}, + {file = "numpy-1.23.5-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:5039f55555e1eab31124a5768898c9e22c25a65c1e0037f4d7c495a45778c9f2"}, + {file = "numpy-1.23.5-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:58f545efd1108e647604a1b5aa809591ccd2540f468a880bedb97247e72db387"}, + {file = "numpy-1.23.5-cp311-cp311-win32.whl", hash = "sha256:b2a9ab7c279c91974f756c84c365a669a887efa287365a8e2c418f8b3ba73fb0"}, + {file = "numpy-1.23.5-cp311-cp311-win_amd64.whl", hash = "sha256:0cbe9848fad08baf71de1a39e12d1b6310f1d5b2d0ea4de051058e6e1076852d"}, + {file = "numpy-1.23.5-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:f063b69b090c9d918f9df0a12116029e274daf0181df392839661c4c7ec9018a"}, + {file = "numpy-1.23.5-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:0aaee12d8883552fadfc41e96b4c82ee7d794949e2a7c3b3a7201e968c7ecab9"}, + {file = "numpy-1.23.5-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:92c8c1e89a1f5028a4c6d9e3ccbe311b6ba53694811269b992c0b224269e2398"}, + {file = "numpy-1.23.5-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:d208a0f8729f3fb790ed18a003f3a57895b989b40ea4dce4717e9cf4af62c6bb"}, + {file = "numpy-1.23.5-cp38-cp38-win32.whl", hash = "sha256:06005a2ef6014e9956c09ba07654f9837d9e26696a0470e42beedadb78c11b07"}, + {file = "numpy-1.23.5-cp38-cp38-win_amd64.whl", hash = "sha256:ca51fcfcc5f9354c45f400059e88bc09215fb71a48d3768fb80e357f3b457e1e"}, + {file = "numpy-1.23.5-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:8969bfd28e85c81f3f94eb4a66bc2cf1dbdc5c18efc320af34bffc54d6b1e38f"}, + {file = "numpy-1.23.5-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:a7ac231a08bb37f852849bbb387a20a57574a97cfc7b6cabb488a4fc8be176de"}, + {file = "numpy-1.23.5-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:bf837dc63ba5c06dc8797c398db1e223a466c7ece27a1f7b5232ba3466aafe3d"}, + {file = "numpy-1.23.5-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:33161613d2269025873025b33e879825ec7b1d831317e68f4f2f0f84ed14c719"}, + {file = "numpy-1.23.5-cp39-cp39-win32.whl", hash = "sha256:af1da88f6bc3d2338ebbf0e22fe487821ea4d8e89053e25fa59d1d79786e7481"}, + {file = "numpy-1.23.5-cp39-cp39-win_amd64.whl", hash = "sha256:09b7847f7e83ca37c6e627682f145856de331049013853f344f37b0c9690e3df"}, + {file = "numpy-1.23.5-pp38-pypy38_pp73-macosx_10_9_x86_64.whl", hash = "sha256:abdde9f795cf292fb9651ed48185503a2ff29be87770c3b8e2a14b0cd7aa16f8"}, + {file = "numpy-1.23.5-pp38-pypy38_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:f9a909a8bae284d46bbfdefbdd4a262ba19d3bc9921b1e76126b1d21c3c34135"}, + {file = "numpy-1.23.5-pp38-pypy38_pp73-win_amd64.whl", hash = "sha256:01dd17cbb340bf0fc23981e52e1d18a9d4050792e8fb8363cecbf066a84b827d"}, + {file = "numpy-1.23.5.tar.gz", hash = "sha256:1b1766d6f397c18153d40015ddfc79ddb715cabadc04d2d228d4e5a8bc4ded1a"}, ] packaging = [ {file = "packaging-21.3-py3-none-any.whl", hash = "sha256:ef103e05f519cdc783ae24ea4e2e0f508a9c99b2d4969652eed6a2e1ea5bd522"}, @@ -1779,10 +1824,6 @@ s3transfer = [ {file = "s3transfer-0.6.0-py3-none-any.whl", hash = "sha256:06176b74f3a15f61f1b4f25a1fc29a4429040b7647133a463da8fa5bd28d5ecd"}, {file = "s3transfer-0.6.0.tar.gz", hash = "sha256:2ed07d3866f523cc561bf4a00fc5535827981b117dd7876f036b0c1aca42c947"}, ] -setuptools = [ - {file = "setuptools-65.5.1-py3-none-any.whl", hash = "sha256:d0b9a8433464d5800cbe05094acf5c6d52a91bfac9b52bcfc4d41382be5d5d31"}, - {file = "setuptools-65.5.1.tar.gz", hash = "sha256:e197a19aa8ec9722928f2206f8de752def0e4c9fc6953527360d1c36d94ddb2f"}, -] six = [ {file = "six-1.16.0-py2.py3-none-any.whl", hash = "sha256:8abb2f1d86890a2dfb989f9a77cfcfd3e47c2a354b01111771326f8aa26e0254"}, {file = "six-1.16.0.tar.gz", hash = "sha256:1e61c37477a1626458e36f7b1d82aa5c9b094fa4802892072e49de9c60c4c926"}, diff --git a/pyiceberg/catalog/__init__.py b/pyiceberg/catalog/__init__.py index 9a88d59e25..52095d6f4c 100644 --- a/pyiceberg/catalog/__init__.py +++ b/pyiceberg/catalog/__init__.py @@ -33,9 +33,9 @@ from pyiceberg.exceptions import NotInstalledError from pyiceberg.io import FileIO, load_file_io from pyiceberg.manifest import ManifestFile +from pyiceberg.partitioning import UNPARTITIONED_PARTITION_SPEC, PartitionSpec from pyiceberg.schema import Schema from pyiceberg.table import Table -from pyiceberg.table.partitioning import UNPARTITIONED_PARTITION_SPEC, PartitionSpec from pyiceberg.table.sorting import UNSORTED_SORT_ORDER, SortOrder from pyiceberg.typedef import ( EMPTY_DICT, diff --git a/pyiceberg/catalog/glue.py b/pyiceberg/catalog/glue.py index 45af6ef97f..4396e0423a 100644 --- a/pyiceberg/catalog/glue.py +++ b/pyiceberg/catalog/glue.py @@ -56,11 +56,11 @@ TableAlreadyExistsError, ) from pyiceberg.io import FileIO, load_file_io +from pyiceberg.partitioning import UNPARTITIONED_PARTITION_SPEC, PartitionSpec from pyiceberg.schema import Schema from pyiceberg.serializers import FromInputFile, ToOutputFile from pyiceberg.table import Table from pyiceberg.table.metadata import TableMetadata, new_table_metadata -from pyiceberg.table.partitioning import UNPARTITIONED_PARTITION_SPEC, PartitionSpec from pyiceberg.table.sorting import UNSORTED_SORT_ORDER, SortOrder from pyiceberg.typedef import EMPTY_DICT @@ -310,7 +310,7 @@ def purge_table(self, identifier: Union[str, Identifier]) -> None: manifest_lists_to_delete = set() manifests_to_delete = [] for snapshot in metadata.snapshots: - manifests_to_delete += snapshot.fetch_manifest_list(io) + manifests_to_delete += snapshot.manifests(io) if snapshot.manifest_list is not None: manifest_lists_to_delete.add(snapshot.manifest_list) diff --git a/pyiceberg/catalog/hive.py b/pyiceberg/catalog/hive.py index 65041697d6..3092be1ecb 100644 --- a/pyiceberg/catalog/hive.py +++ b/pyiceberg/catalog/hive.py @@ -58,11 +58,11 @@ TableAlreadyExistsError, ) from pyiceberg.io import FileIO, load_file_io +from pyiceberg.partitioning import UNPARTITIONED_PARTITION_SPEC, PartitionSpec from pyiceberg.schema import Schema, SchemaVisitor, visit from pyiceberg.serializers import FromInputFile, ToOutputFile from pyiceberg.table import Table from pyiceberg.table.metadata import TableMetadata, new_table_metadata -from pyiceberg.table.partitioning import UNPARTITIONED_PARTITION_SPEC, PartitionSpec from pyiceberg.table.sorting import UNSORTED_SORT_ORDER, SortOrder from pyiceberg.typedef import EMPTY_DICT from pyiceberg.types import ( diff --git a/pyiceberg/catalog/rest.py b/pyiceberg/catalog/rest.py index 40c2abb588..e453440c57 100644 --- a/pyiceberg/catalog/rest.py +++ b/pyiceberg/catalog/rest.py @@ -51,9 +51,9 @@ TableAlreadyExistsError, UnauthorizedError, ) +from pyiceberg.partitioning import UNPARTITIONED_PARTITION_SPEC, PartitionSpec from pyiceberg.schema import Schema from pyiceberg.table import Table, TableMetadata -from pyiceberg.table.partitioning import UNPARTITIONED_PARTITION_SPEC, PartitionSpec from pyiceberg.table.sorting import UNSORTED_SORT_ORDER, SortOrder from pyiceberg.typedef import EMPTY_DICT from pyiceberg.utils.iceberg_base_model import IcebergBaseModel diff --git a/pyiceberg/cli/output.py b/pyiceberg/cli/output.py index 2f83298cfc..5bb61d456b 100644 --- a/pyiceberg/cli/output.py +++ b/pyiceberg/cli/output.py @@ -23,9 +23,9 @@ from rich.table import Table as RichTable from rich.tree import Tree +from pyiceberg.partitioning import PartitionSpec from pyiceberg.schema import Schema from pyiceberg.table import Table, TableMetadata -from pyiceberg.table.partitioning import PartitionSpec from pyiceberg.typedef import Identifier, Properties from pyiceberg.utils.iceberg_base_model import IcebergBaseModel @@ -140,7 +140,7 @@ def files(self, table: Table, history: bool) -> None: manifest_list_str = f": {snapshot.manifest_list}" if snapshot.manifest_list else "" list_tree = snapshot_tree.add(f"Snapshot {snapshot.snapshot_id}, schema {snapshot.schema_id}{manifest_list_str}") - manifest_list = snapshot.fetch_manifest_list(io) + manifest_list = snapshot.manifests(io) for manifest in manifest_list: manifest_tree = list_tree.add(f"Manifest: {manifest.manifest_path}") for manifest_entry in manifest.fetch_manifest_entry(io): diff --git a/pyiceberg/expressions/visitors.py b/pyiceberg/expressions/visitors.py index 366d52d6fe..23b96f71c4 100644 --- a/pyiceberg/expressions/visitors.py +++ b/pyiceberg/expressions/visitors.py @@ -51,8 +51,8 @@ ) from pyiceberg.expressions.literals import Literal from pyiceberg.manifest import ManifestFile, PartitionFieldSummary +from pyiceberg.partitioning import PartitionSpec from pyiceberg.schema import Schema -from pyiceberg.table import PartitionSpec from pyiceberg.typedef import StructProtocol from pyiceberg.types import ( DoubleType, diff --git a/pyiceberg/manifest.py b/pyiceberg/manifest.py index 34c6da924a..75ec0b6a14 100644 --- a/pyiceberg/manifest.py +++ b/pyiceberg/manifest.py @@ -141,6 +141,14 @@ def read_manifest_entry(input_file: InputFile) -> Iterator[ManifestEntry]: yield ManifestEntry(**dict_repr) +def live_entries(input_file: InputFile) -> Iterator[ManifestEntry]: + return (entry for entry in read_manifest_entry(input_file) if entry.status != ManifestEntryStatus.DELETED) + + +def files(input_file: InputFile) -> Iterator[DataFile]: + return (entry.data_file for entry in live_entries(input_file)) + + def read_manifest_list(input_file: InputFile) -> Iterator[ManifestFile]: with AvroFile(input_file) as reader: schema = reader.schema diff --git a/pyiceberg/table/partitioning.py b/pyiceberg/partitioning.py similarity index 100% rename from pyiceberg/table/partitioning.py rename to pyiceberg/partitioning.py diff --git a/pyiceberg/table/__init__.py b/pyiceberg/table/__init__.py index 3af1c13e66..369d02f368 100644 --- a/pyiceberg/table/__init__.py +++ b/pyiceberg/table/__init__.py @@ -16,24 +16,43 @@ # under the License. from __future__ import annotations +from abc import ABC, abstractmethod +from dataclasses import dataclass from typing import ( Any, + Callable, Dict, + Generic, + Iterator, List, Optional, Tuple, + TypeVar, ) from pydantic import Field -from pyiceberg.expressions import AlwaysTrue, And, BooleanExpression +from pyiceberg.expressions import ( + AlwaysTrue, + And, + BooleanExpression, + visitors, +) from pyiceberg.io import FileIO +from pyiceberg.manifest import DataFile, ManifestFile, files +from pyiceberg.partitioning import PartitionSpec from pyiceberg.schema import Schema from pyiceberg.table.metadata import TableMetadata -from pyiceberg.table.partitioning import PartitionSpec from pyiceberg.table.snapshots import Snapshot, SnapshotLogEntry from pyiceberg.table.sorting import SortOrder -from pyiceberg.typedef import EMPTY_DICT, Identifier, Properties +from pyiceberg.typedef import ( + EMPTY_DICT, + Identifier, + KeyDefaultDict, + Properties, + StructProtocol, +) +from pyiceberg.types import StructType class Table: @@ -64,8 +83,8 @@ def scan( case_sensitive: bool = True, snapshot_id: Optional[int] = None, options: Properties = EMPTY_DICT, - ) -> TableScan: - return TableScan( + ) -> TableScan[Any]: + return DataScan( table=self, row_filter=row_filter or AlwaysTrue(), partition_filter=partition_filter or AlwaysTrue(), @@ -138,7 +157,10 @@ def __eq__(self, other: Any) -> bool: ) -class TableScan: +S = TypeVar("S", bound="TableScan", covariant=True) # type: ignore + + +class TableScan(Generic[S], ABC): table: Table row_filter: BooleanExpression partition_filter: BooleanExpression @@ -181,15 +203,17 @@ def projection(self) -> Schema: return snapshot_schema.select(*self.selected_fields, case_sensitive=self.case_sensitive) + @abstractmethod def plan_files(self): - raise NotImplementedError("Not yet implemented") + ... + @abstractmethod def to_arrow(self): - raise NotImplementedError("Not yet implemented") + ... - def update(self, **overrides) -> TableScan: + def update(self: S, **overrides) -> S: """Creates a copy of this table scan with updated fields.""" - return TableScan(**{**self.__dict__, **overrides}) + return type(self)(**{**self.__dict__, **overrides}) def use_ref(self, name: str): if self.snapshot_id: @@ -199,16 +223,144 @@ def use_ref(self, name: str): raise ValueError(f"Cannot scan unknown ref={name}") - def select(self, *field_names: str) -> TableScan: + def select(self, *field_names: str) -> S: if "*" in self.selected_fields: return self.update(selected_fields=field_names) return self.update(selected_fields=tuple(set(self.selected_fields).intersection(set(field_names)))) - def filter_rows(self, new_row_filter: BooleanExpression) -> TableScan: + def filter_rows(self, new_row_filter: BooleanExpression) -> S: return self.update(row_filter=And(self.row_filter, new_row_filter)) - def filter_partitions(self, new_partition_filter: BooleanExpression) -> TableScan: + def filter_partitions(self, new_partition_filter: BooleanExpression) -> S: return self.update(partition_filter=And(self.partition_filter, new_partition_filter)) - def with_case_sensitive(self, case_sensitive: bool = True) -> TableScan: + def with_case_sensitive(self, case_sensitive: bool = True) -> S: return self.update(case_sensitive=case_sensitive) + + +class ScanTask(ABC): + pass + + +@dataclass(init=False) +class FileScanTask(ScanTask): + file: DataFile + start: int + length: int + + def __init__(self, data_file: DataFile, start: Optional[int] = None, length: Optional[int] = None): + self.file = data_file + self.start = start or 0 + self.length = length or data_file.file_size_in_bytes + + +class _DictAsStruct(StructProtocol): + pos_to_name: Dict[int, str] + wrapped: Dict[str, Any] + + def __init__(self, partition_type: StructType): + self.pos_to_name = {pos: field.name for pos, field in enumerate(partition_type.fields)} + + def wrap(self, to_wrap: Dict[str, Any]) -> _DictAsStruct: + self.wrapped = to_wrap + return self + + def get(self, pos: int) -> Any: + return self.wrapped[self.pos_to_name[pos]] + + def set(self, pos: int, value: Any) -> None: + raise NotImplementedError("Cannot set values in DictAsStruct") + + +class DataScan(TableScan["DataScan"]): + def __init__( + self, + table: Table, + row_filter: Optional[BooleanExpression] = None, + partition_filter: Optional[BooleanExpression] = None, + selected_fields: Tuple[str] = ("*",), + case_sensitive: bool = True, + snapshot_id: Optional[int] = None, + options: Properties = EMPTY_DICT, + ): + super().__init__(table, row_filter, partition_filter, selected_fields, case_sensitive, snapshot_id, options) + + def _build_manifest_evaluator(self, spec_id: int) -> Callable[[ManifestFile], bool]: + spec = self.table.specs()[spec_id] + return visitors.manifest_evaluator(spec, self.table.schema(), self.partition_filter, self.case_sensitive) + + def _build_partition_evaluator(self, spec_id: int) -> Callable[[DataFile], bool]: + spec = self.table.specs()[spec_id] + partition_type = spec.partition_type(self.table.schema()) + partition_schema = Schema(*partition_type.fields) + + # TODO: project the row filter # pylint: disable=W0511 + partition_expr = And(self.partition_filter, AlwaysTrue()) + + # TODO: remove the dict to struct wrapper by using a StructProtocol record # pylint: disable=W0511 + wrapper = _DictAsStruct(partition_type) + evaluator = visitors.expression_evaluator(partition_schema, partition_expr, self.case_sensitive) + + return lambda data_file: evaluator(wrapper.wrap(data_file.partition)) + + def plan_files(self) -> Iterator[ScanTask]: + snapshot = self.snapshot() + if not snapshot: + return + + io = self.table.io + + # step 1: filter manifests using partition summaries + # the filter depends on the partition spec used to write the manifest file, so create a cache of filters for each spec id + + manifest_evaluators: Dict[int, Callable[[ManifestFile], bool]] = KeyDefaultDict(self._build_manifest_evaluator) + + manifests = [ + manifest_file + for manifest_file in snapshot.manifests(io) + if manifest_evaluators[manifest_file.partition_spec_id](manifest_file) + ] + + # step 2: filter the data files in each manifest + # this filter depends on the partition spec used to write the manifest file + + partition_evaluators: Dict[int, Callable[[DataFile], bool]] = KeyDefaultDict(self._build_partition_evaluator) + + for manifest in manifests: + partition_filter = partition_evaluators[manifest.partition_spec_id] + all_files = files(io.new_input(manifest.manifest_path)) + matching_partition_files = filter(partition_filter, all_files) + + yield from (FileScanTask(file) for file in matching_partition_files) + + def to_arrow(self): + from pyiceberg.io.pyarrow import PyArrowFileIO + + fs = None + if isinstance(self.table.io, PyArrowFileIO): + scheme, path = PyArrowFileIO.parse_location(self.table.location()) + fs = self.table.io.get_fs(scheme) + + import pyarrow.parquet as pq + + locations = [] + for task in self.plan_files(): + if isinstance(task, FileScanTask): + _, path = PyArrowFileIO.parse_location(task.file.file_path) + locations.append(path) + else: + raise ValueError(f"Cannot read unexpected task: {task}") + + columns = None + if "*" not in self.selected_fields: + columns = list(self.selected_fields) + + return pq.read_table(source=locations, filesystem=fs, columns=columns) + + def to_duckdb(self, table_name: str, connection=None): + import duckdb + + con = connection or duckdb.connect(database=":memory:") + con.register(table_name, self.to_arrow()) + + return con diff --git a/pyiceberg/table/metadata.py b/pyiceberg/table/metadata.py index df96036167..39e60556f7 100644 --- a/pyiceberg/table/metadata.py +++ b/pyiceberg/table/metadata.py @@ -29,8 +29,8 @@ from pydantic import Field, root_validator from pyiceberg.exceptions import ValidationError +from pyiceberg.partitioning import PartitionSpec, assign_fresh_partition_spec_ids from pyiceberg.schema import Schema, assign_fresh_schema_ids -from pyiceberg.table.partitioning import PartitionSpec, assign_fresh_partition_spec_ids from pyiceberg.table.refs import MAIN_BRANCH, SnapshotRef, SnapshotRefType from pyiceberg.table.snapshots import MetadataLogEntry, Snapshot, SnapshotLogEntry from pyiceberg.table.sorting import ( diff --git a/pyiceberg/table/snapshots.py b/pyiceberg/table/snapshots.py index 35dd3c87e7..3fd834af84 100644 --- a/pyiceberg/table/snapshots.py +++ b/pyiceberg/table/snapshots.py @@ -110,7 +110,7 @@ def __str__(self) -> str: result_str = f"{operation}id={self.snapshot_id}{parent_id}{schema_id}" return result_str - def fetch_manifest_list(self, io: FileIO) -> List[ManifestFile]: + def manifests(self, io: FileIO) -> List[ManifestFile]: if self.manifest_list is not None: file = io.new_input(self.manifest_list) return list(read_manifest_list(file)) diff --git a/pyiceberg/typedef.py b/pyiceberg/typedef.py index 97e23e7888..436b1a6e79 100644 --- a/pyiceberg/typedef.py +++ b/pyiceberg/typedef.py @@ -18,6 +18,7 @@ from decimal import Decimal from typing import ( Any, + Callable, Dict, Protocol, Tuple, @@ -38,6 +39,26 @@ def update(self, *args: Any, **kwargs: Any) -> None: EMPTY_DICT = FrozenDict() + +K = TypeVar("K") +V = TypeVar("V") + + +# from https://stackoverflow.com/questions/2912231/is-there-a-clever-way-to-pass-the-key-to-defaultdicts-default-factory +class KeyDefaultDict(Dict[K, V]): + def __init__(self, default_factory: Callable[[K], V]): + super().__init__() + self.default_factory = default_factory + + def __missing__(self, key: K) -> V: + if self.default_factory is None: + raise KeyError(key) + else: + val = self.default_factory(key) + self[key] = val + return val + + Identifier = Tuple[str, ...] Properties = Dict[str, str] RecursiveDict = Dict[str, Union[str, "RecursiveDict"]] diff --git a/pyproject.toml b/pyproject.toml index fda44c0904..1374abff03 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -59,6 +59,8 @@ zstandard = "0.19.0" pyarrow = { version = "10.0.0", optional = true } +duckdb = { version = "0.6.0", optional = true } + python-snappy = { version = "0.6.1", optional = true } thrift = { version = "0.16.0", optional = true } @@ -86,6 +88,7 @@ build-backend = "poetry.core.masonry.api" [tool.poetry.extras] pyarrow = ["pyarrow"] +duckdb = ["duckdb", "pyarrow"] snappy = ["python-snappy"] hive = ["thrift"] s3fs = ["s3fs"] @@ -202,5 +205,9 @@ ignore_missing_imports = true module = "aiohttp.*" ignore_missing_imports = true +[[tool.mypy.overrides]] +module = "duckdb.*" +ignore_missing_imports = true + [tool.coverage.run] source = ['pyiceberg/'] diff --git a/tests/catalog/test_base.py b/tests/catalog/test_base.py index 5b8886ceea..0b7c294fe6 100644 --- a/tests/catalog/test_base.py +++ b/tests/catalog/test_base.py @@ -39,9 +39,9 @@ TableAlreadyExistsError, ) from pyiceberg.io import load_file_io +from pyiceberg.partitioning import UNPARTITIONED_PARTITION_SPEC, PartitionField, PartitionSpec from pyiceberg.schema import Schema from pyiceberg.table import Table -from pyiceberg.table.partitioning import UNPARTITIONED_PARTITION_SPEC, PartitionField, PartitionSpec from pyiceberg.table.sorting import UNSORTED_SORT_ORDER, SortOrder from pyiceberg.transforms import IdentityTransform from pyiceberg.typedef import EMPTY_DICT diff --git a/tests/catalog/test_hive.py b/tests/catalog/test_hive.py index bc45bb457e..a838415fbf 100644 --- a/tests/catalog/test_hive.py +++ b/tests/catalog/test_hive.py @@ -42,10 +42,10 @@ NoSuchNamespaceError, NoSuchTableError, ) +from pyiceberg.partitioning import PartitionField, PartitionSpec from pyiceberg.schema import Schema from pyiceberg.serializers import ToOutputFile from pyiceberg.table.metadata import TableMetadataUtil, TableMetadataV2 -from pyiceberg.table.partitioning import PartitionField, PartitionSpec from pyiceberg.table.refs import SnapshotRef, SnapshotRefType from pyiceberg.table.snapshots import ( MetadataLogEntry, diff --git a/tests/catalog/test_rest.py b/tests/catalog/test_rest.py index c84c24ae5e..45df4fcba1 100644 --- a/tests/catalog/test_rest.py +++ b/tests/catalog/test_rest.py @@ -31,9 +31,9 @@ TableAlreadyExistsError, ) from pyiceberg.io import load_file_io +from pyiceberg.partitioning import PartitionField, PartitionSpec from pyiceberg.schema import Schema from pyiceberg.table.metadata import TableMetadataV1 -from pyiceberg.table.partitioning import PartitionField, PartitionSpec from pyiceberg.table.refs import SnapshotRef, SnapshotRefType from pyiceberg.table.snapshots import Operation, Snapshot, Summary from pyiceberg.table.sorting import SortField, SortOrder diff --git a/tests/cli/test_console.py b/tests/cli/test_console.py index 1c5728831b..f0420bde5c 100644 --- a/tests/cli/test_console.py +++ b/tests/cli/test_console.py @@ -29,10 +29,10 @@ from pyiceberg.cli.console import run from pyiceberg.exceptions import NoSuchNamespaceError, NoSuchTableError from pyiceberg.io import load_file_io +from pyiceberg.partitioning import UNPARTITIONED_PARTITION_SPEC, PartitionSpec from pyiceberg.schema import Schema from pyiceberg.table import Table from pyiceberg.table.metadata import TableMetadataV2 -from pyiceberg.table.partitioning import UNPARTITIONED_PARTITION_SPEC, PartitionSpec from pyiceberg.table.sorting import UNSORTED_SORT_ORDER, SortOrder from pyiceberg.typedef import EMPTY_DICT, Identifier, Properties from tests.conftest import EXAMPLE_TABLE_METADATA_V2 diff --git a/tests/table/test_init.py b/tests/table/test_init.py index 8e4dc7820e..475a600fd9 100644 --- a/tests/table/test_init.py +++ b/tests/table/test_init.py @@ -26,10 +26,10 @@ In, ) from pyiceberg.io import load_file_io +from pyiceberg.partitioning import PartitionField, PartitionSpec from pyiceberg.schema import Schema -from pyiceberg.table import PartitionSpec, Table +from pyiceberg.table import Table from pyiceberg.table.metadata import TableMetadataV2 -from pyiceberg.table.partitioning import PartitionField from pyiceberg.table.snapshots import ( Operation, Snapshot, diff --git a/tests/table/test_metadata.py b/tests/table/test_metadata.py index c88db6a2da..297d58c6ed 100644 --- a/tests/table/test_metadata.py +++ b/tests/table/test_metadata.py @@ -24,6 +24,7 @@ import pytest from pyiceberg.exceptions import ValidationError +from pyiceberg.partitioning import PartitionField, PartitionSpec from pyiceberg.schema import Schema from pyiceberg.serializers import FromByteStream from pyiceberg.table import SortOrder @@ -33,7 +34,6 @@ TableMetadataV2, new_table_metadata, ) -from pyiceberg.table.partitioning import PartitionField, PartitionSpec from pyiceberg.table.refs import SnapshotRef, SnapshotRefType from pyiceberg.table.sorting import NullOrder, SortDirection, SortField from pyiceberg.transforms import IdentityTransform diff --git a/tests/table/test_partitioning.py b/tests/table/test_partitioning.py index b3cbf30f8f..a559dcea30 100644 --- a/tests/table/test_partitioning.py +++ b/tests/table/test_partitioning.py @@ -14,8 +14,8 @@ # KIND, either express or implied. See the License for the # specific language governing permissions and limitations # under the License. +from pyiceberg.partitioning import UNPARTITIONED_PARTITION_SPEC, PartitionField, PartitionSpec from pyiceberg.schema import Schema -from pyiceberg.table.partitioning import UNPARTITIONED_PARTITION_SPEC, PartitionField, PartitionSpec from pyiceberg.transforms import BucketTransform, TruncateTransform from pyiceberg.types import ( IntegerType, diff --git a/tests/table/test_snapshots.py b/tests/table/test_snapshots.py index 6460ba4daf..88ab8718be 100644 --- a/tests/table/test_snapshots.py +++ b/tests/table/test_snapshots.py @@ -134,7 +134,7 @@ def test_fetch_manifest_list(generated_manifest_file_file: str): schema_id=3, ) io = PyArrowFileIO() - actual = snapshot.fetch_manifest_list(io) + actual = snapshot.manifests(io) assert actual == [ ManifestFile( manifest_path=actual[0].manifest_path, # Is a temp path that changes every time diff --git a/tests/utils/test_manifest.py b/tests/utils/test_manifest.py index 7b1d391e25..2f351539eb 100644 --- a/tests/utils/test_manifest.py +++ b/tests/utils/test_manifest.py @@ -303,7 +303,7 @@ def test_read_manifest(generated_manifest_file_file: str, generated_manifest_ent summary=Summary(Operation.APPEND), schema_id=3, ) - manifest_list = snapshot.fetch_manifest_list(io) + manifest_list = snapshot.manifests(io) assert manifest_list == [ ManifestFile( From c32fd4edbb2b352ed7dd75545709329bf2e460df Mon Sep 17 00:00:00 2001 From: Fokko Driesprong Date: Tue, 22 Nov 2022 00:37:27 +0100 Subject: [PATCH 276/642] Python: Implement project in Transform implementations (#6128) --- pyiceberg/expressions/__init__.py | 68 +++++ pyiceberg/expressions/literals.py | 29 +++ pyiceberg/transforms.py | 214 ++++++++++++++-- tests/expressions/test_literals.py | 18 ++ tests/test_transforms.py | 389 ++++++++++++++++++++++++++++- 5 files changed, 701 insertions(+), 17 deletions(-) diff --git a/pyiceberg/expressions/__init__.py b/pyiceberg/expressions/__init__.py index f39e15006b..5d25d33915 100644 --- a/pyiceberg/expressions/__init__.py +++ b/pyiceberg/expressions/__init__.py @@ -309,6 +309,11 @@ def __eq__(self, other: Any) -> bool: return self.term == other.term return False + @property + @abstractmethod + def as_unbound(self) -> Type[UnboundPredicate[Any]]: + ... + class UnboundPredicate(Generic[L], Unbound[BooleanExpression], BooleanExpression, ABC): term: UnboundTerm[Any] @@ -347,6 +352,11 @@ class BoundUnaryPredicate(BoundPredicate[L], ABC): def __repr__(self) -> str: return f"{str(self.__class__.__name__)}(term={repr(self.term)})" + @property + @abstractmethod + def as_unbound(self) -> Type[UnaryPredicate]: + ... + class BoundIsNull(BoundUnaryPredicate[L]): def __new__(cls, term: BoundTerm[L]): # pylint: disable=W0221 @@ -357,6 +367,10 @@ def __new__(cls, term: BoundTerm[L]): # pylint: disable=W0221 def __invert__(self) -> BoundNotNull[L]: return BoundNotNull(self.term) + @property + def as_unbound(self) -> Type[IsNull]: + return IsNull + class BoundNotNull(BoundUnaryPredicate[L]): def __new__(cls, term: BoundTerm[L]): # pylint: disable=W0221 @@ -367,6 +381,10 @@ def __new__(cls, term: BoundTerm[L]): # pylint: disable=W0221 def __invert__(self) -> BoundIsNull[L]: return BoundIsNull(self.term) + @property + def as_unbound(self) -> Type[NotNull]: + return NotNull + class IsNull(UnaryPredicate): def __invert__(self) -> NotNull: @@ -396,6 +414,10 @@ def __new__(cls, term: BoundTerm[L]): # pylint: disable=W0221 def __invert__(self) -> BoundNotNaN[L]: return BoundNotNaN(self.term) + @property + def as_unbound(self) -> Type[IsNaN]: + return IsNaN + class BoundNotNaN(BoundUnaryPredicate[L]): def __new__(cls, term: BoundTerm[L]): # pylint: disable=W0221 @@ -407,6 +429,10 @@ def __new__(cls, term: BoundTerm[L]): # pylint: disable=W0221 def __invert__(self) -> BoundIsNaN[L]: return BoundIsNaN(self.term) + @property + def as_unbound(self) -> Type[NotNaN]: + return NotNaN + class IsNaN(UnaryPredicate): def __invert__(self) -> NotNaN: @@ -477,6 +503,11 @@ def __repr__(self) -> str: def __eq__(self, other: Any) -> bool: return self.term == other.term and self.literals == other.literals if isinstance(other, BoundSetPredicate) else False + @property + @abstractmethod + def as_unbound(self) -> Type[SetPredicate[L]]: + ... + class BoundIn(BoundSetPredicate[L]): def __new__(cls, term: BoundTerm[L], literals: Set[Literal[L]]): # pylint: disable=W0221 @@ -494,6 +525,10 @@ def __invert__(self) -> BoundNotIn[L]: def __eq__(self, other: Any) -> bool: return self.term == other.term and self.literals == other.literals if isinstance(other, BoundIn) else False + @property + def as_unbound(self) -> Type[In[L]]: + return In + class BoundNotIn(BoundSetPredicate[L]): def __new__( # pylint: disable=W0221 @@ -512,6 +547,10 @@ def __new__( # pylint: disable=W0221 def __invert__(self) -> BoundIn[L]: return BoundIn(self.term, self.literals) + @property + def as_unbound(self) -> Type[NotIn[L]]: + return NotIn + class In(SetPredicate[L]): def __new__( @@ -601,36 +640,65 @@ def __eq__(self, other): def __repr__(self) -> str: return f"{str(self.__class__.__name__)}(term={repr(self.term)}, literal={repr(self.literal)})" + @property + @abstractmethod + def as_unbound(self) -> Type[LiteralPredicate[L]]: + ... + class BoundEqualTo(BoundLiteralPredicate[L]): def __invert__(self) -> BoundNotEqualTo[L]: return BoundNotEqualTo[L](self.term, self.literal) + @property + def as_unbound(self) -> Type[EqualTo[L]]: + return EqualTo + class BoundNotEqualTo(BoundLiteralPredicate[L]): def __invert__(self) -> BoundEqualTo[L]: return BoundEqualTo[L](self.term, self.literal) + @property + def as_unbound(self) -> Type[NotEqualTo[L]]: + return NotEqualTo + class BoundGreaterThanOrEqual(BoundLiteralPredicate[L]): def __invert__(self) -> BoundLessThan[L]: return BoundLessThan[L](self.term, self.literal) + @property + def as_unbound(self) -> Type[GreaterThanOrEqual[L]]: + return GreaterThanOrEqual[L] + class BoundGreaterThan(BoundLiteralPredicate[L]): def __invert__(self) -> BoundLessThanOrEqual[L]: return BoundLessThanOrEqual(self.term, self.literal) + @property + def as_unbound(self) -> Type[GreaterThan[L]]: + return GreaterThan[L] + class BoundLessThan(BoundLiteralPredicate[L]): def __invert__(self) -> BoundGreaterThanOrEqual[L]: return BoundGreaterThanOrEqual[L](self.term, self.literal) + @property + def as_unbound(self) -> Type[LessThan[L]]: + return LessThan[L] + class BoundLessThanOrEqual(BoundLiteralPredicate[L]): def __invert__(self) -> BoundGreaterThan[L]: return BoundGreaterThan[L](self.term, self.literal) + @property + def as_unbound(self) -> Type[LessThanOrEqual[L]]: + return LessThanOrEqual[L] + class EqualTo(LiteralPredicate[L]): def __invert__(self) -> NotEqualTo[L]: diff --git a/pyiceberg/expressions/literals.py b/pyiceberg/expressions/literals.py index 44ab9a15e6..c59c6bcf8d 100644 --- a/pyiceberg/expressions/literals.py +++ b/pyiceberg/expressions/literals.py @@ -53,6 +53,7 @@ timestamp_to_micros, timestamptz_to_micros, ) +from pyiceberg.utils.decimal import decimal_to_unscaled, unscaled_to_decimal from pyiceberg.utils.singleton import Singleton @@ -210,6 +211,12 @@ def __init__(self, value: int): def to(self, type_var: IcebergType) -> Literal: # type: ignore raise TypeError(f"Cannot convert LongLiteral into {type_var}") + def increment(self) -> Literal[int]: + return LongLiteral(self.value + 1) + + def decrement(self) -> Literal[int]: + return LongLiteral(self.value - 1) + @to.register(LongType) def _(self, _: LongType) -> Literal[int]: return self @@ -319,6 +326,12 @@ class DateLiteral(Literal[int]): def __init__(self, value: int): super().__init__(value, int) + def increment(self) -> Literal[int]: + return DateLiteral(self.value + 1) + + def decrement(self) -> Literal[int]: + return DateLiteral(self.value - 1) + @singledispatchmethod def to(self, type_var: IcebergType) -> Literal: # type: ignore raise TypeError(f"Cannot convert DateLiteral into {type_var}") @@ -345,6 +358,12 @@ class TimestampLiteral(Literal[int]): def __init__(self, value: int): super().__init__(value, int) + def increment(self) -> Literal[int]: + return TimestampLiteral(self.value + 1) + + def decrement(self) -> Literal[int]: + return TimestampLiteral(self.value - 1) + @singledispatchmethod def to(self, type_var: IcebergType) -> Literal: # type: ignore raise TypeError(f"Cannot convert TimestampLiteral into {type_var}") @@ -362,6 +381,16 @@ class DecimalLiteral(Literal[Decimal]): def __init__(self, value: Decimal): super().__init__(value, Decimal) + def increment(self) -> Literal[Decimal]: + original_scale = abs(self.value.as_tuple().exponent) + unscaled = decimal_to_unscaled(self.value) + return DecimalLiteral(unscaled_to_decimal(unscaled + 1, original_scale)) + + def decrement(self) -> Literal[Decimal]: + original_scale = abs(self.value.as_tuple().exponent) + unscaled = decimal_to_unscaled(self.value) + return DecimalLiteral(unscaled_to_decimal(unscaled - 1, original_scale)) + @singledispatchmethod def to(self, type_var: IcebergType) -> Literal: # type: ignore raise TypeError(f"Cannot convert DecimalLiteral into {type_var}") diff --git a/pyiceberg/transforms.py b/pyiceberg/transforms.py index 14d76fd8cb..79f901913c 100644 --- a/pyiceberg/transforms.py +++ b/pyiceberg/transforms.py @@ -20,18 +20,41 @@ from abc import ABC, abstractmethod from enum import IntEnum from functools import singledispatch -from typing import ( - Any, - Callable, - Generic, - Literal, - Optional, - TypeVar, -) +from typing import Any, Callable, Generic +from typing import Literal as LiteralType +from typing import Optional, TypeVar import mmh3 from pydantic import Field, PositiveInt, PrivateAttr +from pyiceberg.expressions import ( + BoundEqualTo, + BoundGreaterThan, + BoundGreaterThanOrEqual, + BoundIn, + BoundLessThan, + BoundLessThanOrEqual, + BoundLiteralPredicate, + BoundNotIn, + BoundPredicate, + BoundSetPredicate, + BoundTerm, + BoundUnaryPredicate, + EqualTo, + GreaterThanOrEqual, + LessThanOrEqual, + Reference, + UnboundPredicate, +) +from pyiceberg.expressions.literals import ( + DateLiteral, + DecimalLiteral, + Literal, + LongLiteral, + TimestampLiteral, + literal, +) +from pyiceberg.typedef import L from pyiceberg.types import ( BinaryType, DateType, @@ -68,6 +91,11 @@ TRUNCATE_PARSER = ParseNumberFromBrackets(TRUNCATE) +def _transform_literal(func: Callable[[L], L], lit: Literal[L]) -> Literal[L]: + """Small helper to upwrap the value from the literal, and wrap it again""" + return literal(func(lit.value)) + + class Transform(IcebergBaseModel, ABC, Generic[S, T]): """Transform base class for concrete transforms. @@ -121,6 +149,10 @@ def can_transform(self, source: IcebergType) -> bool: def result_type(self, source: IcebergType) -> IcebergType: ... + @abstractmethod + def project(self, name: str, pred: BoundPredicate[L]) -> Optional[UnboundPredicate[Any]]: + ... + @property def preserves_order(self) -> bool: return False @@ -173,6 +205,23 @@ def apply(self, value: Optional[S]) -> Optional[int]: def result_type(self, source: IcebergType) -> IcebergType: return IntegerType() + def project(self, name: str, pred: BoundPredicate[L]) -> Optional[UnboundPredicate[Any]]: + transformer = self.transform(pred.term.ref().field.field_type) + + if isinstance(pred.term, BoundTransform): + return _project_transform_predicate(self, name, pred) + elif isinstance(pred, BoundUnaryPredicate): + return pred.as_unbound(Reference(name)) + elif isinstance(pred, BoundEqualTo): + return pred.as_unbound(Reference(name), _transform_literal(transformer, pred.literal)) + elif isinstance(pred, BoundIn): # NotIn can't be projected + return pred.as_unbound(Reference(name), {_transform_literal(transformer, literal) for literal in pred.literals}) + else: + # - Comparison predicates can't be projected, notEq can't be projected + # - Small ranges can be projected: + # For example, (x > 0) and (x < 3) can be turned into in({1, 2}) and projected. + return None + def can_transform(self, source: IcebergType) -> bool: return type(source) in { IntegerType, @@ -246,9 +295,26 @@ def granularity(self) -> TimeResolution: def satisfies_order_of(self, other: Transform[S, T]) -> bool: return self.granularity <= other.granularity if hasattr(other, "granularity") else False - def result_type(self, source: IcebergType) -> IcebergType: + def result_type(self, source: IcebergType) -> IntegerType: return IntegerType() + @abstractmethod + def transform(self, source: IcebergType) -> Callable[[Optional[Any]], Optional[int]]: + ... + + def project(self, name: str, pred: BoundPredicate[L]) -> Optional[UnboundPredicate[Any]]: + transformer = self.transform(pred.term.ref().field.field_type) + if isinstance(pred.term, BoundTransform): + return _project_transform_predicate(self, name, pred) + elif isinstance(pred, BoundUnaryPredicate): + return pred.as_unbound(Reference(name)) + elif isinstance(pred, BoundLiteralPredicate): + return _truncate_number(name, pred, transformer) + elif isinstance(pred, BoundIn): # NotIn can't be projected + return _set_apply_transform(name, pred, transformer) + else: + return None + @property def dedup_name(self) -> str: return "time" @@ -267,7 +333,7 @@ class YearTransform(TimeTransform[S]): 47 """ - __root__: Literal["year"] = Field(default="year") + __root__: LiteralType["year"] = Field(default="year") # noqa: F821 def transform(self, source: IcebergType) -> Callable[[Optional[S]], Optional[int]]: source_type = type(source) @@ -313,7 +379,7 @@ class MonthTransform(TimeTransform[S]): 575 """ - __root__: Literal["month"] = Field(default="month") + __root__: LiteralType["month"] = Field(default="month") # noqa: F821 def transform(self, source: IcebergType) -> Callable[[Optional[S]], Optional[int]]: source_type = type(source) @@ -359,7 +425,7 @@ class DayTransform(TimeTransform[S]): 17501 """ - __root__: Literal["day"] = Field(default="day") + __root__: LiteralType["day"] = Field(default="day") # noqa: F821 def transform(self, source: IcebergType) -> Callable[[Optional[S]], Optional[int]]: source_type = type(source) @@ -408,7 +474,7 @@ class HourTransform(TimeTransform[S]): 420042 """ - __root__: Literal["hour"] = Field(default="hour") + __root__: LiteralType["hour"] = Field(default="hour") # noqa: F821 def transform(self, source: IcebergType) -> Callable[[Optional[S]], Optional[int]]: if type(source) in {TimestampType, TimestamptzType}: @@ -452,7 +518,7 @@ class IdentityTransform(Transform[S, S]): 'hello-world' """ - __root__: Literal["identity"] = Field(default="identity") + __root__: LiteralType["identity"] = Field(default="identity") # noqa: F821 def transform(self, source: IcebergType) -> Callable[[Optional[S]], Optional[S]]: return lambda v: v @@ -463,6 +529,18 @@ def can_transform(self, source: IcebergType) -> bool: def result_type(self, source: IcebergType) -> IcebergType: return source + def project(self, name: str, pred: BoundPredicate[L]) -> Optional[UnboundPredicate[Any]]: + if isinstance(pred.term, BoundTransform): + return _project_transform_predicate(self, name, pred) + elif isinstance(pred, BoundUnaryPredicate): + return pred.as_unbound(Reference(name)) + elif isinstance(pred, BoundEqualTo): + return pred.as_unbound(Reference(name), pred.literal) + elif isinstance(pred, (BoundIn, BoundNotIn)): + return pred.as_unbound(Reference(name), pred.literals) + else: + raise ValueError(f"Could not project: {self}") + @property def preserves_order(self) -> bool: return True @@ -511,6 +589,29 @@ def preserves_order(self) -> bool: def source_type(self) -> IcebergType: return self._source_type + def project(self, name: str, pred: BoundPredicate[L]) -> Optional[UnboundPredicate[Any]]: + field_type = pred.term.ref().field.field_type + + if isinstance(pred.term, BoundTransform): + return _project_transform_predicate(self, name, pred) + + # Implement startswith and notstartswith for string (and probably binary) + # https://github.com/apache/iceberg/issues/6112 + + if isinstance(pred, BoundUnaryPredicate): + return pred.as_unbound(Reference(name)) + elif isinstance(field_type, (IntegerType, LongType, DecimalType)): + if isinstance(pred, BoundLiteralPredicate): + return _truncate_number(name, pred, self.transform(field_type)) + elif isinstance(pred, BoundIn): + return _set_apply_transform(name, pred, self.transform(field_type)) + elif isinstance(field_type, (BinaryType, StringType)): + if isinstance(pred, BoundLiteralPredicate): + return _truncate_array(name, pred, self.transform(field_type)) + elif isinstance(pred, BoundIn): + return _set_apply_transform(name, pred, self.transform(field_type)) + return None + @property def width(self) -> int: return self._width @@ -610,7 +711,7 @@ class UnknownTransform(Transform[S, T]): AttributeError: If the apply method is called. """ - __root__: Literal["unknown"] = Field(default="unknown") + __root__: LiteralType["unknown"] = Field(default="unknown") # noqa: F821 _transform: str = PrivateAttr() def __init__(self, transform: str, **data: Any): @@ -623,9 +724,12 @@ def transform(self, source: IcebergType) -> Callable[[Optional[S]], Optional[T]] def can_transform(self, source: IcebergType) -> bool: return False - def result_type(self, source: IcebergType) -> IcebergType: + def result_type(self, source: IcebergType) -> StringType: return StringType() + def project(self, name: str, pred: BoundPredicate[L]) -> Optional[UnboundPredicate[Any]]: + return None + def __repr__(self) -> str: return f"UnknownTransform(transform={repr(self._transform)})" @@ -644,8 +748,86 @@ def can_transform(self, _: IcebergType) -> bool: def result_type(self, source: IcebergType) -> IcebergType: return source + def project(self, name: str, pred: BoundPredicate[L]) -> Optional[UnboundPredicate[Any]]: + return None + def to_human_string(self, _: IcebergType, value: Optional[S]) -> str: return "null" def __repr__(self) -> str: return "VoidTransform()" + + +def _truncate_number( + name: str, pred: BoundLiteralPredicate[L], transform: Callable[[Optional[L]], Optional[L]] +) -> Optional[UnboundPredicate[Any]]: + boundary = pred.literal + + if not isinstance(boundary, (LongLiteral, DecimalLiteral, DateLiteral, TimestampLiteral)): + raise ValueError(f"Expected a numeric literal, got: {type(boundary)}") + + if isinstance(pred, BoundLessThan): + return LessThanOrEqual(Reference(name), _transform_literal(transform, boundary.decrement())) # type: ignore + elif isinstance(pred, BoundLessThanOrEqual): + return LessThanOrEqual(Reference(name), _transform_literal(transform, boundary)) + elif isinstance(pred, BoundGreaterThan): + return GreaterThanOrEqual(Reference(name), _transform_literal(transform, boundary.increment())) # type: ignore + elif isinstance(pred, BoundGreaterThanOrEqual): + return GreaterThanOrEqual(Reference(name), _transform_literal(transform, boundary)) + elif isinstance(pred, BoundEqualTo): + return EqualTo(Reference(name), _transform_literal(transform, boundary)) + else: + return None + + +def _truncate_array( + name: str, pred: BoundLiteralPredicate[L], transform: Callable[[Optional[L]], Optional[L]] +) -> Optional[UnboundPredicate[Any]]: + boundary = pred.literal + + if type(pred) in {BoundLessThan, BoundLessThanOrEqual}: + return LessThanOrEqual(Reference(name), _transform_literal(transform, boundary)) + elif type(pred) in {BoundGreaterThan, BoundGreaterThanOrEqual}: + return GreaterThanOrEqual(Reference(name), _transform_literal(transform, boundary)) + if isinstance(pred, BoundEqualTo): + return EqualTo(Reference(name), _transform_literal(transform, boundary)) + else: + return None + + +def _project_transform_predicate( + transform: Transform[Any, Any], partition_name: str, pred: BoundPredicate[L] +) -> Optional[UnboundPredicate[Any]]: + term = pred.term + if isinstance(term, BoundTransform) and transform == term.transform: + return _remove_transform(partition_name, pred) + return None + + +def _remove_transform(partition_name: str, pred: BoundPredicate[L]): + if isinstance(pred, BoundUnaryPredicate): + return pred.as_unbound(Reference(partition_name)) + elif isinstance(pred, BoundLiteralPredicate): + return pred.as_unbound(Reference(partition_name), pred.literal) + elif isinstance(pred, (BoundIn, BoundNotIn)): + return pred.as_unbound(Reference(partition_name), pred.literals) + else: + raise ValueError(f"Cannot replace transform in unknown predicate: {pred}") + + +def _set_apply_transform(name: str, pred: BoundSetPredicate[L], transform: Callable[[L], L]) -> UnboundPredicate[Any]: + literals = pred.literals + if isinstance(pred, BoundSetPredicate): + return pred.as_unbound(Reference(name), {_transform_literal(transform, literal) for literal in literals}) + else: + raise ValueError(f"Unknown BoundSetPredicate: {pred}") + + +class BoundTransform(BoundTerm[L]): + """A transform expression""" + + transform: Transform[L, Any] + + def __init__(self, term: BoundTerm[L], transform: Transform[L, Any]): + self.term: BoundTerm[L] = term + self.transform = transform diff --git a/tests/expressions/test_literals.py b/tests/expressions/test_literals.py index d203195930..efcacc4574 100644 --- a/tests/expressions/test_literals.py +++ b/tests/expressions/test_literals.py @@ -805,6 +805,24 @@ def test_string_to_decimal_type_invalid_value(): assert "Could not convert 18.15 into a decimal(10, 0), scales differ 0 <> 2" in str(e.value) +def test_decimal_literal_increment(): + dec = DecimalLiteral(Decimal("10.123")) + # Twice to check that we don't mutate the value + assert dec.increment() == DecimalLiteral(Decimal("10.124")) + assert dec.increment() == DecimalLiteral(Decimal("10.124")) + # To check that the scale is still the same + assert dec.increment().value.as_tuple() == Decimal("10.124").as_tuple() + + +def test_decimal_literal_dencrement(): + dec = DecimalLiteral(Decimal("10.123")) + # Twice to check that we don't mutate the value + assert dec.decrement() == DecimalLiteral(Decimal("10.122")) + assert dec.decrement() == DecimalLiteral(Decimal("10.122")) + # To check that the scale is still the same + assert dec.decrement().value.as_tuple() == Decimal("10.122").as_tuple() + + # __ __ ___ # | \/ |_ _| _ \_ _ # | |\/| | || | _/ || | diff --git a/tests/test_transforms.py b/tests/test_transforms.py index f8bc42e2dc..9395db13c2 100644 --- a/tests/test_transforms.py +++ b/tests/test_transforms.py @@ -14,7 +14,7 @@ # KIND, either express or implied. See the License for the # specific language governing permissions and limitations # under the License. -# pylint: disable=eval-used,protected-access +# pylint: disable=eval-used,protected-access,redefined-outer-name from datetime import date from decimal import Decimal from typing import Any @@ -24,6 +24,31 @@ import pytest from pyiceberg import transforms +from pyiceberg.expressions import ( + BoundEqualTo, + BoundGreaterThan, + BoundGreaterThanOrEqual, + BoundIn, + BoundLessThan, + BoundLessThanOrEqual, + BoundNotIn, + BoundNotNull, + BoundReference, + EqualTo, + GreaterThanOrEqual, + In, + LessThanOrEqual, + NotIn, + NotNull, + Reference, +) +from pyiceberg.expressions.literals import ( + DateLiteral, + DecimalLiteral, + TimestampLiteral, + literal, +) +from pyiceberg.schema import Accessor from pyiceberg.transforms import ( BucketTransform, DayTransform, @@ -46,6 +71,7 @@ FloatType, IntegerType, LongType, + NestedField, StringType, TimestampType, TimestamptzType, @@ -507,3 +533,364 @@ def test_datetime_transform_str(transform, transform_str): ) def test_datetime_transform_repr(transform, transform_repr): assert repr(transform) == transform_repr + + +@pytest.fixture +def bound_reference_str() -> BoundReference[str]: + return BoundReference(field=NestedField(1, "field", StringType(), required=False), accessor=Accessor(position=0, inner=None)) + + +@pytest.fixture +def bound_reference_date() -> BoundReference[int]: + return BoundReference(field=NestedField(1, "field", DateType(), required=False), accessor=Accessor(position=0, inner=None)) + + +@pytest.fixture +def bound_reference_timestamp() -> BoundReference[int]: + return BoundReference( + field=NestedField(1, "field", TimestampType(), required=False), accessor=Accessor(position=0, inner=None) + ) + + +@pytest.fixture +def bound_reference_decimal() -> BoundReference[Decimal]: + return BoundReference( + field=NestedField(1, "field", DecimalType(8, 2), required=False), accessor=Accessor(position=0, inner=None) + ) + + +@pytest.fixture +def bound_reference_long() -> BoundReference[int]: + return BoundReference( + field=NestedField(1, "field", DecimalType(8, 2), required=False), accessor=Accessor(position=0, inner=None) + ) + + +def test_projection_bucket_unary(bound_reference_str: BoundReference[str]) -> None: + assert BucketTransform(2).project("name", BoundNotNull(term=bound_reference_str)) == NotNull(term=Reference(name="name")) + + +def test_projection_bucket_literal(bound_reference_str: BoundReference[str]) -> None: + assert BucketTransform(2).project("name", BoundEqualTo(term=bound_reference_str, literal=literal("data"))) == EqualTo( + term="name", literal=1 + ) + + +def test_projection_bucket_set_same_bucket(bound_reference_str: BoundReference[str]) -> None: + assert BucketTransform(2).project( + "name", BoundIn(term=bound_reference_str, literals={literal("hello"), literal("world")}) + ) == EqualTo(term="name", literal=1) + + +def test_projection_bucket_set_in(bound_reference_str: BoundReference[str]) -> None: + assert BucketTransform(3).project( + "name", BoundIn(term=bound_reference_str, literals={literal("hello"), literal("world")}) + ) == In(term="name", literals={1, 2}) + + +def test_projection_bucket_set_not_in(bound_reference_str: BoundReference[str]) -> None: + assert ( + BucketTransform(3).project("name", BoundNotIn(term=bound_reference_str, literals={literal("hello"), literal("world")})) + is None + ) + + +def test_projection_year_unary(bound_reference_date: BoundReference[int]) -> None: + assert YearTransform().project("name", BoundNotNull(term=bound_reference_date)) == NotNull(term="name") + + +def test_projection_year_literal(bound_reference_date: BoundReference[int]) -> None: + assert YearTransform().project("name", BoundEqualTo(term=bound_reference_date, literal=DateLiteral(1925))) == EqualTo( + term="name", literal=5 + ) + + +def test_projection_year_set_same_year(bound_reference_date: BoundReference[int]) -> None: + assert YearTransform().project( + "name", BoundIn(term=bound_reference_date, literals={DateLiteral(1925), DateLiteral(1926)}) + ) == EqualTo(term="name", literal=5) + + +def test_projection_year_set_in(bound_reference_date: BoundReference[int]) -> None: + assert YearTransform().project( + "name", BoundIn(term=bound_reference_date, literals={DateLiteral(1925), DateLiteral(2925)}) + ) == In(term="name", literals={8, 5}) + + +def test_projection_year_set_not_in(bound_reference_date: BoundReference[int]) -> None: + assert ( + YearTransform().project("name", BoundNotIn(term=bound_reference_date, literals={DateLiteral(1925), DateLiteral(2925)})) + is None + ) + + +def test_projection_month_unary(bound_reference_date: BoundReference[int]) -> None: + assert MonthTransform().project("name", BoundNotNull(term=bound_reference_date)) == NotNull(term="name") + + +def test_projection_month_literal(bound_reference_date: BoundReference[int]) -> None: + assert MonthTransform().project("name", BoundEqualTo(term=bound_reference_date, literal=DateLiteral(1925))) == EqualTo( + term="name", literal=63 + ) + + +def test_projection_month_set_same_month(bound_reference_date: BoundReference[int]) -> None: + assert MonthTransform().project( + "name", BoundIn(term=bound_reference_date, literals={DateLiteral(1925), DateLiteral(1926)}) + ) == EqualTo(term="name", literal=63) + + +def test_projection_month_set_in(bound_reference_date: BoundReference[int]) -> None: + assert MonthTransform().project( + "name", BoundIn(term=bound_reference_date, literals={DateLiteral(1925), DateLiteral(2925)}) + ) == In(term="name", literals={96, 63}) + + +def test_projection_day_month_not_in(bound_reference_date: BoundReference[int]) -> None: + assert ( + MonthTransform().project("name", BoundNotIn(term=bound_reference_date, literals={DateLiteral(1925), DateLiteral(2925)})) + is None + ) + + +def test_projection_day_unary(bound_reference_timestamp) -> None: + assert DayTransform().project("name", BoundNotNull(term=bound_reference_timestamp)) == NotNull(term="name") + + +def test_projection_day_literal(bound_reference_timestamp) -> None: + assert DayTransform().project( + "name", BoundEqualTo(term=bound_reference_timestamp, literal=TimestampLiteral(1667696874000)) + ) == EqualTo(term="name", literal=19) + + +def test_projection_day_set_same_day(bound_reference_timestamp) -> None: + assert DayTransform().project( + "name", + BoundIn(term=bound_reference_timestamp, literals={TimestampLiteral(1667696874001), TimestampLiteral(1667696874000)}), + ) == EqualTo(term="name", literal=19) + + +def test_projection_day_set_in(bound_reference_timestamp) -> None: + assert DayTransform().project( + "name", + BoundIn(term=bound_reference_timestamp, literals={TimestampLiteral(1667696874001), TimestampLiteral(1567696874000)}), + ) == In(term="name", literals={18, 19}) + + +def test_projection_day_set_not_in(bound_reference_timestamp) -> None: + assert ( + DayTransform().project( + "name", + BoundNotIn(term=bound_reference_timestamp, literals={TimestampLiteral(1567696874), TimestampLiteral(1667696874)}), + ) + is None + ) + + +def test_projection_day_human(bound_reference_date: BoundReference[int]) -> None: + date_literal = DateLiteral(17532) + assert DayTransform().project("dt", BoundEqualTo(term=bound_reference_date, literal=date_literal)) == EqualTo( + term="dt", literal=17532 + ) # == 2018, 1, 1 + + assert DayTransform().project("dt", BoundLessThanOrEqual(term=bound_reference_date, literal=date_literal)) == LessThanOrEqual( + term="dt", literal=17532 + ) # <= 2018, 1, 1 + + assert DayTransform().project("dt", BoundLessThan(term=bound_reference_date, literal=date_literal)) == LessThanOrEqual( + term="dt", literal=17531 + ) # <= 2017, 12, 31 + + assert DayTransform().project( + "dt", BoundGreaterThanOrEqual(term=bound_reference_date, literal=date_literal) + ) == GreaterThanOrEqual( + term="dt", literal=17532 + ) # >= 2018, 1, 1 + + assert DayTransform().project("dt", BoundGreaterThan(term=bound_reference_date, literal=date_literal)) == GreaterThanOrEqual( + term="dt", literal=17533 + ) # >= 2018, 1, 2 + + +def test_projection_hour_unary(bound_reference_timestamp) -> None: + assert HourTransform().project("name", BoundNotNull(term=bound_reference_timestamp)) == NotNull(term="name") + + +TIMESTAMP_EXAMPLE = 1667696874000000 # Sun Nov 06 2022 01:07:54 +HOUR_IN_MICROSECONDS = 60 * 60 * 1000 * 1000 + + +def test_projection_hour_literal(bound_reference_timestamp) -> None: + assert HourTransform().project( + "name", BoundEqualTo(term=bound_reference_timestamp, literal=TimestampLiteral(TIMESTAMP_EXAMPLE)) + ) == EqualTo(term="name", literal=463249) + + +def test_projection_hour_set_same_hour(bound_reference_timestamp) -> None: + assert HourTransform().project( + "name", + BoundIn( + term=bound_reference_timestamp, + literals={TimestampLiteral(TIMESTAMP_EXAMPLE + 1), TimestampLiteral(TIMESTAMP_EXAMPLE)}, + ), + ) == EqualTo(term="name", literal=463249) + + +def test_projection_hour_set_in(bound_reference_timestamp) -> None: + assert HourTransform().project( + "name", + BoundIn( + term=bound_reference_timestamp, + literals={TimestampLiteral(TIMESTAMP_EXAMPLE + HOUR_IN_MICROSECONDS), TimestampLiteral(TIMESTAMP_EXAMPLE)}, + ), + ) == In(term="name", literals={463249, 463250}) + + +def test_projection_hour_set_not_in(bound_reference_timestamp) -> None: + assert ( + HourTransform().project( + "name", + BoundNotIn( + term=bound_reference_timestamp, + literals={TimestampLiteral(TIMESTAMP_EXAMPLE + HOUR_IN_MICROSECONDS), TimestampLiteral(TIMESTAMP_EXAMPLE)}, + ), + ) + is None + ) + + +def test_projection_identity_unary(bound_reference_timestamp) -> None: + assert IdentityTransform().project("name", BoundNotNull(term=bound_reference_timestamp)) == NotNull(term="name") + + +def test_projection_identity_literal(bound_reference_timestamp) -> None: + assert IdentityTransform().project( + "name", BoundEqualTo(term=bound_reference_timestamp, literal=TimestampLiteral(TIMESTAMP_EXAMPLE)) + ) == EqualTo( + term="name", literal=TimestampLiteral(TIMESTAMP_EXAMPLE) # type: ignore + ) + + +def test_projection_identity_set_in(bound_reference_timestamp) -> None: + assert IdentityTransform().project( + "name", + BoundIn( + term=bound_reference_timestamp, + literals={TimestampLiteral(TIMESTAMP_EXAMPLE + HOUR_IN_MICROSECONDS), TimestampLiteral(TIMESTAMP_EXAMPLE)}, + ), + ) == In( + term="name", literals={TimestampLiteral(TIMESTAMP_EXAMPLE + HOUR_IN_MICROSECONDS), TimestampLiteral(TIMESTAMP_EXAMPLE)} # type: ignore + ) + + +def test_projection_identity_set_not_in(bound_reference_timestamp) -> None: + assert IdentityTransform().project( + "name", + BoundNotIn( + term=bound_reference_timestamp, + literals={TimestampLiteral(TIMESTAMP_EXAMPLE + HOUR_IN_MICROSECONDS), TimestampLiteral(TIMESTAMP_EXAMPLE)}, + ), + ) == NotIn( + term="name", literals={TimestampLiteral(TIMESTAMP_EXAMPLE + HOUR_IN_MICROSECONDS), TimestampLiteral(TIMESTAMP_EXAMPLE)} # type: ignore + ) + + +def test_projection_truncate_string_unary(bound_reference_str: BoundReference[str]) -> None: + assert TruncateTransform(2).project("name", BoundNotNull(term=bound_reference_str)) == NotNull(term="name") + + +def test_projection_truncate_string_literal_eq(bound_reference_str: BoundReference[str]) -> None: + assert TruncateTransform(2).project("name", BoundEqualTo(term=bound_reference_str, literal=literal("data"))) == EqualTo( + term="name", literal=literal("da") + ) + + +def test_projection_truncate_string_literal_gt(bound_reference_str: BoundReference[str]) -> None: + assert TruncateTransform(2).project("name", BoundGreaterThan(term=bound_reference_str, literal=literal("data"))) == EqualTo( + term="name", literal=literal("da") + ) + + +def test_projection_truncate_string_literal_gte(bound_reference_str: BoundReference[str]) -> None: + assert TruncateTransform(2).project( + "name", BoundGreaterThanOrEqual(term=bound_reference_str, literal=literal("data")) + ) == EqualTo(term="name", literal=literal("da")) + + +def test_projection_truncate_string_set_same_result(bound_reference_str: BoundReference[str]) -> None: + assert TruncateTransform(2).project( + "name", BoundIn(term=bound_reference_str, literals={literal("hello"), literal("helloworld")}) + ) == EqualTo(term="name", literal=literal("he")) + + +def test_projection_truncate_string_set_in(bound_reference_str: BoundReference[str]) -> None: + assert TruncateTransform(3).project( + "name", BoundIn(term=bound_reference_str, literals={literal("hello"), literal("world")}) + ) == In(term="name", literals={literal("hel"), literal("wor")}) + + +def test_projection_truncate_string_set_not_in(bound_reference_str: BoundReference[str]) -> None: + assert ( + TruncateTransform(3).project("name", BoundNotIn(term=bound_reference_str, literals={literal("hello"), literal("world")})) + is None + ) + + +def test_projection_truncate_decimal_literal_eq(bound_reference_decimal: BoundReference[Decimal]) -> None: + assert TruncateTransform(2).project( + "name", BoundEqualTo(term=bound_reference_decimal, literal=DecimalLiteral(Decimal(19.25))) + ) == EqualTo(term="name", literal=Decimal("19.24")) + + +def test_projection_truncate_decimal_literal_gt(bound_reference_decimal: BoundReference[Decimal]) -> None: + assert TruncateTransform(2).project( + "name", BoundGreaterThan(term=bound_reference_decimal, literal=DecimalLiteral(Decimal(19.25))) + ) == GreaterThanOrEqual(term="name", literal=Decimal("19.26")) + + +def test_projection_truncate_decimal_literal_gte(bound_reference_decimal: BoundReference[Decimal]) -> None: + assert TruncateTransform(2).project( + "name", BoundGreaterThanOrEqual(term=bound_reference_decimal, literal=DecimalLiteral(Decimal(19.25))) + ) == GreaterThanOrEqual(term="name", literal=Decimal("19.24")) + + +def test_projection_truncate_decimal_in(bound_reference_decimal: BoundReference[Decimal]) -> None: + assert TruncateTransform(2).project( + "name", BoundIn(term=bound_reference_decimal, literals={literal(Decimal(19.25)), literal(Decimal(18.15))}) + ) == In( + term="name", + literals={ + Decimal("19.24"), + Decimal("18.14999999999999857891452847979962825775146484374"), + }, + ) + + +def test_projection_truncate_long_literal_eq(bound_reference_decimal: BoundReference[Decimal]) -> None: + assert TruncateTransform(2).project( + "name", BoundEqualTo(term=bound_reference_decimal, literal=DecimalLiteral(Decimal(19.25))) + ) == EqualTo(term="name", literal=Decimal("19.24")) + + +def test_projection_truncate_long_literal_gt(bound_reference_decimal: BoundReference[Decimal]) -> None: + assert TruncateTransform(2).project( + "name", BoundGreaterThan(term=bound_reference_decimal, literal=DecimalLiteral(Decimal(19.25))) + ) == GreaterThanOrEqual(term="name", literal=Decimal("19.26")) + + +def test_projection_truncate_long_literal_gte(bound_reference_decimal: BoundReference[Decimal]) -> None: + assert TruncateTransform(2).project( + "name", BoundGreaterThanOrEqual(term=bound_reference_decimal, literal=DecimalLiteral(Decimal(19.25))) + ) == GreaterThanOrEqual(term="name", literal=Decimal("19.24")) + + +def test_projection_truncate_long_in(bound_reference_decimal: BoundReference[Decimal]) -> None: + assert TruncateTransform(2).project( + "name", BoundIn(term=bound_reference_decimal, literals={DecimalLiteral(Decimal(19.25)), DecimalLiteral(Decimal(18.15))}) + ) == In( + term="name", + literals={ + Decimal("19.24"), + Decimal("18.14999999999999857891452847979962825775146484374"), + }, + ) From 05cf1f6e3c980fa6d213eeedbab83f0d5826ac06 Mon Sep 17 00:00:00 2001 From: Fokko Driesprong Date: Wed, 23 Nov 2022 20:21:30 +0100 Subject: [PATCH 277/642] Python: Pass warehouse location to config (#6260) --- pyiceberg/catalog/__init__.py | 2 +- pyiceberg/catalog/glue.py | 4 ++-- pyiceberg/catalog/rest.py | 7 ++++++- tests/catalog/test_rest.py | 27 +++++++++++++++++++++++++++ 4 files changed, 36 insertions(+), 4 deletions(-) diff --git a/pyiceberg/catalog/__init__.py b/pyiceberg/catalog/__init__.py index 52095d6f4c..64308260be 100644 --- a/pyiceberg/catalog/__init__.py +++ b/pyiceberg/catalog/__init__.py @@ -53,7 +53,7 @@ TYPE = "type" ICEBERG = "iceberg" TABLE_TYPE = "table_type" -WAREHOUSE = "warehouse" +WAREHOUSE_LOCATION = "warehouse" METADATA_LOCATION = "metadata_location" MANIFEST = "manifest" MANIFEST_LIST = "manifest list" diff --git a/pyiceberg/catalog/glue.py b/pyiceberg/catalog/glue.py index 4396e0423a..273a8c613d 100644 --- a/pyiceberg/catalog/glue.py +++ b/pyiceberg/catalog/glue.py @@ -38,7 +38,7 @@ METADATA_LOCATION, PREVIOUS_METADATA, TABLE_TYPE, - WAREHOUSE, + WAREHOUSE_LOCATION, Catalog, Identifier, Properties, @@ -191,7 +191,7 @@ def _default_warehouse_location(self, database_name: str, table_name: str) -> st database_location = database_location.rstrip("/") return f"{database_location}/{table_name}" - if warehouse_path := self.properties.get(WAREHOUSE): + if warehouse_path := self.properties.get(WAREHOUSE_LOCATION): warehouse_path = warehouse_path.rstrip("/") return f"{warehouse_path}/{database_name}.db/{table_name}" diff --git a/pyiceberg/catalog/rest.py b/pyiceberg/catalog/rest.py index e453440c57..71a92b532f 100644 --- a/pyiceberg/catalog/rest.py +++ b/pyiceberg/catalog/rest.py @@ -32,6 +32,7 @@ from pyiceberg.catalog import ( TOKEN, URI, + WAREHOUSE_LOCATION, Catalog, Identifier, Properties, @@ -263,7 +264,11 @@ def _fetch_access_token(self, credential: str) -> str: return TokenResponse(**response.json()).access_token def _fetch_config(self, properties: Properties) -> Properties: - response = self.session.get(self.url(Endpoints.get_config, prefixed=False)) + params = {} + if warehouse_location := properties.get(WAREHOUSE_LOCATION): + params[WAREHOUSE_LOCATION] = warehouse_location + + response = self.session.get(self.url(Endpoints.get_config, prefixed=False), params=params) try: response.raise_for_status() except HTTPError as exc: diff --git a/tests/catalog/test_rest.py b/tests/catalog/test_rest.py index 45df4fcba1..cf840713c2 100644 --- a/tests/catalog/test_rest.py +++ b/tests/catalog/test_rest.py @@ -95,6 +95,33 @@ def test_token_200(rest_mock: Mocker): ) +def test_config_200(requests_mock: Mocker): + requests_mock.get( + f"{TEST_URI}v1/config", + json={"defaults": {}, "overrides": {}}, + status_code=200, + ) + requests_mock.post( + f"{TEST_URI}v1/oauth/tokens", + json={ + "access_token": TEST_TOKEN, + "token_type": "Bearer", + "expires_in": 86400, + "issued_token_type": "urn:ietf:params:oauth:token-type:access_token", + }, + status_code=200, + request_headers=OAUTH_TEST_HEADERS, + ) + RestCatalog("rest", uri=TEST_URI, credential=TEST_CREDENTIALS, warehouse="s3://some-bucket") + + assert requests_mock.called + assert requests_mock.call_count == 2 + + history = requests_mock.request_history + assert history[1].method == "GET" + assert history[1].url == "https://iceberg-test-catalog/v1/config?warehouse=s3%3A%2F%2Fsome-bucket" + + def test_token_400(rest_mock: Mocker): rest_mock.post( f"{TEST_URI}v1/oauth/tokens", From 8ff80a919fb82ccb9eeb0a18a98619a2ec77aeb5 Mon Sep 17 00:00:00 2001 From: Rushan Jiang Date: Sun, 27 Nov 2022 11:59:11 -0500 Subject: [PATCH 278/642] Python: Refactor catalog constants (#6277) --- pyiceberg/catalog/hive.py | 15 +++++++++------ 1 file changed, 9 insertions(+), 6 deletions(-) diff --git a/pyiceberg/catalog/hive.py b/pyiceberg/catalog/hive.py index 3092be1ecb..12601f103f 100644 --- a/pyiceberg/catalog/hive.py +++ b/pyiceberg/catalog/hive.py @@ -45,6 +45,10 @@ from thrift.transport import TSocket, TTransport from pyiceberg.catalog import ( + ICEBERG, + METADATA_LOCATION, + TABLE_TYPE, + WAREHOUSE_LOCATION, Catalog, Identifier, Properties, @@ -53,6 +57,7 @@ from pyiceberg.exceptions import ( NamespaceAlreadyExistsError, NamespaceNotEmptyError, + NoSuchIcebergTableError, NoSuchNamespaceError, NoSuchTableError, TableAlreadyExistsError, @@ -104,11 +109,7 @@ COMMENT = "comment" OWNER = "owner" -TABLE_TYPE = "table_type" -METADATA_LOCATION = "metadata_location" -ICEBERG = "iceberg" LOCATION = "location" -WAREHOUSE = "warehouse" class _HiveClient: @@ -246,7 +247,9 @@ def _convert_hive_into_iceberg(self, table: HiveTable, io: FileIO) -> Table: table_type = properties[TABLE_TYPE] if table_type.lower() != ICEBERG: - raise NoSuchTableError(f"Property table_type is {table_type}, expected {ICEBERG}: {table.dbName}.{table.tableName}") + raise NoSuchIcebergTableError( + f"Property table_type is {table_type}, expected {ICEBERG}: {table.dbName}.{table.tableName}" + ) if prop_metadata_location := properties.get(METADATA_LOCATION): metadata_location = prop_metadata_location @@ -272,7 +275,7 @@ def _resolve_table_location(self, location: Optional[str], database_name: str, t database_location = database_location.rstrip("/") return f"{database_location}/{table_name}" - if warehouse_location := self.properties.get(WAREHOUSE): + if warehouse_location := self.properties.get(WAREHOUSE_LOCATION): warehouse_location = warehouse_location.rstrip("/") return f"{warehouse_location}/{database_name}/{table_name}" raise ValueError("Cannot determine location from warehouse, please provide an explicit location") From dee81adb662c65d511770f6bc43b18b54b54cd5c Mon Sep 17 00:00:00 2001 From: Ryan Blue Date: Sun, 27 Nov 2022 23:19:24 -0800 Subject: [PATCH 279/642] Python: Fix Avro scan performance in PyArrow (#6283) * Python: Fix Avro scan performance. * Python: Fix EOF handling. --- pyiceberg/avro/decoder.py | 5 ++++- pyiceberg/avro/file.py | 20 ++++++++------------ 2 files changed, 12 insertions(+), 13 deletions(-) diff --git a/pyiceberg/avro/decoder.py b/pyiceberg/avro/decoder.py index 776c8bae31..fd39a608ef 100644 --- a/pyiceberg/avro/decoder.py +++ b/pyiceberg/avro/decoder.py @@ -53,7 +53,10 @@ def read(self, n: int) -> bytes: if n < 0: raise ValueError(f"Requested {n} bytes to read, expected positive integer.") read_bytes = self._input_stream.read(n) - if len(read_bytes) != n: + read_len = len(read_bytes) + if read_len <= 0: + raise EOFError + elif read_len != n: raise ValueError(f"Read {len(read_bytes)} bytes, expected {n} bytes") return read_bytes diff --git a/pyiceberg/avro/file.py b/pyiceberg/avro/file.py index f17056640f..5c1d3c63f0 100644 --- a/pyiceberg/avro/file.py +++ b/pyiceberg/avro/file.py @@ -22,8 +22,8 @@ import json from dataclasses import dataclass -from io import SEEK_SET -from typing import Optional, Type +from io import SEEK_SET, BufferedReader +from typing import Optional, Type, cast from pyiceberg.avro.codecs import KNOWN_CODECS, Codec from pyiceberg.avro.decoder import BinaryDecoder @@ -113,7 +113,6 @@ class AvroFile: input_stream: InputStream header: AvroFileHeader schema: Schema - file_length: int reader: StructReader decoder: BinaryDecoder @@ -131,15 +130,14 @@ def __enter__(self): Returns: A generator returning the AvroStructs """ - self.input_stream = self.input_file.open() + self.input_stream = BufferedReader(self.input_file.open()) self.decoder = BinaryDecoder(self.input_stream) self.header = self._read_header() self.schema = self.header.get_schema() - self.file_length = len(self.input_file) if not self.read_schema: self.reader = visit(self.schema, ConstructReader()) else: - self.reader = resolve(self.schema, self.read_schema) + self.reader = cast(StructReader, resolve(self.schema, self.read_schema)) return self @@ -155,8 +153,6 @@ def _read_block(self) -> int: sync_marker = self.decoder.read(SYNC_SIZE) if sync_marker != self.header.sync: raise ValueError(f"Expected sync bytes {self.header.sync!r}, but got {sync_marker!r}") - if self.is_EOF(): - raise StopIteration block_records = self.decoder.read_int() block_bytes_len = self.decoder.read_int() @@ -173,7 +169,10 @@ def __next__(self) -> AvroStruct: if self.block and self.block.has_next(): return next(self.block) - new_block = self._read_block() + try: + new_block = self._read_block() + except EOFError as exc: + raise StopIteration from exc if new_block > 0: return self.__next__() @@ -184,6 +183,3 @@ def _read_header(self) -> AvroFileHeader: reader = visit(META_SCHEMA, ConstructReader()) _header = reader.read(self.decoder) return AvroFileHeader(magic=_header.get(0), meta=_header.get(1), sync=_header.get(2)) - - def is_EOF(self) -> bool: - return self.input_stream.tell() == self.file_length From 0f96bddb05883ce0021d4cf171aa863a1e78bd7c Mon Sep 17 00:00:00 2001 From: Fokko Driesprong Date: Mon, 28 Nov 2022 17:50:24 +0100 Subject: [PATCH 280/642] Python: Implement inclusive projection (#6247) --- pyiceberg/expressions/visitors.py | 64 +++++ pyiceberg/partitioning.py | 2 +- pyiceberg/table/__init__.py | 29 +- pyiceberg/transforms.py | 10 +- tests/expressions/test_projection.py | 378 +++++++++++++++++++++++++++ tests/table/test_init.py | 15 +- tests/table/test_partitioning.py | 2 + 7 files changed, 465 insertions(+), 35 deletions(-) create mode 100644 tests/expressions/test_projection.py diff --git a/pyiceberg/expressions/visitors.py b/pyiceberg/expressions/visitors.py index 23b96f71c4..9aa841097e 100644 --- a/pyiceberg/expressions/visitors.py +++ b/pyiceberg/expressions/visitors.py @@ -17,6 +17,7 @@ from abc import ABC, abstractmethod from functools import singledispatch from typing import ( + Any, Callable, Generic, List, @@ -691,3 +692,66 @@ def manifest_evaluator( partition_schema = Schema(*partition_type.fields) evaluator = _ManifestEvalVisitor(partition_schema, partition_filter, case_sensitive) return evaluator.eval + + +class ProjectionEvaluator(BooleanExpressionVisitor[BooleanExpression], ABC): + schema: Schema + spec: PartitionSpec + case_sensitive: bool + + def __init__(self, schema: Schema, spec: PartitionSpec, case_sensitive: bool): + self.schema = schema + self.spec = spec + self.case_sensitive = case_sensitive + + def project(self, expr: BooleanExpression) -> BooleanExpression: + # projections assume that there are no NOT nodes in the expression tree. to ensure that this + # is the case, the expression is rewritten to push all NOT nodes down to the expression + # leaf nodes. + # this is necessary to ensure that the default expression returned when a predicate can't be + # projected is correct. + return visit(bind(self.schema, rewrite_not(expr), self.case_sensitive), self) + + def visit_true(self) -> BooleanExpression: + return AlwaysTrue() + + def visit_false(self) -> BooleanExpression: + return AlwaysFalse() + + def visit_not(self, child_result: BooleanExpression) -> BooleanExpression: + raise ValueError(f"Cannot project not expression, should be rewritten: {child_result}") + + def visit_and(self, left_result: BooleanExpression, right_result: BooleanExpression) -> BooleanExpression: + return And(left_result, right_result) + + def visit_or(self, left_result: BooleanExpression, right_result: BooleanExpression) -> BooleanExpression: + return Or(left_result, right_result) + + def visit_unbound_predicate(self, predicate: UnboundPredicate[L]) -> BooleanExpression: + raise ValueError(f"Cannot project unbound predicate: {predicate}") + + +class InclusiveProjection(ProjectionEvaluator): + def visit_bound_predicate(self, predicate: BoundPredicate[Any]) -> BooleanExpression: + parts = self.spec.fields_by_source_id(predicate.term.ref().field.field_id) + + result: BooleanExpression = AlwaysTrue() + for part in parts: + # consider (d = 2019-01-01) with bucket(7, d) and bucket(5, d) + # projections: b1 = bucket(7, '2019-01-01') = 5, b2 = bucket(5, '2019-01-01') = 0 + # any value where b1 != 5 or any value where b2 != 0 cannot be the '2019-01-01' + # + # similarly, if partitioning by day(ts) and hour(ts), the more restrictive + # projection should be used. ts = 2019-01-01T01:00:00 produces day=2019-01-01 and + # hour=2019-01-01-01. the value will be in 2019-01-01-01 and not in 2019-01-01-02. + incl_projection = part.transform.project(name=part.name, pred=predicate) + if incl_projection is not None: + result = And(result, incl_projection) + + return result + + +def inclusive_projection( + schema: Schema, spec: PartitionSpec, case_sensitive: bool = True +) -> Callable[[BooleanExpression], BooleanExpression]: + return InclusiveProjection(schema, spec, case_sensitive).project diff --git a/pyiceberg/partitioning.py b/pyiceberg/partitioning.py index 7ec9e3089b..b5dd427197 100644 --- a/pyiceberg/partitioning.py +++ b/pyiceberg/partitioning.py @@ -141,7 +141,7 @@ def source_id_to_fields_map(self) -> Dict[int, List[PartitionField]]: return source_id_to_fields_map def fields_by_source_id(self, field_id: int) -> List[PartitionField]: - return self.source_id_to_fields_map[field_id] + return self.source_id_to_fields_map.get(field_id, []) def compatible_with(self, other: "PartitionSpec") -> bool: """ diff --git a/pyiceberg/table/__init__.py b/pyiceberg/table/__init__.py index 369d02f368..dcc14c03c8 100644 --- a/pyiceberg/table/__init__.py +++ b/pyiceberg/table/__init__.py @@ -18,6 +18,7 @@ from abc import ABC, abstractmethod from dataclasses import dataclass +from functools import cached_property from typing import ( Any, Callable, @@ -38,6 +39,7 @@ BooleanExpression, visitors, ) +from pyiceberg.expressions.visitors import inclusive_projection from pyiceberg.io import FileIO from pyiceberg.manifest import DataFile, ManifestFile, files from pyiceberg.partitioning import PartitionSpec @@ -78,7 +80,6 @@ def name(self) -> Identifier: def scan( self, row_filter: Optional[BooleanExpression] = None, - partition_filter: Optional[BooleanExpression] = None, selected_fields: Tuple[str] = ("*",), case_sensitive: bool = True, snapshot_id: Optional[int] = None, @@ -87,7 +88,6 @@ def scan( return DataScan( table=self, row_filter=row_filter or AlwaysTrue(), - partition_filter=partition_filter or AlwaysTrue(), selected_fields=selected_fields, case_sensitive=case_sensitive, snapshot_id=snapshot_id, @@ -163,7 +163,6 @@ def __eq__(self, other: Any) -> bool: class TableScan(Generic[S], ABC): table: Table row_filter: BooleanExpression - partition_filter: BooleanExpression selected_fields: Tuple[str] case_sensitive: bool snapshot_id: Optional[int] @@ -173,7 +172,6 @@ def __init__( self, table: Table, row_filter: Optional[BooleanExpression] = None, - partition_filter: Optional[BooleanExpression] = None, selected_fields: Tuple[str] = ("*",), case_sensitive: bool = True, snapshot_id: Optional[int] = None, @@ -181,7 +179,6 @@ def __init__( ): self.table = table self.row_filter = row_filter or AlwaysTrue() - self.partition_filter = partition_filter or AlwaysTrue() self.selected_fields = selected_fields self.case_sensitive = case_sensitive self.snapshot_id = snapshot_id @@ -228,12 +225,9 @@ def select(self, *field_names: str) -> S: return self.update(selected_fields=field_names) return self.update(selected_fields=tuple(set(self.selected_fields).intersection(set(field_names)))) - def filter_rows(self, new_row_filter: BooleanExpression) -> S: + def filter(self, new_row_filter: BooleanExpression) -> S: return self.update(row_filter=And(self.row_filter, new_row_filter)) - def filter_partitions(self, new_partition_filter: BooleanExpression) -> S: - return self.update(partition_filter=And(self.partition_filter, new_partition_filter)) - def with_case_sensitive(self, case_sensitive: bool = True) -> S: return self.update(case_sensitive=case_sensitive) @@ -277,25 +271,30 @@ def __init__( self, table: Table, row_filter: Optional[BooleanExpression] = None, - partition_filter: Optional[BooleanExpression] = None, selected_fields: Tuple[str] = ("*",), case_sensitive: bool = True, snapshot_id: Optional[int] = None, options: Properties = EMPTY_DICT, ): - super().__init__(table, row_filter, partition_filter, selected_fields, case_sensitive, snapshot_id, options) + super().__init__(table, row_filter, selected_fields, case_sensitive, snapshot_id, options) + + def _build_partition_projection(self, spec_id: int) -> BooleanExpression: + project = inclusive_projection(self.table.schema(), self.table.specs()[spec_id]) + return project(self.row_filter) + + @cached_property + def partition_filters(self) -> KeyDefaultDict[int, BooleanExpression]: + return KeyDefaultDict(self._build_partition_projection) def _build_manifest_evaluator(self, spec_id: int) -> Callable[[ManifestFile], bool]: spec = self.table.specs()[spec_id] - return visitors.manifest_evaluator(spec, self.table.schema(), self.partition_filter, self.case_sensitive) + return visitors.manifest_evaluator(spec, self.table.schema(), self.partition_filters[spec_id], self.case_sensitive) def _build_partition_evaluator(self, spec_id: int) -> Callable[[DataFile], bool]: spec = self.table.specs()[spec_id] partition_type = spec.partition_type(self.table.schema()) partition_schema = Schema(*partition_type.fields) - - # TODO: project the row filter # pylint: disable=W0511 - partition_expr = And(self.partition_filter, AlwaysTrue()) + partition_expr = self.partition_filters[spec_id] # TODO: remove the dict to struct wrapper by using a StructProtocol record # pylint: disable=W0511 wrapper = _DictAsStruct(partition_type) diff --git a/pyiceberg/transforms.py b/pyiceberg/transforms.py index 79f901913c..0c87c8ac27 100644 --- a/pyiceberg/transforms.py +++ b/pyiceberg/transforms.py @@ -534,12 +534,12 @@ def project(self, name: str, pred: BoundPredicate[L]) -> Optional[UnboundPredica return _project_transform_predicate(self, name, pred) elif isinstance(pred, BoundUnaryPredicate): return pred.as_unbound(Reference(name)) - elif isinstance(pred, BoundEqualTo): + elif isinstance(pred, BoundLiteralPredicate): return pred.as_unbound(Reference(name), pred.literal) elif isinstance(pred, (BoundIn, BoundNotIn)): return pred.as_unbound(Reference(name), pred.literals) else: - raise ValueError(f"Could not project: {self}") + raise ValueError(f"Could not project: {pred}") @property def preserves_order(self) -> bool: @@ -600,16 +600,14 @@ def project(self, name: str, pred: BoundPredicate[L]) -> Optional[UnboundPredica if isinstance(pred, BoundUnaryPredicate): return pred.as_unbound(Reference(name)) + elif isinstance(pred, BoundIn): + return _set_apply_transform(name, pred, self.transform(field_type)) elif isinstance(field_type, (IntegerType, LongType, DecimalType)): if isinstance(pred, BoundLiteralPredicate): return _truncate_number(name, pred, self.transform(field_type)) - elif isinstance(pred, BoundIn): - return _set_apply_transform(name, pred, self.transform(field_type)) elif isinstance(field_type, (BinaryType, StringType)): if isinstance(pred, BoundLiteralPredicate): return _truncate_array(name, pred, self.transform(field_type)) - elif isinstance(pred, BoundIn): - return _set_apply_transform(name, pred, self.transform(field_type)) return None @property diff --git a/tests/expressions/test_projection.py b/tests/expressions/test_projection.py new file mode 100644 index 0000000000..73940f649d --- /dev/null +++ b/tests/expressions/test_projection.py @@ -0,0 +1,378 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# pylint:disable=redefined-outer-name + +import pytest + +from pyiceberg.expressions import ( + AlwaysTrue, + And, + EqualTo, + GreaterThan, + GreaterThanOrEqual, + In, + IsNull, + LessThan, + LessThanOrEqual, + Not, + NotEqualTo, + NotIn, + NotNull, + Or, +) +from pyiceberg.expressions.visitors import inclusive_projection +from pyiceberg.partitioning import PartitionField, PartitionSpec +from pyiceberg.schema import Schema +from pyiceberg.transforms import ( + BucketTransform, + DayTransform, + HourTransform, + IdentityTransform, + TruncateTransform, +) +from pyiceberg.types import ( + DateType, + LongType, + NestedField, + StringType, + TimestampType, +) + + +@pytest.fixture +def schema() -> Schema: + return Schema( + NestedField(1, "id", LongType(), required=False), + NestedField(2, "data", StringType(), required=False), + NestedField(3, "event_date", DateType(), required=False), + NestedField(4, "event_ts", TimestampType(), required=False), + ) + + +@pytest.fixture +def empty_spec() -> PartitionSpec: + return PartitionSpec() + + +@pytest.fixture +def id_spec() -> PartitionSpec: + return PartitionSpec(PartitionField(1, 1000, IdentityTransform(), "id_part")) + + +@pytest.fixture +def bucket_spec() -> PartitionSpec: + return PartitionSpec(PartitionField(2, 1000, BucketTransform(16), "data_bucket")) + + +@pytest.fixture +def day_spec() -> PartitionSpec: + return PartitionSpec(PartitionField(4, 1000, DayTransform(), "date"), PartitionField(3, 1000, DayTransform(), "ddate")) + + +@pytest.fixture +def hour_spec() -> PartitionSpec: + return PartitionSpec(PartitionField(4, 1000, HourTransform(), "hour")) + + +@pytest.fixture +def truncate_str_spec() -> PartitionSpec: + return PartitionSpec(PartitionField(2, 1000, TruncateTransform(2), "data_trunc")) + + +@pytest.fixture +def truncate_int_spec() -> PartitionSpec: + return PartitionSpec(PartitionField(1, 1000, TruncateTransform(10), "id_trunc")) + + +@pytest.fixture +def id_and_bucket_spec() -> PartitionSpec: + return PartitionSpec( + PartitionField(1, 1000, IdentityTransform(), "id_part"), PartitionField(2, 1001, BucketTransform(16), "data_bucket") + ) + + +def test_identity_projection(schema: Schema, id_spec: PartitionSpec): + predicates = [ + NotNull("id"), + IsNull("id"), + LessThan("id", 100), + LessThanOrEqual("id", 101), + GreaterThan("id", 102), + GreaterThanOrEqual("id", 103), + EqualTo("id", 104), + NotEqualTo("id", 105), + In("id", {3, 4, 5}), + NotIn("id", {3, 4, 5}), + ] + + expected = [ + NotNull("id_part"), + IsNull("id_part"), + LessThan("id_part", 100), + LessThanOrEqual("id_part", 101), + GreaterThan("id_part", 102), + GreaterThanOrEqual("id_part", 103), + EqualTo("id_part", 104), + NotEqualTo("id_part", 105), + In("id_part", {3, 4, 5}), + NotIn("id_part", {3, 4, 5}), + ] + + project = inclusive_projection(schema, id_spec) + for index, predicate in enumerate(predicates): + expr = project(predicate) + assert expected[index] == expr + + +def test_bucket_projection(schema: Schema, bucket_spec: PartitionSpec): + predicates = [ + NotNull("data"), + IsNull("data"), + LessThan("data", "val"), + LessThanOrEqual("data", "val"), + GreaterThan("data", "val"), + GreaterThanOrEqual("data", "val"), + EqualTo("data", "val"), + NotEqualTo("data", "val"), + In("data", {"v1", "v2", "v3"}), + NotIn("data", {"v1", "v2", "v3"}), + ] + + expected = [ + NotNull("data_bucket"), + IsNull("data_bucket"), + AlwaysTrue(), + AlwaysTrue(), + AlwaysTrue(), + AlwaysTrue(), + EqualTo("data_bucket", 14), + AlwaysTrue(), + In("data_bucket", {1, 3, 13}), + AlwaysTrue(), + ] + + project = inclusive_projection(schema, bucket_spec) + for index, predicate in enumerate(predicates): + expr = project(predicate) + assert expected[index] == expr + + +def test_hour_projection(schema: Schema, hour_spec: PartitionSpec): + predicates = [ + NotNull("event_ts"), + IsNull("event_ts"), + LessThan("event_ts", "2022-11-27T10:00:00"), + LessThanOrEqual("event_ts", "2022-11-27T10:00:00"), + GreaterThan("event_ts", "2022-11-27T09:59:59.999999"), + GreaterThanOrEqual("event_ts", "2022-11-27T09:59:59.999999"), + EqualTo("event_ts", "2022-11-27T10:00:00"), + NotEqualTo("event_ts", "2022-11-27T10:00:00"), + In("event_ts", {"2022-11-27T10:00:00", "2022-11-27T09:59:59.999999"}), + NotIn("event_ts", {"2022-11-27T10:00:00", "2022-11-27T09:59:59.999999"}), + ] + + expected = [ + NotNull("hour"), + IsNull("hour"), + LessThanOrEqual("hour", 463761), + LessThanOrEqual("hour", 463762), + GreaterThanOrEqual("hour", 463762), + GreaterThanOrEqual("hour", 463761), + EqualTo("hour", 463762), + AlwaysTrue(), + In("hour", {463761, 463762}), + AlwaysTrue(), + ] + + project = inclusive_projection(schema, hour_spec) + for index, predicate in enumerate(predicates): + expr = project(predicate) + assert expected[index] == expr, predicate + + +def test_day_projection(schema: Schema, day_spec: PartitionSpec): + predicates = [ + NotNull("event_ts"), + IsNull("event_ts"), + LessThan("event_ts", "2022-11-27T00:00:00"), + LessThanOrEqual("event_ts", "2022-11-27T00:00:00"), + GreaterThan("event_ts", "2022-11-26T23:59:59.999999"), + GreaterThanOrEqual("event_ts", "2022-11-26T23:59:59.999999"), + EqualTo("event_ts", "2022-11-27T10:00:00"), + NotEqualTo("event_ts", "2022-11-27T10:00:00"), + In("event_ts", {"2022-11-27T00:00:00", "2022-11-26T23:59:59.999999"}), + NotIn("event_ts", {"2022-11-27T00:00:00", "2022-11-26T23:59:59.999999"}), + ] + + expected = [ + NotNull("date"), + IsNull("date"), + LessThanOrEqual("date", 19322), + LessThanOrEqual("date", 19323), + GreaterThanOrEqual("date", 19323), + GreaterThanOrEqual("date", 19322), + EqualTo("date", 19323), + AlwaysTrue(), + In("date", {19322, 19323}), + AlwaysTrue(), + ] + + project = inclusive_projection(schema, day_spec) + for index, predicate in enumerate(predicates): + expr = project(predicate) + assert expected[index] == expr, predicate + + +def test_date_day_projection(schema: Schema, day_spec: PartitionSpec): + predicates = [ + NotNull("event_date"), + IsNull("event_date"), + LessThan("event_date", "2022-11-27"), + LessThanOrEqual("event_date", "2022-11-27"), + GreaterThan("event_date", "2022-11-26"), + GreaterThanOrEqual("event_date", "2022-11-26"), + EqualTo("event_date", "2022-11-27"), + NotEqualTo("event_date", "2022-11-27"), + In("event_date", {"2022-11-26", "2022-11-27"}), + NotIn("event_date", {"2022-11-26", "2022-11-27"}), + ] + + expected = [ + NotNull("ddate"), + IsNull("ddate"), + LessThanOrEqual("ddate", 19322), + LessThanOrEqual("ddate", 19323), + GreaterThanOrEqual("ddate", 19323), + GreaterThanOrEqual("ddate", 19322), + EqualTo("ddate", 19323), + AlwaysTrue(), + In("ddate", {19322, 19323}), + AlwaysTrue(), + ] + + project = inclusive_projection(schema, day_spec) + for index, predicate in enumerate(predicates): + expr = project(predicate) + assert expected[index] == expr, predicate + + +def test_string_truncate_projection(schema: Schema, truncate_str_spec: PartitionSpec): + predicates = [ + NotNull("data"), + IsNull("data"), + LessThan("data", "aaa"), + LessThanOrEqual("data", "aaa"), + GreaterThan("data", "aaa"), + GreaterThanOrEqual("data", "aaa"), + EqualTo("data", "aaa"), + NotEqualTo("data", "aaa"), + In("data", {"aaa", "aab"}), + NotIn("data", {"aaa", "aab"}), + ] + + expected = [ + NotNull("data_trunc"), + IsNull("data_trunc"), + LessThanOrEqual("data_trunc", "aa"), + LessThanOrEqual("data_trunc", "aa"), + GreaterThanOrEqual("data_trunc", "aa"), + GreaterThanOrEqual("data_trunc", "aa"), + EqualTo("data_trunc", "aa"), + AlwaysTrue(), + EqualTo("data_trunc", "aa"), + AlwaysTrue(), + ] + + project = inclusive_projection(schema, truncate_str_spec) + for index, predicate in enumerate(predicates): + expr = project(predicate) + assert expected[index] == expr, predicate + + +def test_int_truncate_projection(schema: Schema, truncate_int_spec: PartitionSpec): + predicates = [ + NotNull("id"), + IsNull("id"), + LessThan("id", 10), + LessThanOrEqual("id", 10), + GreaterThan("id", 9), + GreaterThanOrEqual("id", 10), + EqualTo("id", 15), + NotEqualTo("id", 15), + In("id", {15, 16}), + NotIn("id", {15, 16}), + ] + + expected = [ + NotNull("id_trunc"), + IsNull("id_trunc"), + LessThanOrEqual("id_trunc", 0), + LessThanOrEqual("id_trunc", 10), + GreaterThanOrEqual("id_trunc", 10), + GreaterThanOrEqual("id_trunc", 10), + EqualTo("id_trunc", 10), + AlwaysTrue(), + EqualTo("id_trunc", 10), + AlwaysTrue(), + ] + + project = inclusive_projection(schema, truncate_int_spec) + for index, predicate in enumerate(predicates): + expr = project(predicate) + assert expected[index] == expr, predicate + + +def test_projection_case_sensitive(schema: Schema, id_spec: PartitionSpec): + project = inclusive_projection(schema, id_spec) + with pytest.raises(ValueError) as exc_info: + project(NotNull("ID")) + assert str(exc_info) == "Could not find field with name ID, case_sensitive=True" + + +def test_projection_case_insensitive(schema: Schema, id_spec: PartitionSpec): + project = inclusive_projection(schema, id_spec, case_sensitive=False) + assert NotNull("id_part") == project(NotNull("ID")) + + +def test_projection_empty_spec(schema: Schema, empty_spec: PartitionSpec): + project = inclusive_projection(schema, empty_spec) + assert AlwaysTrue() == project(And(LessThan("id", 5), NotNull("data"))) + + +def test_and_projection_multiple_projected_fields(schema: Schema, id_and_bucket_spec: PartitionSpec): + project = inclusive_projection(schema, id_and_bucket_spec) + assert project(And(LessThan("id", 5), In("data", {"a", "b", "c"}))) == And( + LessThan("id_part", 5), In("data_bucket", {2, 3, 15}) + ) + + +def test_or_projection_multiple_projected_fields(schema: Schema, id_and_bucket_spec: PartitionSpec): + project = inclusive_projection(schema, id_and_bucket_spec) + assert project(Or(LessThan("id", 5), In("data", {"a", "b", "c"}))) == Or( + LessThan("id_part", 5), In("data_bucket", {2, 3, 15}) + ) + + +def test_not_projection_multiple_projected_fields(schema: Schema, id_and_bucket_spec: PartitionSpec): + project = inclusive_projection(schema, id_and_bucket_spec) + # Not causes In to be rewritten to NotIn, which cannot be projected + assert project(Not(Or(LessThan("id", 5), In("data", {"a", "b", "c"})))) == GreaterThanOrEqual("id_part", 5) + + +def test_projection_partial_projected_fields(schema: Schema, id_spec: PartitionSpec): + project = inclusive_projection(schema, id_spec) + assert project(And(LessThan("id", 5), In("data", {"a", "b", "c"}))) == LessThan("id_part", 5) diff --git a/tests/table/test_init.py b/tests/table/test_init.py index 475a600fd9..5ad535693a 100644 --- a/tests/table/test_init.py +++ b/tests/table/test_init.py @@ -199,19 +199,8 @@ def test_table_scan_select(table: Table): def test_table_scan_row_filter(table: Table): scan = table.scan() assert scan.row_filter == AlwaysTrue() - assert scan.filter_rows(EqualTo("x", 10)).row_filter == EqualTo("x", 10) - assert scan.filter_rows(EqualTo("x", 10)).filter_rows(In("y", (10, 11))).row_filter == And( - EqualTo("x", 10), In("y", (10, 11)) - ) - - -def test_table_scan_partition_filter(table: Table): - scan = table.scan() - assert scan.row_filter == AlwaysTrue() - assert scan.filter_partitions(EqualTo("x", 10)).partition_filter == EqualTo("x", 10) - assert scan.filter_partitions(EqualTo("x", 10)).filter_partitions(In("y", (10, 11))).partition_filter == And( - EqualTo("x", 10), In("y", (10, 11)) - ) + assert scan.filter(EqualTo("x", 10)).row_filter == EqualTo("x", 10) + assert scan.filter(EqualTo("x", 10)).filter(In("y", (10, 11))).row_filter == And(EqualTo("x", 10), In("y", (10, 11))) def test_table_scan_ref(table: Table): diff --git a/tests/table/test_partitioning.py b/tests/table/test_partitioning.py index a559dcea30..9c37822360 100644 --- a/tests/table/test_partitioning.py +++ b/tests/table/test_partitioning.py @@ -62,6 +62,8 @@ def test_partition_spec_init(): assert partition_spec1 != partition_spec2 assert partition_spec1.compatible_with(partition_spec2) assert partition_spec1.fields_by_source_id(3) == [id_field1] + # Does not exist + assert partition_spec1.fields_by_source_id(1925) == [] def test_partition_compatible_with(): From 3841e8f480564b785ad89d9483df6cbf97ff47d9 Mon Sep 17 00:00:00 2001 From: Fokko Driesprong Date: Mon, 28 Nov 2022 21:15:43 +0100 Subject: [PATCH 281/642] Python: Implement PyArrow row level filtering (#6258) --- pyiceberg/expressions/visitors.py | 10 +- pyiceberg/io/pyarrow.py | 65 +++++++++ pyiceberg/table/__init__.py | 23 ++- tests/expressions/test_evaluator.py | 34 ++--- tests/expressions/test_visitors.py | 209 +++++++++++++++------------- tests/io/test_pyarrow.py | 162 ++++++++++++++++++++- 6 files changed, 375 insertions(+), 128 deletions(-) diff --git a/pyiceberg/expressions/visitors.py b/pyiceberg/expressions/visitors.py index 9aa841097e..8888b722e0 100644 --- a/pyiceberg/expressions/visitors.py +++ b/pyiceberg/expressions/visitors.py @@ -214,7 +214,7 @@ class BindVisitor(BooleanExpressionVisitor[BooleanExpression]): schema: Schema case_sensitive: bool - def __init__(self, schema: Schema, case_sensitive: bool = True) -> None: + def __init__(self, schema: Schema, case_sensitive: bool) -> None: self.schema = schema self.case_sensitive = case_sensitive @@ -421,9 +421,7 @@ def visit_bound_predicate(self, predicate: BoundPredicate[L]) -> BooleanExpressi return predicate -def expression_evaluator( - schema: Schema, unbound: BooleanExpression, case_sensitive: bool = True -) -> Callable[[StructProtocol], bool]: +def expression_evaluator(schema: Schema, unbound: BooleanExpression, case_sensitive: bool) -> Callable[[StructProtocol], bool]: return _ExpressionEvaluator(schema, unbound, case_sensitive).eval @@ -431,7 +429,7 @@ class _ExpressionEvaluator(BoundBooleanExpressionVisitor[bool]): bound: BooleanExpression struct: StructProtocol - def __init__(self, schema: Schema, unbound: BooleanExpression, case_sensitive: bool = True): + def __init__(self, schema: Schema, unbound: BooleanExpression, case_sensitive: bool): self.bound = bind(schema, unbound, case_sensitive) def eval(self, struct: StructProtocol) -> bool: @@ -507,7 +505,7 @@ class _ManifestEvalVisitor(BoundBooleanExpressionVisitor[bool]): partition_fields: List[PartitionFieldSummary] partition_filter: BooleanExpression - def __init__(self, partition_struct_schema: Schema, partition_filter: BooleanExpression, case_sensitive: bool = True): + def __init__(self, partition_struct_schema: Schema, partition_filter: BooleanExpression, case_sensitive): self.partition_filter = bind(partition_struct_schema, rewrite_not(partition_filter), case_sensitive) def eval(self, manifest: ManifestFile) -> bool: diff --git a/pyiceberg/io/pyarrow.py b/pyiceberg/io/pyarrow.py index d5e23e5c9d..751b2815e3 100644 --- a/pyiceberg/io/pyarrow.py +++ b/pyiceberg/io/pyarrow.py @@ -25,14 +25,17 @@ import os from functools import lru_cache, singledispatch from typing import ( + Any, Callable, List, + Set, Tuple, Union, ) from urllib.parse import urlparse import pyarrow as pa +import pyarrow.compute as pc from pyarrow.fs import ( FileInfo, FileSystem, @@ -41,6 +44,9 @@ S3FileSystem, ) +from pyiceberg.expressions import BooleanExpression, BoundTerm, Literal +from pyiceberg.expressions.visitors import BoundBooleanExpressionVisitor +from pyiceberg.expressions.visitors import visit as boolean_expression_visit from pyiceberg.io import ( FileIO, InputFile, @@ -379,3 +385,62 @@ def _(_: StringType) -> pa.DataType: def _(_: BinaryType) -> pa.DataType: # Variable length by default return pa.binary() + + +class _ConvertToArrowExpression(BoundBooleanExpressionVisitor[pc.Expression]): + def visit_in(self, term: BoundTerm[pc.Expression], literals: Set[Any]) -> pc.Expression: + return pc.field(term.ref().field.name).isin(literals) + + def visit_not_in(self, term: BoundTerm[pc.Expression], literals: Set[Any]) -> pc.Expression: + return ~pc.field(term.ref().field.name).isin(literals) + + def visit_is_nan(self, term: BoundTerm[pc.Expression]) -> pc.Expression: + ref = pc.field(term.ref().field.name) + return ref.is_null(nan_is_null=True) & ref.is_valid() + + def visit_not_nan(self, term: BoundTerm[pc.Expression]) -> pc.Expression: + ref = pc.field(term.ref().field.name) + return ~(ref.is_null(nan_is_null=True) & ref.is_valid()) + + def visit_is_null(self, term: BoundTerm[pc.Expression]) -> pc.Expression: + return pc.field(term.ref().field.name).is_null(nan_is_null=False) + + def visit_not_null(self, term: BoundTerm[Any]) -> pc.Expression: + return pc.field(term.ref().field.name).is_valid() + + def visit_equal(self, term: BoundTerm[Any], literal: Literal[Any]) -> pc.Expression: + return pc.field(term.ref().field.name) == literal.value + + def visit_not_equal(self, term: BoundTerm[Any], literal: Literal[Any]) -> pc.Expression: + return pc.field(term.ref().field.name) != literal.value + + def visit_greater_than_or_equal(self, term: BoundTerm[Any], literal: Literal[Any]) -> pc.Expression: + return pc.field(term.ref().field.name) >= literal.value + + def visit_greater_than(self, term: BoundTerm[Any], literal: Literal[Any]) -> pc.Expression: + return pc.field(term.ref().field.name) > literal.value + + def visit_less_than(self, term: BoundTerm[Any], literal: Literal[Any]) -> pc.Expression: + return pc.field(term.ref().field.name) < literal.value + + def visit_less_than_or_equal(self, term: BoundTerm[Any], literal: Literal[Any]) -> pc.Expression: + return pc.field(term.ref().field.name) <= literal.value + + def visit_true(self) -> pc.Expression: + return pc.scalar(True) + + def visit_false(self) -> pc.Expression: + return pc.scalar(False) + + def visit_not(self, child_result: pc.Expression) -> pc.Expression: + return ~child_result + + def visit_and(self, left_result: pc.Expression, right_result: pc.Expression) -> pc.Expression: + return left_result & right_result + + def visit_or(self, left_result: pc.Expression, right_result: pc.Expression) -> pc.Expression: + return left_result | right_result + + +def expression_to_pyarrow(expr: BooleanExpression) -> pc.Expression: + return boolean_expression_visit(expr, _ConvertToArrowExpression()) diff --git a/pyiceberg/table/__init__.py b/pyiceberg/table/__init__.py index dcc14c03c8..e989a3c911 100644 --- a/pyiceberg/table/__init__.py +++ b/pyiceberg/table/__init__.py @@ -39,8 +39,9 @@ BooleanExpression, visitors, ) -from pyiceberg.expressions.visitors import inclusive_projection +from pyiceberg.expressions.visitors import bind, inclusive_projection from pyiceberg.io import FileIO +from pyiceberg.io.pyarrow import expression_to_pyarrow, schema_to_pyarrow from pyiceberg.manifest import DataFile, ManifestFile, files from pyiceberg.partitioning import PartitionSpec from pyiceberg.schema import Schema @@ -340,8 +341,6 @@ def to_arrow(self): scheme, path = PyArrowFileIO.parse_location(self.table.location()) fs = self.table.io.get_fs(scheme) - import pyarrow.parquet as pq - locations = [] for task in self.plan_files(): if isinstance(task, FileScanTask): @@ -354,7 +353,23 @@ def to_arrow(self): if "*" not in self.selected_fields: columns = list(self.selected_fields) - return pq.read_table(source=locations, filesystem=fs, columns=columns) + pyarrow_filter = None + if self.row_filter is not AlwaysTrue(): + bound_row_filter = bind(self.table.schema(), self.row_filter, case_sensitive=self.case_sensitive) + pyarrow_filter = expression_to_pyarrow(bound_row_filter) + + from pyarrow.dataset import dataset + + ds = dataset( + source=locations, + filesystem=fs, + # Optionally provide the Schema for the Dataset, + # in which case it will not be inferred from the source. + # https://arrow.apache.org/docs/python/generated/pyarrow.dataset.dataset.html#pyarrow.dataset.dataset + schema=schema_to_pyarrow(self.table.schema()), + ) + + return ds.to_table(filter=pyarrow_filter, columns=columns) def to_duckdb(self, table_name: str, connection=None): import duckdb diff --git a/tests/expressions/test_evaluator.py b/tests/expressions/test_evaluator.py index 07a553947e..d2e473e38f 100644 --- a/tests/expressions/test_evaluator.py +++ b/tests/expressions/test_evaluator.py @@ -69,111 +69,111 @@ def set(self, pos: int, value: Any) -> None: def test_true(): - evaluate = expression_evaluator(SIMPLE_SCHEMA, AlwaysTrue()) + evaluate = expression_evaluator(SIMPLE_SCHEMA, AlwaysTrue(), case_sensitive=True) assert evaluate(Record(1, "a")) def test_false(): - evaluate = expression_evaluator(SIMPLE_SCHEMA, AlwaysFalse()) + evaluate = expression_evaluator(SIMPLE_SCHEMA, AlwaysFalse(), case_sensitive=True) assert not evaluate(Record(1, "a")) def test_less_than(): - evaluate = expression_evaluator(SIMPLE_SCHEMA, LessThan("id", 3)) + evaluate = expression_evaluator(SIMPLE_SCHEMA, LessThan("id", 3), case_sensitive=True) assert evaluate(Record(2, "a")) assert not evaluate(Record(3, "a")) def test_less_than_or_equal(): - evaluate = expression_evaluator(SIMPLE_SCHEMA, LessThanOrEqual("id", 3)) + evaluate = expression_evaluator(SIMPLE_SCHEMA, LessThanOrEqual("id", 3), case_sensitive=True) assert evaluate(Record(1, "a")) assert evaluate(Record(3, "a")) assert not evaluate(Record(4, "a")) def test_greater_than(): - evaluate = expression_evaluator(SIMPLE_SCHEMA, GreaterThan("id", 3)) + evaluate = expression_evaluator(SIMPLE_SCHEMA, GreaterThan("id", 3), case_sensitive=True) assert not evaluate(Record(1, "a")) assert not evaluate(Record(3, "a")) assert evaluate(Record(4, "a")) def test_greater_than_or_equal(): - evaluate = expression_evaluator(SIMPLE_SCHEMA, GreaterThanOrEqual("id", 3)) + evaluate = expression_evaluator(SIMPLE_SCHEMA, GreaterThanOrEqual("id", 3), case_sensitive=True) assert not evaluate(Record(2, "a")) assert evaluate(Record(3, "a")) assert evaluate(Record(4, "a")) def test_equal_to(): - evaluate = expression_evaluator(SIMPLE_SCHEMA, EqualTo("id", 3)) + evaluate = expression_evaluator(SIMPLE_SCHEMA, EqualTo("id", 3), case_sensitive=True) assert not evaluate(Record(2, "a")) assert evaluate(Record(3, "a")) assert not evaluate(Record(4, "a")) def test_not_equal_to(): - evaluate = expression_evaluator(SIMPLE_SCHEMA, NotEqualTo("id", 3)) + evaluate = expression_evaluator(SIMPLE_SCHEMA, NotEqualTo("id", 3), case_sensitive=True) assert evaluate(Record(2, "a")) assert not evaluate(Record(3, "a")) assert evaluate(Record(4, "a")) def test_in(): - evaluate = expression_evaluator(SIMPLE_SCHEMA, In("id", [1, 2, 3])) + evaluate = expression_evaluator(SIMPLE_SCHEMA, In("id", [1, 2, 3]), case_sensitive=True) assert evaluate(Record(2, "a")) assert evaluate(Record(3, "a")) assert not evaluate(Record(4, "a")) def test_not_in(): - evaluate = expression_evaluator(SIMPLE_SCHEMA, NotIn("id", [1, 2, 3])) + evaluate = expression_evaluator(SIMPLE_SCHEMA, NotIn("id", [1, 2, 3]), case_sensitive=True) assert not evaluate(Record(2, "a")) assert not evaluate(Record(3, "a")) assert evaluate(Record(4, "a")) def test_is_null(): - evaluate = expression_evaluator(SIMPLE_SCHEMA, IsNull("data")) + evaluate = expression_evaluator(SIMPLE_SCHEMA, IsNull("data"), case_sensitive=True) assert not evaluate(Record(2, "a")) assert evaluate(Record(3, None)) def test_not_null(): - evaluate = expression_evaluator(SIMPLE_SCHEMA, NotNull("data")) + evaluate = expression_evaluator(SIMPLE_SCHEMA, NotNull("data"), case_sensitive=True) assert evaluate(Record(2, "a")) assert not evaluate(Record(3, None)) def test_is_nan(): - evaluate = expression_evaluator(FLOAT_SCHEMA, IsNaN("f")) + evaluate = expression_evaluator(FLOAT_SCHEMA, IsNaN("f"), case_sensitive=True) assert not evaluate(Record(2, 0.0)) assert not evaluate(Record(3, float("infinity"))) assert evaluate(Record(4, float("nan"))) def test_not_nan(): - evaluate = expression_evaluator(FLOAT_SCHEMA, NotNaN("f")) + evaluate = expression_evaluator(FLOAT_SCHEMA, NotNaN("f"), case_sensitive=True) assert evaluate(Record(2, 0.0)) assert evaluate(Record(3, float("infinity"))) assert not evaluate(Record(4, float("nan"))) def test_not(): - evaluate = expression_evaluator(SIMPLE_SCHEMA, Not(LessThan("id", 3))) + evaluate = expression_evaluator(SIMPLE_SCHEMA, Not(LessThan("id", 3)), case_sensitive=True) assert not evaluate(Record(2, "a")) assert evaluate(Record(3, "a")) def test_and(): - evaluate = expression_evaluator(SIMPLE_SCHEMA, And(LessThan("id", 3), GreaterThan("id", 1))) + evaluate = expression_evaluator(SIMPLE_SCHEMA, And(LessThan("id", 3), GreaterThan("id", 1)), case_sensitive=True) assert not evaluate(Record(1, "a")) assert evaluate(Record(2, "a")) assert not evaluate(Record(3, "a")) def test_or(): - evaluate = expression_evaluator(SIMPLE_SCHEMA, Or(LessThan("id", 2), GreaterThan("id", 2))) + evaluate = expression_evaluator(SIMPLE_SCHEMA, Or(LessThan("id", 2), GreaterThan("id", 2)), case_sensitive=True) assert evaluate(Record(1, "a")) assert not evaluate(Record(2, "a")) assert evaluate(Record(3, "a")) diff --git a/tests/expressions/test_visitors.py b/tests/expressions/test_visitors.py index 0f07156be4..0552843413 100644 --- a/tests/expressions/test_visitors.py +++ b/tests/expressions/test_visitors.py @@ -289,7 +289,7 @@ def test_bind_visitor_already_bound(table_schema_simple: Schema): literal=literal("hello"), ) with pytest.raises(TypeError) as exc_info: - visit(bound, visitor=BindVisitor(schema=table_schema_simple)) + visit(bound, visitor=BindVisitor(schema=table_schema_simple, case_sensitive=True)) assert ( "Found already bound predicate: BoundEqualTo(term=BoundReference(field=NestedField(field_id=1, name='foo', field_type=StringType(), required=False), accessor=Accessor(position=0,inner=None)), literal=literal('hello'))" == str(exc_info.value) @@ -305,28 +305,28 @@ def test_visit_bound_visitor_unknown_predicate(): def test_always_true_expression_binding(table_schema_simple: Schema): """Test that visiting an always-true expression returns always-true""" unbound_expression = AlwaysTrue() - bound_expression = visit(unbound_expression, visitor=BindVisitor(schema=table_schema_simple)) + bound_expression = visit(unbound_expression, visitor=BindVisitor(schema=table_schema_simple, case_sensitive=True)) assert bound_expression == AlwaysTrue() def test_always_false_expression_binding(table_schema_simple: Schema): """Test that visiting an always-false expression returns always-false""" unbound_expression = AlwaysFalse() - bound_expression = visit(unbound_expression, visitor=BindVisitor(schema=table_schema_simple)) + bound_expression = visit(unbound_expression, visitor=BindVisitor(schema=table_schema_simple, case_sensitive=True)) assert bound_expression == AlwaysFalse() def test_always_false_and_always_true_expression_binding(table_schema_simple: Schema): """Test that visiting both an always-true AND always-false expression returns always-false""" unbound_expression = And(AlwaysTrue(), AlwaysFalse()) - bound_expression = visit(unbound_expression, visitor=BindVisitor(schema=table_schema_simple)) + bound_expression = visit(unbound_expression, visitor=BindVisitor(schema=table_schema_simple, case_sensitive=True)) assert bound_expression == AlwaysFalse() def test_always_false_or_always_true_expression_binding(table_schema_simple: Schema): """Test that visiting always-true OR always-false expression returns always-true""" unbound_expression = Or(AlwaysTrue(), AlwaysFalse()) - bound_expression = visit(unbound_expression, visitor=BindVisitor(schema=table_schema_simple)) + bound_expression = visit(unbound_expression, visitor=BindVisitor(schema=table_schema_simple, case_sensitive=True)) assert bound_expression == AlwaysTrue() @@ -397,7 +397,7 @@ def test_always_false_or_always_true_expression_binding(table_schema_simple: Sch ) def test_and_expression_binding(unbound_and_expression, expected_bound_expression, table_schema_simple): """Test that visiting an unbound AND expression with a bind-visitor returns the expected bound expression""" - bound_expression = visit(unbound_and_expression, visitor=BindVisitor(schema=table_schema_simple)) + bound_expression = visit(unbound_and_expression, visitor=BindVisitor(schema=table_schema_simple, case_sensitive=True)) assert bound_expression == expected_bound_expression @@ -489,7 +489,7 @@ def test_and_expression_binding(unbound_and_expression, expected_bound_expressio ) def test_or_expression_binding(unbound_or_expression, expected_bound_expression, table_schema_simple): """Test that visiting an unbound OR expression with a bind-visitor returns the expected bound expression""" - bound_expression = visit(unbound_or_expression, visitor=BindVisitor(schema=table_schema_simple)) + bound_expression = visit(unbound_or_expression, visitor=BindVisitor(schema=table_schema_simple, case_sensitive=True)) assert bound_expression == expected_bound_expression @@ -533,7 +533,7 @@ def test_or_expression_binding(unbound_or_expression, expected_bound_expression, ) def test_in_expression_binding(unbound_in_expression, expected_bound_expression, table_schema_simple): """Test that visiting an unbound IN expression with a bind-visitor returns the expected bound expression""" - bound_expression = visit(unbound_in_expression, visitor=BindVisitor(schema=table_schema_simple)) + bound_expression = visit(unbound_in_expression, visitor=BindVisitor(schema=table_schema_simple, case_sensitive=True)) assert bound_expression == expected_bound_expression @@ -582,7 +582,7 @@ def test_in_expression_binding(unbound_in_expression, expected_bound_expression, ) def test_not_expression_binding(unbound_not_expression, expected_bound_expression, table_schema_simple): """Test that visiting an unbound NOT expression with a bind-visitor returns the expected bound expression""" - bound_expression = visit(unbound_not_expression, visitor=BindVisitor(schema=table_schema_simple)) + bound_expression = visit(unbound_not_expression, visitor=BindVisitor(schema=table_schema_simple, case_sensitive=True)) assert bound_expression == expected_bound_expression @@ -950,93 +950,93 @@ def manifest() -> ManifestFile: def test_all_nulls(schema: Schema, manifest: ManifestFile) -> None: - assert not _ManifestEvalVisitor(schema, NotNull(Reference("all_nulls_missing_nan"))).eval( + assert not _ManifestEvalVisitor(schema, NotNull(Reference("all_nulls_missing_nan")), case_sensitive=True).eval( manifest ), "Should skip: all nulls column with non-floating type contains all null" - assert _ManifestEvalVisitor(schema, NotNull(Reference("all_nulls_missing_nan_float"))).eval( + assert _ManifestEvalVisitor(schema, NotNull(Reference("all_nulls_missing_nan_float")), case_sensitive=True).eval( manifest ), "Should read: no NaN information may indicate presence of NaN value" - assert _ManifestEvalVisitor(schema, NotNull(Reference("some_nulls"))).eval( + assert _ManifestEvalVisitor(schema, NotNull(Reference("some_nulls")), case_sensitive=True).eval( manifest ), "Should read: column with some nulls contains a non-null value" - assert _ManifestEvalVisitor(schema, NotNull(Reference("no_nulls"))).eval( + assert _ManifestEvalVisitor(schema, NotNull(Reference("no_nulls")), case_sensitive=True).eval( manifest ), "Should read: non-null column contains a non-null value" def test_no_nulls(schema: Schema, manifest: ManifestFile) -> None: - assert _ManifestEvalVisitor(schema, IsNull(Reference("all_nulls_missing_nan"))).eval( + assert _ManifestEvalVisitor(schema, IsNull(Reference("all_nulls_missing_nan")), case_sensitive=True).eval( manifest ), "Should read: at least one null value in all null column" - assert _ManifestEvalVisitor(schema, IsNull(Reference("some_nulls"))).eval( + assert _ManifestEvalVisitor(schema, IsNull(Reference("some_nulls")), case_sensitive=True).eval( manifest ), "Should read: column with some nulls contains a null value" - assert not _ManifestEvalVisitor(schema, IsNull(Reference("no_nulls"))).eval( + assert not _ManifestEvalVisitor(schema, IsNull(Reference("no_nulls")), case_sensitive=True).eval( manifest ), "Should skip: non-null column contains no null values" - assert _ManifestEvalVisitor(schema, IsNull(Reference("both_nan_and_null"))).eval( + assert _ManifestEvalVisitor(schema, IsNull(Reference("both_nan_and_null")), case_sensitive=True).eval( manifest ), "Should read: both_nan_and_null column contains no null values" def test_is_nan(schema: Schema, manifest: ManifestFile) -> None: - assert _ManifestEvalVisitor(schema, IsNaN(Reference("float"))).eval( + assert _ManifestEvalVisitor(schema, IsNaN(Reference("float")), case_sensitive=True).eval( manifest ), "Should read: no information on if there are nan value in float column" - assert _ManifestEvalVisitor(schema, IsNaN(Reference("all_nulls_double"))).eval( + assert _ManifestEvalVisitor(schema, IsNaN(Reference("all_nulls_double")), case_sensitive=True).eval( manifest ), "Should read: no NaN information may indicate presence of NaN value" - assert _ManifestEvalVisitor(schema, IsNaN(Reference("all_nulls_missing_nan_float"))).eval( + assert _ManifestEvalVisitor(schema, IsNaN(Reference("all_nulls_missing_nan_float")), case_sensitive=True).eval( manifest ), "Should read: no NaN information may indicate presence of NaN value" - assert not _ManifestEvalVisitor(schema, IsNaN(Reference("all_nulls_no_nans"))).eval( + assert not _ManifestEvalVisitor(schema, IsNaN(Reference("all_nulls_no_nans")), case_sensitive=True).eval( manifest ), "Should skip: no nan column doesn't contain nan value" - assert _ManifestEvalVisitor(schema, IsNaN(Reference("all_nans"))).eval( + assert _ManifestEvalVisitor(schema, IsNaN(Reference("all_nans")), case_sensitive=True).eval( manifest ), "Should read: all_nans column contains nan value" - assert _ManifestEvalVisitor(schema, IsNaN(Reference("both_nan_and_null"))).eval( + assert _ManifestEvalVisitor(schema, IsNaN(Reference("both_nan_and_null")), case_sensitive=True).eval( manifest ), "Should read: both_nan_and_null column contains nan value" - assert not _ManifestEvalVisitor(schema, IsNaN(Reference("no_nan_or_null"))).eval( + assert not _ManifestEvalVisitor(schema, IsNaN(Reference("no_nan_or_null")), case_sensitive=True).eval( manifest ), "Should skip: no_nan_or_null column doesn't contain nan value" def test_not_nan(schema: Schema, manifest: ManifestFile) -> None: - assert _ManifestEvalVisitor(schema, NotNaN(Reference("float"))).eval( + assert _ManifestEvalVisitor(schema, NotNaN(Reference("float")), case_sensitive=True).eval( manifest ), "Should read: no information on if there are nan value in float column" - assert _ManifestEvalVisitor(schema, NotNaN(Reference("all_nulls_double"))).eval( + assert _ManifestEvalVisitor(schema, NotNaN(Reference("all_nulls_double")), case_sensitive=True).eval( manifest ), "Should read: all null column contains non nan value" - assert _ManifestEvalVisitor(schema, NotNaN(Reference("all_nulls_no_nans"))).eval( + assert _ManifestEvalVisitor(schema, NotNaN(Reference("all_nulls_no_nans")), case_sensitive=True).eval( manifest ), "Should read: no_nans column contains non nan value" - assert not _ManifestEvalVisitor(schema, NotNaN(Reference("all_nans"))).eval( + assert not _ManifestEvalVisitor(schema, NotNaN(Reference("all_nans")), case_sensitive=True).eval( manifest ), "Should skip: all nans column doesn't contain non nan value" - assert _ManifestEvalVisitor(schema, NotNaN(Reference("both_nan_and_null"))).eval( + assert _ManifestEvalVisitor(schema, NotNaN(Reference("both_nan_and_null")), case_sensitive=True).eval( manifest ), "Should read: both_nan_and_null nans column contains non nan value" - assert _ManifestEvalVisitor(schema, NotNaN(Reference("no_nan_or_null"))).eval( + assert _ManifestEvalVisitor(schema, NotNaN(Reference("no_nan_or_null")), case_sensitive=True).eval( manifest ), "Should read: no_nan_or_null column contains non nan value" @@ -1056,15 +1056,17 @@ def test_missing_stats(schema: Schema, manifest_no_stats: ManifestFile): ] for expr in expressions: - assert _ManifestEvalVisitor(schema, expr).eval(manifest_no_stats), f"Should read when missing stats for expr: {expr}" + assert _ManifestEvalVisitor(schema, expr, case_sensitive=True).eval( + manifest_no_stats + ), f"Should read when missing stats for expr: {expr}" def test_not(schema: Schema, manifest: ManifestFile): - assert _ManifestEvalVisitor(schema, Not(LessThan(Reference("id"), INT_MIN_VALUE - 25))).eval( + assert _ManifestEvalVisitor(schema, Not(LessThan(Reference("id"), INT_MIN_VALUE - 25)), case_sensitive=True).eval( manifest ), "Should read: not(false)" - assert not _ManifestEvalVisitor(schema, Not(GreaterThan(Reference("id"), INT_MIN_VALUE - 25))).eval( + assert not _ManifestEvalVisitor(schema, Not(GreaterThan(Reference("id"), INT_MIN_VALUE - 25)), case_sensitive=True).eval( manifest ), "Should skip: not(true)" @@ -1076,6 +1078,7 @@ def test_and(schema: Schema, manifest: ManifestFile): LessThan(Reference("id"), INT_MIN_VALUE - 25), GreaterThanOrEqual(Reference("id"), INT_MIN_VALUE - 30), ), + case_sensitive=True, ).eval(manifest), "Should skip: and(false, true)" assert not _ManifestEvalVisitor( @@ -1084,6 +1087,7 @@ def test_and(schema: Schema, manifest: ManifestFile): LessThan(Reference("id"), INT_MIN_VALUE - 25), GreaterThanOrEqual(Reference("id"), INT_MAX_VALUE + 1), ), + case_sensitive=True, ).eval(manifest), "Should skip: and(false, false)" assert _ManifestEvalVisitor( @@ -1092,6 +1096,7 @@ def test_and(schema: Schema, manifest: ManifestFile): GreaterThan(Reference("id"), INT_MIN_VALUE - 25), LessThanOrEqual(Reference("id"), INT_MIN_VALUE), ), + case_sensitive=True, ).eval(manifest), "Should read: and(true, true)" @@ -1102,6 +1107,7 @@ def test_or(schema: Schema, manifest: ManifestFile): LessThan(Reference("id"), INT_MIN_VALUE - 25), GreaterThanOrEqual(Reference("id"), INT_MAX_VALUE + 1), ), + case_sensitive=True, ).eval(manifest), "Should skip: or(false, false)" assert _ManifestEvalVisitor( @@ -1110,165 +1116,168 @@ def test_or(schema: Schema, manifest: ManifestFile): LessThan(Reference("id"), INT_MIN_VALUE - 25), GreaterThanOrEqual(Reference("id"), INT_MAX_VALUE - 19), ), + case_sensitive=True, ).eval(manifest), "Should read: or(false, true)" def test_integer_lt(schema: Schema, manifest: ManifestFile): - assert not _ManifestEvalVisitor(schema, LessThan(Reference("id"), INT_MIN_VALUE - 25)).eval( + assert not _ManifestEvalVisitor(schema, LessThan(Reference("id"), INT_MIN_VALUE - 25), case_sensitive=True).eval( manifest ), "Should not read: id range below lower bound (5 < 30)" - assert not _ManifestEvalVisitor(schema, LessThan(Reference("id"), INT_MIN_VALUE)).eval( + assert not _ManifestEvalVisitor(schema, LessThan(Reference("id"), INT_MIN_VALUE), case_sensitive=True).eval( manifest ), "Should not read: id range below lower bound (30 is not < 30)" - assert _ManifestEvalVisitor(schema, LessThan(Reference("id"), INT_MIN_VALUE + 1)).eval( + assert _ManifestEvalVisitor(schema, LessThan(Reference("id"), INT_MIN_VALUE + 1), case_sensitive=True).eval( manifest ), "Should read: one possible id" - assert _ManifestEvalVisitor(schema, LessThan(Reference("id"), INT_MAX_VALUE)).eval(manifest), "Should read: may possible ids" + assert _ManifestEvalVisitor(schema, LessThan(Reference("id"), INT_MAX_VALUE), case_sensitive=True).eval( + manifest + ), "Should read: may possible ids" def test_integer_lt_eq(schema: Schema, manifest: ManifestFile): - assert not _ManifestEvalVisitor(schema, LessThanOrEqual(Reference("id"), INT_MIN_VALUE - 25)).eval( + assert not _ManifestEvalVisitor(schema, LessThanOrEqual(Reference("id"), INT_MIN_VALUE - 25), case_sensitive=True).eval( manifest ), "Should not read: id range below lower bound (5 < 30)" - assert not _ManifestEvalVisitor(schema, LessThanOrEqual(Reference("id"), INT_MIN_VALUE - 1)).eval( + assert not _ManifestEvalVisitor(schema, LessThanOrEqual(Reference("id"), INT_MIN_VALUE - 1), case_sensitive=True).eval( manifest ), "Should not read: id range below lower bound (29 < 30)" - assert _ManifestEvalVisitor(schema, LessThanOrEqual(Reference("id"), INT_MIN_VALUE)).eval( + assert _ManifestEvalVisitor(schema, LessThanOrEqual(Reference("id"), INT_MIN_VALUE), case_sensitive=True).eval( manifest ), "Should read: one possible id" - assert _ManifestEvalVisitor(schema, LessThanOrEqual(Reference("id"), INT_MAX_VALUE)).eval( + assert _ManifestEvalVisitor(schema, LessThanOrEqual(Reference("id"), INT_MAX_VALUE), case_sensitive=True).eval( manifest ), "Should read: many possible ids" def test_integer_gt(schema: Schema, manifest: ManifestFile): - assert not _ManifestEvalVisitor(schema, GreaterThan(Reference("id"), INT_MAX_VALUE + 6)).eval( + assert not _ManifestEvalVisitor(schema, GreaterThan(Reference("id"), INT_MAX_VALUE + 6), case_sensitive=True).eval( manifest ), "Should not read: id range above upper bound (85 < 79)" - assert not _ManifestEvalVisitor(schema, GreaterThan(Reference("id"), INT_MAX_VALUE)).eval( + assert not _ManifestEvalVisitor(schema, GreaterThan(Reference("id"), INT_MAX_VALUE), case_sensitive=True).eval( manifest ), "Should not read: id range above upper bound (79 is not > 79)" - assert _ManifestEvalVisitor(schema, GreaterThan(Reference("id"), INT_MAX_VALUE - 1)).eval( + assert _ManifestEvalVisitor(schema, GreaterThan(Reference("id"), INT_MAX_VALUE - 1), case_sensitive=True).eval( manifest ), "Should read: one possible id" - assert _ManifestEvalVisitor(schema, GreaterThan(Reference("id"), INT_MAX_VALUE - 4)).eval( + assert _ManifestEvalVisitor(schema, GreaterThan(Reference("id"), INT_MAX_VALUE - 4), case_sensitive=True).eval( manifest ), "Should read: may possible ids" def test_integer_gt_eq(schema: Schema, manifest: ManifestFile): - assert not _ManifestEvalVisitor(schema, GreaterThanOrEqual(Reference("id"), INT_MAX_VALUE + 6)).eval( + assert not _ManifestEvalVisitor(schema, GreaterThanOrEqual(Reference("id"), INT_MAX_VALUE + 6), case_sensitive=True).eval( manifest ), "Should not read: id range above upper bound (85 < 79)" - assert not _ManifestEvalVisitor(schema, GreaterThanOrEqual(Reference("id"), INT_MAX_VALUE + 1)).eval( + assert not _ManifestEvalVisitor(schema, GreaterThanOrEqual(Reference("id"), INT_MAX_VALUE + 1), case_sensitive=True).eval( manifest ), "Should not read: id range above upper bound (80 > 79)" - assert _ManifestEvalVisitor(schema, GreaterThanOrEqual(Reference("id"), INT_MAX_VALUE)).eval( + assert _ManifestEvalVisitor(schema, GreaterThanOrEqual(Reference("id"), INT_MAX_VALUE), case_sensitive=True).eval( manifest ), "Should read: one possible id" - assert _ManifestEvalVisitor(schema, GreaterThanOrEqual(Reference("id"), INT_MAX_VALUE)).eval( + assert _ManifestEvalVisitor(schema, GreaterThanOrEqual(Reference("id"), INT_MAX_VALUE), case_sensitive=True).eval( manifest ), "Should read: may possible ids" def test_integer_eq(schema: Schema, manifest: ManifestFile): - assert not _ManifestEvalVisitor(schema, EqualTo(Reference("id"), INT_MIN_VALUE - 25)).eval( + assert not _ManifestEvalVisitor(schema, EqualTo(Reference("id"), INT_MIN_VALUE - 25), case_sensitive=True).eval( manifest ), "Should not read: id below lower bound" - assert not _ManifestEvalVisitor(schema, EqualTo(Reference("id"), INT_MIN_VALUE - 1)).eval( + assert not _ManifestEvalVisitor(schema, EqualTo(Reference("id"), INT_MIN_VALUE - 1), case_sensitive=True).eval( manifest ), "Should not read: id below lower bound" - assert _ManifestEvalVisitor(schema, EqualTo(Reference("id"), INT_MIN_VALUE)).eval( + assert _ManifestEvalVisitor(schema, EqualTo(Reference("id"), INT_MIN_VALUE), case_sensitive=True).eval( manifest ), "Should read: id equal to lower bound" - assert _ManifestEvalVisitor(schema, EqualTo(Reference("id"), INT_MAX_VALUE - 4)).eval( + assert _ManifestEvalVisitor(schema, EqualTo(Reference("id"), INT_MAX_VALUE - 4), case_sensitive=True).eval( manifest ), "Should read: id between lower and upper bounds" - assert _ManifestEvalVisitor(schema, EqualTo(Reference("id"), INT_MAX_VALUE)).eval( + assert _ManifestEvalVisitor(schema, EqualTo(Reference("id"), INT_MAX_VALUE), case_sensitive=True).eval( manifest ), "Should read: id equal to upper bound" - assert not _ManifestEvalVisitor(schema, EqualTo(Reference("id"), INT_MAX_VALUE + 1)).eval( + assert not _ManifestEvalVisitor(schema, EqualTo(Reference("id"), INT_MAX_VALUE + 1), case_sensitive=True).eval( manifest ), "Should not read: id above upper bound" - assert not _ManifestEvalVisitor(schema, EqualTo(Reference("id"), INT_MAX_VALUE + 6)).eval( + assert not _ManifestEvalVisitor(schema, EqualTo(Reference("id"), INT_MAX_VALUE + 6), case_sensitive=True).eval( manifest ), "Should not read: id above upper bound" def test_integer_not_eq(schema: Schema, manifest: ManifestFile): - assert _ManifestEvalVisitor(schema, NotEqualTo(Reference("id"), INT_MIN_VALUE - 25)).eval( + assert _ManifestEvalVisitor(schema, NotEqualTo(Reference("id"), INT_MIN_VALUE - 25), case_sensitive=True).eval( manifest ), "Should read: id below lower bound" - assert _ManifestEvalVisitor(schema, NotEqualTo(Reference("id"), INT_MIN_VALUE - 1)).eval( + assert _ManifestEvalVisitor(schema, NotEqualTo(Reference("id"), INT_MIN_VALUE - 1), case_sensitive=True).eval( manifest ), "Should read: id below lower bound" - assert _ManifestEvalVisitor(schema, NotEqualTo(Reference("id"), INT_MIN_VALUE)).eval( + assert _ManifestEvalVisitor(schema, NotEqualTo(Reference("id"), INT_MIN_VALUE), case_sensitive=True).eval( manifest ), "Should read: id equal to lower bound" - assert _ManifestEvalVisitor(schema, NotEqualTo(Reference("id"), INT_MAX_VALUE - 4)).eval( + assert _ManifestEvalVisitor(schema, NotEqualTo(Reference("id"), INT_MAX_VALUE - 4), case_sensitive=True).eval( manifest ), "Should read: id between lower and upper bounds" - assert _ManifestEvalVisitor(schema, NotEqualTo(Reference("id"), INT_MAX_VALUE)).eval( + assert _ManifestEvalVisitor(schema, NotEqualTo(Reference("id"), INT_MAX_VALUE), case_sensitive=True).eval( manifest ), "Should read: id equal to upper bound" - assert _ManifestEvalVisitor(schema, NotEqualTo(Reference("id"), INT_MAX_VALUE + 1)).eval( + assert _ManifestEvalVisitor(schema, NotEqualTo(Reference("id"), INT_MAX_VALUE + 1), case_sensitive=True).eval( manifest ), "Should read: id above upper bound" - assert _ManifestEvalVisitor(schema, NotEqualTo(Reference("id"), INT_MAX_VALUE + 6)).eval( + assert _ManifestEvalVisitor(schema, NotEqualTo(Reference("id"), INT_MAX_VALUE + 6), case_sensitive=True).eval( manifest ), "Should read: id above upper bound" def test_integer_not_eq_rewritten(schema: Schema, manifest: ManifestFile): - assert _ManifestEvalVisitor(schema, Not(EqualTo(Reference("id"), INT_MIN_VALUE - 25))).eval( + assert _ManifestEvalVisitor(schema, Not(EqualTo(Reference("id"), INT_MIN_VALUE - 25)), case_sensitive=True).eval( manifest ), "Should read: id below lower bound" - assert _ManifestEvalVisitor(schema, Not(EqualTo(Reference("id"), INT_MIN_VALUE - 1))).eval( + assert _ManifestEvalVisitor(schema, Not(EqualTo(Reference("id"), INT_MIN_VALUE - 1)), case_sensitive=True).eval( manifest ), "Should read: id below lower bound" - assert _ManifestEvalVisitor(schema, Not(EqualTo(Reference("id"), INT_MIN_VALUE))).eval( + assert _ManifestEvalVisitor(schema, Not(EqualTo(Reference("id"), INT_MIN_VALUE)), case_sensitive=True).eval( manifest ), "Should read: id equal to lower bound" - assert _ManifestEvalVisitor(schema, Not(EqualTo(Reference("id"), INT_MAX_VALUE - 4))).eval( + assert _ManifestEvalVisitor(schema, Not(EqualTo(Reference("id"), INT_MAX_VALUE - 4)), case_sensitive=True).eval( manifest ), "Should read: id between lower and upper bounds" - assert _ManifestEvalVisitor(schema, Not(EqualTo(Reference("id"), INT_MAX_VALUE))).eval( + assert _ManifestEvalVisitor(schema, Not(EqualTo(Reference("id"), INT_MAX_VALUE)), case_sensitive=True).eval( manifest ), "Should read: id equal to upper bound" - assert _ManifestEvalVisitor(schema, Not(EqualTo(Reference("id"), INT_MAX_VALUE + 1))).eval( + assert _ManifestEvalVisitor(schema, Not(EqualTo(Reference("id"), INT_MAX_VALUE + 1)), case_sensitive=True).eval( manifest ), "Should read: id above upper bound" - assert _ManifestEvalVisitor(schema, Not(EqualTo(Reference("id"), INT_MAX_VALUE + 6))).eval( + assert _ManifestEvalVisitor(schema, Not(EqualTo(Reference("id"), INT_MAX_VALUE + 6)), case_sensitive=True).eval( manifest ), "Should read: id above upper bound" @@ -1304,85 +1313,85 @@ def test_integer_not_eq_rewritten_case_insensitive(schema: Schema, manifest: Man def test_integer_in(schema: Schema, manifest: ManifestFile): - assert not _ManifestEvalVisitor(schema, In(Reference("id"), (INT_MIN_VALUE - 25, INT_MIN_VALUE - 24))).eval( - manifest - ), "Should not read: id below lower bound (5 < 30, 6 < 30)" + assert not _ManifestEvalVisitor( + schema, In(Reference("id"), (INT_MIN_VALUE - 25, INT_MIN_VALUE - 24)), case_sensitive=True + ).eval(manifest), "Should not read: id below lower bound (5 < 30, 6 < 30)" - assert not _ManifestEvalVisitor(schema, In(Reference("id"), (INT_MIN_VALUE - 2, INT_MIN_VALUE - 1))).eval( - manifest - ), "Should not read: id below lower bound (28 < 30, 29 < 30)" + assert not _ManifestEvalVisitor( + schema, In(Reference("id"), (INT_MIN_VALUE - 2, INT_MIN_VALUE - 1)), case_sensitive=True + ).eval(manifest), "Should not read: id below lower bound (28 < 30, 29 < 30)" - assert _ManifestEvalVisitor(schema, In(Reference("id"), (INT_MIN_VALUE - 1, INT_MIN_VALUE))).eval( + assert _ManifestEvalVisitor(schema, In(Reference("id"), (INT_MIN_VALUE - 1, INT_MIN_VALUE)), case_sensitive=True).eval( manifest ), "Should read: id equal to lower bound (30 == 30)" - assert _ManifestEvalVisitor(schema, In(Reference("id"), (INT_MAX_VALUE - 4, INT_MAX_VALUE - 3))).eval( + assert _ManifestEvalVisitor(schema, In(Reference("id"), (INT_MAX_VALUE - 4, INT_MAX_VALUE - 3)), case_sensitive=True).eval( manifest ), "Should read: id between lower and upper bounds (30 < 75 < 79, 30 < 76 < 79)" - assert _ManifestEvalVisitor(schema, In(Reference("id"), (INT_MAX_VALUE, INT_MAX_VALUE + 1))).eval( + assert _ManifestEvalVisitor(schema, In(Reference("id"), (INT_MAX_VALUE, INT_MAX_VALUE + 1)), case_sensitive=True).eval( manifest ), "Should read: id equal to upper bound (79 == 79)" - assert not _ManifestEvalVisitor(schema, In(Reference("id"), (INT_MAX_VALUE + 1, INT_MAX_VALUE + 2))).eval( - manifest - ), "Should not read: id above upper bound (80 > 79, 81 > 79)" + assert not _ManifestEvalVisitor( + schema, In(Reference("id"), (INT_MAX_VALUE + 1, INT_MAX_VALUE + 2)), case_sensitive=True + ).eval(manifest), "Should not read: id above upper bound (80 > 79, 81 > 79)" - assert not _ManifestEvalVisitor(schema, In(Reference("id"), (INT_MAX_VALUE + 6, INT_MAX_VALUE + 7))).eval( - manifest - ), "Should not read: id above upper bound (85 > 79, 86 > 79)" + assert not _ManifestEvalVisitor( + schema, In(Reference("id"), (INT_MAX_VALUE + 6, INT_MAX_VALUE + 7)), case_sensitive=True + ).eval(manifest), "Should not read: id above upper bound (85 > 79, 86 > 79)" - assert not _ManifestEvalVisitor(schema, In(Reference("all_nulls_missing_nan"), ("abc", "def"))).eval( + assert not _ManifestEvalVisitor(schema, In(Reference("all_nulls_missing_nan"), ("abc", "def")), case_sensitive=True).eval( manifest ), "Should skip: in on all nulls column" - assert _ManifestEvalVisitor(schema, In(Reference("some_nulls"), ("abc", "def"))).eval( + assert _ManifestEvalVisitor(schema, In(Reference("some_nulls"), ("abc", "def")), case_sensitive=True).eval( manifest ), "Should read: in on some nulls column" - assert _ManifestEvalVisitor(schema, In(Reference("no_nulls"), ("abc", "def"))).eval( + assert _ManifestEvalVisitor(schema, In(Reference("no_nulls"), ("abc", "def")), case_sensitive=True).eval( manifest ), "Should read: in on no nulls column" def test_integer_not_in(schema: Schema, manifest: ManifestFile): - assert _ManifestEvalVisitor(schema, NotIn(Reference("id"), (INT_MIN_VALUE - 25, INT_MIN_VALUE - 24))).eval( - manifest - ), "Should read: id below lower bound (5 < 30, 6 < 30)" + assert _ManifestEvalVisitor( + schema, NotIn(Reference("id"), (INT_MIN_VALUE - 25, INT_MIN_VALUE - 24)), case_sensitive=True + ).eval(manifest), "Should read: id below lower bound (5 < 30, 6 < 30)" - assert _ManifestEvalVisitor(schema, NotIn(Reference("id"), (INT_MIN_VALUE - 2, INT_MIN_VALUE - 1))).eval( + assert _ManifestEvalVisitor(schema, NotIn(Reference("id"), (INT_MIN_VALUE - 2, INT_MIN_VALUE - 1)), case_sensitive=True).eval( manifest ), "Should read: id below lower bound (28 < 30, 29 < 30)" - assert _ManifestEvalVisitor(schema, NotIn(Reference("id"), (INT_MIN_VALUE - 1, INT_MIN_VALUE))).eval( + assert _ManifestEvalVisitor(schema, NotIn(Reference("id"), (INT_MIN_VALUE - 1, INT_MIN_VALUE)), case_sensitive=True).eval( manifest ), "Should read: id equal to lower bound (30 == 30)" - assert _ManifestEvalVisitor(schema, NotIn(Reference("id"), (INT_MAX_VALUE - 4, INT_MAX_VALUE - 3))).eval( + assert _ManifestEvalVisitor(schema, NotIn(Reference("id"), (INT_MAX_VALUE - 4, INT_MAX_VALUE - 3)), case_sensitive=True).eval( manifest ), "Should read: id between lower and upper bounds (30 < 75 < 79, 30 < 76 < 79)" - assert _ManifestEvalVisitor(schema, NotIn(Reference("id"), (INT_MAX_VALUE, INT_MAX_VALUE + 1))).eval( + assert _ManifestEvalVisitor(schema, NotIn(Reference("id"), (INT_MAX_VALUE, INT_MAX_VALUE + 1)), case_sensitive=True).eval( manifest ), "Should read: id equal to upper bound (79 == 79)" - assert _ManifestEvalVisitor(schema, NotIn(Reference("id"), (INT_MAX_VALUE + 1, INT_MAX_VALUE + 2))).eval( + assert _ManifestEvalVisitor(schema, NotIn(Reference("id"), (INT_MAX_VALUE + 1, INT_MAX_VALUE + 2)), case_sensitive=True).eval( manifest ), "Should read: id above upper bound (80 > 79, 81 > 79)" - assert _ManifestEvalVisitor(schema, NotIn(Reference("id"), (INT_MAX_VALUE + 6, INT_MAX_VALUE + 7))).eval( + assert _ManifestEvalVisitor(schema, NotIn(Reference("id"), (INT_MAX_VALUE + 6, INT_MAX_VALUE + 7)), case_sensitive=True).eval( manifest ), "Should read: id above upper bound (85 > 79, 86 > 79)" - assert _ManifestEvalVisitor(schema, NotIn(Reference("all_nulls_missing_nan"), ("abc", "def"))).eval( + assert _ManifestEvalVisitor(schema, NotIn(Reference("all_nulls_missing_nan"), ("abc", "def")), case_sensitive=True).eval( manifest ), "Should read: notIn on no nulls column" - assert _ManifestEvalVisitor(schema, NotIn(Reference("some_nulls"), ("abc", "def"))).eval( + assert _ManifestEvalVisitor(schema, NotIn(Reference("some_nulls"), ("abc", "def")), case_sensitive=True).eval( manifest ), "Should read: in on some nulls column" - assert _ManifestEvalVisitor(schema, NotIn(Reference("no_nulls"), ("abc", "def"))).eval( + assert _ManifestEvalVisitor(schema, NotIn(Reference("no_nulls"), ("abc", "def")), case_sensitive=True).eval( manifest ), "Should read: in on no nulls column" diff --git a/tests/io/test_pyarrow.py b/tests/io/test_pyarrow.py index db25c0d134..79136be1c2 100644 --- a/tests/io/test_pyarrow.py +++ b/tests/io/test_pyarrow.py @@ -14,7 +14,7 @@ # KIND, either express or implied. See the License for the # specific language governing permissions and limitations # under the License. -# pylint: disable=protected-access,unused-argument +# pylint: disable=protected-access,unused-argument,redefined-outer-name import os import tempfile @@ -24,11 +24,33 @@ import pytest from pyarrow.fs import FileType +from pyiceberg.expressions import ( + AlwaysFalse, + AlwaysTrue, + And, + BoundEqualTo, + BoundGreaterThan, + BoundGreaterThanOrEqual, + BoundIn, + BoundIsNaN, + BoundIsNull, + BoundLessThan, + BoundLessThanOrEqual, + BoundNotEqualTo, + BoundNotIn, + BoundNotNaN, + BoundNotNull, + BoundReference, + Not, + Or, + literal, +) from pyiceberg.io import InputStream, OutputStream from pyiceberg.io.pyarrow import ( PyArrowFile, PyArrowFileIO, _ConvertToArrowSchema, + expression_to_pyarrow, schema_to_pyarrow, ) from pyiceberg.schema import Schema, visit @@ -44,6 +66,7 @@ ListType, LongType, MapType, + NestedField, StringType, TimestampType, TimestamptzType, @@ -411,3 +434,140 @@ def test_list_type_to_pyarrow(): element_required=True, ) assert visit(iceberg_map, _ConvertToArrowSchema()) == pa.list_(pa.int32()) + + +@pytest.fixture +def bound_reference(table_schema_simple: Schema) -> BoundReference[str]: + return BoundReference(table_schema_simple.find_field(1), table_schema_simple.accessor_for_field(1)) + + +@pytest.fixture +def bound_double_reference() -> BoundReference[float]: + schema = Schema( + NestedField(field_id=1, name="foo", field_type=DoubleType(), required=False), + schema_id=1, + identifier_field_ids=[2], + ) + return BoundReference(schema.find_field(1), schema.accessor_for_field(1)) + + +def test_expr_is_null_to_pyarrow(bound_reference: BoundReference[str]) -> None: + assert ( + repr(expression_to_pyarrow(BoundIsNull(bound_reference))) + == "" + ) + + +def test_expr_not_null_to_pyarrow(bound_reference: BoundReference[str]) -> None: + assert repr(expression_to_pyarrow(BoundNotNull(bound_reference))) == "" + + +def test_expr_is_nan_to_pyarrow(bound_double_reference: BoundReference[str]) -> None: + assert ( + repr(expression_to_pyarrow(BoundIsNaN(bound_double_reference))) + == "" + ) + + +def test_expr_not_nan_to_pyarrow(bound_double_reference: BoundReference[str]) -> None: + assert ( + repr(expression_to_pyarrow(BoundNotNaN(bound_double_reference))) + == "" + ) + + +def test_expr_equal_to_pyarrow(bound_reference: BoundReference[str]) -> None: + assert ( + repr(expression_to_pyarrow(BoundEqualTo(bound_reference, literal("hello")))) + == '' + ) + + +def test_expr_not_equal_to_pyarrow(bound_reference: BoundReference[str]) -> None: + assert ( + repr(expression_to_pyarrow(BoundNotEqualTo(bound_reference, literal("hello")))) + == '' + ) + + +def test_expr_greater_than_or_equal_equal_to_pyarrow(bound_reference: BoundReference[str]) -> None: + assert ( + repr(expression_to_pyarrow(BoundGreaterThanOrEqual(bound_reference, literal("hello")))) + == '= "hello")>' + ) + + +def test_expr_greater_than_to_pyarrow(bound_reference: BoundReference[str]) -> None: + assert ( + repr(expression_to_pyarrow(BoundGreaterThan(bound_reference, literal("hello")))) + == ' "hello")>' + ) + + +def test_expr_less_than_to_pyarrow(bound_reference: BoundReference[str]) -> None: + assert ( + repr(expression_to_pyarrow(BoundLessThan(bound_reference, literal("hello")))) + == '' + ) + + +def test_expr_less_than_or_equal_to_pyarrow(bound_reference: BoundReference[str]) -> None: + assert ( + repr(expression_to_pyarrow(BoundLessThanOrEqual(bound_reference, literal("hello")))) + == '' + ) + + +def test_expr_in_to_pyarrow(bound_reference: BoundReference[str]) -> None: + assert repr(expression_to_pyarrow(BoundIn(bound_reference, {literal("hello"), literal("world")}))) in ( + """""", + """""", + ) + + +def test_expr_not_in_to_pyarrow(bound_reference: BoundReference[str]) -> None: + assert repr(expression_to_pyarrow(BoundNotIn(bound_reference, {literal("hello"), literal("world")}))) in ( + """""", + """""", + ) + + +def test_and_to_pyarrow(bound_reference: BoundReference[str]) -> None: + assert ( + repr(expression_to_pyarrow(And(BoundEqualTo(bound_reference, literal("hello")), BoundIsNull(bound_reference)))) + == '' + ) + + +def test_or_to_pyarrow(bound_reference: BoundReference[str]) -> None: + assert ( + repr(expression_to_pyarrow(Or(BoundEqualTo(bound_reference, literal("hello")), BoundIsNull(bound_reference)))) + == '' + ) + + +def test_not_to_pyarrow(bound_reference: BoundReference[str]) -> None: + assert ( + repr(expression_to_pyarrow(Not(BoundEqualTo(bound_reference, literal("hello"))))) + == '' + ) + + +def test_always_true_to_pyarrow(bound_reference: BoundReference[str]) -> None: + assert repr(expression_to_pyarrow(AlwaysTrue())) == "" + + +def test_always_false_to_pyarrow(bound_reference: BoundReference[str]) -> None: + assert repr(expression_to_pyarrow(AlwaysFalse())) == "" From f7467d7260a0ea1efdd8be945228c733b807a205 Mon Sep 17 00:00:00 2001 From: Fokko Driesprong Date: Mon, 28 Nov 2022 21:45:24 +0100 Subject: [PATCH 282/642] Python: Update feature matrix (#6298) We actually don't have support for changing tables We have recently added support for planning scans --- mkdocs/docs/index.md | 26 +++++++++++++++----------- 1 file changed, 15 insertions(+), 11 deletions(-) diff --git a/mkdocs/docs/index.md b/mkdocs/docs/index.md index adbcc0e0c8..53795a8e07 100644 --- a/mkdocs/docs/index.md +++ b/mkdocs/docs/index.md @@ -525,17 +525,21 @@ The goal is that the python library will provide a functional, performant subset ## Metadata -| Operation | Java | Python | -|:------------------------|:-----:|:------:| -| Get Schema | X | X | -| Get Snapshots | X | X | -| Plan Scan | X | | -| Plan Scan for Snapshot | X | | -| Update Current Snapshot | X | | -| Set Table Properties | X | X | -| Create Table | X | X | -| Drop Table | X | X | -| Alter Table | X | | +| Operation | Java | Python | +|:-------------------------|:-----:|:------:| +| Get Schema | X | X | +| Get Snapshots | X | X | +| Plan Scan | X | X | +| Plan Scan for Snapshot | X | X | +| Update Current Snapshot | X | | +| Create Table | X | X | +| Rename Table | X | X | +| Drop Table | X | X | +| Alter Table | X | | +| Set Table Properties | X | | +| Create Namespace | X | X | +| Drop Namespace | X | X | +| Set Namespace Properties | X | X | ## Types From 20f43d1e37d300696a0e2ca958fd03c413162c47 Mon Sep 17 00:00:00 2001 From: Fokko Driesprong Date: Mon, 28 Nov 2022 21:47:27 +0100 Subject: [PATCH 283/642] Python: Bump FastAvro to 1.7.0 (#6294) Bumps [fastavro](https://github.com/fastavro/fastavro) from 1.6.1 to 1.7.0. - [Release notes](https://github.com/fastavro/fastavro/releases) - [Changelog](https://github.com/fastavro/fastavro/blob/master/ChangeLog) - [Commits](https://github.com/fastavro/fastavro/compare/1.6.1...1.7.0) Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- poetry.lock | 124 +++++++++++++++++++++++--------------- pyproject.toml | 2 +- tests/avro/test_reader.py | 22 +++---- tests/conftest.py | 44 +++++++------- 4 files changed, 108 insertions(+), 84 deletions(-) diff --git a/poetry.lock b/poetry.lock index 0338992a40..56f1da99c4 100644 --- a/poetry.lock +++ b/poetry.lock @@ -34,7 +34,7 @@ multidict = ">=4.5,<7.0" yarl = ">=1.0,<2.0" [package.extras] -speedups = ["aiodns", "brotli", "cchardet"] +speedups = ["Brotli", "aiodns", "cchardet"] [[package]] name = "aioitertools" @@ -75,10 +75,10 @@ optional = false python-versions = ">=3.5" [package.extras] -dev = ["coverage[toml] (>=5.0.2)", "hypothesis", "pympler", "pytest (>=4.3.0)", "mypy (>=0.900,!=0.940)", "pytest-mypy-plugins", "zope.interface", "furo", "sphinx", "sphinx-notfound-page", "pre-commit", "cloudpickle"] -docs = ["furo", "sphinx", "zope.interface", "sphinx-notfound-page"] -tests = ["coverage[toml] (>=5.0.2)", "hypothesis", "pympler", "pytest (>=4.3.0)", "mypy (>=0.900,!=0.940)", "pytest-mypy-plugins", "zope.interface", "cloudpickle"] -tests_no_zope = ["coverage[toml] (>=5.0.2)", "hypothesis", "pympler", "pytest (>=4.3.0)", "mypy (>=0.900,!=0.940)", "pytest-mypy-plugins", "cloudpickle"] +dev = ["cloudpickle", "coverage[toml] (>=5.0.2)", "furo", "hypothesis", "mypy (>=0.900,!=0.940)", "pre-commit", "pympler", "pytest (>=4.3.0)", "pytest-mypy-plugins", "sphinx", "sphinx-notfound-page", "zope.interface"] +docs = ["furo", "sphinx", "sphinx-notfound-page", "zope.interface"] +tests = ["cloudpickle", "coverage[toml] (>=5.0.2)", "hypothesis", "mypy (>=0.900,!=0.940)", "pympler", "pytest (>=4.3.0)", "pytest-mypy-plugins", "zope.interface"] +tests-no-zope = ["cloudpickle", "coverage[toml] (>=5.0.2)", "hypothesis", "mypy (>=0.900,!=0.940)", "pympler", "pytest (>=4.3.0)", "pytest-mypy-plugins"] [[package]] name = "boto3" @@ -128,7 +128,7 @@ tomli = {version = ">=1.0.0", markers = "python_version < \"3.11\""} [package.extras] docs = ["furo (>=2021.08.31)", "sphinx (>=4.0,<5.0)", "sphinx-argparse-cli (>=1.5)", "sphinx-autodoc-typehints (>=1.10)"] -test = ["filelock (>=3)", "pytest (>=6.2.4)", "pytest-cov (>=2.12)", "pytest-mock (>=2)", "pytest-rerunfailures (>=9.1)", "pytest-xdist (>=1.34)", "toml (>=0.10.0)", "wheel (>=0.36.0)", "setuptools (>=42.0.0)", "setuptools (>=56.0.0)"] +test = ["filelock (>=3)", "pytest (>=6.2.4)", "pytest-cov (>=2.12)", "pytest-mock (>=2)", "pytest-rerunfailures (>=9.1)", "pytest-xdist (>=1.34)", "setuptools (>=42.0.0)", "setuptools (>=56.0.0)", "toml (>=0.10.0)", "wheel (>=0.36.0)"] typing = ["importlib-metadata (>=4.6.4)", "mypy (==0.950)", "typing-extensions (>=3.7.4.3)"] virtualenv = ["virtualenv (>=20.0.35)"] @@ -168,7 +168,7 @@ optional = false python-versions = ">=3.6.0" [package.extras] -unicode_backport = ["unicodedata2"] +unicode-backport = ["unicodedata2"] [[package]] name = "click" @@ -227,11 +227,11 @@ cffi = ">=1.12" [package.extras] docs = ["sphinx (>=1.6.5,!=1.8.0,!=3.1.0,!=3.1.1)", "sphinx-rtd-theme"] -docstest = ["pyenchant (>=1.6.11)", "twine (>=1.12.0)", "sphinxcontrib-spelling (>=4.0.1)"] +docstest = ["pyenchant (>=1.6.11)", "sphinxcontrib-spelling (>=4.0.1)", "twine (>=1.12.0)"] pep8test = ["black", "flake8", "flake8-import-order", "pep8-naming"] sdist = ["setuptools-rust (>=0.11.4)"] ssh = ["bcrypt (>=3.1.5)"] -test = ["pytest (>=6.2.0)", "pytest-benchmark", "pytest-cov", "pytest-subtests", "pytest-xdist", "pretend", "iso8601", "pytz", "hypothesis (>=1.11.4,!=3.79.2)"] +test = ["hypothesis (>=1.11.4,!=3.79.2)", "iso8601", "pretend", "pytest (>=6.2.0)", "pytest-benchmark", "pytest-cov", "pytest-subtests", "pytest-xdist", "pytz"] [[package]] name = "distlib" @@ -273,14 +273,14 @@ test = ["pytest (>=6)"] [[package]] name = "fastavro" -version = "1.6.1" +version = "1.7.0" description = "Fast read/write of AVRO files" category = "dev" optional = false python-versions = ">=3.7" [package.extras] -codecs = ["python-snappy", "zstandard", "lz4"] +codecs = ["lz4", "python-snappy", "zstandard"] lz4 = ["lz4"] snappy = ["python-snappy"] zstandard = ["zstandard"] @@ -318,7 +318,7 @@ abfs = ["adlfs"] adl = ["adlfs"] arrow = ["pyarrow (>=1)"] dask = ["dask", "distributed"] -dropbox = ["dropboxdrivefs", "requests", "dropbox"] +dropbox = ["dropbox", "dropboxdrivefs", "requests"] entrypoints = ["importlib-metadata"] fuse = ["fusepy"] gcs = ["gcsfs"] @@ -327,7 +327,7 @@ github = ["requests"] gs = ["gcsfs"] gui = ["panel"] hdfs = ["pyarrow (>=1)"] -http = ["requests", "aiohttp (!=4.0.0a0,!=4.0.0a1)"] +http = ["aiohttp (!=4.0.0a0,!=4.0.0a1)", "requests"] libarchive = ["libarchive-c"] oci = ["ocifs"] s3 = ["s3fs"] @@ -367,9 +367,9 @@ python-versions = ">=3.7" zipp = ">=0.5" [package.extras] -docs = ["sphinx (>=3.5)", "jaraco.packaging (>=9)", "rst.linker (>=1.9)", "furo", "jaraco.tidelift (>=1.4)"] +docs = ["furo", "jaraco.packaging (>=9)", "jaraco.tidelift (>=1.4)", "rst.linker (>=1.9)", "sphinx (>=3.5)"] perf = ["ipython"] -testing = ["pytest (>=6)", "pytest-checkdocs (>=2.4)", "pytest-flake8", "flake8 (<5)", "pytest-cov", "pytest-enabler (>=1.3)", "packaging", "pyfakefs", "flufl.flake8", "pytest-perf (>=0.9.2)", "pytest-black (>=0.3.7)", "pytest-mypy (>=0.9.1)", "importlib-resources (>=1.3)"] +testing = ["flake8 (<5)", "flufl.flake8", "importlib-resources (>=1.3)", "packaging", "pyfakefs", "pytest (>=6)", "pytest-black (>=0.3.7)", "pytest-checkdocs (>=2.4)", "pytest-cov", "pytest-enabler (>=1.3)", "pytest-flake8", "pytest-mypy (>=0.9.1)", "pytest-perf (>=0.9.2)"] [[package]] name = "iniconfig" @@ -439,14 +439,14 @@ werkzeug = ">=0.5,<2.2.0 || >2.2.0,<2.2.1 || >2.2.1" xmltodict = "*" [package.extras] -all = ["PyYAML (>=5.1)", "python-jose[cryptography] (>=3.1.0,<4.0.0)", "ecdsa (!=0.15)", "docker (>=2.5.1)", "graphql-core", "jsondiff (>=1.1.2)", "aws-xray-sdk (>=0.93,!=0.96)", "idna (>=2.5,<4)", "cfn-lint (>=0.40.0)", "sshpubkeys (>=3.1.0)", "pyparsing (>=3.0.7)", "openapi-spec-validator (>=0.2.8)", "setuptools"] -apigateway = ["PyYAML (>=5.1)", "python-jose[cryptography] (>=3.1.0,<4.0.0)", "ecdsa (!=0.15)", "openapi-spec-validator (>=0.2.8)"] +all = ["PyYAML (>=5.1)", "aws-xray-sdk (>=0.93,!=0.96)", "cfn-lint (>=0.40.0)", "docker (>=2.5.1)", "ecdsa (!=0.15)", "graphql-core", "idna (>=2.5,<4)", "jsondiff (>=1.1.2)", "openapi-spec-validator (>=0.2.8)", "pyparsing (>=3.0.7)", "python-jose[cryptography] (>=3.1.0,<4.0.0)", "setuptools", "sshpubkeys (>=3.1.0)"] +apigateway = ["PyYAML (>=5.1)", "ecdsa (!=0.15)", "openapi-spec-validator (>=0.2.8)", "python-jose[cryptography] (>=3.1.0,<4.0.0)"] apigatewayv2 = ["PyYAML (>=5.1)"] appsync = ["graphql-core"] awslambda = ["docker (>=2.5.1)"] batch = ["docker (>=2.5.1)"] -cloudformation = ["PyYAML (>=5.1)", "python-jose[cryptography] (>=3.1.0,<4.0.0)", "ecdsa (!=0.15)", "docker (>=2.5.1)", "graphql-core", "jsondiff (>=1.1.2)", "aws-xray-sdk (>=0.93,!=0.96)", "idna (>=2.5,<4)", "cfn-lint (>=0.40.0)", "sshpubkeys (>=3.1.0)", "pyparsing (>=3.0.7)", "openapi-spec-validator (>=0.2.8)", "setuptools"] -cognitoidp = ["python-jose[cryptography] (>=3.1.0,<4.0.0)", "ecdsa (!=0.15)"] +cloudformation = ["PyYAML (>=5.1)", "aws-xray-sdk (>=0.93,!=0.96)", "cfn-lint (>=0.40.0)", "docker (>=2.5.1)", "ecdsa (!=0.15)", "graphql-core", "idna (>=2.5,<4)", "jsondiff (>=1.1.2)", "openapi-spec-validator (>=0.2.8)", "pyparsing (>=3.0.7)", "python-jose[cryptography] (>=3.1.0,<4.0.0)", "setuptools", "sshpubkeys (>=3.1.0)"] +cognitoidp = ["ecdsa (!=0.15)", "python-jose[cryptography] (>=3.1.0,<4.0.0)"] ds = ["sshpubkeys (>=3.1.0)"] dynamodb = ["docker (>=2.5.1)"] dynamodb2 = ["docker (>=2.5.1)"] @@ -458,7 +458,7 @@ glue = ["pyparsing (>=3.0.7)"] iotdata = ["jsondiff (>=1.1.2)"] route53resolver = ["sshpubkeys (>=3.1.0)"] s3 = ["PyYAML (>=5.1)"] -server = ["PyYAML (>=5.1)", "python-jose[cryptography] (>=3.1.0,<4.0.0)", "ecdsa (!=0.15)", "docker (>=2.5.1)", "graphql-core", "jsondiff (>=1.1.2)", "aws-xray-sdk (>=0.93,!=0.96)", "idna (>=2.5,<4)", "cfn-lint (>=0.40.0)", "sshpubkeys (>=3.1.0)", "pyparsing (>=3.0.7)", "openapi-spec-validator (>=0.2.8)", "setuptools", "flask (!=2.2.0,!=2.2.1)", "flask-cors"] +server = ["PyYAML (>=5.1)", "aws-xray-sdk (>=0.93,!=0.96)", "cfn-lint (>=0.40.0)", "docker (>=2.5.1)", "ecdsa (!=0.15)", "flask (!=2.2.0,!=2.2.1)", "flask-cors", "graphql-core", "idna (>=2.5,<4)", "jsondiff (>=1.1.2)", "openapi-spec-validator (>=0.2.8)", "pyparsing (>=3.0.7)", "python-jose[cryptography] (>=3.1.0,<4.0.0)", "setuptools", "sshpubkeys (>=3.1.0)"] ssm = ["PyYAML (>=5.1)", "dataclasses"] xray = ["aws-xray-sdk (>=0.93,!=0.96)", "setuptools"] @@ -478,6 +478,9 @@ category = "dev" optional = false python-versions = ">=2.7,!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,!=3.4.*,!=3.5.*,!=3.6.*" +[package.dependencies] +setuptools = "*" + [[package]] name = "numpy" version = "1.23.5" @@ -517,8 +520,8 @@ optional = false python-versions = ">=3.7" [package.extras] -docs = ["furo (>=2022.9.29)", "proselint (>=0.13)", "sphinx-autodoc-typehints (>=1.19.4)", "sphinx (>=5.3)"] -test = ["appdirs (==1.4.4)", "pytest-cov (>=4)", "pytest-mock (>=3.10)", "pytest (>=7.2)"] +docs = ["furo (>=2022.9.29)", "proselint (>=0.13)", "sphinx (>=5.3)", "sphinx-autodoc-typehints (>=1.19.4)"] +test = ["appdirs (==1.4.4)", "pytest (>=7.2)", "pytest-cov (>=4)", "pytest-mock (>=3.10)"] [[package]] name = "pluggy" @@ -602,7 +605,7 @@ optional = false python-versions = ">=3.6.8" [package.extras] -diagrams = ["railroad-diagrams", "jinja2"] +diagrams = ["jinja2", "railroad-diagrams"] [[package]] name = "pytest" @@ -638,8 +641,8 @@ docutils = ">=0.15" importlib-metadata = {version = ">=4", markers = "python_version < \"3.10\""} [package.extras] -docs = ["sphinx (>=3.5)", "jaraco.packaging (>=9)", "rst.linker (>=1.9)", "furo"] -testing = ["pytest (>=6)", "pytest-checkdocs (>=2.4)", "pytest-flake8", "flake8 (<5)", "pytest-cov", "pytest-enabler (>=1.3)", "types-docutils", "pytest-black (>=0.3.7)", "pytest-mypy (>=0.9.1)"] +docs = ["furo", "jaraco.packaging (>=9)", "rst.linker (>=1.9)", "sphinx (>=3.5)"] +testing = ["flake8 (<5)", "pytest (>=6)", "pytest-black (>=0.3.7)", "pytest-checkdocs (>=2.4)", "pytest-cov", "pytest-enabler (>=1.3)", "pytest-flake8", "pytest-mypy (>=0.9.1)", "types-docutils"] [[package]] name = "python-dateutil" @@ -692,7 +695,7 @@ urllib3 = ">=1.21.1,<1.27" [package.extras] socks = ["PySocks (>=1.5.6,!=1.5.7)"] -use_chardet_on_py3 = ["chardet (>=3.0.2,<6)"] +use-chardet-on-py3 = ["chardet (>=3.0.2,<6)"] [[package]] name = "requests-mock" @@ -708,7 +711,7 @@ six = "*" [package.extras] fixture = ["fixtures"] -test = ["fixtures", "mock", "purl", "pytest", "sphinx", "testrepository (>=0.0.18)", "testtools", "requests-futures"] +test = ["fixtures", "mock", "purl", "pytest", "requests-futures", "sphinx", "testrepository (>=0.0.18)", "testtools"] [[package]] name = "responses" @@ -725,7 +728,7 @@ types-toml = "*" urllib3 = ">=1.25.10" [package.extras] -tests = ["pytest (>=7.0.0)", "coverage (>=6.0.0)", "pytest-cov", "pytest-asyncio", "pytest-httpserver", "flake8", "types-requests", "mypy"] +tests = ["coverage (>=6.0.0)", "flake8", "mypy", "pytest (>=7.0.0)", "pytest-asyncio", "pytest-cov", "pytest-httpserver", "types-requests"] [[package]] name = "rich" @@ -774,6 +777,19 @@ botocore = ">=1.12.36,<2.0a.0" [package.extras] crt = ["botocore[crt] (>=1.20.29,<2.0a.0)"] +[[package]] +name = "setuptools" +version = "65.6.0" +description = "Easily download, build, install, upgrade, and uninstall Python packages" +category = "dev" +optional = false +python-versions = ">=3.7" + +[package.extras] +docs = ["furo", "jaraco.packaging (>=9)", "jaraco.tidelift (>=1.4)", "pygments-github-lexers (==0.0.5)", "rst.linker (>=1.9)", "sphinx (>=3.5)", "sphinx-favicon", "sphinx-hoverxref (<2)", "sphinx-inline-tabs", "sphinx-notfound-page (==0.8.3)", "sphinx-reredirects", "sphinxcontrib-towncrier"] +testing = ["build[virtualenv]", "filelock (>=3.4.0)", "flake8 (<5)", "flake8-2020", "ini2toml[lite] (>=0.9)", "jaraco.envs (>=2.2)", "jaraco.path (>=3.2.0)", "pip (>=19.1)", "pip-run (>=8.8)", "pytest (>=6)", "pytest-black (>=0.3.7)", "pytest-checkdocs (>=2.4)", "pytest-cov", "pytest-enabler (>=1.3)", "pytest-flake8", "pytest-mypy (>=0.9.1)", "pytest-perf", "pytest-timeout", "pytest-xdist", "tomli-w (>=1.0.0)", "virtualenv (>=13.0.0)", "wheel"] +testing-integration = ["build[virtualenv]", "filelock (>=3.4.0)", "jaraco.envs (>=2.2)", "jaraco.path (>=3.2.0)", "pytest", "pytest-enabler", "pytest-xdist", "tomli", "virtualenv (>=13.0.0)", "wheel"] + [[package]] name = "six" version = "1.16.0" @@ -839,8 +855,8 @@ optional = false python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*, !=3.5.*, <4" [package.extras] -brotli = ["brotlicffi (>=0.8.0)", "brotli (>=1.0.9)", "brotlipy (>=0.6.0)"] -secure = ["pyOpenSSL (>=0.14)", "cryptography (>=1.3.4)", "idna (>=2.0.0)", "certifi", "urllib3-secure-extra", "ipaddress"] +brotli = ["brotli (>=1.0.9)", "brotlicffi (>=0.8.0)", "brotlipy (>=0.6.0)"] +secure = ["certifi", "cryptography (>=1.3.4)", "idna (>=2.0.0)", "ipaddress", "pyOpenSSL (>=0.14)", "urllib3-secure-extra"] socks = ["PySocks (>=1.5.6,!=1.5.7,<2.0)"] [[package]] @@ -911,8 +927,8 @@ optional = false python-versions = ">=3.7" [package.extras] -docs = ["sphinx (>=3.5)", "jaraco.packaging (>=9)", "rst.linker (>=1.9)", "furo", "jaraco.tidelift (>=1.4)"] -testing = ["pytest (>=6)", "pytest-checkdocs (>=2.4)", "pytest-flake8", "flake8 (<5)", "pytest-cov", "pytest-enabler (>=1.3)", "jaraco.itertools", "func-timeout", "jaraco.functools", "more-itertools", "pytest-black (>=0.3.7)", "pytest-mypy (>=0.9.1)"] +docs = ["furo", "jaraco.packaging (>=9)", "jaraco.tidelift (>=1.4)", "rst.linker (>=1.9)", "sphinx (>=3.5)"] +testing = ["flake8 (<5)", "func-timeout", "jaraco.functools", "jaraco.itertools", "more-itertools", "pytest (>=6)", "pytest-black (>=0.3.7)", "pytest-checkdocs (>=2.4)", "pytest-cov", "pytest-enabler (>=1.3)", "pytest-flake8", "pytest-mypy (>=0.9.1)"] [[package]] name = "zstandard" @@ -939,7 +955,7 @@ snappy = ["python-snappy"] [metadata] lock-version = "1.1" python-versions = "^3.8" -content-hash = "704760d1f2db5291b48b9fd04e48bdd0f3253b06d1d40a34403cc95d3f12a705" +content-hash = "ad000d110de01a9e85f226ed0fadc5c74570bdb3cc6d30f38f54d18a11d51d36" [metadata.files] aiobotocore = [ @@ -1295,23 +1311,27 @@ exceptiongroup = [ {file = "exceptiongroup-1.0.4.tar.gz", hash = "sha256:bd14967b79cd9bdb54d97323216f8fdf533e278df937aa2a90089e7d6e06e5ec"}, ] fastavro = [ - {file = "fastavro-1.6.1-cp310-cp310-macosx_10_14_x86_64.whl", hash = "sha256:de244146a3d7bc4e60c7be11f317054ee0e57ca807d85923bd8407981488ff18"}, - {file = "fastavro-1.6.1-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:1ed84fb27672e54b0bafecae1f128f6e2950828f451e59f93bb74eeccb31b496"}, - {file = "fastavro-1.6.1-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:98df91eff0c061ffeb722732ba61d4ea9c7afd0f5403534991a984e75e3b2272"}, - {file = "fastavro-1.6.1-cp310-cp310-win_amd64.whl", hash = "sha256:fd60999204f60d5bcc4e9a4461ce867efb23d271e955684b36c04970750e2ccb"}, - {file = "fastavro-1.6.1-cp37-cp37m-macosx_10_15_x86_64.whl", hash = "sha256:19d4e83bf952fc2893d11d4c82cf04c3938c921a7c5bf277256cda1005e59f16"}, - {file = "fastavro-1.6.1-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:945d5f6e7a3076758b52fe1cebecfd86b66e6925a3a38732e93f3e1eab955256"}, - {file = "fastavro-1.6.1-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:315084c45339845273193545a50f3d616787eb3231b0b1f3df229196b22f192e"}, - {file = "fastavro-1.6.1-cp37-cp37m-win_amd64.whl", hash = "sha256:3ef81602dc6ecac3d646d64fe13aeb11252f75b76141b5b3d755bbf548313718"}, - {file = "fastavro-1.6.1-cp38-cp38-macosx_10_15_x86_64.whl", hash = "sha256:462c3177aa231f8b023d4af0a73f41517df8160399564fc750a711faa0889b59"}, - {file = "fastavro-1.6.1-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:3dd8c98d410fbb46a475242cea1b86061d2568db89919eb9079a9a183f04d444"}, - {file = "fastavro-1.6.1-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:b8b77e8f33e55d8f63d65f58a2b2c99248093e2da2e8a86f58320f5715fc3f54"}, - {file = "fastavro-1.6.1-cp38-cp38-win_amd64.whl", hash = "sha256:ac31f490e3030a6c608fffa6e337bd3f1b00f227bcb098df88a6e3e38982dfa5"}, - {file = "fastavro-1.6.1-cp39-cp39-macosx_10_15_x86_64.whl", hash = "sha256:3e479b9e4b5e6184b57daab57f1992e649590e902d26b7e2ef50a605626e9053"}, - {file = "fastavro-1.6.1-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:f2ec2fa5f100e0f1010f550de6de8b1019a683c5034613772b4444d7c85bdb57"}, - {file = "fastavro-1.6.1-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:1b34c4d33e22ada2fa19566a43976b3f5ca937ab6e054e65e1467bf67353f352"}, - {file = "fastavro-1.6.1-cp39-cp39-win_amd64.whl", hash = "sha256:18f0ddf3cc4f09839c08eec67ed3be3d391894874cb87f856a0362540d18df17"}, - {file = "fastavro-1.6.1.tar.gz", hash = "sha256:bc37a6edbde7a04a9df28ab838b94df7527e5eea648d61633233abba59205146"}, + {file = "fastavro-1.7.0-cp310-cp310-macosx_10_14_x86_64.whl", hash = "sha256:ab3387a06e272980fa034f5c62f7063977b77df6416d3d30a4d3b49cc8827566"}, + {file = "fastavro-1.7.0-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:216132bc54da19e97e1531dd69c86282408d4c797749d83b01b3a00862a180de"}, + {file = "fastavro-1.7.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:c20cf6cd8098bb93c2cffd53d03ccea1dcf9ec594a5c83963acf29a2882f8693"}, + {file = "fastavro-1.7.0-cp310-cp310-win_amd64.whl", hash = "sha256:54b60e79506a456bcfc940560fa2c73a7a8e3ddc58a1ae4d94fdd99f6b02aef0"}, + {file = "fastavro-1.7.0-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:4da6d83abd04a804310667f8d1fc23d44e9ed91ed9a9bc9c1fcd906e0c403b12"}, + {file = "fastavro-1.7.0-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:7c664302f94064adeb93403c61c74e07b9710526403eba3b59169f99bb99c55c"}, + {file = "fastavro-1.7.0-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:7aab162f02e64bf82d0282430a3c6ec7a36982b1c5d479e7dcc278e6d62a84b8"}, + {file = "fastavro-1.7.0-cp311-cp311-win_amd64.whl", hash = "sha256:63d0d2a2bb3e85d006c834633be51b105a50b0dc7cc8423b06f30601b532adf4"}, + {file = "fastavro-1.7.0-cp37-cp37m-macosx_10_15_x86_64.whl", hash = "sha256:a07a3245049529d6d9028d664361cc4990e74d035d2303875295e2f7b97eba09"}, + {file = "fastavro-1.7.0-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:7df57d31cb58770a9066790250e9f4ec91242c69e1dc62ea732a6fb2406a8f96"}, + {file = "fastavro-1.7.0-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:b905634b5763ca08c9f7b51486e2c3ae7907f5d9bc48208c69b16ccbe8455e90"}, + {file = "fastavro-1.7.0-cp37-cp37m-win_amd64.whl", hash = "sha256:749206f1cec3d7429546e49db5708f4571497d35181b6b334c4844133f230515"}, + {file = "fastavro-1.7.0-cp38-cp38-macosx_10_15_x86_64.whl", hash = "sha256:a0ebd5c1269085179de4b3f072de274fb66a471ecbc5245bd8684e6f94943c2f"}, + {file = "fastavro-1.7.0-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a7e35b94b692f8cca0096c89abf1937efed66252dea0b3b3165babfb3c289fb7"}, + {file = "fastavro-1.7.0-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:202424a5a7831b773f0f2cc2f82e89ed1726051fd5994f13dc678514144e10d4"}, + {file = "fastavro-1.7.0-cp38-cp38-win_amd64.whl", hash = "sha256:601f8c2ec166bd721f4b12eafe195dd1373d3f8dce4fe2425abd2df3e3968ac7"}, + {file = "fastavro-1.7.0-cp39-cp39-macosx_10_15_x86_64.whl", hash = "sha256:91cc9be740abca894082af4fe3ab9db057d4e5fa783cfa9a94c02ec041bf4346"}, + {file = "fastavro-1.7.0-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:9e4463b00242e4baf52d499aeefab46a26d9dd18d808d4219cd4d21089da540e"}, + {file = "fastavro-1.7.0-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:f7348d318858203bd108e6bcde177d8a6f0590b52bc624d815f26fb6c37029bb"}, + {file = "fastavro-1.7.0-cp39-cp39-win_amd64.whl", hash = "sha256:492e8902063aa1c73170e11c18495fcaa86b71eae3513ef83ba438ca02b16b34"}, + {file = "fastavro-1.7.0.tar.gz", hash = "sha256:4b1205f46489b4032d3155c1ab44d9824be0c7454df98d3a5bd22b78b98f23c8"}, ] filelock = [ {file = "filelock-3.8.0-py3-none-any.whl", hash = "sha256:617eb4e5eedc82fc5f47b6d61e4d11cb837c56cb4544e39081099fa17ad109d4"}, @@ -1824,6 +1844,10 @@ s3transfer = [ {file = "s3transfer-0.6.0-py3-none-any.whl", hash = "sha256:06176b74f3a15f61f1b4f25a1fc29a4429040b7647133a463da8fa5bd28d5ecd"}, {file = "s3transfer-0.6.0.tar.gz", hash = "sha256:2ed07d3866f523cc561bf4a00fc5535827981b117dd7876f036b0c1aca42c947"}, ] +setuptools = [ + {file = "setuptools-65.6.0-py3-none-any.whl", hash = "sha256:6211d2f5eddad8757bd0484923ca7c0a6302ebc4ab32ea5e94357176e0ca0840"}, + {file = "setuptools-65.6.0.tar.gz", hash = "sha256:d1eebf881c6114e51df1664bc2c9133d022f78d12d5f4f665b9191f084e2862d"}, +] six = [ {file = "six-1.16.0-py2.py3-none-any.whl", hash = "sha256:8abb2f1d86890a2dfb989f9a77cfcfd3e47c2a354b01111771326f8aa26e0254"}, {file = "six-1.16.0.tar.gz", hash = "sha256:1e61c37477a1626458e36f7b1d82aa5c9b094fa4802892072e49de9c60c4c926"}, diff --git a/pyproject.toml b/pyproject.toml index 1374abff03..0312018be8 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -72,7 +72,7 @@ boto3 = {version = "1.24.59", optional = true} pytest = "7.2.0" pytest-checkdocs = "2.9.0" pre-commit = "2.20.0" -fastavro = "1.6.1" +fastavro = "1.7.0" coverage = { version = "^6.5.0", extras = ["toml"] } requests-mock = "1.10.0" moto = "^4.0.6" diff --git a/tests/avro/test_reader.py b/tests/avro/test_reader.py index 77e186b4f7..ac0d001abc 100644 --- a/tests/avro/test_reader.py +++ b/tests/avro/test_reader.py @@ -71,7 +71,7 @@ def test_read_header(generated_manifest_entry_file: str, iceberg_manifest_entry_ "name": "manifest_entry", "fields": [ {"field-id": 0, "name": "status", "type": "int"}, - {"field-id": 1, "default": "null", "name": "snapshot_id", "type": ["null", "long"]}, + {"field-id": 1, "default": None, "name": "snapshot_id", "type": ["null", "long"]}, { "field-id": 2, "name": "data_file", @@ -92,7 +92,7 @@ def test_read_header(generated_manifest_entry_file: str, iceberg_manifest_entry_ "type": { "type": "record", "name": "r102", - "fields": [{"field-id": 1000, "default": "null", "name": "VendorID", "type": ["null", "int"]}], + "fields": [{"field-id": 1000, "default": None, "name": "VendorID", "type": ["null", "int"]}], }, }, {"field-id": 103, "doc": "Number of records in the file", "name": "record_count", "type": "long"}, @@ -101,7 +101,7 @@ def test_read_header(generated_manifest_entry_file: str, iceberg_manifest_entry_ { "field-id": 108, "doc": "Map of column id to total size on disk", - "default": "null", + "default": None, "name": "column_sizes", "type": [ "null", @@ -122,7 +122,7 @@ def test_read_header(generated_manifest_entry_file: str, iceberg_manifest_entry_ { "field-id": 109, "doc": "Map of column id to total count, including null and NaN", - "default": "null", + "default": None, "name": "value_counts", "type": [ "null", @@ -143,7 +143,7 @@ def test_read_header(generated_manifest_entry_file: str, iceberg_manifest_entry_ { "field-id": 110, "doc": "Map of column id to null value count", - "default": "null", + "default": None, "name": "null_value_counts", "type": [ "null", @@ -164,7 +164,7 @@ def test_read_header(generated_manifest_entry_file: str, iceberg_manifest_entry_ { "field-id": 137, "doc": "Map of column id to number of NaN values in the column", - "default": "null", + "default": None, "name": "nan_value_counts", "type": [ "null", @@ -185,7 +185,7 @@ def test_read_header(generated_manifest_entry_file: str, iceberg_manifest_entry_ { "field-id": 125, "doc": "Map of column id to lower bound", - "default": "null", + "default": None, "name": "lower_bounds", "type": [ "null", @@ -206,7 +206,7 @@ def test_read_header(generated_manifest_entry_file: str, iceberg_manifest_entry_ { "field-id": 128, "doc": "Map of column id to upper bound", - "default": "null", + "default": None, "name": "upper_bounds", "type": [ "null", @@ -227,21 +227,21 @@ def test_read_header(generated_manifest_entry_file: str, iceberg_manifest_entry_ { "field-id": 131, "doc": "Encryption key metadata blob", - "default": "null", + "default": None, "name": "key_metadata", "type": ["null", "bytes"], }, { "field-id": 132, "doc": "Splittable offsets", - "default": "null", + "default": None, "name": "split_offsets", "type": ["null", {"element-id": 133, "type": "array", "items": "long"}], }, { "field-id": 140, "doc": "Sort order ID", - "default": "null", + "default": None, "name": "sort_order_id", "type": ["null", "int"], }, diff --git a/tests/conftest.py b/tests/conftest.py index d11d155b47..59f7ccc723 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -604,28 +604,28 @@ def avro_schema_manifest_file() -> Dict[str, Any]: "name": "added_snapshot_id", "type": ["null", "long"], "doc": "Snapshot ID that added the manifest", - "default": "null", + "default": None, "field-id": 503, }, { "name": "added_data_files_count", "type": ["null", "int"], "doc": "Added entry count", - "default": "null", + "default": None, "field-id": 504, }, { "name": "existing_data_files_count", "type": ["null", "int"], "doc": "Existing entry count", - "default": "null", + "default": None, "field-id": 505, }, { "name": "deleted_data_files_count", "type": ["null", "int"], "doc": "Deleted entry count", - "default": "null", + "default": None, "field-id": 506, }, { @@ -648,21 +648,21 @@ def avro_schema_manifest_file() -> Dict[str, Any]: "name": "contains_nan", "type": ["null", "boolean"], "doc": "True if any file has a nan partition value", - "default": "null", + "default": None, "field-id": 518, }, { "name": "lower_bound", "type": ["null", "bytes"], "doc": "Partition lower bound for all files", - "default": "null", + "default": None, "field-id": 510, }, { "name": "upper_bound", "type": ["null", "bytes"], "doc": "Partition upper bound for all files", - "default": "null", + "default": None, "field-id": 511, }, ], @@ -671,22 +671,22 @@ def avro_schema_manifest_file() -> Dict[str, Any]: }, ], "doc": "Summary for each partition", - "default": "null", + "default": None, "field-id": 507, }, - {"name": "added_rows_count", "type": ["null", "long"], "doc": "Added rows count", "default": "null", "field-id": 512}, + {"name": "added_rows_count", "type": ["null", "long"], "doc": "Added rows count", "default": None, "field-id": 512}, { "name": "existing_rows_count", "type": ["null", "long"], "doc": "Existing rows count", - "default": "null", + "default": None, "field-id": 513, }, { "name": "deleted_rows_count", "type": ["null", "long"], "doc": "Deleted rows count", - "default": "null", + "default": None, "field-id": 514, }, ], @@ -700,7 +700,7 @@ def avro_schema_manifest_entry() -> Dict[str, Any]: "name": "manifest_entry", "fields": [ {"name": "status", "type": "int", "field-id": 0}, - {"name": "snapshot_id", "type": ["null", "long"], "default": "null", "field-id": 1}, + {"name": "snapshot_id", "type": ["null", "long"], "default": None, "field-id": 1}, { "name": "data_file", "type": { @@ -719,7 +719,7 @@ def avro_schema_manifest_entry() -> Dict[str, Any]: "type": { "type": "record", "name": "r102", - "fields": [{"name": "VendorID", "type": ["null", "int"], "default": "null", "field-id": 1000}], + "fields": [{"name": "VendorID", "type": ["null", "int"], "default": None, "field-id": 1000}], }, "field-id": 102, }, @@ -744,7 +744,7 @@ def avro_schema_manifest_entry() -> Dict[str, Any]: }, ], "doc": "Map of column id to total size on disk", - "default": "null", + "default": None, "field-id": 108, }, { @@ -765,7 +765,7 @@ def avro_schema_manifest_entry() -> Dict[str, Any]: }, ], "doc": "Map of column id to total count, including null and NaN", - "default": "null", + "default": None, "field-id": 109, }, { @@ -786,7 +786,7 @@ def avro_schema_manifest_entry() -> Dict[str, Any]: }, ], "doc": "Map of column id to null value count", - "default": "null", + "default": None, "field-id": 110, }, { @@ -807,7 +807,7 @@ def avro_schema_manifest_entry() -> Dict[str, Any]: }, ], "doc": "Map of column id to number of NaN values in the column", - "default": "null", + "default": None, "field-id": 137, }, { @@ -828,7 +828,7 @@ def avro_schema_manifest_entry() -> Dict[str, Any]: }, ], "doc": "Map of column id to lower bound", - "default": "null", + "default": None, "field-id": 125, }, { @@ -849,28 +849,28 @@ def avro_schema_manifest_entry() -> Dict[str, Any]: }, ], "doc": "Map of column id to upper bound", - "default": "null", + "default": None, "field-id": 128, }, { "name": "key_metadata", "type": ["null", "bytes"], "doc": "Encryption key metadata blob", - "default": "null", + "default": None, "field-id": 131, }, { "name": "split_offsets", "type": ["null", {"type": "array", "items": "long", "element-id": 133}], "doc": "Splittable offsets", - "default": "null", + "default": None, "field-id": 132, }, { "name": "sort_order_id", "type": ["null", "int"], "doc": "Sort order ID", - "default": "null", + "default": None, "field-id": 140, }, ], From 916197a59896c0dbe9793a276406439f91399bf9 Mon Sep 17 00:00:00 2001 From: Ryan Blue Date: Mon, 28 Nov 2022 13:05:09 -0800 Subject: [PATCH 284/642] Python: Add boolean expression parser (#6259) Co-authored-by: Fokko Driesprong --- pyiceberg/expressions/literals.py | 36 +++++ pyiceberg/expressions/parser.py | 237 +++++++++++++++++++++++++++++ pyproject.toml | 4 + tests/expressions/test_literals.py | 4 - tests/expressions/test_parser.py | 151 ++++++++++++++++++ 5 files changed, 428 insertions(+), 4 deletions(-) create mode 100644 pyiceberg/expressions/parser.py create mode 100644 tests/expressions/test_parser.py diff --git a/pyiceberg/expressions/literals.py b/pyiceberg/expressions/literals.py index c59c6bcf8d..44ddc1331e 100644 --- a/pyiceberg/expressions/literals.py +++ b/pyiceberg/expressions/literals.py @@ -86,6 +86,8 @@ def __hash__(self) -> int: return hash(self.value) def __eq__(self, other: Any) -> bool: + if not isinstance(other, Literal): + return False return self.value == other.value def __ne__(self, other) -> bool: @@ -401,6 +403,40 @@ def _(self, type_var: DecimalType) -> Literal[Decimal]: return self raise ValueError(f"Could not convert {self.value} into a {type_var}") + @to.register(IntegerType) + def _(self, _: IntegerType) -> Literal[int]: + value_int = int(self.value.to_integral_value()) + if value_int > IntegerType.max: + return IntAboveMax() + elif value_int < IntegerType.min: + return IntBelowMin() + else: + return LongLiteral(value_int) + + @to.register(LongType) + def _(self, _: LongType) -> Literal[int]: + value_int = int(self.value.to_integral_value()) + if value_int > LongType.max: + return IntAboveMax() + elif value_int < LongType.min: + return IntBelowMin() + else: + return LongLiteral(value_int) + + @to.register(FloatType) + def _(self, _: FloatType): + value_float = float(self.value) + if value_float > FloatType.max: + return FloatAboveMax() + elif value_float < FloatType.min: + return FloatBelowMin() + else: + return FloatLiteral(value_float) + + @to.register(DoubleType) + def _(self, _: DoubleLiteral): + return DoubleLiteral(float(self.value)) + class StringLiteral(Literal[str]): def __init__(self, value: str): diff --git a/pyiceberg/expressions/parser.py b/pyiceberg/expressions/parser.py new file mode 100644 index 0000000000..0a0173da41 --- /dev/null +++ b/pyiceberg/expressions/parser.py @@ -0,0 +1,237 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +from decimal import Decimal + +from pyparsing import ( + CaselessKeyword, + Group, + ParserElement, + ParseResults, + Suppress, + Word, + alphanums, + alphas, + delimited_list, + infix_notation, + one_of, + opAssoc, + sgl_quoted_string, +) +from pyparsing.common import pyparsing_common as common + +from pyiceberg.expressions import ( + AlwaysFalse, + AlwaysTrue, + And, + BooleanExpression, + EqualTo, + GreaterThan, + GreaterThanOrEqual, + In, + IsNaN, + IsNull, + LessThan, + LessThanOrEqual, + Not, + NotEqualTo, + NotIn, + NotNaN, + NotNull, + Or, + Reference, +) +from pyiceberg.expressions.literals import ( + DecimalLiteral, + Literal, + LongLiteral, + StringLiteral, +) +from pyiceberg.typedef import L + +ParserElement.enablePackrat() + +AND = CaselessKeyword("and") +OR = CaselessKeyword("or") +NOT = CaselessKeyword("not") +IS = CaselessKeyword("is") +IN = CaselessKeyword("in") +NULL = CaselessKeyword("null") +NAN = CaselessKeyword("nan") + +identifier = Word(alphas, alphanums + "_$").set_results_name("identifier") +column = delimited_list(identifier, delim=".", combine=True).set_results_name("column") + + +@column.set_parse_action +def _(result: ParseResults) -> Reference: + return Reference(result.column[0]) + + +boolean = one_of(["true", "false"], caseless=True).set_results_name("boolean") +string = sgl_quoted_string.set_results_name("raw_quoted_string") +decimal = common.real().set_results_name("decimal") +integer = common.signed_integer().set_results_name("integer") +literal = Group(string | decimal | integer).set_results_name("literal") +literal_set = Group(delimited_list(string) | delimited_list(decimal) | delimited_list(integer)).set_results_name("literal_set") + + +@boolean.set_parse_action +def _(result: ParseResults) -> BooleanExpression: + if "true" == result.boolean.lower(): + return AlwaysTrue() + else: + return AlwaysFalse() + + +@string.set_parse_action +def _(result: ParseResults) -> Literal[str]: + return StringLiteral(result.raw_quoted_string[1:-1].replace("''", "'")) + + +@decimal.set_parse_action +def _(result: ParseResults) -> Literal[Decimal]: + return DecimalLiteral(Decimal(result.decimal)) + + +@integer.set_parse_action +def _(result: ParseResults) -> Literal[int]: + return LongLiteral(int(result.integer)) + + +@literal.set_parse_action +def _(result: ParseResults) -> Literal[L]: + return result[0][0] + + +@literal_set.set_parse_action +def _(result: ParseResults) -> Literal[L]: + return result[0] + + +comparison_op = one_of(["<", "<=", ">", ">=", "=", "==", "!=", "<>"], caseless=True).set_results_name("op") +left_ref = column + comparison_op + literal +right_ref = literal + comparison_op + column +comparison = left_ref | right_ref + + +@left_ref.set_parse_action +def _(result: ParseResults) -> BooleanExpression: + if result.op == "<": + return LessThan(result.column, result.literal) + elif result.op == "<=": + return LessThanOrEqual(result.column, result.literal) + elif result.op == ">": + return GreaterThan(result.column, result.literal) + elif result.op == ">=": + return GreaterThanOrEqual(result.column, result.literal) + if result.op in ("=", "=="): + return EqualTo(result.column, result.literal) + if result.op in ("!=", "<>"): + return NotEqualTo(result.column, result.literal) + raise ValueError(f"Unsupported operation type: {result.op}") + + +@right_ref.set_parse_action +def _(result: ParseResults) -> BooleanExpression: + if result.op == "<": + return GreaterThan(result.column, result.literal) + elif result.op == "<=": + return GreaterThanOrEqual(result.column, result.literal) + elif result.op == ">": + return LessThan(result.column, result.literal) + elif result.op == ">=": + return LessThanOrEqual(result.column, result.literal) + elif result.op in ("=", "=="): + return EqualTo(result.column, result.literal) + elif result.op in ("!=", "<>"): + return NotEqualTo(result.column, result.literal) + raise ValueError(f"Unsupported operation type: {result.op}") + + +is_null = column + IS + NULL +not_null = column + IS + NOT + NULL +null_check = is_null | not_null + + +@is_null.set_parse_action +def _(result: ParseResults) -> BooleanExpression: + return IsNull(result.column) + + +@not_null.set_parse_action +def _(result: ParseResults) -> BooleanExpression: + return NotNull(result.column) + + +is_nan = column + IS + NAN +not_nan = column + IS + NOT + NAN +nan_check = is_nan | not_nan + + +@is_nan.set_parse_action +def _(result: ParseResults) -> BooleanExpression: + return IsNaN(result.column) + + +@not_nan.set_parse_action +def _(result: ParseResults) -> BooleanExpression: + return NotNaN(result.column) + + +is_in = column + IN + "(" + literal_set + ")" +not_in = column + NOT + IN + "(" + literal_set + ")" +in_check = is_in | not_in + + +@is_in.set_parse_action +def _(result: ParseResults) -> BooleanExpression: + return In(result.column, result.literal_set) + + +@not_in.set_parse_action +def _(result: ParseResults) -> BooleanExpression: + return NotIn(result.column, result.literal_set) + + +predicate = (comparison | in_check | null_check | nan_check | boolean).set_results_name("predicate") + + +def handle_not(result: ParseResults) -> Not: + return Not(result[0][0]) + + +def handle_and(result: ParseResults) -> And: + return And(result[0][0], result[0][1]) + + +def handle_or(result: ParseResults) -> Or: + return Or(result[0][0], result[0][1]) + + +boolean_expression = infix_notation( + predicate, + [ + (Suppress(NOT), 1, opAssoc.RIGHT, handle_not), + (Suppress(AND), 2, opAssoc.LEFT, handle_and), + (Suppress(OR), 2, opAssoc.LEFT, handle_or), + ], +).set_name("expr") + + +def parse(expr: str) -> BooleanExpression: + """Parses a boolean expression""" + return boolean_expression.parse_string(expr)[0] diff --git a/pyproject.toml b/pyproject.toml index 0312018be8..fd8c3fcb52 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -209,5 +209,9 @@ ignore_missing_imports = true module = "duckdb.*" ignore_missing_imports = true +[[tool.mypy.overrides]] +module = "pyparsing.*" +ignore_missing_imports = true + [tool.coverage.run] source = ['pyiceberg/'] diff --git a/tests/expressions/test_literals.py b/tests/expressions/test_literals.py index efcacc4574..ff9ae5629e 100644 --- a/tests/expressions/test_literals.py +++ b/tests/expressions/test_literals.py @@ -667,10 +667,6 @@ def test_invalid_decimal_conversions(): literal(Decimal("34.11")), [ BooleanType(), - IntegerType(), - LongType(), - FloatType(), - DoubleType(), DateType(), TimeType(), TimestampType(), diff --git a/tests/expressions/test_parser.py b/tests/expressions/test_parser.py new file mode 100644 index 0000000000..47704be4c3 --- /dev/null +++ b/tests/expressions/test_parser.py @@ -0,0 +1,151 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +import pytest +from pyparsing import ParseException + +from pyiceberg.expressions import ( + AlwaysFalse, + AlwaysTrue, + And, + EqualTo, + GreaterThan, + GreaterThanOrEqual, + In, + IsNaN, + IsNull, + LessThan, + LessThanOrEqual, + Not, + NotEqualTo, + NotIn, + NotNaN, + NotNull, + Or, + parser, +) + + +def test_true(): + assert AlwaysTrue() == parser.parse("true") + + +def test_false(): + assert AlwaysFalse() == parser.parse("false") + + +def test_is_null(): + assert IsNull("x") == parser.parse("x is null") + assert IsNull("x") == parser.parse("x IS NULL") + + +def test_not_null(): + assert NotNull("x") == parser.parse("x is not null") + assert NotNull("x") == parser.parse("x IS NOT NULL") + + +def test_is_nan(): + assert IsNaN("x") == parser.parse("x is nan") + assert IsNaN("x") == parser.parse("x IS NAN") + + +def test_not_nan(): + assert NotNaN("x") == parser.parse("x is not nan") + assert NotNaN("x") == parser.parse("x IS NOT NaN") + + +def test_less_than(): + assert LessThan("x", 5) == parser.parse("x < 5") + assert LessThan("x", "a") == parser.parse("'a' > x") + + +def test_less_than_or_equal(): + assert LessThanOrEqual("x", 5) == parser.parse("x <= 5") + assert LessThanOrEqual("x", "a") == parser.parse("'a' >= x") + + +def test_greater_than(): + assert GreaterThan("x", 5) == parser.parse("x > 5") + assert GreaterThan("x", "a") == parser.parse("'a' < x") + + +def test_greater_than_or_equal(): + assert GreaterThanOrEqual("x", 5) == parser.parse("x <= 5") + assert GreaterThanOrEqual("x", "a") == parser.parse("'a' >= x") + + +def test_equal_to(): + assert EqualTo("x", 5) == parser.parse("x = 5") + assert EqualTo("x", "a") == parser.parse("'a' = x") + assert EqualTo("x", "a") == parser.parse("x == 'a'") + assert EqualTo("x", 5) == parser.parse("5 == x") + + +def test_not_equal_to(): + assert NotEqualTo("x", 5) == parser.parse("x != 5") + assert NotEqualTo("x", "a") == parser.parse("'a' != x") + assert NotEqualTo("x", "a") == parser.parse("x <> 'a'") + assert NotEqualTo("x", 5) == parser.parse("5 <> x") + + +def test_in(): + assert In("x", {5, 6, 7}) == parser.parse("x in (5, 6, 7)") + assert In("x", {"a", "b", "c"}) == parser.parse("x IN ('a', 'b', 'c')") + + +def test_in_different_types(): + with pytest.raises(ParseException): + parser.parse("x in (5, 'a')") + + +def test_not_in(): + assert NotIn("x", {5, 6, 7}) == parser.parse("x not in (5, 6, 7)") + assert NotIn("x", {"a", "b", "c"}) == parser.parse("x NOT IN ('a', 'b', 'c')") + + +def test_not_in_different_types(): + with pytest.raises(ParseException): + parser.parse("x not in (5, 'a')") + + +def test_simple_and(): + assert And(GreaterThanOrEqual("x", 5), LessThan("x", 10)) == parser.parse("5 <= x and x < 10") + + +def test_and_with_not(): + assert And(Not(GreaterThanOrEqual("x", 5)), LessThan("x", 10)) == parser.parse("not 5 <= x and x < 10") + assert And(GreaterThanOrEqual("x", 5), Not(LessThan("x", 10))) == parser.parse("5 <= x and not x < 10") + + +def test_or_with_not(): + assert Or(Not(LessThan("x", 5)), GreaterThan("x", 10)) == parser.parse("not x < 5 or 10 < x") + assert Or(LessThan("x", 5), Not(GreaterThan("x", 10))) == parser.parse("x < 5 or not 10 < x") + + +def test_simple_or(): + assert Or(LessThan("x", 5), GreaterThan("x", 10)) == parser.parse("x < 5 or 10 < x") + + +def test_and_or_without_parens(): + assert Or(And(NotNull("x"), LessThan("x", 5)), GreaterThan("x", 10)) == parser.parse("x is not null and x < 5 or 10 < x") + assert Or(IsNull("x"), And(GreaterThanOrEqual("x", 5), LessThan("x", 10))) == parser.parse("x is null or 5 <= x and x < 10") + + +def test_and_or_with_parens(): + assert And(NotNull("x"), Or(LessThan("x", 5), GreaterThan("x", 10))) == parser.parse("x is not null and (x < 5 or 10 < x)") + assert Or(IsNull("x"), And(GreaterThanOrEqual("x", 5), Not(LessThan("x", 10)))) == parser.parse( + "(x is null) or (5 <= x) and not(x < 10)" + ) From 8c08259a65c2d12e752ba2e640aebba17d350453 Mon Sep 17 00:00:00 2001 From: Fokko Driesprong Date: Mon, 28 Nov 2022 22:12:05 +0100 Subject: [PATCH 285/642] Python: Produce constants when binding produces AboveMax or BelowMin (#6256) --- pyiceberg/expressions/__init__.py | 22 +++- pyiceberg/expressions/literals.py | 99 +++++++++++++----- tests/expressions/test_expressions.py | 138 +++++++++++++++++++++++++- tests/expressions/test_literals.py | 17 +--- 4 files changed, 235 insertions(+), 41 deletions(-) diff --git a/pyiceberg/expressions/__init__.py b/pyiceberg/expressions/__init__.py index 5d25d33915..528a294545 100644 --- a/pyiceberg/expressions/__init__.py +++ b/pyiceberg/expressions/__init__.py @@ -29,7 +29,12 @@ Union, ) -from pyiceberg.expressions.literals import Literal, literal +from pyiceberg.expressions.literals import ( + AboveMax, + BelowMin, + Literal, + literal, +) from pyiceberg.schema import Accessor, Schema from pyiceberg.typedef import L, StructProtocol from pyiceberg.types import DoubleType, FloatType, NestedField @@ -608,7 +613,20 @@ def __init__(self, term: Union[str, UnboundTerm[Any]], literal: Union[L, Literal def bind(self, schema: Schema, case_sensitive: bool = True) -> BoundLiteralPredicate[L]: bound_term = self.term.bind(schema, case_sensitive) - return self.as_bound(bound_term, self.literal.to(bound_term.ref().field.field_type)) + lit = self.literal.to(bound_term.ref().field.field_type) + + if isinstance(lit, AboveMax): + if isinstance(self, (LessThan, LessThanOrEqual, NotEqualTo)): + return AlwaysTrue() # type: ignore + elif isinstance(self, (GreaterThan, GreaterThanOrEqual, EqualTo)): + return AlwaysFalse() # type: ignore + elif isinstance(lit, BelowMin): + if isinstance(self, (GreaterThan, GreaterThanOrEqual, NotEqualTo)): + return AlwaysTrue() # type: ignore + elif isinstance(self, (LessThan, LessThanOrEqual, EqualTo)): + return AlwaysFalse() # type: ignore + + return self.as_bound(bound_term, lit) def __eq__(self, other): if isinstance(other, LiteralPredicate): diff --git a/pyiceberg/expressions/literals.py b/pyiceberg/expressions/literals.py index 44ddc1331e..8a0aaa08a6 100644 --- a/pyiceberg/expressions/literals.py +++ b/pyiceberg/expressions/literals.py @@ -136,60 +136,98 @@ def literal(value: L) -> Literal[L]: raise TypeError(f"Invalid literal value: {repr(value)}") -class FloatAboveMax(Literal[float], Singleton): +class AboveMax(Literal[L]): + def __repr__(self) -> str: + return f"{self.__class__.__name__}()" + + def __str__(self) -> str: + return self.__class__.__name__ + + +class BelowMin(Literal[L]): + def __repr__(self) -> str: + return f"{self.__class__.__name__}()" + + def __str__(self) -> str: + return self.__class__.__name__ + + +class FloatAboveMax(AboveMax[float], Singleton): def __init__(self): super().__init__(FloatType.max, float) + @singledispatchmethod def to(self, type_var: IcebergType) -> Literal: # type: ignore raise TypeError("Cannot change the type of FloatAboveMax") - def __repr__(self) -> str: - return "FloatAboveMax()" - - def __str__(self) -> str: - return "FloatAboveMax" + @to.register(FloatType) + def _(self, _: FloatType) -> Literal[float]: + return self -class FloatBelowMin(Literal[float], Singleton): +class FloatBelowMin(BelowMin[float], Singleton): def __init__(self): super().__init__(FloatType.min, float) + @singledispatchmethod def to(self, type_var: IcebergType) -> Literal: # type: ignore raise TypeError("Cannot change the type of FloatBelowMin") - def __repr__(self) -> str: - return "FloatBelowMin()" - - def __str__(self) -> str: - return "FloatBelowMin" + @to.register(FloatType) + def _(self, _: FloatType) -> Literal[float]: + return self -class IntAboveMax(Literal[int]): +class IntAboveMax(AboveMax[int], Singleton): def __init__(self): super().__init__(IntegerType.max, int) + @singledispatchmethod def to(self, type_var: IcebergType) -> Literal: # type: ignore raise TypeError("Cannot change the type of IntAboveMax") - def __repr__(self) -> str: - return "IntAboveMax()" - - def __str__(self) -> str: - return "IntAboveMax" + @to.register(IntegerType) + def _(self, _: IntegerType) -> Literal[int]: + return self -class IntBelowMin(Literal[int]): +class IntBelowMin(BelowMin[int], Singleton): def __init__(self): super().__init__(IntegerType.min, int) + @singledispatchmethod def to(self, type_var: IcebergType) -> Literal: # type: ignore raise TypeError("Cannot change the type of IntBelowMin") - def __repr__(self) -> str: - return "IntBelowMin()" + @to.register(IntegerType) + def _(self, _: IntegerType) -> Literal[int]: + return self - def __str__(self) -> str: - return "IntBelowMin" + +class LongAboveMax(AboveMax[int], Singleton): + def __init__(self): + super().__init__(LongType.max, int) + + @singledispatchmethod + def to(self, type_var: IcebergType) -> Literal: # type: ignore + raise TypeError("Cannot change the type of IntAboveMax") + + @to.register(LongType) + def _(self, _: LongType) -> Literal[int]: + return self + + +class LongBelowMin(BelowMin[int], Singleton): + def __init__(self): + super().__init__(LongType.min, int) + + @singledispatchmethod + def to(self, type_var: IcebergType) -> Literal: # type: ignore + raise TypeError("Cannot change the type of IntBelowMin") + + @to.register(LongType) + def _(self, _: LongType) -> Literal[int]: + return self class BooleanLiteral(Literal[bool]): @@ -221,7 +259,12 @@ def decrement(self) -> Literal[int]: @to.register(LongType) def _(self, _: LongType) -> Literal[int]: - return self + if LongType.max < self.value: + return LongAboveMax() + elif LongType.min > self.value: + return LongBelowMin() + else: + return self @to.register(IntegerType) def _(self, _: IntegerType) -> Literal[int]: @@ -466,7 +509,13 @@ def _(self, type_var: IntegerType) -> Literal[int]: @to.register(LongType) def _(self, type_var: LongType) -> Literal[int]: try: - return LongLiteral(int(float(self.value))) + long_value = int(float(self.value)) + if LongType.max < long_value: + return LongAboveMax() + elif LongType.min > long_value: + return LongBelowMin() + else: + return LongLiteral(long_value) except (TypeError, ValueError) as e: raise ValueError(f"Could not convert {self.value} into a {type_var}") from e diff --git a/tests/expressions/test_expressions.py b/tests/expressions/test_expressions.py index d1d0d74581..befa71933c 100644 --- a/tests/expressions/test_expressions.py +++ b/tests/expressions/test_expressions.py @@ -56,7 +56,7 @@ Or, Reference, ) -from pyiceberg.expressions.literals import literal +from pyiceberg.expressions.literals import Literal, literal from pyiceberg.expressions.visitors import _from_byte_buffer from pyiceberg.schema import Accessor, Schema from pyiceberg.types import ( @@ -64,6 +64,7 @@ FloatType, IntegerType, ListType, + LongType, NestedField, StringType, ) @@ -915,6 +916,141 @@ def test_string_argument_unbound_set(): assert In("a", {"b", "c"}) == In(Reference("a"), {"b", "c"}) +@pytest.fixture +def int_schema() -> Schema: + return Schema(NestedField(field_id=1, name="a", field_type=IntegerType(), required=False)) + + +@pytest.fixture +def above_int_max() -> Literal[int]: + return literal(IntegerType.max + 1) + + +@pytest.fixture +def below_int_min() -> Literal[int]: + return literal(IntegerType.min - 1) + + +def test_above_int_bounds_equal_to(int_schema: Schema, above_int_max, below_int_min) -> None: + assert EqualTo("a", above_int_max).bind(int_schema) is AlwaysFalse() + assert EqualTo("a", below_int_min).bind(int_schema) is AlwaysFalse() + + +def test_above_int_bounds_not_equal_to(int_schema: Schema, above_int_max, below_int_min) -> None: + assert NotEqualTo("a", above_int_max).bind(int_schema) is AlwaysTrue() + assert NotEqualTo("a", below_int_min).bind(int_schema) is AlwaysTrue() + + +def test_above_int_bounds_less_than(int_schema: Schema, above_int_max, below_int_min) -> None: + assert LessThan("a", above_int_max).bind(int_schema) is AlwaysTrue() + assert LessThan("a", below_int_min).bind(int_schema) is AlwaysFalse() + + +def test_above_int_bounds_less_than_or_equal(int_schema: Schema, above_int_max, below_int_min) -> None: + assert LessThanOrEqual("a", above_int_max).bind(int_schema) is AlwaysTrue() + assert LessThanOrEqual("a", below_int_min).bind(int_schema) is AlwaysFalse() + + +def test_above_int_bounds_greater_than(int_schema: Schema, above_int_max, below_int_min) -> None: + assert GreaterThan("a", above_int_max).bind(int_schema) is AlwaysFalse() + assert GreaterThan("a", below_int_min).bind(int_schema) is AlwaysTrue() + + +def test_above_int_bounds_greater_than_or_equal(int_schema: Schema, above_int_max, below_int_min) -> None: + assert GreaterThanOrEqual("a", above_int_max).bind(int_schema) is AlwaysFalse() + assert GreaterThanOrEqual("a", below_int_min).bind(int_schema) is AlwaysTrue() + + +@pytest.fixture +def float_schema() -> Schema: + return Schema(NestedField(field_id=1, name="a", field_type=FloatType(), required=False)) + + +@pytest.fixture +def above_float_max() -> Literal[float]: + return literal(FloatType.max * 2) + + +@pytest.fixture +def below_float_min() -> Literal[float]: + return literal(FloatType.min * 2) + + +def test_above_float_bounds_equal_to(float_schema: Schema, above_float_max, below_float_min) -> None: + assert EqualTo("a", above_float_max).bind(float_schema) is AlwaysFalse() + assert EqualTo("a", below_float_min).bind(float_schema) is AlwaysFalse() + + +def test_above_float_bounds_not_equal_to(float_schema: Schema, above_float_max, below_float_min) -> None: + assert NotEqualTo("a", above_float_max).bind(float_schema) is AlwaysTrue() + assert NotEqualTo("a", below_float_min).bind(float_schema) is AlwaysTrue() + + +def test_above_float_bounds_less_than(float_schema: Schema, above_float_max, below_float_min) -> None: + assert LessThan("a", above_float_max).bind(float_schema) is AlwaysTrue() + assert LessThan("a", below_float_min).bind(float_schema) is AlwaysFalse() + + +def test_above_float_bounds_less_than_or_equal(float_schema: Schema, above_float_max, below_float_min) -> None: + assert LessThanOrEqual("a", above_float_max).bind(float_schema) is AlwaysTrue() + assert LessThanOrEqual("a", below_float_min).bind(float_schema) is AlwaysFalse() + + +def test_above_float_bounds_greater_than(float_schema: Schema, above_float_max, below_float_min) -> None: + assert GreaterThan("a", above_float_max).bind(float_schema) is AlwaysFalse() + assert GreaterThan("a", below_float_min).bind(float_schema) is AlwaysTrue() + + +def test_above_float_bounds_greater_than_or_equal(float_schema: Schema, above_float_max, below_float_min) -> None: + assert GreaterThanOrEqual("a", above_float_max).bind(float_schema) is AlwaysFalse() + assert GreaterThanOrEqual("a", below_float_min).bind(float_schema) is AlwaysTrue() + + +@pytest.fixture +def long_schema() -> Schema: + return Schema(NestedField(field_id=1, name="a", field_type=LongType(), required=False)) + + +@pytest.fixture +def above_long_max() -> Literal[float]: + return literal(LongType.max + 1) + + +@pytest.fixture +def below_long_min() -> Literal[float]: + return literal(LongType.min - 1) + + +def test_above_long_bounds_equal_to(long_schema: Schema, above_long_max, below_long_min) -> None: + assert EqualTo("a", above_long_max).bind(long_schema) is AlwaysFalse() + assert EqualTo("a", below_long_min).bind(long_schema) is AlwaysFalse() + + +def test_above_long_bounds_not_equal_to(long_schema: Schema, above_long_max, below_long_min) -> None: + assert NotEqualTo("a", above_long_max).bind(long_schema) is AlwaysTrue() + assert NotEqualTo("a", below_long_min).bind(long_schema) is AlwaysTrue() + + +def test_above_long_bounds_less_than(long_schema: Schema, above_long_max, below_long_min) -> None: + assert LessThan("a", above_long_max).bind(long_schema) is AlwaysTrue() + assert LessThan("a", below_long_min).bind(long_schema) is AlwaysFalse() + + +def test_above_long_bounds_less_than_or_equal(long_schema: Schema, above_long_max, below_long_min) -> None: + assert LessThanOrEqual("a", above_long_max).bind(long_schema) is AlwaysTrue() + assert LessThanOrEqual("a", below_long_min).bind(long_schema) is AlwaysFalse() + + +def test_above_long_bounds_greater_than(long_schema: Schema, above_long_max, below_long_min) -> None: + assert GreaterThan("a", above_long_max).bind(long_schema) is AlwaysFalse() + assert GreaterThan("a", below_long_min).bind(long_schema) is AlwaysTrue() + + +def test_above_long_bounds_greater_than_or_equal(long_schema: Schema, above_long_max, below_long_min) -> None: + assert GreaterThanOrEqual("a", above_long_max).bind(long_schema) is AlwaysFalse() + assert GreaterThanOrEqual("a", below_long_min).bind(long_schema) is AlwaysTrue() + + # __ __ ___ # | \/ |_ _| _ \_ _ # | |\/| | || | _/ || | diff --git a/tests/expressions/test_literals.py b/tests/expressions/test_literals.py index ff9ae5629e..2ca61f9e80 100644 --- a/tests/expressions/test_literals.py +++ b/tests/expressions/test_literals.py @@ -492,9 +492,7 @@ def test_above_max_float(): assert repr(a) == "FloatAboveMax()" assert a.value == FloatType.max assert a == eval(repr(a)) - with pytest.raises(TypeError) as e: - a.to(IntegerType()) - assert "Cannot change the type of FloatAboveMax" in str(e.value) + assert a.to(FloatType()) == FloatAboveMax() def test_below_min_float(): @@ -505,9 +503,7 @@ def test_below_min_float(): assert repr(b) == "FloatBelowMin()" assert b == eval(repr(b)) assert b.value == FloatType.min - with pytest.raises(TypeError) as e: - b.to(IntegerType()) - assert "Cannot change the type of FloatBelowMin" in str(e.value) + assert b.to(FloatType()) == FloatBelowMin() def test_above_max_int(): @@ -518,9 +514,7 @@ def test_above_max_int(): assert repr(a) == "IntAboveMax()" assert a.value == IntegerType.max assert a == eval(repr(a)) - with pytest.raises(TypeError) as e: - a.to(IntegerType()) - assert "Cannot change the type of IntAboveMax" in str(e.value) + assert a.to(IntegerType()) == IntAboveMax() def test_below_min_int(): @@ -530,10 +524,7 @@ def test_below_min_int(): assert str(b) == "IntBelowMin" assert repr(b) == "IntBelowMin()" assert b == eval(repr(b)) - assert b.value == IntegerType.min - with pytest.raises(TypeError) as e: - b.to(IntegerType()) - assert "Cannot change the type of IntBelowMin" in str(e.value) + assert b.to(IntegerType()) == IntBelowMin() def test_invalid_boolean_conversions(): From 8b32188f4298e6f9dd6a8e5f9aa409e2f299de42 Mon Sep 17 00:00:00 2001 From: Fokko Driesprong Date: Wed, 30 Nov 2022 23:25:07 +0100 Subject: [PATCH 286/642] Python: Bump PyArrow (#6295) --- poetry.lock | 360 ++++++++++++------------------------------------- pyproject.toml | 2 +- 2 files changed, 90 insertions(+), 272 deletions(-) diff --git a/poetry.lock b/poetry.lock index 56f1da99c4..3ce6de2222 100644 --- a/poetry.lock +++ b/poetry.lock @@ -1,6 +1,6 @@ [[package]] name = "aiobotocore" -version = "2.4.0" +version = "2.4.1" description = "Async client for aws services using botocore and aiohttp" category = "main" optional = true @@ -34,7 +34,7 @@ multidict = ">=4.5,<7.0" yarl = ">=1.0,<2.0" [package.extras] -speedups = ["Brotli", "aiodns", "cchardet"] +speedups = ["aiodns", "brotli", "cchardet"] [[package]] name = "aioitertools" @@ -78,7 +78,7 @@ python-versions = ">=3.5" dev = ["cloudpickle", "coverage[toml] (>=5.0.2)", "furo", "hypothesis", "mypy (>=0.900,!=0.940)", "pre-commit", "pympler", "pytest (>=4.3.0)", "pytest-mypy-plugins", "sphinx", "sphinx-notfound-page", "zope.interface"] docs = ["furo", "sphinx", "sphinx-notfound-page", "zope.interface"] tests = ["cloudpickle", "coverage[toml] (>=5.0.2)", "hypothesis", "mypy (>=0.900,!=0.940)", "pympler", "pytest (>=4.3.0)", "pytest-mypy-plugins", "zope.interface"] -tests-no-zope = ["cloudpickle", "coverage[toml] (>=5.0.2)", "hypothesis", "mypy (>=0.900,!=0.940)", "pympler", "pytest (>=4.3.0)", "pytest-mypy-plugins"] +tests_no_zope = ["cloudpickle", "coverage[toml] (>=5.0.2)", "hypothesis", "mypy (>=0.900,!=0.940)", "pympler", "pytest (>=4.3.0)", "pytest-mypy-plugins"] [[package]] name = "boto3" @@ -168,7 +168,7 @@ optional = false python-versions = ">=3.6.0" [package.extras] -unicode-backport = ["unicodedata2"] +unicode_backport = ["unicodedata2"] [[package]] name = "click" @@ -216,7 +216,7 @@ toml = ["tomli"] [[package]] name = "cryptography" -version = "38.0.3" +version = "38.0.4" description = "cryptography is a package which provides cryptographic recipes and primitives to Python developers." category = "dev" optional = false @@ -357,7 +357,7 @@ python-versions = ">=3.5" [[package]] name = "importlib-metadata" -version = "5.0.0" +version = "5.1.0" description = "Read metadata from Python packages" category = "dev" optional = false @@ -478,9 +478,6 @@ category = "dev" optional = false python-versions = ">=2.7,!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,!=3.4.*,!=3.5.*,!=3.6.*" -[package.dependencies] -setuptools = "*" - [[package]] name = "numpy" version = "1.23.5" @@ -553,7 +550,7 @@ virtualenv = ">=20.0.8" [[package]] name = "pyarrow" -version = "10.0.0" +version = "10.0.1" description = "Python library for Apache Arrow" category = "main" optional = true @@ -695,7 +692,7 @@ urllib3 = ">=1.21.1,<1.27" [package.extras] socks = ["PySocks (>=1.5.6,!=1.5.7)"] -use-chardet-on-py3 = ["chardet (>=3.0.2,<6)"] +use_chardet_on_py3 = ["chardet (>=3.0.2,<6)"] [[package]] name = "requests-mock" @@ -777,19 +774,6 @@ botocore = ">=1.12.36,<2.0a.0" [package.extras] crt = ["botocore[crt] (>=1.20.29,<2.0a.0)"] -[[package]] -name = "setuptools" -version = "65.6.0" -description = "Easily download, build, install, upgrade, and uninstall Python packages" -category = "dev" -optional = false -python-versions = ">=3.7" - -[package.extras] -docs = ["furo", "jaraco.packaging (>=9)", "jaraco.tidelift (>=1.4)", "pygments-github-lexers (==0.0.5)", "rst.linker (>=1.9)", "sphinx (>=3.5)", "sphinx-favicon", "sphinx-hoverxref (<2)", "sphinx-inline-tabs", "sphinx-notfound-page (==0.8.3)", "sphinx-reredirects", "sphinxcontrib-towncrier"] -testing = ["build[virtualenv]", "filelock (>=3.4.0)", "flake8 (<5)", "flake8-2020", "ini2toml[lite] (>=0.9)", "jaraco.envs (>=2.2)", "jaraco.path (>=3.2.0)", "pip (>=19.1)", "pip-run (>=8.8)", "pytest (>=6)", "pytest-black (>=0.3.7)", "pytest-checkdocs (>=2.4)", "pytest-cov", "pytest-enabler (>=1.3)", "pytest-flake8", "pytest-mypy (>=0.9.1)", "pytest-perf", "pytest-timeout", "pytest-xdist", "tomli-w (>=1.0.0)", "virtualenv (>=13.0.0)", "wheel"] -testing-integration = ["build[virtualenv]", "filelock (>=3.4.0)", "jaraco.envs (>=2.2)", "jaraco.path (>=3.2.0)", "pytest", "pytest-enabler", "pytest-xdist", "tomli", "virtualenv (>=13.0.0)", "wheel"] - [[package]] name = "six" version = "1.16.0" @@ -848,11 +832,11 @@ python-versions = ">=3.7" [[package]] name = "urllib3" -version = "1.26.12" +version = "1.26.13" description = "HTTP library with thread-safe connection pooling, file post, and more." category = "main" optional = false -python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*, !=3.5.*, <4" +python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*, !=3.5.*" [package.extras] brotli = ["brotli (>=1.0.9)", "brotlicffi (>=0.8.0)", "brotlipy (>=0.6.0)"] @@ -861,7 +845,7 @@ socks = ["PySocks (>=1.5.6,!=1.5.7,<2.0)"] [[package]] name = "virtualenv" -version = "20.16.7" +version = "20.17.0" description = "Virtual Python Environment builder" category = "dev" optional = false @@ -920,7 +904,7 @@ multidict = ">=4.0" [[package]] name = "zipp" -version = "3.10.0" +version = "3.11.0" description = "Backport of pathlib-compatible object wrapper for zip files" category = "dev" optional = false @@ -955,12 +939,12 @@ snappy = ["python-snappy"] [metadata] lock-version = "1.1" python-versions = "^3.8" -content-hash = "ad000d110de01a9e85f226ed0fadc5c74570bdb3cc6d30f38f54d18a11d51d36" +content-hash = "064eb484ccbc348f5b0794b51c0365bdc1ee83dd47970652978a781adb6721c3" [metadata.files] aiobotocore = [ - {file = "aiobotocore-2.4.0-py3-none-any.whl", hash = "sha256:6c25381d31b712652bc2f6008683949351c240c56d24b1d8ae252c1034a50f63"}, - {file = "aiobotocore-2.4.0.tar.gz", hash = "sha256:f9fe0698cc497861bdb54cd16161c804031f758ada9480c35540f20c0c078385"}, + {file = "aiobotocore-2.4.1-py3-none-any.whl", hash = "sha256:26e0d55d5901f487c3467616c028abb85036ca33a0f88e279770adae6b865c69"}, + {file = "aiobotocore-2.4.1.tar.gz", hash = "sha256:5c8ee79d1f14273e3f8f3af5c893db889ef163227b178ecacb5aea4547d3b9a7"}, ] aiohttp = [ {file = "aiohttp-3.8.3-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:ba71c9b4dcbb16212f334126cc3d8beb6af377f6703d9dc2d9fb3874fd667ee9"}, @@ -1059,14 +1043,8 @@ aiosignal = [ {file = "aiosignal-1.3.1-py3-none-any.whl", hash = "sha256:f8376fb07dd1e86a584e4fcdec80b36b7f81aac666ebc724e2c090300dd83b17"}, {file = "aiosignal-1.3.1.tar.gz", hash = "sha256:54cd96e15e1649b75d6c87526a6ff0b6c1b0dd3459f43d9ca11d48c339b68cfc"}, ] -async-timeout = [ - {file = "async-timeout-4.0.2.tar.gz", hash = "sha256:2163e1640ddb52b7a8c80d0a67a08587e5d245cc9c553a74a847056bc2976b15"}, - {file = "async_timeout-4.0.2-py3-none-any.whl", hash = "sha256:8ca1e4fcf50d07413d66d1a5e416e42cfdf5851c981d679a09851a6853383b3c"}, -] -attrs = [ - {file = "attrs-22.1.0-py2.py3-none-any.whl", hash = "sha256:86efa402f67bf2df34f51a335487cf46b1ec130d02b8d39fd248abfd30da551c"}, - {file = "attrs-22.1.0.tar.gz", hash = "sha256:29adc2665447e5191d0e7c568fde78b21f9672d344281d0c6e1ab085429b22b6"}, -] +async-timeout = [] +attrs = [] boto3 = [ {file = "boto3-1.24.59-py3-none-any.whl", hash = "sha256:34ab44146a2c4e7f4e72737f4b27e6eb5e0a7855c2f4599e3d9199b6a0a2d575"}, {file = "boto3-1.24.59.tar.gz", hash = "sha256:a50b4323f9579cfe22fcf5531fbd40b567d4d74c1adce06aeb5c95fce2a6fb40"}, @@ -1153,22 +1131,13 @@ cfgv = [ {file = "cfgv-3.3.1-py2.py3-none-any.whl", hash = "sha256:c6a0883f3917a037485059700b9e75da2464e6c27051014ad85ba6aaa5884426"}, {file = "cfgv-3.3.1.tar.gz", hash = "sha256:f5a830efb9ce7a445376bb66ec94c638a9787422f96264c98edc6bdeed8ab736"}, ] -charset-normalizer = [ - {file = "charset-normalizer-2.1.1.tar.gz", hash = "sha256:5a3d016c7c547f69d6f81fb0db9449ce888b418b5b9952cc5e6e66843e9dd845"}, - {file = "charset_normalizer-2.1.1-py3-none-any.whl", hash = "sha256:83e9a75d1911279afd89352c68b45348559d1fc0506b054b346651b5e7fee29f"}, -] -click = [ - {file = "click-8.1.3-py3-none-any.whl", hash = "sha256:bb4d8133cb15a609f44e8213d9b391b0809795062913b383c62be0ee95b1db48"}, - {file = "click-8.1.3.tar.gz", hash = "sha256:7682dc8afb30297001674575ea00d1814d808d6a36af415a82bd481d37ba7b8e"}, -] +charset-normalizer = [] +click = [] colorama = [ {file = "colorama-0.4.6-py2.py3-none-any.whl", hash = "sha256:4f1d9991f5acc0ca119f9d443620b77f9d6b33703e51011c16baf57afb285fc6"}, {file = "colorama-0.4.6.tar.gz", hash = "sha256:08695f5cb7ed6e0531a20572697297273c47b8cae5a63ffc6d6ed5c201be6e44"}, ] -commonmark = [ - {file = "commonmark-0.9.1-py2.py3-none-any.whl", hash = "sha256:da2f38c92590f83de410ba1a3cbceafbc74fee9def35f9251ba9a971d6d66fd9"}, - {file = "commonmark-0.9.1.tar.gz", hash = "sha256:452f9dc859be7f06631ddcb328b6919c67984aca654e5fefb3914d54691aed60"}, -] +commonmark = [] coverage = [ {file = "coverage-6.5.0-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:ef8674b0ee8cc11e2d574e3e2998aea5df5ab242e012286824ea3c6970580e53"}, {file = "coverage-6.5.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:784f53ebc9f3fd0e2a3f6a78b2be1bd1f5575d7863e10c6e12504f240fd06660"}, @@ -1222,32 +1191,32 @@ coverage = [ {file = "coverage-6.5.0.tar.gz", hash = "sha256:f642e90754ee3e06b0e7e51bce3379590e76b7f76b708e1a71ff043f87025c84"}, ] cryptography = [ - {file = "cryptography-38.0.3-cp36-abi3-macosx_10_10_universal2.whl", hash = "sha256:984fe150f350a3c91e84de405fe49e688aa6092b3525f407a18b9646f6612320"}, - {file = "cryptography-38.0.3-cp36-abi3-macosx_10_10_x86_64.whl", hash = "sha256:ed7b00096790213e09eb11c97cc6e2b757f15f3d2f85833cd2d3ec3fe37c1722"}, - {file = "cryptography-38.0.3-cp36-abi3-manylinux_2_17_aarch64.manylinux2014_aarch64.manylinux_2_24_aarch64.whl", hash = "sha256:bbf203f1a814007ce24bd4d51362991d5cb90ba0c177a9c08825f2cc304d871f"}, - {file = "cryptography-38.0.3-cp36-abi3-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:554bec92ee7d1e9d10ded2f7e92a5d70c1f74ba9524947c0ba0c850c7b011828"}, - {file = "cryptography-38.0.3-cp36-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:b1b52c9e5f8aa2b802d48bd693190341fae201ea51c7a167d69fc48b60e8a959"}, - {file = "cryptography-38.0.3-cp36-abi3-manylinux_2_24_x86_64.whl", hash = "sha256:728f2694fa743a996d7784a6194da430f197d5c58e2f4e278612b359f455e4a2"}, - {file = "cryptography-38.0.3-cp36-abi3-manylinux_2_28_aarch64.whl", hash = "sha256:dfb4f4dd568de1b6af9f4cda334adf7d72cf5bc052516e1b2608b683375dd95c"}, - {file = "cryptography-38.0.3-cp36-abi3-manylinux_2_28_x86_64.whl", hash = "sha256:5419a127426084933076132d317911e3c6eb77568a1ce23c3ac1e12d111e61e0"}, - {file = "cryptography-38.0.3-cp36-abi3-musllinux_1_1_aarch64.whl", hash = "sha256:9b24bcff7853ed18a63cfb0c2b008936a9554af24af2fb146e16d8e1aed75748"}, - {file = "cryptography-38.0.3-cp36-abi3-musllinux_1_1_x86_64.whl", hash = "sha256:25c1d1f19729fb09d42e06b4bf9895212292cb27bb50229f5aa64d039ab29146"}, - {file = "cryptography-38.0.3-cp36-abi3-win32.whl", hash = "sha256:7f836217000342d448e1c9a342e9163149e45d5b5eca76a30e84503a5a96cab0"}, - {file = "cryptography-38.0.3-cp36-abi3-win_amd64.whl", hash = "sha256:c46837ea467ed1efea562bbeb543994c2d1f6e800785bd5a2c98bc096f5cb220"}, - {file = "cryptography-38.0.3-pp37-pypy37_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:06fc3cc7b6f6cca87bd56ec80a580c88f1da5306f505876a71c8cfa7050257dd"}, - {file = "cryptography-38.0.3-pp37-pypy37_pp73-manylinux_2_24_x86_64.whl", hash = "sha256:65535bc550b70bd6271984d9863a37741352b4aad6fb1b3344a54e6950249b55"}, - {file = "cryptography-38.0.3-pp37-pypy37_pp73-manylinux_2_28_x86_64.whl", hash = "sha256:5e89468fbd2fcd733b5899333bc54d0d06c80e04cd23d8c6f3e0542358c6060b"}, - {file = "cryptography-38.0.3-pp38-pypy38_pp73-macosx_10_10_x86_64.whl", hash = "sha256:6ab9516b85bebe7aa83f309bacc5f44a61eeb90d0b4ec125d2d003ce41932d36"}, - {file = "cryptography-38.0.3-pp38-pypy38_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:068147f32fa662c81aebab95c74679b401b12b57494872886eb5c1139250ec5d"}, - {file = "cryptography-38.0.3-pp38-pypy38_pp73-manylinux_2_24_x86_64.whl", hash = "sha256:402852a0aea73833d982cabb6d0c3bb582c15483d29fb7085ef2c42bfa7e38d7"}, - {file = "cryptography-38.0.3-pp38-pypy38_pp73-manylinux_2_28_x86_64.whl", hash = "sha256:b1b35d9d3a65542ed2e9d90115dfd16bbc027b3f07ee3304fc83580f26e43249"}, - {file = "cryptography-38.0.3-pp38-pypy38_pp73-win_amd64.whl", hash = "sha256:6addc3b6d593cd980989261dc1cce38263c76954d758c3c94de51f1e010c9a50"}, - {file = "cryptography-38.0.3-pp39-pypy39_pp73-macosx_10_10_x86_64.whl", hash = "sha256:be243c7e2bfcf6cc4cb350c0d5cdf15ca6383bbcb2a8ef51d3c9411a9d4386f0"}, - {file = "cryptography-38.0.3-pp39-pypy39_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:78cf5eefac2b52c10398a42765bfa981ce2372cbc0457e6bf9658f41ec3c41d8"}, - {file = "cryptography-38.0.3-pp39-pypy39_pp73-manylinux_2_24_x86_64.whl", hash = "sha256:4e269dcd9b102c5a3d72be3c45d8ce20377b8076a43cbed6f660a1afe365e436"}, - {file = "cryptography-38.0.3-pp39-pypy39_pp73-manylinux_2_28_x86_64.whl", hash = "sha256:8d41a46251bf0634e21fac50ffd643216ccecfaf3701a063257fe0b2be1b6548"}, - {file = "cryptography-38.0.3-pp39-pypy39_pp73-win_amd64.whl", hash = "sha256:785e4056b5a8b28f05a533fab69febf5004458e20dad7e2e13a3120d8ecec75a"}, - {file = "cryptography-38.0.3.tar.gz", hash = "sha256:bfbe6ee19615b07a98b1d2287d6a6073f734735b49ee45b11324d85efc4d5cbd"}, + {file = "cryptography-38.0.4-cp36-abi3-macosx_10_10_universal2.whl", hash = "sha256:2fa36a7b2cc0998a3a4d5af26ccb6273f3df133d61da2ba13b3286261e7efb70"}, + {file = "cryptography-38.0.4-cp36-abi3-macosx_10_10_x86_64.whl", hash = "sha256:1f13ddda26a04c06eb57119caf27a524ccae20533729f4b1e4a69b54e07035eb"}, + {file = "cryptography-38.0.4-cp36-abi3-manylinux_2_17_aarch64.manylinux2014_aarch64.manylinux_2_24_aarch64.whl", hash = "sha256:2ec2a8714dd005949d4019195d72abed84198d877112abb5a27740e217e0ea8d"}, + {file = "cryptography-38.0.4-cp36-abi3-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:50a1494ed0c3f5b4d07650a68cd6ca62efe8b596ce743a5c94403e6f11bf06c1"}, + {file = "cryptography-38.0.4-cp36-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:a10498349d4c8eab7357a8f9aa3463791292845b79597ad1b98a543686fb1ec8"}, + {file = "cryptography-38.0.4-cp36-abi3-manylinux_2_24_x86_64.whl", hash = "sha256:10652dd7282de17990b88679cb82f832752c4e8237f0c714be518044269415db"}, + {file = "cryptography-38.0.4-cp36-abi3-manylinux_2_28_aarch64.whl", hash = "sha256:bfe6472507986613dc6cc00b3d492b2f7564b02b3b3682d25ca7f40fa3fd321b"}, + {file = "cryptography-38.0.4-cp36-abi3-manylinux_2_28_x86_64.whl", hash = "sha256:ce127dd0a6a0811c251a6cddd014d292728484e530d80e872ad9806cfb1c5b3c"}, + {file = "cryptography-38.0.4-cp36-abi3-musllinux_1_1_aarch64.whl", hash = "sha256:53049f3379ef05182864d13bb9686657659407148f901f3f1eee57a733fb4b00"}, + {file = "cryptography-38.0.4-cp36-abi3-musllinux_1_1_x86_64.whl", hash = "sha256:8a4b2bdb68a447fadebfd7d24855758fe2d6fecc7fed0b78d190b1af39a8e3b0"}, + {file = "cryptography-38.0.4-cp36-abi3-win32.whl", hash = "sha256:1d7e632804a248103b60b16fb145e8df0bc60eed790ece0d12efe8cd3f3e7744"}, + {file = "cryptography-38.0.4-cp36-abi3-win_amd64.whl", hash = "sha256:8e45653fb97eb2f20b8c96f9cd2b3a0654d742b47d638cf2897afbd97f80fa6d"}, + {file = "cryptography-38.0.4-pp37-pypy37_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:ca57eb3ddaccd1112c18fc80abe41db443cc2e9dcb1917078e02dfa010a4f353"}, + {file = "cryptography-38.0.4-pp37-pypy37_pp73-manylinux_2_24_x86_64.whl", hash = "sha256:c9e0d79ee4c56d841bd4ac6e7697c8ff3c8d6da67379057f29e66acffcd1e9a7"}, + {file = "cryptography-38.0.4-pp37-pypy37_pp73-manylinux_2_28_x86_64.whl", hash = "sha256:0e70da4bdff7601b0ef48e6348339e490ebfb0cbe638e083c9c41fb49f00c8bd"}, + {file = "cryptography-38.0.4-pp38-pypy38_pp73-macosx_10_10_x86_64.whl", hash = "sha256:998cd19189d8a747b226d24c0207fdaa1e6658a1d3f2494541cb9dfbf7dcb6d2"}, + {file = "cryptography-38.0.4-pp38-pypy38_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:67461b5ebca2e4c2ab991733f8ab637a7265bb582f07c7c88914b5afb88cb95b"}, + {file = "cryptography-38.0.4-pp38-pypy38_pp73-manylinux_2_24_x86_64.whl", hash = "sha256:4eb85075437f0b1fd8cd66c688469a0c4119e0ba855e3fef86691971b887caf6"}, + {file = "cryptography-38.0.4-pp38-pypy38_pp73-manylinux_2_28_x86_64.whl", hash = "sha256:3178d46f363d4549b9a76264f41c6948752183b3f587666aff0555ac50fd7876"}, + {file = "cryptography-38.0.4-pp38-pypy38_pp73-win_amd64.whl", hash = "sha256:6391e59ebe7c62d9902c24a4d8bcbc79a68e7c4ab65863536127c8a9cd94043b"}, + {file = "cryptography-38.0.4-pp39-pypy39_pp73-macosx_10_10_x86_64.whl", hash = "sha256:78e47e28ddc4ace41dd38c42e6feecfdadf9c3be2af389abbfeef1ff06822285"}, + {file = "cryptography-38.0.4-pp39-pypy39_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:2fb481682873035600b5502f0015b664abc26466153fab5c6bc92c1ea69d478b"}, + {file = "cryptography-38.0.4-pp39-pypy39_pp73-manylinux_2_24_x86_64.whl", hash = "sha256:4367da5705922cf7070462e964f66e4ac24162e22ab0a2e9d31f1b270dd78083"}, + {file = "cryptography-38.0.4-pp39-pypy39_pp73-manylinux_2_28_x86_64.whl", hash = "sha256:b4cad0cea995af760f82820ab4ca54e5471fc782f70a007f31531957f43e9dee"}, + {file = "cryptography-38.0.4-pp39-pypy39_pp73-win_amd64.whl", hash = "sha256:80ca53981ceeb3241998443c4964a387771588c4e4a5d92735a493af868294f9"}, + {file = "cryptography-38.0.4.tar.gz", hash = "sha256:175c1a818b87c9ac80bb7377f5520b7f31b3ef2a0004e2420319beadedb67290"}, ] distlib = [ {file = "distlib-0.3.6-py2.py3-none-any.whl", hash = "sha256:f35c4b692542ca110de7ef0bea44d73981caeb34ca0b9b6b2e6d7790dda8f80e"}, @@ -1333,10 +1302,7 @@ fastavro = [ {file = "fastavro-1.7.0-cp39-cp39-win_amd64.whl", hash = "sha256:492e8902063aa1c73170e11c18495fcaa86b71eae3513ef83ba438ca02b16b34"}, {file = "fastavro-1.7.0.tar.gz", hash = "sha256:4b1205f46489b4032d3155c1ab44d9824be0c7454df98d3a5bd22b78b98f23c8"}, ] -filelock = [ - {file = "filelock-3.8.0-py3-none-any.whl", hash = "sha256:617eb4e5eedc82fc5f47b6d61e4d11cb837c56cb4544e39081099fa17ad109d4"}, - {file = "filelock-3.8.0.tar.gz", hash = "sha256:55447caa666f2198c5b6b13a26d2084d26fa5b115c00d065664b2124680c4edc"}, -] +filelock = [] frozenlist = [ {file = "frozenlist-1.3.3-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:ff8bf625fe85e119553b5383ba0fb6aa3d0ec2ae980295aaefa552374926b3f4"}, {file = "frozenlist-1.3.3-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:dfbac4c2dfcc082fcf8d942d1e49b6aa0766c19d3358bd86e2000bf0fa4a9cf0"}, @@ -1426,8 +1392,8 @@ idna = [ {file = "idna-3.4.tar.gz", hash = "sha256:814f528e8dead7d329833b91c5faa87d60bf71824cd12a7530b5526063d02cb4"}, ] importlib-metadata = [ - {file = "importlib_metadata-5.0.0-py3-none-any.whl", hash = "sha256:ddb0e35065e8938f867ed4928d0ae5bf2a53b7773871bfe6bcc7e4fcdc7dea43"}, - {file = "importlib_metadata-5.0.0.tar.gz", hash = "sha256:da31db32b304314d044d3c12c79bd59e307889b287ad12ff387b3500835fc2ab"}, + {file = "importlib_metadata-5.1.0-py3-none-any.whl", hash = "sha256:d84d17e21670ec07990e1044a99efe8d615d860fd176fc29ef5c306068fda313"}, + {file = "importlib_metadata-5.1.0.tar.gz", hash = "sha256:d5059f9f1e8e41f80e9c56c2ee58811450c31984dfa625329ffd7c0dad88a73b"}, ] iniconfig = [ {file = "iniconfig-1.1.1-py2.py3-none-any.whl", hash = "sha256:011e24c64b7f47f6ebd835bb12a743f2fbe9a26d4cecaa7f53bc4f35ee9da8b3"}, @@ -1437,10 +1403,7 @@ jinja2 = [ {file = "Jinja2-3.1.2-py3-none-any.whl", hash = "sha256:6088930bfe239f0e6710546ab9c19c9ef35e29792895fed6e6e31a023a182a61"}, {file = "Jinja2-3.1.2.tar.gz", hash = "sha256:31351a702a408a9e7595a8fc6150fc3f43bb6bf7e319770cbc0db9df9437e852"}, ] -jmespath = [ - {file = "jmespath-1.0.1-py3-none-any.whl", hash = "sha256:02e2e4cc71b5bcab88332eebf907519190dd9e6e82107fa7f83b1003a6252980"}, - {file = "jmespath-1.0.1.tar.gz", hash = "sha256:90261b206d6defd58fdd5e85f478bf633a2901798906be2ad389150c5c60edbe"}, -] +jmespath = [] markupsafe = [ {file = "MarkupSafe-2.1.1-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:86b1f75c4e7c2ac2ccdaec2b9022845dbb81880ca318bb7a0a01fbf7813e3812"}, {file = "MarkupSafe-2.1.1-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:f121a1420d4e173a5d96e47e9a0c0dcff965afdf1626d28de1460815f7c4ee7a"}, @@ -1523,67 +1486,7 @@ moto = [ {file = "moto-4.0.10-py3-none-any.whl", hash = "sha256:356bf792b439228891c910e2a0fafd4264334cf9000b508c732ff43d8694fb6a"}, {file = "moto-4.0.10.tar.gz", hash = "sha256:9ba96d04a472d5682493cad7fee33337da34ebef18b397af1ea6dfb41efbe148"}, ] -multidict = [ - {file = "multidict-6.0.2-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:0b9e95a740109c6047602f4db4da9949e6c5945cefbad34a1299775ddc9a62e2"}, - {file = "multidict-6.0.2-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:ac0e27844758d7177989ce406acc6a83c16ed4524ebc363c1f748cba184d89d3"}, - {file = "multidict-6.0.2-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:041b81a5f6b38244b34dc18c7b6aba91f9cdaf854d9a39e5ff0b58e2b5773b9c"}, - {file = "multidict-6.0.2-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:5fdda29a3c7e76a064f2477c9aab1ba96fd94e02e386f1e665bca1807fc5386f"}, - {file = "multidict-6.0.2-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:3368bf2398b0e0fcbf46d85795adc4c259299fec50c1416d0f77c0a843a3eed9"}, - {file = "multidict-6.0.2-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:f4f052ee022928d34fe1f4d2bc743f32609fb79ed9c49a1710a5ad6b2198db20"}, - {file = "multidict-6.0.2-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:225383a6603c086e6cef0f2f05564acb4f4d5f019a4e3e983f572b8530f70c88"}, - {file = "multidict-6.0.2-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:50bd442726e288e884f7be9071016c15a8742eb689a593a0cac49ea093eef0a7"}, - {file = "multidict-6.0.2-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:47e6a7e923e9cada7c139531feac59448f1f47727a79076c0b1ee80274cd8eee"}, - {file = "multidict-6.0.2-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:0556a1d4ea2d949efe5fd76a09b4a82e3a4a30700553a6725535098d8d9fb672"}, - {file = "multidict-6.0.2-cp310-cp310-musllinux_1_1_ppc64le.whl", hash = "sha256:626fe10ac87851f4cffecee161fc6f8f9853f0f6f1035b59337a51d29ff3b4f9"}, - {file = "multidict-6.0.2-cp310-cp310-musllinux_1_1_s390x.whl", hash = "sha256:8064b7c6f0af936a741ea1efd18690bacfbae4078c0c385d7c3f611d11f0cf87"}, - {file = "multidict-6.0.2-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:2d36e929d7f6a16d4eb11b250719c39560dd70545356365b494249e2186bc389"}, - {file = "multidict-6.0.2-cp310-cp310-win32.whl", hash = "sha256:fcb91630817aa8b9bc4a74023e4198480587269c272c58b3279875ed7235c293"}, - {file = "multidict-6.0.2-cp310-cp310-win_amd64.whl", hash = "sha256:8cbf0132f3de7cc6c6ce00147cc78e6439ea736cee6bca4f068bcf892b0fd658"}, - {file = "multidict-6.0.2-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:05f6949d6169878a03e607a21e3b862eaf8e356590e8bdae4227eedadacf6e51"}, - {file = "multidict-6.0.2-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:e2c2e459f7050aeb7c1b1276763364884595d47000c1cddb51764c0d8976e608"}, - {file = "multidict-6.0.2-cp37-cp37m-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:d0509e469d48940147e1235d994cd849a8f8195e0bca65f8f5439c56e17872a3"}, - {file = "multidict-6.0.2-cp37-cp37m-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:514fe2b8d750d6cdb4712346a2c5084a80220821a3e91f3f71eec11cf8d28fd4"}, - {file = "multidict-6.0.2-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:19adcfc2a7197cdc3987044e3f415168fc5dc1f720c932eb1ef4f71a2067e08b"}, - {file = "multidict-6.0.2-cp37-cp37m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:b9d153e7f1f9ba0b23ad1568b3b9e17301e23b042c23870f9ee0522dc5cc79e8"}, - {file = "multidict-6.0.2-cp37-cp37m-musllinux_1_1_aarch64.whl", hash = "sha256:aef9cc3d9c7d63d924adac329c33835e0243b5052a6dfcbf7732a921c6e918ba"}, - {file = "multidict-6.0.2-cp37-cp37m-musllinux_1_1_i686.whl", hash = "sha256:4571f1beddff25f3e925eea34268422622963cd8dc395bb8778eb28418248e43"}, - {file = "multidict-6.0.2-cp37-cp37m-musllinux_1_1_ppc64le.whl", hash = "sha256:d48b8ee1d4068561ce8033d2c344cf5232cb29ee1a0206a7b828c79cbc5982b8"}, - {file = "multidict-6.0.2-cp37-cp37m-musllinux_1_1_s390x.whl", hash = "sha256:45183c96ddf61bf96d2684d9fbaf6f3564d86b34cb125761f9a0ef9e36c1d55b"}, - {file = "multidict-6.0.2-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:75bdf08716edde767b09e76829db8c1e5ca9d8bb0a8d4bd94ae1eafe3dac5e15"}, - {file = "multidict-6.0.2-cp37-cp37m-win32.whl", hash = "sha256:a45e1135cb07086833ce969555df39149680e5471c04dfd6a915abd2fc3f6dbc"}, - {file = "multidict-6.0.2-cp37-cp37m-win_amd64.whl", hash = "sha256:6f3cdef8a247d1eafa649085812f8a310e728bdf3900ff6c434eafb2d443b23a"}, - {file = "multidict-6.0.2-cp38-cp38-macosx_10_9_universal2.whl", hash = "sha256:0327292e745a880459ef71be14e709aaea2f783f3537588fb4ed09b6c01bca60"}, - {file = "multidict-6.0.2-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:e875b6086e325bab7e680e4316d667fc0e5e174bb5611eb16b3ea121c8951b86"}, - {file = "multidict-6.0.2-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:feea820722e69451743a3d56ad74948b68bf456984d63c1a92e8347b7b88452d"}, - {file = "multidict-6.0.2-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:9cc57c68cb9139c7cd6fc39f211b02198e69fb90ce4bc4a094cf5fe0d20fd8b0"}, - {file = "multidict-6.0.2-cp38-cp38-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:497988d6b6ec6ed6f87030ec03280b696ca47dbf0648045e4e1d28b80346560d"}, - {file = "multidict-6.0.2-cp38-cp38-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:89171b2c769e03a953d5969b2f272efa931426355b6c0cb508022976a17fd376"}, - {file = "multidict-6.0.2-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:684133b1e1fe91eda8fa7447f137c9490a064c6b7f392aa857bba83a28cfb693"}, - {file = "multidict-6.0.2-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:fd9fc9c4849a07f3635ccffa895d57abce554b467d611a5009ba4f39b78a8849"}, - {file = "multidict-6.0.2-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:e07c8e79d6e6fd37b42f3250dba122053fddb319e84b55dd3a8d6446e1a7ee49"}, - {file = "multidict-6.0.2-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:4070613ea2227da2bfb2c35a6041e4371b0af6b0be57f424fe2318b42a748516"}, - {file = "multidict-6.0.2-cp38-cp38-musllinux_1_1_ppc64le.whl", hash = "sha256:47fbeedbf94bed6547d3aa632075d804867a352d86688c04e606971595460227"}, - {file = "multidict-6.0.2-cp38-cp38-musllinux_1_1_s390x.whl", hash = "sha256:5774d9218d77befa7b70d836004a768fb9aa4fdb53c97498f4d8d3f67bb9cfa9"}, - {file = "multidict-6.0.2-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:2957489cba47c2539a8eb7ab32ff49101439ccf78eab724c828c1a54ff3ff98d"}, - {file = "multidict-6.0.2-cp38-cp38-win32.whl", hash = "sha256:e5b20e9599ba74391ca0cfbd7b328fcc20976823ba19bc573983a25b32e92b57"}, - {file = "multidict-6.0.2-cp38-cp38-win_amd64.whl", hash = "sha256:8004dca28e15b86d1b1372515f32eb6f814bdf6f00952699bdeb541691091f96"}, - {file = "multidict-6.0.2-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:2e4a0785b84fb59e43c18a015ffc575ba93f7d1dbd272b4cdad9f5134b8a006c"}, - {file = "multidict-6.0.2-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:6701bf8a5d03a43375909ac91b6980aea74b0f5402fbe9428fc3f6edf5d9677e"}, - {file = "multidict-6.0.2-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:a007b1638e148c3cfb6bf0bdc4f82776cef0ac487191d093cdc316905e504071"}, - {file = "multidict-6.0.2-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:07a017cfa00c9890011628eab2503bee5872f27144936a52eaab449be5eaf032"}, - {file = "multidict-6.0.2-cp39-cp39-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:c207fff63adcdf5a485969131dc70e4b194327666b7e8a87a97fbc4fd80a53b2"}, - {file = "multidict-6.0.2-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:373ba9d1d061c76462d74e7de1c0c8e267e9791ee8cfefcf6b0b2495762c370c"}, - {file = "multidict-6.0.2-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:bfba7c6d5d7c9099ba21f84662b037a0ffd4a5e6b26ac07d19e423e6fdf965a9"}, - {file = "multidict-6.0.2-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:19d9bad105dfb34eb539c97b132057a4e709919ec4dd883ece5838bcbf262b80"}, - {file = "multidict-6.0.2-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:de989b195c3d636ba000ee4281cd03bb1234635b124bf4cd89eeee9ca8fcb09d"}, - {file = "multidict-6.0.2-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:7c40b7bbece294ae3a87c1bc2abff0ff9beef41d14188cda94ada7bcea99b0fb"}, - {file = "multidict-6.0.2-cp39-cp39-musllinux_1_1_ppc64le.whl", hash = "sha256:d16cce709ebfadc91278a1c005e3c17dd5f71f5098bfae1035149785ea6e9c68"}, - {file = "multidict-6.0.2-cp39-cp39-musllinux_1_1_s390x.whl", hash = "sha256:a2c34a93e1d2aa35fbf1485e5010337c72c6791407d03aa5f4eed920343dd360"}, - {file = "multidict-6.0.2-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:feba80698173761cddd814fa22e88b0661e98cb810f9f986c54aa34d281e4937"}, - {file = "multidict-6.0.2-cp39-cp39-win32.whl", hash = "sha256:23b616fdc3c74c9fe01d76ce0d1ce872d2d396d8fa8e4899398ad64fb5aa214a"}, - {file = "multidict-6.0.2-cp39-cp39-win_amd64.whl", hash = "sha256:4bae31803d708f6f15fd98be6a6ac0b6958fcf68fda3c77a048a4f9073704aae"}, - {file = "multidict-6.0.2.tar.gz", hash = "sha256:5ff3bd75f38e4c43f1f470f2df7a4d430b821c4ce22be384e1459cb57d6bb013"}, -] +multidict = [] nodeenv = [ {file = "nodeenv-1.7.0-py2.py3-none-any.whl", hash = "sha256:27083a7b96a25f2f5e1d8cb4b6317ee8aeda3bdd121394e5ac54e498028a042e"}, {file = "nodeenv-1.7.0.tar.gz", hash = "sha256:e0e7f7dfb85fc5394c6fe1e8fa98131a2473e04311a45afb6508f7cf1836fa2b"}, @@ -1622,10 +1525,7 @@ packaging = [ {file = "packaging-21.3-py3-none-any.whl", hash = "sha256:ef103e05f519cdc783ae24ea4e2e0f508a9c99b2d4969652eed6a2e1ea5bd522"}, {file = "packaging-21.3.tar.gz", hash = "sha256:dd47c42927d89ab911e606518907cc2d3a1f38bbd026385970643f9c5b8ecfeb"}, ] -pep517 = [ - {file = "pep517-0.13.0-py3-none-any.whl", hash = "sha256:4ba4446d80aed5b5eac6509ade100bff3e7943a8489de249654a5ae9b33ee35b"}, - {file = "pep517-0.13.0.tar.gz", hash = "sha256:ae69927c5c172be1add9203726d4b84cf3ebad1edcd5f71fcdc746e66e829f59"}, -] +pep517 = [] platformdirs = [ {file = "platformdirs-2.5.4-py3-none-any.whl", hash = "sha256:af0276409f9a02373d540bf8480021a048711d572745aef4b7842dad245eba10"}, {file = "platformdirs-2.5.4.tar.gz", hash = "sha256:1006647646d80f16130f052404c6b901e80ee4ed6bef6792e1f238a8969106f7"}, @@ -1634,33 +1534,33 @@ pluggy = [ {file = "pluggy-1.0.0-py2.py3-none-any.whl", hash = "sha256:74134bbf457f031a36d68416e1509f34bd5ccc019f0bcc952c7b909d06b37bd3"}, {file = "pluggy-1.0.0.tar.gz", hash = "sha256:4224373bacce55f955a878bf9cfa763c1e360858e330072059e10bad68531159"}, ] -pre-commit = [ - {file = "pre_commit-2.20.0-py2.py3-none-any.whl", hash = "sha256:51a5ba7c480ae8072ecdb6933df22d2f812dc897d5fe848778116129a681aac7"}, - {file = "pre_commit-2.20.0.tar.gz", hash = "sha256:a978dac7bc9ec0bcee55c18a277d553b0f419d259dadb4b9418ff2d00eb43959"}, -] +pre-commit = [] pyarrow = [ - {file = "pyarrow-10.0.0-cp310-cp310-macosx_10_14_universal2.whl", hash = "sha256:10e031794d019425d34406edffe7e32157359e9455f9edb97a1732f8dabf802f"}, - {file = "pyarrow-10.0.0-cp310-cp310-macosx_10_14_x86_64.whl", hash = "sha256:e4c6da9f9e1ff96781ee1478f7cc0860e66c23584887b8e297c4b9905c3c9066"}, - {file = "pyarrow-10.0.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:4051664d354b14939b5da35cfa77821ade594bc1cf56dd2032b3068c96697d74"}, - {file = "pyarrow-10.0.0-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:65d4a312f3ced318423704355acaccc7f7bdfe242472e59bdd54aa0f8837adf8"}, - {file = "pyarrow-10.0.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:758284e1ebd3f2a9abb30544bfec28d151a398bb7c0f2578cbca5ee5b000364a"}, - {file = "pyarrow-10.0.0-cp310-cp310-win_amd64.whl", hash = "sha256:f329951d56b3b943c353f7b27c894e02367a7efbb9fef7979c6b24e02dbfcf55"}, - {file = "pyarrow-10.0.0-cp37-cp37m-macosx_10_14_x86_64.whl", hash = "sha256:511735040b83f2993f78d7fb615e7b88253d75f41500e87e587c40156ff88120"}, - {file = "pyarrow-10.0.0-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:3d2694f08c8d4482d14e3798ff036dbd81ae6b1c47948f52515e1aa90fbec3f0"}, - {file = "pyarrow-10.0.0-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:c79300e1a3e23f2bf4defcf0d70ff5ea25ef6ebf6f121d8670ee14bb662bb7ca"}, - {file = "pyarrow-10.0.0-cp37-cp37m-win_amd64.whl", hash = "sha256:f76157d9579571c865860e5fd004537c03e21139db76692d96fd8a186adab1f2"}, - {file = "pyarrow-10.0.0-cp38-cp38-macosx_10_14_x86_64.whl", hash = "sha256:69b8a1fd99201178799b02f18498633847109b701856ec762f314352a431b7d0"}, - {file = "pyarrow-10.0.0-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:68ccb82c04c0f7abf7a95541d5e9d9d94290fc66a2d36d3f6ea0777f40c15654"}, - {file = "pyarrow-10.0.0-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:b45f969ed924282e9d4ede38f3430630d809c36dbff65452cabce03141943d28"}, - {file = "pyarrow-10.0.0-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:b9f63ceb8346aac0bcb487fafe9faca642ad448ca649fcf66a027c6e120cbc12"}, - {file = "pyarrow-10.0.0-cp38-cp38-win_amd64.whl", hash = "sha256:7ce026274cd5d9934cd3694e89edecde4e036018bbc6cb735fd33b9e967e7d47"}, - {file = "pyarrow-10.0.0-cp39-cp39-macosx_10_14_universal2.whl", hash = "sha256:7e6b837cc44cd62a0e280c8fc4de94ebce503d6d1190e6e94157ab49a8bea67b"}, - {file = "pyarrow-10.0.0-cp39-cp39-macosx_10_14_x86_64.whl", hash = "sha256:7be7f42f713068293308c989a4a3a2de03b70199bdbe753901c6595ff8640c64"}, - {file = "pyarrow-10.0.0-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:b3e3148468d3eed3779d68241f1d13ed4ee7cca4c6dbc7c07e5062b93ad4da33"}, - {file = "pyarrow-10.0.0-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:2d326a9d47ac237d81b8c4337e9d30a0b361835b536fc7ea53991455ce761fbd"}, - {file = "pyarrow-10.0.0-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:25f51dca780fc22cfd7ac30f6bdfe70eb99145aee9acfda987f2c49955d66ed9"}, - {file = "pyarrow-10.0.0-cp39-cp39-win_amd64.whl", hash = "sha256:d45a59e2f47826544c0ca70bc0f7ed8ffa5ad23f93b0458230c7e983bcad1acf"}, - {file = "pyarrow-10.0.0.tar.gz", hash = "sha256:b153b05765393557716e3729cf988442b3ae4f5567364ded40d58c07feed27c2"}, + {file = "pyarrow-10.0.1-cp310-cp310-macosx_10_14_x86_64.whl", hash = "sha256:e00174764a8b4e9d8d5909b6d19ee0c217a6cf0232c5682e31fdfbd5a9f0ae52"}, + {file = "pyarrow-10.0.1-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:6f7a7dbe2f7f65ac1d0bd3163f756deb478a9e9afc2269557ed75b1b25ab3610"}, + {file = "pyarrow-10.0.1-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:cb627673cb98708ef00864e2e243f51ba7b4c1b9f07a1d821f98043eccd3f585"}, + {file = "pyarrow-10.0.1-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:ba71e6fc348c92477586424566110d332f60d9a35cb85278f42e3473bc1373da"}, + {file = "pyarrow-10.0.1-cp310-cp310-win_amd64.whl", hash = "sha256:7b4ede715c004b6fc535de63ef79fa29740b4080639a5ff1ea9ca84e9282f349"}, + {file = "pyarrow-10.0.1-cp311-cp311-macosx_10_14_x86_64.whl", hash = "sha256:e3fe5049d2e9ca661d8e43fab6ad5a4c571af12d20a57dffc392a014caebef65"}, + {file = "pyarrow-10.0.1-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:254017ca43c45c5098b7f2a00e995e1f8346b0fb0be225f042838323bb55283c"}, + {file = "pyarrow-10.0.1-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:70acca1ece4322705652f48db65145b5028f2c01c7e426c5d16a30ba5d739c24"}, + {file = "pyarrow-10.0.1-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:abb57334f2c57979a49b7be2792c31c23430ca02d24becd0b511cbe7b6b08649"}, + {file = "pyarrow-10.0.1-cp311-cp311-win_amd64.whl", hash = "sha256:1765a18205eb1e02ccdedb66049b0ec148c2a0cb52ed1fb3aac322dfc086a6ee"}, + {file = "pyarrow-10.0.1-cp37-cp37m-macosx_10_14_x86_64.whl", hash = "sha256:61f4c37d82fe00d855d0ab522c685262bdeafd3fbcb5fe596fe15025fbc7341b"}, + {file = "pyarrow-10.0.1-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:e141a65705ac98fa52a9113fe574fdaf87fe0316cde2dffe6b94841d3c61544c"}, + {file = "pyarrow-10.0.1-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:bf26f809926a9d74e02d76593026f0aaeac48a65b64f1bb17eed9964bfe7ae1a"}, + {file = "pyarrow-10.0.1-cp37-cp37m-win_amd64.whl", hash = "sha256:443eb9409b0cf78df10ced326490e1a300205a458fbeb0767b6b31ab3ebae6b2"}, + {file = "pyarrow-10.0.1-cp38-cp38-macosx_10_14_x86_64.whl", hash = "sha256:f2d00aa481becf57098e85d99e34a25dba5a9ade2f44eb0b7d80c80f2984fc03"}, + {file = "pyarrow-10.0.1-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:b1fc226d28c7783b52a84d03a66573d5a22e63f8a24b841d5fc68caeed6784d4"}, + {file = "pyarrow-10.0.1-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:efa59933b20183c1c13efc34bd91efc6b2997377c4c6ad9272da92d224e3beb1"}, + {file = "pyarrow-10.0.1-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:668e00e3b19f183394388a687d29c443eb000fb3fe25599c9b4762a0afd37775"}, + {file = "pyarrow-10.0.1-cp38-cp38-win_amd64.whl", hash = "sha256:d1bc6e4d5d6f69e0861d5d7f6cf4d061cf1069cb9d490040129877acf16d4c2a"}, + {file = "pyarrow-10.0.1-cp39-cp39-macosx_10_14_x86_64.whl", hash = "sha256:42ba7c5347ce665338f2bc64685d74855900200dac81a972d49fe127e8132f75"}, + {file = "pyarrow-10.0.1-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:b069602eb1fc09f1adec0a7bdd7897f4d25575611dfa43543c8b8a75d99d6874"}, + {file = "pyarrow-10.0.1-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:94fb4a0c12a2ac1ed8e7e2aa52aade833772cf2d3de9dde685401b22cec30002"}, + {file = "pyarrow-10.0.1-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:db0c5986bf0808927f49640582d2032a07aa49828f14e51f362075f03747d198"}, + {file = "pyarrow-10.0.1-cp39-cp39-win_amd64.whl", hash = "sha256:0ec7587d759153f452d5263dbc8b1af318c4609b607be2bd5127dcda6708cdb1"}, + {file = "pyarrow-10.0.1.tar.gz", hash = "sha256:1a14f57a5f472ce8234f2964cd5184cccaa8df7e04568c64edc33b23eb285dd5"}, ] pycparser = [ {file = "pycparser-2.21-py2.py3-none-any.whl", hash = "sha256:8ee45429555515e1f6b185e78100aea234072576aa43ab53aefcae078162fca9"}, @@ -1704,10 +1604,7 @@ pydantic = [ {file = "pydantic-1.10.2-py3-none-any.whl", hash = "sha256:1b6ee725bd6e83ec78b1aa32c5b1fa67a3a65badddde3976bca5fe4568f27709"}, {file = "pydantic-1.10.2.tar.gz", hash = "sha256:91b8e218852ef6007c2b98cd861601c6a09f1aa32bbbb74fab5b1c33d4a1e410"}, ] -pygments = [ - {file = "Pygments-2.13.0-py3-none-any.whl", hash = "sha256:f643f331ab57ba3c9d89212ee4a2dabc6e94f117cf4eefde99a0574720d14c42"}, - {file = "Pygments-2.13.0.tar.gz", hash = "sha256:56a8508ae95f98e2b9bdf93a6be5ae3f7d8af858b43e02c5a2ff083726be40c1"}, -] +pygments = [] pyparsing = [ {file = "pyparsing-3.0.9-py3-none-any.whl", hash = "sha256:5026bae9a10eeaefb61dab2f09052b9f4307d44aee4eda64b309723d8d206bbc"}, {file = "pyparsing-3.0.9.tar.gz", hash = "sha256:2b020ecf7d21b687f219b71ecad3631f644a47f01403fa1d1036b0c6416d70fb"}, @@ -1720,10 +1617,7 @@ pytest-checkdocs = [ {file = "pytest-checkdocs-2.9.0.tar.gz", hash = "sha256:54b3d1357a289e3f6336d31d4ceb72e99afe0fa4f552990f1b1f015f1925543b"}, {file = "pytest_checkdocs-2.9.0-py3-none-any.whl", hash = "sha256:4231e5426915a02385e386b5eb4a70490b143d97cede4c7c77691bb320b5a73f"}, ] -python-dateutil = [ - {file = "python-dateutil-2.8.2.tar.gz", hash = "sha256:0123cacc1627ae19ddf3c27a5de5bd67ee4586fbdd6440d9748f8abb483d3e86"}, - {file = "python_dateutil-2.8.2-py2.py3-none-any.whl", hash = "sha256:961d03dc3453ebbc59dbdea9e4e11c5651520a876d0f4db161e8674aae935da9"}, -] +python-dateutil = [] python-snappy = [ {file = "python-snappy-0.6.1.tar.gz", hash = "sha256:b6a107ab06206acc5359d4c5632bd9b22d448702a79b3169b0c62e0fb808bb2a"}, {file = "python_snappy-0.6.1-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:b7f920eaf46ebf41bd26f9df51c160d40f9e00b7b48471c3438cb8d027f7fb9b"}, @@ -1786,13 +1680,6 @@ pyyaml = [ {file = "PyYAML-6.0-cp310-cp310-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:f84fbc98b019fef2ee9a1cb3ce93e3187a6df0b2538a651bfb890254ba9f90b5"}, {file = "PyYAML-6.0-cp310-cp310-win32.whl", hash = "sha256:2cd5df3de48857ed0544b34e2d40e9fac445930039f3cfe4bcc592a1f836d513"}, {file = "PyYAML-6.0-cp310-cp310-win_amd64.whl", hash = "sha256:daf496c58a8c52083df09b80c860005194014c3698698d1a57cbcfa182142a3a"}, - {file = "PyYAML-6.0-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:d4b0ba9512519522b118090257be113b9468d804b19d63c71dbcf4a48fa32358"}, - {file = "PyYAML-6.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:81957921f441d50af23654aa6c5e5eaf9b06aba7f0a19c18a538dc7ef291c5a1"}, - {file = "PyYAML-6.0-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:afa17f5bc4d1b10afd4466fd3a44dc0e245382deca5b3c353d8b757f9e3ecb8d"}, - {file = "PyYAML-6.0-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:dbad0e9d368bb989f4515da330b88a057617d16b6a8245084f1b05400f24609f"}, - {file = "PyYAML-6.0-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:432557aa2c09802be39460360ddffd48156e30721f5e8d917f01d31694216782"}, - {file = "PyYAML-6.0-cp311-cp311-win32.whl", hash = "sha256:bfaef573a63ba8923503d27530362590ff4f576c626d86a9fed95822a8255fd7"}, - {file = "PyYAML-6.0-cp311-cp311-win_amd64.whl", hash = "sha256:01b45c0191e6d66c470b6cf1b9531a771a83c1c4208272ead47a3ae4f2f603bf"}, {file = "PyYAML-6.0-cp36-cp36m-macosx_10_9_x86_64.whl", hash = "sha256:897b80890765f037df3403d22bab41627ca8811ae55e9a722fd0392850ec4d86"}, {file = "PyYAML-6.0-cp36-cp36m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:50602afada6d6cbfad699b0c7bb50d5ccffa7e46a3d738092afddc1f9758427f"}, {file = "PyYAML-6.0-cp36-cp36m-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:48c346915c114f5fdb3ead70312bd042a953a8ce5c7106d5bfb1a5254e47da92"}, @@ -1840,21 +1727,12 @@ s3fs = [ {file = "s3fs-2022.10.0-py3-none-any.whl", hash = "sha256:1e134c3577171699feb7c1a0c4713260d5b48296e1708737ff940baef6e2c153"}, {file = "s3fs-2022.10.0.tar.gz", hash = "sha256:e8deb80f20bd0b2059141b874fdb9d6aeb8cce35312ea5f2c02b225a78a00406"}, ] -s3transfer = [ - {file = "s3transfer-0.6.0-py3-none-any.whl", hash = "sha256:06176b74f3a15f61f1b4f25a1fc29a4429040b7647133a463da8fa5bd28d5ecd"}, - {file = "s3transfer-0.6.0.tar.gz", hash = "sha256:2ed07d3866f523cc561bf4a00fc5535827981b117dd7876f036b0c1aca42c947"}, -] -setuptools = [ - {file = "setuptools-65.6.0-py3-none-any.whl", hash = "sha256:6211d2f5eddad8757bd0484923ca7c0a6302ebc4ab32ea5e94357176e0ca0840"}, - {file = "setuptools-65.6.0.tar.gz", hash = "sha256:d1eebf881c6114e51df1664bc2c9133d022f78d12d5f4f665b9191f084e2862d"}, -] +s3transfer = [] six = [ {file = "six-1.16.0-py2.py3-none-any.whl", hash = "sha256:8abb2f1d86890a2dfb989f9a77cfcfd3e47c2a354b01111771326f8aa26e0254"}, {file = "six-1.16.0.tar.gz", hash = "sha256:1e61c37477a1626458e36f7b1d82aa5c9b094fa4802892072e49de9c60c4c926"}, ] -thrift = [ - {file = "thrift-0.16.0.tar.gz", hash = "sha256:2b5b6488fcded21f9d312aa23c9ff6a0195d0f6ae26ddbd5ad9e3e25dfc14408"}, -] +thrift = [] toml = [ {file = "toml-0.10.2-py2.py3-none-any.whl", hash = "sha256:806143ae5bfb6a3c6e736a764057db0e6a0e05e338b5630894a5f779cabb4f9b"}, {file = "toml-0.10.2.tar.gz", hash = "sha256:b3bda1d108d5dd99f4a20d24d9c348e91c4db7ab1b749200bded2f839ccbe68f"}, @@ -1872,12 +1750,12 @@ typing-extensions = [ {file = "typing_extensions-4.4.0.tar.gz", hash = "sha256:1511434bb92bf8dd198c12b1cc812e800d4181cfcb867674e0f8279cc93087aa"}, ] urllib3 = [ - {file = "urllib3-1.26.12-py2.py3-none-any.whl", hash = "sha256:b930dd878d5a8afb066a637fbb35144fe7901e3b209d1cd4f524bd0e9deee997"}, - {file = "urllib3-1.26.12.tar.gz", hash = "sha256:3fa96cf423e6987997fc326ae8df396db2a8b7c667747d47ddd8ecba91f4a74e"}, + {file = "urllib3-1.26.13-py2.py3-none-any.whl", hash = "sha256:47cc05d99aaa09c9e72ed5809b60e7ba354e64b59c9c173ac3018642d8bb41fc"}, + {file = "urllib3-1.26.13.tar.gz", hash = "sha256:c083dd0dce68dbfbe1129d5271cb90f9447dea7d52097c6e0126120c521ddea8"}, ] virtualenv = [ - {file = "virtualenv-20.16.7-py3-none-any.whl", hash = "sha256:efd66b00386fdb7dbe4822d172303f40cd05e50e01740b19ea42425cbe653e29"}, - {file = "virtualenv-20.16.7.tar.gz", hash = "sha256:8691e3ff9387f743e00f6bb20f70121f5e4f596cae754531f2b3b3a1b1ac696e"}, + {file = "virtualenv-20.17.0-py3-none-any.whl", hash = "sha256:40a7e06a98728fd5769e1af6fd1a706005b4bb7e16176a272ed4292473180389"}, + {file = "virtualenv-20.17.0.tar.gz", hash = "sha256:7d6a8d55b2f73b617f684ee40fd85740f062e1f2e379412cb1879c7136f05902"}, ] werkzeug = [ {file = "Werkzeug-2.2.2-py3-none-any.whl", hash = "sha256:f979ab81f58d7318e064e99c4506445d60135ac5cd2e177a2de0089bfd4c9bd5"}, @@ -1953,70 +1831,10 @@ xmltodict = [ {file = "xmltodict-0.13.0-py2.py3-none-any.whl", hash = "sha256:aa89e8fd76320154a40d19a0df04a4695fb9dc5ba977cbb68ab3e4eb225e7852"}, {file = "xmltodict-0.13.0.tar.gz", hash = "sha256:341595a488e3e01a85a9d8911d8912fd922ede5fecc4dce437eb4b6c8d037e56"}, ] -yarl = [ - {file = "yarl-1.8.1-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:abc06b97407868ef38f3d172762f4069323de52f2b70d133d096a48d72215d28"}, - {file = "yarl-1.8.1-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:07b21e274de4c637f3e3b7104694e53260b5fc10d51fb3ec5fed1da8e0f754e3"}, - {file = "yarl-1.8.1-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:9de955d98e02fab288c7718662afb33aab64212ecb368c5dc866d9a57bf48880"}, - {file = "yarl-1.8.1-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:7ec362167e2c9fd178f82f252b6d97669d7245695dc057ee182118042026da40"}, - {file = "yarl-1.8.1-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:20df6ff4089bc86e4a66e3b1380460f864df3dd9dccaf88d6b3385d24405893b"}, - {file = "yarl-1.8.1-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:5999c4662631cb798496535afbd837a102859568adc67d75d2045e31ec3ac497"}, - {file = "yarl-1.8.1-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:ed19b74e81b10b592084a5ad1e70f845f0aacb57577018d31de064e71ffa267a"}, - {file = "yarl-1.8.1-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:1e4808f996ca39a6463f45182e2af2fae55e2560be586d447ce8016f389f626f"}, - {file = "yarl-1.8.1-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:2d800b9c2eaf0684c08be5f50e52bfa2aa920e7163c2ea43f4f431e829b4f0fd"}, - {file = "yarl-1.8.1-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:6628d750041550c5d9da50bb40b5cf28a2e63b9388bac10fedd4f19236ef4957"}, - {file = "yarl-1.8.1-cp310-cp310-musllinux_1_1_ppc64le.whl", hash = "sha256:f5af52738e225fcc526ae64071b7e5342abe03f42e0e8918227b38c9aa711e28"}, - {file = "yarl-1.8.1-cp310-cp310-musllinux_1_1_s390x.whl", hash = "sha256:76577f13333b4fe345c3704811ac7509b31499132ff0181f25ee26619de2c843"}, - {file = "yarl-1.8.1-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:0c03f456522d1ec815893d85fccb5def01ffaa74c1b16ff30f8aaa03eb21e453"}, - {file = "yarl-1.8.1-cp310-cp310-win32.whl", hash = "sha256:ea30a42dc94d42f2ba4d0f7c0ffb4f4f9baa1b23045910c0c32df9c9902cb272"}, - {file = "yarl-1.8.1-cp310-cp310-win_amd64.whl", hash = "sha256:9130ddf1ae9978abe63808b6b60a897e41fccb834408cde79522feb37fb72fb0"}, - {file = "yarl-1.8.1-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:0ab5a138211c1c366404d912824bdcf5545ccba5b3ff52c42c4af4cbdc2c5035"}, - {file = "yarl-1.8.1-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a0fb2cb4204ddb456a8e32381f9a90000429489a25f64e817e6ff94879d432fc"}, - {file = "yarl-1.8.1-cp37-cp37m-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:85cba594433915d5c9a0d14b24cfba0339f57a2fff203a5d4fd070e593307d0b"}, - {file = "yarl-1.8.1-cp37-cp37m-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:1ca7e596c55bd675432b11320b4eacc62310c2145d6801a1f8e9ad160685a231"}, - {file = "yarl-1.8.1-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:d0f77539733e0ec2475ddcd4e26777d08996f8cd55d2aef82ec4d3896687abda"}, - {file = "yarl-1.8.1-cp37-cp37m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:29e256649f42771829974e742061c3501cc50cf16e63f91ed8d1bf98242e5507"}, - {file = "yarl-1.8.1-cp37-cp37m-musllinux_1_1_aarch64.whl", hash = "sha256:7fce6cbc6c170ede0221cc8c91b285f7f3c8b9fe28283b51885ff621bbe0f8ee"}, - {file = "yarl-1.8.1-cp37-cp37m-musllinux_1_1_i686.whl", hash = "sha256:59ddd85a1214862ce7c7c66457f05543b6a275b70a65de366030d56159a979f0"}, - {file = "yarl-1.8.1-cp37-cp37m-musllinux_1_1_ppc64le.whl", hash = "sha256:12768232751689c1a89b0376a96a32bc7633c08da45ad985d0c49ede691f5c0d"}, - {file = "yarl-1.8.1-cp37-cp37m-musllinux_1_1_s390x.whl", hash = "sha256:b19255dde4b4f4c32e012038f2c169bb72e7f081552bea4641cab4d88bc409dd"}, - {file = "yarl-1.8.1-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:6c8148e0b52bf9535c40c48faebb00cb294ee577ca069d21bd5c48d302a83780"}, - {file = "yarl-1.8.1-cp37-cp37m-win32.whl", hash = "sha256:de839c3a1826a909fdbfe05f6fe2167c4ab033f1133757b5936efe2f84904c07"}, - {file = "yarl-1.8.1-cp37-cp37m-win_amd64.whl", hash = "sha256:dd032e8422a52e5a4860e062eb84ac94ea08861d334a4bcaf142a63ce8ad4802"}, - {file = "yarl-1.8.1-cp38-cp38-macosx_10_9_universal2.whl", hash = "sha256:19cd801d6f983918a3f3a39f3a45b553c015c5aac92ccd1fac619bd74beece4a"}, - {file = "yarl-1.8.1-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:6347f1a58e658b97b0a0d1ff7658a03cb79bdbda0331603bed24dd7054a6dea1"}, - {file = "yarl-1.8.1-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:7c0da7e44d0c9108d8b98469338705e07f4bb7dab96dbd8fa4e91b337db42548"}, - {file = "yarl-1.8.1-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:5587bba41399854703212b87071c6d8638fa6e61656385875f8c6dff92b2e461"}, - {file = "yarl-1.8.1-cp38-cp38-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:31a9a04ecccd6b03e2b0e12e82131f1488dea5555a13a4d32f064e22a6003cfe"}, - {file = "yarl-1.8.1-cp38-cp38-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:205904cffd69ae972a1707a1bd3ea7cded594b1d773a0ce66714edf17833cdae"}, - {file = "yarl-1.8.1-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:ea513a25976d21733bff523e0ca836ef1679630ef4ad22d46987d04b372d57fc"}, - {file = "yarl-1.8.1-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:d0b51530877d3ad7a8d47b2fff0c8df3b8f3b8deddf057379ba50b13df2a5eae"}, - {file = "yarl-1.8.1-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:d2b8f245dad9e331540c350285910b20dd913dc86d4ee410c11d48523c4fd546"}, - {file = "yarl-1.8.1-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:ab2a60d57ca88e1d4ca34a10e9fb4ab2ac5ad315543351de3a612bbb0560bead"}, - {file = "yarl-1.8.1-cp38-cp38-musllinux_1_1_ppc64le.whl", hash = "sha256:449c957ffc6bc2309e1fbe67ab7d2c1efca89d3f4912baeb8ead207bb3cc1cd4"}, - {file = "yarl-1.8.1-cp38-cp38-musllinux_1_1_s390x.whl", hash = "sha256:a165442348c211b5dea67c0206fc61366212d7082ba8118c8c5c1c853ea4d82e"}, - {file = "yarl-1.8.1-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:b3ded839a5c5608eec8b6f9ae9a62cb22cd037ea97c627f38ae0841a48f09eae"}, - {file = "yarl-1.8.1-cp38-cp38-win32.whl", hash = "sha256:c1445a0c562ed561d06d8cbc5c8916c6008a31c60bc3655cdd2de1d3bf5174a0"}, - {file = "yarl-1.8.1-cp38-cp38-win_amd64.whl", hash = "sha256:56c11efb0a89700987d05597b08a1efcd78d74c52febe530126785e1b1a285f4"}, - {file = "yarl-1.8.1-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:e80ed5a9939ceb6fda42811542f31c8602be336b1fb977bccb012e83da7e4936"}, - {file = "yarl-1.8.1-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:6afb336e23a793cd3b6476c30f030a0d4c7539cd81649683b5e0c1b0ab0bf350"}, - {file = "yarl-1.8.1-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:4c322cbaa4ed78a8aac89b2174a6df398faf50e5fc12c4c191c40c59d5e28357"}, - {file = "yarl-1.8.1-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:fae37373155f5ef9b403ab48af5136ae9851151f7aacd9926251ab26b953118b"}, - {file = "yarl-1.8.1-cp39-cp39-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:5395da939ffa959974577eff2cbfc24b004a2fb6c346918f39966a5786874e54"}, - {file = "yarl-1.8.1-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:076eede537ab978b605f41db79a56cad2e7efeea2aa6e0fa8f05a26c24a034fb"}, - {file = "yarl-1.8.1-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:3d1a50e461615747dd93c099f297c1994d472b0f4d2db8a64e55b1edf704ec1c"}, - {file = "yarl-1.8.1-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:7de89c8456525650ffa2bb56a3eee6af891e98f498babd43ae307bd42dca98f6"}, - {file = "yarl-1.8.1-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:4a88510731cd8d4befaba5fbd734a7dd914de5ab8132a5b3dde0bbd6c9476c64"}, - {file = "yarl-1.8.1-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:2d93a049d29df172f48bcb09acf9226318e712ce67374f893b460b42cc1380ae"}, - {file = "yarl-1.8.1-cp39-cp39-musllinux_1_1_ppc64le.whl", hash = "sha256:21ac44b763e0eec15746a3d440f5e09ad2ecc8b5f6dcd3ea8cb4773d6d4703e3"}, - {file = "yarl-1.8.1-cp39-cp39-musllinux_1_1_s390x.whl", hash = "sha256:d0272228fabe78ce00a3365ffffd6f643f57a91043e119c289aaba202f4095b0"}, - {file = "yarl-1.8.1-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:99449cd5366fe4608e7226c6cae80873296dfa0cde45d9b498fefa1de315a09e"}, - {file = "yarl-1.8.1-cp39-cp39-win32.whl", hash = "sha256:8b0af1cf36b93cee99a31a545fe91d08223e64390c5ecc5e94c39511832a4bb6"}, - {file = "yarl-1.8.1-cp39-cp39-win_amd64.whl", hash = "sha256:de49d77e968de6626ba7ef4472323f9d2e5a56c1d85b7c0e2a190b2173d3b9be"}, - {file = "yarl-1.8.1.tar.gz", hash = "sha256:af887845b8c2e060eb5605ff72b6f2dd2aab7a761379373fd89d314f4752abbf"}, -] +yarl = [] zipp = [ - {file = "zipp-3.10.0-py3-none-any.whl", hash = "sha256:4fcb6f278987a6605757302a6e40e896257570d11c51628968ccb2a47e80c6c1"}, - {file = "zipp-3.10.0.tar.gz", hash = "sha256:7a7262fd930bd3e36c50b9a64897aec3fafff3dfdeec9623ae22b40e93f99bb8"}, + {file = "zipp-3.11.0-py3-none-any.whl", hash = "sha256:83a28fcb75844b5c0cdaf5aa4003c2d728c77e05f5aeabe8e95e56727005fbaa"}, + {file = "zipp-3.11.0.tar.gz", hash = "sha256:a7a22e05929290a67401440b39690ae6563279bced5f314609d9d03798f56766"}, ] zstandard = [ {file = "zstandard-0.19.0-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:a65e0119ad39e855427520f7829618f78eb2824aa05e63ff19b466080cd99210"}, diff --git a/pyproject.toml b/pyproject.toml index fd8c3fcb52..f6c7126daa 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -57,7 +57,7 @@ fsspec = "2022.10.0" zstandard = "0.19.0" -pyarrow = { version = "10.0.0", optional = true } +pyarrow = { version = "10.0.1", optional = true } duckdb = { version = "0.6.0", optional = true } From 11673b0c8c36bbb5d9fae99cc91f579a31079816 Mon Sep 17 00:00:00 2001 From: Fokko Driesprong Date: Thu, 1 Dec 2022 19:03:41 +0100 Subject: [PATCH 287/642] Python: Set version to 0.2.0 (#6328) --- dev/RELEASE.md | 2 +- pyiceberg/__init__.py | 2 +- pyproject.toml | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/dev/RELEASE.md b/dev/RELEASE.md index b0513f341c..a99683031c 100644 --- a/dev/RELEASE.md +++ b/dev/RELEASE.md @@ -25,7 +25,7 @@ First we're going to release a release candidate (RC) and publish it to the publ ## Running a release candidate -Make sure that you're on the version that you want to release. +Make sure that you're on the version that you want to release. And that the version is correct in `pyproject.toml` and `pyiceberg/__init__.py`. Correct means that it reflects the version that you want to release, and doesn't contain any additional modifiers, such as `dev0`. ```bash export RC=rc1 diff --git a/pyiceberg/__init__.py b/pyiceberg/__init__.py index ba28617441..166b200372 100644 --- a/pyiceberg/__init__.py +++ b/pyiceberg/__init__.py @@ -15,4 +15,4 @@ # specific language governing permissions and limitations # under the License. -__version__ = "0.2.0.dev0" +__version__ = "0.2.0" diff --git a/pyproject.toml b/pyproject.toml index f6c7126daa..86d3bd023a 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -17,7 +17,7 @@ [tool.poetry] name = "pyiceberg" -version = "0.2.0.dev0" +version = "0.2.0" readme = "README.md" homepage = "https://iceberg.apache.org/" repository = "https://github.com/apache/iceberg/" From b127a23bdc8f9c685b74b671e5ef40e6bbb98fbd Mon Sep 17 00:00:00 2001 From: Fokko Driesprong Date: Thu, 1 Dec 2022 19:04:30 +0100 Subject: [PATCH 288/642] Python: Add warning on projection by name (#6334) --- pyiceberg/table/__init__.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/pyiceberg/table/__init__.py b/pyiceberg/table/__init__.py index e989a3c911..3412e8ee2f 100644 --- a/pyiceberg/table/__init__.py +++ b/pyiceberg/table/__init__.py @@ -16,6 +16,7 @@ # under the License. from __future__ import annotations +import warnings from abc import ABC, abstractmethod from dataclasses import dataclass from functools import cached_property @@ -336,6 +337,10 @@ def plan_files(self) -> Iterator[ScanTask]: def to_arrow(self): from pyiceberg.io.pyarrow import PyArrowFileIO + warnings.warn( + "Projection is currently done by name instead of Field ID, this can lead to incorrect results in some cases." + ) + fs = None if isinstance(self.table.io, PyArrowFileIO): scheme, path = PyArrowFileIO.parse_location(self.table.location()) From fe9c3cc38be44fa4ae1adc7e440b07b7dd81867a Mon Sep 17 00:00:00 2001 From: Fokko Driesprong Date: Sun, 4 Dec 2022 19:50:18 +0100 Subject: [PATCH 289/642] Python: Fix PyArrow Type conversion (#6346) * Python: Fix PyArrow Type conversion It turns out that we need to cast the literal to the right type: ``` ArrowNotImplementedError: Function 'greater_equal' has no kernel matching input types (timestamp[us, tz=+00:00], int64) ``` After casting the scalar, it works perfect. One awkward thing to note that we don't have to do this for the `isin` predicate. There this is done for us. Please check the Jupyter notebook. * Explicitly set the type --- pyiceberg/io/pyarrow.py | 25 +++++++++++++++++-------- 1 file changed, 17 insertions(+), 8 deletions(-) diff --git a/pyiceberg/io/pyarrow.py b/pyiceberg/io/pyarrow.py index 751b2815e3..ce4e9b8ae9 100644 --- a/pyiceberg/io/pyarrow.py +++ b/pyiceberg/io/pyarrow.py @@ -64,6 +64,7 @@ DoubleType, FixedType, FloatType, + IcebergType, IntegerType, ListType, LongType, @@ -387,12 +388,20 @@ def _(_: BinaryType) -> pa.DataType: return pa.binary() +def _convert_scalar(value: Any, iceberg_type: IcebergType) -> pa.scalar: + if not isinstance(iceberg_type, PrimitiveType): + raise ValueError(f"Expected primitive type, got: {iceberg_type}") + return pa.scalar(value).cast(_iceberg_to_pyarrow_type(iceberg_type)) + + class _ConvertToArrowExpression(BoundBooleanExpressionVisitor[pc.Expression]): def visit_in(self, term: BoundTerm[pc.Expression], literals: Set[Any]) -> pc.Expression: - return pc.field(term.ref().field.name).isin(literals) + pyarrow_literals = pa.array(literals, type=_iceberg_to_pyarrow_type(term.ref().field.field_type)) + return pc.field(term.ref().field.name).isin(pyarrow_literals) def visit_not_in(self, term: BoundTerm[pc.Expression], literals: Set[Any]) -> pc.Expression: - return ~pc.field(term.ref().field.name).isin(literals) + pyarrow_literals = pa.array(literals, type=_iceberg_to_pyarrow_type(term.ref().field.field_type)) + return ~pc.field(term.ref().field.name).isin(pyarrow_literals) def visit_is_nan(self, term: BoundTerm[pc.Expression]) -> pc.Expression: ref = pc.field(term.ref().field.name) @@ -409,22 +418,22 @@ def visit_not_null(self, term: BoundTerm[Any]) -> pc.Expression: return pc.field(term.ref().field.name).is_valid() def visit_equal(self, term: BoundTerm[Any], literal: Literal[Any]) -> pc.Expression: - return pc.field(term.ref().field.name) == literal.value + return pc.field(term.ref().field.name) == _convert_scalar(literal.value, term.ref().field.field_type) def visit_not_equal(self, term: BoundTerm[Any], literal: Literal[Any]) -> pc.Expression: - return pc.field(term.ref().field.name) != literal.value + return pc.field(term.ref().field.name) != _convert_scalar(literal.value, term.ref().field.field_type) def visit_greater_than_or_equal(self, term: BoundTerm[Any], literal: Literal[Any]) -> pc.Expression: - return pc.field(term.ref().field.name) >= literal.value + return pc.field(term.ref().field.name) >= _convert_scalar(literal.value, term.ref().field.field_type) def visit_greater_than(self, term: BoundTerm[Any], literal: Literal[Any]) -> pc.Expression: - return pc.field(term.ref().field.name) > literal.value + return pc.field(term.ref().field.name) > _convert_scalar(literal.value, term.ref().field.field_type) def visit_less_than(self, term: BoundTerm[Any], literal: Literal[Any]) -> pc.Expression: - return pc.field(term.ref().field.name) < literal.value + return pc.field(term.ref().field.name) < _convert_scalar(literal.value, term.ref().field.field_type) def visit_less_than_or_equal(self, term: BoundTerm[Any], literal: Literal[Any]) -> pc.Expression: - return pc.field(term.ref().field.name) <= literal.value + return pc.field(term.ref().field.name) <= _convert_scalar(literal.value, term.ref().field.field_type) def visit_true(self) -> pc.Expression: return pc.scalar(True) From 1634911b192abc41a9c777a44d65705b9cd351c1 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 6 Dec 2022 09:06:09 +0100 Subject: [PATCH 290/642] Build: Bump moto from 4.0.10 to 4.0.11 in /python (#6356) Bumps [moto](https://github.com/spulec/moto) from 4.0.10 to 4.0.11. - [Release notes](https://github.com/spulec/moto/releases) - [Changelog](https://github.com/spulec/moto/blob/master/CHANGELOG.md) - [Commits](https://github.com/spulec/moto/compare/4.0.10...4.0.11) --- updated-dependencies: - dependency-name: moto dependency-type: direct:development update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- poetry.lock | 231 ++++++++++++++++++++++++++++++++++++++++++++----- pyproject.toml | 2 +- 2 files changed, 209 insertions(+), 24 deletions(-) diff --git a/poetry.lock b/poetry.lock index 3ce6de2222..43fac9650d 100644 --- a/poetry.lock +++ b/poetry.lock @@ -34,7 +34,7 @@ multidict = ">=4.5,<7.0" yarl = ">=1.0,<2.0" [package.extras] -speedups = ["aiodns", "brotli", "cchardet"] +speedups = ["Brotli", "aiodns", "cchardet"] [[package]] name = "aioitertools" @@ -78,7 +78,7 @@ python-versions = ">=3.5" dev = ["cloudpickle", "coverage[toml] (>=5.0.2)", "furo", "hypothesis", "mypy (>=0.900,!=0.940)", "pre-commit", "pympler", "pytest (>=4.3.0)", "pytest-mypy-plugins", "sphinx", "sphinx-notfound-page", "zope.interface"] docs = ["furo", "sphinx", "sphinx-notfound-page", "zope.interface"] tests = ["cloudpickle", "coverage[toml] (>=5.0.2)", "hypothesis", "mypy (>=0.900,!=0.940)", "pympler", "pytest (>=4.3.0)", "pytest-mypy-plugins", "zope.interface"] -tests_no_zope = ["cloudpickle", "coverage[toml] (>=5.0.2)", "hypothesis", "mypy (>=0.900,!=0.940)", "pympler", "pytest (>=4.3.0)", "pytest-mypy-plugins"] +tests-no-zope = ["cloudpickle", "coverage[toml] (>=5.0.2)", "hypothesis", "mypy (>=0.900,!=0.940)", "pympler", "pytest (>=4.3.0)", "pytest-mypy-plugins"] [[package]] name = "boto3" @@ -168,7 +168,7 @@ optional = false python-versions = ">=3.6.0" [package.extras] -unicode_backport = ["unicodedata2"] +unicode-backport = ["unicodedata2"] [[package]] name = "click" @@ -419,7 +419,7 @@ python-versions = "*" [[package]] name = "moto" -version = "4.0.10" +version = "4.0.11" description = "A library that allows your python tests to easily mock out the boto library" category = "dev" optional = false @@ -478,6 +478,9 @@ category = "dev" optional = false python-versions = ">=2.7,!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,!=3.4.*,!=3.5.*,!=3.6.*" +[package.dependencies] +setuptools = "*" + [[package]] name = "numpy" version = "1.23.5" @@ -692,7 +695,7 @@ urllib3 = ">=1.21.1,<1.27" [package.extras] socks = ["PySocks (>=1.5.6,!=1.5.7)"] -use_chardet_on_py3 = ["chardet (>=3.0.2,<6)"] +use-chardet-on-py3 = ["chardet (>=3.0.2,<6)"] [[package]] name = "requests-mock" @@ -774,6 +777,19 @@ botocore = ">=1.12.36,<2.0a.0" [package.extras] crt = ["botocore[crt] (>=1.20.29,<2.0a.0)"] +[[package]] +name = "setuptools" +version = "65.6.3" +description = "Easily download, build, install, upgrade, and uninstall Python packages" +category = "dev" +optional = false +python-versions = ">=3.7" + +[package.extras] +docs = ["furo", "jaraco.packaging (>=9)", "jaraco.tidelift (>=1.4)", "pygments-github-lexers (==0.0.5)", "rst.linker (>=1.9)", "sphinx (>=3.5)", "sphinx-favicon", "sphinx-hoverxref (<2)", "sphinx-inline-tabs", "sphinx-notfound-page (==0.8.3)", "sphinx-reredirects", "sphinxcontrib-towncrier"] +testing = ["build[virtualenv]", "filelock (>=3.4.0)", "flake8 (<5)", "flake8-2020", "ini2toml[lite] (>=0.9)", "jaraco.envs (>=2.2)", "jaraco.path (>=3.2.0)", "pip (>=19.1)", "pip-run (>=8.8)", "pytest (>=6)", "pytest-black (>=0.3.7)", "pytest-checkdocs (>=2.4)", "pytest-cov", "pytest-enabler (>=1.3)", "pytest-flake8", "pytest-mypy (>=0.9.1)", "pytest-perf", "pytest-timeout", "pytest-xdist", "tomli-w (>=1.0.0)", "virtualenv (>=13.0.0)", "wheel"] +testing-integration = ["build[virtualenv]", "filelock (>=3.4.0)", "jaraco.envs (>=2.2)", "jaraco.path (>=3.2.0)", "pytest", "pytest-enabler", "pytest-xdist", "tomli", "virtualenv (>=13.0.0)", "wheel"] + [[package]] name = "six" version = "1.16.0" @@ -939,7 +955,7 @@ snappy = ["python-snappy"] [metadata] lock-version = "1.1" python-versions = "^3.8" -content-hash = "064eb484ccbc348f5b0794b51c0365bdc1ee83dd47970652978a781adb6721c3" +content-hash = "8fa466512e3f74dd8f985ed4c85d2698407bffda8f23858700f5089e904f3982" [metadata.files] aiobotocore = [ @@ -1043,8 +1059,14 @@ aiosignal = [ {file = "aiosignal-1.3.1-py3-none-any.whl", hash = "sha256:f8376fb07dd1e86a584e4fcdec80b36b7f81aac666ebc724e2c090300dd83b17"}, {file = "aiosignal-1.3.1.tar.gz", hash = "sha256:54cd96e15e1649b75d6c87526a6ff0b6c1b0dd3459f43d9ca11d48c339b68cfc"}, ] -async-timeout = [] -attrs = [] +async-timeout = [ + {file = "async-timeout-4.0.2.tar.gz", hash = "sha256:2163e1640ddb52b7a8c80d0a67a08587e5d245cc9c553a74a847056bc2976b15"}, + {file = "async_timeout-4.0.2-py3-none-any.whl", hash = "sha256:8ca1e4fcf50d07413d66d1a5e416e42cfdf5851c981d679a09851a6853383b3c"}, +] +attrs = [ + {file = "attrs-22.1.0-py2.py3-none-any.whl", hash = "sha256:86efa402f67bf2df34f51a335487cf46b1ec130d02b8d39fd248abfd30da551c"}, + {file = "attrs-22.1.0.tar.gz", hash = "sha256:29adc2665447e5191d0e7c568fde78b21f9672d344281d0c6e1ab085429b22b6"}, +] boto3 = [ {file = "boto3-1.24.59-py3-none-any.whl", hash = "sha256:34ab44146a2c4e7f4e72737f4b27e6eb5e0a7855c2f4599e3d9199b6a0a2d575"}, {file = "boto3-1.24.59.tar.gz", hash = "sha256:a50b4323f9579cfe22fcf5531fbd40b567d4d74c1adce06aeb5c95fce2a6fb40"}, @@ -1131,13 +1153,22 @@ cfgv = [ {file = "cfgv-3.3.1-py2.py3-none-any.whl", hash = "sha256:c6a0883f3917a037485059700b9e75da2464e6c27051014ad85ba6aaa5884426"}, {file = "cfgv-3.3.1.tar.gz", hash = "sha256:f5a830efb9ce7a445376bb66ec94c638a9787422f96264c98edc6bdeed8ab736"}, ] -charset-normalizer = [] -click = [] +charset-normalizer = [ + {file = "charset-normalizer-2.1.1.tar.gz", hash = "sha256:5a3d016c7c547f69d6f81fb0db9449ce888b418b5b9952cc5e6e66843e9dd845"}, + {file = "charset_normalizer-2.1.1-py3-none-any.whl", hash = "sha256:83e9a75d1911279afd89352c68b45348559d1fc0506b054b346651b5e7fee29f"}, +] +click = [ + {file = "click-8.1.3-py3-none-any.whl", hash = "sha256:bb4d8133cb15a609f44e8213d9b391b0809795062913b383c62be0ee95b1db48"}, + {file = "click-8.1.3.tar.gz", hash = "sha256:7682dc8afb30297001674575ea00d1814d808d6a36af415a82bd481d37ba7b8e"}, +] colorama = [ {file = "colorama-0.4.6-py2.py3-none-any.whl", hash = "sha256:4f1d9991f5acc0ca119f9d443620b77f9d6b33703e51011c16baf57afb285fc6"}, {file = "colorama-0.4.6.tar.gz", hash = "sha256:08695f5cb7ed6e0531a20572697297273c47b8cae5a63ffc6d6ed5c201be6e44"}, ] -commonmark = [] +commonmark = [ + {file = "commonmark-0.9.1-py2.py3-none-any.whl", hash = "sha256:da2f38c92590f83de410ba1a3cbceafbc74fee9def35f9251ba9a971d6d66fd9"}, + {file = "commonmark-0.9.1.tar.gz", hash = "sha256:452f9dc859be7f06631ddcb328b6919c67984aca654e5fefb3914d54691aed60"}, +] coverage = [ {file = "coverage-6.5.0-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:ef8674b0ee8cc11e2d574e3e2998aea5df5ab242e012286824ea3c6970580e53"}, {file = "coverage-6.5.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:784f53ebc9f3fd0e2a3f6a78b2be1bd1f5575d7863e10c6e12504f240fd06660"}, @@ -1302,7 +1333,10 @@ fastavro = [ {file = "fastavro-1.7.0-cp39-cp39-win_amd64.whl", hash = "sha256:492e8902063aa1c73170e11c18495fcaa86b71eae3513ef83ba438ca02b16b34"}, {file = "fastavro-1.7.0.tar.gz", hash = "sha256:4b1205f46489b4032d3155c1ab44d9824be0c7454df98d3a5bd22b78b98f23c8"}, ] -filelock = [] +filelock = [ + {file = "filelock-3.8.0-py3-none-any.whl", hash = "sha256:617eb4e5eedc82fc5f47b6d61e4d11cb837c56cb4544e39081099fa17ad109d4"}, + {file = "filelock-3.8.0.tar.gz", hash = "sha256:55447caa666f2198c5b6b13a26d2084d26fa5b115c00d065664b2124680c4edc"}, +] frozenlist = [ {file = "frozenlist-1.3.3-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:ff8bf625fe85e119553b5383ba0fb6aa3d0ec2ae980295aaefa552374926b3f4"}, {file = "frozenlist-1.3.3-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:dfbac4c2dfcc082fcf8d942d1e49b6aa0766c19d3358bd86e2000bf0fa4a9cf0"}, @@ -1403,7 +1437,10 @@ jinja2 = [ {file = "Jinja2-3.1.2-py3-none-any.whl", hash = "sha256:6088930bfe239f0e6710546ab9c19c9ef35e29792895fed6e6e31a023a182a61"}, {file = "Jinja2-3.1.2.tar.gz", hash = "sha256:31351a702a408a9e7595a8fc6150fc3f43bb6bf7e319770cbc0db9df9437e852"}, ] -jmespath = [] +jmespath = [ + {file = "jmespath-1.0.1-py3-none-any.whl", hash = "sha256:02e2e4cc71b5bcab88332eebf907519190dd9e6e82107fa7f83b1003a6252980"}, + {file = "jmespath-1.0.1.tar.gz", hash = "sha256:90261b206d6defd58fdd5e85f478bf633a2901798906be2ad389150c5c60edbe"}, +] markupsafe = [ {file = "MarkupSafe-2.1.1-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:86b1f75c4e7c2ac2ccdaec2b9022845dbb81880ca318bb7a0a01fbf7813e3812"}, {file = "MarkupSafe-2.1.1-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:f121a1420d4e173a5d96e47e9a0c0dcff965afdf1626d28de1460815f7c4ee7a"}, @@ -1483,10 +1520,70 @@ mmhash3 = [ {file = "mmhash3-3.0.1.tar.gz", hash = "sha256:a00d68f4a1cc434b9501513c8a29e18ed1ddad383677d72b41d71d0d862348af"}, ] moto = [ - {file = "moto-4.0.10-py3-none-any.whl", hash = "sha256:356bf792b439228891c910e2a0fafd4264334cf9000b508c732ff43d8694fb6a"}, - {file = "moto-4.0.10.tar.gz", hash = "sha256:9ba96d04a472d5682493cad7fee33337da34ebef18b397af1ea6dfb41efbe148"}, + {file = "moto-4.0.11-py3-none-any.whl", hash = "sha256:704d6d38a4e6fe49e1fe9c6b4127ca46c66aac00368149bc1f1d70a0ceff8846"}, + {file = "moto-4.0.11.tar.gz", hash = "sha256:a6388de4a746e0b509286e1d7e70f86900b4f69ec65f6c92c47e570f95d05b14"}, +] +multidict = [ + {file = "multidict-6.0.2-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:0b9e95a740109c6047602f4db4da9949e6c5945cefbad34a1299775ddc9a62e2"}, + {file = "multidict-6.0.2-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:ac0e27844758d7177989ce406acc6a83c16ed4524ebc363c1f748cba184d89d3"}, + {file = "multidict-6.0.2-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:041b81a5f6b38244b34dc18c7b6aba91f9cdaf854d9a39e5ff0b58e2b5773b9c"}, + {file = "multidict-6.0.2-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:5fdda29a3c7e76a064f2477c9aab1ba96fd94e02e386f1e665bca1807fc5386f"}, + {file = "multidict-6.0.2-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:3368bf2398b0e0fcbf46d85795adc4c259299fec50c1416d0f77c0a843a3eed9"}, + {file = "multidict-6.0.2-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:f4f052ee022928d34fe1f4d2bc743f32609fb79ed9c49a1710a5ad6b2198db20"}, + {file = "multidict-6.0.2-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:225383a6603c086e6cef0f2f05564acb4f4d5f019a4e3e983f572b8530f70c88"}, + {file = "multidict-6.0.2-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:50bd442726e288e884f7be9071016c15a8742eb689a593a0cac49ea093eef0a7"}, + {file = "multidict-6.0.2-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:47e6a7e923e9cada7c139531feac59448f1f47727a79076c0b1ee80274cd8eee"}, + {file = "multidict-6.0.2-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:0556a1d4ea2d949efe5fd76a09b4a82e3a4a30700553a6725535098d8d9fb672"}, + {file = "multidict-6.0.2-cp310-cp310-musllinux_1_1_ppc64le.whl", hash = "sha256:626fe10ac87851f4cffecee161fc6f8f9853f0f6f1035b59337a51d29ff3b4f9"}, + {file = "multidict-6.0.2-cp310-cp310-musllinux_1_1_s390x.whl", hash = "sha256:8064b7c6f0af936a741ea1efd18690bacfbae4078c0c385d7c3f611d11f0cf87"}, + {file = "multidict-6.0.2-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:2d36e929d7f6a16d4eb11b250719c39560dd70545356365b494249e2186bc389"}, + {file = "multidict-6.0.2-cp310-cp310-win32.whl", hash = "sha256:fcb91630817aa8b9bc4a74023e4198480587269c272c58b3279875ed7235c293"}, + {file = "multidict-6.0.2-cp310-cp310-win_amd64.whl", hash = "sha256:8cbf0132f3de7cc6c6ce00147cc78e6439ea736cee6bca4f068bcf892b0fd658"}, + {file = "multidict-6.0.2-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:05f6949d6169878a03e607a21e3b862eaf8e356590e8bdae4227eedadacf6e51"}, + {file = "multidict-6.0.2-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:e2c2e459f7050aeb7c1b1276763364884595d47000c1cddb51764c0d8976e608"}, + {file = "multidict-6.0.2-cp37-cp37m-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:d0509e469d48940147e1235d994cd849a8f8195e0bca65f8f5439c56e17872a3"}, + {file = "multidict-6.0.2-cp37-cp37m-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:514fe2b8d750d6cdb4712346a2c5084a80220821a3e91f3f71eec11cf8d28fd4"}, + {file = "multidict-6.0.2-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:19adcfc2a7197cdc3987044e3f415168fc5dc1f720c932eb1ef4f71a2067e08b"}, + {file = "multidict-6.0.2-cp37-cp37m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:b9d153e7f1f9ba0b23ad1568b3b9e17301e23b042c23870f9ee0522dc5cc79e8"}, + {file = "multidict-6.0.2-cp37-cp37m-musllinux_1_1_aarch64.whl", hash = "sha256:aef9cc3d9c7d63d924adac329c33835e0243b5052a6dfcbf7732a921c6e918ba"}, + {file = "multidict-6.0.2-cp37-cp37m-musllinux_1_1_i686.whl", hash = "sha256:4571f1beddff25f3e925eea34268422622963cd8dc395bb8778eb28418248e43"}, + {file = "multidict-6.0.2-cp37-cp37m-musllinux_1_1_ppc64le.whl", hash = "sha256:d48b8ee1d4068561ce8033d2c344cf5232cb29ee1a0206a7b828c79cbc5982b8"}, + {file = "multidict-6.0.2-cp37-cp37m-musllinux_1_1_s390x.whl", hash = "sha256:45183c96ddf61bf96d2684d9fbaf6f3564d86b34cb125761f9a0ef9e36c1d55b"}, + {file = "multidict-6.0.2-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:75bdf08716edde767b09e76829db8c1e5ca9d8bb0a8d4bd94ae1eafe3dac5e15"}, + {file = "multidict-6.0.2-cp37-cp37m-win32.whl", hash = "sha256:a45e1135cb07086833ce969555df39149680e5471c04dfd6a915abd2fc3f6dbc"}, + {file = "multidict-6.0.2-cp37-cp37m-win_amd64.whl", hash = "sha256:6f3cdef8a247d1eafa649085812f8a310e728bdf3900ff6c434eafb2d443b23a"}, + {file = "multidict-6.0.2-cp38-cp38-macosx_10_9_universal2.whl", hash = "sha256:0327292e745a880459ef71be14e709aaea2f783f3537588fb4ed09b6c01bca60"}, + {file = "multidict-6.0.2-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:e875b6086e325bab7e680e4316d667fc0e5e174bb5611eb16b3ea121c8951b86"}, + {file = "multidict-6.0.2-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:feea820722e69451743a3d56ad74948b68bf456984d63c1a92e8347b7b88452d"}, + {file = "multidict-6.0.2-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:9cc57c68cb9139c7cd6fc39f211b02198e69fb90ce4bc4a094cf5fe0d20fd8b0"}, + {file = "multidict-6.0.2-cp38-cp38-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:497988d6b6ec6ed6f87030ec03280b696ca47dbf0648045e4e1d28b80346560d"}, + {file = "multidict-6.0.2-cp38-cp38-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:89171b2c769e03a953d5969b2f272efa931426355b6c0cb508022976a17fd376"}, + {file = "multidict-6.0.2-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:684133b1e1fe91eda8fa7447f137c9490a064c6b7f392aa857bba83a28cfb693"}, + {file = "multidict-6.0.2-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:fd9fc9c4849a07f3635ccffa895d57abce554b467d611a5009ba4f39b78a8849"}, + {file = "multidict-6.0.2-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:e07c8e79d6e6fd37b42f3250dba122053fddb319e84b55dd3a8d6446e1a7ee49"}, + {file = "multidict-6.0.2-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:4070613ea2227da2bfb2c35a6041e4371b0af6b0be57f424fe2318b42a748516"}, + {file = "multidict-6.0.2-cp38-cp38-musllinux_1_1_ppc64le.whl", hash = "sha256:47fbeedbf94bed6547d3aa632075d804867a352d86688c04e606971595460227"}, + {file = "multidict-6.0.2-cp38-cp38-musllinux_1_1_s390x.whl", hash = "sha256:5774d9218d77befa7b70d836004a768fb9aa4fdb53c97498f4d8d3f67bb9cfa9"}, + {file = "multidict-6.0.2-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:2957489cba47c2539a8eb7ab32ff49101439ccf78eab724c828c1a54ff3ff98d"}, + {file = "multidict-6.0.2-cp38-cp38-win32.whl", hash = "sha256:e5b20e9599ba74391ca0cfbd7b328fcc20976823ba19bc573983a25b32e92b57"}, + {file = "multidict-6.0.2-cp38-cp38-win_amd64.whl", hash = "sha256:8004dca28e15b86d1b1372515f32eb6f814bdf6f00952699bdeb541691091f96"}, + {file = "multidict-6.0.2-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:2e4a0785b84fb59e43c18a015ffc575ba93f7d1dbd272b4cdad9f5134b8a006c"}, + {file = "multidict-6.0.2-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:6701bf8a5d03a43375909ac91b6980aea74b0f5402fbe9428fc3f6edf5d9677e"}, + {file = "multidict-6.0.2-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:a007b1638e148c3cfb6bf0bdc4f82776cef0ac487191d093cdc316905e504071"}, + {file = "multidict-6.0.2-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:07a017cfa00c9890011628eab2503bee5872f27144936a52eaab449be5eaf032"}, + {file = "multidict-6.0.2-cp39-cp39-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:c207fff63adcdf5a485969131dc70e4b194327666b7e8a87a97fbc4fd80a53b2"}, + {file = "multidict-6.0.2-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:373ba9d1d061c76462d74e7de1c0c8e267e9791ee8cfefcf6b0b2495762c370c"}, + {file = "multidict-6.0.2-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:bfba7c6d5d7c9099ba21f84662b037a0ffd4a5e6b26ac07d19e423e6fdf965a9"}, + {file = "multidict-6.0.2-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:19d9bad105dfb34eb539c97b132057a4e709919ec4dd883ece5838bcbf262b80"}, + {file = "multidict-6.0.2-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:de989b195c3d636ba000ee4281cd03bb1234635b124bf4cd89eeee9ca8fcb09d"}, + {file = "multidict-6.0.2-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:7c40b7bbece294ae3a87c1bc2abff0ff9beef41d14188cda94ada7bcea99b0fb"}, + {file = "multidict-6.0.2-cp39-cp39-musllinux_1_1_ppc64le.whl", hash = "sha256:d16cce709ebfadc91278a1c005e3c17dd5f71f5098bfae1035149785ea6e9c68"}, + {file = "multidict-6.0.2-cp39-cp39-musllinux_1_1_s390x.whl", hash = "sha256:a2c34a93e1d2aa35fbf1485e5010337c72c6791407d03aa5f4eed920343dd360"}, + {file = "multidict-6.0.2-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:feba80698173761cddd814fa22e88b0661e98cb810f9f986c54aa34d281e4937"}, + {file = "multidict-6.0.2-cp39-cp39-win32.whl", hash = "sha256:23b616fdc3c74c9fe01d76ce0d1ce872d2d396d8fa8e4899398ad64fb5aa214a"}, + {file = "multidict-6.0.2-cp39-cp39-win_amd64.whl", hash = "sha256:4bae31803d708f6f15fd98be6a6ac0b6958fcf68fda3c77a048a4f9073704aae"}, + {file = "multidict-6.0.2.tar.gz", hash = "sha256:5ff3bd75f38e4c43f1f470f2df7a4d430b821c4ce22be384e1459cb57d6bb013"}, ] -multidict = [] nodeenv = [ {file = "nodeenv-1.7.0-py2.py3-none-any.whl", hash = "sha256:27083a7b96a25f2f5e1d8cb4b6317ee8aeda3bdd121394e5ac54e498028a042e"}, {file = "nodeenv-1.7.0.tar.gz", hash = "sha256:e0e7f7dfb85fc5394c6fe1e8fa98131a2473e04311a45afb6508f7cf1836fa2b"}, @@ -1525,7 +1622,10 @@ packaging = [ {file = "packaging-21.3-py3-none-any.whl", hash = "sha256:ef103e05f519cdc783ae24ea4e2e0f508a9c99b2d4969652eed6a2e1ea5bd522"}, {file = "packaging-21.3.tar.gz", hash = "sha256:dd47c42927d89ab911e606518907cc2d3a1f38bbd026385970643f9c5b8ecfeb"}, ] -pep517 = [] +pep517 = [ + {file = "pep517-0.13.0-py3-none-any.whl", hash = "sha256:4ba4446d80aed5b5eac6509ade100bff3e7943a8489de249654a5ae9b33ee35b"}, + {file = "pep517-0.13.0.tar.gz", hash = "sha256:ae69927c5c172be1add9203726d4b84cf3ebad1edcd5f71fcdc746e66e829f59"}, +] platformdirs = [ {file = "platformdirs-2.5.4-py3-none-any.whl", hash = "sha256:af0276409f9a02373d540bf8480021a048711d572745aef4b7842dad245eba10"}, {file = "platformdirs-2.5.4.tar.gz", hash = "sha256:1006647646d80f16130f052404c6b901e80ee4ed6bef6792e1f238a8969106f7"}, @@ -1534,7 +1634,10 @@ pluggy = [ {file = "pluggy-1.0.0-py2.py3-none-any.whl", hash = "sha256:74134bbf457f031a36d68416e1509f34bd5ccc019f0bcc952c7b909d06b37bd3"}, {file = "pluggy-1.0.0.tar.gz", hash = "sha256:4224373bacce55f955a878bf9cfa763c1e360858e330072059e10bad68531159"}, ] -pre-commit = [] +pre-commit = [ + {file = "pre_commit-2.20.0-py2.py3-none-any.whl", hash = "sha256:51a5ba7c480ae8072ecdb6933df22d2f812dc897d5fe848778116129a681aac7"}, + {file = "pre_commit-2.20.0.tar.gz", hash = "sha256:a978dac7bc9ec0bcee55c18a277d553b0f419d259dadb4b9418ff2d00eb43959"}, +] pyarrow = [ {file = "pyarrow-10.0.1-cp310-cp310-macosx_10_14_x86_64.whl", hash = "sha256:e00174764a8b4e9d8d5909b6d19ee0c217a6cf0232c5682e31fdfbd5a9f0ae52"}, {file = "pyarrow-10.0.1-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:6f7a7dbe2f7f65ac1d0bd3163f756deb478a9e9afc2269557ed75b1b25ab3610"}, @@ -1604,7 +1707,10 @@ pydantic = [ {file = "pydantic-1.10.2-py3-none-any.whl", hash = "sha256:1b6ee725bd6e83ec78b1aa32c5b1fa67a3a65badddde3976bca5fe4568f27709"}, {file = "pydantic-1.10.2.tar.gz", hash = "sha256:91b8e218852ef6007c2b98cd861601c6a09f1aa32bbbb74fab5b1c33d4a1e410"}, ] -pygments = [] +pygments = [ + {file = "Pygments-2.13.0-py3-none-any.whl", hash = "sha256:f643f331ab57ba3c9d89212ee4a2dabc6e94f117cf4eefde99a0574720d14c42"}, + {file = "Pygments-2.13.0.tar.gz", hash = "sha256:56a8508ae95f98e2b9bdf93a6be5ae3f7d8af858b43e02c5a2ff083726be40c1"}, +] pyparsing = [ {file = "pyparsing-3.0.9-py3-none-any.whl", hash = "sha256:5026bae9a10eeaefb61dab2f09052b9f4307d44aee4eda64b309723d8d206bbc"}, {file = "pyparsing-3.0.9.tar.gz", hash = "sha256:2b020ecf7d21b687f219b71ecad3631f644a47f01403fa1d1036b0c6416d70fb"}, @@ -1617,7 +1723,10 @@ pytest-checkdocs = [ {file = "pytest-checkdocs-2.9.0.tar.gz", hash = "sha256:54b3d1357a289e3f6336d31d4ceb72e99afe0fa4f552990f1b1f015f1925543b"}, {file = "pytest_checkdocs-2.9.0-py3-none-any.whl", hash = "sha256:4231e5426915a02385e386b5eb4a70490b143d97cede4c7c77691bb320b5a73f"}, ] -python-dateutil = [] +python-dateutil = [ + {file = "python-dateutil-2.8.2.tar.gz", hash = "sha256:0123cacc1627ae19ddf3c27a5de5bd67ee4586fbdd6440d9748f8abb483d3e86"}, + {file = "python_dateutil-2.8.2-py2.py3-none-any.whl", hash = "sha256:961d03dc3453ebbc59dbdea9e4e11c5651520a876d0f4db161e8674aae935da9"}, +] python-snappy = [ {file = "python-snappy-0.6.1.tar.gz", hash = "sha256:b6a107ab06206acc5359d4c5632bd9b22d448702a79b3169b0c62e0fb808bb2a"}, {file = "python_snappy-0.6.1-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:b7f920eaf46ebf41bd26f9df51c160d40f9e00b7b48471c3438cb8d027f7fb9b"}, @@ -1680,6 +1789,13 @@ pyyaml = [ {file = "PyYAML-6.0-cp310-cp310-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:f84fbc98b019fef2ee9a1cb3ce93e3187a6df0b2538a651bfb890254ba9f90b5"}, {file = "PyYAML-6.0-cp310-cp310-win32.whl", hash = "sha256:2cd5df3de48857ed0544b34e2d40e9fac445930039f3cfe4bcc592a1f836d513"}, {file = "PyYAML-6.0-cp310-cp310-win_amd64.whl", hash = "sha256:daf496c58a8c52083df09b80c860005194014c3698698d1a57cbcfa182142a3a"}, + {file = "PyYAML-6.0-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:d4b0ba9512519522b118090257be113b9468d804b19d63c71dbcf4a48fa32358"}, + {file = "PyYAML-6.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:81957921f441d50af23654aa6c5e5eaf9b06aba7f0a19c18a538dc7ef291c5a1"}, + {file = "PyYAML-6.0-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:afa17f5bc4d1b10afd4466fd3a44dc0e245382deca5b3c353d8b757f9e3ecb8d"}, + {file = "PyYAML-6.0-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:dbad0e9d368bb989f4515da330b88a057617d16b6a8245084f1b05400f24609f"}, + {file = "PyYAML-6.0-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:432557aa2c09802be39460360ddffd48156e30721f5e8d917f01d31694216782"}, + {file = "PyYAML-6.0-cp311-cp311-win32.whl", hash = "sha256:bfaef573a63ba8923503d27530362590ff4f576c626d86a9fed95822a8255fd7"}, + {file = "PyYAML-6.0-cp311-cp311-win_amd64.whl", hash = "sha256:01b45c0191e6d66c470b6cf1b9531a771a83c1c4208272ead47a3ae4f2f603bf"}, {file = "PyYAML-6.0-cp36-cp36m-macosx_10_9_x86_64.whl", hash = "sha256:897b80890765f037df3403d22bab41627ca8811ae55e9a722fd0392850ec4d86"}, {file = "PyYAML-6.0-cp36-cp36m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:50602afada6d6cbfad699b0c7bb50d5ccffa7e46a3d738092afddc1f9758427f"}, {file = "PyYAML-6.0-cp36-cp36m-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:48c346915c114f5fdb3ead70312bd042a953a8ce5c7106d5bfb1a5254e47da92"}, @@ -1727,12 +1843,21 @@ s3fs = [ {file = "s3fs-2022.10.0-py3-none-any.whl", hash = "sha256:1e134c3577171699feb7c1a0c4713260d5b48296e1708737ff940baef6e2c153"}, {file = "s3fs-2022.10.0.tar.gz", hash = "sha256:e8deb80f20bd0b2059141b874fdb9d6aeb8cce35312ea5f2c02b225a78a00406"}, ] -s3transfer = [] +s3transfer = [ + {file = "s3transfer-0.6.0-py3-none-any.whl", hash = "sha256:06176b74f3a15f61f1b4f25a1fc29a4429040b7647133a463da8fa5bd28d5ecd"}, + {file = "s3transfer-0.6.0.tar.gz", hash = "sha256:2ed07d3866f523cc561bf4a00fc5535827981b117dd7876f036b0c1aca42c947"}, +] +setuptools = [ + {file = "setuptools-65.6.3-py3-none-any.whl", hash = "sha256:57f6f22bde4e042978bcd50176fdb381d7c21a9efa4041202288d3737a0c6a54"}, + {file = "setuptools-65.6.3.tar.gz", hash = "sha256:a7620757bf984b58deaf32fc8a4577a9bbc0850cf92c20e1ce41c38c19e5fb75"}, +] six = [ {file = "six-1.16.0-py2.py3-none-any.whl", hash = "sha256:8abb2f1d86890a2dfb989f9a77cfcfd3e47c2a354b01111771326f8aa26e0254"}, {file = "six-1.16.0.tar.gz", hash = "sha256:1e61c37477a1626458e36f7b1d82aa5c9b094fa4802892072e49de9c60c4c926"}, ] -thrift = [] +thrift = [ + {file = "thrift-0.16.0.tar.gz", hash = "sha256:2b5b6488fcded21f9d312aa23c9ff6a0195d0f6ae26ddbd5ad9e3e25dfc14408"}, +] toml = [ {file = "toml-0.10.2-py2.py3-none-any.whl", hash = "sha256:806143ae5bfb6a3c6e736a764057db0e6a0e05e338b5630894a5f779cabb4f9b"}, {file = "toml-0.10.2.tar.gz", hash = "sha256:b3bda1d108d5dd99f4a20d24d9c348e91c4db7ab1b749200bded2f839ccbe68f"}, @@ -1831,7 +1956,67 @@ xmltodict = [ {file = "xmltodict-0.13.0-py2.py3-none-any.whl", hash = "sha256:aa89e8fd76320154a40d19a0df04a4695fb9dc5ba977cbb68ab3e4eb225e7852"}, {file = "xmltodict-0.13.0.tar.gz", hash = "sha256:341595a488e3e01a85a9d8911d8912fd922ede5fecc4dce437eb4b6c8d037e56"}, ] -yarl = [] +yarl = [ + {file = "yarl-1.8.1-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:abc06b97407868ef38f3d172762f4069323de52f2b70d133d096a48d72215d28"}, + {file = "yarl-1.8.1-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:07b21e274de4c637f3e3b7104694e53260b5fc10d51fb3ec5fed1da8e0f754e3"}, + {file = "yarl-1.8.1-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:9de955d98e02fab288c7718662afb33aab64212ecb368c5dc866d9a57bf48880"}, + {file = "yarl-1.8.1-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:7ec362167e2c9fd178f82f252b6d97669d7245695dc057ee182118042026da40"}, + {file = "yarl-1.8.1-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:20df6ff4089bc86e4a66e3b1380460f864df3dd9dccaf88d6b3385d24405893b"}, + {file = "yarl-1.8.1-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:5999c4662631cb798496535afbd837a102859568adc67d75d2045e31ec3ac497"}, + {file = "yarl-1.8.1-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:ed19b74e81b10b592084a5ad1e70f845f0aacb57577018d31de064e71ffa267a"}, + {file = "yarl-1.8.1-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:1e4808f996ca39a6463f45182e2af2fae55e2560be586d447ce8016f389f626f"}, + {file = "yarl-1.8.1-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:2d800b9c2eaf0684c08be5f50e52bfa2aa920e7163c2ea43f4f431e829b4f0fd"}, + {file = "yarl-1.8.1-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:6628d750041550c5d9da50bb40b5cf28a2e63b9388bac10fedd4f19236ef4957"}, + {file = "yarl-1.8.1-cp310-cp310-musllinux_1_1_ppc64le.whl", hash = "sha256:f5af52738e225fcc526ae64071b7e5342abe03f42e0e8918227b38c9aa711e28"}, + {file = "yarl-1.8.1-cp310-cp310-musllinux_1_1_s390x.whl", hash = "sha256:76577f13333b4fe345c3704811ac7509b31499132ff0181f25ee26619de2c843"}, + {file = "yarl-1.8.1-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:0c03f456522d1ec815893d85fccb5def01ffaa74c1b16ff30f8aaa03eb21e453"}, + {file = "yarl-1.8.1-cp310-cp310-win32.whl", hash = "sha256:ea30a42dc94d42f2ba4d0f7c0ffb4f4f9baa1b23045910c0c32df9c9902cb272"}, + {file = "yarl-1.8.1-cp310-cp310-win_amd64.whl", hash = "sha256:9130ddf1ae9978abe63808b6b60a897e41fccb834408cde79522feb37fb72fb0"}, + {file = "yarl-1.8.1-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:0ab5a138211c1c366404d912824bdcf5545ccba5b3ff52c42c4af4cbdc2c5035"}, + {file = "yarl-1.8.1-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a0fb2cb4204ddb456a8e32381f9a90000429489a25f64e817e6ff94879d432fc"}, + {file = "yarl-1.8.1-cp37-cp37m-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:85cba594433915d5c9a0d14b24cfba0339f57a2fff203a5d4fd070e593307d0b"}, + {file = "yarl-1.8.1-cp37-cp37m-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:1ca7e596c55bd675432b11320b4eacc62310c2145d6801a1f8e9ad160685a231"}, + {file = "yarl-1.8.1-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:d0f77539733e0ec2475ddcd4e26777d08996f8cd55d2aef82ec4d3896687abda"}, + {file = "yarl-1.8.1-cp37-cp37m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:29e256649f42771829974e742061c3501cc50cf16e63f91ed8d1bf98242e5507"}, + {file = "yarl-1.8.1-cp37-cp37m-musllinux_1_1_aarch64.whl", hash = "sha256:7fce6cbc6c170ede0221cc8c91b285f7f3c8b9fe28283b51885ff621bbe0f8ee"}, + {file = "yarl-1.8.1-cp37-cp37m-musllinux_1_1_i686.whl", hash = "sha256:59ddd85a1214862ce7c7c66457f05543b6a275b70a65de366030d56159a979f0"}, + {file = "yarl-1.8.1-cp37-cp37m-musllinux_1_1_ppc64le.whl", hash = "sha256:12768232751689c1a89b0376a96a32bc7633c08da45ad985d0c49ede691f5c0d"}, + {file = "yarl-1.8.1-cp37-cp37m-musllinux_1_1_s390x.whl", hash = "sha256:b19255dde4b4f4c32e012038f2c169bb72e7f081552bea4641cab4d88bc409dd"}, + {file = "yarl-1.8.1-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:6c8148e0b52bf9535c40c48faebb00cb294ee577ca069d21bd5c48d302a83780"}, + {file = "yarl-1.8.1-cp37-cp37m-win32.whl", hash = "sha256:de839c3a1826a909fdbfe05f6fe2167c4ab033f1133757b5936efe2f84904c07"}, + {file = "yarl-1.8.1-cp37-cp37m-win_amd64.whl", hash = "sha256:dd032e8422a52e5a4860e062eb84ac94ea08861d334a4bcaf142a63ce8ad4802"}, + {file = "yarl-1.8.1-cp38-cp38-macosx_10_9_universal2.whl", hash = "sha256:19cd801d6f983918a3f3a39f3a45b553c015c5aac92ccd1fac619bd74beece4a"}, + {file = "yarl-1.8.1-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:6347f1a58e658b97b0a0d1ff7658a03cb79bdbda0331603bed24dd7054a6dea1"}, + {file = "yarl-1.8.1-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:7c0da7e44d0c9108d8b98469338705e07f4bb7dab96dbd8fa4e91b337db42548"}, + {file = "yarl-1.8.1-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:5587bba41399854703212b87071c6d8638fa6e61656385875f8c6dff92b2e461"}, + {file = "yarl-1.8.1-cp38-cp38-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:31a9a04ecccd6b03e2b0e12e82131f1488dea5555a13a4d32f064e22a6003cfe"}, + {file = "yarl-1.8.1-cp38-cp38-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:205904cffd69ae972a1707a1bd3ea7cded594b1d773a0ce66714edf17833cdae"}, + {file = "yarl-1.8.1-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:ea513a25976d21733bff523e0ca836ef1679630ef4ad22d46987d04b372d57fc"}, + {file = "yarl-1.8.1-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:d0b51530877d3ad7a8d47b2fff0c8df3b8f3b8deddf057379ba50b13df2a5eae"}, + {file = "yarl-1.8.1-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:d2b8f245dad9e331540c350285910b20dd913dc86d4ee410c11d48523c4fd546"}, + {file = "yarl-1.8.1-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:ab2a60d57ca88e1d4ca34a10e9fb4ab2ac5ad315543351de3a612bbb0560bead"}, + {file = "yarl-1.8.1-cp38-cp38-musllinux_1_1_ppc64le.whl", hash = "sha256:449c957ffc6bc2309e1fbe67ab7d2c1efca89d3f4912baeb8ead207bb3cc1cd4"}, + {file = "yarl-1.8.1-cp38-cp38-musllinux_1_1_s390x.whl", hash = "sha256:a165442348c211b5dea67c0206fc61366212d7082ba8118c8c5c1c853ea4d82e"}, + {file = "yarl-1.8.1-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:b3ded839a5c5608eec8b6f9ae9a62cb22cd037ea97c627f38ae0841a48f09eae"}, + {file = "yarl-1.8.1-cp38-cp38-win32.whl", hash = "sha256:c1445a0c562ed561d06d8cbc5c8916c6008a31c60bc3655cdd2de1d3bf5174a0"}, + {file = "yarl-1.8.1-cp38-cp38-win_amd64.whl", hash = "sha256:56c11efb0a89700987d05597b08a1efcd78d74c52febe530126785e1b1a285f4"}, + {file = "yarl-1.8.1-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:e80ed5a9939ceb6fda42811542f31c8602be336b1fb977bccb012e83da7e4936"}, + {file = "yarl-1.8.1-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:6afb336e23a793cd3b6476c30f030a0d4c7539cd81649683b5e0c1b0ab0bf350"}, + {file = "yarl-1.8.1-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:4c322cbaa4ed78a8aac89b2174a6df398faf50e5fc12c4c191c40c59d5e28357"}, + {file = "yarl-1.8.1-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:fae37373155f5ef9b403ab48af5136ae9851151f7aacd9926251ab26b953118b"}, + {file = "yarl-1.8.1-cp39-cp39-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:5395da939ffa959974577eff2cbfc24b004a2fb6c346918f39966a5786874e54"}, + {file = "yarl-1.8.1-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:076eede537ab978b605f41db79a56cad2e7efeea2aa6e0fa8f05a26c24a034fb"}, + {file = "yarl-1.8.1-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:3d1a50e461615747dd93c099f297c1994d472b0f4d2db8a64e55b1edf704ec1c"}, + {file = "yarl-1.8.1-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:7de89c8456525650ffa2bb56a3eee6af891e98f498babd43ae307bd42dca98f6"}, + {file = "yarl-1.8.1-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:4a88510731cd8d4befaba5fbd734a7dd914de5ab8132a5b3dde0bbd6c9476c64"}, + {file = "yarl-1.8.1-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:2d93a049d29df172f48bcb09acf9226318e712ce67374f893b460b42cc1380ae"}, + {file = "yarl-1.8.1-cp39-cp39-musllinux_1_1_ppc64le.whl", hash = "sha256:21ac44b763e0eec15746a3d440f5e09ad2ecc8b5f6dcd3ea8cb4773d6d4703e3"}, + {file = "yarl-1.8.1-cp39-cp39-musllinux_1_1_s390x.whl", hash = "sha256:d0272228fabe78ce00a3365ffffd6f643f57a91043e119c289aaba202f4095b0"}, + {file = "yarl-1.8.1-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:99449cd5366fe4608e7226c6cae80873296dfa0cde45d9b498fefa1de315a09e"}, + {file = "yarl-1.8.1-cp39-cp39-win32.whl", hash = "sha256:8b0af1cf36b93cee99a31a545fe91d08223e64390c5ecc5e94c39511832a4bb6"}, + {file = "yarl-1.8.1-cp39-cp39-win_amd64.whl", hash = "sha256:de49d77e968de6626ba7ef4472323f9d2e5a56c1d85b7c0e2a190b2173d3b9be"}, + {file = "yarl-1.8.1.tar.gz", hash = "sha256:af887845b8c2e060eb5605ff72b6f2dd2aab7a761379373fd89d314f4752abbf"}, +] zipp = [ {file = "zipp-3.11.0-py3-none-any.whl", hash = "sha256:83a28fcb75844b5c0cdaf5aa4003c2d728c77e05f5aeabe8e95e56727005fbaa"}, {file = "zipp-3.11.0.tar.gz", hash = "sha256:a7a22e05929290a67401440b39690ae6563279bced5f314609d9d03798f56766"}, diff --git a/pyproject.toml b/pyproject.toml index 86d3bd023a..f0e68a29ca 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -75,7 +75,7 @@ pre-commit = "2.20.0" fastavro = "1.7.0" coverage = { version = "^6.5.0", extras = ["toml"] } requests-mock = "1.10.0" -moto = "^4.0.6" +moto = "^4.0.11" typing-extensions = '4.4.0' [tool.poetry.scripts] From cd22ceb0d7a648b7ff87792b702a8c5718f39ec9 Mon Sep 17 00:00:00 2001 From: Fokko Driesprong Date: Tue, 6 Dec 2022 21:32:34 +0100 Subject: [PATCH 291/642] Python: Fix PyArrow import (#6362) --- pyiceberg/table/__init__.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/pyiceberg/table/__init__.py b/pyiceberg/table/__init__.py index 3412e8ee2f..0bb76a4f4b 100644 --- a/pyiceberg/table/__init__.py +++ b/pyiceberg/table/__init__.py @@ -42,7 +42,6 @@ ) from pyiceberg.expressions.visitors import bind, inclusive_projection from pyiceberg.io import FileIO -from pyiceberg.io.pyarrow import expression_to_pyarrow, schema_to_pyarrow from pyiceberg.manifest import DataFile, ManifestFile, files from pyiceberg.partitioning import PartitionSpec from pyiceberg.schema import Schema @@ -335,7 +334,7 @@ def plan_files(self) -> Iterator[ScanTask]: yield from (FileScanTask(file) for file in matching_partition_files) def to_arrow(self): - from pyiceberg.io.pyarrow import PyArrowFileIO + from pyiceberg.io.pyarrow import PyArrowFileIO, expression_to_pyarrow, schema_to_pyarrow warnings.warn( "Projection is currently done by name instead of Field ID, this can lead to incorrect results in some cases." From 27409997b15856b3116b81d8927be20972306d18 Mon Sep 17 00:00:00 2001 From: Hongyue/Steve Zhang Date: Wed, 7 Dec 2022 07:57:24 -0800 Subject: [PATCH 292/642] Python: Fix incorrect description when set a property (#6372) * [Python] Fix incorrect description when set a property * Fix wording Co-authored-by: Steve Zhang --- pyiceberg/cli/console.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/pyiceberg/cli/console.py b/pyiceberg/cli/console.py index f1e7802953..38b9e243ad 100644 --- a/pyiceberg/cli/console.py +++ b/pyiceberg/cli/console.py @@ -103,7 +103,7 @@ def list(ctx: Context, parent: Optional[str]): # pylint: disable=redefined-buil @click.pass_context @catch_exception() def describe(ctx: Context, entity: Literal["name", "namespace", "table"], identifier: str): - """Describes a namespace xor table""" + """Describes a namespace or a table""" catalog, output = _catalog_and_output(ctx) identifier_tuple = Catalog.identifier_to_tuple(identifier) @@ -198,7 +198,7 @@ def drop(): @click.pass_context @catch_exception() def table(ctx: Context, identifier: str): # noqa: F811 - """Drop table""" + """Drops a table""" catalog, output = _catalog_and_output(ctx) catalog.drop_table(identifier) @@ -210,7 +210,7 @@ def table(ctx: Context, identifier: str): # noqa: F811 @click.pass_context @catch_exception() def namespace(ctx, identifier: str): - """Drop namespace""" + """Drops a namespace""" catalog, output = _catalog_and_output(ctx) catalog.drop_namespace(identifier) @@ -286,7 +286,7 @@ def get_table(ctx: Context, identifier: str, property_name: str): @properties.group() def set(): - """Removes properties on tables/namespaces""" + """Sets a property on tables/namespaces""" @set.command() # type: ignore @@ -296,7 +296,7 @@ def set(): @click.pass_context @catch_exception() def namespace(ctx: Context, identifier: str, property_name: str, property_value: str): # noqa: F811 - """Sets a property of a namespace or table""" + """Sets a property on a namespace""" catalog, output = _catalog_and_output(ctx) catalog.update_namespace_properties(identifier, updates={property_name: property_value}) @@ -321,7 +321,7 @@ def table(ctx: Context, identifier: str, property_name: str, property_value: str @properties.group() def remove(): - """Removes properties on tables/namespaces""" + """Removes a property from tables/namespaces""" @remove.command() # type: ignore From fc554c1027ff66dc397a37b23e2f20fed9c0d595 Mon Sep 17 00:00:00 2001 From: Fokko Driesprong Date: Wed, 7 Dec 2022 23:00:22 +0100 Subject: [PATCH 293/642] Python: Make types required (#6308) --- pyiceberg/avro/decoder.py | 2 +- pyiceberg/avro/file.py | 21 +- pyiceberg/avro/reader.py | 4 +- pyiceberg/catalog/hive.py | 7 +- pyiceberg/catalog/rest.py | 9 +- pyiceberg/cli/console.py | 58 +++--- pyiceberg/conversions.py | 63 +++--- pyiceberg/expressions/__init__.py | 58 +++--- pyiceberg/expressions/literals.py | 60 +++--- pyiceberg/expressions/visitors.py | 8 +- pyiceberg/io/__init__.py | 17 +- pyiceberg/io/fsspec.py | 21 +- pyiceberg/io/memory.py | 11 +- pyiceberg/io/pyarrow.py | 6 +- pyiceberg/partitioning.py | 4 +- pyiceberg/schema.py | 42 ++-- pyiceberg/table/__init__.py | 21 +- pyiceberg/table/metadata.py | 12 +- pyiceberg/table/snapshots.py | 5 +- pyiceberg/table/sorting.py | 4 +- pyiceberg/transforms.py | 46 +++-- pyiceberg/types.py | 20 +- pyiceberg/utils/config.py | 4 +- pyiceberg/utils/datetime.py | 4 +- pyiceberg/utils/deprecated.py | 6 +- pyiceberg/utils/iceberg_base_model.py | 11 +- pyiceberg/utils/singleton.py | 2 +- pyproject.toml | 1 + tests/avro/test_decoder.py | 58 +++--- tests/avro/test_file.py | 8 +- tests/avro/test_reader.py | 40 ++-- tests/avro/test_resolver.py | 36 ++-- tests/catalog/integration_test_glue.py | 56 ++--- tests/catalog/test_base.py | 50 ++--- tests/catalog/test_glue.py | 84 ++++---- tests/catalog/test_hive.py | 50 ++--- tests/catalog/test_rest.py | 74 +++---- tests/cli/test_console.py | 146 ++++++------- tests/conftest.py | 64 +++--- tests/expressions/test_evaluator.py | 36 ++-- tests/expressions/test_expressions.py | 272 +++++++++++++------------ tests/expressions/test_literals.py | 182 +++++++++-------- tests/expressions/test_parser.py | 44 ++-- tests/expressions/test_projection.py | 28 +-- tests/expressions/test_visitors.py | 132 ++++++------ tests/io/test_fsspec.py | 49 ++--- tests/io/test_io.py | 87 ++++---- tests/io/test_pyarrow.py | 75 +++---- tests/table/test_init.py | 90 ++++---- tests/table/test_metadata.py | 78 +++---- tests/table/test_partitioning.py | 28 ++- tests/table/test_refs.py | 2 +- tests/table/test_snapshots.py | 24 +-- tests/table/test_sorting.py | 16 +- tests/test_conversions.py | 40 ++-- tests/test_schema.py | 82 ++++---- tests/test_transforms.py | 144 ++++++------- tests/test_types.py | 176 ++++++++-------- tests/test_version.py | 2 +- tests/utils/test_bin_packing.py | 4 +- tests/utils/test_config.py | 12 +- tests/utils/test_manifest.py | 6 +- tests/utils/test_schema_conversion.py | 40 ++-- tests/utils/test_singleton.py | 4 +- 64 files changed, 1475 insertions(+), 1371 deletions(-) diff --git a/pyiceberg/avro/decoder.py b/pyiceberg/avro/decoder.py index fd39a608ef..710b888d40 100644 --- a/pyiceberg/avro/decoder.py +++ b/pyiceberg/avro/decoder.py @@ -158,7 +158,7 @@ def read_timestamp_micros(self) -> datetime: """ return micros_to_timestamp(self.read_int()) - def read_timestamptz_micros(self): + def read_timestamptz_micros(self) -> datetime: """ long is decoded as python datetime object which represents the number of microseconds from the unix epoch, 1 January 1970. diff --git a/pyiceberg/avro/file.py b/pyiceberg/avro/file.py index 5c1d3c63f0..68887a61ec 100644 --- a/pyiceberg/avro/file.py +++ b/pyiceberg/avro/file.py @@ -23,11 +23,12 @@ import json from dataclasses import dataclass from io import SEEK_SET, BufferedReader -from typing import Optional, Type, cast +from types import TracebackType +from typing import Optional, Type from pyiceberg.avro.codecs import KNOWN_CODECS, Codec from pyiceberg.avro.decoder import BinaryDecoder -from pyiceberg.avro.reader import AvroStruct, ConstructReader, StructReader +from pyiceberg.avro.reader import AvroStruct, ConstructReader, Reader from pyiceberg.avro.resolver import resolve from pyiceberg.io import InputFile, InputStream from pyiceberg.io.memory import MemoryInputStream @@ -89,12 +90,12 @@ def get_schema(self) -> Schema: @dataclass class Block: - reader: StructReader + reader: Reader block_records: int block_decoder: BinaryDecoder position: int = 0 - def __iter__(self): + def __iter__(self) -> Block: return self def has_next(self) -> bool: @@ -113,7 +114,7 @@ class AvroFile: input_stream: InputStream header: AvroFileHeader schema: Schema - reader: StructReader + reader: Reader decoder: BinaryDecoder block: Optional[Block] = None @@ -122,7 +123,7 @@ def __init__(self, input_file: InputFile, read_schema: Optional[Schema] = None) self.input_file = input_file self.read_schema = read_schema - def __enter__(self): + def __enter__(self) -> AvroFile: """ Opens the file and reads the header and generates a reader tree to start reading the payload @@ -130,18 +131,20 @@ def __enter__(self): Returns: A generator returning the AvroStructs """ - self.input_stream = BufferedReader(self.input_file.open()) + self.input_stream = BufferedReader(self.input_file.open()) # type: ignore self.decoder = BinaryDecoder(self.input_stream) self.header = self._read_header() self.schema = self.header.get_schema() if not self.read_schema: self.reader = visit(self.schema, ConstructReader()) else: - self.reader = cast(StructReader, resolve(self.schema, self.read_schema)) + self.reader = resolve(self.schema, self.read_schema) return self - def __exit__(self, exc_type, exc_val, exc_tb): + def __exit__( + self, exctype: Optional[Type[BaseException]], excinst: Optional[BaseException], exctb: Optional[TracebackType] + ) -> None: self.input_stream.close() def __iter__(self) -> AvroFile: diff --git a/pyiceberg/avro/reader.py b/pyiceberg/avro/reader.py index b193ec91b8..a7631d4888 100644 --- a/pyiceberg/avro/reader.py +++ b/pyiceberg/avro/reader.py @@ -225,7 +225,7 @@ def read(self, decoder: BinaryDecoder) -> bytes: def skip(self, decoder: BinaryDecoder) -> None: decoder.skip(len(self)) - def __len__(self): + def __len__(self) -> int: return self._len @@ -332,7 +332,7 @@ def read(self, decoder: BinaryDecoder) -> Dict[Any, Any]: return read_items def skip(self, decoder: BinaryDecoder) -> None: - def skip(): + def skip() -> None: self.key.skip(decoder) self.value.skip(decoder) diff --git a/pyiceberg/catalog/hive.py b/pyiceberg/catalog/hive.py index 12601f103f..2387e3dc37 100644 --- a/pyiceberg/catalog/hive.py +++ b/pyiceberg/catalog/hive.py @@ -17,6 +17,7 @@ import getpass import time import uuid +from types import TracebackType from typing import ( Any, Dict, @@ -130,7 +131,9 @@ def __enter__(self) -> Client: self._transport.open() return self._client - def __exit__(self, exc_type, exc_val, exc_tb): + def __exit__( + self, exctype: Optional[Type[BaseException]], excinst: Optional[BaseException], exctb: Optional[TracebackType] + ) -> None: self._transport.close() @@ -265,7 +268,7 @@ def _convert_hive_into_iceberg(self, table: HiveTable, io: FileIO) -> Table: io=self._load_file_io(metadata.properties), ) - def _write_metadata(self, metadata: TableMetadata, io: FileIO, metadata_path: str): + def _write_metadata(self, metadata: TableMetadata, io: FileIO, metadata_path: str) -> None: ToOutputFile.table_metadata(metadata, io.new_output(metadata_path)) def _resolve_table_location(self, location: Optional[str], database_name: str, table_name: str) -> str: diff --git a/pyiceberg/catalog/rest.py b/pyiceberg/catalog/rest.py index 71a92b532f..f7bb6b346c 100644 --- a/pyiceberg/catalog/rest.py +++ b/pyiceberg/catalog/rest.py @@ -16,6 +16,7 @@ # under the License. from json import JSONDecodeError from typing import ( + Any, Dict, List, Literal, @@ -227,7 +228,7 @@ def _check_valid_namespace_identifier(self, identifier: Union[str, Identifier]) raise NoSuchNamespaceError(f"Empty namespace identifier: {identifier}") return identifier_tuple - def url(self, endpoint: str, prefixed: bool = True, **kwargs) -> str: + def url(self, endpoint: str, prefixed: bool = True, **kwargs: Any) -> str: """Constructs the endpoint Args: @@ -291,7 +292,7 @@ def _split_identifier_for_json(self, identifier: Union[str, Identifier]) -> Dict raise NoSuchTableError(f"Missing namespace or invalid identifier: {identifier_tuple}") return {"namespace": identifier_tuple[:-1], "name": identifier_tuple[-1]} - def _handle_non_200_response(self, exc: HTTPError, error_handler: Dict[int, Type[Exception]]): + def _handle_non_200_response(self, exc: HTTPError, error_handler: Dict[int, Type[Exception]]) -> None: exception: Type[Exception] code = exc.response.status_code if code in error_handler: @@ -418,7 +419,7 @@ def drop_table(self, identifier: Union[str, Identifier], purge_requested: bool = def purge_table(self, identifier: Union[str, Identifier]) -> None: self.drop_table(identifier=identifier, purge_requested=True) - def rename_table(self, from_identifier: Union[str, Identifier], to_identifier: Union[str, Identifier]): + def rename_table(self, from_identifier: Union[str, Identifier], to_identifier: Union[str, Identifier]) -> Table: payload = { "source": self._split_identifier_for_json(from_identifier), "destination": self._split_identifier_for_json(to_identifier), @@ -429,6 +430,8 @@ def rename_table(self, from_identifier: Union[str, Identifier], to_identifier: U except HTTPError as exc: self._handle_non_200_response(exc, {404: NoSuchTableError, 409: TableAlreadyExistsError}) + return self.load_table(to_identifier) + def create_namespace(self, namespace: Union[str, Identifier], properties: Properties = EMPTY_DICT) -> None: namespace_tuple = self._check_valid_namespace_identifier(namespace) payload = {"namespace": namespace_tuple, "properties": properties} diff --git a/pyiceberg/cli/console.py b/pyiceberg/cli/console.py index 38b9e243ad..e2e9dbe1fc 100644 --- a/pyiceberg/cli/console.py +++ b/pyiceberg/cli/console.py @@ -16,7 +16,13 @@ # under the License. # pylint: disable=broad-except,redefined-builtin,redefined-outer-name from functools import wraps -from typing import Literal, Optional, Tuple +from typing import ( + Any, + Callable, + Literal, + Optional, + Tuple, +) import click from click import Context @@ -26,10 +32,10 @@ from pyiceberg.exceptions import NoSuchNamespaceError, NoSuchPropertyException, NoSuchTableError -def catch_exception(): - def decorator(func): +def catch_exception() -> Callable: # type: ignore + def decorator(func: Callable) -> Callable: # type: ignore @wraps(func) - def wrapper(*args, **kwargs): + def wrapper(*args: Any, **kwargs: Any): # type: ignore try: return func(*args, **kwargs) except Exception as e: @@ -50,7 +56,7 @@ def wrapper(*args, **kwargs): @click.option("--uri") @click.option("--credential") @click.pass_context -def run(ctx: Context, catalog: str, verbose: bool, output: str, uri: Optional[str], credential: Optional[str]): +def run(ctx: Context, catalog: str, verbose: bool, output: str, uri: Optional[str], credential: Optional[str]) -> None: properties = {} if uri: properties["uri"] = uri @@ -87,7 +93,7 @@ def _catalog_and_output(ctx: Context) -> Tuple[Catalog, Output]: @click.pass_context @click.argument("parent", required=False) @catch_exception() -def list(ctx: Context, parent: Optional[str]): # pylint: disable=redefined-builtin +def list(ctx: Context, parent: Optional[str]) -> None: # pylint: disable=redefined-builtin """Lists tables or namespaces""" catalog, output = _catalog_and_output(ctx) @@ -102,7 +108,7 @@ def list(ctx: Context, parent: Optional[str]): # pylint: disable=redefined-buil @click.argument("identifier") @click.pass_context @catch_exception() -def describe(ctx: Context, entity: Literal["name", "namespace", "table"], identifier: str): +def describe(ctx: Context, entity: Literal["name", "namespace", "table"], identifier: str) -> None: """Describes a namespace or a table""" catalog, output = _catalog_and_output(ctx) identifier_tuple = Catalog.identifier_to_tuple(identifier) @@ -136,7 +142,7 @@ def describe(ctx: Context, entity: Literal["name", "namespace", "table"], identi @click.option("--history", is_flag=True) @click.pass_context @catch_exception() -def files(ctx: Context, identifier: str, history: bool): +def files(ctx: Context, identifier: str, history: bool) -> None: """Lists all the files of the table""" catalog, output = _catalog_and_output(ctx) @@ -148,7 +154,7 @@ def files(ctx: Context, identifier: str, history: bool): @click.argument("identifier") @click.pass_context @catch_exception() -def schema(ctx: Context, identifier: str): +def schema(ctx: Context, identifier: str) -> None: """Gets the schema of the table""" catalog, output = _catalog_and_output(ctx) table = catalog.load_table(identifier) @@ -159,7 +165,7 @@ def schema(ctx: Context, identifier: str): @click.argument("identifier") @click.pass_context @catch_exception() -def spec(ctx: Context, identifier: str): +def spec(ctx: Context, identifier: str) -> None: """Returns the partition spec of the table""" catalog, output = _catalog_and_output(ctx) table = catalog.load_table(identifier) @@ -170,7 +176,7 @@ def spec(ctx: Context, identifier: str): @click.argument("identifier") @click.pass_context @catch_exception() -def uuid(ctx: Context, identifier: str): +def uuid(ctx: Context, identifier: str) -> None: """Returns the UUID of the table""" catalog, output = _catalog_and_output(ctx) metadata = catalog.load_table(identifier).metadata @@ -181,7 +187,7 @@ def uuid(ctx: Context, identifier: str): @click.argument("identifier") @click.pass_context @catch_exception() -def location(ctx: Context, identifier: str): +def location(ctx: Context, identifier: str) -> None: """Returns the location of the table""" catalog, output = _catalog_and_output(ctx) table = catalog.load_table(identifier) @@ -189,7 +195,7 @@ def location(ctx: Context, identifier: str): @run.group() -def drop(): +def drop() -> None: """Operations to drop a namespace or table""" @@ -197,7 +203,7 @@ def drop(): @click.argument("identifier") @click.pass_context @catch_exception() -def table(ctx: Context, identifier: str): # noqa: F811 +def table(ctx: Context, identifier: str) -> None: # noqa: F811 """Drops a table""" catalog, output = _catalog_and_output(ctx) @@ -209,7 +215,7 @@ def table(ctx: Context, identifier: str): # noqa: F811 @click.argument("identifier") @click.pass_context @catch_exception() -def namespace(ctx, identifier: str): +def namespace(ctx: Context, identifier: str) -> None: """Drops a namespace""" catalog, output = _catalog_and_output(ctx) @@ -222,7 +228,7 @@ def namespace(ctx, identifier: str): @click.argument("to_identifier") @click.pass_context @catch_exception() -def rename(ctx, from_identifier: str, to_identifier: str): +def rename(ctx: Context, from_identifier: str, to_identifier: str) -> None: """Renames a table""" catalog, output = _catalog_and_output(ctx) @@ -231,12 +237,12 @@ def rename(ctx, from_identifier: str, to_identifier: str): @run.group() -def properties(): +def properties() -> None: """Properties on tables/namespaces""" @properties.group() -def get(): +def get() -> None: """Fetch properties on tables/namespaces""" @@ -245,7 +251,7 @@ def get(): @click.argument("property_name", required=False) @click.pass_context @catch_exception() -def get_namespace(ctx: Context, identifier: str, property_name: str): +def get_namespace(ctx: Context, identifier: str, property_name: str) -> None: """Fetch properties on a namespace""" catalog, output = _catalog_and_output(ctx) identifier_tuple = Catalog.identifier_to_tuple(identifier) @@ -267,7 +273,7 @@ def get_namespace(ctx: Context, identifier: str, property_name: str): @click.argument("property_name", required=False) @click.pass_context @catch_exception() -def get_table(ctx: Context, identifier: str, property_name: str): +def get_table(ctx: Context, identifier: str, property_name: str) -> None: """Fetch properties on a table""" catalog, output = _catalog_and_output(ctx) identifier_tuple = Catalog.identifier_to_tuple(identifier) @@ -285,7 +291,7 @@ def get_table(ctx: Context, identifier: str, property_name: str): @properties.group() -def set(): +def set() -> None: """Sets a property on tables/namespaces""" @@ -295,7 +301,7 @@ def set(): @click.argument("property_value") @click.pass_context @catch_exception() -def namespace(ctx: Context, identifier: str, property_name: str, property_value: str): # noqa: F811 +def namespace(ctx: Context, identifier: str, property_name: str, property_value: str) -> None: # noqa: F811 """Sets a property on a namespace""" catalog, output = _catalog_and_output(ctx) @@ -309,7 +315,7 @@ def namespace(ctx: Context, identifier: str, property_name: str, property_value: @click.argument("property_value") @click.pass_context @catch_exception() -def table(ctx: Context, identifier: str, property_name: str, property_value: str): # noqa: F811 +def table(ctx: Context, identifier: str, property_name: str, property_value: str) -> None: # noqa: F811 """Sets a property on a table""" catalog, output = _catalog_and_output(ctx) identifier_tuple = Catalog.identifier_to_tuple(identifier) @@ -320,7 +326,7 @@ def table(ctx: Context, identifier: str, property_name: str, property_value: str @properties.group() -def remove(): +def remove() -> None: """Removes a property from tables/namespaces""" @@ -329,7 +335,7 @@ def remove(): @click.argument("property_name") @click.pass_context @catch_exception() -def namespace(ctx: Context, identifier: str, property_name: str): # noqa: F811 +def namespace(ctx: Context, identifier: str, property_name: str) -> None: # noqa: F811 """Removes a property from a namespace""" catalog, output = _catalog_and_output(ctx) @@ -346,7 +352,7 @@ def namespace(ctx: Context, identifier: str, property_name: str): # noqa: F811 @click.argument("property_name") @click.pass_context @catch_exception() -def table(ctx: Context, identifier: str, property_name: str): # noqa: F811 +def table(ctx: Context, identifier: str, property_name: str) -> None: # noqa: F811 """Removes a property from a table""" catalog, output = _catalog_and_output(ctx) table = catalog.load_table(identifier) diff --git a/pyiceberg/conversions.py b/pyiceberg/conversions.py index e8c2a4e8cd..4707dab85c 100644 --- a/pyiceberg/conversions.py +++ b/pyiceberg/conversions.py @@ -31,7 +31,12 @@ from decimal import Decimal from functools import singledispatch from struct import Struct -from typing import Union +from typing import ( + Any, + Callable, + Optional, + Union, +) from pyiceberg.types import ( BinaryType, @@ -60,14 +65,14 @@ _UUID_STRUCT = Struct(">QQ") -def handle_none(func): +def handle_none(func: Callable) -> Callable: # type: ignore """A decorator function to handle cases where partition values are `None` or "__HIVE_DEFAULT_PARTITION__" Args: func (Callable): A function registered to the singledispatch function `partition_to_py` """ - def wrapper(primitive_type, value_str): + def wrapper(primitive_type: PrimitiveType, value_str: Optional[str]) -> Any: if value_str is None: return None elif value_str == "__HIVE_DEFAULT_PARTITION__": @@ -78,7 +83,7 @@ def wrapper(primitive_type, value_str): @singledispatch -def partition_to_py(primitive_type, value_str: str): +def partition_to_py(primitive_type: PrimitiveType, value_str: str) -> Union[int, float, str, uuid.UUID, bytes, Decimal]: """A generic function which converts a partition string to a python built-in Args: @@ -90,7 +95,7 @@ def partition_to_py(primitive_type, value_str: str): @partition_to_py.register(BooleanType) @handle_none -def _(primitive_type, value_str: str) -> Union[int, float, str, uuid.UUID]: +def _(primitive_type: BooleanType, value_str: str) -> Union[int, float, str, uuid.UUID]: return value_str.lower() == "true" @@ -101,7 +106,7 @@ def _(primitive_type, value_str: str) -> Union[int, float, str, uuid.UUID]: @partition_to_py.register(TimestampType) @partition_to_py.register(TimestamptzType) @handle_none -def _(primitive_type, value_str: str) -> int: +def _(primitive_type: PrimitiveType, value_str: str) -> int: """ Raises: ValueError: If the scale/exponent is not 0 @@ -115,37 +120,37 @@ def _(primitive_type, value_str: str) -> int: @partition_to_py.register(FloatType) @partition_to_py.register(DoubleType) @handle_none -def _(primitive_type, value_str: str) -> float: +def _(_: PrimitiveType, value_str: str) -> float: return float(value_str) @partition_to_py.register(StringType) @handle_none -def _(primitive_type, value_str: str) -> str: +def _(_: StringType, value_str: str) -> str: return value_str @partition_to_py.register(UUIDType) @handle_none -def _(primitive_type, value_str: str) -> uuid.UUID: +def _(_: UUIDType, value_str: str) -> uuid.UUID: return uuid.UUID(value_str) @partition_to_py.register(FixedType) @partition_to_py.register(BinaryType) @handle_none -def _(primitive_type, value_str: str) -> bytes: +def _(_: PrimitiveType, value_str: str) -> bytes: return bytes(value_str, "UTF-8") @partition_to_py.register(DecimalType) @handle_none -def _(primitive_type, value_str: str) -> Decimal: +def _(_: DecimalType, value_str: str) -> Decimal: return Decimal(value_str) @singledispatch -def to_bytes(primitive_type: PrimitiveType, value: Union[bool, bytes, Decimal, float, int, str, uuid.UUID]) -> bytes: +def to_bytes(primitive_type: PrimitiveType, _: Union[bool, bytes, Decimal, float, int, str, uuid.UUID]) -> bytes: """A generic function which converts a built-in python value to bytes This conversion follows the serialization scheme for storing single values as individual binary values defined in the Iceberg specification that @@ -153,20 +158,20 @@ def to_bytes(primitive_type: PrimitiveType, value: Union[bool, bytes, Decimal, f Args: primitive_type(PrimitiveType): An implementation of the PrimitiveType base class - value: The value to convert to bytes (The type of this value depends on which dispatched function is + _: The value to convert to bytes (The type of this value depends on which dispatched function is used--check dispatchable functions for type hints) """ raise TypeError(f"scale does not match {primitive_type}") @to_bytes.register(BooleanType) -def _(primitive_type, value: bool) -> bytes: +def _(_: BooleanType, value: bool) -> bytes: return _BOOL_STRUCT.pack(1 if value else 0) @to_bytes.register(IntegerType) @to_bytes.register(DateType) -def _(primitive_type, value: int) -> bytes: +def _(_: PrimitiveType, value: int) -> bytes: return _INT_STRUCT.pack(value) @@ -174,12 +179,12 @@ def _(primitive_type, value: int) -> bytes: @to_bytes.register(TimeType) @to_bytes.register(TimestampType) @to_bytes.register(TimestamptzType) -def _(primitive_type, value: int) -> bytes: +def _(_: PrimitiveType, value: int) -> bytes: return _LONG_STRUCT.pack(value) @to_bytes.register(FloatType) -def _(primitive_type, value: float) -> bytes: +def _(_: FloatType, value: float) -> bytes: """ Note: float in python is implemented using a double in C. Therefore this involves a conversion of a 32-bit (single precision) float to a 64-bit (double precision) float which introduces some imprecision. @@ -188,23 +193,23 @@ def _(primitive_type, value: float) -> bytes: @to_bytes.register(DoubleType) -def _(primitive_type, value: float) -> bytes: +def _(_: DoubleType, value: float) -> bytes: return _DOUBLE_STRUCT.pack(value) @to_bytes.register(StringType) -def _(primitive_type, value: str) -> bytes: +def _(_: StringType, value: str) -> bytes: return value.encode("UTF-8") @to_bytes.register(UUIDType) -def _(primitive_type, value: uuid.UUID) -> bytes: +def _(_: UUIDType, value: uuid.UUID) -> bytes: return _UUID_STRUCT.pack((value.int >> 64) & 0xFFFFFFFFFFFFFFFF, value.int & 0xFFFFFFFFFFFFFFFF) @to_bytes.register(BinaryType) @to_bytes.register(FixedType) -def _(primitive_type, value: bytes) -> bytes: +def _(_: PrimitiveType, value: bytes) -> bytes: return value @@ -247,13 +252,13 @@ def from_bytes(primitive_type: PrimitiveType, b: bytes) -> Union[bool, bytes, De @from_bytes.register(BooleanType) -def _(primitive_type, b: bytes) -> bool: +def _(_: BooleanType, b: bytes) -> bool: return _BOOL_STRUCT.unpack(b)[0] != 0 @from_bytes.register(IntegerType) @from_bytes.register(DateType) -def _(primitive_type, b: bytes) -> int: +def _(_: PrimitiveType, b: bytes) -> int: return _INT_STRUCT.unpack(b)[0] @@ -261,34 +266,34 @@ def _(primitive_type, b: bytes) -> int: @from_bytes.register(TimeType) @from_bytes.register(TimestampType) @from_bytes.register(TimestamptzType) -def _(primitive_type, b: bytes) -> int: +def _(_: PrimitiveType, b: bytes) -> int: return _LONG_STRUCT.unpack(b)[0] @from_bytes.register(FloatType) -def _(primitive_type, b: bytes) -> float: +def _(_: FloatType, b: bytes) -> float: return _FLOAT_STRUCT.unpack(b)[0] @from_bytes.register(DoubleType) -def _(primitive_type, b: bytes) -> float: +def _(_: DoubleType, b: bytes) -> float: return _DOUBLE_STRUCT.unpack(b)[0] @from_bytes.register(StringType) -def _(primitive_type: PrimitiveType, b: bytes) -> str: +def _(_: StringType, b: bytes) -> str: return bytes(b).decode("utf-8") @from_bytes.register(UUIDType) -def _(primitive_type, b: bytes) -> uuid.UUID: +def _(_: UUIDType, b: bytes) -> uuid.UUID: unpacked_bytes = _UUID_STRUCT.unpack(b) return uuid.UUID(int=unpacked_bytes[0] << 64 | unpacked_bytes[1]) @from_bytes.register(BinaryType) @from_bytes.register(FixedType) -def _(primitive_type, b: bytes) -> bytes: +def _(_: PrimitiveType, b: bytes) -> bytes: return b diff --git a/pyiceberg/expressions/__init__.py b/pyiceberg/expressions/__init__.py index 528a294545..b7ef368112 100644 --- a/pyiceberg/expressions/__init__.py +++ b/pyiceberg/expressions/__init__.py @@ -125,7 +125,7 @@ def eval(self, struct: StructProtocol) -> L: """ return self.accessor.get(struct) - def __eq__(self, other): + def __eq__(self, other: Any) -> bool: return self.field == other.field if isinstance(other, BoundReference) else False def __repr__(self) -> str: @@ -155,13 +155,13 @@ class Reference(UnboundTerm[Any]): name: str - def __init__(self, name: str): + def __init__(self, name: str) -> None: self.name = name - def __repr__(self): + def __repr__(self) -> str: return f"Reference(name={repr(self.name)})" - def __eq__(self, other): + def __eq__(self, other: Any) -> bool: return self.name == other.name if isinstance(other, Reference) else False def bind(self, schema: Schema, case_sensitive: bool = True) -> BoundReference[L]: @@ -192,7 +192,7 @@ class And(BooleanExpression): left: BooleanExpression right: BooleanExpression - def __new__(cls, left: BooleanExpression, right: BooleanExpression, *rest: BooleanExpression): + def __new__(cls, left: BooleanExpression, right: BooleanExpression, *rest: BooleanExpression) -> BooleanExpression: # type: ignore if rest: return reduce(And, (left, right, *rest)) if left is AlwaysFalse() or right is AlwaysFalse(): @@ -226,7 +226,7 @@ class Or(BooleanExpression): left: BooleanExpression right: BooleanExpression - def __new__(cls, left: BooleanExpression, right: BooleanExpression, *rest: BooleanExpression): + def __new__(cls, left: BooleanExpression, right: BooleanExpression, *rest: BooleanExpression) -> BooleanExpression: # type: ignore if rest: return reduce(Or, (left, right, *rest)) if left is AlwaysTrue() or right is AlwaysTrue(): @@ -256,7 +256,7 @@ class Not(BooleanExpression): child: BooleanExpression - def __new__(cls, child: BooleanExpression): + def __new__(cls, child: BooleanExpression) -> BooleanExpression: # type: ignore if child is AlwaysTrue(): return AlwaysFalse() elif child is AlwaysFalse(): @@ -283,10 +283,10 @@ class AlwaysTrue(BooleanExpression, Singleton): def __invert__(self) -> AlwaysFalse: return AlwaysFalse() - def __str__(self): + def __str__(self) -> str: return "AlwaysTrue()" - def __repr__(self): + def __repr__(self) -> str: return "AlwaysTrue()" @@ -296,10 +296,10 @@ class AlwaysFalse(BooleanExpression, Singleton): def __invert__(self) -> AlwaysTrue: return AlwaysTrue() - def __str__(self): + def __str__(self) -> str: return "AlwaysFalse()" - def __repr__(self): + def __repr__(self) -> str: return "AlwaysFalse()" @@ -326,7 +326,7 @@ class UnboundPredicate(Generic[L], Unbound[BooleanExpression], BooleanExpression def __init__(self, term: Union[str, UnboundTerm[Any]]): self.term = _to_unbound_term(term) - def __eq__(self, other): + def __eq__(self, other: Any) -> bool: return self.term == other.term if isinstance(other, UnboundPredicate) else False @abstractmethod @@ -364,7 +364,7 @@ def as_unbound(self) -> Type[UnaryPredicate]: class BoundIsNull(BoundUnaryPredicate[L]): - def __new__(cls, term: BoundTerm[L]): # pylint: disable=W0221 + def __new__(cls, term: BoundTerm[L]) -> BooleanExpression: # type: ignore # pylint: disable=W0221 if term.ref().field.required: return AlwaysFalse() return super().__new__(cls) @@ -378,7 +378,7 @@ def as_unbound(self) -> Type[IsNull]: class BoundNotNull(BoundUnaryPredicate[L]): - def __new__(cls, term: BoundTerm[L]): # pylint: disable=W0221 + def __new__(cls, term: BoundTerm[L]): # type: ignore # pylint: disable=W0221 if term.ref().field.required: return AlwaysTrue() return super().__new__(cls) @@ -410,7 +410,7 @@ def as_bound(self) -> Type[BoundNotNull[L]]: class BoundIsNaN(BoundUnaryPredicate[L]): - def __new__(cls, term: BoundTerm[L]): # pylint: disable=W0221 + def __new__(cls, term: BoundTerm[L]) -> BooleanExpression: # type: ignore # pylint: disable=W0221 bound_type = term.ref().field.field_type if type(bound_type) in {FloatType, DoubleType}: return super().__new__(cls) @@ -425,7 +425,7 @@ def as_unbound(self) -> Type[IsNaN]: class BoundNotNaN(BoundUnaryPredicate[L]): - def __new__(cls, term: BoundTerm[L]): # pylint: disable=W0221 + def __new__(cls, term: BoundTerm[L]) -> BooleanExpression: # type: ignore # pylint: disable=W0221 bound_type = term.ref().field.field_type if type(bound_type) in {FloatType, DoubleType}: return super().__new__(cls) @@ -468,7 +468,7 @@ def bind(self, schema: Schema, case_sensitive: bool = True) -> BoundSetPredicate bound_term = self.term.bind(schema, case_sensitive) return self.as_bound(bound_term, {lit.to(bound_term.ref().field.field_type) for lit in self.literals}) - def __str__(self): + def __str__(self) -> str: # Sort to make it deterministic return f"{str(self.__class__.__name__)}({str(self.term)}, {{{', '.join(sorted([str(literal) for literal in self.literals]))}}})" @@ -497,7 +497,7 @@ def __init__(self, term: BoundTerm[L], literals: Set[Literal[L]]): def value_set(self) -> Set[L]: return {lit.value for lit in self.literals} - def __str__(self): + def __str__(self) -> str: # Sort to make it deterministic return f"{str(self.__class__.__name__)}({str(self.term)}, {{{', '.join(sorted([str(literal) for literal in self.literals]))}}})" @@ -515,7 +515,7 @@ def as_unbound(self) -> Type[SetPredicate[L]]: class BoundIn(BoundSetPredicate[L]): - def __new__(cls, term: BoundTerm[L], literals: Set[Literal[L]]): # pylint: disable=W0221 + def __new__(cls, term: BoundTerm[L], literals: Set[Literal[L]]) -> BooleanExpression: # type: ignore # pylint: disable=W0221 count = len(literals) if count == 0: return AlwaysFalse() @@ -536,11 +536,11 @@ def as_unbound(self) -> Type[In[L]]: class BoundNotIn(BoundSetPredicate[L]): - def __new__( # pylint: disable=W0221 + def __new__( # type: ignore # pylint: disable=W0221 cls, term: BoundTerm[L], literals: Set[Literal[L]], - ): + ) -> BooleanExpression: count = len(literals) if count == 0: return AlwaysTrue() @@ -558,15 +558,15 @@ def as_unbound(self) -> Type[NotIn[L]]: class In(SetPredicate[L]): - def __new__( + def __new__( # type: ignore # pylint: disable=W0221 cls, term: Union[str, UnboundTerm[Any]], literals: Union[Iterable[L], Iterable[Literal[L]]] - ): # pylint: disable=W0221 + ) -> BooleanExpression: literals_set: Set[Literal[L]] = _to_literal_set(literals) count = len(literals_set) if count == 0: return AlwaysFalse() elif count == 1: - return EqualTo(term, next(iter(literals))) + return EqualTo(term, next(iter(literals))) # type: ignore else: return super().__new__(cls) @@ -579,15 +579,15 @@ def as_bound(self) -> Type[BoundIn[L]]: class NotIn(SetPredicate[L], ABC): - def __new__( + def __new__( # type: ignore # pylint: disable=W0221 cls, term: Union[str, UnboundTerm[Any]], literals: Union[Iterable[L], Iterable[Literal[L]]] - ): # pylint: disable=W0221 + ) -> BooleanExpression: literals_set: Set[Literal[L]] = _to_literal_set(literals) count = len(literals_set) if count == 0: return AlwaysTrue() elif count == 1: - return NotEqualTo(term, next(iter(literals_set))) + return NotEqualTo(term, next(iter(literals_set))) # type: ignore else: return super().__new__(cls) @@ -628,7 +628,7 @@ def bind(self, schema: Schema, case_sensitive: bool = True) -> BoundLiteralPredi return self.as_bound(bound_term, lit) - def __eq__(self, other): + def __eq__(self, other: Any) -> bool: if isinstance(other, LiteralPredicate): return self.term == other.term and self.literal == other.literal return False @@ -650,7 +650,7 @@ def __init__(self, term: BoundTerm[L], literal: Literal[L]): # pylint: disable= super().__init__(term) # type: ignore self.literal = literal # pylint: disable=W0621 - def __eq__(self, other): + def __eq__(self, other: Any) -> bool: if isinstance(other, BoundLiteralPredicate): return self.term == other.term and self.literal == other.literal return False diff --git a/pyiceberg/expressions/literals.py b/pyiceberg/expressions/literals.py index 8a0aaa08a6..4832da591f 100644 --- a/pyiceberg/expressions/literals.py +++ b/pyiceberg/expressions/literals.py @@ -90,19 +90,19 @@ def __eq__(self, other: Any) -> bool: return False return self.value == other.value - def __ne__(self, other) -> bool: + def __ne__(self, other: Any) -> bool: return not self.__eq__(other) - def __lt__(self, other) -> bool: + def __lt__(self, other: Any) -> bool: return self.value < other.value - def __gt__(self, other) -> bool: + def __gt__(self, other: Any) -> bool: return self.value > other.value - def __le__(self, other) -> bool: + def __le__(self, other: Any) -> bool: return self.value <= other.value - def __ge__(self, other) -> bool: + def __ge__(self, other: Any) -> bool: return self.value >= other.value @@ -153,7 +153,7 @@ def __str__(self) -> str: class FloatAboveMax(AboveMax[float], Singleton): - def __init__(self): + def __init__(self) -> None: super().__init__(FloatType.max, float) @singledispatchmethod @@ -166,7 +166,7 @@ def _(self, _: FloatType) -> Literal[float]: class FloatBelowMin(BelowMin[float], Singleton): - def __init__(self): + def __init__(self) -> None: super().__init__(FloatType.min, float) @singledispatchmethod @@ -179,7 +179,7 @@ def _(self, _: FloatType) -> Literal[float]: class IntAboveMax(AboveMax[int], Singleton): - def __init__(self): + def __init__(self) -> None: super().__init__(IntegerType.max, int) @singledispatchmethod @@ -192,7 +192,7 @@ def _(self, _: IntegerType) -> Literal[int]: class IntBelowMin(BelowMin[int], Singleton): - def __init__(self): + def __init__(self) -> None: super().__init__(IntegerType.min, int) @singledispatchmethod @@ -205,7 +205,7 @@ def _(self, _: IntegerType) -> Literal[int]: class LongAboveMax(AboveMax[int], Singleton): - def __init__(self): + def __init__(self) -> None: super().__init__(LongType.max, int) @singledispatchmethod @@ -218,7 +218,7 @@ def _(self, _: LongType) -> Literal[int]: class LongBelowMin(BelowMin[int], Singleton): - def __init__(self): + def __init__(self) -> None: super().__init__(LongType.min, int) @singledispatchmethod @@ -231,7 +231,7 @@ def _(self, _: LongType) -> Literal[int]: class BooleanLiteral(Literal[bool]): - def __init__(self, value: bool): + def __init__(self, value: bool) -> None: super().__init__(value, bool) @singledispatchmethod @@ -244,7 +244,7 @@ def _(self, _: BooleanType) -> Literal[bool]: class LongLiteral(Literal[int]): - def __init__(self, value: int): + def __init__(self, value: int) -> None: super().__init__(value, int) @singledispatchmethod @@ -306,23 +306,23 @@ def _(self, type_var: DecimalType) -> Literal[Decimal]: class FloatLiteral(Literal[float]): - def __init__(self, value: float): + def __init__(self, value: float) -> None: super().__init__(value, float) self._value32 = struct.unpack(" bool: + def __eq__(self, other: Any) -> bool: return self._value32 == other - def __lt__(self, other) -> bool: + def __lt__(self, other: Any) -> bool: return self._value32 < other - def __gt__(self, other) -> bool: + def __gt__(self, other: Any) -> bool: return self._value32 > other - def __le__(self, other) -> bool: + def __le__(self, other: Any) -> bool: return self._value32 <= other - def __ge__(self, other) -> bool: + def __ge__(self, other: Any) -> bool: return self._value32 >= other @singledispatchmethod @@ -343,7 +343,7 @@ def _(self, type_var: DecimalType) -> Literal[Decimal]: class DoubleLiteral(Literal[float]): - def __init__(self, value: float): + def __init__(self, value: float) -> None: super().__init__(value, float) @singledispatchmethod @@ -368,7 +368,7 @@ def _(self, type_var: DecimalType) -> Literal[Decimal]: class DateLiteral(Literal[int]): - def __init__(self, value: int): + def __init__(self, value: int) -> None: super().__init__(value, int) def increment(self) -> Literal[int]: @@ -387,7 +387,7 @@ def _(self, _: DateType) -> Literal[int]: class TimeLiteral(Literal[int]): - def __init__(self, value: int): + def __init__(self, value: int) -> None: super().__init__(value, int) @singledispatchmethod @@ -400,7 +400,7 @@ def _(self, _: TimeType) -> Literal[int]: class TimestampLiteral(Literal[int]): - def __init__(self, value: int): + def __init__(self, value: int) -> None: super().__init__(value, int) def increment(self) -> Literal[int]: @@ -423,7 +423,7 @@ def _(self, _: DateType) -> Literal[int]: class DecimalLiteral(Literal[Decimal]): - def __init__(self, value: Decimal): + def __init__(self, value: Decimal) -> None: super().__init__(value, Decimal) def increment(self) -> Literal[Decimal]: @@ -467,7 +467,7 @@ def _(self, _: LongType) -> Literal[int]: return LongLiteral(value_int) @to.register(FloatType) - def _(self, _: FloatType): + def _(self, _: FloatType) -> Literal[float]: value_float = float(self.value) if value_float > FloatType.max: return FloatAboveMax() @@ -477,12 +477,12 @@ def _(self, _: FloatType): return FloatLiteral(value_float) @to.register(DoubleType) - def _(self, _: DoubleLiteral): + def _(self, _: DoubleLiteral) -> Literal[float]: return DoubleLiteral(float(self.value)) class StringLiteral(Literal[str]): - def __init__(self, value: str): + def __init__(self, value: str) -> None: super().__init__(value, str) @singledispatchmethod @@ -560,7 +560,7 @@ def __repr__(self) -> str: class UUIDLiteral(Literal[UUID]): - def __init__(self, value: UUID): + def __init__(self, value: UUID) -> None: super().__init__(value, UUID) @singledispatchmethod @@ -573,7 +573,7 @@ def _(self, _: UUIDType) -> Literal[UUID]: class FixedLiteral(Literal[bytes]): - def __init__(self, value: bytes): + def __init__(self, value: bytes) -> None: super().__init__(value, bytes) @singledispatchmethod @@ -595,7 +595,7 @@ def _(self, _: BinaryType) -> Literal[bytes]: class BinaryLiteral(Literal[bytes]): - def __init__(self, value: bytes): + def __init__(self, value: bytes) -> None: super().__init__(value, bytes) @singledispatchmethod diff --git a/pyiceberg/expressions/visitors.py b/pyiceberg/expressions/visitors.py index 8888b722e0..f5c11a61cf 100644 --- a/pyiceberg/expressions/visitors.py +++ b/pyiceberg/expressions/visitors.py @@ -124,7 +124,7 @@ def visit_bound_predicate(self, predicate: BoundPredicate[L]) -> T: @singledispatch -def visit(obj, visitor: BooleanExpressionVisitor[T]) -> T: +def visit(obj: BooleanExpression, visitor: BooleanExpressionVisitor[T]) -> T: """A generic function for applying a boolean expression visitor to any point within an expression The function traverses the expression in post-order fashion @@ -309,7 +309,7 @@ def visit_and(self, left_result: T, right_result: T) -> T: def visit_or(self, left_result: T, right_result: T) -> T: """Visit a bound Or predicate""" - def visit_unbound_predicate(self, predicate: UnboundPredicate[L]): + def visit_unbound_predicate(self, predicate: UnboundPredicate[L]) -> T: """Visit an unbound predicate Args: predicate (UnboundPredicate[L]): An unbound predicate @@ -495,7 +495,7 @@ def visit_or(self, left_result: bool, right_result: bool) -> bool: IN_PREDICATE_LIMIT = 200 -def _from_byte_buffer(field_type: IcebergType, val: bytes): +def _from_byte_buffer(field_type: IcebergType, val: bytes) -> Any: if not isinstance(field_type, PrimitiveType): raise ValueError(f"Expected a PrimitiveType, got: {type(field_type)}") return from_bytes(field_type, val) @@ -505,7 +505,7 @@ class _ManifestEvalVisitor(BoundBooleanExpressionVisitor[bool]): partition_fields: List[PartitionFieldSummary] partition_filter: BooleanExpression - def __init__(self, partition_struct_schema: Schema, partition_filter: BooleanExpression, case_sensitive): + def __init__(self, partition_struct_schema: Schema, partition_filter: BooleanExpression, case_sensitive: bool) -> None: self.partition_filter = bind(partition_struct_schema, rewrite_not(partition_filter), case_sensitive) def eval(self, manifest: ManifestFile) -> bool: diff --git a/pyiceberg/io/__init__.py b/pyiceberg/io/__init__.py index 6d734316ef..7c721712d9 100644 --- a/pyiceberg/io/__init__.py +++ b/pyiceberg/io/__init__.py @@ -22,15 +22,19 @@ for returning an InputFile instance, an OutputFile instance, and deleting a file given its location. """ +from __future__ import annotations + import importlib import logging from abc import ABC, abstractmethod from io import SEEK_SET +from types import TracebackType from typing import ( Dict, List, Optional, Protocol, + Type, Union, runtime_checkable, ) @@ -65,12 +69,13 @@ def tell(self) -> int: def close(self) -> None: ... - @abstractmethod - def __enter__(self): + def __enter__(self) -> InputStream: ... @abstractmethod - def __exit__(self, exc_type, exc_val, exc_tb): + def __exit__( + self, exctype: Optional[Type[BaseException]], excinst: Optional[BaseException], exctb: Optional[TracebackType] + ) -> None: ... @@ -91,11 +96,13 @@ def close(self) -> None: ... @abstractmethod - def __enter__(self): + def __enter__(self) -> OutputStream: ... @abstractmethod - def __exit__(self, exc_type, exc_val, exc_tb): + def __exit__( + self, exctype: Optional[Type[BaseException]], excinst: Optional[BaseException], exctb: Optional[TracebackType] + ) -> None: ... diff --git a/pyiceberg/io/fsspec.py b/pyiceberg/io/fsspec.py index 1f5dad686c..34e2246206 100644 --- a/pyiceberg/io/fsspec.py +++ b/pyiceberg/io/fsspec.py @@ -17,7 +17,12 @@ """FileIO implementation for reading and writing table files that uses fsspec compatible filesystems""" import logging from functools import lru_cache, partial -from typing import Callable, Dict, Union +from typing import ( + Any, + Callable, + Dict, + Union, +) from urllib.parse import urlparse import requests @@ -29,13 +34,19 @@ from pyiceberg.catalog import TOKEN from pyiceberg.exceptions import SignError -from pyiceberg.io import FileIO, InputFile, OutputFile +from pyiceberg.io import ( + FileIO, + InputFile, + InputStream, + OutputFile, + OutputStream, +) from pyiceberg.typedef import Properties logger = logging.getLogger(__name__) -def s3v4_rest_signer(properties: Properties, request: AWSRequest, **_) -> AWSRequest: +def s3v4_rest_signer(properties: Properties, request: AWSRequest, **_: Any) -> AWSRequest: if TOKEN not in properties: raise SignError("Signer set, but token is not available") @@ -127,7 +138,7 @@ def exists(self) -> bool: """Checks whether the location exists""" return self._fs.lexists(self.location) - def open(self): + def open(self) -> InputStream: """Create an input stream for reading the contents of the file Returns: @@ -161,7 +172,7 @@ def exists(self) -> bool: """Checks whether the location exists""" return self._fs.lexists(self.location) - def create(self, overwrite: bool = False): + def create(self, overwrite: bool = False) -> OutputStream: """Create an output stream for reading the contents of the file Args: diff --git a/pyiceberg/io/memory.py b/pyiceberg/io/memory.py index 1ea17feac1..a4eb89d76f 100644 --- a/pyiceberg/io/memory.py +++ b/pyiceberg/io/memory.py @@ -14,8 +14,11 @@ # KIND, either express or implied. See the License for the # specific language governing permissions and limitations # under the License. +from __future__ import annotations from io import SEEK_CUR, SEEK_END, SEEK_SET +from types import TracebackType +from typing import Optional, Type from pyiceberg.io import InputStream @@ -71,8 +74,10 @@ def close(self) -> None: del self.buffer self.pos = 0 - def __enter__(self): - pass + def __enter__(self) -> MemoryInputStream: + return self - def __exit__(self, exc_type, exc_val, exc_tb): + def __exit__( + self, exctype: Optional[Type[BaseException]], excinst: Optional[BaseException], exctb: Optional[TracebackType] + ) -> None: self.close() diff --git a/pyiceberg/io/pyarrow.py b/pyiceberg/io/pyarrow.py index ce4e9b8ae9..eeba4945d4 100644 --- a/pyiceberg/io/pyarrow.py +++ b/pyiceberg/io/pyarrow.py @@ -403,15 +403,15 @@ def visit_not_in(self, term: BoundTerm[pc.Expression], literals: Set[Any]) -> pc pyarrow_literals = pa.array(literals, type=_iceberg_to_pyarrow_type(term.ref().field.field_type)) return ~pc.field(term.ref().field.name).isin(pyarrow_literals) - def visit_is_nan(self, term: BoundTerm[pc.Expression]) -> pc.Expression: + def visit_is_nan(self, term: BoundTerm[Any]) -> pc.Expression: ref = pc.field(term.ref().field.name) return ref.is_null(nan_is_null=True) & ref.is_valid() - def visit_not_nan(self, term: BoundTerm[pc.Expression]) -> pc.Expression: + def visit_not_nan(self, term: BoundTerm[Any]) -> pc.Expression: ref = pc.field(term.ref().field.name) return ~(ref.is_null(nan_is_null=True) & ref.is_valid()) - def visit_is_null(self, term: BoundTerm[pc.Expression]) -> pc.Expression: + def visit_is_null(self, term: BoundTerm[Any]) -> pc.Expression: return pc.field(term.ref().field.name).is_null(nan_is_null=False) def visit_not_null(self, term: BoundTerm[Any]) -> pc.Expression: diff --git a/pyiceberg/partitioning.py b/pyiceberg/partitioning.py index b5dd427197..aca79bfa76 100644 --- a/pyiceberg/partitioning.py +++ b/pyiceberg/partitioning.py @@ -69,7 +69,7 @@ def __init__( data["name"] = name super().__init__(**data) - def __str__(self): + def __str__(self) -> str: return f"{self.field_id}: {self.name}: {self.transform}({self.source_id})" @@ -105,7 +105,7 @@ def __eq__(self, other: Any) -> bool: return False return self.spec_id == other.spec_id and self.fields == other.fields - def __str__(self): + def __str__(self) -> str: """ Produce a human-readable string representation of PartitionSpec diff --git a/pyiceberg/schema.py b/pyiceberg/schema.py index ca1d50828b..fa86845caa 100644 --- a/pyiceberg/schema.py +++ b/pyiceberg/schema.py @@ -66,19 +66,19 @@ class Schema(IcebergBaseModel): _name_to_id: Dict[str, int] = PrivateAttr() - def __init__(self, *fields: NestedField, **data): + def __init__(self, *fields: NestedField, **data: Any): if fields: data["fields"] = fields super().__init__(**data) self._name_to_id = index_by_name(self) - def __str__(self): + def __str__(self) -> str: return "table {\n" + "\n".join([" " + str(field) for field in self.columns]) + "\n}" - def __repr__(self): + def __repr__(self) -> str: return f"Schema({', '.join(repr(column) for column in self.columns)}, schema_id={self.schema_id}, identifier_field_ids={self.identifier_field_ids})" - def __eq__(self, other) -> bool: + def __eq__(self, other: Any) -> bool: if not other: return False @@ -178,7 +178,7 @@ def find_type(self, name_or_id: Union[str, int], case_sensitive: bool = True) -> return field.field_type @property - def highest_field_id(self): + def highest_field_id(self) -> int: return visit(self.as_struct(), _FindLastFieldId()) def find_column_name(self, column_id: int) -> Optional[str]: @@ -324,10 +324,10 @@ class Accessor: position: int inner: Optional["Accessor"] = None - def __str__(self): + def __str__(self) -> str: return f"Accessor(position={self.position},inner={self.inner})" - def __repr__(self): + def __repr__(self) -> str: return self.__str__() def get(self, container: StructProtocol) -> Any: @@ -350,7 +350,7 @@ def get(self, container: StructProtocol) -> Any: @singledispatch -def visit(obj, visitor: SchemaVisitor[T]) -> T: +def visit(obj: Union[Schema, IcebergType], visitor: SchemaVisitor[T]) -> T: """A generic function for applying a schema visitor to any point within a schema The function traverses the schema in post-order fashion @@ -417,7 +417,7 @@ def _(obj: PrimitiveType, visitor: SchemaVisitor[T]) -> T: @singledispatch -def pre_order_visit(obj, visitor: PreOrderSchemaVisitor[T]) -> T: +def pre_order_visit(obj: Union[Schema, IcebergType], visitor: PreOrderSchemaVisitor[T]) -> T: """A generic function for applying a schema visitor to any point within a schema The function traverses the schema in pre-order fashion. This is a slimmed down version @@ -479,33 +479,35 @@ class _IndexById(SchemaVisitor[Dict[int, NestedField]]): def __init__(self) -> None: self._index: Dict[int, NestedField] = {} - def schema(self, schema: Schema, struct_result) -> Dict[int, NestedField]: + def schema(self, schema: Schema, struct_result: Dict[int, NestedField]) -> Dict[int, NestedField]: return self._index - def struct(self, struct: StructType, field_results) -> Dict[int, NestedField]: + def struct(self, struct: StructType, field_results: List[Dict[int, NestedField]]) -> Dict[int, NestedField]: return self._index - def field(self, field: NestedField, field_result) -> Dict[int, NestedField]: + def field(self, field: NestedField, field_result: Dict[int, NestedField]) -> Dict[int, NestedField]: """Add the field ID to the index""" self._index[field.field_id] = field return self._index - def list(self, list_type: ListType, element_result) -> Dict[int, NestedField]: + def list(self, list_type: ListType, element_result: Dict[int, NestedField]) -> Dict[int, NestedField]: """Add the list element ID to the index""" self._index[list_type.element_field.field_id] = list_type.element_field return self._index - def map(self, map_type: MapType, key_result, value_result) -> Dict[int, NestedField]: + def map( + self, map_type: MapType, key_result: Dict[int, NestedField], value_result: Dict[int, NestedField] + ) -> Dict[int, NestedField]: """Add the key ID and value ID as individual items in the index""" self._index[map_type.key_field.field_id] = map_type.key_field self._index[map_type.value_field.field_id] = map_type.value_field return self._index - def primitive(self, primitive) -> Dict[int, NestedField]: + def primitive(self, primitive: PrimitiveType) -> Dict[int, NestedField]: return self._index -def index_by_id(schema_or_type) -> Dict[int, NestedField]: +def index_by_id(schema_or_type: Union[Schema, IcebergType]) -> Dict[int, NestedField]: """Generate an index of field IDs to NestedField instances Args: @@ -570,7 +572,7 @@ def map(self, map_type: MapType, key_result: Dict[str, int], value_result: Dict[ self._add_field(map_type.value_field.name, map_type.value_field.field_id) return self._index - def _add_field(self, name: str, field_id: int): + def _add_field(self, name: str, field_id: int) -> None: """Add a field name to the index, mapping its full name to its field ID Args: @@ -593,7 +595,7 @@ def _add_field(self, name: str, field_id: int): short_name = ".".join([".".join(self._short_field_names), name]) self._short_name_to_id[short_name] = field_id - def primitive(self, primitive) -> Dict[str, int]: + def primitive(self, primitive: PrimitiveType) -> Dict[str, int]: return self._index def by_name(self) -> Dict[str, int]: @@ -919,7 +921,7 @@ def _project_selected_struct(projected_field: Optional[IcebergType]) -> StructTy return projected_field @staticmethod - def _project_list(list_type: ListType, element_result: IcebergType): + def _project_list(list_type: ListType, element_result: IcebergType) -> ListType: if list_type.element_type == element_result: return list_type else: @@ -928,7 +930,7 @@ def _project_list(list_type: ListType, element_result: IcebergType): ) @staticmethod - def _project_map(map_type: MapType, value_result: IcebergType): + def _project_map(map_type: MapType, value_result: IcebergType) -> MapType: if map_type.value_type == value_result: return map_type else: diff --git a/pyiceberg/table/__init__.py b/pyiceberg/table/__init__.py index 0bb76a4f4b..904b234f3a 100644 --- a/pyiceberg/table/__init__.py +++ b/pyiceberg/table/__init__.py @@ -21,6 +21,7 @@ from dataclasses import dataclass from functools import cached_property from typing import ( + TYPE_CHECKING, Any, Callable, Dict, @@ -57,6 +58,10 @@ ) from pyiceberg.types import StructType +if TYPE_CHECKING: + import pyarrow as pa + from duckdb import DuckDBPyConnection + class Table: identifier: Identifier = Field() @@ -64,13 +69,13 @@ class Table: metadata_location: str = Field() io: FileIO - def __init__(self, identifier: Identifier, metadata: TableMetadata, metadata_location: str, io: FileIO): + def __init__(self, identifier: Identifier, metadata: TableMetadata, metadata_location: str, io: FileIO) -> None: self.identifier = identifier self.metadata = metadata self.metadata_location = metadata_location self.io = io - def refresh(self): + def refresh(self) -> Table: """Refresh the current table metadata""" raise NotImplementedError("To be implemented") @@ -202,18 +207,18 @@ def projection(self) -> Schema: return snapshot_schema.select(*self.selected_fields, case_sensitive=self.case_sensitive) @abstractmethod - def plan_files(self): + def plan_files(self) -> Iterator[ScanTask]: ... @abstractmethod - def to_arrow(self): + def to_arrow(self) -> pa.table: ... - def update(self: S, **overrides) -> S: + def update(self: S, **overrides: Any) -> S: """Creates a copy of this table scan with updated fields.""" return type(self)(**{**self.__dict__, **overrides}) - def use_ref(self, name: str): + def use_ref(self, name: str) -> S: if self.snapshot_id: raise ValueError(f"Cannot override ref, already set snapshot id={self.snapshot_id}") if snapshot := self.table.snapshot_by_name(name): @@ -333,7 +338,7 @@ def plan_files(self) -> Iterator[ScanTask]: yield from (FileScanTask(file) for file in matching_partition_files) - def to_arrow(self): + def to_arrow(self) -> pa.table: from pyiceberg.io.pyarrow import PyArrowFileIO, expression_to_pyarrow, schema_to_pyarrow warnings.warn( @@ -375,7 +380,7 @@ def to_arrow(self): return ds.to_table(filter=pyarrow_filter, columns=columns) - def to_duckdb(self, table_name: str, connection=None): + def to_duckdb(self, table_name: str, connection: Optional[DuckDBPyConnection] = None) -> DuckDBPyConnection: import duckdb con = connection or duckdb.connect(database=":memory:") diff --git a/pyiceberg/table/metadata.py b/pyiceberg/table/metadata.py index 39e60556f7..5704222d2c 100644 --- a/pyiceberg/table/metadata.py +++ b/pyiceberg/table/metadata.py @@ -99,7 +99,7 @@ class TableMetadataCommonFields(IcebergBaseModel): spec (https://iceberg.apache.org/spec/#iceberg-table-spec)""" @root_validator(skip_on_failure=True) - def cleanup_snapshot_id(cls, data: Dict[str, Any]): + def cleanup_snapshot_id(cls, data: Dict[str, Any]) -> Dict[str, Any]: if data[CURRENT_SNAPSHOT_ID] == -1: # We treat -1 and None the same, by cleaning this up # in a pre-validator, we can simplify the logic later on @@ -107,7 +107,7 @@ def cleanup_snapshot_id(cls, data: Dict[str, Any]): return data @root_validator(skip_on_failure=True) - def construct_refs(cls, data: Dict[str, Any]): + def construct_refs(cls, data: Dict[str, Any]) -> Dict[str, Any]: # This is going to be much nicer as soon as refs is an actual pydantic object if current_snapshot_id := data.get(CURRENT_SNAPSHOT_ID): if MAIN_BRANCH not in data[REFS]: @@ -277,7 +277,7 @@ def construct_partition_specs(cls, data: Dict[str, Any]) -> Dict[str, Any]: return data @root_validator(skip_on_failure=True) - def set_sort_orders(cls, data: Dict[str, Any]): + def set_sort_orders(cls, data: Dict[str, Any]) -> Dict[str, Any]: """Sets the sort_orders if not provided For V1 sort_orders is optional, and if they aren't set, we'll set them @@ -329,15 +329,15 @@ class TableMetadataV2(TableMetadataCommonFields, IcebergBaseModel): """ @root_validator(skip_on_failure=True) - def check_schemas(cls, values: Dict[str, Any]): + def check_schemas(cls, values: Dict[str, Any]) -> Dict[str, Any]: return check_schemas(values) @root_validator - def check_partition_specs(cls, values: Dict[str, Any]): + def check_partition_specs(cls, values: Dict[str, Any]) -> Dict[str, Any]: return check_partition_specs(values) @root_validator(skip_on_failure=True) - def check_sort_orders(cls, values: Dict[str, Any]): + def check_sort_orders(cls, values: Dict[str, Any]) -> Dict[str, Any]: return check_sort_orders(values) format_version: Literal[2] = Field(alias="format-version", default=2) diff --git a/pyiceberg/table/snapshots.py b/pyiceberg/table/snapshots.py index 3fd834af84..af17fd4b4e 100644 --- a/pyiceberg/table/snapshots.py +++ b/pyiceberg/table/snapshots.py @@ -16,6 +16,7 @@ # under the License. from enum import Enum from typing import ( + Any, Dict, List, Optional, @@ -69,8 +70,8 @@ def check_operation(cls, values: Dict[str, Dict[str, Union[str, Operation]]]) -> return values def __init__( - self, operation: Optional[Operation] = None, __root__: Optional[Dict[str, Union[str, Operation]]] = None, **data - ): + self, operation: Optional[Operation] = None, __root__: Optional[Dict[str, Union[str, Operation]]] = None, **data: Any + ) -> None: super().__init__(__root__={"operation": operation, **data} if not __root__ else __root__) self._additional_properties = { k: v for k, v in self.__root__.items() if k != OPERATION # type: ignore # We know that they are all string, and we don't want to check diff --git a/pyiceberg/table/sorting.py b/pyiceberg/table/sorting.py index 22acfac2ed..67f27d82eb 100644 --- a/pyiceberg/table/sorting.py +++ b/pyiceberg/table/sorting.py @@ -96,7 +96,7 @@ def set_null_order(cls, values: Dict[str, Any]) -> Dict[str, Any]: direction: SortDirection = Field() null_order: NullOrder = Field(alias="null-order") - def __str__(self): + def __str__(self) -> str: if type(self.transform) == IdentityTransform: # In the case of an identity transform, we can omit the transform return f"{self.source_id} {self.direction} {self.null_order}" @@ -138,7 +138,7 @@ def __str__(self) -> str: result_str += "]" return result_str - def __repr__(self): + def __repr__(self) -> str: fields = f"{', '.join(repr(column) for column in self.fields)}, " if self.fields else "" return f"SortOrder({fields}order_id={self.order_id})" diff --git a/pyiceberg/transforms.py b/pyiceberg/transforms.py index 0c87c8ac27..b163c02569 100644 --- a/pyiceberg/transforms.py +++ b/pyiceberg/transforms.py @@ -20,12 +20,18 @@ from abc import ABC, abstractmethod from enum import IntEnum from functools import singledispatch -from typing import Any, Callable, Generic +from typing import ( + Any, + Callable, + Generator, + Generic, +) from typing import Literal as LiteralType from typing import Optional, TypeVar import mmh3 from pydantic import Field, PositiveInt, PrivateAttr +from pydantic.typing import AnyCallable from pyiceberg.expressions import ( BoundEqualTo, @@ -106,14 +112,14 @@ class Transform(IcebergBaseModel, ABC, Generic[S, T]): __root__: str = Field() @classmethod - def __get_validators__(cls): + def __get_validators__(cls) -> Generator[AnyCallable, None, None]: # one or more validators may be yielded which will be called in the # order to validate the input, each validator will receive as an input # the value returned from the previous validator yield cls.validate @classmethod - def validate(cls, v: Any): + def validate(cls, v: Any) -> IcebergBaseModel: # When Pydantic is unable to determine the subtype # In this case we'll help pydantic a bit by parsing the transform type ourselves if isinstance(v, str): @@ -157,7 +163,7 @@ def project(self, name: str, pred: BoundPredicate[L]) -> Optional[UnboundPredica def preserves_order(self) -> bool: return False - def satisfies_order_of(self, other) -> bool: + def satisfies_order_of(self, other: Any) -> bool: return self == other def to_human_string(self, _: IcebergType, value: Optional[S]) -> str: @@ -188,7 +194,7 @@ class BucketTransform(Transform[S, int]): _num_buckets: PositiveInt = PrivateAttr() - def __init__(self, num_buckets: int, **data: Any): + def __init__(self, num_buckets: int, **data: Any) -> None: super().__init__(__root__=f"bucket[{num_buckets}]", **data) self._num_buckets = num_buckets @@ -241,22 +247,22 @@ def transform(self, source: IcebergType, bucket: bool = True) -> Callable[[Optio source_type = type(source) if source_type in {IntegerType, LongType, DateType, TimeType, TimestampType, TimestamptzType}: - def hash_func(v): + def hash_func(v: Any) -> int: return mmh3.hash(struct.pack(" int: return mmh3.hash(decimal_to_bytes(v)) elif source_type in {StringType, FixedType, BinaryType}: - def hash_func(v): + def hash_func(v: Any) -> int: return mmh3.hash(v) elif source_type == UUIDType: - def hash_func(v): + def hash_func(v: Any) -> int: return mmh3.hash( struct.pack( ">QQ", @@ -339,12 +345,12 @@ def transform(self, source: IcebergType) -> Callable[[Optional[S]], Optional[int source_type = type(source) if source_type == DateType: - def year_func(v): + def year_func(v: Any) -> int: return datetime.days_to_years(v) elif source_type in {TimestampType, TimestamptzType}: - def year_func(v): + def year_func(v: Any) -> int: return datetime.micros_to_years(v) else: @@ -385,12 +391,12 @@ def transform(self, source: IcebergType) -> Callable[[Optional[S]], Optional[int source_type = type(source) if source_type == DateType: - def month_func(v): + def month_func(v: Any) -> int: return datetime.days_to_months(v) elif source_type in {TimestampType, TimestamptzType}: - def month_func(v): + def month_func(v: Any) -> int: return datetime.micros_to_months(v) else: @@ -431,12 +437,12 @@ def transform(self, source: IcebergType) -> Callable[[Optional[S]], Optional[int source_type = type(source) if source_type == DateType: - def day_func(v): + def day_func(v: Any) -> int: return v elif source_type in {TimestampType, TimestamptzType}: - def day_func(v): + def day_func(v: Any) -> int: return datetime.micros_to_days(v) else: @@ -479,7 +485,7 @@ class HourTransform(TimeTransform[S]): def transform(self, source: IcebergType) -> Callable[[Optional[S]], Optional[int]]: if type(source) in {TimestampType, TimestamptzType}: - def hour_func(v): + def hour_func(v: Any) -> int: return datetime.micros_to_hours(v) else: @@ -618,17 +624,17 @@ def transform(self, source: IcebergType) -> Callable[[Optional[S]], Optional[S]] source_type = type(source) if source_type in {IntegerType, LongType}: - def truncate_func(v): + def truncate_func(v: Any) -> Any: return v - v % self._width elif source_type in {StringType, BinaryType}: - def truncate_func(v): + def truncate_func(v: Any) -> Any: return v[0 : min(self._width, len(v))] elif source_type == DecimalType: - def truncate_func(v): + def truncate_func(v: Any) -> Any: return truncate_decimal(v, self._width) else: @@ -802,7 +808,7 @@ def _project_transform_predicate( return None -def _remove_transform(partition_name: str, pred: BoundPredicate[L]): +def _remove_transform(partition_name: str, pred: BoundPredicate[L]) -> UnboundPredicate[Any]: if isinstance(pred, BoundUnaryPredicate): return pred.as_unbound(Reference(partition_name)) elif isinstance(pred, BoundLiteralPredicate): diff --git a/pyiceberg/types.py b/pyiceberg/types.py index c9c8847435..6236358063 100644 --- a/pyiceberg/types.py +++ b/pyiceberg/types.py @@ -29,17 +29,21 @@ Notes: - https://iceberg.apache.org/#spec/#primitive-types """ +from __future__ import annotations + import re from typing import ( Any, ClassVar, Dict, + Generator, Literal, Optional, Tuple, ) from pydantic import Field, PrivateAttr +from pydantic.typing import AnyCallable from pyiceberg.utils.iceberg_base_model import IcebergBaseModel from pyiceberg.utils.parsing import ParseNumberFromBrackets @@ -61,14 +65,14 @@ class IcebergType(IcebergBaseModel, Singleton): """ @classmethod - def __get_validators__(cls): + def __get_validators__(cls) -> Generator[AnyCallable, None, None]: # one or more validators may be yielded which will be called in the # order to validate the input, each validator will receive as an input # the value returned from the previous validator yield cls.validate @classmethod - def validate(cls, v: Any) -> "IcebergType": + def validate(cls, v: Any) -> IcebergType: # When Pydantic is unable to determine the subtype # In this case we'll help pydantic a bit by parsing the # primitive type ourselves, or pointing it at the correct @@ -129,7 +133,7 @@ class FixedType(PrimitiveType): _len: int = PrivateAttr() @staticmethod - def parse(str_repr: str) -> "FixedType": + def parse(str_repr: str) -> FixedType: return FixedType(length=FIXED_PARSER.match(str_repr)) def __init__(self, length: int): @@ -158,7 +162,7 @@ class DecimalType(PrimitiveType): _scale: int = PrivateAttr() @staticmethod - def parse(str_repr: str) -> "DecimalType": + def parse(str_repr: str) -> DecimalType: matches = DECIMAL_REGEX.search(str_repr) if matches: precision = int(matches.group(1)) @@ -223,7 +227,7 @@ def __init__( field_type: Optional[IcebergType] = None, required: bool = True, doc: Optional[str] = None, - **data, + **data: Any, ): # We need an init when we want to use positional arguments, but # need also to support the aliases. @@ -258,7 +262,7 @@ class StructType(IcebergType): type: Literal["struct"] = "struct" fields: Tuple[NestedField, ...] = Field(default_factory=tuple) - def __init__(self, *fields: NestedField, **data): + def __init__(self, *fields: NestedField, **data: Any): # In case we use positional arguments, instead of keyword args if fields: data["fields"] = fields @@ -289,7 +293,7 @@ class Config: element_field: NestedField = Field(init=False, repr=False) def __init__( - self, element_id: Optional[int] = None, element: Optional[IcebergType] = None, element_required: bool = True, **data + self, element_id: Optional[int] = None, element: Optional[IcebergType] = None, element_required: bool = True, **data: Any ): data["element_id"] = data["element-id"] if "element-id" in data else element_id data["element_type"] = element or data["element_type"] @@ -333,7 +337,7 @@ def __init__( value_id: Optional[int] = None, value_type: Optional[IcebergType] = None, value_required: bool = True, - **data, + **data: Any, ): data["key_id"] = key_id or data["key-id"] data["key_type"] = key_type or data["key"] diff --git a/pyiceberg/utils/config.py b/pyiceberg/utils/config.py index 65aa81d359..6055f75df9 100644 --- a/pyiceberg/utils/config.py +++ b/pyiceberg/utils/config.py @@ -58,7 +58,7 @@ def _lowercase_dictionary_keys(input_dict: RecursiveDict) -> RecursiveDict: class Config: config: RecursiveDict - def __init__(self): + def __init__(self) -> None: config = self._from_configuration_files() or {} config = merge_config(config, self._from_environment_variables(config)) self.config = FrozenDict(**config) @@ -101,7 +101,7 @@ def _from_environment_variables(config: RecursiveDict) -> RecursiveDict: Amended configuration """ - def set_property(_config: RecursiveDict, path: List[str], config_value: str): + def set_property(_config: RecursiveDict, path: List[str], config_value: str) -> None: while len(path) > 0: element = path.pop(0) if len(path) == 0: diff --git a/pyiceberg/utils/datetime.py b/pyiceberg/utils/datetime.py index 7353ebd4ba..2d3664103f 100644 --- a/pyiceberg/utils/datetime.py +++ b/pyiceberg/utils/datetime.py @@ -97,13 +97,13 @@ def timestamptz_to_micros(timestamptz_str: str) -> int: raise ValueError(f"Invalid timestamp with zone: {timestamptz_str} (must be ISO-8601)") -def micros_to_timestamp(micros: int): +def micros_to_timestamp(micros: int) -> datetime: """Converts microseconds from epoch to a timestamp""" dt = timedelta(microseconds=micros) return EPOCH_TIMESTAMP + dt -def micros_to_timestamptz(micros: int): +def micros_to_timestamptz(micros: int) -> datetime: """Converts microseconds from epoch to an utc timestamp""" dt = timedelta(microseconds=micros) return EPOCH_TIMESTAMPTZ + dt diff --git a/pyiceberg/utils/deprecated.py b/pyiceberg/utils/deprecated.py index 4b96881b1b..2fc4ec39ed 100644 --- a/pyiceberg/utils/deprecated.py +++ b/pyiceberg/utils/deprecated.py @@ -16,10 +16,10 @@ # under the License. import functools import warnings -from typing import Callable, Optional +from typing import Any, Callable, Optional -def deprecated(deprecated_in: str, removed_in: str, help_message: Optional[str] = None): +def deprecated(deprecated_in: str, removed_in: str, help_message: Optional[str] = None) -> Callable: # type: ignore """This is a decorator which can be used to mark functions as deprecated. It will result in a warning being emitted when the function is used.""" @@ -29,7 +29,7 @@ def deprecated(deprecated_in: str, removed_in: str, help_message: Optional[str] def decorator(func: Callable): # type: ignore @functools.wraps(func) - def new_func(*args, **kwargs): + def new_func(*args: Any, **kwargs: Any) -> Any: warnings.simplefilter("always", DeprecationWarning) # turn off filter warnings.warn( diff --git a/pyiceberg/utils/iceberg_base_model.py b/pyiceberg/utils/iceberg_base_model.py index e558e4a444..8c5bc94f68 100644 --- a/pyiceberg/utils/iceberg_base_model.py +++ b/pyiceberg/utils/iceberg_base_model.py @@ -15,7 +15,12 @@ # specific language governing permissions and limitations # under the License. from functools import cached_property -from typing import Optional, Set +from typing import ( + Any, + Dict, + Optional, + Set, +) from pydantic import BaseModel @@ -48,10 +53,10 @@ def _exclude_private_properties(self, exclude: Optional[Set[str]] = None) -> Set {field for field in self.__dict__ if field.startswith("_") and not field == "__root__"}, exclude or set() ) - def dict(self, exclude_none: bool = True, exclude: Optional[Set[str]] = None, **kwargs): + def dict(self, exclude_none: bool = True, exclude: Optional[Set[str]] = None, **kwargs: Any) -> Dict[str, Any]: return super().dict(exclude_none=exclude_none, exclude=self._exclude_private_properties(exclude), **kwargs) - def json(self, exclude_none: bool = True, exclude: Optional[Set[str]] = None, by_alias: bool = True, **kwargs): + def json(self, exclude_none: bool = True, exclude: Optional[Set[str]] = None, by_alias: bool = True, **kwargs: Any) -> str: return super().json( exclude_none=exclude_none, exclude=self._exclude_private_properties(exclude), by_alias=by_alias, **kwargs ) diff --git a/pyiceberg/utils/singleton.py b/pyiceberg/utils/singleton.py index ec9eac4a13..709f36a705 100644 --- a/pyiceberg/utils/singleton.py +++ b/pyiceberg/utils/singleton.py @@ -41,7 +41,7 @@ def _convert_to_hashable_type(element: Any) -> Any: class Singleton: _instances: ClassVar[Dict] = {} # type: ignore - def __new__(cls, *args, **kwargs): + def __new__(cls, *args, **kwargs): # type: ignore key = (cls, tuple(args), _convert_to_hashable_type(kwargs)) if key not in cls._instances: cls._instances[key] = super().__new__(cls) diff --git a/pyproject.toml b/pyproject.toml index f0e68a29ca..8b8768afed 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -120,6 +120,7 @@ warn_redundant_casts = true warn_unreachable = true warn_unused_ignores = true disallow_any_generics = true +disallow_untyped_defs = true [[tool.mypy.overrides]] module = "pyarrow.*" diff --git a/tests/avro/test_decoder.py b/tests/avro/test_decoder.py index 1d2a9d4c26..9bbd67db74 100644 --- a/tests/avro/test_decoder.py +++ b/tests/avro/test_decoder.py @@ -14,9 +14,13 @@ # KIND, either express or implied. See the License for the # specific language governing permissions and limitations # under the License. +from __future__ import annotations + from datetime import date, datetime, timezone from decimal import Decimal from io import SEEK_SET +from types import TracebackType +from typing import Optional, Type import pytest @@ -27,7 +31,7 @@ from pyiceberg.types import DoubleType, FloatType -def test_read_decimal_from_fixed(): +def test_read_decimal_from_fixed() -> None: mis = MemoryInputStream(b"\x00\x00\x00\x05\x6A\x48\x1C\xFB\x2C\x7C\x50\x00") decoder = BinaryDecoder(mis) actual = decoder.read_decimal_from_fixed(28, 15, 12) @@ -35,19 +39,19 @@ def test_read_decimal_from_fixed(): assert actual == expected -def test_read_boolean_true(): +def test_read_boolean_true() -> None: mis = MemoryInputStream(b"\x01") decoder = BinaryDecoder(mis) assert decoder.read_boolean() is True -def test_read_boolean_false(): +def test_read_boolean_false() -> None: mis = MemoryInputStream(b"\x00") decoder = BinaryDecoder(mis) assert decoder.read_boolean() is False -def test_skip_boolean(): +def test_skip_boolean() -> None: mis = MemoryInputStream(b"\x00") decoder = BinaryDecoder(mis) assert mis.tell() == 0 @@ -55,13 +59,13 @@ def test_skip_boolean(): assert mis.tell() == 1 -def test_read_int(): +def test_read_int() -> None: mis = MemoryInputStream(b"\x18") decoder = BinaryDecoder(mis) assert decoder.read_int() == 12 -def test_skip_int(): +def test_skip_int() -> None: mis = MemoryInputStream(b"\x18") decoder = BinaryDecoder(mis) assert mis.tell() == 0 @@ -69,7 +73,7 @@ def test_skip_int(): assert mis.tell() == 1 -def test_read_decimal(): +def test_read_decimal() -> None: mis = MemoryInputStream(b"\x18\x00\x00\x00\x05\x6A\x48\x1C\xFB\x2C\x7C\x50\x00") decoder = BinaryDecoder(mis) actual = decoder.read_decimal_from_bytes(28, 15) @@ -77,7 +81,7 @@ def test_read_decimal(): assert actual == expected -def test_decimal_from_fixed_big(): +def test_decimal_from_fixed_big() -> None: mis = MemoryInputStream(b"\x0E\xC2\x02\xE9\x06\x16\x33\x49\x77\x67\xA8\x00") decoder = BinaryDecoder(mis) actual = decoder.read_decimal_from_fixed(28, 15, 12) @@ -85,7 +89,7 @@ def test_decimal_from_fixed_big(): assert actual == expected -def test_read_negative_bytes(): +def test_read_negative_bytes() -> None: mis = MemoryInputStream(b"") decoder = BinaryDecoder(mis) @@ -120,14 +124,16 @@ def closed(self) -> bool: def close(self) -> None: pass - def __enter__(self): - pass + def __enter__(self) -> OneByteAtATimeInputStream: + return self - def __exit__(self, exc_type, exc_val, exc_tb): + def __exit__( + self, exctype: Optional[Type[BaseException]], excinst: Optional[BaseException], exctb: Optional[TracebackType] + ) -> None: self.close() -def test_read_single_byte_at_the_time(): +def test_read_single_byte_at_the_time() -> None: decoder = BinaryDecoder(OneByteAtATimeInputStream()) with pytest.raises(ValueError) as exc_info: @@ -136,13 +142,13 @@ def test_read_single_byte_at_the_time(): assert "Read 1 bytes, expected 2 bytes" in str(exc_info.value) -def test_read_float(): +def test_read_float() -> None: mis = MemoryInputStream(b"\x00\x00\x9A\x41") decoder = BinaryDecoder(mis) assert decoder.read_float() == 19.25 -def test_skip_float(): +def test_skip_float() -> None: mis = MemoryInputStream(b"\x00\x00\x9A\x41") decoder = BinaryDecoder(mis) assert mis.tell() == 0 @@ -150,13 +156,13 @@ def test_skip_float(): assert mis.tell() == 4 -def test_read_double(): +def test_read_double() -> None: mis = MemoryInputStream(b"\x00\x00\x00\x00\x00\x40\x33\x40") decoder = BinaryDecoder(mis) assert decoder.read_double() == 19.25 -def test_skip_double(): +def test_skip_double() -> None: mis = MemoryInputStream(b"\x00\x00\x00\x00\x00\x40\x33\x40") decoder = BinaryDecoder(mis) assert mis.tell() == 0 @@ -164,50 +170,50 @@ def test_skip_double(): assert mis.tell() == 8 -def test_read_date(): +def test_read_date() -> None: mis = MemoryInputStream(b"\xBC\x7D") decoder = BinaryDecoder(mis) assert decoder.read_date_from_int() == date(1991, 12, 27) -def test_read_time_millis(): +def test_read_time_millis() -> None: mis = MemoryInputStream(b"\xBC\x7D") decoder = BinaryDecoder(mis) assert decoder.read_time_millis().microsecond == 30000 -def test_read_time_micros(): +def test_read_time_micros() -> None: mis = MemoryInputStream(b"\xBC\x7D") decoder = BinaryDecoder(mis) assert decoder.read_time_micros().microsecond == 8030 -def test_read_timestamp_micros(): +def test_read_timestamp_micros() -> None: mis = MemoryInputStream(b"\xBC\x7D") decoder = BinaryDecoder(mis) assert decoder.read_timestamp_micros() == datetime(1970, 1, 1, 0, 0, 0, 8030) -def test_read_timestamptz_micros(): +def test_read_timestamptz_micros() -> None: mis = MemoryInputStream(b"\xBC\x7D") decoder = BinaryDecoder(mis) assert decoder.read_timestamptz_micros() == datetime(1970, 1, 1, 0, 0, 0, 8030, tzinfo=timezone.utc) -def test_read_bytes(): +def test_read_bytes() -> None: mis = MemoryInputStream(b"\x08\x01\x02\x03\x04") decoder = BinaryDecoder(mis) actual = decoder.read_bytes() assert actual == b"\x01\x02\x03\x04" -def test_read_utf8(): +def test_read_utf8() -> None: mis = MemoryInputStream(b"\x04\x76\x6F") decoder = BinaryDecoder(mis) assert decoder.read_utf8() == "vo" -def test_skip_utf8(): +def test_skip_utf8() -> None: mis = MemoryInputStream(b"\x04\x76\x6F") decoder = BinaryDecoder(mis) assert mis.tell() == 0 @@ -215,7 +221,7 @@ def test_skip_utf8(): assert mis.tell() == 3 -def test_read_int_as_float(): +def test_read_int_as_float() -> None: mis = MemoryInputStream(b"\x00\x00\x9A\x41") decoder = BinaryDecoder(mis) reader = promote(FloatType(), DoubleType()) diff --git a/tests/avro/test_file.py b/tests/avro/test_file.py index 177f7eb98d..fc46963375 100644 --- a/tests/avro/test_file.py +++ b/tests/avro/test_file.py @@ -20,17 +20,17 @@ from pyiceberg.avro.file import AvroFileHeader -def get_deflate_compressor(): +def get_deflate_compressor() -> None: header = AvroFileHeader(bytes(0), {"avro.codec": "deflate"}, bytes(16)) assert header.compression_codec() == DeflateCodec -def get_null_compressor(): +def get_null_compressor() -> None: header = AvroFileHeader(bytes(0), {"avro.codec": "null"}, bytes(16)) assert header.compression_codec() is None -def test_unknown_codec(): +def test_unknown_codec() -> None: header = AvroFileHeader(bytes(0), {"avro.codec": "unknown"}, bytes(16)) with pytest.raises(ValueError) as exc_info: @@ -39,7 +39,7 @@ def test_unknown_codec(): assert "Unsupported codec: unknown" in str(exc_info.value) -def test_missing_schema(): +def test_missing_schema() -> None: header = AvroFileHeader(bytes(0), {}, bytes(16)) with pytest.raises(ValueError) as exc_info: diff --git a/tests/avro/test_reader.py b/tests/avro/test_reader.py index ac0d001abc..da1613be95 100644 --- a/tests/avro/test_reader.py +++ b/tests/avro/test_reader.py @@ -61,7 +61,7 @@ from tests.io.test_io import LocalInputFile -def test_read_header(generated_manifest_entry_file: str, iceberg_manifest_entry_schema: Schema): +def test_read_header(generated_manifest_entry_file: str, iceberg_manifest_entry_schema: Schema) -> None: with AvroFile(LocalInputFile(generated_manifest_entry_file)) as reader: header = reader._read_header() @@ -254,7 +254,7 @@ def test_read_header(generated_manifest_entry_file: str, iceberg_manifest_entry_ assert header.get_schema() == iceberg_manifest_entry_schema -def test_read_manifest_entry_file(generated_manifest_entry_file: str): +def test_read_manifest_entry_file(generated_manifest_entry_file: str) -> None: with AvroFile(LocalInputFile(generated_manifest_entry_file)) as reader: # Consume the generator records = list(reader) @@ -375,7 +375,7 @@ def test_read_manifest_entry_file(generated_manifest_entry_file: str): ) -def test_read_manifest_file_file(generated_manifest_file_file: str): +def test_read_manifest_file_file(generated_manifest_file_file: str) -> None: with AvroFile(LocalInputFile(generated_manifest_file_file)) as reader: # Consume the generator records = list(reader) @@ -400,7 +400,7 @@ def test_read_manifest_file_file(generated_manifest_file_file: str): assert actual == expected -def test_null_list_convert_pos_to_dict(): +def test_null_list_convert_pos_to_dict() -> None: data = _convert_pos_to_dict( Schema( NestedField(name="field", field_id=1, field_type=ListType(element_id=2, element=StringType(), element_required=False)) @@ -410,7 +410,7 @@ def test_null_list_convert_pos_to_dict(): assert data["field"] is None -def test_null_dict_convert_pos_to_dict(): +def test_null_dict_convert_pos_to_dict() -> None: data = _convert_pos_to_dict( Schema( NestedField( @@ -424,7 +424,7 @@ def test_null_dict_convert_pos_to_dict(): assert data["field"] is None -def test_null_struct_convert_pos_to_dict(): +def test_null_struct_convert_pos_to_dict() -> None: data = _convert_pos_to_dict( Schema( NestedField( @@ -441,59 +441,59 @@ def test_null_struct_convert_pos_to_dict(): assert data["field"] is None -def test_fixed_reader(): +def test_fixed_reader() -> None: assert primitive_reader(FixedType(22)) == FixedReader(22) -def test_decimal_reader(): +def test_decimal_reader() -> None: assert primitive_reader(DecimalType(19, 25)) == DecimalReader(19, 25) -def test_boolean_reader(): +def test_boolean_reader() -> None: assert primitive_reader(BooleanType()) == BooleanReader() -def test_integer_reader(): +def test_integer_reader() -> None: assert primitive_reader(IntegerType()) == IntegerReader() -def test_long_reader(): +def test_long_reader() -> None: assert primitive_reader(LongType()) == IntegerReader() -def test_float_reader(): +def test_float_reader() -> None: assert primitive_reader(FloatType()) == FloatReader() -def test_double_reader(): +def test_double_reader() -> None: assert primitive_reader(DoubleType()) == DoubleReader() -def test_date_reader(): +def test_date_reader() -> None: assert primitive_reader(DateType()) == DateReader() -def test_time_reader(): +def test_time_reader() -> None: assert primitive_reader(TimeType()) == TimeReader() -def test_timestamp_reader(): +def test_timestamp_reader() -> None: assert primitive_reader(TimestampType()) == TimestampReader() -def test_timestamptz_reader(): +def test_timestamptz_reader() -> None: assert primitive_reader(TimestamptzType()) == TimestamptzReader() -def test_string_reader(): +def test_string_reader() -> None: assert primitive_reader(StringType()) == StringReader() -def test_binary_reader(): +def test_binary_reader() -> None: assert primitive_reader(BinaryType()) == BinaryReader() -def test_unknown_type(): +def test_unknown_type() -> None: class UnknownType(PrimitiveType): __root__ = "UnknownType" diff --git a/tests/avro/test_resolver.py b/tests/avro/test_resolver.py index 8c1309d620..853be4e448 100644 --- a/tests/avro/test_resolver.py +++ b/tests/avro/test_resolver.py @@ -42,7 +42,7 @@ ) -def test_resolver(): +def test_resolver() -> None: write_schema = Schema( NestedField(1, "id", LongType()), NestedField(2, "data", StringType()), @@ -90,7 +90,7 @@ def test_resolver(): ) -def test_resolver_new_required_field(): +def test_resolver_new_required_field() -> None: write_schema = Schema( NestedField(1, "id", LongType()), schema_id=1, @@ -107,7 +107,7 @@ def test_resolver_new_required_field(): assert "2: data: required string is non-optional, and not part of the file schema" in str(exc_info.value) -def test_resolver_invalid_evolution(): +def test_resolver_invalid_evolution() -> None: write_schema = Schema( NestedField(1, "id", LongType()), schema_id=1, @@ -123,7 +123,7 @@ def test_resolver_invalid_evolution(): assert "Cannot promote long to double" in str(exc_info.value) -def test_resolver_promotion_string_to_binary(): +def test_resolver_promotion_string_to_binary() -> None: write_schema = Schema( NestedField(1, "id", StringType()), schema_id=1, @@ -135,7 +135,7 @@ def test_resolver_promotion_string_to_binary(): resolve(write_schema, read_schema) -def test_resolver_promotion_binary_to_string(): +def test_resolver_promotion_binary_to_string() -> None: write_schema = Schema( NestedField(1, "id", BinaryType()), schema_id=1, @@ -147,7 +147,7 @@ def test_resolver_promotion_binary_to_string(): resolve(write_schema, read_schema) -def test_resolver_change_type(): +def test_resolver_change_type() -> None: write_schema = Schema( NestedField(1, "properties", ListType(2, StringType())), schema_id=1, @@ -163,61 +163,61 @@ def test_resolver_change_type(): assert "File/read schema are not aligned for list, got map" in str(exc_info.value) -def test_promote_int_to_long(): +def test_promote_int_to_long() -> None: assert promote(IntegerType(), LongType()) == IntegerReader() -def test_promote_float_to_double(): +def test_promote_float_to_double() -> None: # We should still read floats, because it is encoded in 4 bytes assert promote(FloatType(), DoubleType()) == FloatReader() -def test_promote_decimal_to_decimal(): +def test_promote_decimal_to_decimal() -> None: # DecimalType(P, S) to DecimalType(P2, S) where P2 > P assert promote(DecimalType(19, 25), DecimalType(22, 25)) == DecimalReader(22, 25) -def test_struct_not_aligned(): +def test_struct_not_aligned() -> None: with pytest.raises(ResolveException): assert promote(StructType(), StringType()) -def test_map_not_aligned(): +def test_map_not_aligned() -> None: with pytest.raises(ResolveException): assert promote(MapType(1, StringType(), 2, IntegerType()), StringType()) -def test_primitive_not_aligned(): +def test_primitive_not_aligned() -> None: with pytest.raises(ResolveException): assert promote(IntegerType(), MapType(1, StringType(), 2, IntegerType())) -def test_integer_not_aligned(): +def test_integer_not_aligned() -> None: with pytest.raises(ResolveException): assert promote(IntegerType(), StringType()) -def test_float_not_aligned(): +def test_float_not_aligned() -> None: with pytest.raises(ResolveException): assert promote(FloatType(), StringType()) -def test_string_not_aligned(): +def test_string_not_aligned() -> None: with pytest.raises(ResolveException): assert promote(StringType(), FloatType()) -def test_binary_not_aligned(): +def test_binary_not_aligned() -> None: with pytest.raises(ResolveException): assert promote(BinaryType(), FloatType()) -def test_decimal_not_aligned(): +def test_decimal_not_aligned() -> None: with pytest.raises(ResolveException): assert promote(DecimalType(22, 19), StringType()) -def test_promote_decimal_to_decimal_reduce_precision(): +def test_promote_decimal_to_decimal_reduce_precision() -> None: # DecimalType(P, S) to DecimalType(P2, S) where P2 > P with pytest.raises(ResolveException) as exc_info: _ = promote(DecimalType(19, 25), DecimalType(10, 25)) == DecimalReader(22, 25) diff --git a/tests/catalog/integration_test_glue.py b/tests/catalog/integration_test_glue.py index 70ac3a00a4..76b05b2b4d 100644 --- a/tests/catalog/integration_test_glue.py +++ b/tests/catalog/integration_test_glue.py @@ -16,11 +16,13 @@ # under the License. import os +from typing import Generator, Optional import boto3 import pytest from botocore.exceptions import ClientError +from pyiceberg.catalog import Catalog from pyiceberg.catalog.glue import GlueCatalog from pyiceberg.exceptions import ( NamespaceAlreadyExistsError, @@ -43,7 +45,7 @@ LIST_TEST_NUMBER = 2 -def get_bucket_name(): +def get_bucket_name() -> str: """ Set the environment variable AWS_TEST_BUCKET for a default bucket to test """ @@ -53,7 +55,7 @@ def get_bucket_name(): return bucket_name -def get_s3_path(bucket_name, database_name=None, table_name=None): +def get_s3_path(bucket_name: str, database_name: Optional[str] = None, table_name: Optional[str] = None) -> str: result_path = f"s3://{bucket_name}" if database_name is not None: result_path += f"/{database_name}.db" @@ -64,19 +66,19 @@ def get_s3_path(bucket_name, database_name=None, table_name=None): @pytest.fixture(name="s3", scope="module") -def fixture_s3_client(): +def fixture_s3_client() -> boto3.client: yield boto3.client("s3") @pytest.fixture(name="glue", scope="module") -def fixture_glue_client(): +def fixture_glue_client() -> boto3.client: yield boto3.client("glue") -def clean_up(test_catalog): +def clean_up(test_catalog: Catalog) -> None: """Clean all databases and tables created during the integration test""" - for database_name in test_catalog.list_namespaces(): - database_name = database_name[0] + for database_tuple in test_catalog.list_namespaces(): + database_name = database_tuple[0] if "my_iceberg_database-" in database_name: for identifier in test_catalog.list_tables(database_name): test_catalog.purge_table(identifier) @@ -84,14 +86,14 @@ def clean_up(test_catalog): @pytest.fixture(name="test_catalog", scope="module") -def fixture_test_catalog(): +def fixture_test_catalog() -> Generator[Catalog, None, None]: """The pre- and post-setting of aws integration test""" test_catalog = GlueCatalog("glue", warehouse=get_s3_path(get_bucket_name())) yield test_catalog clean_up(test_catalog) -def test_create_table(test_catalog, s3, table_schema_nested: Schema): +def test_create_table(test_catalog: Catalog, s3: boto3.client, table_schema_nested: Schema) -> None: table_name = get_random_table_name() database_name = get_random_database_name() identifier = (database_name, table_name) @@ -103,7 +105,7 @@ def test_create_table(test_catalog, s3, table_schema_nested: Schema): s3.head_object(Bucket=get_bucket_name(), Key=metadata_location) -def test_create_table_with_invalid_location(table_schema_nested: Schema): +def test_create_table_with_invalid_location(table_schema_nested: Schema) -> None: table_name = get_random_table_name() database_name = get_random_database_name() identifier = (database_name, table_name) @@ -114,7 +116,7 @@ def test_create_table_with_invalid_location(table_schema_nested: Schema): test_catalog_no_warehouse.drop_namespace(database_name) -def test_create_table_with_default_location(test_catalog, s3, table_schema_nested: Schema): +def test_create_table_with_default_location(test_catalog: Catalog, s3: boto3.client, table_schema_nested: Schema) -> None: table_name = get_random_table_name() database_name = get_random_database_name() identifier = (database_name, table_name) @@ -126,14 +128,14 @@ def test_create_table_with_default_location(test_catalog, s3, table_schema_neste s3.head_object(Bucket=get_bucket_name(), Key=metadata_location) -def test_create_table_with_invalid_database(test_catalog, table_schema_nested: Schema): +def test_create_table_with_invalid_database(test_catalog: Catalog, table_schema_nested: Schema) -> None: table_name = get_random_table_name() identifier = ("invalid", table_name) with pytest.raises(NoSuchNamespaceError): test_catalog.create_table(identifier, table_schema_nested) -def test_create_duplicated_table(test_catalog, table_schema_nested: Schema): +def test_create_duplicated_table(test_catalog: Catalog, table_schema_nested: Schema) -> None: table_name = get_random_table_name() database_name = get_random_database_name() test_catalog.create_namespace(database_name) @@ -142,7 +144,7 @@ def test_create_duplicated_table(test_catalog, table_schema_nested: Schema): test_catalog.create_table((database_name, table_name), table_schema_nested) -def test_load_table(test_catalog, table_schema_nested): +def test_load_table(test_catalog: Catalog, table_schema_nested: Schema) -> None: table_name = get_random_table_name() database_name = get_random_database_name() identifier = (database_name, table_name) @@ -154,7 +156,7 @@ def test_load_table(test_catalog, table_schema_nested): assert table.metadata == loaded_table.metadata -def test_list_tables(test_catalog, table_schema_nested): +def test_list_tables(test_catalog: Catalog, table_schema_nested: Schema) -> None: test_tables = get_random_tables(LIST_TEST_NUMBER) database_name = get_random_database_name() test_catalog.create_namespace(database_name) @@ -166,7 +168,7 @@ def test_list_tables(test_catalog, table_schema_nested): assert (database_name, table_name) in identifier_list -def test_rename_table(test_catalog, s3, table_schema_nested): +def test_rename_table(test_catalog: Catalog, s3: boto3.client, table_schema_nested: Schema) -> None: table_name = get_random_table_name() database_name = get_random_database_name() new_database_name = get_random_database_name() @@ -187,7 +189,7 @@ def test_rename_table(test_catalog, s3, table_schema_nested): test_catalog.load_table(identifier) -def test_drop_table(test_catalog, table_schema_nested): +def test_drop_table(test_catalog: Catalog, table_schema_nested: Schema) -> None: table_name = get_random_table_name() database_name = get_random_database_name() identifier = (database_name, table_name) @@ -199,7 +201,7 @@ def test_drop_table(test_catalog, table_schema_nested): test_catalog.load_table(identifier) -def test_purge_table(test_catalog, s3, table_schema_nested): +def test_purge_table(test_catalog: Catalog, s3: boto3.client, table_schema_nested: Schema) -> None: table_name = get_random_table_name() database_name = get_random_database_name() identifier = (database_name, table_name) @@ -216,20 +218,20 @@ def test_purge_table(test_catalog, s3, table_schema_nested): s3.head_object(Bucket=get_bucket_name(), Key=metadata_location) -def test_create_namespace(test_catalog): +def test_create_namespace(test_catalog: Catalog) -> None: database_name = get_random_database_name() test_catalog.create_namespace(database_name) assert (database_name,) in test_catalog.list_namespaces() -def test_create_duplicate_namespace(test_catalog): +def test_create_duplicate_namespace(test_catalog: Catalog) -> None: database_name = get_random_database_name() test_catalog.create_namespace(database_name) with pytest.raises(NamespaceAlreadyExistsError): test_catalog.create_namespace(database_name) -def test_create_namespace_with_comment_and_location(test_catalog): +def test_create_namespace_with_comment_and_location(test_catalog: Catalog) -> None: database_name = get_random_database_name() test_location = get_s3_path(get_bucket_name(), database_name) test_properties = { @@ -244,7 +246,7 @@ def test_create_namespace_with_comment_and_location(test_catalog): assert properties["location"] == test_location -def test_list_namespaces(test_catalog): +def test_list_namespaces(test_catalog: Catalog) -> None: database_list = get_random_databases(LIST_TEST_NUMBER) for database_name in database_list: test_catalog.create_namespace(database_name) @@ -254,7 +256,7 @@ def test_list_namespaces(test_catalog): assert len(test_catalog.list_namespaces(list(database_list)[0])) == 0 -def test_drop_namespace(test_catalog, table_schema_nested: Schema): +def test_drop_namespace(test_catalog: Catalog, table_schema_nested: Schema) -> None: table_name = get_random_table_name() database_name = get_random_database_name() test_catalog.create_namespace(database_name) @@ -267,7 +269,7 @@ def test_drop_namespace(test_catalog, table_schema_nested: Schema): assert (database_name,) not in test_catalog.list_namespaces() -def test_load_namespace_properties(test_catalog): +def test_load_namespace_properties(test_catalog: Catalog) -> None: warehouse_location = get_s3_path(get_bucket_name()) database_name = get_random_database_name() test_properties = { @@ -285,14 +287,14 @@ def test_load_namespace_properties(test_catalog): assert v == test_properties[k] -def test_load_empty_namespace_properties(test_catalog): +def test_load_empty_namespace_properties(test_catalog: Catalog) -> None: database_name = get_random_database_name() test_catalog.create_namespace(database_name) listed_properties = test_catalog.load_namespace_properties(database_name) assert listed_properties == {} -def test_load_default_namespace_properties(test_catalog, glue): +def test_load_default_namespace_properties(test_catalog: Catalog, glue: boto3.client) -> None: database_name = get_random_database_name() # simulate creating database with default settings through AWS Glue Web Console glue.create_database(DatabaseInput={"Name": database_name}) @@ -300,7 +302,7 @@ def test_load_default_namespace_properties(test_catalog, glue): assert listed_properties == {} -def test_update_namespace_properties(test_catalog): +def test_update_namespace_properties(test_catalog: Catalog) -> None: warehouse_location = get_s3_path(get_bucket_name()) database_name = get_random_database_name() test_properties = { diff --git a/tests/catalog/test_base.py b/tests/catalog/test_base.py index 0b7c294fe6..ab9557448e 100644 --- a/tests/catalog/test_base.py +++ b/tests/catalog/test_base.py @@ -54,7 +54,7 @@ class InMemoryCatalog(Catalog): __tables: Dict[Identifier, Table] __namespaces: Dict[Identifier, Properties] - def __init__(self, name: str, **properties: str): + def __init__(self, name: str, **properties: str) -> None: super().__init__(name, **properties) self.__tables = {} self.__namespaces = {} @@ -210,7 +210,7 @@ def given_catalog_has_a_table(catalog: InMemoryCatalog) -> Table: ) -def test_namespace_from_tuple(): +def test_namespace_from_tuple() -> None: # Given identifier = ("com", "organization", "department", "my_table") # When @@ -219,7 +219,7 @@ def test_namespace_from_tuple(): assert namespace_from == ("com", "organization", "department") -def test_namespace_from_str(): +def test_namespace_from_str() -> None: # Given identifier = "com.organization.department.my_table" # When @@ -228,7 +228,7 @@ def test_namespace_from_str(): assert namespace_from == ("com", "organization", "department") -def test_name_from_tuple(): +def test_name_from_tuple() -> None: # Given identifier = ("com", "organization", "department", "my_table") # When @@ -237,7 +237,7 @@ def test_name_from_tuple(): assert name_from == "my_table" -def test_name_from_str(): +def test_name_from_str() -> None: # Given identifier = "com.organization.department.my_table" # When @@ -246,7 +246,7 @@ def test_name_from_str(): assert name_from == "my_table" -def test_create_table(catalog: InMemoryCatalog): +def test_create_table(catalog: InMemoryCatalog) -> None: table = catalog.create_table( identifier=TEST_TABLE_IDENTIFIER, schema=TEST_TABLE_SCHEMA, @@ -257,7 +257,7 @@ def test_create_table(catalog: InMemoryCatalog): assert catalog.load_table(TEST_TABLE_IDENTIFIER) == table -def test_create_table_raises_error_when_table_already_exists(catalog: InMemoryCatalog): +def test_create_table_raises_error_when_table_already_exists(catalog: InMemoryCatalog) -> None: # Given given_catalog_has_a_table(catalog) # When @@ -268,7 +268,7 @@ def test_create_table_raises_error_when_table_already_exists(catalog: InMemoryCa ) -def test_load_table(catalog: InMemoryCatalog): +def test_load_table(catalog: InMemoryCatalog) -> None: # Given given_table = given_catalog_has_a_table(catalog) # When @@ -277,12 +277,12 @@ def test_load_table(catalog: InMemoryCatalog): assert table == given_table -def test_table_raises_error_on_table_not_found(catalog: InMemoryCatalog): +def test_table_raises_error_on_table_not_found(catalog: InMemoryCatalog) -> None: with pytest.raises(NoSuchTableError, match=NO_SUCH_TABLE_ERROR): catalog.load_table(TEST_TABLE_IDENTIFIER) -def test_drop_table(catalog: InMemoryCatalog): +def test_drop_table(catalog: InMemoryCatalog) -> None: # Given given_catalog_has_a_table(catalog) # When @@ -292,12 +292,12 @@ def test_drop_table(catalog: InMemoryCatalog): catalog.load_table(TEST_TABLE_IDENTIFIER) -def test_drop_table_that_does_not_exist_raise_error(catalog: InMemoryCatalog): +def test_drop_table_that_does_not_exist_raise_error(catalog: InMemoryCatalog) -> None: with pytest.raises(NoSuchTableError, match=NO_SUCH_TABLE_ERROR): catalog.load_table(TEST_TABLE_IDENTIFIER) -def test_purge_table(catalog: InMemoryCatalog): +def test_purge_table(catalog: InMemoryCatalog) -> None: # Given given_catalog_has_a_table(catalog) # When @@ -307,7 +307,7 @@ def test_purge_table(catalog: InMemoryCatalog): catalog.load_table(TEST_TABLE_IDENTIFIER) -def test_rename_table(catalog: InMemoryCatalog): +def test_rename_table(catalog: InMemoryCatalog) -> None: # Given given_catalog_has_a_table(catalog) @@ -330,7 +330,7 @@ def test_rename_table(catalog: InMemoryCatalog): catalog.load_table(TEST_TABLE_IDENTIFIER) -def test_create_namespace(catalog: InMemoryCatalog): +def test_create_namespace(catalog: InMemoryCatalog) -> None: # When catalog.create_namespace(TEST_TABLE_NAMESPACE, TEST_TABLE_PROPERTIES) @@ -339,7 +339,7 @@ def test_create_namespace(catalog: InMemoryCatalog): assert TEST_TABLE_PROPERTIES == catalog.load_namespace_properties(TEST_TABLE_NAMESPACE) -def test_create_namespace_raises_error_on_existing_namespace(catalog: InMemoryCatalog): +def test_create_namespace_raises_error_on_existing_namespace(catalog: InMemoryCatalog) -> None: # Given catalog.create_namespace(TEST_TABLE_NAMESPACE, TEST_TABLE_PROPERTIES) # When @@ -347,12 +347,12 @@ def test_create_namespace_raises_error_on_existing_namespace(catalog: InMemoryCa catalog.create_namespace(TEST_TABLE_NAMESPACE, TEST_TABLE_PROPERTIES) -def test_get_namespace_metadata_raises_error_when_namespace_does_not_exist(catalog: InMemoryCatalog): +def test_get_namespace_metadata_raises_error_when_namespace_does_not_exist(catalog: InMemoryCatalog) -> None: with pytest.raises(NoSuchNamespaceError, match=NO_SUCH_NAMESPACE_ERROR): catalog.load_namespace_properties(TEST_TABLE_NAMESPACE) -def test_list_namespaces(catalog: InMemoryCatalog): +def test_list_namespaces(catalog: InMemoryCatalog) -> None: # Given catalog.create_namespace(TEST_TABLE_NAMESPACE, TEST_TABLE_PROPERTIES) # When @@ -361,7 +361,7 @@ def test_list_namespaces(catalog: InMemoryCatalog): assert TEST_TABLE_NAMESPACE in namespaces -def test_drop_namespace(catalog: InMemoryCatalog): +def test_drop_namespace(catalog: InMemoryCatalog) -> None: # Given catalog.create_namespace(TEST_TABLE_NAMESPACE, TEST_TABLE_PROPERTIES) # When @@ -370,12 +370,12 @@ def test_drop_namespace(catalog: InMemoryCatalog): assert TEST_TABLE_NAMESPACE not in catalog.list_namespaces() -def test_drop_namespace_raises_error_when_namespace_does_not_exist(catalog: InMemoryCatalog): +def test_drop_namespace_raises_error_when_namespace_does_not_exist(catalog: InMemoryCatalog) -> None: with pytest.raises(NoSuchNamespaceError, match=NO_SUCH_NAMESPACE_ERROR): catalog.drop_namespace(TEST_TABLE_NAMESPACE) -def test_drop_namespace_raises_error_when_namespace_not_empty(catalog: InMemoryCatalog): +def test_drop_namespace_raises_error_when_namespace_not_empty(catalog: InMemoryCatalog) -> None: # Given given_catalog_has_a_table(catalog) # When @@ -383,7 +383,7 @@ def test_drop_namespace_raises_error_when_namespace_not_empty(catalog: InMemoryC catalog.drop_namespace(TEST_TABLE_NAMESPACE) -def test_list_tables(catalog: InMemoryCatalog): +def test_list_tables(catalog: InMemoryCatalog) -> None: # Given given_catalog_has_a_table(catalog) # When @@ -393,7 +393,7 @@ def test_list_tables(catalog: InMemoryCatalog): assert TEST_TABLE_IDENTIFIER in tables -def test_list_tables_under_a_namespace(catalog: InMemoryCatalog): +def test_list_tables_under_a_namespace(catalog: InMemoryCatalog) -> None: # Given given_catalog_has_a_table(catalog) new_namespace = ("new", "namespace") @@ -407,7 +407,7 @@ def test_list_tables_under_a_namespace(catalog: InMemoryCatalog): assert new_namespace_tables == [] -def test_update_namespace_metadata(catalog: InMemoryCatalog): +def test_update_namespace_metadata(catalog: InMemoryCatalog) -> None: # Given catalog.create_namespace(TEST_TABLE_NAMESPACE, TEST_TABLE_PROPERTIES) @@ -421,7 +421,7 @@ def test_update_namespace_metadata(catalog: InMemoryCatalog): assert summary == PropertiesUpdateSummary(removed=[], updated=["key3", "key4"], missing=[]) -def test_update_namespace_metadata_removals(catalog: InMemoryCatalog): +def test_update_namespace_metadata_removals(catalog: InMemoryCatalog) -> None: # Given catalog.create_namespace(TEST_TABLE_NAMESPACE, TEST_TABLE_PROPERTIES) @@ -437,6 +437,6 @@ def test_update_namespace_metadata_removals(catalog: InMemoryCatalog): assert summary == PropertiesUpdateSummary(removed=["key1"], updated=["key3", "key4"], missing=[]) -def test_update_namespace_metadata_raises_error_when_namespace_does_not_exist(catalog: InMemoryCatalog): +def test_update_namespace_metadata_raises_error_when_namespace_does_not_exist(catalog: InMemoryCatalog) -> None: with pytest.raises(NoSuchNamespaceError, match=NO_SUCH_NAMESPACE_ERROR): catalog.update_namespace_properties(TEST_TABLE_NAMESPACE, updates=TEST_TABLE_PROPERTIES) diff --git a/tests/catalog/test_glue.py b/tests/catalog/test_glue.py index ccdeaff033..af12435415 100644 --- a/tests/catalog/test_glue.py +++ b/tests/catalog/test_glue.py @@ -17,6 +17,7 @@ import random import re import string +from typing import Set import pytest from moto import mock_glue @@ -31,6 +32,7 @@ NoSuchTableError, TableAlreadyExistsError, ) +from pyiceberg.schema import Schema BUCKET_NAME = "test_bucket" RANDOM_LENGTH = 20 @@ -43,39 +45,35 @@ ) -def get_random_table_name(): +def get_random_table_name() -> str: prefix = "my_iceberg_table-" random_tag = "".join(random.choice(string.ascii_letters) for _ in range(RANDOM_LENGTH)) return (prefix + random_tag).lower() -def get_random_tables(n): - result = set() - for _ in range(n): - result.add(get_random_table_name()) - return result +def get_random_tables(n: int) -> Set[str]: + return {get_random_table_name() for _ in range(n)} -def get_random_database_name(): +def get_random_database_name() -> str: prefix = "my_iceberg_database-" random_tag = "".join(random.choice(string.ascii_letters) for _ in range(RANDOM_LENGTH)) return (prefix + random_tag).lower() -def get_random_databases(n): - result = set() - for _ in range(n): - result.add(get_random_database_name()) - return result +def get_random_databases(n: int) -> Set[str]: + return {get_random_database_name() for _ in range(n)} @pytest.fixture(name="_bucket_initialize") -def fixture_s3_bucket(_s3): +def fixture_s3_bucket(_s3) -> None: # type: ignore _s3.create_bucket(Bucket=BUCKET_NAME) @mock_glue -def test_create_table_with_database_location(_bucket_initialize, _patch_aiobotocore, table_schema_nested): +def test_create_table_with_database_location( + _bucket_initialize: None, _patch_aiobotocore: None, table_schema_nested: Schema +) -> None: database_name = get_random_database_name() table_name = get_random_table_name() identifier = (database_name, table_name) @@ -87,7 +85,9 @@ def test_create_table_with_database_location(_bucket_initialize, _patch_aiobotoc @mock_glue -def test_create_table_with_default_warehouse(_bucket_initialize, _patch_aiobotocore, table_schema_nested): +def test_create_table_with_default_warehouse( + _bucket_initialize: None, _patch_aiobotocore: None, table_schema_nested: Schema +) -> None: database_name = get_random_database_name() table_name = get_random_table_name() identifier = (database_name, table_name) @@ -99,7 +99,9 @@ def test_create_table_with_default_warehouse(_bucket_initialize, _patch_aiobotoc @mock_glue -def test_create_table_with_given_location(_bucket_initialize, _patch_aiobotocore, table_schema_nested): +def test_create_table_with_given_location( + _bucket_initialize: None, _patch_aiobotocore: None, table_schema_nested: Schema +) -> None: database_name = get_random_database_name() table_name = get_random_table_name() identifier = (database_name, table_name) @@ -113,7 +115,7 @@ def test_create_table_with_given_location(_bucket_initialize, _patch_aiobotocore @mock_glue -def test_create_table_with_no_location(_bucket_initialize, _patch_aiobotocore, table_schema_nested): +def test_create_table_with_no_location(_bucket_initialize: None, _patch_aiobotocore: None, table_schema_nested: Schema) -> None: database_name = get_random_database_name() table_name = get_random_table_name() identifier = (database_name, table_name) @@ -124,7 +126,7 @@ def test_create_table_with_no_location(_bucket_initialize, _patch_aiobotocore, t @mock_glue -def test_create_table_with_strips(_bucket_initialize, _patch_aiobotocore, table_schema_nested): +def test_create_table_with_strips(_bucket_initialize: None, _patch_aiobotocore: None, table_schema_nested: Schema) -> None: database_name = get_random_database_name() table_name = get_random_table_name() identifier = (database_name, table_name) @@ -144,7 +146,7 @@ def test_create_table_with_strips(_bucket_initialize, _patch_aiobotocore, table_ @mock_glue -def test_create_table_with_no_database(_bucket_initialize, _patch_aiobotocore, table_schema_nested): +def test_create_table_with_no_database(_bucket_initialize: None, _patch_aiobotocore: None, table_schema_nested: Schema) -> None: database_name = get_random_database_name() table_name = get_random_table_name() identifier = (database_name, table_name) @@ -154,7 +156,7 @@ def test_create_table_with_no_database(_bucket_initialize, _patch_aiobotocore, t @mock_glue -def test_create_duplicated_table(_bucket_initialize, _patch_aiobotocore, table_schema_nested): +def test_create_duplicated_table(_bucket_initialize: None, _patch_aiobotocore: None, table_schema_nested: Schema) -> None: database_name = get_random_database_name() table_name = get_random_table_name() identifier = (database_name, table_name) @@ -166,7 +168,7 @@ def test_create_duplicated_table(_bucket_initialize, _patch_aiobotocore, table_s @mock_glue -def test_load_table(_bucket_initialize, _patch_aiobotocore, table_schema_nested): +def test_load_table(_bucket_initialize: None, _patch_aiobotocore: None, table_schema_nested: Schema) -> None: database_name = get_random_database_name() table_name = get_random_table_name() identifier = (database_name, table_name) @@ -179,7 +181,7 @@ def test_load_table(_bucket_initialize, _patch_aiobotocore, table_schema_nested) @mock_glue -def test_load_non_exist_table(_bucket_initialize, _patch_aiobotocore): +def test_load_non_exist_table(_bucket_initialize: None, _patch_aiobotocore: None) -> None: database_name = get_random_database_name() table_name = get_random_table_name() identifier = (database_name, table_name) @@ -190,7 +192,7 @@ def test_load_non_exist_table(_bucket_initialize, _patch_aiobotocore): @mock_glue -def test_drop_table(_bucket_initialize, _patch_aiobotocore, table_schema_nested): +def test_drop_table(_bucket_initialize: None, _patch_aiobotocore: None, table_schema_nested: Schema) -> None: database_name = get_random_database_name() table_name = get_random_table_name() identifier = (database_name, table_name) @@ -206,7 +208,7 @@ def test_drop_table(_bucket_initialize, _patch_aiobotocore, table_schema_nested) @mock_glue -def test_drop_non_exist_table(_bucket_initialize, _patch_aiobotocore): +def test_drop_non_exist_table(_bucket_initialize: None, _patch_aiobotocore: None) -> None: database_name = get_random_database_name() table_name = get_random_table_name() identifier = (database_name, table_name) @@ -216,7 +218,7 @@ def test_drop_non_exist_table(_bucket_initialize, _patch_aiobotocore): @mock_glue -def test_rename_table(_bucket_initialize, _patch_aiobotocore, table_schema_nested): +def test_rename_table(_bucket_initialize: None, _patch_aiobotocore: None, table_schema_nested: Schema) -> None: database_name = get_random_database_name() table_name = get_random_table_name() new_table_name = get_random_table_name() @@ -238,7 +240,7 @@ def test_rename_table(_bucket_initialize, _patch_aiobotocore, table_schema_neste @mock_glue -def test_rename_table_no_params(_glue, _bucket_initialize, _patch_aiobotocore): +def test_rename_table_no_params(_glue, _bucket_initialize: None, _patch_aiobotocore: None) -> None: # type: ignore database_name = get_random_database_name() new_database_name = get_random_database_name() table_name = get_random_table_name() @@ -257,7 +259,7 @@ def test_rename_table_no_params(_glue, _bucket_initialize, _patch_aiobotocore): @mock_glue -def test_rename_non_iceberg_table(_glue, _bucket_initialize, _patch_aiobotocore): +def test_rename_non_iceberg_table(_glue, _bucket_initialize: None, _patch_aiobotocore: None) -> None: # type: ignore database_name = get_random_database_name() new_database_name = get_random_database_name() table_name = get_random_table_name() @@ -280,7 +282,7 @@ def test_rename_non_iceberg_table(_glue, _bucket_initialize, _patch_aiobotocore) @mock_glue -def test_list_tables(_bucket_initialize, _patch_aiobotocore, table_schema_nested): +def test_list_tables(_bucket_initialize: None, _patch_aiobotocore: None, table_schema_nested: Schema) -> None: database_name = get_random_database_name() table_list = get_random_tables(LIST_TEST_NUMBER) test_catalog = GlueCatalog("glue", warehouse=f"s3://{BUCKET_NAME}") @@ -293,7 +295,7 @@ def test_list_tables(_bucket_initialize, _patch_aiobotocore, table_schema_nested @mock_glue -def test_list_namespaces(_bucket_initialize, _patch_aiobotocore): +def test_list_namespaces(_bucket_initialize: None, _patch_aiobotocore: None) -> None: database_list = get_random_databases(LIST_TEST_NUMBER) test_catalog = GlueCatalog("glue") for database_name in database_list: @@ -304,7 +306,7 @@ def test_list_namespaces(_bucket_initialize, _patch_aiobotocore): @mock_glue -def test_create_namespace_no_properties(_bucket_initialize, _patch_aiobotocore): +def test_create_namespace_no_properties(_bucket_initialize: None, _patch_aiobotocore: None) -> None: database_name = get_random_database_name() test_catalog = GlueCatalog("glue") test_catalog.create_namespace(namespace=database_name) @@ -316,7 +318,7 @@ def test_create_namespace_no_properties(_bucket_initialize, _patch_aiobotocore): @mock_glue -def test_create_namespace_with_comment_and_location(_bucket_initialize, _patch_aiobotocore): +def test_create_namespace_with_comment_and_location(_bucket_initialize: None, _patch_aiobotocore: None) -> None: database_name = get_random_database_name() test_location = f"s3://{BUCKET_NAME}/{database_name}.db" test_properties = { @@ -334,7 +336,7 @@ def test_create_namespace_with_comment_and_location(_bucket_initialize, _patch_a @mock_glue -def test_create_duplicated_namespace(_bucket_initialize, _patch_aiobotocore): +def test_create_duplicated_namespace(_bucket_initialize: None, _patch_aiobotocore: None) -> None: database_name = get_random_database_name() test_catalog = GlueCatalog("glue") test_catalog.create_namespace(namespace=database_name) @@ -346,7 +348,7 @@ def test_create_duplicated_namespace(_bucket_initialize, _patch_aiobotocore): @mock_glue -def test_drop_namespace(_bucket_initialize, _patch_aiobotocore): +def test_drop_namespace(_bucket_initialize: None, _patch_aiobotocore: None) -> None: database_name = get_random_database_name() test_catalog = GlueCatalog("glue") test_catalog.create_namespace(namespace=database_name) @@ -359,7 +361,7 @@ def test_drop_namespace(_bucket_initialize, _patch_aiobotocore): @mock_glue -def test_drop_non_empty_namespace(_bucket_initialize, _patch_aiobotocore, table_schema_nested): +def test_drop_non_empty_namespace(_bucket_initialize: None, _patch_aiobotocore: None, table_schema_nested: Schema) -> None: database_name = get_random_database_name() table_name = get_random_table_name() identifier = (database_name, table_name) @@ -372,7 +374,7 @@ def test_drop_non_empty_namespace(_bucket_initialize, _patch_aiobotocore, table_ @mock_glue -def test_drop_non_exist_namespace(_bucket_initialize, _patch_aiobotocore): +def test_drop_non_exist_namespace(_bucket_initialize: None, _patch_aiobotocore: None) -> None: database_name = get_random_database_name() test_catalog = GlueCatalog("glue") with pytest.raises(NoSuchNamespaceError): @@ -380,7 +382,7 @@ def test_drop_non_exist_namespace(_bucket_initialize, _patch_aiobotocore): @mock_glue -def test_load_namespace_properties(_bucket_initialize, _patch_aiobotocore): +def test_load_namespace_properties(_bucket_initialize: None, _patch_aiobotocore: None) -> None: database_name = get_random_database_name() test_location = f"s3://{BUCKET_NAME}/{database_name}.db" test_properties = { @@ -399,7 +401,7 @@ def test_load_namespace_properties(_bucket_initialize, _patch_aiobotocore): @mock_glue -def test_load_non_exist_namespace_properties(_bucket_initialize, _patch_aiobotocore): +def test_load_non_exist_namespace_properties(_bucket_initialize: None, _patch_aiobotocore: None) -> None: database_name = get_random_database_name() test_catalog = GlueCatalog("glue") with pytest.raises(NoSuchNamespaceError): @@ -407,7 +409,7 @@ def test_load_non_exist_namespace_properties(_bucket_initialize, _patch_aiobotoc @mock_glue -def test_update_namespace_properties(_bucket_initialize, _patch_aiobotocore): +def test_update_namespace_properties(_bucket_initialize: None, _patch_aiobotocore: None) -> None: database_name = get_random_database_name() test_properties = { "comment": "this is a test description", @@ -433,7 +435,7 @@ def test_update_namespace_properties(_bucket_initialize, _patch_aiobotocore): @mock_glue -def test_load_empty_namespace_properties(_bucket_initialize, _patch_aiobotocore): +def test_load_empty_namespace_properties(_bucket_initialize: None, _patch_aiobotocore: None) -> None: database_name = get_random_database_name() test_catalog = GlueCatalog("glue") test_catalog.create_namespace(database_name) @@ -442,7 +444,7 @@ def test_load_empty_namespace_properties(_bucket_initialize, _patch_aiobotocore) @mock_glue -def test_load_default_namespace_properties(_glue, _bucket_initialize, _patch_aiobotocore): +def test_load_default_namespace_properties(_glue, _bucket_initialize, _patch_aiobotocore) -> None: # type: ignore database_name = get_random_database_name() # simulate creating database with default settings through AWS Glue Web Console _glue.create_database(DatabaseInput={"Name": database_name}) @@ -452,7 +454,7 @@ def test_load_default_namespace_properties(_glue, _bucket_initialize, _patch_aio @mock_glue -def test_update_namespace_properties_overlap_update_removal(_bucket_initialize, _patch_aiobotocore): +def test_update_namespace_properties_overlap_update_removal(_bucket_initialize: None, _patch_aiobotocore: None) -> None: database_name = get_random_database_name() test_properties = { "comment": "this is a test description", diff --git a/tests/catalog/test_hive.py b/tests/catalog/test_hive.py index a838415fbf..f9a47e516c 100644 --- a/tests/catalog/test_hive.py +++ b/tests/catalog/test_hive.py @@ -75,7 +75,7 @@ @pytest.fixture -def hive_table(tmp_path_factory, example_table_metadata_v2: Dict[str, Any]) -> HiveTable: +def hive_table(tmp_path_factory: pytest.TempPathFactory, example_table_metadata_v2: Dict[str, Any]) -> HiveTable: metadata_path = str(tmp_path_factory.mktemp("metadata") / f"{uuid.uuid4()}.metadata.json") metadata = TableMetadataV2(**example_table_metadata_v2) ToOutputFile.table_metadata(metadata, LocalFileIO().new_output(str(metadata_path)), True) @@ -143,7 +143,7 @@ def hive_table(tmp_path_factory, example_table_metadata_v2: Dict[str, Any]) -> H @pytest.fixture(scope="session") -def hive_database(tmp_path_factory) -> HiveDatabase: +def hive_database(tmp_path_factory: pytest.TempPathFactory) -> HiveDatabase: # Pre-create the directory, this has to be done because # of a local FS. Not needed with an actual object store. database_path = tmp_path_factory.mktemp("database") @@ -166,12 +166,12 @@ def hive_database(tmp_path_factory) -> HiveDatabase: ) -def test_no_uri_supplied(): +def test_no_uri_supplied() -> None: with pytest.raises(KeyError): HiveCatalog("production") -def test_check_number_of_namespaces(table_schema_simple: Schema): +def test_check_number_of_namespaces(table_schema_simple: Schema) -> None: catalog = HiveCatalog(HIVE_CATALOG_NAME, uri=HIVE_METASTORE_FAKE_URL) with pytest.raises(ValueError): @@ -188,7 +188,7 @@ def test_check_number_of_namespaces(table_schema_simple: Schema): @patch("time.time", MagicMock(return_value=12345)) -def test_create_table(table_schema_simple: Schema, hive_database: HiveDatabase, hive_table: HiveTable): +def test_create_table(table_schema_simple: Schema, hive_database: HiveDatabase, hive_table: HiveTable) -> None: catalog = HiveCatalog(HIVE_CATALOG_NAME, uri=HIVE_METASTORE_FAKE_URL) catalog._client = MagicMock() @@ -301,7 +301,7 @@ def test_create_table(table_schema_simple: Schema, hive_database: HiveDatabase, assert metadata.dict() == expected.dict() -def test_load_table(hive_table: HiveTable): +def test_load_table(hive_table: HiveTable) -> None: catalog = HiveCatalog(HIVE_CATALOG_NAME, uri=HIVE_METASTORE_FAKE_URL) catalog._client = MagicMock() @@ -401,7 +401,7 @@ def test_load_table(hive_table: HiveTable): assert expected == table.metadata -def test_rename_table_from_does_not_exists(): +def test_rename_table_from_does_not_exists() -> None: catalog = HiveCatalog(HIVE_CATALOG_NAME, uri=HIVE_METASTORE_FAKE_URL) catalog._client = MagicMock() @@ -415,7 +415,7 @@ def test_rename_table_from_does_not_exists(): assert "Table does not exist: does_not_exists" in str(exc_info.value) -def test_rename_table_to_namespace_does_not_exists(): +def test_rename_table_to_namespace_does_not_exists() -> None: catalog = HiveCatalog(HIVE_CATALOG_NAME, uri=HIVE_METASTORE_FAKE_URL) catalog._client = MagicMock() @@ -429,7 +429,7 @@ def test_rename_table_to_namespace_does_not_exists(): assert "Database does not exists: default_does_not_exists" in str(exc_info.value) -def test_drop_database_does_not_empty(): +def test_drop_database_does_not_empty() -> None: catalog = HiveCatalog(HIVE_CATALOG_NAME, uri=HIVE_METASTORE_FAKE_URL) catalog._client = MagicMock() @@ -443,7 +443,7 @@ def test_drop_database_does_not_empty(): assert "Database not_empty is not empty" in str(exc_info.value) -def test_drop_database_does_not_exists(): +def test_drop_database_does_not_exists() -> None: catalog = HiveCatalog(HIVE_CATALOG_NAME, uri=HIVE_METASTORE_FAKE_URL) catalog._client = MagicMock() @@ -455,7 +455,7 @@ def test_drop_database_does_not_exists(): assert "Database does not exists: does_not_exists" in str(exc_info.value) -def test_list_tables(): +def test_list_tables() -> None: catalog = HiveCatalog(HIVE_CATALOG_NAME, uri=HIVE_METASTORE_FAKE_URL) catalog._client = MagicMock() @@ -474,7 +474,7 @@ def test_list_tables(): catalog._client.__enter__().get_all_tables.assert_called_with(db_name="database") -def test_list_namespaces(): +def test_list_namespaces() -> None: catalog = HiveCatalog(HIVE_CATALOG_NAME, uri=HIVE_METASTORE_FAKE_URL) catalog._client = MagicMock() @@ -485,7 +485,7 @@ def test_list_namespaces(): catalog._client.__enter__().get_all_databases.assert_called() -def test_drop_table(): +def test_drop_table() -> None: catalog = HiveCatalog(HIVE_CATALOG_NAME, uri=HIVE_METASTORE_FAKE_URL) catalog._client = MagicMock() @@ -496,7 +496,7 @@ def test_drop_table(): catalog._client.__enter__().drop_table.assert_called_with(dbname="default", name="table", deleteData=False) -def test_drop_table_does_not_exists(): +def test_drop_table_does_not_exists() -> None: catalog = HiveCatalog(HIVE_CATALOG_NAME, uri=HIVE_METASTORE_FAKE_URL) catalog._client = MagicMock() @@ -508,14 +508,14 @@ def test_drop_table_does_not_exists(): assert "Table does not exists: does_not_exists" in str(exc_info.value) -def test_purge_table(): +def test_purge_table() -> None: catalog = HiveCatalog(HIVE_CATALOG_NAME, uri=HIVE_METASTORE_FAKE_URL) with pytest.raises(NotImplementedError): catalog.purge_table(("default", "does_not_exists")) -def test_create_database(): +def test_create_database() -> None: catalog = HiveCatalog(HIVE_CATALOG_NAME, uri=HIVE_METASTORE_FAKE_URL) catalog._client = MagicMock() @@ -542,7 +542,7 @@ def test_create_database(): ) -def test_create_database_already_exists(): +def test_create_database_already_exists() -> None: catalog = HiveCatalog(HIVE_CATALOG_NAME, uri=HIVE_METASTORE_FAKE_URL) catalog._client = MagicMock() @@ -554,7 +554,7 @@ def test_create_database_already_exists(): assert "Database default already exists" in str(exc_info.value) -def test_load_namespace_properties(hive_database: HiveDatabase): +def test_load_namespace_properties(hive_database: HiveDatabase) -> None: catalog = HiveCatalog(HIVE_CATALOG_NAME, uri=HIVE_METASTORE_FAKE_URL) catalog._client = MagicMock() @@ -565,7 +565,7 @@ def test_load_namespace_properties(hive_database: HiveDatabase): catalog._client.__enter__().get_database.assert_called_with(name="default2") -def test_load_namespace_properties_does_not_exists(): +def test_load_namespace_properties_does_not_exists() -> None: catalog = HiveCatalog(HIVE_CATALOG_NAME, uri=HIVE_METASTORE_FAKE_URL) catalog._client = MagicMock() @@ -577,7 +577,7 @@ def test_load_namespace_properties_does_not_exists(): assert "Database does not exists: does_not_exists" in str(exc_info.value) -def test_update_namespace_properties(hive_database: HiveDatabase): +def test_update_namespace_properties(hive_database: HiveDatabase) -> None: catalog = HiveCatalog(HIVE_CATALOG_NAME, uri=HIVE_METASTORE_FAKE_URL) catalog._client = MagicMock() @@ -608,7 +608,7 @@ def test_update_namespace_properties(hive_database: HiveDatabase): ) -def test_update_namespace_properties_namespace_does_not_exists(): +def test_update_namespace_properties_namespace_does_not_exists() -> None: catalog = HiveCatalog(HIVE_CATALOG_NAME, uri=HIVE_METASTORE_FAKE_URL) catalog._client = MagicMock() @@ -620,7 +620,7 @@ def test_update_namespace_properties_namespace_does_not_exists(): assert "Database does not exists: does_not_exists" in str(exc_info.value) -def test_update_namespace_properties_overlap(): +def test_update_namespace_properties_overlap() -> None: catalog = HiveCatalog(HIVE_CATALOG_NAME, uri=HIVE_METASTORE_FAKE_URL) with pytest.raises(ValueError) as exc_info: @@ -629,7 +629,7 @@ def test_update_namespace_properties_overlap(): assert "Updates and deletes have an overlap: {'a'}" in str(exc_info.value) -def test_construct_hive_storage_descriptor_simple(table_schema_simple: Schema): +def test_construct_hive_storage_descriptor_simple(table_schema_simple: Schema) -> None: descriptor = _construct_hive_storage_descriptor(table_schema_simple, "s3://") assert descriptor == StorageDescriptor( cols=[ @@ -659,7 +659,7 @@ def test_construct_hive_storage_descriptor_simple(table_schema_simple: Schema): ) -def test_construct_hive_storage_descriptor_nested(table_schema_nested: Schema): +def test_construct_hive_storage_descriptor_nested(table_schema_nested: Schema) -> None: descriptor = _construct_hive_storage_descriptor(table_schema_nested, "s3://") assert descriptor == StorageDescriptor( cols=[ @@ -693,7 +693,7 @@ def test_construct_hive_storage_descriptor_nested(table_schema_nested: Schema): ) -def test_resolve_table_location_warehouse(hive_database: HiveDatabase): +def test_resolve_table_location_warehouse(hive_database: HiveDatabase) -> None: catalog = HiveCatalog(HIVE_CATALOG_NAME, warehouse="/tmp/warehouse/", uri=HIVE_METASTORE_FAKE_URL) # Set this one to None, so we'll fall back to the properties diff --git a/tests/catalog/test_rest.py b/tests/catalog/test_rest.py index cf840713c2..9c1700d97f 100644 --- a/tests/catalog/test_rest.py +++ b/tests/catalog/test_rest.py @@ -60,7 +60,7 @@ @pytest.fixture -def rest_mock(requests_mock: Mocker): +def rest_mock(requests_mock: Mocker) -> Mocker: """Takes the default requests_mock and adds the config endpoint to it This endpoint is called when initializing the rest catalog @@ -73,12 +73,12 @@ def rest_mock(requests_mock: Mocker): return requests_mock -def test_no_uri_supplied(): +def test_no_uri_supplied() -> None: with pytest.raises(KeyError): RestCatalog("production") -def test_token_200(rest_mock: Mocker): +def test_token_200(rest_mock: Mocker) -> None: rest_mock.post( f"{TEST_URI}v1/oauth/tokens", json={ @@ -95,7 +95,7 @@ def test_token_200(rest_mock: Mocker): ) -def test_config_200(requests_mock: Mocker): +def test_config_200(requests_mock: Mocker) -> None: requests_mock.get( f"{TEST_URI}v1/config", json={"defaults": {}, "overrides": {}}, @@ -122,7 +122,7 @@ def test_config_200(requests_mock: Mocker): assert history[1].url == "https://iceberg-test-catalog/v1/config?warehouse=s3%3A%2F%2Fsome-bucket" -def test_token_400(rest_mock: Mocker): +def test_token_400(rest_mock: Mocker) -> None: rest_mock.post( f"{TEST_URI}v1/oauth/tokens", json={"error": "invalid_client", "error_description": "Credentials for key invalid_key do not match"}, @@ -135,7 +135,7 @@ def test_token_400(rest_mock: Mocker): assert str(e.value) == "invalid_client: Credentials for key invalid_key do not match" -def test_token_401(rest_mock: Mocker): +def test_token_401(rest_mock: Mocker) -> None: message = "invalid_client" rest_mock.post( f"{TEST_URI}v1/oauth/tokens", @@ -149,7 +149,7 @@ def test_token_401(rest_mock: Mocker): assert message in str(e.value) -def test_list_tables_200(rest_mock: Mocker): +def test_list_tables_200(rest_mock: Mocker) -> None: namespace = "examples" rest_mock.get( f"{TEST_URI}v1/namespaces/{namespace}/tables", @@ -161,7 +161,7 @@ def test_list_tables_200(rest_mock: Mocker): assert RestCatalog("rest", uri=TEST_URI, token=TEST_TOKEN).list_tables(namespace) == [("examples", "fooshare")] -def test_list_tables_404(rest_mock: Mocker): +def test_list_tables_404(rest_mock: Mocker) -> None: namespace = "examples" rest_mock.get( f"{TEST_URI}v1/namespaces/{namespace}/tables", @@ -180,7 +180,7 @@ def test_list_tables_404(rest_mock: Mocker): assert "Namespace does not exist" in str(e.value) -def test_list_namespaces_200(rest_mock: Mocker): +def test_list_namespaces_200(rest_mock: Mocker) -> None: rest_mock.get( f"{TEST_URI}v1/namespaces", json={"namespaces": [["default"], ["examples"], ["fokko"], ["system"]]}, @@ -195,7 +195,7 @@ def test_list_namespaces_200(rest_mock: Mocker): ] -def test_list_namespace_with_parent_200(rest_mock: Mocker): +def test_list_namespace_with_parent_200(rest_mock: Mocker) -> None: rest_mock.get( f"{TEST_URI}v1/namespaces?parent=accounting", json={"namespaces": [["tax"]]}, @@ -207,7 +207,7 @@ def test_list_namespace_with_parent_200(rest_mock: Mocker): ] -def test_create_namespace_200(rest_mock: Mocker): +def test_create_namespace_200(rest_mock: Mocker) -> None: namespace = "leden" rest_mock.post( f"{TEST_URI}v1/namespaces", @@ -218,7 +218,7 @@ def test_create_namespace_200(rest_mock: Mocker): RestCatalog("rest", uri=TEST_URI, token=TEST_TOKEN).create_namespace(namespace) -def test_create_namespace_409(rest_mock: Mocker): +def test_create_namespace_409(rest_mock: Mocker) -> None: namespace = "examples" rest_mock.post( f"{TEST_URI}v1/namespaces", @@ -237,7 +237,7 @@ def test_create_namespace_409(rest_mock: Mocker): assert "Namespace already exists" in str(e.value) -def test_drop_namespace_404(rest_mock: Mocker): +def test_drop_namespace_404(rest_mock: Mocker) -> None: namespace = "examples" rest_mock.delete( f"{TEST_URI}v1/namespaces/{namespace}", @@ -256,7 +256,7 @@ def test_drop_namespace_404(rest_mock: Mocker): assert "Namespace does not exist" in str(e.value) -def test_load_namespace_properties_200(rest_mock: Mocker): +def test_load_namespace_properties_200(rest_mock: Mocker) -> None: namespace = "leden" rest_mock.get( f"{TEST_URI}v1/namespaces/{namespace}", @@ -267,7 +267,7 @@ def test_load_namespace_properties_200(rest_mock: Mocker): assert RestCatalog("rest", uri=TEST_URI, token=TEST_TOKEN).load_namespace_properties(namespace) == {"prop": "yes"} -def test_load_namespace_properties_404(rest_mock: Mocker): +def test_load_namespace_properties_404(rest_mock: Mocker) -> None: namespace = "leden" rest_mock.get( f"{TEST_URI}v1/namespaces/{namespace}", @@ -286,7 +286,7 @@ def test_load_namespace_properties_404(rest_mock: Mocker): assert "Namespace does not exist" in str(e.value) -def test_update_namespace_properties_200(rest_mock: Mocker): +def test_update_namespace_properties_200(rest_mock: Mocker) -> None: rest_mock.post( f"{TEST_URI}v1/namespaces/fokko/properties", json={"removed": [], "updated": ["prop"], "missing": ["abc"]}, @@ -300,7 +300,7 @@ def test_update_namespace_properties_200(rest_mock: Mocker): assert response == PropertiesUpdateSummary(removed=[], updated=["prop"], missing=["abc"]) -def test_update_namespace_properties_404(rest_mock: Mocker): +def test_update_namespace_properties_404(rest_mock: Mocker) -> None: rest_mock.post( f"{TEST_URI}v1/namespaces/fokko/properties", json={ @@ -318,7 +318,7 @@ def test_update_namespace_properties_404(rest_mock: Mocker): assert "Namespace does not exist" in str(e.value) -def test_load_table_200(rest_mock: Mocker): +def test_load_table_200(rest_mock: Mocker) -> None: rest_mock.get( f"{TEST_URI}v1/namespaces/fokko/tables/table", json={ @@ -472,7 +472,7 @@ def test_load_table_200(rest_mock: Mocker): assert actual == expected -def test_load_table_404(rest_mock: Mocker): +def test_load_table_404(rest_mock: Mocker) -> None: rest_mock.get( f"{TEST_URI}v1/namespaces/fokko/tables/does_not_exists", json={ @@ -491,7 +491,7 @@ def test_load_table_404(rest_mock: Mocker): assert "Table does not exist" in str(e.value) -def test_drop_table_404(rest_mock: Mocker): +def test_drop_table_404(rest_mock: Mocker) -> None: rest_mock.delete( f"{TEST_URI}v1/namespaces/fokko/tables/does_not_exists", json={ @@ -510,7 +510,7 @@ def test_drop_table_404(rest_mock: Mocker): assert "Table does not exist" in str(e.value) -def test_create_table_200(rest_mock: Mocker, table_schema_simple: Schema): +def test_create_table_200(rest_mock: Mocker, table_schema_simple: Schema) -> None: rest_mock.post( f"{TEST_URI}v1/namespaces/fokko/tables", json={ @@ -626,7 +626,7 @@ def test_create_table_200(rest_mock: Mocker, table_schema_simple: Schema): ) -def test_create_table_409(rest_mock, table_schema_simple: Schema): +def test_create_table_409(rest_mock: Mocker, table_schema_simple: Schema) -> None: rest_mock.post( f"{TEST_URI}v1/namespaces/fokko/tables", json={ @@ -654,7 +654,7 @@ def test_create_table_409(rest_mock, table_schema_simple: Schema): assert "Table already exists" in str(e.value) -def test_delete_namespace_204(rest_mock: Mocker): +def test_delete_namespace_204(rest_mock: Mocker) -> None: namespace = "example" rest_mock.delete( f"{TEST_URI}v1/namespaces/{namespace}", @@ -665,7 +665,7 @@ def test_delete_namespace_204(rest_mock: Mocker): RestCatalog("rest", uri=TEST_URI, token=TEST_TOKEN).drop_namespace(namespace) -def test_delete_table_204(rest_mock: Mocker): +def test_delete_table_204(rest_mock: Mocker) -> None: rest_mock.delete( f"{TEST_URI}v1/namespaces/example/tables/fokko", json={}, @@ -675,7 +675,7 @@ def test_delete_table_204(rest_mock: Mocker): RestCatalog("rest", uri=TEST_URI, token=TEST_TOKEN).drop_table(("example", "fokko")) -def test_delete_table_404(rest_mock: Mocker): +def test_delete_table_404(rest_mock: Mocker) -> None: rest_mock.delete( f"{TEST_URI}v1/namespaces/example/tables/fokko", json={ @@ -693,7 +693,7 @@ def test_delete_table_404(rest_mock: Mocker): assert "Table does not exist" in str(e.value) -def test_create_table_missing_namespace(rest_mock: Mocker, table_schema_simple: Schema): +def test_create_table_missing_namespace(rest_mock: Mocker, table_schema_simple: Schema) -> None: table = "table" with pytest.raises(NoSuchTableError) as e: # Missing namespace @@ -701,7 +701,7 @@ def test_create_table_missing_namespace(rest_mock: Mocker, table_schema_simple: assert f"Missing namespace or invalid identifier: {table}" in str(e.value) -def test_load_table_invalid_namespace(rest_mock: Mocker): +def test_load_table_invalid_namespace(rest_mock: Mocker) -> None: table = "table" with pytest.raises(NoSuchTableError) as e: # Missing namespace @@ -709,7 +709,7 @@ def test_load_table_invalid_namespace(rest_mock: Mocker): assert f"Missing namespace or invalid identifier: {table}" in str(e.value) -def test_drop_table_invalid_namespace(rest_mock: Mocker): +def test_drop_table_invalid_namespace(rest_mock: Mocker) -> None: table = "table" with pytest.raises(NoSuchTableError) as e: # Missing namespace @@ -717,7 +717,7 @@ def test_drop_table_invalid_namespace(rest_mock: Mocker): assert f"Missing namespace or invalid identifier: {table}" in str(e.value) -def test_purge_table_invalid_namespace(rest_mock: Mocker): +def test_purge_table_invalid_namespace(rest_mock: Mocker) -> None: table = "table" with pytest.raises(NoSuchTableError) as e: # Missing namespace @@ -725,35 +725,35 @@ def test_purge_table_invalid_namespace(rest_mock: Mocker): assert f"Missing namespace or invalid identifier: {table}" in str(e.value) -def test_create_namespace_invalid_namespace(rest_mock: Mocker): +def test_create_namespace_invalid_namespace(rest_mock: Mocker) -> None: with pytest.raises(NoSuchNamespaceError) as e: # Missing namespace RestCatalog("rest", uri=TEST_URI, token=TEST_TOKEN).create_namespace(()) assert "Empty namespace identifier" in str(e.value) -def test_drop_namespace_invalid_namespace(rest_mock: Mocker): +def test_drop_namespace_invalid_namespace(rest_mock: Mocker) -> None: with pytest.raises(NoSuchNamespaceError) as e: # Missing namespace RestCatalog("rest", uri=TEST_URI, token=TEST_TOKEN).drop_namespace(()) assert "Empty namespace identifier" in str(e.value) -def test_load_namespace_properties_invalid_namespace(rest_mock: Mocker): +def test_load_namespace_properties_invalid_namespace(rest_mock: Mocker) -> None: with pytest.raises(NoSuchNamespaceError) as e: # Missing namespace RestCatalog("rest", uri=TEST_URI, token=TEST_TOKEN).load_namespace_properties(()) assert "Empty namespace identifier" in str(e.value) -def test_update_namespace_properties_invalid_namespace(rest_mock: Mocker): +def test_update_namespace_properties_invalid_namespace(rest_mock: Mocker) -> None: with pytest.raises(NoSuchNamespaceError) as e: # Missing namespace RestCatalog("rest", uri=TEST_URI, token=TEST_TOKEN).update_namespace_properties(()) assert "Empty namespace identifier" in str(e.value) -def test_request_session_with_ssl_ca_bundle(): +def test_request_session_with_ssl_ca_bundle() -> None: # Given catalog_properties = { "uri": TEST_URI, @@ -764,11 +764,11 @@ def test_request_session_with_ssl_ca_bundle(): } with pytest.raises(OSError) as e: # Missing namespace - RestCatalog("rest", **catalog_properties) + RestCatalog("rest", **catalog_properties) # type: ignore assert "Could not find a suitable TLS CA certificate bundle, invalid path: path_to_ca_bundle" in str(e.value) -def test_request_session_with_ssl_client_cert(): +def test_request_session_with_ssl_client_cert() -> None: # Given catalog_properties = { "uri": TEST_URI, @@ -782,5 +782,5 @@ def test_request_session_with_ssl_client_cert(): } with pytest.raises(OSError) as e: # Missing namespace - RestCatalog("rest", **catalog_properties) + RestCatalog("rest", **catalog_properties) # type: ignore assert "Could not find the TLS certificate file, invalid path: path_to_client_cert" in str(e.value) diff --git a/tests/cli/test_console.py b/tests/cli/test_console.py index f0420bde5c..4e0c3e62cd 100644 --- a/tests/cli/test_console.py +++ b/tests/cli/test_console.py @@ -134,7 +134,7 @@ def update_namespace_properties( MOCK_ENVIRONMENT = {"PYICEBERG_CATALOG__PRODUCTION__URI": "test://doesnotexist"} -def test_missing_uri(): +def test_missing_uri() -> None: runner = CliRunner() result = runner.invoke(run, ["list"]) assert result.exit_code == 1 @@ -146,7 +146,7 @@ def test_missing_uri(): @mock.patch.dict(os.environ, MOCK_ENVIRONMENT) @mock.patch("pyiceberg.cli.console.load_catalog", return_value=MOCK_CATALOG) -def test_list_root(_): +def test_list_root(_: MockCatalog) -> None: runner = CliRunner() result = runner.invoke(run, ["list"]) assert result.exit_code == 0 @@ -155,7 +155,7 @@ def test_list_root(_): @mock.patch.dict(os.environ, MOCK_ENVIRONMENT) @mock.patch("pyiceberg.cli.console.load_catalog", return_value=MOCK_CATALOG) -def test_list_namespace(_): +def test_list_namespace(_: MockCatalog) -> None: runner = CliRunner() result = runner.invoke(run, ["list", "default"]) assert result.exit_code == 0 @@ -164,7 +164,7 @@ def test_list_namespace(_): @mock.patch.dict(os.environ, MOCK_ENVIRONMENT) @mock.patch("pyiceberg.cli.console.load_catalog", return_value=MOCK_CATALOG) -def test_describe_namespace(_): +def test_describe_namespace(_: MockCatalog) -> None: runner = CliRunner() result = runner.invoke(run, ["describe", "default"]) assert result.exit_code == 0 @@ -173,7 +173,7 @@ def test_describe_namespace(_): @mock.patch.dict(os.environ, MOCK_ENVIRONMENT) @mock.patch("pyiceberg.cli.console.load_catalog", return_value=MOCK_CATALOG) -def test_describe_namespace_does_not_exists(_): +def test_describe_namespace_does_not_exists(_: MockCatalog) -> None: runner = CliRunner() result = runner.invoke(run, ["describe", "doesnotexist"]) assert result.exit_code == 1 @@ -182,7 +182,7 @@ def test_describe_namespace_does_not_exists(_): @mock.patch.dict(os.environ, MOCK_ENVIRONMENT) @mock.patch("pyiceberg.cli.console.load_catalog", return_value=MOCK_CATALOG) -def test_describe_table(_): +def test_describe_table(_: MockCatalog) -> None: runner = CliRunner() result = runner.invoke(run, ["describe", "default.foo"]) assert result.exit_code == 0 @@ -218,7 +218,7 @@ def test_describe_table(_): @mock.patch.dict(os.environ, MOCK_ENVIRONMENT) @mock.patch("pyiceberg.cli.console.load_catalog", return_value=MOCK_CATALOG) -def test_describe_table_does_not_exists(_): +def test_describe_table_does_not_exists(_: MockCatalog) -> None: runner = CliRunner() result = runner.invoke(run, ["describe", "default.doesnotexit"]) assert result.exit_code == 1 @@ -227,7 +227,7 @@ def test_describe_table_does_not_exists(_): @mock.patch.dict(os.environ, MOCK_ENVIRONMENT) @mock.patch("pyiceberg.cli.console.load_catalog", return_value=MOCK_CATALOG) -def test_schema(_): +def test_schema(_: MockCatalog) -> None: runner = CliRunner() result = runner.invoke(run, ["schema", "default.foo"]) assert result.exit_code == 0 @@ -242,7 +242,7 @@ def test_schema(_): @mock.patch.dict(os.environ, MOCK_ENVIRONMENT) @mock.patch("pyiceberg.cli.console.load_catalog", return_value=MOCK_CATALOG) -def test_schema_does_not_exists(_): +def test_schema_does_not_exists(_: MockCatalog) -> None: runner = CliRunner() result = runner.invoke(run, ["schema", "default.doesnotexit"]) assert result.exit_code == 1 @@ -251,7 +251,7 @@ def test_schema_does_not_exists(_): @mock.patch.dict(os.environ, MOCK_ENVIRONMENT) @mock.patch("pyiceberg.cli.console.load_catalog", return_value=MOCK_CATALOG) -def test_spec(_): +def test_spec(_: MockCatalog) -> None: runner = CliRunner() result = runner.invoke(run, ["spec", "default.foo"]) assert result.exit_code == 0 @@ -266,7 +266,7 @@ def test_spec(_): @mock.patch.dict(os.environ, MOCK_ENVIRONMENT) @mock.patch("pyiceberg.cli.console.load_catalog", return_value=MOCK_CATALOG) -def test_spec_does_not_exists(_): +def test_spec_does_not_exists(_: MockCatalog) -> None: runner = CliRunner() result = runner.invoke(run, ["spec", "default.doesnotexit"]) assert result.exit_code == 1 @@ -275,7 +275,7 @@ def test_spec_does_not_exists(_): @mock.patch.dict(os.environ, MOCK_ENVIRONMENT) @mock.patch("pyiceberg.cli.console.load_catalog", return_value=MOCK_CATALOG) -def test_uuid(_): +def test_uuid(_: MockCatalog) -> None: runner = CliRunner() result = runner.invoke(run, ["uuid", "default.foo"]) assert result.exit_code == 0 @@ -284,7 +284,7 @@ def test_uuid(_): @mock.patch.dict(os.environ, MOCK_ENVIRONMENT) @mock.patch("pyiceberg.cli.console.load_catalog", return_value=MOCK_CATALOG) -def test_uuid_does_not_exists(_): +def test_uuid_does_not_exists(_: MockCatalog) -> None: runner = CliRunner() result = runner.invoke(run, ["uuid", "default.doesnotexit"]) assert result.exit_code == 1 @@ -293,7 +293,7 @@ def test_uuid_does_not_exists(_): @mock.patch.dict(os.environ, MOCK_ENVIRONMENT) @mock.patch("pyiceberg.cli.console.load_catalog", return_value=MOCK_CATALOG) -def test_location(_): +def test_location(_: MockCatalog) -> None: runner = CliRunner() result = runner.invoke(run, ["location", "default.foo"]) assert result.exit_code == 0 @@ -302,7 +302,7 @@ def test_location(_): @mock.patch.dict(os.environ, MOCK_ENVIRONMENT) @mock.patch("pyiceberg.cli.console.load_catalog", return_value=MOCK_CATALOG) -def test_location_does_not_exists(_): +def test_location_does_not_exists(_: MockCatalog) -> None: runner = CliRunner() result = runner.invoke(run, ["location", "default.doesnotexit"]) assert result.exit_code == 1 @@ -311,7 +311,7 @@ def test_location_does_not_exists(_): @mock.patch.dict(os.environ, MOCK_ENVIRONMENT) @mock.patch("pyiceberg.cli.console.load_catalog", return_value=MOCK_CATALOG) -def test_drop_table(_): +def test_drop_table(_: MockCatalog) -> None: runner = CliRunner() result = runner.invoke(run, ["drop", "table", "default.foo"]) assert result.exit_code == 0 @@ -320,7 +320,7 @@ def test_drop_table(_): @mock.patch.dict(os.environ, MOCK_ENVIRONMENT) @mock.patch("pyiceberg.cli.console.load_catalog", return_value=MOCK_CATALOG) -def test_drop_table_does_not_exists(_): +def test_drop_table_does_not_exists(_: MockCatalog) -> None: runner = CliRunner() result = runner.invoke(run, ["drop", "table", "default.doesnotexit"]) assert result.exit_code == 1 @@ -329,7 +329,7 @@ def test_drop_table_does_not_exists(_): @mock.patch.dict(os.environ, MOCK_ENVIRONMENT) @mock.patch("pyiceberg.cli.console.load_catalog", return_value=MOCK_CATALOG) -def test_drop_namespace(_): +def test_drop_namespace(_: MockCatalog) -> None: runner = CliRunner() result = runner.invoke(run, ["drop", "namespace", "default"]) assert result.exit_code == 0 @@ -338,7 +338,7 @@ def test_drop_namespace(_): @mock.patch.dict(os.environ, MOCK_ENVIRONMENT) @mock.patch("pyiceberg.cli.console.load_catalog", return_value=MOCK_CATALOG) -def test_drop_namespace_does_not_exists(_): +def test_drop_namespace_does_not_exists(_: MockCatalog) -> None: runner = CliRunner() result = runner.invoke(run, ["drop", "namespace", "doesnotexit"]) assert result.exit_code == 1 @@ -347,7 +347,7 @@ def test_drop_namespace_does_not_exists(_): @mock.patch.dict(os.environ, MOCK_ENVIRONMENT) @mock.patch("pyiceberg.cli.console.load_catalog", return_value=MOCK_CATALOG) -def test_rename_table(_): +def test_rename_table(_: MockCatalog) -> None: runner = CliRunner() result = runner.invoke(run, ["rename", "default.foo", "default.bar"]) assert result.exit_code == 0 @@ -356,7 +356,7 @@ def test_rename_table(_): @mock.patch.dict(os.environ, MOCK_ENVIRONMENT) @mock.patch("pyiceberg.cli.console.load_catalog", return_value=MOCK_CATALOG) -def test_rename_table_does_not_exists(_): +def test_rename_table_does_not_exists(_: MockCatalog) -> None: runner = CliRunner() result = runner.invoke(run, ["rename", "default.doesnotexit", "default.bar"]) assert result.exit_code == 1 @@ -365,7 +365,7 @@ def test_rename_table_does_not_exists(_): @mock.patch.dict(os.environ, MOCK_ENVIRONMENT) @mock.patch("pyiceberg.cli.console.load_catalog", return_value=MOCK_CATALOG) -def test_properties_get_table(_): +def test_properties_get_table(_: MockCatalog) -> None: runner = CliRunner() result = runner.invoke(run, ["properties", "get", "table", "default.foo"]) assert result.exit_code == 0 @@ -374,7 +374,7 @@ def test_properties_get_table(_): @mock.patch.dict(os.environ, MOCK_ENVIRONMENT) @mock.patch("pyiceberg.cli.console.load_catalog", return_value=MOCK_CATALOG) -def test_properties_get_table_specific_property(_): +def test_properties_get_table_specific_property(_: MockCatalog) -> None: runner = CliRunner() result = runner.invoke(run, ["properties", "get", "table", "default.foo", "read.split.target.size"]) assert result.exit_code == 0 @@ -383,7 +383,7 @@ def test_properties_get_table_specific_property(_): @mock.patch.dict(os.environ, MOCK_ENVIRONMENT) @mock.patch("pyiceberg.cli.console.load_catalog", return_value=MOCK_CATALOG) -def test_properties_get_table_specific_property_that_doesnt_exist(_): +def test_properties_get_table_specific_property_that_doesnt_exist(_: MockCatalog) -> None: runner = CliRunner() result = runner.invoke(run, ["properties", "get", "table", "default.foo", "doesnotexist"]) assert result.exit_code == 1 @@ -392,7 +392,7 @@ def test_properties_get_table_specific_property_that_doesnt_exist(_): @mock.patch.dict(os.environ, MOCK_ENVIRONMENT) @mock.patch("pyiceberg.cli.console.load_catalog", return_value=MOCK_CATALOG) -def test_properties_get_table_does_not_exist(_): +def test_properties_get_table_does_not_exist(_: MockCatalog) -> None: runner = CliRunner() result = runner.invoke(run, ["properties", "get", "table", "doesnotexist"]) assert result.exit_code == 1 @@ -401,7 +401,7 @@ def test_properties_get_table_does_not_exist(_): @mock.patch.dict(os.environ, MOCK_ENVIRONMENT) @mock.patch("pyiceberg.cli.console.load_catalog", return_value=MOCK_CATALOG) -def test_properties_get_namespace(_): +def test_properties_get_namespace(_: MockCatalog) -> None: runner = CliRunner() result = runner.invoke(run, ["properties", "get", "namespace", "default"]) assert result.exit_code == 0 @@ -410,7 +410,7 @@ def test_properties_get_namespace(_): @mock.patch.dict(os.environ, MOCK_ENVIRONMENT) @mock.patch("pyiceberg.cli.console.load_catalog", return_value=MOCK_CATALOG) -def test_properties_get_namespace_specific_property(_): +def test_properties_get_namespace_specific_property(_: MockCatalog) -> None: runner = CliRunner() result = runner.invoke(run, ["properties", "get", "namespace", "default", "location"]) assert result.exit_code == 0 @@ -419,7 +419,7 @@ def test_properties_get_namespace_specific_property(_): @mock.patch.dict(os.environ, MOCK_ENVIRONMENT) @mock.patch("pyiceberg.cli.console.load_catalog", return_value=MOCK_CATALOG) -def test_properties_get_namespace_does_not_exist(_): +def test_properties_get_namespace_does_not_exist(_: MockCatalog) -> None: runner = CliRunner() result = runner.invoke(run, ["properties", "get", "namespace", "doesnotexist"]) assert result.exit_code == 1 @@ -428,7 +428,7 @@ def test_properties_get_namespace_does_not_exist(_): @mock.patch.dict(os.environ, MOCK_ENVIRONMENT) @mock.patch("pyiceberg.cli.console.load_catalog", return_value=MOCK_CATALOG) -def test_properties_set_namespace(_): +def test_properties_set_namespace(_: MockCatalog) -> None: runner = CliRunner() result = runner.invoke(run, ["properties", "set", "namespace", "default", "location", "s3://new_location"]) assert result.exit_code == 0 @@ -437,7 +437,7 @@ def test_properties_set_namespace(_): @mock.patch.dict(os.environ, MOCK_ENVIRONMENT) @mock.patch("pyiceberg.cli.console.load_catalog", return_value=MOCK_CATALOG) -def test_properties_set_namespace_that_doesnt_exist(_): +def test_properties_set_namespace_that_doesnt_exist(_: MockCatalog) -> None: runner = CliRunner() result = runner.invoke(run, ["properties", "set", "namespace", "doesnotexist", "location", "s3://new_location"]) assert result.exit_code == 1 @@ -446,7 +446,7 @@ def test_properties_set_namespace_that_doesnt_exist(_): @mock.patch.dict(os.environ, MOCK_ENVIRONMENT) @mock.patch("pyiceberg.cli.console.load_catalog", return_value=MOCK_CATALOG) -def test_properties_set_table(_): +def test_properties_set_table(_: MockCatalog) -> None: runner = CliRunner() result = runner.invoke(run, ["properties", "set", "table", "default.foo", "location", "s3://new_location"]) assert result.exit_code == 1 @@ -455,7 +455,7 @@ def test_properties_set_table(_): @mock.patch.dict(os.environ, MOCK_ENVIRONMENT) @mock.patch("pyiceberg.cli.console.load_catalog", return_value=MOCK_CATALOG) -def test_properties_set_table_does_not_exist(_): +def test_properties_set_table_does_not_exist(_: MockCatalog) -> None: runner = CliRunner() result = runner.invoke(run, ["properties", "set", "table", "default.doesnotexist", "location", "s3://new_location"]) assert result.exit_code == 1 @@ -464,7 +464,7 @@ def test_properties_set_table_does_not_exist(_): @mock.patch.dict(os.environ, MOCK_ENVIRONMENT) @mock.patch("pyiceberg.cli.console.load_catalog", return_value=MOCK_CATALOG) -def test_properties_remove_namespace(_): +def test_properties_remove_namespace(_: MockCatalog) -> None: runner = CliRunner() result = runner.invoke(run, ["properties", "remove", "namespace", "default", "location"]) assert result.exit_code == 0 @@ -473,7 +473,7 @@ def test_properties_remove_namespace(_): @mock.patch.dict(os.environ, MOCK_ENVIRONMENT) @mock.patch("pyiceberg.cli.console.load_catalog", return_value=MOCK_CATALOG) -def test_properties_remove_namespace_that_doesnt_exist(_): +def test_properties_remove_namespace_that_doesnt_exist(_: MockCatalog) -> None: runner = CliRunner() result = runner.invoke(run, ["properties", "remove", "namespace", "doesnotexist", "location"]) assert result.exit_code == 1 @@ -482,7 +482,7 @@ def test_properties_remove_namespace_that_doesnt_exist(_): @mock.patch.dict(os.environ, MOCK_ENVIRONMENT) @mock.patch("pyiceberg.cli.console.load_catalog", return_value=MOCK_CATALOG) -def test_properties_remove_table(_): +def test_properties_remove_table(_: MockCatalog) -> None: runner = CliRunner() result = runner.invoke(run, ["properties", "remove", "table", "default.foo", "read.split.target.size"]) assert result.exit_code == 1 @@ -491,7 +491,7 @@ def test_properties_remove_table(_): @mock.patch.dict(os.environ, MOCK_ENVIRONMENT) @mock.patch("pyiceberg.cli.console.load_catalog", return_value=MOCK_CATALOG) -def test_properties_remove_table_property_does_not_exists(_): +def test_properties_remove_table_property_does_not_exists(_: MockCatalog) -> None: runner = CliRunner() result = runner.invoke(run, ["properties", "remove", "table", "default.foo", "doesnotexist"]) assert result.exit_code == 1 @@ -500,7 +500,7 @@ def test_properties_remove_table_property_does_not_exists(_): @mock.patch.dict(os.environ, MOCK_ENVIRONMENT) @mock.patch("pyiceberg.cli.console.load_catalog", return_value=MOCK_CATALOG) -def test_properties_remove_table_does_not_exist(_): +def test_properties_remove_table_does_not_exist(_: MockCatalog) -> None: runner = CliRunner() result = runner.invoke(run, ["properties", "remove", "table", "default.doesnotexist", "location"]) assert result.exit_code == 1 @@ -509,7 +509,7 @@ def test_properties_remove_table_does_not_exist(_): @mock.patch.dict(os.environ, MOCK_ENVIRONMENT) @mock.patch("pyiceberg.cli.console.load_catalog", return_value=MOCK_CATALOG) -def test_json_list_root(_): +def test_json_list_root(_: MockCatalog) -> None: runner = CliRunner() result = runner.invoke(run, ["--output=json", "list"]) assert result.exit_code == 0 @@ -518,7 +518,7 @@ def test_json_list_root(_): @mock.patch.dict(os.environ, MOCK_ENVIRONMENT) @mock.patch("pyiceberg.cli.console.load_catalog", return_value=MOCK_CATALOG) -def test_json_list_namespace(_): +def test_json_list_namespace(_: MockCatalog) -> None: runner = CliRunner() result = runner.invoke(run, ["--output=json", "list", "default"]) assert result.exit_code == 0 @@ -527,7 +527,7 @@ def test_json_list_namespace(_): @mock.patch.dict(os.environ, MOCK_ENVIRONMENT) @mock.patch("pyiceberg.cli.console.load_catalog", return_value=MOCK_CATALOG) -def test_json_describe_namespace(_): +def test_json_describe_namespace(_: MockCatalog) -> None: runner = CliRunner() result = runner.invoke(run, ["--output=json", "describe", "default"]) assert result.exit_code == 0 @@ -536,7 +536,7 @@ def test_json_describe_namespace(_): @mock.patch.dict(os.environ, MOCK_ENVIRONMENT) @mock.patch("pyiceberg.cli.console.load_catalog", return_value=MOCK_CATALOG) -def test_json_describe_namespace_does_not_exists(_): +def test_json_describe_namespace_does_not_exists(_: MockCatalog) -> None: runner = CliRunner() result = runner.invoke(run, ["--output=json", "describe", "doesnotexist"]) assert result.exit_code == 1 @@ -545,7 +545,7 @@ def test_json_describe_namespace_does_not_exists(_): @mock.patch.dict(os.environ, MOCK_ENVIRONMENT) @mock.patch("pyiceberg.cli.console.load_catalog", return_value=MOCK_CATALOG) -def test_json_describe_table(_): +def test_json_describe_table(_: MockCatalog) -> None: runner = CliRunner() result = runner.invoke(run, ["--output=json", "describe", "default.foo"]) assert result.exit_code == 0 @@ -557,7 +557,7 @@ def test_json_describe_table(_): @mock.patch.dict(os.environ, MOCK_ENVIRONMENT) @mock.patch("pyiceberg.cli.console.load_catalog", return_value=MOCK_CATALOG) -def test_json_describe_table_does_not_exists(_): +def test_json_describe_table_does_not_exists(_: MockCatalog) -> None: runner = CliRunner() result = runner.invoke(run, ["--output=json", "describe", "default.doesnotexit"]) assert result.exit_code == 1 @@ -568,7 +568,7 @@ def test_json_describe_table_does_not_exists(_): @mock.patch.dict(os.environ, MOCK_ENVIRONMENT) @mock.patch("pyiceberg.cli.console.load_catalog", return_value=MOCK_CATALOG) -def test_json_schema(_): +def test_json_schema(_: MockCatalog) -> None: runner = CliRunner() result = runner.invoke(run, ["--output=json", "schema", "default.foo"]) assert result.exit_code == 0 @@ -580,7 +580,7 @@ def test_json_schema(_): @mock.patch.dict(os.environ, MOCK_ENVIRONMENT) @mock.patch("pyiceberg.cli.console.load_catalog", return_value=MOCK_CATALOG) -def test_json_schema_does_not_exists(_): +def test_json_schema_does_not_exists(_: MockCatalog) -> None: runner = CliRunner() result = runner.invoke(run, ["--output=json", "schema", "default.doesnotexit"]) assert result.exit_code == 1 @@ -589,7 +589,7 @@ def test_json_schema_does_not_exists(_): @mock.patch.dict(os.environ, MOCK_ENVIRONMENT) @mock.patch("pyiceberg.cli.console.load_catalog", return_value=MOCK_CATALOG) -def test_json_spec(_): +def test_json_spec(_: MockCatalog) -> None: runner = CliRunner() result = runner.invoke(run, ["--output=json", "spec", "default.foo"]) assert result.exit_code == 0 @@ -601,7 +601,7 @@ def test_json_spec(_): @mock.patch.dict(os.environ, MOCK_ENVIRONMENT) @mock.patch("pyiceberg.cli.console.load_catalog", return_value=MOCK_CATALOG) -def test_json_spec_does_not_exists(_): +def test_json_spec_does_not_exists(_: MockCatalog) -> None: runner = CliRunner() result = runner.invoke(run, ["--output=json", "spec", "default.doesnotexit"]) assert result.exit_code == 1 @@ -610,7 +610,7 @@ def test_json_spec_does_not_exists(_): @mock.patch.dict(os.environ, MOCK_ENVIRONMENT) @mock.patch("pyiceberg.cli.console.load_catalog", return_value=MOCK_CATALOG) -def test_json_uuid(_): +def test_json_uuid(_: MockCatalog) -> None: runner = CliRunner() result = runner.invoke(run, ["--output=json", "uuid", "default.foo"]) assert result.exit_code == 0 @@ -619,7 +619,7 @@ def test_json_uuid(_): @mock.patch.dict(os.environ, MOCK_ENVIRONMENT) @mock.patch("pyiceberg.cli.console.load_catalog", return_value=MOCK_CATALOG) -def test_json_uuid_does_not_exists(_): +def test_json_uuid_does_not_exists(_: MockCatalog) -> None: runner = CliRunner() result = runner.invoke(run, ["--output=json", "uuid", "default.doesnotexit"]) assert result.exit_code == 1 @@ -628,7 +628,7 @@ def test_json_uuid_does_not_exists(_): @mock.patch.dict(os.environ, MOCK_ENVIRONMENT) @mock.patch("pyiceberg.cli.console.load_catalog", return_value=MOCK_CATALOG) -def test_json_location(_): +def test_json_location(_: MockCatalog) -> None: runner = CliRunner() result = runner.invoke(run, ["--output=json", "location", "default.foo"]) assert result.exit_code == 0 @@ -637,7 +637,7 @@ def test_json_location(_): @mock.patch.dict(os.environ, MOCK_ENVIRONMENT) @mock.patch("pyiceberg.cli.console.load_catalog", return_value=MOCK_CATALOG) -def test_json_location_does_not_exists(_): +def test_json_location_does_not_exists(_: MockCatalog) -> None: runner = CliRunner() result = runner.invoke(run, ["--output=json", "location", "default.doesnotexist"]) assert result.exit_code == 1 @@ -646,7 +646,7 @@ def test_json_location_does_not_exists(_): @mock.patch.dict(os.environ, MOCK_ENVIRONMENT) @mock.patch("pyiceberg.cli.console.load_catalog", return_value=MOCK_CATALOG) -def test_json_drop_table(_): +def test_json_drop_table(_: MockCatalog) -> None: runner = CliRunner() result = runner.invoke(run, ["--output=json", "drop", "table", "default.foo"]) assert result.exit_code == 0 @@ -655,7 +655,7 @@ def test_json_drop_table(_): @mock.patch.dict(os.environ, MOCK_ENVIRONMENT) @mock.patch("pyiceberg.cli.console.load_catalog", return_value=MOCK_CATALOG) -def test_json_drop_table_does_not_exists(_): +def test_json_drop_table_does_not_exists(_: MockCatalog) -> None: runner = CliRunner() result = runner.invoke(run, ["--output=json", "drop", "table", "default.doesnotexist"]) assert result.exit_code == 1 @@ -664,7 +664,7 @@ def test_json_drop_table_does_not_exists(_): @mock.patch.dict(os.environ, MOCK_ENVIRONMENT) @mock.patch("pyiceberg.cli.console.load_catalog", return_value=MOCK_CATALOG) -def test_json_drop_namespace(_): +def test_json_drop_namespace(_: MockCatalog) -> None: runner = CliRunner() result = runner.invoke(run, ["--output=json", "drop", "namespace", "default"]) assert result.exit_code == 0 @@ -673,7 +673,7 @@ def test_json_drop_namespace(_): @mock.patch.dict(os.environ, MOCK_ENVIRONMENT) @mock.patch("pyiceberg.cli.console.load_catalog", return_value=MOCK_CATALOG) -def test_json_drop_namespace_does_not_exists(_): +def test_json_drop_namespace_does_not_exists(_: MockCatalog) -> None: runner = CliRunner() result = runner.invoke(run, ["--output=json", "drop", "namespace", "doesnotexist"]) assert result.exit_code == 1 @@ -682,7 +682,7 @@ def test_json_drop_namespace_does_not_exists(_): @mock.patch.dict(os.environ, MOCK_ENVIRONMENT) @mock.patch("pyiceberg.cli.console.load_catalog", return_value=MOCK_CATALOG) -def test_json_rename_table(_): +def test_json_rename_table(_: MockCatalog) -> None: runner = CliRunner() result = runner.invoke(run, ["--output=json", "rename", "default.foo", "default.bar"]) assert result.exit_code == 0 @@ -691,7 +691,7 @@ def test_json_rename_table(_): @mock.patch.dict(os.environ, MOCK_ENVIRONMENT) @mock.patch("pyiceberg.cli.console.load_catalog", return_value=MOCK_CATALOG) -def test_json_rename_table_does_not_exists(_): +def test_json_rename_table_does_not_exists(_: MockCatalog) -> None: runner = CliRunner() result = runner.invoke(run, ["--output=json", "rename", "default.doesnotexit", "default.bar"]) assert result.exit_code == 1 @@ -700,7 +700,7 @@ def test_json_rename_table_does_not_exists(_): @mock.patch.dict(os.environ, MOCK_ENVIRONMENT) @mock.patch("pyiceberg.cli.console.load_catalog", return_value=MOCK_CATALOG) -def test_json_properties_get_table(_): +def test_json_properties_get_table(_: MockCatalog) -> None: runner = CliRunner() result = runner.invoke(run, ["--output=json", "properties", "get", "table", "default.foo"]) assert result.exit_code == 0 @@ -709,7 +709,7 @@ def test_json_properties_get_table(_): @mock.patch.dict(os.environ, MOCK_ENVIRONMENT) @mock.patch("pyiceberg.cli.console.load_catalog", return_value=MOCK_CATALOG) -def test_json_properties_get_table_specific_property(_): +def test_json_properties_get_table_specific_property(_: MockCatalog) -> None: runner = CliRunner() result = runner.invoke(run, ["--output=json", "properties", "get", "table", "default.foo", "read.split.target.size"]) assert result.exit_code == 0 @@ -718,7 +718,7 @@ def test_json_properties_get_table_specific_property(_): @mock.patch.dict(os.environ, MOCK_ENVIRONMENT) @mock.patch("pyiceberg.cli.console.load_catalog", return_value=MOCK_CATALOG) -def test_json_properties_get_table_specific_property_that_doesnt_exist(_): +def test_json_properties_get_table_specific_property_that_doesnt_exist(_: MockCatalog) -> None: runner = CliRunner() result = runner.invoke(run, ["--output=json", "properties", "get", "table", "default.foo", "doesnotexist"]) assert result.exit_code == 1 @@ -730,7 +730,7 @@ def test_json_properties_get_table_specific_property_that_doesnt_exist(_): @mock.patch.dict(os.environ, MOCK_ENVIRONMENT) @mock.patch("pyiceberg.cli.console.load_catalog", return_value=MOCK_CATALOG) -def test_json_properties_get_table_does_not_exist(_): +def test_json_properties_get_table_does_not_exist(_: MockCatalog) -> None: runner = CliRunner() result = runner.invoke(run, ["--output=json", "properties", "get", "table", "doesnotexist"]) assert result.exit_code == 1 @@ -739,7 +739,7 @@ def test_json_properties_get_table_does_not_exist(_): @mock.patch.dict(os.environ, MOCK_ENVIRONMENT) @mock.patch("pyiceberg.cli.console.load_catalog", return_value=MOCK_CATALOG) -def test_json_properties_get_namespace(_): +def test_json_properties_get_namespace(_: MockCatalog) -> None: runner = CliRunner() result = runner.invoke(run, ["--output=json", "properties", "get", "namespace", "default"]) assert result.exit_code == 0 @@ -748,7 +748,7 @@ def test_json_properties_get_namespace(_): @mock.patch.dict(os.environ, MOCK_ENVIRONMENT) @mock.patch("pyiceberg.cli.console.load_catalog", return_value=MOCK_CATALOG) -def test_json_properties_get_namespace_specific_property(_): +def test_json_properties_get_namespace_specific_property(_: MockCatalog) -> None: runner = CliRunner() result = runner.invoke(run, ["--output=json", "properties", "get", "namespace", "default", "location"]) assert result.exit_code == 0 @@ -757,7 +757,7 @@ def test_json_properties_get_namespace_specific_property(_): @mock.patch.dict(os.environ, MOCK_ENVIRONMENT) @mock.patch("pyiceberg.cli.console.load_catalog", return_value=MOCK_CATALOG) -def test_json_properties_get_namespace_does_not_exist(_): +def test_json_properties_get_namespace_does_not_exist(_: MockCatalog) -> None: runner = CliRunner() result = runner.invoke(run, ["--output=json", "properties", "get", "namespace", "doesnotexist"]) assert result.exit_code == 1 @@ -766,7 +766,7 @@ def test_json_properties_get_namespace_does_not_exist(_): @mock.patch.dict(os.environ, MOCK_ENVIRONMENT) @mock.patch("pyiceberg.cli.console.load_catalog", return_value=MOCK_CATALOG) -def test_json_properties_set_namespace(_): +def test_json_properties_set_namespace(_: MockCatalog) -> None: runner = CliRunner() result = runner.invoke(run, ["--output=json", "properties", "set", "namespace", "default", "location", "s3://new_location"]) assert result.exit_code == 0 @@ -775,7 +775,7 @@ def test_json_properties_set_namespace(_): @mock.patch.dict(os.environ, MOCK_ENVIRONMENT) @mock.patch("pyiceberg.cli.console.load_catalog", return_value=MOCK_CATALOG) -def test_json_properties_set_namespace_that_doesnt_exist(_): +def test_json_properties_set_namespace_that_doesnt_exist(_: MockCatalog) -> None: runner = CliRunner() result = runner.invoke( run, ["--output=json", "properties", "set", "namespace", "doesnotexist", "location", "s3://new_location"] @@ -786,7 +786,7 @@ def test_json_properties_set_namespace_that_doesnt_exist(_): @mock.patch.dict(os.environ, MOCK_ENVIRONMENT) @mock.patch("pyiceberg.cli.console.load_catalog", return_value=MOCK_CATALOG) -def test_json_properties_set_table(_): +def test_json_properties_set_table(_: MockCatalog) -> None: runner = CliRunner() result = runner.invoke(run, ["--output=json", "properties", "set", "table", "default.foo", "location", "s3://new_location"]) assert result.exit_code == 1 @@ -795,7 +795,7 @@ def test_json_properties_set_table(_): @mock.patch.dict(os.environ, MOCK_ENVIRONMENT) @mock.patch("pyiceberg.cli.console.load_catalog", return_value=MOCK_CATALOG) -def test_json_properties_set_table_does_not_exist(_): +def test_json_properties_set_table_does_not_exist(_: MockCatalog) -> None: runner = CliRunner() result = runner.invoke( run, ["--output=json", "properties", "set", "table", "default.doesnotexist", "location", "s3://new_location"] @@ -806,7 +806,7 @@ def test_json_properties_set_table_does_not_exist(_): @mock.patch.dict(os.environ, MOCK_ENVIRONMENT) @mock.patch("pyiceberg.cli.console.load_catalog", return_value=MOCK_CATALOG) -def test_json_properties_remove_namespace(_): +def test_json_properties_remove_namespace(_: MockCatalog) -> None: runner = CliRunner() result = runner.invoke(run, ["--output=json", "properties", "remove", "namespace", "default", "location"]) assert result.exit_code == 0 @@ -815,7 +815,7 @@ def test_json_properties_remove_namespace(_): @mock.patch.dict(os.environ, MOCK_ENVIRONMENT) @mock.patch("pyiceberg.cli.console.load_catalog", return_value=MOCK_CATALOG) -def test_json_properties_remove_namespace_that_doesnt_exist(_): +def test_json_properties_remove_namespace_that_doesnt_exist(_: MockCatalog) -> None: runner = CliRunner() result = runner.invoke(run, ["--output=json", "properties", "remove", "namespace", "doesnotexist", "location"]) assert result.exit_code == 1 @@ -824,7 +824,7 @@ def test_json_properties_remove_namespace_that_doesnt_exist(_): @mock.patch.dict(os.environ, MOCK_ENVIRONMENT) @mock.patch("pyiceberg.cli.console.load_catalog", return_value=MOCK_CATALOG) -def test_json_properties_remove_table(_): +def test_json_properties_remove_table(_: MockCatalog) -> None: runner = CliRunner() result = runner.invoke(run, ["--output=json", "properties", "remove", "table", "default.foo", "read.split.target.size"]) assert result.exit_code == 1 @@ -833,7 +833,7 @@ def test_json_properties_remove_table(_): @mock.patch.dict(os.environ, MOCK_ENVIRONMENT) @mock.patch("pyiceberg.cli.console.load_catalog", return_value=MOCK_CATALOG) -def test_json_properties_remove_table_property_does_not_exists(_): +def test_json_properties_remove_table_property_does_not_exists(_: MockCatalog) -> None: runner = CliRunner() result = runner.invoke(run, ["--output=json", "properties", "remove", "table", "default.foo", "doesnotexist"]) assert result.exit_code == 1 @@ -845,7 +845,7 @@ def test_json_properties_remove_table_property_does_not_exists(_): @mock.patch.dict(os.environ, MOCK_ENVIRONMENT) @mock.patch("pyiceberg.cli.console.load_catalog", return_value=MOCK_CATALOG) -def test_json_properties_remove_table_does_not_exist(_): +def test_json_properties_remove_table_does_not_exist(_: MockCatalog) -> None: runner = CliRunner() result = runner.invoke(run, ["--output=json", "properties", "remove", "table", "default.doesnotexist", "location"]) assert result.exit_code == 1 diff --git a/tests/conftest.py b/tests/conftest.py index 59f7ccc723..bab7b87bcd 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -31,6 +31,7 @@ Callable, Dict, Generator, + Type, Union, ) from unittest.mock import MagicMock @@ -55,6 +56,7 @@ OutputStream, fsspec, ) +from pyiceberg.io.fsspec import FsspecFileIO from pyiceberg.schema import Schema from pyiceberg.types import ( BinaryType, @@ -73,7 +75,7 @@ from tests.io.test_io import LocalInputFile -def pytest_addoption(parser): +def pytest_addoption(parser: pytest.Parser) -> None: parser.addoption( "--s3.endpoint", action="store", default="http://localhost:9000", help="The S3 endpoint URL for tests marked as s3" ) @@ -86,18 +88,20 @@ def pytest_addoption(parser): class FooStruct: """An example of an object that abides by StructProtocol""" - def __init__(self): + content: Dict[int, Any] + + def __init__(self) -> None: self.content = {} def get(self, pos: int) -> Any: return self.content[pos] - def set(self, pos: int, value) -> None: + def set(self, pos: int, value: Any) -> None: self.content[pos] = value @pytest.fixture(scope="session") -def table_schema_simple(): +def table_schema_simple() -> Schema: return schema.Schema( NestedField(field_id=1, name="foo", field_type=StringType(), required=False), NestedField(field_id=2, name="bar", field_type=IntegerType(), required=True), @@ -108,7 +112,7 @@ def table_schema_simple(): @pytest.fixture(scope="session") -def table_schema_nested(): +def table_schema_nested() -> Schema: return schema.Schema( NestedField(field_id=1, name="foo", field_type=StringType(), required=False), NestedField(field_id=2, name="bar", field_type=IntegerType(), required=True), @@ -159,7 +163,7 @@ def table_schema_nested(): @pytest.fixture(scope="session") -def foo_struct(): +def foo_struct() -> FooStruct: return FooStruct() @@ -882,7 +886,7 @@ def avro_schema_manifest_entry() -> Dict[str, Any]: @pytest.fixture(scope="session") -def simple_struct(): +def simple_struct() -> StructType: return StructType( NestedField(id=1, name="required_field", field_type=StringType(), required=True, doc="this is a doc"), NestedField(id=2, name="optional_field", field_type=IntegerType()), @@ -890,19 +894,19 @@ def simple_struct(): @pytest.fixture(scope="session") -def simple_list(): +def simple_list() -> ListType: return ListType(element_id=22, element=StringType(), element_required=True) @pytest.fixture(scope="session") -def simple_map(): +def simple_map() -> MapType: return MapType(key_id=19, key_type=StringType(), value_id=25, value_type=DoubleType(), value_required=False) class LocalOutputFile(OutputFile): """An OutputFile implementation for local files (for test use only)""" - def __init__(self, location: str): + def __init__(self, location: str) -> None: parsed_location = urlparse(location) # Create a ParseResult from the uri if parsed_location.scheme and parsed_location.scheme != "file": # Validate that a uri is provided with a scheme of `file` raise ValueError("LocalOutputFile location must have a scheme of `file`") @@ -912,13 +916,13 @@ def __init__(self, location: str): super().__init__(location=location) self._path = parsed_location.path - def __len__(self): + def __len__(self) -> int: return os.path.getsize(self._path) - def exists(self): + def exists(self) -> bool: return os.path.exists(self._path) - def to_input_file(self): + def to_input_file(self) -> LocalInputFile: return LocalInputFile(location=self.location) def create(self, overwrite: bool = False) -> OutputStream: @@ -931,19 +935,19 @@ def create(self, overwrite: bool = False) -> OutputStream: class LocalFileIO(FileIO): """A FileIO implementation for local files (for test use only)""" - def new_input(self, location: str): + def new_input(self, location: str) -> LocalInputFile: return LocalInputFile(location=location) - def new_output(self, location: str): + def new_output(self, location: str) -> LocalOutputFile: return LocalOutputFile(location=location) - def delete(self, location: Union[str, InputFile, OutputFile]): + def delete(self, location: Union[str, InputFile, OutputFile]) -> None: location = location.location if isinstance(location, (InputFile, OutputFile)) else location os.remove(location) @pytest.fixture(scope="session", autouse=True) -def LocalFileIOFixture(): +def LocalFileIOFixture() -> Type[LocalFileIO]: return LocalFileIO @@ -1146,7 +1150,7 @@ def iceberg_manifest_entry_schema() -> Schema: @pytest.fixture -def fsspec_fileio(request): +def fsspec_fileio(request: pytest.FixtureRequest) -> FsspecFileIO: properties = { "s3.endpoint": request.config.getoption("--s3.endpoint"), "s3.access-key-id": request.config.getoption("--s3.access-key-id"), @@ -1161,7 +1165,7 @@ class MockAWSResponse(aiobotocore.awsrequest.AioAWSResponse): See https://github.com/aio-libs/aiobotocore/issues/755 """ - def __init__(self, response: botocore.awsrequest.AWSResponse): + def __init__(self, response: botocore.awsrequest.AWSResponse) -> None: self._moto_response = response self.status_code = response.status_code self.raw = MockHttpClientResponse(response) @@ -1180,8 +1184,8 @@ class MockHttpClientResponse(aiohttp.client_reqrep.ClientResponse): See https://github.com/aio-libs/aiobotocore/issues/755 """ - def __init__(self, response: botocore.awsrequest.AWSResponse): - async def read(*_) -> bytes: + def __init__(self, response: botocore.awsrequest.AWSResponse) -> None: + async def read(*_: Any) -> bytes: # streaming/range requests. used by s3fs return response.content @@ -1195,14 +1199,14 @@ def raw_headers(self) -> aiohttp.typedefs.RawHeaders: return {k.encode("utf-8"): str(v).encode("utf-8") for k, v in self.response.headers.items()}.items() -def patch_aiobotocore(): +def patch_aiobotocore() -> None: """ Patch aiobotocore to work with moto See https://github.com/aio-libs/aiobotocore/issues/755 """ - def factory(original: Callable) -> Callable: - def patched_convert_to_response_dict( + def factory(original: Callable) -> Callable: # type: ignore + def patched_convert_to_response_dict( # type: ignore http_response: botocore.awsrequest.AWSResponse, operation_model: botocore.model.OperationModel ): return original(MockAWSResponse(http_response), operation_model) @@ -1213,7 +1217,7 @@ def patched_convert_to_response_dict( @pytest.fixture(name="_patch_aiobotocore") -def fixture_aiobotocore(): +def fixture_aiobotocore(): # type: ignore """ Patch aiobotocore to work with moto pending close of this issue: https://github.com/aio-libs/aiobotocore/issues/755 @@ -1224,7 +1228,7 @@ def fixture_aiobotocore(): aiobotocore.endpoint.convert_to_response_dict = stored_method -def aws_credentials(): +def aws_credentials() -> None: os.environ["AWS_ACCESS_KEY_ID"] = "testing" os.environ["AWS_SECRET_ACCESS_KEY"] = "testing" os.environ["AWS_SECURITY_TOKEN"] = "testing" @@ -1233,9 +1237,9 @@ def aws_credentials(): @pytest.fixture(name="_aws_credentials") -def fixture_aws_credentials(): +def fixture_aws_credentials() -> Generator[None, None, None]: """Mocked AWS Credentials for moto.""" - yield aws_credentials() + yield aws_credentials() # type: ignore os.environ.pop("AWS_ACCESS_KEY_ID") os.environ.pop("AWS_SECRET_ACCESS_KEY") os.environ.pop("AWS_SECURITY_TOKEN") @@ -1244,14 +1248,14 @@ def fixture_aws_credentials(): @pytest.fixture(name="_s3") -def fixture_s3(_aws_credentials): +def fixture_s3(_aws_credentials: None) -> Generator[boto3.client, None, None]: """Mocked S3 client""" with mock_s3(): yield boto3.client("s3", region_name="us-east-1") @pytest.fixture(name="_glue") -def fixture_glue(_aws_credentials): +def fixture_glue(_aws_credentials: None) -> Generator[boto3.client, None, None]: """Mocked glue client""" with mock_glue(): yield boto3.client("glue", region_name="us-east-1") diff --git a/tests/expressions/test_evaluator.py b/tests/expressions/test_evaluator.py index d2e473e38f..4d4025a268 100644 --- a/tests/expressions/test_evaluator.py +++ b/tests/expressions/test_evaluator.py @@ -49,7 +49,7 @@ class Record(StructProtocol): data: List[Any] - def __init__(self, *values): + def __init__(self, *values: Any) -> None: self.data = list(values) def get(self, pos: int) -> Any: @@ -68,111 +68,111 @@ def set(self, pos: int, value: Any) -> None: ) -def test_true(): +def test_true() -> None: evaluate = expression_evaluator(SIMPLE_SCHEMA, AlwaysTrue(), case_sensitive=True) assert evaluate(Record(1, "a")) -def test_false(): +def test_false() -> None: evaluate = expression_evaluator(SIMPLE_SCHEMA, AlwaysFalse(), case_sensitive=True) assert not evaluate(Record(1, "a")) -def test_less_than(): +def test_less_than() -> None: evaluate = expression_evaluator(SIMPLE_SCHEMA, LessThan("id", 3), case_sensitive=True) assert evaluate(Record(2, "a")) assert not evaluate(Record(3, "a")) -def test_less_than_or_equal(): +def test_less_than_or_equal() -> None: evaluate = expression_evaluator(SIMPLE_SCHEMA, LessThanOrEqual("id", 3), case_sensitive=True) assert evaluate(Record(1, "a")) assert evaluate(Record(3, "a")) assert not evaluate(Record(4, "a")) -def test_greater_than(): +def test_greater_than() -> None: evaluate = expression_evaluator(SIMPLE_SCHEMA, GreaterThan("id", 3), case_sensitive=True) assert not evaluate(Record(1, "a")) assert not evaluate(Record(3, "a")) assert evaluate(Record(4, "a")) -def test_greater_than_or_equal(): +def test_greater_than_or_equal() -> None: evaluate = expression_evaluator(SIMPLE_SCHEMA, GreaterThanOrEqual("id", 3), case_sensitive=True) assert not evaluate(Record(2, "a")) assert evaluate(Record(3, "a")) assert evaluate(Record(4, "a")) -def test_equal_to(): +def test_equal_to() -> None: evaluate = expression_evaluator(SIMPLE_SCHEMA, EqualTo("id", 3), case_sensitive=True) assert not evaluate(Record(2, "a")) assert evaluate(Record(3, "a")) assert not evaluate(Record(4, "a")) -def test_not_equal_to(): +def test_not_equal_to() -> None: evaluate = expression_evaluator(SIMPLE_SCHEMA, NotEqualTo("id", 3), case_sensitive=True) assert evaluate(Record(2, "a")) assert not evaluate(Record(3, "a")) assert evaluate(Record(4, "a")) -def test_in(): +def test_in() -> None: evaluate = expression_evaluator(SIMPLE_SCHEMA, In("id", [1, 2, 3]), case_sensitive=True) assert evaluate(Record(2, "a")) assert evaluate(Record(3, "a")) assert not evaluate(Record(4, "a")) -def test_not_in(): +def test_not_in() -> None: evaluate = expression_evaluator(SIMPLE_SCHEMA, NotIn("id", [1, 2, 3]), case_sensitive=True) assert not evaluate(Record(2, "a")) assert not evaluate(Record(3, "a")) assert evaluate(Record(4, "a")) -def test_is_null(): +def test_is_null() -> None: evaluate = expression_evaluator(SIMPLE_SCHEMA, IsNull("data"), case_sensitive=True) assert not evaluate(Record(2, "a")) assert evaluate(Record(3, None)) -def test_not_null(): +def test_not_null() -> None: evaluate = expression_evaluator(SIMPLE_SCHEMA, NotNull("data"), case_sensitive=True) assert evaluate(Record(2, "a")) assert not evaluate(Record(3, None)) -def test_is_nan(): +def test_is_nan() -> None: evaluate = expression_evaluator(FLOAT_SCHEMA, IsNaN("f"), case_sensitive=True) assert not evaluate(Record(2, 0.0)) assert not evaluate(Record(3, float("infinity"))) assert evaluate(Record(4, float("nan"))) -def test_not_nan(): +def test_not_nan() -> None: evaluate = expression_evaluator(FLOAT_SCHEMA, NotNaN("f"), case_sensitive=True) assert evaluate(Record(2, 0.0)) assert evaluate(Record(3, float("infinity"))) assert not evaluate(Record(4, float("nan"))) -def test_not(): +def test_not() -> None: evaluate = expression_evaluator(SIMPLE_SCHEMA, Not(LessThan("id", 3)), case_sensitive=True) assert not evaluate(Record(2, "a")) assert evaluate(Record(3, "a")) -def test_and(): +def test_and() -> None: evaluate = expression_evaluator(SIMPLE_SCHEMA, And(LessThan("id", 3), GreaterThan("id", 1)), case_sensitive=True) assert not evaluate(Record(1, "a")) assert evaluate(Record(2, "a")) assert not evaluate(Record(3, "a")) -def test_or(): +def test_or() -> None: evaluate = expression_evaluator(SIMPLE_SCHEMA, Or(LessThan("id", 2), GreaterThan("id", 2)), case_sensitive=True) assert evaluate(Record(1, "a")) assert not evaluate(Record(2, "a")) diff --git a/tests/expressions/test_expressions.py b/tests/expressions/test_expressions.py index befa71933c..233b80a385 100644 --- a/tests/expressions/test_expressions.py +++ b/tests/expressions/test_expressions.py @@ -27,6 +27,7 @@ AlwaysFalse, AlwaysTrue, And, + BooleanExpression, BoundEqualTo, BoundGreaterThan, BoundGreaterThanOrEqual, @@ -55,6 +56,7 @@ NotNull, Or, Reference, + UnboundPredicate, ) from pyiceberg.expressions.literals import Literal, literal from pyiceberg.expressions.visitors import _from_byte_buffer @@ -72,37 +74,37 @@ from tests.expressions.test_visitors import ExpressionA, ExpressionB -def test_isnull_inverse(): +def test_isnull_inverse() -> None: assert ~IsNull(Reference("a")) == NotNull(Reference("a")) -def test_isnull_bind(): +def test_isnull_bind() -> None: schema = Schema(NestedField(2, "a", IntegerType()), schema_id=1) bound = BoundIsNull(BoundReference(schema.find_field(2), schema.accessor_for_field(2))) assert IsNull(Reference("a")).bind(schema) == bound -def test_invert_is_null_bind(): +def test_invert_is_null_bind() -> None: schema = Schema(NestedField(2, "a", IntegerType(), required=False), schema_id=1) assert ~IsNull(Reference("a")).bind(schema) == NotNull(Reference("a")).bind(schema) -def test_invert_not_null_bind(): +def test_invert_not_null_bind() -> None: schema = Schema(NestedField(2, "a", IntegerType(), required=False), schema_id=1) assert ~NotNull(Reference("a")).bind(schema) == IsNull(Reference("a")).bind(schema) -def test_invert_is_nan_bind(): +def test_invert_is_nan_bind() -> None: schema = Schema(NestedField(2, "a", DoubleType(), required=False), schema_id=1) assert ~IsNaN(Reference("a")).bind(schema) == NotNaN(Reference("a")).bind(schema) -def test_invert_not_nan_bind(): +def test_invert_not_nan_bind() -> None: schema = Schema(NestedField(2, "a", DoubleType(), required=False), schema_id=1) assert ~NotNaN(Reference("a")).bind(schema) == IsNaN(Reference("a")).bind(schema) -def test_bind_expr_does_not_exists(): +def test_bind_expr_does_not_exists() -> None: schema = Schema(NestedField(2, "a", IntegerType()), schema_id=1) with pytest.raises(ValueError) as exc_info: IsNull(Reference("b")).bind(schema) @@ -110,7 +112,7 @@ def test_bind_expr_does_not_exists(): assert str(exc_info.value) == "Could not find field with name b, case_sensitive=True" -def test_bind_does_not_exists(): +def test_bind_does_not_exists() -> None: schema = Schema(NestedField(2, "a", IntegerType()), schema_id=1) with pytest.raises(ValueError) as exc_info: Reference("b").bind(schema) @@ -118,107 +120,107 @@ def test_bind_does_not_exists(): assert str(exc_info.value) == "Could not find field with name b, case_sensitive=True" -def test_isnull_bind_required(): +def test_isnull_bind_required() -> None: schema = Schema(NestedField(2, "a", IntegerType(), required=True), schema_id=1) assert IsNull(Reference("a")).bind(schema) == AlwaysFalse() -def test_notnull_inverse(): +def test_notnull_inverse() -> None: assert ~NotNull(Reference("a")) == IsNull(Reference("a")) -def test_notnull_bind(): +def test_notnull_bind() -> None: schema = Schema(NestedField(2, "a", IntegerType()), schema_id=1) bound = BoundNotNull(BoundReference(schema.find_field(2), schema.accessor_for_field(2))) assert NotNull(Reference("a")).bind(schema) == bound -def test_notnull_bind_required(): +def test_notnull_bind_required() -> None: schema = Schema(NestedField(2, "a", IntegerType(), required=True), schema_id=1) assert NotNull(Reference("a")).bind(schema) == AlwaysTrue() -def test_isnan_inverse(): +def test_isnan_inverse() -> None: assert ~IsNaN(Reference("f")) == NotNaN(Reference("f")) -def test_isnan_bind_float(): +def test_isnan_bind_float() -> None: schema = Schema(NestedField(2, "f", FloatType()), schema_id=1) bound = BoundIsNaN(BoundReference(schema.find_field(2), schema.accessor_for_field(2))) assert IsNaN(Reference("f")).bind(schema) == bound -def test_isnan_bind_double(): +def test_isnan_bind_double() -> None: schema = Schema(NestedField(2, "d", DoubleType()), schema_id=1) bound = BoundIsNaN(BoundReference(schema.find_field(2), schema.accessor_for_field(2))) assert IsNaN(Reference("d")).bind(schema) == bound -def test_isnan_bind_nonfloat(): +def test_isnan_bind_nonfloat() -> None: schema = Schema(NestedField(2, "i", IntegerType()), schema_id=1) assert IsNaN(Reference("i")).bind(schema) == AlwaysFalse() -def test_notnan_inverse(): +def test_notnan_inverse() -> None: assert ~NotNaN(Reference("f")) == IsNaN(Reference("f")) -def test_notnan_bind_float(): +def test_notnan_bind_float() -> None: schema = Schema(NestedField(2, "f", FloatType()), schema_id=1) bound = BoundNotNaN(BoundReference(schema.find_field(2), schema.accessor_for_field(2))) assert NotNaN(Reference("f")).bind(schema) == bound -def test_notnan_bind_double(): +def test_notnan_bind_double() -> None: schema = Schema(NestedField(2, "d", DoubleType()), schema_id=1) bound = BoundNotNaN(BoundReference(schema.find_field(2), schema.accessor_for_field(2))) assert NotNaN(Reference("d")).bind(schema) == bound -def test_notnan_bind_nonfloat(): +def test_notnan_bind_nonfloat() -> None: schema = Schema(NestedField(2, "i", IntegerType()), schema_id=1) assert NotNaN(Reference("i")).bind(schema) == AlwaysTrue() -def test_ref_binding_case_sensitive(table_schema_simple: Schema): +def test_ref_binding_case_sensitive(table_schema_simple: Schema) -> None: ref = Reference("foo") bound = BoundReference(table_schema_simple.find_field(1), table_schema_simple.accessor_for_field(1)) assert ref.bind(table_schema_simple, case_sensitive=True) == bound -def test_ref_binding_case_sensitive_failure(table_schema_simple: Schema): +def test_ref_binding_case_sensitive_failure(table_schema_simple: Schema) -> None: ref = Reference("Foo") with pytest.raises(ValueError): ref.bind(table_schema_simple, case_sensitive=True) -def test_ref_binding_case_insensitive(table_schema_simple: Schema): +def test_ref_binding_case_insensitive(table_schema_simple: Schema) -> None: ref = Reference("Foo") bound = BoundReference(table_schema_simple.find_field(1), table_schema_simple.accessor_for_field(1)) assert ref.bind(table_schema_simple, case_sensitive=False) == bound -def test_ref_binding_case_insensitive_failure(table_schema_simple: Schema): +def test_ref_binding_case_insensitive_failure(table_schema_simple: Schema) -> None: ref = Reference("Foot") with pytest.raises(ValueError): ref.bind(table_schema_simple, case_sensitive=False) -def test_in_to_eq(): +def test_in_to_eq() -> None: assert In("x", (34.56,)) == EqualTo("x", 34.56) -def test_empty_bind_in(table_schema_simple: Schema): +def test_empty_bind_in(table_schema_simple: Schema) -> None: bound = BoundIn(BoundReference(table_schema_simple.find_field(1), table_schema_simple.accessor_for_field(1)), set()) assert bound == AlwaysFalse() -def test_empty_bind_not_in(table_schema_simple: Schema): +def test_empty_bind_not_in(table_schema_simple: Schema) -> None: bound = BoundNotIn(BoundReference(table_schema_simple.find_field(1), table_schema_simple.accessor_for_field(1)), set()) assert bound == AlwaysTrue() -def test_bind_not_in_equal_term(table_schema_simple: Schema): +def test_bind_not_in_equal_term(table_schema_simple: Schema) -> None: bound = BoundNotIn( BoundReference(table_schema_simple.find_field(1), table_schema_simple.accessor_for_field(1)), {literal("hello")} ) @@ -234,31 +236,31 @@ def test_bind_not_in_equal_term(table_schema_simple: Schema): ) -def test_in_empty(): +def test_in_empty() -> None: assert In(Reference("foo"), ()) == AlwaysFalse() -def test_in_set(): +def test_in_set() -> None: assert In(Reference("foo"), {"a", "bc", "def"}).literals == {literal("a"), literal("bc"), literal("def")} -def test_in_tuple(): +def test_in_tuple() -> None: assert In(Reference("foo"), ("a", "bc", "def")).literals == {literal("a"), literal("bc"), literal("def")} -def test_in_list(): +def test_in_list() -> None: assert In(Reference("foo"), ["a", "bc", "def"]).literals == {literal("a"), literal("bc"), literal("def")} -def test_not_in_empty(): +def test_not_in_empty() -> None: assert NotIn(Reference("foo"), ()) == AlwaysTrue() -def test_not_in_equal(): +def test_not_in_equal() -> None: assert NotIn(Reference("foo"), ("hello",)) == NotEqualTo(term=Reference(name="foo"), literal="hello") -def test_bind_in(table_schema_simple: Schema): +def test_bind_in(table_schema_simple: Schema) -> None: bound = BoundIn( BoundReference(table_schema_simple.find_field(1), table_schema_simple.accessor_for_field(1)), {literal("hello"), literal("world")}, @@ -266,7 +268,7 @@ def test_bind_in(table_schema_simple: Schema): assert In(Reference("foo"), ("hello", "world")).bind(table_schema_simple) == bound -def test_bind_in_invert(table_schema_simple: Schema): +def test_bind_in_invert(table_schema_simple: Schema) -> None: bound = BoundIn( BoundReference(table_schema_simple.find_field(1), table_schema_simple.accessor_for_field(1)), {literal("hello"), literal("world")}, @@ -277,7 +279,7 @@ def test_bind_in_invert(table_schema_simple: Schema): ) -def test_bind_not_in_invert(table_schema_simple: Schema): +def test_bind_not_in_invert(table_schema_simple: Schema) -> None: bound = BoundNotIn( BoundReference(table_schema_simple.find_field(1), table_schema_simple.accessor_for_field(1)), {literal("hello"), literal("world")}, @@ -288,7 +290,7 @@ def test_bind_not_in_invert(table_schema_simple: Schema): ) -def test_bind_dedup(table_schema_simple: Schema): +def test_bind_dedup(table_schema_simple: Schema) -> None: bound = BoundIn( BoundReference(table_schema_simple.find_field(1), table_schema_simple.accessor_for_field(1)), {literal("hello"), literal("world")}, @@ -296,14 +298,14 @@ def test_bind_dedup(table_schema_simple: Schema): assert In(Reference("foo"), ("hello", "world", "world")).bind(table_schema_simple) == bound -def test_bind_dedup_to_eq(table_schema_simple: Schema): +def test_bind_dedup_to_eq(table_schema_simple: Schema) -> None: bound = BoundEqualTo( BoundReference(table_schema_simple.find_field(1), table_schema_simple.accessor_for_field(1)), literal("hello") ) assert In(Reference("foo"), ("hello", "hello")).bind(table_schema_simple) == bound -def test_bound_equal_to_invert(table_schema_simple: Schema): +def test_bound_equal_to_invert(table_schema_simple: Schema) -> None: bound = BoundEqualTo( BoundReference(table_schema_simple.find_field(1), table_schema_simple.accessor_for_field(1)), literal("hello") ) @@ -316,7 +318,7 @@ def test_bound_equal_to_invert(table_schema_simple: Schema): ) -def test_bound_not_equal_to_invert(table_schema_simple: Schema): +def test_bound_not_equal_to_invert(table_schema_simple: Schema) -> None: bound = BoundNotEqualTo( BoundReference(table_schema_simple.find_field(1), table_schema_simple.accessor_for_field(1)), literal("hello") ) @@ -329,7 +331,7 @@ def test_bound_not_equal_to_invert(table_schema_simple: Schema): ) -def test_bound_greater_than_or_equal_invert(table_schema_simple: Schema): +def test_bound_greater_than_or_equal_invert(table_schema_simple: Schema) -> None: bound = BoundGreaterThanOrEqual( BoundReference(table_schema_simple.find_field(1), table_schema_simple.accessor_for_field(1)), literal("hello") ) @@ -342,7 +344,7 @@ def test_bound_greater_than_or_equal_invert(table_schema_simple: Schema): ) -def test_bound_greater_than_invert(table_schema_simple: Schema): +def test_bound_greater_than_invert(table_schema_simple: Schema) -> None: bound = BoundGreaterThan( BoundReference(table_schema_simple.find_field(1), table_schema_simple.accessor_for_field(1)), literal("hello") ) @@ -355,7 +357,7 @@ def test_bound_greater_than_invert(table_schema_simple: Schema): ) -def test_bound_less_than_invert(table_schema_simple: Schema): +def test_bound_less_than_invert(table_schema_simple: Schema) -> None: bound = BoundLessThan( BoundReference(table_schema_simple.find_field(1), table_schema_simple.accessor_for_field(1)), literal("hello") ) @@ -368,7 +370,7 @@ def test_bound_less_than_invert(table_schema_simple: Schema): ) -def test_bound_less_than_or_equal_invert(table_schema_simple: Schema): +def test_bound_less_than_or_equal_invert(table_schema_simple: Schema) -> None: bound = BoundLessThanOrEqual( BoundReference(table_schema_simple.find_field(1), table_schema_simple.accessor_for_field(1)), literal("hello") ) @@ -381,16 +383,16 @@ def test_bound_less_than_or_equal_invert(table_schema_simple: Schema): ) -def test_not_equal_to_invert(): +def test_not_equal_to_invert() -> None: bound = NotEqualTo( - term=BoundReference( + term=BoundReference( # type: ignore field=NestedField(field_id=1, name="foo", field_type=StringType(), required=False), accessor=Accessor(position=0, inner=None), ), literal="hello", ) assert ~bound == EqualTo( - term=BoundReference( + term=BoundReference( # type: ignore field=NestedField(field_id=1, name="foo", field_type=StringType(), required=False), accessor=Accessor(position=0, inner=None), ), @@ -398,16 +400,16 @@ def test_not_equal_to_invert(): ) -def test_greater_than_or_equal_invert(): +def test_greater_than_or_equal_invert() -> None: bound = GreaterThanOrEqual( - term=BoundReference( + term=BoundReference( # type: ignore field=NestedField(field_id=1, name="foo", field_type=StringType(), required=False), accessor=Accessor(position=0, inner=None), ), literal="hello", ) assert ~bound == LessThan( - term=BoundReference( + term=BoundReference( # type: ignore field=NestedField(field_id=1, name="foo", field_type=StringType(), required=False), accessor=Accessor(position=0, inner=None), ), @@ -415,16 +417,16 @@ def test_greater_than_or_equal_invert(): ) -def test_less_than_or_equal_invert(): +def test_less_than_or_equal_invert() -> None: bound = LessThanOrEqual( - term=BoundReference( + term=BoundReference( # type: ignore field=NestedField(field_id=1, name="foo", field_type=StringType(), required=False), accessor=Accessor(position=0, inner=None), ), literal="hello", ) assert ~bound == GreaterThan( - term=BoundReference( + term=BoundReference( # type: ignore field=NestedField(field_id=1, name="foo", field_type=StringType(), required=False), accessor=Accessor(position=0, inner=None), ), @@ -444,9 +446,9 @@ def test_less_than_or_equal_invert(): LessThanOrEqual(Reference("foo"), "hello"), ], ) -def test_bind(pred, table_schema_simple: Schema): - assert pred.bind(table_schema_simple, case_sensitive=True).term.field == table_schema_simple.find_field( - pred.term.name, case_sensitive=True +def test_bind(pred: UnboundPredicate[Any], table_schema_simple: Schema) -> None: + assert pred.bind(table_schema_simple, case_sensitive=True).term.field == table_schema_simple.find_field( # type: ignore + pred.term.name, case_sensitive=True # type: ignore ) @@ -463,9 +465,9 @@ def test_bind(pred, table_schema_simple: Schema): LessThanOrEqual(Reference("Bar"), 5), ], ) -def test_bind_case_insensitive(pred, table_schema_simple: Schema): - assert pred.bind(table_schema_simple, case_sensitive=False).term.field == table_schema_simple.find_field( - pred.term.name, case_sensitive=False +def test_bind_case_insensitive(pred: UnboundPredicate[Any], table_schema_simple: Schema) -> None: + assert pred.bind(table_schema_simple, case_sensitive=False).term.field == table_schema_simple.find_field( # type: ignore + pred.term.name, case_sensitive=False # type: ignore ) @@ -497,7 +499,7 @@ def test_bind_case_insensitive(pred, table_schema_simple: Schema): ), ], ) -def test_eq(exp, testexpra, testexprb): +def test_eq(exp: BooleanExpression, testexpra: BooleanExpression, testexprb: BooleanExpression) -> None: assert exp == testexpra and exp != testexprb @@ -533,7 +535,7 @@ def test_eq(exp, testexpra, testexprb): ), ], ) -def test_negate(lhs, rhs): +def test_negate(lhs: BooleanExpression, rhs: BooleanExpression) -> None: assert ~lhs == rhs @@ -551,7 +553,7 @@ def test_negate(lhs, rhs): (Not(Not(ExpressionA())), ExpressionA()), ], ) -def test_reduce(lhs, rhs): +def test_reduce(lhs: BooleanExpression, rhs: BooleanExpression) -> None: assert lhs == rhs @@ -569,16 +571,16 @@ def test_reduce(lhs, rhs): (Not(AlwaysFalse()), AlwaysTrue()), ], ) -def test_base_AlwaysTrue_base_AlwaysFalse(lhs, rhs): +def test_base_AlwaysTrue_base_AlwaysFalse(lhs: BooleanExpression, rhs: BooleanExpression) -> None: assert lhs == rhs -def test_invert_always(): +def test_invert_always() -> None: assert ~AlwaysFalse() == AlwaysTrue() assert ~AlwaysTrue() == AlwaysFalse() -def test_accessor_base_class(foo_struct): +def test_accessor_base_class(foo_struct: FooStruct) -> None: """Test retrieving a value at a position of a container using an accessor""" uuid_value = uuid.uuid4() @@ -683,7 +685,7 @@ def test_always_false() -> None: assert always_false == eval(repr(always_false)) -def test_bound_reference_field_property(): +def test_bound_reference_field_property() -> None: field = NestedField(field_id=1, name="foo", field_type=StringType(), required=False) position1_accessor = Accessor(position=1) bound_ref = BoundReference(field=field, accessor=position1_accessor) @@ -897,22 +899,22 @@ def test_bound_reference_eval(table_schema_simple: Schema, foo_struct: FooStruct assert bound_ref3.eval(foo_struct) is True -def test_non_primitive_from_byte_buffer(): +def test_non_primitive_from_byte_buffer() -> None: with pytest.raises(ValueError) as exc_info: _ = _from_byte_buffer(ListType(element_id=1, element_type=StringType()), b"\0x00") assert str(exc_info.value) == "Expected a PrimitiveType, got: " -def test_string_argument_unbound_unary(): +def test_string_argument_unbound_unary() -> None: assert IsNull("a") == IsNull(Reference("a")) -def test_string_argument_unbound_literal(): +def test_string_argument_unbound_literal() -> None: assert EqualTo("a", "b") == EqualTo(Reference("a"), "b") -def test_string_argument_unbound_set(): +def test_string_argument_unbound_set() -> None: assert In("a", {"b", "c"}) == In(Reference("a"), {"b", "c"}) @@ -931,34 +933,38 @@ def below_int_min() -> Literal[int]: return literal(IntegerType.min - 1) -def test_above_int_bounds_equal_to(int_schema: Schema, above_int_max, below_int_min) -> None: - assert EqualTo("a", above_int_max).bind(int_schema) is AlwaysFalse() - assert EqualTo("a", below_int_min).bind(int_schema) is AlwaysFalse() +def test_above_int_bounds_equal_to(int_schema: Schema, above_int_max: Literal[int], below_int_min: Literal[int]) -> None: + assert EqualTo[int]("a", above_int_max).bind(int_schema) is AlwaysFalse() + assert EqualTo[int]("a", below_int_min).bind(int_schema) is AlwaysFalse() -def test_above_int_bounds_not_equal_to(int_schema: Schema, above_int_max, below_int_min) -> None: - assert NotEqualTo("a", above_int_max).bind(int_schema) is AlwaysTrue() - assert NotEqualTo("a", below_int_min).bind(int_schema) is AlwaysTrue() +def test_above_int_bounds_not_equal_to(int_schema: Schema, above_int_max: Literal[int], below_int_min: Literal[int]) -> None: + assert NotEqualTo[int]("a", above_int_max).bind(int_schema) is AlwaysTrue() + assert NotEqualTo[int]("a", below_int_min).bind(int_schema) is AlwaysTrue() -def test_above_int_bounds_less_than(int_schema: Schema, above_int_max, below_int_min) -> None: - assert LessThan("a", above_int_max).bind(int_schema) is AlwaysTrue() - assert LessThan("a", below_int_min).bind(int_schema) is AlwaysFalse() +def test_above_int_bounds_less_than(int_schema: Schema, above_int_max: Literal[int], below_int_min: Literal[int]) -> None: + assert LessThan[int]("a", above_int_max).bind(int_schema) is AlwaysTrue() + assert LessThan[int]("a", below_int_min).bind(int_schema) is AlwaysFalse() -def test_above_int_bounds_less_than_or_equal(int_schema: Schema, above_int_max, below_int_min) -> None: - assert LessThanOrEqual("a", above_int_max).bind(int_schema) is AlwaysTrue() - assert LessThanOrEqual("a", below_int_min).bind(int_schema) is AlwaysFalse() +def test_above_int_bounds_less_than_or_equal( + int_schema: Schema, above_int_max: Literal[int], below_int_min: Literal[int] +) -> None: + assert LessThanOrEqual[int]("a", above_int_max).bind(int_schema) is AlwaysTrue() + assert LessThanOrEqual[int]("a", below_int_min).bind(int_schema) is AlwaysFalse() -def test_above_int_bounds_greater_than(int_schema: Schema, above_int_max, below_int_min) -> None: - assert GreaterThan("a", above_int_max).bind(int_schema) is AlwaysFalse() - assert GreaterThan("a", below_int_min).bind(int_schema) is AlwaysTrue() +def test_above_int_bounds_greater_than(int_schema: Schema, above_int_max: Literal[int], below_int_min: Literal[int]) -> None: + assert GreaterThan[int]("a", above_int_max).bind(int_schema) is AlwaysFalse() + assert GreaterThan[int]("a", below_int_min).bind(int_schema) is AlwaysTrue() -def test_above_int_bounds_greater_than_or_equal(int_schema: Schema, above_int_max, below_int_min) -> None: - assert GreaterThanOrEqual("a", above_int_max).bind(int_schema) is AlwaysFalse() - assert GreaterThanOrEqual("a", below_int_min).bind(int_schema) is AlwaysTrue() +def test_above_int_bounds_greater_than_or_equal( + int_schema: Schema, above_int_max: Literal[int], below_int_min: Literal[int] +) -> None: + assert GreaterThanOrEqual[int]("a", above_int_max).bind(int_schema) is AlwaysFalse() + assert GreaterThanOrEqual[int]("a", below_int_min).bind(int_schema) is AlwaysTrue() @pytest.fixture @@ -976,34 +982,46 @@ def below_float_min() -> Literal[float]: return literal(FloatType.min * 2) -def test_above_float_bounds_equal_to(float_schema: Schema, above_float_max, below_float_min) -> None: - assert EqualTo("a", above_float_max).bind(float_schema) is AlwaysFalse() - assert EqualTo("a", below_float_min).bind(float_schema) is AlwaysFalse() +def test_above_float_bounds_equal_to( + float_schema: Schema, above_float_max: Literal[float], below_float_min: Literal[float] +) -> None: + assert EqualTo[float]("a", above_float_max).bind(float_schema) is AlwaysFalse() + assert EqualTo[float]("a", below_float_min).bind(float_schema) is AlwaysFalse() -def test_above_float_bounds_not_equal_to(float_schema: Schema, above_float_max, below_float_min) -> None: - assert NotEqualTo("a", above_float_max).bind(float_schema) is AlwaysTrue() - assert NotEqualTo("a", below_float_min).bind(float_schema) is AlwaysTrue() +def test_above_float_bounds_not_equal_to( + float_schema: Schema, above_float_max: Literal[float], below_float_min: Literal[float] +) -> None: + assert NotEqualTo[float]("a", above_float_max).bind(float_schema) is AlwaysTrue() + assert NotEqualTo[float]("a", below_float_min).bind(float_schema) is AlwaysTrue() -def test_above_float_bounds_less_than(float_schema: Schema, above_float_max, below_float_min) -> None: - assert LessThan("a", above_float_max).bind(float_schema) is AlwaysTrue() - assert LessThan("a", below_float_min).bind(float_schema) is AlwaysFalse() +def test_above_float_bounds_less_than( + float_schema: Schema, above_float_max: Literal[float], below_float_min: Literal[float] +) -> None: + assert LessThan[float]("a", above_float_max).bind(float_schema) is AlwaysTrue() + assert LessThan[float]("a", below_float_min).bind(float_schema) is AlwaysFalse() -def test_above_float_bounds_less_than_or_equal(float_schema: Schema, above_float_max, below_float_min) -> None: - assert LessThanOrEqual("a", above_float_max).bind(float_schema) is AlwaysTrue() - assert LessThanOrEqual("a", below_float_min).bind(float_schema) is AlwaysFalse() +def test_above_float_bounds_less_than_or_equal( + float_schema: Schema, above_float_max: Literal[float], below_float_min: Literal[float] +) -> None: + assert LessThanOrEqual[float]("a", above_float_max).bind(float_schema) is AlwaysTrue() + assert LessThanOrEqual[float]("a", below_float_min).bind(float_schema) is AlwaysFalse() -def test_above_float_bounds_greater_than(float_schema: Schema, above_float_max, below_float_min) -> None: - assert GreaterThan("a", above_float_max).bind(float_schema) is AlwaysFalse() - assert GreaterThan("a", below_float_min).bind(float_schema) is AlwaysTrue() +def test_above_float_bounds_greater_than( + float_schema: Schema, above_float_max: Literal[float], below_float_min: Literal[float] +) -> None: + assert GreaterThan[float]("a", above_float_max).bind(float_schema) is AlwaysFalse() + assert GreaterThan[float]("a", below_float_min).bind(float_schema) is AlwaysTrue() -def test_above_float_bounds_greater_than_or_equal(float_schema: Schema, above_float_max, below_float_min) -> None: - assert GreaterThanOrEqual("a", above_float_max).bind(float_schema) is AlwaysFalse() - assert GreaterThanOrEqual("a", below_float_min).bind(float_schema) is AlwaysTrue() +def test_above_float_bounds_greater_than_or_equal( + float_schema: Schema, above_float_max: Literal[float], below_float_min: Literal[float] +) -> None: + assert GreaterThanOrEqual[float]("a", above_float_max).bind(float_schema) is AlwaysFalse() + assert GreaterThanOrEqual[float]("a", below_float_min).bind(float_schema) is AlwaysTrue() @pytest.fixture @@ -1021,34 +1039,38 @@ def below_long_min() -> Literal[float]: return literal(LongType.min - 1) -def test_above_long_bounds_equal_to(long_schema: Schema, above_long_max, below_long_min) -> None: - assert EqualTo("a", above_long_max).bind(long_schema) is AlwaysFalse() - assert EqualTo("a", below_long_min).bind(long_schema) is AlwaysFalse() +def test_above_long_bounds_equal_to(long_schema: Schema, above_long_max: Literal[int], below_long_min: Literal[int]) -> None: + assert EqualTo[int]("a", above_long_max).bind(long_schema) is AlwaysFalse() + assert EqualTo[int]("a", below_long_min).bind(long_schema) is AlwaysFalse() -def test_above_long_bounds_not_equal_to(long_schema: Schema, above_long_max, below_long_min) -> None: - assert NotEqualTo("a", above_long_max).bind(long_schema) is AlwaysTrue() - assert NotEqualTo("a", below_long_min).bind(long_schema) is AlwaysTrue() +def test_above_long_bounds_not_equal_to(long_schema: Schema, above_long_max: Literal[int], below_long_min: Literal[int]) -> None: + assert NotEqualTo[int]("a", above_long_max).bind(long_schema) is AlwaysTrue() + assert NotEqualTo[int]("a", below_long_min).bind(long_schema) is AlwaysTrue() -def test_above_long_bounds_less_than(long_schema: Schema, above_long_max, below_long_min) -> None: - assert LessThan("a", above_long_max).bind(long_schema) is AlwaysTrue() - assert LessThan("a", below_long_min).bind(long_schema) is AlwaysFalse() +def test_above_long_bounds_less_than(long_schema: Schema, above_long_max: Literal[int], below_long_min: Literal[int]) -> None: + assert LessThan[int]("a", above_long_max).bind(long_schema) is AlwaysTrue() + assert LessThan[int]("a", below_long_min).bind(long_schema) is AlwaysFalse() -def test_above_long_bounds_less_than_or_equal(long_schema: Schema, above_long_max, below_long_min) -> None: - assert LessThanOrEqual("a", above_long_max).bind(long_schema) is AlwaysTrue() - assert LessThanOrEqual("a", below_long_min).bind(long_schema) is AlwaysFalse() +def test_above_long_bounds_less_than_or_equal( + long_schema: Schema, above_long_max: Literal[int], below_long_min: Literal[int] +) -> None: + assert LessThanOrEqual[int]("a", above_long_max).bind(long_schema) is AlwaysTrue() + assert LessThanOrEqual[int]("a", below_long_min).bind(long_schema) is AlwaysFalse() -def test_above_long_bounds_greater_than(long_schema: Schema, above_long_max, below_long_min) -> None: - assert GreaterThan("a", above_long_max).bind(long_schema) is AlwaysFalse() - assert GreaterThan("a", below_long_min).bind(long_schema) is AlwaysTrue() +def test_above_long_bounds_greater_than(long_schema: Schema, above_long_max: Literal[int], below_long_min: Literal[int]) -> None: + assert GreaterThan[int]("a", above_long_max).bind(long_schema) is AlwaysFalse() + assert GreaterThan[int]("a", below_long_min).bind(long_schema) is AlwaysTrue() -def test_above_long_bounds_greater_than_or_equal(long_schema: Schema, above_long_max, below_long_min) -> None: - assert GreaterThanOrEqual("a", above_long_max).bind(long_schema) is AlwaysFalse() - assert GreaterThanOrEqual("a", below_long_min).bind(long_schema) is AlwaysTrue() +def test_above_long_bounds_greater_than_or_equal( + long_schema: Schema, above_long_max: Literal[int], below_long_min: Literal[int] +) -> None: + assert GreaterThanOrEqual[int]("a", above_long_max).bind(long_schema) is AlwaysFalse() + assert GreaterThanOrEqual[int]("a", below_long_min).bind(long_schema) is AlwaysTrue() # __ __ ___ diff --git a/tests/expressions/test_literals.py b/tests/expressions/test_literals.py index 2ca61f9e80..453f04f4ee 100644 --- a/tests/expressions/test_literals.py +++ b/tests/expressions/test_literals.py @@ -19,7 +19,12 @@ import datetime import uuid from decimal import Decimal -from typing import Set +from typing import ( + Any, + List, + Set, + Type, +) import pytest from typing_extensions import assert_type @@ -53,6 +58,7 @@ FloatType, IntegerType, LongType, + PrimitiveType, StringType, TimestampType, TimestamptzType, @@ -61,14 +67,14 @@ ) -def test_literal_from_none_error(): +def test_literal_from_none_error() -> None: with pytest.raises(TypeError) as e: - literal(None) + literal(None) # type: ignore assert "Invalid literal value: None" in str(e.value) @pytest.mark.parametrize( - "literalClass", + "literal_class", [ BooleanLiteral, LongLiteral, @@ -83,16 +89,16 @@ def test_literal_from_none_error(): BinaryLiteral, ], ) -def test_string_literal_with_none_value_error(literalClass): +def test_string_literal_with_none_value_error(literal_class: Type[PrimitiveType]) -> None: with pytest.raises(TypeError) as e: - literalClass(None) + literal_class(None) assert "Invalid literal value: None" in str(e.value) # Numeric -def test_numeric_literal_comparison(): +def test_numeric_literal_comparison() -> None: small_lit = literal(10).to(IntegerType()) big_lit = literal(1000).to(IntegerType()) assert small_lit != big_lit @@ -103,21 +109,21 @@ def test_numeric_literal_comparison(): assert big_lit >= small_lit -def test_integer_to_long_conversion(): +def test_integer_to_long_conversion() -> None: lit = literal(34).to(IntegerType()) long_lit = lit.to(LongType()) assert lit.value == long_lit.value -def test_integer_to_float_conversion(): +def test_integer_to_float_conversion() -> None: lit = literal(34).to(IntegerType()) float_lit = lit.to(FloatType()) assert lit.value == float_lit.value -def test_integer_to_double_conversion(): +def test_integer_to_double_conversion() -> None: lit = literal(34).to(IntegerType()) dbl_lit = lit.to(DoubleType()) @@ -125,15 +131,15 @@ def test_integer_to_double_conversion(): @pytest.mark.parametrize( - "decimalType, decimalValue", [(DecimalType(9, 0), "34"), (DecimalType(9, 2), "34.00"), (DecimalType(9, 4), "34.0000")] + "decimal_type, decimal_value", [(DecimalType(9, 0), "34"), (DecimalType(9, 2), "34.00"), (DecimalType(9, 4), "34.0000")] ) -def test_integer_to_decimal_conversion(decimalType, decimalValue): +def test_integer_to_decimal_conversion(decimal_type: DecimalType, decimal_value: str) -> None: lit = literal(34).to(IntegerType()) - assert lit.to(decimalType).value.as_tuple() == Decimal(decimalValue).as_tuple() + assert lit.to(decimal_type).value.as_tuple() == Decimal(decimal_value).as_tuple() # type: ignore -def test_integer_to_date_conversion(): +def test_integer_to_date_conversion() -> None: one_day = "2022-03-28" date_delta = (datetime.date.fromisoformat(one_day) - datetime.date.fromisoformat("1970-01-01")).days date_lit = literal(date_delta).to(DateType()) @@ -142,14 +148,14 @@ def test_integer_to_date_conversion(): assert date_lit.value == date_delta -def test_long_to_integer_within_bound(): +def test_long_to_integer_within_bound() -> None: lit = literal(34).to(LongType()) int_lit = lit.to(IntegerType()) assert lit.value == int_lit.value -def test_long_to_integer_outside_bound(): +def test_long_to_integer_outside_bound() -> None: big_lit = literal(IntegerType.max + 1).to(LongType()) above_max_lit = big_lit.to(IntegerType()) assert above_max_lit == IntAboveMax() @@ -159,21 +165,21 @@ def test_long_to_integer_outside_bound(): assert below_min_lit == IntBelowMin() -def test_long_to_float_conversion(): +def test_long_to_float_conversion() -> None: lit = literal(34).to(LongType()) float_lit = lit.to(FloatType()) assert lit.value == float_lit.value -def test_long_to_double_conversion(): +def test_long_to_double_conversion() -> None: lit = literal(34).to(LongType()) dbl_lit = lit.to(DoubleType()) assert lit.value == dbl_lit.value -def test_long_to_time(): +def test_long_to_time() -> None: long_lit = literal(51661919000).to(LongType()) time_lit = long_lit.to(TimeType()) @@ -181,7 +187,7 @@ def test_long_to_time(): assert time_lit.value == long_lit.value -def test_long_to_timestamp(): +def test_long_to_timestamp() -> None: long_lit = literal(1647305201).to(LongType()) timestamp_lit = long_lit.to(TimestampType()) @@ -189,15 +195,15 @@ def test_long_to_timestamp(): @pytest.mark.parametrize( - "decimalType, decimalValue", [(DecimalType(9, 0), "34"), (DecimalType(9, 2), "34.00"), (DecimalType(9, 4), "34.0000")] + "decimal_type, decimal_value", [(DecimalType(9, 0), "34"), (DecimalType(9, 2), "34.00"), (DecimalType(9, 4), "34.0000")] ) -def test_long_to_decimal_conversion(decimalType, decimalValue): +def test_long_to_decimal_conversion(decimal_type: DecimalType, decimal_value: str) -> None: lit = literal(34).to(LongType()) - assert lit.to(decimalType).value.as_tuple() == Decimal(decimalValue).as_tuple() + assert lit.to(decimal_type).value.as_tuple() == Decimal(decimal_value).as_tuple() # type: ignore -def test_float_to_double(): +def test_float_to_double() -> None: lit = literal(34.56).to(FloatType()) dbl_lit = lit.to(DoubleType()) @@ -205,22 +211,22 @@ def test_float_to_double(): @pytest.mark.parametrize( - "decimalType, decimalValue", [(DecimalType(9, 1), "34.6"), (DecimalType(9, 2), "34.56"), (DecimalType(9, 4), "34.5600")] + "decimal_type, decimal_value", [(DecimalType(9, 1), "34.6"), (DecimalType(9, 2), "34.56"), (DecimalType(9, 4), "34.5600")] ) -def test_float_to_decimal_conversion(decimalType, decimalValue): +def test_float_to_decimal_conversion(decimal_type: DecimalType, decimal_value: str) -> None: lit = literal(34.56).to(FloatType()) - assert lit.to(decimalType).value.as_tuple() == Decimal(decimalValue).as_tuple() + assert lit.to(decimal_type).value.as_tuple() == Decimal(decimal_value).as_tuple() # type: ignore -def test_double_to_float_within_bound(): +def test_double_to_float_within_bound() -> None: lit = literal(34.56).to(DoubleType()) float_lit = lit.to(FloatType()) assert lit.value == float_lit.value -def test_double_to_float_outside_bound(): +def test_double_to_float_outside_bound() -> None: big_lit = literal(FloatType.max + 1.0e37).to(DoubleType()) above_max_lit = big_lit.to(FloatType()) assert above_max_lit == FloatAboveMax() @@ -231,15 +237,15 @@ def test_double_to_float_outside_bound(): @pytest.mark.parametrize( - "decimalType, decimalValue", [(DecimalType(9, 1), "34.6"), (DecimalType(9, 2), "34.56"), (DecimalType(9, 4), "34.5600")] + "decimal_type, decimal_value", [(DecimalType(9, 1), "34.6"), (DecimalType(9, 2), "34.56"), (DecimalType(9, 4), "34.5600")] ) -def test_double_to_decimal_conversion(decimalType, decimalValue): +def test_double_to_decimal_conversion(decimal_type: DecimalType, decimal_value: str) -> None: lit = literal(34.56).to(DoubleType()) - assert lit.to(decimalType).value.as_tuple() == Decimal(decimalValue).as_tuple() + assert lit.to(decimal_type).value.as_tuple() == Decimal(decimal_value).as_tuple() # type: ignore -def test_decimal_to_decimal_conversion(): +def test_decimal_to_decimal_conversion() -> None: lit = literal(Decimal("34.11").quantize(Decimal(".01"))) assert lit.value.as_tuple() == lit.to(DecimalType(9, 2)).value.as_tuple() @@ -255,14 +261,14 @@ def test_decimal_to_decimal_conversion(): assert "Could not convert 34.11 into a decimal(9, 3)" in str(e.value) -def test_timestamp_to_date(): +def test_timestamp_to_date() -> None: epoch_lit = TimestampLiteral(int(datetime.datetime.fromisoformat("1970-01-01T01:23:45.678").timestamp() * 1_000_000)) date_lit = epoch_lit.to(DateType()) assert date_lit.value == 0 -def test_string_literal(): +def test_string_literal() -> None: sqrt2 = literal("1.414").to(StringType()) pi = literal("3.141").to(StringType()) pi_string_lit = StringLiteral("3.141") @@ -282,11 +288,11 @@ def test_string_literal(): assert str(pi) == "3.141" -def test_string_to_string_literal(): +def test_string_to_string_literal() -> None: assert literal("abc") == literal("abc").to(StringType()) -def test_string_to_date_literal(): +def test_string_to_date_literal() -> None: one_day = "2017-08-18" date_lit = literal(one_day).to(DateType()) @@ -294,17 +300,17 @@ def test_string_to_date_literal(): assert date_delta == date_lit.value -def test_string_to_time_literal(): +def test_string_to_time_literal() -> None: time_str = literal("14:21:01.919") time_lit = time_str.to(TimeType()) avro_val = 51661919000 - assert isinstance(time_lit, TimeLiteral) - assert avro_val == time_lit.value + assert isinstance(time_lit, TimeLiteral) # type: ignore + assert avro_val == time_lit.value # type: ignore -def test_string_to_timestamp_literal(): +def test_string_to_timestamp_literal() -> None: timestamp_str = literal("2017-08-18T14:21:01.919234+00:00") timestamp = timestamp_str.to(TimestamptzType()) @@ -321,35 +327,35 @@ def test_string_to_timestamp_literal(): assert avro_val == timestamp.value -def test_timestamp_with_zone_without_zone_in_literal(): +def test_timestamp_with_zone_without_zone_in_literal() -> None: timestamp_str = literal("2017-08-18T14:21:01.919234") with pytest.raises(ValueError) as e: _ = timestamp_str.to(timestamp_str.to(TimestamptzType())) assert "Missing zone offset: 2017-08-18T14:21:01.919234 (must be ISO-8601)" in str(e.value) -def test_invalid_timestamp_in_literal(): +def test_invalid_timestamp_in_literal() -> None: timestamp_str = literal("abc") with pytest.raises(ValueError) as e: _ = timestamp_str.to(timestamp_str.to(TimestamptzType())) assert "Invalid timestamp with zone: abc (must be ISO-8601)" in str(e.value) -def test_timestamp_without_zone_with_zone_in_literal(): +def test_timestamp_without_zone_with_zone_in_literal() -> None: timestamp_str = literal("2017-08-18T14:21:01.919234+07:00") with pytest.raises(ValueError) as e: _ = timestamp_str.to(TimestampType()) assert "Zone offset provided, but not expected: 2017-08-18T14:21:01.919234+07:00" in str(e.value) -def test_invalid_timestamp_with_zone_in_literal(): +def test_invalid_timestamp_with_zone_in_literal() -> None: timestamp_str = literal("abc") with pytest.raises(ValueError) as e: _ = timestamp_str.to(TimestampType()) assert "Invalid timestamp without zone: abc (must be ISO-8601)" in str(e.value) -def test_string_to_uuid_literal(): +def test_string_to_uuid_literal() -> None: expected = uuid.uuid4() uuid_str = literal(str(expected)) uuid_lit = uuid_str.to(UUIDType()) @@ -357,24 +363,24 @@ def test_string_to_uuid_literal(): assert expected == uuid_lit.value -def test_string_to_decimal_literal(): +def test_string_to_decimal_literal() -> None: decimal_str = literal("34.560") decimal_lit = decimal_str.to(DecimalType(9, 3)) - assert 3 == abs(decimal_lit.value.as_tuple().exponent) - assert Decimal("34.560").as_tuple() == decimal_lit.value.as_tuple() + assert 3 == abs(decimal_lit.value.as_tuple().exponent) # type: ignore + assert Decimal("34.560").as_tuple() == decimal_lit.value.as_tuple() # type: ignore # MISC -def test_python_date_conversion(): +def test_python_date_conversion() -> None: one_day_str = "2022-03-28" from_str_lit = literal(one_day_str).to(DateType()) - assert isinstance(from_str_lit, DateLiteral) - assert from_str_lit.value == 19079 + assert isinstance(from_str_lit, DateLiteral) # type: ignore + assert from_str_lit.value == 19079 # type: ignore @pytest.mark.parametrize( @@ -394,12 +400,12 @@ def test_python_date_conversion(): (literal(bytes([0x01, 0x02, 0x03])), FixedType(3)), ], ) -def test_identity_conversions(lit, primitive_type): +def test_identity_conversions(lit: Literal[Any], primitive_type: PrimitiveType) -> None: expected = lit.to(primitive_type) assert expected is expected.to(primitive_type) -def test_fixed_literal(): +def test_fixed_literal() -> None: fixed_lit012 = literal(bytes([0x00, 0x01, 0x02])) fixed_lit013 = literal(bytes([0x00, 0x01, 0x03])) assert fixed_lit012 == fixed_lit012 @@ -410,7 +416,7 @@ def test_fixed_literal(): assert fixed_lit013 >= fixed_lit012 -def test_binary_literal(): +def test_binary_literal() -> None: bin_lit012 = literal(bytes([0x00, 0x01, 0x02])) bin_lit013 = literal(bytes([0x00, 0x01, 0x03])) assert bin_lit012 == bin_lit012 @@ -422,7 +428,7 @@ def test_binary_literal(): # None related -def test_raise_on_comparison_to_none(): +def test_raise_on_comparison_to_none() -> None: bin_lit012 = literal(bytes([0x00, 0x01, 0x02])) fixed_lit012 = literal(bytes([0x00, 0x01, 0x02])) @@ -451,7 +457,7 @@ def test_raise_on_comparison_to_none(): _ = fixed_lit012 >= None -def test_binary_to_fixed(): +def test_binary_to_fixed() -> None: lit = literal(bytes([0x00, 0x01, 0x02])) fixed_lit = lit.to(FixedType(3)) assert fixed_lit is not None @@ -462,7 +468,7 @@ def test_binary_to_fixed(): assert "Cannot convert BinaryLiteral into fixed[4], different length: 4 <> 3" in str(e.value) -def test_binary_to_smaller_fixed_none(): +def test_binary_to_smaller_fixed_none() -> None: lit = literal(bytes([0x00, 0x01, 0x02])) with pytest.raises(TypeError) as e: @@ -470,21 +476,21 @@ def test_binary_to_smaller_fixed_none(): assert "Cannot convert BinaryLiteral into fixed[2], different length: 2 <> 3" in str(e.value) -def test_fixed_to_binary(): +def test_fixed_to_binary() -> None: lit = literal(bytes([0x00, 0x01, 0x02])).to(FixedType(3)) binary_lit = lit.to(BinaryType()) assert binary_lit is not None assert lit.value == binary_lit.value -def test_fixed_to_smaller_fixed_none(): +def test_fixed_to_smaller_fixed_none() -> None: lit = literal(bytes([0x00, 0x01, 0x02])).to(FixedType(3)) with pytest.raises(ValueError) as e: lit.to(lit.to(FixedType(2))) assert "Could not convert b'\\x00\\x01\\x02' into a fixed[2]" in str(e.value) -def test_above_max_float(): +def test_above_max_float() -> None: a = FloatAboveMax() # singleton assert a == FloatAboveMax() @@ -495,7 +501,7 @@ def test_above_max_float(): assert a.to(FloatType()) == FloatAboveMax() -def test_below_min_float(): +def test_below_min_float() -> None: b = FloatBelowMin() # singleton assert b == FloatBelowMin() @@ -506,7 +512,7 @@ def test_below_min_float(): assert b.to(FloatType()) == FloatBelowMin() -def test_above_max_int(): +def test_above_max_int() -> None: a = IntAboveMax() # singleton assert a == IntAboveMax() @@ -517,7 +523,7 @@ def test_above_max_int(): assert a.to(IntegerType()) == IntAboveMax() -def test_below_min_int(): +def test_below_min_int() -> None: b = IntBelowMin() # singleton assert b == IntBelowMin() @@ -527,7 +533,7 @@ def test_below_min_int(): assert b.to(IntegerType()) == IntBelowMin() -def test_invalid_boolean_conversions(): +def test_invalid_boolean_conversions() -> None: assert_invalid_conversions( literal(True), [ @@ -547,7 +553,7 @@ def test_invalid_boolean_conversions(): ) -def test_invalid_long_conversions(): +def test_invalid_long_conversions() -> None: assert_invalid_conversions( literal(34).to(LongType()), [BooleanType(), StringType(), UUIDType(), FixedType(1), BinaryType()], @@ -578,7 +584,7 @@ def test_invalid_long_conversions(): BinaryType(), ], ) -def test_invalid_float_conversions(lit, test_type): +def test_invalid_float_conversions(lit: Literal[Any], test_type: PrimitiveType) -> None: with pytest.raises(TypeError): _ = lit.to(test_type) @@ -602,11 +608,11 @@ def test_invalid_float_conversions(lit, test_type): BinaryType(), ], ) -def test_invalid_datetime_conversions(lit, test_type): - assert_invalid_conversions(lit, (test_type,)) +def test_invalid_datetime_conversions(lit: Literal[Any], test_type: PrimitiveType) -> None: + assert_invalid_conversions(lit, [test_type]) -def test_invalid_time_conversions(): +def test_invalid_time_conversions() -> None: assert_invalid_conversions( literal("14:21:01.919").to(TimeType()), [ @@ -627,7 +633,7 @@ def test_invalid_time_conversions(): ) -def test_invalid_timestamp_conversions(): +def test_invalid_timestamp_conversions() -> None: assert_invalid_conversions( literal("2017-08-18T14:21:01.919").to(TimestampType()), [ @@ -646,14 +652,14 @@ def test_invalid_timestamp_conversions(): ) -def test_invalid_decimal_conversion_scale(): +def test_invalid_decimal_conversion_scale() -> None: lit = literal(Decimal("34.11")) with pytest.raises(ValueError) as e: lit.to(DecimalType(9, 4)) assert "Could not convert 34.11 into a decimal(9, 4)" in str(e.value) -def test_invalid_decimal_conversions(): +def test_invalid_decimal_conversions() -> None: assert_invalid_conversions( literal(Decimal("34.11")), [ @@ -670,14 +676,14 @@ def test_invalid_decimal_conversions(): ) -def test_invalid_string_conversions(): +def test_invalid_string_conversions() -> None: assert_invalid_conversions( literal("abc"), [BooleanType(), FloatType(), DoubleType(), FixedType(1), BinaryType()], ) -def test_invalid_uuid_conversions(): +def test_invalid_uuid_conversions() -> None: assert_invalid_conversions( literal(uuid.uuid4()), [ @@ -698,7 +704,7 @@ def test_invalid_uuid_conversions(): ) -def test_invalid_fixed_conversions(): +def test_invalid_fixed_conversions() -> None: assert_invalid_conversions( literal(bytes([0x00, 0x01, 0x02])).to(FixedType(3)), [ @@ -718,7 +724,7 @@ def test_invalid_fixed_conversions(): ) -def test_invalid_binary_conversions(): +def test_invalid_binary_conversions() -> None: assert_invalid_conversions( literal(bytes([0x00, 0x01, 0x02])), [ @@ -738,13 +744,13 @@ def test_invalid_binary_conversions(): ) -def assert_invalid_conversions(lit, types=None): +def assert_invalid_conversions(lit: Literal[Any], types: List[PrimitiveType]) -> None: for type_var in types: with pytest.raises(TypeError): _ = lit.to(type_var) -def test_compare_floats(): +def test_compare_floats() -> None: lhs = literal(18.15).to(FloatType()) rhs = literal(19.25).to(FloatType()) assert lhs != rhs @@ -754,45 +760,45 @@ def test_compare_floats(): assert not lhs >= rhs -def test_string_to_int_max_value(): +def test_string_to_int_max_value() -> None: assert isinstance(literal(str(IntegerType.max + 1)).to(IntegerType()), IntAboveMax) -def test_string_to_int_min_value(): +def test_string_to_int_min_value() -> None: assert isinstance(literal(str(IntegerType.min - 1)).to(IntegerType()), IntBelowMin) -def test_string_to_integer_type_invalid_value(): +def test_string_to_integer_type_invalid_value() -> None: with pytest.raises(ValueError) as e: _ = literal("abc").to(IntegerType()) assert "Could not convert abc into a int" in str(e.value) -def test_string_to_long_type_invalid_value(): +def test_string_to_long_type_invalid_value() -> None: with pytest.raises(ValueError) as e: _ = literal("abc").to(LongType()) assert "Could not convert abc into a long" in str(e.value) -def test_string_to_date_type_invalid_value(): +def test_string_to_date_type_invalid_value() -> None: with pytest.raises(ValueError) as e: _ = literal("abc").to(DateType()) assert "Could not convert abc into a date" in str(e.value) -def test_string_to_time_type_invalid_value(): +def test_string_to_time_type_invalid_value() -> None: with pytest.raises(ValueError) as e: _ = literal("abc").to(TimeType()) assert "Could not convert abc into a time" in str(e.value) -def test_string_to_decimal_type_invalid_value(): +def test_string_to_decimal_type_invalid_value() -> None: with pytest.raises(ValueError) as e: _ = literal("18.15").to(DecimalType(10, 0)) assert "Could not convert 18.15 into a decimal(10, 0), scales differ 0 <> 2" in str(e.value) -def test_decimal_literal_increment(): +def test_decimal_literal_increment() -> None: dec = DecimalLiteral(Decimal("10.123")) # Twice to check that we don't mutate the value assert dec.increment() == DecimalLiteral(Decimal("10.124")) @@ -801,7 +807,7 @@ def test_decimal_literal_increment(): assert dec.increment().value.as_tuple() == Decimal("10.124").as_tuple() -def test_decimal_literal_dencrement(): +def test_decimal_literal_dencrement() -> None: dec = DecimalLiteral(Decimal("10.123")) # Twice to check that we don't mutate the value assert dec.decrement() == DecimalLiteral(Decimal("10.122")) diff --git a/tests/expressions/test_parser.py b/tests/expressions/test_parser.py index 47704be4c3..3eaa6d2ba7 100644 --- a/tests/expressions/test_parser.py +++ b/tests/expressions/test_parser.py @@ -39,112 +39,112 @@ ) -def test_true(): +def test_true() -> None: assert AlwaysTrue() == parser.parse("true") -def test_false(): +def test_false() -> None: assert AlwaysFalse() == parser.parse("false") -def test_is_null(): +def test_is_null() -> None: assert IsNull("x") == parser.parse("x is null") assert IsNull("x") == parser.parse("x IS NULL") -def test_not_null(): +def test_not_null() -> None: assert NotNull("x") == parser.parse("x is not null") assert NotNull("x") == parser.parse("x IS NOT NULL") -def test_is_nan(): +def test_is_nan() -> None: assert IsNaN("x") == parser.parse("x is nan") assert IsNaN("x") == parser.parse("x IS NAN") -def test_not_nan(): +def test_not_nan() -> None: assert NotNaN("x") == parser.parse("x is not nan") assert NotNaN("x") == parser.parse("x IS NOT NaN") -def test_less_than(): +def test_less_than() -> None: assert LessThan("x", 5) == parser.parse("x < 5") assert LessThan("x", "a") == parser.parse("'a' > x") -def test_less_than_or_equal(): +def test_less_than_or_equal() -> None: assert LessThanOrEqual("x", 5) == parser.parse("x <= 5") assert LessThanOrEqual("x", "a") == parser.parse("'a' >= x") -def test_greater_than(): +def test_greater_than() -> None: assert GreaterThan("x", 5) == parser.parse("x > 5") assert GreaterThan("x", "a") == parser.parse("'a' < x") -def test_greater_than_or_equal(): +def test_greater_than_or_equal() -> None: assert GreaterThanOrEqual("x", 5) == parser.parse("x <= 5") assert GreaterThanOrEqual("x", "a") == parser.parse("'a' >= x") -def test_equal_to(): +def test_equal_to() -> None: assert EqualTo("x", 5) == parser.parse("x = 5") assert EqualTo("x", "a") == parser.parse("'a' = x") assert EqualTo("x", "a") == parser.parse("x == 'a'") assert EqualTo("x", 5) == parser.parse("5 == x") -def test_not_equal_to(): +def test_not_equal_to() -> None: assert NotEqualTo("x", 5) == parser.parse("x != 5") assert NotEqualTo("x", "a") == parser.parse("'a' != x") assert NotEqualTo("x", "a") == parser.parse("x <> 'a'") assert NotEqualTo("x", 5) == parser.parse("5 <> x") -def test_in(): +def test_in() -> None: assert In("x", {5, 6, 7}) == parser.parse("x in (5, 6, 7)") assert In("x", {"a", "b", "c"}) == parser.parse("x IN ('a', 'b', 'c')") -def test_in_different_types(): +def test_in_different_types() -> None: with pytest.raises(ParseException): parser.parse("x in (5, 'a')") -def test_not_in(): +def test_not_in() -> None: assert NotIn("x", {5, 6, 7}) == parser.parse("x not in (5, 6, 7)") assert NotIn("x", {"a", "b", "c"}) == parser.parse("x NOT IN ('a', 'b', 'c')") -def test_not_in_different_types(): +def test_not_in_different_types() -> None: with pytest.raises(ParseException): parser.parse("x not in (5, 'a')") -def test_simple_and(): +def test_simple_and() -> None: assert And(GreaterThanOrEqual("x", 5), LessThan("x", 10)) == parser.parse("5 <= x and x < 10") -def test_and_with_not(): +def test_and_with_not() -> None: assert And(Not(GreaterThanOrEqual("x", 5)), LessThan("x", 10)) == parser.parse("not 5 <= x and x < 10") assert And(GreaterThanOrEqual("x", 5), Not(LessThan("x", 10))) == parser.parse("5 <= x and not x < 10") -def test_or_with_not(): +def test_or_with_not() -> None: assert Or(Not(LessThan("x", 5)), GreaterThan("x", 10)) == parser.parse("not x < 5 or 10 < x") assert Or(LessThan("x", 5), Not(GreaterThan("x", 10))) == parser.parse("x < 5 or not 10 < x") -def test_simple_or(): +def test_simple_or() -> None: assert Or(LessThan("x", 5), GreaterThan("x", 10)) == parser.parse("x < 5 or 10 < x") -def test_and_or_without_parens(): +def test_and_or_without_parens() -> None: assert Or(And(NotNull("x"), LessThan("x", 5)), GreaterThan("x", 10)) == parser.parse("x is not null and x < 5 or 10 < x") assert Or(IsNull("x"), And(GreaterThanOrEqual("x", 5), LessThan("x", 10))) == parser.parse("x is null or 5 <= x and x < 10") -def test_and_or_with_parens(): +def test_and_or_with_parens() -> None: assert And(NotNull("x"), Or(LessThan("x", 5), GreaterThan("x", 10))) == parser.parse("x is not null and (x < 5 or 10 < x)") assert Or(IsNull("x"), And(GreaterThanOrEqual("x", 5), Not(LessThan("x", 10)))) == parser.parse( "(x is null) or (5 <= x) and not(x < 10)" diff --git a/tests/expressions/test_projection.py b/tests/expressions/test_projection.py index 73940f649d..4d0c2c1346 100644 --- a/tests/expressions/test_projection.py +++ b/tests/expressions/test_projection.py @@ -105,7 +105,7 @@ def id_and_bucket_spec() -> PartitionSpec: ) -def test_identity_projection(schema: Schema, id_spec: PartitionSpec): +def test_identity_projection(schema: Schema, id_spec: PartitionSpec) -> None: predicates = [ NotNull("id"), IsNull("id"), @@ -138,7 +138,7 @@ def test_identity_projection(schema: Schema, id_spec: PartitionSpec): assert expected[index] == expr -def test_bucket_projection(schema: Schema, bucket_spec: PartitionSpec): +def test_bucket_projection(schema: Schema, bucket_spec: PartitionSpec) -> None: predicates = [ NotNull("data"), IsNull("data"), @@ -171,7 +171,7 @@ def test_bucket_projection(schema: Schema, bucket_spec: PartitionSpec): assert expected[index] == expr -def test_hour_projection(schema: Schema, hour_spec: PartitionSpec): +def test_hour_projection(schema: Schema, hour_spec: PartitionSpec) -> None: predicates = [ NotNull("event_ts"), IsNull("event_ts"), @@ -204,7 +204,7 @@ def test_hour_projection(schema: Schema, hour_spec: PartitionSpec): assert expected[index] == expr, predicate -def test_day_projection(schema: Schema, day_spec: PartitionSpec): +def test_day_projection(schema: Schema, day_spec: PartitionSpec) -> None: predicates = [ NotNull("event_ts"), IsNull("event_ts"), @@ -237,7 +237,7 @@ def test_day_projection(schema: Schema, day_spec: PartitionSpec): assert expected[index] == expr, predicate -def test_date_day_projection(schema: Schema, day_spec: PartitionSpec): +def test_date_day_projection(schema: Schema, day_spec: PartitionSpec) -> None: predicates = [ NotNull("event_date"), IsNull("event_date"), @@ -270,7 +270,7 @@ def test_date_day_projection(schema: Schema, day_spec: PartitionSpec): assert expected[index] == expr, predicate -def test_string_truncate_projection(schema: Schema, truncate_str_spec: PartitionSpec): +def test_string_truncate_projection(schema: Schema, truncate_str_spec: PartitionSpec) -> None: predicates = [ NotNull("data"), IsNull("data"), @@ -303,7 +303,7 @@ def test_string_truncate_projection(schema: Schema, truncate_str_spec: Partition assert expected[index] == expr, predicate -def test_int_truncate_projection(schema: Schema, truncate_int_spec: PartitionSpec): +def test_int_truncate_projection(schema: Schema, truncate_int_spec: PartitionSpec) -> None: predicates = [ NotNull("id"), IsNull("id"), @@ -336,43 +336,43 @@ def test_int_truncate_projection(schema: Schema, truncate_int_spec: PartitionSpe assert expected[index] == expr, predicate -def test_projection_case_sensitive(schema: Schema, id_spec: PartitionSpec): +def test_projection_case_sensitive(schema: Schema, id_spec: PartitionSpec) -> None: project = inclusive_projection(schema, id_spec) with pytest.raises(ValueError) as exc_info: project(NotNull("ID")) assert str(exc_info) == "Could not find field with name ID, case_sensitive=True" -def test_projection_case_insensitive(schema: Schema, id_spec: PartitionSpec): +def test_projection_case_insensitive(schema: Schema, id_spec: PartitionSpec) -> None: project = inclusive_projection(schema, id_spec, case_sensitive=False) assert NotNull("id_part") == project(NotNull("ID")) -def test_projection_empty_spec(schema: Schema, empty_spec: PartitionSpec): +def test_projection_empty_spec(schema: Schema, empty_spec: PartitionSpec) -> None: project = inclusive_projection(schema, empty_spec) assert AlwaysTrue() == project(And(LessThan("id", 5), NotNull("data"))) -def test_and_projection_multiple_projected_fields(schema: Schema, id_and_bucket_spec: PartitionSpec): +def test_and_projection_multiple_projected_fields(schema: Schema, id_and_bucket_spec: PartitionSpec) -> None: project = inclusive_projection(schema, id_and_bucket_spec) assert project(And(LessThan("id", 5), In("data", {"a", "b", "c"}))) == And( LessThan("id_part", 5), In("data_bucket", {2, 3, 15}) ) -def test_or_projection_multiple_projected_fields(schema: Schema, id_and_bucket_spec: PartitionSpec): +def test_or_projection_multiple_projected_fields(schema: Schema, id_and_bucket_spec: PartitionSpec) -> None: project = inclusive_projection(schema, id_and_bucket_spec) assert project(Or(LessThan("id", 5), In("data", {"a", "b", "c"}))) == Or( LessThan("id_part", 5), In("data_bucket", {2, 3, 15}) ) -def test_not_projection_multiple_projected_fields(schema: Schema, id_and_bucket_spec: PartitionSpec): +def test_not_projection_multiple_projected_fields(schema: Schema, id_and_bucket_spec: PartitionSpec) -> None: project = inclusive_projection(schema, id_and_bucket_spec) # Not causes In to be rewritten to NotIn, which cannot be projected assert project(Not(Or(LessThan("id", 5), In("data", {"a", "b", "c"})))) == GreaterThanOrEqual("id_part", 5) -def test_projection_partial_projected_fields(schema: Schema, id_spec: PartitionSpec): +def test_projection_partial_projected_fields(schema: Schema, id_spec: PartitionSpec) -> None: project = inclusive_projection(schema, id_spec) assert project(And(LessThan("id", 5), In("data", {"a", "b", "c"}))) == LessThan("id_part", 5) diff --git a/tests/expressions/test_visitors.py b/tests/expressions/test_visitors.py index 0552843413..e179ccf05f 100644 --- a/tests/expressions/test_visitors.py +++ b/tests/expressions/test_visitors.py @@ -83,24 +83,24 @@ class ExpressionA(BooleanExpression, Singleton): - def __invert__(self): + def __invert__(self) -> BooleanExpression: return ExpressionB() - def __repr__(self): + def __repr__(self) -> str: return "ExpressionA()" - def __str__(self): + def __str__(self) -> str: return "testexpra" class ExpressionB(BooleanExpression, Singleton): - def __invert__(self): + def __invert__(self) -> BooleanExpression: return ExpressionA() - def __repr__(self): + def __repr__(self) -> str: return "ExpressionB()" - def __str__(self): + def __str__(self) -> str: return "testexprb" @@ -111,7 +111,7 @@ class ExampleVisitor(BooleanExpressionVisitor[List[str]]): visited in an expected order by the `visit` method. """ - def __init__(self): + def __init__(self) -> None: self.visit_history: List[str] = [] def visit_true(self) -> List[str]: @@ -169,7 +169,7 @@ class FooBoundBooleanExpressionVisitor(BoundBooleanExpressionVisitor[List[str]]) visited in an expected order by the `visit` method. """ - def __init__(self): + def __init__(self) -> None: self.visit_history: List[str] = [] def visit_in(self, term: BoundTerm[Any], literals: Set[Any]) -> List[str]: @@ -247,7 +247,7 @@ def visit_or(self, left_result: List[str], right_result: List[str]) -> List[str] return self.visit_history -def test_boolean_expression_visitor(): +def test_boolean_expression_visitor() -> None: """Test post-order traversal of boolean expression visit method""" expr = And( Or(Not(ExpressionA()), Not(ExpressionB()), ExpressionA(), ExpressionB()), @@ -274,16 +274,16 @@ def test_boolean_expression_visitor(): ] -def test_boolean_expression_visit_raise_not_implemented_error(): +def test_boolean_expression_visit_raise_not_implemented_error() -> None: """Test raise NotImplementedError when visiting an unsupported object type""" visitor = ExampleVisitor() with pytest.raises(NotImplementedError) as exc_info: - visit("foo", visitor=visitor) + visit("foo", visitor=visitor) # type: ignore assert str(exc_info.value) == "Cannot visit unsupported expression: foo" -def test_bind_visitor_already_bound(table_schema_simple: Schema): +def test_bind_visitor_already_bound(table_schema_simple: Schema) -> None: bound = BoundEqualTo[str]( term=BoundReference(table_schema_simple.find_field(1), table_schema_simple.accessor_for_field(1)), literal=literal("hello"), @@ -296,34 +296,34 @@ def test_bind_visitor_already_bound(table_schema_simple: Schema): ) -def test_visit_bound_visitor_unknown_predicate(): +def test_visit_bound_visitor_unknown_predicate() -> None: with pytest.raises(TypeError) as exc_info: - visit_bound_predicate({"something"}, FooBoundBooleanExpressionVisitor()) + visit_bound_predicate({"something"}, FooBoundBooleanExpressionVisitor()) # type: ignore assert "Unknown predicate: {'something'}" == str(exc_info.value) -def test_always_true_expression_binding(table_schema_simple: Schema): +def test_always_true_expression_binding(table_schema_simple: Schema) -> None: """Test that visiting an always-true expression returns always-true""" unbound_expression = AlwaysTrue() bound_expression = visit(unbound_expression, visitor=BindVisitor(schema=table_schema_simple, case_sensitive=True)) assert bound_expression == AlwaysTrue() -def test_always_false_expression_binding(table_schema_simple: Schema): +def test_always_false_expression_binding(table_schema_simple: Schema) -> None: """Test that visiting an always-false expression returns always-false""" unbound_expression = AlwaysFalse() bound_expression = visit(unbound_expression, visitor=BindVisitor(schema=table_schema_simple, case_sensitive=True)) assert bound_expression == AlwaysFalse() -def test_always_false_and_always_true_expression_binding(table_schema_simple: Schema): +def test_always_false_and_always_true_expression_binding(table_schema_simple: Schema) -> None: """Test that visiting both an always-true AND always-false expression returns always-false""" unbound_expression = And(AlwaysTrue(), AlwaysFalse()) bound_expression = visit(unbound_expression, visitor=BindVisitor(schema=table_schema_simple, case_sensitive=True)) assert bound_expression == AlwaysFalse() -def test_always_false_or_always_true_expression_binding(table_schema_simple: Schema): +def test_always_false_or_always_true_expression_binding(table_schema_simple: Schema) -> None: """Test that visiting always-true OR always-false expression returns always-true""" unbound_expression = Or(AlwaysTrue(), AlwaysFalse()) bound_expression = visit(unbound_expression, visitor=BindVisitor(schema=table_schema_simple, case_sensitive=True)) @@ -395,7 +395,9 @@ def test_always_false_or_always_true_expression_binding(table_schema_simple: Sch ), ], ) -def test_and_expression_binding(unbound_and_expression, expected_bound_expression, table_schema_simple): +def test_and_expression_binding( + unbound_and_expression: UnboundPredicate[Any], expected_bound_expression: BoundPredicate[Any], table_schema_simple: Schema +) -> None: """Test that visiting an unbound AND expression with a bind-visitor returns the expected bound expression""" bound_expression = visit(unbound_and_expression, visitor=BindVisitor(schema=table_schema_simple, case_sensitive=True)) assert bound_expression == expected_bound_expression @@ -487,7 +489,9 @@ def test_and_expression_binding(unbound_and_expression, expected_bound_expressio ), ], ) -def test_or_expression_binding(unbound_or_expression, expected_bound_expression, table_schema_simple): +def test_or_expression_binding( + unbound_or_expression: UnboundPredicate[Any], expected_bound_expression: BoundPredicate[Any], table_schema_simple: Schema +) -> None: """Test that visiting an unbound OR expression with a bind-visitor returns the expected bound expression""" bound_expression = visit(unbound_or_expression, visitor=BindVisitor(schema=table_schema_simple, case_sensitive=True)) assert bound_expression == expected_bound_expression @@ -531,7 +535,9 @@ def test_or_expression_binding(unbound_or_expression, expected_bound_expression, ), ], ) -def test_in_expression_binding(unbound_in_expression, expected_bound_expression, table_schema_simple): +def test_in_expression_binding( + unbound_in_expression: UnboundPredicate[Any], expected_bound_expression: BoundPredicate[Any], table_schema_simple: Schema +) -> None: """Test that visiting an unbound IN expression with a bind-visitor returns the expected bound expression""" bound_expression = visit(unbound_in_expression, visitor=BindVisitor(schema=table_schema_simple, case_sensitive=True)) assert bound_expression == expected_bound_expression @@ -580,13 +586,15 @@ def test_in_expression_binding(unbound_in_expression, expected_bound_expression, ), ], ) -def test_not_expression_binding(unbound_not_expression, expected_bound_expression, table_schema_simple): +def test_not_expression_binding( + unbound_not_expression: UnboundPredicate[Any], expected_bound_expression: BoundPredicate[Any], table_schema_simple: Schema +) -> None: """Test that visiting an unbound NOT expression with a bind-visitor returns the expected bound expression""" bound_expression = visit(unbound_not_expression, visitor=BindVisitor(schema=table_schema_simple, case_sensitive=True)) assert bound_expression == expected_bound_expression -def test_bound_boolean_expression_visitor_and_in(): +def test_bound_boolean_expression_visitor_and_in() -> None: """Test visiting an And and In expression with a bound boolean expression visitor""" bound_expression = And( BoundIn( @@ -609,7 +617,7 @@ def test_bound_boolean_expression_visitor_and_in(): assert result == ["IN", "IN", "AND"] -def test_bound_boolean_expression_visitor_or(): +def test_bound_boolean_expression_visitor_or() -> None: """Test visiting an Or expression with a bound boolean expression visitor""" bound_expression = Or( Not( @@ -636,7 +644,7 @@ def test_bound_boolean_expression_visitor_or(): assert result == ["IN", "NOT", "IN", "NOT", "OR"] -def test_bound_boolean_expression_visitor_equal(): +def test_bound_boolean_expression_visitor_equal() -> None: bound_expression = BoundEqualTo( term=BoundReference( field=NestedField(field_id=2, name="bar", field_type=StringType(), required=False), @@ -649,7 +657,7 @@ def test_bound_boolean_expression_visitor_equal(): assert result == ["EQUAL"] -def test_bound_boolean_expression_visitor_not_equal(): +def test_bound_boolean_expression_visitor_not_equal() -> None: bound_expression = BoundNotEqualTo( term=BoundReference( field=NestedField(field_id=1, name="foo", field_type=StringType(), required=False), @@ -662,21 +670,21 @@ def test_bound_boolean_expression_visitor_not_equal(): assert result == ["NOT_EQUAL"] -def test_bound_boolean_expression_visitor_always_true(): +def test_bound_boolean_expression_visitor_always_true() -> None: bound_expression = AlwaysTrue() visitor = FooBoundBooleanExpressionVisitor() result = visit(bound_expression, visitor=visitor) assert result == ["TRUE"] -def test_bound_boolean_expression_visitor_always_false(): +def test_bound_boolean_expression_visitor_always_false() -> None: bound_expression = AlwaysFalse() visitor = FooBoundBooleanExpressionVisitor() result = visit(bound_expression, visitor=visitor) assert result == ["FALSE"] -def test_bound_boolean_expression_visitor_in(): +def test_bound_boolean_expression_visitor_in() -> None: bound_expression = BoundIn( term=BoundReference( field=NestedField(field_id=1, name="foo", field_type=StringType(), required=False), @@ -689,7 +697,7 @@ def test_bound_boolean_expression_visitor_in(): assert result == ["IN"] -def test_bound_boolean_expression_visitor_not_in(): +def test_bound_boolean_expression_visitor_not_in() -> None: bound_expression = BoundNotIn( term=BoundReference( field=NestedField(field_id=1, name="foo", field_type=StringType(), required=False), @@ -702,7 +710,7 @@ def test_bound_boolean_expression_visitor_not_in(): assert result == ["NOT_IN"] -def test_bound_boolean_expression_visitor_is_nan(): +def test_bound_boolean_expression_visitor_is_nan() -> None: bound_expression = BoundIsNaN( term=BoundReference( field=NestedField(field_id=3, name="baz", field_type=FloatType(), required=False), @@ -714,7 +722,7 @@ def test_bound_boolean_expression_visitor_is_nan(): assert result == ["IS_NAN"] -def test_bound_boolean_expression_visitor_not_nan(): +def test_bound_boolean_expression_visitor_not_nan() -> None: bound_expression = BoundNotNaN( term=BoundReference( field=NestedField(field_id=3, name="baz", field_type=FloatType(), required=False), @@ -726,7 +734,7 @@ def test_bound_boolean_expression_visitor_not_nan(): assert result == ["NOT_NAN"] -def test_bound_boolean_expression_visitor_is_null(): +def test_bound_boolean_expression_visitor_is_null() -> None: bound_expression = BoundIsNull( term=BoundReference( field=NestedField(field_id=1, name="foo", field_type=StringType(), required=False), @@ -738,7 +746,7 @@ def test_bound_boolean_expression_visitor_is_null(): assert result == ["IS_NULL"] -def test_bound_boolean_expression_visitor_not_null(): +def test_bound_boolean_expression_visitor_not_null() -> None: bound_expression = BoundNotNull( term=BoundReference( field=NestedField(field_id=1, name="foo", field_type=StringType(), required=False), @@ -750,7 +758,7 @@ def test_bound_boolean_expression_visitor_not_null(): assert result == ["NOT_NULL"] -def test_bound_boolean_expression_visitor_greater_than(): +def test_bound_boolean_expression_visitor_greater_than() -> None: bound_expression = BoundGreaterThan( term=BoundReference( field=NestedField(field_id=1, name="foo", field_type=StringType(), required=False), @@ -763,7 +771,7 @@ def test_bound_boolean_expression_visitor_greater_than(): assert result == ["GREATER_THAN"] -def test_bound_boolean_expression_visitor_greater_than_or_equal(): +def test_bound_boolean_expression_visitor_greater_than_or_equal() -> None: bound_expression = BoundGreaterThanOrEqual( term=BoundReference( field=NestedField(field_id=1, name="foo", field_type=StringType(), required=False), @@ -776,7 +784,7 @@ def test_bound_boolean_expression_visitor_greater_than_or_equal(): assert result == ["GREATER_THAN_OR_EQUAL"] -def test_bound_boolean_expression_visitor_less_than(): +def test_bound_boolean_expression_visitor_less_than() -> None: bound_expression = BoundLessThan( term=BoundReference( field=NestedField(field_id=1, name="foo", field_type=StringType(), required=False), @@ -789,7 +797,7 @@ def test_bound_boolean_expression_visitor_less_than(): assert result == ["LESS_THAN"] -def test_bound_boolean_expression_visitor_less_than_or_equal(): +def test_bound_boolean_expression_visitor_less_than_or_equal() -> None: bound_expression = BoundLessThanOrEqual( term=BoundReference( field=NestedField(field_id=1, name="foo", field_type=StringType(), required=False), @@ -802,7 +810,7 @@ def test_bound_boolean_expression_visitor_less_than_or_equal(): assert result == ["LESS_THAN_OR_EQUAL"] -def test_bound_boolean_expression_visitor_raise_on_unbound_predicate(): +def test_bound_boolean_expression_visitor_raise_on_unbound_predicate() -> None: bound_expression = LessThanOrEqual( term=Reference("foo"), literal="foo", @@ -813,7 +821,7 @@ def test_bound_boolean_expression_visitor_raise_on_unbound_predicate(): assert "Not a bound predicate" in str(exc_info.value) -def _to_byte_buffer(field_type: IcebergType, val: Any): +def _to_byte_buffer(field_type: IcebergType, val: Any) -> bytes: if not isinstance(field_type, PrimitiveType): raise ValueError(f"Expected a PrimitiveType, got: {type(field_type)}") return to_bytes(field_type, val) @@ -1041,7 +1049,7 @@ def test_not_nan(schema: Schema, manifest: ManifestFile) -> None: ), "Should read: no_nan_or_null column contains non nan value" -def test_missing_stats(schema: Schema, manifest_no_stats: ManifestFile): +def test_missing_stats(schema: Schema, manifest_no_stats: ManifestFile) -> None: expressions: List[BooleanExpression] = [ LessThan(Reference("id"), 5), LessThanOrEqual(Reference("id"), 30), @@ -1061,7 +1069,7 @@ def test_missing_stats(schema: Schema, manifest_no_stats: ManifestFile): ), f"Should read when missing stats for expr: {expr}" -def test_not(schema: Schema, manifest: ManifestFile): +def test_not(schema: Schema, manifest: ManifestFile) -> None: assert _ManifestEvalVisitor(schema, Not(LessThan(Reference("id"), INT_MIN_VALUE - 25)), case_sensitive=True).eval( manifest ), "Should read: not(false)" @@ -1071,7 +1079,7 @@ def test_not(schema: Schema, manifest: ManifestFile): ), "Should skip: not(true)" -def test_and(schema: Schema, manifest: ManifestFile): +def test_and(schema: Schema, manifest: ManifestFile) -> None: assert not _ManifestEvalVisitor( schema, And( @@ -1100,7 +1108,7 @@ def test_and(schema: Schema, manifest: ManifestFile): ).eval(manifest), "Should read: and(true, true)" -def test_or(schema: Schema, manifest: ManifestFile): +def test_or(schema: Schema, manifest: ManifestFile) -> None: assert not _ManifestEvalVisitor( schema, Or( @@ -1120,7 +1128,7 @@ def test_or(schema: Schema, manifest: ManifestFile): ).eval(manifest), "Should read: or(false, true)" -def test_integer_lt(schema: Schema, manifest: ManifestFile): +def test_integer_lt(schema: Schema, manifest: ManifestFile) -> None: assert not _ManifestEvalVisitor(schema, LessThan(Reference("id"), INT_MIN_VALUE - 25), case_sensitive=True).eval( manifest ), "Should not read: id range below lower bound (5 < 30)" @@ -1138,7 +1146,7 @@ def test_integer_lt(schema: Schema, manifest: ManifestFile): ), "Should read: may possible ids" -def test_integer_lt_eq(schema: Schema, manifest: ManifestFile): +def test_integer_lt_eq(schema: Schema, manifest: ManifestFile) -> None: assert not _ManifestEvalVisitor(schema, LessThanOrEqual(Reference("id"), INT_MIN_VALUE - 25), case_sensitive=True).eval( manifest ), "Should not read: id range below lower bound (5 < 30)" @@ -1156,7 +1164,7 @@ def test_integer_lt_eq(schema: Schema, manifest: ManifestFile): ), "Should read: many possible ids" -def test_integer_gt(schema: Schema, manifest: ManifestFile): +def test_integer_gt(schema: Schema, manifest: ManifestFile) -> None: assert not _ManifestEvalVisitor(schema, GreaterThan(Reference("id"), INT_MAX_VALUE + 6), case_sensitive=True).eval( manifest ), "Should not read: id range above upper bound (85 < 79)" @@ -1174,7 +1182,7 @@ def test_integer_gt(schema: Schema, manifest: ManifestFile): ), "Should read: may possible ids" -def test_integer_gt_eq(schema: Schema, manifest: ManifestFile): +def test_integer_gt_eq(schema: Schema, manifest: ManifestFile) -> None: assert not _ManifestEvalVisitor(schema, GreaterThanOrEqual(Reference("id"), INT_MAX_VALUE + 6), case_sensitive=True).eval( manifest ), "Should not read: id range above upper bound (85 < 79)" @@ -1192,7 +1200,7 @@ def test_integer_gt_eq(schema: Schema, manifest: ManifestFile): ), "Should read: may possible ids" -def test_integer_eq(schema: Schema, manifest: ManifestFile): +def test_integer_eq(schema: Schema, manifest: ManifestFile) -> None: assert not _ManifestEvalVisitor(schema, EqualTo(Reference("id"), INT_MIN_VALUE - 25), case_sensitive=True).eval( manifest ), "Should not read: id below lower bound" @@ -1222,7 +1230,7 @@ def test_integer_eq(schema: Schema, manifest: ManifestFile): ), "Should not read: id above upper bound" -def test_integer_not_eq(schema: Schema, manifest: ManifestFile): +def test_integer_not_eq(schema: Schema, manifest: ManifestFile) -> None: assert _ManifestEvalVisitor(schema, NotEqualTo(Reference("id"), INT_MIN_VALUE - 25), case_sensitive=True).eval( manifest ), "Should read: id below lower bound" @@ -1252,7 +1260,7 @@ def test_integer_not_eq(schema: Schema, manifest: ManifestFile): ), "Should read: id above upper bound" -def test_integer_not_eq_rewritten(schema: Schema, manifest: ManifestFile): +def test_integer_not_eq_rewritten(schema: Schema, manifest: ManifestFile) -> None: assert _ManifestEvalVisitor(schema, Not(EqualTo(Reference("id"), INT_MIN_VALUE - 25)), case_sensitive=True).eval( manifest ), "Should read: id below lower bound" @@ -1282,7 +1290,7 @@ def test_integer_not_eq_rewritten(schema: Schema, manifest: ManifestFile): ), "Should read: id above upper bound" -def test_integer_not_eq_rewritten_case_insensitive(schema: Schema, manifest: ManifestFile): +def test_integer_not_eq_rewritten_case_insensitive(schema: Schema, manifest: ManifestFile) -> None: assert _ManifestEvalVisitor(schema, Not(EqualTo(Reference("ID"), INT_MIN_VALUE - 25)), case_sensitive=False).eval( manifest ), "Should read: id below lower bound" @@ -1312,7 +1320,7 @@ def test_integer_not_eq_rewritten_case_insensitive(schema: Schema, manifest: Man ), "Should read: id above upper bound" -def test_integer_in(schema: Schema, manifest: ManifestFile): +def test_integer_in(schema: Schema, manifest: ManifestFile) -> None: assert not _ManifestEvalVisitor( schema, In(Reference("id"), (INT_MIN_VALUE - 25, INT_MIN_VALUE - 24)), case_sensitive=True ).eval(manifest), "Should not read: id below lower bound (5 < 30, 6 < 30)" @@ -1354,7 +1362,7 @@ def test_integer_in(schema: Schema, manifest: ManifestFile): ), "Should read: in on no nulls column" -def test_integer_not_in(schema: Schema, manifest: ManifestFile): +def test_integer_not_in(schema: Schema, manifest: ManifestFile) -> None: assert _ManifestEvalVisitor( schema, NotIn(Reference("id"), (INT_MIN_VALUE - 25, INT_MIN_VALUE - 24)), case_sensitive=True ).eval(manifest), "Should read: id below lower bound (5 < 30, 6 < 30)" @@ -1396,41 +1404,41 @@ def test_integer_not_in(schema: Schema, manifest: ManifestFile): ), "Should read: in on no nulls column" -def test_rewrite_not_equal_to(): +def test_rewrite_not_equal_to() -> None: assert rewrite_not(Not(EqualTo(Reference("x"), 34.56))) == NotEqualTo(Reference("x"), 34.56) -def test_rewrite_not_not_equal_to(): +def test_rewrite_not_not_equal_to() -> None: assert rewrite_not(Not(NotEqualTo(Reference("x"), 34.56))) == EqualTo(Reference("x"), 34.56) -def test_rewrite_not_in(): +def test_rewrite_not_in() -> None: assert rewrite_not(Not(In(Reference("x"), (34.56,)))) == NotIn(Reference("x"), (34.56,)) -def test_rewrite_and(): +def test_rewrite_and() -> None: assert rewrite_not(Not(And(EqualTo(Reference("x"), 34.56), EqualTo(Reference("y"), 34.56),))) == Or( NotEqualTo(term=Reference(name="x"), literal=34.56), NotEqualTo(term=Reference(name="y"), literal=34.56), ) -def test_rewrite_or(): +def test_rewrite_or() -> None: assert rewrite_not(Not(Or(EqualTo(Reference("x"), 34.56), EqualTo(Reference("y"), 34.56),))) == And( NotEqualTo(term=Reference(name="x"), literal=34.56), NotEqualTo(term=Reference(name="y"), literal=34.56), ) -def test_rewrite_always_false(): +def test_rewrite_always_false() -> None: assert rewrite_not(Not(AlwaysFalse())) == AlwaysTrue() -def test_rewrite_always_true(): +def test_rewrite_always_true() -> None: assert rewrite_not(Not(AlwaysTrue())) == AlwaysFalse() -def test_rewrite_bound(): +def test_rewrite_bound() -> None: schema = Schema(NestedField(2, "a", IntegerType(), required=False), schema_id=1) assert rewrite_not(IsNull(Reference("a")).bind(schema)) == BoundIsNull( term=BoundReference( diff --git a/tests/io/test_fsspec.py b/tests/io/test_fsspec.py index e1ac9f91e3..22cad25ea4 100644 --- a/tests/io/test_fsspec.py +++ b/tests/io/test_fsspec.py @@ -16,6 +16,7 @@ # under the License. import uuid +from typing import Generator import pytest from botocore.awsrequest import AWSRequest @@ -23,12 +24,12 @@ from pyiceberg.exceptions import SignError from pyiceberg.io import fsspec -from pyiceberg.io.fsspec import s3v4_rest_signer +from pyiceberg.io.fsspec import FsspecFileIO, s3v4_rest_signer from tests.io.test_io import LocalInputFile @pytest.mark.s3 -def test_fsspec_new_input_file(fsspec_fileio): +def test_fsspec_new_input_file(fsspec_fileio: FsspecFileIO) -> None: """Test creating a new input file from an fsspec file-io""" filename = str(uuid.uuid4()) @@ -39,7 +40,7 @@ def test_fsspec_new_input_file(fsspec_fileio): @pytest.mark.s3 -def test_fsspec_new_s3_output_file(fsspec_fileio): +def test_fsspec_new_s3_output_file(fsspec_fileio: FsspecFileIO) -> None: """Test creating a new output file from an fsspec file-io""" filename = str(uuid.uuid4()) @@ -50,7 +51,7 @@ def test_fsspec_new_s3_output_file(fsspec_fileio): @pytest.mark.s3 -def test_fsspec_write_and_read_file(fsspec_fileio): +def test_fsspec_write_and_read_file(fsspec_fileio: FsspecFileIO) -> None: """Test writing and reading a file using FsspecInputFile and FsspecOutputFile""" filename = str(uuid.uuid4()) output_file = fsspec_fileio.new_output(location=f"s3://warehouse/{filename}") @@ -64,7 +65,7 @@ def test_fsspec_write_and_read_file(fsspec_fileio): @pytest.mark.s3 -def test_fsspec_getting_length_of_file(fsspec_fileio): +def test_fsspec_getting_length_of_file(fsspec_fileio: FsspecFileIO) -> None: """Test getting the length of an FsspecInputFile and FsspecOutputFile""" filename = str(uuid.uuid4()) @@ -81,14 +82,14 @@ def test_fsspec_getting_length_of_file(fsspec_fileio): @pytest.mark.s3 -def test_fsspec_file_tell(fsspec_fileio): +def test_fsspec_file_tell(fsspec_fileio: FsspecFileIO) -> None: """Test finding cursor position for an fsspec file-io file""" filename = str(uuid.uuid4()) output_file = fsspec_fileio.new_output(location=f"s3://warehouse/{filename}") - with output_file.create() as f: - f.write(b"foobar") + with output_file.create() as write_file: + write_file.write(b"foobar") input_file = fsspec_fileio.new_input(location=f"s3://warehouse/{filename}") f = input_file.open() @@ -104,13 +105,13 @@ def test_fsspec_file_tell(fsspec_fileio): @pytest.mark.s3 -def test_fsspec_read_specified_bytes_for_file(fsspec_fileio): +def test_fsspec_read_specified_bytes_for_file(fsspec_fileio: FsspecFileIO) -> None: """Test reading a specified number of bytes from an fsspec file-io file""" filename = str(uuid.uuid4()) output_file = fsspec_fileio.new_output(location=f"s3://warehouse/{filename}") - with output_file.create() as f: - f.write(b"foo") + with output_file.create() as write_file: + write_file.write(b"foo") input_file = fsspec_fileio.new_input(location=f"s3://warehouse/{filename}") f = input_file.open() @@ -130,7 +131,7 @@ def test_fsspec_read_specified_bytes_for_file(fsspec_fileio): @pytest.mark.s3 -def test_fsspec_raise_on_opening_file_not_found(fsspec_fileio): +def test_fsspec_raise_on_opening_file_not_found(fsspec_fileio: FsspecFileIO) -> None: """Test that an fsppec input file raises appropriately when the s3 file is not found""" filename = str(uuid.uuid4()) @@ -142,7 +143,7 @@ def test_fsspec_raise_on_opening_file_not_found(fsspec_fileio): @pytest.mark.s3 -def test_checking_if_a_file_exists(fsspec_fileio): +def test_checking_if_a_file_exists(fsspec_fileio: FsspecFileIO) -> None: """Test checking if a file exists""" non_existent_file = fsspec_fileio.new_input(location="s3://warehouse/does-not-exist.txt") @@ -164,26 +165,26 @@ def test_checking_if_a_file_exists(fsspec_fileio): @pytest.mark.s3 -def test_closing_a_file(fsspec_fileio): +def test_closing_a_file(fsspec_fileio: FsspecFileIO) -> None: """Test closing an output file and input file""" filename = str(uuid.uuid4()) output_file = fsspec_fileio.new_output(location=f"s3://warehouse/{filename}") - with output_file.create() as f: - f.write(b"foo") - assert not f.closed - assert f.closed + with output_file.create() as write_file: + write_file.write(b"foo") + assert not write_file.closed # type: ignore + assert write_file.closed # type: ignore input_file = fsspec_fileio.new_input(location=f"s3://warehouse/{filename}") f = input_file.open() - assert not f.closed + assert not f.closed # type: ignore f.close() - assert f.closed + assert f.closed # type: ignore fsspec_fileio.delete(f"s3://warehouse/{filename}") @pytest.mark.s3 -def test_fsspec_converting_an_outputfile_to_an_inputfile(fsspec_fileio): +def test_fsspec_converting_an_outputfile_to_an_inputfile(fsspec_fileio: FsspecFileIO) -> None: """Test converting an output file to an input file""" filename = str(uuid.uuid4()) output_file = fsspec_fileio.new_output(location=f"s3://warehouse/{filename}") @@ -192,7 +193,7 @@ def test_fsspec_converting_an_outputfile_to_an_inputfile(fsspec_fileio): @pytest.mark.s3 -def test_writing_avro_file(generated_manifest_entry_file, fsspec_fileio): +def test_writing_avro_file(generated_manifest_entry_file: Generator[str, None, None], fsspec_fileio: FsspecFileIO) -> None: """Test that bytes match when reading a local avro file, writing it using fsspec file-io, and then reading it again""" filename = str(uuid.uuid4()) with LocalInputFile(generated_manifest_entry_file).open() as f: @@ -207,7 +208,7 @@ def test_writing_avro_file(generated_manifest_entry_file, fsspec_fileio): TEST_URI = "https://iceberg-test-signer" -def test_s3v4_rest_signer(requests_mock: Mocker): +def test_s3v4_rest_signer(requests_mock: Mocker) -> None: new_uri = "https://other-bucket/metadata/snap-8048355899640248710-1-a5c8ea2d-aa1f-48e8-89f4-1fa69db8c742.avro" requests_mock.post( f"{TEST_URI}/v1/aws/s3/sign", @@ -259,7 +260,7 @@ def test_s3v4_rest_signer(requests_mock: Mocker): } -def test_s3v4_rest_signer_forbidden(requests_mock: Mocker): +def test_s3v4_rest_signer_forbidden(requests_mock: Mocker) -> None: requests_mock.post( f"{TEST_URI}/v1/aws/s3/sign", json={ diff --git a/tests/io/test_io.py b/tests/io/test_io.py index 24e061dc8b..a83cba1af3 100644 --- a/tests/io/test_io.py +++ b/tests/io/test_io.py @@ -17,7 +17,7 @@ import os import tempfile -from typing import Union +from typing import Type, Union from unittest.mock import patch from urllib.parse import ParseResult, urlparse @@ -41,7 +41,7 @@ class LocalInputFile(InputFile): """An InputFile implementation for local files (for test use only)""" - def __init__(self, location: str): + def __init__(self, location: str) -> None: parsed_location = urlparse(location) # Create a ParseResult from the uri if parsed_location.scheme and parsed_location.scheme != "file": # Validate that a uri is provided with a scheme of `file` @@ -62,10 +62,10 @@ def parsed_location(self) -> ParseResult: """ return self._parsed_location - def __len__(self): + def __len__(self) -> int: return os.path.getsize(self.parsed_location.path) - def exists(self): + def exists(self) -> bool: return os.path.exists(self.parsed_location.path) def open(self) -> InputStream: @@ -78,8 +78,7 @@ def open(self) -> InputStream: class LocalOutputFile(OutputFile): """An OutputFile implementation for local files (for test use only)""" - def __init__(self, location: str): - + def __init__(self, location: str) -> None: parsed_location = urlparse(location) # Create a ParseResult from the uri if parsed_location.scheme and parsed_location.scheme != "file": # Validate that a uri is provided with a scheme of `file` raise ValueError("LocalOutputFile location must have a scheme of `file`") @@ -99,13 +98,13 @@ def parsed_location(self) -> ParseResult: """ return self._parsed_location - def __len__(self): + def __len__(self) -> int: return os.path.getsize(self.parsed_location.path) - def exists(self): + def exists(self) -> bool: return os.path.exists(self.parsed_location.path) - def to_input_file(self): + def to_input_file(self) -> LocalInputFile: return LocalInputFile(location=self.location) def create(self, overwrite: bool = False) -> OutputStream: @@ -118,10 +117,10 @@ def create(self, overwrite: bool = False) -> OutputStream: class LocalFileIO(FileIO): """A FileIO implementation for local files (for test use only)""" - def new_input(self, location: str): + def new_input(self, location: str) -> LocalInputFile: return LocalInputFile(location=location) - def new_output(self, location: str): + def new_output(self, location: str) -> LocalOutputFile: return LocalOutputFile(location=location) def delete(self, location: Union[str, InputFile, OutputFile]) -> None: @@ -134,12 +133,12 @@ def delete(self, location: Union[str, InputFile, OutputFile]) -> None: @pytest.mark.parametrize("CustomFileIO", [LocalFileIO, PyArrowFileIO]) -def test_custom_local_input_file(CustomFileIO): +def test_custom_local_input_file(CustomFileIO: Type[FileIO]) -> None: """Test initializing an InputFile implementation to read a local file""" with tempfile.TemporaryDirectory() as tmpdirname: file_location = os.path.join(tmpdirname, "foo.txt") - with open(file_location, "wb") as f: - f.write(b"foo") + with open(file_location, "wb") as write_file: + write_file.write(b"foo") # Confirm that the file initially exists assert os.path.exists(file_location) @@ -156,7 +155,7 @@ def test_custom_local_input_file(CustomFileIO): @pytest.mark.parametrize("CustomFileIO", [LocalFileIO, PyArrowFileIO]) -def test_custom_local_output_file(CustomFileIO): +def test_custom_local_output_file(CustomFileIO: Type[FileIO]) -> None: """Test initializing an OutputFile implementation to write to a local file""" with tempfile.TemporaryDirectory() as tmpdirname: file_location = os.path.join(tmpdirname, "foo.txt") @@ -177,14 +176,14 @@ def test_custom_local_output_file(CustomFileIO): @pytest.mark.parametrize("CustomFileIO", [LocalFileIO, PyArrowFileIO]) -def test_custom_local_output_file_with_overwrite(CustomFileIO): +def test_custom_local_output_file_with_overwrite(CustomFileIO: Type[FileIO]) -> None: """Test initializing an OutputFile implementation to overwrite a local file""" with tempfile.TemporaryDirectory() as tmpdirname: output_file_location = os.path.join(tmpdirname, "foo.txt") # Create a file in the temporary directory - with open(output_file_location, "wb") as f: - f.write(b"foo") + with open(output_file_location, "wb") as write_file: + write_file.write(b"foo") # Instantiate an output file output_file = CustomFileIO().new_output(location=f"{output_file_location}") @@ -202,7 +201,7 @@ def test_custom_local_output_file_with_overwrite(CustomFileIO): @pytest.mark.parametrize("CustomFileIO", [LocalFileIO, PyArrowFileIO]) -def test_custom_file_exists(CustomFileIO): +def test_custom_file_exists(CustomFileIO: Type[FileIO]) -> None: """Test that the exists property returns the proper value for existing and non-existing files""" with tempfile.TemporaryDirectory() as tmpdirname: file_location = os.path.join(tmpdirname, "foo.txt") @@ -219,12 +218,12 @@ def test_custom_file_exists(CustomFileIO): non_existent_absolute_file_location = os.path.abspath(nonexistent_file_location) # Create InputFile instances - file = CustomFileIO().new_input(location=f"{absolute_file_location}") - non_existent_file = CustomFileIO().new_input(location=f"{non_existent_absolute_file_location}") + input_file = CustomFileIO().new_input(location=f"{absolute_file_location}") + non_existent_input_file = CustomFileIO().new_input(location=f"{non_existent_absolute_file_location}") # Test opening and reading the file - assert file.exists() - assert not non_existent_file.exists() + assert input_file.exists() + assert not non_existent_input_file.exists() # Create OutputFile instances file = CustomFileIO().new_output(location=f"{absolute_file_location}") @@ -236,7 +235,7 @@ def test_custom_file_exists(CustomFileIO): @pytest.mark.parametrize("CustomFileIO", [LocalFileIO, PyArrowFileIO]) -def test_output_file_to_input_file(CustomFileIO): +def test_output_file_to_input_file(CustomFileIO: Type[FileIO]) -> None: """Test initializing an InputFile using the `to_input_file()` method on an OutputFile instance""" with tempfile.TemporaryDirectory() as tmpdirname: output_file_location = os.path.join(tmpdirname, "foo.txt") @@ -245,8 +244,8 @@ def test_output_file_to_input_file(CustomFileIO): output_file = CustomFileIO().new_output(location=f"{output_file_location}") # Create the output file and write to it - f = output_file.create() - f.write(b"foo") + with output_file.create() as output_stream: + output_stream.write(b"foo") # Convert to an input file and confirm the contents input_file = output_file.to_input_file() @@ -265,7 +264,7 @@ def test_output_file_to_input_file(CustomFileIO): (PyArrowFileIO, "file:/foo/bar/baz.parquet"), ], ) -def test_custom_file_io_locations(CustomFileIO, string_uri): +def test_custom_file_io_locations(CustomFileIO: Type[FileIO], string_uri: str) -> None: """Test that the location property is maintained as the value of the location argument""" # Instantiate the file-io and create a new input and output file file_io = CustomFileIO() @@ -280,7 +279,7 @@ def test_custom_file_io_locations(CustomFileIO, string_uri): "string_uri_w_netloc", ["file://localhost:80/foo/bar.parquet", "file://foo/bar.parquet"], ) -def test_raise_on_network_location_in_input_file(string_uri_w_netloc): +def test_raise_on_network_location_in_input_file(string_uri_w_netloc: str) -> None: """Test raising a ValueError when providing a network location to a LocalInputFile""" with pytest.raises(ValueError) as exc_info: LocalInputFile(location=string_uri_w_netloc) @@ -292,7 +291,7 @@ def test_raise_on_network_location_in_input_file(string_uri_w_netloc): "string_uri_w_netloc", ["file://localhost:80/foo/bar.parquet", "file://foo/bar.parquet"], ) -def test_raise_on_network_location_in_output_file(string_uri_w_netloc): +def test_raise_on_network_location_in_output_file(string_uri_w_netloc: str) -> None: """Test raising a ValueError when providing a network location to a LocalOutputFile""" with pytest.raises(ValueError) as exc_info: LocalInputFile(location=string_uri_w_netloc) @@ -301,7 +300,7 @@ def test_raise_on_network_location_in_output_file(string_uri_w_netloc): @pytest.mark.parametrize("CustomFileIO", [LocalFileIO, PyArrowFileIO]) -def test_deleting_local_file_using_file_io(CustomFileIO): +def test_deleting_local_file_using_file_io(CustomFileIO: Type[FileIO]) -> None: """Test deleting a local file using FileIO.delete(...)""" with tempfile.TemporaryDirectory() as tmpdirname: # Write to the temporary file @@ -323,7 +322,7 @@ def test_deleting_local_file_using_file_io(CustomFileIO): @pytest.mark.parametrize("CustomFileIO", [LocalFileIO, PyArrowFileIO]) -def test_raise_file_not_found_error_for_fileio_delete(CustomFileIO): +def test_raise_file_not_found_error_for_fileio_delete(CustomFileIO: Type[FileIO]) -> None: """Test raising a FileNotFound error when trying to delete a non-existent file""" with tempfile.TemporaryDirectory() as tmpdirname: # Write to the temporary file @@ -343,7 +342,7 @@ def test_raise_file_not_found_error_for_fileio_delete(CustomFileIO): @pytest.mark.parametrize("CustomFileIO", [LocalFileIO, PyArrowFileIO]) -def test_deleting_local_file_using_file_io_input_file(CustomFileIO): +def test_deleting_local_file_using_file_io_input_file(CustomFileIO: Type[FileIO]) -> None: """Test deleting a local file by passing an InputFile instance to FileIO.delete(...)""" with tempfile.TemporaryDirectory() as tmpdirname: # Write to the temporary file @@ -368,7 +367,7 @@ def test_deleting_local_file_using_file_io_input_file(CustomFileIO): @pytest.mark.parametrize("CustomFileIO", [LocalFileIO, PyArrowFileIO]) -def test_deleting_local_file_using_file_io_output_file(CustomFileIO): +def test_deleting_local_file_using_file_io_output_file(CustomFileIO: Type[FileIO]) -> None: """Test deleting a local file by passing an OutputFile instance to FileIO.delete(...)""" with tempfile.TemporaryDirectory() as tmpdirname: # Write to the temporary file @@ -392,55 +391,55 @@ def test_deleting_local_file_using_file_io_output_file(CustomFileIO): assert not os.path.exists(file_location) -def test_import_file_io(): +def test_import_file_io() -> None: assert isinstance(_import_file_io(ARROW_FILE_IO, {}), PyArrowFileIO) -def test_import_file_io_does_not_exist(): +def test_import_file_io_does_not_exist() -> None: assert _import_file_io("pyiceberg.does.not.exist.FileIO", {}) is None -def test_load_file(): +def test_load_file() -> None: assert isinstance(load_file_io({PY_IO_IMPL: ARROW_FILE_IO}), PyArrowFileIO) -def test_load_file_io_no_arguments(): +def test_load_file_io_no_arguments() -> None: assert isinstance(load_file_io({}), PyArrowFileIO) -def test_load_file_io_does_not_exist(): +def test_load_file_io_does_not_exist() -> None: with pytest.raises(ValueError) as exc_info: load_file_io({PY_IO_IMPL: "pyiceberg.does.not.exist.FileIO"}) assert "Could not initialize FileIO: pyiceberg.does.not.exist.FileIO" in str(exc_info.value) -def test_load_file_io_warehouse(): +def test_load_file_io_warehouse() -> None: assert isinstance(load_file_io({"warehouse": "s3://some-path/"}), FsspecFileIO) -def test_load_file_io_location(): +def test_load_file_io_location() -> None: assert isinstance(load_file_io({"location": "s3://some-path/"}), PyArrowFileIO) -def test_load_file_io_location_no_schema(): +def test_load_file_io_location_no_schema() -> None: assert isinstance(load_file_io({"location": "/no-schema/"}), PyArrowFileIO) @patch.dict("pyiceberg.io.SCHEMA_TO_FILE_IO", {"test": ["tests.io.test_io.LocalFileIO"]}) -def test_mock_warehouse_location_file_io(): +def test_mock_warehouse_location_file_io() -> None: # For testing the selection logic io = load_file_io({"warehouse": "test://some-path/"}) assert io.properties["warehouse"] == "test://some-path/" @patch.dict("pyiceberg.io.SCHEMA_TO_FILE_IO", {"test": ["tests.io.test_io.LocalFileIO"]}) -def test_mock_table_location_file_io(): +def test_mock_table_location_file_io() -> None: # For testing the selection logic io = load_file_io({}, "test://some-path/") assert io.properties == {} -def test_gibberish_table_location_file_io(): +def test_gibberish_table_location_file_io() -> None: # For testing the selection logic assert isinstance(load_file_io({}, "gibberish"), PyArrowFileIO) diff --git a/tests/io/test_pyarrow.py b/tests/io/test_pyarrow.py index 79136be1c2..674331bf1c 100644 --- a/tests/io/test_pyarrow.py +++ b/tests/io/test_pyarrow.py @@ -18,6 +18,7 @@ import os import tempfile +from typing import Any from unittest.mock import MagicMock, patch import pyarrow as pa @@ -74,7 +75,7 @@ ) -def test_pyarrow_input_file(): +def test_pyarrow_input_file() -> None: """Test reading a file using PyArrowFile""" with tempfile.TemporaryDirectory() as tmpdirname: @@ -90,14 +91,14 @@ def test_pyarrow_input_file(): input_file = PyArrowFileIO().new_input(location=f"{absolute_file_location}") # Test opening and reading the file - f = input_file.open() - assert isinstance(f, InputStream) # Test that the file object abides by the InputStream protocol - data = f.read() + r = input_file.open() + assert isinstance(r, InputStream) # Test that the file object abides by the InputStream protocol + data = r.read() assert data == b"foo" assert len(input_file) == 3 -def test_pyarrow_output_file(): +def test_pyarrow_output_file() -> None: """Test writing a file using PyArrowFile""" with tempfile.TemporaryDirectory() as tmpdirname: @@ -119,21 +120,21 @@ def test_pyarrow_output_file(): assert len(output_file) == 3 -def test_pyarrow_invalid_scheme(): +def test_pyarrow_invalid_scheme() -> None: """Test that a ValueError is raised if a location is provided with an invalid scheme""" with pytest.raises(ValueError) as exc_info: PyArrowFileIO().new_input("foo://bar/baz.txt") - assert ("Unrecognized filesystem type in URI") in str(exc_info.value) + assert "Unrecognized filesystem type in URI" in str(exc_info.value) with pytest.raises(ValueError) as exc_info: PyArrowFileIO().new_output("foo://bar/baz.txt") - assert ("Unrecognized filesystem type in URI") in str(exc_info.value) + assert "Unrecognized filesystem type in URI" in str(exc_info.value) -def test_pyarrow_violating_input_stream_protocol(): +def test_pyarrow_violating_input_stream_protocol() -> None: """Test that a TypeError is raised if an input file is provided that violates the InputStream protocol""" # Missing seek, tell, closed, and close @@ -149,7 +150,7 @@ def test_pyarrow_violating_input_stream_protocol(): assert not isinstance(f, InputStream) -def test_pyarrow_violating_output_stream_protocol(): +def test_pyarrow_violating_output_stream_protocol() -> None: """Test that a TypeError is raised if an output stream is provided that violates the OutputStream protocol""" # Missing closed, and close @@ -171,7 +172,7 @@ def test_pyarrow_violating_output_stream_protocol(): assert not isinstance(f, OutputStream) -def test_raise_on_opening_a_local_file_not_found(): +def test_raise_on_opening_a_local_file_not_found() -> None: """Test that a PyArrowFile raises appropriately when a local file is not found""" with tempfile.TemporaryDirectory() as tmpdirname: @@ -184,7 +185,7 @@ def test_raise_on_opening_a_local_file_not_found(): assert "[Errno 2] Failed to open local file" in str(exc_info.value) -def test_raise_on_opening_a_local_file_no_permission(): +def test_raise_on_opening_a_local_file_no_permission() -> None: """Test that a PyArrowFile raises appropriately when opening a local file without permission""" with tempfile.TemporaryDirectory() as tmpdirname: @@ -198,7 +199,7 @@ def test_raise_on_opening_a_local_file_no_permission(): assert "[Errno 13] Failed to open local file" in str(exc_info.value) -def test_raise_on_checking_if_local_file_exists_no_permission(): +def test_raise_on_checking_if_local_file_exists_no_permission() -> None: """Test that a PyArrowFile raises when checking for existence on a file without permission""" with tempfile.TemporaryDirectory() as tmpdirname: @@ -212,7 +213,7 @@ def test_raise_on_checking_if_local_file_exists_no_permission(): assert "Cannot get file info, access denied:" in str(exc_info.value) -def test_raise_on_creating_a_local_file_no_permission(): +def test_raise_on_creating_a_local_file_no_permission() -> None: """Test that a PyArrowFile raises appropriately when creating a local file without permission""" with tempfile.TemporaryDirectory() as tmpdirname: @@ -226,7 +227,7 @@ def test_raise_on_creating_a_local_file_no_permission(): assert "Cannot get file info, access denied:" in str(exc_info.value) -def test_raise_on_delete_file_with_no_permission(): +def test_raise_on_delete_file_with_no_permission() -> None: """Test that a PyArrowFile raises when deleting a local file without permission""" with tempfile.TemporaryDirectory() as tmpdirname: @@ -240,7 +241,7 @@ def test_raise_on_delete_file_with_no_permission(): assert "Cannot delete file" in str(exc_info.value) -def test_raise_on_opening_an_s3_file_no_permission(): +def test_raise_on_opening_an_s3_file_no_permission() -> None: """Test that opening a PyArrowFile raises a PermissionError when the pyarrow error includes 'AWS Error [code 15]'""" s3fs_mock = MagicMock() @@ -254,7 +255,7 @@ def test_raise_on_opening_an_s3_file_no_permission(): assert "Cannot open file, access denied:" in str(exc_info.value) -def test_raise_on_opening_an_s3_file_not_found(): +def test_raise_on_opening_an_s3_file_not_found() -> None: """Test that a PyArrowFile raises a FileNotFoundError when the pyarrow error includes 'Path does not exist'""" s3fs_mock = MagicMock() @@ -269,7 +270,7 @@ def test_raise_on_opening_an_s3_file_not_found(): @patch("pyiceberg.io.pyarrow.PyArrowFile.exists", return_value=False) -def test_raise_on_creating_an_s3_file_no_permission(_): +def test_raise_on_creating_an_s3_file_no_permission(_: Any) -> None: """Test that creating a PyArrowFile raises a PermissionError when the pyarrow error includes 'AWS Error [code 15]'""" s3fs_mock = MagicMock() @@ -283,7 +284,7 @@ def test_raise_on_creating_an_s3_file_no_permission(_): assert "Cannot create file, access denied:" in str(exc_info.value) -def test_deleting_s3_file_no_permission(): +def test_deleting_s3_file_no_permission() -> None: """Test that a PyArrowFile raises a PermissionError when the pyarrow OSError includes 'AWS Error [code 15]'""" s3fs_mock = MagicMock() @@ -298,7 +299,7 @@ def test_deleting_s3_file_no_permission(): assert "Cannot delete file, access denied:" in str(exc_info.value) -def test_deleting_s3_file_not_found(): +def test_deleting_s3_file_not_found() -> None: """Test that a PyArrowFile raises a PermissionError when the pyarrow error includes 'AWS Error [code 15]'""" s3fs_mock = MagicMock() @@ -313,7 +314,7 @@ def test_deleting_s3_file_not_found(): assert "Cannot delete file, does not exist:" in str(exc_info.value) -def test_schema_to_pyarrow_schema(table_schema_nested: Schema): +def test_schema_to_pyarrow_schema(table_schema_nested: Schema) -> None: actual = schema_to_pyarrow(table_schema_nested) expected = """foo: string bar: int32 not null @@ -337,75 +338,75 @@ def test_schema_to_pyarrow_schema(table_schema_nested: Schema): assert repr(actual) == expected -def test_fixed_type_to_pyarrow(): +def test_fixed_type_to_pyarrow() -> None: length = 22 iceberg_type = FixedType(length) assert visit(iceberg_type, _ConvertToArrowSchema()) == pa.binary(length) -def test_decimal_type_to_pyarrow(): +def test_decimal_type_to_pyarrow() -> None: precision = 25 scale = 19 iceberg_type = DecimalType(precision, scale) assert visit(iceberg_type, _ConvertToArrowSchema()) == pa.decimal128(precision, scale) -def test_boolean_type_to_pyarrow(): +def test_boolean_type_to_pyarrow() -> None: iceberg_type = BooleanType() assert visit(iceberg_type, _ConvertToArrowSchema()) == pa.bool_() -def test_integer_type_to_pyarrow(): +def test_integer_type_to_pyarrow() -> None: iceberg_type = IntegerType() assert visit(iceberg_type, _ConvertToArrowSchema()) == pa.int32() -def test_long_type_to_pyarrow(): +def test_long_type_to_pyarrow() -> None: iceberg_type = LongType() assert visit(iceberg_type, _ConvertToArrowSchema()) == pa.int64() -def test_float_type_to_pyarrow(): +def test_float_type_to_pyarrow() -> None: iceberg_type = FloatType() assert visit(iceberg_type, _ConvertToArrowSchema()) == pa.float32() -def test_double_type_to_pyarrow(): +def test_double_type_to_pyarrow() -> None: iceberg_type = DoubleType() assert visit(iceberg_type, _ConvertToArrowSchema()) == pa.float64() -def test_date_type_to_pyarrow(): +def test_date_type_to_pyarrow() -> None: iceberg_type = DateType() assert visit(iceberg_type, _ConvertToArrowSchema()) == pa.date32() -def test_time_type_to_pyarrow(): +def test_time_type_to_pyarrow() -> None: iceberg_type = TimeType() assert visit(iceberg_type, _ConvertToArrowSchema()) == pa.time64("us") -def test_timestamp_type_to_pyarrow(): +def test_timestamp_type_to_pyarrow() -> None: iceberg_type = TimestampType() assert visit(iceberg_type, _ConvertToArrowSchema()) == pa.timestamp(unit="us") -def test_timestamptz_type_to_pyarrow(): +def test_timestamptz_type_to_pyarrow() -> None: iceberg_type = TimestamptzType() assert visit(iceberg_type, _ConvertToArrowSchema()) == pa.timestamp(unit="us", tz="+00:00") -def test_string_type_to_pyarrow(): +def test_string_type_to_pyarrow() -> None: iceberg_type = StringType() assert visit(iceberg_type, _ConvertToArrowSchema()) == pa.string() -def test_binary_type_to_pyarrow(): +def test_binary_type_to_pyarrow() -> None: iceberg_type = BinaryType() assert visit(iceberg_type, _ConvertToArrowSchema()) == pa.binary() -def test_struct_type_to_pyarrow(table_schema_simple: Schema): +def test_struct_type_to_pyarrow(table_schema_simple: Schema) -> None: expected = pa.struct( [ pa.field("foo", pa.string(), nullable=True, metadata={"id": "1"}), @@ -416,7 +417,7 @@ def test_struct_type_to_pyarrow(table_schema_simple: Schema): assert visit(table_schema_simple.as_struct(), _ConvertToArrowSchema()) == expected -def test_map_type_to_pyarrow(): +def test_map_type_to_pyarrow() -> None: iceberg_map = MapType( key_id=1, key_type=IntegerType(), @@ -427,7 +428,7 @@ def test_map_type_to_pyarrow(): assert visit(iceberg_map, _ConvertToArrowSchema()) == pa.map_(pa.int32(), pa.string()) -def test_list_type_to_pyarrow(): +def test_list_type_to_pyarrow() -> None: iceberg_map = ListType( element_id=1, element_type=IntegerType(), diff --git a/tests/table/test_init.py b/tests/table/test_init.py index 5ad535693a..3c663ceeb4 100644 --- a/tests/table/test_init.py +++ b/tests/table/test_init.py @@ -57,53 +57,61 @@ def table(example_table_metadata_v2: Dict[str, Any]) -> Table: ) -def test_schema(table): +def test_schema(table: Table) -> None: assert table.schema() == Schema( - fields=( - NestedField(field_id=1, name="x", field_type=LongType(), required=True), - NestedField(field_id=2, name="y", field_type=LongType(), required=True, doc="comment"), - NestedField(field_id=3, name="z", field_type=LongType(), required=True), - ), + NestedField(field_id=1, name="x", field_type=LongType(), required=True), + NestedField(field_id=2, name="y", field_type=LongType(), required=True, doc="comment"), + NestedField(field_id=3, name="z", field_type=LongType(), required=True), schema_id=1, identifier_field_ids=[1, 2], ) -def test_schemas(table): +def test_schemas(table: Table) -> None: assert table.schemas() == { 0: Schema( - fields=(NestedField(field_id=1, name="x", field_type=LongType(), required=True),), + NestedField(field_id=1, name="x", field_type=LongType(), required=True), schema_id=0, identifier_field_ids=[], ), 1: Schema( - fields=( - NestedField(field_id=1, name="x", field_type=LongType(), required=True), - NestedField(field_id=2, name="y", field_type=LongType(), required=True, doc="comment"), - NestedField(field_id=3, name="z", field_type=LongType(), required=True), - ), + NestedField(field_id=1, name="x", field_type=LongType(), required=True), + NestedField(field_id=2, name="y", field_type=LongType(), required=True, doc="comment"), + NestedField(field_id=3, name="z", field_type=LongType(), required=True), schema_id=1, identifier_field_ids=[1, 2], ), } -def test_spec(table): +def test_spec(table: Table) -> None: assert table.spec() == PartitionSpec( PartitionField(source_id=1, field_id=1000, transform=IdentityTransform(), name="x"), spec_id=0 ) -def test_specs(table): +def test_specs(table: Table) -> None: assert table.specs() == { 0: PartitionSpec(PartitionField(source_id=1, field_id=1000, transform=IdentityTransform(), name="x"), spec_id=0) } -def test_sort_order(table): +def test_sort_order(table: Table) -> None: assert table.sort_order() == SortOrder( + SortField(source_id=2, transform=IdentityTransform(), direction=SortDirection.ASC, null_order=NullOrder.NULLS_FIRST), + SortField( + source_id=3, + transform=BucketTransform(num_buckets=4), + direction=SortDirection.DESC, + null_order=NullOrder.NULLS_LAST, + ), order_id=3, - fields=[ + ) + + +def test_sort_orders(table: Table) -> None: + assert table.sort_orders() == { + 3: SortOrder( SortField(source_id=2, transform=IdentityTransform(), direction=SortDirection.ASC, null_order=NullOrder.NULLS_FIRST), SortField( source_id=3, @@ -111,34 +119,16 @@ def test_sort_order(table): direction=SortDirection.DESC, null_order=NullOrder.NULLS_LAST, ), - ], - ) - - -def test_sort_orders(table): - assert table.sort_orders() == { - 3: SortOrder( order_id=3, - fields=[ - SortField( - source_id=2, transform=IdentityTransform(), direction=SortDirection.ASC, null_order=NullOrder.NULLS_FIRST - ), - SortField( - source_id=3, - transform=BucketTransform(num_buckets=4), - direction=SortDirection.DESC, - null_order=NullOrder.NULLS_LAST, - ), - ], ) } -def test_location(table): +def test_location(table: Table) -> None: assert table.location() == "s3://bucket/test/location" -def test_current_snapshot(table): +def test_current_snapshot(table: Table) -> None: assert table.current_snapshot() == Snapshot( snapshot_id=3055729675574597004, parent_snapshot_id=3051729675574597004, @@ -150,7 +140,7 @@ def test_current_snapshot(table): ) -def test_snapshot_by_id(table): +def test_snapshot_by_id(table: Table) -> None: assert table.snapshot_by_id(3055729675574597004) == Snapshot( snapshot_id=3055729675574597004, parent_snapshot_id=3051729675574597004, @@ -162,11 +152,11 @@ def test_snapshot_by_id(table): ) -def test_snapshot_by_id_does_not_exist(table): +def test_snapshot_by_id_does_not_exist(table: Table) -> None: assert table.snapshot_by_id(-1) is None -def test_snapshot_by_name(table): +def test_snapshot_by_name(table: Table) -> None: assert table.snapshot_by_name("test") == Snapshot( snapshot_id=3051729675574597004, parent_snapshot_id=None, @@ -178,37 +168,37 @@ def test_snapshot_by_name(table): ) -def test_snapshot_by_name_does_not_exist(table): +def test_snapshot_by_name_does_not_exist(table: Table) -> None: assert table.snapshot_by_name("doesnotexist") is None -def test_history(table): +def test_history(table: Table) -> None: assert table.history() == [ SnapshotLogEntry(snapshot_id="3051729675574597004", timestamp_ms=1515100955770), SnapshotLogEntry(snapshot_id="3055729675574597004", timestamp_ms=1555100955770), ] -def test_table_scan_select(table: Table): +def test_table_scan_select(table: Table) -> None: scan = table.scan() assert scan.selected_fields == ("*",) assert scan.select("a", "b").selected_fields == ("a", "b") assert scan.select("a", "c").select("a").selected_fields == ("a",) -def test_table_scan_row_filter(table: Table): +def test_table_scan_row_filter(table: Table) -> None: scan = table.scan() assert scan.row_filter == AlwaysTrue() assert scan.filter(EqualTo("x", 10)).row_filter == EqualTo("x", 10) assert scan.filter(EqualTo("x", 10)).filter(In("y", (10, 11))).row_filter == And(EqualTo("x", 10), In("y", (10, 11))) -def test_table_scan_ref(table: Table): +def test_table_scan_ref(table: Table) -> None: scan = table.scan() assert scan.use_ref("test").snapshot_id == 3051729675574597004 -def test_table_scan_ref_does_not_exists(table: Table): +def test_table_scan_ref_does_not_exists(table: Table) -> None: scan = table.scan() with pytest.raises(ValueError) as exc_info: @@ -217,7 +207,7 @@ def test_table_scan_ref_does_not_exists(table: Table): assert "Cannot scan unknown ref=boom" in str(exc_info.value) -def test_table_scan_projection_full_schema(table: Table): +def test_table_scan_projection_full_schema(table: Table) -> None: scan = table.scan() assert scan.select("x", "y", "z").projection() == Schema( NestedField(field_id=1, name="x", field_type=LongType(), required=True), @@ -228,7 +218,7 @@ def test_table_scan_projection_full_schema(table: Table): ) -def test_table_scan_projection_single_column(table: Table): +def test_table_scan_projection_single_column(table: Table) -> None: scan = table.scan() assert scan.select("y").projection() == Schema( NestedField(field_id=2, name="y", field_type=LongType(), required=True, doc="comment"), @@ -237,7 +227,7 @@ def test_table_scan_projection_single_column(table: Table): ) -def test_table_scan_projection_single_column_case_sensitive(table: Table): +def test_table_scan_projection_single_column_case_sensitive(table: Table) -> None: scan = table.scan() assert scan.with_case_sensitive(False).select("Y").projection() == Schema( NestedField(field_id=2, name="y", field_type=LongType(), required=True, doc="comment"), @@ -246,7 +236,7 @@ def test_table_scan_projection_single_column_case_sensitive(table: Table): ) -def test_table_scan_projection_unknown_column(table: Table): +def test_table_scan_projection_unknown_column(table: Table) -> None: scan = table.scan() with pytest.raises(ValueError) as exc_info: diff --git a/tests/table/test_metadata.py b/tests/table/test_metadata.py index 297d58c6ed..9f46ca6737 100644 --- a/tests/table/test_metadata.py +++ b/tests/table/test_metadata.py @@ -70,24 +70,24 @@ } -def test_from_dict_v1(): +def test_from_dict_v1() -> None: """Test initialization of a TableMetadata instance from a dictionary""" TableMetadataUtil.parse_obj(EXAMPLE_TABLE_METADATA_V1) -def test_from_dict_v2(example_table_metadata_v2: Dict[str, Any]): +def test_from_dict_v2(example_table_metadata_v2: Dict[str, Any]) -> None: """Test initialization of a TableMetadata instance from a dictionary""" TableMetadataUtil.parse_obj(example_table_metadata_v2) -def test_from_byte_stream(example_table_metadata_v2: Dict[str, Any]): +def test_from_byte_stream(example_table_metadata_v2: Dict[str, Any]) -> None: """Test generating a TableMetadata instance from a file-like byte stream""" data = bytes(json.dumps(example_table_metadata_v2), encoding="utf-8") byte_stream = io.BytesIO(data) FromByteStream.table_metadata(byte_stream=byte_stream) -def test_v2_metadata_parsing(example_table_metadata_v2: Dict[str, Any]): +def test_v2_metadata_parsing(example_table_metadata_v2: Dict[str, Any]) -> None: """Test retrieving values from a TableMetadata instance of version 2""" table_metadata = TableMetadataUtil.parse_obj(example_table_metadata_v2) @@ -110,7 +110,7 @@ def test_v2_metadata_parsing(example_table_metadata_v2: Dict[str, Any]): assert table_metadata.default_sort_order_id == 3 -def test_v1_metadata_parsing_directly(): +def test_v1_metadata_parsing_directly() -> None: """Test retrieving values from a TableMetadata instance of version 1""" table_metadata = TableMetadataV1(**EXAMPLE_TABLE_METADATA_V1) @@ -134,7 +134,7 @@ def test_v1_metadata_parsing_directly(): assert table_metadata.schemas[0].schema_id == 0 assert table_metadata.current_schema_id == 0 assert table_metadata.partition_specs == [ - PartitionSpec(PartitionField(source_id=1, field_id=1000, transform=IdentityTransform(), name="x"), spec_id=0) + PartitionSpec(PartitionField(source_id=1, field_id=1000, transform=IdentityTransform(), name="x")) ] assert table_metadata.default_spec_id == 0 assert table_metadata.last_partition_id == 1000 @@ -142,14 +142,14 @@ def test_v1_metadata_parsing_directly(): assert table_metadata.default_sort_order_id == 0 -def test_parsing_correct_types(example_table_metadata_v2: Dict[str, Any]): +def test_parsing_correct_types(example_table_metadata_v2: Dict[str, Any]) -> None: table_metadata = TableMetadataV2(**example_table_metadata_v2) assert isinstance(table_metadata.schemas[0], Schema) assert isinstance(table_metadata.schemas[0].fields[0], NestedField) assert isinstance(table_metadata.schemas[0].fields[0].field_type, LongType) -def test_updating_metadata(example_table_metadata_v2: Dict[str, Any]): +def test_updating_metadata(example_table_metadata_v2: Dict[str, Any]) -> None: """Test creating a new TableMetadata instance that's an updated version of an existing TableMetadata instance""" table_metadata = TableMetadataV2(**example_table_metadata_v2) @@ -174,20 +174,20 @@ def test_updating_metadata(example_table_metadata_v2: Dict[str, Any]): assert table_metadata.schemas[-1] == Schema(**new_schema) -def test_serialize_v1(): +def test_serialize_v1() -> None: table_metadata = TableMetadataV1(**EXAMPLE_TABLE_METADATA_V1) table_metadata_json = table_metadata.json() expected = """{"location": "s3://bucket/test/location", "table-uuid": "d20125c8-7284-442c-9aea-15fee620737c", "last-updated-ms": 1602638573874, "last-column-id": 3, "schemas": [{"type": "struct", "fields": [{"id": 1, "name": "x", "type": "long", "required": true}, {"id": 2, "name": "y", "type": "long", "required": true, "doc": "comment"}, {"id": 3, "name": "z", "type": "long", "required": true}], "schema-id": 0, "identifier-field-ids": []}], "current-schema-id": 0, "partition-specs": [{"spec-id": 0, "fields": [{"source-id": 1, "field-id": 1000, "transform": "identity", "name": "x"}]}], "default-spec-id": 0, "last-partition-id": 1000, "properties": {}, "snapshots": [{"snapshot-id": 1925, "timestamp-ms": 1602638573822}], "snapshot-log": [], "metadata-log": [], "sort-orders": [{"order-id": 0, "fields": []}], "default-sort-order-id": 0, "refs": {}, "format-version": 1, "schema": {"type": "struct", "fields": [{"id": 1, "name": "x", "type": "long", "required": true}, {"id": 2, "name": "y", "type": "long", "required": true, "doc": "comment"}, {"id": 3, "name": "z", "type": "long", "required": true}], "schema-id": 0, "identifier-field-ids": []}, "partition-spec": [{"name": "x", "transform": "identity", "source-id": 1, "field-id": 1000}]}""" assert table_metadata_json == expected -def test_serialize_v2(example_table_metadata_v2: Dict[str, Any]): +def test_serialize_v2(example_table_metadata_v2: Dict[str, Any]) -> None: table_metadata = TableMetadataV2(**example_table_metadata_v2).json() expected = """{"location": "s3://bucket/test/location", "table-uuid": "9c12d441-03fe-4693-9a96-a0705ddf69c1", "last-updated-ms": 1602638573590, "last-column-id": 3, "schemas": [{"type": "struct", "fields": [{"id": 1, "name": "x", "type": "long", "required": true}], "schema-id": 0, "identifier-field-ids": []}, {"type": "struct", "fields": [{"id": 1, "name": "x", "type": "long", "required": true}, {"id": 2, "name": "y", "type": "long", "required": true, "doc": "comment"}, {"id": 3, "name": "z", "type": "long", "required": true}], "schema-id": 1, "identifier-field-ids": [1, 2]}], "current-schema-id": 1, "partition-specs": [{"spec-id": 0, "fields": [{"source-id": 1, "field-id": 1000, "transform": "identity", "name": "x"}]}], "default-spec-id": 0, "last-partition-id": 1000, "properties": {"read.split.target.size": "134217728"}, "current-snapshot-id": 3055729675574597004, "snapshots": [{"snapshot-id": 3051729675574597004, "sequence-number": 0, "timestamp-ms": 1515100955770, "manifest-list": "s3://a/b/1.avro", "summary": {"operation": "append"}}, {"snapshot-id": 3055729675574597004, "parent-snapshot-id": 3051729675574597004, "sequence-number": 1, "timestamp-ms": 1555100955770, "manifest-list": "s3://a/b/2.avro", "summary": {"operation": "append"}, "schema-id": 1}], "snapshot-log": [{"snapshot-id": "3051729675574597004", "timestamp-ms": 1515100955770}, {"snapshot-id": "3055729675574597004", "timestamp-ms": 1555100955770}], "metadata-log": [{"metadata-file": "s3://bucket/.../v1.json", "timestamp-ms": 1515100}], "sort-orders": [{"order-id": 3, "fields": [{"source-id": 2, "transform": "identity", "direction": "asc", "null-order": "nulls-first"}, {"source-id": 3, "transform": "bucket[4]", "direction": "desc", "null-order": "nulls-last"}]}], "default-sort-order-id": 3, "refs": {"test": {"snapshot-id": 3051729675574597004, "type": "tag", "max-ref-age-ms": 10000000}, "main": {"snapshot-id": 3055729675574597004, "type": "branch"}}, "format-version": 2, "last-sequence-number": 34}""" assert table_metadata == expected -def test_migrate_v1_schemas(): +def test_migrate_v1_schemas() -> None: table_metadata = TableMetadataV1(**EXAMPLE_TABLE_METADATA_V1) assert isinstance(table_metadata, TableMetadataV1) @@ -195,7 +195,7 @@ def test_migrate_v1_schemas(): assert table_metadata.schemas[0] == table_metadata.schema_ -def test_migrate_v1_partition_specs(): +def test_migrate_v1_partition_specs() -> None: # Copy the example, and add a spec table_metadata = TableMetadataV1(**EXAMPLE_TABLE_METADATA_V1) assert isinstance(table_metadata, TableMetadataV1) @@ -206,7 +206,7 @@ def test_migrate_v1_partition_specs(): ] -def test_invalid_format_version(): +def test_invalid_format_version() -> None: """Test the exception when trying to load an unknown version""" table_metadata_invalid_format_version = { "format-version": -1, @@ -234,7 +234,7 @@ def test_invalid_format_version(): assert "Unknown format version: -1" in str(exc_info.value) -def test_current_schema_not_found(): +def test_current_schema_not_found() -> None: """Test that we raise an exception when the schema can't be found""" table_metadata_schema_not_found = { @@ -272,7 +272,7 @@ def test_current_schema_not_found(): assert "current-schema-id 2 can't be found in the schemas" in str(exc_info.value) -def test_sort_order_not_found(): +def test_sort_order_not_found() -> None: """Test that we raise an exception when the schema can't be found""" table_metadata_schema_not_found = { @@ -318,7 +318,7 @@ def test_sort_order_not_found(): assert "default-sort-order-id 4 can't be found" in str(exc_info.value) -def test_sort_order_unsorted(): +def test_sort_order_unsorted() -> None: """Test that we raise an exception when the schema can't be found""" table_metadata_schema_not_found = { @@ -356,7 +356,7 @@ def test_sort_order_unsorted(): assert len(table_metadata.sort_orders) == 0 -def test_invalid_partition_spec(): +def test_invalid_partition_spec() -> None: table_metadata_spec_not_found = { "format-version": 2, "table-uuid": "9c12d441-03fe-4693-9a96-a0705ddf69c1", @@ -390,7 +390,7 @@ def test_invalid_partition_spec(): assert "default-spec-id 1 can't be found" in str(exc_info.value) -def test_v1_writing_metadata(): +def test_v1_writing_metadata() -> None: """ https://iceberg.apache.org/spec/#version-2 @@ -405,7 +405,7 @@ def test_v1_writing_metadata(): assert "last-sequence-number" not in metadata_v1 -def test_v1_metadata_for_v2(): +def test_v1_metadata_for_v2() -> None: """ https://iceberg.apache.org/spec/#version-2 @@ -418,7 +418,7 @@ def test_v1_metadata_for_v2(): assert table_metadata.last_sequence_number == 0 -def test_v1_write_metadata_for_v2(): +def test_v1_write_metadata_for_v2() -> None: """ https://iceberg.apache.org/spec/#version-2 @@ -489,7 +489,7 @@ def test_v1_write_metadata_for_v2(): assert "partition-spec" not in metadata_v2 -def test_v2_ref_creation(example_table_metadata_v2: Dict[str, Any]): +def test_v2_ref_creation(example_table_metadata_v2: Dict[str, Any]) -> None: table_metadata = TableMetadataV2(**example_table_metadata_v2) assert table_metadata.refs == { "main": SnapshotRef( @@ -509,7 +509,7 @@ def test_v2_ref_creation(example_table_metadata_v2: Dict[str, Any]): } -def test_metadata_v1(): +def test_metadata_v1() -> None: valid_v1 = { "format-version": 1, "table-uuid": "bf289591-dcc0-4234-ad4f-5c3eed811a29", @@ -561,7 +561,7 @@ def test_metadata_v1(): @patch("time.time", MagicMock(return_value=12345)) -def test_make_metadata_fresh(): +def test_make_metadata_fresh() -> None: schema = Schema( NestedField(field_id=10, name="foo", field_type=StringType(), required=False), NestedField(field_id=22, name="bar", field_type=IntegerType(), required=True), @@ -611,14 +611,12 @@ def test_make_metadata_fresh(): ) partition_spec = PartitionSpec( - spec_id=10, fields=(PartitionField(source_id=22, field_id=1022, transform=IdentityTransform(), name="bar"),) + PartitionField(source_id=22, field_id=1022, transform=IdentityTransform(), name="bar"), spec_id=10 ) sort_order = SortOrder( + SortField(source_id=10, transform=IdentityTransform(), direction=SortDirection.ASC, null_order=NullOrder.NULLS_LAST), order_id=10, - fields=[ - SortField(source_id=10, transform=IdentityTransform(), direction=SortDirection.ASC, null_order=NullOrder.NULLS_LAST) - ], ) actual = new_table_metadata( @@ -668,10 +666,8 @@ def test_make_metadata_fresh(): type="list", element_id=13, element_type=StructType( - fields=( - NestedField(field_id=14, name="latitude", field_type=FloatType(), required=False), - NestedField(field_id=15, name="longitude", field_type=FloatType(), required=False), - ) + NestedField(field_id=14, name="latitude", field_type=FloatType(), required=False), + NestedField(field_id=15, name="longitude", field_type=FloatType(), required=False), ), element_required=True, ), @@ -681,10 +677,8 @@ def test_make_metadata_fresh(): field_id=7, name="person", field_type=StructType( - fields=( - NestedField(field_id=16, name="name", field_type=StringType(), required=False), - NestedField(field_id=17, name="age", field_type=IntegerType(), required=True), - ) + NestedField(field_id=16, name="name", field_type=StringType(), required=False), + NestedField(field_id=17, name="age", field_type=IntegerType(), required=True), ), required=False, ), @@ -693,11 +687,7 @@ def test_make_metadata_fresh(): ) ], current_schema_id=0, - partition_specs=[ - PartitionSpec( - spec_id=0, fields=(PartitionField(source_id=2, field_id=1000, transform=IdentityTransform(), name="bar"),) - ) - ], + partition_specs=[PartitionSpec(PartitionField(source_id=2, field_id=1000, transform=IdentityTransform(), name="bar"))], default_spec_id=0, last_partition_id=1000, properties={}, @@ -707,12 +697,10 @@ def test_make_metadata_fresh(): metadata_log=[], sort_orders=[ SortOrder( + SortField( + source_id=1, transform=IdentityTransform(), direction=SortDirection.ASC, null_order=NullOrder.NULLS_LAST + ), order_id=1, - fields=[ - SortField( - source_id=1, transform=IdentityTransform(), direction=SortDirection.ASC, null_order=NullOrder.NULLS_LAST - ) - ], ) ], default_sort_order_id=1, diff --git a/tests/table/test_partitioning.py b/tests/table/test_partitioning.py index 9c37822360..b2d78c49ba 100644 --- a/tests/table/test_partitioning.py +++ b/tests/table/test_partitioning.py @@ -25,8 +25,8 @@ ) -def test_partition_field_init(): - bucket_transform = BucketTransform(100) +def test_partition_field_init() -> None: + bucket_transform = BucketTransform(100) # type: ignore partition_field = PartitionField(3, 1000, bucket_transform, "id") assert partition_field.source_id == 3 @@ -41,12 +41,12 @@ def test_partition_field_init(): ) -def test_unpartitioned_partition_spec_repr(): +def test_unpartitioned_partition_spec_repr() -> None: assert repr(PartitionSpec()) == "PartitionSpec(spec_id=0)" -def test_partition_spec_init(): - bucket_transform: BucketTransform = BucketTransform(4) +def test_partition_spec_init() -> None: + bucket_transform: BucketTransform = BucketTransform(4) # type: ignore id_field1 = PartitionField(3, 1001, bucket_transform, "id") partition_spec1 = PartitionSpec(id_field1) @@ -66,8 +66,8 @@ def test_partition_spec_init(): assert partition_spec1.fields_by_source_id(1925) == [] -def test_partition_compatible_with(): - bucket_transform: BucketTransform = BucketTransform(4) +def test_partition_compatible_with() -> None: + bucket_transform: BucketTransform = BucketTransform(4) # type: ignore field1 = PartitionField(3, 100, bucket_transform, "id") field2 = PartitionField(3, 102, bucket_transform, "id") lhs = PartitionSpec( @@ -77,17 +77,17 @@ def test_partition_compatible_with(): assert not lhs.compatible_with(rhs) -def test_unpartitioned(): +def test_unpartitioned() -> None: assert len(UNPARTITIONED_PARTITION_SPEC.fields) == 0 assert UNPARTITIONED_PARTITION_SPEC.is_unpartitioned() assert str(UNPARTITIONED_PARTITION_SPEC) == "[]" -def test_serialize_unpartitioned_spec(): +def test_serialize_unpartitioned_spec() -> None: assert UNPARTITIONED_PARTITION_SPEC.json() == """{"spec-id": 0, "fields": []}""" -def test_serialize_partition_spec(): +def test_serialize_partition_spec() -> None: partitioned = PartitionSpec( PartitionField(source_id=1, field_id=1000, transform=TruncateTransform(width=19), name="str_truncate"), PartitionField(source_id=2, field_id=1001, transform=BucketTransform(num_buckets=25), name="int_bucket"), @@ -99,17 +99,15 @@ def test_serialize_partition_spec(): ) -def test_deserialize_partition_spec(): +def test_deserialize_partition_spec() -> None: json_partition_spec = """{"spec-id": 3, "fields": [{"source-id": 1, "field-id": 1000, "transform": "truncate[19]", "name": "str_truncate"}, {"source-id": 2, "field-id": 1001, "transform": "bucket[25]", "name": "int_bucket"}]}""" spec = PartitionSpec.parse_raw(json_partition_spec) assert spec == PartitionSpec( + PartitionField(source_id=1, field_id=1000, transform=TruncateTransform(width=19), name="str_truncate"), + PartitionField(source_id=2, field_id=1001, transform=BucketTransform(num_buckets=25), name="int_bucket"), spec_id=3, - fields=( - PartitionField(source_id=1, field_id=1000, transform=TruncateTransform(width=19), name="str_truncate"), - PartitionField(source_id=2, field_id=1001, transform=BucketTransform(num_buckets=25), name="int_bucket"), - ), ) diff --git a/tests/table/test_refs.py b/tests/table/test_refs.py index abd71ee110..d106f0237a 100644 --- a/tests/table/test_refs.py +++ b/tests/table/test_refs.py @@ -18,7 +18,7 @@ from pyiceberg.table.refs import SnapshotRef, SnapshotRefType -def test_snapshot_with_properties_repr(): +def test_snapshot_with_properties_repr() -> None: snapshot_ref = SnapshotRef( snapshot_id=3051729675574597004, snapshot_ref_type=SnapshotRefType.TAG, diff --git a/tests/table/test_snapshots.py b/tests/table/test_snapshots.py index 88ab8718be..16ba83153e 100644 --- a/tests/table/test_snapshots.py +++ b/tests/table/test_snapshots.py @@ -48,22 +48,22 @@ def snapshot_with_properties() -> Snapshot: ) -def test_serialize_summary(): +def test_serialize_summary() -> None: assert Summary(Operation.APPEND).json() == """{"operation": "append"}""" -def test_serialize_summary_with_properties(): +def test_serialize_summary_with_properties() -> None: assert Summary(Operation.APPEND, property="yes").json() == """{"operation": "append", "property": "yes"}""" -def test_serialize_snapshot(snapshot: Snapshot): +def test_serialize_snapshot(snapshot: Snapshot) -> None: assert ( snapshot.json() == """{"snapshot-id": 25, "parent-snapshot-id": 19, "sequence-number": 200, "timestamp-ms": 1602638573590, "manifest-list": "s3:/a/b/c.avro", "summary": {"operation": "append"}, "schema-id": 3}""" ) -def test_serialize_snapshot_without_sequence_number(): +def test_serialize_snapshot_without_sequence_number() -> None: snapshot = Snapshot( snapshot_id=25, parent_snapshot_id=19, @@ -77,37 +77,37 @@ def test_serialize_snapshot_without_sequence_number(): assert actual == expected -def test_serialize_snapshot_with_properties(snapshot_with_properties: Snapshot): +def test_serialize_snapshot_with_properties(snapshot_with_properties: Snapshot) -> None: assert ( snapshot_with_properties.json() == """{"snapshot-id": 25, "parent-snapshot-id": 19, "sequence-number": 200, "timestamp-ms": 1602638573590, "manifest-list": "s3:/a/b/c.avro", "summary": {"operation": "append", "foo": "bar"}, "schema-id": 3}""" ) -def test_deserialize_summary(): +def test_deserialize_summary() -> None: summary = Summary.parse_raw("""{"operation": "append"}""") assert summary.operation == Operation.APPEND -def test_deserialize_summary_with_properties(): +def test_deserialize_summary_with_properties() -> None: summary = Summary.parse_raw("""{"operation": "append", "property": "yes"}""") assert summary.operation == Operation.APPEND assert summary.additional_properties == {"property": "yes"} -def test_deserialize_snapshot(snapshot: Snapshot): +def test_deserialize_snapshot(snapshot: Snapshot) -> None: payload = """{"snapshot-id": 25, "parent-snapshot-id": 19, "sequence-number": 200, "timestamp-ms": 1602638573590, "manifest-list": "s3:/a/b/c.avro", "summary": {"operation": "append"}, "schema-id": 3}""" actual = Snapshot.parse_raw(payload) assert actual == snapshot -def test_deserialize_snapshot_with_properties(snapshot_with_properties: Snapshot): +def test_deserialize_snapshot_with_properties(snapshot_with_properties: Snapshot) -> None: payload = """{"snapshot-id": 25, "parent-snapshot-id": 19, "sequence-number": 200, "timestamp-ms": 1602638573590, "manifest-list": "s3:/a/b/c.avro", "summary": {"operation": "append", "foo": "bar"}, "schema-id": 3}""" snapshot = Snapshot.parse_raw(payload) assert snapshot == snapshot_with_properties -def test_snapshot_repr(snapshot: Snapshot): +def test_snapshot_repr(snapshot: Snapshot) -> None: assert ( repr(snapshot) == """Snapshot(snapshot_id=25, parent_snapshot_id=19, sequence_number=200, timestamp_ms=1602638573590, manifest_list='s3:/a/b/c.avro', summary=Summary(Operation.APPEND), schema_id=3)""" @@ -115,7 +115,7 @@ def test_snapshot_repr(snapshot: Snapshot): assert snapshot == eval(repr(snapshot)) -def test_snapshot_with_properties_repr(snapshot_with_properties: Snapshot): +def test_snapshot_with_properties_repr(snapshot_with_properties: Snapshot) -> None: assert ( repr(snapshot_with_properties) == """Snapshot(snapshot_id=25, parent_snapshot_id=19, sequence_number=200, timestamp_ms=1602638573590, manifest_list='s3:/a/b/c.avro', summary=Summary(Operation.APPEND, **{'foo': 'bar'}), schema_id=3)""" @@ -123,7 +123,7 @@ def test_snapshot_with_properties_repr(snapshot_with_properties: Snapshot): assert snapshot_with_properties == eval(repr(snapshot_with_properties)) -def test_fetch_manifest_list(generated_manifest_file_file: str): +def test_fetch_manifest_list(generated_manifest_file_file: str) -> None: snapshot = Snapshot( snapshot_id=25, parent_snapshot_id=19, diff --git a/tests/table/test_sorting.py b/tests/table/test_sorting.py index 00ad34ba80..ea3fdfce53 100644 --- a/tests/table/test_sorting.py +++ b/tests/table/test_sorting.py @@ -41,22 +41,22 @@ def sort_order() -> SortOrder: ) -def test_serialize_sort_order_unsorted(): +def test_serialize_sort_order_unsorted() -> None: assert UNSORTED_SORT_ORDER.json() == '{"order-id": 0, "fields": []}' -def test_serialize_sort_order(sort_order: SortOrder): +def test_serialize_sort_order(sort_order: SortOrder) -> None: expected = '{"order-id": 22, "fields": [{"source-id": 19, "transform": "identity", "direction": "asc", "null-order": "nulls-first"}, {"source-id": 25, "transform": "bucket[4]", "direction": "desc", "null-order": "nulls-last"}, {"source-id": 22, "transform": "void", "direction": "asc", "null-order": "nulls-first"}]}' assert sort_order.json() == expected -def test_deserialize_sort_order(sort_order: SortOrder): +def test_deserialize_sort_order(sort_order: SortOrder) -> None: payload = '{"order-id": 22, "fields": [{"source-id": 19, "transform": "identity", "direction": "asc", "null-order": "nulls-first"}, {"source-id": 25, "transform": "bucket[4]", "direction": "desc", "null-order": "nulls-last"}, {"source-id": 22, "transform": "void", "direction": "asc", "null-order": "nulls-first"}]}' assert SortOrder.parse_raw(payload) == sort_order -def test_sorting_schema(example_table_metadata_v2: Dict[str, Any]): +def test_sorting_schema(example_table_metadata_v2: Dict[str, Any]) -> None: table_metadata = TableMetadataUtil.parse_obj(example_table_metadata_v2) assert table_metadata.sort_orders == [ @@ -73,7 +73,7 @@ def test_sorting_schema(example_table_metadata_v2: Dict[str, Any]): ] -def test_sorting_to_string(sort_order: SortOrder): +def test_sorting_to_string(sort_order: SortOrder) -> None: expected = """[ 19 ASC NULLS FIRST bucket[4](25) DESC NULLS LAST @@ -82,16 +82,16 @@ def test_sorting_to_string(sort_order: SortOrder): assert str(sort_order) == expected -def test_sorting_to_repr(sort_order: SortOrder): +def test_sorting_to_repr(sort_order: SortOrder) -> None: expected = """SortOrder(SortField(source_id=19, transform=IdentityTransform(), direction=SortDirection.ASC, null_order=NullOrder.NULLS_FIRST), SortField(source_id=25, transform=BucketTransform(num_buckets=4), direction=SortDirection.DESC, null_order=NullOrder.NULLS_LAST), SortField(source_id=22, transform=VoidTransform(), direction=SortDirection.ASC, null_order=NullOrder.NULLS_FIRST), order_id=22)""" assert repr(sort_order) == expected -def test_unsorting_to_repr(): +def test_unsorting_to_repr() -> None: expected = """SortOrder(order_id=0)""" assert repr(UNSORTED_SORT_ORDER) == expected -def test_sorting_repr(sort_order: SortOrder): +def test_sorting_repr(sort_order: SortOrder) -> None: """To make sure that the repr converts back to the original object""" assert sort_order == eval(repr(sort_order)) diff --git a/tests/test_conversions.py b/tests/test_conversions.py index 86f015b454..94f553741f 100644 --- a/tests/test_conversions.py +++ b/tests/test_conversions.py @@ -74,6 +74,7 @@ import struct import uuid from decimal import Decimal +from typing import Any import pytest @@ -89,6 +90,7 @@ FloatType, IntegerType, LongType, + PrimitiveType, StringType, TimestampType, TimestamptzType, @@ -111,7 +113,7 @@ (Decimal("0.0000001"), 1), ], ) -def test_decimal_to_unscaled(value, expected_result): +def test_decimal_to_unscaled(value: Decimal, expected_result: int) -> None: """Test converting a decimal to an unscaled value""" assert decimal_util.decimal_to_unscaled(value=value) == expected_result @@ -129,7 +131,7 @@ def test_decimal_to_unscaled(value, expected_result): (1, 7, Decimal("0.0000001")), ], ) -def test_unscaled_to_decimal(unscaled, scale, expected_result): +def test_unscaled_to_decimal(unscaled: int, scale: int, expected_result: Decimal) -> None: """Test converting an unscaled value to a decimal with a specified scale""" assert decimal_util.unscaled_to_decimal(unscaled=unscaled, scale=scale) == expected_result @@ -153,7 +155,7 @@ def test_unscaled_to_decimal(unscaled, scale, expected_result): (BinaryType(), "foo", b"foo"), ], ) -def test_partition_to_py(primitive_type, value_str, expected_result): +def test_partition_to_py(primitive_type: PrimitiveType, value_str: str, expected_result: Any) -> None: """Test converting a partition value to a python built-in""" assert conversions.partition_to_py(primitive_type, value_str) == expected_result @@ -177,9 +179,9 @@ def test_partition_to_py(primitive_type, value_str, expected_result): (UUIDType()), ], ) -def test_none_partition_values(primitive_type): +def test_none_partition_values(primitive_type: PrimitiveType) -> None: """Test converting a partition value to a python built-in""" - assert conversions.partition_to_py(primitive_type, None) is None + assert conversions.partition_to_py(primitive_type, None) is None # type: ignore @pytest.mark.parametrize( @@ -201,7 +203,7 @@ def test_none_partition_values(primitive_type): (UUIDType()), ], ) -def test_hive_default_partition_values(primitive_type): +def test_hive_default_partition_values(primitive_type: PrimitiveType) -> None: """Test converting a partition value to a python built-in""" assert conversions.partition_to_py(primitive_type, "__HIVE_DEFAULT_PARTITION__") is None @@ -227,7 +229,9 @@ def test_hive_default_partition_values(primitive_type): (LongType(), "123456700", False), ], ) -def test_partition_to_py_raise_on_incorrect_precision_or_scale(primitive_type, value, should_raise): +def test_partition_to_py_raise_on_incorrect_precision_or_scale( + primitive_type: PrimitiveType, value: str, should_raise: bool +) -> None: if should_raise: with pytest.raises(ValueError) as exc_info: conversions.partition_to_py(primitive_type, value) @@ -270,7 +274,7 @@ def test_partition_to_py_raise_on_incorrect_precision_or_scale(primitive_type, v (DecimalType(7, 4), b"\xff\xed\x29\x79", Decimal("-123.4567")), ], ) -def test_from_bytes(primitive_type, b, result): +def test_from_bytes(primitive_type: PrimitiveType, b: bytes, result: Any) -> None: """Test converting from bytes""" assert conversions.from_bytes(primitive_type, b) == result @@ -322,7 +326,7 @@ def test_from_bytes(primitive_type, b, result): (FloatType(), b"\x19\x04\x9e?", struct.unpack(" None: """Test round trip conversions of calling `conversions.from_bytes` and then `conversions.to_bytes` on the result""" value_from_bytes = conversions.from_bytes(primitive_type, b) assert value_from_bytes == result @@ -402,7 +406,7 @@ def test_round_trip_conversion(primitive_type, b, result): (DecimalType(31, 26), b"\x0f\x94\xec\xad\xa5C<]\xdePf\xd6N", Decimal("12345.12345678912345678912345678")), ], ) -def test_round_trip_conversion_large_decimals(primitive_type, b, result): +def test_round_trip_conversion_large_decimals(primitive_type: PrimitiveType, b: bytes, result: Any) -> None: """Test round trip conversions of calling `conversions.from_bytes` and then `conversions.to_bytes` on the result""" value_from_bytes = conversions.from_bytes(primitive_type, b) assert value_from_bytes == result @@ -421,7 +425,7 @@ def test_round_trip_conversion_large_decimals(primitive_type, b, result): (DecimalType(20, 1), Decimal("9999999999999999999.9")), ], ) -def test_max_value_round_trip_conversion(primitive_type, expected_max_value): +def test_max_value_round_trip_conversion(primitive_type: DecimalType, expected_max_value: Decimal) -> None: """Test round trip conversions of maximum DecimalType values""" b = conversions.to_bytes(primitive_type, expected_max_value) value_from_bytes = conversions.from_bytes(primitive_type, b) @@ -439,7 +443,7 @@ def test_max_value_round_trip_conversion(primitive_type, expected_max_value): (DecimalType(20, 1), Decimal("-9999999999999999999.9")), ], ) -def test_min_value_round_trip_conversion(primitive_type, expected_min_value): +def test_min_value_round_trip_conversion(primitive_type: DecimalType, expected_min_value: Decimal) -> None: """Test round trip conversions of minimum DecimalType values""" b = conversions.to_bytes(primitive_type, expected_min_value) value_from_bytes = conversions.from_bytes(primitive_type, b) @@ -447,23 +451,23 @@ def test_min_value_round_trip_conversion(primitive_type, expected_min_value): assert value_from_bytes == expected_min_value -def test_raise_on_unregistered_type(): +def test_raise_on_unregistered_type() -> None: """Test raising when a conversion is attempted for a type that has no registered method""" class FooUnknownType: - def __repr__(self): + def __repr__(self) -> str: return "FooUnknownType()" with pytest.raises(TypeError) as exc_info: - conversions.partition_to_py(FooUnknownType(), "foo") + conversions.partition_to_py(FooUnknownType(), "foo") # type: ignore assert "Cannot convert 'foo' to unsupported type: FooUnknownType()" in str(exc_info.value) with pytest.raises(TypeError) as exc_info: - conversions.to_bytes(FooUnknownType(), "foo") + conversions.to_bytes(FooUnknownType(), "foo") # type: ignore assert "scale does not match FooUnknownType()" in str(exc_info.value) with pytest.raises(TypeError) as exc_info: - conversions.from_bytes(FooUnknownType(), b"foo") + conversions.from_bytes(FooUnknownType(), b"foo") # type: ignore assert "Cannot deserialize bytes, type FooUnknownType() not supported: b'foo'" in str(exc_info.value) @@ -498,7 +502,7 @@ def __repr__(self): ), ], ) -def test_raise_on_incorrect_precision_or_scale(primitive_type, value, expected_error_message): +def test_raise_on_incorrect_precision_or_scale(primitive_type: DecimalType, value: Decimal, expected_error_message: str) -> None: with pytest.raises(ValueError) as exc_info: conversions.to_bytes(primitive_type, value) diff --git a/tests/test_schema.py b/tests/test_schema.py index ee8513bee2..a6fba2cff0 100644 --- a/tests/test_schema.py +++ b/tests/test_schema.py @@ -36,7 +36,7 @@ ) -def test_schema_str(table_schema_simple: Schema): +def test_schema_str(table_schema_simple: Schema) -> None: """Test casting a schema to a string""" assert str(table_schema_simple) == dedent( """\ @@ -48,14 +48,14 @@ def test_schema_str(table_schema_simple: Schema): ) -def test_schema_repr_single_field(): +def test_schema_repr_single_field() -> None: """Test schema representation""" actual = repr(schema.Schema(NestedField(1, "foo", StringType()), schema_id=1)) expected = "Schema(NestedField(field_id=1, name='foo', field_type=StringType(), required=True), schema_id=1, identifier_field_ids=[])" assert expected == actual -def test_schema_repr_two_fields(): +def test_schema_repr_two_fields() -> None: """Test schema representation""" actual = repr( schema.Schema(NestedField(1, "foo", StringType()), NestedField(2, "bar", IntegerType(), required=False), schema_id=1) @@ -64,7 +64,7 @@ def test_schema_repr_two_fields(): assert expected == actual -def test_schema_raise_on_duplicate_names(): +def test_schema_raise_on_duplicate_names() -> None: """Test schema representation""" with pytest.raises(ValueError) as exc_info: schema.Schema( @@ -79,7 +79,7 @@ def test_schema_raise_on_duplicate_names(): assert "Invalid schema, multiple fields for name baz: 3 and 4" in str(exc_info.value) -def test_schema_index_by_id_visitor(table_schema_nested): +def test_schema_index_by_id_visitor(table_schema_nested: Schema) -> None: """Test index_by_id visitor function""" index = schema.index_by_id(table_schema_nested) assert index == { @@ -152,7 +152,7 @@ def test_schema_index_by_id_visitor(table_schema_nested): } -def test_schema_index_by_name_visitor(table_schema_nested): +def test_schema_index_by_name_visitor(table_schema_nested: Schema) -> None: """Test index_by_name visitor function""" index = schema.index_by_name(table_schema_nested) assert index == { @@ -178,7 +178,7 @@ def test_schema_index_by_name_visitor(table_schema_nested): } -def test_schema_find_column_name(table_schema_nested): +def test_schema_find_column_name(table_schema_nested: Schema) -> None: """Test finding a column name using its field ID""" assert table_schema_nested.find_column_name(1) == "foo" assert table_schema_nested.find_column_name(2) == "bar" @@ -196,19 +196,19 @@ def test_schema_find_column_name(table_schema_nested): assert table_schema_nested.find_column_name(14) == "location.element.longitude" -def test_schema_find_column_name_on_id_not_found(table_schema_nested): +def test_schema_find_column_name_on_id_not_found(table_schema_nested: Schema) -> None: """Test raising an error when a field ID cannot be found""" assert table_schema_nested.find_column_name(99) is None -def test_schema_find_column_name_by_id(table_schema_simple): +def test_schema_find_column_name_by_id(table_schema_simple: Schema) -> None: """Test finding a column name given its field ID""" assert table_schema_simple.find_column_name(1) == "foo" assert table_schema_simple.find_column_name(2) == "bar" assert table_schema_simple.find_column_name(3) == "baz" -def test_schema_find_field_by_id(table_schema_simple): +def test_schema_find_field_by_id(table_schema_simple: Schema) -> None: """Test finding a column using its field ID""" index = schema.index_by_id(table_schema_simple) @@ -231,7 +231,7 @@ def test_schema_find_field_by_id(table_schema_simple): assert column3.required is False -def test_schema_find_field_by_id_raise_on_unknown_field(table_schema_simple): +def test_schema_find_field_by_id_raise_on_unknown_field(table_schema_simple: Schema) -> None: """Test raising when the field ID is not found among columns""" index = schema.index_by_id(table_schema_simple) with pytest.raises(Exception) as exc_info: @@ -239,7 +239,7 @@ def test_schema_find_field_by_id_raise_on_unknown_field(table_schema_simple): assert str(exc_info.value) == "4" -def test_schema_find_field_type_by_id(table_schema_simple): +def test_schema_find_field_type_by_id(table_schema_simple: Schema) -> None: """Test retrieving a columns' type using its field ID""" index = schema.index_by_id(table_schema_simple) assert index[1] == NestedField(field_id=1, name="foo", field_type=StringType(), required=False) @@ -247,7 +247,7 @@ def test_schema_find_field_type_by_id(table_schema_simple): assert index[3] == NestedField(field_id=3, name="baz", field_type=BooleanType(), required=False) -def test_index_by_id_schema_visitor(table_schema_nested): +def test_index_by_id_schema_visitor(table_schema_nested: Schema) -> None: """Test the index_by_id function that uses the IndexById schema visitor""" assert schema.index_by_id(table_schema_nested) == { 1: NestedField(field_id=1, name="foo", field_type=StringType(), required=False), @@ -319,14 +319,14 @@ def test_index_by_id_schema_visitor(table_schema_nested): } -def test_index_by_id_schema_visitor_raise_on_unregistered_type(): +def test_index_by_id_schema_visitor_raise_on_unregistered_type() -> None: """Test raising a NotImplementedError when an invalid type is provided to the index_by_id function""" with pytest.raises(NotImplementedError) as exc_info: - schema.index_by_id("foo") + schema.index_by_id("foo") # type: ignore assert "Cannot visit non-type: foo" in str(exc_info.value) -def test_schema_find_field(table_schema_simple): +def test_schema_find_field(table_schema_simple: Schema) -> None: """Test finding a field in a schema""" assert ( table_schema_simple.find_field(1) @@ -348,7 +348,7 @@ def test_schema_find_field(table_schema_simple): ) -def test_schema_find_type(table_schema_simple): +def test_schema_find_type(table_schema_simple: Schema) -> None: """Test finding the type of a column given its field ID""" assert ( table_schema_simple.find_type(1) @@ -370,7 +370,7 @@ def test_schema_find_type(table_schema_simple): ) -def test_build_position_accessors(table_schema_nested): +def test_build_position_accessors(table_schema_nested: Schema) -> None: accessors = build_position_accessors(table_schema_nested) assert accessors == { 1: Accessor(position=0, inner=None), @@ -384,12 +384,12 @@ def test_build_position_accessors(table_schema_nested): } -def test_build_position_accessors_with_struct(table_schema_nested: Schema): +def test_build_position_accessors_with_struct(table_schema_nested: Schema) -> None: class TestStruct(StructProtocol): def __init__(self, pos: Dict[int, Any] = EMPTY_DICT): self._pos: Dict[int, Any] = pos - def set(self, pos: int, value) -> None: + def set(self, pos: int, value: Any) -> None: pass def get(self, pos: int) -> Any: @@ -402,13 +402,13 @@ def get(self, pos: int) -> Any: assert inner_accessor.get(container) == "name" -def test_serialize_schema(table_schema_simple: Schema): +def test_serialize_schema(table_schema_simple: Schema) -> None: actual = table_schema_simple.json() expected = """{"type": "struct", "fields": [{"id": 1, "name": "foo", "type": "string", "required": false}, {"id": 2, "name": "bar", "type": "int", "required": true}, {"id": 3, "name": "baz", "type": "boolean", "required": false}], "schema-id": 1, "identifier-field-ids": [2]}""" assert actual == expected -def test_deserialize_schema(table_schema_simple: Schema): +def test_deserialize_schema(table_schema_simple: Schema) -> None: actual = Schema.parse_raw( """{"type": "struct", "fields": [{"id": 1, "name": "foo", "type": "string", "required": false}, {"id": 2, "name": "bar", "type": "int", "required": true}, {"id": 3, "name": "baz", "type": "boolean", "required": false}], "schema-id": 1, "identifier-field-ids": [2]}""" ) @@ -416,19 +416,19 @@ def test_deserialize_schema(table_schema_simple: Schema): assert actual == expected -def test_prune_columns_string(table_schema_nested: Schema): +def test_prune_columns_string(table_schema_nested: Schema) -> None: assert prune_columns(table_schema_nested, {1}, False) == Schema( NestedField(field_id=1, name="foo", field_type=StringType(), required=False), schema_id=1, identifier_field_ids=[1] ) -def test_prune_columns_string_full(table_schema_nested: Schema): +def test_prune_columns_string_full(table_schema_nested: Schema) -> None: assert prune_columns(table_schema_nested, {1}, True) == Schema( NestedField(field_id=1, name="foo", field_type=StringType(), required=False), schema_id=1, identifier_field_ids=[1] ) -def test_prune_columns_list(table_schema_nested: Schema): +def test_prune_columns_list(table_schema_nested: Schema) -> None: assert prune_columns(table_schema_nested, {5}, False) == Schema( NestedField( field_id=4, @@ -441,13 +441,13 @@ def test_prune_columns_list(table_schema_nested: Schema): ) -def test_prune_columns_list_itself(table_schema_nested: Schema): +def test_prune_columns_list_itself(table_schema_nested: Schema) -> None: with pytest.raises(ValueError) as exc_info: assert prune_columns(table_schema_nested, {4}, False) assert "Cannot explicitly project List or Map types, 4:qux of type list was selected" in str(exc_info.value) -def test_prune_columns_list_full(table_schema_nested: Schema): +def test_prune_columns_list_full(table_schema_nested: Schema) -> None: assert prune_columns(table_schema_nested, {5}, True) == Schema( NestedField( field_id=4, @@ -460,7 +460,7 @@ def test_prune_columns_list_full(table_schema_nested: Schema): ) -def test_prune_columns_map(table_schema_nested: Schema): +def test_prune_columns_map(table_schema_nested: Schema) -> None: assert prune_columns(table_schema_nested, {9}, False) == Schema( NestedField( field_id=6, @@ -482,7 +482,7 @@ def test_prune_columns_map(table_schema_nested: Schema): ) -def test_prune_columns_map_itself(table_schema_nested: Schema): +def test_prune_columns_map_itself(table_schema_nested: Schema) -> None: with pytest.raises(ValueError) as exc_info: assert prune_columns(table_schema_nested, {6}, False) assert "Cannot explicitly project List or Map types, 6:quux of type map> was selected" in str( @@ -490,7 +490,7 @@ def test_prune_columns_map_itself(table_schema_nested: Schema): ) -def test_prune_columns_map_full(table_schema_nested: Schema): +def test_prune_columns_map_full(table_schema_nested: Schema) -> None: assert prune_columns(table_schema_nested, {9}, True) == Schema( NestedField( field_id=6, @@ -512,7 +512,7 @@ def test_prune_columns_map_full(table_schema_nested: Schema): ) -def test_prune_columns_map_key(table_schema_nested: Schema): +def test_prune_columns_map_key(table_schema_nested: Schema) -> None: assert prune_columns(table_schema_nested, {10}, False) == Schema( NestedField( field_id=6, @@ -534,7 +534,7 @@ def test_prune_columns_map_key(table_schema_nested: Schema): ) -def test_prune_columns_struct(table_schema_nested: Schema): +def test_prune_columns_struct(table_schema_nested: Schema) -> None: assert prune_columns(table_schema_nested, {16}, False) == Schema( NestedField( field_id=15, @@ -547,7 +547,7 @@ def test_prune_columns_struct(table_schema_nested: Schema): ) -def test_prune_columns_struct_full(table_schema_nested: Schema): +def test_prune_columns_struct_full(table_schema_nested: Schema) -> None: actual = prune_columns(table_schema_nested, {16}, True) assert actual == Schema( NestedField( @@ -561,7 +561,7 @@ def test_prune_columns_struct_full(table_schema_nested: Schema): ) -def test_prune_columns_empty_struct(): +def test_prune_columns_empty_struct() -> None: schema_empty_struct = Schema( NestedField( field_id=15, @@ -575,7 +575,7 @@ def test_prune_columns_empty_struct(): ) -def test_prune_columns_empty_struct_full(): +def test_prune_columns_empty_struct_full() -> None: schema_empty_struct = Schema( NestedField( field_id=15, @@ -589,7 +589,7 @@ def test_prune_columns_empty_struct_full(): ) -def test_prune_columns_struct_in_map(): +def test_prune_columns_struct_in_map() -> None: table_schema_nested = Schema( NestedField( field_id=6, @@ -628,7 +628,7 @@ def test_prune_columns_struct_in_map(): ) -def test_prune_columns_struct_in_map_full(): +def test_prune_columns_struct_in_map_full() -> None: table_schema_nested = Schema( NestedField( field_id=6, @@ -667,12 +667,12 @@ def test_prune_columns_struct_in_map_full(): ) -def test_prune_columns_select_original_schema(table_schema_nested: Schema): +def test_prune_columns_select_original_schema(table_schema_nested: Schema) -> None: ids = set(range(table_schema_nested.highest_field_id)) assert prune_columns(table_schema_nested, ids, True) == table_schema_nested -def test_schema_select(table_schema_nested: Schema): +def test_schema_select(table_schema_nested: Schema) -> None: assert table_schema_nested.select("bar", "baz") == Schema( NestedField(field_id=2, name="bar", field_type=IntegerType(), required=True), NestedField(field_id=3, name="baz", field_type=BooleanType(), required=False), @@ -681,13 +681,13 @@ def test_schema_select(table_schema_nested: Schema): ) -def test_schema_select_case_insensitive(table_schema_nested: Schema): +def test_schema_select_case_insensitive(table_schema_nested: Schema) -> None: assert table_schema_nested.select("BAZ", case_sensitive=False) == Schema( NestedField(field_id=3, name="baz", field_type=BooleanType(), required=False), schema_id=1, identifier_field_ids=[] ) -def test_schema_select_cant_be_found(table_schema_nested: Schema): +def test_schema_select_cant_be_found(table_schema_nested: Schema) -> None: with pytest.raises(ValueError) as exc_info: table_schema_nested.select("BAZ", case_sensitive=True) assert "Could not find column: 'BAZ'" in str(exc_info.value) diff --git a/tests/test_transforms.py b/tests/test_transforms.py index 9395db13c2..72776b8574 100644 --- a/tests/test_transforms.py +++ b/tests/test_transforms.py @@ -17,7 +17,7 @@ # pylint: disable=eval-used,protected-access,redefined-outer-name from datetime import date from decimal import Decimal -from typing import Any +from typing import Any, Callable from uuid import UUID import mmh3 as mmh3 @@ -55,6 +55,7 @@ HourTransform, IdentityTransform, MonthTransform, + TimeTransform, Transform, TruncateTransform, UnknownTransform, @@ -72,6 +73,7 @@ IntegerType, LongType, NestedField, + PrimitiveType, StringType, TimestampType, TimestamptzType, @@ -113,7 +115,7 @@ (UUID("f79c3e09-677c-4bbd-a479-3f349cb785e7"), UUIDType(), 1488055340), ], ) -def test_bucket_hash_values(test_input, test_type, expected): +def test_bucket_hash_values(test_input: Any, test_type: PrimitiveType, expected: Any) -> None: assert BucketTransform(num_buckets=8).transform(test_type, bucket=False)(test_input) == expected @@ -136,7 +138,7 @@ def test_bucket_hash_values(test_input, test_type, expected): (BucketTransform(128).transform(BinaryType()), b"\x00\x01\x02\x03", 57), ], ) -def test_buckets(transform, value, expected): +def test_buckets(transform: Callable[[Any], int], value: Any, expected: int) -> None: assert transform(value) == expected @@ -156,8 +158,8 @@ def test_buckets(transform, value, expected): UUIDType(), ], ) -def test_bucket_method(type_var): - bucket_transform = BucketTransform(8) +def test_bucket_method(type_var: PrimitiveType) -> None: + bucket_transform = BucketTransform(8) # type: ignore assert str(bucket_transform) == str(eval(repr(bucket_transform))) assert bucket_transform.can_transform(type_var) assert bucket_transform.result_type(type_var) == IntegerType() @@ -166,7 +168,7 @@ def test_bucket_method(type_var): assert bucket_transform.to_human_string(type_var, "test") == "test" -def test_string_with_surrogate_pair(): +def test_string_with_surrogate_pair() -> None: string_with_surrogate_pair = "string with a surrogate pair: 💰" as_bytes = bytes(string_with_surrogate_pair, "UTF-8") bucket_transform = BucketTransform(100).transform(StringType(), bucket=False) @@ -181,7 +183,7 @@ def test_string_with_surrogate_pair(): (17501, DayTransform(), "2017-12-01"), ], ) -def test_date_to_human_string(date_val, date_transform, expected): +def test_date_to_human_string(date_val: int, date_transform: TimeTransform[Any], expected: str) -> None: assert date_transform.to_human_string(DateType(), date_val) == expected @@ -193,13 +195,13 @@ def test_date_to_human_string(date_val, date_transform, expected): DayTransform(), ], ) -def test_none_date_to_human_string(date_transform): +def test_none_date_to_human_string(date_transform: TimeTransform[Any]) -> None: assert date_transform.to_human_string(DateType(), None) == "null" -def test_hour_to_human_string(): +def test_hour_to_human_string() -> None: assert HourTransform().to_human_string(TimestampType(), None) == "null" - assert HourTransform().to_human_string(TimestampType(), 420042) == "2017-12-01-18" + assert HourTransform().to_human_string(TimestampType(), 420042) == "2017-12-01-18" # type: ignore @pytest.mark.parametrize( @@ -211,7 +213,7 @@ def test_hour_to_human_string(): (-1, HourTransform(), "1969-12-31-23"), ], ) -def test_negative_value_to_human_string(negative_value, time_transform, expected): +def test_negative_value_to_human_string(negative_value: int, time_transform: TimeTransform[Any], expected: str) -> None: assert time_transform.to_human_string(TimestampType(), negative_value) == expected @@ -223,7 +225,7 @@ def test_negative_value_to_human_string(negative_value, time_transform, expected TimestamptzType(), ], ) -def test_time_methods(type_var): +def test_time_methods(type_var: PrimitiveType) -> None: assert YearTransform().can_transform(type_var) assert MonthTransform().can_transform(type_var) assert DayTransform().can_transform(type_var) @@ -255,7 +257,7 @@ def test_time_methods(type_var): (DayTransform(), TimestampType(), -1, -1), ], ) -def test_time_apply_method(transform, type_var, value, expected): +def test_time_apply_method(transform: TimeTransform[Any], type_var: PrimitiveType, value: int, expected: int) -> None: assert transform.transform(type_var)(value) == expected @@ -266,10 +268,10 @@ def test_time_apply_method(transform, type_var, value, expected): TimestamptzType(), ], ) -def test_hour_method(type_var): +def test_hour_method(type_var: PrimitiveType) -> None: assert HourTransform().can_transform(type_var) assert HourTransform().result_type(type_var) == IntegerType() - assert HourTransform().transform(type_var)(1512151975038194) == 420042 + assert HourTransform().transform(type_var)(1512151975038194) == 420042 # type: ignore assert HourTransform().dedup_name == "time" @@ -284,7 +286,7 @@ def test_hour_method(type_var): (DayTransform(), HourTransform()), ], ) -def test_satisfies_order_of_method(transform, other_transform): +def test_satisfies_order_of_method(transform: TimeTransform[Any], other_transform: TimeTransform[Any]) -> None: assert transform.satisfies_order_of(transform) assert other_transform.satisfies_order_of(transform) assert not transform.satisfies_order_of(other_transform) @@ -306,8 +308,8 @@ def test_satisfies_order_of_method(transform, other_transform): (FixedType(100), b"foo", "Zm9v"), ], ) -def test_identity_human_string(type_var, value, expected): - identity = IdentityTransform() +def test_identity_human_string(type_var: PrimitiveType, value: Any, expected: str) -> None: + identity = IdentityTransform() # type: ignore assert identity.to_human_string(type_var, value) == expected @@ -330,8 +332,8 @@ def test_identity_human_string(type_var, value, expected): UUIDType(), ], ) -def test_identity_method(type_var): - identity_transform = IdentityTransform() +def test_identity_method(type_var: PrimitiveType) -> None: + identity_transform = IdentityTransform() # type: ignore assert str(identity_transform) == str(eval(repr(identity_transform))) assert identity_transform.can_transform(type_var) assert identity_transform.result_type(type_var) == type_var @@ -343,8 +345,8 @@ def test_identity_method(type_var): "input_var,expected", [(1, 0), (5, 0), (9, 0), (10, 10), (11, 10), (-1, -10), (-10, -10), (-12, -20)], ) -def test_truncate_integer(type_var, input_var, expected): - trunc = TruncateTransform(10) +def test_truncate_integer(type_var: PrimitiveType, input_var: int, expected: int) -> None: + trunc = TruncateTransform(10) # type: ignore assert trunc.transform(type_var)(input_var) == expected @@ -358,14 +360,14 @@ def test_truncate_integer(type_var, input_var, expected): (Decimal("-0.05"), Decimal("-0.10")), ], ) -def test_truncate_decimal(input_var, expected): - trunc = TruncateTransform(10) +def test_truncate_decimal(input_var: Decimal, expected: Decimal) -> None: + trunc = TruncateTransform(10) # type: ignore assert trunc.transform(DecimalType(9, 2))(input_var) == expected @pytest.mark.parametrize("input_var,expected", [("abcdefg", "abcde"), ("abc", "abc")]) -def test_truncate_string(input_var, expected): - trunc = TruncateTransform(5) +def test_truncate_string(input_var: str, expected: str) -> None: + trunc = TruncateTransform(5) # type: ignore assert trunc.transform(StringType())(input_var) == expected @@ -381,8 +383,8 @@ def test_truncate_string(input_var, expected): (StringType(), "\u2603de", "\u2603de", "\u2603"), ], ) -def test_truncate_method(type_var, value, expected_human_str, expected): - truncate_transform = TruncateTransform(1) +def test_truncate_method(type_var: PrimitiveType, value: Any, expected_human_str: str, expected: Any) -> None: + truncate_transform = TruncateTransform(1) # type: ignore assert str(truncate_transform) == str(eval(repr(truncate_transform))) assert truncate_transform.can_transform(type_var) assert truncate_transform.result_type(type_var) == type_var @@ -395,8 +397,8 @@ def test_truncate_method(type_var, value, expected_human_str, expected): assert truncate_transform.satisfies_order_of(truncate_transform) -def test_unknown_transform(): - unknown_transform = transforms.UnknownTransform("unknown") +def test_unknown_transform() -> None: + unknown_transform = transforms.UnknownTransform("unknown") # type: ignore assert str(unknown_transform) == str(eval(repr(unknown_transform))) with pytest.raises(AttributeError): unknown_transform.transform(StringType())("test") @@ -404,8 +406,8 @@ def test_unknown_transform(): assert isinstance(unknown_transform.result_type(BooleanType()), StringType) -def test_void_transform(): - void_transform = VoidTransform() +def test_void_transform() -> None: + void_transform = VoidTransform() # type: ignore assert void_transform is VoidTransform() assert void_transform == eval(repr(void_transform)) assert void_transform.transform(StringType())("test") is None @@ -422,89 +424,89 @@ class TestType(IcebergBaseModel): __root__: Transform[Any, Any] -def test_bucket_transform_serialize(): +def test_bucket_transform_serialize() -> None: assert BucketTransform(num_buckets=22).json() == '"bucket[22]"' -def test_bucket_transform_deserialize(): +def test_bucket_transform_deserialize() -> None: transform = TestType.parse_raw('"bucket[22]"').__root__ assert transform == BucketTransform(num_buckets=22) -def test_bucket_transform_str(): +def test_bucket_transform_str() -> None: assert str(BucketTransform(num_buckets=22)) == "bucket[22]" -def test_bucket_transform_repr(): +def test_bucket_transform_repr() -> None: assert repr(BucketTransform(num_buckets=22)) == "BucketTransform(num_buckets=22)" -def test_truncate_transform_serialize(): +def test_truncate_transform_serialize() -> None: assert UnknownTransform("unknown").json() == '"unknown"' -def test_unknown_transform_deserialize(): +def test_unknown_transform_deserialize() -> None: transform = TestType.parse_raw('"unknown"').__root__ assert transform == UnknownTransform("unknown") -def test_unknown_transform_str(): +def test_unknown_transform_str() -> None: assert str(UnknownTransform("unknown")) == "unknown" -def test_unknown_transform_repr(): +def test_unknown_transform_repr() -> None: assert repr(UnknownTransform("unknown")) == "UnknownTransform(transform='unknown')" -def test_void_transform_serialize(): +def test_void_transform_serialize() -> None: assert VoidTransform().json() == '"void"' -def test_void_transform_deserialize(): +def test_void_transform_deserialize() -> None: transform = TestType.parse_raw('"void"').__root__ assert transform == VoidTransform() -def test_void_transform_str(): +def test_void_transform_str() -> None: assert str(VoidTransform()) == "void" -def test_void_transform_repr(): +def test_void_transform_repr() -> None: assert repr(VoidTransform()) == "VoidTransform()" -def test_year_transform_serialize(): +def test_year_transform_serialize() -> None: assert YearTransform().json() == '"year"' -def test_year_transform_deserialize(): +def test_year_transform_deserialize() -> None: transform = TestType.parse_raw('"year"').__root__ assert transform == YearTransform() -def test_month_transform_serialize(): +def test_month_transform_serialize() -> None: assert MonthTransform().json() == '"month"' -def test_month_transform_deserialize(): +def test_month_transform_deserialize() -> None: transform = TestType.parse_raw('"month"').__root__ assert transform == MonthTransform() -def test_day_transform_serialize(): +def test_day_transform_serialize() -> None: assert DayTransform().json() == '"day"' -def test_day_transform_deserialize(): +def test_day_transform_deserialize() -> None: transform = TestType.parse_raw('"day"').__root__ assert transform == DayTransform() -def test_hour_transform_serialize(): +def test_hour_transform_serialize() -> None: assert HourTransform().json() == '"hour"' -def test_hour_transform_deserialize(): +def test_hour_transform_deserialize() -> None: transform = TestType.parse_raw('"hour"').__root__ assert transform == HourTransform() @@ -518,7 +520,7 @@ def test_hour_transform_deserialize(): (HourTransform(), "hour"), ], ) -def test_datetime_transform_str(transform, transform_str): +def test_datetime_transform_str(transform: TimeTransform[Any], transform_str: str) -> None: assert str(transform) == transform_str @@ -531,7 +533,7 @@ def test_datetime_transform_str(transform, transform_str): (HourTransform(), "HourTransform()"), ], ) -def test_datetime_transform_repr(transform, transform_repr): +def test_datetime_transform_repr(transform: TimeTransform[Any], transform_repr: str) -> None: assert repr(transform) == transform_repr @@ -653,31 +655,31 @@ def test_projection_day_month_not_in(bound_reference_date: BoundReference[int]) ) -def test_projection_day_unary(bound_reference_timestamp) -> None: +def test_projection_day_unary(bound_reference_timestamp: BoundReference[int]) -> None: assert DayTransform().project("name", BoundNotNull(term=bound_reference_timestamp)) == NotNull(term="name") -def test_projection_day_literal(bound_reference_timestamp) -> None: +def test_projection_day_literal(bound_reference_timestamp: BoundReference[int]) -> None: assert DayTransform().project( "name", BoundEqualTo(term=bound_reference_timestamp, literal=TimestampLiteral(1667696874000)) ) == EqualTo(term="name", literal=19) -def test_projection_day_set_same_day(bound_reference_timestamp) -> None: +def test_projection_day_set_same_day(bound_reference_timestamp: BoundReference[int]) -> None: assert DayTransform().project( "name", BoundIn(term=bound_reference_timestamp, literals={TimestampLiteral(1667696874001), TimestampLiteral(1667696874000)}), ) == EqualTo(term="name", literal=19) -def test_projection_day_set_in(bound_reference_timestamp) -> None: +def test_projection_day_set_in(bound_reference_timestamp: BoundReference[int]) -> None: assert DayTransform().project( "name", BoundIn(term=bound_reference_timestamp, literals={TimestampLiteral(1667696874001), TimestampLiteral(1567696874000)}), ) == In(term="name", literals={18, 19}) -def test_projection_day_set_not_in(bound_reference_timestamp) -> None: +def test_projection_day_set_not_in(bound_reference_timestamp: BoundReference[int]) -> None: assert ( DayTransform().project( "name", @@ -712,7 +714,7 @@ def test_projection_day_human(bound_reference_date: BoundReference[int]) -> None ) # >= 2018, 1, 2 -def test_projection_hour_unary(bound_reference_timestamp) -> None: +def test_projection_hour_unary(bound_reference_timestamp: BoundReference[int]) -> None: assert HourTransform().project("name", BoundNotNull(term=bound_reference_timestamp)) == NotNull(term="name") @@ -720,13 +722,13 @@ def test_projection_hour_unary(bound_reference_timestamp) -> None: HOUR_IN_MICROSECONDS = 60 * 60 * 1000 * 1000 -def test_projection_hour_literal(bound_reference_timestamp) -> None: +def test_projection_hour_literal(bound_reference_timestamp: BoundReference[int]) -> None: assert HourTransform().project( "name", BoundEqualTo(term=bound_reference_timestamp, literal=TimestampLiteral(TIMESTAMP_EXAMPLE)) ) == EqualTo(term="name", literal=463249) -def test_projection_hour_set_same_hour(bound_reference_timestamp) -> None: +def test_projection_hour_set_same_hour(bound_reference_timestamp: BoundReference[int]) -> None: assert HourTransform().project( "name", BoundIn( @@ -736,7 +738,7 @@ def test_projection_hour_set_same_hour(bound_reference_timestamp) -> None: ) == EqualTo(term="name", literal=463249) -def test_projection_hour_set_in(bound_reference_timestamp) -> None: +def test_projection_hour_set_in(bound_reference_timestamp: BoundReference[int]) -> None: assert HourTransform().project( "name", BoundIn( @@ -746,7 +748,7 @@ def test_projection_hour_set_in(bound_reference_timestamp) -> None: ) == In(term="name", literals={463249, 463250}) -def test_projection_hour_set_not_in(bound_reference_timestamp) -> None: +def test_projection_hour_set_not_in(bound_reference_timestamp: BoundReference[int]) -> None: assert ( HourTransform().project( "name", @@ -759,11 +761,11 @@ def test_projection_hour_set_not_in(bound_reference_timestamp) -> None: ) -def test_projection_identity_unary(bound_reference_timestamp) -> None: +def test_projection_identity_unary(bound_reference_timestamp: BoundReference[int]) -> None: assert IdentityTransform().project("name", BoundNotNull(term=bound_reference_timestamp)) == NotNull(term="name") -def test_projection_identity_literal(bound_reference_timestamp) -> None: +def test_projection_identity_literal(bound_reference_timestamp: BoundReference[int]) -> None: assert IdentityTransform().project( "name", BoundEqualTo(term=bound_reference_timestamp, literal=TimestampLiteral(TIMESTAMP_EXAMPLE)) ) == EqualTo( @@ -771,7 +773,7 @@ def test_projection_identity_literal(bound_reference_timestamp) -> None: ) -def test_projection_identity_set_in(bound_reference_timestamp) -> None: +def test_projection_identity_set_in(bound_reference_timestamp: BoundReference[int]) -> None: assert IdentityTransform().project( "name", BoundIn( @@ -779,11 +781,12 @@ def test_projection_identity_set_in(bound_reference_timestamp) -> None: literals={TimestampLiteral(TIMESTAMP_EXAMPLE + HOUR_IN_MICROSECONDS), TimestampLiteral(TIMESTAMP_EXAMPLE)}, ), ) == In( - term="name", literals={TimestampLiteral(TIMESTAMP_EXAMPLE + HOUR_IN_MICROSECONDS), TimestampLiteral(TIMESTAMP_EXAMPLE)} # type: ignore + term="name", + literals={TimestampLiteral(TIMESTAMP_EXAMPLE + HOUR_IN_MICROSECONDS), TimestampLiteral(TIMESTAMP_EXAMPLE)}, # type: ignore ) -def test_projection_identity_set_not_in(bound_reference_timestamp) -> None: +def test_projection_identity_set_not_in(bound_reference_timestamp: BoundReference[int]) -> None: assert IdentityTransform().project( "name", BoundNotIn( @@ -791,7 +794,8 @@ def test_projection_identity_set_not_in(bound_reference_timestamp) -> None: literals={TimestampLiteral(TIMESTAMP_EXAMPLE + HOUR_IN_MICROSECONDS), TimestampLiteral(TIMESTAMP_EXAMPLE)}, ), ) == NotIn( - term="name", literals={TimestampLiteral(TIMESTAMP_EXAMPLE + HOUR_IN_MICROSECONDS), TimestampLiteral(TIMESTAMP_EXAMPLE)} # type: ignore + term="name", + literals={TimestampLiteral(TIMESTAMP_EXAMPLE + HOUR_IN_MICROSECONDS), TimestampLiteral(TIMESTAMP_EXAMPLE)}, # type: ignore ) diff --git a/tests/test_types.py b/tests/test_types.py index 3bbe1926d6..4b1d7698db 100644 --- a/tests/test_types.py +++ b/tests/test_types.py @@ -15,6 +15,7 @@ # specific language governing permissions and limitations # under the License. # pylint: disable=W0123,W0613 +from typing import Type import pytest @@ -32,6 +33,7 @@ LongType, MapType, NestedField, + PrimitiveType, StringType, StructType, TimestampType, @@ -58,7 +60,7 @@ @pytest.mark.parametrize("input_index, input_type", non_parameterized_types) -def test_repr_primitive_types(input_index, input_type): +def test_repr_primitive_types(input_index: int, input_type: Type[PrimitiveType]) -> None: assert isinstance(eval(repr(input_type())), input_type) @@ -94,11 +96,11 @@ def test_repr_primitive_types(input_index, input_type): (NestedField(1, "required_field", StringType(), required=False), False), ], ) -def test_is_primitive(input_type, result): +def test_is_primitive(input_type: IcebergType, result: bool) -> None: assert input_type.is_primitive == result -def test_fixed_type(): +def test_fixed_type() -> None: type_var = FixedType(length=5) assert len(type_var) == 5 assert str(type_var) == "fixed[5]" @@ -108,7 +110,7 @@ def test_fixed_type(): assert type_var != FixedType(6) -def test_decimal_type(): +def test_decimal_type() -> None: type_var = DecimalType(precision=9, scale=2) assert type_var.precision == 9 assert type_var.scale == 2 @@ -119,7 +121,7 @@ def test_decimal_type(): assert type_var != DecimalType(9, 3) -def test_struct_type(): +def test_struct_type() -> None: type_var = StructType( NestedField(1, "optional_field", IntegerType(), required=True), NestedField(2, "required_field", FixedType(5), required=False), @@ -139,7 +141,7 @@ def test_struct_type(): assert type_var != StructType(NestedField(1, "optional_field", IntegerType(), required=True)) -def test_list_type(): +def test_list_type() -> None: type_var = ListType( 1, StructType( @@ -162,7 +164,7 @@ def test_list_type(): ) -def test_map_type(): +def test_map_type() -> None: type_var = MapType(1, DoubleType(), 2, UUIDType(), False) assert isinstance(type_var.key_field.field_type, DoubleType) assert type_var.key_field.field_id == 1 @@ -174,7 +176,7 @@ def test_map_type(): assert type_var != MapType(1, DoubleType(), 2, StringType(), True) -def test_nested_field(): +def test_nested_field() -> None: field_var = NestedField( 1, "optional_field1", @@ -201,7 +203,9 @@ def test_nested_field(): @pytest.mark.parametrize("input_index,input_type", non_parameterized_types) @pytest.mark.parametrize("check_index,check_type", non_parameterized_types) -def test_non_parameterized_type_equality(input_index, input_type, check_index, check_type): +def test_non_parameterized_type_equality( + input_index: int, input_type: Type[PrimitiveType], check_index: int, check_type: Type[PrimitiveType] +) -> None: if input_index == check_index: assert input_type() == check_type() else: @@ -215,187 +219,187 @@ class IcebergTestType(IcebergBaseModel): __root__: IcebergType -def test_serialization_boolean(): +def test_serialization_boolean() -> None: assert BooleanType().json() == '"boolean"' -def test_deserialization_boolean(): +def test_deserialization_boolean() -> None: assert IcebergTestType.parse_raw('"boolean"') == BooleanType() -def test_str_boolean(): +def test_str_boolean() -> None: assert str(BooleanType()) == "boolean" -def test_repr_boolean(): +def test_repr_boolean() -> None: assert repr(BooleanType()) == "BooleanType()" -def test_serialization_int(): +def test_serialization_int() -> None: assert IntegerType().json() == '"int"' -def test_deserialization_int(): +def test_deserialization_int() -> None: assert IcebergTestType.parse_raw('"int"') == IntegerType() -def test_str_int(): +def test_str_int() -> None: assert str(IntegerType()) == "int" -def test_repr_int(): +def test_repr_int() -> None: assert repr(IntegerType()) == "IntegerType()" -def test_serialization_long(): +def test_serialization_long() -> None: assert LongType().json() == '"long"' -def test_deserialization_long(): +def test_deserialization_long() -> None: assert IcebergTestType.parse_raw('"long"') == LongType() -def test_str_long(): +def test_str_long() -> None: assert str(LongType()) == "long" -def test_repr_long(): +def test_repr_long() -> None: assert repr(LongType()) == "LongType()" -def test_serialization_float(): +def test_serialization_float() -> None: assert FloatType().json() == '"float"' -def test_deserialization_float(): +def test_deserialization_float() -> None: assert IcebergTestType.parse_raw('"float"') == FloatType() -def test_str_float(): +def test_str_float() -> None: assert str(FloatType()) == "float" -def test_repr_float(): +def test_repr_float() -> None: assert repr(FloatType()) == "FloatType()" -def test_serialization_double(): +def test_serialization_double() -> None: assert DoubleType().json() == '"double"' -def test_deserialization_double(): +def test_deserialization_double() -> None: assert IcebergTestType.parse_raw('"double"') == DoubleType() -def test_str_double(): +def test_str_double() -> None: assert str(DoubleType()) == "double" -def test_repr_double(): +def test_repr_double() -> None: assert repr(DoubleType()) == "DoubleType()" -def test_serialization_date(): +def test_serialization_date() -> None: assert DateType().json() == '"date"' -def test_deserialization_date(): +def test_deserialization_date() -> None: assert IcebergTestType.parse_raw('"date"') == DateType() -def test_str_date(): +def test_str_date() -> None: assert str(DateType()) == "date" -def test_repr_date(): +def test_repr_date() -> None: assert repr(DateType()) == "DateType()" -def test_serialization_time(): +def test_serialization_time() -> None: assert TimeType().json() == '"time"' -def test_deserialization_time(): +def test_deserialization_time() -> None: assert IcebergTestType.parse_raw('"time"') == TimeType() -def test_str_time(): +def test_str_time() -> None: assert str(TimeType()) == "time" -def test_repr_time(): +def test_repr_time() -> None: assert repr(TimeType()) == "TimeType()" -def test_serialization_timestamp(): +def test_serialization_timestamp() -> None: assert TimestampType().json() == '"timestamp"' -def test_deserialization_timestamp(): +def test_deserialization_timestamp() -> None: assert IcebergTestType.parse_raw('"timestamp"') == TimestampType() -def test_str_timestamp(): +def test_str_timestamp() -> None: assert str(TimestampType()) == "timestamp" -def test_repr_timestamp(): +def test_repr_timestamp() -> None: assert repr(TimestampType()) == "TimestampType()" -def test_serialization_timestamptz(): +def test_serialization_timestamptz() -> None: assert TimestamptzType().json() == '"timestamptz"' -def test_deserialization_timestamptz(): +def test_deserialization_timestamptz() -> None: assert IcebergTestType.parse_raw('"timestamptz"') == TimestamptzType() -def test_str_timestamptz(): +def test_str_timestamptz() -> None: assert str(TimestamptzType()) == "timestamptz" -def test_repr_timestamptz(): +def test_repr_timestamptz() -> None: assert repr(TimestamptzType()) == "TimestamptzType()" -def test_serialization_string(): +def test_serialization_string() -> None: assert StringType().json() == '"string"' -def test_deserialization_string(): +def test_deserialization_string() -> None: assert IcebergTestType.parse_raw('"string"') == StringType() -def test_str_string(): +def test_str_string() -> None: assert str(StringType()) == "string" -def test_repr_string(): +def test_repr_string() -> None: assert repr(StringType()) == "StringType()" -def test_serialization_uuid(): +def test_serialization_uuid() -> None: assert UUIDType().json() == '"uuid"' -def test_deserialization_uuid(): +def test_deserialization_uuid() -> None: assert IcebergTestType.parse_raw('"uuid"') == UUIDType() -def test_str_uuid(): +def test_str_uuid() -> None: assert str(UUIDType()) == "uuid" -def test_repr_uuid(): +def test_repr_uuid() -> None: assert repr(UUIDType()) == "UUIDType()" -def test_serialization_fixed(): +def test_serialization_fixed() -> None: assert FixedType(22).json() == '"fixed[22]"' -def test_deserialization_fixed(): +def test_deserialization_fixed() -> None: fixed = IcebergTestType.parse_raw('"fixed[22]"') assert fixed == FixedType(22) @@ -404,35 +408,35 @@ def test_deserialization_fixed(): assert len(inner) == 22 -def test_str_fixed(): +def test_str_fixed() -> None: assert str(FixedType(22)) == "fixed[22]" -def test_repr_fixed(): +def test_repr_fixed() -> None: assert repr(FixedType(22)) == "FixedType(length=22)" -def test_serialization_binary(): +def test_serialization_binary() -> None: assert BinaryType().json() == '"binary"' -def test_deserialization_binary(): +def test_deserialization_binary() -> None: assert IcebergTestType.parse_raw('"binary"') == BinaryType() -def test_str_binary(): +def test_str_binary() -> None: assert str(BinaryType()) == "binary" -def test_repr_binary(): +def test_repr_binary() -> None: assert repr(BinaryType()) == "BinaryType()" -def test_serialization_decimal(): +def test_serialization_decimal() -> None: assert DecimalType(19, 25).json() == '"decimal(19, 25)"' -def test_deserialization_decimal(): +def test_deserialization_decimal() -> None: decimal = IcebergTestType.parse_raw('"decimal(19, 25)"') assert decimal == DecimalType(19, 25) @@ -442,45 +446,45 @@ def test_deserialization_decimal(): assert inner.scale == 25 -def test_str_decimal(): +def test_str_decimal() -> None: assert str(DecimalType(19, 25)) == "decimal(19, 25)" -def test_repr_decimal(): +def test_repr_decimal() -> None: assert repr(DecimalType(19, 25)) == "DecimalType(precision=19, scale=25)" -def test_serialization_nestedfield(): +def test_serialization_nestedfield() -> None: expected = '{"id": 1, "name": "required_field", "type": "string", "required": true, "doc": "this is a doc"}' actual = NestedField(1, "required_field", StringType(), True, "this is a doc").json() assert expected == actual -def test_serialization_nestedfield_no_doc(): +def test_serialization_nestedfield_no_doc() -> None: expected = '{"id": 1, "name": "required_field", "type": "string", "required": true}' actual = NestedField(1, "required_field", StringType(), True).json() assert expected == actual -def test_str_nestedfield(): +def test_str_nestedfield() -> None: assert str(NestedField(1, "required_field", StringType(), True)) == "1: required_field: required string" -def test_repr_nestedfield(): +def test_repr_nestedfield() -> None: assert ( repr(NestedField(1, "required_field", StringType(), True)) == "NestedField(field_id=1, name='required_field', field_type=StringType(), required=True)" ) -def test_nestedfield_by_alias(): +def test_nestedfield_by_alias() -> None: # We should be able to initialize a NestedField by alias expected = NestedField(1, "required_field", StringType(), True, "this is a doc") - actual = NestedField(**{"id": 1, "name": "required_field", "type": "string", "required": True, "doc": "this is a doc"}) + actual = NestedField(**{"id": 1, "name": "required_field", "type": "string", "required": True, "doc": "this is a doc"}) # type: ignore assert expected == actual -def test_deserialization_nestedfield(): +def test_deserialization_nestedfield() -> None: expected = NestedField(1, "required_field", StringType(), True, "this is a doc") actual = NestedField.parse_raw( '{"id": 1, "name": "required_field", "type": "string", "required": true, "doc": "this is a doc"}' @@ -488,7 +492,7 @@ def test_deserialization_nestedfield(): assert expected == actual -def test_deserialization_nestedfield_inner(): +def test_deserialization_nestedfield_inner() -> None: expected = NestedField(1, "required_field", StringType(), True, "this is a doc") actual = IcebergTestType.parse_raw( '{"id": 1, "name": "required_field", "type": "string", "required": true, "doc": "this is a doc"}' @@ -496,7 +500,7 @@ def test_deserialization_nestedfield_inner(): assert expected == actual.__root__ -def test_serialization_struct(): +def test_serialization_struct() -> None: actual = StructType( NestedField(1, "required_field", StringType(), True, "this is a doc"), NestedField(2, "optional_field", IntegerType()) ).json() @@ -509,7 +513,7 @@ def test_serialization_struct(): assert actual == expected -def test_deserialization_struct(): +def test_deserialization_struct() -> None: actual = StructType.parse_raw( """ { @@ -539,62 +543,62 @@ def test_deserialization_struct(): assert actual == expected -def test_str_struct(simple_struct: StructType): +def test_str_struct(simple_struct: StructType) -> None: assert str(simple_struct) == "struct<1: required_field: required string (this is a doc), 2: optional_field: required int>" -def test_repr_struct(simple_struct: StructType): +def test_repr_struct(simple_struct: StructType) -> None: assert ( repr(simple_struct) == "StructType(fields=(NestedField(field_id=1, name='required_field', field_type=StringType(), required=True), NestedField(field_id=2, name='optional_field', field_type=IntegerType(), required=True),))" ) -def test_serialization_list(simple_list: ListType): +def test_serialization_list(simple_list: ListType) -> None: actual = simple_list.json() expected = '{"type": "list", "element-id": 22, "element": "string", "element-required": true}' assert actual == expected -def test_deserialization_list(simple_list: ListType): +def test_deserialization_list(simple_list: ListType) -> None: actual = ListType.parse_raw('{"type": "list", "element-id": 22, "element": "string", "element-required": true}') assert actual == simple_list -def test_str_list(simple_list: ListType): +def test_str_list(simple_list: ListType) -> None: assert str(simple_list) == "list" -def test_repr_list(simple_list: ListType): +def test_repr_list(simple_list: ListType) -> None: assert repr(simple_list) == "ListType(type='list', element_id=22, element_type=StringType(), element_required=True)" -def test_serialization_map(simple_map: MapType): +def test_serialization_map(simple_map: MapType) -> None: actual = simple_map.json() expected = """{"type": "map", "key-id": 19, "key": "string", "value-id": 25, "value": "double", "value-required": false}""" assert actual == expected -def test_deserialization_map(simple_map: MapType): +def test_deserialization_map(simple_map: MapType) -> None: actual = MapType.parse_raw( """{"type": "map", "key-id": 19, "key": "string", "value-id": 25, "value": "double", "value-required": false}""" ) assert actual == simple_map -def test_str_map(simple_map: MapType): +def test_str_map(simple_map: MapType) -> None: assert str(simple_map) == "map" -def test_repr_map(simple_map: MapType): +def test_repr_map(simple_map: MapType) -> None: assert ( repr(simple_map) == "MapType(type='map', key_id=19, key_type=StringType(), value_id=25, value_type=DoubleType(), value_required=False)" ) -def test_types_singleton(): +def test_types_singleton() -> None: """The types are immutable so we can return the same instance multiple times""" assert id(BooleanType()) == id(BooleanType()) assert id(FixedType(22)) == id(FixedType(22)) diff --git a/tests/test_version.py b/tests/test_version.py index a1f2191e16..32d96b06a5 100644 --- a/tests/test_version.py +++ b/tests/test_version.py @@ -18,7 +18,7 @@ from pyiceberg import __version__ -def test_version_format(): +def test_version_format() -> None: from importlib import metadata installed_version = metadata.version("pyiceberg") diff --git a/tests/utils/test_bin_packing.py b/tests/utils/test_bin_packing.py index 2c1cda5bdd..c021c94d25 100644 --- a/tests/utils/test_bin_packing.py +++ b/tests/utils/test_bin_packing.py @@ -36,7 +36,7 @@ ), # sparse ], ) -def test_bin_packing(splits, lookback, split_size, open_cost) -> None: +def test_bin_packing(splits: List[int], lookback: int, split_size: int, open_cost: int) -> None: def weight_func(x: int) -> int: return max(x, open_cost) @@ -79,7 +79,7 @@ def weight_func(x: int) -> int: ) def test_bin_packing_lookback( splits: List[int], target_weight: int, lookback: int, largest_bin_first: bool, expected_lists: List[List[int]] -): +) -> None: def weight_func(x: int) -> int: return x diff --git a/tests/utils/test_config.py b/tests/utils/test_config.py index e4a8970876..93831a716e 100644 --- a/tests/utils/test_config.py +++ b/tests/utils/test_config.py @@ -24,22 +24,22 @@ EXAMPLE_ENV = {"PYICEBERG_CATALOG__PRODUCTION__URI": "https://service.io/api"} -def test_config(): +def test_config() -> None: """To check if all the file lookups go well without any mocking""" assert Config() @mock.patch.dict(os.environ, EXAMPLE_ENV) -def test_from_environment_variables(): +def test_from_environment_variables() -> None: assert Config().get_catalog_config("production") == {"uri": "https://service.io/api"} @mock.patch.dict(os.environ, EXAMPLE_ENV) -def test_from_environment_variables_uppercase(): +def test_from_environment_variables_uppercase() -> None: assert Config().get_catalog_config("PRODUCTION") == {"uri": "https://service.io/api"} -def test_from_configuration_files(tmp_path_factory): +def test_from_configuration_files(tmp_path_factory) -> None: # type: ignore config_path = str(tmp_path_factory.mktemp("config")) with open(f"{config_path}/.pyiceberg.yaml", "w", encoding="utf-8") as file: yaml.dump({"catalog": {"production": {"uri": "https://service.io/api"}}}, file) @@ -48,7 +48,7 @@ def test_from_configuration_files(tmp_path_factory): assert Config().get_catalog_config("production") == {"uri": "https://service.io/api"} -def test_lowercase_dictionary_keys(): +def test_lowercase_dictionary_keys() -> None: uppercase_keys = {"UPPER": {"NESTED_UPPER": {"YES"}}} expected = {"upper": {"nested_upper": {"YES"}}} - assert _lowercase_dictionary_keys(uppercase_keys) == expected + assert _lowercase_dictionary_keys(uppercase_keys) == expected # type: ignore diff --git a/tests/utils/test_manifest.py b/tests/utils/test_manifest.py index 2f351539eb..25ba4730dc 100644 --- a/tests/utils/test_manifest.py +++ b/tests/utils/test_manifest.py @@ -33,7 +33,7 @@ from tests.io.test_io import LocalInputFile -def test_read_manifest_entry(generated_manifest_entry_file: str): +def test_read_manifest_entry(generated_manifest_entry_file: str) -> None: input_file = LocalInputFile(generated_manifest_entry_file) assert list(read_manifest_entry(input_file)) == [ ManifestEntry( @@ -267,7 +267,7 @@ def test_read_manifest_entry(generated_manifest_entry_file: str): ] -def test_read_manifest_list(generated_manifest_file_file: str): +def test_read_manifest_list(generated_manifest_file_file: str) -> None: input_file = LocalInputFile(generated_manifest_file_file) actual = list(read_manifest_list(input_file)) expected = [ @@ -292,7 +292,7 @@ def test_read_manifest_list(generated_manifest_file_file: str): assert actual == expected -def test_read_manifest(generated_manifest_file_file: str, generated_manifest_entry_file: str): +def test_read_manifest(generated_manifest_file_file: str, generated_manifest_entry_file: str) -> None: io = load_file_io({}) snapshot = Snapshot( diff --git a/tests/utils/test_schema_conversion.py b/tests/utils/test_schema_conversion.py index 3abf19c6da..9c5df9ef07 100644 --- a/tests/utils/test_schema_conversion.py +++ b/tests/utils/test_schema_conversion.py @@ -37,7 +37,7 @@ from pyiceberg.utils.schema_conversion import AvroSchemaConversion -def test_iceberg_to_avro(avro_schema_manifest_file: Dict[str, Any]): +def test_iceberg_to_avro(avro_schema_manifest_file: Dict[str, Any]) -> None: iceberg_schema = AvroSchemaConversion().avro_to_iceberg(avro_schema_manifest_file) expected_iceberg_schema = Schema( NestedField( @@ -110,7 +110,7 @@ def test_iceberg_to_avro(avro_schema_manifest_file: Dict[str, Any]): assert iceberg_schema == expected_iceberg_schema -def test_avro_list_required_primitive(): +def test_avro_list_required_primitive() -> None: avro_schema = { "type": "record", "name": "avro_schema", @@ -143,7 +143,7 @@ def test_avro_list_required_primitive(): assert expected_iceberg_schema == iceberg_schema -def test_avro_list_wrapped_primitive(): +def test_avro_list_wrapped_primitive() -> None: avro_schema = { "type": "record", "name": "avro_schema", @@ -176,7 +176,7 @@ def test_avro_list_wrapped_primitive(): assert expected_iceberg_schema == iceberg_schema -def test_avro_list_required_record(): +def test_avro_list_required_record() -> None: avro_schema = { "type": "record", "name": "avro_schema", @@ -215,10 +215,8 @@ def test_avro_list_required_record(): field_type=ListType( element_id=101, element_type=StructType( - fields=( - NestedField(field_id=102, name="contains_null", field_type=BooleanType(), required=True), - NestedField(field_id=103, name="contains_nan", field_type=BooleanType(), required=False), - ) + NestedField(field_id=102, name="contains_null", field_type=BooleanType(), required=True), + NestedField(field_id=103, name="contains_nan", field_type=BooleanType(), required=False), ), element_required=True, ), @@ -233,19 +231,19 @@ def test_avro_list_required_record(): assert expected_iceberg_schema == iceberg_schema -def test_resolve_union(): +def test_resolve_union() -> None: with pytest.raises(TypeError) as exc_info: AvroSchemaConversion()._resolve_union(["null", "string", "long"]) assert "Non-optional types aren't part of the Iceberg specification" in str(exc_info.value) -def test_nested_type(): +def test_nested_type() -> None: # In the case a primitive field is nested assert AvroSchemaConversion()._convert_schema({"type": {"type": "string"}}) == StringType() -def test_map_type(): +def test_map_type() -> None: avro_type = { "type": "map", "values": ["null", "long"], @@ -257,21 +255,21 @@ def test_map_type(): assert actual == expected -def test_fixed_type(): +def test_fixed_type() -> None: avro_type = {"type": "fixed", "size": 22} actual = AvroSchemaConversion()._convert_schema(avro_type) expected = FixedType(22) assert actual == expected -def test_unknown_primitive(): +def test_unknown_primitive() -> None: with pytest.raises(TypeError) as exc_info: avro_type = "UnknownType" AvroSchemaConversion()._convert_schema(avro_type) assert "Unknown type: UnknownType" in str(exc_info.value) -def test_unknown_complex_type(): +def test_unknown_complex_type() -> None: with pytest.raises(TypeError) as exc_info: avro_type = { "type": "UnknownType", @@ -280,7 +278,7 @@ def test_unknown_complex_type(): assert "Unknown type: {'type': 'UnknownType'}" in str(exc_info.value) -def test_convert_field_without_field_id(): +def test_convert_field_without_field_id() -> None: with pytest.raises(ValueError) as exc_info: avro_field = { "name": "contains_null", @@ -290,14 +288,14 @@ def test_convert_field_without_field_id(): assert "Cannot convert field, missing field-id" in str(exc_info.value) -def test_convert_record_type_without_record(): +def test_convert_record_type_without_record() -> None: with pytest.raises(ValueError) as exc_info: avro_field = {"type": "non-record", "name": "avro_schema", "fields": []} AvroSchemaConversion()._convert_record_type(avro_field) assert "Expected record type, got" in str(exc_info.value) -def test_avro_list_missing_element_id(): +def test_avro_list_missing_element_id() -> None: avro_type = { "name": "array_with_string", "type": { @@ -315,20 +313,20 @@ def test_avro_list_missing_element_id(): assert "Cannot convert array-type, missing element-id:" in str(exc_info.value) -def test_convert_decimal_type(): +def test_convert_decimal_type() -> None: avro_decimal_type = {"type": "bytes", "logicalType": "decimal", "precision": 19, "scale": 25} actual = AvroSchemaConversion()._convert_logical_type(avro_decimal_type) expected = DecimalType(precision=19, scale=25) assert actual == expected -def test_convert_date_type(): +def test_convert_date_type() -> None: avro_logical_type = {"type": "int", "logicalType": "date"} actual = AvroSchemaConversion()._convert_logical_type(avro_logical_type) assert actual == DateType() -def test_unknown_logical_type(): +def test_unknown_logical_type() -> None: """Test raising a ValueError when converting an unknown logical type as part of an Avro schema conversion""" avro_logical_type = {"type": "bytes", "logicalType": "date"} with pytest.raises(ValueError) as exc_info: @@ -337,7 +335,7 @@ def test_unknown_logical_type(): assert "Unknown logical/physical type combination:" in str(exc_info.value) -def test_logical_map_with_invalid_fields(): +def test_logical_map_with_invalid_fields() -> None: avro_type = { "type": "array", "logicalType": "map", diff --git a/tests/utils/test_singleton.py b/tests/utils/test_singleton.py index b9cf33cc6a..742012886c 100644 --- a/tests/utils/test_singleton.py +++ b/tests/utils/test_singleton.py @@ -18,13 +18,13 @@ from pyiceberg.transforms import VoidTransform -def test_singleton(): +def test_singleton() -> None: """We want to reuse the readers to avoid creating a gazillion of them""" assert id(BooleanReader()) == id(BooleanReader()) assert id(FixedReader(22)) == id(FixedReader(22)) assert id(FixedReader(19)) != id(FixedReader(25)) -def test_singleton_transform(): +def test_singleton_transform() -> None: """We want to reuse VoidTransform since it doesn't carry any state""" assert id(VoidTransform()) == id(VoidTransform()) From 397bf69966e8c51bb403f6fa6ce8f2118981b415 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Fri, 9 Dec 2022 12:12:51 +0100 Subject: [PATCH 294/642] Build: Bump certifi from 2022.9.24 to 2022.12.7 in /python (#6396) Bumps [certifi](https://github.com/certifi/python-certifi) from 2022.9.24 to 2022.12.7. - [Release notes](https://github.com/certifi/python-certifi/releases) - [Commits](https://github.com/certifi/python-certifi/compare/2022.09.24...2022.12.07) --- updated-dependencies: - dependency-name: certifi dependency-type: indirect ... Signed-off-by: dependabot[bot] Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- poetry.lock | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/poetry.lock b/poetry.lock index 43fac9650d..0a54622ac8 100644 --- a/poetry.lock +++ b/poetry.lock @@ -134,7 +134,7 @@ virtualenv = ["virtualenv (>=20.0.35)"] [[package]] name = "certifi" -version = "2022.9.24" +version = "2022.12.7" description = "Python package for providing Mozilla's CA Bundle." category = "main" optional = false @@ -1080,8 +1080,8 @@ build = [ {file = "build-0.9.0.tar.gz", hash = "sha256:1a07724e891cbd898923145eb7752ee7653674c511378eb9c7691aab1612bc3c"}, ] certifi = [ - {file = "certifi-2022.9.24-py3-none-any.whl", hash = "sha256:90c1a32f1d68f940488354e36370f6cca89f0f106db09518524c88d6ed83f382"}, - {file = "certifi-2022.9.24.tar.gz", hash = "sha256:0d9c601124e5a6ba9712dbc60d9c53c21e34f5f641fe83002317394311bdce14"}, + {file = "certifi-2022.12.7-py3-none-any.whl", hash = "sha256:4ad3232f5e926d6718ec31cfc1fcadfde020920e278684144551c91769c7bc18"}, + {file = "certifi-2022.12.7.tar.gz", hash = "sha256:35824b4c3a97115964b408844d64aa14db1cc518f6562e8d7261699d1350a9e3"}, ] cffi = [ {file = "cffi-1.15.1-cp27-cp27m-macosx_10_9_x86_64.whl", hash = "sha256:a66d3508133af6e8548451b25058d5812812ec3798c886bf38ed24a98216fab2"}, From c67f2bbf324ac63e8b1d505fa5a0b82f82160d7f Mon Sep 17 00:00:00 2001 From: Ruben van de Geer <37443208+rubenvdg@users.noreply.github.com> Date: Fri, 9 Dec 2022 12:34:02 +0100 Subject: [PATCH 295/642] Python: Set lower bound pip version (#6384) --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 59cb29f47a..d988be2120 100644 --- a/README.md +++ b/README.md @@ -26,7 +26,7 @@ pyiceberg is currently in development, for development and testing purposes the ``` git clone https://github.com/apache/iceberg.git cd iceberg/python -pip install -e . +pip install -e . # this step requires pip >= 21.3.0 ``` ## Development From d944681aa7bb75a5c76ac33b1a91b6430997fcb2 Mon Sep 17 00:00:00 2001 From: Fokko Driesprong Date: Fri, 9 Dec 2022 13:34:50 +0100 Subject: [PATCH 296/642] Python: Improve docs (#6389) * Python: Improve docs This improves the docs by turning into a multi page site. Also merges in the `CONTRIBUTING.md` and `RELEASE.md` to have everything into a single place. Also changed to the readthedocs theme since that has a much cleaner multi page layout (navigation on the left instead of the top). * Rework the docs --- .pre-commit-config.yaml | 18 +- mkdocs/docs/api.md | 339 +++++++++++ mkdocs/docs/cli.md | 215 +++++++ mkdocs/docs/configuration.md | 84 +++ mkdocs/docs/contributing.md | 135 +++++ mkdocs/docs/feature-support.md | 85 +++ .../docs/how-to-release.md | 22 +- mkdocs/docs/index.md | 550 +----------------- mkdocs/docs/verify-release.md | 100 ++++ mkdocs/mkdocs.yml | 16 + 10 files changed, 1019 insertions(+), 545 deletions(-) create mode 100644 mkdocs/docs/api.md create mode 100644 mkdocs/docs/cli.md create mode 100644 mkdocs/docs/configuration.md create mode 100644 mkdocs/docs/contributing.md create mode 100644 mkdocs/docs/feature-support.md rename dev/RELEASE.md => mkdocs/docs/how-to-release.md (79%) create mode 100644 mkdocs/docs/verify-release.md diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index e6f71b94de..1c367e5644 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -63,11 +63,13 @@ repos: - id: flake8 args: [ "--ignore=E501,W503,E203,B024" ] additional_dependencies: [ flake8-bugbear==22.9.11, flake8-comprehensions==3.10.0 ] - - repo: https://github.com/executablebooks/mdformat - rev: 0.7.16 - hooks: - - id: mdformat - additional_dependencies: - - mdformat-black - - mdformat-config - - mdformat-beautysh +# Disabling this for now, we need mdformat-admin, otherwise it will break the note blocks +# - repo: https://github.com/executablebooks/mdformat +# rev: 0.7.16 +# hooks: +# - id: mdformat +# additional_dependencies: +# - mdformat-black +# - mdformat-config +# - mdformat-beautysh +# - mdformat-admon diff --git a/mkdocs/docs/api.md b/mkdocs/docs/api.md new file mode 100644 index 0000000000..dd6423f61a --- /dev/null +++ b/mkdocs/docs/api.md @@ -0,0 +1,339 @@ + + +# Python API + +PyIceberg is based around catalogs to load tables. First step is to instantiate a catalog that loads tables. Let's use the following configuration: + +```yaml +catalog: + prod: + uri: http://rest-catalog/ws/ + credential: t-1234:secret +``` + +Then load the `prod` catalog: + +```python +from pyiceberg.catalog import load_catalog + +catalog = load_catalog("prod") + +catalog.list_namespaces() +``` + +Returns two namespaces: + +```python +[("default",), ("nyc",)] +``` + +Listing the tables in the `nyc` namespace: + +```python +catalog.list_tables("nyc") +``` + +Returns as list with tuples, containing a single table `taxis`: + +```python +[("nyc", "taxis")] +``` + +## Load a table + +Loading the `taxis` table: + +```python +catalog.load_table("nyc.taxis") +# Equivalent to: +catalog.load_table(("nyc", "taxis")) +# The tuple syntax can be used if the namespace or table contains a dot. +``` + +This returns a `Table` that represents an Iceberg table: + +```python +Table( + identifier=('nyc', 'taxis'), + metadata_location='s3a://warehouse/wh/nyc.db/taxis/metadata/00002-6ea51ce3-62aa-4197-9cf8-43d07c3440ca.metadata.json', + metadata=TableMetadataV2( + location='s3a://warehouse/wh/nyc.db/taxis', + table_uuid=UUID('ebd5d172-2162-453d-b586-1cdce52c1116'), + last_updated_ms=1662633437826, + last_column_id=19, + schemas=[Schema( + NestedField(field_id=1, name='VendorID', field_type=LongType(), required=False), + NestedField(field_id=2, name='tpep_pickup_datetime', field_type=TimestamptzType(), required=False), + NestedField(field_id=3, name='tpep_dropoff_datetime', field_type=TimestamptzType(), required=False), + NestedField(field_id=4, name='passenger_count', field_type=DoubleType(), required=False), + NestedField(field_id=5, name='trip_distance', field_type=DoubleType(), required=False), + NestedField(field_id=6, name='RatecodeID', field_type=DoubleType(), required=False), + NestedField(field_id=7, name='store_and_fwd_flag', field_type=StringType(), required=False), + NestedField(field_id=8, name='PULocationID', field_type=LongType(), required=False), + NestedField(field_id=9, name='DOLocationID', field_type=LongType(), required=False), + NestedField(field_id=10, name='payment_type', field_type=LongType(), required=False), + NestedField(field_id=11, name='fare_amount', field_type=DoubleType(), required=False), + NestedField(field_id=12, name='extra', field_type=DoubleType(), required=False), + NestedField(field_id=13, name='mta_tax', field_type=DoubleType(), required=False), + NestedField(field_id=14, name='tip_amount', field_type=DoubleType(), required=False), + NestedField(field_id=15, name='tolls_amount', field_type=DoubleType(), required=False), + NestedField(field_id=16, name='improvement_surcharge', field_type=DoubleType(), required=False), + NestedField(field_id=17, name='total_amount', field_type=DoubleType(), required=False), + NestedField(field_id=18, name='congestion_surcharge', field_type=DoubleType(), required=False), + NestedField(field_id=19, name='airport_fee', field_type=DoubleType(), required=False) + ), + schema_id=0, + identifier_field_ids=[] + )], + current_schema_id=0, + partition_specs=[PartitionSpec(spec_id=0)], + default_spec_id=0, + last_partition_id=999, + properties={ + 'owner': 'root', + 'write.format.default': 'parquet' + }, + current_snapshot_id=8334458494559715805, + snapshots=[ + Snapshot( + snapshot_id=7910949481055846233, + parent_snapshot_id=None, + sequence_number=None, + timestamp_ms=1662489306555, + manifest_list='s3a://warehouse/wh/nyc.db/taxis/metadata/snap-7910949481055846233-1-3eb7a2e1-5b7a-4e76-a29a-3e29c176eea4.avro', + summary=Summary( + Operation.APPEND, + **{ + 'spark.app.id': 'local-1662489289173', + 'added-data-files': '1', + 'added-records': '2979431', + 'added-files-size': '46600777', + 'changed-partition-count': '1', + 'total-records': '2979431', + 'total-files-size': '46600777', + 'total-data-files': '1', + 'total-delete-files': '0', + 'total-position-deletes': '0', + 'total-equality-deletes': '0' + } + ), + schema_id=0 + ), + ], + snapshot_log=[ + SnapshotLogEntry( + snapshot_id='7910949481055846233', + timestamp_ms=1662489306555 + ) + ], + metadata_log=[ + MetadataLogEntry( + metadata_file='s3a://warehouse/wh/nyc.db/taxis/metadata/00000-b58341ba-6a63-4eea-9b2f-e85e47c7d09f.metadata.json', + timestamp_ms=1662489306555 + ) + ], + sort_orders=[SortOrder(order_id=0)], + default_sort_order_id=0, + refs={ + 'main': SnapshotRef( + snapshot_id=8334458494559715805, + snapshot_ref_type=SnapshotRefType.BRANCH, + min_snapshots_to_keep=None, + max_snapshot_age_ms=None, + max_ref_age_ms=None + ) + }, + format_version=2, + last_sequence_number=1 + ) +) +``` + +## Create a table + +To create a table from a catalog: + +```python +from pyiceberg.catalog import load_catalog +from pyiceberg.schema import Schema +from pyiceberg.types import TimestampType, DoubleType, StringType, NestedField + +schema = Schema( + NestedField( + field_id=1, name="datetime", field_type=TimestampType(), required=False + ), + NestedField(field_id=2, name="bid", field_type=DoubleType(), required=False), + NestedField(field_id=3, name="ask", field_type=DoubleType(), required=False), + NestedField(field_id=4, name="symbol", field_type=StringType(), required=False), +) + +from pyiceberg.partitioning import PartitionSpec, PartitionField +from pyiceberg.transforms import DayTransform + +partition_spec = PartitionSpec( + PartitionField( + source_id=1, field_id=1000, transform=DayTransform(), name="datetime_day" + ) +) + +from pyiceberg.table.sorting import SortOrder, SortField +from pyiceberg.transforms import IdentityTransform + +sort_order = SortOrder(SortField(source_id=4, transform=IdentityTransform())) + +catalog = load_catalog("prod") + +catalog.create_table( + identifier="default.bids", + location="/Users/fokkodriesprong/Desktop/docker-spark-iceberg/wh/bids/", + schema=schema, + partition_spec=partition_spec, + sort_order=sort_order, +) +``` + +Which returns a newly created table: + +```python +Table( + identifier=('default', 'bids'), + metadata_location='/Users/fokkodriesprong/Desktop/docker-spark-iceberg/wh/bids//metadata/00000-c8cd93ab-f784-474d-a167-b1a86b05195f.metadata.json', + metadata=TableMetadataV2( + location='/Users/fokkodriesprong/Desktop/docker-spark-iceberg/wh/bids/', + table_uuid=UUID('38d4cb39-4945-4bf2-b374-984b5c4984d2'), + last_updated_ms=1661847562069, + last_column_id=4, + schemas=[ + Schema( + NestedField(field_id=1, name='datetime', field_type=TimestampType(), required=False), + NestedField(field_id=2, name='bid', field_type=DoubleType(), required=False), + NestedField(field_id=3, name='ask', field_type=DoubleType(), required=False), + NestedField(field_id=4, name='symbol', field_type=StringType(), required=False)), + schema_id=1, + identifier_field_ids=[]) + ], + current_schema_id=1, + partition_specs=[ + PartitionSpec( + PartitionField(source_id=1, field_id=1000, transform=DayTransform(), name='datetime_day'),)) + ], + default_spec_id=0, + last_partition_id=1000, + properties={}, + current_snapshot_id=None, + snapshots=[], + snapshot_log=[], + metadata_log=[], + sort_orders=[ + SortOrder(order_id=1, fields=[SortField(source_id=4, transform=IdentityTransform(), direction=SortDirection.ASC, null_order=NullOrder.NULLS_FIRST)]) + ], + default_sort_order_id=1, + refs={}, + format_version=2, + last_sequence_number=0 + ) +) +``` + +## Query a table + +To query a table, a table scan is needed. A table scan accepts a filter, columns and optionally a snapshot ID: + +```python +from pyiceberg.catalog import load_catalog +from pyiceberg.expressions import GreaterThanOrEqual + +catalog = load_catalog("default") +table = catalog.load_table("nyc.taxis") + +scan = table.scan( + row_filter=GreaterThanOrEqual("trip_distance", 10.0), + selected_fields=("VendorID", "tpep_pickup_datetime", "tpep_dropoff_datetime"), +) + +[task.file.file_path for task in scan.plan_files()] +``` + +The low level API `plan_files` methods returns a set of tasks that provide the files that might contain matching rows: + +```json +['s3a://warehouse/wh/nyc/taxis/data/00003-4-42464649-92dd-41ad-b83b-dea1a2fe4b58-00001.parquet'] +``` + +In this case it is up to the engine itself to filter the file itself. Below, `to_arrow()` and `to_duckdb()` that already do this for you. + +### Apache Arrow + +!!! note "Requirements" + This requires [PyArrow to be installed](index.md) + +Using PyIceberg it is filter out data from a huge table and pull it into a PyArrow table: + +```python +table.scan( + row_filter=GreaterThanOrEqual("trip_distance", 10.0), + selected_fields=("VendorID", "tpep_pickup_datetime", "tpep_dropoff_datetime"), +).to_arrow() +``` + +This will return a PyArrow table: + +``` +pyarrow.Table +VendorID: int64 +tpep_pickup_datetime: timestamp[us, tz=+00:00] +tpep_dropoff_datetime: timestamp[us, tz=+00:00] +---- +VendorID: [[2,1,2,1,1,...,2,2,2,2,2],[2,1,1,1,2,...,1,1,2,1,2],...,[2,2,2,2,2,...,2,6,6,2,2],[2,2,2,2,2,...,2,2,2,2,2]] +tpep_pickup_datetime: [[2021-04-01 00:28:05.000000,...,2021-04-30 23:44:25.000000]] +tpep_dropoff_datetime: [[2021-04-01 00:47:59.000000,...,2021-05-01 00:14:47.000000]] +``` + +This will only pull in the files that that might contain matching rows. + +### DuckDB + +!!! note "Requirements" + This requires [DuckDB to be installed](index.md). + +A table scan can also be converted into a in-memory DuckDB table: + +```python +con = table.scan( + row_filter=GreaterThanOrEqual("trip_distance", 10.0), + selected_fields=("VendorID", "tpep_pickup_datetime", "tpep_dropoff_datetime"), +).to_duckdb(table_name="distant_taxi_trips") +``` + +Using the cursor that we can run queries on the DuckDB table: + +```python +print( + con.execute( + "SELECT tpep_dropoff_datetime - tpep_pickup_datetime AS duration FROM distant_taxi_trips LIMIT 4" + ).fetchall() +) +[ + (datetime.timedelta(seconds=1194),), + (datetime.timedelta(seconds=1118),), + (datetime.timedelta(seconds=1697),), + (datetime.timedelta(seconds=1581),), +] +``` diff --git a/mkdocs/docs/cli.md b/mkdocs/docs/cli.md new file mode 100644 index 0000000000..5b760e1fa6 --- /dev/null +++ b/mkdocs/docs/cli.md @@ -0,0 +1,215 @@ + + +# Python CLI + +Pyiceberg comes with a CLI that's available after installing the `pyiceberg` package. + +You can pass the path to the Catalog using the `--uri` and `--credential` argument, but it is recommended to setup a `~/.pyiceberg.yaml` config as described in the [Catalog](configuration.md) section. + +```sh +➜ pyiceberg --help +Usage: pyiceberg [OPTIONS] COMMAND [ARGS]... + +Options: +--catalog TEXT +--verbose BOOLEAN +--output [text|json] +--uri TEXT +--credential TEXT +--help Show this message and exit. + +Commands: +describe Describes a namespace xor table +drop Operations to drop a namespace or table +list Lists tables or namespaces +location Returns the location of the table +properties Properties on tables/namespaces +rename Renames a table +schema Gets the schema of the table +spec Returns the partition spec of the table +uuid Returns the UUID of the table +``` + +This example assumes that you have a default catalog set. If you want to load another catalog, for example, the rest example above. Then you need to set `--catalog rest`. + +```sh +➜ pyiceberg list +default +nyc +``` + +```sh +➜ pyiceberg list nyc +nyc.taxis +``` + +```sh +➜ pyiceberg describe nyc.taxis +Table format version 1 +Metadata location file:/.../nyc.db/taxis/metadata/00000-aa3a3eac-ea08-4255-b890-383a64a94e42.metadata.json +Table UUID 6cdfda33-bfa3-48a7-a09e-7abb462e3460 +Last Updated 1661783158061 +Partition spec [] +Sort order [] +Current schema Schema, id=0 +├── 1: VendorID: optional long +├── 2: tpep_pickup_datetime: optional timestamptz +├── 3: tpep_dropoff_datetime: optional timestamptz +├── 4: passenger_count: optional double +├── 5: trip_distance: optional double +├── 6: RatecodeID: optional double +├── 7: store_and_fwd_flag: optional string +├── 8: PULocationID: optional long +├── 9: DOLocationID: optional long +├── 10: payment_type: optional long +├── 11: fare_amount: optional double +├── 12: extra: optional double +├── 13: mta_tax: optional double +├── 14: tip_amount: optional double +├── 15: tolls_amount: optional double +├── 16: improvement_surcharge: optional double +├── 17: total_amount: optional double +├── 18: congestion_surcharge: optional double +└── 19: airport_fee: optional double +Current snapshot Operation.APPEND: id=5937117119577207079, schema_id=0 +Snapshots Snapshots +└── Snapshot 5937117119577207079, schema 0: file:/.../nyc.db/taxis/metadata/snap-5937117119577207079-1-94656c4f-4c66-4600-a4ca-f30377300527.avro +Properties owner root +write.format.default parquet +``` + +Or output in JSON for automation: + +```sh +➜ pyiceberg --output json describe nyc.taxis | jq +{ + "identifier": [ + "nyc", + "taxis" + ], + "metadata_location": "file:/.../nyc.db/taxis/metadata/00000-aa3a3eac-ea08-4255-b890-383a64a94e42.metadata.json", + "metadata": { + "location": "file:/.../nyc.db/taxis", + "table-uuid": "6cdfda33-bfa3-48a7-a09e-7abb462e3460", + "last-updated-ms": 1661783158061, + "last-column-id": 19, + "schemas": [ + { + "type": "struct", + "fields": [ + { + "id": 1, + "name": "VendorID", + "type": "long", + "required": false + }, +... + { + "id": 19, + "name": "airport_fee", + "type": "double", + "required": false + } + ], + "schema-id": 0, + "identifier-field-ids": [] + } + ], + "current-schema-id": 0, + "partition-specs": [ + { + "spec-id": 0, + "fields": [] + } + ], + "default-spec-id": 0, + "last-partition-id": 999, + "properties": { + "owner": "root", + "write.format.default": "parquet" + }, + "current-snapshot-id": 5937117119577207000, + "snapshots": [ + { + "snapshot-id": 5937117119577207000, + "timestamp-ms": 1661783158061, + "manifest-list": "file:/.../nyc.db/taxis/metadata/snap-5937117119577207079-1-94656c4f-4c66-4600-a4ca-f30377300527.avro", + "summary": { + "operation": "append", + "spark.app.id": "local-1661783139151", + "added-data-files": "1", + "added-records": "2979431", + "added-files-size": "46600777", + "changed-partition-count": "1", + "total-records": "2979431", + "total-files-size": "46600777", + "total-data-files": "1", + "total-delete-files": "0", + "total-position-deletes": "0", + "total-equality-deletes": "0" + }, + "schema-id": 0 + } + ], + "snapshot-log": [ + { + "snapshot-id": "5937117119577207079", + "timestamp-ms": 1661783158061 + } + ], + "metadata-log": [], + "sort-orders": [ + { + "order-id": 0, + "fields": [] + } + ], + "default-sort-order-id": 0, + "refs": { + "main": { + "snapshot-id": 5937117119577207000, + "type": "branch" + } + }, + "format-version": 1, + "schema": { + "type": "struct", + "fields": [ + { + "id": 1, + "name": "VendorID", + "type": "long", + "required": false + }, +... + { + "id": 19, + "name": "airport_fee", + "type": "double", + "required": false + } + ], + "schema-id": 0, + "identifier-field-ids": [] + }, + "partition-spec": [] + } +} +``` diff --git a/mkdocs/docs/configuration.md b/mkdocs/docs/configuration.md new file mode 100644 index 0000000000..985d258173 --- /dev/null +++ b/mkdocs/docs/configuration.md @@ -0,0 +1,84 @@ + + +# Catalogs + +PyIceberg currently has native support for REST, Hive and Glue. + +There are three ways to pass in configuration: + +- Using the `~/.pyiceberg.yaml` configuration file +- Through environment variables +- By passing in credentials through the CLI or the Python API + +The configuration file is recommended since that's the most transparent way. If you prefer environment configuration: + +```sh +export PYICEBERG_CATALOG__DEFAULT__URI=thrift://localhost:9083 +``` + +The environment variable picked up by Iceberg starts with `PYICEBERG_` and then follows the yaml structure below, where a double underscore `__` represents a nested field. + +For the FileIO there are several configuration options available: + +| Key | Example | Description | +|----------------------|---------------------|-----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| +| s3.endpoint | https://10.0.19.25/ | Configure an alternative endpoint of the S3 service for the FileIO to access. This could be used to use S3FileIO with any s3-compatible object storage service that has a different endpoint, or access a private S3 endpoint in a virtual private cloud. | +| s3.access-key-id | admin | Configure the static secret access key used to access the FileIO. | +| s3.secret-access-key | password | Configure the static session token used to access the FileIO. | +| s3.signer | bearer | Configure the signature version of the FileIO. | + +## REST Catalog + +```yaml +catalog: + default: + uri: http://rest-catalog/ws/ + credential: t-1234:secret + + default-mtls-secured-catalog: + uri: https://rest-catalog/ws/ + ssl: + client: + cert: /absolute/path/to/client.crt + key: /absolute/path/to/client.key + cabundle: /absolute/path/to/cabundle.pem +``` + +## Hive Catalog + +```yaml +catalog: + default: + uri: thrift://localhost:9083 + s3.endpoint: http://localhost:9000 + s3.access-key-id: admin + s3.secret-access-key: password +``` + +## Glue Catalog + +If you want to use AWS Glue as the catalog, you can use the last two ways to configure the pyiceberg and refer +[How to configure AWS credentials](https://docs.aws.amazon.com/cli/latest/userguide/cli-chap-configure.html) to set your AWS account credentials locally. + +```yaml +catalog: + default: + type: glue +``` diff --git a/mkdocs/docs/contributing.md b/mkdocs/docs/contributing.md new file mode 100644 index 0000000000..a87939935f --- /dev/null +++ b/mkdocs/docs/contributing.md @@ -0,0 +1,135 @@ + + +# Contributing to the Iceberg Python library + +For the development, poetry is used for packing and dependency management. You can install this using: + +```bash +pip install poetry +``` + +If you have an older version of pip and virtualenv you need to update these: + +```bash +pip install --upgrade virtualenv pip +``` + +To get started, you can run `make install`, which will install poetry and it will install all the dependencies of the Iceberg library. This will also install the development dependencies. If you don't want to do this, you need to install using `poetry install --no-dev`. + +If you want to install the library on the host, you can simply run `pip3 install -e .`. If you wish to use a virtual environment, you can run `poetry shell`. Poetry will open up a virtual environment with all the dependencies set. + +To set up IDEA with Poetry ([also on Loom](https://www.loom.com/share/6d36464d45f244729d91003e7f671fd2)): + +- Open up the Python project in IntelliJ +- Make sure that you're on latest master (that includes Poetry) +- Go to File -> Project Structure (⌘;) +- Go to Platform Settings -> SDKs +- Click the + sign -> Add Python SDK +- Select Poetry Environment from the left hand side bar and hit OK +- It can take some time to download all the dependencies based on your internet +- Go to Project Settings -> Project +- Select the Poetry SDK from the SDK dropdown, and click OK + +For IDEA ≤2021 you need to install the [Poetry integration as a plugin](https://plugins.jetbrains.com/plugin/14307-poetry/). + +Now you're set using Poetry, and all the tests will run in Poetry, and you'll have syntax highlighting in the pyproject.toml to indicate stale dependencies. + +## Linting + +`pre-commit` is used for autoformatting and linting: + +```bash +make lint +``` + +Pre-commit will automatically fix the violations such as import orders, formatting etc. Pylint errors you need to fix yourself. + +In contrast to the name suggest, it doesn't run the checks on the commit. If this is something that you like, you can set this up by running `pre-commit install`. + +You can bump the integrations to the latest version using `pre-commit autoupdate`. This will check if there is a newer version of `{black,mypy,isort,...}` and update the yaml. + +## Testing + +For Python, `pytest` is used a testing framework in combination with `coverage` to enforce 90%+ code coverage. + +```bash +make test +``` + +By default, s3 tests are ignored because that require minio to be running. To run the s3 suite: + +```bash +make test-s3 +``` + +To pass additional arguments to pytest, you can use `PYTEST_ARGS`. + +*Run pytest in verbose mode* + +```sh +make test PYTEST_ARGS="-v" +``` + +*Run pytest with pdb enabled* + +```sh +make test PYTEST_ARGS="--pdb" +``` + +To see all available pytest arguments, run `make test PYTEST_ARGS="--help"`. + +## Code standards + +Below are the formalized conventions that we adhere to in the PyIceberg project. The goal of this is to have a common agreement on how to evolve the codebase, but also using it as guidelines for newcomers to the project. + +## API Compatibility + +It is important to keep the Python public API compatible across versions. The Python official [PEP-8](https://peps.python.org/pep-0008/) defines Public methods as: _Public attributes should have no leading underscores_. This means not removing any methods without any notice, or removing or renaming any existing parameters. Adding new optional parameters is okay. + +If you want to remove a method, please add a deprecation notice by annotating the function using `@deprecated`: + +```python +from pyiceberg.utils.deprecated import deprecated + + +@deprecated( + deprecated_in="0.1.0", + removed_in="0.2.0", + help_message="Please use load_something_else() instead", +) +def load_something(): + pass +``` + +Which will warn: + +``` +Call to load_something, deprecated in 0.1.0, will be removed in 0.2.0. Please use load_something_else() instead. +``` + +## Type annotations + +For the type annotation the types from the `Typing` package are used. + +PyIceberg offers support from Python 3.8 onwards, we can't use the [type hints from the standard collections](https://peps.python.org/pep-0585/). + +## Third party libraries + +PyIceberg naturally integrates into the rich Python ecosystem, however it is important to be hesistant to add third party packages. Adding a lot of packages makes the library heavyweight, and causes incompatibilities with other projects if they use a different version of the library. Also, big libraries such as `s3fs`, `pyarrow`, `thrift` should be optional to avoid downloading everything, while not being sure if is actually being used. diff --git a/mkdocs/docs/feature-support.md b/mkdocs/docs/feature-support.md new file mode 100644 index 0000000000..366dfbe0c5 --- /dev/null +++ b/mkdocs/docs/feature-support.md @@ -0,0 +1,85 @@ + + +# Feature Support + +The goal is that the python library will provide a functional, performant subset of the Java library. The initial focus has been on reading table metadata and provide a convenient CLI to go through the catalog. + +## Metadata + +| Operation | Java | Python | +|:-------------------------|:-----:|:------:| +| Get Schema | X | X | +| Get Snapshots | X | X | +| Plan Scan | X | X | +| Plan Scan for Snapshot | X | X | +| Update Current Snapshot | X | | +| Create Table | X | X | +| Rename Table | X | X | +| Drop Table | X | X | +| Alter Table | X | | +| Set Table Properties | X | | +| Create Namespace | X | X | +| Drop Namespace | X | X | +| Set Namespace Properties | X | X | + +## Types + +The types are kept in `pyiceberg.types`. + +Primitive types: + +- `BooleanType` +- `StringType` +- `IntegerType` +- `LongType` +- `FloatType` +- `DoubleType` +- `DateType` +- `TimeType` +- `TimestampType` +- `TimestamptzType` +- `BinaryType` +- `UUIDType` + +Complex types: + +- `StructType` +- `ListType` +- `MapType` +- `FixedType(16)` +- `DecimalType(8, 3)` + +## Expressions + +The expressions are kept in `pyiceberg.expressions`. + +- `IsNull` +- `NotNull` +- `IsNaN` +- `NotNaN` +- `In` +- `NotIn` +- `EqualTo` +- `NotEqualTo` +- `GreaterThanOrEqual` +- `GreaterThan` +- `LessThanOrEqual` +- `LessThan` +- `And` +- `Or` +- `Not` diff --git a/dev/RELEASE.md b/mkdocs/docs/how-to-release.md similarity index 79% rename from dev/RELEASE.md rename to mkdocs/docs/how-to-release.md index a99683031c..5f3b6795a9 100644 --- a/dev/RELEASE.md +++ b/mkdocs/docs/how-to-release.md @@ -21,7 +21,7 @@ The guide to release PyIceberg. -First we're going to release a release candidate (RC) and publish it to the public for testing and validation. Once the vote has passed on the RC, we can release the new version. +The first step is to publish a release candidate (RC) and publish it to the public for testing and validation. Once the vote has passed on the RC, the RC turns into the new release. ## Running a release candidate @@ -44,7 +44,7 @@ export LAST_COMMIT_ID=$(git rev-list ${GIT_TAG} 2> /dev/null | head -n 1) The `-s` option will sign the commit. If you don't have a key yet, you can find the instructions [here](http://www.apache.org/dev/openpgp.html#key-gen-generate-key). To install gpg on a M1 based Mac, a couple of additional steps are required: https://gist.github.com/phortuin/cf24b1cca3258720c71ad42977e1ba57 -Next we'll create a source distribution (`sdist`) which will generate a `.tar.gz` with all the source files. So we can upload the files to the Apache SVN. +Next step is to create a source distribution (`sdist`) which will generate a `.tar.gz` with all the source files. These files need to be uploaded to the Apache SVN. ``` poetry build @@ -62,7 +62,7 @@ Building pyiceberg (0.1.0) The `sdist` contains the source which can be used for checking licenses, and the wheel is a compiled version for quick installation. -Before committing the files to the Apache SVN artifact distribution SVN, we need to generate hashes, and we need to sign them using gpg: +Before committing the files to the Apache SVN artifact distribution SVN hashes need to be generated, and those need to be signed with gpg to make sure that they are authentic: ```bash for name in "pyiceberg-${VERSION_WITHOUT_RC}-py3-none-any.whl" "pyiceberg-${VERSION_WITHOUT_RC}.tar.gz" @@ -72,7 +72,7 @@ do done ``` -Next, we'll clone the Apache SVN, copy and commit the files: +Next step is to clone the Apache SVN, copy and commit the files: ```bash export SVN_TMP_DIR=/tmp/iceberg-${VERSION_BRANCH}/ @@ -85,7 +85,7 @@ svn add $SVN_TMP_DIR_VERSIONED svn ci -m "PyIceberg ${VERSION}" ${SVN_TMP_DIR_VERSIONED} ``` -Next, we can upload them to pypi. Please keep in mind that this **won't** bump the version for everyone that hasn't pinned their version, we set it to a RC [pre-release and those are ignored](https://packaging.python.org/en/latest/guides/distributing-packages-using-setuptools/#pre-release-versioning). +Next step is to upload them to pypi. Please keep in mind that this **won't** bump the version for everyone that hasn't pinned their version, since it is set to a RC [pre-release and those are ignored](https://packaging.python.org/en/latest/guides/distributing-packages-using-setuptools/#pre-release-versioning). ``` poetry version ${VERSION} @@ -94,7 +94,7 @@ poetry build twine upload -s dist/* ``` -Finally, we can generate the email what we'll send to the mail list: +Finally step is to generate the email what send to the dev mail list: ```bash cat << EOF > release-announcement-email.txt @@ -134,3 +134,13 @@ EOF cat release-announcement-email.txt ``` + +## Vote has passed + +Once the vote has been passed, the latest version can be pushed to PyPi. Check out the commit associated with the passing vote, and run: + +```bash +rm -rf dist/ +poetry build +twine upload -s dist/* +``` diff --git a/mkdocs/docs/index.md b/mkdocs/docs/index.md index 53795a8e07..e36588e434 100644 --- a/mkdocs/docs/index.md +++ b/mkdocs/docs/index.md @@ -17,9 +17,9 @@ # PyIceberg -Much of the python api conforms to the Java API. You can get more info about the java api [here](https://iceberg.apache.org/docs/latest/java-api-quickstart/). +PyIceberg is a Python implementation for accessing Iceberg tables, without the need of a JVM. -## Installing +## Install You can install the latest release version from pypi: @@ -27,543 +27,31 @@ You can install the latest release version from pypi: pip3 install "pyiceberg[s3fs,hive]" ``` -Or install the latest development version locally: +Install it directly for Github (not recommended), but sometimes handy: -```sh -git clone https://github.com/apache/iceberg.git -cd iceberg/python -pip3 install -e ".[s3fs,hive]" ``` - -You can mix and match optional dependencies: - -| Key | Description: | -|-----------|----------------------------------------------------------------------| -| hive | Support for the Hive metastore | -| glue | Support for AWS Glue | -| pyarrow | PyArrow as a FileIO implementation to interact with the object store | -| s3fs | S3FS as a FileIO implementation to interact with the object store | -| snappy | Support for snappy Avro compression | - -# Python CLI Quickstart - -Pyiceberg comes with a CLI that's available after installing the `pyiceberg` package. - -```sh -➜ pyiceberg --help -Usage: pyiceberg [OPTIONS] COMMAND [ARGS]... - -Options: ---catalog TEXT ---verbose BOOLEAN ---output [text|json] ---uri TEXT ---credential TEXT ---help Show this message and exit. - -Commands: -describe Describes a namespace xor table -drop Operations to drop a namespace or table -list Lists tables or namespaces -location Returns the location of the table -properties Properties on tables/namespaces -rename Renames a table -schema Gets the schema of the table -spec Returns the partition spec of the table -uuid Returns the UUID of the table +pip install "git+https://github.com/apache/iceberg.git#subdirectory=python&egg=pyiceberg[s3fs]" ``` -# Configuration - -There are three ways of setting the configuration. - -For the CLI you can pass it in using `--uri` and `--credential` and it will automatically detect the type based on the scheme (`http(s)` for rest, `thrift` for Hive). - -Secondly, YAML based configuration is supported `cat ~/.pyiceberg.yaml`: - -```yaml -catalog: - default: - uri: thrift://localhost:9083 - s3.endpoint: http://localhost:9000 - s3.access-key-id: admin - s3.secret-access-key: password - - rest: - uri: http://rest-catalog/ws/ - credential: t-1234:secret - - mtls-secured-catalog: - uri: https://rest-catalog/ws/ - ssl: - client: - cert: /absolute/path/to/client.crt - key: /absolute/path/to/client.key - cabundle: /absolute/path/to/cabundle.pem - - glue: - type: glue -``` - -Lastly, you can also set it using environment variables: +Or clone the repository for local development: ```sh -export PYICEBERG_CATALOG__DEFAULT__URI=thrift://localhost:9083 - -export PYICEBERG_CATALOG__REST__URI=http://rest-catalog/ws/ -export PYICEBERG_CATALOG__REST__CREDENTIAL=t-1234:secret - -export PYICEBERG_CATALOG__GLUE__TYPE=glue -``` - -Where the structure is equivalent to the YAML. The levels are separated using a double underscore (`__`). - -If you want to use AWS Glue as the catalog, you can use the last two ways to configure the pyiceberg and refer -[How to configure AWS credentials](https://docs.aws.amazon.com/cli/latest/userguide/cli-chap-configure.html) to set your AWS account credentials locally. - -## FileIO configuration - -For the FileIO there are several configuration options available: - -| Key | Example | Description | -|----------------------|---------------------|-----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| -| s3.endpoint | https://10.0.19.25/ | Configure an alternative endpoint of the S3 service for the FileIO to access. This could be used to use S3FileIO with any s3-compatible object storage service that has a different endpoint, or access a private S3 endpoint in a virtual private cloud. | -| s3.access-key-id | admin | Configure the static secret access key used to access the FileIO. | -| s3.secret-access-key | password | Configure the static session token used to access the FileIO. | -| s3.signer | bearer | Configure the signature version of the FileIO. | - -# CLI Quickstart - -This example assumes that you have a default catalog set. If you want to load another catalog, for example, the rest example above. Then you need to set `--catalog rest`. - -```sh -➜ pyiceberg list -default -nyc -``` - -```sh -➜ pyiceberg list nyc -nyc.taxis -``` - -```sh -➜ pyiceberg describe nyc.taxis -Table format version 1 -Metadata location file:/.../nyc.db/taxis/metadata/00000-aa3a3eac-ea08-4255-b890-383a64a94e42.metadata.json -Table UUID 6cdfda33-bfa3-48a7-a09e-7abb462e3460 -Last Updated 1661783158061 -Partition spec [] -Sort order [] -Current schema Schema, id=0 -├── 1: VendorID: optional long -├── 2: tpep_pickup_datetime: optional timestamptz -├── 3: tpep_dropoff_datetime: optional timestamptz -├── 4: passenger_count: optional double -├── 5: trip_distance: optional double -├── 6: RatecodeID: optional double -├── 7: store_and_fwd_flag: optional string -├── 8: PULocationID: optional long -├── 9: DOLocationID: optional long -├── 10: payment_type: optional long -├── 11: fare_amount: optional double -├── 12: extra: optional double -├── 13: mta_tax: optional double -├── 14: tip_amount: optional double -├── 15: tolls_amount: optional double -├── 16: improvement_surcharge: optional double -├── 17: total_amount: optional double -├── 18: congestion_surcharge: optional double -└── 19: airport_fee: optional double -Current snapshot Operation.APPEND: id=5937117119577207079, schema_id=0 -Snapshots Snapshots -└── Snapshot 5937117119577207079, schema 0: file:/.../nyc.db/taxis/metadata/snap-5937117119577207079-1-94656c4f-4c66-4600-a4ca-f30377300527.avro -Properties owner root -write.format.default parquet -``` - -Or output in JSON for automation: - -```sh -➜ pyiceberg --output json describe nyc.taxis | jq -{ - "identifier": [ - "nyc", - "taxis" - ], - "metadata_location": "file:/.../nyc.db/taxis/metadata/00000-aa3a3eac-ea08-4255-b890-383a64a94e42.metadata.json", - "metadata": { - "location": "file:/.../nyc.db/taxis", - "table-uuid": "6cdfda33-bfa3-48a7-a09e-7abb462e3460", - "last-updated-ms": 1661783158061, - "last-column-id": 19, - "schemas": [ - { - "type": "struct", - "fields": [ - { - "id": 1, - "name": "VendorID", - "type": "long", - "required": false - }, -... - { - "id": 19, - "name": "airport_fee", - "type": "double", - "required": false - } - ], - "schema-id": 0, - "identifier-field-ids": [] - } - ], - "current-schema-id": 0, - "partition-specs": [ - { - "spec-id": 0, - "fields": [] - } - ], - "default-spec-id": 0, - "last-partition-id": 999, - "properties": { - "owner": "root", - "write.format.default": "parquet" - }, - "current-snapshot-id": 5937117119577207000, - "snapshots": [ - { - "snapshot-id": 5937117119577207000, - "timestamp-ms": 1661783158061, - "manifest-list": "file:/.../nyc.db/taxis/metadata/snap-5937117119577207079-1-94656c4f-4c66-4600-a4ca-f30377300527.avro", - "summary": { - "operation": "append", - "spark.app.id": "local-1661783139151", - "added-data-files": "1", - "added-records": "2979431", - "added-files-size": "46600777", - "changed-partition-count": "1", - "total-records": "2979431", - "total-files-size": "46600777", - "total-data-files": "1", - "total-delete-files": "0", - "total-position-deletes": "0", - "total-equality-deletes": "0" - }, - "schema-id": 0 - } - ], - "snapshot-log": [ - { - "snapshot-id": "5937117119577207079", - "timestamp-ms": 1661783158061 - } - ], - "metadata-log": [], - "sort-orders": [ - { - "order-id": 0, - "fields": [] - } - ], - "default-sort-order-id": 0, - "refs": { - "main": { - "snapshot-id": 5937117119577207000, - "type": "branch" - } - }, - "format-version": 1, - "schema": { - "type": "struct", - "fields": [ - { - "id": 1, - "name": "VendorID", - "type": "long", - "required": false - }, -... - { - "id": 19, - "name": "airport_fee", - "type": "double", - "required": false - } - ], - "schema-id": 0, - "identifier-field-ids": [] - }, - "partition-spec": [] - } -} -``` - -# Python API - -To instantiate a catalog: - -```python -from pyiceberg.catalog import load_catalog - -catalog = load_catalog("prod") - -catalog.list_namespaces() -``` - -Returns: - -``` -[('default',), ('nyc',)] -``` - -Listing the tables in the `nyc` namespace: - -```python -catalog.list_tables("nyc") -``` - -Returns: - -``` -[('nyc', 'taxis')] -``` - -Loading the `taxis` table: - -```python -catalog.load_table(("nyc", "taxis")) -``` - -``` -Table( - identifier=('nyc', 'taxis'), - metadata_location='s3a://warehouse/wh/nyc.db/taxis/metadata/00002-6ea51ce3-62aa-4197-9cf8-43d07c3440ca.metadata.json', - metadata=TableMetadataV2( - location='s3a://warehouse/wh/nyc.db/taxis', - table_uuid=UUID('ebd5d172-2162-453d-b586-1cdce52c1116'), - last_updated_ms=1662633437826, - last_column_id=19, - schemas=[Schema( - NestedField(field_id=1, name='VendorID', field_type=LongType(), required=False), - NestedField(field_id=2, name='tpep_pickup_datetime', field_type=TimestamptzType(), required=False), - NestedField(field_id=3, name='tpep_dropoff_datetime', field_type=TimestamptzType(), required=False), - NestedField(field_id=4, name='passenger_count', field_type=DoubleType(), required=False), - NestedField(field_id=5, name='trip_distance', field_type=DoubleType(), required=False), - NestedField(field_id=6, name='RatecodeID', field_type=DoubleType(), required=False), - NestedField(field_id=7, name='store_and_fwd_flag', field_type=StringType(), required=False), - NestedField(field_id=8, name='PULocationID', field_type=LongType(), required=False), - NestedField(field_id=9, name='DOLocationID', field_type=LongType(), required=False), - NestedField(field_id=10, name='payment_type', field_type=LongType(), required=False), - NestedField(field_id=11, name='fare_amount', field_type=DoubleType(), required=False), - NestedField(field_id=12, name='extra', field_type=DoubleType(), required=False), - NestedField(field_id=13, name='mta_tax', field_type=DoubleType(), required=False), - NestedField(field_id=14, name='tip_amount', field_type=DoubleType(), required=False), - NestedField(field_id=15, name='tolls_amount', field_type=DoubleType(), required=False), - NestedField(field_id=16, name='improvement_surcharge', field_type=DoubleType(), required=False), - NestedField(field_id=17, name='total_amount', field_type=DoubleType(), required=False), - NestedField(field_id=18, name='congestion_surcharge', field_type=DoubleType(), required=False), - NestedField(field_id=19, name='airport_fee', field_type=DoubleType(), required=False) - ), - schema_id=0, - identifier_field_ids=[] - )], - current_schema_id=0, - partition_specs=[PartitionSpec(spec_id=0)], - default_spec_id=0, - last_partition_id=999, - properties={ - 'owner': 'root', - 'write.format.default': 'parquet' - }, - current_snapshot_id=8334458494559715805, - snapshots=[ - Snapshot( - snapshot_id=7910949481055846233, - parent_snapshot_id=None, - sequence_number=None, - timestamp_ms=1662489306555, - manifest_list='s3a://warehouse/wh/nyc.db/taxis/metadata/snap-7910949481055846233-1-3eb7a2e1-5b7a-4e76-a29a-3e29c176eea4.avro', - summary=Summary( - Operation.APPEND, - **{ - 'spark.app.id': 'local-1662489289173', - 'added-data-files': '1', - 'added-records': '2979431', - 'added-files-size': '46600777', - 'changed-partition-count': '1', - 'total-records': '2979431', - 'total-files-size': '46600777', - 'total-data-files': '1', - 'total-delete-files': '0', - 'total-position-deletes': '0', - 'total-equality-deletes': '0' - } - ), - schema_id=0 - ), - ], - snapshot_log=[ - SnapshotLogEntry( - snapshot_id='7910949481055846233', - timestamp_ms=1662489306555 - ) - ], - metadata_log=[ - MetadataLogEntry( - metadata_file='s3a://warehouse/wh/nyc.db/taxis/metadata/00000-b58341ba-6a63-4eea-9b2f-e85e47c7d09f.metadata.json', - timestamp_ms=1662489306555 - ) - ], - sort_orders=[SortOrder(order_id=0)], - default_sort_order_id=0, - refs={ - 'main': SnapshotRef( - snapshot_id=8334458494559715805, - snapshot_ref_type=SnapshotRefType.BRANCH, - min_snapshots_to_keep=None, - max_snapshot_age_ms=None, - max_ref_age_ms=None - ) - }, - format_version=2, - last_sequence_number=1 - ) -) -``` - -And to create a table from a catalog: - -```python -from pyiceberg.schema import Schema -from pyiceberg.types import TimestampType, DoubleType, StringType, NestedField - -schema = Schema( - NestedField( - field_id=1, name="datetime", field_type=TimestampType(), required=False - ), - NestedField(field_id=2, name="bid", field_type=DoubleType(), required=False), - NestedField(field_id=3, name="ask", field_type=DoubleType(), required=False), - NestedField(field_id=4, name="symbol", field_type=StringType(), required=False), -) - -from pyiceberg.partitioning import PartitionSpec, PartitionField -from pyiceberg.transforms import DayTransform - -partition_spec = PartitionSpec( - PartitionField( - source_id=1, field_id=1000, transform=DayTransform(), name="datetime_day" - ) -) - -from pyiceberg.table.sorting import SortOrder, SortField -from pyiceberg.transforms import IdentityTransform - -sort_order = SortOrder(SortField(source_id=4, transform=IdentityTransform())) - -from pyiceberg.catalog.hive import HiveCatalog - -catalog = HiveCatalog(name="prod", uri="thrift://localhost:9083/") - -catalog.create_table( - identifier="default.bids", - location="/Users/fokkodriesprong/Desktop/docker-spark-iceberg/wh/bids/", - schema=schema, - partition_spec=partition_spec, - sort_order=sort_order, -) -``` - -Which returns a newly created table: - -``` -Table( - identifier=('default', 'bids'), - metadata_location='/Users/fokkodriesprong/Desktop/docker-spark-iceberg/wh/bids//metadata/00000-c8cd93ab-f784-474d-a167-b1a86b05195f.metadata.json', - metadata=TableMetadataV2( - location='/Users/fokkodriesprong/Desktop/docker-spark-iceberg/wh/bids/', - table_uuid=UUID('38d4cb39-4945-4bf2-b374-984b5c4984d2'), - last_updated_ms=1661847562069, - last_column_id=4, - schemas=[ - Schema( - NestedField(field_id=1, name='datetime', field_type=TimestampType(), required=False), - NestedField(field_id=2, name='bid', field_type=DoubleType(), required=False), - NestedField(field_id=3, name='ask', field_type=DoubleType(), required=False), - NestedField(field_id=4, name='symbol', field_type=StringType(), required=False)), - schema_id=1, - identifier_field_ids=[]) - ], - current_schema_id=1, - partition_specs=[ - PartitionSpec( - PartitionField(source_id=1, field_id=1000, transform=DayTransform(), name='datetime_day'),)) - ], - default_spec_id=0, - last_partition_id=1000, - properties={}, - current_snapshot_id=None, - snapshots=[], - snapshot_log=[], - metadata_log=[], - sort_orders=[ - SortOrder(order_id=1, fields=[SortField(source_id=4, transform=IdentityTransform(), direction=SortDirection.ASC, null_order=NullOrder.NULLS_FIRST)]) - ], - default_sort_order_id=1, - refs={}, - format_version=2, - last_sequence_number=0 - ) -) +git clone https://github.com/apache/iceberg.git +cd iceberg/python +pip3 install -e ".[s3fs,hive]" ``` -# Feature Support - -The goal is that the python library will provide a functional, performant subset of the Java library. The initial focus has been on reading table metadata and provide a convenient CLI to go through the catalog. - -## Metadata - -| Operation | Java | Python | -|:-------------------------|:-----:|:------:| -| Get Schema | X | X | -| Get Snapshots | X | X | -| Plan Scan | X | X | -| Plan Scan for Snapshot | X | X | -| Update Current Snapshot | X | | -| Create Table | X | X | -| Rename Table | X | X | -| Drop Table | X | X | -| Alter Table | X | | -| Set Table Properties | X | | -| Create Namespace | X | X | -| Drop Namespace | X | X | -| Set Namespace Properties | X | X | - -## Types - -The types are kept in `pyiceberg.types`. - -Primitive types: +You can mix and match optional dependencies depending on your needs: -- `BooleanType` -- `StringType` -- `IntegerType` -- `LongType` -- `FloatType` -- `DoubleType` -- `DateType` -- `TimeType` -- `TimestampType` -- `TimestamptzType` -- `BinaryType` -- `UUIDType` +| Key | Description: | +|---------|----------------------------------------------------------------------| +| hive | Support for the Hive metastore | +| glue | Support for AWS Glue | +| pyarrow | PyArrow as a FileIO implementation to interact with the object store | +| duckdb | Installs both PyArrow and DuckDB | +| s3fs | S3FS as a FileIO implementation to interact with the object store | +| snappy | Support for snappy Avro compression | -Complex types: +You either need to install `s3fs` or `pyarrow` for fetching files. -- `StructType` -- `ListType` -- `MapType` -- `FixedType(16)` -- `DecimalType(8, 3)` +There is both a [CLI](cli.md) and [Python API](api.md) available. diff --git a/mkdocs/docs/verify-release.md b/mkdocs/docs/verify-release.md new file mode 100644 index 0000000000..e453eea12e --- /dev/null +++ b/mkdocs/docs/verify-release.md @@ -0,0 +1,100 @@ + + +# Verifying a release + +Each Apache PyIceberg release is validated by the community by holding a vote. A community release manager will prepare a release candidate and call a vote on the Iceberg dev list. To validate the release candidate, community members will test it out in their downstream projects and environments. + +In addition to testing in downstream projects, community members also check the release’s signatures, checksums, and license documentation. + +## Validating a release candidate + +Release announcements include links to the following: + +- A source tarball +- A signature (.asc) +- A checksum (.sha512) +- KEYS file +- GitHub change comparison + +After downloading the source tarball, signature, checksum, and KEYS file, here are instructions on how to verify signatures, checksums, and documentation. + +## Verifying signatures + +First, import the keys. + +```sh +curl https://dist.apache.org/repos/dist/dev/iceberg/KEYS -o KEYS +gpg --import KEYS +``` + +Next, verify the `.asc` file. + +```sh +gpg --verify pyiceberg-0.2.0rc0-py3-none-any.whl.asc pyiceberg-0.2.0rc0-py3-none-any.whl +``` + +## Verifying checksums + +```sh +shasum -a 512 pyiceberg-0.2.0rc0.tar.gz +``` + +## Verifying License Documentation + +```sh +tar xzf pyiceberg-0.2.0rc0.tar.gztar +cd pyiceberg-0.2.0 +``` + +Run RAT checks to validate license header: + +``` +./dev/check-license +``` + +## Testing + +First step is to install the package: + +```sh +make install +``` + +And then run the tests: + +```sh +make test +``` + +To run the full integration tests: + +```sh +make test-s3 +``` + +This will include a Minio S3 container being spun up. + +# Cast the vote + +Votes are cast by replying to the release candidate announcement email on the dev mailing list with either `+1`, `0`, or `-1`. + +> \[ \] +1 Release this as Apache Iceberg 1.1.0 \[ \] +0 \[ \] -1 Do not release this because… + +In addition to your vote, it’s customary to specify if your vote is binding or non-binding. Only members of the Project Management Committee have formally binding votes. If you’re unsure, you can specify that your vote is non-binding. To read more about voting in the Apache framework, checkout the [Voting](https://www.apache.org/foundation/voting.html) information page on the Apache foundation’s website. diff --git a/mkdocs/mkdocs.yml b/mkdocs/mkdocs.yml index c84a2de465..522a45342e 100644 --- a/mkdocs/mkdocs.yml +++ b/mkdocs/mkdocs.yml @@ -16,3 +16,19 @@ # under the License. --- site_name: PyIceberg +site_url: https://py.iceberg.apache.org/ +nav: + - Home: index.md + - Configuration: configuration.md + - CLI: cli.md + - API: api.md + - Contributing: + - Contributing: contributing.md + - Feature support: feature-support.md + - Releases: + - Verify a release: verify-release.md + - How to release: how-to-release.md +theme: + name: readthedocs +markdown_extensions: + - admonition From bf1d50b56354399c16e0534c94ce0a2cce3dbebd Mon Sep 17 00:00:00 2001 From: Fokko Driesprong Date: Fri, 9 Dec 2022 18:54:18 +0100 Subject: [PATCH 297/642] Docs: Fix Python release variable (#6341) --- mkdocs/docs/how-to-release.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mkdocs/docs/how-to-release.md b/mkdocs/docs/how-to-release.md index 5f3b6795a9..12678d2c13 100644 --- a/mkdocs/docs/how-to-release.md +++ b/mkdocs/docs/how-to-release.md @@ -127,7 +127,7 @@ And can be installed using: pip3 install pyiceberg==$VERSION Please download, verify, and test. Please vote in the next 72 hours. -[ ] +1 Release this as PyIceberg $VERSION +[ ] +1 Release this as PyIceberg $VERSION_WITHOUT_RC [ ] +0 [ ] -1 Do not release this because... EOF From 16b8124eb1e8211c12ee416a0a40f57879ceee1e Mon Sep 17 00:00:00 2001 From: Fokko Driesprong Date: Fri, 9 Dec 2022 18:54:54 +0100 Subject: [PATCH 298/642] Python: Add support for 3.11 (#6329) Since https://github.com/apache/iceberg/pull/6295 and https://github.com/apache/iceberg/pull/6294 have been merged, we should be ready to support 3.11 Closes #6124 --- pyproject.toml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index 8b8768afed..5ebe8e196e 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -30,7 +30,8 @@ classifiers = [ "Operating System :: OS Independent", "Programming Language :: Python :: 3.8", "Programming Language :: Python :: 3.9", - "Programming Language :: Python :: 3.10" + "Programming Language :: Python :: 3.10", + "Programming Language :: Python :: 3.11" ] packages = [ From 834aa8d0ee67f2ad1fff6af37ffa8e298e086bba Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C4=90=E1=BA=B7ng=20Minh=20D=C5=A9ng?= Date: Mon, 12 Dec 2022 02:30:20 +0700 Subject: [PATCH 299/642] Python: Implement `to_pandas` (#6254) --- poetry.lock | 54 +++++++++++++++++++++++++++++++++++-- pyiceberg/table/__init__.py | 12 +++++++-- pyproject.toml | 7 +++++ 3 files changed, 69 insertions(+), 4 deletions(-) diff --git a/poetry.lock b/poetry.lock index 0a54622ac8..7c249e2858 100644 --- a/poetry.lock +++ b/poetry.lock @@ -500,6 +500,26 @@ python-versions = ">=3.6" [package.dependencies] pyparsing = ">=2.0.2,<3.0.5 || >3.0.5" +[[package]] +name = "pandas" +version = "1.5.2" +description = "Powerful data structures for data analysis, time series, and statistics" +category = "main" +optional = true +python-versions = ">=3.8" + +[package.dependencies] +numpy = [ + {version = ">=1.20.3", markers = "python_version < \"3.10\""}, + {version = ">=1.21.0", markers = "python_version >= \"3.10\""}, + {version = ">=1.23.2", markers = "python_version >= \"3.11\""}, +] +python-dateutil = ">=2.8.1" +pytz = ">=2020.1" + +[package.extras] +test = ["hypothesis (>=5.5.3)", "pytest (>=6.0)", "pytest-xdist (>=1.31)"] + [[package]] name = "pep517" version = "0.13.0" @@ -667,7 +687,7 @@ python-versions = "*" name = "pytz" version = "2022.6" description = "World timezone definitions, modern and historical" -category = "dev" +category = "main" optional = false python-versions = "*" @@ -948,6 +968,7 @@ cffi = ["cffi (>=1.11)"] duckdb = ["duckdb", "pyarrow"] glue = ["boto3"] hive = ["thrift"] +pandas = ["pandas", "pyarrow"] pyarrow = ["pyarrow"] s3fs = ["s3fs"] snappy = ["python-snappy"] @@ -955,7 +976,7 @@ snappy = ["python-snappy"] [metadata] lock-version = "1.1" python-versions = "^3.8" -content-hash = "8fa466512e3f74dd8f985ed4c85d2698407bffda8f23858700f5089e904f3982" +content-hash = "0a9c1eb50886e25d628f1c78c4dc236a2cb28225340aa853eb91aab7bdc96a9b" [metadata.files] aiobotocore = [ @@ -1622,6 +1643,35 @@ packaging = [ {file = "packaging-21.3-py3-none-any.whl", hash = "sha256:ef103e05f519cdc783ae24ea4e2e0f508a9c99b2d4969652eed6a2e1ea5bd522"}, {file = "packaging-21.3.tar.gz", hash = "sha256:dd47c42927d89ab911e606518907cc2d3a1f38bbd026385970643f9c5b8ecfeb"}, ] +pandas = [ + {file = "pandas-1.5.2-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:e9dbacd22555c2d47f262ef96bb4e30880e5956169741400af8b306bbb24a273"}, + {file = "pandas-1.5.2-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:e2b83abd292194f350bb04e188f9379d36b8dfac24dd445d5c87575f3beaf789"}, + {file = "pandas-1.5.2-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:2552bffc808641c6eb471e55aa6899fa002ac94e4eebfa9ec058649122db5824"}, + {file = "pandas-1.5.2-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:1fc87eac0541a7d24648a001d553406f4256e744d92df1df8ebe41829a915028"}, + {file = "pandas-1.5.2-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:d0d8fd58df5d17ddb8c72a5075d87cd80d71b542571b5f78178fb067fa4e9c72"}, + {file = "pandas-1.5.2-cp310-cp310-win_amd64.whl", hash = "sha256:4aed257c7484d01c9a194d9a94758b37d3d751849c05a0050c087a358c41ad1f"}, + {file = "pandas-1.5.2-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:375262829c8c700c3e7cbb336810b94367b9c4889818bbd910d0ecb4e45dc261"}, + {file = "pandas-1.5.2-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:cc3cd122bea268998b79adebbb8343b735a5511ec14efb70a39e7acbc11ccbdc"}, + {file = "pandas-1.5.2-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:b4f5a82afa4f1ff482ab8ded2ae8a453a2cdfde2001567b3ca24a4c5c5ca0db3"}, + {file = "pandas-1.5.2-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:8092a368d3eb7116e270525329a3e5c15ae796ccdf7ccb17839a73b4f5084a39"}, + {file = "pandas-1.5.2-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:f6257b314fc14958f8122779e5a1557517b0f8e500cfb2bd53fa1f75a8ad0af2"}, + {file = "pandas-1.5.2-cp311-cp311-win_amd64.whl", hash = "sha256:82ae615826da838a8e5d4d630eb70c993ab8636f0eff13cb28aafc4291b632b5"}, + {file = "pandas-1.5.2-cp38-cp38-macosx_10_9_universal2.whl", hash = "sha256:457d8c3d42314ff47cc2d6c54f8fc0d23954b47977b2caed09cd9635cb75388b"}, + {file = "pandas-1.5.2-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:c009a92e81ce836212ce7aa98b219db7961a8b95999b97af566b8dc8c33e9519"}, + {file = "pandas-1.5.2-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:71f510b0efe1629bf2f7c0eadb1ff0b9cf611e87b73cd017e6b7d6adb40e2b3a"}, + {file = "pandas-1.5.2-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a40dd1e9f22e01e66ed534d6a965eb99546b41d4d52dbdb66565608fde48203f"}, + {file = "pandas-1.5.2-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:5ae7e989f12628f41e804847a8cc2943d362440132919a69429d4dea1f164da0"}, + {file = "pandas-1.5.2-cp38-cp38-win32.whl", hash = "sha256:530948945e7b6c95e6fa7aa4be2be25764af53fba93fe76d912e35d1c9ee46f5"}, + {file = "pandas-1.5.2-cp38-cp38-win_amd64.whl", hash = "sha256:73f219fdc1777cf3c45fde7f0708732ec6950dfc598afc50588d0d285fddaefc"}, + {file = "pandas-1.5.2-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:9608000a5a45f663be6af5c70c3cbe634fa19243e720eb380c0d378666bc7702"}, + {file = "pandas-1.5.2-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:315e19a3e5c2ab47a67467fc0362cb36c7c60a93b6457f675d7d9615edad2ebe"}, + {file = "pandas-1.5.2-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:e18bc3764cbb5e118be139b3b611bc3fbc5d3be42a7e827d1096f46087b395eb"}, + {file = "pandas-1.5.2-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:0183cb04a057cc38fde5244909fca9826d5d57c4a5b7390c0cc3fa7acd9fa883"}, + {file = "pandas-1.5.2-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:344021ed3e639e017b452aa8f5f6bf38a8806f5852e217a7594417fb9bbfa00e"}, + {file = "pandas-1.5.2-cp39-cp39-win32.whl", hash = "sha256:e7469271497960b6a781eaa930cba8af400dd59b62ec9ca2f4d31a19f2f91090"}, + {file = "pandas-1.5.2-cp39-cp39-win_amd64.whl", hash = "sha256:c218796d59d5abd8780170c937b812c9637e84c32f8271bbf9845970f8c1351f"}, + {file = "pandas-1.5.2.tar.gz", hash = "sha256:220b98d15cee0b2cd839a6358bd1f273d0356bf964c1a1aeb32d47db0215488b"}, +] pep517 = [ {file = "pep517-0.13.0-py3-none-any.whl", hash = "sha256:4ba4446d80aed5b5eac6509ade100bff3e7943a8489de249654a5ae9b33ee35b"}, {file = "pep517-0.13.0.tar.gz", hash = "sha256:ae69927c5c172be1add9203726d4b84cf3ebad1edcd5f71fcdc746e66e829f59"}, diff --git a/pyiceberg/table/__init__.py b/pyiceberg/table/__init__.py index 904b234f3a..5dbc2f22ae 100644 --- a/pyiceberg/table/__init__.py +++ b/pyiceberg/table/__init__.py @@ -59,6 +59,7 @@ from pyiceberg.types import StructType if TYPE_CHECKING: + import pandas as pd import pyarrow as pa from duckdb import DuckDBPyConnection @@ -211,7 +212,11 @@ def plan_files(self) -> Iterator[ScanTask]: ... @abstractmethod - def to_arrow(self) -> pa.table: + def to_arrow(self) -> pa.Table: + ... + + @abstractmethod + def to_pandas(self, **kwargs: Any) -> pd.DataFrame: ... def update(self: S, **overrides: Any) -> S: @@ -338,7 +343,7 @@ def plan_files(self) -> Iterator[ScanTask]: yield from (FileScanTask(file) for file in matching_partition_files) - def to_arrow(self) -> pa.table: + def to_arrow(self) -> pa.Table: from pyiceberg.io.pyarrow import PyArrowFileIO, expression_to_pyarrow, schema_to_pyarrow warnings.warn( @@ -380,6 +385,9 @@ def to_arrow(self) -> pa.table: return ds.to_table(filter=pyarrow_filter, columns=columns) + def to_pandas(self, **kwargs: Any) -> pd.DataFrame: + return self.to_arrow().to_pandas(**kwargs) + def to_duckdb(self, table_name: str, connection: Optional[DuckDBPyConnection] = None) -> DuckDBPyConnection: import duckdb diff --git a/pyproject.toml b/pyproject.toml index 5ebe8e196e..5f46a26859 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -60,6 +60,8 @@ zstandard = "0.19.0" pyarrow = { version = "10.0.1", optional = true } +pandas = { version = "1.5.2", optional = true } + duckdb = { version = "0.6.0", optional = true } python-snappy = { version = "0.6.1", optional = true } @@ -89,6 +91,7 @@ build-backend = "poetry.core.masonry.api" [tool.poetry.extras] pyarrow = ["pyarrow"] +pandas = ["pandas", "pyarrow"] duckdb = ["duckdb", "pyarrow"] snappy = ["python-snappy"] hive = ["thrift"] @@ -127,6 +130,10 @@ disallow_untyped_defs = true module = "pyarrow.*" ignore_missing_imports = true +[[tool.mypy.overrides]] +module = "pandas.*" +ignore_missing_imports = true + [[tool.mypy.overrides]] module = "snappy.*" ignore_missing_imports = true From 1e7aa7a3832863da01be34f0c957f87e56680912 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Sun, 11 Dec 2022 21:00:39 +0100 Subject: [PATCH 300/642] Build: Bump duckdb from 0.6.0 to 0.6.1 in /python (#6403) Bumps [duckdb](https://github.com/duckdb/duckdb) from 0.6.0 to 0.6.1. - [Release notes](https://github.com/duckdb/duckdb/releases) - [Changelog](https://github.com/duckdb/duckdb/blob/master/tools/release-pip.py) - [Commits](https://github.com/duckdb/duckdb/compare/v0.6.0...v0.6.1) --- updated-dependencies: - dependency-name: duckdb dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- poetry.lock | 98 +++++++++++++++++++++++++------------------------- pyproject.toml | 2 +- 2 files changed, 50 insertions(+), 50 deletions(-) diff --git a/poetry.lock b/poetry.lock index 7c249e2858..7ea806e86b 100644 --- a/poetry.lock +++ b/poetry.lock @@ -251,7 +251,7 @@ python-versions = ">=3.7" [[package]] name = "duckdb" -version = "0.6.0" +version = "0.6.1" description = "DuckDB embedded database" category = "main" optional = true @@ -976,7 +976,7 @@ snappy = ["python-snappy"] [metadata] lock-version = "1.1" python-versions = "^3.8" -content-hash = "0a9c1eb50886e25d628f1c78c4dc236a2cb28225340aa853eb91aab7bdc96a9b" +content-hash = "5c058a9e9a389a4f376baa1d621659fc842293d2a63bb800b111fdd2f6944ef1" [metadata.files] aiobotocore = [ @@ -1279,53 +1279,53 @@ docutils = [ {file = "docutils-0.19.tar.gz", hash = "sha256:33995a6753c30b7f577febfc2c50411fec6aac7f7ffeb7c4cfe5991072dcf9e6"}, ] duckdb = [ - {file = "duckdb-0.6.0-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:0375f174f6ea9e65a5a1db20663d1cee0663ef4021b1591d515fe69822244871"}, - {file = "duckdb-0.6.0-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:6419820349c2c939d740dc1e045df3bc031afb1b86d36e876cec09e6ca84d71b"}, - {file = "duckdb-0.6.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:cfe007492d02ee2b76e530a4b52168d0a92819b5b38be50061665d7ebee7a3d2"}, - {file = "duckdb-0.6.0-cp310-cp310-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:1ae9ff3c8e1d510621888db313dcd808a3e52caedc85c8944100e512b29f6eb6"}, - {file = "duckdb-0.6.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:bcec180d90a61caa790cda3de69bd2ea7a62b898c243d045ea68bfe657a5e99a"}, - {file = "duckdb-0.6.0-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:fe3f54079db38cb7bd2101b6f96519c2bd24f66474ba1b20a987093d6bfa4b82"}, - {file = "duckdb-0.6.0-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:00c6f009ba84745e1afdfe03c7641c4601e6e8b4c3e3ee1f770eada4ae9e29d8"}, - {file = "duckdb-0.6.0-cp310-cp310-win32.whl", hash = "sha256:019096b210c921d01ae0c4ec17deb7a487f20c94ee2a811744bc9d7d23bcee98"}, - {file = "duckdb-0.6.0-cp310-cp310-win_amd64.whl", hash = "sha256:7d7ab4e963141246771d5f15c151dae84a1fd90a986312a77cdc999faa89eae4"}, - {file = "duckdb-0.6.0-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:9ba293125ce3acc6dcec148e22b37c49880e2319e415f322b65ffbcabf762afb"}, - {file = "duckdb-0.6.0-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:f23d5bef667e4631aa9aa57909e2b1eeeb583680ce350c008364894761d3ff55"}, - {file = "duckdb-0.6.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:14972984422d37113fb15a93437db9283b647029db8a7c6c0935977997fe1d7f"}, - {file = "duckdb-0.6.0-cp311-cp311-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:d096d1b623f0a166db621748988bcc0cab9ac8c99c6d5ecc8a72dca71a1a4a49"}, - {file = "duckdb-0.6.0-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:39efe9498309d9e1ce9b693ade5be4ec1d3528c0adc115936717710a396791b0"}, - {file = "duckdb-0.6.0-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:01b03d20f0218fcfbae25b14f06db763ce7951d912e1142a2684fc4613ca546e"}, - {file = "duckdb-0.6.0-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:a7f42d1259bff2d8d010ba064b41bce1ce3d8d79a322e2fe69d21f172f38fe9a"}, - {file = "duckdb-0.6.0-cp311-cp311-win32.whl", hash = "sha256:0df5fc44a3dc31ebc3a42b7f6da510d45e0d8494955a5e22baa497ee1dc5c3f6"}, - {file = "duckdb-0.6.0-cp311-cp311-win_amd64.whl", hash = "sha256:3f36de549f7117f0c30a17b58c2e12c2cf5054a2fe0aef7c04674f1602959c4a"}, - {file = "duckdb-0.6.0-cp36-cp36m-macosx_10_9_x86_64.whl", hash = "sha256:7b312cab0da395aea7bd7b84cb0a6a21dd5e03e5993b9ae2e6c5e9cfa2607b21"}, - {file = "duckdb-0.6.0-cp36-cp36m-win32.whl", hash = "sha256:238e4cc0bc715e688346ae7cd0eaacd9840eabf9ac1f8135e6ac208ce9f07235"}, - {file = "duckdb-0.6.0-cp36-cp36m-win_amd64.whl", hash = "sha256:c746a25d3bcb378c0bed65fd93f8086056529f216b063ac20dd7fe878f6c7438"}, - {file = "duckdb-0.6.0-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:36d45de5d64722768c579fc4fe4ac3c937c65f0ab34a05d1cf2eda449ce79a81"}, - {file = "duckdb-0.6.0-cp37-cp37m-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:902a3a0e712d9114f6b4067c7b53d1d64bddd825d96d9fd61578dc58c13f5524"}, - {file = "duckdb-0.6.0-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:03c13bf558febda114cdd2de9db6d202cbd5bfdbac66dbdc94faa432313b39dd"}, - {file = "duckdb-0.6.0-cp37-cp37m-musllinux_1_1_i686.whl", hash = "sha256:3875b9cc8f008c3a69138a872f2fb9d4e655f889414616755aba700f03a9b399"}, - {file = "duckdb-0.6.0-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:0447fc65e930baef6bbfe11807dd6badd6a24ebb5d739908e50fc0d9f68504f1"}, - {file = "duckdb-0.6.0-cp37-cp37m-win32.whl", hash = "sha256:9796c1359ef31808a5f2e83ab981ba4760da02e0bdbc66d4f46db7e9e2c0fe54"}, - {file = "duckdb-0.6.0-cp37-cp37m-win_amd64.whl", hash = "sha256:853474b71fccb804fc7c338eeca7c795f385dfe3b666cd6184fd5a9c6640958e"}, - {file = "duckdb-0.6.0-cp38-cp38-macosx_10_9_universal2.whl", hash = "sha256:358e2d444619f558563cf1f625680925d67344e62fec81a7a1bf2ed9f959a0b0"}, - {file = "duckdb-0.6.0-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:0c8590f99d63d5407e40e45a90b250ec92f7edaddc67c475af2d82f1a11b18c9"}, - {file = "duckdb-0.6.0-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:ed875f32c3139e84ceb42eeda3b12e56bd803de0015494a25a8176766169ff69"}, - {file = "duckdb-0.6.0-cp38-cp38-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:5f4f0b50f7f4f4c22fc7debd28d1b81705152e52d44425bf777395cdf541b9bb"}, - {file = "duckdb-0.6.0-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:860a62fcf67f8ae86cd946f0ca74d6b22f00ebd845c588fbdd761eca5923000e"}, - {file = "duckdb-0.6.0-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:54de408dcc95f0cd5fffba6d54044b3e97840db93b8e7c961853941d5ec59a30"}, - {file = "duckdb-0.6.0-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:afa8f0f391608c4139752d6f377c4119e2598df3e84087a321bf285348e319e2"}, - {file = "duckdb-0.6.0-cp38-cp38-win32.whl", hash = "sha256:2ddd6f73c42b78fd862ead4df6a730c3087589e842320ec10ad6ce0a4e170b0e"}, - {file = "duckdb-0.6.0-cp38-cp38-win_amd64.whl", hash = "sha256:7ae9d30de3cb881e07b7e95b4ff16b8c121a7714f4ad376c7ef583601a7c1bd9"}, - {file = "duckdb-0.6.0-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:e12b4087cdc48c5c2ed2cc0dbf648df357ace88e3f47dd4152958bd5c5646794"}, - {file = "duckdb-0.6.0-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:89b6c72cd82089c5f3b313bb78de1d8f96cfe87e80bff9b93ee837e29ddf55fe"}, - {file = "duckdb-0.6.0-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:3d859640d49ef856e4d68a086d7c3a17f38b380e9b10387a0419630c17c32b52"}, - {file = "duckdb-0.6.0-cp39-cp39-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:30a2382678a2fc9c0284fb976e3392f50af780dfa404fc18a5d34e443478864f"}, - {file = "duckdb-0.6.0-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:0a9c478f964a45e94338a922af36cd7413aae504d365bb94850270d53bc27182"}, - {file = "duckdb-0.6.0-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:5dc228ef3f0067f312c12a3d14e8ae1c8b4f2cbba637af917979bf73821d6ba0"}, - {file = "duckdb-0.6.0-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:c4851fd0869692f7c3be846bd3325a3f2f45f13821a6fc58dc4e2bd4fecf0b71"}, - {file = "duckdb-0.6.0-cp39-cp39-win32.whl", hash = "sha256:253c1c68635462811f1bef3d10fac36b5907461ee387ba441b7d5dc03844b31e"}, - {file = "duckdb-0.6.0-cp39-cp39-win_amd64.whl", hash = "sha256:5f11a9cd5f860db3ec66dae6f3c5b21df21b82fcbe1a6622acca14c16c0a0cc2"}, - {file = "duckdb-0.6.0.tar.gz", hash = "sha256:74e0e4cd1b77aaec9f76e3a0b4cf8535d80f2282f38c6248d4ec826a9606fe81"}, + {file = "duckdb-0.6.1-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:e566514f9327f89264e98ac14ee7a84fbd9857328028258422c3e8375ee19d25"}, + {file = "duckdb-0.6.1-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:b31c2883de5b19591a2852165e6b3f9821f77af649835f27bc146b26e4aa30cb"}, + {file = "duckdb-0.6.1-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:998165b2fb1f1d2b0ad742096015ea70878f7d40304643c7424c3ed3ddf07bfc"}, + {file = "duckdb-0.6.1-cp310-cp310-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:3941b3a1e8a1cdb7b90ab3917b87af816e71f9692e5ada7f19b6b60969f731e5"}, + {file = "duckdb-0.6.1-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:143611bd1b7c13343f087d4d423a7a8a4f33a114c5326171e867febf3f0fcfe1"}, + {file = "duckdb-0.6.1-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:125ba45e8b08f28858f918ec9cbd3a19975e5d8d9e8275ef4ad924028a616e14"}, + {file = "duckdb-0.6.1-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:e609a65b31c92f2f7166831f74b56f5ed54b33d8c2c4b4c3974c26fdc50464c5"}, + {file = "duckdb-0.6.1-cp310-cp310-win32.whl", hash = "sha256:b39045074fb9a3f068496475a5d627ad4fa572fa3b4980e3b479c11d0b706f2d"}, + {file = "duckdb-0.6.1-cp310-cp310-win_amd64.whl", hash = "sha256:16fa96ffaa3d842a9355a633fb8bc092d119be08d4bc02013946d8594417bc14"}, + {file = "duckdb-0.6.1-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:b4bbe2f6c1b109c626f9318eee80934ad2a5b81a51409c6b5083c6c5f9bdb125"}, + {file = "duckdb-0.6.1-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:cfea36b58928ce778d17280d4fb3bf0a2d7cff407667baedd69c5b41463ac0fd"}, + {file = "duckdb-0.6.1-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:0b64eb53d0d0695814bf1b65c0f91ab7ed66b515f89c88038f65ad5e0762571c"}, + {file = "duckdb-0.6.1-cp311-cp311-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:35b01bc724e1933293f4c34f410d2833bfbb56d5743b515d805bbfed0651476e"}, + {file = "duckdb-0.6.1-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:fec2c2466654ce786843bda2bfba71e0e4719106b41d36b17ceb1901e130aa71"}, + {file = "duckdb-0.6.1-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:82cd30f5cf368658ef879b1c60276bc8650cf67cfe3dc3e3009438ba39251333"}, + {file = "duckdb-0.6.1-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:a782bbfb7f5e97d4a9c834c9e78f023fb8b3f6687c22ca99841e6ed944b724da"}, + {file = "duckdb-0.6.1-cp311-cp311-win32.whl", hash = "sha256:e3702d4a9ade54c6403f6615a98bbec2020a76a60f5db7fcf085df1bd270e66e"}, + {file = "duckdb-0.6.1-cp311-cp311-win_amd64.whl", hash = "sha256:93b074f473d68c944b0eeb2edcafd91ad11da8432b484836efaaab4e26351d48"}, + {file = "duckdb-0.6.1-cp36-cp36m-macosx_10_9_x86_64.whl", hash = "sha256:adae183924d6d479202c39072e37d440b511326e84525bcb7432bca85f86caba"}, + {file = "duckdb-0.6.1-cp36-cp36m-win32.whl", hash = "sha256:546a1cd17595bd1dd009daf6f36705aa6f95337154360ce44932157d353dcd80"}, + {file = "duckdb-0.6.1-cp36-cp36m-win_amd64.whl", hash = "sha256:87b0d00eb9d1a7ebe437276203e0cdc93b4a2154ba9688c65e8d2a8735839ec6"}, + {file = "duckdb-0.6.1-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:8442e074de6e1969c3d2b24363a5a6d7f866d5ac3f4e358e357495b389eff6c1"}, + {file = "duckdb-0.6.1-cp37-cp37m-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:0a6bf2ae7bec803352dade14561cb0b461b2422e70f75d9f09b36ba2dad2613b"}, + {file = "duckdb-0.6.1-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:5054792f22733f89d9cbbced2bafd8772d72d0fe77f159310221cefcf981c680"}, + {file = "duckdb-0.6.1-cp37-cp37m-musllinux_1_1_i686.whl", hash = "sha256:21cc503dffc2c68bb825e4eb3098e82f40e910b3d09e1b3b7f090d39ad53fbea"}, + {file = "duckdb-0.6.1-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:54b3da77ad893e99c073087ff7f75a8c98154ac5139d317149f12b74367211db"}, + {file = "duckdb-0.6.1-cp37-cp37m-win32.whl", hash = "sha256:f1d709aa6a26172a3eab804b57763d5cdc1a4b785ac1fc2b09568578e52032ee"}, + {file = "duckdb-0.6.1-cp37-cp37m-win_amd64.whl", hash = "sha256:f4edcaa471d791393e37f63e3c7c728fa6324e3ac7e768b9dc2ea49065cd37cc"}, + {file = "duckdb-0.6.1-cp38-cp38-macosx_10_9_universal2.whl", hash = "sha256:d218c2dd3bda51fb79e622b7b2266183ac9493834b55010aa01273fa5b7a7105"}, + {file = "duckdb-0.6.1-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:0c7155cb93ab432eca44b651256c359281d26d927ff43badaf1d2276dd770832"}, + {file = "duckdb-0.6.1-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:0925778200090d3d5d8b6bb42b4d05d24db1e8912484ba3b7e7b7f8569f17dcb"}, + {file = "duckdb-0.6.1-cp38-cp38-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:8b544dd04bb851d08bc68b317a7683cec6091547ae75555d075f8c8a7edb626e"}, + {file = "duckdb-0.6.1-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:f2c37d5a0391cf3a3a66e63215968ffb78e6b84f659529fa4bd10478f6203071"}, + {file = "duckdb-0.6.1-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:ce376966260eb5c351fcc6af627a979dbbcae3efeb2e70f85b23aa45a21e289d"}, + {file = "duckdb-0.6.1-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:73c974b09dd08dff5e8bdedba11c7d0aa0fc46ca93954ee7d19e1e18c9883ac1"}, + {file = "duckdb-0.6.1-cp38-cp38-win32.whl", hash = "sha256:bfe39ed3a03e8b1ed764f58f513b37b24afe110d245803a41655d16d391ad9f1"}, + {file = "duckdb-0.6.1-cp38-cp38-win_amd64.whl", hash = "sha256:afa97d982dbe6b125631a17e222142e79bee88f7a13fc4cee92d09285e31ec83"}, + {file = "duckdb-0.6.1-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:c35ff4b1117096ef72d101524df0079da36c3735d52fcf1d907ccffa63bd6202"}, + {file = "duckdb-0.6.1-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:5c54910fbb6de0f21d562e18a5c91540c19876db61b862fc9ffc8e31be8b3f03"}, + {file = "duckdb-0.6.1-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:99a7172563a3ae67d867572ce27cf3962f58e76f491cb7f602f08c2af39213b3"}, + {file = "duckdb-0.6.1-cp39-cp39-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:7363ffe857d00216b659116647fbf1e925cb3895699015d4a4e50b746de13041"}, + {file = "duckdb-0.6.1-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:06c1cef25f896b2284ba048108f645c72fab5c54aa5a6f62f95663f44ff8a79b"}, + {file = "duckdb-0.6.1-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:e92dd6aad7e8c29d002947376b6f5ce28cae29eb3b6b58a64a46cdbfc5cb7943"}, + {file = "duckdb-0.6.1-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:4b280b2d8a01ecd4fe2feab041df70233c534fafbe33a38565b52c1e017529c7"}, + {file = "duckdb-0.6.1-cp39-cp39-win32.whl", hash = "sha256:d9212d76e90b8469743924a4d22bef845be310d0d193d54ae17d9ef1f753cfa7"}, + {file = "duckdb-0.6.1-cp39-cp39-win_amd64.whl", hash = "sha256:00b7be8f67ec1a8edaa8844f521267baa1a795f4c482bfad56c72c26e1862ab2"}, + {file = "duckdb-0.6.1.tar.gz", hash = "sha256:6d26e9f1afcb924a6057785e506810d48332d4764ddc4a5b414d0f2bf0cacfb4"}, ] exceptiongroup = [ {file = "exceptiongroup-1.0.4-py3-none-any.whl", hash = "sha256:542adf9dea4055530d6e1279602fa5cb11dab2395fa650b8674eaec35fc4a828"}, diff --git a/pyproject.toml b/pyproject.toml index 5f46a26859..77a0d8d976 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -62,7 +62,7 @@ pyarrow = { version = "10.0.1", optional = true } pandas = { version = "1.5.2", optional = true } -duckdb = { version = "0.6.0", optional = true } +duckdb = { version = "0.6.1", optional = true } python-snappy = { version = "0.6.1", optional = true } From 353a11f7bed422b48f9dcc0e4b63400d613931c7 Mon Sep 17 00:00:00 2001 From: Fokko Driesprong Date: Mon, 12 Dec 2022 10:35:17 +0100 Subject: [PATCH 301/642] Python: Introduce SchemaVisitorPerPrimitiveType (#6342) * Python: Introduce SchemaVisitorPerPrimitiveType Instead of having another visitor to go over the primitives I think it is nicer to have an extended schema visitor that also goes over the primitive types * Fix merge conflicts --- pyiceberg/io/pyarrow.py | 122 +++++++++++++++++----------------------- pyiceberg/schema.py | 105 ++++++++++++++++++++++++++++++++++ 2 files changed, 158 insertions(+), 69 deletions(-) diff --git a/pyiceberg/io/pyarrow.py b/pyiceberg/io/pyarrow.py index eeba4945d4..1561ccb0e9 100644 --- a/pyiceberg/io/pyarrow.py +++ b/pyiceberg/io/pyarrow.py @@ -23,7 +23,7 @@ """ import os -from functools import lru_cache, singledispatch +from functools import lru_cache from typing import ( Any, Callable, @@ -54,7 +54,7 @@ OutputFile, OutputStream, ) -from pyiceberg.schema import Schema, SchemaVisitor, visit +from pyiceberg.schema import Schema, SchemaVisitorPerPrimitiveType, visit from pyiceberg.typedef import EMPTY_DICT, Properties from pyiceberg.types import ( BinaryType, @@ -76,7 +76,9 @@ TimestampType, TimestamptzType, TimeType, + UUIDType, ) +from pyiceberg.utils.singleton import Singleton class PyArrowFile(InputFile, OutputFile): @@ -289,7 +291,21 @@ def schema_to_pyarrow(schema: Schema) -> pa.schema: return visit(schema, _ConvertToArrowSchema()) -class _ConvertToArrowSchema(SchemaVisitor[pa.DataType]): +class UuidType(pa.PyExtensionType): + """Custom type for UUID + + For more information: + https://arrow.apache.org/docs/python/extending_types.html#defining-extension-types-user-defined-types + """ + + def __init__(self): + pa.PyExtensionType.__init__(self, pa.binary(16)) + + def __reduce__(self): + return UuidType, () + + +class _ConvertToArrowSchema(SchemaVisitorPerPrimitiveType[pa.DataType], Singleton): def schema(self, _: Schema, struct_result: pa.StructType) -> pa.schema: return pa.schema(list(struct_result)) @@ -310,97 +326,65 @@ def list(self, _: ListType, element_result: pa.DataType) -> pa.DataType: def map(self, _: MapType, key_result: pa.DataType, value_result: pa.DataType) -> pa.DataType: return pa.map_(key_type=key_result, item_type=value_result) - def primitive(self, primitive: PrimitiveType) -> pa.DataType: - return _iceberg_to_pyarrow_type(primitive) - - -@singledispatch -def _iceberg_to_pyarrow_type(primitive: PrimitiveType) -> pa.DataType: - raise ValueError(f"Unknown type: {primitive}") - - -@_iceberg_to_pyarrow_type.register -def _(primitive: FixedType) -> pa.DataType: - return pa.binary(len(primitive)) - - -@_iceberg_to_pyarrow_type.register -def _(primitive: DecimalType) -> pa.DataType: - return pa.decimal128(primitive.precision, primitive.scale) - - -@_iceberg_to_pyarrow_type.register -def _(_: BooleanType) -> pa.DataType: - return pa.bool_() - - -@_iceberg_to_pyarrow_type.register -def _(_: IntegerType) -> pa.DataType: - return pa.int32() - - -@_iceberg_to_pyarrow_type.register -def _(_: LongType) -> pa.DataType: - return pa.int64() - - -@_iceberg_to_pyarrow_type.register -def _(_: FloatType) -> pa.DataType: - # 32-bit IEEE 754 floating point - return pa.float32() - + def visit_fixed(self, fixed_type: FixedType) -> pa.DataType: + return pa.binary(len(fixed_type)) -@_iceberg_to_pyarrow_type.register -def _(_: DoubleType) -> pa.DataType: - # 64-bit IEEE 754 floating point - return pa.float64() + def visit_decimal(self, decimal_type: DecimalType) -> pa.DataType: + return pa.decimal128(decimal_type.precision, decimal_type.scale) + def visit_boolean(self, _: BooleanType) -> pa.DataType: + return pa.bool_() -@_iceberg_to_pyarrow_type.register -def _(_: DateType) -> pa.DataType: - # Date encoded as an int - return pa.date32() + def visit_integer(self, _: IntegerType) -> pa.DataType: + return pa.int32() + def visit_long(self, _: LongType) -> pa.DataType: + return pa.int64() -@_iceberg_to_pyarrow_type.register -def _(_: TimeType) -> pa.DataType: - return pa.time64("us") + def visit_float(self, _: FloatType) -> pa.DataType: + # 32-bit IEEE 754 floating point + return pa.float32() + def visit_double(self, _: DoubleType) -> pa.DataType: + # 64-bit IEEE 754 floating point + return pa.float64() -@_iceberg_to_pyarrow_type.register -def _(_: TimestampType) -> pa.DataType: - return pa.timestamp(unit="us") + def visit_date(self, _: DateType) -> pa.DataType: + # Date encoded as an int + return pa.date32() + def visit_time(self, _: TimeType) -> pa.DataType: + return pa.time64("us") -@_iceberg_to_pyarrow_type.register -def _(_: TimestamptzType) -> pa.DataType: - return pa.timestamp(unit="us", tz="+00:00") + def visit_timestamp(self, _: TimestampType) -> pa.DataType: + return pa.timestamp(unit="us") + def visit_timestampz(self, _: TimestamptzType) -> pa.DataType: + return pa.timestamp(unit="us", tz="+00:00") -@_iceberg_to_pyarrow_type.register -def _(_: StringType) -> pa.DataType: - return pa.string() + def visit_string(self, _: StringType) -> pa.DataType: + return pa.string() + def visit_uuid(self, _: UUIDType) -> pa.DataType: + return UuidType() -@_iceberg_to_pyarrow_type.register -def _(_: BinaryType) -> pa.DataType: - # Variable length by default - return pa.binary() + def visit_binary(self, _: BinaryType) -> pa.DataType: + return pa.binary() def _convert_scalar(value: Any, iceberg_type: IcebergType) -> pa.scalar: if not isinstance(iceberg_type, PrimitiveType): raise ValueError(f"Expected primitive type, got: {iceberg_type}") - return pa.scalar(value).cast(_iceberg_to_pyarrow_type(iceberg_type)) + return pa.scalar(value).cast(schema_to_pyarrow(iceberg_type)) class _ConvertToArrowExpression(BoundBooleanExpressionVisitor[pc.Expression]): def visit_in(self, term: BoundTerm[pc.Expression], literals: Set[Any]) -> pc.Expression: - pyarrow_literals = pa.array(literals, type=_iceberg_to_pyarrow_type(term.ref().field.field_type)) + pyarrow_literals = pa.array(literals, type=schema_to_pyarrow(term.ref().field.field_type)) return pc.field(term.ref().field.name).isin(pyarrow_literals) def visit_not_in(self, term: BoundTerm[pc.Expression], literals: Set[Any]) -> pc.Expression: - pyarrow_literals = pa.array(literals, type=_iceberg_to_pyarrow_type(term.ref().field.field_type)) + pyarrow_literals = pa.array(literals, type=schema_to_pyarrow(term.ref().field.field_type)) return ~pc.field(term.ref().field.name).isin(pyarrow_literals) def visit_is_nan(self, term: BoundTerm[Any]) -> pc.Expression: diff --git a/pyiceberg/schema.py b/pyiceberg/schema.py index fa86845caa..ae11b5ba32 100644 --- a/pyiceberg/schema.py +++ b/pyiceberg/schema.py @@ -37,12 +37,26 @@ from pyiceberg.typedef import StructProtocol from pyiceberg.types import ( + BinaryType, + BooleanType, + DateType, + DecimalType, + DoubleType, + FixedType, + FloatType, IcebergType, + IntegerType, ListType, + LongType, MapType, NestedField, PrimitiveType, + StringType, StructType, + TimestampType, + TimestamptzType, + TimeType, + UUIDType, ) from pyiceberg.utils.iceberg_base_model import IcebergBaseModel @@ -317,6 +331,97 @@ def primitive(self, primitive: PrimitiveType) -> T: """Visit a PrimitiveType""" +class SchemaVisitorPerPrimitiveType(SchemaVisitor[T], ABC): + def primitive(self, primitive: PrimitiveType) -> T: + """Visit a PrimitiveType""" + if isinstance(primitive, FixedType): + return self.visit_fixed(primitive) + elif isinstance(primitive, DecimalType): + return self.visit_decimal(primitive) + elif isinstance(primitive, BooleanType): + return self.visit_boolean(primitive) + elif isinstance(primitive, IntegerType): + return self.visit_integer(primitive) + elif isinstance(primitive, LongType): + return self.visit_long(primitive) + elif isinstance(primitive, FloatType): + return self.visit_float(primitive) + elif isinstance(primitive, DoubleType): + return self.visit_double(primitive) + elif isinstance(primitive, DateType): + return self.visit_date(primitive) + elif isinstance(primitive, TimeType): + return self.visit_time(primitive) + elif isinstance(primitive, TimestampType): + return self.visit_timestamp(primitive) + elif isinstance(primitive, TimestamptzType): + return self.visit_timestampz(primitive) + elif isinstance(primitive, StringType): + return self.visit_string(primitive) + elif isinstance(primitive, UUIDType): + return self.visit_uuid(primitive) + elif isinstance(primitive, BinaryType): + return self.visit_binary(primitive) + else: + raise ValueError(f"Found unknown type: {primitive}") + + @abstractmethod + def visit_fixed(self, fixed_type: FixedType) -> T: + """Visit a FixedType""" + + @abstractmethod + def visit_decimal(self, decimal_type: DecimalType) -> T: + """Visit a DecimalType""" + + @abstractmethod + def visit_boolean(self, boolean_type: BooleanType) -> T: + """Visit a BooleanType""" + + @abstractmethod + def visit_integer(self, integer_type: IntegerType) -> T: + """Visit a IntegerType""" + + @abstractmethod + def visit_long(self, long_type: LongType) -> T: + """Visit a LongType""" + + @abstractmethod + def visit_float(self, float_type: FloatType) -> T: + """Visit a FloatType""" + + @abstractmethod + def visit_double(self, double_type: DoubleType) -> T: + """Visit a DoubleType""" + + @abstractmethod + def visit_date(self, date_type: DateType) -> T: + """Visit a DecimalType""" + + @abstractmethod + def visit_time(self, time_type: TimeType) -> T: + """Visit a DecimalType""" + + @abstractmethod + def visit_timestamp(self, timestamp_type: TimestampType) -> T: + """Visit a TimestampType""" + + @abstractmethod + def visit_timestampz(self, timestamptz_type: TimestamptzType) -> T: + """Visit a TimestamptzType""" + + @abstractmethod + def visit_string(self, string_type: StringType) -> T: + """Visit a StringType""" + + @abstractmethod + def visit_uuid(self, uuid_type: UUIDType) -> T: + """Visit a UUIDType""" + + @abstractmethod + def visit_binary(self, binary_ype: BinaryType) -> T: + """Visit a BinaryType""" + + @dataclass(init=True, eq=True, frozen=True) class Accessor: """An accessor for a specific position in a container that implements the StructProtocol""" From 4ef0c0aed0004f868c8210eba7d8d025029236e5 Mon Sep 17 00:00:00 2001 From: Fokko Driesprong Date: Mon, 12 Dec 2022 11:17:00 +0100 Subject: [PATCH 302/642] Python: Add missing types (#6409) --- pyiceberg/io/pyarrow.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pyiceberg/io/pyarrow.py b/pyiceberg/io/pyarrow.py index 1561ccb0e9..f3adfd0f6c 100644 --- a/pyiceberg/io/pyarrow.py +++ b/pyiceberg/io/pyarrow.py @@ -298,10 +298,10 @@ class UuidType(pa.PyExtensionType): https://arrow.apache.org/docs/python/extending_types.html#defining-extension-types-user-defined-types """ - def __init__(self): + def __init__(self) -> None: pa.PyExtensionType.__init__(self, pa.binary(16)) - def __reduce__(self): + def __reduce__(self) -> Tuple[pa.PyExtensionType, Tuple[Any, ...]]: return UuidType, () From 15905a33ed42f4cbc26d1100493e482c65917fe2 Mon Sep 17 00:00:00 2001 From: Ruben van de Geer <37443208+rubenvdg@users.noreply.github.com> Date: Tue, 13 Dec 2022 14:53:39 +0100 Subject: [PATCH 303/642] Python: Textual improvements and cleanup of docs (#6413) --- CONTRIBUTING.md | 135 ------------------------------------ README.md | 36 +--------- mkdocs/docs/contributing.md | 8 +-- 3 files changed, 7 insertions(+), 172 deletions(-) delete mode 100644 CONTRIBUTING.md diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md deleted file mode 100644 index ee2d23a82b..0000000000 --- a/CONTRIBUTING.md +++ /dev/null @@ -1,135 +0,0 @@ - - -# Contributing to the Iceberg Python library - -For the development, we use poetry for packing and dependency management. You can install this using: - -```bash -pip install poetry -``` - -If you have an older version of pip and virtualenv you need to update these: - -```bash -pip install --upgrade virtualenv pip -``` - -To get started, you can run `make install`, which will install poetry and it will install all the dependencies of the Iceberg library. This will also install the development dependencies. If you don't want to do this, you need to install using `poetry install --no-dev`. - -If you want to install the library on the host, you can simply run `pip3 install -e .`. If you wish to use a virtual environment, you can run `poetry shell`. Poetry will open up a virtual environment with all the dependencies set. - -To set up IDEA with Poetry ([also on Loom](https://www.loom.com/share/6d36464d45f244729d91003e7f671fd2)): - -- Open up the Python project in IntelliJ -- Make sure that you're on latest master (that includes Poetry) -- Go to File -> Project Structure (⌘;) -- Go to Platform Settings -> SDKs -- Click the + sign -> Add Python SDK -- Select Poetry Environment from the left hand side bar and hit OK -- It can take some time to download all the dependencies based on your internet -- Go to Project Settings -> Project -- Select the Poetry SDK from the SDK dropdown, and click OK - -For IDEA ≤2021 you need to install the [Poetry integration as a plugin](https://plugins.jetbrains.com/plugin/14307-poetry/). - -Now you're set using Poetry, and all the tests will run in Poetry, and you'll have syntax highlighting in the pyproject.toml to indicate stale dependencies. - -## Linting - -We rely on `pre-commit` to apply autoformatting and linting: - -```bash -make lint -``` - -Pre-commit will automatically fix the violations such as import orders, formatting etc. Pylint errors you need to fix yourself. - -In contrast to the name suggest, it doesn't run the checks on the commit. If this is something that you like, you can set this up by running `pre-commit install`. - -You can bump the integrations to the latest version using `pre-commit autoupdate`. This will check if there is a newer version of `{black,mypy,isort,...}` and update the yaml. - -## Testing - -For Python, we use pytest in combination with coverage to maintain 90% code coverage. - -```bash -make test -``` - -By default we ignore the s3 tests that require minio to be running. To run this suite, we can run: - -```bash -make test-s3 -``` - -To pass additional arguments to pytest, you can use `PYTEST_ARGS`. - -*Run pytest in verbose mode* - -```sh -make test PYTEST_ARGS="-v" -``` - -*Run pytest with pdb enabled* - -```sh -make test PYTEST_ARGS="--pdb" -``` - -To see all available pytest arguments, run `make test PYTEST_ARGS="--help"`. - -# Code standards - -Below are the formalized conventions that we adhere to in the PyIceberg project. The goal of this is to have a common agreement on how to evolve the codebase, but also using it as guidelines for newcomers to the project. - -## API Compatibility - -We try to keep the Python public API compatible across versions. The Python official [PEP-8](https://peps.python.org/pep-0008/) defines Public methods as: _Public attributes should have no leading underscores_. This means not removing any methods without any notice, or removing or renaming any existing parameters. Adding new optional parameters is okay. - -If you want to remove a method, please add a deprecation notice by annotating the function using `@deprecated`: - -```python -from pyiceberg.utils.deprecated import deprecated - - -@deprecated( - deprecated_in="0.1.0", - removed_in="0.2.0", - help_message="Please use load_something_else() instead", -) -def load_something(): - pass -``` - -Which will warn: - -``` -Call to load_something, deprecated in 0.1.0, will be removed in 0.2.0. Please use load_something_else() instead. -``` - -## Type annotations - -For the type annotation we currently rely on the `Typing` package that comes with Python. - -Since we're supporting from Python 3.8 onwards, we can't use the [type hints from the standard collections](https://peps.python.org/pep-0585/). - -## Third party libraries - -Since we expect PyIceberg to be integrated into the Python ecosystem, we want to be hesitant with the use of third party packages. Adding a lot of packages makes the library heavyweight, and causes incompatibilities with other projects if they use a different version of the library. Also, big libraries such as `s3fs`, `pyarrow`, `thrift` should be optional to avoid downloading everything, while not being sure if is actually being used. diff --git a/README.md b/README.md index d988be2120..8fe2f3738d 100644 --- a/README.md +++ b/README.md @@ -17,39 +17,9 @@ # Iceberg Python -pyiceberg is a python library for programmatic access to iceberg table metadata as well as to table data in iceberg format. It is a Python implementation of [iceberg table spec](https://iceberg.apache.org/spec/). Documentation is available at [https://py.iceberg.apache.org/](https://py.iceberg.apache.org/). +PyIceberg is a Python library for programmatic access to Iceberg table metadata as well as to table data in Iceberg format. It is a Python implementation of the [Iceberg table spec](https://iceberg.apache.org/spec/). -## Getting Started - -pyiceberg is currently in development, for development and testing purposes the best way to install the library is to perform the following steps: - -``` -git clone https://github.com/apache/iceberg.git -cd iceberg/python -pip install -e . # this step requires pip >= 21.3.0 -``` - -## Development - -Development is made easy using [Poetry](https://python-poetry.org/docs/#installation). Poetry provides virtual environments for development: - -```bash -poetry shell -make install -make test -``` - -For more information, please refer to the [Manage environments](https://python-poetry.org/docs/managing-environments/) section of Poetry. - -## Testing - -Testing is done using Poetry: - -``` -poetry install -E pyarrow -poetry run pytest -``` - -## Get in Touch +The documentation is available at [https://py.iceberg.apache.org/](https://py.iceberg.apache.org/). +# Get in Touch - [Iceberg community](https://iceberg.apache.org/community/) diff --git a/mkdocs/docs/contributing.md b/mkdocs/docs/contributing.md index a87939935f..6f2236028d 100644 --- a/mkdocs/docs/contributing.md +++ b/mkdocs/docs/contributing.md @@ -19,7 +19,7 @@ # Contributing to the Iceberg Python library -For the development, poetry is used for packing and dependency management. You can install this using: +For the development, Poetry is used for packing and dependency management. You can install this using: ```bash pip install poetry @@ -31,7 +31,7 @@ If you have an older version of pip and virtualenv you need to update these: pip install --upgrade virtualenv pip ``` -To get started, you can run `make install`, which will install poetry and it will install all the dependencies of the Iceberg library. This will also install the development dependencies. If you don't want to do this, you need to install using `poetry install --no-dev`. +To get started, you can run `make install`, which installs Poetry and all the dependencies of the Iceberg library. This also installs the development dependencies. If you don't want to install the development dependencies, you need to install using `poetry install --no-dev`. If you want to install the library on the host, you can simply run `pip3 install -e .`. If you wish to use a virtual environment, you can run `poetry shell`. Poetry will open up a virtual environment with all the dependencies set. @@ -73,7 +73,7 @@ For Python, `pytest` is used a testing framework in combination with `coverage` make test ``` -By default, s3 tests are ignored because that require minio to be running. To run the s3 suite: +By default, S3 tests are ignored because that require minio to be running. To run the S3 suite: ```bash make test-s3 @@ -101,7 +101,7 @@ Below are the formalized conventions that we adhere to in the PyIceberg project. ## API Compatibility -It is important to keep the Python public API compatible across versions. The Python official [PEP-8](https://peps.python.org/pep-0008/) defines Public methods as: _Public attributes should have no leading underscores_. This means not removing any methods without any notice, or removing or renaming any existing parameters. Adding new optional parameters is okay. +It is important to keep the Python public API compatible across versions. The Python official [PEP-8](https://peps.python.org/pep-0008/) defines public methods as: _Public attributes should have no leading underscores_. This means not removing any methods without any notice, or removing or renaming any existing parameters. Adding new optional parameters is okay. If you want to remove a method, please add a deprecation notice by annotating the function using `@deprecated`: From 237634a3396c689d8cf4cb94674ed5550484366e Mon Sep 17 00:00:00 2001 From: Fokko Driesprong Date: Fri, 16 Dec 2022 02:13:18 +0100 Subject: [PATCH 304/642] Python: Add pyparsing (#6439) This one was missing and was being pulled in transitively I presume --- poetry.lock | 407 ++++++++++++++++++++++--------------------------- pyproject.toml | 2 + 2 files changed, 188 insertions(+), 221 deletions(-) diff --git a/poetry.lock b/poetry.lock index 7ea806e86b..4a77b4756c 100644 --- a/poetry.lock +++ b/poetry.lock @@ -34,7 +34,7 @@ multidict = ">=4.5,<7.0" yarl = ">=1.0,<2.0" [package.extras] -speedups = ["Brotli", "aiodns", "cchardet"] +speedups = ["aiodns", "brotli", "cchardet"] [[package]] name = "aioitertools" @@ -78,7 +78,7 @@ python-versions = ">=3.5" dev = ["cloudpickle", "coverage[toml] (>=5.0.2)", "furo", "hypothesis", "mypy (>=0.900,!=0.940)", "pre-commit", "pympler", "pytest (>=4.3.0)", "pytest-mypy-plugins", "sphinx", "sphinx-notfound-page", "zope.interface"] docs = ["furo", "sphinx", "sphinx-notfound-page", "zope.interface"] tests = ["cloudpickle", "coverage[toml] (>=5.0.2)", "hypothesis", "mypy (>=0.900,!=0.940)", "pympler", "pytest (>=4.3.0)", "pytest-mypy-plugins", "zope.interface"] -tests-no-zope = ["cloudpickle", "coverage[toml] (>=5.0.2)", "hypothesis", "mypy (>=0.900,!=0.940)", "pympler", "pytest (>=4.3.0)", "pytest-mypy-plugins"] +tests_no_zope = ["cloudpickle", "coverage[toml] (>=5.0.2)", "hypothesis", "mypy (>=0.900,!=0.940)", "pympler", "pytest (>=4.3.0)", "pytest-mypy-plugins"] [[package]] name = "boto3" @@ -168,7 +168,7 @@ optional = false python-versions = ">=3.6.0" [package.extras] -unicode-backport = ["unicodedata2"] +unicode_backport = ["unicodedata2"] [[package]] name = "click" @@ -287,15 +287,15 @@ zstandard = ["zstandard"] [[package]] name = "filelock" -version = "3.8.0" +version = "3.8.2" description = "A platform independent file lock." category = "dev" optional = false python-versions = ">=3.7" [package.extras] -docs = ["furo (>=2022.6.21)", "sphinx (>=5.1.1)", "sphinx-autodoc-typehints (>=1.19.1)"] -testing = ["covdefaults (>=2.2)", "coverage (>=6.4.2)", "pytest (>=7.1.2)", "pytest-cov (>=3)", "pytest-timeout (>=2.1)"] +docs = ["furo (>=2022.9.29)", "sphinx (>=5.3)", "sphinx-autodoc-typehints (>=1.19.5)"] +testing = ["covdefaults (>=2.2.2)", "coverage (>=6.5)", "pytest (>=7.2)", "pytest-cov (>=4)", "pytest-timeout (>=2.1)"] [[package]] name = "frozenlist" @@ -338,7 +338,7 @@ tqdm = ["tqdm"] [[package]] name = "identify" -version = "2.5.9" +version = "2.5.10" description = "File identification library for Python" category = "dev" optional = false @@ -464,7 +464,7 @@ xray = ["aws-xray-sdk (>=0.93,!=0.96)", "setuptools"] [[package]] name = "multidict" -version = "6.0.2" +version = "6.0.3" description = "multidict implementation" category = "main" optional = true @@ -478,9 +478,6 @@ category = "dev" optional = false python-versions = ">=2.7,!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,!=3.4.*,!=3.5.*,!=3.6.*" -[package.dependencies] -setuptools = "*" - [[package]] name = "numpy" version = "1.23.5" @@ -491,14 +488,11 @@ python-versions = ">=3.8" [[package]] name = "packaging" -version = "21.3" +version = "22.0" description = "Core utilities for Python packages" category = "dev" optional = false -python-versions = ">=3.6" - -[package.dependencies] -pyparsing = ">=2.0.2,<3.0.5 || >3.0.5" +python-versions = ">=3.7" [[package]] name = "pandas" @@ -533,7 +527,7 @@ tomli = {version = ">=1.1.0", markers = "python_version < \"3.11\""} [[package]] name = "platformdirs" -version = "2.5.4" +version = "2.6.0" description = "A small Python package for determining appropriate platform-specific dirs, e.g. a \"user data dir\"." category = "dev" optional = false @@ -620,7 +614,7 @@ plugins = ["importlib-metadata"] name = "pyparsing" version = "3.0.9" description = "pyparsing module - Classes and methods to define and execute parsing grammars" -category = "dev" +category = "main" optional = false python-versions = ">=3.6.8" @@ -715,7 +709,7 @@ urllib3 = ">=1.21.1,<1.27" [package.extras] socks = ["PySocks (>=1.5.6,!=1.5.7)"] -use-chardet-on-py3 = ["chardet (>=3.0.2,<6)"] +use_chardet_on_py3 = ["chardet (>=3.0.2,<6)"] [[package]] name = "requests-mock" @@ -797,19 +791,6 @@ botocore = ">=1.12.36,<2.0a.0" [package.extras] crt = ["botocore[crt] (>=1.20.29,<2.0a.0)"] -[[package]] -name = "setuptools" -version = "65.6.3" -description = "Easily download, build, install, upgrade, and uninstall Python packages" -category = "dev" -optional = false -python-versions = ">=3.7" - -[package.extras] -docs = ["furo", "jaraco.packaging (>=9)", "jaraco.tidelift (>=1.4)", "pygments-github-lexers (==0.0.5)", "rst.linker (>=1.9)", "sphinx (>=3.5)", "sphinx-favicon", "sphinx-hoverxref (<2)", "sphinx-inline-tabs", "sphinx-notfound-page (==0.8.3)", "sphinx-reredirects", "sphinxcontrib-towncrier"] -testing = ["build[virtualenv]", "filelock (>=3.4.0)", "flake8 (<5)", "flake8-2020", "ini2toml[lite] (>=0.9)", "jaraco.envs (>=2.2)", "jaraco.path (>=3.2.0)", "pip (>=19.1)", "pip-run (>=8.8)", "pytest (>=6)", "pytest-black (>=0.3.7)", "pytest-checkdocs (>=2.4)", "pytest-cov", "pytest-enabler (>=1.3)", "pytest-flake8", "pytest-mypy (>=0.9.1)", "pytest-perf", "pytest-timeout", "pytest-xdist", "tomli-w (>=1.0.0)", "virtualenv (>=13.0.0)", "wheel"] -testing-integration = ["build[virtualenv]", "filelock (>=3.4.0)", "jaraco.envs (>=2.2)", "jaraco.path (>=3.2.0)", "pytest", "pytest-enabler", "pytest-xdist", "tomli", "virtualenv (>=13.0.0)", "wheel"] - [[package]] name = "six" version = "1.16.0" @@ -881,7 +862,7 @@ socks = ["PySocks (>=1.5.6,!=1.5.7,<2.0)"] [[package]] name = "virtualenv" -version = "20.17.0" +version = "20.17.1" description = "Virtual Python Environment builder" category = "dev" optional = false @@ -928,7 +909,7 @@ python-versions = ">=3.4" [[package]] name = "yarl" -version = "1.8.1" +version = "1.8.2" description = "Yet another URL library" category = "main" optional = true @@ -976,7 +957,7 @@ snappy = ["python-snappy"] [metadata] lock-version = "1.1" python-versions = "^3.8" -content-hash = "5c058a9e9a389a4f376baa1d621659fc842293d2a63bb800b111fdd2f6944ef1" +content-hash = "5998c3a4d17fc291d61bead0350b2bb830b112fdafea26f6b2f1e456876733db" [metadata.files] aiobotocore = [ @@ -1080,14 +1061,8 @@ aiosignal = [ {file = "aiosignal-1.3.1-py3-none-any.whl", hash = "sha256:f8376fb07dd1e86a584e4fcdec80b36b7f81aac666ebc724e2c090300dd83b17"}, {file = "aiosignal-1.3.1.tar.gz", hash = "sha256:54cd96e15e1649b75d6c87526a6ff0b6c1b0dd3459f43d9ca11d48c339b68cfc"}, ] -async-timeout = [ - {file = "async-timeout-4.0.2.tar.gz", hash = "sha256:2163e1640ddb52b7a8c80d0a67a08587e5d245cc9c553a74a847056bc2976b15"}, - {file = "async_timeout-4.0.2-py3-none-any.whl", hash = "sha256:8ca1e4fcf50d07413d66d1a5e416e42cfdf5851c981d679a09851a6853383b3c"}, -] -attrs = [ - {file = "attrs-22.1.0-py2.py3-none-any.whl", hash = "sha256:86efa402f67bf2df34f51a335487cf46b1ec130d02b8d39fd248abfd30da551c"}, - {file = "attrs-22.1.0.tar.gz", hash = "sha256:29adc2665447e5191d0e7c568fde78b21f9672d344281d0c6e1ab085429b22b6"}, -] +async-timeout = [] +attrs = [] boto3 = [ {file = "boto3-1.24.59-py3-none-any.whl", hash = "sha256:34ab44146a2c4e7f4e72737f4b27e6eb5e0a7855c2f4599e3d9199b6a0a2d575"}, {file = "boto3-1.24.59.tar.gz", hash = "sha256:a50b4323f9579cfe22fcf5531fbd40b567d4d74c1adce06aeb5c95fce2a6fb40"}, @@ -1174,22 +1149,13 @@ cfgv = [ {file = "cfgv-3.3.1-py2.py3-none-any.whl", hash = "sha256:c6a0883f3917a037485059700b9e75da2464e6c27051014ad85ba6aaa5884426"}, {file = "cfgv-3.3.1.tar.gz", hash = "sha256:f5a830efb9ce7a445376bb66ec94c638a9787422f96264c98edc6bdeed8ab736"}, ] -charset-normalizer = [ - {file = "charset-normalizer-2.1.1.tar.gz", hash = "sha256:5a3d016c7c547f69d6f81fb0db9449ce888b418b5b9952cc5e6e66843e9dd845"}, - {file = "charset_normalizer-2.1.1-py3-none-any.whl", hash = "sha256:83e9a75d1911279afd89352c68b45348559d1fc0506b054b346651b5e7fee29f"}, -] -click = [ - {file = "click-8.1.3-py3-none-any.whl", hash = "sha256:bb4d8133cb15a609f44e8213d9b391b0809795062913b383c62be0ee95b1db48"}, - {file = "click-8.1.3.tar.gz", hash = "sha256:7682dc8afb30297001674575ea00d1814d808d6a36af415a82bd481d37ba7b8e"}, -] +charset-normalizer = [] +click = [] colorama = [ {file = "colorama-0.4.6-py2.py3-none-any.whl", hash = "sha256:4f1d9991f5acc0ca119f9d443620b77f9d6b33703e51011c16baf57afb285fc6"}, {file = "colorama-0.4.6.tar.gz", hash = "sha256:08695f5cb7ed6e0531a20572697297273c47b8cae5a63ffc6d6ed5c201be6e44"}, ] -commonmark = [ - {file = "commonmark-0.9.1-py2.py3-none-any.whl", hash = "sha256:da2f38c92590f83de410ba1a3cbceafbc74fee9def35f9251ba9a971d6d66fd9"}, - {file = "commonmark-0.9.1.tar.gz", hash = "sha256:452f9dc859be7f06631ddcb328b6919c67984aca654e5fefb3914d54691aed60"}, -] +commonmark = [] coverage = [ {file = "coverage-6.5.0-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:ef8674b0ee8cc11e2d574e3e2998aea5df5ab242e012286824ea3c6970580e53"}, {file = "coverage-6.5.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:784f53ebc9f3fd0e2a3f6a78b2be1bd1f5575d7863e10c6e12504f240fd06660"}, @@ -1355,8 +1321,8 @@ fastavro = [ {file = "fastavro-1.7.0.tar.gz", hash = "sha256:4b1205f46489b4032d3155c1ab44d9824be0c7454df98d3a5bd22b78b98f23c8"}, ] filelock = [ - {file = "filelock-3.8.0-py3-none-any.whl", hash = "sha256:617eb4e5eedc82fc5f47b6d61e4d11cb837c56cb4544e39081099fa17ad109d4"}, - {file = "filelock-3.8.0.tar.gz", hash = "sha256:55447caa666f2198c5b6b13a26d2084d26fa5b115c00d065664b2124680c4edc"}, + {file = "filelock-3.8.2-py3-none-any.whl", hash = "sha256:8df285554452285f79c035efb0c861eb33a4bcfa5b7a137016e32e6a90f9792c"}, + {file = "filelock-3.8.2.tar.gz", hash = "sha256:7565f628ea56bfcd8e54e42bdc55da899c85c1abfe1b5bcfd147e9188cebb3b2"}, ] frozenlist = [ {file = "frozenlist-1.3.3-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:ff8bf625fe85e119553b5383ba0fb6aa3d0ec2ae980295aaefa552374926b3f4"}, @@ -1439,8 +1405,8 @@ fsspec = [ {file = "fsspec-2022.10.0.tar.gz", hash = "sha256:cb6092474e90487a51de768170f3afa50ca8982c26150a59072b16433879ff1d"}, ] identify = [ - {file = "identify-2.5.9-py2.py3-none-any.whl", hash = "sha256:a390fb696e164dbddb047a0db26e57972ae52fbd037ae68797e5ae2f4492485d"}, - {file = "identify-2.5.9.tar.gz", hash = "sha256:906036344ca769539610436e40a684e170c3648b552194980bb7b617a8daeb9f"}, + {file = "identify-2.5.10-py2.py3-none-any.whl", hash = "sha256:fb7c2feaeca6976a3ffa31ec3236a6911fbc51aec9acc111de2aed99f244ade2"}, + {file = "identify-2.5.10.tar.gz", hash = "sha256:dce9e31fee7dbc45fea36a9e855c316b8fbf807e65a862f160840bb5a2bf5dfd"}, ] idna = [ {file = "idna-3.4-py3-none-any.whl", hash = "sha256:90b77e79eaa3eba6de819a0c442c0b4ceefc341a7a2ab77d7562bf49f425c5c2"}, @@ -1458,10 +1424,7 @@ jinja2 = [ {file = "Jinja2-3.1.2-py3-none-any.whl", hash = "sha256:6088930bfe239f0e6710546ab9c19c9ef35e29792895fed6e6e31a023a182a61"}, {file = "Jinja2-3.1.2.tar.gz", hash = "sha256:31351a702a408a9e7595a8fc6150fc3f43bb6bf7e319770cbc0db9df9437e852"}, ] -jmespath = [ - {file = "jmespath-1.0.1-py3-none-any.whl", hash = "sha256:02e2e4cc71b5bcab88332eebf907519190dd9e6e82107fa7f83b1003a6252980"}, - {file = "jmespath-1.0.1.tar.gz", hash = "sha256:90261b206d6defd58fdd5e85f478bf633a2901798906be2ad389150c5c60edbe"}, -] +jmespath = [] markupsafe = [ {file = "MarkupSafe-2.1.1-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:86b1f75c4e7c2ac2ccdaec2b9022845dbb81880ca318bb7a0a01fbf7813e3812"}, {file = "MarkupSafe-2.1.1-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:f121a1420d4e173a5d96e47e9a0c0dcff965afdf1626d28de1460815f7c4ee7a"}, @@ -1545,65 +1508,80 @@ moto = [ {file = "moto-4.0.11.tar.gz", hash = "sha256:a6388de4a746e0b509286e1d7e70f86900b4f69ec65f6c92c47e570f95d05b14"}, ] multidict = [ - {file = "multidict-6.0.2-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:0b9e95a740109c6047602f4db4da9949e6c5945cefbad34a1299775ddc9a62e2"}, - {file = "multidict-6.0.2-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:ac0e27844758d7177989ce406acc6a83c16ed4524ebc363c1f748cba184d89d3"}, - {file = "multidict-6.0.2-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:041b81a5f6b38244b34dc18c7b6aba91f9cdaf854d9a39e5ff0b58e2b5773b9c"}, - {file = "multidict-6.0.2-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:5fdda29a3c7e76a064f2477c9aab1ba96fd94e02e386f1e665bca1807fc5386f"}, - {file = "multidict-6.0.2-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:3368bf2398b0e0fcbf46d85795adc4c259299fec50c1416d0f77c0a843a3eed9"}, - {file = "multidict-6.0.2-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:f4f052ee022928d34fe1f4d2bc743f32609fb79ed9c49a1710a5ad6b2198db20"}, - {file = "multidict-6.0.2-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:225383a6603c086e6cef0f2f05564acb4f4d5f019a4e3e983f572b8530f70c88"}, - {file = "multidict-6.0.2-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:50bd442726e288e884f7be9071016c15a8742eb689a593a0cac49ea093eef0a7"}, - {file = "multidict-6.0.2-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:47e6a7e923e9cada7c139531feac59448f1f47727a79076c0b1ee80274cd8eee"}, - {file = "multidict-6.0.2-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:0556a1d4ea2d949efe5fd76a09b4a82e3a4a30700553a6725535098d8d9fb672"}, - {file = "multidict-6.0.2-cp310-cp310-musllinux_1_1_ppc64le.whl", hash = "sha256:626fe10ac87851f4cffecee161fc6f8f9853f0f6f1035b59337a51d29ff3b4f9"}, - {file = "multidict-6.0.2-cp310-cp310-musllinux_1_1_s390x.whl", hash = "sha256:8064b7c6f0af936a741ea1efd18690bacfbae4078c0c385d7c3f611d11f0cf87"}, - {file = "multidict-6.0.2-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:2d36e929d7f6a16d4eb11b250719c39560dd70545356365b494249e2186bc389"}, - {file = "multidict-6.0.2-cp310-cp310-win32.whl", hash = "sha256:fcb91630817aa8b9bc4a74023e4198480587269c272c58b3279875ed7235c293"}, - {file = "multidict-6.0.2-cp310-cp310-win_amd64.whl", hash = "sha256:8cbf0132f3de7cc6c6ce00147cc78e6439ea736cee6bca4f068bcf892b0fd658"}, - {file = "multidict-6.0.2-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:05f6949d6169878a03e607a21e3b862eaf8e356590e8bdae4227eedadacf6e51"}, - {file = "multidict-6.0.2-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:e2c2e459f7050aeb7c1b1276763364884595d47000c1cddb51764c0d8976e608"}, - {file = "multidict-6.0.2-cp37-cp37m-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:d0509e469d48940147e1235d994cd849a8f8195e0bca65f8f5439c56e17872a3"}, - {file = "multidict-6.0.2-cp37-cp37m-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:514fe2b8d750d6cdb4712346a2c5084a80220821a3e91f3f71eec11cf8d28fd4"}, - {file = "multidict-6.0.2-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:19adcfc2a7197cdc3987044e3f415168fc5dc1f720c932eb1ef4f71a2067e08b"}, - {file = "multidict-6.0.2-cp37-cp37m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:b9d153e7f1f9ba0b23ad1568b3b9e17301e23b042c23870f9ee0522dc5cc79e8"}, - {file = "multidict-6.0.2-cp37-cp37m-musllinux_1_1_aarch64.whl", hash = "sha256:aef9cc3d9c7d63d924adac329c33835e0243b5052a6dfcbf7732a921c6e918ba"}, - {file = "multidict-6.0.2-cp37-cp37m-musllinux_1_1_i686.whl", hash = "sha256:4571f1beddff25f3e925eea34268422622963cd8dc395bb8778eb28418248e43"}, - {file = "multidict-6.0.2-cp37-cp37m-musllinux_1_1_ppc64le.whl", hash = "sha256:d48b8ee1d4068561ce8033d2c344cf5232cb29ee1a0206a7b828c79cbc5982b8"}, - {file = "multidict-6.0.2-cp37-cp37m-musllinux_1_1_s390x.whl", hash = "sha256:45183c96ddf61bf96d2684d9fbaf6f3564d86b34cb125761f9a0ef9e36c1d55b"}, - {file = "multidict-6.0.2-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:75bdf08716edde767b09e76829db8c1e5ca9d8bb0a8d4bd94ae1eafe3dac5e15"}, - {file = "multidict-6.0.2-cp37-cp37m-win32.whl", hash = "sha256:a45e1135cb07086833ce969555df39149680e5471c04dfd6a915abd2fc3f6dbc"}, - {file = "multidict-6.0.2-cp37-cp37m-win_amd64.whl", hash = "sha256:6f3cdef8a247d1eafa649085812f8a310e728bdf3900ff6c434eafb2d443b23a"}, - {file = "multidict-6.0.2-cp38-cp38-macosx_10_9_universal2.whl", hash = "sha256:0327292e745a880459ef71be14e709aaea2f783f3537588fb4ed09b6c01bca60"}, - {file = "multidict-6.0.2-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:e875b6086e325bab7e680e4316d667fc0e5e174bb5611eb16b3ea121c8951b86"}, - {file = "multidict-6.0.2-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:feea820722e69451743a3d56ad74948b68bf456984d63c1a92e8347b7b88452d"}, - {file = "multidict-6.0.2-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:9cc57c68cb9139c7cd6fc39f211b02198e69fb90ce4bc4a094cf5fe0d20fd8b0"}, - {file = "multidict-6.0.2-cp38-cp38-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:497988d6b6ec6ed6f87030ec03280b696ca47dbf0648045e4e1d28b80346560d"}, - {file = "multidict-6.0.2-cp38-cp38-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:89171b2c769e03a953d5969b2f272efa931426355b6c0cb508022976a17fd376"}, - {file = "multidict-6.0.2-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:684133b1e1fe91eda8fa7447f137c9490a064c6b7f392aa857bba83a28cfb693"}, - {file = "multidict-6.0.2-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:fd9fc9c4849a07f3635ccffa895d57abce554b467d611a5009ba4f39b78a8849"}, - {file = "multidict-6.0.2-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:e07c8e79d6e6fd37b42f3250dba122053fddb319e84b55dd3a8d6446e1a7ee49"}, - {file = "multidict-6.0.2-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:4070613ea2227da2bfb2c35a6041e4371b0af6b0be57f424fe2318b42a748516"}, - {file = "multidict-6.0.2-cp38-cp38-musllinux_1_1_ppc64le.whl", hash = "sha256:47fbeedbf94bed6547d3aa632075d804867a352d86688c04e606971595460227"}, - {file = "multidict-6.0.2-cp38-cp38-musllinux_1_1_s390x.whl", hash = "sha256:5774d9218d77befa7b70d836004a768fb9aa4fdb53c97498f4d8d3f67bb9cfa9"}, - {file = "multidict-6.0.2-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:2957489cba47c2539a8eb7ab32ff49101439ccf78eab724c828c1a54ff3ff98d"}, - {file = "multidict-6.0.2-cp38-cp38-win32.whl", hash = "sha256:e5b20e9599ba74391ca0cfbd7b328fcc20976823ba19bc573983a25b32e92b57"}, - {file = "multidict-6.0.2-cp38-cp38-win_amd64.whl", hash = "sha256:8004dca28e15b86d1b1372515f32eb6f814bdf6f00952699bdeb541691091f96"}, - {file = "multidict-6.0.2-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:2e4a0785b84fb59e43c18a015ffc575ba93f7d1dbd272b4cdad9f5134b8a006c"}, - {file = "multidict-6.0.2-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:6701bf8a5d03a43375909ac91b6980aea74b0f5402fbe9428fc3f6edf5d9677e"}, - {file = "multidict-6.0.2-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:a007b1638e148c3cfb6bf0bdc4f82776cef0ac487191d093cdc316905e504071"}, - {file = "multidict-6.0.2-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:07a017cfa00c9890011628eab2503bee5872f27144936a52eaab449be5eaf032"}, - {file = "multidict-6.0.2-cp39-cp39-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:c207fff63adcdf5a485969131dc70e4b194327666b7e8a87a97fbc4fd80a53b2"}, - {file = "multidict-6.0.2-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:373ba9d1d061c76462d74e7de1c0c8e267e9791ee8cfefcf6b0b2495762c370c"}, - {file = "multidict-6.0.2-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:bfba7c6d5d7c9099ba21f84662b037a0ffd4a5e6b26ac07d19e423e6fdf965a9"}, - {file = "multidict-6.0.2-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:19d9bad105dfb34eb539c97b132057a4e709919ec4dd883ece5838bcbf262b80"}, - {file = "multidict-6.0.2-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:de989b195c3d636ba000ee4281cd03bb1234635b124bf4cd89eeee9ca8fcb09d"}, - {file = "multidict-6.0.2-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:7c40b7bbece294ae3a87c1bc2abff0ff9beef41d14188cda94ada7bcea99b0fb"}, - {file = "multidict-6.0.2-cp39-cp39-musllinux_1_1_ppc64le.whl", hash = "sha256:d16cce709ebfadc91278a1c005e3c17dd5f71f5098bfae1035149785ea6e9c68"}, - {file = "multidict-6.0.2-cp39-cp39-musllinux_1_1_s390x.whl", hash = "sha256:a2c34a93e1d2aa35fbf1485e5010337c72c6791407d03aa5f4eed920343dd360"}, - {file = "multidict-6.0.2-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:feba80698173761cddd814fa22e88b0661e98cb810f9f986c54aa34d281e4937"}, - {file = "multidict-6.0.2-cp39-cp39-win32.whl", hash = "sha256:23b616fdc3c74c9fe01d76ce0d1ce872d2d396d8fa8e4899398ad64fb5aa214a"}, - {file = "multidict-6.0.2-cp39-cp39-win_amd64.whl", hash = "sha256:4bae31803d708f6f15fd98be6a6ac0b6958fcf68fda3c77a048a4f9073704aae"}, - {file = "multidict-6.0.2.tar.gz", hash = "sha256:5ff3bd75f38e4c43f1f470f2df7a4d430b821c4ce22be384e1459cb57d6bb013"}, + {file = "multidict-6.0.3-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:73009ea04205966d47e16d98686ac5c438af23a1bb30b48a2c5da3423ec9ce37"}, + {file = "multidict-6.0.3-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:8b92a9f3ab904397a33b193000dc4de7318ea175c4c460a1e154c415f9008e3d"}, + {file = "multidict-6.0.3-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:578bfcb16f4b8675ef71b960c00f174b0426e0eeb796bab6737389d8288eb827"}, + {file = "multidict-6.0.3-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:f1650ea41c408755da5eed52ac6ccbc8938ccc3e698d81e6f6a1be02ff2a0945"}, + {file = "multidict-6.0.3-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:d52442e7c951e4c9ee591d6047706e66923d248d83958bbf99b8b19515fffaef"}, + {file = "multidict-6.0.3-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:ad7d66422b9cc51125509229693d27e18c08f2dea3ac9de408d821932b1b3759"}, + {file = "multidict-6.0.3-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:6cd14e61f0da2a2cfb9fe05bfced2a1ed7063ce46a7a8cd473be4973de9a7f91"}, + {file = "multidict-6.0.3-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:190626ced82d4cc567a09e7346340d380154a493bac6905e0095d8158cdf1e38"}, + {file = "multidict-6.0.3-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:791458a1f7d1b4ab3bd9e93e0dcd1d59ef7ee9aa051dcd1ea030e62e49b923fd"}, + {file = "multidict-6.0.3-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:b46e79a9f4db53897d17bc64a39d1c7c2be3e3d4f8dba6d6730a2b13ddf0f986"}, + {file = "multidict-6.0.3-cp310-cp310-musllinux_1_1_ppc64le.whl", hash = "sha256:e4a095e18847c12ec20e55326ab8782d9c2d599400a3a2f174fab4796875d0e2"}, + {file = "multidict-6.0.3-cp310-cp310-musllinux_1_1_s390x.whl", hash = "sha256:fb6c3dc3d65014d2c782f5acf0b3ba14e639c6c33d3ed8932ead76b9080b3544"}, + {file = "multidict-6.0.3-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:3541882266247c7cd3dba78d6ef28dbe704774df60c9e4231edaa4493522e614"}, + {file = "multidict-6.0.3-cp310-cp310-win32.whl", hash = "sha256:67090b17a0a5be5704fd109f231ee73cefb1b3802d41288d6378b5df46ae89ba"}, + {file = "multidict-6.0.3-cp310-cp310-win_amd64.whl", hash = "sha256:36df958b15639e40472adaa4f0c2c7828fe680f894a6b48c4ce229f59a6a798b"}, + {file = "multidict-6.0.3-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:5b51969503709415a35754954c2763f536a70b8bf7360322b2edb0c0a44391f6"}, + {file = "multidict-6.0.3-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:24e8d513bfcaadc1f8b0ebece3ff50961951c54b07d5a775008a882966102418"}, + {file = "multidict-6.0.3-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:d325d61cac602976a5d47b19eaa7d04e3daf4efce2164c630219885087234102"}, + {file = "multidict-6.0.3-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:26fbbe17f8a7211b623502d2bf41022a51da3025142401417c765bf9a56fed4c"}, + {file = "multidict-6.0.3-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:4fb3fe591956d8841882c463f934c9f7485cfd5f763a08c0d467b513dc18ef89"}, + {file = "multidict-6.0.3-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:e1925f78a543b94c3d46274c66a366fee8a263747060220ed0188e5f3eeea1c0"}, + {file = "multidict-6.0.3-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:21e1ce0b187c4e93112304dcde2aa18922fdbe8fb4f13d8aa72a5657bce0563a"}, + {file = "multidict-6.0.3-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:e07c24018986fb00d6e7eafca8fcd6e05095649e17fcf0e33a592caaa62a78b9"}, + {file = "multidict-6.0.3-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:114a4ab3e5cfbc56c4b6697686ecb92376c7e8c56893ef20547921552f8bdf57"}, + {file = "multidict-6.0.3-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:4ccf55f28066b4f08666764a957c2b7c241c7547b0921d69c7ceab5f74fe1a45"}, + {file = "multidict-6.0.3-cp311-cp311-musllinux_1_1_ppc64le.whl", hash = "sha256:9d359b0a962e052b713647ac1f13eabf2263167b149ed1e27d5c579f5c8c7d2c"}, + {file = "multidict-6.0.3-cp311-cp311-musllinux_1_1_s390x.whl", hash = "sha256:df7b4cee3ff31b3335aba602f8d70dbc641e5b7164b1e9565570c9d3c536a438"}, + {file = "multidict-6.0.3-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:ee9b1cae9a6c5d023e5a150f6f6b9dbb3c3bbc7887d6ee07d4c0ecb49a473734"}, + {file = "multidict-6.0.3-cp311-cp311-win32.whl", hash = "sha256:960ce1b790952916e682093788696ef7e33ac6a97482f9b983abdc293091b531"}, + {file = "multidict-6.0.3-cp311-cp311-win_amd64.whl", hash = "sha256:2b66d61966b12e6bba500e5cbb2c721a35e119c30ee02495c5629bd0e91eea30"}, + {file = "multidict-6.0.3-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:526f8397fc124674b8f39748680a0ff673bd6a715fecb4866716d36e380f015f"}, + {file = "multidict-6.0.3-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:1f5d5129a937af4e3c4a1d6c139f4051b7d17d43276cefdd8d442a7031f7eef2"}, + {file = "multidict-6.0.3-cp37-cp37m-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:38d394814b39be1c36ac709006d39d50d72a884f9551acd9c8cc1ffae3fc8c4e"}, + {file = "multidict-6.0.3-cp37-cp37m-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:99341ca1f1db9e7f47914cb2461305665a662383765ced6f843712564766956d"}, + {file = "multidict-6.0.3-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:c5790cc603456b6dcf8a9a4765f666895a6afddc88b3d3ba7b53dea2b6e23116"}, + {file = "multidict-6.0.3-cp37-cp37m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:ce8e51774eb03844588d3c279adb94efcd0edeccd2f97516623292445bcc01f9"}, + {file = "multidict-6.0.3-cp37-cp37m-musllinux_1_1_aarch64.whl", hash = "sha256:baa96a3418e27d723064854143b2f414a422c84cc87285a71558722049bebc5a"}, + {file = "multidict-6.0.3-cp37-cp37m-musllinux_1_1_i686.whl", hash = "sha256:cb4a08f0aaaa869f189ffea0e17b86ad0237b51116d494da15ef7991ee6ad2d7"}, + {file = "multidict-6.0.3-cp37-cp37m-musllinux_1_1_ppc64le.whl", hash = "sha256:62db44727d0befea68e8ad2881bb87a9cfb6b87d45dd78609009627167f37b69"}, + {file = "multidict-6.0.3-cp37-cp37m-musllinux_1_1_s390x.whl", hash = "sha256:4cc5c8cd205a9810d16a5cd428cd81bac554ad1477cb87f4ad722b10992e794d"}, + {file = "multidict-6.0.3-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:f76109387e1ec8d8e2137c94c437b89fe002f29e0881aae8ae45529bdff92000"}, + {file = "multidict-6.0.3-cp37-cp37m-win32.whl", hash = "sha256:f8a728511c977df6f3d8af388fcb157e49f11db4a6637dd60131b8b6e40b0253"}, + {file = "multidict-6.0.3-cp37-cp37m-win_amd64.whl", hash = "sha256:c2a1168e5aa7c72499fb03c850e0f03f624fa4a5c8d2e215c518d0a73872eb64"}, + {file = "multidict-6.0.3-cp38-cp38-macosx_10_9_universal2.whl", hash = "sha256:eddf604a3de2ace3d9a4e4d491be7562a1ac095a0a1c95a9ec5781ef0273ef11"}, + {file = "multidict-6.0.3-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:d09daf5c6ce7fc6ed444c9339bbde5ea84e2534d1ca1cd37b60f365c77f00dea"}, + {file = "multidict-6.0.3-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:12e0d396faa6dc55ff5379eee54d1df3b508243ff15bfc8295a6ec7a4483a335"}, + {file = "multidict-6.0.3-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:70740c2bc9ab1c99f7cdcb104f27d16c63860c56d51c5bf0ef82fc1d892a2131"}, + {file = "multidict-6.0.3-cp38-cp38-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:e322c94596054352f5a02771eec71563c018b15699b961aba14d6dd943367022"}, + {file = "multidict-6.0.3-cp38-cp38-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:4159fc1ec9ede8ab93382e0d6ba9b1b3d23c72da39a834db7a116986605c7ab4"}, + {file = "multidict-6.0.3-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:47defc0218682281a52fb1f6346ebb8b68b17538163a89ea24dfe4da37a8a9a3"}, + {file = "multidict-6.0.3-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:7f9511e48bde6b995825e8d35e434fc96296cf07a25f4aae24ff9162be7eaa46"}, + {file = "multidict-6.0.3-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:e0bce9f7c30e7e3a9e683f670314c0144e8d34be6b7019e40604763bd278d84f"}, + {file = "multidict-6.0.3-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:01b456046a05ff7cceefb0e1d2a9d32f05efcb1c7e0d152446304e11557639ce"}, + {file = "multidict-6.0.3-cp38-cp38-musllinux_1_1_ppc64le.whl", hash = "sha256:8230a39bae6c2e8a09e4da6bace5064693b00590a4a213e38f9a9366da10e7dd"}, + {file = "multidict-6.0.3-cp38-cp38-musllinux_1_1_s390x.whl", hash = "sha256:445c0851a1cbc1f2ec3b40bc22f9c4a235edb3c9a0906122a9df6ea8d51f886c"}, + {file = "multidict-6.0.3-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:9aac6881454a750554ed4b280a839dcf9e2133a9d12ab4d417d673fb102289b7"}, + {file = "multidict-6.0.3-cp38-cp38-win32.whl", hash = "sha256:81c3d597591b0940e04949e4e4f79359b2d2e542a686ba0da5e25de33fec13e0"}, + {file = "multidict-6.0.3-cp38-cp38-win_amd64.whl", hash = "sha256:dc4cfef5d899f5f1a15f3d2ac49f71107a01a5a2745b4dd53fa0cede1419385a"}, + {file = "multidict-6.0.3-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:d408172519049e36fb6d29672f060dc8461fc7174eba9883c7026041ef9bfb38"}, + {file = "multidict-6.0.3-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:e068dfeadbce63072b2d8096486713d04db4946aad0a0f849bd4fc300799d0d3"}, + {file = "multidict-6.0.3-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:a8b817d4ed68fd568ec5e45dd75ddf30cc72a47a6b41b74d5bb211374c296f5e"}, + {file = "multidict-6.0.3-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:2cf5d19e12eff855aa198259c0b02fd3f5d07e1291fbd20279c37b3b0e6c9852"}, + {file = "multidict-6.0.3-cp39-cp39-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:e5a811aab1b4aea0b4be669363c19847a8c547510f0e18fb632956369fdbdf67"}, + {file = "multidict-6.0.3-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:2cfda34b7cb99eacada2072e0f69c0ad3285cb6f8e480b11f2b6d6c1c6f92718"}, + {file = "multidict-6.0.3-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:beeca903e4270b4afcd114f371a9602240dc143f9e944edfea00f8d4ad56c40d"}, + {file = "multidict-6.0.3-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:cd5771e8ea325f85cbb361ddbdeb9ae424a68e5dfb6eea786afdcd22e68a7d5d"}, + {file = "multidict-6.0.3-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:9dbab2a7e9c073bc9538824a01f5ed689194db7f55f2b8102766873e906a6c1a"}, + {file = "multidict-6.0.3-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:f2c0957b3e8c66c10d27272709a5299ab3670a0f187c9428f3b90d267119aedb"}, + {file = "multidict-6.0.3-cp39-cp39-musllinux_1_1_ppc64le.whl", hash = "sha256:94cbe5535ef150546b8321aebea22862a3284da51e7b55f6f95b7d73e96d90ee"}, + {file = "multidict-6.0.3-cp39-cp39-musllinux_1_1_s390x.whl", hash = "sha256:d0e798b072cf2aab9daceb43d97c9c527a0c7593e67a7846ad4cc6051de1e303"}, + {file = "multidict-6.0.3-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:a27b029caa3b555a4f3da54bc1e718eb55fcf1a11fda8bf0132147b476cf4c08"}, + {file = "multidict-6.0.3-cp39-cp39-win32.whl", hash = "sha256:018c8e3be7f161a12b3e41741b6721f9baeb2210f4ab25a6359b7d76c1017dce"}, + {file = "multidict-6.0.3-cp39-cp39-win_amd64.whl", hash = "sha256:5e58ec0375803526d395f6f7e730ecc45d06e15f68f7b9cdbf644a2918324e51"}, + {file = "multidict-6.0.3.tar.gz", hash = "sha256:2523a29006c034687eccd3ee70093a697129a3ffe8732535d3b2df6a4ecc279d"}, ] nodeenv = [ {file = "nodeenv-1.7.0-py2.py3-none-any.whl", hash = "sha256:27083a7b96a25f2f5e1d8cb4b6317ee8aeda3bdd121394e5ac54e498028a042e"}, @@ -1640,8 +1618,8 @@ numpy = [ {file = "numpy-1.23.5.tar.gz", hash = "sha256:1b1766d6f397c18153d40015ddfc79ddb715cabadc04d2d228d4e5a8bc4ded1a"}, ] packaging = [ - {file = "packaging-21.3-py3-none-any.whl", hash = "sha256:ef103e05f519cdc783ae24ea4e2e0f508a9c99b2d4969652eed6a2e1ea5bd522"}, - {file = "packaging-21.3.tar.gz", hash = "sha256:dd47c42927d89ab911e606518907cc2d3a1f38bbd026385970643f9c5b8ecfeb"}, + {file = "packaging-22.0-py3-none-any.whl", hash = "sha256:957e2148ba0e1a3b282772e791ef1d8083648bc131c8ab0c1feba110ce1146c3"}, + {file = "packaging-22.0.tar.gz", hash = "sha256:2198ec20bd4c017b8f9717e00f0c8714076fc2fd93816750ab48e2c41de2cfd3"}, ] pandas = [ {file = "pandas-1.5.2-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:e9dbacd22555c2d47f262ef96bb4e30880e5956169741400af8b306bbb24a273"}, @@ -1672,22 +1650,16 @@ pandas = [ {file = "pandas-1.5.2-cp39-cp39-win_amd64.whl", hash = "sha256:c218796d59d5abd8780170c937b812c9637e84c32f8271bbf9845970f8c1351f"}, {file = "pandas-1.5.2.tar.gz", hash = "sha256:220b98d15cee0b2cd839a6358bd1f273d0356bf964c1a1aeb32d47db0215488b"}, ] -pep517 = [ - {file = "pep517-0.13.0-py3-none-any.whl", hash = "sha256:4ba4446d80aed5b5eac6509ade100bff3e7943a8489de249654a5ae9b33ee35b"}, - {file = "pep517-0.13.0.tar.gz", hash = "sha256:ae69927c5c172be1add9203726d4b84cf3ebad1edcd5f71fcdc746e66e829f59"}, -] +pep517 = [] platformdirs = [ - {file = "platformdirs-2.5.4-py3-none-any.whl", hash = "sha256:af0276409f9a02373d540bf8480021a048711d572745aef4b7842dad245eba10"}, - {file = "platformdirs-2.5.4.tar.gz", hash = "sha256:1006647646d80f16130f052404c6b901e80ee4ed6bef6792e1f238a8969106f7"}, + {file = "platformdirs-2.6.0-py3-none-any.whl", hash = "sha256:1a89a12377800c81983db6be069ec068eee989748799b946cce2a6e80dcc54ca"}, + {file = "platformdirs-2.6.0.tar.gz", hash = "sha256:b46ffafa316e6b83b47489d240ce17173f123a9b9c83282141c3daf26ad9ac2e"}, ] pluggy = [ {file = "pluggy-1.0.0-py2.py3-none-any.whl", hash = "sha256:74134bbf457f031a36d68416e1509f34bd5ccc019f0bcc952c7b909d06b37bd3"}, {file = "pluggy-1.0.0.tar.gz", hash = "sha256:4224373bacce55f955a878bf9cfa763c1e360858e330072059e10bad68531159"}, ] -pre-commit = [ - {file = "pre_commit-2.20.0-py2.py3-none-any.whl", hash = "sha256:51a5ba7c480ae8072ecdb6933df22d2f812dc897d5fe848778116129a681aac7"}, - {file = "pre_commit-2.20.0.tar.gz", hash = "sha256:a978dac7bc9ec0bcee55c18a277d553b0f419d259dadb4b9418ff2d00eb43959"}, -] +pre-commit = [] pyarrow = [ {file = "pyarrow-10.0.1-cp310-cp310-macosx_10_14_x86_64.whl", hash = "sha256:e00174764a8b4e9d8d5909b6d19ee0c217a6cf0232c5682e31fdfbd5a9f0ae52"}, {file = "pyarrow-10.0.1-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:6f7a7dbe2f7f65ac1d0bd3163f756deb478a9e9afc2269557ed75b1b25ab3610"}, @@ -1757,10 +1729,7 @@ pydantic = [ {file = "pydantic-1.10.2-py3-none-any.whl", hash = "sha256:1b6ee725bd6e83ec78b1aa32c5b1fa67a3a65badddde3976bca5fe4568f27709"}, {file = "pydantic-1.10.2.tar.gz", hash = "sha256:91b8e218852ef6007c2b98cd861601c6a09f1aa32bbbb74fab5b1c33d4a1e410"}, ] -pygments = [ - {file = "Pygments-2.13.0-py3-none-any.whl", hash = "sha256:f643f331ab57ba3c9d89212ee4a2dabc6e94f117cf4eefde99a0574720d14c42"}, - {file = "Pygments-2.13.0.tar.gz", hash = "sha256:56a8508ae95f98e2b9bdf93a6be5ae3f7d8af858b43e02c5a2ff083726be40c1"}, -] +pygments = [] pyparsing = [ {file = "pyparsing-3.0.9-py3-none-any.whl", hash = "sha256:5026bae9a10eeaefb61dab2f09052b9f4307d44aee4eda64b309723d8d206bbc"}, {file = "pyparsing-3.0.9.tar.gz", hash = "sha256:2b020ecf7d21b687f219b71ecad3631f644a47f01403fa1d1036b0c6416d70fb"}, @@ -1773,10 +1742,7 @@ pytest-checkdocs = [ {file = "pytest-checkdocs-2.9.0.tar.gz", hash = "sha256:54b3d1357a289e3f6336d31d4ceb72e99afe0fa4f552990f1b1f015f1925543b"}, {file = "pytest_checkdocs-2.9.0-py3-none-any.whl", hash = "sha256:4231e5426915a02385e386b5eb4a70490b143d97cede4c7c77691bb320b5a73f"}, ] -python-dateutil = [ - {file = "python-dateutil-2.8.2.tar.gz", hash = "sha256:0123cacc1627ae19ddf3c27a5de5bd67ee4586fbdd6440d9748f8abb483d3e86"}, - {file = "python_dateutil-2.8.2-py2.py3-none-any.whl", hash = "sha256:961d03dc3453ebbc59dbdea9e4e11c5651520a876d0f4db161e8674aae935da9"}, -] +python-dateutil = [] python-snappy = [ {file = "python-snappy-0.6.1.tar.gz", hash = "sha256:b6a107ab06206acc5359d4c5632bd9b22d448702a79b3169b0c62e0fb808bb2a"}, {file = "python_snappy-0.6.1-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:b7f920eaf46ebf41bd26f9df51c160d40f9e00b7b48471c3438cb8d027f7fb9b"}, @@ -1839,13 +1805,6 @@ pyyaml = [ {file = "PyYAML-6.0-cp310-cp310-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:f84fbc98b019fef2ee9a1cb3ce93e3187a6df0b2538a651bfb890254ba9f90b5"}, {file = "PyYAML-6.0-cp310-cp310-win32.whl", hash = "sha256:2cd5df3de48857ed0544b34e2d40e9fac445930039f3cfe4bcc592a1f836d513"}, {file = "PyYAML-6.0-cp310-cp310-win_amd64.whl", hash = "sha256:daf496c58a8c52083df09b80c860005194014c3698698d1a57cbcfa182142a3a"}, - {file = "PyYAML-6.0-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:d4b0ba9512519522b118090257be113b9468d804b19d63c71dbcf4a48fa32358"}, - {file = "PyYAML-6.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:81957921f441d50af23654aa6c5e5eaf9b06aba7f0a19c18a538dc7ef291c5a1"}, - {file = "PyYAML-6.0-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:afa17f5bc4d1b10afd4466fd3a44dc0e245382deca5b3c353d8b757f9e3ecb8d"}, - {file = "PyYAML-6.0-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:dbad0e9d368bb989f4515da330b88a057617d16b6a8245084f1b05400f24609f"}, - {file = "PyYAML-6.0-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:432557aa2c09802be39460360ddffd48156e30721f5e8d917f01d31694216782"}, - {file = "PyYAML-6.0-cp311-cp311-win32.whl", hash = "sha256:bfaef573a63ba8923503d27530362590ff4f576c626d86a9fed95822a8255fd7"}, - {file = "PyYAML-6.0-cp311-cp311-win_amd64.whl", hash = "sha256:01b45c0191e6d66c470b6cf1b9531a771a83c1c4208272ead47a3ae4f2f603bf"}, {file = "PyYAML-6.0-cp36-cp36m-macosx_10_9_x86_64.whl", hash = "sha256:897b80890765f037df3403d22bab41627ca8811ae55e9a722fd0392850ec4d86"}, {file = "PyYAML-6.0-cp36-cp36m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:50602afada6d6cbfad699b0c7bb50d5ccffa7e46a3d738092afddc1f9758427f"}, {file = "PyYAML-6.0-cp36-cp36m-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:48c346915c114f5fdb3ead70312bd042a953a8ce5c7106d5bfb1a5254e47da92"}, @@ -1893,21 +1852,12 @@ s3fs = [ {file = "s3fs-2022.10.0-py3-none-any.whl", hash = "sha256:1e134c3577171699feb7c1a0c4713260d5b48296e1708737ff940baef6e2c153"}, {file = "s3fs-2022.10.0.tar.gz", hash = "sha256:e8deb80f20bd0b2059141b874fdb9d6aeb8cce35312ea5f2c02b225a78a00406"}, ] -s3transfer = [ - {file = "s3transfer-0.6.0-py3-none-any.whl", hash = "sha256:06176b74f3a15f61f1b4f25a1fc29a4429040b7647133a463da8fa5bd28d5ecd"}, - {file = "s3transfer-0.6.0.tar.gz", hash = "sha256:2ed07d3866f523cc561bf4a00fc5535827981b117dd7876f036b0c1aca42c947"}, -] -setuptools = [ - {file = "setuptools-65.6.3-py3-none-any.whl", hash = "sha256:57f6f22bde4e042978bcd50176fdb381d7c21a9efa4041202288d3737a0c6a54"}, - {file = "setuptools-65.6.3.tar.gz", hash = "sha256:a7620757bf984b58deaf32fc8a4577a9bbc0850cf92c20e1ce41c38c19e5fb75"}, -] +s3transfer = [] six = [ {file = "six-1.16.0-py2.py3-none-any.whl", hash = "sha256:8abb2f1d86890a2dfb989f9a77cfcfd3e47c2a354b01111771326f8aa26e0254"}, {file = "six-1.16.0.tar.gz", hash = "sha256:1e61c37477a1626458e36f7b1d82aa5c9b094fa4802892072e49de9c60c4c926"}, ] -thrift = [ - {file = "thrift-0.16.0.tar.gz", hash = "sha256:2b5b6488fcded21f9d312aa23c9ff6a0195d0f6ae26ddbd5ad9e3e25dfc14408"}, -] +thrift = [] toml = [ {file = "toml-0.10.2-py2.py3-none-any.whl", hash = "sha256:806143ae5bfb6a3c6e736a764057db0e6a0e05e338b5630894a5f779cabb4f9b"}, {file = "toml-0.10.2.tar.gz", hash = "sha256:b3bda1d108d5dd99f4a20d24d9c348e91c4db7ab1b749200bded2f839ccbe68f"}, @@ -1929,8 +1879,8 @@ urllib3 = [ {file = "urllib3-1.26.13.tar.gz", hash = "sha256:c083dd0dce68dbfbe1129d5271cb90f9447dea7d52097c6e0126120c521ddea8"}, ] virtualenv = [ - {file = "virtualenv-20.17.0-py3-none-any.whl", hash = "sha256:40a7e06a98728fd5769e1af6fd1a706005b4bb7e16176a272ed4292473180389"}, - {file = "virtualenv-20.17.0.tar.gz", hash = "sha256:7d6a8d55b2f73b617f684ee40fd85740f062e1f2e379412cb1879c7136f05902"}, + {file = "virtualenv-20.17.1-py3-none-any.whl", hash = "sha256:ce3b1684d6e1a20a3e5ed36795a97dfc6af29bc3970ca8dab93e11ac6094b3c4"}, + {file = "virtualenv-20.17.1.tar.gz", hash = "sha256:f8b927684efc6f1cc206c9db297a570ab9ad0e51c16fa9e45487d36d1905c058"}, ] werkzeug = [ {file = "Werkzeug-2.2.2-py3-none-any.whl", hash = "sha256:f979ab81f58d7318e064e99c4506445d60135ac5cd2e177a2de0089bfd4c9bd5"}, @@ -2007,65 +1957,80 @@ xmltodict = [ {file = "xmltodict-0.13.0.tar.gz", hash = "sha256:341595a488e3e01a85a9d8911d8912fd922ede5fecc4dce437eb4b6c8d037e56"}, ] yarl = [ - {file = "yarl-1.8.1-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:abc06b97407868ef38f3d172762f4069323de52f2b70d133d096a48d72215d28"}, - {file = "yarl-1.8.1-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:07b21e274de4c637f3e3b7104694e53260b5fc10d51fb3ec5fed1da8e0f754e3"}, - {file = "yarl-1.8.1-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:9de955d98e02fab288c7718662afb33aab64212ecb368c5dc866d9a57bf48880"}, - {file = "yarl-1.8.1-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:7ec362167e2c9fd178f82f252b6d97669d7245695dc057ee182118042026da40"}, - {file = "yarl-1.8.1-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:20df6ff4089bc86e4a66e3b1380460f864df3dd9dccaf88d6b3385d24405893b"}, - {file = "yarl-1.8.1-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:5999c4662631cb798496535afbd837a102859568adc67d75d2045e31ec3ac497"}, - {file = "yarl-1.8.1-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:ed19b74e81b10b592084a5ad1e70f845f0aacb57577018d31de064e71ffa267a"}, - {file = "yarl-1.8.1-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:1e4808f996ca39a6463f45182e2af2fae55e2560be586d447ce8016f389f626f"}, - {file = "yarl-1.8.1-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:2d800b9c2eaf0684c08be5f50e52bfa2aa920e7163c2ea43f4f431e829b4f0fd"}, - {file = "yarl-1.8.1-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:6628d750041550c5d9da50bb40b5cf28a2e63b9388bac10fedd4f19236ef4957"}, - {file = "yarl-1.8.1-cp310-cp310-musllinux_1_1_ppc64le.whl", hash = "sha256:f5af52738e225fcc526ae64071b7e5342abe03f42e0e8918227b38c9aa711e28"}, - {file = "yarl-1.8.1-cp310-cp310-musllinux_1_1_s390x.whl", hash = "sha256:76577f13333b4fe345c3704811ac7509b31499132ff0181f25ee26619de2c843"}, - {file = "yarl-1.8.1-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:0c03f456522d1ec815893d85fccb5def01ffaa74c1b16ff30f8aaa03eb21e453"}, - {file = "yarl-1.8.1-cp310-cp310-win32.whl", hash = "sha256:ea30a42dc94d42f2ba4d0f7c0ffb4f4f9baa1b23045910c0c32df9c9902cb272"}, - {file = "yarl-1.8.1-cp310-cp310-win_amd64.whl", hash = "sha256:9130ddf1ae9978abe63808b6b60a897e41fccb834408cde79522feb37fb72fb0"}, - {file = "yarl-1.8.1-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:0ab5a138211c1c366404d912824bdcf5545ccba5b3ff52c42c4af4cbdc2c5035"}, - {file = "yarl-1.8.1-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a0fb2cb4204ddb456a8e32381f9a90000429489a25f64e817e6ff94879d432fc"}, - {file = "yarl-1.8.1-cp37-cp37m-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:85cba594433915d5c9a0d14b24cfba0339f57a2fff203a5d4fd070e593307d0b"}, - {file = "yarl-1.8.1-cp37-cp37m-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:1ca7e596c55bd675432b11320b4eacc62310c2145d6801a1f8e9ad160685a231"}, - {file = "yarl-1.8.1-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:d0f77539733e0ec2475ddcd4e26777d08996f8cd55d2aef82ec4d3896687abda"}, - {file = "yarl-1.8.1-cp37-cp37m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:29e256649f42771829974e742061c3501cc50cf16e63f91ed8d1bf98242e5507"}, - {file = "yarl-1.8.1-cp37-cp37m-musllinux_1_1_aarch64.whl", hash = "sha256:7fce6cbc6c170ede0221cc8c91b285f7f3c8b9fe28283b51885ff621bbe0f8ee"}, - {file = "yarl-1.8.1-cp37-cp37m-musllinux_1_1_i686.whl", hash = "sha256:59ddd85a1214862ce7c7c66457f05543b6a275b70a65de366030d56159a979f0"}, - {file = "yarl-1.8.1-cp37-cp37m-musllinux_1_1_ppc64le.whl", hash = "sha256:12768232751689c1a89b0376a96a32bc7633c08da45ad985d0c49ede691f5c0d"}, - {file = "yarl-1.8.1-cp37-cp37m-musllinux_1_1_s390x.whl", hash = "sha256:b19255dde4b4f4c32e012038f2c169bb72e7f081552bea4641cab4d88bc409dd"}, - {file = "yarl-1.8.1-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:6c8148e0b52bf9535c40c48faebb00cb294ee577ca069d21bd5c48d302a83780"}, - {file = "yarl-1.8.1-cp37-cp37m-win32.whl", hash = "sha256:de839c3a1826a909fdbfe05f6fe2167c4ab033f1133757b5936efe2f84904c07"}, - {file = "yarl-1.8.1-cp37-cp37m-win_amd64.whl", hash = "sha256:dd032e8422a52e5a4860e062eb84ac94ea08861d334a4bcaf142a63ce8ad4802"}, - {file = "yarl-1.8.1-cp38-cp38-macosx_10_9_universal2.whl", hash = "sha256:19cd801d6f983918a3f3a39f3a45b553c015c5aac92ccd1fac619bd74beece4a"}, - {file = "yarl-1.8.1-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:6347f1a58e658b97b0a0d1ff7658a03cb79bdbda0331603bed24dd7054a6dea1"}, - {file = "yarl-1.8.1-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:7c0da7e44d0c9108d8b98469338705e07f4bb7dab96dbd8fa4e91b337db42548"}, - {file = "yarl-1.8.1-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:5587bba41399854703212b87071c6d8638fa6e61656385875f8c6dff92b2e461"}, - {file = "yarl-1.8.1-cp38-cp38-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:31a9a04ecccd6b03e2b0e12e82131f1488dea5555a13a4d32f064e22a6003cfe"}, - {file = "yarl-1.8.1-cp38-cp38-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:205904cffd69ae972a1707a1bd3ea7cded594b1d773a0ce66714edf17833cdae"}, - {file = "yarl-1.8.1-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:ea513a25976d21733bff523e0ca836ef1679630ef4ad22d46987d04b372d57fc"}, - {file = "yarl-1.8.1-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:d0b51530877d3ad7a8d47b2fff0c8df3b8f3b8deddf057379ba50b13df2a5eae"}, - {file = "yarl-1.8.1-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:d2b8f245dad9e331540c350285910b20dd913dc86d4ee410c11d48523c4fd546"}, - {file = "yarl-1.8.1-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:ab2a60d57ca88e1d4ca34a10e9fb4ab2ac5ad315543351de3a612bbb0560bead"}, - {file = "yarl-1.8.1-cp38-cp38-musllinux_1_1_ppc64le.whl", hash = "sha256:449c957ffc6bc2309e1fbe67ab7d2c1efca89d3f4912baeb8ead207bb3cc1cd4"}, - {file = "yarl-1.8.1-cp38-cp38-musllinux_1_1_s390x.whl", hash = "sha256:a165442348c211b5dea67c0206fc61366212d7082ba8118c8c5c1c853ea4d82e"}, - {file = "yarl-1.8.1-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:b3ded839a5c5608eec8b6f9ae9a62cb22cd037ea97c627f38ae0841a48f09eae"}, - {file = "yarl-1.8.1-cp38-cp38-win32.whl", hash = "sha256:c1445a0c562ed561d06d8cbc5c8916c6008a31c60bc3655cdd2de1d3bf5174a0"}, - {file = "yarl-1.8.1-cp38-cp38-win_amd64.whl", hash = "sha256:56c11efb0a89700987d05597b08a1efcd78d74c52febe530126785e1b1a285f4"}, - {file = "yarl-1.8.1-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:e80ed5a9939ceb6fda42811542f31c8602be336b1fb977bccb012e83da7e4936"}, - {file = "yarl-1.8.1-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:6afb336e23a793cd3b6476c30f030a0d4c7539cd81649683b5e0c1b0ab0bf350"}, - {file = "yarl-1.8.1-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:4c322cbaa4ed78a8aac89b2174a6df398faf50e5fc12c4c191c40c59d5e28357"}, - {file = "yarl-1.8.1-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:fae37373155f5ef9b403ab48af5136ae9851151f7aacd9926251ab26b953118b"}, - {file = "yarl-1.8.1-cp39-cp39-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:5395da939ffa959974577eff2cbfc24b004a2fb6c346918f39966a5786874e54"}, - {file = "yarl-1.8.1-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:076eede537ab978b605f41db79a56cad2e7efeea2aa6e0fa8f05a26c24a034fb"}, - {file = "yarl-1.8.1-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:3d1a50e461615747dd93c099f297c1994d472b0f4d2db8a64e55b1edf704ec1c"}, - {file = "yarl-1.8.1-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:7de89c8456525650ffa2bb56a3eee6af891e98f498babd43ae307bd42dca98f6"}, - {file = "yarl-1.8.1-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:4a88510731cd8d4befaba5fbd734a7dd914de5ab8132a5b3dde0bbd6c9476c64"}, - {file = "yarl-1.8.1-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:2d93a049d29df172f48bcb09acf9226318e712ce67374f893b460b42cc1380ae"}, - {file = "yarl-1.8.1-cp39-cp39-musllinux_1_1_ppc64le.whl", hash = "sha256:21ac44b763e0eec15746a3d440f5e09ad2ecc8b5f6dcd3ea8cb4773d6d4703e3"}, - {file = "yarl-1.8.1-cp39-cp39-musllinux_1_1_s390x.whl", hash = "sha256:d0272228fabe78ce00a3365ffffd6f643f57a91043e119c289aaba202f4095b0"}, - {file = "yarl-1.8.1-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:99449cd5366fe4608e7226c6cae80873296dfa0cde45d9b498fefa1de315a09e"}, - {file = "yarl-1.8.1-cp39-cp39-win32.whl", hash = "sha256:8b0af1cf36b93cee99a31a545fe91d08223e64390c5ecc5e94c39511832a4bb6"}, - {file = "yarl-1.8.1-cp39-cp39-win_amd64.whl", hash = "sha256:de49d77e968de6626ba7ef4472323f9d2e5a56c1d85b7c0e2a190b2173d3b9be"}, - {file = "yarl-1.8.1.tar.gz", hash = "sha256:af887845b8c2e060eb5605ff72b6f2dd2aab7a761379373fd89d314f4752abbf"}, + {file = "yarl-1.8.2-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:bb81f753c815f6b8e2ddd2eef3c855cf7da193b82396ac013c661aaa6cc6b0a5"}, + {file = "yarl-1.8.2-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:47d49ac96156f0928f002e2424299b2c91d9db73e08c4cd6742923a086f1c863"}, + {file = "yarl-1.8.2-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:3fc056e35fa6fba63248d93ff6e672c096f95f7836938241ebc8260e062832fe"}, + {file = "yarl-1.8.2-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:58a3c13d1c3005dbbac5c9f0d3210b60220a65a999b1833aa46bd6677c69b08e"}, + {file = "yarl-1.8.2-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:10b08293cda921157f1e7c2790999d903b3fd28cd5c208cf8826b3b508026996"}, + {file = "yarl-1.8.2-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:de986979bbd87272fe557e0a8fcb66fd40ae2ddfe28a8b1ce4eae22681728fef"}, + {file = "yarl-1.8.2-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:6c4fcfa71e2c6a3cb568cf81aadc12768b9995323186a10827beccf5fa23d4f8"}, + {file = "yarl-1.8.2-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:ae4d7ff1049f36accde9e1ef7301912a751e5bae0a9d142459646114c70ecba6"}, + {file = "yarl-1.8.2-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:bf071f797aec5b96abfc735ab97da9fd8f8768b43ce2abd85356a3127909d146"}, + {file = "yarl-1.8.2-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:74dece2bfc60f0f70907c34b857ee98f2c6dd0f75185db133770cd67300d505f"}, + {file = "yarl-1.8.2-cp310-cp310-musllinux_1_1_ppc64le.whl", hash = "sha256:df60a94d332158b444301c7f569659c926168e4d4aad2cfbf4bce0e8fb8be826"}, + {file = "yarl-1.8.2-cp310-cp310-musllinux_1_1_s390x.whl", hash = "sha256:63243b21c6e28ec2375f932a10ce7eda65139b5b854c0f6b82ed945ba526bff3"}, + {file = "yarl-1.8.2-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:cfa2bbca929aa742b5084fd4663dd4b87c191c844326fcb21c3afd2d11497f80"}, + {file = "yarl-1.8.2-cp310-cp310-win32.whl", hash = "sha256:b05df9ea7496df11b710081bd90ecc3a3db6adb4fee36f6a411e7bc91a18aa42"}, + {file = "yarl-1.8.2-cp310-cp310-win_amd64.whl", hash = "sha256:24ad1d10c9db1953291f56b5fe76203977f1ed05f82d09ec97acb623a7976574"}, + {file = "yarl-1.8.2-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:2a1fca9588f360036242f379bfea2b8b44cae2721859b1c56d033adfd5893634"}, + {file = "yarl-1.8.2-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:f37db05c6051eff17bc832914fe46869f8849de5b92dc4a3466cd63095d23dfd"}, + {file = "yarl-1.8.2-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:77e913b846a6b9c5f767b14dc1e759e5aff05502fe73079f6f4176359d832581"}, + {file = "yarl-1.8.2-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:0978f29222e649c351b173da2b9b4665ad1feb8d1daa9d971eb90df08702668a"}, + {file = "yarl-1.8.2-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:388a45dc77198b2460eac0aca1efd6a7c09e976ee768b0d5109173e521a19daf"}, + {file = "yarl-1.8.2-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:2305517e332a862ef75be8fad3606ea10108662bc6fe08509d5ca99503ac2aee"}, + {file = "yarl-1.8.2-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:42430ff511571940d51e75cf42f1e4dbdded477e71c1b7a17f4da76c1da8ea76"}, + {file = "yarl-1.8.2-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:3150078118f62371375e1e69b13b48288e44f6691c1069340081c3fd12c94d5b"}, + {file = "yarl-1.8.2-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:c15163b6125db87c8f53c98baa5e785782078fbd2dbeaa04c6141935eb6dab7a"}, + {file = "yarl-1.8.2-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:4d04acba75c72e6eb90745447d69f84e6c9056390f7a9724605ca9c56b4afcc6"}, + {file = "yarl-1.8.2-cp311-cp311-musllinux_1_1_ppc64le.whl", hash = "sha256:e7fd20d6576c10306dea2d6a5765f46f0ac5d6f53436217913e952d19237efc4"}, + {file = "yarl-1.8.2-cp311-cp311-musllinux_1_1_s390x.whl", hash = "sha256:75c16b2a900b3536dfc7014905a128a2bea8fb01f9ee26d2d7d8db0a08e7cb2c"}, + {file = "yarl-1.8.2-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:6d88056a04860a98341a0cf53e950e3ac9f4e51d1b6f61a53b0609df342cc8b2"}, + {file = "yarl-1.8.2-cp311-cp311-win32.whl", hash = "sha256:fb742dcdd5eec9f26b61224c23baea46c9055cf16f62475e11b9b15dfd5c117b"}, + {file = "yarl-1.8.2-cp311-cp311-win_amd64.whl", hash = "sha256:8c46d3d89902c393a1d1e243ac847e0442d0196bbd81aecc94fcebbc2fd5857c"}, + {file = "yarl-1.8.2-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:ceff9722e0df2e0a9e8a79c610842004fa54e5b309fe6d218e47cd52f791d7ef"}, + {file = "yarl-1.8.2-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:3f6b4aca43b602ba0f1459de647af954769919c4714706be36af670a5f44c9c1"}, + {file = "yarl-1.8.2-cp37-cp37m-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:1684a9bd9077e922300ecd48003ddae7a7474e0412bea38d4631443a91d61077"}, + {file = "yarl-1.8.2-cp37-cp37m-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:ebb78745273e51b9832ef90c0898501006670d6e059f2cdb0e999494eb1450c2"}, + {file = "yarl-1.8.2-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:3adeef150d528ded2a8e734ebf9ae2e658f4c49bf413f5f157a470e17a4a2e89"}, + {file = "yarl-1.8.2-cp37-cp37m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:57a7c87927a468e5a1dc60c17caf9597161d66457a34273ab1760219953f7f4c"}, + {file = "yarl-1.8.2-cp37-cp37m-musllinux_1_1_aarch64.whl", hash = "sha256:efff27bd8cbe1f9bd127e7894942ccc20c857aa8b5a0327874f30201e5ce83d0"}, + {file = "yarl-1.8.2-cp37-cp37m-musllinux_1_1_i686.whl", hash = "sha256:a783cd344113cb88c5ff7ca32f1f16532a6f2142185147822187913eb989f739"}, + {file = "yarl-1.8.2-cp37-cp37m-musllinux_1_1_ppc64le.whl", hash = "sha256:705227dccbe96ab02c7cb2c43e1228e2826e7ead880bb19ec94ef279e9555b5b"}, + {file = "yarl-1.8.2-cp37-cp37m-musllinux_1_1_s390x.whl", hash = "sha256:34c09b43bd538bf6c4b891ecce94b6fa4f1f10663a8d4ca589a079a5018f6ed7"}, + {file = "yarl-1.8.2-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:a48f4f7fea9a51098b02209d90297ac324241bf37ff6be6d2b0149ab2bd51b37"}, + {file = "yarl-1.8.2-cp37-cp37m-win32.whl", hash = "sha256:0414fd91ce0b763d4eadb4456795b307a71524dbacd015c657bb2a39db2eab89"}, + {file = "yarl-1.8.2-cp37-cp37m-win_amd64.whl", hash = "sha256:d881d152ae0007809c2c02e22aa534e702f12071e6b285e90945aa3c376463c5"}, + {file = "yarl-1.8.2-cp38-cp38-macosx_10_9_universal2.whl", hash = "sha256:5df5e3d04101c1e5c3b1d69710b0574171cc02fddc4b23d1b2813e75f35a30b1"}, + {file = "yarl-1.8.2-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:7a66c506ec67eb3159eea5096acd05f5e788ceec7b96087d30c7d2865a243918"}, + {file = "yarl-1.8.2-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:2b4fa2606adf392051d990c3b3877d768771adc3faf2e117b9de7eb977741229"}, + {file = "yarl-1.8.2-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:1e21fb44e1eff06dd6ef971d4bdc611807d6bd3691223d9c01a18cec3677939e"}, + {file = "yarl-1.8.2-cp38-cp38-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:93202666046d9edadfe9f2e7bf5e0782ea0d497b6d63da322e541665d65a044e"}, + {file = "yarl-1.8.2-cp38-cp38-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:fc77086ce244453e074e445104f0ecb27530d6fd3a46698e33f6c38951d5a0f1"}, + {file = "yarl-1.8.2-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:64dd68a92cab699a233641f5929a40f02a4ede8c009068ca8aa1fe87b8c20ae3"}, + {file = "yarl-1.8.2-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:1b372aad2b5f81db66ee7ec085cbad72c4da660d994e8e590c997e9b01e44901"}, + {file = "yarl-1.8.2-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:e6f3515aafe0209dd17fb9bdd3b4e892963370b3de781f53e1746a521fb39fc0"}, + {file = "yarl-1.8.2-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:dfef7350ee369197106805e193d420b75467b6cceac646ea5ed3049fcc950a05"}, + {file = "yarl-1.8.2-cp38-cp38-musllinux_1_1_ppc64le.whl", hash = "sha256:728be34f70a190566d20aa13dc1f01dc44b6aa74580e10a3fb159691bc76909d"}, + {file = "yarl-1.8.2-cp38-cp38-musllinux_1_1_s390x.whl", hash = "sha256:ff205b58dc2929191f68162633d5e10e8044398d7a45265f90a0f1d51f85f72c"}, + {file = "yarl-1.8.2-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:baf211dcad448a87a0d9047dc8282d7de59473ade7d7fdf22150b1d23859f946"}, + {file = "yarl-1.8.2-cp38-cp38-win32.whl", hash = "sha256:272b4f1599f1b621bf2aabe4e5b54f39a933971f4e7c9aa311d6d7dc06965165"}, + {file = "yarl-1.8.2-cp38-cp38-win_amd64.whl", hash = "sha256:326dd1d3caf910cd26a26ccbfb84c03b608ba32499b5d6eeb09252c920bcbe4f"}, + {file = "yarl-1.8.2-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:f8ca8ad414c85bbc50f49c0a106f951613dfa5f948ab69c10ce9b128d368baf8"}, + {file = "yarl-1.8.2-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:418857f837347e8aaef682679f41e36c24250097f9e2f315d39bae3a99a34cbf"}, + {file = "yarl-1.8.2-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:ae0eec05ab49e91a78700761777f284c2df119376e391db42c38ab46fd662b77"}, + {file = "yarl-1.8.2-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:009a028127e0a1755c38b03244c0bea9d5565630db9c4cf9572496e947137a87"}, + {file = "yarl-1.8.2-cp39-cp39-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:3edac5d74bb3209c418805bda77f973117836e1de7c000e9755e572c1f7850d0"}, + {file = "yarl-1.8.2-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:da65c3f263729e47351261351b8679c6429151ef9649bba08ef2528ff2c423b2"}, + {file = "yarl-1.8.2-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:0ef8fb25e52663a1c85d608f6dd72e19bd390e2ecaf29c17fb08f730226e3a08"}, + {file = "yarl-1.8.2-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:bcd7bb1e5c45274af9a1dd7494d3c52b2be5e6bd8d7e49c612705fd45420b12d"}, + {file = "yarl-1.8.2-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:44ceac0450e648de86da8e42674f9b7077d763ea80c8ceb9d1c3e41f0f0a9951"}, + {file = "yarl-1.8.2-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:97209cc91189b48e7cfe777237c04af8e7cc51eb369004e061809bcdf4e55220"}, + {file = "yarl-1.8.2-cp39-cp39-musllinux_1_1_ppc64le.whl", hash = "sha256:48dd18adcf98ea9cd721a25313aef49d70d413a999d7d89df44f469edfb38a06"}, + {file = "yarl-1.8.2-cp39-cp39-musllinux_1_1_s390x.whl", hash = "sha256:e59399dda559688461762800d7fb34d9e8a6a7444fd76ec33220a926c8be1516"}, + {file = "yarl-1.8.2-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:d617c241c8c3ad5c4e78a08429fa49e4b04bedfc507b34b4d8dceb83b4af3588"}, + {file = "yarl-1.8.2-cp39-cp39-win32.whl", hash = "sha256:cb6d48d80a41f68de41212f3dfd1a9d9898d7841c8f7ce6696cf2fd9cb57ef83"}, + {file = "yarl-1.8.2-cp39-cp39-win_amd64.whl", hash = "sha256:6604711362f2dbf7160df21c416f81fac0de6dbcf0b5445a2ef25478ecc4c778"}, + {file = "yarl-1.8.2.tar.gz", hash = "sha256:49d43402c6e3013ad0978602bf6bf5328535c48d192304b91b97a3c6790b1562"}, ] zipp = [ {file = "zipp-3.11.0-py3-none-any.whl", hash = "sha256:83a28fcb75844b5c0cdaf5aa4003c2d728c77e05f5aeabe8e95e56727005fbaa"}, diff --git a/pyproject.toml b/pyproject.toml index 77a0d8d976..61cabf1f7a 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -56,6 +56,8 @@ pyyaml = "6.0.0" pydantic = "1.10.2" fsspec = "2022.10.0" +pyparsing = "3.0.9" + zstandard = "0.19.0" pyarrow = { version = "10.0.1", optional = true } From d73a18da42afe9231bc613dcc56ddd082d12b731 Mon Sep 17 00:00:00 2001 From: Ruben van de Geer <37443208+rubenvdg@users.noreply.github.com> Date: Sat, 17 Dec 2022 09:35:18 +0100 Subject: [PATCH 305/642] Python: Mock home folder when running `test_missing_uri` (#6445) --- tests/cli/test_console.py | 19 +++++++++++-------- tests/conftest.py | 6 ++++++ tests/utils/test_config.py | 3 ++- 3 files changed, 19 insertions(+), 9 deletions(-) diff --git a/tests/cli/test_console.py b/tests/cli/test_console.py index 4e0c3e62cd..1a3180e674 100644 --- a/tests/cli/test_console.py +++ b/tests/cli/test_console.py @@ -134,14 +134,17 @@ def update_namespace_properties( MOCK_ENVIRONMENT = {"PYICEBERG_CATALOG__PRODUCTION__URI": "test://doesnotexist"} -def test_missing_uri() -> None: - runner = CliRunner() - result = runner.invoke(run, ["list"]) - assert result.exit_code == 1 - assert ( - result.output - == "URI missing, please provide using --uri, the config or environment variable \nPYICEBERG_CATALOG__DEFAULT__URI\n" - ) +def test_missing_uri(empty_home_dir_path: str) -> None: + + # mock to prevent parsing ~/.pyiceberg.yaml or {PYICEBERG_HOME}/.pyiceberg.yaml + with mock.patch.dict(os.environ, {"HOME": empty_home_dir_path, "PYICEBERG_HOME": empty_home_dir_path}): + runner = CliRunner() + result = runner.invoke(run, ["list"]) + assert result.exit_code == 1 + assert ( + result.output + == "URI missing, please provide using --uri, the config or environment variable \nPYICEBERG_CATALOG__DEFAULT__URI\n" + ) @mock.patch.dict(os.environ, MOCK_ENVIRONMENT) diff --git a/tests/conftest.py b/tests/conftest.py index bab7b87bcd..fffc50547f 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -1259,3 +1259,9 @@ def fixture_glue(_aws_credentials: None) -> Generator[boto3.client, None, None]: """Mocked glue client""" with mock_glue(): yield boto3.client("glue", region_name="us-east-1") + + +@pytest.fixture(scope="session") +def empty_home_dir_path(tmp_path_factory: pytest.TempPathFactory) -> str: + home_path = str(tmp_path_factory.mktemp("home")) + return home_path diff --git a/tests/utils/test_config.py b/tests/utils/test_config.py index 93831a716e..83b5a76295 100644 --- a/tests/utils/test_config.py +++ b/tests/utils/test_config.py @@ -17,6 +17,7 @@ import os from unittest import mock +import pytest import yaml from pyiceberg.utils.config import Config, _lowercase_dictionary_keys @@ -39,7 +40,7 @@ def test_from_environment_variables_uppercase() -> None: assert Config().get_catalog_config("PRODUCTION") == {"uri": "https://service.io/api"} -def test_from_configuration_files(tmp_path_factory) -> None: # type: ignore +def test_from_configuration_files(tmp_path_factory: pytest.TempPathFactory) -> None: config_path = str(tmp_path_factory.mktemp("config")) with open(f"{config_path}/.pyiceberg.yaml", "w", encoding="utf-8") as file: yaml.dump({"catalog": {"production": {"uri": "https://service.io/api"}}}, file) From 6f32f72fc7293c5deeeac18a5074460cb964db41 Mon Sep 17 00:00:00 2001 From: Fokko Driesprong Date: Sun, 18 Dec 2022 21:10:47 +0100 Subject: [PATCH 306/642] Python: Support for UUID (#6446) --- pyiceberg/avro/decoder.py | 5 ++ pyiceberg/avro/reader.py | 100 +++++++++------------------ pyiceberg/avro/resolver.py | 13 ++-- pyiceberg/schema.py | 2 +- pyiceberg/utils/schema_conversion.py | 2 +- tests/avro/test_decoder.py | 7 ++ tests/avro/test_reader.py | 38 +++++----- 7 files changed, 75 insertions(+), 92 deletions(-) diff --git a/pyiceberg/avro/decoder.py b/pyiceberg/avro/decoder.py index 710b888d40..cef9de7c7b 100644 --- a/pyiceberg/avro/decoder.py +++ b/pyiceberg/avro/decoder.py @@ -18,6 +18,7 @@ import struct from datetime import date, datetime, time from io import SEEK_CUR +from uuid import UUID from pyiceberg.io import InputStream from pyiceberg.utils.datetime import ( @@ -136,6 +137,10 @@ def read_date_from_int(self) -> date: """ return days_to_date(self.read_int()) + def read_uuid_from_fixed(self) -> UUID: + """Reads a UUID as a fixed[16]""" + return UUID(bytes=self.read(16)) + def read_time_millis(self) -> time: """ int is decoded as python time object which represents diff --git a/pyiceberg/avro/reader.py b/pyiceberg/avro/reader.py index a7631d4888..6ea2b07ced 100644 --- a/pyiceberg/avro/reader.py +++ b/pyiceberg/avro/reader.py @@ -30,7 +30,6 @@ from dataclasses import field as dataclassfield from datetime import date, datetime, time from decimal import Decimal -from functools import singledispatch from typing import ( Any, Callable, @@ -43,7 +42,7 @@ from uuid import UUID from pyiceberg.avro.decoder import BinaryDecoder -from pyiceberg.schema import Schema, SchemaVisitor +from pyiceberg.schema import Schema, SchemaVisitorPerPrimitiveType from pyiceberg.typedef import StructProtocol from pyiceberg.types import ( BinaryType, @@ -58,12 +57,12 @@ LongType, MapType, NestedField, - PrimitiveType, StringType, StructType, TimestampType, TimestamptzType, TimeType, + UUIDType, ) from pyiceberg.utils.singleton import Singleton @@ -209,10 +208,10 @@ def skip(self, decoder: BinaryDecoder) -> None: class UUIDReader(Reader): def read(self, decoder: BinaryDecoder) -> UUID: - return UUID(decoder.read_utf8()) + return decoder.read_uuid_from_fixed() def skip(self, decoder: BinaryDecoder) -> None: - decoder.skip_utf8() + decoder.skip(16) @dataclass(frozen=True) @@ -339,7 +338,7 @@ def skip() -> None: _skip_map_array(decoder, skip) -class ConstructReader(SchemaVisitor[Reader]): +class ConstructReader(SchemaVisitorPerPrimitiveType[Reader]): def schema(self, schema: Schema, struct_result: Reader) -> Reader: return struct_result @@ -357,77 +356,44 @@ def map(self, map_type: MapType, key_result: Reader, value_result: Reader) -> Re value_reader = value_result if map_type.value_required else OptionReader(value_result) return MapReader(key_result, value_reader) - def primitive(self, primitive: PrimitiveType) -> Reader: - return primitive_reader(primitive) - - -@singledispatch -def primitive_reader(primitive: PrimitiveType) -> Reader: - raise ValueError(f"Unknown type: {primitive}") - - -@primitive_reader.register -def _(primitive: FixedType) -> Reader: - return FixedReader(len(primitive)) - - -@primitive_reader.register -def _(primitive: DecimalType) -> Reader: - return DecimalReader(primitive.precision, primitive.scale) - - -@primitive_reader.register -def _(_: BooleanType) -> Reader: - return BooleanReader() - - -@primitive_reader.register -def _(_: IntegerType) -> Reader: - return IntegerReader() - - -@primitive_reader.register -def _(_: LongType) -> Reader: - # Ints and longs are encoded the same way in Python and - # also binary compatible in Avro - return IntegerReader() - - -@primitive_reader.register -def _(_: FloatType) -> Reader: - return FloatReader() - + def visit_fixed(self, fixed_type: FixedType) -> Reader: + return FixedReader(len(fixed_type)) -@primitive_reader.register -def _(_: DoubleType) -> Reader: - return DoubleReader() + def visit_decimal(self, decimal_type: DecimalType) -> Reader: + return DecimalReader(decimal_type.precision, decimal_type.scale) + def visit_boolean(self, boolean_type: BooleanType) -> Reader: + return BooleanReader() -@primitive_reader.register -def _(_: DateType) -> Reader: - return DateReader() + def visit_integer(self, integer_type: IntegerType) -> Reader: + return IntegerReader() + def visit_long(self, long_type: LongType) -> Reader: + return IntegerReader() -@primitive_reader.register -def _(_: TimeType) -> Reader: - return TimeReader() + def visit_float(self, float_type: FloatType) -> Reader: + return FloatReader() + def visit_double(self, double_type: DoubleType) -> Reader: + return DoubleReader() -@primitive_reader.register -def _(_: TimestampType) -> Reader: - return TimestampReader() + def visit_date(self, date_type: DateType) -> Reader: + return DateReader() + def visit_time(self, time_type: TimeType) -> Reader: + return TimeReader() -@primitive_reader.register -def _(_: TimestamptzType) -> Reader: - return TimestamptzReader() + def visit_timestamp(self, timestamp_type: TimestampType) -> Reader: + return TimestampReader() + def visit_timestampz(self, timestamptz_type: TimestamptzType) -> Reader: + return TimestamptzReader() -@primitive_reader.register -def _(_: StringType) -> Reader: - return StringReader() + def visit_string(self, string_type: StringType) -> Reader: + return StringReader() + def visit_uuid(self, uuid_type: UUIDType) -> Reader: + return UUIDReader() -@primitive_reader.register -def _(_: BinaryType) -> Reader: - return BinaryReader() + def visit_binary(self, binary_ype: BinaryType) -> Reader: + return BinaryReader() diff --git a/pyiceberg/avro/resolver.py b/pyiceberg/avro/resolver.py index 1b710ea191..ca559a2998 100644 --- a/pyiceberg/avro/resolver.py +++ b/pyiceberg/avro/resolver.py @@ -30,7 +30,6 @@ OptionReader, Reader, StructReader, - primitive_reader, ) from pyiceberg.schema import Schema, visit from pyiceberg.types import ( @@ -133,7 +132,7 @@ def _(file_type: PrimitiveType, read_type: IcebergType) -> Reader: # In the case of a promotion, we want to check if it is valid if file_type != read_type: return promote(file_type, read_type) - return primitive_reader(read_type) + return visit(read_type, ConstructReader()) @singledispatch @@ -154,7 +153,7 @@ def promote(file_type: IcebergType, read_type: IcebergType) -> Reader: def _(file_type: IntegerType, read_type: IcebergType) -> Reader: if isinstance(read_type, LongType): # Ints/Longs are binary compatible in Avro, so this is okay - return primitive_reader(read_type) + return visit(read_type, ConstructReader()) else: raise ResolveException(f"Cannot promote an int to {read_type}") @@ -163,7 +162,7 @@ def _(file_type: IntegerType, read_type: IcebergType) -> Reader: def _(file_type: FloatType, read_type: IcebergType) -> Reader: if isinstance(read_type, DoubleType): # We should just read the float, and return it, since it both returns a float - return primitive_reader(file_type) + return visit(file_type, ConstructReader()) else: raise ResolveException(f"Cannot promote an float to {read_type}") @@ -171,7 +170,7 @@ def _(file_type: FloatType, read_type: IcebergType) -> Reader: @promote.register(StringType) def _(file_type: StringType, read_type: IcebergType) -> Reader: if isinstance(read_type, BinaryType): - return primitive_reader(read_type) + return visit(read_type, ConstructReader()) else: raise ResolveException(f"Cannot promote an string to {read_type}") @@ -179,7 +178,7 @@ def _(file_type: StringType, read_type: IcebergType) -> Reader: @promote.register(BinaryType) def _(file_type: BinaryType, read_type: IcebergType) -> Reader: if isinstance(read_type, StringType): - return primitive_reader(read_type) + return visit(read_type, ConstructReader()) else: raise ResolveException(f"Cannot promote an binary to {read_type}") @@ -188,7 +187,7 @@ def _(file_type: BinaryType, read_type: IcebergType) -> Reader: def _(file_type: DecimalType, read_type: IcebergType) -> Reader: if isinstance(read_type, DecimalType): if file_type.precision <= read_type.precision and file_type.scale == file_type.scale: - return primitive_reader(read_type) + return visit(read_type, ConstructReader()) else: raise ResolveException(f"Cannot reduce precision from {file_type} to {read_type}") else: diff --git a/pyiceberg/schema.py b/pyiceberg/schema.py index ae11b5ba32..b265c444f1 100644 --- a/pyiceberg/schema.py +++ b/pyiceberg/schema.py @@ -363,7 +363,7 @@ def primitive(self, primitive: PrimitiveType) -> T: elif isinstance(primitive, BinaryType): return self.visit_binary(primitive) else: - raise ValueError(f"Found unknown type: {primitive}") + raise ValueError(f"Unknown type: {primitive}") @abstractmethod def visit_fixed(self, fixed_type: FixedType) -> T: diff --git a/pyiceberg/utils/schema_conversion.py b/pyiceberg/utils/schema_conversion.py index c2bb5c93a5..2f9c321a13 100644 --- a/pyiceberg/utils/schema_conversion.py +++ b/pyiceberg/utils/schema_conversion.py @@ -68,7 +68,7 @@ ("timestamp-millis", "long"): TimestampType(), ("time-micros", "int"): TimeType(), ("timestamp-micros", "long"): TimestampType(), - ("uuid", "string"): UUIDType(), + ("uuid", "fixed"): UUIDType(), } diff --git a/tests/avro/test_decoder.py b/tests/avro/test_decoder.py index 9bbd67db74..ac38f38868 100644 --- a/tests/avro/test_decoder.py +++ b/tests/avro/test_decoder.py @@ -21,6 +21,7 @@ from io import SEEK_SET from types import TracebackType from typing import Optional, Type +from uuid import UUID import pytest @@ -176,6 +177,12 @@ def test_read_date() -> None: assert decoder.read_date_from_int() == date(1991, 12, 27) +def test_read_uuid_from_fixed() -> None: + mis = MemoryInputStream(b"\x12\x34\x56\x78" * 4) + decoder = BinaryDecoder(mis) + assert decoder.read_uuid_from_fixed() == UUID("{12345678-1234-5678-1234-567812345678}") + + def test_read_time_millis() -> None: mis = MemoryInputStream(b"\xBC\x7D") decoder = BinaryDecoder(mis) diff --git a/tests/avro/test_reader.py b/tests/avro/test_reader.py index da1613be95..fc37f9f9a3 100644 --- a/tests/avro/test_reader.py +++ b/tests/avro/test_reader.py @@ -24,6 +24,7 @@ AvroStruct, BinaryReader, BooleanReader, + ConstructReader, DateReader, DecimalReader, DoubleReader, @@ -34,10 +35,10 @@ TimeReader, TimestampReader, TimestamptzReader, - primitive_reader, + UUIDReader, ) from pyiceberg.manifest import _convert_pos_to_dict -from pyiceberg.schema import Schema +from pyiceberg.schema import Schema, visit from pyiceberg.types import ( BinaryType, BooleanType, @@ -57,6 +58,7 @@ TimestampType, TimestamptzType, TimeType, + UUIDType, ) from tests.io.test_io import LocalInputFile @@ -442,55 +444,55 @@ def test_null_struct_convert_pos_to_dict() -> None: def test_fixed_reader() -> None: - assert primitive_reader(FixedType(22)) == FixedReader(22) + assert visit(FixedType(22), ConstructReader()) == FixedReader(22) def test_decimal_reader() -> None: - assert primitive_reader(DecimalType(19, 25)) == DecimalReader(19, 25) + assert visit(DecimalType(19, 25), ConstructReader()) == DecimalReader(19, 25) def test_boolean_reader() -> None: - assert primitive_reader(BooleanType()) == BooleanReader() + assert visit(BooleanType(), ConstructReader()) == BooleanReader() def test_integer_reader() -> None: - assert primitive_reader(IntegerType()) == IntegerReader() + assert visit(IntegerType(), ConstructReader()) == IntegerReader() def test_long_reader() -> None: - assert primitive_reader(LongType()) == IntegerReader() + assert visit(LongType(), ConstructReader()) == IntegerReader() def test_float_reader() -> None: - assert primitive_reader(FloatType()) == FloatReader() + assert visit(FloatType(), ConstructReader()) == FloatReader() def test_double_reader() -> None: - assert primitive_reader(DoubleType()) == DoubleReader() + assert visit(DoubleType(), ConstructReader()) == DoubleReader() def test_date_reader() -> None: - assert primitive_reader(DateType()) == DateReader() + assert visit(DateType(), ConstructReader()) == DateReader() def test_time_reader() -> None: - assert primitive_reader(TimeType()) == TimeReader() + assert visit(TimeType(), ConstructReader()) == TimeReader() def test_timestamp_reader() -> None: - assert primitive_reader(TimestampType()) == TimestampReader() + assert visit(TimestampType(), ConstructReader()) == TimestampReader() def test_timestamptz_reader() -> None: - assert primitive_reader(TimestamptzType()) == TimestamptzReader() + assert visit(TimestamptzType(), ConstructReader()) == TimestamptzReader() def test_string_reader() -> None: - assert primitive_reader(StringType()) == StringReader() + assert visit(StringType(), ConstructReader()) == StringReader() def test_binary_reader() -> None: - assert primitive_reader(BinaryType()) == BinaryReader() + assert visit(BinaryType(), ConstructReader()) == BinaryReader() def test_unknown_type() -> None: @@ -498,6 +500,10 @@ class UnknownType(PrimitiveType): __root__ = "UnknownType" with pytest.raises(ValueError) as exc_info: - primitive_reader(UnknownType()) + visit(UnknownType(), ConstructReader()) assert "Unknown type:" in str(exc_info.value) + + +def test_uuid_reader() -> None: + assert visit(UUIDType(), ConstructReader()) == UUIDReader() From 39cc6446a6832afc67b9a43a703ca7c912ed81b7 Mon Sep 17 00:00:00 2001 From: Fokko Driesprong Date: Sun, 18 Dec 2022 23:26:26 +0100 Subject: [PATCH 307/642] Python: Reduce the use of mock objects (#6438) --- pyiceberg/avro/file.py | 7 +- pyiceberg/avro/reader.py | 17 +- pyiceberg/manifest.py | 10 +- pyiceberg/typedef.py | 20 ++ tests/avro/test_reader.py | 262 +++++++++++++------------- tests/catalog/test_base.py | 24 ++- tests/catalog/test_hive.py | 5 +- tests/conftest.py | 55 +----- tests/expressions/test_evaluator.py | 18 +- tests/expressions/test_expressions.py | 96 ++++++---- tests/expressions/test_visitors.py | 65 ++----- tests/io/test_fsspec.py | 9 +- tests/io/test_io.py | 198 +++---------------- tests/table/test_metadata.py | 70 +++---- tests/utils/test_manifest.py | 6 +- 15 files changed, 335 insertions(+), 527 deletions(-) diff --git a/pyiceberg/avro/file.py b/pyiceberg/avro/file.py index 68887a61ec..cc725e9db7 100644 --- a/pyiceberg/avro/file.py +++ b/pyiceberg/avro/file.py @@ -28,11 +28,12 @@ from pyiceberg.avro.codecs import KNOWN_CODECS, Codec from pyiceberg.avro.decoder import BinaryDecoder -from pyiceberg.avro.reader import AvroStruct, ConstructReader, Reader +from pyiceberg.avro.reader import ConstructReader, Reader from pyiceberg.avro.resolver import resolve from pyiceberg.io import InputFile, InputStream from pyiceberg.io.memory import MemoryInputStream from pyiceberg.schema import Schema, visit +from pyiceberg.typedef import Record from pyiceberg.types import ( FixedType, MapType, @@ -101,7 +102,7 @@ def __iter__(self) -> Block: def has_next(self) -> bool: return self.position < self.block_records - def __next__(self) -> AvroStruct: + def __next__(self) -> Record: if self.has_next(): self.position += 1 return self.reader.read(self.block_decoder) @@ -168,7 +169,7 @@ def _read_block(self) -> int: ) return block_records - def __next__(self) -> AvroStruct: + def __next__(self) -> Record: if self.block and self.block.has_next(): return next(self.block) diff --git a/pyiceberg/avro/reader.py b/pyiceberg/avro/reader.py index 6ea2b07ced..136498ed34 100644 --- a/pyiceberg/avro/reader.py +++ b/pyiceberg/avro/reader.py @@ -43,7 +43,7 @@ from pyiceberg.avro.decoder import BinaryDecoder from pyiceberg.schema import Schema, SchemaVisitorPerPrimitiveType -from pyiceberg.typedef import StructProtocol +from pyiceberg.typedef import Record, StructProtocol from pyiceberg.types import ( BinaryType, BooleanType, @@ -103,17 +103,6 @@ def _skip_map_array(decoder: BinaryDecoder, skip_entry: Callable[[], None]) -> N block_count = decoder.read_int() -@dataclass(frozen=True) -class AvroStruct(StructProtocol): - _data: List[Union[Any, StructProtocol]] = dataclassfield() - - def set(self, pos: int, value: Any) -> None: - self._data[pos] = value - - def get(self, pos: int) -> Any: - return self._data[pos] - - class Reader(Singleton): @abstractmethod def read(self, decoder: BinaryDecoder) -> Any: @@ -275,7 +264,7 @@ def skip(self, decoder: BinaryDecoder) -> None: class StructReader(Reader): fields: Tuple[Tuple[Optional[int], Reader], ...] = dataclassfield() - def read(self, decoder: BinaryDecoder) -> AvroStruct: + def read(self, decoder: BinaryDecoder) -> Record: result: List[Union[Any, StructProtocol]] = [None] * len(self.fields) for (pos, field) in self.fields: if pos is not None: @@ -283,7 +272,7 @@ def read(self, decoder: BinaryDecoder) -> AvroStruct: else: field.skip(decoder) - return AvroStruct(result) + return Record(*result) def skip(self, decoder: BinaryDecoder) -> None: for _, field in self.fields: diff --git a/pyiceberg/manifest.py b/pyiceberg/manifest.py index 75ec0b6a14..9f7f394159 100644 --- a/pyiceberg/manifest.py +++ b/pyiceberg/manifest.py @@ -28,9 +28,9 @@ from pydantic import Field from pyiceberg.avro.file import AvroFile -from pyiceberg.avro.reader import AvroStruct from pyiceberg.io import FileIO, InputFile from pyiceberg.schema import Schema +from pyiceberg.typedef import Record from pyiceberg.types import ( IcebergType, ListType, @@ -158,14 +158,14 @@ def read_manifest_list(input_file: InputFile) -> Iterator[ManifestFile]: @singledispatch -def _convert_pos_to_dict(schema: Union[Schema, IcebergType], struct: AvroStruct) -> Dict[str, Any]: +def _convert_pos_to_dict(schema: Union[Schema, IcebergType], struct: Record) -> Dict[str, Any]: """Converts the positions in the field names This makes it easy to map it onto a Pydantic model. Might change later on depending on the performance Args: schema (Schema | IcebergType): The schema of the file - struct (AvroStruct): The struct containing the data by positions + struct (Record): The struct containing the data by positions Raises: NotImplementedError: If attempting to handle an unknown type in the schema @@ -174,12 +174,12 @@ def _convert_pos_to_dict(schema: Union[Schema, IcebergType], struct: AvroStruct) @_convert_pos_to_dict.register -def _(schema: Schema, struct: AvroStruct) -> Dict[str, Any]: +def _(schema: Schema, struct: Record) -> Dict[str, Any]: return _convert_pos_to_dict(schema.as_struct(), struct) @_convert_pos_to_dict.register -def _(struct_type: StructType, values: AvroStruct) -> Dict[str, Any]: +def _(struct_type: StructType, values: Record) -> Dict[str, Any]: """Iterates over all the fields in the dict, and gets the data from the struct""" return ( {field.name: _convert_pos_to_dict(field.field_type, values.get(pos)) for pos, field in enumerate(struct_type.fields)} diff --git a/pyiceberg/typedef.py b/pyiceberg/typedef.py index 436b1a6e79..6ad668f006 100644 --- a/pyiceberg/typedef.py +++ b/pyiceberg/typedef.py @@ -14,12 +14,15 @@ # KIND, either express or implied. See the License for the # specific language governing permissions and limitations # under the License. +from __future__ import annotations + from abc import abstractmethod from decimal import Decimal from typing import ( Any, Callable, Dict, + List, Protocol, Tuple, TypeVar, @@ -78,3 +81,20 @@ def get(self, pos: int) -> Any: @abstractmethod def set(self, pos: int, value: Any) -> None: ... + + +class Record(StructProtocol): + _data: List[Union[Any, StructProtocol]] + + def __init__(self, *data: Union[Any, StructProtocol]) -> None: + self._data = list(data) + + def set(self, pos: int, value: Any) -> None: + self._data[pos] = value + + def get(self, pos: int) -> Any: + return self._data[pos] + + def __eq__(self, other: Any) -> bool: + # For testing + return True if isinstance(other, Record) and other._data == self._data else False diff --git a/tests/avro/test_reader.py b/tests/avro/test_reader.py index fc37f9f9a3..2528882333 100644 --- a/tests/avro/test_reader.py +++ b/tests/avro/test_reader.py @@ -21,7 +21,6 @@ from pyiceberg.avro.file import AvroFile from pyiceberg.avro.reader import ( - AvroStruct, BinaryReader, BooleanReader, ConstructReader, @@ -37,8 +36,10 @@ TimestamptzReader, UUIDReader, ) +from pyiceberg.io.pyarrow import PyArrowFileIO from pyiceberg.manifest import _convert_pos_to_dict from pyiceberg.schema import Schema, visit +from pyiceberg.typedef import Record from pyiceberg.types import ( BinaryType, BooleanType, @@ -60,11 +61,10 @@ TimeType, UUIDType, ) -from tests.io.test_io import LocalInputFile def test_read_header(generated_manifest_entry_file: str, iceberg_manifest_entry_schema: Schema) -> None: - with AvroFile(LocalInputFile(generated_manifest_entry_file)) as reader: + with AvroFile(PyArrowFileIO().new_input(generated_manifest_entry_file)) as reader: header = reader._read_header() assert header.magic == b"Obj\x01" @@ -257,147 +257,141 @@ def test_read_header(generated_manifest_entry_file: str, iceberg_manifest_entry_ def test_read_manifest_entry_file(generated_manifest_entry_file: str) -> None: - with AvroFile(LocalInputFile(generated_manifest_entry_file)) as reader: + with AvroFile(PyArrowFileIO().new_input(generated_manifest_entry_file)) as reader: # Consume the generator records = list(reader) assert len(records) == 2, f"Expected 2 records, got {len(records)}" - assert records[0] == AvroStruct( - _data=[ - 1, - 8744736658442914487, - AvroStruct( - _data=[ - "/home/iceberg/warehouse/nyc/taxis_partitioned/data/VendorID=null/00000-633-d8a4223e-dc97-45a1-86e1-adaba6e8abd7-00001.parquet", - "PARQUET", - AvroStruct(_data=[None]), - 19513, - 388872, - 67108864, - { - 1: 53, - 2: 98153, - 3: 98693, - 4: 53, - 5: 53, - 6: 53, - 7: 17425, - 8: 18528, - 9: 53, - 10: 44788, - 11: 35571, - 12: 53, - 13: 1243, - 14: 2355, - 15: 12750, - 16: 4029, - 17: 110, - 18: 47194, - 19: 2948, - }, - { - 1: 19513, - 2: 19513, - 3: 19513, - 4: 19513, - 5: 19513, - 6: 19513, - 7: 19513, - 8: 19513, - 9: 19513, - 10: 19513, - 11: 19513, - 12: 19513, - 13: 19513, - 14: 19513, - 15: 19513, - 16: 19513, - 17: 19513, - 18: 19513, - 19: 19513, - }, - { - 1: 19513, - 2: 0, - 3: 0, - 4: 19513, - 5: 19513, - 6: 19513, - 7: 0, - 8: 0, - 9: 19513, - 10: 0, - 11: 0, - 12: 19513, - 13: 0, - 14: 0, - 15: 0, - 16: 0, - 17: 0, - 18: 0, - 19: 0, - }, - {16: 0, 17: 0, 18: 0, 19: 0, 10: 0, 11: 0, 12: 0, 13: 0, 14: 0, 15: 0}, - { - 2: b"2020-04-01 00:00", - 3: b"2020-04-01 00:12", - 7: b"\x03\x00\x00\x00", - 8: b"\x01\x00\x00\x00", - 10: b"\xf6(\\\x8f\xc2\x05S\xc0", - 11: b"\x00\x00\x00\x00\x00\x00\x00\x00", - 13: b"\x00\x00\x00\x00\x00\x00\x00\x00", - 14: b"\x00\x00\x00\x00\x00\x00\xe0\xbf", - 15: b")\\\x8f\xc2\xf5(\x08\xc0", - 16: b"\x00\x00\x00\x00\x00\x00\x00\x00", - 17: b"\x00\x00\x00\x00\x00\x00\x00\x00", - 18: b"\xf6(\\\x8f\xc2\xc5S\xc0", - 19: b"\x00\x00\x00\x00\x00\x00\x04\xc0", - }, - { - 2: b"2020-04-30 23:5:", - 3: b"2020-05-01 00:41", - 7: b"\t\x01\x00\x00", - 8: b"\t\x01\x00\x00", - 10: b"\xcd\xcc\xcc\xcc\xcc,_@", - 11: b"\x1f\x85\xebQ\\\xe2\xfe@", - 13: b"\x00\x00\x00\x00\x00\x00\x12@", - 14: b"\x00\x00\x00\x00\x00\x00\xe0?", - 15: b"q=\n\xd7\xa3\xf01@", - 16: b"\x00\x00\x00\x00\x00`B@", - 17: b"333333\xd3?", - 18: b"\x00\x00\x00\x00\x00\x18b@", - 19: b"\x00\x00\x00\x00\x00\x00\x04@", - }, - None, - [4], - 0, - ] - ), - ] + assert records[0] == Record( + 1, + 8744736658442914487, + Record( + "/home/iceberg/warehouse/nyc/taxis_partitioned/data/VendorID=null/00000-633-d8a4223e-dc97-45a1-86e1-adaba6e8abd7-00001.parquet", + "PARQUET", + Record(None), + 19513, + 388872, + 67108864, + { + 1: 53, + 2: 98153, + 3: 98693, + 4: 53, + 5: 53, + 6: 53, + 7: 17425, + 8: 18528, + 9: 53, + 10: 44788, + 11: 35571, + 12: 53, + 13: 1243, + 14: 2355, + 15: 12750, + 16: 4029, + 17: 110, + 18: 47194, + 19: 2948, + }, + { + 1: 19513, + 2: 19513, + 3: 19513, + 4: 19513, + 5: 19513, + 6: 19513, + 7: 19513, + 8: 19513, + 9: 19513, + 10: 19513, + 11: 19513, + 12: 19513, + 13: 19513, + 14: 19513, + 15: 19513, + 16: 19513, + 17: 19513, + 18: 19513, + 19: 19513, + }, + { + 1: 19513, + 2: 0, + 3: 0, + 4: 19513, + 5: 19513, + 6: 19513, + 7: 0, + 8: 0, + 9: 19513, + 10: 0, + 11: 0, + 12: 19513, + 13: 0, + 14: 0, + 15: 0, + 16: 0, + 17: 0, + 18: 0, + 19: 0, + }, + {16: 0, 17: 0, 18: 0, 19: 0, 10: 0, 11: 0, 12: 0, 13: 0, 14: 0, 15: 0}, + { + 2: b"2020-04-01 00:00", + 3: b"2020-04-01 00:12", + 7: b"\x03\x00\x00\x00", + 8: b"\x01\x00\x00\x00", + 10: b"\xf6(\\\x8f\xc2\x05S\xc0", + 11: b"\x00\x00\x00\x00\x00\x00\x00\x00", + 13: b"\x00\x00\x00\x00\x00\x00\x00\x00", + 14: b"\x00\x00\x00\x00\x00\x00\xe0\xbf", + 15: b")\\\x8f\xc2\xf5(\x08\xc0", + 16: b"\x00\x00\x00\x00\x00\x00\x00\x00", + 17: b"\x00\x00\x00\x00\x00\x00\x00\x00", + 18: b"\xf6(\\\x8f\xc2\xc5S\xc0", + 19: b"\x00\x00\x00\x00\x00\x00\x04\xc0", + }, + { + 2: b"2020-04-30 23:5:", + 3: b"2020-05-01 00:41", + 7: b"\t\x01\x00\x00", + 8: b"\t\x01\x00\x00", + 10: b"\xcd\xcc\xcc\xcc\xcc,_@", + 11: b"\x1f\x85\xebQ\\\xe2\xfe@", + 13: b"\x00\x00\x00\x00\x00\x00\x12@", + 14: b"\x00\x00\x00\x00\x00\x00\xe0?", + 15: b"q=\n\xd7\xa3\xf01@", + 16: b"\x00\x00\x00\x00\x00`B@", + 17: b"333333\xd3?", + 18: b"\x00\x00\x00\x00\x00\x18b@", + 19: b"\x00\x00\x00\x00\x00\x00\x04@", + }, + None, + [4], + 0, + ), ) def test_read_manifest_file_file(generated_manifest_file_file: str) -> None: - with AvroFile(LocalInputFile(generated_manifest_file_file)) as reader: + with AvroFile(PyArrowFileIO().new_input(generated_manifest_file_file)) as reader: # Consume the generator records = list(reader) assert len(records) == 1, f"Expected 1 records, got {len(records)}" actual = records[0] - expected = AvroStruct( - _data=[ - actual.get(0), - 7989, - 0, - 9182715666859759686, - 3, - 0, - 0, - [AvroStruct(_data=[True, False, b"\x01\x00\x00\x00", b"\x02\x00\x00\x00"])], - 237993, - 0, - 0, - ] + expected = Record( + actual.get(0), + 7989, + 0, + 9182715666859759686, + 3, + 0, + 0, + [Record(True, False, b"\x01\x00\x00\x00", b"\x02\x00\x00\x00")], + 237993, + 0, + 0, ) assert actual == expected @@ -407,7 +401,7 @@ def test_null_list_convert_pos_to_dict() -> None: Schema( NestedField(name="field", field_id=1, field_type=ListType(element_id=2, element=StringType(), element_required=False)) ), - AvroStruct([None]), + Record(None), ) assert data["field"] is None @@ -421,7 +415,7 @@ def test_null_dict_convert_pos_to_dict() -> None: field_type=MapType(key_id=2, key_type=StringType(), value_id=3, value_type=StringType(), value_required=False), ) ), - AvroStruct([None]), + Record(None), ) assert data["field"] is None @@ -438,7 +432,7 @@ def test_null_struct_convert_pos_to_dict() -> None: required=False, ) ), - AvroStruct([None]), + Record(None), ) assert data["field"] is None diff --git a/tests/catalog/test_base.py b/tests/catalog/test_base.py index ab9557448e..7281ff839c 100644 --- a/tests/catalog/test_base.py +++ b/tests/catalog/test_base.py @@ -42,10 +42,10 @@ from pyiceberg.partitioning import UNPARTITIONED_PARTITION_SPEC, PartitionField, PartitionSpec from pyiceberg.schema import Schema from pyiceberg.table import Table +from pyiceberg.table.metadata import TableMetadataV1 from pyiceberg.table.sorting import UNSORTED_SORT_ORDER, SortOrder from pyiceberg.transforms import IdentityTransform from pyiceberg.typedef import EMPTY_DICT -from tests.table.test_metadata import EXAMPLE_TABLE_METADATA_V1 class InMemoryCatalog(Catalog): @@ -80,7 +80,27 @@ def create_table( table = Table( identifier=identifier, - metadata=EXAMPLE_TABLE_METADATA_V1, + metadata=TableMetadataV1( + **{ + "format-version": 1, + "table-uuid": "d20125c8-7284-442c-9aea-15fee620737c", + "location": "s3://bucket/test/location", + "last-updated-ms": 1602638573874, + "last-column-id": 3, + "schema": { + "type": "struct", + "fields": [ + {"id": 1, "name": "x", "required": True, "type": "long"}, + {"id": 2, "name": "y", "required": True, "type": "long", "doc": "comment"}, + {"id": 3, "name": "z", "required": True, "type": "long"}, + ], + }, + "partition-spec": [{"name": "x", "transform": "identity", "source-id": 1, "field-id": 1000}], + "properties": {}, + "current-snapshot-id": -1, + "snapshots": [{"snapshot-id": 1925, "timestamp-ms": 1602638573822}], + } + ), metadata_location=f's3://warehouse/{"/".join(identifier)}/metadata/metadata.json', io=load_file_io(), ) diff --git a/tests/catalog/test_hive.py b/tests/catalog/test_hive.py index f9a47e516c..1cb7cba04b 100644 --- a/tests/catalog/test_hive.py +++ b/tests/catalog/test_hive.py @@ -42,6 +42,7 @@ NoSuchNamespaceError, NoSuchTableError, ) +from pyiceberg.io.pyarrow import PyArrowFileIO from pyiceberg.partitioning import PartitionField, PartitionSpec from pyiceberg.schema import Schema from pyiceberg.serializers import ToOutputFile @@ -68,7 +69,6 @@ NestedField, StringType, ) -from tests.conftest import LocalFileIO HIVE_CATALOG_NAME = "hive" HIVE_METASTORE_FAKE_URL = "thrift://unknown:9083" @@ -78,7 +78,8 @@ def hive_table(tmp_path_factory: pytest.TempPathFactory, example_table_metadata_v2: Dict[str, Any]) -> HiveTable: metadata_path = str(tmp_path_factory.mktemp("metadata") / f"{uuid.uuid4()}.metadata.json") metadata = TableMetadataV2(**example_table_metadata_v2) - ToOutputFile.table_metadata(metadata, LocalFileIO().new_output(str(metadata_path)), True) + + ToOutputFile.table_metadata(metadata, PyArrowFileIO().new_output(location=str(metadata_path)), True) return HiveTable( tableName="new_tabl2e", diff --git a/tests/conftest.py b/tests/conftest.py index fffc50547f..f809cfc503 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -31,8 +31,6 @@ Callable, Dict, Generator, - Type, - Union, ) from unittest.mock import MagicMock from urllib.parse import urlparse @@ -49,14 +47,9 @@ from moto import mock_glue, mock_s3 from pyiceberg import schema -from pyiceberg.io import ( - FileIO, - InputFile, - OutputFile, - OutputStream, - fsspec, -) +from pyiceberg.io import OutputFile, OutputStream, fsspec from pyiceberg.io.fsspec import FsspecFileIO +from pyiceberg.io.pyarrow import PyArrowFile, PyArrowFileIO from pyiceberg.schema import Schema from pyiceberg.types import ( BinaryType, @@ -72,7 +65,6 @@ StructType, ) from tests.catalog.test_base import InMemoryCatalog -from tests.io.test_io import LocalInputFile def pytest_addoption(parser: pytest.Parser) -> None: @@ -85,21 +77,6 @@ def pytest_addoption(parser: pytest.Parser) -> None: ) -class FooStruct: - """An example of an object that abides by StructProtocol""" - - content: Dict[int, Any] - - def __init__(self) -> None: - self.content = {} - - def get(self, pos: int) -> Any: - return self.content[pos] - - def set(self, pos: int, value: Any) -> None: - self.content[pos] = value - - @pytest.fixture(scope="session") def table_schema_simple() -> Schema: return schema.Schema( @@ -162,11 +139,6 @@ def table_schema_nested() -> Schema: ) -@pytest.fixture(scope="session") -def foo_struct() -> FooStruct: - return FooStruct() - - @pytest.fixture(scope="session") def all_avro_types() -> Dict[str, Any]: return { @@ -922,8 +894,8 @@ def __len__(self) -> int: def exists(self) -> bool: return os.path.exists(self._path) - def to_input_file(self) -> LocalInputFile: - return LocalInputFile(location=self.location) + def to_input_file(self) -> PyArrowFile: + return PyArrowFileIO().new_input(location=self.location) def create(self, overwrite: bool = False) -> OutputStream: output_file = open(self._path, "wb" if overwrite else "xb") @@ -932,25 +904,6 @@ def create(self, overwrite: bool = False) -> OutputStream: return output_file -class LocalFileIO(FileIO): - """A FileIO implementation for local files (for test use only)""" - - def new_input(self, location: str) -> LocalInputFile: - return LocalInputFile(location=location) - - def new_output(self, location: str) -> LocalOutputFile: - return LocalOutputFile(location=location) - - def delete(self, location: Union[str, InputFile, OutputFile]) -> None: - location = location.location if isinstance(location, (InputFile, OutputFile)) else location - os.remove(location) - - -@pytest.fixture(scope="session", autouse=True) -def LocalFileIOFixture() -> Type[LocalFileIO]: - return LocalFileIO - - @pytest.fixture(scope="session") def generated_manifest_entry_file(avro_schema_manifest_entry: Dict[str, Any]) -> Generator[str, None, None]: from fastavro import parse_schema, writer diff --git a/tests/expressions/test_evaluator.py b/tests/expressions/test_evaluator.py index 4d4025a268..95766b0024 100644 --- a/tests/expressions/test_evaluator.py +++ b/tests/expressions/test_evaluator.py @@ -14,8 +14,6 @@ # KIND, either express or implied. See the License for the # specific language governing permissions and limitations # under the License. -from typing import Any, List - from pyiceberg.expressions import ( AlwaysFalse, AlwaysTrue, @@ -37,7 +35,7 @@ ) from pyiceberg.expressions.visitors import expression_evaluator from pyiceberg.schema import Schema -from pyiceberg.typedef import StructProtocol +from pyiceberg.typedef import Record from pyiceberg.types import ( DoubleType, LongType, @@ -45,20 +43,6 @@ StringType, ) - -class Record(StructProtocol): - data: List[Any] - - def __init__(self, *values: Any) -> None: - self.data = list(values) - - def get(self, pos: int) -> Any: - return self.data[pos] - - def set(self, pos: int, value: Any) -> None: - self.data[pos] = value - - SIMPLE_SCHEMA = Schema( NestedField(id=1, name="id", field_type=LongType()), NestedField(id=2, name="data", field_type=StringType(), required=False) ) diff --git a/tests/expressions/test_expressions.py b/tests/expressions/test_expressions.py index 233b80a385..7b26d45a28 100644 --- a/tests/expressions/test_expressions.py +++ b/tests/expressions/test_expressions.py @@ -61,6 +61,7 @@ from pyiceberg.expressions.literals import Literal, literal from pyiceberg.expressions.visitors import _from_byte_buffer from pyiceberg.schema import Accessor, Schema +from pyiceberg.typedef import Record from pyiceberg.types import ( DoubleType, FloatType, @@ -70,8 +71,29 @@ NestedField, StringType, ) -from tests.conftest import FooStruct -from tests.expressions.test_visitors import ExpressionA, ExpressionB +from pyiceberg.utils.singleton import Singleton + + +class ExpressionA(BooleanExpression, Singleton): + def __invert__(self) -> BooleanExpression: + return ExpressionB() + + def __repr__(self) -> str: + return "ExpressionA()" + + def __str__(self) -> str: + return "testexpra" + + +class ExpressionB(BooleanExpression, Singleton): + def __invert__(self) -> BooleanExpression: + return ExpressionA() + + def __repr__(self) -> str: + return "ExpressionB()" + + def __str__(self) -> str: + return "testexprb" def test_isnull_inverse() -> None: @@ -580,36 +602,38 @@ def test_invert_always() -> None: assert ~AlwaysTrue() == AlwaysFalse() -def test_accessor_base_class(foo_struct: FooStruct) -> None: +def test_accessor_base_class() -> None: """Test retrieving a value at a position of a container using an accessor""" + struct = Record(*([None] * 12)) + uuid_value = uuid.uuid4() - foo_struct.set(0, "foo") - foo_struct.set(1, "bar") - foo_struct.set(2, "baz") - foo_struct.set(3, 1) - foo_struct.set(4, 2) - foo_struct.set(5, 3) - foo_struct.set(6, 1.234) - foo_struct.set(7, Decimal("1.234")) - foo_struct.set(8, uuid_value) - foo_struct.set(9, True) - foo_struct.set(10, False) - foo_struct.set(11, b"\x19\x04\x9e?") - - assert Accessor(position=0).get(foo_struct) == "foo" - assert Accessor(position=1).get(foo_struct) == "bar" - assert Accessor(position=2).get(foo_struct) == "baz" - assert Accessor(position=3).get(foo_struct) == 1 - assert Accessor(position=4).get(foo_struct) == 2 - assert Accessor(position=5).get(foo_struct) == 3 - assert Accessor(position=6).get(foo_struct) == 1.234 - assert Accessor(position=7).get(foo_struct) == Decimal("1.234") - assert Accessor(position=8).get(foo_struct) == uuid_value - assert Accessor(position=9).get(foo_struct) is True - assert Accessor(position=10).get(foo_struct) is False - assert Accessor(position=11).get(foo_struct) == b"\x19\x04\x9e?" + struct.set(0, "foo") + struct.set(1, "bar") + struct.set(2, "baz") + struct.set(3, 1) + struct.set(4, 2) + struct.set(5, 3) + struct.set(6, 1.234) + struct.set(7, Decimal("1.234")) + struct.set(8, uuid_value) + struct.set(9, True) + struct.set(10, False) + struct.set(11, b"\x19\x04\x9e?") + + assert Accessor(position=0).get(struct) == "foo" + assert Accessor(position=1).get(struct) == "bar" + assert Accessor(position=2).get(struct) == "baz" + assert Accessor(position=3).get(struct) == 1 + assert Accessor(position=4).get(struct) == 2 + assert Accessor(position=5).get(struct) == 3 + assert Accessor(position=6).get(struct) == 1.234 + assert Accessor(position=7).get(struct) == Decimal("1.234") + assert Accessor(position=8).get(struct) == uuid_value + assert Accessor(position=9).get(struct) is True + assert Accessor(position=10).get(struct) is False + assert Accessor(position=11).get(struct) == b"\x19\x04\x9e?" @pytest.fixture @@ -876,11 +900,13 @@ def test_less_than_or_equal() -> None: assert less_than_or_equal == eval(repr(less_than_or_equal)) -def test_bound_reference_eval(table_schema_simple: Schema, foo_struct: FooStruct) -> None: +def test_bound_reference_eval(table_schema_simple: Schema) -> None: """Test creating a BoundReference and evaluating it on a StructProtocol""" - foo_struct.set(pos=1, value="foovalue") - foo_struct.set(pos=2, value=123) - foo_struct.set(pos=3, value=True) + struct = Record(None, None, None, None) + + struct.set(pos=1, value="foovalue") + struct.set(pos=2, value=123) + struct.set(pos=3, value=True) position1_accessor = Accessor(position=1) position2_accessor = Accessor(position=2) @@ -894,9 +920,9 @@ def test_bound_reference_eval(table_schema_simple: Schema, foo_struct: FooStruct bound_ref2 = BoundReference(field=field2, accessor=position2_accessor) bound_ref3 = BoundReference(field=field3, accessor=position3_accessor) - assert bound_ref1.eval(foo_struct) == "foovalue" - assert bound_ref2.eval(foo_struct) == 123 - assert bound_ref3.eval(foo_struct) is True + assert bound_ref1.eval(struct) == "foovalue" + assert bound_ref2.eval(struct) == 123 + assert bound_ref3.eval(struct) is True def test_non_primitive_from_byte_buffer() -> None: diff --git a/tests/expressions/test_visitors.py b/tests/expressions/test_visitors.py index e179ccf05f..19915cc6b9 100644 --- a/tests/expressions/test_visitors.py +++ b/tests/expressions/test_visitors.py @@ -79,29 +79,6 @@ PrimitiveType, StringType, ) -from pyiceberg.utils.singleton import Singleton - - -class ExpressionA(BooleanExpression, Singleton): - def __invert__(self) -> BooleanExpression: - return ExpressionB() - - def __repr__(self) -> str: - return "ExpressionA()" - - def __str__(self) -> str: - return "testexpra" - - -class ExpressionB(BooleanExpression, Singleton): - def __invert__(self) -> BooleanExpression: - return ExpressionA() - - def __repr__(self) -> str: - return "ExpressionB()" - - def __str__(self) -> str: - return "testexprb" class ExampleVisitor(BooleanExpressionVisitor[List[str]]): @@ -135,33 +112,13 @@ def visit_or(self, left_result: List[str], right_result: List[str]) -> List[str] return self.visit_history def visit_unbound_predicate(self, predicate: UnboundPredicate[Any]) -> List[str]: - self.visit_history.append("UNBOUND PREDICATE") + self.visit_history.append(str(predicate.__class__.__name__).upper()) return self.visit_history def visit_bound_predicate(self, predicate: BoundPredicate[Any]) -> List[str]: - self.visit_history.append("BOUND PREDICATE") + self.visit_history.append(str(predicate.__class__.__name__).upper()) return self.visit_history - def visit_test_expression_a(self) -> List[str]: - self.visit_history.append("ExpressionA") - return self.visit_history - - def visit_test_expression_b(self) -> List[str]: - self.visit_history.append("ExpressionB") - return self.visit_history - - -@visit.register(ExpressionA) -def _(obj: ExpressionA, visitor: ExampleVisitor) -> List[str]: - """Visit a ExpressionA with a BooleanExpressionVisitor""" - return visitor.visit_test_expression_a() - - -@visit.register(ExpressionB) -def _(obj: ExpressionB, visitor: ExampleVisitor) -> List[str]: - """Visit a ExpressionB with a BooleanExpressionVisitor""" - return visitor.visit_test_expression_b() - class FooBoundBooleanExpressionVisitor(BoundBooleanExpressionVisitor[List[str]]): """A test implementation of a BoundBooleanExpressionVisitor @@ -250,26 +207,26 @@ def visit_or(self, left_result: List[str], right_result: List[str]) -> List[str] def test_boolean_expression_visitor() -> None: """Test post-order traversal of boolean expression visit method""" expr = And( - Or(Not(ExpressionA()), Not(ExpressionB()), ExpressionA(), ExpressionB()), - Not(ExpressionA()), - ExpressionB(), + Or(Not(EqualTo("a", 1)), Not(NotEqualTo("b", 0)), EqualTo("a", 1), NotEqualTo("b", 0)), + Not(EqualTo("a", 1)), + NotEqualTo("b", 0), ) visitor = ExampleVisitor() result = visit(expr, visitor=visitor) assert result == [ - "ExpressionA", + "EQUALTO", "NOT", - "ExpressionB", + "NOTEQUALTO", "NOT", "OR", - "ExpressionA", + "EQUALTO", "OR", - "ExpressionB", + "NOTEQUALTO", "OR", - "ExpressionA", + "EQUALTO", "NOT", "AND", - "ExpressionB", + "NOTEQUALTO", "AND", ] diff --git a/tests/io/test_fsspec.py b/tests/io/test_fsspec.py index 22cad25ea4..fff308b212 100644 --- a/tests/io/test_fsspec.py +++ b/tests/io/test_fsspec.py @@ -16,7 +16,6 @@ # under the License. import uuid -from typing import Generator import pytest from botocore.awsrequest import AWSRequest @@ -25,12 +24,12 @@ from pyiceberg.exceptions import SignError from pyiceberg.io import fsspec from pyiceberg.io.fsspec import FsspecFileIO, s3v4_rest_signer -from tests.io.test_io import LocalInputFile +from pyiceberg.io.pyarrow import PyArrowFileIO @pytest.mark.s3 def test_fsspec_new_input_file(fsspec_fileio: FsspecFileIO) -> None: - """Test creating a new input file from an fsspec file-io""" + """Test creating a new input file from a fsspec file-io""" filename = str(uuid.uuid4()) input_file = fsspec_fileio.new_input(f"s3://warehouse/{filename}") @@ -193,10 +192,10 @@ def test_fsspec_converting_an_outputfile_to_an_inputfile(fsspec_fileio: FsspecFi @pytest.mark.s3 -def test_writing_avro_file(generated_manifest_entry_file: Generator[str, None, None], fsspec_fileio: FsspecFileIO) -> None: +def test_writing_avro_file(generated_manifest_entry_file: str, fsspec_fileio: FsspecFileIO) -> None: """Test that bytes match when reading a local avro file, writing it using fsspec file-io, and then reading it again""" filename = str(uuid.uuid4()) - with LocalInputFile(generated_manifest_entry_file).open() as f: + with PyArrowFileIO().new_input(location=generated_manifest_entry_file).open() as f: b1 = f.read() with fsspec_fileio.new_output(location=f"s3://warehouse/{filename}").create() as out_f: out_f.write(b1) diff --git a/tests/io/test_io.py b/tests/io/test_io.py index a83cba1af3..6cb5dbc7d3 100644 --- a/tests/io/test_io.py +++ b/tests/io/test_io.py @@ -17,20 +17,12 @@ import os import tempfile -from typing import Type, Union -from unittest.mock import patch -from urllib.parse import ParseResult, urlparse import pytest from pyiceberg.io import ( ARROW_FILE_IO, PY_IO_IMPL, - FileIO, - InputFile, - InputStream, - OutputFile, - OutputStream, _import_file_io, load_file_io, ) @@ -38,102 +30,7 @@ from pyiceberg.io.pyarrow import PyArrowFileIO -class LocalInputFile(InputFile): - """An InputFile implementation for local files (for test use only)""" - - def __init__(self, location: str) -> None: - - parsed_location = urlparse(location) # Create a ParseResult from the uri - if parsed_location.scheme and parsed_location.scheme != "file": # Validate that a uri is provided with a scheme of `file` - raise ValueError("LocalInputFile location must have a scheme of `file`") - elif parsed_location.netloc: - raise ValueError(f"Network location is not allowed for LocalInputFile: {parsed_location.netloc}") - - super().__init__(location=location) - self._parsed_location = parsed_location - - @property - def parsed_location(self) -> ParseResult: - """The parsed location - - Returns: - ParseResult: The parsed results which has attributes `scheme`, `netloc`, `path`, - `params`, `query`, and `fragments`. - """ - return self._parsed_location - - def __len__(self) -> int: - return os.path.getsize(self.parsed_location.path) - - def exists(self) -> bool: - return os.path.exists(self.parsed_location.path) - - def open(self) -> InputStream: - input_file = open(self.parsed_location.path, "rb") - if not isinstance(input_file, InputStream): - raise TypeError("Object returned from LocalInputFile.open() does not match the OutputStream protocol.") - return input_file - - -class LocalOutputFile(OutputFile): - """An OutputFile implementation for local files (for test use only)""" - - def __init__(self, location: str) -> None: - parsed_location = urlparse(location) # Create a ParseResult from the uri - if parsed_location.scheme and parsed_location.scheme != "file": # Validate that a uri is provided with a scheme of `file` - raise ValueError("LocalOutputFile location must have a scheme of `file`") - elif parsed_location.netloc: - raise ValueError(f"Network location is not allowed for LocalOutputFile: {parsed_location.netloc}") - - super().__init__(location=location) - self._parsed_location = parsed_location - - @property - def parsed_location(self) -> ParseResult: - """The parsed location - - Returns: - ParseResult: The parsed results which has attributes `scheme`, `netloc`, `path`, - `params`, `query`, and `fragments`. - """ - return self._parsed_location - - def __len__(self) -> int: - return os.path.getsize(self.parsed_location.path) - - def exists(self) -> bool: - return os.path.exists(self.parsed_location.path) - - def to_input_file(self) -> LocalInputFile: - return LocalInputFile(location=self.location) - - def create(self, overwrite: bool = False) -> OutputStream: - output_file = open(self.parsed_location.path, "wb" if overwrite else "xb") - if not issubclass(type(output_file), OutputStream): - raise TypeError("Object returned from LocalOutputFile.create(...) does not match the OutputStream protocol.") - return output_file - - -class LocalFileIO(FileIO): - """A FileIO implementation for local files (for test use only)""" - - def new_input(self, location: str) -> LocalInputFile: - return LocalInputFile(location=location) - - def new_output(self, location: str) -> LocalOutputFile: - return LocalOutputFile(location=location) - - def delete(self, location: Union[str, InputFile, OutputFile]) -> None: - location = location.location if isinstance(location, (InputFile, OutputFile)) else location - parsed_location = urlparse(location) - try: - os.remove(parsed_location.path) - except FileNotFoundError as e: - raise FileNotFoundError(f"Cannot delete file, does not exist: {parsed_location.path}") from e - - -@pytest.mark.parametrize("CustomFileIO", [LocalFileIO, PyArrowFileIO]) -def test_custom_local_input_file(CustomFileIO: Type[FileIO]) -> None: +def test_custom_local_input_file() -> None: """Test initializing an InputFile implementation to read a local file""" with tempfile.TemporaryDirectory() as tmpdirname: file_location = os.path.join(tmpdirname, "foo.txt") @@ -145,7 +42,7 @@ def test_custom_local_input_file(CustomFileIO: Type[FileIO]) -> None: # Instantiate the input file absolute_file_location = os.path.abspath(file_location) - input_file = CustomFileIO().new_input(location=f"{absolute_file_location}") + input_file = PyArrowFileIO().new_input(location=f"{absolute_file_location}") # Test opening and reading the file f = input_file.open() @@ -154,15 +51,14 @@ def test_custom_local_input_file(CustomFileIO: Type[FileIO]) -> None: assert len(input_file) == 3 -@pytest.mark.parametrize("CustomFileIO", [LocalFileIO, PyArrowFileIO]) -def test_custom_local_output_file(CustomFileIO: Type[FileIO]) -> None: +def test_custom_local_output_file() -> None: """Test initializing an OutputFile implementation to write to a local file""" with tempfile.TemporaryDirectory() as tmpdirname: file_location = os.path.join(tmpdirname, "foo.txt") # Instantiate the output file absolute_file_location = os.path.abspath(file_location) - output_file = CustomFileIO().new_output(location=f"{absolute_file_location}") + output_file = PyArrowFileIO().new_output(location=f"{absolute_file_location}") # Create the output file and write to it f = output_file.create() @@ -175,8 +71,7 @@ def test_custom_local_output_file(CustomFileIO: Type[FileIO]) -> None: assert len(output_file) == 3 -@pytest.mark.parametrize("CustomFileIO", [LocalFileIO, PyArrowFileIO]) -def test_custom_local_output_file_with_overwrite(CustomFileIO: Type[FileIO]) -> None: +def test_custom_local_output_file_with_overwrite() -> None: """Test initializing an OutputFile implementation to overwrite a local file""" with tempfile.TemporaryDirectory() as tmpdirname: output_file_location = os.path.join(tmpdirname, "foo.txt") @@ -186,7 +81,7 @@ def test_custom_local_output_file_with_overwrite(CustomFileIO: Type[FileIO]) -> write_file.write(b"foo") # Instantiate an output file - output_file = CustomFileIO().new_output(location=f"{output_file_location}") + output_file = PyArrowFileIO().new_output(location=f"{output_file_location}") # Confirm that a FileExistsError is raised when overwrite=False with pytest.raises(FileExistsError): @@ -200,8 +95,7 @@ def test_custom_local_output_file_with_overwrite(CustomFileIO: Type[FileIO]) -> assert f.read() == b"bar" -@pytest.mark.parametrize("CustomFileIO", [LocalFileIO, PyArrowFileIO]) -def test_custom_file_exists(CustomFileIO: Type[FileIO]) -> None: +def test_custom_file_exists() -> None: """Test that the exists property returns the proper value for existing and non-existing files""" with tempfile.TemporaryDirectory() as tmpdirname: file_location = os.path.join(tmpdirname, "foo.txt") @@ -218,30 +112,29 @@ def test_custom_file_exists(CustomFileIO: Type[FileIO]) -> None: non_existent_absolute_file_location = os.path.abspath(nonexistent_file_location) # Create InputFile instances - input_file = CustomFileIO().new_input(location=f"{absolute_file_location}") - non_existent_input_file = CustomFileIO().new_input(location=f"{non_existent_absolute_file_location}") + input_file = PyArrowFileIO().new_input(location=f"{absolute_file_location}") + non_existent_input_file = PyArrowFileIO().new_input(location=f"{non_existent_absolute_file_location}") # Test opening and reading the file assert input_file.exists() assert not non_existent_input_file.exists() # Create OutputFile instances - file = CustomFileIO().new_output(location=f"{absolute_file_location}") - non_existent_file = CustomFileIO().new_output(location=f"{non_existent_absolute_file_location}") + file = PyArrowFileIO().new_output(location=f"{absolute_file_location}") + non_existent_file = PyArrowFileIO().new_output(location=f"{non_existent_absolute_file_location}") # Test opening and reading the file assert file.exists() assert not non_existent_file.exists() -@pytest.mark.parametrize("CustomFileIO", [LocalFileIO, PyArrowFileIO]) -def test_output_file_to_input_file(CustomFileIO: Type[FileIO]) -> None: +def test_output_file_to_input_file() -> None: """Test initializing an InputFile using the `to_input_file()` method on an OutputFile instance""" with tempfile.TemporaryDirectory() as tmpdirname: output_file_location = os.path.join(tmpdirname, "foo.txt") # Create an output file instance - output_file = CustomFileIO().new_output(location=f"{output_file_location}") + output_file = PyArrowFileIO().new_output(location=f"{output_file_location}") # Create the output file and write to it with output_file.create() as output_stream: @@ -254,20 +147,17 @@ def test_output_file_to_input_file(CustomFileIO: Type[FileIO]) -> None: @pytest.mark.parametrize( - "CustomFileIO,string_uri", + "string_uri", [ - (LocalFileIO, "foo/bar.parquet"), - (LocalFileIO, "file:///foo/bar.parquet"), - (LocalFileIO, "file:/foo/bar/baz.parquet"), - (PyArrowFileIO, "foo/bar/baz.parquet"), - (PyArrowFileIO, "file:/foo/bar/baz.parquet"), - (PyArrowFileIO, "file:/foo/bar/baz.parquet"), + "foo/bar/baz.parquet", + "file:/foo/bar/baz.parquet", + "file:/foo/bar/baz.parquet", ], ) -def test_custom_file_io_locations(CustomFileIO: Type[FileIO], string_uri: str) -> None: +def test_custom_file_io_locations(string_uri: str) -> None: """Test that the location property is maintained as the value of the location argument""" # Instantiate the file-io and create a new input and output file - file_io = CustomFileIO() + file_io = PyArrowFileIO() input_file = file_io.new_input(location=string_uri) assert input_file.location == string_uri @@ -275,32 +165,7 @@ def test_custom_file_io_locations(CustomFileIO: Type[FileIO], string_uri: str) - assert output_file.location == string_uri -@pytest.mark.parametrize( - "string_uri_w_netloc", - ["file://localhost:80/foo/bar.parquet", "file://foo/bar.parquet"], -) -def test_raise_on_network_location_in_input_file(string_uri_w_netloc: str) -> None: - """Test raising a ValueError when providing a network location to a LocalInputFile""" - with pytest.raises(ValueError) as exc_info: - LocalInputFile(location=string_uri_w_netloc) - - assert ("Network location is not allowed for LocalInputFile") in str(exc_info.value) - - -@pytest.mark.parametrize( - "string_uri_w_netloc", - ["file://localhost:80/foo/bar.parquet", "file://foo/bar.parquet"], -) -def test_raise_on_network_location_in_output_file(string_uri_w_netloc: str) -> None: - """Test raising a ValueError when providing a network location to a LocalOutputFile""" - with pytest.raises(ValueError) as exc_info: - LocalInputFile(location=string_uri_w_netloc) - - assert ("Network location is not allowed for LocalInputFile") in str(exc_info.value) - - -@pytest.mark.parametrize("CustomFileIO", [LocalFileIO, PyArrowFileIO]) -def test_deleting_local_file_using_file_io(CustomFileIO: Type[FileIO]) -> None: +def test_deleting_local_file_using_file_io() -> None: """Test deleting a local file using FileIO.delete(...)""" with tempfile.TemporaryDirectory() as tmpdirname: # Write to the temporary file @@ -309,7 +174,7 @@ def test_deleting_local_file_using_file_io(CustomFileIO: Type[FileIO]) -> None: f.write(b"foo") # Instantiate the file-io - file_io = CustomFileIO() + file_io = PyArrowFileIO() # Confirm that the file initially exists assert os.path.exists(output_file_location) @@ -321,15 +186,14 @@ def test_deleting_local_file_using_file_io(CustomFileIO: Type[FileIO]) -> None: assert not os.path.exists(output_file_location) -@pytest.mark.parametrize("CustomFileIO", [LocalFileIO, PyArrowFileIO]) -def test_raise_file_not_found_error_for_fileio_delete(CustomFileIO: Type[FileIO]) -> None: +def test_raise_file_not_found_error_for_fileio_delete() -> None: """Test raising a FileNotFound error when trying to delete a non-existent file""" with tempfile.TemporaryDirectory() as tmpdirname: # Write to the temporary file output_file_location = os.path.join(tmpdirname, "foo.txt") # Instantiate the file-io - file_io = CustomFileIO() + file_io = PyArrowFileIO() # Delete the non-existent file using the file-io implementations delete method with pytest.raises(FileNotFoundError) as exc_info: @@ -341,8 +205,7 @@ def test_raise_file_not_found_error_for_fileio_delete(CustomFileIO: Type[FileIO] assert not os.path.exists(output_file_location) -@pytest.mark.parametrize("CustomFileIO", [LocalFileIO, PyArrowFileIO]) -def test_deleting_local_file_using_file_io_input_file(CustomFileIO: Type[FileIO]) -> None: +def test_deleting_local_file_using_file_io_input_file() -> None: """Test deleting a local file by passing an InputFile instance to FileIO.delete(...)""" with tempfile.TemporaryDirectory() as tmpdirname: # Write to the temporary file @@ -351,13 +214,13 @@ def test_deleting_local_file_using_file_io_input_file(CustomFileIO: Type[FileIO] f.write(b"foo") # Instantiate the file-io - file_io = CustomFileIO() + file_io = PyArrowFileIO() # Confirm that the file initially exists assert os.path.exists(file_location) # Instantiate the custom InputFile - input_file = CustomFileIO().new_input(location=f"{file_location}") + input_file = PyArrowFileIO().new_input(location=f"{file_location}") # Delete the file using the file-io implementations delete method file_io.delete(input_file) @@ -366,8 +229,7 @@ def test_deleting_local_file_using_file_io_input_file(CustomFileIO: Type[FileIO] assert not os.path.exists(file_location) -@pytest.mark.parametrize("CustomFileIO", [LocalFileIO, PyArrowFileIO]) -def test_deleting_local_file_using_file_io_output_file(CustomFileIO: Type[FileIO]) -> None: +def test_deleting_local_file_using_file_io_output_file() -> None: """Test deleting a local file by passing an OutputFile instance to FileIO.delete(...)""" with tempfile.TemporaryDirectory() as tmpdirname: # Write to the temporary file @@ -376,13 +238,13 @@ def test_deleting_local_file_using_file_io_output_file(CustomFileIO: Type[FileIO f.write(b"foo") # Instantiate the file-io - file_io = CustomFileIO() + file_io = PyArrowFileIO() # Confirm that the file initially exists assert os.path.exists(file_location) # Instantiate the custom OutputFile - output_file = CustomFileIO().new_output(location=f"{file_location}") + output_file = PyArrowFileIO().new_output(location=f"{file_location}") # Delete the file using the file-io implementations delete method file_io.delete(output_file) @@ -426,14 +288,12 @@ def test_load_file_io_location_no_schema() -> None: assert isinstance(load_file_io({"location": "/no-schema/"}), PyArrowFileIO) -@patch.dict("pyiceberg.io.SCHEMA_TO_FILE_IO", {"test": ["tests.io.test_io.LocalFileIO"]}) def test_mock_warehouse_location_file_io() -> None: # For testing the selection logic io = load_file_io({"warehouse": "test://some-path/"}) assert io.properties["warehouse"] == "test://some-path/" -@patch.dict("pyiceberg.io.SCHEMA_TO_FILE_IO", {"test": ["tests.io.test_io.LocalFileIO"]}) def test_mock_table_location_file_io() -> None: # For testing the selection logic io = load_file_io({}, "test://some-path/") diff --git a/tests/table/test_metadata.py b/tests/table/test_metadata.py index 9f46ca6737..8c464c0e5a 100644 --- a/tests/table/test_metadata.py +++ b/tests/table/test_metadata.py @@ -14,6 +14,7 @@ # KIND, either express or implied. See the License for the # specific language governing permissions and limitations # under the License. +# pylint: disable=redefined-outer-name import io import json @@ -49,30 +50,33 @@ StructType, ) -EXAMPLE_TABLE_METADATA_V1 = { - "format-version": 1, - "table-uuid": "d20125c8-7284-442c-9aea-15fee620737c", - "location": "s3://bucket/test/location", - "last-updated-ms": 1602638573874, - "last-column-id": 3, - "schema": { - "type": "struct", - "fields": [ - {"id": 1, "name": "x", "required": True, "type": "long"}, - {"id": 2, "name": "y", "required": True, "type": "long", "doc": "comment"}, - {"id": 3, "name": "z", "required": True, "type": "long"}, - ], - }, - "partition-spec": [{"name": "x", "transform": "identity", "source-id": 1, "field-id": 1000}], - "properties": {}, - "current-snapshot-id": -1, - "snapshots": [{"snapshot-id": 1925, "timestamp-ms": 1602638573822}], -} + +@pytest.fixture(scope="session") +def example_table_metadata_v1() -> Dict[str, Any]: + return { + "format-version": 1, + "table-uuid": "d20125c8-7284-442c-9aea-15fee620737c", + "location": "s3://bucket/test/location", + "last-updated-ms": 1602638573874, + "last-column-id": 3, + "schema": { + "type": "struct", + "fields": [ + {"id": 1, "name": "x", "required": True, "type": "long"}, + {"id": 2, "name": "y", "required": True, "type": "long", "doc": "comment"}, + {"id": 3, "name": "z", "required": True, "type": "long"}, + ], + }, + "partition-spec": [{"name": "x", "transform": "identity", "source-id": 1, "field-id": 1000}], + "properties": {}, + "current-snapshot-id": -1, + "snapshots": [{"snapshot-id": 1925, "timestamp-ms": 1602638573822}], + } -def test_from_dict_v1() -> None: +def test_from_dict_v1(example_table_metadata_v1: Dict[str, Any]) -> None: """Test initialization of a TableMetadata instance from a dictionary""" - TableMetadataUtil.parse_obj(EXAMPLE_TABLE_METADATA_V1) + TableMetadataUtil.parse_obj(example_table_metadata_v1) def test_from_dict_v2(example_table_metadata_v2: Dict[str, Any]) -> None: @@ -110,9 +114,9 @@ def test_v2_metadata_parsing(example_table_metadata_v2: Dict[str, Any]) -> None: assert table_metadata.default_sort_order_id == 3 -def test_v1_metadata_parsing_directly() -> None: +def test_v1_metadata_parsing_directly(example_table_metadata_v1: Dict[str, Any]) -> None: """Test retrieving values from a TableMetadata instance of version 1""" - table_metadata = TableMetadataV1(**EXAMPLE_TABLE_METADATA_V1) + table_metadata = TableMetadataV1(**example_table_metadata_v1) assert isinstance(table_metadata, TableMetadataV1) @@ -174,8 +178,8 @@ def test_updating_metadata(example_table_metadata_v2: Dict[str, Any]) -> None: assert table_metadata.schemas[-1] == Schema(**new_schema) -def test_serialize_v1() -> None: - table_metadata = TableMetadataV1(**EXAMPLE_TABLE_METADATA_V1) +def test_serialize_v1(example_table_metadata_v1: Dict[str, Any]) -> None: + table_metadata = TableMetadataV1(**example_table_metadata_v1) table_metadata_json = table_metadata.json() expected = """{"location": "s3://bucket/test/location", "table-uuid": "d20125c8-7284-442c-9aea-15fee620737c", "last-updated-ms": 1602638573874, "last-column-id": 3, "schemas": [{"type": "struct", "fields": [{"id": 1, "name": "x", "type": "long", "required": true}, {"id": 2, "name": "y", "type": "long", "required": true, "doc": "comment"}, {"id": 3, "name": "z", "type": "long", "required": true}], "schema-id": 0, "identifier-field-ids": []}], "current-schema-id": 0, "partition-specs": [{"spec-id": 0, "fields": [{"source-id": 1, "field-id": 1000, "transform": "identity", "name": "x"}]}], "default-spec-id": 0, "last-partition-id": 1000, "properties": {}, "snapshots": [{"snapshot-id": 1925, "timestamp-ms": 1602638573822}], "snapshot-log": [], "metadata-log": [], "sort-orders": [{"order-id": 0, "fields": []}], "default-sort-order-id": 0, "refs": {}, "format-version": 1, "schema": {"type": "struct", "fields": [{"id": 1, "name": "x", "type": "long", "required": true}, {"id": 2, "name": "y", "type": "long", "required": true, "doc": "comment"}, {"id": 3, "name": "z", "type": "long", "required": true}], "schema-id": 0, "identifier-field-ids": []}, "partition-spec": [{"name": "x", "transform": "identity", "source-id": 1, "field-id": 1000}]}""" assert table_metadata_json == expected @@ -187,17 +191,17 @@ def test_serialize_v2(example_table_metadata_v2: Dict[str, Any]) -> None: assert table_metadata == expected -def test_migrate_v1_schemas() -> None: - table_metadata = TableMetadataV1(**EXAMPLE_TABLE_METADATA_V1) +def test_migrate_v1_schemas(example_table_metadata_v1: Dict[str, Any]) -> None: + table_metadata = TableMetadataV1(**example_table_metadata_v1) assert isinstance(table_metadata, TableMetadataV1) assert len(table_metadata.schemas) == 1 assert table_metadata.schemas[0] == table_metadata.schema_ -def test_migrate_v1_partition_specs() -> None: +def test_migrate_v1_partition_specs(example_table_metadata_v1: Dict[str, Any]) -> None: # Copy the example, and add a spec - table_metadata = TableMetadataV1(**EXAMPLE_TABLE_METADATA_V1) + table_metadata = TableMetadataV1(**example_table_metadata_v1) assert isinstance(table_metadata, TableMetadataV1) assert len(table_metadata.partition_specs) == 1 # Spec ID gets added automatically @@ -390,7 +394,7 @@ def test_invalid_partition_spec() -> None: assert "default-spec-id 1 can't be found" in str(exc_info.value) -def test_v1_writing_metadata() -> None: +def test_v1_writing_metadata(example_table_metadata_v1: Dict[str, Any]) -> None: """ https://iceberg.apache.org/spec/#version-2 @@ -398,14 +402,14 @@ def test_v1_writing_metadata() -> None: - Table metadata field last-sequence-number should not be written """ - table_metadata = TableMetadataV1(**EXAMPLE_TABLE_METADATA_V1) + table_metadata = TableMetadataV1(**example_table_metadata_v1) metadata_v1_json = table_metadata.json() metadata_v1 = json.loads(metadata_v1_json) assert "last-sequence-number" not in metadata_v1 -def test_v1_metadata_for_v2() -> None: +def test_v1_metadata_for_v2(example_table_metadata_v1: Dict[str, Any]) -> None: """ https://iceberg.apache.org/spec/#version-2 @@ -413,7 +417,7 @@ def test_v1_metadata_for_v2() -> None: - Table metadata field last-sequence-number must default to 0 """ - table_metadata = TableMetadataV1(**EXAMPLE_TABLE_METADATA_V1).to_v2() + table_metadata = TableMetadataV1(**example_table_metadata_v1).to_v2() assert table_metadata.last_sequence_number == 0 diff --git a/tests/utils/test_manifest.py b/tests/utils/test_manifest.py index 25ba4730dc..91c111abc1 100644 --- a/tests/utils/test_manifest.py +++ b/tests/utils/test_manifest.py @@ -16,6 +16,7 @@ # under the License. from pyiceberg.io import load_file_io +from pyiceberg.io.pyarrow import PyArrowFileIO from pyiceberg.manifest import ( DataFile, DataFileContent, @@ -30,11 +31,10 @@ ) from pyiceberg.table import Snapshot from pyiceberg.table.snapshots import Operation, Summary -from tests.io.test_io import LocalInputFile def test_read_manifest_entry(generated_manifest_entry_file: str) -> None: - input_file = LocalInputFile(generated_manifest_entry_file) + input_file = PyArrowFileIO().new_input(location=generated_manifest_entry_file) assert list(read_manifest_entry(input_file)) == [ ManifestEntry( status=1, @@ -268,7 +268,7 @@ def test_read_manifest_entry(generated_manifest_entry_file: str) -> None: def test_read_manifest_list(generated_manifest_file_file: str) -> None: - input_file = LocalInputFile(generated_manifest_file_file) + input_file = PyArrowFileIO().new_input(generated_manifest_file_file) actual = list(read_manifest_list(input_file)) expected = [ ManifestFile( From 4ea18bf3d21e7738de39177f65eb31de3b386d11 Mon Sep 17 00:00:00 2001 From: cccs-eric Date: Mon, 19 Dec 2022 07:17:44 -0500 Subject: [PATCH 308/642] Python: Add adlfs support (Azure DataLake FileSystem) (#6392) * Python: Initial code for ADLFS support * Python: Initial code for ADLFS support * Python: fix types and lint errors * Python: fixing lint errors * Python: address PR feedback * Python: address PR feedback about credentials * Python: add adlfs references into documentation * Python: fix linting * Python: rebase master * Python: update test case Co-authored-by: Ryan Blue --- Makefile | 13 +- dev/docker-compose-azurite.yml | 26 + dev/run-azurite.sh | 33 + dev/run-minio.sh | 4 +- mkdocs/docs/configuration.md | 15 +- mkdocs/docs/contributing.md | 11 +- mkdocs/docs/index.md | 3 +- poetry.lock | 2376 ++++++++++++++++++-------------- pyiceberg/io/__init__.py | 2 + pyiceberg/io/fsspec.py | 19 +- pyproject.toml | 14 +- tests/conftest.py | 38 + tests/io/test_fsspec.py | 185 ++- 13 files changed, 1711 insertions(+), 1028 deletions(-) create mode 100644 dev/docker-compose-azurite.yml create mode 100755 dev/run-azurite.sh diff --git a/Makefile b/Makefile index 32cd895f53..9e03b10403 100644 --- a/Makefile +++ b/Makefile @@ -17,7 +17,7 @@ install: pip install poetry - poetry install -E pyarrow -E hive -E s3fs -E glue + poetry install -E pyarrow -E hive -E s3fs -E glue -E adlfs check-license: ./dev/check-license @@ -26,14 +26,21 @@ lint: poetry run pre-commit run --all-files test: - poetry run coverage run --source=pyiceberg/ -m pytest tests/ -m "not s3" ${PYTEST_ARGS} + poetry run coverage run --source=pyiceberg/ -m pytest tests/ -m "not s3 and not adlfs" ${PYTEST_ARGS} poetry run coverage report -m --fail-under=90 poetry run coverage html poetry run coverage xml test-s3: sh ./dev/run-minio.sh - poetry run coverage run --source=pyiceberg/ -m pytest tests/ ${PYTEST_ARGS} + poetry run coverage run --source=pyiceberg/ -m pytest tests/ -m "not adlfs" ${PYTEST_ARGS} + poetry run coverage report -m --fail-under=90 + poetry run coverage html + poetry run coverage xml + +test-adlfs: + sh ./dev/run-azurite.sh + poetry run coverage run --source=pyiceberg/ -m pytest tests/ -m "not s3" ${PYTEST_ARGS} poetry run coverage report -m --fail-under=90 poetry run coverage html poetry run coverage xml diff --git a/dev/docker-compose-azurite.yml b/dev/docker-compose-azurite.yml new file mode 100644 index 0000000000..60f4ff4f94 --- /dev/null +++ b/dev/docker-compose-azurite.yml @@ -0,0 +1,26 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +version: "3" + +services: + azurite: + image: mcr.microsoft.com/azure-storage/azurite + container_name: azurite + hostname: azurite + ports: + - 10000:10000 + command: [ "azurite-blob", "--loose", "--blobHost", "0.0.0.0" ] diff --git a/dev/run-azurite.sh b/dev/run-azurite.sh new file mode 100755 index 0000000000..c218155894 --- /dev/null +++ b/dev/run-azurite.sh @@ -0,0 +1,33 @@ +#!/bin/bash +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# + +set -ex + +if [ $(docker ps -q --filter "name=azurite" --filter "status=running" ) ]; then + echo "Azurite backend running" +else + docker-compose -f dev/docker-compose-azurite.yml kill + docker-compose -f dev/docker-compose-azurite.yml up -d + while [ -z $(docker ps -q --filter "name=azurite" --filter "status=running" ) ] + do + echo "Waiting for Azurite" + sleep 1 + done +fi diff --git a/dev/run-minio.sh b/dev/run-minio.sh index 0e40be4e9d..0db37012e7 100755 --- a/dev/run-minio.sh +++ b/dev/run-minio.sh @@ -20,12 +20,12 @@ set -ex -if [[ $(docker ps -q --filter "name=pyiceberg-minio" --filter "status=running" ) ]]; then +if [ $(docker ps -q --filter "name=pyiceberg-minio" --filter "status=running" ) ]; then echo "Minio backend running" else docker-compose -f dev/docker-compose.yml kill docker-compose -f dev/docker-compose.yml up -d - while [[ -z $(docker ps -q --filter "name=pyiceberg-minio" --filter "status=running" ) ]] + while [ -z $(docker ps -q --filter "name=pyiceberg-minio" --filter "status=running" ) ] do echo "Waiting for Minio" sleep 1 diff --git a/mkdocs/docs/configuration.md b/mkdocs/docs/configuration.md index 985d258173..3780d580e2 100644 --- a/mkdocs/docs/configuration.md +++ b/mkdocs/docs/configuration.md @@ -37,12 +37,15 @@ The environment variable picked up by Iceberg starts with `PYICEBERG_` and then For the FileIO there are several configuration options available: -| Key | Example | Description | -|----------------------|---------------------|-----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| -| s3.endpoint | https://10.0.19.25/ | Configure an alternative endpoint of the S3 service for the FileIO to access. This could be used to use S3FileIO with any s3-compatible object storage service that has a different endpoint, or access a private S3 endpoint in a virtual private cloud. | -| s3.access-key-id | admin | Configure the static secret access key used to access the FileIO. | -| s3.secret-access-key | password | Configure the static session token used to access the FileIO. | -| s3.signer | bearer | Configure the signature version of the FileIO. | +| Key | Example | Description | +|--------------------------|---------------------|-----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| +| s3.endpoint | https://10.0.19.25/ | Configure an alternative endpoint of the S3 service for the FileIO to access. This could be used to use S3FileIO with any s3-compatible object storage service that has a different endpoint, or access a private S3 endpoint in a virtual private cloud. | +| s3.access-key-id | admin | Configure the static secret access key used to access the FileIO. | +| s3.secret-access-key | password | Configure the static session token used to access the FileIO. | +| s3.signer | bearer | Configure the signature version of the FileIO. | +| adlfs.endpoint | http://127.0.0.1/ | Configure an alternative endpoint of the ADLFS service for the FileIO to access. This could be used to use FileIO with any adlfs-compatible object storage service that has a different endpoint (like [azurite](https://github.com/azure/azurite)). | +| adlfs.account-name | devstoreaccount1 | Configure the static storage account name used to access the FileIO. | +| adlfs.account-key | Eby8vdM02xNOcqF... | Configure the static storage account key used to access the FileIO. | ## REST Catalog diff --git a/mkdocs/docs/contributing.md b/mkdocs/docs/contributing.md index 6f2236028d..a170ab1ae8 100644 --- a/mkdocs/docs/contributing.md +++ b/mkdocs/docs/contributing.md @@ -73,12 +73,19 @@ For Python, `pytest` is used a testing framework in combination with `coverage` make test ``` -By default, S3 tests are ignored because that require minio to be running. To run the S3 suite: +By default, S3 and ADLFS tests are ignored because that require minio and azurite to be running. +To run the S3 suite: ```bash make test-s3 ``` +To run the ADLFS suite: + +```bash +make test-adlfs +``` + To pass additional arguments to pytest, you can use `PYTEST_ARGS`. *Run pytest in verbose mode* @@ -132,4 +139,4 @@ PyIceberg offers support from Python 3.8 onwards, we can't use the [type hints f ## Third party libraries -PyIceberg naturally integrates into the rich Python ecosystem, however it is important to be hesistant to add third party packages. Adding a lot of packages makes the library heavyweight, and causes incompatibilities with other projects if they use a different version of the library. Also, big libraries such as `s3fs`, `pyarrow`, `thrift` should be optional to avoid downloading everything, while not being sure if is actually being used. +PyIceberg naturally integrates into the rich Python ecosystem, however it is important to be hesistant to add third party packages. Adding a lot of packages makes the library heavyweight, and causes incompatibilities with other projects if they use a different version of the library. Also, big libraries such as `s3fs`, `adlfs`, `pyarrow`, `thrift` should be optional to avoid downloading everything, while not being sure if is actually being used. diff --git a/mkdocs/docs/index.md b/mkdocs/docs/index.md index e36588e434..2dbbf5159a 100644 --- a/mkdocs/docs/index.md +++ b/mkdocs/docs/index.md @@ -50,8 +50,9 @@ You can mix and match optional dependencies depending on your needs: | pyarrow | PyArrow as a FileIO implementation to interact with the object store | | duckdb | Installs both PyArrow and DuckDB | | s3fs | S3FS as a FileIO implementation to interact with the object store | +| adlfs | ADLFS as a FileIO implementation to interact with the object store | | snappy | Support for snappy Avro compression | -You either need to install `s3fs` or `pyarrow` for fetching files. +You either need to install `s3fs`, `adlfs` or `pyarrow` for fetching files. There is both a [CLI](cli.md) and [Python API](api.md) available. diff --git a/poetry.lock b/poetry.lock index 4a77b4756c..faf506ee56 100644 --- a/poetry.lock +++ b/poetry.lock @@ -1,3 +1,46 @@ +# This file is automatically @generated by Poetry and should not be changed by hand. + +[[package]] +name = "adal" +version = "1.2.7" +description = "Note: This library is already replaced by MSAL Python, available here: https://pypi.org/project/msal/ .ADAL Python remains available here as a legacy. The ADAL for Python library makes it easy for python application to authenticate to Azure Active Directory (AAD) in order to access AAD protected web resources." +category = "main" +optional = true +python-versions = "*" +files = [ + {file = "adal-1.2.7-py2.py3-none-any.whl", hash = "sha256:2a7451ed7441ddbc57703042204a3e30ef747478eea022c70f789fc7f084bc3d"}, + {file = "adal-1.2.7.tar.gz", hash = "sha256:d74f45b81317454d96e982fd1c50e6fb5c99ac2223728aea8764433a39f566f1"}, +] + +[package.dependencies] +cryptography = ">=1.1.0" +PyJWT = ">=1.0.0,<3" +python-dateutil = ">=2.1.0,<3" +requests = ">=2.0.0,<3" + +[[package]] +name = "adlfs" +version = "2022.10.0" +description = "Access Azure Datalake Gen1 with fsspec and dask" +category = "main" +optional = true +python-versions = ">=3.8" +files = [ + {file = "adlfs-2022.10.0-py3-none-any.whl", hash = "sha256:dc63b248f109bca2000d306f7a51c33d6859e4692dda7c6198b6300f8e29faaf"}, + {file = "adlfs-2022.10.0.tar.gz", hash = "sha256:8542f9e6799aeb77adb5cb9854f76b7dce874610dc0f2ff006cce960526ffad6"}, +] + +[package.dependencies] +aiohttp = ">=3.7.0" +azure-core = ">=1.7.0" +azure-datalake-store = ">=0.0.46,<0.1" +azure-identity = "*" +azure-storage-blob = ">=12.12.0" +fsspec = ">=2021.10.1" + +[package.extras] +docs = ["furo", "myst-parser", "numpydoc", "sphinx"] + [[package]] name = "aiobotocore" version = "2.4.1" @@ -5,6 +48,10 @@ description = "Async client for aws services using botocore and aiohttp" category = "main" optional = true python-versions = ">=3.7" +files = [ + {file = "aiobotocore-2.4.1-py3-none-any.whl", hash = "sha256:26e0d55d5901f487c3467616c028abb85036ca33a0f88e279770adae6b865c69"}, + {file = "aiobotocore-2.4.1.tar.gz", hash = "sha256:5c8ee79d1f14273e3f8f3af5c893db889ef163227b178ecacb5aea4547d3b9a7"}, +] [package.dependencies] aiohttp = ">=3.3.1" @@ -23,6 +70,95 @@ description = "Async http client/server framework (asyncio)" category = "main" optional = true python-versions = ">=3.6" +files = [ + {file = "aiohttp-3.8.3-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:ba71c9b4dcbb16212f334126cc3d8beb6af377f6703d9dc2d9fb3874fd667ee9"}, + {file = "aiohttp-3.8.3-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:d24b8bb40d5c61ef2d9b6a8f4528c2f17f1c5d2d31fed62ec860f6006142e83e"}, + {file = "aiohttp-3.8.3-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:f88df3a83cf9df566f171adba39d5bd52814ac0b94778d2448652fc77f9eb491"}, + {file = "aiohttp-3.8.3-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:b97decbb3372d4b69e4d4c8117f44632551c692bb1361b356a02b97b69e18a62"}, + {file = "aiohttp-3.8.3-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:309aa21c1d54b8ef0723181d430347d7452daaff93e8e2363db8e75c72c2fb2d"}, + {file = "aiohttp-3.8.3-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:ad5383a67514e8e76906a06741febd9126fc7c7ff0f599d6fcce3e82b80d026f"}, + {file = "aiohttp-3.8.3-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:20acae4f268317bb975671e375493dbdbc67cddb5f6c71eebdb85b34444ac46b"}, + {file = "aiohttp-3.8.3-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:05a3c31c6d7cd08c149e50dc7aa2568317f5844acd745621983380597f027a18"}, + {file = "aiohttp-3.8.3-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:d6f76310355e9fae637c3162936e9504b4767d5c52ca268331e2756e54fd4ca5"}, + {file = "aiohttp-3.8.3-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:256deb4b29fe5e47893fa32e1de2d73c3afe7407738bd3c63829874661d4822d"}, + {file = "aiohttp-3.8.3-cp310-cp310-musllinux_1_1_ppc64le.whl", hash = "sha256:5c59fcd80b9049b49acd29bd3598cada4afc8d8d69bd4160cd613246912535d7"}, + {file = "aiohttp-3.8.3-cp310-cp310-musllinux_1_1_s390x.whl", hash = "sha256:059a91e88f2c00fe40aed9031b3606c3f311414f86a90d696dd982e7aec48142"}, + {file = "aiohttp-3.8.3-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:2feebbb6074cdbd1ac276dbd737b40e890a1361b3cc30b74ac2f5e24aab41f7b"}, + {file = "aiohttp-3.8.3-cp310-cp310-win32.whl", hash = "sha256:5bf651afd22d5f0c4be16cf39d0482ea494f5c88f03e75e5fef3a85177fecdeb"}, + {file = "aiohttp-3.8.3-cp310-cp310-win_amd64.whl", hash = "sha256:653acc3880459f82a65e27bd6526e47ddf19e643457d36a2250b85b41a564715"}, + {file = "aiohttp-3.8.3-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:86fc24e58ecb32aee09f864cb11bb91bc4c1086615001647dbfc4dc8c32f4008"}, + {file = "aiohttp-3.8.3-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:75e14eac916f024305db517e00a9252714fce0abcb10ad327fb6dcdc0d060f1d"}, + {file = "aiohttp-3.8.3-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:d1fde0f44029e02d02d3993ad55ce93ead9bb9b15c6b7ccd580f90bd7e3de476"}, + {file = "aiohttp-3.8.3-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:4ab94426ddb1ecc6a0b601d832d5d9d421820989b8caa929114811369673235c"}, + {file = "aiohttp-3.8.3-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:89d2e02167fa95172c017732ed7725bc8523c598757f08d13c5acca308e1a061"}, + {file = "aiohttp-3.8.3-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:02f9a2c72fc95d59b881cf38a4b2be9381b9527f9d328771e90f72ac76f31ad8"}, + {file = "aiohttp-3.8.3-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:9c7149272fb5834fc186328e2c1fa01dda3e1fa940ce18fded6d412e8f2cf76d"}, + {file = "aiohttp-3.8.3-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:512bd5ab136b8dc0ffe3fdf2dfb0c4b4f49c8577f6cae55dca862cd37a4564e2"}, + {file = "aiohttp-3.8.3-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:7018ecc5fe97027214556afbc7c502fbd718d0740e87eb1217b17efd05b3d276"}, + {file = "aiohttp-3.8.3-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:88c70ed9da9963d5496d38320160e8eb7e5f1886f9290475a881db12f351ab5d"}, + {file = "aiohttp-3.8.3-cp311-cp311-musllinux_1_1_ppc64le.whl", hash = "sha256:da22885266bbfb3f78218dc40205fed2671909fbd0720aedba39b4515c038091"}, + {file = "aiohttp-3.8.3-cp311-cp311-musllinux_1_1_s390x.whl", hash = "sha256:e65bc19919c910127c06759a63747ebe14f386cda573d95bcc62b427ca1afc73"}, + {file = "aiohttp-3.8.3-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:08c78317e950e0762c2983f4dd58dc5e6c9ff75c8a0efeae299d363d439c8e34"}, + {file = "aiohttp-3.8.3-cp311-cp311-win32.whl", hash = "sha256:45d88b016c849d74ebc6f2b6e8bc17cabf26e7e40c0661ddd8fae4c00f015697"}, + {file = "aiohttp-3.8.3-cp311-cp311-win_amd64.whl", hash = "sha256:96372fc29471646b9b106ee918c8eeb4cca423fcbf9a34daa1b93767a88a2290"}, + {file = "aiohttp-3.8.3-cp36-cp36m-macosx_10_9_x86_64.whl", hash = "sha256:c971bf3786b5fad82ce5ad570dc6ee420f5b12527157929e830f51c55dc8af77"}, + {file = "aiohttp-3.8.3-cp36-cp36m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:ff25f48fc8e623d95eca0670b8cc1469a83783c924a602e0fbd47363bb54aaca"}, + {file = "aiohttp-3.8.3-cp36-cp36m-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:e381581b37db1db7597b62a2e6b8b57c3deec95d93b6d6407c5b61ddc98aca6d"}, + {file = "aiohttp-3.8.3-cp36-cp36m-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:db19d60d846283ee275d0416e2a23493f4e6b6028825b51290ac05afc87a6f97"}, + {file = "aiohttp-3.8.3-cp36-cp36m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:25892c92bee6d9449ffac82c2fe257f3a6f297792cdb18ad784737d61e7a9a85"}, + {file = "aiohttp-3.8.3-cp36-cp36m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:398701865e7a9565d49189f6c90868efaca21be65c725fc87fc305906be915da"}, + {file = "aiohttp-3.8.3-cp36-cp36m-musllinux_1_1_aarch64.whl", hash = "sha256:4a4fbc769ea9b6bd97f4ad0b430a6807f92f0e5eb020f1e42ece59f3ecfc4585"}, + {file = "aiohttp-3.8.3-cp36-cp36m-musllinux_1_1_i686.whl", hash = "sha256:b29bfd650ed8e148f9c515474a6ef0ba1090b7a8faeee26b74a8ff3b33617502"}, + {file = "aiohttp-3.8.3-cp36-cp36m-musllinux_1_1_ppc64le.whl", hash = "sha256:1e56b9cafcd6531bab5d9b2e890bb4937f4165109fe98e2b98ef0dcfcb06ee9d"}, + {file = "aiohttp-3.8.3-cp36-cp36m-musllinux_1_1_s390x.whl", hash = "sha256:ec40170327d4a404b0d91855d41bfe1fe4b699222b2b93e3d833a27330a87a6d"}, + {file = "aiohttp-3.8.3-cp36-cp36m-musllinux_1_1_x86_64.whl", hash = "sha256:2df5f139233060578d8c2c975128fb231a89ca0a462b35d4b5fcf7c501ebdbe1"}, + {file = "aiohttp-3.8.3-cp36-cp36m-win32.whl", hash = "sha256:f973157ffeab5459eefe7b97a804987876dd0a55570b8fa56b4e1954bf11329b"}, + {file = "aiohttp-3.8.3-cp36-cp36m-win_amd64.whl", hash = "sha256:437399385f2abcd634865705bdc180c8314124b98299d54fe1d4c8990f2f9494"}, + {file = "aiohttp-3.8.3-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:09e28f572b21642128ef31f4e8372adb6888846f32fecb288c8b0457597ba61a"}, + {file = "aiohttp-3.8.3-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:6f3553510abdbec67c043ca85727396ceed1272eef029b050677046d3387be8d"}, + {file = "aiohttp-3.8.3-cp37-cp37m-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:e168a7560b7c61342ae0412997b069753f27ac4862ec7867eff74f0fe4ea2ad9"}, + {file = "aiohttp-3.8.3-cp37-cp37m-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:db4c979b0b3e0fa7e9e69ecd11b2b3174c6963cebadeecfb7ad24532ffcdd11a"}, + {file = "aiohttp-3.8.3-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:e164e0a98e92d06da343d17d4e9c4da4654f4a4588a20d6c73548a29f176abe2"}, + {file = "aiohttp-3.8.3-cp37-cp37m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:e8a78079d9a39ca9ca99a8b0ac2fdc0c4d25fc80c8a8a82e5c8211509c523363"}, + {file = "aiohttp-3.8.3-cp37-cp37m-musllinux_1_1_aarch64.whl", hash = "sha256:21b30885a63c3f4ff5b77a5d6caf008b037cb521a5f33eab445dc566f6d092cc"}, + {file = "aiohttp-3.8.3-cp37-cp37m-musllinux_1_1_i686.whl", hash = "sha256:4b0f30372cef3fdc262f33d06e7b411cd59058ce9174ef159ad938c4a34a89da"}, + {file = "aiohttp-3.8.3-cp37-cp37m-musllinux_1_1_ppc64le.whl", hash = "sha256:8135fa153a20d82ffb64f70a1b5c2738684afa197839b34cc3e3c72fa88d302c"}, + {file = "aiohttp-3.8.3-cp37-cp37m-musllinux_1_1_s390x.whl", hash = "sha256:ad61a9639792fd790523ba072c0555cd6be5a0baf03a49a5dd8cfcf20d56df48"}, + {file = "aiohttp-3.8.3-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:978b046ca728073070e9abc074b6299ebf3501e8dee5e26efacb13cec2b2dea0"}, + {file = "aiohttp-3.8.3-cp37-cp37m-win32.whl", hash = "sha256:0d2c6d8c6872df4a6ec37d2ede71eff62395b9e337b4e18efd2177de883a5033"}, + {file = "aiohttp-3.8.3-cp37-cp37m-win_amd64.whl", hash = "sha256:21d69797eb951f155026651f7e9362877334508d39c2fc37bd04ff55b2007091"}, + {file = "aiohttp-3.8.3-cp38-cp38-macosx_10_9_universal2.whl", hash = "sha256:2ca9af5f8f5812d475c5259393f52d712f6d5f0d7fdad9acdb1107dd9e3cb7eb"}, + {file = "aiohttp-3.8.3-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:1d90043c1882067f1bd26196d5d2db9aa6d268def3293ed5fb317e13c9413ea4"}, + {file = "aiohttp-3.8.3-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:d737fc67b9a970f3234754974531dc9afeea11c70791dcb7db53b0cf81b79784"}, + {file = "aiohttp-3.8.3-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:ebf909ea0a3fc9596e40d55d8000702a85e27fd578ff41a5500f68f20fd32e6c"}, + {file = "aiohttp-3.8.3-cp38-cp38-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:5835f258ca9f7c455493a57ee707b76d2d9634d84d5d7f62e77be984ea80b849"}, + {file = "aiohttp-3.8.3-cp38-cp38-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:da37dcfbf4b7f45d80ee386a5f81122501ec75672f475da34784196690762f4b"}, + {file = "aiohttp-3.8.3-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:87f44875f2804bc0511a69ce44a9595d5944837a62caecc8490bbdb0e18b1342"}, + {file = "aiohttp-3.8.3-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:527b3b87b24844ea7865284aabfab08eb0faf599b385b03c2aa91fc6edd6e4b6"}, + {file = "aiohttp-3.8.3-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:d5ba88df9aa5e2f806650fcbeedbe4f6e8736e92fc0e73b0400538fd25a4dd96"}, + {file = "aiohttp-3.8.3-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:e7b8813be97cab8cb52b1375f41f8e6804f6507fe4660152e8ca5c48f0436017"}, + {file = "aiohttp-3.8.3-cp38-cp38-musllinux_1_1_ppc64le.whl", hash = "sha256:2dea10edfa1a54098703cb7acaa665c07b4e7568472a47f4e64e6319d3821ccf"}, + {file = "aiohttp-3.8.3-cp38-cp38-musllinux_1_1_s390x.whl", hash = "sha256:713d22cd9643ba9025d33c4af43943c7a1eb8547729228de18d3e02e278472b6"}, + {file = "aiohttp-3.8.3-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:2d252771fc85e0cf8da0b823157962d70639e63cb9b578b1dec9868dd1f4f937"}, + {file = "aiohttp-3.8.3-cp38-cp38-win32.whl", hash = "sha256:66bd5f950344fb2b3dbdd421aaa4e84f4411a1a13fca3aeb2bcbe667f80c9f76"}, + {file = "aiohttp-3.8.3-cp38-cp38-win_amd64.whl", hash = "sha256:84b14f36e85295fe69c6b9789b51a0903b774046d5f7df538176516c3e422446"}, + {file = "aiohttp-3.8.3-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:16c121ba0b1ec2b44b73e3a8a171c4f999b33929cd2397124a8c7fcfc8cd9e06"}, + {file = "aiohttp-3.8.3-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:8d6aaa4e7155afaf994d7924eb290abbe81a6905b303d8cb61310a2aba1c68ba"}, + {file = "aiohttp-3.8.3-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:43046a319664a04b146f81b40e1545d4c8ac7b7dd04c47e40bf09f65f2437346"}, + {file = "aiohttp-3.8.3-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:599418aaaf88a6d02a8c515e656f6faf3d10618d3dd95866eb4436520096c84b"}, + {file = "aiohttp-3.8.3-cp39-cp39-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:92a2964319d359f494f16011e23434f6f8ef0434acd3cf154a6b7bec511e2fb7"}, + {file = "aiohttp-3.8.3-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:73a4131962e6d91109bca6536416aa067cf6c4efb871975df734f8d2fd821b37"}, + {file = "aiohttp-3.8.3-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:598adde339d2cf7d67beaccda3f2ce7c57b3b412702f29c946708f69cf8222aa"}, + {file = "aiohttp-3.8.3-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:75880ed07be39beff1881d81e4a907cafb802f306efd6d2d15f2b3c69935f6fb"}, + {file = "aiohttp-3.8.3-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:a0239da9fbafd9ff82fd67c16704a7d1bccf0d107a300e790587ad05547681c8"}, + {file = "aiohttp-3.8.3-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:4e3a23ec214e95c9fe85a58470b660efe6534b83e6cbe38b3ed52b053d7cb6ad"}, + {file = "aiohttp-3.8.3-cp39-cp39-musllinux_1_1_ppc64le.whl", hash = "sha256:47841407cc89a4b80b0c52276f3cc8138bbbfba4b179ee3acbd7d77ae33f7ac4"}, + {file = "aiohttp-3.8.3-cp39-cp39-musllinux_1_1_s390x.whl", hash = "sha256:54d107c89a3ebcd13228278d68f1436d3f33f2dd2af5415e3feaeb1156e1a62c"}, + {file = "aiohttp-3.8.3-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:c37c5cce780349d4d51739ae682dec63573847a2a8dcb44381b174c3d9c8d403"}, + {file = "aiohttp-3.8.3-cp39-cp39-win32.whl", hash = "sha256:f178d2aadf0166be4df834c4953da2d7eef24719e8aec9a65289483eeea9d618"}, + {file = "aiohttp-3.8.3-cp39-cp39-win_amd64.whl", hash = "sha256:88e5be56c231981428f4f506c68b6a46fa25c4123a2e86d156c58a8369d31ab7"}, + {file = "aiohttp-3.8.3.tar.gz", hash = "sha256:3828fb41b7203176b82fe5d699e0d845435f2374750a44b480ea6b930f6be269"}, +] [package.dependencies] aiosignal = ">=1.1.2" @@ -34,7 +170,7 @@ multidict = ">=4.5,<7.0" yarl = ">=1.0,<2.0" [package.extras] -speedups = ["aiodns", "brotli", "cchardet"] +speedups = ["Brotli", "aiodns", "cchardet"] [[package]] name = "aioitertools" @@ -43,6 +179,10 @@ description = "itertools and builtins for AsyncIO and mixed iterables" category = "main" optional = true python-versions = ">=3.6" +files = [ + {file = "aioitertools-0.11.0-py3-none-any.whl", hash = "sha256:04b95e3dab25b449def24d7df809411c10e62aab0cbe31a50ca4e68748c43394"}, + {file = "aioitertools-0.11.0.tar.gz", hash = "sha256:42c68b8dd3a69c2bf7f2233bf7df4bb58b557bca5252ac02ed5187bbc67d6831"}, +] [package.dependencies] typing_extensions = {version = ">=4.0", markers = "python_version < \"3.10\""} @@ -54,6 +194,10 @@ description = "aiosignal: a list of registered asynchronous callbacks" category = "main" optional = true python-versions = ">=3.7" +files = [ + {file = "aiosignal-1.3.1-py3-none-any.whl", hash = "sha256:f8376fb07dd1e86a584e4fcdec80b36b7f81aac666ebc724e2c090300dd83b17"}, + {file = "aiosignal-1.3.1.tar.gz", hash = "sha256:54cd96e15e1649b75d6c87526a6ff0b6c1b0dd3459f43d9ca11d48c339b68cfc"}, +] [package.dependencies] frozenlist = ">=1.1.0" @@ -65,6 +209,10 @@ description = "Timeout context manager for asyncio programs" category = "main" optional = true python-versions = ">=3.6" +files = [ + {file = "async-timeout-4.0.2.tar.gz", hash = "sha256:2163e1640ddb52b7a8c80d0a67a08587e5d245cc9c553a74a847056bc2976b15"}, + {file = "async_timeout-4.0.2-py3-none-any.whl", hash = "sha256:8ca1e4fcf50d07413d66d1a5e416e42cfdf5851c981d679a09851a6853383b3c"}, +] [[package]] name = "attrs" @@ -73,12 +221,89 @@ description = "Classes Without Boilerplate" category = "main" optional = false python-versions = ">=3.5" +files = [ + {file = "attrs-22.1.0-py2.py3-none-any.whl", hash = "sha256:86efa402f67bf2df34f51a335487cf46b1ec130d02b8d39fd248abfd30da551c"}, + {file = "attrs-22.1.0.tar.gz", hash = "sha256:29adc2665447e5191d0e7c568fde78b21f9672d344281d0c6e1ab085429b22b6"}, +] [package.extras] dev = ["cloudpickle", "coverage[toml] (>=5.0.2)", "furo", "hypothesis", "mypy (>=0.900,!=0.940)", "pre-commit", "pympler", "pytest (>=4.3.0)", "pytest-mypy-plugins", "sphinx", "sphinx-notfound-page", "zope.interface"] docs = ["furo", "sphinx", "sphinx-notfound-page", "zope.interface"] tests = ["cloudpickle", "coverage[toml] (>=5.0.2)", "hypothesis", "mypy (>=0.900,!=0.940)", "pympler", "pytest (>=4.3.0)", "pytest-mypy-plugins", "zope.interface"] -tests_no_zope = ["cloudpickle", "coverage[toml] (>=5.0.2)", "hypothesis", "mypy (>=0.900,!=0.940)", "pympler", "pytest (>=4.3.0)", "pytest-mypy-plugins"] +tests-no-zope = ["cloudpickle", "coverage[toml] (>=5.0.2)", "hypothesis", "mypy (>=0.900,!=0.940)", "pympler", "pytest (>=4.3.0)", "pytest-mypy-plugins"] + +[[package]] +name = "azure-core" +version = "1.26.1" +description = "Microsoft Azure Core Library for Python" +category = "main" +optional = true +python-versions = ">=3.7" +files = [ + {file = "azure-core-1.26.1.zip", hash = "sha256:223b0e90cbdd1f03c41b195b03239899843f20d00964dbb85e64386873414a2d"}, + {file = "azure_core-1.26.1-py3-none-any.whl", hash = "sha256:726ffd1ded04a2c1cb53f9d9155cbb05ac5c1c2a29af4ef622e93e1c0a8bc92b"}, +] + +[package.dependencies] +requests = ">=2.18.4" +six = ">=1.11.0" +typing-extensions = ">=4.0.1" + +[package.extras] +aio = ["aiohttp (>=3.0)"] + +[[package]] +name = "azure-datalake-store" +version = "0.0.52" +description = "Azure Data Lake Store Filesystem Client Library for Python" +category = "main" +optional = true +python-versions = "*" +files = [ + {file = "azure-datalake-store-0.0.52.tar.gz", hash = "sha256:4198ddb32614d16d4502b43d5c9739f81432b7e0e4d75d30e05149fe6007fea2"}, + {file = "azure_datalake_store-0.0.52-py2.py3-none-any.whl", hash = "sha256:aaed72b9c856824aeab554f4dbe0ef2c6d0ff36700bdd8b93d8298793117c48e"}, +] + +[package.dependencies] +adal = ">=0.4.2" +cffi = "*" +requests = ">=2.20.0" + +[[package]] +name = "azure-identity" +version = "1.12.0" +description = "Microsoft Azure Identity Library for Python" +category = "main" +optional = true +python-versions = ">=3.7" +files = [ + {file = "azure-identity-1.12.0.zip", hash = "sha256:7f9b1ae7d97ea7af3f38dd09305e19ab81a1e16ab66ea186b6579d85c1ca2347"}, + {file = "azure_identity-1.12.0-py3-none-any.whl", hash = "sha256:2a58ce4a209a013e37eaccfd5937570ab99e9118b3e1acf875eed3a85d541b92"}, +] + +[package.dependencies] +azure-core = ">=1.11.0,<2.0.0" +cryptography = ">=2.5" +msal = ">=1.12.0,<2.0.0" +msal-extensions = ">=0.3.0,<2.0.0" +six = ">=1.12.0" + +[[package]] +name = "azure-storage-blob" +version = "12.14.1" +description = "Microsoft Azure Blob Storage Client Library for Python" +category = "main" +optional = true +python-versions = ">=3.7" +files = [ + {file = "azure-storage-blob-12.14.1.zip", hash = "sha256:860d4d82985a4bfc7d3271e71275af330f54f330a754355435a7ba749ccde997"}, + {file = "azure_storage_blob-12.14.1-py3-none-any.whl", hash = "sha256:52b84658e8df7853a3cf1c563814655b5028b979b2a87905b92aa6bb30be240e"}, +] + +[package.dependencies] +azure-core = ">=1.24.2,<2.0.0" +cryptography = ">=2.1.4" +msrest = ">=0.7.1" [[package]] name = "boto3" @@ -87,6 +312,10 @@ description = "The AWS SDK for Python" category = "main" optional = false python-versions = ">= 3.7" +files = [ + {file = "boto3-1.24.59-py3-none-any.whl", hash = "sha256:34ab44146a2c4e7f4e72737f4b27e6eb5e0a7855c2f4599e3d9199b6a0a2d575"}, + {file = "boto3-1.24.59.tar.gz", hash = "sha256:a50b4323f9579cfe22fcf5531fbd40b567d4d74c1adce06aeb5c95fce2a6fb40"}, +] [package.dependencies] botocore = ">=1.27.59,<1.28.0" @@ -103,6 +332,10 @@ description = "Low-level, data-driven core of boto 3." category = "main" optional = false python-versions = ">= 3.7" +files = [ + {file = "botocore-1.27.59-py3-none-any.whl", hash = "sha256:69d756791fc024bda54f6c53f71ae34e695ee41bbbc1743d9179c4837a4929da"}, + {file = "botocore-1.27.59.tar.gz", hash = "sha256:eda4aed6ee719a745d1288eaf1beb12f6f6448ad1fa12f159405db14ba9c92cf"}, +] [package.dependencies] jmespath = ">=0.7.1,<2.0.0" @@ -119,6 +352,10 @@ description = "A simple, correct PEP 517 build frontend" category = "dev" optional = false python-versions = ">=3.6" +files = [ + {file = "build-0.9.0-py3-none-any.whl", hash = "sha256:38a7a2b7a0bdc61a42a0a67509d88c71ecfc37b393baba770fae34e20929ff69"}, + {file = "build-0.9.0.tar.gz", hash = "sha256:1a07724e891cbd898923145eb7752ee7653674c511378eb9c7691aab1612bc3c"}, +] [package.dependencies] colorama = {version = "*", markers = "os_name == \"nt\""} @@ -139,6 +376,10 @@ description = "Python package for providing Mozilla's CA Bundle." category = "main" optional = false python-versions = ">=3.6" +files = [ + {file = "certifi-2022.12.7-py3-none-any.whl", hash = "sha256:4ad3232f5e926d6718ec31cfc1fcadfde020920e278684144551c91769c7bc18"}, + {file = "certifi-2022.12.7.tar.gz", hash = "sha256:35824b4c3a97115964b408844d64aa14db1cc518f6562e8d7261699d1350a9e3"}, +] [[package]] name = "cffi" @@ -147,6 +388,72 @@ description = "Foreign Function Interface for Python calling C code." category = "main" optional = false python-versions = "*" +files = [ + {file = "cffi-1.15.1-cp27-cp27m-macosx_10_9_x86_64.whl", hash = "sha256:a66d3508133af6e8548451b25058d5812812ec3798c886bf38ed24a98216fab2"}, + {file = "cffi-1.15.1-cp27-cp27m-manylinux1_i686.whl", hash = "sha256:470c103ae716238bbe698d67ad020e1db9d9dba34fa5a899b5e21577e6d52ed2"}, + {file = "cffi-1.15.1-cp27-cp27m-manylinux1_x86_64.whl", hash = "sha256:9ad5db27f9cabae298d151c85cf2bad1d359a1b9c686a275df03385758e2f914"}, + {file = "cffi-1.15.1-cp27-cp27m-win32.whl", hash = "sha256:b3bbeb01c2b273cca1e1e0c5df57f12dce9a4dd331b4fa1635b8bec26350bde3"}, + {file = "cffi-1.15.1-cp27-cp27m-win_amd64.whl", hash = "sha256:e00b098126fd45523dd056d2efba6c5a63b71ffe9f2bbe1a4fe1716e1d0c331e"}, + {file = "cffi-1.15.1-cp27-cp27mu-manylinux1_i686.whl", hash = "sha256:d61f4695e6c866a23a21acab0509af1cdfd2c013cf256bbf5b6b5e2695827162"}, + {file = "cffi-1.15.1-cp27-cp27mu-manylinux1_x86_64.whl", hash = "sha256:ed9cb427ba5504c1dc15ede7d516b84757c3e3d7868ccc85121d9310d27eed0b"}, + {file = "cffi-1.15.1-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:39d39875251ca8f612b6f33e6b1195af86d1b3e60086068be9cc053aa4376e21"}, + {file = "cffi-1.15.1-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:285d29981935eb726a4399badae8f0ffdff4f5050eaa6d0cfc3f64b857b77185"}, + {file = "cffi-1.15.1-cp310-cp310-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:3eb6971dcff08619f8d91607cfc726518b6fa2a9eba42856be181c6d0d9515fd"}, + {file = "cffi-1.15.1-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:21157295583fe8943475029ed5abdcf71eb3911894724e360acff1d61c1d54bc"}, + {file = "cffi-1.15.1-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:5635bd9cb9731e6d4a1132a498dd34f764034a8ce60cef4f5319c0541159392f"}, + {file = "cffi-1.15.1-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:2012c72d854c2d03e45d06ae57f40d78e5770d252f195b93f581acf3ba44496e"}, + {file = "cffi-1.15.1-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:dd86c085fae2efd48ac91dd7ccffcfc0571387fe1193d33b6394db7ef31fe2a4"}, + {file = "cffi-1.15.1-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:fa6693661a4c91757f4412306191b6dc88c1703f780c8234035eac011922bc01"}, + {file = "cffi-1.15.1-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:59c0b02d0a6c384d453fece7566d1c7e6b7bae4fc5874ef2ef46d56776d61c9e"}, + {file = "cffi-1.15.1-cp310-cp310-win32.whl", hash = "sha256:cba9d6b9a7d64d4bd46167096fc9d2f835e25d7e4c121fb2ddfc6528fb0413b2"}, + {file = "cffi-1.15.1-cp310-cp310-win_amd64.whl", hash = "sha256:ce4bcc037df4fc5e3d184794f27bdaab018943698f4ca31630bc7f84a7b69c6d"}, + {file = "cffi-1.15.1-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:3d08afd128ddaa624a48cf2b859afef385b720bb4b43df214f85616922e6a5ac"}, + {file = "cffi-1.15.1-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:3799aecf2e17cf585d977b780ce79ff0dc9b78d799fc694221ce814c2c19db83"}, + {file = "cffi-1.15.1-cp311-cp311-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:a591fe9e525846e4d154205572a029f653ada1a78b93697f3b5a8f1f2bc055b9"}, + {file = "cffi-1.15.1-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:3548db281cd7d2561c9ad9984681c95f7b0e38881201e157833a2342c30d5e8c"}, + {file = "cffi-1.15.1-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:91fc98adde3d7881af9b59ed0294046f3806221863722ba7d8d120c575314325"}, + {file = "cffi-1.15.1-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:94411f22c3985acaec6f83c6df553f2dbe17b698cc7f8ae751ff2237d96b9e3c"}, + {file = "cffi-1.15.1-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:03425bdae262c76aad70202debd780501fabeaca237cdfddc008987c0e0f59ef"}, + {file = "cffi-1.15.1-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:cc4d65aeeaa04136a12677d3dd0b1c0c94dc43abac5860ab33cceb42b801c1e8"}, + {file = "cffi-1.15.1-cp311-cp311-win32.whl", hash = "sha256:a0f100c8912c114ff53e1202d0078b425bee3649ae34d7b070e9697f93c5d52d"}, + {file = "cffi-1.15.1-cp311-cp311-win_amd64.whl", hash = "sha256:04ed324bda3cda42b9b695d51bb7d54b680b9719cfab04227cdd1e04e5de3104"}, + {file = "cffi-1.15.1-cp36-cp36m-macosx_10_9_x86_64.whl", hash = "sha256:50a74364d85fd319352182ef59c5c790484a336f6db772c1a9231f1c3ed0cbd7"}, + {file = "cffi-1.15.1-cp36-cp36m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:e263d77ee3dd201c3a142934a086a4450861778baaeeb45db4591ef65550b0a6"}, + {file = "cffi-1.15.1-cp36-cp36m-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:cec7d9412a9102bdc577382c3929b337320c4c4c4849f2c5cdd14d7368c5562d"}, + {file = "cffi-1.15.1-cp36-cp36m-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:4289fc34b2f5316fbb762d75362931e351941fa95fa18789191b33fc4cf9504a"}, + {file = "cffi-1.15.1-cp36-cp36m-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:173379135477dc8cac4bc58f45db08ab45d228b3363adb7af79436135d028405"}, + {file = "cffi-1.15.1-cp36-cp36m-manylinux_2_5_x86_64.manylinux1_x86_64.whl", hash = "sha256:6975a3fac6bc83c4a65c9f9fcab9e47019a11d3d2cf7f3c0d03431bf145a941e"}, + {file = "cffi-1.15.1-cp36-cp36m-win32.whl", hash = "sha256:2470043b93ff09bf8fb1d46d1cb756ce6132c54826661a32d4e4d132e1977adf"}, + {file = "cffi-1.15.1-cp36-cp36m-win_amd64.whl", hash = "sha256:30d78fbc8ebf9c92c9b7823ee18eb92f2e6ef79b45ac84db507f52fbe3ec4497"}, + {file = "cffi-1.15.1-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:198caafb44239b60e252492445da556afafc7d1e3ab7a1fb3f0584ef6d742375"}, + {file = "cffi-1.15.1-cp37-cp37m-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:5ef34d190326c3b1f822a5b7a45f6c4535e2f47ed06fec77d3d799c450b2651e"}, + {file = "cffi-1.15.1-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:8102eaf27e1e448db915d08afa8b41d6c7ca7a04b7d73af6514df10a3e74bd82"}, + {file = "cffi-1.15.1-cp37-cp37m-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:5df2768244d19ab7f60546d0c7c63ce1581f7af8b5de3eb3004b9b6fc8a9f84b"}, + {file = "cffi-1.15.1-cp37-cp37m-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:a8c4917bd7ad33e8eb21e9a5bbba979b49d9a97acb3a803092cbc1133e20343c"}, + {file = "cffi-1.15.1-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:0e2642fe3142e4cc4af0799748233ad6da94c62a8bec3a6648bf8ee68b1c7426"}, + {file = "cffi-1.15.1-cp37-cp37m-win32.whl", hash = "sha256:e229a521186c75c8ad9490854fd8bbdd9a0c9aa3a524326b55be83b54d4e0ad9"}, + {file = "cffi-1.15.1-cp37-cp37m-win_amd64.whl", hash = "sha256:a0b71b1b8fbf2b96e41c4d990244165e2c9be83d54962a9a1d118fd8657d2045"}, + {file = "cffi-1.15.1-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:320dab6e7cb2eacdf0e658569d2575c4dad258c0fcc794f46215e1e39f90f2c3"}, + {file = "cffi-1.15.1-cp38-cp38-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:1e74c6b51a9ed6589199c787bf5f9875612ca4a8a0785fb2d4a84429badaf22a"}, + {file = "cffi-1.15.1-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a5c84c68147988265e60416b57fc83425a78058853509c1b0629c180094904a5"}, + {file = "cffi-1.15.1-cp38-cp38-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:3b926aa83d1edb5aa5b427b4053dc420ec295a08e40911296b9eb1b6170f6cca"}, + {file = "cffi-1.15.1-cp38-cp38-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:87c450779d0914f2861b8526e035c5e6da0a3199d8f1add1a665e1cbc6fc6d02"}, + {file = "cffi-1.15.1-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:4f2c9f67e9821cad2e5f480bc8d83b8742896f1242dba247911072d4fa94c192"}, + {file = "cffi-1.15.1-cp38-cp38-win32.whl", hash = "sha256:8b7ee99e510d7b66cdb6c593f21c043c248537a32e0bedf02e01e9553a172314"}, + {file = "cffi-1.15.1-cp38-cp38-win_amd64.whl", hash = "sha256:00a9ed42e88df81ffae7a8ab6d9356b371399b91dbdf0c3cb1e84c03a13aceb5"}, + {file = "cffi-1.15.1-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:54a2db7b78338edd780e7ef7f9f6c442500fb0d41a5a4ea24fff1c929d5af585"}, + {file = "cffi-1.15.1-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:fcd131dd944808b5bdb38e6f5b53013c5aa4f334c5cad0c72742f6eba4b73db0"}, + {file = "cffi-1.15.1-cp39-cp39-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:7473e861101c9e72452f9bf8acb984947aa1661a7704553a9f6e4baa5ba64415"}, + {file = "cffi-1.15.1-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:6c9a799e985904922a4d207a94eae35c78ebae90e128f0c4e521ce339396be9d"}, + {file = "cffi-1.15.1-cp39-cp39-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:3bcde07039e586f91b45c88f8583ea7cf7a0770df3a1649627bf598332cb6984"}, + {file = "cffi-1.15.1-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:33ab79603146aace82c2427da5ca6e58f2b3f2fb5da893ceac0c42218a40be35"}, + {file = "cffi-1.15.1-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:5d598b938678ebf3c67377cdd45e09d431369c3b1a5b331058c338e201f12b27"}, + {file = "cffi-1.15.1-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:db0fbb9c62743ce59a9ff687eb5f4afbe77e5e8403d6697f7446e5f609976f76"}, + {file = "cffi-1.15.1-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:98d85c6a2bef81588d9227dde12db8a7f47f639f4a17c9ae08e773aa9c697bf3"}, + {file = "cffi-1.15.1-cp39-cp39-win32.whl", hash = "sha256:40f4774f5a9d4f5e344f31a32b5096977b5d48560c5592e2f3d2c4374bd543ee"}, + {file = "cffi-1.15.1-cp39-cp39-win_amd64.whl", hash = "sha256:70df4e3b545a17496c9b3f41f5115e69a4f2e77e94e1d2a8e1070bc0c38c8a3c"}, + {file = "cffi-1.15.1.tar.gz", hash = "sha256:d400bfb9a37b1351253cb402671cea7e89bdecc294e8016a707f6d1d8ac934f9"}, +] [package.dependencies] pycparser = "*" @@ -158,6 +465,10 @@ description = "Validate configuration and produce human readable error messages. category = "dev" optional = false python-versions = ">=3.6.1" +files = [ + {file = "cfgv-3.3.1-py2.py3-none-any.whl", hash = "sha256:c6a0883f3917a037485059700b9e75da2464e6c27051014ad85ba6aaa5884426"}, + {file = "cfgv-3.3.1.tar.gz", hash = "sha256:f5a830efb9ce7a445376bb66ec94c638a9787422f96264c98edc6bdeed8ab736"}, +] [[package]] name = "charset-normalizer" @@ -166,9 +477,13 @@ description = "The Real First Universal Charset Detector. Open, modern and activ category = "main" optional = false python-versions = ">=3.6.0" +files = [ + {file = "charset-normalizer-2.1.1.tar.gz", hash = "sha256:5a3d016c7c547f69d6f81fb0db9449ce888b418b5b9952cc5e6e66843e9dd845"}, + {file = "charset_normalizer-2.1.1-py3-none-any.whl", hash = "sha256:83e9a75d1911279afd89352c68b45348559d1fc0506b054b346651b5e7fee29f"}, +] [package.extras] -unicode_backport = ["unicodedata2"] +unicode-backport = ["unicodedata2"] [[package]] name = "click" @@ -177,6 +492,10 @@ description = "Composable command line interface toolkit" category = "main" optional = false python-versions = ">=3.7" +files = [ + {file = "click-8.1.3-py3-none-any.whl", hash = "sha256:bb4d8133cb15a609f44e8213d9b391b0809795062913b383c62be0ee95b1db48"}, + {file = "click-8.1.3.tar.gz", hash = "sha256:7682dc8afb30297001674575ea00d1814d808d6a36af415a82bd481d37ba7b8e"}, +] [package.dependencies] colorama = {version = "*", markers = "platform_system == \"Windows\""} @@ -188,6 +507,10 @@ description = "Cross-platform colored terminal text." category = "main" optional = false python-versions = "!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,!=3.4.*,!=3.5.*,!=3.6.*,>=2.7" +files = [ + {file = "colorama-0.4.6-py2.py3-none-any.whl", hash = "sha256:4f1d9991f5acc0ca119f9d443620b77f9d6b33703e51011c16baf57afb285fc6"}, + {file = "colorama-0.4.6.tar.gz", hash = "sha256:08695f5cb7ed6e0531a20572697297273c47b8cae5a63ffc6d6ed5c201be6e44"}, +] [[package]] name = "commonmark" @@ -196,6 +519,10 @@ description = "Python parser for the CommonMark Markdown spec" category = "main" optional = false python-versions = "*" +files = [ + {file = "commonmark-0.9.1-py2.py3-none-any.whl", hash = "sha256:da2f38c92590f83de410ba1a3cbceafbc74fee9def35f9251ba9a971d6d66fd9"}, + {file = "commonmark-0.9.1.tar.gz", hash = "sha256:452f9dc859be7f06631ddcb328b6919c67984aca654e5fefb3914d54691aed60"}, +] [package.extras] test = ["flake8 (==3.7.8)", "hypothesis (==3.55.3)"] @@ -207,956 +534,7 @@ description = "Code coverage measurement for Python" category = "dev" optional = false python-versions = ">=3.7" - -[package.dependencies] -tomli = {version = "*", optional = true, markers = "python_full_version <= \"3.11.0a6\" and extra == \"toml\""} - -[package.extras] -toml = ["tomli"] - -[[package]] -name = "cryptography" -version = "38.0.4" -description = "cryptography is a package which provides cryptographic recipes and primitives to Python developers." -category = "dev" -optional = false -python-versions = ">=3.6" - -[package.dependencies] -cffi = ">=1.12" - -[package.extras] -docs = ["sphinx (>=1.6.5,!=1.8.0,!=3.1.0,!=3.1.1)", "sphinx-rtd-theme"] -docstest = ["pyenchant (>=1.6.11)", "sphinxcontrib-spelling (>=4.0.1)", "twine (>=1.12.0)"] -pep8test = ["black", "flake8", "flake8-import-order", "pep8-naming"] -sdist = ["setuptools-rust (>=0.11.4)"] -ssh = ["bcrypt (>=3.1.5)"] -test = ["hypothesis (>=1.11.4,!=3.79.2)", "iso8601", "pretend", "pytest (>=6.2.0)", "pytest-benchmark", "pytest-cov", "pytest-subtests", "pytest-xdist", "pytz"] - -[[package]] -name = "distlib" -version = "0.3.6" -description = "Distribution utilities" -category = "dev" -optional = false -python-versions = "*" - -[[package]] -name = "docutils" -version = "0.19" -description = "Docutils -- Python Documentation Utilities" -category = "dev" -optional = false -python-versions = ">=3.7" - -[[package]] -name = "duckdb" -version = "0.6.1" -description = "DuckDB embedded database" -category = "main" -optional = true -python-versions = "*" - -[package.dependencies] -numpy = ">=1.14" - -[[package]] -name = "exceptiongroup" -version = "1.0.4" -description = "Backport of PEP 654 (exception groups)" -category = "dev" -optional = false -python-versions = ">=3.7" - -[package.extras] -test = ["pytest (>=6)"] - -[[package]] -name = "fastavro" -version = "1.7.0" -description = "Fast read/write of AVRO files" -category = "dev" -optional = false -python-versions = ">=3.7" - -[package.extras] -codecs = ["lz4", "python-snappy", "zstandard"] -lz4 = ["lz4"] -snappy = ["python-snappy"] -zstandard = ["zstandard"] - -[[package]] -name = "filelock" -version = "3.8.2" -description = "A platform independent file lock." -category = "dev" -optional = false -python-versions = ">=3.7" - -[package.extras] -docs = ["furo (>=2022.9.29)", "sphinx (>=5.3)", "sphinx-autodoc-typehints (>=1.19.5)"] -testing = ["covdefaults (>=2.2.2)", "coverage (>=6.5)", "pytest (>=7.2)", "pytest-cov (>=4)", "pytest-timeout (>=2.1)"] - -[[package]] -name = "frozenlist" -version = "1.3.3" -description = "A list-like structure which implements collections.abc.MutableSequence" -category = "main" -optional = true -python-versions = ">=3.7" - -[[package]] -name = "fsspec" -version = "2022.10.0" -description = "File-system specification" -category = "main" -optional = false -python-versions = ">=3.7" - -[package.extras] -abfs = ["adlfs"] -adl = ["adlfs"] -arrow = ["pyarrow (>=1)"] -dask = ["dask", "distributed"] -dropbox = ["dropbox", "dropboxdrivefs", "requests"] -entrypoints = ["importlib-metadata"] -fuse = ["fusepy"] -gcs = ["gcsfs"] -git = ["pygit2"] -github = ["requests"] -gs = ["gcsfs"] -gui = ["panel"] -hdfs = ["pyarrow (>=1)"] -http = ["aiohttp (!=4.0.0a0,!=4.0.0a1)", "requests"] -libarchive = ["libarchive-c"] -oci = ["ocifs"] -s3 = ["s3fs"] -sftp = ["paramiko"] -smb = ["smbprotocol"] -ssh = ["paramiko"] -tqdm = ["tqdm"] - -[[package]] -name = "identify" -version = "2.5.10" -description = "File identification library for Python" -category = "dev" -optional = false -python-versions = ">=3.7" - -[package.extras] -license = ["ukkonen"] - -[[package]] -name = "idna" -version = "3.4" -description = "Internationalized Domain Names in Applications (IDNA)" -category = "main" -optional = false -python-versions = ">=3.5" - -[[package]] -name = "importlib-metadata" -version = "5.1.0" -description = "Read metadata from Python packages" -category = "dev" -optional = false -python-versions = ">=3.7" - -[package.dependencies] -zipp = ">=0.5" - -[package.extras] -docs = ["furo", "jaraco.packaging (>=9)", "jaraco.tidelift (>=1.4)", "rst.linker (>=1.9)", "sphinx (>=3.5)"] -perf = ["ipython"] -testing = ["flake8 (<5)", "flufl.flake8", "importlib-resources (>=1.3)", "packaging", "pyfakefs", "pytest (>=6)", "pytest-black (>=0.3.7)", "pytest-checkdocs (>=2.4)", "pytest-cov", "pytest-enabler (>=1.3)", "pytest-flake8", "pytest-mypy (>=0.9.1)", "pytest-perf (>=0.9.2)"] - -[[package]] -name = "iniconfig" -version = "1.1.1" -description = "iniconfig: brain-dead simple config-ini parsing" -category = "dev" -optional = false -python-versions = "*" - -[[package]] -name = "jinja2" -version = "3.1.2" -description = "A very fast and expressive template engine." -category = "dev" -optional = false -python-versions = ">=3.7" - -[package.dependencies] -MarkupSafe = ">=2.0" - -[package.extras] -i18n = ["Babel (>=2.7)"] - -[[package]] -name = "jmespath" -version = "1.0.1" -description = "JSON Matching Expressions" -category = "main" -optional = false -python-versions = ">=3.7" - -[[package]] -name = "markupsafe" -version = "2.1.1" -description = "Safely add untrusted strings to HTML/XML markup." -category = "dev" -optional = false -python-versions = ">=3.7" - -[[package]] -name = "mmhash3" -version = "3.0.1" -description = "Python wrapper for MurmurHash (MurmurHash3), a set of fast and robust hash functions." -category = "main" -optional = false -python-versions = "*" - -[[package]] -name = "moto" -version = "4.0.11" -description = "A library that allows your python tests to easily mock out the boto library" -category = "dev" -optional = false -python-versions = ">=3.6" - -[package.dependencies] -boto3 = ">=1.9.201" -botocore = ">=1.12.201" -cryptography = ">=3.3.1" -Jinja2 = ">=2.10.1" -MarkupSafe = "!=2.0.0a1" -python-dateutil = ">=2.1,<3.0.0" -pytz = "*" -requests = ">=2.5" -responses = ">=0.13.0" -werkzeug = ">=0.5,<2.2.0 || >2.2.0,<2.2.1 || >2.2.1" -xmltodict = "*" - -[package.extras] -all = ["PyYAML (>=5.1)", "aws-xray-sdk (>=0.93,!=0.96)", "cfn-lint (>=0.40.0)", "docker (>=2.5.1)", "ecdsa (!=0.15)", "graphql-core", "idna (>=2.5,<4)", "jsondiff (>=1.1.2)", "openapi-spec-validator (>=0.2.8)", "pyparsing (>=3.0.7)", "python-jose[cryptography] (>=3.1.0,<4.0.0)", "setuptools", "sshpubkeys (>=3.1.0)"] -apigateway = ["PyYAML (>=5.1)", "ecdsa (!=0.15)", "openapi-spec-validator (>=0.2.8)", "python-jose[cryptography] (>=3.1.0,<4.0.0)"] -apigatewayv2 = ["PyYAML (>=5.1)"] -appsync = ["graphql-core"] -awslambda = ["docker (>=2.5.1)"] -batch = ["docker (>=2.5.1)"] -cloudformation = ["PyYAML (>=5.1)", "aws-xray-sdk (>=0.93,!=0.96)", "cfn-lint (>=0.40.0)", "docker (>=2.5.1)", "ecdsa (!=0.15)", "graphql-core", "idna (>=2.5,<4)", "jsondiff (>=1.1.2)", "openapi-spec-validator (>=0.2.8)", "pyparsing (>=3.0.7)", "python-jose[cryptography] (>=3.1.0,<4.0.0)", "setuptools", "sshpubkeys (>=3.1.0)"] -cognitoidp = ["ecdsa (!=0.15)", "python-jose[cryptography] (>=3.1.0,<4.0.0)"] -ds = ["sshpubkeys (>=3.1.0)"] -dynamodb = ["docker (>=2.5.1)"] -dynamodb2 = ["docker (>=2.5.1)"] -dynamodbstreams = ["docker (>=2.5.1)"] -ebs = ["sshpubkeys (>=3.1.0)"] -ec2 = ["sshpubkeys (>=3.1.0)"] -efs = ["sshpubkeys (>=3.1.0)"] -glue = ["pyparsing (>=3.0.7)"] -iotdata = ["jsondiff (>=1.1.2)"] -route53resolver = ["sshpubkeys (>=3.1.0)"] -s3 = ["PyYAML (>=5.1)"] -server = ["PyYAML (>=5.1)", "aws-xray-sdk (>=0.93,!=0.96)", "cfn-lint (>=0.40.0)", "docker (>=2.5.1)", "ecdsa (!=0.15)", "flask (!=2.2.0,!=2.2.1)", "flask-cors", "graphql-core", "idna (>=2.5,<4)", "jsondiff (>=1.1.2)", "openapi-spec-validator (>=0.2.8)", "pyparsing (>=3.0.7)", "python-jose[cryptography] (>=3.1.0,<4.0.0)", "setuptools", "sshpubkeys (>=3.1.0)"] -ssm = ["PyYAML (>=5.1)", "dataclasses"] -xray = ["aws-xray-sdk (>=0.93,!=0.96)", "setuptools"] - -[[package]] -name = "multidict" -version = "6.0.3" -description = "multidict implementation" -category = "main" -optional = true -python-versions = ">=3.7" - -[[package]] -name = "nodeenv" -version = "1.7.0" -description = "Node.js virtual environment builder" -category = "dev" -optional = false -python-versions = ">=2.7,!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,!=3.4.*,!=3.5.*,!=3.6.*" - -[[package]] -name = "numpy" -version = "1.23.5" -description = "NumPy is the fundamental package for array computing with Python." -category = "main" -optional = true -python-versions = ">=3.8" - -[[package]] -name = "packaging" -version = "22.0" -description = "Core utilities for Python packages" -category = "dev" -optional = false -python-versions = ">=3.7" - -[[package]] -name = "pandas" -version = "1.5.2" -description = "Powerful data structures for data analysis, time series, and statistics" -category = "main" -optional = true -python-versions = ">=3.8" - -[package.dependencies] -numpy = [ - {version = ">=1.20.3", markers = "python_version < \"3.10\""}, - {version = ">=1.21.0", markers = "python_version >= \"3.10\""}, - {version = ">=1.23.2", markers = "python_version >= \"3.11\""}, -] -python-dateutil = ">=2.8.1" -pytz = ">=2020.1" - -[package.extras] -test = ["hypothesis (>=5.5.3)", "pytest (>=6.0)", "pytest-xdist (>=1.31)"] - -[[package]] -name = "pep517" -version = "0.13.0" -description = "Wrappers to build Python packages using PEP 517 hooks" -category = "dev" -optional = false -python-versions = ">=3.6" - -[package.dependencies] -tomli = {version = ">=1.1.0", markers = "python_version < \"3.11\""} - -[[package]] -name = "platformdirs" -version = "2.6.0" -description = "A small Python package for determining appropriate platform-specific dirs, e.g. a \"user data dir\"." -category = "dev" -optional = false -python-versions = ">=3.7" - -[package.extras] -docs = ["furo (>=2022.9.29)", "proselint (>=0.13)", "sphinx (>=5.3)", "sphinx-autodoc-typehints (>=1.19.4)"] -test = ["appdirs (==1.4.4)", "pytest (>=7.2)", "pytest-cov (>=4)", "pytest-mock (>=3.10)"] - -[[package]] -name = "pluggy" -version = "1.0.0" -description = "plugin and hook calling mechanisms for python" -category = "dev" -optional = false -python-versions = ">=3.6" - -[package.extras] -dev = ["pre-commit", "tox"] -testing = ["pytest", "pytest-benchmark"] - -[[package]] -name = "pre-commit" -version = "2.20.0" -description = "A framework for managing and maintaining multi-language pre-commit hooks." -category = "dev" -optional = false -python-versions = ">=3.7" - -[package.dependencies] -cfgv = ">=2.0.0" -identify = ">=1.0.0" -nodeenv = ">=0.11.1" -pyyaml = ">=5.1" -toml = "*" -virtualenv = ">=20.0.8" - -[[package]] -name = "pyarrow" -version = "10.0.1" -description = "Python library for Apache Arrow" -category = "main" -optional = true -python-versions = ">=3.7" - -[package.dependencies] -numpy = ">=1.16.6" - -[[package]] -name = "pycparser" -version = "2.21" -description = "C parser in Python" -category = "main" -optional = false -python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*" - -[[package]] -name = "pydantic" -version = "1.10.2" -description = "Data validation and settings management using python type hints" -category = "main" -optional = false -python-versions = ">=3.7" - -[package.dependencies] -typing-extensions = ">=4.1.0" - -[package.extras] -dotenv = ["python-dotenv (>=0.10.4)"] -email = ["email-validator (>=1.0.3)"] - -[[package]] -name = "pygments" -version = "2.13.0" -description = "Pygments is a syntax highlighting package written in Python." -category = "main" -optional = false -python-versions = ">=3.6" - -[package.extras] -plugins = ["importlib-metadata"] - -[[package]] -name = "pyparsing" -version = "3.0.9" -description = "pyparsing module - Classes and methods to define and execute parsing grammars" -category = "main" -optional = false -python-versions = ">=3.6.8" - -[package.extras] -diagrams = ["jinja2", "railroad-diagrams"] - -[[package]] -name = "pytest" -version = "7.2.0" -description = "pytest: simple powerful testing with Python" -category = "dev" -optional = false -python-versions = ">=3.7" - -[package.dependencies] -attrs = ">=19.2.0" -colorama = {version = "*", markers = "sys_platform == \"win32\""} -exceptiongroup = {version = ">=1.0.0rc8", markers = "python_version < \"3.11\""} -iniconfig = "*" -packaging = "*" -pluggy = ">=0.12,<2.0" -tomli = {version = ">=1.0.0", markers = "python_version < \"3.11\""} - -[package.extras] -testing = ["argcomplete", "hypothesis (>=3.56)", "mock", "nose", "pygments (>=2.7.2)", "requests", "xmlschema"] - -[[package]] -name = "pytest-checkdocs" -version = "2.9.0" -description = "check the README when running tests" -category = "dev" -optional = false -python-versions = ">=3.7" - -[package.dependencies] -build = "*" -docutils = ">=0.15" -importlib-metadata = {version = ">=4", markers = "python_version < \"3.10\""} - -[package.extras] -docs = ["furo", "jaraco.packaging (>=9)", "rst.linker (>=1.9)", "sphinx (>=3.5)"] -testing = ["flake8 (<5)", "pytest (>=6)", "pytest-black (>=0.3.7)", "pytest-checkdocs (>=2.4)", "pytest-cov", "pytest-enabler (>=1.3)", "pytest-flake8", "pytest-mypy (>=0.9.1)", "types-docutils"] - -[[package]] -name = "python-dateutil" -version = "2.8.2" -description = "Extensions to the standard Python datetime module" -category = "main" -optional = false -python-versions = "!=3.0.*,!=3.1.*,!=3.2.*,>=2.7" - -[package.dependencies] -six = ">=1.5" - -[[package]] -name = "python-snappy" -version = "0.6.1" -description = "Python library for the snappy compression library from Google" -category = "main" -optional = true -python-versions = "*" - -[[package]] -name = "pytz" -version = "2022.6" -description = "World timezone definitions, modern and historical" -category = "main" -optional = false -python-versions = "*" - -[[package]] -name = "pyyaml" -version = "6.0" -description = "YAML parser and emitter for Python" -category = "main" -optional = false -python-versions = ">=3.6" - -[[package]] -name = "requests" -version = "2.28.1" -description = "Python HTTP for Humans." -category = "main" -optional = false -python-versions = ">=3.7, <4" - -[package.dependencies] -certifi = ">=2017.4.17" -charset-normalizer = ">=2,<3" -idna = ">=2.5,<4" -urllib3 = ">=1.21.1,<1.27" - -[package.extras] -socks = ["PySocks (>=1.5.6,!=1.5.7)"] -use_chardet_on_py3 = ["chardet (>=3.0.2,<6)"] - -[[package]] -name = "requests-mock" -version = "1.10.0" -description = "Mock out responses from the requests package" -category = "dev" -optional = false -python-versions = "*" - -[package.dependencies] -requests = ">=2.3,<3" -six = "*" - -[package.extras] -fixture = ["fixtures"] -test = ["fixtures", "mock", "purl", "pytest", "requests-futures", "sphinx", "testrepository (>=0.0.18)", "testtools"] - -[[package]] -name = "responses" -version = "0.22.0" -description = "A utility library for mocking out the `requests` Python library." -category = "dev" -optional = false -python-versions = ">=3.7" - -[package.dependencies] -requests = ">=2.22.0,<3.0" -toml = "*" -types-toml = "*" -urllib3 = ">=1.25.10" - -[package.extras] -tests = ["coverage (>=6.0.0)", "flake8", "mypy", "pytest (>=7.0.0)", "pytest-asyncio", "pytest-cov", "pytest-httpserver", "types-requests"] - -[[package]] -name = "rich" -version = "12.6.0" -description = "Render rich text, tables, progress bars, syntax highlighting, markdown and more to the terminal" -category = "main" -optional = false -python-versions = ">=3.6.3,<4.0.0" - -[package.dependencies] -commonmark = ">=0.9.0,<0.10.0" -pygments = ">=2.6.0,<3.0.0" -typing-extensions = {version = ">=4.0.0,<5.0", markers = "python_version < \"3.9\""} - -[package.extras] -jupyter = ["ipywidgets (>=7.5.1,<8.0.0)"] - -[[package]] -name = "s3fs" -version = "2022.10.0" -description = "Convenient Filesystem interface over S3" -category = "main" -optional = true -python-versions = ">= 3.7" - -[package.dependencies] -aiobotocore = ">=2.4.0,<2.5.0" -aiohttp = "<4.0.0a0 || >4.0.0a0,<4.0.0a1 || >4.0.0a1" -fsspec = "2022.10.0" - -[package.extras] -awscli = ["aiobotocore[awscli] (>=2.4.0,<2.5.0)"] -boto3 = ["aiobotocore[boto3] (>=2.4.0,<2.5.0)"] - -[[package]] -name = "s3transfer" -version = "0.6.0" -description = "An Amazon S3 Transfer Manager" -category = "main" -optional = false -python-versions = ">= 3.7" - -[package.dependencies] -botocore = ">=1.12.36,<2.0a.0" - -[package.extras] -crt = ["botocore[crt] (>=1.20.29,<2.0a.0)"] - -[[package]] -name = "six" -version = "1.16.0" -description = "Python 2 and 3 compatibility utilities" -category = "main" -optional = false -python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*" - -[[package]] -name = "thrift" -version = "0.16.0" -description = "Python bindings for the Apache Thrift RPC system" -category = "main" -optional = true -python-versions = "*" - -[package.dependencies] -six = ">=1.7.2" - -[package.extras] -all = ["tornado (>=4.0)", "twisted"] -tornado = ["tornado (>=4.0)"] -twisted = ["twisted"] - -[[package]] -name = "toml" -version = "0.10.2" -description = "Python Library for Tom's Obvious, Minimal Language" -category = "dev" -optional = false -python-versions = ">=2.6, !=3.0.*, !=3.1.*, !=3.2.*" - -[[package]] -name = "tomli" -version = "2.0.1" -description = "A lil' TOML parser" -category = "dev" -optional = false -python-versions = ">=3.7" - -[[package]] -name = "types-toml" -version = "0.10.8.1" -description = "Typing stubs for toml" -category = "dev" -optional = false -python-versions = "*" - -[[package]] -name = "typing-extensions" -version = "4.4.0" -description = "Backported and Experimental Type Hints for Python 3.7+" -category = "main" -optional = false -python-versions = ">=3.7" - -[[package]] -name = "urllib3" -version = "1.26.13" -description = "HTTP library with thread-safe connection pooling, file post, and more." -category = "main" -optional = false -python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*, !=3.5.*" - -[package.extras] -brotli = ["brotli (>=1.0.9)", "brotlicffi (>=0.8.0)", "brotlipy (>=0.6.0)"] -secure = ["certifi", "cryptography (>=1.3.4)", "idna (>=2.0.0)", "ipaddress", "pyOpenSSL (>=0.14)", "urllib3-secure-extra"] -socks = ["PySocks (>=1.5.6,!=1.5.7,<2.0)"] - -[[package]] -name = "virtualenv" -version = "20.17.1" -description = "Virtual Python Environment builder" -category = "dev" -optional = false -python-versions = ">=3.6" - -[package.dependencies] -distlib = ">=0.3.6,<1" -filelock = ">=3.4.1,<4" -platformdirs = ">=2.4,<3" - -[package.extras] -docs = ["proselint (>=0.13)", "sphinx (>=5.3)", "sphinx-argparse (>=0.3.2)", "sphinx-rtd-theme (>=1)", "towncrier (>=22.8)"] -testing = ["coverage (>=6.2)", "coverage-enable-subprocess (>=1)", "flaky (>=3.7)", "packaging (>=21.3)", "pytest (>=7.0.1)", "pytest-env (>=0.6.2)", "pytest-freezegun (>=0.4.2)", "pytest-mock (>=3.6.1)", "pytest-randomly (>=3.10.3)", "pytest-timeout (>=2.1)"] - -[[package]] -name = "werkzeug" -version = "2.2.2" -description = "The comprehensive WSGI web application library." -category = "dev" -optional = false -python-versions = ">=3.7" - -[package.dependencies] -MarkupSafe = ">=2.1.1" - -[package.extras] -watchdog = ["watchdog"] - -[[package]] -name = "wrapt" -version = "1.14.1" -description = "Module for decorators, wrappers and monkey patching." -category = "main" -optional = true -python-versions = "!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,!=3.4.*,>=2.7" - -[[package]] -name = "xmltodict" -version = "0.13.0" -description = "Makes working with XML feel like you are working with JSON" -category = "dev" -optional = false -python-versions = ">=3.4" - -[[package]] -name = "yarl" -version = "1.8.2" -description = "Yet another URL library" -category = "main" -optional = true -python-versions = ">=3.7" - -[package.dependencies] -idna = ">=2.0" -multidict = ">=4.0" - -[[package]] -name = "zipp" -version = "3.11.0" -description = "Backport of pathlib-compatible object wrapper for zip files" -category = "dev" -optional = false -python-versions = ">=3.7" - -[package.extras] -docs = ["furo", "jaraco.packaging (>=9)", "jaraco.tidelift (>=1.4)", "rst.linker (>=1.9)", "sphinx (>=3.5)"] -testing = ["flake8 (<5)", "func-timeout", "jaraco.functools", "jaraco.itertools", "more-itertools", "pytest (>=6)", "pytest-black (>=0.3.7)", "pytest-checkdocs (>=2.4)", "pytest-cov", "pytest-enabler (>=1.3)", "pytest-flake8", "pytest-mypy (>=0.9.1)"] - -[[package]] -name = "zstandard" -version = "0.19.0" -description = "Zstandard bindings for Python" -category = "main" -optional = false -python-versions = ">=3.6" - -[package.dependencies] -cffi = {version = ">=1.11", markers = "platform_python_implementation == \"PyPy\""} - -[package.extras] -cffi = ["cffi (>=1.11)"] - -[extras] -duckdb = ["duckdb", "pyarrow"] -glue = ["boto3"] -hive = ["thrift"] -pandas = ["pandas", "pyarrow"] -pyarrow = ["pyarrow"] -s3fs = ["s3fs"] -snappy = ["python-snappy"] - -[metadata] -lock-version = "1.1" -python-versions = "^3.8" -content-hash = "5998c3a4d17fc291d61bead0350b2bb830b112fdafea26f6b2f1e456876733db" - -[metadata.files] -aiobotocore = [ - {file = "aiobotocore-2.4.1-py3-none-any.whl", hash = "sha256:26e0d55d5901f487c3467616c028abb85036ca33a0f88e279770adae6b865c69"}, - {file = "aiobotocore-2.4.1.tar.gz", hash = "sha256:5c8ee79d1f14273e3f8f3af5c893db889ef163227b178ecacb5aea4547d3b9a7"}, -] -aiohttp = [ - {file = "aiohttp-3.8.3-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:ba71c9b4dcbb16212f334126cc3d8beb6af377f6703d9dc2d9fb3874fd667ee9"}, - {file = "aiohttp-3.8.3-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:d24b8bb40d5c61ef2d9b6a8f4528c2f17f1c5d2d31fed62ec860f6006142e83e"}, - {file = "aiohttp-3.8.3-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:f88df3a83cf9df566f171adba39d5bd52814ac0b94778d2448652fc77f9eb491"}, - {file = "aiohttp-3.8.3-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:b97decbb3372d4b69e4d4c8117f44632551c692bb1361b356a02b97b69e18a62"}, - {file = "aiohttp-3.8.3-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:309aa21c1d54b8ef0723181d430347d7452daaff93e8e2363db8e75c72c2fb2d"}, - {file = "aiohttp-3.8.3-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:ad5383a67514e8e76906a06741febd9126fc7c7ff0f599d6fcce3e82b80d026f"}, - {file = "aiohttp-3.8.3-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:20acae4f268317bb975671e375493dbdbc67cddb5f6c71eebdb85b34444ac46b"}, - {file = "aiohttp-3.8.3-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:05a3c31c6d7cd08c149e50dc7aa2568317f5844acd745621983380597f027a18"}, - {file = "aiohttp-3.8.3-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:d6f76310355e9fae637c3162936e9504b4767d5c52ca268331e2756e54fd4ca5"}, - {file = "aiohttp-3.8.3-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:256deb4b29fe5e47893fa32e1de2d73c3afe7407738bd3c63829874661d4822d"}, - {file = "aiohttp-3.8.3-cp310-cp310-musllinux_1_1_ppc64le.whl", hash = "sha256:5c59fcd80b9049b49acd29bd3598cada4afc8d8d69bd4160cd613246912535d7"}, - {file = "aiohttp-3.8.3-cp310-cp310-musllinux_1_1_s390x.whl", hash = "sha256:059a91e88f2c00fe40aed9031b3606c3f311414f86a90d696dd982e7aec48142"}, - {file = "aiohttp-3.8.3-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:2feebbb6074cdbd1ac276dbd737b40e890a1361b3cc30b74ac2f5e24aab41f7b"}, - {file = "aiohttp-3.8.3-cp310-cp310-win32.whl", hash = "sha256:5bf651afd22d5f0c4be16cf39d0482ea494f5c88f03e75e5fef3a85177fecdeb"}, - {file = "aiohttp-3.8.3-cp310-cp310-win_amd64.whl", hash = "sha256:653acc3880459f82a65e27bd6526e47ddf19e643457d36a2250b85b41a564715"}, - {file = "aiohttp-3.8.3-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:86fc24e58ecb32aee09f864cb11bb91bc4c1086615001647dbfc4dc8c32f4008"}, - {file = "aiohttp-3.8.3-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:75e14eac916f024305db517e00a9252714fce0abcb10ad327fb6dcdc0d060f1d"}, - {file = "aiohttp-3.8.3-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:d1fde0f44029e02d02d3993ad55ce93ead9bb9b15c6b7ccd580f90bd7e3de476"}, - {file = "aiohttp-3.8.3-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:4ab94426ddb1ecc6a0b601d832d5d9d421820989b8caa929114811369673235c"}, - {file = "aiohttp-3.8.3-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:89d2e02167fa95172c017732ed7725bc8523c598757f08d13c5acca308e1a061"}, - {file = "aiohttp-3.8.3-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:02f9a2c72fc95d59b881cf38a4b2be9381b9527f9d328771e90f72ac76f31ad8"}, - {file = "aiohttp-3.8.3-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:9c7149272fb5834fc186328e2c1fa01dda3e1fa940ce18fded6d412e8f2cf76d"}, - {file = "aiohttp-3.8.3-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:512bd5ab136b8dc0ffe3fdf2dfb0c4b4f49c8577f6cae55dca862cd37a4564e2"}, - {file = "aiohttp-3.8.3-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:7018ecc5fe97027214556afbc7c502fbd718d0740e87eb1217b17efd05b3d276"}, - {file = "aiohttp-3.8.3-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:88c70ed9da9963d5496d38320160e8eb7e5f1886f9290475a881db12f351ab5d"}, - {file = "aiohttp-3.8.3-cp311-cp311-musllinux_1_1_ppc64le.whl", hash = "sha256:da22885266bbfb3f78218dc40205fed2671909fbd0720aedba39b4515c038091"}, - {file = "aiohttp-3.8.3-cp311-cp311-musllinux_1_1_s390x.whl", hash = "sha256:e65bc19919c910127c06759a63747ebe14f386cda573d95bcc62b427ca1afc73"}, - {file = "aiohttp-3.8.3-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:08c78317e950e0762c2983f4dd58dc5e6c9ff75c8a0efeae299d363d439c8e34"}, - {file = "aiohttp-3.8.3-cp311-cp311-win32.whl", hash = "sha256:45d88b016c849d74ebc6f2b6e8bc17cabf26e7e40c0661ddd8fae4c00f015697"}, - {file = "aiohttp-3.8.3-cp311-cp311-win_amd64.whl", hash = "sha256:96372fc29471646b9b106ee918c8eeb4cca423fcbf9a34daa1b93767a88a2290"}, - {file = "aiohttp-3.8.3-cp36-cp36m-macosx_10_9_x86_64.whl", hash = "sha256:c971bf3786b5fad82ce5ad570dc6ee420f5b12527157929e830f51c55dc8af77"}, - {file = "aiohttp-3.8.3-cp36-cp36m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:ff25f48fc8e623d95eca0670b8cc1469a83783c924a602e0fbd47363bb54aaca"}, - {file = "aiohttp-3.8.3-cp36-cp36m-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:e381581b37db1db7597b62a2e6b8b57c3deec95d93b6d6407c5b61ddc98aca6d"}, - {file = "aiohttp-3.8.3-cp36-cp36m-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:db19d60d846283ee275d0416e2a23493f4e6b6028825b51290ac05afc87a6f97"}, - {file = "aiohttp-3.8.3-cp36-cp36m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:25892c92bee6d9449ffac82c2fe257f3a6f297792cdb18ad784737d61e7a9a85"}, - {file = "aiohttp-3.8.3-cp36-cp36m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:398701865e7a9565d49189f6c90868efaca21be65c725fc87fc305906be915da"}, - {file = "aiohttp-3.8.3-cp36-cp36m-musllinux_1_1_aarch64.whl", hash = "sha256:4a4fbc769ea9b6bd97f4ad0b430a6807f92f0e5eb020f1e42ece59f3ecfc4585"}, - {file = "aiohttp-3.8.3-cp36-cp36m-musllinux_1_1_i686.whl", hash = "sha256:b29bfd650ed8e148f9c515474a6ef0ba1090b7a8faeee26b74a8ff3b33617502"}, - {file = "aiohttp-3.8.3-cp36-cp36m-musllinux_1_1_ppc64le.whl", hash = "sha256:1e56b9cafcd6531bab5d9b2e890bb4937f4165109fe98e2b98ef0dcfcb06ee9d"}, - {file = "aiohttp-3.8.3-cp36-cp36m-musllinux_1_1_s390x.whl", hash = "sha256:ec40170327d4a404b0d91855d41bfe1fe4b699222b2b93e3d833a27330a87a6d"}, - {file = "aiohttp-3.8.3-cp36-cp36m-musllinux_1_1_x86_64.whl", hash = "sha256:2df5f139233060578d8c2c975128fb231a89ca0a462b35d4b5fcf7c501ebdbe1"}, - {file = "aiohttp-3.8.3-cp36-cp36m-win32.whl", hash = "sha256:f973157ffeab5459eefe7b97a804987876dd0a55570b8fa56b4e1954bf11329b"}, - {file = "aiohttp-3.8.3-cp36-cp36m-win_amd64.whl", hash = "sha256:437399385f2abcd634865705bdc180c8314124b98299d54fe1d4c8990f2f9494"}, - {file = "aiohttp-3.8.3-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:09e28f572b21642128ef31f4e8372adb6888846f32fecb288c8b0457597ba61a"}, - {file = "aiohttp-3.8.3-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:6f3553510abdbec67c043ca85727396ceed1272eef029b050677046d3387be8d"}, - {file = "aiohttp-3.8.3-cp37-cp37m-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:e168a7560b7c61342ae0412997b069753f27ac4862ec7867eff74f0fe4ea2ad9"}, - {file = "aiohttp-3.8.3-cp37-cp37m-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:db4c979b0b3e0fa7e9e69ecd11b2b3174c6963cebadeecfb7ad24532ffcdd11a"}, - {file = "aiohttp-3.8.3-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:e164e0a98e92d06da343d17d4e9c4da4654f4a4588a20d6c73548a29f176abe2"}, - {file = "aiohttp-3.8.3-cp37-cp37m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:e8a78079d9a39ca9ca99a8b0ac2fdc0c4d25fc80c8a8a82e5c8211509c523363"}, - {file = "aiohttp-3.8.3-cp37-cp37m-musllinux_1_1_aarch64.whl", hash = "sha256:21b30885a63c3f4ff5b77a5d6caf008b037cb521a5f33eab445dc566f6d092cc"}, - {file = "aiohttp-3.8.3-cp37-cp37m-musllinux_1_1_i686.whl", hash = "sha256:4b0f30372cef3fdc262f33d06e7b411cd59058ce9174ef159ad938c4a34a89da"}, - {file = "aiohttp-3.8.3-cp37-cp37m-musllinux_1_1_ppc64le.whl", hash = "sha256:8135fa153a20d82ffb64f70a1b5c2738684afa197839b34cc3e3c72fa88d302c"}, - {file = "aiohttp-3.8.3-cp37-cp37m-musllinux_1_1_s390x.whl", hash = "sha256:ad61a9639792fd790523ba072c0555cd6be5a0baf03a49a5dd8cfcf20d56df48"}, - {file = "aiohttp-3.8.3-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:978b046ca728073070e9abc074b6299ebf3501e8dee5e26efacb13cec2b2dea0"}, - {file = "aiohttp-3.8.3-cp37-cp37m-win32.whl", hash = "sha256:0d2c6d8c6872df4a6ec37d2ede71eff62395b9e337b4e18efd2177de883a5033"}, - {file = "aiohttp-3.8.3-cp37-cp37m-win_amd64.whl", hash = "sha256:21d69797eb951f155026651f7e9362877334508d39c2fc37bd04ff55b2007091"}, - {file = "aiohttp-3.8.3-cp38-cp38-macosx_10_9_universal2.whl", hash = "sha256:2ca9af5f8f5812d475c5259393f52d712f6d5f0d7fdad9acdb1107dd9e3cb7eb"}, - {file = "aiohttp-3.8.3-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:1d90043c1882067f1bd26196d5d2db9aa6d268def3293ed5fb317e13c9413ea4"}, - {file = "aiohttp-3.8.3-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:d737fc67b9a970f3234754974531dc9afeea11c70791dcb7db53b0cf81b79784"}, - {file = "aiohttp-3.8.3-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:ebf909ea0a3fc9596e40d55d8000702a85e27fd578ff41a5500f68f20fd32e6c"}, - {file = "aiohttp-3.8.3-cp38-cp38-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:5835f258ca9f7c455493a57ee707b76d2d9634d84d5d7f62e77be984ea80b849"}, - {file = "aiohttp-3.8.3-cp38-cp38-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:da37dcfbf4b7f45d80ee386a5f81122501ec75672f475da34784196690762f4b"}, - {file = "aiohttp-3.8.3-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:87f44875f2804bc0511a69ce44a9595d5944837a62caecc8490bbdb0e18b1342"}, - {file = "aiohttp-3.8.3-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:527b3b87b24844ea7865284aabfab08eb0faf599b385b03c2aa91fc6edd6e4b6"}, - {file = "aiohttp-3.8.3-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:d5ba88df9aa5e2f806650fcbeedbe4f6e8736e92fc0e73b0400538fd25a4dd96"}, - {file = "aiohttp-3.8.3-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:e7b8813be97cab8cb52b1375f41f8e6804f6507fe4660152e8ca5c48f0436017"}, - {file = "aiohttp-3.8.3-cp38-cp38-musllinux_1_1_ppc64le.whl", hash = "sha256:2dea10edfa1a54098703cb7acaa665c07b4e7568472a47f4e64e6319d3821ccf"}, - {file = "aiohttp-3.8.3-cp38-cp38-musllinux_1_1_s390x.whl", hash = "sha256:713d22cd9643ba9025d33c4af43943c7a1eb8547729228de18d3e02e278472b6"}, - {file = "aiohttp-3.8.3-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:2d252771fc85e0cf8da0b823157962d70639e63cb9b578b1dec9868dd1f4f937"}, - {file = "aiohttp-3.8.3-cp38-cp38-win32.whl", hash = "sha256:66bd5f950344fb2b3dbdd421aaa4e84f4411a1a13fca3aeb2bcbe667f80c9f76"}, - {file = "aiohttp-3.8.3-cp38-cp38-win_amd64.whl", hash = "sha256:84b14f36e85295fe69c6b9789b51a0903b774046d5f7df538176516c3e422446"}, - {file = "aiohttp-3.8.3-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:16c121ba0b1ec2b44b73e3a8a171c4f999b33929cd2397124a8c7fcfc8cd9e06"}, - {file = "aiohttp-3.8.3-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:8d6aaa4e7155afaf994d7924eb290abbe81a6905b303d8cb61310a2aba1c68ba"}, - {file = "aiohttp-3.8.3-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:43046a319664a04b146f81b40e1545d4c8ac7b7dd04c47e40bf09f65f2437346"}, - {file = "aiohttp-3.8.3-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:599418aaaf88a6d02a8c515e656f6faf3d10618d3dd95866eb4436520096c84b"}, - {file = "aiohttp-3.8.3-cp39-cp39-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:92a2964319d359f494f16011e23434f6f8ef0434acd3cf154a6b7bec511e2fb7"}, - {file = "aiohttp-3.8.3-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:73a4131962e6d91109bca6536416aa067cf6c4efb871975df734f8d2fd821b37"}, - {file = "aiohttp-3.8.3-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:598adde339d2cf7d67beaccda3f2ce7c57b3b412702f29c946708f69cf8222aa"}, - {file = "aiohttp-3.8.3-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:75880ed07be39beff1881d81e4a907cafb802f306efd6d2d15f2b3c69935f6fb"}, - {file = "aiohttp-3.8.3-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:a0239da9fbafd9ff82fd67c16704a7d1bccf0d107a300e790587ad05547681c8"}, - {file = "aiohttp-3.8.3-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:4e3a23ec214e95c9fe85a58470b660efe6534b83e6cbe38b3ed52b053d7cb6ad"}, - {file = "aiohttp-3.8.3-cp39-cp39-musllinux_1_1_ppc64le.whl", hash = "sha256:47841407cc89a4b80b0c52276f3cc8138bbbfba4b179ee3acbd7d77ae33f7ac4"}, - {file = "aiohttp-3.8.3-cp39-cp39-musllinux_1_1_s390x.whl", hash = "sha256:54d107c89a3ebcd13228278d68f1436d3f33f2dd2af5415e3feaeb1156e1a62c"}, - {file = "aiohttp-3.8.3-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:c37c5cce780349d4d51739ae682dec63573847a2a8dcb44381b174c3d9c8d403"}, - {file = "aiohttp-3.8.3-cp39-cp39-win32.whl", hash = "sha256:f178d2aadf0166be4df834c4953da2d7eef24719e8aec9a65289483eeea9d618"}, - {file = "aiohttp-3.8.3-cp39-cp39-win_amd64.whl", hash = "sha256:88e5be56c231981428f4f506c68b6a46fa25c4123a2e86d156c58a8369d31ab7"}, - {file = "aiohttp-3.8.3.tar.gz", hash = "sha256:3828fb41b7203176b82fe5d699e0d845435f2374750a44b480ea6b930f6be269"}, -] -aioitertools = [ - {file = "aioitertools-0.11.0-py3-none-any.whl", hash = "sha256:04b95e3dab25b449def24d7df809411c10e62aab0cbe31a50ca4e68748c43394"}, - {file = "aioitertools-0.11.0.tar.gz", hash = "sha256:42c68b8dd3a69c2bf7f2233bf7df4bb58b557bca5252ac02ed5187bbc67d6831"}, -] -aiosignal = [ - {file = "aiosignal-1.3.1-py3-none-any.whl", hash = "sha256:f8376fb07dd1e86a584e4fcdec80b36b7f81aac666ebc724e2c090300dd83b17"}, - {file = "aiosignal-1.3.1.tar.gz", hash = "sha256:54cd96e15e1649b75d6c87526a6ff0b6c1b0dd3459f43d9ca11d48c339b68cfc"}, -] -async-timeout = [] -attrs = [] -boto3 = [ - {file = "boto3-1.24.59-py3-none-any.whl", hash = "sha256:34ab44146a2c4e7f4e72737f4b27e6eb5e0a7855c2f4599e3d9199b6a0a2d575"}, - {file = "boto3-1.24.59.tar.gz", hash = "sha256:a50b4323f9579cfe22fcf5531fbd40b567d4d74c1adce06aeb5c95fce2a6fb40"}, -] -botocore = [ - {file = "botocore-1.27.59-py3-none-any.whl", hash = "sha256:69d756791fc024bda54f6c53f71ae34e695ee41bbbc1743d9179c4837a4929da"}, - {file = "botocore-1.27.59.tar.gz", hash = "sha256:eda4aed6ee719a745d1288eaf1beb12f6f6448ad1fa12f159405db14ba9c92cf"}, -] -build = [ - {file = "build-0.9.0-py3-none-any.whl", hash = "sha256:38a7a2b7a0bdc61a42a0a67509d88c71ecfc37b393baba770fae34e20929ff69"}, - {file = "build-0.9.0.tar.gz", hash = "sha256:1a07724e891cbd898923145eb7752ee7653674c511378eb9c7691aab1612bc3c"}, -] -certifi = [ - {file = "certifi-2022.12.7-py3-none-any.whl", hash = "sha256:4ad3232f5e926d6718ec31cfc1fcadfde020920e278684144551c91769c7bc18"}, - {file = "certifi-2022.12.7.tar.gz", hash = "sha256:35824b4c3a97115964b408844d64aa14db1cc518f6562e8d7261699d1350a9e3"}, -] -cffi = [ - {file = "cffi-1.15.1-cp27-cp27m-macosx_10_9_x86_64.whl", hash = "sha256:a66d3508133af6e8548451b25058d5812812ec3798c886bf38ed24a98216fab2"}, - {file = "cffi-1.15.1-cp27-cp27m-manylinux1_i686.whl", hash = "sha256:470c103ae716238bbe698d67ad020e1db9d9dba34fa5a899b5e21577e6d52ed2"}, - {file = "cffi-1.15.1-cp27-cp27m-manylinux1_x86_64.whl", hash = "sha256:9ad5db27f9cabae298d151c85cf2bad1d359a1b9c686a275df03385758e2f914"}, - {file = "cffi-1.15.1-cp27-cp27m-win32.whl", hash = "sha256:b3bbeb01c2b273cca1e1e0c5df57f12dce9a4dd331b4fa1635b8bec26350bde3"}, - {file = "cffi-1.15.1-cp27-cp27m-win_amd64.whl", hash = "sha256:e00b098126fd45523dd056d2efba6c5a63b71ffe9f2bbe1a4fe1716e1d0c331e"}, - {file = "cffi-1.15.1-cp27-cp27mu-manylinux1_i686.whl", hash = "sha256:d61f4695e6c866a23a21acab0509af1cdfd2c013cf256bbf5b6b5e2695827162"}, - {file = "cffi-1.15.1-cp27-cp27mu-manylinux1_x86_64.whl", hash = "sha256:ed9cb427ba5504c1dc15ede7d516b84757c3e3d7868ccc85121d9310d27eed0b"}, - {file = "cffi-1.15.1-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:39d39875251ca8f612b6f33e6b1195af86d1b3e60086068be9cc053aa4376e21"}, - {file = "cffi-1.15.1-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:285d29981935eb726a4399badae8f0ffdff4f5050eaa6d0cfc3f64b857b77185"}, - {file = "cffi-1.15.1-cp310-cp310-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:3eb6971dcff08619f8d91607cfc726518b6fa2a9eba42856be181c6d0d9515fd"}, - {file = "cffi-1.15.1-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:21157295583fe8943475029ed5abdcf71eb3911894724e360acff1d61c1d54bc"}, - {file = "cffi-1.15.1-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:5635bd9cb9731e6d4a1132a498dd34f764034a8ce60cef4f5319c0541159392f"}, - {file = "cffi-1.15.1-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:2012c72d854c2d03e45d06ae57f40d78e5770d252f195b93f581acf3ba44496e"}, - {file = "cffi-1.15.1-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:dd86c085fae2efd48ac91dd7ccffcfc0571387fe1193d33b6394db7ef31fe2a4"}, - {file = "cffi-1.15.1-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:fa6693661a4c91757f4412306191b6dc88c1703f780c8234035eac011922bc01"}, - {file = "cffi-1.15.1-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:59c0b02d0a6c384d453fece7566d1c7e6b7bae4fc5874ef2ef46d56776d61c9e"}, - {file = "cffi-1.15.1-cp310-cp310-win32.whl", hash = "sha256:cba9d6b9a7d64d4bd46167096fc9d2f835e25d7e4c121fb2ddfc6528fb0413b2"}, - {file = "cffi-1.15.1-cp310-cp310-win_amd64.whl", hash = "sha256:ce4bcc037df4fc5e3d184794f27bdaab018943698f4ca31630bc7f84a7b69c6d"}, - {file = "cffi-1.15.1-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:3d08afd128ddaa624a48cf2b859afef385b720bb4b43df214f85616922e6a5ac"}, - {file = "cffi-1.15.1-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:3799aecf2e17cf585d977b780ce79ff0dc9b78d799fc694221ce814c2c19db83"}, - {file = "cffi-1.15.1-cp311-cp311-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:a591fe9e525846e4d154205572a029f653ada1a78b93697f3b5a8f1f2bc055b9"}, - {file = "cffi-1.15.1-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:3548db281cd7d2561c9ad9984681c95f7b0e38881201e157833a2342c30d5e8c"}, - {file = "cffi-1.15.1-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:91fc98adde3d7881af9b59ed0294046f3806221863722ba7d8d120c575314325"}, - {file = "cffi-1.15.1-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:94411f22c3985acaec6f83c6df553f2dbe17b698cc7f8ae751ff2237d96b9e3c"}, - {file = "cffi-1.15.1-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:03425bdae262c76aad70202debd780501fabeaca237cdfddc008987c0e0f59ef"}, - {file = "cffi-1.15.1-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:cc4d65aeeaa04136a12677d3dd0b1c0c94dc43abac5860ab33cceb42b801c1e8"}, - {file = "cffi-1.15.1-cp311-cp311-win32.whl", hash = "sha256:a0f100c8912c114ff53e1202d0078b425bee3649ae34d7b070e9697f93c5d52d"}, - {file = "cffi-1.15.1-cp311-cp311-win_amd64.whl", hash = "sha256:04ed324bda3cda42b9b695d51bb7d54b680b9719cfab04227cdd1e04e5de3104"}, - {file = "cffi-1.15.1-cp36-cp36m-macosx_10_9_x86_64.whl", hash = "sha256:50a74364d85fd319352182ef59c5c790484a336f6db772c1a9231f1c3ed0cbd7"}, - {file = "cffi-1.15.1-cp36-cp36m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:e263d77ee3dd201c3a142934a086a4450861778baaeeb45db4591ef65550b0a6"}, - {file = "cffi-1.15.1-cp36-cp36m-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:cec7d9412a9102bdc577382c3929b337320c4c4c4849f2c5cdd14d7368c5562d"}, - {file = "cffi-1.15.1-cp36-cp36m-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:4289fc34b2f5316fbb762d75362931e351941fa95fa18789191b33fc4cf9504a"}, - {file = "cffi-1.15.1-cp36-cp36m-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:173379135477dc8cac4bc58f45db08ab45d228b3363adb7af79436135d028405"}, - {file = "cffi-1.15.1-cp36-cp36m-manylinux_2_5_x86_64.manylinux1_x86_64.whl", hash = "sha256:6975a3fac6bc83c4a65c9f9fcab9e47019a11d3d2cf7f3c0d03431bf145a941e"}, - {file = "cffi-1.15.1-cp36-cp36m-win32.whl", hash = "sha256:2470043b93ff09bf8fb1d46d1cb756ce6132c54826661a32d4e4d132e1977adf"}, - {file = "cffi-1.15.1-cp36-cp36m-win_amd64.whl", hash = "sha256:30d78fbc8ebf9c92c9b7823ee18eb92f2e6ef79b45ac84db507f52fbe3ec4497"}, - {file = "cffi-1.15.1-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:198caafb44239b60e252492445da556afafc7d1e3ab7a1fb3f0584ef6d742375"}, - {file = "cffi-1.15.1-cp37-cp37m-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:5ef34d190326c3b1f822a5b7a45f6c4535e2f47ed06fec77d3d799c450b2651e"}, - {file = "cffi-1.15.1-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:8102eaf27e1e448db915d08afa8b41d6c7ca7a04b7d73af6514df10a3e74bd82"}, - {file = "cffi-1.15.1-cp37-cp37m-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:5df2768244d19ab7f60546d0c7c63ce1581f7af8b5de3eb3004b9b6fc8a9f84b"}, - {file = "cffi-1.15.1-cp37-cp37m-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:a8c4917bd7ad33e8eb21e9a5bbba979b49d9a97acb3a803092cbc1133e20343c"}, - {file = "cffi-1.15.1-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:0e2642fe3142e4cc4af0799748233ad6da94c62a8bec3a6648bf8ee68b1c7426"}, - {file = "cffi-1.15.1-cp37-cp37m-win32.whl", hash = "sha256:e229a521186c75c8ad9490854fd8bbdd9a0c9aa3a524326b55be83b54d4e0ad9"}, - {file = "cffi-1.15.1-cp37-cp37m-win_amd64.whl", hash = "sha256:a0b71b1b8fbf2b96e41c4d990244165e2c9be83d54962a9a1d118fd8657d2045"}, - {file = "cffi-1.15.1-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:320dab6e7cb2eacdf0e658569d2575c4dad258c0fcc794f46215e1e39f90f2c3"}, - {file = "cffi-1.15.1-cp38-cp38-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:1e74c6b51a9ed6589199c787bf5f9875612ca4a8a0785fb2d4a84429badaf22a"}, - {file = "cffi-1.15.1-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a5c84c68147988265e60416b57fc83425a78058853509c1b0629c180094904a5"}, - {file = "cffi-1.15.1-cp38-cp38-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:3b926aa83d1edb5aa5b427b4053dc420ec295a08e40911296b9eb1b6170f6cca"}, - {file = "cffi-1.15.1-cp38-cp38-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:87c450779d0914f2861b8526e035c5e6da0a3199d8f1add1a665e1cbc6fc6d02"}, - {file = "cffi-1.15.1-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:4f2c9f67e9821cad2e5f480bc8d83b8742896f1242dba247911072d4fa94c192"}, - {file = "cffi-1.15.1-cp38-cp38-win32.whl", hash = "sha256:8b7ee99e510d7b66cdb6c593f21c043c248537a32e0bedf02e01e9553a172314"}, - {file = "cffi-1.15.1-cp38-cp38-win_amd64.whl", hash = "sha256:00a9ed42e88df81ffae7a8ab6d9356b371399b91dbdf0c3cb1e84c03a13aceb5"}, - {file = "cffi-1.15.1-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:54a2db7b78338edd780e7ef7f9f6c442500fb0d41a5a4ea24fff1c929d5af585"}, - {file = "cffi-1.15.1-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:fcd131dd944808b5bdb38e6f5b53013c5aa4f334c5cad0c72742f6eba4b73db0"}, - {file = "cffi-1.15.1-cp39-cp39-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:7473e861101c9e72452f9bf8acb984947aa1661a7704553a9f6e4baa5ba64415"}, - {file = "cffi-1.15.1-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:6c9a799e985904922a4d207a94eae35c78ebae90e128f0c4e521ce339396be9d"}, - {file = "cffi-1.15.1-cp39-cp39-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:3bcde07039e586f91b45c88f8583ea7cf7a0770df3a1649627bf598332cb6984"}, - {file = "cffi-1.15.1-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:33ab79603146aace82c2427da5ca6e58f2b3f2fb5da893ceac0c42218a40be35"}, - {file = "cffi-1.15.1-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:5d598b938678ebf3c67377cdd45e09d431369c3b1a5b331058c338e201f12b27"}, - {file = "cffi-1.15.1-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:db0fbb9c62743ce59a9ff687eb5f4afbe77e5e8403d6697f7446e5f609976f76"}, - {file = "cffi-1.15.1-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:98d85c6a2bef81588d9227dde12db8a7f47f639f4a17c9ae08e773aa9c697bf3"}, - {file = "cffi-1.15.1-cp39-cp39-win32.whl", hash = "sha256:40f4774f5a9d4f5e344f31a32b5096977b5d48560c5592e2f3d2c4374bd543ee"}, - {file = "cffi-1.15.1-cp39-cp39-win_amd64.whl", hash = "sha256:70df4e3b545a17496c9b3f41f5115e69a4f2e77e94e1d2a8e1070bc0c38c8a3c"}, - {file = "cffi-1.15.1.tar.gz", hash = "sha256:d400bfb9a37b1351253cb402671cea7e89bdecc294e8016a707f6d1d8ac934f9"}, -] -cfgv = [ - {file = "cfgv-3.3.1-py2.py3-none-any.whl", hash = "sha256:c6a0883f3917a037485059700b9e75da2464e6c27051014ad85ba6aaa5884426"}, - {file = "cfgv-3.3.1.tar.gz", hash = "sha256:f5a830efb9ce7a445376bb66ec94c638a9787422f96264c98edc6bdeed8ab736"}, -] -charset-normalizer = [] -click = [] -colorama = [ - {file = "colorama-0.4.6-py2.py3-none-any.whl", hash = "sha256:4f1d9991f5acc0ca119f9d443620b77f9d6b33703e51011c16baf57afb285fc6"}, - {file = "colorama-0.4.6.tar.gz", hash = "sha256:08695f5cb7ed6e0531a20572697297273c47b8cae5a63ffc6d6ed5c201be6e44"}, -] -commonmark = [] -coverage = [ +files = [ {file = "coverage-6.5.0-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:ef8674b0ee8cc11e2d574e3e2998aea5df5ab242e012286824ea3c6970580e53"}, {file = "coverage-6.5.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:784f53ebc9f3fd0e2a3f6a78b2be1bd1f5575d7863e10c6e12504f240fd06660"}, {file = "coverage-6.5.0-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:b4a5be1748d538a710f87542f22c2cad22f80545a847ad91ce45e77417293eb4"}, @@ -1208,7 +586,21 @@ coverage = [ {file = "coverage-6.5.0-pp36.pp37.pp38-none-any.whl", hash = "sha256:1431986dac3923c5945271f169f59c45b8802a114c8f548d611f2015133df77a"}, {file = "coverage-6.5.0.tar.gz", hash = "sha256:f642e90754ee3e06b0e7e51bce3379590e76b7f76b708e1a71ff043f87025c84"}, ] -cryptography = [ + +[package.dependencies] +tomli = {version = "*", optional = true, markers = "python_full_version <= \"3.11.0a6\" and extra == \"toml\""} + +[package.extras] +toml = ["tomli"] + +[[package]] +name = "cryptography" +version = "38.0.4" +description = "cryptography is a package which provides cryptographic recipes and primitives to Python developers." +category = "main" +optional = false +python-versions = ">=3.6" +files = [ {file = "cryptography-38.0.4-cp36-abi3-macosx_10_10_universal2.whl", hash = "sha256:2fa36a7b2cc0998a3a4d5af26ccb6273f3df133d61da2ba13b3286261e7efb70"}, {file = "cryptography-38.0.4-cp36-abi3-macosx_10_10_x86_64.whl", hash = "sha256:1f13ddda26a04c06eb57119caf27a524ccae20533729f4b1e4a69b54e07035eb"}, {file = "cryptography-38.0.4-cp36-abi3-manylinux_2_17_aarch64.manylinux2014_aarch64.manylinux_2_24_aarch64.whl", hash = "sha256:2ec2a8714dd005949d4019195d72abed84198d877112abb5a27740e217e0ea8d"}, @@ -1236,15 +628,50 @@ cryptography = [ {file = "cryptography-38.0.4-pp39-pypy39_pp73-win_amd64.whl", hash = "sha256:80ca53981ceeb3241998443c4964a387771588c4e4a5d92735a493af868294f9"}, {file = "cryptography-38.0.4.tar.gz", hash = "sha256:175c1a818b87c9ac80bb7377f5520b7f31b3ef2a0004e2420319beadedb67290"}, ] -distlib = [ + +[package.dependencies] +cffi = ">=1.12" + +[package.extras] +docs = ["sphinx (>=1.6.5,!=1.8.0,!=3.1.0,!=3.1.1)", "sphinx-rtd-theme"] +docstest = ["pyenchant (>=1.6.11)", "sphinxcontrib-spelling (>=4.0.1)", "twine (>=1.12.0)"] +pep8test = ["black", "flake8", "flake8-import-order", "pep8-naming"] +sdist = ["setuptools-rust (>=0.11.4)"] +ssh = ["bcrypt (>=3.1.5)"] +test = ["hypothesis (>=1.11.4,!=3.79.2)", "iso8601", "pretend", "pytest (>=6.2.0)", "pytest-benchmark", "pytest-cov", "pytest-subtests", "pytest-xdist", "pytz"] + +[[package]] +name = "distlib" +version = "0.3.6" +description = "Distribution utilities" +category = "dev" +optional = false +python-versions = "*" +files = [ {file = "distlib-0.3.6-py2.py3-none-any.whl", hash = "sha256:f35c4b692542ca110de7ef0bea44d73981caeb34ca0b9b6b2e6d7790dda8f80e"}, {file = "distlib-0.3.6.tar.gz", hash = "sha256:14bad2d9b04d3a36127ac97f30b12a19268f211063d8f8ee4f47108896e11b46"}, ] -docutils = [ + +[[package]] +name = "docutils" +version = "0.19" +description = "Docutils -- Python Documentation Utilities" +category = "dev" +optional = false +python-versions = ">=3.7" +files = [ {file = "docutils-0.19-py3-none-any.whl", hash = "sha256:5e1de4d849fee02c63b040a4a3fd567f4ab104defd8a5511fbbc24a8a017efbc"}, {file = "docutils-0.19.tar.gz", hash = "sha256:33995a6753c30b7f577febfc2c50411fec6aac7f7ffeb7c4cfe5991072dcf9e6"}, ] -duckdb = [ + +[[package]] +name = "duckdb" +version = "0.6.1" +description = "DuckDB embedded database" +category = "main" +optional = true +python-versions = "*" +files = [ {file = "duckdb-0.6.1-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:e566514f9327f89264e98ac14ee7a84fbd9857328028258422c3e8375ee19d25"}, {file = "duckdb-0.6.1-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:b31c2883de5b19591a2852165e6b3f9821f77af649835f27bc146b26e4aa30cb"}, {file = "duckdb-0.6.1-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:998165b2fb1f1d2b0ad742096015ea70878f7d40304643c7424c3ed3ddf07bfc"}, @@ -1293,11 +720,33 @@ duckdb = [ {file = "duckdb-0.6.1-cp39-cp39-win_amd64.whl", hash = "sha256:00b7be8f67ec1a8edaa8844f521267baa1a795f4c482bfad56c72c26e1862ab2"}, {file = "duckdb-0.6.1.tar.gz", hash = "sha256:6d26e9f1afcb924a6057785e506810d48332d4764ddc4a5b414d0f2bf0cacfb4"}, ] -exceptiongroup = [ + +[package.dependencies] +numpy = ">=1.14" + +[[package]] +name = "exceptiongroup" +version = "1.0.4" +description = "Backport of PEP 654 (exception groups)" +category = "dev" +optional = false +python-versions = ">=3.7" +files = [ {file = "exceptiongroup-1.0.4-py3-none-any.whl", hash = "sha256:542adf9dea4055530d6e1279602fa5cb11dab2395fa650b8674eaec35fc4a828"}, {file = "exceptiongroup-1.0.4.tar.gz", hash = "sha256:bd14967b79cd9bdb54d97323216f8fdf533e278df937aa2a90089e7d6e06e5ec"}, ] -fastavro = [ + +[package.extras] +test = ["pytest (>=6)"] + +[[package]] +name = "fastavro" +version = "1.7.0" +description = "Fast read/write of AVRO files" +category = "dev" +optional = false +python-versions = ">=3.7" +files = [ {file = "fastavro-1.7.0-cp310-cp310-macosx_10_14_x86_64.whl", hash = "sha256:ab3387a06e272980fa034f5c62f7063977b77df6416d3d30a4d3b49cc8827566"}, {file = "fastavro-1.7.0-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:216132bc54da19e97e1531dd69c86282408d4c797749d83b01b3a00862a180de"}, {file = "fastavro-1.7.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:c20cf6cd8098bb93c2cffd53d03ccea1dcf9ec594a5c83963acf29a2882f8693"}, @@ -1320,11 +769,37 @@ fastavro = [ {file = "fastavro-1.7.0-cp39-cp39-win_amd64.whl", hash = "sha256:492e8902063aa1c73170e11c18495fcaa86b71eae3513ef83ba438ca02b16b34"}, {file = "fastavro-1.7.0.tar.gz", hash = "sha256:4b1205f46489b4032d3155c1ab44d9824be0c7454df98d3a5bd22b78b98f23c8"}, ] -filelock = [ + +[package.extras] +codecs = ["lz4", "python-snappy", "zstandard"] +lz4 = ["lz4"] +snappy = ["python-snappy"] +zstandard = ["zstandard"] + +[[package]] +name = "filelock" +version = "3.8.2" +description = "A platform independent file lock." +category = "dev" +optional = false +python-versions = ">=3.7" +files = [ {file = "filelock-3.8.2-py3-none-any.whl", hash = "sha256:8df285554452285f79c035efb0c861eb33a4bcfa5b7a137016e32e6a90f9792c"}, {file = "filelock-3.8.2.tar.gz", hash = "sha256:7565f628ea56bfcd8e54e42bdc55da899c85c1abfe1b5bcfd147e9188cebb3b2"}, ] -frozenlist = [ + +[package.extras] +docs = ["furo (>=2022.9.29)", "sphinx (>=5.3)", "sphinx-autodoc-typehints (>=1.19.5)"] +testing = ["covdefaults (>=2.2.2)", "coverage (>=6.5)", "pytest (>=7.2)", "pytest-cov (>=4)", "pytest-timeout (>=2.1)"] + +[[package]] +name = "frozenlist" +version = "1.3.3" +description = "A list-like structure which implements collections.abc.MutableSequence" +category = "main" +optional = true +python-versions = ">=3.7" +files = [ {file = "frozenlist-1.3.3-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:ff8bf625fe85e119553b5383ba0fb6aa3d0ec2ae980295aaefa552374926b3f4"}, {file = "frozenlist-1.3.3-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:dfbac4c2dfcc082fcf8d942d1e49b6aa0766c19d3358bd86e2000bf0fa4a9cf0"}, {file = "frozenlist-1.3.3-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:b1c63e8d377d039ac769cd0926558bb7068a1f7abb0f003e3717ee003ad85530"}, @@ -1400,32 +875,154 @@ frozenlist = [ {file = "frozenlist-1.3.3-cp39-cp39-win_amd64.whl", hash = "sha256:cfe33efc9cb900a4c46f91a5ceba26d6df370ffddd9ca386eb1d4f0ad97b9ea9"}, {file = "frozenlist-1.3.3.tar.gz", hash = "sha256:58bcc55721e8a90b88332d6cd441261ebb22342e238296bb330968952fbb3a6a"}, ] -fsspec = [ + +[[package]] +name = "fsspec" +version = "2022.10.0" +description = "File-system specification" +category = "main" +optional = false +python-versions = ">=3.7" +files = [ {file = "fsspec-2022.10.0-py3-none-any.whl", hash = "sha256:6b7c6ab3b476cdf17efcfeccde7fca28ef5a48f73a71010aaceec5fc15bf9ebf"}, {file = "fsspec-2022.10.0.tar.gz", hash = "sha256:cb6092474e90487a51de768170f3afa50ca8982c26150a59072b16433879ff1d"}, ] -identify = [ + +[package.extras] +abfs = ["adlfs"] +adl = ["adlfs"] +arrow = ["pyarrow (>=1)"] +dask = ["dask", "distributed"] +dropbox = ["dropbox", "dropboxdrivefs", "requests"] +entrypoints = ["importlib-metadata"] +fuse = ["fusepy"] +gcs = ["gcsfs"] +git = ["pygit2"] +github = ["requests"] +gs = ["gcsfs"] +gui = ["panel"] +hdfs = ["pyarrow (>=1)"] +http = ["aiohttp (!=4.0.0a0,!=4.0.0a1)", "requests"] +libarchive = ["libarchive-c"] +oci = ["ocifs"] +s3 = ["s3fs"] +sftp = ["paramiko"] +smb = ["smbprotocol"] +ssh = ["paramiko"] +tqdm = ["tqdm"] + +[[package]] +name = "identify" +version = "2.5.10" +description = "File identification library for Python" +category = "dev" +optional = false +python-versions = ">=3.7" +files = [ {file = "identify-2.5.10-py2.py3-none-any.whl", hash = "sha256:fb7c2feaeca6976a3ffa31ec3236a6911fbc51aec9acc111de2aed99f244ade2"}, {file = "identify-2.5.10.tar.gz", hash = "sha256:dce9e31fee7dbc45fea36a9e855c316b8fbf807e65a862f160840bb5a2bf5dfd"}, ] -idna = [ + +[package.extras] +license = ["ukkonen"] + +[[package]] +name = "idna" +version = "3.4" +description = "Internationalized Domain Names in Applications (IDNA)" +category = "main" +optional = false +python-versions = ">=3.5" +files = [ {file = "idna-3.4-py3-none-any.whl", hash = "sha256:90b77e79eaa3eba6de819a0c442c0b4ceefc341a7a2ab77d7562bf49f425c5c2"}, {file = "idna-3.4.tar.gz", hash = "sha256:814f528e8dead7d329833b91c5faa87d60bf71824cd12a7530b5526063d02cb4"}, ] -importlib-metadata = [ + +[[package]] +name = "importlib-metadata" +version = "5.1.0" +description = "Read metadata from Python packages" +category = "dev" +optional = false +python-versions = ">=3.7" +files = [ {file = "importlib_metadata-5.1.0-py3-none-any.whl", hash = "sha256:d84d17e21670ec07990e1044a99efe8d615d860fd176fc29ef5c306068fda313"}, {file = "importlib_metadata-5.1.0.tar.gz", hash = "sha256:d5059f9f1e8e41f80e9c56c2ee58811450c31984dfa625329ffd7c0dad88a73b"}, ] -iniconfig = [ + +[package.dependencies] +zipp = ">=0.5" + +[package.extras] +docs = ["furo", "jaraco.packaging (>=9)", "jaraco.tidelift (>=1.4)", "rst.linker (>=1.9)", "sphinx (>=3.5)"] +perf = ["ipython"] +testing = ["flake8 (<5)", "flufl.flake8", "importlib-resources (>=1.3)", "packaging", "pyfakefs", "pytest (>=6)", "pytest-black (>=0.3.7)", "pytest-checkdocs (>=2.4)", "pytest-cov", "pytest-enabler (>=1.3)", "pytest-flake8", "pytest-mypy (>=0.9.1)", "pytest-perf (>=0.9.2)"] + +[[package]] +name = "iniconfig" +version = "1.1.1" +description = "iniconfig: brain-dead simple config-ini parsing" +category = "dev" +optional = false +python-versions = "*" +files = [ {file = "iniconfig-1.1.1-py2.py3-none-any.whl", hash = "sha256:011e24c64b7f47f6ebd835bb12a743f2fbe9a26d4cecaa7f53bc4f35ee9da8b3"}, {file = "iniconfig-1.1.1.tar.gz", hash = "sha256:bc3af051d7d14b2ee5ef9969666def0cd1a000e121eaea580d4a313df4b37f32"}, ] -jinja2 = [ + +[[package]] +name = "isodate" +version = "0.6.1" +description = "An ISO 8601 date/time/duration parser and formatter" +category = "main" +optional = true +python-versions = "*" +files = [ + {file = "isodate-0.6.1-py2.py3-none-any.whl", hash = "sha256:0751eece944162659049d35f4f549ed815792b38793f07cf73381c1c87cbed96"}, + {file = "isodate-0.6.1.tar.gz", hash = "sha256:48c5881de7e8b0a0d648cb024c8062dc84e7b840ed81e864c7614fd3c127bde9"}, +] + +[package.dependencies] +six = "*" + +[[package]] +name = "jinja2" +version = "3.1.2" +description = "A very fast and expressive template engine." +category = "dev" +optional = false +python-versions = ">=3.7" +files = [ {file = "Jinja2-3.1.2-py3-none-any.whl", hash = "sha256:6088930bfe239f0e6710546ab9c19c9ef35e29792895fed6e6e31a023a182a61"}, {file = "Jinja2-3.1.2.tar.gz", hash = "sha256:31351a702a408a9e7595a8fc6150fc3f43bb6bf7e319770cbc0db9df9437e852"}, ] -jmespath = [] -markupsafe = [ + +[package.dependencies] +MarkupSafe = ">=2.0" + +[package.extras] +i18n = ["Babel (>=2.7)"] + +[[package]] +name = "jmespath" +version = "1.0.1" +description = "JSON Matching Expressions" +category = "main" +optional = false +python-versions = ">=3.7" +files = [ + {file = "jmespath-1.0.1-py3-none-any.whl", hash = "sha256:02e2e4cc71b5bcab88332eebf907519190dd9e6e82107fa7f83b1003a6252980"}, + {file = "jmespath-1.0.1.tar.gz", hash = "sha256:90261b206d6defd58fdd5e85f478bf633a2901798906be2ad389150c5c60edbe"}, +] + +[[package]] +name = "markupsafe" +version = "2.1.1" +description = "Safely add untrusted strings to HTML/XML markup." +category = "dev" +optional = false +python-versions = ">=3.7" +files = [ {file = "MarkupSafe-2.1.1-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:86b1f75c4e7c2ac2ccdaec2b9022845dbb81880ca318bb7a0a01fbf7813e3812"}, {file = "MarkupSafe-2.1.1-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:f121a1420d4e173a5d96e47e9a0c0dcff965afdf1626d28de1460815f7c4ee7a"}, {file = "MarkupSafe-2.1.1-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a49907dd8420c5685cfa064a1335b6754b74541bbb3706c259c02ed65b644b3e"}, @@ -1467,7 +1064,15 @@ markupsafe = [ {file = "MarkupSafe-2.1.1-cp39-cp39-win_amd64.whl", hash = "sha256:46d00d6cfecdde84d40e572d63735ef81423ad31184100411e6e3388d405e247"}, {file = "MarkupSafe-2.1.1.tar.gz", hash = "sha256:7f91197cc9e48f989d12e4e6fbc46495c446636dfc81b9ccf50bb0ec74b91d4b"}, ] -mmhash3 = [ + +[[package]] +name = "mmhash3" +version = "3.0.1" +description = "Python wrapper for MurmurHash (MurmurHash3), a set of fast and robust hash functions." +category = "main" +optional = false +python-versions = "*" +files = [ {file = "mmhash3-3.0.1-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:47deea30cd8d3d5cd52dc740902a4c70383bfe8248eac29d0877fe63e03c2713"}, {file = "mmhash3-3.0.1-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:ecdaf4d1de617818bf05cd526ca558db6010beeba7ea9e19f695f2bdcac0e0a4"}, {file = "mmhash3-3.0.1-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:4675585617584e9e1aafa3a90ac0ac9a437257c507748d97de8b21977e9d6745"}, @@ -1503,11 +1108,125 @@ mmhash3 = [ {file = "mmhash3-3.0.1-cp39-cp39-win_amd64.whl", hash = "sha256:0baeaa20cac5f75ed14f28056826bd9d9c8b2354b382073f3fd5190708992a0d"}, {file = "mmhash3-3.0.1.tar.gz", hash = "sha256:a00d68f4a1cc434b9501513c8a29e18ed1ddad383677d72b41d71d0d862348af"}, ] -moto = [ + +[[package]] +name = "moto" +version = "4.0.11" +description = "A library that allows your python tests to easily mock out the boto library" +category = "dev" +optional = false +python-versions = ">=3.6" +files = [ {file = "moto-4.0.11-py3-none-any.whl", hash = "sha256:704d6d38a4e6fe49e1fe9c6b4127ca46c66aac00368149bc1f1d70a0ceff8846"}, {file = "moto-4.0.11.tar.gz", hash = "sha256:a6388de4a746e0b509286e1d7e70f86900b4f69ec65f6c92c47e570f95d05b14"}, ] -multidict = [ + +[package.dependencies] +boto3 = ">=1.9.201" +botocore = ">=1.12.201" +cryptography = ">=3.3.1" +Jinja2 = ">=2.10.1" +MarkupSafe = "!=2.0.0a1" +python-dateutil = ">=2.1,<3.0.0" +pytz = "*" +requests = ">=2.5" +responses = ">=0.13.0" +werkzeug = ">=0.5,<2.2.0 || >2.2.0,<2.2.1 || >2.2.1" +xmltodict = "*" + +[package.extras] +all = ["PyYAML (>=5.1)", "aws-xray-sdk (>=0.93,!=0.96)", "cfn-lint (>=0.40.0)", "docker (>=2.5.1)", "ecdsa (!=0.15)", "graphql-core", "idna (>=2.5,<4)", "jsondiff (>=1.1.2)", "openapi-spec-validator (>=0.2.8)", "pyparsing (>=3.0.7)", "python-jose[cryptography] (>=3.1.0,<4.0.0)", "setuptools", "sshpubkeys (>=3.1.0)"] +apigateway = ["PyYAML (>=5.1)", "ecdsa (!=0.15)", "openapi-spec-validator (>=0.2.8)", "python-jose[cryptography] (>=3.1.0,<4.0.0)"] +apigatewayv2 = ["PyYAML (>=5.1)"] +appsync = ["graphql-core"] +awslambda = ["docker (>=2.5.1)"] +batch = ["docker (>=2.5.1)"] +cloudformation = ["PyYAML (>=5.1)", "aws-xray-sdk (>=0.93,!=0.96)", "cfn-lint (>=0.40.0)", "docker (>=2.5.1)", "ecdsa (!=0.15)", "graphql-core", "idna (>=2.5,<4)", "jsondiff (>=1.1.2)", "openapi-spec-validator (>=0.2.8)", "pyparsing (>=3.0.7)", "python-jose[cryptography] (>=3.1.0,<4.0.0)", "setuptools", "sshpubkeys (>=3.1.0)"] +cognitoidp = ["ecdsa (!=0.15)", "python-jose[cryptography] (>=3.1.0,<4.0.0)"] +ds = ["sshpubkeys (>=3.1.0)"] +dynamodb = ["docker (>=2.5.1)"] +dynamodb2 = ["docker (>=2.5.1)"] +dynamodbstreams = ["docker (>=2.5.1)"] +ebs = ["sshpubkeys (>=3.1.0)"] +ec2 = ["sshpubkeys (>=3.1.0)"] +efs = ["sshpubkeys (>=3.1.0)"] +glue = ["pyparsing (>=3.0.7)"] +iotdata = ["jsondiff (>=1.1.2)"] +route53resolver = ["sshpubkeys (>=3.1.0)"] +s3 = ["PyYAML (>=5.1)"] +server = ["PyYAML (>=5.1)", "aws-xray-sdk (>=0.93,!=0.96)", "cfn-lint (>=0.40.0)", "docker (>=2.5.1)", "ecdsa (!=0.15)", "flask (!=2.2.0,!=2.2.1)", "flask-cors", "graphql-core", "idna (>=2.5,<4)", "jsondiff (>=1.1.2)", "openapi-spec-validator (>=0.2.8)", "pyparsing (>=3.0.7)", "python-jose[cryptography] (>=3.1.0,<4.0.0)", "setuptools", "sshpubkeys (>=3.1.0)"] +ssm = ["PyYAML (>=5.1)", "dataclasses"] +xray = ["aws-xray-sdk (>=0.93,!=0.96)", "setuptools"] + +[[package]] +name = "msal" +version = "1.20.0" +description = "The Microsoft Authentication Library (MSAL) for Python library enables your app to access the Microsoft Cloud by supporting authentication of users with Microsoft Azure Active Directory accounts (AAD) and Microsoft Accounts (MSA) using industry standard OAuth2 and OpenID Connect." +category = "main" +optional = true +python-versions = "*" +files = [ + {file = "msal-1.20.0-py2.py3-none-any.whl", hash = "sha256:d2f1c26368ecdc28c8657d457352faa0b81b1845a7b889d8676787721ba86792"}, + {file = "msal-1.20.0.tar.gz", hash = "sha256:78344cd4c91d6134a593b5e3e45541e666e37b747ff8a6316c3668dd1e6ab6b2"}, +] + +[package.dependencies] +cryptography = ">=0.6,<41" +PyJWT = {version = ">=1.0.0,<3", extras = ["crypto"]} +requests = ">=2.0.0,<3" + +[package.extras] +broker = ["pymsalruntime (>=0.11.2,<0.14)"] + +[[package]] +name = "msal-extensions" +version = "1.0.0" +description = "Microsoft Authentication Library extensions (MSAL EX) provides a persistence API that can save your data on disk, encrypted on Windows, macOS and Linux. Concurrent data access will be coordinated by a file lock mechanism." +category = "main" +optional = true +python-versions = "*" +files = [ + {file = "msal-extensions-1.0.0.tar.gz", hash = "sha256:c676aba56b0cce3783de1b5c5ecfe828db998167875126ca4b47dc6436451354"}, + {file = "msal_extensions-1.0.0-py2.py3-none-any.whl", hash = "sha256:91e3db9620b822d0ed2b4d1850056a0f133cba04455e62f11612e40f5502f2ee"}, +] + +[package.dependencies] +msal = ">=0.4.1,<2.0.0" +portalocker = [ + {version = ">=1.0,<3", markers = "python_version >= \"3.5\" and platform_system != \"Windows\""}, + {version = ">=1.6,<3", markers = "python_version >= \"3.5\" and platform_system == \"Windows\""}, +] + +[[package]] +name = "msrest" +version = "0.7.1" +description = "AutoRest swagger generator Python client runtime." +category = "main" +optional = true +python-versions = ">=3.6" +files = [ + {file = "msrest-0.7.1-py3-none-any.whl", hash = "sha256:21120a810e1233e5e6cc7fe40b474eeb4ec6f757a15d7cf86702c369f9567c32"}, + {file = "msrest-0.7.1.zip", hash = "sha256:6e7661f46f3afd88b75667b7187a92829924446c7ea1d169be8c4bb7eeb788b9"}, +] + +[package.dependencies] +azure-core = ">=1.24.0" +certifi = ">=2017.4.17" +isodate = ">=0.6.0" +requests = ">=2.16,<3.0" +requests-oauthlib = ">=0.5.0" + +[package.extras] +async = ["aiodns", "aiohttp (>=3.0)"] + +[[package]] +name = "multidict" +version = "6.0.3" +description = "multidict implementation" +category = "main" +optional = true +python-versions = ">=3.7" +files = [ {file = "multidict-6.0.3-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:73009ea04205966d47e16d98686ac5c438af23a1bb30b48a2c5da3423ec9ce37"}, {file = "multidict-6.0.3-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:8b92a9f3ab904397a33b193000dc4de7318ea175c4c460a1e154c415f9008e3d"}, {file = "multidict-6.0.3-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:578bfcb16f4b8675ef71b960c00f174b0426e0eeb796bab6737389d8288eb827"}, @@ -1583,11 +1302,30 @@ multidict = [ {file = "multidict-6.0.3-cp39-cp39-win_amd64.whl", hash = "sha256:5e58ec0375803526d395f6f7e730ecc45d06e15f68f7b9cdbf644a2918324e51"}, {file = "multidict-6.0.3.tar.gz", hash = "sha256:2523a29006c034687eccd3ee70093a697129a3ffe8732535d3b2df6a4ecc279d"}, ] -nodeenv = [ + +[[package]] +name = "nodeenv" +version = "1.7.0" +description = "Node.js virtual environment builder" +category = "dev" +optional = false +python-versions = ">=2.7,!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,!=3.4.*,!=3.5.*,!=3.6.*" +files = [ {file = "nodeenv-1.7.0-py2.py3-none-any.whl", hash = "sha256:27083a7b96a25f2f5e1d8cb4b6317ee8aeda3bdd121394e5ac54e498028a042e"}, {file = "nodeenv-1.7.0.tar.gz", hash = "sha256:e0e7f7dfb85fc5394c6fe1e8fa98131a2473e04311a45afb6508f7cf1836fa2b"}, ] -numpy = [ + +[package.dependencies] +setuptools = "*" + +[[package]] +name = "numpy" +version = "1.23.5" +description = "NumPy is the fundamental package for array computing with Python." +category = "main" +optional = true +python-versions = ">=3.8" +files = [ {file = "numpy-1.23.5-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:9c88793f78fca17da0145455f0d7826bcb9f37da4764af27ac945488116efe63"}, {file = "numpy-1.23.5-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:e9f4c4e51567b616be64e05d517c79a8a22f3606499941d97bb76f2ca59f982d"}, {file = "numpy-1.23.5-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:7903ba8ab592b82014713c491f6c5d3a1cde5b4a3bf116404e08f5b52f6daf43"}, @@ -1617,11 +1355,44 @@ numpy = [ {file = "numpy-1.23.5-pp38-pypy38_pp73-win_amd64.whl", hash = "sha256:01dd17cbb340bf0fc23981e52e1d18a9d4050792e8fb8363cecbf066a84b827d"}, {file = "numpy-1.23.5.tar.gz", hash = "sha256:1b1766d6f397c18153d40015ddfc79ddb715cabadc04d2d228d4e5a8bc4ded1a"}, ] -packaging = [ + +[[package]] +name = "oauthlib" +version = "3.2.2" +description = "A generic, spec-compliant, thorough implementation of the OAuth request-signing logic" +category = "main" +optional = true +python-versions = ">=3.6" +files = [ + {file = "oauthlib-3.2.2-py3-none-any.whl", hash = "sha256:8139f29aac13e25d502680e9e19963e83f16838d48a0d71c287fe40e7067fbca"}, + {file = "oauthlib-3.2.2.tar.gz", hash = "sha256:9859c40929662bec5d64f34d01c99e093149682a3f38915dc0655d5a633dd918"}, +] + +[package.extras] +rsa = ["cryptography (>=3.0.0)"] +signals = ["blinker (>=1.4.0)"] +signedtoken = ["cryptography (>=3.0.0)", "pyjwt (>=2.0.0,<3)"] + +[[package]] +name = "packaging" +version = "22.0" +description = "Core utilities for Python packages" +category = "dev" +optional = false +python-versions = ">=3.7" +files = [ {file = "packaging-22.0-py3-none-any.whl", hash = "sha256:957e2148ba0e1a3b282772e791ef1d8083648bc131c8ab0c1feba110ce1146c3"}, {file = "packaging-22.0.tar.gz", hash = "sha256:2198ec20bd4c017b8f9717e00f0c8714076fc2fd93816750ab48e2c41de2cfd3"}, ] -pandas = [ + +[[package]] +name = "pandas" +version = "1.5.2" +description = "Powerful data structures for data analysis, time series, and statistics" +category = "main" +optional = true +python-versions = ">=3.8" +files = [ {file = "pandas-1.5.2-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:e9dbacd22555c2d47f262ef96bb4e30880e5956169741400af8b306bbb24a273"}, {file = "pandas-1.5.2-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:e2b83abd292194f350bb04e188f9379d36b8dfac24dd445d5c87575f3beaf789"}, {file = "pandas-1.5.2-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:2552bffc808641c6eb471e55aa6899fa002ac94e4eebfa9ec058649122db5824"}, @@ -1650,17 +1421,114 @@ pandas = [ {file = "pandas-1.5.2-cp39-cp39-win_amd64.whl", hash = "sha256:c218796d59d5abd8780170c937b812c9637e84c32f8271bbf9845970f8c1351f"}, {file = "pandas-1.5.2.tar.gz", hash = "sha256:220b98d15cee0b2cd839a6358bd1f273d0356bf964c1a1aeb32d47db0215488b"}, ] -pep517 = [] -platformdirs = [ + +[package.dependencies] +numpy = [ + {version = ">=1.20.3", markers = "python_version < \"3.10\""}, + {version = ">=1.21.0", markers = "python_version >= \"3.10\""}, + {version = ">=1.23.2", markers = "python_version >= \"3.11\""}, +] +python-dateutil = ">=2.8.1" +pytz = ">=2020.1" + +[package.extras] +test = ["hypothesis (>=5.5.3)", "pytest (>=6.0)", "pytest-xdist (>=1.31)"] + +[[package]] +name = "pep517" +version = "0.13.0" +description = "Wrappers to build Python packages using PEP 517 hooks" +category = "dev" +optional = false +python-versions = ">=3.6" +files = [ + {file = "pep517-0.13.0-py3-none-any.whl", hash = "sha256:4ba4446d80aed5b5eac6509ade100bff3e7943a8489de249654a5ae9b33ee35b"}, + {file = "pep517-0.13.0.tar.gz", hash = "sha256:ae69927c5c172be1add9203726d4b84cf3ebad1edcd5f71fcdc746e66e829f59"}, +] + +[package.dependencies] +tomli = {version = ">=1.1.0", markers = "python_version < \"3.11\""} + +[[package]] +name = "platformdirs" +version = "2.6.0" +description = "A small Python package for determining appropriate platform-specific dirs, e.g. a \"user data dir\"." +category = "dev" +optional = false +python-versions = ">=3.7" +files = [ {file = "platformdirs-2.6.0-py3-none-any.whl", hash = "sha256:1a89a12377800c81983db6be069ec068eee989748799b946cce2a6e80dcc54ca"}, {file = "platformdirs-2.6.0.tar.gz", hash = "sha256:b46ffafa316e6b83b47489d240ce17173f123a9b9c83282141c3daf26ad9ac2e"}, ] -pluggy = [ + +[package.extras] +docs = ["furo (>=2022.9.29)", "proselint (>=0.13)", "sphinx (>=5.3)", "sphinx-autodoc-typehints (>=1.19.4)"] +test = ["appdirs (==1.4.4)", "pytest (>=7.2)", "pytest-cov (>=4)", "pytest-mock (>=3.10)"] + +[[package]] +name = "pluggy" +version = "1.0.0" +description = "plugin and hook calling mechanisms for python" +category = "dev" +optional = false +python-versions = ">=3.6" +files = [ {file = "pluggy-1.0.0-py2.py3-none-any.whl", hash = "sha256:74134bbf457f031a36d68416e1509f34bd5ccc019f0bcc952c7b909d06b37bd3"}, {file = "pluggy-1.0.0.tar.gz", hash = "sha256:4224373bacce55f955a878bf9cfa763c1e360858e330072059e10bad68531159"}, ] -pre-commit = [] -pyarrow = [ + +[package.extras] +dev = ["pre-commit", "tox"] +testing = ["pytest", "pytest-benchmark"] + +[[package]] +name = "portalocker" +version = "2.6.0" +description = "Wraps the portalocker recipe for easy usage" +category = "main" +optional = true +python-versions = ">=3.5" +files = [ + {file = "portalocker-2.6.0-py2.py3-none-any.whl", hash = "sha256:102ed1f2badd8dec9af3d732ef70e94b215b85ba45a8d7ff3c0003f19b442f4e"}, + {file = "portalocker-2.6.0.tar.gz", hash = "sha256:964f6830fb42a74b5d32bce99ed37d8308c1d7d44ddf18f3dd89f4680de97b39"}, +] + +[package.dependencies] +pywin32 = {version = ">=226", markers = "platform_system == \"Windows\""} + +[package.extras] +docs = ["sphinx (>=1.7.1)"] +redis = ["redis"] +tests = ["pytest (>=5.4.1)", "pytest-cov (>=2.8.1)", "pytest-mypy (>=0.8.0)", "pytest-timeout (>=2.1.0)", "redis", "sphinx (>=3.0.3)"] + +[[package]] +name = "pre-commit" +version = "2.20.0" +description = "A framework for managing and maintaining multi-language pre-commit hooks." +category = "dev" +optional = false +python-versions = ">=3.7" +files = [ + {file = "pre_commit-2.20.0-py2.py3-none-any.whl", hash = "sha256:51a5ba7c480ae8072ecdb6933df22d2f812dc897d5fe848778116129a681aac7"}, + {file = "pre_commit-2.20.0.tar.gz", hash = "sha256:a978dac7bc9ec0bcee55c18a277d553b0f419d259dadb4b9418ff2d00eb43959"}, +] + +[package.dependencies] +cfgv = ">=2.0.0" +identify = ">=1.0.0" +nodeenv = ">=0.11.1" +pyyaml = ">=5.1" +toml = "*" +virtualenv = ">=20.0.8" + +[[package]] +name = "pyarrow" +version = "10.0.1" +description = "Python library for Apache Arrow" +category = "main" +optional = true +python-versions = ">=3.7" +files = [ {file = "pyarrow-10.0.1-cp310-cp310-macosx_10_14_x86_64.whl", hash = "sha256:e00174764a8b4e9d8d5909b6d19ee0c217a6cf0232c5682e31fdfbd5a9f0ae52"}, {file = "pyarrow-10.0.1-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:6f7a7dbe2f7f65ac1d0bd3163f756deb478a9e9afc2269557ed75b1b25ab3610"}, {file = "pyarrow-10.0.1-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:cb627673cb98708ef00864e2e243f51ba7b4c1b9f07a1d821f98043eccd3f585"}, @@ -1687,11 +1555,30 @@ pyarrow = [ {file = "pyarrow-10.0.1-cp39-cp39-win_amd64.whl", hash = "sha256:0ec7587d759153f452d5263dbc8b1af318c4609b607be2bd5127dcda6708cdb1"}, {file = "pyarrow-10.0.1.tar.gz", hash = "sha256:1a14f57a5f472ce8234f2964cd5184cccaa8df7e04568c64edc33b23eb285dd5"}, ] -pycparser = [ + +[package.dependencies] +numpy = ">=1.16.6" + +[[package]] +name = "pycparser" +version = "2.21" +description = "C parser in Python" +category = "main" +optional = false +python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*" +files = [ {file = "pycparser-2.21-py2.py3-none-any.whl", hash = "sha256:8ee45429555515e1f6b185e78100aea234072576aa43ab53aefcae078162fca9"}, {file = "pycparser-2.21.tar.gz", hash = "sha256:e644fdec12f7872f86c58ff790da456218b10f863970249516d60a5eaca77206"}, ] -pydantic = [ + +[[package]] +name = "pydantic" +version = "1.10.2" +description = "Data validation and settings management using python type hints" +category = "main" +optional = false +python-versions = ">=3.7" +files = [ {file = "pydantic-1.10.2-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:bb6ad4489af1bac6955d38ebcb95079a836af31e4c4f74aba1ca05bb9f6027bd"}, {file = "pydantic-1.10.2-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:a1f5a63a6dfe19d719b1b6e6106561869d2efaca6167f84f5ab9347887d78b98"}, {file = "pydantic-1.10.2-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:352aedb1d71b8b0736c6d56ad2bd34c6982720644b0624462059ab29bd6e5912"}, @@ -1729,21 +1616,133 @@ pydantic = [ {file = "pydantic-1.10.2-py3-none-any.whl", hash = "sha256:1b6ee725bd6e83ec78b1aa32c5b1fa67a3a65badddde3976bca5fe4568f27709"}, {file = "pydantic-1.10.2.tar.gz", hash = "sha256:91b8e218852ef6007c2b98cd861601c6a09f1aa32bbbb74fab5b1c33d4a1e410"}, ] -pygments = [] -pyparsing = [ + +[package.dependencies] +typing-extensions = ">=4.1.0" + +[package.extras] +dotenv = ["python-dotenv (>=0.10.4)"] +email = ["email-validator (>=1.0.3)"] + +[[package]] +name = "pygments" +version = "2.13.0" +description = "Pygments is a syntax highlighting package written in Python." +category = "main" +optional = false +python-versions = ">=3.6" +files = [ + {file = "Pygments-2.13.0-py3-none-any.whl", hash = "sha256:f643f331ab57ba3c9d89212ee4a2dabc6e94f117cf4eefde99a0574720d14c42"}, + {file = "Pygments-2.13.0.tar.gz", hash = "sha256:56a8508ae95f98e2b9bdf93a6be5ae3f7d8af858b43e02c5a2ff083726be40c1"}, +] + +[package.extras] +plugins = ["importlib-metadata"] + +[[package]] +name = "pyjwt" +version = "2.6.0" +description = "JSON Web Token implementation in Python" +category = "main" +optional = true +python-versions = ">=3.7" +files = [ + {file = "PyJWT-2.6.0-py3-none-any.whl", hash = "sha256:d83c3d892a77bbb74d3e1a2cfa90afaadb60945205d1095d9221f04466f64c14"}, + {file = "PyJWT-2.6.0.tar.gz", hash = "sha256:69285c7e31fc44f68a1feb309e948e0df53259d579295e6cfe2b1792329f05fd"}, +] + +[package.dependencies] +cryptography = {version = ">=3.4.0", optional = true, markers = "extra == \"crypto\""} + +[package.extras] +crypto = ["cryptography (>=3.4.0)"] +dev = ["coverage[toml] (==5.0.4)", "cryptography (>=3.4.0)", "pre-commit", "pytest (>=6.0.0,<7.0.0)", "sphinx (>=4.5.0,<5.0.0)", "sphinx-rtd-theme", "zope.interface"] +docs = ["sphinx (>=4.5.0,<5.0.0)", "sphinx-rtd-theme", "zope.interface"] +tests = ["coverage[toml] (==5.0.4)", "pytest (>=6.0.0,<7.0.0)"] + +[[package]] +name = "pyparsing" +version = "3.0.9" +description = "pyparsing module - Classes and methods to define and execute parsing grammars" +category = "main" +optional = false +python-versions = ">=3.6.8" +files = [ {file = "pyparsing-3.0.9-py3-none-any.whl", hash = "sha256:5026bae9a10eeaefb61dab2f09052b9f4307d44aee4eda64b309723d8d206bbc"}, {file = "pyparsing-3.0.9.tar.gz", hash = "sha256:2b020ecf7d21b687f219b71ecad3631f644a47f01403fa1d1036b0c6416d70fb"}, ] -pytest = [ + +[package.extras] +diagrams = ["jinja2", "railroad-diagrams"] + +[[package]] +name = "pytest" +version = "7.2.0" +description = "pytest: simple powerful testing with Python" +category = "dev" +optional = false +python-versions = ">=3.7" +files = [ {file = "pytest-7.2.0-py3-none-any.whl", hash = "sha256:892f933d339f068883b6fd5a459f03d85bfcb355e4981e146d2c7616c21fef71"}, {file = "pytest-7.2.0.tar.gz", hash = "sha256:c4014eb40e10f11f355ad4e3c2fb2c6c6d1919c73f3b5a433de4708202cade59"}, ] -pytest-checkdocs = [ + +[package.dependencies] +attrs = ">=19.2.0" +colorama = {version = "*", markers = "sys_platform == \"win32\""} +exceptiongroup = {version = ">=1.0.0rc8", markers = "python_version < \"3.11\""} +iniconfig = "*" +packaging = "*" +pluggy = ">=0.12,<2.0" +tomli = {version = ">=1.0.0", markers = "python_version < \"3.11\""} + +[package.extras] +testing = ["argcomplete", "hypothesis (>=3.56)", "mock", "nose", "pygments (>=2.7.2)", "requests", "xmlschema"] + +[[package]] +name = "pytest-checkdocs" +version = "2.9.0" +description = "check the README when running tests" +category = "dev" +optional = false +python-versions = ">=3.7" +files = [ {file = "pytest-checkdocs-2.9.0.tar.gz", hash = "sha256:54b3d1357a289e3f6336d31d4ceb72e99afe0fa4f552990f1b1f015f1925543b"}, {file = "pytest_checkdocs-2.9.0-py3-none-any.whl", hash = "sha256:4231e5426915a02385e386b5eb4a70490b143d97cede4c7c77691bb320b5a73f"}, ] -python-dateutil = [] -python-snappy = [ + +[package.dependencies] +build = "*" +docutils = ">=0.15" +importlib-metadata = {version = ">=4", markers = "python_version < \"3.10\""} + +[package.extras] +docs = ["furo", "jaraco.packaging (>=9)", "rst.linker (>=1.9)", "sphinx (>=3.5)"] +testing = ["flake8 (<5)", "pytest (>=6)", "pytest-black (>=0.3.7)", "pytest-checkdocs (>=2.4)", "pytest-cov", "pytest-enabler (>=1.3)", "pytest-flake8", "pytest-mypy (>=0.9.1)", "types-docutils"] + +[[package]] +name = "python-dateutil" +version = "2.8.2" +description = "Extensions to the standard Python datetime module" +category = "main" +optional = false +python-versions = "!=3.0.*,!=3.1.*,!=3.2.*,>=2.7" +files = [ + {file = "python-dateutil-2.8.2.tar.gz", hash = "sha256:0123cacc1627ae19ddf3c27a5de5bd67ee4586fbdd6440d9748f8abb483d3e86"}, + {file = "python_dateutil-2.8.2-py2.py3-none-any.whl", hash = "sha256:961d03dc3453ebbc59dbdea9e4e11c5651520a876d0f4db161e8674aae935da9"}, +] + +[package.dependencies] +six = ">=1.5" + +[[package]] +name = "python-snappy" +version = "0.6.1" +description = "Python library for the snappy compression library from Google" +category = "main" +optional = true +python-versions = "*" +files = [ {file = "python-snappy-0.6.1.tar.gz", hash = "sha256:b6a107ab06206acc5359d4c5632bd9b22d448702a79b3169b0c62e0fb808bb2a"}, {file = "python_snappy-0.6.1-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:b7f920eaf46ebf41bd26f9df51c160d40f9e00b7b48471c3438cb8d027f7fb9b"}, {file = "python_snappy-0.6.1-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:4ec533a8c1f8df797bded662ec3e494d225b37855bb63eb0d75464a07947477c"}, @@ -1793,11 +1792,51 @@ python-snappy = [ {file = "python_snappy-0.6.1-pp37-pypy37_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:0bdb6942180660bda7f7d01f4c0def3cfc72b1c6d99aad964801775a3e379aba"}, {file = "python_snappy-0.6.1-pp37-pypy37_pp73-win_amd64.whl", hash = "sha256:03bb511380fca2a13325b6f16fe8234c8e12da9660f0258cd45d9a02ffc916af"}, ] -pytz = [ + +[[package]] +name = "pytz" +version = "2022.6" +description = "World timezone definitions, modern and historical" +category = "main" +optional = false +python-versions = "*" +files = [ {file = "pytz-2022.6-py2.py3-none-any.whl", hash = "sha256:222439474e9c98fced559f1709d89e6c9cbf8d79c794ff3eb9f8800064291427"}, {file = "pytz-2022.6.tar.gz", hash = "sha256:e89512406b793ca39f5971bc999cc538ce125c0e51c27941bef4568b460095e2"}, ] -pyyaml = [ + +[[package]] +name = "pywin32" +version = "305" +description = "Python for Window Extensions" +category = "main" +optional = true +python-versions = "*" +files = [ + {file = "pywin32-305-cp310-cp310-win32.whl", hash = "sha256:421f6cd86e84bbb696d54563c48014b12a23ef95a14e0bdba526be756d89f116"}, + {file = "pywin32-305-cp310-cp310-win_amd64.whl", hash = "sha256:73e819c6bed89f44ff1d690498c0a811948f73777e5f97c494c152b850fad478"}, + {file = "pywin32-305-cp310-cp310-win_arm64.whl", hash = "sha256:742eb905ce2187133a29365b428e6c3b9001d79accdc30aa8969afba1d8470f4"}, + {file = "pywin32-305-cp311-cp311-win32.whl", hash = "sha256:19ca459cd2e66c0e2cc9a09d589f71d827f26d47fe4a9d09175f6aa0256b51c2"}, + {file = "pywin32-305-cp311-cp311-win_amd64.whl", hash = "sha256:326f42ab4cfff56e77e3e595aeaf6c216712bbdd91e464d167c6434b28d65990"}, + {file = "pywin32-305-cp311-cp311-win_arm64.whl", hash = "sha256:4ecd404b2c6eceaca52f8b2e3e91b2187850a1ad3f8b746d0796a98b4cea04db"}, + {file = "pywin32-305-cp36-cp36m-win32.whl", hash = "sha256:48d8b1659284f3c17b68587af047d110d8c44837736b8932c034091683e05863"}, + {file = "pywin32-305-cp36-cp36m-win_amd64.whl", hash = "sha256:13362cc5aa93c2beaf489c9c9017c793722aeb56d3e5166dadd5ef82da021fe1"}, + {file = "pywin32-305-cp37-cp37m-win32.whl", hash = "sha256:a55db448124d1c1484df22fa8bbcbc45c64da5e6eae74ab095b9ea62e6d00496"}, + {file = "pywin32-305-cp37-cp37m-win_amd64.whl", hash = "sha256:109f98980bfb27e78f4df8a51a8198e10b0f347257d1e265bb1a32993d0c973d"}, + {file = "pywin32-305-cp38-cp38-win32.whl", hash = "sha256:9dd98384da775afa009bc04863426cb30596fd78c6f8e4e2e5bbf4edf8029504"}, + {file = "pywin32-305-cp38-cp38-win_amd64.whl", hash = "sha256:56d7a9c6e1a6835f521788f53b5af7912090674bb84ef5611663ee1595860fc7"}, + {file = "pywin32-305-cp39-cp39-win32.whl", hash = "sha256:9d968c677ac4d5cbdaa62fd3014ab241718e619d8e36ef8e11fb930515a1e918"}, + {file = "pywin32-305-cp39-cp39-win_amd64.whl", hash = "sha256:50768c6b7c3f0b38b7fb14dd4104da93ebced5f1a50dc0e834594bff6fbe1271"}, +] + +[[package]] +name = "pyyaml" +version = "6.0" +description = "YAML parser and emitter for Python" +category = "main" +optional = false +python-versions = ">=3.6" +files = [ {file = "PyYAML-6.0-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:d4db7c7aef085872ef65a8fd7d6d09a14ae91f691dec3e87ee5ee0539d516f53"}, {file = "PyYAML-6.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:9df7ed3b3d2e0ecfe09e14741b857df43adb5a3ddadc919a2d94fbdf78fea53c"}, {file = "PyYAML-6.0-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:77f396e6ef4c73fdc33a9157446466f1cff553d979bd00ecb64385760c6babdc"}, @@ -1805,6 +1844,13 @@ pyyaml = [ {file = "PyYAML-6.0-cp310-cp310-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:f84fbc98b019fef2ee9a1cb3ce93e3187a6df0b2538a651bfb890254ba9f90b5"}, {file = "PyYAML-6.0-cp310-cp310-win32.whl", hash = "sha256:2cd5df3de48857ed0544b34e2d40e9fac445930039f3cfe4bcc592a1f836d513"}, {file = "PyYAML-6.0-cp310-cp310-win_amd64.whl", hash = "sha256:daf496c58a8c52083df09b80c860005194014c3698698d1a57cbcfa182142a3a"}, + {file = "PyYAML-6.0-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:d4b0ba9512519522b118090257be113b9468d804b19d63c71dbcf4a48fa32358"}, + {file = "PyYAML-6.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:81957921f441d50af23654aa6c5e5eaf9b06aba7f0a19c18a538dc7ef291c5a1"}, + {file = "PyYAML-6.0-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:afa17f5bc4d1b10afd4466fd3a44dc0e245382deca5b3c353d8b757f9e3ecb8d"}, + {file = "PyYAML-6.0-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:dbad0e9d368bb989f4515da330b88a057617d16b6a8245084f1b05400f24609f"}, + {file = "PyYAML-6.0-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:432557aa2c09802be39460360ddffd48156e30721f5e8d917f01d31694216782"}, + {file = "PyYAML-6.0-cp311-cp311-win32.whl", hash = "sha256:bfaef573a63ba8923503d27530362590ff4f576c626d86a9fed95822a8255fd7"}, + {file = "PyYAML-6.0-cp311-cp311-win_amd64.whl", hash = "sha256:01b45c0191e6d66c470b6cf1b9531a771a83c1c4208272ead47a3ae4f2f603bf"}, {file = "PyYAML-6.0-cp36-cp36m-macosx_10_9_x86_64.whl", hash = "sha256:897b80890765f037df3403d22bab41627ca8811ae55e9a722fd0392850ec4d86"}, {file = "PyYAML-6.0-cp36-cp36m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:50602afada6d6cbfad699b0c7bb50d5ccffa7e46a3d738092afddc1f9758427f"}, {file = "PyYAML-6.0-cp36-cp36m-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:48c346915c114f5fdb3ead70312bd042a953a8ce5c7106d5bfb1a5254e47da92"}, @@ -1832,61 +1878,308 @@ pyyaml = [ {file = "PyYAML-6.0-cp39-cp39-win_amd64.whl", hash = "sha256:b3d267842bf12586ba6c734f89d1f5b871df0273157918b0ccefa29deb05c21c"}, {file = "PyYAML-6.0.tar.gz", hash = "sha256:68fb519c14306fec9720a2a5b45bc9f0c8d1b9c72adf45c37baedfcd949c35a2"}, ] -requests = [ + +[[package]] +name = "requests" +version = "2.28.1" +description = "Python HTTP for Humans." +category = "main" +optional = false +python-versions = ">=3.7, <4" +files = [ {file = "requests-2.28.1-py3-none-any.whl", hash = "sha256:8fefa2a1a1365bf5520aac41836fbee479da67864514bdb821f31ce07ce65349"}, {file = "requests-2.28.1.tar.gz", hash = "sha256:7c5599b102feddaa661c826c56ab4fee28bfd17f5abca1ebbe3e7f19d7c97983"}, ] -requests-mock = [ + +[package.dependencies] +certifi = ">=2017.4.17" +charset-normalizer = ">=2,<3" +idna = ">=2.5,<4" +urllib3 = ">=1.21.1,<1.27" + +[package.extras] +socks = ["PySocks (>=1.5.6,!=1.5.7)"] +use-chardet-on-py3 = ["chardet (>=3.0.2,<6)"] + +[[package]] +name = "requests-mock" +version = "1.10.0" +description = "Mock out responses from the requests package" +category = "dev" +optional = false +python-versions = "*" +files = [ {file = "requests-mock-1.10.0.tar.gz", hash = "sha256:59c9c32419a9fb1ae83ec242d98e889c45bd7d7a65d48375cc243ec08441658b"}, {file = "requests_mock-1.10.0-py2.py3-none-any.whl", hash = "sha256:2fdbb637ad17ee15c06f33d31169e71bf9fe2bdb7bc9da26185be0dd8d842699"}, ] -responses = [ + +[package.dependencies] +requests = ">=2.3,<3" +six = "*" + +[package.extras] +fixture = ["fixtures"] +test = ["fixtures", "mock", "purl", "pytest", "requests-futures", "sphinx", "testrepository (>=0.0.18)", "testtools"] + +[[package]] +name = "requests-oauthlib" +version = "1.3.1" +description = "OAuthlib authentication support for Requests." +category = "main" +optional = true +python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*" +files = [ + {file = "requests-oauthlib-1.3.1.tar.gz", hash = "sha256:75beac4a47881eeb94d5ea5d6ad31ef88856affe2332b9aafb52c6452ccf0d7a"}, + {file = "requests_oauthlib-1.3.1-py2.py3-none-any.whl", hash = "sha256:2577c501a2fb8d05a304c09d090d6e47c306fef15809d102b327cf8364bddab5"}, +] + +[package.dependencies] +oauthlib = ">=3.0.0" +requests = ">=2.0.0" + +[package.extras] +rsa = ["oauthlib[signedtoken] (>=3.0.0)"] + +[[package]] +name = "responses" +version = "0.22.0" +description = "A utility library for mocking out the `requests` Python library." +category = "dev" +optional = false +python-versions = ">=3.7" +files = [ {file = "responses-0.22.0-py3-none-any.whl", hash = "sha256:dcf294d204d14c436fddcc74caefdbc5764795a40ff4e6a7740ed8ddbf3294be"}, {file = "responses-0.22.0.tar.gz", hash = "sha256:396acb2a13d25297789a5866b4881cf4e46ffd49cc26c43ab1117f40b973102e"}, ] -rich = [ + +[package.dependencies] +requests = ">=2.22.0,<3.0" +toml = "*" +types-toml = "*" +urllib3 = ">=1.25.10" + +[package.extras] +tests = ["coverage (>=6.0.0)", "flake8", "mypy", "pytest (>=7.0.0)", "pytest-asyncio", "pytest-cov", "pytest-httpserver", "types-requests"] + +[[package]] +name = "rich" +version = "12.6.0" +description = "Render rich text, tables, progress bars, syntax highlighting, markdown and more to the terminal" +category = "main" +optional = false +python-versions = ">=3.6.3,<4.0.0" +files = [ {file = "rich-12.6.0-py3-none-any.whl", hash = "sha256:a4eb26484f2c82589bd9a17c73d32a010b1e29d89f1604cd9bf3a2097b81bb5e"}, {file = "rich-12.6.0.tar.gz", hash = "sha256:ba3a3775974105c221d31141f2c116f4fd65c5ceb0698657a11e9f295ec93fd0"}, ] -s3fs = [ + +[package.dependencies] +commonmark = ">=0.9.0,<0.10.0" +pygments = ">=2.6.0,<3.0.0" +typing-extensions = {version = ">=4.0.0,<5.0", markers = "python_version < \"3.9\""} + +[package.extras] +jupyter = ["ipywidgets (>=7.5.1,<8.0.0)"] + +[[package]] +name = "s3fs" +version = "2022.10.0" +description = "Convenient Filesystem interface over S3" +category = "main" +optional = true +python-versions = ">= 3.7" +files = [ {file = "s3fs-2022.10.0-py3-none-any.whl", hash = "sha256:1e134c3577171699feb7c1a0c4713260d5b48296e1708737ff940baef6e2c153"}, {file = "s3fs-2022.10.0.tar.gz", hash = "sha256:e8deb80f20bd0b2059141b874fdb9d6aeb8cce35312ea5f2c02b225a78a00406"}, ] -s3transfer = [] -six = [ + +[package.dependencies] +aiobotocore = ">=2.4.0,<2.5.0" +aiohttp = "<4.0.0a0 || >4.0.0a0,<4.0.0a1 || >4.0.0a1" +fsspec = "2022.10.0" + +[package.extras] +awscli = ["aiobotocore[awscli] (>=2.4.0,<2.5.0)"] +boto3 = ["aiobotocore[boto3] (>=2.4.0,<2.5.0)"] + +[[package]] +name = "s3transfer" +version = "0.6.0" +description = "An Amazon S3 Transfer Manager" +category = "main" +optional = false +python-versions = ">= 3.7" +files = [ + {file = "s3transfer-0.6.0-py3-none-any.whl", hash = "sha256:06176b74f3a15f61f1b4f25a1fc29a4429040b7647133a463da8fa5bd28d5ecd"}, + {file = "s3transfer-0.6.0.tar.gz", hash = "sha256:2ed07d3866f523cc561bf4a00fc5535827981b117dd7876f036b0c1aca42c947"}, +] + +[package.dependencies] +botocore = ">=1.12.36,<2.0a.0" + +[package.extras] +crt = ["botocore[crt] (>=1.20.29,<2.0a.0)"] + +[[package]] +name = "setuptools" +version = "65.6.3" +description = "Easily download, build, install, upgrade, and uninstall Python packages" +category = "dev" +optional = false +python-versions = ">=3.7" +files = [ + {file = "setuptools-65.6.3-py3-none-any.whl", hash = "sha256:57f6f22bde4e042978bcd50176fdb381d7c21a9efa4041202288d3737a0c6a54"}, + {file = "setuptools-65.6.3.tar.gz", hash = "sha256:a7620757bf984b58deaf32fc8a4577a9bbc0850cf92c20e1ce41c38c19e5fb75"}, +] + +[package.extras] +docs = ["furo", "jaraco.packaging (>=9)", "jaraco.tidelift (>=1.4)", "pygments-github-lexers (==0.0.5)", "rst.linker (>=1.9)", "sphinx (>=3.5)", "sphinx-favicon", "sphinx-hoverxref (<2)", "sphinx-inline-tabs", "sphinx-notfound-page (==0.8.3)", "sphinx-reredirects", "sphinxcontrib-towncrier"] +testing = ["build[virtualenv]", "filelock (>=3.4.0)", "flake8 (<5)", "flake8-2020", "ini2toml[lite] (>=0.9)", "jaraco.envs (>=2.2)", "jaraco.path (>=3.2.0)", "pip (>=19.1)", "pip-run (>=8.8)", "pytest (>=6)", "pytest-black (>=0.3.7)", "pytest-checkdocs (>=2.4)", "pytest-cov", "pytest-enabler (>=1.3)", "pytest-flake8", "pytest-mypy (>=0.9.1)", "pytest-perf", "pytest-timeout", "pytest-xdist", "tomli-w (>=1.0.0)", "virtualenv (>=13.0.0)", "wheel"] +testing-integration = ["build[virtualenv]", "filelock (>=3.4.0)", "jaraco.envs (>=2.2)", "jaraco.path (>=3.2.0)", "pytest", "pytest-enabler", "pytest-xdist", "tomli", "virtualenv (>=13.0.0)", "wheel"] + +[[package]] +name = "six" +version = "1.16.0" +description = "Python 2 and 3 compatibility utilities" +category = "main" +optional = false +python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*" +files = [ {file = "six-1.16.0-py2.py3-none-any.whl", hash = "sha256:8abb2f1d86890a2dfb989f9a77cfcfd3e47c2a354b01111771326f8aa26e0254"}, {file = "six-1.16.0.tar.gz", hash = "sha256:1e61c37477a1626458e36f7b1d82aa5c9b094fa4802892072e49de9c60c4c926"}, ] -thrift = [] -toml = [ + +[[package]] +name = "thrift" +version = "0.16.0" +description = "Python bindings for the Apache Thrift RPC system" +category = "main" +optional = true +python-versions = "*" +files = [ + {file = "thrift-0.16.0.tar.gz", hash = "sha256:2b5b6488fcded21f9d312aa23c9ff6a0195d0f6ae26ddbd5ad9e3e25dfc14408"}, +] + +[package.dependencies] +six = ">=1.7.2" + +[package.extras] +all = ["tornado (>=4.0)", "twisted"] +tornado = ["tornado (>=4.0)"] +twisted = ["twisted"] + +[[package]] +name = "toml" +version = "0.10.2" +description = "Python Library for Tom's Obvious, Minimal Language" +category = "dev" +optional = false +python-versions = ">=2.6, !=3.0.*, !=3.1.*, !=3.2.*" +files = [ {file = "toml-0.10.2-py2.py3-none-any.whl", hash = "sha256:806143ae5bfb6a3c6e736a764057db0e6a0e05e338b5630894a5f779cabb4f9b"}, {file = "toml-0.10.2.tar.gz", hash = "sha256:b3bda1d108d5dd99f4a20d24d9c348e91c4db7ab1b749200bded2f839ccbe68f"}, ] -tomli = [ + +[[package]] +name = "tomli" +version = "2.0.1" +description = "A lil' TOML parser" +category = "dev" +optional = false +python-versions = ">=3.7" +files = [ {file = "tomli-2.0.1-py3-none-any.whl", hash = "sha256:939de3e7a6161af0c887ef91b7d41a53e7c5a1ca976325f429cb46ea9bc30ecc"}, {file = "tomli-2.0.1.tar.gz", hash = "sha256:de526c12914f0c550d15924c62d72abc48d6fe7364aa87328337a31007fe8a4f"}, ] -types-toml = [ + +[[package]] +name = "types-toml" +version = "0.10.8.1" +description = "Typing stubs for toml" +category = "dev" +optional = false +python-versions = "*" +files = [ {file = "types-toml-0.10.8.1.tar.gz", hash = "sha256:171bdb3163d79a520560f24ba916a9fc9bff81659c5448a9fea89240923722be"}, {file = "types_toml-0.10.8.1-py3-none-any.whl", hash = "sha256:b7b5c4977f96ab7b5ac06d8a6590d17c0bf252a96efc03b109c2711fb3e0eafd"}, ] -typing-extensions = [ + +[[package]] +name = "typing-extensions" +version = "4.4.0" +description = "Backported and Experimental Type Hints for Python 3.7+" +category = "main" +optional = false +python-versions = ">=3.7" +files = [ {file = "typing_extensions-4.4.0-py3-none-any.whl", hash = "sha256:16fa4864408f655d35ec496218b85f79b3437c829e93320c7c9215ccfd92489e"}, {file = "typing_extensions-4.4.0.tar.gz", hash = "sha256:1511434bb92bf8dd198c12b1cc812e800d4181cfcb867674e0f8279cc93087aa"}, ] -urllib3 = [ + +[[package]] +name = "urllib3" +version = "1.26.13" +description = "HTTP library with thread-safe connection pooling, file post, and more." +category = "main" +optional = false +python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*, !=3.5.*" +files = [ {file = "urllib3-1.26.13-py2.py3-none-any.whl", hash = "sha256:47cc05d99aaa09c9e72ed5809b60e7ba354e64b59c9c173ac3018642d8bb41fc"}, {file = "urllib3-1.26.13.tar.gz", hash = "sha256:c083dd0dce68dbfbe1129d5271cb90f9447dea7d52097c6e0126120c521ddea8"}, ] -virtualenv = [ + +[package.extras] +brotli = ["brotli (>=1.0.9)", "brotlicffi (>=0.8.0)", "brotlipy (>=0.6.0)"] +secure = ["certifi", "cryptography (>=1.3.4)", "idna (>=2.0.0)", "ipaddress", "pyOpenSSL (>=0.14)", "urllib3-secure-extra"] +socks = ["PySocks (>=1.5.6,!=1.5.7,<2.0)"] + +[[package]] +name = "virtualenv" +version = "20.17.1" +description = "Virtual Python Environment builder" +category = "dev" +optional = false +python-versions = ">=3.6" +files = [ {file = "virtualenv-20.17.1-py3-none-any.whl", hash = "sha256:ce3b1684d6e1a20a3e5ed36795a97dfc6af29bc3970ca8dab93e11ac6094b3c4"}, {file = "virtualenv-20.17.1.tar.gz", hash = "sha256:f8b927684efc6f1cc206c9db297a570ab9ad0e51c16fa9e45487d36d1905c058"}, ] -werkzeug = [ + +[package.dependencies] +distlib = ">=0.3.6,<1" +filelock = ">=3.4.1,<4" +platformdirs = ">=2.4,<3" + +[package.extras] +docs = ["proselint (>=0.13)", "sphinx (>=5.3)", "sphinx-argparse (>=0.3.2)", "sphinx-rtd-theme (>=1)", "towncrier (>=22.8)"] +testing = ["coverage (>=6.2)", "coverage-enable-subprocess (>=1)", "flaky (>=3.7)", "packaging (>=21.3)", "pytest (>=7.0.1)", "pytest-env (>=0.6.2)", "pytest-freezegun (>=0.4.2)", "pytest-mock (>=3.6.1)", "pytest-randomly (>=3.10.3)", "pytest-timeout (>=2.1)"] + +[[package]] +name = "werkzeug" +version = "2.2.2" +description = "The comprehensive WSGI web application library." +category = "dev" +optional = false +python-versions = ">=3.7" +files = [ {file = "Werkzeug-2.2.2-py3-none-any.whl", hash = "sha256:f979ab81f58d7318e064e99c4506445d60135ac5cd2e177a2de0089bfd4c9bd5"}, {file = "Werkzeug-2.2.2.tar.gz", hash = "sha256:7ea2d48322cc7c0f8b3a215ed73eabd7b5d75d0b50e31ab006286ccff9e00b8f"}, ] -wrapt = [ + +[package.dependencies] +MarkupSafe = ">=2.1.1" + +[package.extras] +watchdog = ["watchdog"] + +[[package]] +name = "wrapt" +version = "1.14.1" +description = "Module for decorators, wrappers and monkey patching." +category = "main" +optional = true +python-versions = "!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,!=3.4.*,>=2.7" +files = [ {file = "wrapt-1.14.1-cp27-cp27m-macosx_10_9_x86_64.whl", hash = "sha256:1b376b3f4896e7930f1f772ac4b064ac12598d1c38d04907e696cc4d794b43d3"}, {file = "wrapt-1.14.1-cp27-cp27m-manylinux1_i686.whl", hash = "sha256:903500616422a40a98a5a3c4ff4ed9d0066f3b4c951fa286018ecdf0750194ef"}, {file = "wrapt-1.14.1-cp27-cp27m-manylinux1_x86_64.whl", hash = "sha256:5a9a0d155deafd9448baff28c08e150d9b24ff010e899311ddd63c45c2445e28"}, @@ -1952,11 +2245,27 @@ wrapt = [ {file = "wrapt-1.14.1-cp39-cp39-win_amd64.whl", hash = "sha256:dee60e1de1898bde3b238f18340eec6148986da0455d8ba7848d50470a7a32fb"}, {file = "wrapt-1.14.1.tar.gz", hash = "sha256:380a85cf89e0e69b7cfbe2ea9f765f004ff419f34194018a6827ac0e3edfed4d"}, ] -xmltodict = [ + +[[package]] +name = "xmltodict" +version = "0.13.0" +description = "Makes working with XML feel like you are working with JSON" +category = "dev" +optional = false +python-versions = ">=3.4" +files = [ {file = "xmltodict-0.13.0-py2.py3-none-any.whl", hash = "sha256:aa89e8fd76320154a40d19a0df04a4695fb9dc5ba977cbb68ab3e4eb225e7852"}, {file = "xmltodict-0.13.0.tar.gz", hash = "sha256:341595a488e3e01a85a9d8911d8912fd922ede5fecc4dce437eb4b6c8d037e56"}, ] -yarl = [ + +[[package]] +name = "yarl" +version = "1.8.2" +description = "Yet another URL library" +category = "main" +optional = true +python-versions = ">=3.7" +files = [ {file = "yarl-1.8.2-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:bb81f753c815f6b8e2ddd2eef3c855cf7da193b82396ac013c661aaa6cc6b0a5"}, {file = "yarl-1.8.2-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:47d49ac96156f0928f002e2424299b2c91d9db73e08c4cd6742923a086f1c863"}, {file = "yarl-1.8.2-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:3fc056e35fa6fba63248d93ff6e672c096f95f7836938241ebc8260e062832fe"}, @@ -2032,11 +2341,35 @@ yarl = [ {file = "yarl-1.8.2-cp39-cp39-win_amd64.whl", hash = "sha256:6604711362f2dbf7160df21c416f81fac0de6dbcf0b5445a2ef25478ecc4c778"}, {file = "yarl-1.8.2.tar.gz", hash = "sha256:49d43402c6e3013ad0978602bf6bf5328535c48d192304b91b97a3c6790b1562"}, ] -zipp = [ + +[package.dependencies] +idna = ">=2.0" +multidict = ">=4.0" + +[[package]] +name = "zipp" +version = "3.11.0" +description = "Backport of pathlib-compatible object wrapper for zip files" +category = "dev" +optional = false +python-versions = ">=3.7" +files = [ {file = "zipp-3.11.0-py3-none-any.whl", hash = "sha256:83a28fcb75844b5c0cdaf5aa4003c2d728c77e05f5aeabe8e95e56727005fbaa"}, {file = "zipp-3.11.0.tar.gz", hash = "sha256:a7a22e05929290a67401440b39690ae6563279bced5f314609d9d03798f56766"}, ] -zstandard = [ + +[package.extras] +docs = ["furo", "jaraco.packaging (>=9)", "jaraco.tidelift (>=1.4)", "rst.linker (>=1.9)", "sphinx (>=3.5)"] +testing = ["flake8 (<5)", "func-timeout", "jaraco.functools", "jaraco.itertools", "more-itertools", "pytest (>=6)", "pytest-black (>=0.3.7)", "pytest-checkdocs (>=2.4)", "pytest-cov", "pytest-enabler (>=1.3)", "pytest-flake8", "pytest-mypy (>=0.9.1)"] + +[[package]] +name = "zstandard" +version = "0.19.0" +description = "Zstandard bindings for Python" +category = "main" +optional = false +python-versions = ">=3.6" +files = [ {file = "zstandard-0.19.0-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:a65e0119ad39e855427520f7829618f78eb2824aa05e63ff19b466080cd99210"}, {file = "zstandard-0.19.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:4fa496d2d674c6e9cffc561639d17009d29adee84a27cf1e12d3c9be14aa8feb"}, {file = "zstandard-0.19.0-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:8f7c68de4f362c1b2f426395fe4e05028c56d0782b2ec3ae18a5416eaf775576"}, @@ -2089,3 +2422,24 @@ zstandard = [ {file = "zstandard-0.19.0-cp39-cp39-win_amd64.whl", hash = "sha256:55a513ec67e85abd8b8b83af8813368036f03e2d29a50fc94033504918273980"}, {file = "zstandard-0.19.0.tar.gz", hash = "sha256:31d12fcd942dd8dbf52ca5f6b1bbe287f44e5d551a081a983ff3ea2082867863"}, ] + +[package.dependencies] +cffi = {version = ">=1.11", markers = "platform_python_implementation == \"PyPy\""} + +[package.extras] +cffi = ["cffi (>=1.11)"] + +[extras] +adlfs = ["adlfs"] +duckdb = ["duckdb", "pyarrow"] +glue = ["boto3"] +hive = ["thrift"] +pandas = ["pandas", "pyarrow"] +pyarrow = ["pyarrow"] +s3fs = ["s3fs"] +snappy = ["python-snappy"] + +[metadata] +lock-version = "2.0" +python-versions = "^3.8" +content-hash = "f56dc6fb65c9d55f693f2fa9b5b02b8a11291b8fb47c2ad55400ae7c84a20de4" diff --git a/pyiceberg/io/__init__.py b/pyiceberg/io/__init__.py index 7c721712d9..3702f47109 100644 --- a/pyiceberg/io/__init__.py +++ b/pyiceberg/io/__init__.py @@ -257,6 +257,8 @@ def delete(self, location: Union[str, InputFile, OutputFile]) -> None: "gcs": [ARROW_FILE_IO], "file": [ARROW_FILE_IO], "hdfs": [ARROW_FILE_IO], + "abfs": [FSSPEC_FILE_IO], + "abfss": [FSSPEC_FILE_IO], } diff --git a/pyiceberg/io/fsspec.py b/pyiceberg/io/fsspec.py index 34e2246206..9624562cb1 100644 --- a/pyiceberg/io/fsspec.py +++ b/pyiceberg/io/fsspec.py @@ -15,7 +15,9 @@ # specific language governing permissions and limitations # under the License. """FileIO implementation for reading and writing table files that uses fsspec compatible filesystems""" +import errno import logging +import os from functools import lru_cache, partial from typing import ( Any, @@ -26,6 +28,7 @@ from urllib.parse import urlparse import requests +from adlfs import AzureBlobFileSystem from botocore import UNSIGNED from botocore.awsrequest import AWSRequest from fsspec import AbstractFileSystem @@ -106,10 +109,17 @@ def _s3(properties: Properties) -> AbstractFileSystem: return fs +def _adlfs(properties: Properties) -> AbstractFileSystem: + fs = AzureBlobFileSystem(**properties) + return fs + + SCHEME_TO_FS = { "s3": _s3, "s3a": _s3, "s3n": _s3, + "abfs": _adlfs, + "abfss": _adlfs, } @@ -143,8 +153,15 @@ def open(self) -> InputStream: Returns: OpenFile: An fsspec compliant file-like object + + Raises: + FileNotFoundError: If the file does not exist """ - return self._fs.open(self.location, "rb") + try: + return self._fs.open(self.location, "rb") + except FileNotFoundError as e: + # To have a consistent error handling experience, make sure exception contains missing file location. + raise e if e.filename else FileNotFoundError(errno.ENOENT, os.strerror(errno.ENOENT), self.location) from e class FsspecOutputFile(OutputFile): diff --git a/pyproject.toml b/pyproject.toml index 61cabf1f7a..4376873cf3 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -73,6 +73,8 @@ thrift = { version = "0.16.0", optional = true } s3fs = { version = "2022.10.0", optional = true } boto3 = {version = "1.24.59", optional = true} +adlfs = { version = "2022.10.0", optional = true } + [tool.poetry.dev-dependencies] pytest = "7.2.0" pytest-checkdocs = "2.9.0" @@ -99,10 +101,12 @@ snappy = ["python-snappy"] hive = ["thrift"] s3fs = ["s3fs"] glue = ["boto3"] +adlfs = ["adlfs"] [tool.pytest.ini_options] markers = [ - "s3: marks a test as requiring access to s3 compliant storage (use with --aws-access-key-id, ----aws-secret-access-key, and --endpoint-url args)" + "s3: marks a test as requiring access to s3 compliant storage (use with --aws-access-key-id, --aws-secret-access-key, and --endpoint-url args)", + "adlfs: marks a test as requiring access to adlfs compliant storage (use with --adlfs.account-name, --adlfs.account-key, and --adlfs.endpoint args)" ] [tool.black] @@ -188,6 +192,14 @@ ignore_missing_imports = true module = "s3fs.*" ignore_missing_imports = true +[[tool.mypy.overrides]] +module = "azure.*" +ignore_missing_imports = true + +[[tool.mypy.overrides]] +module = "adlfs.*" +ignore_missing_imports = true + [[tool.mypy.overrides]] module = "packaging.*" ignore_missing_imports = true diff --git a/tests/conftest.py b/tests/conftest.py index f809cfc503..c656ebf4d0 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -68,6 +68,7 @@ def pytest_addoption(parser: pytest.Parser) -> None: + # S3 options parser.addoption( "--s3.endpoint", action="store", default="http://localhost:9000", help="The S3 endpoint URL for tests marked as s3" ) @@ -75,6 +76,24 @@ def pytest_addoption(parser: pytest.Parser) -> None: parser.addoption( "--s3.secret-access-key", action="store", default="password", help="The AWS secret access key ID for tests marked as s3" ) + # ADLFS options + # Azurite provides default account name and key. Those can be customized using env variables. + # For more information, see README file at https://github.com/azure/azurite#default-storage-account + parser.addoption( + "--adlfs.endpoint", + action="store", + default="http://127.0.0.1:10000", + help="The ADLS endpoint URL for tests marked as adlfs", + ) + parser.addoption( + "--adlfs.account-name", action="store", default="devstoreaccount1", help="The ADLS account key for tests marked as adlfs" + ) + parser.addoption( + "--adlfs.account-key", + action="store", + default="Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw==", + help="The ADLS secret account key for tests marked as adlfs", + ) @pytest.fixture(scope="session") @@ -1214,6 +1233,25 @@ def fixture_glue(_aws_credentials: None) -> Generator[boto3.client, None, None]: yield boto3.client("glue", region_name="us-east-1") +@pytest.fixture +def adlfs_fsspec_fileio(request: pytest.FixtureRequest) -> Generator[FsspecFileIO, None, None]: + from azure.storage.blob import BlobServiceClient + + azurite_url = request.config.getoption("--adlfs.endpoint") + azurite_account_name = request.config.getoption("--adlfs.account-name") + azurite_account_key = request.config.getoption("--adlfs.account-key") + azurite_connection_string = f"DefaultEndpointsProtocol=http;AccountName={azurite_account_name};AccountKey={azurite_account_key};BlobEndpoint={azurite_url}/{azurite_account_name};" + properties = { + "connection_string": azurite_connection_string, + "account_name": azurite_account_name, + } + + bbs = BlobServiceClient.from_connection_string(conn_str=azurite_connection_string) + bbs.create_container("tests") + yield fsspec.FsspecFileIO(properties=properties) + bbs.delete_container("tests") + + @pytest.fixture(scope="session") def empty_home_dir_path(tmp_path_factory: pytest.TempPathFactory) -> str: home_path = str(tmp_path_factory.mktemp("home")) diff --git a/tests/io/test_fsspec.py b/tests/io/test_fsspec.py index fff308b212..a92e383e79 100644 --- a/tests/io/test_fsspec.py +++ b/tests/io/test_fsspec.py @@ -131,7 +131,7 @@ def test_fsspec_read_specified_bytes_for_file(fsspec_fileio: FsspecFileIO) -> No @pytest.mark.s3 def test_fsspec_raise_on_opening_file_not_found(fsspec_fileio: FsspecFileIO) -> None: - """Test that an fsppec input file raises appropriately when the s3 file is not found""" + """Test that an fsspec input file raises appropriately when the s3 file is not found""" filename = str(uuid.uuid4()) input_file = fsspec_fileio.new_input(location=f"s3://warehouse/{filename}") @@ -203,6 +203,189 @@ def test_writing_avro_file(generated_manifest_entry_file: str, fsspec_fileio: Fs b2 = in_f.read() assert b1 == b2 # Check that bytes of read from local avro file match bytes written to s3 + fsspec_fileio.delete(f"s3://warehouse/{filename}") + + +@pytest.mark.adlfs +def test_fsspec_new_input_file_adlfs(adlfs_fsspec_fileio: FsspecFileIO) -> None: + """Test creating a new input file from an fsspec file-io""" + filename = str(uuid.uuid4()) + + input_file = adlfs_fsspec_fileio.new_input(f"abfss://tests/{filename}") + + assert isinstance(input_file, fsspec.FsspecInputFile) + assert input_file.location == f"abfss://tests/{filename}" + + +@pytest.mark.adlfs +def test_fsspec_new_abfss_output_file_adlfs(adlfs_fsspec_fileio: FsspecFileIO) -> None: + """Test creating a new output file from an fsspec file-io""" + filename = str(uuid.uuid4()) + + output_file = adlfs_fsspec_fileio.new_output(f"abfss://tests/{filename}") + + assert isinstance(output_file, fsspec.FsspecOutputFile) + assert output_file.location == f"abfss://tests/{filename}" + + +@pytest.mark.adlfs +def test_fsspec_write_and_read_file_adlfs(adlfs_fsspec_fileio: FsspecFileIO) -> None: + """Test writing and reading a file using FsspecInputFile and FsspecOutputFile""" + filename = str(uuid.uuid4()) + output_file = adlfs_fsspec_fileio.new_output(location=f"abfss://tests/{filename}") + with output_file.create() as f: + f.write(b"foo") + + input_file = adlfs_fsspec_fileio.new_input(f"abfss://tests/{filename}") + assert input_file.open().read() == b"foo" + + adlfs_fsspec_fileio.delete(input_file) + + +@pytest.mark.adlfs +def test_fsspec_getting_length_of_file_adlfs(adlfs_fsspec_fileio: FsspecFileIO) -> None: + """Test getting the length of an FsspecInputFile and FsspecOutputFile""" + filename = str(uuid.uuid4()) + + output_file = adlfs_fsspec_fileio.new_output(location=f"abfss://tests/{filename}") + with output_file.create() as f: + f.write(b"foobar") + + assert len(output_file) == 6 + + input_file = adlfs_fsspec_fileio.new_input(location=f"abfss://tests/{filename}") + assert len(input_file) == 6 + + adlfs_fsspec_fileio.delete(output_file) + + +@pytest.mark.adlfs +def test_fsspec_file_tell_adlfs(adlfs_fsspec_fileio: FsspecFileIO) -> None: + """Test finding cursor position for an fsspec file-io file""" + + filename = str(uuid.uuid4()) + + output_file = adlfs_fsspec_fileio.new_output(location=f"abfss://tests/{filename}") + with output_file.create() as write_file: + write_file.write(b"foobar") + + input_file = adlfs_fsspec_fileio.new_input(location=f"abfss://tests/{filename}") + f = input_file.open() + + f.seek(0) + assert f.tell() == 0 + f.seek(1) + assert f.tell() == 1 + f.seek(3) + assert f.tell() == 3 + f.seek(0) + assert f.tell() == 0 + + adlfs_fsspec_fileio.delete(f"abfss://tests/{filename}") + + +@pytest.mark.adlfs +def test_fsspec_read_specified_bytes_for_file_adlfs(adlfs_fsspec_fileio: FsspecFileIO) -> None: + """Test reading a specified number of bytes from an fsspec file-io file""" + + filename = str(uuid.uuid4()) + output_file = adlfs_fsspec_fileio.new_output(location=f"abfss://tests/{filename}") + with output_file.create() as write_file: + write_file.write(b"foo") + + input_file = adlfs_fsspec_fileio.new_input(location=f"abfss://tests/{filename}") + f = input_file.open() + + f.seek(0) + assert b"f" == f.read(1) + f.seek(0) + assert b"fo" == f.read(2) + f.seek(1) + assert b"o" == f.read(1) + f.seek(1) + assert b"oo" == f.read(2) + f.seek(0) + assert b"foo" == f.read(999) # test reading amount larger than entire content length + + adlfs_fsspec_fileio.delete(input_file) + + +@pytest.mark.adlfs +def test_fsspec_raise_on_opening_file_not_found_adlfs(adlfs_fsspec_fileio: FsspecFileIO) -> None: + """Test that an fsspec input file raises appropriately when the adlfs file is not found""" + + filename = str(uuid.uuid4()) + input_file = adlfs_fsspec_fileio.new_input(location=f"abfss://tests/{filename}") + with pytest.raises(FileNotFoundError) as exc_info: + input_file.open().read() + + assert filename in str(exc_info.value) + + +@pytest.mark.adlfs +def test_checking_if_a_file_exists_adlfs(adlfs_fsspec_fileio: FsspecFileIO) -> None: + """Test checking if a file exists""" + + non_existent_file = adlfs_fsspec_fileio.new_input(location="abfss://tests/does-not-exist.txt") + assert not non_existent_file.exists() + + filename = str(uuid.uuid4()) + output_file = adlfs_fsspec_fileio.new_output(location=f"abfss://tests/{filename}") + assert not output_file.exists() + with output_file.create() as f: + f.write(b"foo") + + existing_input_file = adlfs_fsspec_fileio.new_input(location=f"abfss://tests/{filename}") + assert existing_input_file.exists() + + existing_output_file = adlfs_fsspec_fileio.new_output(location=f"abfss://tests/{filename}") + assert existing_output_file.exists() + + adlfs_fsspec_fileio.delete(existing_output_file) + + +@pytest.mark.adlfs +def test_closing_a_file_adlfs(adlfs_fsspec_fileio: FsspecFileIO) -> None: + """Test closing an output file and input file""" + filename = str(uuid.uuid4()) + output_file = adlfs_fsspec_fileio.new_output(location=f"abfss://tests/{filename}") + with output_file.create() as write_file: + write_file.write(b"foo") + assert not write_file.closed # type: ignore + assert write_file.closed # type: ignore + + input_file = adlfs_fsspec_fileio.new_input(location=f"abfss://tests/{filename}") + f = input_file.open() + assert not f.closed # type: ignore + f.close() + assert f.closed # type: ignore + + adlfs_fsspec_fileio.delete(f"abfss://tests/{filename}") + + +@pytest.mark.adlfs +def test_fsspec_converting_an_outputfile_to_an_inputfile_adlfs(adlfs_fsspec_fileio: FsspecFileIO) -> None: + """Test converting an output file to an input file""" + filename = str(uuid.uuid4()) + output_file = adlfs_fsspec_fileio.new_output(location=f"abfss://tests/{filename}") + input_file = output_file.to_input_file() + assert input_file.location == output_file.location + + +@pytest.mark.adlfs +def test_writing_avro_file_adlfs(generated_manifest_entry_file: str, adlfs_fsspec_fileio: FsspecFileIO) -> None: + """Test that bytes match when reading a local avro file, writing it using fsspec file-io, and then reading it again""" + filename = str(uuid.uuid4()) + with PyArrowFileIO().new_input(location=generated_manifest_entry_file).open() as f: + b1 = f.read() + with adlfs_fsspec_fileio.new_output(location=f"abfss://tests/{filename}").create() as out_f: + out_f.write(b1) + with adlfs_fsspec_fileio.new_input(location=f"abfss://tests/{filename}").open() as in_f: + b2 = in_f.read() + assert b1 == b2 # Check that bytes of read from local avro file match bytes written to adlfs + + adlfs_fsspec_fileio.delete(f"abfss://tests/{filename}") + TEST_URI = "https://iceberg-test-signer" From 54e446d7c2bd0886e2ebaff49360be0dced3d886 Mon Sep 17 00:00:00 2001 From: Ruben van de Geer <37443208+rubenvdg@users.noreply.github.com> Date: Tue, 20 Dec 2022 20:04:30 +0100 Subject: [PATCH 309/642] Python: Check for NaNs when creating literal (#6462) * Check for nans when creating literal * Implement Fokko's suggestions --- pyiceberg/expressions/literals.py | 7 +++++-- tests/expressions/test_literals.py | 15 ++++++++++++++- 2 files changed, 19 insertions(+), 3 deletions(-) diff --git a/pyiceberg/expressions/literals.py b/pyiceberg/expressions/literals.py index 4832da591f..cd53ab6d41 100644 --- a/pyiceberg/expressions/literals.py +++ b/pyiceberg/expressions/literals.py @@ -25,6 +25,7 @@ from abc import ABC, abstractmethod from decimal import ROUND_HALF_UP, Decimal from functools import singledispatchmethod +from math import isnan from typing import Any, Generic, Type from uuid import UUID @@ -65,6 +66,8 @@ class Literal(Generic[L], ABC): def __init__(self, value: L, value_type: Type[L]): if value is None or not isinstance(value, value_type): raise TypeError(f"Invalid literal value: {value!r} (not a {value_type})") + if isinstance(value, float) and isnan(value): + raise ValueError("Cannot create expression literal from NaN.") self._value = value @property @@ -108,10 +111,10 @@ def __ge__(self, other: Any) -> bool: def literal(value: L) -> Literal[L]: """ - A generic Literal factory to construct an iceberg Literal based on python primitive data type + A generic Literal factory to construct an Iceberg Literal based on Python primitive data type Args: - value(python primitive type): the value to be associated with literal + value(Python primitive type): the value to be associated with literal Example: from pyiceberg.expressions.literals import literal diff --git a/tests/expressions/test_literals.py b/tests/expressions/test_literals.py index 453f04f4ee..7373ff040b 100644 --- a/tests/expressions/test_literals.py +++ b/tests/expressions/test_literals.py @@ -73,6 +73,12 @@ def test_literal_from_none_error() -> None: assert "Invalid literal value: None" in str(e.value) +def test_literal_from_nan_error() -> None: + with pytest.raises(ValueError) as e: + literal(float("nan")) + assert "Cannot create expression literal from NaN." in str(e.value) + + @pytest.mark.parametrize( "literal_class", [ @@ -89,12 +95,19 @@ def test_literal_from_none_error() -> None: BinaryLiteral, ], ) -def test_string_literal_with_none_value_error(literal_class: Type[PrimitiveType]) -> None: +def test_literal_classes_with_none_type_error(literal_class: Type[PrimitiveType]) -> None: with pytest.raises(TypeError) as e: literal_class(None) assert "Invalid literal value: None" in str(e.value) +@pytest.mark.parametrize("literal_class", [FloatLiteral, DoubleLiteral]) +def test_literal_classes_with_nan_value_error(literal_class: Type[PrimitiveType]) -> None: + with pytest.raises(ValueError) as e: + literal_class(float("nan")) + assert "Cannot create expression literal from NaN." in str(e.value) + + # Numeric From 63155ce2350f94458aca1151fae2d5146365e701 Mon Sep 17 00:00:00 2001 From: Fokko Driesprong Date: Wed, 21 Dec 2022 23:29:45 +0100 Subject: [PATCH 310/642] Python: Convert UUID to binary(16) in PyArrow (#6468) --- pyiceberg/io/pyarrow.py | 16 +--------------- 1 file changed, 1 insertion(+), 15 deletions(-) diff --git a/pyiceberg/io/pyarrow.py b/pyiceberg/io/pyarrow.py index f3adfd0f6c..d5f7e80f62 100644 --- a/pyiceberg/io/pyarrow.py +++ b/pyiceberg/io/pyarrow.py @@ -291,20 +291,6 @@ def schema_to_pyarrow(schema: Schema) -> pa.schema: return visit(schema, _ConvertToArrowSchema()) -class UuidType(pa.PyExtensionType): - """Custom type for UUID - - For more information: - https://arrow.apache.org/docs/python/extending_types.html#defining-extension-types-user-defined-types - """ - - def __init__(self) -> None: - pa.PyExtensionType.__init__(self, pa.binary(16)) - - def __reduce__(self) -> Tuple[pa.PyExtensionType, Tuple[Any, ...]]: - return UuidType, () - - class _ConvertToArrowSchema(SchemaVisitorPerPrimitiveType[pa.DataType], Singleton): def schema(self, _: Schema, struct_result: pa.StructType) -> pa.schema: return pa.schema(list(struct_result)) @@ -366,7 +352,7 @@ def visit_string(self, _: StringType) -> pa.DataType: return pa.string() def visit_uuid(self, _: UUIDType) -> pa.DataType: - return UuidType() + return pa.binary(16) def visit_binary(self, _: BinaryType) -> pa.DataType: return pa.binary() From 2fd9285e3c5f0ed623f05b046fcdd6408a162c6d Mon Sep 17 00:00:00 2001 From: Fokko Driesprong Date: Thu, 22 Dec 2022 00:47:05 +0100 Subject: [PATCH 311/642] Python: Read a date as an int (#6478) --- pyiceberg/avro/decoder.py | 17 +- pyiceberg/avro/reader.py | 6 +- pyiceberg/typedef.py | 3 + tests/avro/test_decoder.py | 8 +- tests/avro/test_reader.py | 12 +- tests/conftest.py | 21 +- tests/utils/test_manifest.py | 710 +++++++++++++++++++++++------------ 7 files changed, 516 insertions(+), 261 deletions(-) diff --git a/pyiceberg/avro/decoder.py b/pyiceberg/avro/decoder.py index cef9de7c7b..663a193ce8 100644 --- a/pyiceberg/avro/decoder.py +++ b/pyiceberg/avro/decoder.py @@ -16,17 +16,12 @@ # under the License. import decimal import struct -from datetime import date, datetime, time +from datetime import datetime, time from io import SEEK_CUR from uuid import UUID from pyiceberg.io import InputStream -from pyiceberg.utils.datetime import ( - days_to_date, - micros_to_time, - micros_to_timestamp, - micros_to_timestamptz, -) +from pyiceberg.utils.datetime import micros_to_time, micros_to_timestamp, micros_to_timestamptz from pyiceberg.utils.decimal import unscaled_to_decimal STRUCT_FLOAT = struct.Struct(" str: """ return self.read_bytes().decode("utf-8") - def read_date_from_int(self) -> date: - """ - int is decoded as python date object. - int stores the number of days from - the unix epoch, 1 January 1970 (ISO calendar). - """ - return days_to_date(self.read_int()) - def read_uuid_from_fixed(self) -> UUID: """Reads a UUID as a fixed[16]""" return UUID(bytes=self.read(16)) diff --git a/pyiceberg/avro/reader.py b/pyiceberg/avro/reader.py index 136498ed34..d1b114cc5b 100644 --- a/pyiceberg/avro/reader.py +++ b/pyiceberg/avro/reader.py @@ -28,7 +28,7 @@ from abc import abstractmethod from dataclasses import dataclass from dataclasses import field as dataclassfield -from datetime import date, datetime, time +from datetime import datetime, time from decimal import Decimal from typing import ( Any, @@ -156,8 +156,8 @@ def skip(self, decoder: BinaryDecoder) -> None: class DateReader(Reader): - def read(self, decoder: BinaryDecoder) -> date: - return decoder.read_date_from_int() + def read(self, decoder: BinaryDecoder) -> int: + return decoder.read_int() def skip(self, decoder: BinaryDecoder) -> None: decoder.skip_int() diff --git a/pyiceberg/typedef.py b/pyiceberg/typedef.py index 6ad668f006..ccdeacad30 100644 --- a/pyiceberg/typedef.py +++ b/pyiceberg/typedef.py @@ -98,3 +98,6 @@ def get(self, pos: int) -> Any: def __eq__(self, other: Any) -> bool: # For testing return True if isinstance(other, Record) and other._data == self._data else False + + def __repr__(self) -> str: + return "[" + ", ".join([repr(e) for e in self._data]) + "]" diff --git a/tests/avro/test_decoder.py b/tests/avro/test_decoder.py index ac38f38868..e723270376 100644 --- a/tests/avro/test_decoder.py +++ b/tests/avro/test_decoder.py @@ -16,7 +16,7 @@ # under the License. from __future__ import annotations -from datetime import date, datetime, timezone +from datetime import datetime, timezone from decimal import Decimal from io import SEEK_SET from types import TracebackType @@ -171,12 +171,6 @@ def test_skip_double() -> None: assert mis.tell() == 8 -def test_read_date() -> None: - mis = MemoryInputStream(b"\xBC\x7D") - decoder = BinaryDecoder(mis) - assert decoder.read_date_from_int() == date(1991, 12, 27) - - def test_read_uuid_from_fixed() -> None: mis = MemoryInputStream(b"\x12\x34\x56\x78" * 4) decoder = BinaryDecoder(mis) diff --git a/tests/avro/test_reader.py b/tests/avro/test_reader.py index 2528882333..2d54b6e887 100644 --- a/tests/avro/test_reader.py +++ b/tests/avro/test_reader.py @@ -94,7 +94,15 @@ def test_read_header(generated_manifest_entry_file: str, iceberg_manifest_entry_ "type": { "type": "record", "name": "r102", - "fields": [{"field-id": 1000, "default": None, "name": "VendorID", "type": ["null", "int"]}], + "fields": [ + {"field-id": 1000, "default": None, "name": "VendorID", "type": ["null", "int"]}, + { + "field-id": 1001, + "default": None, + "name": "tpep_pickup_datetime", + "type": ["null", {"type": "int", "logicalType": "date"}], + }, + ], }, }, {"field-id": 103, "doc": "Number of records in the file", "name": "record_count", "type": "long"}, @@ -268,7 +276,7 @@ def test_read_manifest_entry_file(generated_manifest_entry_file: str) -> None: Record( "/home/iceberg/warehouse/nyc/taxis_partitioned/data/VendorID=null/00000-633-d8a4223e-dc97-45a1-86e1-adaba6e8abd7-00001.parquet", "PARQUET", - Record(None), + Record(1, 1925), 19513, 388872, 67108864, diff --git a/tests/conftest.py b/tests/conftest.py index c656ebf4d0..888888da9f 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -54,6 +54,7 @@ from pyiceberg.types import ( BinaryType, BooleanType, + DateType, DoubleType, FloatType, IntegerType, @@ -321,7 +322,7 @@ def catalog() -> InMemoryCatalog: "data_file": { "file_path": "/home/iceberg/warehouse/nyc/taxis_partitioned/data/VendorID=null/00000-633-d8a4223e-dc97-45a1-86e1-adaba6e8abd7-00001.parquet", "file_format": "PARQUET", - "partition": {"VendorID": None}, + "partition": {"VendorID": 1, "tpep_pickup_datetime": 1925}, "record_count": 19513, "file_size_in_bytes": 388872, "block_size_in_bytes": 67108864, @@ -441,7 +442,7 @@ def catalog() -> InMemoryCatalog: "data_file": { "file_path": "/home/iceberg/warehouse/nyc/taxis_partitioned/data/VendorID=1/00000-633-d8a4223e-dc97-45a1-86e1-adaba6e8abd7-00002.parquet", "file_format": "PARQUET", - "partition": {"VendorID": 1}, + "partition": {"VendorID": 1, "tpep_pickup_datetime": 1925}, "record_count": 95050, "file_size_in_bytes": 1265950, "block_size_in_bytes": 67108864, @@ -714,7 +715,15 @@ def avro_schema_manifest_entry() -> Dict[str, Any]: "type": { "type": "record", "name": "r102", - "fields": [{"name": "VendorID", "type": ["null", "int"], "default": None, "field-id": 1000}], + "fields": [ + {"field-id": 1000, "default": None, "name": "VendorID", "type": ["null", "int"]}, + { + "field-id": 1001, + "default": None, + "name": "tpep_pickup_datetime", + "type": ["null", {"type": "int", "logicalType": "date"}], + }, + ], }, "field-id": 102, }, @@ -987,6 +996,12 @@ def iceberg_manifest_entry_schema() -> Schema: field_type=IntegerType(), required=False, ), + NestedField( + field_id=1001, + name="tpep_pickup_datetime", + field_type=DateType(), + required=False, + ), ), required=True, ), diff --git a/tests/utils/test_manifest.py b/tests/utils/test_manifest.py index 91c111abc1..e78d0db237 100644 --- a/tests/utils/test_manifest.py +++ b/tests/utils/test_manifest.py @@ -35,236 +35,485 @@ def test_read_manifest_entry(generated_manifest_entry_file: str) -> None: input_file = PyArrowFileIO().new_input(location=generated_manifest_entry_file) - assert list(read_manifest_entry(input_file)) == [ - ManifestEntry( - status=1, - snapshot_id=8744736658442914487, - data_file=DataFile( - file_path="/home/iceberg/warehouse/nyc/taxis_partitioned/data/VendorID=null/00000-633-d8a4223e-dc97-45a1-86e1-adaba6e8abd7-00001.parquet", - file_format=FileFormat.PARQUET, - partition={"VendorID": None}, - record_count=19513, - file_size_in_bytes=388872, - block_size_in_bytes=67108864, - column_sizes={ - 1: 53, - 2: 98153, - 3: 98693, - 4: 53, - 5: 53, - 6: 53, - 7: 17425, - 8: 18528, - 9: 53, - 10: 44788, - 11: 35571, - 12: 53, - 13: 1243, - 14: 2355, - 15: 12750, - 16: 4029, - 17: 110, - 18: 47194, - 19: 2948, - }, - value_counts={ - 1: 19513, - 2: 19513, - 3: 19513, - 4: 19513, - 5: 19513, - 6: 19513, - 7: 19513, - 8: 19513, - 9: 19513, - 10: 19513, - 11: 19513, - 12: 19513, - 13: 19513, - 14: 19513, - 15: 19513, - 16: 19513, - 17: 19513, - 18: 19513, - 19: 19513, - }, - null_value_counts={ - 1: 19513, - 2: 0, - 3: 0, - 4: 19513, - 5: 19513, - 6: 19513, - 7: 0, - 8: 0, - 9: 19513, - 10: 0, - 11: 0, - 12: 19513, - 13: 0, - 14: 0, - 15: 0, - 16: 0, - 17: 0, - 18: 0, - 19: 0, - }, - nan_value_counts={16: 0, 17: 0, 18: 0, 19: 0, 10: 0, 11: 0, 12: 0, 13: 0, 14: 0, 15: 0}, - lower_bounds={ - 2: b"2020-04-01 00:00", - 3: b"2020-04-01 00:12", - 7: b"\x03\x00\x00\x00", - 8: b"\x01\x00\x00\x00", - 10: b"\xf6(\\\x8f\xc2\x05S\xc0", - 11: b"\x00\x00\x00\x00\x00\x00\x00\x00", - 13: b"\x00\x00\x00\x00\x00\x00\x00\x00", - 14: b"\x00\x00\x00\x00\x00\x00\xe0\xbf", - 15: b")\\\x8f\xc2\xf5(\x08\xc0", - 16: b"\x00\x00\x00\x00\x00\x00\x00\x00", - 17: b"\x00\x00\x00\x00\x00\x00\x00\x00", - 18: b"\xf6(\\\x8f\xc2\xc5S\xc0", - 19: b"\x00\x00\x00\x00\x00\x00\x04\xc0", - }, - upper_bounds={ - 2: b"2020-04-30 23:5:", - 3: b"2020-05-01 00:41", - 7: b"\t\x01\x00\x00", - 8: b"\t\x01\x00\x00", - 10: b"\xcd\xcc\xcc\xcc\xcc,_@", - 11: b"\x1f\x85\xebQ\\\xe2\xfe@", - 13: b"\x00\x00\x00\x00\x00\x00\x12@", - 14: b"\x00\x00\x00\x00\x00\x00\xe0?", - 15: b"q=\n\xd7\xa3\xf01@", - 16: b"\x00\x00\x00\x00\x00`B@", - 17: b"333333\xd3?", - 18: b"\x00\x00\x00\x00\x00\x18b@", - 19: b"\x00\x00\x00\x00\x00\x00\x04@", - }, - key_metadata=None, - split_offsets=[4], - sort_order_id=0, + assert ( + list(read_manifest_entry(input_file)) + == [ + ManifestEntry( + status=ManifestEntryStatus.ADDED, + snapshot_id=8744736658442914487, + sequence_number=None, + data_file=DataFile( + content=DataFileContent.DATA, + file_path="/home/iceberg/warehouse/nyc/taxis_partitioned/data/VendorID=null/00000-633-d8a4223e-dc97-45a1-86e1-adaba6e8abd7-00001.parquet", + file_format=FileFormat.PARQUET, + partition={"VendorID": 1, "tpep_pickup_datetime": 1925}, + record_count=19513, + file_size_in_bytes=388872, + block_size_in_bytes=67108864, + column_sizes={ + 1: 53, + 2: 98153, + 3: 98693, + 4: 53, + 5: 53, + 6: 53, + 7: 17425, + 8: 18528, + 9: 53, + 10: 44788, + 11: 35571, + 12: 53, + 13: 1243, + 14: 2355, + 15: 12750, + 16: 4029, + 17: 110, + 18: 47194, + 19: 2948, + }, + value_counts={ + 1: 19513, + 2: 19513, + 3: 19513, + 4: 19513, + 5: 19513, + 6: 19513, + 7: 19513, + 8: 19513, + 9: 19513, + 10: 19513, + 11: 19513, + 12: 19513, + 13: 19513, + 14: 19513, + 15: 19513, + 16: 19513, + 17: 19513, + 18: 19513, + 19: 19513, + }, + null_value_counts={ + 1: 19513, + 2: 0, + 3: 0, + 4: 19513, + 5: 19513, + 6: 19513, + 7: 0, + 8: 0, + 9: 19513, + 10: 0, + 11: 0, + 12: 19513, + 13: 0, + 14: 0, + 15: 0, + 16: 0, + 17: 0, + 18: 0, + 19: 0, + }, + nan_value_counts={16: 0, 17: 0, 18: 0, 19: 0, 10: 0, 11: 0, 12: 0, 13: 0, 14: 0, 15: 0}, + distinct_counts=None, + lower_bounds={ + 2: b"2020-04-01 00:00", + 3: b"2020-04-01 00:12", + 7: b"\x03\x00\x00\x00", + 8: b"\x01\x00\x00\x00", + 10: b"\xf6(\\\x8f\xc2\x05S\xc0", + 11: b"\x00\x00\x00\x00\x00\x00\x00\x00", + 13: b"\x00\x00\x00\x00\x00\x00\x00\x00", + 14: b"\x00\x00\x00\x00\x00\x00\xe0\xbf", + 15: b")\\\x8f\xc2\xf5(\x08\xc0", + 16: b"\x00\x00\x00\x00\x00\x00\x00\x00", + 17: b"\x00\x00\x00\x00\x00\x00\x00\x00", + 18: b"\xf6(\\\x8f\xc2\xc5S\xc0", + 19: b"\x00\x00\x00\x00\x00\x00\x04\xc0", + }, + upper_bounds={ + 2: b"2020-04-30 23:5:", + 3: b"2020-05-01 00:41", + 7: b"\t\x01\x00\x00", + 8: b"\t\x01\x00\x00", + 10: b"\xcd\xcc\xcc\xcc\xcc,_@", + 11: b"\x1f\x85\xebQ\\\xe2\xfe@", + 13: b"\x00\x00\x00\x00\x00\x00\x12@", + 14: b"\x00\x00\x00\x00\x00\x00\xe0?", + 15: b"q=\n\xd7\xa3\xf01@", + 16: b"\x00\x00\x00\x00\x00`B@", + 17: b"333333\xd3?", + 18: b"\x00\x00\x00\x00\x00\x18b@", + 19: b"\x00\x00\x00\x00\x00\x00\x04@", + }, + key_metadata=None, + split_offsets=[4], + equality_ids=None, + sort_order_id=0, + ), ), - ), - ManifestEntry( - status=1, - snapshot_id=8744736658442914487, - data_file=DataFile( - file_path="/home/iceberg/warehouse/nyc/taxis_partitioned/data/VendorID=1/00000-633-d8a4223e-dc97-45a1-86e1-adaba6e8abd7-00002.parquet", - file_format=FileFormat.PARQUET, - partition={"VendorID": 1}, - record_count=95050, - file_size_in_bytes=1265950, - block_size_in_bytes=67108864, - column_sizes={ - 1: 318, - 2: 329806, - 3: 331632, - 4: 15343, - 5: 2351, - 6: 3389, - 7: 71269, - 8: 76429, - 9: 16383, - 10: 86992, - 11: 89608, - 12: 265, - 13: 19377, - 14: 1692, - 15: 76162, - 16: 4354, - 17: 759, - 18: 120650, - 19: 11804, - }, - value_counts={ - 1: 95050, - 2: 95050, - 3: 95050, - 4: 95050, - 5: 95050, - 6: 95050, - 7: 95050, - 8: 95050, - 9: 95050, - 10: 95050, - 11: 95050, - 12: 95050, - 13: 95050, - 14: 95050, - 15: 95050, - 16: 95050, - 17: 95050, - 18: 95050, - 19: 95050, - }, - null_value_counts={ - 1: 0, - 2: 0, - 3: 0, - 4: 0, - 5: 0, - 6: 0, - 7: 0, - 8: 0, - 9: 0, - 10: 0, - 11: 0, - 12: 95050, - 13: 0, - 14: 0, - 15: 0, - 16: 0, - 17: 0, - 18: 0, - 19: 0, - }, - nan_value_counts={16: 0, 17: 0, 18: 0, 19: 0, 10: 0, 11: 0, 12: 0, 13: 0, 14: 0, 15: 0}, - lower_bounds={ - 1: b"\x01\x00\x00\x00", - 2: b"2020-04-01 00:00", - 3: b"2020-04-01 00:03", - 4: b"\x00\x00\x00\x00", - 5: b"\x01\x00\x00\x00", - 6: b"N", - 7: b"\x01\x00\x00\x00", - 8: b"\x01\x00\x00\x00", - 9: b"\x01\x00\x00\x00", - 10: b"\x00\x00\x00\x00\x00\x00\x00\x00", - 11: b"\x00\x00\x00\x00\x00\x00\x00\x00", - 13: b"\x00\x00\x00\x00\x00\x00\x00\x00", - 14: b"\x00\x00\x00\x00\x00\x00\x00\x00", - 15: b"\x00\x00\x00\x00\x00\x00\x00\x00", - 16: b"\x00\x00\x00\x00\x00\x00\x00\x00", - 17: b"\x00\x00\x00\x00\x00\x00\x00\x00", - 18: b"\x00\x00\x00\x00\x00\x00\x00\x00", - 19: b"\x00\x00\x00\x00\x00\x00\x00\x00", - }, - upper_bounds={ - 1: b"\x01\x00\x00\x00", - 2: b"2020-04-30 23:5:", - 3: b"2020-05-01 00:1:", - 4: b"\x06\x00\x00\x00", - 5: b"c\x00\x00\x00", - 6: b"Y", - 7: b"\t\x01\x00\x00", - 8: b"\t\x01\x00\x00", - 9: b"\x04\x00\x00\x00", - 10: b"\\\x8f\xc2\xf5(8\x8c@", - 11: b"\xcd\xcc\xcc\xcc\xcc,f@", - 13: b"\x00\x00\x00\x00\x00\x00\x1c@", - 14: b"\x9a\x99\x99\x99\x99\x99\xf1?", - 15: b"\x00\x00\x00\x00\x00\x00Y@", - 16: b"\x00\x00\x00\x00\x00\xb0X@", - 17: b"333333\xd3?", - 18: b"\xc3\xf5(\\\x8f:\x8c@", - 19: b"\x00\x00\x00\x00\x00\x00\x04@", - }, - key_metadata=None, - split_offsets=[4], - sort_order_id=0, + ManifestEntry( + status=ManifestEntryStatus.ADDED, + snapshot_id=8744736658442914487, + sequence_number=None, + data_file=DataFile( + content=DataFileContent.DATA, + file_path="/home/iceberg/warehouse/nyc/taxis_partitioned/data/VendorID=1/00000-633-d8a4223e-dc97-45a1-86e1-adaba6e8abd7-00002.parquet", + file_format=FileFormat.PARQUET, + partition={"VendorID": 1, "tpep_pickup_datetime": 1925}, + record_count=95050, + file_size_in_bytes=1265950, + block_size_in_bytes=67108864, + column_sizes={ + 1: 318, + 2: 329806, + 3: 331632, + 4: 15343, + 5: 2351, + 6: 3389, + 7: 71269, + 8: 76429, + 9: 16383, + 10: 86992, + 11: 89608, + 12: 265, + 13: 19377, + 14: 1692, + 15: 76162, + 16: 4354, + 17: 759, + 18: 120650, + 19: 11804, + }, + value_counts={ + 1: 95050, + 2: 95050, + 3: 95050, + 4: 95050, + 5: 95050, + 6: 95050, + 7: 95050, + 8: 95050, + 9: 95050, + 10: 95050, + 11: 95050, + 12: 95050, + 13: 95050, + 14: 95050, + 15: 95050, + 16: 95050, + 17: 95050, + 18: 95050, + 19: 95050, + }, + null_value_counts={ + 1: 0, + 2: 0, + 3: 0, + 4: 0, + 5: 0, + 6: 0, + 7: 0, + 8: 0, + 9: 0, + 10: 0, + 11: 0, + 12: 95050, + 13: 0, + 14: 0, + 15: 0, + 16: 0, + 17: 0, + 18: 0, + 19: 0, + }, + nan_value_counts={16: 0, 17: 0, 18: 0, 19: 0, 10: 0, 11: 0, 12: 0, 13: 0, 14: 0, 15: 0}, + distinct_counts=None, + lower_bounds={ + 1: b"\x01\x00\x00\x00", + 2: b"2020-04-01 00:00", + 3: b"2020-04-01 00:03", + 4: b"\x00\x00\x00\x00", + 5: b"\x01\x00\x00\x00", + 6: b"N", + 7: b"\x01\x00\x00\x00", + 8: b"\x01\x00\x00\x00", + 9: b"\x01\x00\x00\x00", + 10: b"\x00\x00\x00\x00\x00\x00\x00\x00", + 11: b"\x00\x00\x00\x00\x00\x00\x00\x00", + 13: b"\x00\x00\x00\x00\x00\x00\x00\x00", + 14: b"\x00\x00\x00\x00\x00\x00\x00\x00", + 15: b"\x00\x00\x00\x00\x00\x00\x00\x00", + 16: b"\x00\x00\x00\x00\x00\x00\x00\x00", + 17: b"\x00\x00\x00\x00\x00\x00\x00\x00", + 18: b"\x00\x00\x00\x00\x00\x00\x00\x00", + 19: b"\x00\x00\x00\x00\x00\x00\x00\x00", + }, + upper_bounds={ + 1: b"\x01\x00\x00\x00", + 2: b"2020-04-30 23:5:", + 3: b"2020-05-01 00:1:", + 4: b"\x06\x00\x00\x00", + 5: b"c\x00\x00\x00", + 6: b"Y", + 7: b"\t\x01\x00\x00", + 8: b"\t\x01\x00\x00", + 9: b"\x04\x00\x00\x00", + 10: b"\\\x8f\xc2\xf5(8\x8c@", + 11: b"\xcd\xcc\xcc\xcc\xcc,f@", + 13: b"\x00\x00\x00\x00\x00\x00\x1c@", + 14: b"\x9a\x99\x99\x99\x99\x99\xf1?", + 15: b"\x00\x00\x00\x00\x00\x00Y@", + 16: b"\x00\x00\x00\x00\x00\xb0X@", + 17: b"333333\xd3?", + 18: b"\xc3\xf5(\\\x8f:\x8c@", + 19: b"\x00\x00\x00\x00\x00\x00\x04@", + }, + key_metadata=None, + split_offsets=[4], + equality_ids=None, + sort_order_id=0, + ), ), - ), - ] + ] + != [ + ManifestEntry( + status=ManifestEntryStatus.ADDED, + snapshot_id=8744736658442914487, + sequence_number=None, + data_file=DataFile( + content=DataFileContent.DATA, + file_path="/home/iceberg/warehouse/nyc/taxis_partitioned/data/VendorID=null/00000-633-d8a4223e-dc97-45a1-86e1-adaba6e8abd7-00001.parquet", + file_format=FileFormat.PARQUET, + partition={"VendorID": 1, "tpep_pickup_datetime": None}, + record_count=19513, + file_size_in_bytes=388872, + block_size_in_bytes=67108864, + column_sizes={ + 1: 53, + 2: 98153, + 3: 98693, + 4: 53, + 5: 53, + 6: 53, + 7: 17425, + 8: 18528, + 9: 53, + 10: 44788, + 11: 35571, + 12: 53, + 13: 1243, + 14: 2355, + 15: 12750, + 16: 4029, + 17: 110, + 18: 47194, + 19: 2948, + }, + value_counts={ + 1: 19513, + 2: 19513, + 3: 19513, + 4: 19513, + 5: 19513, + 6: 19513, + 7: 19513, + 8: 19513, + 9: 19513, + 10: 19513, + 11: 19513, + 12: 19513, + 13: 19513, + 14: 19513, + 15: 19513, + 16: 19513, + 17: 19513, + 18: 19513, + 19: 19513, + }, + null_value_counts={ + 1: 19513, + 2: 0, + 3: 0, + 4: 19513, + 5: 19513, + 6: 19513, + 7: 0, + 8: 0, + 9: 19513, + 10: 0, + 11: 0, + 12: 19513, + 13: 0, + 14: 0, + 15: 0, + 16: 0, + 17: 0, + 18: 0, + 19: 0, + }, + nan_value_counts={16: 0, 17: 0, 18: 0, 19: 0, 10: 0, 11: 0, 12: 0, 13: 0, 14: 0, 15: 0}, + distinct_counts=None, + lower_bounds={ + 2: b"2020-04-01 00:00", + 3: b"2020-04-01 00:12", + 7: b"\x03\x00\x00\x00", + 8: b"\x01\x00\x00\x00", + 10: b"\xf6(\\\x8f\xc2\x05S\xc0", + 11: b"\x00\x00\x00\x00\x00\x00\x00\x00", + 13: b"\x00\x00\x00\x00\x00\x00\x00\x00", + 14: b"\x00\x00\x00\x00\x00\x00\xe0\xbf", + 15: b")\\\x8f\xc2\xf5(\x08\xc0", + 16: b"\x00\x00\x00\x00\x00\x00\x00\x00", + 17: b"\x00\x00\x00\x00\x00\x00\x00\x00", + 18: b"\xf6(\\\x8f\xc2\xc5S\xc0", + 19: b"\x00\x00\x00\x00\x00\x00\x04\xc0", + }, + upper_bounds={ + 2: b"2020-04-30 23:5:", + 3: b"2020-05-01 00:41", + 7: b"\t\x01\x00\x00", + 8: b"\t\x01\x00\x00", + 10: b"\xcd\xcc\xcc\xcc\xcc,_@", + 11: b"\x1f\x85\xebQ\\\xe2\xfe@", + 13: b"\x00\x00\x00\x00\x00\x00\x12@", + 14: b"\x00\x00\x00\x00\x00\x00\xe0?", + 15: b"q=\n\xd7\xa3\xf01@", + 16: b"\x00\x00\x00\x00\x00`B@", + 17: b"333333\xd3?", + 18: b"\x00\x00\x00\x00\x00\x18b@", + 19: b"\x00\x00\x00\x00\x00\x00\x04@", + }, + key_metadata=None, + split_offsets=[4], + equality_ids=None, + sort_order_id=0, + ), + ), + ManifestEntry( + status=ManifestEntryStatus.ADDED, + snapshot_id=8744736658442914487, + sequence_number=None, + data_file=DataFile( + content=DataFileContent.DATA, + file_path="/home/iceberg/warehouse/nyc/taxis_partitioned/data/VendorID=1/00000-633-d8a4223e-dc97-45a1-86e1-adaba6e8abd7-00002.parquet", + file_format=FileFormat.PARQUET, + partition={"VendorID": 1, "tpep_pickup_datetime": 1925}, + record_count=95050, + file_size_in_bytes=1265950, + block_size_in_bytes=67108864, + column_sizes={ + 1: 318, + 2: 329806, + 3: 331632, + 4: 15343, + 5: 2351, + 6: 3389, + 7: 71269, + 8: 76429, + 9: 16383, + 10: 86992, + 11: 89608, + 12: 265, + 13: 19377, + 14: 1692, + 15: 76162, + 16: 4354, + 17: 759, + 18: 120650, + 19: 11804, + }, + value_counts={ + 1: 95050, + 2: 95050, + 3: 95050, + 4: 95050, + 5: 95050, + 6: 95050, + 7: 95050, + 8: 95050, + 9: 95050, + 10: 95050, + 11: 95050, + 12: 95050, + 13: 95050, + 14: 95050, + 15: 95050, + 16: 95050, + 17: 95050, + 18: 95050, + 19: 95050, + }, + null_value_counts={ + 1: 0, + 2: 0, + 3: 0, + 4: 0, + 5: 0, + 6: 0, + 7: 0, + 8: 0, + 9: 0, + 10: 0, + 11: 0, + 12: 95050, + 13: 0, + 14: 0, + 15: 0, + 16: 0, + 17: 0, + 18: 0, + 19: 0, + }, + nan_value_counts={16: 0, 17: 0, 18: 0, 19: 0, 10: 0, 11: 0, 12: 0, 13: 0, 14: 0, 15: 0}, + distinct_counts=None, + lower_bounds={ + 1: b"\x01\x00\x00\x00", + 2: b"2020-04-01 00:00", + 3: b"2020-04-01 00:03", + 4: b"\x00\x00\x00\x00", + 5: b"\x01\x00\x00\x00", + 6: b"N", + 7: b"\x01\x00\x00\x00", + 8: b"\x01\x00\x00\x00", + 9: b"\x01\x00\x00\x00", + 10: b"\x00\x00\x00\x00\x00\x00\x00\x00", + 11: b"\x00\x00\x00\x00\x00\x00\x00\x00", + 13: b"\x00\x00\x00\x00\x00\x00\x00\x00", + 14: b"\x00\x00\x00\x00\x00\x00\x00\x00", + 15: b"\x00\x00\x00\x00\x00\x00\x00\x00", + 16: b"\x00\x00\x00\x00\x00\x00\x00\x00", + 17: b"\x00\x00\x00\x00\x00\x00\x00\x00", + 18: b"\x00\x00\x00\x00\x00\x00\x00\x00", + 19: b"\x00\x00\x00\x00\x00\x00\x00\x00", + }, + upper_bounds={ + 1: b"\x01\x00\x00\x00", + 2: b"2020-04-30 23:5:", + 3: b"2020-05-01 00:1:", + 4: b"\x06\x00\x00\x00", + 5: b"c\x00\x00\x00", + 6: b"Y", + 7: b"\t\x01\x00\x00", + 8: b"\t\x01\x00\x00", + 9: b"\x04\x00\x00\x00", + 10: b"\\\x8f\xc2\xf5(8\x8c@", + 11: b"\xcd\xcc\xcc\xcc\xcc,f@", + 13: b"\x00\x00\x00\x00\x00\x00\x1c@", + 14: b"\x9a\x99\x99\x99\x99\x99\xf1?", + 15: b"\x00\x00\x00\x00\x00\x00Y@", + 16: b"\x00\x00\x00\x00\x00\xb0X@", + 17: b"333333\xd3?", + 18: b"\xc3\xf5(\\\x8f:\x8c@", + 19: b"\x00\x00\x00\x00\x00\x00\x04@", + }, + key_metadata=None, + split_offsets=[4], + equality_ids=None, + sort_order_id=0, + ), + ), + ] + ) def test_read_manifest_list(generated_manifest_file_file: str) -> None: @@ -328,7 +577,6 @@ def test_read_manifest(generated_manifest_file_file: str, generated_manifest_ent key_metadata=None, ) ] - actual = manifest_list[0].fetch_manifest_entry(io) expected = [ ManifestEntry( @@ -339,7 +587,7 @@ def test_read_manifest(generated_manifest_file_file: str, generated_manifest_ent content=DataFileContent.DATA, file_path="/home/iceberg/warehouse/nyc/taxis_partitioned/data/VendorID=null/00000-633-d8a4223e-dc97-45a1-86e1-adaba6e8abd7-00001.parquet", file_format=FileFormat.PARQUET, - partition={"VendorID": None}, + partition={"VendorID": 1, "tpep_pickup_datetime": 1925}, record_count=19513, file_size_in_bytes=388872, block_size_in_bytes=67108864, @@ -452,7 +700,7 @@ def test_read_manifest(generated_manifest_file_file: str, generated_manifest_ent content=DataFileContent.DATA, file_path="/home/iceberg/warehouse/nyc/taxis_partitioned/data/VendorID=1/00000-633-d8a4223e-dc97-45a1-86e1-adaba6e8abd7-00002.parquet", file_format=FileFormat.PARQUET, - partition={"VendorID": 1}, + partition={"VendorID": 1, "tpep_pickup_datetime": 1925}, record_count=95050, file_size_in_bytes=1265950, block_size_in_bytes=67108864, From 56cc12ce7385e9c5a1228ff8d886b7a5b4d08f08 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 26 Dec 2022 21:07:04 +0100 Subject: [PATCH 312/642] Build: Bump coverage from 6.5.0 to 7.0.1 in /python (#6493) Bumps [coverage](https://github.com/nedbat/coveragepy) from 6.5.0 to 7.0.1. - [Release notes](https://github.com/nedbat/coveragepy/releases) - [Changelog](https://github.com/nedbat/coveragepy/blob/master/CHANGES.rst) - [Commits](https://github.com/nedbat/coveragepy/compare/6.5.0...7.0.1) --- updated-dependencies: - dependency-name: coverage dependency-type: direct:development update-type: version-update:semver-major ... Signed-off-by: dependabot[bot] Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- poetry.lock | 2425 ++++++++++++++++++++++++------------------------ pyproject.toml | 2 +- 2 files changed, 1214 insertions(+), 1213 deletions(-) diff --git a/poetry.lock b/poetry.lock index faf506ee56..8044d06b3a 100644 --- a/poetry.lock +++ b/poetry.lock @@ -1,5 +1,3 @@ -# This file is automatically @generated by Poetry and should not be changed by hand. - [[package]] name = "adal" version = "1.2.7" @@ -7,10 +5,6 @@ description = "Note: This library is already replaced by MSAL Python, available category = "main" optional = true python-versions = "*" -files = [ - {file = "adal-1.2.7-py2.py3-none-any.whl", hash = "sha256:2a7451ed7441ddbc57703042204a3e30ef747478eea022c70f789fc7f084bc3d"}, - {file = "adal-1.2.7.tar.gz", hash = "sha256:d74f45b81317454d96e982fd1c50e6fb5c99ac2223728aea8764433a39f566f1"}, -] [package.dependencies] cryptography = ">=1.1.0" @@ -25,10 +19,6 @@ description = "Access Azure Datalake Gen1 with fsspec and dask" category = "main" optional = true python-versions = ">=3.8" -files = [ - {file = "adlfs-2022.10.0-py3-none-any.whl", hash = "sha256:dc63b248f109bca2000d306f7a51c33d6859e4692dda7c6198b6300f8e29faaf"}, - {file = "adlfs-2022.10.0.tar.gz", hash = "sha256:8542f9e6799aeb77adb5cb9854f76b7dce874610dc0f2ff006cce960526ffad6"}, -] [package.dependencies] aiohttp = ">=3.7.0" @@ -48,10 +38,6 @@ description = "Async client for aws services using botocore and aiohttp" category = "main" optional = true python-versions = ">=3.7" -files = [ - {file = "aiobotocore-2.4.1-py3-none-any.whl", hash = "sha256:26e0d55d5901f487c3467616c028abb85036ca33a0f88e279770adae6b865c69"}, - {file = "aiobotocore-2.4.1.tar.gz", hash = "sha256:5c8ee79d1f14273e3f8f3af5c893db889ef163227b178ecacb5aea4547d3b9a7"}, -] [package.dependencies] aiohttp = ">=3.3.1" @@ -70,95 +56,6 @@ description = "Async http client/server framework (asyncio)" category = "main" optional = true python-versions = ">=3.6" -files = [ - {file = "aiohttp-3.8.3-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:ba71c9b4dcbb16212f334126cc3d8beb6af377f6703d9dc2d9fb3874fd667ee9"}, - {file = "aiohttp-3.8.3-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:d24b8bb40d5c61ef2d9b6a8f4528c2f17f1c5d2d31fed62ec860f6006142e83e"}, - {file = "aiohttp-3.8.3-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:f88df3a83cf9df566f171adba39d5bd52814ac0b94778d2448652fc77f9eb491"}, - {file = "aiohttp-3.8.3-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:b97decbb3372d4b69e4d4c8117f44632551c692bb1361b356a02b97b69e18a62"}, - {file = "aiohttp-3.8.3-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:309aa21c1d54b8ef0723181d430347d7452daaff93e8e2363db8e75c72c2fb2d"}, - {file = "aiohttp-3.8.3-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:ad5383a67514e8e76906a06741febd9126fc7c7ff0f599d6fcce3e82b80d026f"}, - {file = "aiohttp-3.8.3-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:20acae4f268317bb975671e375493dbdbc67cddb5f6c71eebdb85b34444ac46b"}, - {file = "aiohttp-3.8.3-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:05a3c31c6d7cd08c149e50dc7aa2568317f5844acd745621983380597f027a18"}, - {file = "aiohttp-3.8.3-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:d6f76310355e9fae637c3162936e9504b4767d5c52ca268331e2756e54fd4ca5"}, - {file = "aiohttp-3.8.3-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:256deb4b29fe5e47893fa32e1de2d73c3afe7407738bd3c63829874661d4822d"}, - {file = "aiohttp-3.8.3-cp310-cp310-musllinux_1_1_ppc64le.whl", hash = "sha256:5c59fcd80b9049b49acd29bd3598cada4afc8d8d69bd4160cd613246912535d7"}, - {file = "aiohttp-3.8.3-cp310-cp310-musllinux_1_1_s390x.whl", hash = "sha256:059a91e88f2c00fe40aed9031b3606c3f311414f86a90d696dd982e7aec48142"}, - {file = "aiohttp-3.8.3-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:2feebbb6074cdbd1ac276dbd737b40e890a1361b3cc30b74ac2f5e24aab41f7b"}, - {file = "aiohttp-3.8.3-cp310-cp310-win32.whl", hash = "sha256:5bf651afd22d5f0c4be16cf39d0482ea494f5c88f03e75e5fef3a85177fecdeb"}, - {file = "aiohttp-3.8.3-cp310-cp310-win_amd64.whl", hash = "sha256:653acc3880459f82a65e27bd6526e47ddf19e643457d36a2250b85b41a564715"}, - {file = "aiohttp-3.8.3-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:86fc24e58ecb32aee09f864cb11bb91bc4c1086615001647dbfc4dc8c32f4008"}, - {file = "aiohttp-3.8.3-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:75e14eac916f024305db517e00a9252714fce0abcb10ad327fb6dcdc0d060f1d"}, - {file = "aiohttp-3.8.3-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:d1fde0f44029e02d02d3993ad55ce93ead9bb9b15c6b7ccd580f90bd7e3de476"}, - {file = "aiohttp-3.8.3-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:4ab94426ddb1ecc6a0b601d832d5d9d421820989b8caa929114811369673235c"}, - {file = "aiohttp-3.8.3-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:89d2e02167fa95172c017732ed7725bc8523c598757f08d13c5acca308e1a061"}, - {file = "aiohttp-3.8.3-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:02f9a2c72fc95d59b881cf38a4b2be9381b9527f9d328771e90f72ac76f31ad8"}, - {file = "aiohttp-3.8.3-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:9c7149272fb5834fc186328e2c1fa01dda3e1fa940ce18fded6d412e8f2cf76d"}, - {file = "aiohttp-3.8.3-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:512bd5ab136b8dc0ffe3fdf2dfb0c4b4f49c8577f6cae55dca862cd37a4564e2"}, - {file = "aiohttp-3.8.3-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:7018ecc5fe97027214556afbc7c502fbd718d0740e87eb1217b17efd05b3d276"}, - {file = "aiohttp-3.8.3-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:88c70ed9da9963d5496d38320160e8eb7e5f1886f9290475a881db12f351ab5d"}, - {file = "aiohttp-3.8.3-cp311-cp311-musllinux_1_1_ppc64le.whl", hash = "sha256:da22885266bbfb3f78218dc40205fed2671909fbd0720aedba39b4515c038091"}, - {file = "aiohttp-3.8.3-cp311-cp311-musllinux_1_1_s390x.whl", hash = "sha256:e65bc19919c910127c06759a63747ebe14f386cda573d95bcc62b427ca1afc73"}, - {file = "aiohttp-3.8.3-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:08c78317e950e0762c2983f4dd58dc5e6c9ff75c8a0efeae299d363d439c8e34"}, - {file = "aiohttp-3.8.3-cp311-cp311-win32.whl", hash = "sha256:45d88b016c849d74ebc6f2b6e8bc17cabf26e7e40c0661ddd8fae4c00f015697"}, - {file = "aiohttp-3.8.3-cp311-cp311-win_amd64.whl", hash = "sha256:96372fc29471646b9b106ee918c8eeb4cca423fcbf9a34daa1b93767a88a2290"}, - {file = "aiohttp-3.8.3-cp36-cp36m-macosx_10_9_x86_64.whl", hash = "sha256:c971bf3786b5fad82ce5ad570dc6ee420f5b12527157929e830f51c55dc8af77"}, - {file = "aiohttp-3.8.3-cp36-cp36m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:ff25f48fc8e623d95eca0670b8cc1469a83783c924a602e0fbd47363bb54aaca"}, - {file = "aiohttp-3.8.3-cp36-cp36m-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:e381581b37db1db7597b62a2e6b8b57c3deec95d93b6d6407c5b61ddc98aca6d"}, - {file = "aiohttp-3.8.3-cp36-cp36m-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:db19d60d846283ee275d0416e2a23493f4e6b6028825b51290ac05afc87a6f97"}, - {file = "aiohttp-3.8.3-cp36-cp36m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:25892c92bee6d9449ffac82c2fe257f3a6f297792cdb18ad784737d61e7a9a85"}, - {file = "aiohttp-3.8.3-cp36-cp36m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:398701865e7a9565d49189f6c90868efaca21be65c725fc87fc305906be915da"}, - {file = "aiohttp-3.8.3-cp36-cp36m-musllinux_1_1_aarch64.whl", hash = "sha256:4a4fbc769ea9b6bd97f4ad0b430a6807f92f0e5eb020f1e42ece59f3ecfc4585"}, - {file = "aiohttp-3.8.3-cp36-cp36m-musllinux_1_1_i686.whl", hash = "sha256:b29bfd650ed8e148f9c515474a6ef0ba1090b7a8faeee26b74a8ff3b33617502"}, - {file = "aiohttp-3.8.3-cp36-cp36m-musllinux_1_1_ppc64le.whl", hash = "sha256:1e56b9cafcd6531bab5d9b2e890bb4937f4165109fe98e2b98ef0dcfcb06ee9d"}, - {file = "aiohttp-3.8.3-cp36-cp36m-musllinux_1_1_s390x.whl", hash = "sha256:ec40170327d4a404b0d91855d41bfe1fe4b699222b2b93e3d833a27330a87a6d"}, - {file = "aiohttp-3.8.3-cp36-cp36m-musllinux_1_1_x86_64.whl", hash = "sha256:2df5f139233060578d8c2c975128fb231a89ca0a462b35d4b5fcf7c501ebdbe1"}, - {file = "aiohttp-3.8.3-cp36-cp36m-win32.whl", hash = "sha256:f973157ffeab5459eefe7b97a804987876dd0a55570b8fa56b4e1954bf11329b"}, - {file = "aiohttp-3.8.3-cp36-cp36m-win_amd64.whl", hash = "sha256:437399385f2abcd634865705bdc180c8314124b98299d54fe1d4c8990f2f9494"}, - {file = "aiohttp-3.8.3-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:09e28f572b21642128ef31f4e8372adb6888846f32fecb288c8b0457597ba61a"}, - {file = "aiohttp-3.8.3-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:6f3553510abdbec67c043ca85727396ceed1272eef029b050677046d3387be8d"}, - {file = "aiohttp-3.8.3-cp37-cp37m-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:e168a7560b7c61342ae0412997b069753f27ac4862ec7867eff74f0fe4ea2ad9"}, - {file = "aiohttp-3.8.3-cp37-cp37m-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:db4c979b0b3e0fa7e9e69ecd11b2b3174c6963cebadeecfb7ad24532ffcdd11a"}, - {file = "aiohttp-3.8.3-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:e164e0a98e92d06da343d17d4e9c4da4654f4a4588a20d6c73548a29f176abe2"}, - {file = "aiohttp-3.8.3-cp37-cp37m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:e8a78079d9a39ca9ca99a8b0ac2fdc0c4d25fc80c8a8a82e5c8211509c523363"}, - {file = "aiohttp-3.8.3-cp37-cp37m-musllinux_1_1_aarch64.whl", hash = "sha256:21b30885a63c3f4ff5b77a5d6caf008b037cb521a5f33eab445dc566f6d092cc"}, - {file = "aiohttp-3.8.3-cp37-cp37m-musllinux_1_1_i686.whl", hash = "sha256:4b0f30372cef3fdc262f33d06e7b411cd59058ce9174ef159ad938c4a34a89da"}, - {file = "aiohttp-3.8.3-cp37-cp37m-musllinux_1_1_ppc64le.whl", hash = "sha256:8135fa153a20d82ffb64f70a1b5c2738684afa197839b34cc3e3c72fa88d302c"}, - {file = "aiohttp-3.8.3-cp37-cp37m-musllinux_1_1_s390x.whl", hash = "sha256:ad61a9639792fd790523ba072c0555cd6be5a0baf03a49a5dd8cfcf20d56df48"}, - {file = "aiohttp-3.8.3-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:978b046ca728073070e9abc074b6299ebf3501e8dee5e26efacb13cec2b2dea0"}, - {file = "aiohttp-3.8.3-cp37-cp37m-win32.whl", hash = "sha256:0d2c6d8c6872df4a6ec37d2ede71eff62395b9e337b4e18efd2177de883a5033"}, - {file = "aiohttp-3.8.3-cp37-cp37m-win_amd64.whl", hash = "sha256:21d69797eb951f155026651f7e9362877334508d39c2fc37bd04ff55b2007091"}, - {file = "aiohttp-3.8.3-cp38-cp38-macosx_10_9_universal2.whl", hash = "sha256:2ca9af5f8f5812d475c5259393f52d712f6d5f0d7fdad9acdb1107dd9e3cb7eb"}, - {file = "aiohttp-3.8.3-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:1d90043c1882067f1bd26196d5d2db9aa6d268def3293ed5fb317e13c9413ea4"}, - {file = "aiohttp-3.8.3-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:d737fc67b9a970f3234754974531dc9afeea11c70791dcb7db53b0cf81b79784"}, - {file = "aiohttp-3.8.3-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:ebf909ea0a3fc9596e40d55d8000702a85e27fd578ff41a5500f68f20fd32e6c"}, - {file = "aiohttp-3.8.3-cp38-cp38-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:5835f258ca9f7c455493a57ee707b76d2d9634d84d5d7f62e77be984ea80b849"}, - {file = "aiohttp-3.8.3-cp38-cp38-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:da37dcfbf4b7f45d80ee386a5f81122501ec75672f475da34784196690762f4b"}, - {file = "aiohttp-3.8.3-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:87f44875f2804bc0511a69ce44a9595d5944837a62caecc8490bbdb0e18b1342"}, - {file = "aiohttp-3.8.3-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:527b3b87b24844ea7865284aabfab08eb0faf599b385b03c2aa91fc6edd6e4b6"}, - {file = "aiohttp-3.8.3-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:d5ba88df9aa5e2f806650fcbeedbe4f6e8736e92fc0e73b0400538fd25a4dd96"}, - {file = "aiohttp-3.8.3-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:e7b8813be97cab8cb52b1375f41f8e6804f6507fe4660152e8ca5c48f0436017"}, - {file = "aiohttp-3.8.3-cp38-cp38-musllinux_1_1_ppc64le.whl", hash = "sha256:2dea10edfa1a54098703cb7acaa665c07b4e7568472a47f4e64e6319d3821ccf"}, - {file = "aiohttp-3.8.3-cp38-cp38-musllinux_1_1_s390x.whl", hash = "sha256:713d22cd9643ba9025d33c4af43943c7a1eb8547729228de18d3e02e278472b6"}, - {file = "aiohttp-3.8.3-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:2d252771fc85e0cf8da0b823157962d70639e63cb9b578b1dec9868dd1f4f937"}, - {file = "aiohttp-3.8.3-cp38-cp38-win32.whl", hash = "sha256:66bd5f950344fb2b3dbdd421aaa4e84f4411a1a13fca3aeb2bcbe667f80c9f76"}, - {file = "aiohttp-3.8.3-cp38-cp38-win_amd64.whl", hash = "sha256:84b14f36e85295fe69c6b9789b51a0903b774046d5f7df538176516c3e422446"}, - {file = "aiohttp-3.8.3-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:16c121ba0b1ec2b44b73e3a8a171c4f999b33929cd2397124a8c7fcfc8cd9e06"}, - {file = "aiohttp-3.8.3-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:8d6aaa4e7155afaf994d7924eb290abbe81a6905b303d8cb61310a2aba1c68ba"}, - {file = "aiohttp-3.8.3-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:43046a319664a04b146f81b40e1545d4c8ac7b7dd04c47e40bf09f65f2437346"}, - {file = "aiohttp-3.8.3-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:599418aaaf88a6d02a8c515e656f6faf3d10618d3dd95866eb4436520096c84b"}, - {file = "aiohttp-3.8.3-cp39-cp39-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:92a2964319d359f494f16011e23434f6f8ef0434acd3cf154a6b7bec511e2fb7"}, - {file = "aiohttp-3.8.3-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:73a4131962e6d91109bca6536416aa067cf6c4efb871975df734f8d2fd821b37"}, - {file = "aiohttp-3.8.3-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:598adde339d2cf7d67beaccda3f2ce7c57b3b412702f29c946708f69cf8222aa"}, - {file = "aiohttp-3.8.3-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:75880ed07be39beff1881d81e4a907cafb802f306efd6d2d15f2b3c69935f6fb"}, - {file = "aiohttp-3.8.3-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:a0239da9fbafd9ff82fd67c16704a7d1bccf0d107a300e790587ad05547681c8"}, - {file = "aiohttp-3.8.3-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:4e3a23ec214e95c9fe85a58470b660efe6534b83e6cbe38b3ed52b053d7cb6ad"}, - {file = "aiohttp-3.8.3-cp39-cp39-musllinux_1_1_ppc64le.whl", hash = "sha256:47841407cc89a4b80b0c52276f3cc8138bbbfba4b179ee3acbd7d77ae33f7ac4"}, - {file = "aiohttp-3.8.3-cp39-cp39-musllinux_1_1_s390x.whl", hash = "sha256:54d107c89a3ebcd13228278d68f1436d3f33f2dd2af5415e3feaeb1156e1a62c"}, - {file = "aiohttp-3.8.3-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:c37c5cce780349d4d51739ae682dec63573847a2a8dcb44381b174c3d9c8d403"}, - {file = "aiohttp-3.8.3-cp39-cp39-win32.whl", hash = "sha256:f178d2aadf0166be4df834c4953da2d7eef24719e8aec9a65289483eeea9d618"}, - {file = "aiohttp-3.8.3-cp39-cp39-win_amd64.whl", hash = "sha256:88e5be56c231981428f4f506c68b6a46fa25c4123a2e86d156c58a8369d31ab7"}, - {file = "aiohttp-3.8.3.tar.gz", hash = "sha256:3828fb41b7203176b82fe5d699e0d845435f2374750a44b480ea6b930f6be269"}, -] [package.dependencies] aiosignal = ">=1.1.2" @@ -179,10 +76,6 @@ description = "itertools and builtins for AsyncIO and mixed iterables" category = "main" optional = true python-versions = ">=3.6" -files = [ - {file = "aioitertools-0.11.0-py3-none-any.whl", hash = "sha256:04b95e3dab25b449def24d7df809411c10e62aab0cbe31a50ca4e68748c43394"}, - {file = "aioitertools-0.11.0.tar.gz", hash = "sha256:42c68b8dd3a69c2bf7f2233bf7df4bb58b557bca5252ac02ed5187bbc67d6831"}, -] [package.dependencies] typing_extensions = {version = ">=4.0", markers = "python_version < \"3.10\""} @@ -194,10 +87,6 @@ description = "aiosignal: a list of registered asynchronous callbacks" category = "main" optional = true python-versions = ">=3.7" -files = [ - {file = "aiosignal-1.3.1-py3-none-any.whl", hash = "sha256:f8376fb07dd1e86a584e4fcdec80b36b7f81aac666ebc724e2c090300dd83b17"}, - {file = "aiosignal-1.3.1.tar.gz", hash = "sha256:54cd96e15e1649b75d6c87526a6ff0b6c1b0dd3459f43d9ca11d48c339b68cfc"}, -] [package.dependencies] frozenlist = ">=1.1.0" @@ -209,10 +98,6 @@ description = "Timeout context manager for asyncio programs" category = "main" optional = true python-versions = ">=3.6" -files = [ - {file = "async-timeout-4.0.2.tar.gz", hash = "sha256:2163e1640ddb52b7a8c80d0a67a08587e5d245cc9c553a74a847056bc2976b15"}, - {file = "async_timeout-4.0.2-py3-none-any.whl", hash = "sha256:8ca1e4fcf50d07413d66d1a5e416e42cfdf5851c981d679a09851a6853383b3c"}, -] [[package]] name = "attrs" @@ -221,10 +106,6 @@ description = "Classes Without Boilerplate" category = "main" optional = false python-versions = ">=3.5" -files = [ - {file = "attrs-22.1.0-py2.py3-none-any.whl", hash = "sha256:86efa402f67bf2df34f51a335487cf46b1ec130d02b8d39fd248abfd30da551c"}, - {file = "attrs-22.1.0.tar.gz", hash = "sha256:29adc2665447e5191d0e7c568fde78b21f9672d344281d0c6e1ab085429b22b6"}, -] [package.extras] dev = ["cloudpickle", "coverage[toml] (>=5.0.2)", "furo", "hypothesis", "mypy (>=0.900,!=0.940)", "pre-commit", "pympler", "pytest (>=4.3.0)", "pytest-mypy-plugins", "sphinx", "sphinx-notfound-page", "zope.interface"] @@ -239,10 +120,6 @@ description = "Microsoft Azure Core Library for Python" category = "main" optional = true python-versions = ">=3.7" -files = [ - {file = "azure-core-1.26.1.zip", hash = "sha256:223b0e90cbdd1f03c41b195b03239899843f20d00964dbb85e64386873414a2d"}, - {file = "azure_core-1.26.1-py3-none-any.whl", hash = "sha256:726ffd1ded04a2c1cb53f9d9155cbb05ac5c1c2a29af4ef622e93e1c0a8bc92b"}, -] [package.dependencies] requests = ">=2.18.4" @@ -259,10 +136,6 @@ description = "Azure Data Lake Store Filesystem Client Library for Python" category = "main" optional = true python-versions = "*" -files = [ - {file = "azure-datalake-store-0.0.52.tar.gz", hash = "sha256:4198ddb32614d16d4502b43d5c9739f81432b7e0e4d75d30e05149fe6007fea2"}, - {file = "azure_datalake_store-0.0.52-py2.py3-none-any.whl", hash = "sha256:aaed72b9c856824aeab554f4dbe0ef2c6d0ff36700bdd8b93d8298793117c48e"}, -] [package.dependencies] adal = ">=0.4.2" @@ -276,10 +149,6 @@ description = "Microsoft Azure Identity Library for Python" category = "main" optional = true python-versions = ">=3.7" -files = [ - {file = "azure-identity-1.12.0.zip", hash = "sha256:7f9b1ae7d97ea7af3f38dd09305e19ab81a1e16ab66ea186b6579d85c1ca2347"}, - {file = "azure_identity-1.12.0-py3-none-any.whl", hash = "sha256:2a58ce4a209a013e37eaccfd5937570ab99e9118b3e1acf875eed3a85d541b92"}, -] [package.dependencies] azure-core = ">=1.11.0,<2.0.0" @@ -295,10 +164,6 @@ description = "Microsoft Azure Blob Storage Client Library for Python" category = "main" optional = true python-versions = ">=3.7" -files = [ - {file = "azure-storage-blob-12.14.1.zip", hash = "sha256:860d4d82985a4bfc7d3271e71275af330f54f330a754355435a7ba749ccde997"}, - {file = "azure_storage_blob-12.14.1-py3-none-any.whl", hash = "sha256:52b84658e8df7853a3cf1c563814655b5028b979b2a87905b92aa6bb30be240e"}, -] [package.dependencies] azure-core = ">=1.24.2,<2.0.0" @@ -312,10 +177,6 @@ description = "The AWS SDK for Python" category = "main" optional = false python-versions = ">= 3.7" -files = [ - {file = "boto3-1.24.59-py3-none-any.whl", hash = "sha256:34ab44146a2c4e7f4e72737f4b27e6eb5e0a7855c2f4599e3d9199b6a0a2d575"}, - {file = "boto3-1.24.59.tar.gz", hash = "sha256:a50b4323f9579cfe22fcf5531fbd40b567d4d74c1adce06aeb5c95fce2a6fb40"}, -] [package.dependencies] botocore = ">=1.27.59,<1.28.0" @@ -332,10 +193,6 @@ description = "Low-level, data-driven core of boto 3." category = "main" optional = false python-versions = ">= 3.7" -files = [ - {file = "botocore-1.27.59-py3-none-any.whl", hash = "sha256:69d756791fc024bda54f6c53f71ae34e695ee41bbbc1743d9179c4837a4929da"}, - {file = "botocore-1.27.59.tar.gz", hash = "sha256:eda4aed6ee719a745d1288eaf1beb12f6f6448ad1fa12f159405db14ba9c92cf"}, -] [package.dependencies] jmespath = ">=0.7.1,<2.0.0" @@ -352,10 +209,6 @@ description = "A simple, correct PEP 517 build frontend" category = "dev" optional = false python-versions = ">=3.6" -files = [ - {file = "build-0.9.0-py3-none-any.whl", hash = "sha256:38a7a2b7a0bdc61a42a0a67509d88c71ecfc37b393baba770fae34e20929ff69"}, - {file = "build-0.9.0.tar.gz", hash = "sha256:1a07724e891cbd898923145eb7752ee7653674c511378eb9c7691aab1612bc3c"}, -] [package.dependencies] colorama = {version = "*", markers = "os_name == \"nt\""} @@ -376,10 +229,6 @@ description = "Python package for providing Mozilla's CA Bundle." category = "main" optional = false python-versions = ">=3.6" -files = [ - {file = "certifi-2022.12.7-py3-none-any.whl", hash = "sha256:4ad3232f5e926d6718ec31cfc1fcadfde020920e278684144551c91769c7bc18"}, - {file = "certifi-2022.12.7.tar.gz", hash = "sha256:35824b4c3a97115964b408844d64aa14db1cc518f6562e8d7261699d1350a9e3"}, -] [[package]] name = "cffi" @@ -388,72 +237,6 @@ description = "Foreign Function Interface for Python calling C code." category = "main" optional = false python-versions = "*" -files = [ - {file = "cffi-1.15.1-cp27-cp27m-macosx_10_9_x86_64.whl", hash = "sha256:a66d3508133af6e8548451b25058d5812812ec3798c886bf38ed24a98216fab2"}, - {file = "cffi-1.15.1-cp27-cp27m-manylinux1_i686.whl", hash = "sha256:470c103ae716238bbe698d67ad020e1db9d9dba34fa5a899b5e21577e6d52ed2"}, - {file = "cffi-1.15.1-cp27-cp27m-manylinux1_x86_64.whl", hash = "sha256:9ad5db27f9cabae298d151c85cf2bad1d359a1b9c686a275df03385758e2f914"}, - {file = "cffi-1.15.1-cp27-cp27m-win32.whl", hash = "sha256:b3bbeb01c2b273cca1e1e0c5df57f12dce9a4dd331b4fa1635b8bec26350bde3"}, - {file = "cffi-1.15.1-cp27-cp27m-win_amd64.whl", hash = "sha256:e00b098126fd45523dd056d2efba6c5a63b71ffe9f2bbe1a4fe1716e1d0c331e"}, - {file = "cffi-1.15.1-cp27-cp27mu-manylinux1_i686.whl", hash = "sha256:d61f4695e6c866a23a21acab0509af1cdfd2c013cf256bbf5b6b5e2695827162"}, - {file = "cffi-1.15.1-cp27-cp27mu-manylinux1_x86_64.whl", hash = "sha256:ed9cb427ba5504c1dc15ede7d516b84757c3e3d7868ccc85121d9310d27eed0b"}, - {file = "cffi-1.15.1-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:39d39875251ca8f612b6f33e6b1195af86d1b3e60086068be9cc053aa4376e21"}, - {file = "cffi-1.15.1-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:285d29981935eb726a4399badae8f0ffdff4f5050eaa6d0cfc3f64b857b77185"}, - {file = "cffi-1.15.1-cp310-cp310-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:3eb6971dcff08619f8d91607cfc726518b6fa2a9eba42856be181c6d0d9515fd"}, - {file = "cffi-1.15.1-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:21157295583fe8943475029ed5abdcf71eb3911894724e360acff1d61c1d54bc"}, - {file = "cffi-1.15.1-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:5635bd9cb9731e6d4a1132a498dd34f764034a8ce60cef4f5319c0541159392f"}, - {file = "cffi-1.15.1-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:2012c72d854c2d03e45d06ae57f40d78e5770d252f195b93f581acf3ba44496e"}, - {file = "cffi-1.15.1-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:dd86c085fae2efd48ac91dd7ccffcfc0571387fe1193d33b6394db7ef31fe2a4"}, - {file = "cffi-1.15.1-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:fa6693661a4c91757f4412306191b6dc88c1703f780c8234035eac011922bc01"}, - {file = "cffi-1.15.1-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:59c0b02d0a6c384d453fece7566d1c7e6b7bae4fc5874ef2ef46d56776d61c9e"}, - {file = "cffi-1.15.1-cp310-cp310-win32.whl", hash = "sha256:cba9d6b9a7d64d4bd46167096fc9d2f835e25d7e4c121fb2ddfc6528fb0413b2"}, - {file = "cffi-1.15.1-cp310-cp310-win_amd64.whl", hash = "sha256:ce4bcc037df4fc5e3d184794f27bdaab018943698f4ca31630bc7f84a7b69c6d"}, - {file = "cffi-1.15.1-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:3d08afd128ddaa624a48cf2b859afef385b720bb4b43df214f85616922e6a5ac"}, - {file = "cffi-1.15.1-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:3799aecf2e17cf585d977b780ce79ff0dc9b78d799fc694221ce814c2c19db83"}, - {file = "cffi-1.15.1-cp311-cp311-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:a591fe9e525846e4d154205572a029f653ada1a78b93697f3b5a8f1f2bc055b9"}, - {file = "cffi-1.15.1-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:3548db281cd7d2561c9ad9984681c95f7b0e38881201e157833a2342c30d5e8c"}, - {file = "cffi-1.15.1-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:91fc98adde3d7881af9b59ed0294046f3806221863722ba7d8d120c575314325"}, - {file = "cffi-1.15.1-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:94411f22c3985acaec6f83c6df553f2dbe17b698cc7f8ae751ff2237d96b9e3c"}, - {file = "cffi-1.15.1-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:03425bdae262c76aad70202debd780501fabeaca237cdfddc008987c0e0f59ef"}, - {file = "cffi-1.15.1-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:cc4d65aeeaa04136a12677d3dd0b1c0c94dc43abac5860ab33cceb42b801c1e8"}, - {file = "cffi-1.15.1-cp311-cp311-win32.whl", hash = "sha256:a0f100c8912c114ff53e1202d0078b425bee3649ae34d7b070e9697f93c5d52d"}, - {file = "cffi-1.15.1-cp311-cp311-win_amd64.whl", hash = "sha256:04ed324bda3cda42b9b695d51bb7d54b680b9719cfab04227cdd1e04e5de3104"}, - {file = "cffi-1.15.1-cp36-cp36m-macosx_10_9_x86_64.whl", hash = "sha256:50a74364d85fd319352182ef59c5c790484a336f6db772c1a9231f1c3ed0cbd7"}, - {file = "cffi-1.15.1-cp36-cp36m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:e263d77ee3dd201c3a142934a086a4450861778baaeeb45db4591ef65550b0a6"}, - {file = "cffi-1.15.1-cp36-cp36m-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:cec7d9412a9102bdc577382c3929b337320c4c4c4849f2c5cdd14d7368c5562d"}, - {file = "cffi-1.15.1-cp36-cp36m-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:4289fc34b2f5316fbb762d75362931e351941fa95fa18789191b33fc4cf9504a"}, - {file = "cffi-1.15.1-cp36-cp36m-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:173379135477dc8cac4bc58f45db08ab45d228b3363adb7af79436135d028405"}, - {file = "cffi-1.15.1-cp36-cp36m-manylinux_2_5_x86_64.manylinux1_x86_64.whl", hash = "sha256:6975a3fac6bc83c4a65c9f9fcab9e47019a11d3d2cf7f3c0d03431bf145a941e"}, - {file = "cffi-1.15.1-cp36-cp36m-win32.whl", hash = "sha256:2470043b93ff09bf8fb1d46d1cb756ce6132c54826661a32d4e4d132e1977adf"}, - {file = "cffi-1.15.1-cp36-cp36m-win_amd64.whl", hash = "sha256:30d78fbc8ebf9c92c9b7823ee18eb92f2e6ef79b45ac84db507f52fbe3ec4497"}, - {file = "cffi-1.15.1-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:198caafb44239b60e252492445da556afafc7d1e3ab7a1fb3f0584ef6d742375"}, - {file = "cffi-1.15.1-cp37-cp37m-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:5ef34d190326c3b1f822a5b7a45f6c4535e2f47ed06fec77d3d799c450b2651e"}, - {file = "cffi-1.15.1-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:8102eaf27e1e448db915d08afa8b41d6c7ca7a04b7d73af6514df10a3e74bd82"}, - {file = "cffi-1.15.1-cp37-cp37m-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:5df2768244d19ab7f60546d0c7c63ce1581f7af8b5de3eb3004b9b6fc8a9f84b"}, - {file = "cffi-1.15.1-cp37-cp37m-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:a8c4917bd7ad33e8eb21e9a5bbba979b49d9a97acb3a803092cbc1133e20343c"}, - {file = "cffi-1.15.1-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:0e2642fe3142e4cc4af0799748233ad6da94c62a8bec3a6648bf8ee68b1c7426"}, - {file = "cffi-1.15.1-cp37-cp37m-win32.whl", hash = "sha256:e229a521186c75c8ad9490854fd8bbdd9a0c9aa3a524326b55be83b54d4e0ad9"}, - {file = "cffi-1.15.1-cp37-cp37m-win_amd64.whl", hash = "sha256:a0b71b1b8fbf2b96e41c4d990244165e2c9be83d54962a9a1d118fd8657d2045"}, - {file = "cffi-1.15.1-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:320dab6e7cb2eacdf0e658569d2575c4dad258c0fcc794f46215e1e39f90f2c3"}, - {file = "cffi-1.15.1-cp38-cp38-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:1e74c6b51a9ed6589199c787bf5f9875612ca4a8a0785fb2d4a84429badaf22a"}, - {file = "cffi-1.15.1-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a5c84c68147988265e60416b57fc83425a78058853509c1b0629c180094904a5"}, - {file = "cffi-1.15.1-cp38-cp38-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:3b926aa83d1edb5aa5b427b4053dc420ec295a08e40911296b9eb1b6170f6cca"}, - {file = "cffi-1.15.1-cp38-cp38-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:87c450779d0914f2861b8526e035c5e6da0a3199d8f1add1a665e1cbc6fc6d02"}, - {file = "cffi-1.15.1-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:4f2c9f67e9821cad2e5f480bc8d83b8742896f1242dba247911072d4fa94c192"}, - {file = "cffi-1.15.1-cp38-cp38-win32.whl", hash = "sha256:8b7ee99e510d7b66cdb6c593f21c043c248537a32e0bedf02e01e9553a172314"}, - {file = "cffi-1.15.1-cp38-cp38-win_amd64.whl", hash = "sha256:00a9ed42e88df81ffae7a8ab6d9356b371399b91dbdf0c3cb1e84c03a13aceb5"}, - {file = "cffi-1.15.1-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:54a2db7b78338edd780e7ef7f9f6c442500fb0d41a5a4ea24fff1c929d5af585"}, - {file = "cffi-1.15.1-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:fcd131dd944808b5bdb38e6f5b53013c5aa4f334c5cad0c72742f6eba4b73db0"}, - {file = "cffi-1.15.1-cp39-cp39-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:7473e861101c9e72452f9bf8acb984947aa1661a7704553a9f6e4baa5ba64415"}, - {file = "cffi-1.15.1-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:6c9a799e985904922a4d207a94eae35c78ebae90e128f0c4e521ce339396be9d"}, - {file = "cffi-1.15.1-cp39-cp39-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:3bcde07039e586f91b45c88f8583ea7cf7a0770df3a1649627bf598332cb6984"}, - {file = "cffi-1.15.1-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:33ab79603146aace82c2427da5ca6e58f2b3f2fb5da893ceac0c42218a40be35"}, - {file = "cffi-1.15.1-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:5d598b938678ebf3c67377cdd45e09d431369c3b1a5b331058c338e201f12b27"}, - {file = "cffi-1.15.1-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:db0fbb9c62743ce59a9ff687eb5f4afbe77e5e8403d6697f7446e5f609976f76"}, - {file = "cffi-1.15.1-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:98d85c6a2bef81588d9227dde12db8a7f47f639f4a17c9ae08e773aa9c697bf3"}, - {file = "cffi-1.15.1-cp39-cp39-win32.whl", hash = "sha256:40f4774f5a9d4f5e344f31a32b5096977b5d48560c5592e2f3d2c4374bd543ee"}, - {file = "cffi-1.15.1-cp39-cp39-win_amd64.whl", hash = "sha256:70df4e3b545a17496c9b3f41f5115e69a4f2e77e94e1d2a8e1070bc0c38c8a3c"}, - {file = "cffi-1.15.1.tar.gz", hash = "sha256:d400bfb9a37b1351253cb402671cea7e89bdecc294e8016a707f6d1d8ac934f9"}, -] [package.dependencies] pycparser = "*" @@ -465,10 +248,6 @@ description = "Validate configuration and produce human readable error messages. category = "dev" optional = false python-versions = ">=3.6.1" -files = [ - {file = "cfgv-3.3.1-py2.py3-none-any.whl", hash = "sha256:c6a0883f3917a037485059700b9e75da2464e6c27051014ad85ba6aaa5884426"}, - {file = "cfgv-3.3.1.tar.gz", hash = "sha256:f5a830efb9ce7a445376bb66ec94c638a9787422f96264c98edc6bdeed8ab736"}, -] [[package]] name = "charset-normalizer" @@ -477,10 +256,6 @@ description = "The Real First Universal Charset Detector. Open, modern and activ category = "main" optional = false python-versions = ">=3.6.0" -files = [ - {file = "charset-normalizer-2.1.1.tar.gz", hash = "sha256:5a3d016c7c547f69d6f81fb0db9449ce888b418b5b9952cc5e6e66843e9dd845"}, - {file = "charset_normalizer-2.1.1-py3-none-any.whl", hash = "sha256:83e9a75d1911279afd89352c68b45348559d1fc0506b054b346651b5e7fee29f"}, -] [package.extras] unicode-backport = ["unicodedata2"] @@ -492,10 +267,6 @@ description = "Composable command line interface toolkit" category = "main" optional = false python-versions = ">=3.7" -files = [ - {file = "click-8.1.3-py3-none-any.whl", hash = "sha256:bb4d8133cb15a609f44e8213d9b391b0809795062913b383c62be0ee95b1db48"}, - {file = "click-8.1.3.tar.gz", hash = "sha256:7682dc8afb30297001674575ea00d1814d808d6a36af415a82bd481d37ba7b8e"}, -] [package.dependencies] colorama = {version = "*", markers = "platform_system == \"Windows\""} @@ -507,10 +278,6 @@ description = "Cross-platform colored terminal text." category = "main" optional = false python-versions = "!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,!=3.4.*,!=3.5.*,!=3.6.*,>=2.7" -files = [ - {file = "colorama-0.4.6-py2.py3-none-any.whl", hash = "sha256:4f1d9991f5acc0ca119f9d443620b77f9d6b33703e51011c16baf57afb285fc6"}, - {file = "colorama-0.4.6.tar.gz", hash = "sha256:08695f5cb7ed6e0531a20572697297273c47b8cae5a63ffc6d6ed5c201be6e44"}, -] [[package]] name = "commonmark" @@ -519,73 +286,17 @@ description = "Python parser for the CommonMark Markdown spec" category = "main" optional = false python-versions = "*" -files = [ - {file = "commonmark-0.9.1-py2.py3-none-any.whl", hash = "sha256:da2f38c92590f83de410ba1a3cbceafbc74fee9def35f9251ba9a971d6d66fd9"}, - {file = "commonmark-0.9.1.tar.gz", hash = "sha256:452f9dc859be7f06631ddcb328b6919c67984aca654e5fefb3914d54691aed60"}, -] [package.extras] test = ["flake8 (==3.7.8)", "hypothesis (==3.55.3)"] [[package]] name = "coverage" -version = "6.5.0" +version = "7.0.1" description = "Code coverage measurement for Python" category = "dev" optional = false python-versions = ">=3.7" -files = [ - {file = "coverage-6.5.0-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:ef8674b0ee8cc11e2d574e3e2998aea5df5ab242e012286824ea3c6970580e53"}, - {file = "coverage-6.5.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:784f53ebc9f3fd0e2a3f6a78b2be1bd1f5575d7863e10c6e12504f240fd06660"}, - {file = "coverage-6.5.0-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:b4a5be1748d538a710f87542f22c2cad22f80545a847ad91ce45e77417293eb4"}, - {file = "coverage-6.5.0-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:83516205e254a0cb77d2d7bb3632ee019d93d9f4005de31dca0a8c3667d5bc04"}, - {file = "coverage-6.5.0-cp310-cp310-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:af4fffaffc4067232253715065e30c5a7ec6faac36f8fc8d6f64263b15f74db0"}, - {file = "coverage-6.5.0-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:97117225cdd992a9c2a5515db1f66b59db634f59d0679ca1fa3fe8da32749cae"}, - {file = "coverage-6.5.0-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:a1170fa54185845505fbfa672f1c1ab175446c887cce8212c44149581cf2d466"}, - {file = "coverage-6.5.0-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:11b990d520ea75e7ee8dcab5bc908072aaada194a794db9f6d7d5cfd19661e5a"}, - {file = "coverage-6.5.0-cp310-cp310-win32.whl", hash = "sha256:5dbec3b9095749390c09ab7c89d314727f18800060d8d24e87f01fb9cfb40b32"}, - {file = "coverage-6.5.0-cp310-cp310-win_amd64.whl", hash = "sha256:59f53f1dc5b656cafb1badd0feb428c1e7bc19b867479ff72f7a9dd9b479f10e"}, - {file = "coverage-6.5.0-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:4a5375e28c5191ac38cca59b38edd33ef4cc914732c916f2929029b4bfb50795"}, - {file = "coverage-6.5.0-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:c4ed2820d919351f4167e52425e096af41bfabacb1857186c1ea32ff9983ed75"}, - {file = "coverage-6.5.0-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:33a7da4376d5977fbf0a8ed91c4dffaaa8dbf0ddbf4c8eea500a2486d8bc4d7b"}, - {file = "coverage-6.5.0-cp311-cp311-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:a8fb6cf131ac4070c9c5a3e21de0f7dc5a0fbe8bc77c9456ced896c12fcdad91"}, - {file = "coverage-6.5.0-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:a6b7d95969b8845250586f269e81e5dfdd8ff828ddeb8567a4a2eaa7313460c4"}, - {file = "coverage-6.5.0-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:1ef221513e6f68b69ee9e159506d583d31aa3567e0ae84eaad9d6ec1107dddaa"}, - {file = "coverage-6.5.0-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:cca4435eebea7962a52bdb216dec27215d0df64cf27fc1dd538415f5d2b9da6b"}, - {file = "coverage-6.5.0-cp311-cp311-win32.whl", hash = "sha256:98e8a10b7a314f454d9eff4216a9a94d143a7ee65018dd12442e898ee2310578"}, - {file = "coverage-6.5.0-cp311-cp311-win_amd64.whl", hash = "sha256:bc8ef5e043a2af066fa8cbfc6e708d58017024dc4345a1f9757b329a249f041b"}, - {file = "coverage-6.5.0-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:4433b90fae13f86fafff0b326453dd42fc9a639a0d9e4eec4d366436d1a41b6d"}, - {file = "coverage-6.5.0-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:f4f05d88d9a80ad3cac6244d36dd89a3c00abc16371769f1340101d3cb899fc3"}, - {file = "coverage-6.5.0-cp37-cp37m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:94e2565443291bd778421856bc975d351738963071e9b8839ca1fc08b42d4bef"}, - {file = "coverage-6.5.0-cp37-cp37m-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:027018943386e7b942fa832372ebc120155fd970837489896099f5cfa2890f79"}, - {file = "coverage-6.5.0-cp37-cp37m-musllinux_1_1_aarch64.whl", hash = "sha256:255758a1e3b61db372ec2736c8e2a1fdfaf563977eedbdf131de003ca5779b7d"}, - {file = "coverage-6.5.0-cp37-cp37m-musllinux_1_1_i686.whl", hash = "sha256:851cf4ff24062c6aec510a454b2584f6e998cada52d4cb58c5e233d07172e50c"}, - {file = "coverage-6.5.0-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:12adf310e4aafddc58afdb04d686795f33f4d7a6fa67a7a9d4ce7d6ae24d949f"}, - {file = "coverage-6.5.0-cp37-cp37m-win32.whl", hash = "sha256:b5604380f3415ba69de87a289a2b56687faa4fe04dbee0754bfcae433489316b"}, - {file = "coverage-6.5.0-cp37-cp37m-win_amd64.whl", hash = "sha256:4a8dbc1f0fbb2ae3de73eb0bdbb914180c7abfbf258e90b311dcd4f585d44bd2"}, - {file = "coverage-6.5.0-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:d900bb429fdfd7f511f868cedd03a6bbb142f3f9118c09b99ef8dc9bf9643c3c"}, - {file = "coverage-6.5.0-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:2198ea6fc548de52adc826f62cb18554caedfb1d26548c1b7c88d8f7faa8f6ba"}, - {file = "coverage-6.5.0-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:6c4459b3de97b75e3bd6b7d4b7f0db13f17f504f3d13e2a7c623786289dd670e"}, - {file = "coverage-6.5.0-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:20c8ac5386253717e5ccc827caad43ed66fea0efe255727b1053a8154d952398"}, - {file = "coverage-6.5.0-cp38-cp38-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:6b07130585d54fe8dff3d97b93b0e20290de974dc8177c320aeaf23459219c0b"}, - {file = "coverage-6.5.0-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:dbdb91cd8c048c2b09eb17713b0c12a54fbd587d79adcebad543bc0cd9a3410b"}, - {file = "coverage-6.5.0-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:de3001a203182842a4630e7b8d1a2c7c07ec1b45d3084a83d5d227a3806f530f"}, - {file = "coverage-6.5.0-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:e07f4a4a9b41583d6eabec04f8b68076ab3cd44c20bd29332c6572dda36f372e"}, - {file = "coverage-6.5.0-cp38-cp38-win32.whl", hash = "sha256:6d4817234349a80dbf03640cec6109cd90cba068330703fa65ddf56b60223a6d"}, - {file = "coverage-6.5.0-cp38-cp38-win_amd64.whl", hash = "sha256:7ccf362abd726b0410bf8911c31fbf97f09f8f1061f8c1cf03dfc4b6372848f6"}, - {file = "coverage-6.5.0-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:633713d70ad6bfc49b34ead4060531658dc6dfc9b3eb7d8a716d5873377ab745"}, - {file = "coverage-6.5.0-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:95203854f974e07af96358c0b261f1048d8e1083f2de9b1c565e1be4a3a48cfc"}, - {file = "coverage-6.5.0-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:b9023e237f4c02ff739581ef35969c3739445fb059b060ca51771e69101efffe"}, - {file = "coverage-6.5.0-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:265de0fa6778d07de30bcf4d9dc471c3dc4314a23a3c6603d356a3c9abc2dfcf"}, - {file = "coverage-6.5.0-cp39-cp39-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:8f830ed581b45b82451a40faabb89c84e1a998124ee4212d440e9c6cf70083e5"}, - {file = "coverage-6.5.0-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:7b6be138d61e458e18d8e6ddcddd36dd96215edfe5f1168de0b1b32635839b62"}, - {file = "coverage-6.5.0-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:42eafe6778551cf006a7c43153af1211c3aaab658d4d66fa5fcc021613d02518"}, - {file = "coverage-6.5.0-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:723e8130d4ecc8f56e9a611e73b31219595baa3bb252d539206f7bbbab6ffc1f"}, - {file = "coverage-6.5.0-cp39-cp39-win32.whl", hash = "sha256:d9ecf0829c6a62b9b573c7bb6d4dcd6ba8b6f80be9ba4fc7ed50bf4ac9aecd72"}, - {file = "coverage-6.5.0-cp39-cp39-win_amd64.whl", hash = "sha256:fc2af30ed0d5ae0b1abdb4ebdce598eafd5b35397d4d75deb341a614d333d987"}, - {file = "coverage-6.5.0-pp36.pp37.pp38-none-any.whl", hash = "sha256:1431986dac3923c5945271f169f59c45b8802a114c8f548d611f2015133df77a"}, - {file = "coverage-6.5.0.tar.gz", hash = "sha256:f642e90754ee3e06b0e7e51bce3379590e76b7f76b708e1a71ff043f87025c84"}, -] [package.dependencies] tomli = {version = "*", optional = true, markers = "python_full_version <= \"3.11.0a6\" and extra == \"toml\""} @@ -600,34 +311,6 @@ description = "cryptography is a package which provides cryptographic recipes an category = "main" optional = false python-versions = ">=3.6" -files = [ - {file = "cryptography-38.0.4-cp36-abi3-macosx_10_10_universal2.whl", hash = "sha256:2fa36a7b2cc0998a3a4d5af26ccb6273f3df133d61da2ba13b3286261e7efb70"}, - {file = "cryptography-38.0.4-cp36-abi3-macosx_10_10_x86_64.whl", hash = "sha256:1f13ddda26a04c06eb57119caf27a524ccae20533729f4b1e4a69b54e07035eb"}, - {file = "cryptography-38.0.4-cp36-abi3-manylinux_2_17_aarch64.manylinux2014_aarch64.manylinux_2_24_aarch64.whl", hash = "sha256:2ec2a8714dd005949d4019195d72abed84198d877112abb5a27740e217e0ea8d"}, - {file = "cryptography-38.0.4-cp36-abi3-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:50a1494ed0c3f5b4d07650a68cd6ca62efe8b596ce743a5c94403e6f11bf06c1"}, - {file = "cryptography-38.0.4-cp36-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:a10498349d4c8eab7357a8f9aa3463791292845b79597ad1b98a543686fb1ec8"}, - {file = "cryptography-38.0.4-cp36-abi3-manylinux_2_24_x86_64.whl", hash = "sha256:10652dd7282de17990b88679cb82f832752c4e8237f0c714be518044269415db"}, - {file = "cryptography-38.0.4-cp36-abi3-manylinux_2_28_aarch64.whl", hash = "sha256:bfe6472507986613dc6cc00b3d492b2f7564b02b3b3682d25ca7f40fa3fd321b"}, - {file = "cryptography-38.0.4-cp36-abi3-manylinux_2_28_x86_64.whl", hash = "sha256:ce127dd0a6a0811c251a6cddd014d292728484e530d80e872ad9806cfb1c5b3c"}, - {file = "cryptography-38.0.4-cp36-abi3-musllinux_1_1_aarch64.whl", hash = "sha256:53049f3379ef05182864d13bb9686657659407148f901f3f1eee57a733fb4b00"}, - {file = "cryptography-38.0.4-cp36-abi3-musllinux_1_1_x86_64.whl", hash = "sha256:8a4b2bdb68a447fadebfd7d24855758fe2d6fecc7fed0b78d190b1af39a8e3b0"}, - {file = "cryptography-38.0.4-cp36-abi3-win32.whl", hash = "sha256:1d7e632804a248103b60b16fb145e8df0bc60eed790ece0d12efe8cd3f3e7744"}, - {file = "cryptography-38.0.4-cp36-abi3-win_amd64.whl", hash = "sha256:8e45653fb97eb2f20b8c96f9cd2b3a0654d742b47d638cf2897afbd97f80fa6d"}, - {file = "cryptography-38.0.4-pp37-pypy37_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:ca57eb3ddaccd1112c18fc80abe41db443cc2e9dcb1917078e02dfa010a4f353"}, - {file = "cryptography-38.0.4-pp37-pypy37_pp73-manylinux_2_24_x86_64.whl", hash = "sha256:c9e0d79ee4c56d841bd4ac6e7697c8ff3c8d6da67379057f29e66acffcd1e9a7"}, - {file = "cryptography-38.0.4-pp37-pypy37_pp73-manylinux_2_28_x86_64.whl", hash = "sha256:0e70da4bdff7601b0ef48e6348339e490ebfb0cbe638e083c9c41fb49f00c8bd"}, - {file = "cryptography-38.0.4-pp38-pypy38_pp73-macosx_10_10_x86_64.whl", hash = "sha256:998cd19189d8a747b226d24c0207fdaa1e6658a1d3f2494541cb9dfbf7dcb6d2"}, - {file = "cryptography-38.0.4-pp38-pypy38_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:67461b5ebca2e4c2ab991733f8ab637a7265bb582f07c7c88914b5afb88cb95b"}, - {file = "cryptography-38.0.4-pp38-pypy38_pp73-manylinux_2_24_x86_64.whl", hash = "sha256:4eb85075437f0b1fd8cd66c688469a0c4119e0ba855e3fef86691971b887caf6"}, - {file = "cryptography-38.0.4-pp38-pypy38_pp73-manylinux_2_28_x86_64.whl", hash = "sha256:3178d46f363d4549b9a76264f41c6948752183b3f587666aff0555ac50fd7876"}, - {file = "cryptography-38.0.4-pp38-pypy38_pp73-win_amd64.whl", hash = "sha256:6391e59ebe7c62d9902c24a4d8bcbc79a68e7c4ab65863536127c8a9cd94043b"}, - {file = "cryptography-38.0.4-pp39-pypy39_pp73-macosx_10_10_x86_64.whl", hash = "sha256:78e47e28ddc4ace41dd38c42e6feecfdadf9c3be2af389abbfeef1ff06822285"}, - {file = "cryptography-38.0.4-pp39-pypy39_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:2fb481682873035600b5502f0015b664abc26466153fab5c6bc92c1ea69d478b"}, - {file = "cryptography-38.0.4-pp39-pypy39_pp73-manylinux_2_24_x86_64.whl", hash = "sha256:4367da5705922cf7070462e964f66e4ac24162e22ab0a2e9d31f1b270dd78083"}, - {file = "cryptography-38.0.4-pp39-pypy39_pp73-manylinux_2_28_x86_64.whl", hash = "sha256:b4cad0cea995af760f82820ab4ca54e5471fc782f70a007f31531957f43e9dee"}, - {file = "cryptography-38.0.4-pp39-pypy39_pp73-win_amd64.whl", hash = "sha256:80ca53981ceeb3241998443c4964a387771588c4e4a5d92735a493af868294f9"}, - {file = "cryptography-38.0.4.tar.gz", hash = "sha256:175c1a818b87c9ac80bb7377f5520b7f31b3ef2a0004e2420319beadedb67290"}, -] [package.dependencies] cffi = ">=1.12" @@ -647,10 +330,6 @@ description = "Distribution utilities" category = "dev" optional = false python-versions = "*" -files = [ - {file = "distlib-0.3.6-py2.py3-none-any.whl", hash = "sha256:f35c4b692542ca110de7ef0bea44d73981caeb34ca0b9b6b2e6d7790dda8f80e"}, - {file = "distlib-0.3.6.tar.gz", hash = "sha256:14bad2d9b04d3a36127ac97f30b12a19268f211063d8f8ee4f47108896e11b46"}, -] [[package]] name = "docutils" @@ -659,10 +338,6 @@ description = "Docutils -- Python Documentation Utilities" category = "dev" optional = false python-versions = ">=3.7" -files = [ - {file = "docutils-0.19-py3-none-any.whl", hash = "sha256:5e1de4d849fee02c63b040a4a3fd567f4ab104defd8a5511fbbc24a8a017efbc"}, - {file = "docutils-0.19.tar.gz", hash = "sha256:33995a6753c30b7f577febfc2c50411fec6aac7f7ffeb7c4cfe5991072dcf9e6"}, -] [[package]] name = "duckdb" @@ -671,55 +346,6 @@ description = "DuckDB embedded database" category = "main" optional = true python-versions = "*" -files = [ - {file = "duckdb-0.6.1-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:e566514f9327f89264e98ac14ee7a84fbd9857328028258422c3e8375ee19d25"}, - {file = "duckdb-0.6.1-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:b31c2883de5b19591a2852165e6b3f9821f77af649835f27bc146b26e4aa30cb"}, - {file = "duckdb-0.6.1-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:998165b2fb1f1d2b0ad742096015ea70878f7d40304643c7424c3ed3ddf07bfc"}, - {file = "duckdb-0.6.1-cp310-cp310-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:3941b3a1e8a1cdb7b90ab3917b87af816e71f9692e5ada7f19b6b60969f731e5"}, - {file = "duckdb-0.6.1-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:143611bd1b7c13343f087d4d423a7a8a4f33a114c5326171e867febf3f0fcfe1"}, - {file = "duckdb-0.6.1-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:125ba45e8b08f28858f918ec9cbd3a19975e5d8d9e8275ef4ad924028a616e14"}, - {file = "duckdb-0.6.1-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:e609a65b31c92f2f7166831f74b56f5ed54b33d8c2c4b4c3974c26fdc50464c5"}, - {file = "duckdb-0.6.1-cp310-cp310-win32.whl", hash = "sha256:b39045074fb9a3f068496475a5d627ad4fa572fa3b4980e3b479c11d0b706f2d"}, - {file = "duckdb-0.6.1-cp310-cp310-win_amd64.whl", hash = "sha256:16fa96ffaa3d842a9355a633fb8bc092d119be08d4bc02013946d8594417bc14"}, - {file = "duckdb-0.6.1-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:b4bbe2f6c1b109c626f9318eee80934ad2a5b81a51409c6b5083c6c5f9bdb125"}, - {file = "duckdb-0.6.1-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:cfea36b58928ce778d17280d4fb3bf0a2d7cff407667baedd69c5b41463ac0fd"}, - {file = "duckdb-0.6.1-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:0b64eb53d0d0695814bf1b65c0f91ab7ed66b515f89c88038f65ad5e0762571c"}, - {file = "duckdb-0.6.1-cp311-cp311-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:35b01bc724e1933293f4c34f410d2833bfbb56d5743b515d805bbfed0651476e"}, - {file = "duckdb-0.6.1-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:fec2c2466654ce786843bda2bfba71e0e4719106b41d36b17ceb1901e130aa71"}, - {file = "duckdb-0.6.1-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:82cd30f5cf368658ef879b1c60276bc8650cf67cfe3dc3e3009438ba39251333"}, - {file = "duckdb-0.6.1-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:a782bbfb7f5e97d4a9c834c9e78f023fb8b3f6687c22ca99841e6ed944b724da"}, - {file = "duckdb-0.6.1-cp311-cp311-win32.whl", hash = "sha256:e3702d4a9ade54c6403f6615a98bbec2020a76a60f5db7fcf085df1bd270e66e"}, - {file = "duckdb-0.6.1-cp311-cp311-win_amd64.whl", hash = "sha256:93b074f473d68c944b0eeb2edcafd91ad11da8432b484836efaaab4e26351d48"}, - {file = "duckdb-0.6.1-cp36-cp36m-macosx_10_9_x86_64.whl", hash = "sha256:adae183924d6d479202c39072e37d440b511326e84525bcb7432bca85f86caba"}, - {file = "duckdb-0.6.1-cp36-cp36m-win32.whl", hash = "sha256:546a1cd17595bd1dd009daf6f36705aa6f95337154360ce44932157d353dcd80"}, - {file = "duckdb-0.6.1-cp36-cp36m-win_amd64.whl", hash = "sha256:87b0d00eb9d1a7ebe437276203e0cdc93b4a2154ba9688c65e8d2a8735839ec6"}, - {file = "duckdb-0.6.1-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:8442e074de6e1969c3d2b24363a5a6d7f866d5ac3f4e358e357495b389eff6c1"}, - {file = "duckdb-0.6.1-cp37-cp37m-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:0a6bf2ae7bec803352dade14561cb0b461b2422e70f75d9f09b36ba2dad2613b"}, - {file = "duckdb-0.6.1-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:5054792f22733f89d9cbbced2bafd8772d72d0fe77f159310221cefcf981c680"}, - {file = "duckdb-0.6.1-cp37-cp37m-musllinux_1_1_i686.whl", hash = "sha256:21cc503dffc2c68bb825e4eb3098e82f40e910b3d09e1b3b7f090d39ad53fbea"}, - {file = "duckdb-0.6.1-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:54b3da77ad893e99c073087ff7f75a8c98154ac5139d317149f12b74367211db"}, - {file = "duckdb-0.6.1-cp37-cp37m-win32.whl", hash = "sha256:f1d709aa6a26172a3eab804b57763d5cdc1a4b785ac1fc2b09568578e52032ee"}, - {file = "duckdb-0.6.1-cp37-cp37m-win_amd64.whl", hash = "sha256:f4edcaa471d791393e37f63e3c7c728fa6324e3ac7e768b9dc2ea49065cd37cc"}, - {file = "duckdb-0.6.1-cp38-cp38-macosx_10_9_universal2.whl", hash = "sha256:d218c2dd3bda51fb79e622b7b2266183ac9493834b55010aa01273fa5b7a7105"}, - {file = "duckdb-0.6.1-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:0c7155cb93ab432eca44b651256c359281d26d927ff43badaf1d2276dd770832"}, - {file = "duckdb-0.6.1-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:0925778200090d3d5d8b6bb42b4d05d24db1e8912484ba3b7e7b7f8569f17dcb"}, - {file = "duckdb-0.6.1-cp38-cp38-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:8b544dd04bb851d08bc68b317a7683cec6091547ae75555d075f8c8a7edb626e"}, - {file = "duckdb-0.6.1-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:f2c37d5a0391cf3a3a66e63215968ffb78e6b84f659529fa4bd10478f6203071"}, - {file = "duckdb-0.6.1-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:ce376966260eb5c351fcc6af627a979dbbcae3efeb2e70f85b23aa45a21e289d"}, - {file = "duckdb-0.6.1-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:73c974b09dd08dff5e8bdedba11c7d0aa0fc46ca93954ee7d19e1e18c9883ac1"}, - {file = "duckdb-0.6.1-cp38-cp38-win32.whl", hash = "sha256:bfe39ed3a03e8b1ed764f58f513b37b24afe110d245803a41655d16d391ad9f1"}, - {file = "duckdb-0.6.1-cp38-cp38-win_amd64.whl", hash = "sha256:afa97d982dbe6b125631a17e222142e79bee88f7a13fc4cee92d09285e31ec83"}, - {file = "duckdb-0.6.1-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:c35ff4b1117096ef72d101524df0079da36c3735d52fcf1d907ccffa63bd6202"}, - {file = "duckdb-0.6.1-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:5c54910fbb6de0f21d562e18a5c91540c19876db61b862fc9ffc8e31be8b3f03"}, - {file = "duckdb-0.6.1-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:99a7172563a3ae67d867572ce27cf3962f58e76f491cb7f602f08c2af39213b3"}, - {file = "duckdb-0.6.1-cp39-cp39-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:7363ffe857d00216b659116647fbf1e925cb3895699015d4a4e50b746de13041"}, - {file = "duckdb-0.6.1-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:06c1cef25f896b2284ba048108f645c72fab5c54aa5a6f62f95663f44ff8a79b"}, - {file = "duckdb-0.6.1-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:e92dd6aad7e8c29d002947376b6f5ce28cae29eb3b6b58a64a46cdbfc5cb7943"}, - {file = "duckdb-0.6.1-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:4b280b2d8a01ecd4fe2feab041df70233c534fafbe33a38565b52c1e017529c7"}, - {file = "duckdb-0.6.1-cp39-cp39-win32.whl", hash = "sha256:d9212d76e90b8469743924a4d22bef845be310d0d193d54ae17d9ef1f753cfa7"}, - {file = "duckdb-0.6.1-cp39-cp39-win_amd64.whl", hash = "sha256:00b7be8f67ec1a8edaa8844f521267baa1a795f4c482bfad56c72c26e1862ab2"}, - {file = "duckdb-0.6.1.tar.gz", hash = "sha256:6d26e9f1afcb924a6057785e506810d48332d4764ddc4a5b414d0f2bf0cacfb4"}, -] [package.dependencies] numpy = ">=1.14" @@ -731,10 +357,6 @@ description = "Backport of PEP 654 (exception groups)" category = "dev" optional = false python-versions = ">=3.7" -files = [ - {file = "exceptiongroup-1.0.4-py3-none-any.whl", hash = "sha256:542adf9dea4055530d6e1279602fa5cb11dab2395fa650b8674eaec35fc4a828"}, - {file = "exceptiongroup-1.0.4.tar.gz", hash = "sha256:bd14967b79cd9bdb54d97323216f8fdf533e278df937aa2a90089e7d6e06e5ec"}, -] [package.extras] test = ["pytest (>=6)"] @@ -746,29 +368,6 @@ description = "Fast read/write of AVRO files" category = "dev" optional = false python-versions = ">=3.7" -files = [ - {file = "fastavro-1.7.0-cp310-cp310-macosx_10_14_x86_64.whl", hash = "sha256:ab3387a06e272980fa034f5c62f7063977b77df6416d3d30a4d3b49cc8827566"}, - {file = "fastavro-1.7.0-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:216132bc54da19e97e1531dd69c86282408d4c797749d83b01b3a00862a180de"}, - {file = "fastavro-1.7.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:c20cf6cd8098bb93c2cffd53d03ccea1dcf9ec594a5c83963acf29a2882f8693"}, - {file = "fastavro-1.7.0-cp310-cp310-win_amd64.whl", hash = "sha256:54b60e79506a456bcfc940560fa2c73a7a8e3ddc58a1ae4d94fdd99f6b02aef0"}, - {file = "fastavro-1.7.0-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:4da6d83abd04a804310667f8d1fc23d44e9ed91ed9a9bc9c1fcd906e0c403b12"}, - {file = "fastavro-1.7.0-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:7c664302f94064adeb93403c61c74e07b9710526403eba3b59169f99bb99c55c"}, - {file = "fastavro-1.7.0-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:7aab162f02e64bf82d0282430a3c6ec7a36982b1c5d479e7dcc278e6d62a84b8"}, - {file = "fastavro-1.7.0-cp311-cp311-win_amd64.whl", hash = "sha256:63d0d2a2bb3e85d006c834633be51b105a50b0dc7cc8423b06f30601b532adf4"}, - {file = "fastavro-1.7.0-cp37-cp37m-macosx_10_15_x86_64.whl", hash = "sha256:a07a3245049529d6d9028d664361cc4990e74d035d2303875295e2f7b97eba09"}, - {file = "fastavro-1.7.0-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:7df57d31cb58770a9066790250e9f4ec91242c69e1dc62ea732a6fb2406a8f96"}, - {file = "fastavro-1.7.0-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:b905634b5763ca08c9f7b51486e2c3ae7907f5d9bc48208c69b16ccbe8455e90"}, - {file = "fastavro-1.7.0-cp37-cp37m-win_amd64.whl", hash = "sha256:749206f1cec3d7429546e49db5708f4571497d35181b6b334c4844133f230515"}, - {file = "fastavro-1.7.0-cp38-cp38-macosx_10_15_x86_64.whl", hash = "sha256:a0ebd5c1269085179de4b3f072de274fb66a471ecbc5245bd8684e6f94943c2f"}, - {file = "fastavro-1.7.0-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a7e35b94b692f8cca0096c89abf1937efed66252dea0b3b3165babfb3c289fb7"}, - {file = "fastavro-1.7.0-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:202424a5a7831b773f0f2cc2f82e89ed1726051fd5994f13dc678514144e10d4"}, - {file = "fastavro-1.7.0-cp38-cp38-win_amd64.whl", hash = "sha256:601f8c2ec166bd721f4b12eafe195dd1373d3f8dce4fe2425abd2df3e3968ac7"}, - {file = "fastavro-1.7.0-cp39-cp39-macosx_10_15_x86_64.whl", hash = "sha256:91cc9be740abca894082af4fe3ab9db057d4e5fa783cfa9a94c02ec041bf4346"}, - {file = "fastavro-1.7.0-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:9e4463b00242e4baf52d499aeefab46a26d9dd18d808d4219cd4d21089da540e"}, - {file = "fastavro-1.7.0-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:f7348d318858203bd108e6bcde177d8a6f0590b52bc624d815f26fb6c37029bb"}, - {file = "fastavro-1.7.0-cp39-cp39-win_amd64.whl", hash = "sha256:492e8902063aa1c73170e11c18495fcaa86b71eae3513ef83ba438ca02b16b34"}, - {file = "fastavro-1.7.0.tar.gz", hash = "sha256:4b1205f46489b4032d3155c1ab44d9824be0c7454df98d3a5bd22b78b98f23c8"}, -] [package.extras] codecs = ["lz4", "python-snappy", "zstandard"] @@ -783,10 +382,6 @@ description = "A platform independent file lock." category = "dev" optional = false python-versions = ">=3.7" -files = [ - {file = "filelock-3.8.2-py3-none-any.whl", hash = "sha256:8df285554452285f79c035efb0c861eb33a4bcfa5b7a137016e32e6a90f9792c"}, - {file = "filelock-3.8.2.tar.gz", hash = "sha256:7565f628ea56bfcd8e54e42bdc55da899c85c1abfe1b5bcfd147e9188cebb3b2"}, -] [package.extras] docs = ["furo (>=2022.9.29)", "sphinx (>=5.3)", "sphinx-autodoc-typehints (>=1.19.5)"] @@ -799,82 +394,6 @@ description = "A list-like structure which implements collections.abc.MutableSeq category = "main" optional = true python-versions = ">=3.7" -files = [ - {file = "frozenlist-1.3.3-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:ff8bf625fe85e119553b5383ba0fb6aa3d0ec2ae980295aaefa552374926b3f4"}, - {file = "frozenlist-1.3.3-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:dfbac4c2dfcc082fcf8d942d1e49b6aa0766c19d3358bd86e2000bf0fa4a9cf0"}, - {file = "frozenlist-1.3.3-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:b1c63e8d377d039ac769cd0926558bb7068a1f7abb0f003e3717ee003ad85530"}, - {file = "frozenlist-1.3.3-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:7fdfc24dcfce5b48109867c13b4cb15e4660e7bd7661741a391f821f23dfdca7"}, - {file = "frozenlist-1.3.3-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:2c926450857408e42f0bbc295e84395722ce74bae69a3b2aa2a65fe22cb14b99"}, - {file = "frozenlist-1.3.3-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:1841e200fdafc3d51f974d9d377c079a0694a8f06de2e67b48150328d66d5483"}, - {file = "frozenlist-1.3.3-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:f470c92737afa7d4c3aacc001e335062d582053d4dbe73cda126f2d7031068dd"}, - {file = "frozenlist-1.3.3-cp310-cp310-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:783263a4eaad7c49983fe4b2e7b53fa9770c136c270d2d4bbb6d2192bf4d9caf"}, - {file = "frozenlist-1.3.3-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:924620eef691990dfb56dc4709f280f40baee568c794b5c1885800c3ecc69816"}, - {file = "frozenlist-1.3.3-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:ae4dc05c465a08a866b7a1baf360747078b362e6a6dbeb0c57f234db0ef88ae0"}, - {file = "frozenlist-1.3.3-cp310-cp310-musllinux_1_1_ppc64le.whl", hash = "sha256:bed331fe18f58d844d39ceb398b77d6ac0b010d571cba8267c2e7165806b00ce"}, - {file = "frozenlist-1.3.3-cp310-cp310-musllinux_1_1_s390x.whl", hash = "sha256:02c9ac843e3390826a265e331105efeab489ffaf4dd86384595ee8ce6d35ae7f"}, - {file = "frozenlist-1.3.3-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:9545a33965d0d377b0bc823dcabf26980e77f1b6a7caa368a365a9497fb09420"}, - {file = "frozenlist-1.3.3-cp310-cp310-win32.whl", hash = "sha256:d5cd3ab21acbdb414bb6c31958d7b06b85eeb40f66463c264a9b343a4e238642"}, - {file = "frozenlist-1.3.3-cp310-cp310-win_amd64.whl", hash = "sha256:b756072364347cb6aa5b60f9bc18e94b2f79632de3b0190253ad770c5df17db1"}, - {file = "frozenlist-1.3.3-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:b4395e2f8d83fbe0c627b2b696acce67868793d7d9750e90e39592b3626691b7"}, - {file = "frozenlist-1.3.3-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:14143ae966a6229350021384870458e4777d1eae4c28d1a7aa47f24d030e6678"}, - {file = "frozenlist-1.3.3-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:5d8860749e813a6f65bad8285a0520607c9500caa23fea6ee407e63debcdbef6"}, - {file = "frozenlist-1.3.3-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:23d16d9f477bb55b6154654e0e74557040575d9d19fe78a161bd33d7d76808e8"}, - {file = "frozenlist-1.3.3-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:eb82dbba47a8318e75f679690190c10a5e1f447fbf9df41cbc4c3afd726d88cb"}, - {file = "frozenlist-1.3.3-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:9309869032abb23d196cb4e4db574232abe8b8be1339026f489eeb34a4acfd91"}, - {file = "frozenlist-1.3.3-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:a97b4fe50b5890d36300820abd305694cb865ddb7885049587a5678215782a6b"}, - {file = "frozenlist-1.3.3-cp311-cp311-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:c188512b43542b1e91cadc3c6c915a82a5eb95929134faf7fd109f14f9892ce4"}, - {file = "frozenlist-1.3.3-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:303e04d422e9b911a09ad499b0368dc551e8c3cd15293c99160c7f1f07b59a48"}, - {file = "frozenlist-1.3.3-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:0771aed7f596c7d73444c847a1c16288937ef988dc04fb9f7be4b2aa91db609d"}, - {file = "frozenlist-1.3.3-cp311-cp311-musllinux_1_1_ppc64le.whl", hash = "sha256:66080ec69883597e4d026f2f71a231a1ee9887835902dbe6b6467d5a89216cf6"}, - {file = "frozenlist-1.3.3-cp311-cp311-musllinux_1_1_s390x.whl", hash = "sha256:41fe21dc74ad3a779c3d73a2786bdf622ea81234bdd4faf90b8b03cad0c2c0b4"}, - {file = "frozenlist-1.3.3-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:f20380df709d91525e4bee04746ba612a4df0972c1b8f8e1e8af997e678c7b81"}, - {file = "frozenlist-1.3.3-cp311-cp311-win32.whl", hash = "sha256:f30f1928162e189091cf4d9da2eac617bfe78ef907a761614ff577ef4edfb3c8"}, - {file = "frozenlist-1.3.3-cp311-cp311-win_amd64.whl", hash = "sha256:a6394d7dadd3cfe3f4b3b186e54d5d8504d44f2d58dcc89d693698e8b7132b32"}, - {file = "frozenlist-1.3.3-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:8df3de3a9ab8325f94f646609a66cbeeede263910c5c0de0101079ad541af332"}, - {file = "frozenlist-1.3.3-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:0693c609e9742c66ba4870bcee1ad5ff35462d5ffec18710b4ac89337ff16e27"}, - {file = "frozenlist-1.3.3-cp37-cp37m-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:cd4210baef299717db0a600d7a3cac81d46ef0e007f88c9335db79f8979c0d3d"}, - {file = "frozenlist-1.3.3-cp37-cp37m-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:394c9c242113bfb4b9aa36e2b80a05ffa163a30691c7b5a29eba82e937895d5e"}, - {file = "frozenlist-1.3.3-cp37-cp37m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:6327eb8e419f7d9c38f333cde41b9ae348bec26d840927332f17e887a8dcb70d"}, - {file = "frozenlist-1.3.3-cp37-cp37m-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:2e24900aa13212e75e5b366cb9065e78bbf3893d4baab6052d1aca10d46d944c"}, - {file = "frozenlist-1.3.3-cp37-cp37m-musllinux_1_1_aarch64.whl", hash = "sha256:3843f84a6c465a36559161e6c59dce2f2ac10943040c2fd021cfb70d58c4ad56"}, - {file = "frozenlist-1.3.3-cp37-cp37m-musllinux_1_1_i686.whl", hash = "sha256:84610c1502b2461255b4c9b7d5e9c48052601a8957cd0aea6ec7a7a1e1fb9420"}, - {file = "frozenlist-1.3.3-cp37-cp37m-musllinux_1_1_ppc64le.whl", hash = "sha256:c21b9aa40e08e4f63a2f92ff3748e6b6c84d717d033c7b3438dd3123ee18f70e"}, - {file = "frozenlist-1.3.3-cp37-cp37m-musllinux_1_1_s390x.whl", hash = "sha256:efce6ae830831ab6a22b9b4091d411698145cb9b8fc869e1397ccf4b4b6455cb"}, - {file = "frozenlist-1.3.3-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:40de71985e9042ca00b7953c4f41eabc3dc514a2d1ff534027f091bc74416401"}, - {file = "frozenlist-1.3.3-cp37-cp37m-win32.whl", hash = "sha256:180c00c66bde6146a860cbb81b54ee0df350d2daf13ca85b275123bbf85de18a"}, - {file = "frozenlist-1.3.3-cp37-cp37m-win_amd64.whl", hash = "sha256:9bbbcedd75acdfecf2159663b87f1bb5cfc80e7cd99f7ddd9d66eb98b14a8411"}, - {file = "frozenlist-1.3.3-cp38-cp38-macosx_10_9_universal2.whl", hash = "sha256:034a5c08d36649591be1cbb10e09da9f531034acfe29275fc5454a3b101ce41a"}, - {file = "frozenlist-1.3.3-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:ba64dc2b3b7b158c6660d49cdb1d872d1d0bf4e42043ad8d5006099479a194e5"}, - {file = "frozenlist-1.3.3-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:47df36a9fe24054b950bbc2db630d508cca3aa27ed0566c0baf661225e52c18e"}, - {file = "frozenlist-1.3.3-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:008a054b75d77c995ea26629ab3a0c0d7281341f2fa7e1e85fa6153ae29ae99c"}, - {file = "frozenlist-1.3.3-cp38-cp38-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:841ea19b43d438a80b4de62ac6ab21cfe6827bb8a9dc62b896acc88eaf9cecba"}, - {file = "frozenlist-1.3.3-cp38-cp38-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:e235688f42b36be2b6b06fc37ac2126a73b75fb8d6bc66dd632aa35286238703"}, - {file = "frozenlist-1.3.3-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:ca713d4af15bae6e5d79b15c10c8522859a9a89d3b361a50b817c98c2fb402a2"}, - {file = "frozenlist-1.3.3-cp38-cp38-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:9ac5995f2b408017b0be26d4a1d7c61bce106ff3d9e3324374d66b5964325448"}, - {file = "frozenlist-1.3.3-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:a4ae8135b11652b08a8baf07631d3ebfe65a4c87909dbef5fa0cdde440444ee4"}, - {file = "frozenlist-1.3.3-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:4ea42116ceb6bb16dbb7d526e242cb6747b08b7710d9782aa3d6732bd8d27649"}, - {file = "frozenlist-1.3.3-cp38-cp38-musllinux_1_1_ppc64le.whl", hash = "sha256:810860bb4bdce7557bc0febb84bbd88198b9dbc2022d8eebe5b3590b2ad6c842"}, - {file = "frozenlist-1.3.3-cp38-cp38-musllinux_1_1_s390x.whl", hash = "sha256:ee78feb9d293c323b59a6f2dd441b63339a30edf35abcb51187d2fc26e696d13"}, - {file = "frozenlist-1.3.3-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:0af2e7c87d35b38732e810befb9d797a99279cbb85374d42ea61c1e9d23094b3"}, - {file = "frozenlist-1.3.3-cp38-cp38-win32.whl", hash = "sha256:899c5e1928eec13fd6f6d8dc51be23f0d09c5281e40d9cf4273d188d9feeaf9b"}, - {file = "frozenlist-1.3.3-cp38-cp38-win_amd64.whl", hash = "sha256:7f44e24fa70f6fbc74aeec3e971f60a14dde85da364aa87f15d1be94ae75aeef"}, - {file = "frozenlist-1.3.3-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:2b07ae0c1edaa0a36339ec6cce700f51b14a3fc6545fdd32930d2c83917332cf"}, - {file = "frozenlist-1.3.3-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:ebb86518203e12e96af765ee89034a1dbb0c3c65052d1b0c19bbbd6af8a145e1"}, - {file = "frozenlist-1.3.3-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:5cf820485f1b4c91e0417ea0afd41ce5cf5965011b3c22c400f6d144296ccbc0"}, - {file = "frozenlist-1.3.3-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:5c11e43016b9024240212d2a65043b70ed8dfd3b52678a1271972702d990ac6d"}, - {file = "frozenlist-1.3.3-cp39-cp39-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:8fa3c6e3305aa1146b59a09b32b2e04074945ffcfb2f0931836d103a2c38f936"}, - {file = "frozenlist-1.3.3-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:352bd4c8c72d508778cf05ab491f6ef36149f4d0cb3c56b1b4302852255d05d5"}, - {file = "frozenlist-1.3.3-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:65a5e4d3aa679610ac6e3569e865425b23b372277f89b5ef06cf2cdaf1ebf22b"}, - {file = "frozenlist-1.3.3-cp39-cp39-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:b1e2c1185858d7e10ff045c496bbf90ae752c28b365fef2c09cf0fa309291669"}, - {file = "frozenlist-1.3.3-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:f163d2fd041c630fed01bc48d28c3ed4a3b003c00acd396900e11ee5316b56bb"}, - {file = "frozenlist-1.3.3-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:05cdb16d09a0832eedf770cb7bd1fe57d8cf4eaf5aced29c4e41e3f20b30a784"}, - {file = "frozenlist-1.3.3-cp39-cp39-musllinux_1_1_ppc64le.whl", hash = "sha256:8bae29d60768bfa8fb92244b74502b18fae55a80eac13c88eb0b496d4268fd2d"}, - {file = "frozenlist-1.3.3-cp39-cp39-musllinux_1_1_s390x.whl", hash = "sha256:eedab4c310c0299961ac285591acd53dc6723a1ebd90a57207c71f6e0c2153ab"}, - {file = "frozenlist-1.3.3-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:3bbdf44855ed8f0fbcd102ef05ec3012d6a4fd7c7562403f76ce6a52aeffb2b1"}, - {file = "frozenlist-1.3.3-cp39-cp39-win32.whl", hash = "sha256:efa568b885bca461f7c7b9e032655c0c143d305bf01c30caf6db2854a4532b38"}, - {file = "frozenlist-1.3.3-cp39-cp39-win_amd64.whl", hash = "sha256:cfe33efc9cb900a4c46f91a5ceba26d6df370ffddd9ca386eb1d4f0ad97b9ea9"}, - {file = "frozenlist-1.3.3.tar.gz", hash = "sha256:58bcc55721e8a90b88332d6cd441261ebb22342e238296bb330968952fbb3a6a"}, -] [[package]] name = "fsspec" @@ -883,10 +402,6 @@ description = "File-system specification" category = "main" optional = false python-versions = ">=3.7" -files = [ - {file = "fsspec-2022.10.0-py3-none-any.whl", hash = "sha256:6b7c6ab3b476cdf17efcfeccde7fca28ef5a48f73a71010aaceec5fc15bf9ebf"}, - {file = "fsspec-2022.10.0.tar.gz", hash = "sha256:cb6092474e90487a51de768170f3afa50ca8982c26150a59072b16433879ff1d"}, -] [package.extras] abfs = ["adlfs"] @@ -918,10 +433,6 @@ description = "File identification library for Python" category = "dev" optional = false python-versions = ">=3.7" -files = [ - {file = "identify-2.5.10-py2.py3-none-any.whl", hash = "sha256:fb7c2feaeca6976a3ffa31ec3236a6911fbc51aec9acc111de2aed99f244ade2"}, - {file = "identify-2.5.10.tar.gz", hash = "sha256:dce9e31fee7dbc45fea36a9e855c316b8fbf807e65a862f160840bb5a2bf5dfd"}, -] [package.extras] license = ["ukkonen"] @@ -933,10 +444,6 @@ description = "Internationalized Domain Names in Applications (IDNA)" category = "main" optional = false python-versions = ">=3.5" -files = [ - {file = "idna-3.4-py3-none-any.whl", hash = "sha256:90b77e79eaa3eba6de819a0c442c0b4ceefc341a7a2ab77d7562bf49f425c5c2"}, - {file = "idna-3.4.tar.gz", hash = "sha256:814f528e8dead7d329833b91c5faa87d60bf71824cd12a7530b5526063d02cb4"}, -] [[package]] name = "importlib-metadata" @@ -945,10 +452,6 @@ description = "Read metadata from Python packages" category = "dev" optional = false python-versions = ">=3.7" -files = [ - {file = "importlib_metadata-5.1.0-py3-none-any.whl", hash = "sha256:d84d17e21670ec07990e1044a99efe8d615d860fd176fc29ef5c306068fda313"}, - {file = "importlib_metadata-5.1.0.tar.gz", hash = "sha256:d5059f9f1e8e41f80e9c56c2ee58811450c31984dfa625329ffd7c0dad88a73b"}, -] [package.dependencies] zipp = ">=0.5" @@ -965,10 +468,6 @@ description = "iniconfig: brain-dead simple config-ini parsing" category = "dev" optional = false python-versions = "*" -files = [ - {file = "iniconfig-1.1.1-py2.py3-none-any.whl", hash = "sha256:011e24c64b7f47f6ebd835bb12a743f2fbe9a26d4cecaa7f53bc4f35ee9da8b3"}, - {file = "iniconfig-1.1.1.tar.gz", hash = "sha256:bc3af051d7d14b2ee5ef9969666def0cd1a000e121eaea580d4a313df4b37f32"}, -] [[package]] name = "isodate" @@ -977,10 +476,6 @@ description = "An ISO 8601 date/time/duration parser and formatter" category = "main" optional = true python-versions = "*" -files = [ - {file = "isodate-0.6.1-py2.py3-none-any.whl", hash = "sha256:0751eece944162659049d35f4f549ed815792b38793f07cf73381c1c87cbed96"}, - {file = "isodate-0.6.1.tar.gz", hash = "sha256:48c5881de7e8b0a0d648cb024c8062dc84e7b840ed81e864c7614fd3c127bde9"}, -] [package.dependencies] six = "*" @@ -992,10 +487,6 @@ description = "A very fast and expressive template engine." category = "dev" optional = false python-versions = ">=3.7" -files = [ - {file = "Jinja2-3.1.2-py3-none-any.whl", hash = "sha256:6088930bfe239f0e6710546ab9c19c9ef35e29792895fed6e6e31a023a182a61"}, - {file = "Jinja2-3.1.2.tar.gz", hash = "sha256:31351a702a408a9e7595a8fc6150fc3f43bb6bf7e319770cbc0db9df9437e852"}, -] [package.dependencies] MarkupSafe = ">=2.0" @@ -1010,10 +501,6 @@ description = "JSON Matching Expressions" category = "main" optional = false python-versions = ">=3.7" -files = [ - {file = "jmespath-1.0.1-py3-none-any.whl", hash = "sha256:02e2e4cc71b5bcab88332eebf907519190dd9e6e82107fa7f83b1003a6252980"}, - {file = "jmespath-1.0.1.tar.gz", hash = "sha256:90261b206d6defd58fdd5e85f478bf633a2901798906be2ad389150c5c60edbe"}, -] [[package]] name = "markupsafe" @@ -1022,48 +509,6 @@ description = "Safely add untrusted strings to HTML/XML markup." category = "dev" optional = false python-versions = ">=3.7" -files = [ - {file = "MarkupSafe-2.1.1-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:86b1f75c4e7c2ac2ccdaec2b9022845dbb81880ca318bb7a0a01fbf7813e3812"}, - {file = "MarkupSafe-2.1.1-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:f121a1420d4e173a5d96e47e9a0c0dcff965afdf1626d28de1460815f7c4ee7a"}, - {file = "MarkupSafe-2.1.1-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a49907dd8420c5685cfa064a1335b6754b74541bbb3706c259c02ed65b644b3e"}, - {file = "MarkupSafe-2.1.1-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:10c1bfff05d95783da83491be968e8fe789263689c02724e0c691933c52994f5"}, - {file = "MarkupSafe-2.1.1-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:b7bd98b796e2b6553da7225aeb61f447f80a1ca64f41d83612e6139ca5213aa4"}, - {file = "MarkupSafe-2.1.1-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:b09bf97215625a311f669476f44b8b318b075847b49316d3e28c08e41a7a573f"}, - {file = "MarkupSafe-2.1.1-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:694deca8d702d5db21ec83983ce0bb4b26a578e71fbdbd4fdcd387daa90e4d5e"}, - {file = "MarkupSafe-2.1.1-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:efc1913fd2ca4f334418481c7e595c00aad186563bbc1ec76067848c7ca0a933"}, - {file = "MarkupSafe-2.1.1-cp310-cp310-win32.whl", hash = "sha256:4a33dea2b688b3190ee12bd7cfa29d39c9ed176bda40bfa11099a3ce5d3a7ac6"}, - {file = "MarkupSafe-2.1.1-cp310-cp310-win_amd64.whl", hash = "sha256:dda30ba7e87fbbb7eab1ec9f58678558fd9a6b8b853530e176eabd064da81417"}, - {file = "MarkupSafe-2.1.1-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:671cd1187ed5e62818414afe79ed29da836dde67166a9fac6d435873c44fdd02"}, - {file = "MarkupSafe-2.1.1-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:3799351e2336dc91ea70b034983ee71cf2f9533cdff7c14c90ea126bfd95d65a"}, - {file = "MarkupSafe-2.1.1-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:e72591e9ecd94d7feb70c1cbd7be7b3ebea3f548870aa91e2732960fa4d57a37"}, - {file = "MarkupSafe-2.1.1-cp37-cp37m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:6fbf47b5d3728c6aea2abb0589b5d30459e369baa772e0f37a0320185e87c980"}, - {file = "MarkupSafe-2.1.1-cp37-cp37m-musllinux_1_1_aarch64.whl", hash = "sha256:d5ee4f386140395a2c818d149221149c54849dfcfcb9f1debfe07a8b8bd63f9a"}, - {file = "MarkupSafe-2.1.1-cp37-cp37m-musllinux_1_1_i686.whl", hash = "sha256:bcb3ed405ed3222f9904899563d6fc492ff75cce56cba05e32eff40e6acbeaa3"}, - {file = "MarkupSafe-2.1.1-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:e1c0b87e09fa55a220f058d1d49d3fb8df88fbfab58558f1198e08c1e1de842a"}, - {file = "MarkupSafe-2.1.1-cp37-cp37m-win32.whl", hash = "sha256:8dc1c72a69aa7e082593c4a203dcf94ddb74bb5c8a731e4e1eb68d031e8498ff"}, - {file = "MarkupSafe-2.1.1-cp37-cp37m-win_amd64.whl", hash = "sha256:97a68e6ada378df82bc9f16b800ab77cbf4b2fada0081794318520138c088e4a"}, - {file = "MarkupSafe-2.1.1-cp38-cp38-macosx_10_9_universal2.whl", hash = "sha256:e8c843bbcda3a2f1e3c2ab25913c80a3c5376cd00c6e8c4a86a89a28c8dc5452"}, - {file = "MarkupSafe-2.1.1-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:0212a68688482dc52b2d45013df70d169f542b7394fc744c02a57374a4207003"}, - {file = "MarkupSafe-2.1.1-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:8e576a51ad59e4bfaac456023a78f6b5e6e7651dcd383bcc3e18d06f9b55d6d1"}, - {file = "MarkupSafe-2.1.1-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:4b9fe39a2ccc108a4accc2676e77da025ce383c108593d65cc909add5c3bd601"}, - {file = "MarkupSafe-2.1.1-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:96e37a3dc86e80bf81758c152fe66dbf60ed5eca3d26305edf01892257049925"}, - {file = "MarkupSafe-2.1.1-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:6d0072fea50feec76a4c418096652f2c3238eaa014b2f94aeb1d56a66b41403f"}, - {file = "MarkupSafe-2.1.1-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:089cf3dbf0cd6c100f02945abeb18484bd1ee57a079aefd52cffd17fba910b88"}, - {file = "MarkupSafe-2.1.1-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:6a074d34ee7a5ce3effbc526b7083ec9731bb3cbf921bbe1d3005d4d2bdb3a63"}, - {file = "MarkupSafe-2.1.1-cp38-cp38-win32.whl", hash = "sha256:421be9fbf0ffe9ffd7a378aafebbf6f4602d564d34be190fc19a193232fd12b1"}, - {file = "MarkupSafe-2.1.1-cp38-cp38-win_amd64.whl", hash = "sha256:fc7b548b17d238737688817ab67deebb30e8073c95749d55538ed473130ec0c7"}, - {file = "MarkupSafe-2.1.1-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:e04e26803c9c3851c931eac40c695602c6295b8d432cbe78609649ad9bd2da8a"}, - {file = "MarkupSafe-2.1.1-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:b87db4360013327109564f0e591bd2a3b318547bcef31b468a92ee504d07ae4f"}, - {file = "MarkupSafe-2.1.1-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:99a2a507ed3ac881b975a2976d59f38c19386d128e7a9a18b7df6fff1fd4c1d6"}, - {file = "MarkupSafe-2.1.1-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:56442863ed2b06d19c37f94d999035e15ee982988920e12a5b4ba29b62ad1f77"}, - {file = "MarkupSafe-2.1.1-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:3ce11ee3f23f79dbd06fb3d63e2f6af7b12db1d46932fe7bd8afa259a5996603"}, - {file = "MarkupSafe-2.1.1-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:33b74d289bd2f5e527beadcaa3f401e0df0a89927c1559c8566c066fa4248ab7"}, - {file = "MarkupSafe-2.1.1-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:43093fb83d8343aac0b1baa75516da6092f58f41200907ef92448ecab8825135"}, - {file = "MarkupSafe-2.1.1-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:8e3dcf21f367459434c18e71b2a9532d96547aef8a871872a5bd69a715c15f96"}, - {file = "MarkupSafe-2.1.1-cp39-cp39-win32.whl", hash = "sha256:d4306c36ca495956b6d568d276ac11fdd9c30a36f1b6eb928070dc5360b22e1c"}, - {file = "MarkupSafe-2.1.1-cp39-cp39-win_amd64.whl", hash = "sha256:46d00d6cfecdde84d40e572d63735ef81423ad31184100411e6e3388d405e247"}, - {file = "MarkupSafe-2.1.1.tar.gz", hash = "sha256:7f91197cc9e48f989d12e4e6fbc46495c446636dfc81b9ccf50bb0ec74b91d4b"}, -] [[package]] name = "mmhash3" @@ -1072,42 +517,6 @@ description = "Python wrapper for MurmurHash (MurmurHash3), a set of fast and ro category = "main" optional = false python-versions = "*" -files = [ - {file = "mmhash3-3.0.1-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:47deea30cd8d3d5cd52dc740902a4c70383bfe8248eac29d0877fe63e03c2713"}, - {file = "mmhash3-3.0.1-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:ecdaf4d1de617818bf05cd526ca558db6010beeba7ea9e19f695f2bdcac0e0a4"}, - {file = "mmhash3-3.0.1-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:4675585617584e9e1aafa3a90ac0ac9a437257c507748d97de8b21977e9d6745"}, - {file = "mmhash3-3.0.1-cp310-cp310-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:ebfd0c2af09b41f0fe1dd260799bda90a0fc7eba4477ccaeb3951527700cd58f"}, - {file = "mmhash3-3.0.1-cp310-cp310-win32.whl", hash = "sha256:68587dec7b8acdb7528fd511e295d8b5ccfe26022923a69867e1822f0fdb4c44"}, - {file = "mmhash3-3.0.1-cp310-cp310-win_amd64.whl", hash = "sha256:54954ebe3a614f19ab4cfe24fa66ea093c493d9fac0533d002dd64c2540a0c99"}, - {file = "mmhash3-3.0.1-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:b172f3bd3220b0bf56fd7cc760fd8a9033def47103611d63fe867003079a1256"}, - {file = "mmhash3-3.0.1-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:de7895eafabc32f7c0c09a81a624921f536768de6e438e3de69e3e954a4d7072"}, - {file = "mmhash3-3.0.1-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:b4b0914effe4ddd8d33149e3508564c17719465b0cc81691c4fa50d5e0e14f80"}, - {file = "mmhash3-3.0.1-cp311-cp311-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:c0575050ac691475938df1ad03d8738c5bd1eadef62093e76157ebb7f2df0946"}, - {file = "mmhash3-3.0.1-cp311-cp311-win32.whl", hash = "sha256:22f92f0f88f28b215357acd346362fa9f7c9fffb436beb42cc4b442b676dbaa3"}, - {file = "mmhash3-3.0.1-cp311-cp311-win_amd64.whl", hash = "sha256:538240ab7936bf71b18304e5a7e7fd3c4c2fab103330ea99584bb4f777299a2b"}, - {file = "mmhash3-3.0.1-cp36-cp36m-macosx_10_9_x86_64.whl", hash = "sha256:ca791bfb311e36ce13998e4632262ed4b95da9d3461941e18b6690760171a045"}, - {file = "mmhash3-3.0.1-cp36-cp36m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:b41708f72c6aa2a49ada1f0b61e85c05cd8389ad31d463fd5bca29999a4d5f9c"}, - {file = "mmhash3-3.0.1-cp36-cp36m-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:a3ce9b4533ddc0a88ba045a27309714c3b127bd05e57fd252d1d5a71d4247ea7"}, - {file = "mmhash3-3.0.1-cp36-cp36m-win32.whl", hash = "sha256:bfafeb96fdeb10db8767d06e1f07b6fdcddba4aaa0dd15058561a49f7ae45345"}, - {file = "mmhash3-3.0.1-cp36-cp36m-win_amd64.whl", hash = "sha256:97fe077b24c948709ed2afc749bf6285d407bc54ff12c63d2dc86678c38a0b8e"}, - {file = "mmhash3-3.0.1-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:0cfd91ccd5fca1ba7ee925297131a15dfb94c714bfe6ba0fb3b1ca78b12bbfec"}, - {file = "mmhash3-3.0.1-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:6d51b1005233141ce7394531af40a3f0fc1f274467bf8dff44dcf7987924fe58"}, - {file = "mmhash3-3.0.1-cp37-cp37m-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:855c67b100e37df166acf79cdff58fa8f9f6c48be0d1e1b6e9ad0fa34a9661ef"}, - {file = "mmhash3-3.0.1-cp37-cp37m-win32.whl", hash = "sha256:bb3030df1334fd665427f8be8e8ce4f04aeab7f6010ce4f2c128f0099bdab96f"}, - {file = "mmhash3-3.0.1-cp37-cp37m-win_amd64.whl", hash = "sha256:1545e1177294afe4912d5a5e401c7fa9b799dd109b30289e7af74d5b07e7c474"}, - {file = "mmhash3-3.0.1-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:2479899e7dda834a671991a1098a691ab1c2eaa20c3e939d691ca4a19361cfe0"}, - {file = "mmhash3-3.0.1-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:c9056196d5e3d3d844433a63d806a683f710ab3aaf1c910550c7746464bc43ae"}, - {file = "mmhash3-3.0.1-cp38-cp38-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:a0d4c307af0bf70207305f70f131898be071d1b19a89f462b13487f5c25e8d4e"}, - {file = "mmhash3-3.0.1-cp38-cp38-win32.whl", hash = "sha256:5f885f65e329fd14bc38debac4a79eacf381e856965d9c65c4d1c6946ea190d0"}, - {file = "mmhash3-3.0.1-cp38-cp38-win_amd64.whl", hash = "sha256:3b42d0bda5e1cd22c18b16887b0521060fb59d0aaaaf033feacbc0a2492d20fe"}, - {file = "mmhash3-3.0.1-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:d3f333286bb87aa9dc6bd8e7960a55a27b011a401f24b889a50e6d219f65e7c9"}, - {file = "mmhash3-3.0.1-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:6b7ef2eb95a18bcd02ce0d3e047adde3a025afd96c1d266a8a0d44574f44a307"}, - {file = "mmhash3-3.0.1-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:d6ac8a5f511c60f341bf9cae462bb4941abb149d98464ba5f4f4548875b601c6"}, - {file = "mmhash3-3.0.1-cp39-cp39-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:efef9e632e6e248e46f52d108a5ebd1f28edaf427b7fd47ebf97dbff4b2cab81"}, - {file = "mmhash3-3.0.1-cp39-cp39-win32.whl", hash = "sha256:bdac06d72e448c67afb12e758b203d875e29d4097bb279a38a5649d44b518ba7"}, - {file = "mmhash3-3.0.1-cp39-cp39-win_amd64.whl", hash = "sha256:0baeaa20cac5f75ed14f28056826bd9d9c8b2354b382073f3fd5190708992a0d"}, - {file = "mmhash3-3.0.1.tar.gz", hash = "sha256:a00d68f4a1cc434b9501513c8a29e18ed1ddad383677d72b41d71d0d862348af"}, -] [[package]] name = "moto" @@ -1116,10 +525,6 @@ description = "A library that allows your python tests to easily mock out the bo category = "dev" optional = false python-versions = ">=3.6" -files = [ - {file = "moto-4.0.11-py3-none-any.whl", hash = "sha256:704d6d38a4e6fe49e1fe9c6b4127ca46c66aac00368149bc1f1d70a0ceff8846"}, - {file = "moto-4.0.11.tar.gz", hash = "sha256:a6388de4a746e0b509286e1d7e70f86900b4f69ec65f6c92c47e570f95d05b14"}, -] [package.dependencies] boto3 = ">=1.9.201" @@ -1165,10 +570,6 @@ description = "The Microsoft Authentication Library (MSAL) for Python library en category = "main" optional = true python-versions = "*" -files = [ - {file = "msal-1.20.0-py2.py3-none-any.whl", hash = "sha256:d2f1c26368ecdc28c8657d457352faa0b81b1845a7b889d8676787721ba86792"}, - {file = "msal-1.20.0.tar.gz", hash = "sha256:78344cd4c91d6134a593b5e3e45541e666e37b747ff8a6316c3668dd1e6ab6b2"}, -] [package.dependencies] cryptography = ">=0.6,<41" @@ -1185,10 +586,6 @@ description = "Microsoft Authentication Library extensions (MSAL EX) provides a category = "main" optional = true python-versions = "*" -files = [ - {file = "msal-extensions-1.0.0.tar.gz", hash = "sha256:c676aba56b0cce3783de1b5c5ecfe828db998167875126ca4b47dc6436451354"}, - {file = "msal_extensions-1.0.0-py2.py3-none-any.whl", hash = "sha256:91e3db9620b822d0ed2b4d1850056a0f133cba04455e62f11612e40f5502f2ee"}, -] [package.dependencies] msal = ">=0.4.1,<2.0.0" @@ -1204,10 +601,6 @@ description = "AutoRest swagger generator Python client runtime." category = "main" optional = true python-versions = ">=3.6" -files = [ - {file = "msrest-0.7.1-py3-none-any.whl", hash = "sha256:21120a810e1233e5e6cc7fe40b474eeb4ec6f757a15d7cf86702c369f9567c32"}, - {file = "msrest-0.7.1.zip", hash = "sha256:6e7661f46f3afd88b75667b7187a92829924446c7ea1d169be8c4bb7eeb788b9"}, -] [package.dependencies] azure-core = ">=1.24.0" @@ -1226,82 +619,6 @@ description = "multidict implementation" category = "main" optional = true python-versions = ">=3.7" -files = [ - {file = "multidict-6.0.3-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:73009ea04205966d47e16d98686ac5c438af23a1bb30b48a2c5da3423ec9ce37"}, - {file = "multidict-6.0.3-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:8b92a9f3ab904397a33b193000dc4de7318ea175c4c460a1e154c415f9008e3d"}, - {file = "multidict-6.0.3-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:578bfcb16f4b8675ef71b960c00f174b0426e0eeb796bab6737389d8288eb827"}, - {file = "multidict-6.0.3-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:f1650ea41c408755da5eed52ac6ccbc8938ccc3e698d81e6f6a1be02ff2a0945"}, - {file = "multidict-6.0.3-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:d52442e7c951e4c9ee591d6047706e66923d248d83958bbf99b8b19515fffaef"}, - {file = "multidict-6.0.3-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:ad7d66422b9cc51125509229693d27e18c08f2dea3ac9de408d821932b1b3759"}, - {file = "multidict-6.0.3-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:6cd14e61f0da2a2cfb9fe05bfced2a1ed7063ce46a7a8cd473be4973de9a7f91"}, - {file = "multidict-6.0.3-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:190626ced82d4cc567a09e7346340d380154a493bac6905e0095d8158cdf1e38"}, - {file = "multidict-6.0.3-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:791458a1f7d1b4ab3bd9e93e0dcd1d59ef7ee9aa051dcd1ea030e62e49b923fd"}, - {file = "multidict-6.0.3-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:b46e79a9f4db53897d17bc64a39d1c7c2be3e3d4f8dba6d6730a2b13ddf0f986"}, - {file = "multidict-6.0.3-cp310-cp310-musllinux_1_1_ppc64le.whl", hash = "sha256:e4a095e18847c12ec20e55326ab8782d9c2d599400a3a2f174fab4796875d0e2"}, - {file = "multidict-6.0.3-cp310-cp310-musllinux_1_1_s390x.whl", hash = "sha256:fb6c3dc3d65014d2c782f5acf0b3ba14e639c6c33d3ed8932ead76b9080b3544"}, - {file = "multidict-6.0.3-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:3541882266247c7cd3dba78d6ef28dbe704774df60c9e4231edaa4493522e614"}, - {file = "multidict-6.0.3-cp310-cp310-win32.whl", hash = "sha256:67090b17a0a5be5704fd109f231ee73cefb1b3802d41288d6378b5df46ae89ba"}, - {file = "multidict-6.0.3-cp310-cp310-win_amd64.whl", hash = "sha256:36df958b15639e40472adaa4f0c2c7828fe680f894a6b48c4ce229f59a6a798b"}, - {file = "multidict-6.0.3-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:5b51969503709415a35754954c2763f536a70b8bf7360322b2edb0c0a44391f6"}, - {file = "multidict-6.0.3-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:24e8d513bfcaadc1f8b0ebece3ff50961951c54b07d5a775008a882966102418"}, - {file = "multidict-6.0.3-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:d325d61cac602976a5d47b19eaa7d04e3daf4efce2164c630219885087234102"}, - {file = "multidict-6.0.3-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:26fbbe17f8a7211b623502d2bf41022a51da3025142401417c765bf9a56fed4c"}, - {file = "multidict-6.0.3-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:4fb3fe591956d8841882c463f934c9f7485cfd5f763a08c0d467b513dc18ef89"}, - {file = "multidict-6.0.3-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:e1925f78a543b94c3d46274c66a366fee8a263747060220ed0188e5f3eeea1c0"}, - {file = "multidict-6.0.3-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:21e1ce0b187c4e93112304dcde2aa18922fdbe8fb4f13d8aa72a5657bce0563a"}, - {file = "multidict-6.0.3-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:e07c24018986fb00d6e7eafca8fcd6e05095649e17fcf0e33a592caaa62a78b9"}, - {file = "multidict-6.0.3-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:114a4ab3e5cfbc56c4b6697686ecb92376c7e8c56893ef20547921552f8bdf57"}, - {file = "multidict-6.0.3-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:4ccf55f28066b4f08666764a957c2b7c241c7547b0921d69c7ceab5f74fe1a45"}, - {file = "multidict-6.0.3-cp311-cp311-musllinux_1_1_ppc64le.whl", hash = "sha256:9d359b0a962e052b713647ac1f13eabf2263167b149ed1e27d5c579f5c8c7d2c"}, - {file = "multidict-6.0.3-cp311-cp311-musllinux_1_1_s390x.whl", hash = "sha256:df7b4cee3ff31b3335aba602f8d70dbc641e5b7164b1e9565570c9d3c536a438"}, - {file = "multidict-6.0.3-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:ee9b1cae9a6c5d023e5a150f6f6b9dbb3c3bbc7887d6ee07d4c0ecb49a473734"}, - {file = "multidict-6.0.3-cp311-cp311-win32.whl", hash = "sha256:960ce1b790952916e682093788696ef7e33ac6a97482f9b983abdc293091b531"}, - {file = "multidict-6.0.3-cp311-cp311-win_amd64.whl", hash = "sha256:2b66d61966b12e6bba500e5cbb2c721a35e119c30ee02495c5629bd0e91eea30"}, - {file = "multidict-6.0.3-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:526f8397fc124674b8f39748680a0ff673bd6a715fecb4866716d36e380f015f"}, - {file = "multidict-6.0.3-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:1f5d5129a937af4e3c4a1d6c139f4051b7d17d43276cefdd8d442a7031f7eef2"}, - {file = "multidict-6.0.3-cp37-cp37m-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:38d394814b39be1c36ac709006d39d50d72a884f9551acd9c8cc1ffae3fc8c4e"}, - {file = "multidict-6.0.3-cp37-cp37m-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:99341ca1f1db9e7f47914cb2461305665a662383765ced6f843712564766956d"}, - {file = "multidict-6.0.3-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:c5790cc603456b6dcf8a9a4765f666895a6afddc88b3d3ba7b53dea2b6e23116"}, - {file = "multidict-6.0.3-cp37-cp37m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:ce8e51774eb03844588d3c279adb94efcd0edeccd2f97516623292445bcc01f9"}, - {file = "multidict-6.0.3-cp37-cp37m-musllinux_1_1_aarch64.whl", hash = "sha256:baa96a3418e27d723064854143b2f414a422c84cc87285a71558722049bebc5a"}, - {file = "multidict-6.0.3-cp37-cp37m-musllinux_1_1_i686.whl", hash = "sha256:cb4a08f0aaaa869f189ffea0e17b86ad0237b51116d494da15ef7991ee6ad2d7"}, - {file = "multidict-6.0.3-cp37-cp37m-musllinux_1_1_ppc64le.whl", hash = "sha256:62db44727d0befea68e8ad2881bb87a9cfb6b87d45dd78609009627167f37b69"}, - {file = "multidict-6.0.3-cp37-cp37m-musllinux_1_1_s390x.whl", hash = "sha256:4cc5c8cd205a9810d16a5cd428cd81bac554ad1477cb87f4ad722b10992e794d"}, - {file = "multidict-6.0.3-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:f76109387e1ec8d8e2137c94c437b89fe002f29e0881aae8ae45529bdff92000"}, - {file = "multidict-6.0.3-cp37-cp37m-win32.whl", hash = "sha256:f8a728511c977df6f3d8af388fcb157e49f11db4a6637dd60131b8b6e40b0253"}, - {file = "multidict-6.0.3-cp37-cp37m-win_amd64.whl", hash = "sha256:c2a1168e5aa7c72499fb03c850e0f03f624fa4a5c8d2e215c518d0a73872eb64"}, - {file = "multidict-6.0.3-cp38-cp38-macosx_10_9_universal2.whl", hash = "sha256:eddf604a3de2ace3d9a4e4d491be7562a1ac095a0a1c95a9ec5781ef0273ef11"}, - {file = "multidict-6.0.3-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:d09daf5c6ce7fc6ed444c9339bbde5ea84e2534d1ca1cd37b60f365c77f00dea"}, - {file = "multidict-6.0.3-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:12e0d396faa6dc55ff5379eee54d1df3b508243ff15bfc8295a6ec7a4483a335"}, - {file = "multidict-6.0.3-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:70740c2bc9ab1c99f7cdcb104f27d16c63860c56d51c5bf0ef82fc1d892a2131"}, - {file = "multidict-6.0.3-cp38-cp38-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:e322c94596054352f5a02771eec71563c018b15699b961aba14d6dd943367022"}, - {file = "multidict-6.0.3-cp38-cp38-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:4159fc1ec9ede8ab93382e0d6ba9b1b3d23c72da39a834db7a116986605c7ab4"}, - {file = "multidict-6.0.3-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:47defc0218682281a52fb1f6346ebb8b68b17538163a89ea24dfe4da37a8a9a3"}, - {file = "multidict-6.0.3-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:7f9511e48bde6b995825e8d35e434fc96296cf07a25f4aae24ff9162be7eaa46"}, - {file = "multidict-6.0.3-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:e0bce9f7c30e7e3a9e683f670314c0144e8d34be6b7019e40604763bd278d84f"}, - {file = "multidict-6.0.3-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:01b456046a05ff7cceefb0e1d2a9d32f05efcb1c7e0d152446304e11557639ce"}, - {file = "multidict-6.0.3-cp38-cp38-musllinux_1_1_ppc64le.whl", hash = "sha256:8230a39bae6c2e8a09e4da6bace5064693b00590a4a213e38f9a9366da10e7dd"}, - {file = "multidict-6.0.3-cp38-cp38-musllinux_1_1_s390x.whl", hash = "sha256:445c0851a1cbc1f2ec3b40bc22f9c4a235edb3c9a0906122a9df6ea8d51f886c"}, - {file = "multidict-6.0.3-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:9aac6881454a750554ed4b280a839dcf9e2133a9d12ab4d417d673fb102289b7"}, - {file = "multidict-6.0.3-cp38-cp38-win32.whl", hash = "sha256:81c3d597591b0940e04949e4e4f79359b2d2e542a686ba0da5e25de33fec13e0"}, - {file = "multidict-6.0.3-cp38-cp38-win_amd64.whl", hash = "sha256:dc4cfef5d899f5f1a15f3d2ac49f71107a01a5a2745b4dd53fa0cede1419385a"}, - {file = "multidict-6.0.3-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:d408172519049e36fb6d29672f060dc8461fc7174eba9883c7026041ef9bfb38"}, - {file = "multidict-6.0.3-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:e068dfeadbce63072b2d8096486713d04db4946aad0a0f849bd4fc300799d0d3"}, - {file = "multidict-6.0.3-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:a8b817d4ed68fd568ec5e45dd75ddf30cc72a47a6b41b74d5bb211374c296f5e"}, - {file = "multidict-6.0.3-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:2cf5d19e12eff855aa198259c0b02fd3f5d07e1291fbd20279c37b3b0e6c9852"}, - {file = "multidict-6.0.3-cp39-cp39-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:e5a811aab1b4aea0b4be669363c19847a8c547510f0e18fb632956369fdbdf67"}, - {file = "multidict-6.0.3-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:2cfda34b7cb99eacada2072e0f69c0ad3285cb6f8e480b11f2b6d6c1c6f92718"}, - {file = "multidict-6.0.3-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:beeca903e4270b4afcd114f371a9602240dc143f9e944edfea00f8d4ad56c40d"}, - {file = "multidict-6.0.3-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:cd5771e8ea325f85cbb361ddbdeb9ae424a68e5dfb6eea786afdcd22e68a7d5d"}, - {file = "multidict-6.0.3-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:9dbab2a7e9c073bc9538824a01f5ed689194db7f55f2b8102766873e906a6c1a"}, - {file = "multidict-6.0.3-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:f2c0957b3e8c66c10d27272709a5299ab3670a0f187c9428f3b90d267119aedb"}, - {file = "multidict-6.0.3-cp39-cp39-musllinux_1_1_ppc64le.whl", hash = "sha256:94cbe5535ef150546b8321aebea22862a3284da51e7b55f6f95b7d73e96d90ee"}, - {file = "multidict-6.0.3-cp39-cp39-musllinux_1_1_s390x.whl", hash = "sha256:d0e798b072cf2aab9daceb43d97c9c527a0c7593e67a7846ad4cc6051de1e303"}, - {file = "multidict-6.0.3-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:a27b029caa3b555a4f3da54bc1e718eb55fcf1a11fda8bf0132147b476cf4c08"}, - {file = "multidict-6.0.3-cp39-cp39-win32.whl", hash = "sha256:018c8e3be7f161a12b3e41741b6721f9baeb2210f4ab25a6359b7d76c1017dce"}, - {file = "multidict-6.0.3-cp39-cp39-win_amd64.whl", hash = "sha256:5e58ec0375803526d395f6f7e730ecc45d06e15f68f7b9cdbf644a2918324e51"}, - {file = "multidict-6.0.3.tar.gz", hash = "sha256:2523a29006c034687eccd3ee70093a697129a3ffe8732535d3b2df6a4ecc279d"}, -] [[package]] name = "nodeenv" @@ -1310,10 +627,6 @@ description = "Node.js virtual environment builder" category = "dev" optional = false python-versions = ">=2.7,!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,!=3.4.*,!=3.5.*,!=3.6.*" -files = [ - {file = "nodeenv-1.7.0-py2.py3-none-any.whl", hash = "sha256:27083a7b96a25f2f5e1d8cb4b6317ee8aeda3bdd121394e5ac54e498028a042e"}, - {file = "nodeenv-1.7.0.tar.gz", hash = "sha256:e0e7f7dfb85fc5394c6fe1e8fa98131a2473e04311a45afb6508f7cf1836fa2b"}, -] [package.dependencies] setuptools = "*" @@ -1325,36 +638,6 @@ description = "NumPy is the fundamental package for array computing with Python. category = "main" optional = true python-versions = ">=3.8" -files = [ - {file = "numpy-1.23.5-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:9c88793f78fca17da0145455f0d7826bcb9f37da4764af27ac945488116efe63"}, - {file = "numpy-1.23.5-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:e9f4c4e51567b616be64e05d517c79a8a22f3606499941d97bb76f2ca59f982d"}, - {file = "numpy-1.23.5-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:7903ba8ab592b82014713c491f6c5d3a1cde5b4a3bf116404e08f5b52f6daf43"}, - {file = "numpy-1.23.5-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:5e05b1c973a9f858c74367553e236f287e749465f773328c8ef31abe18f691e1"}, - {file = "numpy-1.23.5-cp310-cp310-win32.whl", hash = "sha256:522e26bbf6377e4d76403826ed689c295b0b238f46c28a7251ab94716da0b280"}, - {file = "numpy-1.23.5-cp310-cp310-win_amd64.whl", hash = "sha256:dbee87b469018961d1ad79b1a5d50c0ae850000b639bcb1b694e9981083243b6"}, - {file = "numpy-1.23.5-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:ce571367b6dfe60af04e04a1834ca2dc5f46004ac1cc756fb95319f64c095a96"}, - {file = "numpy-1.23.5-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:56e454c7833e94ec9769fa0f86e6ff8e42ee38ce0ce1fa4cbb747ea7e06d56aa"}, - {file = "numpy-1.23.5-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:5039f55555e1eab31124a5768898c9e22c25a65c1e0037f4d7c495a45778c9f2"}, - {file = "numpy-1.23.5-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:58f545efd1108e647604a1b5aa809591ccd2540f468a880bedb97247e72db387"}, - {file = "numpy-1.23.5-cp311-cp311-win32.whl", hash = "sha256:b2a9ab7c279c91974f756c84c365a669a887efa287365a8e2c418f8b3ba73fb0"}, - {file = "numpy-1.23.5-cp311-cp311-win_amd64.whl", hash = "sha256:0cbe9848fad08baf71de1a39e12d1b6310f1d5b2d0ea4de051058e6e1076852d"}, - {file = "numpy-1.23.5-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:f063b69b090c9d918f9df0a12116029e274daf0181df392839661c4c7ec9018a"}, - {file = "numpy-1.23.5-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:0aaee12d8883552fadfc41e96b4c82ee7d794949e2a7c3b3a7201e968c7ecab9"}, - {file = "numpy-1.23.5-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:92c8c1e89a1f5028a4c6d9e3ccbe311b6ba53694811269b992c0b224269e2398"}, - {file = "numpy-1.23.5-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:d208a0f8729f3fb790ed18a003f3a57895b989b40ea4dce4717e9cf4af62c6bb"}, - {file = "numpy-1.23.5-cp38-cp38-win32.whl", hash = "sha256:06005a2ef6014e9956c09ba07654f9837d9e26696a0470e42beedadb78c11b07"}, - {file = "numpy-1.23.5-cp38-cp38-win_amd64.whl", hash = "sha256:ca51fcfcc5f9354c45f400059e88bc09215fb71a48d3768fb80e357f3b457e1e"}, - {file = "numpy-1.23.5-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:8969bfd28e85c81f3f94eb4a66bc2cf1dbdc5c18efc320af34bffc54d6b1e38f"}, - {file = "numpy-1.23.5-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:a7ac231a08bb37f852849bbb387a20a57574a97cfc7b6cabb488a4fc8be176de"}, - {file = "numpy-1.23.5-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:bf837dc63ba5c06dc8797c398db1e223a466c7ece27a1f7b5232ba3466aafe3d"}, - {file = "numpy-1.23.5-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:33161613d2269025873025b33e879825ec7b1d831317e68f4f2f0f84ed14c719"}, - {file = "numpy-1.23.5-cp39-cp39-win32.whl", hash = "sha256:af1da88f6bc3d2338ebbf0e22fe487821ea4d8e89053e25fa59d1d79786e7481"}, - {file = "numpy-1.23.5-cp39-cp39-win_amd64.whl", hash = "sha256:09b7847f7e83ca37c6e627682f145856de331049013853f344f37b0c9690e3df"}, - {file = "numpy-1.23.5-pp38-pypy38_pp73-macosx_10_9_x86_64.whl", hash = "sha256:abdde9f795cf292fb9651ed48185503a2ff29be87770c3b8e2a14b0cd7aa16f8"}, - {file = "numpy-1.23.5-pp38-pypy38_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:f9a909a8bae284d46bbfdefbdd4a262ba19d3bc9921b1e76126b1d21c3c34135"}, - {file = "numpy-1.23.5-pp38-pypy38_pp73-win_amd64.whl", hash = "sha256:01dd17cbb340bf0fc23981e52e1d18a9d4050792e8fb8363cecbf066a84b827d"}, - {file = "numpy-1.23.5.tar.gz", hash = "sha256:1b1766d6f397c18153d40015ddfc79ddb715cabadc04d2d228d4e5a8bc4ded1a"}, -] [[package]] name = "oauthlib" @@ -1363,10 +646,6 @@ description = "A generic, spec-compliant, thorough implementation of the OAuth r category = "main" optional = true python-versions = ">=3.6" -files = [ - {file = "oauthlib-3.2.2-py3-none-any.whl", hash = "sha256:8139f29aac13e25d502680e9e19963e83f16838d48a0d71c287fe40e7067fbca"}, - {file = "oauthlib-3.2.2.tar.gz", hash = "sha256:9859c40929662bec5d64f34d01c99e093149682a3f38915dc0655d5a633dd918"}, -] [package.extras] rsa = ["cryptography (>=3.0.0)"] @@ -1380,10 +659,6 @@ description = "Core utilities for Python packages" category = "dev" optional = false python-versions = ">=3.7" -files = [ - {file = "packaging-22.0-py3-none-any.whl", hash = "sha256:957e2148ba0e1a3b282772e791ef1d8083648bc131c8ab0c1feba110ce1146c3"}, - {file = "packaging-22.0.tar.gz", hash = "sha256:2198ec20bd4c017b8f9717e00f0c8714076fc2fd93816750ab48e2c41de2cfd3"}, -] [[package]] name = "pandas" @@ -1392,35 +667,6 @@ description = "Powerful data structures for data analysis, time series, and stat category = "main" optional = true python-versions = ">=3.8" -files = [ - {file = "pandas-1.5.2-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:e9dbacd22555c2d47f262ef96bb4e30880e5956169741400af8b306bbb24a273"}, - {file = "pandas-1.5.2-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:e2b83abd292194f350bb04e188f9379d36b8dfac24dd445d5c87575f3beaf789"}, - {file = "pandas-1.5.2-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:2552bffc808641c6eb471e55aa6899fa002ac94e4eebfa9ec058649122db5824"}, - {file = "pandas-1.5.2-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:1fc87eac0541a7d24648a001d553406f4256e744d92df1df8ebe41829a915028"}, - {file = "pandas-1.5.2-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:d0d8fd58df5d17ddb8c72a5075d87cd80d71b542571b5f78178fb067fa4e9c72"}, - {file = "pandas-1.5.2-cp310-cp310-win_amd64.whl", hash = "sha256:4aed257c7484d01c9a194d9a94758b37d3d751849c05a0050c087a358c41ad1f"}, - {file = "pandas-1.5.2-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:375262829c8c700c3e7cbb336810b94367b9c4889818bbd910d0ecb4e45dc261"}, - {file = "pandas-1.5.2-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:cc3cd122bea268998b79adebbb8343b735a5511ec14efb70a39e7acbc11ccbdc"}, - {file = "pandas-1.5.2-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:b4f5a82afa4f1ff482ab8ded2ae8a453a2cdfde2001567b3ca24a4c5c5ca0db3"}, - {file = "pandas-1.5.2-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:8092a368d3eb7116e270525329a3e5c15ae796ccdf7ccb17839a73b4f5084a39"}, - {file = "pandas-1.5.2-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:f6257b314fc14958f8122779e5a1557517b0f8e500cfb2bd53fa1f75a8ad0af2"}, - {file = "pandas-1.5.2-cp311-cp311-win_amd64.whl", hash = "sha256:82ae615826da838a8e5d4d630eb70c993ab8636f0eff13cb28aafc4291b632b5"}, - {file = "pandas-1.5.2-cp38-cp38-macosx_10_9_universal2.whl", hash = "sha256:457d8c3d42314ff47cc2d6c54f8fc0d23954b47977b2caed09cd9635cb75388b"}, - {file = "pandas-1.5.2-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:c009a92e81ce836212ce7aa98b219db7961a8b95999b97af566b8dc8c33e9519"}, - {file = "pandas-1.5.2-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:71f510b0efe1629bf2f7c0eadb1ff0b9cf611e87b73cd017e6b7d6adb40e2b3a"}, - {file = "pandas-1.5.2-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a40dd1e9f22e01e66ed534d6a965eb99546b41d4d52dbdb66565608fde48203f"}, - {file = "pandas-1.5.2-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:5ae7e989f12628f41e804847a8cc2943d362440132919a69429d4dea1f164da0"}, - {file = "pandas-1.5.2-cp38-cp38-win32.whl", hash = "sha256:530948945e7b6c95e6fa7aa4be2be25764af53fba93fe76d912e35d1c9ee46f5"}, - {file = "pandas-1.5.2-cp38-cp38-win_amd64.whl", hash = "sha256:73f219fdc1777cf3c45fde7f0708732ec6950dfc598afc50588d0d285fddaefc"}, - {file = "pandas-1.5.2-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:9608000a5a45f663be6af5c70c3cbe634fa19243e720eb380c0d378666bc7702"}, - {file = "pandas-1.5.2-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:315e19a3e5c2ab47a67467fc0362cb36c7c60a93b6457f675d7d9615edad2ebe"}, - {file = "pandas-1.5.2-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:e18bc3764cbb5e118be139b3b611bc3fbc5d3be42a7e827d1096f46087b395eb"}, - {file = "pandas-1.5.2-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:0183cb04a057cc38fde5244909fca9826d5d57c4a5b7390c0cc3fa7acd9fa883"}, - {file = "pandas-1.5.2-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:344021ed3e639e017b452aa8f5f6bf38a8806f5852e217a7594417fb9bbfa00e"}, - {file = "pandas-1.5.2-cp39-cp39-win32.whl", hash = "sha256:e7469271497960b6a781eaa930cba8af400dd59b62ec9ca2f4d31a19f2f91090"}, - {file = "pandas-1.5.2-cp39-cp39-win_amd64.whl", hash = "sha256:c218796d59d5abd8780170c937b812c9637e84c32f8271bbf9845970f8c1351f"}, - {file = "pandas-1.5.2.tar.gz", hash = "sha256:220b98d15cee0b2cd839a6358bd1f273d0356bf964c1a1aeb32d47db0215488b"}, -] [package.dependencies] numpy = [ @@ -1441,10 +687,6 @@ description = "Wrappers to build Python packages using PEP 517 hooks" category = "dev" optional = false python-versions = ">=3.6" -files = [ - {file = "pep517-0.13.0-py3-none-any.whl", hash = "sha256:4ba4446d80aed5b5eac6509ade100bff3e7943a8489de249654a5ae9b33ee35b"}, - {file = "pep517-0.13.0.tar.gz", hash = "sha256:ae69927c5c172be1add9203726d4b84cf3ebad1edcd5f71fcdc746e66e829f59"}, -] [package.dependencies] tomli = {version = ">=1.1.0", markers = "python_version < \"3.11\""} @@ -1456,10 +698,6 @@ description = "A small Python package for determining appropriate platform-speci category = "dev" optional = false python-versions = ">=3.7" -files = [ - {file = "platformdirs-2.6.0-py3-none-any.whl", hash = "sha256:1a89a12377800c81983db6be069ec068eee989748799b946cce2a6e80dcc54ca"}, - {file = "platformdirs-2.6.0.tar.gz", hash = "sha256:b46ffafa316e6b83b47489d240ce17173f123a9b9c83282141c3daf26ad9ac2e"}, -] [package.extras] docs = ["furo (>=2022.9.29)", "proselint (>=0.13)", "sphinx (>=5.3)", "sphinx-autodoc-typehints (>=1.19.4)"] @@ -1472,10 +710,6 @@ description = "plugin and hook calling mechanisms for python" category = "dev" optional = false python-versions = ">=3.6" -files = [ - {file = "pluggy-1.0.0-py2.py3-none-any.whl", hash = "sha256:74134bbf457f031a36d68416e1509f34bd5ccc019f0bcc952c7b909d06b37bd3"}, - {file = "pluggy-1.0.0.tar.gz", hash = "sha256:4224373bacce55f955a878bf9cfa763c1e360858e330072059e10bad68531159"}, -] [package.extras] dev = ["pre-commit", "tox"] @@ -1488,10 +722,6 @@ description = "Wraps the portalocker recipe for easy usage" category = "main" optional = true python-versions = ">=3.5" -files = [ - {file = "portalocker-2.6.0-py2.py3-none-any.whl", hash = "sha256:102ed1f2badd8dec9af3d732ef70e94b215b85ba45a8d7ff3c0003f19b442f4e"}, - {file = "portalocker-2.6.0.tar.gz", hash = "sha256:964f6830fb42a74b5d32bce99ed37d8308c1d7d44ddf18f3dd89f4680de97b39"}, -] [package.dependencies] pywin32 = {version = ">=226", markers = "platform_system == \"Windows\""} @@ -1508,10 +738,6 @@ description = "A framework for managing and maintaining multi-language pre-commi category = "dev" optional = false python-versions = ">=3.7" -files = [ - {file = "pre_commit-2.20.0-py2.py3-none-any.whl", hash = "sha256:51a5ba7c480ae8072ecdb6933df22d2f812dc897d5fe848778116129a681aac7"}, - {file = "pre_commit-2.20.0.tar.gz", hash = "sha256:a978dac7bc9ec0bcee55c18a277d553b0f419d259dadb4b9418ff2d00eb43959"}, -] [package.dependencies] cfgv = ">=2.0.0" @@ -1528,33 +754,6 @@ description = "Python library for Apache Arrow" category = "main" optional = true python-versions = ">=3.7" -files = [ - {file = "pyarrow-10.0.1-cp310-cp310-macosx_10_14_x86_64.whl", hash = "sha256:e00174764a8b4e9d8d5909b6d19ee0c217a6cf0232c5682e31fdfbd5a9f0ae52"}, - {file = "pyarrow-10.0.1-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:6f7a7dbe2f7f65ac1d0bd3163f756deb478a9e9afc2269557ed75b1b25ab3610"}, - {file = "pyarrow-10.0.1-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:cb627673cb98708ef00864e2e243f51ba7b4c1b9f07a1d821f98043eccd3f585"}, - {file = "pyarrow-10.0.1-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:ba71e6fc348c92477586424566110d332f60d9a35cb85278f42e3473bc1373da"}, - {file = "pyarrow-10.0.1-cp310-cp310-win_amd64.whl", hash = "sha256:7b4ede715c004b6fc535de63ef79fa29740b4080639a5ff1ea9ca84e9282f349"}, - {file = "pyarrow-10.0.1-cp311-cp311-macosx_10_14_x86_64.whl", hash = "sha256:e3fe5049d2e9ca661d8e43fab6ad5a4c571af12d20a57dffc392a014caebef65"}, - {file = "pyarrow-10.0.1-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:254017ca43c45c5098b7f2a00e995e1f8346b0fb0be225f042838323bb55283c"}, - {file = "pyarrow-10.0.1-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:70acca1ece4322705652f48db65145b5028f2c01c7e426c5d16a30ba5d739c24"}, - {file = "pyarrow-10.0.1-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:abb57334f2c57979a49b7be2792c31c23430ca02d24becd0b511cbe7b6b08649"}, - {file = "pyarrow-10.0.1-cp311-cp311-win_amd64.whl", hash = "sha256:1765a18205eb1e02ccdedb66049b0ec148c2a0cb52ed1fb3aac322dfc086a6ee"}, - {file = "pyarrow-10.0.1-cp37-cp37m-macosx_10_14_x86_64.whl", hash = "sha256:61f4c37d82fe00d855d0ab522c685262bdeafd3fbcb5fe596fe15025fbc7341b"}, - {file = "pyarrow-10.0.1-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:e141a65705ac98fa52a9113fe574fdaf87fe0316cde2dffe6b94841d3c61544c"}, - {file = "pyarrow-10.0.1-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:bf26f809926a9d74e02d76593026f0aaeac48a65b64f1bb17eed9964bfe7ae1a"}, - {file = "pyarrow-10.0.1-cp37-cp37m-win_amd64.whl", hash = "sha256:443eb9409b0cf78df10ced326490e1a300205a458fbeb0767b6b31ab3ebae6b2"}, - {file = "pyarrow-10.0.1-cp38-cp38-macosx_10_14_x86_64.whl", hash = "sha256:f2d00aa481becf57098e85d99e34a25dba5a9ade2f44eb0b7d80c80f2984fc03"}, - {file = "pyarrow-10.0.1-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:b1fc226d28c7783b52a84d03a66573d5a22e63f8a24b841d5fc68caeed6784d4"}, - {file = "pyarrow-10.0.1-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:efa59933b20183c1c13efc34bd91efc6b2997377c4c6ad9272da92d224e3beb1"}, - {file = "pyarrow-10.0.1-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:668e00e3b19f183394388a687d29c443eb000fb3fe25599c9b4762a0afd37775"}, - {file = "pyarrow-10.0.1-cp38-cp38-win_amd64.whl", hash = "sha256:d1bc6e4d5d6f69e0861d5d7f6cf4d061cf1069cb9d490040129877acf16d4c2a"}, - {file = "pyarrow-10.0.1-cp39-cp39-macosx_10_14_x86_64.whl", hash = "sha256:42ba7c5347ce665338f2bc64685d74855900200dac81a972d49fe127e8132f75"}, - {file = "pyarrow-10.0.1-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:b069602eb1fc09f1adec0a7bdd7897f4d25575611dfa43543c8b8a75d99d6874"}, - {file = "pyarrow-10.0.1-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:94fb4a0c12a2ac1ed8e7e2aa52aade833772cf2d3de9dde685401b22cec30002"}, - {file = "pyarrow-10.0.1-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:db0c5986bf0808927f49640582d2032a07aa49828f14e51f362075f03747d198"}, - {file = "pyarrow-10.0.1-cp39-cp39-win_amd64.whl", hash = "sha256:0ec7587d759153f452d5263dbc8b1af318c4609b607be2bd5127dcda6708cdb1"}, - {file = "pyarrow-10.0.1.tar.gz", hash = "sha256:1a14f57a5f472ce8234f2964cd5184cccaa8df7e04568c64edc33b23eb285dd5"}, -] [package.dependencies] numpy = ">=1.16.6" @@ -1566,10 +765,6 @@ description = "C parser in Python" category = "main" optional = false python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*" -files = [ - {file = "pycparser-2.21-py2.py3-none-any.whl", hash = "sha256:8ee45429555515e1f6b185e78100aea234072576aa43ab53aefcae078162fca9"}, - {file = "pycparser-2.21.tar.gz", hash = "sha256:e644fdec12f7872f86c58ff790da456218b10f863970249516d60a5eaca77206"}, -] [[package]] name = "pydantic" @@ -1578,44 +773,6 @@ description = "Data validation and settings management using python type hints" category = "main" optional = false python-versions = ">=3.7" -files = [ - {file = "pydantic-1.10.2-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:bb6ad4489af1bac6955d38ebcb95079a836af31e4c4f74aba1ca05bb9f6027bd"}, - {file = "pydantic-1.10.2-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:a1f5a63a6dfe19d719b1b6e6106561869d2efaca6167f84f5ab9347887d78b98"}, - {file = "pydantic-1.10.2-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:352aedb1d71b8b0736c6d56ad2bd34c6982720644b0624462059ab29bd6e5912"}, - {file = "pydantic-1.10.2-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:19b3b9ccf97af2b7519c42032441a891a5e05c68368f40865a90eb88833c2559"}, - {file = "pydantic-1.10.2-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:e9069e1b01525a96e6ff49e25876d90d5a563bc31c658289a8772ae186552236"}, - {file = "pydantic-1.10.2-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:355639d9afc76bcb9b0c3000ddcd08472ae75318a6eb67a15866b87e2efa168c"}, - {file = "pydantic-1.10.2-cp310-cp310-win_amd64.whl", hash = "sha256:ae544c47bec47a86bc7d350f965d8b15540e27e5aa4f55170ac6a75e5f73b644"}, - {file = "pydantic-1.10.2-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:a4c805731c33a8db4b6ace45ce440c4ef5336e712508b4d9e1aafa617dc9907f"}, - {file = "pydantic-1.10.2-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:d49f3db871575e0426b12e2f32fdb25e579dea16486a26e5a0474af87cb1ab0a"}, - {file = "pydantic-1.10.2-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:37c90345ec7dd2f1bcef82ce49b6235b40f282b94d3eec47e801baf864d15525"}, - {file = "pydantic-1.10.2-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:7b5ba54d026c2bd2cb769d3468885f23f43710f651688e91f5fb1edcf0ee9283"}, - {file = "pydantic-1.10.2-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:05e00dbebbe810b33c7a7362f231893183bcc4251f3f2ff991c31d5c08240c42"}, - {file = "pydantic-1.10.2-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:2d0567e60eb01bccda3a4df01df677adf6b437958d35c12a3ac3e0f078b0ee52"}, - {file = "pydantic-1.10.2-cp311-cp311-win_amd64.whl", hash = "sha256:c6f981882aea41e021f72779ce2a4e87267458cc4d39ea990729e21ef18f0f8c"}, - {file = "pydantic-1.10.2-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:c4aac8e7103bf598373208f6299fa9a5cfd1fc571f2d40bf1dd1955a63d6eeb5"}, - {file = "pydantic-1.10.2-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:81a7b66c3f499108b448f3f004801fcd7d7165fb4200acb03f1c2402da73ce4c"}, - {file = "pydantic-1.10.2-cp37-cp37m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:bedf309630209e78582ffacda64a21f96f3ed2e51fbf3962d4d488e503420254"}, - {file = "pydantic-1.10.2-cp37-cp37m-musllinux_1_1_i686.whl", hash = "sha256:9300fcbebf85f6339a02c6994b2eb3ff1b9c8c14f502058b5bf349d42447dcf5"}, - {file = "pydantic-1.10.2-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:216f3bcbf19c726b1cc22b099dd409aa371f55c08800bcea4c44c8f74b73478d"}, - {file = "pydantic-1.10.2-cp37-cp37m-win_amd64.whl", hash = "sha256:dd3f9a40c16daf323cf913593083698caee97df2804aa36c4b3175d5ac1b92a2"}, - {file = "pydantic-1.10.2-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:b97890e56a694486f772d36efd2ba31612739bc6f3caeee50e9e7e3ebd2fdd13"}, - {file = "pydantic-1.10.2-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:9cabf4a7f05a776e7793e72793cd92cc865ea0e83a819f9ae4ecccb1b8aa6116"}, - {file = "pydantic-1.10.2-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:06094d18dd5e6f2bbf93efa54991c3240964bb663b87729ac340eb5014310624"}, - {file = "pydantic-1.10.2-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:cc78cc83110d2f275ec1970e7a831f4e371ee92405332ebfe9860a715f8336e1"}, - {file = "pydantic-1.10.2-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:1ee433e274268a4b0c8fde7ad9d58ecba12b069a033ecc4645bb6303c062d2e9"}, - {file = "pydantic-1.10.2-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:7c2abc4393dea97a4ccbb4ec7d8658d4e22c4765b7b9b9445588f16c71ad9965"}, - {file = "pydantic-1.10.2-cp38-cp38-win_amd64.whl", hash = "sha256:0b959f4d8211fc964772b595ebb25f7652da3f22322c007b6fed26846a40685e"}, - {file = "pydantic-1.10.2-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:c33602f93bfb67779f9c507e4d69451664524389546bacfe1bee13cae6dc7488"}, - {file = "pydantic-1.10.2-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:5760e164b807a48a8f25f8aa1a6d857e6ce62e7ec83ea5d5c5a802eac81bad41"}, - {file = "pydantic-1.10.2-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:6eb843dcc411b6a2237a694f5e1d649fc66c6064d02b204a7e9d194dff81eb4b"}, - {file = "pydantic-1.10.2-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:4b8795290deaae348c4eba0cebb196e1c6b98bdbe7f50b2d0d9a4a99716342fe"}, - {file = "pydantic-1.10.2-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:e0bedafe4bc165ad0a56ac0bd7695df25c50f76961da29c050712596cf092d6d"}, - {file = "pydantic-1.10.2-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:2e05aed07fa02231dbf03d0adb1be1d79cabb09025dd45aa094aa8b4e7b9dcda"}, - {file = "pydantic-1.10.2-cp39-cp39-win_amd64.whl", hash = "sha256:c1ba1afb396148bbc70e9eaa8c06c1716fdddabaf86e7027c5988bae2a829ab6"}, - {file = "pydantic-1.10.2-py3-none-any.whl", hash = "sha256:1b6ee725bd6e83ec78b1aa32c5b1fa67a3a65badddde3976bca5fe4568f27709"}, - {file = "pydantic-1.10.2.tar.gz", hash = "sha256:91b8e218852ef6007c2b98cd861601c6a09f1aa32bbbb74fab5b1c33d4a1e410"}, -] [package.dependencies] typing-extensions = ">=4.1.0" @@ -1631,10 +788,6 @@ description = "Pygments is a syntax highlighting package written in Python." category = "main" optional = false python-versions = ">=3.6" -files = [ - {file = "Pygments-2.13.0-py3-none-any.whl", hash = "sha256:f643f331ab57ba3c9d89212ee4a2dabc6e94f117cf4eefde99a0574720d14c42"}, - {file = "Pygments-2.13.0.tar.gz", hash = "sha256:56a8508ae95f98e2b9bdf93a6be5ae3f7d8af858b43e02c5a2ff083726be40c1"}, -] [package.extras] plugins = ["importlib-metadata"] @@ -1646,10 +799,6 @@ description = "JSON Web Token implementation in Python" category = "main" optional = true python-versions = ">=3.7" -files = [ - {file = "PyJWT-2.6.0-py3-none-any.whl", hash = "sha256:d83c3d892a77bbb74d3e1a2cfa90afaadb60945205d1095d9221f04466f64c14"}, - {file = "PyJWT-2.6.0.tar.gz", hash = "sha256:69285c7e31fc44f68a1feb309e948e0df53259d579295e6cfe2b1792329f05fd"}, -] [package.dependencies] cryptography = {version = ">=3.4.0", optional = true, markers = "extra == \"crypto\""} @@ -1667,10 +816,6 @@ description = "pyparsing module - Classes and methods to define and execute pars category = "main" optional = false python-versions = ">=3.6.8" -files = [ - {file = "pyparsing-3.0.9-py3-none-any.whl", hash = "sha256:5026bae9a10eeaefb61dab2f09052b9f4307d44aee4eda64b309723d8d206bbc"}, - {file = "pyparsing-3.0.9.tar.gz", hash = "sha256:2b020ecf7d21b687f219b71ecad3631f644a47f01403fa1d1036b0c6416d70fb"}, -] [package.extras] diagrams = ["jinja2", "railroad-diagrams"] @@ -1682,10 +827,6 @@ description = "pytest: simple powerful testing with Python" category = "dev" optional = false python-versions = ">=3.7" -files = [ - {file = "pytest-7.2.0-py3-none-any.whl", hash = "sha256:892f933d339f068883b6fd5a459f03d85bfcb355e4981e146d2c7616c21fef71"}, - {file = "pytest-7.2.0.tar.gz", hash = "sha256:c4014eb40e10f11f355ad4e3c2fb2c6c6d1919c73f3b5a433de4708202cade59"}, -] [package.dependencies] attrs = ">=19.2.0" @@ -1706,10 +847,6 @@ description = "check the README when running tests" category = "dev" optional = false python-versions = ">=3.7" -files = [ - {file = "pytest-checkdocs-2.9.0.tar.gz", hash = "sha256:54b3d1357a289e3f6336d31d4ceb72e99afe0fa4f552990f1b1f015f1925543b"}, - {file = "pytest_checkdocs-2.9.0-py3-none-any.whl", hash = "sha256:4231e5426915a02385e386b5eb4a70490b143d97cede4c7c77691bb320b5a73f"}, -] [package.dependencies] build = "*" @@ -1727,10 +864,6 @@ description = "Extensions to the standard Python datetime module" category = "main" optional = false python-versions = "!=3.0.*,!=3.1.*,!=3.2.*,>=2.7" -files = [ - {file = "python-dateutil-2.8.2.tar.gz", hash = "sha256:0123cacc1627ae19ddf3c27a5de5bd67ee4586fbdd6440d9748f8abb483d3e86"}, - {file = "python_dateutil-2.8.2-py2.py3-none-any.whl", hash = "sha256:961d03dc3453ebbc59dbdea9e4e11c5651520a876d0f4db161e8674aae935da9"}, -] [package.dependencies] six = ">=1.5" @@ -1742,56 +875,6 @@ description = "Python library for the snappy compression library from Google" category = "main" optional = true python-versions = "*" -files = [ - {file = "python-snappy-0.6.1.tar.gz", hash = "sha256:b6a107ab06206acc5359d4c5632bd9b22d448702a79b3169b0c62e0fb808bb2a"}, - {file = "python_snappy-0.6.1-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:b7f920eaf46ebf41bd26f9df51c160d40f9e00b7b48471c3438cb8d027f7fb9b"}, - {file = "python_snappy-0.6.1-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:4ec533a8c1f8df797bded662ec3e494d225b37855bb63eb0d75464a07947477c"}, - {file = "python_snappy-0.6.1-cp310-cp310-manylinux_2_12_i686.manylinux2010_i686.whl", hash = "sha256:6f8bf4708a11b47517baf962f9a02196478bbb10fdb9582add4aa1459fa82380"}, - {file = "python_snappy-0.6.1-cp310-cp310-manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:8d0c019ee7dcf2c60e240877107cddbd95a5b1081787579bf179938392d66480"}, - {file = "python_snappy-0.6.1-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:cb18d9cd7b3f35a2f5af47bb8ed6a5bdbf4f3ddee37f3daade4ab7864c292f5b"}, - {file = "python_snappy-0.6.1-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:b265cde49774752aec9ca7f5d272e3f98718164afc85521622a8a5394158a2b5"}, - {file = "python_snappy-0.6.1-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:d017775851a778ec9cc32651c4464079d06d927303c2dde9ae9830ccf6fe94e1"}, - {file = "python_snappy-0.6.1-cp310-cp310-win32.whl", hash = "sha256:8277d1f6282463c40761f802b742f833f9f2449fcdbb20a96579aa05c8feb614"}, - {file = "python_snappy-0.6.1-cp310-cp310-win_amd64.whl", hash = "sha256:2aaaf618c68d8c9daebc23a20436bd01b09ee70d7fbf7072b7f38b06d2fab539"}, - {file = "python_snappy-0.6.1-cp36-cp36m-macosx_10_9_x86_64.whl", hash = "sha256:277757d5dad4e239dc1417438a0871b65b1b155beb108888e7438c27ffc6a8cc"}, - {file = "python_snappy-0.6.1-cp36-cp36m-manylinux_2_12_i686.manylinux2010_i686.whl", hash = "sha256:e066a0586833d610c4bbddba0be5ba0e3e4f8e0bc5bb6d82103d8f8fc47bb59a"}, - {file = "python_snappy-0.6.1-cp36-cp36m-manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:0d489b50f49433494160c45048fe806de6b3aeab0586e497ebd22a0bab56e427"}, - {file = "python_snappy-0.6.1-cp36-cp36m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:463fd340a499d47b26ca42d2f36a639188738f6e2098c6dbf80aef0e60f461e1"}, - {file = "python_snappy-0.6.1-cp36-cp36m-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:9837ac1650cc68d22a3cf5f15fb62c6964747d16cecc8b22431f113d6e39555d"}, - {file = "python_snappy-0.6.1-cp36-cp36m-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:5e973e637112391f05581f427659c05b30b6843bc522a65be35ac7b18ce3dedd"}, - {file = "python_snappy-0.6.1-cp36-cp36m-win32.whl", hash = "sha256:c20498bd712b6e31a4402e1d027a1cd64f6a4a0066a3fe3c7344475886d07fdf"}, - {file = "python_snappy-0.6.1-cp36-cp36m-win_amd64.whl", hash = "sha256:59e975be4206cc54d0a112ef72fa3970a57c2b1bcc2c97ed41d6df0ebe518228"}, - {file = "python_snappy-0.6.1-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:2a7e528ab6e09c0d67dcb61a1730a292683e5ff9bb088950638d3170cf2a0a54"}, - {file = "python_snappy-0.6.1-cp37-cp37m-manylinux_2_12_i686.manylinux2010_i686.whl", hash = "sha256:39692bedbe0b717001a99915ac0eb2d9d0bad546440d392a2042b96d813eede1"}, - {file = "python_snappy-0.6.1-cp37-cp37m-manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:6a7620404da966f637b9ce8d4d3d543d363223f7a12452a575189c5355fc2d25"}, - {file = "python_snappy-0.6.1-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:7778c224efc38a40d274da4eb82a04cac27aae20012372a7db3c4bbd8926c4d4"}, - {file = "python_snappy-0.6.1-cp37-cp37m-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:1d029f7051ec1bbeaa3e03030b6d8ed47ceb69cae9016f493c802a08af54e026"}, - {file = "python_snappy-0.6.1-cp37-cp37m-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:a0ad38bc98d0b0497a0b0dbc29409bcabfcecff4511ed7063403c86de16927bc"}, - {file = "python_snappy-0.6.1-cp37-cp37m-win32.whl", hash = "sha256:5a453c45178d7864c1bdd6bfe0ee3ed2883f63b9ba2c9bb967c6b586bf763f96"}, - {file = "python_snappy-0.6.1-cp37-cp37m-win_amd64.whl", hash = "sha256:9f0c0d88b84259f93c3aa46398680646f2c23e43394779758d9f739c34e15295"}, - {file = "python_snappy-0.6.1-cp38-cp38-macosx_10_9_universal2.whl", hash = "sha256:5bb05c28298803a74add08ba496879242ef159c75bc86a5406fac0ffc7dd021b"}, - {file = "python_snappy-0.6.1-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:9eac51307c6a1a38d5f86ebabc26a889fddf20cbba7a116ccb54ba1446601d5b"}, - {file = "python_snappy-0.6.1-cp38-cp38-manylinux_2_12_i686.manylinux2010_i686.whl", hash = "sha256:88b6ea78b83d2796f330b0af1b70cdd3965dbdab02d8ac293260ec2c8fe340ee"}, - {file = "python_snappy-0.6.1-cp38-cp38-manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:8c07220408d3268e8268c9351c5c08041bc6f8c6172e59d398b71020df108541"}, - {file = "python_snappy-0.6.1-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:4038019b1bcaadde726a57430718394076c5a21545ebc5badad2c045a09546cf"}, - {file = "python_snappy-0.6.1-cp38-cp38-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:dc96668d9c7cc656609764275c5f8da58ef56d89bdd6810f6923d36497468ff7"}, - {file = "python_snappy-0.6.1-cp38-cp38-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:cf5bb9254e1c38aacf253d510d3d9be631bba21f3d068b17672b38b5cbf2fff5"}, - {file = "python_snappy-0.6.1-cp38-cp38-win32.whl", hash = "sha256:eaf905a580f2747c4a474040a5063cd5e0cc3d1d2d6edb65f28196186493ad4a"}, - {file = "python_snappy-0.6.1-cp38-cp38-win_amd64.whl", hash = "sha256:546c1a7470ecbf6239101e9aff0f709b68ca0f0268b34d9023019a55baa1f7c6"}, - {file = "python_snappy-0.6.1-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:e3a013895c64352b49d0d8e107a84f99631b16dbab156ded33ebf0becf56c8b2"}, - {file = "python_snappy-0.6.1-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:3fb9a88a4dd6336488f3de67ce75816d0d796dce53c2c6e4d70e0b565633c7fd"}, - {file = "python_snappy-0.6.1-cp39-cp39-manylinux_2_12_i686.manylinux2010_i686.whl", hash = "sha256:735cd4528c55dbe4516d6d2b403331a99fc304f8feded8ae887cf97b67d589bb"}, - {file = "python_snappy-0.6.1-cp39-cp39-manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:90b0186516b7a101c14764b0c25931b741fb0102f21253eff67847b4742dfc72"}, - {file = "python_snappy-0.6.1-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:1a993dc8aadd901915a510fe6af5f20ae4256f527040066c22a154db8946751f"}, - {file = "python_snappy-0.6.1-cp39-cp39-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:530bfb9efebcc1aab8bb4ebcbd92b54477eed11f6cf499355e882970a6d3aa7d"}, - {file = "python_snappy-0.6.1-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:5843feb914796b1f0405ccf31ea0fb51034ceb65a7588edfd5a8250cb369e3b2"}, - {file = "python_snappy-0.6.1-cp39-cp39-win32.whl", hash = "sha256:66c80e9b366012dbee262bb1869e4fc5ba8786cda85928481528bc4a72ec2ee8"}, - {file = "python_snappy-0.6.1-cp39-cp39-win_amd64.whl", hash = "sha256:4d3cafdf454354a621c8ab7408e45aa4e9d5c0b943b61ff4815f71ca6bdf0130"}, - {file = "python_snappy-0.6.1-pp37-pypy37_pp73-manylinux_2_12_i686.manylinux2010_i686.whl", hash = "sha256:586724a0276d7a6083a17259d0b51622e492289a9998848a1b01b6441ca12b2f"}, - {file = "python_snappy-0.6.1-pp37-pypy37_pp73-manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:2be4f4550acd484912441f5f1209ba611ac399aac9355fee73611b9a0d4f949c"}, - {file = "python_snappy-0.6.1-pp37-pypy37_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:0bdb6942180660bda7f7d01f4c0def3cfc72b1c6d99aad964801775a3e379aba"}, - {file = "python_snappy-0.6.1-pp37-pypy37_pp73-win_amd64.whl", hash = "sha256:03bb511380fca2a13325b6f16fe8234c8e12da9660f0258cd45d9a02ffc916af"}, -] [[package]] name = "pytz" @@ -1800,10 +883,6 @@ description = "World timezone definitions, modern and historical" category = "main" optional = false python-versions = "*" -files = [ - {file = "pytz-2022.6-py2.py3-none-any.whl", hash = "sha256:222439474e9c98fced559f1709d89e6c9cbf8d79c794ff3eb9f8800064291427"}, - {file = "pytz-2022.6.tar.gz", hash = "sha256:e89512406b793ca39f5971bc999cc538ce125c0e51c27941bef4568b460095e2"}, -] [[package]] name = "pywin32" @@ -1812,22 +891,6 @@ description = "Python for Window Extensions" category = "main" optional = true python-versions = "*" -files = [ - {file = "pywin32-305-cp310-cp310-win32.whl", hash = "sha256:421f6cd86e84bbb696d54563c48014b12a23ef95a14e0bdba526be756d89f116"}, - {file = "pywin32-305-cp310-cp310-win_amd64.whl", hash = "sha256:73e819c6bed89f44ff1d690498c0a811948f73777e5f97c494c152b850fad478"}, - {file = "pywin32-305-cp310-cp310-win_arm64.whl", hash = "sha256:742eb905ce2187133a29365b428e6c3b9001d79accdc30aa8969afba1d8470f4"}, - {file = "pywin32-305-cp311-cp311-win32.whl", hash = "sha256:19ca459cd2e66c0e2cc9a09d589f71d827f26d47fe4a9d09175f6aa0256b51c2"}, - {file = "pywin32-305-cp311-cp311-win_amd64.whl", hash = "sha256:326f42ab4cfff56e77e3e595aeaf6c216712bbdd91e464d167c6434b28d65990"}, - {file = "pywin32-305-cp311-cp311-win_arm64.whl", hash = "sha256:4ecd404b2c6eceaca52f8b2e3e91b2187850a1ad3f8b746d0796a98b4cea04db"}, - {file = "pywin32-305-cp36-cp36m-win32.whl", hash = "sha256:48d8b1659284f3c17b68587af047d110d8c44837736b8932c034091683e05863"}, - {file = "pywin32-305-cp36-cp36m-win_amd64.whl", hash = "sha256:13362cc5aa93c2beaf489c9c9017c793722aeb56d3e5166dadd5ef82da021fe1"}, - {file = "pywin32-305-cp37-cp37m-win32.whl", hash = "sha256:a55db448124d1c1484df22fa8bbcbc45c64da5e6eae74ab095b9ea62e6d00496"}, - {file = "pywin32-305-cp37-cp37m-win_amd64.whl", hash = "sha256:109f98980bfb27e78f4df8a51a8198e10b0f347257d1e265bb1a32993d0c973d"}, - {file = "pywin32-305-cp38-cp38-win32.whl", hash = "sha256:9dd98384da775afa009bc04863426cb30596fd78c6f8e4e2e5bbf4edf8029504"}, - {file = "pywin32-305-cp38-cp38-win_amd64.whl", hash = "sha256:56d7a9c6e1a6835f521788f53b5af7912090674bb84ef5611663ee1595860fc7"}, - {file = "pywin32-305-cp39-cp39-win32.whl", hash = "sha256:9d968c677ac4d5cbdaa62fd3014ab241718e619d8e36ef8e11fb930515a1e918"}, - {file = "pywin32-305-cp39-cp39-win_amd64.whl", hash = "sha256:50768c6b7c3f0b38b7fb14dd4104da93ebced5f1a50dc0e834594bff6fbe1271"}, -] [[package]] name = "pyyaml" @@ -1836,48 +899,6 @@ description = "YAML parser and emitter for Python" category = "main" optional = false python-versions = ">=3.6" -files = [ - {file = "PyYAML-6.0-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:d4db7c7aef085872ef65a8fd7d6d09a14ae91f691dec3e87ee5ee0539d516f53"}, - {file = "PyYAML-6.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:9df7ed3b3d2e0ecfe09e14741b857df43adb5a3ddadc919a2d94fbdf78fea53c"}, - {file = "PyYAML-6.0-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:77f396e6ef4c73fdc33a9157446466f1cff553d979bd00ecb64385760c6babdc"}, - {file = "PyYAML-6.0-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:a80a78046a72361de73f8f395f1f1e49f956c6be882eed58505a15f3e430962b"}, - {file = "PyYAML-6.0-cp310-cp310-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:f84fbc98b019fef2ee9a1cb3ce93e3187a6df0b2538a651bfb890254ba9f90b5"}, - {file = "PyYAML-6.0-cp310-cp310-win32.whl", hash = "sha256:2cd5df3de48857ed0544b34e2d40e9fac445930039f3cfe4bcc592a1f836d513"}, - {file = "PyYAML-6.0-cp310-cp310-win_amd64.whl", hash = "sha256:daf496c58a8c52083df09b80c860005194014c3698698d1a57cbcfa182142a3a"}, - {file = "PyYAML-6.0-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:d4b0ba9512519522b118090257be113b9468d804b19d63c71dbcf4a48fa32358"}, - {file = "PyYAML-6.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:81957921f441d50af23654aa6c5e5eaf9b06aba7f0a19c18a538dc7ef291c5a1"}, - {file = "PyYAML-6.0-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:afa17f5bc4d1b10afd4466fd3a44dc0e245382deca5b3c353d8b757f9e3ecb8d"}, - {file = "PyYAML-6.0-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:dbad0e9d368bb989f4515da330b88a057617d16b6a8245084f1b05400f24609f"}, - {file = "PyYAML-6.0-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:432557aa2c09802be39460360ddffd48156e30721f5e8d917f01d31694216782"}, - {file = "PyYAML-6.0-cp311-cp311-win32.whl", hash = "sha256:bfaef573a63ba8923503d27530362590ff4f576c626d86a9fed95822a8255fd7"}, - {file = "PyYAML-6.0-cp311-cp311-win_amd64.whl", hash = "sha256:01b45c0191e6d66c470b6cf1b9531a771a83c1c4208272ead47a3ae4f2f603bf"}, - {file = "PyYAML-6.0-cp36-cp36m-macosx_10_9_x86_64.whl", hash = "sha256:897b80890765f037df3403d22bab41627ca8811ae55e9a722fd0392850ec4d86"}, - {file = "PyYAML-6.0-cp36-cp36m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:50602afada6d6cbfad699b0c7bb50d5ccffa7e46a3d738092afddc1f9758427f"}, - {file = "PyYAML-6.0-cp36-cp36m-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:48c346915c114f5fdb3ead70312bd042a953a8ce5c7106d5bfb1a5254e47da92"}, - {file = "PyYAML-6.0-cp36-cp36m-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:98c4d36e99714e55cfbaaee6dd5badbc9a1ec339ebfc3b1f52e293aee6bb71a4"}, - {file = "PyYAML-6.0-cp36-cp36m-win32.whl", hash = "sha256:0283c35a6a9fbf047493e3a0ce8d79ef5030852c51e9d911a27badfde0605293"}, - {file = "PyYAML-6.0-cp36-cp36m-win_amd64.whl", hash = "sha256:07751360502caac1c067a8132d150cf3d61339af5691fe9e87803040dbc5db57"}, - {file = "PyYAML-6.0-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:819b3830a1543db06c4d4b865e70ded25be52a2e0631ccd2f6a47a2822f2fd7c"}, - {file = "PyYAML-6.0-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:473f9edb243cb1935ab5a084eb238d842fb8f404ed2193a915d1784b5a6b5fc0"}, - {file = "PyYAML-6.0-cp37-cp37m-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:0ce82d761c532fe4ec3f87fc45688bdd3a4c1dc5e0b4a19814b9009a29baefd4"}, - {file = "PyYAML-6.0-cp37-cp37m-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:231710d57adfd809ef5d34183b8ed1eeae3f76459c18fb4a0b373ad56bedcdd9"}, - {file = "PyYAML-6.0-cp37-cp37m-win32.whl", hash = "sha256:c5687b8d43cf58545ade1fe3e055f70eac7a5a1a0bf42824308d868289a95737"}, - {file = "PyYAML-6.0-cp37-cp37m-win_amd64.whl", hash = "sha256:d15a181d1ecd0d4270dc32edb46f7cb7733c7c508857278d3d378d14d606db2d"}, - {file = "PyYAML-6.0-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:0b4624f379dab24d3725ffde76559cff63d9ec94e1736b556dacdfebe5ab6d4b"}, - {file = "PyYAML-6.0-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:213c60cd50106436cc818accf5baa1aba61c0189ff610f64f4a3e8c6726218ba"}, - {file = "PyYAML-6.0-cp38-cp38-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:9fa600030013c4de8165339db93d182b9431076eb98eb40ee068700c9c813e34"}, - {file = "PyYAML-6.0-cp38-cp38-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:277a0ef2981ca40581a47093e9e2d13b3f1fbbeffae064c1d21bfceba2030287"}, - {file = "PyYAML-6.0-cp38-cp38-win32.whl", hash = "sha256:d4eccecf9adf6fbcc6861a38015c2a64f38b9d94838ac1810a9023a0609e1b78"}, - {file = "PyYAML-6.0-cp38-cp38-win_amd64.whl", hash = "sha256:1e4747bc279b4f613a09eb64bba2ba602d8a6664c6ce6396a4d0cd413a50ce07"}, - {file = "PyYAML-6.0-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:055d937d65826939cb044fc8c9b08889e8c743fdc6a32b33e2390f66013e449b"}, - {file = "PyYAML-6.0-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:e61ceaab6f49fb8bdfaa0f92c4b57bcfbea54c09277b1b4f7ac376bfb7a7c174"}, - {file = "PyYAML-6.0-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:d67d839ede4ed1b28a4e8909735fc992a923cdb84e618544973d7dfc71540803"}, - {file = "PyYAML-6.0-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:cba8c411ef271aa037d7357a2bc8f9ee8b58b9965831d9e51baf703280dc73d3"}, - {file = "PyYAML-6.0-cp39-cp39-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:40527857252b61eacd1d9af500c3337ba8deb8fc298940291486c465c8b46ec0"}, - {file = "PyYAML-6.0-cp39-cp39-win32.whl", hash = "sha256:b5b9eccad747aabaaffbc6064800670f0c297e52c12754eb1d976c57e4f74dcb"}, - {file = "PyYAML-6.0-cp39-cp39-win_amd64.whl", hash = "sha256:b3d267842bf12586ba6c734f89d1f5b871df0273157918b0ccefa29deb05c21c"}, - {file = "PyYAML-6.0.tar.gz", hash = "sha256:68fb519c14306fec9720a2a5b45bc9f0c8d1b9c72adf45c37baedfcd949c35a2"}, -] [[package]] name = "requests" @@ -1886,10 +907,6 @@ description = "Python HTTP for Humans." category = "main" optional = false python-versions = ">=3.7, <4" -files = [ - {file = "requests-2.28.1-py3-none-any.whl", hash = "sha256:8fefa2a1a1365bf5520aac41836fbee479da67864514bdb821f31ce07ce65349"}, - {file = "requests-2.28.1.tar.gz", hash = "sha256:7c5599b102feddaa661c826c56ab4fee28bfd17f5abca1ebbe3e7f19d7c97983"}, -] [package.dependencies] certifi = ">=2017.4.17" @@ -1908,10 +925,6 @@ description = "Mock out responses from the requests package" category = "dev" optional = false python-versions = "*" -files = [ - {file = "requests-mock-1.10.0.tar.gz", hash = "sha256:59c9c32419a9fb1ae83ec242d98e889c45bd7d7a65d48375cc243ec08441658b"}, - {file = "requests_mock-1.10.0-py2.py3-none-any.whl", hash = "sha256:2fdbb637ad17ee15c06f33d31169e71bf9fe2bdb7bc9da26185be0dd8d842699"}, -] [package.dependencies] requests = ">=2.3,<3" @@ -1928,10 +941,6 @@ description = "OAuthlib authentication support for Requests." category = "main" optional = true python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*" -files = [ - {file = "requests-oauthlib-1.3.1.tar.gz", hash = "sha256:75beac4a47881eeb94d5ea5d6ad31ef88856affe2332b9aafb52c6452ccf0d7a"}, - {file = "requests_oauthlib-1.3.1-py2.py3-none-any.whl", hash = "sha256:2577c501a2fb8d05a304c09d090d6e47c306fef15809d102b327cf8364bddab5"}, -] [package.dependencies] oauthlib = ">=3.0.0" @@ -1947,10 +956,6 @@ description = "A utility library for mocking out the `requests` Python library." category = "dev" optional = false python-versions = ">=3.7" -files = [ - {file = "responses-0.22.0-py3-none-any.whl", hash = "sha256:dcf294d204d14c436fddcc74caefdbc5764795a40ff4e6a7740ed8ddbf3294be"}, - {file = "responses-0.22.0.tar.gz", hash = "sha256:396acb2a13d25297789a5866b4881cf4e46ffd49cc26c43ab1117f40b973102e"}, -] [package.dependencies] requests = ">=2.22.0,<3.0" @@ -1968,10 +973,6 @@ description = "Render rich text, tables, progress bars, syntax highlighting, mar category = "main" optional = false python-versions = ">=3.6.3,<4.0.0" -files = [ - {file = "rich-12.6.0-py3-none-any.whl", hash = "sha256:a4eb26484f2c82589bd9a17c73d32a010b1e29d89f1604cd9bf3a2097b81bb5e"}, - {file = "rich-12.6.0.tar.gz", hash = "sha256:ba3a3775974105c221d31141f2c116f4fd65c5ceb0698657a11e9f295ec93fd0"}, -] [package.dependencies] commonmark = ">=0.9.0,<0.10.0" @@ -1988,10 +989,6 @@ description = "Convenient Filesystem interface over S3" category = "main" optional = true python-versions = ">= 3.7" -files = [ - {file = "s3fs-2022.10.0-py3-none-any.whl", hash = "sha256:1e134c3577171699feb7c1a0c4713260d5b48296e1708737ff940baef6e2c153"}, - {file = "s3fs-2022.10.0.tar.gz", hash = "sha256:e8deb80f20bd0b2059141b874fdb9d6aeb8cce35312ea5f2c02b225a78a00406"}, -] [package.dependencies] aiobotocore = ">=2.4.0,<2.5.0" @@ -2009,10 +1006,6 @@ description = "An Amazon S3 Transfer Manager" category = "main" optional = false python-versions = ">= 3.7" -files = [ - {file = "s3transfer-0.6.0-py3-none-any.whl", hash = "sha256:06176b74f3a15f61f1b4f25a1fc29a4429040b7647133a463da8fa5bd28d5ecd"}, - {file = "s3transfer-0.6.0.tar.gz", hash = "sha256:2ed07d3866f523cc561bf4a00fc5535827981b117dd7876f036b0c1aca42c947"}, -] [package.dependencies] botocore = ">=1.12.36,<2.0a.0" @@ -2027,10 +1020,6 @@ description = "Easily download, build, install, upgrade, and uninstall Python pa category = "dev" optional = false python-versions = ">=3.7" -files = [ - {file = "setuptools-65.6.3-py3-none-any.whl", hash = "sha256:57f6f22bde4e042978bcd50176fdb381d7c21a9efa4041202288d3737a0c6a54"}, - {file = "setuptools-65.6.3.tar.gz", hash = "sha256:a7620757bf984b58deaf32fc8a4577a9bbc0850cf92c20e1ce41c38c19e5fb75"}, -] [package.extras] docs = ["furo", "jaraco.packaging (>=9)", "jaraco.tidelift (>=1.4)", "pygments-github-lexers (==0.0.5)", "rst.linker (>=1.9)", "sphinx (>=3.5)", "sphinx-favicon", "sphinx-hoverxref (<2)", "sphinx-inline-tabs", "sphinx-notfound-page (==0.8.3)", "sphinx-reredirects", "sphinxcontrib-towncrier"] @@ -2044,10 +1033,6 @@ description = "Python 2 and 3 compatibility utilities" category = "main" optional = false python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*" -files = [ - {file = "six-1.16.0-py2.py3-none-any.whl", hash = "sha256:8abb2f1d86890a2dfb989f9a77cfcfd3e47c2a354b01111771326f8aa26e0254"}, - {file = "six-1.16.0.tar.gz", hash = "sha256:1e61c37477a1626458e36f7b1d82aa5c9b094fa4802892072e49de9c60c4c926"}, -] [[package]] name = "thrift" @@ -2056,9 +1041,6 @@ description = "Python bindings for the Apache Thrift RPC system" category = "main" optional = true python-versions = "*" -files = [ - {file = "thrift-0.16.0.tar.gz", hash = "sha256:2b5b6488fcded21f9d312aa23c9ff6a0195d0f6ae26ddbd5ad9e3e25dfc14408"}, -] [package.dependencies] six = ">=1.7.2" @@ -2075,10 +1057,6 @@ description = "Python Library for Tom's Obvious, Minimal Language" category = "dev" optional = false python-versions = ">=2.6, !=3.0.*, !=3.1.*, !=3.2.*" -files = [ - {file = "toml-0.10.2-py2.py3-none-any.whl", hash = "sha256:806143ae5bfb6a3c6e736a764057db0e6a0e05e338b5630894a5f779cabb4f9b"}, - {file = "toml-0.10.2.tar.gz", hash = "sha256:b3bda1d108d5dd99f4a20d24d9c348e91c4db7ab1b749200bded2f839ccbe68f"}, -] [[package]] name = "tomli" @@ -2087,10 +1065,6 @@ description = "A lil' TOML parser" category = "dev" optional = false python-versions = ">=3.7" -files = [ - {file = "tomli-2.0.1-py3-none-any.whl", hash = "sha256:939de3e7a6161af0c887ef91b7d41a53e7c5a1ca976325f429cb46ea9bc30ecc"}, - {file = "tomli-2.0.1.tar.gz", hash = "sha256:de526c12914f0c550d15924c62d72abc48d6fe7364aa87328337a31007fe8a4f"}, -] [[package]] name = "types-toml" @@ -2099,10 +1073,6 @@ description = "Typing stubs for toml" category = "dev" optional = false python-versions = "*" -files = [ - {file = "types-toml-0.10.8.1.tar.gz", hash = "sha256:171bdb3163d79a520560f24ba916a9fc9bff81659c5448a9fea89240923722be"}, - {file = "types_toml-0.10.8.1-py3-none-any.whl", hash = "sha256:b7b5c4977f96ab7b5ac06d8a6590d17c0bf252a96efc03b109c2711fb3e0eafd"}, -] [[package]] name = "typing-extensions" @@ -2111,10 +1081,6 @@ description = "Backported and Experimental Type Hints for Python 3.7+" category = "main" optional = false python-versions = ">=3.7" -files = [ - {file = "typing_extensions-4.4.0-py3-none-any.whl", hash = "sha256:16fa4864408f655d35ec496218b85f79b3437c829e93320c7c9215ccfd92489e"}, - {file = "typing_extensions-4.4.0.tar.gz", hash = "sha256:1511434bb92bf8dd198c12b1cc812e800d4181cfcb867674e0f8279cc93087aa"}, -] [[package]] name = "urllib3" @@ -2123,10 +1089,6 @@ description = "HTTP library with thread-safe connection pooling, file post, and category = "main" optional = false python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*, !=3.5.*" -files = [ - {file = "urllib3-1.26.13-py2.py3-none-any.whl", hash = "sha256:47cc05d99aaa09c9e72ed5809b60e7ba354e64b59c9c173ac3018642d8bb41fc"}, - {file = "urllib3-1.26.13.tar.gz", hash = "sha256:c083dd0dce68dbfbe1129d5271cb90f9447dea7d52097c6e0126120c521ddea8"}, -] [package.extras] brotli = ["brotli (>=1.0.9)", "brotlicffi (>=0.8.0)", "brotlipy (>=0.6.0)"] @@ -2140,10 +1102,6 @@ description = "Virtual Python Environment builder" category = "dev" optional = false python-versions = ">=3.6" -files = [ - {file = "virtualenv-20.17.1-py3-none-any.whl", hash = "sha256:ce3b1684d6e1a20a3e5ed36795a97dfc6af29bc3970ca8dab93e11ac6094b3c4"}, - {file = "virtualenv-20.17.1.tar.gz", hash = "sha256:f8b927684efc6f1cc206c9db297a570ab9ad0e51c16fa9e45487d36d1905c058"}, -] [package.dependencies] distlib = ">=0.3.6,<1" @@ -2161,10 +1119,6 @@ description = "The comprehensive WSGI web application library." category = "dev" optional = false python-versions = ">=3.7" -files = [ - {file = "Werkzeug-2.2.2-py3-none-any.whl", hash = "sha256:f979ab81f58d7318e064e99c4506445d60135ac5cd2e177a2de0089bfd4c9bd5"}, - {file = "Werkzeug-2.2.2.tar.gz", hash = "sha256:7ea2d48322cc7c0f8b3a215ed73eabd7b5d75d0b50e31ab006286ccff9e00b8f"}, -] [package.dependencies] MarkupSafe = ">=2.1.1" @@ -2179,72 +1133,6 @@ description = "Module for decorators, wrappers and monkey patching." category = "main" optional = true python-versions = "!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,!=3.4.*,>=2.7" -files = [ - {file = "wrapt-1.14.1-cp27-cp27m-macosx_10_9_x86_64.whl", hash = "sha256:1b376b3f4896e7930f1f772ac4b064ac12598d1c38d04907e696cc4d794b43d3"}, - {file = "wrapt-1.14.1-cp27-cp27m-manylinux1_i686.whl", hash = "sha256:903500616422a40a98a5a3c4ff4ed9d0066f3b4c951fa286018ecdf0750194ef"}, - {file = "wrapt-1.14.1-cp27-cp27m-manylinux1_x86_64.whl", hash = "sha256:5a9a0d155deafd9448baff28c08e150d9b24ff010e899311ddd63c45c2445e28"}, - {file = "wrapt-1.14.1-cp27-cp27m-manylinux2010_i686.whl", hash = "sha256:ddaea91abf8b0d13443f6dac52e89051a5063c7d014710dcb4d4abb2ff811a59"}, - {file = "wrapt-1.14.1-cp27-cp27m-manylinux2010_x86_64.whl", hash = "sha256:36f582d0c6bc99d5f39cd3ac2a9062e57f3cf606ade29a0a0d6b323462f4dd87"}, - {file = "wrapt-1.14.1-cp27-cp27mu-manylinux1_i686.whl", hash = "sha256:7ef58fb89674095bfc57c4069e95d7a31cfdc0939e2a579882ac7d55aadfd2a1"}, - {file = "wrapt-1.14.1-cp27-cp27mu-manylinux1_x86_64.whl", hash = "sha256:e2f83e18fe2f4c9e7db597e988f72712c0c3676d337d8b101f6758107c42425b"}, - {file = "wrapt-1.14.1-cp27-cp27mu-manylinux2010_i686.whl", hash = "sha256:ee2b1b1769f6707a8a445162ea16dddf74285c3964f605877a20e38545c3c462"}, - {file = "wrapt-1.14.1-cp27-cp27mu-manylinux2010_x86_64.whl", hash = "sha256:833b58d5d0b7e5b9832869f039203389ac7cbf01765639c7309fd50ef619e0b1"}, - {file = "wrapt-1.14.1-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:80bb5c256f1415f747011dc3604b59bc1f91c6e7150bd7db03b19170ee06b320"}, - {file = "wrapt-1.14.1-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:07f7a7d0f388028b2df1d916e94bbb40624c59b48ecc6cbc232546706fac74c2"}, - {file = "wrapt-1.14.1-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:02b41b633c6261feff8ddd8d11c711df6842aba629fdd3da10249a53211a72c4"}, - {file = "wrapt-1.14.1-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:2fe803deacd09a233e4762a1adcea5db5d31e6be577a43352936179d14d90069"}, - {file = "wrapt-1.14.1-cp310-cp310-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:257fd78c513e0fb5cdbe058c27a0624c9884e735bbd131935fd49e9fe719d310"}, - {file = "wrapt-1.14.1-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:4fcc4649dc762cddacd193e6b55bc02edca674067f5f98166d7713b193932b7f"}, - {file = "wrapt-1.14.1-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:11871514607b15cfeb87c547a49bca19fde402f32e2b1c24a632506c0a756656"}, - {file = "wrapt-1.14.1-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:8ad85f7f4e20964db4daadcab70b47ab05c7c1cf2a7c1e51087bfaa83831854c"}, - {file = "wrapt-1.14.1-cp310-cp310-win32.whl", hash = "sha256:a9a52172be0b5aae932bef82a79ec0a0ce87288c7d132946d645eba03f0ad8a8"}, - {file = "wrapt-1.14.1-cp310-cp310-win_amd64.whl", hash = "sha256:6d323e1554b3d22cfc03cd3243b5bb815a51f5249fdcbb86fda4bf62bab9e164"}, - {file = "wrapt-1.14.1-cp35-cp35m-manylinux1_i686.whl", hash = "sha256:43ca3bbbe97af00f49efb06e352eae40434ca9d915906f77def219b88e85d907"}, - {file = "wrapt-1.14.1-cp35-cp35m-manylinux1_x86_64.whl", hash = "sha256:6b1a564e6cb69922c7fe3a678b9f9a3c54e72b469875aa8018f18b4d1dd1adf3"}, - {file = "wrapt-1.14.1-cp35-cp35m-manylinux2010_i686.whl", hash = "sha256:00b6d4ea20a906c0ca56d84f93065b398ab74b927a7a3dbd470f6fc503f95dc3"}, - {file = "wrapt-1.14.1-cp35-cp35m-manylinux2010_x86_64.whl", hash = "sha256:a85d2b46be66a71bedde836d9e41859879cc54a2a04fad1191eb50c2066f6e9d"}, - {file = "wrapt-1.14.1-cp35-cp35m-win32.whl", hash = "sha256:dbcda74c67263139358f4d188ae5faae95c30929281bc6866d00573783c422b7"}, - {file = "wrapt-1.14.1-cp35-cp35m-win_amd64.whl", hash = "sha256:b21bb4c09ffabfa0e85e3a6b623e19b80e7acd709b9f91452b8297ace2a8ab00"}, - {file = "wrapt-1.14.1-cp36-cp36m-macosx_10_9_x86_64.whl", hash = "sha256:9e0fd32e0148dd5dea6af5fee42beb949098564cc23211a88d799e434255a1f4"}, - {file = "wrapt-1.14.1-cp36-cp36m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:9736af4641846491aedb3c3f56b9bc5568d92b0692303b5a305301a95dfd38b1"}, - {file = "wrapt-1.14.1-cp36-cp36m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:5b02d65b9ccf0ef6c34cba6cf5bf2aab1bb2f49c6090bafeecc9cd81ad4ea1c1"}, - {file = "wrapt-1.14.1-cp36-cp36m-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:21ac0156c4b089b330b7666db40feee30a5d52634cc4560e1905d6529a3897ff"}, - {file = "wrapt-1.14.1-cp36-cp36m-musllinux_1_1_aarch64.whl", hash = "sha256:9f3e6f9e05148ff90002b884fbc2a86bd303ae847e472f44ecc06c2cd2fcdb2d"}, - {file = "wrapt-1.14.1-cp36-cp36m-musllinux_1_1_i686.whl", hash = "sha256:6e743de5e9c3d1b7185870f480587b75b1cb604832e380d64f9504a0535912d1"}, - {file = "wrapt-1.14.1-cp36-cp36m-musllinux_1_1_x86_64.whl", hash = "sha256:d79d7d5dc8a32b7093e81e97dad755127ff77bcc899e845f41bf71747af0c569"}, - {file = "wrapt-1.14.1-cp36-cp36m-win32.whl", hash = "sha256:81b19725065dcb43df02b37e03278c011a09e49757287dca60c5aecdd5a0b8ed"}, - {file = "wrapt-1.14.1-cp36-cp36m-win_amd64.whl", hash = "sha256:b014c23646a467558be7da3d6b9fa409b2c567d2110599b7cf9a0c5992b3b471"}, - {file = "wrapt-1.14.1-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:88bd7b6bd70a5b6803c1abf6bca012f7ed963e58c68d76ee20b9d751c74a3248"}, - {file = "wrapt-1.14.1-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:b5901a312f4d14c59918c221323068fad0540e34324925c8475263841dbdfe68"}, - {file = "wrapt-1.14.1-cp37-cp37m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:d77c85fedff92cf788face9bfa3ebaa364448ebb1d765302e9af11bf449ca36d"}, - {file = "wrapt-1.14.1-cp37-cp37m-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:8d649d616e5c6a678b26d15ece345354f7c2286acd6db868e65fcc5ff7c24a77"}, - {file = "wrapt-1.14.1-cp37-cp37m-musllinux_1_1_aarch64.whl", hash = "sha256:7d2872609603cb35ca513d7404a94d6d608fc13211563571117046c9d2bcc3d7"}, - {file = "wrapt-1.14.1-cp37-cp37m-musllinux_1_1_i686.whl", hash = "sha256:ee6acae74a2b91865910eef5e7de37dc6895ad96fa23603d1d27ea69df545015"}, - {file = "wrapt-1.14.1-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:2b39d38039a1fdad98c87279b48bc5dce2c0ca0d73483b12cb72aa9609278e8a"}, - {file = "wrapt-1.14.1-cp37-cp37m-win32.whl", hash = "sha256:60db23fa423575eeb65ea430cee741acb7c26a1365d103f7b0f6ec412b893853"}, - {file = "wrapt-1.14.1-cp37-cp37m-win_amd64.whl", hash = "sha256:709fe01086a55cf79d20f741f39325018f4df051ef39fe921b1ebe780a66184c"}, - {file = "wrapt-1.14.1-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:8c0ce1e99116d5ab21355d8ebe53d9460366704ea38ae4d9f6933188f327b456"}, - {file = "wrapt-1.14.1-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:e3fb1677c720409d5f671e39bac6c9e0e422584e5f518bfd50aa4cbbea02433f"}, - {file = "wrapt-1.14.1-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:642c2e7a804fcf18c222e1060df25fc210b9c58db7c91416fb055897fc27e8cc"}, - {file = "wrapt-1.14.1-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:7b7c050ae976e286906dd3f26009e117eb000fb2cf3533398c5ad9ccc86867b1"}, - {file = "wrapt-1.14.1-cp38-cp38-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:ef3f72c9666bba2bab70d2a8b79f2c6d2c1a42a7f7e2b0ec83bb2f9e383950af"}, - {file = "wrapt-1.14.1-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:01c205616a89d09827986bc4e859bcabd64f5a0662a7fe95e0d359424e0e071b"}, - {file = "wrapt-1.14.1-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:5a0f54ce2c092aaf439813735584b9537cad479575a09892b8352fea5e988dc0"}, - {file = "wrapt-1.14.1-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:2cf71233a0ed05ccdabe209c606fe0bac7379fdcf687f39b944420d2a09fdb57"}, - {file = "wrapt-1.14.1-cp38-cp38-win32.whl", hash = "sha256:aa31fdcc33fef9eb2552cbcbfee7773d5a6792c137b359e82879c101e98584c5"}, - {file = "wrapt-1.14.1-cp38-cp38-win_amd64.whl", hash = "sha256:d1967f46ea8f2db647c786e78d8cc7e4313dbd1b0aca360592d8027b8508e24d"}, - {file = "wrapt-1.14.1-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:3232822c7d98d23895ccc443bbdf57c7412c5a65996c30442ebe6ed3df335383"}, - {file = "wrapt-1.14.1-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:988635d122aaf2bdcef9e795435662bcd65b02f4f4c1ae37fbee7401c440b3a7"}, - {file = "wrapt-1.14.1-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:9cca3c2cdadb362116235fdbd411735de4328c61425b0aa9f872fd76d02c4e86"}, - {file = "wrapt-1.14.1-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:d52a25136894c63de15a35bc0bdc5adb4b0e173b9c0d07a2be9d3ca64a332735"}, - {file = "wrapt-1.14.1-cp39-cp39-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:40e7bc81c9e2b2734ea4bc1aceb8a8f0ceaac7c5299bc5d69e37c44d9081d43b"}, - {file = "wrapt-1.14.1-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:b9b7a708dd92306328117d8c4b62e2194d00c365f18eff11a9b53c6f923b01e3"}, - {file = "wrapt-1.14.1-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:6a9a25751acb379b466ff6be78a315e2b439d4c94c1e99cb7266d40a537995d3"}, - {file = "wrapt-1.14.1-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:34aa51c45f28ba7f12accd624225e2b1e5a3a45206aa191f6f9aac931d9d56fe"}, - {file = "wrapt-1.14.1-cp39-cp39-win32.whl", hash = "sha256:dee0ce50c6a2dd9056c20db781e9c1cfd33e77d2d569f5d1d9321c641bb903d5"}, - {file = "wrapt-1.14.1-cp39-cp39-win_amd64.whl", hash = "sha256:dee60e1de1898bde3b238f18340eec6148986da0455d8ba7848d50470a7a32fb"}, - {file = "wrapt-1.14.1.tar.gz", hash = "sha256:380a85cf89e0e69b7cfbe2ea9f765f004ff419f34194018a6827ac0e3edfed4d"}, -] [[package]] name = "xmltodict" @@ -2253,10 +1141,6 @@ description = "Makes working with XML feel like you are working with JSON" category = "dev" optional = false python-versions = ">=3.4" -files = [ - {file = "xmltodict-0.13.0-py2.py3-none-any.whl", hash = "sha256:aa89e8fd76320154a40d19a0df04a4695fb9dc5ba977cbb68ab3e4eb225e7852"}, - {file = "xmltodict-0.13.0.tar.gz", hash = "sha256:341595a488e3e01a85a9d8911d8912fd922ede5fecc4dce437eb4b6c8d037e56"}, -] [[package]] name = "yarl" @@ -2265,54 +1149,1216 @@ description = "Yet another URL library" category = "main" optional = true python-versions = ">=3.7" -files = [ - {file = "yarl-1.8.2-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:bb81f753c815f6b8e2ddd2eef3c855cf7da193b82396ac013c661aaa6cc6b0a5"}, - {file = "yarl-1.8.2-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:47d49ac96156f0928f002e2424299b2c91d9db73e08c4cd6742923a086f1c863"}, - {file = "yarl-1.8.2-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:3fc056e35fa6fba63248d93ff6e672c096f95f7836938241ebc8260e062832fe"}, - {file = "yarl-1.8.2-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:58a3c13d1c3005dbbac5c9f0d3210b60220a65a999b1833aa46bd6677c69b08e"}, - {file = "yarl-1.8.2-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:10b08293cda921157f1e7c2790999d903b3fd28cd5c208cf8826b3b508026996"}, - {file = "yarl-1.8.2-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:de986979bbd87272fe557e0a8fcb66fd40ae2ddfe28a8b1ce4eae22681728fef"}, - {file = "yarl-1.8.2-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:6c4fcfa71e2c6a3cb568cf81aadc12768b9995323186a10827beccf5fa23d4f8"}, - {file = "yarl-1.8.2-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:ae4d7ff1049f36accde9e1ef7301912a751e5bae0a9d142459646114c70ecba6"}, - {file = "yarl-1.8.2-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:bf071f797aec5b96abfc735ab97da9fd8f8768b43ce2abd85356a3127909d146"}, - {file = "yarl-1.8.2-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:74dece2bfc60f0f70907c34b857ee98f2c6dd0f75185db133770cd67300d505f"}, - {file = "yarl-1.8.2-cp310-cp310-musllinux_1_1_ppc64le.whl", hash = "sha256:df60a94d332158b444301c7f569659c926168e4d4aad2cfbf4bce0e8fb8be826"}, - {file = "yarl-1.8.2-cp310-cp310-musllinux_1_1_s390x.whl", hash = "sha256:63243b21c6e28ec2375f932a10ce7eda65139b5b854c0f6b82ed945ba526bff3"}, - {file = "yarl-1.8.2-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:cfa2bbca929aa742b5084fd4663dd4b87c191c844326fcb21c3afd2d11497f80"}, - {file = "yarl-1.8.2-cp310-cp310-win32.whl", hash = "sha256:b05df9ea7496df11b710081bd90ecc3a3db6adb4fee36f6a411e7bc91a18aa42"}, - {file = "yarl-1.8.2-cp310-cp310-win_amd64.whl", hash = "sha256:24ad1d10c9db1953291f56b5fe76203977f1ed05f82d09ec97acb623a7976574"}, - {file = "yarl-1.8.2-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:2a1fca9588f360036242f379bfea2b8b44cae2721859b1c56d033adfd5893634"}, - {file = "yarl-1.8.2-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:f37db05c6051eff17bc832914fe46869f8849de5b92dc4a3466cd63095d23dfd"}, - {file = "yarl-1.8.2-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:77e913b846a6b9c5f767b14dc1e759e5aff05502fe73079f6f4176359d832581"}, - {file = "yarl-1.8.2-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:0978f29222e649c351b173da2b9b4665ad1feb8d1daa9d971eb90df08702668a"}, - {file = "yarl-1.8.2-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:388a45dc77198b2460eac0aca1efd6a7c09e976ee768b0d5109173e521a19daf"}, - {file = "yarl-1.8.2-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:2305517e332a862ef75be8fad3606ea10108662bc6fe08509d5ca99503ac2aee"}, - {file = "yarl-1.8.2-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:42430ff511571940d51e75cf42f1e4dbdded477e71c1b7a17f4da76c1da8ea76"}, - {file = "yarl-1.8.2-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:3150078118f62371375e1e69b13b48288e44f6691c1069340081c3fd12c94d5b"}, - {file = "yarl-1.8.2-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:c15163b6125db87c8f53c98baa5e785782078fbd2dbeaa04c6141935eb6dab7a"}, - {file = "yarl-1.8.2-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:4d04acba75c72e6eb90745447d69f84e6c9056390f7a9724605ca9c56b4afcc6"}, - {file = "yarl-1.8.2-cp311-cp311-musllinux_1_1_ppc64le.whl", hash = "sha256:e7fd20d6576c10306dea2d6a5765f46f0ac5d6f53436217913e952d19237efc4"}, - {file = "yarl-1.8.2-cp311-cp311-musllinux_1_1_s390x.whl", hash = "sha256:75c16b2a900b3536dfc7014905a128a2bea8fb01f9ee26d2d7d8db0a08e7cb2c"}, - {file = "yarl-1.8.2-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:6d88056a04860a98341a0cf53e950e3ac9f4e51d1b6f61a53b0609df342cc8b2"}, - {file = "yarl-1.8.2-cp311-cp311-win32.whl", hash = "sha256:fb742dcdd5eec9f26b61224c23baea46c9055cf16f62475e11b9b15dfd5c117b"}, - {file = "yarl-1.8.2-cp311-cp311-win_amd64.whl", hash = "sha256:8c46d3d89902c393a1d1e243ac847e0442d0196bbd81aecc94fcebbc2fd5857c"}, - {file = "yarl-1.8.2-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:ceff9722e0df2e0a9e8a79c610842004fa54e5b309fe6d218e47cd52f791d7ef"}, - {file = "yarl-1.8.2-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:3f6b4aca43b602ba0f1459de647af954769919c4714706be36af670a5f44c9c1"}, - {file = "yarl-1.8.2-cp37-cp37m-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:1684a9bd9077e922300ecd48003ddae7a7474e0412bea38d4631443a91d61077"}, - {file = "yarl-1.8.2-cp37-cp37m-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:ebb78745273e51b9832ef90c0898501006670d6e059f2cdb0e999494eb1450c2"}, - {file = "yarl-1.8.2-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:3adeef150d528ded2a8e734ebf9ae2e658f4c49bf413f5f157a470e17a4a2e89"}, - {file = "yarl-1.8.2-cp37-cp37m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:57a7c87927a468e5a1dc60c17caf9597161d66457a34273ab1760219953f7f4c"}, - {file = "yarl-1.8.2-cp37-cp37m-musllinux_1_1_aarch64.whl", hash = "sha256:efff27bd8cbe1f9bd127e7894942ccc20c857aa8b5a0327874f30201e5ce83d0"}, - {file = "yarl-1.8.2-cp37-cp37m-musllinux_1_1_i686.whl", hash = "sha256:a783cd344113cb88c5ff7ca32f1f16532a6f2142185147822187913eb989f739"}, - {file = "yarl-1.8.2-cp37-cp37m-musllinux_1_1_ppc64le.whl", hash = "sha256:705227dccbe96ab02c7cb2c43e1228e2826e7ead880bb19ec94ef279e9555b5b"}, - {file = "yarl-1.8.2-cp37-cp37m-musllinux_1_1_s390x.whl", hash = "sha256:34c09b43bd538bf6c4b891ecce94b6fa4f1f10663a8d4ca589a079a5018f6ed7"}, - {file = "yarl-1.8.2-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:a48f4f7fea9a51098b02209d90297ac324241bf37ff6be6d2b0149ab2bd51b37"}, - {file = "yarl-1.8.2-cp37-cp37m-win32.whl", hash = "sha256:0414fd91ce0b763d4eadb4456795b307a71524dbacd015c657bb2a39db2eab89"}, - {file = "yarl-1.8.2-cp37-cp37m-win_amd64.whl", hash = "sha256:d881d152ae0007809c2c02e22aa534e702f12071e6b285e90945aa3c376463c5"}, - {file = "yarl-1.8.2-cp38-cp38-macosx_10_9_universal2.whl", hash = "sha256:5df5e3d04101c1e5c3b1d69710b0574171cc02fddc4b23d1b2813e75f35a30b1"}, - {file = "yarl-1.8.2-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:7a66c506ec67eb3159eea5096acd05f5e788ceec7b96087d30c7d2865a243918"}, - {file = "yarl-1.8.2-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:2b4fa2606adf392051d990c3b3877d768771adc3faf2e117b9de7eb977741229"}, - {file = "yarl-1.8.2-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:1e21fb44e1eff06dd6ef971d4bdc611807d6bd3691223d9c01a18cec3677939e"}, + +[package.dependencies] +idna = ">=2.0" +multidict = ">=4.0" + +[[package]] +name = "zipp" +version = "3.11.0" +description = "Backport of pathlib-compatible object wrapper for zip files" +category = "dev" +optional = false +python-versions = ">=3.7" + +[package.extras] +docs = ["furo", "jaraco.packaging (>=9)", "jaraco.tidelift (>=1.4)", "rst.linker (>=1.9)", "sphinx (>=3.5)"] +testing = ["flake8 (<5)", "func-timeout", "jaraco.functools", "jaraco.itertools", "more-itertools", "pytest (>=6)", "pytest-black (>=0.3.7)", "pytest-checkdocs (>=2.4)", "pytest-cov", "pytest-enabler (>=1.3)", "pytest-flake8", "pytest-mypy (>=0.9.1)"] + +[[package]] +name = "zstandard" +version = "0.19.0" +description = "Zstandard bindings for Python" +category = "main" +optional = false +python-versions = ">=3.6" + +[package.dependencies] +cffi = {version = ">=1.11", markers = "platform_python_implementation == \"PyPy\""} + +[package.extras] +cffi = ["cffi (>=1.11)"] + +[extras] +adlfs = ["adlfs"] +duckdb = ["duckdb", "pyarrow"] +glue = ["boto3"] +hive = ["thrift"] +pandas = ["pandas", "pyarrow"] +pyarrow = ["pyarrow"] +s3fs = ["s3fs"] +snappy = ["python-snappy"] + +[metadata] +lock-version = "1.1" +python-versions = "^3.8" +content-hash = "f2d6c78025050536a5ef051ddb0023fe16a41d502399f0c4fc5b3e072fef9aec" + +[metadata.files] +adal = [ + {file = "adal-1.2.7-py2.py3-none-any.whl", hash = "sha256:2a7451ed7441ddbc57703042204a3e30ef747478eea022c70f789fc7f084bc3d"}, + {file = "adal-1.2.7.tar.gz", hash = "sha256:d74f45b81317454d96e982fd1c50e6fb5c99ac2223728aea8764433a39f566f1"}, +] +adlfs = [ + {file = "adlfs-2022.10.0-py3-none-any.whl", hash = "sha256:dc63b248f109bca2000d306f7a51c33d6859e4692dda7c6198b6300f8e29faaf"}, + {file = "adlfs-2022.10.0.tar.gz", hash = "sha256:8542f9e6799aeb77adb5cb9854f76b7dce874610dc0f2ff006cce960526ffad6"}, +] +aiobotocore = [ + {file = "aiobotocore-2.4.1-py3-none-any.whl", hash = "sha256:26e0d55d5901f487c3467616c028abb85036ca33a0f88e279770adae6b865c69"}, + {file = "aiobotocore-2.4.1.tar.gz", hash = "sha256:5c8ee79d1f14273e3f8f3af5c893db889ef163227b178ecacb5aea4547d3b9a7"}, +] +aiohttp = [ + {file = "aiohttp-3.8.3-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:ba71c9b4dcbb16212f334126cc3d8beb6af377f6703d9dc2d9fb3874fd667ee9"}, + {file = "aiohttp-3.8.3-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:d24b8bb40d5c61ef2d9b6a8f4528c2f17f1c5d2d31fed62ec860f6006142e83e"}, + {file = "aiohttp-3.8.3-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:f88df3a83cf9df566f171adba39d5bd52814ac0b94778d2448652fc77f9eb491"}, + {file = "aiohttp-3.8.3-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:b97decbb3372d4b69e4d4c8117f44632551c692bb1361b356a02b97b69e18a62"}, + {file = "aiohttp-3.8.3-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:309aa21c1d54b8ef0723181d430347d7452daaff93e8e2363db8e75c72c2fb2d"}, + {file = "aiohttp-3.8.3-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:ad5383a67514e8e76906a06741febd9126fc7c7ff0f599d6fcce3e82b80d026f"}, + {file = "aiohttp-3.8.3-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:20acae4f268317bb975671e375493dbdbc67cddb5f6c71eebdb85b34444ac46b"}, + {file = "aiohttp-3.8.3-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:05a3c31c6d7cd08c149e50dc7aa2568317f5844acd745621983380597f027a18"}, + {file = "aiohttp-3.8.3-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:d6f76310355e9fae637c3162936e9504b4767d5c52ca268331e2756e54fd4ca5"}, + {file = "aiohttp-3.8.3-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:256deb4b29fe5e47893fa32e1de2d73c3afe7407738bd3c63829874661d4822d"}, + {file = "aiohttp-3.8.3-cp310-cp310-musllinux_1_1_ppc64le.whl", hash = "sha256:5c59fcd80b9049b49acd29bd3598cada4afc8d8d69bd4160cd613246912535d7"}, + {file = "aiohttp-3.8.3-cp310-cp310-musllinux_1_1_s390x.whl", hash = "sha256:059a91e88f2c00fe40aed9031b3606c3f311414f86a90d696dd982e7aec48142"}, + {file = "aiohttp-3.8.3-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:2feebbb6074cdbd1ac276dbd737b40e890a1361b3cc30b74ac2f5e24aab41f7b"}, + {file = "aiohttp-3.8.3-cp310-cp310-win32.whl", hash = "sha256:5bf651afd22d5f0c4be16cf39d0482ea494f5c88f03e75e5fef3a85177fecdeb"}, + {file = "aiohttp-3.8.3-cp310-cp310-win_amd64.whl", hash = "sha256:653acc3880459f82a65e27bd6526e47ddf19e643457d36a2250b85b41a564715"}, + {file = "aiohttp-3.8.3-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:86fc24e58ecb32aee09f864cb11bb91bc4c1086615001647dbfc4dc8c32f4008"}, + {file = "aiohttp-3.8.3-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:75e14eac916f024305db517e00a9252714fce0abcb10ad327fb6dcdc0d060f1d"}, + {file = "aiohttp-3.8.3-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:d1fde0f44029e02d02d3993ad55ce93ead9bb9b15c6b7ccd580f90bd7e3de476"}, + {file = "aiohttp-3.8.3-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:4ab94426ddb1ecc6a0b601d832d5d9d421820989b8caa929114811369673235c"}, + {file = "aiohttp-3.8.3-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:89d2e02167fa95172c017732ed7725bc8523c598757f08d13c5acca308e1a061"}, + {file = "aiohttp-3.8.3-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:02f9a2c72fc95d59b881cf38a4b2be9381b9527f9d328771e90f72ac76f31ad8"}, + {file = "aiohttp-3.8.3-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:9c7149272fb5834fc186328e2c1fa01dda3e1fa940ce18fded6d412e8f2cf76d"}, + {file = "aiohttp-3.8.3-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:512bd5ab136b8dc0ffe3fdf2dfb0c4b4f49c8577f6cae55dca862cd37a4564e2"}, + {file = "aiohttp-3.8.3-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:7018ecc5fe97027214556afbc7c502fbd718d0740e87eb1217b17efd05b3d276"}, + {file = "aiohttp-3.8.3-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:88c70ed9da9963d5496d38320160e8eb7e5f1886f9290475a881db12f351ab5d"}, + {file = "aiohttp-3.8.3-cp311-cp311-musllinux_1_1_ppc64le.whl", hash = "sha256:da22885266bbfb3f78218dc40205fed2671909fbd0720aedba39b4515c038091"}, + {file = "aiohttp-3.8.3-cp311-cp311-musllinux_1_1_s390x.whl", hash = "sha256:e65bc19919c910127c06759a63747ebe14f386cda573d95bcc62b427ca1afc73"}, + {file = "aiohttp-3.8.3-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:08c78317e950e0762c2983f4dd58dc5e6c9ff75c8a0efeae299d363d439c8e34"}, + {file = "aiohttp-3.8.3-cp311-cp311-win32.whl", hash = "sha256:45d88b016c849d74ebc6f2b6e8bc17cabf26e7e40c0661ddd8fae4c00f015697"}, + {file = "aiohttp-3.8.3-cp311-cp311-win_amd64.whl", hash = "sha256:96372fc29471646b9b106ee918c8eeb4cca423fcbf9a34daa1b93767a88a2290"}, + {file = "aiohttp-3.8.3-cp36-cp36m-macosx_10_9_x86_64.whl", hash = "sha256:c971bf3786b5fad82ce5ad570dc6ee420f5b12527157929e830f51c55dc8af77"}, + {file = "aiohttp-3.8.3-cp36-cp36m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:ff25f48fc8e623d95eca0670b8cc1469a83783c924a602e0fbd47363bb54aaca"}, + {file = "aiohttp-3.8.3-cp36-cp36m-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:e381581b37db1db7597b62a2e6b8b57c3deec95d93b6d6407c5b61ddc98aca6d"}, + {file = "aiohttp-3.8.3-cp36-cp36m-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:db19d60d846283ee275d0416e2a23493f4e6b6028825b51290ac05afc87a6f97"}, + {file = "aiohttp-3.8.3-cp36-cp36m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:25892c92bee6d9449ffac82c2fe257f3a6f297792cdb18ad784737d61e7a9a85"}, + {file = "aiohttp-3.8.3-cp36-cp36m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:398701865e7a9565d49189f6c90868efaca21be65c725fc87fc305906be915da"}, + {file = "aiohttp-3.8.3-cp36-cp36m-musllinux_1_1_aarch64.whl", hash = "sha256:4a4fbc769ea9b6bd97f4ad0b430a6807f92f0e5eb020f1e42ece59f3ecfc4585"}, + {file = "aiohttp-3.8.3-cp36-cp36m-musllinux_1_1_i686.whl", hash = "sha256:b29bfd650ed8e148f9c515474a6ef0ba1090b7a8faeee26b74a8ff3b33617502"}, + {file = "aiohttp-3.8.3-cp36-cp36m-musllinux_1_1_ppc64le.whl", hash = "sha256:1e56b9cafcd6531bab5d9b2e890bb4937f4165109fe98e2b98ef0dcfcb06ee9d"}, + {file = "aiohttp-3.8.3-cp36-cp36m-musllinux_1_1_s390x.whl", hash = "sha256:ec40170327d4a404b0d91855d41bfe1fe4b699222b2b93e3d833a27330a87a6d"}, + {file = "aiohttp-3.8.3-cp36-cp36m-musllinux_1_1_x86_64.whl", hash = "sha256:2df5f139233060578d8c2c975128fb231a89ca0a462b35d4b5fcf7c501ebdbe1"}, + {file = "aiohttp-3.8.3-cp36-cp36m-win32.whl", hash = "sha256:f973157ffeab5459eefe7b97a804987876dd0a55570b8fa56b4e1954bf11329b"}, + {file = "aiohttp-3.8.3-cp36-cp36m-win_amd64.whl", hash = "sha256:437399385f2abcd634865705bdc180c8314124b98299d54fe1d4c8990f2f9494"}, + {file = "aiohttp-3.8.3-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:09e28f572b21642128ef31f4e8372adb6888846f32fecb288c8b0457597ba61a"}, + {file = "aiohttp-3.8.3-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:6f3553510abdbec67c043ca85727396ceed1272eef029b050677046d3387be8d"}, + {file = "aiohttp-3.8.3-cp37-cp37m-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:e168a7560b7c61342ae0412997b069753f27ac4862ec7867eff74f0fe4ea2ad9"}, + {file = "aiohttp-3.8.3-cp37-cp37m-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:db4c979b0b3e0fa7e9e69ecd11b2b3174c6963cebadeecfb7ad24532ffcdd11a"}, + {file = "aiohttp-3.8.3-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:e164e0a98e92d06da343d17d4e9c4da4654f4a4588a20d6c73548a29f176abe2"}, + {file = "aiohttp-3.8.3-cp37-cp37m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:e8a78079d9a39ca9ca99a8b0ac2fdc0c4d25fc80c8a8a82e5c8211509c523363"}, + {file = "aiohttp-3.8.3-cp37-cp37m-musllinux_1_1_aarch64.whl", hash = "sha256:21b30885a63c3f4ff5b77a5d6caf008b037cb521a5f33eab445dc566f6d092cc"}, + {file = "aiohttp-3.8.3-cp37-cp37m-musllinux_1_1_i686.whl", hash = "sha256:4b0f30372cef3fdc262f33d06e7b411cd59058ce9174ef159ad938c4a34a89da"}, + {file = "aiohttp-3.8.3-cp37-cp37m-musllinux_1_1_ppc64le.whl", hash = "sha256:8135fa153a20d82ffb64f70a1b5c2738684afa197839b34cc3e3c72fa88d302c"}, + {file = "aiohttp-3.8.3-cp37-cp37m-musllinux_1_1_s390x.whl", hash = "sha256:ad61a9639792fd790523ba072c0555cd6be5a0baf03a49a5dd8cfcf20d56df48"}, + {file = "aiohttp-3.8.3-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:978b046ca728073070e9abc074b6299ebf3501e8dee5e26efacb13cec2b2dea0"}, + {file = "aiohttp-3.8.3-cp37-cp37m-win32.whl", hash = "sha256:0d2c6d8c6872df4a6ec37d2ede71eff62395b9e337b4e18efd2177de883a5033"}, + {file = "aiohttp-3.8.3-cp37-cp37m-win_amd64.whl", hash = "sha256:21d69797eb951f155026651f7e9362877334508d39c2fc37bd04ff55b2007091"}, + {file = "aiohttp-3.8.3-cp38-cp38-macosx_10_9_universal2.whl", hash = "sha256:2ca9af5f8f5812d475c5259393f52d712f6d5f0d7fdad9acdb1107dd9e3cb7eb"}, + {file = "aiohttp-3.8.3-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:1d90043c1882067f1bd26196d5d2db9aa6d268def3293ed5fb317e13c9413ea4"}, + {file = "aiohttp-3.8.3-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:d737fc67b9a970f3234754974531dc9afeea11c70791dcb7db53b0cf81b79784"}, + {file = "aiohttp-3.8.3-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:ebf909ea0a3fc9596e40d55d8000702a85e27fd578ff41a5500f68f20fd32e6c"}, + {file = "aiohttp-3.8.3-cp38-cp38-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:5835f258ca9f7c455493a57ee707b76d2d9634d84d5d7f62e77be984ea80b849"}, + {file = "aiohttp-3.8.3-cp38-cp38-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:da37dcfbf4b7f45d80ee386a5f81122501ec75672f475da34784196690762f4b"}, + {file = "aiohttp-3.8.3-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:87f44875f2804bc0511a69ce44a9595d5944837a62caecc8490bbdb0e18b1342"}, + {file = "aiohttp-3.8.3-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:527b3b87b24844ea7865284aabfab08eb0faf599b385b03c2aa91fc6edd6e4b6"}, + {file = "aiohttp-3.8.3-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:d5ba88df9aa5e2f806650fcbeedbe4f6e8736e92fc0e73b0400538fd25a4dd96"}, + {file = "aiohttp-3.8.3-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:e7b8813be97cab8cb52b1375f41f8e6804f6507fe4660152e8ca5c48f0436017"}, + {file = "aiohttp-3.8.3-cp38-cp38-musllinux_1_1_ppc64le.whl", hash = "sha256:2dea10edfa1a54098703cb7acaa665c07b4e7568472a47f4e64e6319d3821ccf"}, + {file = "aiohttp-3.8.3-cp38-cp38-musllinux_1_1_s390x.whl", hash = "sha256:713d22cd9643ba9025d33c4af43943c7a1eb8547729228de18d3e02e278472b6"}, + {file = "aiohttp-3.8.3-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:2d252771fc85e0cf8da0b823157962d70639e63cb9b578b1dec9868dd1f4f937"}, + {file = "aiohttp-3.8.3-cp38-cp38-win32.whl", hash = "sha256:66bd5f950344fb2b3dbdd421aaa4e84f4411a1a13fca3aeb2bcbe667f80c9f76"}, + {file = "aiohttp-3.8.3-cp38-cp38-win_amd64.whl", hash = "sha256:84b14f36e85295fe69c6b9789b51a0903b774046d5f7df538176516c3e422446"}, + {file = "aiohttp-3.8.3-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:16c121ba0b1ec2b44b73e3a8a171c4f999b33929cd2397124a8c7fcfc8cd9e06"}, + {file = "aiohttp-3.8.3-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:8d6aaa4e7155afaf994d7924eb290abbe81a6905b303d8cb61310a2aba1c68ba"}, + {file = "aiohttp-3.8.3-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:43046a319664a04b146f81b40e1545d4c8ac7b7dd04c47e40bf09f65f2437346"}, + {file = "aiohttp-3.8.3-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:599418aaaf88a6d02a8c515e656f6faf3d10618d3dd95866eb4436520096c84b"}, + {file = "aiohttp-3.8.3-cp39-cp39-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:92a2964319d359f494f16011e23434f6f8ef0434acd3cf154a6b7bec511e2fb7"}, + {file = "aiohttp-3.8.3-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:73a4131962e6d91109bca6536416aa067cf6c4efb871975df734f8d2fd821b37"}, + {file = "aiohttp-3.8.3-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:598adde339d2cf7d67beaccda3f2ce7c57b3b412702f29c946708f69cf8222aa"}, + {file = "aiohttp-3.8.3-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:75880ed07be39beff1881d81e4a907cafb802f306efd6d2d15f2b3c69935f6fb"}, + {file = "aiohttp-3.8.3-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:a0239da9fbafd9ff82fd67c16704a7d1bccf0d107a300e790587ad05547681c8"}, + {file = "aiohttp-3.8.3-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:4e3a23ec214e95c9fe85a58470b660efe6534b83e6cbe38b3ed52b053d7cb6ad"}, + {file = "aiohttp-3.8.3-cp39-cp39-musllinux_1_1_ppc64le.whl", hash = "sha256:47841407cc89a4b80b0c52276f3cc8138bbbfba4b179ee3acbd7d77ae33f7ac4"}, + {file = "aiohttp-3.8.3-cp39-cp39-musllinux_1_1_s390x.whl", hash = "sha256:54d107c89a3ebcd13228278d68f1436d3f33f2dd2af5415e3feaeb1156e1a62c"}, + {file = "aiohttp-3.8.3-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:c37c5cce780349d4d51739ae682dec63573847a2a8dcb44381b174c3d9c8d403"}, + {file = "aiohttp-3.8.3-cp39-cp39-win32.whl", hash = "sha256:f178d2aadf0166be4df834c4953da2d7eef24719e8aec9a65289483eeea9d618"}, + {file = "aiohttp-3.8.3-cp39-cp39-win_amd64.whl", hash = "sha256:88e5be56c231981428f4f506c68b6a46fa25c4123a2e86d156c58a8369d31ab7"}, + {file = "aiohttp-3.8.3.tar.gz", hash = "sha256:3828fb41b7203176b82fe5d699e0d845435f2374750a44b480ea6b930f6be269"}, +] +aioitertools = [ + {file = "aioitertools-0.11.0-py3-none-any.whl", hash = "sha256:04b95e3dab25b449def24d7df809411c10e62aab0cbe31a50ca4e68748c43394"}, + {file = "aioitertools-0.11.0.tar.gz", hash = "sha256:42c68b8dd3a69c2bf7f2233bf7df4bb58b557bca5252ac02ed5187bbc67d6831"}, +] +aiosignal = [ + {file = "aiosignal-1.3.1-py3-none-any.whl", hash = "sha256:f8376fb07dd1e86a584e4fcdec80b36b7f81aac666ebc724e2c090300dd83b17"}, + {file = "aiosignal-1.3.1.tar.gz", hash = "sha256:54cd96e15e1649b75d6c87526a6ff0b6c1b0dd3459f43d9ca11d48c339b68cfc"}, +] +async-timeout = [ + {file = "async-timeout-4.0.2.tar.gz", hash = "sha256:2163e1640ddb52b7a8c80d0a67a08587e5d245cc9c553a74a847056bc2976b15"}, + {file = "async_timeout-4.0.2-py3-none-any.whl", hash = "sha256:8ca1e4fcf50d07413d66d1a5e416e42cfdf5851c981d679a09851a6853383b3c"}, +] +attrs = [ + {file = "attrs-22.1.0-py2.py3-none-any.whl", hash = "sha256:86efa402f67bf2df34f51a335487cf46b1ec130d02b8d39fd248abfd30da551c"}, + {file = "attrs-22.1.0.tar.gz", hash = "sha256:29adc2665447e5191d0e7c568fde78b21f9672d344281d0c6e1ab085429b22b6"}, +] +azure-core = [ + {file = "azure-core-1.26.1.zip", hash = "sha256:223b0e90cbdd1f03c41b195b03239899843f20d00964dbb85e64386873414a2d"}, + {file = "azure_core-1.26.1-py3-none-any.whl", hash = "sha256:726ffd1ded04a2c1cb53f9d9155cbb05ac5c1c2a29af4ef622e93e1c0a8bc92b"}, +] +azure-datalake-store = [ + {file = "azure-datalake-store-0.0.52.tar.gz", hash = "sha256:4198ddb32614d16d4502b43d5c9739f81432b7e0e4d75d30e05149fe6007fea2"}, + {file = "azure_datalake_store-0.0.52-py2.py3-none-any.whl", hash = "sha256:aaed72b9c856824aeab554f4dbe0ef2c6d0ff36700bdd8b93d8298793117c48e"}, +] +azure-identity = [ + {file = "azure-identity-1.12.0.zip", hash = "sha256:7f9b1ae7d97ea7af3f38dd09305e19ab81a1e16ab66ea186b6579d85c1ca2347"}, + {file = "azure_identity-1.12.0-py3-none-any.whl", hash = "sha256:2a58ce4a209a013e37eaccfd5937570ab99e9118b3e1acf875eed3a85d541b92"}, +] +azure-storage-blob = [ + {file = "azure-storage-blob-12.14.1.zip", hash = "sha256:860d4d82985a4bfc7d3271e71275af330f54f330a754355435a7ba749ccde997"}, + {file = "azure_storage_blob-12.14.1-py3-none-any.whl", hash = "sha256:52b84658e8df7853a3cf1c563814655b5028b979b2a87905b92aa6bb30be240e"}, +] +boto3 = [ + {file = "boto3-1.24.59-py3-none-any.whl", hash = "sha256:34ab44146a2c4e7f4e72737f4b27e6eb5e0a7855c2f4599e3d9199b6a0a2d575"}, + {file = "boto3-1.24.59.tar.gz", hash = "sha256:a50b4323f9579cfe22fcf5531fbd40b567d4d74c1adce06aeb5c95fce2a6fb40"}, +] +botocore = [ + {file = "botocore-1.27.59-py3-none-any.whl", hash = "sha256:69d756791fc024bda54f6c53f71ae34e695ee41bbbc1743d9179c4837a4929da"}, + {file = "botocore-1.27.59.tar.gz", hash = "sha256:eda4aed6ee719a745d1288eaf1beb12f6f6448ad1fa12f159405db14ba9c92cf"}, +] +build = [ + {file = "build-0.9.0-py3-none-any.whl", hash = "sha256:38a7a2b7a0bdc61a42a0a67509d88c71ecfc37b393baba770fae34e20929ff69"}, + {file = "build-0.9.0.tar.gz", hash = "sha256:1a07724e891cbd898923145eb7752ee7653674c511378eb9c7691aab1612bc3c"}, +] +certifi = [ + {file = "certifi-2022.12.7-py3-none-any.whl", hash = "sha256:4ad3232f5e926d6718ec31cfc1fcadfde020920e278684144551c91769c7bc18"}, + {file = "certifi-2022.12.7.tar.gz", hash = "sha256:35824b4c3a97115964b408844d64aa14db1cc518f6562e8d7261699d1350a9e3"}, +] +cffi = [ + {file = "cffi-1.15.1-cp27-cp27m-macosx_10_9_x86_64.whl", hash = "sha256:a66d3508133af6e8548451b25058d5812812ec3798c886bf38ed24a98216fab2"}, + {file = "cffi-1.15.1-cp27-cp27m-manylinux1_i686.whl", hash = "sha256:470c103ae716238bbe698d67ad020e1db9d9dba34fa5a899b5e21577e6d52ed2"}, + {file = "cffi-1.15.1-cp27-cp27m-manylinux1_x86_64.whl", hash = "sha256:9ad5db27f9cabae298d151c85cf2bad1d359a1b9c686a275df03385758e2f914"}, + {file = "cffi-1.15.1-cp27-cp27m-win32.whl", hash = "sha256:b3bbeb01c2b273cca1e1e0c5df57f12dce9a4dd331b4fa1635b8bec26350bde3"}, + {file = "cffi-1.15.1-cp27-cp27m-win_amd64.whl", hash = "sha256:e00b098126fd45523dd056d2efba6c5a63b71ffe9f2bbe1a4fe1716e1d0c331e"}, + {file = "cffi-1.15.1-cp27-cp27mu-manylinux1_i686.whl", hash = "sha256:d61f4695e6c866a23a21acab0509af1cdfd2c013cf256bbf5b6b5e2695827162"}, + {file = "cffi-1.15.1-cp27-cp27mu-manylinux1_x86_64.whl", hash = "sha256:ed9cb427ba5504c1dc15ede7d516b84757c3e3d7868ccc85121d9310d27eed0b"}, + {file = "cffi-1.15.1-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:39d39875251ca8f612b6f33e6b1195af86d1b3e60086068be9cc053aa4376e21"}, + {file = "cffi-1.15.1-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:285d29981935eb726a4399badae8f0ffdff4f5050eaa6d0cfc3f64b857b77185"}, + {file = "cffi-1.15.1-cp310-cp310-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:3eb6971dcff08619f8d91607cfc726518b6fa2a9eba42856be181c6d0d9515fd"}, + {file = "cffi-1.15.1-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:21157295583fe8943475029ed5abdcf71eb3911894724e360acff1d61c1d54bc"}, + {file = "cffi-1.15.1-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:5635bd9cb9731e6d4a1132a498dd34f764034a8ce60cef4f5319c0541159392f"}, + {file = "cffi-1.15.1-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:2012c72d854c2d03e45d06ae57f40d78e5770d252f195b93f581acf3ba44496e"}, + {file = "cffi-1.15.1-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:dd86c085fae2efd48ac91dd7ccffcfc0571387fe1193d33b6394db7ef31fe2a4"}, + {file = "cffi-1.15.1-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:fa6693661a4c91757f4412306191b6dc88c1703f780c8234035eac011922bc01"}, + {file = "cffi-1.15.1-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:59c0b02d0a6c384d453fece7566d1c7e6b7bae4fc5874ef2ef46d56776d61c9e"}, + {file = "cffi-1.15.1-cp310-cp310-win32.whl", hash = "sha256:cba9d6b9a7d64d4bd46167096fc9d2f835e25d7e4c121fb2ddfc6528fb0413b2"}, + {file = "cffi-1.15.1-cp310-cp310-win_amd64.whl", hash = "sha256:ce4bcc037df4fc5e3d184794f27bdaab018943698f4ca31630bc7f84a7b69c6d"}, + {file = "cffi-1.15.1-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:3d08afd128ddaa624a48cf2b859afef385b720bb4b43df214f85616922e6a5ac"}, + {file = "cffi-1.15.1-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:3799aecf2e17cf585d977b780ce79ff0dc9b78d799fc694221ce814c2c19db83"}, + {file = "cffi-1.15.1-cp311-cp311-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:a591fe9e525846e4d154205572a029f653ada1a78b93697f3b5a8f1f2bc055b9"}, + {file = "cffi-1.15.1-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:3548db281cd7d2561c9ad9984681c95f7b0e38881201e157833a2342c30d5e8c"}, + {file = "cffi-1.15.1-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:91fc98adde3d7881af9b59ed0294046f3806221863722ba7d8d120c575314325"}, + {file = "cffi-1.15.1-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:94411f22c3985acaec6f83c6df553f2dbe17b698cc7f8ae751ff2237d96b9e3c"}, + {file = "cffi-1.15.1-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:03425bdae262c76aad70202debd780501fabeaca237cdfddc008987c0e0f59ef"}, + {file = "cffi-1.15.1-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:cc4d65aeeaa04136a12677d3dd0b1c0c94dc43abac5860ab33cceb42b801c1e8"}, + {file = "cffi-1.15.1-cp311-cp311-win32.whl", hash = "sha256:a0f100c8912c114ff53e1202d0078b425bee3649ae34d7b070e9697f93c5d52d"}, + {file = "cffi-1.15.1-cp311-cp311-win_amd64.whl", hash = "sha256:04ed324bda3cda42b9b695d51bb7d54b680b9719cfab04227cdd1e04e5de3104"}, + {file = "cffi-1.15.1-cp36-cp36m-macosx_10_9_x86_64.whl", hash = "sha256:50a74364d85fd319352182ef59c5c790484a336f6db772c1a9231f1c3ed0cbd7"}, + {file = "cffi-1.15.1-cp36-cp36m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:e263d77ee3dd201c3a142934a086a4450861778baaeeb45db4591ef65550b0a6"}, + {file = "cffi-1.15.1-cp36-cp36m-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:cec7d9412a9102bdc577382c3929b337320c4c4c4849f2c5cdd14d7368c5562d"}, + {file = "cffi-1.15.1-cp36-cp36m-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:4289fc34b2f5316fbb762d75362931e351941fa95fa18789191b33fc4cf9504a"}, + {file = "cffi-1.15.1-cp36-cp36m-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:173379135477dc8cac4bc58f45db08ab45d228b3363adb7af79436135d028405"}, + {file = "cffi-1.15.1-cp36-cp36m-manylinux_2_5_x86_64.manylinux1_x86_64.whl", hash = "sha256:6975a3fac6bc83c4a65c9f9fcab9e47019a11d3d2cf7f3c0d03431bf145a941e"}, + {file = "cffi-1.15.1-cp36-cp36m-win32.whl", hash = "sha256:2470043b93ff09bf8fb1d46d1cb756ce6132c54826661a32d4e4d132e1977adf"}, + {file = "cffi-1.15.1-cp36-cp36m-win_amd64.whl", hash = "sha256:30d78fbc8ebf9c92c9b7823ee18eb92f2e6ef79b45ac84db507f52fbe3ec4497"}, + {file = "cffi-1.15.1-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:198caafb44239b60e252492445da556afafc7d1e3ab7a1fb3f0584ef6d742375"}, + {file = "cffi-1.15.1-cp37-cp37m-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:5ef34d190326c3b1f822a5b7a45f6c4535e2f47ed06fec77d3d799c450b2651e"}, + {file = "cffi-1.15.1-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:8102eaf27e1e448db915d08afa8b41d6c7ca7a04b7d73af6514df10a3e74bd82"}, + {file = "cffi-1.15.1-cp37-cp37m-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:5df2768244d19ab7f60546d0c7c63ce1581f7af8b5de3eb3004b9b6fc8a9f84b"}, + {file = "cffi-1.15.1-cp37-cp37m-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:a8c4917bd7ad33e8eb21e9a5bbba979b49d9a97acb3a803092cbc1133e20343c"}, + {file = "cffi-1.15.1-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:0e2642fe3142e4cc4af0799748233ad6da94c62a8bec3a6648bf8ee68b1c7426"}, + {file = "cffi-1.15.1-cp37-cp37m-win32.whl", hash = "sha256:e229a521186c75c8ad9490854fd8bbdd9a0c9aa3a524326b55be83b54d4e0ad9"}, + {file = "cffi-1.15.1-cp37-cp37m-win_amd64.whl", hash = "sha256:a0b71b1b8fbf2b96e41c4d990244165e2c9be83d54962a9a1d118fd8657d2045"}, + {file = "cffi-1.15.1-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:320dab6e7cb2eacdf0e658569d2575c4dad258c0fcc794f46215e1e39f90f2c3"}, + {file = "cffi-1.15.1-cp38-cp38-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:1e74c6b51a9ed6589199c787bf5f9875612ca4a8a0785fb2d4a84429badaf22a"}, + {file = "cffi-1.15.1-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a5c84c68147988265e60416b57fc83425a78058853509c1b0629c180094904a5"}, + {file = "cffi-1.15.1-cp38-cp38-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:3b926aa83d1edb5aa5b427b4053dc420ec295a08e40911296b9eb1b6170f6cca"}, + {file = "cffi-1.15.1-cp38-cp38-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:87c450779d0914f2861b8526e035c5e6da0a3199d8f1add1a665e1cbc6fc6d02"}, + {file = "cffi-1.15.1-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:4f2c9f67e9821cad2e5f480bc8d83b8742896f1242dba247911072d4fa94c192"}, + {file = "cffi-1.15.1-cp38-cp38-win32.whl", hash = "sha256:8b7ee99e510d7b66cdb6c593f21c043c248537a32e0bedf02e01e9553a172314"}, + {file = "cffi-1.15.1-cp38-cp38-win_amd64.whl", hash = "sha256:00a9ed42e88df81ffae7a8ab6d9356b371399b91dbdf0c3cb1e84c03a13aceb5"}, + {file = "cffi-1.15.1-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:54a2db7b78338edd780e7ef7f9f6c442500fb0d41a5a4ea24fff1c929d5af585"}, + {file = "cffi-1.15.1-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:fcd131dd944808b5bdb38e6f5b53013c5aa4f334c5cad0c72742f6eba4b73db0"}, + {file = "cffi-1.15.1-cp39-cp39-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:7473e861101c9e72452f9bf8acb984947aa1661a7704553a9f6e4baa5ba64415"}, + {file = "cffi-1.15.1-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:6c9a799e985904922a4d207a94eae35c78ebae90e128f0c4e521ce339396be9d"}, + {file = "cffi-1.15.1-cp39-cp39-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:3bcde07039e586f91b45c88f8583ea7cf7a0770df3a1649627bf598332cb6984"}, + {file = "cffi-1.15.1-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:33ab79603146aace82c2427da5ca6e58f2b3f2fb5da893ceac0c42218a40be35"}, + {file = "cffi-1.15.1-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:5d598b938678ebf3c67377cdd45e09d431369c3b1a5b331058c338e201f12b27"}, + {file = "cffi-1.15.1-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:db0fbb9c62743ce59a9ff687eb5f4afbe77e5e8403d6697f7446e5f609976f76"}, + {file = "cffi-1.15.1-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:98d85c6a2bef81588d9227dde12db8a7f47f639f4a17c9ae08e773aa9c697bf3"}, + {file = "cffi-1.15.1-cp39-cp39-win32.whl", hash = "sha256:40f4774f5a9d4f5e344f31a32b5096977b5d48560c5592e2f3d2c4374bd543ee"}, + {file = "cffi-1.15.1-cp39-cp39-win_amd64.whl", hash = "sha256:70df4e3b545a17496c9b3f41f5115e69a4f2e77e94e1d2a8e1070bc0c38c8a3c"}, + {file = "cffi-1.15.1.tar.gz", hash = "sha256:d400bfb9a37b1351253cb402671cea7e89bdecc294e8016a707f6d1d8ac934f9"}, +] +cfgv = [ + {file = "cfgv-3.3.1-py2.py3-none-any.whl", hash = "sha256:c6a0883f3917a037485059700b9e75da2464e6c27051014ad85ba6aaa5884426"}, + {file = "cfgv-3.3.1.tar.gz", hash = "sha256:f5a830efb9ce7a445376bb66ec94c638a9787422f96264c98edc6bdeed8ab736"}, +] +charset-normalizer = [ + {file = "charset-normalizer-2.1.1.tar.gz", hash = "sha256:5a3d016c7c547f69d6f81fb0db9449ce888b418b5b9952cc5e6e66843e9dd845"}, + {file = "charset_normalizer-2.1.1-py3-none-any.whl", hash = "sha256:83e9a75d1911279afd89352c68b45348559d1fc0506b054b346651b5e7fee29f"}, +] +click = [ + {file = "click-8.1.3-py3-none-any.whl", hash = "sha256:bb4d8133cb15a609f44e8213d9b391b0809795062913b383c62be0ee95b1db48"}, + {file = "click-8.1.3.tar.gz", hash = "sha256:7682dc8afb30297001674575ea00d1814d808d6a36af415a82bd481d37ba7b8e"}, +] +colorama = [ + {file = "colorama-0.4.6-py2.py3-none-any.whl", hash = "sha256:4f1d9991f5acc0ca119f9d443620b77f9d6b33703e51011c16baf57afb285fc6"}, + {file = "colorama-0.4.6.tar.gz", hash = "sha256:08695f5cb7ed6e0531a20572697297273c47b8cae5a63ffc6d6ed5c201be6e44"}, +] +commonmark = [ + {file = "commonmark-0.9.1-py2.py3-none-any.whl", hash = "sha256:da2f38c92590f83de410ba1a3cbceafbc74fee9def35f9251ba9a971d6d66fd9"}, + {file = "commonmark-0.9.1.tar.gz", hash = "sha256:452f9dc859be7f06631ddcb328b6919c67984aca654e5fefb3914d54691aed60"}, +] +coverage = [ + {file = "coverage-7.0.1-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:b3695c4f4750bca943b3e1f74ad4be8d29e4aeab927d50772c41359107bd5d5c"}, + {file = "coverage-7.0.1-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:fa6a5a224b7f4cfb226f4fc55a57e8537fcc096f42219128c2c74c0e7d0953e1"}, + {file = "coverage-7.0.1-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:74f70cd92669394eaf8d7756d1b195c8032cf7bbbdfce3bc489d4e15b3b8cf73"}, + {file = "coverage-7.0.1-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:b66bb21a23680dee0be66557dc6b02a3152ddb55edf9f6723fa4a93368f7158d"}, + {file = "coverage-7.0.1-cp310-cp310-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:d87717959d4d0ee9db08a0f1d80d21eb585aafe30f9b0a54ecf779a69cb015f6"}, + {file = "coverage-7.0.1-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:854f22fa361d1ff914c7efa347398374cc7d567bdafa48ac3aa22334650dfba2"}, + {file = "coverage-7.0.1-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:1e414dc32ee5c3f36544ea466b6f52f28a7af788653744b8570d0bf12ff34bc0"}, + {file = "coverage-7.0.1-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:6c5ad996c6fa4d8ed669cfa1e8551348729d008a2caf81489ab9ea67cfbc7498"}, + {file = "coverage-7.0.1-cp310-cp310-win32.whl", hash = "sha256:691571f31ace1837838b7e421d3a09a8c00b4aac32efacb4fc9bd0a5c647d25a"}, + {file = "coverage-7.0.1-cp310-cp310-win_amd64.whl", hash = "sha256:89caf4425fe88889e2973a8e9a3f6f5f9bbe5dd411d7d521e86428c08a873a4a"}, + {file = "coverage-7.0.1-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:63d56165a7c76265468d7e0c5548215a5ba515fc2cba5232d17df97bffa10f6c"}, + {file = "coverage-7.0.1-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:4f943a3b2bc520102dd3e0bb465e1286e12c9a54f58accd71b9e65324d9c7c01"}, + {file = "coverage-7.0.1-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:830525361249dc4cd013652b0efad645a385707a5ae49350c894b67d23fbb07c"}, + {file = "coverage-7.0.1-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:fd1b9c5adc066db699ccf7fa839189a649afcdd9e02cb5dc9d24e67e7922737d"}, + {file = "coverage-7.0.1-cp311-cp311-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:e00c14720b8b3b6c23b487e70bd406abafc976ddc50490f645166f111c419c39"}, + {file = "coverage-7.0.1-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:6d55d840e1b8c0002fce66443e124e8581f30f9ead2e54fbf6709fb593181f2c"}, + {file = "coverage-7.0.1-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:66b18c3cf8bbab0cce0d7b9e4262dc830e93588986865a8c78ab2ae324b3ed56"}, + {file = "coverage-7.0.1-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:12a5aa77783d49e05439fbe6e6b427484f8a0f9f456b46a51d8aac022cfd024d"}, + {file = "coverage-7.0.1-cp311-cp311-win32.whl", hash = "sha256:b77015d1cb8fe941be1222a5a8b4e3fbca88180cfa7e2d4a4e58aeabadef0ab7"}, + {file = "coverage-7.0.1-cp311-cp311-win_amd64.whl", hash = "sha256:fb992c47cb1e5bd6a01e97182400bcc2ba2077080a17fcd7be23aaa6e572e390"}, + {file = "coverage-7.0.1-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:e78e9dcbf4f3853d3ae18a8f9272111242531535ec9e1009fa8ec4a2b74557dc"}, + {file = "coverage-7.0.1-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:e60bef2e2416f15fdc05772bf87db06c6a6f9870d1db08fdd019fbec98ae24a9"}, + {file = "coverage-7.0.1-cp37-cp37m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:9823e4789ab70f3ec88724bba1a203f2856331986cd893dedbe3e23a6cfc1e4e"}, + {file = "coverage-7.0.1-cp37-cp37m-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:9158f8fb06747ac17bd237930c4372336edc85b6e13bdc778e60f9d685c3ca37"}, + {file = "coverage-7.0.1-cp37-cp37m-musllinux_1_1_aarch64.whl", hash = "sha256:486ee81fa694b4b796fc5617e376326a088f7b9729c74d9defa211813f3861e4"}, + {file = "coverage-7.0.1-cp37-cp37m-musllinux_1_1_i686.whl", hash = "sha256:1285648428a6101b5f41a18991c84f1c3959cee359e51b8375c5882fc364a13f"}, + {file = "coverage-7.0.1-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:2c44fcfb3781b41409d0f060a4ed748537557de9362a8a9282182fafb7a76ab4"}, + {file = "coverage-7.0.1-cp37-cp37m-win32.whl", hash = "sha256:d6814854c02cbcd9c873c0f3286a02e3ac1250625cca822ca6bc1018c5b19f1c"}, + {file = "coverage-7.0.1-cp37-cp37m-win_amd64.whl", hash = "sha256:f66460f17c9319ea4f91c165d46840314f0a7c004720b20be58594d162a441d8"}, + {file = "coverage-7.0.1-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:9b373c9345c584bb4b5f5b8840df7f4ab48c4cbb7934b58d52c57020d911b856"}, + {file = "coverage-7.0.1-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:d3022c3007d3267a880b5adcf18c2a9bf1fc64469b394a804886b401959b8742"}, + {file = "coverage-7.0.1-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:92651580bd46519067e36493acb394ea0607b55b45bd81dd4e26379ed1871f55"}, + {file = "coverage-7.0.1-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:3cfc595d2af13856505631be072835c59f1acf30028d1c860b435c5fc9c15b69"}, + {file = "coverage-7.0.1-cp38-cp38-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:0b4b3a4d9915b2be879aff6299c0a6129f3d08a775d5a061f503cf79571f73e4"}, + {file = "coverage-7.0.1-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:b6f22bb64cc39bcb883e5910f99a27b200fdc14cdd79df8696fa96b0005c9444"}, + {file = "coverage-7.0.1-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:72d1507f152abacea81f65fee38e4ef3ac3c02ff8bc16f21d935fd3a8a4ad910"}, + {file = "coverage-7.0.1-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:0a79137fc99815fff6a852c233628e735ec15903cfd16da0f229d9c4d45926ab"}, + {file = "coverage-7.0.1-cp38-cp38-win32.whl", hash = "sha256:b3763e7fcade2ff6c8e62340af9277f54336920489ceb6a8cd6cc96da52fcc62"}, + {file = "coverage-7.0.1-cp38-cp38-win_amd64.whl", hash = "sha256:09f6b5a8415b6b3e136d5fec62b552972187265cb705097bf030eb9d4ffb9b60"}, + {file = "coverage-7.0.1-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:978258fec36c154b5e250d356c59af7d4c3ba02bef4b99cda90b6029441d797d"}, + {file = "coverage-7.0.1-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:19ec666533f0f70a0993f88b8273057b96c07b9d26457b41863ccd021a043b9a"}, + {file = "coverage-7.0.1-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:cfded268092a84605f1cc19e5c737f9ce630a8900a3589e9289622db161967e9"}, + {file = "coverage-7.0.1-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:07bcfb1d8ac94af886b54e18a88b393f6a73d5959bb31e46644a02453c36e475"}, + {file = "coverage-7.0.1-cp39-cp39-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:397b4a923cc7566bbc7ae2dfd0ba5a039b61d19c740f1373791f2ebd11caea59"}, + {file = "coverage-7.0.1-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:aec2d1515d9d39ff270059fd3afbb3b44e6ec5758af73caf18991807138c7118"}, + {file = "coverage-7.0.1-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:c20cfebcc149a4c212f6491a5f9ff56f41829cd4f607b5be71bb2d530ef243b1"}, + {file = "coverage-7.0.1-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:fd556ff16a57a070ce4f31c635953cc44e25244f91a0378c6e9bdfd40fdb249f"}, + {file = "coverage-7.0.1-cp39-cp39-win32.whl", hash = "sha256:b9ea158775c7c2d3e54530a92da79496fb3fb577c876eec761c23e028f1e216c"}, + {file = "coverage-7.0.1-cp39-cp39-win_amd64.whl", hash = "sha256:d1991f1dd95eba69d2cd7708ff6c2bbd2426160ffc73c2b81f617a053ebcb1a8"}, + {file = "coverage-7.0.1-pp37.pp38.pp39-none-any.whl", hash = "sha256:3dd4ee135e08037f458425b8842d24a95a0961831a33f89685ff86b77d378f89"}, + {file = "coverage-7.0.1.tar.gz", hash = "sha256:a4a574a19eeb67575a5328a5760bbbb737faa685616586a9f9da4281f940109c"}, +] +cryptography = [ + {file = "cryptography-38.0.4-cp36-abi3-macosx_10_10_universal2.whl", hash = "sha256:2fa36a7b2cc0998a3a4d5af26ccb6273f3df133d61da2ba13b3286261e7efb70"}, + {file = "cryptography-38.0.4-cp36-abi3-macosx_10_10_x86_64.whl", hash = "sha256:1f13ddda26a04c06eb57119caf27a524ccae20533729f4b1e4a69b54e07035eb"}, + {file = "cryptography-38.0.4-cp36-abi3-manylinux_2_17_aarch64.manylinux2014_aarch64.manylinux_2_24_aarch64.whl", hash = "sha256:2ec2a8714dd005949d4019195d72abed84198d877112abb5a27740e217e0ea8d"}, + {file = "cryptography-38.0.4-cp36-abi3-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:50a1494ed0c3f5b4d07650a68cd6ca62efe8b596ce743a5c94403e6f11bf06c1"}, + {file = "cryptography-38.0.4-cp36-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:a10498349d4c8eab7357a8f9aa3463791292845b79597ad1b98a543686fb1ec8"}, + {file = "cryptography-38.0.4-cp36-abi3-manylinux_2_24_x86_64.whl", hash = "sha256:10652dd7282de17990b88679cb82f832752c4e8237f0c714be518044269415db"}, + {file = "cryptography-38.0.4-cp36-abi3-manylinux_2_28_aarch64.whl", hash = "sha256:bfe6472507986613dc6cc00b3d492b2f7564b02b3b3682d25ca7f40fa3fd321b"}, + {file = "cryptography-38.0.4-cp36-abi3-manylinux_2_28_x86_64.whl", hash = "sha256:ce127dd0a6a0811c251a6cddd014d292728484e530d80e872ad9806cfb1c5b3c"}, + {file = "cryptography-38.0.4-cp36-abi3-musllinux_1_1_aarch64.whl", hash = "sha256:53049f3379ef05182864d13bb9686657659407148f901f3f1eee57a733fb4b00"}, + {file = "cryptography-38.0.4-cp36-abi3-musllinux_1_1_x86_64.whl", hash = "sha256:8a4b2bdb68a447fadebfd7d24855758fe2d6fecc7fed0b78d190b1af39a8e3b0"}, + {file = "cryptography-38.0.4-cp36-abi3-win32.whl", hash = "sha256:1d7e632804a248103b60b16fb145e8df0bc60eed790ece0d12efe8cd3f3e7744"}, + {file = "cryptography-38.0.4-cp36-abi3-win_amd64.whl", hash = "sha256:8e45653fb97eb2f20b8c96f9cd2b3a0654d742b47d638cf2897afbd97f80fa6d"}, + {file = "cryptography-38.0.4-pp37-pypy37_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:ca57eb3ddaccd1112c18fc80abe41db443cc2e9dcb1917078e02dfa010a4f353"}, + {file = "cryptography-38.0.4-pp37-pypy37_pp73-manylinux_2_24_x86_64.whl", hash = "sha256:c9e0d79ee4c56d841bd4ac6e7697c8ff3c8d6da67379057f29e66acffcd1e9a7"}, + {file = "cryptography-38.0.4-pp37-pypy37_pp73-manylinux_2_28_x86_64.whl", hash = "sha256:0e70da4bdff7601b0ef48e6348339e490ebfb0cbe638e083c9c41fb49f00c8bd"}, + {file = "cryptography-38.0.4-pp38-pypy38_pp73-macosx_10_10_x86_64.whl", hash = "sha256:998cd19189d8a747b226d24c0207fdaa1e6658a1d3f2494541cb9dfbf7dcb6d2"}, + {file = "cryptography-38.0.4-pp38-pypy38_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:67461b5ebca2e4c2ab991733f8ab637a7265bb582f07c7c88914b5afb88cb95b"}, + {file = "cryptography-38.0.4-pp38-pypy38_pp73-manylinux_2_24_x86_64.whl", hash = "sha256:4eb85075437f0b1fd8cd66c688469a0c4119e0ba855e3fef86691971b887caf6"}, + {file = "cryptography-38.0.4-pp38-pypy38_pp73-manylinux_2_28_x86_64.whl", hash = "sha256:3178d46f363d4549b9a76264f41c6948752183b3f587666aff0555ac50fd7876"}, + {file = "cryptography-38.0.4-pp38-pypy38_pp73-win_amd64.whl", hash = "sha256:6391e59ebe7c62d9902c24a4d8bcbc79a68e7c4ab65863536127c8a9cd94043b"}, + {file = "cryptography-38.0.4-pp39-pypy39_pp73-macosx_10_10_x86_64.whl", hash = "sha256:78e47e28ddc4ace41dd38c42e6feecfdadf9c3be2af389abbfeef1ff06822285"}, + {file = "cryptography-38.0.4-pp39-pypy39_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:2fb481682873035600b5502f0015b664abc26466153fab5c6bc92c1ea69d478b"}, + {file = "cryptography-38.0.4-pp39-pypy39_pp73-manylinux_2_24_x86_64.whl", hash = "sha256:4367da5705922cf7070462e964f66e4ac24162e22ab0a2e9d31f1b270dd78083"}, + {file = "cryptography-38.0.4-pp39-pypy39_pp73-manylinux_2_28_x86_64.whl", hash = "sha256:b4cad0cea995af760f82820ab4ca54e5471fc782f70a007f31531957f43e9dee"}, + {file = "cryptography-38.0.4-pp39-pypy39_pp73-win_amd64.whl", hash = "sha256:80ca53981ceeb3241998443c4964a387771588c4e4a5d92735a493af868294f9"}, + {file = "cryptography-38.0.4.tar.gz", hash = "sha256:175c1a818b87c9ac80bb7377f5520b7f31b3ef2a0004e2420319beadedb67290"}, +] +distlib = [ + {file = "distlib-0.3.6-py2.py3-none-any.whl", hash = "sha256:f35c4b692542ca110de7ef0bea44d73981caeb34ca0b9b6b2e6d7790dda8f80e"}, + {file = "distlib-0.3.6.tar.gz", hash = "sha256:14bad2d9b04d3a36127ac97f30b12a19268f211063d8f8ee4f47108896e11b46"}, +] +docutils = [ + {file = "docutils-0.19-py3-none-any.whl", hash = "sha256:5e1de4d849fee02c63b040a4a3fd567f4ab104defd8a5511fbbc24a8a017efbc"}, + {file = "docutils-0.19.tar.gz", hash = "sha256:33995a6753c30b7f577febfc2c50411fec6aac7f7ffeb7c4cfe5991072dcf9e6"}, +] +duckdb = [ + {file = "duckdb-0.6.1-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:e566514f9327f89264e98ac14ee7a84fbd9857328028258422c3e8375ee19d25"}, + {file = "duckdb-0.6.1-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:b31c2883de5b19591a2852165e6b3f9821f77af649835f27bc146b26e4aa30cb"}, + {file = "duckdb-0.6.1-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:998165b2fb1f1d2b0ad742096015ea70878f7d40304643c7424c3ed3ddf07bfc"}, + {file = "duckdb-0.6.1-cp310-cp310-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:3941b3a1e8a1cdb7b90ab3917b87af816e71f9692e5ada7f19b6b60969f731e5"}, + {file = "duckdb-0.6.1-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:143611bd1b7c13343f087d4d423a7a8a4f33a114c5326171e867febf3f0fcfe1"}, + {file = "duckdb-0.6.1-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:125ba45e8b08f28858f918ec9cbd3a19975e5d8d9e8275ef4ad924028a616e14"}, + {file = "duckdb-0.6.1-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:e609a65b31c92f2f7166831f74b56f5ed54b33d8c2c4b4c3974c26fdc50464c5"}, + {file = "duckdb-0.6.1-cp310-cp310-win32.whl", hash = "sha256:b39045074fb9a3f068496475a5d627ad4fa572fa3b4980e3b479c11d0b706f2d"}, + {file = "duckdb-0.6.1-cp310-cp310-win_amd64.whl", hash = "sha256:16fa96ffaa3d842a9355a633fb8bc092d119be08d4bc02013946d8594417bc14"}, + {file = "duckdb-0.6.1-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:b4bbe2f6c1b109c626f9318eee80934ad2a5b81a51409c6b5083c6c5f9bdb125"}, + {file = "duckdb-0.6.1-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:cfea36b58928ce778d17280d4fb3bf0a2d7cff407667baedd69c5b41463ac0fd"}, + {file = "duckdb-0.6.1-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:0b64eb53d0d0695814bf1b65c0f91ab7ed66b515f89c88038f65ad5e0762571c"}, + {file = "duckdb-0.6.1-cp311-cp311-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:35b01bc724e1933293f4c34f410d2833bfbb56d5743b515d805bbfed0651476e"}, + {file = "duckdb-0.6.1-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:fec2c2466654ce786843bda2bfba71e0e4719106b41d36b17ceb1901e130aa71"}, + {file = "duckdb-0.6.1-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:82cd30f5cf368658ef879b1c60276bc8650cf67cfe3dc3e3009438ba39251333"}, + {file = "duckdb-0.6.1-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:a782bbfb7f5e97d4a9c834c9e78f023fb8b3f6687c22ca99841e6ed944b724da"}, + {file = "duckdb-0.6.1-cp311-cp311-win32.whl", hash = "sha256:e3702d4a9ade54c6403f6615a98bbec2020a76a60f5db7fcf085df1bd270e66e"}, + {file = "duckdb-0.6.1-cp311-cp311-win_amd64.whl", hash = "sha256:93b074f473d68c944b0eeb2edcafd91ad11da8432b484836efaaab4e26351d48"}, + {file = "duckdb-0.6.1-cp36-cp36m-macosx_10_9_x86_64.whl", hash = "sha256:adae183924d6d479202c39072e37d440b511326e84525bcb7432bca85f86caba"}, + {file = "duckdb-0.6.1-cp36-cp36m-win32.whl", hash = "sha256:546a1cd17595bd1dd009daf6f36705aa6f95337154360ce44932157d353dcd80"}, + {file = "duckdb-0.6.1-cp36-cp36m-win_amd64.whl", hash = "sha256:87b0d00eb9d1a7ebe437276203e0cdc93b4a2154ba9688c65e8d2a8735839ec6"}, + {file = "duckdb-0.6.1-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:8442e074de6e1969c3d2b24363a5a6d7f866d5ac3f4e358e357495b389eff6c1"}, + {file = "duckdb-0.6.1-cp37-cp37m-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:0a6bf2ae7bec803352dade14561cb0b461b2422e70f75d9f09b36ba2dad2613b"}, + {file = "duckdb-0.6.1-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:5054792f22733f89d9cbbced2bafd8772d72d0fe77f159310221cefcf981c680"}, + {file = "duckdb-0.6.1-cp37-cp37m-musllinux_1_1_i686.whl", hash = "sha256:21cc503dffc2c68bb825e4eb3098e82f40e910b3d09e1b3b7f090d39ad53fbea"}, + {file = "duckdb-0.6.1-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:54b3da77ad893e99c073087ff7f75a8c98154ac5139d317149f12b74367211db"}, + {file = "duckdb-0.6.1-cp37-cp37m-win32.whl", hash = "sha256:f1d709aa6a26172a3eab804b57763d5cdc1a4b785ac1fc2b09568578e52032ee"}, + {file = "duckdb-0.6.1-cp37-cp37m-win_amd64.whl", hash = "sha256:f4edcaa471d791393e37f63e3c7c728fa6324e3ac7e768b9dc2ea49065cd37cc"}, + {file = "duckdb-0.6.1-cp38-cp38-macosx_10_9_universal2.whl", hash = "sha256:d218c2dd3bda51fb79e622b7b2266183ac9493834b55010aa01273fa5b7a7105"}, + {file = "duckdb-0.6.1-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:0c7155cb93ab432eca44b651256c359281d26d927ff43badaf1d2276dd770832"}, + {file = "duckdb-0.6.1-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:0925778200090d3d5d8b6bb42b4d05d24db1e8912484ba3b7e7b7f8569f17dcb"}, + {file = "duckdb-0.6.1-cp38-cp38-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:8b544dd04bb851d08bc68b317a7683cec6091547ae75555d075f8c8a7edb626e"}, + {file = "duckdb-0.6.1-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:f2c37d5a0391cf3a3a66e63215968ffb78e6b84f659529fa4bd10478f6203071"}, + {file = "duckdb-0.6.1-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:ce376966260eb5c351fcc6af627a979dbbcae3efeb2e70f85b23aa45a21e289d"}, + {file = "duckdb-0.6.1-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:73c974b09dd08dff5e8bdedba11c7d0aa0fc46ca93954ee7d19e1e18c9883ac1"}, + {file = "duckdb-0.6.1-cp38-cp38-win32.whl", hash = "sha256:bfe39ed3a03e8b1ed764f58f513b37b24afe110d245803a41655d16d391ad9f1"}, + {file = "duckdb-0.6.1-cp38-cp38-win_amd64.whl", hash = "sha256:afa97d982dbe6b125631a17e222142e79bee88f7a13fc4cee92d09285e31ec83"}, + {file = "duckdb-0.6.1-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:c35ff4b1117096ef72d101524df0079da36c3735d52fcf1d907ccffa63bd6202"}, + {file = "duckdb-0.6.1-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:5c54910fbb6de0f21d562e18a5c91540c19876db61b862fc9ffc8e31be8b3f03"}, + {file = "duckdb-0.6.1-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:99a7172563a3ae67d867572ce27cf3962f58e76f491cb7f602f08c2af39213b3"}, + {file = "duckdb-0.6.1-cp39-cp39-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:7363ffe857d00216b659116647fbf1e925cb3895699015d4a4e50b746de13041"}, + {file = "duckdb-0.6.1-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:06c1cef25f896b2284ba048108f645c72fab5c54aa5a6f62f95663f44ff8a79b"}, + {file = "duckdb-0.6.1-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:e92dd6aad7e8c29d002947376b6f5ce28cae29eb3b6b58a64a46cdbfc5cb7943"}, + {file = "duckdb-0.6.1-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:4b280b2d8a01ecd4fe2feab041df70233c534fafbe33a38565b52c1e017529c7"}, + {file = "duckdb-0.6.1-cp39-cp39-win32.whl", hash = "sha256:d9212d76e90b8469743924a4d22bef845be310d0d193d54ae17d9ef1f753cfa7"}, + {file = "duckdb-0.6.1-cp39-cp39-win_amd64.whl", hash = "sha256:00b7be8f67ec1a8edaa8844f521267baa1a795f4c482bfad56c72c26e1862ab2"}, + {file = "duckdb-0.6.1.tar.gz", hash = "sha256:6d26e9f1afcb924a6057785e506810d48332d4764ddc4a5b414d0f2bf0cacfb4"}, +] +exceptiongroup = [ + {file = "exceptiongroup-1.0.4-py3-none-any.whl", hash = "sha256:542adf9dea4055530d6e1279602fa5cb11dab2395fa650b8674eaec35fc4a828"}, + {file = "exceptiongroup-1.0.4.tar.gz", hash = "sha256:bd14967b79cd9bdb54d97323216f8fdf533e278df937aa2a90089e7d6e06e5ec"}, +] +fastavro = [ + {file = "fastavro-1.7.0-cp310-cp310-macosx_10_14_x86_64.whl", hash = "sha256:ab3387a06e272980fa034f5c62f7063977b77df6416d3d30a4d3b49cc8827566"}, + {file = "fastavro-1.7.0-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:216132bc54da19e97e1531dd69c86282408d4c797749d83b01b3a00862a180de"}, + {file = "fastavro-1.7.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:c20cf6cd8098bb93c2cffd53d03ccea1dcf9ec594a5c83963acf29a2882f8693"}, + {file = "fastavro-1.7.0-cp310-cp310-win_amd64.whl", hash = "sha256:54b60e79506a456bcfc940560fa2c73a7a8e3ddc58a1ae4d94fdd99f6b02aef0"}, + {file = "fastavro-1.7.0-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:4da6d83abd04a804310667f8d1fc23d44e9ed91ed9a9bc9c1fcd906e0c403b12"}, + {file = "fastavro-1.7.0-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:7c664302f94064adeb93403c61c74e07b9710526403eba3b59169f99bb99c55c"}, + {file = "fastavro-1.7.0-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:7aab162f02e64bf82d0282430a3c6ec7a36982b1c5d479e7dcc278e6d62a84b8"}, + {file = "fastavro-1.7.0-cp311-cp311-win_amd64.whl", hash = "sha256:63d0d2a2bb3e85d006c834633be51b105a50b0dc7cc8423b06f30601b532adf4"}, + {file = "fastavro-1.7.0-cp37-cp37m-macosx_10_15_x86_64.whl", hash = "sha256:a07a3245049529d6d9028d664361cc4990e74d035d2303875295e2f7b97eba09"}, + {file = "fastavro-1.7.0-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:7df57d31cb58770a9066790250e9f4ec91242c69e1dc62ea732a6fb2406a8f96"}, + {file = "fastavro-1.7.0-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:b905634b5763ca08c9f7b51486e2c3ae7907f5d9bc48208c69b16ccbe8455e90"}, + {file = "fastavro-1.7.0-cp37-cp37m-win_amd64.whl", hash = "sha256:749206f1cec3d7429546e49db5708f4571497d35181b6b334c4844133f230515"}, + {file = "fastavro-1.7.0-cp38-cp38-macosx_10_15_x86_64.whl", hash = "sha256:a0ebd5c1269085179de4b3f072de274fb66a471ecbc5245bd8684e6f94943c2f"}, + {file = "fastavro-1.7.0-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a7e35b94b692f8cca0096c89abf1937efed66252dea0b3b3165babfb3c289fb7"}, + {file = "fastavro-1.7.0-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:202424a5a7831b773f0f2cc2f82e89ed1726051fd5994f13dc678514144e10d4"}, + {file = "fastavro-1.7.0-cp38-cp38-win_amd64.whl", hash = "sha256:601f8c2ec166bd721f4b12eafe195dd1373d3f8dce4fe2425abd2df3e3968ac7"}, + {file = "fastavro-1.7.0-cp39-cp39-macosx_10_15_x86_64.whl", hash = "sha256:91cc9be740abca894082af4fe3ab9db057d4e5fa783cfa9a94c02ec041bf4346"}, + {file = "fastavro-1.7.0-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:9e4463b00242e4baf52d499aeefab46a26d9dd18d808d4219cd4d21089da540e"}, + {file = "fastavro-1.7.0-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:f7348d318858203bd108e6bcde177d8a6f0590b52bc624d815f26fb6c37029bb"}, + {file = "fastavro-1.7.0-cp39-cp39-win_amd64.whl", hash = "sha256:492e8902063aa1c73170e11c18495fcaa86b71eae3513ef83ba438ca02b16b34"}, + {file = "fastavro-1.7.0.tar.gz", hash = "sha256:4b1205f46489b4032d3155c1ab44d9824be0c7454df98d3a5bd22b78b98f23c8"}, +] +filelock = [ + {file = "filelock-3.8.2-py3-none-any.whl", hash = "sha256:8df285554452285f79c035efb0c861eb33a4bcfa5b7a137016e32e6a90f9792c"}, + {file = "filelock-3.8.2.tar.gz", hash = "sha256:7565f628ea56bfcd8e54e42bdc55da899c85c1abfe1b5bcfd147e9188cebb3b2"}, +] +frozenlist = [ + {file = "frozenlist-1.3.3-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:ff8bf625fe85e119553b5383ba0fb6aa3d0ec2ae980295aaefa552374926b3f4"}, + {file = "frozenlist-1.3.3-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:dfbac4c2dfcc082fcf8d942d1e49b6aa0766c19d3358bd86e2000bf0fa4a9cf0"}, + {file = "frozenlist-1.3.3-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:b1c63e8d377d039ac769cd0926558bb7068a1f7abb0f003e3717ee003ad85530"}, + {file = "frozenlist-1.3.3-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:7fdfc24dcfce5b48109867c13b4cb15e4660e7bd7661741a391f821f23dfdca7"}, + {file = "frozenlist-1.3.3-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:2c926450857408e42f0bbc295e84395722ce74bae69a3b2aa2a65fe22cb14b99"}, + {file = "frozenlist-1.3.3-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:1841e200fdafc3d51f974d9d377c079a0694a8f06de2e67b48150328d66d5483"}, + {file = "frozenlist-1.3.3-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:f470c92737afa7d4c3aacc001e335062d582053d4dbe73cda126f2d7031068dd"}, + {file = "frozenlist-1.3.3-cp310-cp310-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:783263a4eaad7c49983fe4b2e7b53fa9770c136c270d2d4bbb6d2192bf4d9caf"}, + {file = "frozenlist-1.3.3-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:924620eef691990dfb56dc4709f280f40baee568c794b5c1885800c3ecc69816"}, + {file = "frozenlist-1.3.3-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:ae4dc05c465a08a866b7a1baf360747078b362e6a6dbeb0c57f234db0ef88ae0"}, + {file = "frozenlist-1.3.3-cp310-cp310-musllinux_1_1_ppc64le.whl", hash = "sha256:bed331fe18f58d844d39ceb398b77d6ac0b010d571cba8267c2e7165806b00ce"}, + {file = "frozenlist-1.3.3-cp310-cp310-musllinux_1_1_s390x.whl", hash = "sha256:02c9ac843e3390826a265e331105efeab489ffaf4dd86384595ee8ce6d35ae7f"}, + {file = "frozenlist-1.3.3-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:9545a33965d0d377b0bc823dcabf26980e77f1b6a7caa368a365a9497fb09420"}, + {file = "frozenlist-1.3.3-cp310-cp310-win32.whl", hash = "sha256:d5cd3ab21acbdb414bb6c31958d7b06b85eeb40f66463c264a9b343a4e238642"}, + {file = "frozenlist-1.3.3-cp310-cp310-win_amd64.whl", hash = "sha256:b756072364347cb6aa5b60f9bc18e94b2f79632de3b0190253ad770c5df17db1"}, + {file = "frozenlist-1.3.3-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:b4395e2f8d83fbe0c627b2b696acce67868793d7d9750e90e39592b3626691b7"}, + {file = "frozenlist-1.3.3-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:14143ae966a6229350021384870458e4777d1eae4c28d1a7aa47f24d030e6678"}, + {file = "frozenlist-1.3.3-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:5d8860749e813a6f65bad8285a0520607c9500caa23fea6ee407e63debcdbef6"}, + {file = "frozenlist-1.3.3-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:23d16d9f477bb55b6154654e0e74557040575d9d19fe78a161bd33d7d76808e8"}, + {file = "frozenlist-1.3.3-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:eb82dbba47a8318e75f679690190c10a5e1f447fbf9df41cbc4c3afd726d88cb"}, + {file = "frozenlist-1.3.3-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:9309869032abb23d196cb4e4db574232abe8b8be1339026f489eeb34a4acfd91"}, + {file = "frozenlist-1.3.3-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:a97b4fe50b5890d36300820abd305694cb865ddb7885049587a5678215782a6b"}, + {file = "frozenlist-1.3.3-cp311-cp311-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:c188512b43542b1e91cadc3c6c915a82a5eb95929134faf7fd109f14f9892ce4"}, + {file = "frozenlist-1.3.3-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:303e04d422e9b911a09ad499b0368dc551e8c3cd15293c99160c7f1f07b59a48"}, + {file = "frozenlist-1.3.3-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:0771aed7f596c7d73444c847a1c16288937ef988dc04fb9f7be4b2aa91db609d"}, + {file = "frozenlist-1.3.3-cp311-cp311-musllinux_1_1_ppc64le.whl", hash = "sha256:66080ec69883597e4d026f2f71a231a1ee9887835902dbe6b6467d5a89216cf6"}, + {file = "frozenlist-1.3.3-cp311-cp311-musllinux_1_1_s390x.whl", hash = "sha256:41fe21dc74ad3a779c3d73a2786bdf622ea81234bdd4faf90b8b03cad0c2c0b4"}, + {file = "frozenlist-1.3.3-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:f20380df709d91525e4bee04746ba612a4df0972c1b8f8e1e8af997e678c7b81"}, + {file = "frozenlist-1.3.3-cp311-cp311-win32.whl", hash = "sha256:f30f1928162e189091cf4d9da2eac617bfe78ef907a761614ff577ef4edfb3c8"}, + {file = "frozenlist-1.3.3-cp311-cp311-win_amd64.whl", hash = "sha256:a6394d7dadd3cfe3f4b3b186e54d5d8504d44f2d58dcc89d693698e8b7132b32"}, + {file = "frozenlist-1.3.3-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:8df3de3a9ab8325f94f646609a66cbeeede263910c5c0de0101079ad541af332"}, + {file = "frozenlist-1.3.3-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:0693c609e9742c66ba4870bcee1ad5ff35462d5ffec18710b4ac89337ff16e27"}, + {file = "frozenlist-1.3.3-cp37-cp37m-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:cd4210baef299717db0a600d7a3cac81d46ef0e007f88c9335db79f8979c0d3d"}, + {file = "frozenlist-1.3.3-cp37-cp37m-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:394c9c242113bfb4b9aa36e2b80a05ffa163a30691c7b5a29eba82e937895d5e"}, + {file = "frozenlist-1.3.3-cp37-cp37m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:6327eb8e419f7d9c38f333cde41b9ae348bec26d840927332f17e887a8dcb70d"}, + {file = "frozenlist-1.3.3-cp37-cp37m-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:2e24900aa13212e75e5b366cb9065e78bbf3893d4baab6052d1aca10d46d944c"}, + {file = "frozenlist-1.3.3-cp37-cp37m-musllinux_1_1_aarch64.whl", hash = "sha256:3843f84a6c465a36559161e6c59dce2f2ac10943040c2fd021cfb70d58c4ad56"}, + {file = "frozenlist-1.3.3-cp37-cp37m-musllinux_1_1_i686.whl", hash = "sha256:84610c1502b2461255b4c9b7d5e9c48052601a8957cd0aea6ec7a7a1e1fb9420"}, + {file = "frozenlist-1.3.3-cp37-cp37m-musllinux_1_1_ppc64le.whl", hash = "sha256:c21b9aa40e08e4f63a2f92ff3748e6b6c84d717d033c7b3438dd3123ee18f70e"}, + {file = "frozenlist-1.3.3-cp37-cp37m-musllinux_1_1_s390x.whl", hash = "sha256:efce6ae830831ab6a22b9b4091d411698145cb9b8fc869e1397ccf4b4b6455cb"}, + {file = "frozenlist-1.3.3-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:40de71985e9042ca00b7953c4f41eabc3dc514a2d1ff534027f091bc74416401"}, + {file = "frozenlist-1.3.3-cp37-cp37m-win32.whl", hash = "sha256:180c00c66bde6146a860cbb81b54ee0df350d2daf13ca85b275123bbf85de18a"}, + {file = "frozenlist-1.3.3-cp37-cp37m-win_amd64.whl", hash = "sha256:9bbbcedd75acdfecf2159663b87f1bb5cfc80e7cd99f7ddd9d66eb98b14a8411"}, + {file = "frozenlist-1.3.3-cp38-cp38-macosx_10_9_universal2.whl", hash = "sha256:034a5c08d36649591be1cbb10e09da9f531034acfe29275fc5454a3b101ce41a"}, + {file = "frozenlist-1.3.3-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:ba64dc2b3b7b158c6660d49cdb1d872d1d0bf4e42043ad8d5006099479a194e5"}, + {file = "frozenlist-1.3.3-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:47df36a9fe24054b950bbc2db630d508cca3aa27ed0566c0baf661225e52c18e"}, + {file = "frozenlist-1.3.3-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:008a054b75d77c995ea26629ab3a0c0d7281341f2fa7e1e85fa6153ae29ae99c"}, + {file = "frozenlist-1.3.3-cp38-cp38-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:841ea19b43d438a80b4de62ac6ab21cfe6827bb8a9dc62b896acc88eaf9cecba"}, + {file = "frozenlist-1.3.3-cp38-cp38-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:e235688f42b36be2b6b06fc37ac2126a73b75fb8d6bc66dd632aa35286238703"}, + {file = "frozenlist-1.3.3-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:ca713d4af15bae6e5d79b15c10c8522859a9a89d3b361a50b817c98c2fb402a2"}, + {file = "frozenlist-1.3.3-cp38-cp38-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:9ac5995f2b408017b0be26d4a1d7c61bce106ff3d9e3324374d66b5964325448"}, + {file = "frozenlist-1.3.3-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:a4ae8135b11652b08a8baf07631d3ebfe65a4c87909dbef5fa0cdde440444ee4"}, + {file = "frozenlist-1.3.3-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:4ea42116ceb6bb16dbb7d526e242cb6747b08b7710d9782aa3d6732bd8d27649"}, + {file = "frozenlist-1.3.3-cp38-cp38-musllinux_1_1_ppc64le.whl", hash = "sha256:810860bb4bdce7557bc0febb84bbd88198b9dbc2022d8eebe5b3590b2ad6c842"}, + {file = "frozenlist-1.3.3-cp38-cp38-musllinux_1_1_s390x.whl", hash = "sha256:ee78feb9d293c323b59a6f2dd441b63339a30edf35abcb51187d2fc26e696d13"}, + {file = "frozenlist-1.3.3-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:0af2e7c87d35b38732e810befb9d797a99279cbb85374d42ea61c1e9d23094b3"}, + {file = "frozenlist-1.3.3-cp38-cp38-win32.whl", hash = "sha256:899c5e1928eec13fd6f6d8dc51be23f0d09c5281e40d9cf4273d188d9feeaf9b"}, + {file = "frozenlist-1.3.3-cp38-cp38-win_amd64.whl", hash = "sha256:7f44e24fa70f6fbc74aeec3e971f60a14dde85da364aa87f15d1be94ae75aeef"}, + {file = "frozenlist-1.3.3-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:2b07ae0c1edaa0a36339ec6cce700f51b14a3fc6545fdd32930d2c83917332cf"}, + {file = "frozenlist-1.3.3-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:ebb86518203e12e96af765ee89034a1dbb0c3c65052d1b0c19bbbd6af8a145e1"}, + {file = "frozenlist-1.3.3-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:5cf820485f1b4c91e0417ea0afd41ce5cf5965011b3c22c400f6d144296ccbc0"}, + {file = "frozenlist-1.3.3-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:5c11e43016b9024240212d2a65043b70ed8dfd3b52678a1271972702d990ac6d"}, + {file = "frozenlist-1.3.3-cp39-cp39-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:8fa3c6e3305aa1146b59a09b32b2e04074945ffcfb2f0931836d103a2c38f936"}, + {file = "frozenlist-1.3.3-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:352bd4c8c72d508778cf05ab491f6ef36149f4d0cb3c56b1b4302852255d05d5"}, + {file = "frozenlist-1.3.3-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:65a5e4d3aa679610ac6e3569e865425b23b372277f89b5ef06cf2cdaf1ebf22b"}, + {file = "frozenlist-1.3.3-cp39-cp39-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:b1e2c1185858d7e10ff045c496bbf90ae752c28b365fef2c09cf0fa309291669"}, + {file = "frozenlist-1.3.3-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:f163d2fd041c630fed01bc48d28c3ed4a3b003c00acd396900e11ee5316b56bb"}, + {file = "frozenlist-1.3.3-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:05cdb16d09a0832eedf770cb7bd1fe57d8cf4eaf5aced29c4e41e3f20b30a784"}, + {file = "frozenlist-1.3.3-cp39-cp39-musllinux_1_1_ppc64le.whl", hash = "sha256:8bae29d60768bfa8fb92244b74502b18fae55a80eac13c88eb0b496d4268fd2d"}, + {file = "frozenlist-1.3.3-cp39-cp39-musllinux_1_1_s390x.whl", hash = "sha256:eedab4c310c0299961ac285591acd53dc6723a1ebd90a57207c71f6e0c2153ab"}, + {file = "frozenlist-1.3.3-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:3bbdf44855ed8f0fbcd102ef05ec3012d6a4fd7c7562403f76ce6a52aeffb2b1"}, + {file = "frozenlist-1.3.3-cp39-cp39-win32.whl", hash = "sha256:efa568b885bca461f7c7b9e032655c0c143d305bf01c30caf6db2854a4532b38"}, + {file = "frozenlist-1.3.3-cp39-cp39-win_amd64.whl", hash = "sha256:cfe33efc9cb900a4c46f91a5ceba26d6df370ffddd9ca386eb1d4f0ad97b9ea9"}, + {file = "frozenlist-1.3.3.tar.gz", hash = "sha256:58bcc55721e8a90b88332d6cd441261ebb22342e238296bb330968952fbb3a6a"}, +] +fsspec = [ + {file = "fsspec-2022.10.0-py3-none-any.whl", hash = "sha256:6b7c6ab3b476cdf17efcfeccde7fca28ef5a48f73a71010aaceec5fc15bf9ebf"}, + {file = "fsspec-2022.10.0.tar.gz", hash = "sha256:cb6092474e90487a51de768170f3afa50ca8982c26150a59072b16433879ff1d"}, +] +identify = [ + {file = "identify-2.5.10-py2.py3-none-any.whl", hash = "sha256:fb7c2feaeca6976a3ffa31ec3236a6911fbc51aec9acc111de2aed99f244ade2"}, + {file = "identify-2.5.10.tar.gz", hash = "sha256:dce9e31fee7dbc45fea36a9e855c316b8fbf807e65a862f160840bb5a2bf5dfd"}, +] +idna = [ + {file = "idna-3.4-py3-none-any.whl", hash = "sha256:90b77e79eaa3eba6de819a0c442c0b4ceefc341a7a2ab77d7562bf49f425c5c2"}, + {file = "idna-3.4.tar.gz", hash = "sha256:814f528e8dead7d329833b91c5faa87d60bf71824cd12a7530b5526063d02cb4"}, +] +importlib-metadata = [ + {file = "importlib_metadata-5.1.0-py3-none-any.whl", hash = "sha256:d84d17e21670ec07990e1044a99efe8d615d860fd176fc29ef5c306068fda313"}, + {file = "importlib_metadata-5.1.0.tar.gz", hash = "sha256:d5059f9f1e8e41f80e9c56c2ee58811450c31984dfa625329ffd7c0dad88a73b"}, +] +iniconfig = [ + {file = "iniconfig-1.1.1-py2.py3-none-any.whl", hash = "sha256:011e24c64b7f47f6ebd835bb12a743f2fbe9a26d4cecaa7f53bc4f35ee9da8b3"}, + {file = "iniconfig-1.1.1.tar.gz", hash = "sha256:bc3af051d7d14b2ee5ef9969666def0cd1a000e121eaea580d4a313df4b37f32"}, +] +isodate = [ + {file = "isodate-0.6.1-py2.py3-none-any.whl", hash = "sha256:0751eece944162659049d35f4f549ed815792b38793f07cf73381c1c87cbed96"}, + {file = "isodate-0.6.1.tar.gz", hash = "sha256:48c5881de7e8b0a0d648cb024c8062dc84e7b840ed81e864c7614fd3c127bde9"}, +] +jinja2 = [ + {file = "Jinja2-3.1.2-py3-none-any.whl", hash = "sha256:6088930bfe239f0e6710546ab9c19c9ef35e29792895fed6e6e31a023a182a61"}, + {file = "Jinja2-3.1.2.tar.gz", hash = "sha256:31351a702a408a9e7595a8fc6150fc3f43bb6bf7e319770cbc0db9df9437e852"}, +] +jmespath = [ + {file = "jmespath-1.0.1-py3-none-any.whl", hash = "sha256:02e2e4cc71b5bcab88332eebf907519190dd9e6e82107fa7f83b1003a6252980"}, + {file = "jmespath-1.0.1.tar.gz", hash = "sha256:90261b206d6defd58fdd5e85f478bf633a2901798906be2ad389150c5c60edbe"}, +] +markupsafe = [ + {file = "MarkupSafe-2.1.1-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:86b1f75c4e7c2ac2ccdaec2b9022845dbb81880ca318bb7a0a01fbf7813e3812"}, + {file = "MarkupSafe-2.1.1-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:f121a1420d4e173a5d96e47e9a0c0dcff965afdf1626d28de1460815f7c4ee7a"}, + {file = "MarkupSafe-2.1.1-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a49907dd8420c5685cfa064a1335b6754b74541bbb3706c259c02ed65b644b3e"}, + {file = "MarkupSafe-2.1.1-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:10c1bfff05d95783da83491be968e8fe789263689c02724e0c691933c52994f5"}, + {file = "MarkupSafe-2.1.1-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:b7bd98b796e2b6553da7225aeb61f447f80a1ca64f41d83612e6139ca5213aa4"}, + {file = "MarkupSafe-2.1.1-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:b09bf97215625a311f669476f44b8b318b075847b49316d3e28c08e41a7a573f"}, + {file = "MarkupSafe-2.1.1-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:694deca8d702d5db21ec83983ce0bb4b26a578e71fbdbd4fdcd387daa90e4d5e"}, + {file = "MarkupSafe-2.1.1-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:efc1913fd2ca4f334418481c7e595c00aad186563bbc1ec76067848c7ca0a933"}, + {file = "MarkupSafe-2.1.1-cp310-cp310-win32.whl", hash = "sha256:4a33dea2b688b3190ee12bd7cfa29d39c9ed176bda40bfa11099a3ce5d3a7ac6"}, + {file = "MarkupSafe-2.1.1-cp310-cp310-win_amd64.whl", hash = "sha256:dda30ba7e87fbbb7eab1ec9f58678558fd9a6b8b853530e176eabd064da81417"}, + {file = "MarkupSafe-2.1.1-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:671cd1187ed5e62818414afe79ed29da836dde67166a9fac6d435873c44fdd02"}, + {file = "MarkupSafe-2.1.1-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:3799351e2336dc91ea70b034983ee71cf2f9533cdff7c14c90ea126bfd95d65a"}, + {file = "MarkupSafe-2.1.1-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:e72591e9ecd94d7feb70c1cbd7be7b3ebea3f548870aa91e2732960fa4d57a37"}, + {file = "MarkupSafe-2.1.1-cp37-cp37m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:6fbf47b5d3728c6aea2abb0589b5d30459e369baa772e0f37a0320185e87c980"}, + {file = "MarkupSafe-2.1.1-cp37-cp37m-musllinux_1_1_aarch64.whl", hash = "sha256:d5ee4f386140395a2c818d149221149c54849dfcfcb9f1debfe07a8b8bd63f9a"}, + {file = "MarkupSafe-2.1.1-cp37-cp37m-musllinux_1_1_i686.whl", hash = "sha256:bcb3ed405ed3222f9904899563d6fc492ff75cce56cba05e32eff40e6acbeaa3"}, + {file = "MarkupSafe-2.1.1-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:e1c0b87e09fa55a220f058d1d49d3fb8df88fbfab58558f1198e08c1e1de842a"}, + {file = "MarkupSafe-2.1.1-cp37-cp37m-win32.whl", hash = "sha256:8dc1c72a69aa7e082593c4a203dcf94ddb74bb5c8a731e4e1eb68d031e8498ff"}, + {file = "MarkupSafe-2.1.1-cp37-cp37m-win_amd64.whl", hash = "sha256:97a68e6ada378df82bc9f16b800ab77cbf4b2fada0081794318520138c088e4a"}, + {file = "MarkupSafe-2.1.1-cp38-cp38-macosx_10_9_universal2.whl", hash = "sha256:e8c843bbcda3a2f1e3c2ab25913c80a3c5376cd00c6e8c4a86a89a28c8dc5452"}, + {file = "MarkupSafe-2.1.1-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:0212a68688482dc52b2d45013df70d169f542b7394fc744c02a57374a4207003"}, + {file = "MarkupSafe-2.1.1-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:8e576a51ad59e4bfaac456023a78f6b5e6e7651dcd383bcc3e18d06f9b55d6d1"}, + {file = "MarkupSafe-2.1.1-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:4b9fe39a2ccc108a4accc2676e77da025ce383c108593d65cc909add5c3bd601"}, + {file = "MarkupSafe-2.1.1-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:96e37a3dc86e80bf81758c152fe66dbf60ed5eca3d26305edf01892257049925"}, + {file = "MarkupSafe-2.1.1-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:6d0072fea50feec76a4c418096652f2c3238eaa014b2f94aeb1d56a66b41403f"}, + {file = "MarkupSafe-2.1.1-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:089cf3dbf0cd6c100f02945abeb18484bd1ee57a079aefd52cffd17fba910b88"}, + {file = "MarkupSafe-2.1.1-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:6a074d34ee7a5ce3effbc526b7083ec9731bb3cbf921bbe1d3005d4d2bdb3a63"}, + {file = "MarkupSafe-2.1.1-cp38-cp38-win32.whl", hash = "sha256:421be9fbf0ffe9ffd7a378aafebbf6f4602d564d34be190fc19a193232fd12b1"}, + {file = "MarkupSafe-2.1.1-cp38-cp38-win_amd64.whl", hash = "sha256:fc7b548b17d238737688817ab67deebb30e8073c95749d55538ed473130ec0c7"}, + {file = "MarkupSafe-2.1.1-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:e04e26803c9c3851c931eac40c695602c6295b8d432cbe78609649ad9bd2da8a"}, + {file = "MarkupSafe-2.1.1-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:b87db4360013327109564f0e591bd2a3b318547bcef31b468a92ee504d07ae4f"}, + {file = "MarkupSafe-2.1.1-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:99a2a507ed3ac881b975a2976d59f38c19386d128e7a9a18b7df6fff1fd4c1d6"}, + {file = "MarkupSafe-2.1.1-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:56442863ed2b06d19c37f94d999035e15ee982988920e12a5b4ba29b62ad1f77"}, + {file = "MarkupSafe-2.1.1-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:3ce11ee3f23f79dbd06fb3d63e2f6af7b12db1d46932fe7bd8afa259a5996603"}, + {file = "MarkupSafe-2.1.1-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:33b74d289bd2f5e527beadcaa3f401e0df0a89927c1559c8566c066fa4248ab7"}, + {file = "MarkupSafe-2.1.1-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:43093fb83d8343aac0b1baa75516da6092f58f41200907ef92448ecab8825135"}, + {file = "MarkupSafe-2.1.1-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:8e3dcf21f367459434c18e71b2a9532d96547aef8a871872a5bd69a715c15f96"}, + {file = "MarkupSafe-2.1.1-cp39-cp39-win32.whl", hash = "sha256:d4306c36ca495956b6d568d276ac11fdd9c30a36f1b6eb928070dc5360b22e1c"}, + {file = "MarkupSafe-2.1.1-cp39-cp39-win_amd64.whl", hash = "sha256:46d00d6cfecdde84d40e572d63735ef81423ad31184100411e6e3388d405e247"}, + {file = "MarkupSafe-2.1.1.tar.gz", hash = "sha256:7f91197cc9e48f989d12e4e6fbc46495c446636dfc81b9ccf50bb0ec74b91d4b"}, +] +mmhash3 = [ + {file = "mmhash3-3.0.1-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:47deea30cd8d3d5cd52dc740902a4c70383bfe8248eac29d0877fe63e03c2713"}, + {file = "mmhash3-3.0.1-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:ecdaf4d1de617818bf05cd526ca558db6010beeba7ea9e19f695f2bdcac0e0a4"}, + {file = "mmhash3-3.0.1-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:4675585617584e9e1aafa3a90ac0ac9a437257c507748d97de8b21977e9d6745"}, + {file = "mmhash3-3.0.1-cp310-cp310-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:ebfd0c2af09b41f0fe1dd260799bda90a0fc7eba4477ccaeb3951527700cd58f"}, + {file = "mmhash3-3.0.1-cp310-cp310-win32.whl", hash = "sha256:68587dec7b8acdb7528fd511e295d8b5ccfe26022923a69867e1822f0fdb4c44"}, + {file = "mmhash3-3.0.1-cp310-cp310-win_amd64.whl", hash = "sha256:54954ebe3a614f19ab4cfe24fa66ea093c493d9fac0533d002dd64c2540a0c99"}, + {file = "mmhash3-3.0.1-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:b172f3bd3220b0bf56fd7cc760fd8a9033def47103611d63fe867003079a1256"}, + {file = "mmhash3-3.0.1-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:de7895eafabc32f7c0c09a81a624921f536768de6e438e3de69e3e954a4d7072"}, + {file = "mmhash3-3.0.1-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:b4b0914effe4ddd8d33149e3508564c17719465b0cc81691c4fa50d5e0e14f80"}, + {file = "mmhash3-3.0.1-cp311-cp311-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:c0575050ac691475938df1ad03d8738c5bd1eadef62093e76157ebb7f2df0946"}, + {file = "mmhash3-3.0.1-cp311-cp311-win32.whl", hash = "sha256:22f92f0f88f28b215357acd346362fa9f7c9fffb436beb42cc4b442b676dbaa3"}, + {file = "mmhash3-3.0.1-cp311-cp311-win_amd64.whl", hash = "sha256:538240ab7936bf71b18304e5a7e7fd3c4c2fab103330ea99584bb4f777299a2b"}, + {file = "mmhash3-3.0.1-cp36-cp36m-macosx_10_9_x86_64.whl", hash = "sha256:ca791bfb311e36ce13998e4632262ed4b95da9d3461941e18b6690760171a045"}, + {file = "mmhash3-3.0.1-cp36-cp36m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:b41708f72c6aa2a49ada1f0b61e85c05cd8389ad31d463fd5bca29999a4d5f9c"}, + {file = "mmhash3-3.0.1-cp36-cp36m-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:a3ce9b4533ddc0a88ba045a27309714c3b127bd05e57fd252d1d5a71d4247ea7"}, + {file = "mmhash3-3.0.1-cp36-cp36m-win32.whl", hash = "sha256:bfafeb96fdeb10db8767d06e1f07b6fdcddba4aaa0dd15058561a49f7ae45345"}, + {file = "mmhash3-3.0.1-cp36-cp36m-win_amd64.whl", hash = "sha256:97fe077b24c948709ed2afc749bf6285d407bc54ff12c63d2dc86678c38a0b8e"}, + {file = "mmhash3-3.0.1-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:0cfd91ccd5fca1ba7ee925297131a15dfb94c714bfe6ba0fb3b1ca78b12bbfec"}, + {file = "mmhash3-3.0.1-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:6d51b1005233141ce7394531af40a3f0fc1f274467bf8dff44dcf7987924fe58"}, + {file = "mmhash3-3.0.1-cp37-cp37m-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:855c67b100e37df166acf79cdff58fa8f9f6c48be0d1e1b6e9ad0fa34a9661ef"}, + {file = "mmhash3-3.0.1-cp37-cp37m-win32.whl", hash = "sha256:bb3030df1334fd665427f8be8e8ce4f04aeab7f6010ce4f2c128f0099bdab96f"}, + {file = "mmhash3-3.0.1-cp37-cp37m-win_amd64.whl", hash = "sha256:1545e1177294afe4912d5a5e401c7fa9b799dd109b30289e7af74d5b07e7c474"}, + {file = "mmhash3-3.0.1-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:2479899e7dda834a671991a1098a691ab1c2eaa20c3e939d691ca4a19361cfe0"}, + {file = "mmhash3-3.0.1-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:c9056196d5e3d3d844433a63d806a683f710ab3aaf1c910550c7746464bc43ae"}, + {file = "mmhash3-3.0.1-cp38-cp38-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:a0d4c307af0bf70207305f70f131898be071d1b19a89f462b13487f5c25e8d4e"}, + {file = "mmhash3-3.0.1-cp38-cp38-win32.whl", hash = "sha256:5f885f65e329fd14bc38debac4a79eacf381e856965d9c65c4d1c6946ea190d0"}, + {file = "mmhash3-3.0.1-cp38-cp38-win_amd64.whl", hash = "sha256:3b42d0bda5e1cd22c18b16887b0521060fb59d0aaaaf033feacbc0a2492d20fe"}, + {file = "mmhash3-3.0.1-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:d3f333286bb87aa9dc6bd8e7960a55a27b011a401f24b889a50e6d219f65e7c9"}, + {file = "mmhash3-3.0.1-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:6b7ef2eb95a18bcd02ce0d3e047adde3a025afd96c1d266a8a0d44574f44a307"}, + {file = "mmhash3-3.0.1-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:d6ac8a5f511c60f341bf9cae462bb4941abb149d98464ba5f4f4548875b601c6"}, + {file = "mmhash3-3.0.1-cp39-cp39-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:efef9e632e6e248e46f52d108a5ebd1f28edaf427b7fd47ebf97dbff4b2cab81"}, + {file = "mmhash3-3.0.1-cp39-cp39-win32.whl", hash = "sha256:bdac06d72e448c67afb12e758b203d875e29d4097bb279a38a5649d44b518ba7"}, + {file = "mmhash3-3.0.1-cp39-cp39-win_amd64.whl", hash = "sha256:0baeaa20cac5f75ed14f28056826bd9d9c8b2354b382073f3fd5190708992a0d"}, + {file = "mmhash3-3.0.1.tar.gz", hash = "sha256:a00d68f4a1cc434b9501513c8a29e18ed1ddad383677d72b41d71d0d862348af"}, +] +moto = [ + {file = "moto-4.0.11-py3-none-any.whl", hash = "sha256:704d6d38a4e6fe49e1fe9c6b4127ca46c66aac00368149bc1f1d70a0ceff8846"}, + {file = "moto-4.0.11.tar.gz", hash = "sha256:a6388de4a746e0b509286e1d7e70f86900b4f69ec65f6c92c47e570f95d05b14"}, +] +msal = [ + {file = "msal-1.20.0-py2.py3-none-any.whl", hash = "sha256:d2f1c26368ecdc28c8657d457352faa0b81b1845a7b889d8676787721ba86792"}, + {file = "msal-1.20.0.tar.gz", hash = "sha256:78344cd4c91d6134a593b5e3e45541e666e37b747ff8a6316c3668dd1e6ab6b2"}, +] +msal-extensions = [ + {file = "msal-extensions-1.0.0.tar.gz", hash = "sha256:c676aba56b0cce3783de1b5c5ecfe828db998167875126ca4b47dc6436451354"}, + {file = "msal_extensions-1.0.0-py2.py3-none-any.whl", hash = "sha256:91e3db9620b822d0ed2b4d1850056a0f133cba04455e62f11612e40f5502f2ee"}, +] +msrest = [ + {file = "msrest-0.7.1-py3-none-any.whl", hash = "sha256:21120a810e1233e5e6cc7fe40b474eeb4ec6f757a15d7cf86702c369f9567c32"}, + {file = "msrest-0.7.1.zip", hash = "sha256:6e7661f46f3afd88b75667b7187a92829924446c7ea1d169be8c4bb7eeb788b9"}, +] +multidict = [ + {file = "multidict-6.0.3-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:73009ea04205966d47e16d98686ac5c438af23a1bb30b48a2c5da3423ec9ce37"}, + {file = "multidict-6.0.3-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:8b92a9f3ab904397a33b193000dc4de7318ea175c4c460a1e154c415f9008e3d"}, + {file = "multidict-6.0.3-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:578bfcb16f4b8675ef71b960c00f174b0426e0eeb796bab6737389d8288eb827"}, + {file = "multidict-6.0.3-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:f1650ea41c408755da5eed52ac6ccbc8938ccc3e698d81e6f6a1be02ff2a0945"}, + {file = "multidict-6.0.3-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:d52442e7c951e4c9ee591d6047706e66923d248d83958bbf99b8b19515fffaef"}, + {file = "multidict-6.0.3-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:ad7d66422b9cc51125509229693d27e18c08f2dea3ac9de408d821932b1b3759"}, + {file = "multidict-6.0.3-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:6cd14e61f0da2a2cfb9fe05bfced2a1ed7063ce46a7a8cd473be4973de9a7f91"}, + {file = "multidict-6.0.3-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:190626ced82d4cc567a09e7346340d380154a493bac6905e0095d8158cdf1e38"}, + {file = "multidict-6.0.3-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:791458a1f7d1b4ab3bd9e93e0dcd1d59ef7ee9aa051dcd1ea030e62e49b923fd"}, + {file = "multidict-6.0.3-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:b46e79a9f4db53897d17bc64a39d1c7c2be3e3d4f8dba6d6730a2b13ddf0f986"}, + {file = "multidict-6.0.3-cp310-cp310-musllinux_1_1_ppc64le.whl", hash = "sha256:e4a095e18847c12ec20e55326ab8782d9c2d599400a3a2f174fab4796875d0e2"}, + {file = "multidict-6.0.3-cp310-cp310-musllinux_1_1_s390x.whl", hash = "sha256:fb6c3dc3d65014d2c782f5acf0b3ba14e639c6c33d3ed8932ead76b9080b3544"}, + {file = "multidict-6.0.3-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:3541882266247c7cd3dba78d6ef28dbe704774df60c9e4231edaa4493522e614"}, + {file = "multidict-6.0.3-cp310-cp310-win32.whl", hash = "sha256:67090b17a0a5be5704fd109f231ee73cefb1b3802d41288d6378b5df46ae89ba"}, + {file = "multidict-6.0.3-cp310-cp310-win_amd64.whl", hash = "sha256:36df958b15639e40472adaa4f0c2c7828fe680f894a6b48c4ce229f59a6a798b"}, + {file = "multidict-6.0.3-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:5b51969503709415a35754954c2763f536a70b8bf7360322b2edb0c0a44391f6"}, + {file = "multidict-6.0.3-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:24e8d513bfcaadc1f8b0ebece3ff50961951c54b07d5a775008a882966102418"}, + {file = "multidict-6.0.3-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:d325d61cac602976a5d47b19eaa7d04e3daf4efce2164c630219885087234102"}, + {file = "multidict-6.0.3-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:26fbbe17f8a7211b623502d2bf41022a51da3025142401417c765bf9a56fed4c"}, + {file = "multidict-6.0.3-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:4fb3fe591956d8841882c463f934c9f7485cfd5f763a08c0d467b513dc18ef89"}, + {file = "multidict-6.0.3-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:e1925f78a543b94c3d46274c66a366fee8a263747060220ed0188e5f3eeea1c0"}, + {file = "multidict-6.0.3-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:21e1ce0b187c4e93112304dcde2aa18922fdbe8fb4f13d8aa72a5657bce0563a"}, + {file = "multidict-6.0.3-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:e07c24018986fb00d6e7eafca8fcd6e05095649e17fcf0e33a592caaa62a78b9"}, + {file = "multidict-6.0.3-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:114a4ab3e5cfbc56c4b6697686ecb92376c7e8c56893ef20547921552f8bdf57"}, + {file = "multidict-6.0.3-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:4ccf55f28066b4f08666764a957c2b7c241c7547b0921d69c7ceab5f74fe1a45"}, + {file = "multidict-6.0.3-cp311-cp311-musllinux_1_1_ppc64le.whl", hash = "sha256:9d359b0a962e052b713647ac1f13eabf2263167b149ed1e27d5c579f5c8c7d2c"}, + {file = "multidict-6.0.3-cp311-cp311-musllinux_1_1_s390x.whl", hash = "sha256:df7b4cee3ff31b3335aba602f8d70dbc641e5b7164b1e9565570c9d3c536a438"}, + {file = "multidict-6.0.3-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:ee9b1cae9a6c5d023e5a150f6f6b9dbb3c3bbc7887d6ee07d4c0ecb49a473734"}, + {file = "multidict-6.0.3-cp311-cp311-win32.whl", hash = "sha256:960ce1b790952916e682093788696ef7e33ac6a97482f9b983abdc293091b531"}, + {file = "multidict-6.0.3-cp311-cp311-win_amd64.whl", hash = "sha256:2b66d61966b12e6bba500e5cbb2c721a35e119c30ee02495c5629bd0e91eea30"}, + {file = "multidict-6.0.3-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:526f8397fc124674b8f39748680a0ff673bd6a715fecb4866716d36e380f015f"}, + {file = "multidict-6.0.3-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:1f5d5129a937af4e3c4a1d6c139f4051b7d17d43276cefdd8d442a7031f7eef2"}, + {file = "multidict-6.0.3-cp37-cp37m-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:38d394814b39be1c36ac709006d39d50d72a884f9551acd9c8cc1ffae3fc8c4e"}, + {file = "multidict-6.0.3-cp37-cp37m-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:99341ca1f1db9e7f47914cb2461305665a662383765ced6f843712564766956d"}, + {file = "multidict-6.0.3-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:c5790cc603456b6dcf8a9a4765f666895a6afddc88b3d3ba7b53dea2b6e23116"}, + {file = "multidict-6.0.3-cp37-cp37m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:ce8e51774eb03844588d3c279adb94efcd0edeccd2f97516623292445bcc01f9"}, + {file = "multidict-6.0.3-cp37-cp37m-musllinux_1_1_aarch64.whl", hash = "sha256:baa96a3418e27d723064854143b2f414a422c84cc87285a71558722049bebc5a"}, + {file = "multidict-6.0.3-cp37-cp37m-musllinux_1_1_i686.whl", hash = "sha256:cb4a08f0aaaa869f189ffea0e17b86ad0237b51116d494da15ef7991ee6ad2d7"}, + {file = "multidict-6.0.3-cp37-cp37m-musllinux_1_1_ppc64le.whl", hash = "sha256:62db44727d0befea68e8ad2881bb87a9cfb6b87d45dd78609009627167f37b69"}, + {file = "multidict-6.0.3-cp37-cp37m-musllinux_1_1_s390x.whl", hash = "sha256:4cc5c8cd205a9810d16a5cd428cd81bac554ad1477cb87f4ad722b10992e794d"}, + {file = "multidict-6.0.3-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:f76109387e1ec8d8e2137c94c437b89fe002f29e0881aae8ae45529bdff92000"}, + {file = "multidict-6.0.3-cp37-cp37m-win32.whl", hash = "sha256:f8a728511c977df6f3d8af388fcb157e49f11db4a6637dd60131b8b6e40b0253"}, + {file = "multidict-6.0.3-cp37-cp37m-win_amd64.whl", hash = "sha256:c2a1168e5aa7c72499fb03c850e0f03f624fa4a5c8d2e215c518d0a73872eb64"}, + {file = "multidict-6.0.3-cp38-cp38-macosx_10_9_universal2.whl", hash = "sha256:eddf604a3de2ace3d9a4e4d491be7562a1ac095a0a1c95a9ec5781ef0273ef11"}, + {file = "multidict-6.0.3-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:d09daf5c6ce7fc6ed444c9339bbde5ea84e2534d1ca1cd37b60f365c77f00dea"}, + {file = "multidict-6.0.3-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:12e0d396faa6dc55ff5379eee54d1df3b508243ff15bfc8295a6ec7a4483a335"}, + {file = "multidict-6.0.3-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:70740c2bc9ab1c99f7cdcb104f27d16c63860c56d51c5bf0ef82fc1d892a2131"}, + {file = "multidict-6.0.3-cp38-cp38-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:e322c94596054352f5a02771eec71563c018b15699b961aba14d6dd943367022"}, + {file = "multidict-6.0.3-cp38-cp38-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:4159fc1ec9ede8ab93382e0d6ba9b1b3d23c72da39a834db7a116986605c7ab4"}, + {file = "multidict-6.0.3-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:47defc0218682281a52fb1f6346ebb8b68b17538163a89ea24dfe4da37a8a9a3"}, + {file = "multidict-6.0.3-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:7f9511e48bde6b995825e8d35e434fc96296cf07a25f4aae24ff9162be7eaa46"}, + {file = "multidict-6.0.3-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:e0bce9f7c30e7e3a9e683f670314c0144e8d34be6b7019e40604763bd278d84f"}, + {file = "multidict-6.0.3-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:01b456046a05ff7cceefb0e1d2a9d32f05efcb1c7e0d152446304e11557639ce"}, + {file = "multidict-6.0.3-cp38-cp38-musllinux_1_1_ppc64le.whl", hash = "sha256:8230a39bae6c2e8a09e4da6bace5064693b00590a4a213e38f9a9366da10e7dd"}, + {file = "multidict-6.0.3-cp38-cp38-musllinux_1_1_s390x.whl", hash = "sha256:445c0851a1cbc1f2ec3b40bc22f9c4a235edb3c9a0906122a9df6ea8d51f886c"}, + {file = "multidict-6.0.3-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:9aac6881454a750554ed4b280a839dcf9e2133a9d12ab4d417d673fb102289b7"}, + {file = "multidict-6.0.3-cp38-cp38-win32.whl", hash = "sha256:81c3d597591b0940e04949e4e4f79359b2d2e542a686ba0da5e25de33fec13e0"}, + {file = "multidict-6.0.3-cp38-cp38-win_amd64.whl", hash = "sha256:dc4cfef5d899f5f1a15f3d2ac49f71107a01a5a2745b4dd53fa0cede1419385a"}, + {file = "multidict-6.0.3-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:d408172519049e36fb6d29672f060dc8461fc7174eba9883c7026041ef9bfb38"}, + {file = "multidict-6.0.3-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:e068dfeadbce63072b2d8096486713d04db4946aad0a0f849bd4fc300799d0d3"}, + {file = "multidict-6.0.3-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:a8b817d4ed68fd568ec5e45dd75ddf30cc72a47a6b41b74d5bb211374c296f5e"}, + {file = "multidict-6.0.3-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:2cf5d19e12eff855aa198259c0b02fd3f5d07e1291fbd20279c37b3b0e6c9852"}, + {file = "multidict-6.0.3-cp39-cp39-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:e5a811aab1b4aea0b4be669363c19847a8c547510f0e18fb632956369fdbdf67"}, + {file = "multidict-6.0.3-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:2cfda34b7cb99eacada2072e0f69c0ad3285cb6f8e480b11f2b6d6c1c6f92718"}, + {file = "multidict-6.0.3-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:beeca903e4270b4afcd114f371a9602240dc143f9e944edfea00f8d4ad56c40d"}, + {file = "multidict-6.0.3-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:cd5771e8ea325f85cbb361ddbdeb9ae424a68e5dfb6eea786afdcd22e68a7d5d"}, + {file = "multidict-6.0.3-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:9dbab2a7e9c073bc9538824a01f5ed689194db7f55f2b8102766873e906a6c1a"}, + {file = "multidict-6.0.3-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:f2c0957b3e8c66c10d27272709a5299ab3670a0f187c9428f3b90d267119aedb"}, + {file = "multidict-6.0.3-cp39-cp39-musllinux_1_1_ppc64le.whl", hash = "sha256:94cbe5535ef150546b8321aebea22862a3284da51e7b55f6f95b7d73e96d90ee"}, + {file = "multidict-6.0.3-cp39-cp39-musllinux_1_1_s390x.whl", hash = "sha256:d0e798b072cf2aab9daceb43d97c9c527a0c7593e67a7846ad4cc6051de1e303"}, + {file = "multidict-6.0.3-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:a27b029caa3b555a4f3da54bc1e718eb55fcf1a11fda8bf0132147b476cf4c08"}, + {file = "multidict-6.0.3-cp39-cp39-win32.whl", hash = "sha256:018c8e3be7f161a12b3e41741b6721f9baeb2210f4ab25a6359b7d76c1017dce"}, + {file = "multidict-6.0.3-cp39-cp39-win_amd64.whl", hash = "sha256:5e58ec0375803526d395f6f7e730ecc45d06e15f68f7b9cdbf644a2918324e51"}, + {file = "multidict-6.0.3.tar.gz", hash = "sha256:2523a29006c034687eccd3ee70093a697129a3ffe8732535d3b2df6a4ecc279d"}, +] +nodeenv = [ + {file = "nodeenv-1.7.0-py2.py3-none-any.whl", hash = "sha256:27083a7b96a25f2f5e1d8cb4b6317ee8aeda3bdd121394e5ac54e498028a042e"}, + {file = "nodeenv-1.7.0.tar.gz", hash = "sha256:e0e7f7dfb85fc5394c6fe1e8fa98131a2473e04311a45afb6508f7cf1836fa2b"}, +] +numpy = [ + {file = "numpy-1.23.5-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:9c88793f78fca17da0145455f0d7826bcb9f37da4764af27ac945488116efe63"}, + {file = "numpy-1.23.5-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:e9f4c4e51567b616be64e05d517c79a8a22f3606499941d97bb76f2ca59f982d"}, + {file = "numpy-1.23.5-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:7903ba8ab592b82014713c491f6c5d3a1cde5b4a3bf116404e08f5b52f6daf43"}, + {file = "numpy-1.23.5-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:5e05b1c973a9f858c74367553e236f287e749465f773328c8ef31abe18f691e1"}, + {file = "numpy-1.23.5-cp310-cp310-win32.whl", hash = "sha256:522e26bbf6377e4d76403826ed689c295b0b238f46c28a7251ab94716da0b280"}, + {file = "numpy-1.23.5-cp310-cp310-win_amd64.whl", hash = "sha256:dbee87b469018961d1ad79b1a5d50c0ae850000b639bcb1b694e9981083243b6"}, + {file = "numpy-1.23.5-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:ce571367b6dfe60af04e04a1834ca2dc5f46004ac1cc756fb95319f64c095a96"}, + {file = "numpy-1.23.5-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:56e454c7833e94ec9769fa0f86e6ff8e42ee38ce0ce1fa4cbb747ea7e06d56aa"}, + {file = "numpy-1.23.5-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:5039f55555e1eab31124a5768898c9e22c25a65c1e0037f4d7c495a45778c9f2"}, + {file = "numpy-1.23.5-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:58f545efd1108e647604a1b5aa809591ccd2540f468a880bedb97247e72db387"}, + {file = "numpy-1.23.5-cp311-cp311-win32.whl", hash = "sha256:b2a9ab7c279c91974f756c84c365a669a887efa287365a8e2c418f8b3ba73fb0"}, + {file = "numpy-1.23.5-cp311-cp311-win_amd64.whl", hash = "sha256:0cbe9848fad08baf71de1a39e12d1b6310f1d5b2d0ea4de051058e6e1076852d"}, + {file = "numpy-1.23.5-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:f063b69b090c9d918f9df0a12116029e274daf0181df392839661c4c7ec9018a"}, + {file = "numpy-1.23.5-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:0aaee12d8883552fadfc41e96b4c82ee7d794949e2a7c3b3a7201e968c7ecab9"}, + {file = "numpy-1.23.5-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:92c8c1e89a1f5028a4c6d9e3ccbe311b6ba53694811269b992c0b224269e2398"}, + {file = "numpy-1.23.5-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:d208a0f8729f3fb790ed18a003f3a57895b989b40ea4dce4717e9cf4af62c6bb"}, + {file = "numpy-1.23.5-cp38-cp38-win32.whl", hash = "sha256:06005a2ef6014e9956c09ba07654f9837d9e26696a0470e42beedadb78c11b07"}, + {file = "numpy-1.23.5-cp38-cp38-win_amd64.whl", hash = "sha256:ca51fcfcc5f9354c45f400059e88bc09215fb71a48d3768fb80e357f3b457e1e"}, + {file = "numpy-1.23.5-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:8969bfd28e85c81f3f94eb4a66bc2cf1dbdc5c18efc320af34bffc54d6b1e38f"}, + {file = "numpy-1.23.5-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:a7ac231a08bb37f852849bbb387a20a57574a97cfc7b6cabb488a4fc8be176de"}, + {file = "numpy-1.23.5-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:bf837dc63ba5c06dc8797c398db1e223a466c7ece27a1f7b5232ba3466aafe3d"}, + {file = "numpy-1.23.5-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:33161613d2269025873025b33e879825ec7b1d831317e68f4f2f0f84ed14c719"}, + {file = "numpy-1.23.5-cp39-cp39-win32.whl", hash = "sha256:af1da88f6bc3d2338ebbf0e22fe487821ea4d8e89053e25fa59d1d79786e7481"}, + {file = "numpy-1.23.5-cp39-cp39-win_amd64.whl", hash = "sha256:09b7847f7e83ca37c6e627682f145856de331049013853f344f37b0c9690e3df"}, + {file = "numpy-1.23.5-pp38-pypy38_pp73-macosx_10_9_x86_64.whl", hash = "sha256:abdde9f795cf292fb9651ed48185503a2ff29be87770c3b8e2a14b0cd7aa16f8"}, + {file = "numpy-1.23.5-pp38-pypy38_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:f9a909a8bae284d46bbfdefbdd4a262ba19d3bc9921b1e76126b1d21c3c34135"}, + {file = "numpy-1.23.5-pp38-pypy38_pp73-win_amd64.whl", hash = "sha256:01dd17cbb340bf0fc23981e52e1d18a9d4050792e8fb8363cecbf066a84b827d"}, + {file = "numpy-1.23.5.tar.gz", hash = "sha256:1b1766d6f397c18153d40015ddfc79ddb715cabadc04d2d228d4e5a8bc4ded1a"}, +] +oauthlib = [ + {file = "oauthlib-3.2.2-py3-none-any.whl", hash = "sha256:8139f29aac13e25d502680e9e19963e83f16838d48a0d71c287fe40e7067fbca"}, + {file = "oauthlib-3.2.2.tar.gz", hash = "sha256:9859c40929662bec5d64f34d01c99e093149682a3f38915dc0655d5a633dd918"}, +] +packaging = [ + {file = "packaging-22.0-py3-none-any.whl", hash = "sha256:957e2148ba0e1a3b282772e791ef1d8083648bc131c8ab0c1feba110ce1146c3"}, + {file = "packaging-22.0.tar.gz", hash = "sha256:2198ec20bd4c017b8f9717e00f0c8714076fc2fd93816750ab48e2c41de2cfd3"}, +] +pandas = [ + {file = "pandas-1.5.2-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:e9dbacd22555c2d47f262ef96bb4e30880e5956169741400af8b306bbb24a273"}, + {file = "pandas-1.5.2-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:e2b83abd292194f350bb04e188f9379d36b8dfac24dd445d5c87575f3beaf789"}, + {file = "pandas-1.5.2-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:2552bffc808641c6eb471e55aa6899fa002ac94e4eebfa9ec058649122db5824"}, + {file = "pandas-1.5.2-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:1fc87eac0541a7d24648a001d553406f4256e744d92df1df8ebe41829a915028"}, + {file = "pandas-1.5.2-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:d0d8fd58df5d17ddb8c72a5075d87cd80d71b542571b5f78178fb067fa4e9c72"}, + {file = "pandas-1.5.2-cp310-cp310-win_amd64.whl", hash = "sha256:4aed257c7484d01c9a194d9a94758b37d3d751849c05a0050c087a358c41ad1f"}, + {file = "pandas-1.5.2-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:375262829c8c700c3e7cbb336810b94367b9c4889818bbd910d0ecb4e45dc261"}, + {file = "pandas-1.5.2-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:cc3cd122bea268998b79adebbb8343b735a5511ec14efb70a39e7acbc11ccbdc"}, + {file = "pandas-1.5.2-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:b4f5a82afa4f1ff482ab8ded2ae8a453a2cdfde2001567b3ca24a4c5c5ca0db3"}, + {file = "pandas-1.5.2-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:8092a368d3eb7116e270525329a3e5c15ae796ccdf7ccb17839a73b4f5084a39"}, + {file = "pandas-1.5.2-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:f6257b314fc14958f8122779e5a1557517b0f8e500cfb2bd53fa1f75a8ad0af2"}, + {file = "pandas-1.5.2-cp311-cp311-win_amd64.whl", hash = "sha256:82ae615826da838a8e5d4d630eb70c993ab8636f0eff13cb28aafc4291b632b5"}, + {file = "pandas-1.5.2-cp38-cp38-macosx_10_9_universal2.whl", hash = "sha256:457d8c3d42314ff47cc2d6c54f8fc0d23954b47977b2caed09cd9635cb75388b"}, + {file = "pandas-1.5.2-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:c009a92e81ce836212ce7aa98b219db7961a8b95999b97af566b8dc8c33e9519"}, + {file = "pandas-1.5.2-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:71f510b0efe1629bf2f7c0eadb1ff0b9cf611e87b73cd017e6b7d6adb40e2b3a"}, + {file = "pandas-1.5.2-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a40dd1e9f22e01e66ed534d6a965eb99546b41d4d52dbdb66565608fde48203f"}, + {file = "pandas-1.5.2-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:5ae7e989f12628f41e804847a8cc2943d362440132919a69429d4dea1f164da0"}, + {file = "pandas-1.5.2-cp38-cp38-win32.whl", hash = "sha256:530948945e7b6c95e6fa7aa4be2be25764af53fba93fe76d912e35d1c9ee46f5"}, + {file = "pandas-1.5.2-cp38-cp38-win_amd64.whl", hash = "sha256:73f219fdc1777cf3c45fde7f0708732ec6950dfc598afc50588d0d285fddaefc"}, + {file = "pandas-1.5.2-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:9608000a5a45f663be6af5c70c3cbe634fa19243e720eb380c0d378666bc7702"}, + {file = "pandas-1.5.2-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:315e19a3e5c2ab47a67467fc0362cb36c7c60a93b6457f675d7d9615edad2ebe"}, + {file = "pandas-1.5.2-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:e18bc3764cbb5e118be139b3b611bc3fbc5d3be42a7e827d1096f46087b395eb"}, + {file = "pandas-1.5.2-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:0183cb04a057cc38fde5244909fca9826d5d57c4a5b7390c0cc3fa7acd9fa883"}, + {file = "pandas-1.5.2-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:344021ed3e639e017b452aa8f5f6bf38a8806f5852e217a7594417fb9bbfa00e"}, + {file = "pandas-1.5.2-cp39-cp39-win32.whl", hash = "sha256:e7469271497960b6a781eaa930cba8af400dd59b62ec9ca2f4d31a19f2f91090"}, + {file = "pandas-1.5.2-cp39-cp39-win_amd64.whl", hash = "sha256:c218796d59d5abd8780170c937b812c9637e84c32f8271bbf9845970f8c1351f"}, + {file = "pandas-1.5.2.tar.gz", hash = "sha256:220b98d15cee0b2cd839a6358bd1f273d0356bf964c1a1aeb32d47db0215488b"}, +] +pep517 = [ + {file = "pep517-0.13.0-py3-none-any.whl", hash = "sha256:4ba4446d80aed5b5eac6509ade100bff3e7943a8489de249654a5ae9b33ee35b"}, + {file = "pep517-0.13.0.tar.gz", hash = "sha256:ae69927c5c172be1add9203726d4b84cf3ebad1edcd5f71fcdc746e66e829f59"}, +] +platformdirs = [ + {file = "platformdirs-2.6.0-py3-none-any.whl", hash = "sha256:1a89a12377800c81983db6be069ec068eee989748799b946cce2a6e80dcc54ca"}, + {file = "platformdirs-2.6.0.tar.gz", hash = "sha256:b46ffafa316e6b83b47489d240ce17173f123a9b9c83282141c3daf26ad9ac2e"}, +] +pluggy = [ + {file = "pluggy-1.0.0-py2.py3-none-any.whl", hash = "sha256:74134bbf457f031a36d68416e1509f34bd5ccc019f0bcc952c7b909d06b37bd3"}, + {file = "pluggy-1.0.0.tar.gz", hash = "sha256:4224373bacce55f955a878bf9cfa763c1e360858e330072059e10bad68531159"}, +] +portalocker = [ + {file = "portalocker-2.6.0-py2.py3-none-any.whl", hash = "sha256:102ed1f2badd8dec9af3d732ef70e94b215b85ba45a8d7ff3c0003f19b442f4e"}, + {file = "portalocker-2.6.0.tar.gz", hash = "sha256:964f6830fb42a74b5d32bce99ed37d8308c1d7d44ddf18f3dd89f4680de97b39"}, +] +pre-commit = [ + {file = "pre_commit-2.20.0-py2.py3-none-any.whl", hash = "sha256:51a5ba7c480ae8072ecdb6933df22d2f812dc897d5fe848778116129a681aac7"}, + {file = "pre_commit-2.20.0.tar.gz", hash = "sha256:a978dac7bc9ec0bcee55c18a277d553b0f419d259dadb4b9418ff2d00eb43959"}, +] +pyarrow = [ + {file = "pyarrow-10.0.1-cp310-cp310-macosx_10_14_x86_64.whl", hash = "sha256:e00174764a8b4e9d8d5909b6d19ee0c217a6cf0232c5682e31fdfbd5a9f0ae52"}, + {file = "pyarrow-10.0.1-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:6f7a7dbe2f7f65ac1d0bd3163f756deb478a9e9afc2269557ed75b1b25ab3610"}, + {file = "pyarrow-10.0.1-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:cb627673cb98708ef00864e2e243f51ba7b4c1b9f07a1d821f98043eccd3f585"}, + {file = "pyarrow-10.0.1-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:ba71e6fc348c92477586424566110d332f60d9a35cb85278f42e3473bc1373da"}, + {file = "pyarrow-10.0.1-cp310-cp310-win_amd64.whl", hash = "sha256:7b4ede715c004b6fc535de63ef79fa29740b4080639a5ff1ea9ca84e9282f349"}, + {file = "pyarrow-10.0.1-cp311-cp311-macosx_10_14_x86_64.whl", hash = "sha256:e3fe5049d2e9ca661d8e43fab6ad5a4c571af12d20a57dffc392a014caebef65"}, + {file = "pyarrow-10.0.1-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:254017ca43c45c5098b7f2a00e995e1f8346b0fb0be225f042838323bb55283c"}, + {file = "pyarrow-10.0.1-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:70acca1ece4322705652f48db65145b5028f2c01c7e426c5d16a30ba5d739c24"}, + {file = "pyarrow-10.0.1-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:abb57334f2c57979a49b7be2792c31c23430ca02d24becd0b511cbe7b6b08649"}, + {file = "pyarrow-10.0.1-cp311-cp311-win_amd64.whl", hash = "sha256:1765a18205eb1e02ccdedb66049b0ec148c2a0cb52ed1fb3aac322dfc086a6ee"}, + {file = "pyarrow-10.0.1-cp37-cp37m-macosx_10_14_x86_64.whl", hash = "sha256:61f4c37d82fe00d855d0ab522c685262bdeafd3fbcb5fe596fe15025fbc7341b"}, + {file = "pyarrow-10.0.1-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:e141a65705ac98fa52a9113fe574fdaf87fe0316cde2dffe6b94841d3c61544c"}, + {file = "pyarrow-10.0.1-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:bf26f809926a9d74e02d76593026f0aaeac48a65b64f1bb17eed9964bfe7ae1a"}, + {file = "pyarrow-10.0.1-cp37-cp37m-win_amd64.whl", hash = "sha256:443eb9409b0cf78df10ced326490e1a300205a458fbeb0767b6b31ab3ebae6b2"}, + {file = "pyarrow-10.0.1-cp38-cp38-macosx_10_14_x86_64.whl", hash = "sha256:f2d00aa481becf57098e85d99e34a25dba5a9ade2f44eb0b7d80c80f2984fc03"}, + {file = "pyarrow-10.0.1-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:b1fc226d28c7783b52a84d03a66573d5a22e63f8a24b841d5fc68caeed6784d4"}, + {file = "pyarrow-10.0.1-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:efa59933b20183c1c13efc34bd91efc6b2997377c4c6ad9272da92d224e3beb1"}, + {file = "pyarrow-10.0.1-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:668e00e3b19f183394388a687d29c443eb000fb3fe25599c9b4762a0afd37775"}, + {file = "pyarrow-10.0.1-cp38-cp38-win_amd64.whl", hash = "sha256:d1bc6e4d5d6f69e0861d5d7f6cf4d061cf1069cb9d490040129877acf16d4c2a"}, + {file = "pyarrow-10.0.1-cp39-cp39-macosx_10_14_x86_64.whl", hash = "sha256:42ba7c5347ce665338f2bc64685d74855900200dac81a972d49fe127e8132f75"}, + {file = "pyarrow-10.0.1-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:b069602eb1fc09f1adec0a7bdd7897f4d25575611dfa43543c8b8a75d99d6874"}, + {file = "pyarrow-10.0.1-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:94fb4a0c12a2ac1ed8e7e2aa52aade833772cf2d3de9dde685401b22cec30002"}, + {file = "pyarrow-10.0.1-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:db0c5986bf0808927f49640582d2032a07aa49828f14e51f362075f03747d198"}, + {file = "pyarrow-10.0.1-cp39-cp39-win_amd64.whl", hash = "sha256:0ec7587d759153f452d5263dbc8b1af318c4609b607be2bd5127dcda6708cdb1"}, + {file = "pyarrow-10.0.1.tar.gz", hash = "sha256:1a14f57a5f472ce8234f2964cd5184cccaa8df7e04568c64edc33b23eb285dd5"}, +] +pycparser = [ + {file = "pycparser-2.21-py2.py3-none-any.whl", hash = "sha256:8ee45429555515e1f6b185e78100aea234072576aa43ab53aefcae078162fca9"}, + {file = "pycparser-2.21.tar.gz", hash = "sha256:e644fdec12f7872f86c58ff790da456218b10f863970249516d60a5eaca77206"}, +] +pydantic = [ + {file = "pydantic-1.10.2-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:bb6ad4489af1bac6955d38ebcb95079a836af31e4c4f74aba1ca05bb9f6027bd"}, + {file = "pydantic-1.10.2-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:a1f5a63a6dfe19d719b1b6e6106561869d2efaca6167f84f5ab9347887d78b98"}, + {file = "pydantic-1.10.2-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:352aedb1d71b8b0736c6d56ad2bd34c6982720644b0624462059ab29bd6e5912"}, + {file = "pydantic-1.10.2-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:19b3b9ccf97af2b7519c42032441a891a5e05c68368f40865a90eb88833c2559"}, + {file = "pydantic-1.10.2-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:e9069e1b01525a96e6ff49e25876d90d5a563bc31c658289a8772ae186552236"}, + {file = "pydantic-1.10.2-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:355639d9afc76bcb9b0c3000ddcd08472ae75318a6eb67a15866b87e2efa168c"}, + {file = "pydantic-1.10.2-cp310-cp310-win_amd64.whl", hash = "sha256:ae544c47bec47a86bc7d350f965d8b15540e27e5aa4f55170ac6a75e5f73b644"}, + {file = "pydantic-1.10.2-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:a4c805731c33a8db4b6ace45ce440c4ef5336e712508b4d9e1aafa617dc9907f"}, + {file = "pydantic-1.10.2-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:d49f3db871575e0426b12e2f32fdb25e579dea16486a26e5a0474af87cb1ab0a"}, + {file = "pydantic-1.10.2-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:37c90345ec7dd2f1bcef82ce49b6235b40f282b94d3eec47e801baf864d15525"}, + {file = "pydantic-1.10.2-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:7b5ba54d026c2bd2cb769d3468885f23f43710f651688e91f5fb1edcf0ee9283"}, + {file = "pydantic-1.10.2-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:05e00dbebbe810b33c7a7362f231893183bcc4251f3f2ff991c31d5c08240c42"}, + {file = "pydantic-1.10.2-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:2d0567e60eb01bccda3a4df01df677adf6b437958d35c12a3ac3e0f078b0ee52"}, + {file = "pydantic-1.10.2-cp311-cp311-win_amd64.whl", hash = "sha256:c6f981882aea41e021f72779ce2a4e87267458cc4d39ea990729e21ef18f0f8c"}, + {file = "pydantic-1.10.2-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:c4aac8e7103bf598373208f6299fa9a5cfd1fc571f2d40bf1dd1955a63d6eeb5"}, + {file = "pydantic-1.10.2-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:81a7b66c3f499108b448f3f004801fcd7d7165fb4200acb03f1c2402da73ce4c"}, + {file = "pydantic-1.10.2-cp37-cp37m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:bedf309630209e78582ffacda64a21f96f3ed2e51fbf3962d4d488e503420254"}, + {file = "pydantic-1.10.2-cp37-cp37m-musllinux_1_1_i686.whl", hash = "sha256:9300fcbebf85f6339a02c6994b2eb3ff1b9c8c14f502058b5bf349d42447dcf5"}, + {file = "pydantic-1.10.2-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:216f3bcbf19c726b1cc22b099dd409aa371f55c08800bcea4c44c8f74b73478d"}, + {file = "pydantic-1.10.2-cp37-cp37m-win_amd64.whl", hash = "sha256:dd3f9a40c16daf323cf913593083698caee97df2804aa36c4b3175d5ac1b92a2"}, + {file = "pydantic-1.10.2-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:b97890e56a694486f772d36efd2ba31612739bc6f3caeee50e9e7e3ebd2fdd13"}, + {file = "pydantic-1.10.2-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:9cabf4a7f05a776e7793e72793cd92cc865ea0e83a819f9ae4ecccb1b8aa6116"}, + {file = "pydantic-1.10.2-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:06094d18dd5e6f2bbf93efa54991c3240964bb663b87729ac340eb5014310624"}, + {file = "pydantic-1.10.2-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:cc78cc83110d2f275ec1970e7a831f4e371ee92405332ebfe9860a715f8336e1"}, + {file = "pydantic-1.10.2-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:1ee433e274268a4b0c8fde7ad9d58ecba12b069a033ecc4645bb6303c062d2e9"}, + {file = "pydantic-1.10.2-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:7c2abc4393dea97a4ccbb4ec7d8658d4e22c4765b7b9b9445588f16c71ad9965"}, + {file = "pydantic-1.10.2-cp38-cp38-win_amd64.whl", hash = "sha256:0b959f4d8211fc964772b595ebb25f7652da3f22322c007b6fed26846a40685e"}, + {file = "pydantic-1.10.2-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:c33602f93bfb67779f9c507e4d69451664524389546bacfe1bee13cae6dc7488"}, + {file = "pydantic-1.10.2-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:5760e164b807a48a8f25f8aa1a6d857e6ce62e7ec83ea5d5c5a802eac81bad41"}, + {file = "pydantic-1.10.2-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:6eb843dcc411b6a2237a694f5e1d649fc66c6064d02b204a7e9d194dff81eb4b"}, + {file = "pydantic-1.10.2-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:4b8795290deaae348c4eba0cebb196e1c6b98bdbe7f50b2d0d9a4a99716342fe"}, + {file = "pydantic-1.10.2-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:e0bedafe4bc165ad0a56ac0bd7695df25c50f76961da29c050712596cf092d6d"}, + {file = "pydantic-1.10.2-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:2e05aed07fa02231dbf03d0adb1be1d79cabb09025dd45aa094aa8b4e7b9dcda"}, + {file = "pydantic-1.10.2-cp39-cp39-win_amd64.whl", hash = "sha256:c1ba1afb396148bbc70e9eaa8c06c1716fdddabaf86e7027c5988bae2a829ab6"}, + {file = "pydantic-1.10.2-py3-none-any.whl", hash = "sha256:1b6ee725bd6e83ec78b1aa32c5b1fa67a3a65badddde3976bca5fe4568f27709"}, + {file = "pydantic-1.10.2.tar.gz", hash = "sha256:91b8e218852ef6007c2b98cd861601c6a09f1aa32bbbb74fab5b1c33d4a1e410"}, +] +pygments = [ + {file = "Pygments-2.13.0-py3-none-any.whl", hash = "sha256:f643f331ab57ba3c9d89212ee4a2dabc6e94f117cf4eefde99a0574720d14c42"}, + {file = "Pygments-2.13.0.tar.gz", hash = "sha256:56a8508ae95f98e2b9bdf93a6be5ae3f7d8af858b43e02c5a2ff083726be40c1"}, +] +pyjwt = [ + {file = "PyJWT-2.6.0-py3-none-any.whl", hash = "sha256:d83c3d892a77bbb74d3e1a2cfa90afaadb60945205d1095d9221f04466f64c14"}, + {file = "PyJWT-2.6.0.tar.gz", hash = "sha256:69285c7e31fc44f68a1feb309e948e0df53259d579295e6cfe2b1792329f05fd"}, +] +pyparsing = [ + {file = "pyparsing-3.0.9-py3-none-any.whl", hash = "sha256:5026bae9a10eeaefb61dab2f09052b9f4307d44aee4eda64b309723d8d206bbc"}, + {file = "pyparsing-3.0.9.tar.gz", hash = "sha256:2b020ecf7d21b687f219b71ecad3631f644a47f01403fa1d1036b0c6416d70fb"}, +] +pytest = [ + {file = "pytest-7.2.0-py3-none-any.whl", hash = "sha256:892f933d339f068883b6fd5a459f03d85bfcb355e4981e146d2c7616c21fef71"}, + {file = "pytest-7.2.0.tar.gz", hash = "sha256:c4014eb40e10f11f355ad4e3c2fb2c6c6d1919c73f3b5a433de4708202cade59"}, +] +pytest-checkdocs = [ + {file = "pytest-checkdocs-2.9.0.tar.gz", hash = "sha256:54b3d1357a289e3f6336d31d4ceb72e99afe0fa4f552990f1b1f015f1925543b"}, + {file = "pytest_checkdocs-2.9.0-py3-none-any.whl", hash = "sha256:4231e5426915a02385e386b5eb4a70490b143d97cede4c7c77691bb320b5a73f"}, +] +python-dateutil = [ + {file = "python-dateutil-2.8.2.tar.gz", hash = "sha256:0123cacc1627ae19ddf3c27a5de5bd67ee4586fbdd6440d9748f8abb483d3e86"}, + {file = "python_dateutil-2.8.2-py2.py3-none-any.whl", hash = "sha256:961d03dc3453ebbc59dbdea9e4e11c5651520a876d0f4db161e8674aae935da9"}, +] +python-snappy = [ + {file = "python-snappy-0.6.1.tar.gz", hash = "sha256:b6a107ab06206acc5359d4c5632bd9b22d448702a79b3169b0c62e0fb808bb2a"}, + {file = "python_snappy-0.6.1-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:b7f920eaf46ebf41bd26f9df51c160d40f9e00b7b48471c3438cb8d027f7fb9b"}, + {file = "python_snappy-0.6.1-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:4ec533a8c1f8df797bded662ec3e494d225b37855bb63eb0d75464a07947477c"}, + {file = "python_snappy-0.6.1-cp310-cp310-manylinux_2_12_i686.manylinux2010_i686.whl", hash = "sha256:6f8bf4708a11b47517baf962f9a02196478bbb10fdb9582add4aa1459fa82380"}, + {file = "python_snappy-0.6.1-cp310-cp310-manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:8d0c019ee7dcf2c60e240877107cddbd95a5b1081787579bf179938392d66480"}, + {file = "python_snappy-0.6.1-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:cb18d9cd7b3f35a2f5af47bb8ed6a5bdbf4f3ddee37f3daade4ab7864c292f5b"}, + {file = "python_snappy-0.6.1-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:b265cde49774752aec9ca7f5d272e3f98718164afc85521622a8a5394158a2b5"}, + {file = "python_snappy-0.6.1-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:d017775851a778ec9cc32651c4464079d06d927303c2dde9ae9830ccf6fe94e1"}, + {file = "python_snappy-0.6.1-cp310-cp310-win32.whl", hash = "sha256:8277d1f6282463c40761f802b742f833f9f2449fcdbb20a96579aa05c8feb614"}, + {file = "python_snappy-0.6.1-cp310-cp310-win_amd64.whl", hash = "sha256:2aaaf618c68d8c9daebc23a20436bd01b09ee70d7fbf7072b7f38b06d2fab539"}, + {file = "python_snappy-0.6.1-cp36-cp36m-macosx_10_9_x86_64.whl", hash = "sha256:277757d5dad4e239dc1417438a0871b65b1b155beb108888e7438c27ffc6a8cc"}, + {file = "python_snappy-0.6.1-cp36-cp36m-manylinux_2_12_i686.manylinux2010_i686.whl", hash = "sha256:e066a0586833d610c4bbddba0be5ba0e3e4f8e0bc5bb6d82103d8f8fc47bb59a"}, + {file = "python_snappy-0.6.1-cp36-cp36m-manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:0d489b50f49433494160c45048fe806de6b3aeab0586e497ebd22a0bab56e427"}, + {file = "python_snappy-0.6.1-cp36-cp36m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:463fd340a499d47b26ca42d2f36a639188738f6e2098c6dbf80aef0e60f461e1"}, + {file = "python_snappy-0.6.1-cp36-cp36m-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:9837ac1650cc68d22a3cf5f15fb62c6964747d16cecc8b22431f113d6e39555d"}, + {file = "python_snappy-0.6.1-cp36-cp36m-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:5e973e637112391f05581f427659c05b30b6843bc522a65be35ac7b18ce3dedd"}, + {file = "python_snappy-0.6.1-cp36-cp36m-win32.whl", hash = "sha256:c20498bd712b6e31a4402e1d027a1cd64f6a4a0066a3fe3c7344475886d07fdf"}, + {file = "python_snappy-0.6.1-cp36-cp36m-win_amd64.whl", hash = "sha256:59e975be4206cc54d0a112ef72fa3970a57c2b1bcc2c97ed41d6df0ebe518228"}, + {file = "python_snappy-0.6.1-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:2a7e528ab6e09c0d67dcb61a1730a292683e5ff9bb088950638d3170cf2a0a54"}, + {file = "python_snappy-0.6.1-cp37-cp37m-manylinux_2_12_i686.manylinux2010_i686.whl", hash = "sha256:39692bedbe0b717001a99915ac0eb2d9d0bad546440d392a2042b96d813eede1"}, + {file = "python_snappy-0.6.1-cp37-cp37m-manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:6a7620404da966f637b9ce8d4d3d543d363223f7a12452a575189c5355fc2d25"}, + {file = "python_snappy-0.6.1-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:7778c224efc38a40d274da4eb82a04cac27aae20012372a7db3c4bbd8926c4d4"}, + {file = "python_snappy-0.6.1-cp37-cp37m-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:1d029f7051ec1bbeaa3e03030b6d8ed47ceb69cae9016f493c802a08af54e026"}, + {file = "python_snappy-0.6.1-cp37-cp37m-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:a0ad38bc98d0b0497a0b0dbc29409bcabfcecff4511ed7063403c86de16927bc"}, + {file = "python_snappy-0.6.1-cp37-cp37m-win32.whl", hash = "sha256:5a453c45178d7864c1bdd6bfe0ee3ed2883f63b9ba2c9bb967c6b586bf763f96"}, + {file = "python_snappy-0.6.1-cp37-cp37m-win_amd64.whl", hash = "sha256:9f0c0d88b84259f93c3aa46398680646f2c23e43394779758d9f739c34e15295"}, + {file = "python_snappy-0.6.1-cp38-cp38-macosx_10_9_universal2.whl", hash = "sha256:5bb05c28298803a74add08ba496879242ef159c75bc86a5406fac0ffc7dd021b"}, + {file = "python_snappy-0.6.1-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:9eac51307c6a1a38d5f86ebabc26a889fddf20cbba7a116ccb54ba1446601d5b"}, + {file = "python_snappy-0.6.1-cp38-cp38-manylinux_2_12_i686.manylinux2010_i686.whl", hash = "sha256:88b6ea78b83d2796f330b0af1b70cdd3965dbdab02d8ac293260ec2c8fe340ee"}, + {file = "python_snappy-0.6.1-cp38-cp38-manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:8c07220408d3268e8268c9351c5c08041bc6f8c6172e59d398b71020df108541"}, + {file = "python_snappy-0.6.1-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:4038019b1bcaadde726a57430718394076c5a21545ebc5badad2c045a09546cf"}, + {file = "python_snappy-0.6.1-cp38-cp38-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:dc96668d9c7cc656609764275c5f8da58ef56d89bdd6810f6923d36497468ff7"}, + {file = "python_snappy-0.6.1-cp38-cp38-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:cf5bb9254e1c38aacf253d510d3d9be631bba21f3d068b17672b38b5cbf2fff5"}, + {file = "python_snappy-0.6.1-cp38-cp38-win32.whl", hash = "sha256:eaf905a580f2747c4a474040a5063cd5e0cc3d1d2d6edb65f28196186493ad4a"}, + {file = "python_snappy-0.6.1-cp38-cp38-win_amd64.whl", hash = "sha256:546c1a7470ecbf6239101e9aff0f709b68ca0f0268b34d9023019a55baa1f7c6"}, + {file = "python_snappy-0.6.1-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:e3a013895c64352b49d0d8e107a84f99631b16dbab156ded33ebf0becf56c8b2"}, + {file = "python_snappy-0.6.1-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:3fb9a88a4dd6336488f3de67ce75816d0d796dce53c2c6e4d70e0b565633c7fd"}, + {file = "python_snappy-0.6.1-cp39-cp39-manylinux_2_12_i686.manylinux2010_i686.whl", hash = "sha256:735cd4528c55dbe4516d6d2b403331a99fc304f8feded8ae887cf97b67d589bb"}, + {file = "python_snappy-0.6.1-cp39-cp39-manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:90b0186516b7a101c14764b0c25931b741fb0102f21253eff67847b4742dfc72"}, + {file = "python_snappy-0.6.1-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:1a993dc8aadd901915a510fe6af5f20ae4256f527040066c22a154db8946751f"}, + {file = "python_snappy-0.6.1-cp39-cp39-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:530bfb9efebcc1aab8bb4ebcbd92b54477eed11f6cf499355e882970a6d3aa7d"}, + {file = "python_snappy-0.6.1-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:5843feb914796b1f0405ccf31ea0fb51034ceb65a7588edfd5a8250cb369e3b2"}, + {file = "python_snappy-0.6.1-cp39-cp39-win32.whl", hash = "sha256:66c80e9b366012dbee262bb1869e4fc5ba8786cda85928481528bc4a72ec2ee8"}, + {file = "python_snappy-0.6.1-cp39-cp39-win_amd64.whl", hash = "sha256:4d3cafdf454354a621c8ab7408e45aa4e9d5c0b943b61ff4815f71ca6bdf0130"}, + {file = "python_snappy-0.6.1-pp37-pypy37_pp73-manylinux_2_12_i686.manylinux2010_i686.whl", hash = "sha256:586724a0276d7a6083a17259d0b51622e492289a9998848a1b01b6441ca12b2f"}, + {file = "python_snappy-0.6.1-pp37-pypy37_pp73-manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:2be4f4550acd484912441f5f1209ba611ac399aac9355fee73611b9a0d4f949c"}, + {file = "python_snappy-0.6.1-pp37-pypy37_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:0bdb6942180660bda7f7d01f4c0def3cfc72b1c6d99aad964801775a3e379aba"}, + {file = "python_snappy-0.6.1-pp37-pypy37_pp73-win_amd64.whl", hash = "sha256:03bb511380fca2a13325b6f16fe8234c8e12da9660f0258cd45d9a02ffc916af"}, +] +pytz = [ + {file = "pytz-2022.6-py2.py3-none-any.whl", hash = "sha256:222439474e9c98fced559f1709d89e6c9cbf8d79c794ff3eb9f8800064291427"}, + {file = "pytz-2022.6.tar.gz", hash = "sha256:e89512406b793ca39f5971bc999cc538ce125c0e51c27941bef4568b460095e2"}, +] +pywin32 = [ + {file = "pywin32-305-cp310-cp310-win32.whl", hash = "sha256:421f6cd86e84bbb696d54563c48014b12a23ef95a14e0bdba526be756d89f116"}, + {file = "pywin32-305-cp310-cp310-win_amd64.whl", hash = "sha256:73e819c6bed89f44ff1d690498c0a811948f73777e5f97c494c152b850fad478"}, + {file = "pywin32-305-cp310-cp310-win_arm64.whl", hash = "sha256:742eb905ce2187133a29365b428e6c3b9001d79accdc30aa8969afba1d8470f4"}, + {file = "pywin32-305-cp311-cp311-win32.whl", hash = "sha256:19ca459cd2e66c0e2cc9a09d589f71d827f26d47fe4a9d09175f6aa0256b51c2"}, + {file = "pywin32-305-cp311-cp311-win_amd64.whl", hash = "sha256:326f42ab4cfff56e77e3e595aeaf6c216712bbdd91e464d167c6434b28d65990"}, + {file = "pywin32-305-cp311-cp311-win_arm64.whl", hash = "sha256:4ecd404b2c6eceaca52f8b2e3e91b2187850a1ad3f8b746d0796a98b4cea04db"}, + {file = "pywin32-305-cp36-cp36m-win32.whl", hash = "sha256:48d8b1659284f3c17b68587af047d110d8c44837736b8932c034091683e05863"}, + {file = "pywin32-305-cp36-cp36m-win_amd64.whl", hash = "sha256:13362cc5aa93c2beaf489c9c9017c793722aeb56d3e5166dadd5ef82da021fe1"}, + {file = "pywin32-305-cp37-cp37m-win32.whl", hash = "sha256:a55db448124d1c1484df22fa8bbcbc45c64da5e6eae74ab095b9ea62e6d00496"}, + {file = "pywin32-305-cp37-cp37m-win_amd64.whl", hash = "sha256:109f98980bfb27e78f4df8a51a8198e10b0f347257d1e265bb1a32993d0c973d"}, + {file = "pywin32-305-cp38-cp38-win32.whl", hash = "sha256:9dd98384da775afa009bc04863426cb30596fd78c6f8e4e2e5bbf4edf8029504"}, + {file = "pywin32-305-cp38-cp38-win_amd64.whl", hash = "sha256:56d7a9c6e1a6835f521788f53b5af7912090674bb84ef5611663ee1595860fc7"}, + {file = "pywin32-305-cp39-cp39-win32.whl", hash = "sha256:9d968c677ac4d5cbdaa62fd3014ab241718e619d8e36ef8e11fb930515a1e918"}, + {file = "pywin32-305-cp39-cp39-win_amd64.whl", hash = "sha256:50768c6b7c3f0b38b7fb14dd4104da93ebced5f1a50dc0e834594bff6fbe1271"}, +] +pyyaml = [ + {file = "PyYAML-6.0-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:d4db7c7aef085872ef65a8fd7d6d09a14ae91f691dec3e87ee5ee0539d516f53"}, + {file = "PyYAML-6.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:9df7ed3b3d2e0ecfe09e14741b857df43adb5a3ddadc919a2d94fbdf78fea53c"}, + {file = "PyYAML-6.0-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:77f396e6ef4c73fdc33a9157446466f1cff553d979bd00ecb64385760c6babdc"}, + {file = "PyYAML-6.0-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:a80a78046a72361de73f8f395f1f1e49f956c6be882eed58505a15f3e430962b"}, + {file = "PyYAML-6.0-cp310-cp310-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:f84fbc98b019fef2ee9a1cb3ce93e3187a6df0b2538a651bfb890254ba9f90b5"}, + {file = "PyYAML-6.0-cp310-cp310-win32.whl", hash = "sha256:2cd5df3de48857ed0544b34e2d40e9fac445930039f3cfe4bcc592a1f836d513"}, + {file = "PyYAML-6.0-cp310-cp310-win_amd64.whl", hash = "sha256:daf496c58a8c52083df09b80c860005194014c3698698d1a57cbcfa182142a3a"}, + {file = "PyYAML-6.0-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:d4b0ba9512519522b118090257be113b9468d804b19d63c71dbcf4a48fa32358"}, + {file = "PyYAML-6.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:81957921f441d50af23654aa6c5e5eaf9b06aba7f0a19c18a538dc7ef291c5a1"}, + {file = "PyYAML-6.0-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:afa17f5bc4d1b10afd4466fd3a44dc0e245382deca5b3c353d8b757f9e3ecb8d"}, + {file = "PyYAML-6.0-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:dbad0e9d368bb989f4515da330b88a057617d16b6a8245084f1b05400f24609f"}, + {file = "PyYAML-6.0-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:432557aa2c09802be39460360ddffd48156e30721f5e8d917f01d31694216782"}, + {file = "PyYAML-6.0-cp311-cp311-win32.whl", hash = "sha256:bfaef573a63ba8923503d27530362590ff4f576c626d86a9fed95822a8255fd7"}, + {file = "PyYAML-6.0-cp311-cp311-win_amd64.whl", hash = "sha256:01b45c0191e6d66c470b6cf1b9531a771a83c1c4208272ead47a3ae4f2f603bf"}, + {file = "PyYAML-6.0-cp36-cp36m-macosx_10_9_x86_64.whl", hash = "sha256:897b80890765f037df3403d22bab41627ca8811ae55e9a722fd0392850ec4d86"}, + {file = "PyYAML-6.0-cp36-cp36m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:50602afada6d6cbfad699b0c7bb50d5ccffa7e46a3d738092afddc1f9758427f"}, + {file = "PyYAML-6.0-cp36-cp36m-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:48c346915c114f5fdb3ead70312bd042a953a8ce5c7106d5bfb1a5254e47da92"}, + {file = "PyYAML-6.0-cp36-cp36m-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:98c4d36e99714e55cfbaaee6dd5badbc9a1ec339ebfc3b1f52e293aee6bb71a4"}, + {file = "PyYAML-6.0-cp36-cp36m-win32.whl", hash = "sha256:0283c35a6a9fbf047493e3a0ce8d79ef5030852c51e9d911a27badfde0605293"}, + {file = "PyYAML-6.0-cp36-cp36m-win_amd64.whl", hash = "sha256:07751360502caac1c067a8132d150cf3d61339af5691fe9e87803040dbc5db57"}, + {file = "PyYAML-6.0-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:819b3830a1543db06c4d4b865e70ded25be52a2e0631ccd2f6a47a2822f2fd7c"}, + {file = "PyYAML-6.0-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:473f9edb243cb1935ab5a084eb238d842fb8f404ed2193a915d1784b5a6b5fc0"}, + {file = "PyYAML-6.0-cp37-cp37m-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:0ce82d761c532fe4ec3f87fc45688bdd3a4c1dc5e0b4a19814b9009a29baefd4"}, + {file = "PyYAML-6.0-cp37-cp37m-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:231710d57adfd809ef5d34183b8ed1eeae3f76459c18fb4a0b373ad56bedcdd9"}, + {file = "PyYAML-6.0-cp37-cp37m-win32.whl", hash = "sha256:c5687b8d43cf58545ade1fe3e055f70eac7a5a1a0bf42824308d868289a95737"}, + {file = "PyYAML-6.0-cp37-cp37m-win_amd64.whl", hash = "sha256:d15a181d1ecd0d4270dc32edb46f7cb7733c7c508857278d3d378d14d606db2d"}, + {file = "PyYAML-6.0-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:0b4624f379dab24d3725ffde76559cff63d9ec94e1736b556dacdfebe5ab6d4b"}, + {file = "PyYAML-6.0-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:213c60cd50106436cc818accf5baa1aba61c0189ff610f64f4a3e8c6726218ba"}, + {file = "PyYAML-6.0-cp38-cp38-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:9fa600030013c4de8165339db93d182b9431076eb98eb40ee068700c9c813e34"}, + {file = "PyYAML-6.0-cp38-cp38-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:277a0ef2981ca40581a47093e9e2d13b3f1fbbeffae064c1d21bfceba2030287"}, + {file = "PyYAML-6.0-cp38-cp38-win32.whl", hash = "sha256:d4eccecf9adf6fbcc6861a38015c2a64f38b9d94838ac1810a9023a0609e1b78"}, + {file = "PyYAML-6.0-cp38-cp38-win_amd64.whl", hash = "sha256:1e4747bc279b4f613a09eb64bba2ba602d8a6664c6ce6396a4d0cd413a50ce07"}, + {file = "PyYAML-6.0-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:055d937d65826939cb044fc8c9b08889e8c743fdc6a32b33e2390f66013e449b"}, + {file = "PyYAML-6.0-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:e61ceaab6f49fb8bdfaa0f92c4b57bcfbea54c09277b1b4f7ac376bfb7a7c174"}, + {file = "PyYAML-6.0-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:d67d839ede4ed1b28a4e8909735fc992a923cdb84e618544973d7dfc71540803"}, + {file = "PyYAML-6.0-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:cba8c411ef271aa037d7357a2bc8f9ee8b58b9965831d9e51baf703280dc73d3"}, + {file = "PyYAML-6.0-cp39-cp39-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:40527857252b61eacd1d9af500c3337ba8deb8fc298940291486c465c8b46ec0"}, + {file = "PyYAML-6.0-cp39-cp39-win32.whl", hash = "sha256:b5b9eccad747aabaaffbc6064800670f0c297e52c12754eb1d976c57e4f74dcb"}, + {file = "PyYAML-6.0-cp39-cp39-win_amd64.whl", hash = "sha256:b3d267842bf12586ba6c734f89d1f5b871df0273157918b0ccefa29deb05c21c"}, + {file = "PyYAML-6.0.tar.gz", hash = "sha256:68fb519c14306fec9720a2a5b45bc9f0c8d1b9c72adf45c37baedfcd949c35a2"}, +] +requests = [ + {file = "requests-2.28.1-py3-none-any.whl", hash = "sha256:8fefa2a1a1365bf5520aac41836fbee479da67864514bdb821f31ce07ce65349"}, + {file = "requests-2.28.1.tar.gz", hash = "sha256:7c5599b102feddaa661c826c56ab4fee28bfd17f5abca1ebbe3e7f19d7c97983"}, +] +requests-mock = [ + {file = "requests-mock-1.10.0.tar.gz", hash = "sha256:59c9c32419a9fb1ae83ec242d98e889c45bd7d7a65d48375cc243ec08441658b"}, + {file = "requests_mock-1.10.0-py2.py3-none-any.whl", hash = "sha256:2fdbb637ad17ee15c06f33d31169e71bf9fe2bdb7bc9da26185be0dd8d842699"}, +] +requests-oauthlib = [ + {file = "requests-oauthlib-1.3.1.tar.gz", hash = "sha256:75beac4a47881eeb94d5ea5d6ad31ef88856affe2332b9aafb52c6452ccf0d7a"}, + {file = "requests_oauthlib-1.3.1-py2.py3-none-any.whl", hash = "sha256:2577c501a2fb8d05a304c09d090d6e47c306fef15809d102b327cf8364bddab5"}, +] +responses = [ + {file = "responses-0.22.0-py3-none-any.whl", hash = "sha256:dcf294d204d14c436fddcc74caefdbc5764795a40ff4e6a7740ed8ddbf3294be"}, + {file = "responses-0.22.0.tar.gz", hash = "sha256:396acb2a13d25297789a5866b4881cf4e46ffd49cc26c43ab1117f40b973102e"}, +] +rich = [ + {file = "rich-12.6.0-py3-none-any.whl", hash = "sha256:a4eb26484f2c82589bd9a17c73d32a010b1e29d89f1604cd9bf3a2097b81bb5e"}, + {file = "rich-12.6.0.tar.gz", hash = "sha256:ba3a3775974105c221d31141f2c116f4fd65c5ceb0698657a11e9f295ec93fd0"}, +] +s3fs = [ + {file = "s3fs-2022.10.0-py3-none-any.whl", hash = "sha256:1e134c3577171699feb7c1a0c4713260d5b48296e1708737ff940baef6e2c153"}, + {file = "s3fs-2022.10.0.tar.gz", hash = "sha256:e8deb80f20bd0b2059141b874fdb9d6aeb8cce35312ea5f2c02b225a78a00406"}, +] +s3transfer = [ + {file = "s3transfer-0.6.0-py3-none-any.whl", hash = "sha256:06176b74f3a15f61f1b4f25a1fc29a4429040b7647133a463da8fa5bd28d5ecd"}, + {file = "s3transfer-0.6.0.tar.gz", hash = "sha256:2ed07d3866f523cc561bf4a00fc5535827981b117dd7876f036b0c1aca42c947"}, +] +setuptools = [ + {file = "setuptools-65.6.3-py3-none-any.whl", hash = "sha256:57f6f22bde4e042978bcd50176fdb381d7c21a9efa4041202288d3737a0c6a54"}, + {file = "setuptools-65.6.3.tar.gz", hash = "sha256:a7620757bf984b58deaf32fc8a4577a9bbc0850cf92c20e1ce41c38c19e5fb75"}, +] +six = [ + {file = "six-1.16.0-py2.py3-none-any.whl", hash = "sha256:8abb2f1d86890a2dfb989f9a77cfcfd3e47c2a354b01111771326f8aa26e0254"}, + {file = "six-1.16.0.tar.gz", hash = "sha256:1e61c37477a1626458e36f7b1d82aa5c9b094fa4802892072e49de9c60c4c926"}, +] +thrift = [ + {file = "thrift-0.16.0.tar.gz", hash = "sha256:2b5b6488fcded21f9d312aa23c9ff6a0195d0f6ae26ddbd5ad9e3e25dfc14408"}, +] +toml = [ + {file = "toml-0.10.2-py2.py3-none-any.whl", hash = "sha256:806143ae5bfb6a3c6e736a764057db0e6a0e05e338b5630894a5f779cabb4f9b"}, + {file = "toml-0.10.2.tar.gz", hash = "sha256:b3bda1d108d5dd99f4a20d24d9c348e91c4db7ab1b749200bded2f839ccbe68f"}, +] +tomli = [ + {file = "tomli-2.0.1-py3-none-any.whl", hash = "sha256:939de3e7a6161af0c887ef91b7d41a53e7c5a1ca976325f429cb46ea9bc30ecc"}, + {file = "tomli-2.0.1.tar.gz", hash = "sha256:de526c12914f0c550d15924c62d72abc48d6fe7364aa87328337a31007fe8a4f"}, +] +types-toml = [ + {file = "types-toml-0.10.8.1.tar.gz", hash = "sha256:171bdb3163d79a520560f24ba916a9fc9bff81659c5448a9fea89240923722be"}, + {file = "types_toml-0.10.8.1-py3-none-any.whl", hash = "sha256:b7b5c4977f96ab7b5ac06d8a6590d17c0bf252a96efc03b109c2711fb3e0eafd"}, +] +typing-extensions = [ + {file = "typing_extensions-4.4.0-py3-none-any.whl", hash = "sha256:16fa4864408f655d35ec496218b85f79b3437c829e93320c7c9215ccfd92489e"}, + {file = "typing_extensions-4.4.0.tar.gz", hash = "sha256:1511434bb92bf8dd198c12b1cc812e800d4181cfcb867674e0f8279cc93087aa"}, +] +urllib3 = [ + {file = "urllib3-1.26.13-py2.py3-none-any.whl", hash = "sha256:47cc05d99aaa09c9e72ed5809b60e7ba354e64b59c9c173ac3018642d8bb41fc"}, + {file = "urllib3-1.26.13.tar.gz", hash = "sha256:c083dd0dce68dbfbe1129d5271cb90f9447dea7d52097c6e0126120c521ddea8"}, +] +virtualenv = [ + {file = "virtualenv-20.17.1-py3-none-any.whl", hash = "sha256:ce3b1684d6e1a20a3e5ed36795a97dfc6af29bc3970ca8dab93e11ac6094b3c4"}, + {file = "virtualenv-20.17.1.tar.gz", hash = "sha256:f8b927684efc6f1cc206c9db297a570ab9ad0e51c16fa9e45487d36d1905c058"}, +] +werkzeug = [ + {file = "Werkzeug-2.2.2-py3-none-any.whl", hash = "sha256:f979ab81f58d7318e064e99c4506445d60135ac5cd2e177a2de0089bfd4c9bd5"}, + {file = "Werkzeug-2.2.2.tar.gz", hash = "sha256:7ea2d48322cc7c0f8b3a215ed73eabd7b5d75d0b50e31ab006286ccff9e00b8f"}, +] +wrapt = [ + {file = "wrapt-1.14.1-cp27-cp27m-macosx_10_9_x86_64.whl", hash = "sha256:1b376b3f4896e7930f1f772ac4b064ac12598d1c38d04907e696cc4d794b43d3"}, + {file = "wrapt-1.14.1-cp27-cp27m-manylinux1_i686.whl", hash = "sha256:903500616422a40a98a5a3c4ff4ed9d0066f3b4c951fa286018ecdf0750194ef"}, + {file = "wrapt-1.14.1-cp27-cp27m-manylinux1_x86_64.whl", hash = "sha256:5a9a0d155deafd9448baff28c08e150d9b24ff010e899311ddd63c45c2445e28"}, + {file = "wrapt-1.14.1-cp27-cp27m-manylinux2010_i686.whl", hash = "sha256:ddaea91abf8b0d13443f6dac52e89051a5063c7d014710dcb4d4abb2ff811a59"}, + {file = "wrapt-1.14.1-cp27-cp27m-manylinux2010_x86_64.whl", hash = "sha256:36f582d0c6bc99d5f39cd3ac2a9062e57f3cf606ade29a0a0d6b323462f4dd87"}, + {file = "wrapt-1.14.1-cp27-cp27mu-manylinux1_i686.whl", hash = "sha256:7ef58fb89674095bfc57c4069e95d7a31cfdc0939e2a579882ac7d55aadfd2a1"}, + {file = "wrapt-1.14.1-cp27-cp27mu-manylinux1_x86_64.whl", hash = "sha256:e2f83e18fe2f4c9e7db597e988f72712c0c3676d337d8b101f6758107c42425b"}, + {file = "wrapt-1.14.1-cp27-cp27mu-manylinux2010_i686.whl", hash = "sha256:ee2b1b1769f6707a8a445162ea16dddf74285c3964f605877a20e38545c3c462"}, + {file = "wrapt-1.14.1-cp27-cp27mu-manylinux2010_x86_64.whl", hash = "sha256:833b58d5d0b7e5b9832869f039203389ac7cbf01765639c7309fd50ef619e0b1"}, + {file = "wrapt-1.14.1-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:80bb5c256f1415f747011dc3604b59bc1f91c6e7150bd7db03b19170ee06b320"}, + {file = "wrapt-1.14.1-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:07f7a7d0f388028b2df1d916e94bbb40624c59b48ecc6cbc232546706fac74c2"}, + {file = "wrapt-1.14.1-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:02b41b633c6261feff8ddd8d11c711df6842aba629fdd3da10249a53211a72c4"}, + {file = "wrapt-1.14.1-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:2fe803deacd09a233e4762a1adcea5db5d31e6be577a43352936179d14d90069"}, + {file = "wrapt-1.14.1-cp310-cp310-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:257fd78c513e0fb5cdbe058c27a0624c9884e735bbd131935fd49e9fe719d310"}, + {file = "wrapt-1.14.1-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:4fcc4649dc762cddacd193e6b55bc02edca674067f5f98166d7713b193932b7f"}, + {file = "wrapt-1.14.1-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:11871514607b15cfeb87c547a49bca19fde402f32e2b1c24a632506c0a756656"}, + {file = "wrapt-1.14.1-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:8ad85f7f4e20964db4daadcab70b47ab05c7c1cf2a7c1e51087bfaa83831854c"}, + {file = "wrapt-1.14.1-cp310-cp310-win32.whl", hash = "sha256:a9a52172be0b5aae932bef82a79ec0a0ce87288c7d132946d645eba03f0ad8a8"}, + {file = "wrapt-1.14.1-cp310-cp310-win_amd64.whl", hash = "sha256:6d323e1554b3d22cfc03cd3243b5bb815a51f5249fdcbb86fda4bf62bab9e164"}, + {file = "wrapt-1.14.1-cp35-cp35m-manylinux1_i686.whl", hash = "sha256:43ca3bbbe97af00f49efb06e352eae40434ca9d915906f77def219b88e85d907"}, + {file = "wrapt-1.14.1-cp35-cp35m-manylinux1_x86_64.whl", hash = "sha256:6b1a564e6cb69922c7fe3a678b9f9a3c54e72b469875aa8018f18b4d1dd1adf3"}, + {file = "wrapt-1.14.1-cp35-cp35m-manylinux2010_i686.whl", hash = "sha256:00b6d4ea20a906c0ca56d84f93065b398ab74b927a7a3dbd470f6fc503f95dc3"}, + {file = "wrapt-1.14.1-cp35-cp35m-manylinux2010_x86_64.whl", hash = "sha256:a85d2b46be66a71bedde836d9e41859879cc54a2a04fad1191eb50c2066f6e9d"}, + {file = "wrapt-1.14.1-cp35-cp35m-win32.whl", hash = "sha256:dbcda74c67263139358f4d188ae5faae95c30929281bc6866d00573783c422b7"}, + {file = "wrapt-1.14.1-cp35-cp35m-win_amd64.whl", hash = "sha256:b21bb4c09ffabfa0e85e3a6b623e19b80e7acd709b9f91452b8297ace2a8ab00"}, + {file = "wrapt-1.14.1-cp36-cp36m-macosx_10_9_x86_64.whl", hash = "sha256:9e0fd32e0148dd5dea6af5fee42beb949098564cc23211a88d799e434255a1f4"}, + {file = "wrapt-1.14.1-cp36-cp36m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:9736af4641846491aedb3c3f56b9bc5568d92b0692303b5a305301a95dfd38b1"}, + {file = "wrapt-1.14.1-cp36-cp36m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:5b02d65b9ccf0ef6c34cba6cf5bf2aab1bb2f49c6090bafeecc9cd81ad4ea1c1"}, + {file = "wrapt-1.14.1-cp36-cp36m-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:21ac0156c4b089b330b7666db40feee30a5d52634cc4560e1905d6529a3897ff"}, + {file = "wrapt-1.14.1-cp36-cp36m-musllinux_1_1_aarch64.whl", hash = "sha256:9f3e6f9e05148ff90002b884fbc2a86bd303ae847e472f44ecc06c2cd2fcdb2d"}, + {file = "wrapt-1.14.1-cp36-cp36m-musllinux_1_1_i686.whl", hash = "sha256:6e743de5e9c3d1b7185870f480587b75b1cb604832e380d64f9504a0535912d1"}, + {file = "wrapt-1.14.1-cp36-cp36m-musllinux_1_1_x86_64.whl", hash = "sha256:d79d7d5dc8a32b7093e81e97dad755127ff77bcc899e845f41bf71747af0c569"}, + {file = "wrapt-1.14.1-cp36-cp36m-win32.whl", hash = "sha256:81b19725065dcb43df02b37e03278c011a09e49757287dca60c5aecdd5a0b8ed"}, + {file = "wrapt-1.14.1-cp36-cp36m-win_amd64.whl", hash = "sha256:b014c23646a467558be7da3d6b9fa409b2c567d2110599b7cf9a0c5992b3b471"}, + {file = "wrapt-1.14.1-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:88bd7b6bd70a5b6803c1abf6bca012f7ed963e58c68d76ee20b9d751c74a3248"}, + {file = "wrapt-1.14.1-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:b5901a312f4d14c59918c221323068fad0540e34324925c8475263841dbdfe68"}, + {file = "wrapt-1.14.1-cp37-cp37m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:d77c85fedff92cf788face9bfa3ebaa364448ebb1d765302e9af11bf449ca36d"}, + {file = "wrapt-1.14.1-cp37-cp37m-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:8d649d616e5c6a678b26d15ece345354f7c2286acd6db868e65fcc5ff7c24a77"}, + {file = "wrapt-1.14.1-cp37-cp37m-musllinux_1_1_aarch64.whl", hash = "sha256:7d2872609603cb35ca513d7404a94d6d608fc13211563571117046c9d2bcc3d7"}, + {file = "wrapt-1.14.1-cp37-cp37m-musllinux_1_1_i686.whl", hash = "sha256:ee6acae74a2b91865910eef5e7de37dc6895ad96fa23603d1d27ea69df545015"}, + {file = "wrapt-1.14.1-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:2b39d38039a1fdad98c87279b48bc5dce2c0ca0d73483b12cb72aa9609278e8a"}, + {file = "wrapt-1.14.1-cp37-cp37m-win32.whl", hash = "sha256:60db23fa423575eeb65ea430cee741acb7c26a1365d103f7b0f6ec412b893853"}, + {file = "wrapt-1.14.1-cp37-cp37m-win_amd64.whl", hash = "sha256:709fe01086a55cf79d20f741f39325018f4df051ef39fe921b1ebe780a66184c"}, + {file = "wrapt-1.14.1-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:8c0ce1e99116d5ab21355d8ebe53d9460366704ea38ae4d9f6933188f327b456"}, + {file = "wrapt-1.14.1-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:e3fb1677c720409d5f671e39bac6c9e0e422584e5f518bfd50aa4cbbea02433f"}, + {file = "wrapt-1.14.1-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:642c2e7a804fcf18c222e1060df25fc210b9c58db7c91416fb055897fc27e8cc"}, + {file = "wrapt-1.14.1-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:7b7c050ae976e286906dd3f26009e117eb000fb2cf3533398c5ad9ccc86867b1"}, + {file = "wrapt-1.14.1-cp38-cp38-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:ef3f72c9666bba2bab70d2a8b79f2c6d2c1a42a7f7e2b0ec83bb2f9e383950af"}, + {file = "wrapt-1.14.1-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:01c205616a89d09827986bc4e859bcabd64f5a0662a7fe95e0d359424e0e071b"}, + {file = "wrapt-1.14.1-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:5a0f54ce2c092aaf439813735584b9537cad479575a09892b8352fea5e988dc0"}, + {file = "wrapt-1.14.1-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:2cf71233a0ed05ccdabe209c606fe0bac7379fdcf687f39b944420d2a09fdb57"}, + {file = "wrapt-1.14.1-cp38-cp38-win32.whl", hash = "sha256:aa31fdcc33fef9eb2552cbcbfee7773d5a6792c137b359e82879c101e98584c5"}, + {file = "wrapt-1.14.1-cp38-cp38-win_amd64.whl", hash = "sha256:d1967f46ea8f2db647c786e78d8cc7e4313dbd1b0aca360592d8027b8508e24d"}, + {file = "wrapt-1.14.1-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:3232822c7d98d23895ccc443bbdf57c7412c5a65996c30442ebe6ed3df335383"}, + {file = "wrapt-1.14.1-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:988635d122aaf2bdcef9e795435662bcd65b02f4f4c1ae37fbee7401c440b3a7"}, + {file = "wrapt-1.14.1-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:9cca3c2cdadb362116235fdbd411735de4328c61425b0aa9f872fd76d02c4e86"}, + {file = "wrapt-1.14.1-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:d52a25136894c63de15a35bc0bdc5adb4b0e173b9c0d07a2be9d3ca64a332735"}, + {file = "wrapt-1.14.1-cp39-cp39-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:40e7bc81c9e2b2734ea4bc1aceb8a8f0ceaac7c5299bc5d69e37c44d9081d43b"}, + {file = "wrapt-1.14.1-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:b9b7a708dd92306328117d8c4b62e2194d00c365f18eff11a9b53c6f923b01e3"}, + {file = "wrapt-1.14.1-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:6a9a25751acb379b466ff6be78a315e2b439d4c94c1e99cb7266d40a537995d3"}, + {file = "wrapt-1.14.1-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:34aa51c45f28ba7f12accd624225e2b1e5a3a45206aa191f6f9aac931d9d56fe"}, + {file = "wrapt-1.14.1-cp39-cp39-win32.whl", hash = "sha256:dee0ce50c6a2dd9056c20db781e9c1cfd33e77d2d569f5d1d9321c641bb903d5"}, + {file = "wrapt-1.14.1-cp39-cp39-win_amd64.whl", hash = "sha256:dee60e1de1898bde3b238f18340eec6148986da0455d8ba7848d50470a7a32fb"}, + {file = "wrapt-1.14.1.tar.gz", hash = "sha256:380a85cf89e0e69b7cfbe2ea9f765f004ff419f34194018a6827ac0e3edfed4d"}, +] +xmltodict = [ + {file = "xmltodict-0.13.0-py2.py3-none-any.whl", hash = "sha256:aa89e8fd76320154a40d19a0df04a4695fb9dc5ba977cbb68ab3e4eb225e7852"}, + {file = "xmltodict-0.13.0.tar.gz", hash = "sha256:341595a488e3e01a85a9d8911d8912fd922ede5fecc4dce437eb4b6c8d037e56"}, +] +yarl = [ + {file = "yarl-1.8.2-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:bb81f753c815f6b8e2ddd2eef3c855cf7da193b82396ac013c661aaa6cc6b0a5"}, + {file = "yarl-1.8.2-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:47d49ac96156f0928f002e2424299b2c91d9db73e08c4cd6742923a086f1c863"}, + {file = "yarl-1.8.2-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:3fc056e35fa6fba63248d93ff6e672c096f95f7836938241ebc8260e062832fe"}, + {file = "yarl-1.8.2-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:58a3c13d1c3005dbbac5c9f0d3210b60220a65a999b1833aa46bd6677c69b08e"}, + {file = "yarl-1.8.2-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:10b08293cda921157f1e7c2790999d903b3fd28cd5c208cf8826b3b508026996"}, + {file = "yarl-1.8.2-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:de986979bbd87272fe557e0a8fcb66fd40ae2ddfe28a8b1ce4eae22681728fef"}, + {file = "yarl-1.8.2-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:6c4fcfa71e2c6a3cb568cf81aadc12768b9995323186a10827beccf5fa23d4f8"}, + {file = "yarl-1.8.2-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:ae4d7ff1049f36accde9e1ef7301912a751e5bae0a9d142459646114c70ecba6"}, + {file = "yarl-1.8.2-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:bf071f797aec5b96abfc735ab97da9fd8f8768b43ce2abd85356a3127909d146"}, + {file = "yarl-1.8.2-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:74dece2bfc60f0f70907c34b857ee98f2c6dd0f75185db133770cd67300d505f"}, + {file = "yarl-1.8.2-cp310-cp310-musllinux_1_1_ppc64le.whl", hash = "sha256:df60a94d332158b444301c7f569659c926168e4d4aad2cfbf4bce0e8fb8be826"}, + {file = "yarl-1.8.2-cp310-cp310-musllinux_1_1_s390x.whl", hash = "sha256:63243b21c6e28ec2375f932a10ce7eda65139b5b854c0f6b82ed945ba526bff3"}, + {file = "yarl-1.8.2-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:cfa2bbca929aa742b5084fd4663dd4b87c191c844326fcb21c3afd2d11497f80"}, + {file = "yarl-1.8.2-cp310-cp310-win32.whl", hash = "sha256:b05df9ea7496df11b710081bd90ecc3a3db6adb4fee36f6a411e7bc91a18aa42"}, + {file = "yarl-1.8.2-cp310-cp310-win_amd64.whl", hash = "sha256:24ad1d10c9db1953291f56b5fe76203977f1ed05f82d09ec97acb623a7976574"}, + {file = "yarl-1.8.2-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:2a1fca9588f360036242f379bfea2b8b44cae2721859b1c56d033adfd5893634"}, + {file = "yarl-1.8.2-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:f37db05c6051eff17bc832914fe46869f8849de5b92dc4a3466cd63095d23dfd"}, + {file = "yarl-1.8.2-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:77e913b846a6b9c5f767b14dc1e759e5aff05502fe73079f6f4176359d832581"}, + {file = "yarl-1.8.2-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:0978f29222e649c351b173da2b9b4665ad1feb8d1daa9d971eb90df08702668a"}, + {file = "yarl-1.8.2-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:388a45dc77198b2460eac0aca1efd6a7c09e976ee768b0d5109173e521a19daf"}, + {file = "yarl-1.8.2-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:2305517e332a862ef75be8fad3606ea10108662bc6fe08509d5ca99503ac2aee"}, + {file = "yarl-1.8.2-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:42430ff511571940d51e75cf42f1e4dbdded477e71c1b7a17f4da76c1da8ea76"}, + {file = "yarl-1.8.2-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:3150078118f62371375e1e69b13b48288e44f6691c1069340081c3fd12c94d5b"}, + {file = "yarl-1.8.2-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:c15163b6125db87c8f53c98baa5e785782078fbd2dbeaa04c6141935eb6dab7a"}, + {file = "yarl-1.8.2-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:4d04acba75c72e6eb90745447d69f84e6c9056390f7a9724605ca9c56b4afcc6"}, + {file = "yarl-1.8.2-cp311-cp311-musllinux_1_1_ppc64le.whl", hash = "sha256:e7fd20d6576c10306dea2d6a5765f46f0ac5d6f53436217913e952d19237efc4"}, + {file = "yarl-1.8.2-cp311-cp311-musllinux_1_1_s390x.whl", hash = "sha256:75c16b2a900b3536dfc7014905a128a2bea8fb01f9ee26d2d7d8db0a08e7cb2c"}, + {file = "yarl-1.8.2-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:6d88056a04860a98341a0cf53e950e3ac9f4e51d1b6f61a53b0609df342cc8b2"}, + {file = "yarl-1.8.2-cp311-cp311-win32.whl", hash = "sha256:fb742dcdd5eec9f26b61224c23baea46c9055cf16f62475e11b9b15dfd5c117b"}, + {file = "yarl-1.8.2-cp311-cp311-win_amd64.whl", hash = "sha256:8c46d3d89902c393a1d1e243ac847e0442d0196bbd81aecc94fcebbc2fd5857c"}, + {file = "yarl-1.8.2-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:ceff9722e0df2e0a9e8a79c610842004fa54e5b309fe6d218e47cd52f791d7ef"}, + {file = "yarl-1.8.2-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:3f6b4aca43b602ba0f1459de647af954769919c4714706be36af670a5f44c9c1"}, + {file = "yarl-1.8.2-cp37-cp37m-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:1684a9bd9077e922300ecd48003ddae7a7474e0412bea38d4631443a91d61077"}, + {file = "yarl-1.8.2-cp37-cp37m-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:ebb78745273e51b9832ef90c0898501006670d6e059f2cdb0e999494eb1450c2"}, + {file = "yarl-1.8.2-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:3adeef150d528ded2a8e734ebf9ae2e658f4c49bf413f5f157a470e17a4a2e89"}, + {file = "yarl-1.8.2-cp37-cp37m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:57a7c87927a468e5a1dc60c17caf9597161d66457a34273ab1760219953f7f4c"}, + {file = "yarl-1.8.2-cp37-cp37m-musllinux_1_1_aarch64.whl", hash = "sha256:efff27bd8cbe1f9bd127e7894942ccc20c857aa8b5a0327874f30201e5ce83d0"}, + {file = "yarl-1.8.2-cp37-cp37m-musllinux_1_1_i686.whl", hash = "sha256:a783cd344113cb88c5ff7ca32f1f16532a6f2142185147822187913eb989f739"}, + {file = "yarl-1.8.2-cp37-cp37m-musllinux_1_1_ppc64le.whl", hash = "sha256:705227dccbe96ab02c7cb2c43e1228e2826e7ead880bb19ec94ef279e9555b5b"}, + {file = "yarl-1.8.2-cp37-cp37m-musllinux_1_1_s390x.whl", hash = "sha256:34c09b43bd538bf6c4b891ecce94b6fa4f1f10663a8d4ca589a079a5018f6ed7"}, + {file = "yarl-1.8.2-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:a48f4f7fea9a51098b02209d90297ac324241bf37ff6be6d2b0149ab2bd51b37"}, + {file = "yarl-1.8.2-cp37-cp37m-win32.whl", hash = "sha256:0414fd91ce0b763d4eadb4456795b307a71524dbacd015c657bb2a39db2eab89"}, + {file = "yarl-1.8.2-cp37-cp37m-win_amd64.whl", hash = "sha256:d881d152ae0007809c2c02e22aa534e702f12071e6b285e90945aa3c376463c5"}, + {file = "yarl-1.8.2-cp38-cp38-macosx_10_9_universal2.whl", hash = "sha256:5df5e3d04101c1e5c3b1d69710b0574171cc02fddc4b23d1b2813e75f35a30b1"}, + {file = "yarl-1.8.2-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:7a66c506ec67eb3159eea5096acd05f5e788ceec7b96087d30c7d2865a243918"}, + {file = "yarl-1.8.2-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:2b4fa2606adf392051d990c3b3877d768771adc3faf2e117b9de7eb977741229"}, + {file = "yarl-1.8.2-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:1e21fb44e1eff06dd6ef971d4bdc611807d6bd3691223d9c01a18cec3677939e"}, {file = "yarl-1.8.2-cp38-cp38-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:93202666046d9edadfe9f2e7bf5e0782ea0d497b6d63da322e541665d65a044e"}, {file = "yarl-1.8.2-cp38-cp38-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:fc77086ce244453e074e445104f0ecb27530d6fd3a46698e33f6c38951d5a0f1"}, {file = "yarl-1.8.2-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:64dd68a92cab699a233641f5929a40f02a4ede8c009068ca8aa1fe87b8c20ae3"}, @@ -2341,35 +2387,11 @@ files = [ {file = "yarl-1.8.2-cp39-cp39-win_amd64.whl", hash = "sha256:6604711362f2dbf7160df21c416f81fac0de6dbcf0b5445a2ef25478ecc4c778"}, {file = "yarl-1.8.2.tar.gz", hash = "sha256:49d43402c6e3013ad0978602bf6bf5328535c48d192304b91b97a3c6790b1562"}, ] - -[package.dependencies] -idna = ">=2.0" -multidict = ">=4.0" - -[[package]] -name = "zipp" -version = "3.11.0" -description = "Backport of pathlib-compatible object wrapper for zip files" -category = "dev" -optional = false -python-versions = ">=3.7" -files = [ +zipp = [ {file = "zipp-3.11.0-py3-none-any.whl", hash = "sha256:83a28fcb75844b5c0cdaf5aa4003c2d728c77e05f5aeabe8e95e56727005fbaa"}, {file = "zipp-3.11.0.tar.gz", hash = "sha256:a7a22e05929290a67401440b39690ae6563279bced5f314609d9d03798f56766"}, ] - -[package.extras] -docs = ["furo", "jaraco.packaging (>=9)", "jaraco.tidelift (>=1.4)", "rst.linker (>=1.9)", "sphinx (>=3.5)"] -testing = ["flake8 (<5)", "func-timeout", "jaraco.functools", "jaraco.itertools", "more-itertools", "pytest (>=6)", "pytest-black (>=0.3.7)", "pytest-checkdocs (>=2.4)", "pytest-cov", "pytest-enabler (>=1.3)", "pytest-flake8", "pytest-mypy (>=0.9.1)"] - -[[package]] -name = "zstandard" -version = "0.19.0" -description = "Zstandard bindings for Python" -category = "main" -optional = false -python-versions = ">=3.6" -files = [ +zstandard = [ {file = "zstandard-0.19.0-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:a65e0119ad39e855427520f7829618f78eb2824aa05e63ff19b466080cd99210"}, {file = "zstandard-0.19.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:4fa496d2d674c6e9cffc561639d17009d29adee84a27cf1e12d3c9be14aa8feb"}, {file = "zstandard-0.19.0-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:8f7c68de4f362c1b2f426395fe4e05028c56d0782b2ec3ae18a5416eaf775576"}, @@ -2422,24 +2444,3 @@ files = [ {file = "zstandard-0.19.0-cp39-cp39-win_amd64.whl", hash = "sha256:55a513ec67e85abd8b8b83af8813368036f03e2d29a50fc94033504918273980"}, {file = "zstandard-0.19.0.tar.gz", hash = "sha256:31d12fcd942dd8dbf52ca5f6b1bbe287f44e5d551a081a983ff3ea2082867863"}, ] - -[package.dependencies] -cffi = {version = ">=1.11", markers = "platform_python_implementation == \"PyPy\""} - -[package.extras] -cffi = ["cffi (>=1.11)"] - -[extras] -adlfs = ["adlfs"] -duckdb = ["duckdb", "pyarrow"] -glue = ["boto3"] -hive = ["thrift"] -pandas = ["pandas", "pyarrow"] -pyarrow = ["pyarrow"] -s3fs = ["s3fs"] -snappy = ["python-snappy"] - -[metadata] -lock-version = "2.0" -python-versions = "^3.8" -content-hash = "f56dc6fb65c9d55f693f2fa9b5b02b8a11291b8fb47c2ad55400ae7c84a20de4" diff --git a/pyproject.toml b/pyproject.toml index 4376873cf3..9cf8704911 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -80,7 +80,7 @@ pytest = "7.2.0" pytest-checkdocs = "2.9.0" pre-commit = "2.20.0" fastavro = "1.7.0" -coverage = { version = "^6.5.0", extras = ["toml"] } +coverage = { version = "^7.0.1", extras = ["toml"] } requests-mock = "1.10.0" moto = "^4.0.11" typing-extensions = '4.4.0' From 406d60b7375d69434f17fee037398d2c0355c807 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 26 Dec 2022 21:25:58 +0100 Subject: [PATCH 313/642] Build: Bump adlfs from 2022.10.0 to 2022.11.2 in /python (#6492) Bumps [adlfs](https://github.com/dask/adlfs) from 2022.10.0 to 2022.11.2. - [Release notes](https://github.com/dask/adlfs/releases) - [Changelog](https://github.com/fsspec/adlfs/blob/main/CHANGELOG.md) - [Commits](https://github.com/dask/adlfs/compare/2022.10.0...2022.11.2) --- updated-dependencies: - dependency-name: adlfs dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- poetry.lock | 10 +++++----- pyproject.toml | 2 +- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/poetry.lock b/poetry.lock index 8044d06b3a..4adbc638e0 100644 --- a/poetry.lock +++ b/poetry.lock @@ -14,7 +14,7 @@ requests = ">=2.0.0,<3" [[package]] name = "adlfs" -version = "2022.10.0" +version = "2022.11.2" description = "Access Azure Datalake Gen1 with fsspec and dask" category = "main" optional = true @@ -22,7 +22,7 @@ python-versions = ">=3.8" [package.dependencies] aiohttp = ">=3.7.0" -azure-core = ">=1.7.0" +azure-core = ">=1.23.1,<2.0.0" azure-datalake-store = ">=0.0.46,<0.1" azure-identity = "*" azure-storage-blob = ">=12.12.0" @@ -1193,7 +1193,7 @@ snappy = ["python-snappy"] [metadata] lock-version = "1.1" python-versions = "^3.8" -content-hash = "f2d6c78025050536a5ef051ddb0023fe16a41d502399f0c4fc5b3e072fef9aec" +content-hash = "07a63f1a7c5a33d18cecc19d86a0d22ed8ba9142ff81f5d0e1f2835a0fbebaf4" [metadata.files] adal = [ @@ -1201,8 +1201,8 @@ adal = [ {file = "adal-1.2.7.tar.gz", hash = "sha256:d74f45b81317454d96e982fd1c50e6fb5c99ac2223728aea8764433a39f566f1"}, ] adlfs = [ - {file = "adlfs-2022.10.0-py3-none-any.whl", hash = "sha256:dc63b248f109bca2000d306f7a51c33d6859e4692dda7c6198b6300f8e29faaf"}, - {file = "adlfs-2022.10.0.tar.gz", hash = "sha256:8542f9e6799aeb77adb5cb9854f76b7dce874610dc0f2ff006cce960526ffad6"}, + {file = "adlfs-2022.11.2-py3-none-any.whl", hash = "sha256:60da4a17260754ee6d8da43e3d756226ade4fd19ada8411e858f5acecd0833e2"}, + {file = "adlfs-2022.11.2.tar.gz", hash = "sha256:920dba10468f186037ca394dcabcba113532d80f52b315211c8e771be40475ea"}, ] aiobotocore = [ {file = "aiobotocore-2.4.1-py3-none-any.whl", hash = "sha256:26e0d55d5901f487c3467616c028abb85036ca33a0f88e279770adae6b865c69"}, diff --git a/pyproject.toml b/pyproject.toml index 9cf8704911..454f52d4e1 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -73,7 +73,7 @@ thrift = { version = "0.16.0", optional = true } s3fs = { version = "2022.10.0", optional = true } boto3 = {version = "1.24.59", optional = true} -adlfs = { version = "2022.10.0", optional = true } +adlfs = { version = "2022.11.2", optional = true } [tool.poetry.dev-dependencies] pytest = "7.2.0" From 6e6f931122352d8ab1d9ff9d93c71dc46b0f7c80 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 26 Dec 2022 21:36:58 +0100 Subject: [PATCH 314/642] Build: Bump moto from 4.0.11 to 4.0.12 in /python (#6494) Bumps [moto](https://github.com/spulec/moto) from 4.0.11 to 4.0.12. - [Release notes](https://github.com/spulec/moto/releases) - [Changelog](https://github.com/spulec/moto/blob/master/CHANGELOG.md) - [Commits](https://github.com/spulec/moto/compare/4.0.11...4.0.12) --- updated-dependencies: - dependency-name: moto dependency-type: direct:development update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- poetry.lock | 11 +++++------ pyproject.toml | 2 +- 2 files changed, 6 insertions(+), 7 deletions(-) diff --git a/poetry.lock b/poetry.lock index 4adbc638e0..e4cb39aece 100644 --- a/poetry.lock +++ b/poetry.lock @@ -520,7 +520,7 @@ python-versions = "*" [[package]] name = "moto" -version = "4.0.11" +version = "4.0.12" description = "A library that allows your python tests to easily mock out the boto library" category = "dev" optional = false @@ -533,7 +533,6 @@ cryptography = ">=3.3.1" Jinja2 = ">=2.10.1" MarkupSafe = "!=2.0.0a1" python-dateutil = ">=2.1,<3.0.0" -pytz = "*" requests = ">=2.5" responses = ">=0.13.0" werkzeug = ">=0.5,<2.2.0 || >2.2.0,<2.2.1 || >2.2.1" @@ -881,7 +880,7 @@ name = "pytz" version = "2022.6" description = "World timezone definitions, modern and historical" category = "main" -optional = false +optional = true python-versions = "*" [[package]] @@ -1193,7 +1192,7 @@ snappy = ["python-snappy"] [metadata] lock-version = "1.1" python-versions = "^3.8" -content-hash = "07a63f1a7c5a33d18cecc19d86a0d22ed8ba9142ff81f5d0e1f2835a0fbebaf4" +content-hash = "565b8068555fa388cea32b84eb0d9a97efc0ef950b4467cbe6d92b0abe34bbb5" [metadata.files] adal = [ @@ -1787,8 +1786,8 @@ mmhash3 = [ {file = "mmhash3-3.0.1.tar.gz", hash = "sha256:a00d68f4a1cc434b9501513c8a29e18ed1ddad383677d72b41d71d0d862348af"}, ] moto = [ - {file = "moto-4.0.11-py3-none-any.whl", hash = "sha256:704d6d38a4e6fe49e1fe9c6b4127ca46c66aac00368149bc1f1d70a0ceff8846"}, - {file = "moto-4.0.11.tar.gz", hash = "sha256:a6388de4a746e0b509286e1d7e70f86900b4f69ec65f6c92c47e570f95d05b14"}, + {file = "moto-4.0.12-py3-none-any.whl", hash = "sha256:e6592727d90a2dc82ca7c7fa61b224e2b195cc0dcdac2f46ccbf0cd8efde1459"}, + {file = "moto-4.0.12.tar.gz", hash = "sha256:30f8f31658f18cdb15eb626c8cb3a07520f0d8c2193226fbcb3326ae12fba246"}, ] msal = [ {file = "msal-1.20.0-py2.py3-none-any.whl", hash = "sha256:d2f1c26368ecdc28c8657d457352faa0b81b1845a7b889d8676787721ba86792"}, diff --git a/pyproject.toml b/pyproject.toml index 454f52d4e1..b46c5c027b 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -82,7 +82,7 @@ pre-commit = "2.20.0" fastavro = "1.7.0" coverage = { version = "^7.0.1", extras = ["toml"] } requests-mock = "1.10.0" -moto = "^4.0.11" +moto = "^4.0.12" typing-extensions = '4.4.0' [tool.poetry.scripts] From 7b17e3b1d8113be571307c68bab4bd7bd2dad14a Mon Sep 17 00:00:00 2001 From: Fokko Driesprong Date: Fri, 30 Dec 2022 19:49:06 +0100 Subject: [PATCH 315/642] Python: Add more tests (#6504) --- pyiceberg/typedef.py | 11 ++++------ tests/test_typedef.py | 15 +++++++++++++- tests/test_types.py | 15 ++++++++++++++ tests/utils/test_deprecated.py | 37 ++++++++++++++++++++++++++++++++++ 4 files changed, 70 insertions(+), 8 deletions(-) create mode 100644 tests/utils/test_deprecated.py diff --git a/pyiceberg/typedef.py b/pyiceberg/typedef.py index ccdeacad30..c924d703bd 100644 --- a/pyiceberg/typedef.py +++ b/pyiceberg/typedef.py @@ -54,12 +54,9 @@ def __init__(self, default_factory: Callable[[K], V]): self.default_factory = default_factory def __missing__(self, key: K) -> V: - if self.default_factory is None: - raise KeyError(key) - else: - val = self.default_factory(key) - self[key] = val - return val + val = self.default_factory(key) + self[key] = val + return val Identifier = Tuple[str, ...] @@ -100,4 +97,4 @@ def __eq__(self, other: Any) -> bool: return True if isinstance(other, Record) and other._data == self._data else False def __repr__(self) -> str: - return "[" + ", ".join([repr(e) for e in self._data]) + "]" + return f"{self.__class__.__name__}[" + ", ".join([repr(e) for e in self._data]) + "]" diff --git a/tests/test_typedef.py b/tests/test_typedef.py index 01eed9cde0..628e674443 100644 --- a/tests/test_typedef.py +++ b/tests/test_typedef.py @@ -16,7 +16,7 @@ # under the License. import pytest -from pyiceberg.typedef import FrozenDict +from pyiceberg.typedef import FrozenDict, KeyDefaultDict, Record def test_setitem_frozendict() -> None: @@ -29,3 +29,16 @@ def test_update_frozendict() -> None: d = FrozenDict(foo=1, bar=2) with pytest.raises(AttributeError): d.update({"yes": 2}) + + +def test_keydefaultdict() -> None: + def one(_: int) -> int: + return 1 + + defaultdict = KeyDefaultDict(one) + assert defaultdict[22] == 1 + + +def test_record_repr() -> None: + r = Record(1, "vo", True) + assert repr(r) == "Record[1, 'vo', True]" diff --git a/tests/test_types.py b/tests/test_types.py index 4b1d7698db..cda1309ede 100644 --- a/tests/test_types.py +++ b/tests/test_types.py @@ -18,6 +18,7 @@ from typing import Type import pytest +from pydantic import ValidationError from pyiceberg.types import ( BinaryType, @@ -408,6 +409,13 @@ def test_deserialization_fixed() -> None: assert len(inner) == 22 +def test_deserialization_fixed_failure() -> None: + with pytest.raises(ValidationError) as exc_info: + _ = IcebergTestType.parse_raw('"fixed[abc]"') + + assert "Could not match fixed[abc], expected format fixed[22]" in str(exc_info.value) + + def test_str_fixed() -> None: assert str(FixedType(22)) == "fixed[22]" @@ -446,6 +454,13 @@ def test_deserialization_decimal() -> None: assert inner.scale == 25 +def test_deserialization_decimal_failure() -> None: + with pytest.raises(ValidationError) as exc_info: + _ = IcebergTestType.parse_raw('"decimal(abc, def)"') + + assert "Could not parse decimal(abc, def) into a DecimalType" in str(exc_info.value) + + def test_str_decimal() -> None: assert str(DecimalType(19, 25)) == "decimal(19, 25)" diff --git a/tests/utils/test_deprecated.py b/tests/utils/test_deprecated.py new file mode 100644 index 0000000000..7c44c45859 --- /dev/null +++ b/tests/utils/test_deprecated.py @@ -0,0 +1,37 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +from unittest.mock import Mock, patch + +from pyiceberg.utils.deprecated import deprecated + + +@patch("warnings.warn") +def test_deprecated(warn: Mock) -> None: + @deprecated( + deprecated_in="0.1.0", + removed_in="0.2.0", + help_message="Please use load_something_else() instead", + ) + def deprecated_method() -> None: + pass + + deprecated_method() + + assert warn.called + assert warn.call_args[0] == ( + "Call to deprecated_method, deprecated in 0.1.0, will be removed in 0.2.0. Please use load_something_else() instead.", + ) From 96562d8f853afb8ee77ab8a205a028465139f6ae Mon Sep 17 00:00:00 2001 From: Fokko Driesprong Date: Fri, 30 Dec 2022 19:54:01 +0100 Subject: [PATCH 316/642] Python: Move adlfs import inline (#6497) --- pyiceberg/io/fsspec.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/pyiceberg/io/fsspec.py b/pyiceberg/io/fsspec.py index 9624562cb1..c5acaadade 100644 --- a/pyiceberg/io/fsspec.py +++ b/pyiceberg/io/fsspec.py @@ -28,12 +28,10 @@ from urllib.parse import urlparse import requests -from adlfs import AzureBlobFileSystem from botocore import UNSIGNED from botocore.awsrequest import AWSRequest from fsspec import AbstractFileSystem from requests import HTTPError -from s3fs import S3FileSystem from pyiceberg.catalog import TOKEN from pyiceberg.exceptions import SignError @@ -81,6 +79,8 @@ def s3v4_rest_signer(properties: Properties, request: AWSRequest, **_: Any) -> A def _s3(properties: Properties) -> AbstractFileSystem: + from s3fs import S3FileSystem + client_kwargs = { "endpoint_url": properties.get("s3.endpoint"), "aws_access_key_id": properties.get("s3.access-key-id"), @@ -110,6 +110,8 @@ def _s3(properties: Properties) -> AbstractFileSystem: def _adlfs(properties: Properties) -> AbstractFileSystem: + from adlfs import AzureBlobFileSystem + fs = AzureBlobFileSystem(**properties) return fs From 2ce966a1cfbd6830f07fe4b815e76a00109d28f7 Mon Sep 17 00:00:00 2001 From: Fokko Driesprong Date: Fri, 30 Dec 2022 23:07:24 +0100 Subject: [PATCH 317/642] Python: Projection by Field ID (#6437) * Python: Projection by Field ID instead of name * Comments * Add tests * WIP * Support nested structures * Cleanup * Add support for filtering on a column that's not projected * Python: Add SchemaWithPartnerVisitor, update pyarrow reads. (#339) * Comments * Fix nested list projection. * Update based on comments * Remove duplicate fixtures * Cleanup Co-authored-by: Ryan Blue --- pyiceberg/avro/resolver.py | 92 +---- pyiceberg/exceptions.py | 4 + pyiceberg/expressions/visitors.py | 89 +++++ pyiceberg/io/pyarrow.py | 217 +++++++++++- pyiceberg/schema.py | 216 ++++++++++- pyiceberg/table/__init__.py | 47 +-- tests/avro/test_decoder.py | 5 +- tests/avro/test_resolver.py | 58 +-- tests/io/test_pyarrow.py | 570 +++++++++++++++++++++++++++++- 9 files changed, 1138 insertions(+), 160 deletions(-) diff --git a/pyiceberg/avro/resolver.py b/pyiceberg/avro/resolver.py index ca559a2998..5542e8de3c 100644 --- a/pyiceberg/avro/resolver.py +++ b/pyiceberg/avro/resolver.py @@ -31,27 +31,19 @@ Reader, StructReader, ) -from pyiceberg.schema import Schema, visit +from pyiceberg.exceptions import ResolveError +from pyiceberg.schema import Schema, promote, visit from pyiceberg.types import ( - BinaryType, - DecimalType, DoubleType, FloatType, IcebergType, - IntegerType, ListType, - LongType, MapType, PrimitiveType, - StringType, StructType, ) -class ResolveException(Exception): - pass - - @singledispatch def resolve(file_schema: Union[Schema, IcebergType], read_schema: Union[Schema, IcebergType]) -> Reader: """This resolves the file and read schema @@ -79,7 +71,7 @@ def _(file_struct: StructType, read_struct: IcebergType) -> Reader: """Iterates over the file schema, and checks if the field is in the read schema""" if not isinstance(read_struct, StructType): - raise ResolveException(f"File/read schema are not aligned for {file_struct}, got {read_struct}") + raise ResolveError(f"File/read schema are not aligned for {file_struct}, got {read_struct}") results: List[Tuple[Optional[int], Reader]] = [] read_fields = {field.field_id: (pos, field) for pos, field in enumerate(read_struct.fields)} @@ -98,7 +90,7 @@ def _(file_struct: StructType, read_struct: IcebergType) -> Reader: for pos, read_field in enumerate(read_struct.fields): if read_field.field_id not in file_fields: if read_field.required: - raise ResolveException(f"{read_field} is non-optional, and not part of the file schema") + raise ResolveError(f"{read_field} is non-optional, and not part of the file schema") # Just set the new field to None results.append((pos, NoneReader())) @@ -108,7 +100,7 @@ def _(file_struct: StructType, read_struct: IcebergType) -> Reader: @resolve.register(ListType) def _(file_list: ListType, read_list: IcebergType) -> Reader: if not isinstance(read_list, ListType): - raise ResolveException(f"File/read schema are not aligned for {file_list}, got {read_list}") + raise ResolveError(f"File/read schema are not aligned for {file_list}, got {read_list}") element_reader = resolve(file_list.element_type, read_list.element_type) return ListReader(element_reader) @@ -116,79 +108,29 @@ def _(file_list: ListType, read_list: IcebergType) -> Reader: @resolve.register(MapType) def _(file_map: MapType, read_map: IcebergType) -> Reader: if not isinstance(read_map, MapType): - raise ResolveException(f"File/read schema are not aligned for {file_map}, got {read_map}") + raise ResolveError(f"File/read schema are not aligned for {file_map}, got {read_map}") key_reader = resolve(file_map.key_type, read_map.key_type) value_reader = resolve(file_map.value_type, read_map.value_type) return MapReader(key_reader, value_reader) +@resolve.register(FloatType) +def _(file_type: PrimitiveType, read_type: IcebergType) -> Reader: + """This is a special case, when we need to adhere to the bytes written""" + if isinstance(read_type, DoubleType): + return visit(file_type, ConstructReader()) + else: + raise ResolveError(f"Cannot promote an float to {read_type}") + + @resolve.register(PrimitiveType) def _(file_type: PrimitiveType, read_type: IcebergType) -> Reader: """Converting the primitive type into an actual reader that will decode the physical data""" if not isinstance(read_type, PrimitiveType): - raise ResolveException(f"Cannot promote {file_type} to {read_type}") + raise ResolveError(f"Cannot promote {file_type} to {read_type}") # In the case of a promotion, we want to check if it is valid if file_type != read_type: - return promote(file_type, read_type) + read_type = promote(file_type, read_type) return visit(read_type, ConstructReader()) - - -@singledispatch -def promote(file_type: IcebergType, read_type: IcebergType) -> Reader: - """Promotes reading a file type to a read type - - Args: - file_type (IcebergType): The type of the Avro file - read_type (IcebergType): The requested read type - - Raises: - ResolveException: If attempting to resolve an unrecognized object type - """ - raise ResolveException(f"Cannot promote {file_type} to {read_type}") - - -@promote.register(IntegerType) -def _(file_type: IntegerType, read_type: IcebergType) -> Reader: - if isinstance(read_type, LongType): - # Ints/Longs are binary compatible in Avro, so this is okay - return visit(read_type, ConstructReader()) - else: - raise ResolveException(f"Cannot promote an int to {read_type}") - - -@promote.register(FloatType) -def _(file_type: FloatType, read_type: IcebergType) -> Reader: - if isinstance(read_type, DoubleType): - # We should just read the float, and return it, since it both returns a float - return visit(file_type, ConstructReader()) - else: - raise ResolveException(f"Cannot promote an float to {read_type}") - - -@promote.register(StringType) -def _(file_type: StringType, read_type: IcebergType) -> Reader: - if isinstance(read_type, BinaryType): - return visit(read_type, ConstructReader()) - else: - raise ResolveException(f"Cannot promote an string to {read_type}") - - -@promote.register(BinaryType) -def _(file_type: BinaryType, read_type: IcebergType) -> Reader: - if isinstance(read_type, StringType): - return visit(read_type, ConstructReader()) - else: - raise ResolveException(f"Cannot promote an binary to {read_type}") - - -@promote.register(DecimalType) -def _(file_type: DecimalType, read_type: IcebergType) -> Reader: - if isinstance(read_type, DecimalType): - if file_type.precision <= read_type.precision and file_type.scale == file_type.scale: - return visit(read_type, ConstructReader()) - else: - raise ResolveException(f"Cannot reduce precision from {file_type} to {read_type}") - else: - raise ResolveException(f"Cannot promote an decimal to {read_type}") diff --git a/pyiceberg/exceptions.py b/pyiceberg/exceptions.py index 0438a5322a..69e40159ce 100644 --- a/pyiceberg/exceptions.py +++ b/pyiceberg/exceptions.py @@ -86,3 +86,7 @@ class NotInstalledError(Exception): class SignError(Exception): """Raises when unable to sign a S3 request""" + + +class ResolveError(Exception): + pass diff --git a/pyiceberg/expressions/visitors.py b/pyiceberg/expressions/visitors.py index f5c11a61cf..de7a489ab3 100644 --- a/pyiceberg/expressions/visitors.py +++ b/pyiceberg/expressions/visitors.py @@ -39,12 +39,15 @@ BoundIsNull, BoundLessThan, BoundLessThanOrEqual, + BoundLiteralPredicate, BoundNotEqualTo, BoundNotIn, BoundNotNaN, BoundNotNull, BoundPredicate, + BoundSetPredicate, BoundTerm, + BoundUnaryPredicate, L, Not, Or, @@ -753,3 +756,89 @@ def inclusive_projection( schema: Schema, spec: PartitionSpec, case_sensitive: bool = True ) -> Callable[[BooleanExpression], BooleanExpression]: return InclusiveProjection(schema, spec, case_sensitive).project + + +class _ColumnNameTranslator(BooleanExpressionVisitor[BooleanExpression]): + """Converts the column names with the ones in the actual file + + Args: + file_schema (Schema): The schema of the file + case_sensitive (bool): Whether to consider case when binding a reference to a field in a schema, defaults to True + + Raises: + TypeError: In the case of an UnboundPredicate + ValueError: When a column name cannot be found + """ + + file_schema: Schema + case_sensitive: bool + + def __init__(self, file_schema: Schema, case_sensitive: bool) -> None: + self.file_schema = file_schema + self.case_sensitive = case_sensitive + + def visit_true(self) -> BooleanExpression: + return AlwaysTrue() + + def visit_false(self) -> BooleanExpression: + return AlwaysFalse() + + def visit_not(self, child_result: BooleanExpression) -> BooleanExpression: + return Not(child=child_result) + + def visit_and(self, left_result: BooleanExpression, right_result: BooleanExpression) -> BooleanExpression: + return And(left=left_result, right=right_result) + + def visit_or(self, left_result: BooleanExpression, right_result: BooleanExpression) -> BooleanExpression: + return Or(left=left_result, right=right_result) + + def visit_unbound_predicate(self, predicate: UnboundPredicate[L]) -> BooleanExpression: + raise TypeError(f"Expected Bound Predicate, got: {predicate.term}") + + def visit_bound_predicate(self, predicate: BoundPredicate[L]) -> BooleanExpression: + file_column_name = self.file_schema.find_column_name(predicate.term.ref().field.field_id) + + if not file_column_name: + raise ValueError(f"Not found in file schema: {file_column_name}") + + if isinstance(predicate, BoundUnaryPredicate): + return predicate.as_unbound(file_column_name) + elif isinstance(predicate, BoundLiteralPredicate): + return predicate.as_unbound(file_column_name, predicate.literal) + elif isinstance(predicate, BoundSetPredicate): + return predicate.as_unbound(file_column_name, predicate.literals) + else: + raise ValueError(f"Unsupported predicate: {predicate}") + + +def translate_column_names(expr: BooleanExpression, file_schema: Schema, case_sensitive: bool) -> BooleanExpression: + return visit(expr, _ColumnNameTranslator(file_schema, case_sensitive)) + + +class _ExpressionFieldIDs(BooleanExpressionVisitor[Set[int]]): + """Extracts the field IDs used in the BooleanExpression""" + + def visit_true(self) -> Set[int]: + return set() + + def visit_false(self) -> Set[int]: + return set() + + def visit_not(self, child_result: Set[int]) -> Set[int]: + return child_result + + def visit_and(self, left_result: Set[int], right_result: Set[int]) -> Set[int]: + return left_result.union(right_result) + + def visit_or(self, left_result: Set[int], right_result: Set[int]) -> Set[int]: + return left_result.union(right_result) + + def visit_unbound_predicate(self, predicate: UnboundPredicate[L]) -> Set[int]: + raise ValueError("Only works on bound records") + + def visit_bound_predicate(self, predicate: BoundPredicate[L]) -> Set[int]: + return {predicate.term.ref().field.field_id} + + +def extract_field_ids(expr: BooleanExpression) -> Set[int]: + return visit(expr, _ExpressionFieldIDs()) diff --git a/pyiceberg/io/pyarrow.py b/pyiceberg/io/pyarrow.py index d5f7e80f62..b4c9024f2d 100644 --- a/pyiceberg/io/pyarrow.py +++ b/pyiceberg/io/pyarrow.py @@ -14,6 +14,7 @@ # KIND, either express or implied. See the License for the # specific language governing permissions and limitations # under the License. +# pylint: disable=redefined-outer-name,arguments-renamed """FileIO implementation for reading and writing table files that uses pyarrow.fs This file contains a FileIO implementation that relies on the filesystem interface provided @@ -21,13 +22,17 @@ type to use. Theoretically, this allows the supported storage types to grow naturally with the pyarrow library. """ +from __future__ import annotations import os from functools import lru_cache from typing import ( + TYPE_CHECKING, Any, Callable, + Iterable, List, + Optional, Set, Tuple, Union, @@ -36,6 +41,8 @@ import pyarrow as pa import pyarrow.compute as pc +import pyarrow.dataset as ds +import pyarrow.parquet as pq from pyarrow.fs import ( FileInfo, FileSystem, @@ -44,8 +51,19 @@ S3FileSystem, ) -from pyiceberg.expressions import BooleanExpression, BoundTerm, Literal -from pyiceberg.expressions.visitors import BoundBooleanExpressionVisitor +from pyiceberg.avro.resolver import ResolveError, promote +from pyiceberg.expressions import ( + AlwaysTrue, + BooleanExpression, + BoundTerm, + Literal, +) +from pyiceberg.expressions.visitors import ( + BoundBooleanExpressionVisitor, + bind, + extract_field_ids, + translate_column_names, +) from pyiceberg.expressions.visitors import visit as boolean_expression_visit from pyiceberg.io import ( FileIO, @@ -54,7 +72,15 @@ OutputFile, OutputStream, ) -from pyiceberg.schema import Schema, SchemaVisitorPerPrimitiveType, visit +from pyiceberg.schema import ( + PartnerAccessor, + Schema, + SchemaVisitorPerPrimitiveType, + SchemaWithPartnerVisitor, + prune_columns, + visit, + visit_with_partner, +) from pyiceberg.typedef import EMPTY_DICT, Properties from pyiceberg.types import ( BinaryType, @@ -80,6 +106,11 @@ ) from pyiceberg.utils.singleton import Singleton +if TYPE_CHECKING: + from pyiceberg.table import FileScanTask, Table + +ICEBERG_SCHEMA = b"iceberg.schema" + class PyArrowFile(InputFile, OutputFile): """A combined InputFile and OutputFile implementation that uses a pyarrow filesystem to generate pyarrow.lib.NativeFile instances @@ -195,7 +226,7 @@ def create(self, overwrite: bool = False) -> OutputStream: raise # pragma: no cover - If some other kind of OSError, raise the raw error return output_file - def to_input_file(self) -> "PyArrowFile": + def to_input_file(self) -> PyArrowFile: """Returns a new PyArrowFile for the location of an existing PyArrowFile instance This method is included to abide by the OutputFile abstract base class. Since this implementation uses a single @@ -287,7 +318,7 @@ def delete(self, location: Union[str, InputFile, OutputFile]) -> None: raise # pragma: no cover - If some other kind of OSError, raise the raw error -def schema_to_pyarrow(schema: Schema) -> pa.schema: +def schema_to_pyarrow(schema: Union[Schema, IcebergType]) -> pa.schema: return visit(schema, _ConvertToArrowSchema()) @@ -302,7 +333,7 @@ def field(self, field: NestedField, field_result: pa.DataType) -> pa.Field: return pa.field( name=field.name, type=field_result, - nullable=not field.required, + nullable=field.optional, metadata={"doc": field.doc, "id": str(field.field_id)} if field.doc else {}, ) @@ -423,3 +454,177 @@ def visit_or(self, left_result: pc.Expression, right_result: pc.Expression) -> p def expression_to_pyarrow(expr: BooleanExpression) -> pc.Expression: return boolean_expression_visit(expr, _ConvertToArrowExpression()) + + +def project_table( + files: Iterable[FileScanTask], table: Table, row_filter: BooleanExpression, projected_schema: Schema, case_sensitive: bool +) -> pa.Table: + """Resolves the right columns based on the identifier + + Args: + files(Iterable[FileScanTask]): A URI or a path to a local file + table(Table): The table that's being queried + row_filter(BooleanExpression): The expression for filtering rows + projected_schema(Schema): The output schema + case_sensitive(bool): Case sensitivity when looking up column names + + Raises: + ResolveError: When an incompatible query is done + """ + + if isinstance(table.io, PyArrowFileIO): + scheme, path = PyArrowFileIO.parse_location(table.location()) + fs = table.io.get_fs(scheme) + else: + raise ValueError(f"Expected PyArrowFileIO, got: {table.io}") + + bound_row_filter = bind(table.schema(), row_filter, case_sensitive=case_sensitive) + + projected_field_ids = { + id for id in projected_schema.field_ids if not isinstance(projected_schema.find_type(id), (MapType, ListType)) + }.union(extract_field_ids(bound_row_filter)) + + tables = [] + for task in files: + _, path = PyArrowFileIO.parse_location(task.file.file_path) + + # Get the schema + with fs.open_input_file(path) as fout: + parquet_schema = pq.read_schema(fout) + schema_raw = parquet_schema.metadata.get(ICEBERG_SCHEMA) + if schema_raw is None: + raise ValueError( + "Iceberg schema is not embedded into the Parquet file, see https://github.com/apache/iceberg/issues/6505" + ) + file_schema = Schema.parse_raw(schema_raw) + + pyarrow_filter = None + if row_filter is not AlwaysTrue(): + translated_row_filter = translate_column_names(bound_row_filter, file_schema, case_sensitive=case_sensitive) + bound_file_filter = bind(file_schema, translated_row_filter, case_sensitive=case_sensitive) + pyarrow_filter = expression_to_pyarrow(bound_file_filter) + + file_project_schema = prune_columns(file_schema, projected_field_ids, select_full_types=False) + + if file_schema is None: + raise ValueError(f"Missing Iceberg schema in Metadata for file: {path}") + + # Prune the stuff that we don't need anyway + file_project_schema_arrow = schema_to_pyarrow(file_project_schema) + + arrow_table = ds.dataset( + source=[path], schema=file_project_schema_arrow, format=ds.ParquetFileFormat(), filesystem=fs + ).to_table(filter=pyarrow_filter) + + tables.append(to_requested_schema(projected_schema, file_project_schema, arrow_table)) + + if len(tables) > 1: + return pa.concat_tables(tables) + else: + return tables[0] + + +def to_requested_schema(requested_schema: Schema, file_schema: Schema, table: pa.Table) -> pa.Table: + struct_array = visit_with_partner(requested_schema, table, ArrowProjectionVisitor(file_schema), ArrowAccessor(file_schema)) + + arrays = [] + fields = [] + for pos, field in enumerate(requested_schema.fields): + array = struct_array.field(pos) + arrays.append(array) + fields.append(pa.field(field.name, array.type, field.optional)) + return pa.Table.from_arrays(arrays, schema=pa.schema(fields)) + + +class ArrowProjectionVisitor(SchemaWithPartnerVisitor[pa.Array, Optional[pa.Array]]): + file_schema: Schema + + def __init__(self, file_schema: Schema): + self.file_schema = file_schema + + def cast_if_needed(self, field: NestedField, values: pa.Array) -> pa.Array: + file_field = self.file_schema.find_field(field.field_id) + if field.field_type.is_primitive and field.field_type != file_field.field_type: + return values.cast(schema_to_pyarrow(promote(file_field.field_type, field.field_type))) + return values + + def schema(self, schema: Schema, schema_partner: Optional[pa.Array], struct_result: Optional[pa.Array]) -> Optional[pa.Array]: + return struct_result + + def struct( + self, struct: StructType, struct_array: Optional[pa.Array], field_results: List[Optional[pa.Array]] + ) -> Optional[pa.Array]: + if struct_array is None: + return None + field_arrays: List[pa.Array] = [] + fields: List[pa.Field] = [] + for field, field_array in zip(struct.fields, field_results): + if field_array is not None: + array = self.cast_if_needed(field, field_array) + field_arrays.append(array) + fields.append(pa.field(field.name, array.type, field.optional)) + elif field.optional: + arrow_type = schema_to_pyarrow(field.field_type) + field_arrays.append(pa.nulls(len(struct_array), type=arrow_type)) + fields.append(pa.field(field.name, arrow_type, field.optional)) + else: + raise ResolveError(f"Field is required, and could not be found in the file: {field}") + + return pa.StructArray.from_arrays(arrays=field_arrays, fields=pa.struct(fields)) + + def field(self, field: NestedField, _: Optional[pa.Array], field_array: Optional[pa.Array]) -> Optional[pa.Array]: + return field_array + + def list(self, list_type: ListType, list_array: Optional[pa.Array], value_array: Optional[pa.Array]) -> Optional[pa.Array]: + return ( + pa.ListArray.from_arrays(list_array.offsets, self.cast_if_needed(list_type.element_field, value_array)) + if isinstance(list_array, pa.ListArray) + else None + ) + + def map( + self, map_type: MapType, map_array: Optional[pa.Array], key_result: Optional[pa.Array], value_result: Optional[pa.Array] + ) -> Optional[pa.Array]: + return ( + pa.MapArray.from_arrays( + map_array.offsets, + self.cast_if_needed(map_type.key_field, key_result), + self.cast_if_needed(map_type.value_field, value_result), + ) + if isinstance(map_array, pa.MapArray) + else None + ) + + def primitive(self, _: PrimitiveType, array: Optional[pa.Array]) -> Optional[pa.Array]: + return array + + +class ArrowAccessor(PartnerAccessor[pa.Array]): + file_schema: Schema + + def __init__(self, file_schema: Schema): + self.file_schema = file_schema + + def field_partner(self, partner_struct: Optional[pa.Array], field_id: int, _: str) -> Optional[pa.Array]: + if partner_struct: + # use the field name from the file schema + try: + name = self.file_schema.find_field(field_id).name + except ValueError: + return None + + if isinstance(partner_struct, pa.StructArray): + return partner_struct.field(name) + elif isinstance(partner_struct, pa.Table): + return partner_struct.column(name).combine_chunks() + + return None + + def list_element_partner(self, partner_list: Optional[pa.Array]) -> Optional[pa.Array]: + return partner_list.values if isinstance(partner_list, pa.ListArray) else None + + def map_key_partner(self, partner_map: Optional[pa.Array]) -> Optional[pa.Array]: + return partner_map.keys if isinstance(partner_map, pa.MapArray) else None + + def map_value_partner(self, partner_map: Optional[pa.Array]) -> Optional[pa.Array]: + return partner_map.items if isinstance(partner_map, pa.MapArray) else None diff --git a/pyiceberg/schema.py b/pyiceberg/schema.py index b265c444f1..33f5cf3dc0 100644 --- a/pyiceberg/schema.py +++ b/pyiceberg/schema.py @@ -35,7 +35,8 @@ from pydantic import Field, PrivateAttr -from pyiceberg.typedef import StructProtocol +from pyiceberg.exceptions import ResolveError +from pyiceberg.typedef import EMPTY_DICT, StructProtocol from pyiceberg.types import ( BinaryType, BooleanType, @@ -61,6 +62,7 @@ from pyiceberg.utils.iceberg_base_model import IcebergBaseModel T = TypeVar("T") +P = TypeVar("P") INITIAL_SCHEMA_ID = 0 @@ -248,6 +250,11 @@ def select(self, *names: str, case_sensitive: bool = True) -> "Schema": return prune_columns(self, ids) + @property + def field_ids(self) -> Set[int]: + """Returns the IDs of the current schema""" + return set(self._name_to_id.values()) + class SchemaVisitor(Generic[T], ABC): def before_field(self, field: NestedField) -> None: @@ -331,6 +338,142 @@ def primitive(self, primitive: PrimitiveType) -> T: """Visit a PrimitiveType""" +class SchemaWithPartnerVisitor(Generic[P, T], ABC): + def before_field(self, field: NestedField, field_partner: Optional[P]) -> None: + """Override this method to perform an action immediately before visiting a field""" + + def after_field(self, field: NestedField, field_partner: Optional[P]) -> None: + """Override this method to perform an action immediately after visiting a field""" + + def before_list_element(self, element: NestedField, element_partner: Optional[P]) -> None: + """Override this method to perform an action immediately before visiting an element within a ListType""" + self.before_field(element, element_partner) + + def after_list_element(self, element: NestedField, element_partner: Optional[P]) -> None: + """Override this method to perform an action immediately after visiting an element within a ListType""" + self.after_field(element, element_partner) + + def before_map_key(self, key: NestedField, key_partner: Optional[P]) -> None: + """Override this method to perform an action immediately before visiting a key within a MapType""" + self.before_field(key, key_partner) + + def after_map_key(self, key: NestedField, key_partner: Optional[P]) -> None: + """Override this method to perform an action immediately after visiting a key within a MapType""" + self.after_field(key, key_partner) + + def before_map_value(self, value: NestedField, value_partner: Optional[P]) -> None: + """Override this method to perform an action immediately before visiting a value within a MapType""" + self.before_field(value, value_partner) + + def after_map_value(self, value: NestedField, value_partner: Optional[P]) -> None: + """Override this method to perform an action immediately after visiting a value within a MapType""" + self.after_field(value, value_partner) + + @abstractmethod + def schema(self, schema: Schema, schema_partner: Optional[P], struct_result: T) -> T: + """Visit a schema with a partner""" + + @abstractmethod + def struct(self, struct: StructType, struct_partner: Optional[P], field_results: List[T]) -> T: + """Visit a struct type with a partner""" + + @abstractmethod + def field(self, field: NestedField, field_partner: Optional[P], field_result: T) -> T: + """Visit a nested field with a partner""" + + @abstractmethod + def list(self, list_type: ListType, list_partner: Optional[P], element_result: T) -> T: + """Visit a list type with a partner""" + + @abstractmethod + def map(self, map_type: MapType, map_partner: Optional[P], key_result: T, value_result: T) -> T: + """Visit a map type with a partner""" + + @abstractmethod + def primitive(self, primitive: PrimitiveType, primitive_partner: Optional[P]) -> T: + """Visit a primitive type with a partner""" + + +class PartnerAccessor(Generic[P], ABC): + @abstractmethod + def field_partner(self, partner_struct: Optional[P], field_id: int, field_name: str) -> Optional[P]: + """Returns the equivalent struct field by name or id in the partner struct""" + + @abstractmethod + def list_element_partner(self, partner_list: Optional[P]) -> Optional[P]: + """Returns the equivalent list element in the partner list""" + + @abstractmethod + def map_key_partner(self, partner_map: Optional[P]) -> Optional[P]: + """Returns the equivalent map key in the partner map""" + + @abstractmethod + def map_value_partner(self, partner_map: Optional[P]) -> Optional[P]: + """Returns the equivalent map value in the partner map""" + + +@singledispatch +def visit_with_partner( + schema_or_type: Union[Schema, IcebergType], partner: P, visitor: SchemaWithPartnerVisitor[T, P], accessor: PartnerAccessor[P] +) -> T: + raise ValueError(f"Unsupported type: {type}") + + +@visit_with_partner.register(Schema) +def _(schema: Schema, partner: P, visitor: SchemaWithPartnerVisitor[P, T], accessor: PartnerAccessor[P]) -> T: + return visitor.schema(schema, partner, visit_with_partner(schema.as_struct(), partner, visitor, accessor)) # type: ignore + + +@visit_with_partner.register(StructType) +def _(struct: StructType, partner: P, visitor: SchemaWithPartnerVisitor[P, T], accessor: PartnerAccessor[P]) -> T: + field_results = [] + for field in struct.fields: + field_partner = accessor.field_partner(partner, field.field_id, field.name) + visitor.before_field(field, field_partner) + try: + field_result = visit_with_partner(field.field_type, field_partner, visitor, accessor) # type: ignore + field_results.append(visitor.field(field, field_partner, field_result)) + finally: + visitor.after_field(field, field_partner) + + return visitor.struct(struct, partner, field_results) + + +@visit_with_partner.register(ListType) +def _(list_type: ListType, partner: P, visitor: SchemaWithPartnerVisitor[P, T], accessor: PartnerAccessor[P]) -> T: + element_partner = accessor.list_element_partner(partner) + visitor.before_list_element(list_type.element_field, element_partner) + try: + element_result = visit_with_partner(list_type.element_type, element_partner, visitor, accessor) # type: ignore + finally: + visitor.after_list_element(list_type.element_field, element_partner) + + return visitor.list(list_type, partner, element_result) + + +@visit_with_partner.register(MapType) +def _(map_type: MapType, partner: P, visitor: SchemaWithPartnerVisitor[P, T], accessor: PartnerAccessor[P]) -> T: + key_partner = accessor.map_key_partner(partner) + visitor.before_map_key(map_type.key_field, key_partner) + try: + key_result = visit_with_partner(map_type.key_type, key_partner, visitor, accessor) # type: ignore + finally: + visitor.after_map_key(map_type.key_field, key_partner) + + value_partner = accessor.map_value_partner(partner) + visitor.before_map_value(map_type.value_field, value_partner) + try: + value_result = visit_with_partner(map_type.value_type, value_partner, visitor, accessor) # type: ignore + finally: + visitor.after_map_value(map_type.value_field, value_partner) + return visitor.map(map_type, partner, key_result, value_result) + + +@visit_with_partner.register(PrimitiveType) +def _(primitive: PrimitiveType, partner: P, visitor: SchemaWithPartnerVisitor[P, T], _: PartnerAccessor[P]) -> T: + return visitor.primitive(primitive, partner) + + class SchemaVisitorPerPrimitiveType(SchemaVisitor[T], ABC): def primitive(self, primitive: PrimitiveType) -> T: """Visit a PrimitiveType""" @@ -727,9 +870,12 @@ def index_by_name(schema_or_type: Union[Schema, IcebergType]) -> Dict[str, int]: Returns: Dict[str, int]: An index of field names to field IDs """ - indexer = _IndexByName() - visit(schema_or_type, indexer) - return indexer.by_name() + if len(schema_or_type.fields) > 0: + indexer = _IndexByName() + visit(schema_or_type, indexer) + return indexer.by_name() + else: + return EMPTY_DICT def index_name_by_id(schema_or_type: Union[Schema, IcebergType]) -> Dict[int, str]: @@ -1046,3 +1192,65 @@ def _project_map(map_type: MapType, value_result: IcebergType) -> MapType: value_type=value_result, value_required=map_type.value_required, ) + + +@singledispatch +def promote(file_type: IcebergType, read_type: IcebergType) -> IcebergType: + """Promotes reading a file type to a read type + + Args: + file_type (IcebergType): The type of the Avro file + read_type (IcebergType): The requested read type + + Raises: + ResolveError: If attempting to resolve an unrecognized object type + """ + if file_type == read_type: + return file_type + else: + raise ResolveError(f"Cannot promote {file_type} to {read_type}") + + +@promote.register(IntegerType) +def _(file_type: IntegerType, read_type: IcebergType) -> IcebergType: + if isinstance(read_type, LongType): + # Ints/Longs are binary compatible in Avro, so this is okay + return read_type + else: + raise ResolveError(f"Cannot promote an int to {read_type}") + + +@promote.register(FloatType) +def _(file_type: FloatType, read_type: IcebergType) -> IcebergType: + if isinstance(read_type, DoubleType): + # A double type is wider + return read_type + else: + raise ResolveError(f"Cannot promote an float to {read_type}") + + +@promote.register(StringType) +def _(file_type: StringType, read_type: IcebergType) -> IcebergType: + if isinstance(read_type, BinaryType): + return read_type + else: + raise ResolveError(f"Cannot promote an string to {read_type}") + + +@promote.register(BinaryType) +def _(file_type: BinaryType, read_type: IcebergType) -> IcebergType: + if isinstance(read_type, StringType): + return read_type + else: + raise ResolveError(f"Cannot promote an binary to {read_type}") + + +@promote.register(DecimalType) +def _(file_type: DecimalType, read_type: IcebergType) -> IcebergType: + if isinstance(read_type, DecimalType): + if file_type.precision <= read_type.precision and file_type.scale == file_type.scale: + return read_type + else: + raise ResolveError(f"Cannot reduce precision from {file_type} to {read_type}") + else: + raise ResolveError(f"Cannot promote an decimal to {read_type}") diff --git a/pyiceberg/table/__init__.py b/pyiceberg/table/__init__.py index 5dbc2f22ae..3fb0702889 100644 --- a/pyiceberg/table/__init__.py +++ b/pyiceberg/table/__init__.py @@ -16,7 +16,6 @@ # under the License. from __future__ import annotations -import warnings from abc import ABC, abstractmethod from dataclasses import dataclass from functools import cached_property @@ -41,8 +40,9 @@ BooleanExpression, visitors, ) -from pyiceberg.expressions.visitors import bind, inclusive_projection +from pyiceberg.expressions.visitors import inclusive_projection from pyiceberg.io import FileIO +from pyiceberg.io.pyarrow import project_table from pyiceberg.manifest import DataFile, ManifestFile, files from pyiceberg.partitioning import PartitionSpec from pyiceberg.schema import Schema @@ -313,7 +313,7 @@ def _build_partition_evaluator(self, spec_id: int) -> Callable[[DataFile], bool] return lambda data_file: evaluator(wrapper.wrap(data_file.partition)) - def plan_files(self) -> Iterator[ScanTask]: + def plan_files(self) -> Iterator[FileScanTask]: snapshot = self.snapshot() if not snapshot: return @@ -344,47 +344,10 @@ def plan_files(self) -> Iterator[ScanTask]: yield from (FileScanTask(file) for file in matching_partition_files) def to_arrow(self) -> pa.Table: - from pyiceberg.io.pyarrow import PyArrowFileIO, expression_to_pyarrow, schema_to_pyarrow - - warnings.warn( - "Projection is currently done by name instead of Field ID, this can lead to incorrect results in some cases." + return project_table( + self.plan_files(), self.table, self.row_filter, self.projection(), case_sensitive=self.case_sensitive ) - fs = None - if isinstance(self.table.io, PyArrowFileIO): - scheme, path = PyArrowFileIO.parse_location(self.table.location()) - fs = self.table.io.get_fs(scheme) - - locations = [] - for task in self.plan_files(): - if isinstance(task, FileScanTask): - _, path = PyArrowFileIO.parse_location(task.file.file_path) - locations.append(path) - else: - raise ValueError(f"Cannot read unexpected task: {task}") - - columns = None - if "*" not in self.selected_fields: - columns = list(self.selected_fields) - - pyarrow_filter = None - if self.row_filter is not AlwaysTrue(): - bound_row_filter = bind(self.table.schema(), self.row_filter, case_sensitive=self.case_sensitive) - pyarrow_filter = expression_to_pyarrow(bound_row_filter) - - from pyarrow.dataset import dataset - - ds = dataset( - source=locations, - filesystem=fs, - # Optionally provide the Schema for the Dataset, - # in which case it will not be inferred from the source. - # https://arrow.apache.org/docs/python/generated/pyarrow.dataset.dataset.html#pyarrow.dataset.dataset - schema=schema_to_pyarrow(self.table.schema()), - ) - - return ds.to_table(filter=pyarrow_filter, columns=columns) - def to_pandas(self, **kwargs: Any) -> pd.DataFrame: return self.to_arrow().to_pandas(**kwargs) diff --git a/tests/avro/test_decoder.py b/tests/avro/test_decoder.py index e723270376..56b9cf7489 100644 --- a/tests/avro/test_decoder.py +++ b/tests/avro/test_decoder.py @@ -26,7 +26,7 @@ import pytest from pyiceberg.avro.decoder import BinaryDecoder -from pyiceberg.avro.resolver import promote +from pyiceberg.avro.resolver import resolve from pyiceberg.io import InputStream from pyiceberg.io.memory import MemoryInputStream from pyiceberg.types import DoubleType, FloatType @@ -225,6 +225,5 @@ def test_skip_utf8() -> None: def test_read_int_as_float() -> None: mis = MemoryInputStream(b"\x00\x00\x9A\x41") decoder = BinaryDecoder(mis) - reader = promote(FloatType(), DoubleType()) - + reader = resolve(FloatType(), DoubleType()) assert reader.read(decoder) == 19.25 diff --git a/tests/avro/test_resolver.py b/tests/avro/test_resolver.py index 853be4e448..f051881e0d 100644 --- a/tests/avro/test_resolver.py +++ b/tests/avro/test_resolver.py @@ -25,7 +25,7 @@ StringReader, StructReader, ) -from pyiceberg.avro.resolver import ResolveException, promote, resolve +from pyiceberg.avro.resolver import ResolveError, resolve from pyiceberg.schema import Schema from pyiceberg.types import ( BinaryType, @@ -101,7 +101,7 @@ def test_resolver_new_required_field() -> None: schema_id=1, ) - with pytest.raises(ResolveException) as exc_info: + with pytest.raises(ResolveError) as exc_info: resolve(write_schema, read_schema) assert "2: data: required string is non-optional, and not part of the file schema" in str(exc_info.value) @@ -117,7 +117,7 @@ def test_resolver_invalid_evolution() -> None: schema_id=1, ) - with pytest.raises(ResolveException) as exc_info: + with pytest.raises(ResolveError) as exc_info: resolve(write_schema, read_schema) assert "Cannot promote long to double" in str(exc_info.value) @@ -157,69 +157,69 @@ def test_resolver_change_type() -> None: schema_id=1, ) - with pytest.raises(ResolveException) as exc_info: + with pytest.raises(ResolveError) as exc_info: resolve(write_schema, read_schema) assert "File/read schema are not aligned for list, got map" in str(exc_info.value) -def test_promote_int_to_long() -> None: - assert promote(IntegerType(), LongType()) == IntegerReader() +def test_resolve_int_to_long() -> None: + assert resolve(IntegerType(), LongType()) == IntegerReader() -def test_promote_float_to_double() -> None: +def test_resolve_float_to_double() -> None: # We should still read floats, because it is encoded in 4 bytes - assert promote(FloatType(), DoubleType()) == FloatReader() + assert resolve(FloatType(), DoubleType()) == FloatReader() -def test_promote_decimal_to_decimal() -> None: +def test_resolve_decimal_to_decimal() -> None: # DecimalType(P, S) to DecimalType(P2, S) where P2 > P - assert promote(DecimalType(19, 25), DecimalType(22, 25)) == DecimalReader(22, 25) + assert resolve(DecimalType(19, 25), DecimalType(22, 25)) == DecimalReader(22, 25) def test_struct_not_aligned() -> None: - with pytest.raises(ResolveException): - assert promote(StructType(), StringType()) + with pytest.raises(ResolveError): + assert resolve(StructType(), StringType()) def test_map_not_aligned() -> None: - with pytest.raises(ResolveException): - assert promote(MapType(1, StringType(), 2, IntegerType()), StringType()) + with pytest.raises(ResolveError): + assert resolve(MapType(1, StringType(), 2, IntegerType()), StringType()) def test_primitive_not_aligned() -> None: - with pytest.raises(ResolveException): - assert promote(IntegerType(), MapType(1, StringType(), 2, IntegerType())) + with pytest.raises(ResolveError): + assert resolve(IntegerType(), MapType(1, StringType(), 2, IntegerType())) def test_integer_not_aligned() -> None: - with pytest.raises(ResolveException): - assert promote(IntegerType(), StringType()) + with pytest.raises(ResolveError): + assert resolve(IntegerType(), StringType()) def test_float_not_aligned() -> None: - with pytest.raises(ResolveException): - assert promote(FloatType(), StringType()) + with pytest.raises(ResolveError): + assert resolve(FloatType(), StringType()) def test_string_not_aligned() -> None: - with pytest.raises(ResolveException): - assert promote(StringType(), FloatType()) + with pytest.raises(ResolveError): + assert resolve(StringType(), FloatType()) def test_binary_not_aligned() -> None: - with pytest.raises(ResolveException): - assert promote(BinaryType(), FloatType()) + with pytest.raises(ResolveError): + assert resolve(BinaryType(), FloatType()) def test_decimal_not_aligned() -> None: - with pytest.raises(ResolveException): - assert promote(DecimalType(22, 19), StringType()) + with pytest.raises(ResolveError): + assert resolve(DecimalType(22, 19), StringType()) -def test_promote_decimal_to_decimal_reduce_precision() -> None: +def test_resolve_decimal_to_decimal_reduce_precision() -> None: # DecimalType(P, S) to DecimalType(P2, S) where P2 > P - with pytest.raises(ResolveException) as exc_info: - _ = promote(DecimalType(19, 25), DecimalType(10, 25)) == DecimalReader(22, 25) + with pytest.raises(ResolveError) as exc_info: + _ = resolve(DecimalType(19, 25), DecimalType(10, 25)) == DecimalReader(22, 25) assert "Cannot reduce precision from decimal(19, 25) to decimal(10, 25)" in str(exc_info.value) diff --git a/tests/io/test_pyarrow.py b/tests/io/test_pyarrow.py index 674331bf1c..02b7d02729 100644 --- a/tests/io/test_pyarrow.py +++ b/tests/io/test_pyarrow.py @@ -18,17 +18,20 @@ import os import tempfile -from typing import Any +from typing import Any, List, Optional from unittest.mock import MagicMock, patch import pyarrow as pa +import pyarrow.parquet as pq import pytest from pyarrow.fs import FileType +from pyiceberg.avro.resolver import ResolveError from pyiceberg.expressions import ( AlwaysFalse, AlwaysTrue, And, + BooleanExpression, BoundEqualTo, BoundGreaterThan, BoundGreaterThanOrEqual, @@ -42,6 +45,7 @@ BoundNotNaN, BoundNotNull, BoundReference, + GreaterThan, Not, Or, literal, @@ -52,9 +56,14 @@ PyArrowFileIO, _ConvertToArrowSchema, expression_to_pyarrow, + project_table, schema_to_pyarrow, ) +from pyiceberg.manifest import DataFile, FileFormat +from pyiceberg.partitioning import PartitionSpec from pyiceberg.schema import Schema, visit +from pyiceberg.table import FileScanTask, Table +from pyiceberg.table.metadata import TableMetadataV2 from pyiceberg.types import ( BinaryType, BooleanType, @@ -69,6 +78,7 @@ MapType, NestedField, StringType, + StructType, TimestampType, TimestamptzType, TimeType, @@ -572,3 +582,561 @@ def test_always_true_to_pyarrow(bound_reference: BoundReference[str]) -> None: def test_always_false_to_pyarrow(bound_reference: BoundReference[str]) -> None: assert repr(expression_to_pyarrow(AlwaysFalse())) == "" + + +@pytest.fixture +def schema_int() -> Schema: + return Schema(NestedField(1, "id", IntegerType(), required=False)) + + +@pytest.fixture +def schema_int_str() -> Schema: + return Schema(NestedField(1, "id", IntegerType(), required=False), NestedField(2, "data", StringType(), required=False)) + + +@pytest.fixture +def schema_str() -> Schema: + return Schema(NestedField(2, "data", StringType(), required=False)) + + +@pytest.fixture +def schema_long() -> Schema: + return Schema(NestedField(3, "id", LongType(), required=False)) + + +@pytest.fixture +def schema_struct() -> Schema: + return Schema( + NestedField( + 4, + "location", + StructType( + NestedField(41, "lat", DoubleType()), + NestedField(42, "long", DoubleType()), + ), + ) + ) + + +@pytest.fixture +def schema_list() -> Schema: + return Schema( + NestedField(5, "ids", ListType(51, IntegerType(), element_required=False), required=False), + ) + + +@pytest.fixture +def schema_list_of_structs() -> Schema: + return Schema( + NestedField( + 5, + "locations", + ListType( + 51, + StructType(NestedField(511, "lat", DoubleType()), NestedField(512, "long", DoubleType())), + element_required=False, + ), + required=False, + ), + ) + + +@pytest.fixture +def schema_map() -> Schema: + return Schema( + NestedField( + 5, + "properties", + MapType( + key_id=51, + key_type=StringType(), + value_id=52, + value_type=StringType(), + value_required=True, + ), + required=False, + ), + ) + + +def _write_table_to_file(filepath: str, schema: pa.Schema, table: pa.Table) -> str: + with pq.ParquetWriter(filepath, schema) as writer: + writer.write_table(table) + return filepath + + +@pytest.fixture +def file_int(schema_int: Schema, tmpdir: str) -> str: + pyarrow_schema = pa.schema(schema_to_pyarrow(schema_int), metadata={"iceberg.schema": schema_int.json()}) + return _write_table_to_file( + f"file:{tmpdir}/a.parquet", pyarrow_schema, pa.Table.from_arrays([pa.array([0, 1, 2])], schema=pyarrow_schema) + ) + + +@pytest.fixture +def file_int_str(schema_int_str: Schema, tmpdir: str) -> str: + pyarrow_schema = pa.schema(schema_to_pyarrow(schema_int_str), metadata={"iceberg.schema": schema_int_str.json()}) + return _write_table_to_file( + f"file:{tmpdir}/a.parquet", + pyarrow_schema, + pa.Table.from_arrays([pa.array([0, 1, 2]), pa.array(["0", "1", "2"])], schema=pyarrow_schema), + ) + + +@pytest.fixture +def file_string(schema_str: Schema, tmpdir: str) -> str: + pyarrow_schema = pa.schema(schema_to_pyarrow(schema_str), metadata={"iceberg.schema": schema_str.json()}) + return _write_table_to_file( + f"file:{tmpdir}/b.parquet", pyarrow_schema, pa.Table.from_arrays([pa.array(["0", "1", "2"])], schema=pyarrow_schema) + ) + + +@pytest.fixture +def file_long(schema_long: Schema, tmpdir: str) -> str: + pyarrow_schema = pa.schema(schema_to_pyarrow(schema_long), metadata={"iceberg.schema": schema_long.json()}) + return _write_table_to_file( + f"file:{tmpdir}/c.parquet", pyarrow_schema, pa.Table.from_arrays([pa.array([0, 1, 2])], schema=pyarrow_schema) + ) + + +@pytest.fixture +def file_struct(schema_struct: Schema, tmpdir: str) -> str: + pyarrow_schema = pa.schema(schema_to_pyarrow(schema_struct), metadata={"iceberg.schema": schema_struct.json()}) + return _write_table_to_file( + f"file:{tmpdir}/d.parquet", + pyarrow_schema, + pa.Table.from_pylist( + [ + {"location": {"lat": 52.371807, "long": 4.896029}}, + {"location": {"lat": 52.387386, "long": 4.646219}}, + {"location": {"lat": 52.078663, "long": 4.288788}}, + ], + schema=pyarrow_schema, + ), + ) + + +@pytest.fixture +def file_list(schema_list: Schema, tmpdir: str) -> str: + pyarrow_schema = pa.schema(schema_to_pyarrow(schema_list), metadata={"iceberg.schema": schema_list.json()}) + return _write_table_to_file( + f"file:{tmpdir}/e.parquet", + pyarrow_schema, + pa.Table.from_pylist( + [ + {"ids": list(range(1, 10))}, + {"ids": list(range(2, 20))}, + {"ids": list(range(3, 30))}, + ], + schema=pyarrow_schema, + ), + ) + + +@pytest.fixture +def file_list_of_structs(schema_list_of_structs: Schema, tmpdir: str) -> str: + pyarrow_schema = pa.schema( + schema_to_pyarrow(schema_list_of_structs), metadata={"iceberg.schema": schema_list_of_structs.json()} + ) + return _write_table_to_file( + f"file:{tmpdir}/e.parquet", + pyarrow_schema, + pa.Table.from_pylist( + [ + {"locations": [{"lat": 52.371807, "long": 4.896029}, {"lat": 52.387386, "long": 4.646219}]}, + {"locations": []}, + {"locations": [{"lat": 52.078663, "long": 4.288788}, {"lat": 52.387386, "long": 4.646219}]}, + ], + schema=pyarrow_schema, + ), + ) + + +@pytest.fixture +def file_map(schema_map: Schema, tmpdir: str) -> str: + pyarrow_schema = pa.schema(schema_to_pyarrow(schema_map), metadata={"iceberg.schema": schema_map.json()}) + return _write_table_to_file( + f"file:{tmpdir}/e.parquet", + pyarrow_schema, + pa.Table.from_pylist( + [ + {"properties": [("a", "b")]}, + {"properties": [("c", "d")]}, + {"properties": [("e", "f"), ("g", "h")]}, + ], + schema=pyarrow_schema, + ), + ) + + +def project( + schema: Schema, files: List[str], expr: Optional[BooleanExpression] = None, table_schema: Optional[Schema] = None +) -> pa.Table: + return project_table( + [ + FileScanTask( + DataFile(file_path=file, file_format=FileFormat.PARQUET, partition={}, record_count=3, file_size_in_bytes=3) + ) + for file in files + ], + Table( + ("namespace", "table"), + metadata=TableMetadataV2( + location="file://a/b/", + last_column_id=1, + format_version=2, + schemas=[table_schema or schema], + partition_specs=[PartitionSpec()], + ), + metadata_location="file://a/b/c.json", + io=PyArrowFileIO(), + ), + expr or AlwaysTrue(), + schema, + case_sensitive=True, + ) + + +def test_projection_add_column(file_int: str) -> None: + schema = Schema( + # All new IDs + NestedField(10, "id", IntegerType(), required=False), + NestedField(20, "list", ListType(21, IntegerType(), element_required=False), required=False), + NestedField( + 30, + "map", + MapType(key_id=31, key_type=IntegerType(), value_id=32, value_type=StringType(), value_required=False), + required=False, + ), + NestedField( + 40, + "location", + StructType( + NestedField(41, "lat", DoubleType(), required=False), NestedField(42, "lon", DoubleType(), required=False) + ), + required=False, + ), + ) + result_table = project(schema, [file_int]) + + for col in result_table.columns: + assert len(col) == 3 + + for actual, expected in zip(result_table.columns[0], [None, None, None]): + assert actual.as_py() == expected + + for actual, expected in zip(result_table.columns[1], [None, None, None]): + assert actual.as_py() == expected + + for actual, expected in zip(result_table.columns[2], [None, None, None]): + assert actual.as_py() == expected + + for actual, expected in zip(result_table.columns[3], [None, None, None]): + assert actual.as_py() == expected + + assert ( + repr(result_table.schema) + == """id: int32 +list: list + child 0, item: int32 +map: map + child 0, entries: struct not null + child 0, key: int32 not null + child 1, value: string +location: struct + child 0, lat: double + child 1, lon: double""" + ) + + +def test_read_list(schema_list: Schema, file_list: str) -> None: + result_table = project(schema_list, [file_list]) + + assert len(result_table.columns[0]) == 3 + for actual, expected in zip(result_table.columns[0], [list(range(1, 10)), list(range(2, 20)), list(range(3, 30))]): + assert actual.as_py() == expected + + assert repr(result_table.schema) == "ids: list\n child 0, item: int32" + + +def test_read_map(schema_map: Schema, file_map: str) -> None: + result_table = project(schema_map, [file_map]) + + assert len(result_table.columns[0]) == 3 + for actual, expected in zip(result_table.columns[0], [[("a", "b")], [("c", "d")], [("e", "f"), ("g", "h")]]): + assert actual.as_py() == expected + + assert ( + repr(result_table.schema) + == """properties: map + child 0, entries: struct not null + child 0, key: string not null + child 1, value: string""" + ) + + +def test_projection_add_column_struct(schema_int: Schema, file_int: str) -> None: + schema = Schema( + # A new ID + NestedField( + 2, + "id", + MapType(key_id=3, key_type=IntegerType(), value_id=4, value_type=StringType(), value_required=False), + required=False, + ) + ) + result_table = project(schema, [file_int]) + # Everything should be None + for r in result_table.columns[0]: + assert r.as_py() is None + + assert ( + repr(result_table.schema) + == """id: map + child 0, entries: struct not null + child 0, key: int32 not null + child 1, value: string""" + ) + + +def test_projection_add_column_struct_required(file_int: str) -> None: + schema = Schema( + # A new ID + NestedField( + 2, + "other_id", + IntegerType(), + required=True, + ) + ) + with pytest.raises(ResolveError) as exc_info: + _ = project(schema, [file_int]) + assert "Field is required, and could not be found in the file: 2: other_id: required int" in str(exc_info.value) + + +def test_projection_rename_column(schema_int: Schema, file_int: str) -> None: + schema = Schema( + # Reuses the id 1 + NestedField(1, "other_name", IntegerType()) + ) + result_table = project(schema, [file_int]) + assert len(result_table.columns[0]) == 3 + for actual, expected in zip(result_table.columns[0], [0, 1, 2]): + assert actual.as_py() == expected + + assert repr(result_table.schema) == "other_name: int32 not null" + + +def test_projection_concat_files(schema_int: Schema, file_int: str) -> None: + result_table = project(schema_int, [file_int, file_int]) + + for actual, expected in zip(result_table.columns[0], [0, 1, 2, 0, 1, 2]): + assert actual.as_py() == expected + assert len(result_table.columns[0]) == 6 + assert repr(result_table.schema) == "id: int32" + + +def test_projection_filter(schema_int: Schema, file_int: str) -> None: + result_table = project(schema_int, [file_int], GreaterThan("id", 4)) + assert len(result_table.columns[0]) == 0 + assert repr(result_table.schema) == "id: int32" + + +def test_projection_filter_renamed_column(file_int: str) -> None: + schema = Schema( + # Reuses the id 1 + NestedField(1, "other_id", IntegerType()) + ) + result_table = project(schema, [file_int], GreaterThan("other_id", 1)) + assert len(result_table.columns[0]) == 1 + assert repr(result_table.schema) == "other_id: int32 not null" + + +def test_projection_filter_add_column(schema_int: Schema, file_int: str, file_string: str) -> None: + """We have one file that has the column, and the other one doesn't""" + result_table = project(schema_int, [file_int, file_string]) + + for actual, expected in zip(result_table.columns[0], [0, 1, 2, None, None, None]): + assert actual.as_py() == expected + assert len(result_table.columns[0]) == 6 + assert repr(result_table.schema) == "id: int32" + + +def test_projection_filter_add_column_promote(file_int: str) -> None: + schema_long = Schema(NestedField(1, "id", LongType())) + result_table = project(schema_long, [file_int]) + + for actual, expected in zip(result_table.columns[0], [0, 1, 2]): + assert actual.as_py() == expected + assert len(result_table.columns[0]) == 3 + assert repr(result_table.schema) == "id: int64 not null" + + +def test_projection_filter_add_column_demote(file_long: str) -> None: + schema_int = Schema(NestedField(3, "id", IntegerType())) + with pytest.raises(ResolveError) as exc_info: + _ = project(schema_int, [file_long]) + assert "Cannot promote long to int" in str(exc_info.value) + + +def test_projection_nested_struct_subset(file_struct: str) -> None: + schema = Schema( + NestedField( + 4, + "location", + StructType( + NestedField(41, "lat", DoubleType()), + # long is missing! + ), + ) + ) + + result_table = project(schema, [file_struct]) + + for actual, expected in zip(result_table.columns[0], [52.371807, 52.387386, 52.078663]): + assert actual.as_py() == {"lat": expected} + + assert len(result_table.columns[0]) == 3 + assert repr(result_table.schema) == "location: struct not null\n child 0, lat: double not null" + + +def test_projection_nested_new_field(file_struct: str) -> None: + schema = Schema( + NestedField( + 4, + "location", + StructType( + NestedField(43, "null", DoubleType(), required=False), # Whoa, this column doesn't exist in the file + ), + ) + ) + + result_table = project(schema, [file_struct]) + + for actual, expected in zip(result_table.columns[0], [None, None, None]): + assert actual.as_py() == {"null": expected} + assert len(result_table.columns[0]) == 3 + assert repr(result_table.schema) == "location: struct not null\n child 0, null: double" + + +def test_projection_nested_struct(schema_struct: Schema, file_struct: str) -> None: + schema = Schema( + NestedField( + 4, + "location", + StructType( + NestedField(41, "lat", DoubleType(), required=False), + NestedField(43, "null", DoubleType(), required=False), + NestedField(42, "long", DoubleType(), required=False), + ), + ) + ) + + result_table = project(schema, [file_struct]) + for actual, expected in zip( + result_table.columns[0], + [ + {"lat": 52.371807, "long": 4.896029, "null": None}, + {"lat": 52.387386, "long": 4.646219, "null": None}, + {"lat": 52.078663, "long": 4.288788, "null": None}, + ], + ): + assert actual.as_py() == expected + assert len(result_table.columns[0]) == 3 + assert ( + repr(result_table.schema) + == "location: struct not null\n child 0, lat: double\n child 1, null: double\n child 2, long: double" + ) + + +def test_projection_list_of_structs(schema_list_of_structs: Schema, file_list_of_structs: str) -> None: + schema = Schema( + NestedField( + 5, + "locations", + ListType( + 51, + StructType( + NestedField(511, "latitude", DoubleType()), + NestedField(512, "longitude", DoubleType()), + NestedField(513, "altitude", DoubleType(), required=False), + ), + element_required=False, + ), + required=False, + ), + ) + + result_table = project(schema, [file_list_of_structs]) + assert len(result_table.columns) == 1 + assert len(result_table.columns[0]) == 3 + for actual, expected in zip( + result_table.columns[0], + [ + [ + {"latitude": 52.371807, "longitude": 4.896029, "altitude": None}, + {"latitude": 52.387386, "longitude": 4.646219, "altitude": None}, + ], + [], + [ + {"latitude": 52.078663, "longitude": 4.288788, "altitude": None}, + {"latitude": 52.387386, "longitude": 4.646219, "altitude": None}, + ], + ], + ): + assert actual.as_py() == expected + assert ( + repr(result_table.schema) + == """locations: list> + child 0, item: struct + child 0, latitude: double not null + child 1, longitude: double not null + child 2, altitude: double""" + ) + + +def test_projection_nested_struct_different_parent_id(file_struct: str) -> None: + schema = Schema( + NestedField( + 5, # 😱 this is 4 in the file, this will be fixed when projecting the file schema + "location", + StructType( + NestedField(41, "lat", DoubleType(), required=False), NestedField(42, "long", DoubleType(), required=False) + ), + required=False, + ) + ) + + result_table = project(schema, [file_struct]) + for actual, expected in zip(result_table.columns[0], [None, None, None]): + assert actual.as_py() == expected + assert len(result_table.columns[0]) == 3 + assert ( + repr(result_table.schema) + == """location: struct + child 0, lat: double + child 1, long: double""" + ) + + +def test_projection_filter_on_unprojected_field(schema_int_str: Schema, file_int_str: str) -> None: + schema = Schema(NestedField(1, "id", IntegerType())) + + result_table = project(schema, [file_int_str], GreaterThan("data", "1"), schema_int_str) + + for actual, expected in zip( + result_table.columns[0], + [2], + ): + assert actual.as_py() == expected + assert len(result_table.columns[0]) == 1 + assert repr(result_table.schema) == "id: int32 not null" + + +def test_projection_filter_on_unknown_field(schema_int_str: Schema, file_int_str: str) -> None: + schema = Schema(NestedField(1, "id", IntegerType())) + + with pytest.raises(ValueError) as exc_info: + _ = project(schema, [file_int_str], GreaterThan("unknown_field", "1"), schema_int_str) + + assert "Could not find field with name unknown_field, case_sensitive=True" in str(exc_info.value) From 237a02773c4920145f7b10795e68dbb63f778e5b Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 2 Jan 2023 09:11:40 +0100 Subject: [PATCH 318/642] Build: Bump pre-commit from 2.20.0 to 2.21.0 in /python (#6509) Bumps [pre-commit](https://github.com/pre-commit/pre-commit) from 2.20.0 to 2.21.0. - [Release notes](https://github.com/pre-commit/pre-commit/releases) - [Changelog](https://github.com/pre-commit/pre-commit/blob/main/CHANGELOG.md) - [Commits](https://github.com/pre-commit/pre-commit/compare/v2.20.0...v2.21.0) --- updated-dependencies: - dependency-name: pre-commit dependency-type: direct:development update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- poetry.lock | 11 +++++------ pyproject.toml | 2 +- 2 files changed, 6 insertions(+), 7 deletions(-) diff --git a/poetry.lock b/poetry.lock index e4cb39aece..0c17a9aefa 100644 --- a/poetry.lock +++ b/poetry.lock @@ -732,7 +732,7 @@ tests = ["pytest (>=5.4.1)", "pytest-cov (>=2.8.1)", "pytest-mypy (>=0.8.0)", "p [[package]] name = "pre-commit" -version = "2.20.0" +version = "2.21.0" description = "A framework for managing and maintaining multi-language pre-commit hooks." category = "dev" optional = false @@ -743,8 +743,7 @@ cfgv = ">=2.0.0" identify = ">=1.0.0" nodeenv = ">=0.11.1" pyyaml = ">=5.1" -toml = "*" -virtualenv = ">=20.0.8" +virtualenv = ">=20.10.0" [[package]] name = "pyarrow" @@ -1192,7 +1191,7 @@ snappy = ["python-snappy"] [metadata] lock-version = "1.1" python-versions = "^3.8" -content-hash = "565b8068555fa388cea32b84eb0d9a97efc0ef950b4467cbe6d92b0abe34bbb5" +content-hash = "d476ca7c2ecc999681b7cc3380488043686338a30e508c69ead925ef05e5b2f4" [metadata.files] adal = [ @@ -1965,8 +1964,8 @@ portalocker = [ {file = "portalocker-2.6.0.tar.gz", hash = "sha256:964f6830fb42a74b5d32bce99ed37d8308c1d7d44ddf18f3dd89f4680de97b39"}, ] pre-commit = [ - {file = "pre_commit-2.20.0-py2.py3-none-any.whl", hash = "sha256:51a5ba7c480ae8072ecdb6933df22d2f812dc897d5fe848778116129a681aac7"}, - {file = "pre_commit-2.20.0.tar.gz", hash = "sha256:a978dac7bc9ec0bcee55c18a277d553b0f419d259dadb4b9418ff2d00eb43959"}, + {file = "pre_commit-2.21.0-py2.py3-none-any.whl", hash = "sha256:e2f91727039fc39a92f58a588a25b87f936de6567eed4f0e673e0507edc75bad"}, + {file = "pre_commit-2.21.0.tar.gz", hash = "sha256:31ef31af7e474a8d8995027fefdfcf509b5c913ff31f2015b4ec4beb26a6f658"}, ] pyarrow = [ {file = "pyarrow-10.0.1-cp310-cp310-macosx_10_14_x86_64.whl", hash = "sha256:e00174764a8b4e9d8d5909b6d19ee0c217a6cf0232c5682e31fdfbd5a9f0ae52"}, diff --git a/pyproject.toml b/pyproject.toml index b46c5c027b..d4fd5cddaf 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -78,7 +78,7 @@ adlfs = { version = "2022.11.2", optional = true } [tool.poetry.dev-dependencies] pytest = "7.2.0" pytest-checkdocs = "2.9.0" -pre-commit = "2.20.0" +pre-commit = "2.21.0" fastavro = "1.7.0" coverage = { version = "^7.0.1", extras = ["toml"] } requests-mock = "1.10.0" From 7269f32d74b1d7c0bf68b3f78fb554a4c2b4cae0 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 2 Jan 2023 09:27:58 +0100 Subject: [PATCH 319/642] Build: Bump pydantic from 1.10.2 to 1.10.4 in /python (#6507) Bumps [pydantic](https://github.com/pydantic/pydantic) from 1.10.2 to 1.10.4. - [Release notes](https://github.com/pydantic/pydantic/releases) - [Changelog](https://github.com/pydantic/pydantic/blob/v1.10.4/HISTORY.md) - [Commits](https://github.com/pydantic/pydantic/compare/v1.10.2...v1.10.4) --- updated-dependencies: - dependency-name: pydantic dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- poetry.lock | 78 +++++++++++++++++++++++++------------------------- pyproject.toml | 2 +- 2 files changed, 40 insertions(+), 40 deletions(-) diff --git a/poetry.lock b/poetry.lock index 0c17a9aefa..d6600130de 100644 --- a/poetry.lock +++ b/poetry.lock @@ -766,14 +766,14 @@ python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*" [[package]] name = "pydantic" -version = "1.10.2" +version = "1.10.4" description = "Data validation and settings management using python type hints" category = "main" optional = false python-versions = ">=3.7" [package.dependencies] -typing-extensions = ">=4.1.0" +typing-extensions = ">=4.2.0" [package.extras] dotenv = ["python-dotenv (>=0.10.4)"] @@ -1191,7 +1191,7 @@ snappy = ["python-snappy"] [metadata] lock-version = "1.1" python-versions = "^3.8" -content-hash = "d476ca7c2ecc999681b7cc3380488043686338a30e508c69ead925ef05e5b2f4" +content-hash = "fbcc62ec8c07a67dc0bd1b043d66317ef5665d84b237bea34525425e257d052a" [metadata.files] adal = [ @@ -1999,42 +1999,42 @@ pycparser = [ {file = "pycparser-2.21.tar.gz", hash = "sha256:e644fdec12f7872f86c58ff790da456218b10f863970249516d60a5eaca77206"}, ] pydantic = [ - {file = "pydantic-1.10.2-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:bb6ad4489af1bac6955d38ebcb95079a836af31e4c4f74aba1ca05bb9f6027bd"}, - {file = "pydantic-1.10.2-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:a1f5a63a6dfe19d719b1b6e6106561869d2efaca6167f84f5ab9347887d78b98"}, - {file = "pydantic-1.10.2-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:352aedb1d71b8b0736c6d56ad2bd34c6982720644b0624462059ab29bd6e5912"}, - {file = "pydantic-1.10.2-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:19b3b9ccf97af2b7519c42032441a891a5e05c68368f40865a90eb88833c2559"}, - {file = "pydantic-1.10.2-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:e9069e1b01525a96e6ff49e25876d90d5a563bc31c658289a8772ae186552236"}, - {file = "pydantic-1.10.2-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:355639d9afc76bcb9b0c3000ddcd08472ae75318a6eb67a15866b87e2efa168c"}, - {file = "pydantic-1.10.2-cp310-cp310-win_amd64.whl", hash = "sha256:ae544c47bec47a86bc7d350f965d8b15540e27e5aa4f55170ac6a75e5f73b644"}, - {file = "pydantic-1.10.2-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:a4c805731c33a8db4b6ace45ce440c4ef5336e712508b4d9e1aafa617dc9907f"}, - {file = "pydantic-1.10.2-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:d49f3db871575e0426b12e2f32fdb25e579dea16486a26e5a0474af87cb1ab0a"}, - {file = "pydantic-1.10.2-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:37c90345ec7dd2f1bcef82ce49b6235b40f282b94d3eec47e801baf864d15525"}, - {file = "pydantic-1.10.2-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:7b5ba54d026c2bd2cb769d3468885f23f43710f651688e91f5fb1edcf0ee9283"}, - {file = "pydantic-1.10.2-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:05e00dbebbe810b33c7a7362f231893183bcc4251f3f2ff991c31d5c08240c42"}, - {file = "pydantic-1.10.2-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:2d0567e60eb01bccda3a4df01df677adf6b437958d35c12a3ac3e0f078b0ee52"}, - {file = "pydantic-1.10.2-cp311-cp311-win_amd64.whl", hash = "sha256:c6f981882aea41e021f72779ce2a4e87267458cc4d39ea990729e21ef18f0f8c"}, - {file = "pydantic-1.10.2-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:c4aac8e7103bf598373208f6299fa9a5cfd1fc571f2d40bf1dd1955a63d6eeb5"}, - {file = "pydantic-1.10.2-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:81a7b66c3f499108b448f3f004801fcd7d7165fb4200acb03f1c2402da73ce4c"}, - {file = "pydantic-1.10.2-cp37-cp37m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:bedf309630209e78582ffacda64a21f96f3ed2e51fbf3962d4d488e503420254"}, - {file = "pydantic-1.10.2-cp37-cp37m-musllinux_1_1_i686.whl", hash = "sha256:9300fcbebf85f6339a02c6994b2eb3ff1b9c8c14f502058b5bf349d42447dcf5"}, - {file = "pydantic-1.10.2-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:216f3bcbf19c726b1cc22b099dd409aa371f55c08800bcea4c44c8f74b73478d"}, - {file = "pydantic-1.10.2-cp37-cp37m-win_amd64.whl", hash = "sha256:dd3f9a40c16daf323cf913593083698caee97df2804aa36c4b3175d5ac1b92a2"}, - {file = "pydantic-1.10.2-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:b97890e56a694486f772d36efd2ba31612739bc6f3caeee50e9e7e3ebd2fdd13"}, - {file = "pydantic-1.10.2-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:9cabf4a7f05a776e7793e72793cd92cc865ea0e83a819f9ae4ecccb1b8aa6116"}, - {file = "pydantic-1.10.2-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:06094d18dd5e6f2bbf93efa54991c3240964bb663b87729ac340eb5014310624"}, - {file = "pydantic-1.10.2-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:cc78cc83110d2f275ec1970e7a831f4e371ee92405332ebfe9860a715f8336e1"}, - {file = "pydantic-1.10.2-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:1ee433e274268a4b0c8fde7ad9d58ecba12b069a033ecc4645bb6303c062d2e9"}, - {file = "pydantic-1.10.2-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:7c2abc4393dea97a4ccbb4ec7d8658d4e22c4765b7b9b9445588f16c71ad9965"}, - {file = "pydantic-1.10.2-cp38-cp38-win_amd64.whl", hash = "sha256:0b959f4d8211fc964772b595ebb25f7652da3f22322c007b6fed26846a40685e"}, - {file = "pydantic-1.10.2-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:c33602f93bfb67779f9c507e4d69451664524389546bacfe1bee13cae6dc7488"}, - {file = "pydantic-1.10.2-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:5760e164b807a48a8f25f8aa1a6d857e6ce62e7ec83ea5d5c5a802eac81bad41"}, - {file = "pydantic-1.10.2-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:6eb843dcc411b6a2237a694f5e1d649fc66c6064d02b204a7e9d194dff81eb4b"}, - {file = "pydantic-1.10.2-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:4b8795290deaae348c4eba0cebb196e1c6b98bdbe7f50b2d0d9a4a99716342fe"}, - {file = "pydantic-1.10.2-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:e0bedafe4bc165ad0a56ac0bd7695df25c50f76961da29c050712596cf092d6d"}, - {file = "pydantic-1.10.2-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:2e05aed07fa02231dbf03d0adb1be1d79cabb09025dd45aa094aa8b4e7b9dcda"}, - {file = "pydantic-1.10.2-cp39-cp39-win_amd64.whl", hash = "sha256:c1ba1afb396148bbc70e9eaa8c06c1716fdddabaf86e7027c5988bae2a829ab6"}, - {file = "pydantic-1.10.2-py3-none-any.whl", hash = "sha256:1b6ee725bd6e83ec78b1aa32c5b1fa67a3a65badddde3976bca5fe4568f27709"}, - {file = "pydantic-1.10.2.tar.gz", hash = "sha256:91b8e218852ef6007c2b98cd861601c6a09f1aa32bbbb74fab5b1c33d4a1e410"}, + {file = "pydantic-1.10.4-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:b5635de53e6686fe7a44b5cf25fcc419a0d5e5c1a1efe73d49d48fe7586db854"}, + {file = "pydantic-1.10.4-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:6dc1cc241440ed7ca9ab59d9929075445da6b7c94ced281b3dd4cfe6c8cff817"}, + {file = "pydantic-1.10.4-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:51bdeb10d2db0f288e71d49c9cefa609bca271720ecd0c58009bd7504a0c464c"}, + {file = "pydantic-1.10.4-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:78cec42b95dbb500a1f7120bdf95c401f6abb616bbe8785ef09887306792e66e"}, + {file = "pydantic-1.10.4-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:8775d4ef5e7299a2f4699501077a0defdaac5b6c4321173bcb0f3c496fbadf85"}, + {file = "pydantic-1.10.4-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:572066051eeac73d23f95ba9a71349c42a3e05999d0ee1572b7860235b850cc6"}, + {file = "pydantic-1.10.4-cp310-cp310-win_amd64.whl", hash = "sha256:7feb6a2d401f4d6863050f58325b8d99c1e56f4512d98b11ac64ad1751dc647d"}, + {file = "pydantic-1.10.4-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:39f4a73e5342b25c2959529f07f026ef58147249f9b7431e1ba8414a36761f53"}, + {file = "pydantic-1.10.4-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:983e720704431a6573d626b00662eb78a07148c9115129f9b4351091ec95ecc3"}, + {file = "pydantic-1.10.4-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:75d52162fe6b2b55964fbb0af2ee58e99791a3138588c482572bb6087953113a"}, + {file = "pydantic-1.10.4-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:fdf8d759ef326962b4678d89e275ffc55b7ce59d917d9f72233762061fd04a2d"}, + {file = "pydantic-1.10.4-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:05a81b006be15655b2a1bae5faa4280cf7c81d0e09fcb49b342ebf826abe5a72"}, + {file = "pydantic-1.10.4-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:d88c4c0e5c5dfd05092a4b271282ef0588e5f4aaf345778056fc5259ba098857"}, + {file = "pydantic-1.10.4-cp311-cp311-win_amd64.whl", hash = "sha256:6a05a9db1ef5be0fe63e988f9617ca2551013f55000289c671f71ec16f4985e3"}, + {file = "pydantic-1.10.4-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:887ca463c3bc47103c123bc06919c86720e80e1214aab79e9b779cda0ff92a00"}, + {file = "pydantic-1.10.4-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:fdf88ab63c3ee282c76d652fc86518aacb737ff35796023fae56a65ced1a5978"}, + {file = "pydantic-1.10.4-cp37-cp37m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:a48f1953c4a1d9bd0b5167ac50da9a79f6072c63c4cef4cf2a3736994903583e"}, + {file = "pydantic-1.10.4-cp37-cp37m-musllinux_1_1_i686.whl", hash = "sha256:a9f2de23bec87ff306aef658384b02aa7c32389766af3c5dee9ce33e80222dfa"}, + {file = "pydantic-1.10.4-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:cd8702c5142afda03dc2b1ee6bc358b62b3735b2cce53fc77b31ca9f728e4bc8"}, + {file = "pydantic-1.10.4-cp37-cp37m-win_amd64.whl", hash = "sha256:6e7124d6855b2780611d9f5e1e145e86667eaa3bd9459192c8dc1a097f5e9903"}, + {file = "pydantic-1.10.4-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:0b53e1d41e97063d51a02821b80538053ee4608b9a181c1005441f1673c55423"}, + {file = "pydantic-1.10.4-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:55b1625899acd33229c4352ce0ae54038529b412bd51c4915349b49ca575258f"}, + {file = "pydantic-1.10.4-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:301d626a59edbe5dfb48fcae245896379a450d04baeed50ef40d8199f2733b06"}, + {file = "pydantic-1.10.4-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:b6f9d649892a6f54a39ed56b8dfd5e08b5f3be5f893da430bed76975f3735d15"}, + {file = "pydantic-1.10.4-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:d7b5a3821225f5c43496c324b0d6875fde910a1c2933d726a743ce328fbb2a8c"}, + {file = "pydantic-1.10.4-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:f2f7eb6273dd12472d7f218e1fef6f7c7c2f00ac2e1ecde4db8824c457300416"}, + {file = "pydantic-1.10.4-cp38-cp38-win_amd64.whl", hash = "sha256:4b05697738e7d2040696b0a66d9f0a10bec0efa1883ca75ee9e55baf511909d6"}, + {file = "pydantic-1.10.4-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:a9a6747cac06c2beb466064dda999a13176b23535e4c496c9d48e6406f92d42d"}, + {file = "pydantic-1.10.4-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:eb992a1ef739cc7b543576337bebfc62c0e6567434e522e97291b251a41dad7f"}, + {file = "pydantic-1.10.4-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:990406d226dea0e8f25f643b370224771878142155b879784ce89f633541a024"}, + {file = "pydantic-1.10.4-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:2e82a6d37a95e0b1b42b82ab340ada3963aea1317fd7f888bb6b9dfbf4fff57c"}, + {file = "pydantic-1.10.4-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:9193d4f4ee8feca58bc56c8306bcb820f5c7905fd919e0750acdeeeef0615b28"}, + {file = "pydantic-1.10.4-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:2b3ce5f16deb45c472dde1a0ee05619298c864a20cded09c4edd820e1454129f"}, + {file = "pydantic-1.10.4-cp39-cp39-win_amd64.whl", hash = "sha256:9cbdc268a62d9a98c56e2452d6c41c0263d64a2009aac69246486f01b4f594c4"}, + {file = "pydantic-1.10.4-py3-none-any.whl", hash = "sha256:4948f264678c703f3877d1c8877c4e3b2e12e549c57795107f08cf70c6ec7774"}, + {file = "pydantic-1.10.4.tar.gz", hash = "sha256:b9a3859f24eb4e097502a3be1fb4b2abb79b6103dd9e2e0edb70613a4459a648"}, ] pygments = [ {file = "Pygments-2.13.0-py3-none-any.whl", hash = "sha256:f643f331ab57ba3c9d89212ee4a2dabc6e94f117cf4eefde99a0574720d14c42"}, diff --git a/pyproject.toml b/pyproject.toml index d4fd5cddaf..d0d089c462 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -53,7 +53,7 @@ click = "8.1.3" rich = "12.6.0" pyyaml = "6.0.0" -pydantic = "1.10.2" +pydantic = "1.10.4" fsspec = "2022.10.0" pyparsing = "3.0.9" From 9f950b2bcc3d5397038dc973aed7681ad9907175 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 2 Jan 2023 09:42:41 +0100 Subject: [PATCH 320/642] Build: Bump rich from 12.6.0 to 13.0.0 in /python (#6508) Bumps [rich](https://github.com/Textualize/rich) from 12.6.0 to 13.0.0. - [Release notes](https://github.com/Textualize/rich/releases) - [Changelog](https://github.com/Textualize/rich/blob/master/CHANGELOG.md) - [Commits](https://github.com/Textualize/rich/compare/v12.6.0...v13.0.0) --- updated-dependencies: - dependency-name: rich dependency-type: direct:production update-type: version-update:semver-major ... Signed-off-by: dependabot[bot] Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- poetry.lock | 10 +++++----- pyproject.toml | 2 +- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/poetry.lock b/poetry.lock index d6600130de..736b84dc30 100644 --- a/poetry.lock +++ b/poetry.lock @@ -966,11 +966,11 @@ tests = ["coverage (>=6.0.0)", "flake8", "mypy", "pytest (>=7.0.0)", "pytest-asy [[package]] name = "rich" -version = "12.6.0" +version = "13.0.0" description = "Render rich text, tables, progress bars, syntax highlighting, markdown and more to the terminal" category = "main" optional = false -python-versions = ">=3.6.3,<4.0.0" +python-versions = ">=3.7.0" [package.dependencies] commonmark = ">=0.9.0,<0.10.0" @@ -1191,7 +1191,7 @@ snappy = ["python-snappy"] [metadata] lock-version = "1.1" python-versions = "^3.8" -content-hash = "fbcc62ec8c07a67dc0bd1b043d66317ef5665d84b237bea34525425e257d052a" +content-hash = "2c443e612fc0e2253a3d1ed8b3c9761817c3ab2b35d593aa6403faa3d46e7195" [metadata.files] adal = [ @@ -2189,8 +2189,8 @@ responses = [ {file = "responses-0.22.0.tar.gz", hash = "sha256:396acb2a13d25297789a5866b4881cf4e46ffd49cc26c43ab1117f40b973102e"}, ] rich = [ - {file = "rich-12.6.0-py3-none-any.whl", hash = "sha256:a4eb26484f2c82589bd9a17c73d32a010b1e29d89f1604cd9bf3a2097b81bb5e"}, - {file = "rich-12.6.0.tar.gz", hash = "sha256:ba3a3775974105c221d31141f2c116f4fd65c5ceb0698657a11e9f295ec93fd0"}, + {file = "rich-13.0.0-py3-none-any.whl", hash = "sha256:12b1d77ee7edf251b741531323f0d990f5f570a4e7c054d0bfb59fb7981ad977"}, + {file = "rich-13.0.0.tar.gz", hash = "sha256:3aa9eba7219b8c575c6494446a59f702552efe1aa261e7eeb95548fa586e1950"}, ] s3fs = [ {file = "s3fs-2022.10.0-py3-none-any.whl", hash = "sha256:1e134c3577171699feb7c1a0c4713260d5b48296e1708737ff940baef6e2c153"}, diff --git a/pyproject.toml b/pyproject.toml index d0d089c462..ce1fa9586d 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -50,7 +50,7 @@ python = "^3.8" mmhash3 = "3.0.1" requests = "2.28.1" click = "8.1.3" -rich = "12.6.0" +rich = "13.0.0" pyyaml = "6.0.0" pydantic = "1.10.4" From 060c1827a6d2d36c639871e5c88af1876e70b333 Mon Sep 17 00:00:00 2001 From: Ryan Blue Date: Mon, 2 Jan 2023 10:18:21 -0800 Subject: [PATCH 321/642] Python: Refactor Avro read path to use a partner visitor (#6506) --- pyiceberg/avro/file.py | 32 +++-- pyiceberg/avro/reader.py | 115 ++++----------- pyiceberg/avro/resolver.py | 271 +++++++++++++++++++++++++----------- pyiceberg/io/pyarrow.py | 6 +- pyiceberg/schema.py | 102 +++++++++++++- pyiceberg/typedef.py | 5 + pyiceberg/types.py | 6 + tests/avro/test_reader.py | 34 ++--- tests/avro/test_resolver.py | 4 +- 9 files changed, 371 insertions(+), 204 deletions(-) diff --git a/pyiceberg/avro/file.py b/pyiceberg/avro/file.py index cc725e9db7..50c85102ad 100644 --- a/pyiceberg/avro/file.py +++ b/pyiceberg/avro/file.py @@ -24,16 +24,21 @@ from dataclasses import dataclass from io import SEEK_SET, BufferedReader from types import TracebackType -from typing import Optional, Type +from typing import ( + Callable, + Dict, + Optional, + Type, +) from pyiceberg.avro.codecs import KNOWN_CODECS, Codec from pyiceberg.avro.decoder import BinaryDecoder -from pyiceberg.avro.reader import ConstructReader, Reader -from pyiceberg.avro.resolver import resolve +from pyiceberg.avro.reader import Reader +from pyiceberg.avro.resolver import construct_reader, resolve from pyiceberg.io import InputFile, InputStream from pyiceberg.io.memory import MemoryInputStream -from pyiceberg.schema import Schema, visit -from pyiceberg.typedef import Record +from pyiceberg.schema import Schema +from pyiceberg.typedef import EMPTY_DICT, Record, StructProtocol from pyiceberg.types import ( FixedType, MapType, @@ -112,6 +117,7 @@ def __next__(self) -> Record: class AvroFile: input_file: InputFile read_schema: Optional[Schema] + read_types: Dict[int, Callable[[Schema], StructProtocol]] input_stream: InputStream header: AvroFileHeader schema: Schema @@ -120,9 +126,15 @@ class AvroFile: decoder: BinaryDecoder block: Optional[Block] = None - def __init__(self, input_file: InputFile, read_schema: Optional[Schema] = None) -> None: + def __init__( + self, + input_file: InputFile, + read_schema: Optional[Schema] = None, + read_types: Dict[int, Callable[[Schema], StructProtocol]] = EMPTY_DICT, + ) -> None: self.input_file = input_file self.read_schema = read_schema + self.read_types = read_types def __enter__(self) -> AvroFile: """ @@ -137,9 +149,9 @@ def __enter__(self) -> AvroFile: self.header = self._read_header() self.schema = self.header.get_schema() if not self.read_schema: - self.reader = visit(self.schema, ConstructReader()) - else: - self.reader = resolve(self.schema, self.read_schema) + self.read_schema = self.schema + + self.reader = resolve(self.schema, self.read_schema, self.read_types) return self @@ -184,6 +196,6 @@ def __next__(self) -> Record: def _read_header(self) -> AvroFileHeader: self.input_stream.seek(0, SEEK_SET) - reader = visit(META_SCHEMA, ConstructReader()) + reader = construct_reader(META_SCHEMA) _header = reader.read(self.decoder) return AvroFileHeader(magic=_header.get(0), meta=_header.get(1), sync=_header.get(2)) diff --git a/pyiceberg/avro/reader.py b/pyiceberg/avro/reader.py index d1b114cc5b..f7a0194a9a 100644 --- a/pyiceberg/avro/reader.py +++ b/pyiceberg/avro/reader.py @@ -37,33 +37,11 @@ List, Optional, Tuple, - Union, ) from uuid import UUID from pyiceberg.avro.decoder import BinaryDecoder -from pyiceberg.schema import Schema, SchemaVisitorPerPrimitiveType from pyiceberg.typedef import Record, StructProtocol -from pyiceberg.types import ( - BinaryType, - BooleanType, - DateType, - DecimalType, - DoubleType, - FixedType, - FloatType, - IntegerType, - ListType, - LongType, - MapType, - NestedField, - StringType, - StructType, - TimestampType, - TimestamptzType, - TimeType, - UUIDType, -) from pyiceberg.utils.singleton import Singleton @@ -260,25 +238,43 @@ def skip(self, decoder: BinaryDecoder) -> None: return self.option.skip(decoder) -@dataclass(frozen=True) -class StructReader(Reader): - fields: Tuple[Tuple[Optional[int], Reader], ...] = dataclassfield() +class StructProtocolReader(Reader): + create_struct: Callable[[], StructProtocol] + fields: Tuple[Tuple[Optional[int], Reader], ...] + + def __init__(self, fields: Tuple[Tuple[Optional[int], Reader], ...], create_struct: Callable[[], StructProtocol]): + self.create_struct = create_struct + self.fields = fields + + def create_or_reuse(self, reuse: Optional[StructProtocol]) -> StructProtocol: + if reuse: + return reuse + else: + return self.create_struct() + + def read(self, decoder: BinaryDecoder) -> Any: + struct = self.create_or_reuse(None) - def read(self, decoder: BinaryDecoder) -> Record: - result: List[Union[Any, StructProtocol]] = [None] * len(self.fields) for (pos, field) in self.fields: if pos is not None: - result[pos] = field.read(decoder) + struct.set(pos, field.read(decoder)) # later: pass reuse in here else: field.skip(decoder) - return Record(*result) + return struct def skip(self, decoder: BinaryDecoder) -> None: for _, field in self.fields: field.skip(decoder) +class StructReader(StructProtocolReader): + fields: Tuple[Tuple[Optional[int], Reader], ...] + + def __init__(self, fields: Tuple[Tuple[Optional[int], Reader], ...]): + super().__init__(fields, lambda: Record.of(len(fields))) + + @dataclass(frozen=True) class ListReader(Reader): element: Reader @@ -325,64 +321,3 @@ def skip() -> None: self.value.skip(decoder) _skip_map_array(decoder, skip) - - -class ConstructReader(SchemaVisitorPerPrimitiveType[Reader]): - def schema(self, schema: Schema, struct_result: Reader) -> Reader: - return struct_result - - def struct(self, struct: StructType, field_results: List[Reader]) -> Reader: - return StructReader(tuple(enumerate(field_results))) - - def field(self, field: NestedField, field_result: Reader) -> Reader: - return field_result if field.required else OptionReader(field_result) - - def list(self, list_type: ListType, element_result: Reader) -> Reader: - element_reader = element_result if list_type.element_required else OptionReader(element_result) - return ListReader(element_reader) - - def map(self, map_type: MapType, key_result: Reader, value_result: Reader) -> Reader: - value_reader = value_result if map_type.value_required else OptionReader(value_result) - return MapReader(key_result, value_reader) - - def visit_fixed(self, fixed_type: FixedType) -> Reader: - return FixedReader(len(fixed_type)) - - def visit_decimal(self, decimal_type: DecimalType) -> Reader: - return DecimalReader(decimal_type.precision, decimal_type.scale) - - def visit_boolean(self, boolean_type: BooleanType) -> Reader: - return BooleanReader() - - def visit_integer(self, integer_type: IntegerType) -> Reader: - return IntegerReader() - - def visit_long(self, long_type: LongType) -> Reader: - return IntegerReader() - - def visit_float(self, float_type: FloatType) -> Reader: - return FloatReader() - - def visit_double(self, double_type: DoubleType) -> Reader: - return DoubleReader() - - def visit_date(self, date_type: DateType) -> Reader: - return DateReader() - - def visit_time(self, time_type: TimeType) -> Reader: - return TimeReader() - - def visit_timestamp(self, timestamp_type: TimestampType) -> Reader: - return TimestampReader() - - def visit_timestampz(self, timestamptz_type: TimestamptzType) -> Reader: - return TimestamptzReader() - - def visit_string(self, string_type: StringType) -> Reader: - return StringReader() - - def visit_uuid(self, uuid_type: UUIDType) -> Reader: - return UUIDReader() - - def visit_binary(self, binary_ype: BinaryType) -> Reader: - return BinaryReader() diff --git a/pyiceberg/avro/resolver.py b/pyiceberg/avro/resolver.py index 5542e8de3c..a53693f415 100644 --- a/pyiceberg/avro/resolver.py +++ b/pyiceberg/avro/resolver.py @@ -14,8 +14,10 @@ # KIND, either express or implied. See the License for the # specific language governing permissions and limitations # under the License. -from functools import singledispatch +# pylint: disable=arguments-renamed,unused-argument from typing import ( + Callable, + Dict, List, Optional, Tuple, @@ -23,114 +25,221 @@ ) from pyiceberg.avro.reader import ( - ConstructReader, + BinaryReader, + BooleanReader, + DateReader, + DecimalReader, + DoubleReader, + FixedReader, + FloatReader, + IntegerReader, ListReader, MapReader, NoneReader, OptionReader, Reader, + StringReader, StructReader, + TimeReader, + TimestampReader, + TimestamptzReader, + UUIDReader, ) from pyiceberg.exceptions import ResolveError -from pyiceberg.schema import Schema, promote, visit +from pyiceberg.schema import ( + PartnerAccessor, + PrimitiveWithPartnerVisitor, + Schema, + promote, + visit_with_partner, +) +from pyiceberg.typedef import EMPTY_DICT, StructProtocol from pyiceberg.types import ( + BinaryType, + BooleanType, + DateType, + DecimalType, DoubleType, + FixedType, FloatType, IcebergType, + IntegerType, ListType, + LongType, MapType, + NestedField, PrimitiveType, + StringType, StructType, + TimestampType, + TimestamptzType, + TimeType, + UUIDType, ) -@singledispatch -def resolve(file_schema: Union[Schema, IcebergType], read_schema: Union[Schema, IcebergType]) -> Reader: - """This resolves the file and read schema +def construct_reader(file_schema: Union[Schema, IcebergType]) -> Reader: + """Constructs a reader from a file schema + + Args: + file_schema (Schema | IcebergType): The schema of the Avro file + + Raises: + NotImplementedError: If attempting to resolve an unrecognized object type + """ + return resolve(file_schema, file_schema) + - The function traverses the schema in post-order fashion +def resolve( + file_schema: Union[Schema, IcebergType], + read_schema: Union[Schema, IcebergType], + read_types: Dict[int, Callable[[Schema], StructProtocol]] = EMPTY_DICT, +) -> Reader: + """Resolves the file and read schema to produce a reader - Args: - file_schema (Schema | IcebergType): The schema of the Avro file - read_schema (Schema | IcebergType): The requested read schema which is equal, subset or superset of the file schema + Args: + file_schema (Schema | IcebergType): The schema of the Avro file + read_schema (Schema | IcebergType): The requested read schema which is equal, subset or superset of the file schema + read_types (Dict[int, Callable[[Schema], StructProtocol]]): A dict of types to use for struct data - Raises: - NotImplementedError: If attempting to resolve an unrecognized object type + Raises: + NotImplementedError: If attempting to resolve an unrecognized object type """ - raise NotImplementedError(f"Cannot resolve non-type: {file_schema}") + return visit_with_partner(file_schema, read_schema, SchemaResolver(read_types), SchemaPartnerAccessor()) # type: ignore + + +class SchemaResolver(PrimitiveWithPartnerVisitor[IcebergType, Reader]): + read_types: Optional[Dict[int, Callable[[Schema], StructProtocol]]] + + def __init__(self, read_types: Optional[Dict[int, Callable[[Schema], StructProtocol]]]): + self.read_types = read_types + + def schema(self, schema: Schema, expected_schema: Optional[IcebergType], result: Reader) -> Reader: + return result + + def struct(self, struct: StructType, expected_struct: Optional[IcebergType], field_readers: List[Reader]) -> Reader: + if not expected_struct: + return StructReader(tuple(enumerate(field_readers))) + + if not isinstance(expected_struct, StructType): + raise ResolveError(f"File/read schema are not aligned for struct, got {expected_struct}") + + results: List[Tuple[Optional[int], Reader]] = [] + expected_positions: Dict[int, int] = {field.field_id: pos for pos, field in enumerate(expected_struct.fields)} + + # first, add readers for the file fields that must be in order + for field, result_reader in zip(struct.fields, field_readers): + read_pos = expected_positions.get(field.field_id) + results.append((read_pos, result_reader)) + + file_fields = {field.field_id: field for field in struct.fields} + for pos, read_field in enumerate(expected_struct.fields): + if read_field.field_id not in file_fields: + if read_field.required: + raise ResolveError(f"{read_field} is non-optional, and not part of the file schema") + # Just set the new field to None + results.append((pos, NoneReader())) + + return StructReader(tuple(results)) + + def field(self, field: NestedField, expected_field: Optional[IcebergType], field_reader: Reader) -> Reader: + return field_reader if field.required else OptionReader(field_reader) + + def list(self, list_type: ListType, expected_list: Optional[IcebergType], element_reader: Reader) -> Reader: + if expected_list and not isinstance(expected_list, ListType): + raise ResolveError(f"File/read schema are not aligned for list, got {expected_list}") + + return ListReader(element_reader if list_type.element_required else OptionReader(element_reader)) + + def map(self, map_type: MapType, expected_map: Optional[IcebergType], key_reader: Reader, value_reader: Reader) -> Reader: + if expected_map and not isinstance(expected_map, MapType): + raise ResolveError(f"File/read schema are not aligned for map, got {expected_map}") + + return MapReader(key_reader, value_reader if map_type.value_required else OptionReader(value_reader)) + + def primitive(self, primitive: PrimitiveType, expected_primitive: Optional[IcebergType]) -> Reader: + if expected_primitive is not None: + if not isinstance(expected_primitive, PrimitiveType): + raise ResolveError(f"File/read schema are not aligned for {primitive}, got {expected_primitive}") + # ensure that the type can be projected to the expected + if primitive != expected_primitive: + promote(primitive, expected_primitive) -@resolve.register(Schema) -def _(file_schema: Schema, read_schema: Schema) -> Reader: - """Visit a Schema and starts resolving it by converting it to a struct""" - return resolve(file_schema.as_struct(), read_schema.as_struct()) + return super().primitive(primitive, expected_primitive) + def visit_boolean(self, boolean_type: BooleanType, partner: Optional[IcebergType]) -> Reader: + return BooleanReader() -@resolve.register(StructType) -def _(file_struct: StructType, read_struct: IcebergType) -> Reader: - """Iterates over the file schema, and checks if the field is in the read schema""" + def visit_integer(self, integer_type: IntegerType, partner: Optional[IcebergType]) -> Reader: + return IntegerReader() - if not isinstance(read_struct, StructType): - raise ResolveError(f"File/read schema are not aligned for {file_struct}, got {read_struct}") + def visit_long(self, long_type: LongType, partner: Optional[IcebergType]) -> Reader: + return IntegerReader() - results: List[Tuple[Optional[int], Reader]] = [] - read_fields = {field.field_id: (pos, field) for pos, field in enumerate(read_struct.fields)} + def visit_float(self, float_type: FloatType, partner: Optional[IcebergType]) -> Reader: + return FloatReader() - for file_field in file_struct.fields: - if file_field.field_id in read_fields: - read_pos, read_field = read_fields[file_field.field_id] - result_reader = resolve(file_field.field_type, read_field.field_type) + def visit_double(self, double_type: DoubleType, partner: Optional[IcebergType]) -> Reader: + return DoubleReader() + + def visit_decimal(self, decimal_type: DecimalType, partner: Optional[IcebergType]) -> Reader: + return DecimalReader(decimal_type.precision, decimal_type.scale) + + def visit_date(self, date_type: DateType, partner: Optional[IcebergType]) -> Reader: + return DateReader() + + def visit_time(self, time_type: TimeType, partner: Optional[IcebergType]) -> Reader: + return TimeReader() + + def visit_timestamp(self, timestamp_type: TimestampType, partner: Optional[IcebergType]) -> Reader: + return TimestampReader() + + def visit_timestampz(self, timestamptz_type: TimestamptzType, partner: Optional[IcebergType]) -> Reader: + return TimestamptzReader() + + def visit_string(self, string_type: StringType, partner: Optional[IcebergType]) -> Reader: + return StringReader() + + def visit_uuid(self, uuid_type: UUIDType, partner: Optional[IcebergType]) -> Reader: + return UUIDReader() + + def visit_fixed(self, fixed_type: FixedType, partner: Optional[IcebergType]) -> Reader: + return FixedReader(len(fixed_type)) + + def visit_binary(self, binary_type: BinaryType, partner: Optional[IcebergType]) -> Reader: + return BinaryReader() + + +class SchemaPartnerAccessor(PartnerAccessor[IcebergType]): + def schema_partner(self, partner: Optional[IcebergType]) -> Optional[IcebergType]: + if isinstance(partner, Schema): + return partner.as_struct() + + raise ResolveError(f"File/read schema are not aligned for schema, got {partner}") + + def field_partner(self, partner: Optional[IcebergType], field_id: int, field_name: str) -> Optional[IcebergType]: + if isinstance(partner, StructType): + field = partner.field(field_id) else: - read_pos = None - result_reader = visit(file_field.field_type, ConstructReader()) - result_reader = result_reader if file_field.required else OptionReader(result_reader) - results.append((read_pos, result_reader)) - - file_fields = {field.field_id: field for field in file_struct.fields} - for pos, read_field in enumerate(read_struct.fields): - if read_field.field_id not in file_fields: - if read_field.required: - raise ResolveError(f"{read_field} is non-optional, and not part of the file schema") - # Just set the new field to None - results.append((pos, NoneReader())) - - return StructReader(tuple(results)) - - -@resolve.register(ListType) -def _(file_list: ListType, read_list: IcebergType) -> Reader: - if not isinstance(read_list, ListType): - raise ResolveError(f"File/read schema are not aligned for {file_list}, got {read_list}") - element_reader = resolve(file_list.element_type, read_list.element_type) - return ListReader(element_reader) - - -@resolve.register(MapType) -def _(file_map: MapType, read_map: IcebergType) -> Reader: - if not isinstance(read_map, MapType): - raise ResolveError(f"File/read schema are not aligned for {file_map}, got {read_map}") - key_reader = resolve(file_map.key_type, read_map.key_type) - value_reader = resolve(file_map.value_type, read_map.value_type) - - return MapReader(key_reader, value_reader) - - -@resolve.register(FloatType) -def _(file_type: PrimitiveType, read_type: IcebergType) -> Reader: - """This is a special case, when we need to adhere to the bytes written""" - if isinstance(read_type, DoubleType): - return visit(file_type, ConstructReader()) - else: - raise ResolveError(f"Cannot promote an float to {read_type}") - - -@resolve.register(PrimitiveType) -def _(file_type: PrimitiveType, read_type: IcebergType) -> Reader: - """Converting the primitive type into an actual reader that will decode the physical data""" - if not isinstance(read_type, PrimitiveType): - raise ResolveError(f"Cannot promote {file_type} to {read_type}") - - # In the case of a promotion, we want to check if it is valid - if file_type != read_type: - read_type = promote(file_type, read_type) - return visit(read_type, ConstructReader()) + raise ResolveError(f"File/read schema are not aligned for struct, got {partner}") + + return field.field_type if field else None + + def list_element_partner(self, partner_list: Optional[IcebergType]) -> Optional[IcebergType]: + if isinstance(partner_list, ListType): + return partner_list.element_type + + raise ResolveError(f"File/read schema are not aligned for list, got {partner_list}") + + def map_key_partner(self, partner_map: Optional[IcebergType]) -> Optional[IcebergType]: + if isinstance(partner_map, MapType): + return partner_map.key_type + + raise ResolveError(f"File/read schema are not aligned for map, got {partner_map}") + + def map_value_partner(self, partner_map: Optional[IcebergType]) -> Optional[IcebergType]: + if isinstance(partner_map, MapType): + return partner_map.value_type + + raise ResolveError(f"File/read schema are not aligned for map, got {partner_map}") diff --git a/pyiceberg/io/pyarrow.py b/pyiceberg/io/pyarrow.py index b4c9024f2d..adddff82fa 100644 --- a/pyiceberg/io/pyarrow.py +++ b/pyiceberg/io/pyarrow.py @@ -51,7 +51,7 @@ S3FileSystem, ) -from pyiceberg.avro.resolver import ResolveError, promote +from pyiceberg.avro.resolver import ResolveError from pyiceberg.expressions import ( AlwaysTrue, BooleanExpression, @@ -77,6 +77,7 @@ Schema, SchemaVisitorPerPrimitiveType, SchemaWithPartnerVisitor, + promote, prune_columns, visit, visit_with_partner, @@ -605,6 +606,9 @@ class ArrowAccessor(PartnerAccessor[pa.Array]): def __init__(self, file_schema: Schema): self.file_schema = file_schema + def schema_partner(self, partner: Optional[pa.Array]) -> Optional[pa.Array]: + return partner + def field_partner(self, partner_struct: Optional[pa.Array], field_id: int, _: str) -> Optional[pa.Array]: if partner_struct: # use the field name from the file schema diff --git a/pyiceberg/schema.py b/pyiceberg/schema.py index 33f5cf3dc0..1a85cb4baa 100644 --- a/pyiceberg/schema.py +++ b/pyiceberg/schema.py @@ -394,7 +394,102 @@ def primitive(self, primitive: PrimitiveType, primitive_partner: Optional[P]) -> """Visit a primitive type with a partner""" +class PrimitiveWithPartnerVisitor(SchemaWithPartnerVisitor[P, T]): + def primitive(self, primitive: PrimitiveType, primitive_partner: Optional[P]) -> T: + """Visit a PrimitiveType""" + if isinstance(primitive, BooleanType): + return self.visit_boolean(primitive, primitive_partner) + elif isinstance(primitive, IntegerType): + return self.visit_integer(primitive, primitive_partner) + elif isinstance(primitive, LongType): + return self.visit_long(primitive, primitive_partner) + elif isinstance(primitive, FloatType): + return self.visit_float(primitive, primitive_partner) + elif isinstance(primitive, DoubleType): + return self.visit_double(primitive, primitive_partner) + elif isinstance(primitive, DecimalType): + return self.visit_decimal(primitive, primitive_partner) + elif isinstance(primitive, DateType): + return self.visit_date(primitive, primitive_partner) + elif isinstance(primitive, TimeType): + return self.visit_time(primitive, primitive_partner) + elif isinstance(primitive, TimestampType): + return self.visit_timestamp(primitive, primitive_partner) + elif isinstance(primitive, TimestamptzType): + return self.visit_timestampz(primitive, primitive_partner) + elif isinstance(primitive, StringType): + return self.visit_string(primitive, primitive_partner) + elif isinstance(primitive, UUIDType): + return self.visit_uuid(primitive, primitive_partner) + elif isinstance(primitive, FixedType): + return self.visit_fixed(primitive, primitive_partner) + elif isinstance(primitive, BinaryType): + return self.visit_binary(primitive, primitive_partner) + else: + raise ValueError(f"Unknown type: {primitive}") + + @abstractmethod + def visit_boolean(self, boolean_type: BooleanType, partner: Optional[P]) -> T: + """Visit a BooleanType""" + + @abstractmethod + def visit_integer(self, integer_type: IntegerType, partner: Optional[P]) -> T: + """Visit a IntegerType""" + + @abstractmethod + def visit_long(self, long_type: LongType, partner: Optional[P]) -> T: + """Visit a LongType""" + + @abstractmethod + def visit_float(self, float_type: FloatType, partner: Optional[P]) -> T: + """Visit a FloatType""" + + @abstractmethod + def visit_double(self, double_type: DoubleType, partner: Optional[P]) -> T: + """Visit a DoubleType""" + + @abstractmethod + def visit_decimal(self, decimal_type: DecimalType, partner: Optional[P]) -> T: + """Visit a DecimalType""" + + @abstractmethod + def visit_date(self, date_type: DateType, partner: Optional[P]) -> T: + """Visit a DecimalType""" + + @abstractmethod + def visit_time(self, time_type: TimeType, partner: Optional[P]) -> T: + """Visit a DecimalType""" + + @abstractmethod + def visit_timestamp(self, timestamp_type: TimestampType, partner: Optional[P]) -> T: + """Visit a TimestampType""" + + @abstractmethod + def visit_timestampz(self, timestamptz_type: TimestamptzType, partner: Optional[P]) -> T: + """Visit a TimestamptzType""" + + @abstractmethod + def visit_string(self, string_type: StringType, partner: Optional[P]) -> T: + """Visit a StringType""" + + @abstractmethod + def visit_uuid(self, uuid_type: UUIDType, partner: Optional[P]) -> T: + """Visit a UUIDType""" + + @abstractmethod + def visit_fixed(self, fixed_type: FixedType, partner: Optional[P]) -> T: + """Visit a FixedType""" + + @abstractmethod + def visit_binary(self, binary_type: BinaryType, partner: Optional[P]) -> T: + """Visit a BinaryType""" + + class PartnerAccessor(Generic[P], ABC): + @abstractmethod + def schema_partner(self, partner: Optional[P]) -> Optional[P]: + """Returns the equivalent of the schema as a struct""" + @abstractmethod def field_partner(self, partner_struct: Optional[P], field_id: int, field_name: str) -> Optional[P]: """Returns the equivalent struct field by name or id in the partner struct""" @@ -416,12 +511,13 @@ def map_value_partner(self, partner_map: Optional[P]) -> Optional[P]: def visit_with_partner( schema_or_type: Union[Schema, IcebergType], partner: P, visitor: SchemaWithPartnerVisitor[T, P], accessor: PartnerAccessor[P] ) -> T: - raise ValueError(f"Unsupported type: {type}") + raise ValueError(f"Unsupported type: {schema_or_type}") @visit_with_partner.register(Schema) def _(schema: Schema, partner: P, visitor: SchemaWithPartnerVisitor[P, T], accessor: PartnerAccessor[P]) -> T: - return visitor.schema(schema, partner, visit_with_partner(schema.as_struct(), partner, visitor, accessor)) # type: ignore + struct_partner = accessor.schema_partner(partner) + return visitor.schema(schema, partner, visit_with_partner(schema.as_struct(), struct_partner, visitor, accessor)) # type: ignore @visit_with_partner.register(StructType) @@ -561,7 +657,7 @@ def visit_uuid(self, uuid_type: UUIDType) -> T: """Visit a UUIDType""" @abstractmethod - def visit_binary(self, binary_ype: BinaryType) -> T: + def visit_binary(self, binary_type: BinaryType) -> T: """Visit a BinaryType""" diff --git a/pyiceberg/typedef.py b/pyiceberg/typedef.py index c924d703bd..228b9a927c 100644 --- a/pyiceberg/typedef.py +++ b/pyiceberg/typedef.py @@ -83,10 +83,15 @@ def set(self, pos: int, value: Any) -> None: class Record(StructProtocol): _data: List[Union[Any, StructProtocol]] + @staticmethod + def of(num_fields: int) -> Record: + return Record(*([None] * num_fields)) + def __init__(self, *data: Union[Any, StructProtocol]) -> None: self._data = list(data) def set(self, pos: int, value: Any) -> None: + print(f"set({pos}, {repr(value)})") self._data[pos] = value def get(self, pos: int) -> Any: diff --git a/pyiceberg/types.py b/pyiceberg/types.py index 6236358063..6a736ed19d 100644 --- a/pyiceberg/types.py +++ b/pyiceberg/types.py @@ -268,6 +268,12 @@ def __init__(self, *fields: NestedField, **data: Any): data["fields"] = fields super().__init__(**data) + def field(self, field_id: int) -> Optional[NestedField]: + for field in self.fields: + if field.field_id == field_id: + return field + return None + def __str__(self) -> str: return f"struct<{', '.join(map(str, self.fields))}>" diff --git a/tests/avro/test_reader.py b/tests/avro/test_reader.py index 2d54b6e887..3fc99ce3f9 100644 --- a/tests/avro/test_reader.py +++ b/tests/avro/test_reader.py @@ -23,7 +23,6 @@ from pyiceberg.avro.reader import ( BinaryReader, BooleanReader, - ConstructReader, DateReader, DecimalReader, DoubleReader, @@ -36,9 +35,10 @@ TimestamptzReader, UUIDReader, ) +from pyiceberg.avro.resolver import construct_reader from pyiceberg.io.pyarrow import PyArrowFileIO from pyiceberg.manifest import _convert_pos_to_dict -from pyiceberg.schema import Schema, visit +from pyiceberg.schema import Schema from pyiceberg.typedef import Record from pyiceberg.types import ( BinaryType, @@ -446,55 +446,55 @@ def test_null_struct_convert_pos_to_dict() -> None: def test_fixed_reader() -> None: - assert visit(FixedType(22), ConstructReader()) == FixedReader(22) + assert construct_reader(FixedType(22)) == FixedReader(22) def test_decimal_reader() -> None: - assert visit(DecimalType(19, 25), ConstructReader()) == DecimalReader(19, 25) + assert construct_reader(DecimalType(19, 25)) == DecimalReader(19, 25) def test_boolean_reader() -> None: - assert visit(BooleanType(), ConstructReader()) == BooleanReader() + assert construct_reader(BooleanType()) == BooleanReader() def test_integer_reader() -> None: - assert visit(IntegerType(), ConstructReader()) == IntegerReader() + assert construct_reader(IntegerType()) == IntegerReader() def test_long_reader() -> None: - assert visit(LongType(), ConstructReader()) == IntegerReader() + assert construct_reader(LongType()) == IntegerReader() def test_float_reader() -> None: - assert visit(FloatType(), ConstructReader()) == FloatReader() + assert construct_reader(FloatType()) == FloatReader() def test_double_reader() -> None: - assert visit(DoubleType(), ConstructReader()) == DoubleReader() + assert construct_reader(DoubleType()) == DoubleReader() def test_date_reader() -> None: - assert visit(DateType(), ConstructReader()) == DateReader() + assert construct_reader(DateType()) == DateReader() def test_time_reader() -> None: - assert visit(TimeType(), ConstructReader()) == TimeReader() + assert construct_reader(TimeType()) == TimeReader() def test_timestamp_reader() -> None: - assert visit(TimestampType(), ConstructReader()) == TimestampReader() + assert construct_reader(TimestampType()) == TimestampReader() def test_timestamptz_reader() -> None: - assert visit(TimestamptzType(), ConstructReader()) == TimestamptzReader() + assert construct_reader(TimestamptzType()) == TimestamptzReader() def test_string_reader() -> None: - assert visit(StringType(), ConstructReader()) == StringReader() + assert construct_reader(StringType()) == StringReader() def test_binary_reader() -> None: - assert visit(BinaryType(), ConstructReader()) == BinaryReader() + assert construct_reader(BinaryType()) == BinaryReader() def test_unknown_type() -> None: @@ -502,10 +502,10 @@ class UnknownType(PrimitiveType): __root__ = "UnknownType" with pytest.raises(ValueError) as exc_info: - visit(UnknownType(), ConstructReader()) + construct_reader(UnknownType()) assert "Unknown type:" in str(exc_info.value) def test_uuid_reader() -> None: - assert visit(UUIDType(), ConstructReader()) == UUIDReader() + assert construct_reader(UUIDType()) == UUIDReader() diff --git a/tests/avro/test_resolver.py b/tests/avro/test_resolver.py index f051881e0d..c36b76922a 100644 --- a/tests/avro/test_resolver.py +++ b/tests/avro/test_resolver.py @@ -160,7 +160,7 @@ def test_resolver_change_type() -> None: with pytest.raises(ResolveError) as exc_info: resolve(write_schema, read_schema) - assert "File/read schema are not aligned for list, got map" in str(exc_info.value) + assert "File/read schema are not aligned for list, got map" in str(exc_info.value) def test_resolve_int_to_long() -> None: @@ -174,7 +174,7 @@ def test_resolve_float_to_double() -> None: def test_resolve_decimal_to_decimal() -> None: # DecimalType(P, S) to DecimalType(P2, S) where P2 > P - assert resolve(DecimalType(19, 25), DecimalType(22, 25)) == DecimalReader(22, 25) + assert resolve(DecimalType(19, 25), DecimalType(22, 25)) == DecimalReader(19, 25) def test_struct_not_aligned() -> None: From 22e5f71c8038e382c30a3c7c293886dd690b06dc Mon Sep 17 00:00:00 2001 From: Fokko Driesprong Date: Tue, 3 Jan 2023 19:19:15 +0100 Subject: [PATCH 322/642] Python: Update license-checker (#6348) --- dev/.rat-excludes | 2 ++ dev/check-license | 14 ++++++++++++-- 2 files changed, 14 insertions(+), 2 deletions(-) create mode 100644 dev/.rat-excludes diff --git a/dev/.rat-excludes b/dev/.rat-excludes new file mode 100644 index 0000000000..fb90e7ed30 --- /dev/null +++ b/dev/.rat-excludes @@ -0,0 +1,2 @@ +.rat-excludes +build diff --git a/dev/check-license b/dev/check-license index 8c9ddb1f1f..6b1a9dfff2 100755 --- a/dev/check-license +++ b/dev/check-license @@ -58,7 +58,7 @@ else declare java_cmd=java fi -export RAT_VERSION=0.12 +export RAT_VERSION=0.15 export rat_jar="$FWDIR"/lib/apache-rat-${RAT_VERSION}.jar mkdir -p "$FWDIR"/lib @@ -68,9 +68,19 @@ mkdir -p "$FWDIR"/lib } mkdir -p build -$java_cmd -jar "$rat_jar" -d "$FWDIR" +$java_cmd -jar "$rat_jar" -E "$FWDIR"/dev/.rat-excludes -d "$FWDIR" > build/rat-results.txt if [ $? -ne 0 ]; then echo "RAT exited abnormally" exit 1 fi + +ERRORS="$(cat build/rat-results.txt | grep -e "??")" + +if test ! -z "$ERRORS"; then + echo "Could not find Apache license headers in the following files:" + echo "$ERRORS" + exit 1 +else + echo -e "RAT checks passed." +fi From b977aa2e46647fbe4c857e8865a0c654bab7ba11 Mon Sep 17 00:00:00 2001 From: Fokko Driesprong Date: Thu, 5 Jan 2023 20:57:12 +0100 Subject: [PATCH 323/642] Python: Use PyArrow to buffer Avro reads (#6501) --- pyiceberg/avro/file.py | 4 +--- pyiceberg/io/__init__.py | 5 ++++- pyiceberg/io/fsspec.py | 5 ++++- pyiceberg/io/pyarrow.py | 25 +++++++++++++++++++------ tests/avro/test_reader.py | 2 +- tests/io/test_pyarrow.py | 32 +++++++++++++++++++++++++++++++- 6 files changed, 60 insertions(+), 13 deletions(-) diff --git a/pyiceberg/avro/file.py b/pyiceberg/avro/file.py index 50c85102ad..5d469a27d7 100644 --- a/pyiceberg/avro/file.py +++ b/pyiceberg/avro/file.py @@ -22,7 +22,6 @@ import json from dataclasses import dataclass -from io import SEEK_SET, BufferedReader from types import TracebackType from typing import ( Callable, @@ -144,7 +143,7 @@ def __enter__(self) -> AvroFile: Returns: A generator returning the AvroStructs """ - self.input_stream = BufferedReader(self.input_file.open()) # type: ignore + self.input_stream = self.input_file.open(seekable=False) self.decoder = BinaryDecoder(self.input_stream) self.header = self._read_header() self.schema = self.header.get_schema() @@ -195,7 +194,6 @@ def __next__(self) -> Record: raise StopIteration def _read_header(self) -> AvroFileHeader: - self.input_stream.seek(0, SEEK_SET) reader = construct_reader(META_SCHEMA) _header = reader.read(self.decoder) return AvroFileHeader(magic=_header.get(0), meta=_header.get(1), sync=_header.get(2)) diff --git a/pyiceberg/io/__init__.py b/pyiceberg/io/__init__.py index 3702f47109..fd86bf5c91 100644 --- a/pyiceberg/io/__init__.py +++ b/pyiceberg/io/__init__.py @@ -139,9 +139,12 @@ def exists(self) -> bool: """ @abstractmethod - def open(self) -> InputStream: + def open(self, seekable: bool = True) -> InputStream: """This method should return an object that matches the InputStream protocol + Args: + seekable: If the stream should support seek, or if it is consumed sequential + Returns: InputStream: An object that matches the InputStream protocol diff --git a/pyiceberg/io/fsspec.py b/pyiceberg/io/fsspec.py index c5acaadade..d3df3a8be8 100644 --- a/pyiceberg/io/fsspec.py +++ b/pyiceberg/io/fsspec.py @@ -150,9 +150,12 @@ def exists(self) -> bool: """Checks whether the location exists""" return self._fs.lexists(self.location) - def open(self) -> InputStream: + def open(self, seekable: bool = True) -> InputStream: """Create an input stream for reading the contents of the file + Args: + seekable: If the stream should support seek, or if it is consumed sequential + Returns: OpenFile: An fsspec compliant file-like object diff --git a/pyiceberg/io/pyarrow.py b/pyiceberg/io/pyarrow.py index adddff82fa..6ee822ba71 100644 --- a/pyiceberg/io/pyarrow.py +++ b/pyiceberg/io/pyarrow.py @@ -110,6 +110,8 @@ if TYPE_CHECKING: from pyiceberg.table import FileScanTask, Table +ONE_MEGABYTE = 1024 * 1024 +BUFFER_SIZE = "buffer-size" ICEBERG_SCHEMA = b"iceberg.schema" @@ -135,9 +137,14 @@ class PyArrowFile(InputFile, OutputFile): >>> # output_file.create().write(b'foobytes') """ - def __init__(self, location: str, path: str, fs: FileSystem): + _fs: FileSystem + _path: str + _buffer_size: int + + def __init__(self, location: str, path: str, fs: FileSystem, buffer_size: int = ONE_MEGABYTE): self._filesystem = fs self._path = path + self._buffer_size = buffer_size super().__init__(location=location) def _file_info(self) -> FileInfo: @@ -171,9 +178,12 @@ def exists(self) -> bool: except FileNotFoundError: return False - def open(self) -> InputStream: + def open(self, seekable: bool = True) -> InputStream: """Opens the location using a PyArrow FileSystem inferred from the location + Args: + seekable: If the stream should support seek, or if it is consumed sequential + Returns: pyarrow.lib.NativeFile: A NativeFile instance for the file located at `self.location` @@ -183,7 +193,10 @@ def open(self) -> InputStream: an AWS error code 15 """ try: - input_file = self._filesystem.open_input_file(self._path) + if seekable: + input_file = self._filesystem.open_input_file(self._path) + else: + input_file = self._filesystem.open_input_stream(self._path, buffer_size=self._buffer_size) except FileNotFoundError: raise except PermissionError: @@ -218,7 +231,7 @@ def create(self, overwrite: bool = False) -> OutputStream: try: if not overwrite and self.exists() is True: raise FileExistsError(f"Cannot create file, already exists: {self.location}") - output_file = self._filesystem.open_output_stream(self._path) + output_file = self._filesystem.open_output_stream(self._path, buffer_size=self._buffer_size) except PermissionError: raise except OSError as e: @@ -273,7 +286,7 @@ def new_input(self, location: str) -> PyArrowFile: """ scheme, path = self.parse_location(location) fs = self._get_fs(scheme) - return PyArrowFile(fs=fs, location=location, path=path) + return PyArrowFile(fs=fs, location=location, path=path, buffer_size=int(self.properties.get(BUFFER_SIZE, ONE_MEGABYTE))) def new_output(self, location: str) -> PyArrowFile: """Get a PyArrowFile instance to write bytes to the file at the given location @@ -286,7 +299,7 @@ def new_output(self, location: str) -> PyArrowFile: """ scheme, path = self.parse_location(location) fs = self._get_fs(scheme) - return PyArrowFile(fs=fs, location=location, path=path) + return PyArrowFile(fs=fs, location=location, path=path, buffer_size=int(self.properties.get(BUFFER_SIZE, ONE_MEGABYTE))) def delete(self, location: Union[str, InputFile, OutputFile]) -> None: """Delete the file at the given location diff --git a/tests/avro/test_reader.py b/tests/avro/test_reader.py index 3fc99ce3f9..f8ec3326ca 100644 --- a/tests/avro/test_reader.py +++ b/tests/avro/test_reader.py @@ -65,7 +65,7 @@ def test_read_header(generated_manifest_entry_file: str, iceberg_manifest_entry_schema: Schema) -> None: with AvroFile(PyArrowFileIO().new_input(generated_manifest_entry_file)) as reader: - header = reader._read_header() + header = reader.header assert header.magic == b"Obj\x01" assert json.loads(header.meta["avro.schema"]) == { diff --git a/tests/io/test_pyarrow.py b/tests/io/test_pyarrow.py index 02b7d02729..45779e63f0 100644 --- a/tests/io/test_pyarrow.py +++ b/tests/io/test_pyarrow.py @@ -101,11 +101,41 @@ def test_pyarrow_input_file() -> None: input_file = PyArrowFileIO().new_input(location=f"{absolute_file_location}") # Test opening and reading the file - r = input_file.open() + r = input_file.open(seekable=False) assert isinstance(r, InputStream) # Test that the file object abides by the InputStream protocol data = r.read() assert data == b"foo" assert len(input_file) == 3 + with pytest.raises(OSError) as exc_info: + r.seek(0, 0) + assert "only valid on seekable files" in str(exc_info.value) + + +def test_pyarrow_input_file_seekable() -> None: + """Test reading a file using PyArrowFile""" + + with tempfile.TemporaryDirectory() as tmpdirname: + file_location = os.path.join(tmpdirname, "foo.txt") + with open(file_location, "wb") as f: + f.write(b"foo") + + # Confirm that the file initially exists + assert os.path.exists(file_location) + + # Instantiate the input file + absolute_file_location = os.path.abspath(file_location) + input_file = PyArrowFileIO().new_input(location=f"{absolute_file_location}") + + # Test opening and reading the file + r = input_file.open(seekable=True) + assert isinstance(r, InputStream) # Test that the file object abides by the InputStream protocol + data = r.read() + assert data == b"foo" + assert len(input_file) == 3 + r.seek(0, 0) + data = r.read() + assert data == b"foo" + assert len(input_file) == 3 def test_pyarrow_output_file() -> None: From b5fe644e88deb24efed9ff437ed65cb7ac9dfc71 Mon Sep 17 00:00:00 2001 From: Fokko Driesprong Date: Sat, 7 Jan 2023 00:01:07 +0100 Subject: [PATCH 324/642] Python: Add Docker suggestion for testing (#6521) --- mkdocs/docs/verify-release.md | 9 ++++++ tests/io/test_pyarrow.py | 56 ----------------------------------- 2 files changed, 9 insertions(+), 56 deletions(-) diff --git a/mkdocs/docs/verify-release.md b/mkdocs/docs/verify-release.md index e453eea12e..8875a4f2dd 100644 --- a/mkdocs/docs/verify-release.md +++ b/mkdocs/docs/verify-release.md @@ -71,6 +71,15 @@ Run RAT checks to validate license header: ## Testing +This section explains how to run the tests of the source distribution. + +!!! note "Clean environment" + To make sure that your environment is fresh is to run the tests in a new Docker container: + ``` + docker run -t -i -v `pwd`:/pyiceberg/ python:3.9 bash + ```. + And change directory: `cd /pyiceberg/` + First step is to install the package: ```sh diff --git a/tests/io/test_pyarrow.py b/tests/io/test_pyarrow.py index 45779e63f0..f894963d94 100644 --- a/tests/io/test_pyarrow.py +++ b/tests/io/test_pyarrow.py @@ -225,62 +225,6 @@ def test_raise_on_opening_a_local_file_not_found() -> None: assert "[Errno 2] Failed to open local file" in str(exc_info.value) -def test_raise_on_opening_a_local_file_no_permission() -> None: - """Test that a PyArrowFile raises appropriately when opening a local file without permission""" - - with tempfile.TemporaryDirectory() as tmpdirname: - os.chmod(tmpdirname, 0o600) - file_location = os.path.join(tmpdirname, "foo.txt") - f = PyArrowFileIO().new_input(file_location) - - with pytest.raises(PermissionError) as exc_info: - f.open() - - assert "[Errno 13] Failed to open local file" in str(exc_info.value) - - -def test_raise_on_checking_if_local_file_exists_no_permission() -> None: - """Test that a PyArrowFile raises when checking for existence on a file without permission""" - - with tempfile.TemporaryDirectory() as tmpdirname: - os.chmod(tmpdirname, 0o600) - file_location = os.path.join(tmpdirname, "foo.txt") - f = PyArrowFileIO().new_input(file_location) - - with pytest.raises(PermissionError) as exc_info: - f.create() - - assert "Cannot get file info, access denied:" in str(exc_info.value) - - -def test_raise_on_creating_a_local_file_no_permission() -> None: - """Test that a PyArrowFile raises appropriately when creating a local file without permission""" - - with tempfile.TemporaryDirectory() as tmpdirname: - os.chmod(tmpdirname, 0o600) - file_location = os.path.join(tmpdirname, "foo.txt") - f = PyArrowFileIO().new_input(file_location) - - with pytest.raises(PermissionError) as exc_info: - f.create() - - assert "Cannot get file info, access denied:" in str(exc_info.value) - - -def test_raise_on_delete_file_with_no_permission() -> None: - """Test that a PyArrowFile raises when deleting a local file without permission""" - - with tempfile.TemporaryDirectory() as tmpdirname: - os.chmod(tmpdirname, 0o600) - file_location = os.path.join(tmpdirname, "foo.txt") - file_io = PyArrowFileIO() - - with pytest.raises(PermissionError) as exc_info: - file_io.delete(file_location) - - assert "Cannot delete file" in str(exc_info.value) - - def test_raise_on_opening_an_s3_file_no_permission() -> None: """Test that opening a PyArrowFile raises a PermissionError when the pyarrow error includes 'AWS Error [code 15]'""" From 2ae10d7b30ed9d5a0267f75c4c9908ab81530f26 Mon Sep 17 00:00:00 2001 From: Fokko Driesprong Date: Sat, 7 Jan 2023 00:01:27 +0100 Subject: [PATCH 325/642] Python: Re-enable mdformat (#6529) --- .pre-commit-config.yaml | 33 ++++++++++++++++----------------- README.md | 1 + 2 files changed, 17 insertions(+), 17 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 1c367e5644..2dfd48e616 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -20,7 +20,7 @@ exclude: ^python/vendor/ repos: - repo: https://github.com/pre-commit/pre-commit-hooks - rev: v4.3.0 + rev: v4.4.0 hooks: - id: trailing-whitespace - id: end-of-file-fixer @@ -29,7 +29,7 @@ repos: - id: check-yaml - id: check-ast - repo: https://github.com/ambv/black - rev: 22.10.0 + rev: 22.12.0 hooks: - id: black - repo: https://github.com/pre-commit/mirrors-isort @@ -43,33 +43,32 @@ repos: - id: mypy args: [ --install-types, --non-interactive, --config=python/pyproject.toml] - repo: https://github.com/hadialqattan/pycln - rev: v2.1.1 + rev: v2.1.2 hooks: - id: pycln args: [--config=python/pyproject.toml] - repo: https://github.com/asottile/pyupgrade - rev: v3.0.0 + rev: v3.3.1 hooks: - id: pyupgrade args: [ --py38-plus, --keep-runtime-typing ] - repo: https://github.com/pycqa/pylint - rev: v2.15.3 + rev: v2.15.9 hooks: - id: pylint args: [ --rcfile=python/pylintrc ] - repo: https://github.com/pycqa/flake8 - rev: '5.0.4' + rev: '6.0.0' hooks: - id: flake8 args: [ "--ignore=E501,W503,E203,B024" ] - additional_dependencies: [ flake8-bugbear==22.9.11, flake8-comprehensions==3.10.0 ] -# Disabling this for now, we need mdformat-admin, otherwise it will break the note blocks -# - repo: https://github.com/executablebooks/mdformat -# rev: 0.7.16 -# hooks: -# - id: mdformat -# additional_dependencies: -# - mdformat-black -# - mdformat-config -# - mdformat-beautysh -# - mdformat-admon + additional_dependencies: [ flake8-bugbear==22.12.6, flake8-comprehensions==3.10.1 ] + - repo: https://github.com/executablebooks/mdformat + rev: 0.7.16 + hooks: + - id: mdformat + additional_dependencies: + - mdformat-black + - mdformat-config + - mdformat-beautysh + - mdformat-admon diff --git a/README.md b/README.md index 8fe2f3738d..11af8f08f8 100644 --- a/README.md +++ b/README.md @@ -20,6 +20,7 @@ PyIceberg is a Python library for programmatic access to Iceberg table metadata as well as to table data in Iceberg format. It is a Python implementation of the [Iceberg table spec](https://iceberg.apache.org/spec/). The documentation is available at [https://py.iceberg.apache.org/](https://py.iceberg.apache.org/). + # Get in Touch - [Iceberg community](https://iceberg.apache.org/community/) From 84065d98d5ba12893964eeb71681751663197698 Mon Sep 17 00:00:00 2001 From: Fokko Driesprong Date: Sat, 7 Jan 2023 00:01:47 +0100 Subject: [PATCH 326/642] Python: Bump version to 0.3.0 (#6526) --- pyiceberg/__init__.py | 2 +- pyproject.toml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/pyiceberg/__init__.py b/pyiceberg/__init__.py index 166b200372..12badee191 100644 --- a/pyiceberg/__init__.py +++ b/pyiceberg/__init__.py @@ -15,4 +15,4 @@ # specific language governing permissions and limitations # under the License. -__version__ = "0.2.0" +__version__ = "0.3.0" diff --git a/pyproject.toml b/pyproject.toml index ce1fa9586d..95ddf40feb 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -17,7 +17,7 @@ [tool.poetry] name = "pyiceberg" -version = "0.2.0" +version = "0.3.0" readme = "README.md" homepage = "https://iceberg.apache.org/" repository = "https://github.com/apache/iceberg/" From b0f1fdf777a134c5b3c69bc28a8b3122757ce025 Mon Sep 17 00:00:00 2001 From: cccs-eric Date: Fri, 6 Jan 2023 18:02:27 -0500 Subject: [PATCH 327/642] Python: Fix type cast in console CLI (#6533) --- pyiceberg/catalog/__init__.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/pyiceberg/catalog/__init__.py b/pyiceberg/catalog/__init__.py index 64308260be..a2577228eb 100644 --- a/pyiceberg/catalog/__init__.py +++ b/pyiceberg/catalog/__init__.py @@ -23,6 +23,7 @@ from enum import Enum from typing import ( Callable, + Dict, List, Optional, Set, @@ -154,7 +155,7 @@ def load_catalog(name: str, **properties: Optional[str]) -> Catalog: catalog_type = infer_catalog_type(name, conf) if catalog_type: - return AVAILABLE_CATALOGS[catalog_type](name, cast(dict[str, str], conf)) + return AVAILABLE_CATALOGS[catalog_type](name, cast(Dict[str, str], conf)) raise ValueError(f"Could not initialize catalog with the following properties: {properties}") From 551cca2705a116c963851d230fadd17c3213c6cf Mon Sep 17 00:00:00 2001 From: Fokko Driesprong Date: Sat, 7 Jan 2023 22:36:27 +0100 Subject: [PATCH 328/642] Python: Fix the mdformat issue (#6540) --- mkdocs/docs/verify-release.md | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/mkdocs/docs/verify-release.md b/mkdocs/docs/verify-release.md index 8875a4f2dd..04c8288aa7 100644 --- a/mkdocs/docs/verify-release.md +++ b/mkdocs/docs/verify-release.md @@ -75,10 +75,7 @@ This section explains how to run the tests of the source distribution. !!! note "Clean environment" To make sure that your environment is fresh is to run the tests in a new Docker container: - ``` - docker run -t -i -v `pwd`:/pyiceberg/ python:3.9 bash - ```. - And change directory: `cd /pyiceberg/` + `docker run -t -i -v $(pwd):/pyiceberg/ python:3.9 bash`. And change directory: `cd /pyiceberg/`. First step is to install the package: From 31290f9d304720ac0de5bf9b8e2993532fc47919 Mon Sep 17 00:00:00 2001 From: Fokko Driesprong Date: Sun, 8 Jan 2023 19:44:38 +0100 Subject: [PATCH 329/642] Python: Add announcement template (#6535) --- mkdocs/docs/how-to-release.md | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/mkdocs/docs/how-to-release.md b/mkdocs/docs/how-to-release.md index 12678d2c13..ca6c917f9c 100644 --- a/mkdocs/docs/how-to-release.md +++ b/mkdocs/docs/how-to-release.md @@ -144,3 +144,21 @@ rm -rf dist/ poetry build twine upload -s dist/* ``` + +Send out an announcement on the dev mail list: + +``` +To: dev@iceberg.apache.org +Subject: [ANNOUNCE] Apache PyIceberg release + +I'm pleased to announce the release of Apache PyIceberg ! + +Apache Iceberg is an open table format for huge analytic datasets. Iceberg +delivers high query performance for tables with tens of petabytes of data, +along with atomic commits, concurrent writes, and SQL-compatible table +evolution. + +This Python release can be downloaded from: https://pypi.org/project/pyiceberg// + +Thanks to everyone for contributing! +``` From cc795e6b7059da18856ba4185e3daa56930c4d20 Mon Sep 17 00:00:00 2001 From: Fokko Driesprong Date: Sun, 8 Jan 2023 20:49:39 +0100 Subject: [PATCH 330/642] Python: Add typing-extensions as a pylint depdency (#6546) It is an issue in a downstream dependency, and is currently being fixed in pylint: https://github.com/PyCQA/pylint/pull/8030 Has been fixed in Astroid in: https://github.com/PyCQA/astroid/pull/1944 --- .pre-commit-config.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 2dfd48e616..c7ffa0c723 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -57,6 +57,7 @@ repos: hooks: - id: pylint args: [ --rcfile=python/pylintrc ] + additional_dependencies: [ typing-extensions==4.4.0 ] - repo: https://github.com/pycqa/flake8 rev: '6.0.0' hooks: From 349cab57156e006b00077977ea8996cc0fc19eda Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Sun, 8 Jan 2023 23:13:55 +0100 Subject: [PATCH 331/642] Build: Bump coverage from 7.0.1 to 7.0.4 in /python (#6545) Bumps [coverage](https://github.com/nedbat/coveragepy) from 7.0.1 to 7.0.4. - [Release notes](https://github.com/nedbat/coveragepy/releases) - [Changelog](https://github.com/nedbat/coveragepy/blob/master/CHANGES.rst) - [Commits](https://github.com/nedbat/coveragepy/compare/7.0.1...7.0.4) --- updated-dependencies: - dependency-name: coverage dependency-type: direct:development update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- poetry.lock | 106 ++++++++++++++++++++++++------------------------- pyproject.toml | 2 +- 2 files changed, 54 insertions(+), 54 deletions(-) diff --git a/poetry.lock b/poetry.lock index 736b84dc30..d1e4480400 100644 --- a/poetry.lock +++ b/poetry.lock @@ -292,7 +292,7 @@ test = ["flake8 (==3.7.8)", "hypothesis (==3.55.3)"] [[package]] name = "coverage" -version = "7.0.1" +version = "7.0.4" description = "Code coverage measurement for Python" category = "dev" optional = false @@ -1191,7 +1191,7 @@ snappy = ["python-snappy"] [metadata] lock-version = "1.1" python-versions = "^3.8" -content-hash = "2c443e612fc0e2253a3d1ed8b3c9761817c3ab2b35d593aa6403faa3d46e7195" +content-hash = "598dc644bf236b1c37688d1cbc61ddc88cabb41cde32a9da3cdc5151bc619b99" [metadata.files] adal = [ @@ -1430,57 +1430,57 @@ commonmark = [ {file = "commonmark-0.9.1.tar.gz", hash = "sha256:452f9dc859be7f06631ddcb328b6919c67984aca654e5fefb3914d54691aed60"}, ] coverage = [ - {file = "coverage-7.0.1-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:b3695c4f4750bca943b3e1f74ad4be8d29e4aeab927d50772c41359107bd5d5c"}, - {file = "coverage-7.0.1-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:fa6a5a224b7f4cfb226f4fc55a57e8537fcc096f42219128c2c74c0e7d0953e1"}, - {file = "coverage-7.0.1-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:74f70cd92669394eaf8d7756d1b195c8032cf7bbbdfce3bc489d4e15b3b8cf73"}, - {file = "coverage-7.0.1-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:b66bb21a23680dee0be66557dc6b02a3152ddb55edf9f6723fa4a93368f7158d"}, - {file = "coverage-7.0.1-cp310-cp310-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:d87717959d4d0ee9db08a0f1d80d21eb585aafe30f9b0a54ecf779a69cb015f6"}, - {file = "coverage-7.0.1-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:854f22fa361d1ff914c7efa347398374cc7d567bdafa48ac3aa22334650dfba2"}, - {file = "coverage-7.0.1-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:1e414dc32ee5c3f36544ea466b6f52f28a7af788653744b8570d0bf12ff34bc0"}, - {file = "coverage-7.0.1-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:6c5ad996c6fa4d8ed669cfa1e8551348729d008a2caf81489ab9ea67cfbc7498"}, - {file = "coverage-7.0.1-cp310-cp310-win32.whl", hash = "sha256:691571f31ace1837838b7e421d3a09a8c00b4aac32efacb4fc9bd0a5c647d25a"}, - {file = "coverage-7.0.1-cp310-cp310-win_amd64.whl", hash = "sha256:89caf4425fe88889e2973a8e9a3f6f5f9bbe5dd411d7d521e86428c08a873a4a"}, - {file = "coverage-7.0.1-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:63d56165a7c76265468d7e0c5548215a5ba515fc2cba5232d17df97bffa10f6c"}, - {file = "coverage-7.0.1-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:4f943a3b2bc520102dd3e0bb465e1286e12c9a54f58accd71b9e65324d9c7c01"}, - {file = "coverage-7.0.1-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:830525361249dc4cd013652b0efad645a385707a5ae49350c894b67d23fbb07c"}, - {file = "coverage-7.0.1-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:fd1b9c5adc066db699ccf7fa839189a649afcdd9e02cb5dc9d24e67e7922737d"}, - {file = "coverage-7.0.1-cp311-cp311-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:e00c14720b8b3b6c23b487e70bd406abafc976ddc50490f645166f111c419c39"}, - {file = "coverage-7.0.1-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:6d55d840e1b8c0002fce66443e124e8581f30f9ead2e54fbf6709fb593181f2c"}, - {file = "coverage-7.0.1-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:66b18c3cf8bbab0cce0d7b9e4262dc830e93588986865a8c78ab2ae324b3ed56"}, - {file = "coverage-7.0.1-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:12a5aa77783d49e05439fbe6e6b427484f8a0f9f456b46a51d8aac022cfd024d"}, - {file = "coverage-7.0.1-cp311-cp311-win32.whl", hash = "sha256:b77015d1cb8fe941be1222a5a8b4e3fbca88180cfa7e2d4a4e58aeabadef0ab7"}, - {file = "coverage-7.0.1-cp311-cp311-win_amd64.whl", hash = "sha256:fb992c47cb1e5bd6a01e97182400bcc2ba2077080a17fcd7be23aaa6e572e390"}, - {file = "coverage-7.0.1-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:e78e9dcbf4f3853d3ae18a8f9272111242531535ec9e1009fa8ec4a2b74557dc"}, - {file = "coverage-7.0.1-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:e60bef2e2416f15fdc05772bf87db06c6a6f9870d1db08fdd019fbec98ae24a9"}, - {file = "coverage-7.0.1-cp37-cp37m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:9823e4789ab70f3ec88724bba1a203f2856331986cd893dedbe3e23a6cfc1e4e"}, - {file = "coverage-7.0.1-cp37-cp37m-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:9158f8fb06747ac17bd237930c4372336edc85b6e13bdc778e60f9d685c3ca37"}, - {file = "coverage-7.0.1-cp37-cp37m-musllinux_1_1_aarch64.whl", hash = "sha256:486ee81fa694b4b796fc5617e376326a088f7b9729c74d9defa211813f3861e4"}, - {file = "coverage-7.0.1-cp37-cp37m-musllinux_1_1_i686.whl", hash = "sha256:1285648428a6101b5f41a18991c84f1c3959cee359e51b8375c5882fc364a13f"}, - {file = "coverage-7.0.1-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:2c44fcfb3781b41409d0f060a4ed748537557de9362a8a9282182fafb7a76ab4"}, - {file = "coverage-7.0.1-cp37-cp37m-win32.whl", hash = "sha256:d6814854c02cbcd9c873c0f3286a02e3ac1250625cca822ca6bc1018c5b19f1c"}, - {file = "coverage-7.0.1-cp37-cp37m-win_amd64.whl", hash = "sha256:f66460f17c9319ea4f91c165d46840314f0a7c004720b20be58594d162a441d8"}, - {file = "coverage-7.0.1-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:9b373c9345c584bb4b5f5b8840df7f4ab48c4cbb7934b58d52c57020d911b856"}, - {file = "coverage-7.0.1-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:d3022c3007d3267a880b5adcf18c2a9bf1fc64469b394a804886b401959b8742"}, - {file = "coverage-7.0.1-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:92651580bd46519067e36493acb394ea0607b55b45bd81dd4e26379ed1871f55"}, - {file = "coverage-7.0.1-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:3cfc595d2af13856505631be072835c59f1acf30028d1c860b435c5fc9c15b69"}, - {file = "coverage-7.0.1-cp38-cp38-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:0b4b3a4d9915b2be879aff6299c0a6129f3d08a775d5a061f503cf79571f73e4"}, - {file = "coverage-7.0.1-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:b6f22bb64cc39bcb883e5910f99a27b200fdc14cdd79df8696fa96b0005c9444"}, - {file = "coverage-7.0.1-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:72d1507f152abacea81f65fee38e4ef3ac3c02ff8bc16f21d935fd3a8a4ad910"}, - {file = "coverage-7.0.1-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:0a79137fc99815fff6a852c233628e735ec15903cfd16da0f229d9c4d45926ab"}, - {file = "coverage-7.0.1-cp38-cp38-win32.whl", hash = "sha256:b3763e7fcade2ff6c8e62340af9277f54336920489ceb6a8cd6cc96da52fcc62"}, - {file = "coverage-7.0.1-cp38-cp38-win_amd64.whl", hash = "sha256:09f6b5a8415b6b3e136d5fec62b552972187265cb705097bf030eb9d4ffb9b60"}, - {file = "coverage-7.0.1-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:978258fec36c154b5e250d356c59af7d4c3ba02bef4b99cda90b6029441d797d"}, - {file = "coverage-7.0.1-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:19ec666533f0f70a0993f88b8273057b96c07b9d26457b41863ccd021a043b9a"}, - {file = "coverage-7.0.1-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:cfded268092a84605f1cc19e5c737f9ce630a8900a3589e9289622db161967e9"}, - {file = "coverage-7.0.1-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:07bcfb1d8ac94af886b54e18a88b393f6a73d5959bb31e46644a02453c36e475"}, - {file = "coverage-7.0.1-cp39-cp39-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:397b4a923cc7566bbc7ae2dfd0ba5a039b61d19c740f1373791f2ebd11caea59"}, - {file = "coverage-7.0.1-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:aec2d1515d9d39ff270059fd3afbb3b44e6ec5758af73caf18991807138c7118"}, - {file = "coverage-7.0.1-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:c20cfebcc149a4c212f6491a5f9ff56f41829cd4f607b5be71bb2d530ef243b1"}, - {file = "coverage-7.0.1-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:fd556ff16a57a070ce4f31c635953cc44e25244f91a0378c6e9bdfd40fdb249f"}, - {file = "coverage-7.0.1-cp39-cp39-win32.whl", hash = "sha256:b9ea158775c7c2d3e54530a92da79496fb3fb577c876eec761c23e028f1e216c"}, - {file = "coverage-7.0.1-cp39-cp39-win_amd64.whl", hash = "sha256:d1991f1dd95eba69d2cd7708ff6c2bbd2426160ffc73c2b81f617a053ebcb1a8"}, - {file = "coverage-7.0.1-pp37.pp38.pp39-none-any.whl", hash = "sha256:3dd4ee135e08037f458425b8842d24a95a0961831a33f89685ff86b77d378f89"}, - {file = "coverage-7.0.1.tar.gz", hash = "sha256:a4a574a19eeb67575a5328a5760bbbb737faa685616586a9f9da4281f940109c"}, + {file = "coverage-7.0.4-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:daf91db39324e9939a9db919ee4fb42a1a23634a056616dae891a030e89f87ba"}, + {file = "coverage-7.0.4-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:55121fe140d7e42cb970999b93cf1c2b24484ce028b32bbd00238bb25c13e34a"}, + {file = "coverage-7.0.4-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:c027fbb83a8c78a6e06a0302ea1799fdb70e5cda9845a5e000545b8e2b47ea39"}, + {file = "coverage-7.0.4-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:caf82db5b7f16b51ec32fe0bd2da0805b177c807aa8bfb478c7e6f893418c284"}, + {file = "coverage-7.0.4-cp310-cp310-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:1ba5cc54baf3c322c4388de2a43cc95f7809366f0600e743e5aae8ea9d1038b2"}, + {file = "coverage-7.0.4-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:260854160083f8275a9d9d49a05ab0ffc7a1f08f2ccccbfaec94a18aae9f407c"}, + {file = "coverage-7.0.4-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:ea45f0dba5a993e93b158f1a9dcfff2770e3bcabf2b80dbe7aa15dce0bcb3bf3"}, + {file = "coverage-7.0.4-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:6abc91f6f8b3cc0ae1034e2c03f38769fba1952ab70d0b26953aa01691265c39"}, + {file = "coverage-7.0.4-cp310-cp310-win32.whl", hash = "sha256:053cdc47cae08257051d7e934a0de4d095b60eb8a3024fa9f1b2322fa1547137"}, + {file = "coverage-7.0.4-cp310-cp310-win_amd64.whl", hash = "sha256:1e9e94f2612ee549a4b3ee79cbc61bceed77e69cf38cfa05858bae939a886d16"}, + {file = "coverage-7.0.4-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:5caa9dd91dcc5f054350dc57a02e053d79633907b9ccffff999568d13dcd19f8"}, + {file = "coverage-7.0.4-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:efc200fa75d9634525b40babc7a16342bd21c101db1a58ef84dc14f4bf6ac0fd"}, + {file = "coverage-7.0.4-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:1791e5f74c5b52f76e83fe9f4bb9571cf76d40ee0c51952ee1e4ee935b7e98b9"}, + {file = "coverage-7.0.4-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:3d9201cfa5a98652b9cef36ab202f17fe3ea83f497b4ba2a8ed39399dfb8fcd4"}, + {file = "coverage-7.0.4-cp311-cp311-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:22d8ef6865cb6834cab2b72fff20747a55c714b57b675f7e11c9624fe4f7cb45"}, + {file = "coverage-7.0.4-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:b84076e3de192fba0f95e279ac017b64c7c6ecd4f09f36f13420f5bed898a9c7"}, + {file = "coverage-7.0.4-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:dcfbf8ffc046f20d75fd775a92c378f6fc7b9bded6c6f2ab88b6b9cb5805a184"}, + {file = "coverage-7.0.4-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:4665a714af31f160403c2e448fb2fef330719d2e04e836b08d60d612707c1041"}, + {file = "coverage-7.0.4-cp311-cp311-win32.whl", hash = "sha256:2e59aef3fba5758059208c9eff10ae7ded3629e797972746ec33b56844f69411"}, + {file = "coverage-7.0.4-cp311-cp311-win_amd64.whl", hash = "sha256:2b854f7985b48122b6fe346631e86d67b63293f8255cb59a93d79e3d9f1574e3"}, + {file = "coverage-7.0.4-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:e44b60b0b49aa85d548d392a2dca2c6a581cd4084e72e9e16bd58bd86ec20816"}, + {file = "coverage-7.0.4-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:2904d7a0388911c61e7e3beefe48c29dfccaba938fc1158f63190101a21e04c2"}, + {file = "coverage-7.0.4-cp37-cp37m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:bc74b64bfa89e2f862ea45dd6ac1def371d7cc883b76680d20bdd61a6f3daa20"}, + {file = "coverage-7.0.4-cp37-cp37m-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:c06046f54e719da21c79f98ecc0962581d1aee0b3798dc6b12b1217da8bf93f4"}, + {file = "coverage-7.0.4-cp37-cp37m-musllinux_1_1_aarch64.whl", hash = "sha256:bc9c77004970a364a1e5454cf7cb884e4277592b959c287689b2a0fd027ef552"}, + {file = "coverage-7.0.4-cp37-cp37m-musllinux_1_1_i686.whl", hash = "sha256:0815a09b32384e8ff00a5939ec9cd10efce8742347e019c2daca1a32f5ac2aae"}, + {file = "coverage-7.0.4-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:a78a80d131c067d67d8a6f9bd3d3f7ea7eac82c1c7259f97d7ab73f723da9d55"}, + {file = "coverage-7.0.4-cp37-cp37m-win32.whl", hash = "sha256:2b5936b624fbe711ed02dfd86edd678822e5ee68da02b6d231e5c01090b64590"}, + {file = "coverage-7.0.4-cp37-cp37m-win_amd64.whl", hash = "sha256:a63922765ee49d5b4c32afb2cd5516812c8665f3b78e64a0dd005bdfabf991b1"}, + {file = "coverage-7.0.4-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:d68f2f7bddb3acdd3b36ef7f334b9d14f30b93e094f808fbbd8d288b8f9e2f9b"}, + {file = "coverage-7.0.4-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:9dafdba3b2b9010abab08cb8c0dc6549bfca6e1630fe14d47b01dca00d39e694"}, + {file = "coverage-7.0.4-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:0322354757b47640535daabd2d56384ff3cad2896248fc84d328c5fad4922d5c"}, + {file = "coverage-7.0.4-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:4e8267466662aff93d66fa72b9591d02122dfc8a729b0a43dd70e0fb07ed9b37"}, + {file = "coverage-7.0.4-cp38-cp38-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:f684d88eb4924ed0630cf488fd5606e334c6835594bb5fe36b50a509b10383ed"}, + {file = "coverage-7.0.4-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:70c294bb15ba576fb96b580db35895bf03749d683df044212b74e938a7f6821f"}, + {file = "coverage-7.0.4-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:34c0457e1ba450ae8b22dc8ea2fd36ada1010af61291e4c96963cd9d9633366f"}, + {file = "coverage-7.0.4-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:b75aff2c35ceaa299691e772f7bf7c8aeab25f46acea2be3dd04cccb914a9860"}, + {file = "coverage-7.0.4-cp38-cp38-win32.whl", hash = "sha256:6c5554d55668381e131577f20e8f620d4882b04ad558f7e7f3f1f55b3124c379"}, + {file = "coverage-7.0.4-cp38-cp38-win_amd64.whl", hash = "sha256:c82f34fafaf5bc05d222fcf84423d6e156432ca35ca78672d4affd0c09c6ef6c"}, + {file = "coverage-7.0.4-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:b8dfb5fed540f77e814bf4ec79619c241af6b4578fa1093c5e3389bbb7beab3f"}, + {file = "coverage-7.0.4-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:ee32a080bab779b71c4d09a3eb5254bfca43ee88828a683dab27dfe8f582516e"}, + {file = "coverage-7.0.4-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:2dfbee0bf0d633be3a2ab068f5a5731a70adf147d0ba17d9f9932b46c7c5782b"}, + {file = "coverage-7.0.4-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:32dc010713455ac0fe2fddb0e48aa43875cc7eb7b09768df10bad8ce45f9c430"}, + {file = "coverage-7.0.4-cp39-cp39-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:9cb88a3019ad042eaa69fc7639ef077793fedbf313e89207aa82fefe92c97ebd"}, + {file = "coverage-7.0.4-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:73bc6114aab7753ca784f87bcd3b7613bc797aa255b5bca45e5654070ae9acfb"}, + {file = "coverage-7.0.4-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:92f135d370fcd7a6fb9659fa2eb716dd2ca364719cbb1756f74d90a221bca1a7"}, + {file = "coverage-7.0.4-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:f3d485e6ec6e09857bf2115ece572d666b7c498377d4c70e66bb06c63ed177c2"}, + {file = "coverage-7.0.4-cp39-cp39-win32.whl", hash = "sha256:c58921fcd9914b56444292e7546fe183d079db99528142c809549ddeaeacd8e9"}, + {file = "coverage-7.0.4-cp39-cp39-win_amd64.whl", hash = "sha256:f092d9f2ddaa30235d33335fbdb61eb8f3657af519ef5f9dd6bdae65272def11"}, + {file = "coverage-7.0.4-pp37.pp38.pp39-none-any.whl", hash = "sha256:cb8cfa3bf3a9f18211279458917fef5edeb5e1fdebe2ea8b11969ec2ebe48884"}, + {file = "coverage-7.0.4.tar.gz", hash = "sha256:f6c4ad409a0caf7e2e12e203348b1a9b19c514e7d078520973147bf2d3dcbc6f"}, ] cryptography = [ {file = "cryptography-38.0.4-cp36-abi3-macosx_10_10_universal2.whl", hash = "sha256:2fa36a7b2cc0998a3a4d5af26ccb6273f3df133d61da2ba13b3286261e7efb70"}, diff --git a/pyproject.toml b/pyproject.toml index 95ddf40feb..6e0ad441c8 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -80,7 +80,7 @@ pytest = "7.2.0" pytest-checkdocs = "2.9.0" pre-commit = "2.21.0" fastavro = "1.7.0" -coverage = { version = "^7.0.1", extras = ["toml"] } +coverage = { version = "^7.0.4", extras = ["toml"] } requests-mock = "1.10.0" moto = "^4.0.12" typing-extensions = '4.4.0' From 369f19acdb0c37a69200663ede1434892f007101 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 9 Jan 2023 06:45:36 +0100 Subject: [PATCH 332/642] Build: Bump rich from 13.0.0 to 13.0.1 in /python (#6544) --- poetry.lock | 8 ++++---- pyproject.toml | 2 +- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/poetry.lock b/poetry.lock index d1e4480400..db0469f73c 100644 --- a/poetry.lock +++ b/poetry.lock @@ -966,7 +966,7 @@ tests = ["coverage (>=6.0.0)", "flake8", "mypy", "pytest (>=7.0.0)", "pytest-asy [[package]] name = "rich" -version = "13.0.0" +version = "13.0.1" description = "Render rich text, tables, progress bars, syntax highlighting, markdown and more to the terminal" category = "main" optional = false @@ -1191,7 +1191,7 @@ snappy = ["python-snappy"] [metadata] lock-version = "1.1" python-versions = "^3.8" -content-hash = "598dc644bf236b1c37688d1cbc61ddc88cabb41cde32a9da3cdc5151bc619b99" +content-hash = "bce00ca640f36394b3b25ea613679d01bdd3565b2bef6a9f6873fe852202d41c" [metadata.files] adal = [ @@ -2189,8 +2189,8 @@ responses = [ {file = "responses-0.22.0.tar.gz", hash = "sha256:396acb2a13d25297789a5866b4881cf4e46ffd49cc26c43ab1117f40b973102e"}, ] rich = [ - {file = "rich-13.0.0-py3-none-any.whl", hash = "sha256:12b1d77ee7edf251b741531323f0d990f5f570a4e7c054d0bfb59fb7981ad977"}, - {file = "rich-13.0.0.tar.gz", hash = "sha256:3aa9eba7219b8c575c6494446a59f702552efe1aa261e7eeb95548fa586e1950"}, + {file = "rich-13.0.1-py3-none-any.whl", hash = "sha256:41fe1d05f433b0f4724cda8345219213d2bfa472ef56b2f64f415b5b94d51b04"}, + {file = "rich-13.0.1.tar.gz", hash = "sha256:25f83363f636995627a99f6e4abc52ed0970ebbd544960cc63cbb43aaac3d6f0"}, ] s3fs = [ {file = "s3fs-2022.10.0-py3-none-any.whl", hash = "sha256:1e134c3577171699feb7c1a0c4713260d5b48296e1708737ff940baef6e2c153"}, diff --git a/pyproject.toml b/pyproject.toml index 6e0ad441c8..8b7bff2955 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -50,7 +50,7 @@ python = "^3.8" mmhash3 = "3.0.1" requests = "2.28.1" click = "8.1.3" -rich = "13.0.0" +rich = "13.0.1" pyyaml = "6.0.0" pydantic = "1.10.4" From 4c98f55b2108e01ce318cbb009c95a2048f871ed Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 9 Jan 2023 08:40:09 +0100 Subject: [PATCH 333/642] Build: Bump moto from 4.0.12 to 4.0.13 in /python (#6543) --- poetry.lock | 8 ++++---- pyproject.toml | 2 +- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/poetry.lock b/poetry.lock index db0469f73c..7d215f79fd 100644 --- a/poetry.lock +++ b/poetry.lock @@ -520,7 +520,7 @@ python-versions = "*" [[package]] name = "moto" -version = "4.0.12" +version = "4.0.13" description = "A library that allows your python tests to easily mock out the boto library" category = "dev" optional = false @@ -1191,7 +1191,7 @@ snappy = ["python-snappy"] [metadata] lock-version = "1.1" python-versions = "^3.8" -content-hash = "bce00ca640f36394b3b25ea613679d01bdd3565b2bef6a9f6873fe852202d41c" +content-hash = "cd9c64bc3b45183df8d3a1689b90fc019dce3ae616c444a74f4129822fc9ffac" [metadata.files] adal = [ @@ -1785,8 +1785,8 @@ mmhash3 = [ {file = "mmhash3-3.0.1.tar.gz", hash = "sha256:a00d68f4a1cc434b9501513c8a29e18ed1ddad383677d72b41d71d0d862348af"}, ] moto = [ - {file = "moto-4.0.12-py3-none-any.whl", hash = "sha256:e6592727d90a2dc82ca7c7fa61b224e2b195cc0dcdac2f46ccbf0cd8efde1459"}, - {file = "moto-4.0.12.tar.gz", hash = "sha256:30f8f31658f18cdb15eb626c8cb3a07520f0d8c2193226fbcb3326ae12fba246"}, + {file = "moto-4.0.13-py3-none-any.whl", hash = "sha256:e73400c6d3fe06028aa7f07bb6f276f14260d289b70f38928a98e3d3d968352d"}, + {file = "moto-4.0.13.tar.gz", hash = "sha256:baf7d6969cf837990c730e6e648315bebc2e1c0038d9d8fc4f59d03561484469"}, ] msal = [ {file = "msal-1.20.0-py2.py3-none-any.whl", hash = "sha256:d2f1c26368ecdc28c8657d457352faa0b81b1845a7b889d8676787721ba86792"}, diff --git a/pyproject.toml b/pyproject.toml index 8b7bff2955..cceedf3dd8 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -82,7 +82,7 @@ pre-commit = "2.21.0" fastavro = "1.7.0" coverage = { version = "^7.0.4", extras = ["toml"] } requests-mock = "1.10.0" -moto = "^4.0.12" +moto = "^4.0.13" typing-extensions = '4.4.0' [tool.poetry.scripts] From 88d2728813d5de9415e6fd9c7c1c4bf6c75cbd6a Mon Sep 17 00:00:00 2001 From: Fokko Driesprong Date: Mon, 9 Jan 2023 08:55:37 +0100 Subject: [PATCH 334/642] Python: Fix reading 0-bytes binary (#6532) * Python: Fix reading 0-bytes binary When a binary field would be zero bytes, we would still read the stream, causing a EOFErrors: ``` Traceback (most recent call last): File "", line 1, in File "/Users/fokkodriesprong/Desktop/iceberg/python/pyiceberg/manifest.py", line 139, in read_manifest_entry for record in reader: File "/Users/fokkodriesprong/Desktop/iceberg/python/pyiceberg/avro/file.py", line 194, in __next__ return self.__next__() File "/Users/fokkodriesprong/Desktop/iceberg/python/pyiceberg/avro/file.py", line 186, in __next__ return next(self.block) File "/Users/fokkodriesprong/Desktop/iceberg/python/pyiceberg/avro/file.py", line 113, in __next__ return self.reader.read(self.block_decoder) File "/Users/fokkodriesprong/Desktop/iceberg/python/pyiceberg/avro/reader.py", line 260, in read struct.set(pos, field.read(decoder)) # later: pass reuse in here File "/Users/fokkodriesprong/Desktop/iceberg/python/pyiceberg/avro/reader.py", line 260, in read struct.set(pos, field.read(decoder)) # later: pass reuse in here File "/Users/fokkodriesprong/Desktop/iceberg/python/pyiceberg/avro/reader.py", line 233, in read return self.option.read(decoder) File "/Users/fokkodriesprong/Desktop/iceberg/python/pyiceberg/avro/reader.py", line 313, in read read_items[key] = self.value.read(decoder) File "/Users/fokkodriesprong/Desktop/iceberg/python/pyiceberg/avro/reader.py", line 200, in read return decoder.read_bytes() File "/Users/fokkodriesprong/Desktop/iceberg/python/pyiceberg/avro/decoder.py", line 118, in read_bytes return self.read(self.read_int()) File "/Users/fokkodriesprong/Desktop/iceberg/python/pyiceberg/avro/decoder.py", line 54, in read raise EOFError EOFError ``` Fixes #6435 * Revert * Read actual bytes * Black * Optimize the code * Remove benchmark * To a literal --- pyiceberg/avro/decoder.py | 27 +++++++++++++++++++-------- tests/avro/test_decoder.py | 12 ++---------- 2 files changed, 21 insertions(+), 18 deletions(-) diff --git a/pyiceberg/avro/decoder.py b/pyiceberg/avro/decoder.py index 663a193ce8..a65a716005 100644 --- a/pyiceberg/avro/decoder.py +++ b/pyiceberg/avro/decoder.py @@ -18,6 +18,7 @@ import struct from datetime import datetime, time from io import SEEK_CUR +from typing import List from uuid import UUID from pyiceberg.io import InputStream @@ -48,13 +49,22 @@ def read(self, n: int) -> bytes: """ if n < 0: raise ValueError(f"Requested {n} bytes to read, expected positive integer.") - read_bytes = self._input_stream.read(n) - read_len = len(read_bytes) - if read_len <= 0: - raise EOFError - elif read_len != n: - raise ValueError(f"Read {len(read_bytes)} bytes, expected {n} bytes") - return read_bytes + data: List[bytes] = [] + + n_remaining = n + while n_remaining > 0: + data_read = self._input_stream.read(n_remaining) + read_len = len(data_read) + if read_len == n: + # If we read everything, we return directly + # otherwise we'll continue to fetch the rest + return data_read + elif read_len <= 0: + raise EOFError(f"EOF: read {read_len} bytes") + data.append(data_read) + n_remaining -= read_len + + return b"".join(data) def skip(self, n: int) -> None: self._input_stream.seek(n, SEEK_CUR) @@ -115,7 +125,8 @@ def read_bytes(self) -> bytes: """ Bytes are encoded as a long followed by that many bytes of data. """ - return self.read(self.read_int()) + num_bytes = self.read_int() + return self.read(num_bytes) if num_bytes > 0 else b"" def read_utf8(self) -> str: """ diff --git a/tests/avro/test_decoder.py b/tests/avro/test_decoder.py index 56b9cf7489..35405c8bad 100644 --- a/tests/avro/test_decoder.py +++ b/tests/avro/test_decoder.py @@ -109,7 +109,7 @@ class OneByteAtATimeInputStream(InputStream): def read(self, size: int = 0) -> bytes: self.pos += 1 - return int.to_bytes(1, self.pos, byteorder="little") + return self.pos.to_bytes(1, byteorder="little") def seek(self, offset: int, whence: int = SEEK_SET) -> int: self.pos = offset @@ -118,10 +118,6 @@ def seek(self, offset: int, whence: int = SEEK_SET) -> int: def tell(self) -> int: return self.pos - @property - def closed(self) -> bool: - return False - def close(self) -> None: pass @@ -136,11 +132,7 @@ def __exit__( def test_read_single_byte_at_the_time() -> None: decoder = BinaryDecoder(OneByteAtATimeInputStream()) - - with pytest.raises(ValueError) as exc_info: - decoder.read(2) - - assert "Read 1 bytes, expected 2 bytes" in str(exc_info.value) + assert decoder.read(2) == b"\x01\x02" def test_read_float() -> None: From f726883234741485e88cba8fd032b2179543134b Mon Sep 17 00:00:00 2001 From: Fokko Driesprong Date: Wed, 11 Jan 2023 12:07:30 +0100 Subject: [PATCH 335/642] Python: Expression to disjunctive normal form (#6555) * Python: Expression to disjunctive normal form Adds a visitor to rewrite an expression to the DNF. This is required for filtering data in Dask. * Run rewriteNot before * Move to the tuple approach * Raise an exception --- pyiceberg/expressions/__init__.py | 6 ++-- pyiceberg/expressions/visitors.py | 39 ++++++++++++++++++++++ tests/expressions/test_visitors.py | 52 ++++++++++++++++++++++++++++++ 3 files changed, 95 insertions(+), 2 deletions(-) diff --git a/pyiceberg/expressions/__init__.py b/pyiceberg/expressions/__init__.py index b7ef368112..14ded66473 100644 --- a/pyiceberg/expressions/__init__.py +++ b/pyiceberg/expressions/__init__.py @@ -216,7 +216,8 @@ def __str__(self) -> str: def __repr__(self) -> str: return f"And(left={repr(self.left)}, right={repr(self.right)})" - def __invert__(self) -> Or: + def __invert__(self) -> BooleanExpression: + # De Morgan's law: not (A and B) = (not A) or (not B) return Or(~self.left, ~self.right) @@ -247,7 +248,8 @@ def __eq__(self, other: Any) -> bool: def __repr__(self) -> str: return f"Or(left={repr(self.left)}, right={repr(self.right)})" - def __invert__(self) -> And: + def __invert__(self) -> BooleanExpression: + # De Morgan's law: not (A or B) = (not A) and (not B) return And(~self.left, ~self.right) diff --git a/pyiceberg/expressions/visitors.py b/pyiceberg/expressions/visitors.py index de7a489ab3..f312f12c3f 100644 --- a/pyiceberg/expressions/visitors.py +++ b/pyiceberg/expressions/visitors.py @@ -22,6 +22,7 @@ Generic, List, Set, + Tuple, TypeVar, ) @@ -842,3 +843,41 @@ def visit_bound_predicate(self, predicate: BoundPredicate[L]) -> Set[int]: def extract_field_ids(expr: BooleanExpression) -> Set[int]: return visit(expr, _ExpressionFieldIDs()) + + +class _RewriteToDNF(BooleanExpressionVisitor[Tuple[BooleanExpression, ...]]): + def visit_true(self) -> Tuple[BooleanExpression, ...]: + return (AlwaysTrue(),) + + def visit_false(self) -> Tuple[BooleanExpression, ...]: + return (AlwaysFalse(),) + + def visit_not(self, child_result: Tuple[BooleanExpression, ...]) -> Tuple[BooleanExpression, ...]: + raise ValueError(f"Not expressions are not allowed: {child_result}") + + def visit_and( + self, left_result: Tuple[BooleanExpression, ...], right_result: Tuple[BooleanExpression, ...] + ) -> Tuple[BooleanExpression, ...]: + # Distributive law: + # ((P OR Q) AND (R OR S)) AND (((P AND R) OR (P AND S)) OR ((Q AND R) OR ((Q AND S))) + # A AND (B OR C) = (A AND B) OR (A AND C) + # (A OR B) AND C = (A AND C) OR (B AND C) + return tuple(And(le, re) for le in left_result for re in right_result) + + def visit_or( + self, left_result: Tuple[BooleanExpression, ...], right_result: Tuple[BooleanExpression, ...] + ) -> Tuple[BooleanExpression, ...]: + return left_result + right_result + + def visit_unbound_predicate(self, predicate: UnboundPredicate[L]) -> Tuple[BooleanExpression, ...]: + return (predicate,) + + def visit_bound_predicate(self, predicate: BoundPredicate[L]) -> Tuple[BooleanExpression, ...]: + return (predicate,) + + +def rewrite_to_dnf(expr: BooleanExpression) -> Tuple[BooleanExpression, ...]: + # Rewrites an arbitrary boolean expression to disjunctive normal form (DNF): + # (A AND NOT(B) AND C) OR (NOT(D) AND E AND F) OR (G) + expr_without_not = rewrite_not(expr) + return visit(expr_without_not, _RewriteToDNF()) diff --git a/tests/expressions/test_visitors.py b/tests/expressions/test_visitors.py index 19915cc6b9..e79c353f81 100644 --- a/tests/expressions/test_visitors.py +++ b/tests/expressions/test_visitors.py @@ -65,6 +65,7 @@ BoundBooleanExpressionVisitor, _ManifestEvalVisitor, rewrite_not, + rewrite_to_dnf, visit, visit_bound_predicate, ) @@ -1403,3 +1404,54 @@ def test_rewrite_bound() -> None: accessor=Accessor(position=0, inner=None), ) ) + + +def test_to_dnf() -> None: + expr = Or(Not(EqualTo("P", "a")), And(EqualTo("Q", "b"), Not(Or(Not(EqualTo("R", "c")), EqualTo("S", "d"))))) + assert rewrite_to_dnf(expr) == (NotEqualTo("P", "a"), And(EqualTo("Q", "b"), And(EqualTo("R", "c"), NotEqualTo("S", "d")))) + + +def test_to_dnf_nested_or() -> None: + expr = Or(EqualTo("P", "a"), And(EqualTo("Q", "b"), Or(EqualTo("R", "c"), EqualTo("S", "d")))) + assert rewrite_to_dnf(expr) == ( + EqualTo("P", "a"), + And(EqualTo("Q", "b"), EqualTo("R", "c")), + And(EqualTo("Q", "b"), EqualTo("S", "d")), + ) + + +def test_to_dnf_double_distribution() -> None: + expr = And(Or(EqualTo("P", "a"), EqualTo("Q", "b")), Or(EqualTo("R", "c"), EqualTo("S", "d"))) + assert rewrite_to_dnf(expr) == ( + And( + left=EqualTo(term=Reference(name="P"), literal=literal("a")), + right=EqualTo(term=Reference(name="R"), literal=literal("c")), + ), + And( + left=EqualTo(term=Reference(name="P"), literal=literal("a")), + right=EqualTo(term=Reference(name="S"), literal=literal("d")), + ), + And( + left=EqualTo(term=Reference(name="Q"), literal=literal("b")), + right=EqualTo(term=Reference(name="R"), literal=literal("c")), + ), + And( + left=EqualTo(term=Reference(name="Q"), literal=literal("b")), + right=EqualTo(term=Reference(name="S"), literal=literal("d")), + ), + ) + + +def test_to_dnf_double_negation() -> None: + expr = rewrite_to_dnf(Not(Not(Not(Not(Not(Not(EqualTo("P", "a")))))))) + assert expr == (EqualTo("P", "a"),) + + +def test_to_dnf_and() -> None: + expr = And(Not(EqualTo("Q", "b")), EqualTo("R", "c")) + assert rewrite_to_dnf(expr) == (And(NotEqualTo("Q", "b"), EqualTo("R", "c")),) + + +def test_to_dnf_not_and() -> None: + expr = Not(And(Not(EqualTo("Q", "b")), EqualTo("R", "c"))) + assert rewrite_to_dnf(expr) == (EqualTo("Q", "b"), NotEqualTo("R", "c")) From d5227e8c3218a8d7a6e913b8ec88d9e782609b08 Mon Sep 17 00:00:00 2001 From: Fokko Driesprong Date: Sat, 14 Jan 2023 13:01:48 +0100 Subject: [PATCH 336/642] Python: Refactor loading manifests (#6525) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * Add ManifestFile StructProtocol implementation. * Python: Make it work * Python: Refactor the reading of the manifest * Comments * Fix named record * Revert some stuff * Extend the test * Always pass in the struct * Refactor the Records into a single one - And remove Pydantic as a base class * Comments * Cleanup * Revert some changes and cleanup * Make CI happy * Fix passing in the struct * It’s easier to ask for forgiveness than permission * Add some more tests * Python: Minor changes to Record to support old test cases. (#351) Co-authored-by: Ryan Blue --- pyiceberg/avro/file.py | 34 +- pyiceberg/avro/reader.py | 69 +- pyiceberg/avro/resolver.py | 37 +- pyiceberg/catalog/rest.py | 3 +- pyiceberg/cli/output.py | 3 +- pyiceberg/manifest.py | 311 +++++---- pyiceberg/partitioning.py | 2 +- pyiceberg/schema.py | 10 +- pyiceberg/table/__init__.py | 25 +- pyiceberg/table/metadata.py | 3 +- pyiceberg/table/refs.py | 2 +- pyiceberg/table/snapshots.py | 2 +- pyiceberg/table/sorting.py | 2 +- pyiceberg/transforms.py | 3 +- pyiceberg/typedef.py | 88 ++- pyiceberg/types.py | 5 +- pyiceberg/utils/iceberg_base_model.py | 62 -- tests/avro/test_file.py | 22 +- tests/avro/test_reader.py | 242 ++----- tests/avro/test_resolver.py | 69 +- tests/expressions/test_evaluator.py | 28 +- tests/expressions/test_expressions.py | 60 +- tests/expressions/test_visitors.py | 8 +- tests/table/test_snapshots.py | 39 -- tests/test_schema.py | 4 +- tests/test_transforms.py | 2 +- tests/test_typedef.py | 51 +- tests/test_types.py | 2 +- tests/utils/test_manifest.py | 929 +++++--------------------- 29 files changed, 785 insertions(+), 1332 deletions(-) delete mode 100644 pyiceberg/utils/iceberg_base_model.py diff --git a/pyiceberg/avro/file.py b/pyiceberg/avro/file.py index 5d469a27d7..56163d7bc7 100644 --- a/pyiceberg/avro/file.py +++ b/pyiceberg/avro/file.py @@ -26,8 +26,10 @@ from typing import ( Callable, Dict, + Generic, Optional, Type, + TypeVar, ) from pyiceberg.avro.codecs import KNOWN_CODECS, Codec @@ -66,10 +68,9 @@ _SCHEMA_KEY = "avro.schema" -@dataclass(frozen=True) -class AvroFileHeader: +class AvroFileHeader(Record): magic: bytes - meta: dict[str, str] + meta: Dict[str, str] sync: bytes def compression_codec(self) -> Optional[Type[Codec]]: @@ -93,49 +94,52 @@ def get_schema(self) -> Schema: raise ValueError("No schema found in Avro file headers") +D = TypeVar("D", bound=StructProtocol) + + @dataclass -class Block: +class Block(Generic[D]): reader: Reader block_records: int block_decoder: BinaryDecoder position: int = 0 - def __iter__(self) -> Block: + def __iter__(self) -> Block[D]: return self def has_next(self) -> bool: return self.position < self.block_records - def __next__(self) -> Record: + def __next__(self) -> D: if self.has_next(): self.position += 1 return self.reader.read(self.block_decoder) raise StopIteration -class AvroFile: +class AvroFile(Generic[D]): input_file: InputFile read_schema: Optional[Schema] - read_types: Dict[int, Callable[[Schema], StructProtocol]] + read_types: Dict[int, Callable[..., StructProtocol]] input_stream: InputStream header: AvroFileHeader schema: Schema reader: Reader decoder: BinaryDecoder - block: Optional[Block] = None + block: Optional[Block[D]] = None def __init__( self, input_file: InputFile, read_schema: Optional[Schema] = None, - read_types: Dict[int, Callable[[Schema], StructProtocol]] = EMPTY_DICT, + read_types: Dict[int, Callable[..., StructProtocol]] = EMPTY_DICT, ) -> None: self.input_file = input_file self.read_schema = read_schema self.read_types = read_types - def __enter__(self) -> AvroFile: + def __enter__(self) -> AvroFile[D]: """ Opens the file and reads the header and generates a reader tree to start reading the payload @@ -159,7 +163,7 @@ def __exit__( ) -> None: self.input_stream.close() - def __iter__(self) -> AvroFile: + def __iter__(self) -> AvroFile[D]: return self def _read_block(self) -> int: @@ -180,7 +184,7 @@ def _read_block(self) -> int: ) return block_records - def __next__(self) -> Record: + def __next__(self) -> D: if self.block and self.block.has_next(): return next(self.block) @@ -194,6 +198,4 @@ def __next__(self) -> Record: raise StopIteration def _read_header(self) -> AvroFileHeader: - reader = construct_reader(META_SCHEMA) - _header = reader.read(self.decoder) - return AvroFileHeader(magic=_header.get(0), meta=_header.get(1), sync=_header.get(2)) + return construct_reader(META_SCHEMA, {-1: AvroFileHeader}).read(self.decoder) diff --git a/pyiceberg/avro/reader.py b/pyiceberg/avro/reader.py index f7a0194a9a..8cc0c4813e 100644 --- a/pyiceberg/avro/reader.py +++ b/pyiceberg/avro/reader.py @@ -41,7 +41,8 @@ from uuid import UUID from pyiceberg.avro.decoder import BinaryDecoder -from pyiceberg.typedef import Record, StructProtocol +from pyiceberg.typedef import StructProtocol +from pyiceberg.types import StructType from pyiceberg.utils.singleton import Singleton @@ -90,6 +91,9 @@ def read(self, decoder: BinaryDecoder) -> Any: def skip(self, decoder: BinaryDecoder) -> None: ... + def __repr__(self) -> str: + return f"{self.__class__.__name__}()" + class NoneReader(Reader): def read(self, _: BinaryDecoder) -> None: @@ -194,6 +198,9 @@ def skip(self, decoder: BinaryDecoder) -> None: def __len__(self) -> int: return self._len + def __repr__(self) -> str: + return f"FixedReader({self._len})" + class BinaryReader(Reader): def read(self, decoder: BinaryDecoder) -> bytes: @@ -214,6 +221,9 @@ def read(self, decoder: BinaryDecoder) -> Decimal: def skip(self, decoder: BinaryDecoder) -> None: decoder.skip_bytes() + def __repr__(self) -> str: + return f"DecimalReader({self.precision}, {self.scale})" + @dataclass(frozen=True) class OptionReader(Reader): @@ -238,41 +248,58 @@ def skip(self, decoder: BinaryDecoder) -> None: return self.option.skip(decoder) -class StructProtocolReader(Reader): - create_struct: Callable[[], StructProtocol] - fields: Tuple[Tuple[Optional[int], Reader], ...] +class StructReader(Reader): + field_readers: Tuple[Tuple[Optional[int], Reader], ...] + create_struct: Callable[..., StructProtocol] + struct: StructType - def __init__(self, fields: Tuple[Tuple[Optional[int], Reader], ...], create_struct: Callable[[], StructProtocol]): + def __init__( + self, + field_readers: Tuple[Tuple[Optional[int], Reader], ...], + create_struct: Callable[..., StructProtocol], + struct: StructType, + ) -> None: + self.field_readers = field_readers self.create_struct = create_struct - self.fields = fields - - def create_or_reuse(self, reuse: Optional[StructProtocol]) -> StructProtocol: - if reuse: - return reuse - else: - return self.create_struct() + self.struct = struct + + def read(self, decoder: BinaryDecoder) -> StructProtocol: + try: + # Try initializing the struct, first with the struct keyword argument + struct = self.create_struct(struct=self.struct) + except TypeError as e: + if "'struct' is an invalid keyword argument for" in str(e): + struct = self.create_struct() + else: + raise ValueError(f"Unable to initialize struct: {self.create_struct}") from e - def read(self, decoder: BinaryDecoder) -> Any: - struct = self.create_or_reuse(None) + if not isinstance(struct, StructProtocol): + raise ValueError(f"Incompatible with StructProtocol: {self.create_struct}") - for (pos, field) in self.fields: + for (pos, field) in self.field_readers: if pos is not None: - struct.set(pos, field.read(decoder)) # later: pass reuse in here + struct[pos] = field.read(decoder) # later: pass reuse in here else: field.skip(decoder) return struct def skip(self, decoder: BinaryDecoder) -> None: - for _, field in self.fields: + for _, field in self.field_readers: field.skip(decoder) + def __eq__(self, other: Any) -> bool: + return ( + self.field_readers == other.field_readers and self.create_struct == other.create_struct + if isinstance(other, StructReader) + else False + ) -class StructReader(StructProtocolReader): - fields: Tuple[Tuple[Optional[int], Reader], ...] + def __repr__(self) -> str: + return f"StructReader(({','.join(repr(field) for field in self.field_readers)}), {repr(self.create_struct)})" - def __init__(self, fields: Tuple[Tuple[Optional[int], Reader], ...]): - super().__init__(fields, lambda: Record.of(len(fields))) + def __hash__(self) -> int: + return hash(self.field_readers) @dataclass(frozen=True) diff --git a/pyiceberg/avro/resolver.py b/pyiceberg/avro/resolver.py index a53693f415..bacd942b33 100644 --- a/pyiceberg/avro/resolver.py +++ b/pyiceberg/avro/resolver.py @@ -53,7 +53,7 @@ promote, visit_with_partner, ) -from pyiceberg.typedef import EMPTY_DICT, StructProtocol +from pyiceberg.typedef import EMPTY_DICT, Record, StructProtocol from pyiceberg.types import ( BinaryType, BooleanType, @@ -78,7 +78,9 @@ ) -def construct_reader(file_schema: Union[Schema, IcebergType]) -> Reader: +def construct_reader( + file_schema: Union[Schema, IcebergType], read_types: Dict[int, Callable[..., StructProtocol]] = EMPTY_DICT +) -> Reader: """Constructs a reader from a file schema Args: @@ -87,13 +89,13 @@ def construct_reader(file_schema: Union[Schema, IcebergType]) -> Reader: Raises: NotImplementedError: If attempting to resolve an unrecognized object type """ - return resolve(file_schema, file_schema) + return resolve(file_schema, file_schema, read_types) def resolve( file_schema: Union[Schema, IcebergType], read_schema: Union[Schema, IcebergType], - read_types: Dict[int, Callable[[Schema], StructProtocol]] = EMPTY_DICT, + read_types: Dict[int, Callable[..., StructProtocol]] = EMPTY_DICT, ) -> Reader: """Resolves the file and read schema to produce a reader @@ -109,28 +111,39 @@ def resolve( class SchemaResolver(PrimitiveWithPartnerVisitor[IcebergType, Reader]): - read_types: Optional[Dict[int, Callable[[Schema], StructProtocol]]] + read_types: Dict[int, Callable[..., StructProtocol]] + context: List[int] - def __init__(self, read_types: Optional[Dict[int, Callable[[Schema], StructProtocol]]]): + def __init__(self, read_types: Dict[int, Callable[..., StructProtocol]] = EMPTY_DICT) -> None: self.read_types = read_types + self.context = [] def schema(self, schema: Schema, expected_schema: Optional[IcebergType], result: Reader) -> Reader: return result + def before_field(self, field: NestedField, field_partner: Optional[NestedField]) -> None: + self.context.append(field.field_id) + + def after_field(self, field: NestedField, field_partner: Optional[NestedField]) -> None: + self.context.pop() + def struct(self, struct: StructType, expected_struct: Optional[IcebergType], field_readers: List[Reader]) -> Reader: + # -1 indicates the struct root + read_struct_id = self.context[-1] if len(self.context) > 0 else -1 + struct_callable = self.read_types.get(read_struct_id, Record) + if not expected_struct: - return StructReader(tuple(enumerate(field_readers))) + return StructReader(tuple(enumerate(field_readers)), struct_callable, struct) if not isinstance(expected_struct, StructType): raise ResolveError(f"File/read schema are not aligned for struct, got {expected_struct}") - results: List[Tuple[Optional[int], Reader]] = [] expected_positions: Dict[int, int] = {field.field_id: pos for pos, field in enumerate(expected_struct.fields)} # first, add readers for the file fields that must be in order - for field, result_reader in zip(struct.fields, field_readers): - read_pos = expected_positions.get(field.field_id) - results.append((read_pos, result_reader)) + results: List[Tuple[Optional[int], Reader]] = [ + (expected_positions.get(field.field_id), result_reader) for field, result_reader in zip(struct.fields, field_readers) + ] file_fields = {field.field_id: field for field in struct.fields} for pos, read_field in enumerate(expected_struct.fields): @@ -140,7 +153,7 @@ def struct(self, struct: StructType, expected_struct: Optional[IcebergType], fie # Just set the new field to None results.append((pos, NoneReader())) - return StructReader(tuple(results)) + return StructReader(tuple(results), struct_callable, expected_struct) def field(self, field: NestedField, expected_field: Optional[IcebergType], field_reader: Reader) -> Reader: return field_reader if field.required else OptionReader(field_reader) diff --git a/pyiceberg/catalog/rest.py b/pyiceberg/catalog/rest.py index f7bb6b346c..2f9eb5df11 100644 --- a/pyiceberg/catalog/rest.py +++ b/pyiceberg/catalog/rest.py @@ -57,8 +57,7 @@ from pyiceberg.schema import Schema from pyiceberg.table import Table, TableMetadata from pyiceberg.table.sorting import UNSORTED_SORT_ORDER, SortOrder -from pyiceberg.typedef import EMPTY_DICT -from pyiceberg.utils.iceberg_base_model import IcebergBaseModel +from pyiceberg.typedef import EMPTY_DICT, IcebergBaseModel ICEBERG_REST_SPEC_VERSION = "0.14.1" diff --git a/pyiceberg/cli/output.py b/pyiceberg/cli/output.py index 5bb61d456b..44c94f6b4c 100644 --- a/pyiceberg/cli/output.py +++ b/pyiceberg/cli/output.py @@ -26,8 +26,7 @@ from pyiceberg.partitioning import PartitionSpec from pyiceberg.schema import Schema from pyiceberg.table import Table, TableMetadata -from pyiceberg.typedef import Identifier, Properties -from pyiceberg.utils.iceberg_base_model import IcebergBaseModel +from pyiceberg.typedef import IcebergBaseModel, Identifier, Properties class Output(ABC): diff --git a/pyiceberg/manifest.py b/pyiceberg/manifest.py index 9f7f394159..757f3bd016 100644 --- a/pyiceberg/manifest.py +++ b/pyiceberg/manifest.py @@ -15,30 +15,29 @@ # specific language governing permissions and limitations # under the License. from enum import Enum -from functools import singledispatch from typing import ( Any, Dict, Iterator, List, Optional, - Union, ) -from pydantic import Field - from pyiceberg.avro.file import AvroFile from pyiceberg.io import FileIO, InputFile from pyiceberg.schema import Schema from pyiceberg.typedef import Record from pyiceberg.types import ( - IcebergType, + BinaryType, + BooleanType, + IntegerType, ListType, + LongType, MapType, - PrimitiveType, + NestedField, + StringType, StructType, ) -from pyiceberg.utils.iceberg_base_model import IcebergBaseModel class DataFileContent(int, Enum): @@ -76,137 +75,207 @@ def __repr__(self) -> str: return f"FileFormat.{self.name}" -class DataFile(IcebergBaseModel): - content: DataFileContent = Field(default=DataFileContent.DATA) - file_path: str = Field() - file_format: FileFormat = Field() - partition: Dict[str, Any] = Field() - record_count: int = Field() - file_size_in_bytes: int = Field() - block_size_in_bytes: Optional[int] = Field() - column_sizes: Optional[Dict[int, int]] = Field() - value_counts: Optional[Dict[int, int]] = Field() - null_value_counts: Optional[Dict[int, int]] = Field() - nan_value_counts: Optional[Dict[int, int]] = Field() - distinct_counts: Optional[Dict[int, int]] = Field() - lower_bounds: Optional[Dict[int, bytes]] = Field() - upper_bounds: Optional[Dict[int, bytes]] = Field() - key_metadata: Optional[bytes] = Field() - split_offsets: Optional[List[int]] = Field() - equality_ids: Optional[List[int]] = Field() - sort_order_id: Optional[int] = Field() - - -class ManifestEntry(IcebergBaseModel): - status: ManifestEntryStatus = Field() - snapshot_id: Optional[int] = Field() - sequence_number: Optional[int] = Field() - data_file: DataFile = Field() - - -class PartitionFieldSummary(IcebergBaseModel): - contains_null: bool = Field() - contains_nan: Optional[bool] = Field() - lower_bound: Optional[bytes] = Field() - upper_bound: Optional[bytes] = Field() - - -class ManifestFile(IcebergBaseModel): - manifest_path: str = Field() - manifest_length: int = Field() - partition_spec_id: int = Field() - content: ManifestContent = Field(default=ManifestContent.DATA) - sequence_number: int = Field(default=0) - min_sequence_number: int = Field(default=0) - added_snapshot_id: Optional[int] = Field() - added_data_files_count: Optional[int] = Field() - existing_data_files_count: Optional[int] = Field() - deleted_data_files_count: Optional[int] = Field() - added_rows_count: Optional[int] = Field() - existing_rows_counts: Optional[int] = Field() - deleted_rows_count: Optional[int] = Field() - partitions: Optional[List[PartitionFieldSummary]] = Field() - key_metadata: Optional[bytes] = Field() - - def fetch_manifest_entry(self, io: FileIO) -> List[ManifestEntry]: - file = io.new_input(self.manifest_path) - return list(read_manifest_entry(file)) - +DATA_FILE_TYPE = StructType( + NestedField( + field_id=134, + name="content", + field_type=IntegerType(), + required=False, + doc="Contents of the file: 0=data, 1=position deletes, 2=equality deletes", + ), + NestedField(field_id=100, name="file_path", field_type=StringType(), required=True, doc="Location URI with FS scheme"), + NestedField( + field_id=101, name="file_format", field_type=StringType(), required=True, doc="File format name: avro, orc, or parquet" + ), + NestedField( + field_id=102, + name="partition", + field_type=StructType(), + required=True, + doc="Partition data tuple, schema based on the partition spec", + ), + NestedField(field_id=103, name="record_count", field_type=LongType(), required=True, doc="Number of records in the file"), + NestedField(field_id=104, name="file_size_in_bytes", field_type=LongType(), required=True, doc="Total file size in bytes"), + NestedField( + field_id=108, + name="column_sizes", + field_type=MapType(key_id=117, key_type=IntegerType(), value_id=118, value_type=LongType()), + required=True, + doc="Map of column id to total size on disk", + ), + NestedField( + field_id=109, + name="value_counts", + field_type=MapType(key_id=119, key_type=IntegerType(), value_id=120, value_type=LongType()), + required=True, + doc="Map of column id to total count, including null and NaN", + ), + NestedField( + field_id=110, + name="null_value_counts", + field_type=MapType(key_id=121, key_type=IntegerType(), value_id=122, value_type=LongType()), + required=False, + doc="Map of column id to null value count", + ), + NestedField( + field_id=137, + name="nan_value_counts", + field_type=MapType(key_id=138, key_type=IntegerType(), value_id=139, value_type=LongType()), + required=False, + doc="Map of column id to number of NaN values in the column", + ), + NestedField( + field_id=125, + name="lower_bounds", + field_type=MapType(key_id=126, key_type=IntegerType(), value_id=127, value_type=BinaryType()), + required=False, + doc="Map of column id to lower bound", + ), + NestedField( + field_id=128, + name="upper_bounds", + field_type=MapType(key_id=129, key_type=IntegerType(), value_id=130, value_type=BinaryType()), + required=False, + doc="Map of column id to upper bound", + ), + NestedField(field_id=131, name="key_metadata", field_type=BinaryType(), required=False, doc="Encryption key metadata blob"), + NestedField( + field_id=132, + name="split_offsets", + field_type=ListType(element_id=133, element_type=LongType(), element_required=True), + required=False, + doc="Splittable offsets", + ), + NestedField( + field_id=135, + name="equality_ids", + field_type=ListType(element_id=136, element_type=LongType(), element_required=True), + required=False, + doc="Equality comparison field IDs", + ), + NestedField(field_id=140, name="sort_order_id", field_type=IntegerType(), required=False, doc="Sort order ID"), + NestedField(field_id=141, name="spec_id", field_type=IntegerType(), required=False, doc="Partition spec ID"), +) -def read_manifest_entry(input_file: InputFile) -> Iterator[ManifestEntry]: - with AvroFile(input_file) as reader: - schema = reader.schema - for record in reader: - dict_repr = _convert_pos_to_dict(schema, record) - yield ManifestEntry(**dict_repr) +class DataFile(Record): + content: Optional[DataFileContent] + file_path: str + file_format: FileFormat + partition: Record + record_count: int + file_size_in_bytes: int + column_sizes: Dict[int, int] + value_counts: Dict[int, int] + null_value_counts: Dict[int, int] + nan_value_counts: Dict[int, int] + lower_bounds: Dict[int, bytes] + upper_bounds: Dict[int, bytes] + key_metadata: Optional[bytes] + split_offsets: Optional[List[int]] + equality_ids: Optional[List[int]] + sort_order_id: Optional[int] + spec_id: Optional[int] + + def __init__(self, *data: Any, **named_data: Any) -> None: + super().__init__(*data, **{"struct": DATA_FILE_TYPE, **named_data}) + + +MANIFEST_ENTRY_SCHEMA = Schema( + NestedField(0, "status", IntegerType(), required=True), + NestedField(1, "snapshot_id", LongType(), required=False), + NestedField(3, "sequence_number", LongType(), required=False), + NestedField(4, "file_sequence_number", LongType(), required=False), + NestedField(2, "data_file", DATA_FILE_TYPE, required=False), +) -def live_entries(input_file: InputFile) -> Iterator[ManifestEntry]: - return (entry for entry in read_manifest_entry(input_file) if entry.status != ManifestEntryStatus.DELETED) +class ManifestEntry(Record): + status: ManifestEntryStatus + snapshot_id: Optional[int] + sequence_number: Optional[int] + file_sequence_number: Optional[int] + data_file: DataFile -def files(input_file: InputFile) -> Iterator[DataFile]: - return (entry.data_file for entry in live_entries(input_file)) + def __init__(self, *data: Any, **named_data: Any) -> None: + super().__init__(*data, **{"struct": MANIFEST_ENTRY_SCHEMA.as_struct(), **named_data}) -def read_manifest_list(input_file: InputFile) -> Iterator[ManifestFile]: - with AvroFile(input_file) as reader: - schema = reader.schema - for record in reader: - dict_repr = _convert_pos_to_dict(schema, record) - yield ManifestFile(**dict_repr) - - -@singledispatch -def _convert_pos_to_dict(schema: Union[Schema, IcebergType], struct: Record) -> Dict[str, Any]: - """Converts the positions in the field names +PARTITION_FIELD_SUMMARY_TYPE = StructType( + NestedField(509, "contains_null", BooleanType(), required=True), + NestedField(518, "contains_nan", BooleanType(), required=False), + NestedField(510, "lower_bound", BinaryType(), required=False), + NestedField(511, "upper_bound", BinaryType(), required=False), +) - This makes it easy to map it onto a Pydantic model. Might change later on depending on the performance - Args: - schema (Schema | IcebergType): The schema of the file - struct (Record): The struct containing the data by positions +class PartitionFieldSummary(Record): + contains_null: bool + contains_nan: Optional[bool] + lower_bound: Optional[bytes] + upper_bound: Optional[bytes] + + def __init__(self, *data: Any, **named_data: Any) -> None: + super().__init__(*data, **{"struct": PARTITION_FIELD_SUMMARY_TYPE, **named_data}) + + +MANIFEST_FILE_SCHEMA: Schema = Schema( + NestedField(500, "manifest_path", StringType(), required=True, doc="Location URI with FS scheme"), + NestedField(501, "manifest_length", LongType(), required=True), + NestedField(502, "partition_spec_id", IntegerType(), required=True), + NestedField(517, "content", IntegerType(), required=False), + NestedField(515, "sequence_number", LongType(), required=False), + NestedField(516, "min_sequence_number", LongType(), required=False), + NestedField(503, "added_snapshot_id", LongType(), required=False), + NestedField(504, "added_files_count", IntegerType(), required=False), + NestedField(505, "existing_files_count", IntegerType(), required=False), + NestedField(506, "deleted_files_count", IntegerType(), required=False), + NestedField(512, "added_rows_count", LongType(), required=False), + NestedField(513, "existing_rows_count", LongType(), required=False), + NestedField(514, "deleted_rows_count", LongType(), required=False), + NestedField(507, "partitions", ListType(508, PARTITION_FIELD_SUMMARY_TYPE, element_required=True), required=False), + NestedField(519, "key_metadata", BinaryType(), required=False), +) - Raises: - NotImplementedError: If attempting to handle an unknown type in the schema - """ - raise NotImplementedError(f"Cannot traverse non-type: {schema}") +class ManifestFile(Record): + manifest_path: str + manifest_length: int + partition_spec_id: int + content: Optional[ManifestContent] + sequence_number: Optional[int] + min_sequence_number: Optional[int] + added_snapshot_id: Optional[int] + added_files_count: Optional[int] + existing_files_count: Optional[int] + deleted_files_count: Optional[int] + added_rows_count: Optional[int] + existing_rows_count: Optional[int] + deleted_rows_count: Optional[int] + partitions: Optional[List[PartitionFieldSummary]] + key_metadata: Optional[bytes] + + def __init__(self, *data: Any, **named_data: Any) -> None: + super().__init__(*data, **{"struct": MANIFEST_FILE_SCHEMA.as_struct(), **named_data}) -@_convert_pos_to_dict.register -def _(schema: Schema, struct: Record) -> Dict[str, Any]: - return _convert_pos_to_dict(schema.as_struct(), struct) + def fetch_manifest_entry(self, io: FileIO) -> List[ManifestEntry]: + file = io.new_input(self.manifest_path) + return list(read_manifest_entry(file)) -@_convert_pos_to_dict.register -def _(struct_type: StructType, values: Record) -> Dict[str, Any]: - """Iterates over all the fields in the dict, and gets the data from the struct""" - return ( - {field.name: _convert_pos_to_dict(field.field_type, values.get(pos)) for pos, field in enumerate(struct_type.fields)} - if values is not None - else None - ) +def read_manifest_entry(input_file: InputFile) -> Iterator[ManifestEntry]: + with AvroFile[ManifestEntry](input_file, MANIFEST_ENTRY_SCHEMA, {-1: ManifestEntry, 2: DataFile}) as reader: + yield from reader -@_convert_pos_to_dict.register -def _(list_type: ListType, values: List[Any]) -> Any: - """In the case of a list, we'll go over the elements in the list to handle complex types""" - return [_convert_pos_to_dict(list_type.element_type, value) for value in values] if values is not None else None +def live_entries(input_file: InputFile) -> Iterator[ManifestEntry]: + return (entry for entry in read_manifest_entry(input_file) if entry.status != ManifestEntryStatus.DELETED) -@_convert_pos_to_dict.register -def _(map_type: MapType, values: Dict[Any, Any]) -> Dict[Any, Any]: - """In the case of a map, we both traverse over the key and value to handle complex types""" - return ( - { - _convert_pos_to_dict(map_type.key_type, key): _convert_pos_to_dict(map_type.value_type, value) - for key, value in values.items() - } - if values is not None - else None - ) +def files(input_file: InputFile) -> Iterator[DataFile]: + return (entry.data_file for entry in live_entries(input_file)) -@_convert_pos_to_dict.register -def _(_: PrimitiveType, value: Any) -> Any: - return value +def read_manifest_list(input_file: InputFile) -> Iterator[ManifestFile]: + with AvroFile[ManifestFile](input_file, MANIFEST_FILE_SCHEMA, {-1: ManifestFile, 508: PartitionFieldSummary}) as reader: + yield from reader diff --git a/pyiceberg/partitioning.py b/pyiceberg/partitioning.py index aca79bfa76..9313b1606c 100644 --- a/pyiceberg/partitioning.py +++ b/pyiceberg/partitioning.py @@ -27,8 +27,8 @@ from pyiceberg.schema import Schema from pyiceberg.transforms import Transform +from pyiceberg.typedef import IcebergBaseModel from pyiceberg.types import NestedField, StructType -from pyiceberg.utils.iceberg_base_model import IcebergBaseModel INITIAL_PARTITION_SPEC_ID = 0 _PARTITION_DATA_ID_START: int = 1000 diff --git a/pyiceberg/schema.py b/pyiceberg/schema.py index 1a85cb4baa..45e9ffe26a 100644 --- a/pyiceberg/schema.py +++ b/pyiceberg/schema.py @@ -36,7 +36,7 @@ from pydantic import Field, PrivateAttr from pyiceberg.exceptions import ResolveError -from pyiceberg.typedef import EMPTY_DICT, StructProtocol +from pyiceberg.typedef import EMPTY_DICT, IcebergBaseModel, StructProtocol from pyiceberg.types import ( BinaryType, BooleanType, @@ -59,7 +59,6 @@ TimeType, UUIDType, ) -from pyiceberg.utils.iceberg_base_model import IcebergBaseModel T = TypeVar("T") P = TypeVar("P") @@ -94,6 +93,9 @@ def __str__(self) -> str: def __repr__(self) -> str: return f"Schema({', '.join(repr(column) for column in self.columns)}, schema_id={self.schema_id}, identifier_field_ids={self.identifier_field_ids})" + def __len__(self) -> int: + return len(self.fields) + def __eq__(self, other: Any) -> bool: if not other: return False @@ -684,11 +686,11 @@ def get(self, container: StructProtocol) -> Any: Any: The value at position `self.position` in the container """ pos = self.position - val = container.get(pos) + val = container[pos] inner = self while inner.inner: inner = inner.inner - val = val.get(inner.position) + val = val[inner.position] return val diff --git a/pyiceberg/table/__init__.py b/pyiceberg/table/__init__.py index 3fb0702889..c4c23100c9 100644 --- a/pyiceberg/table/__init__.py +++ b/pyiceberg/table/__init__.py @@ -54,9 +54,7 @@ Identifier, KeyDefaultDict, Properties, - StructProtocol, ) -from pyiceberg.types import StructType if TYPE_CHECKING: import pandas as pd @@ -259,24 +257,6 @@ def __init__(self, data_file: DataFile, start: Optional[int] = None, length: Opt self.length = length or data_file.file_size_in_bytes -class _DictAsStruct(StructProtocol): - pos_to_name: Dict[int, str] - wrapped: Dict[str, Any] - - def __init__(self, partition_type: StructType): - self.pos_to_name = {pos: field.name for pos, field in enumerate(partition_type.fields)} - - def wrap(self, to_wrap: Dict[str, Any]) -> _DictAsStruct: - self.wrapped = to_wrap - return self - - def get(self, pos: int) -> Any: - return self.wrapped[self.pos_to_name[pos]] - - def set(self, pos: int, value: Any) -> None: - raise NotImplementedError("Cannot set values in DictAsStruct") - - class DataScan(TableScan["DataScan"]): def __init__( self, @@ -307,11 +287,8 @@ def _build_partition_evaluator(self, spec_id: int) -> Callable[[DataFile], bool] partition_schema = Schema(*partition_type.fields) partition_expr = self.partition_filters[spec_id] - # TODO: remove the dict to struct wrapper by using a StructProtocol record # pylint: disable=W0511 - wrapper = _DictAsStruct(partition_type) evaluator = visitors.expression_evaluator(partition_schema, partition_expr, self.case_sensitive) - - return lambda data_file: evaluator(wrapper.wrap(data_file.partition)) + return lambda data_file: evaluator(data_file.partition) def plan_files(self) -> Iterator[FileScanTask]: snapshot = self.snapshot() diff --git a/pyiceberg/table/metadata.py b/pyiceberg/table/metadata.py index 5704222d2c..31e74e7084 100644 --- a/pyiceberg/table/metadata.py +++ b/pyiceberg/table/metadata.py @@ -39,9 +39,8 @@ SortOrder, assign_fresh_sort_order_ids, ) -from pyiceberg.typedef import EMPTY_DICT, Properties +from pyiceberg.typedef import EMPTY_DICT, IcebergBaseModel, Properties from pyiceberg.utils.datetime import datetime_to_micros -from pyiceberg.utils.iceberg_base_model import IcebergBaseModel CURRENT_SNAPSHOT_ID = "current_snapshot_id" CURRENT_SCHEMA_ID = "current_schema_id" diff --git a/pyiceberg/table/refs.py b/pyiceberg/table/refs.py index 58f6121ed9..09cbad27f1 100644 --- a/pyiceberg/table/refs.py +++ b/pyiceberg/table/refs.py @@ -19,7 +19,7 @@ from pydantic import Field -from pyiceberg.utils.iceberg_base_model import IcebergBaseModel +from pyiceberg.typedef import IcebergBaseModel MAIN_BRANCH = "main" diff --git a/pyiceberg/table/snapshots.py b/pyiceberg/table/snapshots.py index af17fd4b4e..83dd3f66a7 100644 --- a/pyiceberg/table/snapshots.py +++ b/pyiceberg/table/snapshots.py @@ -27,7 +27,7 @@ from pyiceberg.io import FileIO from pyiceberg.manifest import ManifestFile, read_manifest_list -from pyiceberg.utils.iceberg_base_model import IcebergBaseModel +from pyiceberg.typedef import IcebergBaseModel OPERATION = "operation" diff --git a/pyiceberg/table/sorting.py b/pyiceberg/table/sorting.py index 67f27d82eb..7ca64399d6 100644 --- a/pyiceberg/table/sorting.py +++ b/pyiceberg/table/sorting.py @@ -29,8 +29,8 @@ from pyiceberg.schema import Schema from pyiceberg.transforms import IdentityTransform, Transform +from pyiceberg.typedef import IcebergBaseModel from pyiceberg.types import IcebergType -from pyiceberg.utils.iceberg_base_model import IcebergBaseModel class SortDirection(Enum): diff --git a/pyiceberg/transforms.py b/pyiceberg/transforms.py index b163c02569..e0197f0639 100644 --- a/pyiceberg/transforms.py +++ b/pyiceberg/transforms.py @@ -60,7 +60,7 @@ TimestampLiteral, literal, ) -from pyiceberg.typedef import L +from pyiceberg.typedef import IcebergBaseModel, L from pyiceberg.types import ( BinaryType, DateType, @@ -77,7 +77,6 @@ ) from pyiceberg.utils import datetime from pyiceberg.utils.decimal import decimal_to_bytes, truncate_decimal -from pyiceberg.utils.iceberg_base_model import IcebergBaseModel from pyiceberg.utils.parsing import ParseNumberFromBrackets from pyiceberg.utils.singleton import Singleton diff --git a/pyiceberg/typedef.py b/pyiceberg/typedef.py index 228b9a927c..bdf467101c 100644 --- a/pyiceberg/typedef.py +++ b/pyiceberg/typedef.py @@ -18,12 +18,15 @@ from abc import abstractmethod from decimal import Decimal +from functools import cached_property from typing import ( + TYPE_CHECKING, Any, Callable, Dict, - List, + Optional, Protocol, + Set, Tuple, TypeVar, Union, @@ -31,6 +34,11 @@ ) from uuid import UUID +from pydantic import BaseModel + +if TYPE_CHECKING: + from pyiceberg.types import StructType + class FrozenDict(Dict[Any, Any]): def __setitem__(self, instance: Any, value: Any) -> None: @@ -42,7 +50,6 @@ def update(self, *args: Any, **kwargs: Any) -> None: EMPTY_DICT = FrozenDict() - K = TypeVar("K") V = TypeVar("V") @@ -72,34 +79,79 @@ class StructProtocol(Protocol): # pragma: no cover """A generic protocol used by accessors to get and set at positions of an object""" @abstractmethod - def get(self, pos: int) -> Any: + def __getitem__(self, pos: int) -> Any: ... @abstractmethod - def set(self, pos: int, value: Any) -> None: + def __setitem__(self, pos: int, value: Any) -> None: ... +class IcebergBaseModel(BaseModel): + """ + This class extends the Pydantic BaseModel to set default values by overriding them. + + This is because we always want to set by_alias to True. In Python, the dash can't + be used in variable names, and this is used throughout the Iceberg spec. + + The same goes for exclude_none, if a field is None we want to omit it from + serialization, for example, the doc attribute on the NestedField object. + Default non-null values will be serialized. + + This is recommended by Pydantic: + https://pydantic-docs.helpmanual.io/usage/model_config/#change-behaviour-globally + """ + + class Config: + keep_untouched = (cached_property,) + allow_population_by_field_name = True + frozen = True + + def _exclude_private_properties(self, exclude: Optional[Set[str]] = None) -> Set[str]: + # A small trick to exclude private properties. Properties are serialized by pydantic, + # regardless if they start with an underscore. + # This will look at the dict, and find the fields and exclude them + return set.union( + {field for field in self.__dict__ if field.startswith("_") and not field == "__root__"}, exclude or set() + ) + + def dict(self, exclude_none: bool = True, exclude: Optional[Set[str]] = None, **kwargs: Any) -> Dict[str, Any]: + return super().dict(exclude_none=exclude_none, exclude=self._exclude_private_properties(exclude), **kwargs) + + def json(self, exclude_none: bool = True, exclude: Optional[Set[str]] = None, by_alias: bool = True, **kwargs: Any) -> str: + return super().json( + exclude_none=exclude_none, exclude=self._exclude_private_properties(exclude), by_alias=by_alias, **kwargs + ) + + class Record(StructProtocol): - _data: List[Union[Any, StructProtocol]] + _position_to_field_name: Dict[int, str] + + def __init__(self, *data: Any, struct: Optional[StructType] = None, **named_data: Any) -> None: + if struct is not None: + self._position_to_field_name = {idx: field.name for idx, field in enumerate(struct.fields)} + elif named_data: + # Order of named_data is preserved (PEP 468) so this can be used to generate the position dict + self._position_to_field_name = {idx: name for idx, name in enumerate(named_data.keys())} + else: + self._position_to_field_name = {idx: f"field{idx + 1}" for idx in range(len(data))} - @staticmethod - def of(num_fields: int) -> Record: - return Record(*([None] * num_fields)) + for idx, d in enumerate(data): + self[idx] = d - def __init__(self, *data: Union[Any, StructProtocol]) -> None: - self._data = list(data) + for field_name, d in named_data.items(): + self.__setattr__(field_name, d) - def set(self, pos: int, value: Any) -> None: - print(f"set({pos}, {repr(value)})") - self._data[pos] = value + def __setitem__(self, pos: int, value: Any) -> None: + self.__setattr__(self._position_to_field_name[pos], value) - def get(self, pos: int) -> Any: - return self._data[pos] + def __getitem__(self, pos: int) -> Any: + return self.__getattribute__(self._position_to_field_name[pos]) def __eq__(self, other: Any) -> bool: - # For testing - return True if isinstance(other, Record) and other._data == self._data else False + if not isinstance(other, Record): + return False + return self.__dict__ == other.__dict__ def __repr__(self) -> str: - return f"{self.__class__.__name__}[" + ", ".join([repr(e) for e in self._data]) + "]" + return f"{self.__class__.__name__}[{', '.join(f'{key}={repr(value)}' for key, value in self.__dict__.items() if not key.startswith('_'))}]" diff --git a/pyiceberg/types.py b/pyiceberg/types.py index 6a736ed19d..e11f6138c9 100644 --- a/pyiceberg/types.py +++ b/pyiceberg/types.py @@ -45,7 +45,7 @@ from pydantic import Field, PrivateAttr from pydantic.typing import AnyCallable -from pyiceberg.utils.iceberg_base_model import IcebergBaseModel +from pyiceberg.typedef import IcebergBaseModel from pyiceberg.utils.parsing import ParseNumberFromBrackets from pyiceberg.utils.singleton import Singleton @@ -280,6 +280,9 @@ def __str__(self) -> str: def __repr__(self) -> str: return f"StructType(fields=({', '.join(map(repr, self.fields))},))" + def __len__(self) -> int: + return len(self.fields) + class ListType(IcebergType): """A list type in Iceberg diff --git a/pyiceberg/utils/iceberg_base_model.py b/pyiceberg/utils/iceberg_base_model.py deleted file mode 100644 index 8c5bc94f68..0000000000 --- a/pyiceberg/utils/iceberg_base_model.py +++ /dev/null @@ -1,62 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. -from functools import cached_property -from typing import ( - Any, - Dict, - Optional, - Set, -) - -from pydantic import BaseModel - - -class IcebergBaseModel(BaseModel): - """ - This class extends the Pydantic BaseModel to set default values by overriding them. - - This is because we always want to set by_alias to True. In Python, the dash can't - be used in variable names, and this is used throughout the Iceberg spec. - - The same goes for exclude_none, if a field is None we want to omit it from - serialization, for example, the doc attribute on the NestedField object. - Default non-null values will be serialized. - - This is recommended by Pydantic: - https://pydantic-docs.helpmanual.io/usage/model_config/#change-behaviour-globally - """ - - class Config: - keep_untouched = (cached_property,) - allow_population_by_field_name = True - frozen = True - - def _exclude_private_properties(self, exclude: Optional[Set[str]] = None) -> Set[str]: - # A small trick to exclude private properties. Properties are serialized by pydantic, - # regardless if they start with an underscore. - # This will look at the dict, and find the fields and exclude them - return set.union( - {field for field in self.__dict__ if field.startswith("_") and not field == "__root__"}, exclude or set() - ) - - def dict(self, exclude_none: bool = True, exclude: Optional[Set[str]] = None, **kwargs: Any) -> Dict[str, Any]: - return super().dict(exclude_none=exclude_none, exclude=self._exclude_private_properties(exclude), **kwargs) - - def json(self, exclude_none: bool = True, exclude: Optional[Set[str]] = None, by_alias: bool = True, **kwargs: Any) -> str: - return super().json( - exclude_none=exclude_none, exclude=self._exclude_private_properties(exclude), by_alias=by_alias, **kwargs - ) diff --git a/tests/avro/test_file.py b/tests/avro/test_file.py index fc46963375..cdb973a395 100644 --- a/tests/avro/test_file.py +++ b/tests/avro/test_file.py @@ -17,21 +17,30 @@ import pytest from pyiceberg.avro.codecs import DeflateCodec -from pyiceberg.avro.file import AvroFileHeader +from pyiceberg.avro.file import META_SCHEMA, AvroFileHeader def get_deflate_compressor() -> None: - header = AvroFileHeader(bytes(0), {"avro.codec": "deflate"}, bytes(16)) + header = AvroFileHeader(struct=META_SCHEMA) + header[0] = bytes(0) + header[1] = {"avro.codec": "deflate"} + header[2] = bytes(16) assert header.compression_codec() == DeflateCodec def get_null_compressor() -> None: - header = AvroFileHeader(bytes(0), {"avro.codec": "null"}, bytes(16)) + header = AvroFileHeader(struct=META_SCHEMA) + header[0] = bytes(0) + header[1] = {"avro.codec": "null"} + header[2] = bytes(16) assert header.compression_codec() is None def test_unknown_codec() -> None: - header = AvroFileHeader(bytes(0), {"avro.codec": "unknown"}, bytes(16)) + header = AvroFileHeader(struct=META_SCHEMA) + header[0] = bytes(0) + header[1] = {"avro.codec": "unknown"} + header[2] = bytes(16) with pytest.raises(ValueError) as exc_info: header.compression_codec() @@ -40,7 +49,10 @@ def test_unknown_codec() -> None: def test_missing_schema() -> None: - header = AvroFileHeader(bytes(0), {}, bytes(16)) + header = AvroFileHeader(struct=META_SCHEMA) + header[0] = bytes(0) + header[1] = {} + header[2] = bytes(16) with pytest.raises(ValueError) as exc_info: header.get_schema() diff --git a/tests/avro/test_reader.py b/tests/avro/test_reader.py index f8ec3326ca..b8736dd69b 100644 --- a/tests/avro/test_reader.py +++ b/tests/avro/test_reader.py @@ -19,6 +19,7 @@ import pytest +from pyiceberg.avro.decoder import BinaryDecoder from pyiceberg.avro.file import AvroFile from pyiceberg.avro.reader import ( BinaryReader, @@ -30,14 +31,16 @@ FloatReader, IntegerReader, StringReader, + StructReader, TimeReader, TimestampReader, TimestamptzReader, UUIDReader, ) from pyiceberg.avro.resolver import construct_reader +from pyiceberg.io.memory import MemoryInputStream from pyiceberg.io.pyarrow import PyArrowFileIO -from pyiceberg.manifest import _convert_pos_to_dict +from pyiceberg.manifest import MANIFEST_ENTRY_SCHEMA, DataFile, ManifestEntry from pyiceberg.schema import Schema from pyiceberg.typedef import Record from pyiceberg.types import ( @@ -49,9 +52,7 @@ FixedType, FloatType, IntegerType, - ListType, LongType, - MapType, NestedField, PrimitiveType, StringType, @@ -64,7 +65,11 @@ def test_read_header(generated_manifest_entry_file: str, iceberg_manifest_entry_schema: Schema) -> None: - with AvroFile(PyArrowFileIO().new_input(generated_manifest_entry_file)) as reader: + with AvroFile[ManifestEntry]( + PyArrowFileIO().new_input(generated_manifest_entry_file), + MANIFEST_ENTRY_SCHEMA, + {-1: ManifestEntry, 2: DataFile}, + ) as reader: header = reader.header assert header.magic == b"Obj\x01" @@ -264,187 +269,6 @@ def test_read_header(generated_manifest_entry_file: str, iceberg_manifest_entry_ assert header.get_schema() == iceberg_manifest_entry_schema -def test_read_manifest_entry_file(generated_manifest_entry_file: str) -> None: - with AvroFile(PyArrowFileIO().new_input(generated_manifest_entry_file)) as reader: - # Consume the generator - records = list(reader) - - assert len(records) == 2, f"Expected 2 records, got {len(records)}" - assert records[0] == Record( - 1, - 8744736658442914487, - Record( - "/home/iceberg/warehouse/nyc/taxis_partitioned/data/VendorID=null/00000-633-d8a4223e-dc97-45a1-86e1-adaba6e8abd7-00001.parquet", - "PARQUET", - Record(1, 1925), - 19513, - 388872, - 67108864, - { - 1: 53, - 2: 98153, - 3: 98693, - 4: 53, - 5: 53, - 6: 53, - 7: 17425, - 8: 18528, - 9: 53, - 10: 44788, - 11: 35571, - 12: 53, - 13: 1243, - 14: 2355, - 15: 12750, - 16: 4029, - 17: 110, - 18: 47194, - 19: 2948, - }, - { - 1: 19513, - 2: 19513, - 3: 19513, - 4: 19513, - 5: 19513, - 6: 19513, - 7: 19513, - 8: 19513, - 9: 19513, - 10: 19513, - 11: 19513, - 12: 19513, - 13: 19513, - 14: 19513, - 15: 19513, - 16: 19513, - 17: 19513, - 18: 19513, - 19: 19513, - }, - { - 1: 19513, - 2: 0, - 3: 0, - 4: 19513, - 5: 19513, - 6: 19513, - 7: 0, - 8: 0, - 9: 19513, - 10: 0, - 11: 0, - 12: 19513, - 13: 0, - 14: 0, - 15: 0, - 16: 0, - 17: 0, - 18: 0, - 19: 0, - }, - {16: 0, 17: 0, 18: 0, 19: 0, 10: 0, 11: 0, 12: 0, 13: 0, 14: 0, 15: 0}, - { - 2: b"2020-04-01 00:00", - 3: b"2020-04-01 00:12", - 7: b"\x03\x00\x00\x00", - 8: b"\x01\x00\x00\x00", - 10: b"\xf6(\\\x8f\xc2\x05S\xc0", - 11: b"\x00\x00\x00\x00\x00\x00\x00\x00", - 13: b"\x00\x00\x00\x00\x00\x00\x00\x00", - 14: b"\x00\x00\x00\x00\x00\x00\xe0\xbf", - 15: b")\\\x8f\xc2\xf5(\x08\xc0", - 16: b"\x00\x00\x00\x00\x00\x00\x00\x00", - 17: b"\x00\x00\x00\x00\x00\x00\x00\x00", - 18: b"\xf6(\\\x8f\xc2\xc5S\xc0", - 19: b"\x00\x00\x00\x00\x00\x00\x04\xc0", - }, - { - 2: b"2020-04-30 23:5:", - 3: b"2020-05-01 00:41", - 7: b"\t\x01\x00\x00", - 8: b"\t\x01\x00\x00", - 10: b"\xcd\xcc\xcc\xcc\xcc,_@", - 11: b"\x1f\x85\xebQ\\\xe2\xfe@", - 13: b"\x00\x00\x00\x00\x00\x00\x12@", - 14: b"\x00\x00\x00\x00\x00\x00\xe0?", - 15: b"q=\n\xd7\xa3\xf01@", - 16: b"\x00\x00\x00\x00\x00`B@", - 17: b"333333\xd3?", - 18: b"\x00\x00\x00\x00\x00\x18b@", - 19: b"\x00\x00\x00\x00\x00\x00\x04@", - }, - None, - [4], - 0, - ), - ) - - -def test_read_manifest_file_file(generated_manifest_file_file: str) -> None: - with AvroFile(PyArrowFileIO().new_input(generated_manifest_file_file)) as reader: - # Consume the generator - records = list(reader) - - assert len(records) == 1, f"Expected 1 records, got {len(records)}" - actual = records[0] - expected = Record( - actual.get(0), - 7989, - 0, - 9182715666859759686, - 3, - 0, - 0, - [Record(True, False, b"\x01\x00\x00\x00", b"\x02\x00\x00\x00")], - 237993, - 0, - 0, - ) - assert actual == expected - - -def test_null_list_convert_pos_to_dict() -> None: - data = _convert_pos_to_dict( - Schema( - NestedField(name="field", field_id=1, field_type=ListType(element_id=2, element=StringType(), element_required=False)) - ), - Record(None), - ) - assert data["field"] is None - - -def test_null_dict_convert_pos_to_dict() -> None: - data = _convert_pos_to_dict( - Schema( - NestedField( - name="field", - field_id=1, - field_type=MapType(key_id=2, key_type=StringType(), value_id=3, value_type=StringType(), value_required=False), - ) - ), - Record(None), - ) - assert data["field"] is None - - -def test_null_struct_convert_pos_to_dict() -> None: - data = _convert_pos_to_dict( - Schema( - NestedField( - name="field", - field_id=1, - field_type=StructType( - NestedField(2, "required_field", StringType(), True), NestedField(3, "optional_field", IntegerType()) - ), - required=False, - ) - ), - Record(None), - ) - assert data["field"] is None - - def test_fixed_reader() -> None: assert construct_reader(FixedType(22)) == FixedReader(22) @@ -509,3 +333,51 @@ class UnknownType(PrimitiveType): def test_uuid_reader() -> None: assert construct_reader(UUIDType()) == UUIDReader() + + +def test_read_struct() -> None: + mis = MemoryInputStream(b"\x18") + decoder = BinaryDecoder(mis) + + struct = StructType(NestedField(1, "id", IntegerType(), required=True)) + result = StructReader(((0, IntegerReader()),), Record, struct).read(decoder) + assert repr(result) == "Record[id=12]" + + +def test_read_struct_lambda() -> None: + mis = MemoryInputStream(b"\x18") + decoder = BinaryDecoder(mis) + + struct = StructType(NestedField(1, "id", IntegerType(), required=True)) + # You can also pass in an arbitrary function that returns a struct + result = StructReader( + ((0, IntegerReader()),), lambda struct: Record(struct=struct), struct # pylint: disable=unnecessary-lambda + ).read(decoder) + assert repr(result) == "Record[id=12]" + + +def test_read_not_struct_type() -> None: + mis = MemoryInputStream(b"\x18") + decoder = BinaryDecoder(mis) + + struct = StructType(NestedField(1, "id", IntegerType(), required=True)) + with pytest.raises(ValueError) as exc_info: + _ = StructReader(((0, IntegerReader()),), str, struct).read(decoder) # type: ignore + + assert "Incompatible with StructProtocol: " in str(exc_info.value) + + +def test_read_struct_exception_handling() -> None: + mis = MemoryInputStream(b"\x18") + decoder = BinaryDecoder(mis) + + def raise_err(struct: StructType) -> None: + raise TypeError("boom") + + struct = StructType(NestedField(1, "id", IntegerType(), required=True)) + # You can also pass in an arbitrary function that returns a struct + + with pytest.raises(ValueError) as exc_info: + _ = StructReader(((0, IntegerReader()),), raise_err, struct).read(decoder) # type: ignore + + assert "Unable to initialize struct:" in str(exc_info.value) diff --git a/tests/avro/test_resolver.py b/tests/avro/test_resolver.py index c36b76922a..42e9146c48 100644 --- a/tests/avro/test_resolver.py +++ b/tests/avro/test_resolver.py @@ -14,8 +14,14 @@ # KIND, either express or implied. See the License for the # specific language governing permissions and limitations # under the License. + +from tempfile import TemporaryDirectory +from typing import Optional + import pytest +from pydantic import Field +from pyiceberg.avro.file import AvroFile from pyiceberg.avro.reader import ( DecimalReader, DoubleReader, @@ -26,7 +32,9 @@ StructReader, ) from pyiceberg.avro.resolver import ResolveError, resolve +from pyiceberg.io.pyarrow import PyArrowFileIO from pyiceberg.schema import Schema +from pyiceberg.typedef import Record from pyiceberg.types import ( BinaryType, DecimalType, @@ -57,14 +65,16 @@ def test_resolver() -> None: NestedField(6, "preferences", MapType(7, StringType(), 8, StringType())), schema_id=1, ) + + location_struct = StructType( + NestedField(4, "lat", DoubleType()), + NestedField(5, "long", DoubleType()), + ) read_schema = Schema( NestedField( 3, "location", - StructType( - NestedField(4, "lat", DoubleType()), - NestedField(5, "long", DoubleType()), - ), + location_struct, ), NestedField(1, "id", LongType()), NestedField(6, "preferences", MapType(7, StringType(), 8, StringType())), @@ -82,11 +92,15 @@ def test_resolver() -> None: ( (0, DoubleReader()), (1, DoubleReader()), - ) + ), + Record, + location_struct, ), ), (2, MapReader(StringReader(), StringReader())), - ) + ), + Record, + read_schema.as_struct(), ) @@ -223,3 +237,46 @@ def test_resolve_decimal_to_decimal_reduce_precision() -> None: _ = resolve(DecimalType(19, 25), DecimalType(10, 25)) == DecimalReader(22, 25) assert "Cannot reduce precision from decimal(19, 25) to decimal(10, 25)" in str(exc_info.value) + + +def test_column_assignment() -> None: + int_schema = { + "type": "record", + "name": "ints", + "fields": [ + {"name": "a", "type": "int", "field-id": 1}, + {"name": "b", "type": "int", "field-id": 2}, + {"name": "c", "type": "int", "field-id": 3}, + ], + } + + from fastavro import parse_schema, writer + + parsed_schema = parse_schema(int_schema) + + int_records = [ + { + "a": 1, + "b": 2, + "c": 3, + } + ] + + with TemporaryDirectory() as tmpdir: + tmp_avro_file = tmpdir + "/manifest.avro" + with open(tmp_avro_file, "wb") as out: + writer(out, parsed_schema, int_records) + + class Ints(Record): + c: int = Field() + d: Optional[int] = Field() + + MANIFEST_ENTRY_SCHEMA = Schema( + NestedField(3, "c", IntegerType(), required=True), + NestedField(4, "d", IntegerType(), required=False), + ) + + with AvroFile[Ints](PyArrowFileIO().new_input(tmp_avro_file), MANIFEST_ENTRY_SCHEMA, {-1: Ints}) as reader: + records = list(reader) + + assert repr(records) == "[Ints[c=3, d=None]]" diff --git a/tests/expressions/test_evaluator.py b/tests/expressions/test_evaluator.py index 95766b0024..b57ede62f2 100644 --- a/tests/expressions/test_evaluator.py +++ b/tests/expressions/test_evaluator.py @@ -14,6 +14,8 @@ # KIND, either express or implied. See the License for the # specific language governing permissions and limitations # under the License. +from typing import Optional + from pyiceberg.expressions import ( AlwaysFalse, AlwaysTrue, @@ -52,6 +54,20 @@ ) +def _record_simple(id: int, data: Optional[str]) -> Record: # pylint: disable=redefined-builtin + r = Record(struct=SIMPLE_SCHEMA.as_struct()) + r[0] = id + r[1] = data + return r + + +def _record_float(id: float, f: float) -> Record: # pylint: disable=redefined-builtin + r = Record(struct=FLOAT_SCHEMA.as_struct()) + r[0] = id + r[1] = f + return r + + def test_true() -> None: evaluate = expression_evaluator(SIMPLE_SCHEMA, AlwaysTrue(), case_sensitive=True) assert evaluate(Record(1, "a")) @@ -131,16 +147,16 @@ def test_not_null() -> None: def test_is_nan() -> None: evaluate = expression_evaluator(FLOAT_SCHEMA, IsNaN("f"), case_sensitive=True) - assert not evaluate(Record(2, 0.0)) - assert not evaluate(Record(3, float("infinity"))) - assert evaluate(Record(4, float("nan"))) + assert not evaluate(_record_float(2, f=0.0)) + assert not evaluate(_record_float(3, f=float("infinity"))) + assert evaluate(_record_float(4, f=float("nan"))) def test_not_nan() -> None: evaluate = expression_evaluator(FLOAT_SCHEMA, NotNaN("f"), case_sensitive=True) - assert evaluate(Record(2, 0.0)) - assert evaluate(Record(3, float("infinity"))) - assert not evaluate(Record(4, float("nan"))) + assert evaluate(_record_float(2, f=0.0)) + assert evaluate(_record_float(3, f=float("infinity"))) + assert not evaluate(_record_float(4, f=float("nan"))) def test_not() -> None: diff --git a/tests/expressions/test_expressions.py b/tests/expressions/test_expressions.py index 7b26d45a28..5b82c14744 100644 --- a/tests/expressions/test_expressions.py +++ b/tests/expressions/test_expressions.py @@ -63,6 +63,9 @@ from pyiceberg.schema import Accessor, Schema from pyiceberg.typedef import Record from pyiceberg.types import ( + BinaryType, + BooleanType, + DecimalType, DoubleType, FloatType, IntegerType, @@ -70,6 +73,8 @@ LongType, NestedField, StringType, + StructType, + UUIDType, ) from pyiceberg.utils.singleton import Singleton @@ -605,22 +610,37 @@ def test_invert_always() -> None: def test_accessor_base_class() -> None: """Test retrieving a value at a position of a container using an accessor""" - struct = Record(*([None] * 12)) + struct = Record( + struct=StructType( + NestedField(1, "a", StringType()), + NestedField(2, "b", StringType()), + NestedField(3, "c", StringType()), + NestedField(4, "d", IntegerType()), + NestedField(5, "e", IntegerType()), + NestedField(6, "f", IntegerType()), + NestedField(7, "g", FloatType()), + NestedField(8, "h", DecimalType(8, 4)), + NestedField(9, "i", UUIDType()), + NestedField(10, "j", BooleanType()), + NestedField(11, "k", BooleanType()), + NestedField(12, "l", BinaryType()), + ) + ) uuid_value = uuid.uuid4() - struct.set(0, "foo") - struct.set(1, "bar") - struct.set(2, "baz") - struct.set(3, 1) - struct.set(4, 2) - struct.set(5, 3) - struct.set(6, 1.234) - struct.set(7, Decimal("1.234")) - struct.set(8, uuid_value) - struct.set(9, True) - struct.set(10, False) - struct.set(11, b"\x19\x04\x9e?") + struct[0] = "foo" + struct[1] = "bar" + struct[2] = "baz" + struct[3] = 1 + struct[4] = 2 + struct[5] = 3 + struct[6] = 1.234 + struct[7] = Decimal("1.234") + struct[8] = uuid_value + struct[9] = True + struct[10] = False + struct[11] = b"\x19\x04\x9e?" assert Accessor(position=0).get(struct) == "foo" assert Accessor(position=1).get(struct) == "bar" @@ -902,15 +922,15 @@ def test_less_than_or_equal() -> None: def test_bound_reference_eval(table_schema_simple: Schema) -> None: """Test creating a BoundReference and evaluating it on a StructProtocol""" - struct = Record(None, None, None, None) + struct = Record(struct=table_schema_simple.as_struct()) - struct.set(pos=1, value="foovalue") - struct.set(pos=2, value=123) - struct.set(pos=3, value=True) + struct[0] = "foovalue" + struct[1] = 123 + struct[2] = True - position1_accessor = Accessor(position=1) - position2_accessor = Accessor(position=2) - position3_accessor = Accessor(position=3) + position1_accessor = Accessor(position=0) + position2_accessor = Accessor(position=1) + position3_accessor = Accessor(position=2) field1 = table_schema_simple.find_field(1) field2 = table_schema_simple.find_field(2) diff --git a/tests/expressions/test_visitors.py b/tests/expressions/test_visitors.py index e79c353f81..13e4d01eea 100644 --- a/tests/expressions/test_visitors.py +++ b/tests/expressions/test_visitors.py @@ -786,12 +786,8 @@ def _to_byte_buffer(field_type: IcebergType, val: Any) -> bytes: def _to_manifest_file(*partitions: PartitionFieldSummary) -> ManifestFile: - return ManifestFile( - manifest_path="", - manifest_length=0, - partition_spec_id=0, - partitions=partitions, - ) + """Helper to create a ManifestFile""" + return ManifestFile(manifest_path="", manifest_length=0, partition_spec_id=0, partitions=partitions) INT_MIN_VALUE = 30 diff --git a/tests/table/test_snapshots.py b/tests/table/test_snapshots.py index 16ba83153e..b119ae9945 100644 --- a/tests/table/test_snapshots.py +++ b/tests/table/test_snapshots.py @@ -17,8 +17,6 @@ # pylint:disable=redefined-outer-name,eval-used import pytest -from pyiceberg.io.pyarrow import PyArrowFileIO -from pyiceberg.manifest import ManifestContent, ManifestFile, PartitionFieldSummary from pyiceberg.table.snapshots import Operation, Snapshot, Summary @@ -121,40 +119,3 @@ def test_snapshot_with_properties_repr(snapshot_with_properties: Snapshot) -> No == """Snapshot(snapshot_id=25, parent_snapshot_id=19, sequence_number=200, timestamp_ms=1602638573590, manifest_list='s3:/a/b/c.avro', summary=Summary(Operation.APPEND, **{'foo': 'bar'}), schema_id=3)""" ) assert snapshot_with_properties == eval(repr(snapshot_with_properties)) - - -def test_fetch_manifest_list(generated_manifest_file_file: str) -> None: - snapshot = Snapshot( - snapshot_id=25, - parent_snapshot_id=19, - sequence_number=200, - timestamp_ms=1602638573590, - manifest_list=generated_manifest_file_file, - summary=Summary(Operation.APPEND), - schema_id=3, - ) - io = PyArrowFileIO() - actual = snapshot.manifests(io) - assert actual == [ - ManifestFile( - manifest_path=actual[0].manifest_path, # Is a temp path that changes every time - manifest_length=7989, - partition_spec_id=0, - content=ManifestContent.DATA, - sequence_number=0, - min_sequence_number=0, - added_snapshot_id=9182715666859759686, - added_data_files_count=3, - existing_data_files_count=0, - deleted_data_files_count=0, - added_rows_count=237993, - existing_rows_counts=None, - deleted_rows_count=0, - partitions=[ - PartitionFieldSummary( - contains_null=True, contains_nan=False, lower_bound=b"\x01\x00\x00\x00", upper_bound=b"\x02\x00\x00\x00" - ) - ], - key_metadata=None, - ) - ] diff --git a/tests/test_schema.py b/tests/test_schema.py index a6fba2cff0..4a59631acd 100644 --- a/tests/test_schema.py +++ b/tests/test_schema.py @@ -389,10 +389,10 @@ class TestStruct(StructProtocol): def __init__(self, pos: Dict[int, Any] = EMPTY_DICT): self._pos: Dict[int, Any] = pos - def set(self, pos: int, value: Any) -> None: + def __setitem__(self, pos: int, value: Any) -> None: pass - def get(self, pos: int) -> Any: + def __getitem__(self, pos: int) -> Any: return self._pos[pos] accessors = build_position_accessors(table_schema_nested) diff --git a/tests/test_transforms.py b/tests/test_transforms.py index 72776b8574..8ba80e396b 100644 --- a/tests/test_transforms.py +++ b/tests/test_transforms.py @@ -62,6 +62,7 @@ VoidTransform, YearTransform, ) +from pyiceberg.typedef import IcebergBaseModel from pyiceberg.types import ( BinaryType, BooleanType, @@ -87,7 +88,6 @@ timestamp_to_micros, timestamptz_to_micros, ) -from pyiceberg.utils.iceberg_base_model import IcebergBaseModel @pytest.mark.parametrize( diff --git a/tests/test_typedef.py b/tests/test_typedef.py index 628e674443..43388addca 100644 --- a/tests/test_typedef.py +++ b/tests/test_typedef.py @@ -16,7 +16,14 @@ # under the License. import pytest +from pyiceberg.schema import Schema from pyiceberg.typedef import FrozenDict, KeyDefaultDict, Record +from pyiceberg.types import ( + IntegerType, + NestedField, + StringType, + StructType, +) def test_setitem_frozendict() -> None: @@ -39,6 +46,44 @@ def one(_: int) -> int: assert defaultdict[22] == 1 -def test_record_repr() -> None: - r = Record(1, "vo", True) - assert repr(r) == "Record[1, 'vo', True]" +def test_record_repr(table_schema_simple: Schema) -> None: + r = Record("vo", 1, True, struct=table_schema_simple.as_struct()) + assert repr(r) == "Record[foo='vo', bar=1, baz=True]" + + +def test_named_record() -> None: + r = Record(struct=StructType(NestedField(0, "id", IntegerType()), NestedField(1, "name", StringType()))) + + with pytest.raises(AttributeError): + assert r.id is None # type: ignore + + with pytest.raises(AttributeError): + assert r.name is None # type: ignore + + r[0] = 123 + r[1] = "abc" + + assert r[0] == 123 + assert r[1] == "abc" + + assert r.id == 123 # type: ignore + assert r.name == "abc" # type: ignore + + +def test_record_positional_args() -> None: + r = Record(1, "a", True) + assert repr(r) == "Record[field1=1, field2='a', field3=True]" + + +def test_record_named_args() -> None: + r = Record(foo=1, bar="a", baz=True) + + assert r.foo == 1 # type: ignore + assert r.bar == "a" # type: ignore + assert r.baz is True # type: ignore + + assert r[0] == 1 + assert r[1] == "a" + assert r[2] is True + + assert repr(r) == "Record[foo=1, bar='a', baz=True]" diff --git a/tests/test_types.py b/tests/test_types.py index cda1309ede..e2c0272b45 100644 --- a/tests/test_types.py +++ b/tests/test_types.py @@ -20,6 +20,7 @@ import pytest from pydantic import ValidationError +from pyiceberg.typedef import IcebergBaseModel from pyiceberg.types import ( BinaryType, BooleanType, @@ -42,7 +43,6 @@ TimeType, UUIDType, ) -from pyiceberg.utils.iceberg_base_model import IcebergBaseModel non_parameterized_types = [ (1, BooleanType), diff --git a/tests/utils/test_manifest.py b/tests/utils/test_manifest.py index e78d0db237..548d94cb8f 100644 --- a/tests/utils/test_manifest.py +++ b/tests/utils/test_manifest.py @@ -19,12 +19,8 @@ from pyiceberg.io.pyarrow import PyArrowFileIO from pyiceberg.manifest import ( DataFile, - DataFileContent, FileFormat, - ManifestContent, - ManifestEntry, ManifestEntryStatus, - ManifestFile, PartitionFieldSummary, read_manifest_entry, read_manifest_list, @@ -35,513 +31,152 @@ def test_read_manifest_entry(generated_manifest_entry_file: str) -> None: input_file = PyArrowFileIO().new_input(location=generated_manifest_entry_file) + manifest_entries = list(read_manifest_entry(input_file)) + manifest_entry = manifest_entries[0] + + assert manifest_entry.status == ManifestEntryStatus.ADDED + assert manifest_entry.snapshot_id == 8744736658442914487 + assert manifest_entry.sequence_number is None + assert isinstance(manifest_entry.data_file, DataFile) + + data_file = manifest_entry.data_file + + assert data_file.content is None assert ( - list(read_manifest_entry(input_file)) - == [ - ManifestEntry( - status=ManifestEntryStatus.ADDED, - snapshot_id=8744736658442914487, - sequence_number=None, - data_file=DataFile( - content=DataFileContent.DATA, - file_path="/home/iceberg/warehouse/nyc/taxis_partitioned/data/VendorID=null/00000-633-d8a4223e-dc97-45a1-86e1-adaba6e8abd7-00001.parquet", - file_format=FileFormat.PARQUET, - partition={"VendorID": 1, "tpep_pickup_datetime": 1925}, - record_count=19513, - file_size_in_bytes=388872, - block_size_in_bytes=67108864, - column_sizes={ - 1: 53, - 2: 98153, - 3: 98693, - 4: 53, - 5: 53, - 6: 53, - 7: 17425, - 8: 18528, - 9: 53, - 10: 44788, - 11: 35571, - 12: 53, - 13: 1243, - 14: 2355, - 15: 12750, - 16: 4029, - 17: 110, - 18: 47194, - 19: 2948, - }, - value_counts={ - 1: 19513, - 2: 19513, - 3: 19513, - 4: 19513, - 5: 19513, - 6: 19513, - 7: 19513, - 8: 19513, - 9: 19513, - 10: 19513, - 11: 19513, - 12: 19513, - 13: 19513, - 14: 19513, - 15: 19513, - 16: 19513, - 17: 19513, - 18: 19513, - 19: 19513, - }, - null_value_counts={ - 1: 19513, - 2: 0, - 3: 0, - 4: 19513, - 5: 19513, - 6: 19513, - 7: 0, - 8: 0, - 9: 19513, - 10: 0, - 11: 0, - 12: 19513, - 13: 0, - 14: 0, - 15: 0, - 16: 0, - 17: 0, - 18: 0, - 19: 0, - }, - nan_value_counts={16: 0, 17: 0, 18: 0, 19: 0, 10: 0, 11: 0, 12: 0, 13: 0, 14: 0, 15: 0}, - distinct_counts=None, - lower_bounds={ - 2: b"2020-04-01 00:00", - 3: b"2020-04-01 00:12", - 7: b"\x03\x00\x00\x00", - 8: b"\x01\x00\x00\x00", - 10: b"\xf6(\\\x8f\xc2\x05S\xc0", - 11: b"\x00\x00\x00\x00\x00\x00\x00\x00", - 13: b"\x00\x00\x00\x00\x00\x00\x00\x00", - 14: b"\x00\x00\x00\x00\x00\x00\xe0\xbf", - 15: b")\\\x8f\xc2\xf5(\x08\xc0", - 16: b"\x00\x00\x00\x00\x00\x00\x00\x00", - 17: b"\x00\x00\x00\x00\x00\x00\x00\x00", - 18: b"\xf6(\\\x8f\xc2\xc5S\xc0", - 19: b"\x00\x00\x00\x00\x00\x00\x04\xc0", - }, - upper_bounds={ - 2: b"2020-04-30 23:5:", - 3: b"2020-05-01 00:41", - 7: b"\t\x01\x00\x00", - 8: b"\t\x01\x00\x00", - 10: b"\xcd\xcc\xcc\xcc\xcc,_@", - 11: b"\x1f\x85\xebQ\\\xe2\xfe@", - 13: b"\x00\x00\x00\x00\x00\x00\x12@", - 14: b"\x00\x00\x00\x00\x00\x00\xe0?", - 15: b"q=\n\xd7\xa3\xf01@", - 16: b"\x00\x00\x00\x00\x00`B@", - 17: b"333333\xd3?", - 18: b"\x00\x00\x00\x00\x00\x18b@", - 19: b"\x00\x00\x00\x00\x00\x00\x04@", - }, - key_metadata=None, - split_offsets=[4], - equality_ids=None, - sort_order_id=0, - ), - ), - ManifestEntry( - status=ManifestEntryStatus.ADDED, - snapshot_id=8744736658442914487, - sequence_number=None, - data_file=DataFile( - content=DataFileContent.DATA, - file_path="/home/iceberg/warehouse/nyc/taxis_partitioned/data/VendorID=1/00000-633-d8a4223e-dc97-45a1-86e1-adaba6e8abd7-00002.parquet", - file_format=FileFormat.PARQUET, - partition={"VendorID": 1, "tpep_pickup_datetime": 1925}, - record_count=95050, - file_size_in_bytes=1265950, - block_size_in_bytes=67108864, - column_sizes={ - 1: 318, - 2: 329806, - 3: 331632, - 4: 15343, - 5: 2351, - 6: 3389, - 7: 71269, - 8: 76429, - 9: 16383, - 10: 86992, - 11: 89608, - 12: 265, - 13: 19377, - 14: 1692, - 15: 76162, - 16: 4354, - 17: 759, - 18: 120650, - 19: 11804, - }, - value_counts={ - 1: 95050, - 2: 95050, - 3: 95050, - 4: 95050, - 5: 95050, - 6: 95050, - 7: 95050, - 8: 95050, - 9: 95050, - 10: 95050, - 11: 95050, - 12: 95050, - 13: 95050, - 14: 95050, - 15: 95050, - 16: 95050, - 17: 95050, - 18: 95050, - 19: 95050, - }, - null_value_counts={ - 1: 0, - 2: 0, - 3: 0, - 4: 0, - 5: 0, - 6: 0, - 7: 0, - 8: 0, - 9: 0, - 10: 0, - 11: 0, - 12: 95050, - 13: 0, - 14: 0, - 15: 0, - 16: 0, - 17: 0, - 18: 0, - 19: 0, - }, - nan_value_counts={16: 0, 17: 0, 18: 0, 19: 0, 10: 0, 11: 0, 12: 0, 13: 0, 14: 0, 15: 0}, - distinct_counts=None, - lower_bounds={ - 1: b"\x01\x00\x00\x00", - 2: b"2020-04-01 00:00", - 3: b"2020-04-01 00:03", - 4: b"\x00\x00\x00\x00", - 5: b"\x01\x00\x00\x00", - 6: b"N", - 7: b"\x01\x00\x00\x00", - 8: b"\x01\x00\x00\x00", - 9: b"\x01\x00\x00\x00", - 10: b"\x00\x00\x00\x00\x00\x00\x00\x00", - 11: b"\x00\x00\x00\x00\x00\x00\x00\x00", - 13: b"\x00\x00\x00\x00\x00\x00\x00\x00", - 14: b"\x00\x00\x00\x00\x00\x00\x00\x00", - 15: b"\x00\x00\x00\x00\x00\x00\x00\x00", - 16: b"\x00\x00\x00\x00\x00\x00\x00\x00", - 17: b"\x00\x00\x00\x00\x00\x00\x00\x00", - 18: b"\x00\x00\x00\x00\x00\x00\x00\x00", - 19: b"\x00\x00\x00\x00\x00\x00\x00\x00", - }, - upper_bounds={ - 1: b"\x01\x00\x00\x00", - 2: b"2020-04-30 23:5:", - 3: b"2020-05-01 00:1:", - 4: b"\x06\x00\x00\x00", - 5: b"c\x00\x00\x00", - 6: b"Y", - 7: b"\t\x01\x00\x00", - 8: b"\t\x01\x00\x00", - 9: b"\x04\x00\x00\x00", - 10: b"\\\x8f\xc2\xf5(8\x8c@", - 11: b"\xcd\xcc\xcc\xcc\xcc,f@", - 13: b"\x00\x00\x00\x00\x00\x00\x1c@", - 14: b"\x9a\x99\x99\x99\x99\x99\xf1?", - 15: b"\x00\x00\x00\x00\x00\x00Y@", - 16: b"\x00\x00\x00\x00\x00\xb0X@", - 17: b"333333\xd3?", - 18: b"\xc3\xf5(\\\x8f:\x8c@", - 19: b"\x00\x00\x00\x00\x00\x00\x04@", - }, - key_metadata=None, - split_offsets=[4], - equality_ids=None, - sort_order_id=0, - ), - ), - ] - != [ - ManifestEntry( - status=ManifestEntryStatus.ADDED, - snapshot_id=8744736658442914487, - sequence_number=None, - data_file=DataFile( - content=DataFileContent.DATA, - file_path="/home/iceberg/warehouse/nyc/taxis_partitioned/data/VendorID=null/00000-633-d8a4223e-dc97-45a1-86e1-adaba6e8abd7-00001.parquet", - file_format=FileFormat.PARQUET, - partition={"VendorID": 1, "tpep_pickup_datetime": None}, - record_count=19513, - file_size_in_bytes=388872, - block_size_in_bytes=67108864, - column_sizes={ - 1: 53, - 2: 98153, - 3: 98693, - 4: 53, - 5: 53, - 6: 53, - 7: 17425, - 8: 18528, - 9: 53, - 10: 44788, - 11: 35571, - 12: 53, - 13: 1243, - 14: 2355, - 15: 12750, - 16: 4029, - 17: 110, - 18: 47194, - 19: 2948, - }, - value_counts={ - 1: 19513, - 2: 19513, - 3: 19513, - 4: 19513, - 5: 19513, - 6: 19513, - 7: 19513, - 8: 19513, - 9: 19513, - 10: 19513, - 11: 19513, - 12: 19513, - 13: 19513, - 14: 19513, - 15: 19513, - 16: 19513, - 17: 19513, - 18: 19513, - 19: 19513, - }, - null_value_counts={ - 1: 19513, - 2: 0, - 3: 0, - 4: 19513, - 5: 19513, - 6: 19513, - 7: 0, - 8: 0, - 9: 19513, - 10: 0, - 11: 0, - 12: 19513, - 13: 0, - 14: 0, - 15: 0, - 16: 0, - 17: 0, - 18: 0, - 19: 0, - }, - nan_value_counts={16: 0, 17: 0, 18: 0, 19: 0, 10: 0, 11: 0, 12: 0, 13: 0, 14: 0, 15: 0}, - distinct_counts=None, - lower_bounds={ - 2: b"2020-04-01 00:00", - 3: b"2020-04-01 00:12", - 7: b"\x03\x00\x00\x00", - 8: b"\x01\x00\x00\x00", - 10: b"\xf6(\\\x8f\xc2\x05S\xc0", - 11: b"\x00\x00\x00\x00\x00\x00\x00\x00", - 13: b"\x00\x00\x00\x00\x00\x00\x00\x00", - 14: b"\x00\x00\x00\x00\x00\x00\xe0\xbf", - 15: b")\\\x8f\xc2\xf5(\x08\xc0", - 16: b"\x00\x00\x00\x00\x00\x00\x00\x00", - 17: b"\x00\x00\x00\x00\x00\x00\x00\x00", - 18: b"\xf6(\\\x8f\xc2\xc5S\xc0", - 19: b"\x00\x00\x00\x00\x00\x00\x04\xc0", - }, - upper_bounds={ - 2: b"2020-04-30 23:5:", - 3: b"2020-05-01 00:41", - 7: b"\t\x01\x00\x00", - 8: b"\t\x01\x00\x00", - 10: b"\xcd\xcc\xcc\xcc\xcc,_@", - 11: b"\x1f\x85\xebQ\\\xe2\xfe@", - 13: b"\x00\x00\x00\x00\x00\x00\x12@", - 14: b"\x00\x00\x00\x00\x00\x00\xe0?", - 15: b"q=\n\xd7\xa3\xf01@", - 16: b"\x00\x00\x00\x00\x00`B@", - 17: b"333333\xd3?", - 18: b"\x00\x00\x00\x00\x00\x18b@", - 19: b"\x00\x00\x00\x00\x00\x00\x04@", - }, - key_metadata=None, - split_offsets=[4], - equality_ids=None, - sort_order_id=0, - ), - ), - ManifestEntry( - status=ManifestEntryStatus.ADDED, - snapshot_id=8744736658442914487, - sequence_number=None, - data_file=DataFile( - content=DataFileContent.DATA, - file_path="/home/iceberg/warehouse/nyc/taxis_partitioned/data/VendorID=1/00000-633-d8a4223e-dc97-45a1-86e1-adaba6e8abd7-00002.parquet", - file_format=FileFormat.PARQUET, - partition={"VendorID": 1, "tpep_pickup_datetime": 1925}, - record_count=95050, - file_size_in_bytes=1265950, - block_size_in_bytes=67108864, - column_sizes={ - 1: 318, - 2: 329806, - 3: 331632, - 4: 15343, - 5: 2351, - 6: 3389, - 7: 71269, - 8: 76429, - 9: 16383, - 10: 86992, - 11: 89608, - 12: 265, - 13: 19377, - 14: 1692, - 15: 76162, - 16: 4354, - 17: 759, - 18: 120650, - 19: 11804, - }, - value_counts={ - 1: 95050, - 2: 95050, - 3: 95050, - 4: 95050, - 5: 95050, - 6: 95050, - 7: 95050, - 8: 95050, - 9: 95050, - 10: 95050, - 11: 95050, - 12: 95050, - 13: 95050, - 14: 95050, - 15: 95050, - 16: 95050, - 17: 95050, - 18: 95050, - 19: 95050, - }, - null_value_counts={ - 1: 0, - 2: 0, - 3: 0, - 4: 0, - 5: 0, - 6: 0, - 7: 0, - 8: 0, - 9: 0, - 10: 0, - 11: 0, - 12: 95050, - 13: 0, - 14: 0, - 15: 0, - 16: 0, - 17: 0, - 18: 0, - 19: 0, - }, - nan_value_counts={16: 0, 17: 0, 18: 0, 19: 0, 10: 0, 11: 0, 12: 0, 13: 0, 14: 0, 15: 0}, - distinct_counts=None, - lower_bounds={ - 1: b"\x01\x00\x00\x00", - 2: b"2020-04-01 00:00", - 3: b"2020-04-01 00:03", - 4: b"\x00\x00\x00\x00", - 5: b"\x01\x00\x00\x00", - 6: b"N", - 7: b"\x01\x00\x00\x00", - 8: b"\x01\x00\x00\x00", - 9: b"\x01\x00\x00\x00", - 10: b"\x00\x00\x00\x00\x00\x00\x00\x00", - 11: b"\x00\x00\x00\x00\x00\x00\x00\x00", - 13: b"\x00\x00\x00\x00\x00\x00\x00\x00", - 14: b"\x00\x00\x00\x00\x00\x00\x00\x00", - 15: b"\x00\x00\x00\x00\x00\x00\x00\x00", - 16: b"\x00\x00\x00\x00\x00\x00\x00\x00", - 17: b"\x00\x00\x00\x00\x00\x00\x00\x00", - 18: b"\x00\x00\x00\x00\x00\x00\x00\x00", - 19: b"\x00\x00\x00\x00\x00\x00\x00\x00", - }, - upper_bounds={ - 1: b"\x01\x00\x00\x00", - 2: b"2020-04-30 23:5:", - 3: b"2020-05-01 00:1:", - 4: b"\x06\x00\x00\x00", - 5: b"c\x00\x00\x00", - 6: b"Y", - 7: b"\t\x01\x00\x00", - 8: b"\t\x01\x00\x00", - 9: b"\x04\x00\x00\x00", - 10: b"\\\x8f\xc2\xf5(8\x8c@", - 11: b"\xcd\xcc\xcc\xcc\xcc,f@", - 13: b"\x00\x00\x00\x00\x00\x00\x1c@", - 14: b"\x9a\x99\x99\x99\x99\x99\xf1?", - 15: b"\x00\x00\x00\x00\x00\x00Y@", - 16: b"\x00\x00\x00\x00\x00\xb0X@", - 17: b"333333\xd3?", - 18: b"\xc3\xf5(\\\x8f:\x8c@", - 19: b"\x00\x00\x00\x00\x00\x00\x04@", - }, - key_metadata=None, - split_offsets=[4], - equality_ids=None, - sort_order_id=0, - ), - ), - ] + data_file.file_path + == "/home/iceberg/warehouse/nyc/taxis_partitioned/data/VendorID=null/00000-633-d8a4223e-dc97-45a1-86e1-adaba6e8abd7-00001.parquet" ) + assert data_file.file_format == FileFormat.PARQUET + assert repr(data_file.partition) == "Record[VendorID=1, tpep_pickup_datetime=1925]" + assert data_file.record_count == 19513 + assert data_file.file_size_in_bytes == 388872 + assert data_file.column_sizes == { + 1: 53, + 2: 98153, + 3: 98693, + 4: 53, + 5: 53, + 6: 53, + 7: 17425, + 8: 18528, + 9: 53, + 10: 44788, + 11: 35571, + 12: 53, + 13: 1243, + 14: 2355, + 15: 12750, + 16: 4029, + 17: 110, + 18: 47194, + 19: 2948, + } + assert data_file.value_counts == { + 1: 19513, + 2: 19513, + 3: 19513, + 4: 19513, + 5: 19513, + 6: 19513, + 7: 19513, + 8: 19513, + 9: 19513, + 10: 19513, + 11: 19513, + 12: 19513, + 13: 19513, + 14: 19513, + 15: 19513, + 16: 19513, + 17: 19513, + 18: 19513, + 19: 19513, + } + assert data_file.null_value_counts == { + 1: 19513, + 2: 0, + 3: 0, + 4: 19513, + 5: 19513, + 6: 19513, + 7: 0, + 8: 0, + 9: 19513, + 10: 0, + 11: 0, + 12: 19513, + 13: 0, + 14: 0, + 15: 0, + 16: 0, + 17: 0, + 18: 0, + 19: 0, + } + assert data_file.nan_value_counts == {16: 0, 17: 0, 18: 0, 19: 0, 10: 0, 11: 0, 12: 0, 13: 0, 14: 0, 15: 0} + assert data_file.lower_bounds == { + 2: b"2020-04-01 00:00", + 3: b"2020-04-01 00:12", + 7: b"\x03\x00\x00\x00", + 8: b"\x01\x00\x00\x00", + 10: b"\xf6(\\\x8f\xc2\x05S\xc0", + 11: b"\x00\x00\x00\x00\x00\x00\x00\x00", + 13: b"\x00\x00\x00\x00\x00\x00\x00\x00", + 14: b"\x00\x00\x00\x00\x00\x00\xe0\xbf", + 15: b")\\\x8f\xc2\xf5(\x08\xc0", + 16: b"\x00\x00\x00\x00\x00\x00\x00\x00", + 17: b"\x00\x00\x00\x00\x00\x00\x00\x00", + 18: b"\xf6(\\\x8f\xc2\xc5S\xc0", + 19: b"\x00\x00\x00\x00\x00\x00\x04\xc0", + } + assert data_file.upper_bounds == { + 2: b"2020-04-30 23:5:", + 3: b"2020-05-01 00:41", + 7: b"\t\x01\x00\x00", + 8: b"\t\x01\x00\x00", + 10: b"\xcd\xcc\xcc\xcc\xcc,_@", + 11: b"\x1f\x85\xebQ\\\xe2\xfe@", + 13: b"\x00\x00\x00\x00\x00\x00\x12@", + 14: b"\x00\x00\x00\x00\x00\x00\xe0?", + 15: b"q=\n\xd7\xa3\xf01@", + 16: b"\x00\x00\x00\x00\x00`B@", + 17: b"333333\xd3?", + 18: b"\x00\x00\x00\x00\x00\x18b@", + 19: b"\x00\x00\x00\x00\x00\x00\x04@", + } + assert data_file.key_metadata is None + assert data_file.split_offsets == [4] + assert data_file.equality_ids is None + assert data_file.sort_order_id == 0 def test_read_manifest_list(generated_manifest_file_file: str) -> None: input_file = PyArrowFileIO().new_input(generated_manifest_file_file) - actual = list(read_manifest_list(input_file)) - expected = [ - ManifestFile( - manifest_path=actual[0].manifest_path, - manifest_length=7989, - partition_spec_id=0, - added_snapshot_id=9182715666859759686, - added_data_files_count=3, - existing_data_files_count=0, - deleted_data_files_count=0, - partitions=[ - PartitionFieldSummary( - contains_null=True, contains_nan=False, lower_bound=b"\x01\x00\x00\x00", upper_bound=b"\x02\x00\x00\x00" - ) - ], - added_rows_count=237993, - existing_rows_counts=None, - deleted_rows_count=0, - ) - ] - assert actual == expected + manifest_list = list(read_manifest_list(input_file))[0] + + assert manifest_list.manifest_length == 7989 + assert manifest_list.partition_spec_id == 0 + assert manifest_list.added_snapshot_id == 9182715666859759686 + assert manifest_list.added_files_count == 3 + assert manifest_list.existing_files_count == 0 + assert manifest_list.deleted_files_count == 0 + + assert isinstance(manifest_list.partitions, list) + + partitions_summary = manifest_list.partitions[0] + assert isinstance(partitions_summary, PartitionFieldSummary) + assert partitions_summary.contains_null is True + assert partitions_summary.contains_nan is False + assert partitions_summary.lower_bound == b"\x01\x00\x00\x00" + assert partitions_summary.upper_bound == b"\x02\x00\x00\x00" -def test_read_manifest(generated_manifest_file_file: str, generated_manifest_entry_file: str) -> None: + assert manifest_list.added_rows_count == 237993 + assert manifest_list.existing_rows_count == 0 + assert manifest_list.deleted_rows_count == 0 + + +def test_read_manifest(generated_manifest_file_file: str) -> None: io = load_file_io({}) snapshot = Snapshot( @@ -552,269 +187,29 @@ def test_read_manifest(generated_manifest_file_file: str, generated_manifest_ent summary=Summary(Operation.APPEND), schema_id=3, ) - manifest_list = snapshot.manifests(io) + manifest_list = snapshot.manifests(io)[0] + + assert manifest_list.manifest_length == 7989 + assert manifest_list.partition_spec_id == 0 + assert manifest_list.content is None + assert manifest_list.sequence_number is None + assert manifest_list.min_sequence_number is None + assert manifest_list.added_snapshot_id == 9182715666859759686 + assert manifest_list.added_files_count == 3 + assert manifest_list.existing_files_count == 0 + assert manifest_list.deleted_files_count == 0 + assert manifest_list.added_rows_count == 237993 + assert manifest_list.existing_rows_count == 0 + assert manifest_list.deleted_rows_count == 0 + assert manifest_list.key_metadata is None + + assert isinstance(manifest_list.partitions, list) + + partition = manifest_list.partitions[0] - assert manifest_list == [ - ManifestFile( - manifest_path=generated_manifest_entry_file, - manifest_length=7989, - partition_spec_id=0, - content=ManifestContent.DATA, - sequence_number=0, - min_sequence_number=0, - added_snapshot_id=9182715666859759686, - added_data_files_count=3, - existing_data_files_count=0, - deleted_data_files_count=0, - added_rows_count=237993, - existing_rows_counts=None, - deleted_rows_count=0, - partitions=[ - PartitionFieldSummary( - contains_null=True, contains_nan=False, lower_bound=b"\x01\x00\x00\x00", upper_bound=b"\x02\x00\x00\x00" - ) - ], - key_metadata=None, - ) - ] - actual = manifest_list[0].fetch_manifest_entry(io) - expected = [ - ManifestEntry( - status=ManifestEntryStatus.ADDED, - snapshot_id=8744736658442914487, - sequence_number=None, - data_file=DataFile( - content=DataFileContent.DATA, - file_path="/home/iceberg/warehouse/nyc/taxis_partitioned/data/VendorID=null/00000-633-d8a4223e-dc97-45a1-86e1-adaba6e8abd7-00001.parquet", - file_format=FileFormat.PARQUET, - partition={"VendorID": 1, "tpep_pickup_datetime": 1925}, - record_count=19513, - file_size_in_bytes=388872, - block_size_in_bytes=67108864, - column_sizes={ - 1: 53, - 2: 98153, - 3: 98693, - 4: 53, - 5: 53, - 6: 53, - 7: 17425, - 8: 18528, - 9: 53, - 10: 44788, - 11: 35571, - 12: 53, - 13: 1243, - 14: 2355, - 15: 12750, - 16: 4029, - 17: 110, - 18: 47194, - 19: 2948, - }, - value_counts={ - 1: 19513, - 2: 19513, - 3: 19513, - 4: 19513, - 5: 19513, - 6: 19513, - 7: 19513, - 8: 19513, - 9: 19513, - 10: 19513, - 11: 19513, - 12: 19513, - 13: 19513, - 14: 19513, - 15: 19513, - 16: 19513, - 17: 19513, - 18: 19513, - 19: 19513, - }, - null_value_counts={ - 1: 19513, - 2: 0, - 3: 0, - 4: 19513, - 5: 19513, - 6: 19513, - 7: 0, - 8: 0, - 9: 19513, - 10: 0, - 11: 0, - 12: 19513, - 13: 0, - 14: 0, - 15: 0, - 16: 0, - 17: 0, - 18: 0, - 19: 0, - }, - nan_value_counts={16: 0, 17: 0, 18: 0, 19: 0, 10: 0, 11: 0, 12: 0, 13: 0, 14: 0, 15: 0}, - distinct_counts=None, - lower_bounds={ - 2: b"2020-04-01 00:00", - 3: b"2020-04-01 00:12", - 7: b"\x03\x00\x00\x00", - 8: b"\x01\x00\x00\x00", - 10: b"\xf6(\\\x8f\xc2\x05S\xc0", - 11: b"\x00\x00\x00\x00\x00\x00\x00\x00", - 13: b"\x00\x00\x00\x00\x00\x00\x00\x00", - 14: b"\x00\x00\x00\x00\x00\x00\xe0\xbf", - 15: b")\\\x8f\xc2\xf5(\x08\xc0", - 16: b"\x00\x00\x00\x00\x00\x00\x00\x00", - 17: b"\x00\x00\x00\x00\x00\x00\x00\x00", - 18: b"\xf6(\\\x8f\xc2\xc5S\xc0", - 19: b"\x00\x00\x00\x00\x00\x00\x04\xc0", - }, - upper_bounds={ - 2: b"2020-04-30 23:5:", - 3: b"2020-05-01 00:41", - 7: b"\t\x01\x00\x00", - 8: b"\t\x01\x00\x00", - 10: b"\xcd\xcc\xcc\xcc\xcc,_@", - 11: b"\x1f\x85\xebQ\\\xe2\xfe@", - 13: b"\x00\x00\x00\x00\x00\x00\x12@", - 14: b"\x00\x00\x00\x00\x00\x00\xe0?", - 15: b"q=\n\xd7\xa3\xf01@", - 16: b"\x00\x00\x00\x00\x00`B@", - 17: b"333333\xd3?", - 18: b"\x00\x00\x00\x00\x00\x18b@", - 19: b"\x00\x00\x00\x00\x00\x00\x04@", - }, - key_metadata=None, - split_offsets=[4], - equality_ids=None, - sort_order_id=0, - ), - ), - ManifestEntry( - status=ManifestEntryStatus.ADDED, - snapshot_id=8744736658442914487, - sequence_number=None, - data_file=DataFile( - content=DataFileContent.DATA, - file_path="/home/iceberg/warehouse/nyc/taxis_partitioned/data/VendorID=1/00000-633-d8a4223e-dc97-45a1-86e1-adaba6e8abd7-00002.parquet", - file_format=FileFormat.PARQUET, - partition={"VendorID": 1, "tpep_pickup_datetime": 1925}, - record_count=95050, - file_size_in_bytes=1265950, - block_size_in_bytes=67108864, - column_sizes={ - 1: 318, - 2: 329806, - 3: 331632, - 4: 15343, - 5: 2351, - 6: 3389, - 7: 71269, - 8: 76429, - 9: 16383, - 10: 86992, - 11: 89608, - 12: 265, - 13: 19377, - 14: 1692, - 15: 76162, - 16: 4354, - 17: 759, - 18: 120650, - 19: 11804, - }, - value_counts={ - 1: 95050, - 2: 95050, - 3: 95050, - 4: 95050, - 5: 95050, - 6: 95050, - 7: 95050, - 8: 95050, - 9: 95050, - 10: 95050, - 11: 95050, - 12: 95050, - 13: 95050, - 14: 95050, - 15: 95050, - 16: 95050, - 17: 95050, - 18: 95050, - 19: 95050, - }, - null_value_counts={ - 1: 0, - 2: 0, - 3: 0, - 4: 0, - 5: 0, - 6: 0, - 7: 0, - 8: 0, - 9: 0, - 10: 0, - 11: 0, - 12: 95050, - 13: 0, - 14: 0, - 15: 0, - 16: 0, - 17: 0, - 18: 0, - 19: 0, - }, - nan_value_counts={16: 0, 17: 0, 18: 0, 19: 0, 10: 0, 11: 0, 12: 0, 13: 0, 14: 0, 15: 0}, - distinct_counts=None, - lower_bounds={ - 1: b"\x01\x00\x00\x00", - 2: b"2020-04-01 00:00", - 3: b"2020-04-01 00:03", - 4: b"\x00\x00\x00\x00", - 5: b"\x01\x00\x00\x00", - 6: b"N", - 7: b"\x01\x00\x00\x00", - 8: b"\x01\x00\x00\x00", - 9: b"\x01\x00\x00\x00", - 10: b"\x00\x00\x00\x00\x00\x00\x00\x00", - 11: b"\x00\x00\x00\x00\x00\x00\x00\x00", - 13: b"\x00\x00\x00\x00\x00\x00\x00\x00", - 14: b"\x00\x00\x00\x00\x00\x00\x00\x00", - 15: b"\x00\x00\x00\x00\x00\x00\x00\x00", - 16: b"\x00\x00\x00\x00\x00\x00\x00\x00", - 17: b"\x00\x00\x00\x00\x00\x00\x00\x00", - 18: b"\x00\x00\x00\x00\x00\x00\x00\x00", - 19: b"\x00\x00\x00\x00\x00\x00\x00\x00", - }, - upper_bounds={ - 1: b"\x01\x00\x00\x00", - 2: b"2020-04-30 23:5:", - 3: b"2020-05-01 00:1:", - 4: b"\x06\x00\x00\x00", - 5: b"c\x00\x00\x00", - 6: b"Y", - 7: b"\t\x01\x00\x00", - 8: b"\t\x01\x00\x00", - 9: b"\x04\x00\x00\x00", - 10: b"\\\x8f\xc2\xf5(8\x8c@", - 11: b"\xcd\xcc\xcc\xcc\xcc,f@", - 13: b"\x00\x00\x00\x00\x00\x00\x1c@", - 14: b"\x9a\x99\x99\x99\x99\x99\xf1?", - 15: b"\x00\x00\x00\x00\x00\x00Y@", - 16: b"\x00\x00\x00\x00\x00\xb0X@", - 17: b"333333\xd3?", - 18: b"\xc3\xf5(\\\x8f:\x8c@", - 19: b"\x00\x00\x00\x00\x00\x00\x04@", - }, - key_metadata=None, - split_offsets=[4], - equality_ids=None, - sort_order_id=0, - ), - ), - ] + assert isinstance(partition, PartitionFieldSummary) - assert actual == expected + assert partition.contains_null is True + assert partition.contains_nan is False + assert partition.lower_bound == b"\x01\x00\x00\x00" + assert partition.upper_bound == b"\x02\x00\x00\x00" From 121aa3a7a36cc0dc4e098cbaab49aae0dd841bf4 Mon Sep 17 00:00:00 2001 From: Fokko Driesprong Date: Sat, 14 Jan 2023 21:08:22 +0100 Subject: [PATCH 337/642] Python: Raise exception on deletes (#6574) --- pyiceberg/table/__init__.py | 21 +++++++++++++++++++-- tests/table/test_init.py | 24 +++++++++++++++++++++++- 2 files changed, 42 insertions(+), 3 deletions(-) diff --git a/pyiceberg/table/__init__.py b/pyiceberg/table/__init__.py index c4c23100c9..910ea7f0e0 100644 --- a/pyiceberg/table/__init__.py +++ b/pyiceberg/table/__init__.py @@ -43,7 +43,12 @@ from pyiceberg.expressions.visitors import inclusive_projection from pyiceberg.io import FileIO from pyiceberg.io.pyarrow import project_table -from pyiceberg.manifest import DataFile, ManifestFile, files +from pyiceberg.manifest import ( + DataFile, + ManifestContent, + ManifestFile, + files, +) from pyiceberg.partitioning import PartitionSpec from pyiceberg.schema import Schema from pyiceberg.table.metadata import TableMetadata @@ -257,6 +262,16 @@ def __init__(self, data_file: DataFile, start: Optional[int] = None, length: Opt self.length = length or data_file.file_size_in_bytes +def _check_content(file: DataFile) -> DataFile: + try: + if file.content == ManifestContent.DELETES: + raise ValueError("PyIceberg does not support deletes: https://github.com/apache/iceberg/issues/6568") + return file + except AttributeError: + # If the attribute is not there, it is a V1 record + return file + + class DataScan(TableScan["DataScan"]): def __init__( self, @@ -318,7 +333,9 @@ def plan_files(self) -> Iterator[FileScanTask]: all_files = files(io.new_input(manifest.manifest_path)) matching_partition_files = filter(partition_filter, all_files) - yield from (FileScanTask(file) for file in matching_partition_files) + matching_partition_data_files = map(_check_content, matching_partition_files) + + yield from (FileScanTask(file) for file in matching_partition_data_files) def to_arrow(self) -> pa.Table: return project_table( diff --git a/tests/table/test_init.py b/tests/table/test_init.py index 3c663ceeb4..506c6609f7 100644 --- a/tests/table/test_init.py +++ b/tests/table/test_init.py @@ -26,9 +26,10 @@ In, ) from pyiceberg.io import load_file_io +from pyiceberg.manifest import DataFile, ManifestContent from pyiceberg.partitioning import PartitionField, PartitionSpec from pyiceberg.schema import Schema -from pyiceberg.table import Table +from pyiceberg.table import Table, _check_content from pyiceberg.table.metadata import TableMetadataV2 from pyiceberg.table.snapshots import ( Operation, @@ -43,6 +44,7 @@ SortOrder, ) from pyiceberg.transforms import BucketTransform, IdentityTransform +from pyiceberg.typedef import Record from pyiceberg.types import LongType, NestedField @@ -243,3 +245,23 @@ def test_table_scan_projection_unknown_column(table: Table) -> None: _ = scan.select("a").projection() assert "Could not find column: 'a'" in str(exc_info.value) + + +def test_check_content_deletes() -> None: + with pytest.raises(ValueError) as exc_info: + _check_content( + DataFile( + content=ManifestContent.DELETES, + ) + ) + assert "PyIceberg does not support deletes: https://github.com/apache/iceberg/issues/6568" in str(exc_info.value) + + +def test_check_content_data() -> None: + manifest_file = DataFile(content=ManifestContent.DATA) + assert _check_content(manifest_file) == manifest_file + + +def test_check_content_missing_attr() -> None: + r = Record(*([None] * 15)) + assert _check_content(r) == r # type: ignore From 1a286b8297f0e4efda0018e3e2b7c07fc34b0d04 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 16 Jan 2023 08:38:02 +0100 Subject: [PATCH 338/642] Build: Bump requests from 2.28.1 to 2.28.2 in /python (#6597) --- poetry.lock | 2406 ++++++++++++++++++++++++------------------------ pyproject.toml | 2 +- 2 files changed, 1204 insertions(+), 1204 deletions(-) diff --git a/poetry.lock b/poetry.lock index 7d215f79fd..0954a7a1b7 100644 --- a/poetry.lock +++ b/poetry.lock @@ -1,3 +1,5 @@ +# This file is automatically @generated by Poetry and should not be changed by hand. + [[package]] name = "adal" version = "1.2.7" @@ -5,6 +7,10 @@ description = "Note: This library is already replaced by MSAL Python, available category = "main" optional = true python-versions = "*" +files = [ + {file = "adal-1.2.7-py2.py3-none-any.whl", hash = "sha256:2a7451ed7441ddbc57703042204a3e30ef747478eea022c70f789fc7f084bc3d"}, + {file = "adal-1.2.7.tar.gz", hash = "sha256:d74f45b81317454d96e982fd1c50e6fb5c99ac2223728aea8764433a39f566f1"}, +] [package.dependencies] cryptography = ">=1.1.0" @@ -19,6 +25,10 @@ description = "Access Azure Datalake Gen1 with fsspec and dask" category = "main" optional = true python-versions = ">=3.8" +files = [ + {file = "adlfs-2022.11.2-py3-none-any.whl", hash = "sha256:60da4a17260754ee6d8da43e3d756226ade4fd19ada8411e858f5acecd0833e2"}, + {file = "adlfs-2022.11.2.tar.gz", hash = "sha256:920dba10468f186037ca394dcabcba113532d80f52b315211c8e771be40475ea"}, +] [package.dependencies] aiohttp = ">=3.7.0" @@ -38,6 +48,10 @@ description = "Async client for aws services using botocore and aiohttp" category = "main" optional = true python-versions = ">=3.7" +files = [ + {file = "aiobotocore-2.4.1-py3-none-any.whl", hash = "sha256:26e0d55d5901f487c3467616c028abb85036ca33a0f88e279770adae6b865c69"}, + {file = "aiobotocore-2.4.1.tar.gz", hash = "sha256:5c8ee79d1f14273e3f8f3af5c893db889ef163227b178ecacb5aea4547d3b9a7"}, +] [package.dependencies] aiohttp = ">=3.3.1" @@ -56,6 +70,95 @@ description = "Async http client/server framework (asyncio)" category = "main" optional = true python-versions = ">=3.6" +files = [ + {file = "aiohttp-3.8.3-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:ba71c9b4dcbb16212f334126cc3d8beb6af377f6703d9dc2d9fb3874fd667ee9"}, + {file = "aiohttp-3.8.3-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:d24b8bb40d5c61ef2d9b6a8f4528c2f17f1c5d2d31fed62ec860f6006142e83e"}, + {file = "aiohttp-3.8.3-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:f88df3a83cf9df566f171adba39d5bd52814ac0b94778d2448652fc77f9eb491"}, + {file = "aiohttp-3.8.3-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:b97decbb3372d4b69e4d4c8117f44632551c692bb1361b356a02b97b69e18a62"}, + {file = "aiohttp-3.8.3-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:309aa21c1d54b8ef0723181d430347d7452daaff93e8e2363db8e75c72c2fb2d"}, + {file = "aiohttp-3.8.3-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:ad5383a67514e8e76906a06741febd9126fc7c7ff0f599d6fcce3e82b80d026f"}, + {file = "aiohttp-3.8.3-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:20acae4f268317bb975671e375493dbdbc67cddb5f6c71eebdb85b34444ac46b"}, + {file = "aiohttp-3.8.3-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:05a3c31c6d7cd08c149e50dc7aa2568317f5844acd745621983380597f027a18"}, + {file = "aiohttp-3.8.3-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:d6f76310355e9fae637c3162936e9504b4767d5c52ca268331e2756e54fd4ca5"}, + {file = "aiohttp-3.8.3-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:256deb4b29fe5e47893fa32e1de2d73c3afe7407738bd3c63829874661d4822d"}, + {file = "aiohttp-3.8.3-cp310-cp310-musllinux_1_1_ppc64le.whl", hash = "sha256:5c59fcd80b9049b49acd29bd3598cada4afc8d8d69bd4160cd613246912535d7"}, + {file = "aiohttp-3.8.3-cp310-cp310-musllinux_1_1_s390x.whl", hash = "sha256:059a91e88f2c00fe40aed9031b3606c3f311414f86a90d696dd982e7aec48142"}, + {file = "aiohttp-3.8.3-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:2feebbb6074cdbd1ac276dbd737b40e890a1361b3cc30b74ac2f5e24aab41f7b"}, + {file = "aiohttp-3.8.3-cp310-cp310-win32.whl", hash = "sha256:5bf651afd22d5f0c4be16cf39d0482ea494f5c88f03e75e5fef3a85177fecdeb"}, + {file = "aiohttp-3.8.3-cp310-cp310-win_amd64.whl", hash = "sha256:653acc3880459f82a65e27bd6526e47ddf19e643457d36a2250b85b41a564715"}, + {file = "aiohttp-3.8.3-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:86fc24e58ecb32aee09f864cb11bb91bc4c1086615001647dbfc4dc8c32f4008"}, + {file = "aiohttp-3.8.3-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:75e14eac916f024305db517e00a9252714fce0abcb10ad327fb6dcdc0d060f1d"}, + {file = "aiohttp-3.8.3-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:d1fde0f44029e02d02d3993ad55ce93ead9bb9b15c6b7ccd580f90bd7e3de476"}, + {file = "aiohttp-3.8.3-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:4ab94426ddb1ecc6a0b601d832d5d9d421820989b8caa929114811369673235c"}, + {file = "aiohttp-3.8.3-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:89d2e02167fa95172c017732ed7725bc8523c598757f08d13c5acca308e1a061"}, + {file = "aiohttp-3.8.3-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:02f9a2c72fc95d59b881cf38a4b2be9381b9527f9d328771e90f72ac76f31ad8"}, + {file = "aiohttp-3.8.3-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:9c7149272fb5834fc186328e2c1fa01dda3e1fa940ce18fded6d412e8f2cf76d"}, + {file = "aiohttp-3.8.3-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:512bd5ab136b8dc0ffe3fdf2dfb0c4b4f49c8577f6cae55dca862cd37a4564e2"}, + {file = "aiohttp-3.8.3-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:7018ecc5fe97027214556afbc7c502fbd718d0740e87eb1217b17efd05b3d276"}, + {file = "aiohttp-3.8.3-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:88c70ed9da9963d5496d38320160e8eb7e5f1886f9290475a881db12f351ab5d"}, + {file = "aiohttp-3.8.3-cp311-cp311-musllinux_1_1_ppc64le.whl", hash = "sha256:da22885266bbfb3f78218dc40205fed2671909fbd0720aedba39b4515c038091"}, + {file = "aiohttp-3.8.3-cp311-cp311-musllinux_1_1_s390x.whl", hash = "sha256:e65bc19919c910127c06759a63747ebe14f386cda573d95bcc62b427ca1afc73"}, + {file = "aiohttp-3.8.3-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:08c78317e950e0762c2983f4dd58dc5e6c9ff75c8a0efeae299d363d439c8e34"}, + {file = "aiohttp-3.8.3-cp311-cp311-win32.whl", hash = "sha256:45d88b016c849d74ebc6f2b6e8bc17cabf26e7e40c0661ddd8fae4c00f015697"}, + {file = "aiohttp-3.8.3-cp311-cp311-win_amd64.whl", hash = "sha256:96372fc29471646b9b106ee918c8eeb4cca423fcbf9a34daa1b93767a88a2290"}, + {file = "aiohttp-3.8.3-cp36-cp36m-macosx_10_9_x86_64.whl", hash = "sha256:c971bf3786b5fad82ce5ad570dc6ee420f5b12527157929e830f51c55dc8af77"}, + {file = "aiohttp-3.8.3-cp36-cp36m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:ff25f48fc8e623d95eca0670b8cc1469a83783c924a602e0fbd47363bb54aaca"}, + {file = "aiohttp-3.8.3-cp36-cp36m-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:e381581b37db1db7597b62a2e6b8b57c3deec95d93b6d6407c5b61ddc98aca6d"}, + {file = "aiohttp-3.8.3-cp36-cp36m-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:db19d60d846283ee275d0416e2a23493f4e6b6028825b51290ac05afc87a6f97"}, + {file = "aiohttp-3.8.3-cp36-cp36m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:25892c92bee6d9449ffac82c2fe257f3a6f297792cdb18ad784737d61e7a9a85"}, + {file = "aiohttp-3.8.3-cp36-cp36m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:398701865e7a9565d49189f6c90868efaca21be65c725fc87fc305906be915da"}, + {file = "aiohttp-3.8.3-cp36-cp36m-musllinux_1_1_aarch64.whl", hash = "sha256:4a4fbc769ea9b6bd97f4ad0b430a6807f92f0e5eb020f1e42ece59f3ecfc4585"}, + {file = "aiohttp-3.8.3-cp36-cp36m-musllinux_1_1_i686.whl", hash = "sha256:b29bfd650ed8e148f9c515474a6ef0ba1090b7a8faeee26b74a8ff3b33617502"}, + {file = "aiohttp-3.8.3-cp36-cp36m-musllinux_1_1_ppc64le.whl", hash = "sha256:1e56b9cafcd6531bab5d9b2e890bb4937f4165109fe98e2b98ef0dcfcb06ee9d"}, + {file = "aiohttp-3.8.3-cp36-cp36m-musllinux_1_1_s390x.whl", hash = "sha256:ec40170327d4a404b0d91855d41bfe1fe4b699222b2b93e3d833a27330a87a6d"}, + {file = "aiohttp-3.8.3-cp36-cp36m-musllinux_1_1_x86_64.whl", hash = "sha256:2df5f139233060578d8c2c975128fb231a89ca0a462b35d4b5fcf7c501ebdbe1"}, + {file = "aiohttp-3.8.3-cp36-cp36m-win32.whl", hash = "sha256:f973157ffeab5459eefe7b97a804987876dd0a55570b8fa56b4e1954bf11329b"}, + {file = "aiohttp-3.8.3-cp36-cp36m-win_amd64.whl", hash = "sha256:437399385f2abcd634865705bdc180c8314124b98299d54fe1d4c8990f2f9494"}, + {file = "aiohttp-3.8.3-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:09e28f572b21642128ef31f4e8372adb6888846f32fecb288c8b0457597ba61a"}, + {file = "aiohttp-3.8.3-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:6f3553510abdbec67c043ca85727396ceed1272eef029b050677046d3387be8d"}, + {file = "aiohttp-3.8.3-cp37-cp37m-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:e168a7560b7c61342ae0412997b069753f27ac4862ec7867eff74f0fe4ea2ad9"}, + {file = "aiohttp-3.8.3-cp37-cp37m-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:db4c979b0b3e0fa7e9e69ecd11b2b3174c6963cebadeecfb7ad24532ffcdd11a"}, + {file = "aiohttp-3.8.3-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:e164e0a98e92d06da343d17d4e9c4da4654f4a4588a20d6c73548a29f176abe2"}, + {file = "aiohttp-3.8.3-cp37-cp37m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:e8a78079d9a39ca9ca99a8b0ac2fdc0c4d25fc80c8a8a82e5c8211509c523363"}, + {file = "aiohttp-3.8.3-cp37-cp37m-musllinux_1_1_aarch64.whl", hash = "sha256:21b30885a63c3f4ff5b77a5d6caf008b037cb521a5f33eab445dc566f6d092cc"}, + {file = "aiohttp-3.8.3-cp37-cp37m-musllinux_1_1_i686.whl", hash = "sha256:4b0f30372cef3fdc262f33d06e7b411cd59058ce9174ef159ad938c4a34a89da"}, + {file = "aiohttp-3.8.3-cp37-cp37m-musllinux_1_1_ppc64le.whl", hash = "sha256:8135fa153a20d82ffb64f70a1b5c2738684afa197839b34cc3e3c72fa88d302c"}, + {file = "aiohttp-3.8.3-cp37-cp37m-musllinux_1_1_s390x.whl", hash = "sha256:ad61a9639792fd790523ba072c0555cd6be5a0baf03a49a5dd8cfcf20d56df48"}, + {file = "aiohttp-3.8.3-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:978b046ca728073070e9abc074b6299ebf3501e8dee5e26efacb13cec2b2dea0"}, + {file = "aiohttp-3.8.3-cp37-cp37m-win32.whl", hash = "sha256:0d2c6d8c6872df4a6ec37d2ede71eff62395b9e337b4e18efd2177de883a5033"}, + {file = "aiohttp-3.8.3-cp37-cp37m-win_amd64.whl", hash = "sha256:21d69797eb951f155026651f7e9362877334508d39c2fc37bd04ff55b2007091"}, + {file = "aiohttp-3.8.3-cp38-cp38-macosx_10_9_universal2.whl", hash = "sha256:2ca9af5f8f5812d475c5259393f52d712f6d5f0d7fdad9acdb1107dd9e3cb7eb"}, + {file = "aiohttp-3.8.3-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:1d90043c1882067f1bd26196d5d2db9aa6d268def3293ed5fb317e13c9413ea4"}, + {file = "aiohttp-3.8.3-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:d737fc67b9a970f3234754974531dc9afeea11c70791dcb7db53b0cf81b79784"}, + {file = "aiohttp-3.8.3-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:ebf909ea0a3fc9596e40d55d8000702a85e27fd578ff41a5500f68f20fd32e6c"}, + {file = "aiohttp-3.8.3-cp38-cp38-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:5835f258ca9f7c455493a57ee707b76d2d9634d84d5d7f62e77be984ea80b849"}, + {file = "aiohttp-3.8.3-cp38-cp38-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:da37dcfbf4b7f45d80ee386a5f81122501ec75672f475da34784196690762f4b"}, + {file = "aiohttp-3.8.3-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:87f44875f2804bc0511a69ce44a9595d5944837a62caecc8490bbdb0e18b1342"}, + {file = "aiohttp-3.8.3-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:527b3b87b24844ea7865284aabfab08eb0faf599b385b03c2aa91fc6edd6e4b6"}, + {file = "aiohttp-3.8.3-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:d5ba88df9aa5e2f806650fcbeedbe4f6e8736e92fc0e73b0400538fd25a4dd96"}, + {file = "aiohttp-3.8.3-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:e7b8813be97cab8cb52b1375f41f8e6804f6507fe4660152e8ca5c48f0436017"}, + {file = "aiohttp-3.8.3-cp38-cp38-musllinux_1_1_ppc64le.whl", hash = "sha256:2dea10edfa1a54098703cb7acaa665c07b4e7568472a47f4e64e6319d3821ccf"}, + {file = "aiohttp-3.8.3-cp38-cp38-musllinux_1_1_s390x.whl", hash = "sha256:713d22cd9643ba9025d33c4af43943c7a1eb8547729228de18d3e02e278472b6"}, + {file = "aiohttp-3.8.3-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:2d252771fc85e0cf8da0b823157962d70639e63cb9b578b1dec9868dd1f4f937"}, + {file = "aiohttp-3.8.3-cp38-cp38-win32.whl", hash = "sha256:66bd5f950344fb2b3dbdd421aaa4e84f4411a1a13fca3aeb2bcbe667f80c9f76"}, + {file = "aiohttp-3.8.3-cp38-cp38-win_amd64.whl", hash = "sha256:84b14f36e85295fe69c6b9789b51a0903b774046d5f7df538176516c3e422446"}, + {file = "aiohttp-3.8.3-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:16c121ba0b1ec2b44b73e3a8a171c4f999b33929cd2397124a8c7fcfc8cd9e06"}, + {file = "aiohttp-3.8.3-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:8d6aaa4e7155afaf994d7924eb290abbe81a6905b303d8cb61310a2aba1c68ba"}, + {file = "aiohttp-3.8.3-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:43046a319664a04b146f81b40e1545d4c8ac7b7dd04c47e40bf09f65f2437346"}, + {file = "aiohttp-3.8.3-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:599418aaaf88a6d02a8c515e656f6faf3d10618d3dd95866eb4436520096c84b"}, + {file = "aiohttp-3.8.3-cp39-cp39-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:92a2964319d359f494f16011e23434f6f8ef0434acd3cf154a6b7bec511e2fb7"}, + {file = "aiohttp-3.8.3-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:73a4131962e6d91109bca6536416aa067cf6c4efb871975df734f8d2fd821b37"}, + {file = "aiohttp-3.8.3-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:598adde339d2cf7d67beaccda3f2ce7c57b3b412702f29c946708f69cf8222aa"}, + {file = "aiohttp-3.8.3-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:75880ed07be39beff1881d81e4a907cafb802f306efd6d2d15f2b3c69935f6fb"}, + {file = "aiohttp-3.8.3-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:a0239da9fbafd9ff82fd67c16704a7d1bccf0d107a300e790587ad05547681c8"}, + {file = "aiohttp-3.8.3-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:4e3a23ec214e95c9fe85a58470b660efe6534b83e6cbe38b3ed52b053d7cb6ad"}, + {file = "aiohttp-3.8.3-cp39-cp39-musllinux_1_1_ppc64le.whl", hash = "sha256:47841407cc89a4b80b0c52276f3cc8138bbbfba4b179ee3acbd7d77ae33f7ac4"}, + {file = "aiohttp-3.8.3-cp39-cp39-musllinux_1_1_s390x.whl", hash = "sha256:54d107c89a3ebcd13228278d68f1436d3f33f2dd2af5415e3feaeb1156e1a62c"}, + {file = "aiohttp-3.8.3-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:c37c5cce780349d4d51739ae682dec63573847a2a8dcb44381b174c3d9c8d403"}, + {file = "aiohttp-3.8.3-cp39-cp39-win32.whl", hash = "sha256:f178d2aadf0166be4df834c4953da2d7eef24719e8aec9a65289483eeea9d618"}, + {file = "aiohttp-3.8.3-cp39-cp39-win_amd64.whl", hash = "sha256:88e5be56c231981428f4f506c68b6a46fa25c4123a2e86d156c58a8369d31ab7"}, + {file = "aiohttp-3.8.3.tar.gz", hash = "sha256:3828fb41b7203176b82fe5d699e0d845435f2374750a44b480ea6b930f6be269"}, +] [package.dependencies] aiosignal = ">=1.1.2" @@ -76,6 +179,10 @@ description = "itertools and builtins for AsyncIO and mixed iterables" category = "main" optional = true python-versions = ">=3.6" +files = [ + {file = "aioitertools-0.11.0-py3-none-any.whl", hash = "sha256:04b95e3dab25b449def24d7df809411c10e62aab0cbe31a50ca4e68748c43394"}, + {file = "aioitertools-0.11.0.tar.gz", hash = "sha256:42c68b8dd3a69c2bf7f2233bf7df4bb58b557bca5252ac02ed5187bbc67d6831"}, +] [package.dependencies] typing_extensions = {version = ">=4.0", markers = "python_version < \"3.10\""} @@ -87,6 +194,10 @@ description = "aiosignal: a list of registered asynchronous callbacks" category = "main" optional = true python-versions = ">=3.7" +files = [ + {file = "aiosignal-1.3.1-py3-none-any.whl", hash = "sha256:f8376fb07dd1e86a584e4fcdec80b36b7f81aac666ebc724e2c090300dd83b17"}, + {file = "aiosignal-1.3.1.tar.gz", hash = "sha256:54cd96e15e1649b75d6c87526a6ff0b6c1b0dd3459f43d9ca11d48c339b68cfc"}, +] [package.dependencies] frozenlist = ">=1.1.0" @@ -98,6 +209,10 @@ description = "Timeout context manager for asyncio programs" category = "main" optional = true python-versions = ">=3.6" +files = [ + {file = "async-timeout-4.0.2.tar.gz", hash = "sha256:2163e1640ddb52b7a8c80d0a67a08587e5d245cc9c553a74a847056bc2976b15"}, + {file = "async_timeout-4.0.2-py3-none-any.whl", hash = "sha256:8ca1e4fcf50d07413d66d1a5e416e42cfdf5851c981d679a09851a6853383b3c"}, +] [[package]] name = "attrs" @@ -106,6 +221,10 @@ description = "Classes Without Boilerplate" category = "main" optional = false python-versions = ">=3.5" +files = [ + {file = "attrs-22.1.0-py2.py3-none-any.whl", hash = "sha256:86efa402f67bf2df34f51a335487cf46b1ec130d02b8d39fd248abfd30da551c"}, + {file = "attrs-22.1.0.tar.gz", hash = "sha256:29adc2665447e5191d0e7c568fde78b21f9672d344281d0c6e1ab085429b22b6"}, +] [package.extras] dev = ["cloudpickle", "coverage[toml] (>=5.0.2)", "furo", "hypothesis", "mypy (>=0.900,!=0.940)", "pre-commit", "pympler", "pytest (>=4.3.0)", "pytest-mypy-plugins", "sphinx", "sphinx-notfound-page", "zope.interface"] @@ -120,6 +239,10 @@ description = "Microsoft Azure Core Library for Python" category = "main" optional = true python-versions = ">=3.7" +files = [ + {file = "azure-core-1.26.1.zip", hash = "sha256:223b0e90cbdd1f03c41b195b03239899843f20d00964dbb85e64386873414a2d"}, + {file = "azure_core-1.26.1-py3-none-any.whl", hash = "sha256:726ffd1ded04a2c1cb53f9d9155cbb05ac5c1c2a29af4ef622e93e1c0a8bc92b"}, +] [package.dependencies] requests = ">=2.18.4" @@ -136,6 +259,10 @@ description = "Azure Data Lake Store Filesystem Client Library for Python" category = "main" optional = true python-versions = "*" +files = [ + {file = "azure-datalake-store-0.0.52.tar.gz", hash = "sha256:4198ddb32614d16d4502b43d5c9739f81432b7e0e4d75d30e05149fe6007fea2"}, + {file = "azure_datalake_store-0.0.52-py2.py3-none-any.whl", hash = "sha256:aaed72b9c856824aeab554f4dbe0ef2c6d0ff36700bdd8b93d8298793117c48e"}, +] [package.dependencies] adal = ">=0.4.2" @@ -149,6 +276,10 @@ description = "Microsoft Azure Identity Library for Python" category = "main" optional = true python-versions = ">=3.7" +files = [ + {file = "azure-identity-1.12.0.zip", hash = "sha256:7f9b1ae7d97ea7af3f38dd09305e19ab81a1e16ab66ea186b6579d85c1ca2347"}, + {file = "azure_identity-1.12.0-py3-none-any.whl", hash = "sha256:2a58ce4a209a013e37eaccfd5937570ab99e9118b3e1acf875eed3a85d541b92"}, +] [package.dependencies] azure-core = ">=1.11.0,<2.0.0" @@ -164,6 +295,10 @@ description = "Microsoft Azure Blob Storage Client Library for Python" category = "main" optional = true python-versions = ">=3.7" +files = [ + {file = "azure-storage-blob-12.14.1.zip", hash = "sha256:860d4d82985a4bfc7d3271e71275af330f54f330a754355435a7ba749ccde997"}, + {file = "azure_storage_blob-12.14.1-py3-none-any.whl", hash = "sha256:52b84658e8df7853a3cf1c563814655b5028b979b2a87905b92aa6bb30be240e"}, +] [package.dependencies] azure-core = ">=1.24.2,<2.0.0" @@ -177,6 +312,10 @@ description = "The AWS SDK for Python" category = "main" optional = false python-versions = ">= 3.7" +files = [ + {file = "boto3-1.24.59-py3-none-any.whl", hash = "sha256:34ab44146a2c4e7f4e72737f4b27e6eb5e0a7855c2f4599e3d9199b6a0a2d575"}, + {file = "boto3-1.24.59.tar.gz", hash = "sha256:a50b4323f9579cfe22fcf5531fbd40b567d4d74c1adce06aeb5c95fce2a6fb40"}, +] [package.dependencies] botocore = ">=1.27.59,<1.28.0" @@ -193,6 +332,10 @@ description = "Low-level, data-driven core of boto 3." category = "main" optional = false python-versions = ">= 3.7" +files = [ + {file = "botocore-1.27.59-py3-none-any.whl", hash = "sha256:69d756791fc024bda54f6c53f71ae34e695ee41bbbc1743d9179c4837a4929da"}, + {file = "botocore-1.27.59.tar.gz", hash = "sha256:eda4aed6ee719a745d1288eaf1beb12f6f6448ad1fa12f159405db14ba9c92cf"}, +] [package.dependencies] jmespath = ">=0.7.1,<2.0.0" @@ -209,6 +352,10 @@ description = "A simple, correct PEP 517 build frontend" category = "dev" optional = false python-versions = ">=3.6" +files = [ + {file = "build-0.9.0-py3-none-any.whl", hash = "sha256:38a7a2b7a0bdc61a42a0a67509d88c71ecfc37b393baba770fae34e20929ff69"}, + {file = "build-0.9.0.tar.gz", hash = "sha256:1a07724e891cbd898923145eb7752ee7653674c511378eb9c7691aab1612bc3c"}, +] [package.dependencies] colorama = {version = "*", markers = "os_name == \"nt\""} @@ -229,6 +376,10 @@ description = "Python package for providing Mozilla's CA Bundle." category = "main" optional = false python-versions = ">=3.6" +files = [ + {file = "certifi-2022.12.7-py3-none-any.whl", hash = "sha256:4ad3232f5e926d6718ec31cfc1fcadfde020920e278684144551c91769c7bc18"}, + {file = "certifi-2022.12.7.tar.gz", hash = "sha256:35824b4c3a97115964b408844d64aa14db1cc518f6562e8d7261699d1350a9e3"}, +] [[package]] name = "cffi" @@ -237,6 +388,72 @@ description = "Foreign Function Interface for Python calling C code." category = "main" optional = false python-versions = "*" +files = [ + {file = "cffi-1.15.1-cp27-cp27m-macosx_10_9_x86_64.whl", hash = "sha256:a66d3508133af6e8548451b25058d5812812ec3798c886bf38ed24a98216fab2"}, + {file = "cffi-1.15.1-cp27-cp27m-manylinux1_i686.whl", hash = "sha256:470c103ae716238bbe698d67ad020e1db9d9dba34fa5a899b5e21577e6d52ed2"}, + {file = "cffi-1.15.1-cp27-cp27m-manylinux1_x86_64.whl", hash = "sha256:9ad5db27f9cabae298d151c85cf2bad1d359a1b9c686a275df03385758e2f914"}, + {file = "cffi-1.15.1-cp27-cp27m-win32.whl", hash = "sha256:b3bbeb01c2b273cca1e1e0c5df57f12dce9a4dd331b4fa1635b8bec26350bde3"}, + {file = "cffi-1.15.1-cp27-cp27m-win_amd64.whl", hash = "sha256:e00b098126fd45523dd056d2efba6c5a63b71ffe9f2bbe1a4fe1716e1d0c331e"}, + {file = "cffi-1.15.1-cp27-cp27mu-manylinux1_i686.whl", hash = "sha256:d61f4695e6c866a23a21acab0509af1cdfd2c013cf256bbf5b6b5e2695827162"}, + {file = "cffi-1.15.1-cp27-cp27mu-manylinux1_x86_64.whl", hash = "sha256:ed9cb427ba5504c1dc15ede7d516b84757c3e3d7868ccc85121d9310d27eed0b"}, + {file = "cffi-1.15.1-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:39d39875251ca8f612b6f33e6b1195af86d1b3e60086068be9cc053aa4376e21"}, + {file = "cffi-1.15.1-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:285d29981935eb726a4399badae8f0ffdff4f5050eaa6d0cfc3f64b857b77185"}, + {file = "cffi-1.15.1-cp310-cp310-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:3eb6971dcff08619f8d91607cfc726518b6fa2a9eba42856be181c6d0d9515fd"}, + {file = "cffi-1.15.1-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:21157295583fe8943475029ed5abdcf71eb3911894724e360acff1d61c1d54bc"}, + {file = "cffi-1.15.1-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:5635bd9cb9731e6d4a1132a498dd34f764034a8ce60cef4f5319c0541159392f"}, + {file = "cffi-1.15.1-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:2012c72d854c2d03e45d06ae57f40d78e5770d252f195b93f581acf3ba44496e"}, + {file = "cffi-1.15.1-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:dd86c085fae2efd48ac91dd7ccffcfc0571387fe1193d33b6394db7ef31fe2a4"}, + {file = "cffi-1.15.1-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:fa6693661a4c91757f4412306191b6dc88c1703f780c8234035eac011922bc01"}, + {file = "cffi-1.15.1-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:59c0b02d0a6c384d453fece7566d1c7e6b7bae4fc5874ef2ef46d56776d61c9e"}, + {file = "cffi-1.15.1-cp310-cp310-win32.whl", hash = "sha256:cba9d6b9a7d64d4bd46167096fc9d2f835e25d7e4c121fb2ddfc6528fb0413b2"}, + {file = "cffi-1.15.1-cp310-cp310-win_amd64.whl", hash = "sha256:ce4bcc037df4fc5e3d184794f27bdaab018943698f4ca31630bc7f84a7b69c6d"}, + {file = "cffi-1.15.1-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:3d08afd128ddaa624a48cf2b859afef385b720bb4b43df214f85616922e6a5ac"}, + {file = "cffi-1.15.1-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:3799aecf2e17cf585d977b780ce79ff0dc9b78d799fc694221ce814c2c19db83"}, + {file = "cffi-1.15.1-cp311-cp311-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:a591fe9e525846e4d154205572a029f653ada1a78b93697f3b5a8f1f2bc055b9"}, + {file = "cffi-1.15.1-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:3548db281cd7d2561c9ad9984681c95f7b0e38881201e157833a2342c30d5e8c"}, + {file = "cffi-1.15.1-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:91fc98adde3d7881af9b59ed0294046f3806221863722ba7d8d120c575314325"}, + {file = "cffi-1.15.1-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:94411f22c3985acaec6f83c6df553f2dbe17b698cc7f8ae751ff2237d96b9e3c"}, + {file = "cffi-1.15.1-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:03425bdae262c76aad70202debd780501fabeaca237cdfddc008987c0e0f59ef"}, + {file = "cffi-1.15.1-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:cc4d65aeeaa04136a12677d3dd0b1c0c94dc43abac5860ab33cceb42b801c1e8"}, + {file = "cffi-1.15.1-cp311-cp311-win32.whl", hash = "sha256:a0f100c8912c114ff53e1202d0078b425bee3649ae34d7b070e9697f93c5d52d"}, + {file = "cffi-1.15.1-cp311-cp311-win_amd64.whl", hash = "sha256:04ed324bda3cda42b9b695d51bb7d54b680b9719cfab04227cdd1e04e5de3104"}, + {file = "cffi-1.15.1-cp36-cp36m-macosx_10_9_x86_64.whl", hash = "sha256:50a74364d85fd319352182ef59c5c790484a336f6db772c1a9231f1c3ed0cbd7"}, + {file = "cffi-1.15.1-cp36-cp36m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:e263d77ee3dd201c3a142934a086a4450861778baaeeb45db4591ef65550b0a6"}, + {file = "cffi-1.15.1-cp36-cp36m-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:cec7d9412a9102bdc577382c3929b337320c4c4c4849f2c5cdd14d7368c5562d"}, + {file = "cffi-1.15.1-cp36-cp36m-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:4289fc34b2f5316fbb762d75362931e351941fa95fa18789191b33fc4cf9504a"}, + {file = "cffi-1.15.1-cp36-cp36m-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:173379135477dc8cac4bc58f45db08ab45d228b3363adb7af79436135d028405"}, + {file = "cffi-1.15.1-cp36-cp36m-manylinux_2_5_x86_64.manylinux1_x86_64.whl", hash = "sha256:6975a3fac6bc83c4a65c9f9fcab9e47019a11d3d2cf7f3c0d03431bf145a941e"}, + {file = "cffi-1.15.1-cp36-cp36m-win32.whl", hash = "sha256:2470043b93ff09bf8fb1d46d1cb756ce6132c54826661a32d4e4d132e1977adf"}, + {file = "cffi-1.15.1-cp36-cp36m-win_amd64.whl", hash = "sha256:30d78fbc8ebf9c92c9b7823ee18eb92f2e6ef79b45ac84db507f52fbe3ec4497"}, + {file = "cffi-1.15.1-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:198caafb44239b60e252492445da556afafc7d1e3ab7a1fb3f0584ef6d742375"}, + {file = "cffi-1.15.1-cp37-cp37m-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:5ef34d190326c3b1f822a5b7a45f6c4535e2f47ed06fec77d3d799c450b2651e"}, + {file = "cffi-1.15.1-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:8102eaf27e1e448db915d08afa8b41d6c7ca7a04b7d73af6514df10a3e74bd82"}, + {file = "cffi-1.15.1-cp37-cp37m-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:5df2768244d19ab7f60546d0c7c63ce1581f7af8b5de3eb3004b9b6fc8a9f84b"}, + {file = "cffi-1.15.1-cp37-cp37m-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:a8c4917bd7ad33e8eb21e9a5bbba979b49d9a97acb3a803092cbc1133e20343c"}, + {file = "cffi-1.15.1-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:0e2642fe3142e4cc4af0799748233ad6da94c62a8bec3a6648bf8ee68b1c7426"}, + {file = "cffi-1.15.1-cp37-cp37m-win32.whl", hash = "sha256:e229a521186c75c8ad9490854fd8bbdd9a0c9aa3a524326b55be83b54d4e0ad9"}, + {file = "cffi-1.15.1-cp37-cp37m-win_amd64.whl", hash = "sha256:a0b71b1b8fbf2b96e41c4d990244165e2c9be83d54962a9a1d118fd8657d2045"}, + {file = "cffi-1.15.1-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:320dab6e7cb2eacdf0e658569d2575c4dad258c0fcc794f46215e1e39f90f2c3"}, + {file = "cffi-1.15.1-cp38-cp38-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:1e74c6b51a9ed6589199c787bf5f9875612ca4a8a0785fb2d4a84429badaf22a"}, + {file = "cffi-1.15.1-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a5c84c68147988265e60416b57fc83425a78058853509c1b0629c180094904a5"}, + {file = "cffi-1.15.1-cp38-cp38-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:3b926aa83d1edb5aa5b427b4053dc420ec295a08e40911296b9eb1b6170f6cca"}, + {file = "cffi-1.15.1-cp38-cp38-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:87c450779d0914f2861b8526e035c5e6da0a3199d8f1add1a665e1cbc6fc6d02"}, + {file = "cffi-1.15.1-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:4f2c9f67e9821cad2e5f480bc8d83b8742896f1242dba247911072d4fa94c192"}, + {file = "cffi-1.15.1-cp38-cp38-win32.whl", hash = "sha256:8b7ee99e510d7b66cdb6c593f21c043c248537a32e0bedf02e01e9553a172314"}, + {file = "cffi-1.15.1-cp38-cp38-win_amd64.whl", hash = "sha256:00a9ed42e88df81ffae7a8ab6d9356b371399b91dbdf0c3cb1e84c03a13aceb5"}, + {file = "cffi-1.15.1-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:54a2db7b78338edd780e7ef7f9f6c442500fb0d41a5a4ea24fff1c929d5af585"}, + {file = "cffi-1.15.1-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:fcd131dd944808b5bdb38e6f5b53013c5aa4f334c5cad0c72742f6eba4b73db0"}, + {file = "cffi-1.15.1-cp39-cp39-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:7473e861101c9e72452f9bf8acb984947aa1661a7704553a9f6e4baa5ba64415"}, + {file = "cffi-1.15.1-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:6c9a799e985904922a4d207a94eae35c78ebae90e128f0c4e521ce339396be9d"}, + {file = "cffi-1.15.1-cp39-cp39-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:3bcde07039e586f91b45c88f8583ea7cf7a0770df3a1649627bf598332cb6984"}, + {file = "cffi-1.15.1-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:33ab79603146aace82c2427da5ca6e58f2b3f2fb5da893ceac0c42218a40be35"}, + {file = "cffi-1.15.1-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:5d598b938678ebf3c67377cdd45e09d431369c3b1a5b331058c338e201f12b27"}, + {file = "cffi-1.15.1-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:db0fbb9c62743ce59a9ff687eb5f4afbe77e5e8403d6697f7446e5f609976f76"}, + {file = "cffi-1.15.1-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:98d85c6a2bef81588d9227dde12db8a7f47f639f4a17c9ae08e773aa9c697bf3"}, + {file = "cffi-1.15.1-cp39-cp39-win32.whl", hash = "sha256:40f4774f5a9d4f5e344f31a32b5096977b5d48560c5592e2f3d2c4374bd543ee"}, + {file = "cffi-1.15.1-cp39-cp39-win_amd64.whl", hash = "sha256:70df4e3b545a17496c9b3f41f5115e69a4f2e77e94e1d2a8e1070bc0c38c8a3c"}, + {file = "cffi-1.15.1.tar.gz", hash = "sha256:d400bfb9a37b1351253cb402671cea7e89bdecc294e8016a707f6d1d8ac934f9"}, +] [package.dependencies] pycparser = "*" @@ -248,6 +465,10 @@ description = "Validate configuration and produce human readable error messages. category = "dev" optional = false python-versions = ">=3.6.1" +files = [ + {file = "cfgv-3.3.1-py2.py3-none-any.whl", hash = "sha256:c6a0883f3917a037485059700b9e75da2464e6c27051014ad85ba6aaa5884426"}, + {file = "cfgv-3.3.1.tar.gz", hash = "sha256:f5a830efb9ce7a445376bb66ec94c638a9787422f96264c98edc6bdeed8ab736"}, +] [[package]] name = "charset-normalizer" @@ -256,6 +477,10 @@ description = "The Real First Universal Charset Detector. Open, modern and activ category = "main" optional = false python-versions = ">=3.6.0" +files = [ + {file = "charset-normalizer-2.1.1.tar.gz", hash = "sha256:5a3d016c7c547f69d6f81fb0db9449ce888b418b5b9952cc5e6e66843e9dd845"}, + {file = "charset_normalizer-2.1.1-py3-none-any.whl", hash = "sha256:83e9a75d1911279afd89352c68b45348559d1fc0506b054b346651b5e7fee29f"}, +] [package.extras] unicode-backport = ["unicodedata2"] @@ -267,6 +492,10 @@ description = "Composable command line interface toolkit" category = "main" optional = false python-versions = ">=3.7" +files = [ + {file = "click-8.1.3-py3-none-any.whl", hash = "sha256:bb4d8133cb15a609f44e8213d9b391b0809795062913b383c62be0ee95b1db48"}, + {file = "click-8.1.3.tar.gz", hash = "sha256:7682dc8afb30297001674575ea00d1814d808d6a36af415a82bd481d37ba7b8e"}, +] [package.dependencies] colorama = {version = "*", markers = "platform_system == \"Windows\""} @@ -278,6 +507,10 @@ description = "Cross-platform colored terminal text." category = "main" optional = false python-versions = "!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,!=3.4.*,!=3.5.*,!=3.6.*,>=2.7" +files = [ + {file = "colorama-0.4.6-py2.py3-none-any.whl", hash = "sha256:4f1d9991f5acc0ca119f9d443620b77f9d6b33703e51011c16baf57afb285fc6"}, + {file = "colorama-0.4.6.tar.gz", hash = "sha256:08695f5cb7ed6e0531a20572697297273c47b8cae5a63ffc6d6ed5c201be6e44"}, +] [[package]] name = "commonmark" @@ -286,6 +519,10 @@ description = "Python parser for the CommonMark Markdown spec" category = "main" optional = false python-versions = "*" +files = [ + {file = "commonmark-0.9.1-py2.py3-none-any.whl", hash = "sha256:da2f38c92590f83de410ba1a3cbceafbc74fee9def35f9251ba9a971d6d66fd9"}, + {file = "commonmark-0.9.1.tar.gz", hash = "sha256:452f9dc859be7f06631ddcb328b6919c67984aca654e5fefb3914d54691aed60"}, +] [package.extras] test = ["flake8 (==3.7.8)", "hypothesis (==3.55.3)"] @@ -297,1139 +534,7 @@ description = "Code coverage measurement for Python" category = "dev" optional = false python-versions = ">=3.7" - -[package.dependencies] -tomli = {version = "*", optional = true, markers = "python_full_version <= \"3.11.0a6\" and extra == \"toml\""} - -[package.extras] -toml = ["tomli"] - -[[package]] -name = "cryptography" -version = "38.0.4" -description = "cryptography is a package which provides cryptographic recipes and primitives to Python developers." -category = "main" -optional = false -python-versions = ">=3.6" - -[package.dependencies] -cffi = ">=1.12" - -[package.extras] -docs = ["sphinx (>=1.6.5,!=1.8.0,!=3.1.0,!=3.1.1)", "sphinx-rtd-theme"] -docstest = ["pyenchant (>=1.6.11)", "sphinxcontrib-spelling (>=4.0.1)", "twine (>=1.12.0)"] -pep8test = ["black", "flake8", "flake8-import-order", "pep8-naming"] -sdist = ["setuptools-rust (>=0.11.4)"] -ssh = ["bcrypt (>=3.1.5)"] -test = ["hypothesis (>=1.11.4,!=3.79.2)", "iso8601", "pretend", "pytest (>=6.2.0)", "pytest-benchmark", "pytest-cov", "pytest-subtests", "pytest-xdist", "pytz"] - -[[package]] -name = "distlib" -version = "0.3.6" -description = "Distribution utilities" -category = "dev" -optional = false -python-versions = "*" - -[[package]] -name = "docutils" -version = "0.19" -description = "Docutils -- Python Documentation Utilities" -category = "dev" -optional = false -python-versions = ">=3.7" - -[[package]] -name = "duckdb" -version = "0.6.1" -description = "DuckDB embedded database" -category = "main" -optional = true -python-versions = "*" - -[package.dependencies] -numpy = ">=1.14" - -[[package]] -name = "exceptiongroup" -version = "1.0.4" -description = "Backport of PEP 654 (exception groups)" -category = "dev" -optional = false -python-versions = ">=3.7" - -[package.extras] -test = ["pytest (>=6)"] - -[[package]] -name = "fastavro" -version = "1.7.0" -description = "Fast read/write of AVRO files" -category = "dev" -optional = false -python-versions = ">=3.7" - -[package.extras] -codecs = ["lz4", "python-snappy", "zstandard"] -lz4 = ["lz4"] -snappy = ["python-snappy"] -zstandard = ["zstandard"] - -[[package]] -name = "filelock" -version = "3.8.2" -description = "A platform independent file lock." -category = "dev" -optional = false -python-versions = ">=3.7" - -[package.extras] -docs = ["furo (>=2022.9.29)", "sphinx (>=5.3)", "sphinx-autodoc-typehints (>=1.19.5)"] -testing = ["covdefaults (>=2.2.2)", "coverage (>=6.5)", "pytest (>=7.2)", "pytest-cov (>=4)", "pytest-timeout (>=2.1)"] - -[[package]] -name = "frozenlist" -version = "1.3.3" -description = "A list-like structure which implements collections.abc.MutableSequence" -category = "main" -optional = true -python-versions = ">=3.7" - -[[package]] -name = "fsspec" -version = "2022.10.0" -description = "File-system specification" -category = "main" -optional = false -python-versions = ">=3.7" - -[package.extras] -abfs = ["adlfs"] -adl = ["adlfs"] -arrow = ["pyarrow (>=1)"] -dask = ["dask", "distributed"] -dropbox = ["dropbox", "dropboxdrivefs", "requests"] -entrypoints = ["importlib-metadata"] -fuse = ["fusepy"] -gcs = ["gcsfs"] -git = ["pygit2"] -github = ["requests"] -gs = ["gcsfs"] -gui = ["panel"] -hdfs = ["pyarrow (>=1)"] -http = ["aiohttp (!=4.0.0a0,!=4.0.0a1)", "requests"] -libarchive = ["libarchive-c"] -oci = ["ocifs"] -s3 = ["s3fs"] -sftp = ["paramiko"] -smb = ["smbprotocol"] -ssh = ["paramiko"] -tqdm = ["tqdm"] - -[[package]] -name = "identify" -version = "2.5.10" -description = "File identification library for Python" -category = "dev" -optional = false -python-versions = ">=3.7" - -[package.extras] -license = ["ukkonen"] - -[[package]] -name = "idna" -version = "3.4" -description = "Internationalized Domain Names in Applications (IDNA)" -category = "main" -optional = false -python-versions = ">=3.5" - -[[package]] -name = "importlib-metadata" -version = "5.1.0" -description = "Read metadata from Python packages" -category = "dev" -optional = false -python-versions = ">=3.7" - -[package.dependencies] -zipp = ">=0.5" - -[package.extras] -docs = ["furo", "jaraco.packaging (>=9)", "jaraco.tidelift (>=1.4)", "rst.linker (>=1.9)", "sphinx (>=3.5)"] -perf = ["ipython"] -testing = ["flake8 (<5)", "flufl.flake8", "importlib-resources (>=1.3)", "packaging", "pyfakefs", "pytest (>=6)", "pytest-black (>=0.3.7)", "pytest-checkdocs (>=2.4)", "pytest-cov", "pytest-enabler (>=1.3)", "pytest-flake8", "pytest-mypy (>=0.9.1)", "pytest-perf (>=0.9.2)"] - -[[package]] -name = "iniconfig" -version = "1.1.1" -description = "iniconfig: brain-dead simple config-ini parsing" -category = "dev" -optional = false -python-versions = "*" - -[[package]] -name = "isodate" -version = "0.6.1" -description = "An ISO 8601 date/time/duration parser and formatter" -category = "main" -optional = true -python-versions = "*" - -[package.dependencies] -six = "*" - -[[package]] -name = "jinja2" -version = "3.1.2" -description = "A very fast and expressive template engine." -category = "dev" -optional = false -python-versions = ">=3.7" - -[package.dependencies] -MarkupSafe = ">=2.0" - -[package.extras] -i18n = ["Babel (>=2.7)"] - -[[package]] -name = "jmespath" -version = "1.0.1" -description = "JSON Matching Expressions" -category = "main" -optional = false -python-versions = ">=3.7" - -[[package]] -name = "markupsafe" -version = "2.1.1" -description = "Safely add untrusted strings to HTML/XML markup." -category = "dev" -optional = false -python-versions = ">=3.7" - -[[package]] -name = "mmhash3" -version = "3.0.1" -description = "Python wrapper for MurmurHash (MurmurHash3), a set of fast and robust hash functions." -category = "main" -optional = false -python-versions = "*" - -[[package]] -name = "moto" -version = "4.0.13" -description = "A library that allows your python tests to easily mock out the boto library" -category = "dev" -optional = false -python-versions = ">=3.6" - -[package.dependencies] -boto3 = ">=1.9.201" -botocore = ">=1.12.201" -cryptography = ">=3.3.1" -Jinja2 = ">=2.10.1" -MarkupSafe = "!=2.0.0a1" -python-dateutil = ">=2.1,<3.0.0" -requests = ">=2.5" -responses = ">=0.13.0" -werkzeug = ">=0.5,<2.2.0 || >2.2.0,<2.2.1 || >2.2.1" -xmltodict = "*" - -[package.extras] -all = ["PyYAML (>=5.1)", "aws-xray-sdk (>=0.93,!=0.96)", "cfn-lint (>=0.40.0)", "docker (>=2.5.1)", "ecdsa (!=0.15)", "graphql-core", "idna (>=2.5,<4)", "jsondiff (>=1.1.2)", "openapi-spec-validator (>=0.2.8)", "pyparsing (>=3.0.7)", "python-jose[cryptography] (>=3.1.0,<4.0.0)", "setuptools", "sshpubkeys (>=3.1.0)"] -apigateway = ["PyYAML (>=5.1)", "ecdsa (!=0.15)", "openapi-spec-validator (>=0.2.8)", "python-jose[cryptography] (>=3.1.0,<4.0.0)"] -apigatewayv2 = ["PyYAML (>=5.1)"] -appsync = ["graphql-core"] -awslambda = ["docker (>=2.5.1)"] -batch = ["docker (>=2.5.1)"] -cloudformation = ["PyYAML (>=5.1)", "aws-xray-sdk (>=0.93,!=0.96)", "cfn-lint (>=0.40.0)", "docker (>=2.5.1)", "ecdsa (!=0.15)", "graphql-core", "idna (>=2.5,<4)", "jsondiff (>=1.1.2)", "openapi-spec-validator (>=0.2.8)", "pyparsing (>=3.0.7)", "python-jose[cryptography] (>=3.1.0,<4.0.0)", "setuptools", "sshpubkeys (>=3.1.0)"] -cognitoidp = ["ecdsa (!=0.15)", "python-jose[cryptography] (>=3.1.0,<4.0.0)"] -ds = ["sshpubkeys (>=3.1.0)"] -dynamodb = ["docker (>=2.5.1)"] -dynamodb2 = ["docker (>=2.5.1)"] -dynamodbstreams = ["docker (>=2.5.1)"] -ebs = ["sshpubkeys (>=3.1.0)"] -ec2 = ["sshpubkeys (>=3.1.0)"] -efs = ["sshpubkeys (>=3.1.0)"] -glue = ["pyparsing (>=3.0.7)"] -iotdata = ["jsondiff (>=1.1.2)"] -route53resolver = ["sshpubkeys (>=3.1.0)"] -s3 = ["PyYAML (>=5.1)"] -server = ["PyYAML (>=5.1)", "aws-xray-sdk (>=0.93,!=0.96)", "cfn-lint (>=0.40.0)", "docker (>=2.5.1)", "ecdsa (!=0.15)", "flask (!=2.2.0,!=2.2.1)", "flask-cors", "graphql-core", "idna (>=2.5,<4)", "jsondiff (>=1.1.2)", "openapi-spec-validator (>=0.2.8)", "pyparsing (>=3.0.7)", "python-jose[cryptography] (>=3.1.0,<4.0.0)", "setuptools", "sshpubkeys (>=3.1.0)"] -ssm = ["PyYAML (>=5.1)", "dataclasses"] -xray = ["aws-xray-sdk (>=0.93,!=0.96)", "setuptools"] - -[[package]] -name = "msal" -version = "1.20.0" -description = "The Microsoft Authentication Library (MSAL) for Python library enables your app to access the Microsoft Cloud by supporting authentication of users with Microsoft Azure Active Directory accounts (AAD) and Microsoft Accounts (MSA) using industry standard OAuth2 and OpenID Connect." -category = "main" -optional = true -python-versions = "*" - -[package.dependencies] -cryptography = ">=0.6,<41" -PyJWT = {version = ">=1.0.0,<3", extras = ["crypto"]} -requests = ">=2.0.0,<3" - -[package.extras] -broker = ["pymsalruntime (>=0.11.2,<0.14)"] - -[[package]] -name = "msal-extensions" -version = "1.0.0" -description = "Microsoft Authentication Library extensions (MSAL EX) provides a persistence API that can save your data on disk, encrypted on Windows, macOS and Linux. Concurrent data access will be coordinated by a file lock mechanism." -category = "main" -optional = true -python-versions = "*" - -[package.dependencies] -msal = ">=0.4.1,<2.0.0" -portalocker = [ - {version = ">=1.0,<3", markers = "python_version >= \"3.5\" and platform_system != \"Windows\""}, - {version = ">=1.6,<3", markers = "python_version >= \"3.5\" and platform_system == \"Windows\""}, -] - -[[package]] -name = "msrest" -version = "0.7.1" -description = "AutoRest swagger generator Python client runtime." -category = "main" -optional = true -python-versions = ">=3.6" - -[package.dependencies] -azure-core = ">=1.24.0" -certifi = ">=2017.4.17" -isodate = ">=0.6.0" -requests = ">=2.16,<3.0" -requests-oauthlib = ">=0.5.0" - -[package.extras] -async = ["aiodns", "aiohttp (>=3.0)"] - -[[package]] -name = "multidict" -version = "6.0.3" -description = "multidict implementation" -category = "main" -optional = true -python-versions = ">=3.7" - -[[package]] -name = "nodeenv" -version = "1.7.0" -description = "Node.js virtual environment builder" -category = "dev" -optional = false -python-versions = ">=2.7,!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,!=3.4.*,!=3.5.*,!=3.6.*" - -[package.dependencies] -setuptools = "*" - -[[package]] -name = "numpy" -version = "1.23.5" -description = "NumPy is the fundamental package for array computing with Python." -category = "main" -optional = true -python-versions = ">=3.8" - -[[package]] -name = "oauthlib" -version = "3.2.2" -description = "A generic, spec-compliant, thorough implementation of the OAuth request-signing logic" -category = "main" -optional = true -python-versions = ">=3.6" - -[package.extras] -rsa = ["cryptography (>=3.0.0)"] -signals = ["blinker (>=1.4.0)"] -signedtoken = ["cryptography (>=3.0.0)", "pyjwt (>=2.0.0,<3)"] - -[[package]] -name = "packaging" -version = "22.0" -description = "Core utilities for Python packages" -category = "dev" -optional = false -python-versions = ">=3.7" - -[[package]] -name = "pandas" -version = "1.5.2" -description = "Powerful data structures for data analysis, time series, and statistics" -category = "main" -optional = true -python-versions = ">=3.8" - -[package.dependencies] -numpy = [ - {version = ">=1.20.3", markers = "python_version < \"3.10\""}, - {version = ">=1.21.0", markers = "python_version >= \"3.10\""}, - {version = ">=1.23.2", markers = "python_version >= \"3.11\""}, -] -python-dateutil = ">=2.8.1" -pytz = ">=2020.1" - -[package.extras] -test = ["hypothesis (>=5.5.3)", "pytest (>=6.0)", "pytest-xdist (>=1.31)"] - -[[package]] -name = "pep517" -version = "0.13.0" -description = "Wrappers to build Python packages using PEP 517 hooks" -category = "dev" -optional = false -python-versions = ">=3.6" - -[package.dependencies] -tomli = {version = ">=1.1.0", markers = "python_version < \"3.11\""} - -[[package]] -name = "platformdirs" -version = "2.6.0" -description = "A small Python package for determining appropriate platform-specific dirs, e.g. a \"user data dir\"." -category = "dev" -optional = false -python-versions = ">=3.7" - -[package.extras] -docs = ["furo (>=2022.9.29)", "proselint (>=0.13)", "sphinx (>=5.3)", "sphinx-autodoc-typehints (>=1.19.4)"] -test = ["appdirs (==1.4.4)", "pytest (>=7.2)", "pytest-cov (>=4)", "pytest-mock (>=3.10)"] - -[[package]] -name = "pluggy" -version = "1.0.0" -description = "plugin and hook calling mechanisms for python" -category = "dev" -optional = false -python-versions = ">=3.6" - -[package.extras] -dev = ["pre-commit", "tox"] -testing = ["pytest", "pytest-benchmark"] - -[[package]] -name = "portalocker" -version = "2.6.0" -description = "Wraps the portalocker recipe for easy usage" -category = "main" -optional = true -python-versions = ">=3.5" - -[package.dependencies] -pywin32 = {version = ">=226", markers = "platform_system == \"Windows\""} - -[package.extras] -docs = ["sphinx (>=1.7.1)"] -redis = ["redis"] -tests = ["pytest (>=5.4.1)", "pytest-cov (>=2.8.1)", "pytest-mypy (>=0.8.0)", "pytest-timeout (>=2.1.0)", "redis", "sphinx (>=3.0.3)"] - -[[package]] -name = "pre-commit" -version = "2.21.0" -description = "A framework for managing and maintaining multi-language pre-commit hooks." -category = "dev" -optional = false -python-versions = ">=3.7" - -[package.dependencies] -cfgv = ">=2.0.0" -identify = ">=1.0.0" -nodeenv = ">=0.11.1" -pyyaml = ">=5.1" -virtualenv = ">=20.10.0" - -[[package]] -name = "pyarrow" -version = "10.0.1" -description = "Python library for Apache Arrow" -category = "main" -optional = true -python-versions = ">=3.7" - -[package.dependencies] -numpy = ">=1.16.6" - -[[package]] -name = "pycparser" -version = "2.21" -description = "C parser in Python" -category = "main" -optional = false -python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*" - -[[package]] -name = "pydantic" -version = "1.10.4" -description = "Data validation and settings management using python type hints" -category = "main" -optional = false -python-versions = ">=3.7" - -[package.dependencies] -typing-extensions = ">=4.2.0" - -[package.extras] -dotenv = ["python-dotenv (>=0.10.4)"] -email = ["email-validator (>=1.0.3)"] - -[[package]] -name = "pygments" -version = "2.13.0" -description = "Pygments is a syntax highlighting package written in Python." -category = "main" -optional = false -python-versions = ">=3.6" - -[package.extras] -plugins = ["importlib-metadata"] - -[[package]] -name = "pyjwt" -version = "2.6.0" -description = "JSON Web Token implementation in Python" -category = "main" -optional = true -python-versions = ">=3.7" - -[package.dependencies] -cryptography = {version = ">=3.4.0", optional = true, markers = "extra == \"crypto\""} - -[package.extras] -crypto = ["cryptography (>=3.4.0)"] -dev = ["coverage[toml] (==5.0.4)", "cryptography (>=3.4.0)", "pre-commit", "pytest (>=6.0.0,<7.0.0)", "sphinx (>=4.5.0,<5.0.0)", "sphinx-rtd-theme", "zope.interface"] -docs = ["sphinx (>=4.5.0,<5.0.0)", "sphinx-rtd-theme", "zope.interface"] -tests = ["coverage[toml] (==5.0.4)", "pytest (>=6.0.0,<7.0.0)"] - -[[package]] -name = "pyparsing" -version = "3.0.9" -description = "pyparsing module - Classes and methods to define and execute parsing grammars" -category = "main" -optional = false -python-versions = ">=3.6.8" - -[package.extras] -diagrams = ["jinja2", "railroad-diagrams"] - -[[package]] -name = "pytest" -version = "7.2.0" -description = "pytest: simple powerful testing with Python" -category = "dev" -optional = false -python-versions = ">=3.7" - -[package.dependencies] -attrs = ">=19.2.0" -colorama = {version = "*", markers = "sys_platform == \"win32\""} -exceptiongroup = {version = ">=1.0.0rc8", markers = "python_version < \"3.11\""} -iniconfig = "*" -packaging = "*" -pluggy = ">=0.12,<2.0" -tomli = {version = ">=1.0.0", markers = "python_version < \"3.11\""} - -[package.extras] -testing = ["argcomplete", "hypothesis (>=3.56)", "mock", "nose", "pygments (>=2.7.2)", "requests", "xmlschema"] - -[[package]] -name = "pytest-checkdocs" -version = "2.9.0" -description = "check the README when running tests" -category = "dev" -optional = false -python-versions = ">=3.7" - -[package.dependencies] -build = "*" -docutils = ">=0.15" -importlib-metadata = {version = ">=4", markers = "python_version < \"3.10\""} - -[package.extras] -docs = ["furo", "jaraco.packaging (>=9)", "rst.linker (>=1.9)", "sphinx (>=3.5)"] -testing = ["flake8 (<5)", "pytest (>=6)", "pytest-black (>=0.3.7)", "pytest-checkdocs (>=2.4)", "pytest-cov", "pytest-enabler (>=1.3)", "pytest-flake8", "pytest-mypy (>=0.9.1)", "types-docutils"] - -[[package]] -name = "python-dateutil" -version = "2.8.2" -description = "Extensions to the standard Python datetime module" -category = "main" -optional = false -python-versions = "!=3.0.*,!=3.1.*,!=3.2.*,>=2.7" - -[package.dependencies] -six = ">=1.5" - -[[package]] -name = "python-snappy" -version = "0.6.1" -description = "Python library for the snappy compression library from Google" -category = "main" -optional = true -python-versions = "*" - -[[package]] -name = "pytz" -version = "2022.6" -description = "World timezone definitions, modern and historical" -category = "main" -optional = true -python-versions = "*" - -[[package]] -name = "pywin32" -version = "305" -description = "Python for Window Extensions" -category = "main" -optional = true -python-versions = "*" - -[[package]] -name = "pyyaml" -version = "6.0" -description = "YAML parser and emitter for Python" -category = "main" -optional = false -python-versions = ">=3.6" - -[[package]] -name = "requests" -version = "2.28.1" -description = "Python HTTP for Humans." -category = "main" -optional = false -python-versions = ">=3.7, <4" - -[package.dependencies] -certifi = ">=2017.4.17" -charset-normalizer = ">=2,<3" -idna = ">=2.5,<4" -urllib3 = ">=1.21.1,<1.27" - -[package.extras] -socks = ["PySocks (>=1.5.6,!=1.5.7)"] -use-chardet-on-py3 = ["chardet (>=3.0.2,<6)"] - -[[package]] -name = "requests-mock" -version = "1.10.0" -description = "Mock out responses from the requests package" -category = "dev" -optional = false -python-versions = "*" - -[package.dependencies] -requests = ">=2.3,<3" -six = "*" - -[package.extras] -fixture = ["fixtures"] -test = ["fixtures", "mock", "purl", "pytest", "requests-futures", "sphinx", "testrepository (>=0.0.18)", "testtools"] - -[[package]] -name = "requests-oauthlib" -version = "1.3.1" -description = "OAuthlib authentication support for Requests." -category = "main" -optional = true -python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*" - -[package.dependencies] -oauthlib = ">=3.0.0" -requests = ">=2.0.0" - -[package.extras] -rsa = ["oauthlib[signedtoken] (>=3.0.0)"] - -[[package]] -name = "responses" -version = "0.22.0" -description = "A utility library for mocking out the `requests` Python library." -category = "dev" -optional = false -python-versions = ">=3.7" - -[package.dependencies] -requests = ">=2.22.0,<3.0" -toml = "*" -types-toml = "*" -urllib3 = ">=1.25.10" - -[package.extras] -tests = ["coverage (>=6.0.0)", "flake8", "mypy", "pytest (>=7.0.0)", "pytest-asyncio", "pytest-cov", "pytest-httpserver", "types-requests"] - -[[package]] -name = "rich" -version = "13.0.1" -description = "Render rich text, tables, progress bars, syntax highlighting, markdown and more to the terminal" -category = "main" -optional = false -python-versions = ">=3.7.0" - -[package.dependencies] -commonmark = ">=0.9.0,<0.10.0" -pygments = ">=2.6.0,<3.0.0" -typing-extensions = {version = ">=4.0.0,<5.0", markers = "python_version < \"3.9\""} - -[package.extras] -jupyter = ["ipywidgets (>=7.5.1,<8.0.0)"] - -[[package]] -name = "s3fs" -version = "2022.10.0" -description = "Convenient Filesystem interface over S3" -category = "main" -optional = true -python-versions = ">= 3.7" - -[package.dependencies] -aiobotocore = ">=2.4.0,<2.5.0" -aiohttp = "<4.0.0a0 || >4.0.0a0,<4.0.0a1 || >4.0.0a1" -fsspec = "2022.10.0" - -[package.extras] -awscli = ["aiobotocore[awscli] (>=2.4.0,<2.5.0)"] -boto3 = ["aiobotocore[boto3] (>=2.4.0,<2.5.0)"] - -[[package]] -name = "s3transfer" -version = "0.6.0" -description = "An Amazon S3 Transfer Manager" -category = "main" -optional = false -python-versions = ">= 3.7" - -[package.dependencies] -botocore = ">=1.12.36,<2.0a.0" - -[package.extras] -crt = ["botocore[crt] (>=1.20.29,<2.0a.0)"] - -[[package]] -name = "setuptools" -version = "65.6.3" -description = "Easily download, build, install, upgrade, and uninstall Python packages" -category = "dev" -optional = false -python-versions = ">=3.7" - -[package.extras] -docs = ["furo", "jaraco.packaging (>=9)", "jaraco.tidelift (>=1.4)", "pygments-github-lexers (==0.0.5)", "rst.linker (>=1.9)", "sphinx (>=3.5)", "sphinx-favicon", "sphinx-hoverxref (<2)", "sphinx-inline-tabs", "sphinx-notfound-page (==0.8.3)", "sphinx-reredirects", "sphinxcontrib-towncrier"] -testing = ["build[virtualenv]", "filelock (>=3.4.0)", "flake8 (<5)", "flake8-2020", "ini2toml[lite] (>=0.9)", "jaraco.envs (>=2.2)", "jaraco.path (>=3.2.0)", "pip (>=19.1)", "pip-run (>=8.8)", "pytest (>=6)", "pytest-black (>=0.3.7)", "pytest-checkdocs (>=2.4)", "pytest-cov", "pytest-enabler (>=1.3)", "pytest-flake8", "pytest-mypy (>=0.9.1)", "pytest-perf", "pytest-timeout", "pytest-xdist", "tomli-w (>=1.0.0)", "virtualenv (>=13.0.0)", "wheel"] -testing-integration = ["build[virtualenv]", "filelock (>=3.4.0)", "jaraco.envs (>=2.2)", "jaraco.path (>=3.2.0)", "pytest", "pytest-enabler", "pytest-xdist", "tomli", "virtualenv (>=13.0.0)", "wheel"] - -[[package]] -name = "six" -version = "1.16.0" -description = "Python 2 and 3 compatibility utilities" -category = "main" -optional = false -python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*" - -[[package]] -name = "thrift" -version = "0.16.0" -description = "Python bindings for the Apache Thrift RPC system" -category = "main" -optional = true -python-versions = "*" - -[package.dependencies] -six = ">=1.7.2" - -[package.extras] -all = ["tornado (>=4.0)", "twisted"] -tornado = ["tornado (>=4.0)"] -twisted = ["twisted"] - -[[package]] -name = "toml" -version = "0.10.2" -description = "Python Library for Tom's Obvious, Minimal Language" -category = "dev" -optional = false -python-versions = ">=2.6, !=3.0.*, !=3.1.*, !=3.2.*" - -[[package]] -name = "tomli" -version = "2.0.1" -description = "A lil' TOML parser" -category = "dev" -optional = false -python-versions = ">=3.7" - -[[package]] -name = "types-toml" -version = "0.10.8.1" -description = "Typing stubs for toml" -category = "dev" -optional = false -python-versions = "*" - -[[package]] -name = "typing-extensions" -version = "4.4.0" -description = "Backported and Experimental Type Hints for Python 3.7+" -category = "main" -optional = false -python-versions = ">=3.7" - -[[package]] -name = "urllib3" -version = "1.26.13" -description = "HTTP library with thread-safe connection pooling, file post, and more." -category = "main" -optional = false -python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*, !=3.5.*" - -[package.extras] -brotli = ["brotli (>=1.0.9)", "brotlicffi (>=0.8.0)", "brotlipy (>=0.6.0)"] -secure = ["certifi", "cryptography (>=1.3.4)", "idna (>=2.0.0)", "ipaddress", "pyOpenSSL (>=0.14)", "urllib3-secure-extra"] -socks = ["PySocks (>=1.5.6,!=1.5.7,<2.0)"] - -[[package]] -name = "virtualenv" -version = "20.17.1" -description = "Virtual Python Environment builder" -category = "dev" -optional = false -python-versions = ">=3.6" - -[package.dependencies] -distlib = ">=0.3.6,<1" -filelock = ">=3.4.1,<4" -platformdirs = ">=2.4,<3" - -[package.extras] -docs = ["proselint (>=0.13)", "sphinx (>=5.3)", "sphinx-argparse (>=0.3.2)", "sphinx-rtd-theme (>=1)", "towncrier (>=22.8)"] -testing = ["coverage (>=6.2)", "coverage-enable-subprocess (>=1)", "flaky (>=3.7)", "packaging (>=21.3)", "pytest (>=7.0.1)", "pytest-env (>=0.6.2)", "pytest-freezegun (>=0.4.2)", "pytest-mock (>=3.6.1)", "pytest-randomly (>=3.10.3)", "pytest-timeout (>=2.1)"] - -[[package]] -name = "werkzeug" -version = "2.2.2" -description = "The comprehensive WSGI web application library." -category = "dev" -optional = false -python-versions = ">=3.7" - -[package.dependencies] -MarkupSafe = ">=2.1.1" - -[package.extras] -watchdog = ["watchdog"] - -[[package]] -name = "wrapt" -version = "1.14.1" -description = "Module for decorators, wrappers and monkey patching." -category = "main" -optional = true -python-versions = "!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,!=3.4.*,>=2.7" - -[[package]] -name = "xmltodict" -version = "0.13.0" -description = "Makes working with XML feel like you are working with JSON" -category = "dev" -optional = false -python-versions = ">=3.4" - -[[package]] -name = "yarl" -version = "1.8.2" -description = "Yet another URL library" -category = "main" -optional = true -python-versions = ">=3.7" - -[package.dependencies] -idna = ">=2.0" -multidict = ">=4.0" - -[[package]] -name = "zipp" -version = "3.11.0" -description = "Backport of pathlib-compatible object wrapper for zip files" -category = "dev" -optional = false -python-versions = ">=3.7" - -[package.extras] -docs = ["furo", "jaraco.packaging (>=9)", "jaraco.tidelift (>=1.4)", "rst.linker (>=1.9)", "sphinx (>=3.5)"] -testing = ["flake8 (<5)", "func-timeout", "jaraco.functools", "jaraco.itertools", "more-itertools", "pytest (>=6)", "pytest-black (>=0.3.7)", "pytest-checkdocs (>=2.4)", "pytest-cov", "pytest-enabler (>=1.3)", "pytest-flake8", "pytest-mypy (>=0.9.1)"] - -[[package]] -name = "zstandard" -version = "0.19.0" -description = "Zstandard bindings for Python" -category = "main" -optional = false -python-versions = ">=3.6" - -[package.dependencies] -cffi = {version = ">=1.11", markers = "platform_python_implementation == \"PyPy\""} - -[package.extras] -cffi = ["cffi (>=1.11)"] - -[extras] -adlfs = ["adlfs"] -duckdb = ["duckdb", "pyarrow"] -glue = ["boto3"] -hive = ["thrift"] -pandas = ["pandas", "pyarrow"] -pyarrow = ["pyarrow"] -s3fs = ["s3fs"] -snappy = ["python-snappy"] - -[metadata] -lock-version = "1.1" -python-versions = "^3.8" -content-hash = "cd9c64bc3b45183df8d3a1689b90fc019dce3ae616c444a74f4129822fc9ffac" - -[metadata.files] -adal = [ - {file = "adal-1.2.7-py2.py3-none-any.whl", hash = "sha256:2a7451ed7441ddbc57703042204a3e30ef747478eea022c70f789fc7f084bc3d"}, - {file = "adal-1.2.7.tar.gz", hash = "sha256:d74f45b81317454d96e982fd1c50e6fb5c99ac2223728aea8764433a39f566f1"}, -] -adlfs = [ - {file = "adlfs-2022.11.2-py3-none-any.whl", hash = "sha256:60da4a17260754ee6d8da43e3d756226ade4fd19ada8411e858f5acecd0833e2"}, - {file = "adlfs-2022.11.2.tar.gz", hash = "sha256:920dba10468f186037ca394dcabcba113532d80f52b315211c8e771be40475ea"}, -] -aiobotocore = [ - {file = "aiobotocore-2.4.1-py3-none-any.whl", hash = "sha256:26e0d55d5901f487c3467616c028abb85036ca33a0f88e279770adae6b865c69"}, - {file = "aiobotocore-2.4.1.tar.gz", hash = "sha256:5c8ee79d1f14273e3f8f3af5c893db889ef163227b178ecacb5aea4547d3b9a7"}, -] -aiohttp = [ - {file = "aiohttp-3.8.3-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:ba71c9b4dcbb16212f334126cc3d8beb6af377f6703d9dc2d9fb3874fd667ee9"}, - {file = "aiohttp-3.8.3-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:d24b8bb40d5c61ef2d9b6a8f4528c2f17f1c5d2d31fed62ec860f6006142e83e"}, - {file = "aiohttp-3.8.3-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:f88df3a83cf9df566f171adba39d5bd52814ac0b94778d2448652fc77f9eb491"}, - {file = "aiohttp-3.8.3-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:b97decbb3372d4b69e4d4c8117f44632551c692bb1361b356a02b97b69e18a62"}, - {file = "aiohttp-3.8.3-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:309aa21c1d54b8ef0723181d430347d7452daaff93e8e2363db8e75c72c2fb2d"}, - {file = "aiohttp-3.8.3-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:ad5383a67514e8e76906a06741febd9126fc7c7ff0f599d6fcce3e82b80d026f"}, - {file = "aiohttp-3.8.3-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:20acae4f268317bb975671e375493dbdbc67cddb5f6c71eebdb85b34444ac46b"}, - {file = "aiohttp-3.8.3-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:05a3c31c6d7cd08c149e50dc7aa2568317f5844acd745621983380597f027a18"}, - {file = "aiohttp-3.8.3-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:d6f76310355e9fae637c3162936e9504b4767d5c52ca268331e2756e54fd4ca5"}, - {file = "aiohttp-3.8.3-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:256deb4b29fe5e47893fa32e1de2d73c3afe7407738bd3c63829874661d4822d"}, - {file = "aiohttp-3.8.3-cp310-cp310-musllinux_1_1_ppc64le.whl", hash = "sha256:5c59fcd80b9049b49acd29bd3598cada4afc8d8d69bd4160cd613246912535d7"}, - {file = "aiohttp-3.8.3-cp310-cp310-musllinux_1_1_s390x.whl", hash = "sha256:059a91e88f2c00fe40aed9031b3606c3f311414f86a90d696dd982e7aec48142"}, - {file = "aiohttp-3.8.3-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:2feebbb6074cdbd1ac276dbd737b40e890a1361b3cc30b74ac2f5e24aab41f7b"}, - {file = "aiohttp-3.8.3-cp310-cp310-win32.whl", hash = "sha256:5bf651afd22d5f0c4be16cf39d0482ea494f5c88f03e75e5fef3a85177fecdeb"}, - {file = "aiohttp-3.8.3-cp310-cp310-win_amd64.whl", hash = "sha256:653acc3880459f82a65e27bd6526e47ddf19e643457d36a2250b85b41a564715"}, - {file = "aiohttp-3.8.3-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:86fc24e58ecb32aee09f864cb11bb91bc4c1086615001647dbfc4dc8c32f4008"}, - {file = "aiohttp-3.8.3-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:75e14eac916f024305db517e00a9252714fce0abcb10ad327fb6dcdc0d060f1d"}, - {file = "aiohttp-3.8.3-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:d1fde0f44029e02d02d3993ad55ce93ead9bb9b15c6b7ccd580f90bd7e3de476"}, - {file = "aiohttp-3.8.3-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:4ab94426ddb1ecc6a0b601d832d5d9d421820989b8caa929114811369673235c"}, - {file = "aiohttp-3.8.3-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:89d2e02167fa95172c017732ed7725bc8523c598757f08d13c5acca308e1a061"}, - {file = "aiohttp-3.8.3-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:02f9a2c72fc95d59b881cf38a4b2be9381b9527f9d328771e90f72ac76f31ad8"}, - {file = "aiohttp-3.8.3-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:9c7149272fb5834fc186328e2c1fa01dda3e1fa940ce18fded6d412e8f2cf76d"}, - {file = "aiohttp-3.8.3-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:512bd5ab136b8dc0ffe3fdf2dfb0c4b4f49c8577f6cae55dca862cd37a4564e2"}, - {file = "aiohttp-3.8.3-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:7018ecc5fe97027214556afbc7c502fbd718d0740e87eb1217b17efd05b3d276"}, - {file = "aiohttp-3.8.3-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:88c70ed9da9963d5496d38320160e8eb7e5f1886f9290475a881db12f351ab5d"}, - {file = "aiohttp-3.8.3-cp311-cp311-musllinux_1_1_ppc64le.whl", hash = "sha256:da22885266bbfb3f78218dc40205fed2671909fbd0720aedba39b4515c038091"}, - {file = "aiohttp-3.8.3-cp311-cp311-musllinux_1_1_s390x.whl", hash = "sha256:e65bc19919c910127c06759a63747ebe14f386cda573d95bcc62b427ca1afc73"}, - {file = "aiohttp-3.8.3-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:08c78317e950e0762c2983f4dd58dc5e6c9ff75c8a0efeae299d363d439c8e34"}, - {file = "aiohttp-3.8.3-cp311-cp311-win32.whl", hash = "sha256:45d88b016c849d74ebc6f2b6e8bc17cabf26e7e40c0661ddd8fae4c00f015697"}, - {file = "aiohttp-3.8.3-cp311-cp311-win_amd64.whl", hash = "sha256:96372fc29471646b9b106ee918c8eeb4cca423fcbf9a34daa1b93767a88a2290"}, - {file = "aiohttp-3.8.3-cp36-cp36m-macosx_10_9_x86_64.whl", hash = "sha256:c971bf3786b5fad82ce5ad570dc6ee420f5b12527157929e830f51c55dc8af77"}, - {file = "aiohttp-3.8.3-cp36-cp36m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:ff25f48fc8e623d95eca0670b8cc1469a83783c924a602e0fbd47363bb54aaca"}, - {file = "aiohttp-3.8.3-cp36-cp36m-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:e381581b37db1db7597b62a2e6b8b57c3deec95d93b6d6407c5b61ddc98aca6d"}, - {file = "aiohttp-3.8.3-cp36-cp36m-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:db19d60d846283ee275d0416e2a23493f4e6b6028825b51290ac05afc87a6f97"}, - {file = "aiohttp-3.8.3-cp36-cp36m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:25892c92bee6d9449ffac82c2fe257f3a6f297792cdb18ad784737d61e7a9a85"}, - {file = "aiohttp-3.8.3-cp36-cp36m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:398701865e7a9565d49189f6c90868efaca21be65c725fc87fc305906be915da"}, - {file = "aiohttp-3.8.3-cp36-cp36m-musllinux_1_1_aarch64.whl", hash = "sha256:4a4fbc769ea9b6bd97f4ad0b430a6807f92f0e5eb020f1e42ece59f3ecfc4585"}, - {file = "aiohttp-3.8.3-cp36-cp36m-musllinux_1_1_i686.whl", hash = "sha256:b29bfd650ed8e148f9c515474a6ef0ba1090b7a8faeee26b74a8ff3b33617502"}, - {file = "aiohttp-3.8.3-cp36-cp36m-musllinux_1_1_ppc64le.whl", hash = "sha256:1e56b9cafcd6531bab5d9b2e890bb4937f4165109fe98e2b98ef0dcfcb06ee9d"}, - {file = "aiohttp-3.8.3-cp36-cp36m-musllinux_1_1_s390x.whl", hash = "sha256:ec40170327d4a404b0d91855d41bfe1fe4b699222b2b93e3d833a27330a87a6d"}, - {file = "aiohttp-3.8.3-cp36-cp36m-musllinux_1_1_x86_64.whl", hash = "sha256:2df5f139233060578d8c2c975128fb231a89ca0a462b35d4b5fcf7c501ebdbe1"}, - {file = "aiohttp-3.8.3-cp36-cp36m-win32.whl", hash = "sha256:f973157ffeab5459eefe7b97a804987876dd0a55570b8fa56b4e1954bf11329b"}, - {file = "aiohttp-3.8.3-cp36-cp36m-win_amd64.whl", hash = "sha256:437399385f2abcd634865705bdc180c8314124b98299d54fe1d4c8990f2f9494"}, - {file = "aiohttp-3.8.3-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:09e28f572b21642128ef31f4e8372adb6888846f32fecb288c8b0457597ba61a"}, - {file = "aiohttp-3.8.3-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:6f3553510abdbec67c043ca85727396ceed1272eef029b050677046d3387be8d"}, - {file = "aiohttp-3.8.3-cp37-cp37m-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:e168a7560b7c61342ae0412997b069753f27ac4862ec7867eff74f0fe4ea2ad9"}, - {file = "aiohttp-3.8.3-cp37-cp37m-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:db4c979b0b3e0fa7e9e69ecd11b2b3174c6963cebadeecfb7ad24532ffcdd11a"}, - {file = "aiohttp-3.8.3-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:e164e0a98e92d06da343d17d4e9c4da4654f4a4588a20d6c73548a29f176abe2"}, - {file = "aiohttp-3.8.3-cp37-cp37m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:e8a78079d9a39ca9ca99a8b0ac2fdc0c4d25fc80c8a8a82e5c8211509c523363"}, - {file = "aiohttp-3.8.3-cp37-cp37m-musllinux_1_1_aarch64.whl", hash = "sha256:21b30885a63c3f4ff5b77a5d6caf008b037cb521a5f33eab445dc566f6d092cc"}, - {file = "aiohttp-3.8.3-cp37-cp37m-musllinux_1_1_i686.whl", hash = "sha256:4b0f30372cef3fdc262f33d06e7b411cd59058ce9174ef159ad938c4a34a89da"}, - {file = "aiohttp-3.8.3-cp37-cp37m-musllinux_1_1_ppc64le.whl", hash = "sha256:8135fa153a20d82ffb64f70a1b5c2738684afa197839b34cc3e3c72fa88d302c"}, - {file = "aiohttp-3.8.3-cp37-cp37m-musllinux_1_1_s390x.whl", hash = "sha256:ad61a9639792fd790523ba072c0555cd6be5a0baf03a49a5dd8cfcf20d56df48"}, - {file = "aiohttp-3.8.3-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:978b046ca728073070e9abc074b6299ebf3501e8dee5e26efacb13cec2b2dea0"}, - {file = "aiohttp-3.8.3-cp37-cp37m-win32.whl", hash = "sha256:0d2c6d8c6872df4a6ec37d2ede71eff62395b9e337b4e18efd2177de883a5033"}, - {file = "aiohttp-3.8.3-cp37-cp37m-win_amd64.whl", hash = "sha256:21d69797eb951f155026651f7e9362877334508d39c2fc37bd04ff55b2007091"}, - {file = "aiohttp-3.8.3-cp38-cp38-macosx_10_9_universal2.whl", hash = "sha256:2ca9af5f8f5812d475c5259393f52d712f6d5f0d7fdad9acdb1107dd9e3cb7eb"}, - {file = "aiohttp-3.8.3-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:1d90043c1882067f1bd26196d5d2db9aa6d268def3293ed5fb317e13c9413ea4"}, - {file = "aiohttp-3.8.3-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:d737fc67b9a970f3234754974531dc9afeea11c70791dcb7db53b0cf81b79784"}, - {file = "aiohttp-3.8.3-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:ebf909ea0a3fc9596e40d55d8000702a85e27fd578ff41a5500f68f20fd32e6c"}, - {file = "aiohttp-3.8.3-cp38-cp38-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:5835f258ca9f7c455493a57ee707b76d2d9634d84d5d7f62e77be984ea80b849"}, - {file = "aiohttp-3.8.3-cp38-cp38-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:da37dcfbf4b7f45d80ee386a5f81122501ec75672f475da34784196690762f4b"}, - {file = "aiohttp-3.8.3-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:87f44875f2804bc0511a69ce44a9595d5944837a62caecc8490bbdb0e18b1342"}, - {file = "aiohttp-3.8.3-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:527b3b87b24844ea7865284aabfab08eb0faf599b385b03c2aa91fc6edd6e4b6"}, - {file = "aiohttp-3.8.3-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:d5ba88df9aa5e2f806650fcbeedbe4f6e8736e92fc0e73b0400538fd25a4dd96"}, - {file = "aiohttp-3.8.3-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:e7b8813be97cab8cb52b1375f41f8e6804f6507fe4660152e8ca5c48f0436017"}, - {file = "aiohttp-3.8.3-cp38-cp38-musllinux_1_1_ppc64le.whl", hash = "sha256:2dea10edfa1a54098703cb7acaa665c07b4e7568472a47f4e64e6319d3821ccf"}, - {file = "aiohttp-3.8.3-cp38-cp38-musllinux_1_1_s390x.whl", hash = "sha256:713d22cd9643ba9025d33c4af43943c7a1eb8547729228de18d3e02e278472b6"}, - {file = "aiohttp-3.8.3-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:2d252771fc85e0cf8da0b823157962d70639e63cb9b578b1dec9868dd1f4f937"}, - {file = "aiohttp-3.8.3-cp38-cp38-win32.whl", hash = "sha256:66bd5f950344fb2b3dbdd421aaa4e84f4411a1a13fca3aeb2bcbe667f80c9f76"}, - {file = "aiohttp-3.8.3-cp38-cp38-win_amd64.whl", hash = "sha256:84b14f36e85295fe69c6b9789b51a0903b774046d5f7df538176516c3e422446"}, - {file = "aiohttp-3.8.3-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:16c121ba0b1ec2b44b73e3a8a171c4f999b33929cd2397124a8c7fcfc8cd9e06"}, - {file = "aiohttp-3.8.3-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:8d6aaa4e7155afaf994d7924eb290abbe81a6905b303d8cb61310a2aba1c68ba"}, - {file = "aiohttp-3.8.3-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:43046a319664a04b146f81b40e1545d4c8ac7b7dd04c47e40bf09f65f2437346"}, - {file = "aiohttp-3.8.3-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:599418aaaf88a6d02a8c515e656f6faf3d10618d3dd95866eb4436520096c84b"}, - {file = "aiohttp-3.8.3-cp39-cp39-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:92a2964319d359f494f16011e23434f6f8ef0434acd3cf154a6b7bec511e2fb7"}, - {file = "aiohttp-3.8.3-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:73a4131962e6d91109bca6536416aa067cf6c4efb871975df734f8d2fd821b37"}, - {file = "aiohttp-3.8.3-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:598adde339d2cf7d67beaccda3f2ce7c57b3b412702f29c946708f69cf8222aa"}, - {file = "aiohttp-3.8.3-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:75880ed07be39beff1881d81e4a907cafb802f306efd6d2d15f2b3c69935f6fb"}, - {file = "aiohttp-3.8.3-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:a0239da9fbafd9ff82fd67c16704a7d1bccf0d107a300e790587ad05547681c8"}, - {file = "aiohttp-3.8.3-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:4e3a23ec214e95c9fe85a58470b660efe6534b83e6cbe38b3ed52b053d7cb6ad"}, - {file = "aiohttp-3.8.3-cp39-cp39-musllinux_1_1_ppc64le.whl", hash = "sha256:47841407cc89a4b80b0c52276f3cc8138bbbfba4b179ee3acbd7d77ae33f7ac4"}, - {file = "aiohttp-3.8.3-cp39-cp39-musllinux_1_1_s390x.whl", hash = "sha256:54d107c89a3ebcd13228278d68f1436d3f33f2dd2af5415e3feaeb1156e1a62c"}, - {file = "aiohttp-3.8.3-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:c37c5cce780349d4d51739ae682dec63573847a2a8dcb44381b174c3d9c8d403"}, - {file = "aiohttp-3.8.3-cp39-cp39-win32.whl", hash = "sha256:f178d2aadf0166be4df834c4953da2d7eef24719e8aec9a65289483eeea9d618"}, - {file = "aiohttp-3.8.3-cp39-cp39-win_amd64.whl", hash = "sha256:88e5be56c231981428f4f506c68b6a46fa25c4123a2e86d156c58a8369d31ab7"}, - {file = "aiohttp-3.8.3.tar.gz", hash = "sha256:3828fb41b7203176b82fe5d699e0d845435f2374750a44b480ea6b930f6be269"}, -] -aioitertools = [ - {file = "aioitertools-0.11.0-py3-none-any.whl", hash = "sha256:04b95e3dab25b449def24d7df809411c10e62aab0cbe31a50ca4e68748c43394"}, - {file = "aioitertools-0.11.0.tar.gz", hash = "sha256:42c68b8dd3a69c2bf7f2233bf7df4bb58b557bca5252ac02ed5187bbc67d6831"}, -] -aiosignal = [ - {file = "aiosignal-1.3.1-py3-none-any.whl", hash = "sha256:f8376fb07dd1e86a584e4fcdec80b36b7f81aac666ebc724e2c090300dd83b17"}, - {file = "aiosignal-1.3.1.tar.gz", hash = "sha256:54cd96e15e1649b75d6c87526a6ff0b6c1b0dd3459f43d9ca11d48c339b68cfc"}, -] -async-timeout = [ - {file = "async-timeout-4.0.2.tar.gz", hash = "sha256:2163e1640ddb52b7a8c80d0a67a08587e5d245cc9c553a74a847056bc2976b15"}, - {file = "async_timeout-4.0.2-py3-none-any.whl", hash = "sha256:8ca1e4fcf50d07413d66d1a5e416e42cfdf5851c981d679a09851a6853383b3c"}, -] -attrs = [ - {file = "attrs-22.1.0-py2.py3-none-any.whl", hash = "sha256:86efa402f67bf2df34f51a335487cf46b1ec130d02b8d39fd248abfd30da551c"}, - {file = "attrs-22.1.0.tar.gz", hash = "sha256:29adc2665447e5191d0e7c568fde78b21f9672d344281d0c6e1ab085429b22b6"}, -] -azure-core = [ - {file = "azure-core-1.26.1.zip", hash = "sha256:223b0e90cbdd1f03c41b195b03239899843f20d00964dbb85e64386873414a2d"}, - {file = "azure_core-1.26.1-py3-none-any.whl", hash = "sha256:726ffd1ded04a2c1cb53f9d9155cbb05ac5c1c2a29af4ef622e93e1c0a8bc92b"}, -] -azure-datalake-store = [ - {file = "azure-datalake-store-0.0.52.tar.gz", hash = "sha256:4198ddb32614d16d4502b43d5c9739f81432b7e0e4d75d30e05149fe6007fea2"}, - {file = "azure_datalake_store-0.0.52-py2.py3-none-any.whl", hash = "sha256:aaed72b9c856824aeab554f4dbe0ef2c6d0ff36700bdd8b93d8298793117c48e"}, -] -azure-identity = [ - {file = "azure-identity-1.12.0.zip", hash = "sha256:7f9b1ae7d97ea7af3f38dd09305e19ab81a1e16ab66ea186b6579d85c1ca2347"}, - {file = "azure_identity-1.12.0-py3-none-any.whl", hash = "sha256:2a58ce4a209a013e37eaccfd5937570ab99e9118b3e1acf875eed3a85d541b92"}, -] -azure-storage-blob = [ - {file = "azure-storage-blob-12.14.1.zip", hash = "sha256:860d4d82985a4bfc7d3271e71275af330f54f330a754355435a7ba749ccde997"}, - {file = "azure_storage_blob-12.14.1-py3-none-any.whl", hash = "sha256:52b84658e8df7853a3cf1c563814655b5028b979b2a87905b92aa6bb30be240e"}, -] -boto3 = [ - {file = "boto3-1.24.59-py3-none-any.whl", hash = "sha256:34ab44146a2c4e7f4e72737f4b27e6eb5e0a7855c2f4599e3d9199b6a0a2d575"}, - {file = "boto3-1.24.59.tar.gz", hash = "sha256:a50b4323f9579cfe22fcf5531fbd40b567d4d74c1adce06aeb5c95fce2a6fb40"}, -] -botocore = [ - {file = "botocore-1.27.59-py3-none-any.whl", hash = "sha256:69d756791fc024bda54f6c53f71ae34e695ee41bbbc1743d9179c4837a4929da"}, - {file = "botocore-1.27.59.tar.gz", hash = "sha256:eda4aed6ee719a745d1288eaf1beb12f6f6448ad1fa12f159405db14ba9c92cf"}, -] -build = [ - {file = "build-0.9.0-py3-none-any.whl", hash = "sha256:38a7a2b7a0bdc61a42a0a67509d88c71ecfc37b393baba770fae34e20929ff69"}, - {file = "build-0.9.0.tar.gz", hash = "sha256:1a07724e891cbd898923145eb7752ee7653674c511378eb9c7691aab1612bc3c"}, -] -certifi = [ - {file = "certifi-2022.12.7-py3-none-any.whl", hash = "sha256:4ad3232f5e926d6718ec31cfc1fcadfde020920e278684144551c91769c7bc18"}, - {file = "certifi-2022.12.7.tar.gz", hash = "sha256:35824b4c3a97115964b408844d64aa14db1cc518f6562e8d7261699d1350a9e3"}, -] -cffi = [ - {file = "cffi-1.15.1-cp27-cp27m-macosx_10_9_x86_64.whl", hash = "sha256:a66d3508133af6e8548451b25058d5812812ec3798c886bf38ed24a98216fab2"}, - {file = "cffi-1.15.1-cp27-cp27m-manylinux1_i686.whl", hash = "sha256:470c103ae716238bbe698d67ad020e1db9d9dba34fa5a899b5e21577e6d52ed2"}, - {file = "cffi-1.15.1-cp27-cp27m-manylinux1_x86_64.whl", hash = "sha256:9ad5db27f9cabae298d151c85cf2bad1d359a1b9c686a275df03385758e2f914"}, - {file = "cffi-1.15.1-cp27-cp27m-win32.whl", hash = "sha256:b3bbeb01c2b273cca1e1e0c5df57f12dce9a4dd331b4fa1635b8bec26350bde3"}, - {file = "cffi-1.15.1-cp27-cp27m-win_amd64.whl", hash = "sha256:e00b098126fd45523dd056d2efba6c5a63b71ffe9f2bbe1a4fe1716e1d0c331e"}, - {file = "cffi-1.15.1-cp27-cp27mu-manylinux1_i686.whl", hash = "sha256:d61f4695e6c866a23a21acab0509af1cdfd2c013cf256bbf5b6b5e2695827162"}, - {file = "cffi-1.15.1-cp27-cp27mu-manylinux1_x86_64.whl", hash = "sha256:ed9cb427ba5504c1dc15ede7d516b84757c3e3d7868ccc85121d9310d27eed0b"}, - {file = "cffi-1.15.1-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:39d39875251ca8f612b6f33e6b1195af86d1b3e60086068be9cc053aa4376e21"}, - {file = "cffi-1.15.1-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:285d29981935eb726a4399badae8f0ffdff4f5050eaa6d0cfc3f64b857b77185"}, - {file = "cffi-1.15.1-cp310-cp310-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:3eb6971dcff08619f8d91607cfc726518b6fa2a9eba42856be181c6d0d9515fd"}, - {file = "cffi-1.15.1-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:21157295583fe8943475029ed5abdcf71eb3911894724e360acff1d61c1d54bc"}, - {file = "cffi-1.15.1-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:5635bd9cb9731e6d4a1132a498dd34f764034a8ce60cef4f5319c0541159392f"}, - {file = "cffi-1.15.1-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:2012c72d854c2d03e45d06ae57f40d78e5770d252f195b93f581acf3ba44496e"}, - {file = "cffi-1.15.1-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:dd86c085fae2efd48ac91dd7ccffcfc0571387fe1193d33b6394db7ef31fe2a4"}, - {file = "cffi-1.15.1-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:fa6693661a4c91757f4412306191b6dc88c1703f780c8234035eac011922bc01"}, - {file = "cffi-1.15.1-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:59c0b02d0a6c384d453fece7566d1c7e6b7bae4fc5874ef2ef46d56776d61c9e"}, - {file = "cffi-1.15.1-cp310-cp310-win32.whl", hash = "sha256:cba9d6b9a7d64d4bd46167096fc9d2f835e25d7e4c121fb2ddfc6528fb0413b2"}, - {file = "cffi-1.15.1-cp310-cp310-win_amd64.whl", hash = "sha256:ce4bcc037df4fc5e3d184794f27bdaab018943698f4ca31630bc7f84a7b69c6d"}, - {file = "cffi-1.15.1-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:3d08afd128ddaa624a48cf2b859afef385b720bb4b43df214f85616922e6a5ac"}, - {file = "cffi-1.15.1-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:3799aecf2e17cf585d977b780ce79ff0dc9b78d799fc694221ce814c2c19db83"}, - {file = "cffi-1.15.1-cp311-cp311-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:a591fe9e525846e4d154205572a029f653ada1a78b93697f3b5a8f1f2bc055b9"}, - {file = "cffi-1.15.1-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:3548db281cd7d2561c9ad9984681c95f7b0e38881201e157833a2342c30d5e8c"}, - {file = "cffi-1.15.1-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:91fc98adde3d7881af9b59ed0294046f3806221863722ba7d8d120c575314325"}, - {file = "cffi-1.15.1-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:94411f22c3985acaec6f83c6df553f2dbe17b698cc7f8ae751ff2237d96b9e3c"}, - {file = "cffi-1.15.1-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:03425bdae262c76aad70202debd780501fabeaca237cdfddc008987c0e0f59ef"}, - {file = "cffi-1.15.1-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:cc4d65aeeaa04136a12677d3dd0b1c0c94dc43abac5860ab33cceb42b801c1e8"}, - {file = "cffi-1.15.1-cp311-cp311-win32.whl", hash = "sha256:a0f100c8912c114ff53e1202d0078b425bee3649ae34d7b070e9697f93c5d52d"}, - {file = "cffi-1.15.1-cp311-cp311-win_amd64.whl", hash = "sha256:04ed324bda3cda42b9b695d51bb7d54b680b9719cfab04227cdd1e04e5de3104"}, - {file = "cffi-1.15.1-cp36-cp36m-macosx_10_9_x86_64.whl", hash = "sha256:50a74364d85fd319352182ef59c5c790484a336f6db772c1a9231f1c3ed0cbd7"}, - {file = "cffi-1.15.1-cp36-cp36m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:e263d77ee3dd201c3a142934a086a4450861778baaeeb45db4591ef65550b0a6"}, - {file = "cffi-1.15.1-cp36-cp36m-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:cec7d9412a9102bdc577382c3929b337320c4c4c4849f2c5cdd14d7368c5562d"}, - {file = "cffi-1.15.1-cp36-cp36m-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:4289fc34b2f5316fbb762d75362931e351941fa95fa18789191b33fc4cf9504a"}, - {file = "cffi-1.15.1-cp36-cp36m-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:173379135477dc8cac4bc58f45db08ab45d228b3363adb7af79436135d028405"}, - {file = "cffi-1.15.1-cp36-cp36m-manylinux_2_5_x86_64.manylinux1_x86_64.whl", hash = "sha256:6975a3fac6bc83c4a65c9f9fcab9e47019a11d3d2cf7f3c0d03431bf145a941e"}, - {file = "cffi-1.15.1-cp36-cp36m-win32.whl", hash = "sha256:2470043b93ff09bf8fb1d46d1cb756ce6132c54826661a32d4e4d132e1977adf"}, - {file = "cffi-1.15.1-cp36-cp36m-win_amd64.whl", hash = "sha256:30d78fbc8ebf9c92c9b7823ee18eb92f2e6ef79b45ac84db507f52fbe3ec4497"}, - {file = "cffi-1.15.1-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:198caafb44239b60e252492445da556afafc7d1e3ab7a1fb3f0584ef6d742375"}, - {file = "cffi-1.15.1-cp37-cp37m-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:5ef34d190326c3b1f822a5b7a45f6c4535e2f47ed06fec77d3d799c450b2651e"}, - {file = "cffi-1.15.1-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:8102eaf27e1e448db915d08afa8b41d6c7ca7a04b7d73af6514df10a3e74bd82"}, - {file = "cffi-1.15.1-cp37-cp37m-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:5df2768244d19ab7f60546d0c7c63ce1581f7af8b5de3eb3004b9b6fc8a9f84b"}, - {file = "cffi-1.15.1-cp37-cp37m-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:a8c4917bd7ad33e8eb21e9a5bbba979b49d9a97acb3a803092cbc1133e20343c"}, - {file = "cffi-1.15.1-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:0e2642fe3142e4cc4af0799748233ad6da94c62a8bec3a6648bf8ee68b1c7426"}, - {file = "cffi-1.15.1-cp37-cp37m-win32.whl", hash = "sha256:e229a521186c75c8ad9490854fd8bbdd9a0c9aa3a524326b55be83b54d4e0ad9"}, - {file = "cffi-1.15.1-cp37-cp37m-win_amd64.whl", hash = "sha256:a0b71b1b8fbf2b96e41c4d990244165e2c9be83d54962a9a1d118fd8657d2045"}, - {file = "cffi-1.15.1-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:320dab6e7cb2eacdf0e658569d2575c4dad258c0fcc794f46215e1e39f90f2c3"}, - {file = "cffi-1.15.1-cp38-cp38-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:1e74c6b51a9ed6589199c787bf5f9875612ca4a8a0785fb2d4a84429badaf22a"}, - {file = "cffi-1.15.1-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a5c84c68147988265e60416b57fc83425a78058853509c1b0629c180094904a5"}, - {file = "cffi-1.15.1-cp38-cp38-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:3b926aa83d1edb5aa5b427b4053dc420ec295a08e40911296b9eb1b6170f6cca"}, - {file = "cffi-1.15.1-cp38-cp38-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:87c450779d0914f2861b8526e035c5e6da0a3199d8f1add1a665e1cbc6fc6d02"}, - {file = "cffi-1.15.1-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:4f2c9f67e9821cad2e5f480bc8d83b8742896f1242dba247911072d4fa94c192"}, - {file = "cffi-1.15.1-cp38-cp38-win32.whl", hash = "sha256:8b7ee99e510d7b66cdb6c593f21c043c248537a32e0bedf02e01e9553a172314"}, - {file = "cffi-1.15.1-cp38-cp38-win_amd64.whl", hash = "sha256:00a9ed42e88df81ffae7a8ab6d9356b371399b91dbdf0c3cb1e84c03a13aceb5"}, - {file = "cffi-1.15.1-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:54a2db7b78338edd780e7ef7f9f6c442500fb0d41a5a4ea24fff1c929d5af585"}, - {file = "cffi-1.15.1-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:fcd131dd944808b5bdb38e6f5b53013c5aa4f334c5cad0c72742f6eba4b73db0"}, - {file = "cffi-1.15.1-cp39-cp39-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:7473e861101c9e72452f9bf8acb984947aa1661a7704553a9f6e4baa5ba64415"}, - {file = "cffi-1.15.1-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:6c9a799e985904922a4d207a94eae35c78ebae90e128f0c4e521ce339396be9d"}, - {file = "cffi-1.15.1-cp39-cp39-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:3bcde07039e586f91b45c88f8583ea7cf7a0770df3a1649627bf598332cb6984"}, - {file = "cffi-1.15.1-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:33ab79603146aace82c2427da5ca6e58f2b3f2fb5da893ceac0c42218a40be35"}, - {file = "cffi-1.15.1-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:5d598b938678ebf3c67377cdd45e09d431369c3b1a5b331058c338e201f12b27"}, - {file = "cffi-1.15.1-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:db0fbb9c62743ce59a9ff687eb5f4afbe77e5e8403d6697f7446e5f609976f76"}, - {file = "cffi-1.15.1-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:98d85c6a2bef81588d9227dde12db8a7f47f639f4a17c9ae08e773aa9c697bf3"}, - {file = "cffi-1.15.1-cp39-cp39-win32.whl", hash = "sha256:40f4774f5a9d4f5e344f31a32b5096977b5d48560c5592e2f3d2c4374bd543ee"}, - {file = "cffi-1.15.1-cp39-cp39-win_amd64.whl", hash = "sha256:70df4e3b545a17496c9b3f41f5115e69a4f2e77e94e1d2a8e1070bc0c38c8a3c"}, - {file = "cffi-1.15.1.tar.gz", hash = "sha256:d400bfb9a37b1351253cb402671cea7e89bdecc294e8016a707f6d1d8ac934f9"}, -] -cfgv = [ - {file = "cfgv-3.3.1-py2.py3-none-any.whl", hash = "sha256:c6a0883f3917a037485059700b9e75da2464e6c27051014ad85ba6aaa5884426"}, - {file = "cfgv-3.3.1.tar.gz", hash = "sha256:f5a830efb9ce7a445376bb66ec94c638a9787422f96264c98edc6bdeed8ab736"}, -] -charset-normalizer = [ - {file = "charset-normalizer-2.1.1.tar.gz", hash = "sha256:5a3d016c7c547f69d6f81fb0db9449ce888b418b5b9952cc5e6e66843e9dd845"}, - {file = "charset_normalizer-2.1.1-py3-none-any.whl", hash = "sha256:83e9a75d1911279afd89352c68b45348559d1fc0506b054b346651b5e7fee29f"}, -] -click = [ - {file = "click-8.1.3-py3-none-any.whl", hash = "sha256:bb4d8133cb15a609f44e8213d9b391b0809795062913b383c62be0ee95b1db48"}, - {file = "click-8.1.3.tar.gz", hash = "sha256:7682dc8afb30297001674575ea00d1814d808d6a36af415a82bd481d37ba7b8e"}, -] -colorama = [ - {file = "colorama-0.4.6-py2.py3-none-any.whl", hash = "sha256:4f1d9991f5acc0ca119f9d443620b77f9d6b33703e51011c16baf57afb285fc6"}, - {file = "colorama-0.4.6.tar.gz", hash = "sha256:08695f5cb7ed6e0531a20572697297273c47b8cae5a63ffc6d6ed5c201be6e44"}, -] -commonmark = [ - {file = "commonmark-0.9.1-py2.py3-none-any.whl", hash = "sha256:da2f38c92590f83de410ba1a3cbceafbc74fee9def35f9251ba9a971d6d66fd9"}, - {file = "commonmark-0.9.1.tar.gz", hash = "sha256:452f9dc859be7f06631ddcb328b6919c67984aca654e5fefb3914d54691aed60"}, -] -coverage = [ +files = [ {file = "coverage-7.0.4-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:daf91db39324e9939a9db919ee4fb42a1a23634a056616dae891a030e89f87ba"}, {file = "coverage-7.0.4-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:55121fe140d7e42cb970999b93cf1c2b24484ce028b32bbd00238bb25c13e34a"}, {file = "coverage-7.0.4-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:c027fbb83a8c78a6e06a0302ea1799fdb70e5cda9845a5e000545b8e2b47ea39"}, @@ -1482,7 +587,21 @@ coverage = [ {file = "coverage-7.0.4-pp37.pp38.pp39-none-any.whl", hash = "sha256:cb8cfa3bf3a9f18211279458917fef5edeb5e1fdebe2ea8b11969ec2ebe48884"}, {file = "coverage-7.0.4.tar.gz", hash = "sha256:f6c4ad409a0caf7e2e12e203348b1a9b19c514e7d078520973147bf2d3dcbc6f"}, ] -cryptography = [ + +[package.dependencies] +tomli = {version = "*", optional = true, markers = "python_full_version <= \"3.11.0a6\" and extra == \"toml\""} + +[package.extras] +toml = ["tomli"] + +[[package]] +name = "cryptography" +version = "38.0.4" +description = "cryptography is a package which provides cryptographic recipes and primitives to Python developers." +category = "main" +optional = false +python-versions = ">=3.6" +files = [ {file = "cryptography-38.0.4-cp36-abi3-macosx_10_10_universal2.whl", hash = "sha256:2fa36a7b2cc0998a3a4d5af26ccb6273f3df133d61da2ba13b3286261e7efb70"}, {file = "cryptography-38.0.4-cp36-abi3-macosx_10_10_x86_64.whl", hash = "sha256:1f13ddda26a04c06eb57119caf27a524ccae20533729f4b1e4a69b54e07035eb"}, {file = "cryptography-38.0.4-cp36-abi3-manylinux_2_17_aarch64.manylinux2014_aarch64.manylinux_2_24_aarch64.whl", hash = "sha256:2ec2a8714dd005949d4019195d72abed84198d877112abb5a27740e217e0ea8d"}, @@ -1510,15 +629,50 @@ cryptography = [ {file = "cryptography-38.0.4-pp39-pypy39_pp73-win_amd64.whl", hash = "sha256:80ca53981ceeb3241998443c4964a387771588c4e4a5d92735a493af868294f9"}, {file = "cryptography-38.0.4.tar.gz", hash = "sha256:175c1a818b87c9ac80bb7377f5520b7f31b3ef2a0004e2420319beadedb67290"}, ] -distlib = [ + +[package.dependencies] +cffi = ">=1.12" + +[package.extras] +docs = ["sphinx (>=1.6.5,!=1.8.0,!=3.1.0,!=3.1.1)", "sphinx-rtd-theme"] +docstest = ["pyenchant (>=1.6.11)", "sphinxcontrib-spelling (>=4.0.1)", "twine (>=1.12.0)"] +pep8test = ["black", "flake8", "flake8-import-order", "pep8-naming"] +sdist = ["setuptools-rust (>=0.11.4)"] +ssh = ["bcrypt (>=3.1.5)"] +test = ["hypothesis (>=1.11.4,!=3.79.2)", "iso8601", "pretend", "pytest (>=6.2.0)", "pytest-benchmark", "pytest-cov", "pytest-subtests", "pytest-xdist", "pytz"] + +[[package]] +name = "distlib" +version = "0.3.6" +description = "Distribution utilities" +category = "dev" +optional = false +python-versions = "*" +files = [ {file = "distlib-0.3.6-py2.py3-none-any.whl", hash = "sha256:f35c4b692542ca110de7ef0bea44d73981caeb34ca0b9b6b2e6d7790dda8f80e"}, {file = "distlib-0.3.6.tar.gz", hash = "sha256:14bad2d9b04d3a36127ac97f30b12a19268f211063d8f8ee4f47108896e11b46"}, ] -docutils = [ + +[[package]] +name = "docutils" +version = "0.19" +description = "Docutils -- Python Documentation Utilities" +category = "dev" +optional = false +python-versions = ">=3.7" +files = [ {file = "docutils-0.19-py3-none-any.whl", hash = "sha256:5e1de4d849fee02c63b040a4a3fd567f4ab104defd8a5511fbbc24a8a017efbc"}, {file = "docutils-0.19.tar.gz", hash = "sha256:33995a6753c30b7f577febfc2c50411fec6aac7f7ffeb7c4cfe5991072dcf9e6"}, ] -duckdb = [ + +[[package]] +name = "duckdb" +version = "0.6.1" +description = "DuckDB embedded database" +category = "main" +optional = true +python-versions = "*" +files = [ {file = "duckdb-0.6.1-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:e566514f9327f89264e98ac14ee7a84fbd9857328028258422c3e8375ee19d25"}, {file = "duckdb-0.6.1-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:b31c2883de5b19591a2852165e6b3f9821f77af649835f27bc146b26e4aa30cb"}, {file = "duckdb-0.6.1-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:998165b2fb1f1d2b0ad742096015ea70878f7d40304643c7424c3ed3ddf07bfc"}, @@ -1567,11 +721,33 @@ duckdb = [ {file = "duckdb-0.6.1-cp39-cp39-win_amd64.whl", hash = "sha256:00b7be8f67ec1a8edaa8844f521267baa1a795f4c482bfad56c72c26e1862ab2"}, {file = "duckdb-0.6.1.tar.gz", hash = "sha256:6d26e9f1afcb924a6057785e506810d48332d4764ddc4a5b414d0f2bf0cacfb4"}, ] -exceptiongroup = [ + +[package.dependencies] +numpy = ">=1.14" + +[[package]] +name = "exceptiongroup" +version = "1.0.4" +description = "Backport of PEP 654 (exception groups)" +category = "dev" +optional = false +python-versions = ">=3.7" +files = [ {file = "exceptiongroup-1.0.4-py3-none-any.whl", hash = "sha256:542adf9dea4055530d6e1279602fa5cb11dab2395fa650b8674eaec35fc4a828"}, {file = "exceptiongroup-1.0.4.tar.gz", hash = "sha256:bd14967b79cd9bdb54d97323216f8fdf533e278df937aa2a90089e7d6e06e5ec"}, ] -fastavro = [ + +[package.extras] +test = ["pytest (>=6)"] + +[[package]] +name = "fastavro" +version = "1.7.0" +description = "Fast read/write of AVRO files" +category = "dev" +optional = false +python-versions = ">=3.7" +files = [ {file = "fastavro-1.7.0-cp310-cp310-macosx_10_14_x86_64.whl", hash = "sha256:ab3387a06e272980fa034f5c62f7063977b77df6416d3d30a4d3b49cc8827566"}, {file = "fastavro-1.7.0-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:216132bc54da19e97e1531dd69c86282408d4c797749d83b01b3a00862a180de"}, {file = "fastavro-1.7.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:c20cf6cd8098bb93c2cffd53d03ccea1dcf9ec594a5c83963acf29a2882f8693"}, @@ -1594,11 +770,37 @@ fastavro = [ {file = "fastavro-1.7.0-cp39-cp39-win_amd64.whl", hash = "sha256:492e8902063aa1c73170e11c18495fcaa86b71eae3513ef83ba438ca02b16b34"}, {file = "fastavro-1.7.0.tar.gz", hash = "sha256:4b1205f46489b4032d3155c1ab44d9824be0c7454df98d3a5bd22b78b98f23c8"}, ] -filelock = [ + +[package.extras] +codecs = ["lz4", "python-snappy", "zstandard"] +lz4 = ["lz4"] +snappy = ["python-snappy"] +zstandard = ["zstandard"] + +[[package]] +name = "filelock" +version = "3.8.2" +description = "A platform independent file lock." +category = "dev" +optional = false +python-versions = ">=3.7" +files = [ {file = "filelock-3.8.2-py3-none-any.whl", hash = "sha256:8df285554452285f79c035efb0c861eb33a4bcfa5b7a137016e32e6a90f9792c"}, {file = "filelock-3.8.2.tar.gz", hash = "sha256:7565f628ea56bfcd8e54e42bdc55da899c85c1abfe1b5bcfd147e9188cebb3b2"}, ] -frozenlist = [ + +[package.extras] +docs = ["furo (>=2022.9.29)", "sphinx (>=5.3)", "sphinx-autodoc-typehints (>=1.19.5)"] +testing = ["covdefaults (>=2.2.2)", "coverage (>=6.5)", "pytest (>=7.2)", "pytest-cov (>=4)", "pytest-timeout (>=2.1)"] + +[[package]] +name = "frozenlist" +version = "1.3.3" +description = "A list-like structure which implements collections.abc.MutableSequence" +category = "main" +optional = true +python-versions = ">=3.7" +files = [ {file = "frozenlist-1.3.3-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:ff8bf625fe85e119553b5383ba0fb6aa3d0ec2ae980295aaefa552374926b3f4"}, {file = "frozenlist-1.3.3-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:dfbac4c2dfcc082fcf8d942d1e49b6aa0766c19d3358bd86e2000bf0fa4a9cf0"}, {file = "frozenlist-1.3.3-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:b1c63e8d377d039ac769cd0926558bb7068a1f7abb0f003e3717ee003ad85530"}, @@ -1674,39 +876,154 @@ frozenlist = [ {file = "frozenlist-1.3.3-cp39-cp39-win_amd64.whl", hash = "sha256:cfe33efc9cb900a4c46f91a5ceba26d6df370ffddd9ca386eb1d4f0ad97b9ea9"}, {file = "frozenlist-1.3.3.tar.gz", hash = "sha256:58bcc55721e8a90b88332d6cd441261ebb22342e238296bb330968952fbb3a6a"}, ] -fsspec = [ + +[[package]] +name = "fsspec" +version = "2022.10.0" +description = "File-system specification" +category = "main" +optional = false +python-versions = ">=3.7" +files = [ {file = "fsspec-2022.10.0-py3-none-any.whl", hash = "sha256:6b7c6ab3b476cdf17efcfeccde7fca28ef5a48f73a71010aaceec5fc15bf9ebf"}, {file = "fsspec-2022.10.0.tar.gz", hash = "sha256:cb6092474e90487a51de768170f3afa50ca8982c26150a59072b16433879ff1d"}, ] -identify = [ + +[package.extras] +abfs = ["adlfs"] +adl = ["adlfs"] +arrow = ["pyarrow (>=1)"] +dask = ["dask", "distributed"] +dropbox = ["dropbox", "dropboxdrivefs", "requests"] +entrypoints = ["importlib-metadata"] +fuse = ["fusepy"] +gcs = ["gcsfs"] +git = ["pygit2"] +github = ["requests"] +gs = ["gcsfs"] +gui = ["panel"] +hdfs = ["pyarrow (>=1)"] +http = ["aiohttp (!=4.0.0a0,!=4.0.0a1)", "requests"] +libarchive = ["libarchive-c"] +oci = ["ocifs"] +s3 = ["s3fs"] +sftp = ["paramiko"] +smb = ["smbprotocol"] +ssh = ["paramiko"] +tqdm = ["tqdm"] + +[[package]] +name = "identify" +version = "2.5.10" +description = "File identification library for Python" +category = "dev" +optional = false +python-versions = ">=3.7" +files = [ {file = "identify-2.5.10-py2.py3-none-any.whl", hash = "sha256:fb7c2feaeca6976a3ffa31ec3236a6911fbc51aec9acc111de2aed99f244ade2"}, {file = "identify-2.5.10.tar.gz", hash = "sha256:dce9e31fee7dbc45fea36a9e855c316b8fbf807e65a862f160840bb5a2bf5dfd"}, ] -idna = [ + +[package.extras] +license = ["ukkonen"] + +[[package]] +name = "idna" +version = "3.4" +description = "Internationalized Domain Names in Applications (IDNA)" +category = "main" +optional = false +python-versions = ">=3.5" +files = [ {file = "idna-3.4-py3-none-any.whl", hash = "sha256:90b77e79eaa3eba6de819a0c442c0b4ceefc341a7a2ab77d7562bf49f425c5c2"}, {file = "idna-3.4.tar.gz", hash = "sha256:814f528e8dead7d329833b91c5faa87d60bf71824cd12a7530b5526063d02cb4"}, ] -importlib-metadata = [ + +[[package]] +name = "importlib-metadata" +version = "5.1.0" +description = "Read metadata from Python packages" +category = "dev" +optional = false +python-versions = ">=3.7" +files = [ {file = "importlib_metadata-5.1.0-py3-none-any.whl", hash = "sha256:d84d17e21670ec07990e1044a99efe8d615d860fd176fc29ef5c306068fda313"}, {file = "importlib_metadata-5.1.0.tar.gz", hash = "sha256:d5059f9f1e8e41f80e9c56c2ee58811450c31984dfa625329ffd7c0dad88a73b"}, ] -iniconfig = [ + +[package.dependencies] +zipp = ">=0.5" + +[package.extras] +docs = ["furo", "jaraco.packaging (>=9)", "jaraco.tidelift (>=1.4)", "rst.linker (>=1.9)", "sphinx (>=3.5)"] +perf = ["ipython"] +testing = ["flake8 (<5)", "flufl.flake8", "importlib-resources (>=1.3)", "packaging", "pyfakefs", "pytest (>=6)", "pytest-black (>=0.3.7)", "pytest-checkdocs (>=2.4)", "pytest-cov", "pytest-enabler (>=1.3)", "pytest-flake8", "pytest-mypy (>=0.9.1)", "pytest-perf (>=0.9.2)"] + +[[package]] +name = "iniconfig" +version = "1.1.1" +description = "iniconfig: brain-dead simple config-ini parsing" +category = "dev" +optional = false +python-versions = "*" +files = [ {file = "iniconfig-1.1.1-py2.py3-none-any.whl", hash = "sha256:011e24c64b7f47f6ebd835bb12a743f2fbe9a26d4cecaa7f53bc4f35ee9da8b3"}, {file = "iniconfig-1.1.1.tar.gz", hash = "sha256:bc3af051d7d14b2ee5ef9969666def0cd1a000e121eaea580d4a313df4b37f32"}, ] -isodate = [ + +[[package]] +name = "isodate" +version = "0.6.1" +description = "An ISO 8601 date/time/duration parser and formatter" +category = "main" +optional = true +python-versions = "*" +files = [ {file = "isodate-0.6.1-py2.py3-none-any.whl", hash = "sha256:0751eece944162659049d35f4f549ed815792b38793f07cf73381c1c87cbed96"}, {file = "isodate-0.6.1.tar.gz", hash = "sha256:48c5881de7e8b0a0d648cb024c8062dc84e7b840ed81e864c7614fd3c127bde9"}, ] -jinja2 = [ + +[package.dependencies] +six = "*" + +[[package]] +name = "jinja2" +version = "3.1.2" +description = "A very fast and expressive template engine." +category = "dev" +optional = false +python-versions = ">=3.7" +files = [ {file = "Jinja2-3.1.2-py3-none-any.whl", hash = "sha256:6088930bfe239f0e6710546ab9c19c9ef35e29792895fed6e6e31a023a182a61"}, {file = "Jinja2-3.1.2.tar.gz", hash = "sha256:31351a702a408a9e7595a8fc6150fc3f43bb6bf7e319770cbc0db9df9437e852"}, ] -jmespath = [ + +[package.dependencies] +MarkupSafe = ">=2.0" + +[package.extras] +i18n = ["Babel (>=2.7)"] + +[[package]] +name = "jmespath" +version = "1.0.1" +description = "JSON Matching Expressions" +category = "main" +optional = false +python-versions = ">=3.7" +files = [ {file = "jmespath-1.0.1-py3-none-any.whl", hash = "sha256:02e2e4cc71b5bcab88332eebf907519190dd9e6e82107fa7f83b1003a6252980"}, {file = "jmespath-1.0.1.tar.gz", hash = "sha256:90261b206d6defd58fdd5e85f478bf633a2901798906be2ad389150c5c60edbe"}, ] -markupsafe = [ + +[[package]] +name = "markupsafe" +version = "2.1.1" +description = "Safely add untrusted strings to HTML/XML markup." +category = "dev" +optional = false +python-versions = ">=3.7" +files = [ {file = "MarkupSafe-2.1.1-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:86b1f75c4e7c2ac2ccdaec2b9022845dbb81880ca318bb7a0a01fbf7813e3812"}, {file = "MarkupSafe-2.1.1-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:f121a1420d4e173a5d96e47e9a0c0dcff965afdf1626d28de1460815f7c4ee7a"}, {file = "MarkupSafe-2.1.1-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a49907dd8420c5685cfa064a1335b6754b74541bbb3706c259c02ed65b644b3e"}, @@ -1748,7 +1065,15 @@ markupsafe = [ {file = "MarkupSafe-2.1.1-cp39-cp39-win_amd64.whl", hash = "sha256:46d00d6cfecdde84d40e572d63735ef81423ad31184100411e6e3388d405e247"}, {file = "MarkupSafe-2.1.1.tar.gz", hash = "sha256:7f91197cc9e48f989d12e4e6fbc46495c446636dfc81b9ccf50bb0ec74b91d4b"}, ] -mmhash3 = [ + +[[package]] +name = "mmhash3" +version = "3.0.1" +description = "Python wrapper for MurmurHash (MurmurHash3), a set of fast and robust hash functions." +category = "main" +optional = false +python-versions = "*" +files = [ {file = "mmhash3-3.0.1-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:47deea30cd8d3d5cd52dc740902a4c70383bfe8248eac29d0877fe63e03c2713"}, {file = "mmhash3-3.0.1-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:ecdaf4d1de617818bf05cd526ca558db6010beeba7ea9e19f695f2bdcac0e0a4"}, {file = "mmhash3-3.0.1-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:4675585617584e9e1aafa3a90ac0ac9a437257c507748d97de8b21977e9d6745"}, @@ -1784,23 +1109,124 @@ mmhash3 = [ {file = "mmhash3-3.0.1-cp39-cp39-win_amd64.whl", hash = "sha256:0baeaa20cac5f75ed14f28056826bd9d9c8b2354b382073f3fd5190708992a0d"}, {file = "mmhash3-3.0.1.tar.gz", hash = "sha256:a00d68f4a1cc434b9501513c8a29e18ed1ddad383677d72b41d71d0d862348af"}, ] -moto = [ + +[[package]] +name = "moto" +version = "4.0.13" +description = "A library that allows your python tests to easily mock out the boto library" +category = "dev" +optional = false +python-versions = ">=3.6" +files = [ {file = "moto-4.0.13-py3-none-any.whl", hash = "sha256:e73400c6d3fe06028aa7f07bb6f276f14260d289b70f38928a98e3d3d968352d"}, {file = "moto-4.0.13.tar.gz", hash = "sha256:baf7d6969cf837990c730e6e648315bebc2e1c0038d9d8fc4f59d03561484469"}, ] -msal = [ + +[package.dependencies] +boto3 = ">=1.9.201" +botocore = ">=1.12.201" +cryptography = ">=3.3.1" +Jinja2 = ">=2.10.1" +MarkupSafe = "!=2.0.0a1" +python-dateutil = ">=2.1,<3.0.0" +requests = ">=2.5" +responses = ">=0.13.0" +werkzeug = ">=0.5,<2.2.0 || >2.2.0,<2.2.1 || >2.2.1" +xmltodict = "*" + +[package.extras] +all = ["PyYAML (>=5.1)", "aws-xray-sdk (>=0.93,!=0.96)", "cfn-lint (>=0.40.0)", "docker (>=2.5.1)", "ecdsa (!=0.15)", "graphql-core", "idna (>=2.5,<4)", "jsondiff (>=1.1.2)", "openapi-spec-validator (>=0.2.8)", "pyparsing (>=3.0.7)", "python-jose[cryptography] (>=3.1.0,<4.0.0)", "setuptools", "sshpubkeys (>=3.1.0)"] +apigateway = ["PyYAML (>=5.1)", "ecdsa (!=0.15)", "openapi-spec-validator (>=0.2.8)", "python-jose[cryptography] (>=3.1.0,<4.0.0)"] +apigatewayv2 = ["PyYAML (>=5.1)"] +appsync = ["graphql-core"] +awslambda = ["docker (>=2.5.1)"] +batch = ["docker (>=2.5.1)"] +cloudformation = ["PyYAML (>=5.1)", "aws-xray-sdk (>=0.93,!=0.96)", "cfn-lint (>=0.40.0)", "docker (>=2.5.1)", "ecdsa (!=0.15)", "graphql-core", "idna (>=2.5,<4)", "jsondiff (>=1.1.2)", "openapi-spec-validator (>=0.2.8)", "pyparsing (>=3.0.7)", "python-jose[cryptography] (>=3.1.0,<4.0.0)", "setuptools", "sshpubkeys (>=3.1.0)"] +cognitoidp = ["ecdsa (!=0.15)", "python-jose[cryptography] (>=3.1.0,<4.0.0)"] +ds = ["sshpubkeys (>=3.1.0)"] +dynamodb = ["docker (>=2.5.1)"] +dynamodb2 = ["docker (>=2.5.1)"] +dynamodbstreams = ["docker (>=2.5.1)"] +ebs = ["sshpubkeys (>=3.1.0)"] +ec2 = ["sshpubkeys (>=3.1.0)"] +efs = ["sshpubkeys (>=3.1.0)"] +glue = ["pyparsing (>=3.0.7)"] +iotdata = ["jsondiff (>=1.1.2)"] +route53resolver = ["sshpubkeys (>=3.1.0)"] +s3 = ["PyYAML (>=5.1)"] +server = ["PyYAML (>=5.1)", "aws-xray-sdk (>=0.93,!=0.96)", "cfn-lint (>=0.40.0)", "docker (>=2.5.1)", "ecdsa (!=0.15)", "flask (!=2.2.0,!=2.2.1)", "flask-cors", "graphql-core", "idna (>=2.5,<4)", "jsondiff (>=1.1.2)", "openapi-spec-validator (>=0.2.8)", "pyparsing (>=3.0.7)", "python-jose[cryptography] (>=3.1.0,<4.0.0)", "setuptools", "sshpubkeys (>=3.1.0)"] +ssm = ["PyYAML (>=5.1)", "dataclasses"] +xray = ["aws-xray-sdk (>=0.93,!=0.96)", "setuptools"] + +[[package]] +name = "msal" +version = "1.20.0" +description = "The Microsoft Authentication Library (MSAL) for Python library enables your app to access the Microsoft Cloud by supporting authentication of users with Microsoft Azure Active Directory accounts (AAD) and Microsoft Accounts (MSA) using industry standard OAuth2 and OpenID Connect." +category = "main" +optional = true +python-versions = "*" +files = [ {file = "msal-1.20.0-py2.py3-none-any.whl", hash = "sha256:d2f1c26368ecdc28c8657d457352faa0b81b1845a7b889d8676787721ba86792"}, {file = "msal-1.20.0.tar.gz", hash = "sha256:78344cd4c91d6134a593b5e3e45541e666e37b747ff8a6316c3668dd1e6ab6b2"}, ] -msal-extensions = [ + +[package.dependencies] +cryptography = ">=0.6,<41" +PyJWT = {version = ">=1.0.0,<3", extras = ["crypto"]} +requests = ">=2.0.0,<3" + +[package.extras] +broker = ["pymsalruntime (>=0.11.2,<0.14)"] + +[[package]] +name = "msal-extensions" +version = "1.0.0" +description = "Microsoft Authentication Library extensions (MSAL EX) provides a persistence API that can save your data on disk, encrypted on Windows, macOS and Linux. Concurrent data access will be coordinated by a file lock mechanism." +category = "main" +optional = true +python-versions = "*" +files = [ {file = "msal-extensions-1.0.0.tar.gz", hash = "sha256:c676aba56b0cce3783de1b5c5ecfe828db998167875126ca4b47dc6436451354"}, {file = "msal_extensions-1.0.0-py2.py3-none-any.whl", hash = "sha256:91e3db9620b822d0ed2b4d1850056a0f133cba04455e62f11612e40f5502f2ee"}, ] -msrest = [ + +[package.dependencies] +msal = ">=0.4.1,<2.0.0" +portalocker = [ + {version = ">=1.0,<3", markers = "python_version >= \"3.5\" and platform_system != \"Windows\""}, + {version = ">=1.6,<3", markers = "python_version >= \"3.5\" and platform_system == \"Windows\""}, +] + +[[package]] +name = "msrest" +version = "0.7.1" +description = "AutoRest swagger generator Python client runtime." +category = "main" +optional = true +python-versions = ">=3.6" +files = [ {file = "msrest-0.7.1-py3-none-any.whl", hash = "sha256:21120a810e1233e5e6cc7fe40b474eeb4ec6f757a15d7cf86702c369f9567c32"}, {file = "msrest-0.7.1.zip", hash = "sha256:6e7661f46f3afd88b75667b7187a92829924446c7ea1d169be8c4bb7eeb788b9"}, ] -multidict = [ + +[package.dependencies] +azure-core = ">=1.24.0" +certifi = ">=2017.4.17" +isodate = ">=0.6.0" +requests = ">=2.16,<3.0" +requests-oauthlib = ">=0.5.0" + +[package.extras] +async = ["aiodns", "aiohttp (>=3.0)"] + +[[package]] +name = "multidict" +version = "6.0.3" +description = "multidict implementation" +category = "main" +optional = true +python-versions = ">=3.7" +files = [ {file = "multidict-6.0.3-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:73009ea04205966d47e16d98686ac5c438af23a1bb30b48a2c5da3423ec9ce37"}, {file = "multidict-6.0.3-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:8b92a9f3ab904397a33b193000dc4de7318ea175c4c460a1e154c415f9008e3d"}, {file = "multidict-6.0.3-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:578bfcb16f4b8675ef71b960c00f174b0426e0eeb796bab6737389d8288eb827"}, @@ -1876,11 +1302,30 @@ multidict = [ {file = "multidict-6.0.3-cp39-cp39-win_amd64.whl", hash = "sha256:5e58ec0375803526d395f6f7e730ecc45d06e15f68f7b9cdbf644a2918324e51"}, {file = "multidict-6.0.3.tar.gz", hash = "sha256:2523a29006c034687eccd3ee70093a697129a3ffe8732535d3b2df6a4ecc279d"}, ] -nodeenv = [ + +[[package]] +name = "nodeenv" +version = "1.7.0" +description = "Node.js virtual environment builder" +category = "dev" +optional = false +python-versions = ">=2.7,!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,!=3.4.*,!=3.5.*,!=3.6.*" +files = [ {file = "nodeenv-1.7.0-py2.py3-none-any.whl", hash = "sha256:27083a7b96a25f2f5e1d8cb4b6317ee8aeda3bdd121394e5ac54e498028a042e"}, {file = "nodeenv-1.7.0.tar.gz", hash = "sha256:e0e7f7dfb85fc5394c6fe1e8fa98131a2473e04311a45afb6508f7cf1836fa2b"}, ] -numpy = [ + +[package.dependencies] +setuptools = "*" + +[[package]] +name = "numpy" +version = "1.23.5" +description = "NumPy is the fundamental package for array computing with Python." +category = "main" +optional = true +python-versions = ">=3.8" +files = [ {file = "numpy-1.23.5-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:9c88793f78fca17da0145455f0d7826bcb9f37da4764af27ac945488116efe63"}, {file = "numpy-1.23.5-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:e9f4c4e51567b616be64e05d517c79a8a22f3606499941d97bb76f2ca59f982d"}, {file = "numpy-1.23.5-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:7903ba8ab592b82014713c491f6c5d3a1cde5b4a3bf116404e08f5b52f6daf43"}, @@ -1910,15 +1355,44 @@ numpy = [ {file = "numpy-1.23.5-pp38-pypy38_pp73-win_amd64.whl", hash = "sha256:01dd17cbb340bf0fc23981e52e1d18a9d4050792e8fb8363cecbf066a84b827d"}, {file = "numpy-1.23.5.tar.gz", hash = "sha256:1b1766d6f397c18153d40015ddfc79ddb715cabadc04d2d228d4e5a8bc4ded1a"}, ] -oauthlib = [ + +[[package]] +name = "oauthlib" +version = "3.2.2" +description = "A generic, spec-compliant, thorough implementation of the OAuth request-signing logic" +category = "main" +optional = true +python-versions = ">=3.6" +files = [ {file = "oauthlib-3.2.2-py3-none-any.whl", hash = "sha256:8139f29aac13e25d502680e9e19963e83f16838d48a0d71c287fe40e7067fbca"}, {file = "oauthlib-3.2.2.tar.gz", hash = "sha256:9859c40929662bec5d64f34d01c99e093149682a3f38915dc0655d5a633dd918"}, ] -packaging = [ + +[package.extras] +rsa = ["cryptography (>=3.0.0)"] +signals = ["blinker (>=1.4.0)"] +signedtoken = ["cryptography (>=3.0.0)", "pyjwt (>=2.0.0,<3)"] + +[[package]] +name = "packaging" +version = "22.0" +description = "Core utilities for Python packages" +category = "dev" +optional = false +python-versions = ">=3.7" +files = [ {file = "packaging-22.0-py3-none-any.whl", hash = "sha256:957e2148ba0e1a3b282772e791ef1d8083648bc131c8ab0c1feba110ce1146c3"}, {file = "packaging-22.0.tar.gz", hash = "sha256:2198ec20bd4c017b8f9717e00f0c8714076fc2fd93816750ab48e2c41de2cfd3"}, ] -pandas = [ + +[[package]] +name = "pandas" +version = "1.5.2" +description = "Powerful data structures for data analysis, time series, and statistics" +category = "main" +optional = true +python-versions = ">=3.8" +files = [ {file = "pandas-1.5.2-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:e9dbacd22555c2d47f262ef96bb4e30880e5956169741400af8b306bbb24a273"}, {file = "pandas-1.5.2-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:e2b83abd292194f350bb04e188f9379d36b8dfac24dd445d5c87575f3beaf789"}, {file = "pandas-1.5.2-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:2552bffc808641c6eb471e55aa6899fa002ac94e4eebfa9ec058649122db5824"}, @@ -1947,27 +1421,113 @@ pandas = [ {file = "pandas-1.5.2-cp39-cp39-win_amd64.whl", hash = "sha256:c218796d59d5abd8780170c937b812c9637e84c32f8271bbf9845970f8c1351f"}, {file = "pandas-1.5.2.tar.gz", hash = "sha256:220b98d15cee0b2cd839a6358bd1f273d0356bf964c1a1aeb32d47db0215488b"}, ] -pep517 = [ + +[package.dependencies] +numpy = [ + {version = ">=1.20.3", markers = "python_version < \"3.10\""}, + {version = ">=1.21.0", markers = "python_version >= \"3.10\""}, + {version = ">=1.23.2", markers = "python_version >= \"3.11\""}, +] +python-dateutil = ">=2.8.1" +pytz = ">=2020.1" + +[package.extras] +test = ["hypothesis (>=5.5.3)", "pytest (>=6.0)", "pytest-xdist (>=1.31)"] + +[[package]] +name = "pep517" +version = "0.13.0" +description = "Wrappers to build Python packages using PEP 517 hooks" +category = "dev" +optional = false +python-versions = ">=3.6" +files = [ {file = "pep517-0.13.0-py3-none-any.whl", hash = "sha256:4ba4446d80aed5b5eac6509ade100bff3e7943a8489de249654a5ae9b33ee35b"}, {file = "pep517-0.13.0.tar.gz", hash = "sha256:ae69927c5c172be1add9203726d4b84cf3ebad1edcd5f71fcdc746e66e829f59"}, ] -platformdirs = [ + +[package.dependencies] +tomli = {version = ">=1.1.0", markers = "python_version < \"3.11\""} + +[[package]] +name = "platformdirs" +version = "2.6.0" +description = "A small Python package for determining appropriate platform-specific dirs, e.g. a \"user data dir\"." +category = "dev" +optional = false +python-versions = ">=3.7" +files = [ {file = "platformdirs-2.6.0-py3-none-any.whl", hash = "sha256:1a89a12377800c81983db6be069ec068eee989748799b946cce2a6e80dcc54ca"}, {file = "platformdirs-2.6.0.tar.gz", hash = "sha256:b46ffafa316e6b83b47489d240ce17173f123a9b9c83282141c3daf26ad9ac2e"}, ] -pluggy = [ + +[package.extras] +docs = ["furo (>=2022.9.29)", "proselint (>=0.13)", "sphinx (>=5.3)", "sphinx-autodoc-typehints (>=1.19.4)"] +test = ["appdirs (==1.4.4)", "pytest (>=7.2)", "pytest-cov (>=4)", "pytest-mock (>=3.10)"] + +[[package]] +name = "pluggy" +version = "1.0.0" +description = "plugin and hook calling mechanisms for python" +category = "dev" +optional = false +python-versions = ">=3.6" +files = [ {file = "pluggy-1.0.0-py2.py3-none-any.whl", hash = "sha256:74134bbf457f031a36d68416e1509f34bd5ccc019f0bcc952c7b909d06b37bd3"}, {file = "pluggy-1.0.0.tar.gz", hash = "sha256:4224373bacce55f955a878bf9cfa763c1e360858e330072059e10bad68531159"}, ] -portalocker = [ + +[package.extras] +dev = ["pre-commit", "tox"] +testing = ["pytest", "pytest-benchmark"] + +[[package]] +name = "portalocker" +version = "2.6.0" +description = "Wraps the portalocker recipe for easy usage" +category = "main" +optional = true +python-versions = ">=3.5" +files = [ {file = "portalocker-2.6.0-py2.py3-none-any.whl", hash = "sha256:102ed1f2badd8dec9af3d732ef70e94b215b85ba45a8d7ff3c0003f19b442f4e"}, {file = "portalocker-2.6.0.tar.gz", hash = "sha256:964f6830fb42a74b5d32bce99ed37d8308c1d7d44ddf18f3dd89f4680de97b39"}, ] -pre-commit = [ + +[package.dependencies] +pywin32 = {version = ">=226", markers = "platform_system == \"Windows\""} + +[package.extras] +docs = ["sphinx (>=1.7.1)"] +redis = ["redis"] +tests = ["pytest (>=5.4.1)", "pytest-cov (>=2.8.1)", "pytest-mypy (>=0.8.0)", "pytest-timeout (>=2.1.0)", "redis", "sphinx (>=3.0.3)"] + +[[package]] +name = "pre-commit" +version = "2.21.0" +description = "A framework for managing and maintaining multi-language pre-commit hooks." +category = "dev" +optional = false +python-versions = ">=3.7" +files = [ {file = "pre_commit-2.21.0-py2.py3-none-any.whl", hash = "sha256:e2f91727039fc39a92f58a588a25b87f936de6567eed4f0e673e0507edc75bad"}, {file = "pre_commit-2.21.0.tar.gz", hash = "sha256:31ef31af7e474a8d8995027fefdfcf509b5c913ff31f2015b4ec4beb26a6f658"}, ] -pyarrow = [ + +[package.dependencies] +cfgv = ">=2.0.0" +identify = ">=1.0.0" +nodeenv = ">=0.11.1" +pyyaml = ">=5.1" +virtualenv = ">=20.10.0" + +[[package]] +name = "pyarrow" +version = "10.0.1" +description = "Python library for Apache Arrow" +category = "main" +optional = true +python-versions = ">=3.7" +files = [ {file = "pyarrow-10.0.1-cp310-cp310-macosx_10_14_x86_64.whl", hash = "sha256:e00174764a8b4e9d8d5909b6d19ee0c217a6cf0232c5682e31fdfbd5a9f0ae52"}, {file = "pyarrow-10.0.1-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:6f7a7dbe2f7f65ac1d0bd3163f756deb478a9e9afc2269557ed75b1b25ab3610"}, {file = "pyarrow-10.0.1-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:cb627673cb98708ef00864e2e243f51ba7b4c1b9f07a1d821f98043eccd3f585"}, @@ -1994,11 +1554,30 @@ pyarrow = [ {file = "pyarrow-10.0.1-cp39-cp39-win_amd64.whl", hash = "sha256:0ec7587d759153f452d5263dbc8b1af318c4609b607be2bd5127dcda6708cdb1"}, {file = "pyarrow-10.0.1.tar.gz", hash = "sha256:1a14f57a5f472ce8234f2964cd5184cccaa8df7e04568c64edc33b23eb285dd5"}, ] -pycparser = [ + +[package.dependencies] +numpy = ">=1.16.6" + +[[package]] +name = "pycparser" +version = "2.21" +description = "C parser in Python" +category = "main" +optional = false +python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*" +files = [ {file = "pycparser-2.21-py2.py3-none-any.whl", hash = "sha256:8ee45429555515e1f6b185e78100aea234072576aa43ab53aefcae078162fca9"}, {file = "pycparser-2.21.tar.gz", hash = "sha256:e644fdec12f7872f86c58ff790da456218b10f863970249516d60a5eaca77206"}, ] -pydantic = [ + +[[package]] +name = "pydantic" +version = "1.10.4" +description = "Data validation and settings management using python type hints" +category = "main" +optional = false +python-versions = ">=3.7" +files = [ {file = "pydantic-1.10.4-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:b5635de53e6686fe7a44b5cf25fcc419a0d5e5c1a1efe73d49d48fe7586db854"}, {file = "pydantic-1.10.4-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:6dc1cc241440ed7ca9ab59d9929075445da6b7c94ced281b3dd4cfe6c8cff817"}, {file = "pydantic-1.10.4-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:51bdeb10d2db0f288e71d49c9cefa609bca271720ecd0c58009bd7504a0c464c"}, @@ -2036,31 +1615,133 @@ pydantic = [ {file = "pydantic-1.10.4-py3-none-any.whl", hash = "sha256:4948f264678c703f3877d1c8877c4e3b2e12e549c57795107f08cf70c6ec7774"}, {file = "pydantic-1.10.4.tar.gz", hash = "sha256:b9a3859f24eb4e097502a3be1fb4b2abb79b6103dd9e2e0edb70613a4459a648"}, ] -pygments = [ + +[package.dependencies] +typing-extensions = ">=4.2.0" + +[package.extras] +dotenv = ["python-dotenv (>=0.10.4)"] +email = ["email-validator (>=1.0.3)"] + +[[package]] +name = "pygments" +version = "2.13.0" +description = "Pygments is a syntax highlighting package written in Python." +category = "main" +optional = false +python-versions = ">=3.6" +files = [ {file = "Pygments-2.13.0-py3-none-any.whl", hash = "sha256:f643f331ab57ba3c9d89212ee4a2dabc6e94f117cf4eefde99a0574720d14c42"}, {file = "Pygments-2.13.0.tar.gz", hash = "sha256:56a8508ae95f98e2b9bdf93a6be5ae3f7d8af858b43e02c5a2ff083726be40c1"}, ] -pyjwt = [ + +[package.extras] +plugins = ["importlib-metadata"] + +[[package]] +name = "pyjwt" +version = "2.6.0" +description = "JSON Web Token implementation in Python" +category = "main" +optional = true +python-versions = ">=3.7" +files = [ {file = "PyJWT-2.6.0-py3-none-any.whl", hash = "sha256:d83c3d892a77bbb74d3e1a2cfa90afaadb60945205d1095d9221f04466f64c14"}, {file = "PyJWT-2.6.0.tar.gz", hash = "sha256:69285c7e31fc44f68a1feb309e948e0df53259d579295e6cfe2b1792329f05fd"}, ] -pyparsing = [ + +[package.dependencies] +cryptography = {version = ">=3.4.0", optional = true, markers = "extra == \"crypto\""} + +[package.extras] +crypto = ["cryptography (>=3.4.0)"] +dev = ["coverage[toml] (==5.0.4)", "cryptography (>=3.4.0)", "pre-commit", "pytest (>=6.0.0,<7.0.0)", "sphinx (>=4.5.0,<5.0.0)", "sphinx-rtd-theme", "zope.interface"] +docs = ["sphinx (>=4.5.0,<5.0.0)", "sphinx-rtd-theme", "zope.interface"] +tests = ["coverage[toml] (==5.0.4)", "pytest (>=6.0.0,<7.0.0)"] + +[[package]] +name = "pyparsing" +version = "3.0.9" +description = "pyparsing module - Classes and methods to define and execute parsing grammars" +category = "main" +optional = false +python-versions = ">=3.6.8" +files = [ {file = "pyparsing-3.0.9-py3-none-any.whl", hash = "sha256:5026bae9a10eeaefb61dab2f09052b9f4307d44aee4eda64b309723d8d206bbc"}, {file = "pyparsing-3.0.9.tar.gz", hash = "sha256:2b020ecf7d21b687f219b71ecad3631f644a47f01403fa1d1036b0c6416d70fb"}, ] -pytest = [ + +[package.extras] +diagrams = ["jinja2", "railroad-diagrams"] + +[[package]] +name = "pytest" +version = "7.2.0" +description = "pytest: simple powerful testing with Python" +category = "dev" +optional = false +python-versions = ">=3.7" +files = [ {file = "pytest-7.2.0-py3-none-any.whl", hash = "sha256:892f933d339f068883b6fd5a459f03d85bfcb355e4981e146d2c7616c21fef71"}, {file = "pytest-7.2.0.tar.gz", hash = "sha256:c4014eb40e10f11f355ad4e3c2fb2c6c6d1919c73f3b5a433de4708202cade59"}, ] -pytest-checkdocs = [ + +[package.dependencies] +attrs = ">=19.2.0" +colorama = {version = "*", markers = "sys_platform == \"win32\""} +exceptiongroup = {version = ">=1.0.0rc8", markers = "python_version < \"3.11\""} +iniconfig = "*" +packaging = "*" +pluggy = ">=0.12,<2.0" +tomli = {version = ">=1.0.0", markers = "python_version < \"3.11\""} + +[package.extras] +testing = ["argcomplete", "hypothesis (>=3.56)", "mock", "nose", "pygments (>=2.7.2)", "requests", "xmlschema"] + +[[package]] +name = "pytest-checkdocs" +version = "2.9.0" +description = "check the README when running tests" +category = "dev" +optional = false +python-versions = ">=3.7" +files = [ {file = "pytest-checkdocs-2.9.0.tar.gz", hash = "sha256:54b3d1357a289e3f6336d31d4ceb72e99afe0fa4f552990f1b1f015f1925543b"}, {file = "pytest_checkdocs-2.9.0-py3-none-any.whl", hash = "sha256:4231e5426915a02385e386b5eb4a70490b143d97cede4c7c77691bb320b5a73f"}, ] -python-dateutil = [ + +[package.dependencies] +build = "*" +docutils = ">=0.15" +importlib-metadata = {version = ">=4", markers = "python_version < \"3.10\""} + +[package.extras] +docs = ["furo", "jaraco.packaging (>=9)", "rst.linker (>=1.9)", "sphinx (>=3.5)"] +testing = ["flake8 (<5)", "pytest (>=6)", "pytest-black (>=0.3.7)", "pytest-checkdocs (>=2.4)", "pytest-cov", "pytest-enabler (>=1.3)", "pytest-flake8", "pytest-mypy (>=0.9.1)", "types-docutils"] + +[[package]] +name = "python-dateutil" +version = "2.8.2" +description = "Extensions to the standard Python datetime module" +category = "main" +optional = false +python-versions = "!=3.0.*,!=3.1.*,!=3.2.*,>=2.7" +files = [ {file = "python-dateutil-2.8.2.tar.gz", hash = "sha256:0123cacc1627ae19ddf3c27a5de5bd67ee4586fbdd6440d9748f8abb483d3e86"}, {file = "python_dateutil-2.8.2-py2.py3-none-any.whl", hash = "sha256:961d03dc3453ebbc59dbdea9e4e11c5651520a876d0f4db161e8674aae935da9"}, ] -python-snappy = [ + +[package.dependencies] +six = ">=1.5" + +[[package]] +name = "python-snappy" +version = "0.6.1" +description = "Python library for the snappy compression library from Google" +category = "main" +optional = true +python-versions = "*" +files = [ {file = "python-snappy-0.6.1.tar.gz", hash = "sha256:b6a107ab06206acc5359d4c5632bd9b22d448702a79b3169b0c62e0fb808bb2a"}, {file = "python_snappy-0.6.1-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:b7f920eaf46ebf41bd26f9df51c160d40f9e00b7b48471c3438cb8d027f7fb9b"}, {file = "python_snappy-0.6.1-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:4ec533a8c1f8df797bded662ec3e494d225b37855bb63eb0d75464a07947477c"}, @@ -2110,11 +1791,27 @@ python-snappy = [ {file = "python_snappy-0.6.1-pp37-pypy37_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:0bdb6942180660bda7f7d01f4c0def3cfc72b1c6d99aad964801775a3e379aba"}, {file = "python_snappy-0.6.1-pp37-pypy37_pp73-win_amd64.whl", hash = "sha256:03bb511380fca2a13325b6f16fe8234c8e12da9660f0258cd45d9a02ffc916af"}, ] -pytz = [ + +[[package]] +name = "pytz" +version = "2022.6" +description = "World timezone definitions, modern and historical" +category = "main" +optional = true +python-versions = "*" +files = [ {file = "pytz-2022.6-py2.py3-none-any.whl", hash = "sha256:222439474e9c98fced559f1709d89e6c9cbf8d79c794ff3eb9f8800064291427"}, {file = "pytz-2022.6.tar.gz", hash = "sha256:e89512406b793ca39f5971bc999cc538ce125c0e51c27941bef4568b460095e2"}, ] -pywin32 = [ + +[[package]] +name = "pywin32" +version = "305" +description = "Python for Window Extensions" +category = "main" +optional = true +python-versions = "*" +files = [ {file = "pywin32-305-cp310-cp310-win32.whl", hash = "sha256:421f6cd86e84bbb696d54563c48014b12a23ef95a14e0bdba526be756d89f116"}, {file = "pywin32-305-cp310-cp310-win_amd64.whl", hash = "sha256:73e819c6bed89f44ff1d690498c0a811948f73777e5f97c494c152b850fad478"}, {file = "pywin32-305-cp310-cp310-win_arm64.whl", hash = "sha256:742eb905ce2187133a29365b428e6c3b9001d79accdc30aa8969afba1d8470f4"}, @@ -2130,7 +1827,15 @@ pywin32 = [ {file = "pywin32-305-cp39-cp39-win32.whl", hash = "sha256:9d968c677ac4d5cbdaa62fd3014ab241718e619d8e36ef8e11fb930515a1e918"}, {file = "pywin32-305-cp39-cp39-win_amd64.whl", hash = "sha256:50768c6b7c3f0b38b7fb14dd4104da93ebced5f1a50dc0e834594bff6fbe1271"}, ] -pyyaml = [ + +[[package]] +name = "pyyaml" +version = "6.0" +description = "YAML parser and emitter for Python" +category = "main" +optional = false +python-versions = ">=3.6" +files = [ {file = "PyYAML-6.0-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:d4db7c7aef085872ef65a8fd7d6d09a14ae91f691dec3e87ee5ee0539d516f53"}, {file = "PyYAML-6.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:9df7ed3b3d2e0ecfe09e14741b857df43adb5a3ddadc919a2d94fbdf78fea53c"}, {file = "PyYAML-6.0-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:77f396e6ef4c73fdc33a9157446466f1cff553d979bd00ecb64385760c6babdc"}, @@ -2172,74 +1877,308 @@ pyyaml = [ {file = "PyYAML-6.0-cp39-cp39-win_amd64.whl", hash = "sha256:b3d267842bf12586ba6c734f89d1f5b871df0273157918b0ccefa29deb05c21c"}, {file = "PyYAML-6.0.tar.gz", hash = "sha256:68fb519c14306fec9720a2a5b45bc9f0c8d1b9c72adf45c37baedfcd949c35a2"}, ] -requests = [ - {file = "requests-2.28.1-py3-none-any.whl", hash = "sha256:8fefa2a1a1365bf5520aac41836fbee479da67864514bdb821f31ce07ce65349"}, - {file = "requests-2.28.1.tar.gz", hash = "sha256:7c5599b102feddaa661c826c56ab4fee28bfd17f5abca1ebbe3e7f19d7c97983"}, + +[[package]] +name = "requests" +version = "2.28.2" +description = "Python HTTP for Humans." +category = "main" +optional = false +python-versions = ">=3.7, <4" +files = [ + {file = "requests-2.28.2-py3-none-any.whl", hash = "sha256:64299f4909223da747622c030b781c0d7811e359c37124b4bd368fb8c6518baa"}, + {file = "requests-2.28.2.tar.gz", hash = "sha256:98b1b2782e3c6c4904938b84c0eb932721069dfdb9134313beff7c83c2df24bf"}, ] -requests-mock = [ + +[package.dependencies] +certifi = ">=2017.4.17" +charset-normalizer = ">=2,<4" +idna = ">=2.5,<4" +urllib3 = ">=1.21.1,<1.27" + +[package.extras] +socks = ["PySocks (>=1.5.6,!=1.5.7)"] +use-chardet-on-py3 = ["chardet (>=3.0.2,<6)"] + +[[package]] +name = "requests-mock" +version = "1.10.0" +description = "Mock out responses from the requests package" +category = "dev" +optional = false +python-versions = "*" +files = [ {file = "requests-mock-1.10.0.tar.gz", hash = "sha256:59c9c32419a9fb1ae83ec242d98e889c45bd7d7a65d48375cc243ec08441658b"}, {file = "requests_mock-1.10.0-py2.py3-none-any.whl", hash = "sha256:2fdbb637ad17ee15c06f33d31169e71bf9fe2bdb7bc9da26185be0dd8d842699"}, ] -requests-oauthlib = [ + +[package.dependencies] +requests = ">=2.3,<3" +six = "*" + +[package.extras] +fixture = ["fixtures"] +test = ["fixtures", "mock", "purl", "pytest", "requests-futures", "sphinx", "testrepository (>=0.0.18)", "testtools"] + +[[package]] +name = "requests-oauthlib" +version = "1.3.1" +description = "OAuthlib authentication support for Requests." +category = "main" +optional = true +python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*" +files = [ {file = "requests-oauthlib-1.3.1.tar.gz", hash = "sha256:75beac4a47881eeb94d5ea5d6ad31ef88856affe2332b9aafb52c6452ccf0d7a"}, {file = "requests_oauthlib-1.3.1-py2.py3-none-any.whl", hash = "sha256:2577c501a2fb8d05a304c09d090d6e47c306fef15809d102b327cf8364bddab5"}, ] -responses = [ + +[package.dependencies] +oauthlib = ">=3.0.0" +requests = ">=2.0.0" + +[package.extras] +rsa = ["oauthlib[signedtoken] (>=3.0.0)"] + +[[package]] +name = "responses" +version = "0.22.0" +description = "A utility library for mocking out the `requests` Python library." +category = "dev" +optional = false +python-versions = ">=3.7" +files = [ {file = "responses-0.22.0-py3-none-any.whl", hash = "sha256:dcf294d204d14c436fddcc74caefdbc5764795a40ff4e6a7740ed8ddbf3294be"}, {file = "responses-0.22.0.tar.gz", hash = "sha256:396acb2a13d25297789a5866b4881cf4e46ffd49cc26c43ab1117f40b973102e"}, ] -rich = [ + +[package.dependencies] +requests = ">=2.22.0,<3.0" +toml = "*" +types-toml = "*" +urllib3 = ">=1.25.10" + +[package.extras] +tests = ["coverage (>=6.0.0)", "flake8", "mypy", "pytest (>=7.0.0)", "pytest-asyncio", "pytest-cov", "pytest-httpserver", "types-requests"] + +[[package]] +name = "rich" +version = "13.0.1" +description = "Render rich text, tables, progress bars, syntax highlighting, markdown and more to the terminal" +category = "main" +optional = false +python-versions = ">=3.7.0" +files = [ {file = "rich-13.0.1-py3-none-any.whl", hash = "sha256:41fe1d05f433b0f4724cda8345219213d2bfa472ef56b2f64f415b5b94d51b04"}, {file = "rich-13.0.1.tar.gz", hash = "sha256:25f83363f636995627a99f6e4abc52ed0970ebbd544960cc63cbb43aaac3d6f0"}, ] -s3fs = [ + +[package.dependencies] +commonmark = ">=0.9.0,<0.10.0" +pygments = ">=2.6.0,<3.0.0" +typing-extensions = {version = ">=4.0.0,<5.0", markers = "python_version < \"3.9\""} + +[package.extras] +jupyter = ["ipywidgets (>=7.5.1,<8.0.0)"] + +[[package]] +name = "s3fs" +version = "2022.10.0" +description = "Convenient Filesystem interface over S3" +category = "main" +optional = true +python-versions = ">= 3.7" +files = [ {file = "s3fs-2022.10.0-py3-none-any.whl", hash = "sha256:1e134c3577171699feb7c1a0c4713260d5b48296e1708737ff940baef6e2c153"}, {file = "s3fs-2022.10.0.tar.gz", hash = "sha256:e8deb80f20bd0b2059141b874fdb9d6aeb8cce35312ea5f2c02b225a78a00406"}, ] -s3transfer = [ + +[package.dependencies] +aiobotocore = ">=2.4.0,<2.5.0" +aiohttp = "<4.0.0a0 || >4.0.0a0,<4.0.0a1 || >4.0.0a1" +fsspec = "2022.10.0" + +[package.extras] +awscli = ["aiobotocore[awscli] (>=2.4.0,<2.5.0)"] +boto3 = ["aiobotocore[boto3] (>=2.4.0,<2.5.0)"] + +[[package]] +name = "s3transfer" +version = "0.6.0" +description = "An Amazon S3 Transfer Manager" +category = "main" +optional = false +python-versions = ">= 3.7" +files = [ {file = "s3transfer-0.6.0-py3-none-any.whl", hash = "sha256:06176b74f3a15f61f1b4f25a1fc29a4429040b7647133a463da8fa5bd28d5ecd"}, {file = "s3transfer-0.6.0.tar.gz", hash = "sha256:2ed07d3866f523cc561bf4a00fc5535827981b117dd7876f036b0c1aca42c947"}, ] -setuptools = [ + +[package.dependencies] +botocore = ">=1.12.36,<2.0a.0" + +[package.extras] +crt = ["botocore[crt] (>=1.20.29,<2.0a.0)"] + +[[package]] +name = "setuptools" +version = "65.6.3" +description = "Easily download, build, install, upgrade, and uninstall Python packages" +category = "dev" +optional = false +python-versions = ">=3.7" +files = [ {file = "setuptools-65.6.3-py3-none-any.whl", hash = "sha256:57f6f22bde4e042978bcd50176fdb381d7c21a9efa4041202288d3737a0c6a54"}, {file = "setuptools-65.6.3.tar.gz", hash = "sha256:a7620757bf984b58deaf32fc8a4577a9bbc0850cf92c20e1ce41c38c19e5fb75"}, ] -six = [ + +[package.extras] +docs = ["furo", "jaraco.packaging (>=9)", "jaraco.tidelift (>=1.4)", "pygments-github-lexers (==0.0.5)", "rst.linker (>=1.9)", "sphinx (>=3.5)", "sphinx-favicon", "sphinx-hoverxref (<2)", "sphinx-inline-tabs", "sphinx-notfound-page (==0.8.3)", "sphinx-reredirects", "sphinxcontrib-towncrier"] +testing = ["build[virtualenv]", "filelock (>=3.4.0)", "flake8 (<5)", "flake8-2020", "ini2toml[lite] (>=0.9)", "jaraco.envs (>=2.2)", "jaraco.path (>=3.2.0)", "pip (>=19.1)", "pip-run (>=8.8)", "pytest (>=6)", "pytest-black (>=0.3.7)", "pytest-checkdocs (>=2.4)", "pytest-cov", "pytest-enabler (>=1.3)", "pytest-flake8", "pytest-mypy (>=0.9.1)", "pytest-perf", "pytest-timeout", "pytest-xdist", "tomli-w (>=1.0.0)", "virtualenv (>=13.0.0)", "wheel"] +testing-integration = ["build[virtualenv]", "filelock (>=3.4.0)", "jaraco.envs (>=2.2)", "jaraco.path (>=3.2.0)", "pytest", "pytest-enabler", "pytest-xdist", "tomli", "virtualenv (>=13.0.0)", "wheel"] + +[[package]] +name = "six" +version = "1.16.0" +description = "Python 2 and 3 compatibility utilities" +category = "main" +optional = false +python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*" +files = [ {file = "six-1.16.0-py2.py3-none-any.whl", hash = "sha256:8abb2f1d86890a2dfb989f9a77cfcfd3e47c2a354b01111771326f8aa26e0254"}, {file = "six-1.16.0.tar.gz", hash = "sha256:1e61c37477a1626458e36f7b1d82aa5c9b094fa4802892072e49de9c60c4c926"}, ] -thrift = [ + +[[package]] +name = "thrift" +version = "0.16.0" +description = "Python bindings for the Apache Thrift RPC system" +category = "main" +optional = true +python-versions = "*" +files = [ {file = "thrift-0.16.0.tar.gz", hash = "sha256:2b5b6488fcded21f9d312aa23c9ff6a0195d0f6ae26ddbd5ad9e3e25dfc14408"}, ] -toml = [ + +[package.dependencies] +six = ">=1.7.2" + +[package.extras] +all = ["tornado (>=4.0)", "twisted"] +tornado = ["tornado (>=4.0)"] +twisted = ["twisted"] + +[[package]] +name = "toml" +version = "0.10.2" +description = "Python Library for Tom's Obvious, Minimal Language" +category = "dev" +optional = false +python-versions = ">=2.6, !=3.0.*, !=3.1.*, !=3.2.*" +files = [ {file = "toml-0.10.2-py2.py3-none-any.whl", hash = "sha256:806143ae5bfb6a3c6e736a764057db0e6a0e05e338b5630894a5f779cabb4f9b"}, {file = "toml-0.10.2.tar.gz", hash = "sha256:b3bda1d108d5dd99f4a20d24d9c348e91c4db7ab1b749200bded2f839ccbe68f"}, ] -tomli = [ + +[[package]] +name = "tomli" +version = "2.0.1" +description = "A lil' TOML parser" +category = "dev" +optional = false +python-versions = ">=3.7" +files = [ {file = "tomli-2.0.1-py3-none-any.whl", hash = "sha256:939de3e7a6161af0c887ef91b7d41a53e7c5a1ca976325f429cb46ea9bc30ecc"}, {file = "tomli-2.0.1.tar.gz", hash = "sha256:de526c12914f0c550d15924c62d72abc48d6fe7364aa87328337a31007fe8a4f"}, ] -types-toml = [ + +[[package]] +name = "types-toml" +version = "0.10.8.1" +description = "Typing stubs for toml" +category = "dev" +optional = false +python-versions = "*" +files = [ {file = "types-toml-0.10.8.1.tar.gz", hash = "sha256:171bdb3163d79a520560f24ba916a9fc9bff81659c5448a9fea89240923722be"}, {file = "types_toml-0.10.8.1-py3-none-any.whl", hash = "sha256:b7b5c4977f96ab7b5ac06d8a6590d17c0bf252a96efc03b109c2711fb3e0eafd"}, ] -typing-extensions = [ + +[[package]] +name = "typing-extensions" +version = "4.4.0" +description = "Backported and Experimental Type Hints for Python 3.7+" +category = "main" +optional = false +python-versions = ">=3.7" +files = [ {file = "typing_extensions-4.4.0-py3-none-any.whl", hash = "sha256:16fa4864408f655d35ec496218b85f79b3437c829e93320c7c9215ccfd92489e"}, {file = "typing_extensions-4.4.0.tar.gz", hash = "sha256:1511434bb92bf8dd198c12b1cc812e800d4181cfcb867674e0f8279cc93087aa"}, ] -urllib3 = [ + +[[package]] +name = "urllib3" +version = "1.26.13" +description = "HTTP library with thread-safe connection pooling, file post, and more." +category = "main" +optional = false +python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*, !=3.5.*" +files = [ {file = "urllib3-1.26.13-py2.py3-none-any.whl", hash = "sha256:47cc05d99aaa09c9e72ed5809b60e7ba354e64b59c9c173ac3018642d8bb41fc"}, {file = "urllib3-1.26.13.tar.gz", hash = "sha256:c083dd0dce68dbfbe1129d5271cb90f9447dea7d52097c6e0126120c521ddea8"}, ] -virtualenv = [ + +[package.extras] +brotli = ["brotli (>=1.0.9)", "brotlicffi (>=0.8.0)", "brotlipy (>=0.6.0)"] +secure = ["certifi", "cryptography (>=1.3.4)", "idna (>=2.0.0)", "ipaddress", "pyOpenSSL (>=0.14)", "urllib3-secure-extra"] +socks = ["PySocks (>=1.5.6,!=1.5.7,<2.0)"] + +[[package]] +name = "virtualenv" +version = "20.17.1" +description = "Virtual Python Environment builder" +category = "dev" +optional = false +python-versions = ">=3.6" +files = [ {file = "virtualenv-20.17.1-py3-none-any.whl", hash = "sha256:ce3b1684d6e1a20a3e5ed36795a97dfc6af29bc3970ca8dab93e11ac6094b3c4"}, {file = "virtualenv-20.17.1.tar.gz", hash = "sha256:f8b927684efc6f1cc206c9db297a570ab9ad0e51c16fa9e45487d36d1905c058"}, ] -werkzeug = [ + +[package.dependencies] +distlib = ">=0.3.6,<1" +filelock = ">=3.4.1,<4" +platformdirs = ">=2.4,<3" + +[package.extras] +docs = ["proselint (>=0.13)", "sphinx (>=5.3)", "sphinx-argparse (>=0.3.2)", "sphinx-rtd-theme (>=1)", "towncrier (>=22.8)"] +testing = ["coverage (>=6.2)", "coverage-enable-subprocess (>=1)", "flaky (>=3.7)", "packaging (>=21.3)", "pytest (>=7.0.1)", "pytest-env (>=0.6.2)", "pytest-freezegun (>=0.4.2)", "pytest-mock (>=3.6.1)", "pytest-randomly (>=3.10.3)", "pytest-timeout (>=2.1)"] + +[[package]] +name = "werkzeug" +version = "2.2.2" +description = "The comprehensive WSGI web application library." +category = "dev" +optional = false +python-versions = ">=3.7" +files = [ {file = "Werkzeug-2.2.2-py3-none-any.whl", hash = "sha256:f979ab81f58d7318e064e99c4506445d60135ac5cd2e177a2de0089bfd4c9bd5"}, {file = "Werkzeug-2.2.2.tar.gz", hash = "sha256:7ea2d48322cc7c0f8b3a215ed73eabd7b5d75d0b50e31ab006286ccff9e00b8f"}, ] -wrapt = [ + +[package.dependencies] +MarkupSafe = ">=2.1.1" + +[package.extras] +watchdog = ["watchdog"] + +[[package]] +name = "wrapt" +version = "1.14.1" +description = "Module for decorators, wrappers and monkey patching." +category = "main" +optional = true +python-versions = "!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,!=3.4.*,>=2.7" +files = [ {file = "wrapt-1.14.1-cp27-cp27m-macosx_10_9_x86_64.whl", hash = "sha256:1b376b3f4896e7930f1f772ac4b064ac12598d1c38d04907e696cc4d794b43d3"}, {file = "wrapt-1.14.1-cp27-cp27m-manylinux1_i686.whl", hash = "sha256:903500616422a40a98a5a3c4ff4ed9d0066f3b4c951fa286018ecdf0750194ef"}, {file = "wrapt-1.14.1-cp27-cp27m-manylinux1_x86_64.whl", hash = "sha256:5a9a0d155deafd9448baff28c08e150d9b24ff010e899311ddd63c45c2445e28"}, @@ -2305,11 +2244,27 @@ wrapt = [ {file = "wrapt-1.14.1-cp39-cp39-win_amd64.whl", hash = "sha256:dee60e1de1898bde3b238f18340eec6148986da0455d8ba7848d50470a7a32fb"}, {file = "wrapt-1.14.1.tar.gz", hash = "sha256:380a85cf89e0e69b7cfbe2ea9f765f004ff419f34194018a6827ac0e3edfed4d"}, ] -xmltodict = [ + +[[package]] +name = "xmltodict" +version = "0.13.0" +description = "Makes working with XML feel like you are working with JSON" +category = "dev" +optional = false +python-versions = ">=3.4" +files = [ {file = "xmltodict-0.13.0-py2.py3-none-any.whl", hash = "sha256:aa89e8fd76320154a40d19a0df04a4695fb9dc5ba977cbb68ab3e4eb225e7852"}, {file = "xmltodict-0.13.0.tar.gz", hash = "sha256:341595a488e3e01a85a9d8911d8912fd922ede5fecc4dce437eb4b6c8d037e56"}, ] -yarl = [ + +[[package]] +name = "yarl" +version = "1.8.2" +description = "Yet another URL library" +category = "main" +optional = true +python-versions = ">=3.7" +files = [ {file = "yarl-1.8.2-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:bb81f753c815f6b8e2ddd2eef3c855cf7da193b82396ac013c661aaa6cc6b0a5"}, {file = "yarl-1.8.2-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:47d49ac96156f0928f002e2424299b2c91d9db73e08c4cd6742923a086f1c863"}, {file = "yarl-1.8.2-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:3fc056e35fa6fba63248d93ff6e672c096f95f7836938241ebc8260e062832fe"}, @@ -2385,11 +2340,35 @@ yarl = [ {file = "yarl-1.8.2-cp39-cp39-win_amd64.whl", hash = "sha256:6604711362f2dbf7160df21c416f81fac0de6dbcf0b5445a2ef25478ecc4c778"}, {file = "yarl-1.8.2.tar.gz", hash = "sha256:49d43402c6e3013ad0978602bf6bf5328535c48d192304b91b97a3c6790b1562"}, ] -zipp = [ + +[package.dependencies] +idna = ">=2.0" +multidict = ">=4.0" + +[[package]] +name = "zipp" +version = "3.11.0" +description = "Backport of pathlib-compatible object wrapper for zip files" +category = "dev" +optional = false +python-versions = ">=3.7" +files = [ {file = "zipp-3.11.0-py3-none-any.whl", hash = "sha256:83a28fcb75844b5c0cdaf5aa4003c2d728c77e05f5aeabe8e95e56727005fbaa"}, {file = "zipp-3.11.0.tar.gz", hash = "sha256:a7a22e05929290a67401440b39690ae6563279bced5f314609d9d03798f56766"}, ] -zstandard = [ + +[package.extras] +docs = ["furo", "jaraco.packaging (>=9)", "jaraco.tidelift (>=1.4)", "rst.linker (>=1.9)", "sphinx (>=3.5)"] +testing = ["flake8 (<5)", "func-timeout", "jaraco.functools", "jaraco.itertools", "more-itertools", "pytest (>=6)", "pytest-black (>=0.3.7)", "pytest-checkdocs (>=2.4)", "pytest-cov", "pytest-enabler (>=1.3)", "pytest-flake8", "pytest-mypy (>=0.9.1)"] + +[[package]] +name = "zstandard" +version = "0.19.0" +description = "Zstandard bindings for Python" +category = "main" +optional = false +python-versions = ">=3.6" +files = [ {file = "zstandard-0.19.0-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:a65e0119ad39e855427520f7829618f78eb2824aa05e63ff19b466080cd99210"}, {file = "zstandard-0.19.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:4fa496d2d674c6e9cffc561639d17009d29adee84a27cf1e12d3c9be14aa8feb"}, {file = "zstandard-0.19.0-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:8f7c68de4f362c1b2f426395fe4e05028c56d0782b2ec3ae18a5416eaf775576"}, @@ -2442,3 +2421,24 @@ zstandard = [ {file = "zstandard-0.19.0-cp39-cp39-win_amd64.whl", hash = "sha256:55a513ec67e85abd8b8b83af8813368036f03e2d29a50fc94033504918273980"}, {file = "zstandard-0.19.0.tar.gz", hash = "sha256:31d12fcd942dd8dbf52ca5f6b1bbe287f44e5d551a081a983ff3ea2082867863"}, ] + +[package.dependencies] +cffi = {version = ">=1.11", markers = "platform_python_implementation == \"PyPy\""} + +[package.extras] +cffi = ["cffi (>=1.11)"] + +[extras] +adlfs = ["adlfs"] +duckdb = ["duckdb", "pyarrow"] +glue = ["boto3"] +hive = ["thrift"] +pandas = ["pandas", "pyarrow"] +pyarrow = ["pyarrow"] +s3fs = ["s3fs"] +snappy = ["python-snappy"] + +[metadata] +lock-version = "2.0" +python-versions = "^3.8" +content-hash = "4e0047e7a526078a4a54c4d1f1dcaeaf3fd5056942124a9cebb451866b625ff9" diff --git a/pyproject.toml b/pyproject.toml index cceedf3dd8..fcb695083e 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -48,7 +48,7 @@ packages = [ [tool.poetry.dependencies] python = "^3.8" mmhash3 = "3.0.1" -requests = "2.28.1" +requests = "2.28.2" click = "8.1.3" rich = "13.0.1" pyyaml = "6.0.0" From a4d3bca96c3fab74e6fdc6116d78d9c75356c041 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 16 Jan 2023 09:16:20 +0100 Subject: [PATCH 339/642] Build: Bump coverage from 7.0.4 to 7.0.5 in /python (#6596) --- poetry.lock | 106 ++++++++++++++++++++++++------------------------- pyproject.toml | 2 +- 2 files changed, 54 insertions(+), 54 deletions(-) diff --git a/poetry.lock b/poetry.lock index 0954a7a1b7..4836004b92 100644 --- a/poetry.lock +++ b/poetry.lock @@ -529,63 +529,63 @@ test = ["flake8 (==3.7.8)", "hypothesis (==3.55.3)"] [[package]] name = "coverage" -version = "7.0.4" +version = "7.0.5" description = "Code coverage measurement for Python" category = "dev" optional = false python-versions = ">=3.7" files = [ - {file = "coverage-7.0.4-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:daf91db39324e9939a9db919ee4fb42a1a23634a056616dae891a030e89f87ba"}, - {file = "coverage-7.0.4-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:55121fe140d7e42cb970999b93cf1c2b24484ce028b32bbd00238bb25c13e34a"}, - {file = "coverage-7.0.4-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:c027fbb83a8c78a6e06a0302ea1799fdb70e5cda9845a5e000545b8e2b47ea39"}, - {file = "coverage-7.0.4-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:caf82db5b7f16b51ec32fe0bd2da0805b177c807aa8bfb478c7e6f893418c284"}, - {file = "coverage-7.0.4-cp310-cp310-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:1ba5cc54baf3c322c4388de2a43cc95f7809366f0600e743e5aae8ea9d1038b2"}, - {file = "coverage-7.0.4-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:260854160083f8275a9d9d49a05ab0ffc7a1f08f2ccccbfaec94a18aae9f407c"}, - {file = "coverage-7.0.4-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:ea45f0dba5a993e93b158f1a9dcfff2770e3bcabf2b80dbe7aa15dce0bcb3bf3"}, - {file = "coverage-7.0.4-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:6abc91f6f8b3cc0ae1034e2c03f38769fba1952ab70d0b26953aa01691265c39"}, - {file = "coverage-7.0.4-cp310-cp310-win32.whl", hash = "sha256:053cdc47cae08257051d7e934a0de4d095b60eb8a3024fa9f1b2322fa1547137"}, - {file = "coverage-7.0.4-cp310-cp310-win_amd64.whl", hash = "sha256:1e9e94f2612ee549a4b3ee79cbc61bceed77e69cf38cfa05858bae939a886d16"}, - {file = "coverage-7.0.4-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:5caa9dd91dcc5f054350dc57a02e053d79633907b9ccffff999568d13dcd19f8"}, - {file = "coverage-7.0.4-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:efc200fa75d9634525b40babc7a16342bd21c101db1a58ef84dc14f4bf6ac0fd"}, - {file = "coverage-7.0.4-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:1791e5f74c5b52f76e83fe9f4bb9571cf76d40ee0c51952ee1e4ee935b7e98b9"}, - {file = "coverage-7.0.4-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:3d9201cfa5a98652b9cef36ab202f17fe3ea83f497b4ba2a8ed39399dfb8fcd4"}, - {file = "coverage-7.0.4-cp311-cp311-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:22d8ef6865cb6834cab2b72fff20747a55c714b57b675f7e11c9624fe4f7cb45"}, - {file = "coverage-7.0.4-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:b84076e3de192fba0f95e279ac017b64c7c6ecd4f09f36f13420f5bed898a9c7"}, - {file = "coverage-7.0.4-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:dcfbf8ffc046f20d75fd775a92c378f6fc7b9bded6c6f2ab88b6b9cb5805a184"}, - {file = "coverage-7.0.4-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:4665a714af31f160403c2e448fb2fef330719d2e04e836b08d60d612707c1041"}, - {file = "coverage-7.0.4-cp311-cp311-win32.whl", hash = "sha256:2e59aef3fba5758059208c9eff10ae7ded3629e797972746ec33b56844f69411"}, - {file = "coverage-7.0.4-cp311-cp311-win_amd64.whl", hash = "sha256:2b854f7985b48122b6fe346631e86d67b63293f8255cb59a93d79e3d9f1574e3"}, - {file = "coverage-7.0.4-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:e44b60b0b49aa85d548d392a2dca2c6a581cd4084e72e9e16bd58bd86ec20816"}, - {file = "coverage-7.0.4-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:2904d7a0388911c61e7e3beefe48c29dfccaba938fc1158f63190101a21e04c2"}, - {file = "coverage-7.0.4-cp37-cp37m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:bc74b64bfa89e2f862ea45dd6ac1def371d7cc883b76680d20bdd61a6f3daa20"}, - {file = "coverage-7.0.4-cp37-cp37m-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:c06046f54e719da21c79f98ecc0962581d1aee0b3798dc6b12b1217da8bf93f4"}, - {file = "coverage-7.0.4-cp37-cp37m-musllinux_1_1_aarch64.whl", hash = "sha256:bc9c77004970a364a1e5454cf7cb884e4277592b959c287689b2a0fd027ef552"}, - {file = "coverage-7.0.4-cp37-cp37m-musllinux_1_1_i686.whl", hash = "sha256:0815a09b32384e8ff00a5939ec9cd10efce8742347e019c2daca1a32f5ac2aae"}, - {file = "coverage-7.0.4-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:a78a80d131c067d67d8a6f9bd3d3f7ea7eac82c1c7259f97d7ab73f723da9d55"}, - {file = "coverage-7.0.4-cp37-cp37m-win32.whl", hash = "sha256:2b5936b624fbe711ed02dfd86edd678822e5ee68da02b6d231e5c01090b64590"}, - {file = "coverage-7.0.4-cp37-cp37m-win_amd64.whl", hash = "sha256:a63922765ee49d5b4c32afb2cd5516812c8665f3b78e64a0dd005bdfabf991b1"}, - {file = "coverage-7.0.4-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:d68f2f7bddb3acdd3b36ef7f334b9d14f30b93e094f808fbbd8d288b8f9e2f9b"}, - {file = "coverage-7.0.4-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:9dafdba3b2b9010abab08cb8c0dc6549bfca6e1630fe14d47b01dca00d39e694"}, - {file = "coverage-7.0.4-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:0322354757b47640535daabd2d56384ff3cad2896248fc84d328c5fad4922d5c"}, - {file = "coverage-7.0.4-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:4e8267466662aff93d66fa72b9591d02122dfc8a729b0a43dd70e0fb07ed9b37"}, - {file = "coverage-7.0.4-cp38-cp38-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:f684d88eb4924ed0630cf488fd5606e334c6835594bb5fe36b50a509b10383ed"}, - {file = "coverage-7.0.4-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:70c294bb15ba576fb96b580db35895bf03749d683df044212b74e938a7f6821f"}, - {file = "coverage-7.0.4-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:34c0457e1ba450ae8b22dc8ea2fd36ada1010af61291e4c96963cd9d9633366f"}, - {file = "coverage-7.0.4-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:b75aff2c35ceaa299691e772f7bf7c8aeab25f46acea2be3dd04cccb914a9860"}, - {file = "coverage-7.0.4-cp38-cp38-win32.whl", hash = "sha256:6c5554d55668381e131577f20e8f620d4882b04ad558f7e7f3f1f55b3124c379"}, - {file = "coverage-7.0.4-cp38-cp38-win_amd64.whl", hash = "sha256:c82f34fafaf5bc05d222fcf84423d6e156432ca35ca78672d4affd0c09c6ef6c"}, - {file = "coverage-7.0.4-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:b8dfb5fed540f77e814bf4ec79619c241af6b4578fa1093c5e3389bbb7beab3f"}, - {file = "coverage-7.0.4-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:ee32a080bab779b71c4d09a3eb5254bfca43ee88828a683dab27dfe8f582516e"}, - {file = "coverage-7.0.4-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:2dfbee0bf0d633be3a2ab068f5a5731a70adf147d0ba17d9f9932b46c7c5782b"}, - {file = "coverage-7.0.4-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:32dc010713455ac0fe2fddb0e48aa43875cc7eb7b09768df10bad8ce45f9c430"}, - {file = "coverage-7.0.4-cp39-cp39-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:9cb88a3019ad042eaa69fc7639ef077793fedbf313e89207aa82fefe92c97ebd"}, - {file = "coverage-7.0.4-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:73bc6114aab7753ca784f87bcd3b7613bc797aa255b5bca45e5654070ae9acfb"}, - {file = "coverage-7.0.4-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:92f135d370fcd7a6fb9659fa2eb716dd2ca364719cbb1756f74d90a221bca1a7"}, - {file = "coverage-7.0.4-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:f3d485e6ec6e09857bf2115ece572d666b7c498377d4c70e66bb06c63ed177c2"}, - {file = "coverage-7.0.4-cp39-cp39-win32.whl", hash = "sha256:c58921fcd9914b56444292e7546fe183d079db99528142c809549ddeaeacd8e9"}, - {file = "coverage-7.0.4-cp39-cp39-win_amd64.whl", hash = "sha256:f092d9f2ddaa30235d33335fbdb61eb8f3657af519ef5f9dd6bdae65272def11"}, - {file = "coverage-7.0.4-pp37.pp38.pp39-none-any.whl", hash = "sha256:cb8cfa3bf3a9f18211279458917fef5edeb5e1fdebe2ea8b11969ec2ebe48884"}, - {file = "coverage-7.0.4.tar.gz", hash = "sha256:f6c4ad409a0caf7e2e12e203348b1a9b19c514e7d078520973147bf2d3dcbc6f"}, + {file = "coverage-7.0.5-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:2a7f23bbaeb2a87f90f607730b45564076d870f1fb07b9318d0c21f36871932b"}, + {file = "coverage-7.0.5-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:c18d47f314b950dbf24a41787ced1474e01ca816011925976d90a88b27c22b89"}, + {file = "coverage-7.0.5-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:ef14d75d86f104f03dea66c13188487151760ef25dd6b2dbd541885185f05f40"}, + {file = "coverage-7.0.5-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:66e50680e888840c0995f2ad766e726ce71ca682e3c5f4eee82272c7671d38a2"}, + {file = "coverage-7.0.5-cp310-cp310-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:a9fed35ca8c6e946e877893bbac022e8563b94404a605af1d1e6accc7eb73289"}, + {file = "coverage-7.0.5-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:d8d04e755934195bdc1db45ba9e040b8d20d046d04d6d77e71b3b34a8cc002d0"}, + {file = "coverage-7.0.5-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:7e109f1c9a3ece676597831874126555997c48f62bddbcace6ed17be3e372de8"}, + {file = "coverage-7.0.5-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:0a1890fca2962c4f1ad16551d660b46ea77291fba2cc21c024cd527b9d9c8809"}, + {file = "coverage-7.0.5-cp310-cp310-win32.whl", hash = "sha256:be9fcf32c010da0ba40bf4ee01889d6c737658f4ddff160bd7eb9cac8f094b21"}, + {file = "coverage-7.0.5-cp310-cp310-win_amd64.whl", hash = "sha256:cbfcba14a3225b055a28b3199c3d81cd0ab37d2353ffd7f6fd64844cebab31ad"}, + {file = "coverage-7.0.5-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:30b5fec1d34cc932c1bc04017b538ce16bf84e239378b8f75220478645d11fca"}, + {file = "coverage-7.0.5-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:1caed2367b32cc80a2b7f58a9f46658218a19c6cfe5bc234021966dc3daa01f0"}, + {file = "coverage-7.0.5-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:d254666d29540a72d17cc0175746cfb03d5123db33e67d1020e42dae611dc196"}, + {file = "coverage-7.0.5-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:19245c249aa711d954623d94f23cc94c0fd65865661f20b7781210cb97c471c0"}, + {file = "coverage-7.0.5-cp311-cp311-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:7b05ed4b35bf6ee790832f68932baf1f00caa32283d66cc4d455c9e9d115aafc"}, + {file = "coverage-7.0.5-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:29de916ba1099ba2aab76aca101580006adfac5646de9b7c010a0f13867cba45"}, + {file = "coverage-7.0.5-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:e057e74e53db78122a3979f908973e171909a58ac20df05c33998d52e6d35757"}, + {file = "coverage-7.0.5-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:411d4ff9d041be08fdfc02adf62e89c735b9468f6d8f6427f8a14b6bb0a85095"}, + {file = "coverage-7.0.5-cp311-cp311-win32.whl", hash = "sha256:52ab14b9e09ce052237dfe12d6892dd39b0401690856bcfe75d5baba4bfe2831"}, + {file = "coverage-7.0.5-cp311-cp311-win_amd64.whl", hash = "sha256:1f66862d3a41674ebd8d1a7b6f5387fe5ce353f8719040a986551a545d7d83ea"}, + {file = "coverage-7.0.5-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:b69522b168a6b64edf0c33ba53eac491c0a8f5cc94fa4337f9c6f4c8f2f5296c"}, + {file = "coverage-7.0.5-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:436e103950d05b7d7f55e39beeb4d5be298ca3e119e0589c0227e6d0b01ee8c7"}, + {file = "coverage-7.0.5-cp37-cp37m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:b8c56bec53d6e3154eaff6ea941226e7bd7cc0d99f9b3756c2520fc7a94e6d96"}, + {file = "coverage-7.0.5-cp37-cp37m-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:7a38362528a9115a4e276e65eeabf67dcfaf57698e17ae388599568a78dcb029"}, + {file = "coverage-7.0.5-cp37-cp37m-musllinux_1_1_aarch64.whl", hash = "sha256:f67472c09a0c7486e27f3275f617c964d25e35727af952869dd496b9b5b7f6a3"}, + {file = "coverage-7.0.5-cp37-cp37m-musllinux_1_1_i686.whl", hash = "sha256:220e3fa77d14c8a507b2d951e463b57a1f7810a6443a26f9b7591ef39047b1b2"}, + {file = "coverage-7.0.5-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:ecb0f73954892f98611e183f50acdc9e21a4653f294dfbe079da73c6378a6f47"}, + {file = "coverage-7.0.5-cp37-cp37m-win32.whl", hash = "sha256:d8f3e2e0a1d6777e58e834fd5a04657f66affa615dae61dd67c35d1568c38882"}, + {file = "coverage-7.0.5-cp37-cp37m-win_amd64.whl", hash = "sha256:9e662e6fc4f513b79da5d10a23edd2b87685815b337b1a30cd11307a6679148d"}, + {file = "coverage-7.0.5-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:790e4433962c9f454e213b21b0fd4b42310ade9c077e8edcb5113db0818450cb"}, + {file = "coverage-7.0.5-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:49640bda9bda35b057b0e65b7c43ba706fa2335c9a9896652aebe0fa399e80e6"}, + {file = "coverage-7.0.5-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:d66187792bfe56f8c18ba986a0e4ae44856b1c645336bd2c776e3386da91e1dd"}, + {file = "coverage-7.0.5-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:276f4cd0001cd83b00817c8db76730938b1ee40f4993b6a905f40a7278103b3a"}, + {file = "coverage-7.0.5-cp38-cp38-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:95304068686545aa368b35dfda1cdfbbdbe2f6fe43de4a2e9baa8ebd71be46e2"}, + {file = "coverage-7.0.5-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:17e01dd8666c445025c29684d4aabf5a90dc6ef1ab25328aa52bedaa95b65ad7"}, + {file = "coverage-7.0.5-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:ea76dbcad0b7b0deb265d8c36e0801abcddf6cc1395940a24e3595288b405ca0"}, + {file = "coverage-7.0.5-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:50a6adc2be8edd7ee67d1abc3cd20678987c7b9d79cd265de55941e3d0d56499"}, + {file = "coverage-7.0.5-cp38-cp38-win32.whl", hash = "sha256:e4ce984133b888cc3a46867c8b4372c7dee9cee300335e2925e197bcd45b9e16"}, + {file = "coverage-7.0.5-cp38-cp38-win_amd64.whl", hash = "sha256:4a950f83fd3f9bca23b77442f3a2b2ea4ac900944d8af9993743774c4fdc57af"}, + {file = "coverage-7.0.5-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:3c2155943896ac78b9b0fd910fb381186d0c345911f5333ee46ac44c8f0e43ab"}, + {file = "coverage-7.0.5-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:54f7e9705e14b2c9f6abdeb127c390f679f6dbe64ba732788d3015f7f76ef637"}, + {file = "coverage-7.0.5-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:0ee30375b409d9a7ea0f30c50645d436b6f5dfee254edffd27e45a980ad2c7f4"}, + {file = "coverage-7.0.5-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:b78729038abea6a5df0d2708dce21e82073463b2d79d10884d7d591e0f385ded"}, + {file = "coverage-7.0.5-cp39-cp39-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:13250b1f0bd023e0c9f11838bdeb60214dd5b6aaf8e8d2f110c7e232a1bff83b"}, + {file = "coverage-7.0.5-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:2c407b1950b2d2ffa091f4e225ca19a66a9bd81222f27c56bd12658fc5ca1209"}, + {file = "coverage-7.0.5-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:c76a3075e96b9c9ff00df8b5f7f560f5634dffd1658bafb79eb2682867e94f78"}, + {file = "coverage-7.0.5-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:f26648e1b3b03b6022b48a9b910d0ae209e2d51f50441db5dce5b530fad6d9b1"}, + {file = "coverage-7.0.5-cp39-cp39-win32.whl", hash = "sha256:ba3027deb7abf02859aca49c865ece538aee56dcb4871b4cced23ba4d5088904"}, + {file = "coverage-7.0.5-cp39-cp39-win_amd64.whl", hash = "sha256:949844af60ee96a376aac1ded2a27e134b8c8d35cc006a52903fc06c24a3296f"}, + {file = "coverage-7.0.5-pp37.pp38.pp39-none-any.whl", hash = "sha256:b9727ac4f5cf2cbf87880a63870b5b9730a8ae3a4a360241a0fdaa2f71240ff0"}, + {file = "coverage-7.0.5.tar.gz", hash = "sha256:051afcbd6d2ac39298d62d340f94dbb6a1f31de06dfaf6fcef7b759dd3860c45"}, ] [package.dependencies] @@ -2441,4 +2441,4 @@ snappy = ["python-snappy"] [metadata] lock-version = "2.0" python-versions = "^3.8" -content-hash = "4e0047e7a526078a4a54c4d1f1dcaeaf3fd5056942124a9cebb451866b625ff9" +content-hash = "83b02246dcb98756c3814c8190c687e010d766379ff2e85683939e927c3220bc" diff --git a/pyproject.toml b/pyproject.toml index fcb695083e..90e69084ef 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -80,7 +80,7 @@ pytest = "7.2.0" pytest-checkdocs = "2.9.0" pre-commit = "2.21.0" fastavro = "1.7.0" -coverage = { version = "^7.0.4", extras = ["toml"] } +coverage = { version = "^7.0.5", extras = ["toml"] } requests-mock = "1.10.0" moto = "^4.0.13" typing-extensions = '4.4.0' From 821a319e13333aeed3661406cd137ce9e768b6e1 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 16 Jan 2023 09:35:04 +0100 Subject: [PATCH 340/642] Build: Bump pytest from 7.2.0 to 7.2.1 in /python (#6595) --- poetry.lock | 8 ++++---- pyproject.toml | 2 +- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/poetry.lock b/poetry.lock index 4836004b92..0496980a1f 100644 --- a/poetry.lock +++ b/poetry.lock @@ -1676,14 +1676,14 @@ diagrams = ["jinja2", "railroad-diagrams"] [[package]] name = "pytest" -version = "7.2.0" +version = "7.2.1" description = "pytest: simple powerful testing with Python" category = "dev" optional = false python-versions = ">=3.7" files = [ - {file = "pytest-7.2.0-py3-none-any.whl", hash = "sha256:892f933d339f068883b6fd5a459f03d85bfcb355e4981e146d2c7616c21fef71"}, - {file = "pytest-7.2.0.tar.gz", hash = "sha256:c4014eb40e10f11f355ad4e3c2fb2c6c6d1919c73f3b5a433de4708202cade59"}, + {file = "pytest-7.2.1-py3-none-any.whl", hash = "sha256:c7c6ca206e93355074ae32f7403e8ea12163b1163c976fee7d4d84027c162be5"}, + {file = "pytest-7.2.1.tar.gz", hash = "sha256:d45e0952f3727241918b8fd0f376f5ff6b301cc0777c6f9a556935c92d8a7d42"}, ] [package.dependencies] @@ -2441,4 +2441,4 @@ snappy = ["python-snappy"] [metadata] lock-version = "2.0" python-versions = "^3.8" -content-hash = "83b02246dcb98756c3814c8190c687e010d766379ff2e85683939e927c3220bc" +content-hash = "a833233e69ccfe5dba8cc1dc3076dbfd3fa7de187a2a2dfb4258f9fa58ae21e0" diff --git a/pyproject.toml b/pyproject.toml index 90e69084ef..93d3c9bb97 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -76,7 +76,7 @@ boto3 = {version = "1.24.59", optional = true} adlfs = { version = "2022.11.2", optional = true } [tool.poetry.dev-dependencies] -pytest = "7.2.0" +pytest = "7.2.1" pytest-checkdocs = "2.9.0" pre-commit = "2.21.0" fastavro = "1.7.0" From 3f77c681f683c540f734abbae3128bdb8b4be30d Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 16 Jan 2023 11:40:37 +0100 Subject: [PATCH 341/642] Build: Bump rich from 13.0.1 to 13.1.0 in /python (#6594) --- poetry.lock | 8 ++++---- pyproject.toml | 2 +- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/poetry.lock b/poetry.lock index 0496980a1f..e04885c675 100644 --- a/poetry.lock +++ b/poetry.lock @@ -1962,14 +1962,14 @@ tests = ["coverage (>=6.0.0)", "flake8", "mypy", "pytest (>=7.0.0)", "pytest-asy [[package]] name = "rich" -version = "13.0.1" +version = "13.1.0" description = "Render rich text, tables, progress bars, syntax highlighting, markdown and more to the terminal" category = "main" optional = false python-versions = ">=3.7.0" files = [ - {file = "rich-13.0.1-py3-none-any.whl", hash = "sha256:41fe1d05f433b0f4724cda8345219213d2bfa472ef56b2f64f415b5b94d51b04"}, - {file = "rich-13.0.1.tar.gz", hash = "sha256:25f83363f636995627a99f6e4abc52ed0970ebbd544960cc63cbb43aaac3d6f0"}, + {file = "rich-13.1.0-py3-none-any.whl", hash = "sha256:f846bff22a43e8508aebf3f0f2410ce1c6f4cde429098bd58d91fde038c57299"}, + {file = "rich-13.1.0.tar.gz", hash = "sha256:81c73a30b144bbcdedc13f4ea0b6ffd7fdc3b0d3cc259a9402309c8e4aee1964"}, ] [package.dependencies] @@ -2441,4 +2441,4 @@ snappy = ["python-snappy"] [metadata] lock-version = "2.0" python-versions = "^3.8" -content-hash = "a833233e69ccfe5dba8cc1dc3076dbfd3fa7de187a2a2dfb4258f9fa58ae21e0" +content-hash = "7190f3393420a23ff2612371ec3cbbdcae6eff027948d8c822b9f59c8a6984e8" diff --git a/pyproject.toml b/pyproject.toml index 93d3c9bb97..75668ea230 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -50,7 +50,7 @@ python = "^3.8" mmhash3 = "3.0.1" requests = "2.28.2" click = "8.1.3" -rich = "13.0.1" +rich = "13.1.0" pyyaml = "6.0.0" pydantic = "1.10.4" From e713b57ab949ef061eab85297642bade02a8ce87 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 17 Jan 2023 07:26:56 +0100 Subject: [PATCH 342/642] Build: Bump moto from 4.0.13 to 4.1.0 in /python (#6593) --- poetry.lock | 22 ++++++++++------------ pyproject.toml | 2 +- 2 files changed, 11 insertions(+), 13 deletions(-) diff --git a/poetry.lock b/poetry.lock index e04885c675..766d393080 100644 --- a/poetry.lock +++ b/poetry.lock @@ -1112,14 +1112,14 @@ files = [ [[package]] name = "moto" -version = "4.0.13" -description = "A library that allows your python tests to easily mock out the boto library" +version = "4.1.0" +description = "" category = "dev" optional = false -python-versions = ">=3.6" +python-versions = ">=3.7" files = [ - {file = "moto-4.0.13-py3-none-any.whl", hash = "sha256:e73400c6d3fe06028aa7f07bb6f276f14260d289b70f38928a98e3d3d968352d"}, - {file = "moto-4.0.13.tar.gz", hash = "sha256:baf7d6969cf837990c730e6e648315bebc2e1c0038d9d8fc4f59d03561484469"}, + {file = "moto-4.1.0-py2.py3-none-any.whl", hash = "sha256:12f231196ebf22d19774f2a75738fdedae10a8c82abb8fa0877e1819aa799f4f"}, + {file = "moto-4.1.0.tar.gz", hash = "sha256:e65a692df57abb631b06c8f3f7827c9deceae3416887527e266c775ad722a2fa"}, ] [package.dependencies] @@ -1127,7 +1127,6 @@ boto3 = ">=1.9.201" botocore = ">=1.12.201" cryptography = ">=3.3.1" Jinja2 = ">=2.10.1" -MarkupSafe = "!=2.0.0a1" python-dateutil = ">=2.1,<3.0.0" requests = ">=2.5" responses = ">=0.13.0" @@ -1135,17 +1134,16 @@ werkzeug = ">=0.5,<2.2.0 || >2.2.0,<2.2.1 || >2.2.1" xmltodict = "*" [package.extras] -all = ["PyYAML (>=5.1)", "aws-xray-sdk (>=0.93,!=0.96)", "cfn-lint (>=0.40.0)", "docker (>=2.5.1)", "ecdsa (!=0.15)", "graphql-core", "idna (>=2.5,<4)", "jsondiff (>=1.1.2)", "openapi-spec-validator (>=0.2.8)", "pyparsing (>=3.0.7)", "python-jose[cryptography] (>=3.1.0,<4.0.0)", "setuptools", "sshpubkeys (>=3.1.0)"] +all = ["PyYAML (>=5.1)", "aws-xray-sdk (>=0.93,!=0.96)", "cfn-lint (>=0.40.0)", "docker (>=2.5.1)", "ecdsa (!=0.15)", "graphql-core", "jsondiff (>=1.1.2)", "openapi-spec-validator (>=0.2.8)", "pyparsing (>=3.0.7)", "python-jose[cryptography] (>=3.1.0,<4.0.0)", "setuptools", "sshpubkeys (>=3.1.0)"] apigateway = ["PyYAML (>=5.1)", "ecdsa (!=0.15)", "openapi-spec-validator (>=0.2.8)", "python-jose[cryptography] (>=3.1.0,<4.0.0)"] apigatewayv2 = ["PyYAML (>=5.1)"] appsync = ["graphql-core"] awslambda = ["docker (>=2.5.1)"] batch = ["docker (>=2.5.1)"] -cloudformation = ["PyYAML (>=5.1)", "aws-xray-sdk (>=0.93,!=0.96)", "cfn-lint (>=0.40.0)", "docker (>=2.5.1)", "ecdsa (!=0.15)", "graphql-core", "idna (>=2.5,<4)", "jsondiff (>=1.1.2)", "openapi-spec-validator (>=0.2.8)", "pyparsing (>=3.0.7)", "python-jose[cryptography] (>=3.1.0,<4.0.0)", "setuptools", "sshpubkeys (>=3.1.0)"] +cloudformation = ["PyYAML (>=5.1)", "aws-xray-sdk (>=0.93,!=0.96)", "cfn-lint (>=0.40.0)", "docker (>=2.5.1)", "ecdsa (!=0.15)", "graphql-core", "jsondiff (>=1.1.2)", "openapi-spec-validator (>=0.2.8)", "pyparsing (>=3.0.7)", "python-jose[cryptography] (>=3.1.0,<4.0.0)", "setuptools", "sshpubkeys (>=3.1.0)"] cognitoidp = ["ecdsa (!=0.15)", "python-jose[cryptography] (>=3.1.0,<4.0.0)"] ds = ["sshpubkeys (>=3.1.0)"] dynamodb = ["docker (>=2.5.1)"] -dynamodb2 = ["docker (>=2.5.1)"] dynamodbstreams = ["docker (>=2.5.1)"] ebs = ["sshpubkeys (>=3.1.0)"] ec2 = ["sshpubkeys (>=3.1.0)"] @@ -1154,8 +1152,8 @@ glue = ["pyparsing (>=3.0.7)"] iotdata = ["jsondiff (>=1.1.2)"] route53resolver = ["sshpubkeys (>=3.1.0)"] s3 = ["PyYAML (>=5.1)"] -server = ["PyYAML (>=5.1)", "aws-xray-sdk (>=0.93,!=0.96)", "cfn-lint (>=0.40.0)", "docker (>=2.5.1)", "ecdsa (!=0.15)", "flask (!=2.2.0,!=2.2.1)", "flask-cors", "graphql-core", "idna (>=2.5,<4)", "jsondiff (>=1.1.2)", "openapi-spec-validator (>=0.2.8)", "pyparsing (>=3.0.7)", "python-jose[cryptography] (>=3.1.0,<4.0.0)", "setuptools", "sshpubkeys (>=3.1.0)"] -ssm = ["PyYAML (>=5.1)", "dataclasses"] +server = ["PyYAML (>=5.1)", "aws-xray-sdk (>=0.93,!=0.96)", "cfn-lint (>=0.40.0)", "docker (>=2.5.1)", "ecdsa (!=0.15)", "flask (!=2.2.0,!=2.2.1)", "flask-cors", "graphql-core", "jsondiff (>=1.1.2)", "openapi-spec-validator (>=0.2.8)", "pyparsing (>=3.0.7)", "python-jose[cryptography] (>=3.1.0,<4.0.0)", "setuptools", "sshpubkeys (>=3.1.0)"] +ssm = ["PyYAML (>=5.1)"] xray = ["aws-xray-sdk (>=0.93,!=0.96)", "setuptools"] [[package]] @@ -2441,4 +2439,4 @@ snappy = ["python-snappy"] [metadata] lock-version = "2.0" python-versions = "^3.8" -content-hash = "7190f3393420a23ff2612371ec3cbbdcae6eff027948d8c822b9f59c8a6984e8" +content-hash = "b8a5b73e631bd8204dc4ee672f9fb07e0b6454842d80d0488c69848943a53405" diff --git a/pyproject.toml b/pyproject.toml index 75668ea230..778e3b3e94 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -82,7 +82,7 @@ pre-commit = "2.21.0" fastavro = "1.7.0" coverage = { version = "^7.0.5", extras = ["toml"] } requests-mock = "1.10.0" -moto = "^4.0.13" +moto = "^4.1.0" typing-extensions = '4.4.0' [tool.poetry.scripts] From a486d11f2e301533da39267337906be0fbaf9be2 Mon Sep 17 00:00:00 2001 From: Armin Najafi Date: Sat, 21 Jan 2023 11:53:20 -0800 Subject: [PATCH 343/642] Python: Fix `test_missing_uri` unit test (#6607) * Fix test_missing_uri unit test * Fix lint error --- tests/cli/test_console.py | 16 +++++++++------- 1 file changed, 9 insertions(+), 7 deletions(-) diff --git a/tests/cli/test_console.py b/tests/cli/test_console.py index 1a3180e674..4605cf3d49 100644 --- a/tests/cli/test_console.py +++ b/tests/cli/test_console.py @@ -35,6 +35,7 @@ from pyiceberg.table.metadata import TableMetadataV2 from pyiceberg.table.sorting import UNSORTED_SORT_ORDER, SortOrder from pyiceberg.typedef import EMPTY_DICT, Identifier, Properties +from pyiceberg.utils.config import Config from tests.conftest import EXAMPLE_TABLE_METADATA_V2 @@ -138,13 +139,14 @@ def test_missing_uri(empty_home_dir_path: str) -> None: # mock to prevent parsing ~/.pyiceberg.yaml or {PYICEBERG_HOME}/.pyiceberg.yaml with mock.patch.dict(os.environ, {"HOME": empty_home_dir_path, "PYICEBERG_HOME": empty_home_dir_path}): - runner = CliRunner() - result = runner.invoke(run, ["list"]) - assert result.exit_code == 1 - assert ( - result.output - == "URI missing, please provide using --uri, the config or environment variable \nPYICEBERG_CATALOG__DEFAULT__URI\n" - ) + with mock.patch("pyiceberg.catalog._ENV_CONFIG", Config()): + runner = CliRunner() + result = runner.invoke(run, ["list"]) + assert result.exit_code == 1 + assert ( + result.output + == "URI missing, please provide using --uri, the config or environment variable \nPYICEBERG_CATALOG__DEFAULT__URI\n" + ) @mock.patch.dict(os.environ, MOCK_ENVIRONMENT) From 098fdeaf3df73b87d683c560c36f3b9e9d104bc6 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Sun, 22 Jan 2023 10:51:53 +0100 Subject: [PATCH 344/642] Build: Bump rich from 13.1.0 to 13.2.0 in /python (#6641) Bumps [rich](https://github.com/Textualize/rich) from 13.1.0 to 13.2.0. - [Release notes](https://github.com/Textualize/rich/releases) - [Changelog](https://github.com/Textualize/rich/blob/master/CHANGELOG.md) - [Commits](https://github.com/Textualize/rich/compare/v13.1.0...v13.2.0) --- updated-dependencies: - dependency-name: rich dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- poetry.lock | 62 ++++++++++++++++++++++++++++++++++---------------- pyproject.toml | 2 +- 2 files changed, 43 insertions(+), 21 deletions(-) diff --git a/poetry.lock b/poetry.lock index 766d393080..5302839037 100644 --- a/poetry.lock +++ b/poetry.lock @@ -512,21 +512,6 @@ files = [ {file = "colorama-0.4.6.tar.gz", hash = "sha256:08695f5cb7ed6e0531a20572697297273c47b8cae5a63ffc6d6ed5c201be6e44"}, ] -[[package]] -name = "commonmark" -version = "0.9.1" -description = "Python parser for the CommonMark Markdown spec" -category = "main" -optional = false -python-versions = "*" -files = [ - {file = "commonmark-0.9.1-py2.py3-none-any.whl", hash = "sha256:da2f38c92590f83de410ba1a3cbceafbc74fee9def35f9251ba9a971d6d66fd9"}, - {file = "commonmark-0.9.1.tar.gz", hash = "sha256:452f9dc859be7f06631ddcb328b6919c67984aca654e5fefb3914d54691aed60"}, -] - -[package.extras] -test = ["flake8 (==3.7.8)", "hypothesis (==3.55.3)"] - [[package]] name = "coverage" version = "7.0.5" @@ -1016,6 +1001,31 @@ files = [ {file = "jmespath-1.0.1.tar.gz", hash = "sha256:90261b206d6defd58fdd5e85f478bf633a2901798906be2ad389150c5c60edbe"}, ] +[[package]] +name = "markdown-it-py" +version = "2.1.0" +description = "Python port of markdown-it. Markdown parsing, done right!" +category = "main" +optional = false +python-versions = ">=3.7" +files = [ + {file = "markdown-it-py-2.1.0.tar.gz", hash = "sha256:cf7e59fed14b5ae17c0006eff14a2d9a00ed5f3a846148153899a0224e2c07da"}, + {file = "markdown_it_py-2.1.0-py3-none-any.whl", hash = "sha256:93de681e5c021a432c63147656fe21790bc01231e0cd2da73626f1aa3ac0fe27"}, +] + +[package.dependencies] +mdurl = ">=0.1,<1.0" + +[package.extras] +benchmarking = ["psutil", "pytest", "pytest-benchmark (>=3.2,<4.0)"] +code-style = ["pre-commit (==2.6)"] +compare = ["commonmark (>=0.9.1,<0.10.0)", "markdown (>=3.3.6,<3.4.0)", "mistletoe (>=0.8.1,<0.9.0)", "mistune (>=2.0.2,<2.1.0)", "panflute (>=2.1.3,<2.2.0)"] +linkify = ["linkify-it-py (>=1.0,<2.0)"] +plugins = ["mdit-py-plugins"] +profiling = ["gprof2dot"] +rtd = ["attrs", "myst-parser", "pyyaml", "sphinx", "sphinx-copybutton", "sphinx-design", "sphinx_book_theme"] +testing = ["coverage", "pytest", "pytest-cov", "pytest-regressions"] + [[package]] name = "markupsafe" version = "2.1.1" @@ -1066,6 +1076,18 @@ files = [ {file = "MarkupSafe-2.1.1.tar.gz", hash = "sha256:7f91197cc9e48f989d12e4e6fbc46495c446636dfc81b9ccf50bb0ec74b91d4b"}, ] +[[package]] +name = "mdurl" +version = "0.1.2" +description = "Markdown URL utilities" +category = "main" +optional = false +python-versions = ">=3.7" +files = [ + {file = "mdurl-0.1.2-py3-none-any.whl", hash = "sha256:84008a41e51615a49fc9966191ff91509e3c40b939176e643fd50a5c2196b8f8"}, + {file = "mdurl-0.1.2.tar.gz", hash = "sha256:bb413d29f5eea38f31dd4754dd7377d4465116fb207585f97bf925588687c1ba"}, +] + [[package]] name = "mmhash3" version = "3.0.1" @@ -1960,18 +1982,18 @@ tests = ["coverage (>=6.0.0)", "flake8", "mypy", "pytest (>=7.0.0)", "pytest-asy [[package]] name = "rich" -version = "13.1.0" +version = "13.2.0" description = "Render rich text, tables, progress bars, syntax highlighting, markdown and more to the terminal" category = "main" optional = false python-versions = ">=3.7.0" files = [ - {file = "rich-13.1.0-py3-none-any.whl", hash = "sha256:f846bff22a43e8508aebf3f0f2410ce1c6f4cde429098bd58d91fde038c57299"}, - {file = "rich-13.1.0.tar.gz", hash = "sha256:81c73a30b144bbcdedc13f4ea0b6ffd7fdc3b0d3cc259a9402309c8e4aee1964"}, + {file = "rich-13.2.0-py3-none-any.whl", hash = "sha256:7c963f0d03819221e9ac561e1bc866e3f95a02248c1234daa48954e6d381c003"}, + {file = "rich-13.2.0.tar.gz", hash = "sha256:f1a00cdd3eebf999a15d85ec498bfe0b1a77efe9b34f645768a54132ef444ac5"}, ] [package.dependencies] -commonmark = ">=0.9.0,<0.10.0" +markdown-it-py = ">=2.1.0,<3.0.0" pygments = ">=2.6.0,<3.0.0" typing-extensions = {version = ">=4.0.0,<5.0", markers = "python_version < \"3.9\""} @@ -2439,4 +2461,4 @@ snappy = ["python-snappy"] [metadata] lock-version = "2.0" python-versions = "^3.8" -content-hash = "b8a5b73e631bd8204dc4ee672f9fb07e0b6454842d80d0488c69848943a53405" +content-hash = "ac31dc8fb10a5e9638d132d8e83ac1cd5b40162c370e7b4f7cc4abb283ef575b" diff --git a/pyproject.toml b/pyproject.toml index 778e3b3e94..10686ba00e 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -50,7 +50,7 @@ python = "^3.8" mmhash3 = "3.0.1" requests = "2.28.2" click = "8.1.3" -rich = "13.1.0" +rich = "13.2.0" pyyaml = "6.0.0" pydantic = "1.10.4" From 21bcf1aa8cf6415d6b3270531853733a21cd08f3 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Sun, 22 Jan 2023 11:09:50 +0100 Subject: [PATCH 345/642] Build: Bump pandas from 1.5.2 to 1.5.3 in /python (#6639) Bumps [pandas](https://github.com/pandas-dev/pandas) from 1.5.2 to 1.5.3. - [Release notes](https://github.com/pandas-dev/pandas/releases) - [Changelog](https://github.com/pandas-dev/pandas/blob/main/RELEASE.md) - [Commits](https://github.com/pandas-dev/pandas/compare/v1.5.2...v1.5.3) --- updated-dependencies: - dependency-name: pandas dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- poetry.lock | 58 +++++++++++++++++++++++++------------------------- pyproject.toml | 2 +- 2 files changed, 30 insertions(+), 30 deletions(-) diff --git a/poetry.lock b/poetry.lock index 5302839037..1d3b1d8047 100644 --- a/poetry.lock +++ b/poetry.lock @@ -1407,39 +1407,39 @@ files = [ [[package]] name = "pandas" -version = "1.5.2" +version = "1.5.3" description = "Powerful data structures for data analysis, time series, and statistics" category = "main" optional = true python-versions = ">=3.8" files = [ - {file = "pandas-1.5.2-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:e9dbacd22555c2d47f262ef96bb4e30880e5956169741400af8b306bbb24a273"}, - {file = "pandas-1.5.2-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:e2b83abd292194f350bb04e188f9379d36b8dfac24dd445d5c87575f3beaf789"}, - {file = "pandas-1.5.2-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:2552bffc808641c6eb471e55aa6899fa002ac94e4eebfa9ec058649122db5824"}, - {file = "pandas-1.5.2-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:1fc87eac0541a7d24648a001d553406f4256e744d92df1df8ebe41829a915028"}, - {file = "pandas-1.5.2-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:d0d8fd58df5d17ddb8c72a5075d87cd80d71b542571b5f78178fb067fa4e9c72"}, - {file = "pandas-1.5.2-cp310-cp310-win_amd64.whl", hash = "sha256:4aed257c7484d01c9a194d9a94758b37d3d751849c05a0050c087a358c41ad1f"}, - {file = "pandas-1.5.2-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:375262829c8c700c3e7cbb336810b94367b9c4889818bbd910d0ecb4e45dc261"}, - {file = "pandas-1.5.2-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:cc3cd122bea268998b79adebbb8343b735a5511ec14efb70a39e7acbc11ccbdc"}, - {file = "pandas-1.5.2-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:b4f5a82afa4f1ff482ab8ded2ae8a453a2cdfde2001567b3ca24a4c5c5ca0db3"}, - {file = "pandas-1.5.2-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:8092a368d3eb7116e270525329a3e5c15ae796ccdf7ccb17839a73b4f5084a39"}, - {file = "pandas-1.5.2-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:f6257b314fc14958f8122779e5a1557517b0f8e500cfb2bd53fa1f75a8ad0af2"}, - {file = "pandas-1.5.2-cp311-cp311-win_amd64.whl", hash = "sha256:82ae615826da838a8e5d4d630eb70c993ab8636f0eff13cb28aafc4291b632b5"}, - {file = "pandas-1.5.2-cp38-cp38-macosx_10_9_universal2.whl", hash = "sha256:457d8c3d42314ff47cc2d6c54f8fc0d23954b47977b2caed09cd9635cb75388b"}, - {file = "pandas-1.5.2-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:c009a92e81ce836212ce7aa98b219db7961a8b95999b97af566b8dc8c33e9519"}, - {file = "pandas-1.5.2-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:71f510b0efe1629bf2f7c0eadb1ff0b9cf611e87b73cd017e6b7d6adb40e2b3a"}, - {file = "pandas-1.5.2-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a40dd1e9f22e01e66ed534d6a965eb99546b41d4d52dbdb66565608fde48203f"}, - {file = "pandas-1.5.2-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:5ae7e989f12628f41e804847a8cc2943d362440132919a69429d4dea1f164da0"}, - {file = "pandas-1.5.2-cp38-cp38-win32.whl", hash = "sha256:530948945e7b6c95e6fa7aa4be2be25764af53fba93fe76d912e35d1c9ee46f5"}, - {file = "pandas-1.5.2-cp38-cp38-win_amd64.whl", hash = "sha256:73f219fdc1777cf3c45fde7f0708732ec6950dfc598afc50588d0d285fddaefc"}, - {file = "pandas-1.5.2-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:9608000a5a45f663be6af5c70c3cbe634fa19243e720eb380c0d378666bc7702"}, - {file = "pandas-1.5.2-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:315e19a3e5c2ab47a67467fc0362cb36c7c60a93b6457f675d7d9615edad2ebe"}, - {file = "pandas-1.5.2-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:e18bc3764cbb5e118be139b3b611bc3fbc5d3be42a7e827d1096f46087b395eb"}, - {file = "pandas-1.5.2-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:0183cb04a057cc38fde5244909fca9826d5d57c4a5b7390c0cc3fa7acd9fa883"}, - {file = "pandas-1.5.2-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:344021ed3e639e017b452aa8f5f6bf38a8806f5852e217a7594417fb9bbfa00e"}, - {file = "pandas-1.5.2-cp39-cp39-win32.whl", hash = "sha256:e7469271497960b6a781eaa930cba8af400dd59b62ec9ca2f4d31a19f2f91090"}, - {file = "pandas-1.5.2-cp39-cp39-win_amd64.whl", hash = "sha256:c218796d59d5abd8780170c937b812c9637e84c32f8271bbf9845970f8c1351f"}, - {file = "pandas-1.5.2.tar.gz", hash = "sha256:220b98d15cee0b2cd839a6358bd1f273d0356bf964c1a1aeb32d47db0215488b"}, + {file = "pandas-1.5.3-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:3749077d86e3a2f0ed51367f30bf5b82e131cc0f14260c4d3e499186fccc4406"}, + {file = "pandas-1.5.3-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:972d8a45395f2a2d26733eb8d0f629b2f90bebe8e8eddbb8829b180c09639572"}, + {file = "pandas-1.5.3-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:50869a35cbb0f2e0cd5ec04b191e7b12ed688874bd05dd777c19b28cbea90996"}, + {file = "pandas-1.5.3-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:c3ac844a0fe00bfaeb2c9b51ab1424e5c8744f89860b138434a363b1f620f354"}, + {file = "pandas-1.5.3-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:7a0a56cef15fd1586726dace5616db75ebcfec9179a3a55e78f72c5639fa2a23"}, + {file = "pandas-1.5.3-cp310-cp310-win_amd64.whl", hash = "sha256:478ff646ca42b20376e4ed3fa2e8d7341e8a63105586efe54fa2508ee087f328"}, + {file = "pandas-1.5.3-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:6973549c01ca91ec96199e940495219c887ea815b2083722821f1d7abfa2b4dc"}, + {file = "pandas-1.5.3-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:c39a8da13cede5adcd3be1182883aea1c925476f4e84b2807a46e2775306305d"}, + {file = "pandas-1.5.3-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:f76d097d12c82a535fda9dfe5e8dd4127952b45fea9b0276cb30cca5ea313fbc"}, + {file = "pandas-1.5.3-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:e474390e60ed609cec869b0da796ad94f420bb057d86784191eefc62b65819ae"}, + {file = "pandas-1.5.3-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:5f2b952406a1588ad4cad5b3f55f520e82e902388a6d5a4a91baa8d38d23c7f6"}, + {file = "pandas-1.5.3-cp311-cp311-win_amd64.whl", hash = "sha256:bc4c368f42b551bf72fac35c5128963a171b40dce866fb066540eeaf46faa003"}, + {file = "pandas-1.5.3-cp38-cp38-macosx_10_9_universal2.whl", hash = "sha256:14e45300521902689a81f3f41386dc86f19b8ba8dd5ac5a3c7010ef8d2932813"}, + {file = "pandas-1.5.3-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:9842b6f4b8479e41968eced654487258ed81df7d1c9b7b870ceea24ed9459b31"}, + {file = "pandas-1.5.3-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:26d9c71772c7afb9d5046e6e9cf42d83dd147b5cf5bcb9d97252077118543792"}, + {file = "pandas-1.5.3-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:5fbcb19d6fceb9e946b3e23258757c7b225ba450990d9ed63ccceeb8cae609f7"}, + {file = "pandas-1.5.3-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:565fa34a5434d38e9d250af3c12ff931abaf88050551d9fbcdfafca50d62babf"}, + {file = "pandas-1.5.3-cp38-cp38-win32.whl", hash = "sha256:87bd9c03da1ac870a6d2c8902a0e1fd4267ca00f13bc494c9e5a9020920e1d51"}, + {file = "pandas-1.5.3-cp38-cp38-win_amd64.whl", hash = "sha256:41179ce559943d83a9b4bbacb736b04c928b095b5f25dd2b7389eda08f46f373"}, + {file = "pandas-1.5.3-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:c74a62747864ed568f5a82a49a23a8d7fe171d0c69038b38cedf0976831296fa"}, + {file = "pandas-1.5.3-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:c4c00e0b0597c8e4f59e8d461f797e5d70b4d025880516a8261b2817c47759ee"}, + {file = "pandas-1.5.3-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:a50d9a4336a9621cab7b8eb3fb11adb82de58f9b91d84c2cd526576b881a0c5a"}, + {file = "pandas-1.5.3-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:dd05f7783b3274aa206a1af06f0ceed3f9b412cf665b7247eacd83be41cf7bf0"}, + {file = "pandas-1.5.3-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:9f69c4029613de47816b1bb30ff5ac778686688751a5e9c99ad8c7031f6508e5"}, + {file = "pandas-1.5.3-cp39-cp39-win32.whl", hash = "sha256:7cec0bee9f294e5de5bbfc14d0573f65526071029d036b753ee6507d2a21480a"}, + {file = "pandas-1.5.3-cp39-cp39-win_amd64.whl", hash = "sha256:dfd681c5dc216037e0b0a2c821f5ed99ba9f03ebcf119c7dac0e9a7b960b9ec9"}, + {file = "pandas-1.5.3.tar.gz", hash = "sha256:74a3fd7e5a7ec052f183273dc7b0acd3a863edf7520f5d3a1765c04ffdb3b0b1"}, ] [package.dependencies] @@ -2461,4 +2461,4 @@ snappy = ["python-snappy"] [metadata] lock-version = "2.0" python-versions = "^3.8" -content-hash = "ac31dc8fb10a5e9638d132d8e83ac1cd5b40162c370e7b4f7cc4abb283ef575b" +content-hash = "61159a32a55ee5d6065910bbabdbde732a99b81a4415fd6f51a586055dd76223" diff --git a/pyproject.toml b/pyproject.toml index 10686ba00e..5084b53d36 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -62,7 +62,7 @@ zstandard = "0.19.0" pyarrow = { version = "10.0.1", optional = true } -pandas = { version = "1.5.2", optional = true } +pandas = { version = "1.5.3", optional = true } duckdb = { version = "0.6.1", optional = true } From 9ce8bec586a22f853605330de48d9b7d929e174b Mon Sep 17 00:00:00 2001 From: Amogh Jahagirdar Date: Tue, 24 Jan 2023 10:52:37 -0800 Subject: [PATCH 346/642] Python: Check if optional file kv metadata is None before reading Iceberg Schema (#6654) --- pyiceberg/io/pyarrow.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/pyiceberg/io/pyarrow.py b/pyiceberg/io/pyarrow.py index 6ee822ba71..12edc6e3e2 100644 --- a/pyiceberg/io/pyarrow.py +++ b/pyiceberg/io/pyarrow.py @@ -505,7 +505,9 @@ def project_table( # Get the schema with fs.open_input_file(path) as fout: parquet_schema = pq.read_schema(fout) - schema_raw = parquet_schema.metadata.get(ICEBERG_SCHEMA) + schema_raw = None + if metadata := parquet_schema.metadata: + schema_raw = metadata.get(ICEBERG_SCHEMA) if schema_raw is None: raise ValueError( "Iceberg schema is not embedded into the Parquet file, see https://github.com/apache/iceberg/issues/6505" From d1c0a6a430118526e9cb3951ed78b69e02ed6ac8 Mon Sep 17 00:00:00 2001 From: Fokko Driesprong Date: Tue, 24 Jan 2023 21:27:36 +0100 Subject: [PATCH 347/642] Python: Bump s3fs (#6643) --- poetry.lock | 2404 ++++++++++++++++++++++++------------------------ pyproject.toml | 7 +- 2 files changed, 1187 insertions(+), 1224 deletions(-) diff --git a/poetry.lock b/poetry.lock index 1d3b1d8047..c6e78d271f 100644 --- a/poetry.lock +++ b/poetry.lock @@ -1,5 +1,3 @@ -# This file is automatically @generated by Poetry and should not be changed by hand. - [[package]] name = "adal" version = "1.2.7" @@ -7,10 +5,6 @@ description = "Note: This library is already replaced by MSAL Python, available category = "main" optional = true python-versions = "*" -files = [ - {file = "adal-1.2.7-py2.py3-none-any.whl", hash = "sha256:2a7451ed7441ddbc57703042204a3e30ef747478eea022c70f789fc7f084bc3d"}, - {file = "adal-1.2.7.tar.gz", hash = "sha256:d74f45b81317454d96e982fd1c50e6fb5c99ac2223728aea8764433a39f566f1"}, -] [package.dependencies] cryptography = ">=1.1.0" @@ -20,15 +14,11 @@ requests = ">=2.0.0,<3" [[package]] name = "adlfs" -version = "2022.11.2" +version = "2023.1.0" description = "Access Azure Datalake Gen1 with fsspec and dask" category = "main" optional = true python-versions = ">=3.8" -files = [ - {file = "adlfs-2022.11.2-py3-none-any.whl", hash = "sha256:60da4a17260754ee6d8da43e3d756226ade4fd19ada8411e858f5acecd0833e2"}, - {file = "adlfs-2022.11.2.tar.gz", hash = "sha256:920dba10468f186037ca394dcabcba113532d80f52b315211c8e771be40475ea"}, -] [package.dependencies] aiohttp = ">=3.7.0" @@ -43,15 +33,11 @@ docs = ["furo", "myst-parser", "numpydoc", "sphinx"] [[package]] name = "aiobotocore" -version = "2.4.1" +version = "2.4.2" description = "Async client for aws services using botocore and aiohttp" category = "main" optional = true python-versions = ">=3.7" -files = [ - {file = "aiobotocore-2.4.1-py3-none-any.whl", hash = "sha256:26e0d55d5901f487c3467616c028abb85036ca33a0f88e279770adae6b865c69"}, - {file = "aiobotocore-2.4.1.tar.gz", hash = "sha256:5c8ee79d1f14273e3f8f3af5c893db889ef163227b178ecacb5aea4547d3b9a7"}, -] [package.dependencies] aiohttp = ">=3.3.1" @@ -70,95 +56,6 @@ description = "Async http client/server framework (asyncio)" category = "main" optional = true python-versions = ">=3.6" -files = [ - {file = "aiohttp-3.8.3-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:ba71c9b4dcbb16212f334126cc3d8beb6af377f6703d9dc2d9fb3874fd667ee9"}, - {file = "aiohttp-3.8.3-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:d24b8bb40d5c61ef2d9b6a8f4528c2f17f1c5d2d31fed62ec860f6006142e83e"}, - {file = "aiohttp-3.8.3-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:f88df3a83cf9df566f171adba39d5bd52814ac0b94778d2448652fc77f9eb491"}, - {file = "aiohttp-3.8.3-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:b97decbb3372d4b69e4d4c8117f44632551c692bb1361b356a02b97b69e18a62"}, - {file = "aiohttp-3.8.3-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:309aa21c1d54b8ef0723181d430347d7452daaff93e8e2363db8e75c72c2fb2d"}, - {file = "aiohttp-3.8.3-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:ad5383a67514e8e76906a06741febd9126fc7c7ff0f599d6fcce3e82b80d026f"}, - {file = "aiohttp-3.8.3-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:20acae4f268317bb975671e375493dbdbc67cddb5f6c71eebdb85b34444ac46b"}, - {file = "aiohttp-3.8.3-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:05a3c31c6d7cd08c149e50dc7aa2568317f5844acd745621983380597f027a18"}, - {file = "aiohttp-3.8.3-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:d6f76310355e9fae637c3162936e9504b4767d5c52ca268331e2756e54fd4ca5"}, - {file = "aiohttp-3.8.3-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:256deb4b29fe5e47893fa32e1de2d73c3afe7407738bd3c63829874661d4822d"}, - {file = "aiohttp-3.8.3-cp310-cp310-musllinux_1_1_ppc64le.whl", hash = "sha256:5c59fcd80b9049b49acd29bd3598cada4afc8d8d69bd4160cd613246912535d7"}, - {file = "aiohttp-3.8.3-cp310-cp310-musllinux_1_1_s390x.whl", hash = "sha256:059a91e88f2c00fe40aed9031b3606c3f311414f86a90d696dd982e7aec48142"}, - {file = "aiohttp-3.8.3-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:2feebbb6074cdbd1ac276dbd737b40e890a1361b3cc30b74ac2f5e24aab41f7b"}, - {file = "aiohttp-3.8.3-cp310-cp310-win32.whl", hash = "sha256:5bf651afd22d5f0c4be16cf39d0482ea494f5c88f03e75e5fef3a85177fecdeb"}, - {file = "aiohttp-3.8.3-cp310-cp310-win_amd64.whl", hash = "sha256:653acc3880459f82a65e27bd6526e47ddf19e643457d36a2250b85b41a564715"}, - {file = "aiohttp-3.8.3-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:86fc24e58ecb32aee09f864cb11bb91bc4c1086615001647dbfc4dc8c32f4008"}, - {file = "aiohttp-3.8.3-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:75e14eac916f024305db517e00a9252714fce0abcb10ad327fb6dcdc0d060f1d"}, - {file = "aiohttp-3.8.3-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:d1fde0f44029e02d02d3993ad55ce93ead9bb9b15c6b7ccd580f90bd7e3de476"}, - {file = "aiohttp-3.8.3-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:4ab94426ddb1ecc6a0b601d832d5d9d421820989b8caa929114811369673235c"}, - {file = "aiohttp-3.8.3-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:89d2e02167fa95172c017732ed7725bc8523c598757f08d13c5acca308e1a061"}, - {file = "aiohttp-3.8.3-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:02f9a2c72fc95d59b881cf38a4b2be9381b9527f9d328771e90f72ac76f31ad8"}, - {file = "aiohttp-3.8.3-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:9c7149272fb5834fc186328e2c1fa01dda3e1fa940ce18fded6d412e8f2cf76d"}, - {file = "aiohttp-3.8.3-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:512bd5ab136b8dc0ffe3fdf2dfb0c4b4f49c8577f6cae55dca862cd37a4564e2"}, - {file = "aiohttp-3.8.3-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:7018ecc5fe97027214556afbc7c502fbd718d0740e87eb1217b17efd05b3d276"}, - {file = "aiohttp-3.8.3-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:88c70ed9da9963d5496d38320160e8eb7e5f1886f9290475a881db12f351ab5d"}, - {file = "aiohttp-3.8.3-cp311-cp311-musllinux_1_1_ppc64le.whl", hash = "sha256:da22885266bbfb3f78218dc40205fed2671909fbd0720aedba39b4515c038091"}, - {file = "aiohttp-3.8.3-cp311-cp311-musllinux_1_1_s390x.whl", hash = "sha256:e65bc19919c910127c06759a63747ebe14f386cda573d95bcc62b427ca1afc73"}, - {file = "aiohttp-3.8.3-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:08c78317e950e0762c2983f4dd58dc5e6c9ff75c8a0efeae299d363d439c8e34"}, - {file = "aiohttp-3.8.3-cp311-cp311-win32.whl", hash = "sha256:45d88b016c849d74ebc6f2b6e8bc17cabf26e7e40c0661ddd8fae4c00f015697"}, - {file = "aiohttp-3.8.3-cp311-cp311-win_amd64.whl", hash = "sha256:96372fc29471646b9b106ee918c8eeb4cca423fcbf9a34daa1b93767a88a2290"}, - {file = "aiohttp-3.8.3-cp36-cp36m-macosx_10_9_x86_64.whl", hash = "sha256:c971bf3786b5fad82ce5ad570dc6ee420f5b12527157929e830f51c55dc8af77"}, - {file = "aiohttp-3.8.3-cp36-cp36m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:ff25f48fc8e623d95eca0670b8cc1469a83783c924a602e0fbd47363bb54aaca"}, - {file = "aiohttp-3.8.3-cp36-cp36m-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:e381581b37db1db7597b62a2e6b8b57c3deec95d93b6d6407c5b61ddc98aca6d"}, - {file = "aiohttp-3.8.3-cp36-cp36m-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:db19d60d846283ee275d0416e2a23493f4e6b6028825b51290ac05afc87a6f97"}, - {file = "aiohttp-3.8.3-cp36-cp36m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:25892c92bee6d9449ffac82c2fe257f3a6f297792cdb18ad784737d61e7a9a85"}, - {file = "aiohttp-3.8.3-cp36-cp36m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:398701865e7a9565d49189f6c90868efaca21be65c725fc87fc305906be915da"}, - {file = "aiohttp-3.8.3-cp36-cp36m-musllinux_1_1_aarch64.whl", hash = "sha256:4a4fbc769ea9b6bd97f4ad0b430a6807f92f0e5eb020f1e42ece59f3ecfc4585"}, - {file = "aiohttp-3.8.3-cp36-cp36m-musllinux_1_1_i686.whl", hash = "sha256:b29bfd650ed8e148f9c515474a6ef0ba1090b7a8faeee26b74a8ff3b33617502"}, - {file = "aiohttp-3.8.3-cp36-cp36m-musllinux_1_1_ppc64le.whl", hash = "sha256:1e56b9cafcd6531bab5d9b2e890bb4937f4165109fe98e2b98ef0dcfcb06ee9d"}, - {file = "aiohttp-3.8.3-cp36-cp36m-musllinux_1_1_s390x.whl", hash = "sha256:ec40170327d4a404b0d91855d41bfe1fe4b699222b2b93e3d833a27330a87a6d"}, - {file = "aiohttp-3.8.3-cp36-cp36m-musllinux_1_1_x86_64.whl", hash = "sha256:2df5f139233060578d8c2c975128fb231a89ca0a462b35d4b5fcf7c501ebdbe1"}, - {file = "aiohttp-3.8.3-cp36-cp36m-win32.whl", hash = "sha256:f973157ffeab5459eefe7b97a804987876dd0a55570b8fa56b4e1954bf11329b"}, - {file = "aiohttp-3.8.3-cp36-cp36m-win_amd64.whl", hash = "sha256:437399385f2abcd634865705bdc180c8314124b98299d54fe1d4c8990f2f9494"}, - {file = "aiohttp-3.8.3-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:09e28f572b21642128ef31f4e8372adb6888846f32fecb288c8b0457597ba61a"}, - {file = "aiohttp-3.8.3-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:6f3553510abdbec67c043ca85727396ceed1272eef029b050677046d3387be8d"}, - {file = "aiohttp-3.8.3-cp37-cp37m-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:e168a7560b7c61342ae0412997b069753f27ac4862ec7867eff74f0fe4ea2ad9"}, - {file = "aiohttp-3.8.3-cp37-cp37m-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:db4c979b0b3e0fa7e9e69ecd11b2b3174c6963cebadeecfb7ad24532ffcdd11a"}, - {file = "aiohttp-3.8.3-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:e164e0a98e92d06da343d17d4e9c4da4654f4a4588a20d6c73548a29f176abe2"}, - {file = "aiohttp-3.8.3-cp37-cp37m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:e8a78079d9a39ca9ca99a8b0ac2fdc0c4d25fc80c8a8a82e5c8211509c523363"}, - {file = "aiohttp-3.8.3-cp37-cp37m-musllinux_1_1_aarch64.whl", hash = "sha256:21b30885a63c3f4ff5b77a5d6caf008b037cb521a5f33eab445dc566f6d092cc"}, - {file = "aiohttp-3.8.3-cp37-cp37m-musllinux_1_1_i686.whl", hash = "sha256:4b0f30372cef3fdc262f33d06e7b411cd59058ce9174ef159ad938c4a34a89da"}, - {file = "aiohttp-3.8.3-cp37-cp37m-musllinux_1_1_ppc64le.whl", hash = "sha256:8135fa153a20d82ffb64f70a1b5c2738684afa197839b34cc3e3c72fa88d302c"}, - {file = "aiohttp-3.8.3-cp37-cp37m-musllinux_1_1_s390x.whl", hash = "sha256:ad61a9639792fd790523ba072c0555cd6be5a0baf03a49a5dd8cfcf20d56df48"}, - {file = "aiohttp-3.8.3-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:978b046ca728073070e9abc074b6299ebf3501e8dee5e26efacb13cec2b2dea0"}, - {file = "aiohttp-3.8.3-cp37-cp37m-win32.whl", hash = "sha256:0d2c6d8c6872df4a6ec37d2ede71eff62395b9e337b4e18efd2177de883a5033"}, - {file = "aiohttp-3.8.3-cp37-cp37m-win_amd64.whl", hash = "sha256:21d69797eb951f155026651f7e9362877334508d39c2fc37bd04ff55b2007091"}, - {file = "aiohttp-3.8.3-cp38-cp38-macosx_10_9_universal2.whl", hash = "sha256:2ca9af5f8f5812d475c5259393f52d712f6d5f0d7fdad9acdb1107dd9e3cb7eb"}, - {file = "aiohttp-3.8.3-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:1d90043c1882067f1bd26196d5d2db9aa6d268def3293ed5fb317e13c9413ea4"}, - {file = "aiohttp-3.8.3-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:d737fc67b9a970f3234754974531dc9afeea11c70791dcb7db53b0cf81b79784"}, - {file = "aiohttp-3.8.3-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:ebf909ea0a3fc9596e40d55d8000702a85e27fd578ff41a5500f68f20fd32e6c"}, - {file = "aiohttp-3.8.3-cp38-cp38-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:5835f258ca9f7c455493a57ee707b76d2d9634d84d5d7f62e77be984ea80b849"}, - {file = "aiohttp-3.8.3-cp38-cp38-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:da37dcfbf4b7f45d80ee386a5f81122501ec75672f475da34784196690762f4b"}, - {file = "aiohttp-3.8.3-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:87f44875f2804bc0511a69ce44a9595d5944837a62caecc8490bbdb0e18b1342"}, - {file = "aiohttp-3.8.3-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:527b3b87b24844ea7865284aabfab08eb0faf599b385b03c2aa91fc6edd6e4b6"}, - {file = "aiohttp-3.8.3-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:d5ba88df9aa5e2f806650fcbeedbe4f6e8736e92fc0e73b0400538fd25a4dd96"}, - {file = "aiohttp-3.8.3-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:e7b8813be97cab8cb52b1375f41f8e6804f6507fe4660152e8ca5c48f0436017"}, - {file = "aiohttp-3.8.3-cp38-cp38-musllinux_1_1_ppc64le.whl", hash = "sha256:2dea10edfa1a54098703cb7acaa665c07b4e7568472a47f4e64e6319d3821ccf"}, - {file = "aiohttp-3.8.3-cp38-cp38-musllinux_1_1_s390x.whl", hash = "sha256:713d22cd9643ba9025d33c4af43943c7a1eb8547729228de18d3e02e278472b6"}, - {file = "aiohttp-3.8.3-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:2d252771fc85e0cf8da0b823157962d70639e63cb9b578b1dec9868dd1f4f937"}, - {file = "aiohttp-3.8.3-cp38-cp38-win32.whl", hash = "sha256:66bd5f950344fb2b3dbdd421aaa4e84f4411a1a13fca3aeb2bcbe667f80c9f76"}, - {file = "aiohttp-3.8.3-cp38-cp38-win_amd64.whl", hash = "sha256:84b14f36e85295fe69c6b9789b51a0903b774046d5f7df538176516c3e422446"}, - {file = "aiohttp-3.8.3-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:16c121ba0b1ec2b44b73e3a8a171c4f999b33929cd2397124a8c7fcfc8cd9e06"}, - {file = "aiohttp-3.8.3-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:8d6aaa4e7155afaf994d7924eb290abbe81a6905b303d8cb61310a2aba1c68ba"}, - {file = "aiohttp-3.8.3-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:43046a319664a04b146f81b40e1545d4c8ac7b7dd04c47e40bf09f65f2437346"}, - {file = "aiohttp-3.8.3-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:599418aaaf88a6d02a8c515e656f6faf3d10618d3dd95866eb4436520096c84b"}, - {file = "aiohttp-3.8.3-cp39-cp39-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:92a2964319d359f494f16011e23434f6f8ef0434acd3cf154a6b7bec511e2fb7"}, - {file = "aiohttp-3.8.3-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:73a4131962e6d91109bca6536416aa067cf6c4efb871975df734f8d2fd821b37"}, - {file = "aiohttp-3.8.3-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:598adde339d2cf7d67beaccda3f2ce7c57b3b412702f29c946708f69cf8222aa"}, - {file = "aiohttp-3.8.3-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:75880ed07be39beff1881d81e4a907cafb802f306efd6d2d15f2b3c69935f6fb"}, - {file = "aiohttp-3.8.3-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:a0239da9fbafd9ff82fd67c16704a7d1bccf0d107a300e790587ad05547681c8"}, - {file = "aiohttp-3.8.3-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:4e3a23ec214e95c9fe85a58470b660efe6534b83e6cbe38b3ed52b053d7cb6ad"}, - {file = "aiohttp-3.8.3-cp39-cp39-musllinux_1_1_ppc64le.whl", hash = "sha256:47841407cc89a4b80b0c52276f3cc8138bbbfba4b179ee3acbd7d77ae33f7ac4"}, - {file = "aiohttp-3.8.3-cp39-cp39-musllinux_1_1_s390x.whl", hash = "sha256:54d107c89a3ebcd13228278d68f1436d3f33f2dd2af5415e3feaeb1156e1a62c"}, - {file = "aiohttp-3.8.3-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:c37c5cce780349d4d51739ae682dec63573847a2a8dcb44381b174c3d9c8d403"}, - {file = "aiohttp-3.8.3-cp39-cp39-win32.whl", hash = "sha256:f178d2aadf0166be4df834c4953da2d7eef24719e8aec9a65289483eeea9d618"}, - {file = "aiohttp-3.8.3-cp39-cp39-win_amd64.whl", hash = "sha256:88e5be56c231981428f4f506c68b6a46fa25c4123a2e86d156c58a8369d31ab7"}, - {file = "aiohttp-3.8.3.tar.gz", hash = "sha256:3828fb41b7203176b82fe5d699e0d845435f2374750a44b480ea6b930f6be269"}, -] [package.dependencies] aiosignal = ">=1.1.2" @@ -170,7 +67,7 @@ multidict = ">=4.5,<7.0" yarl = ">=1.0,<2.0" [package.extras] -speedups = ["Brotli", "aiodns", "cchardet"] +speedups = ["aiodns", "brotli", "cchardet"] [[package]] name = "aioitertools" @@ -179,10 +76,6 @@ description = "itertools and builtins for AsyncIO and mixed iterables" category = "main" optional = true python-versions = ">=3.6" -files = [ - {file = "aioitertools-0.11.0-py3-none-any.whl", hash = "sha256:04b95e3dab25b449def24d7df809411c10e62aab0cbe31a50ca4e68748c43394"}, - {file = "aioitertools-0.11.0.tar.gz", hash = "sha256:42c68b8dd3a69c2bf7f2233bf7df4bb58b557bca5252ac02ed5187bbc67d6831"}, -] [package.dependencies] typing_extensions = {version = ">=4.0", markers = "python_version < \"3.10\""} @@ -194,10 +87,6 @@ description = "aiosignal: a list of registered asynchronous callbacks" category = "main" optional = true python-versions = ">=3.7" -files = [ - {file = "aiosignal-1.3.1-py3-none-any.whl", hash = "sha256:f8376fb07dd1e86a584e4fcdec80b36b7f81aac666ebc724e2c090300dd83b17"}, - {file = "aiosignal-1.3.1.tar.gz", hash = "sha256:54cd96e15e1649b75d6c87526a6ff0b6c1b0dd3459f43d9ca11d48c339b68cfc"}, -] [package.dependencies] frozenlist = ">=1.1.0" @@ -209,40 +98,30 @@ description = "Timeout context manager for asyncio programs" category = "main" optional = true python-versions = ">=3.6" -files = [ - {file = "async-timeout-4.0.2.tar.gz", hash = "sha256:2163e1640ddb52b7a8c80d0a67a08587e5d245cc9c553a74a847056bc2976b15"}, - {file = "async_timeout-4.0.2-py3-none-any.whl", hash = "sha256:8ca1e4fcf50d07413d66d1a5e416e42cfdf5851c981d679a09851a6853383b3c"}, -] [[package]] name = "attrs" -version = "22.1.0" +version = "22.2.0" description = "Classes Without Boilerplate" category = "main" optional = false -python-versions = ">=3.5" -files = [ - {file = "attrs-22.1.0-py2.py3-none-any.whl", hash = "sha256:86efa402f67bf2df34f51a335487cf46b1ec130d02b8d39fd248abfd30da551c"}, - {file = "attrs-22.1.0.tar.gz", hash = "sha256:29adc2665447e5191d0e7c568fde78b21f9672d344281d0c6e1ab085429b22b6"}, -] +python-versions = ">=3.6" [package.extras] -dev = ["cloudpickle", "coverage[toml] (>=5.0.2)", "furo", "hypothesis", "mypy (>=0.900,!=0.940)", "pre-commit", "pympler", "pytest (>=4.3.0)", "pytest-mypy-plugins", "sphinx", "sphinx-notfound-page", "zope.interface"] -docs = ["furo", "sphinx", "sphinx-notfound-page", "zope.interface"] -tests = ["cloudpickle", "coverage[toml] (>=5.0.2)", "hypothesis", "mypy (>=0.900,!=0.940)", "pympler", "pytest (>=4.3.0)", "pytest-mypy-plugins", "zope.interface"] -tests-no-zope = ["cloudpickle", "coverage[toml] (>=5.0.2)", "hypothesis", "mypy (>=0.900,!=0.940)", "pympler", "pytest (>=4.3.0)", "pytest-mypy-plugins"] +cov = ["attrs", "coverage-enable-subprocess", "coverage[toml] (>=5.3)"] +dev = ["attrs"] +docs = ["furo", "myst-parser", "sphinx", "sphinx-notfound-page", "sphinxcontrib-towncrier", "towncrier", "zope.interface"] +tests = ["attrs", "zope.interface"] +tests-no-zope = ["cloudpickle", "hypothesis", "mypy (>=0.971,<0.990)", "pympler", "pytest (>=4.3.0)", "pytest-mypy-plugins", "pytest-xdist"] +tests_no_zope = ["cloudpickle", "hypothesis", "mypy (>=0.971,<0.990)", "pympler", "pytest (>=4.3.0)", "pytest-mypy-plugins", "pytest-xdist"] [[package]] name = "azure-core" -version = "1.26.1" +version = "1.26.2" description = "Microsoft Azure Core Library for Python" category = "main" optional = true python-versions = ">=3.7" -files = [ - {file = "azure-core-1.26.1.zip", hash = "sha256:223b0e90cbdd1f03c41b195b03239899843f20d00964dbb85e64386873414a2d"}, - {file = "azure_core-1.26.1-py3-none-any.whl", hash = "sha256:726ffd1ded04a2c1cb53f9d9155cbb05ac5c1c2a29af4ef622e93e1c0a8bc92b"}, -] [package.dependencies] requests = ">=2.18.4" @@ -259,10 +138,6 @@ description = "Azure Data Lake Store Filesystem Client Library for Python" category = "main" optional = true python-versions = "*" -files = [ - {file = "azure-datalake-store-0.0.52.tar.gz", hash = "sha256:4198ddb32614d16d4502b43d5c9739f81432b7e0e4d75d30e05149fe6007fea2"}, - {file = "azure_datalake_store-0.0.52-py2.py3-none-any.whl", hash = "sha256:aaed72b9c856824aeab554f4dbe0ef2c6d0ff36700bdd8b93d8298793117c48e"}, -] [package.dependencies] adal = ">=0.4.2" @@ -276,10 +151,6 @@ description = "Microsoft Azure Identity Library for Python" category = "main" optional = true python-versions = ">=3.7" -files = [ - {file = "azure-identity-1.12.0.zip", hash = "sha256:7f9b1ae7d97ea7af3f38dd09305e19ab81a1e16ab66ea186b6579d85c1ca2347"}, - {file = "azure_identity-1.12.0-py3-none-any.whl", hash = "sha256:2a58ce4a209a013e37eaccfd5937570ab99e9118b3e1acf875eed3a85d541b92"}, -] [package.dependencies] azure-core = ">=1.11.0,<2.0.0" @@ -295,10 +166,6 @@ description = "Microsoft Azure Blob Storage Client Library for Python" category = "main" optional = true python-versions = ">=3.7" -files = [ - {file = "azure-storage-blob-12.14.1.zip", hash = "sha256:860d4d82985a4bfc7d3271e71275af330f54f330a754355435a7ba749ccde997"}, - {file = "azure_storage_blob-12.14.1-py3-none-any.whl", hash = "sha256:52b84658e8df7853a3cf1c563814655b5028b979b2a87905b92aa6bb30be240e"}, -] [package.dependencies] azure-core = ">=1.24.2,<2.0.0" @@ -312,10 +179,6 @@ description = "The AWS SDK for Python" category = "main" optional = false python-versions = ">= 3.7" -files = [ - {file = "boto3-1.24.59-py3-none-any.whl", hash = "sha256:34ab44146a2c4e7f4e72737f4b27e6eb5e0a7855c2f4599e3d9199b6a0a2d575"}, - {file = "boto3-1.24.59.tar.gz", hash = "sha256:a50b4323f9579cfe22fcf5531fbd40b567d4d74c1adce06aeb5c95fce2a6fb40"}, -] [package.dependencies] botocore = ">=1.27.59,<1.28.0" @@ -332,10 +195,6 @@ description = "Low-level, data-driven core of boto 3." category = "main" optional = false python-versions = ">= 3.7" -files = [ - {file = "botocore-1.27.59-py3-none-any.whl", hash = "sha256:69d756791fc024bda54f6c53f71ae34e695ee41bbbc1743d9179c4837a4929da"}, - {file = "botocore-1.27.59.tar.gz", hash = "sha256:eda4aed6ee719a745d1288eaf1beb12f6f6448ad1fa12f159405db14ba9c92cf"}, -] [package.dependencies] jmespath = ">=0.7.1,<2.0.0" @@ -347,26 +206,22 @@ crt = ["awscrt (==0.14.0)"] [[package]] name = "build" -version = "0.9.0" -description = "A simple, correct PEP 517 build frontend" +version = "0.10.0" +description = "A simple, correct Python build frontend" category = "dev" optional = false -python-versions = ">=3.6" -files = [ - {file = "build-0.9.0-py3-none-any.whl", hash = "sha256:38a7a2b7a0bdc61a42a0a67509d88c71ecfc37b393baba770fae34e20929ff69"}, - {file = "build-0.9.0.tar.gz", hash = "sha256:1a07724e891cbd898923145eb7752ee7653674c511378eb9c7691aab1612bc3c"}, -] +python-versions = ">= 3.7" [package.dependencies] colorama = {version = "*", markers = "os_name == \"nt\""} packaging = ">=19.0" -pep517 = ">=0.9.1" -tomli = {version = ">=1.0.0", markers = "python_version < \"3.11\""} +pyproject_hooks = "*" +tomli = {version = ">=1.1.0", markers = "python_version < \"3.11\""} [package.extras] docs = ["furo (>=2021.08.31)", "sphinx (>=4.0,<5.0)", "sphinx-argparse-cli (>=1.5)", "sphinx-autodoc-typehints (>=1.10)"] test = ["filelock (>=3)", "pytest (>=6.2.4)", "pytest-cov (>=2.12)", "pytest-mock (>=2)", "pytest-rerunfailures (>=9.1)", "pytest-xdist (>=1.34)", "setuptools (>=42.0.0)", "setuptools (>=56.0.0)", "toml (>=0.10.0)", "wheel (>=0.36.0)"] -typing = ["importlib-metadata (>=4.6.4)", "mypy (==0.950)", "typing-extensions (>=3.7.4.3)"] +typing = ["importlib-metadata (>=5.1)", "mypy (==0.991)", "tomli", "typing-extensions (>=3.7.4.3)"] virtualenv = ["virtualenv (>=20.0.35)"] [[package]] @@ -376,10 +231,6 @@ description = "Python package for providing Mozilla's CA Bundle." category = "main" optional = false python-versions = ">=3.6" -files = [ - {file = "certifi-2022.12.7-py3-none-any.whl", hash = "sha256:4ad3232f5e926d6718ec31cfc1fcadfde020920e278684144551c91769c7bc18"}, - {file = "certifi-2022.12.7.tar.gz", hash = "sha256:35824b4c3a97115964b408844d64aa14db1cc518f6562e8d7261699d1350a9e3"}, -] [[package]] name = "cffi" @@ -388,72 +239,6 @@ description = "Foreign Function Interface for Python calling C code." category = "main" optional = false python-versions = "*" -files = [ - {file = "cffi-1.15.1-cp27-cp27m-macosx_10_9_x86_64.whl", hash = "sha256:a66d3508133af6e8548451b25058d5812812ec3798c886bf38ed24a98216fab2"}, - {file = "cffi-1.15.1-cp27-cp27m-manylinux1_i686.whl", hash = "sha256:470c103ae716238bbe698d67ad020e1db9d9dba34fa5a899b5e21577e6d52ed2"}, - {file = "cffi-1.15.1-cp27-cp27m-manylinux1_x86_64.whl", hash = "sha256:9ad5db27f9cabae298d151c85cf2bad1d359a1b9c686a275df03385758e2f914"}, - {file = "cffi-1.15.1-cp27-cp27m-win32.whl", hash = "sha256:b3bbeb01c2b273cca1e1e0c5df57f12dce9a4dd331b4fa1635b8bec26350bde3"}, - {file = "cffi-1.15.1-cp27-cp27m-win_amd64.whl", hash = "sha256:e00b098126fd45523dd056d2efba6c5a63b71ffe9f2bbe1a4fe1716e1d0c331e"}, - {file = "cffi-1.15.1-cp27-cp27mu-manylinux1_i686.whl", hash = "sha256:d61f4695e6c866a23a21acab0509af1cdfd2c013cf256bbf5b6b5e2695827162"}, - {file = "cffi-1.15.1-cp27-cp27mu-manylinux1_x86_64.whl", hash = "sha256:ed9cb427ba5504c1dc15ede7d516b84757c3e3d7868ccc85121d9310d27eed0b"}, - {file = "cffi-1.15.1-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:39d39875251ca8f612b6f33e6b1195af86d1b3e60086068be9cc053aa4376e21"}, - {file = "cffi-1.15.1-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:285d29981935eb726a4399badae8f0ffdff4f5050eaa6d0cfc3f64b857b77185"}, - {file = "cffi-1.15.1-cp310-cp310-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:3eb6971dcff08619f8d91607cfc726518b6fa2a9eba42856be181c6d0d9515fd"}, - {file = "cffi-1.15.1-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:21157295583fe8943475029ed5abdcf71eb3911894724e360acff1d61c1d54bc"}, - {file = "cffi-1.15.1-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:5635bd9cb9731e6d4a1132a498dd34f764034a8ce60cef4f5319c0541159392f"}, - {file = "cffi-1.15.1-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:2012c72d854c2d03e45d06ae57f40d78e5770d252f195b93f581acf3ba44496e"}, - {file = "cffi-1.15.1-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:dd86c085fae2efd48ac91dd7ccffcfc0571387fe1193d33b6394db7ef31fe2a4"}, - {file = "cffi-1.15.1-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:fa6693661a4c91757f4412306191b6dc88c1703f780c8234035eac011922bc01"}, - {file = "cffi-1.15.1-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:59c0b02d0a6c384d453fece7566d1c7e6b7bae4fc5874ef2ef46d56776d61c9e"}, - {file = "cffi-1.15.1-cp310-cp310-win32.whl", hash = "sha256:cba9d6b9a7d64d4bd46167096fc9d2f835e25d7e4c121fb2ddfc6528fb0413b2"}, - {file = "cffi-1.15.1-cp310-cp310-win_amd64.whl", hash = "sha256:ce4bcc037df4fc5e3d184794f27bdaab018943698f4ca31630bc7f84a7b69c6d"}, - {file = "cffi-1.15.1-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:3d08afd128ddaa624a48cf2b859afef385b720bb4b43df214f85616922e6a5ac"}, - {file = "cffi-1.15.1-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:3799aecf2e17cf585d977b780ce79ff0dc9b78d799fc694221ce814c2c19db83"}, - {file = "cffi-1.15.1-cp311-cp311-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:a591fe9e525846e4d154205572a029f653ada1a78b93697f3b5a8f1f2bc055b9"}, - {file = "cffi-1.15.1-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:3548db281cd7d2561c9ad9984681c95f7b0e38881201e157833a2342c30d5e8c"}, - {file = "cffi-1.15.1-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:91fc98adde3d7881af9b59ed0294046f3806221863722ba7d8d120c575314325"}, - {file = "cffi-1.15.1-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:94411f22c3985acaec6f83c6df553f2dbe17b698cc7f8ae751ff2237d96b9e3c"}, - {file = "cffi-1.15.1-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:03425bdae262c76aad70202debd780501fabeaca237cdfddc008987c0e0f59ef"}, - {file = "cffi-1.15.1-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:cc4d65aeeaa04136a12677d3dd0b1c0c94dc43abac5860ab33cceb42b801c1e8"}, - {file = "cffi-1.15.1-cp311-cp311-win32.whl", hash = "sha256:a0f100c8912c114ff53e1202d0078b425bee3649ae34d7b070e9697f93c5d52d"}, - {file = "cffi-1.15.1-cp311-cp311-win_amd64.whl", hash = "sha256:04ed324bda3cda42b9b695d51bb7d54b680b9719cfab04227cdd1e04e5de3104"}, - {file = "cffi-1.15.1-cp36-cp36m-macosx_10_9_x86_64.whl", hash = "sha256:50a74364d85fd319352182ef59c5c790484a336f6db772c1a9231f1c3ed0cbd7"}, - {file = "cffi-1.15.1-cp36-cp36m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:e263d77ee3dd201c3a142934a086a4450861778baaeeb45db4591ef65550b0a6"}, - {file = "cffi-1.15.1-cp36-cp36m-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:cec7d9412a9102bdc577382c3929b337320c4c4c4849f2c5cdd14d7368c5562d"}, - {file = "cffi-1.15.1-cp36-cp36m-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:4289fc34b2f5316fbb762d75362931e351941fa95fa18789191b33fc4cf9504a"}, - {file = "cffi-1.15.1-cp36-cp36m-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:173379135477dc8cac4bc58f45db08ab45d228b3363adb7af79436135d028405"}, - {file = "cffi-1.15.1-cp36-cp36m-manylinux_2_5_x86_64.manylinux1_x86_64.whl", hash = "sha256:6975a3fac6bc83c4a65c9f9fcab9e47019a11d3d2cf7f3c0d03431bf145a941e"}, - {file = "cffi-1.15.1-cp36-cp36m-win32.whl", hash = "sha256:2470043b93ff09bf8fb1d46d1cb756ce6132c54826661a32d4e4d132e1977adf"}, - {file = "cffi-1.15.1-cp36-cp36m-win_amd64.whl", hash = "sha256:30d78fbc8ebf9c92c9b7823ee18eb92f2e6ef79b45ac84db507f52fbe3ec4497"}, - {file = "cffi-1.15.1-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:198caafb44239b60e252492445da556afafc7d1e3ab7a1fb3f0584ef6d742375"}, - {file = "cffi-1.15.1-cp37-cp37m-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:5ef34d190326c3b1f822a5b7a45f6c4535e2f47ed06fec77d3d799c450b2651e"}, - {file = "cffi-1.15.1-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:8102eaf27e1e448db915d08afa8b41d6c7ca7a04b7d73af6514df10a3e74bd82"}, - {file = "cffi-1.15.1-cp37-cp37m-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:5df2768244d19ab7f60546d0c7c63ce1581f7af8b5de3eb3004b9b6fc8a9f84b"}, - {file = "cffi-1.15.1-cp37-cp37m-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:a8c4917bd7ad33e8eb21e9a5bbba979b49d9a97acb3a803092cbc1133e20343c"}, - {file = "cffi-1.15.1-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:0e2642fe3142e4cc4af0799748233ad6da94c62a8bec3a6648bf8ee68b1c7426"}, - {file = "cffi-1.15.1-cp37-cp37m-win32.whl", hash = "sha256:e229a521186c75c8ad9490854fd8bbdd9a0c9aa3a524326b55be83b54d4e0ad9"}, - {file = "cffi-1.15.1-cp37-cp37m-win_amd64.whl", hash = "sha256:a0b71b1b8fbf2b96e41c4d990244165e2c9be83d54962a9a1d118fd8657d2045"}, - {file = "cffi-1.15.1-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:320dab6e7cb2eacdf0e658569d2575c4dad258c0fcc794f46215e1e39f90f2c3"}, - {file = "cffi-1.15.1-cp38-cp38-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:1e74c6b51a9ed6589199c787bf5f9875612ca4a8a0785fb2d4a84429badaf22a"}, - {file = "cffi-1.15.1-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a5c84c68147988265e60416b57fc83425a78058853509c1b0629c180094904a5"}, - {file = "cffi-1.15.1-cp38-cp38-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:3b926aa83d1edb5aa5b427b4053dc420ec295a08e40911296b9eb1b6170f6cca"}, - {file = "cffi-1.15.1-cp38-cp38-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:87c450779d0914f2861b8526e035c5e6da0a3199d8f1add1a665e1cbc6fc6d02"}, - {file = "cffi-1.15.1-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:4f2c9f67e9821cad2e5f480bc8d83b8742896f1242dba247911072d4fa94c192"}, - {file = "cffi-1.15.1-cp38-cp38-win32.whl", hash = "sha256:8b7ee99e510d7b66cdb6c593f21c043c248537a32e0bedf02e01e9553a172314"}, - {file = "cffi-1.15.1-cp38-cp38-win_amd64.whl", hash = "sha256:00a9ed42e88df81ffae7a8ab6d9356b371399b91dbdf0c3cb1e84c03a13aceb5"}, - {file = "cffi-1.15.1-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:54a2db7b78338edd780e7ef7f9f6c442500fb0d41a5a4ea24fff1c929d5af585"}, - {file = "cffi-1.15.1-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:fcd131dd944808b5bdb38e6f5b53013c5aa4f334c5cad0c72742f6eba4b73db0"}, - {file = "cffi-1.15.1-cp39-cp39-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:7473e861101c9e72452f9bf8acb984947aa1661a7704553a9f6e4baa5ba64415"}, - {file = "cffi-1.15.1-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:6c9a799e985904922a4d207a94eae35c78ebae90e128f0c4e521ce339396be9d"}, - {file = "cffi-1.15.1-cp39-cp39-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:3bcde07039e586f91b45c88f8583ea7cf7a0770df3a1649627bf598332cb6984"}, - {file = "cffi-1.15.1-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:33ab79603146aace82c2427da5ca6e58f2b3f2fb5da893ceac0c42218a40be35"}, - {file = "cffi-1.15.1-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:5d598b938678ebf3c67377cdd45e09d431369c3b1a5b331058c338e201f12b27"}, - {file = "cffi-1.15.1-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:db0fbb9c62743ce59a9ff687eb5f4afbe77e5e8403d6697f7446e5f609976f76"}, - {file = "cffi-1.15.1-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:98d85c6a2bef81588d9227dde12db8a7f47f639f4a17c9ae08e773aa9c697bf3"}, - {file = "cffi-1.15.1-cp39-cp39-win32.whl", hash = "sha256:40f4774f5a9d4f5e344f31a32b5096977b5d48560c5592e2f3d2c4374bd543ee"}, - {file = "cffi-1.15.1-cp39-cp39-win_amd64.whl", hash = "sha256:70df4e3b545a17496c9b3f41f5115e69a4f2e77e94e1d2a8e1070bc0c38c8a3c"}, - {file = "cffi-1.15.1.tar.gz", hash = "sha256:d400bfb9a37b1351253cb402671cea7e89bdecc294e8016a707f6d1d8ac934f9"}, -] [package.dependencies] pycparser = "*" @@ -465,10 +250,6 @@ description = "Validate configuration and produce human readable error messages. category = "dev" optional = false python-versions = ">=3.6.1" -files = [ - {file = "cfgv-3.3.1-py2.py3-none-any.whl", hash = "sha256:c6a0883f3917a037485059700b9e75da2464e6c27051014ad85ba6aaa5884426"}, - {file = "cfgv-3.3.1.tar.gz", hash = "sha256:f5a830efb9ce7a445376bb66ec94c638a9787422f96264c98edc6bdeed8ab736"}, -] [[package]] name = "charset-normalizer" @@ -477,13 +258,9 @@ description = "The Real First Universal Charset Detector. Open, modern and activ category = "main" optional = false python-versions = ">=3.6.0" -files = [ - {file = "charset-normalizer-2.1.1.tar.gz", hash = "sha256:5a3d016c7c547f69d6f81fb0db9449ce888b418b5b9952cc5e6e66843e9dd845"}, - {file = "charset_normalizer-2.1.1-py3-none-any.whl", hash = "sha256:83e9a75d1911279afd89352c68b45348559d1fc0506b054b346651b5e7fee29f"}, -] [package.extras] -unicode-backport = ["unicodedata2"] +unicode_backport = ["unicodedata2"] [[package]] name = "click" @@ -492,10 +269,6 @@ description = "Composable command line interface toolkit" category = "main" optional = false python-versions = ">=3.7" -files = [ - {file = "click-8.1.3-py3-none-any.whl", hash = "sha256:bb4d8133cb15a609f44e8213d9b391b0809795062913b383c62be0ee95b1db48"}, - {file = "click-8.1.3.tar.gz", hash = "sha256:7682dc8afb30297001674575ea00d1814d808d6a36af415a82bd481d37ba7b8e"}, -] [package.dependencies] colorama = {version = "*", markers = "platform_system == \"Windows\""} @@ -507,10 +280,6 @@ description = "Cross-platform colored terminal text." category = "main" optional = false python-versions = "!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,!=3.4.*,!=3.5.*,!=3.6.*,>=2.7" -files = [ - {file = "colorama-0.4.6-py2.py3-none-any.whl", hash = "sha256:4f1d9991f5acc0ca119f9d443620b77f9d6b33703e51011c16baf57afb285fc6"}, - {file = "colorama-0.4.6.tar.gz", hash = "sha256:08695f5cb7ed6e0531a20572697297273c47b8cae5a63ffc6d6ed5c201be6e44"}, -] [[package]] name = "coverage" @@ -519,59 +288,6 @@ description = "Code coverage measurement for Python" category = "dev" optional = false python-versions = ">=3.7" -files = [ - {file = "coverage-7.0.5-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:2a7f23bbaeb2a87f90f607730b45564076d870f1fb07b9318d0c21f36871932b"}, - {file = "coverage-7.0.5-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:c18d47f314b950dbf24a41787ced1474e01ca816011925976d90a88b27c22b89"}, - {file = "coverage-7.0.5-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:ef14d75d86f104f03dea66c13188487151760ef25dd6b2dbd541885185f05f40"}, - {file = "coverage-7.0.5-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:66e50680e888840c0995f2ad766e726ce71ca682e3c5f4eee82272c7671d38a2"}, - {file = "coverage-7.0.5-cp310-cp310-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:a9fed35ca8c6e946e877893bbac022e8563b94404a605af1d1e6accc7eb73289"}, - {file = "coverage-7.0.5-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:d8d04e755934195bdc1db45ba9e040b8d20d046d04d6d77e71b3b34a8cc002d0"}, - {file = "coverage-7.0.5-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:7e109f1c9a3ece676597831874126555997c48f62bddbcace6ed17be3e372de8"}, - {file = "coverage-7.0.5-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:0a1890fca2962c4f1ad16551d660b46ea77291fba2cc21c024cd527b9d9c8809"}, - {file = "coverage-7.0.5-cp310-cp310-win32.whl", hash = "sha256:be9fcf32c010da0ba40bf4ee01889d6c737658f4ddff160bd7eb9cac8f094b21"}, - {file = "coverage-7.0.5-cp310-cp310-win_amd64.whl", hash = "sha256:cbfcba14a3225b055a28b3199c3d81cd0ab37d2353ffd7f6fd64844cebab31ad"}, - {file = "coverage-7.0.5-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:30b5fec1d34cc932c1bc04017b538ce16bf84e239378b8f75220478645d11fca"}, - {file = "coverage-7.0.5-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:1caed2367b32cc80a2b7f58a9f46658218a19c6cfe5bc234021966dc3daa01f0"}, - {file = "coverage-7.0.5-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:d254666d29540a72d17cc0175746cfb03d5123db33e67d1020e42dae611dc196"}, - {file = "coverage-7.0.5-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:19245c249aa711d954623d94f23cc94c0fd65865661f20b7781210cb97c471c0"}, - {file = "coverage-7.0.5-cp311-cp311-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:7b05ed4b35bf6ee790832f68932baf1f00caa32283d66cc4d455c9e9d115aafc"}, - {file = "coverage-7.0.5-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:29de916ba1099ba2aab76aca101580006adfac5646de9b7c010a0f13867cba45"}, - {file = "coverage-7.0.5-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:e057e74e53db78122a3979f908973e171909a58ac20df05c33998d52e6d35757"}, - {file = "coverage-7.0.5-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:411d4ff9d041be08fdfc02adf62e89c735b9468f6d8f6427f8a14b6bb0a85095"}, - {file = "coverage-7.0.5-cp311-cp311-win32.whl", hash = "sha256:52ab14b9e09ce052237dfe12d6892dd39b0401690856bcfe75d5baba4bfe2831"}, - {file = "coverage-7.0.5-cp311-cp311-win_amd64.whl", hash = "sha256:1f66862d3a41674ebd8d1a7b6f5387fe5ce353f8719040a986551a545d7d83ea"}, - {file = "coverage-7.0.5-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:b69522b168a6b64edf0c33ba53eac491c0a8f5cc94fa4337f9c6f4c8f2f5296c"}, - {file = "coverage-7.0.5-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:436e103950d05b7d7f55e39beeb4d5be298ca3e119e0589c0227e6d0b01ee8c7"}, - {file = "coverage-7.0.5-cp37-cp37m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:b8c56bec53d6e3154eaff6ea941226e7bd7cc0d99f9b3756c2520fc7a94e6d96"}, - {file = "coverage-7.0.5-cp37-cp37m-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:7a38362528a9115a4e276e65eeabf67dcfaf57698e17ae388599568a78dcb029"}, - {file = "coverage-7.0.5-cp37-cp37m-musllinux_1_1_aarch64.whl", hash = "sha256:f67472c09a0c7486e27f3275f617c964d25e35727af952869dd496b9b5b7f6a3"}, - {file = "coverage-7.0.5-cp37-cp37m-musllinux_1_1_i686.whl", hash = "sha256:220e3fa77d14c8a507b2d951e463b57a1f7810a6443a26f9b7591ef39047b1b2"}, - {file = "coverage-7.0.5-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:ecb0f73954892f98611e183f50acdc9e21a4653f294dfbe079da73c6378a6f47"}, - {file = "coverage-7.0.5-cp37-cp37m-win32.whl", hash = "sha256:d8f3e2e0a1d6777e58e834fd5a04657f66affa615dae61dd67c35d1568c38882"}, - {file = "coverage-7.0.5-cp37-cp37m-win_amd64.whl", hash = "sha256:9e662e6fc4f513b79da5d10a23edd2b87685815b337b1a30cd11307a6679148d"}, - {file = "coverage-7.0.5-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:790e4433962c9f454e213b21b0fd4b42310ade9c077e8edcb5113db0818450cb"}, - {file = "coverage-7.0.5-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:49640bda9bda35b057b0e65b7c43ba706fa2335c9a9896652aebe0fa399e80e6"}, - {file = "coverage-7.0.5-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:d66187792bfe56f8c18ba986a0e4ae44856b1c645336bd2c776e3386da91e1dd"}, - {file = "coverage-7.0.5-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:276f4cd0001cd83b00817c8db76730938b1ee40f4993b6a905f40a7278103b3a"}, - {file = "coverage-7.0.5-cp38-cp38-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:95304068686545aa368b35dfda1cdfbbdbe2f6fe43de4a2e9baa8ebd71be46e2"}, - {file = "coverage-7.0.5-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:17e01dd8666c445025c29684d4aabf5a90dc6ef1ab25328aa52bedaa95b65ad7"}, - {file = "coverage-7.0.5-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:ea76dbcad0b7b0deb265d8c36e0801abcddf6cc1395940a24e3595288b405ca0"}, - {file = "coverage-7.0.5-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:50a6adc2be8edd7ee67d1abc3cd20678987c7b9d79cd265de55941e3d0d56499"}, - {file = "coverage-7.0.5-cp38-cp38-win32.whl", hash = "sha256:e4ce984133b888cc3a46867c8b4372c7dee9cee300335e2925e197bcd45b9e16"}, - {file = "coverage-7.0.5-cp38-cp38-win_amd64.whl", hash = "sha256:4a950f83fd3f9bca23b77442f3a2b2ea4ac900944d8af9993743774c4fdc57af"}, - {file = "coverage-7.0.5-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:3c2155943896ac78b9b0fd910fb381186d0c345911f5333ee46ac44c8f0e43ab"}, - {file = "coverage-7.0.5-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:54f7e9705e14b2c9f6abdeb127c390f679f6dbe64ba732788d3015f7f76ef637"}, - {file = "coverage-7.0.5-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:0ee30375b409d9a7ea0f30c50645d436b6f5dfee254edffd27e45a980ad2c7f4"}, - {file = "coverage-7.0.5-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:b78729038abea6a5df0d2708dce21e82073463b2d79d10884d7d591e0f385ded"}, - {file = "coverage-7.0.5-cp39-cp39-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:13250b1f0bd023e0c9f11838bdeb60214dd5b6aaf8e8d2f110c7e232a1bff83b"}, - {file = "coverage-7.0.5-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:2c407b1950b2d2ffa091f4e225ca19a66a9bd81222f27c56bd12658fc5ca1209"}, - {file = "coverage-7.0.5-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:c76a3075e96b9c9ff00df8b5f7f560f5634dffd1658bafb79eb2682867e94f78"}, - {file = "coverage-7.0.5-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:f26648e1b3b03b6022b48a9b910d0ae209e2d51f50441db5dce5b530fad6d9b1"}, - {file = "coverage-7.0.5-cp39-cp39-win32.whl", hash = "sha256:ba3027deb7abf02859aca49c865ece538aee56dcb4871b4cced23ba4d5088904"}, - {file = "coverage-7.0.5-cp39-cp39-win_amd64.whl", hash = "sha256:949844af60ee96a376aac1ded2a27e134b8c8d35cc006a52903fc06c24a3296f"}, - {file = "coverage-7.0.5-pp37.pp38.pp39-none-any.whl", hash = "sha256:b9727ac4f5cf2cbf87880a63870b5b9730a8ae3a4a360241a0fdaa2f71240ff0"}, - {file = "coverage-7.0.5.tar.gz", hash = "sha256:051afcbd6d2ac39298d62d340f94dbb6a1f31de06dfaf6fcef7b759dd3860c45"}, -] [package.dependencies] tomli = {version = "*", optional = true, markers = "python_full_version <= \"3.11.0a6\" and extra == \"toml\""} @@ -581,47 +297,19 @@ toml = ["tomli"] [[package]] name = "cryptography" -version = "38.0.4" +version = "39.0.0" description = "cryptography is a package which provides cryptographic recipes and primitives to Python developers." category = "main" optional = false python-versions = ">=3.6" -files = [ - {file = "cryptography-38.0.4-cp36-abi3-macosx_10_10_universal2.whl", hash = "sha256:2fa36a7b2cc0998a3a4d5af26ccb6273f3df133d61da2ba13b3286261e7efb70"}, - {file = "cryptography-38.0.4-cp36-abi3-macosx_10_10_x86_64.whl", hash = "sha256:1f13ddda26a04c06eb57119caf27a524ccae20533729f4b1e4a69b54e07035eb"}, - {file = "cryptography-38.0.4-cp36-abi3-manylinux_2_17_aarch64.manylinux2014_aarch64.manylinux_2_24_aarch64.whl", hash = "sha256:2ec2a8714dd005949d4019195d72abed84198d877112abb5a27740e217e0ea8d"}, - {file = "cryptography-38.0.4-cp36-abi3-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:50a1494ed0c3f5b4d07650a68cd6ca62efe8b596ce743a5c94403e6f11bf06c1"}, - {file = "cryptography-38.0.4-cp36-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:a10498349d4c8eab7357a8f9aa3463791292845b79597ad1b98a543686fb1ec8"}, - {file = "cryptography-38.0.4-cp36-abi3-manylinux_2_24_x86_64.whl", hash = "sha256:10652dd7282de17990b88679cb82f832752c4e8237f0c714be518044269415db"}, - {file = "cryptography-38.0.4-cp36-abi3-manylinux_2_28_aarch64.whl", hash = "sha256:bfe6472507986613dc6cc00b3d492b2f7564b02b3b3682d25ca7f40fa3fd321b"}, - {file = "cryptography-38.0.4-cp36-abi3-manylinux_2_28_x86_64.whl", hash = "sha256:ce127dd0a6a0811c251a6cddd014d292728484e530d80e872ad9806cfb1c5b3c"}, - {file = "cryptography-38.0.4-cp36-abi3-musllinux_1_1_aarch64.whl", hash = "sha256:53049f3379ef05182864d13bb9686657659407148f901f3f1eee57a733fb4b00"}, - {file = "cryptography-38.0.4-cp36-abi3-musllinux_1_1_x86_64.whl", hash = "sha256:8a4b2bdb68a447fadebfd7d24855758fe2d6fecc7fed0b78d190b1af39a8e3b0"}, - {file = "cryptography-38.0.4-cp36-abi3-win32.whl", hash = "sha256:1d7e632804a248103b60b16fb145e8df0bc60eed790ece0d12efe8cd3f3e7744"}, - {file = "cryptography-38.0.4-cp36-abi3-win_amd64.whl", hash = "sha256:8e45653fb97eb2f20b8c96f9cd2b3a0654d742b47d638cf2897afbd97f80fa6d"}, - {file = "cryptography-38.0.4-pp37-pypy37_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:ca57eb3ddaccd1112c18fc80abe41db443cc2e9dcb1917078e02dfa010a4f353"}, - {file = "cryptography-38.0.4-pp37-pypy37_pp73-manylinux_2_24_x86_64.whl", hash = "sha256:c9e0d79ee4c56d841bd4ac6e7697c8ff3c8d6da67379057f29e66acffcd1e9a7"}, - {file = "cryptography-38.0.4-pp37-pypy37_pp73-manylinux_2_28_x86_64.whl", hash = "sha256:0e70da4bdff7601b0ef48e6348339e490ebfb0cbe638e083c9c41fb49f00c8bd"}, - {file = "cryptography-38.0.4-pp38-pypy38_pp73-macosx_10_10_x86_64.whl", hash = "sha256:998cd19189d8a747b226d24c0207fdaa1e6658a1d3f2494541cb9dfbf7dcb6d2"}, - {file = "cryptography-38.0.4-pp38-pypy38_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:67461b5ebca2e4c2ab991733f8ab637a7265bb582f07c7c88914b5afb88cb95b"}, - {file = "cryptography-38.0.4-pp38-pypy38_pp73-manylinux_2_24_x86_64.whl", hash = "sha256:4eb85075437f0b1fd8cd66c688469a0c4119e0ba855e3fef86691971b887caf6"}, - {file = "cryptography-38.0.4-pp38-pypy38_pp73-manylinux_2_28_x86_64.whl", hash = "sha256:3178d46f363d4549b9a76264f41c6948752183b3f587666aff0555ac50fd7876"}, - {file = "cryptography-38.0.4-pp38-pypy38_pp73-win_amd64.whl", hash = "sha256:6391e59ebe7c62d9902c24a4d8bcbc79a68e7c4ab65863536127c8a9cd94043b"}, - {file = "cryptography-38.0.4-pp39-pypy39_pp73-macosx_10_10_x86_64.whl", hash = "sha256:78e47e28ddc4ace41dd38c42e6feecfdadf9c3be2af389abbfeef1ff06822285"}, - {file = "cryptography-38.0.4-pp39-pypy39_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:2fb481682873035600b5502f0015b664abc26466153fab5c6bc92c1ea69d478b"}, - {file = "cryptography-38.0.4-pp39-pypy39_pp73-manylinux_2_24_x86_64.whl", hash = "sha256:4367da5705922cf7070462e964f66e4ac24162e22ab0a2e9d31f1b270dd78083"}, - {file = "cryptography-38.0.4-pp39-pypy39_pp73-manylinux_2_28_x86_64.whl", hash = "sha256:b4cad0cea995af760f82820ab4ca54e5471fc782f70a007f31531957f43e9dee"}, - {file = "cryptography-38.0.4-pp39-pypy39_pp73-win_amd64.whl", hash = "sha256:80ca53981ceeb3241998443c4964a387771588c4e4a5d92735a493af868294f9"}, - {file = "cryptography-38.0.4.tar.gz", hash = "sha256:175c1a818b87c9ac80bb7377f5520b7f31b3ef2a0004e2420319beadedb67290"}, -] [package.dependencies] cffi = ">=1.12" [package.extras] -docs = ["sphinx (>=1.6.5,!=1.8.0,!=3.1.0,!=3.1.1)", "sphinx-rtd-theme"] +docs = ["sphinx (>=1.6.5,!=1.8.0,!=3.1.0,!=3.1.1,!=5.2.0,!=5.2.0.post0)", "sphinx-rtd-theme"] docstest = ["pyenchant (>=1.6.11)", "sphinxcontrib-spelling (>=4.0.1)", "twine (>=1.12.0)"] -pep8test = ["black", "flake8", "flake8-import-order", "pep8-naming"] +pep8test = ["black", "ruff"] sdist = ["setuptools-rust (>=0.11.4)"] ssh = ["bcrypt (>=3.1.5)"] test = ["hypothesis (>=1.11.4,!=3.79.2)", "iso8601", "pretend", "pytest (>=6.2.0)", "pytest-benchmark", "pytest-cov", "pytest-subtests", "pytest-xdist", "pytz"] @@ -633,10 +321,6 @@ description = "Distribution utilities" category = "dev" optional = false python-versions = "*" -files = [ - {file = "distlib-0.3.6-py2.py3-none-any.whl", hash = "sha256:f35c4b692542ca110de7ef0bea44d73981caeb34ca0b9b6b2e6d7790dda8f80e"}, - {file = "distlib-0.3.6.tar.gz", hash = "sha256:14bad2d9b04d3a36127ac97f30b12a19268f211063d8f8ee4f47108896e11b46"}, -] [[package]] name = "docutils" @@ -645,10 +329,6 @@ description = "Docutils -- Python Documentation Utilities" category = "dev" optional = false python-versions = ">=3.7" -files = [ - {file = "docutils-0.19-py3-none-any.whl", hash = "sha256:5e1de4d849fee02c63b040a4a3fd567f4ab104defd8a5511fbbc24a8a017efbc"}, - {file = "docutils-0.19.tar.gz", hash = "sha256:33995a6753c30b7f577febfc2c50411fec6aac7f7ffeb7c4cfe5991072dcf9e6"}, -] [[package]] name = "duckdb" @@ -657,70 +337,17 @@ description = "DuckDB embedded database" category = "main" optional = true python-versions = "*" -files = [ - {file = "duckdb-0.6.1-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:e566514f9327f89264e98ac14ee7a84fbd9857328028258422c3e8375ee19d25"}, - {file = "duckdb-0.6.1-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:b31c2883de5b19591a2852165e6b3f9821f77af649835f27bc146b26e4aa30cb"}, - {file = "duckdb-0.6.1-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:998165b2fb1f1d2b0ad742096015ea70878f7d40304643c7424c3ed3ddf07bfc"}, - {file = "duckdb-0.6.1-cp310-cp310-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:3941b3a1e8a1cdb7b90ab3917b87af816e71f9692e5ada7f19b6b60969f731e5"}, - {file = "duckdb-0.6.1-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:143611bd1b7c13343f087d4d423a7a8a4f33a114c5326171e867febf3f0fcfe1"}, - {file = "duckdb-0.6.1-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:125ba45e8b08f28858f918ec9cbd3a19975e5d8d9e8275ef4ad924028a616e14"}, - {file = "duckdb-0.6.1-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:e609a65b31c92f2f7166831f74b56f5ed54b33d8c2c4b4c3974c26fdc50464c5"}, - {file = "duckdb-0.6.1-cp310-cp310-win32.whl", hash = "sha256:b39045074fb9a3f068496475a5d627ad4fa572fa3b4980e3b479c11d0b706f2d"}, - {file = "duckdb-0.6.1-cp310-cp310-win_amd64.whl", hash = "sha256:16fa96ffaa3d842a9355a633fb8bc092d119be08d4bc02013946d8594417bc14"}, - {file = "duckdb-0.6.1-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:b4bbe2f6c1b109c626f9318eee80934ad2a5b81a51409c6b5083c6c5f9bdb125"}, - {file = "duckdb-0.6.1-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:cfea36b58928ce778d17280d4fb3bf0a2d7cff407667baedd69c5b41463ac0fd"}, - {file = "duckdb-0.6.1-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:0b64eb53d0d0695814bf1b65c0f91ab7ed66b515f89c88038f65ad5e0762571c"}, - {file = "duckdb-0.6.1-cp311-cp311-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:35b01bc724e1933293f4c34f410d2833bfbb56d5743b515d805bbfed0651476e"}, - {file = "duckdb-0.6.1-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:fec2c2466654ce786843bda2bfba71e0e4719106b41d36b17ceb1901e130aa71"}, - {file = "duckdb-0.6.1-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:82cd30f5cf368658ef879b1c60276bc8650cf67cfe3dc3e3009438ba39251333"}, - {file = "duckdb-0.6.1-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:a782bbfb7f5e97d4a9c834c9e78f023fb8b3f6687c22ca99841e6ed944b724da"}, - {file = "duckdb-0.6.1-cp311-cp311-win32.whl", hash = "sha256:e3702d4a9ade54c6403f6615a98bbec2020a76a60f5db7fcf085df1bd270e66e"}, - {file = "duckdb-0.6.1-cp311-cp311-win_amd64.whl", hash = "sha256:93b074f473d68c944b0eeb2edcafd91ad11da8432b484836efaaab4e26351d48"}, - {file = "duckdb-0.6.1-cp36-cp36m-macosx_10_9_x86_64.whl", hash = "sha256:adae183924d6d479202c39072e37d440b511326e84525bcb7432bca85f86caba"}, - {file = "duckdb-0.6.1-cp36-cp36m-win32.whl", hash = "sha256:546a1cd17595bd1dd009daf6f36705aa6f95337154360ce44932157d353dcd80"}, - {file = "duckdb-0.6.1-cp36-cp36m-win_amd64.whl", hash = "sha256:87b0d00eb9d1a7ebe437276203e0cdc93b4a2154ba9688c65e8d2a8735839ec6"}, - {file = "duckdb-0.6.1-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:8442e074de6e1969c3d2b24363a5a6d7f866d5ac3f4e358e357495b389eff6c1"}, - {file = "duckdb-0.6.1-cp37-cp37m-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:0a6bf2ae7bec803352dade14561cb0b461b2422e70f75d9f09b36ba2dad2613b"}, - {file = "duckdb-0.6.1-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:5054792f22733f89d9cbbced2bafd8772d72d0fe77f159310221cefcf981c680"}, - {file = "duckdb-0.6.1-cp37-cp37m-musllinux_1_1_i686.whl", hash = "sha256:21cc503dffc2c68bb825e4eb3098e82f40e910b3d09e1b3b7f090d39ad53fbea"}, - {file = "duckdb-0.6.1-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:54b3da77ad893e99c073087ff7f75a8c98154ac5139d317149f12b74367211db"}, - {file = "duckdb-0.6.1-cp37-cp37m-win32.whl", hash = "sha256:f1d709aa6a26172a3eab804b57763d5cdc1a4b785ac1fc2b09568578e52032ee"}, - {file = "duckdb-0.6.1-cp37-cp37m-win_amd64.whl", hash = "sha256:f4edcaa471d791393e37f63e3c7c728fa6324e3ac7e768b9dc2ea49065cd37cc"}, - {file = "duckdb-0.6.1-cp38-cp38-macosx_10_9_universal2.whl", hash = "sha256:d218c2dd3bda51fb79e622b7b2266183ac9493834b55010aa01273fa5b7a7105"}, - {file = "duckdb-0.6.1-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:0c7155cb93ab432eca44b651256c359281d26d927ff43badaf1d2276dd770832"}, - {file = "duckdb-0.6.1-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:0925778200090d3d5d8b6bb42b4d05d24db1e8912484ba3b7e7b7f8569f17dcb"}, - {file = "duckdb-0.6.1-cp38-cp38-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:8b544dd04bb851d08bc68b317a7683cec6091547ae75555d075f8c8a7edb626e"}, - {file = "duckdb-0.6.1-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:f2c37d5a0391cf3a3a66e63215968ffb78e6b84f659529fa4bd10478f6203071"}, - {file = "duckdb-0.6.1-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:ce376966260eb5c351fcc6af627a979dbbcae3efeb2e70f85b23aa45a21e289d"}, - {file = "duckdb-0.6.1-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:73c974b09dd08dff5e8bdedba11c7d0aa0fc46ca93954ee7d19e1e18c9883ac1"}, - {file = "duckdb-0.6.1-cp38-cp38-win32.whl", hash = "sha256:bfe39ed3a03e8b1ed764f58f513b37b24afe110d245803a41655d16d391ad9f1"}, - {file = "duckdb-0.6.1-cp38-cp38-win_amd64.whl", hash = "sha256:afa97d982dbe6b125631a17e222142e79bee88f7a13fc4cee92d09285e31ec83"}, - {file = "duckdb-0.6.1-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:c35ff4b1117096ef72d101524df0079da36c3735d52fcf1d907ccffa63bd6202"}, - {file = "duckdb-0.6.1-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:5c54910fbb6de0f21d562e18a5c91540c19876db61b862fc9ffc8e31be8b3f03"}, - {file = "duckdb-0.6.1-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:99a7172563a3ae67d867572ce27cf3962f58e76f491cb7f602f08c2af39213b3"}, - {file = "duckdb-0.6.1-cp39-cp39-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:7363ffe857d00216b659116647fbf1e925cb3895699015d4a4e50b746de13041"}, - {file = "duckdb-0.6.1-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:06c1cef25f896b2284ba048108f645c72fab5c54aa5a6f62f95663f44ff8a79b"}, - {file = "duckdb-0.6.1-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:e92dd6aad7e8c29d002947376b6f5ce28cae29eb3b6b58a64a46cdbfc5cb7943"}, - {file = "duckdb-0.6.1-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:4b280b2d8a01ecd4fe2feab041df70233c534fafbe33a38565b52c1e017529c7"}, - {file = "duckdb-0.6.1-cp39-cp39-win32.whl", hash = "sha256:d9212d76e90b8469743924a4d22bef845be310d0d193d54ae17d9ef1f753cfa7"}, - {file = "duckdb-0.6.1-cp39-cp39-win_amd64.whl", hash = "sha256:00b7be8f67ec1a8edaa8844f521267baa1a795f4c482bfad56c72c26e1862ab2"}, - {file = "duckdb-0.6.1.tar.gz", hash = "sha256:6d26e9f1afcb924a6057785e506810d48332d4764ddc4a5b414d0f2bf0cacfb4"}, -] [package.dependencies] numpy = ">=1.14" [[package]] name = "exceptiongroup" -version = "1.0.4" +version = "1.1.0" description = "Backport of PEP 654 (exception groups)" category = "dev" optional = false python-versions = ">=3.7" -files = [ - {file = "exceptiongroup-1.0.4-py3-none-any.whl", hash = "sha256:542adf9dea4055530d6e1279602fa5cb11dab2395fa650b8674eaec35fc4a828"}, - {file = "exceptiongroup-1.0.4.tar.gz", hash = "sha256:bd14967b79cd9bdb54d97323216f8fdf533e278df937aa2a90089e7d6e06e5ec"}, -] [package.extras] test = ["pytest (>=6)"] @@ -732,29 +359,6 @@ description = "Fast read/write of AVRO files" category = "dev" optional = false python-versions = ">=3.7" -files = [ - {file = "fastavro-1.7.0-cp310-cp310-macosx_10_14_x86_64.whl", hash = "sha256:ab3387a06e272980fa034f5c62f7063977b77df6416d3d30a4d3b49cc8827566"}, - {file = "fastavro-1.7.0-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:216132bc54da19e97e1531dd69c86282408d4c797749d83b01b3a00862a180de"}, - {file = "fastavro-1.7.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:c20cf6cd8098bb93c2cffd53d03ccea1dcf9ec594a5c83963acf29a2882f8693"}, - {file = "fastavro-1.7.0-cp310-cp310-win_amd64.whl", hash = "sha256:54b60e79506a456bcfc940560fa2c73a7a8e3ddc58a1ae4d94fdd99f6b02aef0"}, - {file = "fastavro-1.7.0-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:4da6d83abd04a804310667f8d1fc23d44e9ed91ed9a9bc9c1fcd906e0c403b12"}, - {file = "fastavro-1.7.0-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:7c664302f94064adeb93403c61c74e07b9710526403eba3b59169f99bb99c55c"}, - {file = "fastavro-1.7.0-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:7aab162f02e64bf82d0282430a3c6ec7a36982b1c5d479e7dcc278e6d62a84b8"}, - {file = "fastavro-1.7.0-cp311-cp311-win_amd64.whl", hash = "sha256:63d0d2a2bb3e85d006c834633be51b105a50b0dc7cc8423b06f30601b532adf4"}, - {file = "fastavro-1.7.0-cp37-cp37m-macosx_10_15_x86_64.whl", hash = "sha256:a07a3245049529d6d9028d664361cc4990e74d035d2303875295e2f7b97eba09"}, - {file = "fastavro-1.7.0-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:7df57d31cb58770a9066790250e9f4ec91242c69e1dc62ea732a6fb2406a8f96"}, - {file = "fastavro-1.7.0-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:b905634b5763ca08c9f7b51486e2c3ae7907f5d9bc48208c69b16ccbe8455e90"}, - {file = "fastavro-1.7.0-cp37-cp37m-win_amd64.whl", hash = "sha256:749206f1cec3d7429546e49db5708f4571497d35181b6b334c4844133f230515"}, - {file = "fastavro-1.7.0-cp38-cp38-macosx_10_15_x86_64.whl", hash = "sha256:a0ebd5c1269085179de4b3f072de274fb66a471ecbc5245bd8684e6f94943c2f"}, - {file = "fastavro-1.7.0-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a7e35b94b692f8cca0096c89abf1937efed66252dea0b3b3165babfb3c289fb7"}, - {file = "fastavro-1.7.0-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:202424a5a7831b773f0f2cc2f82e89ed1726051fd5994f13dc678514144e10d4"}, - {file = "fastavro-1.7.0-cp38-cp38-win_amd64.whl", hash = "sha256:601f8c2ec166bd721f4b12eafe195dd1373d3f8dce4fe2425abd2df3e3968ac7"}, - {file = "fastavro-1.7.0-cp39-cp39-macosx_10_15_x86_64.whl", hash = "sha256:91cc9be740abca894082af4fe3ab9db057d4e5fa783cfa9a94c02ec041bf4346"}, - {file = "fastavro-1.7.0-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:9e4463b00242e4baf52d499aeefab46a26d9dd18d808d4219cd4d21089da540e"}, - {file = "fastavro-1.7.0-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:f7348d318858203bd108e6bcde177d8a6f0590b52bc624d815f26fb6c37029bb"}, - {file = "fastavro-1.7.0-cp39-cp39-win_amd64.whl", hash = "sha256:492e8902063aa1c73170e11c18495fcaa86b71eae3513ef83ba438ca02b16b34"}, - {file = "fastavro-1.7.0.tar.gz", hash = "sha256:4b1205f46489b4032d3155c1ab44d9824be0c7454df98d3a5bd22b78b98f23c8"}, -] [package.extras] codecs = ["lz4", "python-snappy", "zstandard"] @@ -764,19 +368,15 @@ zstandard = ["zstandard"] [[package]] name = "filelock" -version = "3.8.2" +version = "3.9.0" description = "A platform independent file lock." category = "dev" optional = false python-versions = ">=3.7" -files = [ - {file = "filelock-3.8.2-py3-none-any.whl", hash = "sha256:8df285554452285f79c035efb0c861eb33a4bcfa5b7a137016e32e6a90f9792c"}, - {file = "filelock-3.8.2.tar.gz", hash = "sha256:7565f628ea56bfcd8e54e42bdc55da899c85c1abfe1b5bcfd147e9188cebb3b2"}, -] [package.extras] -docs = ["furo (>=2022.9.29)", "sphinx (>=5.3)", "sphinx-autodoc-typehints (>=1.19.5)"] -testing = ["covdefaults (>=2.2.2)", "coverage (>=6.5)", "pytest (>=7.2)", "pytest-cov (>=4)", "pytest-timeout (>=2.1)"] +docs = ["furo (>=2022.12.7)", "sphinx (>=5.3)", "sphinx-autodoc-typehints (>=1.19.5)"] +testing = ["covdefaults (>=2.2.2)", "coverage (>=7.0.1)", "pytest (>=7.2)", "pytest-cov (>=4)", "pytest-timeout (>=2.1)"] [[package]] name = "frozenlist" @@ -785,94 +385,14 @@ description = "A list-like structure which implements collections.abc.MutableSeq category = "main" optional = true python-versions = ">=3.7" -files = [ - {file = "frozenlist-1.3.3-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:ff8bf625fe85e119553b5383ba0fb6aa3d0ec2ae980295aaefa552374926b3f4"}, - {file = "frozenlist-1.3.3-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:dfbac4c2dfcc082fcf8d942d1e49b6aa0766c19d3358bd86e2000bf0fa4a9cf0"}, - {file = "frozenlist-1.3.3-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:b1c63e8d377d039ac769cd0926558bb7068a1f7abb0f003e3717ee003ad85530"}, - {file = "frozenlist-1.3.3-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:7fdfc24dcfce5b48109867c13b4cb15e4660e7bd7661741a391f821f23dfdca7"}, - {file = "frozenlist-1.3.3-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:2c926450857408e42f0bbc295e84395722ce74bae69a3b2aa2a65fe22cb14b99"}, - {file = "frozenlist-1.3.3-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:1841e200fdafc3d51f974d9d377c079a0694a8f06de2e67b48150328d66d5483"}, - {file = "frozenlist-1.3.3-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:f470c92737afa7d4c3aacc001e335062d582053d4dbe73cda126f2d7031068dd"}, - {file = "frozenlist-1.3.3-cp310-cp310-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:783263a4eaad7c49983fe4b2e7b53fa9770c136c270d2d4bbb6d2192bf4d9caf"}, - {file = "frozenlist-1.3.3-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:924620eef691990dfb56dc4709f280f40baee568c794b5c1885800c3ecc69816"}, - {file = "frozenlist-1.3.3-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:ae4dc05c465a08a866b7a1baf360747078b362e6a6dbeb0c57f234db0ef88ae0"}, - {file = "frozenlist-1.3.3-cp310-cp310-musllinux_1_1_ppc64le.whl", hash = "sha256:bed331fe18f58d844d39ceb398b77d6ac0b010d571cba8267c2e7165806b00ce"}, - {file = "frozenlist-1.3.3-cp310-cp310-musllinux_1_1_s390x.whl", hash = "sha256:02c9ac843e3390826a265e331105efeab489ffaf4dd86384595ee8ce6d35ae7f"}, - {file = "frozenlist-1.3.3-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:9545a33965d0d377b0bc823dcabf26980e77f1b6a7caa368a365a9497fb09420"}, - {file = "frozenlist-1.3.3-cp310-cp310-win32.whl", hash = "sha256:d5cd3ab21acbdb414bb6c31958d7b06b85eeb40f66463c264a9b343a4e238642"}, - {file = "frozenlist-1.3.3-cp310-cp310-win_amd64.whl", hash = "sha256:b756072364347cb6aa5b60f9bc18e94b2f79632de3b0190253ad770c5df17db1"}, - {file = "frozenlist-1.3.3-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:b4395e2f8d83fbe0c627b2b696acce67868793d7d9750e90e39592b3626691b7"}, - {file = "frozenlist-1.3.3-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:14143ae966a6229350021384870458e4777d1eae4c28d1a7aa47f24d030e6678"}, - {file = "frozenlist-1.3.3-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:5d8860749e813a6f65bad8285a0520607c9500caa23fea6ee407e63debcdbef6"}, - {file = "frozenlist-1.3.3-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:23d16d9f477bb55b6154654e0e74557040575d9d19fe78a161bd33d7d76808e8"}, - {file = "frozenlist-1.3.3-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:eb82dbba47a8318e75f679690190c10a5e1f447fbf9df41cbc4c3afd726d88cb"}, - {file = "frozenlist-1.3.3-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:9309869032abb23d196cb4e4db574232abe8b8be1339026f489eeb34a4acfd91"}, - {file = "frozenlist-1.3.3-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:a97b4fe50b5890d36300820abd305694cb865ddb7885049587a5678215782a6b"}, - {file = "frozenlist-1.3.3-cp311-cp311-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:c188512b43542b1e91cadc3c6c915a82a5eb95929134faf7fd109f14f9892ce4"}, - {file = "frozenlist-1.3.3-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:303e04d422e9b911a09ad499b0368dc551e8c3cd15293c99160c7f1f07b59a48"}, - {file = "frozenlist-1.3.3-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:0771aed7f596c7d73444c847a1c16288937ef988dc04fb9f7be4b2aa91db609d"}, - {file = "frozenlist-1.3.3-cp311-cp311-musllinux_1_1_ppc64le.whl", hash = "sha256:66080ec69883597e4d026f2f71a231a1ee9887835902dbe6b6467d5a89216cf6"}, - {file = "frozenlist-1.3.3-cp311-cp311-musllinux_1_1_s390x.whl", hash = "sha256:41fe21dc74ad3a779c3d73a2786bdf622ea81234bdd4faf90b8b03cad0c2c0b4"}, - {file = "frozenlist-1.3.3-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:f20380df709d91525e4bee04746ba612a4df0972c1b8f8e1e8af997e678c7b81"}, - {file = "frozenlist-1.3.3-cp311-cp311-win32.whl", hash = "sha256:f30f1928162e189091cf4d9da2eac617bfe78ef907a761614ff577ef4edfb3c8"}, - {file = "frozenlist-1.3.3-cp311-cp311-win_amd64.whl", hash = "sha256:a6394d7dadd3cfe3f4b3b186e54d5d8504d44f2d58dcc89d693698e8b7132b32"}, - {file = "frozenlist-1.3.3-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:8df3de3a9ab8325f94f646609a66cbeeede263910c5c0de0101079ad541af332"}, - {file = "frozenlist-1.3.3-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:0693c609e9742c66ba4870bcee1ad5ff35462d5ffec18710b4ac89337ff16e27"}, - {file = "frozenlist-1.3.3-cp37-cp37m-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:cd4210baef299717db0a600d7a3cac81d46ef0e007f88c9335db79f8979c0d3d"}, - {file = "frozenlist-1.3.3-cp37-cp37m-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:394c9c242113bfb4b9aa36e2b80a05ffa163a30691c7b5a29eba82e937895d5e"}, - {file = "frozenlist-1.3.3-cp37-cp37m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:6327eb8e419f7d9c38f333cde41b9ae348bec26d840927332f17e887a8dcb70d"}, - {file = "frozenlist-1.3.3-cp37-cp37m-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:2e24900aa13212e75e5b366cb9065e78bbf3893d4baab6052d1aca10d46d944c"}, - {file = "frozenlist-1.3.3-cp37-cp37m-musllinux_1_1_aarch64.whl", hash = "sha256:3843f84a6c465a36559161e6c59dce2f2ac10943040c2fd021cfb70d58c4ad56"}, - {file = "frozenlist-1.3.3-cp37-cp37m-musllinux_1_1_i686.whl", hash = "sha256:84610c1502b2461255b4c9b7d5e9c48052601a8957cd0aea6ec7a7a1e1fb9420"}, - {file = "frozenlist-1.3.3-cp37-cp37m-musllinux_1_1_ppc64le.whl", hash = "sha256:c21b9aa40e08e4f63a2f92ff3748e6b6c84d717d033c7b3438dd3123ee18f70e"}, - {file = "frozenlist-1.3.3-cp37-cp37m-musllinux_1_1_s390x.whl", hash = "sha256:efce6ae830831ab6a22b9b4091d411698145cb9b8fc869e1397ccf4b4b6455cb"}, - {file = "frozenlist-1.3.3-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:40de71985e9042ca00b7953c4f41eabc3dc514a2d1ff534027f091bc74416401"}, - {file = "frozenlist-1.3.3-cp37-cp37m-win32.whl", hash = "sha256:180c00c66bde6146a860cbb81b54ee0df350d2daf13ca85b275123bbf85de18a"}, - {file = "frozenlist-1.3.3-cp37-cp37m-win_amd64.whl", hash = "sha256:9bbbcedd75acdfecf2159663b87f1bb5cfc80e7cd99f7ddd9d66eb98b14a8411"}, - {file = "frozenlist-1.3.3-cp38-cp38-macosx_10_9_universal2.whl", hash = "sha256:034a5c08d36649591be1cbb10e09da9f531034acfe29275fc5454a3b101ce41a"}, - {file = "frozenlist-1.3.3-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:ba64dc2b3b7b158c6660d49cdb1d872d1d0bf4e42043ad8d5006099479a194e5"}, - {file = "frozenlist-1.3.3-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:47df36a9fe24054b950bbc2db630d508cca3aa27ed0566c0baf661225e52c18e"}, - {file = "frozenlist-1.3.3-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:008a054b75d77c995ea26629ab3a0c0d7281341f2fa7e1e85fa6153ae29ae99c"}, - {file = "frozenlist-1.3.3-cp38-cp38-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:841ea19b43d438a80b4de62ac6ab21cfe6827bb8a9dc62b896acc88eaf9cecba"}, - {file = "frozenlist-1.3.3-cp38-cp38-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:e235688f42b36be2b6b06fc37ac2126a73b75fb8d6bc66dd632aa35286238703"}, - {file = "frozenlist-1.3.3-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:ca713d4af15bae6e5d79b15c10c8522859a9a89d3b361a50b817c98c2fb402a2"}, - {file = "frozenlist-1.3.3-cp38-cp38-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:9ac5995f2b408017b0be26d4a1d7c61bce106ff3d9e3324374d66b5964325448"}, - {file = "frozenlist-1.3.3-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:a4ae8135b11652b08a8baf07631d3ebfe65a4c87909dbef5fa0cdde440444ee4"}, - {file = "frozenlist-1.3.3-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:4ea42116ceb6bb16dbb7d526e242cb6747b08b7710d9782aa3d6732bd8d27649"}, - {file = "frozenlist-1.3.3-cp38-cp38-musllinux_1_1_ppc64le.whl", hash = "sha256:810860bb4bdce7557bc0febb84bbd88198b9dbc2022d8eebe5b3590b2ad6c842"}, - {file = "frozenlist-1.3.3-cp38-cp38-musllinux_1_1_s390x.whl", hash = "sha256:ee78feb9d293c323b59a6f2dd441b63339a30edf35abcb51187d2fc26e696d13"}, - {file = "frozenlist-1.3.3-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:0af2e7c87d35b38732e810befb9d797a99279cbb85374d42ea61c1e9d23094b3"}, - {file = "frozenlist-1.3.3-cp38-cp38-win32.whl", hash = "sha256:899c5e1928eec13fd6f6d8dc51be23f0d09c5281e40d9cf4273d188d9feeaf9b"}, - {file = "frozenlist-1.3.3-cp38-cp38-win_amd64.whl", hash = "sha256:7f44e24fa70f6fbc74aeec3e971f60a14dde85da364aa87f15d1be94ae75aeef"}, - {file = "frozenlist-1.3.3-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:2b07ae0c1edaa0a36339ec6cce700f51b14a3fc6545fdd32930d2c83917332cf"}, - {file = "frozenlist-1.3.3-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:ebb86518203e12e96af765ee89034a1dbb0c3c65052d1b0c19bbbd6af8a145e1"}, - {file = "frozenlist-1.3.3-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:5cf820485f1b4c91e0417ea0afd41ce5cf5965011b3c22c400f6d144296ccbc0"}, - {file = "frozenlist-1.3.3-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:5c11e43016b9024240212d2a65043b70ed8dfd3b52678a1271972702d990ac6d"}, - {file = "frozenlist-1.3.3-cp39-cp39-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:8fa3c6e3305aa1146b59a09b32b2e04074945ffcfb2f0931836d103a2c38f936"}, - {file = "frozenlist-1.3.3-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:352bd4c8c72d508778cf05ab491f6ef36149f4d0cb3c56b1b4302852255d05d5"}, - {file = "frozenlist-1.3.3-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:65a5e4d3aa679610ac6e3569e865425b23b372277f89b5ef06cf2cdaf1ebf22b"}, - {file = "frozenlist-1.3.3-cp39-cp39-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:b1e2c1185858d7e10ff045c496bbf90ae752c28b365fef2c09cf0fa309291669"}, - {file = "frozenlist-1.3.3-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:f163d2fd041c630fed01bc48d28c3ed4a3b003c00acd396900e11ee5316b56bb"}, - {file = "frozenlist-1.3.3-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:05cdb16d09a0832eedf770cb7bd1fe57d8cf4eaf5aced29c4e41e3f20b30a784"}, - {file = "frozenlist-1.3.3-cp39-cp39-musllinux_1_1_ppc64le.whl", hash = "sha256:8bae29d60768bfa8fb92244b74502b18fae55a80eac13c88eb0b496d4268fd2d"}, - {file = "frozenlist-1.3.3-cp39-cp39-musllinux_1_1_s390x.whl", hash = "sha256:eedab4c310c0299961ac285591acd53dc6723a1ebd90a57207c71f6e0c2153ab"}, - {file = "frozenlist-1.3.3-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:3bbdf44855ed8f0fbcd102ef05ec3012d6a4fd7c7562403f76ce6a52aeffb2b1"}, - {file = "frozenlist-1.3.3-cp39-cp39-win32.whl", hash = "sha256:efa568b885bca461f7c7b9e032655c0c143d305bf01c30caf6db2854a4532b38"}, - {file = "frozenlist-1.3.3-cp39-cp39-win_amd64.whl", hash = "sha256:cfe33efc9cb900a4c46f91a5ceba26d6df370ffddd9ca386eb1d4f0ad97b9ea9"}, - {file = "frozenlist-1.3.3.tar.gz", hash = "sha256:58bcc55721e8a90b88332d6cd441261ebb22342e238296bb330968952fbb3a6a"}, -] [[package]] name = "fsspec" -version = "2022.10.0" +version = "2023.1.0" description = "File-system specification" category = "main" optional = false python-versions = ">=3.7" -files = [ - {file = "fsspec-2022.10.0-py3-none-any.whl", hash = "sha256:6b7c6ab3b476cdf17efcfeccde7fca28ef5a48f73a71010aaceec5fc15bf9ebf"}, - {file = "fsspec-2022.10.0.tar.gz", hash = "sha256:cb6092474e90487a51de768170f3afa50ca8982c26150a59072b16433879ff1d"}, -] [package.extras] abfs = ["adlfs"] @@ -899,15 +419,11 @@ tqdm = ["tqdm"] [[package]] name = "identify" -version = "2.5.10" +version = "2.5.13" description = "File identification library for Python" category = "dev" optional = false python-versions = ">=3.7" -files = [ - {file = "identify-2.5.10-py2.py3-none-any.whl", hash = "sha256:fb7c2feaeca6976a3ffa31ec3236a6911fbc51aec9acc111de2aed99f244ade2"}, - {file = "identify-2.5.10.tar.gz", hash = "sha256:dce9e31fee7dbc45fea36a9e855c316b8fbf807e65a862f160840bb5a2bf5dfd"}, -] [package.extras] license = ["ukkonen"] @@ -919,42 +435,30 @@ description = "Internationalized Domain Names in Applications (IDNA)" category = "main" optional = false python-versions = ">=3.5" -files = [ - {file = "idna-3.4-py3-none-any.whl", hash = "sha256:90b77e79eaa3eba6de819a0c442c0b4ceefc341a7a2ab77d7562bf49f425c5c2"}, - {file = "idna-3.4.tar.gz", hash = "sha256:814f528e8dead7d329833b91c5faa87d60bf71824cd12a7530b5526063d02cb4"}, -] [[package]] name = "importlib-metadata" -version = "5.1.0" +version = "6.0.0" description = "Read metadata from Python packages" category = "dev" optional = false python-versions = ">=3.7" -files = [ - {file = "importlib_metadata-5.1.0-py3-none-any.whl", hash = "sha256:d84d17e21670ec07990e1044a99efe8d615d860fd176fc29ef5c306068fda313"}, - {file = "importlib_metadata-5.1.0.tar.gz", hash = "sha256:d5059f9f1e8e41f80e9c56c2ee58811450c31984dfa625329ffd7c0dad88a73b"}, -] [package.dependencies] zipp = ">=0.5" [package.extras] -docs = ["furo", "jaraco.packaging (>=9)", "jaraco.tidelift (>=1.4)", "rst.linker (>=1.9)", "sphinx (>=3.5)"] +docs = ["furo", "jaraco.packaging (>=9)", "jaraco.tidelift (>=1.4)", "rst.linker (>=1.9)", "sphinx (>=3.5)", "sphinx-lint"] perf = ["ipython"] testing = ["flake8 (<5)", "flufl.flake8", "importlib-resources (>=1.3)", "packaging", "pyfakefs", "pytest (>=6)", "pytest-black (>=0.3.7)", "pytest-checkdocs (>=2.4)", "pytest-cov", "pytest-enabler (>=1.3)", "pytest-flake8", "pytest-mypy (>=0.9.1)", "pytest-perf (>=0.9.2)"] [[package]] name = "iniconfig" -version = "1.1.1" -description = "iniconfig: brain-dead simple config-ini parsing" +version = "2.0.0" +description = "brain-dead simple config-ini parsing" category = "dev" optional = false -python-versions = "*" -files = [ - {file = "iniconfig-1.1.1-py2.py3-none-any.whl", hash = "sha256:011e24c64b7f47f6ebd835bb12a743f2fbe9a26d4cecaa7f53bc4f35ee9da8b3"}, - {file = "iniconfig-1.1.1.tar.gz", hash = "sha256:bc3af051d7d14b2ee5ef9969666def0cd1a000e121eaea580d4a313df4b37f32"}, -] +python-versions = ">=3.7" [[package]] name = "isodate" @@ -963,10 +467,6 @@ description = "An ISO 8601 date/time/duration parser and formatter" category = "main" optional = true python-versions = "*" -files = [ - {file = "isodate-0.6.1-py2.py3-none-any.whl", hash = "sha256:0751eece944162659049d35f4f549ed815792b38793f07cf73381c1c87cbed96"}, - {file = "isodate-0.6.1.tar.gz", hash = "sha256:48c5881de7e8b0a0d648cb024c8062dc84e7b840ed81e864c7614fd3c127bde9"}, -] [package.dependencies] six = "*" @@ -978,10 +478,6 @@ description = "A very fast and expressive template engine." category = "dev" optional = false python-versions = ">=3.7" -files = [ - {file = "Jinja2-3.1.2-py3-none-any.whl", hash = "sha256:6088930bfe239f0e6710546ab9c19c9ef35e29792895fed6e6e31a023a182a61"}, - {file = "Jinja2-3.1.2.tar.gz", hash = "sha256:31351a702a408a9e7595a8fc6150fc3f43bb6bf7e319770cbc0db9df9437e852"}, -] [package.dependencies] MarkupSafe = ">=2.0" @@ -996,10 +492,6 @@ description = "JSON Matching Expressions" category = "main" optional = false python-versions = ">=3.7" -files = [ - {file = "jmespath-1.0.1-py3-none-any.whl", hash = "sha256:02e2e4cc71b5bcab88332eebf907519190dd9e6e82107fa7f83b1003a6252980"}, - {file = "jmespath-1.0.1.tar.gz", hash = "sha256:90261b206d6defd58fdd5e85f478bf633a2901798906be2ad389150c5c60edbe"}, -] [[package]] name = "markdown-it-py" @@ -1008,73 +500,27 @@ description = "Python port of markdown-it. Markdown parsing, done right!" category = "main" optional = false python-versions = ">=3.7" -files = [ - {file = "markdown-it-py-2.1.0.tar.gz", hash = "sha256:cf7e59fed14b5ae17c0006eff14a2d9a00ed5f3a846148153899a0224e2c07da"}, - {file = "markdown_it_py-2.1.0-py3-none-any.whl", hash = "sha256:93de681e5c021a432c63147656fe21790bc01231e0cd2da73626f1aa3ac0fe27"}, -] [package.dependencies] mdurl = ">=0.1,<1.0" [package.extras] benchmarking = ["psutil", "pytest", "pytest-benchmark (>=3.2,<4.0)"] -code-style = ["pre-commit (==2.6)"] +code_style = ["pre-commit (==2.6)"] compare = ["commonmark (>=0.9.1,<0.10.0)", "markdown (>=3.3.6,<3.4.0)", "mistletoe (>=0.8.1,<0.9.0)", "mistune (>=2.0.2,<2.1.0)", "panflute (>=2.1.3,<2.2.0)"] linkify = ["linkify-it-py (>=1.0,<2.0)"] plugins = ["mdit-py-plugins"] profiling = ["gprof2dot"] -rtd = ["attrs", "myst-parser", "pyyaml", "sphinx", "sphinx-copybutton", "sphinx-design", "sphinx_book_theme"] +rtd = ["attrs", "myst-parser", "pyyaml", "sphinx", "sphinx-book-theme", "sphinx-copybutton", "sphinx-design"] testing = ["coverage", "pytest", "pytest-cov", "pytest-regressions"] [[package]] name = "markupsafe" -version = "2.1.1" +version = "2.1.2" description = "Safely add untrusted strings to HTML/XML markup." category = "dev" optional = false python-versions = ">=3.7" -files = [ - {file = "MarkupSafe-2.1.1-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:86b1f75c4e7c2ac2ccdaec2b9022845dbb81880ca318bb7a0a01fbf7813e3812"}, - {file = "MarkupSafe-2.1.1-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:f121a1420d4e173a5d96e47e9a0c0dcff965afdf1626d28de1460815f7c4ee7a"}, - {file = "MarkupSafe-2.1.1-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a49907dd8420c5685cfa064a1335b6754b74541bbb3706c259c02ed65b644b3e"}, - {file = "MarkupSafe-2.1.1-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:10c1bfff05d95783da83491be968e8fe789263689c02724e0c691933c52994f5"}, - {file = "MarkupSafe-2.1.1-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:b7bd98b796e2b6553da7225aeb61f447f80a1ca64f41d83612e6139ca5213aa4"}, - {file = "MarkupSafe-2.1.1-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:b09bf97215625a311f669476f44b8b318b075847b49316d3e28c08e41a7a573f"}, - {file = "MarkupSafe-2.1.1-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:694deca8d702d5db21ec83983ce0bb4b26a578e71fbdbd4fdcd387daa90e4d5e"}, - {file = "MarkupSafe-2.1.1-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:efc1913fd2ca4f334418481c7e595c00aad186563bbc1ec76067848c7ca0a933"}, - {file = "MarkupSafe-2.1.1-cp310-cp310-win32.whl", hash = "sha256:4a33dea2b688b3190ee12bd7cfa29d39c9ed176bda40bfa11099a3ce5d3a7ac6"}, - {file = "MarkupSafe-2.1.1-cp310-cp310-win_amd64.whl", hash = "sha256:dda30ba7e87fbbb7eab1ec9f58678558fd9a6b8b853530e176eabd064da81417"}, - {file = "MarkupSafe-2.1.1-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:671cd1187ed5e62818414afe79ed29da836dde67166a9fac6d435873c44fdd02"}, - {file = "MarkupSafe-2.1.1-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:3799351e2336dc91ea70b034983ee71cf2f9533cdff7c14c90ea126bfd95d65a"}, - {file = "MarkupSafe-2.1.1-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:e72591e9ecd94d7feb70c1cbd7be7b3ebea3f548870aa91e2732960fa4d57a37"}, - {file = "MarkupSafe-2.1.1-cp37-cp37m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:6fbf47b5d3728c6aea2abb0589b5d30459e369baa772e0f37a0320185e87c980"}, - {file = "MarkupSafe-2.1.1-cp37-cp37m-musllinux_1_1_aarch64.whl", hash = "sha256:d5ee4f386140395a2c818d149221149c54849dfcfcb9f1debfe07a8b8bd63f9a"}, - {file = "MarkupSafe-2.1.1-cp37-cp37m-musllinux_1_1_i686.whl", hash = "sha256:bcb3ed405ed3222f9904899563d6fc492ff75cce56cba05e32eff40e6acbeaa3"}, - {file = "MarkupSafe-2.1.1-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:e1c0b87e09fa55a220f058d1d49d3fb8df88fbfab58558f1198e08c1e1de842a"}, - {file = "MarkupSafe-2.1.1-cp37-cp37m-win32.whl", hash = "sha256:8dc1c72a69aa7e082593c4a203dcf94ddb74bb5c8a731e4e1eb68d031e8498ff"}, - {file = "MarkupSafe-2.1.1-cp37-cp37m-win_amd64.whl", hash = "sha256:97a68e6ada378df82bc9f16b800ab77cbf4b2fada0081794318520138c088e4a"}, - {file = "MarkupSafe-2.1.1-cp38-cp38-macosx_10_9_universal2.whl", hash = "sha256:e8c843bbcda3a2f1e3c2ab25913c80a3c5376cd00c6e8c4a86a89a28c8dc5452"}, - {file = "MarkupSafe-2.1.1-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:0212a68688482dc52b2d45013df70d169f542b7394fc744c02a57374a4207003"}, - {file = "MarkupSafe-2.1.1-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:8e576a51ad59e4bfaac456023a78f6b5e6e7651dcd383bcc3e18d06f9b55d6d1"}, - {file = "MarkupSafe-2.1.1-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:4b9fe39a2ccc108a4accc2676e77da025ce383c108593d65cc909add5c3bd601"}, - {file = "MarkupSafe-2.1.1-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:96e37a3dc86e80bf81758c152fe66dbf60ed5eca3d26305edf01892257049925"}, - {file = "MarkupSafe-2.1.1-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:6d0072fea50feec76a4c418096652f2c3238eaa014b2f94aeb1d56a66b41403f"}, - {file = "MarkupSafe-2.1.1-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:089cf3dbf0cd6c100f02945abeb18484bd1ee57a079aefd52cffd17fba910b88"}, - {file = "MarkupSafe-2.1.1-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:6a074d34ee7a5ce3effbc526b7083ec9731bb3cbf921bbe1d3005d4d2bdb3a63"}, - {file = "MarkupSafe-2.1.1-cp38-cp38-win32.whl", hash = "sha256:421be9fbf0ffe9ffd7a378aafebbf6f4602d564d34be190fc19a193232fd12b1"}, - {file = "MarkupSafe-2.1.1-cp38-cp38-win_amd64.whl", hash = "sha256:fc7b548b17d238737688817ab67deebb30e8073c95749d55538ed473130ec0c7"}, - {file = "MarkupSafe-2.1.1-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:e04e26803c9c3851c931eac40c695602c6295b8d432cbe78609649ad9bd2da8a"}, - {file = "MarkupSafe-2.1.1-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:b87db4360013327109564f0e591bd2a3b318547bcef31b468a92ee504d07ae4f"}, - {file = "MarkupSafe-2.1.1-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:99a2a507ed3ac881b975a2976d59f38c19386d128e7a9a18b7df6fff1fd4c1d6"}, - {file = "MarkupSafe-2.1.1-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:56442863ed2b06d19c37f94d999035e15ee982988920e12a5b4ba29b62ad1f77"}, - {file = "MarkupSafe-2.1.1-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:3ce11ee3f23f79dbd06fb3d63e2f6af7b12db1d46932fe7bd8afa259a5996603"}, - {file = "MarkupSafe-2.1.1-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:33b74d289bd2f5e527beadcaa3f401e0df0a89927c1559c8566c066fa4248ab7"}, - {file = "MarkupSafe-2.1.1-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:43093fb83d8343aac0b1baa75516da6092f58f41200907ef92448ecab8825135"}, - {file = "MarkupSafe-2.1.1-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:8e3dcf21f367459434c18e71b2a9532d96547aef8a871872a5bd69a715c15f96"}, - {file = "MarkupSafe-2.1.1-cp39-cp39-win32.whl", hash = "sha256:d4306c36ca495956b6d568d276ac11fdd9c30a36f1b6eb928070dc5360b22e1c"}, - {file = "MarkupSafe-2.1.1-cp39-cp39-win_amd64.whl", hash = "sha256:46d00d6cfecdde84d40e572d63735ef81423ad31184100411e6e3388d405e247"}, - {file = "MarkupSafe-2.1.1.tar.gz", hash = "sha256:7f91197cc9e48f989d12e4e6fbc46495c446636dfc81b9ccf50bb0ec74b91d4b"}, -] [[package]] name = "mdurl" @@ -1083,10 +529,6 @@ description = "Markdown URL utilities" category = "main" optional = false python-versions = ">=3.7" -files = [ - {file = "mdurl-0.1.2-py3-none-any.whl", hash = "sha256:84008a41e51615a49fc9966191ff91509e3c40b939176e643fd50a5c2196b8f8"}, - {file = "mdurl-0.1.2.tar.gz", hash = "sha256:bb413d29f5eea38f31dd4754dd7377d4465116fb207585f97bf925588687c1ba"}, -] [[package]] name = "mmhash3" @@ -1095,42 +537,6 @@ description = "Python wrapper for MurmurHash (MurmurHash3), a set of fast and ro category = "main" optional = false python-versions = "*" -files = [ - {file = "mmhash3-3.0.1-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:47deea30cd8d3d5cd52dc740902a4c70383bfe8248eac29d0877fe63e03c2713"}, - {file = "mmhash3-3.0.1-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:ecdaf4d1de617818bf05cd526ca558db6010beeba7ea9e19f695f2bdcac0e0a4"}, - {file = "mmhash3-3.0.1-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:4675585617584e9e1aafa3a90ac0ac9a437257c507748d97de8b21977e9d6745"}, - {file = "mmhash3-3.0.1-cp310-cp310-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:ebfd0c2af09b41f0fe1dd260799bda90a0fc7eba4477ccaeb3951527700cd58f"}, - {file = "mmhash3-3.0.1-cp310-cp310-win32.whl", hash = "sha256:68587dec7b8acdb7528fd511e295d8b5ccfe26022923a69867e1822f0fdb4c44"}, - {file = "mmhash3-3.0.1-cp310-cp310-win_amd64.whl", hash = "sha256:54954ebe3a614f19ab4cfe24fa66ea093c493d9fac0533d002dd64c2540a0c99"}, - {file = "mmhash3-3.0.1-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:b172f3bd3220b0bf56fd7cc760fd8a9033def47103611d63fe867003079a1256"}, - {file = "mmhash3-3.0.1-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:de7895eafabc32f7c0c09a81a624921f536768de6e438e3de69e3e954a4d7072"}, - {file = "mmhash3-3.0.1-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:b4b0914effe4ddd8d33149e3508564c17719465b0cc81691c4fa50d5e0e14f80"}, - {file = "mmhash3-3.0.1-cp311-cp311-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:c0575050ac691475938df1ad03d8738c5bd1eadef62093e76157ebb7f2df0946"}, - {file = "mmhash3-3.0.1-cp311-cp311-win32.whl", hash = "sha256:22f92f0f88f28b215357acd346362fa9f7c9fffb436beb42cc4b442b676dbaa3"}, - {file = "mmhash3-3.0.1-cp311-cp311-win_amd64.whl", hash = "sha256:538240ab7936bf71b18304e5a7e7fd3c4c2fab103330ea99584bb4f777299a2b"}, - {file = "mmhash3-3.0.1-cp36-cp36m-macosx_10_9_x86_64.whl", hash = "sha256:ca791bfb311e36ce13998e4632262ed4b95da9d3461941e18b6690760171a045"}, - {file = "mmhash3-3.0.1-cp36-cp36m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:b41708f72c6aa2a49ada1f0b61e85c05cd8389ad31d463fd5bca29999a4d5f9c"}, - {file = "mmhash3-3.0.1-cp36-cp36m-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:a3ce9b4533ddc0a88ba045a27309714c3b127bd05e57fd252d1d5a71d4247ea7"}, - {file = "mmhash3-3.0.1-cp36-cp36m-win32.whl", hash = "sha256:bfafeb96fdeb10db8767d06e1f07b6fdcddba4aaa0dd15058561a49f7ae45345"}, - {file = "mmhash3-3.0.1-cp36-cp36m-win_amd64.whl", hash = "sha256:97fe077b24c948709ed2afc749bf6285d407bc54ff12c63d2dc86678c38a0b8e"}, - {file = "mmhash3-3.0.1-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:0cfd91ccd5fca1ba7ee925297131a15dfb94c714bfe6ba0fb3b1ca78b12bbfec"}, - {file = "mmhash3-3.0.1-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:6d51b1005233141ce7394531af40a3f0fc1f274467bf8dff44dcf7987924fe58"}, - {file = "mmhash3-3.0.1-cp37-cp37m-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:855c67b100e37df166acf79cdff58fa8f9f6c48be0d1e1b6e9ad0fa34a9661ef"}, - {file = "mmhash3-3.0.1-cp37-cp37m-win32.whl", hash = "sha256:bb3030df1334fd665427f8be8e8ce4f04aeab7f6010ce4f2c128f0099bdab96f"}, - {file = "mmhash3-3.0.1-cp37-cp37m-win_amd64.whl", hash = "sha256:1545e1177294afe4912d5a5e401c7fa9b799dd109b30289e7af74d5b07e7c474"}, - {file = "mmhash3-3.0.1-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:2479899e7dda834a671991a1098a691ab1c2eaa20c3e939d691ca4a19361cfe0"}, - {file = "mmhash3-3.0.1-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:c9056196d5e3d3d844433a63d806a683f710ab3aaf1c910550c7746464bc43ae"}, - {file = "mmhash3-3.0.1-cp38-cp38-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:a0d4c307af0bf70207305f70f131898be071d1b19a89f462b13487f5c25e8d4e"}, - {file = "mmhash3-3.0.1-cp38-cp38-win32.whl", hash = "sha256:5f885f65e329fd14bc38debac4a79eacf381e856965d9c65c4d1c6946ea190d0"}, - {file = "mmhash3-3.0.1-cp38-cp38-win_amd64.whl", hash = "sha256:3b42d0bda5e1cd22c18b16887b0521060fb59d0aaaaf033feacbc0a2492d20fe"}, - {file = "mmhash3-3.0.1-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:d3f333286bb87aa9dc6bd8e7960a55a27b011a401f24b889a50e6d219f65e7c9"}, - {file = "mmhash3-3.0.1-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:6b7ef2eb95a18bcd02ce0d3e047adde3a025afd96c1d266a8a0d44574f44a307"}, - {file = "mmhash3-3.0.1-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:d6ac8a5f511c60f341bf9cae462bb4941abb149d98464ba5f4f4548875b601c6"}, - {file = "mmhash3-3.0.1-cp39-cp39-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:efef9e632e6e248e46f52d108a5ebd1f28edaf427b7fd47ebf97dbff4b2cab81"}, - {file = "mmhash3-3.0.1-cp39-cp39-win32.whl", hash = "sha256:bdac06d72e448c67afb12e758b203d875e29d4097bb279a38a5649d44b518ba7"}, - {file = "mmhash3-3.0.1-cp39-cp39-win_amd64.whl", hash = "sha256:0baeaa20cac5f75ed14f28056826bd9d9c8b2354b382073f3fd5190708992a0d"}, - {file = "mmhash3-3.0.1.tar.gz", hash = "sha256:a00d68f4a1cc434b9501513c8a29e18ed1ddad383677d72b41d71d0d862348af"}, -] [[package]] name = "moto" @@ -1139,10 +545,6 @@ description = "" category = "dev" optional = false python-versions = ">=3.7" -files = [ - {file = "moto-4.1.0-py2.py3-none-any.whl", hash = "sha256:12f231196ebf22d19774f2a75738fdedae10a8c82abb8fa0877e1819aa799f4f"}, - {file = "moto-4.1.0.tar.gz", hash = "sha256:e65a692df57abb631b06c8f3f7827c9deceae3416887527e266c775ad722a2fa"}, -] [package.dependencies] boto3 = ">=1.9.201" @@ -1185,10 +587,6 @@ description = "The Microsoft Authentication Library (MSAL) for Python library en category = "main" optional = true python-versions = "*" -files = [ - {file = "msal-1.20.0-py2.py3-none-any.whl", hash = "sha256:d2f1c26368ecdc28c8657d457352faa0b81b1845a7b889d8676787721ba86792"}, - {file = "msal-1.20.0.tar.gz", hash = "sha256:78344cd4c91d6134a593b5e3e45541e666e37b747ff8a6316c3668dd1e6ab6b2"}, -] [package.dependencies] cryptography = ">=0.6,<41" @@ -1205,10 +603,6 @@ description = "Microsoft Authentication Library extensions (MSAL EX) provides a category = "main" optional = true python-versions = "*" -files = [ - {file = "msal-extensions-1.0.0.tar.gz", hash = "sha256:c676aba56b0cce3783de1b5c5ecfe828db998167875126ca4b47dc6436451354"}, - {file = "msal_extensions-1.0.0-py2.py3-none-any.whl", hash = "sha256:91e3db9620b822d0ed2b4d1850056a0f133cba04455e62f11612e40f5502f2ee"}, -] [package.dependencies] msal = ">=0.4.1,<2.0.0" @@ -1224,10 +618,6 @@ description = "AutoRest swagger generator Python client runtime." category = "main" optional = true python-versions = ">=3.6" -files = [ - {file = "msrest-0.7.1-py3-none-any.whl", hash = "sha256:21120a810e1233e5e6cc7fe40b474eeb4ec6f757a15d7cf86702c369f9567c32"}, - {file = "msrest-0.7.1.zip", hash = "sha256:6e7661f46f3afd88b75667b7187a92829924446c7ea1d169be8c4bb7eeb788b9"}, -] [package.dependencies] azure-core = ">=1.24.0" @@ -1241,87 +631,11 @@ async = ["aiodns", "aiohttp (>=3.0)"] [[package]] name = "multidict" -version = "6.0.3" +version = "6.0.4" description = "multidict implementation" category = "main" optional = true python-versions = ">=3.7" -files = [ - {file = "multidict-6.0.3-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:73009ea04205966d47e16d98686ac5c438af23a1bb30b48a2c5da3423ec9ce37"}, - {file = "multidict-6.0.3-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:8b92a9f3ab904397a33b193000dc4de7318ea175c4c460a1e154c415f9008e3d"}, - {file = "multidict-6.0.3-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:578bfcb16f4b8675ef71b960c00f174b0426e0eeb796bab6737389d8288eb827"}, - {file = "multidict-6.0.3-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:f1650ea41c408755da5eed52ac6ccbc8938ccc3e698d81e6f6a1be02ff2a0945"}, - {file = "multidict-6.0.3-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:d52442e7c951e4c9ee591d6047706e66923d248d83958bbf99b8b19515fffaef"}, - {file = "multidict-6.0.3-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:ad7d66422b9cc51125509229693d27e18c08f2dea3ac9de408d821932b1b3759"}, - {file = "multidict-6.0.3-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:6cd14e61f0da2a2cfb9fe05bfced2a1ed7063ce46a7a8cd473be4973de9a7f91"}, - {file = "multidict-6.0.3-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:190626ced82d4cc567a09e7346340d380154a493bac6905e0095d8158cdf1e38"}, - {file = "multidict-6.0.3-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:791458a1f7d1b4ab3bd9e93e0dcd1d59ef7ee9aa051dcd1ea030e62e49b923fd"}, - {file = "multidict-6.0.3-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:b46e79a9f4db53897d17bc64a39d1c7c2be3e3d4f8dba6d6730a2b13ddf0f986"}, - {file = "multidict-6.0.3-cp310-cp310-musllinux_1_1_ppc64le.whl", hash = "sha256:e4a095e18847c12ec20e55326ab8782d9c2d599400a3a2f174fab4796875d0e2"}, - {file = "multidict-6.0.3-cp310-cp310-musllinux_1_1_s390x.whl", hash = "sha256:fb6c3dc3d65014d2c782f5acf0b3ba14e639c6c33d3ed8932ead76b9080b3544"}, - {file = "multidict-6.0.3-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:3541882266247c7cd3dba78d6ef28dbe704774df60c9e4231edaa4493522e614"}, - {file = "multidict-6.0.3-cp310-cp310-win32.whl", hash = "sha256:67090b17a0a5be5704fd109f231ee73cefb1b3802d41288d6378b5df46ae89ba"}, - {file = "multidict-6.0.3-cp310-cp310-win_amd64.whl", hash = "sha256:36df958b15639e40472adaa4f0c2c7828fe680f894a6b48c4ce229f59a6a798b"}, - {file = "multidict-6.0.3-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:5b51969503709415a35754954c2763f536a70b8bf7360322b2edb0c0a44391f6"}, - {file = "multidict-6.0.3-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:24e8d513bfcaadc1f8b0ebece3ff50961951c54b07d5a775008a882966102418"}, - {file = "multidict-6.0.3-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:d325d61cac602976a5d47b19eaa7d04e3daf4efce2164c630219885087234102"}, - {file = "multidict-6.0.3-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:26fbbe17f8a7211b623502d2bf41022a51da3025142401417c765bf9a56fed4c"}, - {file = "multidict-6.0.3-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:4fb3fe591956d8841882c463f934c9f7485cfd5f763a08c0d467b513dc18ef89"}, - {file = "multidict-6.0.3-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:e1925f78a543b94c3d46274c66a366fee8a263747060220ed0188e5f3eeea1c0"}, - {file = "multidict-6.0.3-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:21e1ce0b187c4e93112304dcde2aa18922fdbe8fb4f13d8aa72a5657bce0563a"}, - {file = "multidict-6.0.3-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:e07c24018986fb00d6e7eafca8fcd6e05095649e17fcf0e33a592caaa62a78b9"}, - {file = "multidict-6.0.3-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:114a4ab3e5cfbc56c4b6697686ecb92376c7e8c56893ef20547921552f8bdf57"}, - {file = "multidict-6.0.3-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:4ccf55f28066b4f08666764a957c2b7c241c7547b0921d69c7ceab5f74fe1a45"}, - {file = "multidict-6.0.3-cp311-cp311-musllinux_1_1_ppc64le.whl", hash = "sha256:9d359b0a962e052b713647ac1f13eabf2263167b149ed1e27d5c579f5c8c7d2c"}, - {file = "multidict-6.0.3-cp311-cp311-musllinux_1_1_s390x.whl", hash = "sha256:df7b4cee3ff31b3335aba602f8d70dbc641e5b7164b1e9565570c9d3c536a438"}, - {file = "multidict-6.0.3-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:ee9b1cae9a6c5d023e5a150f6f6b9dbb3c3bbc7887d6ee07d4c0ecb49a473734"}, - {file = "multidict-6.0.3-cp311-cp311-win32.whl", hash = "sha256:960ce1b790952916e682093788696ef7e33ac6a97482f9b983abdc293091b531"}, - {file = "multidict-6.0.3-cp311-cp311-win_amd64.whl", hash = "sha256:2b66d61966b12e6bba500e5cbb2c721a35e119c30ee02495c5629bd0e91eea30"}, - {file = "multidict-6.0.3-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:526f8397fc124674b8f39748680a0ff673bd6a715fecb4866716d36e380f015f"}, - {file = "multidict-6.0.3-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:1f5d5129a937af4e3c4a1d6c139f4051b7d17d43276cefdd8d442a7031f7eef2"}, - {file = "multidict-6.0.3-cp37-cp37m-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:38d394814b39be1c36ac709006d39d50d72a884f9551acd9c8cc1ffae3fc8c4e"}, - {file = "multidict-6.0.3-cp37-cp37m-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:99341ca1f1db9e7f47914cb2461305665a662383765ced6f843712564766956d"}, - {file = "multidict-6.0.3-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:c5790cc603456b6dcf8a9a4765f666895a6afddc88b3d3ba7b53dea2b6e23116"}, - {file = "multidict-6.0.3-cp37-cp37m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:ce8e51774eb03844588d3c279adb94efcd0edeccd2f97516623292445bcc01f9"}, - {file = "multidict-6.0.3-cp37-cp37m-musllinux_1_1_aarch64.whl", hash = "sha256:baa96a3418e27d723064854143b2f414a422c84cc87285a71558722049bebc5a"}, - {file = "multidict-6.0.3-cp37-cp37m-musllinux_1_1_i686.whl", hash = "sha256:cb4a08f0aaaa869f189ffea0e17b86ad0237b51116d494da15ef7991ee6ad2d7"}, - {file = "multidict-6.0.3-cp37-cp37m-musllinux_1_1_ppc64le.whl", hash = "sha256:62db44727d0befea68e8ad2881bb87a9cfb6b87d45dd78609009627167f37b69"}, - {file = "multidict-6.0.3-cp37-cp37m-musllinux_1_1_s390x.whl", hash = "sha256:4cc5c8cd205a9810d16a5cd428cd81bac554ad1477cb87f4ad722b10992e794d"}, - {file = "multidict-6.0.3-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:f76109387e1ec8d8e2137c94c437b89fe002f29e0881aae8ae45529bdff92000"}, - {file = "multidict-6.0.3-cp37-cp37m-win32.whl", hash = "sha256:f8a728511c977df6f3d8af388fcb157e49f11db4a6637dd60131b8b6e40b0253"}, - {file = "multidict-6.0.3-cp37-cp37m-win_amd64.whl", hash = "sha256:c2a1168e5aa7c72499fb03c850e0f03f624fa4a5c8d2e215c518d0a73872eb64"}, - {file = "multidict-6.0.3-cp38-cp38-macosx_10_9_universal2.whl", hash = "sha256:eddf604a3de2ace3d9a4e4d491be7562a1ac095a0a1c95a9ec5781ef0273ef11"}, - {file = "multidict-6.0.3-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:d09daf5c6ce7fc6ed444c9339bbde5ea84e2534d1ca1cd37b60f365c77f00dea"}, - {file = "multidict-6.0.3-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:12e0d396faa6dc55ff5379eee54d1df3b508243ff15bfc8295a6ec7a4483a335"}, - {file = "multidict-6.0.3-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:70740c2bc9ab1c99f7cdcb104f27d16c63860c56d51c5bf0ef82fc1d892a2131"}, - {file = "multidict-6.0.3-cp38-cp38-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:e322c94596054352f5a02771eec71563c018b15699b961aba14d6dd943367022"}, - {file = "multidict-6.0.3-cp38-cp38-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:4159fc1ec9ede8ab93382e0d6ba9b1b3d23c72da39a834db7a116986605c7ab4"}, - {file = "multidict-6.0.3-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:47defc0218682281a52fb1f6346ebb8b68b17538163a89ea24dfe4da37a8a9a3"}, - {file = "multidict-6.0.3-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:7f9511e48bde6b995825e8d35e434fc96296cf07a25f4aae24ff9162be7eaa46"}, - {file = "multidict-6.0.3-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:e0bce9f7c30e7e3a9e683f670314c0144e8d34be6b7019e40604763bd278d84f"}, - {file = "multidict-6.0.3-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:01b456046a05ff7cceefb0e1d2a9d32f05efcb1c7e0d152446304e11557639ce"}, - {file = "multidict-6.0.3-cp38-cp38-musllinux_1_1_ppc64le.whl", hash = "sha256:8230a39bae6c2e8a09e4da6bace5064693b00590a4a213e38f9a9366da10e7dd"}, - {file = "multidict-6.0.3-cp38-cp38-musllinux_1_1_s390x.whl", hash = "sha256:445c0851a1cbc1f2ec3b40bc22f9c4a235edb3c9a0906122a9df6ea8d51f886c"}, - {file = "multidict-6.0.3-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:9aac6881454a750554ed4b280a839dcf9e2133a9d12ab4d417d673fb102289b7"}, - {file = "multidict-6.0.3-cp38-cp38-win32.whl", hash = "sha256:81c3d597591b0940e04949e4e4f79359b2d2e542a686ba0da5e25de33fec13e0"}, - {file = "multidict-6.0.3-cp38-cp38-win_amd64.whl", hash = "sha256:dc4cfef5d899f5f1a15f3d2ac49f71107a01a5a2745b4dd53fa0cede1419385a"}, - {file = "multidict-6.0.3-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:d408172519049e36fb6d29672f060dc8461fc7174eba9883c7026041ef9bfb38"}, - {file = "multidict-6.0.3-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:e068dfeadbce63072b2d8096486713d04db4946aad0a0f849bd4fc300799d0d3"}, - {file = "multidict-6.0.3-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:a8b817d4ed68fd568ec5e45dd75ddf30cc72a47a6b41b74d5bb211374c296f5e"}, - {file = "multidict-6.0.3-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:2cf5d19e12eff855aa198259c0b02fd3f5d07e1291fbd20279c37b3b0e6c9852"}, - {file = "multidict-6.0.3-cp39-cp39-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:e5a811aab1b4aea0b4be669363c19847a8c547510f0e18fb632956369fdbdf67"}, - {file = "multidict-6.0.3-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:2cfda34b7cb99eacada2072e0f69c0ad3285cb6f8e480b11f2b6d6c1c6f92718"}, - {file = "multidict-6.0.3-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:beeca903e4270b4afcd114f371a9602240dc143f9e944edfea00f8d4ad56c40d"}, - {file = "multidict-6.0.3-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:cd5771e8ea325f85cbb361ddbdeb9ae424a68e5dfb6eea786afdcd22e68a7d5d"}, - {file = "multidict-6.0.3-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:9dbab2a7e9c073bc9538824a01f5ed689194db7f55f2b8102766873e906a6c1a"}, - {file = "multidict-6.0.3-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:f2c0957b3e8c66c10d27272709a5299ab3670a0f187c9428f3b90d267119aedb"}, - {file = "multidict-6.0.3-cp39-cp39-musllinux_1_1_ppc64le.whl", hash = "sha256:94cbe5535ef150546b8321aebea22862a3284da51e7b55f6f95b7d73e96d90ee"}, - {file = "multidict-6.0.3-cp39-cp39-musllinux_1_1_s390x.whl", hash = "sha256:d0e798b072cf2aab9daceb43d97c9c527a0c7593e67a7846ad4cc6051de1e303"}, - {file = "multidict-6.0.3-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:a27b029caa3b555a4f3da54bc1e718eb55fcf1a11fda8bf0132147b476cf4c08"}, - {file = "multidict-6.0.3-cp39-cp39-win32.whl", hash = "sha256:018c8e3be7f161a12b3e41741b6721f9baeb2210f4ab25a6359b7d76c1017dce"}, - {file = "multidict-6.0.3-cp39-cp39-win_amd64.whl", hash = "sha256:5e58ec0375803526d395f6f7e730ecc45d06e15f68f7b9cdbf644a2918324e51"}, - {file = "multidict-6.0.3.tar.gz", hash = "sha256:2523a29006c034687eccd3ee70093a697129a3ffe8732535d3b2df6a4ecc279d"}, -] [[package]] name = "nodeenv" @@ -1330,51 +644,14 @@ description = "Node.js virtual environment builder" category = "dev" optional = false python-versions = ">=2.7,!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,!=3.4.*,!=3.5.*,!=3.6.*" -files = [ - {file = "nodeenv-1.7.0-py2.py3-none-any.whl", hash = "sha256:27083a7b96a25f2f5e1d8cb4b6317ee8aeda3bdd121394e5ac54e498028a042e"}, - {file = "nodeenv-1.7.0.tar.gz", hash = "sha256:e0e7f7dfb85fc5394c6fe1e8fa98131a2473e04311a45afb6508f7cf1836fa2b"}, -] - -[package.dependencies] -setuptools = "*" [[package]] name = "numpy" -version = "1.23.5" -description = "NumPy is the fundamental package for array computing with Python." +version = "1.24.1" +description = "Fundamental package for array computing in Python" category = "main" optional = true python-versions = ">=3.8" -files = [ - {file = "numpy-1.23.5-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:9c88793f78fca17da0145455f0d7826bcb9f37da4764af27ac945488116efe63"}, - {file = "numpy-1.23.5-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:e9f4c4e51567b616be64e05d517c79a8a22f3606499941d97bb76f2ca59f982d"}, - {file = "numpy-1.23.5-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:7903ba8ab592b82014713c491f6c5d3a1cde5b4a3bf116404e08f5b52f6daf43"}, - {file = "numpy-1.23.5-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:5e05b1c973a9f858c74367553e236f287e749465f773328c8ef31abe18f691e1"}, - {file = "numpy-1.23.5-cp310-cp310-win32.whl", hash = "sha256:522e26bbf6377e4d76403826ed689c295b0b238f46c28a7251ab94716da0b280"}, - {file = "numpy-1.23.5-cp310-cp310-win_amd64.whl", hash = "sha256:dbee87b469018961d1ad79b1a5d50c0ae850000b639bcb1b694e9981083243b6"}, - {file = "numpy-1.23.5-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:ce571367b6dfe60af04e04a1834ca2dc5f46004ac1cc756fb95319f64c095a96"}, - {file = "numpy-1.23.5-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:56e454c7833e94ec9769fa0f86e6ff8e42ee38ce0ce1fa4cbb747ea7e06d56aa"}, - {file = "numpy-1.23.5-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:5039f55555e1eab31124a5768898c9e22c25a65c1e0037f4d7c495a45778c9f2"}, - {file = "numpy-1.23.5-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:58f545efd1108e647604a1b5aa809591ccd2540f468a880bedb97247e72db387"}, - {file = "numpy-1.23.5-cp311-cp311-win32.whl", hash = "sha256:b2a9ab7c279c91974f756c84c365a669a887efa287365a8e2c418f8b3ba73fb0"}, - {file = "numpy-1.23.5-cp311-cp311-win_amd64.whl", hash = "sha256:0cbe9848fad08baf71de1a39e12d1b6310f1d5b2d0ea4de051058e6e1076852d"}, - {file = "numpy-1.23.5-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:f063b69b090c9d918f9df0a12116029e274daf0181df392839661c4c7ec9018a"}, - {file = "numpy-1.23.5-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:0aaee12d8883552fadfc41e96b4c82ee7d794949e2a7c3b3a7201e968c7ecab9"}, - {file = "numpy-1.23.5-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:92c8c1e89a1f5028a4c6d9e3ccbe311b6ba53694811269b992c0b224269e2398"}, - {file = "numpy-1.23.5-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:d208a0f8729f3fb790ed18a003f3a57895b989b40ea4dce4717e9cf4af62c6bb"}, - {file = "numpy-1.23.5-cp38-cp38-win32.whl", hash = "sha256:06005a2ef6014e9956c09ba07654f9837d9e26696a0470e42beedadb78c11b07"}, - {file = "numpy-1.23.5-cp38-cp38-win_amd64.whl", hash = "sha256:ca51fcfcc5f9354c45f400059e88bc09215fb71a48d3768fb80e357f3b457e1e"}, - {file = "numpy-1.23.5-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:8969bfd28e85c81f3f94eb4a66bc2cf1dbdc5c18efc320af34bffc54d6b1e38f"}, - {file = "numpy-1.23.5-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:a7ac231a08bb37f852849bbb387a20a57574a97cfc7b6cabb488a4fc8be176de"}, - {file = "numpy-1.23.5-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:bf837dc63ba5c06dc8797c398db1e223a466c7ece27a1f7b5232ba3466aafe3d"}, - {file = "numpy-1.23.5-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:33161613d2269025873025b33e879825ec7b1d831317e68f4f2f0f84ed14c719"}, - {file = "numpy-1.23.5-cp39-cp39-win32.whl", hash = "sha256:af1da88f6bc3d2338ebbf0e22fe487821ea4d8e89053e25fa59d1d79786e7481"}, - {file = "numpy-1.23.5-cp39-cp39-win_amd64.whl", hash = "sha256:09b7847f7e83ca37c6e627682f145856de331049013853f344f37b0c9690e3df"}, - {file = "numpy-1.23.5-pp38-pypy38_pp73-macosx_10_9_x86_64.whl", hash = "sha256:abdde9f795cf292fb9651ed48185503a2ff29be87770c3b8e2a14b0cd7aa16f8"}, - {file = "numpy-1.23.5-pp38-pypy38_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:f9a909a8bae284d46bbfdefbdd4a262ba19d3bc9921b1e76126b1d21c3c34135"}, - {file = "numpy-1.23.5-pp38-pypy38_pp73-win_amd64.whl", hash = "sha256:01dd17cbb340bf0fc23981e52e1d18a9d4050792e8fb8363cecbf066a84b827d"}, - {file = "numpy-1.23.5.tar.gz", hash = "sha256:1b1766d6f397c18153d40015ddfc79ddb715cabadc04d2d228d4e5a8bc4ded1a"}, -] [[package]] name = "oauthlib" @@ -1383,10 +660,6 @@ description = "A generic, spec-compliant, thorough implementation of the OAuth r category = "main" optional = true python-versions = ">=3.6" -files = [ - {file = "oauthlib-3.2.2-py3-none-any.whl", hash = "sha256:8139f29aac13e25d502680e9e19963e83f16838d48a0d71c287fe40e7067fbca"}, - {file = "oauthlib-3.2.2.tar.gz", hash = "sha256:9859c40929662bec5d64f34d01c99e093149682a3f38915dc0655d5a633dd918"}, -] [package.extras] rsa = ["cryptography (>=3.0.0)"] @@ -1395,15 +668,11 @@ signedtoken = ["cryptography (>=3.0.0)", "pyjwt (>=2.0.0,<3)"] [[package]] name = "packaging" -version = "22.0" +version = "23.0" description = "Core utilities for Python packages" category = "dev" optional = false python-versions = ">=3.7" -files = [ - {file = "packaging-22.0-py3-none-any.whl", hash = "sha256:957e2148ba0e1a3b282772e791ef1d8083648bc131c8ab0c1feba110ce1146c3"}, - {file = "packaging-22.0.tar.gz", hash = "sha256:2198ec20bd4c017b8f9717e00f0c8714076fc2fd93816750ab48e2c41de2cfd3"}, -] [[package]] name = "pandas" @@ -1412,35 +681,6 @@ description = "Powerful data structures for data analysis, time series, and stat category = "main" optional = true python-versions = ">=3.8" -files = [ - {file = "pandas-1.5.3-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:3749077d86e3a2f0ed51367f30bf5b82e131cc0f14260c4d3e499186fccc4406"}, - {file = "pandas-1.5.3-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:972d8a45395f2a2d26733eb8d0f629b2f90bebe8e8eddbb8829b180c09639572"}, - {file = "pandas-1.5.3-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:50869a35cbb0f2e0cd5ec04b191e7b12ed688874bd05dd777c19b28cbea90996"}, - {file = "pandas-1.5.3-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:c3ac844a0fe00bfaeb2c9b51ab1424e5c8744f89860b138434a363b1f620f354"}, - {file = "pandas-1.5.3-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:7a0a56cef15fd1586726dace5616db75ebcfec9179a3a55e78f72c5639fa2a23"}, - {file = "pandas-1.5.3-cp310-cp310-win_amd64.whl", hash = "sha256:478ff646ca42b20376e4ed3fa2e8d7341e8a63105586efe54fa2508ee087f328"}, - {file = "pandas-1.5.3-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:6973549c01ca91ec96199e940495219c887ea815b2083722821f1d7abfa2b4dc"}, - {file = "pandas-1.5.3-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:c39a8da13cede5adcd3be1182883aea1c925476f4e84b2807a46e2775306305d"}, - {file = "pandas-1.5.3-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:f76d097d12c82a535fda9dfe5e8dd4127952b45fea9b0276cb30cca5ea313fbc"}, - {file = "pandas-1.5.3-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:e474390e60ed609cec869b0da796ad94f420bb057d86784191eefc62b65819ae"}, - {file = "pandas-1.5.3-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:5f2b952406a1588ad4cad5b3f55f520e82e902388a6d5a4a91baa8d38d23c7f6"}, - {file = "pandas-1.5.3-cp311-cp311-win_amd64.whl", hash = "sha256:bc4c368f42b551bf72fac35c5128963a171b40dce866fb066540eeaf46faa003"}, - {file = "pandas-1.5.3-cp38-cp38-macosx_10_9_universal2.whl", hash = "sha256:14e45300521902689a81f3f41386dc86f19b8ba8dd5ac5a3c7010ef8d2932813"}, - {file = "pandas-1.5.3-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:9842b6f4b8479e41968eced654487258ed81df7d1c9b7b870ceea24ed9459b31"}, - {file = "pandas-1.5.3-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:26d9c71772c7afb9d5046e6e9cf42d83dd147b5cf5bcb9d97252077118543792"}, - {file = "pandas-1.5.3-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:5fbcb19d6fceb9e946b3e23258757c7b225ba450990d9ed63ccceeb8cae609f7"}, - {file = "pandas-1.5.3-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:565fa34a5434d38e9d250af3c12ff931abaf88050551d9fbcdfafca50d62babf"}, - {file = "pandas-1.5.3-cp38-cp38-win32.whl", hash = "sha256:87bd9c03da1ac870a6d2c8902a0e1fd4267ca00f13bc494c9e5a9020920e1d51"}, - {file = "pandas-1.5.3-cp38-cp38-win_amd64.whl", hash = "sha256:41179ce559943d83a9b4bbacb736b04c928b095b5f25dd2b7389eda08f46f373"}, - {file = "pandas-1.5.3-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:c74a62747864ed568f5a82a49a23a8d7fe171d0c69038b38cedf0976831296fa"}, - {file = "pandas-1.5.3-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:c4c00e0b0597c8e4f59e8d461f797e5d70b4d025880516a8261b2817c47759ee"}, - {file = "pandas-1.5.3-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:a50d9a4336a9621cab7b8eb3fb11adb82de58f9b91d84c2cd526576b881a0c5a"}, - {file = "pandas-1.5.3-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:dd05f7783b3274aa206a1af06f0ceed3f9b412cf665b7247eacd83be41cf7bf0"}, - {file = "pandas-1.5.3-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:9f69c4029613de47816b1bb30ff5ac778686688751a5e9c99ad8c7031f6508e5"}, - {file = "pandas-1.5.3-cp39-cp39-win32.whl", hash = "sha256:7cec0bee9f294e5de5bbfc14d0573f65526071029d036b753ee6507d2a21480a"}, - {file = "pandas-1.5.3-cp39-cp39-win_amd64.whl", hash = "sha256:dfd681c5dc216037e0b0a2c821f5ed99ba9f03ebcf119c7dac0e9a7b960b9ec9"}, - {file = "pandas-1.5.3.tar.gz", hash = "sha256:74a3fd7e5a7ec052f183273dc7b0acd3a863edf7520f5d3a1765c04ffdb3b0b1"}, -] [package.dependencies] numpy = [ @@ -1454,36 +694,17 @@ pytz = ">=2020.1" [package.extras] test = ["hypothesis (>=5.5.3)", "pytest (>=6.0)", "pytest-xdist (>=1.31)"] -[[package]] -name = "pep517" -version = "0.13.0" -description = "Wrappers to build Python packages using PEP 517 hooks" -category = "dev" -optional = false -python-versions = ">=3.6" -files = [ - {file = "pep517-0.13.0-py3-none-any.whl", hash = "sha256:4ba4446d80aed5b5eac6509ade100bff3e7943a8489de249654a5ae9b33ee35b"}, - {file = "pep517-0.13.0.tar.gz", hash = "sha256:ae69927c5c172be1add9203726d4b84cf3ebad1edcd5f71fcdc746e66e829f59"}, -] - -[package.dependencies] -tomli = {version = ">=1.1.0", markers = "python_version < \"3.11\""} - [[package]] name = "platformdirs" -version = "2.6.0" +version = "2.6.2" description = "A small Python package for determining appropriate platform-specific dirs, e.g. a \"user data dir\"." category = "dev" optional = false python-versions = ">=3.7" -files = [ - {file = "platformdirs-2.6.0-py3-none-any.whl", hash = "sha256:1a89a12377800c81983db6be069ec068eee989748799b946cce2a6e80dcc54ca"}, - {file = "platformdirs-2.6.0.tar.gz", hash = "sha256:b46ffafa316e6b83b47489d240ce17173f123a9b9c83282141c3daf26ad9ac2e"}, -] [package.extras] -docs = ["furo (>=2022.9.29)", "proselint (>=0.13)", "sphinx (>=5.3)", "sphinx-autodoc-typehints (>=1.19.4)"] -test = ["appdirs (==1.4.4)", "pytest (>=7.2)", "pytest-cov (>=4)", "pytest-mock (>=3.10)"] +docs = ["furo (>=2022.12.7)", "proselint (>=0.13)", "sphinx (>=5.3)", "sphinx-autodoc-typehints (>=1.19.5)"] +test = ["appdirs (==1.4.4)", "covdefaults (>=2.2.2)", "pytest (>=7.2)", "pytest-cov (>=4)", "pytest-mock (>=3.10)"] [[package]] name = "pluggy" @@ -1492,10 +713,6 @@ description = "plugin and hook calling mechanisms for python" category = "dev" optional = false python-versions = ">=3.6" -files = [ - {file = "pluggy-1.0.0-py2.py3-none-any.whl", hash = "sha256:74134bbf457f031a36d68416e1509f34bd5ccc019f0bcc952c7b909d06b37bd3"}, - {file = "pluggy-1.0.0.tar.gz", hash = "sha256:4224373bacce55f955a878bf9cfa763c1e360858e330072059e10bad68531159"}, -] [package.extras] dev = ["pre-commit", "tox"] @@ -1503,15 +720,11 @@ testing = ["pytest", "pytest-benchmark"] [[package]] name = "portalocker" -version = "2.6.0" +version = "2.7.0" description = "Wraps the portalocker recipe for easy usage" category = "main" optional = true python-versions = ">=3.5" -files = [ - {file = "portalocker-2.6.0-py2.py3-none-any.whl", hash = "sha256:102ed1f2badd8dec9af3d732ef70e94b215b85ba45a8d7ff3c0003f19b442f4e"}, - {file = "portalocker-2.6.0.tar.gz", hash = "sha256:964f6830fb42a74b5d32bce99ed37d8308c1d7d44ddf18f3dd89f4680de97b39"}, -] [package.dependencies] pywin32 = {version = ">=226", markers = "platform_system == \"Windows\""} @@ -1519,7 +732,7 @@ pywin32 = {version = ">=226", markers = "platform_system == \"Windows\""} [package.extras] docs = ["sphinx (>=1.7.1)"] redis = ["redis"] -tests = ["pytest (>=5.4.1)", "pytest-cov (>=2.8.1)", "pytest-mypy (>=0.8.0)", "pytest-timeout (>=2.1.0)", "redis", "sphinx (>=3.0.3)"] +tests = ["pytest (>=5.4.1)", "pytest-cov (>=2.8.1)", "pytest-mypy (>=0.8.0)", "pytest-timeout (>=2.1.0)", "redis", "sphinx (>=6.0.0)"] [[package]] name = "pre-commit" @@ -1528,10 +741,6 @@ description = "A framework for managing and maintaining multi-language pre-commi category = "dev" optional = false python-versions = ">=3.7" -files = [ - {file = "pre_commit-2.21.0-py2.py3-none-any.whl", hash = "sha256:e2f91727039fc39a92f58a588a25b87f936de6567eed4f0e673e0507edc75bad"}, - {file = "pre_commit-2.21.0.tar.gz", hash = "sha256:31ef31af7e474a8d8995027fefdfcf509b5c913ff31f2015b4ec4beb26a6f658"}, -] [package.dependencies] cfgv = ">=2.0.0" @@ -1547,33 +756,6 @@ description = "Python library for Apache Arrow" category = "main" optional = true python-versions = ">=3.7" -files = [ - {file = "pyarrow-10.0.1-cp310-cp310-macosx_10_14_x86_64.whl", hash = "sha256:e00174764a8b4e9d8d5909b6d19ee0c217a6cf0232c5682e31fdfbd5a9f0ae52"}, - {file = "pyarrow-10.0.1-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:6f7a7dbe2f7f65ac1d0bd3163f756deb478a9e9afc2269557ed75b1b25ab3610"}, - {file = "pyarrow-10.0.1-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:cb627673cb98708ef00864e2e243f51ba7b4c1b9f07a1d821f98043eccd3f585"}, - {file = "pyarrow-10.0.1-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:ba71e6fc348c92477586424566110d332f60d9a35cb85278f42e3473bc1373da"}, - {file = "pyarrow-10.0.1-cp310-cp310-win_amd64.whl", hash = "sha256:7b4ede715c004b6fc535de63ef79fa29740b4080639a5ff1ea9ca84e9282f349"}, - {file = "pyarrow-10.0.1-cp311-cp311-macosx_10_14_x86_64.whl", hash = "sha256:e3fe5049d2e9ca661d8e43fab6ad5a4c571af12d20a57dffc392a014caebef65"}, - {file = "pyarrow-10.0.1-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:254017ca43c45c5098b7f2a00e995e1f8346b0fb0be225f042838323bb55283c"}, - {file = "pyarrow-10.0.1-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:70acca1ece4322705652f48db65145b5028f2c01c7e426c5d16a30ba5d739c24"}, - {file = "pyarrow-10.0.1-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:abb57334f2c57979a49b7be2792c31c23430ca02d24becd0b511cbe7b6b08649"}, - {file = "pyarrow-10.0.1-cp311-cp311-win_amd64.whl", hash = "sha256:1765a18205eb1e02ccdedb66049b0ec148c2a0cb52ed1fb3aac322dfc086a6ee"}, - {file = "pyarrow-10.0.1-cp37-cp37m-macosx_10_14_x86_64.whl", hash = "sha256:61f4c37d82fe00d855d0ab522c685262bdeafd3fbcb5fe596fe15025fbc7341b"}, - {file = "pyarrow-10.0.1-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:e141a65705ac98fa52a9113fe574fdaf87fe0316cde2dffe6b94841d3c61544c"}, - {file = "pyarrow-10.0.1-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:bf26f809926a9d74e02d76593026f0aaeac48a65b64f1bb17eed9964bfe7ae1a"}, - {file = "pyarrow-10.0.1-cp37-cp37m-win_amd64.whl", hash = "sha256:443eb9409b0cf78df10ced326490e1a300205a458fbeb0767b6b31ab3ebae6b2"}, - {file = "pyarrow-10.0.1-cp38-cp38-macosx_10_14_x86_64.whl", hash = "sha256:f2d00aa481becf57098e85d99e34a25dba5a9ade2f44eb0b7d80c80f2984fc03"}, - {file = "pyarrow-10.0.1-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:b1fc226d28c7783b52a84d03a66573d5a22e63f8a24b841d5fc68caeed6784d4"}, - {file = "pyarrow-10.0.1-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:efa59933b20183c1c13efc34bd91efc6b2997377c4c6ad9272da92d224e3beb1"}, - {file = "pyarrow-10.0.1-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:668e00e3b19f183394388a687d29c443eb000fb3fe25599c9b4762a0afd37775"}, - {file = "pyarrow-10.0.1-cp38-cp38-win_amd64.whl", hash = "sha256:d1bc6e4d5d6f69e0861d5d7f6cf4d061cf1069cb9d490040129877acf16d4c2a"}, - {file = "pyarrow-10.0.1-cp39-cp39-macosx_10_14_x86_64.whl", hash = "sha256:42ba7c5347ce665338f2bc64685d74855900200dac81a972d49fe127e8132f75"}, - {file = "pyarrow-10.0.1-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:b069602eb1fc09f1adec0a7bdd7897f4d25575611dfa43543c8b8a75d99d6874"}, - {file = "pyarrow-10.0.1-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:94fb4a0c12a2ac1ed8e7e2aa52aade833772cf2d3de9dde685401b22cec30002"}, - {file = "pyarrow-10.0.1-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:db0c5986bf0808927f49640582d2032a07aa49828f14e51f362075f03747d198"}, - {file = "pyarrow-10.0.1-cp39-cp39-win_amd64.whl", hash = "sha256:0ec7587d759153f452d5263dbc8b1af318c4609b607be2bd5127dcda6708cdb1"}, - {file = "pyarrow-10.0.1.tar.gz", hash = "sha256:1a14f57a5f472ce8234f2964cd5184cccaa8df7e04568c64edc33b23eb285dd5"}, -] [package.dependencies] numpy = ">=1.16.6" @@ -1585,10 +767,6 @@ description = "C parser in Python" category = "main" optional = false python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*" -files = [ - {file = "pycparser-2.21-py2.py3-none-any.whl", hash = "sha256:8ee45429555515e1f6b185e78100aea234072576aa43ab53aefcae078162fca9"}, - {file = "pycparser-2.21.tar.gz", hash = "sha256:e644fdec12f7872f86c58ff790da456218b10f863970249516d60a5eaca77206"}, -] [[package]] name = "pydantic" @@ -1597,44 +775,6 @@ description = "Data validation and settings management using python type hints" category = "main" optional = false python-versions = ">=3.7" -files = [ - {file = "pydantic-1.10.4-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:b5635de53e6686fe7a44b5cf25fcc419a0d5e5c1a1efe73d49d48fe7586db854"}, - {file = "pydantic-1.10.4-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:6dc1cc241440ed7ca9ab59d9929075445da6b7c94ced281b3dd4cfe6c8cff817"}, - {file = "pydantic-1.10.4-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:51bdeb10d2db0f288e71d49c9cefa609bca271720ecd0c58009bd7504a0c464c"}, - {file = "pydantic-1.10.4-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:78cec42b95dbb500a1f7120bdf95c401f6abb616bbe8785ef09887306792e66e"}, - {file = "pydantic-1.10.4-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:8775d4ef5e7299a2f4699501077a0defdaac5b6c4321173bcb0f3c496fbadf85"}, - {file = "pydantic-1.10.4-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:572066051eeac73d23f95ba9a71349c42a3e05999d0ee1572b7860235b850cc6"}, - {file = "pydantic-1.10.4-cp310-cp310-win_amd64.whl", hash = "sha256:7feb6a2d401f4d6863050f58325b8d99c1e56f4512d98b11ac64ad1751dc647d"}, - {file = "pydantic-1.10.4-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:39f4a73e5342b25c2959529f07f026ef58147249f9b7431e1ba8414a36761f53"}, - {file = "pydantic-1.10.4-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:983e720704431a6573d626b00662eb78a07148c9115129f9b4351091ec95ecc3"}, - {file = "pydantic-1.10.4-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:75d52162fe6b2b55964fbb0af2ee58e99791a3138588c482572bb6087953113a"}, - {file = "pydantic-1.10.4-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:fdf8d759ef326962b4678d89e275ffc55b7ce59d917d9f72233762061fd04a2d"}, - {file = "pydantic-1.10.4-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:05a81b006be15655b2a1bae5faa4280cf7c81d0e09fcb49b342ebf826abe5a72"}, - {file = "pydantic-1.10.4-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:d88c4c0e5c5dfd05092a4b271282ef0588e5f4aaf345778056fc5259ba098857"}, - {file = "pydantic-1.10.4-cp311-cp311-win_amd64.whl", hash = "sha256:6a05a9db1ef5be0fe63e988f9617ca2551013f55000289c671f71ec16f4985e3"}, - {file = "pydantic-1.10.4-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:887ca463c3bc47103c123bc06919c86720e80e1214aab79e9b779cda0ff92a00"}, - {file = "pydantic-1.10.4-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:fdf88ab63c3ee282c76d652fc86518aacb737ff35796023fae56a65ced1a5978"}, - {file = "pydantic-1.10.4-cp37-cp37m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:a48f1953c4a1d9bd0b5167ac50da9a79f6072c63c4cef4cf2a3736994903583e"}, - {file = "pydantic-1.10.4-cp37-cp37m-musllinux_1_1_i686.whl", hash = "sha256:a9f2de23bec87ff306aef658384b02aa7c32389766af3c5dee9ce33e80222dfa"}, - {file = "pydantic-1.10.4-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:cd8702c5142afda03dc2b1ee6bc358b62b3735b2cce53fc77b31ca9f728e4bc8"}, - {file = "pydantic-1.10.4-cp37-cp37m-win_amd64.whl", hash = "sha256:6e7124d6855b2780611d9f5e1e145e86667eaa3bd9459192c8dc1a097f5e9903"}, - {file = "pydantic-1.10.4-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:0b53e1d41e97063d51a02821b80538053ee4608b9a181c1005441f1673c55423"}, - {file = "pydantic-1.10.4-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:55b1625899acd33229c4352ce0ae54038529b412bd51c4915349b49ca575258f"}, - {file = "pydantic-1.10.4-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:301d626a59edbe5dfb48fcae245896379a450d04baeed50ef40d8199f2733b06"}, - {file = "pydantic-1.10.4-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:b6f9d649892a6f54a39ed56b8dfd5e08b5f3be5f893da430bed76975f3735d15"}, - {file = "pydantic-1.10.4-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:d7b5a3821225f5c43496c324b0d6875fde910a1c2933d726a743ce328fbb2a8c"}, - {file = "pydantic-1.10.4-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:f2f7eb6273dd12472d7f218e1fef6f7c7c2f00ac2e1ecde4db8824c457300416"}, - {file = "pydantic-1.10.4-cp38-cp38-win_amd64.whl", hash = "sha256:4b05697738e7d2040696b0a66d9f0a10bec0efa1883ca75ee9e55baf511909d6"}, - {file = "pydantic-1.10.4-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:a9a6747cac06c2beb466064dda999a13176b23535e4c496c9d48e6406f92d42d"}, - {file = "pydantic-1.10.4-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:eb992a1ef739cc7b543576337bebfc62c0e6567434e522e97291b251a41dad7f"}, - {file = "pydantic-1.10.4-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:990406d226dea0e8f25f643b370224771878142155b879784ce89f633541a024"}, - {file = "pydantic-1.10.4-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:2e82a6d37a95e0b1b42b82ab340ada3963aea1317fd7f888bb6b9dfbf4fff57c"}, - {file = "pydantic-1.10.4-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:9193d4f4ee8feca58bc56c8306bcb820f5c7905fd919e0750acdeeeef0615b28"}, - {file = "pydantic-1.10.4-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:2b3ce5f16deb45c472dde1a0ee05619298c864a20cded09c4edd820e1454129f"}, - {file = "pydantic-1.10.4-cp39-cp39-win_amd64.whl", hash = "sha256:9cbdc268a62d9a98c56e2452d6c41c0263d64a2009aac69246486f01b4f594c4"}, - {file = "pydantic-1.10.4-py3-none-any.whl", hash = "sha256:4948f264678c703f3877d1c8877c4e3b2e12e549c57795107f08cf70c6ec7774"}, - {file = "pydantic-1.10.4.tar.gz", hash = "sha256:b9a3859f24eb4e097502a3be1fb4b2abb79b6103dd9e2e0edb70613a4459a648"}, -] [package.dependencies] typing-extensions = ">=4.2.0" @@ -1645,15 +785,11 @@ email = ["email-validator (>=1.0.3)"] [[package]] name = "pygments" -version = "2.13.0" +version = "2.14.0" description = "Pygments is a syntax highlighting package written in Python." category = "main" optional = false python-versions = ">=3.6" -files = [ - {file = "Pygments-2.13.0-py3-none-any.whl", hash = "sha256:f643f331ab57ba3c9d89212ee4a2dabc6e94f117cf4eefde99a0574720d14c42"}, - {file = "Pygments-2.13.0.tar.gz", hash = "sha256:56a8508ae95f98e2b9bdf93a6be5ae3f7d8af858b43e02c5a2ff083726be40c1"}, -] [package.extras] plugins = ["importlib-metadata"] @@ -1665,10 +801,6 @@ description = "JSON Web Token implementation in Python" category = "main" optional = true python-versions = ">=3.7" -files = [ - {file = "PyJWT-2.6.0-py3-none-any.whl", hash = "sha256:d83c3d892a77bbb74d3e1a2cfa90afaadb60945205d1095d9221f04466f64c14"}, - {file = "PyJWT-2.6.0.tar.gz", hash = "sha256:69285c7e31fc44f68a1feb309e948e0df53259d579295e6cfe2b1792329f05fd"}, -] [package.dependencies] cryptography = {version = ">=3.4.0", optional = true, markers = "extra == \"crypto\""} @@ -1686,14 +818,21 @@ description = "pyparsing module - Classes and methods to define and execute pars category = "main" optional = false python-versions = ">=3.6.8" -files = [ - {file = "pyparsing-3.0.9-py3-none-any.whl", hash = "sha256:5026bae9a10eeaefb61dab2f09052b9f4307d44aee4eda64b309723d8d206bbc"}, - {file = "pyparsing-3.0.9.tar.gz", hash = "sha256:2b020ecf7d21b687f219b71ecad3631f644a47f01403fa1d1036b0c6416d70fb"}, -] [package.extras] diagrams = ["jinja2", "railroad-diagrams"] +[[package]] +name = "pyproject-hooks" +version = "1.0.0" +description = "Wrappers to call pyproject.toml-based build backend hooks." +category = "dev" +optional = false +python-versions = ">=3.7" + +[package.dependencies] +tomli = {version = ">=1.1.0", markers = "python_version < \"3.11\""} + [[package]] name = "pytest" version = "7.2.1" @@ -1701,10 +840,6 @@ description = "pytest: simple powerful testing with Python" category = "dev" optional = false python-versions = ">=3.7" -files = [ - {file = "pytest-7.2.1-py3-none-any.whl", hash = "sha256:c7c6ca206e93355074ae32f7403e8ea12163b1163c976fee7d4d84027c162be5"}, - {file = "pytest-7.2.1.tar.gz", hash = "sha256:d45e0952f3727241918b8fd0f376f5ff6b301cc0777c6f9a556935c92d8a7d42"}, -] [package.dependencies] attrs = ">=19.2.0" @@ -1725,10 +860,6 @@ description = "check the README when running tests" category = "dev" optional = false python-versions = ">=3.7" -files = [ - {file = "pytest-checkdocs-2.9.0.tar.gz", hash = "sha256:54b3d1357a289e3f6336d31d4ceb72e99afe0fa4f552990f1b1f015f1925543b"}, - {file = "pytest_checkdocs-2.9.0-py3-none-any.whl", hash = "sha256:4231e5426915a02385e386b5eb4a70490b143d97cede4c7c77691bb320b5a73f"}, -] [package.dependencies] build = "*" @@ -1746,10 +877,6 @@ description = "Extensions to the standard Python datetime module" category = "main" optional = false python-versions = "!=3.0.*,!=3.1.*,!=3.2.*,>=2.7" -files = [ - {file = "python-dateutil-2.8.2.tar.gz", hash = "sha256:0123cacc1627ae19ddf3c27a5de5bd67ee4586fbdd6440d9748f8abb483d3e86"}, - {file = "python_dateutil-2.8.2-py2.py3-none-any.whl", hash = "sha256:961d03dc3453ebbc59dbdea9e4e11c5651520a876d0f4db161e8674aae935da9"}, -] [package.dependencies] six = ">=1.5" @@ -1761,68 +888,14 @@ description = "Python library for the snappy compression library from Google" category = "main" optional = true python-versions = "*" -files = [ - {file = "python-snappy-0.6.1.tar.gz", hash = "sha256:b6a107ab06206acc5359d4c5632bd9b22d448702a79b3169b0c62e0fb808bb2a"}, - {file = "python_snappy-0.6.1-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:b7f920eaf46ebf41bd26f9df51c160d40f9e00b7b48471c3438cb8d027f7fb9b"}, - {file = "python_snappy-0.6.1-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:4ec533a8c1f8df797bded662ec3e494d225b37855bb63eb0d75464a07947477c"}, - {file = "python_snappy-0.6.1-cp310-cp310-manylinux_2_12_i686.manylinux2010_i686.whl", hash = "sha256:6f8bf4708a11b47517baf962f9a02196478bbb10fdb9582add4aa1459fa82380"}, - {file = "python_snappy-0.6.1-cp310-cp310-manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:8d0c019ee7dcf2c60e240877107cddbd95a5b1081787579bf179938392d66480"}, - {file = "python_snappy-0.6.1-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:cb18d9cd7b3f35a2f5af47bb8ed6a5bdbf4f3ddee37f3daade4ab7864c292f5b"}, - {file = "python_snappy-0.6.1-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:b265cde49774752aec9ca7f5d272e3f98718164afc85521622a8a5394158a2b5"}, - {file = "python_snappy-0.6.1-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:d017775851a778ec9cc32651c4464079d06d927303c2dde9ae9830ccf6fe94e1"}, - {file = "python_snappy-0.6.1-cp310-cp310-win32.whl", hash = "sha256:8277d1f6282463c40761f802b742f833f9f2449fcdbb20a96579aa05c8feb614"}, - {file = "python_snappy-0.6.1-cp310-cp310-win_amd64.whl", hash = "sha256:2aaaf618c68d8c9daebc23a20436bd01b09ee70d7fbf7072b7f38b06d2fab539"}, - {file = "python_snappy-0.6.1-cp36-cp36m-macosx_10_9_x86_64.whl", hash = "sha256:277757d5dad4e239dc1417438a0871b65b1b155beb108888e7438c27ffc6a8cc"}, - {file = "python_snappy-0.6.1-cp36-cp36m-manylinux_2_12_i686.manylinux2010_i686.whl", hash = "sha256:e066a0586833d610c4bbddba0be5ba0e3e4f8e0bc5bb6d82103d8f8fc47bb59a"}, - {file = "python_snappy-0.6.1-cp36-cp36m-manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:0d489b50f49433494160c45048fe806de6b3aeab0586e497ebd22a0bab56e427"}, - {file = "python_snappy-0.6.1-cp36-cp36m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:463fd340a499d47b26ca42d2f36a639188738f6e2098c6dbf80aef0e60f461e1"}, - {file = "python_snappy-0.6.1-cp36-cp36m-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:9837ac1650cc68d22a3cf5f15fb62c6964747d16cecc8b22431f113d6e39555d"}, - {file = "python_snappy-0.6.1-cp36-cp36m-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:5e973e637112391f05581f427659c05b30b6843bc522a65be35ac7b18ce3dedd"}, - {file = "python_snappy-0.6.1-cp36-cp36m-win32.whl", hash = "sha256:c20498bd712b6e31a4402e1d027a1cd64f6a4a0066a3fe3c7344475886d07fdf"}, - {file = "python_snappy-0.6.1-cp36-cp36m-win_amd64.whl", hash = "sha256:59e975be4206cc54d0a112ef72fa3970a57c2b1bcc2c97ed41d6df0ebe518228"}, - {file = "python_snappy-0.6.1-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:2a7e528ab6e09c0d67dcb61a1730a292683e5ff9bb088950638d3170cf2a0a54"}, - {file = "python_snappy-0.6.1-cp37-cp37m-manylinux_2_12_i686.manylinux2010_i686.whl", hash = "sha256:39692bedbe0b717001a99915ac0eb2d9d0bad546440d392a2042b96d813eede1"}, - {file = "python_snappy-0.6.1-cp37-cp37m-manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:6a7620404da966f637b9ce8d4d3d543d363223f7a12452a575189c5355fc2d25"}, - {file = "python_snappy-0.6.1-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:7778c224efc38a40d274da4eb82a04cac27aae20012372a7db3c4bbd8926c4d4"}, - {file = "python_snappy-0.6.1-cp37-cp37m-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:1d029f7051ec1bbeaa3e03030b6d8ed47ceb69cae9016f493c802a08af54e026"}, - {file = "python_snappy-0.6.1-cp37-cp37m-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:a0ad38bc98d0b0497a0b0dbc29409bcabfcecff4511ed7063403c86de16927bc"}, - {file = "python_snappy-0.6.1-cp37-cp37m-win32.whl", hash = "sha256:5a453c45178d7864c1bdd6bfe0ee3ed2883f63b9ba2c9bb967c6b586bf763f96"}, - {file = "python_snappy-0.6.1-cp37-cp37m-win_amd64.whl", hash = "sha256:9f0c0d88b84259f93c3aa46398680646f2c23e43394779758d9f739c34e15295"}, - {file = "python_snappy-0.6.1-cp38-cp38-macosx_10_9_universal2.whl", hash = "sha256:5bb05c28298803a74add08ba496879242ef159c75bc86a5406fac0ffc7dd021b"}, - {file = "python_snappy-0.6.1-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:9eac51307c6a1a38d5f86ebabc26a889fddf20cbba7a116ccb54ba1446601d5b"}, - {file = "python_snappy-0.6.1-cp38-cp38-manylinux_2_12_i686.manylinux2010_i686.whl", hash = "sha256:88b6ea78b83d2796f330b0af1b70cdd3965dbdab02d8ac293260ec2c8fe340ee"}, - {file = "python_snappy-0.6.1-cp38-cp38-manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:8c07220408d3268e8268c9351c5c08041bc6f8c6172e59d398b71020df108541"}, - {file = "python_snappy-0.6.1-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:4038019b1bcaadde726a57430718394076c5a21545ebc5badad2c045a09546cf"}, - {file = "python_snappy-0.6.1-cp38-cp38-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:dc96668d9c7cc656609764275c5f8da58ef56d89bdd6810f6923d36497468ff7"}, - {file = "python_snappy-0.6.1-cp38-cp38-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:cf5bb9254e1c38aacf253d510d3d9be631bba21f3d068b17672b38b5cbf2fff5"}, - {file = "python_snappy-0.6.1-cp38-cp38-win32.whl", hash = "sha256:eaf905a580f2747c4a474040a5063cd5e0cc3d1d2d6edb65f28196186493ad4a"}, - {file = "python_snappy-0.6.1-cp38-cp38-win_amd64.whl", hash = "sha256:546c1a7470ecbf6239101e9aff0f709b68ca0f0268b34d9023019a55baa1f7c6"}, - {file = "python_snappy-0.6.1-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:e3a013895c64352b49d0d8e107a84f99631b16dbab156ded33ebf0becf56c8b2"}, - {file = "python_snappy-0.6.1-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:3fb9a88a4dd6336488f3de67ce75816d0d796dce53c2c6e4d70e0b565633c7fd"}, - {file = "python_snappy-0.6.1-cp39-cp39-manylinux_2_12_i686.manylinux2010_i686.whl", hash = "sha256:735cd4528c55dbe4516d6d2b403331a99fc304f8feded8ae887cf97b67d589bb"}, - {file = "python_snappy-0.6.1-cp39-cp39-manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:90b0186516b7a101c14764b0c25931b741fb0102f21253eff67847b4742dfc72"}, - {file = "python_snappy-0.6.1-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:1a993dc8aadd901915a510fe6af5f20ae4256f527040066c22a154db8946751f"}, - {file = "python_snappy-0.6.1-cp39-cp39-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:530bfb9efebcc1aab8bb4ebcbd92b54477eed11f6cf499355e882970a6d3aa7d"}, - {file = "python_snappy-0.6.1-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:5843feb914796b1f0405ccf31ea0fb51034ceb65a7588edfd5a8250cb369e3b2"}, - {file = "python_snappy-0.6.1-cp39-cp39-win32.whl", hash = "sha256:66c80e9b366012dbee262bb1869e4fc5ba8786cda85928481528bc4a72ec2ee8"}, - {file = "python_snappy-0.6.1-cp39-cp39-win_amd64.whl", hash = "sha256:4d3cafdf454354a621c8ab7408e45aa4e9d5c0b943b61ff4815f71ca6bdf0130"}, - {file = "python_snappy-0.6.1-pp37-pypy37_pp73-manylinux_2_12_i686.manylinux2010_i686.whl", hash = "sha256:586724a0276d7a6083a17259d0b51622e492289a9998848a1b01b6441ca12b2f"}, - {file = "python_snappy-0.6.1-pp37-pypy37_pp73-manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:2be4f4550acd484912441f5f1209ba611ac399aac9355fee73611b9a0d4f949c"}, - {file = "python_snappy-0.6.1-pp37-pypy37_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:0bdb6942180660bda7f7d01f4c0def3cfc72b1c6d99aad964801775a3e379aba"}, - {file = "python_snappy-0.6.1-pp37-pypy37_pp73-win_amd64.whl", hash = "sha256:03bb511380fca2a13325b6f16fe8234c8e12da9660f0258cd45d9a02ffc916af"}, -] [[package]] name = "pytz" -version = "2022.6" +version = "2022.7.1" description = "World timezone definitions, modern and historical" category = "main" optional = true python-versions = "*" -files = [ - {file = "pytz-2022.6-py2.py3-none-any.whl", hash = "sha256:222439474e9c98fced559f1709d89e6c9cbf8d79c794ff3eb9f8800064291427"}, - {file = "pytz-2022.6.tar.gz", hash = "sha256:e89512406b793ca39f5971bc999cc538ce125c0e51c27941bef4568b460095e2"}, -] [[package]] name = "pywin32" @@ -1831,22 +904,6 @@ description = "Python for Window Extensions" category = "main" optional = true python-versions = "*" -files = [ - {file = "pywin32-305-cp310-cp310-win32.whl", hash = "sha256:421f6cd86e84bbb696d54563c48014b12a23ef95a14e0bdba526be756d89f116"}, - {file = "pywin32-305-cp310-cp310-win_amd64.whl", hash = "sha256:73e819c6bed89f44ff1d690498c0a811948f73777e5f97c494c152b850fad478"}, - {file = "pywin32-305-cp310-cp310-win_arm64.whl", hash = "sha256:742eb905ce2187133a29365b428e6c3b9001d79accdc30aa8969afba1d8470f4"}, - {file = "pywin32-305-cp311-cp311-win32.whl", hash = "sha256:19ca459cd2e66c0e2cc9a09d589f71d827f26d47fe4a9d09175f6aa0256b51c2"}, - {file = "pywin32-305-cp311-cp311-win_amd64.whl", hash = "sha256:326f42ab4cfff56e77e3e595aeaf6c216712bbdd91e464d167c6434b28d65990"}, - {file = "pywin32-305-cp311-cp311-win_arm64.whl", hash = "sha256:4ecd404b2c6eceaca52f8b2e3e91b2187850a1ad3f8b746d0796a98b4cea04db"}, - {file = "pywin32-305-cp36-cp36m-win32.whl", hash = "sha256:48d8b1659284f3c17b68587af047d110d8c44837736b8932c034091683e05863"}, - {file = "pywin32-305-cp36-cp36m-win_amd64.whl", hash = "sha256:13362cc5aa93c2beaf489c9c9017c793722aeb56d3e5166dadd5ef82da021fe1"}, - {file = "pywin32-305-cp37-cp37m-win32.whl", hash = "sha256:a55db448124d1c1484df22fa8bbcbc45c64da5e6eae74ab095b9ea62e6d00496"}, - {file = "pywin32-305-cp37-cp37m-win_amd64.whl", hash = "sha256:109f98980bfb27e78f4df8a51a8198e10b0f347257d1e265bb1a32993d0c973d"}, - {file = "pywin32-305-cp38-cp38-win32.whl", hash = "sha256:9dd98384da775afa009bc04863426cb30596fd78c6f8e4e2e5bbf4edf8029504"}, - {file = "pywin32-305-cp38-cp38-win_amd64.whl", hash = "sha256:56d7a9c6e1a6835f521788f53b5af7912090674bb84ef5611663ee1595860fc7"}, - {file = "pywin32-305-cp39-cp39-win32.whl", hash = "sha256:9d968c677ac4d5cbdaa62fd3014ab241718e619d8e36ef8e11fb930515a1e918"}, - {file = "pywin32-305-cp39-cp39-win_amd64.whl", hash = "sha256:50768c6b7c3f0b38b7fb14dd4104da93ebced5f1a50dc0e834594bff6fbe1271"}, -] [[package]] name = "pyyaml" @@ -1855,48 +912,6 @@ description = "YAML parser and emitter for Python" category = "main" optional = false python-versions = ">=3.6" -files = [ - {file = "PyYAML-6.0-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:d4db7c7aef085872ef65a8fd7d6d09a14ae91f691dec3e87ee5ee0539d516f53"}, - {file = "PyYAML-6.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:9df7ed3b3d2e0ecfe09e14741b857df43adb5a3ddadc919a2d94fbdf78fea53c"}, - {file = "PyYAML-6.0-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:77f396e6ef4c73fdc33a9157446466f1cff553d979bd00ecb64385760c6babdc"}, - {file = "PyYAML-6.0-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:a80a78046a72361de73f8f395f1f1e49f956c6be882eed58505a15f3e430962b"}, - {file = "PyYAML-6.0-cp310-cp310-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:f84fbc98b019fef2ee9a1cb3ce93e3187a6df0b2538a651bfb890254ba9f90b5"}, - {file = "PyYAML-6.0-cp310-cp310-win32.whl", hash = "sha256:2cd5df3de48857ed0544b34e2d40e9fac445930039f3cfe4bcc592a1f836d513"}, - {file = "PyYAML-6.0-cp310-cp310-win_amd64.whl", hash = "sha256:daf496c58a8c52083df09b80c860005194014c3698698d1a57cbcfa182142a3a"}, - {file = "PyYAML-6.0-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:d4b0ba9512519522b118090257be113b9468d804b19d63c71dbcf4a48fa32358"}, - {file = "PyYAML-6.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:81957921f441d50af23654aa6c5e5eaf9b06aba7f0a19c18a538dc7ef291c5a1"}, - {file = "PyYAML-6.0-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:afa17f5bc4d1b10afd4466fd3a44dc0e245382deca5b3c353d8b757f9e3ecb8d"}, - {file = "PyYAML-6.0-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:dbad0e9d368bb989f4515da330b88a057617d16b6a8245084f1b05400f24609f"}, - {file = "PyYAML-6.0-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:432557aa2c09802be39460360ddffd48156e30721f5e8d917f01d31694216782"}, - {file = "PyYAML-6.0-cp311-cp311-win32.whl", hash = "sha256:bfaef573a63ba8923503d27530362590ff4f576c626d86a9fed95822a8255fd7"}, - {file = "PyYAML-6.0-cp311-cp311-win_amd64.whl", hash = "sha256:01b45c0191e6d66c470b6cf1b9531a771a83c1c4208272ead47a3ae4f2f603bf"}, - {file = "PyYAML-6.0-cp36-cp36m-macosx_10_9_x86_64.whl", hash = "sha256:897b80890765f037df3403d22bab41627ca8811ae55e9a722fd0392850ec4d86"}, - {file = "PyYAML-6.0-cp36-cp36m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:50602afada6d6cbfad699b0c7bb50d5ccffa7e46a3d738092afddc1f9758427f"}, - {file = "PyYAML-6.0-cp36-cp36m-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:48c346915c114f5fdb3ead70312bd042a953a8ce5c7106d5bfb1a5254e47da92"}, - {file = "PyYAML-6.0-cp36-cp36m-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:98c4d36e99714e55cfbaaee6dd5badbc9a1ec339ebfc3b1f52e293aee6bb71a4"}, - {file = "PyYAML-6.0-cp36-cp36m-win32.whl", hash = "sha256:0283c35a6a9fbf047493e3a0ce8d79ef5030852c51e9d911a27badfde0605293"}, - {file = "PyYAML-6.0-cp36-cp36m-win_amd64.whl", hash = "sha256:07751360502caac1c067a8132d150cf3d61339af5691fe9e87803040dbc5db57"}, - {file = "PyYAML-6.0-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:819b3830a1543db06c4d4b865e70ded25be52a2e0631ccd2f6a47a2822f2fd7c"}, - {file = "PyYAML-6.0-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:473f9edb243cb1935ab5a084eb238d842fb8f404ed2193a915d1784b5a6b5fc0"}, - {file = "PyYAML-6.0-cp37-cp37m-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:0ce82d761c532fe4ec3f87fc45688bdd3a4c1dc5e0b4a19814b9009a29baefd4"}, - {file = "PyYAML-6.0-cp37-cp37m-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:231710d57adfd809ef5d34183b8ed1eeae3f76459c18fb4a0b373ad56bedcdd9"}, - {file = "PyYAML-6.0-cp37-cp37m-win32.whl", hash = "sha256:c5687b8d43cf58545ade1fe3e055f70eac7a5a1a0bf42824308d868289a95737"}, - {file = "PyYAML-6.0-cp37-cp37m-win_amd64.whl", hash = "sha256:d15a181d1ecd0d4270dc32edb46f7cb7733c7c508857278d3d378d14d606db2d"}, - {file = "PyYAML-6.0-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:0b4624f379dab24d3725ffde76559cff63d9ec94e1736b556dacdfebe5ab6d4b"}, - {file = "PyYAML-6.0-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:213c60cd50106436cc818accf5baa1aba61c0189ff610f64f4a3e8c6726218ba"}, - {file = "PyYAML-6.0-cp38-cp38-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:9fa600030013c4de8165339db93d182b9431076eb98eb40ee068700c9c813e34"}, - {file = "PyYAML-6.0-cp38-cp38-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:277a0ef2981ca40581a47093e9e2d13b3f1fbbeffae064c1d21bfceba2030287"}, - {file = "PyYAML-6.0-cp38-cp38-win32.whl", hash = "sha256:d4eccecf9adf6fbcc6861a38015c2a64f38b9d94838ac1810a9023a0609e1b78"}, - {file = "PyYAML-6.0-cp38-cp38-win_amd64.whl", hash = "sha256:1e4747bc279b4f613a09eb64bba2ba602d8a6664c6ce6396a4d0cd413a50ce07"}, - {file = "PyYAML-6.0-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:055d937d65826939cb044fc8c9b08889e8c743fdc6a32b33e2390f66013e449b"}, - {file = "PyYAML-6.0-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:e61ceaab6f49fb8bdfaa0f92c4b57bcfbea54c09277b1b4f7ac376bfb7a7c174"}, - {file = "PyYAML-6.0-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:d67d839ede4ed1b28a4e8909735fc992a923cdb84e618544973d7dfc71540803"}, - {file = "PyYAML-6.0-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:cba8c411ef271aa037d7357a2bc8f9ee8b58b9965831d9e51baf703280dc73d3"}, - {file = "PyYAML-6.0-cp39-cp39-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:40527857252b61eacd1d9af500c3337ba8deb8fc298940291486c465c8b46ec0"}, - {file = "PyYAML-6.0-cp39-cp39-win32.whl", hash = "sha256:b5b9eccad747aabaaffbc6064800670f0c297e52c12754eb1d976c57e4f74dcb"}, - {file = "PyYAML-6.0-cp39-cp39-win_amd64.whl", hash = "sha256:b3d267842bf12586ba6c734f89d1f5b871df0273157918b0ccefa29deb05c21c"}, - {file = "PyYAML-6.0.tar.gz", hash = "sha256:68fb519c14306fec9720a2a5b45bc9f0c8d1b9c72adf45c37baedfcd949c35a2"}, -] [[package]] name = "requests" @@ -1905,10 +920,6 @@ description = "Python HTTP for Humans." category = "main" optional = false python-versions = ">=3.7, <4" -files = [ - {file = "requests-2.28.2-py3-none-any.whl", hash = "sha256:64299f4909223da747622c030b781c0d7811e359c37124b4bd368fb8c6518baa"}, - {file = "requests-2.28.2.tar.gz", hash = "sha256:98b1b2782e3c6c4904938b84c0eb932721069dfdb9134313beff7c83c2df24bf"}, -] [package.dependencies] certifi = ">=2017.4.17" @@ -1918,7 +929,7 @@ urllib3 = ">=1.21.1,<1.27" [package.extras] socks = ["PySocks (>=1.5.6,!=1.5.7)"] -use-chardet-on-py3 = ["chardet (>=3.0.2,<6)"] +use_chardet_on_py3 = ["chardet (>=3.0.2,<6)"] [[package]] name = "requests-mock" @@ -1927,10 +938,6 @@ description = "Mock out responses from the requests package" category = "dev" optional = false python-versions = "*" -files = [ - {file = "requests-mock-1.10.0.tar.gz", hash = "sha256:59c9c32419a9fb1ae83ec242d98e889c45bd7d7a65d48375cc243ec08441658b"}, - {file = "requests_mock-1.10.0-py2.py3-none-any.whl", hash = "sha256:2fdbb637ad17ee15c06f33d31169e71bf9fe2bdb7bc9da26185be0dd8d842699"}, -] [package.dependencies] requests = ">=2.3,<3" @@ -1947,10 +954,6 @@ description = "OAuthlib authentication support for Requests." category = "main" optional = true python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*" -files = [ - {file = "requests-oauthlib-1.3.1.tar.gz", hash = "sha256:75beac4a47881eeb94d5ea5d6ad31ef88856affe2332b9aafb52c6452ccf0d7a"}, - {file = "requests_oauthlib-1.3.1-py2.py3-none-any.whl", hash = "sha256:2577c501a2fb8d05a304c09d090d6e47c306fef15809d102b327cf8364bddab5"}, -] [package.dependencies] oauthlib = ">=3.0.0" @@ -1966,10 +969,6 @@ description = "A utility library for mocking out the `requests` Python library." category = "dev" optional = false python-versions = ">=3.7" -files = [ - {file = "responses-0.22.0-py3-none-any.whl", hash = "sha256:dcf294d204d14c436fddcc74caefdbc5764795a40ff4e6a7740ed8ddbf3294be"}, - {file = "responses-0.22.0.tar.gz", hash = "sha256:396acb2a13d25297789a5866b4881cf4e46ffd49cc26c43ab1117f40b973102e"}, -] [package.dependencies] requests = ">=2.22.0,<3.0" @@ -1987,10 +986,6 @@ description = "Render rich text, tables, progress bars, syntax highlighting, mar category = "main" optional = false python-versions = ">=3.7.0" -files = [ - {file = "rich-13.2.0-py3-none-any.whl", hash = "sha256:7c963f0d03819221e9ac561e1bc866e3f95a02248c1234daa48954e6d381c003"}, - {file = "rich-13.2.0.tar.gz", hash = "sha256:f1a00cdd3eebf999a15d85ec498bfe0b1a77efe9b34f645768a54132ef444ac5"}, -] [package.dependencies] markdown-it-py = ">=2.1.0,<3.0.0" @@ -2002,24 +997,20 @@ jupyter = ["ipywidgets (>=7.5.1,<8.0.0)"] [[package]] name = "s3fs" -version = "2022.10.0" +version = "2023.1.0" description = "Convenient Filesystem interface over S3" category = "main" optional = true python-versions = ">= 3.7" -files = [ - {file = "s3fs-2022.10.0-py3-none-any.whl", hash = "sha256:1e134c3577171699feb7c1a0c4713260d5b48296e1708737ff940baef6e2c153"}, - {file = "s3fs-2022.10.0.tar.gz", hash = "sha256:e8deb80f20bd0b2059141b874fdb9d6aeb8cce35312ea5f2c02b225a78a00406"}, -] [package.dependencies] -aiobotocore = ">=2.4.0,<2.5.0" +aiobotocore = ">=2.4.2,<2.5.0" aiohttp = "<4.0.0a0 || >4.0.0a0,<4.0.0a1 || >4.0.0a1" -fsspec = "2022.10.0" +fsspec = "2023.1.0" [package.extras] -awscli = ["aiobotocore[awscli] (>=2.4.0,<2.5.0)"] -boto3 = ["aiobotocore[boto3] (>=2.4.0,<2.5.0)"] +awscli = ["aiobotocore[awscli] (>=2.4.2,<2.5.0)"] +boto3 = ["aiobotocore[boto3] (>=2.4.2,<2.5.0)"] [[package]] name = "s3transfer" @@ -2028,10 +1019,6 @@ description = "An Amazon S3 Transfer Manager" category = "main" optional = false python-versions = ">= 3.7" -files = [ - {file = "s3transfer-0.6.0-py3-none-any.whl", hash = "sha256:06176b74f3a15f61f1b4f25a1fc29a4429040b7647133a463da8fa5bd28d5ecd"}, - {file = "s3transfer-0.6.0.tar.gz", hash = "sha256:2ed07d3866f523cc561bf4a00fc5535827981b117dd7876f036b0c1aca42c947"}, -] [package.dependencies] botocore = ">=1.12.36,<2.0a.0" @@ -2039,23 +1026,6 @@ botocore = ">=1.12.36,<2.0a.0" [package.extras] crt = ["botocore[crt] (>=1.20.29,<2.0a.0)"] -[[package]] -name = "setuptools" -version = "65.6.3" -description = "Easily download, build, install, upgrade, and uninstall Python packages" -category = "dev" -optional = false -python-versions = ">=3.7" -files = [ - {file = "setuptools-65.6.3-py3-none-any.whl", hash = "sha256:57f6f22bde4e042978bcd50176fdb381d7c21a9efa4041202288d3737a0c6a54"}, - {file = "setuptools-65.6.3.tar.gz", hash = "sha256:a7620757bf984b58deaf32fc8a4577a9bbc0850cf92c20e1ce41c38c19e5fb75"}, -] - -[package.extras] -docs = ["furo", "jaraco.packaging (>=9)", "jaraco.tidelift (>=1.4)", "pygments-github-lexers (==0.0.5)", "rst.linker (>=1.9)", "sphinx (>=3.5)", "sphinx-favicon", "sphinx-hoverxref (<2)", "sphinx-inline-tabs", "sphinx-notfound-page (==0.8.3)", "sphinx-reredirects", "sphinxcontrib-towncrier"] -testing = ["build[virtualenv]", "filelock (>=3.4.0)", "flake8 (<5)", "flake8-2020", "ini2toml[lite] (>=0.9)", "jaraco.envs (>=2.2)", "jaraco.path (>=3.2.0)", "pip (>=19.1)", "pip-run (>=8.8)", "pytest (>=6)", "pytest-black (>=0.3.7)", "pytest-checkdocs (>=2.4)", "pytest-cov", "pytest-enabler (>=1.3)", "pytest-flake8", "pytest-mypy (>=0.9.1)", "pytest-perf", "pytest-timeout", "pytest-xdist", "tomli-w (>=1.0.0)", "virtualenv (>=13.0.0)", "wheel"] -testing-integration = ["build[virtualenv]", "filelock (>=3.4.0)", "jaraco.envs (>=2.2)", "jaraco.path (>=3.2.0)", "pytest", "pytest-enabler", "pytest-xdist", "tomli", "virtualenv (>=13.0.0)", "wheel"] - [[package]] name = "six" version = "1.16.0" @@ -2063,10 +1033,6 @@ description = "Python 2 and 3 compatibility utilities" category = "main" optional = false python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*" -files = [ - {file = "six-1.16.0-py2.py3-none-any.whl", hash = "sha256:8abb2f1d86890a2dfb989f9a77cfcfd3e47c2a354b01111771326f8aa26e0254"}, - {file = "six-1.16.0.tar.gz", hash = "sha256:1e61c37477a1626458e36f7b1d82aa5c9b094fa4802892072e49de9c60c4c926"}, -] [[package]] name = "thrift" @@ -2075,9 +1041,6 @@ description = "Python bindings for the Apache Thrift RPC system" category = "main" optional = true python-versions = "*" -files = [ - {file = "thrift-0.16.0.tar.gz", hash = "sha256:2b5b6488fcded21f9d312aa23c9ff6a0195d0f6ae26ddbd5ad9e3e25dfc14408"}, -] [package.dependencies] six = ">=1.7.2" @@ -2094,10 +1057,6 @@ description = "Python Library for Tom's Obvious, Minimal Language" category = "dev" optional = false python-versions = ">=2.6, !=3.0.*, !=3.1.*, !=3.2.*" -files = [ - {file = "toml-0.10.2-py2.py3-none-any.whl", hash = "sha256:806143ae5bfb6a3c6e736a764057db0e6a0e05e338b5630894a5f779cabb4f9b"}, - {file = "toml-0.10.2.tar.gz", hash = "sha256:b3bda1d108d5dd99f4a20d24d9c348e91c4db7ab1b749200bded2f839ccbe68f"}, -] [[package]] name = "tomli" @@ -2106,10 +1065,6 @@ description = "A lil' TOML parser" category = "dev" optional = false python-versions = ">=3.7" -files = [ - {file = "tomli-2.0.1-py3-none-any.whl", hash = "sha256:939de3e7a6161af0c887ef91b7d41a53e7c5a1ca976325f429cb46ea9bc30ecc"}, - {file = "tomli-2.0.1.tar.gz", hash = "sha256:de526c12914f0c550d15924c62d72abc48d6fe7364aa87328337a31007fe8a4f"}, -] [[package]] name = "types-toml" @@ -2118,10 +1073,6 @@ description = "Typing stubs for toml" category = "dev" optional = false python-versions = "*" -files = [ - {file = "types-toml-0.10.8.1.tar.gz", hash = "sha256:171bdb3163d79a520560f24ba916a9fc9bff81659c5448a9fea89240923722be"}, - {file = "types_toml-0.10.8.1-py3-none-any.whl", hash = "sha256:b7b5c4977f96ab7b5ac06d8a6590d17c0bf252a96efc03b109c2711fb3e0eafd"}, -] [[package]] name = "typing-extensions" @@ -2130,22 +1081,14 @@ description = "Backported and Experimental Type Hints for Python 3.7+" category = "main" optional = false python-versions = ">=3.7" -files = [ - {file = "typing_extensions-4.4.0-py3-none-any.whl", hash = "sha256:16fa4864408f655d35ec496218b85f79b3437c829e93320c7c9215ccfd92489e"}, - {file = "typing_extensions-4.4.0.tar.gz", hash = "sha256:1511434bb92bf8dd198c12b1cc812e800d4181cfcb867674e0f8279cc93087aa"}, -] [[package]] name = "urllib3" -version = "1.26.13" +version = "1.26.14" description = "HTTP library with thread-safe connection pooling, file post, and more." category = "main" optional = false python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*, !=3.5.*" -files = [ - {file = "urllib3-1.26.13-py2.py3-none-any.whl", hash = "sha256:47cc05d99aaa09c9e72ed5809b60e7ba354e64b59c9c173ac3018642d8bb41fc"}, - {file = "urllib3-1.26.13.tar.gz", hash = "sha256:c083dd0dce68dbfbe1129d5271cb90f9447dea7d52097c6e0126120c521ddea8"}, -] [package.extras] brotli = ["brotli (>=1.0.9)", "brotlicffi (>=0.8.0)", "brotlipy (>=0.6.0)"] @@ -2159,46 +1102,1126 @@ description = "Virtual Python Environment builder" category = "dev" optional = false python-versions = ">=3.6" -files = [ + +[package.dependencies] +distlib = ">=0.3.6,<1" +filelock = ">=3.4.1,<4" +platformdirs = ">=2.4,<3" + +[package.extras] +docs = ["proselint (>=0.13)", "sphinx (>=5.3)", "sphinx-argparse (>=0.3.2)", "sphinx-rtd-theme (>=1)", "towncrier (>=22.8)"] +testing = ["coverage (>=6.2)", "coverage-enable-subprocess (>=1)", "flaky (>=3.7)", "packaging (>=21.3)", "pytest (>=7.0.1)", "pytest-env (>=0.6.2)", "pytest-freezegun (>=0.4.2)", "pytest-mock (>=3.6.1)", "pytest-randomly (>=3.10.3)", "pytest-timeout (>=2.1)"] + +[[package]] +name = "werkzeug" +version = "2.2.2" +description = "The comprehensive WSGI web application library." +category = "dev" +optional = false +python-versions = ">=3.7" + +[package.dependencies] +MarkupSafe = ">=2.1.1" + +[package.extras] +watchdog = ["watchdog"] + +[[package]] +name = "wrapt" +version = "1.14.1" +description = "Module for decorators, wrappers and monkey patching." +category = "main" +optional = true +python-versions = "!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,!=3.4.*,>=2.7" + +[[package]] +name = "xmltodict" +version = "0.13.0" +description = "Makes working with XML feel like you are working with JSON" +category = "dev" +optional = false +python-versions = ">=3.4" + +[[package]] +name = "yarl" +version = "1.8.2" +description = "Yet another URL library" +category = "main" +optional = true +python-versions = ">=3.7" + +[package.dependencies] +idna = ">=2.0" +multidict = ">=4.0" + +[[package]] +name = "zipp" +version = "3.11.0" +description = "Backport of pathlib-compatible object wrapper for zip files" +category = "dev" +optional = false +python-versions = ">=3.7" + +[package.extras] +docs = ["furo", "jaraco.packaging (>=9)", "jaraco.tidelift (>=1.4)", "rst.linker (>=1.9)", "sphinx (>=3.5)"] +testing = ["flake8 (<5)", "func-timeout", "jaraco.functools", "jaraco.itertools", "more-itertools", "pytest (>=6)", "pytest-black (>=0.3.7)", "pytest-checkdocs (>=2.4)", "pytest-cov", "pytest-enabler (>=1.3)", "pytest-flake8", "pytest-mypy (>=0.9.1)"] + +[[package]] +name = "zstandard" +version = "0.19.0" +description = "Zstandard bindings for Python" +category = "main" +optional = false +python-versions = ">=3.6" + +[package.dependencies] +cffi = {version = ">=1.11", markers = "platform_python_implementation == \"PyPy\""} + +[package.extras] +cffi = ["cffi (>=1.11)"] + +[extras] +adlfs = ["adlfs"] +duckdb = ["duckdb", "pyarrow"] +glue = ["boto3"] +hive = ["thrift"] +pandas = ["pandas", "pyarrow"] +pyarrow = ["pyarrow"] +s3fs = ["s3fs"] +snappy = ["python-snappy"] + +[metadata] +lock-version = "1.1" +python-versions = "^3.8" +content-hash = "dd8d3af780197201f569d17aeff10b22d5b807bdb7949d5e9e234851b88ce0ce" + +[metadata.files] +adal = [ + {file = "adal-1.2.7-py2.py3-none-any.whl", hash = "sha256:2a7451ed7441ddbc57703042204a3e30ef747478eea022c70f789fc7f084bc3d"}, + {file = "adal-1.2.7.tar.gz", hash = "sha256:d74f45b81317454d96e982fd1c50e6fb5c99ac2223728aea8764433a39f566f1"}, +] +adlfs = [ + {file = "adlfs-2023.1.0-py3-none-any.whl", hash = "sha256:ccbdd6d33d5b7bce99a4a07c884c82fb135745d39b2d9b7b672f8e9d4d04407f"}, + {file = "adlfs-2023.1.0.tar.gz", hash = "sha256:eca53f53d88fc8e2e7a2f1d8f5a40b8a1c56aa3b541e4aa2e7eaf55a6a789262"}, +] +aiobotocore = [ + {file = "aiobotocore-2.4.2-py3-none-any.whl", hash = "sha256:4acd1ebe2e44be4b100aa553910bda899f6dc090b3da2bc1cf3d5de2146ed208"}, + {file = "aiobotocore-2.4.2.tar.gz", hash = "sha256:0603b74a582dffa7511ce7548d07dc9b10ec87bc5fb657eb0b34f9bd490958bf"}, +] +aiohttp = [ + {file = "aiohttp-3.8.3-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:ba71c9b4dcbb16212f334126cc3d8beb6af377f6703d9dc2d9fb3874fd667ee9"}, + {file = "aiohttp-3.8.3-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:d24b8bb40d5c61ef2d9b6a8f4528c2f17f1c5d2d31fed62ec860f6006142e83e"}, + {file = "aiohttp-3.8.3-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:f88df3a83cf9df566f171adba39d5bd52814ac0b94778d2448652fc77f9eb491"}, + {file = "aiohttp-3.8.3-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:b97decbb3372d4b69e4d4c8117f44632551c692bb1361b356a02b97b69e18a62"}, + {file = "aiohttp-3.8.3-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:309aa21c1d54b8ef0723181d430347d7452daaff93e8e2363db8e75c72c2fb2d"}, + {file = "aiohttp-3.8.3-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:ad5383a67514e8e76906a06741febd9126fc7c7ff0f599d6fcce3e82b80d026f"}, + {file = "aiohttp-3.8.3-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:20acae4f268317bb975671e375493dbdbc67cddb5f6c71eebdb85b34444ac46b"}, + {file = "aiohttp-3.8.3-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:05a3c31c6d7cd08c149e50dc7aa2568317f5844acd745621983380597f027a18"}, + {file = "aiohttp-3.8.3-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:d6f76310355e9fae637c3162936e9504b4767d5c52ca268331e2756e54fd4ca5"}, + {file = "aiohttp-3.8.3-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:256deb4b29fe5e47893fa32e1de2d73c3afe7407738bd3c63829874661d4822d"}, + {file = "aiohttp-3.8.3-cp310-cp310-musllinux_1_1_ppc64le.whl", hash = "sha256:5c59fcd80b9049b49acd29bd3598cada4afc8d8d69bd4160cd613246912535d7"}, + {file = "aiohttp-3.8.3-cp310-cp310-musllinux_1_1_s390x.whl", hash = "sha256:059a91e88f2c00fe40aed9031b3606c3f311414f86a90d696dd982e7aec48142"}, + {file = "aiohttp-3.8.3-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:2feebbb6074cdbd1ac276dbd737b40e890a1361b3cc30b74ac2f5e24aab41f7b"}, + {file = "aiohttp-3.8.3-cp310-cp310-win32.whl", hash = "sha256:5bf651afd22d5f0c4be16cf39d0482ea494f5c88f03e75e5fef3a85177fecdeb"}, + {file = "aiohttp-3.8.3-cp310-cp310-win_amd64.whl", hash = "sha256:653acc3880459f82a65e27bd6526e47ddf19e643457d36a2250b85b41a564715"}, + {file = "aiohttp-3.8.3-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:86fc24e58ecb32aee09f864cb11bb91bc4c1086615001647dbfc4dc8c32f4008"}, + {file = "aiohttp-3.8.3-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:75e14eac916f024305db517e00a9252714fce0abcb10ad327fb6dcdc0d060f1d"}, + {file = "aiohttp-3.8.3-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:d1fde0f44029e02d02d3993ad55ce93ead9bb9b15c6b7ccd580f90bd7e3de476"}, + {file = "aiohttp-3.8.3-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:4ab94426ddb1ecc6a0b601d832d5d9d421820989b8caa929114811369673235c"}, + {file = "aiohttp-3.8.3-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:89d2e02167fa95172c017732ed7725bc8523c598757f08d13c5acca308e1a061"}, + {file = "aiohttp-3.8.3-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:02f9a2c72fc95d59b881cf38a4b2be9381b9527f9d328771e90f72ac76f31ad8"}, + {file = "aiohttp-3.8.3-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:9c7149272fb5834fc186328e2c1fa01dda3e1fa940ce18fded6d412e8f2cf76d"}, + {file = "aiohttp-3.8.3-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:512bd5ab136b8dc0ffe3fdf2dfb0c4b4f49c8577f6cae55dca862cd37a4564e2"}, + {file = "aiohttp-3.8.3-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:7018ecc5fe97027214556afbc7c502fbd718d0740e87eb1217b17efd05b3d276"}, + {file = "aiohttp-3.8.3-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:88c70ed9da9963d5496d38320160e8eb7e5f1886f9290475a881db12f351ab5d"}, + {file = "aiohttp-3.8.3-cp311-cp311-musllinux_1_1_ppc64le.whl", hash = "sha256:da22885266bbfb3f78218dc40205fed2671909fbd0720aedba39b4515c038091"}, + {file = "aiohttp-3.8.3-cp311-cp311-musllinux_1_1_s390x.whl", hash = "sha256:e65bc19919c910127c06759a63747ebe14f386cda573d95bcc62b427ca1afc73"}, + {file = "aiohttp-3.8.3-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:08c78317e950e0762c2983f4dd58dc5e6c9ff75c8a0efeae299d363d439c8e34"}, + {file = "aiohttp-3.8.3-cp311-cp311-win32.whl", hash = "sha256:45d88b016c849d74ebc6f2b6e8bc17cabf26e7e40c0661ddd8fae4c00f015697"}, + {file = "aiohttp-3.8.3-cp311-cp311-win_amd64.whl", hash = "sha256:96372fc29471646b9b106ee918c8eeb4cca423fcbf9a34daa1b93767a88a2290"}, + {file = "aiohttp-3.8.3-cp36-cp36m-macosx_10_9_x86_64.whl", hash = "sha256:c971bf3786b5fad82ce5ad570dc6ee420f5b12527157929e830f51c55dc8af77"}, + {file = "aiohttp-3.8.3-cp36-cp36m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:ff25f48fc8e623d95eca0670b8cc1469a83783c924a602e0fbd47363bb54aaca"}, + {file = "aiohttp-3.8.3-cp36-cp36m-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:e381581b37db1db7597b62a2e6b8b57c3deec95d93b6d6407c5b61ddc98aca6d"}, + {file = "aiohttp-3.8.3-cp36-cp36m-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:db19d60d846283ee275d0416e2a23493f4e6b6028825b51290ac05afc87a6f97"}, + {file = "aiohttp-3.8.3-cp36-cp36m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:25892c92bee6d9449ffac82c2fe257f3a6f297792cdb18ad784737d61e7a9a85"}, + {file = "aiohttp-3.8.3-cp36-cp36m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:398701865e7a9565d49189f6c90868efaca21be65c725fc87fc305906be915da"}, + {file = "aiohttp-3.8.3-cp36-cp36m-musllinux_1_1_aarch64.whl", hash = "sha256:4a4fbc769ea9b6bd97f4ad0b430a6807f92f0e5eb020f1e42ece59f3ecfc4585"}, + {file = "aiohttp-3.8.3-cp36-cp36m-musllinux_1_1_i686.whl", hash = "sha256:b29bfd650ed8e148f9c515474a6ef0ba1090b7a8faeee26b74a8ff3b33617502"}, + {file = "aiohttp-3.8.3-cp36-cp36m-musllinux_1_1_ppc64le.whl", hash = "sha256:1e56b9cafcd6531bab5d9b2e890bb4937f4165109fe98e2b98ef0dcfcb06ee9d"}, + {file = "aiohttp-3.8.3-cp36-cp36m-musllinux_1_1_s390x.whl", hash = "sha256:ec40170327d4a404b0d91855d41bfe1fe4b699222b2b93e3d833a27330a87a6d"}, + {file = "aiohttp-3.8.3-cp36-cp36m-musllinux_1_1_x86_64.whl", hash = "sha256:2df5f139233060578d8c2c975128fb231a89ca0a462b35d4b5fcf7c501ebdbe1"}, + {file = "aiohttp-3.8.3-cp36-cp36m-win32.whl", hash = "sha256:f973157ffeab5459eefe7b97a804987876dd0a55570b8fa56b4e1954bf11329b"}, + {file = "aiohttp-3.8.3-cp36-cp36m-win_amd64.whl", hash = "sha256:437399385f2abcd634865705bdc180c8314124b98299d54fe1d4c8990f2f9494"}, + {file = "aiohttp-3.8.3-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:09e28f572b21642128ef31f4e8372adb6888846f32fecb288c8b0457597ba61a"}, + {file = "aiohttp-3.8.3-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:6f3553510abdbec67c043ca85727396ceed1272eef029b050677046d3387be8d"}, + {file = "aiohttp-3.8.3-cp37-cp37m-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:e168a7560b7c61342ae0412997b069753f27ac4862ec7867eff74f0fe4ea2ad9"}, + {file = "aiohttp-3.8.3-cp37-cp37m-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:db4c979b0b3e0fa7e9e69ecd11b2b3174c6963cebadeecfb7ad24532ffcdd11a"}, + {file = "aiohttp-3.8.3-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:e164e0a98e92d06da343d17d4e9c4da4654f4a4588a20d6c73548a29f176abe2"}, + {file = "aiohttp-3.8.3-cp37-cp37m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:e8a78079d9a39ca9ca99a8b0ac2fdc0c4d25fc80c8a8a82e5c8211509c523363"}, + {file = "aiohttp-3.8.3-cp37-cp37m-musllinux_1_1_aarch64.whl", hash = "sha256:21b30885a63c3f4ff5b77a5d6caf008b037cb521a5f33eab445dc566f6d092cc"}, + {file = "aiohttp-3.8.3-cp37-cp37m-musllinux_1_1_i686.whl", hash = "sha256:4b0f30372cef3fdc262f33d06e7b411cd59058ce9174ef159ad938c4a34a89da"}, + {file = "aiohttp-3.8.3-cp37-cp37m-musllinux_1_1_ppc64le.whl", hash = "sha256:8135fa153a20d82ffb64f70a1b5c2738684afa197839b34cc3e3c72fa88d302c"}, + {file = "aiohttp-3.8.3-cp37-cp37m-musllinux_1_1_s390x.whl", hash = "sha256:ad61a9639792fd790523ba072c0555cd6be5a0baf03a49a5dd8cfcf20d56df48"}, + {file = "aiohttp-3.8.3-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:978b046ca728073070e9abc074b6299ebf3501e8dee5e26efacb13cec2b2dea0"}, + {file = "aiohttp-3.8.3-cp37-cp37m-win32.whl", hash = "sha256:0d2c6d8c6872df4a6ec37d2ede71eff62395b9e337b4e18efd2177de883a5033"}, + {file = "aiohttp-3.8.3-cp37-cp37m-win_amd64.whl", hash = "sha256:21d69797eb951f155026651f7e9362877334508d39c2fc37bd04ff55b2007091"}, + {file = "aiohttp-3.8.3-cp38-cp38-macosx_10_9_universal2.whl", hash = "sha256:2ca9af5f8f5812d475c5259393f52d712f6d5f0d7fdad9acdb1107dd9e3cb7eb"}, + {file = "aiohttp-3.8.3-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:1d90043c1882067f1bd26196d5d2db9aa6d268def3293ed5fb317e13c9413ea4"}, + {file = "aiohttp-3.8.3-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:d737fc67b9a970f3234754974531dc9afeea11c70791dcb7db53b0cf81b79784"}, + {file = "aiohttp-3.8.3-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:ebf909ea0a3fc9596e40d55d8000702a85e27fd578ff41a5500f68f20fd32e6c"}, + {file = "aiohttp-3.8.3-cp38-cp38-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:5835f258ca9f7c455493a57ee707b76d2d9634d84d5d7f62e77be984ea80b849"}, + {file = "aiohttp-3.8.3-cp38-cp38-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:da37dcfbf4b7f45d80ee386a5f81122501ec75672f475da34784196690762f4b"}, + {file = "aiohttp-3.8.3-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:87f44875f2804bc0511a69ce44a9595d5944837a62caecc8490bbdb0e18b1342"}, + {file = "aiohttp-3.8.3-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:527b3b87b24844ea7865284aabfab08eb0faf599b385b03c2aa91fc6edd6e4b6"}, + {file = "aiohttp-3.8.3-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:d5ba88df9aa5e2f806650fcbeedbe4f6e8736e92fc0e73b0400538fd25a4dd96"}, + {file = "aiohttp-3.8.3-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:e7b8813be97cab8cb52b1375f41f8e6804f6507fe4660152e8ca5c48f0436017"}, + {file = "aiohttp-3.8.3-cp38-cp38-musllinux_1_1_ppc64le.whl", hash = "sha256:2dea10edfa1a54098703cb7acaa665c07b4e7568472a47f4e64e6319d3821ccf"}, + {file = "aiohttp-3.8.3-cp38-cp38-musllinux_1_1_s390x.whl", hash = "sha256:713d22cd9643ba9025d33c4af43943c7a1eb8547729228de18d3e02e278472b6"}, + {file = "aiohttp-3.8.3-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:2d252771fc85e0cf8da0b823157962d70639e63cb9b578b1dec9868dd1f4f937"}, + {file = "aiohttp-3.8.3-cp38-cp38-win32.whl", hash = "sha256:66bd5f950344fb2b3dbdd421aaa4e84f4411a1a13fca3aeb2bcbe667f80c9f76"}, + {file = "aiohttp-3.8.3-cp38-cp38-win_amd64.whl", hash = "sha256:84b14f36e85295fe69c6b9789b51a0903b774046d5f7df538176516c3e422446"}, + {file = "aiohttp-3.8.3-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:16c121ba0b1ec2b44b73e3a8a171c4f999b33929cd2397124a8c7fcfc8cd9e06"}, + {file = "aiohttp-3.8.3-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:8d6aaa4e7155afaf994d7924eb290abbe81a6905b303d8cb61310a2aba1c68ba"}, + {file = "aiohttp-3.8.3-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:43046a319664a04b146f81b40e1545d4c8ac7b7dd04c47e40bf09f65f2437346"}, + {file = "aiohttp-3.8.3-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:599418aaaf88a6d02a8c515e656f6faf3d10618d3dd95866eb4436520096c84b"}, + {file = "aiohttp-3.8.3-cp39-cp39-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:92a2964319d359f494f16011e23434f6f8ef0434acd3cf154a6b7bec511e2fb7"}, + {file = "aiohttp-3.8.3-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:73a4131962e6d91109bca6536416aa067cf6c4efb871975df734f8d2fd821b37"}, + {file = "aiohttp-3.8.3-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:598adde339d2cf7d67beaccda3f2ce7c57b3b412702f29c946708f69cf8222aa"}, + {file = "aiohttp-3.8.3-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:75880ed07be39beff1881d81e4a907cafb802f306efd6d2d15f2b3c69935f6fb"}, + {file = "aiohttp-3.8.3-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:a0239da9fbafd9ff82fd67c16704a7d1bccf0d107a300e790587ad05547681c8"}, + {file = "aiohttp-3.8.3-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:4e3a23ec214e95c9fe85a58470b660efe6534b83e6cbe38b3ed52b053d7cb6ad"}, + {file = "aiohttp-3.8.3-cp39-cp39-musllinux_1_1_ppc64le.whl", hash = "sha256:47841407cc89a4b80b0c52276f3cc8138bbbfba4b179ee3acbd7d77ae33f7ac4"}, + {file = "aiohttp-3.8.3-cp39-cp39-musllinux_1_1_s390x.whl", hash = "sha256:54d107c89a3ebcd13228278d68f1436d3f33f2dd2af5415e3feaeb1156e1a62c"}, + {file = "aiohttp-3.8.3-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:c37c5cce780349d4d51739ae682dec63573847a2a8dcb44381b174c3d9c8d403"}, + {file = "aiohttp-3.8.3-cp39-cp39-win32.whl", hash = "sha256:f178d2aadf0166be4df834c4953da2d7eef24719e8aec9a65289483eeea9d618"}, + {file = "aiohttp-3.8.3-cp39-cp39-win_amd64.whl", hash = "sha256:88e5be56c231981428f4f506c68b6a46fa25c4123a2e86d156c58a8369d31ab7"}, + {file = "aiohttp-3.8.3.tar.gz", hash = "sha256:3828fb41b7203176b82fe5d699e0d845435f2374750a44b480ea6b930f6be269"}, +] +aioitertools = [ + {file = "aioitertools-0.11.0-py3-none-any.whl", hash = "sha256:04b95e3dab25b449def24d7df809411c10e62aab0cbe31a50ca4e68748c43394"}, + {file = "aioitertools-0.11.0.tar.gz", hash = "sha256:42c68b8dd3a69c2bf7f2233bf7df4bb58b557bca5252ac02ed5187bbc67d6831"}, +] +aiosignal = [ + {file = "aiosignal-1.3.1-py3-none-any.whl", hash = "sha256:f8376fb07dd1e86a584e4fcdec80b36b7f81aac666ebc724e2c090300dd83b17"}, + {file = "aiosignal-1.3.1.tar.gz", hash = "sha256:54cd96e15e1649b75d6c87526a6ff0b6c1b0dd3459f43d9ca11d48c339b68cfc"}, +] +async-timeout = [] +attrs = [ + {file = "attrs-22.2.0-py3-none-any.whl", hash = "sha256:29e95c7f6778868dbd49170f98f8818f78f3dc5e0e37c0b1f474e3561b240836"}, + {file = "attrs-22.2.0.tar.gz", hash = "sha256:c9227bfc2f01993c03f68db37d1d15c9690188323c067c641f1a35ca58185f99"}, +] +azure-core = [ + {file = "azure-core-1.26.2.zip", hash = "sha256:986bfd8687889782d79481d4c5d0af04ab4a18ca2f210364804a88e4eaa1586a"}, + {file = "azure_core-1.26.2-py3-none-any.whl", hash = "sha256:df306e6e4abc145610ca6744aef943129a6fd7a11977e56731f69ac0e00724f9"}, +] +azure-datalake-store = [ + {file = "azure-datalake-store-0.0.52.tar.gz", hash = "sha256:4198ddb32614d16d4502b43d5c9739f81432b7e0e4d75d30e05149fe6007fea2"}, + {file = "azure_datalake_store-0.0.52-py2.py3-none-any.whl", hash = "sha256:aaed72b9c856824aeab554f4dbe0ef2c6d0ff36700bdd8b93d8298793117c48e"}, +] +azure-identity = [ + {file = "azure-identity-1.12.0.zip", hash = "sha256:7f9b1ae7d97ea7af3f38dd09305e19ab81a1e16ab66ea186b6579d85c1ca2347"}, + {file = "azure_identity-1.12.0-py3-none-any.whl", hash = "sha256:2a58ce4a209a013e37eaccfd5937570ab99e9118b3e1acf875eed3a85d541b92"}, +] +azure-storage-blob = [ + {file = "azure-storage-blob-12.14.1.zip", hash = "sha256:860d4d82985a4bfc7d3271e71275af330f54f330a754355435a7ba749ccde997"}, + {file = "azure_storage_blob-12.14.1-py3-none-any.whl", hash = "sha256:52b84658e8df7853a3cf1c563814655b5028b979b2a87905b92aa6bb30be240e"}, +] +boto3 = [ + {file = "boto3-1.24.59-py3-none-any.whl", hash = "sha256:34ab44146a2c4e7f4e72737f4b27e6eb5e0a7855c2f4599e3d9199b6a0a2d575"}, + {file = "boto3-1.24.59.tar.gz", hash = "sha256:a50b4323f9579cfe22fcf5531fbd40b567d4d74c1adce06aeb5c95fce2a6fb40"}, +] +botocore = [ + {file = "botocore-1.27.59-py3-none-any.whl", hash = "sha256:69d756791fc024bda54f6c53f71ae34e695ee41bbbc1743d9179c4837a4929da"}, + {file = "botocore-1.27.59.tar.gz", hash = "sha256:eda4aed6ee719a745d1288eaf1beb12f6f6448ad1fa12f159405db14ba9c92cf"}, +] +build = [ + {file = "build-0.10.0-py3-none-any.whl", hash = "sha256:af266720050a66c893a6096a2f410989eeac74ff9a68ba194b3f6473e8e26171"}, + {file = "build-0.10.0.tar.gz", hash = "sha256:d5b71264afdb5951d6704482aac78de887c80691c52b88a9ad195983ca2c9269"}, +] +certifi = [ + {file = "certifi-2022.12.7-py3-none-any.whl", hash = "sha256:4ad3232f5e926d6718ec31cfc1fcadfde020920e278684144551c91769c7bc18"}, + {file = "certifi-2022.12.7.tar.gz", hash = "sha256:35824b4c3a97115964b408844d64aa14db1cc518f6562e8d7261699d1350a9e3"}, +] +cffi = [ + {file = "cffi-1.15.1-cp27-cp27m-macosx_10_9_x86_64.whl", hash = "sha256:a66d3508133af6e8548451b25058d5812812ec3798c886bf38ed24a98216fab2"}, + {file = "cffi-1.15.1-cp27-cp27m-manylinux1_i686.whl", hash = "sha256:470c103ae716238bbe698d67ad020e1db9d9dba34fa5a899b5e21577e6d52ed2"}, + {file = "cffi-1.15.1-cp27-cp27m-manylinux1_x86_64.whl", hash = "sha256:9ad5db27f9cabae298d151c85cf2bad1d359a1b9c686a275df03385758e2f914"}, + {file = "cffi-1.15.1-cp27-cp27m-win32.whl", hash = "sha256:b3bbeb01c2b273cca1e1e0c5df57f12dce9a4dd331b4fa1635b8bec26350bde3"}, + {file = "cffi-1.15.1-cp27-cp27m-win_amd64.whl", hash = "sha256:e00b098126fd45523dd056d2efba6c5a63b71ffe9f2bbe1a4fe1716e1d0c331e"}, + {file = "cffi-1.15.1-cp27-cp27mu-manylinux1_i686.whl", hash = "sha256:d61f4695e6c866a23a21acab0509af1cdfd2c013cf256bbf5b6b5e2695827162"}, + {file = "cffi-1.15.1-cp27-cp27mu-manylinux1_x86_64.whl", hash = "sha256:ed9cb427ba5504c1dc15ede7d516b84757c3e3d7868ccc85121d9310d27eed0b"}, + {file = "cffi-1.15.1-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:39d39875251ca8f612b6f33e6b1195af86d1b3e60086068be9cc053aa4376e21"}, + {file = "cffi-1.15.1-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:285d29981935eb726a4399badae8f0ffdff4f5050eaa6d0cfc3f64b857b77185"}, + {file = "cffi-1.15.1-cp310-cp310-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:3eb6971dcff08619f8d91607cfc726518b6fa2a9eba42856be181c6d0d9515fd"}, + {file = "cffi-1.15.1-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:21157295583fe8943475029ed5abdcf71eb3911894724e360acff1d61c1d54bc"}, + {file = "cffi-1.15.1-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:5635bd9cb9731e6d4a1132a498dd34f764034a8ce60cef4f5319c0541159392f"}, + {file = "cffi-1.15.1-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:2012c72d854c2d03e45d06ae57f40d78e5770d252f195b93f581acf3ba44496e"}, + {file = "cffi-1.15.1-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:dd86c085fae2efd48ac91dd7ccffcfc0571387fe1193d33b6394db7ef31fe2a4"}, + {file = "cffi-1.15.1-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:fa6693661a4c91757f4412306191b6dc88c1703f780c8234035eac011922bc01"}, + {file = "cffi-1.15.1-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:59c0b02d0a6c384d453fece7566d1c7e6b7bae4fc5874ef2ef46d56776d61c9e"}, + {file = "cffi-1.15.1-cp310-cp310-win32.whl", hash = "sha256:cba9d6b9a7d64d4bd46167096fc9d2f835e25d7e4c121fb2ddfc6528fb0413b2"}, + {file = "cffi-1.15.1-cp310-cp310-win_amd64.whl", hash = "sha256:ce4bcc037df4fc5e3d184794f27bdaab018943698f4ca31630bc7f84a7b69c6d"}, + {file = "cffi-1.15.1-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:3d08afd128ddaa624a48cf2b859afef385b720bb4b43df214f85616922e6a5ac"}, + {file = "cffi-1.15.1-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:3799aecf2e17cf585d977b780ce79ff0dc9b78d799fc694221ce814c2c19db83"}, + {file = "cffi-1.15.1-cp311-cp311-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:a591fe9e525846e4d154205572a029f653ada1a78b93697f3b5a8f1f2bc055b9"}, + {file = "cffi-1.15.1-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:3548db281cd7d2561c9ad9984681c95f7b0e38881201e157833a2342c30d5e8c"}, + {file = "cffi-1.15.1-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:91fc98adde3d7881af9b59ed0294046f3806221863722ba7d8d120c575314325"}, + {file = "cffi-1.15.1-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:94411f22c3985acaec6f83c6df553f2dbe17b698cc7f8ae751ff2237d96b9e3c"}, + {file = "cffi-1.15.1-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:03425bdae262c76aad70202debd780501fabeaca237cdfddc008987c0e0f59ef"}, + {file = "cffi-1.15.1-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:cc4d65aeeaa04136a12677d3dd0b1c0c94dc43abac5860ab33cceb42b801c1e8"}, + {file = "cffi-1.15.1-cp311-cp311-win32.whl", hash = "sha256:a0f100c8912c114ff53e1202d0078b425bee3649ae34d7b070e9697f93c5d52d"}, + {file = "cffi-1.15.1-cp311-cp311-win_amd64.whl", hash = "sha256:04ed324bda3cda42b9b695d51bb7d54b680b9719cfab04227cdd1e04e5de3104"}, + {file = "cffi-1.15.1-cp36-cp36m-macosx_10_9_x86_64.whl", hash = "sha256:50a74364d85fd319352182ef59c5c790484a336f6db772c1a9231f1c3ed0cbd7"}, + {file = "cffi-1.15.1-cp36-cp36m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:e263d77ee3dd201c3a142934a086a4450861778baaeeb45db4591ef65550b0a6"}, + {file = "cffi-1.15.1-cp36-cp36m-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:cec7d9412a9102bdc577382c3929b337320c4c4c4849f2c5cdd14d7368c5562d"}, + {file = "cffi-1.15.1-cp36-cp36m-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:4289fc34b2f5316fbb762d75362931e351941fa95fa18789191b33fc4cf9504a"}, + {file = "cffi-1.15.1-cp36-cp36m-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:173379135477dc8cac4bc58f45db08ab45d228b3363adb7af79436135d028405"}, + {file = "cffi-1.15.1-cp36-cp36m-manylinux_2_5_x86_64.manylinux1_x86_64.whl", hash = "sha256:6975a3fac6bc83c4a65c9f9fcab9e47019a11d3d2cf7f3c0d03431bf145a941e"}, + {file = "cffi-1.15.1-cp36-cp36m-win32.whl", hash = "sha256:2470043b93ff09bf8fb1d46d1cb756ce6132c54826661a32d4e4d132e1977adf"}, + {file = "cffi-1.15.1-cp36-cp36m-win_amd64.whl", hash = "sha256:30d78fbc8ebf9c92c9b7823ee18eb92f2e6ef79b45ac84db507f52fbe3ec4497"}, + {file = "cffi-1.15.1-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:198caafb44239b60e252492445da556afafc7d1e3ab7a1fb3f0584ef6d742375"}, + {file = "cffi-1.15.1-cp37-cp37m-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:5ef34d190326c3b1f822a5b7a45f6c4535e2f47ed06fec77d3d799c450b2651e"}, + {file = "cffi-1.15.1-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:8102eaf27e1e448db915d08afa8b41d6c7ca7a04b7d73af6514df10a3e74bd82"}, + {file = "cffi-1.15.1-cp37-cp37m-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:5df2768244d19ab7f60546d0c7c63ce1581f7af8b5de3eb3004b9b6fc8a9f84b"}, + {file = "cffi-1.15.1-cp37-cp37m-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:a8c4917bd7ad33e8eb21e9a5bbba979b49d9a97acb3a803092cbc1133e20343c"}, + {file = "cffi-1.15.1-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:0e2642fe3142e4cc4af0799748233ad6da94c62a8bec3a6648bf8ee68b1c7426"}, + {file = "cffi-1.15.1-cp37-cp37m-win32.whl", hash = "sha256:e229a521186c75c8ad9490854fd8bbdd9a0c9aa3a524326b55be83b54d4e0ad9"}, + {file = "cffi-1.15.1-cp37-cp37m-win_amd64.whl", hash = "sha256:a0b71b1b8fbf2b96e41c4d990244165e2c9be83d54962a9a1d118fd8657d2045"}, + {file = "cffi-1.15.1-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:320dab6e7cb2eacdf0e658569d2575c4dad258c0fcc794f46215e1e39f90f2c3"}, + {file = "cffi-1.15.1-cp38-cp38-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:1e74c6b51a9ed6589199c787bf5f9875612ca4a8a0785fb2d4a84429badaf22a"}, + {file = "cffi-1.15.1-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a5c84c68147988265e60416b57fc83425a78058853509c1b0629c180094904a5"}, + {file = "cffi-1.15.1-cp38-cp38-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:3b926aa83d1edb5aa5b427b4053dc420ec295a08e40911296b9eb1b6170f6cca"}, + {file = "cffi-1.15.1-cp38-cp38-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:87c450779d0914f2861b8526e035c5e6da0a3199d8f1add1a665e1cbc6fc6d02"}, + {file = "cffi-1.15.1-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:4f2c9f67e9821cad2e5f480bc8d83b8742896f1242dba247911072d4fa94c192"}, + {file = "cffi-1.15.1-cp38-cp38-win32.whl", hash = "sha256:8b7ee99e510d7b66cdb6c593f21c043c248537a32e0bedf02e01e9553a172314"}, + {file = "cffi-1.15.1-cp38-cp38-win_amd64.whl", hash = "sha256:00a9ed42e88df81ffae7a8ab6d9356b371399b91dbdf0c3cb1e84c03a13aceb5"}, + {file = "cffi-1.15.1-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:54a2db7b78338edd780e7ef7f9f6c442500fb0d41a5a4ea24fff1c929d5af585"}, + {file = "cffi-1.15.1-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:fcd131dd944808b5bdb38e6f5b53013c5aa4f334c5cad0c72742f6eba4b73db0"}, + {file = "cffi-1.15.1-cp39-cp39-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:7473e861101c9e72452f9bf8acb984947aa1661a7704553a9f6e4baa5ba64415"}, + {file = "cffi-1.15.1-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:6c9a799e985904922a4d207a94eae35c78ebae90e128f0c4e521ce339396be9d"}, + {file = "cffi-1.15.1-cp39-cp39-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:3bcde07039e586f91b45c88f8583ea7cf7a0770df3a1649627bf598332cb6984"}, + {file = "cffi-1.15.1-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:33ab79603146aace82c2427da5ca6e58f2b3f2fb5da893ceac0c42218a40be35"}, + {file = "cffi-1.15.1-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:5d598b938678ebf3c67377cdd45e09d431369c3b1a5b331058c338e201f12b27"}, + {file = "cffi-1.15.1-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:db0fbb9c62743ce59a9ff687eb5f4afbe77e5e8403d6697f7446e5f609976f76"}, + {file = "cffi-1.15.1-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:98d85c6a2bef81588d9227dde12db8a7f47f639f4a17c9ae08e773aa9c697bf3"}, + {file = "cffi-1.15.1-cp39-cp39-win32.whl", hash = "sha256:40f4774f5a9d4f5e344f31a32b5096977b5d48560c5592e2f3d2c4374bd543ee"}, + {file = "cffi-1.15.1-cp39-cp39-win_amd64.whl", hash = "sha256:70df4e3b545a17496c9b3f41f5115e69a4f2e77e94e1d2a8e1070bc0c38c8a3c"}, + {file = "cffi-1.15.1.tar.gz", hash = "sha256:d400bfb9a37b1351253cb402671cea7e89bdecc294e8016a707f6d1d8ac934f9"}, +] +cfgv = [ + {file = "cfgv-3.3.1-py2.py3-none-any.whl", hash = "sha256:c6a0883f3917a037485059700b9e75da2464e6c27051014ad85ba6aaa5884426"}, + {file = "cfgv-3.3.1.tar.gz", hash = "sha256:f5a830efb9ce7a445376bb66ec94c638a9787422f96264c98edc6bdeed8ab736"}, +] +charset-normalizer = [] +click = [] +colorama = [ + {file = "colorama-0.4.6-py2.py3-none-any.whl", hash = "sha256:4f1d9991f5acc0ca119f9d443620b77f9d6b33703e51011c16baf57afb285fc6"}, + {file = "colorama-0.4.6.tar.gz", hash = "sha256:08695f5cb7ed6e0531a20572697297273c47b8cae5a63ffc6d6ed5c201be6e44"}, +] +coverage = [ + {file = "coverage-7.0.5-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:2a7f23bbaeb2a87f90f607730b45564076d870f1fb07b9318d0c21f36871932b"}, + {file = "coverage-7.0.5-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:c18d47f314b950dbf24a41787ced1474e01ca816011925976d90a88b27c22b89"}, + {file = "coverage-7.0.5-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:ef14d75d86f104f03dea66c13188487151760ef25dd6b2dbd541885185f05f40"}, + {file = "coverage-7.0.5-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:66e50680e888840c0995f2ad766e726ce71ca682e3c5f4eee82272c7671d38a2"}, + {file = "coverage-7.0.5-cp310-cp310-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:a9fed35ca8c6e946e877893bbac022e8563b94404a605af1d1e6accc7eb73289"}, + {file = "coverage-7.0.5-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:d8d04e755934195bdc1db45ba9e040b8d20d046d04d6d77e71b3b34a8cc002d0"}, + {file = "coverage-7.0.5-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:7e109f1c9a3ece676597831874126555997c48f62bddbcace6ed17be3e372de8"}, + {file = "coverage-7.0.5-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:0a1890fca2962c4f1ad16551d660b46ea77291fba2cc21c024cd527b9d9c8809"}, + {file = "coverage-7.0.5-cp310-cp310-win32.whl", hash = "sha256:be9fcf32c010da0ba40bf4ee01889d6c737658f4ddff160bd7eb9cac8f094b21"}, + {file = "coverage-7.0.5-cp310-cp310-win_amd64.whl", hash = "sha256:cbfcba14a3225b055a28b3199c3d81cd0ab37d2353ffd7f6fd64844cebab31ad"}, + {file = "coverage-7.0.5-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:30b5fec1d34cc932c1bc04017b538ce16bf84e239378b8f75220478645d11fca"}, + {file = "coverage-7.0.5-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:1caed2367b32cc80a2b7f58a9f46658218a19c6cfe5bc234021966dc3daa01f0"}, + {file = "coverage-7.0.5-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:d254666d29540a72d17cc0175746cfb03d5123db33e67d1020e42dae611dc196"}, + {file = "coverage-7.0.5-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:19245c249aa711d954623d94f23cc94c0fd65865661f20b7781210cb97c471c0"}, + {file = "coverage-7.0.5-cp311-cp311-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:7b05ed4b35bf6ee790832f68932baf1f00caa32283d66cc4d455c9e9d115aafc"}, + {file = "coverage-7.0.5-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:29de916ba1099ba2aab76aca101580006adfac5646de9b7c010a0f13867cba45"}, + {file = "coverage-7.0.5-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:e057e74e53db78122a3979f908973e171909a58ac20df05c33998d52e6d35757"}, + {file = "coverage-7.0.5-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:411d4ff9d041be08fdfc02adf62e89c735b9468f6d8f6427f8a14b6bb0a85095"}, + {file = "coverage-7.0.5-cp311-cp311-win32.whl", hash = "sha256:52ab14b9e09ce052237dfe12d6892dd39b0401690856bcfe75d5baba4bfe2831"}, + {file = "coverage-7.0.5-cp311-cp311-win_amd64.whl", hash = "sha256:1f66862d3a41674ebd8d1a7b6f5387fe5ce353f8719040a986551a545d7d83ea"}, + {file = "coverage-7.0.5-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:b69522b168a6b64edf0c33ba53eac491c0a8f5cc94fa4337f9c6f4c8f2f5296c"}, + {file = "coverage-7.0.5-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:436e103950d05b7d7f55e39beeb4d5be298ca3e119e0589c0227e6d0b01ee8c7"}, + {file = "coverage-7.0.5-cp37-cp37m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:b8c56bec53d6e3154eaff6ea941226e7bd7cc0d99f9b3756c2520fc7a94e6d96"}, + {file = "coverage-7.0.5-cp37-cp37m-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:7a38362528a9115a4e276e65eeabf67dcfaf57698e17ae388599568a78dcb029"}, + {file = "coverage-7.0.5-cp37-cp37m-musllinux_1_1_aarch64.whl", hash = "sha256:f67472c09a0c7486e27f3275f617c964d25e35727af952869dd496b9b5b7f6a3"}, + {file = "coverage-7.0.5-cp37-cp37m-musllinux_1_1_i686.whl", hash = "sha256:220e3fa77d14c8a507b2d951e463b57a1f7810a6443a26f9b7591ef39047b1b2"}, + {file = "coverage-7.0.5-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:ecb0f73954892f98611e183f50acdc9e21a4653f294dfbe079da73c6378a6f47"}, + {file = "coverage-7.0.5-cp37-cp37m-win32.whl", hash = "sha256:d8f3e2e0a1d6777e58e834fd5a04657f66affa615dae61dd67c35d1568c38882"}, + {file = "coverage-7.0.5-cp37-cp37m-win_amd64.whl", hash = "sha256:9e662e6fc4f513b79da5d10a23edd2b87685815b337b1a30cd11307a6679148d"}, + {file = "coverage-7.0.5-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:790e4433962c9f454e213b21b0fd4b42310ade9c077e8edcb5113db0818450cb"}, + {file = "coverage-7.0.5-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:49640bda9bda35b057b0e65b7c43ba706fa2335c9a9896652aebe0fa399e80e6"}, + {file = "coverage-7.0.5-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:d66187792bfe56f8c18ba986a0e4ae44856b1c645336bd2c776e3386da91e1dd"}, + {file = "coverage-7.0.5-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:276f4cd0001cd83b00817c8db76730938b1ee40f4993b6a905f40a7278103b3a"}, + {file = "coverage-7.0.5-cp38-cp38-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:95304068686545aa368b35dfda1cdfbbdbe2f6fe43de4a2e9baa8ebd71be46e2"}, + {file = "coverage-7.0.5-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:17e01dd8666c445025c29684d4aabf5a90dc6ef1ab25328aa52bedaa95b65ad7"}, + {file = "coverage-7.0.5-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:ea76dbcad0b7b0deb265d8c36e0801abcddf6cc1395940a24e3595288b405ca0"}, + {file = "coverage-7.0.5-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:50a6adc2be8edd7ee67d1abc3cd20678987c7b9d79cd265de55941e3d0d56499"}, + {file = "coverage-7.0.5-cp38-cp38-win32.whl", hash = "sha256:e4ce984133b888cc3a46867c8b4372c7dee9cee300335e2925e197bcd45b9e16"}, + {file = "coverage-7.0.5-cp38-cp38-win_amd64.whl", hash = "sha256:4a950f83fd3f9bca23b77442f3a2b2ea4ac900944d8af9993743774c4fdc57af"}, + {file = "coverage-7.0.5-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:3c2155943896ac78b9b0fd910fb381186d0c345911f5333ee46ac44c8f0e43ab"}, + {file = "coverage-7.0.5-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:54f7e9705e14b2c9f6abdeb127c390f679f6dbe64ba732788d3015f7f76ef637"}, + {file = "coverage-7.0.5-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:0ee30375b409d9a7ea0f30c50645d436b6f5dfee254edffd27e45a980ad2c7f4"}, + {file = "coverage-7.0.5-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:b78729038abea6a5df0d2708dce21e82073463b2d79d10884d7d591e0f385ded"}, + {file = "coverage-7.0.5-cp39-cp39-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:13250b1f0bd023e0c9f11838bdeb60214dd5b6aaf8e8d2f110c7e232a1bff83b"}, + {file = "coverage-7.0.5-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:2c407b1950b2d2ffa091f4e225ca19a66a9bd81222f27c56bd12658fc5ca1209"}, + {file = "coverage-7.0.5-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:c76a3075e96b9c9ff00df8b5f7f560f5634dffd1658bafb79eb2682867e94f78"}, + {file = "coverage-7.0.5-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:f26648e1b3b03b6022b48a9b910d0ae209e2d51f50441db5dce5b530fad6d9b1"}, + {file = "coverage-7.0.5-cp39-cp39-win32.whl", hash = "sha256:ba3027deb7abf02859aca49c865ece538aee56dcb4871b4cced23ba4d5088904"}, + {file = "coverage-7.0.5-cp39-cp39-win_amd64.whl", hash = "sha256:949844af60ee96a376aac1ded2a27e134b8c8d35cc006a52903fc06c24a3296f"}, + {file = "coverage-7.0.5-pp37.pp38.pp39-none-any.whl", hash = "sha256:b9727ac4f5cf2cbf87880a63870b5b9730a8ae3a4a360241a0fdaa2f71240ff0"}, + {file = "coverage-7.0.5.tar.gz", hash = "sha256:051afcbd6d2ac39298d62d340f94dbb6a1f31de06dfaf6fcef7b759dd3860c45"}, +] +cryptography = [ + {file = "cryptography-39.0.0-cp36-abi3-macosx_10_12_universal2.whl", hash = "sha256:c52a1a6f81e738d07f43dab57831c29e57d21c81a942f4602fac7ee21b27f288"}, + {file = "cryptography-39.0.0-cp36-abi3-macosx_10_12_x86_64.whl", hash = "sha256:80ee674c08aaef194bc4627b7f2956e5ba7ef29c3cc3ca488cf15854838a8f72"}, + {file = "cryptography-39.0.0-cp36-abi3-manylinux_2_17_aarch64.manylinux2014_aarch64.manylinux_2_24_aarch64.whl", hash = "sha256:887cbc1ea60786e534b00ba8b04d1095f4272d380ebd5f7a7eb4cc274710fad9"}, + {file = "cryptography-39.0.0-cp36-abi3-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:6f97109336df5c178ee7c9c711b264c502b905c2d2a29ace99ed761533a3460f"}, + {file = "cryptography-39.0.0-cp36-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:1a6915075c6d3a5e1215eab5d99bcec0da26036ff2102a1038401d6ef5bef25b"}, + {file = "cryptography-39.0.0-cp36-abi3-manylinux_2_24_x86_64.whl", hash = "sha256:76c24dd4fd196a80f9f2f5405a778a8ca132f16b10af113474005635fe7e066c"}, + {file = "cryptography-39.0.0-cp36-abi3-manylinux_2_28_aarch64.whl", hash = "sha256:bae6c7f4a36a25291b619ad064a30a07110a805d08dc89984f4f441f6c1f3f96"}, + {file = "cryptography-39.0.0-cp36-abi3-manylinux_2_28_x86_64.whl", hash = "sha256:875aea1039d78557c7c6b4db2fe0e9d2413439f4676310a5f269dd342ca7a717"}, + {file = "cryptography-39.0.0-cp36-abi3-musllinux_1_1_aarch64.whl", hash = "sha256:f6c0db08d81ead9576c4d94bbb27aed8d7a430fa27890f39084c2d0e2ec6b0df"}, + {file = "cryptography-39.0.0-cp36-abi3-musllinux_1_1_x86_64.whl", hash = "sha256:f3ed2d864a2fa1666e749fe52fb8e23d8e06b8012e8bd8147c73797c506e86f1"}, + {file = "cryptography-39.0.0-cp36-abi3-win32.whl", hash = "sha256:f671c1bb0d6088e94d61d80c606d65baacc0d374e67bf895148883461cd848de"}, + {file = "cryptography-39.0.0-cp36-abi3-win_amd64.whl", hash = "sha256:e324de6972b151f99dc078defe8fb1b0a82c6498e37bff335f5bc6b1e3ab5a1e"}, + {file = "cryptography-39.0.0-pp38-pypy38_pp73-macosx_10_12_x86_64.whl", hash = "sha256:754978da4d0457e7ca176f58c57b1f9de6556591c19b25b8bcce3c77d314f5eb"}, + {file = "cryptography-39.0.0-pp38-pypy38_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:1ee1fd0de9851ff32dbbb9362a4d833b579b4a6cc96883e8e6d2ff2a6bc7104f"}, + {file = "cryptography-39.0.0-pp38-pypy38_pp73-manylinux_2_24_x86_64.whl", hash = "sha256:fec8b932f51ae245121c4671b4bbc030880f363354b2f0e0bd1366017d891458"}, + {file = "cryptography-39.0.0-pp38-pypy38_pp73-manylinux_2_28_x86_64.whl", hash = "sha256:407cec680e811b4fc829de966f88a7c62a596faa250fc1a4b520a0355b9bc190"}, + {file = "cryptography-39.0.0-pp38-pypy38_pp73-win_amd64.whl", hash = "sha256:7dacfdeee048814563eaaec7c4743c8aea529fe3dd53127313a792f0dadc1773"}, + {file = "cryptography-39.0.0-pp39-pypy39_pp73-macosx_10_12_x86_64.whl", hash = "sha256:ad04f413436b0781f20c52a661660f1e23bcd89a0e9bb1d6d20822d048cf2856"}, + {file = "cryptography-39.0.0-pp39-pypy39_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:50386acb40fbabbceeb2986332f0287f50f29ccf1497bae31cf5c3e7b4f4b34f"}, + {file = "cryptography-39.0.0-pp39-pypy39_pp73-manylinux_2_24_x86_64.whl", hash = "sha256:e5d71c5d5bd5b5c3eebcf7c5c2bb332d62ec68921a8c593bea8c394911a005ce"}, + {file = "cryptography-39.0.0-pp39-pypy39_pp73-manylinux_2_28_x86_64.whl", hash = "sha256:844ad4d7c3850081dffba91cdd91950038ee4ac525c575509a42d3fc806b83c8"}, + {file = "cryptography-39.0.0-pp39-pypy39_pp73-win_amd64.whl", hash = "sha256:e0a05aee6a82d944f9b4edd6a001178787d1546ec7c6223ee9a848a7ade92e39"}, + {file = "cryptography-39.0.0.tar.gz", hash = "sha256:f964c7dcf7802d133e8dbd1565914fa0194f9d683d82411989889ecd701e8adf"}, +] +distlib = [ + {file = "distlib-0.3.6-py2.py3-none-any.whl", hash = "sha256:f35c4b692542ca110de7ef0bea44d73981caeb34ca0b9b6b2e6d7790dda8f80e"}, + {file = "distlib-0.3.6.tar.gz", hash = "sha256:14bad2d9b04d3a36127ac97f30b12a19268f211063d8f8ee4f47108896e11b46"}, +] +docutils = [ + {file = "docutils-0.19-py3-none-any.whl", hash = "sha256:5e1de4d849fee02c63b040a4a3fd567f4ab104defd8a5511fbbc24a8a017efbc"}, + {file = "docutils-0.19.tar.gz", hash = "sha256:33995a6753c30b7f577febfc2c50411fec6aac7f7ffeb7c4cfe5991072dcf9e6"}, +] +duckdb = [ + {file = "duckdb-0.6.1-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:e566514f9327f89264e98ac14ee7a84fbd9857328028258422c3e8375ee19d25"}, + {file = "duckdb-0.6.1-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:b31c2883de5b19591a2852165e6b3f9821f77af649835f27bc146b26e4aa30cb"}, + {file = "duckdb-0.6.1-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:998165b2fb1f1d2b0ad742096015ea70878f7d40304643c7424c3ed3ddf07bfc"}, + {file = "duckdb-0.6.1-cp310-cp310-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:3941b3a1e8a1cdb7b90ab3917b87af816e71f9692e5ada7f19b6b60969f731e5"}, + {file = "duckdb-0.6.1-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:143611bd1b7c13343f087d4d423a7a8a4f33a114c5326171e867febf3f0fcfe1"}, + {file = "duckdb-0.6.1-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:125ba45e8b08f28858f918ec9cbd3a19975e5d8d9e8275ef4ad924028a616e14"}, + {file = "duckdb-0.6.1-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:e609a65b31c92f2f7166831f74b56f5ed54b33d8c2c4b4c3974c26fdc50464c5"}, + {file = "duckdb-0.6.1-cp310-cp310-win32.whl", hash = "sha256:b39045074fb9a3f068496475a5d627ad4fa572fa3b4980e3b479c11d0b706f2d"}, + {file = "duckdb-0.6.1-cp310-cp310-win_amd64.whl", hash = "sha256:16fa96ffaa3d842a9355a633fb8bc092d119be08d4bc02013946d8594417bc14"}, + {file = "duckdb-0.6.1-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:b4bbe2f6c1b109c626f9318eee80934ad2a5b81a51409c6b5083c6c5f9bdb125"}, + {file = "duckdb-0.6.1-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:cfea36b58928ce778d17280d4fb3bf0a2d7cff407667baedd69c5b41463ac0fd"}, + {file = "duckdb-0.6.1-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:0b64eb53d0d0695814bf1b65c0f91ab7ed66b515f89c88038f65ad5e0762571c"}, + {file = "duckdb-0.6.1-cp311-cp311-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:35b01bc724e1933293f4c34f410d2833bfbb56d5743b515d805bbfed0651476e"}, + {file = "duckdb-0.6.1-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:fec2c2466654ce786843bda2bfba71e0e4719106b41d36b17ceb1901e130aa71"}, + {file = "duckdb-0.6.1-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:82cd30f5cf368658ef879b1c60276bc8650cf67cfe3dc3e3009438ba39251333"}, + {file = "duckdb-0.6.1-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:a782bbfb7f5e97d4a9c834c9e78f023fb8b3f6687c22ca99841e6ed944b724da"}, + {file = "duckdb-0.6.1-cp311-cp311-win32.whl", hash = "sha256:e3702d4a9ade54c6403f6615a98bbec2020a76a60f5db7fcf085df1bd270e66e"}, + {file = "duckdb-0.6.1-cp311-cp311-win_amd64.whl", hash = "sha256:93b074f473d68c944b0eeb2edcafd91ad11da8432b484836efaaab4e26351d48"}, + {file = "duckdb-0.6.1-cp36-cp36m-macosx_10_9_x86_64.whl", hash = "sha256:adae183924d6d479202c39072e37d440b511326e84525bcb7432bca85f86caba"}, + {file = "duckdb-0.6.1-cp36-cp36m-win32.whl", hash = "sha256:546a1cd17595bd1dd009daf6f36705aa6f95337154360ce44932157d353dcd80"}, + {file = "duckdb-0.6.1-cp36-cp36m-win_amd64.whl", hash = "sha256:87b0d00eb9d1a7ebe437276203e0cdc93b4a2154ba9688c65e8d2a8735839ec6"}, + {file = "duckdb-0.6.1-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:8442e074de6e1969c3d2b24363a5a6d7f866d5ac3f4e358e357495b389eff6c1"}, + {file = "duckdb-0.6.1-cp37-cp37m-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:0a6bf2ae7bec803352dade14561cb0b461b2422e70f75d9f09b36ba2dad2613b"}, + {file = "duckdb-0.6.1-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:5054792f22733f89d9cbbced2bafd8772d72d0fe77f159310221cefcf981c680"}, + {file = "duckdb-0.6.1-cp37-cp37m-musllinux_1_1_i686.whl", hash = "sha256:21cc503dffc2c68bb825e4eb3098e82f40e910b3d09e1b3b7f090d39ad53fbea"}, + {file = "duckdb-0.6.1-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:54b3da77ad893e99c073087ff7f75a8c98154ac5139d317149f12b74367211db"}, + {file = "duckdb-0.6.1-cp37-cp37m-win32.whl", hash = "sha256:f1d709aa6a26172a3eab804b57763d5cdc1a4b785ac1fc2b09568578e52032ee"}, + {file = "duckdb-0.6.1-cp37-cp37m-win_amd64.whl", hash = "sha256:f4edcaa471d791393e37f63e3c7c728fa6324e3ac7e768b9dc2ea49065cd37cc"}, + {file = "duckdb-0.6.1-cp38-cp38-macosx_10_9_universal2.whl", hash = "sha256:d218c2dd3bda51fb79e622b7b2266183ac9493834b55010aa01273fa5b7a7105"}, + {file = "duckdb-0.6.1-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:0c7155cb93ab432eca44b651256c359281d26d927ff43badaf1d2276dd770832"}, + {file = "duckdb-0.6.1-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:0925778200090d3d5d8b6bb42b4d05d24db1e8912484ba3b7e7b7f8569f17dcb"}, + {file = "duckdb-0.6.1-cp38-cp38-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:8b544dd04bb851d08bc68b317a7683cec6091547ae75555d075f8c8a7edb626e"}, + {file = "duckdb-0.6.1-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:f2c37d5a0391cf3a3a66e63215968ffb78e6b84f659529fa4bd10478f6203071"}, + {file = "duckdb-0.6.1-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:ce376966260eb5c351fcc6af627a979dbbcae3efeb2e70f85b23aa45a21e289d"}, + {file = "duckdb-0.6.1-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:73c974b09dd08dff5e8bdedba11c7d0aa0fc46ca93954ee7d19e1e18c9883ac1"}, + {file = "duckdb-0.6.1-cp38-cp38-win32.whl", hash = "sha256:bfe39ed3a03e8b1ed764f58f513b37b24afe110d245803a41655d16d391ad9f1"}, + {file = "duckdb-0.6.1-cp38-cp38-win_amd64.whl", hash = "sha256:afa97d982dbe6b125631a17e222142e79bee88f7a13fc4cee92d09285e31ec83"}, + {file = "duckdb-0.6.1-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:c35ff4b1117096ef72d101524df0079da36c3735d52fcf1d907ccffa63bd6202"}, + {file = "duckdb-0.6.1-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:5c54910fbb6de0f21d562e18a5c91540c19876db61b862fc9ffc8e31be8b3f03"}, + {file = "duckdb-0.6.1-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:99a7172563a3ae67d867572ce27cf3962f58e76f491cb7f602f08c2af39213b3"}, + {file = "duckdb-0.6.1-cp39-cp39-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:7363ffe857d00216b659116647fbf1e925cb3895699015d4a4e50b746de13041"}, + {file = "duckdb-0.6.1-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:06c1cef25f896b2284ba048108f645c72fab5c54aa5a6f62f95663f44ff8a79b"}, + {file = "duckdb-0.6.1-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:e92dd6aad7e8c29d002947376b6f5ce28cae29eb3b6b58a64a46cdbfc5cb7943"}, + {file = "duckdb-0.6.1-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:4b280b2d8a01ecd4fe2feab041df70233c534fafbe33a38565b52c1e017529c7"}, + {file = "duckdb-0.6.1-cp39-cp39-win32.whl", hash = "sha256:d9212d76e90b8469743924a4d22bef845be310d0d193d54ae17d9ef1f753cfa7"}, + {file = "duckdb-0.6.1-cp39-cp39-win_amd64.whl", hash = "sha256:00b7be8f67ec1a8edaa8844f521267baa1a795f4c482bfad56c72c26e1862ab2"}, + {file = "duckdb-0.6.1.tar.gz", hash = "sha256:6d26e9f1afcb924a6057785e506810d48332d4764ddc4a5b414d0f2bf0cacfb4"}, +] +exceptiongroup = [ + {file = "exceptiongroup-1.1.0-py3-none-any.whl", hash = "sha256:327cbda3da756e2de031a3107b81ab7b3770a602c4d16ca618298c526f4bec1e"}, + {file = "exceptiongroup-1.1.0.tar.gz", hash = "sha256:bcb67d800a4497e1b404c2dd44fca47d3b7a5e5433dbab67f96c1a685cdfdf23"}, +] +fastavro = [ + {file = "fastavro-1.7.0-cp310-cp310-macosx_10_14_x86_64.whl", hash = "sha256:ab3387a06e272980fa034f5c62f7063977b77df6416d3d30a4d3b49cc8827566"}, + {file = "fastavro-1.7.0-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:216132bc54da19e97e1531dd69c86282408d4c797749d83b01b3a00862a180de"}, + {file = "fastavro-1.7.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:c20cf6cd8098bb93c2cffd53d03ccea1dcf9ec594a5c83963acf29a2882f8693"}, + {file = "fastavro-1.7.0-cp310-cp310-win_amd64.whl", hash = "sha256:54b60e79506a456bcfc940560fa2c73a7a8e3ddc58a1ae4d94fdd99f6b02aef0"}, + {file = "fastavro-1.7.0-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:4da6d83abd04a804310667f8d1fc23d44e9ed91ed9a9bc9c1fcd906e0c403b12"}, + {file = "fastavro-1.7.0-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:7c664302f94064adeb93403c61c74e07b9710526403eba3b59169f99bb99c55c"}, + {file = "fastavro-1.7.0-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:7aab162f02e64bf82d0282430a3c6ec7a36982b1c5d479e7dcc278e6d62a84b8"}, + {file = "fastavro-1.7.0-cp311-cp311-win_amd64.whl", hash = "sha256:63d0d2a2bb3e85d006c834633be51b105a50b0dc7cc8423b06f30601b532adf4"}, + {file = "fastavro-1.7.0-cp37-cp37m-macosx_10_15_x86_64.whl", hash = "sha256:a07a3245049529d6d9028d664361cc4990e74d035d2303875295e2f7b97eba09"}, + {file = "fastavro-1.7.0-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:7df57d31cb58770a9066790250e9f4ec91242c69e1dc62ea732a6fb2406a8f96"}, + {file = "fastavro-1.7.0-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:b905634b5763ca08c9f7b51486e2c3ae7907f5d9bc48208c69b16ccbe8455e90"}, + {file = "fastavro-1.7.0-cp37-cp37m-win_amd64.whl", hash = "sha256:749206f1cec3d7429546e49db5708f4571497d35181b6b334c4844133f230515"}, + {file = "fastavro-1.7.0-cp38-cp38-macosx_10_15_x86_64.whl", hash = "sha256:a0ebd5c1269085179de4b3f072de274fb66a471ecbc5245bd8684e6f94943c2f"}, + {file = "fastavro-1.7.0-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a7e35b94b692f8cca0096c89abf1937efed66252dea0b3b3165babfb3c289fb7"}, + {file = "fastavro-1.7.0-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:202424a5a7831b773f0f2cc2f82e89ed1726051fd5994f13dc678514144e10d4"}, + {file = "fastavro-1.7.0-cp38-cp38-win_amd64.whl", hash = "sha256:601f8c2ec166bd721f4b12eafe195dd1373d3f8dce4fe2425abd2df3e3968ac7"}, + {file = "fastavro-1.7.0-cp39-cp39-macosx_10_15_x86_64.whl", hash = "sha256:91cc9be740abca894082af4fe3ab9db057d4e5fa783cfa9a94c02ec041bf4346"}, + {file = "fastavro-1.7.0-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:9e4463b00242e4baf52d499aeefab46a26d9dd18d808d4219cd4d21089da540e"}, + {file = "fastavro-1.7.0-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:f7348d318858203bd108e6bcde177d8a6f0590b52bc624d815f26fb6c37029bb"}, + {file = "fastavro-1.7.0-cp39-cp39-win_amd64.whl", hash = "sha256:492e8902063aa1c73170e11c18495fcaa86b71eae3513ef83ba438ca02b16b34"}, + {file = "fastavro-1.7.0.tar.gz", hash = "sha256:4b1205f46489b4032d3155c1ab44d9824be0c7454df98d3a5bd22b78b98f23c8"}, +] +filelock = [ + {file = "filelock-3.9.0-py3-none-any.whl", hash = "sha256:f58d535af89bb9ad5cd4df046f741f8553a418c01a7856bf0d173bbc9f6bd16d"}, + {file = "filelock-3.9.0.tar.gz", hash = "sha256:7b319f24340b51f55a2bf7a12ac0755a9b03e718311dac567a0f4f7fabd2f5de"}, +] +frozenlist = [ + {file = "frozenlist-1.3.3-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:ff8bf625fe85e119553b5383ba0fb6aa3d0ec2ae980295aaefa552374926b3f4"}, + {file = "frozenlist-1.3.3-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:dfbac4c2dfcc082fcf8d942d1e49b6aa0766c19d3358bd86e2000bf0fa4a9cf0"}, + {file = "frozenlist-1.3.3-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:b1c63e8d377d039ac769cd0926558bb7068a1f7abb0f003e3717ee003ad85530"}, + {file = "frozenlist-1.3.3-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:7fdfc24dcfce5b48109867c13b4cb15e4660e7bd7661741a391f821f23dfdca7"}, + {file = "frozenlist-1.3.3-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:2c926450857408e42f0bbc295e84395722ce74bae69a3b2aa2a65fe22cb14b99"}, + {file = "frozenlist-1.3.3-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:1841e200fdafc3d51f974d9d377c079a0694a8f06de2e67b48150328d66d5483"}, + {file = "frozenlist-1.3.3-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:f470c92737afa7d4c3aacc001e335062d582053d4dbe73cda126f2d7031068dd"}, + {file = "frozenlist-1.3.3-cp310-cp310-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:783263a4eaad7c49983fe4b2e7b53fa9770c136c270d2d4bbb6d2192bf4d9caf"}, + {file = "frozenlist-1.3.3-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:924620eef691990dfb56dc4709f280f40baee568c794b5c1885800c3ecc69816"}, + {file = "frozenlist-1.3.3-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:ae4dc05c465a08a866b7a1baf360747078b362e6a6dbeb0c57f234db0ef88ae0"}, + {file = "frozenlist-1.3.3-cp310-cp310-musllinux_1_1_ppc64le.whl", hash = "sha256:bed331fe18f58d844d39ceb398b77d6ac0b010d571cba8267c2e7165806b00ce"}, + {file = "frozenlist-1.3.3-cp310-cp310-musllinux_1_1_s390x.whl", hash = "sha256:02c9ac843e3390826a265e331105efeab489ffaf4dd86384595ee8ce6d35ae7f"}, + {file = "frozenlist-1.3.3-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:9545a33965d0d377b0bc823dcabf26980e77f1b6a7caa368a365a9497fb09420"}, + {file = "frozenlist-1.3.3-cp310-cp310-win32.whl", hash = "sha256:d5cd3ab21acbdb414bb6c31958d7b06b85eeb40f66463c264a9b343a4e238642"}, + {file = "frozenlist-1.3.3-cp310-cp310-win_amd64.whl", hash = "sha256:b756072364347cb6aa5b60f9bc18e94b2f79632de3b0190253ad770c5df17db1"}, + {file = "frozenlist-1.3.3-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:b4395e2f8d83fbe0c627b2b696acce67868793d7d9750e90e39592b3626691b7"}, + {file = "frozenlist-1.3.3-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:14143ae966a6229350021384870458e4777d1eae4c28d1a7aa47f24d030e6678"}, + {file = "frozenlist-1.3.3-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:5d8860749e813a6f65bad8285a0520607c9500caa23fea6ee407e63debcdbef6"}, + {file = "frozenlist-1.3.3-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:23d16d9f477bb55b6154654e0e74557040575d9d19fe78a161bd33d7d76808e8"}, + {file = "frozenlist-1.3.3-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:eb82dbba47a8318e75f679690190c10a5e1f447fbf9df41cbc4c3afd726d88cb"}, + {file = "frozenlist-1.3.3-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:9309869032abb23d196cb4e4db574232abe8b8be1339026f489eeb34a4acfd91"}, + {file = "frozenlist-1.3.3-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:a97b4fe50b5890d36300820abd305694cb865ddb7885049587a5678215782a6b"}, + {file = "frozenlist-1.3.3-cp311-cp311-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:c188512b43542b1e91cadc3c6c915a82a5eb95929134faf7fd109f14f9892ce4"}, + {file = "frozenlist-1.3.3-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:303e04d422e9b911a09ad499b0368dc551e8c3cd15293c99160c7f1f07b59a48"}, + {file = "frozenlist-1.3.3-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:0771aed7f596c7d73444c847a1c16288937ef988dc04fb9f7be4b2aa91db609d"}, + {file = "frozenlist-1.3.3-cp311-cp311-musllinux_1_1_ppc64le.whl", hash = "sha256:66080ec69883597e4d026f2f71a231a1ee9887835902dbe6b6467d5a89216cf6"}, + {file = "frozenlist-1.3.3-cp311-cp311-musllinux_1_1_s390x.whl", hash = "sha256:41fe21dc74ad3a779c3d73a2786bdf622ea81234bdd4faf90b8b03cad0c2c0b4"}, + {file = "frozenlist-1.3.3-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:f20380df709d91525e4bee04746ba612a4df0972c1b8f8e1e8af997e678c7b81"}, + {file = "frozenlist-1.3.3-cp311-cp311-win32.whl", hash = "sha256:f30f1928162e189091cf4d9da2eac617bfe78ef907a761614ff577ef4edfb3c8"}, + {file = "frozenlist-1.3.3-cp311-cp311-win_amd64.whl", hash = "sha256:a6394d7dadd3cfe3f4b3b186e54d5d8504d44f2d58dcc89d693698e8b7132b32"}, + {file = "frozenlist-1.3.3-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:8df3de3a9ab8325f94f646609a66cbeeede263910c5c0de0101079ad541af332"}, + {file = "frozenlist-1.3.3-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:0693c609e9742c66ba4870bcee1ad5ff35462d5ffec18710b4ac89337ff16e27"}, + {file = "frozenlist-1.3.3-cp37-cp37m-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:cd4210baef299717db0a600d7a3cac81d46ef0e007f88c9335db79f8979c0d3d"}, + {file = "frozenlist-1.3.3-cp37-cp37m-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:394c9c242113bfb4b9aa36e2b80a05ffa163a30691c7b5a29eba82e937895d5e"}, + {file = "frozenlist-1.3.3-cp37-cp37m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:6327eb8e419f7d9c38f333cde41b9ae348bec26d840927332f17e887a8dcb70d"}, + {file = "frozenlist-1.3.3-cp37-cp37m-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:2e24900aa13212e75e5b366cb9065e78bbf3893d4baab6052d1aca10d46d944c"}, + {file = "frozenlist-1.3.3-cp37-cp37m-musllinux_1_1_aarch64.whl", hash = "sha256:3843f84a6c465a36559161e6c59dce2f2ac10943040c2fd021cfb70d58c4ad56"}, + {file = "frozenlist-1.3.3-cp37-cp37m-musllinux_1_1_i686.whl", hash = "sha256:84610c1502b2461255b4c9b7d5e9c48052601a8957cd0aea6ec7a7a1e1fb9420"}, + {file = "frozenlist-1.3.3-cp37-cp37m-musllinux_1_1_ppc64le.whl", hash = "sha256:c21b9aa40e08e4f63a2f92ff3748e6b6c84d717d033c7b3438dd3123ee18f70e"}, + {file = "frozenlist-1.3.3-cp37-cp37m-musllinux_1_1_s390x.whl", hash = "sha256:efce6ae830831ab6a22b9b4091d411698145cb9b8fc869e1397ccf4b4b6455cb"}, + {file = "frozenlist-1.3.3-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:40de71985e9042ca00b7953c4f41eabc3dc514a2d1ff534027f091bc74416401"}, + {file = "frozenlist-1.3.3-cp37-cp37m-win32.whl", hash = "sha256:180c00c66bde6146a860cbb81b54ee0df350d2daf13ca85b275123bbf85de18a"}, + {file = "frozenlist-1.3.3-cp37-cp37m-win_amd64.whl", hash = "sha256:9bbbcedd75acdfecf2159663b87f1bb5cfc80e7cd99f7ddd9d66eb98b14a8411"}, + {file = "frozenlist-1.3.3-cp38-cp38-macosx_10_9_universal2.whl", hash = "sha256:034a5c08d36649591be1cbb10e09da9f531034acfe29275fc5454a3b101ce41a"}, + {file = "frozenlist-1.3.3-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:ba64dc2b3b7b158c6660d49cdb1d872d1d0bf4e42043ad8d5006099479a194e5"}, + {file = "frozenlist-1.3.3-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:47df36a9fe24054b950bbc2db630d508cca3aa27ed0566c0baf661225e52c18e"}, + {file = "frozenlist-1.3.3-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:008a054b75d77c995ea26629ab3a0c0d7281341f2fa7e1e85fa6153ae29ae99c"}, + {file = "frozenlist-1.3.3-cp38-cp38-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:841ea19b43d438a80b4de62ac6ab21cfe6827bb8a9dc62b896acc88eaf9cecba"}, + {file = "frozenlist-1.3.3-cp38-cp38-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:e235688f42b36be2b6b06fc37ac2126a73b75fb8d6bc66dd632aa35286238703"}, + {file = "frozenlist-1.3.3-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:ca713d4af15bae6e5d79b15c10c8522859a9a89d3b361a50b817c98c2fb402a2"}, + {file = "frozenlist-1.3.3-cp38-cp38-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:9ac5995f2b408017b0be26d4a1d7c61bce106ff3d9e3324374d66b5964325448"}, + {file = "frozenlist-1.3.3-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:a4ae8135b11652b08a8baf07631d3ebfe65a4c87909dbef5fa0cdde440444ee4"}, + {file = "frozenlist-1.3.3-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:4ea42116ceb6bb16dbb7d526e242cb6747b08b7710d9782aa3d6732bd8d27649"}, + {file = "frozenlist-1.3.3-cp38-cp38-musllinux_1_1_ppc64le.whl", hash = "sha256:810860bb4bdce7557bc0febb84bbd88198b9dbc2022d8eebe5b3590b2ad6c842"}, + {file = "frozenlist-1.3.3-cp38-cp38-musllinux_1_1_s390x.whl", hash = "sha256:ee78feb9d293c323b59a6f2dd441b63339a30edf35abcb51187d2fc26e696d13"}, + {file = "frozenlist-1.3.3-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:0af2e7c87d35b38732e810befb9d797a99279cbb85374d42ea61c1e9d23094b3"}, + {file = "frozenlist-1.3.3-cp38-cp38-win32.whl", hash = "sha256:899c5e1928eec13fd6f6d8dc51be23f0d09c5281e40d9cf4273d188d9feeaf9b"}, + {file = "frozenlist-1.3.3-cp38-cp38-win_amd64.whl", hash = "sha256:7f44e24fa70f6fbc74aeec3e971f60a14dde85da364aa87f15d1be94ae75aeef"}, + {file = "frozenlist-1.3.3-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:2b07ae0c1edaa0a36339ec6cce700f51b14a3fc6545fdd32930d2c83917332cf"}, + {file = "frozenlist-1.3.3-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:ebb86518203e12e96af765ee89034a1dbb0c3c65052d1b0c19bbbd6af8a145e1"}, + {file = "frozenlist-1.3.3-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:5cf820485f1b4c91e0417ea0afd41ce5cf5965011b3c22c400f6d144296ccbc0"}, + {file = "frozenlist-1.3.3-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:5c11e43016b9024240212d2a65043b70ed8dfd3b52678a1271972702d990ac6d"}, + {file = "frozenlist-1.3.3-cp39-cp39-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:8fa3c6e3305aa1146b59a09b32b2e04074945ffcfb2f0931836d103a2c38f936"}, + {file = "frozenlist-1.3.3-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:352bd4c8c72d508778cf05ab491f6ef36149f4d0cb3c56b1b4302852255d05d5"}, + {file = "frozenlist-1.3.3-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:65a5e4d3aa679610ac6e3569e865425b23b372277f89b5ef06cf2cdaf1ebf22b"}, + {file = "frozenlist-1.3.3-cp39-cp39-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:b1e2c1185858d7e10ff045c496bbf90ae752c28b365fef2c09cf0fa309291669"}, + {file = "frozenlist-1.3.3-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:f163d2fd041c630fed01bc48d28c3ed4a3b003c00acd396900e11ee5316b56bb"}, + {file = "frozenlist-1.3.3-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:05cdb16d09a0832eedf770cb7bd1fe57d8cf4eaf5aced29c4e41e3f20b30a784"}, + {file = "frozenlist-1.3.3-cp39-cp39-musllinux_1_1_ppc64le.whl", hash = "sha256:8bae29d60768bfa8fb92244b74502b18fae55a80eac13c88eb0b496d4268fd2d"}, + {file = "frozenlist-1.3.3-cp39-cp39-musllinux_1_1_s390x.whl", hash = "sha256:eedab4c310c0299961ac285591acd53dc6723a1ebd90a57207c71f6e0c2153ab"}, + {file = "frozenlist-1.3.3-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:3bbdf44855ed8f0fbcd102ef05ec3012d6a4fd7c7562403f76ce6a52aeffb2b1"}, + {file = "frozenlist-1.3.3-cp39-cp39-win32.whl", hash = "sha256:efa568b885bca461f7c7b9e032655c0c143d305bf01c30caf6db2854a4532b38"}, + {file = "frozenlist-1.3.3-cp39-cp39-win_amd64.whl", hash = "sha256:cfe33efc9cb900a4c46f91a5ceba26d6df370ffddd9ca386eb1d4f0ad97b9ea9"}, + {file = "frozenlist-1.3.3.tar.gz", hash = "sha256:58bcc55721e8a90b88332d6cd441261ebb22342e238296bb330968952fbb3a6a"}, +] +fsspec = [ + {file = "fsspec-2023.1.0-py3-none-any.whl", hash = "sha256:b833e2e541e9e8cde0ab549414187871243177feb3d344f9d27b25a93f5d8139"}, + {file = "fsspec-2023.1.0.tar.gz", hash = "sha256:fbae7f20ff801eb5f7d0bedf81f25c787c0dfac5e982d98fa3884a9cde2b5411"}, +] +identify = [ + {file = "identify-2.5.13-py2.py3-none-any.whl", hash = "sha256:8aa48ce56e38c28b6faa9f261075dea0a942dfbb42b341b4e711896cbb40f3f7"}, + {file = "identify-2.5.13.tar.gz", hash = "sha256:abb546bca6f470228785338a01b539de8a85bbf46491250ae03363956d8ebb10"}, +] +idna = [ + {file = "idna-3.4-py3-none-any.whl", hash = "sha256:90b77e79eaa3eba6de819a0c442c0b4ceefc341a7a2ab77d7562bf49f425c5c2"}, + {file = "idna-3.4.tar.gz", hash = "sha256:814f528e8dead7d329833b91c5faa87d60bf71824cd12a7530b5526063d02cb4"}, +] +importlib-metadata = [ + {file = "importlib_metadata-6.0.0-py3-none-any.whl", hash = "sha256:7efb448ec9a5e313a57655d35aa54cd3e01b7e1fbcf72dce1bf06119420f5bad"}, + {file = "importlib_metadata-6.0.0.tar.gz", hash = "sha256:e354bedeb60efa6affdcc8ae121b73544a7aa74156d047311948f6d711cd378d"}, +] +iniconfig = [ + {file = "iniconfig-2.0.0-py3-none-any.whl", hash = "sha256:b6a85871a79d2e3b22d2d1b94ac2824226a63c6b741c88f7ae975f18b6778374"}, + {file = "iniconfig-2.0.0.tar.gz", hash = "sha256:2d91e135bf72d31a410b17c16da610a82cb55f6b0477d1a902134b24a455b8b3"}, +] +isodate = [ + {file = "isodate-0.6.1-py2.py3-none-any.whl", hash = "sha256:0751eece944162659049d35f4f549ed815792b38793f07cf73381c1c87cbed96"}, + {file = "isodate-0.6.1.tar.gz", hash = "sha256:48c5881de7e8b0a0d648cb024c8062dc84e7b840ed81e864c7614fd3c127bde9"}, +] +jinja2 = [ + {file = "Jinja2-3.1.2-py3-none-any.whl", hash = "sha256:6088930bfe239f0e6710546ab9c19c9ef35e29792895fed6e6e31a023a182a61"}, + {file = "Jinja2-3.1.2.tar.gz", hash = "sha256:31351a702a408a9e7595a8fc6150fc3f43bb6bf7e319770cbc0db9df9437e852"}, +] +jmespath = [] +markdown-it-py = [ + {file = "markdown-it-py-2.1.0.tar.gz", hash = "sha256:cf7e59fed14b5ae17c0006eff14a2d9a00ed5f3a846148153899a0224e2c07da"}, + {file = "markdown_it_py-2.1.0-py3-none-any.whl", hash = "sha256:93de681e5c021a432c63147656fe21790bc01231e0cd2da73626f1aa3ac0fe27"}, +] +markupsafe = [ + {file = "MarkupSafe-2.1.2-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:665a36ae6f8f20a4676b53224e33d456a6f5a72657d9c83c2aa00765072f31f7"}, + {file = "MarkupSafe-2.1.2-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:340bea174e9761308703ae988e982005aedf427de816d1afe98147668cc03036"}, + {file = "MarkupSafe-2.1.2-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:22152d00bf4a9c7c83960521fc558f55a1adbc0631fbb00a9471e097b19d72e1"}, + {file = "MarkupSafe-2.1.2-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:28057e985dace2f478e042eaa15606c7efccb700797660629da387eb289b9323"}, + {file = "MarkupSafe-2.1.2-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:ca244fa73f50a800cf8c3ebf7fd93149ec37f5cb9596aa8873ae2c1d23498601"}, + {file = "MarkupSafe-2.1.2-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:d9d971ec1e79906046aa3ca266de79eac42f1dbf3612a05dc9368125952bd1a1"}, + {file = "MarkupSafe-2.1.2-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:7e007132af78ea9df29495dbf7b5824cb71648d7133cf7848a2a5dd00d36f9ff"}, + {file = "MarkupSafe-2.1.2-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:7313ce6a199651c4ed9d7e4cfb4aa56fe923b1adf9af3b420ee14e6d9a73df65"}, + {file = "MarkupSafe-2.1.2-cp310-cp310-win32.whl", hash = "sha256:c4a549890a45f57f1ebf99c067a4ad0cb423a05544accaf2b065246827ed9603"}, + {file = "MarkupSafe-2.1.2-cp310-cp310-win_amd64.whl", hash = "sha256:835fb5e38fd89328e9c81067fd642b3593c33e1e17e2fdbf77f5676abb14a156"}, + {file = "MarkupSafe-2.1.2-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:2ec4f2d48ae59bbb9d1f9d7efb9236ab81429a764dedca114f5fdabbc3788013"}, + {file = "MarkupSafe-2.1.2-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:608e7073dfa9e38a85d38474c082d4281f4ce276ac0010224eaba11e929dd53a"}, + {file = "MarkupSafe-2.1.2-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:65608c35bfb8a76763f37036547f7adfd09270fbdbf96608be2bead319728fcd"}, + {file = "MarkupSafe-2.1.2-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:f2bfb563d0211ce16b63c7cb9395d2c682a23187f54c3d79bfec33e6705473c6"}, + {file = "MarkupSafe-2.1.2-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:da25303d91526aac3672ee6d49a2f3db2d9502a4a60b55519feb1a4c7714e07d"}, + {file = "MarkupSafe-2.1.2-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:9cad97ab29dfc3f0249b483412c85c8ef4766d96cdf9dcf5a1e3caa3f3661cf1"}, + {file = "MarkupSafe-2.1.2-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:085fd3201e7b12809f9e6e9bc1e5c96a368c8523fad5afb02afe3c051ae4afcc"}, + {file = "MarkupSafe-2.1.2-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:1bea30e9bf331f3fef67e0a3877b2288593c98a21ccb2cf29b74c581a4eb3af0"}, + {file = "MarkupSafe-2.1.2-cp311-cp311-win32.whl", hash = "sha256:7df70907e00c970c60b9ef2938d894a9381f38e6b9db73c5be35e59d92e06625"}, + {file = "MarkupSafe-2.1.2-cp311-cp311-win_amd64.whl", hash = "sha256:e55e40ff0cc8cc5c07996915ad367fa47da6b3fc091fdadca7f5403239c5fec3"}, + {file = "MarkupSafe-2.1.2-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:a6e40afa7f45939ca356f348c8e23048e02cb109ced1eb8420961b2f40fb373a"}, + {file = "MarkupSafe-2.1.2-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:cf877ab4ed6e302ec1d04952ca358b381a882fbd9d1b07cccbfd61783561f98a"}, + {file = "MarkupSafe-2.1.2-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:63ba06c9941e46fa389d389644e2d8225e0e3e5ebcc4ff1ea8506dce646f8c8a"}, + {file = "MarkupSafe-2.1.2-cp37-cp37m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:f1cd098434e83e656abf198f103a8207a8187c0fc110306691a2e94a78d0abb2"}, + {file = "MarkupSafe-2.1.2-cp37-cp37m-musllinux_1_1_aarch64.whl", hash = "sha256:55f44b440d491028addb3b88f72207d71eeebfb7b5dbf0643f7c023ae1fba619"}, + {file = "MarkupSafe-2.1.2-cp37-cp37m-musllinux_1_1_i686.whl", hash = "sha256:a6f2fcca746e8d5910e18782f976489939d54a91f9411c32051b4aab2bd7c513"}, + {file = "MarkupSafe-2.1.2-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:0b462104ba25f1ac006fdab8b6a01ebbfbce9ed37fd37fd4acd70c67c973e460"}, + {file = "MarkupSafe-2.1.2-cp37-cp37m-win32.whl", hash = "sha256:7668b52e102d0ed87cb082380a7e2e1e78737ddecdde129acadb0eccc5423859"}, + {file = "MarkupSafe-2.1.2-cp37-cp37m-win_amd64.whl", hash = "sha256:6d6607f98fcf17e534162f0709aaad3ab7a96032723d8ac8750ffe17ae5a0666"}, + {file = "MarkupSafe-2.1.2-cp38-cp38-macosx_10_9_universal2.whl", hash = "sha256:a806db027852538d2ad7555b203300173dd1b77ba116de92da9afbc3a3be3eed"}, + {file = "MarkupSafe-2.1.2-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:a4abaec6ca3ad8660690236d11bfe28dfd707778e2442b45addd2f086d6ef094"}, + {file = "MarkupSafe-2.1.2-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:f03a532d7dee1bed20bc4884194a16160a2de9ffc6354b3878ec9682bb623c54"}, + {file = "MarkupSafe-2.1.2-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:4cf06cdc1dda95223e9d2d3c58d3b178aa5dacb35ee7e3bbac10e4e1faacb419"}, + {file = "MarkupSafe-2.1.2-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:22731d79ed2eb25059ae3df1dfc9cb1546691cc41f4e3130fe6bfbc3ecbbecfa"}, + {file = "MarkupSafe-2.1.2-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:f8ffb705ffcf5ddd0e80b65ddf7bed7ee4f5a441ea7d3419e861a12eaf41af58"}, + {file = "MarkupSafe-2.1.2-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:8db032bf0ce9022a8e41a22598eefc802314e81b879ae093f36ce9ddf39ab1ba"}, + {file = "MarkupSafe-2.1.2-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:2298c859cfc5463f1b64bd55cb3e602528db6fa0f3cfd568d3605c50678f8f03"}, + {file = "MarkupSafe-2.1.2-cp38-cp38-win32.whl", hash = "sha256:50c42830a633fa0cf9e7d27664637532791bfc31c731a87b202d2d8ac40c3ea2"}, + {file = "MarkupSafe-2.1.2-cp38-cp38-win_amd64.whl", hash = "sha256:bb06feb762bade6bf3c8b844462274db0c76acc95c52abe8dbed28ae3d44a147"}, + {file = "MarkupSafe-2.1.2-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:99625a92da8229df6d44335e6fcc558a5037dd0a760e11d84be2260e6f37002f"}, + {file = "MarkupSafe-2.1.2-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:8bca7e26c1dd751236cfb0c6c72d4ad61d986e9a41bbf76cb445f69488b2a2bd"}, + {file = "MarkupSafe-2.1.2-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:40627dcf047dadb22cd25ea7ecfe9cbf3bbbad0482ee5920b582f3809c97654f"}, + {file = "MarkupSafe-2.1.2-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:40dfd3fefbef579ee058f139733ac336312663c6706d1163b82b3003fb1925c4"}, + {file = "MarkupSafe-2.1.2-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:090376d812fb6ac5f171e5938e82e7f2d7adc2b629101cec0db8b267815c85e2"}, + {file = "MarkupSafe-2.1.2-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:2e7821bffe00aa6bd07a23913b7f4e01328c3d5cc0b40b36c0bd81d362faeb65"}, + {file = "MarkupSafe-2.1.2-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:c0a33bc9f02c2b17c3ea382f91b4db0e6cde90b63b296422a939886a7a80de1c"}, + {file = "MarkupSafe-2.1.2-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:b8526c6d437855442cdd3d87eede9c425c4445ea011ca38d937db299382e6fa3"}, + {file = "MarkupSafe-2.1.2-cp39-cp39-win32.whl", hash = "sha256:137678c63c977754abe9086a3ec011e8fd985ab90631145dfb9294ad09c102a7"}, + {file = "MarkupSafe-2.1.2-cp39-cp39-win_amd64.whl", hash = "sha256:0576fe974b40a400449768941d5d0858cc624e3249dfd1e0c33674e5c7ca7aed"}, + {file = "MarkupSafe-2.1.2.tar.gz", hash = "sha256:abcabc8c2b26036d62d4c746381a6f7cf60aafcc653198ad678306986b09450d"}, +] +mdurl = [ + {file = "mdurl-0.1.2-py3-none-any.whl", hash = "sha256:84008a41e51615a49fc9966191ff91509e3c40b939176e643fd50a5c2196b8f8"}, + {file = "mdurl-0.1.2.tar.gz", hash = "sha256:bb413d29f5eea38f31dd4754dd7377d4465116fb207585f97bf925588687c1ba"}, +] +mmhash3 = [ + {file = "mmhash3-3.0.1-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:47deea30cd8d3d5cd52dc740902a4c70383bfe8248eac29d0877fe63e03c2713"}, + {file = "mmhash3-3.0.1-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:ecdaf4d1de617818bf05cd526ca558db6010beeba7ea9e19f695f2bdcac0e0a4"}, + {file = "mmhash3-3.0.1-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:4675585617584e9e1aafa3a90ac0ac9a437257c507748d97de8b21977e9d6745"}, + {file = "mmhash3-3.0.1-cp310-cp310-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:ebfd0c2af09b41f0fe1dd260799bda90a0fc7eba4477ccaeb3951527700cd58f"}, + {file = "mmhash3-3.0.1-cp310-cp310-win32.whl", hash = "sha256:68587dec7b8acdb7528fd511e295d8b5ccfe26022923a69867e1822f0fdb4c44"}, + {file = "mmhash3-3.0.1-cp310-cp310-win_amd64.whl", hash = "sha256:54954ebe3a614f19ab4cfe24fa66ea093c493d9fac0533d002dd64c2540a0c99"}, + {file = "mmhash3-3.0.1-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:b172f3bd3220b0bf56fd7cc760fd8a9033def47103611d63fe867003079a1256"}, + {file = "mmhash3-3.0.1-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:de7895eafabc32f7c0c09a81a624921f536768de6e438e3de69e3e954a4d7072"}, + {file = "mmhash3-3.0.1-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:b4b0914effe4ddd8d33149e3508564c17719465b0cc81691c4fa50d5e0e14f80"}, + {file = "mmhash3-3.0.1-cp311-cp311-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:c0575050ac691475938df1ad03d8738c5bd1eadef62093e76157ebb7f2df0946"}, + {file = "mmhash3-3.0.1-cp311-cp311-win32.whl", hash = "sha256:22f92f0f88f28b215357acd346362fa9f7c9fffb436beb42cc4b442b676dbaa3"}, + {file = "mmhash3-3.0.1-cp311-cp311-win_amd64.whl", hash = "sha256:538240ab7936bf71b18304e5a7e7fd3c4c2fab103330ea99584bb4f777299a2b"}, + {file = "mmhash3-3.0.1-cp36-cp36m-macosx_10_9_x86_64.whl", hash = "sha256:ca791bfb311e36ce13998e4632262ed4b95da9d3461941e18b6690760171a045"}, + {file = "mmhash3-3.0.1-cp36-cp36m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:b41708f72c6aa2a49ada1f0b61e85c05cd8389ad31d463fd5bca29999a4d5f9c"}, + {file = "mmhash3-3.0.1-cp36-cp36m-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:a3ce9b4533ddc0a88ba045a27309714c3b127bd05e57fd252d1d5a71d4247ea7"}, + {file = "mmhash3-3.0.1-cp36-cp36m-win32.whl", hash = "sha256:bfafeb96fdeb10db8767d06e1f07b6fdcddba4aaa0dd15058561a49f7ae45345"}, + {file = "mmhash3-3.0.1-cp36-cp36m-win_amd64.whl", hash = "sha256:97fe077b24c948709ed2afc749bf6285d407bc54ff12c63d2dc86678c38a0b8e"}, + {file = "mmhash3-3.0.1-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:0cfd91ccd5fca1ba7ee925297131a15dfb94c714bfe6ba0fb3b1ca78b12bbfec"}, + {file = "mmhash3-3.0.1-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:6d51b1005233141ce7394531af40a3f0fc1f274467bf8dff44dcf7987924fe58"}, + {file = "mmhash3-3.0.1-cp37-cp37m-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:855c67b100e37df166acf79cdff58fa8f9f6c48be0d1e1b6e9ad0fa34a9661ef"}, + {file = "mmhash3-3.0.1-cp37-cp37m-win32.whl", hash = "sha256:bb3030df1334fd665427f8be8e8ce4f04aeab7f6010ce4f2c128f0099bdab96f"}, + {file = "mmhash3-3.0.1-cp37-cp37m-win_amd64.whl", hash = "sha256:1545e1177294afe4912d5a5e401c7fa9b799dd109b30289e7af74d5b07e7c474"}, + {file = "mmhash3-3.0.1-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:2479899e7dda834a671991a1098a691ab1c2eaa20c3e939d691ca4a19361cfe0"}, + {file = "mmhash3-3.0.1-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:c9056196d5e3d3d844433a63d806a683f710ab3aaf1c910550c7746464bc43ae"}, + {file = "mmhash3-3.0.1-cp38-cp38-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:a0d4c307af0bf70207305f70f131898be071d1b19a89f462b13487f5c25e8d4e"}, + {file = "mmhash3-3.0.1-cp38-cp38-win32.whl", hash = "sha256:5f885f65e329fd14bc38debac4a79eacf381e856965d9c65c4d1c6946ea190d0"}, + {file = "mmhash3-3.0.1-cp38-cp38-win_amd64.whl", hash = "sha256:3b42d0bda5e1cd22c18b16887b0521060fb59d0aaaaf033feacbc0a2492d20fe"}, + {file = "mmhash3-3.0.1-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:d3f333286bb87aa9dc6bd8e7960a55a27b011a401f24b889a50e6d219f65e7c9"}, + {file = "mmhash3-3.0.1-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:6b7ef2eb95a18bcd02ce0d3e047adde3a025afd96c1d266a8a0d44574f44a307"}, + {file = "mmhash3-3.0.1-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:d6ac8a5f511c60f341bf9cae462bb4941abb149d98464ba5f4f4548875b601c6"}, + {file = "mmhash3-3.0.1-cp39-cp39-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:efef9e632e6e248e46f52d108a5ebd1f28edaf427b7fd47ebf97dbff4b2cab81"}, + {file = "mmhash3-3.0.1-cp39-cp39-win32.whl", hash = "sha256:bdac06d72e448c67afb12e758b203d875e29d4097bb279a38a5649d44b518ba7"}, + {file = "mmhash3-3.0.1-cp39-cp39-win_amd64.whl", hash = "sha256:0baeaa20cac5f75ed14f28056826bd9d9c8b2354b382073f3fd5190708992a0d"}, + {file = "mmhash3-3.0.1.tar.gz", hash = "sha256:a00d68f4a1cc434b9501513c8a29e18ed1ddad383677d72b41d71d0d862348af"}, +] +moto = [ + {file = "moto-4.1.0-py2.py3-none-any.whl", hash = "sha256:12f231196ebf22d19774f2a75738fdedae10a8c82abb8fa0877e1819aa799f4f"}, + {file = "moto-4.1.0.tar.gz", hash = "sha256:e65a692df57abb631b06c8f3f7827c9deceae3416887527e266c775ad722a2fa"}, +] +msal = [ + {file = "msal-1.20.0-py2.py3-none-any.whl", hash = "sha256:d2f1c26368ecdc28c8657d457352faa0b81b1845a7b889d8676787721ba86792"}, + {file = "msal-1.20.0.tar.gz", hash = "sha256:78344cd4c91d6134a593b5e3e45541e666e37b747ff8a6316c3668dd1e6ab6b2"}, +] +msal-extensions = [ + {file = "msal-extensions-1.0.0.tar.gz", hash = "sha256:c676aba56b0cce3783de1b5c5ecfe828db998167875126ca4b47dc6436451354"}, + {file = "msal_extensions-1.0.0-py2.py3-none-any.whl", hash = "sha256:91e3db9620b822d0ed2b4d1850056a0f133cba04455e62f11612e40f5502f2ee"}, +] +msrest = [ + {file = "msrest-0.7.1-py3-none-any.whl", hash = "sha256:21120a810e1233e5e6cc7fe40b474eeb4ec6f757a15d7cf86702c369f9567c32"}, + {file = "msrest-0.7.1.zip", hash = "sha256:6e7661f46f3afd88b75667b7187a92829924446c7ea1d169be8c4bb7eeb788b9"}, +] +multidict = [ + {file = "multidict-6.0.4-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:0b1a97283e0c85772d613878028fec909f003993e1007eafa715b24b377cb9b8"}, + {file = "multidict-6.0.4-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:eeb6dcc05e911516ae3d1f207d4b0520d07f54484c49dfc294d6e7d63b734171"}, + {file = "multidict-6.0.4-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:d6d635d5209b82a3492508cf5b365f3446afb65ae7ebd755e70e18f287b0adf7"}, + {file = "multidict-6.0.4-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:c048099e4c9e9d615545e2001d3d8a4380bd403e1a0578734e0d31703d1b0c0b"}, + {file = "multidict-6.0.4-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:ea20853c6dbbb53ed34cb4d080382169b6f4554d394015f1bef35e881bf83547"}, + {file = "multidict-6.0.4-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:16d232d4e5396c2efbbf4f6d4df89bfa905eb0d4dc5b3549d872ab898451f569"}, + {file = "multidict-6.0.4-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:36c63aaa167f6c6b04ef2c85704e93af16c11d20de1d133e39de6a0e84582a93"}, + {file = "multidict-6.0.4-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:64bdf1086b6043bf519869678f5f2757f473dee970d7abf6da91ec00acb9cb98"}, + {file = "multidict-6.0.4-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:43644e38f42e3af682690876cff722d301ac585c5b9e1eacc013b7a3f7b696a0"}, + {file = "multidict-6.0.4-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:7582a1d1030e15422262de9f58711774e02fa80df0d1578995c76214f6954988"}, + {file = "multidict-6.0.4-cp310-cp310-musllinux_1_1_ppc64le.whl", hash = "sha256:ddff9c4e225a63a5afab9dd15590432c22e8057e1a9a13d28ed128ecf047bbdc"}, + {file = "multidict-6.0.4-cp310-cp310-musllinux_1_1_s390x.whl", hash = "sha256:ee2a1ece51b9b9e7752e742cfb661d2a29e7bcdba2d27e66e28a99f1890e4fa0"}, + {file = "multidict-6.0.4-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:a2e4369eb3d47d2034032a26c7a80fcb21a2cb22e1173d761a162f11e562caa5"}, + {file = "multidict-6.0.4-cp310-cp310-win32.whl", hash = "sha256:574b7eae1ab267e5f8285f0fe881f17efe4b98c39a40858247720935b893bba8"}, + {file = "multidict-6.0.4-cp310-cp310-win_amd64.whl", hash = "sha256:4dcbb0906e38440fa3e325df2359ac6cb043df8e58c965bb45f4e406ecb162cc"}, + {file = "multidict-6.0.4-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:0dfad7a5a1e39c53ed00d2dd0c2e36aed4650936dc18fd9a1826a5ae1cad6f03"}, + {file = "multidict-6.0.4-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:64da238a09d6039e3bd39bb3aee9c21a5e34f28bfa5aa22518581f910ff94af3"}, + {file = "multidict-6.0.4-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:ff959bee35038c4624250473988b24f846cbeb2c6639de3602c073f10410ceba"}, + {file = "multidict-6.0.4-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:01a3a55bd90018c9c080fbb0b9f4891db37d148a0a18722b42f94694f8b6d4c9"}, + {file = "multidict-6.0.4-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:c5cb09abb18c1ea940fb99360ea0396f34d46566f157122c92dfa069d3e0e982"}, + {file = "multidict-6.0.4-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:666daae833559deb2d609afa4490b85830ab0dfca811a98b70a205621a6109fe"}, + {file = "multidict-6.0.4-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:11bdf3f5e1518b24530b8241529d2050014c884cf18b6fc69c0c2b30ca248710"}, + {file = "multidict-6.0.4-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:7d18748f2d30f94f498e852c67d61261c643b349b9d2a581131725595c45ec6c"}, + {file = "multidict-6.0.4-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:458f37be2d9e4c95e2d8866a851663cbc76e865b78395090786f6cd9b3bbf4f4"}, + {file = "multidict-6.0.4-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:b1a2eeedcead3a41694130495593a559a668f382eee0727352b9a41e1c45759a"}, + {file = "multidict-6.0.4-cp311-cp311-musllinux_1_1_ppc64le.whl", hash = "sha256:7d6ae9d593ef8641544d6263c7fa6408cc90370c8cb2bbb65f8d43e5b0351d9c"}, + {file = "multidict-6.0.4-cp311-cp311-musllinux_1_1_s390x.whl", hash = "sha256:5979b5632c3e3534e42ca6ff856bb24b2e3071b37861c2c727ce220d80eee9ed"}, + {file = "multidict-6.0.4-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:dcfe792765fab89c365123c81046ad4103fcabbc4f56d1c1997e6715e8015461"}, + {file = "multidict-6.0.4-cp311-cp311-win32.whl", hash = "sha256:3601a3cece3819534b11d4efc1eb76047488fddd0c85a3948099d5da4d504636"}, + {file = "multidict-6.0.4-cp311-cp311-win_amd64.whl", hash = "sha256:81a4f0b34bd92df3da93315c6a59034df95866014ac08535fc819f043bfd51f0"}, + {file = "multidict-6.0.4-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:67040058f37a2a51ed8ea8f6b0e6ee5bd78ca67f169ce6122f3e2ec80dfe9b78"}, + {file = "multidict-6.0.4-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:853888594621e6604c978ce2a0444a1e6e70c8d253ab65ba11657659dcc9100f"}, + {file = "multidict-6.0.4-cp37-cp37m-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:39ff62e7d0f26c248b15e364517a72932a611a9b75f35b45be078d81bdb86603"}, + {file = "multidict-6.0.4-cp37-cp37m-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:af048912e045a2dc732847d33821a9d84ba553f5c5f028adbd364dd4765092ac"}, + {file = "multidict-6.0.4-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:b1e8b901e607795ec06c9e42530788c45ac21ef3aaa11dbd0c69de543bfb79a9"}, + {file = "multidict-6.0.4-cp37-cp37m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:62501642008a8b9871ddfccbf83e4222cf8ac0d5aeedf73da36153ef2ec222d2"}, + {file = "multidict-6.0.4-cp37-cp37m-musllinux_1_1_aarch64.whl", hash = "sha256:99b76c052e9f1bc0721f7541e5e8c05db3941eb9ebe7b8553c625ef88d6eefde"}, + {file = "multidict-6.0.4-cp37-cp37m-musllinux_1_1_i686.whl", hash = "sha256:509eac6cf09c794aa27bcacfd4d62c885cce62bef7b2c3e8b2e49d365b5003fe"}, + {file = "multidict-6.0.4-cp37-cp37m-musllinux_1_1_ppc64le.whl", hash = "sha256:21a12c4eb6ddc9952c415f24eef97e3e55ba3af61f67c7bc388dcdec1404a067"}, + {file = "multidict-6.0.4-cp37-cp37m-musllinux_1_1_s390x.whl", hash = "sha256:5cad9430ab3e2e4fa4a2ef4450f548768400a2ac635841bc2a56a2052cdbeb87"}, + {file = "multidict-6.0.4-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:ab55edc2e84460694295f401215f4a58597f8f7c9466faec545093045476327d"}, + {file = "multidict-6.0.4-cp37-cp37m-win32.whl", hash = "sha256:5a4dcf02b908c3b8b17a45fb0f15b695bf117a67b76b7ad18b73cf8e92608775"}, + {file = "multidict-6.0.4-cp37-cp37m-win_amd64.whl", hash = "sha256:6ed5f161328b7df384d71b07317f4d8656434e34591f20552c7bcef27b0ab88e"}, + {file = "multidict-6.0.4-cp38-cp38-macosx_10_9_universal2.whl", hash = "sha256:5fc1b16f586f049820c5c5b17bb4ee7583092fa0d1c4e28b5239181ff9532e0c"}, + {file = "multidict-6.0.4-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:1502e24330eb681bdaa3eb70d6358e818e8e8f908a22a1851dfd4e15bc2f8161"}, + {file = "multidict-6.0.4-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:b692f419760c0e65d060959df05f2a531945af31fda0c8a3b3195d4efd06de11"}, + {file = "multidict-6.0.4-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:45e1ecb0379bfaab5eef059f50115b54571acfbe422a14f668fc8c27ba410e7e"}, + {file = "multidict-6.0.4-cp38-cp38-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:ddd3915998d93fbcd2566ddf9cf62cdb35c9e093075f862935573d265cf8f65d"}, + {file = "multidict-6.0.4-cp38-cp38-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:59d43b61c59d82f2effb39a93c48b845efe23a3852d201ed2d24ba830d0b4cf2"}, + {file = "multidict-6.0.4-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:cc8e1d0c705233c5dd0c5e6460fbad7827d5d36f310a0fadfd45cc3029762258"}, + {file = "multidict-6.0.4-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:d6aa0418fcc838522256761b3415822626f866758ee0bc6632c9486b179d0b52"}, + {file = "multidict-6.0.4-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:6748717bb10339c4760c1e63da040f5f29f5ed6e59d76daee30305894069a660"}, + {file = "multidict-6.0.4-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:4d1a3d7ef5e96b1c9e92f973e43aa5e5b96c659c9bc3124acbbd81b0b9c8a951"}, + {file = "multidict-6.0.4-cp38-cp38-musllinux_1_1_ppc64le.whl", hash = "sha256:4372381634485bec7e46718edc71528024fcdc6f835baefe517b34a33c731d60"}, + {file = "multidict-6.0.4-cp38-cp38-musllinux_1_1_s390x.whl", hash = "sha256:fc35cb4676846ef752816d5be2193a1e8367b4c1397b74a565a9d0389c433a1d"}, + {file = "multidict-6.0.4-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:4b9d9e4e2b37daddb5c23ea33a3417901fa7c7b3dee2d855f63ee67a0b21e5b1"}, + {file = "multidict-6.0.4-cp38-cp38-win32.whl", hash = "sha256:e41b7e2b59679edfa309e8db64fdf22399eec4b0b24694e1b2104fb789207779"}, + {file = "multidict-6.0.4-cp38-cp38-win_amd64.whl", hash = "sha256:d6c254ba6e45d8e72739281ebc46ea5eb5f101234f3ce171f0e9f5cc86991480"}, + {file = "multidict-6.0.4-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:16ab77bbeb596e14212e7bab8429f24c1579234a3a462105cda4a66904998664"}, + {file = "multidict-6.0.4-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:bc779e9e6f7fda81b3f9aa58e3a6091d49ad528b11ed19f6621408806204ad35"}, + {file = "multidict-6.0.4-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:4ceef517eca3e03c1cceb22030a3e39cb399ac86bff4e426d4fc6ae49052cc60"}, + {file = "multidict-6.0.4-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:281af09f488903fde97923c7744bb001a9b23b039a909460d0f14edc7bf59706"}, + {file = "multidict-6.0.4-cp39-cp39-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:52f2dffc8acaba9a2f27174c41c9e57f60b907bb9f096b36b1a1f3be71c6284d"}, + {file = "multidict-6.0.4-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:b41156839806aecb3641f3208c0dafd3ac7775b9c4c422d82ee2a45c34ba81ca"}, + {file = "multidict-6.0.4-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:d5e3fc56f88cc98ef8139255cf8cd63eb2c586531e43310ff859d6bb3a6b51f1"}, + {file = "multidict-6.0.4-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:8316a77808c501004802f9beebde51c9f857054a0c871bd6da8280e718444449"}, + {file = "multidict-6.0.4-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:f70b98cd94886b49d91170ef23ec5c0e8ebb6f242d734ed7ed677b24d50c82cf"}, + {file = "multidict-6.0.4-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:bf6774e60d67a9efe02b3616fee22441d86fab4c6d335f9d2051d19d90a40063"}, + {file = "multidict-6.0.4-cp39-cp39-musllinux_1_1_ppc64le.whl", hash = "sha256:e69924bfcdda39b722ef4d9aa762b2dd38e4632b3641b1d9a57ca9cd18f2f83a"}, + {file = "multidict-6.0.4-cp39-cp39-musllinux_1_1_s390x.whl", hash = "sha256:6b181d8c23da913d4ff585afd1155a0e1194c0b50c54fcfe286f70cdaf2b7176"}, + {file = "multidict-6.0.4-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:52509b5be062d9eafc8170e53026fbc54cf3b32759a23d07fd935fb04fc22d95"}, + {file = "multidict-6.0.4-cp39-cp39-win32.whl", hash = "sha256:27c523fbfbdfd19c6867af7346332b62b586eed663887392cff78d614f9ec313"}, + {file = "multidict-6.0.4-cp39-cp39-win_amd64.whl", hash = "sha256:33029f5734336aa0d4c0384525da0387ef89148dc7191aae00ca5fb23d7aafc2"}, + {file = "multidict-6.0.4.tar.gz", hash = "sha256:3666906492efb76453c0e7b97f2cf459b0682e7402c0489a95484965dbc1da49"}, +] +nodeenv = [ + {file = "nodeenv-1.7.0-py2.py3-none-any.whl", hash = "sha256:27083a7b96a25f2f5e1d8cb4b6317ee8aeda3bdd121394e5ac54e498028a042e"}, + {file = "nodeenv-1.7.0.tar.gz", hash = "sha256:e0e7f7dfb85fc5394c6fe1e8fa98131a2473e04311a45afb6508f7cf1836fa2b"}, +] +numpy = [ + {file = "numpy-1.24.1-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:179a7ef0889ab769cc03573b6217f54c8bd8e16cef80aad369e1e8185f994cd7"}, + {file = "numpy-1.24.1-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:b09804ff570b907da323b3d762e74432fb07955701b17b08ff1b5ebaa8cfe6a9"}, + {file = "numpy-1.24.1-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:f1b739841821968798947d3afcefd386fa56da0caf97722a5de53e07c4ccedc7"}, + {file = "numpy-1.24.1-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:0e3463e6ac25313462e04aea3fb8a0a30fb906d5d300f58b3bc2c23da6a15398"}, + {file = "numpy-1.24.1-cp310-cp310-win32.whl", hash = "sha256:b31da69ed0c18be8b77bfce48d234e55d040793cebb25398e2a7d84199fbc7e2"}, + {file = "numpy-1.24.1-cp310-cp310-win_amd64.whl", hash = "sha256:b07b40f5fb4fa034120a5796288f24c1fe0e0580bbfff99897ba6267af42def2"}, + {file = "numpy-1.24.1-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:7094891dcf79ccc6bc2a1f30428fa5edb1e6fb955411ffff3401fb4ea93780a8"}, + {file = "numpy-1.24.1-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:28e418681372520c992805bb723e29d69d6b7aa411065f48216d8329d02ba032"}, + {file = "numpy-1.24.1-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:e274f0f6c7efd0d577744f52032fdd24344f11c5ae668fe8d01aac0422611df1"}, + {file = "numpy-1.24.1-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:0044f7d944ee882400890f9ae955220d29b33d809a038923d88e4e01d652acd9"}, + {file = "numpy-1.24.1-cp311-cp311-win32.whl", hash = "sha256:442feb5e5bada8408e8fcd43f3360b78683ff12a4444670a7d9e9824c1817d36"}, + {file = "numpy-1.24.1-cp311-cp311-win_amd64.whl", hash = "sha256:de92efa737875329b052982e37bd4371d52cabf469f83e7b8be9bb7752d67e51"}, + {file = "numpy-1.24.1-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:b162ac10ca38850510caf8ea33f89edcb7b0bb0dfa5592d59909419986b72407"}, + {file = "numpy-1.24.1-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:26089487086f2648944f17adaa1a97ca6aee57f513ba5f1c0b7ebdabbe2b9954"}, + {file = "numpy-1.24.1-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:caf65a396c0d1f9809596be2e444e3bd4190d86d5c1ce21f5fc4be60a3bc5b36"}, + {file = "numpy-1.24.1-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:b0677a52f5d896e84414761531947c7a330d1adc07c3a4372262f25d84af7bf7"}, + {file = "numpy-1.24.1-cp38-cp38-win32.whl", hash = "sha256:dae46bed2cb79a58d6496ff6d8da1e3b95ba09afeca2e277628171ca99b99db1"}, + {file = "numpy-1.24.1-cp38-cp38-win_amd64.whl", hash = "sha256:6ec0c021cd9fe732e5bab6401adea5a409214ca5592cd92a114f7067febcba0c"}, + {file = "numpy-1.24.1-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:28bc9750ae1f75264ee0f10561709b1462d450a4808cd97c013046073ae64ab6"}, + {file = "numpy-1.24.1-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:84e789a085aabef2f36c0515f45e459f02f570c4b4c4c108ac1179c34d475ed7"}, + {file = "numpy-1.24.1-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:8e669fbdcdd1e945691079c2cae335f3e3a56554e06bbd45d7609a6cf568c700"}, + {file = "numpy-1.24.1-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:ef85cf1f693c88c1fd229ccd1055570cb41cdf4875873b7728b6301f12cd05bf"}, + {file = "numpy-1.24.1-cp39-cp39-win32.whl", hash = "sha256:87a118968fba001b248aac90e502c0b13606721b1343cdaddbc6e552e8dfb56f"}, + {file = "numpy-1.24.1-cp39-cp39-win_amd64.whl", hash = "sha256:ddc7ab52b322eb1e40521eb422c4e0a20716c271a306860979d450decbb51b8e"}, + {file = "numpy-1.24.1-pp38-pypy38_pp73-macosx_10_9_x86_64.whl", hash = "sha256:ed5fb71d79e771ec930566fae9c02626b939e37271ec285e9efaf1b5d4370e7d"}, + {file = "numpy-1.24.1-pp38-pypy38_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:ad2925567f43643f51255220424c23d204024ed428afc5aad0f86f3ffc080086"}, + {file = "numpy-1.24.1-pp38-pypy38_pp73-win_amd64.whl", hash = "sha256:cfa1161c6ac8f92dea03d625c2d0c05e084668f4a06568b77a25a89111621566"}, + {file = "numpy-1.24.1.tar.gz", hash = "sha256:2386da9a471cc00a1f47845e27d916d5ec5346ae9696e01a8a34760858fe9dd2"}, +] +oauthlib = [ + {file = "oauthlib-3.2.2-py3-none-any.whl", hash = "sha256:8139f29aac13e25d502680e9e19963e83f16838d48a0d71c287fe40e7067fbca"}, + {file = "oauthlib-3.2.2.tar.gz", hash = "sha256:9859c40929662bec5d64f34d01c99e093149682a3f38915dc0655d5a633dd918"}, +] +packaging = [ + {file = "packaging-23.0-py3-none-any.whl", hash = "sha256:714ac14496c3e68c99c29b00845f7a2b85f3bb6f1078fd9f72fd20f0570002b2"}, + {file = "packaging-23.0.tar.gz", hash = "sha256:b6ad297f8907de0fa2fe1ccbd26fdaf387f5f47c7275fedf8cce89f99446cf97"}, +] +pandas = [ + {file = "pandas-1.5.3-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:3749077d86e3a2f0ed51367f30bf5b82e131cc0f14260c4d3e499186fccc4406"}, + {file = "pandas-1.5.3-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:972d8a45395f2a2d26733eb8d0f629b2f90bebe8e8eddbb8829b180c09639572"}, + {file = "pandas-1.5.3-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:50869a35cbb0f2e0cd5ec04b191e7b12ed688874bd05dd777c19b28cbea90996"}, + {file = "pandas-1.5.3-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:c3ac844a0fe00bfaeb2c9b51ab1424e5c8744f89860b138434a363b1f620f354"}, + {file = "pandas-1.5.3-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:7a0a56cef15fd1586726dace5616db75ebcfec9179a3a55e78f72c5639fa2a23"}, + {file = "pandas-1.5.3-cp310-cp310-win_amd64.whl", hash = "sha256:478ff646ca42b20376e4ed3fa2e8d7341e8a63105586efe54fa2508ee087f328"}, + {file = "pandas-1.5.3-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:6973549c01ca91ec96199e940495219c887ea815b2083722821f1d7abfa2b4dc"}, + {file = "pandas-1.5.3-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:c39a8da13cede5adcd3be1182883aea1c925476f4e84b2807a46e2775306305d"}, + {file = "pandas-1.5.3-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:f76d097d12c82a535fda9dfe5e8dd4127952b45fea9b0276cb30cca5ea313fbc"}, + {file = "pandas-1.5.3-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:e474390e60ed609cec869b0da796ad94f420bb057d86784191eefc62b65819ae"}, + {file = "pandas-1.5.3-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:5f2b952406a1588ad4cad5b3f55f520e82e902388a6d5a4a91baa8d38d23c7f6"}, + {file = "pandas-1.5.3-cp311-cp311-win_amd64.whl", hash = "sha256:bc4c368f42b551bf72fac35c5128963a171b40dce866fb066540eeaf46faa003"}, + {file = "pandas-1.5.3-cp38-cp38-macosx_10_9_universal2.whl", hash = "sha256:14e45300521902689a81f3f41386dc86f19b8ba8dd5ac5a3c7010ef8d2932813"}, + {file = "pandas-1.5.3-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:9842b6f4b8479e41968eced654487258ed81df7d1c9b7b870ceea24ed9459b31"}, + {file = "pandas-1.5.3-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:26d9c71772c7afb9d5046e6e9cf42d83dd147b5cf5bcb9d97252077118543792"}, + {file = "pandas-1.5.3-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:5fbcb19d6fceb9e946b3e23258757c7b225ba450990d9ed63ccceeb8cae609f7"}, + {file = "pandas-1.5.3-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:565fa34a5434d38e9d250af3c12ff931abaf88050551d9fbcdfafca50d62babf"}, + {file = "pandas-1.5.3-cp38-cp38-win32.whl", hash = "sha256:87bd9c03da1ac870a6d2c8902a0e1fd4267ca00f13bc494c9e5a9020920e1d51"}, + {file = "pandas-1.5.3-cp38-cp38-win_amd64.whl", hash = "sha256:41179ce559943d83a9b4bbacb736b04c928b095b5f25dd2b7389eda08f46f373"}, + {file = "pandas-1.5.3-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:c74a62747864ed568f5a82a49a23a8d7fe171d0c69038b38cedf0976831296fa"}, + {file = "pandas-1.5.3-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:c4c00e0b0597c8e4f59e8d461f797e5d70b4d025880516a8261b2817c47759ee"}, + {file = "pandas-1.5.3-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:a50d9a4336a9621cab7b8eb3fb11adb82de58f9b91d84c2cd526576b881a0c5a"}, + {file = "pandas-1.5.3-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:dd05f7783b3274aa206a1af06f0ceed3f9b412cf665b7247eacd83be41cf7bf0"}, + {file = "pandas-1.5.3-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:9f69c4029613de47816b1bb30ff5ac778686688751a5e9c99ad8c7031f6508e5"}, + {file = "pandas-1.5.3-cp39-cp39-win32.whl", hash = "sha256:7cec0bee9f294e5de5bbfc14d0573f65526071029d036b753ee6507d2a21480a"}, + {file = "pandas-1.5.3-cp39-cp39-win_amd64.whl", hash = "sha256:dfd681c5dc216037e0b0a2c821f5ed99ba9f03ebcf119c7dac0e9a7b960b9ec9"}, + {file = "pandas-1.5.3.tar.gz", hash = "sha256:74a3fd7e5a7ec052f183273dc7b0acd3a863edf7520f5d3a1765c04ffdb3b0b1"}, +] +platformdirs = [ + {file = "platformdirs-2.6.2-py3-none-any.whl", hash = "sha256:83c8f6d04389165de7c9b6f0c682439697887bca0aa2f1c87ef1826be3584490"}, + {file = "platformdirs-2.6.2.tar.gz", hash = "sha256:e1fea1fe471b9ff8332e229df3cb7de4f53eeea4998d3b6bfff542115e998bd2"}, +] +pluggy = [ + {file = "pluggy-1.0.0-py2.py3-none-any.whl", hash = "sha256:74134bbf457f031a36d68416e1509f34bd5ccc019f0bcc952c7b909d06b37bd3"}, + {file = "pluggy-1.0.0.tar.gz", hash = "sha256:4224373bacce55f955a878bf9cfa763c1e360858e330072059e10bad68531159"}, +] +portalocker = [ + {file = "portalocker-2.7.0-py2.py3-none-any.whl", hash = "sha256:a07c5b4f3985c3cf4798369631fb7011adb498e2a46d8440efc75a8f29a0f983"}, + {file = "portalocker-2.7.0.tar.gz", hash = "sha256:032e81d534a88ec1736d03f780ba073f047a06c478b06e2937486f334e955c51"}, +] +pre-commit = [ + {file = "pre_commit-2.21.0-py2.py3-none-any.whl", hash = "sha256:e2f91727039fc39a92f58a588a25b87f936de6567eed4f0e673e0507edc75bad"}, + {file = "pre_commit-2.21.0.tar.gz", hash = "sha256:31ef31af7e474a8d8995027fefdfcf509b5c913ff31f2015b4ec4beb26a6f658"}, +] +pyarrow = [ + {file = "pyarrow-10.0.1-cp310-cp310-macosx_10_14_x86_64.whl", hash = "sha256:e00174764a8b4e9d8d5909b6d19ee0c217a6cf0232c5682e31fdfbd5a9f0ae52"}, + {file = "pyarrow-10.0.1-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:6f7a7dbe2f7f65ac1d0bd3163f756deb478a9e9afc2269557ed75b1b25ab3610"}, + {file = "pyarrow-10.0.1-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:cb627673cb98708ef00864e2e243f51ba7b4c1b9f07a1d821f98043eccd3f585"}, + {file = "pyarrow-10.0.1-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:ba71e6fc348c92477586424566110d332f60d9a35cb85278f42e3473bc1373da"}, + {file = "pyarrow-10.0.1-cp310-cp310-win_amd64.whl", hash = "sha256:7b4ede715c004b6fc535de63ef79fa29740b4080639a5ff1ea9ca84e9282f349"}, + {file = "pyarrow-10.0.1-cp311-cp311-macosx_10_14_x86_64.whl", hash = "sha256:e3fe5049d2e9ca661d8e43fab6ad5a4c571af12d20a57dffc392a014caebef65"}, + {file = "pyarrow-10.0.1-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:254017ca43c45c5098b7f2a00e995e1f8346b0fb0be225f042838323bb55283c"}, + {file = "pyarrow-10.0.1-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:70acca1ece4322705652f48db65145b5028f2c01c7e426c5d16a30ba5d739c24"}, + {file = "pyarrow-10.0.1-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:abb57334f2c57979a49b7be2792c31c23430ca02d24becd0b511cbe7b6b08649"}, + {file = "pyarrow-10.0.1-cp311-cp311-win_amd64.whl", hash = "sha256:1765a18205eb1e02ccdedb66049b0ec148c2a0cb52ed1fb3aac322dfc086a6ee"}, + {file = "pyarrow-10.0.1-cp37-cp37m-macosx_10_14_x86_64.whl", hash = "sha256:61f4c37d82fe00d855d0ab522c685262bdeafd3fbcb5fe596fe15025fbc7341b"}, + {file = "pyarrow-10.0.1-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:e141a65705ac98fa52a9113fe574fdaf87fe0316cde2dffe6b94841d3c61544c"}, + {file = "pyarrow-10.0.1-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:bf26f809926a9d74e02d76593026f0aaeac48a65b64f1bb17eed9964bfe7ae1a"}, + {file = "pyarrow-10.0.1-cp37-cp37m-win_amd64.whl", hash = "sha256:443eb9409b0cf78df10ced326490e1a300205a458fbeb0767b6b31ab3ebae6b2"}, + {file = "pyarrow-10.0.1-cp38-cp38-macosx_10_14_x86_64.whl", hash = "sha256:f2d00aa481becf57098e85d99e34a25dba5a9ade2f44eb0b7d80c80f2984fc03"}, + {file = "pyarrow-10.0.1-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:b1fc226d28c7783b52a84d03a66573d5a22e63f8a24b841d5fc68caeed6784d4"}, + {file = "pyarrow-10.0.1-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:efa59933b20183c1c13efc34bd91efc6b2997377c4c6ad9272da92d224e3beb1"}, + {file = "pyarrow-10.0.1-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:668e00e3b19f183394388a687d29c443eb000fb3fe25599c9b4762a0afd37775"}, + {file = "pyarrow-10.0.1-cp38-cp38-win_amd64.whl", hash = "sha256:d1bc6e4d5d6f69e0861d5d7f6cf4d061cf1069cb9d490040129877acf16d4c2a"}, + {file = "pyarrow-10.0.1-cp39-cp39-macosx_10_14_x86_64.whl", hash = "sha256:42ba7c5347ce665338f2bc64685d74855900200dac81a972d49fe127e8132f75"}, + {file = "pyarrow-10.0.1-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:b069602eb1fc09f1adec0a7bdd7897f4d25575611dfa43543c8b8a75d99d6874"}, + {file = "pyarrow-10.0.1-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:94fb4a0c12a2ac1ed8e7e2aa52aade833772cf2d3de9dde685401b22cec30002"}, + {file = "pyarrow-10.0.1-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:db0c5986bf0808927f49640582d2032a07aa49828f14e51f362075f03747d198"}, + {file = "pyarrow-10.0.1-cp39-cp39-win_amd64.whl", hash = "sha256:0ec7587d759153f452d5263dbc8b1af318c4609b607be2bd5127dcda6708cdb1"}, + {file = "pyarrow-10.0.1.tar.gz", hash = "sha256:1a14f57a5f472ce8234f2964cd5184cccaa8df7e04568c64edc33b23eb285dd5"}, +] +pycparser = [ + {file = "pycparser-2.21-py2.py3-none-any.whl", hash = "sha256:8ee45429555515e1f6b185e78100aea234072576aa43ab53aefcae078162fca9"}, + {file = "pycparser-2.21.tar.gz", hash = "sha256:e644fdec12f7872f86c58ff790da456218b10f863970249516d60a5eaca77206"}, +] +pydantic = [ + {file = "pydantic-1.10.4-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:b5635de53e6686fe7a44b5cf25fcc419a0d5e5c1a1efe73d49d48fe7586db854"}, + {file = "pydantic-1.10.4-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:6dc1cc241440ed7ca9ab59d9929075445da6b7c94ced281b3dd4cfe6c8cff817"}, + {file = "pydantic-1.10.4-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:51bdeb10d2db0f288e71d49c9cefa609bca271720ecd0c58009bd7504a0c464c"}, + {file = "pydantic-1.10.4-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:78cec42b95dbb500a1f7120bdf95c401f6abb616bbe8785ef09887306792e66e"}, + {file = "pydantic-1.10.4-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:8775d4ef5e7299a2f4699501077a0defdaac5b6c4321173bcb0f3c496fbadf85"}, + {file = "pydantic-1.10.4-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:572066051eeac73d23f95ba9a71349c42a3e05999d0ee1572b7860235b850cc6"}, + {file = "pydantic-1.10.4-cp310-cp310-win_amd64.whl", hash = "sha256:7feb6a2d401f4d6863050f58325b8d99c1e56f4512d98b11ac64ad1751dc647d"}, + {file = "pydantic-1.10.4-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:39f4a73e5342b25c2959529f07f026ef58147249f9b7431e1ba8414a36761f53"}, + {file = "pydantic-1.10.4-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:983e720704431a6573d626b00662eb78a07148c9115129f9b4351091ec95ecc3"}, + {file = "pydantic-1.10.4-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:75d52162fe6b2b55964fbb0af2ee58e99791a3138588c482572bb6087953113a"}, + {file = "pydantic-1.10.4-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:fdf8d759ef326962b4678d89e275ffc55b7ce59d917d9f72233762061fd04a2d"}, + {file = "pydantic-1.10.4-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:05a81b006be15655b2a1bae5faa4280cf7c81d0e09fcb49b342ebf826abe5a72"}, + {file = "pydantic-1.10.4-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:d88c4c0e5c5dfd05092a4b271282ef0588e5f4aaf345778056fc5259ba098857"}, + {file = "pydantic-1.10.4-cp311-cp311-win_amd64.whl", hash = "sha256:6a05a9db1ef5be0fe63e988f9617ca2551013f55000289c671f71ec16f4985e3"}, + {file = "pydantic-1.10.4-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:887ca463c3bc47103c123bc06919c86720e80e1214aab79e9b779cda0ff92a00"}, + {file = "pydantic-1.10.4-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:fdf88ab63c3ee282c76d652fc86518aacb737ff35796023fae56a65ced1a5978"}, + {file = "pydantic-1.10.4-cp37-cp37m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:a48f1953c4a1d9bd0b5167ac50da9a79f6072c63c4cef4cf2a3736994903583e"}, + {file = "pydantic-1.10.4-cp37-cp37m-musllinux_1_1_i686.whl", hash = "sha256:a9f2de23bec87ff306aef658384b02aa7c32389766af3c5dee9ce33e80222dfa"}, + {file = "pydantic-1.10.4-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:cd8702c5142afda03dc2b1ee6bc358b62b3735b2cce53fc77b31ca9f728e4bc8"}, + {file = "pydantic-1.10.4-cp37-cp37m-win_amd64.whl", hash = "sha256:6e7124d6855b2780611d9f5e1e145e86667eaa3bd9459192c8dc1a097f5e9903"}, + {file = "pydantic-1.10.4-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:0b53e1d41e97063d51a02821b80538053ee4608b9a181c1005441f1673c55423"}, + {file = "pydantic-1.10.4-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:55b1625899acd33229c4352ce0ae54038529b412bd51c4915349b49ca575258f"}, + {file = "pydantic-1.10.4-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:301d626a59edbe5dfb48fcae245896379a450d04baeed50ef40d8199f2733b06"}, + {file = "pydantic-1.10.4-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:b6f9d649892a6f54a39ed56b8dfd5e08b5f3be5f893da430bed76975f3735d15"}, + {file = "pydantic-1.10.4-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:d7b5a3821225f5c43496c324b0d6875fde910a1c2933d726a743ce328fbb2a8c"}, + {file = "pydantic-1.10.4-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:f2f7eb6273dd12472d7f218e1fef6f7c7c2f00ac2e1ecde4db8824c457300416"}, + {file = "pydantic-1.10.4-cp38-cp38-win_amd64.whl", hash = "sha256:4b05697738e7d2040696b0a66d9f0a10bec0efa1883ca75ee9e55baf511909d6"}, + {file = "pydantic-1.10.4-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:a9a6747cac06c2beb466064dda999a13176b23535e4c496c9d48e6406f92d42d"}, + {file = "pydantic-1.10.4-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:eb992a1ef739cc7b543576337bebfc62c0e6567434e522e97291b251a41dad7f"}, + {file = "pydantic-1.10.4-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:990406d226dea0e8f25f643b370224771878142155b879784ce89f633541a024"}, + {file = "pydantic-1.10.4-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:2e82a6d37a95e0b1b42b82ab340ada3963aea1317fd7f888bb6b9dfbf4fff57c"}, + {file = "pydantic-1.10.4-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:9193d4f4ee8feca58bc56c8306bcb820f5c7905fd919e0750acdeeeef0615b28"}, + {file = "pydantic-1.10.4-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:2b3ce5f16deb45c472dde1a0ee05619298c864a20cded09c4edd820e1454129f"}, + {file = "pydantic-1.10.4-cp39-cp39-win_amd64.whl", hash = "sha256:9cbdc268a62d9a98c56e2452d6c41c0263d64a2009aac69246486f01b4f594c4"}, + {file = "pydantic-1.10.4-py3-none-any.whl", hash = "sha256:4948f264678c703f3877d1c8877c4e3b2e12e549c57795107f08cf70c6ec7774"}, + {file = "pydantic-1.10.4.tar.gz", hash = "sha256:b9a3859f24eb4e097502a3be1fb4b2abb79b6103dd9e2e0edb70613a4459a648"}, +] +pygments = [ + {file = "Pygments-2.14.0-py3-none-any.whl", hash = "sha256:fa7bd7bd2771287c0de303af8bfdfc731f51bd2c6a47ab69d117138893b82717"}, + {file = "Pygments-2.14.0.tar.gz", hash = "sha256:b3ed06a9e8ac9a9aae5a6f5dbe78a8a58655d17b43b93c078f094ddc476ae297"}, +] +pyjwt = [ + {file = "PyJWT-2.6.0-py3-none-any.whl", hash = "sha256:d83c3d892a77bbb74d3e1a2cfa90afaadb60945205d1095d9221f04466f64c14"}, + {file = "PyJWT-2.6.0.tar.gz", hash = "sha256:69285c7e31fc44f68a1feb309e948e0df53259d579295e6cfe2b1792329f05fd"}, +] +pyparsing = [ + {file = "pyparsing-3.0.9-py3-none-any.whl", hash = "sha256:5026bae9a10eeaefb61dab2f09052b9f4307d44aee4eda64b309723d8d206bbc"}, + {file = "pyparsing-3.0.9.tar.gz", hash = "sha256:2b020ecf7d21b687f219b71ecad3631f644a47f01403fa1d1036b0c6416d70fb"}, +] +pyproject-hooks = [ + {file = "pyproject_hooks-1.0.0-py3-none-any.whl", hash = "sha256:283c11acd6b928d2f6a7c73fa0d01cb2bdc5f07c57a2eeb6e83d5e56b97976f8"}, + {file = "pyproject_hooks-1.0.0.tar.gz", hash = "sha256:f271b298b97f5955d53fb12b72c1fb1948c22c1a6b70b315c54cedaca0264ef5"}, +] +pytest = [ + {file = "pytest-7.2.1-py3-none-any.whl", hash = "sha256:c7c6ca206e93355074ae32f7403e8ea12163b1163c976fee7d4d84027c162be5"}, + {file = "pytest-7.2.1.tar.gz", hash = "sha256:d45e0952f3727241918b8fd0f376f5ff6b301cc0777c6f9a556935c92d8a7d42"}, +] +pytest-checkdocs = [ + {file = "pytest-checkdocs-2.9.0.tar.gz", hash = "sha256:54b3d1357a289e3f6336d31d4ceb72e99afe0fa4f552990f1b1f015f1925543b"}, + {file = "pytest_checkdocs-2.9.0-py3-none-any.whl", hash = "sha256:4231e5426915a02385e386b5eb4a70490b143d97cede4c7c77691bb320b5a73f"}, +] +python-dateutil = [] +python-snappy = [ + {file = "python-snappy-0.6.1.tar.gz", hash = "sha256:b6a107ab06206acc5359d4c5632bd9b22d448702a79b3169b0c62e0fb808bb2a"}, + {file = "python_snappy-0.6.1-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:b7f920eaf46ebf41bd26f9df51c160d40f9e00b7b48471c3438cb8d027f7fb9b"}, + {file = "python_snappy-0.6.1-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:4ec533a8c1f8df797bded662ec3e494d225b37855bb63eb0d75464a07947477c"}, + {file = "python_snappy-0.6.1-cp310-cp310-manylinux_2_12_i686.manylinux2010_i686.whl", hash = "sha256:6f8bf4708a11b47517baf962f9a02196478bbb10fdb9582add4aa1459fa82380"}, + {file = "python_snappy-0.6.1-cp310-cp310-manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:8d0c019ee7dcf2c60e240877107cddbd95a5b1081787579bf179938392d66480"}, + {file = "python_snappy-0.6.1-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:cb18d9cd7b3f35a2f5af47bb8ed6a5bdbf4f3ddee37f3daade4ab7864c292f5b"}, + {file = "python_snappy-0.6.1-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:b265cde49774752aec9ca7f5d272e3f98718164afc85521622a8a5394158a2b5"}, + {file = "python_snappy-0.6.1-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:d017775851a778ec9cc32651c4464079d06d927303c2dde9ae9830ccf6fe94e1"}, + {file = "python_snappy-0.6.1-cp310-cp310-win32.whl", hash = "sha256:8277d1f6282463c40761f802b742f833f9f2449fcdbb20a96579aa05c8feb614"}, + {file = "python_snappy-0.6.1-cp310-cp310-win_amd64.whl", hash = "sha256:2aaaf618c68d8c9daebc23a20436bd01b09ee70d7fbf7072b7f38b06d2fab539"}, + {file = "python_snappy-0.6.1-cp36-cp36m-macosx_10_9_x86_64.whl", hash = "sha256:277757d5dad4e239dc1417438a0871b65b1b155beb108888e7438c27ffc6a8cc"}, + {file = "python_snappy-0.6.1-cp36-cp36m-manylinux_2_12_i686.manylinux2010_i686.whl", hash = "sha256:e066a0586833d610c4bbddba0be5ba0e3e4f8e0bc5bb6d82103d8f8fc47bb59a"}, + {file = "python_snappy-0.6.1-cp36-cp36m-manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:0d489b50f49433494160c45048fe806de6b3aeab0586e497ebd22a0bab56e427"}, + {file = "python_snappy-0.6.1-cp36-cp36m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:463fd340a499d47b26ca42d2f36a639188738f6e2098c6dbf80aef0e60f461e1"}, + {file = "python_snappy-0.6.1-cp36-cp36m-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:9837ac1650cc68d22a3cf5f15fb62c6964747d16cecc8b22431f113d6e39555d"}, + {file = "python_snappy-0.6.1-cp36-cp36m-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:5e973e637112391f05581f427659c05b30b6843bc522a65be35ac7b18ce3dedd"}, + {file = "python_snappy-0.6.1-cp36-cp36m-win32.whl", hash = "sha256:c20498bd712b6e31a4402e1d027a1cd64f6a4a0066a3fe3c7344475886d07fdf"}, + {file = "python_snappy-0.6.1-cp36-cp36m-win_amd64.whl", hash = "sha256:59e975be4206cc54d0a112ef72fa3970a57c2b1bcc2c97ed41d6df0ebe518228"}, + {file = "python_snappy-0.6.1-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:2a7e528ab6e09c0d67dcb61a1730a292683e5ff9bb088950638d3170cf2a0a54"}, + {file = "python_snappy-0.6.1-cp37-cp37m-manylinux_2_12_i686.manylinux2010_i686.whl", hash = "sha256:39692bedbe0b717001a99915ac0eb2d9d0bad546440d392a2042b96d813eede1"}, + {file = "python_snappy-0.6.1-cp37-cp37m-manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:6a7620404da966f637b9ce8d4d3d543d363223f7a12452a575189c5355fc2d25"}, + {file = "python_snappy-0.6.1-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:7778c224efc38a40d274da4eb82a04cac27aae20012372a7db3c4bbd8926c4d4"}, + {file = "python_snappy-0.6.1-cp37-cp37m-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:1d029f7051ec1bbeaa3e03030b6d8ed47ceb69cae9016f493c802a08af54e026"}, + {file = "python_snappy-0.6.1-cp37-cp37m-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:a0ad38bc98d0b0497a0b0dbc29409bcabfcecff4511ed7063403c86de16927bc"}, + {file = "python_snappy-0.6.1-cp37-cp37m-win32.whl", hash = "sha256:5a453c45178d7864c1bdd6bfe0ee3ed2883f63b9ba2c9bb967c6b586bf763f96"}, + {file = "python_snappy-0.6.1-cp37-cp37m-win_amd64.whl", hash = "sha256:9f0c0d88b84259f93c3aa46398680646f2c23e43394779758d9f739c34e15295"}, + {file = "python_snappy-0.6.1-cp38-cp38-macosx_10_9_universal2.whl", hash = "sha256:5bb05c28298803a74add08ba496879242ef159c75bc86a5406fac0ffc7dd021b"}, + {file = "python_snappy-0.6.1-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:9eac51307c6a1a38d5f86ebabc26a889fddf20cbba7a116ccb54ba1446601d5b"}, + {file = "python_snappy-0.6.1-cp38-cp38-manylinux_2_12_i686.manylinux2010_i686.whl", hash = "sha256:88b6ea78b83d2796f330b0af1b70cdd3965dbdab02d8ac293260ec2c8fe340ee"}, + {file = "python_snappy-0.6.1-cp38-cp38-manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:8c07220408d3268e8268c9351c5c08041bc6f8c6172e59d398b71020df108541"}, + {file = "python_snappy-0.6.1-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:4038019b1bcaadde726a57430718394076c5a21545ebc5badad2c045a09546cf"}, + {file = "python_snappy-0.6.1-cp38-cp38-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:dc96668d9c7cc656609764275c5f8da58ef56d89bdd6810f6923d36497468ff7"}, + {file = "python_snappy-0.6.1-cp38-cp38-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:cf5bb9254e1c38aacf253d510d3d9be631bba21f3d068b17672b38b5cbf2fff5"}, + {file = "python_snappy-0.6.1-cp38-cp38-win32.whl", hash = "sha256:eaf905a580f2747c4a474040a5063cd5e0cc3d1d2d6edb65f28196186493ad4a"}, + {file = "python_snappy-0.6.1-cp38-cp38-win_amd64.whl", hash = "sha256:546c1a7470ecbf6239101e9aff0f709b68ca0f0268b34d9023019a55baa1f7c6"}, + {file = "python_snappy-0.6.1-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:e3a013895c64352b49d0d8e107a84f99631b16dbab156ded33ebf0becf56c8b2"}, + {file = "python_snappy-0.6.1-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:3fb9a88a4dd6336488f3de67ce75816d0d796dce53c2c6e4d70e0b565633c7fd"}, + {file = "python_snappy-0.6.1-cp39-cp39-manylinux_2_12_i686.manylinux2010_i686.whl", hash = "sha256:735cd4528c55dbe4516d6d2b403331a99fc304f8feded8ae887cf97b67d589bb"}, + {file = "python_snappy-0.6.1-cp39-cp39-manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:90b0186516b7a101c14764b0c25931b741fb0102f21253eff67847b4742dfc72"}, + {file = "python_snappy-0.6.1-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:1a993dc8aadd901915a510fe6af5f20ae4256f527040066c22a154db8946751f"}, + {file = "python_snappy-0.6.1-cp39-cp39-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:530bfb9efebcc1aab8bb4ebcbd92b54477eed11f6cf499355e882970a6d3aa7d"}, + {file = "python_snappy-0.6.1-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:5843feb914796b1f0405ccf31ea0fb51034ceb65a7588edfd5a8250cb369e3b2"}, + {file = "python_snappy-0.6.1-cp39-cp39-win32.whl", hash = "sha256:66c80e9b366012dbee262bb1869e4fc5ba8786cda85928481528bc4a72ec2ee8"}, + {file = "python_snappy-0.6.1-cp39-cp39-win_amd64.whl", hash = "sha256:4d3cafdf454354a621c8ab7408e45aa4e9d5c0b943b61ff4815f71ca6bdf0130"}, + {file = "python_snappy-0.6.1-pp37-pypy37_pp73-manylinux_2_12_i686.manylinux2010_i686.whl", hash = "sha256:586724a0276d7a6083a17259d0b51622e492289a9998848a1b01b6441ca12b2f"}, + {file = "python_snappy-0.6.1-pp37-pypy37_pp73-manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:2be4f4550acd484912441f5f1209ba611ac399aac9355fee73611b9a0d4f949c"}, + {file = "python_snappy-0.6.1-pp37-pypy37_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:0bdb6942180660bda7f7d01f4c0def3cfc72b1c6d99aad964801775a3e379aba"}, + {file = "python_snappy-0.6.1-pp37-pypy37_pp73-win_amd64.whl", hash = "sha256:03bb511380fca2a13325b6f16fe8234c8e12da9660f0258cd45d9a02ffc916af"}, +] +pytz = [ + {file = "pytz-2022.7.1-py2.py3-none-any.whl", hash = "sha256:78f4f37d8198e0627c5f1143240bb0206b8691d8d7ac6d78fee88b78733f8c4a"}, + {file = "pytz-2022.7.1.tar.gz", hash = "sha256:01a0681c4b9684a28304615eba55d1ab31ae00bf68ec157ec3708a8182dbbcd0"}, +] +pywin32 = [ + {file = "pywin32-305-cp310-cp310-win32.whl", hash = "sha256:421f6cd86e84bbb696d54563c48014b12a23ef95a14e0bdba526be756d89f116"}, + {file = "pywin32-305-cp310-cp310-win_amd64.whl", hash = "sha256:73e819c6bed89f44ff1d690498c0a811948f73777e5f97c494c152b850fad478"}, + {file = "pywin32-305-cp310-cp310-win_arm64.whl", hash = "sha256:742eb905ce2187133a29365b428e6c3b9001d79accdc30aa8969afba1d8470f4"}, + {file = "pywin32-305-cp311-cp311-win32.whl", hash = "sha256:19ca459cd2e66c0e2cc9a09d589f71d827f26d47fe4a9d09175f6aa0256b51c2"}, + {file = "pywin32-305-cp311-cp311-win_amd64.whl", hash = "sha256:326f42ab4cfff56e77e3e595aeaf6c216712bbdd91e464d167c6434b28d65990"}, + {file = "pywin32-305-cp311-cp311-win_arm64.whl", hash = "sha256:4ecd404b2c6eceaca52f8b2e3e91b2187850a1ad3f8b746d0796a98b4cea04db"}, + {file = "pywin32-305-cp36-cp36m-win32.whl", hash = "sha256:48d8b1659284f3c17b68587af047d110d8c44837736b8932c034091683e05863"}, + {file = "pywin32-305-cp36-cp36m-win_amd64.whl", hash = "sha256:13362cc5aa93c2beaf489c9c9017c793722aeb56d3e5166dadd5ef82da021fe1"}, + {file = "pywin32-305-cp37-cp37m-win32.whl", hash = "sha256:a55db448124d1c1484df22fa8bbcbc45c64da5e6eae74ab095b9ea62e6d00496"}, + {file = "pywin32-305-cp37-cp37m-win_amd64.whl", hash = "sha256:109f98980bfb27e78f4df8a51a8198e10b0f347257d1e265bb1a32993d0c973d"}, + {file = "pywin32-305-cp38-cp38-win32.whl", hash = "sha256:9dd98384da775afa009bc04863426cb30596fd78c6f8e4e2e5bbf4edf8029504"}, + {file = "pywin32-305-cp38-cp38-win_amd64.whl", hash = "sha256:56d7a9c6e1a6835f521788f53b5af7912090674bb84ef5611663ee1595860fc7"}, + {file = "pywin32-305-cp39-cp39-win32.whl", hash = "sha256:9d968c677ac4d5cbdaa62fd3014ab241718e619d8e36ef8e11fb930515a1e918"}, + {file = "pywin32-305-cp39-cp39-win_amd64.whl", hash = "sha256:50768c6b7c3f0b38b7fb14dd4104da93ebced5f1a50dc0e834594bff6fbe1271"}, +] +pyyaml = [ + {file = "PyYAML-6.0-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:d4db7c7aef085872ef65a8fd7d6d09a14ae91f691dec3e87ee5ee0539d516f53"}, + {file = "PyYAML-6.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:9df7ed3b3d2e0ecfe09e14741b857df43adb5a3ddadc919a2d94fbdf78fea53c"}, + {file = "PyYAML-6.0-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:77f396e6ef4c73fdc33a9157446466f1cff553d979bd00ecb64385760c6babdc"}, + {file = "PyYAML-6.0-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:a80a78046a72361de73f8f395f1f1e49f956c6be882eed58505a15f3e430962b"}, + {file = "PyYAML-6.0-cp310-cp310-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:f84fbc98b019fef2ee9a1cb3ce93e3187a6df0b2538a651bfb890254ba9f90b5"}, + {file = "PyYAML-6.0-cp310-cp310-win32.whl", hash = "sha256:2cd5df3de48857ed0544b34e2d40e9fac445930039f3cfe4bcc592a1f836d513"}, + {file = "PyYAML-6.0-cp310-cp310-win_amd64.whl", hash = "sha256:daf496c58a8c52083df09b80c860005194014c3698698d1a57cbcfa182142a3a"}, + {file = "PyYAML-6.0-cp36-cp36m-macosx_10_9_x86_64.whl", hash = "sha256:897b80890765f037df3403d22bab41627ca8811ae55e9a722fd0392850ec4d86"}, + {file = "PyYAML-6.0-cp36-cp36m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:50602afada6d6cbfad699b0c7bb50d5ccffa7e46a3d738092afddc1f9758427f"}, + {file = "PyYAML-6.0-cp36-cp36m-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:48c346915c114f5fdb3ead70312bd042a953a8ce5c7106d5bfb1a5254e47da92"}, + {file = "PyYAML-6.0-cp36-cp36m-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:98c4d36e99714e55cfbaaee6dd5badbc9a1ec339ebfc3b1f52e293aee6bb71a4"}, + {file = "PyYAML-6.0-cp36-cp36m-win32.whl", hash = "sha256:0283c35a6a9fbf047493e3a0ce8d79ef5030852c51e9d911a27badfde0605293"}, + {file = "PyYAML-6.0-cp36-cp36m-win_amd64.whl", hash = "sha256:07751360502caac1c067a8132d150cf3d61339af5691fe9e87803040dbc5db57"}, + {file = "PyYAML-6.0-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:819b3830a1543db06c4d4b865e70ded25be52a2e0631ccd2f6a47a2822f2fd7c"}, + {file = "PyYAML-6.0-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:473f9edb243cb1935ab5a084eb238d842fb8f404ed2193a915d1784b5a6b5fc0"}, + {file = "PyYAML-6.0-cp37-cp37m-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:0ce82d761c532fe4ec3f87fc45688bdd3a4c1dc5e0b4a19814b9009a29baefd4"}, + {file = "PyYAML-6.0-cp37-cp37m-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:231710d57adfd809ef5d34183b8ed1eeae3f76459c18fb4a0b373ad56bedcdd9"}, + {file = "PyYAML-6.0-cp37-cp37m-win32.whl", hash = "sha256:c5687b8d43cf58545ade1fe3e055f70eac7a5a1a0bf42824308d868289a95737"}, + {file = "PyYAML-6.0-cp37-cp37m-win_amd64.whl", hash = "sha256:d15a181d1ecd0d4270dc32edb46f7cb7733c7c508857278d3d378d14d606db2d"}, + {file = "PyYAML-6.0-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:0b4624f379dab24d3725ffde76559cff63d9ec94e1736b556dacdfebe5ab6d4b"}, + {file = "PyYAML-6.0-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:213c60cd50106436cc818accf5baa1aba61c0189ff610f64f4a3e8c6726218ba"}, + {file = "PyYAML-6.0-cp38-cp38-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:9fa600030013c4de8165339db93d182b9431076eb98eb40ee068700c9c813e34"}, + {file = "PyYAML-6.0-cp38-cp38-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:277a0ef2981ca40581a47093e9e2d13b3f1fbbeffae064c1d21bfceba2030287"}, + {file = "PyYAML-6.0-cp38-cp38-win32.whl", hash = "sha256:d4eccecf9adf6fbcc6861a38015c2a64f38b9d94838ac1810a9023a0609e1b78"}, + {file = "PyYAML-6.0-cp38-cp38-win_amd64.whl", hash = "sha256:1e4747bc279b4f613a09eb64bba2ba602d8a6664c6ce6396a4d0cd413a50ce07"}, + {file = "PyYAML-6.0-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:055d937d65826939cb044fc8c9b08889e8c743fdc6a32b33e2390f66013e449b"}, + {file = "PyYAML-6.0-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:e61ceaab6f49fb8bdfaa0f92c4b57bcfbea54c09277b1b4f7ac376bfb7a7c174"}, + {file = "PyYAML-6.0-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:d67d839ede4ed1b28a4e8909735fc992a923cdb84e618544973d7dfc71540803"}, + {file = "PyYAML-6.0-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:cba8c411ef271aa037d7357a2bc8f9ee8b58b9965831d9e51baf703280dc73d3"}, + {file = "PyYAML-6.0-cp39-cp39-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:40527857252b61eacd1d9af500c3337ba8deb8fc298940291486c465c8b46ec0"}, + {file = "PyYAML-6.0-cp39-cp39-win32.whl", hash = "sha256:b5b9eccad747aabaaffbc6064800670f0c297e52c12754eb1d976c57e4f74dcb"}, + {file = "PyYAML-6.0-cp39-cp39-win_amd64.whl", hash = "sha256:b3d267842bf12586ba6c734f89d1f5b871df0273157918b0ccefa29deb05c21c"}, + {file = "PyYAML-6.0.tar.gz", hash = "sha256:68fb519c14306fec9720a2a5b45bc9f0c8d1b9c72adf45c37baedfcd949c35a2"}, +] +requests = [ + {file = "requests-2.28.2-py3-none-any.whl", hash = "sha256:64299f4909223da747622c030b781c0d7811e359c37124b4bd368fb8c6518baa"}, + {file = "requests-2.28.2.tar.gz", hash = "sha256:98b1b2782e3c6c4904938b84c0eb932721069dfdb9134313beff7c83c2df24bf"}, +] +requests-mock = [ + {file = "requests-mock-1.10.0.tar.gz", hash = "sha256:59c9c32419a9fb1ae83ec242d98e889c45bd7d7a65d48375cc243ec08441658b"}, + {file = "requests_mock-1.10.0-py2.py3-none-any.whl", hash = "sha256:2fdbb637ad17ee15c06f33d31169e71bf9fe2bdb7bc9da26185be0dd8d842699"}, +] +requests-oauthlib = [ + {file = "requests-oauthlib-1.3.1.tar.gz", hash = "sha256:75beac4a47881eeb94d5ea5d6ad31ef88856affe2332b9aafb52c6452ccf0d7a"}, + {file = "requests_oauthlib-1.3.1-py2.py3-none-any.whl", hash = "sha256:2577c501a2fb8d05a304c09d090d6e47c306fef15809d102b327cf8364bddab5"}, +] +responses = [ + {file = "responses-0.22.0-py3-none-any.whl", hash = "sha256:dcf294d204d14c436fddcc74caefdbc5764795a40ff4e6a7740ed8ddbf3294be"}, + {file = "responses-0.22.0.tar.gz", hash = "sha256:396acb2a13d25297789a5866b4881cf4e46ffd49cc26c43ab1117f40b973102e"}, +] +rich = [ + {file = "rich-13.2.0-py3-none-any.whl", hash = "sha256:7c963f0d03819221e9ac561e1bc866e3f95a02248c1234daa48954e6d381c003"}, + {file = "rich-13.2.0.tar.gz", hash = "sha256:f1a00cdd3eebf999a15d85ec498bfe0b1a77efe9b34f645768a54132ef444ac5"}, +] +s3fs = [ + {file = "s3fs-2023.1.0-py3-none-any.whl", hash = "sha256:a549ae518fff4388bd6fca5248575c29f521e7e39efcd2bfab476651701fd114"}, + {file = "s3fs-2023.1.0.tar.gz", hash = "sha256:8b2e28372423e93f26312208a9272e22a962ddd0a79d63f9a68693b6af5ff187"}, +] +s3transfer = [] +six = [ + {file = "six-1.16.0-py2.py3-none-any.whl", hash = "sha256:8abb2f1d86890a2dfb989f9a77cfcfd3e47c2a354b01111771326f8aa26e0254"}, + {file = "six-1.16.0.tar.gz", hash = "sha256:1e61c37477a1626458e36f7b1d82aa5c9b094fa4802892072e49de9c60c4c926"}, +] +thrift = [] +toml = [ + {file = "toml-0.10.2-py2.py3-none-any.whl", hash = "sha256:806143ae5bfb6a3c6e736a764057db0e6a0e05e338b5630894a5f779cabb4f9b"}, + {file = "toml-0.10.2.tar.gz", hash = "sha256:b3bda1d108d5dd99f4a20d24d9c348e91c4db7ab1b749200bded2f839ccbe68f"}, +] +tomli = [ + {file = "tomli-2.0.1-py3-none-any.whl", hash = "sha256:939de3e7a6161af0c887ef91b7d41a53e7c5a1ca976325f429cb46ea9bc30ecc"}, + {file = "tomli-2.0.1.tar.gz", hash = "sha256:de526c12914f0c550d15924c62d72abc48d6fe7364aa87328337a31007fe8a4f"}, +] +types-toml = [ + {file = "types-toml-0.10.8.1.tar.gz", hash = "sha256:171bdb3163d79a520560f24ba916a9fc9bff81659c5448a9fea89240923722be"}, + {file = "types_toml-0.10.8.1-py3-none-any.whl", hash = "sha256:b7b5c4977f96ab7b5ac06d8a6590d17c0bf252a96efc03b109c2711fb3e0eafd"}, +] +typing-extensions = [ + {file = "typing_extensions-4.4.0-py3-none-any.whl", hash = "sha256:16fa4864408f655d35ec496218b85f79b3437c829e93320c7c9215ccfd92489e"}, + {file = "typing_extensions-4.4.0.tar.gz", hash = "sha256:1511434bb92bf8dd198c12b1cc812e800d4181cfcb867674e0f8279cc93087aa"}, +] +urllib3 = [ + {file = "urllib3-1.26.14-py2.py3-none-any.whl", hash = "sha256:75edcdc2f7d85b137124a6c3c9fc3933cdeaa12ecb9a6a959f22797a0feca7e1"}, + {file = "urllib3-1.26.14.tar.gz", hash = "sha256:076907bf8fd355cde77728471316625a4d2f7e713c125f51953bb5b3eecf4f72"}, +] +virtualenv = [ {file = "virtualenv-20.17.1-py3-none-any.whl", hash = "sha256:ce3b1684d6e1a20a3e5ed36795a97dfc6af29bc3970ca8dab93e11ac6094b3c4"}, {file = "virtualenv-20.17.1.tar.gz", hash = "sha256:f8b927684efc6f1cc206c9db297a570ab9ad0e51c16fa9e45487d36d1905c058"}, ] - -[package.dependencies] -distlib = ">=0.3.6,<1" -filelock = ">=3.4.1,<4" -platformdirs = ">=2.4,<3" - -[package.extras] -docs = ["proselint (>=0.13)", "sphinx (>=5.3)", "sphinx-argparse (>=0.3.2)", "sphinx-rtd-theme (>=1)", "towncrier (>=22.8)"] -testing = ["coverage (>=6.2)", "coverage-enable-subprocess (>=1)", "flaky (>=3.7)", "packaging (>=21.3)", "pytest (>=7.0.1)", "pytest-env (>=0.6.2)", "pytest-freezegun (>=0.4.2)", "pytest-mock (>=3.6.1)", "pytest-randomly (>=3.10.3)", "pytest-timeout (>=2.1)"] - -[[package]] -name = "werkzeug" -version = "2.2.2" -description = "The comprehensive WSGI web application library." -category = "dev" -optional = false -python-versions = ">=3.7" -files = [ +werkzeug = [ {file = "Werkzeug-2.2.2-py3-none-any.whl", hash = "sha256:f979ab81f58d7318e064e99c4506445d60135ac5cd2e177a2de0089bfd4c9bd5"}, {file = "Werkzeug-2.2.2.tar.gz", hash = "sha256:7ea2d48322cc7c0f8b3a215ed73eabd7b5d75d0b50e31ab006286ccff9e00b8f"}, ] - -[package.dependencies] -MarkupSafe = ">=2.1.1" - -[package.extras] -watchdog = ["watchdog"] - -[[package]] -name = "wrapt" -version = "1.14.1" -description = "Module for decorators, wrappers and monkey patching." -category = "main" -optional = true -python-versions = "!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,!=3.4.*,>=2.7" -files = [ +wrapt = [ {file = "wrapt-1.14.1-cp27-cp27m-macosx_10_9_x86_64.whl", hash = "sha256:1b376b3f4896e7930f1f772ac4b064ac12598d1c38d04907e696cc4d794b43d3"}, {file = "wrapt-1.14.1-cp27-cp27m-manylinux1_i686.whl", hash = "sha256:903500616422a40a98a5a3c4ff4ed9d0066f3b4c951fa286018ecdf0750194ef"}, {file = "wrapt-1.14.1-cp27-cp27m-manylinux1_x86_64.whl", hash = "sha256:5a9a0d155deafd9448baff28c08e150d9b24ff010e899311ddd63c45c2445e28"}, @@ -2264,27 +2287,11 @@ files = [ {file = "wrapt-1.14.1-cp39-cp39-win_amd64.whl", hash = "sha256:dee60e1de1898bde3b238f18340eec6148986da0455d8ba7848d50470a7a32fb"}, {file = "wrapt-1.14.1.tar.gz", hash = "sha256:380a85cf89e0e69b7cfbe2ea9f765f004ff419f34194018a6827ac0e3edfed4d"}, ] - -[[package]] -name = "xmltodict" -version = "0.13.0" -description = "Makes working with XML feel like you are working with JSON" -category = "dev" -optional = false -python-versions = ">=3.4" -files = [ +xmltodict = [ {file = "xmltodict-0.13.0-py2.py3-none-any.whl", hash = "sha256:aa89e8fd76320154a40d19a0df04a4695fb9dc5ba977cbb68ab3e4eb225e7852"}, {file = "xmltodict-0.13.0.tar.gz", hash = "sha256:341595a488e3e01a85a9d8911d8912fd922ede5fecc4dce437eb4b6c8d037e56"}, ] - -[[package]] -name = "yarl" -version = "1.8.2" -description = "Yet another URL library" -category = "main" -optional = true -python-versions = ">=3.7" -files = [ +yarl = [ {file = "yarl-1.8.2-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:bb81f753c815f6b8e2ddd2eef3c855cf7da193b82396ac013c661aaa6cc6b0a5"}, {file = "yarl-1.8.2-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:47d49ac96156f0928f002e2424299b2c91d9db73e08c4cd6742923a086f1c863"}, {file = "yarl-1.8.2-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:3fc056e35fa6fba63248d93ff6e672c096f95f7836938241ebc8260e062832fe"}, @@ -2360,35 +2367,11 @@ files = [ {file = "yarl-1.8.2-cp39-cp39-win_amd64.whl", hash = "sha256:6604711362f2dbf7160df21c416f81fac0de6dbcf0b5445a2ef25478ecc4c778"}, {file = "yarl-1.8.2.tar.gz", hash = "sha256:49d43402c6e3013ad0978602bf6bf5328535c48d192304b91b97a3c6790b1562"}, ] - -[package.dependencies] -idna = ">=2.0" -multidict = ">=4.0" - -[[package]] -name = "zipp" -version = "3.11.0" -description = "Backport of pathlib-compatible object wrapper for zip files" -category = "dev" -optional = false -python-versions = ">=3.7" -files = [ +zipp = [ {file = "zipp-3.11.0-py3-none-any.whl", hash = "sha256:83a28fcb75844b5c0cdaf5aa4003c2d728c77e05f5aeabe8e95e56727005fbaa"}, {file = "zipp-3.11.0.tar.gz", hash = "sha256:a7a22e05929290a67401440b39690ae6563279bced5f314609d9d03798f56766"}, ] - -[package.extras] -docs = ["furo", "jaraco.packaging (>=9)", "jaraco.tidelift (>=1.4)", "rst.linker (>=1.9)", "sphinx (>=3.5)"] -testing = ["flake8 (<5)", "func-timeout", "jaraco.functools", "jaraco.itertools", "more-itertools", "pytest (>=6)", "pytest-black (>=0.3.7)", "pytest-checkdocs (>=2.4)", "pytest-cov", "pytest-enabler (>=1.3)", "pytest-flake8", "pytest-mypy (>=0.9.1)"] - -[[package]] -name = "zstandard" -version = "0.19.0" -description = "Zstandard bindings for Python" -category = "main" -optional = false -python-versions = ">=3.6" -files = [ +zstandard = [ {file = "zstandard-0.19.0-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:a65e0119ad39e855427520f7829618f78eb2824aa05e63ff19b466080cd99210"}, {file = "zstandard-0.19.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:4fa496d2d674c6e9cffc561639d17009d29adee84a27cf1e12d3c9be14aa8feb"}, {file = "zstandard-0.19.0-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:8f7c68de4f362c1b2f426395fe4e05028c56d0782b2ec3ae18a5416eaf775576"}, @@ -2441,24 +2424,3 @@ files = [ {file = "zstandard-0.19.0-cp39-cp39-win_amd64.whl", hash = "sha256:55a513ec67e85abd8b8b83af8813368036f03e2d29a50fc94033504918273980"}, {file = "zstandard-0.19.0.tar.gz", hash = "sha256:31d12fcd942dd8dbf52ca5f6b1bbe287f44e5d551a081a983ff3ea2082867863"}, ] - -[package.dependencies] -cffi = {version = ">=1.11", markers = "platform_python_implementation == \"PyPy\""} - -[package.extras] -cffi = ["cffi (>=1.11)"] - -[extras] -adlfs = ["adlfs"] -duckdb = ["duckdb", "pyarrow"] -glue = ["boto3"] -hive = ["thrift"] -pandas = ["pandas", "pyarrow"] -pyarrow = ["pyarrow"] -s3fs = ["s3fs"] -snappy = ["python-snappy"] - -[metadata] -lock-version = "2.0" -python-versions = "^3.8" -content-hash = "61159a32a55ee5d6065910bbabdbde732a99b81a4415fd6f51a586055dd76223" diff --git a/pyproject.toml b/pyproject.toml index 5084b53d36..a3dbca59c4 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -54,7 +54,7 @@ rich = "13.2.0" pyyaml = "6.0.0" pydantic = "1.10.4" -fsspec = "2022.10.0" +fsspec = "2023.1.0" pyparsing = "3.0.9" @@ -70,10 +70,11 @@ python-snappy = { version = "0.6.1", optional = true } thrift = { version = "0.16.0", optional = true } -s3fs = { version = "2022.10.0", optional = true } boto3 = {version = "1.24.59", optional = true} -adlfs = { version = "2022.11.2", optional = true } +# The versions of the fsspec implementations should run in sync with fsspec above +s3fs = { version = "2023.1.0", optional = true } +adlfs = { version = "2023.1.0", optional = true } [tool.poetry.dev-dependencies] pytest = "7.2.1" From 1af133b490a15902d5af18d3c9f8ab2e7e94455c Mon Sep 17 00:00:00 2001 From: Fokko Driesprong Date: Tue, 24 Jan 2023 21:28:14 +0100 Subject: [PATCH 348/642] Python: Bump pylint (#6580) And remove the temporary hack --- .pre-commit-config.yaml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index c7ffa0c723..932e84cd90 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -53,11 +53,10 @@ repos: - id: pyupgrade args: [ --py38-plus, --keep-runtime-typing ] - repo: https://github.com/pycqa/pylint - rev: v2.15.9 + rev: v2.15.10 hooks: - id: pylint args: [ --rcfile=python/pylintrc ] - additional_dependencies: [ typing-extensions==4.4.0 ] - repo: https://github.com/pycqa/flake8 rev: '6.0.0' hooks: From 8fbe85364bf84dc108bc1734699d04ef8969c166 Mon Sep 17 00:00:00 2001 From: Fokko Driesprong Date: Tue, 24 Jan 2023 23:31:48 +0100 Subject: [PATCH 349/642] Python: Parallelize IO (#6645) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * Python: Parallelize IO Alternative for https://github.com/apache/iceberg/pull/6590 This uses the ThreadPool approach instead of ThreadPoolExecutor. The ThreadPoolExecutor is more flexible and works well with heterogeneous tasks. This allows the user to handle exceptions per task and able to cancel individual tasks. But the ThreadPoolExecutor also has some limitation such as not able to forcefully terminate all the tasks. For reading tasks I think the ThreadPool might be more appriopriate, but for writing the ThreadPoolExecutor might be more applicable. A very nice writeup of the differences is available in this blog: https://superfastpython.com/threadpool-vs-threadpoolexecutor/ Before: ``` ➜ python git:(fd-threadpool) time python3 /tmp/test.py python3 /tmp/test.py 3.45s user 2.84s system 2% cpu 3:34.19 total ``` After: ``` ➜ python git:(fd-threadpool) ✗ time python3 /tmp/test.py python3 /tmp/test.py 3.13s user 2.83s system 19% cpu 31.369 total ➜ python git:(fd-threadpool) ✗ time python3 /tmp/test.py python3 /tmp/test.py 2.94s user 3.08s system 18% cpu 32.538 total ➜ python git:(fd-threadpool) ✗ time python3 /tmp/test.py python3 /tmp/test.py 2.84s user 3.14s system 20% cpu 29.033 total ``` Longlining the requests from EU to the USA, so this might impact the results a bit, but makes IO more dominant. * Set read options --- pyiceberg/expressions/literals.py | 4 ++ pyiceberg/io/pyarrow.py | 97 +++++++++++++++++++------------ pyiceberg/table/__init__.py | 26 ++++++--- 3 files changed, 80 insertions(+), 47 deletions(-) diff --git a/pyiceberg/expressions/literals.py b/pyiceberg/expressions/literals.py index cd53ab6d41..fcfaaf7fe9 100644 --- a/pyiceberg/expressions/literals.py +++ b/pyiceberg/expressions/literals.py @@ -420,6 +420,10 @@ def to(self, type_var: IcebergType) -> Literal: # type: ignore def _(self, _: TimestampType) -> Literal[int]: return self + @to.register(TimestamptzType) + def _(self, _: TimestamptzType) -> Literal[int]: + return self + @to.register(DateType) def _(self, _: DateType) -> Literal[int]: return DateLiteral(micros_to_days(self.value)) diff --git a/pyiceberg/io/pyarrow.py b/pyiceberg/io/pyarrow.py index 12edc6e3e2..848ec4e0c8 100644 --- a/pyiceberg/io/pyarrow.py +++ b/pyiceberg/io/pyarrow.py @@ -26,6 +26,7 @@ import os from functools import lru_cache +from multiprocessing.pool import ThreadPool from typing import ( TYPE_CHECKING, Any, @@ -470,13 +471,62 @@ def expression_to_pyarrow(expr: BooleanExpression) -> pc.Expression: return boolean_expression_visit(expr, _ConvertToArrowExpression()) +def _file_to_table( + fs: FileSystem, + task: FileScanTask, + bound_row_filter: BooleanExpression, + projected_schema: Schema, + projected_field_ids: Set[int], + case_sensitive: bool, +) -> pa.Table: + _, path = PyArrowFileIO.parse_location(task.file.file_path) + + # Get the schema + with fs.open_input_file(path) as fout: + parquet_schema = pq.read_schema(fout) + schema_raw = None + if metadata := parquet_schema.metadata: + schema_raw = metadata.get(ICEBERG_SCHEMA) + if schema_raw is None: + raise ValueError( + "Iceberg schema is not embedded into the Parquet file, see https://github.com/apache/iceberg/issues/6505" + ) + file_schema = Schema.parse_raw(schema_raw) + + pyarrow_filter = None + if bound_row_filter is not AlwaysTrue(): + translated_row_filter = translate_column_names(bound_row_filter, file_schema, case_sensitive=case_sensitive) + bound_file_filter = bind(file_schema, translated_row_filter, case_sensitive=case_sensitive) + pyarrow_filter = expression_to_pyarrow(bound_file_filter) + + file_project_schema = prune_columns(file_schema, projected_field_ids, select_full_types=False) + + if file_schema is None: + raise ValueError(f"Missing Iceberg schema in Metadata for file: {path}") + + # Prune the stuff that we don't need anyway + file_project_schema_arrow = schema_to_pyarrow(file_project_schema) + + read_options = { + "pre_buffer": True, + "use_buffered_stream": True, + "buffer_size": 8388608, + } + + arrow_table = ds.dataset( + source=[path], schema=file_project_schema_arrow, format=ds.ParquetFileFormat(**read_options), filesystem=fs + ).to_table(filter=pyarrow_filter) + + return to_requested_schema(projected_schema, file_project_schema, arrow_table) + + def project_table( - files: Iterable[FileScanTask], table: Table, row_filter: BooleanExpression, projected_schema: Schema, case_sensitive: bool + tasks: Iterable[FileScanTask], table: Table, row_filter: BooleanExpression, projected_schema: Schema, case_sensitive: bool ) -> pa.Table: """Resolves the right columns based on the identifier Args: - files(Iterable[FileScanTask]): A URI or a path to a local file + tasks(Iterable[FileScanTask]): A URI or a path to a local file table(Table): The table that's being queried row_filter(BooleanExpression): The expression for filtering rows projected_schema(Schema): The output schema @@ -487,7 +537,7 @@ def project_table( """ if isinstance(table.io, PyArrowFileIO): - scheme, path = PyArrowFileIO.parse_location(table.location()) + scheme, _ = PyArrowFileIO.parse_location(table.location()) fs = table.io.get_fs(scheme) else: raise ValueError(f"Expected PyArrowFileIO, got: {table.io}") @@ -498,41 +548,12 @@ def project_table( id for id in projected_schema.field_ids if not isinstance(projected_schema.find_type(id), (MapType, ListType)) }.union(extract_field_ids(bound_row_filter)) - tables = [] - for task in files: - _, path = PyArrowFileIO.parse_location(task.file.file_path) - - # Get the schema - with fs.open_input_file(path) as fout: - parquet_schema = pq.read_schema(fout) - schema_raw = None - if metadata := parquet_schema.metadata: - schema_raw = metadata.get(ICEBERG_SCHEMA) - if schema_raw is None: - raise ValueError( - "Iceberg schema is not embedded into the Parquet file, see https://github.com/apache/iceberg/issues/6505" - ) - file_schema = Schema.parse_raw(schema_raw) - - pyarrow_filter = None - if row_filter is not AlwaysTrue(): - translated_row_filter = translate_column_names(bound_row_filter, file_schema, case_sensitive=case_sensitive) - bound_file_filter = bind(file_schema, translated_row_filter, case_sensitive=case_sensitive) - pyarrow_filter = expression_to_pyarrow(bound_file_filter) - - file_project_schema = prune_columns(file_schema, projected_field_ids, select_full_types=False) - - if file_schema is None: - raise ValueError(f"Missing Iceberg schema in Metadata for file: {path}") - - # Prune the stuff that we don't need anyway - file_project_schema_arrow = schema_to_pyarrow(file_project_schema) - - arrow_table = ds.dataset( - source=[path], schema=file_project_schema_arrow, format=ds.ParquetFileFormat(), filesystem=fs - ).to_table(filter=pyarrow_filter) - - tables.append(to_requested_schema(projected_schema, file_project_schema, arrow_table)) + with ThreadPool() as pool: + tables = pool.starmap( + func=_file_to_table, + iterable=[(fs, task, bound_row_filter, projected_schema, projected_field_ids, case_sensitive) for task in tasks], + chunksize=None, # we could use this to control how to materialize the generator of tasks (we should also make the expression above lazy) + ) if len(tables) > 1: return pa.concat_tables(tables) diff --git a/pyiceberg/table/__init__.py b/pyiceberg/table/__init__.py index 910ea7f0e0..db42540496 100644 --- a/pyiceberg/table/__init__.py +++ b/pyiceberg/table/__init__.py @@ -19,6 +19,8 @@ from abc import ABC, abstractmethod from dataclasses import dataclass from functools import cached_property +from itertools import chain +from multiprocessing.pool import ThreadPool from typing import ( TYPE_CHECKING, Any, @@ -272,6 +274,13 @@ def _check_content(file: DataFile) -> DataFile: return file +def _open_manifest(io: FileIO, manifest: ManifestFile, partition_filter: Callable[[DataFile], bool]) -> List[FileScanTask]: + all_files = files(io.new_input(manifest.manifest_path)) + matching_partition_files = filter(partition_filter, all_files) + matching_partition_data_files = map(_check_content, matching_partition_files) + return [FileScanTask(file) for file in matching_partition_data_files] + + class DataScan(TableScan["DataScan"]): def __init__( self, @@ -308,7 +317,7 @@ def _build_partition_evaluator(self, spec_id: int) -> Callable[[DataFile], bool] def plan_files(self) -> Iterator[FileScanTask]: snapshot = self.snapshot() if not snapshot: - return + return iter([]) io = self.table.io @@ -328,14 +337,13 @@ def plan_files(self) -> Iterator[FileScanTask]: partition_evaluators: Dict[int, Callable[[DataFile], bool]] = KeyDefaultDict(self._build_partition_evaluator) - for manifest in manifests: - partition_filter = partition_evaluators[manifest.partition_spec_id] - all_files = files(io.new_input(manifest.manifest_path)) - matching_partition_files = filter(partition_filter, all_files) - - matching_partition_data_files = map(_check_content, matching_partition_files) - - yield from (FileScanTask(file) for file in matching_partition_data_files) + with ThreadPool() as pool: + return chain( + *pool.starmap( + func=_open_manifest, + iterable=[(io, manifest, partition_evaluators[manifest.partition_spec_id]) for manifest in manifests], + ) + ) def to_arrow(self) -> pa.Table: return project_table( From bff5401ced0b889cf0eaf646fc9e360d80752ccd Mon Sep 17 00:00:00 2001 From: Fokko Driesprong Date: Sat, 28 Jan 2023 00:08:05 +0100 Subject: [PATCH 350/642] Python: Add visitor to DNF expr into Dask/PyArrow format (#6566) --- pyiceberg/expressions/visitors.py | 79 ++++++++++++++++++++++++++++++ tests/expressions/test_visitors.py | 21 ++++++++ 2 files changed, 100 insertions(+) diff --git a/pyiceberg/expressions/visitors.py b/pyiceberg/expressions/visitors.py index f312f12c3f..a76b183a98 100644 --- a/pyiceberg/expressions/visitors.py +++ b/pyiceberg/expressions/visitors.py @@ -881,3 +881,82 @@ def rewrite_to_dnf(expr: BooleanExpression) -> Tuple[BooleanExpression, ...]: # (A AND NOT(B) AND C) OR (NOT(D) AND E AND F) OR (G) expr_without_not = rewrite_not(expr) return visit(expr_without_not, _RewriteToDNF()) + + +class ExpressionToPlainFormat(BoundBooleanExpressionVisitor[List[Tuple[str, str, Any]]]): + def visit_in(self, term: BoundTerm[L], literals: Set[L]) -> List[Tuple[str, str, Any]]: + return [(term.ref().field.name, "in", literals)] + + def visit_not_in(self, term: BoundTerm[L], literals: Set[L]) -> List[Tuple[str, str, Any]]: + return [(term.ref().field.name, "not in", literals)] + + def visit_is_nan(self, term: BoundTerm[L]) -> List[Tuple[str, str, Any]]: + return [(term.ref().field.name, "==", float("nan"))] + + def visit_not_nan(self, term: BoundTerm[L]) -> List[Tuple[str, str, Any]]: + return [(term.ref().field.name, "!=", float("nan"))] + + def visit_is_null(self, term: BoundTerm[L]) -> List[Tuple[str, str, Any]]: + return [(term.ref().field.name, "==", None)] + + def visit_not_null(self, term: BoundTerm[L]) -> List[Tuple[str, str, Any]]: + return [(term.ref().field.name, "!=", None)] + + def visit_equal(self, term: BoundTerm[L], literal: Literal[L]) -> List[Tuple[str, str, Any]]: + return [(term.ref().field.name, "==", literal.value)] + + def visit_not_equal(self, term: BoundTerm[L], literal: Literal[L]) -> List[Tuple[str, str, Any]]: + return [(term.ref().field.name, "!=", literal.value)] + + def visit_greater_than_or_equal(self, term: BoundTerm[L], literal: Literal[L]) -> List[Tuple[str, str, Any]]: + return [(term.ref().field.name, ">=", literal.value)] + + def visit_greater_than(self, term: BoundTerm[L], literal: Literal[L]) -> List[Tuple[str, str, Any]]: + return [(term.ref().field.name, ">", literal.value)] + + def visit_less_than(self, term: BoundTerm[L], literal: Literal[L]) -> List[Tuple[str, str, Any]]: + return [(term.ref().field.name, "<", literal.value)] + + def visit_less_than_or_equal(self, term: BoundTerm[L], literal: Literal[L]) -> List[Tuple[str, str, Any]]: + return [(term.ref().field.name, "<=", literal.value)] + + def visit_true(self) -> List[Tuple[str, str, Any]]: + return [] # Not supported + + def visit_false(self) -> List[Tuple[str, str, Any]]: + raise ValueError("Not supported: AlwaysFalse") + + def visit_not(self, child_result: List[Tuple[str, str, Any]]) -> List[Tuple[str, str, Any]]: + raise ValueError(f"Not allowed: {child_result}") + + def visit_and( + self, left_result: List[Tuple[str, str, Any]], right_result: List[Tuple[str, str, Any]] + ) -> List[Tuple[str, str, Any]]: + return left_result + right_result + + def visit_or( + self, left_result: List[Tuple[str, str, Any]], right_result: List[Tuple[str, str, Any]] + ) -> List[Tuple[str, str, Any]]: + raise ValueError(f"Not allowed: {left_result} || {right_result}") + + +def expression_to_plain_format(expressions: Tuple[BooleanExpression, ...]) -> List[List[Tuple[str, str, Any]]]: + """Formats a Disjunctive Normal Form expression into the format that can be fed into: + + - https://arrow.apache.org/docs/python/generated/pyarrow.parquet.read_table.html + - https://docs.dask.org/en/stable/generated/dask.dataframe.read_parquet.html + + Contrary to normal DNF that may contain Not expressions, but here they should have + been rewritten. This can be done using ``rewrite_not(...)``. + + Keep in mind that this is only used for page skipping, and still needs to filter + on a row level. + + Args: + expressions: Expression in Disjunctive Normal Form + + Returns: + Formatter filter compatible with Dask and PyArrow + """ + # In the form of expr1 ∨ expr2 ∨ ... ∨ exprN + return [visit(expression, ExpressionToPlainFormat()) for expression in expressions] diff --git a/tests/expressions/test_visitors.py b/tests/expressions/test_visitors.py index 13e4d01eea..40e97004ad 100644 --- a/tests/expressions/test_visitors.py +++ b/tests/expressions/test_visitors.py @@ -64,6 +64,7 @@ BooleanExpressionVisitor, BoundBooleanExpressionVisitor, _ManifestEvalVisitor, + expression_to_plain_format, rewrite_not, rewrite_to_dnf, visit, @@ -1451,3 +1452,23 @@ def test_to_dnf_and() -> None: def test_to_dnf_not_and() -> None: expr = Not(And(Not(EqualTo("Q", "b")), EqualTo("R", "c"))) assert rewrite_to_dnf(expr) == (EqualTo("Q", "b"), NotEqualTo("R", "c")) + + +def test_dnf_to_dask(table_schema_simple: Schema) -> None: + expr = ( + BoundGreaterThan[str]( + term=BoundReference(table_schema_simple.find_field(1), table_schema_simple.accessor_for_field(1)), + literal=literal("hello"), + ), + And( + BoundIn[int]( + term=BoundReference(table_schema_simple.find_field(2), table_schema_simple.accessor_for_field(2)), + literals={literal(1), literal(2), literal(3)}, + ), + BoundEqualTo[bool]( + term=BoundReference(table_schema_simple.find_field(3), table_schema_simple.accessor_for_field(3)), + literal=literal(True), + ), + ), + ) + assert expression_to_plain_format(expr) == [[("foo", ">", "hello")], [("bar", "in", {1, 2, 3}), ("baz", "==", True)]] From 856bd44051c8820cb22d9c3ce3b9190eada181be Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 30 Jan 2023 05:49:35 +0100 Subject: [PATCH 351/642] Build: Bump pre-commit from 2.21.0 to 3.0.1 in /python (#6688) Bumps [pre-commit](https://github.com/pre-commit/pre-commit) from 2.21.0 to 3.0.1. - [Release notes](https://github.com/pre-commit/pre-commit/releases) - [Changelog](https://github.com/pre-commit/pre-commit/blob/main/CHANGELOG.md) - [Commits](https://github.com/pre-commit/pre-commit/compare/v2.21.0...v3.0.1) --- updated-dependencies: - dependency-name: pre-commit dependency-type: direct:development update-type: version-update:semver-major ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- poetry.lock | 2464 ++++++++++++++++++++++++------------------------ pyproject.toml | 2 +- 2 files changed, 1256 insertions(+), 1210 deletions(-) diff --git a/poetry.lock b/poetry.lock index c6e78d271f..91f41d758f 100644 --- a/poetry.lock +++ b/poetry.lock @@ -1,3 +1,5 @@ +# This file is automatically @generated by Poetry and should not be changed by hand. + [[package]] name = "adal" version = "1.2.7" @@ -5,6 +7,10 @@ description = "Note: This library is already replaced by MSAL Python, available category = "main" optional = true python-versions = "*" +files = [ + {file = "adal-1.2.7-py2.py3-none-any.whl", hash = "sha256:2a7451ed7441ddbc57703042204a3e30ef747478eea022c70f789fc7f084bc3d"}, + {file = "adal-1.2.7.tar.gz", hash = "sha256:d74f45b81317454d96e982fd1c50e6fb5c99ac2223728aea8764433a39f566f1"}, +] [package.dependencies] cryptography = ">=1.1.0" @@ -19,6 +25,10 @@ description = "Access Azure Datalake Gen1 with fsspec and dask" category = "main" optional = true python-versions = ">=3.8" +files = [ + {file = "adlfs-2023.1.0-py3-none-any.whl", hash = "sha256:ccbdd6d33d5b7bce99a4a07c884c82fb135745d39b2d9b7b672f8e9d4d04407f"}, + {file = "adlfs-2023.1.0.tar.gz", hash = "sha256:eca53f53d88fc8e2e7a2f1d8f5a40b8a1c56aa3b541e4aa2e7eaf55a6a789262"}, +] [package.dependencies] aiohttp = ">=3.7.0" @@ -38,6 +48,10 @@ description = "Async client for aws services using botocore and aiohttp" category = "main" optional = true python-versions = ">=3.7" +files = [ + {file = "aiobotocore-2.4.2-py3-none-any.whl", hash = "sha256:4acd1ebe2e44be4b100aa553910bda899f6dc090b3da2bc1cf3d5de2146ed208"}, + {file = "aiobotocore-2.4.2.tar.gz", hash = "sha256:0603b74a582dffa7511ce7548d07dc9b10ec87bc5fb657eb0b34f9bd490958bf"}, +] [package.dependencies] aiohttp = ">=3.3.1" @@ -56,6 +70,95 @@ description = "Async http client/server framework (asyncio)" category = "main" optional = true python-versions = ">=3.6" +files = [ + {file = "aiohttp-3.8.3-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:ba71c9b4dcbb16212f334126cc3d8beb6af377f6703d9dc2d9fb3874fd667ee9"}, + {file = "aiohttp-3.8.3-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:d24b8bb40d5c61ef2d9b6a8f4528c2f17f1c5d2d31fed62ec860f6006142e83e"}, + {file = "aiohttp-3.8.3-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:f88df3a83cf9df566f171adba39d5bd52814ac0b94778d2448652fc77f9eb491"}, + {file = "aiohttp-3.8.3-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:b97decbb3372d4b69e4d4c8117f44632551c692bb1361b356a02b97b69e18a62"}, + {file = "aiohttp-3.8.3-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:309aa21c1d54b8ef0723181d430347d7452daaff93e8e2363db8e75c72c2fb2d"}, + {file = "aiohttp-3.8.3-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:ad5383a67514e8e76906a06741febd9126fc7c7ff0f599d6fcce3e82b80d026f"}, + {file = "aiohttp-3.8.3-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:20acae4f268317bb975671e375493dbdbc67cddb5f6c71eebdb85b34444ac46b"}, + {file = "aiohttp-3.8.3-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:05a3c31c6d7cd08c149e50dc7aa2568317f5844acd745621983380597f027a18"}, + {file = "aiohttp-3.8.3-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:d6f76310355e9fae637c3162936e9504b4767d5c52ca268331e2756e54fd4ca5"}, + {file = "aiohttp-3.8.3-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:256deb4b29fe5e47893fa32e1de2d73c3afe7407738bd3c63829874661d4822d"}, + {file = "aiohttp-3.8.3-cp310-cp310-musllinux_1_1_ppc64le.whl", hash = "sha256:5c59fcd80b9049b49acd29bd3598cada4afc8d8d69bd4160cd613246912535d7"}, + {file = "aiohttp-3.8.3-cp310-cp310-musllinux_1_1_s390x.whl", hash = "sha256:059a91e88f2c00fe40aed9031b3606c3f311414f86a90d696dd982e7aec48142"}, + {file = "aiohttp-3.8.3-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:2feebbb6074cdbd1ac276dbd737b40e890a1361b3cc30b74ac2f5e24aab41f7b"}, + {file = "aiohttp-3.8.3-cp310-cp310-win32.whl", hash = "sha256:5bf651afd22d5f0c4be16cf39d0482ea494f5c88f03e75e5fef3a85177fecdeb"}, + {file = "aiohttp-3.8.3-cp310-cp310-win_amd64.whl", hash = "sha256:653acc3880459f82a65e27bd6526e47ddf19e643457d36a2250b85b41a564715"}, + {file = "aiohttp-3.8.3-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:86fc24e58ecb32aee09f864cb11bb91bc4c1086615001647dbfc4dc8c32f4008"}, + {file = "aiohttp-3.8.3-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:75e14eac916f024305db517e00a9252714fce0abcb10ad327fb6dcdc0d060f1d"}, + {file = "aiohttp-3.8.3-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:d1fde0f44029e02d02d3993ad55ce93ead9bb9b15c6b7ccd580f90bd7e3de476"}, + {file = "aiohttp-3.8.3-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:4ab94426ddb1ecc6a0b601d832d5d9d421820989b8caa929114811369673235c"}, + {file = "aiohttp-3.8.3-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:89d2e02167fa95172c017732ed7725bc8523c598757f08d13c5acca308e1a061"}, + {file = "aiohttp-3.8.3-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:02f9a2c72fc95d59b881cf38a4b2be9381b9527f9d328771e90f72ac76f31ad8"}, + {file = "aiohttp-3.8.3-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:9c7149272fb5834fc186328e2c1fa01dda3e1fa940ce18fded6d412e8f2cf76d"}, + {file = "aiohttp-3.8.3-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:512bd5ab136b8dc0ffe3fdf2dfb0c4b4f49c8577f6cae55dca862cd37a4564e2"}, + {file = "aiohttp-3.8.3-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:7018ecc5fe97027214556afbc7c502fbd718d0740e87eb1217b17efd05b3d276"}, + {file = "aiohttp-3.8.3-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:88c70ed9da9963d5496d38320160e8eb7e5f1886f9290475a881db12f351ab5d"}, + {file = "aiohttp-3.8.3-cp311-cp311-musllinux_1_1_ppc64le.whl", hash = "sha256:da22885266bbfb3f78218dc40205fed2671909fbd0720aedba39b4515c038091"}, + {file = "aiohttp-3.8.3-cp311-cp311-musllinux_1_1_s390x.whl", hash = "sha256:e65bc19919c910127c06759a63747ebe14f386cda573d95bcc62b427ca1afc73"}, + {file = "aiohttp-3.8.3-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:08c78317e950e0762c2983f4dd58dc5e6c9ff75c8a0efeae299d363d439c8e34"}, + {file = "aiohttp-3.8.3-cp311-cp311-win32.whl", hash = "sha256:45d88b016c849d74ebc6f2b6e8bc17cabf26e7e40c0661ddd8fae4c00f015697"}, + {file = "aiohttp-3.8.3-cp311-cp311-win_amd64.whl", hash = "sha256:96372fc29471646b9b106ee918c8eeb4cca423fcbf9a34daa1b93767a88a2290"}, + {file = "aiohttp-3.8.3-cp36-cp36m-macosx_10_9_x86_64.whl", hash = "sha256:c971bf3786b5fad82ce5ad570dc6ee420f5b12527157929e830f51c55dc8af77"}, + {file = "aiohttp-3.8.3-cp36-cp36m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:ff25f48fc8e623d95eca0670b8cc1469a83783c924a602e0fbd47363bb54aaca"}, + {file = "aiohttp-3.8.3-cp36-cp36m-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:e381581b37db1db7597b62a2e6b8b57c3deec95d93b6d6407c5b61ddc98aca6d"}, + {file = "aiohttp-3.8.3-cp36-cp36m-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:db19d60d846283ee275d0416e2a23493f4e6b6028825b51290ac05afc87a6f97"}, + {file = "aiohttp-3.8.3-cp36-cp36m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:25892c92bee6d9449ffac82c2fe257f3a6f297792cdb18ad784737d61e7a9a85"}, + {file = "aiohttp-3.8.3-cp36-cp36m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:398701865e7a9565d49189f6c90868efaca21be65c725fc87fc305906be915da"}, + {file = "aiohttp-3.8.3-cp36-cp36m-musllinux_1_1_aarch64.whl", hash = "sha256:4a4fbc769ea9b6bd97f4ad0b430a6807f92f0e5eb020f1e42ece59f3ecfc4585"}, + {file = "aiohttp-3.8.3-cp36-cp36m-musllinux_1_1_i686.whl", hash = "sha256:b29bfd650ed8e148f9c515474a6ef0ba1090b7a8faeee26b74a8ff3b33617502"}, + {file = "aiohttp-3.8.3-cp36-cp36m-musllinux_1_1_ppc64le.whl", hash = "sha256:1e56b9cafcd6531bab5d9b2e890bb4937f4165109fe98e2b98ef0dcfcb06ee9d"}, + {file = "aiohttp-3.8.3-cp36-cp36m-musllinux_1_1_s390x.whl", hash = "sha256:ec40170327d4a404b0d91855d41bfe1fe4b699222b2b93e3d833a27330a87a6d"}, + {file = "aiohttp-3.8.3-cp36-cp36m-musllinux_1_1_x86_64.whl", hash = "sha256:2df5f139233060578d8c2c975128fb231a89ca0a462b35d4b5fcf7c501ebdbe1"}, + {file = "aiohttp-3.8.3-cp36-cp36m-win32.whl", hash = "sha256:f973157ffeab5459eefe7b97a804987876dd0a55570b8fa56b4e1954bf11329b"}, + {file = "aiohttp-3.8.3-cp36-cp36m-win_amd64.whl", hash = "sha256:437399385f2abcd634865705bdc180c8314124b98299d54fe1d4c8990f2f9494"}, + {file = "aiohttp-3.8.3-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:09e28f572b21642128ef31f4e8372adb6888846f32fecb288c8b0457597ba61a"}, + {file = "aiohttp-3.8.3-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:6f3553510abdbec67c043ca85727396ceed1272eef029b050677046d3387be8d"}, + {file = "aiohttp-3.8.3-cp37-cp37m-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:e168a7560b7c61342ae0412997b069753f27ac4862ec7867eff74f0fe4ea2ad9"}, + {file = "aiohttp-3.8.3-cp37-cp37m-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:db4c979b0b3e0fa7e9e69ecd11b2b3174c6963cebadeecfb7ad24532ffcdd11a"}, + {file = "aiohttp-3.8.3-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:e164e0a98e92d06da343d17d4e9c4da4654f4a4588a20d6c73548a29f176abe2"}, + {file = "aiohttp-3.8.3-cp37-cp37m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:e8a78079d9a39ca9ca99a8b0ac2fdc0c4d25fc80c8a8a82e5c8211509c523363"}, + {file = "aiohttp-3.8.3-cp37-cp37m-musllinux_1_1_aarch64.whl", hash = "sha256:21b30885a63c3f4ff5b77a5d6caf008b037cb521a5f33eab445dc566f6d092cc"}, + {file = "aiohttp-3.8.3-cp37-cp37m-musllinux_1_1_i686.whl", hash = "sha256:4b0f30372cef3fdc262f33d06e7b411cd59058ce9174ef159ad938c4a34a89da"}, + {file = "aiohttp-3.8.3-cp37-cp37m-musllinux_1_1_ppc64le.whl", hash = "sha256:8135fa153a20d82ffb64f70a1b5c2738684afa197839b34cc3e3c72fa88d302c"}, + {file = "aiohttp-3.8.3-cp37-cp37m-musllinux_1_1_s390x.whl", hash = "sha256:ad61a9639792fd790523ba072c0555cd6be5a0baf03a49a5dd8cfcf20d56df48"}, + {file = "aiohttp-3.8.3-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:978b046ca728073070e9abc074b6299ebf3501e8dee5e26efacb13cec2b2dea0"}, + {file = "aiohttp-3.8.3-cp37-cp37m-win32.whl", hash = "sha256:0d2c6d8c6872df4a6ec37d2ede71eff62395b9e337b4e18efd2177de883a5033"}, + {file = "aiohttp-3.8.3-cp37-cp37m-win_amd64.whl", hash = "sha256:21d69797eb951f155026651f7e9362877334508d39c2fc37bd04ff55b2007091"}, + {file = "aiohttp-3.8.3-cp38-cp38-macosx_10_9_universal2.whl", hash = "sha256:2ca9af5f8f5812d475c5259393f52d712f6d5f0d7fdad9acdb1107dd9e3cb7eb"}, + {file = "aiohttp-3.8.3-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:1d90043c1882067f1bd26196d5d2db9aa6d268def3293ed5fb317e13c9413ea4"}, + {file = "aiohttp-3.8.3-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:d737fc67b9a970f3234754974531dc9afeea11c70791dcb7db53b0cf81b79784"}, + {file = "aiohttp-3.8.3-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:ebf909ea0a3fc9596e40d55d8000702a85e27fd578ff41a5500f68f20fd32e6c"}, + {file = "aiohttp-3.8.3-cp38-cp38-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:5835f258ca9f7c455493a57ee707b76d2d9634d84d5d7f62e77be984ea80b849"}, + {file = "aiohttp-3.8.3-cp38-cp38-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:da37dcfbf4b7f45d80ee386a5f81122501ec75672f475da34784196690762f4b"}, + {file = "aiohttp-3.8.3-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:87f44875f2804bc0511a69ce44a9595d5944837a62caecc8490bbdb0e18b1342"}, + {file = "aiohttp-3.8.3-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:527b3b87b24844ea7865284aabfab08eb0faf599b385b03c2aa91fc6edd6e4b6"}, + {file = "aiohttp-3.8.3-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:d5ba88df9aa5e2f806650fcbeedbe4f6e8736e92fc0e73b0400538fd25a4dd96"}, + {file = "aiohttp-3.8.3-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:e7b8813be97cab8cb52b1375f41f8e6804f6507fe4660152e8ca5c48f0436017"}, + {file = "aiohttp-3.8.3-cp38-cp38-musllinux_1_1_ppc64le.whl", hash = "sha256:2dea10edfa1a54098703cb7acaa665c07b4e7568472a47f4e64e6319d3821ccf"}, + {file = "aiohttp-3.8.3-cp38-cp38-musllinux_1_1_s390x.whl", hash = "sha256:713d22cd9643ba9025d33c4af43943c7a1eb8547729228de18d3e02e278472b6"}, + {file = "aiohttp-3.8.3-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:2d252771fc85e0cf8da0b823157962d70639e63cb9b578b1dec9868dd1f4f937"}, + {file = "aiohttp-3.8.3-cp38-cp38-win32.whl", hash = "sha256:66bd5f950344fb2b3dbdd421aaa4e84f4411a1a13fca3aeb2bcbe667f80c9f76"}, + {file = "aiohttp-3.8.3-cp38-cp38-win_amd64.whl", hash = "sha256:84b14f36e85295fe69c6b9789b51a0903b774046d5f7df538176516c3e422446"}, + {file = "aiohttp-3.8.3-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:16c121ba0b1ec2b44b73e3a8a171c4f999b33929cd2397124a8c7fcfc8cd9e06"}, + {file = "aiohttp-3.8.3-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:8d6aaa4e7155afaf994d7924eb290abbe81a6905b303d8cb61310a2aba1c68ba"}, + {file = "aiohttp-3.8.3-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:43046a319664a04b146f81b40e1545d4c8ac7b7dd04c47e40bf09f65f2437346"}, + {file = "aiohttp-3.8.3-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:599418aaaf88a6d02a8c515e656f6faf3d10618d3dd95866eb4436520096c84b"}, + {file = "aiohttp-3.8.3-cp39-cp39-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:92a2964319d359f494f16011e23434f6f8ef0434acd3cf154a6b7bec511e2fb7"}, + {file = "aiohttp-3.8.3-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:73a4131962e6d91109bca6536416aa067cf6c4efb871975df734f8d2fd821b37"}, + {file = "aiohttp-3.8.3-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:598adde339d2cf7d67beaccda3f2ce7c57b3b412702f29c946708f69cf8222aa"}, + {file = "aiohttp-3.8.3-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:75880ed07be39beff1881d81e4a907cafb802f306efd6d2d15f2b3c69935f6fb"}, + {file = "aiohttp-3.8.3-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:a0239da9fbafd9ff82fd67c16704a7d1bccf0d107a300e790587ad05547681c8"}, + {file = "aiohttp-3.8.3-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:4e3a23ec214e95c9fe85a58470b660efe6534b83e6cbe38b3ed52b053d7cb6ad"}, + {file = "aiohttp-3.8.3-cp39-cp39-musllinux_1_1_ppc64le.whl", hash = "sha256:47841407cc89a4b80b0c52276f3cc8138bbbfba4b179ee3acbd7d77ae33f7ac4"}, + {file = "aiohttp-3.8.3-cp39-cp39-musllinux_1_1_s390x.whl", hash = "sha256:54d107c89a3ebcd13228278d68f1436d3f33f2dd2af5415e3feaeb1156e1a62c"}, + {file = "aiohttp-3.8.3-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:c37c5cce780349d4d51739ae682dec63573847a2a8dcb44381b174c3d9c8d403"}, + {file = "aiohttp-3.8.3-cp39-cp39-win32.whl", hash = "sha256:f178d2aadf0166be4df834c4953da2d7eef24719e8aec9a65289483eeea9d618"}, + {file = "aiohttp-3.8.3-cp39-cp39-win_amd64.whl", hash = "sha256:88e5be56c231981428f4f506c68b6a46fa25c4123a2e86d156c58a8369d31ab7"}, + {file = "aiohttp-3.8.3.tar.gz", hash = "sha256:3828fb41b7203176b82fe5d699e0d845435f2374750a44b480ea6b930f6be269"}, +] [package.dependencies] aiosignal = ">=1.1.2" @@ -67,7 +170,7 @@ multidict = ">=4.5,<7.0" yarl = ">=1.0,<2.0" [package.extras] -speedups = ["aiodns", "brotli", "cchardet"] +speedups = ["Brotli", "aiodns", "cchardet"] [[package]] name = "aioitertools" @@ -76,6 +179,10 @@ description = "itertools and builtins for AsyncIO and mixed iterables" category = "main" optional = true python-versions = ">=3.6" +files = [ + {file = "aioitertools-0.11.0-py3-none-any.whl", hash = "sha256:04b95e3dab25b449def24d7df809411c10e62aab0cbe31a50ca4e68748c43394"}, + {file = "aioitertools-0.11.0.tar.gz", hash = "sha256:42c68b8dd3a69c2bf7f2233bf7df4bb58b557bca5252ac02ed5187bbc67d6831"}, +] [package.dependencies] typing_extensions = {version = ">=4.0", markers = "python_version < \"3.10\""} @@ -87,6 +194,10 @@ description = "aiosignal: a list of registered asynchronous callbacks" category = "main" optional = true python-versions = ">=3.7" +files = [ + {file = "aiosignal-1.3.1-py3-none-any.whl", hash = "sha256:f8376fb07dd1e86a584e4fcdec80b36b7f81aac666ebc724e2c090300dd83b17"}, + {file = "aiosignal-1.3.1.tar.gz", hash = "sha256:54cd96e15e1649b75d6c87526a6ff0b6c1b0dd3459f43d9ca11d48c339b68cfc"}, +] [package.dependencies] frozenlist = ">=1.1.0" @@ -98,6 +209,10 @@ description = "Timeout context manager for asyncio programs" category = "main" optional = true python-versions = ">=3.6" +files = [ + {file = "async-timeout-4.0.2.tar.gz", hash = "sha256:2163e1640ddb52b7a8c80d0a67a08587e5d245cc9c553a74a847056bc2976b15"}, + {file = "async_timeout-4.0.2-py3-none-any.whl", hash = "sha256:8ca1e4fcf50d07413d66d1a5e416e42cfdf5851c981d679a09851a6853383b3c"}, +] [[package]] name = "attrs" @@ -106,14 +221,17 @@ description = "Classes Without Boilerplate" category = "main" optional = false python-versions = ">=3.6" +files = [ + {file = "attrs-22.2.0-py3-none-any.whl", hash = "sha256:29e95c7f6778868dbd49170f98f8818f78f3dc5e0e37c0b1f474e3561b240836"}, + {file = "attrs-22.2.0.tar.gz", hash = "sha256:c9227bfc2f01993c03f68db37d1d15c9690188323c067c641f1a35ca58185f99"}, +] [package.extras] -cov = ["attrs", "coverage-enable-subprocess", "coverage[toml] (>=5.3)"] -dev = ["attrs"] +cov = ["attrs[tests]", "coverage-enable-subprocess", "coverage[toml] (>=5.3)"] +dev = ["attrs[docs,tests]"] docs = ["furo", "myst-parser", "sphinx", "sphinx-notfound-page", "sphinxcontrib-towncrier", "towncrier", "zope.interface"] -tests = ["attrs", "zope.interface"] -tests-no-zope = ["cloudpickle", "hypothesis", "mypy (>=0.971,<0.990)", "pympler", "pytest (>=4.3.0)", "pytest-mypy-plugins", "pytest-xdist"] -tests_no_zope = ["cloudpickle", "hypothesis", "mypy (>=0.971,<0.990)", "pympler", "pytest (>=4.3.0)", "pytest-mypy-plugins", "pytest-xdist"] +tests = ["attrs[tests-no-zope]", "zope.interface"] +tests-no-zope = ["cloudpickle", "cloudpickle", "hypothesis", "hypothesis", "mypy (>=0.971,<0.990)", "mypy (>=0.971,<0.990)", "pympler", "pympler", "pytest (>=4.3.0)", "pytest (>=4.3.0)", "pytest-mypy-plugins", "pytest-mypy-plugins", "pytest-xdist[psutil]", "pytest-xdist[psutil]"] [[package]] name = "azure-core" @@ -122,6 +240,10 @@ description = "Microsoft Azure Core Library for Python" category = "main" optional = true python-versions = ">=3.7" +files = [ + {file = "azure-core-1.26.2.zip", hash = "sha256:986bfd8687889782d79481d4c5d0af04ab4a18ca2f210364804a88e4eaa1586a"}, + {file = "azure_core-1.26.2-py3-none-any.whl", hash = "sha256:df306e6e4abc145610ca6744aef943129a6fd7a11977e56731f69ac0e00724f9"}, +] [package.dependencies] requests = ">=2.18.4" @@ -138,6 +260,10 @@ description = "Azure Data Lake Store Filesystem Client Library for Python" category = "main" optional = true python-versions = "*" +files = [ + {file = "azure-datalake-store-0.0.52.tar.gz", hash = "sha256:4198ddb32614d16d4502b43d5c9739f81432b7e0e4d75d30e05149fe6007fea2"}, + {file = "azure_datalake_store-0.0.52-py2.py3-none-any.whl", hash = "sha256:aaed72b9c856824aeab554f4dbe0ef2c6d0ff36700bdd8b93d8298793117c48e"}, +] [package.dependencies] adal = ">=0.4.2" @@ -151,6 +277,10 @@ description = "Microsoft Azure Identity Library for Python" category = "main" optional = true python-versions = ">=3.7" +files = [ + {file = "azure-identity-1.12.0.zip", hash = "sha256:7f9b1ae7d97ea7af3f38dd09305e19ab81a1e16ab66ea186b6579d85c1ca2347"}, + {file = "azure_identity-1.12.0-py3-none-any.whl", hash = "sha256:2a58ce4a209a013e37eaccfd5937570ab99e9118b3e1acf875eed3a85d541b92"}, +] [package.dependencies] azure-core = ">=1.11.0,<2.0.0" @@ -166,6 +296,10 @@ description = "Microsoft Azure Blob Storage Client Library for Python" category = "main" optional = true python-versions = ">=3.7" +files = [ + {file = "azure-storage-blob-12.14.1.zip", hash = "sha256:860d4d82985a4bfc7d3271e71275af330f54f330a754355435a7ba749ccde997"}, + {file = "azure_storage_blob-12.14.1-py3-none-any.whl", hash = "sha256:52b84658e8df7853a3cf1c563814655b5028b979b2a87905b92aa6bb30be240e"}, +] [package.dependencies] azure-core = ">=1.24.2,<2.0.0" @@ -179,6 +313,10 @@ description = "The AWS SDK for Python" category = "main" optional = false python-versions = ">= 3.7" +files = [ + {file = "boto3-1.24.59-py3-none-any.whl", hash = "sha256:34ab44146a2c4e7f4e72737f4b27e6eb5e0a7855c2f4599e3d9199b6a0a2d575"}, + {file = "boto3-1.24.59.tar.gz", hash = "sha256:a50b4323f9579cfe22fcf5531fbd40b567d4d74c1adce06aeb5c95fce2a6fb40"}, +] [package.dependencies] botocore = ">=1.27.59,<1.28.0" @@ -195,6 +333,10 @@ description = "Low-level, data-driven core of boto 3." category = "main" optional = false python-versions = ">= 3.7" +files = [ + {file = "botocore-1.27.59-py3-none-any.whl", hash = "sha256:69d756791fc024bda54f6c53f71ae34e695ee41bbbc1743d9179c4837a4929da"}, + {file = "botocore-1.27.59.tar.gz", hash = "sha256:eda4aed6ee719a745d1288eaf1beb12f6f6448ad1fa12f159405db14ba9c92cf"}, +] [package.dependencies] jmespath = ">=0.7.1,<2.0.0" @@ -211,6 +353,10 @@ description = "A simple, correct Python build frontend" category = "dev" optional = false python-versions = ">= 3.7" +files = [ + {file = "build-0.10.0-py3-none-any.whl", hash = "sha256:af266720050a66c893a6096a2f410989eeac74ff9a68ba194b3f6473e8e26171"}, + {file = "build-0.10.0.tar.gz", hash = "sha256:d5b71264afdb5951d6704482aac78de887c80691c52b88a9ad195983ca2c9269"}, +] [package.dependencies] colorama = {version = "*", markers = "os_name == \"nt\""} @@ -231,6 +377,10 @@ description = "Python package for providing Mozilla's CA Bundle." category = "main" optional = false python-versions = ">=3.6" +files = [ + {file = "certifi-2022.12.7-py3-none-any.whl", hash = "sha256:4ad3232f5e926d6718ec31cfc1fcadfde020920e278684144551c91769c7bc18"}, + {file = "certifi-2022.12.7.tar.gz", hash = "sha256:35824b4c3a97115964b408844d64aa14db1cc518f6562e8d7261699d1350a9e3"}, +] [[package]] name = "cffi" @@ -239,6 +389,72 @@ description = "Foreign Function Interface for Python calling C code." category = "main" optional = false python-versions = "*" +files = [ + {file = "cffi-1.15.1-cp27-cp27m-macosx_10_9_x86_64.whl", hash = "sha256:a66d3508133af6e8548451b25058d5812812ec3798c886bf38ed24a98216fab2"}, + {file = "cffi-1.15.1-cp27-cp27m-manylinux1_i686.whl", hash = "sha256:470c103ae716238bbe698d67ad020e1db9d9dba34fa5a899b5e21577e6d52ed2"}, + {file = "cffi-1.15.1-cp27-cp27m-manylinux1_x86_64.whl", hash = "sha256:9ad5db27f9cabae298d151c85cf2bad1d359a1b9c686a275df03385758e2f914"}, + {file = "cffi-1.15.1-cp27-cp27m-win32.whl", hash = "sha256:b3bbeb01c2b273cca1e1e0c5df57f12dce9a4dd331b4fa1635b8bec26350bde3"}, + {file = "cffi-1.15.1-cp27-cp27m-win_amd64.whl", hash = "sha256:e00b098126fd45523dd056d2efba6c5a63b71ffe9f2bbe1a4fe1716e1d0c331e"}, + {file = "cffi-1.15.1-cp27-cp27mu-manylinux1_i686.whl", hash = "sha256:d61f4695e6c866a23a21acab0509af1cdfd2c013cf256bbf5b6b5e2695827162"}, + {file = "cffi-1.15.1-cp27-cp27mu-manylinux1_x86_64.whl", hash = "sha256:ed9cb427ba5504c1dc15ede7d516b84757c3e3d7868ccc85121d9310d27eed0b"}, + {file = "cffi-1.15.1-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:39d39875251ca8f612b6f33e6b1195af86d1b3e60086068be9cc053aa4376e21"}, + {file = "cffi-1.15.1-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:285d29981935eb726a4399badae8f0ffdff4f5050eaa6d0cfc3f64b857b77185"}, + {file = "cffi-1.15.1-cp310-cp310-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:3eb6971dcff08619f8d91607cfc726518b6fa2a9eba42856be181c6d0d9515fd"}, + {file = "cffi-1.15.1-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:21157295583fe8943475029ed5abdcf71eb3911894724e360acff1d61c1d54bc"}, + {file = "cffi-1.15.1-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:5635bd9cb9731e6d4a1132a498dd34f764034a8ce60cef4f5319c0541159392f"}, + {file = "cffi-1.15.1-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:2012c72d854c2d03e45d06ae57f40d78e5770d252f195b93f581acf3ba44496e"}, + {file = "cffi-1.15.1-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:dd86c085fae2efd48ac91dd7ccffcfc0571387fe1193d33b6394db7ef31fe2a4"}, + {file = "cffi-1.15.1-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:fa6693661a4c91757f4412306191b6dc88c1703f780c8234035eac011922bc01"}, + {file = "cffi-1.15.1-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:59c0b02d0a6c384d453fece7566d1c7e6b7bae4fc5874ef2ef46d56776d61c9e"}, + {file = "cffi-1.15.1-cp310-cp310-win32.whl", hash = "sha256:cba9d6b9a7d64d4bd46167096fc9d2f835e25d7e4c121fb2ddfc6528fb0413b2"}, + {file = "cffi-1.15.1-cp310-cp310-win_amd64.whl", hash = "sha256:ce4bcc037df4fc5e3d184794f27bdaab018943698f4ca31630bc7f84a7b69c6d"}, + {file = "cffi-1.15.1-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:3d08afd128ddaa624a48cf2b859afef385b720bb4b43df214f85616922e6a5ac"}, + {file = "cffi-1.15.1-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:3799aecf2e17cf585d977b780ce79ff0dc9b78d799fc694221ce814c2c19db83"}, + {file = "cffi-1.15.1-cp311-cp311-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:a591fe9e525846e4d154205572a029f653ada1a78b93697f3b5a8f1f2bc055b9"}, + {file = "cffi-1.15.1-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:3548db281cd7d2561c9ad9984681c95f7b0e38881201e157833a2342c30d5e8c"}, + {file = "cffi-1.15.1-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:91fc98adde3d7881af9b59ed0294046f3806221863722ba7d8d120c575314325"}, + {file = "cffi-1.15.1-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:94411f22c3985acaec6f83c6df553f2dbe17b698cc7f8ae751ff2237d96b9e3c"}, + {file = "cffi-1.15.1-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:03425bdae262c76aad70202debd780501fabeaca237cdfddc008987c0e0f59ef"}, + {file = "cffi-1.15.1-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:cc4d65aeeaa04136a12677d3dd0b1c0c94dc43abac5860ab33cceb42b801c1e8"}, + {file = "cffi-1.15.1-cp311-cp311-win32.whl", hash = "sha256:a0f100c8912c114ff53e1202d0078b425bee3649ae34d7b070e9697f93c5d52d"}, + {file = "cffi-1.15.1-cp311-cp311-win_amd64.whl", hash = "sha256:04ed324bda3cda42b9b695d51bb7d54b680b9719cfab04227cdd1e04e5de3104"}, + {file = "cffi-1.15.1-cp36-cp36m-macosx_10_9_x86_64.whl", hash = "sha256:50a74364d85fd319352182ef59c5c790484a336f6db772c1a9231f1c3ed0cbd7"}, + {file = "cffi-1.15.1-cp36-cp36m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:e263d77ee3dd201c3a142934a086a4450861778baaeeb45db4591ef65550b0a6"}, + {file = "cffi-1.15.1-cp36-cp36m-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:cec7d9412a9102bdc577382c3929b337320c4c4c4849f2c5cdd14d7368c5562d"}, + {file = "cffi-1.15.1-cp36-cp36m-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:4289fc34b2f5316fbb762d75362931e351941fa95fa18789191b33fc4cf9504a"}, + {file = "cffi-1.15.1-cp36-cp36m-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:173379135477dc8cac4bc58f45db08ab45d228b3363adb7af79436135d028405"}, + {file = "cffi-1.15.1-cp36-cp36m-manylinux_2_5_x86_64.manylinux1_x86_64.whl", hash = "sha256:6975a3fac6bc83c4a65c9f9fcab9e47019a11d3d2cf7f3c0d03431bf145a941e"}, + {file = "cffi-1.15.1-cp36-cp36m-win32.whl", hash = "sha256:2470043b93ff09bf8fb1d46d1cb756ce6132c54826661a32d4e4d132e1977adf"}, + {file = "cffi-1.15.1-cp36-cp36m-win_amd64.whl", hash = "sha256:30d78fbc8ebf9c92c9b7823ee18eb92f2e6ef79b45ac84db507f52fbe3ec4497"}, + {file = "cffi-1.15.1-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:198caafb44239b60e252492445da556afafc7d1e3ab7a1fb3f0584ef6d742375"}, + {file = "cffi-1.15.1-cp37-cp37m-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:5ef34d190326c3b1f822a5b7a45f6c4535e2f47ed06fec77d3d799c450b2651e"}, + {file = "cffi-1.15.1-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:8102eaf27e1e448db915d08afa8b41d6c7ca7a04b7d73af6514df10a3e74bd82"}, + {file = "cffi-1.15.1-cp37-cp37m-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:5df2768244d19ab7f60546d0c7c63ce1581f7af8b5de3eb3004b9b6fc8a9f84b"}, + {file = "cffi-1.15.1-cp37-cp37m-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:a8c4917bd7ad33e8eb21e9a5bbba979b49d9a97acb3a803092cbc1133e20343c"}, + {file = "cffi-1.15.1-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:0e2642fe3142e4cc4af0799748233ad6da94c62a8bec3a6648bf8ee68b1c7426"}, + {file = "cffi-1.15.1-cp37-cp37m-win32.whl", hash = "sha256:e229a521186c75c8ad9490854fd8bbdd9a0c9aa3a524326b55be83b54d4e0ad9"}, + {file = "cffi-1.15.1-cp37-cp37m-win_amd64.whl", hash = "sha256:a0b71b1b8fbf2b96e41c4d990244165e2c9be83d54962a9a1d118fd8657d2045"}, + {file = "cffi-1.15.1-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:320dab6e7cb2eacdf0e658569d2575c4dad258c0fcc794f46215e1e39f90f2c3"}, + {file = "cffi-1.15.1-cp38-cp38-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:1e74c6b51a9ed6589199c787bf5f9875612ca4a8a0785fb2d4a84429badaf22a"}, + {file = "cffi-1.15.1-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a5c84c68147988265e60416b57fc83425a78058853509c1b0629c180094904a5"}, + {file = "cffi-1.15.1-cp38-cp38-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:3b926aa83d1edb5aa5b427b4053dc420ec295a08e40911296b9eb1b6170f6cca"}, + {file = "cffi-1.15.1-cp38-cp38-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:87c450779d0914f2861b8526e035c5e6da0a3199d8f1add1a665e1cbc6fc6d02"}, + {file = "cffi-1.15.1-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:4f2c9f67e9821cad2e5f480bc8d83b8742896f1242dba247911072d4fa94c192"}, + {file = "cffi-1.15.1-cp38-cp38-win32.whl", hash = "sha256:8b7ee99e510d7b66cdb6c593f21c043c248537a32e0bedf02e01e9553a172314"}, + {file = "cffi-1.15.1-cp38-cp38-win_amd64.whl", hash = "sha256:00a9ed42e88df81ffae7a8ab6d9356b371399b91dbdf0c3cb1e84c03a13aceb5"}, + {file = "cffi-1.15.1-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:54a2db7b78338edd780e7ef7f9f6c442500fb0d41a5a4ea24fff1c929d5af585"}, + {file = "cffi-1.15.1-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:fcd131dd944808b5bdb38e6f5b53013c5aa4f334c5cad0c72742f6eba4b73db0"}, + {file = "cffi-1.15.1-cp39-cp39-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:7473e861101c9e72452f9bf8acb984947aa1661a7704553a9f6e4baa5ba64415"}, + {file = "cffi-1.15.1-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:6c9a799e985904922a4d207a94eae35c78ebae90e128f0c4e521ce339396be9d"}, + {file = "cffi-1.15.1-cp39-cp39-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:3bcde07039e586f91b45c88f8583ea7cf7a0770df3a1649627bf598332cb6984"}, + {file = "cffi-1.15.1-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:33ab79603146aace82c2427da5ca6e58f2b3f2fb5da893ceac0c42218a40be35"}, + {file = "cffi-1.15.1-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:5d598b938678ebf3c67377cdd45e09d431369c3b1a5b331058c338e201f12b27"}, + {file = "cffi-1.15.1-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:db0fbb9c62743ce59a9ff687eb5f4afbe77e5e8403d6697f7446e5f609976f76"}, + {file = "cffi-1.15.1-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:98d85c6a2bef81588d9227dde12db8a7f47f639f4a17c9ae08e773aa9c697bf3"}, + {file = "cffi-1.15.1-cp39-cp39-win32.whl", hash = "sha256:40f4774f5a9d4f5e344f31a32b5096977b5d48560c5592e2f3d2c4374bd543ee"}, + {file = "cffi-1.15.1-cp39-cp39-win_amd64.whl", hash = "sha256:70df4e3b545a17496c9b3f41f5115e69a4f2e77e94e1d2a8e1070bc0c38c8a3c"}, + {file = "cffi-1.15.1.tar.gz", hash = "sha256:d400bfb9a37b1351253cb402671cea7e89bdecc294e8016a707f6d1d8ac934f9"}, +] [package.dependencies] pycparser = "*" @@ -250,6 +466,10 @@ description = "Validate configuration and produce human readable error messages. category = "dev" optional = false python-versions = ">=3.6.1" +files = [ + {file = "cfgv-3.3.1-py2.py3-none-any.whl", hash = "sha256:c6a0883f3917a037485059700b9e75da2464e6c27051014ad85ba6aaa5884426"}, + {file = "cfgv-3.3.1.tar.gz", hash = "sha256:f5a830efb9ce7a445376bb66ec94c638a9787422f96264c98edc6bdeed8ab736"}, +] [[package]] name = "charset-normalizer" @@ -258,9 +478,13 @@ description = "The Real First Universal Charset Detector. Open, modern and activ category = "main" optional = false python-versions = ">=3.6.0" +files = [ + {file = "charset-normalizer-2.1.1.tar.gz", hash = "sha256:5a3d016c7c547f69d6f81fb0db9449ce888b418b5b9952cc5e6e66843e9dd845"}, + {file = "charset_normalizer-2.1.1-py3-none-any.whl", hash = "sha256:83e9a75d1911279afd89352c68b45348559d1fc0506b054b346651b5e7fee29f"}, +] [package.extras] -unicode_backport = ["unicodedata2"] +unicode-backport = ["unicodedata2"] [[package]] name = "click" @@ -269,6 +493,10 @@ description = "Composable command line interface toolkit" category = "main" optional = false python-versions = ">=3.7" +files = [ + {file = "click-8.1.3-py3-none-any.whl", hash = "sha256:bb4d8133cb15a609f44e8213d9b391b0809795062913b383c62be0ee95b1db48"}, + {file = "click-8.1.3.tar.gz", hash = "sha256:7682dc8afb30297001674575ea00d1814d808d6a36af415a82bd481d37ba7b8e"}, +] [package.dependencies] colorama = {version = "*", markers = "platform_system == \"Windows\""} @@ -280,6 +508,10 @@ description = "Cross-platform colored terminal text." category = "main" optional = false python-versions = "!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,!=3.4.*,!=3.5.*,!=3.6.*,>=2.7" +files = [ + {file = "colorama-0.4.6-py2.py3-none-any.whl", hash = "sha256:4f1d9991f5acc0ca119f9d443620b77f9d6b33703e51011c16baf57afb285fc6"}, + {file = "colorama-0.4.6.tar.gz", hash = "sha256:08695f5cb7ed6e0531a20572697297273c47b8cae5a63ffc6d6ed5c201be6e44"}, +] [[package]] name = "coverage" @@ -288,1137 +520,7 @@ description = "Code coverage measurement for Python" category = "dev" optional = false python-versions = ">=3.7" - -[package.dependencies] -tomli = {version = "*", optional = true, markers = "python_full_version <= \"3.11.0a6\" and extra == \"toml\""} - -[package.extras] -toml = ["tomli"] - -[[package]] -name = "cryptography" -version = "39.0.0" -description = "cryptography is a package which provides cryptographic recipes and primitives to Python developers." -category = "main" -optional = false -python-versions = ">=3.6" - -[package.dependencies] -cffi = ">=1.12" - -[package.extras] -docs = ["sphinx (>=1.6.5,!=1.8.0,!=3.1.0,!=3.1.1,!=5.2.0,!=5.2.0.post0)", "sphinx-rtd-theme"] -docstest = ["pyenchant (>=1.6.11)", "sphinxcontrib-spelling (>=4.0.1)", "twine (>=1.12.0)"] -pep8test = ["black", "ruff"] -sdist = ["setuptools-rust (>=0.11.4)"] -ssh = ["bcrypt (>=3.1.5)"] -test = ["hypothesis (>=1.11.4,!=3.79.2)", "iso8601", "pretend", "pytest (>=6.2.0)", "pytest-benchmark", "pytest-cov", "pytest-subtests", "pytest-xdist", "pytz"] - -[[package]] -name = "distlib" -version = "0.3.6" -description = "Distribution utilities" -category = "dev" -optional = false -python-versions = "*" - -[[package]] -name = "docutils" -version = "0.19" -description = "Docutils -- Python Documentation Utilities" -category = "dev" -optional = false -python-versions = ">=3.7" - -[[package]] -name = "duckdb" -version = "0.6.1" -description = "DuckDB embedded database" -category = "main" -optional = true -python-versions = "*" - -[package.dependencies] -numpy = ">=1.14" - -[[package]] -name = "exceptiongroup" -version = "1.1.0" -description = "Backport of PEP 654 (exception groups)" -category = "dev" -optional = false -python-versions = ">=3.7" - -[package.extras] -test = ["pytest (>=6)"] - -[[package]] -name = "fastavro" -version = "1.7.0" -description = "Fast read/write of AVRO files" -category = "dev" -optional = false -python-versions = ">=3.7" - -[package.extras] -codecs = ["lz4", "python-snappy", "zstandard"] -lz4 = ["lz4"] -snappy = ["python-snappy"] -zstandard = ["zstandard"] - -[[package]] -name = "filelock" -version = "3.9.0" -description = "A platform independent file lock." -category = "dev" -optional = false -python-versions = ">=3.7" - -[package.extras] -docs = ["furo (>=2022.12.7)", "sphinx (>=5.3)", "sphinx-autodoc-typehints (>=1.19.5)"] -testing = ["covdefaults (>=2.2.2)", "coverage (>=7.0.1)", "pytest (>=7.2)", "pytest-cov (>=4)", "pytest-timeout (>=2.1)"] - -[[package]] -name = "frozenlist" -version = "1.3.3" -description = "A list-like structure which implements collections.abc.MutableSequence" -category = "main" -optional = true -python-versions = ">=3.7" - -[[package]] -name = "fsspec" -version = "2023.1.0" -description = "File-system specification" -category = "main" -optional = false -python-versions = ">=3.7" - -[package.extras] -abfs = ["adlfs"] -adl = ["adlfs"] -arrow = ["pyarrow (>=1)"] -dask = ["dask", "distributed"] -dropbox = ["dropbox", "dropboxdrivefs", "requests"] -entrypoints = ["importlib-metadata"] -fuse = ["fusepy"] -gcs = ["gcsfs"] -git = ["pygit2"] -github = ["requests"] -gs = ["gcsfs"] -gui = ["panel"] -hdfs = ["pyarrow (>=1)"] -http = ["aiohttp (!=4.0.0a0,!=4.0.0a1)", "requests"] -libarchive = ["libarchive-c"] -oci = ["ocifs"] -s3 = ["s3fs"] -sftp = ["paramiko"] -smb = ["smbprotocol"] -ssh = ["paramiko"] -tqdm = ["tqdm"] - -[[package]] -name = "identify" -version = "2.5.13" -description = "File identification library for Python" -category = "dev" -optional = false -python-versions = ">=3.7" - -[package.extras] -license = ["ukkonen"] - -[[package]] -name = "idna" -version = "3.4" -description = "Internationalized Domain Names in Applications (IDNA)" -category = "main" -optional = false -python-versions = ">=3.5" - -[[package]] -name = "importlib-metadata" -version = "6.0.0" -description = "Read metadata from Python packages" -category = "dev" -optional = false -python-versions = ">=3.7" - -[package.dependencies] -zipp = ">=0.5" - -[package.extras] -docs = ["furo", "jaraco.packaging (>=9)", "jaraco.tidelift (>=1.4)", "rst.linker (>=1.9)", "sphinx (>=3.5)", "sphinx-lint"] -perf = ["ipython"] -testing = ["flake8 (<5)", "flufl.flake8", "importlib-resources (>=1.3)", "packaging", "pyfakefs", "pytest (>=6)", "pytest-black (>=0.3.7)", "pytest-checkdocs (>=2.4)", "pytest-cov", "pytest-enabler (>=1.3)", "pytest-flake8", "pytest-mypy (>=0.9.1)", "pytest-perf (>=0.9.2)"] - -[[package]] -name = "iniconfig" -version = "2.0.0" -description = "brain-dead simple config-ini parsing" -category = "dev" -optional = false -python-versions = ">=3.7" - -[[package]] -name = "isodate" -version = "0.6.1" -description = "An ISO 8601 date/time/duration parser and formatter" -category = "main" -optional = true -python-versions = "*" - -[package.dependencies] -six = "*" - -[[package]] -name = "jinja2" -version = "3.1.2" -description = "A very fast and expressive template engine." -category = "dev" -optional = false -python-versions = ">=3.7" - -[package.dependencies] -MarkupSafe = ">=2.0" - -[package.extras] -i18n = ["Babel (>=2.7)"] - -[[package]] -name = "jmespath" -version = "1.0.1" -description = "JSON Matching Expressions" -category = "main" -optional = false -python-versions = ">=3.7" - -[[package]] -name = "markdown-it-py" -version = "2.1.0" -description = "Python port of markdown-it. Markdown parsing, done right!" -category = "main" -optional = false -python-versions = ">=3.7" - -[package.dependencies] -mdurl = ">=0.1,<1.0" - -[package.extras] -benchmarking = ["psutil", "pytest", "pytest-benchmark (>=3.2,<4.0)"] -code_style = ["pre-commit (==2.6)"] -compare = ["commonmark (>=0.9.1,<0.10.0)", "markdown (>=3.3.6,<3.4.0)", "mistletoe (>=0.8.1,<0.9.0)", "mistune (>=2.0.2,<2.1.0)", "panflute (>=2.1.3,<2.2.0)"] -linkify = ["linkify-it-py (>=1.0,<2.0)"] -plugins = ["mdit-py-plugins"] -profiling = ["gprof2dot"] -rtd = ["attrs", "myst-parser", "pyyaml", "sphinx", "sphinx-book-theme", "sphinx-copybutton", "sphinx-design"] -testing = ["coverage", "pytest", "pytest-cov", "pytest-regressions"] - -[[package]] -name = "markupsafe" -version = "2.1.2" -description = "Safely add untrusted strings to HTML/XML markup." -category = "dev" -optional = false -python-versions = ">=3.7" - -[[package]] -name = "mdurl" -version = "0.1.2" -description = "Markdown URL utilities" -category = "main" -optional = false -python-versions = ">=3.7" - -[[package]] -name = "mmhash3" -version = "3.0.1" -description = "Python wrapper for MurmurHash (MurmurHash3), a set of fast and robust hash functions." -category = "main" -optional = false -python-versions = "*" - -[[package]] -name = "moto" -version = "4.1.0" -description = "" -category = "dev" -optional = false -python-versions = ">=3.7" - -[package.dependencies] -boto3 = ">=1.9.201" -botocore = ">=1.12.201" -cryptography = ">=3.3.1" -Jinja2 = ">=2.10.1" -python-dateutil = ">=2.1,<3.0.0" -requests = ">=2.5" -responses = ">=0.13.0" -werkzeug = ">=0.5,<2.2.0 || >2.2.0,<2.2.1 || >2.2.1" -xmltodict = "*" - -[package.extras] -all = ["PyYAML (>=5.1)", "aws-xray-sdk (>=0.93,!=0.96)", "cfn-lint (>=0.40.0)", "docker (>=2.5.1)", "ecdsa (!=0.15)", "graphql-core", "jsondiff (>=1.1.2)", "openapi-spec-validator (>=0.2.8)", "pyparsing (>=3.0.7)", "python-jose[cryptography] (>=3.1.0,<4.0.0)", "setuptools", "sshpubkeys (>=3.1.0)"] -apigateway = ["PyYAML (>=5.1)", "ecdsa (!=0.15)", "openapi-spec-validator (>=0.2.8)", "python-jose[cryptography] (>=3.1.0,<4.0.0)"] -apigatewayv2 = ["PyYAML (>=5.1)"] -appsync = ["graphql-core"] -awslambda = ["docker (>=2.5.1)"] -batch = ["docker (>=2.5.1)"] -cloudformation = ["PyYAML (>=5.1)", "aws-xray-sdk (>=0.93,!=0.96)", "cfn-lint (>=0.40.0)", "docker (>=2.5.1)", "ecdsa (!=0.15)", "graphql-core", "jsondiff (>=1.1.2)", "openapi-spec-validator (>=0.2.8)", "pyparsing (>=3.0.7)", "python-jose[cryptography] (>=3.1.0,<4.0.0)", "setuptools", "sshpubkeys (>=3.1.0)"] -cognitoidp = ["ecdsa (!=0.15)", "python-jose[cryptography] (>=3.1.0,<4.0.0)"] -ds = ["sshpubkeys (>=3.1.0)"] -dynamodb = ["docker (>=2.5.1)"] -dynamodbstreams = ["docker (>=2.5.1)"] -ebs = ["sshpubkeys (>=3.1.0)"] -ec2 = ["sshpubkeys (>=3.1.0)"] -efs = ["sshpubkeys (>=3.1.0)"] -glue = ["pyparsing (>=3.0.7)"] -iotdata = ["jsondiff (>=1.1.2)"] -route53resolver = ["sshpubkeys (>=3.1.0)"] -s3 = ["PyYAML (>=5.1)"] -server = ["PyYAML (>=5.1)", "aws-xray-sdk (>=0.93,!=0.96)", "cfn-lint (>=0.40.0)", "docker (>=2.5.1)", "ecdsa (!=0.15)", "flask (!=2.2.0,!=2.2.1)", "flask-cors", "graphql-core", "jsondiff (>=1.1.2)", "openapi-spec-validator (>=0.2.8)", "pyparsing (>=3.0.7)", "python-jose[cryptography] (>=3.1.0,<4.0.0)", "setuptools", "sshpubkeys (>=3.1.0)"] -ssm = ["PyYAML (>=5.1)"] -xray = ["aws-xray-sdk (>=0.93,!=0.96)", "setuptools"] - -[[package]] -name = "msal" -version = "1.20.0" -description = "The Microsoft Authentication Library (MSAL) for Python library enables your app to access the Microsoft Cloud by supporting authentication of users with Microsoft Azure Active Directory accounts (AAD) and Microsoft Accounts (MSA) using industry standard OAuth2 and OpenID Connect." -category = "main" -optional = true -python-versions = "*" - -[package.dependencies] -cryptography = ">=0.6,<41" -PyJWT = {version = ">=1.0.0,<3", extras = ["crypto"]} -requests = ">=2.0.0,<3" - -[package.extras] -broker = ["pymsalruntime (>=0.11.2,<0.14)"] - -[[package]] -name = "msal-extensions" -version = "1.0.0" -description = "Microsoft Authentication Library extensions (MSAL EX) provides a persistence API that can save your data on disk, encrypted on Windows, macOS and Linux. Concurrent data access will be coordinated by a file lock mechanism." -category = "main" -optional = true -python-versions = "*" - -[package.dependencies] -msal = ">=0.4.1,<2.0.0" -portalocker = [ - {version = ">=1.0,<3", markers = "python_version >= \"3.5\" and platform_system != \"Windows\""}, - {version = ">=1.6,<3", markers = "python_version >= \"3.5\" and platform_system == \"Windows\""}, -] - -[[package]] -name = "msrest" -version = "0.7.1" -description = "AutoRest swagger generator Python client runtime." -category = "main" -optional = true -python-versions = ">=3.6" - -[package.dependencies] -azure-core = ">=1.24.0" -certifi = ">=2017.4.17" -isodate = ">=0.6.0" -requests = ">=2.16,<3.0" -requests-oauthlib = ">=0.5.0" - -[package.extras] -async = ["aiodns", "aiohttp (>=3.0)"] - -[[package]] -name = "multidict" -version = "6.0.4" -description = "multidict implementation" -category = "main" -optional = true -python-versions = ">=3.7" - -[[package]] -name = "nodeenv" -version = "1.7.0" -description = "Node.js virtual environment builder" -category = "dev" -optional = false -python-versions = ">=2.7,!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,!=3.4.*,!=3.5.*,!=3.6.*" - -[[package]] -name = "numpy" -version = "1.24.1" -description = "Fundamental package for array computing in Python" -category = "main" -optional = true -python-versions = ">=3.8" - -[[package]] -name = "oauthlib" -version = "3.2.2" -description = "A generic, spec-compliant, thorough implementation of the OAuth request-signing logic" -category = "main" -optional = true -python-versions = ">=3.6" - -[package.extras] -rsa = ["cryptography (>=3.0.0)"] -signals = ["blinker (>=1.4.0)"] -signedtoken = ["cryptography (>=3.0.0)", "pyjwt (>=2.0.0,<3)"] - -[[package]] -name = "packaging" -version = "23.0" -description = "Core utilities for Python packages" -category = "dev" -optional = false -python-versions = ">=3.7" - -[[package]] -name = "pandas" -version = "1.5.3" -description = "Powerful data structures for data analysis, time series, and statistics" -category = "main" -optional = true -python-versions = ">=3.8" - -[package.dependencies] -numpy = [ - {version = ">=1.20.3", markers = "python_version < \"3.10\""}, - {version = ">=1.21.0", markers = "python_version >= \"3.10\""}, - {version = ">=1.23.2", markers = "python_version >= \"3.11\""}, -] -python-dateutil = ">=2.8.1" -pytz = ">=2020.1" - -[package.extras] -test = ["hypothesis (>=5.5.3)", "pytest (>=6.0)", "pytest-xdist (>=1.31)"] - -[[package]] -name = "platformdirs" -version = "2.6.2" -description = "A small Python package for determining appropriate platform-specific dirs, e.g. a \"user data dir\"." -category = "dev" -optional = false -python-versions = ">=3.7" - -[package.extras] -docs = ["furo (>=2022.12.7)", "proselint (>=0.13)", "sphinx (>=5.3)", "sphinx-autodoc-typehints (>=1.19.5)"] -test = ["appdirs (==1.4.4)", "covdefaults (>=2.2.2)", "pytest (>=7.2)", "pytest-cov (>=4)", "pytest-mock (>=3.10)"] - -[[package]] -name = "pluggy" -version = "1.0.0" -description = "plugin and hook calling mechanisms for python" -category = "dev" -optional = false -python-versions = ">=3.6" - -[package.extras] -dev = ["pre-commit", "tox"] -testing = ["pytest", "pytest-benchmark"] - -[[package]] -name = "portalocker" -version = "2.7.0" -description = "Wraps the portalocker recipe for easy usage" -category = "main" -optional = true -python-versions = ">=3.5" - -[package.dependencies] -pywin32 = {version = ">=226", markers = "platform_system == \"Windows\""} - -[package.extras] -docs = ["sphinx (>=1.7.1)"] -redis = ["redis"] -tests = ["pytest (>=5.4.1)", "pytest-cov (>=2.8.1)", "pytest-mypy (>=0.8.0)", "pytest-timeout (>=2.1.0)", "redis", "sphinx (>=6.0.0)"] - -[[package]] -name = "pre-commit" -version = "2.21.0" -description = "A framework for managing and maintaining multi-language pre-commit hooks." -category = "dev" -optional = false -python-versions = ">=3.7" - -[package.dependencies] -cfgv = ">=2.0.0" -identify = ">=1.0.0" -nodeenv = ">=0.11.1" -pyyaml = ">=5.1" -virtualenv = ">=20.10.0" - -[[package]] -name = "pyarrow" -version = "10.0.1" -description = "Python library for Apache Arrow" -category = "main" -optional = true -python-versions = ">=3.7" - -[package.dependencies] -numpy = ">=1.16.6" - -[[package]] -name = "pycparser" -version = "2.21" -description = "C parser in Python" -category = "main" -optional = false -python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*" - -[[package]] -name = "pydantic" -version = "1.10.4" -description = "Data validation and settings management using python type hints" -category = "main" -optional = false -python-versions = ">=3.7" - -[package.dependencies] -typing-extensions = ">=4.2.0" - -[package.extras] -dotenv = ["python-dotenv (>=0.10.4)"] -email = ["email-validator (>=1.0.3)"] - -[[package]] -name = "pygments" -version = "2.14.0" -description = "Pygments is a syntax highlighting package written in Python." -category = "main" -optional = false -python-versions = ">=3.6" - -[package.extras] -plugins = ["importlib-metadata"] - -[[package]] -name = "pyjwt" -version = "2.6.0" -description = "JSON Web Token implementation in Python" -category = "main" -optional = true -python-versions = ">=3.7" - -[package.dependencies] -cryptography = {version = ">=3.4.0", optional = true, markers = "extra == \"crypto\""} - -[package.extras] -crypto = ["cryptography (>=3.4.0)"] -dev = ["coverage[toml] (==5.0.4)", "cryptography (>=3.4.0)", "pre-commit", "pytest (>=6.0.0,<7.0.0)", "sphinx (>=4.5.0,<5.0.0)", "sphinx-rtd-theme", "zope.interface"] -docs = ["sphinx (>=4.5.0,<5.0.0)", "sphinx-rtd-theme", "zope.interface"] -tests = ["coverage[toml] (==5.0.4)", "pytest (>=6.0.0,<7.0.0)"] - -[[package]] -name = "pyparsing" -version = "3.0.9" -description = "pyparsing module - Classes and methods to define and execute parsing grammars" -category = "main" -optional = false -python-versions = ">=3.6.8" - -[package.extras] -diagrams = ["jinja2", "railroad-diagrams"] - -[[package]] -name = "pyproject-hooks" -version = "1.0.0" -description = "Wrappers to call pyproject.toml-based build backend hooks." -category = "dev" -optional = false -python-versions = ">=3.7" - -[package.dependencies] -tomli = {version = ">=1.1.0", markers = "python_version < \"3.11\""} - -[[package]] -name = "pytest" -version = "7.2.1" -description = "pytest: simple powerful testing with Python" -category = "dev" -optional = false -python-versions = ">=3.7" - -[package.dependencies] -attrs = ">=19.2.0" -colorama = {version = "*", markers = "sys_platform == \"win32\""} -exceptiongroup = {version = ">=1.0.0rc8", markers = "python_version < \"3.11\""} -iniconfig = "*" -packaging = "*" -pluggy = ">=0.12,<2.0" -tomli = {version = ">=1.0.0", markers = "python_version < \"3.11\""} - -[package.extras] -testing = ["argcomplete", "hypothesis (>=3.56)", "mock", "nose", "pygments (>=2.7.2)", "requests", "xmlschema"] - -[[package]] -name = "pytest-checkdocs" -version = "2.9.0" -description = "check the README when running tests" -category = "dev" -optional = false -python-versions = ">=3.7" - -[package.dependencies] -build = "*" -docutils = ">=0.15" -importlib-metadata = {version = ">=4", markers = "python_version < \"3.10\""} - -[package.extras] -docs = ["furo", "jaraco.packaging (>=9)", "rst.linker (>=1.9)", "sphinx (>=3.5)"] -testing = ["flake8 (<5)", "pytest (>=6)", "pytest-black (>=0.3.7)", "pytest-checkdocs (>=2.4)", "pytest-cov", "pytest-enabler (>=1.3)", "pytest-flake8", "pytest-mypy (>=0.9.1)", "types-docutils"] - -[[package]] -name = "python-dateutil" -version = "2.8.2" -description = "Extensions to the standard Python datetime module" -category = "main" -optional = false -python-versions = "!=3.0.*,!=3.1.*,!=3.2.*,>=2.7" - -[package.dependencies] -six = ">=1.5" - -[[package]] -name = "python-snappy" -version = "0.6.1" -description = "Python library for the snappy compression library from Google" -category = "main" -optional = true -python-versions = "*" - -[[package]] -name = "pytz" -version = "2022.7.1" -description = "World timezone definitions, modern and historical" -category = "main" -optional = true -python-versions = "*" - -[[package]] -name = "pywin32" -version = "305" -description = "Python for Window Extensions" -category = "main" -optional = true -python-versions = "*" - -[[package]] -name = "pyyaml" -version = "6.0" -description = "YAML parser and emitter for Python" -category = "main" -optional = false -python-versions = ">=3.6" - -[[package]] -name = "requests" -version = "2.28.2" -description = "Python HTTP for Humans." -category = "main" -optional = false -python-versions = ">=3.7, <4" - -[package.dependencies] -certifi = ">=2017.4.17" -charset-normalizer = ">=2,<4" -idna = ">=2.5,<4" -urllib3 = ">=1.21.1,<1.27" - -[package.extras] -socks = ["PySocks (>=1.5.6,!=1.5.7)"] -use_chardet_on_py3 = ["chardet (>=3.0.2,<6)"] - -[[package]] -name = "requests-mock" -version = "1.10.0" -description = "Mock out responses from the requests package" -category = "dev" -optional = false -python-versions = "*" - -[package.dependencies] -requests = ">=2.3,<3" -six = "*" - -[package.extras] -fixture = ["fixtures"] -test = ["fixtures", "mock", "purl", "pytest", "requests-futures", "sphinx", "testrepository (>=0.0.18)", "testtools"] - -[[package]] -name = "requests-oauthlib" -version = "1.3.1" -description = "OAuthlib authentication support for Requests." -category = "main" -optional = true -python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*" - -[package.dependencies] -oauthlib = ">=3.0.0" -requests = ">=2.0.0" - -[package.extras] -rsa = ["oauthlib[signedtoken] (>=3.0.0)"] - -[[package]] -name = "responses" -version = "0.22.0" -description = "A utility library for mocking out the `requests` Python library." -category = "dev" -optional = false -python-versions = ">=3.7" - -[package.dependencies] -requests = ">=2.22.0,<3.0" -toml = "*" -types-toml = "*" -urllib3 = ">=1.25.10" - -[package.extras] -tests = ["coverage (>=6.0.0)", "flake8", "mypy", "pytest (>=7.0.0)", "pytest-asyncio", "pytest-cov", "pytest-httpserver", "types-requests"] - -[[package]] -name = "rich" -version = "13.2.0" -description = "Render rich text, tables, progress bars, syntax highlighting, markdown and more to the terminal" -category = "main" -optional = false -python-versions = ">=3.7.0" - -[package.dependencies] -markdown-it-py = ">=2.1.0,<3.0.0" -pygments = ">=2.6.0,<3.0.0" -typing-extensions = {version = ">=4.0.0,<5.0", markers = "python_version < \"3.9\""} - -[package.extras] -jupyter = ["ipywidgets (>=7.5.1,<8.0.0)"] - -[[package]] -name = "s3fs" -version = "2023.1.0" -description = "Convenient Filesystem interface over S3" -category = "main" -optional = true -python-versions = ">= 3.7" - -[package.dependencies] -aiobotocore = ">=2.4.2,<2.5.0" -aiohttp = "<4.0.0a0 || >4.0.0a0,<4.0.0a1 || >4.0.0a1" -fsspec = "2023.1.0" - -[package.extras] -awscli = ["aiobotocore[awscli] (>=2.4.2,<2.5.0)"] -boto3 = ["aiobotocore[boto3] (>=2.4.2,<2.5.0)"] - -[[package]] -name = "s3transfer" -version = "0.6.0" -description = "An Amazon S3 Transfer Manager" -category = "main" -optional = false -python-versions = ">= 3.7" - -[package.dependencies] -botocore = ">=1.12.36,<2.0a.0" - -[package.extras] -crt = ["botocore[crt] (>=1.20.29,<2.0a.0)"] - -[[package]] -name = "six" -version = "1.16.0" -description = "Python 2 and 3 compatibility utilities" -category = "main" -optional = false -python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*" - -[[package]] -name = "thrift" -version = "0.16.0" -description = "Python bindings for the Apache Thrift RPC system" -category = "main" -optional = true -python-versions = "*" - -[package.dependencies] -six = ">=1.7.2" - -[package.extras] -all = ["tornado (>=4.0)", "twisted"] -tornado = ["tornado (>=4.0)"] -twisted = ["twisted"] - -[[package]] -name = "toml" -version = "0.10.2" -description = "Python Library for Tom's Obvious, Minimal Language" -category = "dev" -optional = false -python-versions = ">=2.6, !=3.0.*, !=3.1.*, !=3.2.*" - -[[package]] -name = "tomli" -version = "2.0.1" -description = "A lil' TOML parser" -category = "dev" -optional = false -python-versions = ">=3.7" - -[[package]] -name = "types-toml" -version = "0.10.8.1" -description = "Typing stubs for toml" -category = "dev" -optional = false -python-versions = "*" - -[[package]] -name = "typing-extensions" -version = "4.4.0" -description = "Backported and Experimental Type Hints for Python 3.7+" -category = "main" -optional = false -python-versions = ">=3.7" - -[[package]] -name = "urllib3" -version = "1.26.14" -description = "HTTP library with thread-safe connection pooling, file post, and more." -category = "main" -optional = false -python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*, !=3.5.*" - -[package.extras] -brotli = ["brotli (>=1.0.9)", "brotlicffi (>=0.8.0)", "brotlipy (>=0.6.0)"] -secure = ["certifi", "cryptography (>=1.3.4)", "idna (>=2.0.0)", "ipaddress", "pyOpenSSL (>=0.14)", "urllib3-secure-extra"] -socks = ["PySocks (>=1.5.6,!=1.5.7,<2.0)"] - -[[package]] -name = "virtualenv" -version = "20.17.1" -description = "Virtual Python Environment builder" -category = "dev" -optional = false -python-versions = ">=3.6" - -[package.dependencies] -distlib = ">=0.3.6,<1" -filelock = ">=3.4.1,<4" -platformdirs = ">=2.4,<3" - -[package.extras] -docs = ["proselint (>=0.13)", "sphinx (>=5.3)", "sphinx-argparse (>=0.3.2)", "sphinx-rtd-theme (>=1)", "towncrier (>=22.8)"] -testing = ["coverage (>=6.2)", "coverage-enable-subprocess (>=1)", "flaky (>=3.7)", "packaging (>=21.3)", "pytest (>=7.0.1)", "pytest-env (>=0.6.2)", "pytest-freezegun (>=0.4.2)", "pytest-mock (>=3.6.1)", "pytest-randomly (>=3.10.3)", "pytest-timeout (>=2.1)"] - -[[package]] -name = "werkzeug" -version = "2.2.2" -description = "The comprehensive WSGI web application library." -category = "dev" -optional = false -python-versions = ">=3.7" - -[package.dependencies] -MarkupSafe = ">=2.1.1" - -[package.extras] -watchdog = ["watchdog"] - -[[package]] -name = "wrapt" -version = "1.14.1" -description = "Module for decorators, wrappers and monkey patching." -category = "main" -optional = true -python-versions = "!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,!=3.4.*,>=2.7" - -[[package]] -name = "xmltodict" -version = "0.13.0" -description = "Makes working with XML feel like you are working with JSON" -category = "dev" -optional = false -python-versions = ">=3.4" - -[[package]] -name = "yarl" -version = "1.8.2" -description = "Yet another URL library" -category = "main" -optional = true -python-versions = ">=3.7" - -[package.dependencies] -idna = ">=2.0" -multidict = ">=4.0" - -[[package]] -name = "zipp" -version = "3.11.0" -description = "Backport of pathlib-compatible object wrapper for zip files" -category = "dev" -optional = false -python-versions = ">=3.7" - -[package.extras] -docs = ["furo", "jaraco.packaging (>=9)", "jaraco.tidelift (>=1.4)", "rst.linker (>=1.9)", "sphinx (>=3.5)"] -testing = ["flake8 (<5)", "func-timeout", "jaraco.functools", "jaraco.itertools", "more-itertools", "pytest (>=6)", "pytest-black (>=0.3.7)", "pytest-checkdocs (>=2.4)", "pytest-cov", "pytest-enabler (>=1.3)", "pytest-flake8", "pytest-mypy (>=0.9.1)"] - -[[package]] -name = "zstandard" -version = "0.19.0" -description = "Zstandard bindings for Python" -category = "main" -optional = false -python-versions = ">=3.6" - -[package.dependencies] -cffi = {version = ">=1.11", markers = "platform_python_implementation == \"PyPy\""} - -[package.extras] -cffi = ["cffi (>=1.11)"] - -[extras] -adlfs = ["adlfs"] -duckdb = ["duckdb", "pyarrow"] -glue = ["boto3"] -hive = ["thrift"] -pandas = ["pandas", "pyarrow"] -pyarrow = ["pyarrow"] -s3fs = ["s3fs"] -snappy = ["python-snappy"] - -[metadata] -lock-version = "1.1" -python-versions = "^3.8" -content-hash = "dd8d3af780197201f569d17aeff10b22d5b807bdb7949d5e9e234851b88ce0ce" - -[metadata.files] -adal = [ - {file = "adal-1.2.7-py2.py3-none-any.whl", hash = "sha256:2a7451ed7441ddbc57703042204a3e30ef747478eea022c70f789fc7f084bc3d"}, - {file = "adal-1.2.7.tar.gz", hash = "sha256:d74f45b81317454d96e982fd1c50e6fb5c99ac2223728aea8764433a39f566f1"}, -] -adlfs = [ - {file = "adlfs-2023.1.0-py3-none-any.whl", hash = "sha256:ccbdd6d33d5b7bce99a4a07c884c82fb135745d39b2d9b7b672f8e9d4d04407f"}, - {file = "adlfs-2023.1.0.tar.gz", hash = "sha256:eca53f53d88fc8e2e7a2f1d8f5a40b8a1c56aa3b541e4aa2e7eaf55a6a789262"}, -] -aiobotocore = [ - {file = "aiobotocore-2.4.2-py3-none-any.whl", hash = "sha256:4acd1ebe2e44be4b100aa553910bda899f6dc090b3da2bc1cf3d5de2146ed208"}, - {file = "aiobotocore-2.4.2.tar.gz", hash = "sha256:0603b74a582dffa7511ce7548d07dc9b10ec87bc5fb657eb0b34f9bd490958bf"}, -] -aiohttp = [ - {file = "aiohttp-3.8.3-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:ba71c9b4dcbb16212f334126cc3d8beb6af377f6703d9dc2d9fb3874fd667ee9"}, - {file = "aiohttp-3.8.3-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:d24b8bb40d5c61ef2d9b6a8f4528c2f17f1c5d2d31fed62ec860f6006142e83e"}, - {file = "aiohttp-3.8.3-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:f88df3a83cf9df566f171adba39d5bd52814ac0b94778d2448652fc77f9eb491"}, - {file = "aiohttp-3.8.3-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:b97decbb3372d4b69e4d4c8117f44632551c692bb1361b356a02b97b69e18a62"}, - {file = "aiohttp-3.8.3-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:309aa21c1d54b8ef0723181d430347d7452daaff93e8e2363db8e75c72c2fb2d"}, - {file = "aiohttp-3.8.3-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:ad5383a67514e8e76906a06741febd9126fc7c7ff0f599d6fcce3e82b80d026f"}, - {file = "aiohttp-3.8.3-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:20acae4f268317bb975671e375493dbdbc67cddb5f6c71eebdb85b34444ac46b"}, - {file = "aiohttp-3.8.3-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:05a3c31c6d7cd08c149e50dc7aa2568317f5844acd745621983380597f027a18"}, - {file = "aiohttp-3.8.3-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:d6f76310355e9fae637c3162936e9504b4767d5c52ca268331e2756e54fd4ca5"}, - {file = "aiohttp-3.8.3-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:256deb4b29fe5e47893fa32e1de2d73c3afe7407738bd3c63829874661d4822d"}, - {file = "aiohttp-3.8.3-cp310-cp310-musllinux_1_1_ppc64le.whl", hash = "sha256:5c59fcd80b9049b49acd29bd3598cada4afc8d8d69bd4160cd613246912535d7"}, - {file = "aiohttp-3.8.3-cp310-cp310-musllinux_1_1_s390x.whl", hash = "sha256:059a91e88f2c00fe40aed9031b3606c3f311414f86a90d696dd982e7aec48142"}, - {file = "aiohttp-3.8.3-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:2feebbb6074cdbd1ac276dbd737b40e890a1361b3cc30b74ac2f5e24aab41f7b"}, - {file = "aiohttp-3.8.3-cp310-cp310-win32.whl", hash = "sha256:5bf651afd22d5f0c4be16cf39d0482ea494f5c88f03e75e5fef3a85177fecdeb"}, - {file = "aiohttp-3.8.3-cp310-cp310-win_amd64.whl", hash = "sha256:653acc3880459f82a65e27bd6526e47ddf19e643457d36a2250b85b41a564715"}, - {file = "aiohttp-3.8.3-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:86fc24e58ecb32aee09f864cb11bb91bc4c1086615001647dbfc4dc8c32f4008"}, - {file = "aiohttp-3.8.3-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:75e14eac916f024305db517e00a9252714fce0abcb10ad327fb6dcdc0d060f1d"}, - {file = "aiohttp-3.8.3-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:d1fde0f44029e02d02d3993ad55ce93ead9bb9b15c6b7ccd580f90bd7e3de476"}, - {file = "aiohttp-3.8.3-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:4ab94426ddb1ecc6a0b601d832d5d9d421820989b8caa929114811369673235c"}, - {file = "aiohttp-3.8.3-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:89d2e02167fa95172c017732ed7725bc8523c598757f08d13c5acca308e1a061"}, - {file = "aiohttp-3.8.3-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:02f9a2c72fc95d59b881cf38a4b2be9381b9527f9d328771e90f72ac76f31ad8"}, - {file = "aiohttp-3.8.3-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:9c7149272fb5834fc186328e2c1fa01dda3e1fa940ce18fded6d412e8f2cf76d"}, - {file = "aiohttp-3.8.3-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:512bd5ab136b8dc0ffe3fdf2dfb0c4b4f49c8577f6cae55dca862cd37a4564e2"}, - {file = "aiohttp-3.8.3-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:7018ecc5fe97027214556afbc7c502fbd718d0740e87eb1217b17efd05b3d276"}, - {file = "aiohttp-3.8.3-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:88c70ed9da9963d5496d38320160e8eb7e5f1886f9290475a881db12f351ab5d"}, - {file = "aiohttp-3.8.3-cp311-cp311-musllinux_1_1_ppc64le.whl", hash = "sha256:da22885266bbfb3f78218dc40205fed2671909fbd0720aedba39b4515c038091"}, - {file = "aiohttp-3.8.3-cp311-cp311-musllinux_1_1_s390x.whl", hash = "sha256:e65bc19919c910127c06759a63747ebe14f386cda573d95bcc62b427ca1afc73"}, - {file = "aiohttp-3.8.3-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:08c78317e950e0762c2983f4dd58dc5e6c9ff75c8a0efeae299d363d439c8e34"}, - {file = "aiohttp-3.8.3-cp311-cp311-win32.whl", hash = "sha256:45d88b016c849d74ebc6f2b6e8bc17cabf26e7e40c0661ddd8fae4c00f015697"}, - {file = "aiohttp-3.8.3-cp311-cp311-win_amd64.whl", hash = "sha256:96372fc29471646b9b106ee918c8eeb4cca423fcbf9a34daa1b93767a88a2290"}, - {file = "aiohttp-3.8.3-cp36-cp36m-macosx_10_9_x86_64.whl", hash = "sha256:c971bf3786b5fad82ce5ad570dc6ee420f5b12527157929e830f51c55dc8af77"}, - {file = "aiohttp-3.8.3-cp36-cp36m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:ff25f48fc8e623d95eca0670b8cc1469a83783c924a602e0fbd47363bb54aaca"}, - {file = "aiohttp-3.8.3-cp36-cp36m-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:e381581b37db1db7597b62a2e6b8b57c3deec95d93b6d6407c5b61ddc98aca6d"}, - {file = "aiohttp-3.8.3-cp36-cp36m-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:db19d60d846283ee275d0416e2a23493f4e6b6028825b51290ac05afc87a6f97"}, - {file = "aiohttp-3.8.3-cp36-cp36m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:25892c92bee6d9449ffac82c2fe257f3a6f297792cdb18ad784737d61e7a9a85"}, - {file = "aiohttp-3.8.3-cp36-cp36m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:398701865e7a9565d49189f6c90868efaca21be65c725fc87fc305906be915da"}, - {file = "aiohttp-3.8.3-cp36-cp36m-musllinux_1_1_aarch64.whl", hash = "sha256:4a4fbc769ea9b6bd97f4ad0b430a6807f92f0e5eb020f1e42ece59f3ecfc4585"}, - {file = "aiohttp-3.8.3-cp36-cp36m-musllinux_1_1_i686.whl", hash = "sha256:b29bfd650ed8e148f9c515474a6ef0ba1090b7a8faeee26b74a8ff3b33617502"}, - {file = "aiohttp-3.8.3-cp36-cp36m-musllinux_1_1_ppc64le.whl", hash = "sha256:1e56b9cafcd6531bab5d9b2e890bb4937f4165109fe98e2b98ef0dcfcb06ee9d"}, - {file = "aiohttp-3.8.3-cp36-cp36m-musllinux_1_1_s390x.whl", hash = "sha256:ec40170327d4a404b0d91855d41bfe1fe4b699222b2b93e3d833a27330a87a6d"}, - {file = "aiohttp-3.8.3-cp36-cp36m-musllinux_1_1_x86_64.whl", hash = "sha256:2df5f139233060578d8c2c975128fb231a89ca0a462b35d4b5fcf7c501ebdbe1"}, - {file = "aiohttp-3.8.3-cp36-cp36m-win32.whl", hash = "sha256:f973157ffeab5459eefe7b97a804987876dd0a55570b8fa56b4e1954bf11329b"}, - {file = "aiohttp-3.8.3-cp36-cp36m-win_amd64.whl", hash = "sha256:437399385f2abcd634865705bdc180c8314124b98299d54fe1d4c8990f2f9494"}, - {file = "aiohttp-3.8.3-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:09e28f572b21642128ef31f4e8372adb6888846f32fecb288c8b0457597ba61a"}, - {file = "aiohttp-3.8.3-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:6f3553510abdbec67c043ca85727396ceed1272eef029b050677046d3387be8d"}, - {file = "aiohttp-3.8.3-cp37-cp37m-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:e168a7560b7c61342ae0412997b069753f27ac4862ec7867eff74f0fe4ea2ad9"}, - {file = "aiohttp-3.8.3-cp37-cp37m-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:db4c979b0b3e0fa7e9e69ecd11b2b3174c6963cebadeecfb7ad24532ffcdd11a"}, - {file = "aiohttp-3.8.3-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:e164e0a98e92d06da343d17d4e9c4da4654f4a4588a20d6c73548a29f176abe2"}, - {file = "aiohttp-3.8.3-cp37-cp37m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:e8a78079d9a39ca9ca99a8b0ac2fdc0c4d25fc80c8a8a82e5c8211509c523363"}, - {file = "aiohttp-3.8.3-cp37-cp37m-musllinux_1_1_aarch64.whl", hash = "sha256:21b30885a63c3f4ff5b77a5d6caf008b037cb521a5f33eab445dc566f6d092cc"}, - {file = "aiohttp-3.8.3-cp37-cp37m-musllinux_1_1_i686.whl", hash = "sha256:4b0f30372cef3fdc262f33d06e7b411cd59058ce9174ef159ad938c4a34a89da"}, - {file = "aiohttp-3.8.3-cp37-cp37m-musllinux_1_1_ppc64le.whl", hash = "sha256:8135fa153a20d82ffb64f70a1b5c2738684afa197839b34cc3e3c72fa88d302c"}, - {file = "aiohttp-3.8.3-cp37-cp37m-musllinux_1_1_s390x.whl", hash = "sha256:ad61a9639792fd790523ba072c0555cd6be5a0baf03a49a5dd8cfcf20d56df48"}, - {file = "aiohttp-3.8.3-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:978b046ca728073070e9abc074b6299ebf3501e8dee5e26efacb13cec2b2dea0"}, - {file = "aiohttp-3.8.3-cp37-cp37m-win32.whl", hash = "sha256:0d2c6d8c6872df4a6ec37d2ede71eff62395b9e337b4e18efd2177de883a5033"}, - {file = "aiohttp-3.8.3-cp37-cp37m-win_amd64.whl", hash = "sha256:21d69797eb951f155026651f7e9362877334508d39c2fc37bd04ff55b2007091"}, - {file = "aiohttp-3.8.3-cp38-cp38-macosx_10_9_universal2.whl", hash = "sha256:2ca9af5f8f5812d475c5259393f52d712f6d5f0d7fdad9acdb1107dd9e3cb7eb"}, - {file = "aiohttp-3.8.3-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:1d90043c1882067f1bd26196d5d2db9aa6d268def3293ed5fb317e13c9413ea4"}, - {file = "aiohttp-3.8.3-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:d737fc67b9a970f3234754974531dc9afeea11c70791dcb7db53b0cf81b79784"}, - {file = "aiohttp-3.8.3-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:ebf909ea0a3fc9596e40d55d8000702a85e27fd578ff41a5500f68f20fd32e6c"}, - {file = "aiohttp-3.8.3-cp38-cp38-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:5835f258ca9f7c455493a57ee707b76d2d9634d84d5d7f62e77be984ea80b849"}, - {file = "aiohttp-3.8.3-cp38-cp38-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:da37dcfbf4b7f45d80ee386a5f81122501ec75672f475da34784196690762f4b"}, - {file = "aiohttp-3.8.3-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:87f44875f2804bc0511a69ce44a9595d5944837a62caecc8490bbdb0e18b1342"}, - {file = "aiohttp-3.8.3-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:527b3b87b24844ea7865284aabfab08eb0faf599b385b03c2aa91fc6edd6e4b6"}, - {file = "aiohttp-3.8.3-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:d5ba88df9aa5e2f806650fcbeedbe4f6e8736e92fc0e73b0400538fd25a4dd96"}, - {file = "aiohttp-3.8.3-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:e7b8813be97cab8cb52b1375f41f8e6804f6507fe4660152e8ca5c48f0436017"}, - {file = "aiohttp-3.8.3-cp38-cp38-musllinux_1_1_ppc64le.whl", hash = "sha256:2dea10edfa1a54098703cb7acaa665c07b4e7568472a47f4e64e6319d3821ccf"}, - {file = "aiohttp-3.8.3-cp38-cp38-musllinux_1_1_s390x.whl", hash = "sha256:713d22cd9643ba9025d33c4af43943c7a1eb8547729228de18d3e02e278472b6"}, - {file = "aiohttp-3.8.3-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:2d252771fc85e0cf8da0b823157962d70639e63cb9b578b1dec9868dd1f4f937"}, - {file = "aiohttp-3.8.3-cp38-cp38-win32.whl", hash = "sha256:66bd5f950344fb2b3dbdd421aaa4e84f4411a1a13fca3aeb2bcbe667f80c9f76"}, - {file = "aiohttp-3.8.3-cp38-cp38-win_amd64.whl", hash = "sha256:84b14f36e85295fe69c6b9789b51a0903b774046d5f7df538176516c3e422446"}, - {file = "aiohttp-3.8.3-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:16c121ba0b1ec2b44b73e3a8a171c4f999b33929cd2397124a8c7fcfc8cd9e06"}, - {file = "aiohttp-3.8.3-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:8d6aaa4e7155afaf994d7924eb290abbe81a6905b303d8cb61310a2aba1c68ba"}, - {file = "aiohttp-3.8.3-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:43046a319664a04b146f81b40e1545d4c8ac7b7dd04c47e40bf09f65f2437346"}, - {file = "aiohttp-3.8.3-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:599418aaaf88a6d02a8c515e656f6faf3d10618d3dd95866eb4436520096c84b"}, - {file = "aiohttp-3.8.3-cp39-cp39-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:92a2964319d359f494f16011e23434f6f8ef0434acd3cf154a6b7bec511e2fb7"}, - {file = "aiohttp-3.8.3-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:73a4131962e6d91109bca6536416aa067cf6c4efb871975df734f8d2fd821b37"}, - {file = "aiohttp-3.8.3-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:598adde339d2cf7d67beaccda3f2ce7c57b3b412702f29c946708f69cf8222aa"}, - {file = "aiohttp-3.8.3-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:75880ed07be39beff1881d81e4a907cafb802f306efd6d2d15f2b3c69935f6fb"}, - {file = "aiohttp-3.8.3-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:a0239da9fbafd9ff82fd67c16704a7d1bccf0d107a300e790587ad05547681c8"}, - {file = "aiohttp-3.8.3-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:4e3a23ec214e95c9fe85a58470b660efe6534b83e6cbe38b3ed52b053d7cb6ad"}, - {file = "aiohttp-3.8.3-cp39-cp39-musllinux_1_1_ppc64le.whl", hash = "sha256:47841407cc89a4b80b0c52276f3cc8138bbbfba4b179ee3acbd7d77ae33f7ac4"}, - {file = "aiohttp-3.8.3-cp39-cp39-musllinux_1_1_s390x.whl", hash = "sha256:54d107c89a3ebcd13228278d68f1436d3f33f2dd2af5415e3feaeb1156e1a62c"}, - {file = "aiohttp-3.8.3-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:c37c5cce780349d4d51739ae682dec63573847a2a8dcb44381b174c3d9c8d403"}, - {file = "aiohttp-3.8.3-cp39-cp39-win32.whl", hash = "sha256:f178d2aadf0166be4df834c4953da2d7eef24719e8aec9a65289483eeea9d618"}, - {file = "aiohttp-3.8.3-cp39-cp39-win_amd64.whl", hash = "sha256:88e5be56c231981428f4f506c68b6a46fa25c4123a2e86d156c58a8369d31ab7"}, - {file = "aiohttp-3.8.3.tar.gz", hash = "sha256:3828fb41b7203176b82fe5d699e0d845435f2374750a44b480ea6b930f6be269"}, -] -aioitertools = [ - {file = "aioitertools-0.11.0-py3-none-any.whl", hash = "sha256:04b95e3dab25b449def24d7df809411c10e62aab0cbe31a50ca4e68748c43394"}, - {file = "aioitertools-0.11.0.tar.gz", hash = "sha256:42c68b8dd3a69c2bf7f2233bf7df4bb58b557bca5252ac02ed5187bbc67d6831"}, -] -aiosignal = [ - {file = "aiosignal-1.3.1-py3-none-any.whl", hash = "sha256:f8376fb07dd1e86a584e4fcdec80b36b7f81aac666ebc724e2c090300dd83b17"}, - {file = "aiosignal-1.3.1.tar.gz", hash = "sha256:54cd96e15e1649b75d6c87526a6ff0b6c1b0dd3459f43d9ca11d48c339b68cfc"}, -] -async-timeout = [] -attrs = [ - {file = "attrs-22.2.0-py3-none-any.whl", hash = "sha256:29e95c7f6778868dbd49170f98f8818f78f3dc5e0e37c0b1f474e3561b240836"}, - {file = "attrs-22.2.0.tar.gz", hash = "sha256:c9227bfc2f01993c03f68db37d1d15c9690188323c067c641f1a35ca58185f99"}, -] -azure-core = [ - {file = "azure-core-1.26.2.zip", hash = "sha256:986bfd8687889782d79481d4c5d0af04ab4a18ca2f210364804a88e4eaa1586a"}, - {file = "azure_core-1.26.2-py3-none-any.whl", hash = "sha256:df306e6e4abc145610ca6744aef943129a6fd7a11977e56731f69ac0e00724f9"}, -] -azure-datalake-store = [ - {file = "azure-datalake-store-0.0.52.tar.gz", hash = "sha256:4198ddb32614d16d4502b43d5c9739f81432b7e0e4d75d30e05149fe6007fea2"}, - {file = "azure_datalake_store-0.0.52-py2.py3-none-any.whl", hash = "sha256:aaed72b9c856824aeab554f4dbe0ef2c6d0ff36700bdd8b93d8298793117c48e"}, -] -azure-identity = [ - {file = "azure-identity-1.12.0.zip", hash = "sha256:7f9b1ae7d97ea7af3f38dd09305e19ab81a1e16ab66ea186b6579d85c1ca2347"}, - {file = "azure_identity-1.12.0-py3-none-any.whl", hash = "sha256:2a58ce4a209a013e37eaccfd5937570ab99e9118b3e1acf875eed3a85d541b92"}, -] -azure-storage-blob = [ - {file = "azure-storage-blob-12.14.1.zip", hash = "sha256:860d4d82985a4bfc7d3271e71275af330f54f330a754355435a7ba749ccde997"}, - {file = "azure_storage_blob-12.14.1-py3-none-any.whl", hash = "sha256:52b84658e8df7853a3cf1c563814655b5028b979b2a87905b92aa6bb30be240e"}, -] -boto3 = [ - {file = "boto3-1.24.59-py3-none-any.whl", hash = "sha256:34ab44146a2c4e7f4e72737f4b27e6eb5e0a7855c2f4599e3d9199b6a0a2d575"}, - {file = "boto3-1.24.59.tar.gz", hash = "sha256:a50b4323f9579cfe22fcf5531fbd40b567d4d74c1adce06aeb5c95fce2a6fb40"}, -] -botocore = [ - {file = "botocore-1.27.59-py3-none-any.whl", hash = "sha256:69d756791fc024bda54f6c53f71ae34e695ee41bbbc1743d9179c4837a4929da"}, - {file = "botocore-1.27.59.tar.gz", hash = "sha256:eda4aed6ee719a745d1288eaf1beb12f6f6448ad1fa12f159405db14ba9c92cf"}, -] -build = [ - {file = "build-0.10.0-py3-none-any.whl", hash = "sha256:af266720050a66c893a6096a2f410989eeac74ff9a68ba194b3f6473e8e26171"}, - {file = "build-0.10.0.tar.gz", hash = "sha256:d5b71264afdb5951d6704482aac78de887c80691c52b88a9ad195983ca2c9269"}, -] -certifi = [ - {file = "certifi-2022.12.7-py3-none-any.whl", hash = "sha256:4ad3232f5e926d6718ec31cfc1fcadfde020920e278684144551c91769c7bc18"}, - {file = "certifi-2022.12.7.tar.gz", hash = "sha256:35824b4c3a97115964b408844d64aa14db1cc518f6562e8d7261699d1350a9e3"}, -] -cffi = [ - {file = "cffi-1.15.1-cp27-cp27m-macosx_10_9_x86_64.whl", hash = "sha256:a66d3508133af6e8548451b25058d5812812ec3798c886bf38ed24a98216fab2"}, - {file = "cffi-1.15.1-cp27-cp27m-manylinux1_i686.whl", hash = "sha256:470c103ae716238bbe698d67ad020e1db9d9dba34fa5a899b5e21577e6d52ed2"}, - {file = "cffi-1.15.1-cp27-cp27m-manylinux1_x86_64.whl", hash = "sha256:9ad5db27f9cabae298d151c85cf2bad1d359a1b9c686a275df03385758e2f914"}, - {file = "cffi-1.15.1-cp27-cp27m-win32.whl", hash = "sha256:b3bbeb01c2b273cca1e1e0c5df57f12dce9a4dd331b4fa1635b8bec26350bde3"}, - {file = "cffi-1.15.1-cp27-cp27m-win_amd64.whl", hash = "sha256:e00b098126fd45523dd056d2efba6c5a63b71ffe9f2bbe1a4fe1716e1d0c331e"}, - {file = "cffi-1.15.1-cp27-cp27mu-manylinux1_i686.whl", hash = "sha256:d61f4695e6c866a23a21acab0509af1cdfd2c013cf256bbf5b6b5e2695827162"}, - {file = "cffi-1.15.1-cp27-cp27mu-manylinux1_x86_64.whl", hash = "sha256:ed9cb427ba5504c1dc15ede7d516b84757c3e3d7868ccc85121d9310d27eed0b"}, - {file = "cffi-1.15.1-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:39d39875251ca8f612b6f33e6b1195af86d1b3e60086068be9cc053aa4376e21"}, - {file = "cffi-1.15.1-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:285d29981935eb726a4399badae8f0ffdff4f5050eaa6d0cfc3f64b857b77185"}, - {file = "cffi-1.15.1-cp310-cp310-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:3eb6971dcff08619f8d91607cfc726518b6fa2a9eba42856be181c6d0d9515fd"}, - {file = "cffi-1.15.1-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:21157295583fe8943475029ed5abdcf71eb3911894724e360acff1d61c1d54bc"}, - {file = "cffi-1.15.1-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:5635bd9cb9731e6d4a1132a498dd34f764034a8ce60cef4f5319c0541159392f"}, - {file = "cffi-1.15.1-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:2012c72d854c2d03e45d06ae57f40d78e5770d252f195b93f581acf3ba44496e"}, - {file = "cffi-1.15.1-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:dd86c085fae2efd48ac91dd7ccffcfc0571387fe1193d33b6394db7ef31fe2a4"}, - {file = "cffi-1.15.1-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:fa6693661a4c91757f4412306191b6dc88c1703f780c8234035eac011922bc01"}, - {file = "cffi-1.15.1-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:59c0b02d0a6c384d453fece7566d1c7e6b7bae4fc5874ef2ef46d56776d61c9e"}, - {file = "cffi-1.15.1-cp310-cp310-win32.whl", hash = "sha256:cba9d6b9a7d64d4bd46167096fc9d2f835e25d7e4c121fb2ddfc6528fb0413b2"}, - {file = "cffi-1.15.1-cp310-cp310-win_amd64.whl", hash = "sha256:ce4bcc037df4fc5e3d184794f27bdaab018943698f4ca31630bc7f84a7b69c6d"}, - {file = "cffi-1.15.1-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:3d08afd128ddaa624a48cf2b859afef385b720bb4b43df214f85616922e6a5ac"}, - {file = "cffi-1.15.1-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:3799aecf2e17cf585d977b780ce79ff0dc9b78d799fc694221ce814c2c19db83"}, - {file = "cffi-1.15.1-cp311-cp311-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:a591fe9e525846e4d154205572a029f653ada1a78b93697f3b5a8f1f2bc055b9"}, - {file = "cffi-1.15.1-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:3548db281cd7d2561c9ad9984681c95f7b0e38881201e157833a2342c30d5e8c"}, - {file = "cffi-1.15.1-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:91fc98adde3d7881af9b59ed0294046f3806221863722ba7d8d120c575314325"}, - {file = "cffi-1.15.1-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:94411f22c3985acaec6f83c6df553f2dbe17b698cc7f8ae751ff2237d96b9e3c"}, - {file = "cffi-1.15.1-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:03425bdae262c76aad70202debd780501fabeaca237cdfddc008987c0e0f59ef"}, - {file = "cffi-1.15.1-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:cc4d65aeeaa04136a12677d3dd0b1c0c94dc43abac5860ab33cceb42b801c1e8"}, - {file = "cffi-1.15.1-cp311-cp311-win32.whl", hash = "sha256:a0f100c8912c114ff53e1202d0078b425bee3649ae34d7b070e9697f93c5d52d"}, - {file = "cffi-1.15.1-cp311-cp311-win_amd64.whl", hash = "sha256:04ed324bda3cda42b9b695d51bb7d54b680b9719cfab04227cdd1e04e5de3104"}, - {file = "cffi-1.15.1-cp36-cp36m-macosx_10_9_x86_64.whl", hash = "sha256:50a74364d85fd319352182ef59c5c790484a336f6db772c1a9231f1c3ed0cbd7"}, - {file = "cffi-1.15.1-cp36-cp36m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:e263d77ee3dd201c3a142934a086a4450861778baaeeb45db4591ef65550b0a6"}, - {file = "cffi-1.15.1-cp36-cp36m-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:cec7d9412a9102bdc577382c3929b337320c4c4c4849f2c5cdd14d7368c5562d"}, - {file = "cffi-1.15.1-cp36-cp36m-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:4289fc34b2f5316fbb762d75362931e351941fa95fa18789191b33fc4cf9504a"}, - {file = "cffi-1.15.1-cp36-cp36m-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:173379135477dc8cac4bc58f45db08ab45d228b3363adb7af79436135d028405"}, - {file = "cffi-1.15.1-cp36-cp36m-manylinux_2_5_x86_64.manylinux1_x86_64.whl", hash = "sha256:6975a3fac6bc83c4a65c9f9fcab9e47019a11d3d2cf7f3c0d03431bf145a941e"}, - {file = "cffi-1.15.1-cp36-cp36m-win32.whl", hash = "sha256:2470043b93ff09bf8fb1d46d1cb756ce6132c54826661a32d4e4d132e1977adf"}, - {file = "cffi-1.15.1-cp36-cp36m-win_amd64.whl", hash = "sha256:30d78fbc8ebf9c92c9b7823ee18eb92f2e6ef79b45ac84db507f52fbe3ec4497"}, - {file = "cffi-1.15.1-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:198caafb44239b60e252492445da556afafc7d1e3ab7a1fb3f0584ef6d742375"}, - {file = "cffi-1.15.1-cp37-cp37m-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:5ef34d190326c3b1f822a5b7a45f6c4535e2f47ed06fec77d3d799c450b2651e"}, - {file = "cffi-1.15.1-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:8102eaf27e1e448db915d08afa8b41d6c7ca7a04b7d73af6514df10a3e74bd82"}, - {file = "cffi-1.15.1-cp37-cp37m-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:5df2768244d19ab7f60546d0c7c63ce1581f7af8b5de3eb3004b9b6fc8a9f84b"}, - {file = "cffi-1.15.1-cp37-cp37m-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:a8c4917bd7ad33e8eb21e9a5bbba979b49d9a97acb3a803092cbc1133e20343c"}, - {file = "cffi-1.15.1-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:0e2642fe3142e4cc4af0799748233ad6da94c62a8bec3a6648bf8ee68b1c7426"}, - {file = "cffi-1.15.1-cp37-cp37m-win32.whl", hash = "sha256:e229a521186c75c8ad9490854fd8bbdd9a0c9aa3a524326b55be83b54d4e0ad9"}, - {file = "cffi-1.15.1-cp37-cp37m-win_amd64.whl", hash = "sha256:a0b71b1b8fbf2b96e41c4d990244165e2c9be83d54962a9a1d118fd8657d2045"}, - {file = "cffi-1.15.1-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:320dab6e7cb2eacdf0e658569d2575c4dad258c0fcc794f46215e1e39f90f2c3"}, - {file = "cffi-1.15.1-cp38-cp38-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:1e74c6b51a9ed6589199c787bf5f9875612ca4a8a0785fb2d4a84429badaf22a"}, - {file = "cffi-1.15.1-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a5c84c68147988265e60416b57fc83425a78058853509c1b0629c180094904a5"}, - {file = "cffi-1.15.1-cp38-cp38-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:3b926aa83d1edb5aa5b427b4053dc420ec295a08e40911296b9eb1b6170f6cca"}, - {file = "cffi-1.15.1-cp38-cp38-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:87c450779d0914f2861b8526e035c5e6da0a3199d8f1add1a665e1cbc6fc6d02"}, - {file = "cffi-1.15.1-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:4f2c9f67e9821cad2e5f480bc8d83b8742896f1242dba247911072d4fa94c192"}, - {file = "cffi-1.15.1-cp38-cp38-win32.whl", hash = "sha256:8b7ee99e510d7b66cdb6c593f21c043c248537a32e0bedf02e01e9553a172314"}, - {file = "cffi-1.15.1-cp38-cp38-win_amd64.whl", hash = "sha256:00a9ed42e88df81ffae7a8ab6d9356b371399b91dbdf0c3cb1e84c03a13aceb5"}, - {file = "cffi-1.15.1-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:54a2db7b78338edd780e7ef7f9f6c442500fb0d41a5a4ea24fff1c929d5af585"}, - {file = "cffi-1.15.1-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:fcd131dd944808b5bdb38e6f5b53013c5aa4f334c5cad0c72742f6eba4b73db0"}, - {file = "cffi-1.15.1-cp39-cp39-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:7473e861101c9e72452f9bf8acb984947aa1661a7704553a9f6e4baa5ba64415"}, - {file = "cffi-1.15.1-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:6c9a799e985904922a4d207a94eae35c78ebae90e128f0c4e521ce339396be9d"}, - {file = "cffi-1.15.1-cp39-cp39-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:3bcde07039e586f91b45c88f8583ea7cf7a0770df3a1649627bf598332cb6984"}, - {file = "cffi-1.15.1-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:33ab79603146aace82c2427da5ca6e58f2b3f2fb5da893ceac0c42218a40be35"}, - {file = "cffi-1.15.1-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:5d598b938678ebf3c67377cdd45e09d431369c3b1a5b331058c338e201f12b27"}, - {file = "cffi-1.15.1-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:db0fbb9c62743ce59a9ff687eb5f4afbe77e5e8403d6697f7446e5f609976f76"}, - {file = "cffi-1.15.1-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:98d85c6a2bef81588d9227dde12db8a7f47f639f4a17c9ae08e773aa9c697bf3"}, - {file = "cffi-1.15.1-cp39-cp39-win32.whl", hash = "sha256:40f4774f5a9d4f5e344f31a32b5096977b5d48560c5592e2f3d2c4374bd543ee"}, - {file = "cffi-1.15.1-cp39-cp39-win_amd64.whl", hash = "sha256:70df4e3b545a17496c9b3f41f5115e69a4f2e77e94e1d2a8e1070bc0c38c8a3c"}, - {file = "cffi-1.15.1.tar.gz", hash = "sha256:d400bfb9a37b1351253cb402671cea7e89bdecc294e8016a707f6d1d8ac934f9"}, -] -cfgv = [ - {file = "cfgv-3.3.1-py2.py3-none-any.whl", hash = "sha256:c6a0883f3917a037485059700b9e75da2464e6c27051014ad85ba6aaa5884426"}, - {file = "cfgv-3.3.1.tar.gz", hash = "sha256:f5a830efb9ce7a445376bb66ec94c638a9787422f96264c98edc6bdeed8ab736"}, -] -charset-normalizer = [] -click = [] -colorama = [ - {file = "colorama-0.4.6-py2.py3-none-any.whl", hash = "sha256:4f1d9991f5acc0ca119f9d443620b77f9d6b33703e51011c16baf57afb285fc6"}, - {file = "colorama-0.4.6.tar.gz", hash = "sha256:08695f5cb7ed6e0531a20572697297273c47b8cae5a63ffc6d6ed5c201be6e44"}, -] -coverage = [ +files = [ {file = "coverage-7.0.5-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:2a7f23bbaeb2a87f90f607730b45564076d870f1fb07b9318d0c21f36871932b"}, {file = "coverage-7.0.5-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:c18d47f314b950dbf24a41787ced1474e01ca816011925976d90a88b27c22b89"}, {file = "coverage-7.0.5-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:ef14d75d86f104f03dea66c13188487151760ef25dd6b2dbd541885185f05f40"}, @@ -1471,7 +573,21 @@ coverage = [ {file = "coverage-7.0.5-pp37.pp38.pp39-none-any.whl", hash = "sha256:b9727ac4f5cf2cbf87880a63870b5b9730a8ae3a4a360241a0fdaa2f71240ff0"}, {file = "coverage-7.0.5.tar.gz", hash = "sha256:051afcbd6d2ac39298d62d340f94dbb6a1f31de06dfaf6fcef7b759dd3860c45"}, ] -cryptography = [ + +[package.dependencies] +tomli = {version = "*", optional = true, markers = "python_full_version <= \"3.11.0a6\" and extra == \"toml\""} + +[package.extras] +toml = ["tomli"] + +[[package]] +name = "cryptography" +version = "39.0.0" +description = "cryptography is a package which provides cryptographic recipes and primitives to Python developers." +category = "main" +optional = false +python-versions = ">=3.6" +files = [ {file = "cryptography-39.0.0-cp36-abi3-macosx_10_12_universal2.whl", hash = "sha256:c52a1a6f81e738d07f43dab57831c29e57d21c81a942f4602fac7ee21b27f288"}, {file = "cryptography-39.0.0-cp36-abi3-macosx_10_12_x86_64.whl", hash = "sha256:80ee674c08aaef194bc4627b7f2956e5ba7ef29c3cc3ca488cf15854838a8f72"}, {file = "cryptography-39.0.0-cp36-abi3-manylinux_2_17_aarch64.manylinux2014_aarch64.manylinux_2_24_aarch64.whl", hash = "sha256:887cbc1ea60786e534b00ba8b04d1095f4272d380ebd5f7a7eb4cc274710fad9"}, @@ -1496,15 +612,50 @@ cryptography = [ {file = "cryptography-39.0.0-pp39-pypy39_pp73-win_amd64.whl", hash = "sha256:e0a05aee6a82d944f9b4edd6a001178787d1546ec7c6223ee9a848a7ade92e39"}, {file = "cryptography-39.0.0.tar.gz", hash = "sha256:f964c7dcf7802d133e8dbd1565914fa0194f9d683d82411989889ecd701e8adf"}, ] -distlib = [ + +[package.dependencies] +cffi = ">=1.12" + +[package.extras] +docs = ["sphinx (>=1.6.5,!=1.8.0,!=3.1.0,!=3.1.1,!=5.2.0,!=5.2.0.post0)", "sphinx-rtd-theme"] +docstest = ["pyenchant (>=1.6.11)", "sphinxcontrib-spelling (>=4.0.1)", "twine (>=1.12.0)"] +pep8test = ["black", "ruff"] +sdist = ["setuptools-rust (>=0.11.4)"] +ssh = ["bcrypt (>=3.1.5)"] +test = ["hypothesis (>=1.11.4,!=3.79.2)", "iso8601", "pretend", "pytest (>=6.2.0)", "pytest-benchmark", "pytest-cov", "pytest-subtests", "pytest-xdist", "pytz"] + +[[package]] +name = "distlib" +version = "0.3.6" +description = "Distribution utilities" +category = "dev" +optional = false +python-versions = "*" +files = [ {file = "distlib-0.3.6-py2.py3-none-any.whl", hash = "sha256:f35c4b692542ca110de7ef0bea44d73981caeb34ca0b9b6b2e6d7790dda8f80e"}, {file = "distlib-0.3.6.tar.gz", hash = "sha256:14bad2d9b04d3a36127ac97f30b12a19268f211063d8f8ee4f47108896e11b46"}, ] -docutils = [ + +[[package]] +name = "docutils" +version = "0.19" +description = "Docutils -- Python Documentation Utilities" +category = "dev" +optional = false +python-versions = ">=3.7" +files = [ {file = "docutils-0.19-py3-none-any.whl", hash = "sha256:5e1de4d849fee02c63b040a4a3fd567f4ab104defd8a5511fbbc24a8a017efbc"}, {file = "docutils-0.19.tar.gz", hash = "sha256:33995a6753c30b7f577febfc2c50411fec6aac7f7ffeb7c4cfe5991072dcf9e6"}, ] -duckdb = [ + +[[package]] +name = "duckdb" +version = "0.6.1" +description = "DuckDB embedded database" +category = "main" +optional = true +python-versions = "*" +files = [ {file = "duckdb-0.6.1-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:e566514f9327f89264e98ac14ee7a84fbd9857328028258422c3e8375ee19d25"}, {file = "duckdb-0.6.1-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:b31c2883de5b19591a2852165e6b3f9821f77af649835f27bc146b26e4aa30cb"}, {file = "duckdb-0.6.1-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:998165b2fb1f1d2b0ad742096015ea70878f7d40304643c7424c3ed3ddf07bfc"}, @@ -1553,11 +704,33 @@ duckdb = [ {file = "duckdb-0.6.1-cp39-cp39-win_amd64.whl", hash = "sha256:00b7be8f67ec1a8edaa8844f521267baa1a795f4c482bfad56c72c26e1862ab2"}, {file = "duckdb-0.6.1.tar.gz", hash = "sha256:6d26e9f1afcb924a6057785e506810d48332d4764ddc4a5b414d0f2bf0cacfb4"}, ] -exceptiongroup = [ + +[package.dependencies] +numpy = ">=1.14" + +[[package]] +name = "exceptiongroup" +version = "1.1.0" +description = "Backport of PEP 654 (exception groups)" +category = "dev" +optional = false +python-versions = ">=3.7" +files = [ {file = "exceptiongroup-1.1.0-py3-none-any.whl", hash = "sha256:327cbda3da756e2de031a3107b81ab7b3770a602c4d16ca618298c526f4bec1e"}, {file = "exceptiongroup-1.1.0.tar.gz", hash = "sha256:bcb67d800a4497e1b404c2dd44fca47d3b7a5e5433dbab67f96c1a685cdfdf23"}, ] -fastavro = [ + +[package.extras] +test = ["pytest (>=6)"] + +[[package]] +name = "fastavro" +version = "1.7.0" +description = "Fast read/write of AVRO files" +category = "dev" +optional = false +python-versions = ">=3.7" +files = [ {file = "fastavro-1.7.0-cp310-cp310-macosx_10_14_x86_64.whl", hash = "sha256:ab3387a06e272980fa034f5c62f7063977b77df6416d3d30a4d3b49cc8827566"}, {file = "fastavro-1.7.0-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:216132bc54da19e97e1531dd69c86282408d4c797749d83b01b3a00862a180de"}, {file = "fastavro-1.7.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:c20cf6cd8098bb93c2cffd53d03ccea1dcf9ec594a5c83963acf29a2882f8693"}, @@ -1580,11 +753,37 @@ fastavro = [ {file = "fastavro-1.7.0-cp39-cp39-win_amd64.whl", hash = "sha256:492e8902063aa1c73170e11c18495fcaa86b71eae3513ef83ba438ca02b16b34"}, {file = "fastavro-1.7.0.tar.gz", hash = "sha256:4b1205f46489b4032d3155c1ab44d9824be0c7454df98d3a5bd22b78b98f23c8"}, ] -filelock = [ + +[package.extras] +codecs = ["lz4", "python-snappy", "zstandard"] +lz4 = ["lz4"] +snappy = ["python-snappy"] +zstandard = ["zstandard"] + +[[package]] +name = "filelock" +version = "3.9.0" +description = "A platform independent file lock." +category = "dev" +optional = false +python-versions = ">=3.7" +files = [ {file = "filelock-3.9.0-py3-none-any.whl", hash = "sha256:f58d535af89bb9ad5cd4df046f741f8553a418c01a7856bf0d173bbc9f6bd16d"}, {file = "filelock-3.9.0.tar.gz", hash = "sha256:7b319f24340b51f55a2bf7a12ac0755a9b03e718311dac567a0f4f7fabd2f5de"}, ] -frozenlist = [ + +[package.extras] +docs = ["furo (>=2022.12.7)", "sphinx (>=5.3)", "sphinx-autodoc-typehints (>=1.19.5)"] +testing = ["covdefaults (>=2.2.2)", "coverage (>=7.0.1)", "pytest (>=7.2)", "pytest-cov (>=4)", "pytest-timeout (>=2.1)"] + +[[package]] +name = "frozenlist" +version = "1.3.3" +description = "A list-like structure which implements collections.abc.MutableSequence" +category = "main" +optional = true +python-versions = ">=3.7" +files = [ {file = "frozenlist-1.3.3-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:ff8bf625fe85e119553b5383ba0fb6aa3d0ec2ae980295aaefa552374926b3f4"}, {file = "frozenlist-1.3.3-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:dfbac4c2dfcc082fcf8d942d1e49b6aa0766c19d3358bd86e2000bf0fa4a9cf0"}, {file = "frozenlist-1.3.3-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:b1c63e8d377d039ac769cd0926558bb7068a1f7abb0f003e3717ee003ad85530"}, @@ -1660,40 +859,179 @@ frozenlist = [ {file = "frozenlist-1.3.3-cp39-cp39-win_amd64.whl", hash = "sha256:cfe33efc9cb900a4c46f91a5ceba26d6df370ffddd9ca386eb1d4f0ad97b9ea9"}, {file = "frozenlist-1.3.3.tar.gz", hash = "sha256:58bcc55721e8a90b88332d6cd441261ebb22342e238296bb330968952fbb3a6a"}, ] -fsspec = [ + +[[package]] +name = "fsspec" +version = "2023.1.0" +description = "File-system specification" +category = "main" +optional = false +python-versions = ">=3.7" +files = [ {file = "fsspec-2023.1.0-py3-none-any.whl", hash = "sha256:b833e2e541e9e8cde0ab549414187871243177feb3d344f9d27b25a93f5d8139"}, {file = "fsspec-2023.1.0.tar.gz", hash = "sha256:fbae7f20ff801eb5f7d0bedf81f25c787c0dfac5e982d98fa3884a9cde2b5411"}, ] -identify = [ + +[package.extras] +abfs = ["adlfs"] +adl = ["adlfs"] +arrow = ["pyarrow (>=1)"] +dask = ["dask", "distributed"] +dropbox = ["dropbox", "dropboxdrivefs", "requests"] +entrypoints = ["importlib-metadata"] +fuse = ["fusepy"] +gcs = ["gcsfs"] +git = ["pygit2"] +github = ["requests"] +gs = ["gcsfs"] +gui = ["panel"] +hdfs = ["pyarrow (>=1)"] +http = ["aiohttp (!=4.0.0a0,!=4.0.0a1)", "requests"] +libarchive = ["libarchive-c"] +oci = ["ocifs"] +s3 = ["s3fs"] +sftp = ["paramiko"] +smb = ["smbprotocol"] +ssh = ["paramiko"] +tqdm = ["tqdm"] + +[[package]] +name = "identify" +version = "2.5.13" +description = "File identification library for Python" +category = "dev" +optional = false +python-versions = ">=3.7" +files = [ {file = "identify-2.5.13-py2.py3-none-any.whl", hash = "sha256:8aa48ce56e38c28b6faa9f261075dea0a942dfbb42b341b4e711896cbb40f3f7"}, {file = "identify-2.5.13.tar.gz", hash = "sha256:abb546bca6f470228785338a01b539de8a85bbf46491250ae03363956d8ebb10"}, ] -idna = [ + +[package.extras] +license = ["ukkonen"] + +[[package]] +name = "idna" +version = "3.4" +description = "Internationalized Domain Names in Applications (IDNA)" +category = "main" +optional = false +python-versions = ">=3.5" +files = [ {file = "idna-3.4-py3-none-any.whl", hash = "sha256:90b77e79eaa3eba6de819a0c442c0b4ceefc341a7a2ab77d7562bf49f425c5c2"}, {file = "idna-3.4.tar.gz", hash = "sha256:814f528e8dead7d329833b91c5faa87d60bf71824cd12a7530b5526063d02cb4"}, ] -importlib-metadata = [ + +[[package]] +name = "importlib-metadata" +version = "6.0.0" +description = "Read metadata from Python packages" +category = "dev" +optional = false +python-versions = ">=3.7" +files = [ {file = "importlib_metadata-6.0.0-py3-none-any.whl", hash = "sha256:7efb448ec9a5e313a57655d35aa54cd3e01b7e1fbcf72dce1bf06119420f5bad"}, {file = "importlib_metadata-6.0.0.tar.gz", hash = "sha256:e354bedeb60efa6affdcc8ae121b73544a7aa74156d047311948f6d711cd378d"}, ] -iniconfig = [ + +[package.dependencies] +zipp = ">=0.5" + +[package.extras] +docs = ["furo", "jaraco.packaging (>=9)", "jaraco.tidelift (>=1.4)", "rst.linker (>=1.9)", "sphinx (>=3.5)", "sphinx-lint"] +perf = ["ipython"] +testing = ["flake8 (<5)", "flufl.flake8", "importlib-resources (>=1.3)", "packaging", "pyfakefs", "pytest (>=6)", "pytest-black (>=0.3.7)", "pytest-checkdocs (>=2.4)", "pytest-cov", "pytest-enabler (>=1.3)", "pytest-flake8", "pytest-mypy (>=0.9.1)", "pytest-perf (>=0.9.2)"] + +[[package]] +name = "iniconfig" +version = "2.0.0" +description = "brain-dead simple config-ini parsing" +category = "dev" +optional = false +python-versions = ">=3.7" +files = [ {file = "iniconfig-2.0.0-py3-none-any.whl", hash = "sha256:b6a85871a79d2e3b22d2d1b94ac2824226a63c6b741c88f7ae975f18b6778374"}, {file = "iniconfig-2.0.0.tar.gz", hash = "sha256:2d91e135bf72d31a410b17c16da610a82cb55f6b0477d1a902134b24a455b8b3"}, ] -isodate = [ + +[[package]] +name = "isodate" +version = "0.6.1" +description = "An ISO 8601 date/time/duration parser and formatter" +category = "main" +optional = true +python-versions = "*" +files = [ {file = "isodate-0.6.1-py2.py3-none-any.whl", hash = "sha256:0751eece944162659049d35f4f549ed815792b38793f07cf73381c1c87cbed96"}, {file = "isodate-0.6.1.tar.gz", hash = "sha256:48c5881de7e8b0a0d648cb024c8062dc84e7b840ed81e864c7614fd3c127bde9"}, ] -jinja2 = [ + +[package.dependencies] +six = "*" + +[[package]] +name = "jinja2" +version = "3.1.2" +description = "A very fast and expressive template engine." +category = "dev" +optional = false +python-versions = ">=3.7" +files = [ {file = "Jinja2-3.1.2-py3-none-any.whl", hash = "sha256:6088930bfe239f0e6710546ab9c19c9ef35e29792895fed6e6e31a023a182a61"}, {file = "Jinja2-3.1.2.tar.gz", hash = "sha256:31351a702a408a9e7595a8fc6150fc3f43bb6bf7e319770cbc0db9df9437e852"}, ] -jmespath = [] -markdown-it-py = [ + +[package.dependencies] +MarkupSafe = ">=2.0" + +[package.extras] +i18n = ["Babel (>=2.7)"] + +[[package]] +name = "jmespath" +version = "1.0.1" +description = "JSON Matching Expressions" +category = "main" +optional = false +python-versions = ">=3.7" +files = [ + {file = "jmespath-1.0.1-py3-none-any.whl", hash = "sha256:02e2e4cc71b5bcab88332eebf907519190dd9e6e82107fa7f83b1003a6252980"}, + {file = "jmespath-1.0.1.tar.gz", hash = "sha256:90261b206d6defd58fdd5e85f478bf633a2901798906be2ad389150c5c60edbe"}, +] + +[[package]] +name = "markdown-it-py" +version = "2.1.0" +description = "Python port of markdown-it. Markdown parsing, done right!" +category = "main" +optional = false +python-versions = ">=3.7" +files = [ {file = "markdown-it-py-2.1.0.tar.gz", hash = "sha256:cf7e59fed14b5ae17c0006eff14a2d9a00ed5f3a846148153899a0224e2c07da"}, {file = "markdown_it_py-2.1.0-py3-none-any.whl", hash = "sha256:93de681e5c021a432c63147656fe21790bc01231e0cd2da73626f1aa3ac0fe27"}, ] -markupsafe = [ + +[package.dependencies] +mdurl = ">=0.1,<1.0" + +[package.extras] +benchmarking = ["psutil", "pytest", "pytest-benchmark (>=3.2,<4.0)"] +code-style = ["pre-commit (==2.6)"] +compare = ["commonmark (>=0.9.1,<0.10.0)", "markdown (>=3.3.6,<3.4.0)", "mistletoe (>=0.8.1,<0.9.0)", "mistune (>=2.0.2,<2.1.0)", "panflute (>=2.1.3,<2.2.0)"] +linkify = ["linkify-it-py (>=1.0,<2.0)"] +plugins = ["mdit-py-plugins"] +profiling = ["gprof2dot"] +rtd = ["attrs", "myst-parser", "pyyaml", "sphinx", "sphinx-copybutton", "sphinx-design", "sphinx_book_theme"] +testing = ["coverage", "pytest", "pytest-cov", "pytest-regressions"] + +[[package]] +name = "markupsafe" +version = "2.1.2" +description = "Safely add untrusted strings to HTML/XML markup." +category = "dev" +optional = false +python-versions = ">=3.7" +files = [ {file = "MarkupSafe-2.1.2-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:665a36ae6f8f20a4676b53224e33d456a6f5a72657d9c83c2aa00765072f31f7"}, {file = "MarkupSafe-2.1.2-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:340bea174e9761308703ae988e982005aedf427de816d1afe98147668cc03036"}, {file = "MarkupSafe-2.1.2-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:22152d00bf4a9c7c83960521fc558f55a1adbc0631fbb00a9471e097b19d72e1"}, @@ -1745,11 +1083,27 @@ markupsafe = [ {file = "MarkupSafe-2.1.2-cp39-cp39-win_amd64.whl", hash = "sha256:0576fe974b40a400449768941d5d0858cc624e3249dfd1e0c33674e5c7ca7aed"}, {file = "MarkupSafe-2.1.2.tar.gz", hash = "sha256:abcabc8c2b26036d62d4c746381a6f7cf60aafcc653198ad678306986b09450d"}, ] -mdurl = [ + +[[package]] +name = "mdurl" +version = "0.1.2" +description = "Markdown URL utilities" +category = "main" +optional = false +python-versions = ">=3.7" +files = [ {file = "mdurl-0.1.2-py3-none-any.whl", hash = "sha256:84008a41e51615a49fc9966191ff91509e3c40b939176e643fd50a5c2196b8f8"}, {file = "mdurl-0.1.2.tar.gz", hash = "sha256:bb413d29f5eea38f31dd4754dd7377d4465116fb207585f97bf925588687c1ba"}, ] -mmhash3 = [ + +[[package]] +name = "mmhash3" +version = "3.0.1" +description = "Python wrapper for MurmurHash (MurmurHash3), a set of fast and robust hash functions." +category = "main" +optional = false +python-versions = "*" +files = [ {file = "mmhash3-3.0.1-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:47deea30cd8d3d5cd52dc740902a4c70383bfe8248eac29d0877fe63e03c2713"}, {file = "mmhash3-3.0.1-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:ecdaf4d1de617818bf05cd526ca558db6010beeba7ea9e19f695f2bdcac0e0a4"}, {file = "mmhash3-3.0.1-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:4675585617584e9e1aafa3a90ac0ac9a437257c507748d97de8b21977e9d6745"}, @@ -1785,23 +1139,122 @@ mmhash3 = [ {file = "mmhash3-3.0.1-cp39-cp39-win_amd64.whl", hash = "sha256:0baeaa20cac5f75ed14f28056826bd9d9c8b2354b382073f3fd5190708992a0d"}, {file = "mmhash3-3.0.1.tar.gz", hash = "sha256:a00d68f4a1cc434b9501513c8a29e18ed1ddad383677d72b41d71d0d862348af"}, ] -moto = [ + +[[package]] +name = "moto" +version = "4.1.0" +description = "" +category = "dev" +optional = false +python-versions = ">=3.7" +files = [ {file = "moto-4.1.0-py2.py3-none-any.whl", hash = "sha256:12f231196ebf22d19774f2a75738fdedae10a8c82abb8fa0877e1819aa799f4f"}, {file = "moto-4.1.0.tar.gz", hash = "sha256:e65a692df57abb631b06c8f3f7827c9deceae3416887527e266c775ad722a2fa"}, ] -msal = [ + +[package.dependencies] +boto3 = ">=1.9.201" +botocore = ">=1.12.201" +cryptography = ">=3.3.1" +Jinja2 = ">=2.10.1" +python-dateutil = ">=2.1,<3.0.0" +requests = ">=2.5" +responses = ">=0.13.0" +werkzeug = ">=0.5,<2.2.0 || >2.2.0,<2.2.1 || >2.2.1" +xmltodict = "*" + +[package.extras] +all = ["PyYAML (>=5.1)", "aws-xray-sdk (>=0.93,!=0.96)", "cfn-lint (>=0.40.0)", "docker (>=2.5.1)", "ecdsa (!=0.15)", "graphql-core", "jsondiff (>=1.1.2)", "openapi-spec-validator (>=0.2.8)", "pyparsing (>=3.0.7)", "python-jose[cryptography] (>=3.1.0,<4.0.0)", "setuptools", "sshpubkeys (>=3.1.0)"] +apigateway = ["PyYAML (>=5.1)", "ecdsa (!=0.15)", "openapi-spec-validator (>=0.2.8)", "python-jose[cryptography] (>=3.1.0,<4.0.0)"] +apigatewayv2 = ["PyYAML (>=5.1)"] +appsync = ["graphql-core"] +awslambda = ["docker (>=2.5.1)"] +batch = ["docker (>=2.5.1)"] +cloudformation = ["PyYAML (>=5.1)", "aws-xray-sdk (>=0.93,!=0.96)", "cfn-lint (>=0.40.0)", "docker (>=2.5.1)", "ecdsa (!=0.15)", "graphql-core", "jsondiff (>=1.1.2)", "openapi-spec-validator (>=0.2.8)", "pyparsing (>=3.0.7)", "python-jose[cryptography] (>=3.1.0,<4.0.0)", "setuptools", "sshpubkeys (>=3.1.0)"] +cognitoidp = ["ecdsa (!=0.15)", "python-jose[cryptography] (>=3.1.0,<4.0.0)"] +ds = ["sshpubkeys (>=3.1.0)"] +dynamodb = ["docker (>=2.5.1)"] +dynamodbstreams = ["docker (>=2.5.1)"] +ebs = ["sshpubkeys (>=3.1.0)"] +ec2 = ["sshpubkeys (>=3.1.0)"] +efs = ["sshpubkeys (>=3.1.0)"] +glue = ["pyparsing (>=3.0.7)"] +iotdata = ["jsondiff (>=1.1.2)"] +route53resolver = ["sshpubkeys (>=3.1.0)"] +s3 = ["PyYAML (>=5.1)"] +server = ["PyYAML (>=5.1)", "aws-xray-sdk (>=0.93,!=0.96)", "cfn-lint (>=0.40.0)", "docker (>=2.5.1)", "ecdsa (!=0.15)", "flask (!=2.2.0,!=2.2.1)", "flask-cors", "graphql-core", "jsondiff (>=1.1.2)", "openapi-spec-validator (>=0.2.8)", "pyparsing (>=3.0.7)", "python-jose[cryptography] (>=3.1.0,<4.0.0)", "setuptools", "sshpubkeys (>=3.1.0)"] +ssm = ["PyYAML (>=5.1)"] +xray = ["aws-xray-sdk (>=0.93,!=0.96)", "setuptools"] + +[[package]] +name = "msal" +version = "1.20.0" +description = "The Microsoft Authentication Library (MSAL) for Python library enables your app to access the Microsoft Cloud by supporting authentication of users with Microsoft Azure Active Directory accounts (AAD) and Microsoft Accounts (MSA) using industry standard OAuth2 and OpenID Connect." +category = "main" +optional = true +python-versions = "*" +files = [ {file = "msal-1.20.0-py2.py3-none-any.whl", hash = "sha256:d2f1c26368ecdc28c8657d457352faa0b81b1845a7b889d8676787721ba86792"}, {file = "msal-1.20.0.tar.gz", hash = "sha256:78344cd4c91d6134a593b5e3e45541e666e37b747ff8a6316c3668dd1e6ab6b2"}, ] -msal-extensions = [ + +[package.dependencies] +cryptography = ">=0.6,<41" +PyJWT = {version = ">=1.0.0,<3", extras = ["crypto"]} +requests = ">=2.0.0,<3" + +[package.extras] +broker = ["pymsalruntime (>=0.11.2,<0.14)"] + +[[package]] +name = "msal-extensions" +version = "1.0.0" +description = "Microsoft Authentication Library extensions (MSAL EX) provides a persistence API that can save your data on disk, encrypted on Windows, macOS and Linux. Concurrent data access will be coordinated by a file lock mechanism." +category = "main" +optional = true +python-versions = "*" +files = [ {file = "msal-extensions-1.0.0.tar.gz", hash = "sha256:c676aba56b0cce3783de1b5c5ecfe828db998167875126ca4b47dc6436451354"}, {file = "msal_extensions-1.0.0-py2.py3-none-any.whl", hash = "sha256:91e3db9620b822d0ed2b4d1850056a0f133cba04455e62f11612e40f5502f2ee"}, ] -msrest = [ + +[package.dependencies] +msal = ">=0.4.1,<2.0.0" +portalocker = [ + {version = ">=1.0,<3", markers = "python_version >= \"3.5\" and platform_system != \"Windows\""}, + {version = ">=1.6,<3", markers = "python_version >= \"3.5\" and platform_system == \"Windows\""}, +] + +[[package]] +name = "msrest" +version = "0.7.1" +description = "AutoRest swagger generator Python client runtime." +category = "main" +optional = true +python-versions = ">=3.6" +files = [ {file = "msrest-0.7.1-py3-none-any.whl", hash = "sha256:21120a810e1233e5e6cc7fe40b474eeb4ec6f757a15d7cf86702c369f9567c32"}, {file = "msrest-0.7.1.zip", hash = "sha256:6e7661f46f3afd88b75667b7187a92829924446c7ea1d169be8c4bb7eeb788b9"}, ] -multidict = [ + +[package.dependencies] +azure-core = ">=1.24.0" +certifi = ">=2017.4.17" +isodate = ">=0.6.0" +requests = ">=2.16,<3.0" +requests-oauthlib = ">=0.5.0" + +[package.extras] +async = ["aiodns", "aiohttp (>=3.0)"] + +[[package]] +name = "multidict" +version = "6.0.4" +description = "multidict implementation" +category = "main" +optional = true +python-versions = ">=3.7" +files = [ {file = "multidict-6.0.4-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:0b1a97283e0c85772d613878028fec909f003993e1007eafa715b24b377cb9b8"}, {file = "multidict-6.0.4-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:eeb6dcc05e911516ae3d1f207d4b0520d07f54484c49dfc294d6e7d63b734171"}, {file = "multidict-6.0.4-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:d6d635d5209b82a3492508cf5b365f3446afb65ae7ebd755e70e18f287b0adf7"}, @@ -1877,11 +1330,30 @@ multidict = [ {file = "multidict-6.0.4-cp39-cp39-win_amd64.whl", hash = "sha256:33029f5734336aa0d4c0384525da0387ef89148dc7191aae00ca5fb23d7aafc2"}, {file = "multidict-6.0.4.tar.gz", hash = "sha256:3666906492efb76453c0e7b97f2cf459b0682e7402c0489a95484965dbc1da49"}, ] -nodeenv = [ + +[[package]] +name = "nodeenv" +version = "1.7.0" +description = "Node.js virtual environment builder" +category = "dev" +optional = false +python-versions = ">=2.7,!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,!=3.4.*,!=3.5.*,!=3.6.*" +files = [ {file = "nodeenv-1.7.0-py2.py3-none-any.whl", hash = "sha256:27083a7b96a25f2f5e1d8cb4b6317ee8aeda3bdd121394e5ac54e498028a042e"}, {file = "nodeenv-1.7.0.tar.gz", hash = "sha256:e0e7f7dfb85fc5394c6fe1e8fa98131a2473e04311a45afb6508f7cf1836fa2b"}, ] -numpy = [ + +[package.dependencies] +setuptools = "*" + +[[package]] +name = "numpy" +version = "1.24.1" +description = "Fundamental package for array computing in Python" +category = "main" +optional = true +python-versions = ">=3.8" +files = [ {file = "numpy-1.24.1-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:179a7ef0889ab769cc03573b6217f54c8bd8e16cef80aad369e1e8185f994cd7"}, {file = "numpy-1.24.1-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:b09804ff570b907da323b3d762e74432fb07955701b17b08ff1b5ebaa8cfe6a9"}, {file = "numpy-1.24.1-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:f1b739841821968798947d3afcefd386fa56da0caf97722a5de53e07c4ccedc7"}, @@ -1911,15 +1383,44 @@ numpy = [ {file = "numpy-1.24.1-pp38-pypy38_pp73-win_amd64.whl", hash = "sha256:cfa1161c6ac8f92dea03d625c2d0c05e084668f4a06568b77a25a89111621566"}, {file = "numpy-1.24.1.tar.gz", hash = "sha256:2386da9a471cc00a1f47845e27d916d5ec5346ae9696e01a8a34760858fe9dd2"}, ] -oauthlib = [ + +[[package]] +name = "oauthlib" +version = "3.2.2" +description = "A generic, spec-compliant, thorough implementation of the OAuth request-signing logic" +category = "main" +optional = true +python-versions = ">=3.6" +files = [ {file = "oauthlib-3.2.2-py3-none-any.whl", hash = "sha256:8139f29aac13e25d502680e9e19963e83f16838d48a0d71c287fe40e7067fbca"}, {file = "oauthlib-3.2.2.tar.gz", hash = "sha256:9859c40929662bec5d64f34d01c99e093149682a3f38915dc0655d5a633dd918"}, ] -packaging = [ + +[package.extras] +rsa = ["cryptography (>=3.0.0)"] +signals = ["blinker (>=1.4.0)"] +signedtoken = ["cryptography (>=3.0.0)", "pyjwt (>=2.0.0,<3)"] + +[[package]] +name = "packaging" +version = "23.0" +description = "Core utilities for Python packages" +category = "dev" +optional = false +python-versions = ">=3.7" +files = [ {file = "packaging-23.0-py3-none-any.whl", hash = "sha256:714ac14496c3e68c99c29b00845f7a2b85f3bb6f1078fd9f72fd20f0570002b2"}, {file = "packaging-23.0.tar.gz", hash = "sha256:b6ad297f8907de0fa2fe1ccbd26fdaf387f5f47c7275fedf8cce89f99446cf97"}, ] -pandas = [ + +[[package]] +name = "pandas" +version = "1.5.3" +description = "Powerful data structures for data analysis, time series, and statistics" +category = "main" +optional = true +python-versions = ">=3.8" +files = [ {file = "pandas-1.5.3-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:3749077d86e3a2f0ed51367f30bf5b82e131cc0f14260c4d3e499186fccc4406"}, {file = "pandas-1.5.3-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:972d8a45395f2a2d26733eb8d0f629b2f90bebe8e8eddbb8829b180c09639572"}, {file = "pandas-1.5.3-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:50869a35cbb0f2e0cd5ec04b191e7b12ed688874bd05dd777c19b28cbea90996"}, @@ -1948,23 +1449,98 @@ pandas = [ {file = "pandas-1.5.3-cp39-cp39-win_amd64.whl", hash = "sha256:dfd681c5dc216037e0b0a2c821f5ed99ba9f03ebcf119c7dac0e9a7b960b9ec9"}, {file = "pandas-1.5.3.tar.gz", hash = "sha256:74a3fd7e5a7ec052f183273dc7b0acd3a863edf7520f5d3a1765c04ffdb3b0b1"}, ] -platformdirs = [ + +[package.dependencies] +numpy = [ + {version = ">=1.20.3", markers = "python_version < \"3.10\""}, + {version = ">=1.21.0", markers = "python_version >= \"3.10\""}, + {version = ">=1.23.2", markers = "python_version >= \"3.11\""}, +] +python-dateutil = ">=2.8.1" +pytz = ">=2020.1" + +[package.extras] +test = ["hypothesis (>=5.5.3)", "pytest (>=6.0)", "pytest-xdist (>=1.31)"] + +[[package]] +name = "platformdirs" +version = "2.6.2" +description = "A small Python package for determining appropriate platform-specific dirs, e.g. a \"user data dir\"." +category = "dev" +optional = false +python-versions = ">=3.7" +files = [ {file = "platformdirs-2.6.2-py3-none-any.whl", hash = "sha256:83c8f6d04389165de7c9b6f0c682439697887bca0aa2f1c87ef1826be3584490"}, {file = "platformdirs-2.6.2.tar.gz", hash = "sha256:e1fea1fe471b9ff8332e229df3cb7de4f53eeea4998d3b6bfff542115e998bd2"}, ] -pluggy = [ + +[package.extras] +docs = ["furo (>=2022.12.7)", "proselint (>=0.13)", "sphinx (>=5.3)", "sphinx-autodoc-typehints (>=1.19.5)"] +test = ["appdirs (==1.4.4)", "covdefaults (>=2.2.2)", "pytest (>=7.2)", "pytest-cov (>=4)", "pytest-mock (>=3.10)"] + +[[package]] +name = "pluggy" +version = "1.0.0" +description = "plugin and hook calling mechanisms for python" +category = "dev" +optional = false +python-versions = ">=3.6" +files = [ {file = "pluggy-1.0.0-py2.py3-none-any.whl", hash = "sha256:74134bbf457f031a36d68416e1509f34bd5ccc019f0bcc952c7b909d06b37bd3"}, {file = "pluggy-1.0.0.tar.gz", hash = "sha256:4224373bacce55f955a878bf9cfa763c1e360858e330072059e10bad68531159"}, ] -portalocker = [ + +[package.extras] +dev = ["pre-commit", "tox"] +testing = ["pytest", "pytest-benchmark"] + +[[package]] +name = "portalocker" +version = "2.7.0" +description = "Wraps the portalocker recipe for easy usage" +category = "main" +optional = true +python-versions = ">=3.5" +files = [ {file = "portalocker-2.7.0-py2.py3-none-any.whl", hash = "sha256:a07c5b4f3985c3cf4798369631fb7011adb498e2a46d8440efc75a8f29a0f983"}, {file = "portalocker-2.7.0.tar.gz", hash = "sha256:032e81d534a88ec1736d03f780ba073f047a06c478b06e2937486f334e955c51"}, ] -pre-commit = [ - {file = "pre_commit-2.21.0-py2.py3-none-any.whl", hash = "sha256:e2f91727039fc39a92f58a588a25b87f936de6567eed4f0e673e0507edc75bad"}, - {file = "pre_commit-2.21.0.tar.gz", hash = "sha256:31ef31af7e474a8d8995027fefdfcf509b5c913ff31f2015b4ec4beb26a6f658"}, + +[package.dependencies] +pywin32 = {version = ">=226", markers = "platform_system == \"Windows\""} + +[package.extras] +docs = ["sphinx (>=1.7.1)"] +redis = ["redis"] +tests = ["pytest (>=5.4.1)", "pytest-cov (>=2.8.1)", "pytest-mypy (>=0.8.0)", "pytest-timeout (>=2.1.0)", "redis", "sphinx (>=6.0.0)"] + +[[package]] +name = "pre-commit" +version = "3.0.1" +description = "A framework for managing and maintaining multi-language pre-commit hooks." +category = "dev" +optional = false +python-versions = ">=3.8" +files = [ + {file = "pre_commit-3.0.1-py2.py3-none-any.whl", hash = "sha256:61ecb75e0e99939cc30c79181c0394545657855e9172c42ff98ebecb0e02fcb7"}, + {file = "pre_commit-3.0.1.tar.gz", hash = "sha256:3a3f9229e8c19a626a7f91be25b3c8c135e52de1a678da98eb015c0d0baea7a5"}, ] -pyarrow = [ + +[package.dependencies] +cfgv = ">=2.0.0" +identify = ">=1.0.0" +nodeenv = ">=0.11.1" +pyyaml = ">=5.1" +virtualenv = ">=20.10.0" + +[[package]] +name = "pyarrow" +version = "10.0.1" +description = "Python library for Apache Arrow" +category = "main" +optional = true +python-versions = ">=3.7" +files = [ {file = "pyarrow-10.0.1-cp310-cp310-macosx_10_14_x86_64.whl", hash = "sha256:e00174764a8b4e9d8d5909b6d19ee0c217a6cf0232c5682e31fdfbd5a9f0ae52"}, {file = "pyarrow-10.0.1-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:6f7a7dbe2f7f65ac1d0bd3163f756deb478a9e9afc2269557ed75b1b25ab3610"}, {file = "pyarrow-10.0.1-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:cb627673cb98708ef00864e2e243f51ba7b4c1b9f07a1d821f98043eccd3f585"}, @@ -1991,11 +1567,30 @@ pyarrow = [ {file = "pyarrow-10.0.1-cp39-cp39-win_amd64.whl", hash = "sha256:0ec7587d759153f452d5263dbc8b1af318c4609b607be2bd5127dcda6708cdb1"}, {file = "pyarrow-10.0.1.tar.gz", hash = "sha256:1a14f57a5f472ce8234f2964cd5184cccaa8df7e04568c64edc33b23eb285dd5"}, ] -pycparser = [ + +[package.dependencies] +numpy = ">=1.16.6" + +[[package]] +name = "pycparser" +version = "2.21" +description = "C parser in Python" +category = "main" +optional = false +python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*" +files = [ {file = "pycparser-2.21-py2.py3-none-any.whl", hash = "sha256:8ee45429555515e1f6b185e78100aea234072576aa43ab53aefcae078162fca9"}, {file = "pycparser-2.21.tar.gz", hash = "sha256:e644fdec12f7872f86c58ff790da456218b10f863970249516d60a5eaca77206"}, ] -pydantic = [ + +[[package]] +name = "pydantic" +version = "1.10.4" +description = "Data validation and settings management using python type hints" +category = "main" +optional = false +python-versions = ">=3.7" +files = [ {file = "pydantic-1.10.4-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:b5635de53e6686fe7a44b5cf25fcc419a0d5e5c1a1efe73d49d48fe7586db854"}, {file = "pydantic-1.10.4-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:6dc1cc241440ed7ca9ab59d9929075445da6b7c94ced281b3dd4cfe6c8cff817"}, {file = "pydantic-1.10.4-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:51bdeb10d2db0f288e71d49c9cefa609bca271720ecd0c58009bd7504a0c464c"}, @@ -2033,32 +1628,148 @@ pydantic = [ {file = "pydantic-1.10.4-py3-none-any.whl", hash = "sha256:4948f264678c703f3877d1c8877c4e3b2e12e549c57795107f08cf70c6ec7774"}, {file = "pydantic-1.10.4.tar.gz", hash = "sha256:b9a3859f24eb4e097502a3be1fb4b2abb79b6103dd9e2e0edb70613a4459a648"}, ] -pygments = [ + +[package.dependencies] +typing-extensions = ">=4.2.0" + +[package.extras] +dotenv = ["python-dotenv (>=0.10.4)"] +email = ["email-validator (>=1.0.3)"] + +[[package]] +name = "pygments" +version = "2.14.0" +description = "Pygments is a syntax highlighting package written in Python." +category = "main" +optional = false +python-versions = ">=3.6" +files = [ {file = "Pygments-2.14.0-py3-none-any.whl", hash = "sha256:fa7bd7bd2771287c0de303af8bfdfc731f51bd2c6a47ab69d117138893b82717"}, {file = "Pygments-2.14.0.tar.gz", hash = "sha256:b3ed06a9e8ac9a9aae5a6f5dbe78a8a58655d17b43b93c078f094ddc476ae297"}, ] -pyjwt = [ + +[package.extras] +plugins = ["importlib-metadata"] + +[[package]] +name = "pyjwt" +version = "2.6.0" +description = "JSON Web Token implementation in Python" +category = "main" +optional = true +python-versions = ">=3.7" +files = [ {file = "PyJWT-2.6.0-py3-none-any.whl", hash = "sha256:d83c3d892a77bbb74d3e1a2cfa90afaadb60945205d1095d9221f04466f64c14"}, {file = "PyJWT-2.6.0.tar.gz", hash = "sha256:69285c7e31fc44f68a1feb309e948e0df53259d579295e6cfe2b1792329f05fd"}, ] -pyparsing = [ + +[package.dependencies] +cryptography = {version = ">=3.4.0", optional = true, markers = "extra == \"crypto\""} + +[package.extras] +crypto = ["cryptography (>=3.4.0)"] +dev = ["coverage[toml] (==5.0.4)", "cryptography (>=3.4.0)", "pre-commit", "pytest (>=6.0.0,<7.0.0)", "sphinx (>=4.5.0,<5.0.0)", "sphinx-rtd-theme", "zope.interface"] +docs = ["sphinx (>=4.5.0,<5.0.0)", "sphinx-rtd-theme", "zope.interface"] +tests = ["coverage[toml] (==5.0.4)", "pytest (>=6.0.0,<7.0.0)"] + +[[package]] +name = "pyparsing" +version = "3.0.9" +description = "pyparsing module - Classes and methods to define and execute parsing grammars" +category = "main" +optional = false +python-versions = ">=3.6.8" +files = [ {file = "pyparsing-3.0.9-py3-none-any.whl", hash = "sha256:5026bae9a10eeaefb61dab2f09052b9f4307d44aee4eda64b309723d8d206bbc"}, {file = "pyparsing-3.0.9.tar.gz", hash = "sha256:2b020ecf7d21b687f219b71ecad3631f644a47f01403fa1d1036b0c6416d70fb"}, ] -pyproject-hooks = [ + +[package.extras] +diagrams = ["jinja2", "railroad-diagrams"] + +[[package]] +name = "pyproject-hooks" +version = "1.0.0" +description = "Wrappers to call pyproject.toml-based build backend hooks." +category = "dev" +optional = false +python-versions = ">=3.7" +files = [ {file = "pyproject_hooks-1.0.0-py3-none-any.whl", hash = "sha256:283c11acd6b928d2f6a7c73fa0d01cb2bdc5f07c57a2eeb6e83d5e56b97976f8"}, {file = "pyproject_hooks-1.0.0.tar.gz", hash = "sha256:f271b298b97f5955d53fb12b72c1fb1948c22c1a6b70b315c54cedaca0264ef5"}, ] -pytest = [ + +[package.dependencies] +tomli = {version = ">=1.1.0", markers = "python_version < \"3.11\""} + +[[package]] +name = "pytest" +version = "7.2.1" +description = "pytest: simple powerful testing with Python" +category = "dev" +optional = false +python-versions = ">=3.7" +files = [ {file = "pytest-7.2.1-py3-none-any.whl", hash = "sha256:c7c6ca206e93355074ae32f7403e8ea12163b1163c976fee7d4d84027c162be5"}, {file = "pytest-7.2.1.tar.gz", hash = "sha256:d45e0952f3727241918b8fd0f376f5ff6b301cc0777c6f9a556935c92d8a7d42"}, ] -pytest-checkdocs = [ + +[package.dependencies] +attrs = ">=19.2.0" +colorama = {version = "*", markers = "sys_platform == \"win32\""} +exceptiongroup = {version = ">=1.0.0rc8", markers = "python_version < \"3.11\""} +iniconfig = "*" +packaging = "*" +pluggy = ">=0.12,<2.0" +tomli = {version = ">=1.0.0", markers = "python_version < \"3.11\""} + +[package.extras] +testing = ["argcomplete", "hypothesis (>=3.56)", "mock", "nose", "pygments (>=2.7.2)", "requests", "xmlschema"] + +[[package]] +name = "pytest-checkdocs" +version = "2.9.0" +description = "check the README when running tests" +category = "dev" +optional = false +python-versions = ">=3.7" +files = [ {file = "pytest-checkdocs-2.9.0.tar.gz", hash = "sha256:54b3d1357a289e3f6336d31d4ceb72e99afe0fa4f552990f1b1f015f1925543b"}, {file = "pytest_checkdocs-2.9.0-py3-none-any.whl", hash = "sha256:4231e5426915a02385e386b5eb4a70490b143d97cede4c7c77691bb320b5a73f"}, ] -python-dateutil = [] -python-snappy = [ + +[package.dependencies] +build = "*" +docutils = ">=0.15" +importlib-metadata = {version = ">=4", markers = "python_version < \"3.10\""} + +[package.extras] +docs = ["furo", "jaraco.packaging (>=9)", "rst.linker (>=1.9)", "sphinx (>=3.5)"] +testing = ["flake8 (<5)", "pytest (>=6)", "pytest-black (>=0.3.7)", "pytest-checkdocs (>=2.4)", "pytest-cov", "pytest-enabler (>=1.3)", "pytest-flake8", "pytest-mypy (>=0.9.1)", "types-docutils"] + +[[package]] +name = "python-dateutil" +version = "2.8.2" +description = "Extensions to the standard Python datetime module" +category = "main" +optional = false +python-versions = "!=3.0.*,!=3.1.*,!=3.2.*,>=2.7" +files = [ + {file = "python-dateutil-2.8.2.tar.gz", hash = "sha256:0123cacc1627ae19ddf3c27a5de5bd67ee4586fbdd6440d9748f8abb483d3e86"}, + {file = "python_dateutil-2.8.2-py2.py3-none-any.whl", hash = "sha256:961d03dc3453ebbc59dbdea9e4e11c5651520a876d0f4db161e8674aae935da9"}, +] + +[package.dependencies] +six = ">=1.5" + +[[package]] +name = "python-snappy" +version = "0.6.1" +description = "Python library for the snappy compression library from Google" +category = "main" +optional = true +python-versions = "*" +files = [ {file = "python-snappy-0.6.1.tar.gz", hash = "sha256:b6a107ab06206acc5359d4c5632bd9b22d448702a79b3169b0c62e0fb808bb2a"}, {file = "python_snappy-0.6.1-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:b7f920eaf46ebf41bd26f9df51c160d40f9e00b7b48471c3438cb8d027f7fb9b"}, {file = "python_snappy-0.6.1-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:4ec533a8c1f8df797bded662ec3e494d225b37855bb63eb0d75464a07947477c"}, @@ -2108,11 +1819,27 @@ python-snappy = [ {file = "python_snappy-0.6.1-pp37-pypy37_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:0bdb6942180660bda7f7d01f4c0def3cfc72b1c6d99aad964801775a3e379aba"}, {file = "python_snappy-0.6.1-pp37-pypy37_pp73-win_amd64.whl", hash = "sha256:03bb511380fca2a13325b6f16fe8234c8e12da9660f0258cd45d9a02ffc916af"}, ] -pytz = [ + +[[package]] +name = "pytz" +version = "2022.7.1" +description = "World timezone definitions, modern and historical" +category = "main" +optional = true +python-versions = "*" +files = [ {file = "pytz-2022.7.1-py2.py3-none-any.whl", hash = "sha256:78f4f37d8198e0627c5f1143240bb0206b8691d8d7ac6d78fee88b78733f8c4a"}, {file = "pytz-2022.7.1.tar.gz", hash = "sha256:01a0681c4b9684a28304615eba55d1ab31ae00bf68ec157ec3708a8182dbbcd0"}, ] -pywin32 = [ + +[[package]] +name = "pywin32" +version = "305" +description = "Python for Window Extensions" +category = "main" +optional = true +python-versions = "*" +files = [ {file = "pywin32-305-cp310-cp310-win32.whl", hash = "sha256:421f6cd86e84bbb696d54563c48014b12a23ef95a14e0bdba526be756d89f116"}, {file = "pywin32-305-cp310-cp310-win_amd64.whl", hash = "sha256:73e819c6bed89f44ff1d690498c0a811948f73777e5f97c494c152b850fad478"}, {file = "pywin32-305-cp310-cp310-win_arm64.whl", hash = "sha256:742eb905ce2187133a29365b428e6c3b9001d79accdc30aa8969afba1d8470f4"}, @@ -2128,7 +1855,15 @@ pywin32 = [ {file = "pywin32-305-cp39-cp39-win32.whl", hash = "sha256:9d968c677ac4d5cbdaa62fd3014ab241718e619d8e36ef8e11fb930515a1e918"}, {file = "pywin32-305-cp39-cp39-win_amd64.whl", hash = "sha256:50768c6b7c3f0b38b7fb14dd4104da93ebced5f1a50dc0e834594bff6fbe1271"}, ] -pyyaml = [ + +[[package]] +name = "pyyaml" +version = "6.0" +description = "YAML parser and emitter for Python" +category = "main" +optional = false +python-versions = ">=3.6" +files = [ {file = "PyYAML-6.0-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:d4db7c7aef085872ef65a8fd7d6d09a14ae91f691dec3e87ee5ee0539d516f53"}, {file = "PyYAML-6.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:9df7ed3b3d2e0ecfe09e14741b857df43adb5a3ddadc919a2d94fbdf78fea53c"}, {file = "PyYAML-6.0-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:77f396e6ef4c73fdc33a9157446466f1cff553d979bd00ecb64385760c6babdc"}, @@ -2136,6 +1871,13 @@ pyyaml = [ {file = "PyYAML-6.0-cp310-cp310-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:f84fbc98b019fef2ee9a1cb3ce93e3187a6df0b2538a651bfb890254ba9f90b5"}, {file = "PyYAML-6.0-cp310-cp310-win32.whl", hash = "sha256:2cd5df3de48857ed0544b34e2d40e9fac445930039f3cfe4bcc592a1f836d513"}, {file = "PyYAML-6.0-cp310-cp310-win_amd64.whl", hash = "sha256:daf496c58a8c52083df09b80c860005194014c3698698d1a57cbcfa182142a3a"}, + {file = "PyYAML-6.0-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:d4b0ba9512519522b118090257be113b9468d804b19d63c71dbcf4a48fa32358"}, + {file = "PyYAML-6.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:81957921f441d50af23654aa6c5e5eaf9b06aba7f0a19c18a538dc7ef291c5a1"}, + {file = "PyYAML-6.0-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:afa17f5bc4d1b10afd4466fd3a44dc0e245382deca5b3c353d8b757f9e3ecb8d"}, + {file = "PyYAML-6.0-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:dbad0e9d368bb989f4515da330b88a057617d16b6a8245084f1b05400f24609f"}, + {file = "PyYAML-6.0-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:432557aa2c09802be39460360ddffd48156e30721f5e8d917f01d31694216782"}, + {file = "PyYAML-6.0-cp311-cp311-win32.whl", hash = "sha256:bfaef573a63ba8923503d27530362590ff4f576c626d86a9fed95822a8255fd7"}, + {file = "PyYAML-6.0-cp311-cp311-win_amd64.whl", hash = "sha256:01b45c0191e6d66c470b6cf1b9531a771a83c1c4208272ead47a3ae4f2f603bf"}, {file = "PyYAML-6.0-cp36-cp36m-macosx_10_9_x86_64.whl", hash = "sha256:897b80890765f037df3403d22bab41627ca8811ae55e9a722fd0392850ec4d86"}, {file = "PyYAML-6.0-cp36-cp36m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:50602afada6d6cbfad699b0c7bb50d5ccffa7e46a3d738092afddc1f9758427f"}, {file = "PyYAML-6.0-cp36-cp36m-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:48c346915c114f5fdb3ead70312bd042a953a8ce5c7106d5bfb1a5254e47da92"}, @@ -2163,65 +1905,308 @@ pyyaml = [ {file = "PyYAML-6.0-cp39-cp39-win_amd64.whl", hash = "sha256:b3d267842bf12586ba6c734f89d1f5b871df0273157918b0ccefa29deb05c21c"}, {file = "PyYAML-6.0.tar.gz", hash = "sha256:68fb519c14306fec9720a2a5b45bc9f0c8d1b9c72adf45c37baedfcd949c35a2"}, ] -requests = [ + +[[package]] +name = "requests" +version = "2.28.2" +description = "Python HTTP for Humans." +category = "main" +optional = false +python-versions = ">=3.7, <4" +files = [ {file = "requests-2.28.2-py3-none-any.whl", hash = "sha256:64299f4909223da747622c030b781c0d7811e359c37124b4bd368fb8c6518baa"}, {file = "requests-2.28.2.tar.gz", hash = "sha256:98b1b2782e3c6c4904938b84c0eb932721069dfdb9134313beff7c83c2df24bf"}, ] -requests-mock = [ + +[package.dependencies] +certifi = ">=2017.4.17" +charset-normalizer = ">=2,<4" +idna = ">=2.5,<4" +urllib3 = ">=1.21.1,<1.27" + +[package.extras] +socks = ["PySocks (>=1.5.6,!=1.5.7)"] +use-chardet-on-py3 = ["chardet (>=3.0.2,<6)"] + +[[package]] +name = "requests-mock" +version = "1.10.0" +description = "Mock out responses from the requests package" +category = "dev" +optional = false +python-versions = "*" +files = [ {file = "requests-mock-1.10.0.tar.gz", hash = "sha256:59c9c32419a9fb1ae83ec242d98e889c45bd7d7a65d48375cc243ec08441658b"}, {file = "requests_mock-1.10.0-py2.py3-none-any.whl", hash = "sha256:2fdbb637ad17ee15c06f33d31169e71bf9fe2bdb7bc9da26185be0dd8d842699"}, ] -requests-oauthlib = [ + +[package.dependencies] +requests = ">=2.3,<3" +six = "*" + +[package.extras] +fixture = ["fixtures"] +test = ["fixtures", "mock", "purl", "pytest", "requests-futures", "sphinx", "testrepository (>=0.0.18)", "testtools"] + +[[package]] +name = "requests-oauthlib" +version = "1.3.1" +description = "OAuthlib authentication support for Requests." +category = "main" +optional = true +python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*" +files = [ {file = "requests-oauthlib-1.3.1.tar.gz", hash = "sha256:75beac4a47881eeb94d5ea5d6ad31ef88856affe2332b9aafb52c6452ccf0d7a"}, {file = "requests_oauthlib-1.3.1-py2.py3-none-any.whl", hash = "sha256:2577c501a2fb8d05a304c09d090d6e47c306fef15809d102b327cf8364bddab5"}, ] -responses = [ + +[package.dependencies] +oauthlib = ">=3.0.0" +requests = ">=2.0.0" + +[package.extras] +rsa = ["oauthlib[signedtoken] (>=3.0.0)"] + +[[package]] +name = "responses" +version = "0.22.0" +description = "A utility library for mocking out the `requests` Python library." +category = "dev" +optional = false +python-versions = ">=3.7" +files = [ {file = "responses-0.22.0-py3-none-any.whl", hash = "sha256:dcf294d204d14c436fddcc74caefdbc5764795a40ff4e6a7740ed8ddbf3294be"}, {file = "responses-0.22.0.tar.gz", hash = "sha256:396acb2a13d25297789a5866b4881cf4e46ffd49cc26c43ab1117f40b973102e"}, ] -rich = [ + +[package.dependencies] +requests = ">=2.22.0,<3.0" +toml = "*" +types-toml = "*" +urllib3 = ">=1.25.10" + +[package.extras] +tests = ["coverage (>=6.0.0)", "flake8", "mypy", "pytest (>=7.0.0)", "pytest-asyncio", "pytest-cov", "pytest-httpserver", "types-requests"] + +[[package]] +name = "rich" +version = "13.2.0" +description = "Render rich text, tables, progress bars, syntax highlighting, markdown and more to the terminal" +category = "main" +optional = false +python-versions = ">=3.7.0" +files = [ {file = "rich-13.2.0-py3-none-any.whl", hash = "sha256:7c963f0d03819221e9ac561e1bc866e3f95a02248c1234daa48954e6d381c003"}, {file = "rich-13.2.0.tar.gz", hash = "sha256:f1a00cdd3eebf999a15d85ec498bfe0b1a77efe9b34f645768a54132ef444ac5"}, ] -s3fs = [ + +[package.dependencies] +markdown-it-py = ">=2.1.0,<3.0.0" +pygments = ">=2.6.0,<3.0.0" +typing-extensions = {version = ">=4.0.0,<5.0", markers = "python_version < \"3.9\""} + +[package.extras] +jupyter = ["ipywidgets (>=7.5.1,<8.0.0)"] + +[[package]] +name = "s3fs" +version = "2023.1.0" +description = "Convenient Filesystem interface over S3" +category = "main" +optional = true +python-versions = ">= 3.7" +files = [ {file = "s3fs-2023.1.0-py3-none-any.whl", hash = "sha256:a549ae518fff4388bd6fca5248575c29f521e7e39efcd2bfab476651701fd114"}, {file = "s3fs-2023.1.0.tar.gz", hash = "sha256:8b2e28372423e93f26312208a9272e22a962ddd0a79d63f9a68693b6af5ff187"}, ] -s3transfer = [] -six = [ + +[package.dependencies] +aiobotocore = ">=2.4.2,<2.5.0" +aiohttp = "<4.0.0a0 || >4.0.0a0,<4.0.0a1 || >4.0.0a1" +fsspec = "2023.1.0" + +[package.extras] +awscli = ["aiobotocore[awscli] (>=2.4.2,<2.5.0)"] +boto3 = ["aiobotocore[boto3] (>=2.4.2,<2.5.0)"] + +[[package]] +name = "s3transfer" +version = "0.6.0" +description = "An Amazon S3 Transfer Manager" +category = "main" +optional = false +python-versions = ">= 3.7" +files = [ + {file = "s3transfer-0.6.0-py3-none-any.whl", hash = "sha256:06176b74f3a15f61f1b4f25a1fc29a4429040b7647133a463da8fa5bd28d5ecd"}, + {file = "s3transfer-0.6.0.tar.gz", hash = "sha256:2ed07d3866f523cc561bf4a00fc5535827981b117dd7876f036b0c1aca42c947"}, +] + +[package.dependencies] +botocore = ">=1.12.36,<2.0a.0" + +[package.extras] +crt = ["botocore[crt] (>=1.20.29,<2.0a.0)"] + +[[package]] +name = "setuptools" +version = "67.0.0" +description = "Easily download, build, install, upgrade, and uninstall Python packages" +category = "dev" +optional = false +python-versions = ">=3.7" +files = [ + {file = "setuptools-67.0.0-py3-none-any.whl", hash = "sha256:9d790961ba6219e9ff7d9557622d2fe136816a264dd01d5997cfc057d804853d"}, + {file = "setuptools-67.0.0.tar.gz", hash = "sha256:883131c5b6efa70b9101c7ef30b2b7b780a4283d5fc1616383cdf22c83cbefe6"}, +] + +[package.extras] +docs = ["furo", "jaraco.packaging (>=9)", "jaraco.tidelift (>=1.4)", "pygments-github-lexers (==0.0.5)", "rst.linker (>=1.9)", "sphinx (>=3.5)", "sphinx-favicon", "sphinx-hoverxref (<2)", "sphinx-inline-tabs", "sphinx-lint", "sphinx-notfound-page (==0.8.3)", "sphinx-reredirects", "sphinxcontrib-towncrier"] +testing = ["build[virtualenv]", "filelock (>=3.4.0)", "flake8 (<5)", "flake8-2020", "ini2toml[lite] (>=0.9)", "jaraco.envs (>=2.2)", "jaraco.path (>=3.2.0)", "pip (>=19.1)", "pip-run (>=8.8)", "pytest (>=6)", "pytest-black (>=0.3.7)", "pytest-checkdocs (>=2.4)", "pytest-cov", "pytest-enabler (>=1.3)", "pytest-flake8", "pytest-mypy (>=0.9.1)", "pytest-perf", "pytest-timeout", "pytest-xdist", "tomli-w (>=1.0.0)", "virtualenv (>=13.0.0)", "wheel"] +testing-integration = ["build[virtualenv]", "filelock (>=3.4.0)", "jaraco.envs (>=2.2)", "jaraco.path (>=3.2.0)", "pytest", "pytest-enabler", "pytest-xdist", "tomli", "virtualenv (>=13.0.0)", "wheel"] + +[[package]] +name = "six" +version = "1.16.0" +description = "Python 2 and 3 compatibility utilities" +category = "main" +optional = false +python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*" +files = [ {file = "six-1.16.0-py2.py3-none-any.whl", hash = "sha256:8abb2f1d86890a2dfb989f9a77cfcfd3e47c2a354b01111771326f8aa26e0254"}, {file = "six-1.16.0.tar.gz", hash = "sha256:1e61c37477a1626458e36f7b1d82aa5c9b094fa4802892072e49de9c60c4c926"}, ] -thrift = [] -toml = [ + +[[package]] +name = "thrift" +version = "0.16.0" +description = "Python bindings for the Apache Thrift RPC system" +category = "main" +optional = true +python-versions = "*" +files = [ + {file = "thrift-0.16.0.tar.gz", hash = "sha256:2b5b6488fcded21f9d312aa23c9ff6a0195d0f6ae26ddbd5ad9e3e25dfc14408"}, +] + +[package.dependencies] +six = ">=1.7.2" + +[package.extras] +all = ["tornado (>=4.0)", "twisted"] +tornado = ["tornado (>=4.0)"] +twisted = ["twisted"] + +[[package]] +name = "toml" +version = "0.10.2" +description = "Python Library for Tom's Obvious, Minimal Language" +category = "dev" +optional = false +python-versions = ">=2.6, !=3.0.*, !=3.1.*, !=3.2.*" +files = [ {file = "toml-0.10.2-py2.py3-none-any.whl", hash = "sha256:806143ae5bfb6a3c6e736a764057db0e6a0e05e338b5630894a5f779cabb4f9b"}, {file = "toml-0.10.2.tar.gz", hash = "sha256:b3bda1d108d5dd99f4a20d24d9c348e91c4db7ab1b749200bded2f839ccbe68f"}, ] -tomli = [ + +[[package]] +name = "tomli" +version = "2.0.1" +description = "A lil' TOML parser" +category = "dev" +optional = false +python-versions = ">=3.7" +files = [ {file = "tomli-2.0.1-py3-none-any.whl", hash = "sha256:939de3e7a6161af0c887ef91b7d41a53e7c5a1ca976325f429cb46ea9bc30ecc"}, {file = "tomli-2.0.1.tar.gz", hash = "sha256:de526c12914f0c550d15924c62d72abc48d6fe7364aa87328337a31007fe8a4f"}, ] -types-toml = [ + +[[package]] +name = "types-toml" +version = "0.10.8.1" +description = "Typing stubs for toml" +category = "dev" +optional = false +python-versions = "*" +files = [ {file = "types-toml-0.10.8.1.tar.gz", hash = "sha256:171bdb3163d79a520560f24ba916a9fc9bff81659c5448a9fea89240923722be"}, {file = "types_toml-0.10.8.1-py3-none-any.whl", hash = "sha256:b7b5c4977f96ab7b5ac06d8a6590d17c0bf252a96efc03b109c2711fb3e0eafd"}, ] -typing-extensions = [ + +[[package]] +name = "typing-extensions" +version = "4.4.0" +description = "Backported and Experimental Type Hints for Python 3.7+" +category = "main" +optional = false +python-versions = ">=3.7" +files = [ {file = "typing_extensions-4.4.0-py3-none-any.whl", hash = "sha256:16fa4864408f655d35ec496218b85f79b3437c829e93320c7c9215ccfd92489e"}, {file = "typing_extensions-4.4.0.tar.gz", hash = "sha256:1511434bb92bf8dd198c12b1cc812e800d4181cfcb867674e0f8279cc93087aa"}, ] -urllib3 = [ + +[[package]] +name = "urllib3" +version = "1.26.14" +description = "HTTP library with thread-safe connection pooling, file post, and more." +category = "main" +optional = false +python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*, !=3.5.*" +files = [ {file = "urllib3-1.26.14-py2.py3-none-any.whl", hash = "sha256:75edcdc2f7d85b137124a6c3c9fc3933cdeaa12ecb9a6a959f22797a0feca7e1"}, {file = "urllib3-1.26.14.tar.gz", hash = "sha256:076907bf8fd355cde77728471316625a4d2f7e713c125f51953bb5b3eecf4f72"}, ] -virtualenv = [ + +[package.extras] +brotli = ["brotli (>=1.0.9)", "brotlicffi (>=0.8.0)", "brotlipy (>=0.6.0)"] +secure = ["certifi", "cryptography (>=1.3.4)", "idna (>=2.0.0)", "ipaddress", "pyOpenSSL (>=0.14)", "urllib3-secure-extra"] +socks = ["PySocks (>=1.5.6,!=1.5.7,<2.0)"] + +[[package]] +name = "virtualenv" +version = "20.17.1" +description = "Virtual Python Environment builder" +category = "dev" +optional = false +python-versions = ">=3.6" +files = [ {file = "virtualenv-20.17.1-py3-none-any.whl", hash = "sha256:ce3b1684d6e1a20a3e5ed36795a97dfc6af29bc3970ca8dab93e11ac6094b3c4"}, {file = "virtualenv-20.17.1.tar.gz", hash = "sha256:f8b927684efc6f1cc206c9db297a570ab9ad0e51c16fa9e45487d36d1905c058"}, ] -werkzeug = [ + +[package.dependencies] +distlib = ">=0.3.6,<1" +filelock = ">=3.4.1,<4" +platformdirs = ">=2.4,<3" + +[package.extras] +docs = ["proselint (>=0.13)", "sphinx (>=5.3)", "sphinx-argparse (>=0.3.2)", "sphinx-rtd-theme (>=1)", "towncrier (>=22.8)"] +testing = ["coverage (>=6.2)", "coverage-enable-subprocess (>=1)", "flaky (>=3.7)", "packaging (>=21.3)", "pytest (>=7.0.1)", "pytest-env (>=0.6.2)", "pytest-freezegun (>=0.4.2)", "pytest-mock (>=3.6.1)", "pytest-randomly (>=3.10.3)", "pytest-timeout (>=2.1)"] + +[[package]] +name = "werkzeug" +version = "2.2.2" +description = "The comprehensive WSGI web application library." +category = "dev" +optional = false +python-versions = ">=3.7" +files = [ {file = "Werkzeug-2.2.2-py3-none-any.whl", hash = "sha256:f979ab81f58d7318e064e99c4506445d60135ac5cd2e177a2de0089bfd4c9bd5"}, {file = "Werkzeug-2.2.2.tar.gz", hash = "sha256:7ea2d48322cc7c0f8b3a215ed73eabd7b5d75d0b50e31ab006286ccff9e00b8f"}, ] -wrapt = [ + +[package.dependencies] +MarkupSafe = ">=2.1.1" + +[package.extras] +watchdog = ["watchdog"] + +[[package]] +name = "wrapt" +version = "1.14.1" +description = "Module for decorators, wrappers and monkey patching." +category = "main" +optional = true +python-versions = "!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,!=3.4.*,>=2.7" +files = [ {file = "wrapt-1.14.1-cp27-cp27m-macosx_10_9_x86_64.whl", hash = "sha256:1b376b3f4896e7930f1f772ac4b064ac12598d1c38d04907e696cc4d794b43d3"}, {file = "wrapt-1.14.1-cp27-cp27m-manylinux1_i686.whl", hash = "sha256:903500616422a40a98a5a3c4ff4ed9d0066f3b4c951fa286018ecdf0750194ef"}, {file = "wrapt-1.14.1-cp27-cp27m-manylinux1_x86_64.whl", hash = "sha256:5a9a0d155deafd9448baff28c08e150d9b24ff010e899311ddd63c45c2445e28"}, @@ -2287,11 +2272,27 @@ wrapt = [ {file = "wrapt-1.14.1-cp39-cp39-win_amd64.whl", hash = "sha256:dee60e1de1898bde3b238f18340eec6148986da0455d8ba7848d50470a7a32fb"}, {file = "wrapt-1.14.1.tar.gz", hash = "sha256:380a85cf89e0e69b7cfbe2ea9f765f004ff419f34194018a6827ac0e3edfed4d"}, ] -xmltodict = [ + +[[package]] +name = "xmltodict" +version = "0.13.0" +description = "Makes working with XML feel like you are working with JSON" +category = "dev" +optional = false +python-versions = ">=3.4" +files = [ {file = "xmltodict-0.13.0-py2.py3-none-any.whl", hash = "sha256:aa89e8fd76320154a40d19a0df04a4695fb9dc5ba977cbb68ab3e4eb225e7852"}, {file = "xmltodict-0.13.0.tar.gz", hash = "sha256:341595a488e3e01a85a9d8911d8912fd922ede5fecc4dce437eb4b6c8d037e56"}, ] -yarl = [ + +[[package]] +name = "yarl" +version = "1.8.2" +description = "Yet another URL library" +category = "main" +optional = true +python-versions = ">=3.7" +files = [ {file = "yarl-1.8.2-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:bb81f753c815f6b8e2ddd2eef3c855cf7da193b82396ac013c661aaa6cc6b0a5"}, {file = "yarl-1.8.2-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:47d49ac96156f0928f002e2424299b2c91d9db73e08c4cd6742923a086f1c863"}, {file = "yarl-1.8.2-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:3fc056e35fa6fba63248d93ff6e672c096f95f7836938241ebc8260e062832fe"}, @@ -2367,11 +2368,35 @@ yarl = [ {file = "yarl-1.8.2-cp39-cp39-win_amd64.whl", hash = "sha256:6604711362f2dbf7160df21c416f81fac0de6dbcf0b5445a2ef25478ecc4c778"}, {file = "yarl-1.8.2.tar.gz", hash = "sha256:49d43402c6e3013ad0978602bf6bf5328535c48d192304b91b97a3c6790b1562"}, ] -zipp = [ + +[package.dependencies] +idna = ">=2.0" +multidict = ">=4.0" + +[[package]] +name = "zipp" +version = "3.11.0" +description = "Backport of pathlib-compatible object wrapper for zip files" +category = "dev" +optional = false +python-versions = ">=3.7" +files = [ {file = "zipp-3.11.0-py3-none-any.whl", hash = "sha256:83a28fcb75844b5c0cdaf5aa4003c2d728c77e05f5aeabe8e95e56727005fbaa"}, {file = "zipp-3.11.0.tar.gz", hash = "sha256:a7a22e05929290a67401440b39690ae6563279bced5f314609d9d03798f56766"}, ] -zstandard = [ + +[package.extras] +docs = ["furo", "jaraco.packaging (>=9)", "jaraco.tidelift (>=1.4)", "rst.linker (>=1.9)", "sphinx (>=3.5)"] +testing = ["flake8 (<5)", "func-timeout", "jaraco.functools", "jaraco.itertools", "more-itertools", "pytest (>=6)", "pytest-black (>=0.3.7)", "pytest-checkdocs (>=2.4)", "pytest-cov", "pytest-enabler (>=1.3)", "pytest-flake8", "pytest-mypy (>=0.9.1)"] + +[[package]] +name = "zstandard" +version = "0.19.0" +description = "Zstandard bindings for Python" +category = "main" +optional = false +python-versions = ">=3.6" +files = [ {file = "zstandard-0.19.0-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:a65e0119ad39e855427520f7829618f78eb2824aa05e63ff19b466080cd99210"}, {file = "zstandard-0.19.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:4fa496d2d674c6e9cffc561639d17009d29adee84a27cf1e12d3c9be14aa8feb"}, {file = "zstandard-0.19.0-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:8f7c68de4f362c1b2f426395fe4e05028c56d0782b2ec3ae18a5416eaf775576"}, @@ -2424,3 +2449,24 @@ zstandard = [ {file = "zstandard-0.19.0-cp39-cp39-win_amd64.whl", hash = "sha256:55a513ec67e85abd8b8b83af8813368036f03e2d29a50fc94033504918273980"}, {file = "zstandard-0.19.0.tar.gz", hash = "sha256:31d12fcd942dd8dbf52ca5f6b1bbe287f44e5d551a081a983ff3ea2082867863"}, ] + +[package.dependencies] +cffi = {version = ">=1.11", markers = "platform_python_implementation == \"PyPy\""} + +[package.extras] +cffi = ["cffi (>=1.11)"] + +[extras] +adlfs = ["adlfs"] +duckdb = ["duckdb", "pyarrow"] +glue = ["boto3"] +hive = ["thrift"] +pandas = ["pandas", "pyarrow"] +pyarrow = ["pyarrow"] +s3fs = ["s3fs"] +snappy = ["python-snappy"] + +[metadata] +lock-version = "2.0" +python-versions = "^3.8" +content-hash = "4baf674169ac2e5d6004512d941bcd860e7436f3dd767f5b28b6d21ee7c75949" diff --git a/pyproject.toml b/pyproject.toml index a3dbca59c4..33afb6faef 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -79,7 +79,7 @@ adlfs = { version = "2023.1.0", optional = true } [tool.poetry.dev-dependencies] pytest = "7.2.1" pytest-checkdocs = "2.9.0" -pre-commit = "2.21.0" +pre-commit = "3.0.1" fastavro = "1.7.0" coverage = { version = "^7.0.5", extras = ["toml"] } requests-mock = "1.10.0" From 03484454311779dcf08d0923a6f65c08a4ad880c Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 30 Jan 2023 05:59:40 +0100 Subject: [PATCH 352/642] Build: Bump fastavro from 1.7.0 to 1.7.1 in /python (#6689) Bumps [fastavro](https://github.com/fastavro/fastavro) from 1.7.0 to 1.7.1. - [Release notes](https://github.com/fastavro/fastavro/releases) - [Changelog](https://github.com/fastavro/fastavro/blob/master/ChangeLog) - [Commits](https://github.com/fastavro/fastavro/compare/1.7.0...1.7.1) --- updated-dependencies: - dependency-name: fastavro dependency-type: direct:development update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- poetry.lock | 46 +++++++++++++++++++++++----------------------- pyproject.toml | 2 +- 2 files changed, 24 insertions(+), 24 deletions(-) diff --git a/poetry.lock b/poetry.lock index 91f41d758f..650c094a48 100644 --- a/poetry.lock +++ b/poetry.lock @@ -725,33 +725,33 @@ test = ["pytest (>=6)"] [[package]] name = "fastavro" -version = "1.7.0" +version = "1.7.1" description = "Fast read/write of AVRO files" category = "dev" optional = false python-versions = ">=3.7" files = [ - {file = "fastavro-1.7.0-cp310-cp310-macosx_10_14_x86_64.whl", hash = "sha256:ab3387a06e272980fa034f5c62f7063977b77df6416d3d30a4d3b49cc8827566"}, - {file = "fastavro-1.7.0-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:216132bc54da19e97e1531dd69c86282408d4c797749d83b01b3a00862a180de"}, - {file = "fastavro-1.7.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:c20cf6cd8098bb93c2cffd53d03ccea1dcf9ec594a5c83963acf29a2882f8693"}, - {file = "fastavro-1.7.0-cp310-cp310-win_amd64.whl", hash = "sha256:54b60e79506a456bcfc940560fa2c73a7a8e3ddc58a1ae4d94fdd99f6b02aef0"}, - {file = "fastavro-1.7.0-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:4da6d83abd04a804310667f8d1fc23d44e9ed91ed9a9bc9c1fcd906e0c403b12"}, - {file = "fastavro-1.7.0-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:7c664302f94064adeb93403c61c74e07b9710526403eba3b59169f99bb99c55c"}, - {file = "fastavro-1.7.0-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:7aab162f02e64bf82d0282430a3c6ec7a36982b1c5d479e7dcc278e6d62a84b8"}, - {file = "fastavro-1.7.0-cp311-cp311-win_amd64.whl", hash = "sha256:63d0d2a2bb3e85d006c834633be51b105a50b0dc7cc8423b06f30601b532adf4"}, - {file = "fastavro-1.7.0-cp37-cp37m-macosx_10_15_x86_64.whl", hash = "sha256:a07a3245049529d6d9028d664361cc4990e74d035d2303875295e2f7b97eba09"}, - {file = "fastavro-1.7.0-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:7df57d31cb58770a9066790250e9f4ec91242c69e1dc62ea732a6fb2406a8f96"}, - {file = "fastavro-1.7.0-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:b905634b5763ca08c9f7b51486e2c3ae7907f5d9bc48208c69b16ccbe8455e90"}, - {file = "fastavro-1.7.0-cp37-cp37m-win_amd64.whl", hash = "sha256:749206f1cec3d7429546e49db5708f4571497d35181b6b334c4844133f230515"}, - {file = "fastavro-1.7.0-cp38-cp38-macosx_10_15_x86_64.whl", hash = "sha256:a0ebd5c1269085179de4b3f072de274fb66a471ecbc5245bd8684e6f94943c2f"}, - {file = "fastavro-1.7.0-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a7e35b94b692f8cca0096c89abf1937efed66252dea0b3b3165babfb3c289fb7"}, - {file = "fastavro-1.7.0-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:202424a5a7831b773f0f2cc2f82e89ed1726051fd5994f13dc678514144e10d4"}, - {file = "fastavro-1.7.0-cp38-cp38-win_amd64.whl", hash = "sha256:601f8c2ec166bd721f4b12eafe195dd1373d3f8dce4fe2425abd2df3e3968ac7"}, - {file = "fastavro-1.7.0-cp39-cp39-macosx_10_15_x86_64.whl", hash = "sha256:91cc9be740abca894082af4fe3ab9db057d4e5fa783cfa9a94c02ec041bf4346"}, - {file = "fastavro-1.7.0-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:9e4463b00242e4baf52d499aeefab46a26d9dd18d808d4219cd4d21089da540e"}, - {file = "fastavro-1.7.0-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:f7348d318858203bd108e6bcde177d8a6f0590b52bc624d815f26fb6c37029bb"}, - {file = "fastavro-1.7.0-cp39-cp39-win_amd64.whl", hash = "sha256:492e8902063aa1c73170e11c18495fcaa86b71eae3513ef83ba438ca02b16b34"}, - {file = "fastavro-1.7.0.tar.gz", hash = "sha256:4b1205f46489b4032d3155c1ab44d9824be0c7454df98d3a5bd22b78b98f23c8"}, + {file = "fastavro-1.7.1-cp310-cp310-macosx_11_0_x86_64.whl", hash = "sha256:f8f7f11af8c2c074341217d6247b7ba09cadcd55f899e046c14e3a44afa5fc95"}, + {file = "fastavro-1.7.1-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:11159cf23f5ff4b752b028a77bd2a7941599932527e8a6512779e25b0503f037"}, + {file = "fastavro-1.7.1-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:86ce89743711355eee7d3fec34847de1ab574f4567aa4a667e966711bb2e0cd9"}, + {file = "fastavro-1.7.1-cp310-cp310-win_amd64.whl", hash = "sha256:2e3f5ae42e033dbb6e0efa788c4b8a4e5a59bc2b9cb83f717b6d85176727faed"}, + {file = "fastavro-1.7.1-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:8b671a97e864e4c024061c0d6f93f2768ba0595f917317ca4fe3e99dca6fcf3d"}, + {file = "fastavro-1.7.1-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:e9be910ca1645e0e5c4192d667cfdfa58dff4f8690db5af1ae33602643d41a78"}, + {file = "fastavro-1.7.1-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:6f233a1d0d95265f81a25c274f55017bd39d9b8f7f1387a4235bf8e01841a9ff"}, + {file = "fastavro-1.7.1-cp311-cp311-win_amd64.whl", hash = "sha256:31c924ed2fdad56182b543941cdec9cc384443cc3ca9462e0580afeb4dc64f4b"}, + {file = "fastavro-1.7.1-cp37-cp37m-macosx_10_15_x86_64.whl", hash = "sha256:8ad522c7f2c7791cfe54a7e87af8ac09634872f4fdeef28deec97fab8de38b24"}, + {file = "fastavro-1.7.1-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:74fe0fb24a489e2fd567775935eb55a461fc6c4da8b5e3467245752ac2098284"}, + {file = "fastavro-1.7.1-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:4f7a744569b4b44ea1f4f87e7ea8298e1e2bf23779aa6ef255b95f9f38faad48"}, + {file = "fastavro-1.7.1-cp37-cp37m-win_amd64.whl", hash = "sha256:b793252a6325341890bbbbdca2804d4db3d5d29ff7f15a58fcf84dda440808fa"}, + {file = "fastavro-1.7.1-cp38-cp38-macosx_10_15_x86_64.whl", hash = "sha256:a107a8b333628c0f8a2ece5d5a57c69923719a139b106ace4050206250df4b13"}, + {file = "fastavro-1.7.1-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:182671595683054ec234beec03f485b5c889c21c08e429577ae7929480703409"}, + {file = "fastavro-1.7.1-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:980ce7fdccf328d287e8b789f4e3b422f64c84ed9cd81c05dd7c560c3d8076b1"}, + {file = "fastavro-1.7.1-cp38-cp38-win_amd64.whl", hash = "sha256:a5f104bc5b4986bbbcab170918c4d8ffa4f8efa3ebe8ec190954178630074d5a"}, + {file = "fastavro-1.7.1-cp39-cp39-macosx_11_0_x86_64.whl", hash = "sha256:5ff701d6b228218a3d9c09b392205dd0899afa501a4f14724bef0ce13a719700"}, + {file = "fastavro-1.7.1-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:5cb2d8ea7b21493bb18ff7c68401edbc663bbd8a57016d6cf2c4b0a2dc4464e7"}, + {file = "fastavro-1.7.1-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:e15e8163581983f07a3156902761cf746bbe1e646abd9553cc9a1cede6e23ae9"}, + {file = "fastavro-1.7.1-cp39-cp39-win_amd64.whl", hash = "sha256:4d56cc05ceb1c9f4de19c1f330a4f140af3b944834d63cd0e11517deaea21ee1"}, + {file = "fastavro-1.7.1.tar.gz", hash = "sha256:4b8bcae4ed6343af186e638061cdfbc5331cdb5e026d055099c91d4f07be838c"}, ] [package.extras] @@ -2469,4 +2469,4 @@ snappy = ["python-snappy"] [metadata] lock-version = "2.0" python-versions = "^3.8" -content-hash = "4baf674169ac2e5d6004512d941bcd860e7436f3dd767f5b28b6d21ee7c75949" +content-hash = "6b74903d15252bb9327c9b6867617bf00ca7fb7c971740c4cc5be7e5d6f19a2d" diff --git a/pyproject.toml b/pyproject.toml index 33afb6faef..7a1ecae2f1 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -80,7 +80,7 @@ adlfs = { version = "2023.1.0", optional = true } pytest = "7.2.1" pytest-checkdocs = "2.9.0" pre-commit = "3.0.1" -fastavro = "1.7.0" +fastavro = "1.7.1" coverage = { version = "^7.0.5", extras = ["toml"] } requests-mock = "1.10.0" moto = "^4.1.0" From 49b70624b1f81cf2a7065b94af7f226314dd0558 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 30 Jan 2023 07:31:24 +0100 Subject: [PATCH 353/642] Build: Bump rich from 13.2.0 to 13.3.1 in /python (#6690) --- poetry.lock | 12 ++++++------ pyproject.toml | 2 +- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/poetry.lock b/poetry.lock index 650c094a48..29b21a12ae 100644 --- a/poetry.lock +++ b/poetry.lock @@ -1990,23 +1990,23 @@ tests = ["coverage (>=6.0.0)", "flake8", "mypy", "pytest (>=7.0.0)", "pytest-asy [[package]] name = "rich" -version = "13.2.0" +version = "13.3.1" description = "Render rich text, tables, progress bars, syntax highlighting, markdown and more to the terminal" category = "main" optional = false python-versions = ">=3.7.0" files = [ - {file = "rich-13.2.0-py3-none-any.whl", hash = "sha256:7c963f0d03819221e9ac561e1bc866e3f95a02248c1234daa48954e6d381c003"}, - {file = "rich-13.2.0.tar.gz", hash = "sha256:f1a00cdd3eebf999a15d85ec498bfe0b1a77efe9b34f645768a54132ef444ac5"}, + {file = "rich-13.3.1-py3-none-any.whl", hash = "sha256:8aa57747f3fc3e977684f0176a88e789be314a99f99b43b75d1e9cb5dc6db9e9"}, + {file = "rich-13.3.1.tar.gz", hash = "sha256:125d96d20c92b946b983d0d392b84ff945461e5a06d3867e9f9e575f8697b67f"}, ] [package.dependencies] markdown-it-py = ">=2.1.0,<3.0.0" -pygments = ">=2.6.0,<3.0.0" +pygments = ">=2.14.0,<3.0.0" typing-extensions = {version = ">=4.0.0,<5.0", markers = "python_version < \"3.9\""} [package.extras] -jupyter = ["ipywidgets (>=7.5.1,<8.0.0)"] +jupyter = ["ipywidgets (>=7.5.1,<9)"] [[package]] name = "s3fs" @@ -2469,4 +2469,4 @@ snappy = ["python-snappy"] [metadata] lock-version = "2.0" python-versions = "^3.8" -content-hash = "6b74903d15252bb9327c9b6867617bf00ca7fb7c971740c4cc5be7e5d6f19a2d" +content-hash = "fa445af21e9b4ff93574979f5ad6c84930ecadeb5afc3661e6fd19a76a54c77d" diff --git a/pyproject.toml b/pyproject.toml index 7a1ecae2f1..1d21ba7c45 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -50,7 +50,7 @@ python = "^3.8" mmhash3 = "3.0.1" requests = "2.28.2" click = "8.1.3" -rich = "13.2.0" +rich = "13.3.1" pyyaml = "6.0.0" pydantic = "1.10.4" From ef4927f440be9a54baf3cea64d6d7f02352acf8b Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 30 Jan 2023 07:42:52 +0100 Subject: [PATCH 354/642] Build: Bump coverage from 7.0.5 to 7.1.0 in /python (#6691) --- poetry.lock | 106 ++++++++++++++++++++++++------------------------- pyproject.toml | 2 +- 2 files changed, 54 insertions(+), 54 deletions(-) diff --git a/poetry.lock b/poetry.lock index 29b21a12ae..b9f1a81599 100644 --- a/poetry.lock +++ b/poetry.lock @@ -515,63 +515,63 @@ files = [ [[package]] name = "coverage" -version = "7.0.5" +version = "7.1.0" description = "Code coverage measurement for Python" category = "dev" optional = false python-versions = ">=3.7" files = [ - {file = "coverage-7.0.5-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:2a7f23bbaeb2a87f90f607730b45564076d870f1fb07b9318d0c21f36871932b"}, - {file = "coverage-7.0.5-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:c18d47f314b950dbf24a41787ced1474e01ca816011925976d90a88b27c22b89"}, - {file = "coverage-7.0.5-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:ef14d75d86f104f03dea66c13188487151760ef25dd6b2dbd541885185f05f40"}, - {file = "coverage-7.0.5-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:66e50680e888840c0995f2ad766e726ce71ca682e3c5f4eee82272c7671d38a2"}, - {file = "coverage-7.0.5-cp310-cp310-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:a9fed35ca8c6e946e877893bbac022e8563b94404a605af1d1e6accc7eb73289"}, - {file = "coverage-7.0.5-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:d8d04e755934195bdc1db45ba9e040b8d20d046d04d6d77e71b3b34a8cc002d0"}, - {file = "coverage-7.0.5-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:7e109f1c9a3ece676597831874126555997c48f62bddbcace6ed17be3e372de8"}, - {file = "coverage-7.0.5-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:0a1890fca2962c4f1ad16551d660b46ea77291fba2cc21c024cd527b9d9c8809"}, - {file = "coverage-7.0.5-cp310-cp310-win32.whl", hash = "sha256:be9fcf32c010da0ba40bf4ee01889d6c737658f4ddff160bd7eb9cac8f094b21"}, - {file = "coverage-7.0.5-cp310-cp310-win_amd64.whl", hash = "sha256:cbfcba14a3225b055a28b3199c3d81cd0ab37d2353ffd7f6fd64844cebab31ad"}, - {file = "coverage-7.0.5-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:30b5fec1d34cc932c1bc04017b538ce16bf84e239378b8f75220478645d11fca"}, - {file = "coverage-7.0.5-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:1caed2367b32cc80a2b7f58a9f46658218a19c6cfe5bc234021966dc3daa01f0"}, - {file = "coverage-7.0.5-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:d254666d29540a72d17cc0175746cfb03d5123db33e67d1020e42dae611dc196"}, - {file = "coverage-7.0.5-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:19245c249aa711d954623d94f23cc94c0fd65865661f20b7781210cb97c471c0"}, - {file = "coverage-7.0.5-cp311-cp311-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:7b05ed4b35bf6ee790832f68932baf1f00caa32283d66cc4d455c9e9d115aafc"}, - {file = "coverage-7.0.5-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:29de916ba1099ba2aab76aca101580006adfac5646de9b7c010a0f13867cba45"}, - {file = "coverage-7.0.5-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:e057e74e53db78122a3979f908973e171909a58ac20df05c33998d52e6d35757"}, - {file = "coverage-7.0.5-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:411d4ff9d041be08fdfc02adf62e89c735b9468f6d8f6427f8a14b6bb0a85095"}, - {file = "coverage-7.0.5-cp311-cp311-win32.whl", hash = "sha256:52ab14b9e09ce052237dfe12d6892dd39b0401690856bcfe75d5baba4bfe2831"}, - {file = "coverage-7.0.5-cp311-cp311-win_amd64.whl", hash = "sha256:1f66862d3a41674ebd8d1a7b6f5387fe5ce353f8719040a986551a545d7d83ea"}, - {file = "coverage-7.0.5-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:b69522b168a6b64edf0c33ba53eac491c0a8f5cc94fa4337f9c6f4c8f2f5296c"}, - {file = "coverage-7.0.5-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:436e103950d05b7d7f55e39beeb4d5be298ca3e119e0589c0227e6d0b01ee8c7"}, - {file = "coverage-7.0.5-cp37-cp37m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:b8c56bec53d6e3154eaff6ea941226e7bd7cc0d99f9b3756c2520fc7a94e6d96"}, - {file = "coverage-7.0.5-cp37-cp37m-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:7a38362528a9115a4e276e65eeabf67dcfaf57698e17ae388599568a78dcb029"}, - {file = "coverage-7.0.5-cp37-cp37m-musllinux_1_1_aarch64.whl", hash = "sha256:f67472c09a0c7486e27f3275f617c964d25e35727af952869dd496b9b5b7f6a3"}, - {file = "coverage-7.0.5-cp37-cp37m-musllinux_1_1_i686.whl", hash = "sha256:220e3fa77d14c8a507b2d951e463b57a1f7810a6443a26f9b7591ef39047b1b2"}, - {file = "coverage-7.0.5-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:ecb0f73954892f98611e183f50acdc9e21a4653f294dfbe079da73c6378a6f47"}, - {file = "coverage-7.0.5-cp37-cp37m-win32.whl", hash = "sha256:d8f3e2e0a1d6777e58e834fd5a04657f66affa615dae61dd67c35d1568c38882"}, - {file = "coverage-7.0.5-cp37-cp37m-win_amd64.whl", hash = "sha256:9e662e6fc4f513b79da5d10a23edd2b87685815b337b1a30cd11307a6679148d"}, - {file = "coverage-7.0.5-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:790e4433962c9f454e213b21b0fd4b42310ade9c077e8edcb5113db0818450cb"}, - {file = "coverage-7.0.5-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:49640bda9bda35b057b0e65b7c43ba706fa2335c9a9896652aebe0fa399e80e6"}, - {file = "coverage-7.0.5-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:d66187792bfe56f8c18ba986a0e4ae44856b1c645336bd2c776e3386da91e1dd"}, - {file = "coverage-7.0.5-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:276f4cd0001cd83b00817c8db76730938b1ee40f4993b6a905f40a7278103b3a"}, - {file = "coverage-7.0.5-cp38-cp38-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:95304068686545aa368b35dfda1cdfbbdbe2f6fe43de4a2e9baa8ebd71be46e2"}, - {file = "coverage-7.0.5-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:17e01dd8666c445025c29684d4aabf5a90dc6ef1ab25328aa52bedaa95b65ad7"}, - {file = "coverage-7.0.5-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:ea76dbcad0b7b0deb265d8c36e0801abcddf6cc1395940a24e3595288b405ca0"}, - {file = "coverage-7.0.5-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:50a6adc2be8edd7ee67d1abc3cd20678987c7b9d79cd265de55941e3d0d56499"}, - {file = "coverage-7.0.5-cp38-cp38-win32.whl", hash = "sha256:e4ce984133b888cc3a46867c8b4372c7dee9cee300335e2925e197bcd45b9e16"}, - {file = "coverage-7.0.5-cp38-cp38-win_amd64.whl", hash = "sha256:4a950f83fd3f9bca23b77442f3a2b2ea4ac900944d8af9993743774c4fdc57af"}, - {file = "coverage-7.0.5-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:3c2155943896ac78b9b0fd910fb381186d0c345911f5333ee46ac44c8f0e43ab"}, - {file = "coverage-7.0.5-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:54f7e9705e14b2c9f6abdeb127c390f679f6dbe64ba732788d3015f7f76ef637"}, - {file = "coverage-7.0.5-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:0ee30375b409d9a7ea0f30c50645d436b6f5dfee254edffd27e45a980ad2c7f4"}, - {file = "coverage-7.0.5-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:b78729038abea6a5df0d2708dce21e82073463b2d79d10884d7d591e0f385ded"}, - {file = "coverage-7.0.5-cp39-cp39-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:13250b1f0bd023e0c9f11838bdeb60214dd5b6aaf8e8d2f110c7e232a1bff83b"}, - {file = "coverage-7.0.5-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:2c407b1950b2d2ffa091f4e225ca19a66a9bd81222f27c56bd12658fc5ca1209"}, - {file = "coverage-7.0.5-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:c76a3075e96b9c9ff00df8b5f7f560f5634dffd1658bafb79eb2682867e94f78"}, - {file = "coverage-7.0.5-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:f26648e1b3b03b6022b48a9b910d0ae209e2d51f50441db5dce5b530fad6d9b1"}, - {file = "coverage-7.0.5-cp39-cp39-win32.whl", hash = "sha256:ba3027deb7abf02859aca49c865ece538aee56dcb4871b4cced23ba4d5088904"}, - {file = "coverage-7.0.5-cp39-cp39-win_amd64.whl", hash = "sha256:949844af60ee96a376aac1ded2a27e134b8c8d35cc006a52903fc06c24a3296f"}, - {file = "coverage-7.0.5-pp37.pp38.pp39-none-any.whl", hash = "sha256:b9727ac4f5cf2cbf87880a63870b5b9730a8ae3a4a360241a0fdaa2f71240ff0"}, - {file = "coverage-7.0.5.tar.gz", hash = "sha256:051afcbd6d2ac39298d62d340f94dbb6a1f31de06dfaf6fcef7b759dd3860c45"}, + {file = "coverage-7.1.0-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:3b946bbcd5a8231383450b195cfb58cb01cbe7f8949f5758566b881df4b33baf"}, + {file = "coverage-7.1.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:ec8e767f13be637d056f7e07e61d089e555f719b387a7070154ad80a0ff31801"}, + {file = "coverage-7.1.0-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:d4a5a5879a939cb84959d86869132b00176197ca561c664fc21478c1eee60d75"}, + {file = "coverage-7.1.0-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:b643cb30821e7570c0aaf54feaf0bfb630b79059f85741843e9dc23f33aaca2c"}, + {file = "coverage-7.1.0-cp310-cp310-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:32df215215f3af2c1617a55dbdfb403b772d463d54d219985ac7cd3bf124cada"}, + {file = "coverage-7.1.0-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:33d1ae9d4079e05ac4cc1ef9e20c648f5afabf1a92adfaf2ccf509c50b85717f"}, + {file = "coverage-7.1.0-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:29571503c37f2ef2138a306d23e7270687c0efb9cab4bd8038d609b5c2393a3a"}, + {file = "coverage-7.1.0-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:63ffd21aa133ff48c4dff7adcc46b7ec8b565491bfc371212122dd999812ea1c"}, + {file = "coverage-7.1.0-cp310-cp310-win32.whl", hash = "sha256:4b14d5e09c656de5038a3f9bfe5228f53439282abcab87317c9f7f1acb280352"}, + {file = "coverage-7.1.0-cp310-cp310-win_amd64.whl", hash = "sha256:8361be1c2c073919500b6601220a6f2f98ea0b6d2fec5014c1d9cfa23dd07038"}, + {file = "coverage-7.1.0-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:da9b41d4539eefd408c46725fb76ecba3a50a3367cafb7dea5f250d0653c1040"}, + {file = "coverage-7.1.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:c5b15ed7644ae4bee0ecf74fee95808dcc34ba6ace87e8dfbf5cb0dc20eab45a"}, + {file = "coverage-7.1.0-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:d12d076582507ea460ea2a89a8c85cb558f83406c8a41dd641d7be9a32e1274f"}, + {file = "coverage-7.1.0-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:e2617759031dae1bf183c16cef8fcfb3de7617f394c813fa5e8e46e9b82d4222"}, + {file = "coverage-7.1.0-cp311-cp311-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:c4e4881fa9e9667afcc742f0c244d9364d197490fbc91d12ac3b5de0bf2df146"}, + {file = "coverage-7.1.0-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:9d58885215094ab4a86a6aef044e42994a2bd76a446dc59b352622655ba6621b"}, + {file = "coverage-7.1.0-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:ffeeb38ee4a80a30a6877c5c4c359e5498eec095878f1581453202bfacc8fbc2"}, + {file = "coverage-7.1.0-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:3baf5f126f30781b5e93dbefcc8271cb2491647f8283f20ac54d12161dff080e"}, + {file = "coverage-7.1.0-cp311-cp311-win32.whl", hash = "sha256:ded59300d6330be27bc6cf0b74b89ada58069ced87c48eaf9344e5e84b0072f7"}, + {file = "coverage-7.1.0-cp311-cp311-win_amd64.whl", hash = "sha256:6a43c7823cd7427b4ed763aa7fb63901ca8288591323b58c9cd6ec31ad910f3c"}, + {file = "coverage-7.1.0-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:7a726d742816cb3a8973c8c9a97539c734b3a309345236cd533c4883dda05b8d"}, + {file = "coverage-7.1.0-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:bc7c85a150501286f8b56bd8ed3aa4093f4b88fb68c0843d21ff9656f0009d6a"}, + {file = "coverage-7.1.0-cp37-cp37m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:f5b4198d85a3755d27e64c52f8c95d6333119e49fd001ae5798dac872c95e0f8"}, + {file = "coverage-7.1.0-cp37-cp37m-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:ddb726cb861c3117a553f940372a495fe1078249ff5f8a5478c0576c7be12050"}, + {file = "coverage-7.1.0-cp37-cp37m-musllinux_1_1_aarch64.whl", hash = "sha256:51b236e764840a6df0661b67e50697aaa0e7d4124ca95e5058fa3d7cbc240b7c"}, + {file = "coverage-7.1.0-cp37-cp37m-musllinux_1_1_i686.whl", hash = "sha256:7ee5c9bb51695f80878faaa5598040dd6c9e172ddcf490382e8aedb8ec3fec8d"}, + {file = "coverage-7.1.0-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:c31b75ae466c053a98bf26843563b3b3517b8f37da4d47b1c582fdc703112bc3"}, + {file = "coverage-7.1.0-cp37-cp37m-win32.whl", hash = "sha256:3b155caf3760408d1cb903b21e6a97ad4e2bdad43cbc265e3ce0afb8e0057e73"}, + {file = "coverage-7.1.0-cp37-cp37m-win_amd64.whl", hash = "sha256:2a60d6513781e87047c3e630b33b4d1e89f39836dac6e069ffee28c4786715f5"}, + {file = "coverage-7.1.0-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:f2cba5c6db29ce991029b5e4ac51eb36774458f0a3b8d3137241b32d1bb91f06"}, + {file = "coverage-7.1.0-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:beeb129cacea34490ffd4d6153af70509aa3cda20fdda2ea1a2be870dfec8d52"}, + {file = "coverage-7.1.0-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:0c45948f613d5d18c9ec5eaa203ce06a653334cf1bd47c783a12d0dd4fd9c851"}, + {file = "coverage-7.1.0-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:ef382417db92ba23dfb5864a3fc9be27ea4894e86620d342a116b243ade5d35d"}, + {file = "coverage-7.1.0-cp38-cp38-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:7c7c0d0827e853315c9bbd43c1162c006dd808dbbe297db7ae66cd17b07830f0"}, + {file = "coverage-7.1.0-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:e5cdbb5cafcedea04924568d990e20ce7f1945a1dd54b560f879ee2d57226912"}, + {file = "coverage-7.1.0-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:9817733f0d3ea91bea80de0f79ef971ae94f81ca52f9b66500c6a2fea8e4b4f8"}, + {file = "coverage-7.1.0-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:218fe982371ac7387304153ecd51205f14e9d731b34fb0568181abaf7b443ba0"}, + {file = "coverage-7.1.0-cp38-cp38-win32.whl", hash = "sha256:04481245ef966fbd24ae9b9e537ce899ae584d521dfbe78f89cad003c38ca2ab"}, + {file = "coverage-7.1.0-cp38-cp38-win_amd64.whl", hash = "sha256:8ae125d1134bf236acba8b83e74c603d1b30e207266121e76484562bc816344c"}, + {file = "coverage-7.1.0-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:2bf1d5f2084c3932b56b962a683074a3692bce7cabd3aa023c987a2a8e7612f6"}, + {file = "coverage-7.1.0-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:98b85dd86514d889a2e3dd22ab3c18c9d0019e696478391d86708b805f4ea0fa"}, + {file = "coverage-7.1.0-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:38da2db80cc505a611938d8624801158e409928b136c8916cd2e203970dde4dc"}, + {file = "coverage-7.1.0-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:3164d31078fa9efe406e198aecd2a02d32a62fecbdef74f76dad6a46c7e48311"}, + {file = "coverage-7.1.0-cp39-cp39-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:db61a79c07331e88b9a9974815c075fbd812bc9dbc4dc44b366b5368a2936063"}, + {file = "coverage-7.1.0-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:9ccb092c9ede70b2517a57382a601619d20981f56f440eae7e4d7eaafd1d1d09"}, + {file = "coverage-7.1.0-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:33ff26d0f6cc3ca8de13d14fde1ff8efe1456b53e3f0273e63cc8b3c84a063d8"}, + {file = "coverage-7.1.0-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:d47dd659a4ee952e90dc56c97d78132573dc5c7b09d61b416a9deef4ebe01a0c"}, + {file = "coverage-7.1.0-cp39-cp39-win32.whl", hash = "sha256:d248cd4a92065a4d4543b8331660121b31c4148dd00a691bfb7a5cdc7483cfa4"}, + {file = "coverage-7.1.0-cp39-cp39-win_amd64.whl", hash = "sha256:7ed681b0f8e8bcbbffa58ba26fcf5dbc8f79e7997595bf071ed5430d8c08d6f3"}, + {file = "coverage-7.1.0-pp37.pp38.pp39-none-any.whl", hash = "sha256:755e89e32376c850f826c425ece2c35a4fc266c081490eb0a841e7c1cb0d3bda"}, + {file = "coverage-7.1.0.tar.gz", hash = "sha256:10188fe543560ec4874f974b5305cd1a8bdcfa885ee00ea3a03733464c4ca265"}, ] [package.dependencies] @@ -2469,4 +2469,4 @@ snappy = ["python-snappy"] [metadata] lock-version = "2.0" python-versions = "^3.8" -content-hash = "fa445af21e9b4ff93574979f5ad6c84930ecadeb5afc3661e6fd19a76a54c77d" +content-hash = "2eb5bd49695043878f3c76ffc40a115a1ccc6d4482e32f57b1d8cfae417b2fff" diff --git a/pyproject.toml b/pyproject.toml index 1d21ba7c45..af672d86c2 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -81,7 +81,7 @@ pytest = "7.2.1" pytest-checkdocs = "2.9.0" pre-commit = "3.0.1" fastavro = "1.7.1" -coverage = { version = "^7.0.5", extras = ["toml"] } +coverage = { version = "^7.1.0", extras = ["toml"] } requests-mock = "1.10.0" moto = "^4.1.0" typing-extensions = '4.4.0' From 89017525a46ca27b998a5639b7b576f22c6c7844 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 30 Jan 2023 15:00:46 +0100 Subject: [PATCH 355/642] Build: Bump pyarrow from 10.0.1 to 11.0.0 in /python (#6692) Bumps [pyarrow](https://github.com/apache/arrow) from 10.0.1 to 11.0.0. - [Release notes](https://github.com/apache/arrow/releases) - [Commits](https://github.com/apache/arrow/compare/go/v10.0.1...apache-arrow-11.0.0) --- updated-dependencies: - dependency-name: pyarrow dependency-type: direct:production update-type: version-update:semver-major ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- poetry.lock | 54 +++++++++++++++++++++++++------------------------- pyproject.toml | 2 +- 2 files changed, 28 insertions(+), 28 deletions(-) diff --git a/poetry.lock b/poetry.lock index b9f1a81599..74fc860281 100644 --- a/poetry.lock +++ b/poetry.lock @@ -1535,37 +1535,37 @@ virtualenv = ">=20.10.0" [[package]] name = "pyarrow" -version = "10.0.1" +version = "11.0.0" description = "Python library for Apache Arrow" category = "main" optional = true python-versions = ">=3.7" files = [ - {file = "pyarrow-10.0.1-cp310-cp310-macosx_10_14_x86_64.whl", hash = "sha256:e00174764a8b4e9d8d5909b6d19ee0c217a6cf0232c5682e31fdfbd5a9f0ae52"}, - {file = "pyarrow-10.0.1-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:6f7a7dbe2f7f65ac1d0bd3163f756deb478a9e9afc2269557ed75b1b25ab3610"}, - {file = "pyarrow-10.0.1-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:cb627673cb98708ef00864e2e243f51ba7b4c1b9f07a1d821f98043eccd3f585"}, - {file = "pyarrow-10.0.1-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:ba71e6fc348c92477586424566110d332f60d9a35cb85278f42e3473bc1373da"}, - {file = "pyarrow-10.0.1-cp310-cp310-win_amd64.whl", hash = "sha256:7b4ede715c004b6fc535de63ef79fa29740b4080639a5ff1ea9ca84e9282f349"}, - {file = "pyarrow-10.0.1-cp311-cp311-macosx_10_14_x86_64.whl", hash = "sha256:e3fe5049d2e9ca661d8e43fab6ad5a4c571af12d20a57dffc392a014caebef65"}, - {file = "pyarrow-10.0.1-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:254017ca43c45c5098b7f2a00e995e1f8346b0fb0be225f042838323bb55283c"}, - {file = "pyarrow-10.0.1-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:70acca1ece4322705652f48db65145b5028f2c01c7e426c5d16a30ba5d739c24"}, - {file = "pyarrow-10.0.1-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:abb57334f2c57979a49b7be2792c31c23430ca02d24becd0b511cbe7b6b08649"}, - {file = "pyarrow-10.0.1-cp311-cp311-win_amd64.whl", hash = "sha256:1765a18205eb1e02ccdedb66049b0ec148c2a0cb52ed1fb3aac322dfc086a6ee"}, - {file = "pyarrow-10.0.1-cp37-cp37m-macosx_10_14_x86_64.whl", hash = "sha256:61f4c37d82fe00d855d0ab522c685262bdeafd3fbcb5fe596fe15025fbc7341b"}, - {file = "pyarrow-10.0.1-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:e141a65705ac98fa52a9113fe574fdaf87fe0316cde2dffe6b94841d3c61544c"}, - {file = "pyarrow-10.0.1-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:bf26f809926a9d74e02d76593026f0aaeac48a65b64f1bb17eed9964bfe7ae1a"}, - {file = "pyarrow-10.0.1-cp37-cp37m-win_amd64.whl", hash = "sha256:443eb9409b0cf78df10ced326490e1a300205a458fbeb0767b6b31ab3ebae6b2"}, - {file = "pyarrow-10.0.1-cp38-cp38-macosx_10_14_x86_64.whl", hash = "sha256:f2d00aa481becf57098e85d99e34a25dba5a9ade2f44eb0b7d80c80f2984fc03"}, - {file = "pyarrow-10.0.1-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:b1fc226d28c7783b52a84d03a66573d5a22e63f8a24b841d5fc68caeed6784d4"}, - {file = "pyarrow-10.0.1-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:efa59933b20183c1c13efc34bd91efc6b2997377c4c6ad9272da92d224e3beb1"}, - {file = "pyarrow-10.0.1-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:668e00e3b19f183394388a687d29c443eb000fb3fe25599c9b4762a0afd37775"}, - {file = "pyarrow-10.0.1-cp38-cp38-win_amd64.whl", hash = "sha256:d1bc6e4d5d6f69e0861d5d7f6cf4d061cf1069cb9d490040129877acf16d4c2a"}, - {file = "pyarrow-10.0.1-cp39-cp39-macosx_10_14_x86_64.whl", hash = "sha256:42ba7c5347ce665338f2bc64685d74855900200dac81a972d49fe127e8132f75"}, - {file = "pyarrow-10.0.1-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:b069602eb1fc09f1adec0a7bdd7897f4d25575611dfa43543c8b8a75d99d6874"}, - {file = "pyarrow-10.0.1-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:94fb4a0c12a2ac1ed8e7e2aa52aade833772cf2d3de9dde685401b22cec30002"}, - {file = "pyarrow-10.0.1-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:db0c5986bf0808927f49640582d2032a07aa49828f14e51f362075f03747d198"}, - {file = "pyarrow-10.0.1-cp39-cp39-win_amd64.whl", hash = "sha256:0ec7587d759153f452d5263dbc8b1af318c4609b607be2bd5127dcda6708cdb1"}, - {file = "pyarrow-10.0.1.tar.gz", hash = "sha256:1a14f57a5f472ce8234f2964cd5184cccaa8df7e04568c64edc33b23eb285dd5"}, + {file = "pyarrow-11.0.0-cp310-cp310-macosx_10_14_x86_64.whl", hash = "sha256:40bb42afa1053c35c749befbe72f6429b7b5f45710e85059cdd534553ebcf4f2"}, + {file = "pyarrow-11.0.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:7c28b5f248e08dea3b3e0c828b91945f431f4202f1a9fe84d1012a761324e1ba"}, + {file = "pyarrow-11.0.0-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a37bc81f6c9435da3c9c1e767324ac3064ffbe110c4e460660c43e144be4ed85"}, + {file = "pyarrow-11.0.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:ad7c53def8dbbc810282ad308cc46a523ec81e653e60a91c609c2233ae407689"}, + {file = "pyarrow-11.0.0-cp310-cp310-win_amd64.whl", hash = "sha256:25aa11c443b934078bfd60ed63e4e2d42461682b5ac10f67275ea21e60e6042c"}, + {file = "pyarrow-11.0.0-cp311-cp311-macosx_10_14_x86_64.whl", hash = "sha256:e217d001e6389b20a6759392a5ec49d670757af80101ee6b5f2c8ff0172e02ca"}, + {file = "pyarrow-11.0.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:ad42bb24fc44c48f74f0d8c72a9af16ba9a01a2ccda5739a517aa860fa7e3d56"}, + {file = "pyarrow-11.0.0-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:2d942c690ff24a08b07cb3df818f542a90e4d359381fbff71b8f2aea5bf58841"}, + {file = "pyarrow-11.0.0-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:f010ce497ca1b0f17a8243df3048055c0d18dcadbcc70895d5baf8921f753de5"}, + {file = "pyarrow-11.0.0-cp311-cp311-win_amd64.whl", hash = "sha256:2f51dc7ca940fdf17893227edb46b6784d37522ce08d21afc56466898cb213b2"}, + {file = "pyarrow-11.0.0-cp37-cp37m-macosx_10_14_x86_64.whl", hash = "sha256:1cbcfcbb0e74b4d94f0b7dde447b835a01bc1d16510edb8bb7d6224b9bf5bafc"}, + {file = "pyarrow-11.0.0-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:aaee8f79d2a120bf3e032d6d64ad20b3af6f56241b0ffc38d201aebfee879d00"}, + {file = "pyarrow-11.0.0-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:410624da0708c37e6a27eba321a72f29d277091c8f8d23f72c92bada4092eb5e"}, + {file = "pyarrow-11.0.0-cp37-cp37m-win_amd64.whl", hash = "sha256:2d53ba72917fdb71e3584ffc23ee4fcc487218f8ff29dd6df3a34c5c48fe8c06"}, + {file = "pyarrow-11.0.0-cp38-cp38-macosx_10_14_x86_64.whl", hash = "sha256:f12932e5a6feb5c58192209af1d2607d488cb1d404fbc038ac12ada60327fa34"}, + {file = "pyarrow-11.0.0-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:41a1451dd895c0b2964b83d91019e46f15b5564c7ecd5dcb812dadd3f05acc97"}, + {file = "pyarrow-11.0.0-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:becc2344be80e5dce4e1b80b7c650d2fc2061b9eb339045035a1baa34d5b8f1c"}, + {file = "pyarrow-11.0.0-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:8f40be0d7381112a398b93c45a7e69f60261e7b0269cc324e9f739ce272f4f70"}, + {file = "pyarrow-11.0.0-cp38-cp38-win_amd64.whl", hash = "sha256:362a7c881b32dc6b0eccf83411a97acba2774c10edcec715ccaab5ebf3bb0835"}, + {file = "pyarrow-11.0.0-cp39-cp39-macosx_10_14_x86_64.whl", hash = "sha256:ccbf29a0dadfcdd97632b4f7cca20a966bb552853ba254e874c66934931b9841"}, + {file = "pyarrow-11.0.0-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:3e99be85973592051e46412accea31828da324531a060bd4585046a74ba45854"}, + {file = "pyarrow-11.0.0-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:69309be84dcc36422574d19c7d3a30a7ea43804f12552356d1ab2a82a713c418"}, + {file = "pyarrow-11.0.0-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:da93340fbf6f4e2a62815064383605b7ffa3e9eeb320ec839995b1660d69f89b"}, + {file = "pyarrow-11.0.0-cp39-cp39-win_amd64.whl", hash = "sha256:caad867121f182d0d3e1a0d36f197df604655d0b466f1bc9bafa903aa95083e4"}, + {file = "pyarrow-11.0.0.tar.gz", hash = "sha256:5461c57dbdb211a632a48facb9b39bbeb8a7905ec95d768078525283caef5f6d"}, ] [package.dependencies] @@ -2469,4 +2469,4 @@ snappy = ["python-snappy"] [metadata] lock-version = "2.0" python-versions = "^3.8" -content-hash = "2eb5bd49695043878f3c76ffc40a115a1ccc6d4482e32f57b1d8cfae417b2fff" +content-hash = "936a690f66edaa27e6491b6a2f43585be56e38fbda0faf274cd398a3489db10c" diff --git a/pyproject.toml b/pyproject.toml index af672d86c2..735e963170 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -60,7 +60,7 @@ pyparsing = "3.0.9" zstandard = "0.19.0" -pyarrow = { version = "10.0.1", optional = true } +pyarrow = { version = "11.0.0", optional = true } pandas = { version = "1.5.3", optional = true } From 137ea70bcfb896711e3087c7bddb09867d978b39 Mon Sep 17 00:00:00 2001 From: Amogh Jahagirdar Date: Tue, 31 Jan 2023 00:36:02 -0800 Subject: [PATCH 356/642] Python: Update pyproject.toml to include dev folder (#6705) --- pyproject.toml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index 735e963170..8b30778a39 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -39,11 +39,13 @@ packages = [ { from = "vendor", include = "fb303" }, { from = "vendor", include = "hive_metastore" }, { include = "tests", format = "sdist" }, - { include = "dev/check-license", format = "sdist" }, { include = "Makefile", format = "sdist" }, { include = "NOTICE", format = ["sdist", "wheel"] } ] +include = [ + {path = "dev", format = "sdist"} +] [tool.poetry.dependencies] python = "^3.8" From 7b0e40eb8f62c323713a44da30129315cfee9378 Mon Sep 17 00:00:00 2001 From: Fokko Driesprong Date: Tue, 31 Jan 2023 19:33:58 +0000 Subject: [PATCH 357/642] Python: Optimize PyArrow reads (#6673) --- pyiceberg/expressions/visitors.py | 46 +++++++++++++++++++++------- pyiceberg/io/pyarrow.py | 50 ++++++++++++++++--------------- pyiceberg/schema.py | 12 ++++++++ 3 files changed, 74 insertions(+), 34 deletions(-) diff --git a/pyiceberg/expressions/visitors.py b/pyiceberg/expressions/visitors.py index a76b183a98..81e0ebc559 100644 --- a/pyiceberg/expressions/visitors.py +++ b/pyiceberg/expressions/visitors.py @@ -24,6 +24,7 @@ Set, Tuple, TypeVar, + Union, ) from pyiceberg.conversions import from_bytes @@ -64,7 +65,10 @@ FloatType, IcebergType, PrimitiveType, + TimestampType, + TimestamptzType, ) +from pyiceberg.utils.datetime import micros_to_timestamp, micros_to_timestamptz T = TypeVar("T") @@ -884,11 +888,30 @@ def rewrite_to_dnf(expr: BooleanExpression) -> Tuple[BooleanExpression, ...]: class ExpressionToPlainFormat(BoundBooleanExpressionVisitor[List[Tuple[str, str, Any]]]): + cast_int_to_date: bool + + def __init__(self, cast_int_to_date: bool = False) -> None: + self.cast_int_to_date = cast_int_to_date + + def _cast_if_necessary(self, iceberg_type: IcebergType, literal: Union[L, Set[L]]) -> Union[L, Set[L]]: + if self.cast_int_to_date: + iceberg_type_class = type(iceberg_type) + conversions = {TimestampType: micros_to_timestamp, TimestamptzType: micros_to_timestamptz} + if iceberg_type_class in conversions: + conversion_function = conversions[iceberg_type_class] + if isinstance(literal, set): + return {conversion_function(lit) for lit in literal} # type: ignore + else: + return conversion_function(literal) # type: ignore + return literal + def visit_in(self, term: BoundTerm[L], literals: Set[L]) -> List[Tuple[str, str, Any]]: - return [(term.ref().field.name, "in", literals)] + field = term.ref().field + return [(term.ref().field.name, "in", self._cast_if_necessary(field.field_type, literals))] def visit_not_in(self, term: BoundTerm[L], literals: Set[L]) -> List[Tuple[str, str, Any]]: - return [(term.ref().field.name, "not in", literals)] + field = term.ref().field + return [(field.name, "not in", self._cast_if_necessary(field.field_type, literals))] def visit_is_nan(self, term: BoundTerm[L]) -> List[Tuple[str, str, Any]]: return [(term.ref().field.name, "==", float("nan"))] @@ -903,22 +926,22 @@ def visit_not_null(self, term: BoundTerm[L]) -> List[Tuple[str, str, Any]]: return [(term.ref().field.name, "!=", None)] def visit_equal(self, term: BoundTerm[L], literal: Literal[L]) -> List[Tuple[str, str, Any]]: - return [(term.ref().field.name, "==", literal.value)] + return [(term.ref().field.name, "==", self._cast_if_necessary(term.ref().field.field_type, literal.value))] def visit_not_equal(self, term: BoundTerm[L], literal: Literal[L]) -> List[Tuple[str, str, Any]]: - return [(term.ref().field.name, "!=", literal.value)] + return [(term.ref().field.name, "!=", self._cast_if_necessary(term.ref().field.field_type, literal.value))] def visit_greater_than_or_equal(self, term: BoundTerm[L], literal: Literal[L]) -> List[Tuple[str, str, Any]]: - return [(term.ref().field.name, ">=", literal.value)] + return [(term.ref().field.name, ">=", self._cast_if_necessary(term.ref().field.field_type, literal.value))] def visit_greater_than(self, term: BoundTerm[L], literal: Literal[L]) -> List[Tuple[str, str, Any]]: - return [(term.ref().field.name, ">", literal.value)] + return [(term.ref().field.name, ">", self._cast_if_necessary(term.ref().field.field_type, literal.value))] def visit_less_than(self, term: BoundTerm[L], literal: Literal[L]) -> List[Tuple[str, str, Any]]: - return [(term.ref().field.name, "<", literal.value)] + return [(term.ref().field.name, "<", self._cast_if_necessary(term.ref().field.field_type, literal.value))] def visit_less_than_or_equal(self, term: BoundTerm[L], literal: Literal[L]) -> List[Tuple[str, str, Any]]: - return [(term.ref().field.name, "<=", literal.value)] + return [(term.ref().field.name, "<=", self._cast_if_necessary(term.ref().field.field_type, literal.value))] def visit_true(self) -> List[Tuple[str, str, Any]]: return [] # Not supported @@ -940,7 +963,9 @@ def visit_or( raise ValueError(f"Not allowed: {left_result} || {right_result}") -def expression_to_plain_format(expressions: Tuple[BooleanExpression, ...]) -> List[List[Tuple[str, str, Any]]]: +def expression_to_plain_format( + expressions: Tuple[BooleanExpression, ...], cast_int_to_datetime: bool = False +) -> List[List[Tuple[str, str, Any]]]: """Formats a Disjunctive Normal Form expression into the format that can be fed into: - https://arrow.apache.org/docs/python/generated/pyarrow.parquet.read_table.html @@ -959,4 +984,5 @@ def expression_to_plain_format(expressions: Tuple[BooleanExpression, ...]) -> Li Formatter filter compatible with Dask and PyArrow """ # In the form of expr1 ∨ expr2 ∨ ... ∨ exprN - return [visit(expression, ExpressionToPlainFormat()) for expression in expressions] + visitor = ExpressionToPlainFormat(cast_int_to_datetime) + return [visit(expression, visitor) for expression in expressions] diff --git a/pyiceberg/io/pyarrow.py b/pyiceberg/io/pyarrow.py index 848ec4e0c8..e65a292d71 100644 --- a/pyiceberg/io/pyarrow.py +++ b/pyiceberg/io/pyarrow.py @@ -42,7 +42,6 @@ import pyarrow as pa import pyarrow.compute as pc -import pyarrow.dataset as ds import pyarrow.parquet as pq from pyarrow.fs import ( FileInfo, @@ -62,7 +61,9 @@ from pyiceberg.expressions.visitors import ( BoundBooleanExpressionVisitor, bind, + expression_to_plain_format, extract_field_ids, + rewrite_to_dnf, translate_column_names, ) from pyiceberg.expressions.visitors import visit as boolean_expression_visit @@ -493,31 +494,32 @@ def _file_to_table( ) file_schema = Schema.parse_raw(schema_raw) - pyarrow_filter = None - if bound_row_filter is not AlwaysTrue(): - translated_row_filter = translate_column_names(bound_row_filter, file_schema, case_sensitive=case_sensitive) - bound_file_filter = bind(file_schema, translated_row_filter, case_sensitive=case_sensitive) - pyarrow_filter = expression_to_pyarrow(bound_file_filter) - - file_project_schema = prune_columns(file_schema, projected_field_ids, select_full_types=False) - - if file_schema is None: - raise ValueError(f"Missing Iceberg schema in Metadata for file: {path}") - - # Prune the stuff that we don't need anyway - file_project_schema_arrow = schema_to_pyarrow(file_project_schema) - - read_options = { - "pre_buffer": True, - "use_buffered_stream": True, - "buffer_size": 8388608, - } + pyarrow_filter = None + dnf_filter = None + if bound_row_filter is not AlwaysTrue(): + translated_row_filter = translate_column_names(bound_row_filter, file_schema, case_sensitive=case_sensitive) + bound_file_filter = bind(file_schema, translated_row_filter, case_sensitive=case_sensitive) + dnf_filter = expression_to_plain_format(rewrite_to_dnf(bound_file_filter), cast_int_to_datetime=True) + pyarrow_filter = expression_to_pyarrow(bound_file_filter) + + file_project_schema = prune_columns(file_schema, projected_field_ids, select_full_types=False) + + if file_schema is None: + raise ValueError(f"Missing Iceberg schema in Metadata for file: {path}") + + arrow_table = pq.read_table( + source=fout, + schema=parquet_schema, + pre_buffer=True, + buffer_size=8 * ONE_MEGABYTE, + filters=dnf_filter, + columns=[col.name for col in file_project_schema.columns], + ) - arrow_table = ds.dataset( - source=[path], schema=file_project_schema_arrow, format=ds.ParquetFileFormat(**read_options), filesystem=fs - ).to_table(filter=pyarrow_filter) + if pyarrow_filter is not None: + arrow_table = arrow_table.filter(pyarrow_filter) - return to_requested_schema(projected_schema, file_project_schema, arrow_table) + return to_requested_schema(projected_schema, file_project_schema, arrow_table) def project_table( diff --git a/pyiceberg/schema.py b/pyiceberg/schema.py index 45e9ffe26a..ad18a98f01 100644 --- a/pyiceberg/schema.py +++ b/pyiceberg/schema.py @@ -210,6 +210,18 @@ def find_column_name(self, column_id: int) -> Optional[str]: """ return self._lazy_id_to_name.get(column_id) + @property + def column_names(self) -> List[str]: + """ + Returns a list of all the column names, including nested fields + + Excludes short names + + Returns: + List[str]: The column names + """ + return list(self._lazy_id_to_name.values()) + def accessor_for_field(self, field_id: int) -> "Accessor": """Find a schema position accessor given a field ID From 14dde008820ac2e5da94fd7e94e955e03222b09e Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Sun, 5 Feb 2023 05:13:44 +0000 Subject: [PATCH 358/642] Build: Bump pre-commit from 3.0.1 to 3.0.4 in /python (#6748) --- poetry.lock | 8 ++++---- pyproject.toml | 2 +- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/poetry.lock b/poetry.lock index 74fc860281..19c6fb743b 100644 --- a/poetry.lock +++ b/poetry.lock @@ -1516,14 +1516,14 @@ tests = ["pytest (>=5.4.1)", "pytest-cov (>=2.8.1)", "pytest-mypy (>=0.8.0)", "p [[package]] name = "pre-commit" -version = "3.0.1" +version = "3.0.4" description = "A framework for managing and maintaining multi-language pre-commit hooks." category = "dev" optional = false python-versions = ">=3.8" files = [ - {file = "pre_commit-3.0.1-py2.py3-none-any.whl", hash = "sha256:61ecb75e0e99939cc30c79181c0394545657855e9172c42ff98ebecb0e02fcb7"}, - {file = "pre_commit-3.0.1.tar.gz", hash = "sha256:3a3f9229e8c19a626a7f91be25b3c8c135e52de1a678da98eb015c0d0baea7a5"}, + {file = "pre_commit-3.0.4-py2.py3-none-any.whl", hash = "sha256:9e3255edb0c9e7fe9b4f328cb3dc86069f8fdc38026f1bf521018a05eaf4d67b"}, + {file = "pre_commit-3.0.4.tar.gz", hash = "sha256:bc4687478d55578c4ac37272fe96df66f73d9b5cf81be6f28627d4e712e752d5"}, ] [package.dependencies] @@ -2469,4 +2469,4 @@ snappy = ["python-snappy"] [metadata] lock-version = "2.0" python-versions = "^3.8" -content-hash = "936a690f66edaa27e6491b6a2f43585be56e38fbda0faf274cd398a3489db10c" +content-hash = "c41254b0c11e3d2094ae1b7ea9752ccb74a7b8c3bb5bc2165cc61befe693f9fd" diff --git a/pyproject.toml b/pyproject.toml index 8b30778a39..4d0f866ada 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -81,7 +81,7 @@ adlfs = { version = "2023.1.0", optional = true } [tool.poetry.dev-dependencies] pytest = "7.2.1" pytest-checkdocs = "2.9.0" -pre-commit = "3.0.1" +pre-commit = "3.0.4" fastavro = "1.7.1" coverage = { version = "^7.1.0", extras = ["toml"] } requests-mock = "1.10.0" From 176d1b6e64367a2d1bef3e3e88314b1327027c19 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Sun, 5 Feb 2023 06:21:55 +0000 Subject: [PATCH 359/642] Build: Bump moto from 4.1.0 to 4.1.2 in /python (#6747) --- poetry.lock | 9 +++++---- pyproject.toml | 2 +- 2 files changed, 6 insertions(+), 5 deletions(-) diff --git a/poetry.lock b/poetry.lock index 19c6fb743b..133783a794 100644 --- a/poetry.lock +++ b/poetry.lock @@ -1142,14 +1142,14 @@ files = [ [[package]] name = "moto" -version = "4.1.0" +version = "4.1.2" description = "" category = "dev" optional = false python-versions = ">=3.7" files = [ - {file = "moto-4.1.0-py2.py3-none-any.whl", hash = "sha256:12f231196ebf22d19774f2a75738fdedae10a8c82abb8fa0877e1819aa799f4f"}, - {file = "moto-4.1.0.tar.gz", hash = "sha256:e65a692df57abb631b06c8f3f7827c9deceae3416887527e266c775ad722a2fa"}, + {file = "moto-4.1.2-py2.py3-none-any.whl", hash = "sha256:1b361ece638c74a657325378a259276f368aafce2f8be84f8143e69fa93ce8ec"}, + {file = "moto-4.1.2.tar.gz", hash = "sha256:63431733d2a02c7bd652ad71ec1da442a0e0d580cbac5eeb50d440a2ce066eac"}, ] [package.dependencies] @@ -1178,6 +1178,7 @@ dynamodbstreams = ["docker (>=2.5.1)"] ebs = ["sshpubkeys (>=3.1.0)"] ec2 = ["sshpubkeys (>=3.1.0)"] efs = ["sshpubkeys (>=3.1.0)"] +eks = ["sshpubkeys (>=3.1.0)"] glue = ["pyparsing (>=3.0.7)"] iotdata = ["jsondiff (>=1.1.2)"] route53resolver = ["sshpubkeys (>=3.1.0)"] @@ -2469,4 +2470,4 @@ snappy = ["python-snappy"] [metadata] lock-version = "2.0" python-versions = "^3.8" -content-hash = "c41254b0c11e3d2094ae1b7ea9752ccb74a7b8c3bb5bc2165cc61befe693f9fd" +content-hash = "9294e496c5be2025b0ea54dbd1e068730349b47337d459553c0080f41b3a3abd" diff --git a/pyproject.toml b/pyproject.toml index 4d0f866ada..1aace8eaba 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -85,7 +85,7 @@ pre-commit = "3.0.4" fastavro = "1.7.1" coverage = { version = "^7.1.0", extras = ["toml"] } requests-mock = "1.10.0" -moto = "^4.1.0" +moto = "^4.1.2" typing-extensions = '4.4.0' [tool.poetry.scripts] From 11360a6c27692e5621dacd49df9b6bb4aa167213 Mon Sep 17 00:00:00 2001 From: Fokko Driesprong Date: Tue, 7 Feb 2023 17:24:07 +0100 Subject: [PATCH 360/642] Python: Bump pre-commit plugins to the latest version (#6738) --- .pre-commit-config.yaml | 6 +++--- pyiceberg/avro/reader.py | 2 +- pyiceberg/utils/bin_packing.py | 1 - tests/catalog/test_base.py | 1 - tests/cli/test_console.py | 1 - tests/expressions/test_visitors.py | 18 ++++++++++++++++-- 6 files changed, 20 insertions(+), 9 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 932e84cd90..c4a6fdbaad 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -29,7 +29,7 @@ repos: - id: check-yaml - id: check-ast - repo: https://github.com/ambv/black - rev: 22.12.0 + rev: 23.1.0 hooks: - id: black - repo: https://github.com/pre-commit/mirrors-isort @@ -43,7 +43,7 @@ repos: - id: mypy args: [ --install-types, --non-interactive, --config=python/pyproject.toml] - repo: https://github.com/hadialqattan/pycln - rev: v2.1.2 + rev: v2.1.3 hooks: - id: pycln args: [--config=python/pyproject.toml] @@ -53,7 +53,7 @@ repos: - id: pyupgrade args: [ --py38-plus, --keep-runtime-typing ] - repo: https://github.com/pycqa/pylint - rev: v2.15.10 + rev: v2.16.0 hooks: - id: pylint args: [ --rcfile=python/pylintrc ] diff --git a/pyiceberg/avro/reader.py b/pyiceberg/avro/reader.py index 8cc0c4813e..9156a08116 100644 --- a/pyiceberg/avro/reader.py +++ b/pyiceberg/avro/reader.py @@ -276,7 +276,7 @@ def read(self, decoder: BinaryDecoder) -> StructProtocol: if not isinstance(struct, StructProtocol): raise ValueError(f"Incompatible with StructProtocol: {self.create_struct}") - for (pos, field) in self.field_readers: + for pos, field in self.field_readers: if pos is not None: struct[pos] = field.read(decoder) # later: pass reuse in here else: diff --git a/pyiceberg/utils/bin_packing.py b/pyiceberg/utils/bin_packing.py index 23049c03e2..10e21693ae 100644 --- a/pyiceberg/utils/bin_packing.py +++ b/pyiceberg/utils/bin_packing.py @@ -46,7 +46,6 @@ def add(self, item: T, weight: int) -> None: class PackingIterator(Generic[T]): - bins: List[Bin[T]] def __init__( diff --git a/tests/catalog/test_base.py b/tests/catalog/test_base.py index 7281ff839c..d2277a6cb8 100644 --- a/tests/catalog/test_base.py +++ b/tests/catalog/test_base.py @@ -68,7 +68,6 @@ def create_table( sort_order: SortOrder = UNSORTED_SORT_ORDER, properties: Properties = EMPTY_DICT, ) -> Table: - identifier = Catalog.identifier_to_tuple(identifier) namespace = Catalog.namespace_from(identifier) diff --git a/tests/cli/test_console.py b/tests/cli/test_console.py index 4605cf3d49..0686daa52a 100644 --- a/tests/cli/test_console.py +++ b/tests/cli/test_console.py @@ -136,7 +136,6 @@ def update_namespace_properties( def test_missing_uri(empty_home_dir_path: str) -> None: - # mock to prevent parsing ~/.pyiceberg.yaml or {PYICEBERG_HOME}/.pyiceberg.yaml with mock.patch.dict(os.environ, {"HOME": empty_home_dir_path, "PYICEBERG_HOME": empty_home_dir_path}): with mock.patch("pyiceberg.catalog._ENV_CONFIG", Config()): diff --git a/tests/expressions/test_visitors.py b/tests/expressions/test_visitors.py index 40e97004ad..1b28080e32 100644 --- a/tests/expressions/test_visitors.py +++ b/tests/expressions/test_visitors.py @@ -1372,14 +1372,28 @@ def test_rewrite_not_in() -> None: def test_rewrite_and() -> None: - assert rewrite_not(Not(And(EqualTo(Reference("x"), 34.56), EqualTo(Reference("y"), 34.56),))) == Or( + assert rewrite_not( + Not( + And( + EqualTo(Reference("x"), 34.56), + EqualTo(Reference("y"), 34.56), + ) + ) + ) == Or( NotEqualTo(term=Reference(name="x"), literal=34.56), NotEqualTo(term=Reference(name="y"), literal=34.56), ) def test_rewrite_or() -> None: - assert rewrite_not(Not(Or(EqualTo(Reference("x"), 34.56), EqualTo(Reference("y"), 34.56),))) == And( + assert rewrite_not( + Not( + Or( + EqualTo(Reference("x"), 34.56), + EqualTo(Reference("y"), 34.56), + ) + ) + ) == And( NotEqualTo(term=Reference(name="x"), literal=34.56), NotEqualTo(term=Reference(name="y"), literal=34.56), ) From 589bef30065167592a7a63feabddb2faf2b25531 Mon Sep 17 00:00:00 2001 From: Fokko Driesprong Date: Tue, 7 Feb 2023 17:26:41 +0100 Subject: [PATCH 361/642] Python: Avoid local imports in test (#6737) --- tests/catalog/integration_test_glue.py | 91 +++++-------- tests/catalog/test_base.py | 6 + tests/catalog/test_glue.py | 176 ++++++++++--------------- tests/cli/test_console.py | 63 ++++++++- tests/conftest.py | 37 +++++- 5 files changed, 200 insertions(+), 173 deletions(-) diff --git a/tests/catalog/integration_test_glue.py b/tests/catalog/integration_test_glue.py index 76b05b2b4d..be4cd2e826 100644 --- a/tests/catalog/integration_test_glue.py +++ b/tests/catalog/integration_test_glue.py @@ -16,7 +16,7 @@ # under the License. import os -from typing import Generator, Optional +from typing import Generator, List, Optional import boto3 import pytest @@ -32,15 +32,7 @@ TableAlreadyExistsError, ) from pyiceberg.schema import Schema -from tests.catalog.test_glue import ( - get_random_database_name, - get_random_databases, - get_random_table_name, - get_random_tables, -) -# The number of random characters in generated table/database name -RANDOM_LENGTH = 20 # The number of tables/databases used in list_table/namespace test LIST_TEST_NUMBER = 2 @@ -93,9 +85,9 @@ def fixture_test_catalog() -> Generator[Catalog, None, None]: clean_up(test_catalog) -def test_create_table(test_catalog: Catalog, s3: boto3.client, table_schema_nested: Schema) -> None: - table_name = get_random_table_name() - database_name = get_random_database_name() +def test_create_table( + test_catalog: Catalog, s3: boto3.client, table_schema_nested: Schema, table_name: str, database_name: str +) -> None: identifier = (database_name, table_name) test_catalog.create_namespace(database_name) test_catalog.create_table(identifier, table_schema_nested, get_s3_path(get_bucket_name(), database_name, table_name)) @@ -105,9 +97,7 @@ def test_create_table(test_catalog: Catalog, s3: boto3.client, table_schema_nest s3.head_object(Bucket=get_bucket_name(), Key=metadata_location) -def test_create_table_with_invalid_location(table_schema_nested: Schema) -> None: - table_name = get_random_table_name() - database_name = get_random_database_name() +def test_create_table_with_invalid_location(table_schema_nested: Schema, table_name: str, database_name: str) -> None: identifier = (database_name, table_name) test_catalog_no_warehouse = GlueCatalog("glue") test_catalog_no_warehouse.create_namespace(database_name) @@ -116,9 +106,9 @@ def test_create_table_with_invalid_location(table_schema_nested: Schema) -> None test_catalog_no_warehouse.drop_namespace(database_name) -def test_create_table_with_default_location(test_catalog: Catalog, s3: boto3.client, table_schema_nested: Schema) -> None: - table_name = get_random_table_name() - database_name = get_random_database_name() +def test_create_table_with_default_location( + test_catalog: Catalog, s3: boto3.client, table_schema_nested: Schema, table_name: str, database_name: str +) -> None: identifier = (database_name, table_name) test_catalog.create_namespace(database_name) test_catalog.create_table(identifier, table_schema_nested) @@ -128,25 +118,20 @@ def test_create_table_with_default_location(test_catalog: Catalog, s3: boto3.cli s3.head_object(Bucket=get_bucket_name(), Key=metadata_location) -def test_create_table_with_invalid_database(test_catalog: Catalog, table_schema_nested: Schema) -> None: - table_name = get_random_table_name() +def test_create_table_with_invalid_database(test_catalog: Catalog, table_schema_nested: Schema, table_name: str) -> None: identifier = ("invalid", table_name) with pytest.raises(NoSuchNamespaceError): test_catalog.create_table(identifier, table_schema_nested) -def test_create_duplicated_table(test_catalog: Catalog, table_schema_nested: Schema) -> None: - table_name = get_random_table_name() - database_name = get_random_database_name() +def test_create_duplicated_table(test_catalog: Catalog, table_schema_nested: Schema, table_name: str, database_name: str) -> None: test_catalog.create_namespace(database_name) test_catalog.create_table((database_name, table_name), table_schema_nested) with pytest.raises(TableAlreadyExistsError): test_catalog.create_table((database_name, table_name), table_schema_nested) -def test_load_table(test_catalog: Catalog, table_schema_nested: Schema) -> None: - table_name = get_random_table_name() - database_name = get_random_database_name() +def test_load_table(test_catalog: Catalog, table_schema_nested: Schema, table_name: str, database_name: str) -> None: identifier = (database_name, table_name) test_catalog.create_namespace(database_name) table = test_catalog.create_table(identifier, table_schema_nested) @@ -156,22 +141,20 @@ def test_load_table(test_catalog: Catalog, table_schema_nested: Schema) -> None: assert table.metadata == loaded_table.metadata -def test_list_tables(test_catalog: Catalog, table_schema_nested: Schema) -> None: - test_tables = get_random_tables(LIST_TEST_NUMBER) - database_name = get_random_database_name() +def test_list_tables(test_catalog: Catalog, table_schema_nested: Schema, database_name: str, table_list: List[str]) -> None: test_catalog.create_namespace(database_name) - for table_name in test_tables: + for table_name in table_list: test_catalog.create_table((database_name, table_name), table_schema_nested) identifier_list = test_catalog.list_tables(database_name) assert len(identifier_list) == LIST_TEST_NUMBER - for table_name in test_tables: + for table_name in table_list: assert (database_name, table_name) in identifier_list -def test_rename_table(test_catalog: Catalog, s3: boto3.client, table_schema_nested: Schema) -> None: - table_name = get_random_table_name() - database_name = get_random_database_name() - new_database_name = get_random_database_name() +def test_rename_table( + test_catalog: Catalog, s3: boto3.client, table_schema_nested: Schema, table_name: str, database_name: str +) -> None: + new_database_name = f"{database_name}_new" test_catalog.create_namespace(database_name) test_catalog.create_namespace(new_database_name) new_table_name = f"rename-{table_name}" @@ -189,9 +172,7 @@ def test_rename_table(test_catalog: Catalog, s3: boto3.client, table_schema_nest test_catalog.load_table(identifier) -def test_drop_table(test_catalog: Catalog, table_schema_nested: Schema) -> None: - table_name = get_random_table_name() - database_name = get_random_database_name() +def test_drop_table(test_catalog: Catalog, table_schema_nested: Schema, table_name: str, database_name: str) -> None: identifier = (database_name, table_name) test_catalog.create_namespace(database_name) table = test_catalog.create_table(identifier, table_schema_nested) @@ -201,9 +182,9 @@ def test_drop_table(test_catalog: Catalog, table_schema_nested: Schema) -> None: test_catalog.load_table(identifier) -def test_purge_table(test_catalog: Catalog, s3: boto3.client, table_schema_nested: Schema) -> None: - table_name = get_random_table_name() - database_name = get_random_database_name() +def test_purge_table( + test_catalog: Catalog, s3: boto3.client, table_schema_nested: Schema, table_name: str, database_name: str +) -> None: identifier = (database_name, table_name) test_catalog.create_namespace(database_name) test_catalog.create_table(identifier, table_schema_nested) @@ -218,21 +199,18 @@ def test_purge_table(test_catalog: Catalog, s3: boto3.client, table_schema_neste s3.head_object(Bucket=get_bucket_name(), Key=metadata_location) -def test_create_namespace(test_catalog: Catalog) -> None: - database_name = get_random_database_name() +def test_create_namespace(test_catalog: Catalog, database_name: str) -> None: test_catalog.create_namespace(database_name) assert (database_name,) in test_catalog.list_namespaces() -def test_create_duplicate_namespace(test_catalog: Catalog) -> None: - database_name = get_random_database_name() +def test_create_duplicate_namespace(test_catalog: Catalog, database_name: str) -> None: test_catalog.create_namespace(database_name) with pytest.raises(NamespaceAlreadyExistsError): test_catalog.create_namespace(database_name) -def test_create_namespace_with_comment_and_location(test_catalog: Catalog) -> None: - database_name = get_random_database_name() +def test_create_namespace_with_comment_and_location(test_catalog: Catalog, database_name: str) -> None: test_location = get_s3_path(get_bucket_name(), database_name) test_properties = { "comment": "this is a test description", @@ -246,8 +224,7 @@ def test_create_namespace_with_comment_and_location(test_catalog: Catalog) -> No assert properties["location"] == test_location -def test_list_namespaces(test_catalog: Catalog) -> None: - database_list = get_random_databases(LIST_TEST_NUMBER) +def test_list_namespaces(test_catalog: Catalog, database_list: List[str]) -> None: for database_name in database_list: test_catalog.create_namespace(database_name) db_list = test_catalog.list_namespaces() @@ -256,9 +233,7 @@ def test_list_namespaces(test_catalog: Catalog) -> None: assert len(test_catalog.list_namespaces(list(database_list)[0])) == 0 -def test_drop_namespace(test_catalog: Catalog, table_schema_nested: Schema) -> None: - table_name = get_random_table_name() - database_name = get_random_database_name() +def test_drop_namespace(test_catalog: Catalog, table_schema_nested: Schema, database_name: str, table_name: str) -> None: test_catalog.create_namespace(database_name) assert (database_name,) in test_catalog.list_namespaces() test_catalog.create_table((database_name, table_name), table_schema_nested) @@ -269,9 +244,8 @@ def test_drop_namespace(test_catalog: Catalog, table_schema_nested: Schema) -> N assert (database_name,) not in test_catalog.list_namespaces() -def test_load_namespace_properties(test_catalog: Catalog) -> None: +def test_load_namespace_properties(test_catalog: Catalog, database_name: str) -> None: warehouse_location = get_s3_path(get_bucket_name()) - database_name = get_random_database_name() test_properties = { "comment": "this is a test description", "location": f"{warehouse_location}/{database_name}.db", @@ -287,24 +261,21 @@ def test_load_namespace_properties(test_catalog: Catalog) -> None: assert v == test_properties[k] -def test_load_empty_namespace_properties(test_catalog: Catalog) -> None: - database_name = get_random_database_name() +def test_load_empty_namespace_properties(test_catalog: Catalog, database_name: str) -> None: test_catalog.create_namespace(database_name) listed_properties = test_catalog.load_namespace_properties(database_name) assert listed_properties == {} -def test_load_default_namespace_properties(test_catalog: Catalog, glue: boto3.client) -> None: - database_name = get_random_database_name() +def test_load_default_namespace_properties(test_catalog: Catalog, glue: boto3.client, database_name: str) -> None: # simulate creating database with default settings through AWS Glue Web Console glue.create_database(DatabaseInput={"Name": database_name}) listed_properties = test_catalog.load_namespace_properties(database_name) assert listed_properties == {} -def test_update_namespace_properties(test_catalog: Catalog) -> None: +def test_update_namespace_properties(test_catalog: Catalog, database_name: str) -> None: warehouse_location = get_s3_path(get_bucket_name()) - database_name = get_random_database_name() test_properties = { "comment": "this is a test description", "location": f"{warehouse_location}/{database_name}.db", diff --git a/tests/catalog/test_base.py b/tests/catalog/test_base.py index d2277a6cb8..742549a654 100644 --- a/tests/catalog/test_base.py +++ b/tests/catalog/test_base.py @@ -14,6 +14,7 @@ # KIND, either express or implied. See the License for the # specific language governing permissions and limitations # under the License. +# pylint:disable=redefined-outer-name from typing import ( Dict, @@ -205,6 +206,11 @@ def update_namespace_properties( ) +@pytest.fixture +def catalog() -> InMemoryCatalog: + return InMemoryCatalog("test.in.memory.catalog", **{"test.key": "test.value"}) + + TEST_TABLE_IDENTIFIER = ("com", "organization", "department", "my_table") TEST_TABLE_NAMESPACE = ("com", "organization", "department") TEST_TABLE_NAME = "my_table" diff --git a/tests/catalog/test_glue.py b/tests/catalog/test_glue.py index af12435415..bdf64b9ef1 100644 --- a/tests/catalog/test_glue.py +++ b/tests/catalog/test_glue.py @@ -14,10 +14,8 @@ # KIND, either express or implied. See the License for the # specific language governing permissions and limitations # under the License. -import random import re -import string -from typing import Set +from typing import List import pytest from moto import mock_glue @@ -35,7 +33,6 @@ from pyiceberg.schema import Schema BUCKET_NAME = "test_bucket" -RANDOM_LENGTH = 20 LIST_TEST_NUMBER = 100 table_metadata_location_regex = re.compile( r"""s3://test_bucket/my_iceberg_database-[a-z]{20}.db/ @@ -45,26 +42,6 @@ ) -def get_random_table_name() -> str: - prefix = "my_iceberg_table-" - random_tag = "".join(random.choice(string.ascii_letters) for _ in range(RANDOM_LENGTH)) - return (prefix + random_tag).lower() - - -def get_random_tables(n: int) -> Set[str]: - return {get_random_table_name() for _ in range(n)} - - -def get_random_database_name() -> str: - prefix = "my_iceberg_database-" - random_tag = "".join(random.choice(string.ascii_letters) for _ in range(RANDOM_LENGTH)) - return (prefix + random_tag).lower() - - -def get_random_databases(n: int) -> Set[str]: - return {get_random_database_name() for _ in range(n)} - - @pytest.fixture(name="_bucket_initialize") def fixture_s3_bucket(_s3) -> None: # type: ignore _s3.create_bucket(Bucket=BUCKET_NAME) @@ -72,10 +49,8 @@ def fixture_s3_bucket(_s3) -> None: # type: ignore @mock_glue def test_create_table_with_database_location( - _bucket_initialize: None, _patch_aiobotocore: None, table_schema_nested: Schema + _bucket_initialize: None, _patch_aiobotocore: None, table_schema_nested: Schema, database_name: str, table_name: str ) -> None: - database_name = get_random_database_name() - table_name = get_random_table_name() identifier = (database_name, table_name) test_catalog = GlueCatalog("glue") test_catalog.create_namespace(namespace=database_name, properties={"location": f"s3://{BUCKET_NAME}/{database_name}.db"}) @@ -86,10 +61,8 @@ def test_create_table_with_database_location( @mock_glue def test_create_table_with_default_warehouse( - _bucket_initialize: None, _patch_aiobotocore: None, table_schema_nested: Schema + _bucket_initialize: None, _patch_aiobotocore: None, table_schema_nested: Schema, database_name: str, table_name: str ) -> None: - database_name = get_random_database_name() - table_name = get_random_table_name() identifier = (database_name, table_name) test_catalog = GlueCatalog("glue", warehouse=f"s3://{BUCKET_NAME}") test_catalog.create_namespace(namespace=database_name) @@ -100,10 +73,8 @@ def test_create_table_with_default_warehouse( @mock_glue def test_create_table_with_given_location( - _bucket_initialize: None, _patch_aiobotocore: None, table_schema_nested: Schema + _bucket_initialize: None, _patch_aiobotocore: None, table_schema_nested: Schema, database_name: str, table_name: str ) -> None: - database_name = get_random_database_name() - table_name = get_random_table_name() identifier = (database_name, table_name) test_catalog = GlueCatalog("glue") test_catalog.create_namespace(namespace=database_name) @@ -115,9 +86,9 @@ def test_create_table_with_given_location( @mock_glue -def test_create_table_with_no_location(_bucket_initialize: None, _patch_aiobotocore: None, table_schema_nested: Schema) -> None: - database_name = get_random_database_name() - table_name = get_random_table_name() +def test_create_table_with_no_location( + _bucket_initialize: None, _patch_aiobotocore: None, table_schema_nested: Schema, database_name: str, table_name: str +) -> None: identifier = (database_name, table_name) test_catalog = GlueCatalog("glue") test_catalog.create_namespace(namespace=database_name) @@ -126,29 +97,33 @@ def test_create_table_with_no_location(_bucket_initialize: None, _patch_aiobotoc @mock_glue -def test_create_table_with_strips(_bucket_initialize: None, _patch_aiobotocore: None, table_schema_nested: Schema) -> None: - database_name = get_random_database_name() - table_name = get_random_table_name() +def test_create_table_with_strips( + _bucket_initialize: None, _patch_aiobotocore: None, table_schema_nested: Schema, database_name: str, table_name: str +) -> None: identifier = (database_name, table_name) test_catalog = GlueCatalog("glue") test_catalog.create_namespace(namespace=database_name, properties={"location": f"s3://{BUCKET_NAME}/{database_name}.db/"}) table = test_catalog.create_table(identifier, table_schema_nested) assert table.identifier == identifier assert table_metadata_location_regex.match(table.metadata_location) - database_name = get_random_database_name() - table_name = get_random_table_name() + + +@mock_glue +def test_create_table_with_strips_bucket_root( + _bucket_initialize: None, _patch_aiobotocore: None, table_schema_nested: Schema, database_name: str, table_name: str +) -> None: identifier = (database_name, table_name) - test_catalog_strip = GlueCatalog("glue", warehouse=f"s3://{BUCKET_NAME}/") - test_catalog_strip.create_namespace(namespace=database_name) - table_strip = test_catalog_strip.create_table(identifier, table_schema_nested) + test_catalog = GlueCatalog("glue", warehouse=f"s3://{BUCKET_NAME}/") + test_catalog.create_namespace(namespace=database_name) + table_strip = test_catalog.create_table(identifier, table_schema_nested) assert table_strip.identifier == identifier assert table_metadata_location_regex.match(table_strip.metadata_location) @mock_glue -def test_create_table_with_no_database(_bucket_initialize: None, _patch_aiobotocore: None, table_schema_nested: Schema) -> None: - database_name = get_random_database_name() - table_name = get_random_table_name() +def test_create_table_with_no_database( + _bucket_initialize: None, _patch_aiobotocore: None, table_schema_nested: Schema, database_name: str, table_name: str +) -> None: identifier = (database_name, table_name) test_catalog = GlueCatalog("glue") with pytest.raises(NoSuchNamespaceError): @@ -156,9 +131,9 @@ def test_create_table_with_no_database(_bucket_initialize: None, _patch_aiobotoc @mock_glue -def test_create_duplicated_table(_bucket_initialize: None, _patch_aiobotocore: None, table_schema_nested: Schema) -> None: - database_name = get_random_database_name() - table_name = get_random_table_name() +def test_create_duplicated_table( + _bucket_initialize: None, _patch_aiobotocore: None, table_schema_nested: Schema, database_name: str, table_name: str +) -> None: identifier = (database_name, table_name) test_catalog = GlueCatalog("glue", warehouse=f"s3://{BUCKET_NAME}") test_catalog.create_namespace(namespace=database_name) @@ -168,9 +143,9 @@ def test_create_duplicated_table(_bucket_initialize: None, _patch_aiobotocore: N @mock_glue -def test_load_table(_bucket_initialize: None, _patch_aiobotocore: None, table_schema_nested: Schema) -> None: - database_name = get_random_database_name() - table_name = get_random_table_name() +def test_load_table( + _bucket_initialize: None, _patch_aiobotocore: None, table_schema_nested: Schema, database_name: str, table_name: str +) -> None: identifier = (database_name, table_name) test_catalog = GlueCatalog("glue", warehouse=f"s3://{BUCKET_NAME}") test_catalog.create_namespace(namespace=database_name) @@ -181,9 +156,7 @@ def test_load_table(_bucket_initialize: None, _patch_aiobotocore: None, table_sc @mock_glue -def test_load_non_exist_table(_bucket_initialize: None, _patch_aiobotocore: None) -> None: - database_name = get_random_database_name() - table_name = get_random_table_name() +def test_load_non_exist_table(_bucket_initialize: None, _patch_aiobotocore: None, database_name: str, table_name: str) -> None: identifier = (database_name, table_name) test_catalog = GlueCatalog("glue", warehouse=f"s3://{BUCKET_NAME}") test_catalog.create_namespace(namespace=database_name) @@ -192,9 +165,9 @@ def test_load_non_exist_table(_bucket_initialize: None, _patch_aiobotocore: None @mock_glue -def test_drop_table(_bucket_initialize: None, _patch_aiobotocore: None, table_schema_nested: Schema) -> None: - database_name = get_random_database_name() - table_name = get_random_table_name() +def test_drop_table( + _bucket_initialize: None, _patch_aiobotocore: None, table_schema_nested: Schema, database_name: str, table_name: str +) -> None: identifier = (database_name, table_name) test_catalog = GlueCatalog("glue", warehouse=f"s3://{BUCKET_NAME}") test_catalog.create_namespace(namespace=database_name) @@ -208,9 +181,7 @@ def test_drop_table(_bucket_initialize: None, _patch_aiobotocore: None, table_sc @mock_glue -def test_drop_non_exist_table(_bucket_initialize: None, _patch_aiobotocore: None) -> None: - database_name = get_random_database_name() - table_name = get_random_table_name() +def test_drop_non_exist_table(_bucket_initialize: None, _patch_aiobotocore: None, database_name: str, table_name: str) -> None: identifier = (database_name, table_name) test_catalog = GlueCatalog("glue", warehouse=f"s3://{BUCKET_NAME}") with pytest.raises(NoSuchTableError): @@ -218,10 +189,10 @@ def test_drop_non_exist_table(_bucket_initialize: None, _patch_aiobotocore: None @mock_glue -def test_rename_table(_bucket_initialize: None, _patch_aiobotocore: None, table_schema_nested: Schema) -> None: - database_name = get_random_database_name() - table_name = get_random_table_name() - new_table_name = get_random_table_name() +def test_rename_table( + _bucket_initialize: None, _patch_aiobotocore: None, table_schema_nested: Schema, database_name: str, table_name: str +) -> None: + new_table_name = f"{table_name}_new" identifier = (database_name, table_name) new_identifier = (database_name, new_table_name) test_catalog = GlueCatalog("glue", warehouse=f"s3://{BUCKET_NAME}") @@ -240,11 +211,9 @@ def test_rename_table(_bucket_initialize: None, _patch_aiobotocore: None, table_ @mock_glue -def test_rename_table_no_params(_glue, _bucket_initialize: None, _patch_aiobotocore: None) -> None: # type: ignore - database_name = get_random_database_name() - new_database_name = get_random_database_name() - table_name = get_random_table_name() - new_table_name = get_random_table_name() +def test_rename_table_no_params(_glue, _bucket_initialize: None, _patch_aiobotocore: None, database_name: str, table_name: str) -> None: # type: ignore + new_database_name = f"{database_name}_new" + new_table_name = f"{table_name}_new" identifier = (database_name, table_name) new_identifier = (new_database_name, new_table_name) test_catalog = GlueCatalog("glue", warehouse=f"s3://{BUCKET_NAME}") @@ -259,11 +228,9 @@ def test_rename_table_no_params(_glue, _bucket_initialize: None, _patch_aiobotoc @mock_glue -def test_rename_non_iceberg_table(_glue, _bucket_initialize: None, _patch_aiobotocore: None) -> None: # type: ignore - database_name = get_random_database_name() - new_database_name = get_random_database_name() - table_name = get_random_table_name() - new_table_name = get_random_table_name() +def test_rename_non_iceberg_table(_glue, _bucket_initialize: None, _patch_aiobotocore: None, database_name: str, table_name: str) -> None: # type: ignore + new_database_name = f"{database_name}_new" + new_table_name = f"{table_name}_new" identifier = (database_name, table_name) new_identifier = (new_database_name, new_table_name) test_catalog = GlueCatalog("glue", warehouse=f"s3://{BUCKET_NAME}") @@ -282,9 +249,14 @@ def test_rename_non_iceberg_table(_glue, _bucket_initialize: None, _patch_aiobot @mock_glue -def test_list_tables(_bucket_initialize: None, _patch_aiobotocore: None, table_schema_nested: Schema) -> None: - database_name = get_random_database_name() - table_list = get_random_tables(LIST_TEST_NUMBER) +def test_list_tables( + _bucket_initialize: None, + _patch_aiobotocore: None, + table_schema_nested: Schema, + database_name: str, + table_name: str, + table_list: List[str], +) -> None: test_catalog = GlueCatalog("glue", warehouse=f"s3://{BUCKET_NAME}") test_catalog.create_namespace(namespace=database_name) for table_name in table_list: @@ -295,8 +267,7 @@ def test_list_tables(_bucket_initialize: None, _patch_aiobotocore: None, table_s @mock_glue -def test_list_namespaces(_bucket_initialize: None, _patch_aiobotocore: None) -> None: - database_list = get_random_databases(LIST_TEST_NUMBER) +def test_list_namespaces(_bucket_initialize: None, _patch_aiobotocore: None, database_list: List[str]) -> None: test_catalog = GlueCatalog("glue") for database_name in database_list: test_catalog.create_namespace(namespace=database_name) @@ -306,8 +277,7 @@ def test_list_namespaces(_bucket_initialize: None, _patch_aiobotocore: None) -> @mock_glue -def test_create_namespace_no_properties(_bucket_initialize: None, _patch_aiobotocore: None) -> None: - database_name = get_random_database_name() +def test_create_namespace_no_properties(_bucket_initialize: None, _patch_aiobotocore: None, database_name: str) -> None: test_catalog = GlueCatalog("glue") test_catalog.create_namespace(namespace=database_name) loaded_database_list = test_catalog.list_namespaces() @@ -318,8 +288,9 @@ def test_create_namespace_no_properties(_bucket_initialize: None, _patch_aioboto @mock_glue -def test_create_namespace_with_comment_and_location(_bucket_initialize: None, _patch_aiobotocore: None) -> None: - database_name = get_random_database_name() +def test_create_namespace_with_comment_and_location( + _bucket_initialize: None, _patch_aiobotocore: None, database_name: str +) -> None: test_location = f"s3://{BUCKET_NAME}/{database_name}.db" test_properties = { "comment": "this is a test description", @@ -336,8 +307,7 @@ def test_create_namespace_with_comment_and_location(_bucket_initialize: None, _p @mock_glue -def test_create_duplicated_namespace(_bucket_initialize: None, _patch_aiobotocore: None) -> None: - database_name = get_random_database_name() +def test_create_duplicated_namespace(_bucket_initialize: None, _patch_aiobotocore: None, database_name: str) -> None: test_catalog = GlueCatalog("glue") test_catalog.create_namespace(namespace=database_name) loaded_database_list = test_catalog.list_namespaces() @@ -348,8 +318,7 @@ def test_create_duplicated_namespace(_bucket_initialize: None, _patch_aiobotocor @mock_glue -def test_drop_namespace(_bucket_initialize: None, _patch_aiobotocore: None) -> None: - database_name = get_random_database_name() +def test_drop_namespace(_bucket_initialize: None, _patch_aiobotocore: None, database_name: str) -> None: test_catalog = GlueCatalog("glue") test_catalog.create_namespace(namespace=database_name) loaded_database_list = test_catalog.list_namespaces() @@ -361,9 +330,9 @@ def test_drop_namespace(_bucket_initialize: None, _patch_aiobotocore: None) -> N @mock_glue -def test_drop_non_empty_namespace(_bucket_initialize: None, _patch_aiobotocore: None, table_schema_nested: Schema) -> None: - database_name = get_random_database_name() - table_name = get_random_table_name() +def test_drop_non_empty_namespace( + _bucket_initialize: None, _patch_aiobotocore: None, table_schema_nested: Schema, database_name: str, table_name: str +) -> None: identifier = (database_name, table_name) test_catalog = GlueCatalog("glue", warehouse=f"s3://{BUCKET_NAME}") test_catalog.create_namespace(namespace=database_name) @@ -374,16 +343,14 @@ def test_drop_non_empty_namespace(_bucket_initialize: None, _patch_aiobotocore: @mock_glue -def test_drop_non_exist_namespace(_bucket_initialize: None, _patch_aiobotocore: None) -> None: - database_name = get_random_database_name() +def test_drop_non_exist_namespace(_bucket_initialize: None, _patch_aiobotocore: None, database_name: str) -> None: test_catalog = GlueCatalog("glue") with pytest.raises(NoSuchNamespaceError): test_catalog.drop_namespace(database_name) @mock_glue -def test_load_namespace_properties(_bucket_initialize: None, _patch_aiobotocore: None) -> None: - database_name = get_random_database_name() +def test_load_namespace_properties(_bucket_initialize: None, _patch_aiobotocore: None, database_name: str) -> None: test_location = f"s3://{BUCKET_NAME}/{database_name}.db" test_properties = { "comment": "this is a test description", @@ -401,16 +368,14 @@ def test_load_namespace_properties(_bucket_initialize: None, _patch_aiobotocore: @mock_glue -def test_load_non_exist_namespace_properties(_bucket_initialize: None, _patch_aiobotocore: None) -> None: - database_name = get_random_database_name() +def test_load_non_exist_namespace_properties(_bucket_initialize: None, _patch_aiobotocore: None, database_name: str) -> None: test_catalog = GlueCatalog("glue") with pytest.raises(NoSuchNamespaceError): test_catalog.load_namespace_properties(database_name) @mock_glue -def test_update_namespace_properties(_bucket_initialize: None, _patch_aiobotocore: None) -> None: - database_name = get_random_database_name() +def test_update_namespace_properties(_bucket_initialize: None, _patch_aiobotocore: None, database_name: str) -> None: test_properties = { "comment": "this is a test description", "location": f"s3://{BUCKET_NAME}/{database_name}.db", @@ -435,8 +400,7 @@ def test_update_namespace_properties(_bucket_initialize: None, _patch_aiobotocor @mock_glue -def test_load_empty_namespace_properties(_bucket_initialize: None, _patch_aiobotocore: None) -> None: - database_name = get_random_database_name() +def test_load_empty_namespace_properties(_bucket_initialize: None, _patch_aiobotocore: None, database_name: str) -> None: test_catalog = GlueCatalog("glue") test_catalog.create_namespace(database_name) listed_properties = test_catalog.load_namespace_properties(database_name) @@ -444,8 +408,7 @@ def test_load_empty_namespace_properties(_bucket_initialize: None, _patch_aiobot @mock_glue -def test_load_default_namespace_properties(_glue, _bucket_initialize, _patch_aiobotocore) -> None: # type: ignore - database_name = get_random_database_name() +def test_load_default_namespace_properties(_glue, _bucket_initialize, _patch_aiobotocore, database_name: str) -> None: # type: ignore # simulate creating database with default settings through AWS Glue Web Console _glue.create_database(DatabaseInput={"Name": database_name}) test_catalog = GlueCatalog("glue") @@ -454,8 +417,9 @@ def test_load_default_namespace_properties(_glue, _bucket_initialize, _patch_aio @mock_glue -def test_update_namespace_properties_overlap_update_removal(_bucket_initialize: None, _patch_aiobotocore: None) -> None: - database_name = get_random_database_name() +def test_update_namespace_properties_overlap_update_removal( + _bucket_initialize: None, _patch_aiobotocore: None, database_name: str +) -> None: test_properties = { "comment": "this is a test description", "location": f"s3://{BUCKET_NAME}/{database_name}.db", diff --git a/tests/cli/test_console.py b/tests/cli/test_console.py index 0686daa52a..555071dd0d 100644 --- a/tests/cli/test_console.py +++ b/tests/cli/test_console.py @@ -36,7 +36,68 @@ from pyiceberg.table.sorting import UNSORTED_SORT_ORDER, SortOrder from pyiceberg.typedef import EMPTY_DICT, Identifier, Properties from pyiceberg.utils.config import Config -from tests.conftest import EXAMPLE_TABLE_METADATA_V2 + +EXAMPLE_TABLE_METADATA_V2 = { + "format-version": 2, + "table-uuid": "9c12d441-03fe-4693-9a96-a0705ddf69c1", + "location": "s3://bucket/test/location", + "last-sequence-number": 34, + "last-updated-ms": 1602638573590, + "last-column-id": 3, + "current-schema-id": 1, + "schemas": [ + {"type": "struct", "schema-id": 0, "fields": [{"id": 1, "name": "x", "required": True, "type": "long"}]}, + { + "type": "struct", + "schema-id": 1, + "identifier-field-ids": [1, 2], + "fields": [ + {"id": 1, "name": "x", "required": True, "type": "long"}, + {"id": 2, "name": "y", "required": True, "type": "long", "doc": "comment"}, + {"id": 3, "name": "z", "required": True, "type": "long"}, + ], + }, + ], + "default-spec-id": 0, + "partition-specs": [{"spec-id": 0, "fields": [{"name": "x", "transform": "identity", "source-id": 1, "field-id": 1000}]}], + "last-partition-id": 1000, + "default-sort-order-id": 3, + "sort-orders": [ + { + "order-id": 3, + "fields": [ + {"transform": "identity", "source-id": 2, "direction": "asc", "null-order": "nulls-first"}, + {"transform": "bucket[4]", "source-id": 3, "direction": "desc", "null-order": "nulls-last"}, + ], + } + ], + "properties": {"read.split.target.size": 134217728}, + "current-snapshot-id": 3055729675574597004, + "snapshots": [ + { + "snapshot-id": 3051729675574597004, + "timestamp-ms": 1515100955770, + "sequence-number": 0, + "summary": {"operation": "append"}, + "manifest-list": "s3://a/b/1.avro", + }, + { + "snapshot-id": 3055729675574597004, + "parent-snapshot-id": 3051729675574597004, + "timestamp-ms": 1555100955770, + "sequence-number": 1, + "summary": {"operation": "append"}, + "manifest-list": "s3://a/b/2.avro", + "schema-id": 1, + }, + ], + "snapshot-log": [ + {"snapshot-id": 3051729675574597004, "timestamp-ms": 1515100955770}, + {"snapshot-id": 3055729675574597004, "timestamp-ms": 1555100955770}, + ], + "metadata-log": [{"metadata-file": "s3://bucket/.../v1.json", "timestamp-ms": 1515100}], + "refs": {"test": {"snapshot-id": 3051729675574597004, "type": "tag", "max-ref-age-ms": 10000000}}, +} class MockCatalog(Catalog): diff --git a/tests/conftest.py b/tests/conftest.py index 888888da9f..2ce71f5f12 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -25,12 +25,15 @@ retrieved using `request.getfixturevalue(fixture_name)`. """ import os +import string +from random import choice from tempfile import TemporaryDirectory from typing import ( Any, Callable, Dict, Generator, + List, ) from unittest.mock import MagicMock from urllib.parse import urlparse @@ -65,7 +68,6 @@ StringType, StructType, ) -from tests.catalog.test_base import InMemoryCatalog def pytest_addoption(parser: pytest.Parser) -> None: @@ -310,11 +312,6 @@ def example_table_metadata_v2() -> Dict[str, Any]: return EXAMPLE_TABLE_METADATA_V2 -@pytest.fixture -def catalog() -> InMemoryCatalog: - return InMemoryCatalog("test.in.memory.catalog", **{"test.key": "test.value"}) - - manifest_entry_records = [ { "status": 1, @@ -1271,3 +1268,31 @@ def adlfs_fsspec_fileio(request: pytest.FixtureRequest) -> Generator[FsspecFileI def empty_home_dir_path(tmp_path_factory: pytest.TempPathFactory) -> str: home_path = str(tmp_path_factory.mktemp("home")) return home_path + + +RANDOM_LENGTH = 20 +NUM_TABLES = 2 + + +@pytest.fixture() +def table_name() -> str: + prefix = "my_iceberg_table-" + random_tag = "".join(choice(string.ascii_letters) for _ in range(RANDOM_LENGTH)) + return (prefix + random_tag).lower() + + +@pytest.fixture() +def table_list(table_name: str) -> List[str]: + return [f"{table_name}_{idx}" for idx in range(NUM_TABLES)] + + +@pytest.fixture() +def database_name() -> str: + prefix = "my_iceberg_database-" + random_tag = "".join(choice(string.ascii_letters) for _ in range(RANDOM_LENGTH)) + return (prefix + random_tag).lower() + + +@pytest.fixture() +def database_list(database_name: str) -> List[str]: + return [f"{database_name}_{idx}" for idx in range(NUM_TABLES)] From f7fd93f0e4430ccf3ae63606eef08cbffb758874 Mon Sep 17 00:00:00 2001 From: Fokko Driesprong Date: Tue, 7 Feb 2023 17:27:19 +0100 Subject: [PATCH 362/642] Python: Publish the docs by hand (#6719) Currently everything that goes into master will directly be pushed to Github Pages. This was fine for a while, but now we have PRs that publish new functionality, but then it isn't available in the release for example: https://github.com/apache/iceberg/pull/6644 I think we should now publish by hand, and move to multi version docs once PyIceberg stabilizes a bit. --- mkdocs/docs/how-to-release.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/mkdocs/docs/how-to-release.md b/mkdocs/docs/how-to-release.md index ca6c917f9c..4b5c6345a9 100644 --- a/mkdocs/docs/how-to-release.md +++ b/mkdocs/docs/how-to-release.md @@ -162,3 +162,7 @@ This Python release can be downloaded from: https://pypi.org/project/pyiceberg/< Thanks to everyone for contributing! ``` + +## Release the docs + +A committer triggers the `Python Docs` Github Actions through the UI by selecting the branch that just has been released. This will publish the new docs. From e03aceaca664700e3f8c96783e817c52356a2205 Mon Sep 17 00:00:00 2001 From: Fokko Driesprong Date: Tue, 7 Feb 2023 17:28:56 +0100 Subject: [PATCH 363/642] Python: Allow to pass in a string as filter (#6657) --- mkdocs/docs/api.md | 5 +++++ pyiceberg/table/__init__.py | 32 +++++++++++++++++++++++++------- 2 files changed, 30 insertions(+), 7 deletions(-) diff --git a/mkdocs/docs/api.md b/mkdocs/docs/api.md index dd6423f61a..114362a9ab 100644 --- a/mkdocs/docs/api.md +++ b/mkdocs/docs/api.md @@ -268,6 +268,11 @@ scan = table.scan( selected_fields=("VendorID", "tpep_pickup_datetime", "tpep_dropoff_datetime"), ) +# Or filter using a string predicate +scan = table.scan( + row_filter="trip_distance > 10.0", +) + [task.file.file_path for task in scan.plan_files()] ``` diff --git a/pyiceberg/table/__init__.py b/pyiceberg/table/__init__.py index db42540496..dc9cc9e036 100644 --- a/pyiceberg/table/__init__.py +++ b/pyiceberg/table/__init__.py @@ -32,6 +32,7 @@ Optional, Tuple, TypeVar, + Union, ) from pydantic import Field @@ -40,6 +41,7 @@ AlwaysTrue, And, BooleanExpression, + parser, visitors, ) from pyiceberg.expressions.visitors import inclusive_projection @@ -69,6 +71,9 @@ from duckdb import DuckDBPyConnection +ALWAYS_TRUE = AlwaysTrue() + + class Table: identifier: Identifier = Field() metadata: TableMetadata = Field() @@ -91,7 +96,7 @@ def name(self) -> Identifier: def scan( self, - row_filter: Optional[BooleanExpression] = None, + row_filter: Union[str, BooleanExpression] = ALWAYS_TRUE, selected_fields: Tuple[str] = ("*",), case_sensitive: bool = True, snapshot_id: Optional[int] = None, @@ -99,7 +104,7 @@ def scan( ) -> TableScan[Any]: return DataScan( table=self, - row_filter=row_filter or AlwaysTrue(), + row_filter=row_filter, selected_fields=selected_fields, case_sensitive=case_sensitive, snapshot_id=snapshot_id, @@ -172,6 +177,19 @@ def __eq__(self, other: Any) -> bool: S = TypeVar("S", bound="TableScan", covariant=True) # type: ignore +def _parse_row_filter(expr: Union[str, BooleanExpression]) -> BooleanExpression: + """Accepts an expression in the form of a BooleanExpression or a string + + In the case of a string, it will be converted into a unbound BooleanExpression + + Args: + expr: Expression as a BooleanExpression or a string + + Returns: An unbound BooleanExpression + """ + return parser.parse(expr) if isinstance(expr, str) else expr + + class TableScan(Generic[S], ABC): table: Table row_filter: BooleanExpression @@ -183,14 +201,14 @@ class TableScan(Generic[S], ABC): def __init__( self, table: Table, - row_filter: Optional[BooleanExpression] = None, + row_filter: Union[str, BooleanExpression] = ALWAYS_TRUE, selected_fields: Tuple[str] = ("*",), case_sensitive: bool = True, snapshot_id: Optional[int] = None, options: Properties = EMPTY_DICT, ): self.table = table - self.row_filter = row_filter or AlwaysTrue() + self.row_filter = _parse_row_filter(row_filter) self.selected_fields = selected_fields self.case_sensitive = case_sensitive self.snapshot_id = snapshot_id @@ -241,8 +259,8 @@ def select(self, *field_names: str) -> S: return self.update(selected_fields=field_names) return self.update(selected_fields=tuple(set(self.selected_fields).intersection(set(field_names)))) - def filter(self, new_row_filter: BooleanExpression) -> S: - return self.update(row_filter=And(self.row_filter, new_row_filter)) + def filter(self, expr: Union[str, BooleanExpression]) -> S: + return self.update(row_filter=And(self.row_filter, _parse_row_filter(expr))) def with_case_sensitive(self, case_sensitive: bool = True) -> S: return self.update(case_sensitive=case_sensitive) @@ -285,7 +303,7 @@ class DataScan(TableScan["DataScan"]): def __init__( self, table: Table, - row_filter: Optional[BooleanExpression] = None, + row_filter: Union[str, BooleanExpression] = ALWAYS_TRUE, selected_fields: Tuple[str] = ("*",), case_sensitive: bool = True, snapshot_id: Optional[int] = None, From c23786afda9b3fe0c32b92e0d73b6ad17faea0a7 Mon Sep 17 00:00:00 2001 From: Fokko Driesprong Date: Tue, 7 Feb 2023 18:14:35 +0100 Subject: [PATCH 364/642] Python: Remove the DNF conversion (#6721) This was not needed after all. The Pythondoc was outdated, see: https://github.com/apache/arrow/issues/33973 --- pyiceberg/io/pyarrow.py | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/pyiceberg/io/pyarrow.py b/pyiceberg/io/pyarrow.py index e65a292d71..a064ae3bdf 100644 --- a/pyiceberg/io/pyarrow.py +++ b/pyiceberg/io/pyarrow.py @@ -61,9 +61,7 @@ from pyiceberg.expressions.visitors import ( BoundBooleanExpressionVisitor, bind, - expression_to_plain_format, extract_field_ids, - rewrite_to_dnf, translate_column_names, ) from pyiceberg.expressions.visitors import visit as boolean_expression_visit @@ -495,11 +493,9 @@ def _file_to_table( file_schema = Schema.parse_raw(schema_raw) pyarrow_filter = None - dnf_filter = None if bound_row_filter is not AlwaysTrue(): translated_row_filter = translate_column_names(bound_row_filter, file_schema, case_sensitive=case_sensitive) bound_file_filter = bind(file_schema, translated_row_filter, case_sensitive=case_sensitive) - dnf_filter = expression_to_plain_format(rewrite_to_dnf(bound_file_filter), cast_int_to_datetime=True) pyarrow_filter = expression_to_pyarrow(bound_file_filter) file_project_schema = prune_columns(file_schema, projected_field_ids, select_full_types=False) @@ -512,7 +508,7 @@ def _file_to_table( schema=parquet_schema, pre_buffer=True, buffer_size=8 * ONE_MEGABYTE, - filters=dnf_filter, + filters=pyarrow_filter, columns=[col.name for col in file_project_schema.columns], ) From 99849a24a21b0fcb32bda7a22dd682460abcc580 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 8 Feb 2023 07:41:37 +0100 Subject: [PATCH 365/642] Build: Bump cryptography from 39.0.0 to 39.0.1 in /python (#6767) --- poetry.lock | 54 ++++++++++++++++++++++++++--------------------------- 1 file changed, 27 insertions(+), 27 deletions(-) diff --git a/poetry.lock b/poetry.lock index 133783a794..e1e74c2a88 100644 --- a/poetry.lock +++ b/poetry.lock @@ -582,47 +582,47 @@ toml = ["tomli"] [[package]] name = "cryptography" -version = "39.0.0" +version = "39.0.1" description = "cryptography is a package which provides cryptographic recipes and primitives to Python developers." category = "main" optional = false python-versions = ">=3.6" files = [ - {file = "cryptography-39.0.0-cp36-abi3-macosx_10_12_universal2.whl", hash = "sha256:c52a1a6f81e738d07f43dab57831c29e57d21c81a942f4602fac7ee21b27f288"}, - {file = "cryptography-39.0.0-cp36-abi3-macosx_10_12_x86_64.whl", hash = "sha256:80ee674c08aaef194bc4627b7f2956e5ba7ef29c3cc3ca488cf15854838a8f72"}, - {file = "cryptography-39.0.0-cp36-abi3-manylinux_2_17_aarch64.manylinux2014_aarch64.manylinux_2_24_aarch64.whl", hash = "sha256:887cbc1ea60786e534b00ba8b04d1095f4272d380ebd5f7a7eb4cc274710fad9"}, - {file = "cryptography-39.0.0-cp36-abi3-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:6f97109336df5c178ee7c9c711b264c502b905c2d2a29ace99ed761533a3460f"}, - {file = "cryptography-39.0.0-cp36-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:1a6915075c6d3a5e1215eab5d99bcec0da26036ff2102a1038401d6ef5bef25b"}, - {file = "cryptography-39.0.0-cp36-abi3-manylinux_2_24_x86_64.whl", hash = "sha256:76c24dd4fd196a80f9f2f5405a778a8ca132f16b10af113474005635fe7e066c"}, - {file = "cryptography-39.0.0-cp36-abi3-manylinux_2_28_aarch64.whl", hash = "sha256:bae6c7f4a36a25291b619ad064a30a07110a805d08dc89984f4f441f6c1f3f96"}, - {file = "cryptography-39.0.0-cp36-abi3-manylinux_2_28_x86_64.whl", hash = "sha256:875aea1039d78557c7c6b4db2fe0e9d2413439f4676310a5f269dd342ca7a717"}, - {file = "cryptography-39.0.0-cp36-abi3-musllinux_1_1_aarch64.whl", hash = "sha256:f6c0db08d81ead9576c4d94bbb27aed8d7a430fa27890f39084c2d0e2ec6b0df"}, - {file = "cryptography-39.0.0-cp36-abi3-musllinux_1_1_x86_64.whl", hash = "sha256:f3ed2d864a2fa1666e749fe52fb8e23d8e06b8012e8bd8147c73797c506e86f1"}, - {file = "cryptography-39.0.0-cp36-abi3-win32.whl", hash = "sha256:f671c1bb0d6088e94d61d80c606d65baacc0d374e67bf895148883461cd848de"}, - {file = "cryptography-39.0.0-cp36-abi3-win_amd64.whl", hash = "sha256:e324de6972b151f99dc078defe8fb1b0a82c6498e37bff335f5bc6b1e3ab5a1e"}, - {file = "cryptography-39.0.0-pp38-pypy38_pp73-macosx_10_12_x86_64.whl", hash = "sha256:754978da4d0457e7ca176f58c57b1f9de6556591c19b25b8bcce3c77d314f5eb"}, - {file = "cryptography-39.0.0-pp38-pypy38_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:1ee1fd0de9851ff32dbbb9362a4d833b579b4a6cc96883e8e6d2ff2a6bc7104f"}, - {file = "cryptography-39.0.0-pp38-pypy38_pp73-manylinux_2_24_x86_64.whl", hash = "sha256:fec8b932f51ae245121c4671b4bbc030880f363354b2f0e0bd1366017d891458"}, - {file = "cryptography-39.0.0-pp38-pypy38_pp73-manylinux_2_28_x86_64.whl", hash = "sha256:407cec680e811b4fc829de966f88a7c62a596faa250fc1a4b520a0355b9bc190"}, - {file = "cryptography-39.0.0-pp38-pypy38_pp73-win_amd64.whl", hash = "sha256:7dacfdeee048814563eaaec7c4743c8aea529fe3dd53127313a792f0dadc1773"}, - {file = "cryptography-39.0.0-pp39-pypy39_pp73-macosx_10_12_x86_64.whl", hash = "sha256:ad04f413436b0781f20c52a661660f1e23bcd89a0e9bb1d6d20822d048cf2856"}, - {file = "cryptography-39.0.0-pp39-pypy39_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:50386acb40fbabbceeb2986332f0287f50f29ccf1497bae31cf5c3e7b4f4b34f"}, - {file = "cryptography-39.0.0-pp39-pypy39_pp73-manylinux_2_24_x86_64.whl", hash = "sha256:e5d71c5d5bd5b5c3eebcf7c5c2bb332d62ec68921a8c593bea8c394911a005ce"}, - {file = "cryptography-39.0.0-pp39-pypy39_pp73-manylinux_2_28_x86_64.whl", hash = "sha256:844ad4d7c3850081dffba91cdd91950038ee4ac525c575509a42d3fc806b83c8"}, - {file = "cryptography-39.0.0-pp39-pypy39_pp73-win_amd64.whl", hash = "sha256:e0a05aee6a82d944f9b4edd6a001178787d1546ec7c6223ee9a848a7ade92e39"}, - {file = "cryptography-39.0.0.tar.gz", hash = "sha256:f964c7dcf7802d133e8dbd1565914fa0194f9d683d82411989889ecd701e8adf"}, + {file = "cryptography-39.0.1-cp36-abi3-macosx_10_12_universal2.whl", hash = "sha256:6687ef6d0a6497e2b58e7c5b852b53f62142cfa7cd1555795758934da363a965"}, + {file = "cryptography-39.0.1-cp36-abi3-macosx_10_12_x86_64.whl", hash = "sha256:706843b48f9a3f9b9911979761c91541e3d90db1ca905fd63fee540a217698bc"}, + {file = "cryptography-39.0.1-cp36-abi3-manylinux_2_17_aarch64.manylinux2014_aarch64.manylinux_2_24_aarch64.whl", hash = "sha256:5d2d8b87a490bfcd407ed9d49093793d0f75198a35e6eb1a923ce1ee86c62b41"}, + {file = "cryptography-39.0.1-cp36-abi3-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:83e17b26de248c33f3acffb922748151d71827d6021d98c70e6c1a25ddd78505"}, + {file = "cryptography-39.0.1-cp36-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:e124352fd3db36a9d4a21c1aa27fd5d051e621845cb87fb851c08f4f75ce8be6"}, + {file = "cryptography-39.0.1-cp36-abi3-manylinux_2_24_x86_64.whl", hash = "sha256:5aa67414fcdfa22cf052e640cb5ddc461924a045cacf325cd164e65312d99502"}, + {file = "cryptography-39.0.1-cp36-abi3-manylinux_2_28_aarch64.whl", hash = "sha256:35f7c7d015d474f4011e859e93e789c87d21f6f4880ebdc29896a60403328f1f"}, + {file = "cryptography-39.0.1-cp36-abi3-manylinux_2_28_x86_64.whl", hash = "sha256:f24077a3b5298a5a06a8e0536e3ea9ec60e4c7ac486755e5fb6e6ea9b3500106"}, + {file = "cryptography-39.0.1-cp36-abi3-musllinux_1_1_aarch64.whl", hash = "sha256:f0c64d1bd842ca2633e74a1a28033d139368ad959872533b1bab8c80e8240a0c"}, + {file = "cryptography-39.0.1-cp36-abi3-musllinux_1_1_x86_64.whl", hash = "sha256:0f8da300b5c8af9f98111ffd512910bc792b4c77392a9523624680f7956a99d4"}, + {file = "cryptography-39.0.1-cp36-abi3-win32.whl", hash = "sha256:fe913f20024eb2cb2f323e42a64bdf2911bb9738a15dba7d3cce48151034e3a8"}, + {file = "cryptography-39.0.1-cp36-abi3-win_amd64.whl", hash = "sha256:ced4e447ae29ca194449a3f1ce132ded8fcab06971ef5f618605aacaa612beac"}, + {file = "cryptography-39.0.1-pp38-pypy38_pp73-macosx_10_12_x86_64.whl", hash = "sha256:807ce09d4434881ca3a7594733669bd834f5b2c6d5c7e36f8c00f691887042ad"}, + {file = "cryptography-39.0.1-pp38-pypy38_pp73-win_amd64.whl", hash = "sha256:96f1157a7c08b5b189b16b47bc9db2332269d6680a196341bf30046330d15388"}, + {file = "cryptography-39.0.1-pp39-pypy39_pp73-macosx_10_12_x86_64.whl", hash = "sha256:e422abdec8b5fa8462aa016786680720d78bdce7a30c652b7fadf83a4ba35336"}, + {file = "cryptography-39.0.1-pp39-pypy39_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.manylinux_2_24_aarch64.whl", hash = "sha256:b0afd054cd42f3d213bf82c629efb1ee5f22eba35bf0eec88ea9ea7304f511a2"}, + {file = "cryptography-39.0.1-pp39-pypy39_pp73-manylinux_2_24_x86_64.whl", hash = "sha256:6f8ba7f0328b79f08bdacc3e4e66fb4d7aab0c3584e0bd41328dce5262e26b2e"}, + {file = "cryptography-39.0.1-pp39-pypy39_pp73-manylinux_2_28_aarch64.whl", hash = "sha256:ef8b72fa70b348724ff1218267e7f7375b8de4e8194d1636ee60510aae104cd0"}, + {file = "cryptography-39.0.1-pp39-pypy39_pp73-manylinux_2_28_x86_64.whl", hash = "sha256:aec5a6c9864be7df2240c382740fcf3b96928c46604eaa7f3091f58b878c0bb6"}, + {file = "cryptography-39.0.1-pp39-pypy39_pp73-win_amd64.whl", hash = "sha256:fdd188c8a6ef8769f148f88f859884507b954cc64db6b52f66ef199bb9ad660a"}, + {file = "cryptography-39.0.1.tar.gz", hash = "sha256:d1f6198ee6d9148405e49887803907fe8962a23e6c6f83ea7d98f1c0de375695"}, ] [package.dependencies] cffi = ">=1.12" [package.extras] -docs = ["sphinx (>=1.6.5,!=1.8.0,!=3.1.0,!=3.1.1,!=5.2.0,!=5.2.0.post0)", "sphinx-rtd-theme"] +docs = ["sphinx (>=5.3.0)", "sphinx-rtd-theme (>=1.1.1)"] docstest = ["pyenchant (>=1.6.11)", "sphinxcontrib-spelling (>=4.0.1)", "twine (>=1.12.0)"] -pep8test = ["black", "ruff"] +pep8test = ["black", "check-manifest", "mypy", "ruff", "types-pytz", "types-requests"] sdist = ["setuptools-rust (>=0.11.4)"] ssh = ["bcrypt (>=3.1.5)"] -test = ["hypothesis (>=1.11.4,!=3.79.2)", "iso8601", "pretend", "pytest (>=6.2.0)", "pytest-benchmark", "pytest-cov", "pytest-subtests", "pytest-xdist", "pytz"] +test = ["hypothesis (>=1.11.4,!=3.79.2)", "iso8601", "pretend", "pytest (>=6.2.0)", "pytest-benchmark", "pytest-cov", "pytest-shard (>=0.1.2)", "pytest-subtests", "pytest-xdist", "pytz"] +test-randomorder = ["pytest-randomly"] +tox = ["tox"] [[package]] name = "distlib" From c01475c5492e052c91a89854af225aa6504e6be5 Mon Sep 17 00:00:00 2001 From: Srinivas Lade Date: Thu, 9 Feb 2023 10:20:00 -0500 Subject: [PATCH 366/642] Python: Loosen the version requirements for dependencies (#6745) --- poetry.lock | 2 +- pyproject.toml | 20 ++++++++++---------- 2 files changed, 11 insertions(+), 11 deletions(-) diff --git a/poetry.lock b/poetry.lock index e1e74c2a88..12eebbb607 100644 --- a/poetry.lock +++ b/poetry.lock @@ -2470,4 +2470,4 @@ snappy = ["python-snappy"] [metadata] lock-version = "2.0" python-versions = "^3.8" -content-hash = "9294e496c5be2025b0ea54dbd1e068730349b47337d459553c0080f41b3a3abd" +content-hash = "9ed787b3f081bd27cee8c90fa1ad3461d8734f5251edcc0339c62b37c3a275f9" diff --git a/pyproject.toml b/pyproject.toml index 1aace8eaba..735df4c5d0 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -50,33 +50,33 @@ include = [ [tool.poetry.dependencies] python = "^3.8" mmhash3 = "3.0.1" -requests = "2.28.2" +requests = ">=2.28.1,<=2.28.2" click = "8.1.3" -rich = "13.3.1" -pyyaml = "6.0.0" +rich = ">=13.0.0,<=13.3.1" +pyyaml = ">=5.4.0,<=6.0.0" pydantic = "1.10.4" -fsspec = "2023.1.0" +fsspec = ">=2022.8.2,<=2023.1.0" pyparsing = "3.0.9" zstandard = "0.19.0" -pyarrow = { version = "11.0.0", optional = true } +pyarrow = { version = ">=8.0.0,<=11.0.0", optional = true } -pandas = { version = "1.5.3", optional = true } +pandas = { version = ">=1.4.4,<=1.5.3", optional = true } -duckdb = { version = "0.6.1", optional = true } +duckdb = { version = ">=0.6.0,<=0.6.1", optional = true } python-snappy = { version = "0.6.1", optional = true } thrift = { version = "0.16.0", optional = true } -boto3 = {version = "1.24.59", optional = true} +boto3 = { version = "1.24.59", optional = true } # The versions of the fsspec implementations should run in sync with fsspec above -s3fs = { version = "2023.1.0", optional = true } -adlfs = { version = "2023.1.0", optional = true } +s3fs = { version = ">=2022.8.2,<=2023.1.0", optional = true } +adlfs = { version = ">=2022.8.2,<=2023.1.0", optional = true } [tool.poetry.dev-dependencies] pytest = "7.2.1" From 034c2e28939542cd72f8593ece338221715c9c77 Mon Sep 17 00:00:00 2001 From: Guillem Orellana Trullols Date: Mon, 13 Feb 2023 16:51:19 +0100 Subject: [PATCH 367/642] Python: TypeVar for bounding TableScan (#6819) * Properly use TypeVar for bounding TableScan return * Fix isort and mypy error * Update __init__.py --------- Co-authored-by: Fokko Driesprong --- pyiceberg/table/__init__.py | 21 ++++++++++----------- 1 file changed, 10 insertions(+), 11 deletions(-) diff --git a/pyiceberg/table/__init__.py b/pyiceberg/table/__init__.py index dc9cc9e036..df2e09ab3a 100644 --- a/pyiceberg/table/__init__.py +++ b/pyiceberg/table/__init__.py @@ -26,7 +26,6 @@ Any, Callable, Dict, - Generic, Iterator, List, Optional, @@ -101,7 +100,7 @@ def scan( case_sensitive: bool = True, snapshot_id: Optional[int] = None, options: Properties = EMPTY_DICT, - ) -> TableScan[Any]: + ) -> DataScan: return DataScan( table=self, row_filter=row_filter, @@ -174,9 +173,6 @@ def __eq__(self, other: Any) -> bool: ) -S = TypeVar("S", bound="TableScan", covariant=True) # type: ignore - - def _parse_row_filter(expr: Union[str, BooleanExpression]) -> BooleanExpression: """Accepts an expression in the form of a BooleanExpression or a string @@ -190,7 +186,10 @@ def _parse_row_filter(expr: Union[str, BooleanExpression]) -> BooleanExpression: return parser.parse(expr) if isinstance(expr, str) else expr -class TableScan(Generic[S], ABC): +S = TypeVar("S", bound="TableScan", covariant=True) + + +class TableScan(ABC): table: Table row_filter: BooleanExpression selected_fields: Tuple[str] @@ -246,7 +245,7 @@ def update(self: S, **overrides: Any) -> S: """Creates a copy of this table scan with updated fields.""" return type(self)(**{**self.__dict__, **overrides}) - def use_ref(self, name: str) -> S: + def use_ref(self: S, name: str) -> S: if self.snapshot_id: raise ValueError(f"Cannot override ref, already set snapshot id={self.snapshot_id}") if snapshot := self.table.snapshot_by_name(name): @@ -254,15 +253,15 @@ def use_ref(self, name: str) -> S: raise ValueError(f"Cannot scan unknown ref={name}") - def select(self, *field_names: str) -> S: + def select(self: S, *field_names: str) -> S: if "*" in self.selected_fields: return self.update(selected_fields=field_names) return self.update(selected_fields=tuple(set(self.selected_fields).intersection(set(field_names)))) - def filter(self, expr: Union[str, BooleanExpression]) -> S: + def filter(self: S, expr: Union[str, BooleanExpression]) -> S: return self.update(row_filter=And(self.row_filter, _parse_row_filter(expr))) - def with_case_sensitive(self, case_sensitive: bool = True) -> S: + def with_case_sensitive(self: S, case_sensitive: bool = True) -> S: return self.update(case_sensitive=case_sensitive) @@ -299,7 +298,7 @@ def _open_manifest(io: FileIO, manifest: ManifestFile, partition_filter: Callabl return [FileScanTask(file) for file in matching_partition_data_files] -class DataScan(TableScan["DataScan"]): +class DataScan(TableScan): def __init__( self, table: Table, From fe4ea5f0a9ba3adb09477613dcd18c70b8a70146 Mon Sep 17 00:00:00 2001 From: Ruben van de Geer <37443208+rubenvdg@users.noreply.github.com> Date: Mon, 13 Feb 2023 21:02:14 +0100 Subject: [PATCH 368/642] Python: Add support for static table (#6644) * save some lines * make metadata_location fixture * Add StaticTable * Fokko's feedback * linterrrrrr * Update __init__.py --------- Co-authored-by: Fokko Driesprong --- mkdocs/docs/api.md | 23 +++++++++++++++++++++++ pyiceberg/catalog/rest.py | 6 +----- pyiceberg/table/__init__.py | 24 +++++++++++++++++++++++- tests/catalog/test_hive.py | 12 ++---------- tests/conftest.py | 11 +++++++++++ tests/table/test_init.py | 19 +++++++++++++++++-- 6 files changed, 77 insertions(+), 18 deletions(-) diff --git a/mkdocs/docs/api.md b/mkdocs/docs/api.md index 114362a9ab..f6a7b0f646 100644 --- a/mkdocs/docs/api.md +++ b/mkdocs/docs/api.md @@ -58,6 +58,8 @@ Returns as list with tuples, containing a single table `taxis`: ## Load a table +### From a catalog + Loading the `taxis` table: ```python @@ -166,6 +168,27 @@ Table( ) ``` +### Directly from a metadata file + +To load a table directly from a metadata file (i.e., **without** using a catalog), you can use a `StaticTable` as follows: + +```python +table = StaticTable.from_metadata( + "s3a://warehouse/wh/nyc.db/taxis/metadata/00002-6ea51ce3-62aa-4197-9cf8-43d07c3440ca.metadata.json" +) +``` + +For the rest, this table behaves similarly as a table loaded using a catalog. Note that `StaticTable` is intended to be _read only_. + +Any properties related to file IO can be passed accordingly: + +```python +table = StaticTable.from_metadata( + "s3a://warehouse/wh/nyc.db/taxis/metadata/00002-6ea51ce3-62aa-4197-9cf8-43d07c3440ca.metadata.json", + {PY_IO_IMPL: "pyiceberg.some.FileIO.class"}, +) +``` + ## Create a table To create a table from a catalog: diff --git a/pyiceberg/catalog/rest.py b/pyiceberg/catalog/rest.py index 2f9eb5df11..89e28bd7ca 100644 --- a/pyiceberg/catalog/rest.py +++ b/pyiceberg/catalog/rest.py @@ -175,11 +175,7 @@ class RestCatalog(Catalog): session: Session properties: Properties - def __init__( - self, - name: str, - **properties: str, - ): + def __init__(self, name: str, **properties: str): """Rest Catalog You either need to provide a client_id and client_secret, or an already valid token. diff --git a/pyiceberg/table/__init__.py b/pyiceberg/table/__init__.py index df2e09ab3a..4e5d4fd262 100644 --- a/pyiceberg/table/__init__.py +++ b/pyiceberg/table/__init__.py @@ -44,7 +44,7 @@ visitors, ) from pyiceberg.expressions.visitors import inclusive_projection -from pyiceberg.io import FileIO +from pyiceberg.io import FileIO, load_file_io from pyiceberg.io.pyarrow import project_table from pyiceberg.manifest import ( DataFile, @@ -54,6 +54,7 @@ ) from pyiceberg.partitioning import PartitionSpec from pyiceberg.schema import Schema +from pyiceberg.serializers import FromInputFile from pyiceberg.table.metadata import TableMetadata from pyiceberg.table.snapshots import Snapshot, SnapshotLogEntry from pyiceberg.table.sorting import SortOrder @@ -173,6 +174,27 @@ def __eq__(self, other: Any) -> bool: ) +class StaticTable(Table): + """Load a table directly from a metadata file (i.e., without using a catalog).""" + + def refresh(self) -> Table: + """Refresh the current table metadata""" + raise NotImplementedError("To be implemented") + + @classmethod + def from_metadata(cls, metadata_location: str, properties: Properties = EMPTY_DICT) -> StaticTable: + io = load_file_io(properties=properties, location=metadata_location) + file = io.new_input(metadata_location) + metadata = FromInputFile.table_metadata(file) + + return cls( + identifier=("static-table", metadata_location), + metadata_location=metadata_location, + metadata=metadata, + io=load_file_io({**properties, **metadata.properties}), + ) + + def _parse_row_filter(expr: Union[str, BooleanExpression]) -> BooleanExpression: """Accepts an expression in the form of a BooleanExpression or a string diff --git a/tests/catalog/test_hive.py b/tests/catalog/test_hive.py index 1cb7cba04b..4525edc7ed 100644 --- a/tests/catalog/test_hive.py +++ b/tests/catalog/test_hive.py @@ -17,7 +17,6 @@ # pylint: disable=protected-access,redefined-outer-name import json import uuid -from typing import Any, Dict from unittest.mock import MagicMock, patch import pytest @@ -42,10 +41,8 @@ NoSuchNamespaceError, NoSuchTableError, ) -from pyiceberg.io.pyarrow import PyArrowFileIO from pyiceberg.partitioning import PartitionField, PartitionSpec from pyiceberg.schema import Schema -from pyiceberg.serializers import ToOutputFile from pyiceberg.table.metadata import TableMetadataUtil, TableMetadataV2 from pyiceberg.table.refs import SnapshotRef, SnapshotRefType from pyiceberg.table.snapshots import ( @@ -75,12 +72,7 @@ @pytest.fixture -def hive_table(tmp_path_factory: pytest.TempPathFactory, example_table_metadata_v2: Dict[str, Any]) -> HiveTable: - metadata_path = str(tmp_path_factory.mktemp("metadata") / f"{uuid.uuid4()}.metadata.json") - metadata = TableMetadataV2(**example_table_metadata_v2) - - ToOutputFile.table_metadata(metadata, PyArrowFileIO().new_output(location=str(metadata_path)), True) - +def hive_table(metadata_location: str) -> HiveTable: return HiveTable( tableName="new_tabl2e", dbName="default", @@ -119,7 +111,7 @@ def hive_table(tmp_path_factory: pytest.TempPathFactory, example_table_metadata_ "EXTERNAL": "TRUE", "transient_lastDdlTime": "1659092339", "table_type": "ICEBERG", - "metadata_location": metadata_path, + "metadata_location": metadata_location, }, viewOriginalText=None, viewExpandedText=None, diff --git a/tests/conftest.py b/tests/conftest.py index 2ce71f5f12..c9bb26854a 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -26,6 +26,7 @@ """ import os import string +import uuid from random import choice from tempfile import TemporaryDirectory from typing import ( @@ -54,6 +55,8 @@ from pyiceberg.io.fsspec import FsspecFileIO from pyiceberg.io.pyarrow import PyArrowFile, PyArrowFileIO from pyiceberg.schema import Schema +from pyiceberg.serializers import ToOutputFile +from pyiceberg.table.metadata import TableMetadataV2 from pyiceberg.types import ( BinaryType, BooleanType, @@ -312,6 +315,14 @@ def example_table_metadata_v2() -> Dict[str, Any]: return EXAMPLE_TABLE_METADATA_V2 +@pytest.fixture(scope="session") +def metadata_location(tmp_path_factory: pytest.TempPathFactory) -> str: + metadata_location = str(tmp_path_factory.mktemp("metadata") / f"{uuid.uuid4()}.metadata.json") + metadata = TableMetadataV2(**EXAMPLE_TABLE_METADATA_V2) + ToOutputFile.table_metadata(metadata, PyArrowFileIO().new_output(location=metadata_location), overwrite=True) + return metadata_location + + manifest_entry_records = [ { "status": 1, diff --git a/tests/table/test_init.py b/tests/table/test_init.py index 506c6609f7..bf4fafdc30 100644 --- a/tests/table/test_init.py +++ b/tests/table/test_init.py @@ -25,11 +25,11 @@ EqualTo, In, ) -from pyiceberg.io import load_file_io +from pyiceberg.io import PY_IO_IMPL, load_file_io from pyiceberg.manifest import DataFile, ManifestContent from pyiceberg.partitioning import PartitionField, PartitionSpec from pyiceberg.schema import Schema -from pyiceberg.table import Table, _check_content +from pyiceberg.table import StaticTable, Table, _check_content from pyiceberg.table.metadata import TableMetadataV2 from pyiceberg.table.snapshots import ( Operation, @@ -59,6 +59,11 @@ def table(example_table_metadata_v2: Dict[str, Any]) -> Table: ) +@pytest.fixture +def static_table(metadata_location: str) -> StaticTable: + return StaticTable.from_metadata(metadata_location) + + def test_schema(table: Table) -> None: assert table.schema() == Schema( NestedField(field_id=1, name="x", field_type=LongType(), required=True), @@ -265,3 +270,13 @@ def test_check_content_data() -> None: def test_check_content_missing_attr() -> None: r = Record(*([None] * 15)) assert _check_content(r) == r # type: ignore + + +def test_static_table_same_as_table(table: Table, static_table: StaticTable) -> None: + assert isinstance(static_table, Table) + assert static_table.metadata == table.metadata + + +def test_static_table_io_does_not_exist(metadata_location: str) -> None: + with pytest.raises(ValueError): + StaticTable.from_metadata(metadata_location, {PY_IO_IMPL: "pyiceberg.does.not.exist.FileIO"}) From ec4b4fc838fe9811e1f4fd11ab82bd5731bf4512 Mon Sep 17 00:00:00 2001 From: Fokko Driesprong Date: Wed, 15 Feb 2023 10:25:33 +0100 Subject: [PATCH 369/642] Python: Inline the PyArrow import (#6827) --- pyiceberg/table/__init__.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/pyiceberg/table/__init__.py b/pyiceberg/table/__init__.py index 4e5d4fd262..0bce2a4e86 100644 --- a/pyiceberg/table/__init__.py +++ b/pyiceberg/table/__init__.py @@ -45,7 +45,6 @@ ) from pyiceberg.expressions.visitors import inclusive_projection from pyiceberg.io import FileIO, load_file_io -from pyiceberg.io.pyarrow import project_table from pyiceberg.manifest import ( DataFile, ManifestContent, @@ -385,6 +384,8 @@ def plan_files(self) -> Iterator[FileScanTask]: ) def to_arrow(self) -> pa.Table: + from pyiceberg.io.pyarrow import project_table + return project_table( self.plan_files(), self.table, self.row_filter, self.projection(), case_sensitive=self.case_sensitive ) From 2689892fda953a59fd0cfb9714032dba54866e83 Mon Sep 17 00:00:00 2001 From: Fokko Driesprong Date: Wed, 15 Feb 2023 10:25:59 +0100 Subject: [PATCH 370/642] Python: Set PyArrow as the default FileIO (#6822) PyArrow is the most feature complete FileIO, and it can be a bit confusing. --- mkdocs/docs/configuration.md | 23 ++++++++++++++ pyiceberg/io/__init__.py | 8 ++--- tests/catalog/test_glue.py | 58 ++++++++++++++++++------------------ tests/io/test_io.py | 3 +- 4 files changed, 57 insertions(+), 35 deletions(-) diff --git a/mkdocs/docs/configuration.md b/mkdocs/docs/configuration.md index 3780d580e2..23aec50554 100644 --- a/mkdocs/docs/configuration.md +++ b/mkdocs/docs/configuration.md @@ -35,14 +35,37 @@ export PYICEBERG_CATALOG__DEFAULT__URI=thrift://localhost:9083 The environment variable picked up by Iceberg starts with `PYICEBERG_` and then follows the yaml structure below, where a double underscore `__` represents a nested field. +## FileIO + +Iceberg works with the concept of a FileIO which is a pluggable module for reading, writing, and deleting files. By default, PyIceberg will try to initialize the FileIO that's suitable for the scheme (`s3://`, `gs://`, etc.) and will use the first one that's installed. + +- **s3**, **s3a**, **s3n**: `PyArrowFileIO`, `FsspecFileIO` +- **gs**: `PyArrowFileIO` +- **file**: `PyArrowFileIO` +- **hdfs**: `PyArrowFileIO` +- **abfs**, **abfss**: `FsspecFileIO` + +You can also set the FileIO explicitly: + +| Key | Example | Description | +|----------------------|----------------------------------|-------------------------------------------------------------------------------------------------| +| py-io-impl | pyiceberg.io.fsspec.FsspecFileIO | Sets the FileIO explicitly to an implementation, and will fail explicitly if it can't be loaded | + For the FileIO there are several configuration options available: +### S3 + | Key | Example | Description | |--------------------------|---------------------|-----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| | s3.endpoint | https://10.0.19.25/ | Configure an alternative endpoint of the S3 service for the FileIO to access. This could be used to use S3FileIO with any s3-compatible object storage service that has a different endpoint, or access a private S3 endpoint in a virtual private cloud. | | s3.access-key-id | admin | Configure the static secret access key used to access the FileIO. | | s3.secret-access-key | password | Configure the static session token used to access the FileIO. | | s3.signer | bearer | Configure the signature version of the FileIO. | + +### Azure Data lake + +| Key | Example | Description | +|--------------------------|---------------------|-----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| | adlfs.endpoint | http://127.0.0.1/ | Configure an alternative endpoint of the ADLFS service for the FileIO to access. This could be used to use FileIO with any adlfs-compatible object storage service that has a different endpoint (like [azurite](https://github.com/azure/azurite)). | | adlfs.account-name | devstoreaccount1 | Configure the static storage account name used to access the FileIO. | | adlfs.account-key | Eby8vdM02xNOcqF... | Configure the static storage account key used to access the FileIO. | diff --git a/pyiceberg/io/__init__.py b/pyiceberg/io/__init__.py index fd86bf5c91..9690e53c67 100644 --- a/pyiceberg/io/__init__.py +++ b/pyiceberg/io/__init__.py @@ -254,10 +254,10 @@ def delete(self, location: Union[str, InputFile, OutputFile]) -> None: # Mappings from the Java FileIO impl to a Python one. The list is ordered by preference. # If an implementation isn't installed, it will fall back to the next one. SCHEMA_TO_FILE_IO: Dict[str, List[str]] = { - "s3": [FSSPEC_FILE_IO, ARROW_FILE_IO], - "s3a": [FSSPEC_FILE_IO, ARROW_FILE_IO], - "s3n": [FSSPEC_FILE_IO, ARROW_FILE_IO], - "gcs": [ARROW_FILE_IO], + "s3": [ARROW_FILE_IO, FSSPEC_FILE_IO], + "s3a": [ARROW_FILE_IO, FSSPEC_FILE_IO], + "s3n": [ARROW_FILE_IO, FSSPEC_FILE_IO], + "gs": [ARROW_FILE_IO], "file": [ARROW_FILE_IO], "hdfs": [ARROW_FILE_IO], "abfs": [FSSPEC_FILE_IO], diff --git a/tests/catalog/test_glue.py b/tests/catalog/test_glue.py index bdf64b9ef1..26825c052a 100644 --- a/tests/catalog/test_glue.py +++ b/tests/catalog/test_glue.py @@ -52,7 +52,7 @@ def test_create_table_with_database_location( _bucket_initialize: None, _patch_aiobotocore: None, table_schema_nested: Schema, database_name: str, table_name: str ) -> None: identifier = (database_name, table_name) - test_catalog = GlueCatalog("glue") + test_catalog = GlueCatalog("glue", **{"py-io-impl": "pyiceberg.io.fsspec.FsspecFileIO"}) test_catalog.create_namespace(namespace=database_name, properties={"location": f"s3://{BUCKET_NAME}/{database_name}.db"}) table = test_catalog.create_table(identifier, table_schema_nested) assert table.identifier == identifier @@ -64,7 +64,7 @@ def test_create_table_with_default_warehouse( _bucket_initialize: None, _patch_aiobotocore: None, table_schema_nested: Schema, database_name: str, table_name: str ) -> None: identifier = (database_name, table_name) - test_catalog = GlueCatalog("glue", warehouse=f"s3://{BUCKET_NAME}") + test_catalog = GlueCatalog("glue", **{"py-io-impl": "pyiceberg.io.fsspec.FsspecFileIO", "warehouse": f"s3://{BUCKET_NAME}"}) test_catalog.create_namespace(namespace=database_name) table = test_catalog.create_table(identifier, table_schema_nested) assert table.identifier == identifier @@ -76,7 +76,7 @@ def test_create_table_with_given_location( _bucket_initialize: None, _patch_aiobotocore: None, table_schema_nested: Schema, database_name: str, table_name: str ) -> None: identifier = (database_name, table_name) - test_catalog = GlueCatalog("glue") + test_catalog = GlueCatalog("glue", **{"py-io-impl": "pyiceberg.io.fsspec.FsspecFileIO"}) test_catalog.create_namespace(namespace=database_name) table = test_catalog.create_table( identifier=identifier, schema=table_schema_nested, location=f"s3://{BUCKET_NAME}/{database_name}.db/{table_name}" @@ -90,7 +90,7 @@ def test_create_table_with_no_location( _bucket_initialize: None, _patch_aiobotocore: None, table_schema_nested: Schema, database_name: str, table_name: str ) -> None: identifier = (database_name, table_name) - test_catalog = GlueCatalog("glue") + test_catalog = GlueCatalog("glue", **{"py-io-impl": "pyiceberg.io.fsspec.FsspecFileIO"}) test_catalog.create_namespace(namespace=database_name) with pytest.raises(ValueError): test_catalog.create_table(identifier=identifier, schema=table_schema_nested) @@ -101,7 +101,7 @@ def test_create_table_with_strips( _bucket_initialize: None, _patch_aiobotocore: None, table_schema_nested: Schema, database_name: str, table_name: str ) -> None: identifier = (database_name, table_name) - test_catalog = GlueCatalog("glue") + test_catalog = GlueCatalog("glue", **{"py-io-impl": "pyiceberg.io.fsspec.FsspecFileIO"}) test_catalog.create_namespace(namespace=database_name, properties={"location": f"s3://{BUCKET_NAME}/{database_name}.db/"}) table = test_catalog.create_table(identifier, table_schema_nested) assert table.identifier == identifier @@ -113,7 +113,7 @@ def test_create_table_with_strips_bucket_root( _bucket_initialize: None, _patch_aiobotocore: None, table_schema_nested: Schema, database_name: str, table_name: str ) -> None: identifier = (database_name, table_name) - test_catalog = GlueCatalog("glue", warehouse=f"s3://{BUCKET_NAME}/") + test_catalog = GlueCatalog("glue", **{"py-io-impl": "pyiceberg.io.fsspec.FsspecFileIO", "warehouse": f"s3://{BUCKET_NAME}/"}) test_catalog.create_namespace(namespace=database_name) table_strip = test_catalog.create_table(identifier, table_schema_nested) assert table_strip.identifier == identifier @@ -125,7 +125,7 @@ def test_create_table_with_no_database( _bucket_initialize: None, _patch_aiobotocore: None, table_schema_nested: Schema, database_name: str, table_name: str ) -> None: identifier = (database_name, table_name) - test_catalog = GlueCatalog("glue") + test_catalog = GlueCatalog("glue", **{"py-io-impl": "pyiceberg.io.fsspec.FsspecFileIO"}) with pytest.raises(NoSuchNamespaceError): test_catalog.create_table(identifier=identifier, schema=table_schema_nested) @@ -135,7 +135,7 @@ def test_create_duplicated_table( _bucket_initialize: None, _patch_aiobotocore: None, table_schema_nested: Schema, database_name: str, table_name: str ) -> None: identifier = (database_name, table_name) - test_catalog = GlueCatalog("glue", warehouse=f"s3://{BUCKET_NAME}") + test_catalog = GlueCatalog("glue", **{"py-io-impl": "pyiceberg.io.fsspec.FsspecFileIO", "warehouse": f"s3://{BUCKET_NAME}/"}) test_catalog.create_namespace(namespace=database_name) test_catalog.create_table(identifier, table_schema_nested) with pytest.raises(TableAlreadyExistsError): @@ -147,7 +147,7 @@ def test_load_table( _bucket_initialize: None, _patch_aiobotocore: None, table_schema_nested: Schema, database_name: str, table_name: str ) -> None: identifier = (database_name, table_name) - test_catalog = GlueCatalog("glue", warehouse=f"s3://{BUCKET_NAME}") + test_catalog = GlueCatalog("glue", **{"py-io-impl": "pyiceberg.io.fsspec.FsspecFileIO", "warehouse": f"s3://{BUCKET_NAME}/"}) test_catalog.create_namespace(namespace=database_name) test_catalog.create_table(identifier, table_schema_nested) table = test_catalog.load_table(identifier) @@ -158,7 +158,7 @@ def test_load_table( @mock_glue def test_load_non_exist_table(_bucket_initialize: None, _patch_aiobotocore: None, database_name: str, table_name: str) -> None: identifier = (database_name, table_name) - test_catalog = GlueCatalog("glue", warehouse=f"s3://{BUCKET_NAME}") + test_catalog = GlueCatalog("glue", **{"py-io-impl": "pyiceberg.io.fsspec.FsspecFileIO", "warehouse": f"s3://{BUCKET_NAME}/"}) test_catalog.create_namespace(namespace=database_name) with pytest.raises(NoSuchTableError): test_catalog.load_table(identifier) @@ -169,7 +169,7 @@ def test_drop_table( _bucket_initialize: None, _patch_aiobotocore: None, table_schema_nested: Schema, database_name: str, table_name: str ) -> None: identifier = (database_name, table_name) - test_catalog = GlueCatalog("glue", warehouse=f"s3://{BUCKET_NAME}") + test_catalog = GlueCatalog("glue", **{"py-io-impl": "pyiceberg.io.fsspec.FsspecFileIO", "warehouse": f"s3://{BUCKET_NAME}/"}) test_catalog.create_namespace(namespace=database_name) test_catalog.create_table(identifier, table_schema_nested) table = test_catalog.load_table(identifier) @@ -183,7 +183,7 @@ def test_drop_table( @mock_glue def test_drop_non_exist_table(_bucket_initialize: None, _patch_aiobotocore: None, database_name: str, table_name: str) -> None: identifier = (database_name, table_name) - test_catalog = GlueCatalog("glue", warehouse=f"s3://{BUCKET_NAME}") + test_catalog = GlueCatalog("glue", **{"py-io-impl": "pyiceberg.io.fsspec.FsspecFileIO", "warehouse": f"s3://{BUCKET_NAME}/"}) with pytest.raises(NoSuchTableError): test_catalog.drop_table(identifier) @@ -195,7 +195,7 @@ def test_rename_table( new_table_name = f"{table_name}_new" identifier = (database_name, table_name) new_identifier = (database_name, new_table_name) - test_catalog = GlueCatalog("glue", warehouse=f"s3://{BUCKET_NAME}") + test_catalog = GlueCatalog("glue", **{"py-io-impl": "pyiceberg.io.fsspec.FsspecFileIO", "warehouse": f"s3://{BUCKET_NAME}/"}) test_catalog.create_namespace(namespace=database_name) table = test_catalog.create_table(identifier, table_schema_nested) assert table.identifier == identifier @@ -216,7 +216,7 @@ def test_rename_table_no_params(_glue, _bucket_initialize: None, _patch_aiobotoc new_table_name = f"{table_name}_new" identifier = (database_name, table_name) new_identifier = (new_database_name, new_table_name) - test_catalog = GlueCatalog("glue", warehouse=f"s3://{BUCKET_NAME}") + test_catalog = GlueCatalog("glue", **{"py-io-impl": "pyiceberg.io.fsspec.FsspecFileIO", "warehouse": f"s3://{BUCKET_NAME}/"}) test_catalog.create_namespace(namespace=database_name) test_catalog.create_namespace(namespace=new_database_name) _glue.create_table( @@ -233,7 +233,7 @@ def test_rename_non_iceberg_table(_glue, _bucket_initialize: None, _patch_aiobot new_table_name = f"{table_name}_new" identifier = (database_name, table_name) new_identifier = (new_database_name, new_table_name) - test_catalog = GlueCatalog("glue", warehouse=f"s3://{BUCKET_NAME}") + test_catalog = GlueCatalog("glue", **{"py-io-impl": "pyiceberg.io.fsspec.FsspecFileIO", "warehouse": f"s3://{BUCKET_NAME}/"}) test_catalog.create_namespace(namespace=database_name) test_catalog.create_namespace(namespace=new_database_name) _glue.create_table( @@ -257,7 +257,7 @@ def test_list_tables( table_name: str, table_list: List[str], ) -> None: - test_catalog = GlueCatalog("glue", warehouse=f"s3://{BUCKET_NAME}") + test_catalog = GlueCatalog("glue", **{"py-io-impl": "pyiceberg.io.fsspec.FsspecFileIO", "warehouse": f"s3://{BUCKET_NAME}/"}) test_catalog.create_namespace(namespace=database_name) for table_name in table_list: test_catalog.create_table((database_name, table_name), table_schema_nested) @@ -268,7 +268,7 @@ def test_list_tables( @mock_glue def test_list_namespaces(_bucket_initialize: None, _patch_aiobotocore: None, database_list: List[str]) -> None: - test_catalog = GlueCatalog("glue") + test_catalog = GlueCatalog("glue", **{"py-io-impl": "pyiceberg.io.fsspec.FsspecFileIO"}) for database_name in database_list: test_catalog.create_namespace(namespace=database_name) loaded_database_list = test_catalog.list_namespaces() @@ -278,7 +278,7 @@ def test_list_namespaces(_bucket_initialize: None, _patch_aiobotocore: None, dat @mock_glue def test_create_namespace_no_properties(_bucket_initialize: None, _patch_aiobotocore: None, database_name: str) -> None: - test_catalog = GlueCatalog("glue") + test_catalog = GlueCatalog("glue", **{"py-io-impl": "pyiceberg.io.fsspec.FsspecFileIO"}) test_catalog.create_namespace(namespace=database_name) loaded_database_list = test_catalog.list_namespaces() assert len(loaded_database_list) == 1 @@ -296,7 +296,7 @@ def test_create_namespace_with_comment_and_location( "comment": "this is a test description", "location": test_location, } - test_catalog = GlueCatalog("glue") + test_catalog = GlueCatalog("glue", **{"py-io-impl": "pyiceberg.io.fsspec.FsspecFileIO"}) test_catalog.create_namespace(namespace=database_name, properties=test_properties) loaded_database_list = test_catalog.list_namespaces() assert len(loaded_database_list) == 1 @@ -308,7 +308,7 @@ def test_create_namespace_with_comment_and_location( @mock_glue def test_create_duplicated_namespace(_bucket_initialize: None, _patch_aiobotocore: None, database_name: str) -> None: - test_catalog = GlueCatalog("glue") + test_catalog = GlueCatalog("glue", **{"py-io-impl": "pyiceberg.io.fsspec.FsspecFileIO"}) test_catalog.create_namespace(namespace=database_name) loaded_database_list = test_catalog.list_namespaces() assert len(loaded_database_list) == 1 @@ -319,7 +319,7 @@ def test_create_duplicated_namespace(_bucket_initialize: None, _patch_aiobotocor @mock_glue def test_drop_namespace(_bucket_initialize: None, _patch_aiobotocore: None, database_name: str) -> None: - test_catalog = GlueCatalog("glue") + test_catalog = GlueCatalog("glue", **{"py-io-impl": "pyiceberg.io.fsspec.FsspecFileIO"}) test_catalog.create_namespace(namespace=database_name) loaded_database_list = test_catalog.list_namespaces() assert len(loaded_database_list) == 1 @@ -334,7 +334,7 @@ def test_drop_non_empty_namespace( _bucket_initialize: None, _patch_aiobotocore: None, table_schema_nested: Schema, database_name: str, table_name: str ) -> None: identifier = (database_name, table_name) - test_catalog = GlueCatalog("glue", warehouse=f"s3://{BUCKET_NAME}") + test_catalog = GlueCatalog("glue", **{"py-io-impl": "pyiceberg.io.fsspec.FsspecFileIO", "warehouse": f"s3://{BUCKET_NAME}/"}) test_catalog.create_namespace(namespace=database_name) test_catalog.create_table(identifier, table_schema_nested) assert len(test_catalog.list_tables(database_name)) == 1 @@ -344,7 +344,7 @@ def test_drop_non_empty_namespace( @mock_glue def test_drop_non_exist_namespace(_bucket_initialize: None, _patch_aiobotocore: None, database_name: str) -> None: - test_catalog = GlueCatalog("glue") + test_catalog = GlueCatalog("glue", **{"py-io-impl": "pyiceberg.io.fsspec.FsspecFileIO"}) with pytest.raises(NoSuchNamespaceError): test_catalog.drop_namespace(database_name) @@ -359,7 +359,7 @@ def test_load_namespace_properties(_bucket_initialize: None, _patch_aiobotocore: "test_property2": "2", "test_property3": "3", } - test_catalog = GlueCatalog("glue") + test_catalog = GlueCatalog("glue", **{"py-io-impl": "pyiceberg.io.fsspec.FsspecFileIO"}) test_catalog.create_namespace(database_name, test_properties) listed_properties = test_catalog.load_namespace_properties(database_name) for k, v in listed_properties.items(): @@ -369,7 +369,7 @@ def test_load_namespace_properties(_bucket_initialize: None, _patch_aiobotocore: @mock_glue def test_load_non_exist_namespace_properties(_bucket_initialize: None, _patch_aiobotocore: None, database_name: str) -> None: - test_catalog = GlueCatalog("glue") + test_catalog = GlueCatalog("glue", **{"py-io-impl": "pyiceberg.io.fsspec.FsspecFileIO"}) with pytest.raises(NoSuchNamespaceError): test_catalog.load_namespace_properties(database_name) @@ -385,7 +385,7 @@ def test_update_namespace_properties(_bucket_initialize: None, _patch_aiobotocor } removals = {"test_property1", "test_property2", "test_property3", "should_not_removed"} updates = {"test_property4": "4", "test_property5": "5", "comment": "updated test description"} - test_catalog = GlueCatalog("glue") + test_catalog = GlueCatalog("glue", **{"py-io-impl": "pyiceberg.io.fsspec.FsspecFileIO"}) test_catalog.create_namespace(database_name, test_properties) update_report = test_catalog.update_namespace_properties(database_name, removals, updates) for k in updates.keys(): @@ -401,7 +401,7 @@ def test_update_namespace_properties(_bucket_initialize: None, _patch_aiobotocor @mock_glue def test_load_empty_namespace_properties(_bucket_initialize: None, _patch_aiobotocore: None, database_name: str) -> None: - test_catalog = GlueCatalog("glue") + test_catalog = GlueCatalog("glue", **{"py-io-impl": "pyiceberg.io.fsspec.FsspecFileIO"}) test_catalog.create_namespace(database_name) listed_properties = test_catalog.load_namespace_properties(database_name) assert listed_properties == {} @@ -411,7 +411,7 @@ def test_load_empty_namespace_properties(_bucket_initialize: None, _patch_aiobot def test_load_default_namespace_properties(_glue, _bucket_initialize, _patch_aiobotocore, database_name: str) -> None: # type: ignore # simulate creating database with default settings through AWS Glue Web Console _glue.create_database(DatabaseInput={"Name": database_name}) - test_catalog = GlueCatalog("glue") + test_catalog = GlueCatalog("glue", **{"py-io-impl": "pyiceberg.io.fsspec.FsspecFileIO"}) listed_properties = test_catalog.load_namespace_properties(database_name) assert listed_properties == {} @@ -429,7 +429,7 @@ def test_update_namespace_properties_overlap_update_removal( } removals = {"test_property1", "test_property2", "test_property3", "should_not_removed"} updates = {"test_property1": "4", "test_property5": "5", "comment": "updated test description"} - test_catalog = GlueCatalog("glue") + test_catalog = GlueCatalog("glue", **{"py-io-impl": "pyiceberg.io.fsspec.FsspecFileIO"}) test_catalog.create_namespace(database_name, test_properties) with pytest.raises(ValueError): test_catalog.update_namespace_properties(database_name, removals, updates) diff --git a/tests/io/test_io.py b/tests/io/test_io.py index 6cb5dbc7d3..c872ae1c7c 100644 --- a/tests/io/test_io.py +++ b/tests/io/test_io.py @@ -26,7 +26,6 @@ _import_file_io, load_file_io, ) -from pyiceberg.io.fsspec import FsspecFileIO from pyiceberg.io.pyarrow import PyArrowFileIO @@ -277,7 +276,7 @@ def test_load_file_io_does_not_exist() -> None: def test_load_file_io_warehouse() -> None: - assert isinstance(load_file_io({"warehouse": "s3://some-path/"}), FsspecFileIO) + assert isinstance(load_file_io({"warehouse": "s3://some-path/"}), PyArrowFileIO) def test_load_file_io_location() -> None: From dc90c85357c19e44b88177ed240cecbec2bda1cb Mon Sep 17 00:00:00 2001 From: Armin Najafi Date: Wed, 15 Feb 2023 01:32:10 -0800 Subject: [PATCH 371/642] Python: Support for DynamoDB Catalog (#6646) * Implement support for DynamoDB catalog * Python: Support DynamoDB catalog #6541 * Fix licence * Remove BaseAwsCatalog class * Add custom DynamoDB table name * Remove __init__.py from tests/catalog/ * Apply comments from PR * Rebase and fix tests * Return empty list for hierarchical namespace * Return empty list for hierarchical namespace * Apply comments from PR * Rebase and fix circular import * Fix linter issue * Fix integ tests --- mkdocs/docs/configuration.md | 13 + mkdocs/docs/index.md | 19 +- pyiceberg/catalog/__init__.py | 145 +++- pyiceberg/catalog/dynamodb.py | 769 +++++++++++++++++++++ pyiceberg/catalog/glue.py | 228 ++---- pyiceberg/catalog/hive.py | 76 +- pyiceberg/exceptions.py | 12 + pyiceberg/table/__init__.py | 4 +- pyproject.toml | 1 + tests/catalog/integration_test_dynamodb.py | 255 +++++++ tests/catalog/integration_test_glue.py | 39 +- tests/catalog/test_dynamodb.py | 452 ++++++++++++ tests/catalog/test_glue.py | 32 +- tests/catalog/test_hive.py | 2 +- tests/conftest.py | 65 +- 15 files changed, 1809 insertions(+), 303 deletions(-) create mode 100644 pyiceberg/catalog/dynamodb.py create mode 100644 tests/catalog/integration_test_dynamodb.py create mode 100644 tests/catalog/test_dynamodb.py diff --git a/mkdocs/docs/configuration.md b/mkdocs/docs/configuration.md index 23aec50554..9691bd073a 100644 --- a/mkdocs/docs/configuration.md +++ b/mkdocs/docs/configuration.md @@ -108,3 +108,16 @@ catalog: default: type: glue ``` + +## DynamoDB Catalog + +If you want to use AWS DynamoDB as the catalog, you can use the last two ways to configure the pyiceberg and refer +[How to configure AWS credentials](https://docs.aws.amazon.com/cli/latest/userguide/cli-chap-configure.html) +to set your AWS account credentials locally. + +```yaml +catalog: + default: + type: dynamodb + table-name: iceberg +``` diff --git a/mkdocs/docs/index.md b/mkdocs/docs/index.md index 2dbbf5159a..c35196813f 100644 --- a/mkdocs/docs/index.md +++ b/mkdocs/docs/index.md @@ -43,15 +43,16 @@ pip3 install -e ".[s3fs,hive]" You can mix and match optional dependencies depending on your needs: -| Key | Description: | -|---------|----------------------------------------------------------------------| -| hive | Support for the Hive metastore | -| glue | Support for AWS Glue | -| pyarrow | PyArrow as a FileIO implementation to interact with the object store | -| duckdb | Installs both PyArrow and DuckDB | -| s3fs | S3FS as a FileIO implementation to interact with the object store | -| adlfs | ADLFS as a FileIO implementation to interact with the object store | -| snappy | Support for snappy Avro compression | +| Key | Description: | +|----------|----------------------------------------------------------------------| +| hive | Support for the Hive metastore | +| glue | Support for AWS Glue | +| dynamodb | Support for AWS DynamoDB | +| pyarrow | PyArrow as a FileIO implementation to interact with the object store | +| duckdb | Installs both PyArrow and DuckDB | +| s3fs | S3FS as a FileIO implementation to interact with the object store | +| adlfs | ADLFS as a FileIO implementation to interact with the object store | +| snappy | Support for snappy Avro compression | You either need to install `s3fs`, `adlfs` or `pyarrow` for fetching files. diff --git a/pyiceberg/catalog/__init__.py b/pyiceberg/catalog/__init__.py index a2577228eb..c908a3b462 100644 --- a/pyiceberg/catalog/__init__.py +++ b/pyiceberg/catalog/__init__.py @@ -18,6 +18,7 @@ from __future__ import annotations import logging +import uuid from abc import ABC, abstractmethod from dataclasses import dataclass from enum import Enum @@ -27,16 +28,19 @@ List, Optional, Set, + Tuple, + Type, Union, cast, ) -from pyiceberg.exceptions import NotInstalledError +from pyiceberg.exceptions import NoSuchNamespaceError, NoSuchTableError, NotInstalledError from pyiceberg.io import FileIO, load_file_io from pyiceberg.manifest import ManifestFile from pyiceberg.partitioning import UNPARTITIONED_PARTITION_SPEC, PartitionSpec from pyiceberg.schema import Schema -from pyiceberg.table import Table +from pyiceberg.serializers import ToOutputFile +from pyiceberg.table import Table, TableMetadata from pyiceberg.table.sorting import UNSORTED_SORT_ORDER, SortOrder from pyiceberg.typedef import ( EMPTY_DICT, @@ -56,17 +60,21 @@ TABLE_TYPE = "table_type" WAREHOUSE_LOCATION = "warehouse" METADATA_LOCATION = "metadata_location" +PREVIOUS_METADATA_LOCATION = "previous_metadata_location" MANIFEST = "manifest" MANIFEST_LIST = "manifest list" PREVIOUS_METADATA = "previous metadata" METADATA = "metadata" URI = "uri" +LOCATION = "location" +EXTERNAL_TABLE = "EXTERNAL_TABLE" class CatalogType(Enum): REST = "rest" HIVE = "hive" GLUE = "glue" + DYNAMODB = "dynamodb" def load_rest(name: str, conf: Properties) -> Catalog: @@ -93,10 +101,20 @@ def load_glue(name: str, conf: Properties) -> Catalog: raise NotInstalledError("AWS glue support not installed: pip install 'pyiceberg[glue]'") from exc +def load_dynamodb(name: str, conf: Properties) -> Catalog: + try: + from pyiceberg.catalog.dynamodb import DynamoDbCatalog + + return DynamoDbCatalog(name, **conf) + except ImportError as exc: + raise NotInstalledError("AWS DynamoDB support not installed: pip install 'pyiceberg[dynamodb]'") from exc + + AVAILABLE_CATALOGS: dict[CatalogType, Callable[[str, Properties], Catalog]] = { CatalogType.REST: load_rest, CatalogType.HIVE: load_hive, CatalogType.GLUE: load_glue, + CatalogType.DYNAMODB: load_dynamodb, } @@ -104,6 +122,7 @@ def infer_catalog_type(name: str, catalog_properties: RecursiveDict) -> Optional """Tries to infer the type based on the dict Args: + name: Name of the catalog catalog_properties: Catalog properties Returns: @@ -149,6 +168,7 @@ def load_catalog(name: str, **properties: Optional[str]) -> Catalog: catalog_type: Optional[CatalogType] provided_catalog_type = conf.get(TYPE) + catalog_type = None if provided_catalog_type and isinstance(provided_catalog_type, str): catalog_type = CatalogType[provided_catalog_type.upper()] elif not provided_catalog_type: @@ -284,17 +304,6 @@ def drop_table(self, identifier: Union[str, Identifier]) -> None: NoSuchTableError: If a table with the name does not exist """ - @abstractmethod - def purge_table(self, identifier: Union[str, Identifier]) -> None: - """Drop a table and purge all data and metadata files. - - Args: - identifier (str | Identifier): Table identifier. - - Raises: - NoSuchTableError: If a table with the name does not exist - """ - @abstractmethod def rename_table(self, from_identifier: Union[str, Identifier], to_identifier: Union[str, Identifier]) -> Table: """Rename a fully classified table name @@ -431,3 +440,113 @@ def namespace_from(identifier: Union[str, Identifier]) -> Identifier: Identifier: Namespace identifier """ return Catalog.identifier_to_tuple(identifier)[:-1] + + @staticmethod + def _check_for_overlap(removals: Optional[Set[str]], updates: Properties) -> None: + if updates and removals: + overlap = set(removals) & set(updates.keys()) + if overlap: + raise ValueError(f"Updates and deletes have an overlap: {overlap}") + + def _resolve_table_location(self, location: Optional[str], database_name: str, table_name: str) -> str: + if not location: + return self._get_default_warehouse_location(database_name, table_name) + return location + + def _get_default_warehouse_location(self, database_name: str, table_name: str) -> str: + database_properties = self.load_namespace_properties(database_name) + if database_location := database_properties.get(LOCATION): + database_location = database_location.rstrip("/") + return f"{database_location}/{table_name}" + + if warehouse_path := self.properties.get(WAREHOUSE_LOCATION): + warehouse_path = warehouse_path.rstrip("/") + return f"{warehouse_path}/{database_name}.db/{table_name}" + + raise ValueError("No default path is set, please specify a location when creating a table") + + @staticmethod + def identifier_to_database( + identifier: Union[str, Identifier], err: Union[Type[ValueError], Type[NoSuchNamespaceError]] = ValueError + ) -> str: + tuple_identifier = Catalog.identifier_to_tuple(identifier) + if len(tuple_identifier) != 1: + raise err(f"Invalid database, hierarchical namespaces are not supported: {identifier}") + + return tuple_identifier[0] + + @staticmethod + def identifier_to_database_and_table( + identifier: Union[str, Identifier], + err: Union[Type[ValueError], Type[NoSuchTableError], Type[NoSuchNamespaceError]] = ValueError, + ) -> Tuple[str, str]: + tuple_identifier = Catalog.identifier_to_tuple(identifier) + if len(tuple_identifier) != 2: + raise err(f"Invalid path, hierarchical namespaces are not supported: {identifier}") + + return tuple_identifier[0], tuple_identifier[1] + + def purge_table(self, identifier: Union[str, Identifier]) -> None: + """Drop a table and purge all data and metadata files. + + Note: This method only logs warning rather than raise exception when encountering file deletion failure + + Args: + identifier (str | Identifier): Table identifier. + + Raises: + NoSuchTableError: If a table with the name does not exist, or the identifier is invalid + """ + table = self.load_table(identifier) + self.drop_table(identifier) + io = load_file_io(self.properties, table.metadata_location) + metadata = table.metadata + manifest_lists_to_delete = set() + manifests_to_delete = [] + for snapshot in metadata.snapshots: + manifests_to_delete += snapshot.manifests(io) + if snapshot.manifest_list is not None: + manifest_lists_to_delete.add(snapshot.manifest_list) + + manifest_paths_to_delete = {manifest.manifest_path for manifest in manifests_to_delete} + prev_metadata_files = {log.metadata_file for log in metadata.metadata_log} + + delete_data_files(io, manifests_to_delete) + delete_files(io, manifest_paths_to_delete, MANIFEST) + delete_files(io, manifest_lists_to_delete, MANIFEST_LIST) + delete_files(io, prev_metadata_files, PREVIOUS_METADATA) + delete_files(io, {table.metadata_location}, METADATA) + + @staticmethod + def _write_metadata(metadata: TableMetadata, io: FileIO, metadata_path: str) -> None: + ToOutputFile.table_metadata(metadata, io.new_output(metadata_path)) + + @staticmethod + def _get_metadata_location(location: str) -> str: + return f"{location}/metadata/00000-{uuid.uuid4()}.metadata.json" + + def _get_updated_props_and_update_summary( + self, current_properties: Properties, removals: Optional[Set[str]], updates: Properties + ) -> Tuple[PropertiesUpdateSummary, Properties]: + self._check_for_overlap(updates=updates, removals=removals) + updated_properties = dict(current_properties) + + removed: Set[str] = set() + updated: Set[str] = set() + + if removals: + for key in removals: + if key in updated_properties: + updated_properties.pop(key) + removed.add(key) + if updates: + for key, value in updates.items(): + updated_properties[key] = value + updated.add(key) + + expected_to_change = (removals or set()).difference(removed) + properties_update_summary = PropertiesUpdateSummary( + removed=list(removed or []), updated=list(updated or []), missing=list(expected_to_change) + ) + + return properties_update_summary, updated_properties diff --git a/pyiceberg/catalog/dynamodb.py b/pyiceberg/catalog/dynamodb.py new file mode 100644 index 0000000000..2c29d5bb1d --- /dev/null +++ b/pyiceberg/catalog/dynamodb.py @@ -0,0 +1,769 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +import uuid +from time import time +from typing import ( + Any, + Dict, + List, + Optional, + Set, + Union, +) + +import boto3 + +from pyiceberg.catalog import ( + ICEBERG, + METADATA_LOCATION, + PREVIOUS_METADATA_LOCATION, + TABLE_TYPE, + Catalog, + Identifier, + Properties, + PropertiesUpdateSummary, +) +from pyiceberg.exceptions import ( + ConditionalCheckFailedException, + GenericDynamoDbError, + NamespaceAlreadyExistsError, + NamespaceNotEmptyError, + NoSuchIcebergTableError, + NoSuchNamespaceError, + NoSuchPropertyException, + NoSuchTableError, + TableAlreadyExistsError, +) +from pyiceberg.io import load_file_io +from pyiceberg.partitioning import UNPARTITIONED_PARTITION_SPEC, PartitionSpec +from pyiceberg.schema import Schema +from pyiceberg.serializers import FromInputFile +from pyiceberg.table import Table +from pyiceberg.table.metadata import new_table_metadata +from pyiceberg.table.sorting import UNSORTED_SORT_ORDER, SortOrder +from pyiceberg.typedef import EMPTY_DICT + +DYNAMODB_CLIENT = "dynamodb" + +DYNAMODB_COL_IDENTIFIER = "identifier" +DYNAMODB_COL_NAMESPACE = "namespace" +DYNAMODB_COL_VERSION = "v" +DYNAMODB_COL_UPDATED_AT = "updated_at" +DYNAMODB_COL_CREATED_AT = "created_at" +DYNAMODB_NAMESPACE = "NAMESPACE" +DYNAMODB_NAMESPACE_GSI = "namespace-identifier" +DYNAMODB_PAY_PER_REQUEST = "PAY_PER_REQUEST" + +DYNAMODB_TABLE_NAME = "table-name" +DYNAMODB_TABLE_NAME_DEFAULT = "iceberg" + +PROPERTY_KEY_PREFIX = "p." + +ACTIVE = "ACTIVE" +ITEM = "Item" + + +class DynamoDbCatalog(Catalog): + def __init__(self, name: str, **properties: str): + super().__init__(name, **properties) + self.dynamodb = boto3.client(DYNAMODB_CLIENT) + self.dynamodb_table_name = self.properties.get(DYNAMODB_TABLE_NAME, DYNAMODB_TABLE_NAME_DEFAULT) + self._ensure_catalog_table_exists_or_create() + + def _ensure_catalog_table_exists_or_create(self) -> None: + if self._dynamodb_table_exists(): + return None + + try: + self.dynamodb.create_table( + TableName=self.dynamodb_table_name, + AttributeDefinitions=CREATE_CATALOG_ATTRIBUTE_DEFINITIONS, + KeySchema=CREATE_CATALOG_KEY_SCHEMA, + GlobalSecondaryIndexes=CREATE_CATALOG_GLOBAL_SECONDARY_INDEXES, + BillingMode=DYNAMODB_PAY_PER_REQUEST, + ) + except ( + self.dynamodb.exceptions.ResourceInUseException, + self.dynamodb.exceptions.LimitExceededException, + self.dynamodb.exceptions.InternalServerError, + ) as e: + raise GenericDynamoDbError(e.message) from e + + def _dynamodb_table_exists(self) -> bool: + try: + response = self.dynamodb.describe_table(TableName=self.dynamodb_table_name) + except self.dynamodb.exceptions.ResourceNotFoundException: + return False + except self.dynamodb.exceptions.InternalServerError as e: + raise GenericDynamoDbError(e.message) from e + + if response["Table"]["TableStatus"] != ACTIVE: + raise GenericDynamoDbError(f"DynamoDB table for catalog {self.dynamodb_table_name} is not {ACTIVE}") + else: + return True + + def create_table( + self, + identifier: Union[str, Identifier], + schema: Schema, + location: Optional[str] = None, + partition_spec: PartitionSpec = UNPARTITIONED_PARTITION_SPEC, + sort_order: SortOrder = UNSORTED_SORT_ORDER, + properties: Properties = EMPTY_DICT, + ) -> Table: + """ + Create an Iceberg table + + Args: + identifier: Table identifier. + schema: Table's schema. + location: Location for the table. Optional Argument. + partition_spec: PartitionSpec for the table. + sort_order: SortOrder for the table. + properties: Table properties that can be a string based dictionary. + + Returns: + Table: the created table instance + + Raises: + AlreadyExistsError: If a table with the name already exists + ValueError: If the identifier is invalid, or no path is given to store metadata + + """ + database_name, table_name = self.identifier_to_database_and_table(identifier) + + location = self._resolve_table_location(location, database_name, table_name) + metadata_location = self._get_metadata_location(location=location) + metadata = new_table_metadata( + location=location, schema=schema, partition_spec=partition_spec, sort_order=sort_order, properties=properties + ) + io = load_file_io(properties=self.properties, location=metadata_location) + self._write_metadata(metadata, io, metadata_location) + + self._ensure_namespace_exists(database_name=database_name) + + try: + self._put_dynamo_item( + item=_get_create_table_item( + database_name=database_name, table_name=table_name, properties=properties, metadata_location=metadata_location + ), + condition_expression=f"attribute_not_exists({DYNAMODB_COL_IDENTIFIER})", + ) + except ConditionalCheckFailedException as e: + raise TableAlreadyExistsError(f"Table {database_name}.{table_name} already exists") from e + + return self.load_table(identifier=identifier) + + def load_table(self, identifier: Union[str, Identifier]) -> Table: + """ + Loads the table's metadata and returns the table instance. + + You can also use this method to check for table existence using 'try catalog.table() except TableNotFoundError' + Note: This method doesn't scan data stored in the table. + + Args: + identifier: Table identifier. + + Returns: + Table: the table instance with its metadata + + Raises: + NoSuchTableError: If a table with the name does not exist, or the identifier is invalid + """ + database_name, table_name = self.identifier_to_database_and_table(identifier, NoSuchTableError) + dynamo_table_item = self._get_iceberg_table_item(database_name=database_name, table_name=table_name) + return self._convert_dynamo_table_item_to_iceberg_table(dynamo_table_item=dynamo_table_item) + + def drop_table(self, identifier: Union[str, Identifier]) -> None: + """Drop a table. + + Args: + identifier: Table identifier. + + Raises: + NoSuchTableError: If a table with the name does not exist, or the identifier is invalid + """ + database_name, table_name = self.identifier_to_database_and_table(identifier, NoSuchTableError) + + try: + self._delete_dynamo_item( + namespace=database_name, + identifier=f"{database_name}.{table_name}", + condition_expression=f"attribute_exists({DYNAMODB_COL_IDENTIFIER})", + ) + except ConditionalCheckFailedException as e: + raise NoSuchTableError(f"Table does not exist: {database_name}.{table_name}") from e + + def rename_table(self, from_identifier: Union[str, Identifier], to_identifier: Union[str, Identifier]) -> Table: + """Rename a fully classified table name + + This method can only rename Iceberg tables in AWS Glue + + Args: + from_identifier: Existing table identifier. + to_identifier: New table identifier. + + Returns: + Table: the updated table instance with its metadata + + Raises: + ValueError: When from table identifier is invalid + NoSuchTableError: When a table with the name does not exist + NoSuchIcebergTableError: When from table is not a valid iceberg table + NoSuchPropertyException: When from table miss some required properties + NoSuchNamespaceError: When the destination namespace doesn't exist + """ + from_database_name, from_table_name = self.identifier_to_database_and_table(from_identifier, NoSuchTableError) + to_database_name, to_table_name = self.identifier_to_database_and_table(to_identifier) + + from_table_item = self._get_iceberg_table_item(database_name=from_database_name, table_name=from_table_name) + + try: + # Verify that from_identifier is a valid iceberg table + self._convert_dynamo_table_item_to_iceberg_table(dynamo_table_item=from_table_item) + except NoSuchPropertyException as e: + raise NoSuchPropertyException( + f"Failed to rename table {from_database_name}.{from_table_name} since it is missing required properties" + ) from e + except NoSuchIcebergTableError as e: + raise NoSuchIcebergTableError( + f"Failed to rename table {from_database_name}.{from_table_name} since it is not a valid iceberg table" + ) from e + + self._ensure_namespace_exists(database_name=from_database_name) + self._ensure_namespace_exists(database_name=to_database_name) + + try: + self._put_dynamo_item( + item=_get_rename_table_item( + from_dynamo_table_item=from_table_item, to_database_name=to_database_name, to_table_name=to_table_name + ), + condition_expression=f"attribute_not_exists({DYNAMODB_COL_IDENTIFIER})", + ) + except ConditionalCheckFailedException as e: + raise TableAlreadyExistsError(f"Table {to_database_name}.{to_table_name} already exists") from e + + try: + self.drop_table(from_identifier) + except (NoSuchTableError, GenericDynamoDbError) as e: + log_message = f"Failed to drop old table {from_database_name}.{from_table_name}. " + + try: + self.drop_table(to_identifier) + log_message += f"Rolled back table creation for {to_database_name}.{to_table_name}." + except (NoSuchTableError, GenericDynamoDbError): + log_message += ( + f"Failed to roll back table creation for {to_database_name}.{to_table_name}. " f"Please clean up manually" + ) + + raise ValueError(log_message) from e + + return self.load_table(to_identifier) + + def create_namespace(self, namespace: Union[str, Identifier], properties: Properties = EMPTY_DICT) -> None: + """Create a namespace in the catalog. + + Args: + namespace: Namespace identifier + properties: A string dictionary of properties for the given namespace + + Raises: + ValueError: If the identifier is invalid + AlreadyExistsError: If a namespace with the given name already exists + """ + database_name = self.identifier_to_database(namespace) + + try: + self._put_dynamo_item( + item=_get_create_database_item(database_name=database_name, properties=properties), + condition_expression=f"attribute_not_exists({DYNAMODB_COL_NAMESPACE})", + ) + except ConditionalCheckFailedException as e: + raise NamespaceAlreadyExistsError(f"Database {database_name} already exists") from e + + def drop_namespace(self, namespace: Union[str, Identifier]) -> None: + """Drop a namespace. + + A Glue namespace can only be dropped if it is empty + + Args: + namespace: Namespace identifier + + Raises: + NoSuchNamespaceError: If a namespace with the given name does not exist, or the identifier is invalid + NamespaceNotEmptyError: If the namespace is not empty + """ + database_name = self.identifier_to_database(namespace, NoSuchNamespaceError) + table_identifiers = self.list_tables(namespace=database_name) + + if len(table_identifiers) > 0: + raise NamespaceNotEmptyError(f"Database {database_name} is not empty") + + try: + self._delete_dynamo_item( + namespace=database_name, + identifier=DYNAMODB_NAMESPACE, + condition_expression=f"attribute_exists({DYNAMODB_COL_IDENTIFIER})", + ) + except ConditionalCheckFailedException as e: + raise NoSuchNamespaceError(f"Database does not exist: {database_name}") from e + + def list_tables(self, namespace: Union[str, Identifier]) -> List[Identifier]: + """List tables under the given namespace in the catalog (including non-Iceberg tables) + + Args: + namespace (str | Identifier): Namespace identifier to search. + + Returns: + List[Identifier]: list of table identifiers. + """ + + database_name = self.identifier_to_database(namespace, NoSuchNamespaceError) + + paginator = self.dynamodb.get_paginator("query") + + try: + page_iterator = paginator.paginate( + TableName=self.dynamodb_table_name, + IndexName=DYNAMODB_NAMESPACE_GSI, + KeyConditionExpression=f"{DYNAMODB_COL_NAMESPACE} = :namespace ", + ExpressionAttributeValues={ + ":namespace": { + "S": database_name, + } + }, + ) + except ( + self.dynamodb.exceptions.ProvisionedThroughputExceededException, + self.dynamodb.exceptions.RequestLimitExceeded, + self.dynamodb.exceptions.InternalServerError, + self.dynamodb.exceptions.ResourceNotFoundException, + ) as e: + raise GenericDynamoDbError(e.message) from e + + table_identifiers = [] + for page in page_iterator: + for item in page["Items"]: + _dict = _convert_dynamo_item_to_regular_dict(item) + identifier_col = _dict[DYNAMODB_COL_IDENTIFIER] + if identifier_col == DYNAMODB_NAMESPACE: + continue + + table_identifiers.append(self.identifier_to_tuple(identifier_col)) + + return table_identifiers + + def list_namespaces(self, namespace: Union[str, Identifier] = ()) -> List[Identifier]: + """ + List top-level namespaces from the catalog. + We do not support hierarchical namespace. + + Returns: + List[Identifier]: a List of namespace identifiers + """ + + # Hierarchical namespace is not supported. Return an empty list + if namespace: + return [] + + paginator = self.dynamodb.get_paginator("query") + + try: + page_iterator = paginator.paginate( + TableName=self.dynamodb_table_name, + ConsistentRead=True, + KeyConditionExpression=f"{DYNAMODB_COL_IDENTIFIER} = :identifier", + ExpressionAttributeValues={ + ":identifier": { + "S": DYNAMODB_NAMESPACE, + } + }, + ) + except ( + self.dynamodb.exceptions.ProvisionedThroughputExceededException, + self.dynamodb.exceptions.RequestLimitExceeded, + self.dynamodb.exceptions.InternalServerError, + self.dynamodb.exceptions.ResourceNotFoundException, + ) as e: + raise GenericDynamoDbError(e.message) from e + + database_identifiers = [] + for page in page_iterator: + for item in page["Items"]: + _dict = _convert_dynamo_item_to_regular_dict(item) + namespace_col = _dict[DYNAMODB_COL_NAMESPACE] + database_identifiers.append(self.identifier_to_tuple(namespace_col)) + + return database_identifiers + + def load_namespace_properties(self, namespace: Union[str, Identifier]) -> Properties: + """ + Get properties for a namespace. + + Args: + namespace: Namespace identifier + + Returns: + Properties: Properties for the given namespace + + Raises: + NoSuchNamespaceError: If a namespace with the given name does not exist, or identifier is invalid + """ + database_name = self.identifier_to_database(namespace, NoSuchNamespaceError) + namespace_item = self._get_iceberg_namespace_item(database_name=database_name) + namespace_dict = _convert_dynamo_item_to_regular_dict(namespace_item) + return _get_namespace_properties(namespace_dict=namespace_dict) + + def update_namespace_properties( + self, namespace: Union[str, Identifier], removals: Optional[Set[str]] = None, updates: Properties = EMPTY_DICT + ) -> PropertiesUpdateSummary: + """ + Removes or updates provided property keys for a namespace. + + Args: + namespace: Namespace identifier + removals: Set of property keys that need to be removed. Optional Argument. + updates: Properties to be updated for the given namespace. + + Raises: + NoSuchNamespaceError: If a namespace with the given name does not exist， or identifier is invalid + ValueError: If removals and updates have overlapping keys. + """ + + database_name = self.identifier_to_database(namespace, NoSuchNamespaceError) + namespace_item = self._get_iceberg_namespace_item(database_name=database_name) + namespace_dict = _convert_dynamo_item_to_regular_dict(namespace_item) + current_properties = _get_namespace_properties(namespace_dict=namespace_dict) + + properties_update_summary, updated_properties = self._get_updated_props_and_update_summary( + current_properties=current_properties, removals=removals, updates=updates + ) + + try: + self._put_dynamo_item( + item=_get_update_database_item( + namespace_item=namespace_item, + updated_properties=updated_properties, + ), + condition_expression=f"attribute_exists({DYNAMODB_COL_NAMESPACE})", + ) + except ConditionalCheckFailedException as e: + raise NoSuchNamespaceError(f"Database {database_name} does not exist") from e + + return properties_update_summary + + def _get_iceberg_table_item(self, database_name: str, table_name: str) -> Dict[str, Any]: + try: + return self._get_dynamo_item(identifier=f"{database_name}.{table_name}", namespace=database_name) + except ValueError as e: + raise NoSuchTableError(f"Table does not exist: {database_name}.{table_name}") from e + + def _get_iceberg_namespace_item(self, database_name: str) -> Dict[str, Any]: + try: + return self._get_dynamo_item(identifier=DYNAMODB_NAMESPACE, namespace=database_name) + except ValueError as e: + raise NoSuchNamespaceError(f"Namespace does not exist: {database_name}") from e + + def _ensure_namespace_exists(self, database_name: str) -> Dict[str, Any]: + return self._get_iceberg_namespace_item(database_name) + + def _get_dynamo_item(self, identifier: str, namespace: str) -> Dict[str, Any]: + try: + response = self.dynamodb.get_item( + TableName=self.dynamodb_table_name, + ConsistentRead=True, + Key={ + DYNAMODB_COL_IDENTIFIER: { + "S": identifier, + }, + DYNAMODB_COL_NAMESPACE: { + "S": namespace, + }, + }, + ) + if ITEM in response: + return response[ITEM] + else: + raise ValueError(f"Item not found. identifier: {identifier} - namespace: {namespace}") + except self.dynamodb.exceptions.ResourceNotFoundException as e: + raise ValueError(f"Item not found. identifier: {identifier} - namespace: {namespace}") from e + except ( + self.dynamodb.exceptions.ProvisionedThroughputExceededException, + self.dynamodb.exceptions.RequestLimitExceeded, + self.dynamodb.exceptions.InternalServerError, + ) as e: + raise GenericDynamoDbError(e.message) from e + + def _put_dynamo_item(self, item: Dict[str, Any], condition_expression: str) -> None: + try: + self.dynamodb.put_item(TableName=self.dynamodb_table_name, Item=item, ConditionExpression=condition_expression) + except self.dynamodb.exceptions.ConditionalCheckFailedException as e: + raise ConditionalCheckFailedException(f"Condition expression check failed: {condition_expression} - {item}") from e + except ( + self.dynamodb.exceptions.ProvisionedThroughputExceededException, + self.dynamodb.exceptions.RequestLimitExceeded, + self.dynamodb.exceptions.InternalServerError, + self.dynamodb.exceptions.ResourceNotFoundException, + self.dynamodb.exceptions.ItemCollectionSizeLimitExceededException, + self.dynamodb.exceptions.TransactionConflictException, + ) as e: + raise GenericDynamoDbError(e.message) from e + + def _delete_dynamo_item(self, namespace: str, identifier: str, condition_expression: str) -> None: + try: + self.dynamodb.delete_item( + TableName=self.dynamodb_table_name, + Key={ + DYNAMODB_COL_IDENTIFIER: { + "S": identifier, + }, + DYNAMODB_COL_NAMESPACE: { + "S": namespace, + }, + }, + ConditionExpression=condition_expression, + ) + except self.dynamodb.exceptions.ConditionalCheckFailedException as e: + raise ConditionalCheckFailedException( + f"Condition expression check failed: {condition_expression} - {identifier}" + ) from e + except ( + self.dynamodb.exceptions.ProvisionedThroughputExceededException, + self.dynamodb.exceptions.RequestLimitExceeded, + self.dynamodb.exceptions.InternalServerError, + self.dynamodb.exceptions.ResourceNotFoundException, + self.dynamodb.exceptions.ItemCollectionSizeLimitExceededException, + self.dynamodb.exceptions.TransactionConflictException, + ) as e: + raise GenericDynamoDbError(e.message) from e + + def _convert_dynamo_table_item_to_iceberg_table(self, dynamo_table_item: Dict[str, Any]) -> Table: + table_dict = _convert_dynamo_item_to_regular_dict(dynamo_table_item) + + for prop in [_add_property_prefix(prop) for prop in (TABLE_TYPE, METADATA_LOCATION)] + [ + DYNAMODB_COL_IDENTIFIER, + DYNAMODB_COL_NAMESPACE, + DYNAMODB_COL_CREATED_AT, + ]: + if prop not in table_dict.keys(): + raise NoSuchPropertyException(f"Iceberg required property {prop} is missing: {dynamo_table_item}") + + table_type = table_dict[_add_property_prefix(TABLE_TYPE)] + identifier = table_dict[DYNAMODB_COL_IDENTIFIER] + metadata_location = table_dict[_add_property_prefix(METADATA_LOCATION)] + database_name, table_name = self.identifier_to_database_and_table(identifier, NoSuchTableError) + + if table_type.lower() != ICEBERG: + raise NoSuchIcebergTableError( + f"Property table_type is {table_type}, expected {ICEBERG}: " f"{database_name}.{table_name}" + ) + + io = load_file_io(properties=self.properties, location=metadata_location) + file = io.new_input(metadata_location) + metadata = FromInputFile.table_metadata(file) + return Table( + identifier=(self.name, database_name, table_name), + metadata=metadata, + metadata_location=metadata_location, + io=self._load_file_io(metadata.properties), + ) + + +def _get_create_table_item(database_name: str, table_name: str, properties: Properties, metadata_location: str) -> Dict[str, Any]: + current_timestamp_ms = str(round(time() * 1000)) + _dict = { + DYNAMODB_COL_IDENTIFIER: { + "S": f"{database_name}.{table_name}", + }, + DYNAMODB_COL_NAMESPACE: { + "S": database_name, + }, + DYNAMODB_COL_VERSION: { + "S": str(uuid.uuid4()), + }, + DYNAMODB_COL_CREATED_AT: { + "N": current_timestamp_ms, + }, + DYNAMODB_COL_UPDATED_AT: { + "N": current_timestamp_ms, + }, + } + + for key, val in properties.items(): + _dict[_add_property_prefix(key)] = {"S": val} + + _dict[_add_property_prefix(TABLE_TYPE)] = {"S": ICEBERG.upper()} + _dict[_add_property_prefix(METADATA_LOCATION)] = {"S": metadata_location} + _dict[_add_property_prefix(PREVIOUS_METADATA_LOCATION)] = {"S": ""} + + return _dict + + +def _get_rename_table_item(from_dynamo_table_item: Dict[str, Any], to_database_name: str, to_table_name: str) -> Dict[str, Any]: + _dict = from_dynamo_table_item + current_timestamp_ms = str(round(time() * 1000)) + _dict[DYNAMODB_COL_IDENTIFIER]["S"] = f"{to_database_name}.{to_table_name}" + _dict[DYNAMODB_COL_NAMESPACE]["S"] = to_database_name + _dict[DYNAMODB_COL_VERSION]["S"] = str(uuid.uuid4()) + _dict[DYNAMODB_COL_UPDATED_AT]["N"] = current_timestamp_ms + return _dict + + +def _get_create_database_item(database_name: str, properties: Properties) -> Dict[str, Any]: + current_timestamp_ms = str(round(time() * 1000)) + _dict = { + DYNAMODB_COL_IDENTIFIER: { + "S": DYNAMODB_NAMESPACE, + }, + DYNAMODB_COL_NAMESPACE: { + "S": database_name, + }, + DYNAMODB_COL_VERSION: { + "S": str(uuid.uuid4()), + }, + DYNAMODB_COL_CREATED_AT: { + "N": current_timestamp_ms, + }, + DYNAMODB_COL_UPDATED_AT: { + "N": current_timestamp_ms, + }, + } + + for key, val in properties.items(): + _dict[_add_property_prefix(key)] = {"S": val} + + return _dict + + +def _get_update_database_item(namespace_item: Dict[str, Any], updated_properties: Properties) -> Dict[str, Any]: + current_timestamp_ms = str(round(time() * 1000)) + + _dict = { + DYNAMODB_COL_IDENTIFIER: namespace_item[DYNAMODB_COL_IDENTIFIER], + DYNAMODB_COL_NAMESPACE: namespace_item[DYNAMODB_COL_NAMESPACE], + DYNAMODB_COL_VERSION: { + "S": str(uuid.uuid4()), + }, + DYNAMODB_COL_CREATED_AT: namespace_item[DYNAMODB_COL_CREATED_AT], + DYNAMODB_COL_UPDATED_AT: { + "N": current_timestamp_ms, + }, + } + + for key, val in updated_properties.items(): + _dict[_add_property_prefix(key)] = {"S": val} + + return _dict + + +CREATE_CATALOG_ATTRIBUTE_DEFINITIONS = [ + { + "AttributeName": DYNAMODB_COL_IDENTIFIER, + "AttributeType": "S", + }, + { + "AttributeName": DYNAMODB_COL_NAMESPACE, + "AttributeType": "S", + }, +] + +CREATE_CATALOG_KEY_SCHEMA = [ + { + "AttributeName": DYNAMODB_COL_IDENTIFIER, + "KeyType": "HASH", + }, + { + "AttributeName": DYNAMODB_COL_NAMESPACE, + "KeyType": "RANGE", + }, +] + + +CREATE_CATALOG_GLOBAL_SECONDARY_INDEXES = [ + { + "IndexName": DYNAMODB_NAMESPACE_GSI, + "KeySchema": [ + { + "AttributeName": DYNAMODB_COL_NAMESPACE, + "KeyType": "HASH", + }, + { + "AttributeName": DYNAMODB_COL_IDENTIFIER, + "KeyType": "RANGE", + }, + ], + "Projection": { + "ProjectionType": "KEYS_ONLY", + }, + } +] + + +def _get_namespace_properties(namespace_dict: Dict[str, str]) -> Properties: + return {_remove_property_prefix(key): val for key, val in namespace_dict.items() if key.startswith(PROPERTY_KEY_PREFIX)} + + +def _convert_dynamo_item_to_regular_dict(dynamo_json: Dict[str, Any]) -> Dict[str, str]: + """ + Converts a dynamo json to a regular json. Example of a dynamo json: + { + "AlbumTitle": { + "S": "Songs About Life", + }, + "Artist": { + "S": "Acme Band", + }, + "SongTitle": { + "S": "Happy Day", + } + } + + Converted to regular json: + { + "AlbumTitle": "Songs About Life", + "Artist": "Acme Band", + "SongTitle": "Happy Day" + } + + Only "S" and "N" data types are supported since those are the only ones that Iceberg is utilizing. + """ + + regular_json = {} + for column_name, val_dict in dynamo_json.items(): + keys = list(val_dict.keys()) + + if len(keys) != 1: + raise ValueError(f"Expecting only 1 key: {keys}") + + data_type = keys[0] + if data_type not in ("S", "N"): + raise ValueError("Only S and N data types are supported.") + + values = list(val_dict.values()) + assert len(values) == 1 + column_value = values[0] + regular_json[column_name] = column_value + + return regular_json + + +def _add_property_prefix(prop: str) -> str: + return PROPERTY_KEY_PREFIX + prop + + +def _remove_property_prefix(prop: str) -> str: + return prop.lstrip(PROPERTY_KEY_PREFIX) diff --git a/pyiceberg/catalog/glue.py b/pyiceberg/catalog/glue.py index 273a8c613d..763ad6a283 100644 --- a/pyiceberg/catalog/glue.py +++ b/pyiceberg/catalog/glue.py @@ -16,35 +16,27 @@ # under the License. -import uuid from typing import ( Any, Dict, List, Optional, Set, - Tuple, - Type, Union, ) import boto3 from pyiceberg.catalog import ( + EXTERNAL_TABLE, ICEBERG, - MANIFEST, - MANIFEST_LIST, - METADATA, + LOCATION, METADATA_LOCATION, - PREVIOUS_METADATA, TABLE_TYPE, - WAREHOUSE_LOCATION, Catalog, Identifier, Properties, PropertiesUpdateSummary, - delete_data_files, - delete_files, ) from pyiceberg.exceptions import ( NamespaceAlreadyExistsError, @@ -55,16 +47,15 @@ NoSuchTableError, TableAlreadyExistsError, ) -from pyiceberg.io import FileIO, load_file_io +from pyiceberg.io import load_file_io from pyiceberg.partitioning import UNPARTITIONED_PARTITION_SPEC, PartitionSpec from pyiceberg.schema import Schema -from pyiceberg.serializers import FromInputFile, ToOutputFile +from pyiceberg.serializers import FromInputFile from pyiceberg.table import Table -from pyiceberg.table.metadata import TableMetadata, new_table_metadata +from pyiceberg.table.metadata import new_table_metadata from pyiceberg.table.sorting import UNSORTED_SORT_ORDER, SortOrder from pyiceberg.typedef import EMPTY_DICT -EXTERNAL_TABLE_TYPE = "EXTERNAL_TABLE" GLUE_CLIENT = "glue" PROP_GLUE_TABLE = "Table" @@ -88,17 +79,16 @@ PROP_GLUE_NEXT_TOKEN = "NextToken" GLUE_DESCRIPTION_KEY = "comment" -GLUE_DATABASE_LOCATION_KEY = "location" def _construct_parameters(metadata_location: str) -> Properties: return {TABLE_TYPE: ICEBERG.upper(), METADATA_LOCATION: metadata_location} -def _construct_table_input(table_name: str, metadata_location: str, properties: Properties) -> Dict[str, Any]: +def _construct_create_table_input(table_name: str, metadata_location: str, properties: Properties) -> Dict[str, Any]: table_input = { PROP_GLUE_TABLE_NAME: table_name, - PROP_GLUE_TABLE_TYPE: EXTERNAL_TABLE_TYPE, + PROP_GLUE_TABLE_TYPE: EXTERNAL_TABLE, PROP_GLUE_TABLE_PARAMETERS: _construct_parameters(metadata_location), } @@ -108,13 +98,29 @@ def _construct_table_input(table_name: str, metadata_location: str, properties: return table_input +def _construct_rename_table_input(to_table_name: str, glue_table: Dict[str, Any]) -> Dict[str, Any]: + rename_table_input = {PROP_GLUE_TABLE_NAME: to_table_name} + # use the same Glue info to create the new table, pointing to the old metadata + if table_type := glue_table.get(PROP_GLUE_TABLE_TYPE): + rename_table_input[PROP_GLUE_TABLE_TYPE] = table_type + if table_parameters := glue_table.get(PROP_GLUE_TABLE_PARAMETERS): + rename_table_input[PROP_GLUE_TABLE_PARAMETERS] = table_parameters + if table_owner := glue_table.get(PROP_GLUE_TABLE_OWNER): + rename_table_input[PROP_GLUE_TABLE_OWNER] = table_owner + if table_storage_descriptor := glue_table.get(PROP_GLUE_TABLE_STORAGE_DESCRIPTOR): + rename_table_input[PROP_GLUE_TABLE_STORAGE_DESCRIPTOR] = table_storage_descriptor + if table_description := glue_table.get(PROP_GLUE_TABLE_DESCRIPTION): + rename_table_input[PROP_GLUE_TABLE_DESCRIPTION] = table_description + return rename_table_input + + def _construct_database_input(database_name: str, properties: Properties) -> Dict[str, Any]: database_input: Dict[str, Any] = {PROP_GLUE_DATABASE_NAME: database_name} parameters = {} for k, v in properties.items(): if k == GLUE_DESCRIPTION_KEY: database_input[PROP_GLUE_DATABASE_DESCRIPTION] = v - elif k == GLUE_DATABASE_LOCATION_KEY: + elif k == LOCATION: database_input[PROP_GLUE_DATABASE_LOCATION] = v else: parameters[k] = v @@ -122,32 +128,7 @@ def _construct_database_input(database_name: str, properties: Properties) -> Dic return database_input -def _write_metadata(metadata: TableMetadata, io: FileIO, metadate_path: str) -> None: - ToOutputFile.table_metadata(metadata, io.new_output(metadate_path)) - - class GlueCatalog(Catalog): - @staticmethod - def identifier_to_database( - identifier: Union[str, Identifier], err: Union[Type[ValueError], Type[NoSuchNamespaceError]] = ValueError - ) -> str: - tuple_identifier = Catalog.identifier_to_tuple(identifier) - if len(tuple_identifier) != 1: - raise err(f"Invalid database, hierarchical namespaces are not supported: {identifier}") - - return tuple_identifier[0] - - @staticmethod - def identifier_to_database_and_table( - identifier: Union[str, Identifier], - err: Union[Type[ValueError], Type[NoSuchTableError], Type[NoSuchNamespaceError]] = ValueError, - ) -> Tuple[str, str]: - tuple_identifier = Catalog.identifier_to_tuple(identifier) - if len(tuple_identifier) != 2: - raise err(f"Invalid path, hierarchical namespaces are not supported: {identifier}") - - return tuple_identifier[0], tuple_identifier[1] - def __init__(self, name: str, **properties: str): super().__init__(name, **properties) self.glue = boto3.client(GLUE_CLIENT) @@ -185,25 +166,7 @@ def _convert_glue_to_iceberg(self, glue_table: Dict[str, Any]) -> Table: io=self._load_file_io(metadata.properties), ) - def _default_warehouse_location(self, database_name: str, table_name: str) -> str: - database_properties = self.load_namespace_properties(database_name) - if database_location := database_properties.get(GLUE_DATABASE_LOCATION_KEY): - database_location = database_location.rstrip("/") - return f"{database_location}/{table_name}" - - if warehouse_path := self.properties.get(WAREHOUSE_LOCATION): - warehouse_path = warehouse_path.rstrip("/") - return f"{warehouse_path}/{database_name}.db/{table_name}" - - raise ValueError("No default path is set, please specify a location when creating a table") - - def _resolve_table_location(self, location: Optional[str], database_name: str, table_name: str) -> str: - if not location: - return self._default_warehouse_location(database_name, table_name) - return location - - def _create_glue_table(self, identifier: Union[str, Identifier], table_input: Dict[str, Any]) -> None: - database_name, table_name = self.identifier_to_database_and_table(identifier) + def _create_glue_table(self, database_name: str, table_name: str, table_input: Dict[str, Any]) -> None: try: self.glue.create_table(DatabaseName=database_name, TableInput=table_input) except self.glue.exceptions.AlreadyExistsException as e: @@ -220,7 +183,8 @@ def create_table( sort_order: SortOrder = UNSORTED_SORT_ORDER, properties: Properties = EMPTY_DICT, ) -> Table: - """Create an Iceberg table in Glue catalog + """ + Create an Iceberg table Args: identifier: Table identifier. @@ -241,18 +205,18 @@ def create_table( database_name, table_name = self.identifier_to_database_and_table(identifier) location = self._resolve_table_location(location, database_name, table_name) - metadata_location = f"{location}/metadata/00000-{uuid.uuid4()}.metadata.json" + metadata_location = self._get_metadata_location(location=location) metadata = new_table_metadata( location=location, schema=schema, partition_spec=partition_spec, sort_order=sort_order, properties=properties ) io = load_file_io(properties=self.properties, location=metadata_location) - _write_metadata(metadata, io, metadata_location) + self._write_metadata(metadata, io, metadata_location) - self._create_glue_table( - identifier=identifier, table_input=_construct_table_input(table_name, metadata_location, properties) - ) - loaded_table = self.load_table(identifier=(database_name, table_name)) - return loaded_table + table_input = _construct_create_table_input(table_name, metadata_location, properties) + database_name, table_name = self.identifier_to_database_and_table(identifier) + self._create_glue_table(database_name=database_name, table_name=table_name, table_input=table_input) + + return self.load_table(identifier=identifier) def load_table(self, identifier: Union[str, Identifier]) -> Table: """Loads the table's metadata and returns the table instance. @@ -273,7 +237,7 @@ def load_table(self, identifier: Union[str, Identifier]) -> Table: try: load_table_response = self.glue.get_table(DatabaseName=database_name, Name=table_name) except self.glue.exceptions.EntityNotFoundException as e: - raise NoSuchTableError(f"Table does not exists: {database_name}.{table_name}") from e + raise NoSuchTableError(f"Table does not exist: {database_name}.{table_name}") from e return self._convert_glue_to_iceberg(load_table_response.get(PROP_GLUE_TABLE, {})) @@ -290,38 +254,7 @@ def drop_table(self, identifier: Union[str, Identifier]) -> None: try: self.glue.delete_table(DatabaseName=database_name, Name=table_name) except self.glue.exceptions.EntityNotFoundException as e: - raise NoSuchTableError(f"Table does not exists: {database_name}.{table_name}") from e - - def purge_table(self, identifier: Union[str, Identifier]) -> None: - """Drop a table and purge all data and metadata files. - - Note: This method only logs warning rather than raise exception when encountering file deletion failure - - Args: - identifier (str | Identifier): Table identifier. - - Raises: - NoSuchTableError: If a table with the name does not exist, or the identifier is invalid - """ - table = self.load_table(identifier) - self.drop_table(identifier) - io = load_file_io(self.properties, table.metadata_location) - metadata = table.metadata - manifest_lists_to_delete = set() - manifests_to_delete = [] - for snapshot in metadata.snapshots: - manifests_to_delete += snapshot.manifests(io) - if snapshot.manifest_list is not None: - manifest_lists_to_delete.add(snapshot.manifest_list) - - manifest_paths_to_delete = {manifest.manifest_path for manifest in manifests_to_delete} - prev_metadata_files = {log.metadata_file for log in metadata.metadata_log} - - delete_data_files(io, manifests_to_delete) - delete_files(io, manifest_paths_to_delete, MANIFEST) - delete_files(io, manifest_lists_to_delete, MANIFEST_LIST) - delete_files(io, prev_metadata_files, PREVIOUS_METADATA) - delete_files(io, {table.metadata_location}, METADATA) + raise NoSuchTableError(f"Table does not exist: {database_name}.{table_name}") from e def rename_table(self, from_identifier: Union[str, Identifier], to_identifier: Union[str, Identifier]) -> Table: """Rename a fully classified table name @@ -336,10 +269,10 @@ def rename_table(self, from_identifier: Union[str, Identifier], to_identifier: U Table: the updated table instance with its metadata Raises: - ValueError: When the from table identifier is invalid + ValueError: When from table identifier is invalid NoSuchTableError: When a table with the name does not exist - NoSuchIcebergTableError: When the from table is not a valid iceberg table - NoSuchPropertyException: When the from table miss some required properties + NoSuchIcebergTableError: When from table is not a valid iceberg table + NoSuchPropertyException: When from table miss some required properties NoSuchNamespaceError: When the destination namespace doesn't exist """ from_database_name, from_table_name = self.identifier_to_database_and_table(from_identifier, NoSuchTableError) @@ -347,44 +280,40 @@ def rename_table(self, from_identifier: Union[str, Identifier], to_identifier: U try: get_table_response = self.glue.get_table(DatabaseName=from_database_name, Name=from_table_name) except self.glue.exceptions.EntityNotFoundException as e: - raise NoSuchTableError(f"Table does not exists: {from_database_name}.{from_table_name}") from e + raise NoSuchTableError(f"Table does not exist: {from_database_name}.{from_table_name}") from e glue_table = get_table_response[PROP_GLUE_TABLE] try: # verify that from_identifier is a valid iceberg table - self._convert_glue_to_iceberg(glue_table) + self._convert_glue_to_iceberg(glue_table=glue_table) except NoSuchPropertyException as e: raise NoSuchPropertyException( - f"Failed to rename table {from_database_name}.{from_table_name} since it miss required properties" + f"Failed to rename table {from_database_name}.{from_table_name} since it is missing required properties" ) from e except NoSuchIcebergTableError as e: raise NoSuchIcebergTableError( f"Failed to rename table {from_database_name}.{from_table_name} since it is not a valid iceberg table" ) from e - new_table_input = {PROP_GLUE_TABLE_NAME: to_table_name} - # use the same Glue info to create the new table, pointing to the old metadata - if table_type := glue_table.get(PROP_GLUE_TABLE_TYPE): - new_table_input[PROP_GLUE_TABLE_TYPE] = table_type - if table_parameters := glue_table.get(PROP_GLUE_TABLE_PARAMETERS): - new_table_input[PROP_GLUE_TABLE_PARAMETERS] = table_parameters - if table_owner := glue_table.get(PROP_GLUE_TABLE_OWNER): - new_table_input[PROP_GLUE_TABLE_OWNER] = table_owner - if table_storage_descriptor := glue_table.get(PROP_GLUE_TABLE_STORAGE_DESCRIPTOR): - new_table_input[PROP_GLUE_TABLE_STORAGE_DESCRIPTOR] = table_storage_descriptor - if table_description := glue_table.get(PROP_GLUE_TABLE_DESCRIPTION): - new_table_input[PROP_GLUE_TABLE_DESCRIPTION] = table_description - - self._create_glue_table(identifier=to_identifier, table_input=new_table_input) + rename_table_input = _construct_rename_table_input(to_table_name=to_table_name, glue_table=glue_table) + self._create_glue_table(database_name=to_database_name, table_name=to_table_name, table_input=rename_table_input) + try: self.drop_table(from_identifier) except Exception as e: - self.drop_table(to_identifier) - raise ValueError( - f"Fail to drop old table {from_database_name}.{from_table_name}, " - f"after renaming to {to_database_name}.{to_table_name} roll back to use the old one" - ) from e + log_message = f"Failed to drop old table {from_database_name}.{from_table_name}. " + + try: + self.drop_table(to_identifier) + log_message += f"Rolled back table creation for {to_database_name}.{to_table_name}." + except NoSuchTableError: + log_message += ( + f"Failed to roll back table creation for {to_database_name}.{to_table_name}. " f"Please clean up manually" + ) + + raise ValueError(log_message) from e + return self.load_table(to_identifier) def create_namespace(self, namespace: Union[str, Identifier], properties: Properties = EMPTY_DICT) -> None: @@ -420,7 +349,7 @@ def drop_namespace(self, namespace: Union[str, Identifier]) -> None: try: table_list = self.list_tables(namespace=database_name) except NoSuchNamespaceError as e: - raise NoSuchNamespaceError(f"Database does not exists: {database_name}") from e + raise NoSuchNamespaceError(f"Database does not exist: {database_name}") from e if len(table_list) > 0: raise NamespaceNotEmptyError(f"Database {database_name} is not empty") @@ -451,7 +380,7 @@ def list_tables(self, namespace: Union[str, Identifier]) -> List[Identifier]: next_token = table_list_response.get(PROP_GLUE_NEXT_TOKEN) table_list += table_list_response.get(PROP_GLUE_TABLELIST, []) except self.glue.exceptions.EntityNotFoundException as e: - raise NoSuchNamespaceError(f"Database does not exists: {database_name}") from e + raise NoSuchNamespaceError(f"Database does not exist: {database_name}") from e return [(database_name, table.get(PROP_GLUE_TABLE_NAME)) for table in table_list] def list_namespaces(self, namespace: Union[str, Identifier] = ()) -> List[Identifier]: @@ -460,9 +389,10 @@ def list_namespaces(self, namespace: Union[str, Identifier] = ()) -> List[Identi Returns: List[Identifier]: a List of namespace identifiers """ - # Glue does not support hierarchical namespace, therefore return an empty list + # Hierarchical namespace is not supported. Return an empty list if namespace: return [] + database_list = [] databases_response = self.glue.get_databases() next_token = databases_response.get(PROP_GLUE_NEXT_TOKEN) @@ -489,7 +419,7 @@ def load_namespace_properties(self, namespace: Union[str, Identifier]) -> Proper try: database_response = self.glue.get_database(Name=database_name) except self.glue.exceptions.EntityNotFoundException as e: - raise NoSuchNamespaceError(f"Database does not exists: {database_name}") from e + raise NoSuchNamespaceError(f"Database does not exist: {database_name}") from e except self.glue.exceptions.InvalidInputException as e: raise NoSuchNamespaceError(f"Invalid input for namespace {database_name}") from e @@ -499,7 +429,7 @@ def load_namespace_properties(self, namespace: Union[str, Identifier]) -> Proper properties = dict(database[PROP_GLUE_DATABASE_PARAMETERS]) if database_location := database.get(PROP_GLUE_DATABASE_LOCATION): - properties[GLUE_DATABASE_LOCATION_KEY] = database_location + properties[LOCATION] = database_location if database_description := database.get(PROP_GLUE_DATABASE_DESCRIPTION): properties[GLUE_DESCRIPTION_KEY] = database_description @@ -519,31 +449,13 @@ def update_namespace_properties( NoSuchNamespaceError: If a namespace with the given name does not exist， or identifier is invalid ValueError: If removals and updates have overlapping keys. """ - removed: Set[str] = set() - updated: Set[str] = set() - - if updates and removals: - overlap = set(removals) & set(updates.keys()) - if overlap: - raise ValueError(f"Updates and deletes have an overlap: {overlap}") - database_name = self.identifier_to_database(namespace, NoSuchNamespaceError) - current_properties = self.load_namespace_properties(namespace=database_name) - new_properties = dict(current_properties) - if removals: - for key in removals: - if key in new_properties: - new_properties.pop(key) - removed.add(key) - if updates: - for key, value in updates.items(): - new_properties[key] = value - updated.add(key) - - self.glue.update_database(Name=database_name, DatabaseInput=_construct_database_input(database_name, new_properties)) + current_properties = self.load_namespace_properties(namespace=namespace) + properties_update_summary, updated_properties = self._get_updated_props_and_update_summary( + current_properties=current_properties, removals=removals, updates=updates + ) - expected_to_change = (removals or set()).difference(removed) + database_name = self.identifier_to_database(namespace, NoSuchNamespaceError) + self.glue.update_database(Name=database_name, DatabaseInput=_construct_database_input(database_name, updated_properties)) - return PropertiesUpdateSummary( - removed=list(removed or []), updated=list(updates.keys() if updates else []), missing=list(expected_to_change) - ) + return properties_update_summary diff --git a/pyiceberg/catalog/hive.py b/pyiceberg/catalog/hive.py index 2387e3dc37..b49967caa8 100644 --- a/pyiceberg/catalog/hive.py +++ b/pyiceberg/catalog/hive.py @@ -16,7 +16,6 @@ # under the License. import getpass import time -import uuid from types import TracebackType from typing import ( Any, @@ -24,7 +23,6 @@ List, Optional, Set, - Tuple, Type, Union, ) @@ -46,10 +44,11 @@ from thrift.transport import TSocket, TTransport from pyiceberg.catalog import ( + EXTERNAL_TABLE, ICEBERG, + LOCATION, METADATA_LOCATION, TABLE_TYPE, - WAREHOUSE_LOCATION, Catalog, Identifier, Properties, @@ -66,9 +65,9 @@ from pyiceberg.io import FileIO, load_file_io from pyiceberg.partitioning import UNPARTITIONED_PARTITION_SPEC, PartitionSpec from pyiceberg.schema import Schema, SchemaVisitor, visit -from pyiceberg.serializers import FromInputFile, ToOutputFile +from pyiceberg.serializers import FromInputFile from pyiceberg.table import Table -from pyiceberg.table.metadata import TableMetadata, new_table_metadata +from pyiceberg.table.metadata import new_table_metadata from pyiceberg.table.sorting import UNSORTED_SORT_ORDER, SortOrder from pyiceberg.typedef import EMPTY_DICT from pyiceberg.types import ( @@ -110,7 +109,6 @@ COMMENT = "comment" OWNER = "owner" -LOCATION = "location" class _HiveClient: @@ -218,27 +216,6 @@ def primitive(self, primitive: PrimitiveType) -> str: class HiveCatalog(Catalog): _client: _HiveClient - @staticmethod - def identifier_to_database( - identifier: Union[str, Identifier], err: Union[Type[ValueError], Type[NoSuchNamespaceError]] = ValueError - ) -> str: - tuple_identifier = Catalog.identifier_to_tuple(identifier) - if len(tuple_identifier) != 1: - raise err(f"Invalid database, hierarchical namespaces are not supported: {identifier}") - - return tuple_identifier[0] - - @staticmethod - def identifier_to_database_and_table( - identifier: Union[str, Identifier], - err: Union[Type[ValueError], Type[NoSuchTableError], Type[NoSuchNamespaceError]] = ValueError, - ) -> Tuple[str, str]: - tuple_identifier = Catalog.identifier_to_tuple(identifier) - if len(tuple_identifier) != 2: - raise err(f"Invalid path, hierarchical namespaces are not supported: {identifier}") - - return tuple_identifier[0], tuple_identifier[1] - def __init__(self, name: str, **properties: str): super().__init__(name, **properties) self._client = _HiveClient(properties["uri"]) @@ -268,22 +245,6 @@ def _convert_hive_into_iceberg(self, table: HiveTable, io: FileIO) -> Table: io=self._load_file_io(metadata.properties), ) - def _write_metadata(self, metadata: TableMetadata, io: FileIO, metadata_path: str) -> None: - ToOutputFile.table_metadata(metadata, io.new_output(metadata_path)) - - def _resolve_table_location(self, location: Optional[str], database_name: str, table_name: str) -> str: - if not location: - database_properties = self.load_namespace_properties(database_name) - if database_location := database_properties.get(LOCATION): - database_location = database_location.rstrip("/") - return f"{database_location}/{table_name}" - - if warehouse_location := self.properties.get(WAREHOUSE_LOCATION): - warehouse_location = warehouse_location.rstrip("/") - return f"{warehouse_location}/{database_name}/{table_name}" - raise ValueError("Cannot determine location from warehouse, please provide an explicit location") - return location - def create_table( self, identifier: Union[str, Identifier], @@ -315,7 +276,7 @@ def create_table( location = self._resolve_table_location(location, database_name, table_name) - metadata_location = f"{location}/metadata/00000-{uuid.uuid4()}.metadata.json" + metadata_location = self._get_metadata_location(location=location) metadata = new_table_metadata( location=location, schema=schema, partition_spec=partition_spec, sort_order=sort_order, properties=properties ) @@ -329,7 +290,7 @@ def create_table( createTime=current_time_millis // 1000, lastAccessTime=current_time_millis // 1000, sd=_construct_hive_storage_descriptor(schema, location), - tableType="EXTERNAL_TABLE", + tableType=EXTERNAL_TABLE, parameters=_construct_parameters(metadata_location), ) try: @@ -380,7 +341,7 @@ def drop_table(self, identifier: Union[str, Identifier]) -> None: with self._client as open_client: open_client.drop_table(dbname=database_name, name=table_name, deleteData=False) except NoSuchObjectException as e: - # When the namespace doesn't exists, it throws the same error + # When the namespace doesn't exist, it throws the same error raise NoSuchTableError(f"Table does not exists: {table_name}") from e def purge_table(self, identifier: Union[str, Identifier]) -> None: @@ -398,9 +359,9 @@ def rename_table(self, from_identifier: Union[str, Identifier], to_identifier: U Table: the updated table instance with its metadata Raises: - ValueError: When the from table identifier is invalid + ValueError: When from table identifier is invalid NoSuchTableError: When a table with the name does not exist - NoSuchNamespaceError: When the destination namespace doesn't exists + NoSuchNamespaceError: When the destination namespace doesn't exist """ from_database_name, from_table_name = self.identifier_to_database_and_table(from_identifier, NoSuchTableError) to_database_name, to_table_name = self.identifier_to_database_and_table(to_identifier) @@ -479,7 +440,7 @@ def list_namespaces(self, namespace: Union[str, Identifier] = ()) -> List[Identi Returns: List[Identifier]: a List of namespace identifiers """ - # Hive does not support hierarchical namespaces, therefore return an empty list + # Hierarchical namespace is not supported. Return an empty list if namespace: return [] @@ -524,14 +485,8 @@ def update_namespace_properties( NoSuchNamespaceError: If a namespace with the given name does not exist ValueError: If removals and updates have overlapping keys. """ - removed: Set[str] = set() - updated: Set[str] = set() - - if updates and removals: - overlap = set(removals) & set(updates.keys()) - if overlap: - raise ValueError(f"Updates and deletes have an overlap: {overlap}") + self._check_for_overlap(updates=updates, removals=removals) database_name = self.identifier_to_database(namespace, NoSuchNamespaceError) with self._client as open_client: try: @@ -539,6 +494,10 @@ def update_namespace_properties( parameters = database.parameters except NoSuchObjectException as e: raise NoSuchNamespaceError(f"Database does not exists: {database_name}") from e + + removed: Set[str] = set() + updated: Set[str] = set() + if removals: for key in removals: if key in parameters: @@ -548,10 +507,9 @@ def update_namespace_properties( for key, value in updates.items(): parameters[key] = value updated.add(key) + open_client.alter_database(database_name, _annotate_namespace(database, parameters)) expected_to_change = (removals or set()).difference(removed) - return PropertiesUpdateSummary( - removed=list(removed or []), updated=list(updates.keys() if updates else []), missing=list(expected_to_change) - ) + return PropertiesUpdateSummary(removed=list(removed or []), updated=list(updated or []), missing=list(expected_to_change)) diff --git a/pyiceberg/exceptions.py b/pyiceberg/exceptions.py index 69e40159ce..813b34c3ba 100644 --- a/pyiceberg/exceptions.py +++ b/pyiceberg/exceptions.py @@ -90,3 +90,15 @@ class SignError(Exception): class ResolveError(Exception): pass + + +class DynamoDbError(Exception): + pass + + +class ConditionalCheckFailedException(DynamoDbError): + pass + + +class GenericDynamoDbError(DynamoDbError): + pass diff --git a/pyiceberg/table/__init__.py b/pyiceberg/table/__init__.py index 0bce2a4e86..7c53fd0967 100644 --- a/pyiceberg/table/__init__.py +++ b/pyiceberg/table/__init__.py @@ -53,7 +53,6 @@ ) from pyiceberg.partitioning import PartitionSpec from pyiceberg.schema import Schema -from pyiceberg.serializers import FromInputFile from pyiceberg.table.metadata import TableMetadata from pyiceberg.table.snapshots import Snapshot, SnapshotLogEntry from pyiceberg.table.sorting import SortOrder @@ -184,6 +183,9 @@ def refresh(self) -> Table: def from_metadata(cls, metadata_location: str, properties: Properties = EMPTY_DICT) -> StaticTable: io = load_file_io(properties=properties, location=metadata_location) file = io.new_input(metadata_location) + + from pyiceberg.serializers import FromInputFile + metadata = FromInputFile.table_metadata(file) return cls( diff --git a/pyproject.toml b/pyproject.toml index 735df4c5d0..92bbb1266c 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -105,6 +105,7 @@ hive = ["thrift"] s3fs = ["s3fs"] glue = ["boto3"] adlfs = ["adlfs"] +dynamodb = ["boto3"] [tool.pytest.ini_options] markers = [ diff --git a/tests/catalog/integration_test_dynamodb.py b/tests/catalog/integration_test_dynamodb.py new file mode 100644 index 0000000000..a430175c5a --- /dev/null +++ b/tests/catalog/integration_test_dynamodb.py @@ -0,0 +1,255 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +from typing import Generator, List + +import boto3 +import pytest +from botocore.exceptions import ClientError + +from pyiceberg.catalog import Catalog +from pyiceberg.catalog.dynamodb import DynamoDbCatalog +from pyiceberg.exceptions import ( + NamespaceAlreadyExistsError, + NamespaceNotEmptyError, + NoSuchNamespaceError, + NoSuchTableError, + TableAlreadyExistsError, +) +from pyiceberg.schema import Schema +from tests.conftest import clean_up, get_bucket_name, get_s3_path + +# The number of tables/databases used in list_table/namespace test +LIST_TEST_NUMBER = 2 + + +@pytest.fixture(name="dynamodb", scope="module") +def fixture_dynamodb_client() -> boto3.client: + yield boto3.client("dynamodb") + + +@pytest.fixture(name="test_catalog", scope="module") +def fixture_test_catalog() -> Generator[Catalog, None, None]: + """The pre- and post-setting of aws integration test""" + test_catalog = DynamoDbCatalog("test_ddb_catalog", warehouse=get_s3_path(get_bucket_name())) + yield test_catalog + clean_up(test_catalog) + + +def test_create_table( + test_catalog: Catalog, s3: boto3.client, table_schema_nested: Schema, database_name: str, table_name: str +) -> None: + identifier = (database_name, table_name) + test_catalog.create_namespace(database_name) + test_catalog.create_table(identifier, table_schema_nested, get_s3_path(get_bucket_name(), database_name, table_name)) + table = test_catalog.load_table(identifier) + assert table.identifier == (test_catalog.name,) + identifier + metadata_location = table.metadata_location.split(get_bucket_name())[1][1:] + s3.head_object(Bucket=get_bucket_name(), Key=metadata_location) + + +def test_create_table_with_invalid_location(table_schema_nested: Schema, database_name: str, table_name: str) -> None: + identifier = (database_name, table_name) + test_catalog_no_warehouse = DynamoDbCatalog("test_ddb_catalog") + test_catalog_no_warehouse.create_namespace(database_name) + with pytest.raises(ValueError): + test_catalog_no_warehouse.create_table(identifier, table_schema_nested) + test_catalog_no_warehouse.drop_namespace(database_name) + + +def test_create_table_with_default_location( + test_catalog: Catalog, s3: boto3.client, table_schema_nested: Schema, database_name: str, table_name: str +) -> None: + identifier = (database_name, table_name) + test_catalog.create_namespace(database_name) + test_catalog.create_table(identifier, table_schema_nested) + table = test_catalog.load_table(identifier) + assert table.identifier == (test_catalog.name,) + identifier + metadata_location = table.metadata_location.split(get_bucket_name())[1][1:] + s3.head_object(Bucket=get_bucket_name(), Key=metadata_location) + + +def test_create_table_with_invalid_database(test_catalog: Catalog, table_schema_nested: Schema, table_name: str) -> None: + identifier = ("invalid", table_name) + with pytest.raises(NoSuchNamespaceError): + test_catalog.create_table(identifier, table_schema_nested) + + +def test_create_duplicated_table(test_catalog: Catalog, table_schema_nested: Schema, database_name: str, table_name: str) -> None: + test_catalog.create_namespace(database_name) + test_catalog.create_table((database_name, table_name), table_schema_nested) + with pytest.raises(TableAlreadyExistsError): + test_catalog.create_table((database_name, table_name), table_schema_nested) + + +def test_load_table(test_catalog: Catalog, table_schema_nested: Schema, database_name: str, table_name: str) -> None: + identifier = (database_name, table_name) + test_catalog.create_namespace(database_name) + table = test_catalog.create_table(identifier, table_schema_nested) + loaded_table = test_catalog.load_table(identifier) + assert table.identifier == loaded_table.identifier + assert table.metadata_location == loaded_table.metadata_location + assert table.metadata == loaded_table.metadata + + +def test_list_tables(test_catalog: Catalog, table_schema_nested: Schema, database_name: str, table_list: List[str]) -> None: + test_catalog.create_namespace(database_name) + for table_name in table_list: + test_catalog.create_table((database_name, table_name), table_schema_nested) + identifier_list = test_catalog.list_tables(database_name) + assert len(identifier_list) == LIST_TEST_NUMBER + for table_name in table_list: + assert (database_name, table_name) in identifier_list + + +def test_rename_table( + test_catalog: Catalog, s3: boto3.client, table_schema_nested: Schema, table_name: str, database_name: str +) -> None: + new_database_name = f"{database_name}_new" + test_catalog.create_namespace(database_name) + test_catalog.create_namespace(new_database_name) + new_table_name = f"rename-{table_name}" + identifier = (database_name, table_name) + table = test_catalog.create_table(identifier, table_schema_nested) + assert table.identifier == (test_catalog.name,) + identifier + new_identifier = (new_database_name, new_table_name) + test_catalog.rename_table(identifier, new_identifier) + new_table = test_catalog.load_table(new_identifier) + assert new_table.identifier == (test_catalog.name,) + new_identifier + assert new_table.metadata_location == table.metadata_location + metadata_location = new_table.metadata_location.split(get_bucket_name())[1][1:] + s3.head_object(Bucket=get_bucket_name(), Key=metadata_location) + with pytest.raises(NoSuchTableError): + test_catalog.load_table(identifier) + + +def test_drop_table(test_catalog: Catalog, table_schema_nested: Schema, table_name: str, database_name: str) -> None: + identifier = (database_name, table_name) + test_catalog.create_namespace(database_name) + table = test_catalog.create_table(identifier, table_schema_nested) + assert table.identifier == (test_catalog.name,) + identifier + test_catalog.drop_table(identifier) + with pytest.raises(NoSuchTableError): + test_catalog.load_table(identifier) + + +def test_purge_table( + test_catalog: Catalog, s3: boto3.client, table_schema_nested: Schema, table_name: str, database_name: str +) -> None: + identifier = (database_name, table_name) + test_catalog.create_namespace(database_name) + test_catalog.create_table(identifier, table_schema_nested) + table = test_catalog.load_table(identifier) + assert table.identifier == (test_catalog.name,) + identifier + metadata_location = table.metadata_location.split(get_bucket_name())[1][1:] + s3.head_object(Bucket=get_bucket_name(), Key=metadata_location) + test_catalog.purge_table(identifier) + with pytest.raises(NoSuchTableError): + test_catalog.load_table(identifier) + with pytest.raises(ClientError): + s3.head_object(Bucket=get_bucket_name(), Key=metadata_location) + + +def test_create_namespace(test_catalog: Catalog, database_name: str) -> None: + test_catalog.create_namespace(database_name) + assert (database_name,) in test_catalog.list_namespaces() + + +def test_create_duplicate_namespace(test_catalog: Catalog, database_name: str) -> None: + test_catalog.create_namespace(database_name) + with pytest.raises(NamespaceAlreadyExistsError): + test_catalog.create_namespace(database_name) + + +def test_create_namespace_with_comment_and_location(test_catalog: Catalog, database_name: str) -> None: + test_location = get_s3_path(get_bucket_name(), database_name) + test_properties = { + "comment": "this is a test description", + "location": test_location, + } + test_catalog.create_namespace(namespace=database_name, properties=test_properties) + loaded_database_list = test_catalog.list_namespaces() + assert (database_name,) in loaded_database_list + properties = test_catalog.load_namespace_properties(database_name) + assert properties["comment"] == "this is a test description" + assert properties["location"] == test_location + + +def test_list_namespaces(test_catalog: Catalog, database_list: List[str]) -> None: + for database_name in database_list: + test_catalog.create_namespace(database_name) + db_list = test_catalog.list_namespaces() + for database_name in database_list: + assert (database_name,) in db_list + assert len(test_catalog.list_namespaces(list(database_list)[0])) == 0 + + +def test_drop_namespace(test_catalog: Catalog, table_schema_nested: Schema, table_name: str, database_name: str) -> None: + test_catalog.create_namespace(database_name) + assert (database_name,) in test_catalog.list_namespaces() + test_catalog.create_table((database_name, table_name), table_schema_nested) + with pytest.raises(NamespaceNotEmptyError): + test_catalog.drop_namespace(database_name) + test_catalog.drop_table((database_name, table_name)) + test_catalog.drop_namespace(database_name) + assert (database_name,) not in test_catalog.list_namespaces() + + +def test_load_namespace_properties(test_catalog: Catalog, database_name: str) -> None: + warehouse_location = get_s3_path(get_bucket_name()) + test_properties = { + "comment": "this is a test description", + "location": f"{warehouse_location}/{database_name}.db", + "test_property1": "1", + "test_property2": "2", + "test_property3": "3", + } + + test_catalog.create_namespace(database_name, test_properties) + listed_properties = test_catalog.load_namespace_properties(database_name) + for k, v in listed_properties.items(): + assert k in test_properties + assert v == test_properties[k] + + +def test_load_empty_namespace_properties(test_catalog: Catalog, database_name: str) -> None: + test_catalog.create_namespace(database_name) + listed_properties = test_catalog.load_namespace_properties(database_name) + assert listed_properties == {} + + +def test_update_namespace_properties(test_catalog: Catalog, database_name: str) -> None: + warehouse_location = get_s3_path(get_bucket_name()) + test_properties = { + "comment": "this is a test description", + "location": f"{warehouse_location}/{database_name}.db", + "test_property1": "1", + "test_property2": "2", + "test_property3": "3", + } + removals = {"test_property1", "test_property2", "test_property3", "should_not_removed"} + updates = {"test_property4": "4", "test_property5": "5", "comment": "updated test description"} + test_catalog.create_namespace(database_name, test_properties) + update_report = test_catalog.update_namespace_properties(database_name, removals, updates) + for k in updates.keys(): + assert k in update_report.updated + for k in removals: + if k == "should_not_removed": + assert k in update_report.missing + else: + assert k in update_report.removed + assert "updated test description" == test_catalog.load_namespace_properties(database_name)["comment"] diff --git a/tests/catalog/integration_test_glue.py b/tests/catalog/integration_test_glue.py index be4cd2e826..62870390b3 100644 --- a/tests/catalog/integration_test_glue.py +++ b/tests/catalog/integration_test_glue.py @@ -15,8 +15,7 @@ # specific language governing permissions and limitations # under the License. -import os -from typing import Generator, List, Optional +from typing import Generator, List import boto3 import pytest @@ -32,51 +31,17 @@ TableAlreadyExistsError, ) from pyiceberg.schema import Schema +from tests.conftest import clean_up, get_bucket_name, get_s3_path # The number of tables/databases used in list_table/namespace test LIST_TEST_NUMBER = 2 -def get_bucket_name() -> str: - """ - Set the environment variable AWS_TEST_BUCKET for a default bucket to test - """ - bucket_name = os.getenv("AWS_TEST_BUCKET") - if bucket_name is None: - raise ValueError("Please specify a bucket to run the test by setting environment variable AWS_TEST_BUCKET") - return bucket_name - - -def get_s3_path(bucket_name: str, database_name: Optional[str] = None, table_name: Optional[str] = None) -> str: - result_path = f"s3://{bucket_name}" - if database_name is not None: - result_path += f"/{database_name}.db" - - if table_name is not None: - result_path += f"/{table_name}" - return result_path - - -@pytest.fixture(name="s3", scope="module") -def fixture_s3_client() -> boto3.client: - yield boto3.client("s3") - - @pytest.fixture(name="glue", scope="module") def fixture_glue_client() -> boto3.client: yield boto3.client("glue") -def clean_up(test_catalog: Catalog) -> None: - """Clean all databases and tables created during the integration test""" - for database_tuple in test_catalog.list_namespaces(): - database_name = database_tuple[0] - if "my_iceberg_database-" in database_name: - for identifier in test_catalog.list_tables(database_name): - test_catalog.purge_table(identifier) - test_catalog.drop_namespace(database_name) - - @pytest.fixture(name="test_catalog", scope="module") def fixture_test_catalog() -> Generator[Catalog, None, None]: """The pre- and post-setting of aws integration test""" diff --git a/tests/catalog/test_dynamodb.py b/tests/catalog/test_dynamodb.py new file mode 100644 index 0000000000..e2d9b67de9 --- /dev/null +++ b/tests/catalog/test_dynamodb.py @@ -0,0 +1,452 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +from typing import List + +import pytest +from moto import mock_dynamodb + +from pyiceberg.catalog import METADATA_LOCATION, TABLE_TYPE +from pyiceberg.catalog.dynamodb import ( + DYNAMODB_COL_CREATED_AT, + DYNAMODB_COL_IDENTIFIER, + DYNAMODB_COL_NAMESPACE, + DYNAMODB_TABLE_NAME_DEFAULT, + DynamoDbCatalog, + _add_property_prefix, +) +from pyiceberg.exceptions import ( + NamespaceAlreadyExistsError, + NamespaceNotEmptyError, + NoSuchIcebergTableError, + NoSuchNamespaceError, + NoSuchPropertyException, + NoSuchTableError, + TableAlreadyExistsError, +) +from pyiceberg.schema import Schema +from tests.conftest import BUCKET_NAME, TABLE_METADATA_LOCATION_REGEX + + +@mock_dynamodb +def test_create_dynamodb_catalog_with_table_name(_dynamodb, _bucket_initialize: None, _patch_aiobotocore: None) -> None: # type: ignore + DynamoDbCatalog("test_ddb_catalog") + response = _dynamodb.describe_table(TableName=DYNAMODB_TABLE_NAME_DEFAULT) + assert response["Table"]["TableName"] == DYNAMODB_TABLE_NAME_DEFAULT + + custom_table_name = "custom_table_name" + DynamoDbCatalog("test_ddb_catalog", **{"table-name": custom_table_name}) + response = _dynamodb.describe_table(TableName=custom_table_name) + assert response["Table"]["TableName"] == custom_table_name + + +@mock_dynamodb +def test_create_table_with_database_location( + _bucket_initialize: None, _patch_aiobotocore: None, table_schema_nested: Schema, database_name: str, table_name: str +) -> None: + catalog_name = "test_ddb_catalog" + identifier = (database_name, table_name) + test_catalog = DynamoDbCatalog(catalog_name) + test_catalog.create_namespace(namespace=database_name, properties={"location": f"s3://{BUCKET_NAME}/{database_name}.db"}) + table = test_catalog.create_table(identifier, table_schema_nested) + assert table.identifier == (catalog_name,) + identifier + assert TABLE_METADATA_LOCATION_REGEX.match(table.metadata_location) + + +@mock_dynamodb +def test_create_table_with_default_warehouse( + _bucket_initialize: None, _patch_aiobotocore: None, table_schema_nested: Schema, database_name: str, table_name: str +) -> None: + catalog_name = "test_ddb_catalog" + identifier = (database_name, table_name) + test_catalog = DynamoDbCatalog(catalog_name, warehouse=f"s3://{BUCKET_NAME}") + test_catalog.create_namespace(namespace=database_name) + table = test_catalog.create_table(identifier, table_schema_nested) + assert table.identifier == (catalog_name,) + identifier + assert TABLE_METADATA_LOCATION_REGEX.match(table.metadata_location) + + +@mock_dynamodb +def test_create_table_with_given_location( + _bucket_initialize: None, _patch_aiobotocore: None, table_schema_nested: Schema, database_name: str, table_name: str +) -> None: + catalog_name = "test_ddb_catalog" + identifier = (database_name, table_name) + test_catalog = DynamoDbCatalog(catalog_name) + test_catalog.create_namespace(namespace=database_name) + table = test_catalog.create_table( + identifier=identifier, schema=table_schema_nested, location=f"s3://{BUCKET_NAME}/{database_name}.db/{table_name}" + ) + assert table.identifier == (catalog_name,) + identifier + assert TABLE_METADATA_LOCATION_REGEX.match(table.metadata_location) + + +@mock_dynamodb +def test_create_table_with_no_location( + _bucket_initialize: None, _patch_aiobotocore: None, table_schema_nested: Schema, database_name: str, table_name: str +) -> None: + identifier = (database_name, table_name) + test_catalog = DynamoDbCatalog("test_ddb_catalog") + test_catalog.create_namespace(namespace=database_name) + with pytest.raises(ValueError): + test_catalog.create_table(identifier=identifier, schema=table_schema_nested) + + +@mock_dynamodb +def test_create_table_with_strips( + _bucket_initialize: None, _patch_aiobotocore: None, table_schema_nested: Schema, database_name: str, table_name: str +) -> None: + catalog_name = "test_ddb_catalog" + identifier = (database_name, table_name) + test_catalog = DynamoDbCatalog(catalog_name) + test_catalog.create_namespace(namespace=database_name, properties={"location": f"s3://{BUCKET_NAME}/{database_name}.db/"}) + table = test_catalog.create_table(identifier, table_schema_nested) + assert table.identifier == (catalog_name,) + identifier + assert TABLE_METADATA_LOCATION_REGEX.match(table.metadata_location) + + +@mock_dynamodb +def test_create_table_with_strips_bucket_root( + _bucket_initialize: None, _patch_aiobotocore: None, table_schema_nested: Schema, database_name: str, table_name: str +) -> None: + catalog_name = "test_ddb_catalog" + identifier = (database_name, table_name) + test_catalog = DynamoDbCatalog(catalog_name, warehouse=f"s3://{BUCKET_NAME}/") + test_catalog.create_namespace(namespace=database_name) + table_strip = test_catalog.create_table(identifier, table_schema_nested) + assert table_strip.identifier == (catalog_name,) + identifier + assert TABLE_METADATA_LOCATION_REGEX.match(table_strip.metadata_location) + + +@mock_dynamodb +def test_create_table_with_no_database( + _bucket_initialize: None, _patch_aiobotocore: None, table_schema_nested: Schema, database_name: str, table_name: str +) -> None: + identifier = (database_name, table_name) + test_catalog = DynamoDbCatalog("test_ddb_catalog") + with pytest.raises(NoSuchNamespaceError): + test_catalog.create_table(identifier=identifier, schema=table_schema_nested) + + +@mock_dynamodb +def test_create_duplicated_table( + _bucket_initialize: None, _patch_aiobotocore: None, table_schema_nested: Schema, database_name: str, table_name: str +) -> None: + identifier = (database_name, table_name) + test_catalog = DynamoDbCatalog("test_ddb_catalog", warehouse=f"s3://{BUCKET_NAME}") + test_catalog.create_namespace(namespace=database_name) + test_catalog.create_table(identifier, table_schema_nested) + with pytest.raises(TableAlreadyExistsError): + test_catalog.create_table(identifier, table_schema_nested) + + +@mock_dynamodb +def test_load_table( + _bucket_initialize: None, _patch_aiobotocore: None, table_schema_nested: Schema, database_name: str, table_name: str +) -> None: + catalog_name = "test_ddb_catalog" + identifier = (database_name, table_name) + test_catalog = DynamoDbCatalog(catalog_name, warehouse=f"s3://{BUCKET_NAME}") + test_catalog.create_namespace(namespace=database_name) + test_catalog.create_table(identifier, table_schema_nested) + table = test_catalog.load_table(identifier) + assert table.identifier == (catalog_name,) + identifier + assert TABLE_METADATA_LOCATION_REGEX.match(table.metadata_location) + + +@mock_dynamodb +def test_load_non_exist_table(_bucket_initialize: None, _patch_aiobotocore: None, database_name: str, table_name: str) -> None: + identifier = (database_name, table_name) + test_catalog = DynamoDbCatalog("test_ddb_catalog", warehouse=f"s3://{BUCKET_NAME}") + test_catalog.create_namespace(namespace=database_name) + with pytest.raises(NoSuchTableError): + test_catalog.load_table(identifier) + + +@mock_dynamodb +def test_drop_table( + _bucket_initialize: None, _patch_aiobotocore: None, table_schema_nested: Schema, database_name: str, table_name: str +) -> None: + catalog_name = "test_ddb_catalog" + identifier = (database_name, table_name) + test_catalog = DynamoDbCatalog(catalog_name, warehouse=f"s3://{BUCKET_NAME}") + test_catalog.create_namespace(namespace=database_name) + test_catalog.create_table(identifier, table_schema_nested) + table = test_catalog.load_table(identifier) + assert table.identifier == (catalog_name,) + identifier + assert TABLE_METADATA_LOCATION_REGEX.match(table.metadata_location) + test_catalog.drop_table(identifier) + with pytest.raises(NoSuchTableError): + test_catalog.load_table(identifier) + + +@mock_dynamodb +def test_drop_non_exist_table(_bucket_initialize: None, _patch_aiobotocore: None, database_name: str, table_name: str) -> None: + identifier = (database_name, table_name) + test_catalog = DynamoDbCatalog("test_ddb_catalog", warehouse=f"s3://{BUCKET_NAME}") + with pytest.raises(NoSuchTableError): + test_catalog.drop_table(identifier) + + +@mock_dynamodb +def test_rename_table( + _bucket_initialize: None, _patch_aiobotocore: None, table_schema_nested: Schema, database_name: str, table_name: str +) -> None: + catalog_name = "test_ddb_catalog" + new_table_name = f"{table_name}_new" + identifier = (database_name, table_name) + new_identifier = (database_name, new_table_name) + test_catalog = DynamoDbCatalog(catalog_name, warehouse=f"s3://{BUCKET_NAME}") + test_catalog.create_namespace(namespace=database_name) + table = test_catalog.create_table(identifier, table_schema_nested) + assert table.identifier == (catalog_name,) + identifier + assert TABLE_METADATA_LOCATION_REGEX.match(table.metadata_location) + test_catalog.rename_table(identifier, new_identifier) + new_table = test_catalog.load_table(new_identifier) + assert new_table.identifier == (catalog_name,) + new_identifier + # the metadata_location should not change + assert new_table.metadata_location == table.metadata_location + # old table should be dropped + with pytest.raises(NoSuchTableError): + test_catalog.load_table(identifier) + + +@mock_dynamodb +def test_fail_on_rename_table_with_missing_required_params( + _bucket_initialize: None, _patch_aiobotocore: None, database_name: str, table_name: str +) -> None: + new_database_name = f"{database_name}_new" + new_table_name = f"{table_name}_new" + identifier = (database_name, table_name) + new_identifier = (new_database_name, new_table_name) + test_catalog = DynamoDbCatalog("test_ddb_catalog", warehouse=f"s3://{BUCKET_NAME}") + test_catalog.create_namespace(namespace=database_name) + test_catalog.create_namespace(namespace=new_database_name) + + # Missing required params + # pylint: disable=W0212 + test_catalog._put_dynamo_item( + item={ + DYNAMODB_COL_IDENTIFIER: {"S": f"{database_name}.{table_name}"}, + DYNAMODB_COL_NAMESPACE: {"S": database_name}, + }, + condition_expression=f"attribute_not_exists({DYNAMODB_COL_IDENTIFIER})", + ) + + with pytest.raises(NoSuchPropertyException): + test_catalog.rename_table(identifier, new_identifier) + + +@mock_dynamodb +def test_fail_on_rename_non_iceberg_table(_dynamodb, _bucket_initialize: None, _patch_aiobotocore: None, database_name: str, table_name: str) -> None: # type: ignore + new_database_name = f"{database_name}_new" + new_table_name = f"{table_name}_new" + identifier = (database_name, table_name) + new_identifier = (new_database_name, new_table_name) + test_catalog = DynamoDbCatalog("test_ddb_catalog", warehouse=f"s3://{BUCKET_NAME}") + test_catalog.create_namespace(namespace=database_name) + test_catalog.create_namespace(namespace=new_database_name) + + # Wrong TABLE_TYPE param + # pylint: disable=W0212 + test_catalog._put_dynamo_item( + item={ + DYNAMODB_COL_IDENTIFIER: {"S": f"{database_name}.{table_name}"}, + DYNAMODB_COL_NAMESPACE: {"S": database_name}, + DYNAMODB_COL_CREATED_AT: {"S": "test-1873287263487623"}, + _add_property_prefix(TABLE_TYPE): {"S": "non-iceberg-table-type"}, + _add_property_prefix(METADATA_LOCATION): {"S": "test-metadata-location"}, + }, + condition_expression=f"attribute_not_exists({DYNAMODB_COL_IDENTIFIER})", + ) + + with pytest.raises(NoSuchIcebergTableError): + test_catalog.rename_table(identifier, new_identifier) + + +@mock_dynamodb +def test_list_tables( + _bucket_initialize: None, _patch_aiobotocore: None, table_schema_nested: Schema, database_name: str, table_list: List[str] +) -> None: + test_catalog = DynamoDbCatalog("test_ddb_catalog", warehouse=f"s3://{BUCKET_NAME}") + test_catalog.create_namespace(namespace=database_name) + for table_name in table_list: + test_catalog.create_table((database_name, table_name), table_schema_nested) + loaded_table_list = test_catalog.list_tables(database_name) + for table_name in table_list: + assert (database_name, table_name) in loaded_table_list + + +@mock_dynamodb +def test_list_namespaces(_bucket_initialize: None, _patch_aiobotocore: None, database_list: List[str]) -> None: + test_catalog = DynamoDbCatalog("test_ddb_catalog") + for database_name in database_list: + test_catalog.create_namespace(namespace=database_name) + loaded_database_list = test_catalog.list_namespaces() + for database_name in database_list: + assert (database_name,) in loaded_database_list + + +@mock_dynamodb +def test_create_namespace_no_properties(_bucket_initialize: None, _patch_aiobotocore: None, database_name: str) -> None: + test_catalog = DynamoDbCatalog("test_ddb_catalog") + test_catalog.create_namespace(namespace=database_name) + loaded_database_list = test_catalog.list_namespaces() + assert len(loaded_database_list) == 1 + assert (database_name,) in loaded_database_list + properties = test_catalog.load_namespace_properties(database_name) + assert properties == {} + + +@mock_dynamodb +def test_create_namespace_with_comment_and_location( + _bucket_initialize: None, _patch_aiobotocore: None, database_name: str +) -> None: + test_location = f"s3://{BUCKET_NAME}/{database_name}.db" + test_properties = { + "comment": "this is a test description", + "location": test_location, + } + test_catalog = DynamoDbCatalog("test_ddb_catalog") + test_catalog.create_namespace(namespace=database_name, properties=test_properties) + loaded_database_list = test_catalog.list_namespaces() + assert len(loaded_database_list) == 1 + assert (database_name,) in loaded_database_list + properties = test_catalog.load_namespace_properties(database_name) + assert properties["comment"] == "this is a test description" + assert properties["location"] == test_location + + +@mock_dynamodb +def test_create_duplicated_namespace(_bucket_initialize: None, _patch_aiobotocore: None, database_name: str) -> None: + test_catalog = DynamoDbCatalog("test_ddb_catalog") + test_catalog.create_namespace(namespace=database_name) + loaded_database_list = test_catalog.list_namespaces() + assert len(loaded_database_list) == 1 + assert (database_name,) in loaded_database_list + with pytest.raises(NamespaceAlreadyExistsError): + test_catalog.create_namespace(namespace=database_name, properties={"test": "test"}) + + +@mock_dynamodb +def test_drop_namespace(_bucket_initialize: None, _patch_aiobotocore: None, database_name: str) -> None: + test_catalog = DynamoDbCatalog("test_ddb_catalog") + test_catalog.create_namespace(namespace=database_name) + loaded_database_list = test_catalog.list_namespaces() + assert len(loaded_database_list) == 1 + assert (database_name,) in loaded_database_list + test_catalog.drop_namespace(database_name) + loaded_database_list = test_catalog.list_namespaces() + assert len(loaded_database_list) == 0 + + +@mock_dynamodb +def test_drop_non_empty_namespace( + _bucket_initialize: None, _patch_aiobotocore: None, table_schema_nested: Schema, database_name: str, table_name: str +) -> None: + identifier = (database_name, table_name) + test_catalog = DynamoDbCatalog("test_ddb_catalog", warehouse=f"s3://{BUCKET_NAME}") + test_catalog.create_namespace(namespace=database_name) + test_catalog.create_table(identifier, table_schema_nested) + assert len(test_catalog.list_tables(database_name)) == 1 + with pytest.raises(NamespaceNotEmptyError): + test_catalog.drop_namespace(database_name) + + +@mock_dynamodb +def test_drop_non_exist_namespace(_bucket_initialize: None, _patch_aiobotocore: None, database_name: str) -> None: + test_catalog = DynamoDbCatalog("test_ddb_catalog") + with pytest.raises(NoSuchNamespaceError): + test_catalog.drop_namespace(database_name) + + +@mock_dynamodb +def test_load_namespace_properties(_bucket_initialize: None, _patch_aiobotocore: None, database_name: str) -> None: + test_location = f"s3://{BUCKET_NAME}/{database_name}.db" + test_properties = { + "comment": "this is a test description", + "location": test_location, + "test_property1": "1", + "test_property2": "2", + "test_property3": "3", + } + test_catalog = DynamoDbCatalog("test_ddb_catalog") + test_catalog.create_namespace(database_name, test_properties) + listed_properties = test_catalog.load_namespace_properties(database_name) + for k, v in listed_properties.items(): + assert k in test_properties + assert v == test_properties[k] + + +@mock_dynamodb +def test_load_non_exist_namespace_properties(_bucket_initialize: None, _patch_aiobotocore: None, database_name: str) -> None: + test_catalog = DynamoDbCatalog("test_ddb_catalog") + with pytest.raises(NoSuchNamespaceError): + test_catalog.load_namespace_properties(database_name) + + +@mock_dynamodb +def test_update_namespace_properties(_bucket_initialize: None, _patch_aiobotocore: None, database_name: str) -> None: + test_properties = { + "comment": "this is a test description", + "location": f"s3://{BUCKET_NAME}/{database_name}.db", + "test_property1": "1", + "test_property2": "2", + "test_property3": "3", + } + removals = {"test_property1", "test_property2", "test_property3", "should_not_removed"} + updates = {"test_property4": "4", "test_property5": "5", "comment": "updated test description"} + test_catalog = DynamoDbCatalog("test_ddb_catalog") + test_catalog.create_namespace(database_name, test_properties) + update_report = test_catalog.update_namespace_properties(database_name, removals, updates) + for k in updates.keys(): + assert k in update_report.updated + for k in removals: + if k == "should_not_removed": + assert k in update_report.missing + else: + assert k in update_report.removed + assert "updated test description" == test_catalog.load_namespace_properties(database_name)["comment"] + test_catalog.drop_namespace(database_name) + + +@mock_dynamodb +def test_load_empty_namespace_properties(_bucket_initialize: None, _patch_aiobotocore: None, database_name: str) -> None: + test_catalog = DynamoDbCatalog("test_ddb_catalog") + test_catalog.create_namespace(database_name) + listed_properties = test_catalog.load_namespace_properties(database_name) + assert listed_properties == {} + + +@mock_dynamodb +def test_update_namespace_properties_overlap_update_removal( + _bucket_initialize: None, _patch_aiobotocore: None, database_name: str +) -> None: + test_properties = { + "comment": "this is a test description", + "location": f"s3://{BUCKET_NAME}/{database_name}.db", + "test_property1": "1", + "test_property2": "2", + "test_property3": "3", + } + removals = {"test_property1", "test_property2", "test_property3", "should_not_removed"} + updates = {"test_property1": "4", "test_property5": "5", "comment": "updated test description"} + test_catalog = DynamoDbCatalog("test_ddb_catalog") + test_catalog.create_namespace(database_name, test_properties) + with pytest.raises(ValueError): + test_catalog.update_namespace_properties(database_name, removals, updates) + # should not modify the properties + assert test_catalog.load_namespace_properties(database_name) == test_properties diff --git a/tests/catalog/test_glue.py b/tests/catalog/test_glue.py index 26825c052a..0fef1fad5f 100644 --- a/tests/catalog/test_glue.py +++ b/tests/catalog/test_glue.py @@ -14,7 +14,6 @@ # KIND, either express or implied. See the License for the # specific language governing permissions and limitations # under the License. -import re from typing import List import pytest @@ -31,20 +30,7 @@ TableAlreadyExistsError, ) from pyiceberg.schema import Schema - -BUCKET_NAME = "test_bucket" -LIST_TEST_NUMBER = 100 -table_metadata_location_regex = re.compile( - r"""s3://test_bucket/my_iceberg_database-[a-z]{20}.db/ - my_iceberg_table-[a-z]{20}/metadata/ - 00000-[a-f0-9]{8}-?[a-f0-9]{4}-?4[a-f0-9]{3}-?[89ab][a-f0-9]{3}-?[a-f0-9]{12}.metadata.json""", - re.X, -) - - -@pytest.fixture(name="_bucket_initialize") -def fixture_s3_bucket(_s3) -> None: # type: ignore - _s3.create_bucket(Bucket=BUCKET_NAME) +from tests.conftest import BUCKET_NAME, TABLE_METADATA_LOCATION_REGEX @mock_glue @@ -56,7 +42,7 @@ def test_create_table_with_database_location( test_catalog.create_namespace(namespace=database_name, properties={"location": f"s3://{BUCKET_NAME}/{database_name}.db"}) table = test_catalog.create_table(identifier, table_schema_nested) assert table.identifier == identifier - assert table_metadata_location_regex.match(table.metadata_location) + assert TABLE_METADATA_LOCATION_REGEX.match(table.metadata_location) @mock_glue @@ -68,7 +54,7 @@ def test_create_table_with_default_warehouse( test_catalog.create_namespace(namespace=database_name) table = test_catalog.create_table(identifier, table_schema_nested) assert table.identifier == identifier - assert table_metadata_location_regex.match(table.metadata_location) + assert TABLE_METADATA_LOCATION_REGEX.match(table.metadata_location) @mock_glue @@ -82,7 +68,7 @@ def test_create_table_with_given_location( identifier=identifier, schema=table_schema_nested, location=f"s3://{BUCKET_NAME}/{database_name}.db/{table_name}" ) assert table.identifier == identifier - assert table_metadata_location_regex.match(table.metadata_location) + assert TABLE_METADATA_LOCATION_REGEX.match(table.metadata_location) @mock_glue @@ -105,7 +91,7 @@ def test_create_table_with_strips( test_catalog.create_namespace(namespace=database_name, properties={"location": f"s3://{BUCKET_NAME}/{database_name}.db/"}) table = test_catalog.create_table(identifier, table_schema_nested) assert table.identifier == identifier - assert table_metadata_location_regex.match(table.metadata_location) + assert TABLE_METADATA_LOCATION_REGEX.match(table.metadata_location) @mock_glue @@ -117,7 +103,7 @@ def test_create_table_with_strips_bucket_root( test_catalog.create_namespace(namespace=database_name) table_strip = test_catalog.create_table(identifier, table_schema_nested) assert table_strip.identifier == identifier - assert table_metadata_location_regex.match(table_strip.metadata_location) + assert TABLE_METADATA_LOCATION_REGEX.match(table_strip.metadata_location) @mock_glue @@ -152,7 +138,7 @@ def test_load_table( test_catalog.create_table(identifier, table_schema_nested) table = test_catalog.load_table(identifier) assert table.identifier == identifier - assert table_metadata_location_regex.match(table.metadata_location) + assert TABLE_METADATA_LOCATION_REGEX.match(table.metadata_location) @mock_glue @@ -174,7 +160,7 @@ def test_drop_table( test_catalog.create_table(identifier, table_schema_nested) table = test_catalog.load_table(identifier) assert table.identifier == identifier - assert table_metadata_location_regex.match(table.metadata_location) + assert TABLE_METADATA_LOCATION_REGEX.match(table.metadata_location) test_catalog.drop_table(identifier) with pytest.raises(NoSuchTableError): test_catalog.load_table(identifier) @@ -199,7 +185,7 @@ def test_rename_table( test_catalog.create_namespace(namespace=database_name) table = test_catalog.create_table(identifier, table_schema_nested) assert table.identifier == identifier - assert table_metadata_location_regex.match(table.metadata_location) + assert TABLE_METADATA_LOCATION_REGEX.match(table.metadata_location) test_catalog.rename_table(identifier, new_identifier) new_table = test_catalog.load_table(new_identifier) assert new_table.identifier == new_identifier diff --git a/tests/catalog/test_hive.py b/tests/catalog/test_hive.py index 4525edc7ed..fee3a4f731 100644 --- a/tests/catalog/test_hive.py +++ b/tests/catalog/test_hive.py @@ -696,4 +696,4 @@ def test_resolve_table_location_warehouse(hive_database: HiveDatabase) -> None: catalog._client.__enter__().get_database.return_value = hive_database location = catalog._resolve_table_location(None, "database", "table") - assert location == "/tmp/warehouse/database/table" + assert location == "/tmp/warehouse/database.db/table" diff --git a/tests/conftest.py b/tests/conftest.py index c9bb26854a..c42cfc435d 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -25,6 +25,7 @@ retrieved using `request.getfixturevalue(fixture_name)`. """ import os +import re import string import uuid from random import choice @@ -35,6 +36,7 @@ Dict, Generator, List, + Optional, ) from unittest.mock import MagicMock from urllib.parse import urlparse @@ -48,9 +50,10 @@ import botocore.awsrequest import botocore.model import pytest -from moto import mock_glue, mock_s3 +from moto import mock_dynamodb, mock_glue, mock_s3 from pyiceberg import schema +from pyiceberg.catalog import Catalog from pyiceberg.io import OutputFile, OutputStream, fsspec from pyiceberg.io.fsspec import FsspecFileIO from pyiceberg.io.pyarrow import PyArrowFile, PyArrowFileIO @@ -916,7 +919,9 @@ class LocalOutputFile(OutputFile): def __init__(self, location: str) -> None: parsed_location = urlparse(location) # Create a ParseResult from the uri - if parsed_location.scheme and parsed_location.scheme != "file": # Validate that a uri is provided with a scheme of `file` + if ( + parsed_location.scheme and parsed_location.scheme != "file" + ): # Validate that an uri is provided with a scheme of `file` raise ValueError("LocalOutputFile location must have a scheme of `file`") elif parsed_location.netloc: raise ValueError(f"Network location is not allowed for LocalOutputFile: {parsed_location.netloc}") @@ -1256,6 +1261,13 @@ def fixture_glue(_aws_credentials: None) -> Generator[boto3.client, None, None]: yield boto3.client("glue", region_name="us-east-1") +@pytest.fixture(name="_dynamodb") +def fixture_dynamodb(_aws_credentials: None) -> Generator[boto3.client, None, None]: + """Mocked DynamoDB client""" + with mock_dynamodb(): + yield boto3.client("dynamodb", region_name="us-east-1") + + @pytest.fixture def adlfs_fsspec_fileio(request: pytest.FixtureRequest) -> Generator[FsspecFileIO, None, None]: from azure.storage.blob import BlobServiceClient @@ -1307,3 +1319,52 @@ def database_name() -> str: @pytest.fixture() def database_list(database_name: str) -> List[str]: return [f"{database_name}_{idx}" for idx in range(NUM_TABLES)] + + +BUCKET_NAME = "test_bucket" +TABLE_METADATA_LOCATION_REGEX = re.compile( + r"""s3://test_bucket/my_iceberg_database-[a-z]{20}.db/ + my_iceberg_table-[a-z]{20}/metadata/ + 00000-[a-f0-9]{8}-?[a-f0-9]{4}-?4[a-f0-9]{3}-?[89ab][a-f0-9]{3}-?[a-f0-9]{12}.metadata.json""", + re.X, +) + + +@pytest.fixture(name="_bucket_initialize") +def fixture_s3_bucket(_s3) -> None: # type: ignore + _s3.create_bucket(Bucket=BUCKET_NAME) + + +def get_bucket_name() -> str: + """ + Set the environment variable AWS_TEST_BUCKET for a default bucket to test + """ + bucket_name = os.getenv("AWS_TEST_BUCKET") + if bucket_name is None: + raise ValueError("Please specify a bucket to run the test by setting environment variable AWS_TEST_BUCKET") + return bucket_name + + +def get_s3_path(bucket_name: str, database_name: Optional[str] = None, table_name: Optional[str] = None) -> str: + result_path = f"s3://{bucket_name}" + if database_name is not None: + result_path += f"/{database_name}.db" + + if table_name is not None: + result_path += f"/{table_name}" + return result_path + + +@pytest.fixture(name="s3", scope="module") +def fixture_s3_client() -> boto3.client: + yield boto3.client("s3") + + +def clean_up(test_catalog: Catalog) -> None: + """Clean all databases and tables created during the integration test""" + for database_tuple in test_catalog.list_namespaces(): + database_name = database_tuple[0] + if "my_iceberg_database-" in database_name: + for identifier in test_catalog.list_tables(database_name): + test_catalog.purge_table(identifier) + test_catalog.drop_namespace(database_name) From 5baad95498375e0c37714a2859f8b040f4f11166 Mon Sep 17 00:00:00 2001 From: Fokko Driesprong Date: Wed, 15 Feb 2023 17:58:07 +0100 Subject: [PATCH 372/642] Python: Fix the CI (#6842) Changing the default FileIO from Arrow to Fsspec broke the dynamodb tests. --- tests/catalog/test_dynamodb.py | 38 ++++++++++++++++++++++++---------- 1 file changed, 27 insertions(+), 11 deletions(-) diff --git a/tests/catalog/test_dynamodb.py b/tests/catalog/test_dynamodb.py index e2d9b67de9..582cb034e8 100644 --- a/tests/catalog/test_dynamodb.py +++ b/tests/catalog/test_dynamodb.py @@ -59,7 +59,7 @@ def test_create_table_with_database_location( ) -> None: catalog_name = "test_ddb_catalog" identifier = (database_name, table_name) - test_catalog = DynamoDbCatalog(catalog_name) + test_catalog = DynamoDbCatalog(catalog_name, **{"py-io-impl": "pyiceberg.io.fsspec.FsspecFileIO"}) test_catalog.create_namespace(namespace=database_name, properties={"location": f"s3://{BUCKET_NAME}/{database_name}.db"}) table = test_catalog.create_table(identifier, table_schema_nested) assert table.identifier == (catalog_name,) + identifier @@ -72,7 +72,9 @@ def test_create_table_with_default_warehouse( ) -> None: catalog_name = "test_ddb_catalog" identifier = (database_name, table_name) - test_catalog = DynamoDbCatalog(catalog_name, warehouse=f"s3://{BUCKET_NAME}") + test_catalog = DynamoDbCatalog( + catalog_name, **{"py-io-impl": "pyiceberg.io.fsspec.FsspecFileIO", "warehouse": f"s3://{BUCKET_NAME}"} + ) test_catalog.create_namespace(namespace=database_name) table = test_catalog.create_table(identifier, table_schema_nested) assert table.identifier == (catalog_name,) + identifier @@ -85,7 +87,7 @@ def test_create_table_with_given_location( ) -> None: catalog_name = "test_ddb_catalog" identifier = (database_name, table_name) - test_catalog = DynamoDbCatalog(catalog_name) + test_catalog = DynamoDbCatalog(catalog_name, **{"py-io-impl": "pyiceberg.io.fsspec.FsspecFileIO"}) test_catalog.create_namespace(namespace=database_name) table = test_catalog.create_table( identifier=identifier, schema=table_schema_nested, location=f"s3://{BUCKET_NAME}/{database_name}.db/{table_name}" @@ -111,7 +113,7 @@ def test_create_table_with_strips( ) -> None: catalog_name = "test_ddb_catalog" identifier = (database_name, table_name) - test_catalog = DynamoDbCatalog(catalog_name) + test_catalog = DynamoDbCatalog(catalog_name, **{"py-io-impl": "pyiceberg.io.fsspec.FsspecFileIO"}) test_catalog.create_namespace(namespace=database_name, properties={"location": f"s3://{BUCKET_NAME}/{database_name}.db/"}) table = test_catalog.create_table(identifier, table_schema_nested) assert table.identifier == (catalog_name,) + identifier @@ -124,7 +126,9 @@ def test_create_table_with_strips_bucket_root( ) -> None: catalog_name = "test_ddb_catalog" identifier = (database_name, table_name) - test_catalog = DynamoDbCatalog(catalog_name, warehouse=f"s3://{BUCKET_NAME}/") + test_catalog = DynamoDbCatalog( + catalog_name, **{"py-io-impl": "pyiceberg.io.fsspec.FsspecFileIO", "warehouse": f"s3://{BUCKET_NAME}/"} + ) test_catalog.create_namespace(namespace=database_name) table_strip = test_catalog.create_table(identifier, table_schema_nested) assert table_strip.identifier == (catalog_name,) + identifier @@ -146,7 +150,9 @@ def test_create_duplicated_table( _bucket_initialize: None, _patch_aiobotocore: None, table_schema_nested: Schema, database_name: str, table_name: str ) -> None: identifier = (database_name, table_name) - test_catalog = DynamoDbCatalog("test_ddb_catalog", warehouse=f"s3://{BUCKET_NAME}") + test_catalog = DynamoDbCatalog( + "test_ddb_catalog", **{"warehouse": f"s3://{BUCKET_NAME}", "py-io-impl": "pyiceberg.io.fsspec.FsspecFileIO"} + ) test_catalog.create_namespace(namespace=database_name) test_catalog.create_table(identifier, table_schema_nested) with pytest.raises(TableAlreadyExistsError): @@ -159,7 +165,9 @@ def test_load_table( ) -> None: catalog_name = "test_ddb_catalog" identifier = (database_name, table_name) - test_catalog = DynamoDbCatalog(catalog_name, warehouse=f"s3://{BUCKET_NAME}") + test_catalog = DynamoDbCatalog( + catalog_name, **{"warehouse": f"s3://{BUCKET_NAME}", "py-io-impl": "pyiceberg.io.fsspec.FsspecFileIO"} + ) test_catalog.create_namespace(namespace=database_name) test_catalog.create_table(identifier, table_schema_nested) table = test_catalog.load_table(identifier) @@ -182,7 +190,9 @@ def test_drop_table( ) -> None: catalog_name = "test_ddb_catalog" identifier = (database_name, table_name) - test_catalog = DynamoDbCatalog(catalog_name, warehouse=f"s3://{BUCKET_NAME}") + test_catalog = DynamoDbCatalog( + catalog_name, **{"warehouse": f"s3://{BUCKET_NAME}", "py-io-impl": "pyiceberg.io.fsspec.FsspecFileIO"} + ) test_catalog.create_namespace(namespace=database_name) test_catalog.create_table(identifier, table_schema_nested) table = test_catalog.load_table(identifier) @@ -209,7 +219,9 @@ def test_rename_table( new_table_name = f"{table_name}_new" identifier = (database_name, table_name) new_identifier = (database_name, new_table_name) - test_catalog = DynamoDbCatalog(catalog_name, warehouse=f"s3://{BUCKET_NAME}") + test_catalog = DynamoDbCatalog( + catalog_name, **{"warehouse": f"s3://{BUCKET_NAME}", "py-io-impl": "pyiceberg.io.fsspec.FsspecFileIO"} + ) test_catalog.create_namespace(namespace=database_name) table = test_catalog.create_table(identifier, table_schema_nested) assert table.identifier == (catalog_name,) + identifier @@ -281,7 +293,9 @@ def test_fail_on_rename_non_iceberg_table(_dynamodb, _bucket_initialize: None, _ def test_list_tables( _bucket_initialize: None, _patch_aiobotocore: None, table_schema_nested: Schema, database_name: str, table_list: List[str] ) -> None: - test_catalog = DynamoDbCatalog("test_ddb_catalog", warehouse=f"s3://{BUCKET_NAME}") + test_catalog = DynamoDbCatalog( + "test_ddb_catalog", **{"warehouse": f"s3://{BUCKET_NAME}", "py-io-impl": "pyiceberg.io.fsspec.FsspecFileIO"} + ) test_catalog.create_namespace(namespace=database_name) for table_name in table_list: test_catalog.create_table((database_name, table_name), table_schema_nested) @@ -358,7 +372,9 @@ def test_drop_non_empty_namespace( _bucket_initialize: None, _patch_aiobotocore: None, table_schema_nested: Schema, database_name: str, table_name: str ) -> None: identifier = (database_name, table_name) - test_catalog = DynamoDbCatalog("test_ddb_catalog", warehouse=f"s3://{BUCKET_NAME}") + test_catalog = DynamoDbCatalog( + "test_ddb_catalog", **{"warehouse": f"s3://{BUCKET_NAME}", "py-io-impl": "pyiceberg.io.fsspec.FsspecFileIO"} + ) test_catalog.create_namespace(namespace=database_name) test_catalog.create_table(identifier, table_schema_nested) assert len(test_catalog.list_tables(database_name)) == 1 From 210f0718900e92be2970bf6b50682dd18771a8f9 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 16 Feb 2023 08:45:52 +0100 Subject: [PATCH 373/642] Build: Bump werkzeug from 2.2.2 to 2.2.3 in /python (#6854) Bumps [werkzeug](https://github.com/pallets/werkzeug) from 2.2.2 to 2.2.3. - [Release notes](https://github.com/pallets/werkzeug/releases) - [Changelog](https://github.com/pallets/werkzeug/blob/main/CHANGES.rst) - [Commits](https://github.com/pallets/werkzeug/compare/2.2.2...2.2.3) --- updated-dependencies: - dependency-name: werkzeug dependency-type: indirect ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- poetry.lock | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/poetry.lock b/poetry.lock index 12eebbb607..3087614073 100644 --- a/poetry.lock +++ b/poetry.lock @@ -601,6 +601,8 @@ files = [ {file = "cryptography-39.0.1-cp36-abi3-win32.whl", hash = "sha256:fe913f20024eb2cb2f323e42a64bdf2911bb9738a15dba7d3cce48151034e3a8"}, {file = "cryptography-39.0.1-cp36-abi3-win_amd64.whl", hash = "sha256:ced4e447ae29ca194449a3f1ce132ded8fcab06971ef5f618605aacaa612beac"}, {file = "cryptography-39.0.1-pp38-pypy38_pp73-macosx_10_12_x86_64.whl", hash = "sha256:807ce09d4434881ca3a7594733669bd834f5b2c6d5c7e36f8c00f691887042ad"}, + {file = "cryptography-39.0.1-pp38-pypy38_pp73-manylinux_2_28_aarch64.whl", hash = "sha256:c5caeb8188c24888c90b5108a441c106f7faa4c4c075a2bcae438c6e8ca73cef"}, + {file = "cryptography-39.0.1-pp38-pypy38_pp73-manylinux_2_28_x86_64.whl", hash = "sha256:4789d1e3e257965e960232345002262ede4d094d1a19f4d3b52e48d4d8f3b885"}, {file = "cryptography-39.0.1-pp38-pypy38_pp73-win_amd64.whl", hash = "sha256:96f1157a7c08b5b189b16b47bc9db2332269d6680a196341bf30046330d15388"}, {file = "cryptography-39.0.1-pp39-pypy39_pp73-macosx_10_12_x86_64.whl", hash = "sha256:e422abdec8b5fa8462aa016786680720d78bdce7a30c652b7fadf83a4ba35336"}, {file = "cryptography-39.0.1-pp39-pypy39_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.manylinux_2_24_aarch64.whl", hash = "sha256:b0afd054cd42f3d213bf82c629efb1ee5f22eba35bf0eec88ea9ea7304f511a2"}, @@ -2184,14 +2186,14 @@ testing = ["coverage (>=6.2)", "coverage-enable-subprocess (>=1)", "flaky (>=3.7 [[package]] name = "werkzeug" -version = "2.2.2" +version = "2.2.3" description = "The comprehensive WSGI web application library." category = "dev" optional = false python-versions = ">=3.7" files = [ - {file = "Werkzeug-2.2.2-py3-none-any.whl", hash = "sha256:f979ab81f58d7318e064e99c4506445d60135ac5cd2e177a2de0089bfd4c9bd5"}, - {file = "Werkzeug-2.2.2.tar.gz", hash = "sha256:7ea2d48322cc7c0f8b3a215ed73eabd7b5d75d0b50e31ab006286ccff9e00b8f"}, + {file = "Werkzeug-2.2.3-py3-none-any.whl", hash = "sha256:56433961bc1f12533306c624f3be5e744389ac61d722175d543e1751285da612"}, + {file = "Werkzeug-2.2.3.tar.gz", hash = "sha256:2e1ccc9417d4da358b9de6f174e3ac094391ea1d4fbef2d667865d819dfd0afe"}, ] [package.dependencies] @@ -2460,6 +2462,7 @@ cffi = ["cffi (>=1.11)"] [extras] adlfs = ["adlfs"] duckdb = ["duckdb", "pyarrow"] +dynamodb = ["boto3"] glue = ["boto3"] hive = ["thrift"] pandas = ["pandas", "pyarrow"] @@ -2470,4 +2473,4 @@ snappy = ["python-snappy"] [metadata] lock-version = "2.0" python-versions = "^3.8" -content-hash = "9ed787b3f081bd27cee8c90fa1ad3461d8734f5251edcc0339c62b37c3a275f9" +content-hash = "4cdeb2c3ef66160ac86dc5016eee2cf482ecec825cbfd33f582cdbbb2a063bfb" From 83c918886308361123dc9f127d5b7fdb1416a7d1 Mon Sep 17 00:00:00 2001 From: Pritam <28836358+pritampan@users.noreply.github.com> Date: Thu, 16 Feb 2023 08:05:04 +0000 Subject: [PATCH 374/642] Python: Add String to Boolean literal conversion (#6851) * Convert string to boolean if the binding variable is Boolean * Update python/pyiceberg/expressions/literals.py Co-authored-by: Fokko Driesprong * addressed review comments * corrected lint failure * corrected test failure --------- Co-authored-by: pritampan Co-authored-by: Fokko Driesprong --- pyiceberg/expressions/literals.py | 8 ++++++++ tests/expressions/test_literals.py | 16 +++++++++++++++- 2 files changed, 23 insertions(+), 1 deletion(-) diff --git a/pyiceberg/expressions/literals.py b/pyiceberg/expressions/literals.py index fcfaaf7fe9..3e3233af0d 100644 --- a/pyiceberg/expressions/literals.py +++ b/pyiceberg/expressions/literals.py @@ -562,6 +562,14 @@ def _(self, type_var: DecimalType) -> Literal[Decimal]: f"Could not convert {self.value} into a {type_var}, scales differ {type_var.scale} <> {abs(dec.as_tuple().exponent)}" ) + @to.register(BooleanType) + def _(self, type_var: BooleanType) -> Literal[bool]: + value_upper = self.value.upper() + if value_upper in ["TRUE", "FALSE"]: + return BooleanLiteral(value_upper == "TRUE") + else: + raise ValueError(f"Could not convert {self.value} into a {type_var}") + def __repr__(self) -> str: return f"literal({repr(self.value)})" diff --git a/tests/expressions/test_literals.py b/tests/expressions/test_literals.py index 7373ff040b..16aee4dbc3 100644 --- a/tests/expressions/test_literals.py +++ b/tests/expressions/test_literals.py @@ -384,6 +384,20 @@ def test_string_to_decimal_literal() -> None: assert Decimal("34.560").as_tuple() == decimal_lit.value.as_tuple() # type: ignore +def test_string_to_boolean_literal() -> None: + assert literal(True) == literal("true").to(BooleanType()) + assert literal(True) == literal("True").to(BooleanType()) + assert literal(False) == literal("false").to(BooleanType()) + assert literal(False) == literal("False").to(BooleanType()) + + +def test_invalid_string_to_boolean_literal() -> None: + invalid_boolean_str = literal("unknown") + with pytest.raises(ValueError) as e: + _ = invalid_boolean_str.to(BooleanType()) + assert "Could not convert unknown into a boolean" in str(e.value) + + # MISC @@ -692,7 +706,7 @@ def test_invalid_decimal_conversions() -> None: def test_invalid_string_conversions() -> None: assert_invalid_conversions( literal("abc"), - [BooleanType(), FloatType(), DoubleType(), FixedType(1), BinaryType()], + [FloatType(), DoubleType(), FixedType(1), BinaryType()], ) From ca388e2d45b7013a8bd7f1babf9d594949f8c5e4 Mon Sep 17 00:00:00 2001 From: Fokko Driesprong Date: Thu, 16 Feb 2023 23:00:37 +0100 Subject: [PATCH 375/642] Python: Add `default-catalog` option to the config (#6864) If you set `default-catalog` it will default to that catalog. --- pyiceberg/catalog/__init__.py | 6 +++++- pyiceberg/cli/console.py | 4 ++-- pyiceberg/utils/config.py | 16 ++++++++++++++++ 3 files changed, 23 insertions(+), 3 deletions(-) diff --git a/pyiceberg/catalog/__init__.py b/pyiceberg/catalog/__init__.py index c908a3b462..041018f127 100644 --- a/pyiceberg/catalog/__init__.py +++ b/pyiceberg/catalog/__init__.py @@ -146,7 +146,7 @@ def infer_catalog_type(name: str, catalog_properties: RecursiveDict) -> Optional ) -def load_catalog(name: str, **properties: Optional[str]) -> Catalog: +def load_catalog(name: Optional[str], **properties: Optional[str]) -> Catalog: """Load the catalog based on the properties Will look up the properties from the config, based on the name @@ -162,6 +162,10 @@ def load_catalog(name: str, **properties: Optional[str]) -> Catalog: ValueError: Raises a ValueError in case properties are missing or malformed, or if it could not determine the catalog based on the properties """ + + if name is None: + name = _ENV_CONFIG.get_default_catalog_name() + env = _ENV_CONFIG.get_catalog_config(name) conf: RecursiveDict = merge_config(env or {}, cast(RecursiveDict, properties)) diff --git a/pyiceberg/cli/console.py b/pyiceberg/cli/console.py index e2e9dbe1fc..7ece1b1bbe 100644 --- a/pyiceberg/cli/console.py +++ b/pyiceberg/cli/console.py @@ -50,13 +50,13 @@ def wrapper(*args: Any, **kwargs: Any): # type: ignore @click.group() -@click.option("--catalog", default="default") +@click.option("--catalog") @click.option("--verbose", type=click.BOOL) @click.option("--output", type=click.Choice(["text", "json"]), default="text") @click.option("--uri") @click.option("--credential") @click.pass_context -def run(ctx: Context, catalog: str, verbose: bool, output: str, uri: Optional[str], credential: Optional[str]) -> None: +def run(ctx: Context, catalog: Optional[str], verbose: bool, output: str, uri: Optional[str], credential: Optional[str]) -> None: properties = {} if uri: properties["uri"] = uri diff --git a/pyiceberg/utils/config.py b/pyiceberg/utils/config.py index 6055f75df9..1af3e13aa0 100644 --- a/pyiceberg/utils/config.py +++ b/pyiceberg/utils/config.py @@ -23,7 +23,9 @@ from pyiceberg.typedef import FrozenDict, RecursiveDict PYICEBERG = "pyiceberg_" +DEFAULT = "default" CATALOG = "catalog" +DEFAULT_CATALOG = f"{DEFAULT}-{CATALOG}" HOME = "HOME" PYICEBERG_HOME = "PYICEBERG_HOME" PYICEBERG_YML = ".pyiceberg.yaml" @@ -129,6 +131,20 @@ def set_property(_config: RecursiveDict, path: List[str], config_value: str) -> return config + def get_default_catalog_name(self) -> str: + """ + Looks into the configuration file for `default-catalog` + and returns the name as the default catalog + + Returns: The name of the default catalog in `default-catalog` + Returns `default` when the key cannot be found. + """ + if default_catalog_name := self.config.get(DEFAULT_CATALOG): + if not isinstance(default_catalog_name, str): + raise ValueError(f"Default catalog name should be a str: {default_catalog_name}") + return default_catalog_name + return DEFAULT + def get_catalog_config(self, catalog_name: str) -> Optional[RecursiveDict]: if CATALOG in self.config: catalog_name_lower = catalog_name.lower() From ea6fccc195881690404a09f6a21054c4e4e52c17 Mon Sep 17 00:00:00 2001 From: Fokko Driesprong Date: Sun, 19 Feb 2023 22:02:48 +0100 Subject: [PATCH 376/642] Python: Bump DuckDB to 0.7.0 (#6848) New release: https://github.com/duckdb/duckdb/releases/tag/v0.7.0 That has some nice Python API improvements: https://duckdb.org/2023/02/13/announcing-duckdb-070.html#python-api-improvements --- poetry.lock | 2537 ++++++++++++++++++++++++------------------------ pyproject.toml | 2 +- 2 files changed, 1287 insertions(+), 1252 deletions(-) diff --git a/poetry.lock b/poetry.lock index 3087614073..def42af828 100644 --- a/poetry.lock +++ b/poetry.lock @@ -1,5 +1,3 @@ -# This file is automatically @generated by Poetry and should not be changed by hand. - [[package]] name = "adal" version = "1.2.7" @@ -7,10 +5,6 @@ description = "Note: This library is already replaced by MSAL Python, available category = "main" optional = true python-versions = "*" -files = [ - {file = "adal-1.2.7-py2.py3-none-any.whl", hash = "sha256:2a7451ed7441ddbc57703042204a3e30ef747478eea022c70f789fc7f084bc3d"}, - {file = "adal-1.2.7.tar.gz", hash = "sha256:d74f45b81317454d96e982fd1c50e6fb5c99ac2223728aea8764433a39f566f1"}, -] [package.dependencies] cryptography = ">=1.1.0" @@ -25,10 +19,6 @@ description = "Access Azure Datalake Gen1 with fsspec and dask" category = "main" optional = true python-versions = ">=3.8" -files = [ - {file = "adlfs-2023.1.0-py3-none-any.whl", hash = "sha256:ccbdd6d33d5b7bce99a4a07c884c82fb135745d39b2d9b7b672f8e9d4d04407f"}, - {file = "adlfs-2023.1.0.tar.gz", hash = "sha256:eca53f53d88fc8e2e7a2f1d8f5a40b8a1c56aa3b541e4aa2e7eaf55a6a789262"}, -] [package.dependencies] aiohttp = ">=3.7.0" @@ -48,10 +38,6 @@ description = "Async client for aws services using botocore and aiohttp" category = "main" optional = true python-versions = ">=3.7" -files = [ - {file = "aiobotocore-2.4.2-py3-none-any.whl", hash = "sha256:4acd1ebe2e44be4b100aa553910bda899f6dc090b3da2bc1cf3d5de2146ed208"}, - {file = "aiobotocore-2.4.2.tar.gz", hash = "sha256:0603b74a582dffa7511ce7548d07dc9b10ec87bc5fb657eb0b34f9bd490958bf"}, -] [package.dependencies] aiohttp = ">=3.3.1" @@ -65,112 +51,23 @@ boto3 = ["boto3 (>=1.24.59,<1.24.60)"] [[package]] name = "aiohttp" -version = "3.8.3" +version = "3.8.4" description = "Async http client/server framework (asyncio)" category = "main" optional = true python-versions = ">=3.6" -files = [ - {file = "aiohttp-3.8.3-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:ba71c9b4dcbb16212f334126cc3d8beb6af377f6703d9dc2d9fb3874fd667ee9"}, - {file = "aiohttp-3.8.3-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:d24b8bb40d5c61ef2d9b6a8f4528c2f17f1c5d2d31fed62ec860f6006142e83e"}, - {file = "aiohttp-3.8.3-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:f88df3a83cf9df566f171adba39d5bd52814ac0b94778d2448652fc77f9eb491"}, - {file = "aiohttp-3.8.3-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:b97decbb3372d4b69e4d4c8117f44632551c692bb1361b356a02b97b69e18a62"}, - {file = "aiohttp-3.8.3-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:309aa21c1d54b8ef0723181d430347d7452daaff93e8e2363db8e75c72c2fb2d"}, - {file = "aiohttp-3.8.3-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:ad5383a67514e8e76906a06741febd9126fc7c7ff0f599d6fcce3e82b80d026f"}, - {file = "aiohttp-3.8.3-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:20acae4f268317bb975671e375493dbdbc67cddb5f6c71eebdb85b34444ac46b"}, - {file = "aiohttp-3.8.3-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:05a3c31c6d7cd08c149e50dc7aa2568317f5844acd745621983380597f027a18"}, - {file = "aiohttp-3.8.3-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:d6f76310355e9fae637c3162936e9504b4767d5c52ca268331e2756e54fd4ca5"}, - {file = "aiohttp-3.8.3-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:256deb4b29fe5e47893fa32e1de2d73c3afe7407738bd3c63829874661d4822d"}, - {file = "aiohttp-3.8.3-cp310-cp310-musllinux_1_1_ppc64le.whl", hash = "sha256:5c59fcd80b9049b49acd29bd3598cada4afc8d8d69bd4160cd613246912535d7"}, - {file = "aiohttp-3.8.3-cp310-cp310-musllinux_1_1_s390x.whl", hash = "sha256:059a91e88f2c00fe40aed9031b3606c3f311414f86a90d696dd982e7aec48142"}, - {file = "aiohttp-3.8.3-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:2feebbb6074cdbd1ac276dbd737b40e890a1361b3cc30b74ac2f5e24aab41f7b"}, - {file = "aiohttp-3.8.3-cp310-cp310-win32.whl", hash = "sha256:5bf651afd22d5f0c4be16cf39d0482ea494f5c88f03e75e5fef3a85177fecdeb"}, - {file = "aiohttp-3.8.3-cp310-cp310-win_amd64.whl", hash = "sha256:653acc3880459f82a65e27bd6526e47ddf19e643457d36a2250b85b41a564715"}, - {file = "aiohttp-3.8.3-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:86fc24e58ecb32aee09f864cb11bb91bc4c1086615001647dbfc4dc8c32f4008"}, - {file = "aiohttp-3.8.3-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:75e14eac916f024305db517e00a9252714fce0abcb10ad327fb6dcdc0d060f1d"}, - {file = "aiohttp-3.8.3-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:d1fde0f44029e02d02d3993ad55ce93ead9bb9b15c6b7ccd580f90bd7e3de476"}, - {file = "aiohttp-3.8.3-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:4ab94426ddb1ecc6a0b601d832d5d9d421820989b8caa929114811369673235c"}, - {file = "aiohttp-3.8.3-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:89d2e02167fa95172c017732ed7725bc8523c598757f08d13c5acca308e1a061"}, - {file = "aiohttp-3.8.3-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:02f9a2c72fc95d59b881cf38a4b2be9381b9527f9d328771e90f72ac76f31ad8"}, - {file = "aiohttp-3.8.3-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:9c7149272fb5834fc186328e2c1fa01dda3e1fa940ce18fded6d412e8f2cf76d"}, - {file = "aiohttp-3.8.3-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:512bd5ab136b8dc0ffe3fdf2dfb0c4b4f49c8577f6cae55dca862cd37a4564e2"}, - {file = "aiohttp-3.8.3-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:7018ecc5fe97027214556afbc7c502fbd718d0740e87eb1217b17efd05b3d276"}, - {file = "aiohttp-3.8.3-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:88c70ed9da9963d5496d38320160e8eb7e5f1886f9290475a881db12f351ab5d"}, - {file = "aiohttp-3.8.3-cp311-cp311-musllinux_1_1_ppc64le.whl", hash = "sha256:da22885266bbfb3f78218dc40205fed2671909fbd0720aedba39b4515c038091"}, - {file = "aiohttp-3.8.3-cp311-cp311-musllinux_1_1_s390x.whl", hash = "sha256:e65bc19919c910127c06759a63747ebe14f386cda573d95bcc62b427ca1afc73"}, - {file = "aiohttp-3.8.3-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:08c78317e950e0762c2983f4dd58dc5e6c9ff75c8a0efeae299d363d439c8e34"}, - {file = "aiohttp-3.8.3-cp311-cp311-win32.whl", hash = "sha256:45d88b016c849d74ebc6f2b6e8bc17cabf26e7e40c0661ddd8fae4c00f015697"}, - {file = "aiohttp-3.8.3-cp311-cp311-win_amd64.whl", hash = "sha256:96372fc29471646b9b106ee918c8eeb4cca423fcbf9a34daa1b93767a88a2290"}, - {file = "aiohttp-3.8.3-cp36-cp36m-macosx_10_9_x86_64.whl", hash = "sha256:c971bf3786b5fad82ce5ad570dc6ee420f5b12527157929e830f51c55dc8af77"}, - {file = "aiohttp-3.8.3-cp36-cp36m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:ff25f48fc8e623d95eca0670b8cc1469a83783c924a602e0fbd47363bb54aaca"}, - {file = "aiohttp-3.8.3-cp36-cp36m-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:e381581b37db1db7597b62a2e6b8b57c3deec95d93b6d6407c5b61ddc98aca6d"}, - {file = "aiohttp-3.8.3-cp36-cp36m-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:db19d60d846283ee275d0416e2a23493f4e6b6028825b51290ac05afc87a6f97"}, - {file = "aiohttp-3.8.3-cp36-cp36m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:25892c92bee6d9449ffac82c2fe257f3a6f297792cdb18ad784737d61e7a9a85"}, - {file = "aiohttp-3.8.3-cp36-cp36m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:398701865e7a9565d49189f6c90868efaca21be65c725fc87fc305906be915da"}, - {file = "aiohttp-3.8.3-cp36-cp36m-musllinux_1_1_aarch64.whl", hash = "sha256:4a4fbc769ea9b6bd97f4ad0b430a6807f92f0e5eb020f1e42ece59f3ecfc4585"}, - {file = "aiohttp-3.8.3-cp36-cp36m-musllinux_1_1_i686.whl", hash = "sha256:b29bfd650ed8e148f9c515474a6ef0ba1090b7a8faeee26b74a8ff3b33617502"}, - {file = "aiohttp-3.8.3-cp36-cp36m-musllinux_1_1_ppc64le.whl", hash = "sha256:1e56b9cafcd6531bab5d9b2e890bb4937f4165109fe98e2b98ef0dcfcb06ee9d"}, - {file = "aiohttp-3.8.3-cp36-cp36m-musllinux_1_1_s390x.whl", hash = "sha256:ec40170327d4a404b0d91855d41bfe1fe4b699222b2b93e3d833a27330a87a6d"}, - {file = "aiohttp-3.8.3-cp36-cp36m-musllinux_1_1_x86_64.whl", hash = "sha256:2df5f139233060578d8c2c975128fb231a89ca0a462b35d4b5fcf7c501ebdbe1"}, - {file = "aiohttp-3.8.3-cp36-cp36m-win32.whl", hash = "sha256:f973157ffeab5459eefe7b97a804987876dd0a55570b8fa56b4e1954bf11329b"}, - {file = "aiohttp-3.8.3-cp36-cp36m-win_amd64.whl", hash = "sha256:437399385f2abcd634865705bdc180c8314124b98299d54fe1d4c8990f2f9494"}, - {file = "aiohttp-3.8.3-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:09e28f572b21642128ef31f4e8372adb6888846f32fecb288c8b0457597ba61a"}, - {file = "aiohttp-3.8.3-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:6f3553510abdbec67c043ca85727396ceed1272eef029b050677046d3387be8d"}, - {file = "aiohttp-3.8.3-cp37-cp37m-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:e168a7560b7c61342ae0412997b069753f27ac4862ec7867eff74f0fe4ea2ad9"}, - {file = "aiohttp-3.8.3-cp37-cp37m-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:db4c979b0b3e0fa7e9e69ecd11b2b3174c6963cebadeecfb7ad24532ffcdd11a"}, - {file = "aiohttp-3.8.3-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:e164e0a98e92d06da343d17d4e9c4da4654f4a4588a20d6c73548a29f176abe2"}, - {file = "aiohttp-3.8.3-cp37-cp37m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:e8a78079d9a39ca9ca99a8b0ac2fdc0c4d25fc80c8a8a82e5c8211509c523363"}, - {file = "aiohttp-3.8.3-cp37-cp37m-musllinux_1_1_aarch64.whl", hash = "sha256:21b30885a63c3f4ff5b77a5d6caf008b037cb521a5f33eab445dc566f6d092cc"}, - {file = "aiohttp-3.8.3-cp37-cp37m-musllinux_1_1_i686.whl", hash = "sha256:4b0f30372cef3fdc262f33d06e7b411cd59058ce9174ef159ad938c4a34a89da"}, - {file = "aiohttp-3.8.3-cp37-cp37m-musllinux_1_1_ppc64le.whl", hash = "sha256:8135fa153a20d82ffb64f70a1b5c2738684afa197839b34cc3e3c72fa88d302c"}, - {file = "aiohttp-3.8.3-cp37-cp37m-musllinux_1_1_s390x.whl", hash = "sha256:ad61a9639792fd790523ba072c0555cd6be5a0baf03a49a5dd8cfcf20d56df48"}, - {file = "aiohttp-3.8.3-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:978b046ca728073070e9abc074b6299ebf3501e8dee5e26efacb13cec2b2dea0"}, - {file = "aiohttp-3.8.3-cp37-cp37m-win32.whl", hash = "sha256:0d2c6d8c6872df4a6ec37d2ede71eff62395b9e337b4e18efd2177de883a5033"}, - {file = "aiohttp-3.8.3-cp37-cp37m-win_amd64.whl", hash = "sha256:21d69797eb951f155026651f7e9362877334508d39c2fc37bd04ff55b2007091"}, - {file = "aiohttp-3.8.3-cp38-cp38-macosx_10_9_universal2.whl", hash = "sha256:2ca9af5f8f5812d475c5259393f52d712f6d5f0d7fdad9acdb1107dd9e3cb7eb"}, - {file = "aiohttp-3.8.3-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:1d90043c1882067f1bd26196d5d2db9aa6d268def3293ed5fb317e13c9413ea4"}, - {file = "aiohttp-3.8.3-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:d737fc67b9a970f3234754974531dc9afeea11c70791dcb7db53b0cf81b79784"}, - {file = "aiohttp-3.8.3-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:ebf909ea0a3fc9596e40d55d8000702a85e27fd578ff41a5500f68f20fd32e6c"}, - {file = "aiohttp-3.8.3-cp38-cp38-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:5835f258ca9f7c455493a57ee707b76d2d9634d84d5d7f62e77be984ea80b849"}, - {file = "aiohttp-3.8.3-cp38-cp38-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:da37dcfbf4b7f45d80ee386a5f81122501ec75672f475da34784196690762f4b"}, - {file = "aiohttp-3.8.3-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:87f44875f2804bc0511a69ce44a9595d5944837a62caecc8490bbdb0e18b1342"}, - {file = "aiohttp-3.8.3-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:527b3b87b24844ea7865284aabfab08eb0faf599b385b03c2aa91fc6edd6e4b6"}, - {file = "aiohttp-3.8.3-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:d5ba88df9aa5e2f806650fcbeedbe4f6e8736e92fc0e73b0400538fd25a4dd96"}, - {file = "aiohttp-3.8.3-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:e7b8813be97cab8cb52b1375f41f8e6804f6507fe4660152e8ca5c48f0436017"}, - {file = "aiohttp-3.8.3-cp38-cp38-musllinux_1_1_ppc64le.whl", hash = "sha256:2dea10edfa1a54098703cb7acaa665c07b4e7568472a47f4e64e6319d3821ccf"}, - {file = "aiohttp-3.8.3-cp38-cp38-musllinux_1_1_s390x.whl", hash = "sha256:713d22cd9643ba9025d33c4af43943c7a1eb8547729228de18d3e02e278472b6"}, - {file = "aiohttp-3.8.3-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:2d252771fc85e0cf8da0b823157962d70639e63cb9b578b1dec9868dd1f4f937"}, - {file = "aiohttp-3.8.3-cp38-cp38-win32.whl", hash = "sha256:66bd5f950344fb2b3dbdd421aaa4e84f4411a1a13fca3aeb2bcbe667f80c9f76"}, - {file = "aiohttp-3.8.3-cp38-cp38-win_amd64.whl", hash = "sha256:84b14f36e85295fe69c6b9789b51a0903b774046d5f7df538176516c3e422446"}, - {file = "aiohttp-3.8.3-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:16c121ba0b1ec2b44b73e3a8a171c4f999b33929cd2397124a8c7fcfc8cd9e06"}, - {file = "aiohttp-3.8.3-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:8d6aaa4e7155afaf994d7924eb290abbe81a6905b303d8cb61310a2aba1c68ba"}, - {file = "aiohttp-3.8.3-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:43046a319664a04b146f81b40e1545d4c8ac7b7dd04c47e40bf09f65f2437346"}, - {file = "aiohttp-3.8.3-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:599418aaaf88a6d02a8c515e656f6faf3d10618d3dd95866eb4436520096c84b"}, - {file = "aiohttp-3.8.3-cp39-cp39-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:92a2964319d359f494f16011e23434f6f8ef0434acd3cf154a6b7bec511e2fb7"}, - {file = "aiohttp-3.8.3-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:73a4131962e6d91109bca6536416aa067cf6c4efb871975df734f8d2fd821b37"}, - {file = "aiohttp-3.8.3-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:598adde339d2cf7d67beaccda3f2ce7c57b3b412702f29c946708f69cf8222aa"}, - {file = "aiohttp-3.8.3-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:75880ed07be39beff1881d81e4a907cafb802f306efd6d2d15f2b3c69935f6fb"}, - {file = "aiohttp-3.8.3-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:a0239da9fbafd9ff82fd67c16704a7d1bccf0d107a300e790587ad05547681c8"}, - {file = "aiohttp-3.8.3-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:4e3a23ec214e95c9fe85a58470b660efe6534b83e6cbe38b3ed52b053d7cb6ad"}, - {file = "aiohttp-3.8.3-cp39-cp39-musllinux_1_1_ppc64le.whl", hash = "sha256:47841407cc89a4b80b0c52276f3cc8138bbbfba4b179ee3acbd7d77ae33f7ac4"}, - {file = "aiohttp-3.8.3-cp39-cp39-musllinux_1_1_s390x.whl", hash = "sha256:54d107c89a3ebcd13228278d68f1436d3f33f2dd2af5415e3feaeb1156e1a62c"}, - {file = "aiohttp-3.8.3-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:c37c5cce780349d4d51739ae682dec63573847a2a8dcb44381b174c3d9c8d403"}, - {file = "aiohttp-3.8.3-cp39-cp39-win32.whl", hash = "sha256:f178d2aadf0166be4df834c4953da2d7eef24719e8aec9a65289483eeea9d618"}, - {file = "aiohttp-3.8.3-cp39-cp39-win_amd64.whl", hash = "sha256:88e5be56c231981428f4f506c68b6a46fa25c4123a2e86d156c58a8369d31ab7"}, - {file = "aiohttp-3.8.3.tar.gz", hash = "sha256:3828fb41b7203176b82fe5d699e0d845435f2374750a44b480ea6b930f6be269"}, -] [package.dependencies] aiosignal = ">=1.1.2" async-timeout = ">=4.0.0a3,<5.0" attrs = ">=17.3.0" -charset-normalizer = ">=2.0,<3.0" +charset-normalizer = ">=2.0,<4.0" frozenlist = ">=1.1.1" multidict = ">=4.5,<7.0" yarl = ">=1.0,<2.0" [package.extras] -speedups = ["Brotli", "aiodns", "cchardet"] +speedups = ["aiodns", "brotli", "cchardet"] [[package]] name = "aioitertools" @@ -179,10 +76,6 @@ description = "itertools and builtins for AsyncIO and mixed iterables" category = "main" optional = true python-versions = ">=3.6" -files = [ - {file = "aioitertools-0.11.0-py3-none-any.whl", hash = "sha256:04b95e3dab25b449def24d7df809411c10e62aab0cbe31a50ca4e68748c43394"}, - {file = "aioitertools-0.11.0.tar.gz", hash = "sha256:42c68b8dd3a69c2bf7f2233bf7df4bb58b557bca5252ac02ed5187bbc67d6831"}, -] [package.dependencies] typing_extensions = {version = ">=4.0", markers = "python_version < \"3.10\""} @@ -194,10 +87,6 @@ description = "aiosignal: a list of registered asynchronous callbacks" category = "main" optional = true python-versions = ">=3.7" -files = [ - {file = "aiosignal-1.3.1-py3-none-any.whl", hash = "sha256:f8376fb07dd1e86a584e4fcdec80b36b7f81aac666ebc724e2c090300dd83b17"}, - {file = "aiosignal-1.3.1.tar.gz", hash = "sha256:54cd96e15e1649b75d6c87526a6ff0b6c1b0dd3459f43d9ca11d48c339b68cfc"}, -] [package.dependencies] frozenlist = ">=1.1.0" @@ -209,10 +98,6 @@ description = "Timeout context manager for asyncio programs" category = "main" optional = true python-versions = ">=3.6" -files = [ - {file = "async-timeout-4.0.2.tar.gz", hash = "sha256:2163e1640ddb52b7a8c80d0a67a08587e5d245cc9c553a74a847056bc2976b15"}, - {file = "async_timeout-4.0.2-py3-none-any.whl", hash = "sha256:8ca1e4fcf50d07413d66d1a5e416e42cfdf5851c981d679a09851a6853383b3c"}, -] [[package]] name = "attrs" @@ -221,29 +106,22 @@ description = "Classes Without Boilerplate" category = "main" optional = false python-versions = ">=3.6" -files = [ - {file = "attrs-22.2.0-py3-none-any.whl", hash = "sha256:29e95c7f6778868dbd49170f98f8818f78f3dc5e0e37c0b1f474e3561b240836"}, - {file = "attrs-22.2.0.tar.gz", hash = "sha256:c9227bfc2f01993c03f68db37d1d15c9690188323c067c641f1a35ca58185f99"}, -] [package.extras] -cov = ["attrs[tests]", "coverage-enable-subprocess", "coverage[toml] (>=5.3)"] -dev = ["attrs[docs,tests]"] +cov = ["attrs", "coverage-enable-subprocess", "coverage[toml] (>=5.3)"] +dev = ["attrs"] docs = ["furo", "myst-parser", "sphinx", "sphinx-notfound-page", "sphinxcontrib-towncrier", "towncrier", "zope.interface"] -tests = ["attrs[tests-no-zope]", "zope.interface"] -tests-no-zope = ["cloudpickle", "cloudpickle", "hypothesis", "hypothesis", "mypy (>=0.971,<0.990)", "mypy (>=0.971,<0.990)", "pympler", "pympler", "pytest (>=4.3.0)", "pytest (>=4.3.0)", "pytest-mypy-plugins", "pytest-mypy-plugins", "pytest-xdist[psutil]", "pytest-xdist[psutil]"] +tests = ["attrs", "zope.interface"] +tests-no-zope = ["cloudpickle", "hypothesis", "mypy (>=0.971,<0.990)", "pympler", "pytest (>=4.3.0)", "pytest-mypy-plugins", "pytest-xdist"] +tests_no_zope = ["cloudpickle", "hypothesis", "mypy (>=0.971,<0.990)", "pympler", "pytest (>=4.3.0)", "pytest-mypy-plugins", "pytest-xdist"] [[package]] name = "azure-core" -version = "1.26.2" +version = "1.26.3" description = "Microsoft Azure Core Library for Python" category = "main" optional = true python-versions = ">=3.7" -files = [ - {file = "azure-core-1.26.2.zip", hash = "sha256:986bfd8687889782d79481d4c5d0af04ab4a18ca2f210364804a88e4eaa1586a"}, - {file = "azure_core-1.26.2-py3-none-any.whl", hash = "sha256:df306e6e4abc145610ca6744aef943129a6fd7a11977e56731f69ac0e00724f9"}, -] [package.dependencies] requests = ">=2.18.4" @@ -260,10 +138,6 @@ description = "Azure Data Lake Store Filesystem Client Library for Python" category = "main" optional = true python-versions = "*" -files = [ - {file = "azure-datalake-store-0.0.52.tar.gz", hash = "sha256:4198ddb32614d16d4502b43d5c9739f81432b7e0e4d75d30e05149fe6007fea2"}, - {file = "azure_datalake_store-0.0.52-py2.py3-none-any.whl", hash = "sha256:aaed72b9c856824aeab554f4dbe0ef2c6d0ff36700bdd8b93d8298793117c48e"}, -] [package.dependencies] adal = ">=0.4.2" @@ -277,10 +151,6 @@ description = "Microsoft Azure Identity Library for Python" category = "main" optional = true python-versions = ">=3.7" -files = [ - {file = "azure-identity-1.12.0.zip", hash = "sha256:7f9b1ae7d97ea7af3f38dd09305e19ab81a1e16ab66ea186b6579d85c1ca2347"}, - {file = "azure_identity-1.12.0-py3-none-any.whl", hash = "sha256:2a58ce4a209a013e37eaccfd5937570ab99e9118b3e1acf875eed3a85d541b92"}, -] [package.dependencies] azure-core = ">=1.11.0,<2.0.0" @@ -296,10 +166,6 @@ description = "Microsoft Azure Blob Storage Client Library for Python" category = "main" optional = true python-versions = ">=3.7" -files = [ - {file = "azure-storage-blob-12.14.1.zip", hash = "sha256:860d4d82985a4bfc7d3271e71275af330f54f330a754355435a7ba749ccde997"}, - {file = "azure_storage_blob-12.14.1-py3-none-any.whl", hash = "sha256:52b84658e8df7853a3cf1c563814655b5028b979b2a87905b92aa6bb30be240e"}, -] [package.dependencies] azure-core = ">=1.24.2,<2.0.0" @@ -313,10 +179,6 @@ description = "The AWS SDK for Python" category = "main" optional = false python-versions = ">= 3.7" -files = [ - {file = "boto3-1.24.59-py3-none-any.whl", hash = "sha256:34ab44146a2c4e7f4e72737f4b27e6eb5e0a7855c2f4599e3d9199b6a0a2d575"}, - {file = "boto3-1.24.59.tar.gz", hash = "sha256:a50b4323f9579cfe22fcf5531fbd40b567d4d74c1adce06aeb5c95fce2a6fb40"}, -] [package.dependencies] botocore = ">=1.27.59,<1.28.0" @@ -333,10 +195,6 @@ description = "Low-level, data-driven core of boto 3." category = "main" optional = false python-versions = ">= 3.7" -files = [ - {file = "botocore-1.27.59-py3-none-any.whl", hash = "sha256:69d756791fc024bda54f6c53f71ae34e695ee41bbbc1743d9179c4837a4929da"}, - {file = "botocore-1.27.59.tar.gz", hash = "sha256:eda4aed6ee719a745d1288eaf1beb12f6f6448ad1fa12f159405db14ba9c92cf"}, -] [package.dependencies] jmespath = ">=0.7.1,<2.0.0" @@ -353,10 +211,6 @@ description = "A simple, correct Python build frontend" category = "dev" optional = false python-versions = ">= 3.7" -files = [ - {file = "build-0.10.0-py3-none-any.whl", hash = "sha256:af266720050a66c893a6096a2f410989eeac74ff9a68ba194b3f6473e8e26171"}, - {file = "build-0.10.0.tar.gz", hash = "sha256:d5b71264afdb5951d6704482aac78de887c80691c52b88a9ad195983ca2c9269"}, -] [package.dependencies] colorama = {version = "*", markers = "os_name == \"nt\""} @@ -377,10 +231,6 @@ description = "Python package for providing Mozilla's CA Bundle." category = "main" optional = false python-versions = ">=3.6" -files = [ - {file = "certifi-2022.12.7-py3-none-any.whl", hash = "sha256:4ad3232f5e926d6718ec31cfc1fcadfde020920e278684144551c91769c7bc18"}, - {file = "certifi-2022.12.7.tar.gz", hash = "sha256:35824b4c3a97115964b408844d64aa14db1cc518f6562e8d7261699d1350a9e3"}, -] [[package]] name = "cffi" @@ -389,72 +239,6 @@ description = "Foreign Function Interface for Python calling C code." category = "main" optional = false python-versions = "*" -files = [ - {file = "cffi-1.15.1-cp27-cp27m-macosx_10_9_x86_64.whl", hash = "sha256:a66d3508133af6e8548451b25058d5812812ec3798c886bf38ed24a98216fab2"}, - {file = "cffi-1.15.1-cp27-cp27m-manylinux1_i686.whl", hash = "sha256:470c103ae716238bbe698d67ad020e1db9d9dba34fa5a899b5e21577e6d52ed2"}, - {file = "cffi-1.15.1-cp27-cp27m-manylinux1_x86_64.whl", hash = "sha256:9ad5db27f9cabae298d151c85cf2bad1d359a1b9c686a275df03385758e2f914"}, - {file = "cffi-1.15.1-cp27-cp27m-win32.whl", hash = "sha256:b3bbeb01c2b273cca1e1e0c5df57f12dce9a4dd331b4fa1635b8bec26350bde3"}, - {file = "cffi-1.15.1-cp27-cp27m-win_amd64.whl", hash = "sha256:e00b098126fd45523dd056d2efba6c5a63b71ffe9f2bbe1a4fe1716e1d0c331e"}, - {file = "cffi-1.15.1-cp27-cp27mu-manylinux1_i686.whl", hash = "sha256:d61f4695e6c866a23a21acab0509af1cdfd2c013cf256bbf5b6b5e2695827162"}, - {file = "cffi-1.15.1-cp27-cp27mu-manylinux1_x86_64.whl", hash = "sha256:ed9cb427ba5504c1dc15ede7d516b84757c3e3d7868ccc85121d9310d27eed0b"}, - {file = "cffi-1.15.1-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:39d39875251ca8f612b6f33e6b1195af86d1b3e60086068be9cc053aa4376e21"}, - {file = "cffi-1.15.1-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:285d29981935eb726a4399badae8f0ffdff4f5050eaa6d0cfc3f64b857b77185"}, - {file = "cffi-1.15.1-cp310-cp310-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:3eb6971dcff08619f8d91607cfc726518b6fa2a9eba42856be181c6d0d9515fd"}, - {file = "cffi-1.15.1-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:21157295583fe8943475029ed5abdcf71eb3911894724e360acff1d61c1d54bc"}, - {file = "cffi-1.15.1-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:5635bd9cb9731e6d4a1132a498dd34f764034a8ce60cef4f5319c0541159392f"}, - {file = "cffi-1.15.1-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:2012c72d854c2d03e45d06ae57f40d78e5770d252f195b93f581acf3ba44496e"}, - {file = "cffi-1.15.1-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:dd86c085fae2efd48ac91dd7ccffcfc0571387fe1193d33b6394db7ef31fe2a4"}, - {file = "cffi-1.15.1-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:fa6693661a4c91757f4412306191b6dc88c1703f780c8234035eac011922bc01"}, - {file = "cffi-1.15.1-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:59c0b02d0a6c384d453fece7566d1c7e6b7bae4fc5874ef2ef46d56776d61c9e"}, - {file = "cffi-1.15.1-cp310-cp310-win32.whl", hash = "sha256:cba9d6b9a7d64d4bd46167096fc9d2f835e25d7e4c121fb2ddfc6528fb0413b2"}, - {file = "cffi-1.15.1-cp310-cp310-win_amd64.whl", hash = "sha256:ce4bcc037df4fc5e3d184794f27bdaab018943698f4ca31630bc7f84a7b69c6d"}, - {file = "cffi-1.15.1-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:3d08afd128ddaa624a48cf2b859afef385b720bb4b43df214f85616922e6a5ac"}, - {file = "cffi-1.15.1-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:3799aecf2e17cf585d977b780ce79ff0dc9b78d799fc694221ce814c2c19db83"}, - {file = "cffi-1.15.1-cp311-cp311-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:a591fe9e525846e4d154205572a029f653ada1a78b93697f3b5a8f1f2bc055b9"}, - {file = "cffi-1.15.1-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:3548db281cd7d2561c9ad9984681c95f7b0e38881201e157833a2342c30d5e8c"}, - {file = "cffi-1.15.1-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:91fc98adde3d7881af9b59ed0294046f3806221863722ba7d8d120c575314325"}, - {file = "cffi-1.15.1-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:94411f22c3985acaec6f83c6df553f2dbe17b698cc7f8ae751ff2237d96b9e3c"}, - {file = "cffi-1.15.1-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:03425bdae262c76aad70202debd780501fabeaca237cdfddc008987c0e0f59ef"}, - {file = "cffi-1.15.1-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:cc4d65aeeaa04136a12677d3dd0b1c0c94dc43abac5860ab33cceb42b801c1e8"}, - {file = "cffi-1.15.1-cp311-cp311-win32.whl", hash = "sha256:a0f100c8912c114ff53e1202d0078b425bee3649ae34d7b070e9697f93c5d52d"}, - {file = "cffi-1.15.1-cp311-cp311-win_amd64.whl", hash = "sha256:04ed324bda3cda42b9b695d51bb7d54b680b9719cfab04227cdd1e04e5de3104"}, - {file = "cffi-1.15.1-cp36-cp36m-macosx_10_9_x86_64.whl", hash = "sha256:50a74364d85fd319352182ef59c5c790484a336f6db772c1a9231f1c3ed0cbd7"}, - {file = "cffi-1.15.1-cp36-cp36m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:e263d77ee3dd201c3a142934a086a4450861778baaeeb45db4591ef65550b0a6"}, - {file = "cffi-1.15.1-cp36-cp36m-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:cec7d9412a9102bdc577382c3929b337320c4c4c4849f2c5cdd14d7368c5562d"}, - {file = "cffi-1.15.1-cp36-cp36m-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:4289fc34b2f5316fbb762d75362931e351941fa95fa18789191b33fc4cf9504a"}, - {file = "cffi-1.15.1-cp36-cp36m-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:173379135477dc8cac4bc58f45db08ab45d228b3363adb7af79436135d028405"}, - {file = "cffi-1.15.1-cp36-cp36m-manylinux_2_5_x86_64.manylinux1_x86_64.whl", hash = "sha256:6975a3fac6bc83c4a65c9f9fcab9e47019a11d3d2cf7f3c0d03431bf145a941e"}, - {file = "cffi-1.15.1-cp36-cp36m-win32.whl", hash = "sha256:2470043b93ff09bf8fb1d46d1cb756ce6132c54826661a32d4e4d132e1977adf"}, - {file = "cffi-1.15.1-cp36-cp36m-win_amd64.whl", hash = "sha256:30d78fbc8ebf9c92c9b7823ee18eb92f2e6ef79b45ac84db507f52fbe3ec4497"}, - {file = "cffi-1.15.1-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:198caafb44239b60e252492445da556afafc7d1e3ab7a1fb3f0584ef6d742375"}, - {file = "cffi-1.15.1-cp37-cp37m-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:5ef34d190326c3b1f822a5b7a45f6c4535e2f47ed06fec77d3d799c450b2651e"}, - {file = "cffi-1.15.1-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:8102eaf27e1e448db915d08afa8b41d6c7ca7a04b7d73af6514df10a3e74bd82"}, - {file = "cffi-1.15.1-cp37-cp37m-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:5df2768244d19ab7f60546d0c7c63ce1581f7af8b5de3eb3004b9b6fc8a9f84b"}, - {file = "cffi-1.15.1-cp37-cp37m-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:a8c4917bd7ad33e8eb21e9a5bbba979b49d9a97acb3a803092cbc1133e20343c"}, - {file = "cffi-1.15.1-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:0e2642fe3142e4cc4af0799748233ad6da94c62a8bec3a6648bf8ee68b1c7426"}, - {file = "cffi-1.15.1-cp37-cp37m-win32.whl", hash = "sha256:e229a521186c75c8ad9490854fd8bbdd9a0c9aa3a524326b55be83b54d4e0ad9"}, - {file = "cffi-1.15.1-cp37-cp37m-win_amd64.whl", hash = "sha256:a0b71b1b8fbf2b96e41c4d990244165e2c9be83d54962a9a1d118fd8657d2045"}, - {file = "cffi-1.15.1-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:320dab6e7cb2eacdf0e658569d2575c4dad258c0fcc794f46215e1e39f90f2c3"}, - {file = "cffi-1.15.1-cp38-cp38-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:1e74c6b51a9ed6589199c787bf5f9875612ca4a8a0785fb2d4a84429badaf22a"}, - {file = "cffi-1.15.1-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a5c84c68147988265e60416b57fc83425a78058853509c1b0629c180094904a5"}, - {file = "cffi-1.15.1-cp38-cp38-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:3b926aa83d1edb5aa5b427b4053dc420ec295a08e40911296b9eb1b6170f6cca"}, - {file = "cffi-1.15.1-cp38-cp38-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:87c450779d0914f2861b8526e035c5e6da0a3199d8f1add1a665e1cbc6fc6d02"}, - {file = "cffi-1.15.1-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:4f2c9f67e9821cad2e5f480bc8d83b8742896f1242dba247911072d4fa94c192"}, - {file = "cffi-1.15.1-cp38-cp38-win32.whl", hash = "sha256:8b7ee99e510d7b66cdb6c593f21c043c248537a32e0bedf02e01e9553a172314"}, - {file = "cffi-1.15.1-cp38-cp38-win_amd64.whl", hash = "sha256:00a9ed42e88df81ffae7a8ab6d9356b371399b91dbdf0c3cb1e84c03a13aceb5"}, - {file = "cffi-1.15.1-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:54a2db7b78338edd780e7ef7f9f6c442500fb0d41a5a4ea24fff1c929d5af585"}, - {file = "cffi-1.15.1-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:fcd131dd944808b5bdb38e6f5b53013c5aa4f334c5cad0c72742f6eba4b73db0"}, - {file = "cffi-1.15.1-cp39-cp39-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:7473e861101c9e72452f9bf8acb984947aa1661a7704553a9f6e4baa5ba64415"}, - {file = "cffi-1.15.1-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:6c9a799e985904922a4d207a94eae35c78ebae90e128f0c4e521ce339396be9d"}, - {file = "cffi-1.15.1-cp39-cp39-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:3bcde07039e586f91b45c88f8583ea7cf7a0770df3a1649627bf598332cb6984"}, - {file = "cffi-1.15.1-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:33ab79603146aace82c2427da5ca6e58f2b3f2fb5da893ceac0c42218a40be35"}, - {file = "cffi-1.15.1-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:5d598b938678ebf3c67377cdd45e09d431369c3b1a5b331058c338e201f12b27"}, - {file = "cffi-1.15.1-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:db0fbb9c62743ce59a9ff687eb5f4afbe77e5e8403d6697f7446e5f609976f76"}, - {file = "cffi-1.15.1-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:98d85c6a2bef81588d9227dde12db8a7f47f639f4a17c9ae08e773aa9c697bf3"}, - {file = "cffi-1.15.1-cp39-cp39-win32.whl", hash = "sha256:40f4774f5a9d4f5e344f31a32b5096977b5d48560c5592e2f3d2c4374bd543ee"}, - {file = "cffi-1.15.1-cp39-cp39-win_amd64.whl", hash = "sha256:70df4e3b545a17496c9b3f41f5115e69a4f2e77e94e1d2a8e1070bc0c38c8a3c"}, - {file = "cffi-1.15.1.tar.gz", hash = "sha256:d400bfb9a37b1351253cb402671cea7e89bdecc294e8016a707f6d1d8ac934f9"}, -] [package.dependencies] pycparser = "*" @@ -466,25 +250,14 @@ description = "Validate configuration and produce human readable error messages. category = "dev" optional = false python-versions = ">=3.6.1" -files = [ - {file = "cfgv-3.3.1-py2.py3-none-any.whl", hash = "sha256:c6a0883f3917a037485059700b9e75da2464e6c27051014ad85ba6aaa5884426"}, - {file = "cfgv-3.3.1.tar.gz", hash = "sha256:f5a830efb9ce7a445376bb66ec94c638a9787422f96264c98edc6bdeed8ab736"}, -] [[package]] name = "charset-normalizer" -version = "2.1.1" +version = "3.0.1" description = "The Real First Universal Charset Detector. Open, modern and actively maintained alternative to Chardet." category = "main" optional = false -python-versions = ">=3.6.0" -files = [ - {file = "charset-normalizer-2.1.1.tar.gz", hash = "sha256:5a3d016c7c547f69d6f81fb0db9449ce888b418b5b9952cc5e6e66843e9dd845"}, - {file = "charset_normalizer-2.1.1-py3-none-any.whl", hash = "sha256:83e9a75d1911279afd89352c68b45348559d1fc0506b054b346651b5e7fee29f"}, -] - -[package.extras] -unicode-backport = ["unicodedata2"] +python-versions = "*" [[package]] name = "click" @@ -493,10 +266,6 @@ description = "Composable command line interface toolkit" category = "main" optional = false python-versions = ">=3.7" -files = [ - {file = "click-8.1.3-py3-none-any.whl", hash = "sha256:bb4d8133cb15a609f44e8213d9b391b0809795062913b383c62be0ee95b1db48"}, - {file = "click-8.1.3.tar.gz", hash = "sha256:7682dc8afb30297001674575ea00d1814d808d6a36af415a82bd481d37ba7b8e"}, -] [package.dependencies] colorama = {version = "*", markers = "platform_system == \"Windows\""} @@ -508,10 +277,6 @@ description = "Cross-platform colored terminal text." category = "main" optional = false python-versions = "!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,!=3.4.*,!=3.5.*,!=3.6.*,>=2.7" -files = [ - {file = "colorama-0.4.6-py2.py3-none-any.whl", hash = "sha256:4f1d9991f5acc0ca119f9d443620b77f9d6b33703e51011c16baf57afb285fc6"}, - {file = "colorama-0.4.6.tar.gz", hash = "sha256:08695f5cb7ed6e0531a20572697297273c47b8cae5a63ffc6d6ed5c201be6e44"}, -] [[package]] name = "coverage" @@ -520,59 +285,6 @@ description = "Code coverage measurement for Python" category = "dev" optional = false python-versions = ">=3.7" -files = [ - {file = "coverage-7.1.0-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:3b946bbcd5a8231383450b195cfb58cb01cbe7f8949f5758566b881df4b33baf"}, - {file = "coverage-7.1.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:ec8e767f13be637d056f7e07e61d089e555f719b387a7070154ad80a0ff31801"}, - {file = "coverage-7.1.0-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:d4a5a5879a939cb84959d86869132b00176197ca561c664fc21478c1eee60d75"}, - {file = "coverage-7.1.0-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:b643cb30821e7570c0aaf54feaf0bfb630b79059f85741843e9dc23f33aaca2c"}, - {file = "coverage-7.1.0-cp310-cp310-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:32df215215f3af2c1617a55dbdfb403b772d463d54d219985ac7cd3bf124cada"}, - {file = "coverage-7.1.0-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:33d1ae9d4079e05ac4cc1ef9e20c648f5afabf1a92adfaf2ccf509c50b85717f"}, - {file = "coverage-7.1.0-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:29571503c37f2ef2138a306d23e7270687c0efb9cab4bd8038d609b5c2393a3a"}, - {file = "coverage-7.1.0-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:63ffd21aa133ff48c4dff7adcc46b7ec8b565491bfc371212122dd999812ea1c"}, - {file = "coverage-7.1.0-cp310-cp310-win32.whl", hash = "sha256:4b14d5e09c656de5038a3f9bfe5228f53439282abcab87317c9f7f1acb280352"}, - {file = "coverage-7.1.0-cp310-cp310-win_amd64.whl", hash = "sha256:8361be1c2c073919500b6601220a6f2f98ea0b6d2fec5014c1d9cfa23dd07038"}, - {file = "coverage-7.1.0-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:da9b41d4539eefd408c46725fb76ecba3a50a3367cafb7dea5f250d0653c1040"}, - {file = "coverage-7.1.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:c5b15ed7644ae4bee0ecf74fee95808dcc34ba6ace87e8dfbf5cb0dc20eab45a"}, - {file = "coverage-7.1.0-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:d12d076582507ea460ea2a89a8c85cb558f83406c8a41dd641d7be9a32e1274f"}, - {file = "coverage-7.1.0-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:e2617759031dae1bf183c16cef8fcfb3de7617f394c813fa5e8e46e9b82d4222"}, - {file = "coverage-7.1.0-cp311-cp311-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:c4e4881fa9e9667afcc742f0c244d9364d197490fbc91d12ac3b5de0bf2df146"}, - {file = "coverage-7.1.0-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:9d58885215094ab4a86a6aef044e42994a2bd76a446dc59b352622655ba6621b"}, - {file = "coverage-7.1.0-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:ffeeb38ee4a80a30a6877c5c4c359e5498eec095878f1581453202bfacc8fbc2"}, - {file = "coverage-7.1.0-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:3baf5f126f30781b5e93dbefcc8271cb2491647f8283f20ac54d12161dff080e"}, - {file = "coverage-7.1.0-cp311-cp311-win32.whl", hash = "sha256:ded59300d6330be27bc6cf0b74b89ada58069ced87c48eaf9344e5e84b0072f7"}, - {file = "coverage-7.1.0-cp311-cp311-win_amd64.whl", hash = "sha256:6a43c7823cd7427b4ed763aa7fb63901ca8288591323b58c9cd6ec31ad910f3c"}, - {file = "coverage-7.1.0-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:7a726d742816cb3a8973c8c9a97539c734b3a309345236cd533c4883dda05b8d"}, - {file = "coverage-7.1.0-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:bc7c85a150501286f8b56bd8ed3aa4093f4b88fb68c0843d21ff9656f0009d6a"}, - {file = "coverage-7.1.0-cp37-cp37m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:f5b4198d85a3755d27e64c52f8c95d6333119e49fd001ae5798dac872c95e0f8"}, - {file = "coverage-7.1.0-cp37-cp37m-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:ddb726cb861c3117a553f940372a495fe1078249ff5f8a5478c0576c7be12050"}, - {file = "coverage-7.1.0-cp37-cp37m-musllinux_1_1_aarch64.whl", hash = "sha256:51b236e764840a6df0661b67e50697aaa0e7d4124ca95e5058fa3d7cbc240b7c"}, - {file = "coverage-7.1.0-cp37-cp37m-musllinux_1_1_i686.whl", hash = "sha256:7ee5c9bb51695f80878faaa5598040dd6c9e172ddcf490382e8aedb8ec3fec8d"}, - {file = "coverage-7.1.0-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:c31b75ae466c053a98bf26843563b3b3517b8f37da4d47b1c582fdc703112bc3"}, - {file = "coverage-7.1.0-cp37-cp37m-win32.whl", hash = "sha256:3b155caf3760408d1cb903b21e6a97ad4e2bdad43cbc265e3ce0afb8e0057e73"}, - {file = "coverage-7.1.0-cp37-cp37m-win_amd64.whl", hash = "sha256:2a60d6513781e87047c3e630b33b4d1e89f39836dac6e069ffee28c4786715f5"}, - {file = "coverage-7.1.0-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:f2cba5c6db29ce991029b5e4ac51eb36774458f0a3b8d3137241b32d1bb91f06"}, - {file = "coverage-7.1.0-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:beeb129cacea34490ffd4d6153af70509aa3cda20fdda2ea1a2be870dfec8d52"}, - {file = "coverage-7.1.0-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:0c45948f613d5d18c9ec5eaa203ce06a653334cf1bd47c783a12d0dd4fd9c851"}, - {file = "coverage-7.1.0-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:ef382417db92ba23dfb5864a3fc9be27ea4894e86620d342a116b243ade5d35d"}, - {file = "coverage-7.1.0-cp38-cp38-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:7c7c0d0827e853315c9bbd43c1162c006dd808dbbe297db7ae66cd17b07830f0"}, - {file = "coverage-7.1.0-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:e5cdbb5cafcedea04924568d990e20ce7f1945a1dd54b560f879ee2d57226912"}, - {file = "coverage-7.1.0-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:9817733f0d3ea91bea80de0f79ef971ae94f81ca52f9b66500c6a2fea8e4b4f8"}, - {file = "coverage-7.1.0-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:218fe982371ac7387304153ecd51205f14e9d731b34fb0568181abaf7b443ba0"}, - {file = "coverage-7.1.0-cp38-cp38-win32.whl", hash = "sha256:04481245ef966fbd24ae9b9e537ce899ae584d521dfbe78f89cad003c38ca2ab"}, - {file = "coverage-7.1.0-cp38-cp38-win_amd64.whl", hash = "sha256:8ae125d1134bf236acba8b83e74c603d1b30e207266121e76484562bc816344c"}, - {file = "coverage-7.1.0-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:2bf1d5f2084c3932b56b962a683074a3692bce7cabd3aa023c987a2a8e7612f6"}, - {file = "coverage-7.1.0-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:98b85dd86514d889a2e3dd22ab3c18c9d0019e696478391d86708b805f4ea0fa"}, - {file = "coverage-7.1.0-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:38da2db80cc505a611938d8624801158e409928b136c8916cd2e203970dde4dc"}, - {file = "coverage-7.1.0-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:3164d31078fa9efe406e198aecd2a02d32a62fecbdef74f76dad6a46c7e48311"}, - {file = "coverage-7.1.0-cp39-cp39-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:db61a79c07331e88b9a9974815c075fbd812bc9dbc4dc44b366b5368a2936063"}, - {file = "coverage-7.1.0-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:9ccb092c9ede70b2517a57382a601619d20981f56f440eae7e4d7eaafd1d1d09"}, - {file = "coverage-7.1.0-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:33ff26d0f6cc3ca8de13d14fde1ff8efe1456b53e3f0273e63cc8b3c84a063d8"}, - {file = "coverage-7.1.0-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:d47dd659a4ee952e90dc56c97d78132573dc5c7b09d61b416a9deef4ebe01a0c"}, - {file = "coverage-7.1.0-cp39-cp39-win32.whl", hash = "sha256:d248cd4a92065a4d4543b8331660121b31c4148dd00a691bfb7a5cdc7483cfa4"}, - {file = "coverage-7.1.0-cp39-cp39-win_amd64.whl", hash = "sha256:7ed681b0f8e8bcbbffa58ba26fcf5dbc8f79e7997595bf071ed5430d8c08d6f3"}, - {file = "coverage-7.1.0-pp37.pp38.pp39-none-any.whl", hash = "sha256:755e89e32376c850f826c425ece2c35a4fc266c081490eb0a841e7c1cb0d3bda"}, - {file = "coverage-7.1.0.tar.gz", hash = "sha256:10188fe543560ec4874f974b5305cd1a8bdcfa885ee00ea3a03733464c4ca265"}, -] [package.dependencies] tomli = {version = "*", optional = true, markers = "python_full_version <= \"3.11.0a6\" and extra == \"toml\""} @@ -587,31 +299,6 @@ description = "cryptography is a package which provides cryptographic recipes an category = "main" optional = false python-versions = ">=3.6" -files = [ - {file = "cryptography-39.0.1-cp36-abi3-macosx_10_12_universal2.whl", hash = "sha256:6687ef6d0a6497e2b58e7c5b852b53f62142cfa7cd1555795758934da363a965"}, - {file = "cryptography-39.0.1-cp36-abi3-macosx_10_12_x86_64.whl", hash = "sha256:706843b48f9a3f9b9911979761c91541e3d90db1ca905fd63fee540a217698bc"}, - {file = "cryptography-39.0.1-cp36-abi3-manylinux_2_17_aarch64.manylinux2014_aarch64.manylinux_2_24_aarch64.whl", hash = "sha256:5d2d8b87a490bfcd407ed9d49093793d0f75198a35e6eb1a923ce1ee86c62b41"}, - {file = "cryptography-39.0.1-cp36-abi3-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:83e17b26de248c33f3acffb922748151d71827d6021d98c70e6c1a25ddd78505"}, - {file = "cryptography-39.0.1-cp36-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:e124352fd3db36a9d4a21c1aa27fd5d051e621845cb87fb851c08f4f75ce8be6"}, - {file = "cryptography-39.0.1-cp36-abi3-manylinux_2_24_x86_64.whl", hash = "sha256:5aa67414fcdfa22cf052e640cb5ddc461924a045cacf325cd164e65312d99502"}, - {file = "cryptography-39.0.1-cp36-abi3-manylinux_2_28_aarch64.whl", hash = "sha256:35f7c7d015d474f4011e859e93e789c87d21f6f4880ebdc29896a60403328f1f"}, - {file = "cryptography-39.0.1-cp36-abi3-manylinux_2_28_x86_64.whl", hash = "sha256:f24077a3b5298a5a06a8e0536e3ea9ec60e4c7ac486755e5fb6e6ea9b3500106"}, - {file = "cryptography-39.0.1-cp36-abi3-musllinux_1_1_aarch64.whl", hash = "sha256:f0c64d1bd842ca2633e74a1a28033d139368ad959872533b1bab8c80e8240a0c"}, - {file = "cryptography-39.0.1-cp36-abi3-musllinux_1_1_x86_64.whl", hash = "sha256:0f8da300b5c8af9f98111ffd512910bc792b4c77392a9523624680f7956a99d4"}, - {file = "cryptography-39.0.1-cp36-abi3-win32.whl", hash = "sha256:fe913f20024eb2cb2f323e42a64bdf2911bb9738a15dba7d3cce48151034e3a8"}, - {file = "cryptography-39.0.1-cp36-abi3-win_amd64.whl", hash = "sha256:ced4e447ae29ca194449a3f1ce132ded8fcab06971ef5f618605aacaa612beac"}, - {file = "cryptography-39.0.1-pp38-pypy38_pp73-macosx_10_12_x86_64.whl", hash = "sha256:807ce09d4434881ca3a7594733669bd834f5b2c6d5c7e36f8c00f691887042ad"}, - {file = "cryptography-39.0.1-pp38-pypy38_pp73-manylinux_2_28_aarch64.whl", hash = "sha256:c5caeb8188c24888c90b5108a441c106f7faa4c4c075a2bcae438c6e8ca73cef"}, - {file = "cryptography-39.0.1-pp38-pypy38_pp73-manylinux_2_28_x86_64.whl", hash = "sha256:4789d1e3e257965e960232345002262ede4d094d1a19f4d3b52e48d4d8f3b885"}, - {file = "cryptography-39.0.1-pp38-pypy38_pp73-win_amd64.whl", hash = "sha256:96f1157a7c08b5b189b16b47bc9db2332269d6680a196341bf30046330d15388"}, - {file = "cryptography-39.0.1-pp39-pypy39_pp73-macosx_10_12_x86_64.whl", hash = "sha256:e422abdec8b5fa8462aa016786680720d78bdce7a30c652b7fadf83a4ba35336"}, - {file = "cryptography-39.0.1-pp39-pypy39_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.manylinux_2_24_aarch64.whl", hash = "sha256:b0afd054cd42f3d213bf82c629efb1ee5f22eba35bf0eec88ea9ea7304f511a2"}, - {file = "cryptography-39.0.1-pp39-pypy39_pp73-manylinux_2_24_x86_64.whl", hash = "sha256:6f8ba7f0328b79f08bdacc3e4e66fb4d7aab0c3584e0bd41328dce5262e26b2e"}, - {file = "cryptography-39.0.1-pp39-pypy39_pp73-manylinux_2_28_aarch64.whl", hash = "sha256:ef8b72fa70b348724ff1218267e7f7375b8de4e8194d1636ee60510aae104cd0"}, - {file = "cryptography-39.0.1-pp39-pypy39_pp73-manylinux_2_28_x86_64.whl", hash = "sha256:aec5a6c9864be7df2240c382740fcf3b96928c46604eaa7f3091f58b878c0bb6"}, - {file = "cryptography-39.0.1-pp39-pypy39_pp73-win_amd64.whl", hash = "sha256:fdd188c8a6ef8769f148f88f859884507b954cc64db6b52f66ef199bb9ad660a"}, - {file = "cryptography-39.0.1.tar.gz", hash = "sha256:d1f6198ee6d9148405e49887803907fe8962a23e6c6f83ea7d98f1c0de375695"}, -] [package.dependencies] cffi = ">=1.12" @@ -633,10 +320,6 @@ description = "Distribution utilities" category = "dev" optional = false python-versions = "*" -files = [ - {file = "distlib-0.3.6-py2.py3-none-any.whl", hash = "sha256:f35c4b692542ca110de7ef0bea44d73981caeb34ca0b9b6b2e6d7790dda8f80e"}, - {file = "distlib-0.3.6.tar.gz", hash = "sha256:14bad2d9b04d3a36127ac97f30b12a19268f211063d8f8ee4f47108896e11b46"}, -] [[package]] name = "docutils" @@ -645,70 +328,14 @@ description = "Docutils -- Python Documentation Utilities" category = "dev" optional = false python-versions = ">=3.7" -files = [ - {file = "docutils-0.19-py3-none-any.whl", hash = "sha256:5e1de4d849fee02c63b040a4a3fd567f4ab104defd8a5511fbbc24a8a017efbc"}, - {file = "docutils-0.19.tar.gz", hash = "sha256:33995a6753c30b7f577febfc2c50411fec6aac7f7ffeb7c4cfe5991072dcf9e6"}, -] [[package]] name = "duckdb" -version = "0.6.1" +version = "0.7.0" description = "DuckDB embedded database" category = "main" optional = true python-versions = "*" -files = [ - {file = "duckdb-0.6.1-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:e566514f9327f89264e98ac14ee7a84fbd9857328028258422c3e8375ee19d25"}, - {file = "duckdb-0.6.1-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:b31c2883de5b19591a2852165e6b3f9821f77af649835f27bc146b26e4aa30cb"}, - {file = "duckdb-0.6.1-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:998165b2fb1f1d2b0ad742096015ea70878f7d40304643c7424c3ed3ddf07bfc"}, - {file = "duckdb-0.6.1-cp310-cp310-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:3941b3a1e8a1cdb7b90ab3917b87af816e71f9692e5ada7f19b6b60969f731e5"}, - {file = "duckdb-0.6.1-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:143611bd1b7c13343f087d4d423a7a8a4f33a114c5326171e867febf3f0fcfe1"}, - {file = "duckdb-0.6.1-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:125ba45e8b08f28858f918ec9cbd3a19975e5d8d9e8275ef4ad924028a616e14"}, - {file = "duckdb-0.6.1-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:e609a65b31c92f2f7166831f74b56f5ed54b33d8c2c4b4c3974c26fdc50464c5"}, - {file = "duckdb-0.6.1-cp310-cp310-win32.whl", hash = "sha256:b39045074fb9a3f068496475a5d627ad4fa572fa3b4980e3b479c11d0b706f2d"}, - {file = "duckdb-0.6.1-cp310-cp310-win_amd64.whl", hash = "sha256:16fa96ffaa3d842a9355a633fb8bc092d119be08d4bc02013946d8594417bc14"}, - {file = "duckdb-0.6.1-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:b4bbe2f6c1b109c626f9318eee80934ad2a5b81a51409c6b5083c6c5f9bdb125"}, - {file = "duckdb-0.6.1-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:cfea36b58928ce778d17280d4fb3bf0a2d7cff407667baedd69c5b41463ac0fd"}, - {file = "duckdb-0.6.1-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:0b64eb53d0d0695814bf1b65c0f91ab7ed66b515f89c88038f65ad5e0762571c"}, - {file = "duckdb-0.6.1-cp311-cp311-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:35b01bc724e1933293f4c34f410d2833bfbb56d5743b515d805bbfed0651476e"}, - {file = "duckdb-0.6.1-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:fec2c2466654ce786843bda2bfba71e0e4719106b41d36b17ceb1901e130aa71"}, - {file = "duckdb-0.6.1-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:82cd30f5cf368658ef879b1c60276bc8650cf67cfe3dc3e3009438ba39251333"}, - {file = "duckdb-0.6.1-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:a782bbfb7f5e97d4a9c834c9e78f023fb8b3f6687c22ca99841e6ed944b724da"}, - {file = "duckdb-0.6.1-cp311-cp311-win32.whl", hash = "sha256:e3702d4a9ade54c6403f6615a98bbec2020a76a60f5db7fcf085df1bd270e66e"}, - {file = "duckdb-0.6.1-cp311-cp311-win_amd64.whl", hash = "sha256:93b074f473d68c944b0eeb2edcafd91ad11da8432b484836efaaab4e26351d48"}, - {file = "duckdb-0.6.1-cp36-cp36m-macosx_10_9_x86_64.whl", hash = "sha256:adae183924d6d479202c39072e37d440b511326e84525bcb7432bca85f86caba"}, - {file = "duckdb-0.6.1-cp36-cp36m-win32.whl", hash = "sha256:546a1cd17595bd1dd009daf6f36705aa6f95337154360ce44932157d353dcd80"}, - {file = "duckdb-0.6.1-cp36-cp36m-win_amd64.whl", hash = "sha256:87b0d00eb9d1a7ebe437276203e0cdc93b4a2154ba9688c65e8d2a8735839ec6"}, - {file = "duckdb-0.6.1-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:8442e074de6e1969c3d2b24363a5a6d7f866d5ac3f4e358e357495b389eff6c1"}, - {file = "duckdb-0.6.1-cp37-cp37m-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:0a6bf2ae7bec803352dade14561cb0b461b2422e70f75d9f09b36ba2dad2613b"}, - {file = "duckdb-0.6.1-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:5054792f22733f89d9cbbced2bafd8772d72d0fe77f159310221cefcf981c680"}, - {file = "duckdb-0.6.1-cp37-cp37m-musllinux_1_1_i686.whl", hash = "sha256:21cc503dffc2c68bb825e4eb3098e82f40e910b3d09e1b3b7f090d39ad53fbea"}, - {file = "duckdb-0.6.1-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:54b3da77ad893e99c073087ff7f75a8c98154ac5139d317149f12b74367211db"}, - {file = "duckdb-0.6.1-cp37-cp37m-win32.whl", hash = "sha256:f1d709aa6a26172a3eab804b57763d5cdc1a4b785ac1fc2b09568578e52032ee"}, - {file = "duckdb-0.6.1-cp37-cp37m-win_amd64.whl", hash = "sha256:f4edcaa471d791393e37f63e3c7c728fa6324e3ac7e768b9dc2ea49065cd37cc"}, - {file = "duckdb-0.6.1-cp38-cp38-macosx_10_9_universal2.whl", hash = "sha256:d218c2dd3bda51fb79e622b7b2266183ac9493834b55010aa01273fa5b7a7105"}, - {file = "duckdb-0.6.1-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:0c7155cb93ab432eca44b651256c359281d26d927ff43badaf1d2276dd770832"}, - {file = "duckdb-0.6.1-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:0925778200090d3d5d8b6bb42b4d05d24db1e8912484ba3b7e7b7f8569f17dcb"}, - {file = "duckdb-0.6.1-cp38-cp38-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:8b544dd04bb851d08bc68b317a7683cec6091547ae75555d075f8c8a7edb626e"}, - {file = "duckdb-0.6.1-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:f2c37d5a0391cf3a3a66e63215968ffb78e6b84f659529fa4bd10478f6203071"}, - {file = "duckdb-0.6.1-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:ce376966260eb5c351fcc6af627a979dbbcae3efeb2e70f85b23aa45a21e289d"}, - {file = "duckdb-0.6.1-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:73c974b09dd08dff5e8bdedba11c7d0aa0fc46ca93954ee7d19e1e18c9883ac1"}, - {file = "duckdb-0.6.1-cp38-cp38-win32.whl", hash = "sha256:bfe39ed3a03e8b1ed764f58f513b37b24afe110d245803a41655d16d391ad9f1"}, - {file = "duckdb-0.6.1-cp38-cp38-win_amd64.whl", hash = "sha256:afa97d982dbe6b125631a17e222142e79bee88f7a13fc4cee92d09285e31ec83"}, - {file = "duckdb-0.6.1-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:c35ff4b1117096ef72d101524df0079da36c3735d52fcf1d907ccffa63bd6202"}, - {file = "duckdb-0.6.1-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:5c54910fbb6de0f21d562e18a5c91540c19876db61b862fc9ffc8e31be8b3f03"}, - {file = "duckdb-0.6.1-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:99a7172563a3ae67d867572ce27cf3962f58e76f491cb7f602f08c2af39213b3"}, - {file = "duckdb-0.6.1-cp39-cp39-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:7363ffe857d00216b659116647fbf1e925cb3895699015d4a4e50b746de13041"}, - {file = "duckdb-0.6.1-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:06c1cef25f896b2284ba048108f645c72fab5c54aa5a6f62f95663f44ff8a79b"}, - {file = "duckdb-0.6.1-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:e92dd6aad7e8c29d002947376b6f5ce28cae29eb3b6b58a64a46cdbfc5cb7943"}, - {file = "duckdb-0.6.1-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:4b280b2d8a01ecd4fe2feab041df70233c534fafbe33a38565b52c1e017529c7"}, - {file = "duckdb-0.6.1-cp39-cp39-win32.whl", hash = "sha256:d9212d76e90b8469743924a4d22bef845be310d0d193d54ae17d9ef1f753cfa7"}, - {file = "duckdb-0.6.1-cp39-cp39-win_amd64.whl", hash = "sha256:00b7be8f67ec1a8edaa8844f521267baa1a795f4c482bfad56c72c26e1862ab2"}, - {file = "duckdb-0.6.1.tar.gz", hash = "sha256:6d26e9f1afcb924a6057785e506810d48332d4764ddc4a5b414d0f2bf0cacfb4"}, -] - -[package.dependencies] -numpy = ">=1.14" [[package]] name = "exceptiongroup" @@ -717,10 +344,6 @@ description = "Backport of PEP 654 (exception groups)" category = "dev" optional = false python-versions = ">=3.7" -files = [ - {file = "exceptiongroup-1.1.0-py3-none-any.whl", hash = "sha256:327cbda3da756e2de031a3107b81ab7b3770a602c4d16ca618298c526f4bec1e"}, - {file = "exceptiongroup-1.1.0.tar.gz", hash = "sha256:bcb67d800a4497e1b404c2dd44fca47d3b7a5e5433dbab67f96c1a685cdfdf23"}, -] [package.extras] test = ["pytest (>=6)"] @@ -732,29 +355,6 @@ description = "Fast read/write of AVRO files" category = "dev" optional = false python-versions = ">=3.7" -files = [ - {file = "fastavro-1.7.1-cp310-cp310-macosx_11_0_x86_64.whl", hash = "sha256:f8f7f11af8c2c074341217d6247b7ba09cadcd55f899e046c14e3a44afa5fc95"}, - {file = "fastavro-1.7.1-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:11159cf23f5ff4b752b028a77bd2a7941599932527e8a6512779e25b0503f037"}, - {file = "fastavro-1.7.1-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:86ce89743711355eee7d3fec34847de1ab574f4567aa4a667e966711bb2e0cd9"}, - {file = "fastavro-1.7.1-cp310-cp310-win_amd64.whl", hash = "sha256:2e3f5ae42e033dbb6e0efa788c4b8a4e5a59bc2b9cb83f717b6d85176727faed"}, - {file = "fastavro-1.7.1-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:8b671a97e864e4c024061c0d6f93f2768ba0595f917317ca4fe3e99dca6fcf3d"}, - {file = "fastavro-1.7.1-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:e9be910ca1645e0e5c4192d667cfdfa58dff4f8690db5af1ae33602643d41a78"}, - {file = "fastavro-1.7.1-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:6f233a1d0d95265f81a25c274f55017bd39d9b8f7f1387a4235bf8e01841a9ff"}, - {file = "fastavro-1.7.1-cp311-cp311-win_amd64.whl", hash = "sha256:31c924ed2fdad56182b543941cdec9cc384443cc3ca9462e0580afeb4dc64f4b"}, - {file = "fastavro-1.7.1-cp37-cp37m-macosx_10_15_x86_64.whl", hash = "sha256:8ad522c7f2c7791cfe54a7e87af8ac09634872f4fdeef28deec97fab8de38b24"}, - {file = "fastavro-1.7.1-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:74fe0fb24a489e2fd567775935eb55a461fc6c4da8b5e3467245752ac2098284"}, - {file = "fastavro-1.7.1-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:4f7a744569b4b44ea1f4f87e7ea8298e1e2bf23779aa6ef255b95f9f38faad48"}, - {file = "fastavro-1.7.1-cp37-cp37m-win_amd64.whl", hash = "sha256:b793252a6325341890bbbbdca2804d4db3d5d29ff7f15a58fcf84dda440808fa"}, - {file = "fastavro-1.7.1-cp38-cp38-macosx_10_15_x86_64.whl", hash = "sha256:a107a8b333628c0f8a2ece5d5a57c69923719a139b106ace4050206250df4b13"}, - {file = "fastavro-1.7.1-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:182671595683054ec234beec03f485b5c889c21c08e429577ae7929480703409"}, - {file = "fastavro-1.7.1-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:980ce7fdccf328d287e8b789f4e3b422f64c84ed9cd81c05dd7c560c3d8076b1"}, - {file = "fastavro-1.7.1-cp38-cp38-win_amd64.whl", hash = "sha256:a5f104bc5b4986bbbcab170918c4d8ffa4f8efa3ebe8ec190954178630074d5a"}, - {file = "fastavro-1.7.1-cp39-cp39-macosx_11_0_x86_64.whl", hash = "sha256:5ff701d6b228218a3d9c09b392205dd0899afa501a4f14724bef0ce13a719700"}, - {file = "fastavro-1.7.1-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:5cb2d8ea7b21493bb18ff7c68401edbc663bbd8a57016d6cf2c4b0a2dc4464e7"}, - {file = "fastavro-1.7.1-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:e15e8163581983f07a3156902761cf746bbe1e646abd9553cc9a1cede6e23ae9"}, - {file = "fastavro-1.7.1-cp39-cp39-win_amd64.whl", hash = "sha256:4d56cc05ceb1c9f4de19c1f330a4f140af3b944834d63cd0e11517deaea21ee1"}, - {file = "fastavro-1.7.1.tar.gz", hash = "sha256:4b8bcae4ed6343af186e638061cdfbc5331cdb5e026d055099c91d4f07be838c"}, -] [package.extras] codecs = ["lz4", "python-snappy", "zstandard"] @@ -769,10 +369,6 @@ description = "A platform independent file lock." category = "dev" optional = false python-versions = ">=3.7" -files = [ - {file = "filelock-3.9.0-py3-none-any.whl", hash = "sha256:f58d535af89bb9ad5cd4df046f741f8553a418c01a7856bf0d173bbc9f6bd16d"}, - {file = "filelock-3.9.0.tar.gz", hash = "sha256:7b319f24340b51f55a2bf7a12ac0755a9b03e718311dac567a0f4f7fabd2f5de"}, -] [package.extras] docs = ["furo (>=2022.12.7)", "sphinx (>=5.3)", "sphinx-autodoc-typehints (>=1.19.5)"] @@ -785,82 +381,6 @@ description = "A list-like structure which implements collections.abc.MutableSeq category = "main" optional = true python-versions = ">=3.7" -files = [ - {file = "frozenlist-1.3.3-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:ff8bf625fe85e119553b5383ba0fb6aa3d0ec2ae980295aaefa552374926b3f4"}, - {file = "frozenlist-1.3.3-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:dfbac4c2dfcc082fcf8d942d1e49b6aa0766c19d3358bd86e2000bf0fa4a9cf0"}, - {file = "frozenlist-1.3.3-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:b1c63e8d377d039ac769cd0926558bb7068a1f7abb0f003e3717ee003ad85530"}, - {file = "frozenlist-1.3.3-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:7fdfc24dcfce5b48109867c13b4cb15e4660e7bd7661741a391f821f23dfdca7"}, - {file = "frozenlist-1.3.3-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:2c926450857408e42f0bbc295e84395722ce74bae69a3b2aa2a65fe22cb14b99"}, - {file = "frozenlist-1.3.3-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:1841e200fdafc3d51f974d9d377c079a0694a8f06de2e67b48150328d66d5483"}, - {file = "frozenlist-1.3.3-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:f470c92737afa7d4c3aacc001e335062d582053d4dbe73cda126f2d7031068dd"}, - {file = "frozenlist-1.3.3-cp310-cp310-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:783263a4eaad7c49983fe4b2e7b53fa9770c136c270d2d4bbb6d2192bf4d9caf"}, - {file = "frozenlist-1.3.3-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:924620eef691990dfb56dc4709f280f40baee568c794b5c1885800c3ecc69816"}, - {file = "frozenlist-1.3.3-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:ae4dc05c465a08a866b7a1baf360747078b362e6a6dbeb0c57f234db0ef88ae0"}, - {file = "frozenlist-1.3.3-cp310-cp310-musllinux_1_1_ppc64le.whl", hash = "sha256:bed331fe18f58d844d39ceb398b77d6ac0b010d571cba8267c2e7165806b00ce"}, - {file = "frozenlist-1.3.3-cp310-cp310-musllinux_1_1_s390x.whl", hash = "sha256:02c9ac843e3390826a265e331105efeab489ffaf4dd86384595ee8ce6d35ae7f"}, - {file = "frozenlist-1.3.3-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:9545a33965d0d377b0bc823dcabf26980e77f1b6a7caa368a365a9497fb09420"}, - {file = "frozenlist-1.3.3-cp310-cp310-win32.whl", hash = "sha256:d5cd3ab21acbdb414bb6c31958d7b06b85eeb40f66463c264a9b343a4e238642"}, - {file = "frozenlist-1.3.3-cp310-cp310-win_amd64.whl", hash = "sha256:b756072364347cb6aa5b60f9bc18e94b2f79632de3b0190253ad770c5df17db1"}, - {file = "frozenlist-1.3.3-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:b4395e2f8d83fbe0c627b2b696acce67868793d7d9750e90e39592b3626691b7"}, - {file = "frozenlist-1.3.3-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:14143ae966a6229350021384870458e4777d1eae4c28d1a7aa47f24d030e6678"}, - {file = "frozenlist-1.3.3-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:5d8860749e813a6f65bad8285a0520607c9500caa23fea6ee407e63debcdbef6"}, - {file = "frozenlist-1.3.3-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:23d16d9f477bb55b6154654e0e74557040575d9d19fe78a161bd33d7d76808e8"}, - {file = "frozenlist-1.3.3-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:eb82dbba47a8318e75f679690190c10a5e1f447fbf9df41cbc4c3afd726d88cb"}, - {file = "frozenlist-1.3.3-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:9309869032abb23d196cb4e4db574232abe8b8be1339026f489eeb34a4acfd91"}, - {file = "frozenlist-1.3.3-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:a97b4fe50b5890d36300820abd305694cb865ddb7885049587a5678215782a6b"}, - {file = "frozenlist-1.3.3-cp311-cp311-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:c188512b43542b1e91cadc3c6c915a82a5eb95929134faf7fd109f14f9892ce4"}, - {file = "frozenlist-1.3.3-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:303e04d422e9b911a09ad499b0368dc551e8c3cd15293c99160c7f1f07b59a48"}, - {file = "frozenlist-1.3.3-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:0771aed7f596c7d73444c847a1c16288937ef988dc04fb9f7be4b2aa91db609d"}, - {file = "frozenlist-1.3.3-cp311-cp311-musllinux_1_1_ppc64le.whl", hash = "sha256:66080ec69883597e4d026f2f71a231a1ee9887835902dbe6b6467d5a89216cf6"}, - {file = "frozenlist-1.3.3-cp311-cp311-musllinux_1_1_s390x.whl", hash = "sha256:41fe21dc74ad3a779c3d73a2786bdf622ea81234bdd4faf90b8b03cad0c2c0b4"}, - {file = "frozenlist-1.3.3-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:f20380df709d91525e4bee04746ba612a4df0972c1b8f8e1e8af997e678c7b81"}, - {file = "frozenlist-1.3.3-cp311-cp311-win32.whl", hash = "sha256:f30f1928162e189091cf4d9da2eac617bfe78ef907a761614ff577ef4edfb3c8"}, - {file = "frozenlist-1.3.3-cp311-cp311-win_amd64.whl", hash = "sha256:a6394d7dadd3cfe3f4b3b186e54d5d8504d44f2d58dcc89d693698e8b7132b32"}, - {file = "frozenlist-1.3.3-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:8df3de3a9ab8325f94f646609a66cbeeede263910c5c0de0101079ad541af332"}, - {file = "frozenlist-1.3.3-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:0693c609e9742c66ba4870bcee1ad5ff35462d5ffec18710b4ac89337ff16e27"}, - {file = "frozenlist-1.3.3-cp37-cp37m-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:cd4210baef299717db0a600d7a3cac81d46ef0e007f88c9335db79f8979c0d3d"}, - {file = "frozenlist-1.3.3-cp37-cp37m-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:394c9c242113bfb4b9aa36e2b80a05ffa163a30691c7b5a29eba82e937895d5e"}, - {file = "frozenlist-1.3.3-cp37-cp37m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:6327eb8e419f7d9c38f333cde41b9ae348bec26d840927332f17e887a8dcb70d"}, - {file = "frozenlist-1.3.3-cp37-cp37m-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:2e24900aa13212e75e5b366cb9065e78bbf3893d4baab6052d1aca10d46d944c"}, - {file = "frozenlist-1.3.3-cp37-cp37m-musllinux_1_1_aarch64.whl", hash = "sha256:3843f84a6c465a36559161e6c59dce2f2ac10943040c2fd021cfb70d58c4ad56"}, - {file = "frozenlist-1.3.3-cp37-cp37m-musllinux_1_1_i686.whl", hash = "sha256:84610c1502b2461255b4c9b7d5e9c48052601a8957cd0aea6ec7a7a1e1fb9420"}, - {file = "frozenlist-1.3.3-cp37-cp37m-musllinux_1_1_ppc64le.whl", hash = "sha256:c21b9aa40e08e4f63a2f92ff3748e6b6c84d717d033c7b3438dd3123ee18f70e"}, - {file = "frozenlist-1.3.3-cp37-cp37m-musllinux_1_1_s390x.whl", hash = "sha256:efce6ae830831ab6a22b9b4091d411698145cb9b8fc869e1397ccf4b4b6455cb"}, - {file = "frozenlist-1.3.3-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:40de71985e9042ca00b7953c4f41eabc3dc514a2d1ff534027f091bc74416401"}, - {file = "frozenlist-1.3.3-cp37-cp37m-win32.whl", hash = "sha256:180c00c66bde6146a860cbb81b54ee0df350d2daf13ca85b275123bbf85de18a"}, - {file = "frozenlist-1.3.3-cp37-cp37m-win_amd64.whl", hash = "sha256:9bbbcedd75acdfecf2159663b87f1bb5cfc80e7cd99f7ddd9d66eb98b14a8411"}, - {file = "frozenlist-1.3.3-cp38-cp38-macosx_10_9_universal2.whl", hash = "sha256:034a5c08d36649591be1cbb10e09da9f531034acfe29275fc5454a3b101ce41a"}, - {file = "frozenlist-1.3.3-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:ba64dc2b3b7b158c6660d49cdb1d872d1d0bf4e42043ad8d5006099479a194e5"}, - {file = "frozenlist-1.3.3-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:47df36a9fe24054b950bbc2db630d508cca3aa27ed0566c0baf661225e52c18e"}, - {file = "frozenlist-1.3.3-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:008a054b75d77c995ea26629ab3a0c0d7281341f2fa7e1e85fa6153ae29ae99c"}, - {file = "frozenlist-1.3.3-cp38-cp38-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:841ea19b43d438a80b4de62ac6ab21cfe6827bb8a9dc62b896acc88eaf9cecba"}, - {file = "frozenlist-1.3.3-cp38-cp38-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:e235688f42b36be2b6b06fc37ac2126a73b75fb8d6bc66dd632aa35286238703"}, - {file = "frozenlist-1.3.3-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:ca713d4af15bae6e5d79b15c10c8522859a9a89d3b361a50b817c98c2fb402a2"}, - {file = "frozenlist-1.3.3-cp38-cp38-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:9ac5995f2b408017b0be26d4a1d7c61bce106ff3d9e3324374d66b5964325448"}, - {file = "frozenlist-1.3.3-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:a4ae8135b11652b08a8baf07631d3ebfe65a4c87909dbef5fa0cdde440444ee4"}, - {file = "frozenlist-1.3.3-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:4ea42116ceb6bb16dbb7d526e242cb6747b08b7710d9782aa3d6732bd8d27649"}, - {file = "frozenlist-1.3.3-cp38-cp38-musllinux_1_1_ppc64le.whl", hash = "sha256:810860bb4bdce7557bc0febb84bbd88198b9dbc2022d8eebe5b3590b2ad6c842"}, - {file = "frozenlist-1.3.3-cp38-cp38-musllinux_1_1_s390x.whl", hash = "sha256:ee78feb9d293c323b59a6f2dd441b63339a30edf35abcb51187d2fc26e696d13"}, - {file = "frozenlist-1.3.3-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:0af2e7c87d35b38732e810befb9d797a99279cbb85374d42ea61c1e9d23094b3"}, - {file = "frozenlist-1.3.3-cp38-cp38-win32.whl", hash = "sha256:899c5e1928eec13fd6f6d8dc51be23f0d09c5281e40d9cf4273d188d9feeaf9b"}, - {file = "frozenlist-1.3.3-cp38-cp38-win_amd64.whl", hash = "sha256:7f44e24fa70f6fbc74aeec3e971f60a14dde85da364aa87f15d1be94ae75aeef"}, - {file = "frozenlist-1.3.3-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:2b07ae0c1edaa0a36339ec6cce700f51b14a3fc6545fdd32930d2c83917332cf"}, - {file = "frozenlist-1.3.3-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:ebb86518203e12e96af765ee89034a1dbb0c3c65052d1b0c19bbbd6af8a145e1"}, - {file = "frozenlist-1.3.3-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:5cf820485f1b4c91e0417ea0afd41ce5cf5965011b3c22c400f6d144296ccbc0"}, - {file = "frozenlist-1.3.3-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:5c11e43016b9024240212d2a65043b70ed8dfd3b52678a1271972702d990ac6d"}, - {file = "frozenlist-1.3.3-cp39-cp39-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:8fa3c6e3305aa1146b59a09b32b2e04074945ffcfb2f0931836d103a2c38f936"}, - {file = "frozenlist-1.3.3-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:352bd4c8c72d508778cf05ab491f6ef36149f4d0cb3c56b1b4302852255d05d5"}, - {file = "frozenlist-1.3.3-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:65a5e4d3aa679610ac6e3569e865425b23b372277f89b5ef06cf2cdaf1ebf22b"}, - {file = "frozenlist-1.3.3-cp39-cp39-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:b1e2c1185858d7e10ff045c496bbf90ae752c28b365fef2c09cf0fa309291669"}, - {file = "frozenlist-1.3.3-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:f163d2fd041c630fed01bc48d28c3ed4a3b003c00acd396900e11ee5316b56bb"}, - {file = "frozenlist-1.3.3-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:05cdb16d09a0832eedf770cb7bd1fe57d8cf4eaf5aced29c4e41e3f20b30a784"}, - {file = "frozenlist-1.3.3-cp39-cp39-musllinux_1_1_ppc64le.whl", hash = "sha256:8bae29d60768bfa8fb92244b74502b18fae55a80eac13c88eb0b496d4268fd2d"}, - {file = "frozenlist-1.3.3-cp39-cp39-musllinux_1_1_s390x.whl", hash = "sha256:eedab4c310c0299961ac285591acd53dc6723a1ebd90a57207c71f6e0c2153ab"}, - {file = "frozenlist-1.3.3-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:3bbdf44855ed8f0fbcd102ef05ec3012d6a4fd7c7562403f76ce6a52aeffb2b1"}, - {file = "frozenlist-1.3.3-cp39-cp39-win32.whl", hash = "sha256:efa568b885bca461f7c7b9e032655c0c143d305bf01c30caf6db2854a4532b38"}, - {file = "frozenlist-1.3.3-cp39-cp39-win_amd64.whl", hash = "sha256:cfe33efc9cb900a4c46f91a5ceba26d6df370ffddd9ca386eb1d4f0ad97b9ea9"}, - {file = "frozenlist-1.3.3.tar.gz", hash = "sha256:58bcc55721e8a90b88332d6cd441261ebb22342e238296bb330968952fbb3a6a"}, -] [[package]] name = "fsspec" @@ -869,10 +389,6 @@ description = "File-system specification" category = "main" optional = false python-versions = ">=3.7" -files = [ - {file = "fsspec-2023.1.0-py3-none-any.whl", hash = "sha256:b833e2e541e9e8cde0ab549414187871243177feb3d344f9d27b25a93f5d8139"}, - {file = "fsspec-2023.1.0.tar.gz", hash = "sha256:fbae7f20ff801eb5f7d0bedf81f25c787c0dfac5e982d98fa3884a9cde2b5411"}, -] [package.extras] abfs = ["adlfs"] @@ -899,15 +415,11 @@ tqdm = ["tqdm"] [[package]] name = "identify" -version = "2.5.13" +version = "2.5.18" description = "File identification library for Python" category = "dev" optional = false python-versions = ">=3.7" -files = [ - {file = "identify-2.5.13-py2.py3-none-any.whl", hash = "sha256:8aa48ce56e38c28b6faa9f261075dea0a942dfbb42b341b4e711896cbb40f3f7"}, - {file = "identify-2.5.13.tar.gz", hash = "sha256:abb546bca6f470228785338a01b539de8a85bbf46491250ae03363956d8ebb10"}, -] [package.extras] license = ["ukkonen"] @@ -919,10 +431,6 @@ description = "Internationalized Domain Names in Applications (IDNA)" category = "main" optional = false python-versions = ">=3.5" -files = [ - {file = "idna-3.4-py3-none-any.whl", hash = "sha256:90b77e79eaa3eba6de819a0c442c0b4ceefc341a7a2ab77d7562bf49f425c5c2"}, - {file = "idna-3.4.tar.gz", hash = "sha256:814f528e8dead7d329833b91c5faa87d60bf71824cd12a7530b5526063d02cb4"}, -] [[package]] name = "importlib-metadata" @@ -931,10 +439,6 @@ description = "Read metadata from Python packages" category = "dev" optional = false python-versions = ">=3.7" -files = [ - {file = "importlib_metadata-6.0.0-py3-none-any.whl", hash = "sha256:7efb448ec9a5e313a57655d35aa54cd3e01b7e1fbcf72dce1bf06119420f5bad"}, - {file = "importlib_metadata-6.0.0.tar.gz", hash = "sha256:e354bedeb60efa6affdcc8ae121b73544a7aa74156d047311948f6d711cd378d"}, -] [package.dependencies] zipp = ">=0.5" @@ -951,10 +455,6 @@ description = "brain-dead simple config-ini parsing" category = "dev" optional = false python-versions = ">=3.7" -files = [ - {file = "iniconfig-2.0.0-py3-none-any.whl", hash = "sha256:b6a85871a79d2e3b22d2d1b94ac2824226a63c6b741c88f7ae975f18b6778374"}, - {file = "iniconfig-2.0.0.tar.gz", hash = "sha256:2d91e135bf72d31a410b17c16da610a82cb55f6b0477d1a902134b24a455b8b3"}, -] [[package]] name = "isodate" @@ -963,10 +463,6 @@ description = "An ISO 8601 date/time/duration parser and formatter" category = "main" optional = true python-versions = "*" -files = [ - {file = "isodate-0.6.1-py2.py3-none-any.whl", hash = "sha256:0751eece944162659049d35f4f549ed815792b38793f07cf73381c1c87cbed96"}, - {file = "isodate-0.6.1.tar.gz", hash = "sha256:48c5881de7e8b0a0d648cb024c8062dc84e7b840ed81e864c7614fd3c127bde9"}, -] [package.dependencies] six = "*" @@ -978,10 +474,6 @@ description = "A very fast and expressive template engine." category = "dev" optional = false python-versions = ">=3.7" -files = [ - {file = "Jinja2-3.1.2-py3-none-any.whl", hash = "sha256:6088930bfe239f0e6710546ab9c19c9ef35e29792895fed6e6e31a023a182a61"}, - {file = "Jinja2-3.1.2.tar.gz", hash = "sha256:31351a702a408a9e7595a8fc6150fc3f43bb6bf7e319770cbc0db9df9437e852"}, -] [package.dependencies] MarkupSafe = ">=2.0" @@ -996,10 +488,6 @@ description = "JSON Matching Expressions" category = "main" optional = false python-versions = ">=3.7" -files = [ - {file = "jmespath-1.0.1-py3-none-any.whl", hash = "sha256:02e2e4cc71b5bcab88332eebf907519190dd9e6e82107fa7f83b1003a6252980"}, - {file = "jmespath-1.0.1.tar.gz", hash = "sha256:90261b206d6defd58fdd5e85f478bf633a2901798906be2ad389150c5c60edbe"}, -] [[package]] name = "markdown-it-py" @@ -1008,22 +496,18 @@ description = "Python port of markdown-it. Markdown parsing, done right!" category = "main" optional = false python-versions = ">=3.7" -files = [ - {file = "markdown-it-py-2.1.0.tar.gz", hash = "sha256:cf7e59fed14b5ae17c0006eff14a2d9a00ed5f3a846148153899a0224e2c07da"}, - {file = "markdown_it_py-2.1.0-py3-none-any.whl", hash = "sha256:93de681e5c021a432c63147656fe21790bc01231e0cd2da73626f1aa3ac0fe27"}, -] [package.dependencies] mdurl = ">=0.1,<1.0" [package.extras] benchmarking = ["psutil", "pytest", "pytest-benchmark (>=3.2,<4.0)"] -code-style = ["pre-commit (==2.6)"] +code_style = ["pre-commit (==2.6)"] compare = ["commonmark (>=0.9.1,<0.10.0)", "markdown (>=3.3.6,<3.4.0)", "mistletoe (>=0.8.1,<0.9.0)", "mistune (>=2.0.2,<2.1.0)", "panflute (>=2.1.3,<2.2.0)"] linkify = ["linkify-it-py (>=1.0,<2.0)"] plugins = ["mdit-py-plugins"] profiling = ["gprof2dot"] -rtd = ["attrs", "myst-parser", "pyyaml", "sphinx", "sphinx-copybutton", "sphinx-design", "sphinx_book_theme"] +rtd = ["attrs", "myst-parser", "pyyaml", "sphinx", "sphinx-book-theme", "sphinx-copybutton", "sphinx-design"] testing = ["coverage", "pytest", "pytest-cov", "pytest-regressions"] [[package]] @@ -1033,58 +517,6 @@ description = "Safely add untrusted strings to HTML/XML markup." category = "dev" optional = false python-versions = ">=3.7" -files = [ - {file = "MarkupSafe-2.1.2-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:665a36ae6f8f20a4676b53224e33d456a6f5a72657d9c83c2aa00765072f31f7"}, - {file = "MarkupSafe-2.1.2-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:340bea174e9761308703ae988e982005aedf427de816d1afe98147668cc03036"}, - {file = "MarkupSafe-2.1.2-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:22152d00bf4a9c7c83960521fc558f55a1adbc0631fbb00a9471e097b19d72e1"}, - {file = "MarkupSafe-2.1.2-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:28057e985dace2f478e042eaa15606c7efccb700797660629da387eb289b9323"}, - {file = "MarkupSafe-2.1.2-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:ca244fa73f50a800cf8c3ebf7fd93149ec37f5cb9596aa8873ae2c1d23498601"}, - {file = "MarkupSafe-2.1.2-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:d9d971ec1e79906046aa3ca266de79eac42f1dbf3612a05dc9368125952bd1a1"}, - {file = "MarkupSafe-2.1.2-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:7e007132af78ea9df29495dbf7b5824cb71648d7133cf7848a2a5dd00d36f9ff"}, - {file = "MarkupSafe-2.1.2-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:7313ce6a199651c4ed9d7e4cfb4aa56fe923b1adf9af3b420ee14e6d9a73df65"}, - {file = "MarkupSafe-2.1.2-cp310-cp310-win32.whl", hash = "sha256:c4a549890a45f57f1ebf99c067a4ad0cb423a05544accaf2b065246827ed9603"}, - {file = "MarkupSafe-2.1.2-cp310-cp310-win_amd64.whl", hash = "sha256:835fb5e38fd89328e9c81067fd642b3593c33e1e17e2fdbf77f5676abb14a156"}, - {file = "MarkupSafe-2.1.2-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:2ec4f2d48ae59bbb9d1f9d7efb9236ab81429a764dedca114f5fdabbc3788013"}, - {file = "MarkupSafe-2.1.2-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:608e7073dfa9e38a85d38474c082d4281f4ce276ac0010224eaba11e929dd53a"}, - {file = "MarkupSafe-2.1.2-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:65608c35bfb8a76763f37036547f7adfd09270fbdbf96608be2bead319728fcd"}, - {file = "MarkupSafe-2.1.2-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:f2bfb563d0211ce16b63c7cb9395d2c682a23187f54c3d79bfec33e6705473c6"}, - {file = "MarkupSafe-2.1.2-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:da25303d91526aac3672ee6d49a2f3db2d9502a4a60b55519feb1a4c7714e07d"}, - {file = "MarkupSafe-2.1.2-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:9cad97ab29dfc3f0249b483412c85c8ef4766d96cdf9dcf5a1e3caa3f3661cf1"}, - {file = "MarkupSafe-2.1.2-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:085fd3201e7b12809f9e6e9bc1e5c96a368c8523fad5afb02afe3c051ae4afcc"}, - {file = "MarkupSafe-2.1.2-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:1bea30e9bf331f3fef67e0a3877b2288593c98a21ccb2cf29b74c581a4eb3af0"}, - {file = "MarkupSafe-2.1.2-cp311-cp311-win32.whl", hash = "sha256:7df70907e00c970c60b9ef2938d894a9381f38e6b9db73c5be35e59d92e06625"}, - {file = "MarkupSafe-2.1.2-cp311-cp311-win_amd64.whl", hash = "sha256:e55e40ff0cc8cc5c07996915ad367fa47da6b3fc091fdadca7f5403239c5fec3"}, - {file = "MarkupSafe-2.1.2-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:a6e40afa7f45939ca356f348c8e23048e02cb109ced1eb8420961b2f40fb373a"}, - {file = "MarkupSafe-2.1.2-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:cf877ab4ed6e302ec1d04952ca358b381a882fbd9d1b07cccbfd61783561f98a"}, - {file = "MarkupSafe-2.1.2-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:63ba06c9941e46fa389d389644e2d8225e0e3e5ebcc4ff1ea8506dce646f8c8a"}, - {file = "MarkupSafe-2.1.2-cp37-cp37m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:f1cd098434e83e656abf198f103a8207a8187c0fc110306691a2e94a78d0abb2"}, - {file = "MarkupSafe-2.1.2-cp37-cp37m-musllinux_1_1_aarch64.whl", hash = "sha256:55f44b440d491028addb3b88f72207d71eeebfb7b5dbf0643f7c023ae1fba619"}, - {file = "MarkupSafe-2.1.2-cp37-cp37m-musllinux_1_1_i686.whl", hash = "sha256:a6f2fcca746e8d5910e18782f976489939d54a91f9411c32051b4aab2bd7c513"}, - {file = "MarkupSafe-2.1.2-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:0b462104ba25f1ac006fdab8b6a01ebbfbce9ed37fd37fd4acd70c67c973e460"}, - {file = "MarkupSafe-2.1.2-cp37-cp37m-win32.whl", hash = "sha256:7668b52e102d0ed87cb082380a7e2e1e78737ddecdde129acadb0eccc5423859"}, - {file = "MarkupSafe-2.1.2-cp37-cp37m-win_amd64.whl", hash = "sha256:6d6607f98fcf17e534162f0709aaad3ab7a96032723d8ac8750ffe17ae5a0666"}, - {file = "MarkupSafe-2.1.2-cp38-cp38-macosx_10_9_universal2.whl", hash = "sha256:a806db027852538d2ad7555b203300173dd1b77ba116de92da9afbc3a3be3eed"}, - {file = "MarkupSafe-2.1.2-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:a4abaec6ca3ad8660690236d11bfe28dfd707778e2442b45addd2f086d6ef094"}, - {file = "MarkupSafe-2.1.2-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:f03a532d7dee1bed20bc4884194a16160a2de9ffc6354b3878ec9682bb623c54"}, - {file = "MarkupSafe-2.1.2-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:4cf06cdc1dda95223e9d2d3c58d3b178aa5dacb35ee7e3bbac10e4e1faacb419"}, - {file = "MarkupSafe-2.1.2-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:22731d79ed2eb25059ae3df1dfc9cb1546691cc41f4e3130fe6bfbc3ecbbecfa"}, - {file = "MarkupSafe-2.1.2-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:f8ffb705ffcf5ddd0e80b65ddf7bed7ee4f5a441ea7d3419e861a12eaf41af58"}, - {file = "MarkupSafe-2.1.2-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:8db032bf0ce9022a8e41a22598eefc802314e81b879ae093f36ce9ddf39ab1ba"}, - {file = "MarkupSafe-2.1.2-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:2298c859cfc5463f1b64bd55cb3e602528db6fa0f3cfd568d3605c50678f8f03"}, - {file = "MarkupSafe-2.1.2-cp38-cp38-win32.whl", hash = "sha256:50c42830a633fa0cf9e7d27664637532791bfc31c731a87b202d2d8ac40c3ea2"}, - {file = "MarkupSafe-2.1.2-cp38-cp38-win_amd64.whl", hash = "sha256:bb06feb762bade6bf3c8b844462274db0c76acc95c52abe8dbed28ae3d44a147"}, - {file = "MarkupSafe-2.1.2-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:99625a92da8229df6d44335e6fcc558a5037dd0a760e11d84be2260e6f37002f"}, - {file = "MarkupSafe-2.1.2-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:8bca7e26c1dd751236cfb0c6c72d4ad61d986e9a41bbf76cb445f69488b2a2bd"}, - {file = "MarkupSafe-2.1.2-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:40627dcf047dadb22cd25ea7ecfe9cbf3bbbad0482ee5920b582f3809c97654f"}, - {file = "MarkupSafe-2.1.2-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:40dfd3fefbef579ee058f139733ac336312663c6706d1163b82b3003fb1925c4"}, - {file = "MarkupSafe-2.1.2-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:090376d812fb6ac5f171e5938e82e7f2d7adc2b629101cec0db8b267815c85e2"}, - {file = "MarkupSafe-2.1.2-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:2e7821bffe00aa6bd07a23913b7f4e01328c3d5cc0b40b36c0bd81d362faeb65"}, - {file = "MarkupSafe-2.1.2-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:c0a33bc9f02c2b17c3ea382f91b4db0e6cde90b63b296422a939886a7a80de1c"}, - {file = "MarkupSafe-2.1.2-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:b8526c6d437855442cdd3d87eede9c425c4445ea011ca38d937db299382e6fa3"}, - {file = "MarkupSafe-2.1.2-cp39-cp39-win32.whl", hash = "sha256:137678c63c977754abe9086a3ec011e8fd985ab90631145dfb9294ad09c102a7"}, - {file = "MarkupSafe-2.1.2-cp39-cp39-win_amd64.whl", hash = "sha256:0576fe974b40a400449768941d5d0858cc624e3249dfd1e0c33674e5c7ca7aed"}, - {file = "MarkupSafe-2.1.2.tar.gz", hash = "sha256:abcabc8c2b26036d62d4c746381a6f7cf60aafcc653198ad678306986b09450d"}, -] [[package]] name = "mdurl" @@ -1093,10 +525,6 @@ description = "Markdown URL utilities" category = "main" optional = false python-versions = ">=3.7" -files = [ - {file = "mdurl-0.1.2-py3-none-any.whl", hash = "sha256:84008a41e51615a49fc9966191ff91509e3c40b939176e643fd50a5c2196b8f8"}, - {file = "mdurl-0.1.2.tar.gz", hash = "sha256:bb413d29f5eea38f31dd4754dd7377d4465116fb207585f97bf925588687c1ba"}, -] [[package]] name = "mmhash3" @@ -1105,42 +533,6 @@ description = "Python wrapper for MurmurHash (MurmurHash3), a set of fast and ro category = "main" optional = false python-versions = "*" -files = [ - {file = "mmhash3-3.0.1-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:47deea30cd8d3d5cd52dc740902a4c70383bfe8248eac29d0877fe63e03c2713"}, - {file = "mmhash3-3.0.1-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:ecdaf4d1de617818bf05cd526ca558db6010beeba7ea9e19f695f2bdcac0e0a4"}, - {file = "mmhash3-3.0.1-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:4675585617584e9e1aafa3a90ac0ac9a437257c507748d97de8b21977e9d6745"}, - {file = "mmhash3-3.0.1-cp310-cp310-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:ebfd0c2af09b41f0fe1dd260799bda90a0fc7eba4477ccaeb3951527700cd58f"}, - {file = "mmhash3-3.0.1-cp310-cp310-win32.whl", hash = "sha256:68587dec7b8acdb7528fd511e295d8b5ccfe26022923a69867e1822f0fdb4c44"}, - {file = "mmhash3-3.0.1-cp310-cp310-win_amd64.whl", hash = "sha256:54954ebe3a614f19ab4cfe24fa66ea093c493d9fac0533d002dd64c2540a0c99"}, - {file = "mmhash3-3.0.1-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:b172f3bd3220b0bf56fd7cc760fd8a9033def47103611d63fe867003079a1256"}, - {file = "mmhash3-3.0.1-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:de7895eafabc32f7c0c09a81a624921f536768de6e438e3de69e3e954a4d7072"}, - {file = "mmhash3-3.0.1-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:b4b0914effe4ddd8d33149e3508564c17719465b0cc81691c4fa50d5e0e14f80"}, - {file = "mmhash3-3.0.1-cp311-cp311-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:c0575050ac691475938df1ad03d8738c5bd1eadef62093e76157ebb7f2df0946"}, - {file = "mmhash3-3.0.1-cp311-cp311-win32.whl", hash = "sha256:22f92f0f88f28b215357acd346362fa9f7c9fffb436beb42cc4b442b676dbaa3"}, - {file = "mmhash3-3.0.1-cp311-cp311-win_amd64.whl", hash = "sha256:538240ab7936bf71b18304e5a7e7fd3c4c2fab103330ea99584bb4f777299a2b"}, - {file = "mmhash3-3.0.1-cp36-cp36m-macosx_10_9_x86_64.whl", hash = "sha256:ca791bfb311e36ce13998e4632262ed4b95da9d3461941e18b6690760171a045"}, - {file = "mmhash3-3.0.1-cp36-cp36m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:b41708f72c6aa2a49ada1f0b61e85c05cd8389ad31d463fd5bca29999a4d5f9c"}, - {file = "mmhash3-3.0.1-cp36-cp36m-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:a3ce9b4533ddc0a88ba045a27309714c3b127bd05e57fd252d1d5a71d4247ea7"}, - {file = "mmhash3-3.0.1-cp36-cp36m-win32.whl", hash = "sha256:bfafeb96fdeb10db8767d06e1f07b6fdcddba4aaa0dd15058561a49f7ae45345"}, - {file = "mmhash3-3.0.1-cp36-cp36m-win_amd64.whl", hash = "sha256:97fe077b24c948709ed2afc749bf6285d407bc54ff12c63d2dc86678c38a0b8e"}, - {file = "mmhash3-3.0.1-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:0cfd91ccd5fca1ba7ee925297131a15dfb94c714bfe6ba0fb3b1ca78b12bbfec"}, - {file = "mmhash3-3.0.1-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:6d51b1005233141ce7394531af40a3f0fc1f274467bf8dff44dcf7987924fe58"}, - {file = "mmhash3-3.0.1-cp37-cp37m-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:855c67b100e37df166acf79cdff58fa8f9f6c48be0d1e1b6e9ad0fa34a9661ef"}, - {file = "mmhash3-3.0.1-cp37-cp37m-win32.whl", hash = "sha256:bb3030df1334fd665427f8be8e8ce4f04aeab7f6010ce4f2c128f0099bdab96f"}, - {file = "mmhash3-3.0.1-cp37-cp37m-win_amd64.whl", hash = "sha256:1545e1177294afe4912d5a5e401c7fa9b799dd109b30289e7af74d5b07e7c474"}, - {file = "mmhash3-3.0.1-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:2479899e7dda834a671991a1098a691ab1c2eaa20c3e939d691ca4a19361cfe0"}, - {file = "mmhash3-3.0.1-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:c9056196d5e3d3d844433a63d806a683f710ab3aaf1c910550c7746464bc43ae"}, - {file = "mmhash3-3.0.1-cp38-cp38-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:a0d4c307af0bf70207305f70f131898be071d1b19a89f462b13487f5c25e8d4e"}, - {file = "mmhash3-3.0.1-cp38-cp38-win32.whl", hash = "sha256:5f885f65e329fd14bc38debac4a79eacf381e856965d9c65c4d1c6946ea190d0"}, - {file = "mmhash3-3.0.1-cp38-cp38-win_amd64.whl", hash = "sha256:3b42d0bda5e1cd22c18b16887b0521060fb59d0aaaaf033feacbc0a2492d20fe"}, - {file = "mmhash3-3.0.1-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:d3f333286bb87aa9dc6bd8e7960a55a27b011a401f24b889a50e6d219f65e7c9"}, - {file = "mmhash3-3.0.1-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:6b7ef2eb95a18bcd02ce0d3e047adde3a025afd96c1d266a8a0d44574f44a307"}, - {file = "mmhash3-3.0.1-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:d6ac8a5f511c60f341bf9cae462bb4941abb149d98464ba5f4f4548875b601c6"}, - {file = "mmhash3-3.0.1-cp39-cp39-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:efef9e632e6e248e46f52d108a5ebd1f28edaf427b7fd47ebf97dbff4b2cab81"}, - {file = "mmhash3-3.0.1-cp39-cp39-win32.whl", hash = "sha256:bdac06d72e448c67afb12e758b203d875e29d4097bb279a38a5649d44b518ba7"}, - {file = "mmhash3-3.0.1-cp39-cp39-win_amd64.whl", hash = "sha256:0baeaa20cac5f75ed14f28056826bd9d9c8b2354b382073f3fd5190708992a0d"}, - {file = "mmhash3-3.0.1.tar.gz", hash = "sha256:a00d68f4a1cc434b9501513c8a29e18ed1ddad383677d72b41d71d0d862348af"}, -] [[package]] name = "moto" @@ -1149,10 +541,6 @@ description = "" category = "dev" optional = false python-versions = ">=3.7" -files = [ - {file = "moto-4.1.2-py2.py3-none-any.whl", hash = "sha256:1b361ece638c74a657325378a259276f368aafce2f8be84f8143e69fa93ce8ec"}, - {file = "moto-4.1.2.tar.gz", hash = "sha256:63431733d2a02c7bd652ad71ec1da442a0e0d580cbac5eeb50d440a2ce066eac"}, -] [package.dependencies] boto3 = ">=1.9.201" @@ -1191,15 +579,11 @@ xray = ["aws-xray-sdk (>=0.93,!=0.96)", "setuptools"] [[package]] name = "msal" -version = "1.20.0" +version = "1.21.0" description = "The Microsoft Authentication Library (MSAL) for Python library enables your app to access the Microsoft Cloud by supporting authentication of users with Microsoft Azure Active Directory accounts (AAD) and Microsoft Accounts (MSA) using industry standard OAuth2 and OpenID Connect." category = "main" optional = true python-versions = "*" -files = [ - {file = "msal-1.20.0-py2.py3-none-any.whl", hash = "sha256:d2f1c26368ecdc28c8657d457352faa0b81b1845a7b889d8676787721ba86792"}, - {file = "msal-1.20.0.tar.gz", hash = "sha256:78344cd4c91d6134a593b5e3e45541e666e37b747ff8a6316c3668dd1e6ab6b2"}, -] [package.dependencies] cryptography = ">=0.6,<41" @@ -1207,7 +591,7 @@ PyJWT = {version = ">=1.0.0,<3", extras = ["crypto"]} requests = ">=2.0.0,<3" [package.extras] -broker = ["pymsalruntime (>=0.11.2,<0.14)"] +broker = ["pymsalruntime (>=0.13.2,<0.14)"] [[package]] name = "msal-extensions" @@ -1216,10 +600,6 @@ description = "Microsoft Authentication Library extensions (MSAL EX) provides a category = "main" optional = true python-versions = "*" -files = [ - {file = "msal-extensions-1.0.0.tar.gz", hash = "sha256:c676aba56b0cce3783de1b5c5ecfe828db998167875126ca4b47dc6436451354"}, - {file = "msal_extensions-1.0.0-py2.py3-none-any.whl", hash = "sha256:91e3db9620b822d0ed2b4d1850056a0f133cba04455e62f11612e40f5502f2ee"}, -] [package.dependencies] msal = ">=0.4.1,<2.0.0" @@ -1235,10 +615,6 @@ description = "AutoRest swagger generator Python client runtime." category = "main" optional = true python-versions = ">=3.6" -files = [ - {file = "msrest-0.7.1-py3-none-any.whl", hash = "sha256:21120a810e1233e5e6cc7fe40b474eeb4ec6f757a15d7cf86702c369f9567c32"}, - {file = "msrest-0.7.1.zip", hash = "sha256:6e7661f46f3afd88b75667b7187a92829924446c7ea1d169be8c4bb7eeb788b9"}, -] [package.dependencies] azure-core = ">=1.24.0" @@ -1257,82 +633,6 @@ description = "multidict implementation" category = "main" optional = true python-versions = ">=3.7" -files = [ - {file = "multidict-6.0.4-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:0b1a97283e0c85772d613878028fec909f003993e1007eafa715b24b377cb9b8"}, - {file = "multidict-6.0.4-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:eeb6dcc05e911516ae3d1f207d4b0520d07f54484c49dfc294d6e7d63b734171"}, - {file = "multidict-6.0.4-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:d6d635d5209b82a3492508cf5b365f3446afb65ae7ebd755e70e18f287b0adf7"}, - {file = "multidict-6.0.4-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:c048099e4c9e9d615545e2001d3d8a4380bd403e1a0578734e0d31703d1b0c0b"}, - {file = "multidict-6.0.4-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:ea20853c6dbbb53ed34cb4d080382169b6f4554d394015f1bef35e881bf83547"}, - {file = "multidict-6.0.4-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:16d232d4e5396c2efbbf4f6d4df89bfa905eb0d4dc5b3549d872ab898451f569"}, - {file = "multidict-6.0.4-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:36c63aaa167f6c6b04ef2c85704e93af16c11d20de1d133e39de6a0e84582a93"}, - {file = "multidict-6.0.4-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:64bdf1086b6043bf519869678f5f2757f473dee970d7abf6da91ec00acb9cb98"}, - {file = "multidict-6.0.4-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:43644e38f42e3af682690876cff722d301ac585c5b9e1eacc013b7a3f7b696a0"}, - {file = "multidict-6.0.4-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:7582a1d1030e15422262de9f58711774e02fa80df0d1578995c76214f6954988"}, - {file = "multidict-6.0.4-cp310-cp310-musllinux_1_1_ppc64le.whl", hash = "sha256:ddff9c4e225a63a5afab9dd15590432c22e8057e1a9a13d28ed128ecf047bbdc"}, - {file = "multidict-6.0.4-cp310-cp310-musllinux_1_1_s390x.whl", hash = "sha256:ee2a1ece51b9b9e7752e742cfb661d2a29e7bcdba2d27e66e28a99f1890e4fa0"}, - {file = "multidict-6.0.4-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:a2e4369eb3d47d2034032a26c7a80fcb21a2cb22e1173d761a162f11e562caa5"}, - {file = "multidict-6.0.4-cp310-cp310-win32.whl", hash = "sha256:574b7eae1ab267e5f8285f0fe881f17efe4b98c39a40858247720935b893bba8"}, - {file = "multidict-6.0.4-cp310-cp310-win_amd64.whl", hash = "sha256:4dcbb0906e38440fa3e325df2359ac6cb043df8e58c965bb45f4e406ecb162cc"}, - {file = "multidict-6.0.4-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:0dfad7a5a1e39c53ed00d2dd0c2e36aed4650936dc18fd9a1826a5ae1cad6f03"}, - {file = "multidict-6.0.4-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:64da238a09d6039e3bd39bb3aee9c21a5e34f28bfa5aa22518581f910ff94af3"}, - {file = "multidict-6.0.4-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:ff959bee35038c4624250473988b24f846cbeb2c6639de3602c073f10410ceba"}, - {file = "multidict-6.0.4-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:01a3a55bd90018c9c080fbb0b9f4891db37d148a0a18722b42f94694f8b6d4c9"}, - {file = "multidict-6.0.4-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:c5cb09abb18c1ea940fb99360ea0396f34d46566f157122c92dfa069d3e0e982"}, - {file = "multidict-6.0.4-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:666daae833559deb2d609afa4490b85830ab0dfca811a98b70a205621a6109fe"}, - {file = "multidict-6.0.4-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:11bdf3f5e1518b24530b8241529d2050014c884cf18b6fc69c0c2b30ca248710"}, - {file = "multidict-6.0.4-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:7d18748f2d30f94f498e852c67d61261c643b349b9d2a581131725595c45ec6c"}, - {file = "multidict-6.0.4-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:458f37be2d9e4c95e2d8866a851663cbc76e865b78395090786f6cd9b3bbf4f4"}, - {file = "multidict-6.0.4-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:b1a2eeedcead3a41694130495593a559a668f382eee0727352b9a41e1c45759a"}, - {file = "multidict-6.0.4-cp311-cp311-musllinux_1_1_ppc64le.whl", hash = "sha256:7d6ae9d593ef8641544d6263c7fa6408cc90370c8cb2bbb65f8d43e5b0351d9c"}, - {file = "multidict-6.0.4-cp311-cp311-musllinux_1_1_s390x.whl", hash = "sha256:5979b5632c3e3534e42ca6ff856bb24b2e3071b37861c2c727ce220d80eee9ed"}, - {file = "multidict-6.0.4-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:dcfe792765fab89c365123c81046ad4103fcabbc4f56d1c1997e6715e8015461"}, - {file = "multidict-6.0.4-cp311-cp311-win32.whl", hash = "sha256:3601a3cece3819534b11d4efc1eb76047488fddd0c85a3948099d5da4d504636"}, - {file = "multidict-6.0.4-cp311-cp311-win_amd64.whl", hash = "sha256:81a4f0b34bd92df3da93315c6a59034df95866014ac08535fc819f043bfd51f0"}, - {file = "multidict-6.0.4-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:67040058f37a2a51ed8ea8f6b0e6ee5bd78ca67f169ce6122f3e2ec80dfe9b78"}, - {file = "multidict-6.0.4-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:853888594621e6604c978ce2a0444a1e6e70c8d253ab65ba11657659dcc9100f"}, - {file = "multidict-6.0.4-cp37-cp37m-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:39ff62e7d0f26c248b15e364517a72932a611a9b75f35b45be078d81bdb86603"}, - {file = "multidict-6.0.4-cp37-cp37m-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:af048912e045a2dc732847d33821a9d84ba553f5c5f028adbd364dd4765092ac"}, - {file = "multidict-6.0.4-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:b1e8b901e607795ec06c9e42530788c45ac21ef3aaa11dbd0c69de543bfb79a9"}, - {file = "multidict-6.0.4-cp37-cp37m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:62501642008a8b9871ddfccbf83e4222cf8ac0d5aeedf73da36153ef2ec222d2"}, - {file = "multidict-6.0.4-cp37-cp37m-musllinux_1_1_aarch64.whl", hash = "sha256:99b76c052e9f1bc0721f7541e5e8c05db3941eb9ebe7b8553c625ef88d6eefde"}, - {file = "multidict-6.0.4-cp37-cp37m-musllinux_1_1_i686.whl", hash = "sha256:509eac6cf09c794aa27bcacfd4d62c885cce62bef7b2c3e8b2e49d365b5003fe"}, - {file = "multidict-6.0.4-cp37-cp37m-musllinux_1_1_ppc64le.whl", hash = "sha256:21a12c4eb6ddc9952c415f24eef97e3e55ba3af61f67c7bc388dcdec1404a067"}, - {file = "multidict-6.0.4-cp37-cp37m-musllinux_1_1_s390x.whl", hash = "sha256:5cad9430ab3e2e4fa4a2ef4450f548768400a2ac635841bc2a56a2052cdbeb87"}, - {file = "multidict-6.0.4-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:ab55edc2e84460694295f401215f4a58597f8f7c9466faec545093045476327d"}, - {file = "multidict-6.0.4-cp37-cp37m-win32.whl", hash = "sha256:5a4dcf02b908c3b8b17a45fb0f15b695bf117a67b76b7ad18b73cf8e92608775"}, - {file = "multidict-6.0.4-cp37-cp37m-win_amd64.whl", hash = "sha256:6ed5f161328b7df384d71b07317f4d8656434e34591f20552c7bcef27b0ab88e"}, - {file = "multidict-6.0.4-cp38-cp38-macosx_10_9_universal2.whl", hash = "sha256:5fc1b16f586f049820c5c5b17bb4ee7583092fa0d1c4e28b5239181ff9532e0c"}, - {file = "multidict-6.0.4-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:1502e24330eb681bdaa3eb70d6358e818e8e8f908a22a1851dfd4e15bc2f8161"}, - {file = "multidict-6.0.4-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:b692f419760c0e65d060959df05f2a531945af31fda0c8a3b3195d4efd06de11"}, - {file = "multidict-6.0.4-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:45e1ecb0379bfaab5eef059f50115b54571acfbe422a14f668fc8c27ba410e7e"}, - {file = "multidict-6.0.4-cp38-cp38-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:ddd3915998d93fbcd2566ddf9cf62cdb35c9e093075f862935573d265cf8f65d"}, - {file = "multidict-6.0.4-cp38-cp38-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:59d43b61c59d82f2effb39a93c48b845efe23a3852d201ed2d24ba830d0b4cf2"}, - {file = "multidict-6.0.4-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:cc8e1d0c705233c5dd0c5e6460fbad7827d5d36f310a0fadfd45cc3029762258"}, - {file = "multidict-6.0.4-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:d6aa0418fcc838522256761b3415822626f866758ee0bc6632c9486b179d0b52"}, - {file = "multidict-6.0.4-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:6748717bb10339c4760c1e63da040f5f29f5ed6e59d76daee30305894069a660"}, - {file = "multidict-6.0.4-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:4d1a3d7ef5e96b1c9e92f973e43aa5e5b96c659c9bc3124acbbd81b0b9c8a951"}, - {file = "multidict-6.0.4-cp38-cp38-musllinux_1_1_ppc64le.whl", hash = "sha256:4372381634485bec7e46718edc71528024fcdc6f835baefe517b34a33c731d60"}, - {file = "multidict-6.0.4-cp38-cp38-musllinux_1_1_s390x.whl", hash = "sha256:fc35cb4676846ef752816d5be2193a1e8367b4c1397b74a565a9d0389c433a1d"}, - {file = "multidict-6.0.4-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:4b9d9e4e2b37daddb5c23ea33a3417901fa7c7b3dee2d855f63ee67a0b21e5b1"}, - {file = "multidict-6.0.4-cp38-cp38-win32.whl", hash = "sha256:e41b7e2b59679edfa309e8db64fdf22399eec4b0b24694e1b2104fb789207779"}, - {file = "multidict-6.0.4-cp38-cp38-win_amd64.whl", hash = "sha256:d6c254ba6e45d8e72739281ebc46ea5eb5f101234f3ce171f0e9f5cc86991480"}, - {file = "multidict-6.0.4-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:16ab77bbeb596e14212e7bab8429f24c1579234a3a462105cda4a66904998664"}, - {file = "multidict-6.0.4-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:bc779e9e6f7fda81b3f9aa58e3a6091d49ad528b11ed19f6621408806204ad35"}, - {file = "multidict-6.0.4-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:4ceef517eca3e03c1cceb22030a3e39cb399ac86bff4e426d4fc6ae49052cc60"}, - {file = "multidict-6.0.4-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:281af09f488903fde97923c7744bb001a9b23b039a909460d0f14edc7bf59706"}, - {file = "multidict-6.0.4-cp39-cp39-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:52f2dffc8acaba9a2f27174c41c9e57f60b907bb9f096b36b1a1f3be71c6284d"}, - {file = "multidict-6.0.4-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:b41156839806aecb3641f3208c0dafd3ac7775b9c4c422d82ee2a45c34ba81ca"}, - {file = "multidict-6.0.4-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:d5e3fc56f88cc98ef8139255cf8cd63eb2c586531e43310ff859d6bb3a6b51f1"}, - {file = "multidict-6.0.4-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:8316a77808c501004802f9beebde51c9f857054a0c871bd6da8280e718444449"}, - {file = "multidict-6.0.4-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:f70b98cd94886b49d91170ef23ec5c0e8ebb6f242d734ed7ed677b24d50c82cf"}, - {file = "multidict-6.0.4-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:bf6774e60d67a9efe02b3616fee22441d86fab4c6d335f9d2051d19d90a40063"}, - {file = "multidict-6.0.4-cp39-cp39-musllinux_1_1_ppc64le.whl", hash = "sha256:e69924bfcdda39b722ef4d9aa762b2dd38e4632b3641b1d9a57ca9cd18f2f83a"}, - {file = "multidict-6.0.4-cp39-cp39-musllinux_1_1_s390x.whl", hash = "sha256:6b181d8c23da913d4ff585afd1155a0e1194c0b50c54fcfe286f70cdaf2b7176"}, - {file = "multidict-6.0.4-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:52509b5be062d9eafc8170e53026fbc54cf3b32759a23d07fd935fb04fc22d95"}, - {file = "multidict-6.0.4-cp39-cp39-win32.whl", hash = "sha256:27c523fbfbdfd19c6867af7346332b62b586eed663887392cff78d614f9ec313"}, - {file = "multidict-6.0.4-cp39-cp39-win_amd64.whl", hash = "sha256:33029f5734336aa0d4c0384525da0387ef89148dc7191aae00ca5fb23d7aafc2"}, - {file = "multidict-6.0.4.tar.gz", hash = "sha256:3666906492efb76453c0e7b97f2cf459b0682e7402c0489a95484965dbc1da49"}, -] [[package]] name = "nodeenv" @@ -1341,51 +641,14 @@ description = "Node.js virtual environment builder" category = "dev" optional = false python-versions = ">=2.7,!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,!=3.4.*,!=3.5.*,!=3.6.*" -files = [ - {file = "nodeenv-1.7.0-py2.py3-none-any.whl", hash = "sha256:27083a7b96a25f2f5e1d8cb4b6317ee8aeda3bdd121394e5ac54e498028a042e"}, - {file = "nodeenv-1.7.0.tar.gz", hash = "sha256:e0e7f7dfb85fc5394c6fe1e8fa98131a2473e04311a45afb6508f7cf1836fa2b"}, -] - -[package.dependencies] -setuptools = "*" [[package]] name = "numpy" -version = "1.24.1" +version = "1.24.2" description = "Fundamental package for array computing in Python" category = "main" optional = true python-versions = ">=3.8" -files = [ - {file = "numpy-1.24.1-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:179a7ef0889ab769cc03573b6217f54c8bd8e16cef80aad369e1e8185f994cd7"}, - {file = "numpy-1.24.1-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:b09804ff570b907da323b3d762e74432fb07955701b17b08ff1b5ebaa8cfe6a9"}, - {file = "numpy-1.24.1-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:f1b739841821968798947d3afcefd386fa56da0caf97722a5de53e07c4ccedc7"}, - {file = "numpy-1.24.1-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:0e3463e6ac25313462e04aea3fb8a0a30fb906d5d300f58b3bc2c23da6a15398"}, - {file = "numpy-1.24.1-cp310-cp310-win32.whl", hash = "sha256:b31da69ed0c18be8b77bfce48d234e55d040793cebb25398e2a7d84199fbc7e2"}, - {file = "numpy-1.24.1-cp310-cp310-win_amd64.whl", hash = "sha256:b07b40f5fb4fa034120a5796288f24c1fe0e0580bbfff99897ba6267af42def2"}, - {file = "numpy-1.24.1-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:7094891dcf79ccc6bc2a1f30428fa5edb1e6fb955411ffff3401fb4ea93780a8"}, - {file = "numpy-1.24.1-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:28e418681372520c992805bb723e29d69d6b7aa411065f48216d8329d02ba032"}, - {file = "numpy-1.24.1-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:e274f0f6c7efd0d577744f52032fdd24344f11c5ae668fe8d01aac0422611df1"}, - {file = "numpy-1.24.1-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:0044f7d944ee882400890f9ae955220d29b33d809a038923d88e4e01d652acd9"}, - {file = "numpy-1.24.1-cp311-cp311-win32.whl", hash = "sha256:442feb5e5bada8408e8fcd43f3360b78683ff12a4444670a7d9e9824c1817d36"}, - {file = "numpy-1.24.1-cp311-cp311-win_amd64.whl", hash = "sha256:de92efa737875329b052982e37bd4371d52cabf469f83e7b8be9bb7752d67e51"}, - {file = "numpy-1.24.1-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:b162ac10ca38850510caf8ea33f89edcb7b0bb0dfa5592d59909419986b72407"}, - {file = "numpy-1.24.1-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:26089487086f2648944f17adaa1a97ca6aee57f513ba5f1c0b7ebdabbe2b9954"}, - {file = "numpy-1.24.1-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:caf65a396c0d1f9809596be2e444e3bd4190d86d5c1ce21f5fc4be60a3bc5b36"}, - {file = "numpy-1.24.1-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:b0677a52f5d896e84414761531947c7a330d1adc07c3a4372262f25d84af7bf7"}, - {file = "numpy-1.24.1-cp38-cp38-win32.whl", hash = "sha256:dae46bed2cb79a58d6496ff6d8da1e3b95ba09afeca2e277628171ca99b99db1"}, - {file = "numpy-1.24.1-cp38-cp38-win_amd64.whl", hash = "sha256:6ec0c021cd9fe732e5bab6401adea5a409214ca5592cd92a114f7067febcba0c"}, - {file = "numpy-1.24.1-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:28bc9750ae1f75264ee0f10561709b1462d450a4808cd97c013046073ae64ab6"}, - {file = "numpy-1.24.1-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:84e789a085aabef2f36c0515f45e459f02f570c4b4c4c108ac1179c34d475ed7"}, - {file = "numpy-1.24.1-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:8e669fbdcdd1e945691079c2cae335f3e3a56554e06bbd45d7609a6cf568c700"}, - {file = "numpy-1.24.1-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:ef85cf1f693c88c1fd229ccd1055570cb41cdf4875873b7728b6301f12cd05bf"}, - {file = "numpy-1.24.1-cp39-cp39-win32.whl", hash = "sha256:87a118968fba001b248aac90e502c0b13606721b1343cdaddbc6e552e8dfb56f"}, - {file = "numpy-1.24.1-cp39-cp39-win_amd64.whl", hash = "sha256:ddc7ab52b322eb1e40521eb422c4e0a20716c271a306860979d450decbb51b8e"}, - {file = "numpy-1.24.1-pp38-pypy38_pp73-macosx_10_9_x86_64.whl", hash = "sha256:ed5fb71d79e771ec930566fae9c02626b939e37271ec285e9efaf1b5d4370e7d"}, - {file = "numpy-1.24.1-pp38-pypy38_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:ad2925567f43643f51255220424c23d204024ed428afc5aad0f86f3ffc080086"}, - {file = "numpy-1.24.1-pp38-pypy38_pp73-win_amd64.whl", hash = "sha256:cfa1161c6ac8f92dea03d625c2d0c05e084668f4a06568b77a25a89111621566"}, - {file = "numpy-1.24.1.tar.gz", hash = "sha256:2386da9a471cc00a1f47845e27d916d5ec5346ae9696e01a8a34760858fe9dd2"}, -] [[package]] name = "oauthlib" @@ -1394,10 +657,6 @@ description = "A generic, spec-compliant, thorough implementation of the OAuth r category = "main" optional = true python-versions = ">=3.6" -files = [ - {file = "oauthlib-3.2.2-py3-none-any.whl", hash = "sha256:8139f29aac13e25d502680e9e19963e83f16838d48a0d71c287fe40e7067fbca"}, - {file = "oauthlib-3.2.2.tar.gz", hash = "sha256:9859c40929662bec5d64f34d01c99e093149682a3f38915dc0655d5a633dd918"}, -] [package.extras] rsa = ["cryptography (>=3.0.0)"] @@ -1411,10 +670,6 @@ description = "Core utilities for Python packages" category = "dev" optional = false python-versions = ">=3.7" -files = [ - {file = "packaging-23.0-py3-none-any.whl", hash = "sha256:714ac14496c3e68c99c29b00845f7a2b85f3bb6f1078fd9f72fd20f0570002b2"}, - {file = "packaging-23.0.tar.gz", hash = "sha256:b6ad297f8907de0fa2fe1ccbd26fdaf387f5f47c7275fedf8cce89f99446cf97"}, -] [[package]] name = "pandas" @@ -1423,35 +678,6 @@ description = "Powerful data structures for data analysis, time series, and stat category = "main" optional = true python-versions = ">=3.8" -files = [ - {file = "pandas-1.5.3-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:3749077d86e3a2f0ed51367f30bf5b82e131cc0f14260c4d3e499186fccc4406"}, - {file = "pandas-1.5.3-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:972d8a45395f2a2d26733eb8d0f629b2f90bebe8e8eddbb8829b180c09639572"}, - {file = "pandas-1.5.3-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:50869a35cbb0f2e0cd5ec04b191e7b12ed688874bd05dd777c19b28cbea90996"}, - {file = "pandas-1.5.3-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:c3ac844a0fe00bfaeb2c9b51ab1424e5c8744f89860b138434a363b1f620f354"}, - {file = "pandas-1.5.3-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:7a0a56cef15fd1586726dace5616db75ebcfec9179a3a55e78f72c5639fa2a23"}, - {file = "pandas-1.5.3-cp310-cp310-win_amd64.whl", hash = "sha256:478ff646ca42b20376e4ed3fa2e8d7341e8a63105586efe54fa2508ee087f328"}, - {file = "pandas-1.5.3-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:6973549c01ca91ec96199e940495219c887ea815b2083722821f1d7abfa2b4dc"}, - {file = "pandas-1.5.3-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:c39a8da13cede5adcd3be1182883aea1c925476f4e84b2807a46e2775306305d"}, - {file = "pandas-1.5.3-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:f76d097d12c82a535fda9dfe5e8dd4127952b45fea9b0276cb30cca5ea313fbc"}, - {file = "pandas-1.5.3-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:e474390e60ed609cec869b0da796ad94f420bb057d86784191eefc62b65819ae"}, - {file = "pandas-1.5.3-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:5f2b952406a1588ad4cad5b3f55f520e82e902388a6d5a4a91baa8d38d23c7f6"}, - {file = "pandas-1.5.3-cp311-cp311-win_amd64.whl", hash = "sha256:bc4c368f42b551bf72fac35c5128963a171b40dce866fb066540eeaf46faa003"}, - {file = "pandas-1.5.3-cp38-cp38-macosx_10_9_universal2.whl", hash = "sha256:14e45300521902689a81f3f41386dc86f19b8ba8dd5ac5a3c7010ef8d2932813"}, - {file = "pandas-1.5.3-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:9842b6f4b8479e41968eced654487258ed81df7d1c9b7b870ceea24ed9459b31"}, - {file = "pandas-1.5.3-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:26d9c71772c7afb9d5046e6e9cf42d83dd147b5cf5bcb9d97252077118543792"}, - {file = "pandas-1.5.3-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:5fbcb19d6fceb9e946b3e23258757c7b225ba450990d9ed63ccceeb8cae609f7"}, - {file = "pandas-1.5.3-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:565fa34a5434d38e9d250af3c12ff931abaf88050551d9fbcdfafca50d62babf"}, - {file = "pandas-1.5.3-cp38-cp38-win32.whl", hash = "sha256:87bd9c03da1ac870a6d2c8902a0e1fd4267ca00f13bc494c9e5a9020920e1d51"}, - {file = "pandas-1.5.3-cp38-cp38-win_amd64.whl", hash = "sha256:41179ce559943d83a9b4bbacb736b04c928b095b5f25dd2b7389eda08f46f373"}, - {file = "pandas-1.5.3-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:c74a62747864ed568f5a82a49a23a8d7fe171d0c69038b38cedf0976831296fa"}, - {file = "pandas-1.5.3-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:c4c00e0b0597c8e4f59e8d461f797e5d70b4d025880516a8261b2817c47759ee"}, - {file = "pandas-1.5.3-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:a50d9a4336a9621cab7b8eb3fb11adb82de58f9b91d84c2cd526576b881a0c5a"}, - {file = "pandas-1.5.3-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:dd05f7783b3274aa206a1af06f0ceed3f9b412cf665b7247eacd83be41cf7bf0"}, - {file = "pandas-1.5.3-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:9f69c4029613de47816b1bb30ff5ac778686688751a5e9c99ad8c7031f6508e5"}, - {file = "pandas-1.5.3-cp39-cp39-win32.whl", hash = "sha256:7cec0bee9f294e5de5bbfc14d0573f65526071029d036b753ee6507d2a21480a"}, - {file = "pandas-1.5.3-cp39-cp39-win_amd64.whl", hash = "sha256:dfd681c5dc216037e0b0a2c821f5ed99ba9f03ebcf119c7dac0e9a7b960b9ec9"}, - {file = "pandas-1.5.3.tar.gz", hash = "sha256:74a3fd7e5a7ec052f183273dc7b0acd3a863edf7520f5d3a1765c04ffdb3b0b1"}, -] [package.dependencies] numpy = [ @@ -1467,19 +693,15 @@ test = ["hypothesis (>=5.5.3)", "pytest (>=6.0)", "pytest-xdist (>=1.31)"] [[package]] name = "platformdirs" -version = "2.6.2" +version = "3.0.0" description = "A small Python package for determining appropriate platform-specific dirs, e.g. a \"user data dir\"." category = "dev" optional = false python-versions = ">=3.7" -files = [ - {file = "platformdirs-2.6.2-py3-none-any.whl", hash = "sha256:83c8f6d04389165de7c9b6f0c682439697887bca0aa2f1c87ef1826be3584490"}, - {file = "platformdirs-2.6.2.tar.gz", hash = "sha256:e1fea1fe471b9ff8332e229df3cb7de4f53eeea4998d3b6bfff542115e998bd2"}, -] [package.extras] -docs = ["furo (>=2022.12.7)", "proselint (>=0.13)", "sphinx (>=5.3)", "sphinx-autodoc-typehints (>=1.19.5)"] -test = ["appdirs (==1.4.4)", "covdefaults (>=2.2.2)", "pytest (>=7.2)", "pytest-cov (>=4)", "pytest-mock (>=3.10)"] +docs = ["furo (>=2022.12.7)", "proselint (>=0.13)", "sphinx (>=6.1.3)", "sphinx-autodoc-typehints (>=1.22,!=1.23.4)"] +test = ["appdirs (==1.4.4)", "covdefaults (>=2.2.2)", "pytest (>=7.2.1)", "pytest-cov (>=4)", "pytest-mock (>=3.10)"] [[package]] name = "pluggy" @@ -1488,10 +710,6 @@ description = "plugin and hook calling mechanisms for python" category = "dev" optional = false python-versions = ">=3.6" -files = [ - {file = "pluggy-1.0.0-py2.py3-none-any.whl", hash = "sha256:74134bbf457f031a36d68416e1509f34bd5ccc019f0bcc952c7b909d06b37bd3"}, - {file = "pluggy-1.0.0.tar.gz", hash = "sha256:4224373bacce55f955a878bf9cfa763c1e360858e330072059e10bad68531159"}, -] [package.extras] dev = ["pre-commit", "tox"] @@ -1504,10 +722,6 @@ description = "Wraps the portalocker recipe for easy usage" category = "main" optional = true python-versions = ">=3.5" -files = [ - {file = "portalocker-2.7.0-py2.py3-none-any.whl", hash = "sha256:a07c5b4f3985c3cf4798369631fb7011adb498e2a46d8440efc75a8f29a0f983"}, - {file = "portalocker-2.7.0.tar.gz", hash = "sha256:032e81d534a88ec1736d03f780ba073f047a06c478b06e2937486f334e955c51"}, -] [package.dependencies] pywin32 = {version = ">=226", markers = "platform_system == \"Windows\""} @@ -1524,10 +738,6 @@ description = "A framework for managing and maintaining multi-language pre-commi category = "dev" optional = false python-versions = ">=3.8" -files = [ - {file = "pre_commit-3.0.4-py2.py3-none-any.whl", hash = "sha256:9e3255edb0c9e7fe9b4f328cb3dc86069f8fdc38026f1bf521018a05eaf4d67b"}, - {file = "pre_commit-3.0.4.tar.gz", hash = "sha256:bc4687478d55578c4ac37272fe96df66f73d9b5cf81be6f28627d4e712e752d5"}, -] [package.dependencies] cfgv = ">=2.0.0" @@ -1543,33 +753,6 @@ description = "Python library for Apache Arrow" category = "main" optional = true python-versions = ">=3.7" -files = [ - {file = "pyarrow-11.0.0-cp310-cp310-macosx_10_14_x86_64.whl", hash = "sha256:40bb42afa1053c35c749befbe72f6429b7b5f45710e85059cdd534553ebcf4f2"}, - {file = "pyarrow-11.0.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:7c28b5f248e08dea3b3e0c828b91945f431f4202f1a9fe84d1012a761324e1ba"}, - {file = "pyarrow-11.0.0-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a37bc81f6c9435da3c9c1e767324ac3064ffbe110c4e460660c43e144be4ed85"}, - {file = "pyarrow-11.0.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:ad7c53def8dbbc810282ad308cc46a523ec81e653e60a91c609c2233ae407689"}, - {file = "pyarrow-11.0.0-cp310-cp310-win_amd64.whl", hash = "sha256:25aa11c443b934078bfd60ed63e4e2d42461682b5ac10f67275ea21e60e6042c"}, - {file = "pyarrow-11.0.0-cp311-cp311-macosx_10_14_x86_64.whl", hash = "sha256:e217d001e6389b20a6759392a5ec49d670757af80101ee6b5f2c8ff0172e02ca"}, - {file = "pyarrow-11.0.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:ad42bb24fc44c48f74f0d8c72a9af16ba9a01a2ccda5739a517aa860fa7e3d56"}, - {file = "pyarrow-11.0.0-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:2d942c690ff24a08b07cb3df818f542a90e4d359381fbff71b8f2aea5bf58841"}, - {file = "pyarrow-11.0.0-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:f010ce497ca1b0f17a8243df3048055c0d18dcadbcc70895d5baf8921f753de5"}, - {file = "pyarrow-11.0.0-cp311-cp311-win_amd64.whl", hash = "sha256:2f51dc7ca940fdf17893227edb46b6784d37522ce08d21afc56466898cb213b2"}, - {file = "pyarrow-11.0.0-cp37-cp37m-macosx_10_14_x86_64.whl", hash = "sha256:1cbcfcbb0e74b4d94f0b7dde447b835a01bc1d16510edb8bb7d6224b9bf5bafc"}, - {file = "pyarrow-11.0.0-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:aaee8f79d2a120bf3e032d6d64ad20b3af6f56241b0ffc38d201aebfee879d00"}, - {file = "pyarrow-11.0.0-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:410624da0708c37e6a27eba321a72f29d277091c8f8d23f72c92bada4092eb5e"}, - {file = "pyarrow-11.0.0-cp37-cp37m-win_amd64.whl", hash = "sha256:2d53ba72917fdb71e3584ffc23ee4fcc487218f8ff29dd6df3a34c5c48fe8c06"}, - {file = "pyarrow-11.0.0-cp38-cp38-macosx_10_14_x86_64.whl", hash = "sha256:f12932e5a6feb5c58192209af1d2607d488cb1d404fbc038ac12ada60327fa34"}, - {file = "pyarrow-11.0.0-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:41a1451dd895c0b2964b83d91019e46f15b5564c7ecd5dcb812dadd3f05acc97"}, - {file = "pyarrow-11.0.0-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:becc2344be80e5dce4e1b80b7c650d2fc2061b9eb339045035a1baa34d5b8f1c"}, - {file = "pyarrow-11.0.0-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:8f40be0d7381112a398b93c45a7e69f60261e7b0269cc324e9f739ce272f4f70"}, - {file = "pyarrow-11.0.0-cp38-cp38-win_amd64.whl", hash = "sha256:362a7c881b32dc6b0eccf83411a97acba2774c10edcec715ccaab5ebf3bb0835"}, - {file = "pyarrow-11.0.0-cp39-cp39-macosx_10_14_x86_64.whl", hash = "sha256:ccbf29a0dadfcdd97632b4f7cca20a966bb552853ba254e874c66934931b9841"}, - {file = "pyarrow-11.0.0-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:3e99be85973592051e46412accea31828da324531a060bd4585046a74ba45854"}, - {file = "pyarrow-11.0.0-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:69309be84dcc36422574d19c7d3a30a7ea43804f12552356d1ab2a82a713c418"}, - {file = "pyarrow-11.0.0-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:da93340fbf6f4e2a62815064383605b7ffa3e9eeb320ec839995b1660d69f89b"}, - {file = "pyarrow-11.0.0-cp39-cp39-win_amd64.whl", hash = "sha256:caad867121f182d0d3e1a0d36f197df604655d0b466f1bc9bafa903aa95083e4"}, - {file = "pyarrow-11.0.0.tar.gz", hash = "sha256:5461c57dbdb211a632a48facb9b39bbeb8a7905ec95d768078525283caef5f6d"}, -] [package.dependencies] numpy = ">=1.16.6" @@ -1581,10 +764,6 @@ description = "C parser in Python" category = "main" optional = false python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*" -files = [ - {file = "pycparser-2.21-py2.py3-none-any.whl", hash = "sha256:8ee45429555515e1f6b185e78100aea234072576aa43ab53aefcae078162fca9"}, - {file = "pycparser-2.21.tar.gz", hash = "sha256:e644fdec12f7872f86c58ff790da456218b10f863970249516d60a5eaca77206"}, -] [[package]] name = "pydantic" @@ -1593,44 +772,6 @@ description = "Data validation and settings management using python type hints" category = "main" optional = false python-versions = ">=3.7" -files = [ - {file = "pydantic-1.10.4-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:b5635de53e6686fe7a44b5cf25fcc419a0d5e5c1a1efe73d49d48fe7586db854"}, - {file = "pydantic-1.10.4-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:6dc1cc241440ed7ca9ab59d9929075445da6b7c94ced281b3dd4cfe6c8cff817"}, - {file = "pydantic-1.10.4-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:51bdeb10d2db0f288e71d49c9cefa609bca271720ecd0c58009bd7504a0c464c"}, - {file = "pydantic-1.10.4-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:78cec42b95dbb500a1f7120bdf95c401f6abb616bbe8785ef09887306792e66e"}, - {file = "pydantic-1.10.4-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:8775d4ef5e7299a2f4699501077a0defdaac5b6c4321173bcb0f3c496fbadf85"}, - {file = "pydantic-1.10.4-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:572066051eeac73d23f95ba9a71349c42a3e05999d0ee1572b7860235b850cc6"}, - {file = "pydantic-1.10.4-cp310-cp310-win_amd64.whl", hash = "sha256:7feb6a2d401f4d6863050f58325b8d99c1e56f4512d98b11ac64ad1751dc647d"}, - {file = "pydantic-1.10.4-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:39f4a73e5342b25c2959529f07f026ef58147249f9b7431e1ba8414a36761f53"}, - {file = "pydantic-1.10.4-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:983e720704431a6573d626b00662eb78a07148c9115129f9b4351091ec95ecc3"}, - {file = "pydantic-1.10.4-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:75d52162fe6b2b55964fbb0af2ee58e99791a3138588c482572bb6087953113a"}, - {file = "pydantic-1.10.4-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:fdf8d759ef326962b4678d89e275ffc55b7ce59d917d9f72233762061fd04a2d"}, - {file = "pydantic-1.10.4-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:05a81b006be15655b2a1bae5faa4280cf7c81d0e09fcb49b342ebf826abe5a72"}, - {file = "pydantic-1.10.4-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:d88c4c0e5c5dfd05092a4b271282ef0588e5f4aaf345778056fc5259ba098857"}, - {file = "pydantic-1.10.4-cp311-cp311-win_amd64.whl", hash = "sha256:6a05a9db1ef5be0fe63e988f9617ca2551013f55000289c671f71ec16f4985e3"}, - {file = "pydantic-1.10.4-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:887ca463c3bc47103c123bc06919c86720e80e1214aab79e9b779cda0ff92a00"}, - {file = "pydantic-1.10.4-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:fdf88ab63c3ee282c76d652fc86518aacb737ff35796023fae56a65ced1a5978"}, - {file = "pydantic-1.10.4-cp37-cp37m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:a48f1953c4a1d9bd0b5167ac50da9a79f6072c63c4cef4cf2a3736994903583e"}, - {file = "pydantic-1.10.4-cp37-cp37m-musllinux_1_1_i686.whl", hash = "sha256:a9f2de23bec87ff306aef658384b02aa7c32389766af3c5dee9ce33e80222dfa"}, - {file = "pydantic-1.10.4-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:cd8702c5142afda03dc2b1ee6bc358b62b3735b2cce53fc77b31ca9f728e4bc8"}, - {file = "pydantic-1.10.4-cp37-cp37m-win_amd64.whl", hash = "sha256:6e7124d6855b2780611d9f5e1e145e86667eaa3bd9459192c8dc1a097f5e9903"}, - {file = "pydantic-1.10.4-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:0b53e1d41e97063d51a02821b80538053ee4608b9a181c1005441f1673c55423"}, - {file = "pydantic-1.10.4-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:55b1625899acd33229c4352ce0ae54038529b412bd51c4915349b49ca575258f"}, - {file = "pydantic-1.10.4-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:301d626a59edbe5dfb48fcae245896379a450d04baeed50ef40d8199f2733b06"}, - {file = "pydantic-1.10.4-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:b6f9d649892a6f54a39ed56b8dfd5e08b5f3be5f893da430bed76975f3735d15"}, - {file = "pydantic-1.10.4-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:d7b5a3821225f5c43496c324b0d6875fde910a1c2933d726a743ce328fbb2a8c"}, - {file = "pydantic-1.10.4-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:f2f7eb6273dd12472d7f218e1fef6f7c7c2f00ac2e1ecde4db8824c457300416"}, - {file = "pydantic-1.10.4-cp38-cp38-win_amd64.whl", hash = "sha256:4b05697738e7d2040696b0a66d9f0a10bec0efa1883ca75ee9e55baf511909d6"}, - {file = "pydantic-1.10.4-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:a9a6747cac06c2beb466064dda999a13176b23535e4c496c9d48e6406f92d42d"}, - {file = "pydantic-1.10.4-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:eb992a1ef739cc7b543576337bebfc62c0e6567434e522e97291b251a41dad7f"}, - {file = "pydantic-1.10.4-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:990406d226dea0e8f25f643b370224771878142155b879784ce89f633541a024"}, - {file = "pydantic-1.10.4-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:2e82a6d37a95e0b1b42b82ab340ada3963aea1317fd7f888bb6b9dfbf4fff57c"}, - {file = "pydantic-1.10.4-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:9193d4f4ee8feca58bc56c8306bcb820f5c7905fd919e0750acdeeeef0615b28"}, - {file = "pydantic-1.10.4-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:2b3ce5f16deb45c472dde1a0ee05619298c864a20cded09c4edd820e1454129f"}, - {file = "pydantic-1.10.4-cp39-cp39-win_amd64.whl", hash = "sha256:9cbdc268a62d9a98c56e2452d6c41c0263d64a2009aac69246486f01b4f594c4"}, - {file = "pydantic-1.10.4-py3-none-any.whl", hash = "sha256:4948f264678c703f3877d1c8877c4e3b2e12e549c57795107f08cf70c6ec7774"}, - {file = "pydantic-1.10.4.tar.gz", hash = "sha256:b9a3859f24eb4e097502a3be1fb4b2abb79b6103dd9e2e0edb70613a4459a648"}, -] [package.dependencies] typing-extensions = ">=4.2.0" @@ -1646,10 +787,6 @@ description = "Pygments is a syntax highlighting package written in Python." category = "main" optional = false python-versions = ">=3.6" -files = [ - {file = "Pygments-2.14.0-py3-none-any.whl", hash = "sha256:fa7bd7bd2771287c0de303af8bfdfc731f51bd2c6a47ab69d117138893b82717"}, - {file = "Pygments-2.14.0.tar.gz", hash = "sha256:b3ed06a9e8ac9a9aae5a6f5dbe78a8a58655d17b43b93c078f094ddc476ae297"}, -] [package.extras] plugins = ["importlib-metadata"] @@ -1661,10 +798,6 @@ description = "JSON Web Token implementation in Python" category = "main" optional = true python-versions = ">=3.7" -files = [ - {file = "PyJWT-2.6.0-py3-none-any.whl", hash = "sha256:d83c3d892a77bbb74d3e1a2cfa90afaadb60945205d1095d9221f04466f64c14"}, - {file = "PyJWT-2.6.0.tar.gz", hash = "sha256:69285c7e31fc44f68a1feb309e948e0df53259d579295e6cfe2b1792329f05fd"}, -] [package.dependencies] cryptography = {version = ">=3.4.0", optional = true, markers = "extra == \"crypto\""} @@ -1682,10 +815,6 @@ description = "pyparsing module - Classes and methods to define and execute pars category = "main" optional = false python-versions = ">=3.6.8" -files = [ - {file = "pyparsing-3.0.9-py3-none-any.whl", hash = "sha256:5026bae9a10eeaefb61dab2f09052b9f4307d44aee4eda64b309723d8d206bbc"}, - {file = "pyparsing-3.0.9.tar.gz", hash = "sha256:2b020ecf7d21b687f219b71ecad3631f644a47f01403fa1d1036b0c6416d70fb"}, -] [package.extras] diagrams = ["jinja2", "railroad-diagrams"] @@ -1697,10 +826,6 @@ description = "Wrappers to call pyproject.toml-based build backend hooks." category = "dev" optional = false python-versions = ">=3.7" -files = [ - {file = "pyproject_hooks-1.0.0-py3-none-any.whl", hash = "sha256:283c11acd6b928d2f6a7c73fa0d01cb2bdc5f07c57a2eeb6e83d5e56b97976f8"}, - {file = "pyproject_hooks-1.0.0.tar.gz", hash = "sha256:f271b298b97f5955d53fb12b72c1fb1948c22c1a6b70b315c54cedaca0264ef5"}, -] [package.dependencies] tomli = {version = ">=1.1.0", markers = "python_version < \"3.11\""} @@ -1712,10 +837,6 @@ description = "pytest: simple powerful testing with Python" category = "dev" optional = false python-versions = ">=3.7" -files = [ - {file = "pytest-7.2.1-py3-none-any.whl", hash = "sha256:c7c6ca206e93355074ae32f7403e8ea12163b1163c976fee7d4d84027c162be5"}, - {file = "pytest-7.2.1.tar.gz", hash = "sha256:d45e0952f3727241918b8fd0f376f5ff6b301cc0777c6f9a556935c92d8a7d42"}, -] [package.dependencies] attrs = ">=19.2.0" @@ -1736,10 +857,6 @@ description = "check the README when running tests" category = "dev" optional = false python-versions = ">=3.7" -files = [ - {file = "pytest-checkdocs-2.9.0.tar.gz", hash = "sha256:54b3d1357a289e3f6336d31d4ceb72e99afe0fa4f552990f1b1f015f1925543b"}, - {file = "pytest_checkdocs-2.9.0-py3-none-any.whl", hash = "sha256:4231e5426915a02385e386b5eb4a70490b143d97cede4c7c77691bb320b5a73f"}, -] [package.dependencies] build = "*" @@ -1757,10 +874,6 @@ description = "Extensions to the standard Python datetime module" category = "main" optional = false python-versions = "!=3.0.*,!=3.1.*,!=3.2.*,>=2.7" -files = [ - {file = "python-dateutil-2.8.2.tar.gz", hash = "sha256:0123cacc1627ae19ddf3c27a5de5bd67ee4586fbdd6440d9748f8abb483d3e86"}, - {file = "python_dateutil-2.8.2-py2.py3-none-any.whl", hash = "sha256:961d03dc3453ebbc59dbdea9e4e11c5651520a876d0f4db161e8674aae935da9"}, -] [package.dependencies] six = ">=1.5" @@ -1772,56 +885,6 @@ description = "Python library for the snappy compression library from Google" category = "main" optional = true python-versions = "*" -files = [ - {file = "python-snappy-0.6.1.tar.gz", hash = "sha256:b6a107ab06206acc5359d4c5632bd9b22d448702a79b3169b0c62e0fb808bb2a"}, - {file = "python_snappy-0.6.1-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:b7f920eaf46ebf41bd26f9df51c160d40f9e00b7b48471c3438cb8d027f7fb9b"}, - {file = "python_snappy-0.6.1-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:4ec533a8c1f8df797bded662ec3e494d225b37855bb63eb0d75464a07947477c"}, - {file = "python_snappy-0.6.1-cp310-cp310-manylinux_2_12_i686.manylinux2010_i686.whl", hash = "sha256:6f8bf4708a11b47517baf962f9a02196478bbb10fdb9582add4aa1459fa82380"}, - {file = "python_snappy-0.6.1-cp310-cp310-manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:8d0c019ee7dcf2c60e240877107cddbd95a5b1081787579bf179938392d66480"}, - {file = "python_snappy-0.6.1-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:cb18d9cd7b3f35a2f5af47bb8ed6a5bdbf4f3ddee37f3daade4ab7864c292f5b"}, - {file = "python_snappy-0.6.1-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:b265cde49774752aec9ca7f5d272e3f98718164afc85521622a8a5394158a2b5"}, - {file = "python_snappy-0.6.1-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:d017775851a778ec9cc32651c4464079d06d927303c2dde9ae9830ccf6fe94e1"}, - {file = "python_snappy-0.6.1-cp310-cp310-win32.whl", hash = "sha256:8277d1f6282463c40761f802b742f833f9f2449fcdbb20a96579aa05c8feb614"}, - {file = "python_snappy-0.6.1-cp310-cp310-win_amd64.whl", hash = "sha256:2aaaf618c68d8c9daebc23a20436bd01b09ee70d7fbf7072b7f38b06d2fab539"}, - {file = "python_snappy-0.6.1-cp36-cp36m-macosx_10_9_x86_64.whl", hash = "sha256:277757d5dad4e239dc1417438a0871b65b1b155beb108888e7438c27ffc6a8cc"}, - {file = "python_snappy-0.6.1-cp36-cp36m-manylinux_2_12_i686.manylinux2010_i686.whl", hash = "sha256:e066a0586833d610c4bbddba0be5ba0e3e4f8e0bc5bb6d82103d8f8fc47bb59a"}, - {file = "python_snappy-0.6.1-cp36-cp36m-manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:0d489b50f49433494160c45048fe806de6b3aeab0586e497ebd22a0bab56e427"}, - {file = "python_snappy-0.6.1-cp36-cp36m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:463fd340a499d47b26ca42d2f36a639188738f6e2098c6dbf80aef0e60f461e1"}, - {file = "python_snappy-0.6.1-cp36-cp36m-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:9837ac1650cc68d22a3cf5f15fb62c6964747d16cecc8b22431f113d6e39555d"}, - {file = "python_snappy-0.6.1-cp36-cp36m-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:5e973e637112391f05581f427659c05b30b6843bc522a65be35ac7b18ce3dedd"}, - {file = "python_snappy-0.6.1-cp36-cp36m-win32.whl", hash = "sha256:c20498bd712b6e31a4402e1d027a1cd64f6a4a0066a3fe3c7344475886d07fdf"}, - {file = "python_snappy-0.6.1-cp36-cp36m-win_amd64.whl", hash = "sha256:59e975be4206cc54d0a112ef72fa3970a57c2b1bcc2c97ed41d6df0ebe518228"}, - {file = "python_snappy-0.6.1-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:2a7e528ab6e09c0d67dcb61a1730a292683e5ff9bb088950638d3170cf2a0a54"}, - {file = "python_snappy-0.6.1-cp37-cp37m-manylinux_2_12_i686.manylinux2010_i686.whl", hash = "sha256:39692bedbe0b717001a99915ac0eb2d9d0bad546440d392a2042b96d813eede1"}, - {file = "python_snappy-0.6.1-cp37-cp37m-manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:6a7620404da966f637b9ce8d4d3d543d363223f7a12452a575189c5355fc2d25"}, - {file = "python_snappy-0.6.1-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:7778c224efc38a40d274da4eb82a04cac27aae20012372a7db3c4bbd8926c4d4"}, - {file = "python_snappy-0.6.1-cp37-cp37m-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:1d029f7051ec1bbeaa3e03030b6d8ed47ceb69cae9016f493c802a08af54e026"}, - {file = "python_snappy-0.6.1-cp37-cp37m-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:a0ad38bc98d0b0497a0b0dbc29409bcabfcecff4511ed7063403c86de16927bc"}, - {file = "python_snappy-0.6.1-cp37-cp37m-win32.whl", hash = "sha256:5a453c45178d7864c1bdd6bfe0ee3ed2883f63b9ba2c9bb967c6b586bf763f96"}, - {file = "python_snappy-0.6.1-cp37-cp37m-win_amd64.whl", hash = "sha256:9f0c0d88b84259f93c3aa46398680646f2c23e43394779758d9f739c34e15295"}, - {file = "python_snappy-0.6.1-cp38-cp38-macosx_10_9_universal2.whl", hash = "sha256:5bb05c28298803a74add08ba496879242ef159c75bc86a5406fac0ffc7dd021b"}, - {file = "python_snappy-0.6.1-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:9eac51307c6a1a38d5f86ebabc26a889fddf20cbba7a116ccb54ba1446601d5b"}, - {file = "python_snappy-0.6.1-cp38-cp38-manylinux_2_12_i686.manylinux2010_i686.whl", hash = "sha256:88b6ea78b83d2796f330b0af1b70cdd3965dbdab02d8ac293260ec2c8fe340ee"}, - {file = "python_snappy-0.6.1-cp38-cp38-manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:8c07220408d3268e8268c9351c5c08041bc6f8c6172e59d398b71020df108541"}, - {file = "python_snappy-0.6.1-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:4038019b1bcaadde726a57430718394076c5a21545ebc5badad2c045a09546cf"}, - {file = "python_snappy-0.6.1-cp38-cp38-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:dc96668d9c7cc656609764275c5f8da58ef56d89bdd6810f6923d36497468ff7"}, - {file = "python_snappy-0.6.1-cp38-cp38-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:cf5bb9254e1c38aacf253d510d3d9be631bba21f3d068b17672b38b5cbf2fff5"}, - {file = "python_snappy-0.6.1-cp38-cp38-win32.whl", hash = "sha256:eaf905a580f2747c4a474040a5063cd5e0cc3d1d2d6edb65f28196186493ad4a"}, - {file = "python_snappy-0.6.1-cp38-cp38-win_amd64.whl", hash = "sha256:546c1a7470ecbf6239101e9aff0f709b68ca0f0268b34d9023019a55baa1f7c6"}, - {file = "python_snappy-0.6.1-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:e3a013895c64352b49d0d8e107a84f99631b16dbab156ded33ebf0becf56c8b2"}, - {file = "python_snappy-0.6.1-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:3fb9a88a4dd6336488f3de67ce75816d0d796dce53c2c6e4d70e0b565633c7fd"}, - {file = "python_snappy-0.6.1-cp39-cp39-manylinux_2_12_i686.manylinux2010_i686.whl", hash = "sha256:735cd4528c55dbe4516d6d2b403331a99fc304f8feded8ae887cf97b67d589bb"}, - {file = "python_snappy-0.6.1-cp39-cp39-manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:90b0186516b7a101c14764b0c25931b741fb0102f21253eff67847b4742dfc72"}, - {file = "python_snappy-0.6.1-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:1a993dc8aadd901915a510fe6af5f20ae4256f527040066c22a154db8946751f"}, - {file = "python_snappy-0.6.1-cp39-cp39-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:530bfb9efebcc1aab8bb4ebcbd92b54477eed11f6cf499355e882970a6d3aa7d"}, - {file = "python_snappy-0.6.1-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:5843feb914796b1f0405ccf31ea0fb51034ceb65a7588edfd5a8250cb369e3b2"}, - {file = "python_snappy-0.6.1-cp39-cp39-win32.whl", hash = "sha256:66c80e9b366012dbee262bb1869e4fc5ba8786cda85928481528bc4a72ec2ee8"}, - {file = "python_snappy-0.6.1-cp39-cp39-win_amd64.whl", hash = "sha256:4d3cafdf454354a621c8ab7408e45aa4e9d5c0b943b61ff4815f71ca6bdf0130"}, - {file = "python_snappy-0.6.1-pp37-pypy37_pp73-manylinux_2_12_i686.manylinux2010_i686.whl", hash = "sha256:586724a0276d7a6083a17259d0b51622e492289a9998848a1b01b6441ca12b2f"}, - {file = "python_snappy-0.6.1-pp37-pypy37_pp73-manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:2be4f4550acd484912441f5f1209ba611ac399aac9355fee73611b9a0d4f949c"}, - {file = "python_snappy-0.6.1-pp37-pypy37_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:0bdb6942180660bda7f7d01f4c0def3cfc72b1c6d99aad964801775a3e379aba"}, - {file = "python_snappy-0.6.1-pp37-pypy37_pp73-win_amd64.whl", hash = "sha256:03bb511380fca2a13325b6f16fe8234c8e12da9660f0258cd45d9a02ffc916af"}, -] [[package]] name = "pytz" @@ -1830,10 +893,6 @@ description = "World timezone definitions, modern and historical" category = "main" optional = true python-versions = "*" -files = [ - {file = "pytz-2022.7.1-py2.py3-none-any.whl", hash = "sha256:78f4f37d8198e0627c5f1143240bb0206b8691d8d7ac6d78fee88b78733f8c4a"}, - {file = "pytz-2022.7.1.tar.gz", hash = "sha256:01a0681c4b9684a28304615eba55d1ab31ae00bf68ec157ec3708a8182dbbcd0"}, -] [[package]] name = "pywin32" @@ -1842,22 +901,6 @@ description = "Python for Window Extensions" category = "main" optional = true python-versions = "*" -files = [ - {file = "pywin32-305-cp310-cp310-win32.whl", hash = "sha256:421f6cd86e84bbb696d54563c48014b12a23ef95a14e0bdba526be756d89f116"}, - {file = "pywin32-305-cp310-cp310-win_amd64.whl", hash = "sha256:73e819c6bed89f44ff1d690498c0a811948f73777e5f97c494c152b850fad478"}, - {file = "pywin32-305-cp310-cp310-win_arm64.whl", hash = "sha256:742eb905ce2187133a29365b428e6c3b9001d79accdc30aa8969afba1d8470f4"}, - {file = "pywin32-305-cp311-cp311-win32.whl", hash = "sha256:19ca459cd2e66c0e2cc9a09d589f71d827f26d47fe4a9d09175f6aa0256b51c2"}, - {file = "pywin32-305-cp311-cp311-win_amd64.whl", hash = "sha256:326f42ab4cfff56e77e3e595aeaf6c216712bbdd91e464d167c6434b28d65990"}, - {file = "pywin32-305-cp311-cp311-win_arm64.whl", hash = "sha256:4ecd404b2c6eceaca52f8b2e3e91b2187850a1ad3f8b746d0796a98b4cea04db"}, - {file = "pywin32-305-cp36-cp36m-win32.whl", hash = "sha256:48d8b1659284f3c17b68587af047d110d8c44837736b8932c034091683e05863"}, - {file = "pywin32-305-cp36-cp36m-win_amd64.whl", hash = "sha256:13362cc5aa93c2beaf489c9c9017c793722aeb56d3e5166dadd5ef82da021fe1"}, - {file = "pywin32-305-cp37-cp37m-win32.whl", hash = "sha256:a55db448124d1c1484df22fa8bbcbc45c64da5e6eae74ab095b9ea62e6d00496"}, - {file = "pywin32-305-cp37-cp37m-win_amd64.whl", hash = "sha256:109f98980bfb27e78f4df8a51a8198e10b0f347257d1e265bb1a32993d0c973d"}, - {file = "pywin32-305-cp38-cp38-win32.whl", hash = "sha256:9dd98384da775afa009bc04863426cb30596fd78c6f8e4e2e5bbf4edf8029504"}, - {file = "pywin32-305-cp38-cp38-win_amd64.whl", hash = "sha256:56d7a9c6e1a6835f521788f53b5af7912090674bb84ef5611663ee1595860fc7"}, - {file = "pywin32-305-cp39-cp39-win32.whl", hash = "sha256:9d968c677ac4d5cbdaa62fd3014ab241718e619d8e36ef8e11fb930515a1e918"}, - {file = "pywin32-305-cp39-cp39-win_amd64.whl", hash = "sha256:50768c6b7c3f0b38b7fb14dd4104da93ebced5f1a50dc0e834594bff6fbe1271"}, -] [[package]] name = "pyyaml" @@ -1866,48 +909,6 @@ description = "YAML parser and emitter for Python" category = "main" optional = false python-versions = ">=3.6" -files = [ - {file = "PyYAML-6.0-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:d4db7c7aef085872ef65a8fd7d6d09a14ae91f691dec3e87ee5ee0539d516f53"}, - {file = "PyYAML-6.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:9df7ed3b3d2e0ecfe09e14741b857df43adb5a3ddadc919a2d94fbdf78fea53c"}, - {file = "PyYAML-6.0-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:77f396e6ef4c73fdc33a9157446466f1cff553d979bd00ecb64385760c6babdc"}, - {file = "PyYAML-6.0-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:a80a78046a72361de73f8f395f1f1e49f956c6be882eed58505a15f3e430962b"}, - {file = "PyYAML-6.0-cp310-cp310-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:f84fbc98b019fef2ee9a1cb3ce93e3187a6df0b2538a651bfb890254ba9f90b5"}, - {file = "PyYAML-6.0-cp310-cp310-win32.whl", hash = "sha256:2cd5df3de48857ed0544b34e2d40e9fac445930039f3cfe4bcc592a1f836d513"}, - {file = "PyYAML-6.0-cp310-cp310-win_amd64.whl", hash = "sha256:daf496c58a8c52083df09b80c860005194014c3698698d1a57cbcfa182142a3a"}, - {file = "PyYAML-6.0-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:d4b0ba9512519522b118090257be113b9468d804b19d63c71dbcf4a48fa32358"}, - {file = "PyYAML-6.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:81957921f441d50af23654aa6c5e5eaf9b06aba7f0a19c18a538dc7ef291c5a1"}, - {file = "PyYAML-6.0-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:afa17f5bc4d1b10afd4466fd3a44dc0e245382deca5b3c353d8b757f9e3ecb8d"}, - {file = "PyYAML-6.0-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:dbad0e9d368bb989f4515da330b88a057617d16b6a8245084f1b05400f24609f"}, - {file = "PyYAML-6.0-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:432557aa2c09802be39460360ddffd48156e30721f5e8d917f01d31694216782"}, - {file = "PyYAML-6.0-cp311-cp311-win32.whl", hash = "sha256:bfaef573a63ba8923503d27530362590ff4f576c626d86a9fed95822a8255fd7"}, - {file = "PyYAML-6.0-cp311-cp311-win_amd64.whl", hash = "sha256:01b45c0191e6d66c470b6cf1b9531a771a83c1c4208272ead47a3ae4f2f603bf"}, - {file = "PyYAML-6.0-cp36-cp36m-macosx_10_9_x86_64.whl", hash = "sha256:897b80890765f037df3403d22bab41627ca8811ae55e9a722fd0392850ec4d86"}, - {file = "PyYAML-6.0-cp36-cp36m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:50602afada6d6cbfad699b0c7bb50d5ccffa7e46a3d738092afddc1f9758427f"}, - {file = "PyYAML-6.0-cp36-cp36m-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:48c346915c114f5fdb3ead70312bd042a953a8ce5c7106d5bfb1a5254e47da92"}, - {file = "PyYAML-6.0-cp36-cp36m-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:98c4d36e99714e55cfbaaee6dd5badbc9a1ec339ebfc3b1f52e293aee6bb71a4"}, - {file = "PyYAML-6.0-cp36-cp36m-win32.whl", hash = "sha256:0283c35a6a9fbf047493e3a0ce8d79ef5030852c51e9d911a27badfde0605293"}, - {file = "PyYAML-6.0-cp36-cp36m-win_amd64.whl", hash = "sha256:07751360502caac1c067a8132d150cf3d61339af5691fe9e87803040dbc5db57"}, - {file = "PyYAML-6.0-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:819b3830a1543db06c4d4b865e70ded25be52a2e0631ccd2f6a47a2822f2fd7c"}, - {file = "PyYAML-6.0-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:473f9edb243cb1935ab5a084eb238d842fb8f404ed2193a915d1784b5a6b5fc0"}, - {file = "PyYAML-6.0-cp37-cp37m-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:0ce82d761c532fe4ec3f87fc45688bdd3a4c1dc5e0b4a19814b9009a29baefd4"}, - {file = "PyYAML-6.0-cp37-cp37m-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:231710d57adfd809ef5d34183b8ed1eeae3f76459c18fb4a0b373ad56bedcdd9"}, - {file = "PyYAML-6.0-cp37-cp37m-win32.whl", hash = "sha256:c5687b8d43cf58545ade1fe3e055f70eac7a5a1a0bf42824308d868289a95737"}, - {file = "PyYAML-6.0-cp37-cp37m-win_amd64.whl", hash = "sha256:d15a181d1ecd0d4270dc32edb46f7cb7733c7c508857278d3d378d14d606db2d"}, - {file = "PyYAML-6.0-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:0b4624f379dab24d3725ffde76559cff63d9ec94e1736b556dacdfebe5ab6d4b"}, - {file = "PyYAML-6.0-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:213c60cd50106436cc818accf5baa1aba61c0189ff610f64f4a3e8c6726218ba"}, - {file = "PyYAML-6.0-cp38-cp38-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:9fa600030013c4de8165339db93d182b9431076eb98eb40ee068700c9c813e34"}, - {file = "PyYAML-6.0-cp38-cp38-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:277a0ef2981ca40581a47093e9e2d13b3f1fbbeffae064c1d21bfceba2030287"}, - {file = "PyYAML-6.0-cp38-cp38-win32.whl", hash = "sha256:d4eccecf9adf6fbcc6861a38015c2a64f38b9d94838ac1810a9023a0609e1b78"}, - {file = "PyYAML-6.0-cp38-cp38-win_amd64.whl", hash = "sha256:1e4747bc279b4f613a09eb64bba2ba602d8a6664c6ce6396a4d0cd413a50ce07"}, - {file = "PyYAML-6.0-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:055d937d65826939cb044fc8c9b08889e8c743fdc6a32b33e2390f66013e449b"}, - {file = "PyYAML-6.0-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:e61ceaab6f49fb8bdfaa0f92c4b57bcfbea54c09277b1b4f7ac376bfb7a7c174"}, - {file = "PyYAML-6.0-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:d67d839ede4ed1b28a4e8909735fc992a923cdb84e618544973d7dfc71540803"}, - {file = "PyYAML-6.0-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:cba8c411ef271aa037d7357a2bc8f9ee8b58b9965831d9e51baf703280dc73d3"}, - {file = "PyYAML-6.0-cp39-cp39-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:40527857252b61eacd1d9af500c3337ba8deb8fc298940291486c465c8b46ec0"}, - {file = "PyYAML-6.0-cp39-cp39-win32.whl", hash = "sha256:b5b9eccad747aabaaffbc6064800670f0c297e52c12754eb1d976c57e4f74dcb"}, - {file = "PyYAML-6.0-cp39-cp39-win_amd64.whl", hash = "sha256:b3d267842bf12586ba6c734f89d1f5b871df0273157918b0ccefa29deb05c21c"}, - {file = "PyYAML-6.0.tar.gz", hash = "sha256:68fb519c14306fec9720a2a5b45bc9f0c8d1b9c72adf45c37baedfcd949c35a2"}, -] [[package]] name = "requests" @@ -1916,10 +917,6 @@ description = "Python HTTP for Humans." category = "main" optional = false python-versions = ">=3.7, <4" -files = [ - {file = "requests-2.28.2-py3-none-any.whl", hash = "sha256:64299f4909223da747622c030b781c0d7811e359c37124b4bd368fb8c6518baa"}, - {file = "requests-2.28.2.tar.gz", hash = "sha256:98b1b2782e3c6c4904938b84c0eb932721069dfdb9134313beff7c83c2df24bf"}, -] [package.dependencies] certifi = ">=2017.4.17" @@ -1929,7 +926,7 @@ urllib3 = ">=1.21.1,<1.27" [package.extras] socks = ["PySocks (>=1.5.6,!=1.5.7)"] -use-chardet-on-py3 = ["chardet (>=3.0.2,<6)"] +use_chardet_on_py3 = ["chardet (>=3.0.2,<6)"] [[package]] name = "requests-mock" @@ -1938,10 +935,6 @@ description = "Mock out responses from the requests package" category = "dev" optional = false python-versions = "*" -files = [ - {file = "requests-mock-1.10.0.tar.gz", hash = "sha256:59c9c32419a9fb1ae83ec242d98e889c45bd7d7a65d48375cc243ec08441658b"}, - {file = "requests_mock-1.10.0-py2.py3-none-any.whl", hash = "sha256:2fdbb637ad17ee15c06f33d31169e71bf9fe2bdb7bc9da26185be0dd8d842699"}, -] [package.dependencies] requests = ">=2.3,<3" @@ -1958,10 +951,6 @@ description = "OAuthlib authentication support for Requests." category = "main" optional = true python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*" -files = [ - {file = "requests-oauthlib-1.3.1.tar.gz", hash = "sha256:75beac4a47881eeb94d5ea5d6ad31ef88856affe2332b9aafb52c6452ccf0d7a"}, - {file = "requests_oauthlib-1.3.1-py2.py3-none-any.whl", hash = "sha256:2577c501a2fb8d05a304c09d090d6e47c306fef15809d102b327cf8364bddab5"}, -] [package.dependencies] oauthlib = ">=3.0.0" @@ -1977,10 +966,6 @@ description = "A utility library for mocking out the `requests` Python library." category = "dev" optional = false python-versions = ">=3.7" -files = [ - {file = "responses-0.22.0-py3-none-any.whl", hash = "sha256:dcf294d204d14c436fddcc74caefdbc5764795a40ff4e6a7740ed8ddbf3294be"}, - {file = "responses-0.22.0.tar.gz", hash = "sha256:396acb2a13d25297789a5866b4881cf4e46ffd49cc26c43ab1117f40b973102e"}, -] [package.dependencies] requests = ">=2.22.0,<3.0" @@ -1998,10 +983,6 @@ description = "Render rich text, tables, progress bars, syntax highlighting, mar category = "main" optional = false python-versions = ">=3.7.0" -files = [ - {file = "rich-13.3.1-py3-none-any.whl", hash = "sha256:8aa57747f3fc3e977684f0176a88e789be314a99f99b43b75d1e9cb5dc6db9e9"}, - {file = "rich-13.3.1.tar.gz", hash = "sha256:125d96d20c92b946b983d0d392b84ff945461e5a06d3867e9f9e575f8697b67f"}, -] [package.dependencies] markdown-it-py = ">=2.1.0,<3.0.0" @@ -2018,10 +999,6 @@ description = "Convenient Filesystem interface over S3" category = "main" optional = true python-versions = ">= 3.7" -files = [ - {file = "s3fs-2023.1.0-py3-none-any.whl", hash = "sha256:a549ae518fff4388bd6fca5248575c29f521e7e39efcd2bfab476651701fd114"}, - {file = "s3fs-2023.1.0.tar.gz", hash = "sha256:8b2e28372423e93f26312208a9272e22a962ddd0a79d63f9a68693b6af5ff187"}, -] [package.dependencies] aiobotocore = ">=2.4.2,<2.5.0" @@ -2039,10 +1016,6 @@ description = "An Amazon S3 Transfer Manager" category = "main" optional = false python-versions = ">= 3.7" -files = [ - {file = "s3transfer-0.6.0-py3-none-any.whl", hash = "sha256:06176b74f3a15f61f1b4f25a1fc29a4429040b7647133a463da8fa5bd28d5ecd"}, - {file = "s3transfer-0.6.0.tar.gz", hash = "sha256:2ed07d3866f523cc561bf4a00fc5535827981b117dd7876f036b0c1aca42c947"}, -] [package.dependencies] botocore = ">=1.12.36,<2.0a.0" @@ -2050,23 +1023,6 @@ botocore = ">=1.12.36,<2.0a.0" [package.extras] crt = ["botocore[crt] (>=1.20.29,<2.0a.0)"] -[[package]] -name = "setuptools" -version = "67.0.0" -description = "Easily download, build, install, upgrade, and uninstall Python packages" -category = "dev" -optional = false -python-versions = ">=3.7" -files = [ - {file = "setuptools-67.0.0-py3-none-any.whl", hash = "sha256:9d790961ba6219e9ff7d9557622d2fe136816a264dd01d5997cfc057d804853d"}, - {file = "setuptools-67.0.0.tar.gz", hash = "sha256:883131c5b6efa70b9101c7ef30b2b7b780a4283d5fc1616383cdf22c83cbefe6"}, -] - -[package.extras] -docs = ["furo", "jaraco.packaging (>=9)", "jaraco.tidelift (>=1.4)", "pygments-github-lexers (==0.0.5)", "rst.linker (>=1.9)", "sphinx (>=3.5)", "sphinx-favicon", "sphinx-hoverxref (<2)", "sphinx-inline-tabs", "sphinx-lint", "sphinx-notfound-page (==0.8.3)", "sphinx-reredirects", "sphinxcontrib-towncrier"] -testing = ["build[virtualenv]", "filelock (>=3.4.0)", "flake8 (<5)", "flake8-2020", "ini2toml[lite] (>=0.9)", "jaraco.envs (>=2.2)", "jaraco.path (>=3.2.0)", "pip (>=19.1)", "pip-run (>=8.8)", "pytest (>=6)", "pytest-black (>=0.3.7)", "pytest-checkdocs (>=2.4)", "pytest-cov", "pytest-enabler (>=1.3)", "pytest-flake8", "pytest-mypy (>=0.9.1)", "pytest-perf", "pytest-timeout", "pytest-xdist", "tomli-w (>=1.0.0)", "virtualenv (>=13.0.0)", "wheel"] -testing-integration = ["build[virtualenv]", "filelock (>=3.4.0)", "jaraco.envs (>=2.2)", "jaraco.path (>=3.2.0)", "pytest", "pytest-enabler", "pytest-xdist", "tomli", "virtualenv (>=13.0.0)", "wheel"] - [[package]] name = "six" version = "1.16.0" @@ -2074,142 +1030,1283 @@ description = "Python 2 and 3 compatibility utilities" category = "main" optional = false python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*" -files = [ - {file = "six-1.16.0-py2.py3-none-any.whl", hash = "sha256:8abb2f1d86890a2dfb989f9a77cfcfd3e47c2a354b01111771326f8aa26e0254"}, - {file = "six-1.16.0.tar.gz", hash = "sha256:1e61c37477a1626458e36f7b1d82aa5c9b094fa4802892072e49de9c60c4c926"}, + +[[package]] +name = "thrift" +version = "0.16.0" +description = "Python bindings for the Apache Thrift RPC system" +category = "main" +optional = true +python-versions = "*" + +[package.dependencies] +six = ">=1.7.2" + +[package.extras] +all = ["tornado (>=4.0)", "twisted"] +tornado = ["tornado (>=4.0)"] +twisted = ["twisted"] + +[[package]] +name = "toml" +version = "0.10.2" +description = "Python Library for Tom's Obvious, Minimal Language" +category = "dev" +optional = false +python-versions = ">=2.6, !=3.0.*, !=3.1.*, !=3.2.*" + +[[package]] +name = "tomli" +version = "2.0.1" +description = "A lil' TOML parser" +category = "dev" +optional = false +python-versions = ">=3.7" + +[[package]] +name = "types-toml" +version = "0.10.8.4" +description = "Typing stubs for toml" +category = "dev" +optional = false +python-versions = "*" + +[[package]] +name = "typing-extensions" +version = "4.4.0" +description = "Backported and Experimental Type Hints for Python 3.7+" +category = "main" +optional = false +python-versions = ">=3.7" + +[[package]] +name = "urllib3" +version = "1.26.14" +description = "HTTP library with thread-safe connection pooling, file post, and more." +category = "main" +optional = false +python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*, !=3.5.*" + +[package.extras] +brotli = ["brotli (>=1.0.9)", "brotlicffi (>=0.8.0)", "brotlipy (>=0.6.0)"] +secure = ["certifi", "cryptography (>=1.3.4)", "idna (>=2.0.0)", "ipaddress", "pyOpenSSL (>=0.14)", "urllib3-secure-extra"] +socks = ["PySocks (>=1.5.6,!=1.5.7,<2.0)"] + +[[package]] +name = "virtualenv" +version = "20.19.0" +description = "Virtual Python Environment builder" +category = "dev" +optional = false +python-versions = ">=3.7" + +[package.dependencies] +distlib = ">=0.3.6,<1" +filelock = ">=3.4.1,<4" +platformdirs = ">=2.4,<4" + +[package.extras] +docs = ["furo (>=2022.12.7)", "proselint (>=0.13)", "sphinx (>=6.1.3)", "sphinx-argparse (>=0.4)", "sphinxcontrib-towncrier (>=0.2.1a0)", "towncrier (>=22.12)"] +test = ["covdefaults (>=2.2.2)", "coverage (>=7.1)", "coverage-enable-subprocess (>=1)", "flaky (>=3.7)", "packaging (>=23)", "pytest (>=7.2.1)", "pytest-env (>=0.8.1)", "pytest-freezegun (>=0.4.2)", "pytest-mock (>=3.10)", "pytest-randomly (>=3.12)", "pytest-timeout (>=2.1)"] + +[[package]] +name = "werkzeug" +version = "2.2.3" +description = "The comprehensive WSGI web application library." +category = "dev" +optional = false +python-versions = ">=3.7" + +[package.dependencies] +MarkupSafe = ">=2.1.1" + +[package.extras] +watchdog = ["watchdog"] + +[[package]] +name = "wrapt" +version = "1.14.1" +description = "Module for decorators, wrappers and monkey patching." +category = "main" +optional = true +python-versions = "!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,!=3.4.*,>=2.7" + +[[package]] +name = "xmltodict" +version = "0.13.0" +description = "Makes working with XML feel like you are working with JSON" +category = "dev" +optional = false +python-versions = ">=3.4" + +[[package]] +name = "yarl" +version = "1.8.2" +description = "Yet another URL library" +category = "main" +optional = true +python-versions = ">=3.7" + +[package.dependencies] +idna = ">=2.0" +multidict = ">=4.0" + +[[package]] +name = "zipp" +version = "3.13.0" +description = "Backport of pathlib-compatible object wrapper for zip files" +category = "dev" +optional = false +python-versions = ">=3.7" + +[package.extras] +docs = ["furo", "jaraco.packaging (>=9)", "jaraco.tidelift (>=1.4)", "rst.linker (>=1.9)", "sphinx (>=3.5)", "sphinx-lint"] +testing = ["flake8 (<5)", "func-timeout", "jaraco.functools", "jaraco.itertools", "more-itertools", "pytest (>=6)", "pytest-black (>=0.3.7)", "pytest-checkdocs (>=2.4)", "pytest-cov", "pytest-enabler (>=1.3)", "pytest-flake8", "pytest-mypy (>=0.9.1)"] + +[[package]] +name = "zstandard" +version = "0.19.0" +description = "Zstandard bindings for Python" +category = "main" +optional = false +python-versions = ">=3.6" + +[package.dependencies] +cffi = {version = ">=1.11", markers = "platform_python_implementation == \"PyPy\""} + +[package.extras] +cffi = ["cffi (>=1.11)"] + +[extras] +adlfs = ["adlfs"] +duckdb = ["duckdb", "pyarrow"] +dynamodb = ["boto3"] +glue = ["boto3"] +hive = ["thrift"] +pandas = ["pandas", "pyarrow"] +pyarrow = ["pyarrow"] +s3fs = ["s3fs"] +snappy = ["python-snappy"] + +[metadata] +lock-version = "1.1" +python-versions = "^3.8" +content-hash = "f2c5be6a05f8b56a16a6aee60c46aceeff565711cf5450c83bdebc88cb35b89f" + +[metadata.files] +adal = [ + {file = "adal-1.2.7-py2.py3-none-any.whl", hash = "sha256:2a7451ed7441ddbc57703042204a3e30ef747478eea022c70f789fc7f084bc3d"}, + {file = "adal-1.2.7.tar.gz", hash = "sha256:d74f45b81317454d96e982fd1c50e6fb5c99ac2223728aea8764433a39f566f1"}, +] +adlfs = [ + {file = "adlfs-2023.1.0-py3-none-any.whl", hash = "sha256:ccbdd6d33d5b7bce99a4a07c884c82fb135745d39b2d9b7b672f8e9d4d04407f"}, + {file = "adlfs-2023.1.0.tar.gz", hash = "sha256:eca53f53d88fc8e2e7a2f1d8f5a40b8a1c56aa3b541e4aa2e7eaf55a6a789262"}, +] +aiobotocore = [ + {file = "aiobotocore-2.4.2-py3-none-any.whl", hash = "sha256:4acd1ebe2e44be4b100aa553910bda899f6dc090b3da2bc1cf3d5de2146ed208"}, + {file = "aiobotocore-2.4.2.tar.gz", hash = "sha256:0603b74a582dffa7511ce7548d07dc9b10ec87bc5fb657eb0b34f9bd490958bf"}, +] +aiohttp = [ + {file = "aiohttp-3.8.4-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:5ce45967538fb747370308d3145aa68a074bdecb4f3a300869590f725ced69c1"}, + {file = "aiohttp-3.8.4-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:b744c33b6f14ca26b7544e8d8aadff6b765a80ad6164fb1a430bbadd593dfb1a"}, + {file = "aiohttp-3.8.4-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:1a45865451439eb320784918617ba54b7a377e3501fb70402ab84d38c2cd891b"}, + {file = "aiohttp-3.8.4-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a86d42d7cba1cec432d47ab13b6637bee393a10f664c425ea7b305d1301ca1a3"}, + {file = "aiohttp-3.8.4-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:ee3c36df21b5714d49fc4580247947aa64bcbe2939d1b77b4c8dcb8f6c9faecc"}, + {file = "aiohttp-3.8.4-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:176a64b24c0935869d5bbc4c96e82f89f643bcdf08ec947701b9dbb3c956b7dd"}, + {file = "aiohttp-3.8.4-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:c844fd628851c0bc309f3c801b3a3d58ce430b2ce5b359cd918a5a76d0b20cb5"}, + {file = "aiohttp-3.8.4-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:5393fb786a9e23e4799fec788e7e735de18052f83682ce2dfcabaf1c00c2c08e"}, + {file = "aiohttp-3.8.4-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:e4b09863aae0dc965c3ef36500d891a3ff495a2ea9ae9171e4519963c12ceefd"}, + {file = "aiohttp-3.8.4-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:adfbc22e87365a6e564c804c58fc44ff7727deea782d175c33602737b7feadb6"}, + {file = "aiohttp-3.8.4-cp310-cp310-musllinux_1_1_ppc64le.whl", hash = "sha256:147ae376f14b55f4f3c2b118b95be50a369b89b38a971e80a17c3fd623f280c9"}, + {file = "aiohttp-3.8.4-cp310-cp310-musllinux_1_1_s390x.whl", hash = "sha256:eafb3e874816ebe2a92f5e155f17260034c8c341dad1df25672fb710627c6949"}, + {file = "aiohttp-3.8.4-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:c6cc15d58053c76eacac5fa9152d7d84b8d67b3fde92709195cb984cfb3475ea"}, + {file = "aiohttp-3.8.4-cp310-cp310-win32.whl", hash = "sha256:59f029a5f6e2d679296db7bee982bb3d20c088e52a2977e3175faf31d6fb75d1"}, + {file = "aiohttp-3.8.4-cp310-cp310-win_amd64.whl", hash = "sha256:fe7ba4a51f33ab275515f66b0a236bcde4fb5561498fe8f898d4e549b2e4509f"}, + {file = "aiohttp-3.8.4-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:3d8ef1a630519a26d6760bc695842579cb09e373c5f227a21b67dc3eb16cfea4"}, + {file = "aiohttp-3.8.4-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:5b3f2e06a512e94722886c0827bee9807c86a9f698fac6b3aee841fab49bbfb4"}, + {file = "aiohttp-3.8.4-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:3a80464982d41b1fbfe3154e440ba4904b71c1a53e9cd584098cd41efdb188ef"}, + {file = "aiohttp-3.8.4-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:8b631e26df63e52f7cce0cce6507b7a7f1bc9b0c501fcde69742130b32e8782f"}, + {file = "aiohttp-3.8.4-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:3f43255086fe25e36fd5ed8f2ee47477408a73ef00e804cb2b5cba4bf2ac7f5e"}, + {file = "aiohttp-3.8.4-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:4d347a172f866cd1d93126d9b239fcbe682acb39b48ee0873c73c933dd23bd0f"}, + {file = "aiohttp-3.8.4-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:a3fec6a4cb5551721cdd70473eb009d90935b4063acc5f40905d40ecfea23e05"}, + {file = "aiohttp-3.8.4-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:80a37fe8f7c1e6ce8f2d9c411676e4bc633a8462844e38f46156d07a7d401654"}, + {file = "aiohttp-3.8.4-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:d1e6a862b76f34395a985b3cd39a0d949ca80a70b6ebdea37d3ab39ceea6698a"}, + {file = "aiohttp-3.8.4-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:cd468460eefef601ece4428d3cf4562459157c0f6523db89365202c31b6daebb"}, + {file = "aiohttp-3.8.4-cp311-cp311-musllinux_1_1_ppc64le.whl", hash = "sha256:618c901dd3aad4ace71dfa0f5e82e88b46ef57e3239fc7027773cb6d4ed53531"}, + {file = "aiohttp-3.8.4-cp311-cp311-musllinux_1_1_s390x.whl", hash = "sha256:652b1bff4f15f6287550b4670546a2947f2a4575b6c6dff7760eafb22eacbf0b"}, + {file = "aiohttp-3.8.4-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:80575ba9377c5171407a06d0196b2310b679dc752d02a1fcaa2bc20b235dbf24"}, + {file = "aiohttp-3.8.4-cp311-cp311-win32.whl", hash = "sha256:bbcf1a76cf6f6dacf2c7f4d2ebd411438c275faa1dc0c68e46eb84eebd05dd7d"}, + {file = "aiohttp-3.8.4-cp311-cp311-win_amd64.whl", hash = "sha256:6e74dd54f7239fcffe07913ff8b964e28b712f09846e20de78676ce2a3dc0bfc"}, + {file = "aiohttp-3.8.4-cp36-cp36m-macosx_10_9_x86_64.whl", hash = "sha256:880e15bb6dad90549b43f796b391cfffd7af373f4646784795e20d92606b7a51"}, + {file = "aiohttp-3.8.4-cp36-cp36m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:bb96fa6b56bb536c42d6a4a87dfca570ff8e52de2d63cabebfd6fb67049c34b6"}, + {file = "aiohttp-3.8.4-cp36-cp36m-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:4a6cadebe132e90cefa77e45f2d2f1a4b2ce5c6b1bfc1656c1ddafcfe4ba8131"}, + {file = "aiohttp-3.8.4-cp36-cp36m-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:f352b62b45dff37b55ddd7b9c0c8672c4dd2eb9c0f9c11d395075a84e2c40f75"}, + {file = "aiohttp-3.8.4-cp36-cp36m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:7ab43061a0c81198d88f39aaf90dae9a7744620978f7ef3e3708339b8ed2ef01"}, + {file = "aiohttp-3.8.4-cp36-cp36m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:c9cb1565a7ad52e096a6988e2ee0397f72fe056dadf75d17fa6b5aebaea05622"}, + {file = "aiohttp-3.8.4-cp36-cp36m-musllinux_1_1_aarch64.whl", hash = "sha256:1b3ea7edd2d24538959c1c1abf97c744d879d4e541d38305f9bd7d9b10c9ec41"}, + {file = "aiohttp-3.8.4-cp36-cp36m-musllinux_1_1_i686.whl", hash = "sha256:7c7837fe8037e96b6dd5cfcf47263c1620a9d332a87ec06a6ca4564e56bd0f36"}, + {file = "aiohttp-3.8.4-cp36-cp36m-musllinux_1_1_ppc64le.whl", hash = "sha256:3b90467ebc3d9fa5b0f9b6489dfb2c304a1db7b9946fa92aa76a831b9d587e99"}, + {file = "aiohttp-3.8.4-cp36-cp36m-musllinux_1_1_s390x.whl", hash = "sha256:cab9401de3ea52b4b4c6971db5fb5c999bd4260898af972bf23de1c6b5dd9d71"}, + {file = "aiohttp-3.8.4-cp36-cp36m-musllinux_1_1_x86_64.whl", hash = "sha256:d1f9282c5f2b5e241034a009779e7b2a1aa045f667ff521e7948ea9b56e0c5ff"}, + {file = "aiohttp-3.8.4-cp36-cp36m-win32.whl", hash = "sha256:5e14f25765a578a0a634d5f0cd1e2c3f53964553a00347998dfdf96b8137f777"}, + {file = "aiohttp-3.8.4-cp36-cp36m-win_amd64.whl", hash = "sha256:4c745b109057e7e5f1848c689ee4fb3a016c8d4d92da52b312f8a509f83aa05e"}, + {file = "aiohttp-3.8.4-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:aede4df4eeb926c8fa70de46c340a1bc2c6079e1c40ccf7b0eae1313ffd33519"}, + {file = "aiohttp-3.8.4-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:4ddaae3f3d32fc2cb4c53fab020b69a05c8ab1f02e0e59665c6f7a0d3a5be54f"}, + {file = "aiohttp-3.8.4-cp37-cp37m-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:c4eb3b82ca349cf6fadcdc7abcc8b3a50ab74a62e9113ab7a8ebc268aad35bb9"}, + {file = "aiohttp-3.8.4-cp37-cp37m-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:9bcb89336efa095ea21b30f9e686763f2be4478f1b0a616969551982c4ee4c3b"}, + {file = "aiohttp-3.8.4-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:6c08e8ed6fa3d477e501ec9db169bfac8140e830aa372d77e4a43084d8dd91ab"}, + {file = "aiohttp-3.8.4-cp37-cp37m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:c6cd05ea06daca6ad6a4ca3ba7fe7dc5b5de063ff4daec6170ec0f9979f6c332"}, + {file = "aiohttp-3.8.4-cp37-cp37m-musllinux_1_1_aarch64.whl", hash = "sha256:b7a00a9ed8d6e725b55ef98b1b35c88013245f35f68b1b12c5cd4100dddac333"}, + {file = "aiohttp-3.8.4-cp37-cp37m-musllinux_1_1_i686.whl", hash = "sha256:de04b491d0e5007ee1b63a309956eaed959a49f5bb4e84b26c8f5d49de140fa9"}, + {file = "aiohttp-3.8.4-cp37-cp37m-musllinux_1_1_ppc64le.whl", hash = "sha256:40653609b3bf50611356e6b6554e3a331f6879fa7116f3959b20e3528783e699"}, + {file = "aiohttp-3.8.4-cp37-cp37m-musllinux_1_1_s390x.whl", hash = "sha256:dbf3a08a06b3f433013c143ebd72c15cac33d2914b8ea4bea7ac2c23578815d6"}, + {file = "aiohttp-3.8.4-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:854f422ac44af92bfe172d8e73229c270dc09b96535e8a548f99c84f82dde241"}, + {file = "aiohttp-3.8.4-cp37-cp37m-win32.whl", hash = "sha256:aeb29c84bb53a84b1a81c6c09d24cf33bb8432cc5c39979021cc0f98c1292a1a"}, + {file = "aiohttp-3.8.4-cp37-cp37m-win_amd64.whl", hash = "sha256:db3fc6120bce9f446d13b1b834ea5b15341ca9ff3f335e4a951a6ead31105480"}, + {file = "aiohttp-3.8.4-cp38-cp38-macosx_10_9_universal2.whl", hash = "sha256:fabb87dd8850ef0f7fe2b366d44b77d7e6fa2ea87861ab3844da99291e81e60f"}, + {file = "aiohttp-3.8.4-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:91f6d540163f90bbaef9387e65f18f73ffd7c79f5225ac3d3f61df7b0d01ad15"}, + {file = "aiohttp-3.8.4-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:d265f09a75a79a788237d7f9054f929ced2e69eb0bb79de3798c468d8a90f945"}, + {file = "aiohttp-3.8.4-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:3d89efa095ca7d442a6d0cbc755f9e08190ba40069b235c9886a8763b03785da"}, + {file = "aiohttp-3.8.4-cp38-cp38-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:4dac314662f4e2aa5009977b652d9b8db7121b46c38f2073bfeed9f4049732cd"}, + {file = "aiohttp-3.8.4-cp38-cp38-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:fe11310ae1e4cd560035598c3f29d86cef39a83d244c7466f95c27ae04850f10"}, + {file = "aiohttp-3.8.4-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:6ddb2a2026c3f6a68c3998a6c47ab6795e4127315d2e35a09997da21865757f8"}, + {file = "aiohttp-3.8.4-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:e75b89ac3bd27d2d043b234aa7b734c38ba1b0e43f07787130a0ecac1e12228a"}, + {file = "aiohttp-3.8.4-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:6e601588f2b502c93c30cd5a45bfc665faaf37bbe835b7cfd461753068232074"}, + {file = "aiohttp-3.8.4-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:a5d794d1ae64e7753e405ba58e08fcfa73e3fad93ef9b7e31112ef3c9a0efb52"}, + {file = "aiohttp-3.8.4-cp38-cp38-musllinux_1_1_ppc64le.whl", hash = "sha256:a1f4689c9a1462f3df0a1f7e797791cd6b124ddbee2b570d34e7f38ade0e2c71"}, + {file = "aiohttp-3.8.4-cp38-cp38-musllinux_1_1_s390x.whl", hash = "sha256:3032dcb1c35bc330134a5b8a5d4f68c1a87252dfc6e1262c65a7e30e62298275"}, + {file = "aiohttp-3.8.4-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:8189c56eb0ddbb95bfadb8f60ea1b22fcfa659396ea36f6adcc521213cd7b44d"}, + {file = "aiohttp-3.8.4-cp38-cp38-win32.whl", hash = "sha256:33587f26dcee66efb2fff3c177547bd0449ab7edf1b73a7f5dea1e38609a0c54"}, + {file = "aiohttp-3.8.4-cp38-cp38-win_amd64.whl", hash = "sha256:e595432ac259af2d4630008bf638873d69346372d38255774c0e286951e8b79f"}, + {file = "aiohttp-3.8.4-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:5a7bdf9e57126dc345b683c3632e8ba317c31d2a41acd5800c10640387d193ed"}, + {file = "aiohttp-3.8.4-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:22f6eab15b6db242499a16de87939a342f5a950ad0abaf1532038e2ce7d31567"}, + {file = "aiohttp-3.8.4-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:7235604476a76ef249bd64cb8274ed24ccf6995c4a8b51a237005ee7a57e8643"}, + {file = "aiohttp-3.8.4-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:ea9eb976ffdd79d0e893869cfe179a8f60f152d42cb64622fca418cd9b18dc2a"}, + {file = "aiohttp-3.8.4-cp39-cp39-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:92c0cea74a2a81c4c76b62ea1cac163ecb20fb3ba3a75c909b9fa71b4ad493cf"}, + {file = "aiohttp-3.8.4-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:493f5bc2f8307286b7799c6d899d388bbaa7dfa6c4caf4f97ef7521b9cb13719"}, + {file = "aiohttp-3.8.4-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:0a63f03189a6fa7c900226e3ef5ba4d3bd047e18f445e69adbd65af433add5a2"}, + {file = "aiohttp-3.8.4-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:10c8cefcff98fd9168cdd86c4da8b84baaa90bf2da2269c6161984e6737bf23e"}, + {file = "aiohttp-3.8.4-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:bca5f24726e2919de94f047739d0a4fc01372801a3672708260546aa2601bf57"}, + {file = "aiohttp-3.8.4-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:03baa76b730e4e15a45f81dfe29a8d910314143414e528737f8589ec60cf7391"}, + {file = "aiohttp-3.8.4-cp39-cp39-musllinux_1_1_ppc64le.whl", hash = "sha256:8c29c77cc57e40f84acef9bfb904373a4e89a4e8b74e71aa8075c021ec9078c2"}, + {file = "aiohttp-3.8.4-cp39-cp39-musllinux_1_1_s390x.whl", hash = "sha256:03543dcf98a6619254b409be2d22b51f21ec66272be4ebda7b04e6412e4b2e14"}, + {file = "aiohttp-3.8.4-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:17b79c2963db82086229012cff93ea55196ed31f6493bb1ccd2c62f1724324e4"}, + {file = "aiohttp-3.8.4-cp39-cp39-win32.whl", hash = "sha256:34ce9f93a4a68d1272d26030655dd1b58ff727b3ed2a33d80ec433561b03d67a"}, + {file = "aiohttp-3.8.4-cp39-cp39-win_amd64.whl", hash = "sha256:41a86a69bb63bb2fc3dc9ad5ea9f10f1c9c8e282b471931be0268ddd09430b04"}, + {file = "aiohttp-3.8.4.tar.gz", hash = "sha256:bf2e1a9162c1e441bf805a1fd166e249d574ca04e03b34f97e2928769e91ab5c"}, +] +aioitertools = [ + {file = "aioitertools-0.11.0-py3-none-any.whl", hash = "sha256:04b95e3dab25b449def24d7df809411c10e62aab0cbe31a50ca4e68748c43394"}, + {file = "aioitertools-0.11.0.tar.gz", hash = "sha256:42c68b8dd3a69c2bf7f2233bf7df4bb58b557bca5252ac02ed5187bbc67d6831"}, +] +aiosignal = [ + {file = "aiosignal-1.3.1-py3-none-any.whl", hash = "sha256:f8376fb07dd1e86a584e4fcdec80b36b7f81aac666ebc724e2c090300dd83b17"}, + {file = "aiosignal-1.3.1.tar.gz", hash = "sha256:54cd96e15e1649b75d6c87526a6ff0b6c1b0dd3459f43d9ca11d48c339b68cfc"}, +] +async-timeout = [] +attrs = [ + {file = "attrs-22.2.0-py3-none-any.whl", hash = "sha256:29e95c7f6778868dbd49170f98f8818f78f3dc5e0e37c0b1f474e3561b240836"}, + {file = "attrs-22.2.0.tar.gz", hash = "sha256:c9227bfc2f01993c03f68db37d1d15c9690188323c067c641f1a35ca58185f99"}, +] +azure-core = [ + {file = "azure-core-1.26.3.zip", hash = "sha256:acbd0daa9675ce88623da35c80d819cdafa91731dee6b2695c64d7ca9da82db4"}, + {file = "azure_core-1.26.3-py3-none-any.whl", hash = "sha256:f7bad0a7b4d800d6e73733ea7e13a616016221ac111ff9a344fe4cba41e51bbe"}, +] +azure-datalake-store = [ + {file = "azure-datalake-store-0.0.52.tar.gz", hash = "sha256:4198ddb32614d16d4502b43d5c9739f81432b7e0e4d75d30e05149fe6007fea2"}, + {file = "azure_datalake_store-0.0.52-py2.py3-none-any.whl", hash = "sha256:aaed72b9c856824aeab554f4dbe0ef2c6d0ff36700bdd8b93d8298793117c48e"}, +] +azure-identity = [ + {file = "azure-identity-1.12.0.zip", hash = "sha256:7f9b1ae7d97ea7af3f38dd09305e19ab81a1e16ab66ea186b6579d85c1ca2347"}, + {file = "azure_identity-1.12.0-py3-none-any.whl", hash = "sha256:2a58ce4a209a013e37eaccfd5937570ab99e9118b3e1acf875eed3a85d541b92"}, +] +azure-storage-blob = [ + {file = "azure-storage-blob-12.14.1.zip", hash = "sha256:860d4d82985a4bfc7d3271e71275af330f54f330a754355435a7ba749ccde997"}, + {file = "azure_storage_blob-12.14.1-py3-none-any.whl", hash = "sha256:52b84658e8df7853a3cf1c563814655b5028b979b2a87905b92aa6bb30be240e"}, +] +boto3 = [ + {file = "boto3-1.24.59-py3-none-any.whl", hash = "sha256:34ab44146a2c4e7f4e72737f4b27e6eb5e0a7855c2f4599e3d9199b6a0a2d575"}, + {file = "boto3-1.24.59.tar.gz", hash = "sha256:a50b4323f9579cfe22fcf5531fbd40b567d4d74c1adce06aeb5c95fce2a6fb40"}, +] +botocore = [ + {file = "botocore-1.27.59-py3-none-any.whl", hash = "sha256:69d756791fc024bda54f6c53f71ae34e695ee41bbbc1743d9179c4837a4929da"}, + {file = "botocore-1.27.59.tar.gz", hash = "sha256:eda4aed6ee719a745d1288eaf1beb12f6f6448ad1fa12f159405db14ba9c92cf"}, +] +build = [ + {file = "build-0.10.0-py3-none-any.whl", hash = "sha256:af266720050a66c893a6096a2f410989eeac74ff9a68ba194b3f6473e8e26171"}, + {file = "build-0.10.0.tar.gz", hash = "sha256:d5b71264afdb5951d6704482aac78de887c80691c52b88a9ad195983ca2c9269"}, +] +certifi = [ + {file = "certifi-2022.12.7-py3-none-any.whl", hash = "sha256:4ad3232f5e926d6718ec31cfc1fcadfde020920e278684144551c91769c7bc18"}, + {file = "certifi-2022.12.7.tar.gz", hash = "sha256:35824b4c3a97115964b408844d64aa14db1cc518f6562e8d7261699d1350a9e3"}, +] +cffi = [ + {file = "cffi-1.15.1-cp27-cp27m-macosx_10_9_x86_64.whl", hash = "sha256:a66d3508133af6e8548451b25058d5812812ec3798c886bf38ed24a98216fab2"}, + {file = "cffi-1.15.1-cp27-cp27m-manylinux1_i686.whl", hash = "sha256:470c103ae716238bbe698d67ad020e1db9d9dba34fa5a899b5e21577e6d52ed2"}, + {file = "cffi-1.15.1-cp27-cp27m-manylinux1_x86_64.whl", hash = "sha256:9ad5db27f9cabae298d151c85cf2bad1d359a1b9c686a275df03385758e2f914"}, + {file = "cffi-1.15.1-cp27-cp27m-win32.whl", hash = "sha256:b3bbeb01c2b273cca1e1e0c5df57f12dce9a4dd331b4fa1635b8bec26350bde3"}, + {file = "cffi-1.15.1-cp27-cp27m-win_amd64.whl", hash = "sha256:e00b098126fd45523dd056d2efba6c5a63b71ffe9f2bbe1a4fe1716e1d0c331e"}, + {file = "cffi-1.15.1-cp27-cp27mu-manylinux1_i686.whl", hash = "sha256:d61f4695e6c866a23a21acab0509af1cdfd2c013cf256bbf5b6b5e2695827162"}, + {file = "cffi-1.15.1-cp27-cp27mu-manylinux1_x86_64.whl", hash = "sha256:ed9cb427ba5504c1dc15ede7d516b84757c3e3d7868ccc85121d9310d27eed0b"}, + {file = "cffi-1.15.1-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:39d39875251ca8f612b6f33e6b1195af86d1b3e60086068be9cc053aa4376e21"}, + {file = "cffi-1.15.1-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:285d29981935eb726a4399badae8f0ffdff4f5050eaa6d0cfc3f64b857b77185"}, + {file = "cffi-1.15.1-cp310-cp310-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:3eb6971dcff08619f8d91607cfc726518b6fa2a9eba42856be181c6d0d9515fd"}, + {file = "cffi-1.15.1-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:21157295583fe8943475029ed5abdcf71eb3911894724e360acff1d61c1d54bc"}, + {file = "cffi-1.15.1-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:5635bd9cb9731e6d4a1132a498dd34f764034a8ce60cef4f5319c0541159392f"}, + {file = "cffi-1.15.1-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:2012c72d854c2d03e45d06ae57f40d78e5770d252f195b93f581acf3ba44496e"}, + {file = "cffi-1.15.1-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:dd86c085fae2efd48ac91dd7ccffcfc0571387fe1193d33b6394db7ef31fe2a4"}, + {file = "cffi-1.15.1-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:fa6693661a4c91757f4412306191b6dc88c1703f780c8234035eac011922bc01"}, + {file = "cffi-1.15.1-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:59c0b02d0a6c384d453fece7566d1c7e6b7bae4fc5874ef2ef46d56776d61c9e"}, + {file = "cffi-1.15.1-cp310-cp310-win32.whl", hash = "sha256:cba9d6b9a7d64d4bd46167096fc9d2f835e25d7e4c121fb2ddfc6528fb0413b2"}, + {file = "cffi-1.15.1-cp310-cp310-win_amd64.whl", hash = "sha256:ce4bcc037df4fc5e3d184794f27bdaab018943698f4ca31630bc7f84a7b69c6d"}, + {file = "cffi-1.15.1-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:3d08afd128ddaa624a48cf2b859afef385b720bb4b43df214f85616922e6a5ac"}, + {file = "cffi-1.15.1-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:3799aecf2e17cf585d977b780ce79ff0dc9b78d799fc694221ce814c2c19db83"}, + {file = "cffi-1.15.1-cp311-cp311-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:a591fe9e525846e4d154205572a029f653ada1a78b93697f3b5a8f1f2bc055b9"}, + {file = "cffi-1.15.1-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:3548db281cd7d2561c9ad9984681c95f7b0e38881201e157833a2342c30d5e8c"}, + {file = "cffi-1.15.1-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:91fc98adde3d7881af9b59ed0294046f3806221863722ba7d8d120c575314325"}, + {file = "cffi-1.15.1-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:94411f22c3985acaec6f83c6df553f2dbe17b698cc7f8ae751ff2237d96b9e3c"}, + {file = "cffi-1.15.1-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:03425bdae262c76aad70202debd780501fabeaca237cdfddc008987c0e0f59ef"}, + {file = "cffi-1.15.1-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:cc4d65aeeaa04136a12677d3dd0b1c0c94dc43abac5860ab33cceb42b801c1e8"}, + {file = "cffi-1.15.1-cp311-cp311-win32.whl", hash = "sha256:a0f100c8912c114ff53e1202d0078b425bee3649ae34d7b070e9697f93c5d52d"}, + {file = "cffi-1.15.1-cp311-cp311-win_amd64.whl", hash = "sha256:04ed324bda3cda42b9b695d51bb7d54b680b9719cfab04227cdd1e04e5de3104"}, + {file = "cffi-1.15.1-cp36-cp36m-macosx_10_9_x86_64.whl", hash = "sha256:50a74364d85fd319352182ef59c5c790484a336f6db772c1a9231f1c3ed0cbd7"}, + {file = "cffi-1.15.1-cp36-cp36m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:e263d77ee3dd201c3a142934a086a4450861778baaeeb45db4591ef65550b0a6"}, + {file = "cffi-1.15.1-cp36-cp36m-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:cec7d9412a9102bdc577382c3929b337320c4c4c4849f2c5cdd14d7368c5562d"}, + {file = "cffi-1.15.1-cp36-cp36m-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:4289fc34b2f5316fbb762d75362931e351941fa95fa18789191b33fc4cf9504a"}, + {file = "cffi-1.15.1-cp36-cp36m-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:173379135477dc8cac4bc58f45db08ab45d228b3363adb7af79436135d028405"}, + {file = "cffi-1.15.1-cp36-cp36m-manylinux_2_5_x86_64.manylinux1_x86_64.whl", hash = "sha256:6975a3fac6bc83c4a65c9f9fcab9e47019a11d3d2cf7f3c0d03431bf145a941e"}, + {file = "cffi-1.15.1-cp36-cp36m-win32.whl", hash = "sha256:2470043b93ff09bf8fb1d46d1cb756ce6132c54826661a32d4e4d132e1977adf"}, + {file = "cffi-1.15.1-cp36-cp36m-win_amd64.whl", hash = "sha256:30d78fbc8ebf9c92c9b7823ee18eb92f2e6ef79b45ac84db507f52fbe3ec4497"}, + {file = "cffi-1.15.1-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:198caafb44239b60e252492445da556afafc7d1e3ab7a1fb3f0584ef6d742375"}, + {file = "cffi-1.15.1-cp37-cp37m-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:5ef34d190326c3b1f822a5b7a45f6c4535e2f47ed06fec77d3d799c450b2651e"}, + {file = "cffi-1.15.1-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:8102eaf27e1e448db915d08afa8b41d6c7ca7a04b7d73af6514df10a3e74bd82"}, + {file = "cffi-1.15.1-cp37-cp37m-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:5df2768244d19ab7f60546d0c7c63ce1581f7af8b5de3eb3004b9b6fc8a9f84b"}, + {file = "cffi-1.15.1-cp37-cp37m-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:a8c4917bd7ad33e8eb21e9a5bbba979b49d9a97acb3a803092cbc1133e20343c"}, + {file = "cffi-1.15.1-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:0e2642fe3142e4cc4af0799748233ad6da94c62a8bec3a6648bf8ee68b1c7426"}, + {file = "cffi-1.15.1-cp37-cp37m-win32.whl", hash = "sha256:e229a521186c75c8ad9490854fd8bbdd9a0c9aa3a524326b55be83b54d4e0ad9"}, + {file = "cffi-1.15.1-cp37-cp37m-win_amd64.whl", hash = "sha256:a0b71b1b8fbf2b96e41c4d990244165e2c9be83d54962a9a1d118fd8657d2045"}, + {file = "cffi-1.15.1-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:320dab6e7cb2eacdf0e658569d2575c4dad258c0fcc794f46215e1e39f90f2c3"}, + {file = "cffi-1.15.1-cp38-cp38-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:1e74c6b51a9ed6589199c787bf5f9875612ca4a8a0785fb2d4a84429badaf22a"}, + {file = "cffi-1.15.1-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a5c84c68147988265e60416b57fc83425a78058853509c1b0629c180094904a5"}, + {file = "cffi-1.15.1-cp38-cp38-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:3b926aa83d1edb5aa5b427b4053dc420ec295a08e40911296b9eb1b6170f6cca"}, + {file = "cffi-1.15.1-cp38-cp38-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:87c450779d0914f2861b8526e035c5e6da0a3199d8f1add1a665e1cbc6fc6d02"}, + {file = "cffi-1.15.1-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:4f2c9f67e9821cad2e5f480bc8d83b8742896f1242dba247911072d4fa94c192"}, + {file = "cffi-1.15.1-cp38-cp38-win32.whl", hash = "sha256:8b7ee99e510d7b66cdb6c593f21c043c248537a32e0bedf02e01e9553a172314"}, + {file = "cffi-1.15.1-cp38-cp38-win_amd64.whl", hash = "sha256:00a9ed42e88df81ffae7a8ab6d9356b371399b91dbdf0c3cb1e84c03a13aceb5"}, + {file = "cffi-1.15.1-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:54a2db7b78338edd780e7ef7f9f6c442500fb0d41a5a4ea24fff1c929d5af585"}, + {file = "cffi-1.15.1-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:fcd131dd944808b5bdb38e6f5b53013c5aa4f334c5cad0c72742f6eba4b73db0"}, + {file = "cffi-1.15.1-cp39-cp39-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:7473e861101c9e72452f9bf8acb984947aa1661a7704553a9f6e4baa5ba64415"}, + {file = "cffi-1.15.1-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:6c9a799e985904922a4d207a94eae35c78ebae90e128f0c4e521ce339396be9d"}, + {file = "cffi-1.15.1-cp39-cp39-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:3bcde07039e586f91b45c88f8583ea7cf7a0770df3a1649627bf598332cb6984"}, + {file = "cffi-1.15.1-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:33ab79603146aace82c2427da5ca6e58f2b3f2fb5da893ceac0c42218a40be35"}, + {file = "cffi-1.15.1-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:5d598b938678ebf3c67377cdd45e09d431369c3b1a5b331058c338e201f12b27"}, + {file = "cffi-1.15.1-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:db0fbb9c62743ce59a9ff687eb5f4afbe77e5e8403d6697f7446e5f609976f76"}, + {file = "cffi-1.15.1-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:98d85c6a2bef81588d9227dde12db8a7f47f639f4a17c9ae08e773aa9c697bf3"}, + {file = "cffi-1.15.1-cp39-cp39-win32.whl", hash = "sha256:40f4774f5a9d4f5e344f31a32b5096977b5d48560c5592e2f3d2c4374bd543ee"}, + {file = "cffi-1.15.1-cp39-cp39-win_amd64.whl", hash = "sha256:70df4e3b545a17496c9b3f41f5115e69a4f2e77e94e1d2a8e1070bc0c38c8a3c"}, + {file = "cffi-1.15.1.tar.gz", hash = "sha256:d400bfb9a37b1351253cb402671cea7e89bdecc294e8016a707f6d1d8ac934f9"}, +] +cfgv = [ + {file = "cfgv-3.3.1-py2.py3-none-any.whl", hash = "sha256:c6a0883f3917a037485059700b9e75da2464e6c27051014ad85ba6aaa5884426"}, + {file = "cfgv-3.3.1.tar.gz", hash = "sha256:f5a830efb9ce7a445376bb66ec94c638a9787422f96264c98edc6bdeed8ab736"}, +] +charset-normalizer = [ + {file = "charset-normalizer-3.0.1.tar.gz", hash = "sha256:ebea339af930f8ca5d7a699b921106c6e29c617fe9606fa7baa043c1cdae326f"}, + {file = "charset_normalizer-3.0.1-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:88600c72ef7587fe1708fd242b385b6ed4b8904976d5da0893e31df8b3480cb6"}, + {file = "charset_normalizer-3.0.1-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:c75ffc45f25324e68ab238cb4b5c0a38cd1c3d7f1fb1f72b5541de469e2247db"}, + {file = "charset_normalizer-3.0.1-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:db72b07027db150f468fbada4d85b3b2729a3db39178abf5c543b784c1254539"}, + {file = "charset_normalizer-3.0.1-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:62595ab75873d50d57323a91dd03e6966eb79c41fa834b7a1661ed043b2d404d"}, + {file = "charset_normalizer-3.0.1-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:ff6f3db31555657f3163b15a6b7c6938d08df7adbfc9dd13d9d19edad678f1e8"}, + {file = "charset_normalizer-3.0.1-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:772b87914ff1152b92a197ef4ea40efe27a378606c39446ded52c8f80f79702e"}, + {file = "charset_normalizer-3.0.1-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:70990b9c51340e4044cfc394a81f614f3f90d41397104d226f21e66de668730d"}, + {file = "charset_normalizer-3.0.1-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:292d5e8ba896bbfd6334b096e34bffb56161c81408d6d036a7dfa6929cff8783"}, + {file = "charset_normalizer-3.0.1-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:2edb64ee7bf1ed524a1da60cdcd2e1f6e2b4f66ef7c077680739f1641f62f555"}, + {file = "charset_normalizer-3.0.1-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:31a9ddf4718d10ae04d9b18801bd776693487cbb57d74cc3458a7673f6f34639"}, + {file = "charset_normalizer-3.0.1-cp310-cp310-musllinux_1_1_ppc64le.whl", hash = "sha256:44ba614de5361b3e5278e1241fda3dc1838deed864b50a10d7ce92983797fa76"}, + {file = "charset_normalizer-3.0.1-cp310-cp310-musllinux_1_1_s390x.whl", hash = "sha256:12db3b2c533c23ab812c2b25934f60383361f8a376ae272665f8e48b88e8e1c6"}, + {file = "charset_normalizer-3.0.1-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:c512accbd6ff0270939b9ac214b84fb5ada5f0409c44298361b2f5e13f9aed9e"}, + {file = "charset_normalizer-3.0.1-cp310-cp310-win32.whl", hash = "sha256:502218f52498a36d6bf5ea77081844017bf7982cdbe521ad85e64cabee1b608b"}, + {file = "charset_normalizer-3.0.1-cp310-cp310-win_amd64.whl", hash = "sha256:601f36512f9e28f029d9481bdaf8e89e5148ac5d89cffd3b05cd533eeb423b59"}, + {file = "charset_normalizer-3.0.1-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:0298eafff88c99982a4cf66ba2efa1128e4ddaca0b05eec4c456bbc7db691d8d"}, + {file = "charset_normalizer-3.0.1-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:a8d0fc946c784ff7f7c3742310cc8a57c5c6dc31631269876a88b809dbeff3d3"}, + {file = "charset_normalizer-3.0.1-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:87701167f2a5c930b403e9756fab1d31d4d4da52856143b609e30a1ce7160f3c"}, + {file = "charset_normalizer-3.0.1-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:14e76c0f23218b8f46c4d87018ca2e441535aed3632ca134b10239dfb6dadd6b"}, + {file = "charset_normalizer-3.0.1-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:0c0a590235ccd933d9892c627dec5bc7511ce6ad6c1011fdf5b11363022746c1"}, + {file = "charset_normalizer-3.0.1-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:8c7fe7afa480e3e82eed58e0ca89f751cd14d767638e2550c77a92a9e749c317"}, + {file = "charset_normalizer-3.0.1-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:79909e27e8e4fcc9db4addea88aa63f6423ebb171db091fb4373e3312cb6d603"}, + {file = "charset_normalizer-3.0.1-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:8ac7b6a045b814cf0c47f3623d21ebd88b3e8cf216a14790b455ea7ff0135d18"}, + {file = "charset_normalizer-3.0.1-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:72966d1b297c741541ca8cf1223ff262a6febe52481af742036a0b296e35fa5a"}, + {file = "charset_normalizer-3.0.1-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:f9d0c5c045a3ca9bedfc35dca8526798eb91a07aa7a2c0fee134c6c6f321cbd7"}, + {file = "charset_normalizer-3.0.1-cp311-cp311-musllinux_1_1_ppc64le.whl", hash = "sha256:5995f0164fa7df59db4746112fec3f49c461dd6b31b841873443bdb077c13cfc"}, + {file = "charset_normalizer-3.0.1-cp311-cp311-musllinux_1_1_s390x.whl", hash = "sha256:4a8fcf28c05c1f6d7e177a9a46a1c52798bfe2ad80681d275b10dcf317deaf0b"}, + {file = "charset_normalizer-3.0.1-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:761e8904c07ad053d285670f36dd94e1b6ab7f16ce62b9805c475b7aa1cffde6"}, + {file = "charset_normalizer-3.0.1-cp311-cp311-win32.whl", hash = "sha256:71140351489970dfe5e60fc621ada3e0f41104a5eddaca47a7acb3c1b851d6d3"}, + {file = "charset_normalizer-3.0.1-cp311-cp311-win_amd64.whl", hash = "sha256:9ab77acb98eba3fd2a85cd160851816bfce6871d944d885febf012713f06659c"}, + {file = "charset_normalizer-3.0.1-cp36-cp36m-macosx_10_9_x86_64.whl", hash = "sha256:84c3990934bae40ea69a82034912ffe5a62c60bbf6ec5bc9691419641d7d5c9a"}, + {file = "charset_normalizer-3.0.1-cp36-cp36m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:74292fc76c905c0ef095fe11e188a32ebd03bc38f3f3e9bcb85e4e6db177b7ea"}, + {file = "charset_normalizer-3.0.1-cp36-cp36m-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:c95a03c79bbe30eec3ec2b7f076074f4281526724c8685a42872974ef4d36b72"}, + {file = "charset_normalizer-3.0.1-cp36-cp36m-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:f4c39b0e3eac288fedc2b43055cfc2ca7a60362d0e5e87a637beac5d801ef478"}, + {file = "charset_normalizer-3.0.1-cp36-cp36m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:df2c707231459e8a4028eabcd3cfc827befd635b3ef72eada84ab13b52e1574d"}, + {file = "charset_normalizer-3.0.1-cp36-cp36m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:93ad6d87ac18e2a90b0fe89df7c65263b9a99a0eb98f0a3d2e079f12a0735837"}, + {file = "charset_normalizer-3.0.1-cp36-cp36m-musllinux_1_1_aarch64.whl", hash = "sha256:59e5686dd847347e55dffcc191a96622f016bc0ad89105e24c14e0d6305acbc6"}, + {file = "charset_normalizer-3.0.1-cp36-cp36m-musllinux_1_1_i686.whl", hash = "sha256:cd6056167405314a4dc3c173943f11249fa0f1b204f8b51ed4bde1a9cd1834dc"}, + {file = "charset_normalizer-3.0.1-cp36-cp36m-musllinux_1_1_ppc64le.whl", hash = "sha256:083c8d17153ecb403e5e1eb76a7ef4babfc2c48d58899c98fcaa04833e7a2f9a"}, + {file = "charset_normalizer-3.0.1-cp36-cp36m-musllinux_1_1_s390x.whl", hash = "sha256:f5057856d21e7586765171eac8b9fc3f7d44ef39425f85dbcccb13b3ebea806c"}, + {file = "charset_normalizer-3.0.1-cp36-cp36m-musllinux_1_1_x86_64.whl", hash = "sha256:7eb33a30d75562222b64f569c642ff3dc6689e09adda43a082208397f016c39a"}, + {file = "charset_normalizer-3.0.1-cp36-cp36m-win32.whl", hash = "sha256:95dea361dd73757c6f1c0a1480ac499952c16ac83f7f5f4f84f0658a01b8ef41"}, + {file = "charset_normalizer-3.0.1-cp36-cp36m-win_amd64.whl", hash = "sha256:eaa379fcd227ca235d04152ca6704c7cb55564116f8bc52545ff357628e10602"}, + {file = "charset_normalizer-3.0.1-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:3e45867f1f2ab0711d60c6c71746ac53537f1684baa699f4f668d4c6f6ce8e14"}, + {file = "charset_normalizer-3.0.1-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:cadaeaba78750d58d3cc6ac4d1fd867da6fc73c88156b7a3212a3cd4819d679d"}, + {file = "charset_normalizer-3.0.1-cp37-cp37m-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:911d8a40b2bef5b8bbae2e36a0b103f142ac53557ab421dc16ac4aafee6f53dc"}, + {file = "charset_normalizer-3.0.1-cp37-cp37m-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:503e65837c71b875ecdd733877d852adbc465bd82c768a067badd953bf1bc5a3"}, + {file = "charset_normalizer-3.0.1-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:a60332922359f920193b1d4826953c507a877b523b2395ad7bc716ddd386d866"}, + {file = "charset_normalizer-3.0.1-cp37-cp37m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:16a8663d6e281208d78806dbe14ee9903715361cf81f6d4309944e4d1e59ac5b"}, + {file = "charset_normalizer-3.0.1-cp37-cp37m-musllinux_1_1_aarch64.whl", hash = "sha256:a16418ecf1329f71df119e8a65f3aa68004a3f9383821edcb20f0702934d8087"}, + {file = "charset_normalizer-3.0.1-cp37-cp37m-musllinux_1_1_i686.whl", hash = "sha256:9d9153257a3f70d5f69edf2325357251ed20f772b12e593f3b3377b5f78e7ef8"}, + {file = "charset_normalizer-3.0.1-cp37-cp37m-musllinux_1_1_ppc64le.whl", hash = "sha256:02a51034802cbf38db3f89c66fb5d2ec57e6fe7ef2f4a44d070a593c3688667b"}, + {file = "charset_normalizer-3.0.1-cp37-cp37m-musllinux_1_1_s390x.whl", hash = "sha256:2e396d70bc4ef5325b72b593a72c8979999aa52fb8bcf03f701c1b03e1166918"}, + {file = "charset_normalizer-3.0.1-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:11b53acf2411c3b09e6af37e4b9005cba376c872503c8f28218c7243582df45d"}, + {file = "charset_normalizer-3.0.1-cp37-cp37m-win32.whl", hash = "sha256:0bf2dae5291758b6f84cf923bfaa285632816007db0330002fa1de38bfcb7154"}, + {file = "charset_normalizer-3.0.1-cp37-cp37m-win_amd64.whl", hash = "sha256:2c03cc56021a4bd59be889c2b9257dae13bf55041a3372d3295416f86b295fb5"}, + {file = "charset_normalizer-3.0.1-cp38-cp38-macosx_10_9_universal2.whl", hash = "sha256:024e606be3ed92216e2b6952ed859d86b4cfa52cd5bc5f050e7dc28f9b43ec42"}, + {file = "charset_normalizer-3.0.1-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:4b0d02d7102dd0f997580b51edc4cebcf2ab6397a7edf89f1c73b586c614272c"}, + {file = "charset_normalizer-3.0.1-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:358a7c4cb8ba9b46c453b1dd8d9e431452d5249072e4f56cfda3149f6ab1405e"}, + {file = "charset_normalizer-3.0.1-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:81d6741ab457d14fdedc215516665050f3822d3e56508921cc7239f8c8e66a58"}, + {file = "charset_normalizer-3.0.1-cp38-cp38-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:8b8af03d2e37866d023ad0ddea594edefc31e827fee64f8de5611a1dbc373174"}, + {file = "charset_normalizer-3.0.1-cp38-cp38-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:9cf4e8ad252f7c38dd1f676b46514f92dc0ebeb0db5552f5f403509705e24753"}, + {file = "charset_normalizer-3.0.1-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:e696f0dd336161fca9adbb846875d40752e6eba585843c768935ba5c9960722b"}, + {file = "charset_normalizer-3.0.1-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:c22d3fe05ce11d3671297dc8973267daa0f938b93ec716e12e0f6dee81591dc1"}, + {file = "charset_normalizer-3.0.1-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:109487860ef6a328f3eec66f2bf78b0b72400280d8f8ea05f69c51644ba6521a"}, + {file = "charset_normalizer-3.0.1-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:37f8febc8ec50c14f3ec9637505f28e58d4f66752207ea177c1d67df25da5aed"}, + {file = "charset_normalizer-3.0.1-cp38-cp38-musllinux_1_1_ppc64le.whl", hash = "sha256:f97e83fa6c25693c7a35de154681fcc257c1c41b38beb0304b9c4d2d9e164479"}, + {file = "charset_normalizer-3.0.1-cp38-cp38-musllinux_1_1_s390x.whl", hash = "sha256:a152f5f33d64a6be73f1d30c9cc82dfc73cec6477ec268e7c6e4c7d23c2d2291"}, + {file = "charset_normalizer-3.0.1-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:39049da0ffb96c8cbb65cbf5c5f3ca3168990adf3551bd1dee10c48fce8ae820"}, + {file = "charset_normalizer-3.0.1-cp38-cp38-win32.whl", hash = "sha256:4457ea6774b5611f4bed5eaa5df55f70abde42364d498c5134b7ef4c6958e20e"}, + {file = "charset_normalizer-3.0.1-cp38-cp38-win_amd64.whl", hash = "sha256:e62164b50f84e20601c1ff8eb55620d2ad25fb81b59e3cd776a1902527a788af"}, + {file = "charset_normalizer-3.0.1-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:8eade758719add78ec36dc13201483f8e9b5d940329285edcd5f70c0a9edbd7f"}, + {file = "charset_normalizer-3.0.1-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:8499ca8f4502af841f68135133d8258f7b32a53a1d594aa98cc52013fff55678"}, + {file = "charset_normalizer-3.0.1-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:3fc1c4a2ffd64890aebdb3f97e1278b0cc72579a08ca4de8cd2c04799a3a22be"}, + {file = "charset_normalizer-3.0.1-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:00d3ffdaafe92a5dc603cb9bd5111aaa36dfa187c8285c543be562e61b755f6b"}, + {file = "charset_normalizer-3.0.1-cp39-cp39-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:c2ac1b08635a8cd4e0cbeaf6f5e922085908d48eb05d44c5ae9eabab148512ca"}, + {file = "charset_normalizer-3.0.1-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:f6f45710b4459401609ebebdbcfb34515da4fc2aa886f95107f556ac69a9147e"}, + {file = "charset_normalizer-3.0.1-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:3ae1de54a77dc0d6d5fcf623290af4266412a7c4be0b1ff7444394f03f5c54e3"}, + {file = "charset_normalizer-3.0.1-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:3b590df687e3c5ee0deef9fc8c547d81986d9a1b56073d82de008744452d6541"}, + {file = "charset_normalizer-3.0.1-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:ab5de034a886f616a5668aa5d098af2b5385ed70142090e2a31bcbd0af0fdb3d"}, + {file = "charset_normalizer-3.0.1-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:9cb3032517f1627cc012dbc80a8ec976ae76d93ea2b5feaa9d2a5b8882597579"}, + {file = "charset_normalizer-3.0.1-cp39-cp39-musllinux_1_1_ppc64le.whl", hash = "sha256:608862a7bf6957f2333fc54ab4399e405baad0163dc9f8d99cb236816db169d4"}, + {file = "charset_normalizer-3.0.1-cp39-cp39-musllinux_1_1_s390x.whl", hash = "sha256:0f438ae3532723fb6ead77e7c604be7c8374094ef4ee2c5e03a3a17f1fca256c"}, + {file = "charset_normalizer-3.0.1-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:356541bf4381fa35856dafa6a965916e54bed415ad8a24ee6de6e37deccf2786"}, + {file = "charset_normalizer-3.0.1-cp39-cp39-win32.whl", hash = "sha256:39cf9ed17fe3b1bc81f33c9ceb6ce67683ee7526e65fde1447c772afc54a1bb8"}, + {file = "charset_normalizer-3.0.1-cp39-cp39-win_amd64.whl", hash = "sha256:0a11e971ed097d24c534c037d298ad32c6ce81a45736d31e0ff0ad37ab437d59"}, + {file = "charset_normalizer-3.0.1-py3-none-any.whl", hash = "sha256:7e189e2e1d3ed2f4aebabd2d5b0f931e883676e51c7624826e0a4e5fe8a0bf24"}, +] +click = [] +colorama = [ + {file = "colorama-0.4.6-py2.py3-none-any.whl", hash = "sha256:4f1d9991f5acc0ca119f9d443620b77f9d6b33703e51011c16baf57afb285fc6"}, + {file = "colorama-0.4.6.tar.gz", hash = "sha256:08695f5cb7ed6e0531a20572697297273c47b8cae5a63ffc6d6ed5c201be6e44"}, +] +coverage = [ + {file = "coverage-7.1.0-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:3b946bbcd5a8231383450b195cfb58cb01cbe7f8949f5758566b881df4b33baf"}, + {file = "coverage-7.1.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:ec8e767f13be637d056f7e07e61d089e555f719b387a7070154ad80a0ff31801"}, + {file = "coverage-7.1.0-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:d4a5a5879a939cb84959d86869132b00176197ca561c664fc21478c1eee60d75"}, + {file = "coverage-7.1.0-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:b643cb30821e7570c0aaf54feaf0bfb630b79059f85741843e9dc23f33aaca2c"}, + {file = "coverage-7.1.0-cp310-cp310-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:32df215215f3af2c1617a55dbdfb403b772d463d54d219985ac7cd3bf124cada"}, + {file = "coverage-7.1.0-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:33d1ae9d4079e05ac4cc1ef9e20c648f5afabf1a92adfaf2ccf509c50b85717f"}, + {file = "coverage-7.1.0-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:29571503c37f2ef2138a306d23e7270687c0efb9cab4bd8038d609b5c2393a3a"}, + {file = "coverage-7.1.0-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:63ffd21aa133ff48c4dff7adcc46b7ec8b565491bfc371212122dd999812ea1c"}, + {file = "coverage-7.1.0-cp310-cp310-win32.whl", hash = "sha256:4b14d5e09c656de5038a3f9bfe5228f53439282abcab87317c9f7f1acb280352"}, + {file = "coverage-7.1.0-cp310-cp310-win_amd64.whl", hash = "sha256:8361be1c2c073919500b6601220a6f2f98ea0b6d2fec5014c1d9cfa23dd07038"}, + {file = "coverage-7.1.0-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:da9b41d4539eefd408c46725fb76ecba3a50a3367cafb7dea5f250d0653c1040"}, + {file = "coverage-7.1.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:c5b15ed7644ae4bee0ecf74fee95808dcc34ba6ace87e8dfbf5cb0dc20eab45a"}, + {file = "coverage-7.1.0-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:d12d076582507ea460ea2a89a8c85cb558f83406c8a41dd641d7be9a32e1274f"}, + {file = "coverage-7.1.0-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:e2617759031dae1bf183c16cef8fcfb3de7617f394c813fa5e8e46e9b82d4222"}, + {file = "coverage-7.1.0-cp311-cp311-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:c4e4881fa9e9667afcc742f0c244d9364d197490fbc91d12ac3b5de0bf2df146"}, + {file = "coverage-7.1.0-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:9d58885215094ab4a86a6aef044e42994a2bd76a446dc59b352622655ba6621b"}, + {file = "coverage-7.1.0-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:ffeeb38ee4a80a30a6877c5c4c359e5498eec095878f1581453202bfacc8fbc2"}, + {file = "coverage-7.1.0-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:3baf5f126f30781b5e93dbefcc8271cb2491647f8283f20ac54d12161dff080e"}, + {file = "coverage-7.1.0-cp311-cp311-win32.whl", hash = "sha256:ded59300d6330be27bc6cf0b74b89ada58069ced87c48eaf9344e5e84b0072f7"}, + {file = "coverage-7.1.0-cp311-cp311-win_amd64.whl", hash = "sha256:6a43c7823cd7427b4ed763aa7fb63901ca8288591323b58c9cd6ec31ad910f3c"}, + {file = "coverage-7.1.0-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:7a726d742816cb3a8973c8c9a97539c734b3a309345236cd533c4883dda05b8d"}, + {file = "coverage-7.1.0-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:bc7c85a150501286f8b56bd8ed3aa4093f4b88fb68c0843d21ff9656f0009d6a"}, + {file = "coverage-7.1.0-cp37-cp37m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:f5b4198d85a3755d27e64c52f8c95d6333119e49fd001ae5798dac872c95e0f8"}, + {file = "coverage-7.1.0-cp37-cp37m-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:ddb726cb861c3117a553f940372a495fe1078249ff5f8a5478c0576c7be12050"}, + {file = "coverage-7.1.0-cp37-cp37m-musllinux_1_1_aarch64.whl", hash = "sha256:51b236e764840a6df0661b67e50697aaa0e7d4124ca95e5058fa3d7cbc240b7c"}, + {file = "coverage-7.1.0-cp37-cp37m-musllinux_1_1_i686.whl", hash = "sha256:7ee5c9bb51695f80878faaa5598040dd6c9e172ddcf490382e8aedb8ec3fec8d"}, + {file = "coverage-7.1.0-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:c31b75ae466c053a98bf26843563b3b3517b8f37da4d47b1c582fdc703112bc3"}, + {file = "coverage-7.1.0-cp37-cp37m-win32.whl", hash = "sha256:3b155caf3760408d1cb903b21e6a97ad4e2bdad43cbc265e3ce0afb8e0057e73"}, + {file = "coverage-7.1.0-cp37-cp37m-win_amd64.whl", hash = "sha256:2a60d6513781e87047c3e630b33b4d1e89f39836dac6e069ffee28c4786715f5"}, + {file = "coverage-7.1.0-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:f2cba5c6db29ce991029b5e4ac51eb36774458f0a3b8d3137241b32d1bb91f06"}, + {file = "coverage-7.1.0-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:beeb129cacea34490ffd4d6153af70509aa3cda20fdda2ea1a2be870dfec8d52"}, + {file = "coverage-7.1.0-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:0c45948f613d5d18c9ec5eaa203ce06a653334cf1bd47c783a12d0dd4fd9c851"}, + {file = "coverage-7.1.0-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:ef382417db92ba23dfb5864a3fc9be27ea4894e86620d342a116b243ade5d35d"}, + {file = "coverage-7.1.0-cp38-cp38-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:7c7c0d0827e853315c9bbd43c1162c006dd808dbbe297db7ae66cd17b07830f0"}, + {file = "coverage-7.1.0-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:e5cdbb5cafcedea04924568d990e20ce7f1945a1dd54b560f879ee2d57226912"}, + {file = "coverage-7.1.0-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:9817733f0d3ea91bea80de0f79ef971ae94f81ca52f9b66500c6a2fea8e4b4f8"}, + {file = "coverage-7.1.0-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:218fe982371ac7387304153ecd51205f14e9d731b34fb0568181abaf7b443ba0"}, + {file = "coverage-7.1.0-cp38-cp38-win32.whl", hash = "sha256:04481245ef966fbd24ae9b9e537ce899ae584d521dfbe78f89cad003c38ca2ab"}, + {file = "coverage-7.1.0-cp38-cp38-win_amd64.whl", hash = "sha256:8ae125d1134bf236acba8b83e74c603d1b30e207266121e76484562bc816344c"}, + {file = "coverage-7.1.0-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:2bf1d5f2084c3932b56b962a683074a3692bce7cabd3aa023c987a2a8e7612f6"}, + {file = "coverage-7.1.0-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:98b85dd86514d889a2e3dd22ab3c18c9d0019e696478391d86708b805f4ea0fa"}, + {file = "coverage-7.1.0-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:38da2db80cc505a611938d8624801158e409928b136c8916cd2e203970dde4dc"}, + {file = "coverage-7.1.0-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:3164d31078fa9efe406e198aecd2a02d32a62fecbdef74f76dad6a46c7e48311"}, + {file = "coverage-7.1.0-cp39-cp39-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:db61a79c07331e88b9a9974815c075fbd812bc9dbc4dc44b366b5368a2936063"}, + {file = "coverage-7.1.0-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:9ccb092c9ede70b2517a57382a601619d20981f56f440eae7e4d7eaafd1d1d09"}, + {file = "coverage-7.1.0-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:33ff26d0f6cc3ca8de13d14fde1ff8efe1456b53e3f0273e63cc8b3c84a063d8"}, + {file = "coverage-7.1.0-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:d47dd659a4ee952e90dc56c97d78132573dc5c7b09d61b416a9deef4ebe01a0c"}, + {file = "coverage-7.1.0-cp39-cp39-win32.whl", hash = "sha256:d248cd4a92065a4d4543b8331660121b31c4148dd00a691bfb7a5cdc7483cfa4"}, + {file = "coverage-7.1.0-cp39-cp39-win_amd64.whl", hash = "sha256:7ed681b0f8e8bcbbffa58ba26fcf5dbc8f79e7997595bf071ed5430d8c08d6f3"}, + {file = "coverage-7.1.0-pp37.pp38.pp39-none-any.whl", hash = "sha256:755e89e32376c850f826c425ece2c35a4fc266c081490eb0a841e7c1cb0d3bda"}, + {file = "coverage-7.1.0.tar.gz", hash = "sha256:10188fe543560ec4874f974b5305cd1a8bdcfa885ee00ea3a03733464c4ca265"}, +] +cryptography = [ + {file = "cryptography-39.0.1-cp36-abi3-macosx_10_12_universal2.whl", hash = "sha256:6687ef6d0a6497e2b58e7c5b852b53f62142cfa7cd1555795758934da363a965"}, + {file = "cryptography-39.0.1-cp36-abi3-macosx_10_12_x86_64.whl", hash = "sha256:706843b48f9a3f9b9911979761c91541e3d90db1ca905fd63fee540a217698bc"}, + {file = "cryptography-39.0.1-cp36-abi3-manylinux_2_17_aarch64.manylinux2014_aarch64.manylinux_2_24_aarch64.whl", hash = "sha256:5d2d8b87a490bfcd407ed9d49093793d0f75198a35e6eb1a923ce1ee86c62b41"}, + {file = "cryptography-39.0.1-cp36-abi3-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:83e17b26de248c33f3acffb922748151d71827d6021d98c70e6c1a25ddd78505"}, + {file = "cryptography-39.0.1-cp36-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:e124352fd3db36a9d4a21c1aa27fd5d051e621845cb87fb851c08f4f75ce8be6"}, + {file = "cryptography-39.0.1-cp36-abi3-manylinux_2_24_x86_64.whl", hash = "sha256:5aa67414fcdfa22cf052e640cb5ddc461924a045cacf325cd164e65312d99502"}, + {file = "cryptography-39.0.1-cp36-abi3-manylinux_2_28_aarch64.whl", hash = "sha256:35f7c7d015d474f4011e859e93e789c87d21f6f4880ebdc29896a60403328f1f"}, + {file = "cryptography-39.0.1-cp36-abi3-manylinux_2_28_x86_64.whl", hash = "sha256:f24077a3b5298a5a06a8e0536e3ea9ec60e4c7ac486755e5fb6e6ea9b3500106"}, + {file = "cryptography-39.0.1-cp36-abi3-musllinux_1_1_aarch64.whl", hash = "sha256:f0c64d1bd842ca2633e74a1a28033d139368ad959872533b1bab8c80e8240a0c"}, + {file = "cryptography-39.0.1-cp36-abi3-musllinux_1_1_x86_64.whl", hash = "sha256:0f8da300b5c8af9f98111ffd512910bc792b4c77392a9523624680f7956a99d4"}, + {file = "cryptography-39.0.1-cp36-abi3-win32.whl", hash = "sha256:fe913f20024eb2cb2f323e42a64bdf2911bb9738a15dba7d3cce48151034e3a8"}, + {file = "cryptography-39.0.1-cp36-abi3-win_amd64.whl", hash = "sha256:ced4e447ae29ca194449a3f1ce132ded8fcab06971ef5f618605aacaa612beac"}, + {file = "cryptography-39.0.1-pp38-pypy38_pp73-macosx_10_12_x86_64.whl", hash = "sha256:807ce09d4434881ca3a7594733669bd834f5b2c6d5c7e36f8c00f691887042ad"}, + {file = "cryptography-39.0.1-pp38-pypy38_pp73-win_amd64.whl", hash = "sha256:96f1157a7c08b5b189b16b47bc9db2332269d6680a196341bf30046330d15388"}, + {file = "cryptography-39.0.1-pp39-pypy39_pp73-macosx_10_12_x86_64.whl", hash = "sha256:e422abdec8b5fa8462aa016786680720d78bdce7a30c652b7fadf83a4ba35336"}, + {file = "cryptography-39.0.1-pp39-pypy39_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.manylinux_2_24_aarch64.whl", hash = "sha256:b0afd054cd42f3d213bf82c629efb1ee5f22eba35bf0eec88ea9ea7304f511a2"}, + {file = "cryptography-39.0.1-pp39-pypy39_pp73-manylinux_2_24_x86_64.whl", hash = "sha256:6f8ba7f0328b79f08bdacc3e4e66fb4d7aab0c3584e0bd41328dce5262e26b2e"}, + {file = "cryptography-39.0.1-pp39-pypy39_pp73-manylinux_2_28_aarch64.whl", hash = "sha256:ef8b72fa70b348724ff1218267e7f7375b8de4e8194d1636ee60510aae104cd0"}, + {file = "cryptography-39.0.1-pp39-pypy39_pp73-manylinux_2_28_x86_64.whl", hash = "sha256:aec5a6c9864be7df2240c382740fcf3b96928c46604eaa7f3091f58b878c0bb6"}, + {file = "cryptography-39.0.1-pp39-pypy39_pp73-win_amd64.whl", hash = "sha256:fdd188c8a6ef8769f148f88f859884507b954cc64db6b52f66ef199bb9ad660a"}, + {file = "cryptography-39.0.1.tar.gz", hash = "sha256:d1f6198ee6d9148405e49887803907fe8962a23e6c6f83ea7d98f1c0de375695"}, +] +distlib = [ + {file = "distlib-0.3.6-py2.py3-none-any.whl", hash = "sha256:f35c4b692542ca110de7ef0bea44d73981caeb34ca0b9b6b2e6d7790dda8f80e"}, + {file = "distlib-0.3.6.tar.gz", hash = "sha256:14bad2d9b04d3a36127ac97f30b12a19268f211063d8f8ee4f47108896e11b46"}, +] +docutils = [ + {file = "docutils-0.19-py3-none-any.whl", hash = "sha256:5e1de4d849fee02c63b040a4a3fd567f4ab104defd8a5511fbbc24a8a017efbc"}, + {file = "docutils-0.19.tar.gz", hash = "sha256:33995a6753c30b7f577febfc2c50411fec6aac7f7ffeb7c4cfe5991072dcf9e6"}, +] +duckdb = [ + {file = "duckdb-0.7.0-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:3fcae66586bd858c522559e85068d80414a27f23839705d5b400156c0fdbbeca"}, + {file = "duckdb-0.7.0-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:5d07d37edd7e2d1078bf500d11d068d6c1b62868bf372beca4d856cb3fa58c02"}, + {file = "duckdb-0.7.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:5b86e4bff0f5d49b6ac0aaae5fd6d4bd3d65f1aeeb77529d717515a0e4e16e60"}, + {file = "duckdb-0.7.0-cp310-cp310-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:4b9fce20c0e43089c6892f6af0112f6532ec9e163dbfd744ce6579a3ed5c63e6"}, + {file = "duckdb-0.7.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:5c730ccfef672b8fe04010862e5b766d1698c2a7401413958bd09a2640260966"}, + {file = "duckdb-0.7.0-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:36add2ce4747331bcc650d50d656604cc5442a1de9e88ab52481f51e53f10632"}, + {file = "duckdb-0.7.0-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:115e5e4ed087a129510c34dbac5bc8b30e05f95b05e34b3589771ccaab7215ec"}, + {file = "duckdb-0.7.0-cp310-cp310-win32.whl", hash = "sha256:5fa43a479c68937c7cad9ee13a6fd99417820a1f44aaaaf6d8675ea916278226"}, + {file = "duckdb-0.7.0-cp310-cp310-win_amd64.whl", hash = "sha256:864432628afbc8c4309c2c26b60cd97fdfc7891acdbc633a09e7961bdf5a1dcf"}, + {file = "duckdb-0.7.0-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:a09f52996921b8341a06a7425d8a4b6bc05fbdb8cf39ed1e915a0c9fdd3186ee"}, + {file = "duckdb-0.7.0-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:e3956bed611b1a9b543f3fed1d01ee41cfcb68cc150bebd2b3152b817541ec01"}, + {file = "duckdb-0.7.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:bc9ec4507b0c09aa163d27e3c4fcb8d3d904f0d02bb99f628534741cae45cf08"}, + {file = "duckdb-0.7.0-cp311-cp311-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:ac9e94c402d149a894febe76012cf86f6ed2b56232dd89a9ea06ac0843aacf69"}, + {file = "duckdb-0.7.0-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:6d735ba4d054cd05f7cceb4bc591151f0934460bfc6346d84063c58f5156c16a"}, + {file = "duckdb-0.7.0-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:49d33c0beaf075a8347bcab7589066f30d537bf1cba44bc87b2812343a080691"}, + {file = "duckdb-0.7.0-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:264fd585c5a6ccd0fb6ca3a957a0b82fc195afa0533392c77f7ac6b58a8cc617"}, + {file = "duckdb-0.7.0-cp311-cp311-win32.whl", hash = "sha256:5f93ec36c1b038cc09ac57c58b8506c13c722eca68e9f24af04ca0ee76b6fb1b"}, + {file = "duckdb-0.7.0-cp311-cp311-win_amd64.whl", hash = "sha256:cee0f977015dd7a35e7374ee341d56101c6561363adde213a87595e49719284c"}, + {file = "duckdb-0.7.0-cp36-cp36m-macosx_10_9_x86_64.whl", hash = "sha256:f0c9c54a6ce29ce45efc271ad4bd4d8f8b6ceb5e7e9ee66a236cda4c15ea1be6"}, + {file = "duckdb-0.7.0-cp36-cp36m-win32.whl", hash = "sha256:fe2ea12990ecc5eaf2de4f349d9c8c23b4855f8037c6e8fd1c659d22464d32ff"}, + {file = "duckdb-0.7.0-cp36-cp36m-win_amd64.whl", hash = "sha256:cc473a5ebd7efbe1c16fd72885031c167e925a2c7afbf6fb1ff66b9f4bfb93b9"}, + {file = "duckdb-0.7.0-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:462df10e74205ee45532315d239bcc37d5372c5169496ac5f227ab521d97a8a1"}, + {file = "duckdb-0.7.0-cp37-cp37m-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:22fb9d39b4cac968b2c80683be7be70138d49df0917cb6a3d141fb147afe4bb4"}, + {file = "duckdb-0.7.0-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:309ba76ac99f5a9a38579205406f13d117d9506e4a097b4bbb0a3484a103291c"}, + {file = "duckdb-0.7.0-cp37-cp37m-musllinux_1_1_i686.whl", hash = "sha256:78fb8e34d4bb73d21104c5c9cd170a0f2d79ab37bd1e7d03bd92ee65e50b80f1"}, + {file = "duckdb-0.7.0-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:5766c9d08d9cffd755153eca6c8f61f70c3b6a6d3c58ed2b2d4057b0a50f7850"}, + {file = "duckdb-0.7.0-cp37-cp37m-win32.whl", hash = "sha256:0d733bc19806e45782c79a0a6b9a6e88cd8502020533175b83db75d1fa564246"}, + {file = "duckdb-0.7.0-cp37-cp37m-win_amd64.whl", hash = "sha256:136bdc26d5f571d4ff3c09d867923e5f9ac03cc62db732d4b2ca2d4d9f84dd1d"}, + {file = "duckdb-0.7.0-cp38-cp38-macosx_10_9_universal2.whl", hash = "sha256:a22607efe25bc3b85e078fc5619095bf0d1cc8d39f0039a0e424075e2c77197b"}, + {file = "duckdb-0.7.0-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:1db7cacd682f465a3770ca97928edb9d245ef3ebe4e195c3cb6b812f099fae4f"}, + {file = "duckdb-0.7.0-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:ef71d92a738b1e4f64073e355dae90122793d5008c5e728e35b17bc023dbc7a7"}, + {file = "duckdb-0.7.0-cp38-cp38-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:65458c1fac678e1e03cf9e0aa00666b3990792adb318fe41bd49b15fe8a6e78a"}, + {file = "duckdb-0.7.0-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:2f21317cc8cc7f331936cc42294f52b8212a8e6a221b09672f0bfa7f290898a5"}, + {file = "duckdb-0.7.0-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:b9e0010145b0abe510bb9ea34782fa82df4ef4f2e6c9b87877df99ff20b8c993"}, + {file = "duckdb-0.7.0-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:b0e308354b54d235cd262c52036da56b3d49969c7cad47f784aff618cb3bed7a"}, + {file = "duckdb-0.7.0-cp38-cp38-win32.whl", hash = "sha256:8ae0e5ec024de474f12613524b5c2466955b5e42a68ea2ec788ac50040d44b88"}, + {file = "duckdb-0.7.0-cp38-cp38-win_amd64.whl", hash = "sha256:99ba303d81d8145edcbb5cae27e873e3c1c403532041cbe94b514af8e7db3009"}, + {file = "duckdb-0.7.0-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:1ce42f93dd35e6b256abe1c7344445913536da4604f6e8ee3c886fe7c1eb5580"}, + {file = "duckdb-0.7.0-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:b09ba609cf1744f87c1c7143db46c668e24bab893f422e22c7d059bc07a18b3d"}, + {file = "duckdb-0.7.0-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:fdb88cccd4a9446e67a375ac7e8058c35df62f4199b781e2e408ad95af10deb3"}, + {file = "duckdb-0.7.0-cp39-cp39-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:94d3cf46267a698f5e98dadf9cb0ce5c4f68e7a4c9702c2a37b57d56d2f944d7"}, + {file = "duckdb-0.7.0-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:5ff5ff3326faa087892d6529e700774878227c80ce8e16b018a62cb50d5bd4ac"}, + {file = "duckdb-0.7.0-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:ee3333009b828faf767582b23e0b9c1c4f888e8f1ae42de614f3ed0c40b1fe49"}, + {file = "duckdb-0.7.0-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:1521f430409ac6be7b959bb6f3ebfa9597f6a8c8837e13e060022fd9f6b45726"}, + {file = "duckdb-0.7.0-cp39-cp39-win32.whl", hash = "sha256:77e4607798d196c601795032b50cfb0d564ff42cbd10cf8ff47a5eb992bb442f"}, + {file = "duckdb-0.7.0-cp39-cp39-win_amd64.whl", hash = "sha256:726a3d555c6387e6653bc46d94645eac8139f825adfd0b7876ea858b5717796b"}, + {file = "duckdb-0.7.0.tar.gz", hash = "sha256:08ebfcc67a6a073fa2efd0453de9a7e453637bba522e67968fe0c03ccbfc2ec5"}, +] +exceptiongroup = [ + {file = "exceptiongroup-1.1.0-py3-none-any.whl", hash = "sha256:327cbda3da756e2de031a3107b81ab7b3770a602c4d16ca618298c526f4bec1e"}, + {file = "exceptiongroup-1.1.0.tar.gz", hash = "sha256:bcb67d800a4497e1b404c2dd44fca47d3b7a5e5433dbab67f96c1a685cdfdf23"}, +] +fastavro = [ + {file = "fastavro-1.7.1-cp310-cp310-macosx_11_0_x86_64.whl", hash = "sha256:f8f7f11af8c2c074341217d6247b7ba09cadcd55f899e046c14e3a44afa5fc95"}, + {file = "fastavro-1.7.1-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:11159cf23f5ff4b752b028a77bd2a7941599932527e8a6512779e25b0503f037"}, + {file = "fastavro-1.7.1-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:86ce89743711355eee7d3fec34847de1ab574f4567aa4a667e966711bb2e0cd9"}, + {file = "fastavro-1.7.1-cp310-cp310-win_amd64.whl", hash = "sha256:2e3f5ae42e033dbb6e0efa788c4b8a4e5a59bc2b9cb83f717b6d85176727faed"}, + {file = "fastavro-1.7.1-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:8b671a97e864e4c024061c0d6f93f2768ba0595f917317ca4fe3e99dca6fcf3d"}, + {file = "fastavro-1.7.1-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:e9be910ca1645e0e5c4192d667cfdfa58dff4f8690db5af1ae33602643d41a78"}, + {file = "fastavro-1.7.1-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:6f233a1d0d95265f81a25c274f55017bd39d9b8f7f1387a4235bf8e01841a9ff"}, + {file = "fastavro-1.7.1-cp311-cp311-win_amd64.whl", hash = "sha256:31c924ed2fdad56182b543941cdec9cc384443cc3ca9462e0580afeb4dc64f4b"}, + {file = "fastavro-1.7.1-cp37-cp37m-macosx_10_15_x86_64.whl", hash = "sha256:8ad522c7f2c7791cfe54a7e87af8ac09634872f4fdeef28deec97fab8de38b24"}, + {file = "fastavro-1.7.1-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:74fe0fb24a489e2fd567775935eb55a461fc6c4da8b5e3467245752ac2098284"}, + {file = "fastavro-1.7.1-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:4f7a744569b4b44ea1f4f87e7ea8298e1e2bf23779aa6ef255b95f9f38faad48"}, + {file = "fastavro-1.7.1-cp37-cp37m-win_amd64.whl", hash = "sha256:b793252a6325341890bbbbdca2804d4db3d5d29ff7f15a58fcf84dda440808fa"}, + {file = "fastavro-1.7.1-cp38-cp38-macosx_10_15_x86_64.whl", hash = "sha256:a107a8b333628c0f8a2ece5d5a57c69923719a139b106ace4050206250df4b13"}, + {file = "fastavro-1.7.1-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:182671595683054ec234beec03f485b5c889c21c08e429577ae7929480703409"}, + {file = "fastavro-1.7.1-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:980ce7fdccf328d287e8b789f4e3b422f64c84ed9cd81c05dd7c560c3d8076b1"}, + {file = "fastavro-1.7.1-cp38-cp38-win_amd64.whl", hash = "sha256:a5f104bc5b4986bbbcab170918c4d8ffa4f8efa3ebe8ec190954178630074d5a"}, + {file = "fastavro-1.7.1-cp39-cp39-macosx_11_0_x86_64.whl", hash = "sha256:5ff701d6b228218a3d9c09b392205dd0899afa501a4f14724bef0ce13a719700"}, + {file = "fastavro-1.7.1-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:5cb2d8ea7b21493bb18ff7c68401edbc663bbd8a57016d6cf2c4b0a2dc4464e7"}, + {file = "fastavro-1.7.1-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:e15e8163581983f07a3156902761cf746bbe1e646abd9553cc9a1cede6e23ae9"}, + {file = "fastavro-1.7.1-cp39-cp39-win_amd64.whl", hash = "sha256:4d56cc05ceb1c9f4de19c1f330a4f140af3b944834d63cd0e11517deaea21ee1"}, + {file = "fastavro-1.7.1.tar.gz", hash = "sha256:4b8bcae4ed6343af186e638061cdfbc5331cdb5e026d055099c91d4f07be838c"}, +] +filelock = [ + {file = "filelock-3.9.0-py3-none-any.whl", hash = "sha256:f58d535af89bb9ad5cd4df046f741f8553a418c01a7856bf0d173bbc9f6bd16d"}, + {file = "filelock-3.9.0.tar.gz", hash = "sha256:7b319f24340b51f55a2bf7a12ac0755a9b03e718311dac567a0f4f7fabd2f5de"}, +] +frozenlist = [ + {file = "frozenlist-1.3.3-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:ff8bf625fe85e119553b5383ba0fb6aa3d0ec2ae980295aaefa552374926b3f4"}, + {file = "frozenlist-1.3.3-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:dfbac4c2dfcc082fcf8d942d1e49b6aa0766c19d3358bd86e2000bf0fa4a9cf0"}, + {file = "frozenlist-1.3.3-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:b1c63e8d377d039ac769cd0926558bb7068a1f7abb0f003e3717ee003ad85530"}, + {file = "frozenlist-1.3.3-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:7fdfc24dcfce5b48109867c13b4cb15e4660e7bd7661741a391f821f23dfdca7"}, + {file = "frozenlist-1.3.3-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:2c926450857408e42f0bbc295e84395722ce74bae69a3b2aa2a65fe22cb14b99"}, + {file = "frozenlist-1.3.3-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:1841e200fdafc3d51f974d9d377c079a0694a8f06de2e67b48150328d66d5483"}, + {file = "frozenlist-1.3.3-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:f470c92737afa7d4c3aacc001e335062d582053d4dbe73cda126f2d7031068dd"}, + {file = "frozenlist-1.3.3-cp310-cp310-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:783263a4eaad7c49983fe4b2e7b53fa9770c136c270d2d4bbb6d2192bf4d9caf"}, + {file = "frozenlist-1.3.3-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:924620eef691990dfb56dc4709f280f40baee568c794b5c1885800c3ecc69816"}, + {file = "frozenlist-1.3.3-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:ae4dc05c465a08a866b7a1baf360747078b362e6a6dbeb0c57f234db0ef88ae0"}, + {file = "frozenlist-1.3.3-cp310-cp310-musllinux_1_1_ppc64le.whl", hash = "sha256:bed331fe18f58d844d39ceb398b77d6ac0b010d571cba8267c2e7165806b00ce"}, + {file = "frozenlist-1.3.3-cp310-cp310-musllinux_1_1_s390x.whl", hash = "sha256:02c9ac843e3390826a265e331105efeab489ffaf4dd86384595ee8ce6d35ae7f"}, + {file = "frozenlist-1.3.3-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:9545a33965d0d377b0bc823dcabf26980e77f1b6a7caa368a365a9497fb09420"}, + {file = "frozenlist-1.3.3-cp310-cp310-win32.whl", hash = "sha256:d5cd3ab21acbdb414bb6c31958d7b06b85eeb40f66463c264a9b343a4e238642"}, + {file = "frozenlist-1.3.3-cp310-cp310-win_amd64.whl", hash = "sha256:b756072364347cb6aa5b60f9bc18e94b2f79632de3b0190253ad770c5df17db1"}, + {file = "frozenlist-1.3.3-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:b4395e2f8d83fbe0c627b2b696acce67868793d7d9750e90e39592b3626691b7"}, + {file = "frozenlist-1.3.3-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:14143ae966a6229350021384870458e4777d1eae4c28d1a7aa47f24d030e6678"}, + {file = "frozenlist-1.3.3-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:5d8860749e813a6f65bad8285a0520607c9500caa23fea6ee407e63debcdbef6"}, + {file = "frozenlist-1.3.3-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:23d16d9f477bb55b6154654e0e74557040575d9d19fe78a161bd33d7d76808e8"}, + {file = "frozenlist-1.3.3-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:eb82dbba47a8318e75f679690190c10a5e1f447fbf9df41cbc4c3afd726d88cb"}, + {file = "frozenlist-1.3.3-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:9309869032abb23d196cb4e4db574232abe8b8be1339026f489eeb34a4acfd91"}, + {file = "frozenlist-1.3.3-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:a97b4fe50b5890d36300820abd305694cb865ddb7885049587a5678215782a6b"}, + {file = "frozenlist-1.3.3-cp311-cp311-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:c188512b43542b1e91cadc3c6c915a82a5eb95929134faf7fd109f14f9892ce4"}, + {file = "frozenlist-1.3.3-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:303e04d422e9b911a09ad499b0368dc551e8c3cd15293c99160c7f1f07b59a48"}, + {file = "frozenlist-1.3.3-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:0771aed7f596c7d73444c847a1c16288937ef988dc04fb9f7be4b2aa91db609d"}, + {file = "frozenlist-1.3.3-cp311-cp311-musllinux_1_1_ppc64le.whl", hash = "sha256:66080ec69883597e4d026f2f71a231a1ee9887835902dbe6b6467d5a89216cf6"}, + {file = "frozenlist-1.3.3-cp311-cp311-musllinux_1_1_s390x.whl", hash = "sha256:41fe21dc74ad3a779c3d73a2786bdf622ea81234bdd4faf90b8b03cad0c2c0b4"}, + {file = "frozenlist-1.3.3-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:f20380df709d91525e4bee04746ba612a4df0972c1b8f8e1e8af997e678c7b81"}, + {file = "frozenlist-1.3.3-cp311-cp311-win32.whl", hash = "sha256:f30f1928162e189091cf4d9da2eac617bfe78ef907a761614ff577ef4edfb3c8"}, + {file = "frozenlist-1.3.3-cp311-cp311-win_amd64.whl", hash = "sha256:a6394d7dadd3cfe3f4b3b186e54d5d8504d44f2d58dcc89d693698e8b7132b32"}, + {file = "frozenlist-1.3.3-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:8df3de3a9ab8325f94f646609a66cbeeede263910c5c0de0101079ad541af332"}, + {file = "frozenlist-1.3.3-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:0693c609e9742c66ba4870bcee1ad5ff35462d5ffec18710b4ac89337ff16e27"}, + {file = "frozenlist-1.3.3-cp37-cp37m-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:cd4210baef299717db0a600d7a3cac81d46ef0e007f88c9335db79f8979c0d3d"}, + {file = "frozenlist-1.3.3-cp37-cp37m-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:394c9c242113bfb4b9aa36e2b80a05ffa163a30691c7b5a29eba82e937895d5e"}, + {file = "frozenlist-1.3.3-cp37-cp37m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:6327eb8e419f7d9c38f333cde41b9ae348bec26d840927332f17e887a8dcb70d"}, + {file = "frozenlist-1.3.3-cp37-cp37m-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:2e24900aa13212e75e5b366cb9065e78bbf3893d4baab6052d1aca10d46d944c"}, + {file = "frozenlist-1.3.3-cp37-cp37m-musllinux_1_1_aarch64.whl", hash = "sha256:3843f84a6c465a36559161e6c59dce2f2ac10943040c2fd021cfb70d58c4ad56"}, + {file = "frozenlist-1.3.3-cp37-cp37m-musllinux_1_1_i686.whl", hash = "sha256:84610c1502b2461255b4c9b7d5e9c48052601a8957cd0aea6ec7a7a1e1fb9420"}, + {file = "frozenlist-1.3.3-cp37-cp37m-musllinux_1_1_ppc64le.whl", hash = "sha256:c21b9aa40e08e4f63a2f92ff3748e6b6c84d717d033c7b3438dd3123ee18f70e"}, + {file = "frozenlist-1.3.3-cp37-cp37m-musllinux_1_1_s390x.whl", hash = "sha256:efce6ae830831ab6a22b9b4091d411698145cb9b8fc869e1397ccf4b4b6455cb"}, + {file = "frozenlist-1.3.3-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:40de71985e9042ca00b7953c4f41eabc3dc514a2d1ff534027f091bc74416401"}, + {file = "frozenlist-1.3.3-cp37-cp37m-win32.whl", hash = "sha256:180c00c66bde6146a860cbb81b54ee0df350d2daf13ca85b275123bbf85de18a"}, + {file = "frozenlist-1.3.3-cp37-cp37m-win_amd64.whl", hash = "sha256:9bbbcedd75acdfecf2159663b87f1bb5cfc80e7cd99f7ddd9d66eb98b14a8411"}, + {file = "frozenlist-1.3.3-cp38-cp38-macosx_10_9_universal2.whl", hash = "sha256:034a5c08d36649591be1cbb10e09da9f531034acfe29275fc5454a3b101ce41a"}, + {file = "frozenlist-1.3.3-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:ba64dc2b3b7b158c6660d49cdb1d872d1d0bf4e42043ad8d5006099479a194e5"}, + {file = "frozenlist-1.3.3-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:47df36a9fe24054b950bbc2db630d508cca3aa27ed0566c0baf661225e52c18e"}, + {file = "frozenlist-1.3.3-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:008a054b75d77c995ea26629ab3a0c0d7281341f2fa7e1e85fa6153ae29ae99c"}, + {file = "frozenlist-1.3.3-cp38-cp38-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:841ea19b43d438a80b4de62ac6ab21cfe6827bb8a9dc62b896acc88eaf9cecba"}, + {file = "frozenlist-1.3.3-cp38-cp38-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:e235688f42b36be2b6b06fc37ac2126a73b75fb8d6bc66dd632aa35286238703"}, + {file = "frozenlist-1.3.3-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:ca713d4af15bae6e5d79b15c10c8522859a9a89d3b361a50b817c98c2fb402a2"}, + {file = "frozenlist-1.3.3-cp38-cp38-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:9ac5995f2b408017b0be26d4a1d7c61bce106ff3d9e3324374d66b5964325448"}, + {file = "frozenlist-1.3.3-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:a4ae8135b11652b08a8baf07631d3ebfe65a4c87909dbef5fa0cdde440444ee4"}, + {file = "frozenlist-1.3.3-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:4ea42116ceb6bb16dbb7d526e242cb6747b08b7710d9782aa3d6732bd8d27649"}, + {file = "frozenlist-1.3.3-cp38-cp38-musllinux_1_1_ppc64le.whl", hash = "sha256:810860bb4bdce7557bc0febb84bbd88198b9dbc2022d8eebe5b3590b2ad6c842"}, + {file = "frozenlist-1.3.3-cp38-cp38-musllinux_1_1_s390x.whl", hash = "sha256:ee78feb9d293c323b59a6f2dd441b63339a30edf35abcb51187d2fc26e696d13"}, + {file = "frozenlist-1.3.3-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:0af2e7c87d35b38732e810befb9d797a99279cbb85374d42ea61c1e9d23094b3"}, + {file = "frozenlist-1.3.3-cp38-cp38-win32.whl", hash = "sha256:899c5e1928eec13fd6f6d8dc51be23f0d09c5281e40d9cf4273d188d9feeaf9b"}, + {file = "frozenlist-1.3.3-cp38-cp38-win_amd64.whl", hash = "sha256:7f44e24fa70f6fbc74aeec3e971f60a14dde85da364aa87f15d1be94ae75aeef"}, + {file = "frozenlist-1.3.3-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:2b07ae0c1edaa0a36339ec6cce700f51b14a3fc6545fdd32930d2c83917332cf"}, + {file = "frozenlist-1.3.3-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:ebb86518203e12e96af765ee89034a1dbb0c3c65052d1b0c19bbbd6af8a145e1"}, + {file = "frozenlist-1.3.3-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:5cf820485f1b4c91e0417ea0afd41ce5cf5965011b3c22c400f6d144296ccbc0"}, + {file = "frozenlist-1.3.3-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:5c11e43016b9024240212d2a65043b70ed8dfd3b52678a1271972702d990ac6d"}, + {file = "frozenlist-1.3.3-cp39-cp39-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:8fa3c6e3305aa1146b59a09b32b2e04074945ffcfb2f0931836d103a2c38f936"}, + {file = "frozenlist-1.3.3-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:352bd4c8c72d508778cf05ab491f6ef36149f4d0cb3c56b1b4302852255d05d5"}, + {file = "frozenlist-1.3.3-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:65a5e4d3aa679610ac6e3569e865425b23b372277f89b5ef06cf2cdaf1ebf22b"}, + {file = "frozenlist-1.3.3-cp39-cp39-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:b1e2c1185858d7e10ff045c496bbf90ae752c28b365fef2c09cf0fa309291669"}, + {file = "frozenlist-1.3.3-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:f163d2fd041c630fed01bc48d28c3ed4a3b003c00acd396900e11ee5316b56bb"}, + {file = "frozenlist-1.3.3-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:05cdb16d09a0832eedf770cb7bd1fe57d8cf4eaf5aced29c4e41e3f20b30a784"}, + {file = "frozenlist-1.3.3-cp39-cp39-musllinux_1_1_ppc64le.whl", hash = "sha256:8bae29d60768bfa8fb92244b74502b18fae55a80eac13c88eb0b496d4268fd2d"}, + {file = "frozenlist-1.3.3-cp39-cp39-musllinux_1_1_s390x.whl", hash = "sha256:eedab4c310c0299961ac285591acd53dc6723a1ebd90a57207c71f6e0c2153ab"}, + {file = "frozenlist-1.3.3-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:3bbdf44855ed8f0fbcd102ef05ec3012d6a4fd7c7562403f76ce6a52aeffb2b1"}, + {file = "frozenlist-1.3.3-cp39-cp39-win32.whl", hash = "sha256:efa568b885bca461f7c7b9e032655c0c143d305bf01c30caf6db2854a4532b38"}, + {file = "frozenlist-1.3.3-cp39-cp39-win_amd64.whl", hash = "sha256:cfe33efc9cb900a4c46f91a5ceba26d6df370ffddd9ca386eb1d4f0ad97b9ea9"}, + {file = "frozenlist-1.3.3.tar.gz", hash = "sha256:58bcc55721e8a90b88332d6cd441261ebb22342e238296bb330968952fbb3a6a"}, +] +fsspec = [ + {file = "fsspec-2023.1.0-py3-none-any.whl", hash = "sha256:b833e2e541e9e8cde0ab549414187871243177feb3d344f9d27b25a93f5d8139"}, + {file = "fsspec-2023.1.0.tar.gz", hash = "sha256:fbae7f20ff801eb5f7d0bedf81f25c787c0dfac5e982d98fa3884a9cde2b5411"}, +] +identify = [ + {file = "identify-2.5.18-py2.py3-none-any.whl", hash = "sha256:93aac7ecf2f6abf879b8f29a8002d3c6de7086b8c28d88e1ad15045a15ab63f9"}, + {file = "identify-2.5.18.tar.gz", hash = "sha256:89e144fa560cc4cffb6ef2ab5e9fb18ed9f9b3cb054384bab4b95c12f6c309fe"}, +] +idna = [ + {file = "idna-3.4-py3-none-any.whl", hash = "sha256:90b77e79eaa3eba6de819a0c442c0b4ceefc341a7a2ab77d7562bf49f425c5c2"}, + {file = "idna-3.4.tar.gz", hash = "sha256:814f528e8dead7d329833b91c5faa87d60bf71824cd12a7530b5526063d02cb4"}, +] +importlib-metadata = [ + {file = "importlib_metadata-6.0.0-py3-none-any.whl", hash = "sha256:7efb448ec9a5e313a57655d35aa54cd3e01b7e1fbcf72dce1bf06119420f5bad"}, + {file = "importlib_metadata-6.0.0.tar.gz", hash = "sha256:e354bedeb60efa6affdcc8ae121b73544a7aa74156d047311948f6d711cd378d"}, +] +iniconfig = [ + {file = "iniconfig-2.0.0-py3-none-any.whl", hash = "sha256:b6a85871a79d2e3b22d2d1b94ac2824226a63c6b741c88f7ae975f18b6778374"}, + {file = "iniconfig-2.0.0.tar.gz", hash = "sha256:2d91e135bf72d31a410b17c16da610a82cb55f6b0477d1a902134b24a455b8b3"}, +] +isodate = [ + {file = "isodate-0.6.1-py2.py3-none-any.whl", hash = "sha256:0751eece944162659049d35f4f549ed815792b38793f07cf73381c1c87cbed96"}, + {file = "isodate-0.6.1.tar.gz", hash = "sha256:48c5881de7e8b0a0d648cb024c8062dc84e7b840ed81e864c7614fd3c127bde9"}, +] +jinja2 = [ + {file = "Jinja2-3.1.2-py3-none-any.whl", hash = "sha256:6088930bfe239f0e6710546ab9c19c9ef35e29792895fed6e6e31a023a182a61"}, + {file = "Jinja2-3.1.2.tar.gz", hash = "sha256:31351a702a408a9e7595a8fc6150fc3f43bb6bf7e319770cbc0db9df9437e852"}, +] +jmespath = [] +markdown-it-py = [ + {file = "markdown-it-py-2.1.0.tar.gz", hash = "sha256:cf7e59fed14b5ae17c0006eff14a2d9a00ed5f3a846148153899a0224e2c07da"}, + {file = "markdown_it_py-2.1.0-py3-none-any.whl", hash = "sha256:93de681e5c021a432c63147656fe21790bc01231e0cd2da73626f1aa3ac0fe27"}, +] +markupsafe = [ + {file = "MarkupSafe-2.1.2-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:665a36ae6f8f20a4676b53224e33d456a6f5a72657d9c83c2aa00765072f31f7"}, + {file = "MarkupSafe-2.1.2-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:340bea174e9761308703ae988e982005aedf427de816d1afe98147668cc03036"}, + {file = "MarkupSafe-2.1.2-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:22152d00bf4a9c7c83960521fc558f55a1adbc0631fbb00a9471e097b19d72e1"}, + {file = "MarkupSafe-2.1.2-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:28057e985dace2f478e042eaa15606c7efccb700797660629da387eb289b9323"}, + {file = "MarkupSafe-2.1.2-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:ca244fa73f50a800cf8c3ebf7fd93149ec37f5cb9596aa8873ae2c1d23498601"}, + {file = "MarkupSafe-2.1.2-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:d9d971ec1e79906046aa3ca266de79eac42f1dbf3612a05dc9368125952bd1a1"}, + {file = "MarkupSafe-2.1.2-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:7e007132af78ea9df29495dbf7b5824cb71648d7133cf7848a2a5dd00d36f9ff"}, + {file = "MarkupSafe-2.1.2-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:7313ce6a199651c4ed9d7e4cfb4aa56fe923b1adf9af3b420ee14e6d9a73df65"}, + {file = "MarkupSafe-2.1.2-cp310-cp310-win32.whl", hash = "sha256:c4a549890a45f57f1ebf99c067a4ad0cb423a05544accaf2b065246827ed9603"}, + {file = "MarkupSafe-2.1.2-cp310-cp310-win_amd64.whl", hash = "sha256:835fb5e38fd89328e9c81067fd642b3593c33e1e17e2fdbf77f5676abb14a156"}, + {file = "MarkupSafe-2.1.2-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:2ec4f2d48ae59bbb9d1f9d7efb9236ab81429a764dedca114f5fdabbc3788013"}, + {file = "MarkupSafe-2.1.2-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:608e7073dfa9e38a85d38474c082d4281f4ce276ac0010224eaba11e929dd53a"}, + {file = "MarkupSafe-2.1.2-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:65608c35bfb8a76763f37036547f7adfd09270fbdbf96608be2bead319728fcd"}, + {file = "MarkupSafe-2.1.2-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:f2bfb563d0211ce16b63c7cb9395d2c682a23187f54c3d79bfec33e6705473c6"}, + {file = "MarkupSafe-2.1.2-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:da25303d91526aac3672ee6d49a2f3db2d9502a4a60b55519feb1a4c7714e07d"}, + {file = "MarkupSafe-2.1.2-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:9cad97ab29dfc3f0249b483412c85c8ef4766d96cdf9dcf5a1e3caa3f3661cf1"}, + {file = "MarkupSafe-2.1.2-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:085fd3201e7b12809f9e6e9bc1e5c96a368c8523fad5afb02afe3c051ae4afcc"}, + {file = "MarkupSafe-2.1.2-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:1bea30e9bf331f3fef67e0a3877b2288593c98a21ccb2cf29b74c581a4eb3af0"}, + {file = "MarkupSafe-2.1.2-cp311-cp311-win32.whl", hash = "sha256:7df70907e00c970c60b9ef2938d894a9381f38e6b9db73c5be35e59d92e06625"}, + {file = "MarkupSafe-2.1.2-cp311-cp311-win_amd64.whl", hash = "sha256:e55e40ff0cc8cc5c07996915ad367fa47da6b3fc091fdadca7f5403239c5fec3"}, + {file = "MarkupSafe-2.1.2-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:a6e40afa7f45939ca356f348c8e23048e02cb109ced1eb8420961b2f40fb373a"}, + {file = "MarkupSafe-2.1.2-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:cf877ab4ed6e302ec1d04952ca358b381a882fbd9d1b07cccbfd61783561f98a"}, + {file = "MarkupSafe-2.1.2-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:63ba06c9941e46fa389d389644e2d8225e0e3e5ebcc4ff1ea8506dce646f8c8a"}, + {file = "MarkupSafe-2.1.2-cp37-cp37m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:f1cd098434e83e656abf198f103a8207a8187c0fc110306691a2e94a78d0abb2"}, + {file = "MarkupSafe-2.1.2-cp37-cp37m-musllinux_1_1_aarch64.whl", hash = "sha256:55f44b440d491028addb3b88f72207d71eeebfb7b5dbf0643f7c023ae1fba619"}, + {file = "MarkupSafe-2.1.2-cp37-cp37m-musllinux_1_1_i686.whl", hash = "sha256:a6f2fcca746e8d5910e18782f976489939d54a91f9411c32051b4aab2bd7c513"}, + {file = "MarkupSafe-2.1.2-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:0b462104ba25f1ac006fdab8b6a01ebbfbce9ed37fd37fd4acd70c67c973e460"}, + {file = "MarkupSafe-2.1.2-cp37-cp37m-win32.whl", hash = "sha256:7668b52e102d0ed87cb082380a7e2e1e78737ddecdde129acadb0eccc5423859"}, + {file = "MarkupSafe-2.1.2-cp37-cp37m-win_amd64.whl", hash = "sha256:6d6607f98fcf17e534162f0709aaad3ab7a96032723d8ac8750ffe17ae5a0666"}, + {file = "MarkupSafe-2.1.2-cp38-cp38-macosx_10_9_universal2.whl", hash = "sha256:a806db027852538d2ad7555b203300173dd1b77ba116de92da9afbc3a3be3eed"}, + {file = "MarkupSafe-2.1.2-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:a4abaec6ca3ad8660690236d11bfe28dfd707778e2442b45addd2f086d6ef094"}, + {file = "MarkupSafe-2.1.2-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:f03a532d7dee1bed20bc4884194a16160a2de9ffc6354b3878ec9682bb623c54"}, + {file = "MarkupSafe-2.1.2-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:4cf06cdc1dda95223e9d2d3c58d3b178aa5dacb35ee7e3bbac10e4e1faacb419"}, + {file = "MarkupSafe-2.1.2-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:22731d79ed2eb25059ae3df1dfc9cb1546691cc41f4e3130fe6bfbc3ecbbecfa"}, + {file = "MarkupSafe-2.1.2-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:f8ffb705ffcf5ddd0e80b65ddf7bed7ee4f5a441ea7d3419e861a12eaf41af58"}, + {file = "MarkupSafe-2.1.2-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:8db032bf0ce9022a8e41a22598eefc802314e81b879ae093f36ce9ddf39ab1ba"}, + {file = "MarkupSafe-2.1.2-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:2298c859cfc5463f1b64bd55cb3e602528db6fa0f3cfd568d3605c50678f8f03"}, + {file = "MarkupSafe-2.1.2-cp38-cp38-win32.whl", hash = "sha256:50c42830a633fa0cf9e7d27664637532791bfc31c731a87b202d2d8ac40c3ea2"}, + {file = "MarkupSafe-2.1.2-cp38-cp38-win_amd64.whl", hash = "sha256:bb06feb762bade6bf3c8b844462274db0c76acc95c52abe8dbed28ae3d44a147"}, + {file = "MarkupSafe-2.1.2-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:99625a92da8229df6d44335e6fcc558a5037dd0a760e11d84be2260e6f37002f"}, + {file = "MarkupSafe-2.1.2-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:8bca7e26c1dd751236cfb0c6c72d4ad61d986e9a41bbf76cb445f69488b2a2bd"}, + {file = "MarkupSafe-2.1.2-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:40627dcf047dadb22cd25ea7ecfe9cbf3bbbad0482ee5920b582f3809c97654f"}, + {file = "MarkupSafe-2.1.2-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:40dfd3fefbef579ee058f139733ac336312663c6706d1163b82b3003fb1925c4"}, + {file = "MarkupSafe-2.1.2-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:090376d812fb6ac5f171e5938e82e7f2d7adc2b629101cec0db8b267815c85e2"}, + {file = "MarkupSafe-2.1.2-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:2e7821bffe00aa6bd07a23913b7f4e01328c3d5cc0b40b36c0bd81d362faeb65"}, + {file = "MarkupSafe-2.1.2-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:c0a33bc9f02c2b17c3ea382f91b4db0e6cde90b63b296422a939886a7a80de1c"}, + {file = "MarkupSafe-2.1.2-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:b8526c6d437855442cdd3d87eede9c425c4445ea011ca38d937db299382e6fa3"}, + {file = "MarkupSafe-2.1.2-cp39-cp39-win32.whl", hash = "sha256:137678c63c977754abe9086a3ec011e8fd985ab90631145dfb9294ad09c102a7"}, + {file = "MarkupSafe-2.1.2-cp39-cp39-win_amd64.whl", hash = "sha256:0576fe974b40a400449768941d5d0858cc624e3249dfd1e0c33674e5c7ca7aed"}, + {file = "MarkupSafe-2.1.2.tar.gz", hash = "sha256:abcabc8c2b26036d62d4c746381a6f7cf60aafcc653198ad678306986b09450d"}, +] +mdurl = [ + {file = "mdurl-0.1.2-py3-none-any.whl", hash = "sha256:84008a41e51615a49fc9966191ff91509e3c40b939176e643fd50a5c2196b8f8"}, + {file = "mdurl-0.1.2.tar.gz", hash = "sha256:bb413d29f5eea38f31dd4754dd7377d4465116fb207585f97bf925588687c1ba"}, +] +mmhash3 = [ + {file = "mmhash3-3.0.1-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:47deea30cd8d3d5cd52dc740902a4c70383bfe8248eac29d0877fe63e03c2713"}, + {file = "mmhash3-3.0.1-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:ecdaf4d1de617818bf05cd526ca558db6010beeba7ea9e19f695f2bdcac0e0a4"}, + {file = "mmhash3-3.0.1-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:4675585617584e9e1aafa3a90ac0ac9a437257c507748d97de8b21977e9d6745"}, + {file = "mmhash3-3.0.1-cp310-cp310-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:ebfd0c2af09b41f0fe1dd260799bda90a0fc7eba4477ccaeb3951527700cd58f"}, + {file = "mmhash3-3.0.1-cp310-cp310-win32.whl", hash = "sha256:68587dec7b8acdb7528fd511e295d8b5ccfe26022923a69867e1822f0fdb4c44"}, + {file = "mmhash3-3.0.1-cp310-cp310-win_amd64.whl", hash = "sha256:54954ebe3a614f19ab4cfe24fa66ea093c493d9fac0533d002dd64c2540a0c99"}, + {file = "mmhash3-3.0.1-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:b172f3bd3220b0bf56fd7cc760fd8a9033def47103611d63fe867003079a1256"}, + {file = "mmhash3-3.0.1-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:de7895eafabc32f7c0c09a81a624921f536768de6e438e3de69e3e954a4d7072"}, + {file = "mmhash3-3.0.1-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:b4b0914effe4ddd8d33149e3508564c17719465b0cc81691c4fa50d5e0e14f80"}, + {file = "mmhash3-3.0.1-cp311-cp311-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:c0575050ac691475938df1ad03d8738c5bd1eadef62093e76157ebb7f2df0946"}, + {file = "mmhash3-3.0.1-cp311-cp311-win32.whl", hash = "sha256:22f92f0f88f28b215357acd346362fa9f7c9fffb436beb42cc4b442b676dbaa3"}, + {file = "mmhash3-3.0.1-cp311-cp311-win_amd64.whl", hash = "sha256:538240ab7936bf71b18304e5a7e7fd3c4c2fab103330ea99584bb4f777299a2b"}, + {file = "mmhash3-3.0.1-cp36-cp36m-macosx_10_9_x86_64.whl", hash = "sha256:ca791bfb311e36ce13998e4632262ed4b95da9d3461941e18b6690760171a045"}, + {file = "mmhash3-3.0.1-cp36-cp36m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:b41708f72c6aa2a49ada1f0b61e85c05cd8389ad31d463fd5bca29999a4d5f9c"}, + {file = "mmhash3-3.0.1-cp36-cp36m-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:a3ce9b4533ddc0a88ba045a27309714c3b127bd05e57fd252d1d5a71d4247ea7"}, + {file = "mmhash3-3.0.1-cp36-cp36m-win32.whl", hash = "sha256:bfafeb96fdeb10db8767d06e1f07b6fdcddba4aaa0dd15058561a49f7ae45345"}, + {file = "mmhash3-3.0.1-cp36-cp36m-win_amd64.whl", hash = "sha256:97fe077b24c948709ed2afc749bf6285d407bc54ff12c63d2dc86678c38a0b8e"}, + {file = "mmhash3-3.0.1-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:0cfd91ccd5fca1ba7ee925297131a15dfb94c714bfe6ba0fb3b1ca78b12bbfec"}, + {file = "mmhash3-3.0.1-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:6d51b1005233141ce7394531af40a3f0fc1f274467bf8dff44dcf7987924fe58"}, + {file = "mmhash3-3.0.1-cp37-cp37m-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:855c67b100e37df166acf79cdff58fa8f9f6c48be0d1e1b6e9ad0fa34a9661ef"}, + {file = "mmhash3-3.0.1-cp37-cp37m-win32.whl", hash = "sha256:bb3030df1334fd665427f8be8e8ce4f04aeab7f6010ce4f2c128f0099bdab96f"}, + {file = "mmhash3-3.0.1-cp37-cp37m-win_amd64.whl", hash = "sha256:1545e1177294afe4912d5a5e401c7fa9b799dd109b30289e7af74d5b07e7c474"}, + {file = "mmhash3-3.0.1-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:2479899e7dda834a671991a1098a691ab1c2eaa20c3e939d691ca4a19361cfe0"}, + {file = "mmhash3-3.0.1-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:c9056196d5e3d3d844433a63d806a683f710ab3aaf1c910550c7746464bc43ae"}, + {file = "mmhash3-3.0.1-cp38-cp38-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:a0d4c307af0bf70207305f70f131898be071d1b19a89f462b13487f5c25e8d4e"}, + {file = "mmhash3-3.0.1-cp38-cp38-win32.whl", hash = "sha256:5f885f65e329fd14bc38debac4a79eacf381e856965d9c65c4d1c6946ea190d0"}, + {file = "mmhash3-3.0.1-cp38-cp38-win_amd64.whl", hash = "sha256:3b42d0bda5e1cd22c18b16887b0521060fb59d0aaaaf033feacbc0a2492d20fe"}, + {file = "mmhash3-3.0.1-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:d3f333286bb87aa9dc6bd8e7960a55a27b011a401f24b889a50e6d219f65e7c9"}, + {file = "mmhash3-3.0.1-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:6b7ef2eb95a18bcd02ce0d3e047adde3a025afd96c1d266a8a0d44574f44a307"}, + {file = "mmhash3-3.0.1-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:d6ac8a5f511c60f341bf9cae462bb4941abb149d98464ba5f4f4548875b601c6"}, + {file = "mmhash3-3.0.1-cp39-cp39-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:efef9e632e6e248e46f52d108a5ebd1f28edaf427b7fd47ebf97dbff4b2cab81"}, + {file = "mmhash3-3.0.1-cp39-cp39-win32.whl", hash = "sha256:bdac06d72e448c67afb12e758b203d875e29d4097bb279a38a5649d44b518ba7"}, + {file = "mmhash3-3.0.1-cp39-cp39-win_amd64.whl", hash = "sha256:0baeaa20cac5f75ed14f28056826bd9d9c8b2354b382073f3fd5190708992a0d"}, + {file = "mmhash3-3.0.1.tar.gz", hash = "sha256:a00d68f4a1cc434b9501513c8a29e18ed1ddad383677d72b41d71d0d862348af"}, +] +moto = [ + {file = "moto-4.1.2-py2.py3-none-any.whl", hash = "sha256:1b361ece638c74a657325378a259276f368aafce2f8be84f8143e69fa93ce8ec"}, + {file = "moto-4.1.2.tar.gz", hash = "sha256:63431733d2a02c7bd652ad71ec1da442a0e0d580cbac5eeb50d440a2ce066eac"}, +] +msal = [ + {file = "msal-1.21.0-py2.py3-none-any.whl", hash = "sha256:e8444617c1eccdff7bb73f5d4f94036002accea4a2c05f8f39c9efb5bd2b0c6a"}, + {file = "msal-1.21.0.tar.gz", hash = "sha256:96b5c867830fd116e5f7d0ec8ef1b238b4cda4d1aea86d8fecf518260e136fbf"}, +] +msal-extensions = [ + {file = "msal-extensions-1.0.0.tar.gz", hash = "sha256:c676aba56b0cce3783de1b5c5ecfe828db998167875126ca4b47dc6436451354"}, + {file = "msal_extensions-1.0.0-py2.py3-none-any.whl", hash = "sha256:91e3db9620b822d0ed2b4d1850056a0f133cba04455e62f11612e40f5502f2ee"}, +] +msrest = [ + {file = "msrest-0.7.1-py3-none-any.whl", hash = "sha256:21120a810e1233e5e6cc7fe40b474eeb4ec6f757a15d7cf86702c369f9567c32"}, + {file = "msrest-0.7.1.zip", hash = "sha256:6e7661f46f3afd88b75667b7187a92829924446c7ea1d169be8c4bb7eeb788b9"}, +] +multidict = [ + {file = "multidict-6.0.4-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:0b1a97283e0c85772d613878028fec909f003993e1007eafa715b24b377cb9b8"}, + {file = "multidict-6.0.4-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:eeb6dcc05e911516ae3d1f207d4b0520d07f54484c49dfc294d6e7d63b734171"}, + {file = "multidict-6.0.4-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:d6d635d5209b82a3492508cf5b365f3446afb65ae7ebd755e70e18f287b0adf7"}, + {file = "multidict-6.0.4-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:c048099e4c9e9d615545e2001d3d8a4380bd403e1a0578734e0d31703d1b0c0b"}, + {file = "multidict-6.0.4-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:ea20853c6dbbb53ed34cb4d080382169b6f4554d394015f1bef35e881bf83547"}, + {file = "multidict-6.0.4-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:16d232d4e5396c2efbbf4f6d4df89bfa905eb0d4dc5b3549d872ab898451f569"}, + {file = "multidict-6.0.4-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:36c63aaa167f6c6b04ef2c85704e93af16c11d20de1d133e39de6a0e84582a93"}, + {file = "multidict-6.0.4-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:64bdf1086b6043bf519869678f5f2757f473dee970d7abf6da91ec00acb9cb98"}, + {file = "multidict-6.0.4-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:43644e38f42e3af682690876cff722d301ac585c5b9e1eacc013b7a3f7b696a0"}, + {file = "multidict-6.0.4-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:7582a1d1030e15422262de9f58711774e02fa80df0d1578995c76214f6954988"}, + {file = "multidict-6.0.4-cp310-cp310-musllinux_1_1_ppc64le.whl", hash = "sha256:ddff9c4e225a63a5afab9dd15590432c22e8057e1a9a13d28ed128ecf047bbdc"}, + {file = "multidict-6.0.4-cp310-cp310-musllinux_1_1_s390x.whl", hash = "sha256:ee2a1ece51b9b9e7752e742cfb661d2a29e7bcdba2d27e66e28a99f1890e4fa0"}, + {file = "multidict-6.0.4-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:a2e4369eb3d47d2034032a26c7a80fcb21a2cb22e1173d761a162f11e562caa5"}, + {file = "multidict-6.0.4-cp310-cp310-win32.whl", hash = "sha256:574b7eae1ab267e5f8285f0fe881f17efe4b98c39a40858247720935b893bba8"}, + {file = "multidict-6.0.4-cp310-cp310-win_amd64.whl", hash = "sha256:4dcbb0906e38440fa3e325df2359ac6cb043df8e58c965bb45f4e406ecb162cc"}, + {file = "multidict-6.0.4-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:0dfad7a5a1e39c53ed00d2dd0c2e36aed4650936dc18fd9a1826a5ae1cad6f03"}, + {file = "multidict-6.0.4-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:64da238a09d6039e3bd39bb3aee9c21a5e34f28bfa5aa22518581f910ff94af3"}, + {file = "multidict-6.0.4-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:ff959bee35038c4624250473988b24f846cbeb2c6639de3602c073f10410ceba"}, + {file = "multidict-6.0.4-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:01a3a55bd90018c9c080fbb0b9f4891db37d148a0a18722b42f94694f8b6d4c9"}, + {file = "multidict-6.0.4-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:c5cb09abb18c1ea940fb99360ea0396f34d46566f157122c92dfa069d3e0e982"}, + {file = "multidict-6.0.4-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:666daae833559deb2d609afa4490b85830ab0dfca811a98b70a205621a6109fe"}, + {file = "multidict-6.0.4-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:11bdf3f5e1518b24530b8241529d2050014c884cf18b6fc69c0c2b30ca248710"}, + {file = "multidict-6.0.4-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:7d18748f2d30f94f498e852c67d61261c643b349b9d2a581131725595c45ec6c"}, + {file = "multidict-6.0.4-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:458f37be2d9e4c95e2d8866a851663cbc76e865b78395090786f6cd9b3bbf4f4"}, + {file = "multidict-6.0.4-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:b1a2eeedcead3a41694130495593a559a668f382eee0727352b9a41e1c45759a"}, + {file = "multidict-6.0.4-cp311-cp311-musllinux_1_1_ppc64le.whl", hash = "sha256:7d6ae9d593ef8641544d6263c7fa6408cc90370c8cb2bbb65f8d43e5b0351d9c"}, + {file = "multidict-6.0.4-cp311-cp311-musllinux_1_1_s390x.whl", hash = "sha256:5979b5632c3e3534e42ca6ff856bb24b2e3071b37861c2c727ce220d80eee9ed"}, + {file = "multidict-6.0.4-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:dcfe792765fab89c365123c81046ad4103fcabbc4f56d1c1997e6715e8015461"}, + {file = "multidict-6.0.4-cp311-cp311-win32.whl", hash = "sha256:3601a3cece3819534b11d4efc1eb76047488fddd0c85a3948099d5da4d504636"}, + {file = "multidict-6.0.4-cp311-cp311-win_amd64.whl", hash = "sha256:81a4f0b34bd92df3da93315c6a59034df95866014ac08535fc819f043bfd51f0"}, + {file = "multidict-6.0.4-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:67040058f37a2a51ed8ea8f6b0e6ee5bd78ca67f169ce6122f3e2ec80dfe9b78"}, + {file = "multidict-6.0.4-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:853888594621e6604c978ce2a0444a1e6e70c8d253ab65ba11657659dcc9100f"}, + {file = "multidict-6.0.4-cp37-cp37m-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:39ff62e7d0f26c248b15e364517a72932a611a9b75f35b45be078d81bdb86603"}, + {file = "multidict-6.0.4-cp37-cp37m-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:af048912e045a2dc732847d33821a9d84ba553f5c5f028adbd364dd4765092ac"}, + {file = "multidict-6.0.4-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:b1e8b901e607795ec06c9e42530788c45ac21ef3aaa11dbd0c69de543bfb79a9"}, + {file = "multidict-6.0.4-cp37-cp37m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:62501642008a8b9871ddfccbf83e4222cf8ac0d5aeedf73da36153ef2ec222d2"}, + {file = "multidict-6.0.4-cp37-cp37m-musllinux_1_1_aarch64.whl", hash = "sha256:99b76c052e9f1bc0721f7541e5e8c05db3941eb9ebe7b8553c625ef88d6eefde"}, + {file = "multidict-6.0.4-cp37-cp37m-musllinux_1_1_i686.whl", hash = "sha256:509eac6cf09c794aa27bcacfd4d62c885cce62bef7b2c3e8b2e49d365b5003fe"}, + {file = "multidict-6.0.4-cp37-cp37m-musllinux_1_1_ppc64le.whl", hash = "sha256:21a12c4eb6ddc9952c415f24eef97e3e55ba3af61f67c7bc388dcdec1404a067"}, + {file = "multidict-6.0.4-cp37-cp37m-musllinux_1_1_s390x.whl", hash = "sha256:5cad9430ab3e2e4fa4a2ef4450f548768400a2ac635841bc2a56a2052cdbeb87"}, + {file = "multidict-6.0.4-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:ab55edc2e84460694295f401215f4a58597f8f7c9466faec545093045476327d"}, + {file = "multidict-6.0.4-cp37-cp37m-win32.whl", hash = "sha256:5a4dcf02b908c3b8b17a45fb0f15b695bf117a67b76b7ad18b73cf8e92608775"}, + {file = "multidict-6.0.4-cp37-cp37m-win_amd64.whl", hash = "sha256:6ed5f161328b7df384d71b07317f4d8656434e34591f20552c7bcef27b0ab88e"}, + {file = "multidict-6.0.4-cp38-cp38-macosx_10_9_universal2.whl", hash = "sha256:5fc1b16f586f049820c5c5b17bb4ee7583092fa0d1c4e28b5239181ff9532e0c"}, + {file = "multidict-6.0.4-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:1502e24330eb681bdaa3eb70d6358e818e8e8f908a22a1851dfd4e15bc2f8161"}, + {file = "multidict-6.0.4-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:b692f419760c0e65d060959df05f2a531945af31fda0c8a3b3195d4efd06de11"}, + {file = "multidict-6.0.4-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:45e1ecb0379bfaab5eef059f50115b54571acfbe422a14f668fc8c27ba410e7e"}, + {file = "multidict-6.0.4-cp38-cp38-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:ddd3915998d93fbcd2566ddf9cf62cdb35c9e093075f862935573d265cf8f65d"}, + {file = "multidict-6.0.4-cp38-cp38-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:59d43b61c59d82f2effb39a93c48b845efe23a3852d201ed2d24ba830d0b4cf2"}, + {file = "multidict-6.0.4-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:cc8e1d0c705233c5dd0c5e6460fbad7827d5d36f310a0fadfd45cc3029762258"}, + {file = "multidict-6.0.4-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:d6aa0418fcc838522256761b3415822626f866758ee0bc6632c9486b179d0b52"}, + {file = "multidict-6.0.4-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:6748717bb10339c4760c1e63da040f5f29f5ed6e59d76daee30305894069a660"}, + {file = "multidict-6.0.4-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:4d1a3d7ef5e96b1c9e92f973e43aa5e5b96c659c9bc3124acbbd81b0b9c8a951"}, + {file = "multidict-6.0.4-cp38-cp38-musllinux_1_1_ppc64le.whl", hash = "sha256:4372381634485bec7e46718edc71528024fcdc6f835baefe517b34a33c731d60"}, + {file = "multidict-6.0.4-cp38-cp38-musllinux_1_1_s390x.whl", hash = "sha256:fc35cb4676846ef752816d5be2193a1e8367b4c1397b74a565a9d0389c433a1d"}, + {file = "multidict-6.0.4-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:4b9d9e4e2b37daddb5c23ea33a3417901fa7c7b3dee2d855f63ee67a0b21e5b1"}, + {file = "multidict-6.0.4-cp38-cp38-win32.whl", hash = "sha256:e41b7e2b59679edfa309e8db64fdf22399eec4b0b24694e1b2104fb789207779"}, + {file = "multidict-6.0.4-cp38-cp38-win_amd64.whl", hash = "sha256:d6c254ba6e45d8e72739281ebc46ea5eb5f101234f3ce171f0e9f5cc86991480"}, + {file = "multidict-6.0.4-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:16ab77bbeb596e14212e7bab8429f24c1579234a3a462105cda4a66904998664"}, + {file = "multidict-6.0.4-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:bc779e9e6f7fda81b3f9aa58e3a6091d49ad528b11ed19f6621408806204ad35"}, + {file = "multidict-6.0.4-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:4ceef517eca3e03c1cceb22030a3e39cb399ac86bff4e426d4fc6ae49052cc60"}, + {file = "multidict-6.0.4-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:281af09f488903fde97923c7744bb001a9b23b039a909460d0f14edc7bf59706"}, + {file = "multidict-6.0.4-cp39-cp39-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:52f2dffc8acaba9a2f27174c41c9e57f60b907bb9f096b36b1a1f3be71c6284d"}, + {file = "multidict-6.0.4-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:b41156839806aecb3641f3208c0dafd3ac7775b9c4c422d82ee2a45c34ba81ca"}, + {file = "multidict-6.0.4-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:d5e3fc56f88cc98ef8139255cf8cd63eb2c586531e43310ff859d6bb3a6b51f1"}, + {file = "multidict-6.0.4-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:8316a77808c501004802f9beebde51c9f857054a0c871bd6da8280e718444449"}, + {file = "multidict-6.0.4-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:f70b98cd94886b49d91170ef23ec5c0e8ebb6f242d734ed7ed677b24d50c82cf"}, + {file = "multidict-6.0.4-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:bf6774e60d67a9efe02b3616fee22441d86fab4c6d335f9d2051d19d90a40063"}, + {file = "multidict-6.0.4-cp39-cp39-musllinux_1_1_ppc64le.whl", hash = "sha256:e69924bfcdda39b722ef4d9aa762b2dd38e4632b3641b1d9a57ca9cd18f2f83a"}, + {file = "multidict-6.0.4-cp39-cp39-musllinux_1_1_s390x.whl", hash = "sha256:6b181d8c23da913d4ff585afd1155a0e1194c0b50c54fcfe286f70cdaf2b7176"}, + {file = "multidict-6.0.4-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:52509b5be062d9eafc8170e53026fbc54cf3b32759a23d07fd935fb04fc22d95"}, + {file = "multidict-6.0.4-cp39-cp39-win32.whl", hash = "sha256:27c523fbfbdfd19c6867af7346332b62b586eed663887392cff78d614f9ec313"}, + {file = "multidict-6.0.4-cp39-cp39-win_amd64.whl", hash = "sha256:33029f5734336aa0d4c0384525da0387ef89148dc7191aae00ca5fb23d7aafc2"}, + {file = "multidict-6.0.4.tar.gz", hash = "sha256:3666906492efb76453c0e7b97f2cf459b0682e7402c0489a95484965dbc1da49"}, +] +nodeenv = [ + {file = "nodeenv-1.7.0-py2.py3-none-any.whl", hash = "sha256:27083a7b96a25f2f5e1d8cb4b6317ee8aeda3bdd121394e5ac54e498028a042e"}, + {file = "nodeenv-1.7.0.tar.gz", hash = "sha256:e0e7f7dfb85fc5394c6fe1e8fa98131a2473e04311a45afb6508f7cf1836fa2b"}, +] +numpy = [ + {file = "numpy-1.24.2-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:eef70b4fc1e872ebddc38cddacc87c19a3709c0e3e5d20bf3954c147b1dd941d"}, + {file = "numpy-1.24.2-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:e8d2859428712785e8a8b7d2b3ef0a1d1565892367b32f915c4a4df44d0e64f5"}, + {file = "numpy-1.24.2-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:6524630f71631be2dabe0c541e7675db82651eb998496bbe16bc4f77f0772253"}, + {file = "numpy-1.24.2-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:a51725a815a6188c662fb66fb32077709a9ca38053f0274640293a14fdd22978"}, + {file = "numpy-1.24.2-cp310-cp310-win32.whl", hash = "sha256:2620e8592136e073bd12ee4536149380695fbe9ebeae845b81237f986479ffc9"}, + {file = "numpy-1.24.2-cp310-cp310-win_amd64.whl", hash = "sha256:97cf27e51fa078078c649a51d7ade3c92d9e709ba2bfb97493007103c741f1d0"}, + {file = "numpy-1.24.2-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:7de8fdde0003f4294655aa5d5f0a89c26b9f22c0a58790c38fae1ed392d44a5a"}, + {file = "numpy-1.24.2-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:4173bde9fa2a005c2c6e2ea8ac1618e2ed2c1c6ec8a7657237854d42094123a0"}, + {file = "numpy-1.24.2-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:4cecaed30dc14123020f77b03601559fff3e6cd0c048f8b5289f4eeabb0eb281"}, + {file = "numpy-1.24.2-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:9a23f8440561a633204a67fb44617ce2a299beecf3295f0d13c495518908e910"}, + {file = "numpy-1.24.2-cp311-cp311-win32.whl", hash = "sha256:e428c4fbfa085f947b536706a2fc349245d7baa8334f0c5723c56a10595f9b95"}, + {file = "numpy-1.24.2-cp311-cp311-win_amd64.whl", hash = "sha256:557d42778a6869c2162deb40ad82612645e21d79e11c1dc62c6e82a2220ffb04"}, + {file = "numpy-1.24.2-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:d0a2db9d20117bf523dde15858398e7c0858aadca7c0f088ac0d6edd360e9ad2"}, + {file = "numpy-1.24.2-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:c72a6b2f4af1adfe193f7beb91ddf708ff867a3f977ef2ec53c0ffb8283ab9f5"}, + {file = "numpy-1.24.2-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:c29e6bd0ec49a44d7690ecb623a8eac5ab8a923bce0bea6293953992edf3a76a"}, + {file = "numpy-1.24.2-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:2eabd64ddb96a1239791da78fa5f4e1693ae2dadc82a76bc76a14cbb2b966e96"}, + {file = "numpy-1.24.2-cp38-cp38-win32.whl", hash = "sha256:e3ab5d32784e843fc0dd3ab6dcafc67ef806e6b6828dc6af2f689be0eb4d781d"}, + {file = "numpy-1.24.2-cp38-cp38-win_amd64.whl", hash = "sha256:76807b4063f0002c8532cfeac47a3068a69561e9c8715efdad3c642eb27c0756"}, + {file = "numpy-1.24.2-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:4199e7cfc307a778f72d293372736223e39ec9ac096ff0a2e64853b866a8e18a"}, + {file = "numpy-1.24.2-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:adbdce121896fd3a17a77ab0b0b5eedf05a9834a18699db6829a64e1dfccca7f"}, + {file = "numpy-1.24.2-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:889b2cc88b837d86eda1b17008ebeb679d82875022200c6e8e4ce6cf549b7acb"}, + {file = "numpy-1.24.2-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:f64bb98ac59b3ea3bf74b02f13836eb2e24e48e0ab0145bbda646295769bd780"}, + {file = "numpy-1.24.2-cp39-cp39-win32.whl", hash = "sha256:63e45511ee4d9d976637d11e6c9864eae50e12dc9598f531c035265991910468"}, + {file = "numpy-1.24.2-cp39-cp39-win_amd64.whl", hash = "sha256:a77d3e1163a7770164404607b7ba3967fb49b24782a6ef85d9b5f54126cc39e5"}, + {file = "numpy-1.24.2-pp38-pypy38_pp73-macosx_10_9_x86_64.whl", hash = "sha256:92011118955724465fb6853def593cf397b4a1367495e0b59a7e69d40c4eb71d"}, + {file = "numpy-1.24.2-pp38-pypy38_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:f9006288bcf4895917d02583cf3411f98631275bc67cce355a7f39f8c14338fa"}, + {file = "numpy-1.24.2-pp38-pypy38_pp73-win_amd64.whl", hash = "sha256:150947adbdfeceec4e5926d956a06865c1c690f2fd902efede4ca6fe2e657c3f"}, + {file = "numpy-1.24.2.tar.gz", hash = "sha256:003a9f530e880cb2cd177cba1af7220b9aa42def9c4afc2a2fc3ee6be7eb2b22"}, +] +oauthlib = [ + {file = "oauthlib-3.2.2-py3-none-any.whl", hash = "sha256:8139f29aac13e25d502680e9e19963e83f16838d48a0d71c287fe40e7067fbca"}, + {file = "oauthlib-3.2.2.tar.gz", hash = "sha256:9859c40929662bec5d64f34d01c99e093149682a3f38915dc0655d5a633dd918"}, +] +packaging = [ + {file = "packaging-23.0-py3-none-any.whl", hash = "sha256:714ac14496c3e68c99c29b00845f7a2b85f3bb6f1078fd9f72fd20f0570002b2"}, + {file = "packaging-23.0.tar.gz", hash = "sha256:b6ad297f8907de0fa2fe1ccbd26fdaf387f5f47c7275fedf8cce89f99446cf97"}, +] +pandas = [ + {file = "pandas-1.5.3-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:3749077d86e3a2f0ed51367f30bf5b82e131cc0f14260c4d3e499186fccc4406"}, + {file = "pandas-1.5.3-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:972d8a45395f2a2d26733eb8d0f629b2f90bebe8e8eddbb8829b180c09639572"}, + {file = "pandas-1.5.3-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:50869a35cbb0f2e0cd5ec04b191e7b12ed688874bd05dd777c19b28cbea90996"}, + {file = "pandas-1.5.3-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:c3ac844a0fe00bfaeb2c9b51ab1424e5c8744f89860b138434a363b1f620f354"}, + {file = "pandas-1.5.3-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:7a0a56cef15fd1586726dace5616db75ebcfec9179a3a55e78f72c5639fa2a23"}, + {file = "pandas-1.5.3-cp310-cp310-win_amd64.whl", hash = "sha256:478ff646ca42b20376e4ed3fa2e8d7341e8a63105586efe54fa2508ee087f328"}, + {file = "pandas-1.5.3-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:6973549c01ca91ec96199e940495219c887ea815b2083722821f1d7abfa2b4dc"}, + {file = "pandas-1.5.3-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:c39a8da13cede5adcd3be1182883aea1c925476f4e84b2807a46e2775306305d"}, + {file = "pandas-1.5.3-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:f76d097d12c82a535fda9dfe5e8dd4127952b45fea9b0276cb30cca5ea313fbc"}, + {file = "pandas-1.5.3-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:e474390e60ed609cec869b0da796ad94f420bb057d86784191eefc62b65819ae"}, + {file = "pandas-1.5.3-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:5f2b952406a1588ad4cad5b3f55f520e82e902388a6d5a4a91baa8d38d23c7f6"}, + {file = "pandas-1.5.3-cp311-cp311-win_amd64.whl", hash = "sha256:bc4c368f42b551bf72fac35c5128963a171b40dce866fb066540eeaf46faa003"}, + {file = "pandas-1.5.3-cp38-cp38-macosx_10_9_universal2.whl", hash = "sha256:14e45300521902689a81f3f41386dc86f19b8ba8dd5ac5a3c7010ef8d2932813"}, + {file = "pandas-1.5.3-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:9842b6f4b8479e41968eced654487258ed81df7d1c9b7b870ceea24ed9459b31"}, + {file = "pandas-1.5.3-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:26d9c71772c7afb9d5046e6e9cf42d83dd147b5cf5bcb9d97252077118543792"}, + {file = "pandas-1.5.3-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:5fbcb19d6fceb9e946b3e23258757c7b225ba450990d9ed63ccceeb8cae609f7"}, + {file = "pandas-1.5.3-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:565fa34a5434d38e9d250af3c12ff931abaf88050551d9fbcdfafca50d62babf"}, + {file = "pandas-1.5.3-cp38-cp38-win32.whl", hash = "sha256:87bd9c03da1ac870a6d2c8902a0e1fd4267ca00f13bc494c9e5a9020920e1d51"}, + {file = "pandas-1.5.3-cp38-cp38-win_amd64.whl", hash = "sha256:41179ce559943d83a9b4bbacb736b04c928b095b5f25dd2b7389eda08f46f373"}, + {file = "pandas-1.5.3-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:c74a62747864ed568f5a82a49a23a8d7fe171d0c69038b38cedf0976831296fa"}, + {file = "pandas-1.5.3-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:c4c00e0b0597c8e4f59e8d461f797e5d70b4d025880516a8261b2817c47759ee"}, + {file = "pandas-1.5.3-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:a50d9a4336a9621cab7b8eb3fb11adb82de58f9b91d84c2cd526576b881a0c5a"}, + {file = "pandas-1.5.3-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:dd05f7783b3274aa206a1af06f0ceed3f9b412cf665b7247eacd83be41cf7bf0"}, + {file = "pandas-1.5.3-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:9f69c4029613de47816b1bb30ff5ac778686688751a5e9c99ad8c7031f6508e5"}, + {file = "pandas-1.5.3-cp39-cp39-win32.whl", hash = "sha256:7cec0bee9f294e5de5bbfc14d0573f65526071029d036b753ee6507d2a21480a"}, + {file = "pandas-1.5.3-cp39-cp39-win_amd64.whl", hash = "sha256:dfd681c5dc216037e0b0a2c821f5ed99ba9f03ebcf119c7dac0e9a7b960b9ec9"}, + {file = "pandas-1.5.3.tar.gz", hash = "sha256:74a3fd7e5a7ec052f183273dc7b0acd3a863edf7520f5d3a1765c04ffdb3b0b1"}, +] +platformdirs = [ + {file = "platformdirs-3.0.0-py3-none-any.whl", hash = "sha256:b1d5eb14f221506f50d6604a561f4c5786d9e80355219694a1b244bcd96f4567"}, + {file = "platformdirs-3.0.0.tar.gz", hash = "sha256:8a1228abb1ef82d788f74139988b137e78692984ec7b08eaa6c65f1723af28f9"}, +] +pluggy = [ + {file = "pluggy-1.0.0-py2.py3-none-any.whl", hash = "sha256:74134bbf457f031a36d68416e1509f34bd5ccc019f0bcc952c7b909d06b37bd3"}, + {file = "pluggy-1.0.0.tar.gz", hash = "sha256:4224373bacce55f955a878bf9cfa763c1e360858e330072059e10bad68531159"}, +] +portalocker = [ + {file = "portalocker-2.7.0-py2.py3-none-any.whl", hash = "sha256:a07c5b4f3985c3cf4798369631fb7011adb498e2a46d8440efc75a8f29a0f983"}, + {file = "portalocker-2.7.0.tar.gz", hash = "sha256:032e81d534a88ec1736d03f780ba073f047a06c478b06e2937486f334e955c51"}, +] +pre-commit = [ + {file = "pre_commit-3.0.4-py2.py3-none-any.whl", hash = "sha256:9e3255edb0c9e7fe9b4f328cb3dc86069f8fdc38026f1bf521018a05eaf4d67b"}, + {file = "pre_commit-3.0.4.tar.gz", hash = "sha256:bc4687478d55578c4ac37272fe96df66f73d9b5cf81be6f28627d4e712e752d5"}, +] +pyarrow = [ + {file = "pyarrow-11.0.0-cp310-cp310-macosx_10_14_x86_64.whl", hash = "sha256:40bb42afa1053c35c749befbe72f6429b7b5f45710e85059cdd534553ebcf4f2"}, + {file = "pyarrow-11.0.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:7c28b5f248e08dea3b3e0c828b91945f431f4202f1a9fe84d1012a761324e1ba"}, + {file = "pyarrow-11.0.0-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a37bc81f6c9435da3c9c1e767324ac3064ffbe110c4e460660c43e144be4ed85"}, + {file = "pyarrow-11.0.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:ad7c53def8dbbc810282ad308cc46a523ec81e653e60a91c609c2233ae407689"}, + {file = "pyarrow-11.0.0-cp310-cp310-win_amd64.whl", hash = "sha256:25aa11c443b934078bfd60ed63e4e2d42461682b5ac10f67275ea21e60e6042c"}, + {file = "pyarrow-11.0.0-cp311-cp311-macosx_10_14_x86_64.whl", hash = "sha256:e217d001e6389b20a6759392a5ec49d670757af80101ee6b5f2c8ff0172e02ca"}, + {file = "pyarrow-11.0.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:ad42bb24fc44c48f74f0d8c72a9af16ba9a01a2ccda5739a517aa860fa7e3d56"}, + {file = "pyarrow-11.0.0-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:2d942c690ff24a08b07cb3df818f542a90e4d359381fbff71b8f2aea5bf58841"}, + {file = "pyarrow-11.0.0-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:f010ce497ca1b0f17a8243df3048055c0d18dcadbcc70895d5baf8921f753de5"}, + {file = "pyarrow-11.0.0-cp311-cp311-win_amd64.whl", hash = "sha256:2f51dc7ca940fdf17893227edb46b6784d37522ce08d21afc56466898cb213b2"}, + {file = "pyarrow-11.0.0-cp37-cp37m-macosx_10_14_x86_64.whl", hash = "sha256:1cbcfcbb0e74b4d94f0b7dde447b835a01bc1d16510edb8bb7d6224b9bf5bafc"}, + {file = "pyarrow-11.0.0-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:aaee8f79d2a120bf3e032d6d64ad20b3af6f56241b0ffc38d201aebfee879d00"}, + {file = "pyarrow-11.0.0-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:410624da0708c37e6a27eba321a72f29d277091c8f8d23f72c92bada4092eb5e"}, + {file = "pyarrow-11.0.0-cp37-cp37m-win_amd64.whl", hash = "sha256:2d53ba72917fdb71e3584ffc23ee4fcc487218f8ff29dd6df3a34c5c48fe8c06"}, + {file = "pyarrow-11.0.0-cp38-cp38-macosx_10_14_x86_64.whl", hash = "sha256:f12932e5a6feb5c58192209af1d2607d488cb1d404fbc038ac12ada60327fa34"}, + {file = "pyarrow-11.0.0-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:41a1451dd895c0b2964b83d91019e46f15b5564c7ecd5dcb812dadd3f05acc97"}, + {file = "pyarrow-11.0.0-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:becc2344be80e5dce4e1b80b7c650d2fc2061b9eb339045035a1baa34d5b8f1c"}, + {file = "pyarrow-11.0.0-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:8f40be0d7381112a398b93c45a7e69f60261e7b0269cc324e9f739ce272f4f70"}, + {file = "pyarrow-11.0.0-cp38-cp38-win_amd64.whl", hash = "sha256:362a7c881b32dc6b0eccf83411a97acba2774c10edcec715ccaab5ebf3bb0835"}, + {file = "pyarrow-11.0.0-cp39-cp39-macosx_10_14_x86_64.whl", hash = "sha256:ccbf29a0dadfcdd97632b4f7cca20a966bb552853ba254e874c66934931b9841"}, + {file = "pyarrow-11.0.0-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:3e99be85973592051e46412accea31828da324531a060bd4585046a74ba45854"}, + {file = "pyarrow-11.0.0-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:69309be84dcc36422574d19c7d3a30a7ea43804f12552356d1ab2a82a713c418"}, + {file = "pyarrow-11.0.0-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:da93340fbf6f4e2a62815064383605b7ffa3e9eeb320ec839995b1660d69f89b"}, + {file = "pyarrow-11.0.0-cp39-cp39-win_amd64.whl", hash = "sha256:caad867121f182d0d3e1a0d36f197df604655d0b466f1bc9bafa903aa95083e4"}, + {file = "pyarrow-11.0.0.tar.gz", hash = "sha256:5461c57dbdb211a632a48facb9b39bbeb8a7905ec95d768078525283caef5f6d"}, +] +pycparser = [ + {file = "pycparser-2.21-py2.py3-none-any.whl", hash = "sha256:8ee45429555515e1f6b185e78100aea234072576aa43ab53aefcae078162fca9"}, + {file = "pycparser-2.21.tar.gz", hash = "sha256:e644fdec12f7872f86c58ff790da456218b10f863970249516d60a5eaca77206"}, +] +pydantic = [ + {file = "pydantic-1.10.4-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:b5635de53e6686fe7a44b5cf25fcc419a0d5e5c1a1efe73d49d48fe7586db854"}, + {file = "pydantic-1.10.4-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:6dc1cc241440ed7ca9ab59d9929075445da6b7c94ced281b3dd4cfe6c8cff817"}, + {file = "pydantic-1.10.4-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:51bdeb10d2db0f288e71d49c9cefa609bca271720ecd0c58009bd7504a0c464c"}, + {file = "pydantic-1.10.4-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:78cec42b95dbb500a1f7120bdf95c401f6abb616bbe8785ef09887306792e66e"}, + {file = "pydantic-1.10.4-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:8775d4ef5e7299a2f4699501077a0defdaac5b6c4321173bcb0f3c496fbadf85"}, + {file = "pydantic-1.10.4-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:572066051eeac73d23f95ba9a71349c42a3e05999d0ee1572b7860235b850cc6"}, + {file = "pydantic-1.10.4-cp310-cp310-win_amd64.whl", hash = "sha256:7feb6a2d401f4d6863050f58325b8d99c1e56f4512d98b11ac64ad1751dc647d"}, + {file = "pydantic-1.10.4-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:39f4a73e5342b25c2959529f07f026ef58147249f9b7431e1ba8414a36761f53"}, + {file = "pydantic-1.10.4-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:983e720704431a6573d626b00662eb78a07148c9115129f9b4351091ec95ecc3"}, + {file = "pydantic-1.10.4-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:75d52162fe6b2b55964fbb0af2ee58e99791a3138588c482572bb6087953113a"}, + {file = "pydantic-1.10.4-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:fdf8d759ef326962b4678d89e275ffc55b7ce59d917d9f72233762061fd04a2d"}, + {file = "pydantic-1.10.4-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:05a81b006be15655b2a1bae5faa4280cf7c81d0e09fcb49b342ebf826abe5a72"}, + {file = "pydantic-1.10.4-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:d88c4c0e5c5dfd05092a4b271282ef0588e5f4aaf345778056fc5259ba098857"}, + {file = "pydantic-1.10.4-cp311-cp311-win_amd64.whl", hash = "sha256:6a05a9db1ef5be0fe63e988f9617ca2551013f55000289c671f71ec16f4985e3"}, + {file = "pydantic-1.10.4-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:887ca463c3bc47103c123bc06919c86720e80e1214aab79e9b779cda0ff92a00"}, + {file = "pydantic-1.10.4-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:fdf88ab63c3ee282c76d652fc86518aacb737ff35796023fae56a65ced1a5978"}, + {file = "pydantic-1.10.4-cp37-cp37m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:a48f1953c4a1d9bd0b5167ac50da9a79f6072c63c4cef4cf2a3736994903583e"}, + {file = "pydantic-1.10.4-cp37-cp37m-musllinux_1_1_i686.whl", hash = "sha256:a9f2de23bec87ff306aef658384b02aa7c32389766af3c5dee9ce33e80222dfa"}, + {file = "pydantic-1.10.4-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:cd8702c5142afda03dc2b1ee6bc358b62b3735b2cce53fc77b31ca9f728e4bc8"}, + {file = "pydantic-1.10.4-cp37-cp37m-win_amd64.whl", hash = "sha256:6e7124d6855b2780611d9f5e1e145e86667eaa3bd9459192c8dc1a097f5e9903"}, + {file = "pydantic-1.10.4-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:0b53e1d41e97063d51a02821b80538053ee4608b9a181c1005441f1673c55423"}, + {file = "pydantic-1.10.4-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:55b1625899acd33229c4352ce0ae54038529b412bd51c4915349b49ca575258f"}, + {file = "pydantic-1.10.4-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:301d626a59edbe5dfb48fcae245896379a450d04baeed50ef40d8199f2733b06"}, + {file = "pydantic-1.10.4-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:b6f9d649892a6f54a39ed56b8dfd5e08b5f3be5f893da430bed76975f3735d15"}, + {file = "pydantic-1.10.4-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:d7b5a3821225f5c43496c324b0d6875fde910a1c2933d726a743ce328fbb2a8c"}, + {file = "pydantic-1.10.4-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:f2f7eb6273dd12472d7f218e1fef6f7c7c2f00ac2e1ecde4db8824c457300416"}, + {file = "pydantic-1.10.4-cp38-cp38-win_amd64.whl", hash = "sha256:4b05697738e7d2040696b0a66d9f0a10bec0efa1883ca75ee9e55baf511909d6"}, + {file = "pydantic-1.10.4-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:a9a6747cac06c2beb466064dda999a13176b23535e4c496c9d48e6406f92d42d"}, + {file = "pydantic-1.10.4-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:eb992a1ef739cc7b543576337bebfc62c0e6567434e522e97291b251a41dad7f"}, + {file = "pydantic-1.10.4-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:990406d226dea0e8f25f643b370224771878142155b879784ce89f633541a024"}, + {file = "pydantic-1.10.4-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:2e82a6d37a95e0b1b42b82ab340ada3963aea1317fd7f888bb6b9dfbf4fff57c"}, + {file = "pydantic-1.10.4-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:9193d4f4ee8feca58bc56c8306bcb820f5c7905fd919e0750acdeeeef0615b28"}, + {file = "pydantic-1.10.4-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:2b3ce5f16deb45c472dde1a0ee05619298c864a20cded09c4edd820e1454129f"}, + {file = "pydantic-1.10.4-cp39-cp39-win_amd64.whl", hash = "sha256:9cbdc268a62d9a98c56e2452d6c41c0263d64a2009aac69246486f01b4f594c4"}, + {file = "pydantic-1.10.4-py3-none-any.whl", hash = "sha256:4948f264678c703f3877d1c8877c4e3b2e12e549c57795107f08cf70c6ec7774"}, + {file = "pydantic-1.10.4.tar.gz", hash = "sha256:b9a3859f24eb4e097502a3be1fb4b2abb79b6103dd9e2e0edb70613a4459a648"}, +] +pygments = [ + {file = "Pygments-2.14.0-py3-none-any.whl", hash = "sha256:fa7bd7bd2771287c0de303af8bfdfc731f51bd2c6a47ab69d117138893b82717"}, + {file = "Pygments-2.14.0.tar.gz", hash = "sha256:b3ed06a9e8ac9a9aae5a6f5dbe78a8a58655d17b43b93c078f094ddc476ae297"}, +] +pyjwt = [ + {file = "PyJWT-2.6.0-py3-none-any.whl", hash = "sha256:d83c3d892a77bbb74d3e1a2cfa90afaadb60945205d1095d9221f04466f64c14"}, + {file = "PyJWT-2.6.0.tar.gz", hash = "sha256:69285c7e31fc44f68a1feb309e948e0df53259d579295e6cfe2b1792329f05fd"}, +] +pyparsing = [ + {file = "pyparsing-3.0.9-py3-none-any.whl", hash = "sha256:5026bae9a10eeaefb61dab2f09052b9f4307d44aee4eda64b309723d8d206bbc"}, + {file = "pyparsing-3.0.9.tar.gz", hash = "sha256:2b020ecf7d21b687f219b71ecad3631f644a47f01403fa1d1036b0c6416d70fb"}, +] +pyproject-hooks = [ + {file = "pyproject_hooks-1.0.0-py3-none-any.whl", hash = "sha256:283c11acd6b928d2f6a7c73fa0d01cb2bdc5f07c57a2eeb6e83d5e56b97976f8"}, + {file = "pyproject_hooks-1.0.0.tar.gz", hash = "sha256:f271b298b97f5955d53fb12b72c1fb1948c22c1a6b70b315c54cedaca0264ef5"}, +] +pytest = [ + {file = "pytest-7.2.1-py3-none-any.whl", hash = "sha256:c7c6ca206e93355074ae32f7403e8ea12163b1163c976fee7d4d84027c162be5"}, + {file = "pytest-7.2.1.tar.gz", hash = "sha256:d45e0952f3727241918b8fd0f376f5ff6b301cc0777c6f9a556935c92d8a7d42"}, +] +pytest-checkdocs = [ + {file = "pytest-checkdocs-2.9.0.tar.gz", hash = "sha256:54b3d1357a289e3f6336d31d4ceb72e99afe0fa4f552990f1b1f015f1925543b"}, + {file = "pytest_checkdocs-2.9.0-py3-none-any.whl", hash = "sha256:4231e5426915a02385e386b5eb4a70490b143d97cede4c7c77691bb320b5a73f"}, +] +python-dateutil = [] +python-snappy = [ + {file = "python-snappy-0.6.1.tar.gz", hash = "sha256:b6a107ab06206acc5359d4c5632bd9b22d448702a79b3169b0c62e0fb808bb2a"}, + {file = "python_snappy-0.6.1-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:b7f920eaf46ebf41bd26f9df51c160d40f9e00b7b48471c3438cb8d027f7fb9b"}, + {file = "python_snappy-0.6.1-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:4ec533a8c1f8df797bded662ec3e494d225b37855bb63eb0d75464a07947477c"}, + {file = "python_snappy-0.6.1-cp310-cp310-manylinux_2_12_i686.manylinux2010_i686.whl", hash = "sha256:6f8bf4708a11b47517baf962f9a02196478bbb10fdb9582add4aa1459fa82380"}, + {file = "python_snappy-0.6.1-cp310-cp310-manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:8d0c019ee7dcf2c60e240877107cddbd95a5b1081787579bf179938392d66480"}, + {file = "python_snappy-0.6.1-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:cb18d9cd7b3f35a2f5af47bb8ed6a5bdbf4f3ddee37f3daade4ab7864c292f5b"}, + {file = "python_snappy-0.6.1-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:b265cde49774752aec9ca7f5d272e3f98718164afc85521622a8a5394158a2b5"}, + {file = "python_snappy-0.6.1-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:d017775851a778ec9cc32651c4464079d06d927303c2dde9ae9830ccf6fe94e1"}, + {file = "python_snappy-0.6.1-cp310-cp310-win32.whl", hash = "sha256:8277d1f6282463c40761f802b742f833f9f2449fcdbb20a96579aa05c8feb614"}, + {file = "python_snappy-0.6.1-cp310-cp310-win_amd64.whl", hash = "sha256:2aaaf618c68d8c9daebc23a20436bd01b09ee70d7fbf7072b7f38b06d2fab539"}, + {file = "python_snappy-0.6.1-cp36-cp36m-macosx_10_9_x86_64.whl", hash = "sha256:277757d5dad4e239dc1417438a0871b65b1b155beb108888e7438c27ffc6a8cc"}, + {file = "python_snappy-0.6.1-cp36-cp36m-manylinux_2_12_i686.manylinux2010_i686.whl", hash = "sha256:e066a0586833d610c4bbddba0be5ba0e3e4f8e0bc5bb6d82103d8f8fc47bb59a"}, + {file = "python_snappy-0.6.1-cp36-cp36m-manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:0d489b50f49433494160c45048fe806de6b3aeab0586e497ebd22a0bab56e427"}, + {file = "python_snappy-0.6.1-cp36-cp36m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:463fd340a499d47b26ca42d2f36a639188738f6e2098c6dbf80aef0e60f461e1"}, + {file = "python_snappy-0.6.1-cp36-cp36m-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:9837ac1650cc68d22a3cf5f15fb62c6964747d16cecc8b22431f113d6e39555d"}, + {file = "python_snappy-0.6.1-cp36-cp36m-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:5e973e637112391f05581f427659c05b30b6843bc522a65be35ac7b18ce3dedd"}, + {file = "python_snappy-0.6.1-cp36-cp36m-win32.whl", hash = "sha256:c20498bd712b6e31a4402e1d027a1cd64f6a4a0066a3fe3c7344475886d07fdf"}, + {file = "python_snappy-0.6.1-cp36-cp36m-win_amd64.whl", hash = "sha256:59e975be4206cc54d0a112ef72fa3970a57c2b1bcc2c97ed41d6df0ebe518228"}, + {file = "python_snappy-0.6.1-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:2a7e528ab6e09c0d67dcb61a1730a292683e5ff9bb088950638d3170cf2a0a54"}, + {file = "python_snappy-0.6.1-cp37-cp37m-manylinux_2_12_i686.manylinux2010_i686.whl", hash = "sha256:39692bedbe0b717001a99915ac0eb2d9d0bad546440d392a2042b96d813eede1"}, + {file = "python_snappy-0.6.1-cp37-cp37m-manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:6a7620404da966f637b9ce8d4d3d543d363223f7a12452a575189c5355fc2d25"}, + {file = "python_snappy-0.6.1-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:7778c224efc38a40d274da4eb82a04cac27aae20012372a7db3c4bbd8926c4d4"}, + {file = "python_snappy-0.6.1-cp37-cp37m-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:1d029f7051ec1bbeaa3e03030b6d8ed47ceb69cae9016f493c802a08af54e026"}, + {file = "python_snappy-0.6.1-cp37-cp37m-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:a0ad38bc98d0b0497a0b0dbc29409bcabfcecff4511ed7063403c86de16927bc"}, + {file = "python_snappy-0.6.1-cp37-cp37m-win32.whl", hash = "sha256:5a453c45178d7864c1bdd6bfe0ee3ed2883f63b9ba2c9bb967c6b586bf763f96"}, + {file = "python_snappy-0.6.1-cp37-cp37m-win_amd64.whl", hash = "sha256:9f0c0d88b84259f93c3aa46398680646f2c23e43394779758d9f739c34e15295"}, + {file = "python_snappy-0.6.1-cp38-cp38-macosx_10_9_universal2.whl", hash = "sha256:5bb05c28298803a74add08ba496879242ef159c75bc86a5406fac0ffc7dd021b"}, + {file = "python_snappy-0.6.1-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:9eac51307c6a1a38d5f86ebabc26a889fddf20cbba7a116ccb54ba1446601d5b"}, + {file = "python_snappy-0.6.1-cp38-cp38-manylinux_2_12_i686.manylinux2010_i686.whl", hash = "sha256:88b6ea78b83d2796f330b0af1b70cdd3965dbdab02d8ac293260ec2c8fe340ee"}, + {file = "python_snappy-0.6.1-cp38-cp38-manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:8c07220408d3268e8268c9351c5c08041bc6f8c6172e59d398b71020df108541"}, + {file = "python_snappy-0.6.1-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:4038019b1bcaadde726a57430718394076c5a21545ebc5badad2c045a09546cf"}, + {file = "python_snappy-0.6.1-cp38-cp38-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:dc96668d9c7cc656609764275c5f8da58ef56d89bdd6810f6923d36497468ff7"}, + {file = "python_snappy-0.6.1-cp38-cp38-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:cf5bb9254e1c38aacf253d510d3d9be631bba21f3d068b17672b38b5cbf2fff5"}, + {file = "python_snappy-0.6.1-cp38-cp38-win32.whl", hash = "sha256:eaf905a580f2747c4a474040a5063cd5e0cc3d1d2d6edb65f28196186493ad4a"}, + {file = "python_snappy-0.6.1-cp38-cp38-win_amd64.whl", hash = "sha256:546c1a7470ecbf6239101e9aff0f709b68ca0f0268b34d9023019a55baa1f7c6"}, + {file = "python_snappy-0.6.1-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:e3a013895c64352b49d0d8e107a84f99631b16dbab156ded33ebf0becf56c8b2"}, + {file = "python_snappy-0.6.1-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:3fb9a88a4dd6336488f3de67ce75816d0d796dce53c2c6e4d70e0b565633c7fd"}, + {file = "python_snappy-0.6.1-cp39-cp39-manylinux_2_12_i686.manylinux2010_i686.whl", hash = "sha256:735cd4528c55dbe4516d6d2b403331a99fc304f8feded8ae887cf97b67d589bb"}, + {file = "python_snappy-0.6.1-cp39-cp39-manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:90b0186516b7a101c14764b0c25931b741fb0102f21253eff67847b4742dfc72"}, + {file = "python_snappy-0.6.1-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:1a993dc8aadd901915a510fe6af5f20ae4256f527040066c22a154db8946751f"}, + {file = "python_snappy-0.6.1-cp39-cp39-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:530bfb9efebcc1aab8bb4ebcbd92b54477eed11f6cf499355e882970a6d3aa7d"}, + {file = "python_snappy-0.6.1-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:5843feb914796b1f0405ccf31ea0fb51034ceb65a7588edfd5a8250cb369e3b2"}, + {file = "python_snappy-0.6.1-cp39-cp39-win32.whl", hash = "sha256:66c80e9b366012dbee262bb1869e4fc5ba8786cda85928481528bc4a72ec2ee8"}, + {file = "python_snappy-0.6.1-cp39-cp39-win_amd64.whl", hash = "sha256:4d3cafdf454354a621c8ab7408e45aa4e9d5c0b943b61ff4815f71ca6bdf0130"}, + {file = "python_snappy-0.6.1-pp37-pypy37_pp73-manylinux_2_12_i686.manylinux2010_i686.whl", hash = "sha256:586724a0276d7a6083a17259d0b51622e492289a9998848a1b01b6441ca12b2f"}, + {file = "python_snappy-0.6.1-pp37-pypy37_pp73-manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:2be4f4550acd484912441f5f1209ba611ac399aac9355fee73611b9a0d4f949c"}, + {file = "python_snappy-0.6.1-pp37-pypy37_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:0bdb6942180660bda7f7d01f4c0def3cfc72b1c6d99aad964801775a3e379aba"}, + {file = "python_snappy-0.6.1-pp37-pypy37_pp73-win_amd64.whl", hash = "sha256:03bb511380fca2a13325b6f16fe8234c8e12da9660f0258cd45d9a02ffc916af"}, ] - -[[package]] -name = "thrift" -version = "0.16.0" -description = "Python bindings for the Apache Thrift RPC system" -category = "main" -optional = true -python-versions = "*" -files = [ - {file = "thrift-0.16.0.tar.gz", hash = "sha256:2b5b6488fcded21f9d312aa23c9ff6a0195d0f6ae26ddbd5ad9e3e25dfc14408"}, +pytz = [ + {file = "pytz-2022.7.1-py2.py3-none-any.whl", hash = "sha256:78f4f37d8198e0627c5f1143240bb0206b8691d8d7ac6d78fee88b78733f8c4a"}, + {file = "pytz-2022.7.1.tar.gz", hash = "sha256:01a0681c4b9684a28304615eba55d1ab31ae00bf68ec157ec3708a8182dbbcd0"}, ] - -[package.dependencies] -six = ">=1.7.2" - -[package.extras] -all = ["tornado (>=4.0)", "twisted"] -tornado = ["tornado (>=4.0)"] -twisted = ["twisted"] - -[[package]] -name = "toml" -version = "0.10.2" -description = "Python Library for Tom's Obvious, Minimal Language" -category = "dev" -optional = false -python-versions = ">=2.6, !=3.0.*, !=3.1.*, !=3.2.*" -files = [ +pywin32 = [ + {file = "pywin32-305-cp310-cp310-win32.whl", hash = "sha256:421f6cd86e84bbb696d54563c48014b12a23ef95a14e0bdba526be756d89f116"}, + {file = "pywin32-305-cp310-cp310-win_amd64.whl", hash = "sha256:73e819c6bed89f44ff1d690498c0a811948f73777e5f97c494c152b850fad478"}, + {file = "pywin32-305-cp310-cp310-win_arm64.whl", hash = "sha256:742eb905ce2187133a29365b428e6c3b9001d79accdc30aa8969afba1d8470f4"}, + {file = "pywin32-305-cp311-cp311-win32.whl", hash = "sha256:19ca459cd2e66c0e2cc9a09d589f71d827f26d47fe4a9d09175f6aa0256b51c2"}, + {file = "pywin32-305-cp311-cp311-win_amd64.whl", hash = "sha256:326f42ab4cfff56e77e3e595aeaf6c216712bbdd91e464d167c6434b28d65990"}, + {file = "pywin32-305-cp311-cp311-win_arm64.whl", hash = "sha256:4ecd404b2c6eceaca52f8b2e3e91b2187850a1ad3f8b746d0796a98b4cea04db"}, + {file = "pywin32-305-cp36-cp36m-win32.whl", hash = "sha256:48d8b1659284f3c17b68587af047d110d8c44837736b8932c034091683e05863"}, + {file = "pywin32-305-cp36-cp36m-win_amd64.whl", hash = "sha256:13362cc5aa93c2beaf489c9c9017c793722aeb56d3e5166dadd5ef82da021fe1"}, + {file = "pywin32-305-cp37-cp37m-win32.whl", hash = "sha256:a55db448124d1c1484df22fa8bbcbc45c64da5e6eae74ab095b9ea62e6d00496"}, + {file = "pywin32-305-cp37-cp37m-win_amd64.whl", hash = "sha256:109f98980bfb27e78f4df8a51a8198e10b0f347257d1e265bb1a32993d0c973d"}, + {file = "pywin32-305-cp38-cp38-win32.whl", hash = "sha256:9dd98384da775afa009bc04863426cb30596fd78c6f8e4e2e5bbf4edf8029504"}, + {file = "pywin32-305-cp38-cp38-win_amd64.whl", hash = "sha256:56d7a9c6e1a6835f521788f53b5af7912090674bb84ef5611663ee1595860fc7"}, + {file = "pywin32-305-cp39-cp39-win32.whl", hash = "sha256:9d968c677ac4d5cbdaa62fd3014ab241718e619d8e36ef8e11fb930515a1e918"}, + {file = "pywin32-305-cp39-cp39-win_amd64.whl", hash = "sha256:50768c6b7c3f0b38b7fb14dd4104da93ebced5f1a50dc0e834594bff6fbe1271"}, +] +pyyaml = [ + {file = "PyYAML-6.0-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:d4db7c7aef085872ef65a8fd7d6d09a14ae91f691dec3e87ee5ee0539d516f53"}, + {file = "PyYAML-6.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:9df7ed3b3d2e0ecfe09e14741b857df43adb5a3ddadc919a2d94fbdf78fea53c"}, + {file = "PyYAML-6.0-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:77f396e6ef4c73fdc33a9157446466f1cff553d979bd00ecb64385760c6babdc"}, + {file = "PyYAML-6.0-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:a80a78046a72361de73f8f395f1f1e49f956c6be882eed58505a15f3e430962b"}, + {file = "PyYAML-6.0-cp310-cp310-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:f84fbc98b019fef2ee9a1cb3ce93e3187a6df0b2538a651bfb890254ba9f90b5"}, + {file = "PyYAML-6.0-cp310-cp310-win32.whl", hash = "sha256:2cd5df3de48857ed0544b34e2d40e9fac445930039f3cfe4bcc592a1f836d513"}, + {file = "PyYAML-6.0-cp310-cp310-win_amd64.whl", hash = "sha256:daf496c58a8c52083df09b80c860005194014c3698698d1a57cbcfa182142a3a"}, + {file = "PyYAML-6.0-cp36-cp36m-macosx_10_9_x86_64.whl", hash = "sha256:897b80890765f037df3403d22bab41627ca8811ae55e9a722fd0392850ec4d86"}, + {file = "PyYAML-6.0-cp36-cp36m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:50602afada6d6cbfad699b0c7bb50d5ccffa7e46a3d738092afddc1f9758427f"}, + {file = "PyYAML-6.0-cp36-cp36m-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:48c346915c114f5fdb3ead70312bd042a953a8ce5c7106d5bfb1a5254e47da92"}, + {file = "PyYAML-6.0-cp36-cp36m-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:98c4d36e99714e55cfbaaee6dd5badbc9a1ec339ebfc3b1f52e293aee6bb71a4"}, + {file = "PyYAML-6.0-cp36-cp36m-win32.whl", hash = "sha256:0283c35a6a9fbf047493e3a0ce8d79ef5030852c51e9d911a27badfde0605293"}, + {file = "PyYAML-6.0-cp36-cp36m-win_amd64.whl", hash = "sha256:07751360502caac1c067a8132d150cf3d61339af5691fe9e87803040dbc5db57"}, + {file = "PyYAML-6.0-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:819b3830a1543db06c4d4b865e70ded25be52a2e0631ccd2f6a47a2822f2fd7c"}, + {file = "PyYAML-6.0-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:473f9edb243cb1935ab5a084eb238d842fb8f404ed2193a915d1784b5a6b5fc0"}, + {file = "PyYAML-6.0-cp37-cp37m-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:0ce82d761c532fe4ec3f87fc45688bdd3a4c1dc5e0b4a19814b9009a29baefd4"}, + {file = "PyYAML-6.0-cp37-cp37m-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:231710d57adfd809ef5d34183b8ed1eeae3f76459c18fb4a0b373ad56bedcdd9"}, + {file = "PyYAML-6.0-cp37-cp37m-win32.whl", hash = "sha256:c5687b8d43cf58545ade1fe3e055f70eac7a5a1a0bf42824308d868289a95737"}, + {file = "PyYAML-6.0-cp37-cp37m-win_amd64.whl", hash = "sha256:d15a181d1ecd0d4270dc32edb46f7cb7733c7c508857278d3d378d14d606db2d"}, + {file = "PyYAML-6.0-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:0b4624f379dab24d3725ffde76559cff63d9ec94e1736b556dacdfebe5ab6d4b"}, + {file = "PyYAML-6.0-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:213c60cd50106436cc818accf5baa1aba61c0189ff610f64f4a3e8c6726218ba"}, + {file = "PyYAML-6.0-cp38-cp38-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:9fa600030013c4de8165339db93d182b9431076eb98eb40ee068700c9c813e34"}, + {file = "PyYAML-6.0-cp38-cp38-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:277a0ef2981ca40581a47093e9e2d13b3f1fbbeffae064c1d21bfceba2030287"}, + {file = "PyYAML-6.0-cp38-cp38-win32.whl", hash = "sha256:d4eccecf9adf6fbcc6861a38015c2a64f38b9d94838ac1810a9023a0609e1b78"}, + {file = "PyYAML-6.0-cp38-cp38-win_amd64.whl", hash = "sha256:1e4747bc279b4f613a09eb64bba2ba602d8a6664c6ce6396a4d0cd413a50ce07"}, + {file = "PyYAML-6.0-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:055d937d65826939cb044fc8c9b08889e8c743fdc6a32b33e2390f66013e449b"}, + {file = "PyYAML-6.0-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:e61ceaab6f49fb8bdfaa0f92c4b57bcfbea54c09277b1b4f7ac376bfb7a7c174"}, + {file = "PyYAML-6.0-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:d67d839ede4ed1b28a4e8909735fc992a923cdb84e618544973d7dfc71540803"}, + {file = "PyYAML-6.0-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:cba8c411ef271aa037d7357a2bc8f9ee8b58b9965831d9e51baf703280dc73d3"}, + {file = "PyYAML-6.0-cp39-cp39-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:40527857252b61eacd1d9af500c3337ba8deb8fc298940291486c465c8b46ec0"}, + {file = "PyYAML-6.0-cp39-cp39-win32.whl", hash = "sha256:b5b9eccad747aabaaffbc6064800670f0c297e52c12754eb1d976c57e4f74dcb"}, + {file = "PyYAML-6.0-cp39-cp39-win_amd64.whl", hash = "sha256:b3d267842bf12586ba6c734f89d1f5b871df0273157918b0ccefa29deb05c21c"}, + {file = "PyYAML-6.0.tar.gz", hash = "sha256:68fb519c14306fec9720a2a5b45bc9f0c8d1b9c72adf45c37baedfcd949c35a2"}, +] +requests = [ + {file = "requests-2.28.2-py3-none-any.whl", hash = "sha256:64299f4909223da747622c030b781c0d7811e359c37124b4bd368fb8c6518baa"}, + {file = "requests-2.28.2.tar.gz", hash = "sha256:98b1b2782e3c6c4904938b84c0eb932721069dfdb9134313beff7c83c2df24bf"}, +] +requests-mock = [ + {file = "requests-mock-1.10.0.tar.gz", hash = "sha256:59c9c32419a9fb1ae83ec242d98e889c45bd7d7a65d48375cc243ec08441658b"}, + {file = "requests_mock-1.10.0-py2.py3-none-any.whl", hash = "sha256:2fdbb637ad17ee15c06f33d31169e71bf9fe2bdb7bc9da26185be0dd8d842699"}, +] +requests-oauthlib = [ + {file = "requests-oauthlib-1.3.1.tar.gz", hash = "sha256:75beac4a47881eeb94d5ea5d6ad31ef88856affe2332b9aafb52c6452ccf0d7a"}, + {file = "requests_oauthlib-1.3.1-py2.py3-none-any.whl", hash = "sha256:2577c501a2fb8d05a304c09d090d6e47c306fef15809d102b327cf8364bddab5"}, +] +responses = [ + {file = "responses-0.22.0-py3-none-any.whl", hash = "sha256:dcf294d204d14c436fddcc74caefdbc5764795a40ff4e6a7740ed8ddbf3294be"}, + {file = "responses-0.22.0.tar.gz", hash = "sha256:396acb2a13d25297789a5866b4881cf4e46ffd49cc26c43ab1117f40b973102e"}, +] +rich = [ + {file = "rich-13.3.1-py3-none-any.whl", hash = "sha256:8aa57747f3fc3e977684f0176a88e789be314a99f99b43b75d1e9cb5dc6db9e9"}, + {file = "rich-13.3.1.tar.gz", hash = "sha256:125d96d20c92b946b983d0d392b84ff945461e5a06d3867e9f9e575f8697b67f"}, +] +s3fs = [ + {file = "s3fs-2023.1.0-py3-none-any.whl", hash = "sha256:a549ae518fff4388bd6fca5248575c29f521e7e39efcd2bfab476651701fd114"}, + {file = "s3fs-2023.1.0.tar.gz", hash = "sha256:8b2e28372423e93f26312208a9272e22a962ddd0a79d63f9a68693b6af5ff187"}, +] +s3transfer = [] +six = [ + {file = "six-1.16.0-py2.py3-none-any.whl", hash = "sha256:8abb2f1d86890a2dfb989f9a77cfcfd3e47c2a354b01111771326f8aa26e0254"}, + {file = "six-1.16.0.tar.gz", hash = "sha256:1e61c37477a1626458e36f7b1d82aa5c9b094fa4802892072e49de9c60c4c926"}, +] +thrift = [] +toml = [ {file = "toml-0.10.2-py2.py3-none-any.whl", hash = "sha256:806143ae5bfb6a3c6e736a764057db0e6a0e05e338b5630894a5f779cabb4f9b"}, {file = "toml-0.10.2.tar.gz", hash = "sha256:b3bda1d108d5dd99f4a20d24d9c348e91c4db7ab1b749200bded2f839ccbe68f"}, ] - -[[package]] -name = "tomli" -version = "2.0.1" -description = "A lil' TOML parser" -category = "dev" -optional = false -python-versions = ">=3.7" -files = [ +tomli = [ {file = "tomli-2.0.1-py3-none-any.whl", hash = "sha256:939de3e7a6161af0c887ef91b7d41a53e7c5a1ca976325f429cb46ea9bc30ecc"}, {file = "tomli-2.0.1.tar.gz", hash = "sha256:de526c12914f0c550d15924c62d72abc48d6fe7364aa87328337a31007fe8a4f"}, ] - -[[package]] -name = "types-toml" -version = "0.10.8.1" -description = "Typing stubs for toml" -category = "dev" -optional = false -python-versions = "*" -files = [ - {file = "types-toml-0.10.8.1.tar.gz", hash = "sha256:171bdb3163d79a520560f24ba916a9fc9bff81659c5448a9fea89240923722be"}, - {file = "types_toml-0.10.8.1-py3-none-any.whl", hash = "sha256:b7b5c4977f96ab7b5ac06d8a6590d17c0bf252a96efc03b109c2711fb3e0eafd"}, +types-toml = [ + {file = "types-toml-0.10.8.4.tar.gz", hash = "sha256:c8748dd225b28eb80ce712e2d7d61b57599815e7b48d07ef53df51ed148fa6b1"}, + {file = "types_toml-0.10.8.4-py3-none-any.whl", hash = "sha256:306b1bb8b5bbc5f1b60387dbcc4b489e79f8490ce20e93af5f422a68b470d94b"}, ] - -[[package]] -name = "typing-extensions" -version = "4.4.0" -description = "Backported and Experimental Type Hints for Python 3.7+" -category = "main" -optional = false -python-versions = ">=3.7" -files = [ +typing-extensions = [ {file = "typing_extensions-4.4.0-py3-none-any.whl", hash = "sha256:16fa4864408f655d35ec496218b85f79b3437c829e93320c7c9215ccfd92489e"}, {file = "typing_extensions-4.4.0.tar.gz", hash = "sha256:1511434bb92bf8dd198c12b1cc812e800d4181cfcb867674e0f8279cc93087aa"}, ] - -[[package]] -name = "urllib3" -version = "1.26.14" -description = "HTTP library with thread-safe connection pooling, file post, and more." -category = "main" -optional = false -python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*, !=3.5.*" -files = [ +urllib3 = [ {file = "urllib3-1.26.14-py2.py3-none-any.whl", hash = "sha256:75edcdc2f7d85b137124a6c3c9fc3933cdeaa12ecb9a6a959f22797a0feca7e1"}, {file = "urllib3-1.26.14.tar.gz", hash = "sha256:076907bf8fd355cde77728471316625a4d2f7e713c125f51953bb5b3eecf4f72"}, ] - -[package.extras] -brotli = ["brotli (>=1.0.9)", "brotlicffi (>=0.8.0)", "brotlipy (>=0.6.0)"] -secure = ["certifi", "cryptography (>=1.3.4)", "idna (>=2.0.0)", "ipaddress", "pyOpenSSL (>=0.14)", "urllib3-secure-extra"] -socks = ["PySocks (>=1.5.6,!=1.5.7,<2.0)"] - -[[package]] -name = "virtualenv" -version = "20.17.1" -description = "Virtual Python Environment builder" -category = "dev" -optional = false -python-versions = ">=3.6" -files = [ - {file = "virtualenv-20.17.1-py3-none-any.whl", hash = "sha256:ce3b1684d6e1a20a3e5ed36795a97dfc6af29bc3970ca8dab93e11ac6094b3c4"}, - {file = "virtualenv-20.17.1.tar.gz", hash = "sha256:f8b927684efc6f1cc206c9db297a570ab9ad0e51c16fa9e45487d36d1905c058"}, +virtualenv = [ + {file = "virtualenv-20.19.0-py3-none-any.whl", hash = "sha256:54eb59e7352b573aa04d53f80fc9736ed0ad5143af445a1e539aada6eb947dd1"}, + {file = "virtualenv-20.19.0.tar.gz", hash = "sha256:37a640ba82ed40b226599c522d411e4be5edb339a0c0de030c0dc7b646d61590"}, ] - -[package.dependencies] -distlib = ">=0.3.6,<1" -filelock = ">=3.4.1,<4" -platformdirs = ">=2.4,<3" - -[package.extras] -docs = ["proselint (>=0.13)", "sphinx (>=5.3)", "sphinx-argparse (>=0.3.2)", "sphinx-rtd-theme (>=1)", "towncrier (>=22.8)"] -testing = ["coverage (>=6.2)", "coverage-enable-subprocess (>=1)", "flaky (>=3.7)", "packaging (>=21.3)", "pytest (>=7.0.1)", "pytest-env (>=0.6.2)", "pytest-freezegun (>=0.4.2)", "pytest-mock (>=3.6.1)", "pytest-randomly (>=3.10.3)", "pytest-timeout (>=2.1)"] - -[[package]] -name = "werkzeug" -version = "2.2.3" -description = "The comprehensive WSGI web application library." -category = "dev" -optional = false -python-versions = ">=3.7" -files = [ +werkzeug = [ {file = "Werkzeug-2.2.3-py3-none-any.whl", hash = "sha256:56433961bc1f12533306c624f3be5e744389ac61d722175d543e1751285da612"}, {file = "Werkzeug-2.2.3.tar.gz", hash = "sha256:2e1ccc9417d4da358b9de6f174e3ac094391ea1d4fbef2d667865d819dfd0afe"}, ] - -[package.dependencies] -MarkupSafe = ">=2.1.1" - -[package.extras] -watchdog = ["watchdog"] - -[[package]] -name = "wrapt" -version = "1.14.1" -description = "Module for decorators, wrappers and monkey patching." -category = "main" -optional = true -python-versions = "!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,!=3.4.*,>=2.7" -files = [ +wrapt = [ {file = "wrapt-1.14.1-cp27-cp27m-macosx_10_9_x86_64.whl", hash = "sha256:1b376b3f4896e7930f1f772ac4b064ac12598d1c38d04907e696cc4d794b43d3"}, {file = "wrapt-1.14.1-cp27-cp27m-manylinux1_i686.whl", hash = "sha256:903500616422a40a98a5a3c4ff4ed9d0066f3b4c951fa286018ecdf0750194ef"}, {file = "wrapt-1.14.1-cp27-cp27m-manylinux1_x86_64.whl", hash = "sha256:5a9a0d155deafd9448baff28c08e150d9b24ff010e899311ddd63c45c2445e28"}, @@ -2275,27 +2372,11 @@ files = [ {file = "wrapt-1.14.1-cp39-cp39-win_amd64.whl", hash = "sha256:dee60e1de1898bde3b238f18340eec6148986da0455d8ba7848d50470a7a32fb"}, {file = "wrapt-1.14.1.tar.gz", hash = "sha256:380a85cf89e0e69b7cfbe2ea9f765f004ff419f34194018a6827ac0e3edfed4d"}, ] - -[[package]] -name = "xmltodict" -version = "0.13.0" -description = "Makes working with XML feel like you are working with JSON" -category = "dev" -optional = false -python-versions = ">=3.4" -files = [ +xmltodict = [ {file = "xmltodict-0.13.0-py2.py3-none-any.whl", hash = "sha256:aa89e8fd76320154a40d19a0df04a4695fb9dc5ba977cbb68ab3e4eb225e7852"}, {file = "xmltodict-0.13.0.tar.gz", hash = "sha256:341595a488e3e01a85a9d8911d8912fd922ede5fecc4dce437eb4b6c8d037e56"}, ] - -[[package]] -name = "yarl" -version = "1.8.2" -description = "Yet another URL library" -category = "main" -optional = true -python-versions = ">=3.7" -files = [ +yarl = [ {file = "yarl-1.8.2-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:bb81f753c815f6b8e2ddd2eef3c855cf7da193b82396ac013c661aaa6cc6b0a5"}, {file = "yarl-1.8.2-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:47d49ac96156f0928f002e2424299b2c91d9db73e08c4cd6742923a086f1c863"}, {file = "yarl-1.8.2-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:3fc056e35fa6fba63248d93ff6e672c096f95f7836938241ebc8260e062832fe"}, @@ -2371,35 +2452,11 @@ files = [ {file = "yarl-1.8.2-cp39-cp39-win_amd64.whl", hash = "sha256:6604711362f2dbf7160df21c416f81fac0de6dbcf0b5445a2ef25478ecc4c778"}, {file = "yarl-1.8.2.tar.gz", hash = "sha256:49d43402c6e3013ad0978602bf6bf5328535c48d192304b91b97a3c6790b1562"}, ] - -[package.dependencies] -idna = ">=2.0" -multidict = ">=4.0" - -[[package]] -name = "zipp" -version = "3.11.0" -description = "Backport of pathlib-compatible object wrapper for zip files" -category = "dev" -optional = false -python-versions = ">=3.7" -files = [ - {file = "zipp-3.11.0-py3-none-any.whl", hash = "sha256:83a28fcb75844b5c0cdaf5aa4003c2d728c77e05f5aeabe8e95e56727005fbaa"}, - {file = "zipp-3.11.0.tar.gz", hash = "sha256:a7a22e05929290a67401440b39690ae6563279bced5f314609d9d03798f56766"}, +zipp = [ + {file = "zipp-3.13.0-py3-none-any.whl", hash = "sha256:e8b2a36ea17df80ffe9e2c4fda3f693c3dad6df1697d3cd3af232db680950b0b"}, + {file = "zipp-3.13.0.tar.gz", hash = "sha256:23f70e964bc11a34cef175bc90ba2914e1e4545ea1e3e2f67c079671883f9cb6"}, ] - -[package.extras] -docs = ["furo", "jaraco.packaging (>=9)", "jaraco.tidelift (>=1.4)", "rst.linker (>=1.9)", "sphinx (>=3.5)"] -testing = ["flake8 (<5)", "func-timeout", "jaraco.functools", "jaraco.itertools", "more-itertools", "pytest (>=6)", "pytest-black (>=0.3.7)", "pytest-checkdocs (>=2.4)", "pytest-cov", "pytest-enabler (>=1.3)", "pytest-flake8", "pytest-mypy (>=0.9.1)"] - -[[package]] -name = "zstandard" -version = "0.19.0" -description = "Zstandard bindings for Python" -category = "main" -optional = false -python-versions = ">=3.6" -files = [ +zstandard = [ {file = "zstandard-0.19.0-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:a65e0119ad39e855427520f7829618f78eb2824aa05e63ff19b466080cd99210"}, {file = "zstandard-0.19.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:4fa496d2d674c6e9cffc561639d17009d29adee84a27cf1e12d3c9be14aa8feb"}, {file = "zstandard-0.19.0-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:8f7c68de4f362c1b2f426395fe4e05028c56d0782b2ec3ae18a5416eaf775576"}, @@ -2452,25 +2509,3 @@ files = [ {file = "zstandard-0.19.0-cp39-cp39-win_amd64.whl", hash = "sha256:55a513ec67e85abd8b8b83af8813368036f03e2d29a50fc94033504918273980"}, {file = "zstandard-0.19.0.tar.gz", hash = "sha256:31d12fcd942dd8dbf52ca5f6b1bbe287f44e5d551a081a983ff3ea2082867863"}, ] - -[package.dependencies] -cffi = {version = ">=1.11", markers = "platform_python_implementation == \"PyPy\""} - -[package.extras] -cffi = ["cffi (>=1.11)"] - -[extras] -adlfs = ["adlfs"] -duckdb = ["duckdb", "pyarrow"] -dynamodb = ["boto3"] -glue = ["boto3"] -hive = ["thrift"] -pandas = ["pandas", "pyarrow"] -pyarrow = ["pyarrow"] -s3fs = ["s3fs"] -snappy = ["python-snappy"] - -[metadata] -lock-version = "2.0" -python-versions = "^3.8" -content-hash = "4cdeb2c3ef66160ac86dc5016eee2cf482ecec825cbfd33f582cdbbb2a063bfb" diff --git a/pyproject.toml b/pyproject.toml index 92bbb1266c..555564f1a0 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -66,7 +66,7 @@ pyarrow = { version = ">=8.0.0,<=11.0.0", optional = true } pandas = { version = ">=1.4.4,<=1.5.3", optional = true } -duckdb = { version = ">=0.6.0,<=0.6.1", optional = true } +duckdb = { version = ">=0.6.0,<=0.7.0", optional = true } python-snappy = { version = "0.6.1", optional = true } From 5a9c96f7051801058aa288489f6e75d4e8299fe9 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Sun, 19 Feb 2023 22:11:32 +0100 Subject: [PATCH 377/642] Build: Bump pydantic from 1.10.4 to 1.10.5 in /python (#6881) Bumps [pydantic](https://github.com/pydantic/pydantic) from 1.10.4 to 1.10.5. - [Release notes](https://github.com/pydantic/pydantic/releases) - [Changelog](https://github.com/pydantic/pydantic/blob/v1.10.5/HISTORY.md) - [Commits](https://github.com/pydantic/pydantic/compare/v1.10.4...v1.10.5) --- updated-dependencies: - dependency-name: pydantic dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- poetry.lock | 2581 ++++++++++++++++++++++++------------------------ pyproject.toml | 2 +- 2 files changed, 1314 insertions(+), 1269 deletions(-) diff --git a/poetry.lock b/poetry.lock index def42af828..39e594f713 100644 --- a/poetry.lock +++ b/poetry.lock @@ -1,3 +1,5 @@ +# This file is automatically @generated by Poetry and should not be changed by hand. + [[package]] name = "adal" version = "1.2.7" @@ -5,6 +7,10 @@ description = "Note: This library is already replaced by MSAL Python, available category = "main" optional = true python-versions = "*" +files = [ + {file = "adal-1.2.7-py2.py3-none-any.whl", hash = "sha256:2a7451ed7441ddbc57703042204a3e30ef747478eea022c70f789fc7f084bc3d"}, + {file = "adal-1.2.7.tar.gz", hash = "sha256:d74f45b81317454d96e982fd1c50e6fb5c99ac2223728aea8764433a39f566f1"}, +] [package.dependencies] cryptography = ">=1.1.0" @@ -19,6 +25,10 @@ description = "Access Azure Datalake Gen1 with fsspec and dask" category = "main" optional = true python-versions = ">=3.8" +files = [ + {file = "adlfs-2023.1.0-py3-none-any.whl", hash = "sha256:ccbdd6d33d5b7bce99a4a07c884c82fb135745d39b2d9b7b672f8e9d4d04407f"}, + {file = "adlfs-2023.1.0.tar.gz", hash = "sha256:eca53f53d88fc8e2e7a2f1d8f5a40b8a1c56aa3b541e4aa2e7eaf55a6a789262"}, +] [package.dependencies] aiohttp = ">=3.7.0" @@ -38,6 +48,10 @@ description = "Async client for aws services using botocore and aiohttp" category = "main" optional = true python-versions = ">=3.7" +files = [ + {file = "aiobotocore-2.4.2-py3-none-any.whl", hash = "sha256:4acd1ebe2e44be4b100aa553910bda899f6dc090b3da2bc1cf3d5de2146ed208"}, + {file = "aiobotocore-2.4.2.tar.gz", hash = "sha256:0603b74a582dffa7511ce7548d07dc9b10ec87bc5fb657eb0b34f9bd490958bf"}, +] [package.dependencies] aiohttp = ">=3.3.1" @@ -56,6 +70,95 @@ description = "Async http client/server framework (asyncio)" category = "main" optional = true python-versions = ">=3.6" +files = [ + {file = "aiohttp-3.8.4-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:5ce45967538fb747370308d3145aa68a074bdecb4f3a300869590f725ced69c1"}, + {file = "aiohttp-3.8.4-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:b744c33b6f14ca26b7544e8d8aadff6b765a80ad6164fb1a430bbadd593dfb1a"}, + {file = "aiohttp-3.8.4-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:1a45865451439eb320784918617ba54b7a377e3501fb70402ab84d38c2cd891b"}, + {file = "aiohttp-3.8.4-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a86d42d7cba1cec432d47ab13b6637bee393a10f664c425ea7b305d1301ca1a3"}, + {file = "aiohttp-3.8.4-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:ee3c36df21b5714d49fc4580247947aa64bcbe2939d1b77b4c8dcb8f6c9faecc"}, + {file = "aiohttp-3.8.4-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:176a64b24c0935869d5bbc4c96e82f89f643bcdf08ec947701b9dbb3c956b7dd"}, + {file = "aiohttp-3.8.4-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:c844fd628851c0bc309f3c801b3a3d58ce430b2ce5b359cd918a5a76d0b20cb5"}, + {file = "aiohttp-3.8.4-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:5393fb786a9e23e4799fec788e7e735de18052f83682ce2dfcabaf1c00c2c08e"}, + {file = "aiohttp-3.8.4-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:e4b09863aae0dc965c3ef36500d891a3ff495a2ea9ae9171e4519963c12ceefd"}, + {file = "aiohttp-3.8.4-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:adfbc22e87365a6e564c804c58fc44ff7727deea782d175c33602737b7feadb6"}, + {file = "aiohttp-3.8.4-cp310-cp310-musllinux_1_1_ppc64le.whl", hash = "sha256:147ae376f14b55f4f3c2b118b95be50a369b89b38a971e80a17c3fd623f280c9"}, + {file = "aiohttp-3.8.4-cp310-cp310-musllinux_1_1_s390x.whl", hash = "sha256:eafb3e874816ebe2a92f5e155f17260034c8c341dad1df25672fb710627c6949"}, + {file = "aiohttp-3.8.4-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:c6cc15d58053c76eacac5fa9152d7d84b8d67b3fde92709195cb984cfb3475ea"}, + {file = "aiohttp-3.8.4-cp310-cp310-win32.whl", hash = "sha256:59f029a5f6e2d679296db7bee982bb3d20c088e52a2977e3175faf31d6fb75d1"}, + {file = "aiohttp-3.8.4-cp310-cp310-win_amd64.whl", hash = "sha256:fe7ba4a51f33ab275515f66b0a236bcde4fb5561498fe8f898d4e549b2e4509f"}, + {file = "aiohttp-3.8.4-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:3d8ef1a630519a26d6760bc695842579cb09e373c5f227a21b67dc3eb16cfea4"}, + {file = "aiohttp-3.8.4-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:5b3f2e06a512e94722886c0827bee9807c86a9f698fac6b3aee841fab49bbfb4"}, + {file = "aiohttp-3.8.4-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:3a80464982d41b1fbfe3154e440ba4904b71c1a53e9cd584098cd41efdb188ef"}, + {file = "aiohttp-3.8.4-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:8b631e26df63e52f7cce0cce6507b7a7f1bc9b0c501fcde69742130b32e8782f"}, + {file = "aiohttp-3.8.4-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:3f43255086fe25e36fd5ed8f2ee47477408a73ef00e804cb2b5cba4bf2ac7f5e"}, + {file = "aiohttp-3.8.4-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:4d347a172f866cd1d93126d9b239fcbe682acb39b48ee0873c73c933dd23bd0f"}, + {file = "aiohttp-3.8.4-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:a3fec6a4cb5551721cdd70473eb009d90935b4063acc5f40905d40ecfea23e05"}, + {file = "aiohttp-3.8.4-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:80a37fe8f7c1e6ce8f2d9c411676e4bc633a8462844e38f46156d07a7d401654"}, + {file = "aiohttp-3.8.4-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:d1e6a862b76f34395a985b3cd39a0d949ca80a70b6ebdea37d3ab39ceea6698a"}, + {file = "aiohttp-3.8.4-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:cd468460eefef601ece4428d3cf4562459157c0f6523db89365202c31b6daebb"}, + {file = "aiohttp-3.8.4-cp311-cp311-musllinux_1_1_ppc64le.whl", hash = "sha256:618c901dd3aad4ace71dfa0f5e82e88b46ef57e3239fc7027773cb6d4ed53531"}, + {file = "aiohttp-3.8.4-cp311-cp311-musllinux_1_1_s390x.whl", hash = "sha256:652b1bff4f15f6287550b4670546a2947f2a4575b6c6dff7760eafb22eacbf0b"}, + {file = "aiohttp-3.8.4-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:80575ba9377c5171407a06d0196b2310b679dc752d02a1fcaa2bc20b235dbf24"}, + {file = "aiohttp-3.8.4-cp311-cp311-win32.whl", hash = "sha256:bbcf1a76cf6f6dacf2c7f4d2ebd411438c275faa1dc0c68e46eb84eebd05dd7d"}, + {file = "aiohttp-3.8.4-cp311-cp311-win_amd64.whl", hash = "sha256:6e74dd54f7239fcffe07913ff8b964e28b712f09846e20de78676ce2a3dc0bfc"}, + {file = "aiohttp-3.8.4-cp36-cp36m-macosx_10_9_x86_64.whl", hash = "sha256:880e15bb6dad90549b43f796b391cfffd7af373f4646784795e20d92606b7a51"}, + {file = "aiohttp-3.8.4-cp36-cp36m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:bb96fa6b56bb536c42d6a4a87dfca570ff8e52de2d63cabebfd6fb67049c34b6"}, + {file = "aiohttp-3.8.4-cp36-cp36m-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:4a6cadebe132e90cefa77e45f2d2f1a4b2ce5c6b1bfc1656c1ddafcfe4ba8131"}, + {file = "aiohttp-3.8.4-cp36-cp36m-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:f352b62b45dff37b55ddd7b9c0c8672c4dd2eb9c0f9c11d395075a84e2c40f75"}, + {file = "aiohttp-3.8.4-cp36-cp36m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:7ab43061a0c81198d88f39aaf90dae9a7744620978f7ef3e3708339b8ed2ef01"}, + {file = "aiohttp-3.8.4-cp36-cp36m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:c9cb1565a7ad52e096a6988e2ee0397f72fe056dadf75d17fa6b5aebaea05622"}, + {file = "aiohttp-3.8.4-cp36-cp36m-musllinux_1_1_aarch64.whl", hash = "sha256:1b3ea7edd2d24538959c1c1abf97c744d879d4e541d38305f9bd7d9b10c9ec41"}, + {file = "aiohttp-3.8.4-cp36-cp36m-musllinux_1_1_i686.whl", hash = "sha256:7c7837fe8037e96b6dd5cfcf47263c1620a9d332a87ec06a6ca4564e56bd0f36"}, + {file = "aiohttp-3.8.4-cp36-cp36m-musllinux_1_1_ppc64le.whl", hash = "sha256:3b90467ebc3d9fa5b0f9b6489dfb2c304a1db7b9946fa92aa76a831b9d587e99"}, + {file = "aiohttp-3.8.4-cp36-cp36m-musllinux_1_1_s390x.whl", hash = "sha256:cab9401de3ea52b4b4c6971db5fb5c999bd4260898af972bf23de1c6b5dd9d71"}, + {file = "aiohttp-3.8.4-cp36-cp36m-musllinux_1_1_x86_64.whl", hash = "sha256:d1f9282c5f2b5e241034a009779e7b2a1aa045f667ff521e7948ea9b56e0c5ff"}, + {file = "aiohttp-3.8.4-cp36-cp36m-win32.whl", hash = "sha256:5e14f25765a578a0a634d5f0cd1e2c3f53964553a00347998dfdf96b8137f777"}, + {file = "aiohttp-3.8.4-cp36-cp36m-win_amd64.whl", hash = "sha256:4c745b109057e7e5f1848c689ee4fb3a016c8d4d92da52b312f8a509f83aa05e"}, + {file = "aiohttp-3.8.4-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:aede4df4eeb926c8fa70de46c340a1bc2c6079e1c40ccf7b0eae1313ffd33519"}, + {file = "aiohttp-3.8.4-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:4ddaae3f3d32fc2cb4c53fab020b69a05c8ab1f02e0e59665c6f7a0d3a5be54f"}, + {file = "aiohttp-3.8.4-cp37-cp37m-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:c4eb3b82ca349cf6fadcdc7abcc8b3a50ab74a62e9113ab7a8ebc268aad35bb9"}, + {file = "aiohttp-3.8.4-cp37-cp37m-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:9bcb89336efa095ea21b30f9e686763f2be4478f1b0a616969551982c4ee4c3b"}, + {file = "aiohttp-3.8.4-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:6c08e8ed6fa3d477e501ec9db169bfac8140e830aa372d77e4a43084d8dd91ab"}, + {file = "aiohttp-3.8.4-cp37-cp37m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:c6cd05ea06daca6ad6a4ca3ba7fe7dc5b5de063ff4daec6170ec0f9979f6c332"}, + {file = "aiohttp-3.8.4-cp37-cp37m-musllinux_1_1_aarch64.whl", hash = "sha256:b7a00a9ed8d6e725b55ef98b1b35c88013245f35f68b1b12c5cd4100dddac333"}, + {file = "aiohttp-3.8.4-cp37-cp37m-musllinux_1_1_i686.whl", hash = "sha256:de04b491d0e5007ee1b63a309956eaed959a49f5bb4e84b26c8f5d49de140fa9"}, + {file = "aiohttp-3.8.4-cp37-cp37m-musllinux_1_1_ppc64le.whl", hash = "sha256:40653609b3bf50611356e6b6554e3a331f6879fa7116f3959b20e3528783e699"}, + {file = "aiohttp-3.8.4-cp37-cp37m-musllinux_1_1_s390x.whl", hash = "sha256:dbf3a08a06b3f433013c143ebd72c15cac33d2914b8ea4bea7ac2c23578815d6"}, + {file = "aiohttp-3.8.4-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:854f422ac44af92bfe172d8e73229c270dc09b96535e8a548f99c84f82dde241"}, + {file = "aiohttp-3.8.4-cp37-cp37m-win32.whl", hash = "sha256:aeb29c84bb53a84b1a81c6c09d24cf33bb8432cc5c39979021cc0f98c1292a1a"}, + {file = "aiohttp-3.8.4-cp37-cp37m-win_amd64.whl", hash = "sha256:db3fc6120bce9f446d13b1b834ea5b15341ca9ff3f335e4a951a6ead31105480"}, + {file = "aiohttp-3.8.4-cp38-cp38-macosx_10_9_universal2.whl", hash = "sha256:fabb87dd8850ef0f7fe2b366d44b77d7e6fa2ea87861ab3844da99291e81e60f"}, + {file = "aiohttp-3.8.4-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:91f6d540163f90bbaef9387e65f18f73ffd7c79f5225ac3d3f61df7b0d01ad15"}, + {file = "aiohttp-3.8.4-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:d265f09a75a79a788237d7f9054f929ced2e69eb0bb79de3798c468d8a90f945"}, + {file = "aiohttp-3.8.4-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:3d89efa095ca7d442a6d0cbc755f9e08190ba40069b235c9886a8763b03785da"}, + {file = "aiohttp-3.8.4-cp38-cp38-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:4dac314662f4e2aa5009977b652d9b8db7121b46c38f2073bfeed9f4049732cd"}, + {file = "aiohttp-3.8.4-cp38-cp38-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:fe11310ae1e4cd560035598c3f29d86cef39a83d244c7466f95c27ae04850f10"}, + {file = "aiohttp-3.8.4-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:6ddb2a2026c3f6a68c3998a6c47ab6795e4127315d2e35a09997da21865757f8"}, + {file = "aiohttp-3.8.4-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:e75b89ac3bd27d2d043b234aa7b734c38ba1b0e43f07787130a0ecac1e12228a"}, + {file = "aiohttp-3.8.4-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:6e601588f2b502c93c30cd5a45bfc665faaf37bbe835b7cfd461753068232074"}, + {file = "aiohttp-3.8.4-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:a5d794d1ae64e7753e405ba58e08fcfa73e3fad93ef9b7e31112ef3c9a0efb52"}, + {file = "aiohttp-3.8.4-cp38-cp38-musllinux_1_1_ppc64le.whl", hash = "sha256:a1f4689c9a1462f3df0a1f7e797791cd6b124ddbee2b570d34e7f38ade0e2c71"}, + {file = "aiohttp-3.8.4-cp38-cp38-musllinux_1_1_s390x.whl", hash = "sha256:3032dcb1c35bc330134a5b8a5d4f68c1a87252dfc6e1262c65a7e30e62298275"}, + {file = "aiohttp-3.8.4-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:8189c56eb0ddbb95bfadb8f60ea1b22fcfa659396ea36f6adcc521213cd7b44d"}, + {file = "aiohttp-3.8.4-cp38-cp38-win32.whl", hash = "sha256:33587f26dcee66efb2fff3c177547bd0449ab7edf1b73a7f5dea1e38609a0c54"}, + {file = "aiohttp-3.8.4-cp38-cp38-win_amd64.whl", hash = "sha256:e595432ac259af2d4630008bf638873d69346372d38255774c0e286951e8b79f"}, + {file = "aiohttp-3.8.4-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:5a7bdf9e57126dc345b683c3632e8ba317c31d2a41acd5800c10640387d193ed"}, + {file = "aiohttp-3.8.4-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:22f6eab15b6db242499a16de87939a342f5a950ad0abaf1532038e2ce7d31567"}, + {file = "aiohttp-3.8.4-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:7235604476a76ef249bd64cb8274ed24ccf6995c4a8b51a237005ee7a57e8643"}, + {file = "aiohttp-3.8.4-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:ea9eb976ffdd79d0e893869cfe179a8f60f152d42cb64622fca418cd9b18dc2a"}, + {file = "aiohttp-3.8.4-cp39-cp39-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:92c0cea74a2a81c4c76b62ea1cac163ecb20fb3ba3a75c909b9fa71b4ad493cf"}, + {file = "aiohttp-3.8.4-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:493f5bc2f8307286b7799c6d899d388bbaa7dfa6c4caf4f97ef7521b9cb13719"}, + {file = "aiohttp-3.8.4-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:0a63f03189a6fa7c900226e3ef5ba4d3bd047e18f445e69adbd65af433add5a2"}, + {file = "aiohttp-3.8.4-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:10c8cefcff98fd9168cdd86c4da8b84baaa90bf2da2269c6161984e6737bf23e"}, + {file = "aiohttp-3.8.4-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:bca5f24726e2919de94f047739d0a4fc01372801a3672708260546aa2601bf57"}, + {file = "aiohttp-3.8.4-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:03baa76b730e4e15a45f81dfe29a8d910314143414e528737f8589ec60cf7391"}, + {file = "aiohttp-3.8.4-cp39-cp39-musllinux_1_1_ppc64le.whl", hash = "sha256:8c29c77cc57e40f84acef9bfb904373a4e89a4e8b74e71aa8075c021ec9078c2"}, + {file = "aiohttp-3.8.4-cp39-cp39-musllinux_1_1_s390x.whl", hash = "sha256:03543dcf98a6619254b409be2d22b51f21ec66272be4ebda7b04e6412e4b2e14"}, + {file = "aiohttp-3.8.4-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:17b79c2963db82086229012cff93ea55196ed31f6493bb1ccd2c62f1724324e4"}, + {file = "aiohttp-3.8.4-cp39-cp39-win32.whl", hash = "sha256:34ce9f93a4a68d1272d26030655dd1b58ff727b3ed2a33d80ec433561b03d67a"}, + {file = "aiohttp-3.8.4-cp39-cp39-win_amd64.whl", hash = "sha256:41a86a69bb63bb2fc3dc9ad5ea9f10f1c9c8e282b471931be0268ddd09430b04"}, + {file = "aiohttp-3.8.4.tar.gz", hash = "sha256:bf2e1a9162c1e441bf805a1fd166e249d574ca04e03b34f97e2928769e91ab5c"}, +] [package.dependencies] aiosignal = ">=1.1.2" @@ -67,7 +170,7 @@ multidict = ">=4.5,<7.0" yarl = ">=1.0,<2.0" [package.extras] -speedups = ["aiodns", "brotli", "cchardet"] +speedups = ["Brotli", "aiodns", "cchardet"] [[package]] name = "aioitertools" @@ -76,6 +179,10 @@ description = "itertools and builtins for AsyncIO and mixed iterables" category = "main" optional = true python-versions = ">=3.6" +files = [ + {file = "aioitertools-0.11.0-py3-none-any.whl", hash = "sha256:04b95e3dab25b449def24d7df809411c10e62aab0cbe31a50ca4e68748c43394"}, + {file = "aioitertools-0.11.0.tar.gz", hash = "sha256:42c68b8dd3a69c2bf7f2233bf7df4bb58b557bca5252ac02ed5187bbc67d6831"}, +] [package.dependencies] typing_extensions = {version = ">=4.0", markers = "python_version < \"3.10\""} @@ -87,6 +194,10 @@ description = "aiosignal: a list of registered asynchronous callbacks" category = "main" optional = true python-versions = ">=3.7" +files = [ + {file = "aiosignal-1.3.1-py3-none-any.whl", hash = "sha256:f8376fb07dd1e86a584e4fcdec80b36b7f81aac666ebc724e2c090300dd83b17"}, + {file = "aiosignal-1.3.1.tar.gz", hash = "sha256:54cd96e15e1649b75d6c87526a6ff0b6c1b0dd3459f43d9ca11d48c339b68cfc"}, +] [package.dependencies] frozenlist = ">=1.1.0" @@ -98,6 +209,10 @@ description = "Timeout context manager for asyncio programs" category = "main" optional = true python-versions = ">=3.6" +files = [ + {file = "async-timeout-4.0.2.tar.gz", hash = "sha256:2163e1640ddb52b7a8c80d0a67a08587e5d245cc9c553a74a847056bc2976b15"}, + {file = "async_timeout-4.0.2-py3-none-any.whl", hash = "sha256:8ca1e4fcf50d07413d66d1a5e416e42cfdf5851c981d679a09851a6853383b3c"}, +] [[package]] name = "attrs" @@ -106,14 +221,17 @@ description = "Classes Without Boilerplate" category = "main" optional = false python-versions = ">=3.6" +files = [ + {file = "attrs-22.2.0-py3-none-any.whl", hash = "sha256:29e95c7f6778868dbd49170f98f8818f78f3dc5e0e37c0b1f474e3561b240836"}, + {file = "attrs-22.2.0.tar.gz", hash = "sha256:c9227bfc2f01993c03f68db37d1d15c9690188323c067c641f1a35ca58185f99"}, +] [package.extras] -cov = ["attrs", "coverage-enable-subprocess", "coverage[toml] (>=5.3)"] -dev = ["attrs"] +cov = ["attrs[tests]", "coverage-enable-subprocess", "coverage[toml] (>=5.3)"] +dev = ["attrs[docs,tests]"] docs = ["furo", "myst-parser", "sphinx", "sphinx-notfound-page", "sphinxcontrib-towncrier", "towncrier", "zope.interface"] -tests = ["attrs", "zope.interface"] -tests-no-zope = ["cloudpickle", "hypothesis", "mypy (>=0.971,<0.990)", "pympler", "pytest (>=4.3.0)", "pytest-mypy-plugins", "pytest-xdist"] -tests_no_zope = ["cloudpickle", "hypothesis", "mypy (>=0.971,<0.990)", "pympler", "pytest (>=4.3.0)", "pytest-mypy-plugins", "pytest-xdist"] +tests = ["attrs[tests-no-zope]", "zope.interface"] +tests-no-zope = ["cloudpickle", "cloudpickle", "hypothesis", "hypothesis", "mypy (>=0.971,<0.990)", "mypy (>=0.971,<0.990)", "pympler", "pympler", "pytest (>=4.3.0)", "pytest (>=4.3.0)", "pytest-mypy-plugins", "pytest-mypy-plugins", "pytest-xdist[psutil]", "pytest-xdist[psutil]"] [[package]] name = "azure-core" @@ -122,6 +240,10 @@ description = "Microsoft Azure Core Library for Python" category = "main" optional = true python-versions = ">=3.7" +files = [ + {file = "azure-core-1.26.3.zip", hash = "sha256:acbd0daa9675ce88623da35c80d819cdafa91731dee6b2695c64d7ca9da82db4"}, + {file = "azure_core-1.26.3-py3-none-any.whl", hash = "sha256:f7bad0a7b4d800d6e73733ea7e13a616016221ac111ff9a344fe4cba41e51bbe"}, +] [package.dependencies] requests = ">=2.18.4" @@ -138,6 +260,10 @@ description = "Azure Data Lake Store Filesystem Client Library for Python" category = "main" optional = true python-versions = "*" +files = [ + {file = "azure-datalake-store-0.0.52.tar.gz", hash = "sha256:4198ddb32614d16d4502b43d5c9739f81432b7e0e4d75d30e05149fe6007fea2"}, + {file = "azure_datalake_store-0.0.52-py2.py3-none-any.whl", hash = "sha256:aaed72b9c856824aeab554f4dbe0ef2c6d0ff36700bdd8b93d8298793117c48e"}, +] [package.dependencies] adal = ">=0.4.2" @@ -151,6 +277,10 @@ description = "Microsoft Azure Identity Library for Python" category = "main" optional = true python-versions = ">=3.7" +files = [ + {file = "azure-identity-1.12.0.zip", hash = "sha256:7f9b1ae7d97ea7af3f38dd09305e19ab81a1e16ab66ea186b6579d85c1ca2347"}, + {file = "azure_identity-1.12.0-py3-none-any.whl", hash = "sha256:2a58ce4a209a013e37eaccfd5937570ab99e9118b3e1acf875eed3a85d541b92"}, +] [package.dependencies] azure-core = ">=1.11.0,<2.0.0" @@ -166,6 +296,10 @@ description = "Microsoft Azure Blob Storage Client Library for Python" category = "main" optional = true python-versions = ">=3.7" +files = [ + {file = "azure-storage-blob-12.14.1.zip", hash = "sha256:860d4d82985a4bfc7d3271e71275af330f54f330a754355435a7ba749ccde997"}, + {file = "azure_storage_blob-12.14.1-py3-none-any.whl", hash = "sha256:52b84658e8df7853a3cf1c563814655b5028b979b2a87905b92aa6bb30be240e"}, +] [package.dependencies] azure-core = ">=1.24.2,<2.0.0" @@ -179,6 +313,10 @@ description = "The AWS SDK for Python" category = "main" optional = false python-versions = ">= 3.7" +files = [ + {file = "boto3-1.24.59-py3-none-any.whl", hash = "sha256:34ab44146a2c4e7f4e72737f4b27e6eb5e0a7855c2f4599e3d9199b6a0a2d575"}, + {file = "boto3-1.24.59.tar.gz", hash = "sha256:a50b4323f9579cfe22fcf5531fbd40b567d4d74c1adce06aeb5c95fce2a6fb40"}, +] [package.dependencies] botocore = ">=1.27.59,<1.28.0" @@ -195,6 +333,10 @@ description = "Low-level, data-driven core of boto 3." category = "main" optional = false python-versions = ">= 3.7" +files = [ + {file = "botocore-1.27.59-py3-none-any.whl", hash = "sha256:69d756791fc024bda54f6c53f71ae34e695ee41bbbc1743d9179c4837a4929da"}, + {file = "botocore-1.27.59.tar.gz", hash = "sha256:eda4aed6ee719a745d1288eaf1beb12f6f6448ad1fa12f159405db14ba9c92cf"}, +] [package.dependencies] jmespath = ">=0.7.1,<2.0.0" @@ -211,6 +353,10 @@ description = "A simple, correct Python build frontend" category = "dev" optional = false python-versions = ">= 3.7" +files = [ + {file = "build-0.10.0-py3-none-any.whl", hash = "sha256:af266720050a66c893a6096a2f410989eeac74ff9a68ba194b3f6473e8e26171"}, + {file = "build-0.10.0.tar.gz", hash = "sha256:d5b71264afdb5951d6704482aac78de887c80691c52b88a9ad195983ca2c9269"}, +] [package.dependencies] colorama = {version = "*", markers = "os_name == \"nt\""} @@ -231,6 +377,10 @@ description = "Python package for providing Mozilla's CA Bundle." category = "main" optional = false python-versions = ">=3.6" +files = [ + {file = "certifi-2022.12.7-py3-none-any.whl", hash = "sha256:4ad3232f5e926d6718ec31cfc1fcadfde020920e278684144551c91769c7bc18"}, + {file = "certifi-2022.12.7.tar.gz", hash = "sha256:35824b4c3a97115964b408844d64aa14db1cc518f6562e8d7261699d1350a9e3"}, +] [[package]] name = "cffi" @@ -239,6 +389,72 @@ description = "Foreign Function Interface for Python calling C code." category = "main" optional = false python-versions = "*" +files = [ + {file = "cffi-1.15.1-cp27-cp27m-macosx_10_9_x86_64.whl", hash = "sha256:a66d3508133af6e8548451b25058d5812812ec3798c886bf38ed24a98216fab2"}, + {file = "cffi-1.15.1-cp27-cp27m-manylinux1_i686.whl", hash = "sha256:470c103ae716238bbe698d67ad020e1db9d9dba34fa5a899b5e21577e6d52ed2"}, + {file = "cffi-1.15.1-cp27-cp27m-manylinux1_x86_64.whl", hash = "sha256:9ad5db27f9cabae298d151c85cf2bad1d359a1b9c686a275df03385758e2f914"}, + {file = "cffi-1.15.1-cp27-cp27m-win32.whl", hash = "sha256:b3bbeb01c2b273cca1e1e0c5df57f12dce9a4dd331b4fa1635b8bec26350bde3"}, + {file = "cffi-1.15.1-cp27-cp27m-win_amd64.whl", hash = "sha256:e00b098126fd45523dd056d2efba6c5a63b71ffe9f2bbe1a4fe1716e1d0c331e"}, + {file = "cffi-1.15.1-cp27-cp27mu-manylinux1_i686.whl", hash = "sha256:d61f4695e6c866a23a21acab0509af1cdfd2c013cf256bbf5b6b5e2695827162"}, + {file = "cffi-1.15.1-cp27-cp27mu-manylinux1_x86_64.whl", hash = "sha256:ed9cb427ba5504c1dc15ede7d516b84757c3e3d7868ccc85121d9310d27eed0b"}, + {file = "cffi-1.15.1-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:39d39875251ca8f612b6f33e6b1195af86d1b3e60086068be9cc053aa4376e21"}, + {file = "cffi-1.15.1-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:285d29981935eb726a4399badae8f0ffdff4f5050eaa6d0cfc3f64b857b77185"}, + {file = "cffi-1.15.1-cp310-cp310-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:3eb6971dcff08619f8d91607cfc726518b6fa2a9eba42856be181c6d0d9515fd"}, + {file = "cffi-1.15.1-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:21157295583fe8943475029ed5abdcf71eb3911894724e360acff1d61c1d54bc"}, + {file = "cffi-1.15.1-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:5635bd9cb9731e6d4a1132a498dd34f764034a8ce60cef4f5319c0541159392f"}, + {file = "cffi-1.15.1-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:2012c72d854c2d03e45d06ae57f40d78e5770d252f195b93f581acf3ba44496e"}, + {file = "cffi-1.15.1-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:dd86c085fae2efd48ac91dd7ccffcfc0571387fe1193d33b6394db7ef31fe2a4"}, + {file = "cffi-1.15.1-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:fa6693661a4c91757f4412306191b6dc88c1703f780c8234035eac011922bc01"}, + {file = "cffi-1.15.1-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:59c0b02d0a6c384d453fece7566d1c7e6b7bae4fc5874ef2ef46d56776d61c9e"}, + {file = "cffi-1.15.1-cp310-cp310-win32.whl", hash = "sha256:cba9d6b9a7d64d4bd46167096fc9d2f835e25d7e4c121fb2ddfc6528fb0413b2"}, + {file = "cffi-1.15.1-cp310-cp310-win_amd64.whl", hash = "sha256:ce4bcc037df4fc5e3d184794f27bdaab018943698f4ca31630bc7f84a7b69c6d"}, + {file = "cffi-1.15.1-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:3d08afd128ddaa624a48cf2b859afef385b720bb4b43df214f85616922e6a5ac"}, + {file = "cffi-1.15.1-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:3799aecf2e17cf585d977b780ce79ff0dc9b78d799fc694221ce814c2c19db83"}, + {file = "cffi-1.15.1-cp311-cp311-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:a591fe9e525846e4d154205572a029f653ada1a78b93697f3b5a8f1f2bc055b9"}, + {file = "cffi-1.15.1-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:3548db281cd7d2561c9ad9984681c95f7b0e38881201e157833a2342c30d5e8c"}, + {file = "cffi-1.15.1-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:91fc98adde3d7881af9b59ed0294046f3806221863722ba7d8d120c575314325"}, + {file = "cffi-1.15.1-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:94411f22c3985acaec6f83c6df553f2dbe17b698cc7f8ae751ff2237d96b9e3c"}, + {file = "cffi-1.15.1-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:03425bdae262c76aad70202debd780501fabeaca237cdfddc008987c0e0f59ef"}, + {file = "cffi-1.15.1-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:cc4d65aeeaa04136a12677d3dd0b1c0c94dc43abac5860ab33cceb42b801c1e8"}, + {file = "cffi-1.15.1-cp311-cp311-win32.whl", hash = "sha256:a0f100c8912c114ff53e1202d0078b425bee3649ae34d7b070e9697f93c5d52d"}, + {file = "cffi-1.15.1-cp311-cp311-win_amd64.whl", hash = "sha256:04ed324bda3cda42b9b695d51bb7d54b680b9719cfab04227cdd1e04e5de3104"}, + {file = "cffi-1.15.1-cp36-cp36m-macosx_10_9_x86_64.whl", hash = "sha256:50a74364d85fd319352182ef59c5c790484a336f6db772c1a9231f1c3ed0cbd7"}, + {file = "cffi-1.15.1-cp36-cp36m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:e263d77ee3dd201c3a142934a086a4450861778baaeeb45db4591ef65550b0a6"}, + {file = "cffi-1.15.1-cp36-cp36m-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:cec7d9412a9102bdc577382c3929b337320c4c4c4849f2c5cdd14d7368c5562d"}, + {file = "cffi-1.15.1-cp36-cp36m-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:4289fc34b2f5316fbb762d75362931e351941fa95fa18789191b33fc4cf9504a"}, + {file = "cffi-1.15.1-cp36-cp36m-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:173379135477dc8cac4bc58f45db08ab45d228b3363adb7af79436135d028405"}, + {file = "cffi-1.15.1-cp36-cp36m-manylinux_2_5_x86_64.manylinux1_x86_64.whl", hash = "sha256:6975a3fac6bc83c4a65c9f9fcab9e47019a11d3d2cf7f3c0d03431bf145a941e"}, + {file = "cffi-1.15.1-cp36-cp36m-win32.whl", hash = "sha256:2470043b93ff09bf8fb1d46d1cb756ce6132c54826661a32d4e4d132e1977adf"}, + {file = "cffi-1.15.1-cp36-cp36m-win_amd64.whl", hash = "sha256:30d78fbc8ebf9c92c9b7823ee18eb92f2e6ef79b45ac84db507f52fbe3ec4497"}, + {file = "cffi-1.15.1-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:198caafb44239b60e252492445da556afafc7d1e3ab7a1fb3f0584ef6d742375"}, + {file = "cffi-1.15.1-cp37-cp37m-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:5ef34d190326c3b1f822a5b7a45f6c4535e2f47ed06fec77d3d799c450b2651e"}, + {file = "cffi-1.15.1-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:8102eaf27e1e448db915d08afa8b41d6c7ca7a04b7d73af6514df10a3e74bd82"}, + {file = "cffi-1.15.1-cp37-cp37m-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:5df2768244d19ab7f60546d0c7c63ce1581f7af8b5de3eb3004b9b6fc8a9f84b"}, + {file = "cffi-1.15.1-cp37-cp37m-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:a8c4917bd7ad33e8eb21e9a5bbba979b49d9a97acb3a803092cbc1133e20343c"}, + {file = "cffi-1.15.1-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:0e2642fe3142e4cc4af0799748233ad6da94c62a8bec3a6648bf8ee68b1c7426"}, + {file = "cffi-1.15.1-cp37-cp37m-win32.whl", hash = "sha256:e229a521186c75c8ad9490854fd8bbdd9a0c9aa3a524326b55be83b54d4e0ad9"}, + {file = "cffi-1.15.1-cp37-cp37m-win_amd64.whl", hash = "sha256:a0b71b1b8fbf2b96e41c4d990244165e2c9be83d54962a9a1d118fd8657d2045"}, + {file = "cffi-1.15.1-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:320dab6e7cb2eacdf0e658569d2575c4dad258c0fcc794f46215e1e39f90f2c3"}, + {file = "cffi-1.15.1-cp38-cp38-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:1e74c6b51a9ed6589199c787bf5f9875612ca4a8a0785fb2d4a84429badaf22a"}, + {file = "cffi-1.15.1-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a5c84c68147988265e60416b57fc83425a78058853509c1b0629c180094904a5"}, + {file = "cffi-1.15.1-cp38-cp38-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:3b926aa83d1edb5aa5b427b4053dc420ec295a08e40911296b9eb1b6170f6cca"}, + {file = "cffi-1.15.1-cp38-cp38-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:87c450779d0914f2861b8526e035c5e6da0a3199d8f1add1a665e1cbc6fc6d02"}, + {file = "cffi-1.15.1-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:4f2c9f67e9821cad2e5f480bc8d83b8742896f1242dba247911072d4fa94c192"}, + {file = "cffi-1.15.1-cp38-cp38-win32.whl", hash = "sha256:8b7ee99e510d7b66cdb6c593f21c043c248537a32e0bedf02e01e9553a172314"}, + {file = "cffi-1.15.1-cp38-cp38-win_amd64.whl", hash = "sha256:00a9ed42e88df81ffae7a8ab6d9356b371399b91dbdf0c3cb1e84c03a13aceb5"}, + {file = "cffi-1.15.1-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:54a2db7b78338edd780e7ef7f9f6c442500fb0d41a5a4ea24fff1c929d5af585"}, + {file = "cffi-1.15.1-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:fcd131dd944808b5bdb38e6f5b53013c5aa4f334c5cad0c72742f6eba4b73db0"}, + {file = "cffi-1.15.1-cp39-cp39-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:7473e861101c9e72452f9bf8acb984947aa1661a7704553a9f6e4baa5ba64415"}, + {file = "cffi-1.15.1-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:6c9a799e985904922a4d207a94eae35c78ebae90e128f0c4e521ce339396be9d"}, + {file = "cffi-1.15.1-cp39-cp39-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:3bcde07039e586f91b45c88f8583ea7cf7a0770df3a1649627bf598332cb6984"}, + {file = "cffi-1.15.1-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:33ab79603146aace82c2427da5ca6e58f2b3f2fb5da893ceac0c42218a40be35"}, + {file = "cffi-1.15.1-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:5d598b938678ebf3c67377cdd45e09d431369c3b1a5b331058c338e201f12b27"}, + {file = "cffi-1.15.1-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:db0fbb9c62743ce59a9ff687eb5f4afbe77e5e8403d6697f7446e5f609976f76"}, + {file = "cffi-1.15.1-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:98d85c6a2bef81588d9227dde12db8a7f47f639f4a17c9ae08e773aa9c697bf3"}, + {file = "cffi-1.15.1-cp39-cp39-win32.whl", hash = "sha256:40f4774f5a9d4f5e344f31a32b5096977b5d48560c5592e2f3d2c4374bd543ee"}, + {file = "cffi-1.15.1-cp39-cp39-win_amd64.whl", hash = "sha256:70df4e3b545a17496c9b3f41f5115e69a4f2e77e94e1d2a8e1070bc0c38c8a3c"}, + {file = "cffi-1.15.1.tar.gz", hash = "sha256:d400bfb9a37b1351253cb402671cea7e89bdecc294e8016a707f6d1d8ac934f9"}, +] [package.dependencies] pycparser = "*" @@ -250,6 +466,10 @@ description = "Validate configuration and produce human readable error messages. category = "dev" optional = false python-versions = ">=3.6.1" +files = [ + {file = "cfgv-3.3.1-py2.py3-none-any.whl", hash = "sha256:c6a0883f3917a037485059700b9e75da2464e6c27051014ad85ba6aaa5884426"}, + {file = "cfgv-3.3.1.tar.gz", hash = "sha256:f5a830efb9ce7a445376bb66ec94c638a9787422f96264c98edc6bdeed8ab736"}, +] [[package]] name = "charset-normalizer" @@ -258,1159 +478,7 @@ description = "The Real First Universal Charset Detector. Open, modern and activ category = "main" optional = false python-versions = "*" - -[[package]] -name = "click" -version = "8.1.3" -description = "Composable command line interface toolkit" -category = "main" -optional = false -python-versions = ">=3.7" - -[package.dependencies] -colorama = {version = "*", markers = "platform_system == \"Windows\""} - -[[package]] -name = "colorama" -version = "0.4.6" -description = "Cross-platform colored terminal text." -category = "main" -optional = false -python-versions = "!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,!=3.4.*,!=3.5.*,!=3.6.*,>=2.7" - -[[package]] -name = "coverage" -version = "7.1.0" -description = "Code coverage measurement for Python" -category = "dev" -optional = false -python-versions = ">=3.7" - -[package.dependencies] -tomli = {version = "*", optional = true, markers = "python_full_version <= \"3.11.0a6\" and extra == \"toml\""} - -[package.extras] -toml = ["tomli"] - -[[package]] -name = "cryptography" -version = "39.0.1" -description = "cryptography is a package which provides cryptographic recipes and primitives to Python developers." -category = "main" -optional = false -python-versions = ">=3.6" - -[package.dependencies] -cffi = ">=1.12" - -[package.extras] -docs = ["sphinx (>=5.3.0)", "sphinx-rtd-theme (>=1.1.1)"] -docstest = ["pyenchant (>=1.6.11)", "sphinxcontrib-spelling (>=4.0.1)", "twine (>=1.12.0)"] -pep8test = ["black", "check-manifest", "mypy", "ruff", "types-pytz", "types-requests"] -sdist = ["setuptools-rust (>=0.11.4)"] -ssh = ["bcrypt (>=3.1.5)"] -test = ["hypothesis (>=1.11.4,!=3.79.2)", "iso8601", "pretend", "pytest (>=6.2.0)", "pytest-benchmark", "pytest-cov", "pytest-shard (>=0.1.2)", "pytest-subtests", "pytest-xdist", "pytz"] -test-randomorder = ["pytest-randomly"] -tox = ["tox"] - -[[package]] -name = "distlib" -version = "0.3.6" -description = "Distribution utilities" -category = "dev" -optional = false -python-versions = "*" - -[[package]] -name = "docutils" -version = "0.19" -description = "Docutils -- Python Documentation Utilities" -category = "dev" -optional = false -python-versions = ">=3.7" - -[[package]] -name = "duckdb" -version = "0.7.0" -description = "DuckDB embedded database" -category = "main" -optional = true -python-versions = "*" - -[[package]] -name = "exceptiongroup" -version = "1.1.0" -description = "Backport of PEP 654 (exception groups)" -category = "dev" -optional = false -python-versions = ">=3.7" - -[package.extras] -test = ["pytest (>=6)"] - -[[package]] -name = "fastavro" -version = "1.7.1" -description = "Fast read/write of AVRO files" -category = "dev" -optional = false -python-versions = ">=3.7" - -[package.extras] -codecs = ["lz4", "python-snappy", "zstandard"] -lz4 = ["lz4"] -snappy = ["python-snappy"] -zstandard = ["zstandard"] - -[[package]] -name = "filelock" -version = "3.9.0" -description = "A platform independent file lock." -category = "dev" -optional = false -python-versions = ">=3.7" - -[package.extras] -docs = ["furo (>=2022.12.7)", "sphinx (>=5.3)", "sphinx-autodoc-typehints (>=1.19.5)"] -testing = ["covdefaults (>=2.2.2)", "coverage (>=7.0.1)", "pytest (>=7.2)", "pytest-cov (>=4)", "pytest-timeout (>=2.1)"] - -[[package]] -name = "frozenlist" -version = "1.3.3" -description = "A list-like structure which implements collections.abc.MutableSequence" -category = "main" -optional = true -python-versions = ">=3.7" - -[[package]] -name = "fsspec" -version = "2023.1.0" -description = "File-system specification" -category = "main" -optional = false -python-versions = ">=3.7" - -[package.extras] -abfs = ["adlfs"] -adl = ["adlfs"] -arrow = ["pyarrow (>=1)"] -dask = ["dask", "distributed"] -dropbox = ["dropbox", "dropboxdrivefs", "requests"] -entrypoints = ["importlib-metadata"] -fuse = ["fusepy"] -gcs = ["gcsfs"] -git = ["pygit2"] -github = ["requests"] -gs = ["gcsfs"] -gui = ["panel"] -hdfs = ["pyarrow (>=1)"] -http = ["aiohttp (!=4.0.0a0,!=4.0.0a1)", "requests"] -libarchive = ["libarchive-c"] -oci = ["ocifs"] -s3 = ["s3fs"] -sftp = ["paramiko"] -smb = ["smbprotocol"] -ssh = ["paramiko"] -tqdm = ["tqdm"] - -[[package]] -name = "identify" -version = "2.5.18" -description = "File identification library for Python" -category = "dev" -optional = false -python-versions = ">=3.7" - -[package.extras] -license = ["ukkonen"] - -[[package]] -name = "idna" -version = "3.4" -description = "Internationalized Domain Names in Applications (IDNA)" -category = "main" -optional = false -python-versions = ">=3.5" - -[[package]] -name = "importlib-metadata" -version = "6.0.0" -description = "Read metadata from Python packages" -category = "dev" -optional = false -python-versions = ">=3.7" - -[package.dependencies] -zipp = ">=0.5" - -[package.extras] -docs = ["furo", "jaraco.packaging (>=9)", "jaraco.tidelift (>=1.4)", "rst.linker (>=1.9)", "sphinx (>=3.5)", "sphinx-lint"] -perf = ["ipython"] -testing = ["flake8 (<5)", "flufl.flake8", "importlib-resources (>=1.3)", "packaging", "pyfakefs", "pytest (>=6)", "pytest-black (>=0.3.7)", "pytest-checkdocs (>=2.4)", "pytest-cov", "pytest-enabler (>=1.3)", "pytest-flake8", "pytest-mypy (>=0.9.1)", "pytest-perf (>=0.9.2)"] - -[[package]] -name = "iniconfig" -version = "2.0.0" -description = "brain-dead simple config-ini parsing" -category = "dev" -optional = false -python-versions = ">=3.7" - -[[package]] -name = "isodate" -version = "0.6.1" -description = "An ISO 8601 date/time/duration parser and formatter" -category = "main" -optional = true -python-versions = "*" - -[package.dependencies] -six = "*" - -[[package]] -name = "jinja2" -version = "3.1.2" -description = "A very fast and expressive template engine." -category = "dev" -optional = false -python-versions = ">=3.7" - -[package.dependencies] -MarkupSafe = ">=2.0" - -[package.extras] -i18n = ["Babel (>=2.7)"] - -[[package]] -name = "jmespath" -version = "1.0.1" -description = "JSON Matching Expressions" -category = "main" -optional = false -python-versions = ">=3.7" - -[[package]] -name = "markdown-it-py" -version = "2.1.0" -description = "Python port of markdown-it. Markdown parsing, done right!" -category = "main" -optional = false -python-versions = ">=3.7" - -[package.dependencies] -mdurl = ">=0.1,<1.0" - -[package.extras] -benchmarking = ["psutil", "pytest", "pytest-benchmark (>=3.2,<4.0)"] -code_style = ["pre-commit (==2.6)"] -compare = ["commonmark (>=0.9.1,<0.10.0)", "markdown (>=3.3.6,<3.4.0)", "mistletoe (>=0.8.1,<0.9.0)", "mistune (>=2.0.2,<2.1.0)", "panflute (>=2.1.3,<2.2.0)"] -linkify = ["linkify-it-py (>=1.0,<2.0)"] -plugins = ["mdit-py-plugins"] -profiling = ["gprof2dot"] -rtd = ["attrs", "myst-parser", "pyyaml", "sphinx", "sphinx-book-theme", "sphinx-copybutton", "sphinx-design"] -testing = ["coverage", "pytest", "pytest-cov", "pytest-regressions"] - -[[package]] -name = "markupsafe" -version = "2.1.2" -description = "Safely add untrusted strings to HTML/XML markup." -category = "dev" -optional = false -python-versions = ">=3.7" - -[[package]] -name = "mdurl" -version = "0.1.2" -description = "Markdown URL utilities" -category = "main" -optional = false -python-versions = ">=3.7" - -[[package]] -name = "mmhash3" -version = "3.0.1" -description = "Python wrapper for MurmurHash (MurmurHash3), a set of fast and robust hash functions." -category = "main" -optional = false -python-versions = "*" - -[[package]] -name = "moto" -version = "4.1.2" -description = "" -category = "dev" -optional = false -python-versions = ">=3.7" - -[package.dependencies] -boto3 = ">=1.9.201" -botocore = ">=1.12.201" -cryptography = ">=3.3.1" -Jinja2 = ">=2.10.1" -python-dateutil = ">=2.1,<3.0.0" -requests = ">=2.5" -responses = ">=0.13.0" -werkzeug = ">=0.5,<2.2.0 || >2.2.0,<2.2.1 || >2.2.1" -xmltodict = "*" - -[package.extras] -all = ["PyYAML (>=5.1)", "aws-xray-sdk (>=0.93,!=0.96)", "cfn-lint (>=0.40.0)", "docker (>=2.5.1)", "ecdsa (!=0.15)", "graphql-core", "jsondiff (>=1.1.2)", "openapi-spec-validator (>=0.2.8)", "pyparsing (>=3.0.7)", "python-jose[cryptography] (>=3.1.0,<4.0.0)", "setuptools", "sshpubkeys (>=3.1.0)"] -apigateway = ["PyYAML (>=5.1)", "ecdsa (!=0.15)", "openapi-spec-validator (>=0.2.8)", "python-jose[cryptography] (>=3.1.0,<4.0.0)"] -apigatewayv2 = ["PyYAML (>=5.1)"] -appsync = ["graphql-core"] -awslambda = ["docker (>=2.5.1)"] -batch = ["docker (>=2.5.1)"] -cloudformation = ["PyYAML (>=5.1)", "aws-xray-sdk (>=0.93,!=0.96)", "cfn-lint (>=0.40.0)", "docker (>=2.5.1)", "ecdsa (!=0.15)", "graphql-core", "jsondiff (>=1.1.2)", "openapi-spec-validator (>=0.2.8)", "pyparsing (>=3.0.7)", "python-jose[cryptography] (>=3.1.0,<4.0.0)", "setuptools", "sshpubkeys (>=3.1.0)"] -cognitoidp = ["ecdsa (!=0.15)", "python-jose[cryptography] (>=3.1.0,<4.0.0)"] -ds = ["sshpubkeys (>=3.1.0)"] -dynamodb = ["docker (>=2.5.1)"] -dynamodbstreams = ["docker (>=2.5.1)"] -ebs = ["sshpubkeys (>=3.1.0)"] -ec2 = ["sshpubkeys (>=3.1.0)"] -efs = ["sshpubkeys (>=3.1.0)"] -eks = ["sshpubkeys (>=3.1.0)"] -glue = ["pyparsing (>=3.0.7)"] -iotdata = ["jsondiff (>=1.1.2)"] -route53resolver = ["sshpubkeys (>=3.1.0)"] -s3 = ["PyYAML (>=5.1)"] -server = ["PyYAML (>=5.1)", "aws-xray-sdk (>=0.93,!=0.96)", "cfn-lint (>=0.40.0)", "docker (>=2.5.1)", "ecdsa (!=0.15)", "flask (!=2.2.0,!=2.2.1)", "flask-cors", "graphql-core", "jsondiff (>=1.1.2)", "openapi-spec-validator (>=0.2.8)", "pyparsing (>=3.0.7)", "python-jose[cryptography] (>=3.1.0,<4.0.0)", "setuptools", "sshpubkeys (>=3.1.0)"] -ssm = ["PyYAML (>=5.1)"] -xray = ["aws-xray-sdk (>=0.93,!=0.96)", "setuptools"] - -[[package]] -name = "msal" -version = "1.21.0" -description = "The Microsoft Authentication Library (MSAL) for Python library enables your app to access the Microsoft Cloud by supporting authentication of users with Microsoft Azure Active Directory accounts (AAD) and Microsoft Accounts (MSA) using industry standard OAuth2 and OpenID Connect." -category = "main" -optional = true -python-versions = "*" - -[package.dependencies] -cryptography = ">=0.6,<41" -PyJWT = {version = ">=1.0.0,<3", extras = ["crypto"]} -requests = ">=2.0.0,<3" - -[package.extras] -broker = ["pymsalruntime (>=0.13.2,<0.14)"] - -[[package]] -name = "msal-extensions" -version = "1.0.0" -description = "Microsoft Authentication Library extensions (MSAL EX) provides a persistence API that can save your data on disk, encrypted on Windows, macOS and Linux. Concurrent data access will be coordinated by a file lock mechanism." -category = "main" -optional = true -python-versions = "*" - -[package.dependencies] -msal = ">=0.4.1,<2.0.0" -portalocker = [ - {version = ">=1.0,<3", markers = "python_version >= \"3.5\" and platform_system != \"Windows\""}, - {version = ">=1.6,<3", markers = "python_version >= \"3.5\" and platform_system == \"Windows\""}, -] - -[[package]] -name = "msrest" -version = "0.7.1" -description = "AutoRest swagger generator Python client runtime." -category = "main" -optional = true -python-versions = ">=3.6" - -[package.dependencies] -azure-core = ">=1.24.0" -certifi = ">=2017.4.17" -isodate = ">=0.6.0" -requests = ">=2.16,<3.0" -requests-oauthlib = ">=0.5.0" - -[package.extras] -async = ["aiodns", "aiohttp (>=3.0)"] - -[[package]] -name = "multidict" -version = "6.0.4" -description = "multidict implementation" -category = "main" -optional = true -python-versions = ">=3.7" - -[[package]] -name = "nodeenv" -version = "1.7.0" -description = "Node.js virtual environment builder" -category = "dev" -optional = false -python-versions = ">=2.7,!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,!=3.4.*,!=3.5.*,!=3.6.*" - -[[package]] -name = "numpy" -version = "1.24.2" -description = "Fundamental package for array computing in Python" -category = "main" -optional = true -python-versions = ">=3.8" - -[[package]] -name = "oauthlib" -version = "3.2.2" -description = "A generic, spec-compliant, thorough implementation of the OAuth request-signing logic" -category = "main" -optional = true -python-versions = ">=3.6" - -[package.extras] -rsa = ["cryptography (>=3.0.0)"] -signals = ["blinker (>=1.4.0)"] -signedtoken = ["cryptography (>=3.0.0)", "pyjwt (>=2.0.0,<3)"] - -[[package]] -name = "packaging" -version = "23.0" -description = "Core utilities for Python packages" -category = "dev" -optional = false -python-versions = ">=3.7" - -[[package]] -name = "pandas" -version = "1.5.3" -description = "Powerful data structures for data analysis, time series, and statistics" -category = "main" -optional = true -python-versions = ">=3.8" - -[package.dependencies] -numpy = [ - {version = ">=1.20.3", markers = "python_version < \"3.10\""}, - {version = ">=1.21.0", markers = "python_version >= \"3.10\""}, - {version = ">=1.23.2", markers = "python_version >= \"3.11\""}, -] -python-dateutil = ">=2.8.1" -pytz = ">=2020.1" - -[package.extras] -test = ["hypothesis (>=5.5.3)", "pytest (>=6.0)", "pytest-xdist (>=1.31)"] - -[[package]] -name = "platformdirs" -version = "3.0.0" -description = "A small Python package for determining appropriate platform-specific dirs, e.g. a \"user data dir\"." -category = "dev" -optional = false -python-versions = ">=3.7" - -[package.extras] -docs = ["furo (>=2022.12.7)", "proselint (>=0.13)", "sphinx (>=6.1.3)", "sphinx-autodoc-typehints (>=1.22,!=1.23.4)"] -test = ["appdirs (==1.4.4)", "covdefaults (>=2.2.2)", "pytest (>=7.2.1)", "pytest-cov (>=4)", "pytest-mock (>=3.10)"] - -[[package]] -name = "pluggy" -version = "1.0.0" -description = "plugin and hook calling mechanisms for python" -category = "dev" -optional = false -python-versions = ">=3.6" - -[package.extras] -dev = ["pre-commit", "tox"] -testing = ["pytest", "pytest-benchmark"] - -[[package]] -name = "portalocker" -version = "2.7.0" -description = "Wraps the portalocker recipe for easy usage" -category = "main" -optional = true -python-versions = ">=3.5" - -[package.dependencies] -pywin32 = {version = ">=226", markers = "platform_system == \"Windows\""} - -[package.extras] -docs = ["sphinx (>=1.7.1)"] -redis = ["redis"] -tests = ["pytest (>=5.4.1)", "pytest-cov (>=2.8.1)", "pytest-mypy (>=0.8.0)", "pytest-timeout (>=2.1.0)", "redis", "sphinx (>=6.0.0)"] - -[[package]] -name = "pre-commit" -version = "3.0.4" -description = "A framework for managing and maintaining multi-language pre-commit hooks." -category = "dev" -optional = false -python-versions = ">=3.8" - -[package.dependencies] -cfgv = ">=2.0.0" -identify = ">=1.0.0" -nodeenv = ">=0.11.1" -pyyaml = ">=5.1" -virtualenv = ">=20.10.0" - -[[package]] -name = "pyarrow" -version = "11.0.0" -description = "Python library for Apache Arrow" -category = "main" -optional = true -python-versions = ">=3.7" - -[package.dependencies] -numpy = ">=1.16.6" - -[[package]] -name = "pycparser" -version = "2.21" -description = "C parser in Python" -category = "main" -optional = false -python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*" - -[[package]] -name = "pydantic" -version = "1.10.4" -description = "Data validation and settings management using python type hints" -category = "main" -optional = false -python-versions = ">=3.7" - -[package.dependencies] -typing-extensions = ">=4.2.0" - -[package.extras] -dotenv = ["python-dotenv (>=0.10.4)"] -email = ["email-validator (>=1.0.3)"] - -[[package]] -name = "pygments" -version = "2.14.0" -description = "Pygments is a syntax highlighting package written in Python." -category = "main" -optional = false -python-versions = ">=3.6" - -[package.extras] -plugins = ["importlib-metadata"] - -[[package]] -name = "pyjwt" -version = "2.6.0" -description = "JSON Web Token implementation in Python" -category = "main" -optional = true -python-versions = ">=3.7" - -[package.dependencies] -cryptography = {version = ">=3.4.0", optional = true, markers = "extra == \"crypto\""} - -[package.extras] -crypto = ["cryptography (>=3.4.0)"] -dev = ["coverage[toml] (==5.0.4)", "cryptography (>=3.4.0)", "pre-commit", "pytest (>=6.0.0,<7.0.0)", "sphinx (>=4.5.0,<5.0.0)", "sphinx-rtd-theme", "zope.interface"] -docs = ["sphinx (>=4.5.0,<5.0.0)", "sphinx-rtd-theme", "zope.interface"] -tests = ["coverage[toml] (==5.0.4)", "pytest (>=6.0.0,<7.0.0)"] - -[[package]] -name = "pyparsing" -version = "3.0.9" -description = "pyparsing module - Classes and methods to define and execute parsing grammars" -category = "main" -optional = false -python-versions = ">=3.6.8" - -[package.extras] -diagrams = ["jinja2", "railroad-diagrams"] - -[[package]] -name = "pyproject-hooks" -version = "1.0.0" -description = "Wrappers to call pyproject.toml-based build backend hooks." -category = "dev" -optional = false -python-versions = ">=3.7" - -[package.dependencies] -tomli = {version = ">=1.1.0", markers = "python_version < \"3.11\""} - -[[package]] -name = "pytest" -version = "7.2.1" -description = "pytest: simple powerful testing with Python" -category = "dev" -optional = false -python-versions = ">=3.7" - -[package.dependencies] -attrs = ">=19.2.0" -colorama = {version = "*", markers = "sys_platform == \"win32\""} -exceptiongroup = {version = ">=1.0.0rc8", markers = "python_version < \"3.11\""} -iniconfig = "*" -packaging = "*" -pluggy = ">=0.12,<2.0" -tomli = {version = ">=1.0.0", markers = "python_version < \"3.11\""} - -[package.extras] -testing = ["argcomplete", "hypothesis (>=3.56)", "mock", "nose", "pygments (>=2.7.2)", "requests", "xmlschema"] - -[[package]] -name = "pytest-checkdocs" -version = "2.9.0" -description = "check the README when running tests" -category = "dev" -optional = false -python-versions = ">=3.7" - -[package.dependencies] -build = "*" -docutils = ">=0.15" -importlib-metadata = {version = ">=4", markers = "python_version < \"3.10\""} - -[package.extras] -docs = ["furo", "jaraco.packaging (>=9)", "rst.linker (>=1.9)", "sphinx (>=3.5)"] -testing = ["flake8 (<5)", "pytest (>=6)", "pytest-black (>=0.3.7)", "pytest-checkdocs (>=2.4)", "pytest-cov", "pytest-enabler (>=1.3)", "pytest-flake8", "pytest-mypy (>=0.9.1)", "types-docutils"] - -[[package]] -name = "python-dateutil" -version = "2.8.2" -description = "Extensions to the standard Python datetime module" -category = "main" -optional = false -python-versions = "!=3.0.*,!=3.1.*,!=3.2.*,>=2.7" - -[package.dependencies] -six = ">=1.5" - -[[package]] -name = "python-snappy" -version = "0.6.1" -description = "Python library for the snappy compression library from Google" -category = "main" -optional = true -python-versions = "*" - -[[package]] -name = "pytz" -version = "2022.7.1" -description = "World timezone definitions, modern and historical" -category = "main" -optional = true -python-versions = "*" - -[[package]] -name = "pywin32" -version = "305" -description = "Python for Window Extensions" -category = "main" -optional = true -python-versions = "*" - -[[package]] -name = "pyyaml" -version = "6.0" -description = "YAML parser and emitter for Python" -category = "main" -optional = false -python-versions = ">=3.6" - -[[package]] -name = "requests" -version = "2.28.2" -description = "Python HTTP for Humans." -category = "main" -optional = false -python-versions = ">=3.7, <4" - -[package.dependencies] -certifi = ">=2017.4.17" -charset-normalizer = ">=2,<4" -idna = ">=2.5,<4" -urllib3 = ">=1.21.1,<1.27" - -[package.extras] -socks = ["PySocks (>=1.5.6,!=1.5.7)"] -use_chardet_on_py3 = ["chardet (>=3.0.2,<6)"] - -[[package]] -name = "requests-mock" -version = "1.10.0" -description = "Mock out responses from the requests package" -category = "dev" -optional = false -python-versions = "*" - -[package.dependencies] -requests = ">=2.3,<3" -six = "*" - -[package.extras] -fixture = ["fixtures"] -test = ["fixtures", "mock", "purl", "pytest", "requests-futures", "sphinx", "testrepository (>=0.0.18)", "testtools"] - -[[package]] -name = "requests-oauthlib" -version = "1.3.1" -description = "OAuthlib authentication support for Requests." -category = "main" -optional = true -python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*" - -[package.dependencies] -oauthlib = ">=3.0.0" -requests = ">=2.0.0" - -[package.extras] -rsa = ["oauthlib[signedtoken] (>=3.0.0)"] - -[[package]] -name = "responses" -version = "0.22.0" -description = "A utility library for mocking out the `requests` Python library." -category = "dev" -optional = false -python-versions = ">=3.7" - -[package.dependencies] -requests = ">=2.22.0,<3.0" -toml = "*" -types-toml = "*" -urllib3 = ">=1.25.10" - -[package.extras] -tests = ["coverage (>=6.0.0)", "flake8", "mypy", "pytest (>=7.0.0)", "pytest-asyncio", "pytest-cov", "pytest-httpserver", "types-requests"] - -[[package]] -name = "rich" -version = "13.3.1" -description = "Render rich text, tables, progress bars, syntax highlighting, markdown and more to the terminal" -category = "main" -optional = false -python-versions = ">=3.7.0" - -[package.dependencies] -markdown-it-py = ">=2.1.0,<3.0.0" -pygments = ">=2.14.0,<3.0.0" -typing-extensions = {version = ">=4.0.0,<5.0", markers = "python_version < \"3.9\""} - -[package.extras] -jupyter = ["ipywidgets (>=7.5.1,<9)"] - -[[package]] -name = "s3fs" -version = "2023.1.0" -description = "Convenient Filesystem interface over S3" -category = "main" -optional = true -python-versions = ">= 3.7" - -[package.dependencies] -aiobotocore = ">=2.4.2,<2.5.0" -aiohttp = "<4.0.0a0 || >4.0.0a0,<4.0.0a1 || >4.0.0a1" -fsspec = "2023.1.0" - -[package.extras] -awscli = ["aiobotocore[awscli] (>=2.4.2,<2.5.0)"] -boto3 = ["aiobotocore[boto3] (>=2.4.2,<2.5.0)"] - -[[package]] -name = "s3transfer" -version = "0.6.0" -description = "An Amazon S3 Transfer Manager" -category = "main" -optional = false -python-versions = ">= 3.7" - -[package.dependencies] -botocore = ">=1.12.36,<2.0a.0" - -[package.extras] -crt = ["botocore[crt] (>=1.20.29,<2.0a.0)"] - -[[package]] -name = "six" -version = "1.16.0" -description = "Python 2 and 3 compatibility utilities" -category = "main" -optional = false -python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*" - -[[package]] -name = "thrift" -version = "0.16.0" -description = "Python bindings for the Apache Thrift RPC system" -category = "main" -optional = true -python-versions = "*" - -[package.dependencies] -six = ">=1.7.2" - -[package.extras] -all = ["tornado (>=4.0)", "twisted"] -tornado = ["tornado (>=4.0)"] -twisted = ["twisted"] - -[[package]] -name = "toml" -version = "0.10.2" -description = "Python Library for Tom's Obvious, Minimal Language" -category = "dev" -optional = false -python-versions = ">=2.6, !=3.0.*, !=3.1.*, !=3.2.*" - -[[package]] -name = "tomli" -version = "2.0.1" -description = "A lil' TOML parser" -category = "dev" -optional = false -python-versions = ">=3.7" - -[[package]] -name = "types-toml" -version = "0.10.8.4" -description = "Typing stubs for toml" -category = "dev" -optional = false -python-versions = "*" - -[[package]] -name = "typing-extensions" -version = "4.4.0" -description = "Backported and Experimental Type Hints for Python 3.7+" -category = "main" -optional = false -python-versions = ">=3.7" - -[[package]] -name = "urllib3" -version = "1.26.14" -description = "HTTP library with thread-safe connection pooling, file post, and more." -category = "main" -optional = false -python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*, !=3.5.*" - -[package.extras] -brotli = ["brotli (>=1.0.9)", "brotlicffi (>=0.8.0)", "brotlipy (>=0.6.0)"] -secure = ["certifi", "cryptography (>=1.3.4)", "idna (>=2.0.0)", "ipaddress", "pyOpenSSL (>=0.14)", "urllib3-secure-extra"] -socks = ["PySocks (>=1.5.6,!=1.5.7,<2.0)"] - -[[package]] -name = "virtualenv" -version = "20.19.0" -description = "Virtual Python Environment builder" -category = "dev" -optional = false -python-versions = ">=3.7" - -[package.dependencies] -distlib = ">=0.3.6,<1" -filelock = ">=3.4.1,<4" -platformdirs = ">=2.4,<4" - -[package.extras] -docs = ["furo (>=2022.12.7)", "proselint (>=0.13)", "sphinx (>=6.1.3)", "sphinx-argparse (>=0.4)", "sphinxcontrib-towncrier (>=0.2.1a0)", "towncrier (>=22.12)"] -test = ["covdefaults (>=2.2.2)", "coverage (>=7.1)", "coverage-enable-subprocess (>=1)", "flaky (>=3.7)", "packaging (>=23)", "pytest (>=7.2.1)", "pytest-env (>=0.8.1)", "pytest-freezegun (>=0.4.2)", "pytest-mock (>=3.10)", "pytest-randomly (>=3.12)", "pytest-timeout (>=2.1)"] - -[[package]] -name = "werkzeug" -version = "2.2.3" -description = "The comprehensive WSGI web application library." -category = "dev" -optional = false -python-versions = ">=3.7" - -[package.dependencies] -MarkupSafe = ">=2.1.1" - -[package.extras] -watchdog = ["watchdog"] - -[[package]] -name = "wrapt" -version = "1.14.1" -description = "Module for decorators, wrappers and monkey patching." -category = "main" -optional = true -python-versions = "!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,!=3.4.*,>=2.7" - -[[package]] -name = "xmltodict" -version = "0.13.0" -description = "Makes working with XML feel like you are working with JSON" -category = "dev" -optional = false -python-versions = ">=3.4" - -[[package]] -name = "yarl" -version = "1.8.2" -description = "Yet another URL library" -category = "main" -optional = true -python-versions = ">=3.7" - -[package.dependencies] -idna = ">=2.0" -multidict = ">=4.0" - -[[package]] -name = "zipp" -version = "3.13.0" -description = "Backport of pathlib-compatible object wrapper for zip files" -category = "dev" -optional = false -python-versions = ">=3.7" - -[package.extras] -docs = ["furo", "jaraco.packaging (>=9)", "jaraco.tidelift (>=1.4)", "rst.linker (>=1.9)", "sphinx (>=3.5)", "sphinx-lint"] -testing = ["flake8 (<5)", "func-timeout", "jaraco.functools", "jaraco.itertools", "more-itertools", "pytest (>=6)", "pytest-black (>=0.3.7)", "pytest-checkdocs (>=2.4)", "pytest-cov", "pytest-enabler (>=1.3)", "pytest-flake8", "pytest-mypy (>=0.9.1)"] - -[[package]] -name = "zstandard" -version = "0.19.0" -description = "Zstandard bindings for Python" -category = "main" -optional = false -python-versions = ">=3.6" - -[package.dependencies] -cffi = {version = ">=1.11", markers = "platform_python_implementation == \"PyPy\""} - -[package.extras] -cffi = ["cffi (>=1.11)"] - -[extras] -adlfs = ["adlfs"] -duckdb = ["duckdb", "pyarrow"] -dynamodb = ["boto3"] -glue = ["boto3"] -hive = ["thrift"] -pandas = ["pandas", "pyarrow"] -pyarrow = ["pyarrow"] -s3fs = ["s3fs"] -snappy = ["python-snappy"] - -[metadata] -lock-version = "1.1" -python-versions = "^3.8" -content-hash = "f2c5be6a05f8b56a16a6aee60c46aceeff565711cf5450c83bdebc88cb35b89f" - -[metadata.files] -adal = [ - {file = "adal-1.2.7-py2.py3-none-any.whl", hash = "sha256:2a7451ed7441ddbc57703042204a3e30ef747478eea022c70f789fc7f084bc3d"}, - {file = "adal-1.2.7.tar.gz", hash = "sha256:d74f45b81317454d96e982fd1c50e6fb5c99ac2223728aea8764433a39f566f1"}, -] -adlfs = [ - {file = "adlfs-2023.1.0-py3-none-any.whl", hash = "sha256:ccbdd6d33d5b7bce99a4a07c884c82fb135745d39b2d9b7b672f8e9d4d04407f"}, - {file = "adlfs-2023.1.0.tar.gz", hash = "sha256:eca53f53d88fc8e2e7a2f1d8f5a40b8a1c56aa3b541e4aa2e7eaf55a6a789262"}, -] -aiobotocore = [ - {file = "aiobotocore-2.4.2-py3-none-any.whl", hash = "sha256:4acd1ebe2e44be4b100aa553910bda899f6dc090b3da2bc1cf3d5de2146ed208"}, - {file = "aiobotocore-2.4.2.tar.gz", hash = "sha256:0603b74a582dffa7511ce7548d07dc9b10ec87bc5fb657eb0b34f9bd490958bf"}, -] -aiohttp = [ - {file = "aiohttp-3.8.4-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:5ce45967538fb747370308d3145aa68a074bdecb4f3a300869590f725ced69c1"}, - {file = "aiohttp-3.8.4-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:b744c33b6f14ca26b7544e8d8aadff6b765a80ad6164fb1a430bbadd593dfb1a"}, - {file = "aiohttp-3.8.4-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:1a45865451439eb320784918617ba54b7a377e3501fb70402ab84d38c2cd891b"}, - {file = "aiohttp-3.8.4-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a86d42d7cba1cec432d47ab13b6637bee393a10f664c425ea7b305d1301ca1a3"}, - {file = "aiohttp-3.8.4-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:ee3c36df21b5714d49fc4580247947aa64bcbe2939d1b77b4c8dcb8f6c9faecc"}, - {file = "aiohttp-3.8.4-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:176a64b24c0935869d5bbc4c96e82f89f643bcdf08ec947701b9dbb3c956b7dd"}, - {file = "aiohttp-3.8.4-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:c844fd628851c0bc309f3c801b3a3d58ce430b2ce5b359cd918a5a76d0b20cb5"}, - {file = "aiohttp-3.8.4-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:5393fb786a9e23e4799fec788e7e735de18052f83682ce2dfcabaf1c00c2c08e"}, - {file = "aiohttp-3.8.4-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:e4b09863aae0dc965c3ef36500d891a3ff495a2ea9ae9171e4519963c12ceefd"}, - {file = "aiohttp-3.8.4-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:adfbc22e87365a6e564c804c58fc44ff7727deea782d175c33602737b7feadb6"}, - {file = "aiohttp-3.8.4-cp310-cp310-musllinux_1_1_ppc64le.whl", hash = "sha256:147ae376f14b55f4f3c2b118b95be50a369b89b38a971e80a17c3fd623f280c9"}, - {file = "aiohttp-3.8.4-cp310-cp310-musllinux_1_1_s390x.whl", hash = "sha256:eafb3e874816ebe2a92f5e155f17260034c8c341dad1df25672fb710627c6949"}, - {file = "aiohttp-3.8.4-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:c6cc15d58053c76eacac5fa9152d7d84b8d67b3fde92709195cb984cfb3475ea"}, - {file = "aiohttp-3.8.4-cp310-cp310-win32.whl", hash = "sha256:59f029a5f6e2d679296db7bee982bb3d20c088e52a2977e3175faf31d6fb75d1"}, - {file = "aiohttp-3.8.4-cp310-cp310-win_amd64.whl", hash = "sha256:fe7ba4a51f33ab275515f66b0a236bcde4fb5561498fe8f898d4e549b2e4509f"}, - {file = "aiohttp-3.8.4-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:3d8ef1a630519a26d6760bc695842579cb09e373c5f227a21b67dc3eb16cfea4"}, - {file = "aiohttp-3.8.4-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:5b3f2e06a512e94722886c0827bee9807c86a9f698fac6b3aee841fab49bbfb4"}, - {file = "aiohttp-3.8.4-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:3a80464982d41b1fbfe3154e440ba4904b71c1a53e9cd584098cd41efdb188ef"}, - {file = "aiohttp-3.8.4-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:8b631e26df63e52f7cce0cce6507b7a7f1bc9b0c501fcde69742130b32e8782f"}, - {file = "aiohttp-3.8.4-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:3f43255086fe25e36fd5ed8f2ee47477408a73ef00e804cb2b5cba4bf2ac7f5e"}, - {file = "aiohttp-3.8.4-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:4d347a172f866cd1d93126d9b239fcbe682acb39b48ee0873c73c933dd23bd0f"}, - {file = "aiohttp-3.8.4-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:a3fec6a4cb5551721cdd70473eb009d90935b4063acc5f40905d40ecfea23e05"}, - {file = "aiohttp-3.8.4-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:80a37fe8f7c1e6ce8f2d9c411676e4bc633a8462844e38f46156d07a7d401654"}, - {file = "aiohttp-3.8.4-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:d1e6a862b76f34395a985b3cd39a0d949ca80a70b6ebdea37d3ab39ceea6698a"}, - {file = "aiohttp-3.8.4-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:cd468460eefef601ece4428d3cf4562459157c0f6523db89365202c31b6daebb"}, - {file = "aiohttp-3.8.4-cp311-cp311-musllinux_1_1_ppc64le.whl", hash = "sha256:618c901dd3aad4ace71dfa0f5e82e88b46ef57e3239fc7027773cb6d4ed53531"}, - {file = "aiohttp-3.8.4-cp311-cp311-musllinux_1_1_s390x.whl", hash = "sha256:652b1bff4f15f6287550b4670546a2947f2a4575b6c6dff7760eafb22eacbf0b"}, - {file = "aiohttp-3.8.4-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:80575ba9377c5171407a06d0196b2310b679dc752d02a1fcaa2bc20b235dbf24"}, - {file = "aiohttp-3.8.4-cp311-cp311-win32.whl", hash = "sha256:bbcf1a76cf6f6dacf2c7f4d2ebd411438c275faa1dc0c68e46eb84eebd05dd7d"}, - {file = "aiohttp-3.8.4-cp311-cp311-win_amd64.whl", hash = "sha256:6e74dd54f7239fcffe07913ff8b964e28b712f09846e20de78676ce2a3dc0bfc"}, - {file = "aiohttp-3.8.4-cp36-cp36m-macosx_10_9_x86_64.whl", hash = "sha256:880e15bb6dad90549b43f796b391cfffd7af373f4646784795e20d92606b7a51"}, - {file = "aiohttp-3.8.4-cp36-cp36m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:bb96fa6b56bb536c42d6a4a87dfca570ff8e52de2d63cabebfd6fb67049c34b6"}, - {file = "aiohttp-3.8.4-cp36-cp36m-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:4a6cadebe132e90cefa77e45f2d2f1a4b2ce5c6b1bfc1656c1ddafcfe4ba8131"}, - {file = "aiohttp-3.8.4-cp36-cp36m-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:f352b62b45dff37b55ddd7b9c0c8672c4dd2eb9c0f9c11d395075a84e2c40f75"}, - {file = "aiohttp-3.8.4-cp36-cp36m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:7ab43061a0c81198d88f39aaf90dae9a7744620978f7ef3e3708339b8ed2ef01"}, - {file = "aiohttp-3.8.4-cp36-cp36m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:c9cb1565a7ad52e096a6988e2ee0397f72fe056dadf75d17fa6b5aebaea05622"}, - {file = "aiohttp-3.8.4-cp36-cp36m-musllinux_1_1_aarch64.whl", hash = "sha256:1b3ea7edd2d24538959c1c1abf97c744d879d4e541d38305f9bd7d9b10c9ec41"}, - {file = "aiohttp-3.8.4-cp36-cp36m-musllinux_1_1_i686.whl", hash = "sha256:7c7837fe8037e96b6dd5cfcf47263c1620a9d332a87ec06a6ca4564e56bd0f36"}, - {file = "aiohttp-3.8.4-cp36-cp36m-musllinux_1_1_ppc64le.whl", hash = "sha256:3b90467ebc3d9fa5b0f9b6489dfb2c304a1db7b9946fa92aa76a831b9d587e99"}, - {file = "aiohttp-3.8.4-cp36-cp36m-musllinux_1_1_s390x.whl", hash = "sha256:cab9401de3ea52b4b4c6971db5fb5c999bd4260898af972bf23de1c6b5dd9d71"}, - {file = "aiohttp-3.8.4-cp36-cp36m-musllinux_1_1_x86_64.whl", hash = "sha256:d1f9282c5f2b5e241034a009779e7b2a1aa045f667ff521e7948ea9b56e0c5ff"}, - {file = "aiohttp-3.8.4-cp36-cp36m-win32.whl", hash = "sha256:5e14f25765a578a0a634d5f0cd1e2c3f53964553a00347998dfdf96b8137f777"}, - {file = "aiohttp-3.8.4-cp36-cp36m-win_amd64.whl", hash = "sha256:4c745b109057e7e5f1848c689ee4fb3a016c8d4d92da52b312f8a509f83aa05e"}, - {file = "aiohttp-3.8.4-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:aede4df4eeb926c8fa70de46c340a1bc2c6079e1c40ccf7b0eae1313ffd33519"}, - {file = "aiohttp-3.8.4-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:4ddaae3f3d32fc2cb4c53fab020b69a05c8ab1f02e0e59665c6f7a0d3a5be54f"}, - {file = "aiohttp-3.8.4-cp37-cp37m-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:c4eb3b82ca349cf6fadcdc7abcc8b3a50ab74a62e9113ab7a8ebc268aad35bb9"}, - {file = "aiohttp-3.8.4-cp37-cp37m-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:9bcb89336efa095ea21b30f9e686763f2be4478f1b0a616969551982c4ee4c3b"}, - {file = "aiohttp-3.8.4-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:6c08e8ed6fa3d477e501ec9db169bfac8140e830aa372d77e4a43084d8dd91ab"}, - {file = "aiohttp-3.8.4-cp37-cp37m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:c6cd05ea06daca6ad6a4ca3ba7fe7dc5b5de063ff4daec6170ec0f9979f6c332"}, - {file = "aiohttp-3.8.4-cp37-cp37m-musllinux_1_1_aarch64.whl", hash = "sha256:b7a00a9ed8d6e725b55ef98b1b35c88013245f35f68b1b12c5cd4100dddac333"}, - {file = "aiohttp-3.8.4-cp37-cp37m-musllinux_1_1_i686.whl", hash = "sha256:de04b491d0e5007ee1b63a309956eaed959a49f5bb4e84b26c8f5d49de140fa9"}, - {file = "aiohttp-3.8.4-cp37-cp37m-musllinux_1_1_ppc64le.whl", hash = "sha256:40653609b3bf50611356e6b6554e3a331f6879fa7116f3959b20e3528783e699"}, - {file = "aiohttp-3.8.4-cp37-cp37m-musllinux_1_1_s390x.whl", hash = "sha256:dbf3a08a06b3f433013c143ebd72c15cac33d2914b8ea4bea7ac2c23578815d6"}, - {file = "aiohttp-3.8.4-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:854f422ac44af92bfe172d8e73229c270dc09b96535e8a548f99c84f82dde241"}, - {file = "aiohttp-3.8.4-cp37-cp37m-win32.whl", hash = "sha256:aeb29c84bb53a84b1a81c6c09d24cf33bb8432cc5c39979021cc0f98c1292a1a"}, - {file = "aiohttp-3.8.4-cp37-cp37m-win_amd64.whl", hash = "sha256:db3fc6120bce9f446d13b1b834ea5b15341ca9ff3f335e4a951a6ead31105480"}, - {file = "aiohttp-3.8.4-cp38-cp38-macosx_10_9_universal2.whl", hash = "sha256:fabb87dd8850ef0f7fe2b366d44b77d7e6fa2ea87861ab3844da99291e81e60f"}, - {file = "aiohttp-3.8.4-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:91f6d540163f90bbaef9387e65f18f73ffd7c79f5225ac3d3f61df7b0d01ad15"}, - {file = "aiohttp-3.8.4-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:d265f09a75a79a788237d7f9054f929ced2e69eb0bb79de3798c468d8a90f945"}, - {file = "aiohttp-3.8.4-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:3d89efa095ca7d442a6d0cbc755f9e08190ba40069b235c9886a8763b03785da"}, - {file = "aiohttp-3.8.4-cp38-cp38-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:4dac314662f4e2aa5009977b652d9b8db7121b46c38f2073bfeed9f4049732cd"}, - {file = "aiohttp-3.8.4-cp38-cp38-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:fe11310ae1e4cd560035598c3f29d86cef39a83d244c7466f95c27ae04850f10"}, - {file = "aiohttp-3.8.4-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:6ddb2a2026c3f6a68c3998a6c47ab6795e4127315d2e35a09997da21865757f8"}, - {file = "aiohttp-3.8.4-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:e75b89ac3bd27d2d043b234aa7b734c38ba1b0e43f07787130a0ecac1e12228a"}, - {file = "aiohttp-3.8.4-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:6e601588f2b502c93c30cd5a45bfc665faaf37bbe835b7cfd461753068232074"}, - {file = "aiohttp-3.8.4-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:a5d794d1ae64e7753e405ba58e08fcfa73e3fad93ef9b7e31112ef3c9a0efb52"}, - {file = "aiohttp-3.8.4-cp38-cp38-musllinux_1_1_ppc64le.whl", hash = "sha256:a1f4689c9a1462f3df0a1f7e797791cd6b124ddbee2b570d34e7f38ade0e2c71"}, - {file = "aiohttp-3.8.4-cp38-cp38-musllinux_1_1_s390x.whl", hash = "sha256:3032dcb1c35bc330134a5b8a5d4f68c1a87252dfc6e1262c65a7e30e62298275"}, - {file = "aiohttp-3.8.4-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:8189c56eb0ddbb95bfadb8f60ea1b22fcfa659396ea36f6adcc521213cd7b44d"}, - {file = "aiohttp-3.8.4-cp38-cp38-win32.whl", hash = "sha256:33587f26dcee66efb2fff3c177547bd0449ab7edf1b73a7f5dea1e38609a0c54"}, - {file = "aiohttp-3.8.4-cp38-cp38-win_amd64.whl", hash = "sha256:e595432ac259af2d4630008bf638873d69346372d38255774c0e286951e8b79f"}, - {file = "aiohttp-3.8.4-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:5a7bdf9e57126dc345b683c3632e8ba317c31d2a41acd5800c10640387d193ed"}, - {file = "aiohttp-3.8.4-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:22f6eab15b6db242499a16de87939a342f5a950ad0abaf1532038e2ce7d31567"}, - {file = "aiohttp-3.8.4-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:7235604476a76ef249bd64cb8274ed24ccf6995c4a8b51a237005ee7a57e8643"}, - {file = "aiohttp-3.8.4-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:ea9eb976ffdd79d0e893869cfe179a8f60f152d42cb64622fca418cd9b18dc2a"}, - {file = "aiohttp-3.8.4-cp39-cp39-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:92c0cea74a2a81c4c76b62ea1cac163ecb20fb3ba3a75c909b9fa71b4ad493cf"}, - {file = "aiohttp-3.8.4-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:493f5bc2f8307286b7799c6d899d388bbaa7dfa6c4caf4f97ef7521b9cb13719"}, - {file = "aiohttp-3.8.4-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:0a63f03189a6fa7c900226e3ef5ba4d3bd047e18f445e69adbd65af433add5a2"}, - {file = "aiohttp-3.8.4-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:10c8cefcff98fd9168cdd86c4da8b84baaa90bf2da2269c6161984e6737bf23e"}, - {file = "aiohttp-3.8.4-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:bca5f24726e2919de94f047739d0a4fc01372801a3672708260546aa2601bf57"}, - {file = "aiohttp-3.8.4-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:03baa76b730e4e15a45f81dfe29a8d910314143414e528737f8589ec60cf7391"}, - {file = "aiohttp-3.8.4-cp39-cp39-musllinux_1_1_ppc64le.whl", hash = "sha256:8c29c77cc57e40f84acef9bfb904373a4e89a4e8b74e71aa8075c021ec9078c2"}, - {file = "aiohttp-3.8.4-cp39-cp39-musllinux_1_1_s390x.whl", hash = "sha256:03543dcf98a6619254b409be2d22b51f21ec66272be4ebda7b04e6412e4b2e14"}, - {file = "aiohttp-3.8.4-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:17b79c2963db82086229012cff93ea55196ed31f6493bb1ccd2c62f1724324e4"}, - {file = "aiohttp-3.8.4-cp39-cp39-win32.whl", hash = "sha256:34ce9f93a4a68d1272d26030655dd1b58ff727b3ed2a33d80ec433561b03d67a"}, - {file = "aiohttp-3.8.4-cp39-cp39-win_amd64.whl", hash = "sha256:41a86a69bb63bb2fc3dc9ad5ea9f10f1c9c8e282b471931be0268ddd09430b04"}, - {file = "aiohttp-3.8.4.tar.gz", hash = "sha256:bf2e1a9162c1e441bf805a1fd166e249d574ca04e03b34f97e2928769e91ab5c"}, -] -aioitertools = [ - {file = "aioitertools-0.11.0-py3-none-any.whl", hash = "sha256:04b95e3dab25b449def24d7df809411c10e62aab0cbe31a50ca4e68748c43394"}, - {file = "aioitertools-0.11.0.tar.gz", hash = "sha256:42c68b8dd3a69c2bf7f2233bf7df4bb58b557bca5252ac02ed5187bbc67d6831"}, -] -aiosignal = [ - {file = "aiosignal-1.3.1-py3-none-any.whl", hash = "sha256:f8376fb07dd1e86a584e4fcdec80b36b7f81aac666ebc724e2c090300dd83b17"}, - {file = "aiosignal-1.3.1.tar.gz", hash = "sha256:54cd96e15e1649b75d6c87526a6ff0b6c1b0dd3459f43d9ca11d48c339b68cfc"}, -] -async-timeout = [] -attrs = [ - {file = "attrs-22.2.0-py3-none-any.whl", hash = "sha256:29e95c7f6778868dbd49170f98f8818f78f3dc5e0e37c0b1f474e3561b240836"}, - {file = "attrs-22.2.0.tar.gz", hash = "sha256:c9227bfc2f01993c03f68db37d1d15c9690188323c067c641f1a35ca58185f99"}, -] -azure-core = [ - {file = "azure-core-1.26.3.zip", hash = "sha256:acbd0daa9675ce88623da35c80d819cdafa91731dee6b2695c64d7ca9da82db4"}, - {file = "azure_core-1.26.3-py3-none-any.whl", hash = "sha256:f7bad0a7b4d800d6e73733ea7e13a616016221ac111ff9a344fe4cba41e51bbe"}, -] -azure-datalake-store = [ - {file = "azure-datalake-store-0.0.52.tar.gz", hash = "sha256:4198ddb32614d16d4502b43d5c9739f81432b7e0e4d75d30e05149fe6007fea2"}, - {file = "azure_datalake_store-0.0.52-py2.py3-none-any.whl", hash = "sha256:aaed72b9c856824aeab554f4dbe0ef2c6d0ff36700bdd8b93d8298793117c48e"}, -] -azure-identity = [ - {file = "azure-identity-1.12.0.zip", hash = "sha256:7f9b1ae7d97ea7af3f38dd09305e19ab81a1e16ab66ea186b6579d85c1ca2347"}, - {file = "azure_identity-1.12.0-py3-none-any.whl", hash = "sha256:2a58ce4a209a013e37eaccfd5937570ab99e9118b3e1acf875eed3a85d541b92"}, -] -azure-storage-blob = [ - {file = "azure-storage-blob-12.14.1.zip", hash = "sha256:860d4d82985a4bfc7d3271e71275af330f54f330a754355435a7ba749ccde997"}, - {file = "azure_storage_blob-12.14.1-py3-none-any.whl", hash = "sha256:52b84658e8df7853a3cf1c563814655b5028b979b2a87905b92aa6bb30be240e"}, -] -boto3 = [ - {file = "boto3-1.24.59-py3-none-any.whl", hash = "sha256:34ab44146a2c4e7f4e72737f4b27e6eb5e0a7855c2f4599e3d9199b6a0a2d575"}, - {file = "boto3-1.24.59.tar.gz", hash = "sha256:a50b4323f9579cfe22fcf5531fbd40b567d4d74c1adce06aeb5c95fce2a6fb40"}, -] -botocore = [ - {file = "botocore-1.27.59-py3-none-any.whl", hash = "sha256:69d756791fc024bda54f6c53f71ae34e695ee41bbbc1743d9179c4837a4929da"}, - {file = "botocore-1.27.59.tar.gz", hash = "sha256:eda4aed6ee719a745d1288eaf1beb12f6f6448ad1fa12f159405db14ba9c92cf"}, -] -build = [ - {file = "build-0.10.0-py3-none-any.whl", hash = "sha256:af266720050a66c893a6096a2f410989eeac74ff9a68ba194b3f6473e8e26171"}, - {file = "build-0.10.0.tar.gz", hash = "sha256:d5b71264afdb5951d6704482aac78de887c80691c52b88a9ad195983ca2c9269"}, -] -certifi = [ - {file = "certifi-2022.12.7-py3-none-any.whl", hash = "sha256:4ad3232f5e926d6718ec31cfc1fcadfde020920e278684144551c91769c7bc18"}, - {file = "certifi-2022.12.7.tar.gz", hash = "sha256:35824b4c3a97115964b408844d64aa14db1cc518f6562e8d7261699d1350a9e3"}, -] -cffi = [ - {file = "cffi-1.15.1-cp27-cp27m-macosx_10_9_x86_64.whl", hash = "sha256:a66d3508133af6e8548451b25058d5812812ec3798c886bf38ed24a98216fab2"}, - {file = "cffi-1.15.1-cp27-cp27m-manylinux1_i686.whl", hash = "sha256:470c103ae716238bbe698d67ad020e1db9d9dba34fa5a899b5e21577e6d52ed2"}, - {file = "cffi-1.15.1-cp27-cp27m-manylinux1_x86_64.whl", hash = "sha256:9ad5db27f9cabae298d151c85cf2bad1d359a1b9c686a275df03385758e2f914"}, - {file = "cffi-1.15.1-cp27-cp27m-win32.whl", hash = "sha256:b3bbeb01c2b273cca1e1e0c5df57f12dce9a4dd331b4fa1635b8bec26350bde3"}, - {file = "cffi-1.15.1-cp27-cp27m-win_amd64.whl", hash = "sha256:e00b098126fd45523dd056d2efba6c5a63b71ffe9f2bbe1a4fe1716e1d0c331e"}, - {file = "cffi-1.15.1-cp27-cp27mu-manylinux1_i686.whl", hash = "sha256:d61f4695e6c866a23a21acab0509af1cdfd2c013cf256bbf5b6b5e2695827162"}, - {file = "cffi-1.15.1-cp27-cp27mu-manylinux1_x86_64.whl", hash = "sha256:ed9cb427ba5504c1dc15ede7d516b84757c3e3d7868ccc85121d9310d27eed0b"}, - {file = "cffi-1.15.1-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:39d39875251ca8f612b6f33e6b1195af86d1b3e60086068be9cc053aa4376e21"}, - {file = "cffi-1.15.1-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:285d29981935eb726a4399badae8f0ffdff4f5050eaa6d0cfc3f64b857b77185"}, - {file = "cffi-1.15.1-cp310-cp310-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:3eb6971dcff08619f8d91607cfc726518b6fa2a9eba42856be181c6d0d9515fd"}, - {file = "cffi-1.15.1-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:21157295583fe8943475029ed5abdcf71eb3911894724e360acff1d61c1d54bc"}, - {file = "cffi-1.15.1-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:5635bd9cb9731e6d4a1132a498dd34f764034a8ce60cef4f5319c0541159392f"}, - {file = "cffi-1.15.1-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:2012c72d854c2d03e45d06ae57f40d78e5770d252f195b93f581acf3ba44496e"}, - {file = "cffi-1.15.1-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:dd86c085fae2efd48ac91dd7ccffcfc0571387fe1193d33b6394db7ef31fe2a4"}, - {file = "cffi-1.15.1-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:fa6693661a4c91757f4412306191b6dc88c1703f780c8234035eac011922bc01"}, - {file = "cffi-1.15.1-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:59c0b02d0a6c384d453fece7566d1c7e6b7bae4fc5874ef2ef46d56776d61c9e"}, - {file = "cffi-1.15.1-cp310-cp310-win32.whl", hash = "sha256:cba9d6b9a7d64d4bd46167096fc9d2f835e25d7e4c121fb2ddfc6528fb0413b2"}, - {file = "cffi-1.15.1-cp310-cp310-win_amd64.whl", hash = "sha256:ce4bcc037df4fc5e3d184794f27bdaab018943698f4ca31630bc7f84a7b69c6d"}, - {file = "cffi-1.15.1-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:3d08afd128ddaa624a48cf2b859afef385b720bb4b43df214f85616922e6a5ac"}, - {file = "cffi-1.15.1-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:3799aecf2e17cf585d977b780ce79ff0dc9b78d799fc694221ce814c2c19db83"}, - {file = "cffi-1.15.1-cp311-cp311-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:a591fe9e525846e4d154205572a029f653ada1a78b93697f3b5a8f1f2bc055b9"}, - {file = "cffi-1.15.1-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:3548db281cd7d2561c9ad9984681c95f7b0e38881201e157833a2342c30d5e8c"}, - {file = "cffi-1.15.1-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:91fc98adde3d7881af9b59ed0294046f3806221863722ba7d8d120c575314325"}, - {file = "cffi-1.15.1-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:94411f22c3985acaec6f83c6df553f2dbe17b698cc7f8ae751ff2237d96b9e3c"}, - {file = "cffi-1.15.1-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:03425bdae262c76aad70202debd780501fabeaca237cdfddc008987c0e0f59ef"}, - {file = "cffi-1.15.1-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:cc4d65aeeaa04136a12677d3dd0b1c0c94dc43abac5860ab33cceb42b801c1e8"}, - {file = "cffi-1.15.1-cp311-cp311-win32.whl", hash = "sha256:a0f100c8912c114ff53e1202d0078b425bee3649ae34d7b070e9697f93c5d52d"}, - {file = "cffi-1.15.1-cp311-cp311-win_amd64.whl", hash = "sha256:04ed324bda3cda42b9b695d51bb7d54b680b9719cfab04227cdd1e04e5de3104"}, - {file = "cffi-1.15.1-cp36-cp36m-macosx_10_9_x86_64.whl", hash = "sha256:50a74364d85fd319352182ef59c5c790484a336f6db772c1a9231f1c3ed0cbd7"}, - {file = "cffi-1.15.1-cp36-cp36m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:e263d77ee3dd201c3a142934a086a4450861778baaeeb45db4591ef65550b0a6"}, - {file = "cffi-1.15.1-cp36-cp36m-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:cec7d9412a9102bdc577382c3929b337320c4c4c4849f2c5cdd14d7368c5562d"}, - {file = "cffi-1.15.1-cp36-cp36m-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:4289fc34b2f5316fbb762d75362931e351941fa95fa18789191b33fc4cf9504a"}, - {file = "cffi-1.15.1-cp36-cp36m-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:173379135477dc8cac4bc58f45db08ab45d228b3363adb7af79436135d028405"}, - {file = "cffi-1.15.1-cp36-cp36m-manylinux_2_5_x86_64.manylinux1_x86_64.whl", hash = "sha256:6975a3fac6bc83c4a65c9f9fcab9e47019a11d3d2cf7f3c0d03431bf145a941e"}, - {file = "cffi-1.15.1-cp36-cp36m-win32.whl", hash = "sha256:2470043b93ff09bf8fb1d46d1cb756ce6132c54826661a32d4e4d132e1977adf"}, - {file = "cffi-1.15.1-cp36-cp36m-win_amd64.whl", hash = "sha256:30d78fbc8ebf9c92c9b7823ee18eb92f2e6ef79b45ac84db507f52fbe3ec4497"}, - {file = "cffi-1.15.1-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:198caafb44239b60e252492445da556afafc7d1e3ab7a1fb3f0584ef6d742375"}, - {file = "cffi-1.15.1-cp37-cp37m-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:5ef34d190326c3b1f822a5b7a45f6c4535e2f47ed06fec77d3d799c450b2651e"}, - {file = "cffi-1.15.1-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:8102eaf27e1e448db915d08afa8b41d6c7ca7a04b7d73af6514df10a3e74bd82"}, - {file = "cffi-1.15.1-cp37-cp37m-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:5df2768244d19ab7f60546d0c7c63ce1581f7af8b5de3eb3004b9b6fc8a9f84b"}, - {file = "cffi-1.15.1-cp37-cp37m-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:a8c4917bd7ad33e8eb21e9a5bbba979b49d9a97acb3a803092cbc1133e20343c"}, - {file = "cffi-1.15.1-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:0e2642fe3142e4cc4af0799748233ad6da94c62a8bec3a6648bf8ee68b1c7426"}, - {file = "cffi-1.15.1-cp37-cp37m-win32.whl", hash = "sha256:e229a521186c75c8ad9490854fd8bbdd9a0c9aa3a524326b55be83b54d4e0ad9"}, - {file = "cffi-1.15.1-cp37-cp37m-win_amd64.whl", hash = "sha256:a0b71b1b8fbf2b96e41c4d990244165e2c9be83d54962a9a1d118fd8657d2045"}, - {file = "cffi-1.15.1-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:320dab6e7cb2eacdf0e658569d2575c4dad258c0fcc794f46215e1e39f90f2c3"}, - {file = "cffi-1.15.1-cp38-cp38-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:1e74c6b51a9ed6589199c787bf5f9875612ca4a8a0785fb2d4a84429badaf22a"}, - {file = "cffi-1.15.1-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a5c84c68147988265e60416b57fc83425a78058853509c1b0629c180094904a5"}, - {file = "cffi-1.15.1-cp38-cp38-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:3b926aa83d1edb5aa5b427b4053dc420ec295a08e40911296b9eb1b6170f6cca"}, - {file = "cffi-1.15.1-cp38-cp38-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:87c450779d0914f2861b8526e035c5e6da0a3199d8f1add1a665e1cbc6fc6d02"}, - {file = "cffi-1.15.1-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:4f2c9f67e9821cad2e5f480bc8d83b8742896f1242dba247911072d4fa94c192"}, - {file = "cffi-1.15.1-cp38-cp38-win32.whl", hash = "sha256:8b7ee99e510d7b66cdb6c593f21c043c248537a32e0bedf02e01e9553a172314"}, - {file = "cffi-1.15.1-cp38-cp38-win_amd64.whl", hash = "sha256:00a9ed42e88df81ffae7a8ab6d9356b371399b91dbdf0c3cb1e84c03a13aceb5"}, - {file = "cffi-1.15.1-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:54a2db7b78338edd780e7ef7f9f6c442500fb0d41a5a4ea24fff1c929d5af585"}, - {file = "cffi-1.15.1-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:fcd131dd944808b5bdb38e6f5b53013c5aa4f334c5cad0c72742f6eba4b73db0"}, - {file = "cffi-1.15.1-cp39-cp39-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:7473e861101c9e72452f9bf8acb984947aa1661a7704553a9f6e4baa5ba64415"}, - {file = "cffi-1.15.1-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:6c9a799e985904922a4d207a94eae35c78ebae90e128f0c4e521ce339396be9d"}, - {file = "cffi-1.15.1-cp39-cp39-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:3bcde07039e586f91b45c88f8583ea7cf7a0770df3a1649627bf598332cb6984"}, - {file = "cffi-1.15.1-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:33ab79603146aace82c2427da5ca6e58f2b3f2fb5da893ceac0c42218a40be35"}, - {file = "cffi-1.15.1-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:5d598b938678ebf3c67377cdd45e09d431369c3b1a5b331058c338e201f12b27"}, - {file = "cffi-1.15.1-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:db0fbb9c62743ce59a9ff687eb5f4afbe77e5e8403d6697f7446e5f609976f76"}, - {file = "cffi-1.15.1-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:98d85c6a2bef81588d9227dde12db8a7f47f639f4a17c9ae08e773aa9c697bf3"}, - {file = "cffi-1.15.1-cp39-cp39-win32.whl", hash = "sha256:40f4774f5a9d4f5e344f31a32b5096977b5d48560c5592e2f3d2c4374bd543ee"}, - {file = "cffi-1.15.1-cp39-cp39-win_amd64.whl", hash = "sha256:70df4e3b545a17496c9b3f41f5115e69a4f2e77e94e1d2a8e1070bc0c38c8a3c"}, - {file = "cffi-1.15.1.tar.gz", hash = "sha256:d400bfb9a37b1351253cb402671cea7e89bdecc294e8016a707f6d1d8ac934f9"}, -] -cfgv = [ - {file = "cfgv-3.3.1-py2.py3-none-any.whl", hash = "sha256:c6a0883f3917a037485059700b9e75da2464e6c27051014ad85ba6aaa5884426"}, - {file = "cfgv-3.3.1.tar.gz", hash = "sha256:f5a830efb9ce7a445376bb66ec94c638a9787422f96264c98edc6bdeed8ab736"}, -] -charset-normalizer = [ +files = [ {file = "charset-normalizer-3.0.1.tar.gz", hash = "sha256:ebea339af930f8ca5d7a699b921106c6e29c617fe9606fa7baa043c1cdae326f"}, {file = "charset_normalizer-3.0.1-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:88600c72ef7587fe1708fd242b385b6ed4b8904976d5da0893e31df8b3480cb6"}, {file = "charset_normalizer-3.0.1-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:c75ffc45f25324e68ab238cb4b5c0a38cd1c3d7f1fb1f72b5541de469e2247db"}, @@ -1500,12 +568,42 @@ charset-normalizer = [ {file = "charset_normalizer-3.0.1-cp39-cp39-win_amd64.whl", hash = "sha256:0a11e971ed097d24c534c037d298ad32c6ce81a45736d31e0ff0ad37ab437d59"}, {file = "charset_normalizer-3.0.1-py3-none-any.whl", hash = "sha256:7e189e2e1d3ed2f4aebabd2d5b0f931e883676e51c7624826e0a4e5fe8a0bf24"}, ] -click = [] -colorama = [ + +[[package]] +name = "click" +version = "8.1.3" +description = "Composable command line interface toolkit" +category = "main" +optional = false +python-versions = ">=3.7" +files = [ + {file = "click-8.1.3-py3-none-any.whl", hash = "sha256:bb4d8133cb15a609f44e8213d9b391b0809795062913b383c62be0ee95b1db48"}, + {file = "click-8.1.3.tar.gz", hash = "sha256:7682dc8afb30297001674575ea00d1814d808d6a36af415a82bd481d37ba7b8e"}, +] + +[package.dependencies] +colorama = {version = "*", markers = "platform_system == \"Windows\""} + +[[package]] +name = "colorama" +version = "0.4.6" +description = "Cross-platform colored terminal text." +category = "main" +optional = false +python-versions = "!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,!=3.4.*,!=3.5.*,!=3.6.*,>=2.7" +files = [ {file = "colorama-0.4.6-py2.py3-none-any.whl", hash = "sha256:4f1d9991f5acc0ca119f9d443620b77f9d6b33703e51011c16baf57afb285fc6"}, {file = "colorama-0.4.6.tar.gz", hash = "sha256:08695f5cb7ed6e0531a20572697297273c47b8cae5a63ffc6d6ed5c201be6e44"}, ] -coverage = [ + +[[package]] +name = "coverage" +version = "7.1.0" +description = "Code coverage measurement for Python" +category = "dev" +optional = false +python-versions = ">=3.7" +files = [ {file = "coverage-7.1.0-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:3b946bbcd5a8231383450b195cfb58cb01cbe7f8949f5758566b881df4b33baf"}, {file = "coverage-7.1.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:ec8e767f13be637d056f7e07e61d089e555f719b387a7070154ad80a0ff31801"}, {file = "coverage-7.1.0-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:d4a5a5879a939cb84959d86869132b00176197ca561c664fc21478c1eee60d75"}, @@ -1558,7 +656,21 @@ coverage = [ {file = "coverage-7.1.0-pp37.pp38.pp39-none-any.whl", hash = "sha256:755e89e32376c850f826c425ece2c35a4fc266c081490eb0a841e7c1cb0d3bda"}, {file = "coverage-7.1.0.tar.gz", hash = "sha256:10188fe543560ec4874f974b5305cd1a8bdcfa885ee00ea3a03733464c4ca265"}, ] -cryptography = [ + +[package.dependencies] +tomli = {version = "*", optional = true, markers = "python_full_version <= \"3.11.0a6\" and extra == \"toml\""} + +[package.extras] +toml = ["tomli"] + +[[package]] +name = "cryptography" +version = "39.0.1" +description = "cryptography is a package which provides cryptographic recipes and primitives to Python developers." +category = "main" +optional = false +python-versions = ">=3.6" +files = [ {file = "cryptography-39.0.1-cp36-abi3-macosx_10_12_universal2.whl", hash = "sha256:6687ef6d0a6497e2b58e7c5b852b53f62142cfa7cd1555795758934da363a965"}, {file = "cryptography-39.0.1-cp36-abi3-macosx_10_12_x86_64.whl", hash = "sha256:706843b48f9a3f9b9911979761c91541e3d90db1ca905fd63fee540a217698bc"}, {file = "cryptography-39.0.1-cp36-abi3-manylinux_2_17_aarch64.manylinux2014_aarch64.manylinux_2_24_aarch64.whl", hash = "sha256:5d2d8b87a490bfcd407ed9d49093793d0f75198a35e6eb1a923ce1ee86c62b41"}, @@ -1572,6 +684,8 @@ cryptography = [ {file = "cryptography-39.0.1-cp36-abi3-win32.whl", hash = "sha256:fe913f20024eb2cb2f323e42a64bdf2911bb9738a15dba7d3cce48151034e3a8"}, {file = "cryptography-39.0.1-cp36-abi3-win_amd64.whl", hash = "sha256:ced4e447ae29ca194449a3f1ce132ded8fcab06971ef5f618605aacaa612beac"}, {file = "cryptography-39.0.1-pp38-pypy38_pp73-macosx_10_12_x86_64.whl", hash = "sha256:807ce09d4434881ca3a7594733669bd834f5b2c6d5c7e36f8c00f691887042ad"}, + {file = "cryptography-39.0.1-pp38-pypy38_pp73-manylinux_2_28_aarch64.whl", hash = "sha256:c5caeb8188c24888c90b5108a441c106f7faa4c4c075a2bcae438c6e8ca73cef"}, + {file = "cryptography-39.0.1-pp38-pypy38_pp73-manylinux_2_28_x86_64.whl", hash = "sha256:4789d1e3e257965e960232345002262ede4d094d1a19f4d3b52e48d4d8f3b885"}, {file = "cryptography-39.0.1-pp38-pypy38_pp73-win_amd64.whl", hash = "sha256:96f1157a7c08b5b189b16b47bc9db2332269d6680a196341bf30046330d15388"}, {file = "cryptography-39.0.1-pp39-pypy39_pp73-macosx_10_12_x86_64.whl", hash = "sha256:e422abdec8b5fa8462aa016786680720d78bdce7a30c652b7fadf83a4ba35336"}, {file = "cryptography-39.0.1-pp39-pypy39_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.manylinux_2_24_aarch64.whl", hash = "sha256:b0afd054cd42f3d213bf82c629efb1ee5f22eba35bf0eec88ea9ea7304f511a2"}, @@ -1581,15 +695,52 @@ cryptography = [ {file = "cryptography-39.0.1-pp39-pypy39_pp73-win_amd64.whl", hash = "sha256:fdd188c8a6ef8769f148f88f859884507b954cc64db6b52f66ef199bb9ad660a"}, {file = "cryptography-39.0.1.tar.gz", hash = "sha256:d1f6198ee6d9148405e49887803907fe8962a23e6c6f83ea7d98f1c0de375695"}, ] -distlib = [ + +[package.dependencies] +cffi = ">=1.12" + +[package.extras] +docs = ["sphinx (>=5.3.0)", "sphinx-rtd-theme (>=1.1.1)"] +docstest = ["pyenchant (>=1.6.11)", "sphinxcontrib-spelling (>=4.0.1)", "twine (>=1.12.0)"] +pep8test = ["black", "check-manifest", "mypy", "ruff", "types-pytz", "types-requests"] +sdist = ["setuptools-rust (>=0.11.4)"] +ssh = ["bcrypt (>=3.1.5)"] +test = ["hypothesis (>=1.11.4,!=3.79.2)", "iso8601", "pretend", "pytest (>=6.2.0)", "pytest-benchmark", "pytest-cov", "pytest-shard (>=0.1.2)", "pytest-subtests", "pytest-xdist", "pytz"] +test-randomorder = ["pytest-randomly"] +tox = ["tox"] + +[[package]] +name = "distlib" +version = "0.3.6" +description = "Distribution utilities" +category = "dev" +optional = false +python-versions = "*" +files = [ {file = "distlib-0.3.6-py2.py3-none-any.whl", hash = "sha256:f35c4b692542ca110de7ef0bea44d73981caeb34ca0b9b6b2e6d7790dda8f80e"}, {file = "distlib-0.3.6.tar.gz", hash = "sha256:14bad2d9b04d3a36127ac97f30b12a19268f211063d8f8ee4f47108896e11b46"}, ] -docutils = [ + +[[package]] +name = "docutils" +version = "0.19" +description = "Docutils -- Python Documentation Utilities" +category = "dev" +optional = false +python-versions = ">=3.7" +files = [ {file = "docutils-0.19-py3-none-any.whl", hash = "sha256:5e1de4d849fee02c63b040a4a3fd567f4ab104defd8a5511fbbc24a8a017efbc"}, {file = "docutils-0.19.tar.gz", hash = "sha256:33995a6753c30b7f577febfc2c50411fec6aac7f7ffeb7c4cfe5991072dcf9e6"}, ] -duckdb = [ + +[[package]] +name = "duckdb" +version = "0.7.0" +description = "DuckDB embedded database" +category = "main" +optional = true +python-versions = "*" +files = [ {file = "duckdb-0.7.0-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:3fcae66586bd858c522559e85068d80414a27f23839705d5b400156c0fdbbeca"}, {file = "duckdb-0.7.0-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:5d07d37edd7e2d1078bf500d11d068d6c1b62868bf372beca4d856cb3fa58c02"}, {file = "duckdb-0.7.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:5b86e4bff0f5d49b6ac0aaae5fd6d4bd3d65f1aeeb77529d717515a0e4e16e60"}, @@ -1638,11 +789,30 @@ duckdb = [ {file = "duckdb-0.7.0-cp39-cp39-win_amd64.whl", hash = "sha256:726a3d555c6387e6653bc46d94645eac8139f825adfd0b7876ea858b5717796b"}, {file = "duckdb-0.7.0.tar.gz", hash = "sha256:08ebfcc67a6a073fa2efd0453de9a7e453637bba522e67968fe0c03ccbfc2ec5"}, ] -exceptiongroup = [ + +[[package]] +name = "exceptiongroup" +version = "1.1.0" +description = "Backport of PEP 654 (exception groups)" +category = "dev" +optional = false +python-versions = ">=3.7" +files = [ {file = "exceptiongroup-1.1.0-py3-none-any.whl", hash = "sha256:327cbda3da756e2de031a3107b81ab7b3770a602c4d16ca618298c526f4bec1e"}, {file = "exceptiongroup-1.1.0.tar.gz", hash = "sha256:bcb67d800a4497e1b404c2dd44fca47d3b7a5e5433dbab67f96c1a685cdfdf23"}, ] -fastavro = [ + +[package.extras] +test = ["pytest (>=6)"] + +[[package]] +name = "fastavro" +version = "1.7.1" +description = "Fast read/write of AVRO files" +category = "dev" +optional = false +python-versions = ">=3.7" +files = [ {file = "fastavro-1.7.1-cp310-cp310-macosx_11_0_x86_64.whl", hash = "sha256:f8f7f11af8c2c074341217d6247b7ba09cadcd55f899e046c14e3a44afa5fc95"}, {file = "fastavro-1.7.1-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:11159cf23f5ff4b752b028a77bd2a7941599932527e8a6512779e25b0503f037"}, {file = "fastavro-1.7.1-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:86ce89743711355eee7d3fec34847de1ab574f4567aa4a667e966711bb2e0cd9"}, @@ -1665,11 +835,37 @@ fastavro = [ {file = "fastavro-1.7.1-cp39-cp39-win_amd64.whl", hash = "sha256:4d56cc05ceb1c9f4de19c1f330a4f140af3b944834d63cd0e11517deaea21ee1"}, {file = "fastavro-1.7.1.tar.gz", hash = "sha256:4b8bcae4ed6343af186e638061cdfbc5331cdb5e026d055099c91d4f07be838c"}, ] -filelock = [ + +[package.extras] +codecs = ["lz4", "python-snappy", "zstandard"] +lz4 = ["lz4"] +snappy = ["python-snappy"] +zstandard = ["zstandard"] + +[[package]] +name = "filelock" +version = "3.9.0" +description = "A platform independent file lock." +category = "dev" +optional = false +python-versions = ">=3.7" +files = [ {file = "filelock-3.9.0-py3-none-any.whl", hash = "sha256:f58d535af89bb9ad5cd4df046f741f8553a418c01a7856bf0d173bbc9f6bd16d"}, {file = "filelock-3.9.0.tar.gz", hash = "sha256:7b319f24340b51f55a2bf7a12ac0755a9b03e718311dac567a0f4f7fabd2f5de"}, ] -frozenlist = [ + +[package.extras] +docs = ["furo (>=2022.12.7)", "sphinx (>=5.3)", "sphinx-autodoc-typehints (>=1.19.5)"] +testing = ["covdefaults (>=2.2.2)", "coverage (>=7.0.1)", "pytest (>=7.2)", "pytest-cov (>=4)", "pytest-timeout (>=2.1)"] + +[[package]] +name = "frozenlist" +version = "1.3.3" +description = "A list-like structure which implements collections.abc.MutableSequence" +category = "main" +optional = true +python-versions = ">=3.7" +files = [ {file = "frozenlist-1.3.3-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:ff8bf625fe85e119553b5383ba0fb6aa3d0ec2ae980295aaefa552374926b3f4"}, {file = "frozenlist-1.3.3-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:dfbac4c2dfcc082fcf8d942d1e49b6aa0766c19d3358bd86e2000bf0fa4a9cf0"}, {file = "frozenlist-1.3.3-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:b1c63e8d377d039ac769cd0926558bb7068a1f7abb0f003e3717ee003ad85530"}, @@ -1745,40 +941,179 @@ frozenlist = [ {file = "frozenlist-1.3.3-cp39-cp39-win_amd64.whl", hash = "sha256:cfe33efc9cb900a4c46f91a5ceba26d6df370ffddd9ca386eb1d4f0ad97b9ea9"}, {file = "frozenlist-1.3.3.tar.gz", hash = "sha256:58bcc55721e8a90b88332d6cd441261ebb22342e238296bb330968952fbb3a6a"}, ] -fsspec = [ + +[[package]] +name = "fsspec" +version = "2023.1.0" +description = "File-system specification" +category = "main" +optional = false +python-versions = ">=3.7" +files = [ {file = "fsspec-2023.1.0-py3-none-any.whl", hash = "sha256:b833e2e541e9e8cde0ab549414187871243177feb3d344f9d27b25a93f5d8139"}, {file = "fsspec-2023.1.0.tar.gz", hash = "sha256:fbae7f20ff801eb5f7d0bedf81f25c787c0dfac5e982d98fa3884a9cde2b5411"}, ] -identify = [ + +[package.extras] +abfs = ["adlfs"] +adl = ["adlfs"] +arrow = ["pyarrow (>=1)"] +dask = ["dask", "distributed"] +dropbox = ["dropbox", "dropboxdrivefs", "requests"] +entrypoints = ["importlib-metadata"] +fuse = ["fusepy"] +gcs = ["gcsfs"] +git = ["pygit2"] +github = ["requests"] +gs = ["gcsfs"] +gui = ["panel"] +hdfs = ["pyarrow (>=1)"] +http = ["aiohttp (!=4.0.0a0,!=4.0.0a1)", "requests"] +libarchive = ["libarchive-c"] +oci = ["ocifs"] +s3 = ["s3fs"] +sftp = ["paramiko"] +smb = ["smbprotocol"] +ssh = ["paramiko"] +tqdm = ["tqdm"] + +[[package]] +name = "identify" +version = "2.5.18" +description = "File identification library for Python" +category = "dev" +optional = false +python-versions = ">=3.7" +files = [ {file = "identify-2.5.18-py2.py3-none-any.whl", hash = "sha256:93aac7ecf2f6abf879b8f29a8002d3c6de7086b8c28d88e1ad15045a15ab63f9"}, {file = "identify-2.5.18.tar.gz", hash = "sha256:89e144fa560cc4cffb6ef2ab5e9fb18ed9f9b3cb054384bab4b95c12f6c309fe"}, ] -idna = [ + +[package.extras] +license = ["ukkonen"] + +[[package]] +name = "idna" +version = "3.4" +description = "Internationalized Domain Names in Applications (IDNA)" +category = "main" +optional = false +python-versions = ">=3.5" +files = [ {file = "idna-3.4-py3-none-any.whl", hash = "sha256:90b77e79eaa3eba6de819a0c442c0b4ceefc341a7a2ab77d7562bf49f425c5c2"}, {file = "idna-3.4.tar.gz", hash = "sha256:814f528e8dead7d329833b91c5faa87d60bf71824cd12a7530b5526063d02cb4"}, ] -importlib-metadata = [ + +[[package]] +name = "importlib-metadata" +version = "6.0.0" +description = "Read metadata from Python packages" +category = "dev" +optional = false +python-versions = ">=3.7" +files = [ {file = "importlib_metadata-6.0.0-py3-none-any.whl", hash = "sha256:7efb448ec9a5e313a57655d35aa54cd3e01b7e1fbcf72dce1bf06119420f5bad"}, {file = "importlib_metadata-6.0.0.tar.gz", hash = "sha256:e354bedeb60efa6affdcc8ae121b73544a7aa74156d047311948f6d711cd378d"}, ] -iniconfig = [ + +[package.dependencies] +zipp = ">=0.5" + +[package.extras] +docs = ["furo", "jaraco.packaging (>=9)", "jaraco.tidelift (>=1.4)", "rst.linker (>=1.9)", "sphinx (>=3.5)", "sphinx-lint"] +perf = ["ipython"] +testing = ["flake8 (<5)", "flufl.flake8", "importlib-resources (>=1.3)", "packaging", "pyfakefs", "pytest (>=6)", "pytest-black (>=0.3.7)", "pytest-checkdocs (>=2.4)", "pytest-cov", "pytest-enabler (>=1.3)", "pytest-flake8", "pytest-mypy (>=0.9.1)", "pytest-perf (>=0.9.2)"] + +[[package]] +name = "iniconfig" +version = "2.0.0" +description = "brain-dead simple config-ini parsing" +category = "dev" +optional = false +python-versions = ">=3.7" +files = [ {file = "iniconfig-2.0.0-py3-none-any.whl", hash = "sha256:b6a85871a79d2e3b22d2d1b94ac2824226a63c6b741c88f7ae975f18b6778374"}, {file = "iniconfig-2.0.0.tar.gz", hash = "sha256:2d91e135bf72d31a410b17c16da610a82cb55f6b0477d1a902134b24a455b8b3"}, ] -isodate = [ + +[[package]] +name = "isodate" +version = "0.6.1" +description = "An ISO 8601 date/time/duration parser and formatter" +category = "main" +optional = true +python-versions = "*" +files = [ {file = "isodate-0.6.1-py2.py3-none-any.whl", hash = "sha256:0751eece944162659049d35f4f549ed815792b38793f07cf73381c1c87cbed96"}, {file = "isodate-0.6.1.tar.gz", hash = "sha256:48c5881de7e8b0a0d648cb024c8062dc84e7b840ed81e864c7614fd3c127bde9"}, ] -jinja2 = [ + +[package.dependencies] +six = "*" + +[[package]] +name = "jinja2" +version = "3.1.2" +description = "A very fast and expressive template engine." +category = "dev" +optional = false +python-versions = ">=3.7" +files = [ {file = "Jinja2-3.1.2-py3-none-any.whl", hash = "sha256:6088930bfe239f0e6710546ab9c19c9ef35e29792895fed6e6e31a023a182a61"}, {file = "Jinja2-3.1.2.tar.gz", hash = "sha256:31351a702a408a9e7595a8fc6150fc3f43bb6bf7e319770cbc0db9df9437e852"}, ] -jmespath = [] -markdown-it-py = [ + +[package.dependencies] +MarkupSafe = ">=2.0" + +[package.extras] +i18n = ["Babel (>=2.7)"] + +[[package]] +name = "jmespath" +version = "1.0.1" +description = "JSON Matching Expressions" +category = "main" +optional = false +python-versions = ">=3.7" +files = [ + {file = "jmespath-1.0.1-py3-none-any.whl", hash = "sha256:02e2e4cc71b5bcab88332eebf907519190dd9e6e82107fa7f83b1003a6252980"}, + {file = "jmespath-1.0.1.tar.gz", hash = "sha256:90261b206d6defd58fdd5e85f478bf633a2901798906be2ad389150c5c60edbe"}, +] + +[[package]] +name = "markdown-it-py" +version = "2.1.0" +description = "Python port of markdown-it. Markdown parsing, done right!" +category = "main" +optional = false +python-versions = ">=3.7" +files = [ {file = "markdown-it-py-2.1.0.tar.gz", hash = "sha256:cf7e59fed14b5ae17c0006eff14a2d9a00ed5f3a846148153899a0224e2c07da"}, {file = "markdown_it_py-2.1.0-py3-none-any.whl", hash = "sha256:93de681e5c021a432c63147656fe21790bc01231e0cd2da73626f1aa3ac0fe27"}, ] -markupsafe = [ + +[package.dependencies] +mdurl = ">=0.1,<1.0" + +[package.extras] +benchmarking = ["psutil", "pytest", "pytest-benchmark (>=3.2,<4.0)"] +code-style = ["pre-commit (==2.6)"] +compare = ["commonmark (>=0.9.1,<0.10.0)", "markdown (>=3.3.6,<3.4.0)", "mistletoe (>=0.8.1,<0.9.0)", "mistune (>=2.0.2,<2.1.0)", "panflute (>=2.1.3,<2.2.0)"] +linkify = ["linkify-it-py (>=1.0,<2.0)"] +plugins = ["mdit-py-plugins"] +profiling = ["gprof2dot"] +rtd = ["attrs", "myst-parser", "pyyaml", "sphinx", "sphinx-copybutton", "sphinx-design", "sphinx_book_theme"] +testing = ["coverage", "pytest", "pytest-cov", "pytest-regressions"] + +[[package]] +name = "markupsafe" +version = "2.1.2" +description = "Safely add untrusted strings to HTML/XML markup." +category = "dev" +optional = false +python-versions = ">=3.7" +files = [ {file = "MarkupSafe-2.1.2-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:665a36ae6f8f20a4676b53224e33d456a6f5a72657d9c83c2aa00765072f31f7"}, {file = "MarkupSafe-2.1.2-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:340bea174e9761308703ae988e982005aedf427de816d1afe98147668cc03036"}, {file = "MarkupSafe-2.1.2-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:22152d00bf4a9c7c83960521fc558f55a1adbc0631fbb00a9471e097b19d72e1"}, @@ -1830,11 +1165,27 @@ markupsafe = [ {file = "MarkupSafe-2.1.2-cp39-cp39-win_amd64.whl", hash = "sha256:0576fe974b40a400449768941d5d0858cc624e3249dfd1e0c33674e5c7ca7aed"}, {file = "MarkupSafe-2.1.2.tar.gz", hash = "sha256:abcabc8c2b26036d62d4c746381a6f7cf60aafcc653198ad678306986b09450d"}, ] -mdurl = [ + +[[package]] +name = "mdurl" +version = "0.1.2" +description = "Markdown URL utilities" +category = "main" +optional = false +python-versions = ">=3.7" +files = [ {file = "mdurl-0.1.2-py3-none-any.whl", hash = "sha256:84008a41e51615a49fc9966191ff91509e3c40b939176e643fd50a5c2196b8f8"}, {file = "mdurl-0.1.2.tar.gz", hash = "sha256:bb413d29f5eea38f31dd4754dd7377d4465116fb207585f97bf925588687c1ba"}, ] -mmhash3 = [ + +[[package]] +name = "mmhash3" +version = "3.0.1" +description = "Python wrapper for MurmurHash (MurmurHash3), a set of fast and robust hash functions." +category = "main" +optional = false +python-versions = "*" +files = [ {file = "mmhash3-3.0.1-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:47deea30cd8d3d5cd52dc740902a4c70383bfe8248eac29d0877fe63e03c2713"}, {file = "mmhash3-3.0.1-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:ecdaf4d1de617818bf05cd526ca558db6010beeba7ea9e19f695f2bdcac0e0a4"}, {file = "mmhash3-3.0.1-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:4675585617584e9e1aafa3a90ac0ac9a437257c507748d97de8b21977e9d6745"}, @@ -1870,23 +1221,123 @@ mmhash3 = [ {file = "mmhash3-3.0.1-cp39-cp39-win_amd64.whl", hash = "sha256:0baeaa20cac5f75ed14f28056826bd9d9c8b2354b382073f3fd5190708992a0d"}, {file = "mmhash3-3.0.1.tar.gz", hash = "sha256:a00d68f4a1cc434b9501513c8a29e18ed1ddad383677d72b41d71d0d862348af"}, ] -moto = [ + +[[package]] +name = "moto" +version = "4.1.2" +description = "" +category = "dev" +optional = false +python-versions = ">=3.7" +files = [ {file = "moto-4.1.2-py2.py3-none-any.whl", hash = "sha256:1b361ece638c74a657325378a259276f368aafce2f8be84f8143e69fa93ce8ec"}, {file = "moto-4.1.2.tar.gz", hash = "sha256:63431733d2a02c7bd652ad71ec1da442a0e0d580cbac5eeb50d440a2ce066eac"}, ] -msal = [ + +[package.dependencies] +boto3 = ">=1.9.201" +botocore = ">=1.12.201" +cryptography = ">=3.3.1" +Jinja2 = ">=2.10.1" +python-dateutil = ">=2.1,<3.0.0" +requests = ">=2.5" +responses = ">=0.13.0" +werkzeug = ">=0.5,<2.2.0 || >2.2.0,<2.2.1 || >2.2.1" +xmltodict = "*" + +[package.extras] +all = ["PyYAML (>=5.1)", "aws-xray-sdk (>=0.93,!=0.96)", "cfn-lint (>=0.40.0)", "docker (>=2.5.1)", "ecdsa (!=0.15)", "graphql-core", "jsondiff (>=1.1.2)", "openapi-spec-validator (>=0.2.8)", "pyparsing (>=3.0.7)", "python-jose[cryptography] (>=3.1.0,<4.0.0)", "setuptools", "sshpubkeys (>=3.1.0)"] +apigateway = ["PyYAML (>=5.1)", "ecdsa (!=0.15)", "openapi-spec-validator (>=0.2.8)", "python-jose[cryptography] (>=3.1.0,<4.0.0)"] +apigatewayv2 = ["PyYAML (>=5.1)"] +appsync = ["graphql-core"] +awslambda = ["docker (>=2.5.1)"] +batch = ["docker (>=2.5.1)"] +cloudformation = ["PyYAML (>=5.1)", "aws-xray-sdk (>=0.93,!=0.96)", "cfn-lint (>=0.40.0)", "docker (>=2.5.1)", "ecdsa (!=0.15)", "graphql-core", "jsondiff (>=1.1.2)", "openapi-spec-validator (>=0.2.8)", "pyparsing (>=3.0.7)", "python-jose[cryptography] (>=3.1.0,<4.0.0)", "setuptools", "sshpubkeys (>=3.1.0)"] +cognitoidp = ["ecdsa (!=0.15)", "python-jose[cryptography] (>=3.1.0,<4.0.0)"] +ds = ["sshpubkeys (>=3.1.0)"] +dynamodb = ["docker (>=2.5.1)"] +dynamodbstreams = ["docker (>=2.5.1)"] +ebs = ["sshpubkeys (>=3.1.0)"] +ec2 = ["sshpubkeys (>=3.1.0)"] +efs = ["sshpubkeys (>=3.1.0)"] +eks = ["sshpubkeys (>=3.1.0)"] +glue = ["pyparsing (>=3.0.7)"] +iotdata = ["jsondiff (>=1.1.2)"] +route53resolver = ["sshpubkeys (>=3.1.0)"] +s3 = ["PyYAML (>=5.1)"] +server = ["PyYAML (>=5.1)", "aws-xray-sdk (>=0.93,!=0.96)", "cfn-lint (>=0.40.0)", "docker (>=2.5.1)", "ecdsa (!=0.15)", "flask (!=2.2.0,!=2.2.1)", "flask-cors", "graphql-core", "jsondiff (>=1.1.2)", "openapi-spec-validator (>=0.2.8)", "pyparsing (>=3.0.7)", "python-jose[cryptography] (>=3.1.0,<4.0.0)", "setuptools", "sshpubkeys (>=3.1.0)"] +ssm = ["PyYAML (>=5.1)"] +xray = ["aws-xray-sdk (>=0.93,!=0.96)", "setuptools"] + +[[package]] +name = "msal" +version = "1.21.0" +description = "The Microsoft Authentication Library (MSAL) for Python library enables your app to access the Microsoft Cloud by supporting authentication of users with Microsoft Azure Active Directory accounts (AAD) and Microsoft Accounts (MSA) using industry standard OAuth2 and OpenID Connect." +category = "main" +optional = true +python-versions = "*" +files = [ {file = "msal-1.21.0-py2.py3-none-any.whl", hash = "sha256:e8444617c1eccdff7bb73f5d4f94036002accea4a2c05f8f39c9efb5bd2b0c6a"}, {file = "msal-1.21.0.tar.gz", hash = "sha256:96b5c867830fd116e5f7d0ec8ef1b238b4cda4d1aea86d8fecf518260e136fbf"}, ] -msal-extensions = [ + +[package.dependencies] +cryptography = ">=0.6,<41" +PyJWT = {version = ">=1.0.0,<3", extras = ["crypto"]} +requests = ">=2.0.0,<3" + +[package.extras] +broker = ["pymsalruntime (>=0.13.2,<0.14)"] + +[[package]] +name = "msal-extensions" +version = "1.0.0" +description = "Microsoft Authentication Library extensions (MSAL EX) provides a persistence API that can save your data on disk, encrypted on Windows, macOS and Linux. Concurrent data access will be coordinated by a file lock mechanism." +category = "main" +optional = true +python-versions = "*" +files = [ {file = "msal-extensions-1.0.0.tar.gz", hash = "sha256:c676aba56b0cce3783de1b5c5ecfe828db998167875126ca4b47dc6436451354"}, {file = "msal_extensions-1.0.0-py2.py3-none-any.whl", hash = "sha256:91e3db9620b822d0ed2b4d1850056a0f133cba04455e62f11612e40f5502f2ee"}, ] -msrest = [ + +[package.dependencies] +msal = ">=0.4.1,<2.0.0" +portalocker = [ + {version = ">=1.0,<3", markers = "python_version >= \"3.5\" and platform_system != \"Windows\""}, + {version = ">=1.6,<3", markers = "python_version >= \"3.5\" and platform_system == \"Windows\""}, +] + +[[package]] +name = "msrest" +version = "0.7.1" +description = "AutoRest swagger generator Python client runtime." +category = "main" +optional = true +python-versions = ">=3.6" +files = [ {file = "msrest-0.7.1-py3-none-any.whl", hash = "sha256:21120a810e1233e5e6cc7fe40b474eeb4ec6f757a15d7cf86702c369f9567c32"}, {file = "msrest-0.7.1.zip", hash = "sha256:6e7661f46f3afd88b75667b7187a92829924446c7ea1d169be8c4bb7eeb788b9"}, ] -multidict = [ + +[package.dependencies] +azure-core = ">=1.24.0" +certifi = ">=2017.4.17" +isodate = ">=0.6.0" +requests = ">=2.16,<3.0" +requests-oauthlib = ">=0.5.0" + +[package.extras] +async = ["aiodns", "aiohttp (>=3.0)"] + +[[package]] +name = "multidict" +version = "6.0.4" +description = "multidict implementation" +category = "main" +optional = true +python-versions = ">=3.7" +files = [ {file = "multidict-6.0.4-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:0b1a97283e0c85772d613878028fec909f003993e1007eafa715b24b377cb9b8"}, {file = "multidict-6.0.4-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:eeb6dcc05e911516ae3d1f207d4b0520d07f54484c49dfc294d6e7d63b734171"}, {file = "multidict-6.0.4-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:d6d635d5209b82a3492508cf5b365f3446afb65ae7ebd755e70e18f287b0adf7"}, @@ -1962,11 +1413,30 @@ multidict = [ {file = "multidict-6.0.4-cp39-cp39-win_amd64.whl", hash = "sha256:33029f5734336aa0d4c0384525da0387ef89148dc7191aae00ca5fb23d7aafc2"}, {file = "multidict-6.0.4.tar.gz", hash = "sha256:3666906492efb76453c0e7b97f2cf459b0682e7402c0489a95484965dbc1da49"}, ] -nodeenv = [ + +[[package]] +name = "nodeenv" +version = "1.7.0" +description = "Node.js virtual environment builder" +category = "dev" +optional = false +python-versions = ">=2.7,!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,!=3.4.*,!=3.5.*,!=3.6.*" +files = [ {file = "nodeenv-1.7.0-py2.py3-none-any.whl", hash = "sha256:27083a7b96a25f2f5e1d8cb4b6317ee8aeda3bdd121394e5ac54e498028a042e"}, {file = "nodeenv-1.7.0.tar.gz", hash = "sha256:e0e7f7dfb85fc5394c6fe1e8fa98131a2473e04311a45afb6508f7cf1836fa2b"}, ] -numpy = [ + +[package.dependencies] +setuptools = "*" + +[[package]] +name = "numpy" +version = "1.24.2" +description = "Fundamental package for array computing in Python" +category = "main" +optional = true +python-versions = ">=3.8" +files = [ {file = "numpy-1.24.2-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:eef70b4fc1e872ebddc38cddacc87c19a3709c0e3e5d20bf3954c147b1dd941d"}, {file = "numpy-1.24.2-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:e8d2859428712785e8a8b7d2b3ef0a1d1565892367b32f915c4a4df44d0e64f5"}, {file = "numpy-1.24.2-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:6524630f71631be2dabe0c541e7675db82651eb998496bbe16bc4f77f0772253"}, @@ -1996,15 +1466,44 @@ numpy = [ {file = "numpy-1.24.2-pp38-pypy38_pp73-win_amd64.whl", hash = "sha256:150947adbdfeceec4e5926d956a06865c1c690f2fd902efede4ca6fe2e657c3f"}, {file = "numpy-1.24.2.tar.gz", hash = "sha256:003a9f530e880cb2cd177cba1af7220b9aa42def9c4afc2a2fc3ee6be7eb2b22"}, ] -oauthlib = [ + +[[package]] +name = "oauthlib" +version = "3.2.2" +description = "A generic, spec-compliant, thorough implementation of the OAuth request-signing logic" +category = "main" +optional = true +python-versions = ">=3.6" +files = [ {file = "oauthlib-3.2.2-py3-none-any.whl", hash = "sha256:8139f29aac13e25d502680e9e19963e83f16838d48a0d71c287fe40e7067fbca"}, {file = "oauthlib-3.2.2.tar.gz", hash = "sha256:9859c40929662bec5d64f34d01c99e093149682a3f38915dc0655d5a633dd918"}, ] -packaging = [ + +[package.extras] +rsa = ["cryptography (>=3.0.0)"] +signals = ["blinker (>=1.4.0)"] +signedtoken = ["cryptography (>=3.0.0)", "pyjwt (>=2.0.0,<3)"] + +[[package]] +name = "packaging" +version = "23.0" +description = "Core utilities for Python packages" +category = "dev" +optional = false +python-versions = ">=3.7" +files = [ {file = "packaging-23.0-py3-none-any.whl", hash = "sha256:714ac14496c3e68c99c29b00845f7a2b85f3bb6f1078fd9f72fd20f0570002b2"}, {file = "packaging-23.0.tar.gz", hash = "sha256:b6ad297f8907de0fa2fe1ccbd26fdaf387f5f47c7275fedf8cce89f99446cf97"}, ] -pandas = [ + +[[package]] +name = "pandas" +version = "1.5.3" +description = "Powerful data structures for data analysis, time series, and statistics" +category = "main" +optional = true +python-versions = ">=3.8" +files = [ {file = "pandas-1.5.3-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:3749077d86e3a2f0ed51367f30bf5b82e131cc0f14260c4d3e499186fccc4406"}, {file = "pandas-1.5.3-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:972d8a45395f2a2d26733eb8d0f629b2f90bebe8e8eddbb8829b180c09639572"}, {file = "pandas-1.5.3-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:50869a35cbb0f2e0cd5ec04b191e7b12ed688874bd05dd777c19b28cbea90996"}, @@ -2033,23 +1532,98 @@ pandas = [ {file = "pandas-1.5.3-cp39-cp39-win_amd64.whl", hash = "sha256:dfd681c5dc216037e0b0a2c821f5ed99ba9f03ebcf119c7dac0e9a7b960b9ec9"}, {file = "pandas-1.5.3.tar.gz", hash = "sha256:74a3fd7e5a7ec052f183273dc7b0acd3a863edf7520f5d3a1765c04ffdb3b0b1"}, ] -platformdirs = [ + +[package.dependencies] +numpy = [ + {version = ">=1.20.3", markers = "python_version < \"3.10\""}, + {version = ">=1.21.0", markers = "python_version >= \"3.10\""}, + {version = ">=1.23.2", markers = "python_version >= \"3.11\""}, +] +python-dateutil = ">=2.8.1" +pytz = ">=2020.1" + +[package.extras] +test = ["hypothesis (>=5.5.3)", "pytest (>=6.0)", "pytest-xdist (>=1.31)"] + +[[package]] +name = "platformdirs" +version = "3.0.0" +description = "A small Python package for determining appropriate platform-specific dirs, e.g. a \"user data dir\"." +category = "dev" +optional = false +python-versions = ">=3.7" +files = [ {file = "platformdirs-3.0.0-py3-none-any.whl", hash = "sha256:b1d5eb14f221506f50d6604a561f4c5786d9e80355219694a1b244bcd96f4567"}, {file = "platformdirs-3.0.0.tar.gz", hash = "sha256:8a1228abb1ef82d788f74139988b137e78692984ec7b08eaa6c65f1723af28f9"}, ] -pluggy = [ + +[package.extras] +docs = ["furo (>=2022.12.7)", "proselint (>=0.13)", "sphinx (>=6.1.3)", "sphinx-autodoc-typehints (>=1.22,!=1.23.4)"] +test = ["appdirs (==1.4.4)", "covdefaults (>=2.2.2)", "pytest (>=7.2.1)", "pytest-cov (>=4)", "pytest-mock (>=3.10)"] + +[[package]] +name = "pluggy" +version = "1.0.0" +description = "plugin and hook calling mechanisms for python" +category = "dev" +optional = false +python-versions = ">=3.6" +files = [ {file = "pluggy-1.0.0-py2.py3-none-any.whl", hash = "sha256:74134bbf457f031a36d68416e1509f34bd5ccc019f0bcc952c7b909d06b37bd3"}, {file = "pluggy-1.0.0.tar.gz", hash = "sha256:4224373bacce55f955a878bf9cfa763c1e360858e330072059e10bad68531159"}, ] -portalocker = [ + +[package.extras] +dev = ["pre-commit", "tox"] +testing = ["pytest", "pytest-benchmark"] + +[[package]] +name = "portalocker" +version = "2.7.0" +description = "Wraps the portalocker recipe for easy usage" +category = "main" +optional = true +python-versions = ">=3.5" +files = [ {file = "portalocker-2.7.0-py2.py3-none-any.whl", hash = "sha256:a07c5b4f3985c3cf4798369631fb7011adb498e2a46d8440efc75a8f29a0f983"}, {file = "portalocker-2.7.0.tar.gz", hash = "sha256:032e81d534a88ec1736d03f780ba073f047a06c478b06e2937486f334e955c51"}, ] -pre-commit = [ + +[package.dependencies] +pywin32 = {version = ">=226", markers = "platform_system == \"Windows\""} + +[package.extras] +docs = ["sphinx (>=1.7.1)"] +redis = ["redis"] +tests = ["pytest (>=5.4.1)", "pytest-cov (>=2.8.1)", "pytest-mypy (>=0.8.0)", "pytest-timeout (>=2.1.0)", "redis", "sphinx (>=6.0.0)"] + +[[package]] +name = "pre-commit" +version = "3.0.4" +description = "A framework for managing and maintaining multi-language pre-commit hooks." +category = "dev" +optional = false +python-versions = ">=3.8" +files = [ {file = "pre_commit-3.0.4-py2.py3-none-any.whl", hash = "sha256:9e3255edb0c9e7fe9b4f328cb3dc86069f8fdc38026f1bf521018a05eaf4d67b"}, {file = "pre_commit-3.0.4.tar.gz", hash = "sha256:bc4687478d55578c4ac37272fe96df66f73d9b5cf81be6f28627d4e712e752d5"}, ] -pyarrow = [ + +[package.dependencies] +cfgv = ">=2.0.0" +identify = ">=1.0.0" +nodeenv = ">=0.11.1" +pyyaml = ">=5.1" +virtualenv = ">=20.10.0" + +[[package]] +name = "pyarrow" +version = "11.0.0" +description = "Python library for Apache Arrow" +category = "main" +optional = true +python-versions = ">=3.7" +files = [ {file = "pyarrow-11.0.0-cp310-cp310-macosx_10_14_x86_64.whl", hash = "sha256:40bb42afa1053c35c749befbe72f6429b7b5f45710e85059cdd534553ebcf4f2"}, {file = "pyarrow-11.0.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:7c28b5f248e08dea3b3e0c828b91945f431f4202f1a9fe84d1012a761324e1ba"}, {file = "pyarrow-11.0.0-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a37bc81f6c9435da3c9c1e767324ac3064ffbe110c4e460660c43e144be4ed85"}, @@ -2076,74 +1650,209 @@ pyarrow = [ {file = "pyarrow-11.0.0-cp39-cp39-win_amd64.whl", hash = "sha256:caad867121f182d0d3e1a0d36f197df604655d0b466f1bc9bafa903aa95083e4"}, {file = "pyarrow-11.0.0.tar.gz", hash = "sha256:5461c57dbdb211a632a48facb9b39bbeb8a7905ec95d768078525283caef5f6d"}, ] -pycparser = [ + +[package.dependencies] +numpy = ">=1.16.6" + +[[package]] +name = "pycparser" +version = "2.21" +description = "C parser in Python" +category = "main" +optional = false +python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*" +files = [ {file = "pycparser-2.21-py2.py3-none-any.whl", hash = "sha256:8ee45429555515e1f6b185e78100aea234072576aa43ab53aefcae078162fca9"}, {file = "pycparser-2.21.tar.gz", hash = "sha256:e644fdec12f7872f86c58ff790da456218b10f863970249516d60a5eaca77206"}, ] -pydantic = [ - {file = "pydantic-1.10.4-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:b5635de53e6686fe7a44b5cf25fcc419a0d5e5c1a1efe73d49d48fe7586db854"}, - {file = "pydantic-1.10.4-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:6dc1cc241440ed7ca9ab59d9929075445da6b7c94ced281b3dd4cfe6c8cff817"}, - {file = "pydantic-1.10.4-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:51bdeb10d2db0f288e71d49c9cefa609bca271720ecd0c58009bd7504a0c464c"}, - {file = "pydantic-1.10.4-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:78cec42b95dbb500a1f7120bdf95c401f6abb616bbe8785ef09887306792e66e"}, - {file = "pydantic-1.10.4-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:8775d4ef5e7299a2f4699501077a0defdaac5b6c4321173bcb0f3c496fbadf85"}, - {file = "pydantic-1.10.4-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:572066051eeac73d23f95ba9a71349c42a3e05999d0ee1572b7860235b850cc6"}, - {file = "pydantic-1.10.4-cp310-cp310-win_amd64.whl", hash = "sha256:7feb6a2d401f4d6863050f58325b8d99c1e56f4512d98b11ac64ad1751dc647d"}, - {file = "pydantic-1.10.4-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:39f4a73e5342b25c2959529f07f026ef58147249f9b7431e1ba8414a36761f53"}, - {file = "pydantic-1.10.4-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:983e720704431a6573d626b00662eb78a07148c9115129f9b4351091ec95ecc3"}, - {file = "pydantic-1.10.4-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:75d52162fe6b2b55964fbb0af2ee58e99791a3138588c482572bb6087953113a"}, - {file = "pydantic-1.10.4-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:fdf8d759ef326962b4678d89e275ffc55b7ce59d917d9f72233762061fd04a2d"}, - {file = "pydantic-1.10.4-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:05a81b006be15655b2a1bae5faa4280cf7c81d0e09fcb49b342ebf826abe5a72"}, - {file = "pydantic-1.10.4-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:d88c4c0e5c5dfd05092a4b271282ef0588e5f4aaf345778056fc5259ba098857"}, - {file = "pydantic-1.10.4-cp311-cp311-win_amd64.whl", hash = "sha256:6a05a9db1ef5be0fe63e988f9617ca2551013f55000289c671f71ec16f4985e3"}, - {file = "pydantic-1.10.4-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:887ca463c3bc47103c123bc06919c86720e80e1214aab79e9b779cda0ff92a00"}, - {file = "pydantic-1.10.4-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:fdf88ab63c3ee282c76d652fc86518aacb737ff35796023fae56a65ced1a5978"}, - {file = "pydantic-1.10.4-cp37-cp37m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:a48f1953c4a1d9bd0b5167ac50da9a79f6072c63c4cef4cf2a3736994903583e"}, - {file = "pydantic-1.10.4-cp37-cp37m-musllinux_1_1_i686.whl", hash = "sha256:a9f2de23bec87ff306aef658384b02aa7c32389766af3c5dee9ce33e80222dfa"}, - {file = "pydantic-1.10.4-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:cd8702c5142afda03dc2b1ee6bc358b62b3735b2cce53fc77b31ca9f728e4bc8"}, - {file = "pydantic-1.10.4-cp37-cp37m-win_amd64.whl", hash = "sha256:6e7124d6855b2780611d9f5e1e145e86667eaa3bd9459192c8dc1a097f5e9903"}, - {file = "pydantic-1.10.4-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:0b53e1d41e97063d51a02821b80538053ee4608b9a181c1005441f1673c55423"}, - {file = "pydantic-1.10.4-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:55b1625899acd33229c4352ce0ae54038529b412bd51c4915349b49ca575258f"}, - {file = "pydantic-1.10.4-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:301d626a59edbe5dfb48fcae245896379a450d04baeed50ef40d8199f2733b06"}, - {file = "pydantic-1.10.4-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:b6f9d649892a6f54a39ed56b8dfd5e08b5f3be5f893da430bed76975f3735d15"}, - {file = "pydantic-1.10.4-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:d7b5a3821225f5c43496c324b0d6875fde910a1c2933d726a743ce328fbb2a8c"}, - {file = "pydantic-1.10.4-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:f2f7eb6273dd12472d7f218e1fef6f7c7c2f00ac2e1ecde4db8824c457300416"}, - {file = "pydantic-1.10.4-cp38-cp38-win_amd64.whl", hash = "sha256:4b05697738e7d2040696b0a66d9f0a10bec0efa1883ca75ee9e55baf511909d6"}, - {file = "pydantic-1.10.4-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:a9a6747cac06c2beb466064dda999a13176b23535e4c496c9d48e6406f92d42d"}, - {file = "pydantic-1.10.4-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:eb992a1ef739cc7b543576337bebfc62c0e6567434e522e97291b251a41dad7f"}, - {file = "pydantic-1.10.4-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:990406d226dea0e8f25f643b370224771878142155b879784ce89f633541a024"}, - {file = "pydantic-1.10.4-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:2e82a6d37a95e0b1b42b82ab340ada3963aea1317fd7f888bb6b9dfbf4fff57c"}, - {file = "pydantic-1.10.4-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:9193d4f4ee8feca58bc56c8306bcb820f5c7905fd919e0750acdeeeef0615b28"}, - {file = "pydantic-1.10.4-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:2b3ce5f16deb45c472dde1a0ee05619298c864a20cded09c4edd820e1454129f"}, - {file = "pydantic-1.10.4-cp39-cp39-win_amd64.whl", hash = "sha256:9cbdc268a62d9a98c56e2452d6c41c0263d64a2009aac69246486f01b4f594c4"}, - {file = "pydantic-1.10.4-py3-none-any.whl", hash = "sha256:4948f264678c703f3877d1c8877c4e3b2e12e549c57795107f08cf70c6ec7774"}, - {file = "pydantic-1.10.4.tar.gz", hash = "sha256:b9a3859f24eb4e097502a3be1fb4b2abb79b6103dd9e2e0edb70613a4459a648"}, -] -pygments = [ + +[[package]] +name = "pydantic" +version = "1.10.5" +description = "Data validation and settings management using python type hints" +category = "main" +optional = false +python-versions = ">=3.7" +files = [ + {file = "pydantic-1.10.5-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:5920824fe1e21cbb3e38cf0f3dd24857c8959801d1031ce1fac1d50857a03bfb"}, + {file = "pydantic-1.10.5-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:3bb99cf9655b377db1a9e47fa4479e3330ea96f4123c6c8200e482704bf1eda2"}, + {file = "pydantic-1.10.5-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:2185a3b3d98ab4506a3f6707569802d2d92c3a7ba3a9a35683a7709ea6c2aaa2"}, + {file = "pydantic-1.10.5-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:f582cac9d11c227c652d3ce8ee223d94eb06f4228b52a8adaafa9fa62e73d5c9"}, + {file = "pydantic-1.10.5-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:c9e5b778b6842f135902e2d82624008c6a79710207e28e86966cd136c621bfee"}, + {file = "pydantic-1.10.5-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:72ef3783be8cbdef6bca034606a5de3862be6b72415dc5cb1fb8ddbac110049a"}, + {file = "pydantic-1.10.5-cp310-cp310-win_amd64.whl", hash = "sha256:45edea10b75d3da43cfda12f3792833a3fa70b6eee4db1ed6aed528cef17c74e"}, + {file = "pydantic-1.10.5-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:63200cd8af1af2c07964546b7bc8f217e8bda9d0a2ef0ee0c797b36353914984"}, + {file = "pydantic-1.10.5-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:305d0376c516b0dfa1dbefeae8c21042b57b496892d721905a6ec6b79494a66d"}, + {file = "pydantic-1.10.5-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:1fd326aff5d6c36f05735c7c9b3d5b0e933b4ca52ad0b6e4b38038d82703d35b"}, + {file = "pydantic-1.10.5-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:6bb0452d7b8516178c969d305d9630a3c9b8cf16fcf4713261c9ebd465af0d73"}, + {file = "pydantic-1.10.5-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:9a9d9155e2a9f38b2eb9374c88f02fd4d6851ae17b65ee786a87d032f87008f8"}, + {file = "pydantic-1.10.5-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:f836444b4c5ece128b23ec36a446c9ab7f9b0f7981d0d27e13a7c366ee163f8a"}, + {file = "pydantic-1.10.5-cp311-cp311-win_amd64.whl", hash = "sha256:8481dca324e1c7b715ce091a698b181054d22072e848b6fc7895cd86f79b4449"}, + {file = "pydantic-1.10.5-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:87f831e81ea0589cd18257f84386bf30154c5f4bed373b7b75e5cb0b5d53ea87"}, + {file = "pydantic-1.10.5-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:7ce1612e98c6326f10888df951a26ec1a577d8df49ddcaea87773bfbe23ba5cc"}, + {file = "pydantic-1.10.5-cp37-cp37m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:58e41dd1e977531ac6073b11baac8c013f3cd8706a01d3dc74e86955be8b2c0c"}, + {file = "pydantic-1.10.5-cp37-cp37m-musllinux_1_1_i686.whl", hash = "sha256:6a4b0aab29061262065bbdede617ef99cc5914d1bf0ddc8bcd8e3d7928d85bd6"}, + {file = "pydantic-1.10.5-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:36e44a4de37b8aecffa81c081dbfe42c4d2bf9f6dff34d03dce157ec65eb0f15"}, + {file = "pydantic-1.10.5-cp37-cp37m-win_amd64.whl", hash = "sha256:261f357f0aecda005934e413dfd7aa4077004a174dafe414a8325e6098a8e419"}, + {file = "pydantic-1.10.5-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:b429f7c457aebb7fbe7cd69c418d1cd7c6fdc4d3c8697f45af78b8d5a7955760"}, + {file = "pydantic-1.10.5-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:663d2dd78596c5fa3eb996bc3f34b8c2a592648ad10008f98d1348be7ae212fb"}, + {file = "pydantic-1.10.5-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:51782fd81f09edcf265823c3bf43ff36d00db246eca39ee765ef58dc8421a642"}, + {file = "pydantic-1.10.5-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:c428c0f64a86661fb4873495c4fac430ec7a7cef2b8c1c28f3d1a7277f9ea5ab"}, + {file = "pydantic-1.10.5-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:76c930ad0746c70f0368c4596020b736ab65b473c1f9b3872310a835d852eb19"}, + {file = "pydantic-1.10.5-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:3257bd714de9db2102b742570a56bf7978e90441193acac109b1f500290f5718"}, + {file = "pydantic-1.10.5-cp38-cp38-win_amd64.whl", hash = "sha256:f5bee6c523d13944a1fdc6f0525bc86dbbd94372f17b83fa6331aabacc8fd08e"}, + {file = "pydantic-1.10.5-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:532e97c35719f137ee5405bd3eeddc5c06eb91a032bc755a44e34a712420daf3"}, + {file = "pydantic-1.10.5-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:ca9075ab3de9e48b75fa8ccb897c34ccc1519177ad8841d99f7fd74cf43be5bf"}, + {file = "pydantic-1.10.5-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:bd46a0e6296346c477e59a954da57beaf9c538da37b9df482e50f836e4a7d4bb"}, + {file = "pydantic-1.10.5-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:3353072625ea2a9a6c81ad01b91e5c07fa70deb06368c71307529abf70d23325"}, + {file = "pydantic-1.10.5-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:3f9d9b2be177c3cb6027cd67fbf323586417868c06c3c85d0d101703136e6b31"}, + {file = "pydantic-1.10.5-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:b473d00ccd5c2061fd896ac127b7755baad233f8d996ea288af14ae09f8e0d1e"}, + {file = "pydantic-1.10.5-cp39-cp39-win_amd64.whl", hash = "sha256:5f3bc8f103b56a8c88021d481410874b1f13edf6e838da607dcb57ecff9b4594"}, + {file = "pydantic-1.10.5-py3-none-any.whl", hash = "sha256:7c5b94d598c90f2f46b3a983ffb46ab806a67099d118ae0da7ef21a2a4033b28"}, + {file = "pydantic-1.10.5.tar.gz", hash = "sha256:9e337ac83686645a46db0e825acceea8e02fca4062483f40e9ae178e8bd1103a"}, +] + +[package.dependencies] +typing-extensions = ">=4.2.0" + +[package.extras] +dotenv = ["python-dotenv (>=0.10.4)"] +email = ["email-validator (>=1.0.3)"] + +[[package]] +name = "pygments" +version = "2.14.0" +description = "Pygments is a syntax highlighting package written in Python." +category = "main" +optional = false +python-versions = ">=3.6" +files = [ {file = "Pygments-2.14.0-py3-none-any.whl", hash = "sha256:fa7bd7bd2771287c0de303af8bfdfc731f51bd2c6a47ab69d117138893b82717"}, {file = "Pygments-2.14.0.tar.gz", hash = "sha256:b3ed06a9e8ac9a9aae5a6f5dbe78a8a58655d17b43b93c078f094ddc476ae297"}, ] -pyjwt = [ + +[package.extras] +plugins = ["importlib-metadata"] + +[[package]] +name = "pyjwt" +version = "2.6.0" +description = "JSON Web Token implementation in Python" +category = "main" +optional = true +python-versions = ">=3.7" +files = [ {file = "PyJWT-2.6.0-py3-none-any.whl", hash = "sha256:d83c3d892a77bbb74d3e1a2cfa90afaadb60945205d1095d9221f04466f64c14"}, {file = "PyJWT-2.6.0.tar.gz", hash = "sha256:69285c7e31fc44f68a1feb309e948e0df53259d579295e6cfe2b1792329f05fd"}, ] -pyparsing = [ + +[package.dependencies] +cryptography = {version = ">=3.4.0", optional = true, markers = "extra == \"crypto\""} + +[package.extras] +crypto = ["cryptography (>=3.4.0)"] +dev = ["coverage[toml] (==5.0.4)", "cryptography (>=3.4.0)", "pre-commit", "pytest (>=6.0.0,<7.0.0)", "sphinx (>=4.5.0,<5.0.0)", "sphinx-rtd-theme", "zope.interface"] +docs = ["sphinx (>=4.5.0,<5.0.0)", "sphinx-rtd-theme", "zope.interface"] +tests = ["coverage[toml] (==5.0.4)", "pytest (>=6.0.0,<7.0.0)"] + +[[package]] +name = "pyparsing" +version = "3.0.9" +description = "pyparsing module - Classes and methods to define and execute parsing grammars" +category = "main" +optional = false +python-versions = ">=3.6.8" +files = [ {file = "pyparsing-3.0.9-py3-none-any.whl", hash = "sha256:5026bae9a10eeaefb61dab2f09052b9f4307d44aee4eda64b309723d8d206bbc"}, {file = "pyparsing-3.0.9.tar.gz", hash = "sha256:2b020ecf7d21b687f219b71ecad3631f644a47f01403fa1d1036b0c6416d70fb"}, ] -pyproject-hooks = [ + +[package.extras] +diagrams = ["jinja2", "railroad-diagrams"] + +[[package]] +name = "pyproject-hooks" +version = "1.0.0" +description = "Wrappers to call pyproject.toml-based build backend hooks." +category = "dev" +optional = false +python-versions = ">=3.7" +files = [ {file = "pyproject_hooks-1.0.0-py3-none-any.whl", hash = "sha256:283c11acd6b928d2f6a7c73fa0d01cb2bdc5f07c57a2eeb6e83d5e56b97976f8"}, {file = "pyproject_hooks-1.0.0.tar.gz", hash = "sha256:f271b298b97f5955d53fb12b72c1fb1948c22c1a6b70b315c54cedaca0264ef5"}, ] -pytest = [ + +[package.dependencies] +tomli = {version = ">=1.1.0", markers = "python_version < \"3.11\""} + +[[package]] +name = "pytest" +version = "7.2.1" +description = "pytest: simple powerful testing with Python" +category = "dev" +optional = false +python-versions = ">=3.7" +files = [ {file = "pytest-7.2.1-py3-none-any.whl", hash = "sha256:c7c6ca206e93355074ae32f7403e8ea12163b1163c976fee7d4d84027c162be5"}, {file = "pytest-7.2.1.tar.gz", hash = "sha256:d45e0952f3727241918b8fd0f376f5ff6b301cc0777c6f9a556935c92d8a7d42"}, ] -pytest-checkdocs = [ + +[package.dependencies] +attrs = ">=19.2.0" +colorama = {version = "*", markers = "sys_platform == \"win32\""} +exceptiongroup = {version = ">=1.0.0rc8", markers = "python_version < \"3.11\""} +iniconfig = "*" +packaging = "*" +pluggy = ">=0.12,<2.0" +tomli = {version = ">=1.0.0", markers = "python_version < \"3.11\""} + +[package.extras] +testing = ["argcomplete", "hypothesis (>=3.56)", "mock", "nose", "pygments (>=2.7.2)", "requests", "xmlschema"] + +[[package]] +name = "pytest-checkdocs" +version = "2.9.0" +description = "check the README when running tests" +category = "dev" +optional = false +python-versions = ">=3.7" +files = [ {file = "pytest-checkdocs-2.9.0.tar.gz", hash = "sha256:54b3d1357a289e3f6336d31d4ceb72e99afe0fa4f552990f1b1f015f1925543b"}, {file = "pytest_checkdocs-2.9.0-py3-none-any.whl", hash = "sha256:4231e5426915a02385e386b5eb4a70490b143d97cede4c7c77691bb320b5a73f"}, ] -python-dateutil = [] -python-snappy = [ + +[package.dependencies] +build = "*" +docutils = ">=0.15" +importlib-metadata = {version = ">=4", markers = "python_version < \"3.10\""} + +[package.extras] +docs = ["furo", "jaraco.packaging (>=9)", "rst.linker (>=1.9)", "sphinx (>=3.5)"] +testing = ["flake8 (<5)", "pytest (>=6)", "pytest-black (>=0.3.7)", "pytest-checkdocs (>=2.4)", "pytest-cov", "pytest-enabler (>=1.3)", "pytest-flake8", "pytest-mypy (>=0.9.1)", "types-docutils"] + +[[package]] +name = "python-dateutil" +version = "2.8.2" +description = "Extensions to the standard Python datetime module" +category = "main" +optional = false +python-versions = "!=3.0.*,!=3.1.*,!=3.2.*,>=2.7" +files = [ + {file = "python-dateutil-2.8.2.tar.gz", hash = "sha256:0123cacc1627ae19ddf3c27a5de5bd67ee4586fbdd6440d9748f8abb483d3e86"}, + {file = "python_dateutil-2.8.2-py2.py3-none-any.whl", hash = "sha256:961d03dc3453ebbc59dbdea9e4e11c5651520a876d0f4db161e8674aae935da9"}, +] + +[package.dependencies] +six = ">=1.5" + +[[package]] +name = "python-snappy" +version = "0.6.1" +description = "Python library for the snappy compression library from Google" +category = "main" +optional = true +python-versions = "*" +files = [ {file = "python-snappy-0.6.1.tar.gz", hash = "sha256:b6a107ab06206acc5359d4c5632bd9b22d448702a79b3169b0c62e0fb808bb2a"}, {file = "python_snappy-0.6.1-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:b7f920eaf46ebf41bd26f9df51c160d40f9e00b7b48471c3438cb8d027f7fb9b"}, {file = "python_snappy-0.6.1-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:4ec533a8c1f8df797bded662ec3e494d225b37855bb63eb0d75464a07947477c"}, @@ -2193,11 +1902,27 @@ python-snappy = [ {file = "python_snappy-0.6.1-pp37-pypy37_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:0bdb6942180660bda7f7d01f4c0def3cfc72b1c6d99aad964801775a3e379aba"}, {file = "python_snappy-0.6.1-pp37-pypy37_pp73-win_amd64.whl", hash = "sha256:03bb511380fca2a13325b6f16fe8234c8e12da9660f0258cd45d9a02ffc916af"}, ] -pytz = [ + +[[package]] +name = "pytz" +version = "2022.7.1" +description = "World timezone definitions, modern and historical" +category = "main" +optional = true +python-versions = "*" +files = [ {file = "pytz-2022.7.1-py2.py3-none-any.whl", hash = "sha256:78f4f37d8198e0627c5f1143240bb0206b8691d8d7ac6d78fee88b78733f8c4a"}, {file = "pytz-2022.7.1.tar.gz", hash = "sha256:01a0681c4b9684a28304615eba55d1ab31ae00bf68ec157ec3708a8182dbbcd0"}, ] -pywin32 = [ + +[[package]] +name = "pywin32" +version = "305" +description = "Python for Window Extensions" +category = "main" +optional = true +python-versions = "*" +files = [ {file = "pywin32-305-cp310-cp310-win32.whl", hash = "sha256:421f6cd86e84bbb696d54563c48014b12a23ef95a14e0bdba526be756d89f116"}, {file = "pywin32-305-cp310-cp310-win_amd64.whl", hash = "sha256:73e819c6bed89f44ff1d690498c0a811948f73777e5f97c494c152b850fad478"}, {file = "pywin32-305-cp310-cp310-win_arm64.whl", hash = "sha256:742eb905ce2187133a29365b428e6c3b9001d79accdc30aa8969afba1d8470f4"}, @@ -2213,7 +1938,15 @@ pywin32 = [ {file = "pywin32-305-cp39-cp39-win32.whl", hash = "sha256:9d968c677ac4d5cbdaa62fd3014ab241718e619d8e36ef8e11fb930515a1e918"}, {file = "pywin32-305-cp39-cp39-win_amd64.whl", hash = "sha256:50768c6b7c3f0b38b7fb14dd4104da93ebced5f1a50dc0e834594bff6fbe1271"}, ] -pyyaml = [ + +[[package]] +name = "pyyaml" +version = "6.0" +description = "YAML parser and emitter for Python" +category = "main" +optional = false +python-versions = ">=3.6" +files = [ {file = "PyYAML-6.0-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:d4db7c7aef085872ef65a8fd7d6d09a14ae91f691dec3e87ee5ee0539d516f53"}, {file = "PyYAML-6.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:9df7ed3b3d2e0ecfe09e14741b857df43adb5a3ddadc919a2d94fbdf78fea53c"}, {file = "PyYAML-6.0-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:77f396e6ef4c73fdc33a9157446466f1cff553d979bd00ecb64385760c6babdc"}, @@ -2221,6 +1954,13 @@ pyyaml = [ {file = "PyYAML-6.0-cp310-cp310-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:f84fbc98b019fef2ee9a1cb3ce93e3187a6df0b2538a651bfb890254ba9f90b5"}, {file = "PyYAML-6.0-cp310-cp310-win32.whl", hash = "sha256:2cd5df3de48857ed0544b34e2d40e9fac445930039f3cfe4bcc592a1f836d513"}, {file = "PyYAML-6.0-cp310-cp310-win_amd64.whl", hash = "sha256:daf496c58a8c52083df09b80c860005194014c3698698d1a57cbcfa182142a3a"}, + {file = "PyYAML-6.0-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:d4b0ba9512519522b118090257be113b9468d804b19d63c71dbcf4a48fa32358"}, + {file = "PyYAML-6.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:81957921f441d50af23654aa6c5e5eaf9b06aba7f0a19c18a538dc7ef291c5a1"}, + {file = "PyYAML-6.0-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:afa17f5bc4d1b10afd4466fd3a44dc0e245382deca5b3c353d8b757f9e3ecb8d"}, + {file = "PyYAML-6.0-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:dbad0e9d368bb989f4515da330b88a057617d16b6a8245084f1b05400f24609f"}, + {file = "PyYAML-6.0-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:432557aa2c09802be39460360ddffd48156e30721f5e8d917f01d31694216782"}, + {file = "PyYAML-6.0-cp311-cp311-win32.whl", hash = "sha256:bfaef573a63ba8923503d27530362590ff4f576c626d86a9fed95822a8255fd7"}, + {file = "PyYAML-6.0-cp311-cp311-win_amd64.whl", hash = "sha256:01b45c0191e6d66c470b6cf1b9531a771a83c1c4208272ead47a3ae4f2f603bf"}, {file = "PyYAML-6.0-cp36-cp36m-macosx_10_9_x86_64.whl", hash = "sha256:897b80890765f037df3403d22bab41627ca8811ae55e9a722fd0392850ec4d86"}, {file = "PyYAML-6.0-cp36-cp36m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:50602afada6d6cbfad699b0c7bb50d5ccffa7e46a3d738092afddc1f9758427f"}, {file = "PyYAML-6.0-cp36-cp36m-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:48c346915c114f5fdb3ead70312bd042a953a8ce5c7106d5bfb1a5254e47da92"}, @@ -2248,65 +1988,308 @@ pyyaml = [ {file = "PyYAML-6.0-cp39-cp39-win_amd64.whl", hash = "sha256:b3d267842bf12586ba6c734f89d1f5b871df0273157918b0ccefa29deb05c21c"}, {file = "PyYAML-6.0.tar.gz", hash = "sha256:68fb519c14306fec9720a2a5b45bc9f0c8d1b9c72adf45c37baedfcd949c35a2"}, ] -requests = [ + +[[package]] +name = "requests" +version = "2.28.2" +description = "Python HTTP for Humans." +category = "main" +optional = false +python-versions = ">=3.7, <4" +files = [ {file = "requests-2.28.2-py3-none-any.whl", hash = "sha256:64299f4909223da747622c030b781c0d7811e359c37124b4bd368fb8c6518baa"}, {file = "requests-2.28.2.tar.gz", hash = "sha256:98b1b2782e3c6c4904938b84c0eb932721069dfdb9134313beff7c83c2df24bf"}, ] -requests-mock = [ + +[package.dependencies] +certifi = ">=2017.4.17" +charset-normalizer = ">=2,<4" +idna = ">=2.5,<4" +urllib3 = ">=1.21.1,<1.27" + +[package.extras] +socks = ["PySocks (>=1.5.6,!=1.5.7)"] +use-chardet-on-py3 = ["chardet (>=3.0.2,<6)"] + +[[package]] +name = "requests-mock" +version = "1.10.0" +description = "Mock out responses from the requests package" +category = "dev" +optional = false +python-versions = "*" +files = [ {file = "requests-mock-1.10.0.tar.gz", hash = "sha256:59c9c32419a9fb1ae83ec242d98e889c45bd7d7a65d48375cc243ec08441658b"}, {file = "requests_mock-1.10.0-py2.py3-none-any.whl", hash = "sha256:2fdbb637ad17ee15c06f33d31169e71bf9fe2bdb7bc9da26185be0dd8d842699"}, ] -requests-oauthlib = [ + +[package.dependencies] +requests = ">=2.3,<3" +six = "*" + +[package.extras] +fixture = ["fixtures"] +test = ["fixtures", "mock", "purl", "pytest", "requests-futures", "sphinx", "testrepository (>=0.0.18)", "testtools"] + +[[package]] +name = "requests-oauthlib" +version = "1.3.1" +description = "OAuthlib authentication support for Requests." +category = "main" +optional = true +python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*" +files = [ {file = "requests-oauthlib-1.3.1.tar.gz", hash = "sha256:75beac4a47881eeb94d5ea5d6ad31ef88856affe2332b9aafb52c6452ccf0d7a"}, {file = "requests_oauthlib-1.3.1-py2.py3-none-any.whl", hash = "sha256:2577c501a2fb8d05a304c09d090d6e47c306fef15809d102b327cf8364bddab5"}, ] -responses = [ + +[package.dependencies] +oauthlib = ">=3.0.0" +requests = ">=2.0.0" + +[package.extras] +rsa = ["oauthlib[signedtoken] (>=3.0.0)"] + +[[package]] +name = "responses" +version = "0.22.0" +description = "A utility library for mocking out the `requests` Python library." +category = "dev" +optional = false +python-versions = ">=3.7" +files = [ {file = "responses-0.22.0-py3-none-any.whl", hash = "sha256:dcf294d204d14c436fddcc74caefdbc5764795a40ff4e6a7740ed8ddbf3294be"}, {file = "responses-0.22.0.tar.gz", hash = "sha256:396acb2a13d25297789a5866b4881cf4e46ffd49cc26c43ab1117f40b973102e"}, ] -rich = [ + +[package.dependencies] +requests = ">=2.22.0,<3.0" +toml = "*" +types-toml = "*" +urllib3 = ">=1.25.10" + +[package.extras] +tests = ["coverage (>=6.0.0)", "flake8", "mypy", "pytest (>=7.0.0)", "pytest-asyncio", "pytest-cov", "pytest-httpserver", "types-requests"] + +[[package]] +name = "rich" +version = "13.3.1" +description = "Render rich text, tables, progress bars, syntax highlighting, markdown and more to the terminal" +category = "main" +optional = false +python-versions = ">=3.7.0" +files = [ {file = "rich-13.3.1-py3-none-any.whl", hash = "sha256:8aa57747f3fc3e977684f0176a88e789be314a99f99b43b75d1e9cb5dc6db9e9"}, {file = "rich-13.3.1.tar.gz", hash = "sha256:125d96d20c92b946b983d0d392b84ff945461e5a06d3867e9f9e575f8697b67f"}, ] -s3fs = [ + +[package.dependencies] +markdown-it-py = ">=2.1.0,<3.0.0" +pygments = ">=2.14.0,<3.0.0" +typing-extensions = {version = ">=4.0.0,<5.0", markers = "python_version < \"3.9\""} + +[package.extras] +jupyter = ["ipywidgets (>=7.5.1,<9)"] + +[[package]] +name = "s3fs" +version = "2023.1.0" +description = "Convenient Filesystem interface over S3" +category = "main" +optional = true +python-versions = ">= 3.7" +files = [ {file = "s3fs-2023.1.0-py3-none-any.whl", hash = "sha256:a549ae518fff4388bd6fca5248575c29f521e7e39efcd2bfab476651701fd114"}, {file = "s3fs-2023.1.0.tar.gz", hash = "sha256:8b2e28372423e93f26312208a9272e22a962ddd0a79d63f9a68693b6af5ff187"}, ] -s3transfer = [] -six = [ + +[package.dependencies] +aiobotocore = ">=2.4.2,<2.5.0" +aiohttp = "<4.0.0a0 || >4.0.0a0,<4.0.0a1 || >4.0.0a1" +fsspec = "2023.1.0" + +[package.extras] +awscli = ["aiobotocore[awscli] (>=2.4.2,<2.5.0)"] +boto3 = ["aiobotocore[boto3] (>=2.4.2,<2.5.0)"] + +[[package]] +name = "s3transfer" +version = "0.6.0" +description = "An Amazon S3 Transfer Manager" +category = "main" +optional = false +python-versions = ">= 3.7" +files = [ + {file = "s3transfer-0.6.0-py3-none-any.whl", hash = "sha256:06176b74f3a15f61f1b4f25a1fc29a4429040b7647133a463da8fa5bd28d5ecd"}, + {file = "s3transfer-0.6.0.tar.gz", hash = "sha256:2ed07d3866f523cc561bf4a00fc5535827981b117dd7876f036b0c1aca42c947"}, +] + +[package.dependencies] +botocore = ">=1.12.36,<2.0a.0" + +[package.extras] +crt = ["botocore[crt] (>=1.20.29,<2.0a.0)"] + +[[package]] +name = "setuptools" +version = "67.3.2" +description = "Easily download, build, install, upgrade, and uninstall Python packages" +category = "dev" +optional = false +python-versions = ">=3.7" +files = [ + {file = "setuptools-67.3.2-py3-none-any.whl", hash = "sha256:bb6d8e508de562768f2027902929f8523932fcd1fb784e6d573d2cafac995a48"}, + {file = "setuptools-67.3.2.tar.gz", hash = "sha256:95f00380ef2ffa41d9bba85d95b27689d923c93dfbafed4aecd7cf988a25e012"}, +] + +[package.extras] +docs = ["furo", "jaraco.packaging (>=9)", "jaraco.tidelift (>=1.4)", "pygments-github-lexers (==0.0.5)", "rst.linker (>=1.9)", "sphinx (>=3.5)", "sphinx-favicon", "sphinx-hoverxref (<2)", "sphinx-inline-tabs", "sphinx-lint", "sphinx-notfound-page (==0.8.3)", "sphinx-reredirects", "sphinxcontrib-towncrier"] +testing = ["build[virtualenv]", "filelock (>=3.4.0)", "flake8 (<5)", "flake8-2020", "ini2toml[lite] (>=0.9)", "jaraco.envs (>=2.2)", "jaraco.path (>=3.2.0)", "pip (>=19.1)", "pip-run (>=8.8)", "pytest (>=6)", "pytest-black (>=0.3.7)", "pytest-checkdocs (>=2.4)", "pytest-cov", "pytest-enabler (>=1.3)", "pytest-flake8", "pytest-mypy (>=0.9.1)", "pytest-perf", "pytest-timeout", "pytest-xdist", "tomli-w (>=1.0.0)", "virtualenv (>=13.0.0)", "wheel"] +testing-integration = ["build[virtualenv]", "filelock (>=3.4.0)", "jaraco.envs (>=2.2)", "jaraco.path (>=3.2.0)", "pytest", "pytest-enabler", "pytest-xdist", "tomli", "virtualenv (>=13.0.0)", "wheel"] + +[[package]] +name = "six" +version = "1.16.0" +description = "Python 2 and 3 compatibility utilities" +category = "main" +optional = false +python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*" +files = [ {file = "six-1.16.0-py2.py3-none-any.whl", hash = "sha256:8abb2f1d86890a2dfb989f9a77cfcfd3e47c2a354b01111771326f8aa26e0254"}, {file = "six-1.16.0.tar.gz", hash = "sha256:1e61c37477a1626458e36f7b1d82aa5c9b094fa4802892072e49de9c60c4c926"}, ] -thrift = [] -toml = [ + +[[package]] +name = "thrift" +version = "0.16.0" +description = "Python bindings for the Apache Thrift RPC system" +category = "main" +optional = true +python-versions = "*" +files = [ + {file = "thrift-0.16.0.tar.gz", hash = "sha256:2b5b6488fcded21f9d312aa23c9ff6a0195d0f6ae26ddbd5ad9e3e25dfc14408"}, +] + +[package.dependencies] +six = ">=1.7.2" + +[package.extras] +all = ["tornado (>=4.0)", "twisted"] +tornado = ["tornado (>=4.0)"] +twisted = ["twisted"] + +[[package]] +name = "toml" +version = "0.10.2" +description = "Python Library for Tom's Obvious, Minimal Language" +category = "dev" +optional = false +python-versions = ">=2.6, !=3.0.*, !=3.1.*, !=3.2.*" +files = [ {file = "toml-0.10.2-py2.py3-none-any.whl", hash = "sha256:806143ae5bfb6a3c6e736a764057db0e6a0e05e338b5630894a5f779cabb4f9b"}, {file = "toml-0.10.2.tar.gz", hash = "sha256:b3bda1d108d5dd99f4a20d24d9c348e91c4db7ab1b749200bded2f839ccbe68f"}, ] -tomli = [ + +[[package]] +name = "tomli" +version = "2.0.1" +description = "A lil' TOML parser" +category = "dev" +optional = false +python-versions = ">=3.7" +files = [ {file = "tomli-2.0.1-py3-none-any.whl", hash = "sha256:939de3e7a6161af0c887ef91b7d41a53e7c5a1ca976325f429cb46ea9bc30ecc"}, {file = "tomli-2.0.1.tar.gz", hash = "sha256:de526c12914f0c550d15924c62d72abc48d6fe7364aa87328337a31007fe8a4f"}, ] -types-toml = [ + +[[package]] +name = "types-toml" +version = "0.10.8.4" +description = "Typing stubs for toml" +category = "dev" +optional = false +python-versions = "*" +files = [ {file = "types-toml-0.10.8.4.tar.gz", hash = "sha256:c8748dd225b28eb80ce712e2d7d61b57599815e7b48d07ef53df51ed148fa6b1"}, {file = "types_toml-0.10.8.4-py3-none-any.whl", hash = "sha256:306b1bb8b5bbc5f1b60387dbcc4b489e79f8490ce20e93af5f422a68b470d94b"}, ] -typing-extensions = [ + +[[package]] +name = "typing-extensions" +version = "4.4.0" +description = "Backported and Experimental Type Hints for Python 3.7+" +category = "main" +optional = false +python-versions = ">=3.7" +files = [ {file = "typing_extensions-4.4.0-py3-none-any.whl", hash = "sha256:16fa4864408f655d35ec496218b85f79b3437c829e93320c7c9215ccfd92489e"}, {file = "typing_extensions-4.4.0.tar.gz", hash = "sha256:1511434bb92bf8dd198c12b1cc812e800d4181cfcb867674e0f8279cc93087aa"}, ] -urllib3 = [ + +[[package]] +name = "urllib3" +version = "1.26.14" +description = "HTTP library with thread-safe connection pooling, file post, and more." +category = "main" +optional = false +python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*, !=3.5.*" +files = [ {file = "urllib3-1.26.14-py2.py3-none-any.whl", hash = "sha256:75edcdc2f7d85b137124a6c3c9fc3933cdeaa12ecb9a6a959f22797a0feca7e1"}, {file = "urllib3-1.26.14.tar.gz", hash = "sha256:076907bf8fd355cde77728471316625a4d2f7e713c125f51953bb5b3eecf4f72"}, ] -virtualenv = [ + +[package.extras] +brotli = ["brotli (>=1.0.9)", "brotlicffi (>=0.8.0)", "brotlipy (>=0.6.0)"] +secure = ["certifi", "cryptography (>=1.3.4)", "idna (>=2.0.0)", "ipaddress", "pyOpenSSL (>=0.14)", "urllib3-secure-extra"] +socks = ["PySocks (>=1.5.6,!=1.5.7,<2.0)"] + +[[package]] +name = "virtualenv" +version = "20.19.0" +description = "Virtual Python Environment builder" +category = "dev" +optional = false +python-versions = ">=3.7" +files = [ {file = "virtualenv-20.19.0-py3-none-any.whl", hash = "sha256:54eb59e7352b573aa04d53f80fc9736ed0ad5143af445a1e539aada6eb947dd1"}, {file = "virtualenv-20.19.0.tar.gz", hash = "sha256:37a640ba82ed40b226599c522d411e4be5edb339a0c0de030c0dc7b646d61590"}, ] -werkzeug = [ + +[package.dependencies] +distlib = ">=0.3.6,<1" +filelock = ">=3.4.1,<4" +platformdirs = ">=2.4,<4" + +[package.extras] +docs = ["furo (>=2022.12.7)", "proselint (>=0.13)", "sphinx (>=6.1.3)", "sphinx-argparse (>=0.4)", "sphinxcontrib-towncrier (>=0.2.1a0)", "towncrier (>=22.12)"] +test = ["covdefaults (>=2.2.2)", "coverage (>=7.1)", "coverage-enable-subprocess (>=1)", "flaky (>=3.7)", "packaging (>=23)", "pytest (>=7.2.1)", "pytest-env (>=0.8.1)", "pytest-freezegun (>=0.4.2)", "pytest-mock (>=3.10)", "pytest-randomly (>=3.12)", "pytest-timeout (>=2.1)"] + +[[package]] +name = "werkzeug" +version = "2.2.3" +description = "The comprehensive WSGI web application library." +category = "dev" +optional = false +python-versions = ">=3.7" +files = [ {file = "Werkzeug-2.2.3-py3-none-any.whl", hash = "sha256:56433961bc1f12533306c624f3be5e744389ac61d722175d543e1751285da612"}, {file = "Werkzeug-2.2.3.tar.gz", hash = "sha256:2e1ccc9417d4da358b9de6f174e3ac094391ea1d4fbef2d667865d819dfd0afe"}, ] -wrapt = [ + +[package.dependencies] +MarkupSafe = ">=2.1.1" + +[package.extras] +watchdog = ["watchdog"] + +[[package]] +name = "wrapt" +version = "1.14.1" +description = "Module for decorators, wrappers and monkey patching." +category = "main" +optional = true +python-versions = "!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,!=3.4.*,>=2.7" +files = [ {file = "wrapt-1.14.1-cp27-cp27m-macosx_10_9_x86_64.whl", hash = "sha256:1b376b3f4896e7930f1f772ac4b064ac12598d1c38d04907e696cc4d794b43d3"}, {file = "wrapt-1.14.1-cp27-cp27m-manylinux1_i686.whl", hash = "sha256:903500616422a40a98a5a3c4ff4ed9d0066f3b4c951fa286018ecdf0750194ef"}, {file = "wrapt-1.14.1-cp27-cp27m-manylinux1_x86_64.whl", hash = "sha256:5a9a0d155deafd9448baff28c08e150d9b24ff010e899311ddd63c45c2445e28"}, @@ -2372,11 +2355,27 @@ wrapt = [ {file = "wrapt-1.14.1-cp39-cp39-win_amd64.whl", hash = "sha256:dee60e1de1898bde3b238f18340eec6148986da0455d8ba7848d50470a7a32fb"}, {file = "wrapt-1.14.1.tar.gz", hash = "sha256:380a85cf89e0e69b7cfbe2ea9f765f004ff419f34194018a6827ac0e3edfed4d"}, ] -xmltodict = [ + +[[package]] +name = "xmltodict" +version = "0.13.0" +description = "Makes working with XML feel like you are working with JSON" +category = "dev" +optional = false +python-versions = ">=3.4" +files = [ {file = "xmltodict-0.13.0-py2.py3-none-any.whl", hash = "sha256:aa89e8fd76320154a40d19a0df04a4695fb9dc5ba977cbb68ab3e4eb225e7852"}, {file = "xmltodict-0.13.0.tar.gz", hash = "sha256:341595a488e3e01a85a9d8911d8912fd922ede5fecc4dce437eb4b6c8d037e56"}, ] -yarl = [ + +[[package]] +name = "yarl" +version = "1.8.2" +description = "Yet another URL library" +category = "main" +optional = true +python-versions = ">=3.7" +files = [ {file = "yarl-1.8.2-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:bb81f753c815f6b8e2ddd2eef3c855cf7da193b82396ac013c661aaa6cc6b0a5"}, {file = "yarl-1.8.2-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:47d49ac96156f0928f002e2424299b2c91d9db73e08c4cd6742923a086f1c863"}, {file = "yarl-1.8.2-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:3fc056e35fa6fba63248d93ff6e672c096f95f7836938241ebc8260e062832fe"}, @@ -2452,11 +2451,35 @@ yarl = [ {file = "yarl-1.8.2-cp39-cp39-win_amd64.whl", hash = "sha256:6604711362f2dbf7160df21c416f81fac0de6dbcf0b5445a2ef25478ecc4c778"}, {file = "yarl-1.8.2.tar.gz", hash = "sha256:49d43402c6e3013ad0978602bf6bf5328535c48d192304b91b97a3c6790b1562"}, ] -zipp = [ + +[package.dependencies] +idna = ">=2.0" +multidict = ">=4.0" + +[[package]] +name = "zipp" +version = "3.13.0" +description = "Backport of pathlib-compatible object wrapper for zip files" +category = "dev" +optional = false +python-versions = ">=3.7" +files = [ {file = "zipp-3.13.0-py3-none-any.whl", hash = "sha256:e8b2a36ea17df80ffe9e2c4fda3f693c3dad6df1697d3cd3af232db680950b0b"}, {file = "zipp-3.13.0.tar.gz", hash = "sha256:23f70e964bc11a34cef175bc90ba2914e1e4545ea1e3e2f67c079671883f9cb6"}, ] -zstandard = [ + +[package.extras] +docs = ["furo", "jaraco.packaging (>=9)", "jaraco.tidelift (>=1.4)", "rst.linker (>=1.9)", "sphinx (>=3.5)", "sphinx-lint"] +testing = ["flake8 (<5)", "func-timeout", "jaraco.functools", "jaraco.itertools", "more-itertools", "pytest (>=6)", "pytest-black (>=0.3.7)", "pytest-checkdocs (>=2.4)", "pytest-cov", "pytest-enabler (>=1.3)", "pytest-flake8", "pytest-mypy (>=0.9.1)"] + +[[package]] +name = "zstandard" +version = "0.19.0" +description = "Zstandard bindings for Python" +category = "main" +optional = false +python-versions = ">=3.6" +files = [ {file = "zstandard-0.19.0-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:a65e0119ad39e855427520f7829618f78eb2824aa05e63ff19b466080cd99210"}, {file = "zstandard-0.19.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:4fa496d2d674c6e9cffc561639d17009d29adee84a27cf1e12d3c9be14aa8feb"}, {file = "zstandard-0.19.0-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:8f7c68de4f362c1b2f426395fe4e05028c56d0782b2ec3ae18a5416eaf775576"}, @@ -2509,3 +2532,25 @@ zstandard = [ {file = "zstandard-0.19.0-cp39-cp39-win_amd64.whl", hash = "sha256:55a513ec67e85abd8b8b83af8813368036f03e2d29a50fc94033504918273980"}, {file = "zstandard-0.19.0.tar.gz", hash = "sha256:31d12fcd942dd8dbf52ca5f6b1bbe287f44e5d551a081a983ff3ea2082867863"}, ] + +[package.dependencies] +cffi = {version = ">=1.11", markers = "platform_python_implementation == \"PyPy\""} + +[package.extras] +cffi = ["cffi (>=1.11)"] + +[extras] +adlfs = ["adlfs"] +duckdb = ["duckdb", "pyarrow"] +dynamodb = ["boto3"] +glue = ["boto3"] +hive = ["thrift"] +pandas = ["pandas", "pyarrow"] +pyarrow = ["pyarrow"] +s3fs = ["s3fs"] +snappy = ["python-snappy"] + +[metadata] +lock-version = "2.0" +python-versions = "^3.8" +content-hash = "c917970a52346ac5720aa2ac0b8548986f6e673c838a06883dbe3bb2be2bd9af" diff --git a/pyproject.toml b/pyproject.toml index 555564f1a0..71bb8578b3 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -55,7 +55,7 @@ click = "8.1.3" rich = ">=13.0.0,<=13.3.1" pyyaml = ">=5.4.0,<=6.0.0" -pydantic = "1.10.4" +pydantic = "1.10.5" fsspec = ">=2022.8.2,<=2023.1.0" pyparsing = "3.0.9" From f8b6e1ac4ce622d3c33562e55a0f9bc1adba9061 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Sun, 19 Feb 2023 22:20:10 +0100 Subject: [PATCH 378/642] Build: Bump typing-extensions from 4.4.0 to 4.5.0 in /python (#6883) Bumps [typing-extensions](https://github.com/python/typing_extensions) from 4.4.0 to 4.5.0. - [Release notes](https://github.com/python/typing_extensions/releases) - [Changelog](https://github.com/python/typing_extensions/blob/main/CHANGELOG.md) - [Commits](https://github.com/python/typing_extensions/compare/4.4.0...4.5.0) --- updated-dependencies: - dependency-name: typing-extensions dependency-type: direct:development update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- poetry.lock | 8 ++++---- pyproject.toml | 2 +- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/poetry.lock b/poetry.lock index 39e594f713..56cbf9307d 100644 --- a/poetry.lock +++ b/poetry.lock @@ -2216,14 +2216,14 @@ files = [ [[package]] name = "typing-extensions" -version = "4.4.0" +version = "4.5.0" description = "Backported and Experimental Type Hints for Python 3.7+" category = "main" optional = false python-versions = ">=3.7" files = [ - {file = "typing_extensions-4.4.0-py3-none-any.whl", hash = "sha256:16fa4864408f655d35ec496218b85f79b3437c829e93320c7c9215ccfd92489e"}, - {file = "typing_extensions-4.4.0.tar.gz", hash = "sha256:1511434bb92bf8dd198c12b1cc812e800d4181cfcb867674e0f8279cc93087aa"}, + {file = "typing_extensions-4.5.0-py3-none-any.whl", hash = "sha256:fb33085c39dd998ac16d1431ebc293a8b3eedd00fd4a32de0ff79002c19511b4"}, + {file = "typing_extensions-4.5.0.tar.gz", hash = "sha256:5cb5f4a79139d699607b3ef622a1dedafa84e115ab0024e0d9c044a9479ca7cb"}, ] [[package]] @@ -2553,4 +2553,4 @@ snappy = ["python-snappy"] [metadata] lock-version = "2.0" python-versions = "^3.8" -content-hash = "c917970a52346ac5720aa2ac0b8548986f6e673c838a06883dbe3bb2be2bd9af" +content-hash = "90e62ebb24ecac51f053de99f42ffe1f968fa1097bf6c475e0f189589d39870b" diff --git a/pyproject.toml b/pyproject.toml index 71bb8578b3..c5bbbd6823 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -86,7 +86,7 @@ fastavro = "1.7.1" coverage = { version = "^7.1.0", extras = ["toml"] } requests-mock = "1.10.0" moto = "^4.1.2" -typing-extensions = '4.4.0' +typing-extensions = '4.5.0' [tool.poetry.scripts] pyiceberg = "pyiceberg.cli.console:run" From fbb1995a8fbdf62fb855dc09f118f2d3e1675708 Mon Sep 17 00:00:00 2001 From: Luigi Cerone Date: Mon, 20 Feb 2023 12:17:47 +0100 Subject: [PATCH 379/642] Python: Add more pyiceberg docs (#6874) --- mkdocs/docs/api.md | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/mkdocs/docs/api.md b/mkdocs/docs/api.md index f6a7b0f646..a3d6edf015 100644 --- a/mkdocs/docs/api.md +++ b/mkdocs/docs/api.md @@ -19,7 +19,7 @@ # Python API -PyIceberg is based around catalogs to load tables. First step is to instantiate a catalog that loads tables. Let's use the following configuration: +PyIceberg is based around catalogs to load tables. First step is to instantiate a catalog that loads tables. Let's use the following configuration to define a catalog called `prod`: ```yaml catalog: @@ -28,6 +28,10 @@ catalog: credential: t-1234:secret ``` +These info must be placed inside a file called `.pyiceberg.yaml` located either in the `$HOME` directory or in the `$PYICEBERG_HOME` directory (if var is set). + +For more details on possible configurations refer to the [specific page](https://py.iceberg.apache.org/configuration/). + Then load the `prod` catalog: ```python From b9929ac474a04c913b410d5621b6e6291168870b Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 23 Feb 2023 21:50:05 +0100 Subject: [PATCH 380/642] Build: Bump markdown-it-py from 2.1.0 to 2.2.0 in /python (#6922) Bumps [markdown-it-py](https://github.com/executablebooks/markdown-it-py) from 2.1.0 to 2.2.0. - [Release notes](https://github.com/executablebooks/markdown-it-py/releases) - [Changelog](https://github.com/executablebooks/markdown-it-py/blob/master/CHANGELOG.md) - [Commits](https://github.com/executablebooks/markdown-it-py/compare/v2.1.0...v2.2.0) --- updated-dependencies: - dependency-name: markdown-it-py dependency-type: indirect ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- poetry.lock | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/poetry.lock b/poetry.lock index 56cbf9307d..e38996051a 100644 --- a/poetry.lock +++ b/poetry.lock @@ -1083,24 +1083,24 @@ files = [ [[package]] name = "markdown-it-py" -version = "2.1.0" +version = "2.2.0" description = "Python port of markdown-it. Markdown parsing, done right!" category = "main" optional = false python-versions = ">=3.7" files = [ - {file = "markdown-it-py-2.1.0.tar.gz", hash = "sha256:cf7e59fed14b5ae17c0006eff14a2d9a00ed5f3a846148153899a0224e2c07da"}, - {file = "markdown_it_py-2.1.0-py3-none-any.whl", hash = "sha256:93de681e5c021a432c63147656fe21790bc01231e0cd2da73626f1aa3ac0fe27"}, + {file = "markdown-it-py-2.2.0.tar.gz", hash = "sha256:7c9a5e412688bc771c67432cbfebcdd686c93ce6484913dccf06cb5a0bea35a1"}, + {file = "markdown_it_py-2.2.0-py3-none-any.whl", hash = "sha256:5a35f8d1870171d9acc47b99612dc146129b631baf04970128b568f190d0cc30"}, ] [package.dependencies] mdurl = ">=0.1,<1.0" [package.extras] -benchmarking = ["psutil", "pytest", "pytest-benchmark (>=3.2,<4.0)"] -code-style = ["pre-commit (==2.6)"] -compare = ["commonmark (>=0.9.1,<0.10.0)", "markdown (>=3.3.6,<3.4.0)", "mistletoe (>=0.8.1,<0.9.0)", "mistune (>=2.0.2,<2.1.0)", "panflute (>=2.1.3,<2.2.0)"] -linkify = ["linkify-it-py (>=1.0,<2.0)"] +benchmarking = ["psutil", "pytest", "pytest-benchmark"] +code-style = ["pre-commit (>=3.0,<4.0)"] +compare = ["commonmark (>=0.9,<1.0)", "markdown (>=3.4,<4.0)", "mistletoe (>=1.0,<2.0)", "mistune (>=2.0,<3.0)", "panflute (>=2.3,<3.0)"] +linkify = ["linkify-it-py (>=1,<3)"] plugins = ["mdit-py-plugins"] profiling = ["gprof2dot"] rtd = ["attrs", "myst-parser", "pyyaml", "sphinx", "sphinx-copybutton", "sphinx-design", "sphinx_book_theme"] From 694c4a0836840387b4f5f23f68e248a4c886559c Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Sun, 26 Feb 2023 07:39:23 +0100 Subject: [PATCH 381/642] Build: Bump coverage from 7.1.0 to 7.2.0 in /python (#6944) --- poetry.lock | 106 ++++++++++++++++++++++++------------------------- pyproject.toml | 2 +- 2 files changed, 54 insertions(+), 54 deletions(-) diff --git a/poetry.lock b/poetry.lock index e38996051a..0ee28fa157 100644 --- a/poetry.lock +++ b/poetry.lock @@ -598,63 +598,63 @@ files = [ [[package]] name = "coverage" -version = "7.1.0" +version = "7.2.0" description = "Code coverage measurement for Python" category = "dev" optional = false python-versions = ">=3.7" files = [ - {file = "coverage-7.1.0-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:3b946bbcd5a8231383450b195cfb58cb01cbe7f8949f5758566b881df4b33baf"}, - {file = "coverage-7.1.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:ec8e767f13be637d056f7e07e61d089e555f719b387a7070154ad80a0ff31801"}, - {file = "coverage-7.1.0-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:d4a5a5879a939cb84959d86869132b00176197ca561c664fc21478c1eee60d75"}, - {file = "coverage-7.1.0-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:b643cb30821e7570c0aaf54feaf0bfb630b79059f85741843e9dc23f33aaca2c"}, - {file = "coverage-7.1.0-cp310-cp310-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:32df215215f3af2c1617a55dbdfb403b772d463d54d219985ac7cd3bf124cada"}, - {file = "coverage-7.1.0-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:33d1ae9d4079e05ac4cc1ef9e20c648f5afabf1a92adfaf2ccf509c50b85717f"}, - {file = "coverage-7.1.0-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:29571503c37f2ef2138a306d23e7270687c0efb9cab4bd8038d609b5c2393a3a"}, - {file = "coverage-7.1.0-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:63ffd21aa133ff48c4dff7adcc46b7ec8b565491bfc371212122dd999812ea1c"}, - {file = "coverage-7.1.0-cp310-cp310-win32.whl", hash = "sha256:4b14d5e09c656de5038a3f9bfe5228f53439282abcab87317c9f7f1acb280352"}, - {file = "coverage-7.1.0-cp310-cp310-win_amd64.whl", hash = "sha256:8361be1c2c073919500b6601220a6f2f98ea0b6d2fec5014c1d9cfa23dd07038"}, - {file = "coverage-7.1.0-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:da9b41d4539eefd408c46725fb76ecba3a50a3367cafb7dea5f250d0653c1040"}, - {file = "coverage-7.1.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:c5b15ed7644ae4bee0ecf74fee95808dcc34ba6ace87e8dfbf5cb0dc20eab45a"}, - {file = "coverage-7.1.0-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:d12d076582507ea460ea2a89a8c85cb558f83406c8a41dd641d7be9a32e1274f"}, - {file = "coverage-7.1.0-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:e2617759031dae1bf183c16cef8fcfb3de7617f394c813fa5e8e46e9b82d4222"}, - {file = "coverage-7.1.0-cp311-cp311-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:c4e4881fa9e9667afcc742f0c244d9364d197490fbc91d12ac3b5de0bf2df146"}, - {file = "coverage-7.1.0-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:9d58885215094ab4a86a6aef044e42994a2bd76a446dc59b352622655ba6621b"}, - {file = "coverage-7.1.0-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:ffeeb38ee4a80a30a6877c5c4c359e5498eec095878f1581453202bfacc8fbc2"}, - {file = "coverage-7.1.0-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:3baf5f126f30781b5e93dbefcc8271cb2491647f8283f20ac54d12161dff080e"}, - {file = "coverage-7.1.0-cp311-cp311-win32.whl", hash = "sha256:ded59300d6330be27bc6cf0b74b89ada58069ced87c48eaf9344e5e84b0072f7"}, - {file = "coverage-7.1.0-cp311-cp311-win_amd64.whl", hash = "sha256:6a43c7823cd7427b4ed763aa7fb63901ca8288591323b58c9cd6ec31ad910f3c"}, - {file = "coverage-7.1.0-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:7a726d742816cb3a8973c8c9a97539c734b3a309345236cd533c4883dda05b8d"}, - {file = "coverage-7.1.0-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:bc7c85a150501286f8b56bd8ed3aa4093f4b88fb68c0843d21ff9656f0009d6a"}, - {file = "coverage-7.1.0-cp37-cp37m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:f5b4198d85a3755d27e64c52f8c95d6333119e49fd001ae5798dac872c95e0f8"}, - {file = "coverage-7.1.0-cp37-cp37m-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:ddb726cb861c3117a553f940372a495fe1078249ff5f8a5478c0576c7be12050"}, - {file = "coverage-7.1.0-cp37-cp37m-musllinux_1_1_aarch64.whl", hash = "sha256:51b236e764840a6df0661b67e50697aaa0e7d4124ca95e5058fa3d7cbc240b7c"}, - {file = "coverage-7.1.0-cp37-cp37m-musllinux_1_1_i686.whl", hash = "sha256:7ee5c9bb51695f80878faaa5598040dd6c9e172ddcf490382e8aedb8ec3fec8d"}, - {file = "coverage-7.1.0-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:c31b75ae466c053a98bf26843563b3b3517b8f37da4d47b1c582fdc703112bc3"}, - {file = "coverage-7.1.0-cp37-cp37m-win32.whl", hash = "sha256:3b155caf3760408d1cb903b21e6a97ad4e2bdad43cbc265e3ce0afb8e0057e73"}, - {file = "coverage-7.1.0-cp37-cp37m-win_amd64.whl", hash = "sha256:2a60d6513781e87047c3e630b33b4d1e89f39836dac6e069ffee28c4786715f5"}, - {file = "coverage-7.1.0-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:f2cba5c6db29ce991029b5e4ac51eb36774458f0a3b8d3137241b32d1bb91f06"}, - {file = "coverage-7.1.0-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:beeb129cacea34490ffd4d6153af70509aa3cda20fdda2ea1a2be870dfec8d52"}, - {file = "coverage-7.1.0-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:0c45948f613d5d18c9ec5eaa203ce06a653334cf1bd47c783a12d0dd4fd9c851"}, - {file = "coverage-7.1.0-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:ef382417db92ba23dfb5864a3fc9be27ea4894e86620d342a116b243ade5d35d"}, - {file = "coverage-7.1.0-cp38-cp38-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:7c7c0d0827e853315c9bbd43c1162c006dd808dbbe297db7ae66cd17b07830f0"}, - {file = "coverage-7.1.0-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:e5cdbb5cafcedea04924568d990e20ce7f1945a1dd54b560f879ee2d57226912"}, - {file = "coverage-7.1.0-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:9817733f0d3ea91bea80de0f79ef971ae94f81ca52f9b66500c6a2fea8e4b4f8"}, - {file = "coverage-7.1.0-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:218fe982371ac7387304153ecd51205f14e9d731b34fb0568181abaf7b443ba0"}, - {file = "coverage-7.1.0-cp38-cp38-win32.whl", hash = "sha256:04481245ef966fbd24ae9b9e537ce899ae584d521dfbe78f89cad003c38ca2ab"}, - {file = "coverage-7.1.0-cp38-cp38-win_amd64.whl", hash = "sha256:8ae125d1134bf236acba8b83e74c603d1b30e207266121e76484562bc816344c"}, - {file = "coverage-7.1.0-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:2bf1d5f2084c3932b56b962a683074a3692bce7cabd3aa023c987a2a8e7612f6"}, - {file = "coverage-7.1.0-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:98b85dd86514d889a2e3dd22ab3c18c9d0019e696478391d86708b805f4ea0fa"}, - {file = "coverage-7.1.0-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:38da2db80cc505a611938d8624801158e409928b136c8916cd2e203970dde4dc"}, - {file = "coverage-7.1.0-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:3164d31078fa9efe406e198aecd2a02d32a62fecbdef74f76dad6a46c7e48311"}, - {file = "coverage-7.1.0-cp39-cp39-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:db61a79c07331e88b9a9974815c075fbd812bc9dbc4dc44b366b5368a2936063"}, - {file = "coverage-7.1.0-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:9ccb092c9ede70b2517a57382a601619d20981f56f440eae7e4d7eaafd1d1d09"}, - {file = "coverage-7.1.0-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:33ff26d0f6cc3ca8de13d14fde1ff8efe1456b53e3f0273e63cc8b3c84a063d8"}, - {file = "coverage-7.1.0-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:d47dd659a4ee952e90dc56c97d78132573dc5c7b09d61b416a9deef4ebe01a0c"}, - {file = "coverage-7.1.0-cp39-cp39-win32.whl", hash = "sha256:d248cd4a92065a4d4543b8331660121b31c4148dd00a691bfb7a5cdc7483cfa4"}, - {file = "coverage-7.1.0-cp39-cp39-win_amd64.whl", hash = "sha256:7ed681b0f8e8bcbbffa58ba26fcf5dbc8f79e7997595bf071ed5430d8c08d6f3"}, - {file = "coverage-7.1.0-pp37.pp38.pp39-none-any.whl", hash = "sha256:755e89e32376c850f826c425ece2c35a4fc266c081490eb0a841e7c1cb0d3bda"}, - {file = "coverage-7.1.0.tar.gz", hash = "sha256:10188fe543560ec4874f974b5305cd1a8bdcfa885ee00ea3a03733464c4ca265"}, + {file = "coverage-7.2.0-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:90e7a4cbbb7b1916937d380beb1315b12957b8e895d7d9fb032e2038ac367525"}, + {file = "coverage-7.2.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:34d7211be69b215ad92298a962b2cd5a4ef4b17c7871d85e15d3d1b6dc8d8c96"}, + {file = "coverage-7.2.0-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:971b49dbf713044c3e5f6451b39f65615d4d1c1d9a19948fa0f41b0245a98765"}, + {file = "coverage-7.2.0-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:f0557289260125a6c453ad5673ba79e5b6841d9a20c9e101f758bfbedf928a77"}, + {file = "coverage-7.2.0-cp310-cp310-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:049806ae2df69468c130f04f0fab4212c46b34ba5590296281423bb1ae379df2"}, + {file = "coverage-7.2.0-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:875b03d92ac939fbfa8ae74a35b2c468fc4f070f613d5b1692f9980099a3a210"}, + {file = "coverage-7.2.0-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:c160e34e388277f10c50dc2c7b5e78abe6d07357d9fe7fcb2f3c156713fd647e"}, + {file = "coverage-7.2.0-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:32e6a730fd18b2556716039ab93278ccebbefa1af81e6aa0c8dba888cf659e6e"}, + {file = "coverage-7.2.0-cp310-cp310-win32.whl", hash = "sha256:f3ff4205aff999164834792a3949f82435bc7c7655c849226d5836c3242d7451"}, + {file = "coverage-7.2.0-cp310-cp310-win_amd64.whl", hash = "sha256:93db11da6e728587e943dff8ae1b739002311f035831b6ecdb15e308224a4247"}, + {file = "coverage-7.2.0-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:cd38140b56538855d3d5722c6d1b752b35237e7ea3f360047ce57f3fade82d98"}, + {file = "coverage-7.2.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:9dbb21561b0e04acabe62d2c274f02df0d715e8769485353ddf3cf84727e31ce"}, + {file = "coverage-7.2.0-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:171dd3aa71a49274a7e4fc26f5bc167bfae5a4421a668bc074e21a0522a0af4b"}, + {file = "coverage-7.2.0-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:4655ecd813f4ba44857af3e9cffd133ab409774e9d2a7d8fdaf4fdfd2941b789"}, + {file = "coverage-7.2.0-cp311-cp311-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:1856a8c4aa77eb7ca0d42c996d0ca395ecafae658c1432b9da4528c429f2575c"}, + {file = "coverage-7.2.0-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:bd67df6b48db18c10790635060858e2ea4109601e84a1e9bfdd92e898dc7dc79"}, + {file = "coverage-7.2.0-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:2d7daf3da9c7e0ed742b3e6b4de6cc464552e787b8a6449d16517b31bbdaddf5"}, + {file = "coverage-7.2.0-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:bf9e02bc3dee792b9d145af30db8686f328e781bd212fdef499db5e9e4dd8377"}, + {file = "coverage-7.2.0-cp311-cp311-win32.whl", hash = "sha256:3713a8ec18781fda408f0e853bf8c85963e2d3327c99a82a22e5c91baffcb934"}, + {file = "coverage-7.2.0-cp311-cp311-win_amd64.whl", hash = "sha256:88ae5929f0ef668b582fd7cad09b5e7277f50f912183cf969b36e82a1c26e49a"}, + {file = "coverage-7.2.0-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:5e29a64e9586194ea271048bc80c83cdd4587830110d1e07b109e6ff435e5dbc"}, + {file = "coverage-7.2.0-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:8d5302eb84c61e758c9d68b8a2f93a398b272073a046d07da83d77b0edc8d76b"}, + {file = "coverage-7.2.0-cp37-cp37m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:2c9fffbc39dc4a6277e1525cab06c161d11ee3995bbc97543dc74fcec33e045b"}, + {file = "coverage-7.2.0-cp37-cp37m-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:a6ceeab5fca62bca072eba6865a12d881f281c74231d2990f8a398226e1a5d96"}, + {file = "coverage-7.2.0-cp37-cp37m-musllinux_1_1_aarch64.whl", hash = "sha256:28563a35ef4a82b5bc5160a01853ce62b9fceee00760e583ffc8acf9e3413753"}, + {file = "coverage-7.2.0-cp37-cp37m-musllinux_1_1_i686.whl", hash = "sha256:bfa065307667f1c6e1f4c3e13f415b0925e34e56441f5fda2c84110a4a1d8bda"}, + {file = "coverage-7.2.0-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:7f992b32286c86c38f07a8b5c3fc88384199e82434040a729ec06b067ee0d52c"}, + {file = "coverage-7.2.0-cp37-cp37m-win32.whl", hash = "sha256:2c15bd09fd5009f3a79c8b3682b52973df29761030b692043f9834fc780947c4"}, + {file = "coverage-7.2.0-cp37-cp37m-win_amd64.whl", hash = "sha256:f332d61fbff353e2ef0f3130a166f499c3fad3a196e7f7ae72076d41a6bfb259"}, + {file = "coverage-7.2.0-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:577a8bc40c01ad88bb9ab1b3a1814f2f860ff5c5099827da2a3cafc5522dadea"}, + {file = "coverage-7.2.0-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:9240a0335365c29c968131bdf624bb25a8a653a9c0d8c5dbfcabf80b59c1973c"}, + {file = "coverage-7.2.0-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:358d3bce1468f298b19a3e35183bdb13c06cdda029643537a0cc37e55e74e8f1"}, + {file = "coverage-7.2.0-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:932048364ff9c39030c6ba360c31bf4500036d4e15c02a2afc5a76e7623140d4"}, + {file = "coverage-7.2.0-cp38-cp38-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:7efa21611ffc91156e6f053997285c6fe88cfef3fb7533692d0692d2cb30c846"}, + {file = "coverage-7.2.0-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:465ea431c3b78a87e32d7d9ea6d081a1003c43a442982375cf2c247a19971961"}, + {file = "coverage-7.2.0-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:0f03c229f1453b936916f68a47b3dfb5e84e7ad48e160488168a5e35115320c8"}, + {file = "coverage-7.2.0-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:40785553d68c61e61100262b73f665024fd2bb3c6f0f8e2cd5b13e10e4df027b"}, + {file = "coverage-7.2.0-cp38-cp38-win32.whl", hash = "sha256:b09dd7bef59448c66e6b490cc3f3c25c14bc85d4e3c193b81a6204be8dd355de"}, + {file = "coverage-7.2.0-cp38-cp38-win_amd64.whl", hash = "sha256:dc4f9a89c82faf6254d646180b2e3aa4daf5ff75bdb2c296b9f6a6cf547e26a7"}, + {file = "coverage-7.2.0-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:c243b25051440386179591a8d5a5caff4484f92c980fb6e061b9559da7cc3f64"}, + {file = "coverage-7.2.0-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:4b8fd32f85b256fc096deeb4872aeb8137474da0c0351236f93cbedc359353d6"}, + {file = "coverage-7.2.0-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:d7f2a7df523791e6a63b40360afa6792a11869651307031160dc10802df9a252"}, + {file = "coverage-7.2.0-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:da32526326e8da0effb452dc32a21ffad282c485a85a02aeff2393156f69c1c3"}, + {file = "coverage-7.2.0-cp39-cp39-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:4c1153a6156715db9d6ae8283480ae67fb67452aa693a56d7dae9ffe8f7a80da"}, + {file = "coverage-7.2.0-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:74cd60fa00f46f28bd40048d6ca26bd58e9bee61d2b0eb4ec18cea13493c003f"}, + {file = "coverage-7.2.0-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:59a427f8a005aa7254074719441acb25ac2c2f60c1f1026d43f846d4254c1c2f"}, + {file = "coverage-7.2.0-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:c3c4beddee01c8125a75cde3b71be273995e2e9ec08fbc260dd206b46bb99969"}, + {file = "coverage-7.2.0-cp39-cp39-win32.whl", hash = "sha256:08e3dd256b8d3e07bb230896c8c96ec6c5dffbe5a133ba21f8be82b275b900e8"}, + {file = "coverage-7.2.0-cp39-cp39-win_amd64.whl", hash = "sha256:ad12c74c6ce53a027f5a5ecbac9be20758a41c85425c1bbab7078441794b04ee"}, + {file = "coverage-7.2.0-pp37.pp38.pp39-none-any.whl", hash = "sha256:ffa637a2d5883298449a5434b699b22ef98dd8e2ef8a1d9e60fa9cfe79813411"}, + {file = "coverage-7.2.0.tar.gz", hash = "sha256:9cc9c41aa5af16d845b53287051340c363dd03b7ef408e45eec3af52be77810d"}, ] [package.dependencies] @@ -2553,4 +2553,4 @@ snappy = ["python-snappy"] [metadata] lock-version = "2.0" python-versions = "^3.8" -content-hash = "90e62ebb24ecac51f053de99f42ffe1f968fa1097bf6c475e0f189589d39870b" +content-hash = "6cb6fa744af952d9ea6f3c4a1316dcf00ea0f7df719913be29ea8551d03df2e6" diff --git a/pyproject.toml b/pyproject.toml index c5bbbd6823..9e5c768142 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -83,7 +83,7 @@ pytest = "7.2.1" pytest-checkdocs = "2.9.0" pre-commit = "3.0.4" fastavro = "1.7.1" -coverage = { version = "^7.1.0", extras = ["toml"] } +coverage = { version = "^7.2.0", extras = ["toml"] } requests-mock = "1.10.0" moto = "^4.1.2" typing-extensions = '4.5.0' From ec06fc2560512dd8e9623959e0457470121e346a Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Sun, 26 Feb 2023 10:27:22 +0100 Subject: [PATCH 382/642] Build: Bump zstandard from 0.19.0 to 0.20.0 in /python (#6943) --- poetry.lock | 106 ++++++++++++++++++++++++------------------------- pyproject.toml | 2 +- 2 files changed, 54 insertions(+), 54 deletions(-) diff --git a/poetry.lock b/poetry.lock index 0ee28fa157..28488a56f8 100644 --- a/poetry.lock +++ b/poetry.lock @@ -2474,63 +2474,63 @@ testing = ["flake8 (<5)", "func-timeout", "jaraco.functools", "jaraco.itertools" [[package]] name = "zstandard" -version = "0.19.0" +version = "0.20.0" description = "Zstandard bindings for Python" category = "main" optional = false python-versions = ">=3.6" files = [ - {file = "zstandard-0.19.0-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:a65e0119ad39e855427520f7829618f78eb2824aa05e63ff19b466080cd99210"}, - {file = "zstandard-0.19.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:4fa496d2d674c6e9cffc561639d17009d29adee84a27cf1e12d3c9be14aa8feb"}, - {file = "zstandard-0.19.0-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:8f7c68de4f362c1b2f426395fe4e05028c56d0782b2ec3ae18a5416eaf775576"}, - {file = "zstandard-0.19.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:d1a7a716bb04b1c3c4a707e38e2dee46ac544fff931e66d7ae944f3019fc55b8"}, - {file = "zstandard-0.19.0-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_12_i686.manylinux2010_i686.whl", hash = "sha256:72758c9f785831d9d744af282d54c3e0f9db34f7eae521c33798695464993da2"}, - {file = "zstandard-0.19.0-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:04c298d381a3b6274b0a8001f0da0ec7819d052ad9c3b0863fe8c7f154061f76"}, - {file = "zstandard-0.19.0-cp310-cp310-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:aef0889417eda2db000d791f9739f5cecb9ccdd45c98f82c6be531bdc67ff0f2"}, - {file = "zstandard-0.19.0-cp310-cp310-win32.whl", hash = "sha256:9d97c713433087ba5cee61a3e8edb54029753d45a4288ad61a176fa4718033ce"}, - {file = "zstandard-0.19.0-cp310-cp310-win_amd64.whl", hash = "sha256:81ab21d03e3b0351847a86a0b298b297fde1e152752614138021d6d16a476ea6"}, - {file = "zstandard-0.19.0-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:593f96718ad906e24d6534187fdade28b611f8ed06e27ba972ba48aecec45fc6"}, - {file = "zstandard-0.19.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:5e21032efe673b887464667d09406bab6e16d96b09ad87e80859e3a20b6745b6"}, - {file = "zstandard-0.19.0-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:876567136b0359f6581ecd892bdb4ca03a0eead0265db73206c78cff03bcdb0f"}, - {file = "zstandard-0.19.0-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:aa9087571729c968cd853d54b3f6e9d0ec61e45cd2c31e0eb8a0d4bdbbe6da2f"}, - {file = "zstandard-0.19.0-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:8371217dff635cfc0220db2720fc3ce728cd47e72bb7572cca035332823dbdfc"}, - {file = "zstandard-0.19.0-cp311-cp311-win32.whl", hash = "sha256:126aa8433773efad0871f624339c7984a9c43913952f77d5abeee7f95a0c0860"}, - {file = "zstandard-0.19.0-cp311-cp311-win_amd64.whl", hash = "sha256:0fde1c56ec118940974e726c2a27e5b54e71e16c6f81d0b4722112b91d2d9009"}, - {file = "zstandard-0.19.0-cp36-cp36m-macosx_10_9_x86_64.whl", hash = "sha256:898500957ae5e7f31b7271ace4e6f3625b38c0ac84e8cedde8de3a77a7fdae5e"}, - {file = "zstandard-0.19.0-cp36-cp36m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:660b91eca10ee1b44c47843894abe3e6cfd80e50c90dee3123befbf7ca486bd3"}, - {file = "zstandard-0.19.0-cp36-cp36m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:55b3187e0bed004533149882ef8c24e954321f3be81f8a9ceffe35099b82a0d0"}, - {file = "zstandard-0.19.0-cp36-cp36m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_12_i686.manylinux2010_i686.whl", hash = "sha256:6d2182e648e79213b3881998b30225b3f4b1f3e681f1c1eaf4cacf19bde1040d"}, - {file = "zstandard-0.19.0-cp36-cp36m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:8ec2c146e10b59c376b6bc0369929647fcd95404a503a7aa0990f21c16462248"}, - {file = "zstandard-0.19.0-cp36-cp36m-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:67710d220af405f5ce22712fa741d85e8b3ada7a457ea419b038469ba379837c"}, - {file = "zstandard-0.19.0-cp36-cp36m-win32.whl", hash = "sha256:f097dda5d4f9b9b01b3c9fa2069f9c02929365f48f341feddf3d6b32510a2f93"}, - {file = "zstandard-0.19.0-cp36-cp36m-win_amd64.whl", hash = "sha256:f4ebfe03cbae821ef994b2e58e4df6a087470cc522aca502614e82a143365d45"}, - {file = "zstandard-0.19.0-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:b80f6f6478f9d4ca26daee6c61584499493bf97950cfaa1a02b16bb5c2c17e70"}, - {file = "zstandard-0.19.0-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:909bdd4e19ea437eb9b45d6695d722f6f0fd9d8f493e837d70f92062b9f39faf"}, - {file = "zstandard-0.19.0-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:e9c90a44470f2999779057aeaf33461cbd8bb59d8f15e983150d10bb260e16e0"}, - {file = "zstandard-0.19.0-cp37-cp37m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_12_i686.manylinux2010_i686.whl", hash = "sha256:401508efe02341ae681752a87e8ac9ef76df85ef1a238a7a21786a489d2c983d"}, - {file = "zstandard-0.19.0-cp37-cp37m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:47dfa52bed3097c705451bafd56dac26535545a987b6759fa39da1602349d7ba"}, - {file = "zstandard-0.19.0-cp37-cp37m-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:1a4fb8b4ac6772e4d656103ccaf2e43e45bd16b5da324b963d58ef360d09eb73"}, - {file = "zstandard-0.19.0-cp37-cp37m-win32.whl", hash = "sha256:d63b04e16df8ea21dfcedbf5a60e11cbba9d835d44cb3cbff233cfd037a916d5"}, - {file = "zstandard-0.19.0-cp37-cp37m-win_amd64.whl", hash = "sha256:74c2637d12eaacb503b0b06efdf55199a11b1d7c580bd3dd9dfe84cac97ef2f6"}, - {file = "zstandard-0.19.0-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:2e4812720582d0803e84aefa2ac48ce1e1e6e200ca3ce1ae2be6d410c1d637ae"}, - {file = "zstandard-0.19.0-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:4514b19abe6dbd36d6c5d75c54faca24b1ceb3999193c5b1f4b685abeabde3d0"}, - {file = "zstandard-0.19.0-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:6caed86cd47ae93915d9031dc04be5283c275e1a2af2ceff33932071f3eeff4d"}, - {file = "zstandard-0.19.0-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:7ccc4727300f223184520a6064c161a90b5d0283accd72d1455bcd85ec44dd0d"}, - {file = "zstandard-0.19.0-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_12_i686.manylinux2010_i686.whl", hash = "sha256:879411d04068bd489db57dcf6b82ffad3c5fb2a1fdd30817c566d8b7bedee442"}, - {file = "zstandard-0.19.0-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:8c9ca56345b0c5574db47560603de9d05f63cce5dfeb3a456eb60f3fec737ff2"}, - {file = "zstandard-0.19.0-cp38-cp38-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:d777d239036815e9b3a093fa9208ad314c040c26d7246617e70e23025b60083a"}, - {file = "zstandard-0.19.0-cp38-cp38-win32.whl", hash = "sha256:be6329b5ba18ec5d32dc26181e0148e423347ed936dda48bf49fb243895d1566"}, - {file = "zstandard-0.19.0-cp38-cp38-win_amd64.whl", hash = "sha256:3d5bb598963ac1f1f5b72dd006adb46ca6203e4fb7269a5b6e1f99e85b07ad38"}, - {file = "zstandard-0.19.0-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:619f9bf37cdb4c3dc9d4120d2a1003f5db9446f3618a323219f408f6a9df6725"}, - {file = "zstandard-0.19.0-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:b253d0c53c8ee12c3e53d181fb9ef6ce2cd9c41cbca1c56a535e4fc8ec41e241"}, - {file = "zstandard-0.19.0-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:3c927b6aa682c6d96225e1c797f4a5d0b9f777b327dea912b23471aaf5385376"}, - {file = "zstandard-0.19.0-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:2f01b27d0b453f07cbcff01405cdd007e71f5d6410eb01303a16ba19213e58e4"}, - {file = "zstandard-0.19.0-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_12_i686.manylinux2010_i686.whl", hash = "sha256:c7560f622e3849cc8f3e999791a915addd08fafe80b47fcf3ffbda5b5151047c"}, - {file = "zstandard-0.19.0-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:e892d3177380ec080550b56a7ffeab680af25575d291766bdd875147ba246a91"}, - {file = "zstandard-0.19.0-cp39-cp39-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:60a86b7b2b1c300779167cf595e019e61afcc0e20c4838692983a921db9006ac"}, - {file = "zstandard-0.19.0-cp39-cp39-win32.whl", hash = "sha256:755020d5aeb1b10bffd93d119e7709a2a7475b6ad79c8d5226cea3f76d152ce0"}, - {file = "zstandard-0.19.0-cp39-cp39-win_amd64.whl", hash = "sha256:55a513ec67e85abd8b8b83af8813368036f03e2d29a50fc94033504918273980"}, - {file = "zstandard-0.19.0.tar.gz", hash = "sha256:31d12fcd942dd8dbf52ca5f6b1bbe287f44e5d551a081a983ff3ea2082867863"}, + {file = "zstandard-0.20.0-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:c4efa051799703dc37c072e22af1f0e4c77069a78fb37caf70e26414c738ca1d"}, + {file = "zstandard-0.20.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:f847701d77371d90783c0ce6cfdb7ebde4053882c2aaba7255c70ae3c3eb7af0"}, + {file = "zstandard-0.20.0-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:0aa4d178560d7ee32092ddfd415c2cdc6ab5ddce9554985c75f1a019a0ff4c55"}, + {file = "zstandard-0.20.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:0488f2a238b4560828b3a595f3337daac4d3725c2a1637ffe2a0d187c091da59"}, + {file = "zstandard-0.20.0-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_12_i686.manylinux2010_i686.whl", hash = "sha256:cd0aa9a043c38901925ae1bba49e1e638f2d9c3cdf1b8000868993c642deb7f2"}, + {file = "zstandard-0.20.0-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:cdd769da7add8498658d881ce0eeb4c35ea1baac62e24c5a030c50f859f29724"}, + {file = "zstandard-0.20.0-cp310-cp310-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:9aea3c7bab4276212e5ac63d28e6bd72a79ff058d57e06926dfe30a52451d943"}, + {file = "zstandard-0.20.0-cp310-cp310-win32.whl", hash = "sha256:0d213353d58ad37fb5070314b156fb983b4d680ed5f3fce76ab013484cf3cf12"}, + {file = "zstandard-0.20.0-cp310-cp310-win_amd64.whl", hash = "sha256:d08459f7f7748398a6cc65eb7f88aa7ef5731097be2ddfba544be4b558acd900"}, + {file = "zstandard-0.20.0-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:c1929afea64da48ec59eca9055d7ec7e5955801489ac40ac2a19dde19e7edad9"}, + {file = "zstandard-0.20.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:b6d718f1b7cd30adb02c2a46dde0f25a84a9de8865126e0fff7d0162332d6b92"}, + {file = "zstandard-0.20.0-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:5499d65d4a1978dccf0a9c2c0d12415e16d4995ffad7a0bc4f72cc66691cf9f2"}, + {file = "zstandard-0.20.0-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:302a31400de0280f17c4ce67a73444a7a069f228db64048e4ce555cd0c02fbc4"}, + {file = "zstandard-0.20.0-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:39ae788dcdc404c07ef7aac9b11925185ea0831b985db0bbc43f95acdbd1c2ce"}, + {file = "zstandard-0.20.0-cp311-cp311-win32.whl", hash = "sha256:e3f6887d2bdfb5752d5544860bd6b778e53ebfaf4ab6c3f9d7fd388445429d41"}, + {file = "zstandard-0.20.0-cp311-cp311-win_amd64.whl", hash = "sha256:4abf9a9e0841b844736d1ae8ead2b583d2cd212815eab15391b702bde17477a7"}, + {file = "zstandard-0.20.0-cp36-cp36m-macosx_10_9_x86_64.whl", hash = "sha256:dc47cec184e66953f635254e5381df8a22012a2308168c069230b1a95079ccd0"}, + {file = "zstandard-0.20.0-cp36-cp36m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:84c1dae0c0a21eea245b5691286fe6470dc797d5e86e0c26b57a3afd1e750b48"}, + {file = "zstandard-0.20.0-cp36-cp36m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:059316f07e39b7214cd9eed565d26ab239035d2c76835deeff381995f7a27ba8"}, + {file = "zstandard-0.20.0-cp36-cp36m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_12_i686.manylinux2010_i686.whl", hash = "sha256:9aca916724d0802d3e70dc68adeff893efece01dffe7252ee3ae0053f1f1990f"}, + {file = "zstandard-0.20.0-cp36-cp36m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:b07f391fd85e3d07514c05fb40c5573b398d0063ab2bada6eb09949ec6004772"}, + {file = "zstandard-0.20.0-cp36-cp36m-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:2adf65cfce73ce94ef4c482f6cc01f08ddf5e1ca0c1ec95f2b63840f9e4c226c"}, + {file = "zstandard-0.20.0-cp36-cp36m-win32.whl", hash = "sha256:ee2a1510e06dfc7706ea9afad363efe222818a1eafa59abc32d9bbcd8465fba7"}, + {file = "zstandard-0.20.0-cp36-cp36m-win_amd64.whl", hash = "sha256:29699746fae2760d3963a4ffb603968e77da55150ee0a3326c0569f4e35f319f"}, + {file = "zstandard-0.20.0-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:78fb35d07423f25efd0fc90d0d4710ae83cfc86443a32192b0c6cb8475ec79a5"}, + {file = "zstandard-0.20.0-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:40466adfa071f58bfa448d90f9623d6aff67c6d86de6fc60be47a26388f6c74d"}, + {file = "zstandard-0.20.0-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:ba86f931bf925e9561ccd6cb978acb163e38c425990927feb38be10c894fa937"}, + {file = "zstandard-0.20.0-cp37-cp37m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_12_i686.manylinux2010_i686.whl", hash = "sha256:b671b75ae88139b1dd022fa4aa66ba419abd66f98869af55a342cb9257a1831e"}, + {file = "zstandard-0.20.0-cp37-cp37m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:cc98c8bcaa07150d3f5d7c4bd264eaa4fdd4a4dfb8fd3f9d62565ae5c4aba227"}, + {file = "zstandard-0.20.0-cp37-cp37m-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:0b815dec62e2d5a1bf7a373388f2616f21a27047b9b999de328bca7462033708"}, + {file = "zstandard-0.20.0-cp37-cp37m-win32.whl", hash = "sha256:5a3578b182c21b8af3c49619eb4cd0b9127fa60791e621b34217d65209722002"}, + {file = "zstandard-0.20.0-cp37-cp37m-win_amd64.whl", hash = "sha256:f1ba6bbd28ad926d130f0af8016f3a2930baa013c2128cfff46ca76432f50669"}, + {file = "zstandard-0.20.0-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:b0f556c74c6f0f481b61d917e48c341cdfbb80cc3391511345aed4ce6fb52fdc"}, + {file = "zstandard-0.20.0-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:862ad0a5c94670f2bd6f64fff671bd2045af5f4ed428a3f2f69fa5e52483f86a"}, + {file = "zstandard-0.20.0-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a56036c08645aa6041d435a50103428f0682effdc67f5038de47cea5e4221d6f"}, + {file = "zstandard-0.20.0-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:4af5d1891eebef430038ea4981957d31b1eb70aca14b906660c3ac1c3e7a8612"}, + {file = "zstandard-0.20.0-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_12_i686.manylinux2010_i686.whl", hash = "sha256:489959e2d52f7f1fe8ea275fecde6911d454df465265bf3ec51b3e755e769a5e"}, + {file = "zstandard-0.20.0-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:7041efe3a93d0975d2ad16451720932e8a3d164be8521bfd0873b27ac917b77a"}, + {file = "zstandard-0.20.0-cp38-cp38-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:c28c7441638c472bfb794f424bd560a22c7afce764cd99196e8d70fbc4d14e85"}, + {file = "zstandard-0.20.0-cp38-cp38-win32.whl", hash = "sha256:ba4bb4c5a0cac802ff485fa1e57f7763df5efa0ad4ee10c2693ecc5a018d2c1a"}, + {file = "zstandard-0.20.0-cp38-cp38-win_amd64.whl", hash = "sha256:a5efe366bf0545a1a5a917787659b445ba16442ae4093f102204f42a9da1ecbc"}, + {file = "zstandard-0.20.0-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:79c3058ccbe1fa37356a73c9d3c0475ec935ab528f5b76d56fc002a5a23407c7"}, + {file = "zstandard-0.20.0-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:39cbaf8fe3fa3515d35fb790465db4dc1ff45e58e1e00cbaf8b714e85437f039"}, + {file = "zstandard-0.20.0-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:f199d58f3fd7dfa0d447bc255ff22571f2e4e5e5748bfd1c41370454723cb053"}, + {file = "zstandard-0.20.0-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:0f32a8f3a697ef87e67c0d0c0673b245babee6682b2c95e46eb30208ffb720bd"}, + {file = "zstandard-0.20.0-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_12_i686.manylinux2010_i686.whl", hash = "sha256:4a3c36284c219a4d2694e52b2582fe5d5f0ecaf94a22cf0ea959b527dbd8a2a6"}, + {file = "zstandard-0.20.0-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:2eeb9e1ecd48ac1d352608bfe0dc1ed78a397698035a1796cf72f0c9d905d219"}, + {file = "zstandard-0.20.0-cp39-cp39-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:6179808ebd1ebc42b1e2f221a23c28a22d3bc8f79209ae4a3cc114693c380bff"}, + {file = "zstandard-0.20.0-cp39-cp39-win32.whl", hash = "sha256:afbcd2ed0c1145e24dd3df8440a429688a1614b83424bc871371b176bed429f9"}, + {file = "zstandard-0.20.0-cp39-cp39-win_amd64.whl", hash = "sha256:e6b4de1ba2f3028fafa0d82222d1e91b729334c8d65fbf04290c65c09d7457e1"}, + {file = "zstandard-0.20.0.tar.gz", hash = "sha256:613daadd72c71b1488742cafb2c3b381c39d0c9bb8c6cc157aa2d5ea45cc2efc"}, ] [package.dependencies] @@ -2553,4 +2553,4 @@ snappy = ["python-snappy"] [metadata] lock-version = "2.0" python-versions = "^3.8" -content-hash = "6cb6fa744af952d9ea6f3c4a1316dcf00ea0f7df719913be29ea8551d03df2e6" +content-hash = "890387c19732f824d2b1b1461b27807b559c8d140a5e2593962e184178cd7b58" diff --git a/pyproject.toml b/pyproject.toml index 9e5c768142..d8e3604bac 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -60,7 +60,7 @@ fsspec = ">=2022.8.2,<=2023.1.0" pyparsing = "3.0.9" -zstandard = "0.19.0" +zstandard = "0.20.0" pyarrow = { version = ">=8.0.0,<=11.0.0", optional = true } From 6a81df6577e2f9b67f73c7dcfed61d70f9a9907d Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Sun, 26 Feb 2023 11:34:24 +0100 Subject: [PATCH 383/642] Build: Bump fastavro from 1.7.1 to 1.7.2 in /python (#6942) --- poetry.lock | 46 +++++++++++++++++++++++----------------------- pyproject.toml | 2 +- 2 files changed, 24 insertions(+), 24 deletions(-) diff --git a/poetry.lock b/poetry.lock index 28488a56f8..c70652e35a 100644 --- a/poetry.lock +++ b/poetry.lock @@ -807,33 +807,33 @@ test = ["pytest (>=6)"] [[package]] name = "fastavro" -version = "1.7.1" +version = "1.7.2" description = "Fast read/write of AVRO files" category = "dev" optional = false python-versions = ">=3.7" files = [ - {file = "fastavro-1.7.1-cp310-cp310-macosx_11_0_x86_64.whl", hash = "sha256:f8f7f11af8c2c074341217d6247b7ba09cadcd55f899e046c14e3a44afa5fc95"}, - {file = "fastavro-1.7.1-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:11159cf23f5ff4b752b028a77bd2a7941599932527e8a6512779e25b0503f037"}, - {file = "fastavro-1.7.1-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:86ce89743711355eee7d3fec34847de1ab574f4567aa4a667e966711bb2e0cd9"}, - {file = "fastavro-1.7.1-cp310-cp310-win_amd64.whl", hash = "sha256:2e3f5ae42e033dbb6e0efa788c4b8a4e5a59bc2b9cb83f717b6d85176727faed"}, - {file = "fastavro-1.7.1-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:8b671a97e864e4c024061c0d6f93f2768ba0595f917317ca4fe3e99dca6fcf3d"}, - {file = "fastavro-1.7.1-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:e9be910ca1645e0e5c4192d667cfdfa58dff4f8690db5af1ae33602643d41a78"}, - {file = "fastavro-1.7.1-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:6f233a1d0d95265f81a25c274f55017bd39d9b8f7f1387a4235bf8e01841a9ff"}, - {file = "fastavro-1.7.1-cp311-cp311-win_amd64.whl", hash = "sha256:31c924ed2fdad56182b543941cdec9cc384443cc3ca9462e0580afeb4dc64f4b"}, - {file = "fastavro-1.7.1-cp37-cp37m-macosx_10_15_x86_64.whl", hash = "sha256:8ad522c7f2c7791cfe54a7e87af8ac09634872f4fdeef28deec97fab8de38b24"}, - {file = "fastavro-1.7.1-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:74fe0fb24a489e2fd567775935eb55a461fc6c4da8b5e3467245752ac2098284"}, - {file = "fastavro-1.7.1-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:4f7a744569b4b44ea1f4f87e7ea8298e1e2bf23779aa6ef255b95f9f38faad48"}, - {file = "fastavro-1.7.1-cp37-cp37m-win_amd64.whl", hash = "sha256:b793252a6325341890bbbbdca2804d4db3d5d29ff7f15a58fcf84dda440808fa"}, - {file = "fastavro-1.7.1-cp38-cp38-macosx_10_15_x86_64.whl", hash = "sha256:a107a8b333628c0f8a2ece5d5a57c69923719a139b106ace4050206250df4b13"}, - {file = "fastavro-1.7.1-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:182671595683054ec234beec03f485b5c889c21c08e429577ae7929480703409"}, - {file = "fastavro-1.7.1-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:980ce7fdccf328d287e8b789f4e3b422f64c84ed9cd81c05dd7c560c3d8076b1"}, - {file = "fastavro-1.7.1-cp38-cp38-win_amd64.whl", hash = "sha256:a5f104bc5b4986bbbcab170918c4d8ffa4f8efa3ebe8ec190954178630074d5a"}, - {file = "fastavro-1.7.1-cp39-cp39-macosx_11_0_x86_64.whl", hash = "sha256:5ff701d6b228218a3d9c09b392205dd0899afa501a4f14724bef0ce13a719700"}, - {file = "fastavro-1.7.1-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:5cb2d8ea7b21493bb18ff7c68401edbc663bbd8a57016d6cf2c4b0a2dc4464e7"}, - {file = "fastavro-1.7.1-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:e15e8163581983f07a3156902761cf746bbe1e646abd9553cc9a1cede6e23ae9"}, - {file = "fastavro-1.7.1-cp39-cp39-win_amd64.whl", hash = "sha256:4d56cc05ceb1c9f4de19c1f330a4f140af3b944834d63cd0e11517deaea21ee1"}, - {file = "fastavro-1.7.1.tar.gz", hash = "sha256:4b8bcae4ed6343af186e638061cdfbc5331cdb5e026d055099c91d4f07be838c"}, + {file = "fastavro-1.7.2-cp310-cp310-macosx_11_0_x86_64.whl", hash = "sha256:20a94269bc0910c41f06106424429289e5c8cdce65b006e48d600c9bb6c442a5"}, + {file = "fastavro-1.7.2-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:b34e9cd755e4d9d12377fe151ad95b9bff08215f0a3756601b21ed31f4d127ab"}, + {file = "fastavro-1.7.2-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:8993e2f34e9ae195f417105d495fdd3d7f6d1532bacf57dc5dea600082f9b1a0"}, + {file = "fastavro-1.7.2-cp310-cp310-win_amd64.whl", hash = "sha256:e83069f971f307175ab806e9929a15826e586ef05a620fcfc7768ceaa3ec19c8"}, + {file = "fastavro-1.7.2-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:4c44f72a1a7b2d28cc21c95686916b782354dcc1f1aa1c8ca044c66d5ada3e14"}, + {file = "fastavro-1.7.2-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:88005d5c6fe436e80f756399a52aaa6d0cdc506c1ee195277287a09c7617aec3"}, + {file = "fastavro-1.7.2-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:a5c5d466631f455213b8bfd54888d2d8691cfc240cd6dbb40eb3694bf05e562e"}, + {file = "fastavro-1.7.2-cp311-cp311-win_amd64.whl", hash = "sha256:4c2523c5e2279debb5fa788bf665a5abbd16a1e889338b973e3b318d50fcce77"}, + {file = "fastavro-1.7.2-cp37-cp37m-macosx_10_15_x86_64.whl", hash = "sha256:db3d9b6237934be5001fc18c07d25c09d3008ce6ab3448daf4b30608b84488e0"}, + {file = "fastavro-1.7.2-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:e82862cb7d3bf99fdca65048b8b09db295795d7fb1eb7c5dae29c0a4d7dcf1f2"}, + {file = "fastavro-1.7.2-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:5417dfbf99ac5bf963a929f2541c643d35e1a5a22876ba5ae0dca7b86326d9c2"}, + {file = "fastavro-1.7.2-cp37-cp37m-win_amd64.whl", hash = "sha256:b8cd8bd9a8ad4b687bd8dbf12effc2996bdf41ec04960875c19a7a90c949e8a8"}, + {file = "fastavro-1.7.2-cp38-cp38-macosx_10_15_x86_64.whl", hash = "sha256:f07d1f2ebdcc18938908e710a79ab3731061936fbea27534e436f9e12342450b"}, + {file = "fastavro-1.7.2-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:9919d866319b1e123374fced84afb4728802872bd5938ba7e12a691c54593542"}, + {file = "fastavro-1.7.2-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:8da1a9d4f0b871e7c968c8413caef19e19002100036b2714e037543c36550b8d"}, + {file = "fastavro-1.7.2-cp38-cp38-win_amd64.whl", hash = "sha256:e842beff23b0b207c45f99fa79d50cfc0d84ac137d84a7e283e9e59a8368b115"}, + {file = "fastavro-1.7.2-cp39-cp39-macosx_11_0_x86_64.whl", hash = "sha256:c1d5805c50a9507c99b814cb014da2812e6e0ee2770f4b793ce56b82a5b0af21"}, + {file = "fastavro-1.7.2-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:22cb5d2bf87c1ed9a1dcc97b6b13dc6b7ee4395dbf1fd5903dac01cdde0291c2"}, + {file = "fastavro-1.7.2-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:06bd51962df9b3f28c7081c0a3c8663d840df05d964b7817d85bd37c1d101236"}, + {file = "fastavro-1.7.2-cp39-cp39-win_amd64.whl", hash = "sha256:d787e7739d93db59f378c0e44a4eca8e56ff979e5b43dea6378e87c48548818c"}, + {file = "fastavro-1.7.2.tar.gz", hash = "sha256:3c524c65a5d578c020997800fce992cd90cc88ef13148b70f01a3429de34c8a4"}, ] [package.extras] @@ -2553,4 +2553,4 @@ snappy = ["python-snappy"] [metadata] lock-version = "2.0" python-versions = "^3.8" -content-hash = "890387c19732f824d2b1b1461b27807b559c8d140a5e2593962e184178cd7b58" +content-hash = "418ac8ae266f29e7a88e16962a7f9416b35936f702a1403050c323a8643836c5" diff --git a/pyproject.toml b/pyproject.toml index d8e3604bac..39d81a4e46 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -82,7 +82,7 @@ adlfs = { version = ">=2022.8.2,<=2023.1.0", optional = true } pytest = "7.2.1" pytest-checkdocs = "2.9.0" pre-commit = "3.0.4" -fastavro = "1.7.1" +fastavro = "1.7.2" coverage = { version = "^7.2.0", extras = ["toml"] } requests-mock = "1.10.0" moto = "^4.1.2" From 70ff3a1f7f96b1d0d5b30dacb30336e8e2bf0175 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Sun, 26 Feb 2023 12:07:37 +0100 Subject: [PATCH 384/642] Build: Bump pre-commit from 3.0.4 to 3.1.0 in /python (#6941) --- poetry.lock | 8 ++++---- pyproject.toml | 2 +- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/poetry.lock b/poetry.lock index c70652e35a..6eeb4a644b 100644 --- a/poetry.lock +++ b/poetry.lock @@ -1599,14 +1599,14 @@ tests = ["pytest (>=5.4.1)", "pytest-cov (>=2.8.1)", "pytest-mypy (>=0.8.0)", "p [[package]] name = "pre-commit" -version = "3.0.4" +version = "3.1.0" description = "A framework for managing and maintaining multi-language pre-commit hooks." category = "dev" optional = false python-versions = ">=3.8" files = [ - {file = "pre_commit-3.0.4-py2.py3-none-any.whl", hash = "sha256:9e3255edb0c9e7fe9b4f328cb3dc86069f8fdc38026f1bf521018a05eaf4d67b"}, - {file = "pre_commit-3.0.4.tar.gz", hash = "sha256:bc4687478d55578c4ac37272fe96df66f73d9b5cf81be6f28627d4e712e752d5"}, + {file = "pre_commit-3.1.0-py2.py3-none-any.whl", hash = "sha256:7001dfcd174540658822b1fd3630ceadf4f41375a5d1844b5c3b3830f227348c"}, + {file = "pre_commit-3.1.0.tar.gz", hash = "sha256:61bd9f1b96d3d1e763f2a9a0f8522aed341646800642ff6803c73fac5781f5b7"}, ] [package.dependencies] @@ -2553,4 +2553,4 @@ snappy = ["python-snappy"] [metadata] lock-version = "2.0" python-versions = "^3.8" -content-hash = "418ac8ae266f29e7a88e16962a7f9416b35936f702a1403050c323a8643836c5" +content-hash = "579733f53f7b75a10b5bc22ff141ce46f9cd1bdca2df28f7122cb0f1c0e0c379" diff --git a/pyproject.toml b/pyproject.toml index 39d81a4e46..e73365eac7 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -81,7 +81,7 @@ adlfs = { version = ">=2022.8.2,<=2023.1.0", optional = true } [tool.poetry.dev-dependencies] pytest = "7.2.1" pytest-checkdocs = "2.9.0" -pre-commit = "3.0.4" +pre-commit = "3.1.0" fastavro = "1.7.2" coverage = { version = "^7.2.0", extras = ["toml"] } requests-mock = "1.10.0" From ecdde0ccadc8d8f71a596646dce0ef03e571ea71 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Sun, 26 Feb 2023 13:07:06 +0100 Subject: [PATCH 385/642] Build: Bump moto from 4.1.2 to 4.1.3 in /python (#6940) --- poetry.lock | 8 ++++---- pyproject.toml | 2 +- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/poetry.lock b/poetry.lock index 6eeb4a644b..e462387423 100644 --- a/poetry.lock +++ b/poetry.lock @@ -1224,14 +1224,14 @@ files = [ [[package]] name = "moto" -version = "4.1.2" +version = "4.1.3" description = "" category = "dev" optional = false python-versions = ">=3.7" files = [ - {file = "moto-4.1.2-py2.py3-none-any.whl", hash = "sha256:1b361ece638c74a657325378a259276f368aafce2f8be84f8143e69fa93ce8ec"}, - {file = "moto-4.1.2.tar.gz", hash = "sha256:63431733d2a02c7bd652ad71ec1da442a0e0d580cbac5eeb50d440a2ce066eac"}, + {file = "moto-4.1.3-py2.py3-none-any.whl", hash = "sha256:dcd1d06662982cf3c94f36d6348251ccdcf62a1c5de5650425cb4e6f260ae7a0"}, + {file = "moto-4.1.3.tar.gz", hash = "sha256:c8200ccaa9440c2e9daa0bd5e0bd768a719db5a2c82ea8d782f0e3fa09a3c5e2"}, ] [package.dependencies] @@ -2553,4 +2553,4 @@ snappy = ["python-snappy"] [metadata] lock-version = "2.0" python-versions = "^3.8" -content-hash = "579733f53f7b75a10b5bc22ff141ce46f9cd1bdca2df28f7122cb0f1c0e0c379" +content-hash = "a9347540fc649099f8f9837333893efffb537957f95f458cf1175b68549e81ac" diff --git a/pyproject.toml b/pyproject.toml index e73365eac7..9cfdc9aa1d 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -85,7 +85,7 @@ pre-commit = "3.1.0" fastavro = "1.7.2" coverage = { version = "^7.2.0", extras = ["toml"] } requests-mock = "1.10.0" -moto = "^4.1.2" +moto = "^4.1.3" typing-extensions = '4.5.0' [tool.poetry.scripts] From 8aa8610590bd9d81dda40924b88fe214040d1c34 Mon Sep 17 00:00:00 2001 From: Fokko Driesprong Date: Mon, 27 Feb 2023 23:02:49 +0100 Subject: [PATCH 386/642] Python: Filter on Datafile metrics (#6714) --- pyiceberg/conversions.py | 3 +- pyiceberg/expressions/literals.py | 3 + pyiceberg/expressions/visitors.py | 274 +++++++++- pyiceberg/io/pyarrow.py | 4 +- pyiceberg/table/__init__.py | 23 +- tests/expressions/test_evaluator.py | 803 ++++++++++++++++++++++++---- 6 files changed, 1002 insertions(+), 108 deletions(-) diff --git a/pyiceberg/conversions.py b/pyiceberg/conversions.py index 4707dab85c..948ddcdcad 100644 --- a/pyiceberg/conversions.py +++ b/pyiceberg/conversions.py @@ -38,6 +38,7 @@ Union, ) +from pyiceberg.typedef import L from pyiceberg.types import ( BinaryType, BooleanType, @@ -241,7 +242,7 @@ def _(primitive_type: DecimalType, value: Decimal) -> bytes: @singledispatch -def from_bytes(primitive_type: PrimitiveType, b: bytes) -> Union[bool, bytes, Decimal, float, int, str, uuid.UUID]: +def from_bytes(primitive_type: PrimitiveType, b: bytes) -> L: """A generic function which converts bytes to a built-in python value Args: diff --git a/pyiceberg/expressions/literals.py b/pyiceberg/expressions/literals.py index 3e3233af0d..ba380cfd67 100644 --- a/pyiceberg/expressions/literals.py +++ b/pyiceberg/expressions/literals.py @@ -328,6 +328,9 @@ def __le__(self, other: Any) -> bool: def __ge__(self, other: Any) -> bool: return self._value32 >= other + def __hash__(self) -> int: + return hash(self._value32) + @singledispatchmethod def to(self, type_var: IcebergType) -> Literal: # type: ignore raise TypeError(f"Cannot convert FloatLiteral into {type_var}") diff --git a/pyiceberg/expressions/visitors.py b/pyiceberg/expressions/visitors.py index 81e0ebc559..4fc666bcd9 100644 --- a/pyiceberg/expressions/visitors.py +++ b/pyiceberg/expressions/visitors.py @@ -14,11 +14,13 @@ # KIND, either express or implied. See the License for the # specific language governing permissions and limitations # under the License. +import math from abc import ABC, abstractmethod from functools import singledispatch from typing import ( Any, Callable, + Dict, Generic, List, Set, @@ -56,15 +58,16 @@ UnboundPredicate, ) from pyiceberg.expressions.literals import Literal -from pyiceberg.manifest import ManifestFile, PartitionFieldSummary +from pyiceberg.manifest import DataFile, ManifestFile, PartitionFieldSummary from pyiceberg.partitioning import PartitionSpec from pyiceberg.schema import Schema -from pyiceberg.typedef import StructProtocol +from pyiceberg.typedef import EMPTY_DICT, StructProtocol from pyiceberg.types import ( DoubleType, FloatType, IcebergType, PrimitiveType, + StructType, TimestampType, TimestamptzType, ) @@ -986,3 +989,270 @@ def expression_to_plain_format( # In the form of expr1 ∨ expr2 ∨ ... ∨ exprN visitor = ExpressionToPlainFormat(cast_int_to_datetime) return [visit(expression, visitor) for expression in expressions] + + +class _InclusiveMetricsEvaluator(BoundBooleanExpressionVisitor[bool]): + struct: StructType + expr: BooleanExpression + + value_counts: Dict[int, int] + null_counts: Dict[int, int] + nan_counts: Dict[int, int] + lower_bounds: Dict[int, bytes] + upper_bounds: Dict[int, bytes] + + def __init__(self, schema: Schema, expr: BooleanExpression, case_sensitive: bool = True) -> None: + self.struct = schema.as_struct() + self.expr = bind(schema, rewrite_not(expr), case_sensitive) + + def eval(self, file: DataFile) -> bool: + """Test whether the file may contain records that match the expression.""" + + if file.record_count == 0: + return ROWS_CANNOT_MATCH + + if file.record_count < 0: + # Older version don't correctly implement record count from avro file and thus + # set record count -1 when importing avro tables to iceberg tables. This should + # be updated once we implemented and set correct record count. + return ROWS_MIGHT_MATCH + + self.value_counts = file.value_counts or EMPTY_DICT + self.null_counts = file.null_value_counts or EMPTY_DICT + self.nan_counts = file.nan_value_counts or EMPTY_DICT + self.lower_bounds = file.lower_bounds or EMPTY_DICT + self.upper_bounds = file.upper_bounds or EMPTY_DICT + + return visit(self.expr, self) + + def _contains_nulls_only(self, field_id: int) -> bool: + if (value_count := self.value_counts.get(field_id)) and (null_count := self.null_counts.get(field_id)): + return value_count == null_count + return False + + def _contains_nans_only(self, field_id: int) -> bool: + if (nan_count := self.nan_counts.get(field_id)) and (value_count := self.value_counts.get(field_id)): + return nan_count == value_count + return False + + def _is_nan(self, val: Any) -> bool: + try: + return math.isnan(val) + except TypeError: + # In the case of None or other non-numeric types + return False + + def visit_true(self) -> bool: + # all rows match + return ROWS_MIGHT_MATCH + + def visit_false(self) -> bool: + # all rows fail + return ROWS_CANNOT_MATCH + + def visit_not(self, child_result: bool) -> bool: + raise ValueError(f"NOT should be rewritten: {child_result}") + + def visit_and(self, left_result: bool, right_result: bool) -> bool: + return left_result and right_result + + def visit_or(self, left_result: bool, right_result: bool) -> bool: + return left_result or right_result + + def visit_is_null(self, term: BoundTerm[L]) -> bool: + field_id = term.ref().field.field_id + + if self.null_counts.get(field_id) == 0: + return ROWS_CANNOT_MATCH + + return ROWS_MIGHT_MATCH + + def visit_not_null(self, term: BoundTerm[L]) -> bool: + # no need to check whether the field is required because binding evaluates that case + # if the column has no non-null values, the expression cannot match + field_id = term.ref().field.field_id + + if self._contains_nulls_only(field_id): + return ROWS_CANNOT_MATCH + + return ROWS_MIGHT_MATCH + + def visit_is_nan(self, term: BoundTerm[L]) -> bool: + field_id = term.ref().field.field_id + + if self.nan_counts.get(field_id) == 0: + return ROWS_CANNOT_MATCH + + # when there's no nanCounts information, but we already know the column only contains null, + # it's guaranteed that there's no NaN value + if self._contains_nulls_only(field_id): + return ROWS_CANNOT_MATCH + + return ROWS_MIGHT_MATCH + + def visit_not_nan(self, term: BoundTerm[L]) -> bool: + field_id = term.ref().field.field_id + + if self._contains_nans_only(field_id): + return ROWS_CANNOT_MATCH + + return ROWS_MIGHT_MATCH + + def visit_less_than(self, term: BoundTerm[L], literal: Literal[L]) -> bool: + field = term.ref().field + field_id = field.field_id + + if self._contains_nulls_only(field_id) or self._contains_nans_only(field_id): + return ROWS_CANNOT_MATCH + + if not isinstance(field.field_type, PrimitiveType): + raise ValueError(f"Expected PrimitiveType: {field.field_type}") + + if lower_bound_bytes := self.lower_bounds.get(field_id): # type: ignore + lower_bound = from_bytes(field.field_type, lower_bound_bytes) # type: ignore + + if self._is_nan(lower_bound): + # NaN indicates unreliable bounds. See the InclusiveMetricsEvaluator docs for more. + return ROWS_MIGHT_MATCH + + if lower_bound >= literal.value: + return ROWS_CANNOT_MATCH + + return ROWS_MIGHT_MATCH + + def visit_less_than_or_equal(self, term: BoundTerm[L], literal: Literal[L]) -> bool: + field = term.ref().field + field_id = field.field_id + + if self._contains_nulls_only(field_id) or self._contains_nans_only(field_id): + return ROWS_CANNOT_MATCH + + if not isinstance(field.field_type, PrimitiveType): + raise ValueError(f"Expected PrimitiveType: {field.field_type}") + + if lower_bound_bytes := self.lower_bounds.get(field_id): # type: ignore + lower_bound = from_bytes(field.field_type, lower_bound_bytes) # type: ignore + if self._is_nan(lower_bound): + # NaN indicates unreliable bounds. See the InclusiveMetricsEvaluator docs for more. + return ROWS_MIGHT_MATCH + + if lower_bound > literal.value: + return ROWS_CANNOT_MATCH + + return ROWS_MIGHT_MATCH + + def visit_greater_than(self, term: BoundTerm[L], literal: Literal[L]) -> bool: + field = term.ref().field + field_id = field.field_id + + if self._contains_nulls_only(field_id) or self._contains_nans_only(field_id): + return ROWS_CANNOT_MATCH + + if not isinstance(field.field_type, PrimitiveType): + raise ValueError(f"Expected PrimitiveType: {field.field_type}") + + if upper_bound_bytes := self.upper_bounds.get(field_id): # type: ignore + upper_bound = from_bytes(field.field_type, upper_bound_bytes) # type: ignore + if upper_bound <= literal.value: + if self._is_nan(upper_bound): + # NaN indicates unreliable bounds. See the InclusiveMetricsEvaluator docs for more. + return ROWS_MIGHT_MATCH + + return ROWS_CANNOT_MATCH + + return ROWS_MIGHT_MATCH + + def visit_greater_than_or_equal(self, term: BoundTerm[L], literal: Literal[L]) -> bool: + field = term.ref().field + field_id = field.field_id + + if self._contains_nulls_only(field_id) or self._contains_nans_only(field_id): + return ROWS_CANNOT_MATCH + + if not isinstance(field.field_type, PrimitiveType): + raise ValueError(f"Expected PrimitiveType: {field.field_type}") + + if upper_bound_bytes := self.upper_bounds.get(field_id): # type: ignore + upper_bound = from_bytes(field.field_type, upper_bound_bytes) # type: ignore + if upper_bound < literal.value: + if self._is_nan(upper_bound): + # NaN indicates unreliable bounds. See the InclusiveMetricsEvaluator docs for more. + return ROWS_MIGHT_MATCH + + return ROWS_CANNOT_MATCH + + return ROWS_MIGHT_MATCH + + def visit_equal(self, term: BoundTerm[L], literal: Literal[L]) -> bool: + field = term.ref().field + field_id = field.field_id + + if self._contains_nulls_only(field_id) or self._contains_nans_only(field_id): + return ROWS_CANNOT_MATCH + + if not isinstance(field.field_type, PrimitiveType): + raise ValueError(f"Expected PrimitiveType: {field.field_type}") + + if lower_bound_bytes := self.lower_bounds.get(field_id): # type: ignore + lower_bound = from_bytes(field.field_type, lower_bound_bytes) # type: ignore + if self._is_nan(lower_bound): + # NaN indicates unreliable bounds. See the InclusiveMetricsEvaluator docs for more. + return ROWS_MIGHT_MATCH + + if lower_bound > literal.value: + return ROWS_CANNOT_MATCH + + if upper_bound_bytes := self.upper_bounds.get(field_id): # type: ignore + upper_bound = from_bytes(field.field_type, upper_bound_bytes) # type: ignore + if self._is_nan(upper_bound): + # NaN indicates unreliable bounds. See the InclusiveMetricsEvaluator docs for more. + return ROWS_MIGHT_MATCH + + if upper_bound < literal.value: + return ROWS_CANNOT_MATCH + + return ROWS_MIGHT_MATCH + + def visit_not_equal(self, term: BoundTerm[L], literal: Literal[L]) -> bool: + return ROWS_MIGHT_MATCH + + def visit_in(self, term: BoundTerm[L], literals: Set[L]) -> bool: + field = term.ref().field + field_id = field.field_id + + if self._contains_nulls_only(field_id) or self._contains_nans_only(field_id): + return ROWS_CANNOT_MATCH + + if len(literals) > IN_PREDICATE_LIMIT: + # skip evaluating the predicate if the number of values is too big + return ROWS_MIGHT_MATCH + + if not isinstance(field.field_type, PrimitiveType): + raise ValueError(f"Expected PrimitiveType: {field.field_type}") + + if lower_bound_bytes := self.lower_bounds.get(field_id): # type: ignore + lower_bound = from_bytes(field.field_type, lower_bound_bytes) # type: ignore + if self._is_nan(lower_bound): + # NaN indicates unreliable bounds. See the InclusiveMetricsEvaluator docs for more. + return ROWS_MIGHT_MATCH + + literals = {lit for lit in literals if lower_bound <= lit} + if len(literals) == 0: + return ROWS_CANNOT_MATCH + + if upper_bound_bytes := self.upper_bounds.get(field_id): # type: ignore + upper_bound = from_bytes(field.field_type, upper_bound_bytes) # type: ignore + # this is different from Java, here NaN is always larger + if self._is_nan(upper_bound): + return ROWS_MIGHT_MATCH + + literals = {lit for lit in literals if upper_bound >= lit} + if len(literals) == 0: + return ROWS_CANNOT_MATCH + + return ROWS_MIGHT_MATCH + + def visit_not_in(self, term: BoundTerm[L], literals: Set[L]) -> bool: + # because the bounds are not necessarily a min or max value, this cannot be answered using + # them. notIn(col, {X, ...}) with (X, Y) doesn't guarantee that X is a value in col. + return ROWS_MIGHT_MATCH diff --git a/pyiceberg/io/pyarrow.py b/pyiceberg/io/pyarrow.py index a064ae3bdf..d2cd762273 100644 --- a/pyiceberg/io/pyarrow.py +++ b/pyiceberg/io/pyarrow.py @@ -555,8 +555,10 @@ def project_table( if len(tables) > 1: return pa.concat_tables(tables) - else: + elif len(tables) == 1: return tables[0] + else: + return pa.Table.from_batches([], schema=schema_to_pyarrow(projected_schema)) def to_requested_schema(requested_schema: Schema, file_schema: Schema, table: pa.Table) -> pa.Table: diff --git a/pyiceberg/table/__init__.py b/pyiceberg/table/__init__.py index 7c53fd0967..69ce08f457 100644 --- a/pyiceberg/table/__init__.py +++ b/pyiceberg/table/__init__.py @@ -43,7 +43,7 @@ parser, visitors, ) -from pyiceberg.expressions.visitors import inclusive_projection +from pyiceberg.expressions.visitors import _InclusiveMetricsEvaluator, inclusive_projection from pyiceberg.io import FileIO, load_file_io from pyiceberg.manifest import ( DataFile, @@ -314,11 +314,16 @@ def _check_content(file: DataFile) -> DataFile: return file -def _open_manifest(io: FileIO, manifest: ManifestFile, partition_filter: Callable[[DataFile], bool]) -> List[FileScanTask]: +def _open_manifest( + io: FileIO, + manifest: ManifestFile, + partition_filter: Callable[[DataFile], bool], + metrics_evaluator: Callable[[DataFile], bool], +) -> List[FileScanTask]: all_files = files(io.new_input(manifest.manifest_path)) matching_partition_files = filter(partition_filter, all_files) matching_partition_data_files = map(_check_content, matching_partition_files) - return [FileScanTask(file) for file in matching_partition_data_files] + return [FileScanTask(file) for file in matching_partition_data_files if metrics_evaluator(file)] class DataScan(TableScan): @@ -376,12 +381,20 @@ def plan_files(self) -> Iterator[FileScanTask]: # this filter depends on the partition spec used to write the manifest file partition_evaluators: Dict[int, Callable[[DataFile], bool]] = KeyDefaultDict(self._build_partition_evaluator) - + metrics_evaluator = _InclusiveMetricsEvaluator(self.table.schema(), self.row_filter, self.case_sensitive).eval with ThreadPool() as pool: return chain( *pool.starmap( func=_open_manifest, - iterable=[(io, manifest, partition_evaluators[manifest.partition_spec_id]) for manifest in manifests], + iterable=[ + ( + io, + manifest, + partition_evaluators[manifest.partition_spec_id], + metrics_evaluator, + ) + for manifest in manifests + ], ) ) diff --git a/tests/expressions/test_evaluator.py b/tests/expressions/test_evaluator.py index b57ede62f2..7de9c04fb7 100644 --- a/tests/expressions/test_evaluator.py +++ b/tests/expressions/test_evaluator.py @@ -14,11 +14,13 @@ # KIND, either express or implied. See the License for the # specific language governing permissions and limitations # under the License. -from typing import Optional +# pylint:disable=redefined-outer-name +from typing import Any +import pytest + +from pyiceberg.conversions import to_bytes from pyiceberg.expressions import ( - AlwaysFalse, - AlwaysTrue, And, EqualTo, GreaterThan, @@ -35,145 +37,748 @@ NotNull, Or, ) -from pyiceberg.expressions.visitors import expression_evaluator +from pyiceberg.expressions.visitors import _InclusiveMetricsEvaluator +from pyiceberg.manifest import DataFile, FileFormat from pyiceberg.schema import Schema -from pyiceberg.typedef import Record from pyiceberg.types import ( DoubleType, - LongType, + FloatType, + IcebergType, + IntegerType, NestedField, + PrimitiveType, StringType, ) -SIMPLE_SCHEMA = Schema( - NestedField(id=1, name="id", field_type=LongType()), NestedField(id=2, name="data", field_type=StringType(), required=False) -) +INT_MIN_VALUE = 30 +INT_MAX_VALUE = 79 + + +def _to_byte_buffer(field_type: IcebergType, val: Any) -> bytes: + if not isinstance(field_type, PrimitiveType): + raise ValueError(f"Expected a PrimitiveType, got: {type(field_type)}") + return to_bytes(field_type, val) + + +INT_MIN = _to_byte_buffer(IntegerType(), INT_MIN_VALUE) +INT_MAX = _to_byte_buffer(IntegerType(), INT_MAX_VALUE) + +STRING_MIN = _to_byte_buffer(StringType(), "a") +STRING_MAX = _to_byte_buffer(StringType(), "z") + + +@pytest.fixture +def schema_data_file() -> Schema: + return Schema( + NestedField(1, "id", IntegerType(), required=True), + NestedField(2, "no_stats", IntegerType(), required=False), + NestedField(3, "required", StringType(), required=True), + NestedField(4, "all_nulls", StringType(), required=False), + NestedField(5, "some_nulls", StringType(), required=False), + NestedField(6, "no_nulls", StringType(), required=False), + NestedField(7, "all_nans", DoubleType(), required=False), + NestedField(8, "some_nans", FloatType(), required=False), + NestedField(9, "no_nans", FloatType(), required=False), + NestedField(10, "all_nulls_double", DoubleType(), required=False), + NestedField(11, "all_nans_v1_stats", FloatType(), required=False), + NestedField(12, "nan_and_null_only", DoubleType(), required=False), + NestedField(13, "no_nan_stats", DoubleType(), required=False), + NestedField(14, "some_empty", StringType(), required=False), + ) + + +@pytest.fixture +def data_file() -> DataFile: + return DataFile( + file_path="file_1.parquet", + file_format=FileFormat.PARQUET, + partition={}, + record_count=50, + file_size_in_bytes=3, + value_counts={ + 1: 50, + 2: 50, + 3: 50, + 4: 50, + 5: 50, + 6: 50, + 7: 50, + 8: 50, + 9: 50, + 10: 50, + 11: 50, + 12: 50, + 13: 50, + 14: 50, + }, + null_value_counts={4: 50, 5: 10, 6: 0, 10: 50, 11: 0, 12: 1, 14: 8}, + nan_value_counts={ + 7: 50, + 8: 10, + 9: 0, + }, + lower_bounds={ + 1: to_bytes(IntegerType(), INT_MIN_VALUE), + 11: to_bytes(FloatType(), float("nan")), + 12: to_bytes(DoubleType(), float("nan")), + 14: to_bytes(StringType(), ""), + }, + upper_bounds={ + 1: to_bytes(IntegerType(), INT_MAX_VALUE), + 11: to_bytes(FloatType(), float("nan")), + 12: to_bytes(DoubleType(), float("nan")), + 14: to_bytes(StringType(), "房东整租霍营小区二层两居室"), + }, + ) -FLOAT_SCHEMA = Schema( - NestedField(id=1, name="id", field_type=LongType()), NestedField(id=2, name="f", field_type=DoubleType(), required=False) -) +def test_all_null(schema_data_file: Schema, data_file: DataFile) -> None: + should_read = _InclusiveMetricsEvaluator(schema_data_file, NotNull("all_nulls")).eval(data_file) + assert not should_read, "Should skip: no non-null value in all null column" + + should_read = _InclusiveMetricsEvaluator(schema_data_file, LessThan("all_nulls", "a")).eval(data_file) + assert not should_read, "Should skip: lessThan on all null column" + + should_read = _InclusiveMetricsEvaluator(schema_data_file, LessThanOrEqual("all_nulls", "a")).eval(data_file) + assert not should_read, "Should skip: lessThanOrEqual on all null column" + + should_read = _InclusiveMetricsEvaluator(schema_data_file, GreaterThan("all_nulls", "a")).eval(data_file) + assert not should_read, "Should skip: greaterThan on all null column" + + should_read = _InclusiveMetricsEvaluator(schema_data_file, GreaterThanOrEqual("all_nulls", "a")).eval(data_file) + assert not should_read, "Should skip: greaterThanOrEqual on all null column" + + should_read = _InclusiveMetricsEvaluator(schema_data_file, EqualTo("all_nulls", "a")).eval(data_file) + assert not should_read, "Should skip: equal on all null column" + + should_read = _InclusiveMetricsEvaluator(schema_data_file, NotNull("some_nulls")).eval(data_file) + assert should_read, "Should read: column with some nulls contains a non-null value" + + should_read = _InclusiveMetricsEvaluator(schema_data_file, NotNull("no_nulls")).eval(data_file) + assert should_read, "Should read: non-null column contains a non-null value" + + +def test_no_nulls(schema_data_file: Schema, data_file: DataFile) -> None: + should_read = _InclusiveMetricsEvaluator(schema_data_file, IsNull("all_nulls")).eval(data_file) + assert should_read, "Should read: at least one null value in all null column" + + should_read = _InclusiveMetricsEvaluator(schema_data_file, IsNull("some_nulls")).eval(data_file) + assert should_read, "Should read: column with some nulls contains a null value" + + should_read = _InclusiveMetricsEvaluator(schema_data_file, IsNull("no_nulls")).eval(data_file) + assert not should_read, "Should skip: non-null column contains no null values" + + +def test_is_nan(schema_data_file: Schema, data_file: DataFile) -> None: + should_read = _InclusiveMetricsEvaluator(schema_data_file, IsNaN("all_nans")).eval(data_file) + assert should_read, "Should read: at least one nan value in all nan column" + + should_read = _InclusiveMetricsEvaluator(schema_data_file, IsNaN("some_nans")).eval(data_file) + assert should_read, "Should read: at least one nan value in some nan column" + + should_read = _InclusiveMetricsEvaluator(schema_data_file, IsNaN("no_nans")).eval(data_file) + assert not should_read, "Should skip: no-nans column contains no nan values" + + should_read = _InclusiveMetricsEvaluator(schema_data_file, IsNaN("all_nulls_double")).eval(data_file) + assert not should_read, "Should skip: all-null column doesn't contain nan value" + + should_read = _InclusiveMetricsEvaluator(schema_data_file, IsNaN("no_nan_stats")).eval(data_file) + assert should_read, "Should read: no guarantee on if contains nan value without nan stats" + + should_read = _InclusiveMetricsEvaluator(schema_data_file, IsNaN("all_nans_v1_stats")).eval(data_file) + assert should_read, "Should read: at least one nan value in all nan column" + + should_read = _InclusiveMetricsEvaluator(schema_data_file, IsNaN("nan_and_null_only")).eval(data_file) + assert should_read, "Should read: at least one nan value in nan and nulls only column" + + +def test_not_nan(schema_data_file: Schema, data_file: DataFile) -> None: + should_read = _InclusiveMetricsEvaluator(schema_data_file, NotNaN("all_nans")).eval(data_file) + assert not should_read, "Should skip: column with all nans will not contain non-nan" + + should_read = _InclusiveMetricsEvaluator(schema_data_file, NotNaN("some_nans")).eval(data_file) + assert should_read, "Should read: at least one non-nan value in some nan column" + + should_read = _InclusiveMetricsEvaluator(schema_data_file, NotNaN("no_nans")).eval(data_file) + assert should_read, "Should read: at least one non-nan value in no nan column" + + should_read = _InclusiveMetricsEvaluator(schema_data_file, NotNaN("all_nulls_double")).eval(data_file) + assert should_read, "Should read: at least one non-nan value in all null column" + + should_read = _InclusiveMetricsEvaluator(schema_data_file, NotNaN("no_nan_stats")).eval(data_file) + assert should_read, "Should read: no guarantee on if contains nan value without nan stats" + + should_read = _InclusiveMetricsEvaluator(schema_data_file, NotNaN("all_nans_v1_stats")).eval(data_file) + assert should_read, "Should read: no guarantee on if contains nan value without nan stats" + + should_read = _InclusiveMetricsEvaluator(schema_data_file, NotNaN("nan_and_null_only")).eval(data_file) + assert should_read, "Should read: at least one null value in nan and nulls only column" + + +def test_required_column(schema_data_file: Schema, data_file: DataFile) -> None: + should_read = _InclusiveMetricsEvaluator(schema_data_file, NotNull("required")).eval(data_file) + assert should_read, "Should read: required columns are always non-null" + + should_read = _InclusiveMetricsEvaluator(schema_data_file, IsNull("required")).eval(data_file) + assert not should_read, "Should skip: required columns are always non-null" + + +def test_missing_column(schema_data_file: Schema, data_file: DataFile) -> None: + with pytest.raises(ValueError) as exc_info: + _ = _InclusiveMetricsEvaluator(schema_data_file, LessThan("missing", 22)).eval(data_file) + + assert str(exc_info.value) == "Could not find field with name missing, case_sensitive=True" + + +def test_missing_stats() -> None: + no_stats_schema = Schema( + NestedField(2, "no_stats", DoubleType(), required=False), + ) + + no_stats_file = DataFile( + file_path="file_1.parquet", + file_format=FileFormat.PARQUET, + partition={}, + record_count=50, + value_counts=None, + null_value_counts=None, + nan_value_counts=None, + lower_bounds=None, + upper_bounds=None, + ) + + expressions = [ + LessThan("no_stats", 5), + LessThanOrEqual("no_stats", 30), + EqualTo("no_stats", 70), + GreaterThan("no_stats", 78), + GreaterThanOrEqual("no_stats", 90), + NotEqualTo("no_stats", 101), + IsNull("no_stats"), + NotNull("no_stats"), + IsNaN("no_stats"), + NotNaN("no_stats"), + ] + + for expression in expressions: + should_read = _InclusiveMetricsEvaluator(no_stats_schema, expression).eval(no_stats_file) + assert should_read, f"Should read when stats are missing for: {expression}" + + +def test_zero_record_file_stats(schema_data_file: Schema) -> None: + zero_record_data_file = DataFile(file_path="file_1.parquet", file_format=FileFormat.PARQUET, partition={}, record_count=0) + + expressions = [ + LessThan("no_stats", 5), + LessThanOrEqual("no_stats", 30), + EqualTo("no_stats", 70), + GreaterThan("no_stats", 78), + GreaterThanOrEqual("no_stats", 90), + NotEqualTo("no_stats", 101), + IsNull("no_stats"), + NotNull("no_stats"), + IsNaN("no_stats"), + NotNaN("no_stats"), + ] + + for expression in expressions: + should_read = _InclusiveMetricsEvaluator(schema_data_file, expression).eval(zero_record_data_file) + assert not should_read, f"Should skip a datafile without records: {expression}" + + +def test_not(schema_data_file: Schema, data_file: DataFile) -> None: + should_read = _InclusiveMetricsEvaluator(schema_data_file, Not(LessThan("id", INT_MIN_VALUE - 25))).eval(data_file) + assert should_read, "Should read: not(false)" + + should_read = _InclusiveMetricsEvaluator(schema_data_file, Not(GreaterThan("id", INT_MIN_VALUE - 25))).eval(data_file) + assert not should_read, "Should skip: not(true)" + + +def test_and(schema_data_file: Schema, data_file: DataFile) -> None: + should_read = _InclusiveMetricsEvaluator( + schema_data_file, And(LessThan("id", INT_MIN_VALUE - 25), GreaterThanOrEqual("id", INT_MIN_VALUE - 30)) + ).eval(data_file) + assert not should_read, "Should skip: and(false, true)" + + should_read = _InclusiveMetricsEvaluator( + schema_data_file, And(LessThan("id", INT_MIN_VALUE - 25), GreaterThanOrEqual("id", INT_MIN_VALUE + 1)) + ).eval(data_file) + assert not should_read, "Should skip: and(false, false)" + + should_read = _InclusiveMetricsEvaluator( + schema_data_file, And(GreaterThan("id", INT_MIN_VALUE - 25), LessThanOrEqual("id", INT_MIN_VALUE)) + ).eval(data_file) + assert should_read, "Should read: and(true, true)" + + +def test_or(schema_data_file: Schema, data_file: DataFile) -> None: + should_read = _InclusiveMetricsEvaluator( + schema_data_file, Or(LessThan("id", INT_MIN_VALUE - 25), GreaterThanOrEqual("id", INT_MAX_VALUE + 1)) + ).eval(data_file) + assert not should_read, "Should skip: or(false, false)" + + should_read = _InclusiveMetricsEvaluator( + schema_data_file, Or(LessThan("id", INT_MIN_VALUE - 25), GreaterThanOrEqual("id", INT_MAX_VALUE - 19)) + ).eval(data_file) + assert should_read, "Should read: or(false, true)" + + +def test_integer_lt(schema_data_file: Schema, data_file: DataFile) -> None: + should_read = _InclusiveMetricsEvaluator(schema_data_file, LessThan("id", INT_MIN_VALUE - 25)).eval(data_file) + assert not should_read, "Should not read: id range below lower bound (5 < 30)" + + should_read = _InclusiveMetricsEvaluator(schema_data_file, LessThan("id", INT_MIN_VALUE)).eval(data_file) + assert not should_read, "Should not read: id range below lower bound (30 is not < 30)" + + should_read = _InclusiveMetricsEvaluator(schema_data_file, LessThan("id", INT_MIN_VALUE + 1)).eval(data_file) + assert should_read, "Should read: one possible id" + + should_read = _InclusiveMetricsEvaluator(schema_data_file, LessThan("id", INT_MAX_VALUE)).eval(data_file) + assert should_read, "Should read: may possible ids" + + +def test_integer_lt_eq(schema_data_file: Schema, data_file: DataFile) -> None: + should_read = _InclusiveMetricsEvaluator(schema_data_file, LessThanOrEqual("id", INT_MIN_VALUE - 25)).eval(data_file) + assert not should_read, "Should not read: id range below lower bound (5 < 30)" + + should_read = _InclusiveMetricsEvaluator(schema_data_file, LessThanOrEqual("id", INT_MIN_VALUE - 1)).eval(data_file) + assert not should_read, "Should not read: id range below lower bound (30 is not < 30)" + + should_read = _InclusiveMetricsEvaluator(schema_data_file, LessThanOrEqual("id", INT_MIN_VALUE)).eval(data_file) + assert should_read, "Should read: one possible id" + + should_read = _InclusiveMetricsEvaluator(schema_data_file, LessThanOrEqual("id", INT_MAX_VALUE)).eval(data_file) + assert should_read, "Should read: may possible ids" + + +def test_integer_gt(schema_data_file: Schema, data_file: DataFile) -> None: + should_read = _InclusiveMetricsEvaluator(schema_data_file, GreaterThan("id", INT_MAX_VALUE + 6)).eval(data_file) + assert not should_read, "Should not read: id range above upper bound (85 < 79)" + + should_read = _InclusiveMetricsEvaluator(schema_data_file, GreaterThan("id", INT_MAX_VALUE)).eval(data_file) + assert not should_read, "Should not read: id range above upper bound (79 is not > 79)" + + should_read = _InclusiveMetricsEvaluator(schema_data_file, GreaterThan("id", INT_MIN_VALUE - 1)).eval(data_file) + assert should_read, "Should read: one possible id" + + should_read = _InclusiveMetricsEvaluator(schema_data_file, GreaterThan("id", INT_MAX_VALUE - 4)).eval(data_file) + assert should_read, "Should read: may possible ids" + + +def test_integer_gt_eq(schema_data_file: Schema, data_file: DataFile) -> None: + should_read = _InclusiveMetricsEvaluator(schema_data_file, GreaterThanOrEqual("id", INT_MAX_VALUE + 6)).eval(data_file) + assert not should_read, "Should not read: id range above upper bound (85 < 79)" + + should_read = _InclusiveMetricsEvaluator(schema_data_file, GreaterThanOrEqual("id", INT_MAX_VALUE + 1)).eval(data_file) + assert not should_read, "Should not read: id range above upper bound (80 > 79)" + + should_read = _InclusiveMetricsEvaluator(schema_data_file, GreaterThanOrEqual("id", INT_MAX_VALUE)).eval(data_file) + assert should_read, "Should read: one possible id" + + should_read = _InclusiveMetricsEvaluator(schema_data_file, GreaterThanOrEqual("id", INT_MAX_VALUE - 4)).eval(data_file) + assert should_read, "Should read: may possible ids" + + +def test_integer_eq(schema_data_file: Schema, data_file: DataFile) -> None: + should_read = _InclusiveMetricsEvaluator(schema_data_file, EqualTo("id", INT_MIN_VALUE - 25)).eval(data_file) + assert not should_read, "Should not read: id below lower bound" + + should_read = _InclusiveMetricsEvaluator(schema_data_file, EqualTo("id", INT_MIN_VALUE - 1)).eval(data_file) + assert not should_read, "Should not read: id below lower bound" + + should_read = _InclusiveMetricsEvaluator(schema_data_file, EqualTo("id", INT_MIN_VALUE)).eval(data_file) + assert should_read, "Should read: id equal to lower bound" + + should_read = _InclusiveMetricsEvaluator(schema_data_file, EqualTo("id", INT_MAX_VALUE - 4)).eval(data_file) + assert should_read, "Should read: id between lower and upper bounds" + + should_read = _InclusiveMetricsEvaluator(schema_data_file, EqualTo("id", INT_MAX_VALUE)).eval(data_file) + assert should_read, "Should read: id equal to upper bound" + + should_read = _InclusiveMetricsEvaluator(schema_data_file, EqualTo("id", INT_MAX_VALUE + 1)).eval(data_file) + assert not should_read, "Should not read: id above upper bound" + + should_read = _InclusiveMetricsEvaluator(schema_data_file, EqualTo("id", INT_MAX_VALUE + 6)).eval(data_file) + assert not should_read, "Should not read: id above upper bound" + + +def test_integer_not_eq(schema_data_file: Schema, data_file: DataFile) -> None: + should_read = _InclusiveMetricsEvaluator(schema_data_file, NotEqualTo("id", INT_MIN_VALUE - 25)).eval(data_file) + assert should_read, "Should read: id below lower bound" + + should_read = _InclusiveMetricsEvaluator(schema_data_file, NotEqualTo("id", INT_MIN_VALUE - 1)).eval(data_file) + assert should_read, "Should read: id below lower bound" + + should_read = _InclusiveMetricsEvaluator(schema_data_file, NotEqualTo("id", INT_MIN_VALUE)).eval(data_file) + assert should_read, "Should read: id equal to lower bound" + + should_read = _InclusiveMetricsEvaluator(schema_data_file, NotEqualTo("id", INT_MAX_VALUE - 4)).eval(data_file) + assert should_read, "Should read: id between lower and upper bounds" + + should_read = _InclusiveMetricsEvaluator(schema_data_file, NotEqualTo("id", INT_MAX_VALUE)).eval(data_file) + assert should_read, "Should read: id equal to upper bound" + + should_read = _InclusiveMetricsEvaluator(schema_data_file, NotEqualTo("id", INT_MAX_VALUE + 1)).eval(data_file) + assert should_read, "Should read: id above upper bound" + + should_read = _InclusiveMetricsEvaluator(schema_data_file, NotEqualTo("id", INT_MAX_VALUE + 6)).eval(data_file) + assert should_read, "Should read: id above upper bound" + + +def test_integer_not_eq_rewritten(schema_data_file: Schema, data_file: DataFile) -> None: + should_read = _InclusiveMetricsEvaluator(schema_data_file, Not(EqualTo("id", INT_MIN_VALUE - 25))).eval(data_file) + assert should_read, "Should read: id below lower bound" + + should_read = _InclusiveMetricsEvaluator(schema_data_file, Not(EqualTo("id", INT_MIN_VALUE - 1))).eval(data_file) + assert should_read, "Should read: id below lower bound" + + should_read = _InclusiveMetricsEvaluator(schema_data_file, Not(EqualTo("id", INT_MIN_VALUE))).eval(data_file) + assert should_read, "Should read: id equal to lower bound" + + should_read = _InclusiveMetricsEvaluator(schema_data_file, Not(EqualTo("id", INT_MAX_VALUE - 4))).eval(data_file) + assert should_read, "Should read: id between lower and upper bounds" + + should_read = _InclusiveMetricsEvaluator(schema_data_file, Not(EqualTo("id", INT_MAX_VALUE))).eval(data_file) + assert should_read, "Should read: id equal to upper bound" + + should_read = _InclusiveMetricsEvaluator(schema_data_file, Not(EqualTo("id", INT_MAX_VALUE + 1))).eval(data_file) + assert should_read, "Should read: id above upper bound" + + should_read = _InclusiveMetricsEvaluator(schema_data_file, Not(EqualTo("id", INT_MAX_VALUE + 6))).eval(data_file) + assert should_read, "Should read: id above upper bound" + + +def test_integer_case_insensitive_not_eq_rewritten(schema_data_file: Schema, data_file: DataFile) -> None: + should_read = _InclusiveMetricsEvaluator(schema_data_file, Not(EqualTo("ID", INT_MIN_VALUE - 25)), case_sensitive=False).eval( + data_file + ) + assert should_read, "Should read: id below lower bound" + + should_read = _InclusiveMetricsEvaluator(schema_data_file, Not(EqualTo("ID", INT_MIN_VALUE - 1)), case_sensitive=False).eval( + data_file + ) + assert should_read, "Should read: id below lower bound" + + should_read = _InclusiveMetricsEvaluator(schema_data_file, Not(EqualTo("ID", INT_MIN_VALUE)), case_sensitive=False).eval( + data_file + ) + assert should_read, "Should read: id equal to lower bound" + + should_read = _InclusiveMetricsEvaluator(schema_data_file, Not(EqualTo("ID", INT_MAX_VALUE - 4)), case_sensitive=False).eval( + data_file + ) + assert should_read, "Should read: id between lower and upper bounds" + + should_read = _InclusiveMetricsEvaluator(schema_data_file, Not(EqualTo("ID", INT_MAX_VALUE)), case_sensitive=False).eval( + data_file + ) + assert should_read, "Should read: id equal to upper bound" + + should_read = _InclusiveMetricsEvaluator(schema_data_file, Not(EqualTo("ID", INT_MAX_VALUE + 1)), case_sensitive=False).eval( + data_file + ) + assert should_read, "Should read: id above upper bound" + + should_read = _InclusiveMetricsEvaluator(schema_data_file, Not(EqualTo("ID", INT_MAX_VALUE + 6)), case_sensitive=False).eval( + data_file + ) + assert should_read, "Should read: id above upper bound" + + +def test_missing_column_case_sensitive(schema_data_file: Schema, data_file: DataFile) -> None: + with pytest.raises(ValueError) as exc_info: + _ = _InclusiveMetricsEvaluator(schema_data_file, LessThan("ID", 22), case_sensitive=True).eval(data_file) + + assert str(exc_info.value) == "Could not find field with name ID, case_sensitive=True" + + +def test_integer_in(schema_data_file: Schema, data_file: DataFile) -> None: + should_read = _InclusiveMetricsEvaluator(schema_data_file, In("id", {INT_MIN_VALUE - 25, INT_MIN_VALUE - 24})).eval(data_file) + assert not should_read, "Should not read: id below lower bound (5 < 30, 6 < 30)" + + should_read = _InclusiveMetricsEvaluator(schema_data_file, In("id", {INT_MIN_VALUE - 2, INT_MIN_VALUE - 1})).eval(data_file) + assert not should_read, "Should not read: id below lower bound (28 < 30, 29 < 30)" + + should_read = _InclusiveMetricsEvaluator(schema_data_file, In("id", {INT_MIN_VALUE - 1, INT_MIN_VALUE})).eval(data_file) + assert should_read, "Should read: id equal to lower bound (30 == 30)" + + should_read = _InclusiveMetricsEvaluator(schema_data_file, In("id", {INT_MAX_VALUE - 4, INT_MAX_VALUE - 3})).eval(data_file) + assert should_read, "Should read: id between lower and upper bounds (30 < 75 < 79, 30 < 76 < 79)" + + should_read = _InclusiveMetricsEvaluator(schema_data_file, In("id", {INT_MAX_VALUE, INT_MAX_VALUE + 1})).eval(data_file) + assert should_read, "Should read: id equal to upper bound (79 == 79)" + + should_read = _InclusiveMetricsEvaluator(schema_data_file, In("id", {INT_MAX_VALUE + 1, INT_MAX_VALUE + 2})).eval(data_file) + assert not should_read, "Should not read: id above upper bound (80 > 79, 81 > 79)" + + should_read = _InclusiveMetricsEvaluator(schema_data_file, In("id", {INT_MAX_VALUE + 6, INT_MAX_VALUE + 7})).eval(data_file) + assert not should_read, "Should not read: id above upper bound (85 > 79, 86 > 79)" + + should_read = _InclusiveMetricsEvaluator(schema_data_file, In("all_nulls", {"abc", "def"})).eval(data_file) + assert not should_read, "Should skip: in on all nulls column" + + should_read = _InclusiveMetricsEvaluator(schema_data_file, In("some_nulls", {"abc", "def"})).eval(data_file) + assert should_read, "Should read: in on some nulls column" + + should_read = _InclusiveMetricsEvaluator(schema_data_file, In("no_nulls", {"abc", "def"})).eval(data_file) + assert should_read, "Should read: in on no nulls column" + + ids = list(range(400)) + should_read = _InclusiveMetricsEvaluator(schema_data_file, In("id", ids)).eval(data_file) + assert should_read, "Should read: large in expression" + + +def test_integer_not_in(schema_data_file: Schema, data_file: DataFile) -> None: + should_read = _InclusiveMetricsEvaluator(schema_data_file, NotIn("id", {INT_MIN_VALUE - 25, INT_MIN_VALUE - 24})).eval( + data_file + ) + assert should_read, "Should read: id below lower bound (5 < 30, 6 < 30)" + + should_read = _InclusiveMetricsEvaluator(schema_data_file, NotIn("id", {INT_MIN_VALUE - 2, INT_MIN_VALUE - 1})).eval( + data_file + ) + assert should_read, "Should read: id below lower bound (28 < 30, 29 < 30)" + + should_read = _InclusiveMetricsEvaluator(schema_data_file, NotIn("id", {INT_MIN_VALUE - 1, INT_MIN_VALUE})).eval(data_file) + assert should_read, "Should read: id equal to lower bound (30 == 30)" + + should_read = _InclusiveMetricsEvaluator(schema_data_file, NotIn("id", {INT_MAX_VALUE - 4, INT_MAX_VALUE - 3})).eval( + data_file + ) + assert should_read, "Should read: id between lower and upper bounds (30 < 75 < 79, 30 < 76 < 79)" + + should_read = _InclusiveMetricsEvaluator(schema_data_file, NotIn("id", {INT_MAX_VALUE, INT_MAX_VALUE + 1})).eval(data_file) + assert should_read, "Should read: id equal to upper bound (79 == 79)" + + should_read = _InclusiveMetricsEvaluator(schema_data_file, NotIn("id", {INT_MAX_VALUE + 1, INT_MAX_VALUE + 2})).eval( + data_file + ) + assert should_read, "Should read: id above upper bound (80 > 79, 81 > 79)" + + should_read = _InclusiveMetricsEvaluator(schema_data_file, NotIn("id", {INT_MAX_VALUE + 6, INT_MAX_VALUE + 7})).eval( + data_file + ) + assert should_read, "Should read: id above upper bound (85 > 79, 86 > 79)" + + should_read = _InclusiveMetricsEvaluator(schema_data_file, NotIn("all_nulls", {"abc", "def"})).eval(data_file) + assert should_read, "Should read: notIn on all nulls column" + + should_read = _InclusiveMetricsEvaluator(schema_data_file, NotIn("some_nulls", {"abc", "def"})).eval(data_file) + assert should_read, "Should read: in on some nulls column" + + should_read = _InclusiveMetricsEvaluator(schema_data_file, NotIn("no_nulls", {"abc", "def"})).eval(data_file) + assert should_read, "Should read: in on no nulls column" + + +@pytest.fixture +def schema_data_file_nan() -> Schema: + return Schema( + NestedField(1, "all_nan", DoubleType(), required=True), + NestedField(2, "max_nan", DoubleType(), required=True), + NestedField(3, "min_max_nan", FloatType(), required=False), + NestedField(4, "all_nan_null_bounds", DoubleType(), required=True), + NestedField(5, "some_nan_correct_bounds", FloatType(), required=False), + ) + + +@pytest.fixture +def data_file_nan() -> DataFile: + return DataFile( + file_path="file.avro", + file_format=FileFormat.PARQUET, + partition={}, + record_count=50, + file_size_in_bytes=3, + column_sizes={ + 1: 10, + 2: 10, + 3: 10, + 4: 10, + 5: 10, + }, + value_counts={ + 1: 10, + 2: 10, + 3: 10, + 4: 10, + 5: 10, + }, + null_value_counts={ + 1: 0, + 2: 0, + 3: 0, + 4: 0, + 5: 0, + }, + nan_value_counts={1: 10, 4: 10, 5: 5}, + lower_bounds={ + 1: to_bytes(DoubleType(), float("nan")), + 2: to_bytes(DoubleType(), 7), + 3: to_bytes(FloatType(), float("nan")), + 5: to_bytes(FloatType(), 7), + }, + upper_bounds={ + 1: to_bytes(DoubleType(), float("nan")), + 2: to_bytes(DoubleType(), float("nan")), + 3: to_bytes(FloatType(), float("nan")), + 5: to_bytes(FloatType(), 22), + }, + ) + + +def test_inclusive_metrics_evaluator_less_than_and_less_than_equal(schema_data_file_nan: Schema, data_file_nan: DataFile) -> None: + for operator in [LessThan, LessThanOrEqual]: + should_read = _InclusiveMetricsEvaluator(schema_data_file_nan, operator("all_nan", 1)).eval(data_file_nan) + assert not should_read, "Should not match: all nan column doesn't contain number" + + should_read = _InclusiveMetricsEvaluator(schema_data_file_nan, operator("max_nan", 1)).eval(data_file_nan) + assert not should_read, "Should not match: 1 is smaller than lower bound" + + should_read = _InclusiveMetricsEvaluator(schema_data_file_nan, operator("max_nan", 10)).eval(data_file_nan) + assert should_read, "Should match: 10 is larger than lower bound" + + should_read = _InclusiveMetricsEvaluator(schema_data_file_nan, operator("min_max_nan", 1)).eval(data_file_nan) + assert should_read, "Should match: no visibility" + + should_read = _InclusiveMetricsEvaluator(schema_data_file_nan, operator("all_nan_null_bounds", 1)).eval(data_file_nan) + assert not should_read, "Should not match: all nan column doesn't contain number" + + should_read = _InclusiveMetricsEvaluator(schema_data_file_nan, operator("some_nan_correct_bounds", 1)).eval(data_file_nan) + assert not should_read, "Should not match: 1 is smaller than lower bound" + + should_read = _InclusiveMetricsEvaluator(schema_data_file_nan, operator("some_nan_correct_bounds", 10)).eval( + data_file_nan + ) + assert should_read, "Should match: 10 larger than lower bound" + + +def test_inclusive_metrics_evaluator_greater_than_and_greater_than_equal( + schema_data_file_nan: Schema, data_file_nan: DataFile +) -> None: + for operator in [GreaterThan, GreaterThanOrEqual]: + should_read = _InclusiveMetricsEvaluator(schema_data_file_nan, operator("all_nan", 1)).eval(data_file_nan) + assert not should_read, "Should not match: all nan column doesn't contain number" + + should_read = _InclusiveMetricsEvaluator(schema_data_file_nan, operator("max_nan", 1)).eval(data_file_nan) + assert should_read, "Should match: upper bound is larger than 1" + + should_read = _InclusiveMetricsEvaluator(schema_data_file_nan, operator("max_nan", 10)).eval(data_file_nan) + assert should_read, "Should match: upper bound is larger than 10" + + should_read = _InclusiveMetricsEvaluator(schema_data_file_nan, operator("min_max_nan", 1)).eval(data_file_nan) + assert should_read, "Should match: no visibility" + + should_read = _InclusiveMetricsEvaluator(schema_data_file_nan, operator("all_nan_null_bounds", 1)).eval(data_file_nan) + assert not should_read, "Should not match: all nan column doesn't contain number" + + should_read = _InclusiveMetricsEvaluator(schema_data_file_nan, operator("some_nan_correct_bounds", 1)).eval(data_file_nan) + assert should_read, "Should match: 1 is smaller than upper bound" -def _record_simple(id: int, data: Optional[str]) -> Record: # pylint: disable=redefined-builtin - r = Record(struct=SIMPLE_SCHEMA.as_struct()) - r[0] = id - r[1] = data - return r + should_read = _InclusiveMetricsEvaluator(schema_data_file_nan, operator("some_nan_correct_bounds", 10)).eval( + data_file_nan + ) + assert should_read, "Should match: 10 is smaller than upper bound" + should_read = _InclusiveMetricsEvaluator(schema_data_file_nan, operator("all_nan", 30)).eval(data_file_nan) + assert not should_read, "Should not match: 30 is greater than upper bound" -def _record_float(id: float, f: float) -> Record: # pylint: disable=redefined-builtin - r = Record(struct=FLOAT_SCHEMA.as_struct()) - r[0] = id - r[1] = f - return r +def test_inclusive_metrics_evaluator_equals(schema_data_file_nan: Schema, data_file_nan: DataFile) -> None: + should_read = _InclusiveMetricsEvaluator(schema_data_file_nan, EqualTo("all_nan", 1)).eval(data_file_nan) + assert not should_read, "Should not match: all nan column doesn't contain number" -def test_true() -> None: - evaluate = expression_evaluator(SIMPLE_SCHEMA, AlwaysTrue(), case_sensitive=True) - assert evaluate(Record(1, "a")) + should_read = _InclusiveMetricsEvaluator(schema_data_file_nan, EqualTo("max_nan", 1)).eval(data_file_nan) + assert not should_read, "Should not match: 1 is smaller than lower bound" + should_read = _InclusiveMetricsEvaluator(schema_data_file_nan, EqualTo("max_nan", 10)).eval(data_file_nan) + assert should_read, "Should match: 10 is within bounds" -def test_false() -> None: - evaluate = expression_evaluator(SIMPLE_SCHEMA, AlwaysFalse(), case_sensitive=True) - assert not evaluate(Record(1, "a")) + should_read = _InclusiveMetricsEvaluator(schema_data_file_nan, EqualTo("min_max_nan", 1)).eval(data_file_nan) + assert should_read, "Should match: no visibility" + should_read = _InclusiveMetricsEvaluator(schema_data_file_nan, EqualTo("all_nan_null_bounds", 1)).eval(data_file_nan) + assert not should_read, "Should not match: all nan column doesn't contain number" -def test_less_than() -> None: - evaluate = expression_evaluator(SIMPLE_SCHEMA, LessThan("id", 3), case_sensitive=True) - assert evaluate(Record(2, "a")) - assert not evaluate(Record(3, "a")) + should_read = _InclusiveMetricsEvaluator(schema_data_file_nan, EqualTo("some_nan_correct_bounds", 1)).eval(data_file_nan) + assert not should_read, "Should not match: 1 is smaller than lower bound" + should_read = _InclusiveMetricsEvaluator(schema_data_file_nan, EqualTo("some_nan_correct_bounds", 10)).eval(data_file_nan) + assert should_read, "Should match: 10 is within bounds" -def test_less_than_or_equal() -> None: - evaluate = expression_evaluator(SIMPLE_SCHEMA, LessThanOrEqual("id", 3), case_sensitive=True) - assert evaluate(Record(1, "a")) - assert evaluate(Record(3, "a")) - assert not evaluate(Record(4, "a")) + should_read = _InclusiveMetricsEvaluator(schema_data_file_nan, EqualTo("all_nan", 30)).eval(data_file_nan) + assert not should_read, "Should not match: 30 is greater than upper bound" -def test_greater_than() -> None: - evaluate = expression_evaluator(SIMPLE_SCHEMA, GreaterThan("id", 3), case_sensitive=True) - assert not evaluate(Record(1, "a")) - assert not evaluate(Record(3, "a")) - assert evaluate(Record(4, "a")) +def test_inclusive_metrics_evaluator_not_equals(schema_data_file_nan: Schema, data_file_nan: DataFile) -> None: + should_read = _InclusiveMetricsEvaluator(schema_data_file_nan, NotEqualTo("all_nan", 1)).eval(data_file_nan) + assert should_read, "Should match: no visibility" + should_read = _InclusiveMetricsEvaluator(schema_data_file_nan, NotEqualTo("max_nan", 10)).eval(data_file_nan) + assert should_read, "Should match: no visibility" -def test_greater_than_or_equal() -> None: - evaluate = expression_evaluator(SIMPLE_SCHEMA, GreaterThanOrEqual("id", 3), case_sensitive=True) - assert not evaluate(Record(2, "a")) - assert evaluate(Record(3, "a")) - assert evaluate(Record(4, "a")) + should_read = _InclusiveMetricsEvaluator(schema_data_file_nan, NotEqualTo("max_nan", 10)).eval(data_file_nan) + assert should_read, "Should match: no visibility" + should_read = _InclusiveMetricsEvaluator(schema_data_file_nan, NotEqualTo("min_max_nan", 1)).eval(data_file_nan) + assert should_read, "Should match: no visibility" -def test_equal_to() -> None: - evaluate = expression_evaluator(SIMPLE_SCHEMA, EqualTo("id", 3), case_sensitive=True) - assert not evaluate(Record(2, "a")) - assert evaluate(Record(3, "a")) - assert not evaluate(Record(4, "a")) + should_read = _InclusiveMetricsEvaluator(schema_data_file_nan, NotEqualTo("all_nan_null_bounds", 1)).eval(data_file_nan) + assert should_read, "Should match: no visibility" + should_read = _InclusiveMetricsEvaluator(schema_data_file_nan, NotEqualTo("some_nan_correct_bounds", 1)).eval(data_file_nan) + assert should_read, "Should match: no visibility" -def test_not_equal_to() -> None: - evaluate = expression_evaluator(SIMPLE_SCHEMA, NotEqualTo("id", 3), case_sensitive=True) - assert evaluate(Record(2, "a")) - assert not evaluate(Record(3, "a")) - assert evaluate(Record(4, "a")) + should_read = _InclusiveMetricsEvaluator(schema_data_file_nan, NotEqualTo("some_nan_correct_bounds", 10)).eval(data_file_nan) + assert should_read, "Should match: no visibility" + should_read = _InclusiveMetricsEvaluator(schema_data_file_nan, NotEqualTo("some_nan_correct_bounds", 30)).eval(data_file_nan) + assert should_read, "Should match: no visibility" -def test_in() -> None: - evaluate = expression_evaluator(SIMPLE_SCHEMA, In("id", [1, 2, 3]), case_sensitive=True) - assert evaluate(Record(2, "a")) - assert evaluate(Record(3, "a")) - assert not evaluate(Record(4, "a")) +def test_inclusive_metrics_evaluator_in(schema_data_file_nan: Schema, data_file_nan: DataFile) -> None: + should_read = _InclusiveMetricsEvaluator(schema_data_file_nan, In("all_nan", (1, 10, 30))).eval(data_file_nan) + assert not should_read, "Should not match: all nan column doesn't contain number" -def test_not_in() -> None: - evaluate = expression_evaluator(SIMPLE_SCHEMA, NotIn("id", [1, 2, 3]), case_sensitive=True) - assert not evaluate(Record(2, "a")) - assert not evaluate(Record(3, "a")) - assert evaluate(Record(4, "a")) + should_read = _InclusiveMetricsEvaluator(schema_data_file_nan, In("max_nan", (1, 10, 30))).eval(data_file_nan) + assert should_read, "Should match: 10 and 30 are greater than lower bound" + should_read = _InclusiveMetricsEvaluator(schema_data_file_nan, In("min_max_nan", (1, 10, 30))).eval(data_file_nan) + assert should_read, "Should match: no visibility" -def test_is_null() -> None: - evaluate = expression_evaluator(SIMPLE_SCHEMA, IsNull("data"), case_sensitive=True) - assert not evaluate(Record(2, "a")) - assert evaluate(Record(3, None)) + should_read = _InclusiveMetricsEvaluator(schema_data_file_nan, In("all_nan_null_bounds", (1, 10, 30))).eval(data_file_nan) + assert not should_read, "Should not match: all nan column doesn't contain number" + should_read = _InclusiveMetricsEvaluator(schema_data_file_nan, In("some_nan_correct_bounds", (1, 10, 30))).eval(data_file_nan) + assert should_read, "Should match: 10 within bounds" -def test_not_null() -> None: - evaluate = expression_evaluator(SIMPLE_SCHEMA, NotNull("data"), case_sensitive=True) - assert evaluate(Record(2, "a")) - assert not evaluate(Record(3, None)) + should_read = _InclusiveMetricsEvaluator(schema_data_file_nan, In("some_nan_correct_bounds", (1, 30))).eval(data_file_nan) + assert not should_read, "Should not match: 1 and 30 not within bounds" + should_read = _InclusiveMetricsEvaluator(schema_data_file_nan, In("some_nan_correct_bounds", (5, 7))).eval(data_file_nan) + assert should_read, "Should match: overlap with lower bound" -def test_is_nan() -> None: - evaluate = expression_evaluator(FLOAT_SCHEMA, IsNaN("f"), case_sensitive=True) - assert not evaluate(_record_float(2, f=0.0)) - assert not evaluate(_record_float(3, f=float("infinity"))) - assert evaluate(_record_float(4, f=float("nan"))) + should_read = _InclusiveMetricsEvaluator(schema_data_file_nan, In("some_nan_correct_bounds", (22, 25))).eval(data_file_nan) + assert should_read, "Should match: overlap with upper bounds" -def test_not_nan() -> None: - evaluate = expression_evaluator(FLOAT_SCHEMA, NotNaN("f"), case_sensitive=True) - assert evaluate(_record_float(2, f=0.0)) - assert evaluate(_record_float(3, f=float("infinity"))) - assert not evaluate(_record_float(4, f=float("nan"))) +def test_inclusive_metrics_evaluator_not_in(schema_data_file_nan: Schema, data_file_nan: DataFile) -> None: + should_read = _InclusiveMetricsEvaluator(schema_data_file_nan, NotIn("all_nan", (1, 10, 30))).eval(data_file_nan) + assert should_read, "Should match: no visibility" + should_read = _InclusiveMetricsEvaluator(schema_data_file_nan, NotIn("max_nan", (1, 10, 30))).eval(data_file_nan) + assert should_read, "Should match: no visibility" -def test_not() -> None: - evaluate = expression_evaluator(SIMPLE_SCHEMA, Not(LessThan("id", 3)), case_sensitive=True) - assert not evaluate(Record(2, "a")) - assert evaluate(Record(3, "a")) + should_read = _InclusiveMetricsEvaluator(schema_data_file_nan, NotIn("max_nan", (1, 10, 30))).eval(data_file_nan) + assert should_read, "Should match: no visibility" + should_read = _InclusiveMetricsEvaluator(schema_data_file_nan, NotIn("min_max_nan", (1, 10, 30))).eval(data_file_nan) + assert should_read, "Should match: no visibility" -def test_and() -> None: - evaluate = expression_evaluator(SIMPLE_SCHEMA, And(LessThan("id", 3), GreaterThan("id", 1)), case_sensitive=True) - assert not evaluate(Record(1, "a")) - assert evaluate(Record(2, "a")) - assert not evaluate(Record(3, "a")) + should_read = _InclusiveMetricsEvaluator(schema_data_file_nan, NotIn("all_nan_null_bounds", (1, 10, 30))).eval(data_file_nan) + assert should_read, "Should match: no visibility" + should_read = _InclusiveMetricsEvaluator(schema_data_file_nan, NotIn("some_nan_correct_bounds", (1, 30))).eval(data_file_nan) + assert should_read, "Should match: no visibility" -def test_or() -> None: - evaluate = expression_evaluator(SIMPLE_SCHEMA, Or(LessThan("id", 2), GreaterThan("id", 2)), case_sensitive=True) - assert evaluate(Record(1, "a")) - assert not evaluate(Record(2, "a")) - assert evaluate(Record(3, "a")) + should_read = _InclusiveMetricsEvaluator(schema_data_file_nan, NotIn("some_nan_correct_bounds", (1, 30))).eval(data_file_nan) + assert should_read, "Should match: no visibility" From e95ecd5cb6be3ff640906892c51775e67576f572 Mon Sep 17 00:00:00 2001 From: Fokko Driesprong Date: Tue, 28 Feb 2023 18:17:10 +0100 Subject: [PATCH 387/642] Python: Fix timezone concat issue (#6946) Resolves #6945 --- pyiceberg/io/pyarrow.py | 27 ++++++++++++++++----------- tests/io/test_pyarrow.py | 2 +- 2 files changed, 17 insertions(+), 12 deletions(-) diff --git a/pyiceberg/io/pyarrow.py b/pyiceberg/io/pyarrow.py index d2cd762273..46e384bc17 100644 --- a/pyiceberg/io/pyarrow.py +++ b/pyiceberg/io/pyarrow.py @@ -391,7 +391,7 @@ def visit_timestamp(self, _: TimestampType) -> pa.DataType: return pa.timestamp(unit="us") def visit_timestampz(self, _: TimestamptzType) -> pa.DataType: - return pa.timestamp(unit="us", tz="+00:00") + return pa.timestamp(unit="us", tz="UTC") def visit_string(self, _: StringType) -> pa.DataType: return pa.string() @@ -477,7 +477,7 @@ def _file_to_table( projected_schema: Schema, projected_field_ids: Set[int], case_sensitive: bool, -) -> pa.Table: +) -> Optional[pa.Table]: _, path = PyArrowFileIO.parse_location(task.file.file_path) # Get the schema @@ -512,10 +512,11 @@ def _file_to_table( columns=[col.name for col in file_project_schema.columns], ) - if pyarrow_filter is not None: - arrow_table = arrow_table.filter(pyarrow_filter) - - return to_requested_schema(projected_schema, file_project_schema, arrow_table) + # If there is no data, we don't have to go through the schema + if len(arrow_table) > 0: + return to_requested_schema(projected_schema, file_project_schema, arrow_table) + else: + return None def project_table( @@ -547,11 +548,15 @@ def project_table( }.union(extract_field_ids(bound_row_filter)) with ThreadPool() as pool: - tables = pool.starmap( - func=_file_to_table, - iterable=[(fs, task, bound_row_filter, projected_schema, projected_field_ids, case_sensitive) for task in tasks], - chunksize=None, # we could use this to control how to materialize the generator of tasks (we should also make the expression above lazy) - ) + tables = [ + table + for table in pool.starmap( + func=_file_to_table, + iterable=[(fs, task, bound_row_filter, projected_schema, projected_field_ids, case_sensitive) for task in tasks], + chunksize=None, # we could use this to control how to materialize the generator of tasks (we should also make the expression above lazy) + ) + if table is not None + ] if len(tables) > 1: return pa.concat_tables(tables) diff --git a/tests/io/test_pyarrow.py b/tests/io/test_pyarrow.py index f894963d94..c07890053d 100644 --- a/tests/io/test_pyarrow.py +++ b/tests/io/test_pyarrow.py @@ -377,7 +377,7 @@ def test_timestamp_type_to_pyarrow() -> None: def test_timestamptz_type_to_pyarrow() -> None: iceberg_type = TimestamptzType() - assert visit(iceberg_type, _ConvertToArrowSchema()) == pa.timestamp(unit="us", tz="+00:00") + assert visit(iceberg_type, _ConvertToArrowSchema()) == pa.timestamp(unit="us", tz="UTC") def test_string_type_to_pyarrow() -> None: From 2600937445507672cfbfc821ed7a0de866867a82 Mon Sep 17 00:00:00 2001 From: Luigi Cerone Date: Thu, 2 Mar 2023 13:32:52 +0100 Subject: [PATCH 388/642] Python: `starts_with` and `not_starts_with` expressions (#6892) * Add new expressions * Add test for new expressions * Fix tests and test visitors * Add py arrow tests * Fix visitors, fix mypy * Fix problem on ManifestEvalVisitor * Update python/pyiceberg/expressions/visitors.py Co-authored-by: Fokko Driesprong * Update python/tests/expressions/test_parser.py Co-authored-by: Fokko Driesprong * Fix test_parser tests and add more test_evaluator tests for new python operators * Fix ManifestEvalVisitor logic, add more tests * Fixes and minors from review * Add MetricsEvaluator starts_with logic and fix tests * Apply suggestions from code review Co-authored-by: Fokko Driesprong * Apply suggestions from code review Co-authored-by: Fokko Driesprong * Fix mypy error --------- Co-authored-by: Luigi Cerone Co-authored-by: Fokko Driesprong --- pyiceberg/expressions/__init__.py | 36 +++++++ pyiceberg/expressions/parser.py | 20 +++- pyiceberg/expressions/visitors.py | 151 ++++++++++++++++++++++++++++ pyiceberg/io/pyarrow.py | 6 ++ pyiceberg/transforms.py | 11 +- tests/expressions/test_evaluator.py | 149 ++++++++++++++++++++++++++- tests/expressions/test_parser.py | 10 ++ tests/expressions/test_visitors.py | 122 ++++++++++++++++++++++ tests/io/test_pyarrow.py | 16 +++ tests/test_transforms.py | 16 +++ 10 files changed, 530 insertions(+), 7 deletions(-) diff --git a/pyiceberg/expressions/__init__.py b/pyiceberg/expressions/__init__.py index 14ded66473..6780c6dca4 100644 --- a/pyiceberg/expressions/__init__.py +++ b/pyiceberg/expressions/__init__.py @@ -720,6 +720,24 @@ def as_unbound(self) -> Type[LessThanOrEqual[L]]: return LessThanOrEqual[L] +class BoundStartsWith(BoundLiteralPredicate[L]): + def __invert__(self) -> BoundNotStartsWith[L]: + return BoundNotStartsWith[L](self.term, self.literal) + + @property + def as_unbound(self) -> Type[StartsWith[L]]: + return StartsWith[L] + + +class BoundNotStartsWith(BoundLiteralPredicate[L]): + def __invert__(self) -> BoundStartsWith[L]: + return BoundStartsWith[L](self.term, self.literal) + + @property + def as_unbound(self) -> Type[NotStartsWith[L]]: + return NotStartsWith[L] + + class EqualTo(LiteralPredicate[L]): def __invert__(self) -> NotEqualTo[L]: return NotEqualTo[L](self.term, self.literal) @@ -772,3 +790,21 @@ def __invert__(self) -> GreaterThan[L]: @property def as_bound(self) -> Type[BoundLessThanOrEqual[L]]: return BoundLessThanOrEqual[L] + + +class StartsWith(LiteralPredicate[L]): + def __invert__(self) -> NotStartsWith[L]: + return NotStartsWith[L](self.term, self.literal) + + @property + def as_bound(self) -> Type[BoundStartsWith[L]]: + return BoundStartsWith[L] + + +class NotStartsWith(LiteralPredicate[L]): + def __invert__(self) -> NotStartsWith[L]: + return NotStartsWith[L](self.term, self.literal) + + @property + def as_bound(self) -> Type[BoundNotStartsWith[L]]: + return BoundNotStartsWith[L] diff --git a/pyiceberg/expressions/parser.py b/pyiceberg/expressions/parser.py index 0a0173da41..02dd64d1a1 100644 --- a/pyiceberg/expressions/parser.py +++ b/pyiceberg/expressions/parser.py @@ -51,8 +51,10 @@ NotIn, NotNaN, NotNull, + NotStartsWith, Or, Reference, + StartsWith, ) from pyiceberg.expressions.literals import ( DecimalLiteral, @@ -71,6 +73,7 @@ IN = CaselessKeyword("in") NULL = CaselessKeyword("null") NAN = CaselessKeyword("nan") +LIKE = CaselessKeyword("like") identifier = Word(alphas, alphanums + "_$").set_results_name("identifier") column = delimited_list(identifier, delim=".", combine=True).set_results_name("column") @@ -207,7 +210,22 @@ def _(result: ParseResults) -> BooleanExpression: return NotIn(result.column, result.literal_set) -predicate = (comparison | in_check | null_check | nan_check | boolean).set_results_name("predicate") +starts_with = column + LIKE + string +not_starts_with = column + NOT + LIKE + string +starts_check = starts_with | not_starts_with + + +@starts_with.set_parse_action +def _(result: ParseResults) -> BooleanExpression: + return StartsWith(result.column, result.raw_quoted_string) + + +@not_starts_with.set_parse_action +def _(result: ParseResults) -> BooleanExpression: + return NotStartsWith(result.column, result.raw_quoted_string) + + +predicate = (comparison | in_check | null_check | nan_check | starts_check | boolean).set_results_name("predicate") def handle_not(result: ParseResults) -> Not: diff --git a/pyiceberg/expressions/visitors.py b/pyiceberg/expressions/visitors.py index 4fc666bcd9..901c451198 100644 --- a/pyiceberg/expressions/visitors.py +++ b/pyiceberg/expressions/visitors.py @@ -48,8 +48,10 @@ BoundNotIn, BoundNotNaN, BoundNotNull, + BoundNotStartsWith, BoundPredicate, BoundSetPredicate, + BoundStartsWith, BoundTerm, BoundUnaryPredicate, L, @@ -320,6 +322,14 @@ def visit_and(self, left_result: T, right_result: T) -> T: def visit_or(self, left_result: T, right_result: T) -> T: """Visit a bound Or predicate""" + @abstractmethod + def visit_starts_with(self, term: BoundTerm[L], literal: Literal[L]) -> T: + """Visit bound StartsWith predicate""" + + @abstractmethod + def visit_not_starts_with(self, term: BoundTerm[L], literal: Literal[L]) -> T: + """Visit bound NotStartsWith predicate""" + def visit_unbound_predicate(self, predicate: UnboundPredicate[L]) -> T: """Visit an unbound predicate Args: @@ -403,6 +413,16 @@ def _(expr: BoundLessThanOrEqual[L], visitor: BoundBooleanExpressionVisitor[T]) return visitor.visit_less_than_or_equal(term=expr.term, literal=expr.literal) +@visit_bound_predicate.register(BoundStartsWith) +def _(expr: BoundStartsWith[L], visitor: BoundBooleanExpressionVisitor[T]) -> T: + return visitor.visit_starts_with(term=expr.term, literal=expr.literal) + + +@visit_bound_predicate.register(BoundNotStartsWith) +def _(expr: BoundNotStartsWith[L], visitor: BoundBooleanExpressionVisitor[T]) -> T: + return visitor.visit_not_starts_with(term=expr.term, literal=expr.literal) + + def rewrite_not(expr: BooleanExpression) -> BooleanExpression: return visit(expr, _RewriteNotVisitor()) @@ -485,6 +505,13 @@ def visit_less_than(self, term: BoundTerm[L], literal: Literal[L]) -> bool: def visit_less_than_or_equal(self, term: BoundTerm[L], literal: Literal[L]) -> bool: return term.eval(self.struct) <= literal.value + def visit_starts_with(self, term: BoundTerm[L], literal: Literal[L]) -> bool: + eval_res = term.eval(self.struct) + return eval_res is not None and str(eval_res).startswith(str(literal.value)) + + def visit_not_starts_with(self, term: BoundTerm[L], literal: Literal[L]) -> bool: + return not self.visit_starts_with(term, literal) + def visit_true(self) -> bool: return True @@ -678,6 +705,59 @@ def visit_less_than_or_equal(self, term: BoundTerm[L], literal: Literal[L]) -> b return ROWS_MIGHT_MATCH + def visit_starts_with(self, term: BoundTerm[L], literal: Literal[L]) -> bool: + pos = term.ref().accessor.position + field = self.partition_fields[pos] + prefix = str(literal.value) + len_prefix = len(prefix) + + if field.lower_bound is None: + return ROWS_CANNOT_MATCH + + lower = _from_byte_buffer(term.ref().field.field_type, field.lower_bound) + # truncate lower bound so that its length is not greater than the length of prefix + if lower is not None and lower[:len_prefix] > prefix: + return ROWS_CANNOT_MATCH + + if field.upper_bound is None: + return ROWS_CANNOT_MATCH + + upper = _from_byte_buffer(term.ref().field.field_type, field.upper_bound) + # truncate upper bound so that its length is not greater than the length of prefix + if upper is not None and upper[:len_prefix] < prefix: + return ROWS_CANNOT_MATCH + + return ROWS_MIGHT_MATCH + + def visit_not_starts_with(self, term: BoundTerm[L], literal: Literal[L]) -> bool: + pos = term.ref().accessor.position + field = self.partition_fields[pos] + prefix = str(literal.value) + len_prefix = len(prefix) + + if field.contains_null or field.lower_bound is None or field.upper_bound is None: + return ROWS_MIGHT_MATCH + + # not_starts_with will match unless all values must start with the prefix. This happens when + # the lower and upper bounds both start with the prefix. + lower = _from_byte_buffer(term.ref().field.field_type, field.lower_bound) + upper = _from_byte_buffer(term.ref().field.field_type, field.upper_bound) + + if lower is not None and upper is not None: + # if lower is shorter than the prefix then lower doesn't start with the prefix + if len(lower) < len_prefix: + return ROWS_MIGHT_MATCH + + if lower[:len_prefix] == prefix: + # if upper is shorter than the prefix then upper can't start with the prefix + if len(upper) < len_prefix: + return ROWS_MIGHT_MATCH + + if upper[:len_prefix] == prefix: + return ROWS_CANNOT_MATCH + + return ROWS_MIGHT_MATCH + def visit_true(self) -> bool: return ROWS_MIGHT_MATCH @@ -946,6 +1026,12 @@ def visit_less_than(self, term: BoundTerm[L], literal: Literal[L]) -> List[Tuple def visit_less_than_or_equal(self, term: BoundTerm[L], literal: Literal[L]) -> List[Tuple[str, str, Any]]: return [(term.ref().field.name, "<=", self._cast_if_necessary(term.ref().field.field_type, literal.value))] + def visit_starts_with(self, term: BoundTerm[L], literal: Literal[L]) -> List[Tuple[str, str, Any]]: + return [] + + def visit_not_starts_with(self, term: BoundTerm[L], literal: Literal[L]) -> List[Tuple[str, str, Any]]: + return [] + def visit_true(self) -> List[Tuple[str, str, Any]]: return [] # Not supported @@ -1025,6 +1111,9 @@ def eval(self, file: DataFile) -> bool: return visit(self.expr, self) + def _may_contain_null(self, field_id: int) -> bool: + return self.null_counts is None or (field_id in self.null_counts and self.null_counts.get(field_id) is not None) + def _contains_nulls_only(self, field_id: int) -> bool: if (value_count := self.value_counts.get(field_id)) and (null_count := self.null_counts.get(field_id)): return value_count == null_count @@ -1256,3 +1345,65 @@ def visit_not_in(self, term: BoundTerm[L], literals: Set[L]) -> bool: # because the bounds are not necessarily a min or max value, this cannot be answered using # them. notIn(col, {X, ...}) with (X, Y) doesn't guarantee that X is a value in col. return ROWS_MIGHT_MATCH + + def visit_starts_with(self, term: BoundTerm[L], literal: Literal[L]) -> bool: + field = term.ref().field + field_id: int = field.field_id + + if self._contains_nulls_only(field_id): + return ROWS_CANNOT_MATCH + + if not isinstance(field.field_type, PrimitiveType): + raise ValueError(f"Expected PrimitiveType: {field.field_type}") + + prefix = str(literal.value) + len_prefix = len(prefix) + + if lower_bound_bytes := self.lower_bounds.get(field_id): + lower_bound = str(from_bytes(field.field_type, lower_bound_bytes)) + + # truncate lower bound so that its length is not greater than the length of prefix + if lower_bound and lower_bound[:len_prefix] > prefix: + return ROWS_CANNOT_MATCH + + if upper_bound_bytes := self.upper_bounds.get(field_id): + upper_bound = str(from_bytes(field.field_type, upper_bound_bytes)) # type: ignore + + # truncate upper bound so that its length is not greater than the length of prefix + if upper_bound is not None and upper_bound[:len_prefix] < prefix: + return ROWS_CANNOT_MATCH + + return ROWS_MIGHT_MATCH + + def visit_not_starts_with(self, term: BoundTerm[L], literal: Literal[L]) -> bool: + field = term.ref().field + field_id: int = field.field_id + + if self._may_contain_null(field_id): + return ROWS_MIGHT_MATCH + + if not isinstance(field.field_type, PrimitiveType): + raise ValueError(f"Expected PrimitiveType: {field.field_type}") + + prefix = str(literal.value) + len_prefix = len(prefix) + + # not_starts_with will match unless all values must start with the prefix. This happens when + # the lower and upper bounds both start with the prefix. + if (lower_bound_bytes := self.lower_bounds.get(field_id)) and (upper_bound_bytes := self.upper_bounds.get(field_id)): + lower_bound = str(from_bytes(field.field_type, lower_bound_bytes)) + upper_bound = str(from_bytes(field.field_type, upper_bound_bytes)) + + # if lower is shorter than the prefix then lower doesn't start with the prefix + if len(lower_bound) < len_prefix: + return ROWS_MIGHT_MATCH + + if lower_bound[:len_prefix] == prefix: + # if upper is shorter than the prefix then upper can't start with the prefix + if len(upper_bound) < len_prefix: + return ROWS_MIGHT_MATCH + + if upper_bound[:len_prefix] == prefix: + return ROWS_CANNOT_MATCH + + return ROWS_MIGHT_MATCH diff --git a/pyiceberg/io/pyarrow.py b/pyiceberg/io/pyarrow.py index 46e384bc17..ce9ce70344 100644 --- a/pyiceberg/io/pyarrow.py +++ b/pyiceberg/io/pyarrow.py @@ -450,6 +450,12 @@ def visit_less_than(self, term: BoundTerm[Any], literal: Literal[Any]) -> pc.Exp def visit_less_than_or_equal(self, term: BoundTerm[Any], literal: Literal[Any]) -> pc.Expression: return pc.field(term.ref().field.name) <= _convert_scalar(literal.value, term.ref().field.field_type) + def visit_starts_with(self, term: BoundTerm[Any], literal: Literal[Any]) -> pc.Expression: + return pc.starts_with(pc.field(term.ref().field.name), literal.value) + + def visit_not_starts_with(self, term: BoundTerm[Any], literal: Literal[Any]) -> pc.Expression: + return ~pc.starts_with(pc.field(term.ref().field.name), literal.value) + def visit_true(self) -> pc.Expression: return pc.scalar(True) diff --git a/pyiceberg/transforms.py b/pyiceberg/transforms.py index e0197f0639..b362cfbf88 100644 --- a/pyiceberg/transforms.py +++ b/pyiceberg/transforms.py @@ -42,14 +42,18 @@ BoundLessThanOrEqual, BoundLiteralPredicate, BoundNotIn, + BoundNotStartsWith, BoundPredicate, BoundSetPredicate, + BoundStartsWith, BoundTerm, BoundUnaryPredicate, EqualTo, GreaterThanOrEqual, LessThanOrEqual, + NotStartsWith, Reference, + StartsWith, UnboundPredicate, ) from pyiceberg.expressions.literals import ( @@ -600,9 +604,6 @@ def project(self, name: str, pred: BoundPredicate[L]) -> Optional[UnboundPredica if isinstance(pred.term, BoundTransform): return _project_transform_predicate(self, name, pred) - # Implement startswith and notstartswith for string (and probably binary) - # https://github.com/apache/iceberg/issues/6112 - if isinstance(pred, BoundUnaryPredicate): return pred.as_unbound(Reference(name)) elif isinstance(pred, BoundIn): @@ -794,6 +795,10 @@ def _truncate_array( return GreaterThanOrEqual(Reference(name), _transform_literal(transform, boundary)) if isinstance(pred, BoundEqualTo): return EqualTo(Reference(name), _transform_literal(transform, boundary)) + elif isinstance(pred, BoundStartsWith): + return StartsWith(Reference(name), _transform_literal(transform, boundary)) + elif isinstance(pred, BoundNotStartsWith): + return NotStartsWith(Reference(name), _transform_literal(transform, boundary)) else: return None diff --git a/tests/expressions/test_evaluator.py b/tests/expressions/test_evaluator.py index 7de9c04fb7..7d97a6d2d2 100644 --- a/tests/expressions/test_evaluator.py +++ b/tests/expressions/test_evaluator.py @@ -35,7 +35,9 @@ NotIn, NotNaN, NotNull, + NotStartsWith, Or, + StartsWith, ) from pyiceberg.expressions.visitors import _InclusiveMetricsEvaluator from pyiceberg.manifest import DataFile, FileFormat @@ -96,9 +98,6 @@ def data_file() -> DataFile: record_count=50, file_size_in_bytes=3, value_counts={ - 1: 50, - 2: 50, - 3: 50, 4: 50, 5: 50, 6: 50, @@ -132,6 +131,54 @@ def data_file() -> DataFile: ) +@pytest.fixture +def data_file_2() -> DataFile: + return DataFile( + file_path="file_2.parquet", + file_format=FileFormat.PARQUET, + partition={}, + record_count=50, + file_size_in_bytes=3, + value_counts={3: 20}, + null_value_counts={3: 2}, + nan_value_counts=None, + lower_bounds={3: to_bytes(StringType(), "aa")}, + upper_bounds={3: to_bytes(StringType(), "dC")}, + ) + + +@pytest.fixture +def data_file_3() -> DataFile: + return DataFile( + file_path="file_3.parquet", + file_format=FileFormat.PARQUET, + partition={}, + record_count=50, + file_size_in_bytes=3, + value_counts={3: 20}, + null_value_counts={3: 2}, + nan_value_counts=None, + lower_bounds={3: to_bytes(StringType(), "1str1")}, + upper_bounds={3: to_bytes(StringType(), "3str3")}, + ) + + +@pytest.fixture +def data_file_4() -> DataFile: + return DataFile( + file_path="file_4.parquet", + file_format=FileFormat.PARQUET, + partition={}, + record_count=50, + file_size_in_bytes=3, + value_counts={3: 20}, + null_value_counts={3: 2}, + nan_value_counts=None, + lower_bounds={3: to_bytes(StringType(), "abc")}, + upper_bounds={3: to_bytes(StringType(), "イロハニホヘト")}, + ) + + def test_all_null(schema_data_file: Schema, data_file: DataFile) -> None: should_read = _InclusiveMetricsEvaluator(schema_data_file, NotNull("all_nulls")).eval(data_file) assert not should_read, "Should skip: no non-null value in all null column" @@ -157,6 +204,12 @@ def test_all_null(schema_data_file: Schema, data_file: DataFile) -> None: should_read = _InclusiveMetricsEvaluator(schema_data_file, NotNull("no_nulls")).eval(data_file) assert should_read, "Should read: non-null column contains a non-null value" + should_read = _InclusiveMetricsEvaluator(schema_data_file, StartsWith("all_nulls", "asad")).eval(data_file) + assert not should_read, "Should skip: startsWith on all null column" + + should_read = _InclusiveMetricsEvaluator(schema_data_file, NotStartsWith("all_nulls", "asad")).eval(data_file) + assert should_read, "Should read: notStartsWith on all null column" + def test_no_nulls(schema_data_file: Schema, data_file: DataFile) -> None: should_read = _InclusiveMetricsEvaluator(schema_data_file, IsNull("all_nulls")).eval(data_file) @@ -782,3 +835,93 @@ def test_inclusive_metrics_evaluator_not_in(schema_data_file_nan: Schema, data_f should_read = _InclusiveMetricsEvaluator(schema_data_file_nan, NotIn("some_nan_correct_bounds", (1, 30))).eval(data_file_nan) assert should_read, "Should match: no visibility" + + +def test_string_starts_with( + schema_data_file: Schema, data_file: DataFile, data_file_2: DataFile, data_file_3: DataFile, data_file_4: DataFile +) -> None: + should_read = _InclusiveMetricsEvaluator(schema_data_file, StartsWith("required", "a")).eval(data_file) + assert should_read, "Should read: no stats" + + should_read = _InclusiveMetricsEvaluator(schema_data_file, StartsWith("required", "a")).eval(data_file_2) + assert should_read, "Should read: range matches" + + should_read = _InclusiveMetricsEvaluator(schema_data_file, StartsWith("required", "aa")).eval(data_file_2) + assert should_read, "Should read: range matches" + + should_read = _InclusiveMetricsEvaluator(schema_data_file, StartsWith("required", "aaa")).eval(data_file_2) + assert should_read, "Should read: range matches" + + should_read = _InclusiveMetricsEvaluator(schema_data_file, StartsWith("required", "1s")).eval(data_file_3) + assert should_read, "Should read: range matches" + + should_read = _InclusiveMetricsEvaluator(schema_data_file, StartsWith("required", "1str1x")).eval(data_file_3) + assert should_read, "Should read: range matches" + + should_read = _InclusiveMetricsEvaluator(schema_data_file, StartsWith("required", "ff")).eval(data_file_4) + assert should_read, "Should read: range matches" + + should_read = _InclusiveMetricsEvaluator(schema_data_file, StartsWith("required", "aB")).eval(data_file_2) + assert not should_read, "Should not read: range doesn't match" + + should_read = _InclusiveMetricsEvaluator(schema_data_file, StartsWith("required", "dWX")).eval(data_file_2) + assert not should_read, "Should not read: range doesn't match" + + should_read = _InclusiveMetricsEvaluator(schema_data_file, StartsWith("required", "5")).eval(data_file_3) + assert not should_read, "Should not read: range doesn't match" + + should_read = _InclusiveMetricsEvaluator(schema_data_file, StartsWith("required", "3str3x")).eval(data_file_3) + assert not should_read, "Should not read: range doesn't match" + + should_read = _InclusiveMetricsEvaluator(schema_data_file, StartsWith("some_empty", "房东整租霍")).eval(data_file) + assert should_read, "Should read: range matches" + + should_read = _InclusiveMetricsEvaluator(schema_data_file, StartsWith("all_nulls", "")).eval(data_file) + assert not should_read, "Should not read: range doesn't match" + + # above_max = UnicodeUtil.truncateStringMax(Literal.of("イロハニホヘト"), 4).value().toString(); + + # should_read = _InclusiveMetricsEvaluator(schema_data_file, StartsWith("required", above_max)).eval(data_file_4) + # assert not should_read, "Should not read: range doesn't match" + + +def test_string_not_starts_with( + schema_data_file: Schema, data_file: DataFile, data_file_2: DataFile, data_file_3: DataFile, data_file_4: DataFile +) -> None: + should_read = _InclusiveMetricsEvaluator(schema_data_file, NotStartsWith("required", "a")).eval(data_file) + assert should_read, "Should read: no stats" + + should_read = _InclusiveMetricsEvaluator(schema_data_file, NotStartsWith("required", "a")).eval(data_file_2) + assert should_read, "Should read: range matches" + + should_read = _InclusiveMetricsEvaluator(schema_data_file, NotStartsWith("required", "aa")).eval(data_file_2) + assert should_read, "Should read: range matches" + + should_read = _InclusiveMetricsEvaluator(schema_data_file, NotStartsWith("required", "aaa")).eval(data_file_2) + assert should_read, "Should read: range matches" + + should_read = _InclusiveMetricsEvaluator(schema_data_file, NotStartsWith("required", "1s")).eval(data_file_3) + assert should_read, "Should read: range matches" + + should_read = _InclusiveMetricsEvaluator(schema_data_file, NotStartsWith("required", "1str1x")).eval(data_file_3) + assert should_read, "Should read: range matches" + + should_read = _InclusiveMetricsEvaluator(schema_data_file, NotStartsWith("required", "ff")).eval(data_file_4) + assert should_read, "Should read: range matches" + + should_read = _InclusiveMetricsEvaluator(schema_data_file, NotStartsWith("required", "aB")).eval(data_file_2) + assert should_read, "Should not read: range doesn't match" + + should_read = _InclusiveMetricsEvaluator(schema_data_file, NotStartsWith("required", "dWX")).eval(data_file_2) + assert should_read, "Should not read: range doesn't match" + + should_read = _InclusiveMetricsEvaluator(schema_data_file, NotStartsWith("required", "5")).eval(data_file_3) + assert should_read, "Should not read: range doesn't match" + + should_read = _InclusiveMetricsEvaluator(schema_data_file, NotStartsWith("required", "3str3x")).eval(data_file_3) + assert should_read, "Should not read: range doesn't match" + + # above_max = UnicodeUtil.truncateStringMax(Literal.of("イロハニホヘト"), 4).value().toString(); + + # should_read = _InclusiveMetricsEvaluator(schema_data_file, NotStartsWith("required", above_max)).eval(data_file_4) + # assert should_read, "Should not read: range doesn't match" diff --git a/tests/expressions/test_parser.py b/tests/expressions/test_parser.py index 3eaa6d2ba7..80b6c8d0ed 100644 --- a/tests/expressions/test_parser.py +++ b/tests/expressions/test_parser.py @@ -34,7 +34,9 @@ NotIn, NotNaN, NotNull, + NotStartsWith, Or, + StartsWith, parser, ) @@ -149,3 +151,11 @@ def test_and_or_with_parens() -> None: assert Or(IsNull("x"), And(GreaterThanOrEqual("x", 5), Not(LessThan("x", 10)))) == parser.parse( "(x is null) or (5 <= x) and not(x < 10)" ) + + +def test_starts_with() -> None: + assert StartsWith("x", "data") == parser.parse("x LIKE 'data'") + + +def test_not_starts_with() -> None: + assert NotStartsWith("x", "data") == parser.parse("x NOT LIKE 'data'") diff --git a/tests/expressions/test_visitors.py b/tests/expressions/test_visitors.py index 1b28080e32..001d83c719 100644 --- a/tests/expressions/test_visitors.py +++ b/tests/expressions/test_visitors.py @@ -38,8 +38,10 @@ BoundNotIn, BoundNotNaN, BoundNotNull, + BoundNotStartsWith, BoundPredicate, BoundReference, + BoundStartsWith, BoundTerm, EqualTo, GreaterThan, @@ -54,8 +56,10 @@ NotIn, NotNaN, NotNull, + NotStartsWith, Or, Reference, + StartsWith, UnboundPredicate, ) from pyiceberg.expressions.literals import Literal, literal @@ -205,6 +209,14 @@ def visit_or(self, left_result: List[str], right_result: List[str]) -> List[str] self.visit_history.append("OR") return self.visit_history + def visit_starts_with(self, term: BoundTerm[Any], literal: Literal[Any]) -> List[str]: + self.visit_history.append("STARTS_WITH") + return self.visit_history + + def visit_not_starts_with(self, term: BoundTerm[Any], literal: Literal[Any]) -> List[str]: + self.visit_history.append("NOT_STARTS_WITH") + return self.visit_history + def test_boolean_expression_visitor() -> None: """Test post-order traversal of boolean expression visit method""" @@ -780,6 +792,32 @@ def test_bound_boolean_expression_visitor_raise_on_unbound_predicate() -> None: assert "Not a bound predicate" in str(exc_info.value) +def test_bound_boolean_expression_visitor_starts_with() -> None: + bound_expression = BoundStartsWith( + term=BoundReference( + field=NestedField(field_id=1, name="foo", field_type=StringType(), required=False), + accessor=Accessor(position=0, inner=None), + ), + literal=literal("foo"), + ) + visitor = FooBoundBooleanExpressionVisitor() + result = visit(bound_expression, visitor=visitor) + assert result == ["STARTS_WITH"] + + +def test_bound_boolean_expression_visitor_not_starts_with() -> None: + bound_expression = BoundNotStartsWith( + term=BoundReference( + field=NestedField(field_id=1, name="foo", field_type=StringType(), required=False), + accessor=Accessor(position=0, inner=None), + ), + literal=literal("foo"), + ) + visitor = FooBoundBooleanExpressionVisitor() + result = visit(bound_expression, visitor=visitor) + assert result == ["NOT_STARTS_WITH"] + + def _to_byte_buffer(field_type: IcebergType, val: Any) -> bytes: if not isinstance(field_type, PrimitiveType): raise ValueError(f"Expected a PrimitiveType, got: {type(field_type)}") @@ -1359,6 +1397,90 @@ def test_integer_not_in(schema: Schema, manifest: ManifestFile) -> None: ), "Should read: in on no nulls column" +def test_string_starts_with(schema: Schema, manifest: ManifestFile) -> None: + assert _ManifestEvalVisitor(schema, StartsWith(Reference("some_nulls"), "a"), case_sensitive=False).eval( + manifest + ), "Should read: range matches" + + assert _ManifestEvalVisitor(schema, StartsWith(Reference("some_nulls"), "aa"), case_sensitive=False).eval( + manifest + ), "Should read: range matches" + + assert _ManifestEvalVisitor(schema, StartsWith(Reference("some_nulls"), "dddd"), case_sensitive=False).eval( + manifest + ), "Should read: range matches" + + assert _ManifestEvalVisitor(schema, StartsWith(Reference("some_nulls"), "z"), case_sensitive=False).eval( + manifest + ), "Should read: range matches" + + assert _ManifestEvalVisitor(schema, StartsWith(Reference("no_nulls"), "a"), case_sensitive=False).eval( + manifest + ), "Should read: range matches" + + assert not _ManifestEvalVisitor(schema, StartsWith(Reference("some_nulls"), "zzzz"), case_sensitive=False).eval( + manifest + ), "Should skip: range doesn't match" + + assert not _ManifestEvalVisitor(schema, StartsWith(Reference("some_nulls"), "1"), case_sensitive=False).eval( + manifest + ), "Should skip: range doesn't match" + + +def test_string_not_starts_with(schema: Schema, manifest: ManifestFile) -> None: + assert _ManifestEvalVisitor(schema, NotStartsWith(Reference("some_nulls"), "a"), case_sensitive=False).eval( + manifest + ), "Should read: range matches" + + assert _ManifestEvalVisitor(schema, NotStartsWith(Reference("some_nulls"), "aa"), case_sensitive=False).eval( + manifest + ), "Should read: range matches" + + assert _ManifestEvalVisitor(schema, NotStartsWith(Reference("some_nulls"), "dddd"), case_sensitive=False).eval( + manifest + ), "Should read: range matches" + + assert _ManifestEvalVisitor(schema, NotStartsWith(Reference("some_nulls"), "z"), case_sensitive=False).eval( + manifest + ), "Should read: range matches" + + assert _ManifestEvalVisitor(schema, NotStartsWith(Reference("no_nulls"), "a"), case_sensitive=False).eval( + manifest + ), "Should read: range matches" + + assert _ManifestEvalVisitor(schema, NotStartsWith(Reference("some_nulls"), "zzzz"), case_sensitive=False).eval( + manifest + ), "Should read: range matches" + + assert _ManifestEvalVisitor(schema, NotStartsWith(Reference("some_nulls"), "1"), case_sensitive=False).eval( + manifest + ), "Should read: range matches" + + assert _ManifestEvalVisitor(schema, NotStartsWith(Reference("all_same_value_or_null"), "a"), case_sensitive=False).eval( + manifest + ), "Should read: range matches" + + assert _ManifestEvalVisitor(schema, NotStartsWith(Reference("all_same_value_or_null"), "aa"), case_sensitive=False).eval( + manifest + ), "Should read: range matches" + + assert _ManifestEvalVisitor(schema, NotStartsWith(Reference("all_same_value_or_null"), "A"), case_sensitive=False).eval( + manifest + ), "Should read: range matches" + + # Iceberg does not implement SQL's 3-way boolean logic, so the choice of an all null column + # matching is + # by definition in order to surface more values to the query engine to allow it to make its own + # decision. + assert _ManifestEvalVisitor(schema, NotStartsWith(Reference("all_nulls_missing_nan"), "A"), case_sensitive=False).eval( + manifest + ), "Should read: range matches" + + assert not _ManifestEvalVisitor(schema, NotStartsWith(Reference("no_nulls_same_value_a"), "a"), case_sensitive=False).eval( + manifest + ), "Should not read: all values start with the prefix" + + def test_rewrite_not_equal_to() -> None: assert rewrite_not(Not(EqualTo(Reference("x"), 34.56))) == NotEqualTo(Reference("x"), 34.56) diff --git a/tests/io/test_pyarrow.py b/tests/io/test_pyarrow.py index c07890053d..a95f0c1a27 100644 --- a/tests/io/test_pyarrow.py +++ b/tests/io/test_pyarrow.py @@ -44,7 +44,9 @@ BoundNotIn, BoundNotNaN, BoundNotNull, + BoundNotStartsWith, BoundReference, + BoundStartsWith, GreaterThan, Not, Or, @@ -529,6 +531,20 @@ def test_expr_not_in_to_pyarrow(bound_reference: BoundReference[str]) -> None: ) +def test_expr_starts_with_to_pyarrow(bound_reference: BoundReference[str]) -> None: + assert ( + repr(expression_to_pyarrow(BoundStartsWith(bound_reference, literal("he")))) + == '' + ) + + +def test_expr_not_starts_with_to_pyarrow(bound_reference: BoundReference[str]) -> None: + assert ( + repr(expression_to_pyarrow(BoundNotStartsWith(bound_reference, literal("he")))) + == '' + ) + + def test_and_to_pyarrow(bound_reference: BoundReference[str]) -> None: assert ( repr(expression_to_pyarrow(And(BoundEqualTo(bound_reference, literal("hello")), BoundIsNull(bound_reference)))) diff --git a/tests/test_transforms.py b/tests/test_transforms.py index 8ba80e396b..ba03393dd8 100644 --- a/tests/test_transforms.py +++ b/tests/test_transforms.py @@ -33,14 +33,18 @@ BoundLessThanOrEqual, BoundNotIn, BoundNotNull, + BoundNotStartsWith, BoundReference, + BoundStartsWith, EqualTo, GreaterThanOrEqual, In, LessThanOrEqual, NotIn, NotNull, + NotStartsWith, Reference, + StartsWith, ) from pyiceberg.expressions.literals import ( DateLiteral, @@ -898,3 +902,15 @@ def test_projection_truncate_long_in(bound_reference_decimal: BoundReference[Dec Decimal("18.14999999999999857891452847979962825775146484374"), }, ) + + +def test_projection_truncate_string_starts_with(bound_reference_str: BoundReference[str]) -> None: + assert TruncateTransform(2).project( + "name", BoundStartsWith(term=bound_reference_str, literal=literal("hello")) + ) == StartsWith(term="name", literal=literal("he")) + + +def test_projection_truncate_string_not_starts_with(bound_reference_str: BoundReference[str]) -> None: + assert TruncateTransform(2).project( + "name", BoundNotStartsWith(term=bound_reference_str, literal=literal("hello")) + ) == NotStartsWith(term="name", literal=literal("he")) From 1ea1ab53db18fb6e3f5b831a6b1db3a394ec9982 Mon Sep 17 00:00:00 2001 From: Isa Inalcik Date: Fri, 3 Mar 2023 21:41:33 +0100 Subject: [PATCH 389/642] Python: Config loading on windows (#7002) * Python: fix config loading on windows * Python: add backticks to docs * Python: remove HOME env variable and us os.path.expanduser() * Python: remove HOME env variable and use os.path.expanduser() * Update config.py --------- Co-authored-by: Isa.Inalcik Co-authored-by: Fokko Driesprong --- mkdocs/docs/api.md | 2 +- pyiceberg/utils/config.py | 5 ++--- 2 files changed, 3 insertions(+), 4 deletions(-) diff --git a/mkdocs/docs/api.md b/mkdocs/docs/api.md index a3d6edf015..302c6fd88a 100644 --- a/mkdocs/docs/api.md +++ b/mkdocs/docs/api.md @@ -28,7 +28,7 @@ catalog: credential: t-1234:secret ``` -These info must be placed inside a file called `.pyiceberg.yaml` located either in the `$HOME` directory or in the `$PYICEBERG_HOME` directory (if var is set). +This information must be placed inside a file called `.pyiceberg.yaml` located either in the `$HOME` or `%USERPROFILE%` directory (depending on whether the operating system is Unix-based or Windows-based, respectively) or in the `$PYICEBERG_HOME` directory (if the corresponding environment variable is set). For more details on possible configurations refer to the [specific page](https://py.iceberg.apache.org/configuration/). diff --git a/pyiceberg/utils/config.py b/pyiceberg/utils/config.py index 1af3e13aa0..764054d7f9 100644 --- a/pyiceberg/utils/config.py +++ b/pyiceberg/utils/config.py @@ -26,7 +26,6 @@ DEFAULT = "default" CATALOG = "catalog" DEFAULT_CATALOG = f"{DEFAULT}-{CATALOG}" -HOME = "HOME" PYICEBERG_HOME = "PYICEBERG_HOME" PYICEBERG_YML = ".pyiceberg.yaml" @@ -75,7 +74,7 @@ def _from_configuration_files() -> Optional[RecursiveDict]: def _load_yaml(directory: Optional[str]) -> Optional[RecursiveDict]: if directory: - path = f"{directory.rstrip('/')}/{PYICEBERG_YML}" + path = os.path.join(directory, PYICEBERG_YML) if os.path.isfile(path): with open(path, encoding="utf-8") as f: file_config = yaml.safe_load(f) @@ -87,7 +86,7 @@ def _load_yaml(directory: Optional[str]) -> Optional[RecursiveDict]: if pyiceberg_home_config := _load_yaml(os.environ.get(PYICEBERG_HOME)): return pyiceberg_home_config # Look into the home directory - if pyiceberg_home_config := _load_yaml(os.environ.get(HOME)): + if pyiceberg_home_config := _load_yaml(os.path.expanduser("~")): return pyiceberg_home_config # Didn't find a config return None From b78d0c01abcd36c22ddccf49c36b0814927eebe1 Mon Sep 17 00:00:00 2001 From: Fokko Driesprong Date: Sat, 4 Mar 2023 10:19:58 +0100 Subject: [PATCH 390/642] Python: Wrap FsspecFileIO in PyArrowFileIO (#6972) * Python: Wrap FsspecFileIO in PyArrowFileIO This allows us to read PyArrow tables using FsspecFileIO. For ADLS users this allows them to pull data from ADLS. * Add test * Make CI happy --- pyiceberg/io/fsspec.py | 6 ++++++ pyiceberg/io/pyarrow.py | 15 +++++++++++++-- tests/conftest.py | 23 +++++++++++++++++++++++ tests/io/test_pyarrow.py | 37 ++++++++++++++++++++++++++++++++++++- 4 files changed, 78 insertions(+), 3 deletions(-) diff --git a/pyiceberg/io/fsspec.py b/pyiceberg/io/fsspec.py index d3df3a8be8..ce81853508 100644 --- a/pyiceberg/io/fsspec.py +++ b/pyiceberg/io/fsspec.py @@ -31,6 +31,7 @@ from botocore import UNSIGNED from botocore.awsrequest import AWSRequest from fsspec import AbstractFileSystem +from pyarrow.filesystem import LocalFileSystem from requests import HTTPError from pyiceberg.catalog import TOKEN @@ -78,6 +79,10 @@ def s3v4_rest_signer(properties: Properties, request: AWSRequest, **_: Any) -> A SIGNERS: Dict[str, Callable[[Properties, AWSRequest], AWSRequest]] = {"S3V4RestSigner": s3v4_rest_signer} +def _file(_: Properties) -> LocalFileSystem: + return LocalFileSystem() + + def _s3(properties: Properties) -> AbstractFileSystem: from s3fs import S3FileSystem @@ -117,6 +122,7 @@ def _adlfs(properties: Properties) -> AbstractFileSystem: SCHEME_TO_FS = { + "file": _file, "s3": _s3, "s3a": _s3, "s3n": _s3, diff --git a/pyiceberg/io/pyarrow.py b/pyiceberg/io/pyarrow.py index ce9ce70344..4cf81dadd8 100644 --- a/pyiceberg/io/pyarrow.py +++ b/pyiceberg/io/pyarrow.py @@ -47,7 +47,9 @@ FileInfo, FileSystem, FileType, + FSSpecHandler, LocalFileSystem, + PyFileSystem, S3FileSystem, ) @@ -541,11 +543,20 @@ def project_table( ResolveError: When an incompatible query is done """ + scheme, _ = PyArrowFileIO.parse_location(table.location()) if isinstance(table.io, PyArrowFileIO): - scheme, _ = PyArrowFileIO.parse_location(table.location()) fs = table.io.get_fs(scheme) else: - raise ValueError(f"Expected PyArrowFileIO, got: {table.io}") + try: + from pyiceberg.io.fsspec import FsspecFileIO + + if isinstance(table.io, FsspecFileIO): + fs = PyFileSystem(FSSpecHandler(table.io.get_fs(scheme))) + else: + raise ValueError(f"Expected PyArrowFileIO or FsspecFileIO, got: {table.io}") + except ModuleNotFoundError as e: + # When FsSpec is not installed + raise ValueError(f"Expected PyArrowFileIO or FsspecFileIO, got: {table.io}") from e bound_row_filter = bind(table.schema(), row_filter, case_sensitive=case_sensitive) diff --git a/tests/conftest.py b/tests/conftest.py index c42cfc435d..0b8c18ca28 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -49,16 +49,20 @@ import boto3 import botocore.awsrequest import botocore.model +import pyarrow as pa import pytest from moto import mock_dynamodb, mock_glue, mock_s3 +from pyarrow import parquet as pq from pyiceberg import schema from pyiceberg.catalog import Catalog from pyiceberg.io import OutputFile, OutputStream, fsspec from pyiceberg.io.fsspec import FsspecFileIO from pyiceberg.io.pyarrow import PyArrowFile, PyArrowFileIO +from pyiceberg.manifest import DataFile, FileFormat from pyiceberg.schema import Schema from pyiceberg.serializers import ToOutputFile +from pyiceberg.table import FileScanTask from pyiceberg.table.metadata import TableMetadataV2 from pyiceberg.types import ( BinaryType, @@ -1368,3 +1372,22 @@ def clean_up(test_catalog: Catalog) -> None: for identifier in test_catalog.list_tables(database_name): test_catalog.purge_table(identifier) test_catalog.drop_namespace(database_name) + + +@pytest.fixture +def data_file(table_schema_simple: Schema, tmp_path: str) -> str: + table = pa.table( + {"foo": ["a", "b", "c"], "bar": [1, 2, 3], "baz": [True, False, None]}, + metadata={"iceberg.schema": table_schema_simple.json()}, + ) + + file_path = f"{tmp_path}/0000-data.parquet" + pq.write_table(table=table, where=file_path) + return file_path + + +@pytest.fixture +def example_task(data_file: str) -> FileScanTask: + return FileScanTask( + data_file=DataFile(file_path=data_file, file_format=FileFormat.PARQUET, file_size_in_bytes=1925), + ) diff --git a/tests/io/test_pyarrow.py b/tests/io/test_pyarrow.py index a95f0c1a27..4b363d0d44 100644 --- a/tests/io/test_pyarrow.py +++ b/tests/io/test_pyarrow.py @@ -52,7 +52,7 @@ Or, literal, ) -from pyiceberg.io import InputStream, OutputStream +from pyiceberg.io import InputStream, OutputStream, load_file_io from pyiceberg.io.pyarrow import ( PyArrowFile, PyArrowFileIO, @@ -1130,3 +1130,38 @@ def test_projection_filter_on_unknown_field(schema_int_str: Schema, file_int_str _ = project(schema, [file_int_str], GreaterThan("unknown_field", "1"), schema_int_str) assert "Could not find field with name unknown_field, case_sensitive=True" in str(exc_info.value) + + +def test_pyarrow_wrap_fsspec(example_task: FileScanTask, table_schema_simple: Schema) -> None: + metadata_location = "file://a/b/c.json" + projection = project_table( + [example_task], + Table( + ("namespace", "table"), + metadata=TableMetadataV2( + location=metadata_location, + last_column_id=1, + format_version=2, + current_schema_id=1, + schemas=[table_schema_simple], + partition_specs=[PartitionSpec()], + ), + metadata_location=metadata_location, + io=load_file_io(properties={"py-io-impl": "pyiceberg.io.fsspec.FsspecFileIO"}, location=metadata_location), + ), + case_sensitive=True, + projected_schema=table_schema_simple, + row_filter=AlwaysTrue(), + ) + + assert ( + str(projection) + == """pyarrow.Table +foo: string +bar: int64 not null +baz: bool +---- +foo: [["a","b","c"]] +bar: [[1,2,3]] +baz: [[true,false,null]]""" + ) From aa10b3717e6a2d9e7fdad50f7d89d28ce27deeea Mon Sep 17 00:00:00 2001 From: Fokko Driesprong Date: Sat, 4 Mar 2023 10:26:01 +0100 Subject: [PATCH 391/642] Python: Pass location when recreating the FileIO (#6971) We need to pass in the metadata location when we recreate the file io for the table. We re-create the table because there can be new configuration, but we don't supply the location. This can cause issues when you have both PyArrow and FSspec installed, you don't have the FileIO set explicitly in your config: ```yaml catalog: default: ... py-io-impl: pyiceberg.io.fsspec.FsspecFileIO ``` When you fetch the metadata from ADLS using FSSpec, but then the re-created FileIO is PyArrow. When we pass in the url, the schema gets taken into account and PyArrow won't be considered for ADLS. --- pyiceberg/catalog/__init__.py | 4 ++-- pyiceberg/catalog/dynamodb.py | 2 +- pyiceberg/catalog/glue.py | 2 +- pyiceberg/catalog/hive.py | 2 +- pyiceberg/catalog/rest.py | 8 ++++++-- 5 files changed, 11 insertions(+), 7 deletions(-) diff --git a/pyiceberg/catalog/__init__.py b/pyiceberg/catalog/__init__.py index 041018f127..cd71d685b9 100644 --- a/pyiceberg/catalog/__init__.py +++ b/pyiceberg/catalog/__init__.py @@ -250,8 +250,8 @@ def __init__(self, name: str, **properties: str): self.name = name self.properties = properties - def _load_file_io(self, properties: Properties = EMPTY_DICT) -> FileIO: - return load_file_io({**self.properties, **properties}) + def _load_file_io(self, properties: Properties = EMPTY_DICT, location: Optional[str] = None) -> FileIO: + return load_file_io({**self.properties, **properties}, location) @abstractmethod def create_table( diff --git a/pyiceberg/catalog/dynamodb.py b/pyiceberg/catalog/dynamodb.py index 2c29d5bb1d..50f3b07571 100644 --- a/pyiceberg/catalog/dynamodb.py +++ b/pyiceberg/catalog/dynamodb.py @@ -579,7 +579,7 @@ def _convert_dynamo_table_item_to_iceberg_table(self, dynamo_table_item: Dict[st identifier=(self.name, database_name, table_name), metadata=metadata, metadata_location=metadata_location, - io=self._load_file_io(metadata.properties), + io=self._load_file_io(metadata.properties, metadata_location), ) diff --git a/pyiceberg/catalog/glue.py b/pyiceberg/catalog/glue.py index 763ad6a283..bb83eababb 100644 --- a/pyiceberg/catalog/glue.py +++ b/pyiceberg/catalog/glue.py @@ -163,7 +163,7 @@ def _convert_glue_to_iceberg(self, glue_table: Dict[str, Any]) -> Table: identifier=(glue_table[PROP_GLUE_TABLE_DATABASE_NAME], glue_table[PROP_GLUE_TABLE_NAME]), metadata=metadata, metadata_location=metadata_location, - io=self._load_file_io(metadata.properties), + io=self._load_file_io(metadata.properties, metadata_location), ) def _create_glue_table(self, database_name: str, table_name: str, table_input: Dict[str, Any]) -> None: diff --git a/pyiceberg/catalog/hive.py b/pyiceberg/catalog/hive.py index b49967caa8..3baf55864d 100644 --- a/pyiceberg/catalog/hive.py +++ b/pyiceberg/catalog/hive.py @@ -242,7 +242,7 @@ def _convert_hive_into_iceberg(self, table: HiveTable, io: FileIO) -> Table: identifier=(table.dbName, table.tableName), metadata=metadata, metadata_location=metadata_location, - io=self._load_file_io(metadata.properties), + io=self._load_file_io(metadata.properties, metadata_location), ) def create_table( diff --git a/pyiceberg/catalog/rest.py b/pyiceberg/catalog/rest.py index 89e28bd7ca..2c0cf634a9 100644 --- a/pyiceberg/catalog/rest.py +++ b/pyiceberg/catalog/rest.py @@ -369,7 +369,9 @@ def create_table( identifier=(self.name,) + self.identifier_to_tuple(identifier), metadata_location=table_response.metadata_location, metadata=table_response.metadata, - io=self._load_file_io({**table_response.metadata.properties, **table_response.config}), + io=self._load_file_io( + {**table_response.metadata.properties, **table_response.config}, table_response.metadata_location + ), ) def list_tables(self, namespace: Union[str, Identifier]) -> List[Identifier]: @@ -399,7 +401,9 @@ def load_table(self, identifier: Union[str, Identifier]) -> Table: identifier=(self.name,) + identifier_tuple if self.name else identifier_tuple, metadata_location=table_response.metadata_location, metadata=table_response.metadata, - io=self._load_file_io({**table_response.metadata.properties, **table_response.config}), + io=self._load_file_io( + {**table_response.metadata.properties, **table_response.config}, table_response.metadata_location + ), ) def drop_table(self, identifier: Union[str, Identifier], purge_requested: bool = False) -> None: From 0a45b0f1a0fdb69248c49a06f7811b16a6de0e95 Mon Sep 17 00:00:00 2001 From: Fokko Driesprong Date: Sat, 4 Mar 2023 10:28:33 +0100 Subject: [PATCH 392/642] Python: Fix ADLS configuration (#6970) The configuration keys weren't extracted according to the documentation. --- mkdocs/docs/configuration.md | 14 +++++++++----- pyiceberg/io/fsspec.py | 11 +++++++++-- tests/conftest.py | 4 ++-- 3 files changed, 20 insertions(+), 9 deletions(-) diff --git a/mkdocs/docs/configuration.md b/mkdocs/docs/configuration.md index 9691bd073a..ee13133653 100644 --- a/mkdocs/docs/configuration.md +++ b/mkdocs/docs/configuration.md @@ -64,11 +64,15 @@ For the FileIO there are several configuration options available: ### Azure Data lake -| Key | Example | Description | -|--------------------------|---------------------|-----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| -| adlfs.endpoint | http://127.0.0.1/ | Configure an alternative endpoint of the ADLFS service for the FileIO to access. This could be used to use FileIO with any adlfs-compatible object storage service that has a different endpoint (like [azurite](https://github.com/azure/azurite)). | -| adlfs.account-name | devstoreaccount1 | Configure the static storage account name used to access the FileIO. | -| adlfs.account-key | Eby8vdM02xNOcqF... | Configure the static storage account key used to access the FileIO. | +| Key | Example | Description | +|-------------------------|-------------------------------------------------------------------------------------------|--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| +| adlfs.connection-string | AccountName=devstoreaccount1;AccountKey=Eby8vdM02xNOcqF...;BlobEndpoint=http://localhost/ | A [connection string](https://learn.microsoft.com/en-us/azure/storage/common/storage-configure-connection-string). This could be used to use FileIO with any adlfs-compatible object storage service that has a different endpoint (like [azurite](https://github.com/azure/azurite)). | +| adlfs.account-name | devstoreaccount1 | The account that you want to connect to | +| adlfs.account-key | Eby8vdM02xNOcqF... | The key to authentication against the account. | +| adlfs.sas-token | NuHOuuzdQN7VRM%2FOpOeqBlawRCA845IY05h9eu1Yte4%3D | The shared access signature | +| adlfs.tenant-id | ad667be4-b811-11ed-afa1-0242ac120002 | The tenant-id | +| adlfs.client-id | ad667be4-b811-11ed-afa1-0242ac120002 | The client-id | +| adlfs.client-secret | oCA3R6P\*ka#oa1Sms2J74z... | The client-secret | ## REST Catalog diff --git a/pyiceberg/io/fsspec.py b/pyiceberg/io/fsspec.py index ce81853508..5c7e5cb873 100644 --- a/pyiceberg/io/fsspec.py +++ b/pyiceberg/io/fsspec.py @@ -117,8 +117,15 @@ def _s3(properties: Properties) -> AbstractFileSystem: def _adlfs(properties: Properties) -> AbstractFileSystem: from adlfs import AzureBlobFileSystem - fs = AzureBlobFileSystem(**properties) - return fs + return AzureBlobFileSystem( + connection_string=properties.get("adlfs.connection-string"), + account_name=properties.get("adlfs.account-name"), + account_key=properties.get("adlfs.account-key"), + sas_token=properties.get("adlfs.sas-token"), + tenant_id=properties.get("adlfs.tenant-id"), + client_id=properties.get("adlfs.client-id"), + client_secret=properties.get("adlfs.client-secret"), + ) SCHEME_TO_FS = { diff --git a/tests/conftest.py b/tests/conftest.py index 0b8c18ca28..8987150b55 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -1281,8 +1281,8 @@ def adlfs_fsspec_fileio(request: pytest.FixtureRequest) -> Generator[FsspecFileI azurite_account_key = request.config.getoption("--adlfs.account-key") azurite_connection_string = f"DefaultEndpointsProtocol=http;AccountName={azurite_account_name};AccountKey={azurite_account_key};BlobEndpoint={azurite_url}/{azurite_account_name};" properties = { - "connection_string": azurite_connection_string, - "account_name": azurite_account_name, + "adlfs.connection-string": azurite_connection_string, + "adlfs.account-name": azurite_account_name, } bbs = BlobServiceClient.from_connection_string(conn_str=azurite_connection_string) From a2fe4f21378e4d7ace244848b9a6c27e6333f809 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Sun, 5 Mar 2023 17:50:06 +0100 Subject: [PATCH 393/642] Build: Bump pytest from 7.2.1 to 7.2.2 in /python (#7019) Bumps [pytest](https://github.com/pytest-dev/pytest) from 7.2.1 to 7.2.2. - [Release notes](https://github.com/pytest-dev/pytest/releases) - [Changelog](https://github.com/pytest-dev/pytest/blob/main/CHANGELOG.rst) - [Commits](https://github.com/pytest-dev/pytest/compare/7.2.1...7.2.2) --- updated-dependencies: - dependency-name: pytest dependency-type: direct:development update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- poetry.lock | 8 ++++---- pyproject.toml | 2 +- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/poetry.lock b/poetry.lock index e462387423..dd0a466a07 100644 --- a/poetry.lock +++ b/poetry.lock @@ -1787,14 +1787,14 @@ tomli = {version = ">=1.1.0", markers = "python_version < \"3.11\""} [[package]] name = "pytest" -version = "7.2.1" +version = "7.2.2" description = "pytest: simple powerful testing with Python" category = "dev" optional = false python-versions = ">=3.7" files = [ - {file = "pytest-7.2.1-py3-none-any.whl", hash = "sha256:c7c6ca206e93355074ae32f7403e8ea12163b1163c976fee7d4d84027c162be5"}, - {file = "pytest-7.2.1.tar.gz", hash = "sha256:d45e0952f3727241918b8fd0f376f5ff6b301cc0777c6f9a556935c92d8a7d42"}, + {file = "pytest-7.2.2-py3-none-any.whl", hash = "sha256:130328f552dcfac0b1cec75c12e3f005619dc5f874f0a06e8ff7263f0ee6225e"}, + {file = "pytest-7.2.2.tar.gz", hash = "sha256:c99ab0c73aceb050f68929bc93af19ab6db0558791c6a0715723abe9d0ade9d4"}, ] [package.dependencies] @@ -2553,4 +2553,4 @@ snappy = ["python-snappy"] [metadata] lock-version = "2.0" python-versions = "^3.8" -content-hash = "a9347540fc649099f8f9837333893efffb537957f95f458cf1175b68549e81ac" +content-hash = "2214e8c9b8f4b55cfb670178ef818eb4d0cecb4cc3d773f527ade7c752c62ef6" diff --git a/pyproject.toml b/pyproject.toml index 9cfdc9aa1d..1e5b0fc57e 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -79,7 +79,7 @@ s3fs = { version = ">=2022.8.2,<=2023.1.0", optional = true } adlfs = { version = ">=2022.8.2,<=2023.1.0", optional = true } [tool.poetry.dev-dependencies] -pytest = "7.2.1" +pytest = "7.2.2" pytest-checkdocs = "2.9.0" pre-commit = "3.1.0" fastavro = "1.7.2" From f2048e6c94d5995fae28106161993d3b443933a6 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Sun, 5 Mar 2023 18:50:38 +0100 Subject: [PATCH 394/642] Build: Bump duckdb from 0.7.0 to 0.7.1 in /python (#7018) Bumps [duckdb](https://github.com/duckdb/duckdb) from 0.7.0 to 0.7.1. - [Release notes](https://github.com/duckdb/duckdb/releases) - [Changelog](https://github.com/duckdb/duckdb/blob/master/tools/release-pip.py) - [Commits](https://github.com/duckdb/duckdb/compare/v0.7.0...v0.7.1) --- updated-dependencies: - dependency-name: duckdb dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- poetry.lock | 98 +++++++++++++++++++++++++------------------------- pyproject.toml | 2 +- 2 files changed, 50 insertions(+), 50 deletions(-) diff --git a/poetry.lock b/poetry.lock index dd0a466a07..6aec0aea5f 100644 --- a/poetry.lock +++ b/poetry.lock @@ -735,59 +735,59 @@ files = [ [[package]] name = "duckdb" -version = "0.7.0" +version = "0.7.1" description = "DuckDB embedded database" category = "main" optional = true python-versions = "*" files = [ - {file = "duckdb-0.7.0-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:3fcae66586bd858c522559e85068d80414a27f23839705d5b400156c0fdbbeca"}, - {file = "duckdb-0.7.0-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:5d07d37edd7e2d1078bf500d11d068d6c1b62868bf372beca4d856cb3fa58c02"}, - {file = "duckdb-0.7.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:5b86e4bff0f5d49b6ac0aaae5fd6d4bd3d65f1aeeb77529d717515a0e4e16e60"}, - {file = "duckdb-0.7.0-cp310-cp310-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:4b9fce20c0e43089c6892f6af0112f6532ec9e163dbfd744ce6579a3ed5c63e6"}, - {file = "duckdb-0.7.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:5c730ccfef672b8fe04010862e5b766d1698c2a7401413958bd09a2640260966"}, - {file = "duckdb-0.7.0-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:36add2ce4747331bcc650d50d656604cc5442a1de9e88ab52481f51e53f10632"}, - {file = "duckdb-0.7.0-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:115e5e4ed087a129510c34dbac5bc8b30e05f95b05e34b3589771ccaab7215ec"}, - {file = "duckdb-0.7.0-cp310-cp310-win32.whl", hash = "sha256:5fa43a479c68937c7cad9ee13a6fd99417820a1f44aaaaf6d8675ea916278226"}, - {file = "duckdb-0.7.0-cp310-cp310-win_amd64.whl", hash = "sha256:864432628afbc8c4309c2c26b60cd97fdfc7891acdbc633a09e7961bdf5a1dcf"}, - {file = "duckdb-0.7.0-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:a09f52996921b8341a06a7425d8a4b6bc05fbdb8cf39ed1e915a0c9fdd3186ee"}, - {file = "duckdb-0.7.0-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:e3956bed611b1a9b543f3fed1d01ee41cfcb68cc150bebd2b3152b817541ec01"}, - {file = "duckdb-0.7.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:bc9ec4507b0c09aa163d27e3c4fcb8d3d904f0d02bb99f628534741cae45cf08"}, - {file = "duckdb-0.7.0-cp311-cp311-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:ac9e94c402d149a894febe76012cf86f6ed2b56232dd89a9ea06ac0843aacf69"}, - {file = "duckdb-0.7.0-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:6d735ba4d054cd05f7cceb4bc591151f0934460bfc6346d84063c58f5156c16a"}, - {file = "duckdb-0.7.0-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:49d33c0beaf075a8347bcab7589066f30d537bf1cba44bc87b2812343a080691"}, - {file = "duckdb-0.7.0-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:264fd585c5a6ccd0fb6ca3a957a0b82fc195afa0533392c77f7ac6b58a8cc617"}, - {file = "duckdb-0.7.0-cp311-cp311-win32.whl", hash = "sha256:5f93ec36c1b038cc09ac57c58b8506c13c722eca68e9f24af04ca0ee76b6fb1b"}, - {file = "duckdb-0.7.0-cp311-cp311-win_amd64.whl", hash = "sha256:cee0f977015dd7a35e7374ee341d56101c6561363adde213a87595e49719284c"}, - {file = "duckdb-0.7.0-cp36-cp36m-macosx_10_9_x86_64.whl", hash = "sha256:f0c9c54a6ce29ce45efc271ad4bd4d8f8b6ceb5e7e9ee66a236cda4c15ea1be6"}, - {file = "duckdb-0.7.0-cp36-cp36m-win32.whl", hash = "sha256:fe2ea12990ecc5eaf2de4f349d9c8c23b4855f8037c6e8fd1c659d22464d32ff"}, - {file = "duckdb-0.7.0-cp36-cp36m-win_amd64.whl", hash = "sha256:cc473a5ebd7efbe1c16fd72885031c167e925a2c7afbf6fb1ff66b9f4bfb93b9"}, - {file = "duckdb-0.7.0-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:462df10e74205ee45532315d239bcc37d5372c5169496ac5f227ab521d97a8a1"}, - {file = "duckdb-0.7.0-cp37-cp37m-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:22fb9d39b4cac968b2c80683be7be70138d49df0917cb6a3d141fb147afe4bb4"}, - {file = "duckdb-0.7.0-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:309ba76ac99f5a9a38579205406f13d117d9506e4a097b4bbb0a3484a103291c"}, - {file = "duckdb-0.7.0-cp37-cp37m-musllinux_1_1_i686.whl", hash = "sha256:78fb8e34d4bb73d21104c5c9cd170a0f2d79ab37bd1e7d03bd92ee65e50b80f1"}, - {file = "duckdb-0.7.0-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:5766c9d08d9cffd755153eca6c8f61f70c3b6a6d3c58ed2b2d4057b0a50f7850"}, - {file = "duckdb-0.7.0-cp37-cp37m-win32.whl", hash = "sha256:0d733bc19806e45782c79a0a6b9a6e88cd8502020533175b83db75d1fa564246"}, - {file = "duckdb-0.7.0-cp37-cp37m-win_amd64.whl", hash = "sha256:136bdc26d5f571d4ff3c09d867923e5f9ac03cc62db732d4b2ca2d4d9f84dd1d"}, - {file = "duckdb-0.7.0-cp38-cp38-macosx_10_9_universal2.whl", hash = "sha256:a22607efe25bc3b85e078fc5619095bf0d1cc8d39f0039a0e424075e2c77197b"}, - {file = "duckdb-0.7.0-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:1db7cacd682f465a3770ca97928edb9d245ef3ebe4e195c3cb6b812f099fae4f"}, - {file = "duckdb-0.7.0-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:ef71d92a738b1e4f64073e355dae90122793d5008c5e728e35b17bc023dbc7a7"}, - {file = "duckdb-0.7.0-cp38-cp38-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:65458c1fac678e1e03cf9e0aa00666b3990792adb318fe41bd49b15fe8a6e78a"}, - {file = "duckdb-0.7.0-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:2f21317cc8cc7f331936cc42294f52b8212a8e6a221b09672f0bfa7f290898a5"}, - {file = "duckdb-0.7.0-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:b9e0010145b0abe510bb9ea34782fa82df4ef4f2e6c9b87877df99ff20b8c993"}, - {file = "duckdb-0.7.0-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:b0e308354b54d235cd262c52036da56b3d49969c7cad47f784aff618cb3bed7a"}, - {file = "duckdb-0.7.0-cp38-cp38-win32.whl", hash = "sha256:8ae0e5ec024de474f12613524b5c2466955b5e42a68ea2ec788ac50040d44b88"}, - {file = "duckdb-0.7.0-cp38-cp38-win_amd64.whl", hash = "sha256:99ba303d81d8145edcbb5cae27e873e3c1c403532041cbe94b514af8e7db3009"}, - {file = "duckdb-0.7.0-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:1ce42f93dd35e6b256abe1c7344445913536da4604f6e8ee3c886fe7c1eb5580"}, - {file = "duckdb-0.7.0-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:b09ba609cf1744f87c1c7143db46c668e24bab893f422e22c7d059bc07a18b3d"}, - {file = "duckdb-0.7.0-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:fdb88cccd4a9446e67a375ac7e8058c35df62f4199b781e2e408ad95af10deb3"}, - {file = "duckdb-0.7.0-cp39-cp39-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:94d3cf46267a698f5e98dadf9cb0ce5c4f68e7a4c9702c2a37b57d56d2f944d7"}, - {file = "duckdb-0.7.0-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:5ff5ff3326faa087892d6529e700774878227c80ce8e16b018a62cb50d5bd4ac"}, - {file = "duckdb-0.7.0-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:ee3333009b828faf767582b23e0b9c1c4f888e8f1ae42de614f3ed0c40b1fe49"}, - {file = "duckdb-0.7.0-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:1521f430409ac6be7b959bb6f3ebfa9597f6a8c8837e13e060022fd9f6b45726"}, - {file = "duckdb-0.7.0-cp39-cp39-win32.whl", hash = "sha256:77e4607798d196c601795032b50cfb0d564ff42cbd10cf8ff47a5eb992bb442f"}, - {file = "duckdb-0.7.0-cp39-cp39-win_amd64.whl", hash = "sha256:726a3d555c6387e6653bc46d94645eac8139f825adfd0b7876ea858b5717796b"}, - {file = "duckdb-0.7.0.tar.gz", hash = "sha256:08ebfcc67a6a073fa2efd0453de9a7e453637bba522e67968fe0c03ccbfc2ec5"}, + {file = "duckdb-0.7.1-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:3e0170be6cc315c179169dfa3e06485ef7009ef8ce399cd2908f29105ef2c67b"}, + {file = "duckdb-0.7.1-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:6360d41023e726646507d5479ba60960989a09f04527b36abeef3643c61d8c48"}, + {file = "duckdb-0.7.1-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:578c269d7aa27184e8d45421694f89deda3f41fe6bd2a8ce48b262b9fc975326"}, + {file = "duckdb-0.7.1-cp310-cp310-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:36aae9a923c9f78da1cf3fcf75873f62d32ea017d4cef7c706d16d3eca527ca2"}, + {file = "duckdb-0.7.1-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:630e0122a02f19bb1fafae00786350b2c31ae8422fce97c827bd3686e7c386af"}, + {file = "duckdb-0.7.1-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:9b9ca2d294725e523ce207bc37f28787478ae6f7a223e2cf3a213a2d498596c3"}, + {file = "duckdb-0.7.1-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:0bd89f388205b6c99b62650169efe9a02933555ee1d46ddf79fbd0fb9e62652b"}, + {file = "duckdb-0.7.1-cp310-cp310-win32.whl", hash = "sha256:a9e987565a268fd8da9f65e54621d28f39c13105b8aee34c96643074babe6d9c"}, + {file = "duckdb-0.7.1-cp310-cp310-win_amd64.whl", hash = "sha256:5d986b5ad1307b069309f9707c0c5051323e29865aefa059eb6c3b22dc9751b6"}, + {file = "duckdb-0.7.1-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:54606dfd24d7181d3098030ca6858f6be52f3ccbf42fff05f7587f2d9cdf4343"}, + {file = "duckdb-0.7.1-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:bd9367ae650b6605ffe00412183cf0edb688a5fc9fbb03ed757e8310e7ec3b6c"}, + {file = "duckdb-0.7.1-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:aaf33aeb543c7816bd915cd10141866d54f92f698e1b5712de9d8b7076da19df"}, + {file = "duckdb-0.7.1-cp311-cp311-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:2e56b0329c38c0356b40449917bab6fce6ac27d356257b9a9da613d2a0f064e0"}, + {file = "duckdb-0.7.1-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:604b8b476d6cc6bf91625d8c2722ef9c50c402b3d64bc518c838d6c279e6d93b"}, + {file = "duckdb-0.7.1-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:32a268508c6d7fdc99d5442736051de74c28a5166c4cc3dcbbf35d383299b941"}, + {file = "duckdb-0.7.1-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:90794406fa2111414877ee9db154fef940911f3920c312c1cf69947621737c8d"}, + {file = "duckdb-0.7.1-cp311-cp311-win32.whl", hash = "sha256:bf20c5ee62cbbf10b39ebdfd70d454ce914e70545c7cb6cb78cb5befef96328a"}, + {file = "duckdb-0.7.1-cp311-cp311-win_amd64.whl", hash = "sha256:bb2700785cab37cd1e7a76c4547a5ab0f8a7c28ad3f3e4d02a8fae52be223090"}, + {file = "duckdb-0.7.1-cp36-cp36m-macosx_10_9_x86_64.whl", hash = "sha256:b09741cfa31388b8f9cdf5c5200e0995d55a5b54d2d1a75b54784e2f5c042f7f"}, + {file = "duckdb-0.7.1-cp36-cp36m-win32.whl", hash = "sha256:766e6390f7ace7f1e322085c2ca5d0ad94767bde78a38d168253d2b0b4d5cd5c"}, + {file = "duckdb-0.7.1-cp36-cp36m-win_amd64.whl", hash = "sha256:6a3f3315e2b553db3463f07324f62dfebaf3b97656a87558e59e2f1f816eaf15"}, + {file = "duckdb-0.7.1-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:278edb8c912d836b3b77fd1695887e1dbd736137c3912478af3608c9d7307bb0"}, + {file = "duckdb-0.7.1-cp37-cp37m-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:e189b558d10b58fe6ed85ce79f728e143eb4115db1e63147a44db613cd4dd0d9"}, + {file = "duckdb-0.7.1-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:6b91ec3544ee4dc9e6abbdf2669475d5adedaaea51987c67acf161673e6b7443"}, + {file = "duckdb-0.7.1-cp37-cp37m-musllinux_1_1_i686.whl", hash = "sha256:3fe3f3dbd62b76a773144eef31aa29794578c359da932e77fef04516535318ca"}, + {file = "duckdb-0.7.1-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:1e78c7f59325e99f0b3d9fe7c2bad4aaadf42d2c7711925cc26331d7647a91b2"}, + {file = "duckdb-0.7.1-cp37-cp37m-win32.whl", hash = "sha256:bc2a12d9f4fc8ef2fd1022d610287c9fc9972ea06b7510fc87387f1fa256a390"}, + {file = "duckdb-0.7.1-cp37-cp37m-win_amd64.whl", hash = "sha256:53e3db1bc0f445ee48b23cde47bfba08c7fa5a69976c740ec8cdf89543d2405d"}, + {file = "duckdb-0.7.1-cp38-cp38-macosx_10_9_universal2.whl", hash = "sha256:1247cc11bac17f2585d11681329806c86295e32242f84a10a604665e697d5c81"}, + {file = "duckdb-0.7.1-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:5feaff16a012075b49dfa09d4cb24455938d6b0e06b08e1404ec00089119dba2"}, + {file = "duckdb-0.7.1-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:b411a0c361eab9b26dcd0d0c7a0d1bc0ad6b214068555de7e946fbdd2619961a"}, + {file = "duckdb-0.7.1-cp38-cp38-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:c7c76d8694ecdb579241ecfeaf03c51d640b984dbbe8e1d9f919089ebf3cdea6"}, + {file = "duckdb-0.7.1-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:193b896eed44d8751a755ccf002a137630020af0bc3505affa21bf19fdc90df3"}, + {file = "duckdb-0.7.1-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:7da132ee452c80a3784b8daffd86429fa698e1b0e3ecb84660db96d36c27ad55"}, + {file = "duckdb-0.7.1-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:5fd08c97c3e8cb5bec3822cf78b966b489213dcaab24b25c05a99f7caf8db467"}, + {file = "duckdb-0.7.1-cp38-cp38-win32.whl", hash = "sha256:9cb956f94fa55c4782352dac7cc7572a58312bd7ce97332bb14591d6059f0ea4"}, + {file = "duckdb-0.7.1-cp38-cp38-win_amd64.whl", hash = "sha256:289a5f65213e66d320ebcd51a94787e7097b9d1c3492d01a121a2c809812bf19"}, + {file = "duckdb-0.7.1-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:8085ad58c9b5854ee3820804fa1797e6b3134429c1506c3faab3cb96e71b07e9"}, + {file = "duckdb-0.7.1-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:b47c19d1f2f662a5951fc6c5f6939d0d3b96689604b529cdcffd9afdcc95bff2"}, + {file = "duckdb-0.7.1-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:6a611f598226fd634b7190f509cc6dd668132ffe436b0a6b43847b4b32b99e4a"}, + {file = "duckdb-0.7.1-cp39-cp39-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:6730f03b5b78f3943b752c90bdf37b62ae3ac52302282a942cc675825b4a8dc9"}, + {file = "duckdb-0.7.1-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:fe23e938d29cd8ea6953d77dc828b7f5b95a4dbc7cd7fe5bcc3531da8cec3dba"}, + {file = "duckdb-0.7.1-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:feffe503c2e2a99480e1e5e15176f37796b3675e4dadad446fe7c2cc672aed3c"}, + {file = "duckdb-0.7.1-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:72fceb06f5bf24ad6bb5974c60d397a7a7e61b3d847507a22276de076f3392e2"}, + {file = "duckdb-0.7.1-cp39-cp39-win32.whl", hash = "sha256:c4d5217437d20d05fe23317bbc161befa1f9363f3622887cd1d2f4719b407936"}, + {file = "duckdb-0.7.1-cp39-cp39-win_amd64.whl", hash = "sha256:066885e1883464ce3b7d1fd844f9431227dcffe1ee39bfd2a05cd6d53f304557"}, + {file = "duckdb-0.7.1.tar.gz", hash = "sha256:a7db6da0366b239ea1e4541fcc19556b286872f5015c9a54c2e347146e25a2ad"}, ] [[package]] @@ -2553,4 +2553,4 @@ snappy = ["python-snappy"] [metadata] lock-version = "2.0" python-versions = "^3.8" -content-hash = "2214e8c9b8f4b55cfb670178ef818eb4d0cecb4cc3d773f527ade7c752c62ef6" +content-hash = "223bfc7ef30e7ce32751766ce64b231b297f98478a489d18eeb7b1a6fd15d709" diff --git a/pyproject.toml b/pyproject.toml index 1e5b0fc57e..1f6b1d7f8b 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -66,7 +66,7 @@ pyarrow = { version = ">=8.0.0,<=11.0.0", optional = true } pandas = { version = ">=1.4.4,<=1.5.3", optional = true } -duckdb = { version = ">=0.6.0,<=0.7.0", optional = true } +duckdb = { version = ">=0.6.0,<=0.7.1", optional = true } python-snappy = { version = "0.6.1", optional = true } From d9bebc48982b8033a8bce657bc1b7c9802f4fc81 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Sun, 5 Mar 2023 19:20:51 +0100 Subject: [PATCH 395/642] Build: Bump pre-commit from 3.1.0 to 3.1.1 in /python (#7015) --- poetry.lock | 8 ++++---- pyproject.toml | 2 +- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/poetry.lock b/poetry.lock index 6aec0aea5f..b748f34aa8 100644 --- a/poetry.lock +++ b/poetry.lock @@ -1599,14 +1599,14 @@ tests = ["pytest (>=5.4.1)", "pytest-cov (>=2.8.1)", "pytest-mypy (>=0.8.0)", "p [[package]] name = "pre-commit" -version = "3.1.0" +version = "3.1.1" description = "A framework for managing and maintaining multi-language pre-commit hooks." category = "dev" optional = false python-versions = ">=3.8" files = [ - {file = "pre_commit-3.1.0-py2.py3-none-any.whl", hash = "sha256:7001dfcd174540658822b1fd3630ceadf4f41375a5d1844b5c3b3830f227348c"}, - {file = "pre_commit-3.1.0.tar.gz", hash = "sha256:61bd9f1b96d3d1e763f2a9a0f8522aed341646800642ff6803c73fac5781f5b7"}, + {file = "pre_commit-3.1.1-py2.py3-none-any.whl", hash = "sha256:b80254e60668e1dd1f5c03a1c9e0413941d61f568a57d745add265945f65bfe8"}, + {file = "pre_commit-3.1.1.tar.gz", hash = "sha256:d63e6537f9252d99f65755ae5b79c989b462d511ebbc481b561db6a297e1e865"}, ] [package.dependencies] @@ -2553,4 +2553,4 @@ snappy = ["python-snappy"] [metadata] lock-version = "2.0" python-versions = "^3.8" -content-hash = "223bfc7ef30e7ce32751766ce64b231b297f98478a489d18eeb7b1a6fd15d709" +content-hash = "f5eae7249c3e28f5f917e9f81f931a7136abb3668e0f6eef5ecd41dcb7193287" diff --git a/pyproject.toml b/pyproject.toml index 1f6b1d7f8b..94cd4ecd96 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -81,7 +81,7 @@ adlfs = { version = ">=2022.8.2,<=2023.1.0", optional = true } [tool.poetry.dev-dependencies] pytest = "7.2.2" pytest-checkdocs = "2.9.0" -pre-commit = "3.1.0" +pre-commit = "3.1.1" fastavro = "1.7.2" coverage = { version = "^7.2.0", extras = ["toml"] } requests-mock = "1.10.0" From 627c82d7219b601f642e09576f41dd976bc45a86 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Sun, 5 Mar 2023 19:38:06 +0100 Subject: [PATCH 396/642] Build: Bump rich from 13.3.1 to 13.3.2 in /python (#7016) --- poetry.lock | 12 ++++++------ pyproject.toml | 2 +- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/poetry.lock b/poetry.lock index b748f34aa8..34cc19b973 100644 --- a/poetry.lock +++ b/poetry.lock @@ -2073,19 +2073,19 @@ tests = ["coverage (>=6.0.0)", "flake8", "mypy", "pytest (>=7.0.0)", "pytest-asy [[package]] name = "rich" -version = "13.3.1" +version = "13.3.2" description = "Render rich text, tables, progress bars, syntax highlighting, markdown and more to the terminal" category = "main" optional = false python-versions = ">=3.7.0" files = [ - {file = "rich-13.3.1-py3-none-any.whl", hash = "sha256:8aa57747f3fc3e977684f0176a88e789be314a99f99b43b75d1e9cb5dc6db9e9"}, - {file = "rich-13.3.1.tar.gz", hash = "sha256:125d96d20c92b946b983d0d392b84ff945461e5a06d3867e9f9e575f8697b67f"}, + {file = "rich-13.3.2-py3-none-any.whl", hash = "sha256:a104f37270bf677148d8acb07d33be1569eeee87e2d1beb286a4e9113caf6f2f"}, + {file = "rich-13.3.2.tar.gz", hash = "sha256:91954fe80cfb7985727a467ca98a7618e5dd15178cc2da10f553b36a93859001"}, ] [package.dependencies] -markdown-it-py = ">=2.1.0,<3.0.0" -pygments = ">=2.14.0,<3.0.0" +markdown-it-py = ">=2.2.0,<3.0.0" +pygments = ">=2.13.0,<3.0.0" typing-extensions = {version = ">=4.0.0,<5.0", markers = "python_version < \"3.9\""} [package.extras] @@ -2553,4 +2553,4 @@ snappy = ["python-snappy"] [metadata] lock-version = "2.0" python-versions = "^3.8" -content-hash = "f5eae7249c3e28f5f917e9f81f931a7136abb3668e0f6eef5ecd41dcb7193287" +content-hash = "e466c6266c05f51de03bc7c767d733d9050289cafb191435e789e541e0e1047b" diff --git a/pyproject.toml b/pyproject.toml index 94cd4ecd96..f0b10fa3da 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -52,7 +52,7 @@ python = "^3.8" mmhash3 = "3.0.1" requests = ">=2.28.1,<=2.28.2" click = "8.1.3" -rich = ">=13.0.0,<=13.3.1" +rich = ">=13.0.0,<=13.3.2" pyyaml = ">=5.4.0,<=6.0.0" pydantic = "1.10.5" From 900d5a470aafb7915c5412208d92fba95ec27384 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Sun, 5 Mar 2023 20:36:42 +0100 Subject: [PATCH 397/642] Build: Bump coverage from 7.2.0 to 7.2.1 in /python (#7017) Bumps [coverage](https://github.com/nedbat/coveragepy) from 7.2.0 to 7.2.1. - [Release notes](https://github.com/nedbat/coveragepy/releases) - [Changelog](https://github.com/nedbat/coveragepy/blob/master/CHANGES.rst) - [Commits](https://github.com/nedbat/coveragepy/compare/7.2.0...7.2.1) --- updated-dependencies: - dependency-name: coverage dependency-type: direct:development update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- poetry.lock | 106 ++++++++++++++++++++++++------------------------- pyproject.toml | 2 +- 2 files changed, 54 insertions(+), 54 deletions(-) diff --git a/poetry.lock b/poetry.lock index 34cc19b973..10820e3ac3 100644 --- a/poetry.lock +++ b/poetry.lock @@ -598,63 +598,63 @@ files = [ [[package]] name = "coverage" -version = "7.2.0" +version = "7.2.1" description = "Code coverage measurement for Python" category = "dev" optional = false python-versions = ">=3.7" files = [ - {file = "coverage-7.2.0-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:90e7a4cbbb7b1916937d380beb1315b12957b8e895d7d9fb032e2038ac367525"}, - {file = "coverage-7.2.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:34d7211be69b215ad92298a962b2cd5a4ef4b17c7871d85e15d3d1b6dc8d8c96"}, - {file = "coverage-7.2.0-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:971b49dbf713044c3e5f6451b39f65615d4d1c1d9a19948fa0f41b0245a98765"}, - {file = "coverage-7.2.0-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:f0557289260125a6c453ad5673ba79e5b6841d9a20c9e101f758bfbedf928a77"}, - {file = "coverage-7.2.0-cp310-cp310-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:049806ae2df69468c130f04f0fab4212c46b34ba5590296281423bb1ae379df2"}, - {file = "coverage-7.2.0-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:875b03d92ac939fbfa8ae74a35b2c468fc4f070f613d5b1692f9980099a3a210"}, - {file = "coverage-7.2.0-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:c160e34e388277f10c50dc2c7b5e78abe6d07357d9fe7fcb2f3c156713fd647e"}, - {file = "coverage-7.2.0-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:32e6a730fd18b2556716039ab93278ccebbefa1af81e6aa0c8dba888cf659e6e"}, - {file = "coverage-7.2.0-cp310-cp310-win32.whl", hash = "sha256:f3ff4205aff999164834792a3949f82435bc7c7655c849226d5836c3242d7451"}, - {file = "coverage-7.2.0-cp310-cp310-win_amd64.whl", hash = "sha256:93db11da6e728587e943dff8ae1b739002311f035831b6ecdb15e308224a4247"}, - {file = "coverage-7.2.0-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:cd38140b56538855d3d5722c6d1b752b35237e7ea3f360047ce57f3fade82d98"}, - {file = "coverage-7.2.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:9dbb21561b0e04acabe62d2c274f02df0d715e8769485353ddf3cf84727e31ce"}, - {file = "coverage-7.2.0-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:171dd3aa71a49274a7e4fc26f5bc167bfae5a4421a668bc074e21a0522a0af4b"}, - {file = "coverage-7.2.0-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:4655ecd813f4ba44857af3e9cffd133ab409774e9d2a7d8fdaf4fdfd2941b789"}, - {file = "coverage-7.2.0-cp311-cp311-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:1856a8c4aa77eb7ca0d42c996d0ca395ecafae658c1432b9da4528c429f2575c"}, - {file = "coverage-7.2.0-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:bd67df6b48db18c10790635060858e2ea4109601e84a1e9bfdd92e898dc7dc79"}, - {file = "coverage-7.2.0-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:2d7daf3da9c7e0ed742b3e6b4de6cc464552e787b8a6449d16517b31bbdaddf5"}, - {file = "coverage-7.2.0-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:bf9e02bc3dee792b9d145af30db8686f328e781bd212fdef499db5e9e4dd8377"}, - {file = "coverage-7.2.0-cp311-cp311-win32.whl", hash = "sha256:3713a8ec18781fda408f0e853bf8c85963e2d3327c99a82a22e5c91baffcb934"}, - {file = "coverage-7.2.0-cp311-cp311-win_amd64.whl", hash = "sha256:88ae5929f0ef668b582fd7cad09b5e7277f50f912183cf969b36e82a1c26e49a"}, - {file = "coverage-7.2.0-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:5e29a64e9586194ea271048bc80c83cdd4587830110d1e07b109e6ff435e5dbc"}, - {file = "coverage-7.2.0-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:8d5302eb84c61e758c9d68b8a2f93a398b272073a046d07da83d77b0edc8d76b"}, - {file = "coverage-7.2.0-cp37-cp37m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:2c9fffbc39dc4a6277e1525cab06c161d11ee3995bbc97543dc74fcec33e045b"}, - {file = "coverage-7.2.0-cp37-cp37m-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:a6ceeab5fca62bca072eba6865a12d881f281c74231d2990f8a398226e1a5d96"}, - {file = "coverage-7.2.0-cp37-cp37m-musllinux_1_1_aarch64.whl", hash = "sha256:28563a35ef4a82b5bc5160a01853ce62b9fceee00760e583ffc8acf9e3413753"}, - {file = "coverage-7.2.0-cp37-cp37m-musllinux_1_1_i686.whl", hash = "sha256:bfa065307667f1c6e1f4c3e13f415b0925e34e56441f5fda2c84110a4a1d8bda"}, - {file = "coverage-7.2.0-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:7f992b32286c86c38f07a8b5c3fc88384199e82434040a729ec06b067ee0d52c"}, - {file = "coverage-7.2.0-cp37-cp37m-win32.whl", hash = "sha256:2c15bd09fd5009f3a79c8b3682b52973df29761030b692043f9834fc780947c4"}, - {file = "coverage-7.2.0-cp37-cp37m-win_amd64.whl", hash = "sha256:f332d61fbff353e2ef0f3130a166f499c3fad3a196e7f7ae72076d41a6bfb259"}, - {file = "coverage-7.2.0-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:577a8bc40c01ad88bb9ab1b3a1814f2f860ff5c5099827da2a3cafc5522dadea"}, - {file = "coverage-7.2.0-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:9240a0335365c29c968131bdf624bb25a8a653a9c0d8c5dbfcabf80b59c1973c"}, - {file = "coverage-7.2.0-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:358d3bce1468f298b19a3e35183bdb13c06cdda029643537a0cc37e55e74e8f1"}, - {file = "coverage-7.2.0-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:932048364ff9c39030c6ba360c31bf4500036d4e15c02a2afc5a76e7623140d4"}, - {file = "coverage-7.2.0-cp38-cp38-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:7efa21611ffc91156e6f053997285c6fe88cfef3fb7533692d0692d2cb30c846"}, - {file = "coverage-7.2.0-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:465ea431c3b78a87e32d7d9ea6d081a1003c43a442982375cf2c247a19971961"}, - {file = "coverage-7.2.0-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:0f03c229f1453b936916f68a47b3dfb5e84e7ad48e160488168a5e35115320c8"}, - {file = "coverage-7.2.0-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:40785553d68c61e61100262b73f665024fd2bb3c6f0f8e2cd5b13e10e4df027b"}, - {file = "coverage-7.2.0-cp38-cp38-win32.whl", hash = "sha256:b09dd7bef59448c66e6b490cc3f3c25c14bc85d4e3c193b81a6204be8dd355de"}, - {file = "coverage-7.2.0-cp38-cp38-win_amd64.whl", hash = "sha256:dc4f9a89c82faf6254d646180b2e3aa4daf5ff75bdb2c296b9f6a6cf547e26a7"}, - {file = "coverage-7.2.0-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:c243b25051440386179591a8d5a5caff4484f92c980fb6e061b9559da7cc3f64"}, - {file = "coverage-7.2.0-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:4b8fd32f85b256fc096deeb4872aeb8137474da0c0351236f93cbedc359353d6"}, - {file = "coverage-7.2.0-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:d7f2a7df523791e6a63b40360afa6792a11869651307031160dc10802df9a252"}, - {file = "coverage-7.2.0-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:da32526326e8da0effb452dc32a21ffad282c485a85a02aeff2393156f69c1c3"}, - {file = "coverage-7.2.0-cp39-cp39-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:4c1153a6156715db9d6ae8283480ae67fb67452aa693a56d7dae9ffe8f7a80da"}, - {file = "coverage-7.2.0-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:74cd60fa00f46f28bd40048d6ca26bd58e9bee61d2b0eb4ec18cea13493c003f"}, - {file = "coverage-7.2.0-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:59a427f8a005aa7254074719441acb25ac2c2f60c1f1026d43f846d4254c1c2f"}, - {file = "coverage-7.2.0-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:c3c4beddee01c8125a75cde3b71be273995e2e9ec08fbc260dd206b46bb99969"}, - {file = "coverage-7.2.0-cp39-cp39-win32.whl", hash = "sha256:08e3dd256b8d3e07bb230896c8c96ec6c5dffbe5a133ba21f8be82b275b900e8"}, - {file = "coverage-7.2.0-cp39-cp39-win_amd64.whl", hash = "sha256:ad12c74c6ce53a027f5a5ecbac9be20758a41c85425c1bbab7078441794b04ee"}, - {file = "coverage-7.2.0-pp37.pp38.pp39-none-any.whl", hash = "sha256:ffa637a2d5883298449a5434b699b22ef98dd8e2ef8a1d9e60fa9cfe79813411"}, - {file = "coverage-7.2.0.tar.gz", hash = "sha256:9cc9c41aa5af16d845b53287051340c363dd03b7ef408e45eec3af52be77810d"}, + {file = "coverage-7.2.1-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:49567ec91fc5e0b15356da07a2feabb421d62f52a9fff4b1ec40e9e19772f5f8"}, + {file = "coverage-7.2.1-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:d2ef6cae70168815ed91388948b5f4fcc69681480a0061114db737f957719f03"}, + {file = "coverage-7.2.1-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:3004765bca3acd9e015794e5c2f0c9a05587f5e698127ff95e9cfba0d3f29339"}, + {file = "coverage-7.2.1-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:cca7c0b7f5881dfe0291ef09ba7bb1582cb92ab0aeffd8afb00c700bf692415a"}, + {file = "coverage-7.2.1-cp310-cp310-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:b2167d116309f564af56f9aa5e75ef710ef871c5f9b313a83050035097b56820"}, + {file = "coverage-7.2.1-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:cb5f152fb14857cbe7f3e8c9a5d98979c4c66319a33cad6e617f0067c9accdc4"}, + {file = "coverage-7.2.1-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:87dc37f16fb5e3a28429e094145bf7c1753e32bb50f662722e378c5851f7fdc6"}, + {file = "coverage-7.2.1-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:e191a63a05851f8bce77bc875e75457f9b01d42843f8bd7feed2fc26bbe60833"}, + {file = "coverage-7.2.1-cp310-cp310-win32.whl", hash = "sha256:e3ea04b23b114572b98a88c85379e9e9ae031272ba1fb9b532aa934c621626d4"}, + {file = "coverage-7.2.1-cp310-cp310-win_amd64.whl", hash = "sha256:0cf557827be7eca1c38a2480484d706693e7bb1929e129785fe59ec155a59de6"}, + {file = "coverage-7.2.1-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:570c21a29493b350f591a4b04c158ce1601e8d18bdcd21db136fbb135d75efa6"}, + {file = "coverage-7.2.1-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:9e872b082b32065ac2834149dc0adc2a2e6d8203080501e1e3c3c77851b466f9"}, + {file = "coverage-7.2.1-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:fac6343bae03b176e9b58104a9810df3cdccd5cfed19f99adfa807ffbf43cf9b"}, + {file = "coverage-7.2.1-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:abacd0a738e71b20e224861bc87e819ef46fedba2fb01bc1af83dfd122e9c319"}, + {file = "coverage-7.2.1-cp311-cp311-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:d9256d4c60c4bbfec92721b51579c50f9e5062c21c12bec56b55292464873508"}, + {file = "coverage-7.2.1-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:80559eaf6c15ce3da10edb7977a1548b393db36cbc6cf417633eca05d84dd1ed"}, + {file = "coverage-7.2.1-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:0bd7e628f6c3ec4e7d2d24ec0e50aae4e5ae95ea644e849d92ae4805650b4c4e"}, + {file = "coverage-7.2.1-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:09643fb0df8e29f7417adc3f40aaf379d071ee8f0350ab290517c7004f05360b"}, + {file = "coverage-7.2.1-cp311-cp311-win32.whl", hash = "sha256:1b7fb13850ecb29b62a447ac3516c777b0e7a09ecb0f4bb6718a8654c87dfc80"}, + {file = "coverage-7.2.1-cp311-cp311-win_amd64.whl", hash = "sha256:617a94ada56bbfe547aa8d1b1a2b8299e2ec1ba14aac1d4b26a9f7d6158e1273"}, + {file = "coverage-7.2.1-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:8649371570551d2fd7dee22cfbf0b61f1747cdfb2b7587bb551e4beaaa44cb97"}, + {file = "coverage-7.2.1-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:5d2b9b5e70a21474c105a133ba227c61bc95f2ac3b66861143ce39a5ea4b3f84"}, + {file = "coverage-7.2.1-cp37-cp37m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:ae82c988954722fa07ec5045c57b6d55bc1a0890defb57cf4a712ced65b26ddd"}, + {file = "coverage-7.2.1-cp37-cp37m-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:861cc85dfbf55a7a768443d90a07e0ac5207704a9f97a8eb753292a7fcbdfcfc"}, + {file = "coverage-7.2.1-cp37-cp37m-musllinux_1_1_aarch64.whl", hash = "sha256:0339dc3237c0d31c3b574f19c57985fcbe494280153bbcad33f2cdf469f4ac3e"}, + {file = "coverage-7.2.1-cp37-cp37m-musllinux_1_1_i686.whl", hash = "sha256:5928b85416a388dd557ddc006425b0c37e8468bd1c3dc118c1a3de42f59e2a54"}, + {file = "coverage-7.2.1-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:8d3843ca645f62c426c3d272902b9de90558e9886f15ddf5efe757b12dd376f5"}, + {file = "coverage-7.2.1-cp37-cp37m-win32.whl", hash = "sha256:6a034480e9ebd4e83d1aa0453fd78986414b5d237aea89a8fdc35d330aa13bae"}, + {file = "coverage-7.2.1-cp37-cp37m-win_amd64.whl", hash = "sha256:6fce673f79a0e017a4dc35e18dc7bb90bf6d307c67a11ad5e61ca8d42b87cbff"}, + {file = "coverage-7.2.1-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:7f099da6958ddfa2ed84bddea7515cb248583292e16bb9231d151cd528eab657"}, + {file = "coverage-7.2.1-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:97a3189e019d27e914ecf5c5247ea9f13261d22c3bb0cfcfd2a9b179bb36f8b1"}, + {file = "coverage-7.2.1-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a81dbcf6c6c877986083d00b834ac1e84b375220207a059ad45d12f6e518a4e3"}, + {file = "coverage-7.2.1-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:78d2c3dde4c0b9be4b02067185136b7ee4681978228ad5ec1278fa74f5ca3e99"}, + {file = "coverage-7.2.1-cp38-cp38-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:3a209d512d157379cc9ab697cbdbb4cfd18daa3e7eebaa84c3d20b6af0037384"}, + {file = "coverage-7.2.1-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:f3d07edb912a978915576a776756069dede66d012baa503022d3a0adba1b6afa"}, + {file = "coverage-7.2.1-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:8dca3c1706670297851bca1acff9618455122246bdae623be31eca744ade05ec"}, + {file = "coverage-7.2.1-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:b1991a6d64231a3e5bbe3099fb0dd7c9aeaa4275ad0e0aeff4cb9ef885c62ba2"}, + {file = "coverage-7.2.1-cp38-cp38-win32.whl", hash = "sha256:22c308bc508372576ffa3d2dbc4824bb70d28eeb4fcd79d4d1aed663a06630d0"}, + {file = "coverage-7.2.1-cp38-cp38-win_amd64.whl", hash = "sha256:b0c0d46de5dd97f6c2d1b560bf0fcf0215658097b604f1840365296302a9d1fb"}, + {file = "coverage-7.2.1-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:4dd34a935de268a133e4741827ae951283a28c0125ddcdbcbba41c4b98f2dfef"}, + {file = "coverage-7.2.1-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:0f8318ed0f3c376cfad8d3520f496946977abde080439d6689d7799791457454"}, + {file = "coverage-7.2.1-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:834c2172edff5a08d78e2f53cf5e7164aacabeb66b369f76e7bb367ca4e2d993"}, + {file = "coverage-7.2.1-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:e4d70c853f0546855f027890b77854508bdb4d6a81242a9d804482e667fff6e6"}, + {file = "coverage-7.2.1-cp39-cp39-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:8a6450da4c7afc4534305b2b7d8650131e130610cea448ff240b6ab73d7eab63"}, + {file = "coverage-7.2.1-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:99f4dd81b2bb8fc67c3da68b1f5ee1650aca06faa585cbc6818dbf67893c6d58"}, + {file = "coverage-7.2.1-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:bdd3f2f285ddcf2e75174248b2406189261a79e7fedee2ceeadc76219b6faa0e"}, + {file = "coverage-7.2.1-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:f29351393eb05e6326f044a7b45ed8e38cb4dcc38570d12791f271399dc41431"}, + {file = "coverage-7.2.1-cp39-cp39-win32.whl", hash = "sha256:e2b50ebc2b6121edf352336d503357321b9d8738bb7a72d06fc56153fd3f4cd8"}, + {file = "coverage-7.2.1-cp39-cp39-win_amd64.whl", hash = "sha256:bd5a12239c0006252244f94863f1c518ac256160cd316ea5c47fb1a11b25889a"}, + {file = "coverage-7.2.1-pp37.pp38.pp39-none-any.whl", hash = "sha256:436313d129db7cf5b4ac355dd2bd3f7c7e5294af077b090b85de75f8458b8616"}, + {file = "coverage-7.2.1.tar.gz", hash = "sha256:c77f2a9093ccf329dd523a9b2b3c854c20d2a3d968b6def3b820272ca6732242"}, ] [package.dependencies] @@ -2553,4 +2553,4 @@ snappy = ["python-snappy"] [metadata] lock-version = "2.0" python-versions = "^3.8" -content-hash = "e466c6266c05f51de03bc7c767d733d9050289cafb191435e789e541e0e1047b" +content-hash = "324d67e5f9fc6fdc8f818d90380fd0c0d41b4d957726a40bb7002fef1ea9b1d2" diff --git a/pyproject.toml b/pyproject.toml index f0b10fa3da..a6bd85300a 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -83,7 +83,7 @@ pytest = "7.2.2" pytest-checkdocs = "2.9.0" pre-commit = "3.1.1" fastavro = "1.7.2" -coverage = { version = "^7.2.0", extras = ["toml"] } +coverage = { version = "^7.2.1", extras = ["toml"] } requests-mock = "1.10.0" moto = "^4.1.3" typing-extensions = '4.5.0' From 6a42a1ad26caf923932cfcdaf485bbfcb5631c13 Mon Sep 17 00:00:00 2001 From: Fokko Driesprong Date: Mon, 6 Mar 2023 08:47:13 +0100 Subject: [PATCH 398/642] Python: Don't warn on an identifier (#6844) When we pass in a warehouse identifier, we don't want to emit a warning. Also, the help message won't be helpful: ``` No preferred file implementation for scheme: ``` Since there is no scheme. --- pyiceberg/io/__init__.py | 22 ++++++++++++---------- tests/io/test_io.py | 9 +++++++++ 2 files changed, 21 insertions(+), 10 deletions(-) diff --git a/pyiceberg/io/__init__.py b/pyiceberg/io/__init__.py index 9690e53c67..f1287b4f4a 100644 --- a/pyiceberg/io/__init__.py +++ b/pyiceberg/io/__init__.py @@ -26,6 +26,7 @@ import importlib import logging +import warnings from abc import ABC, abstractmethod from io import SEEK_SET from types import TracebackType @@ -275,21 +276,22 @@ def _import_file_io(io_impl: str, properties: Properties) -> Optional[FileIO]: class_ = getattr(module, class_name) return class_(properties) except ModuleNotFoundError: - logger.warning("Could not initialize FileIO: %s", io_impl) + warnings.warn(f"Could not initialize FileIO: {io_impl}") return None PY_IO_IMPL = "py-io-impl" -def _infer_file_io_from_schema(path: str, properties: Properties) -> Optional[FileIO]: +def _infer_file_io_from_scheme(path: str, properties: Properties) -> Optional[FileIO]: parsed_url = urlparse(path) - if file_ios := SCHEMA_TO_FILE_IO.get(parsed_url.scheme): - for file_io_path in file_ios: - if file_io := _import_file_io(file_io_path, properties): - return file_io - else: - logger.warning("No preferred file implementation for schema: %s", parsed_url.scheme) + if parsed_url.scheme: + if file_ios := SCHEMA_TO_FILE_IO.get(parsed_url.scheme): + for file_io_path in file_ios: + if file_io := _import_file_io(file_io_path, properties): + return file_io + else: + warnings.warn(f"No preferred file implementation for scheme: {parsed_url.scheme}") return None @@ -304,12 +306,12 @@ def load_file_io(properties: Properties = EMPTY_DICT, location: Optional[str] = # Check the table location if location: - if file_io := _infer_file_io_from_schema(location, properties): + if file_io := _infer_file_io_from_scheme(location, properties): return file_io # Look at the schema of the warehouse if warehouse_location := properties.get(WAREHOUSE): - if file_io := _infer_file_io_from_schema(warehouse_location, properties): + if file_io := _infer_file_io_from_scheme(warehouse_location, properties): return file_io try: diff --git a/tests/io/test_io.py b/tests/io/test_io.py index c872ae1c7c..c4dc3d45a5 100644 --- a/tests/io/test_io.py +++ b/tests/io/test_io.py @@ -24,6 +24,7 @@ ARROW_FILE_IO, PY_IO_IMPL, _import_file_io, + _infer_file_io_from_scheme, load_file_io, ) from pyiceberg.io.pyarrow import PyArrowFileIO @@ -302,3 +303,11 @@ def test_mock_table_location_file_io() -> None: def test_gibberish_table_location_file_io() -> None: # For testing the selection logic assert isinstance(load_file_io({}, "gibberish"), PyArrowFileIO) + + +def test_infer_file_io_from_schema_unknown() -> None: + # When we have an unknown scheme, we would like to know + with pytest.warns(UserWarning) as w: + _infer_file_io_from_scheme("unknown://bucket/path/", {}) + + assert str(w[0].message) == "No preferred file implementation for scheme: unknown" From c686682e7c354453d249f6963f1fa507c05c1860 Mon Sep 17 00:00:00 2001 From: Fokko Driesprong Date: Tue, 7 Mar 2023 19:43:26 +0100 Subject: [PATCH 399/642] Python: Allow setting the AWS Region (#6847) --- mkdocs/docs/configuration.md | 13 +++++++------ pyiceberg/io/__init__.py | 6 ++++++ pyiceberg/io/fsspec.py | 14 ++++++++++---- pyiceberg/io/pyarrow.py | 14 ++++++++++---- 4 files changed, 33 insertions(+), 14 deletions(-) diff --git a/mkdocs/docs/configuration.md b/mkdocs/docs/configuration.md index ee13133653..1ea034ed2d 100644 --- a/mkdocs/docs/configuration.md +++ b/mkdocs/docs/configuration.md @@ -55,12 +55,13 @@ For the FileIO there are several configuration options available: ### S3 -| Key | Example | Description | -|--------------------------|---------------------|-----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| -| s3.endpoint | https://10.0.19.25/ | Configure an alternative endpoint of the S3 service for the FileIO to access. This could be used to use S3FileIO with any s3-compatible object storage service that has a different endpoint, or access a private S3 endpoint in a virtual private cloud. | -| s3.access-key-id | admin | Configure the static secret access key used to access the FileIO. | -| s3.secret-access-key | password | Configure the static session token used to access the FileIO. | -| s3.signer | bearer | Configure the signature version of the FileIO. | +| Key | Example | Description | +|----------------------|---------------------|-----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| +| s3.endpoint | https://10.0.19.25/ | Configure an alternative endpoint of the S3 service for the FileIO to access. This could be used to use S3FileIO with any s3-compatible object storage service that has a different endpoint, or access a private S3 endpoint in a virtual private cloud. | +| s3.access-key-id | admin | Configure the static secret access key used to access the FileIO. | +| s3.secret-access-key | password | Configure the static session token used to access the FileIO. | +| s3.signer | bearer | Configure the signature version of the FileIO. | +| s3.region | us-west-2 | Sets the region of the bucket | ### Azure Data lake diff --git a/pyiceberg/io/__init__.py b/pyiceberg/io/__init__.py index f1287b4f4a..b48c0186ac 100644 --- a/pyiceberg/io/__init__.py +++ b/pyiceberg/io/__init__.py @@ -45,6 +45,12 @@ logger = logging.getLogger(__name__) +S3_ENDPOINT = "s3.endpoint" +S3_ACCESS_KEY_ID = "s3.access-key-id" +S3_SECRET_ACCESS_KEY = "s3.secret-access-key" +S3_SESSION_TOKEN = "s3.session-token" +S3_REGION = "s3.region" + @runtime_checkable class InputStream(Protocol): diff --git a/pyiceberg/io/fsspec.py b/pyiceberg/io/fsspec.py index 5c7e5cb873..35d7d5e6b4 100644 --- a/pyiceberg/io/fsspec.py +++ b/pyiceberg/io/fsspec.py @@ -37,6 +37,11 @@ from pyiceberg.catalog import TOKEN from pyiceberg.exceptions import SignError from pyiceberg.io import ( + S3_ACCESS_KEY_ID, + S3_ENDPOINT, + S3_REGION, + S3_SECRET_ACCESS_KEY, + S3_SESSION_TOKEN, FileIO, InputFile, InputStream, @@ -87,10 +92,11 @@ def _s3(properties: Properties) -> AbstractFileSystem: from s3fs import S3FileSystem client_kwargs = { - "endpoint_url": properties.get("s3.endpoint"), - "aws_access_key_id": properties.get("s3.access-key-id"), - "aws_secret_access_key": properties.get("s3.secret-access-key"), - "aws_session_token": properties.get("s3.session-token"), + "endpoint_url": properties.get(S3_ENDPOINT), + "aws_access_key_id": properties.get(S3_ACCESS_KEY_ID), + "aws_secret_access_key": properties.get(S3_SECRET_ACCESS_KEY), + "aws_session_token": properties.get(S3_SESSION_TOKEN), + "region_name": properties.get(S3_REGION), } config_kwargs = {} register_events: Dict[str, Callable[[Properties], None]] = {} diff --git a/pyiceberg/io/pyarrow.py b/pyiceberg/io/pyarrow.py index 4cf81dadd8..37bf298b3e 100644 --- a/pyiceberg/io/pyarrow.py +++ b/pyiceberg/io/pyarrow.py @@ -68,6 +68,11 @@ ) from pyiceberg.expressions.visitors import visit as boolean_expression_visit from pyiceberg.io import ( + S3_ACCESS_KEY_ID, + S3_ENDPOINT, + S3_REGION, + S3_SECRET_ACCESS_KEY, + S3_SESSION_TOKEN, FileIO, InputFile, InputStream, @@ -266,10 +271,11 @@ def parse_location(location: str) -> Tuple[str, str]: def _get_fs(self, scheme: str) -> FileSystem: if scheme in {"s3", "s3a", "s3n"}: client_kwargs = { - "endpoint_override": self.properties.get("s3.endpoint"), - "access_key": self.properties.get("s3.access-key-id"), - "secret_key": self.properties.get("s3.secret-access-key"), - "session_token": self.properties.get("s3.session-token"), + "endpoint_override": self.properties.get(S3_ENDPOINT), + "access_key": self.properties.get(S3_ACCESS_KEY_ID), + "secret_key": self.properties.get(S3_SECRET_ACCESS_KEY), + "session_token": self.properties.get(S3_SESSION_TOKEN), + "region": self.properties.get(S3_REGION), } return S3FileSystem(**client_kwargs) elif scheme == "file": From 572921f7dea3c71fd176a9dafd6d99817cbcd100 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Sun, 12 Mar 2023 18:01:41 +0100 Subject: [PATCH 400/642] Build: Bump moto from 4.1.3 to 4.1.4 in /python (#7082) Bumps [moto](https://github.com/getmoto/moto) from 4.1.3 to 4.1.4. - [Release notes](https://github.com/getmoto/moto/releases) - [Changelog](https://github.com/getmoto/moto/blob/master/CHANGELOG.md) - [Commits](https://github.com/getmoto/moto/compare/4.1.3...4.1.4) --- updated-dependencies: - dependency-name: moto dependency-type: direct:development update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- poetry.lock | 8 ++++---- pyproject.toml | 2 +- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/poetry.lock b/poetry.lock index 10820e3ac3..e604497c6d 100644 --- a/poetry.lock +++ b/poetry.lock @@ -1224,14 +1224,14 @@ files = [ [[package]] name = "moto" -version = "4.1.3" +version = "4.1.4" description = "" category = "dev" optional = false python-versions = ">=3.7" files = [ - {file = "moto-4.1.3-py2.py3-none-any.whl", hash = "sha256:dcd1d06662982cf3c94f36d6348251ccdcf62a1c5de5650425cb4e6f260ae7a0"}, - {file = "moto-4.1.3.tar.gz", hash = "sha256:c8200ccaa9440c2e9daa0bd5e0bd768a719db5a2c82ea8d782f0e3fa09a3c5e2"}, + {file = "moto-4.1.4-py2.py3-none-any.whl", hash = "sha256:f9bf72aec6aea49ebb1a46c8096b2c9629be58ecca3ecd8b51f781fea78148e2"}, + {file = "moto-4.1.4.tar.gz", hash = "sha256:304cb19eee0019cd13f7d87ca590a4694677469c383ab8f8439fcb6717c47037"}, ] [package.dependencies] @@ -2553,4 +2553,4 @@ snappy = ["python-snappy"] [metadata] lock-version = "2.0" python-versions = "^3.8" -content-hash = "324d67e5f9fc6fdc8f818d90380fd0c0d41b4d957726a40bb7002fef1ea9b1d2" +content-hash = "d7acc6a2ba42e36170ca09152223e8fce0c394929189fa39da271b058e4314d7" diff --git a/pyproject.toml b/pyproject.toml index a6bd85300a..6cdcbbad84 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -85,7 +85,7 @@ pre-commit = "3.1.1" fastavro = "1.7.2" coverage = { version = "^7.2.1", extras = ["toml"] } requests-mock = "1.10.0" -moto = "^4.1.3" +moto = "^4.1.4" typing-extensions = '4.5.0' [tool.poetry.scripts] From 133a9ef012913da7395e5282014ce690a96ffcf2 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Sun, 12 Mar 2023 19:31:32 +0100 Subject: [PATCH 401/642] Build: Bump pydantic from 1.10.5 to 1.10.6 in /python (#7083) Bumps [pydantic](https://github.com/pydantic/pydantic) from 1.10.5 to 1.10.6. - [Release notes](https://github.com/pydantic/pydantic/releases) - [Changelog](https://github.com/pydantic/pydantic/blob/v1.10.6/HISTORY.md) - [Commits](https://github.com/pydantic/pydantic/compare/v1.10.5...v1.10.6) --- updated-dependencies: - dependency-name: pydantic dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- poetry.lock | 76 +++++++++++++++++++++++++------------------------- pyproject.toml | 2 +- 2 files changed, 39 insertions(+), 39 deletions(-) diff --git a/poetry.lock b/poetry.lock index e604497c6d..ddb9dd0dc1 100644 --- a/poetry.lock +++ b/poetry.lock @@ -1668,48 +1668,48 @@ files = [ [[package]] name = "pydantic" -version = "1.10.5" +version = "1.10.6" description = "Data validation and settings management using python type hints" category = "main" optional = false python-versions = ">=3.7" files = [ - {file = "pydantic-1.10.5-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:5920824fe1e21cbb3e38cf0f3dd24857c8959801d1031ce1fac1d50857a03bfb"}, - {file = "pydantic-1.10.5-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:3bb99cf9655b377db1a9e47fa4479e3330ea96f4123c6c8200e482704bf1eda2"}, - {file = "pydantic-1.10.5-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:2185a3b3d98ab4506a3f6707569802d2d92c3a7ba3a9a35683a7709ea6c2aaa2"}, - {file = "pydantic-1.10.5-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:f582cac9d11c227c652d3ce8ee223d94eb06f4228b52a8adaafa9fa62e73d5c9"}, - {file = "pydantic-1.10.5-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:c9e5b778b6842f135902e2d82624008c6a79710207e28e86966cd136c621bfee"}, - {file = "pydantic-1.10.5-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:72ef3783be8cbdef6bca034606a5de3862be6b72415dc5cb1fb8ddbac110049a"}, - {file = "pydantic-1.10.5-cp310-cp310-win_amd64.whl", hash = "sha256:45edea10b75d3da43cfda12f3792833a3fa70b6eee4db1ed6aed528cef17c74e"}, - {file = "pydantic-1.10.5-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:63200cd8af1af2c07964546b7bc8f217e8bda9d0a2ef0ee0c797b36353914984"}, - {file = "pydantic-1.10.5-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:305d0376c516b0dfa1dbefeae8c21042b57b496892d721905a6ec6b79494a66d"}, - {file = "pydantic-1.10.5-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:1fd326aff5d6c36f05735c7c9b3d5b0e933b4ca52ad0b6e4b38038d82703d35b"}, - {file = "pydantic-1.10.5-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:6bb0452d7b8516178c969d305d9630a3c9b8cf16fcf4713261c9ebd465af0d73"}, - {file = "pydantic-1.10.5-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:9a9d9155e2a9f38b2eb9374c88f02fd4d6851ae17b65ee786a87d032f87008f8"}, - {file = "pydantic-1.10.5-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:f836444b4c5ece128b23ec36a446c9ab7f9b0f7981d0d27e13a7c366ee163f8a"}, - {file = "pydantic-1.10.5-cp311-cp311-win_amd64.whl", hash = "sha256:8481dca324e1c7b715ce091a698b181054d22072e848b6fc7895cd86f79b4449"}, - {file = "pydantic-1.10.5-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:87f831e81ea0589cd18257f84386bf30154c5f4bed373b7b75e5cb0b5d53ea87"}, - {file = "pydantic-1.10.5-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:7ce1612e98c6326f10888df951a26ec1a577d8df49ddcaea87773bfbe23ba5cc"}, - {file = "pydantic-1.10.5-cp37-cp37m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:58e41dd1e977531ac6073b11baac8c013f3cd8706a01d3dc74e86955be8b2c0c"}, - {file = "pydantic-1.10.5-cp37-cp37m-musllinux_1_1_i686.whl", hash = "sha256:6a4b0aab29061262065bbdede617ef99cc5914d1bf0ddc8bcd8e3d7928d85bd6"}, - {file = "pydantic-1.10.5-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:36e44a4de37b8aecffa81c081dbfe42c4d2bf9f6dff34d03dce157ec65eb0f15"}, - {file = "pydantic-1.10.5-cp37-cp37m-win_amd64.whl", hash = "sha256:261f357f0aecda005934e413dfd7aa4077004a174dafe414a8325e6098a8e419"}, - {file = "pydantic-1.10.5-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:b429f7c457aebb7fbe7cd69c418d1cd7c6fdc4d3c8697f45af78b8d5a7955760"}, - {file = "pydantic-1.10.5-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:663d2dd78596c5fa3eb996bc3f34b8c2a592648ad10008f98d1348be7ae212fb"}, - {file = "pydantic-1.10.5-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:51782fd81f09edcf265823c3bf43ff36d00db246eca39ee765ef58dc8421a642"}, - {file = "pydantic-1.10.5-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:c428c0f64a86661fb4873495c4fac430ec7a7cef2b8c1c28f3d1a7277f9ea5ab"}, - {file = "pydantic-1.10.5-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:76c930ad0746c70f0368c4596020b736ab65b473c1f9b3872310a835d852eb19"}, - {file = "pydantic-1.10.5-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:3257bd714de9db2102b742570a56bf7978e90441193acac109b1f500290f5718"}, - {file = "pydantic-1.10.5-cp38-cp38-win_amd64.whl", hash = "sha256:f5bee6c523d13944a1fdc6f0525bc86dbbd94372f17b83fa6331aabacc8fd08e"}, - {file = "pydantic-1.10.5-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:532e97c35719f137ee5405bd3eeddc5c06eb91a032bc755a44e34a712420daf3"}, - {file = "pydantic-1.10.5-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:ca9075ab3de9e48b75fa8ccb897c34ccc1519177ad8841d99f7fd74cf43be5bf"}, - {file = "pydantic-1.10.5-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:bd46a0e6296346c477e59a954da57beaf9c538da37b9df482e50f836e4a7d4bb"}, - {file = "pydantic-1.10.5-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:3353072625ea2a9a6c81ad01b91e5c07fa70deb06368c71307529abf70d23325"}, - {file = "pydantic-1.10.5-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:3f9d9b2be177c3cb6027cd67fbf323586417868c06c3c85d0d101703136e6b31"}, - {file = "pydantic-1.10.5-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:b473d00ccd5c2061fd896ac127b7755baad233f8d996ea288af14ae09f8e0d1e"}, - {file = "pydantic-1.10.5-cp39-cp39-win_amd64.whl", hash = "sha256:5f3bc8f103b56a8c88021d481410874b1f13edf6e838da607dcb57ecff9b4594"}, - {file = "pydantic-1.10.5-py3-none-any.whl", hash = "sha256:7c5b94d598c90f2f46b3a983ffb46ab806a67099d118ae0da7ef21a2a4033b28"}, - {file = "pydantic-1.10.5.tar.gz", hash = "sha256:9e337ac83686645a46db0e825acceea8e02fca4062483f40e9ae178e8bd1103a"}, + {file = "pydantic-1.10.6-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:f9289065611c48147c1dd1fd344e9d57ab45f1d99b0fb26c51f1cf72cd9bcd31"}, + {file = "pydantic-1.10.6-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:8c32b6bba301490d9bb2bf5f631907803135e8085b6aa3e5fe5a770d46dd0160"}, + {file = "pydantic-1.10.6-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:fd9b9e98068fa1068edfc9eabde70a7132017bdd4f362f8b4fd0abed79c33083"}, + {file = "pydantic-1.10.6-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:1c84583b9df62522829cbc46e2b22e0ec11445625b5acd70c5681ce09c9b11c4"}, + {file = "pydantic-1.10.6-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:b41822064585fea56d0116aa431fbd5137ce69dfe837b599e310034171996084"}, + {file = "pydantic-1.10.6-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:61f1f08adfaa9cc02e0cbc94f478140385cbd52d5b3c5a657c2fceb15de8d1fb"}, + {file = "pydantic-1.10.6-cp310-cp310-win_amd64.whl", hash = "sha256:32937835e525d92c98a1512218db4eed9ddc8f4ee2a78382d77f54341972c0e7"}, + {file = "pydantic-1.10.6-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:bbd5c531b22928e63d0cb1868dee76123456e1de2f1cb45879e9e7a3f3f1779b"}, + {file = "pydantic-1.10.6-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:e277bd18339177daa62a294256869bbe84df1fb592be2716ec62627bb8d7c81d"}, + {file = "pydantic-1.10.6-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:89f15277d720aa57e173954d237628a8d304896364b9de745dcb722f584812c7"}, + {file = "pydantic-1.10.6-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:b243b564cea2576725e77aeeda54e3e0229a168bc587d536cd69941e6797543d"}, + {file = "pydantic-1.10.6-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:3ce13a558b484c9ae48a6a7c184b1ba0e5588c5525482681db418268e5f86186"}, + {file = "pydantic-1.10.6-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:3ac1cd4deed871dfe0c5f63721e29debf03e2deefa41b3ed5eb5f5df287c7b70"}, + {file = "pydantic-1.10.6-cp311-cp311-win_amd64.whl", hash = "sha256:b1eb6610330a1dfba9ce142ada792f26bbef1255b75f538196a39e9e90388bf4"}, + {file = "pydantic-1.10.6-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:4ca83739c1263a044ec8b79df4eefc34bbac87191f0a513d00dd47d46e307a65"}, + {file = "pydantic-1.10.6-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:ea4e2a7cb409951988e79a469f609bba998a576e6d7b9791ae5d1e0619e1c0f2"}, + {file = "pydantic-1.10.6-cp37-cp37m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:53de12b4608290992a943801d7756f18a37b7aee284b9ffa794ee8ea8153f8e2"}, + {file = "pydantic-1.10.6-cp37-cp37m-musllinux_1_1_i686.whl", hash = "sha256:60184e80aac3b56933c71c48d6181e630b0fbc61ae455a63322a66a23c14731a"}, + {file = "pydantic-1.10.6-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:415a3f719ce518e95a92effc7ee30118a25c3d032455d13e121e3840985f2efd"}, + {file = "pydantic-1.10.6-cp37-cp37m-win_amd64.whl", hash = "sha256:72cb30894a34d3a7ab6d959b45a70abac8a2a93b6480fc5a7bfbd9c935bdc4fb"}, + {file = "pydantic-1.10.6-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:3091d2eaeda25391405e36c2fc2ed102b48bac4b384d42b2267310abae350ca6"}, + {file = "pydantic-1.10.6-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:751f008cd2afe812a781fd6aa2fb66c620ca2e1a13b6a2152b1ad51553cb4b77"}, + {file = "pydantic-1.10.6-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:12e837fd320dd30bd625be1b101e3b62edc096a49835392dcf418f1a5ac2b832"}, + {file = "pydantic-1.10.6-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:587d92831d0115874d766b1f5fddcdde0c5b6c60f8c6111a394078ec227fca6d"}, + {file = "pydantic-1.10.6-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:476f6674303ae7965730a382a8e8d7fae18b8004b7b69a56c3d8fa93968aa21c"}, + {file = "pydantic-1.10.6-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:3a2be0a0f32c83265fd71a45027201e1278beaa82ea88ea5b345eea6afa9ac7f"}, + {file = "pydantic-1.10.6-cp38-cp38-win_amd64.whl", hash = "sha256:0abd9c60eee6201b853b6c4be104edfba4f8f6c5f3623f8e1dba90634d63eb35"}, + {file = "pydantic-1.10.6-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:6195ca908045054dd2d57eb9c39a5fe86409968b8040de8c2240186da0769da7"}, + {file = "pydantic-1.10.6-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:43cdeca8d30de9a897440e3fb8866f827c4c31f6c73838e3a01a14b03b067b1d"}, + {file = "pydantic-1.10.6-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:4c19eb5163167489cb1e0161ae9220dadd4fc609a42649e7e84a8fa8fff7a80f"}, + {file = "pydantic-1.10.6-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:012c99a9c0d18cfde7469aa1ebff922e24b0c706d03ead96940f5465f2c9cf62"}, + {file = "pydantic-1.10.6-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:528dcf7ec49fb5a84bf6fe346c1cc3c55b0e7603c2123881996ca3ad79db5bfc"}, + {file = "pydantic-1.10.6-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:163e79386c3547c49366e959d01e37fc30252285a70619ffc1b10ede4758250a"}, + {file = "pydantic-1.10.6-cp39-cp39-win_amd64.whl", hash = "sha256:189318051c3d57821f7233ecc94708767dd67687a614a4e8f92b4a020d4ffd06"}, + {file = "pydantic-1.10.6-py3-none-any.whl", hash = "sha256:acc6783751ac9c9bc4680379edd6d286468a1dc8d7d9906cd6f1186ed682b2b0"}, + {file = "pydantic-1.10.6.tar.gz", hash = "sha256:cf95adb0d1671fc38d8c43dd921ad5814a735e7d9b4d9e437c088002863854fd"}, ] [package.dependencies] @@ -2553,4 +2553,4 @@ snappy = ["python-snappy"] [metadata] lock-version = "2.0" python-versions = "^3.8" -content-hash = "d7acc6a2ba42e36170ca09152223e8fce0c394929189fa39da271b058e4314d7" +content-hash = "c32275bee42410ea29cb95758abc87b5bd040eb6f672415f77cdba3fc97b343c" diff --git a/pyproject.toml b/pyproject.toml index 6cdcbbad84..efc0ebd215 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -55,7 +55,7 @@ click = "8.1.3" rich = ">=13.0.0,<=13.3.2" pyyaml = ">=5.4.0,<=6.0.0" -pydantic = "1.10.5" +pydantic = "1.10.6" fsspec = ">=2022.8.2,<=2023.1.0" pyparsing = "3.0.9" From 8918bb8b1388719ba2a6150a394bb58289fd8ebf Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Sun, 12 Mar 2023 20:50:56 +0100 Subject: [PATCH 402/642] Build: Bump fastavro from 1.7.2 to 1.7.3 in /python (#7084) Bumps [fastavro](https://github.com/fastavro/fastavro) from 1.7.2 to 1.7.3. - [Release notes](https://github.com/fastavro/fastavro/releases) - [Changelog](https://github.com/fastavro/fastavro/blob/master/ChangeLog) - [Commits](https://github.com/fastavro/fastavro/compare/1.7.2...1.7.3) --- updated-dependencies: - dependency-name: fastavro dependency-type: direct:development update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- poetry.lock | 56 +++++++++++++++++++++++++++++--------------------- pyproject.toml | 2 +- 2 files changed, 34 insertions(+), 24 deletions(-) diff --git a/poetry.lock b/poetry.lock index ddb9dd0dc1..bf90680172 100644 --- a/poetry.lock +++ b/poetry.lock @@ -807,33 +807,43 @@ test = ["pytest (>=6)"] [[package]] name = "fastavro" -version = "1.7.2" +version = "1.7.3" description = "Fast read/write of AVRO files" category = "dev" optional = false python-versions = ">=3.7" files = [ - {file = "fastavro-1.7.2-cp310-cp310-macosx_11_0_x86_64.whl", hash = "sha256:20a94269bc0910c41f06106424429289e5c8cdce65b006e48d600c9bb6c442a5"}, - {file = "fastavro-1.7.2-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:b34e9cd755e4d9d12377fe151ad95b9bff08215f0a3756601b21ed31f4d127ab"}, - {file = "fastavro-1.7.2-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:8993e2f34e9ae195f417105d495fdd3d7f6d1532bacf57dc5dea600082f9b1a0"}, - {file = "fastavro-1.7.2-cp310-cp310-win_amd64.whl", hash = "sha256:e83069f971f307175ab806e9929a15826e586ef05a620fcfc7768ceaa3ec19c8"}, - {file = "fastavro-1.7.2-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:4c44f72a1a7b2d28cc21c95686916b782354dcc1f1aa1c8ca044c66d5ada3e14"}, - {file = "fastavro-1.7.2-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:88005d5c6fe436e80f756399a52aaa6d0cdc506c1ee195277287a09c7617aec3"}, - {file = "fastavro-1.7.2-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:a5c5d466631f455213b8bfd54888d2d8691cfc240cd6dbb40eb3694bf05e562e"}, - {file = "fastavro-1.7.2-cp311-cp311-win_amd64.whl", hash = "sha256:4c2523c5e2279debb5fa788bf665a5abbd16a1e889338b973e3b318d50fcce77"}, - {file = "fastavro-1.7.2-cp37-cp37m-macosx_10_15_x86_64.whl", hash = "sha256:db3d9b6237934be5001fc18c07d25c09d3008ce6ab3448daf4b30608b84488e0"}, - {file = "fastavro-1.7.2-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:e82862cb7d3bf99fdca65048b8b09db295795d7fb1eb7c5dae29c0a4d7dcf1f2"}, - {file = "fastavro-1.7.2-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:5417dfbf99ac5bf963a929f2541c643d35e1a5a22876ba5ae0dca7b86326d9c2"}, - {file = "fastavro-1.7.2-cp37-cp37m-win_amd64.whl", hash = "sha256:b8cd8bd9a8ad4b687bd8dbf12effc2996bdf41ec04960875c19a7a90c949e8a8"}, - {file = "fastavro-1.7.2-cp38-cp38-macosx_10_15_x86_64.whl", hash = "sha256:f07d1f2ebdcc18938908e710a79ab3731061936fbea27534e436f9e12342450b"}, - {file = "fastavro-1.7.2-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:9919d866319b1e123374fced84afb4728802872bd5938ba7e12a691c54593542"}, - {file = "fastavro-1.7.2-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:8da1a9d4f0b871e7c968c8413caef19e19002100036b2714e037543c36550b8d"}, - {file = "fastavro-1.7.2-cp38-cp38-win_amd64.whl", hash = "sha256:e842beff23b0b207c45f99fa79d50cfc0d84ac137d84a7e283e9e59a8368b115"}, - {file = "fastavro-1.7.2-cp39-cp39-macosx_11_0_x86_64.whl", hash = "sha256:c1d5805c50a9507c99b814cb014da2812e6e0ee2770f4b793ce56b82a5b0af21"}, - {file = "fastavro-1.7.2-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:22cb5d2bf87c1ed9a1dcc97b6b13dc6b7ee4395dbf1fd5903dac01cdde0291c2"}, - {file = "fastavro-1.7.2-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:06bd51962df9b3f28c7081c0a3c8663d840df05d964b7817d85bd37c1d101236"}, - {file = "fastavro-1.7.2-cp39-cp39-win_amd64.whl", hash = "sha256:d787e7739d93db59f378c0e44a4eca8e56ff979e5b43dea6378e87c48548818c"}, - {file = "fastavro-1.7.2.tar.gz", hash = "sha256:3c524c65a5d578c020997800fce992cd90cc88ef13148b70f01a3429de34c8a4"}, + {file = "fastavro-1.7.3-cp310-cp310-macosx_11_0_x86_64.whl", hash = "sha256:10a5ac9d8c66d4ba24f25ad7313e2dab56d98ceebcf53ba9cfa88acdd135c794"}, + {file = "fastavro-1.7.3-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:b2e6d8bb79e53dc39e620c777f14b5f7122f1bf21309a9fcf60085f8e062e49c"}, + {file = "fastavro-1.7.3-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:9a0ba2f43844eb784f8abf5324a0c10474287beaecb14fb736e47136464e3044"}, + {file = "fastavro-1.7.3-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:e20db96b092d7b6208f3063a424d35bb48c283e2d8b4e7ad4ee6541dc1fac2ed"}, + {file = "fastavro-1.7.3-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:52ba6bb5525561df577ebd94819784626caac9d8ad2ed167030403ba1bf73159"}, + {file = "fastavro-1.7.3-cp310-cp310-win_amd64.whl", hash = "sha256:22d6f3e73f471e2b4ba0785cb60df939792e8904db4ba93037ba6b7858f7d6f9"}, + {file = "fastavro-1.7.3-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:5dd5299cbc5bc2aa15f1c619f4cc55c054c6fe9ccd614f93eb1d6ab22cf314dd"}, + {file = "fastavro-1.7.3-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:f4706a77038bf31ad2e8cc752a0c007894bd39ffb0b775c7824113743182c5f6"}, + {file = "fastavro-1.7.3-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:b6b0e58e7dd34906d21738c3461cddef760de3b7845779169a378b2757afa693"}, + {file = "fastavro-1.7.3-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:04740e2dd27084b4155337d082f2a232cf1d801a1b009f772e50c8306a8f8aaf"}, + {file = "fastavro-1.7.3-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:2f95c767bf97e896640f24d58931b3a19df3d84ccaf0606c92e603c79de60f16"}, + {file = "fastavro-1.7.3-cp311-cp311-win_amd64.whl", hash = "sha256:a727e07007230267e25702d5f3738854cb315747fc58b84839699db30dedf490"}, + {file = "fastavro-1.7.3-cp37-cp37m-macosx_10_15_x86_64.whl", hash = "sha256:da71b9db7718f4682cc11e0f25b5e395d5f3bc17ddaf0224f39be3bac5309cfa"}, + {file = "fastavro-1.7.3-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:d1a62c359f4c9472c3ebe2be478e203ff434cc1d6bebaf61181a4a121c0899a6"}, + {file = "fastavro-1.7.3-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:9e2ba001864607b46fc2f6124d690731b19db215a84751c4b3b155e70b615d05"}, + {file = "fastavro-1.7.3-cp37-cp37m-musllinux_1_1_aarch64.whl", hash = "sha256:cd10bffc26457402da9727663de71c40dd717d90e8ab3d3b893bc227cad5e410"}, + {file = "fastavro-1.7.3-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:e5d0f16b85104aa0e2899a47c186be1082a10cecf6b331571afa92a4b8e6061a"}, + {file = "fastavro-1.7.3-cp37-cp37m-win_amd64.whl", hash = "sha256:b86d1c1188ec47aeb76d6195e36ab52665984e8e98f69a224ab550c82991fe07"}, + {file = "fastavro-1.7.3-cp38-cp38-macosx_10_15_x86_64.whl", hash = "sha256:a007151cc2a08e61dd5ea5b48989849d056a8d63b04d7e6799c36fdf0b702bf4"}, + {file = "fastavro-1.7.3-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:7b176d5731f336c2c9c88d95225f71f862b2512c33ef917b1fe7f87379cc92fd"}, + {file = "fastavro-1.7.3-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:c52e4a1b05306f82916eacf83c732a4637a5be748bc2ef2ff6fed1506535d692"}, + {file = "fastavro-1.7.3-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:207bc7663133ca766eaf9033806da4cf08071dacf2e9779aa9427df40815f846"}, + {file = "fastavro-1.7.3-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:cc98be4ad3d8fb9000abeeae0ecb0f8e62ec7898b791da5ec2f6de81dd2a73e2"}, + {file = "fastavro-1.7.3-cp38-cp38-win_amd64.whl", hash = "sha256:fb3879aaeb3b56ee5b3a22ffa11cbdf4ba65c04be4688ee8bd152aa6535a00ee"}, + {file = "fastavro-1.7.3-cp39-cp39-macosx_11_0_x86_64.whl", hash = "sha256:8c45f7fdfab351431d106f5981fdc2313a8cbfdb82d2b1172b2a144bfba376b7"}, + {file = "fastavro-1.7.3-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:127e928604753d845fa0f2ae758c1640215ff901a5ce20cdf7e9f154500c3212"}, + {file = "fastavro-1.7.3-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:b82a71a6c014ec5d03293d8dc8b698220380266d5503779fd3712a94e4497069"}, + {file = "fastavro-1.7.3-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:08bfd749cce456f925203895d6732f6b68c973d63ff886733f27db3c2d3c0b9a"}, + {file = "fastavro-1.7.3-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:dc1e88d5db17e7ebc3fc764a1091f6c05a42e3cb0e2c8eaf49126743c7ca1bb5"}, + {file = "fastavro-1.7.3-cp39-cp39-win_amd64.whl", hash = "sha256:7d525f3f99cc49a5e245e08d7ab947195a18cbdd5c43af75c0989fbe14a32597"}, + {file = "fastavro-1.7.3.tar.gz", hash = "sha256:8b08bd3cba45830b64adda32ccc5b027a71b6941a99cc39f90d7019a7986cc19"}, ] [package.extras] @@ -2553,4 +2563,4 @@ snappy = ["python-snappy"] [metadata] lock-version = "2.0" python-versions = "^3.8" -content-hash = "c32275bee42410ea29cb95758abc87b5bd040eb6f672415f77cdba3fc97b343c" +content-hash = "dc2372a1fa549a351cbb80296b8364377f5977896daccaf6866392696f524d3c" diff --git a/pyproject.toml b/pyproject.toml index efc0ebd215..6746f8a7de 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -82,7 +82,7 @@ adlfs = { version = ">=2022.8.2,<=2023.1.0", optional = true } pytest = "7.2.2" pytest-checkdocs = "2.9.0" pre-commit = "3.1.1" -fastavro = "1.7.2" +fastavro = "1.7.3" coverage = { version = "^7.2.1", extras = ["toml"] } requests-mock = "1.10.0" moto = "^4.1.4" From 3d78ecbd3c60e75411818d7dbb578dd2d7cc3883 Mon Sep 17 00:00:00 2001 From: Fokko Driesprong Date: Wed, 15 Mar 2023 20:04:53 +0100 Subject: [PATCH 403/642] Python: Integration tests (#6398) * Integration tests * First version * Add caching * Add caching * Restore pyproject * WIP * NaN seems to be broken * WIP * Coming along * Cleanup * Install duckdb * Cleanup * Revert changes to poetry * Make it even nicer * Revert unneeded change * Update Spark version * Make test passing * comments --- Makefile | 21 ++++--- dev/Dockerfile | 67 ++++++++++++++++++++ dev/docker-compose-integration.yml | 76 +++++++++++++++++++++++ dev/entrypoint.sh | 25 ++++++++ dev/provision.py | 98 ++++++++++++++++++++++++++++++ dev/spark-defaults.conf | 29 +++++++++ pyiceberg/io/pyarrow.py | 4 +- pyiceberg/table/__init__.py | 8 +-- pyproject.toml | 7 ++- tests/test_integration.py | 82 +++++++++++++++++++++++++ 10 files changed, 400 insertions(+), 17 deletions(-) create mode 100644 dev/Dockerfile create mode 100644 dev/docker-compose-integration.yml create mode 100755 dev/entrypoint.sh create mode 100644 dev/provision.py create mode 100644 dev/spark-defaults.conf create mode 100644 tests/test_integration.py diff --git a/Makefile b/Makefile index 9e03b10403..d04466dff8 100644 --- a/Makefile +++ b/Makefile @@ -17,7 +17,7 @@ install: pip install poetry - poetry install -E pyarrow -E hive -E s3fs -E glue -E adlfs + poetry install -E pyarrow -E hive -E s3fs -E glue -E adlfs -E duckdb check-license: ./dev/check-license @@ -26,21 +26,22 @@ lint: poetry run pre-commit run --all-files test: - poetry run coverage run --source=pyiceberg/ -m pytest tests/ -m "not s3 and not adlfs" ${PYTEST_ARGS} + poetry run coverage run --source=pyiceberg/ -m pytest tests/ -m unmarked ${PYTEST_ARGS} poetry run coverage report -m --fail-under=90 poetry run coverage html poetry run coverage xml test-s3: sh ./dev/run-minio.sh - poetry run coverage run --source=pyiceberg/ -m pytest tests/ -m "not adlfs" ${PYTEST_ARGS} - poetry run coverage report -m --fail-under=90 - poetry run coverage html - poetry run coverage xml + poetry run coverage run --source=pyiceberg/ -m pytest tests/ -m s3 ${PYTEST_ARGS} + +test-integration: + docker-compose -f dev/docker-compose-integration.yml kill + docker-compose -f dev/docker-compose-integration.yml build + docker-compose -f dev/docker-compose-integration.yml up -d + sleep 20 + poetry run coverage run --source=pyiceberg/ -m pytest tests/ -m integration ${PYTEST_ARGS} test-adlfs: sh ./dev/run-azurite.sh - poetry run coverage run --source=pyiceberg/ -m pytest tests/ -m "not s3" ${PYTEST_ARGS} - poetry run coverage report -m --fail-under=90 - poetry run coverage html - poetry run coverage xml + poetry run coverage run --source=pyiceberg/ -m pytest tests/ -m adlfs ${PYTEST_ARGS} diff --git a/dev/Dockerfile b/dev/Dockerfile new file mode 100644 index 0000000000..13508dd668 --- /dev/null +++ b/dev/Dockerfile @@ -0,0 +1,67 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +FROM python:3.9-bullseye + +RUN apt-get -qq update && \ + apt-get -qq install -y --no-install-recommends \ + sudo \ + curl \ + vim \ + unzip \ + openjdk-11-jdk \ + build-essential \ + software-properties-common \ + ssh && \ + apt-get -qq clean && \ + rm -rf /var/lib/apt/lists/* + +# Optional env variables +ENV SPARK_HOME=${SPARK_HOME:-"/opt/spark"} +ENV HADOOP_HOME=${HADOOP_HOME:-"/opt/hadoop"} +ENV PYTHONPATH=$SPARK_HOME/python:$SPARK_HOME/python/lib/py4j-0.10.9.5-src.zip:$PYTHONPATH + +RUN mkdir -p ${HADOOP_HOME} && mkdir -p ${SPARK_HOME} && mkdir -p /home/iceberg/spark-events +WORKDIR ${SPARK_HOME} + +ENV SPARK_VERSION=3.3.2 + +RUN curl -s https://dlcdn.apache.org/spark/spark-${SPARK_VERSION}/spark-${SPARK_VERSION}-bin-hadoop3.tgz -o spark-${SPARK_VERSION}-bin-hadoop3.tgz \ + && tar xzf spark-${SPARK_VERSION}-bin-hadoop3.tgz --directory /opt/spark --strip-components 1 \ + && rm -rf spark-${SPARK_VERSION}-bin-hadoop3.tgz + +# Download iceberg spark runtime +RUN curl -s https://repo1.maven.org/maven2/org/apache/iceberg/iceberg-spark-runtime-3.3_2.12/1.1.0/iceberg-spark-runtime-3.3_2.12-1.1.0.jar -Lo iceberg-spark-runtime-3.3_2.12-1.1.0.jar \ + && mv iceberg-spark-runtime-3.3_2.12-1.1.0.jar /opt/spark/jars + +# Download Java AWS SDK +RUN curl -s https://repo1.maven.org/maven2/software/amazon/awssdk/bundle/2.17.165/bundle-2.17.165.jar -Lo bundle-2.17.165.jar \ + && mv bundle-2.17.165.jar /opt/spark/jars + +# Download URL connection client required for S3FileIO +RUN curl -s https://repo1.maven.org/maven2/software/amazon/awssdk/url-connection-client/2.17.165/url-connection-client-2.17.165.jar -Lo url-connection-client-2.17.165.jar \ + && mv url-connection-client-2.17.165.jar /opt/spark/jars + +COPY spark-defaults.conf /opt/spark/conf +ENV PATH="/opt/spark/sbin:/opt/spark/bin:${PATH}" + +RUN chmod u+x /opt/spark/sbin/* && \ + chmod u+x /opt/spark/bin/* + +COPY entrypoint.sh . +COPY provision.py . + +ENTRYPOINT ["./entrypoint.sh"] +CMD ["notebook"] diff --git a/dev/docker-compose-integration.yml b/dev/docker-compose-integration.yml new file mode 100644 index 0000000000..663303ed69 --- /dev/null +++ b/dev/docker-compose-integration.yml @@ -0,0 +1,76 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +version: "3" + +services: + spark-iceberg: + image: python-integration + container_name: pyiceberg-spark + build: . + depends_on: + - rest + - minio + volumes: + - ./warehouse:/home/iceberg/warehouse + environment: + - AWS_ACCESS_KEY_ID=admin + - AWS_SECRET_ACCESS_KEY=password + - AWS_REGION=us-east-1 + ports: + - 8888:8888 + - 8080:8080 + links: + - rest:rest + - minio:minio + rest: + image: tabulario/iceberg-rest:0.2.0 + container_name: pyiceberg-rest + ports: + - 8181:8181 + environment: + - AWS_ACCESS_KEY_ID=admin + - AWS_SECRET_ACCESS_KEY=password + - AWS_REGION=us-east-1 + - CATALOG_WAREHOUSE=s3a://warehouse/wh/ + - CATALOG_IO__IMPL=org.apache.iceberg.aws.s3.S3FileIO + - CATALOG_S3_ENDPOINT=http://minio:9000 + minio: + image: minio/minio + container_name: pyiceberg-minio + environment: + - MINIO_ROOT_USER=admin + - MINIO_ROOT_PASSWORD=password + ports: + - 9001:9001 + - 9000:9000 + command: [ "server", "/data", "--console-address", ":9001" ] + mc: + depends_on: + - minio + image: minio/mc + container_name: pyiceberg-mc + environment: + - AWS_ACCESS_KEY_ID=admin + - AWS_SECRET_ACCESS_KEY=password + - AWS_REGION=us-east-1 + entrypoint: > + /bin/sh -c " + until (/usr/bin/mc config host add minio http://minio:9000 admin password) do echo '...waiting...' && sleep 1; done; + /usr/bin/mc mb minio/warehouse; + /usr/bin/mc policy set public minio/warehouse; + tail -f /dev/null + " diff --git a/dev/entrypoint.sh b/dev/entrypoint.sh new file mode 100755 index 0000000000..d777f8f5a2 --- /dev/null +++ b/dev/entrypoint.sh @@ -0,0 +1,25 @@ +#!/bin/bash +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# + +start-master.sh -p 7077 +start-worker.sh spark://spark-iceberg:7077 +start-history-server.sh + +python3 ./provision.py diff --git a/dev/provision.py b/dev/provision.py new file mode 100644 index 0000000000..ec84e87a23 --- /dev/null +++ b/dev/provision.py @@ -0,0 +1,98 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +import time + +from pyspark.sql import SparkSession + +spark = SparkSession.builder.getOrCreate() + +print("Create database") + +spark.sql( + """ + CREATE DATABASE IF NOT EXISTS default; +""" +) + +spark.sql( + """ + use default; +""" +) + +spark.sql( + """ + DROP TABLE IF EXISTS test_null_nan; +""" +) + +spark.sql( + """ + CREATE TABLE test_null_nan + USING iceberg + AS SELECT + 1 AS idx, + float('NaN') AS col_numeric +UNION ALL SELECT + 2 AS idx, + null AS col_numeric +UNION ALL SELECT + 3 AS idx, + 1 AS col_numeric +""" +) + +spark.sql( + """ + CREATE TABLE test_null_nan_rewritten + USING iceberg + AS SELECT * FROM test_null_nan +""" +) + +spark.sql( + """ + DROP TABLE IF EXISTS test_deletes; +""" +) + +spark.sql( + """ + CREATE TABLE test_deletes + USING iceberg + TBLPROPERTIES ( + 'write.delete.mode'='merge-on-read', + 'write.update.mode'='merge-on-read', + 'write.merge.mode'='merge-on-read' + ) + AS SELECT + 1 AS idx, + True AS deleted +UNION ALL SELECT + 2 AS idx, + False AS deleted; +""" +) + +spark.sql( + """ + DELETE FROM test_deletes WHERE deleted = True; +""" +) + +while True: + time.sleep(1) diff --git a/dev/spark-defaults.conf b/dev/spark-defaults.conf new file mode 100644 index 0000000000..28f93b15a6 --- /dev/null +++ b/dev/spark-defaults.conf @@ -0,0 +1,29 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +spark.sql.extensions org.apache.iceberg.spark.extensions.IcebergSparkSessionExtensions +spark.sql.catalog.demo org.apache.iceberg.spark.SparkCatalog +spark.sql.catalog.demo.type rest +spark.sql.catalog.demo.uri http://rest:8181 +spark.sql.catalog.demo.io-impl org.apache.iceberg.aws.s3.S3FileIO +spark.sql.catalog.demo.warehouse s3a://warehouse/wh/ +spark.sql.catalog.demo.s3.endpoint http://minio:9000 +spark.sql.defaultCatalog demo +spark.eventLog.enabled true +spark.eventLog.dir /home/iceberg/spark-events +spark.history.fs.logDirectory /home/iceberg/spark-events +spark.sql.catalogImplementation in-memory diff --git a/pyiceberg/io/pyarrow.py b/pyiceberg/io/pyarrow.py index 37bf298b3e..07b59258dd 100644 --- a/pyiceberg/io/pyarrow.py +++ b/pyiceberg/io/pyarrow.py @@ -428,11 +428,11 @@ def visit_not_in(self, term: BoundTerm[pc.Expression], literals: Set[Any]) -> pc def visit_is_nan(self, term: BoundTerm[Any]) -> pc.Expression: ref = pc.field(term.ref().field.name) - return ref.is_null(nan_is_null=True) & ref.is_valid() + return pc.is_nan(ref) def visit_not_nan(self, term: BoundTerm[Any]) -> pc.Expression: ref = pc.field(term.ref().field.name) - return ~(ref.is_null(nan_is_null=True) & ref.is_valid()) + return ~pc.is_nan(ref) def visit_is_null(self, term: BoundTerm[Any]) -> pc.Expression: return pc.field(term.ref().field.name).is_null(nan_is_null=False) diff --git a/pyiceberg/table/__init__.py b/pyiceberg/table/__init__.py index 69ce08f457..89f3af847b 100644 --- a/pyiceberg/table/__init__.py +++ b/pyiceberg/table/__init__.py @@ -95,7 +95,7 @@ def name(self) -> Identifier: def scan( self, row_filter: Union[str, BooleanExpression] = ALWAYS_TRUE, - selected_fields: Tuple[str] = ("*",), + selected_fields: Tuple[str, ...] = ("*",), case_sensitive: bool = True, snapshot_id: Optional[int] = None, options: Properties = EMPTY_DICT, @@ -215,7 +215,7 @@ def _parse_row_filter(expr: Union[str, BooleanExpression]) -> BooleanExpression: class TableScan(ABC): table: Table row_filter: BooleanExpression - selected_fields: Tuple[str] + selected_fields: Tuple[str, ...] case_sensitive: bool snapshot_id: Optional[int] options: Properties @@ -224,7 +224,7 @@ def __init__( self, table: Table, row_filter: Union[str, BooleanExpression] = ALWAYS_TRUE, - selected_fields: Tuple[str] = ("*",), + selected_fields: Tuple[str, ...] = ("*",), case_sensitive: bool = True, snapshot_id: Optional[int] = None, options: Properties = EMPTY_DICT, @@ -331,7 +331,7 @@ def __init__( self, table: Table, row_filter: Union[str, BooleanExpression] = ALWAYS_TRUE, - selected_fields: Tuple[str] = ("*",), + selected_fields: Tuple[str, ...] = ("*",), case_sensitive: bool = True, snapshot_id: Optional[int] = None, options: Properties = EMPTY_DICT, diff --git a/pyproject.toml b/pyproject.toml index 6746f8a7de..f65bc14dbb 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -110,7 +110,8 @@ dynamodb = ["boto3"] [tool.pytest.ini_options] markers = [ "s3: marks a test as requiring access to s3 compliant storage (use with --aws-access-key-id, --aws-secret-access-key, and --endpoint-url args)", - "adlfs: marks a test as requiring access to adlfs compliant storage (use with --adlfs.account-name, --adlfs.account-key, and --adlfs.endpoint args)" + "adlfs: marks a test as requiring access to adlfs compliant storage (use with --adlfs.account-name, --adlfs.account-key, and --adlfs.endpoint args)", + "integration: marks integration tests against Apache Spark" ] [tool.black] @@ -240,5 +241,9 @@ ignore_missing_imports = true module = "pyparsing.*" ignore_missing_imports = true +[[tool.mypy.overrides]] +module = "pyspark.*" +ignore_missing_imports = true + [tool.coverage.run] source = ['pyiceberg/'] diff --git a/tests/test_integration.py b/tests/test_integration.py new file mode 100644 index 0000000000..3d498f0ec8 --- /dev/null +++ b/tests/test_integration.py @@ -0,0 +1,82 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# pylint:disable=redefined-outer-name + +import math + +import pytest + +from pyiceberg.catalog import Catalog, load_catalog +from pyiceberg.expressions import IsNaN, NotNaN +from pyiceberg.table import Table + + +@pytest.fixture() +def catalog() -> Catalog: + return load_catalog( + "local", + **{ + "type": "rest", + "uri": "http://localhost:8181", + "s3.endpoint": "http://localhost:9000", + "s3.access-key-id": "admin", + "s3.secret-access-key": "password", + }, + ) + + +@pytest.fixture() +def table_test_null_nan(catalog: Catalog) -> Table: + return catalog.load_table("default.test_null_nan") + + +@pytest.fixture() +def table_test_null_nan_rewritten(catalog: Catalog) -> Table: + return catalog.load_table("default.test_null_nan_rewritten") + + +@pytest.mark.integration +def test_pyarrow_nan(table_test_null_nan: Table) -> None: + arrow_table = table_test_null_nan.scan(row_filter=IsNaN("col_numeric"), selected_fields=("idx", "col_numeric")).to_arrow() + assert len(arrow_table) == 1 + assert arrow_table["idx"][0].as_py() == 1 + assert math.isnan(arrow_table["col_numeric"][0].as_py()) + + +@pytest.mark.integration +def test_pyarrow_nan_rewritten(table_test_null_nan_rewritten: Table) -> None: + arrow_table = table_test_null_nan_rewritten.scan( + row_filter=IsNaN("col_numeric"), selected_fields=("idx", "col_numeric") + ).to_arrow() + assert len(arrow_table) == 1 + assert arrow_table["idx"][0].as_py() == 1 + assert math.isnan(arrow_table["col_numeric"][0].as_py()) + + +@pytest.mark.integration +@pytest.mark.skip(reason="Fixing issues with NaN's: https://github.com/apache/arrow/issues/34162") +def test_pyarrow_not_nan_count(table_test_null_nan: Table) -> None: + not_nan = table_test_null_nan.scan(row_filter=NotNaN("col_numeric"), selected_fields=("idx",)).to_arrow() + assert len(not_nan) == 2 + + +@pytest.mark.integration +def test_duckdb_nan(table_test_null_nan_rewritten: Table) -> None: + con = table_test_null_nan_rewritten.scan().to_duckdb("table_test_null_nan") + result = con.query("SELECT idx, col_numeric FROM table_test_null_nan WHERE isnan(col_numeric)").fetchone() + assert result[0] == 1 + assert math.isnan(result[1]) From 9546da76d2f3a47fcbda85b3c6b8eab547835bd3 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Sun, 19 Mar 2023 12:00:00 +0100 Subject: [PATCH 404/642] Build: Bump pre-commit from 3.1.1 to 3.2.0 in /python (#7141) --- poetry.lock | 8 ++++---- pyproject.toml | 2 +- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/poetry.lock b/poetry.lock index bf90680172..609c48a934 100644 --- a/poetry.lock +++ b/poetry.lock @@ -1609,14 +1609,14 @@ tests = ["pytest (>=5.4.1)", "pytest-cov (>=2.8.1)", "pytest-mypy (>=0.8.0)", "p [[package]] name = "pre-commit" -version = "3.1.1" +version = "3.2.0" description = "A framework for managing and maintaining multi-language pre-commit hooks." category = "dev" optional = false python-versions = ">=3.8" files = [ - {file = "pre_commit-3.1.1-py2.py3-none-any.whl", hash = "sha256:b80254e60668e1dd1f5c03a1c9e0413941d61f568a57d745add265945f65bfe8"}, - {file = "pre_commit-3.1.1.tar.gz", hash = "sha256:d63e6537f9252d99f65755ae5b79c989b462d511ebbc481b561db6a297e1e865"}, + {file = "pre_commit-3.2.0-py2.py3-none-any.whl", hash = "sha256:f712d3688102e13c8e66b7d7dbd8934a6dda157e58635d89f7d6fecdca39ce8a"}, + {file = "pre_commit-3.2.0.tar.gz", hash = "sha256:818f0d998059934d0f81bb3667e3ccdc32da6ed7ccaac33e43dc231561ddaaa9"}, ] [package.dependencies] @@ -2563,4 +2563,4 @@ snappy = ["python-snappy"] [metadata] lock-version = "2.0" python-versions = "^3.8" -content-hash = "dc2372a1fa549a351cbb80296b8364377f5977896daccaf6866392696f524d3c" +content-hash = "bc2c7ca4a0b4c28fab0c7db7227bf7b7d5565a3e814ce14b7e111f9a7e9c8871" diff --git a/pyproject.toml b/pyproject.toml index f65bc14dbb..ea8353b69f 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -81,7 +81,7 @@ adlfs = { version = ">=2022.8.2,<=2023.1.0", optional = true } [tool.poetry.dev-dependencies] pytest = "7.2.2" pytest-checkdocs = "2.9.0" -pre-commit = "3.1.1" +pre-commit = "3.2.0" fastavro = "1.7.3" coverage = { version = "^7.2.1", extras = ["toml"] } requests-mock = "1.10.0" From dd3c6ba53b39da566e0b29fb97ddf30bac36117a Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Sun, 19 Mar 2023 16:03:56 +0100 Subject: [PATCH 405/642] Build: Bump coverage from 7.2.1 to 7.2.2 in /python (#7142) Bumps [coverage](https://github.com/nedbat/coveragepy) from 7.2.1 to 7.2.2. - [Release notes](https://github.com/nedbat/coveragepy/releases) - [Changelog](https://github.com/nedbat/coveragepy/blob/master/CHANGES.rst) - [Commits](https://github.com/nedbat/coveragepy/compare/7.2.1...7.2.2) --- updated-dependencies: - dependency-name: coverage dependency-type: direct:development update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- poetry.lock | 106 ++++++++++++++++++++++++------------------------- pyproject.toml | 2 +- 2 files changed, 54 insertions(+), 54 deletions(-) diff --git a/poetry.lock b/poetry.lock index 609c48a934..bf2b9ad15c 100644 --- a/poetry.lock +++ b/poetry.lock @@ -598,63 +598,63 @@ files = [ [[package]] name = "coverage" -version = "7.2.1" +version = "7.2.2" description = "Code coverage measurement for Python" category = "dev" optional = false python-versions = ">=3.7" files = [ - {file = "coverage-7.2.1-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:49567ec91fc5e0b15356da07a2feabb421d62f52a9fff4b1ec40e9e19772f5f8"}, - {file = "coverage-7.2.1-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:d2ef6cae70168815ed91388948b5f4fcc69681480a0061114db737f957719f03"}, - {file = "coverage-7.2.1-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:3004765bca3acd9e015794e5c2f0c9a05587f5e698127ff95e9cfba0d3f29339"}, - {file = "coverage-7.2.1-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:cca7c0b7f5881dfe0291ef09ba7bb1582cb92ab0aeffd8afb00c700bf692415a"}, - {file = "coverage-7.2.1-cp310-cp310-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:b2167d116309f564af56f9aa5e75ef710ef871c5f9b313a83050035097b56820"}, - {file = "coverage-7.2.1-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:cb5f152fb14857cbe7f3e8c9a5d98979c4c66319a33cad6e617f0067c9accdc4"}, - {file = "coverage-7.2.1-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:87dc37f16fb5e3a28429e094145bf7c1753e32bb50f662722e378c5851f7fdc6"}, - {file = "coverage-7.2.1-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:e191a63a05851f8bce77bc875e75457f9b01d42843f8bd7feed2fc26bbe60833"}, - {file = "coverage-7.2.1-cp310-cp310-win32.whl", hash = "sha256:e3ea04b23b114572b98a88c85379e9e9ae031272ba1fb9b532aa934c621626d4"}, - {file = "coverage-7.2.1-cp310-cp310-win_amd64.whl", hash = "sha256:0cf557827be7eca1c38a2480484d706693e7bb1929e129785fe59ec155a59de6"}, - {file = "coverage-7.2.1-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:570c21a29493b350f591a4b04c158ce1601e8d18bdcd21db136fbb135d75efa6"}, - {file = "coverage-7.2.1-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:9e872b082b32065ac2834149dc0adc2a2e6d8203080501e1e3c3c77851b466f9"}, - {file = "coverage-7.2.1-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:fac6343bae03b176e9b58104a9810df3cdccd5cfed19f99adfa807ffbf43cf9b"}, - {file = "coverage-7.2.1-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:abacd0a738e71b20e224861bc87e819ef46fedba2fb01bc1af83dfd122e9c319"}, - {file = "coverage-7.2.1-cp311-cp311-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:d9256d4c60c4bbfec92721b51579c50f9e5062c21c12bec56b55292464873508"}, - {file = "coverage-7.2.1-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:80559eaf6c15ce3da10edb7977a1548b393db36cbc6cf417633eca05d84dd1ed"}, - {file = "coverage-7.2.1-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:0bd7e628f6c3ec4e7d2d24ec0e50aae4e5ae95ea644e849d92ae4805650b4c4e"}, - {file = "coverage-7.2.1-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:09643fb0df8e29f7417adc3f40aaf379d071ee8f0350ab290517c7004f05360b"}, - {file = "coverage-7.2.1-cp311-cp311-win32.whl", hash = "sha256:1b7fb13850ecb29b62a447ac3516c777b0e7a09ecb0f4bb6718a8654c87dfc80"}, - {file = "coverage-7.2.1-cp311-cp311-win_amd64.whl", hash = "sha256:617a94ada56bbfe547aa8d1b1a2b8299e2ec1ba14aac1d4b26a9f7d6158e1273"}, - {file = "coverage-7.2.1-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:8649371570551d2fd7dee22cfbf0b61f1747cdfb2b7587bb551e4beaaa44cb97"}, - {file = "coverage-7.2.1-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:5d2b9b5e70a21474c105a133ba227c61bc95f2ac3b66861143ce39a5ea4b3f84"}, - {file = "coverage-7.2.1-cp37-cp37m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:ae82c988954722fa07ec5045c57b6d55bc1a0890defb57cf4a712ced65b26ddd"}, - {file = "coverage-7.2.1-cp37-cp37m-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:861cc85dfbf55a7a768443d90a07e0ac5207704a9f97a8eb753292a7fcbdfcfc"}, - {file = "coverage-7.2.1-cp37-cp37m-musllinux_1_1_aarch64.whl", hash = "sha256:0339dc3237c0d31c3b574f19c57985fcbe494280153bbcad33f2cdf469f4ac3e"}, - {file = "coverage-7.2.1-cp37-cp37m-musllinux_1_1_i686.whl", hash = "sha256:5928b85416a388dd557ddc006425b0c37e8468bd1c3dc118c1a3de42f59e2a54"}, - {file = "coverage-7.2.1-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:8d3843ca645f62c426c3d272902b9de90558e9886f15ddf5efe757b12dd376f5"}, - {file = "coverage-7.2.1-cp37-cp37m-win32.whl", hash = "sha256:6a034480e9ebd4e83d1aa0453fd78986414b5d237aea89a8fdc35d330aa13bae"}, - {file = "coverage-7.2.1-cp37-cp37m-win_amd64.whl", hash = "sha256:6fce673f79a0e017a4dc35e18dc7bb90bf6d307c67a11ad5e61ca8d42b87cbff"}, - {file = "coverage-7.2.1-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:7f099da6958ddfa2ed84bddea7515cb248583292e16bb9231d151cd528eab657"}, - {file = "coverage-7.2.1-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:97a3189e019d27e914ecf5c5247ea9f13261d22c3bb0cfcfd2a9b179bb36f8b1"}, - {file = "coverage-7.2.1-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a81dbcf6c6c877986083d00b834ac1e84b375220207a059ad45d12f6e518a4e3"}, - {file = "coverage-7.2.1-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:78d2c3dde4c0b9be4b02067185136b7ee4681978228ad5ec1278fa74f5ca3e99"}, - {file = "coverage-7.2.1-cp38-cp38-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:3a209d512d157379cc9ab697cbdbb4cfd18daa3e7eebaa84c3d20b6af0037384"}, - {file = "coverage-7.2.1-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:f3d07edb912a978915576a776756069dede66d012baa503022d3a0adba1b6afa"}, - {file = "coverage-7.2.1-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:8dca3c1706670297851bca1acff9618455122246bdae623be31eca744ade05ec"}, - {file = "coverage-7.2.1-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:b1991a6d64231a3e5bbe3099fb0dd7c9aeaa4275ad0e0aeff4cb9ef885c62ba2"}, - {file = "coverage-7.2.1-cp38-cp38-win32.whl", hash = "sha256:22c308bc508372576ffa3d2dbc4824bb70d28eeb4fcd79d4d1aed663a06630d0"}, - {file = "coverage-7.2.1-cp38-cp38-win_amd64.whl", hash = "sha256:b0c0d46de5dd97f6c2d1b560bf0fcf0215658097b604f1840365296302a9d1fb"}, - {file = "coverage-7.2.1-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:4dd34a935de268a133e4741827ae951283a28c0125ddcdbcbba41c4b98f2dfef"}, - {file = "coverage-7.2.1-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:0f8318ed0f3c376cfad8d3520f496946977abde080439d6689d7799791457454"}, - {file = "coverage-7.2.1-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:834c2172edff5a08d78e2f53cf5e7164aacabeb66b369f76e7bb367ca4e2d993"}, - {file = "coverage-7.2.1-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:e4d70c853f0546855f027890b77854508bdb4d6a81242a9d804482e667fff6e6"}, - {file = "coverage-7.2.1-cp39-cp39-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:8a6450da4c7afc4534305b2b7d8650131e130610cea448ff240b6ab73d7eab63"}, - {file = "coverage-7.2.1-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:99f4dd81b2bb8fc67c3da68b1f5ee1650aca06faa585cbc6818dbf67893c6d58"}, - {file = "coverage-7.2.1-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:bdd3f2f285ddcf2e75174248b2406189261a79e7fedee2ceeadc76219b6faa0e"}, - {file = "coverage-7.2.1-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:f29351393eb05e6326f044a7b45ed8e38cb4dcc38570d12791f271399dc41431"}, - {file = "coverage-7.2.1-cp39-cp39-win32.whl", hash = "sha256:e2b50ebc2b6121edf352336d503357321b9d8738bb7a72d06fc56153fd3f4cd8"}, - {file = "coverage-7.2.1-cp39-cp39-win_amd64.whl", hash = "sha256:bd5a12239c0006252244f94863f1c518ac256160cd316ea5c47fb1a11b25889a"}, - {file = "coverage-7.2.1-pp37.pp38.pp39-none-any.whl", hash = "sha256:436313d129db7cf5b4ac355dd2bd3f7c7e5294af077b090b85de75f8458b8616"}, - {file = "coverage-7.2.1.tar.gz", hash = "sha256:c77f2a9093ccf329dd523a9b2b3c854c20d2a3d968b6def3b820272ca6732242"}, + {file = "coverage-7.2.2-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:c90e73bdecb7b0d1cea65a08cb41e9d672ac6d7995603d6465ed4914b98b9ad7"}, + {file = "coverage-7.2.2-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:e2926b8abedf750c2ecf5035c07515770944acf02e1c46ab08f6348d24c5f94d"}, + {file = "coverage-7.2.2-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:57b77b9099f172804e695a40ebaa374f79e4fb8b92f3e167f66facbf92e8e7f5"}, + {file = "coverage-7.2.2-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:efe1c0adad110bf0ad7fb59f833880e489a61e39d699d37249bdf42f80590169"}, + {file = "coverage-7.2.2-cp310-cp310-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:2199988e0bc8325d941b209f4fd1c6fa007024b1442c5576f1a32ca2e48941e6"}, + {file = "coverage-7.2.2-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:81f63e0fb74effd5be736cfe07d710307cc0a3ccb8f4741f7f053c057615a137"}, + {file = "coverage-7.2.2-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:186e0fc9cf497365036d51d4d2ab76113fb74f729bd25da0975daab2e107fd90"}, + {file = "coverage-7.2.2-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:420f94a35e3e00a2b43ad5740f935358e24478354ce41c99407cddd283be00d2"}, + {file = "coverage-7.2.2-cp310-cp310-win32.whl", hash = "sha256:38004671848b5745bb05d4d621526fca30cee164db42a1f185615f39dc997292"}, + {file = "coverage-7.2.2-cp310-cp310-win_amd64.whl", hash = "sha256:0ce383d5f56d0729d2dd40e53fe3afeb8f2237244b0975e1427bfb2cf0d32bab"}, + {file = "coverage-7.2.2-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:3eb55b7b26389dd4f8ae911ba9bc8c027411163839dea4c8b8be54c4ee9ae10b"}, + {file = "coverage-7.2.2-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:d2b96123a453a2d7f3995ddb9f28d01fd112319a7a4d5ca99796a7ff43f02af5"}, + {file = "coverage-7.2.2-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:299bc75cb2a41e6741b5e470b8c9fb78d931edbd0cd009c58e5c84de57c06731"}, + {file = "coverage-7.2.2-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:5e1df45c23d4230e3d56d04414f9057eba501f78db60d4eeecfcb940501b08fd"}, + {file = "coverage-7.2.2-cp311-cp311-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:006ed5582e9cbc8115d2e22d6d2144a0725db542f654d9d4fda86793832f873d"}, + {file = "coverage-7.2.2-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:d683d230b5774816e7d784d7ed8444f2a40e7a450e5720d58af593cb0b94a212"}, + {file = "coverage-7.2.2-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:8efb48fa743d1c1a65ee8787b5b552681610f06c40a40b7ef94a5b517d885c54"}, + {file = "coverage-7.2.2-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:4c752d5264053a7cf2fe81c9e14f8a4fb261370a7bb344c2a011836a96fb3f57"}, + {file = "coverage-7.2.2-cp311-cp311-win32.whl", hash = "sha256:55272f33da9a5d7cccd3774aeca7a01e500a614eaea2a77091e9be000ecd401d"}, + {file = "coverage-7.2.2-cp311-cp311-win_amd64.whl", hash = "sha256:92ebc1619650409da324d001b3a36f14f63644c7f0a588e331f3b0f67491f512"}, + {file = "coverage-7.2.2-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:5afdad4cc4cc199fdf3e18088812edcf8f4c5a3c8e6cb69127513ad4cb7471a9"}, + {file = "coverage-7.2.2-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:0484d9dd1e6f481b24070c87561c8d7151bdd8b044c93ac99faafd01f695c78e"}, + {file = "coverage-7.2.2-cp37-cp37m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:d530191aa9c66ab4f190be8ac8cc7cfd8f4f3217da379606f3dd4e3d83feba69"}, + {file = "coverage-7.2.2-cp37-cp37m-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:4ac0f522c3b6109c4b764ffec71bf04ebc0523e926ca7cbe6c5ac88f84faced0"}, + {file = "coverage-7.2.2-cp37-cp37m-musllinux_1_1_aarch64.whl", hash = "sha256:ba279aae162b20444881fc3ed4e4f934c1cf8620f3dab3b531480cf602c76b7f"}, + {file = "coverage-7.2.2-cp37-cp37m-musllinux_1_1_i686.whl", hash = "sha256:53d0fd4c17175aded9c633e319360d41a1f3c6e352ba94edcb0fa5167e2bad67"}, + {file = "coverage-7.2.2-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:8c99cb7c26a3039a8a4ee3ca1efdde471e61b4837108847fb7d5be7789ed8fd9"}, + {file = "coverage-7.2.2-cp37-cp37m-win32.whl", hash = "sha256:5cc0783844c84af2522e3a99b9b761a979a3ef10fb87fc4048d1ee174e18a7d8"}, + {file = "coverage-7.2.2-cp37-cp37m-win_amd64.whl", hash = "sha256:817295f06eacdc8623dc4df7d8b49cea65925030d4e1e2a7c7218380c0072c25"}, + {file = "coverage-7.2.2-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:6146910231ece63facfc5984234ad1b06a36cecc9fd0c028e59ac7c9b18c38c6"}, + {file = "coverage-7.2.2-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:387fb46cb8e53ba7304d80aadca5dca84a2fbf6fe3faf6951d8cf2d46485d1e5"}, + {file = "coverage-7.2.2-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:046936ab032a2810dcaafd39cc4ef6dd295df1a7cbead08fe996d4765fca9fe4"}, + {file = "coverage-7.2.2-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:e627dee428a176ffb13697a2c4318d3f60b2ccdde3acdc9b3f304206ec130ccd"}, + {file = "coverage-7.2.2-cp38-cp38-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:4fa54fb483decc45f94011898727802309a109d89446a3c76387d016057d2c84"}, + {file = "coverage-7.2.2-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:3668291b50b69a0c1ef9f462c7df2c235da3c4073f49543b01e7eb1dee7dd540"}, + {file = "coverage-7.2.2-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:7c20b731211261dc9739bbe080c579a1835b0c2d9b274e5fcd903c3a7821cf88"}, + {file = "coverage-7.2.2-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:5764e1f7471cb8f64b8cda0554f3d4c4085ae4b417bfeab236799863703e5de2"}, + {file = "coverage-7.2.2-cp38-cp38-win32.whl", hash = "sha256:4f01911c010122f49a3e9bdc730eccc66f9b72bd410a3a9d3cb8448bb50d65d3"}, + {file = "coverage-7.2.2-cp38-cp38-win_amd64.whl", hash = "sha256:c448b5c9e3df5448a362208b8d4b9ed85305528313fca1b479f14f9fe0d873b8"}, + {file = "coverage-7.2.2-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:bfe7085783cda55e53510482fa7b5efc761fad1abe4d653b32710eb548ebdd2d"}, + {file = "coverage-7.2.2-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:9d22e94e6dc86de981b1b684b342bec5e331401599ce652900ec59db52940005"}, + {file = "coverage-7.2.2-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:507e4720791977934bba016101579b8c500fb21c5fa3cd4cf256477331ddd988"}, + {file = "coverage-7.2.2-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:bc4803779f0e4b06a2361f666e76f5c2e3715e8e379889d02251ec911befd149"}, + {file = "coverage-7.2.2-cp39-cp39-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:db8c2c5ace167fd25ab5dd732714c51d4633f58bac21fb0ff63b0349f62755a8"}, + {file = "coverage-7.2.2-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:4f68ee32d7c4164f1e2c8797535a6d0a3733355f5861e0f667e37df2d4b07140"}, + {file = "coverage-7.2.2-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:d52f0a114b6a58305b11a5cdecd42b2e7f1ec77eb20e2b33969d702feafdd016"}, + {file = "coverage-7.2.2-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:797aad79e7b6182cb49c08cc5d2f7aa7b2128133b0926060d0a8889ac43843be"}, + {file = "coverage-7.2.2-cp39-cp39-win32.whl", hash = "sha256:db45eec1dfccdadb179b0f9ca616872c6f700d23945ecc8f21bb105d74b1c5fc"}, + {file = "coverage-7.2.2-cp39-cp39-win_amd64.whl", hash = "sha256:8dbe2647bf58d2c5a6c5bcc685f23b5f371909a5624e9f5cd51436d6a9f6c6ef"}, + {file = "coverage-7.2.2-pp37.pp38.pp39-none-any.whl", hash = "sha256:872d6ce1f5be73f05bea4df498c140b9e7ee5418bfa2cc8204e7f9b817caa968"}, + {file = "coverage-7.2.2.tar.gz", hash = "sha256:36dd42da34fe94ed98c39887b86db9d06777b1c8f860520e21126a75507024f2"}, ] [package.dependencies] @@ -2563,4 +2563,4 @@ snappy = ["python-snappy"] [metadata] lock-version = "2.0" python-versions = "^3.8" -content-hash = "bc2c7ca4a0b4c28fab0c7db7227bf7b7d5565a3e814ce14b7e111f9a7e9c8871" +content-hash = "690692d9d932f3d1b5fe540aea1f4bcf7dbcc12fedb0c6f1ab343440d8a1abfc" diff --git a/pyproject.toml b/pyproject.toml index ea8353b69f..f4c4f6981e 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -83,7 +83,7 @@ pytest = "7.2.2" pytest-checkdocs = "2.9.0" pre-commit = "3.2.0" fastavro = "1.7.3" -coverage = { version = "^7.2.1", extras = ["toml"] } +coverage = { version = "^7.2.2", extras = ["toml"] } requests-mock = "1.10.0" moto = "^4.1.4" typing-extensions = '4.5.0' From 7c7809303bd9430b7c3c12a69f6703c43f244bcb Mon Sep 17 00:00:00 2001 From: "Jonas(Rushan) Jiang" Date: Tue, 21 Mar 2023 17:47:15 -0400 Subject: [PATCH 406/642] Python: Make Python CI collect most of the unit tests and Fix CI name conflict (#7136) --- Makefile | 19 ++++++++++++------- tests/conftest.py | 6 ++++++ tests/io/test_pyarrow.py | 10 ++-------- 3 files changed, 20 insertions(+), 15 deletions(-) diff --git a/Makefile b/Makefile index d04466dff8..c693ab7fd7 100644 --- a/Makefile +++ b/Makefile @@ -26,22 +26,27 @@ lint: poetry run pre-commit run --all-files test: - poetry run coverage run --source=pyiceberg/ -m pytest tests/ -m unmarked ${PYTEST_ARGS} - poetry run coverage report -m --fail-under=90 - poetry run coverage html - poetry run coverage xml + poetry run pytest tests/ -m "unmarked or parametrize" ${PYTEST_ARGS} test-s3: sh ./dev/run-minio.sh - poetry run coverage run --source=pyiceberg/ -m pytest tests/ -m s3 ${PYTEST_ARGS} + poetry run pytest tests/ -m s3 ${PYTEST_ARGS} test-integration: docker-compose -f dev/docker-compose-integration.yml kill docker-compose -f dev/docker-compose-integration.yml build docker-compose -f dev/docker-compose-integration.yml up -d sleep 20 - poetry run coverage run --source=pyiceberg/ -m pytest tests/ -m integration ${PYTEST_ARGS} + poetry run pytest tests/ -m integration ${PYTEST_ARGS} test-adlfs: sh ./dev/run-azurite.sh - poetry run coverage run --source=pyiceberg/ -m pytest tests/ -m adlfs ${PYTEST_ARGS} + poetry run pytest tests/ -m adlfs ${PYTEST_ARGS} + +test-coverage: + sh ./dev/run-minio.sh + sh ./dev/run-azurite.sh + poetry run coverage run --source=pyiceberg/ -m pytest tests/ -m "not integration" ${PYTEST_ARGS} + poetry run coverage report -m --fail-under=90 + poetry run coverage html + poetry run coverage xml diff --git a/tests/conftest.py b/tests/conftest.py index 8987150b55..b4db30597a 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -80,6 +80,12 @@ ) +def pytest_collection_modifyitems(items: List[pytest.Item]) -> None: + for item in items: + if not any(item.iter_markers()): + item.add_marker("unmarked") + + def pytest_addoption(parser: pytest.Parser) -> None: # S3 options parser.addoption( diff --git a/tests/io/test_pyarrow.py b/tests/io/test_pyarrow.py index 4b363d0d44..7a78b1c0a9 100644 --- a/tests/io/test_pyarrow.py +++ b/tests/io/test_pyarrow.py @@ -450,17 +450,11 @@ def test_expr_not_null_to_pyarrow(bound_reference: BoundReference[str]) -> None: def test_expr_is_nan_to_pyarrow(bound_double_reference: BoundReference[str]) -> None: - assert ( - repr(expression_to_pyarrow(BoundIsNaN(bound_double_reference))) - == "" - ) + assert repr(expression_to_pyarrow(BoundIsNaN(bound_double_reference))) == "" def test_expr_not_nan_to_pyarrow(bound_double_reference: BoundReference[str]) -> None: - assert ( - repr(expression_to_pyarrow(BoundNotNaN(bound_double_reference))) - == "" - ) + assert repr(expression_to_pyarrow(BoundNotNaN(bound_double_reference))) == "" def test_expr_equal_to_pyarrow(bound_reference: BoundReference[str]) -> None: From 6559b8ef7f03b5054a46629dda0398a411ec620a Mon Sep 17 00:00:00 2001 From: Prashant Singh <35593236+singhpk234@users.noreply.github.com> Date: Wed, 22 Mar 2023 15:32:08 -0700 Subject: [PATCH 407/642] Python: Update docker file (#7164) --- dev/Dockerfile | 14 ++++++++------ dev/docker-compose-integration.yml | 17 +++++++++++++++-- dev/docker-compose.yml | 1 + mkdocs/docs/verify-release.md | 4 ++-- 4 files changed, 26 insertions(+), 10 deletions(-) diff --git a/dev/Dockerfile b/dev/Dockerfile index 13508dd668..65d5503b57 100644 --- a/dev/Dockerfile +++ b/dev/Dockerfile @@ -37,22 +37,24 @@ RUN mkdir -p ${HADOOP_HOME} && mkdir -p ${SPARK_HOME} && mkdir -p /home/iceberg/ WORKDIR ${SPARK_HOME} ENV SPARK_VERSION=3.3.2 +ENV ICEBERG_VERSION=1.2.0 +ENV AWS_SDK_VERSION=2.20.18 RUN curl -s https://dlcdn.apache.org/spark/spark-${SPARK_VERSION}/spark-${SPARK_VERSION}-bin-hadoop3.tgz -o spark-${SPARK_VERSION}-bin-hadoop3.tgz \ && tar xzf spark-${SPARK_VERSION}-bin-hadoop3.tgz --directory /opt/spark --strip-components 1 \ && rm -rf spark-${SPARK_VERSION}-bin-hadoop3.tgz # Download iceberg spark runtime -RUN curl -s https://repo1.maven.org/maven2/org/apache/iceberg/iceberg-spark-runtime-3.3_2.12/1.1.0/iceberg-spark-runtime-3.3_2.12-1.1.0.jar -Lo iceberg-spark-runtime-3.3_2.12-1.1.0.jar \ - && mv iceberg-spark-runtime-3.3_2.12-1.1.0.jar /opt/spark/jars +RUN curl -s https://repo1.maven.org/maven2/org/apache/iceberg/iceberg-spark-runtime-3.3_2.12/${ICEBERG_VERSION}/iceberg-spark-runtime-3.3_2.12-${ICEBERG_VERSION}.jar -Lo iceberg-spark-runtime-3.3_2.12-${ICEBERG_VERSION}.jar \ + && mv iceberg-spark-runtime-3.3_2.12-${ICEBERG_VERSION}.jar /opt/spark/jars # Download Java AWS SDK -RUN curl -s https://repo1.maven.org/maven2/software/amazon/awssdk/bundle/2.17.165/bundle-2.17.165.jar -Lo bundle-2.17.165.jar \ - && mv bundle-2.17.165.jar /opt/spark/jars +RUN curl -s https://repo1.maven.org/maven2/software/amazon/awssdk/bundle/${AWS_SDK_VERSION}/bundle-${AWS_SDK_VERSION}.jar -Lo bundle-${AWS_SDK_VERSION}.jar \ + && mv bundle-${AWS_SDK_VERSION}.jar /opt/spark/jars # Download URL connection client required for S3FileIO -RUN curl -s https://repo1.maven.org/maven2/software/amazon/awssdk/url-connection-client/2.17.165/url-connection-client-2.17.165.jar -Lo url-connection-client-2.17.165.jar \ - && mv url-connection-client-2.17.165.jar /opt/spark/jars +RUN curl -s https://repo1.maven.org/maven2/software/amazon/awssdk/url-connection-client/${AWS_SDK_VERSION}/url-connection-client-${AWS_SDK_VERSION}.jar -Lo url-connection-client-${AWS_SDK_VERSION}.jar \ + && mv url-connection-client-${AWS_SDK_VERSION}.jar /opt/spark/jars COPY spark-defaults.conf /opt/spark/conf ENV PATH="/opt/spark/sbin:/opt/spark/bin:${PATH}" diff --git a/dev/docker-compose-integration.yml b/dev/docker-compose-integration.yml index 663303ed69..562a3c7f46 100644 --- a/dev/docker-compose-integration.yml +++ b/dev/docker-compose-integration.yml @@ -21,6 +21,8 @@ services: image: python-integration container_name: pyiceberg-spark build: . + networks: + iceberg_net: depends_on: - rest - minio @@ -37,15 +39,17 @@ services: - rest:rest - minio:minio rest: - image: tabulario/iceberg-rest:0.2.0 + image: tabulario/iceberg-rest container_name: pyiceberg-rest + networks: + iceberg_net: ports: - 8181:8181 environment: - AWS_ACCESS_KEY_ID=admin - AWS_SECRET_ACCESS_KEY=password - AWS_REGION=us-east-1 - - CATALOG_WAREHOUSE=s3a://warehouse/wh/ + - CATALOG_WAREHOUSE=s3://warehouse/ - CATALOG_IO__IMPL=org.apache.iceberg.aws.s3.S3FileIO - CATALOG_S3_ENDPOINT=http://minio:9000 minio: @@ -54,6 +58,11 @@ services: environment: - MINIO_ROOT_USER=admin - MINIO_ROOT_PASSWORD=password + - MINIO_DOMAIN=minio + networks: + iceberg_net: + aliases: + - warehouse.minio ports: - 9001:9001 - 9000:9000 @@ -63,6 +72,8 @@ services: - minio image: minio/mc container_name: pyiceberg-mc + networks: + iceberg_net: environment: - AWS_ACCESS_KEY_ID=admin - AWS_SECRET_ACCESS_KEY=password @@ -74,3 +85,5 @@ services: /usr/bin/mc policy set public minio/warehouse; tail -f /dev/null " +networks: + iceberg_net: diff --git a/dev/docker-compose.yml b/dev/docker-compose.yml index f7fbd6deaa..423ea62f04 100644 --- a/dev/docker-compose.yml +++ b/dev/docker-compose.yml @@ -23,6 +23,7 @@ services: environment: - MINIO_ROOT_USER=admin - MINIO_ROOT_PASSWORD=password + - MINIO_DOMAIN=minio ports: - 9001:9001 - 9000:9000 diff --git a/mkdocs/docs/verify-release.md b/mkdocs/docs/verify-release.md index 04c8288aa7..962d0ba111 100644 --- a/mkdocs/docs/verify-release.md +++ b/mkdocs/docs/verify-release.md @@ -99,8 +99,8 @@ This will include a Minio S3 container being spun up. # Cast the vote -Votes are cast by replying to the release candidate announcement email on the dev mailing list with either `+1`, `0`, or `-1`. +Votes are cast by replying to the release candidate announcement email on the dev mailing list with either `+1`, `0`, or `-1`. For example : -> \[ \] +1 Release this as Apache Iceberg 1.1.0 \[ \] +0 \[ \] -1 Do not release this because… +> \[ \] +1 Release this as PyIceberg 0.3.0 \[ \] +0 \[ \] -1 Do not release this because… In addition to your vote, it’s customary to specify if your vote is binding or non-binding. Only members of the Project Management Committee have formally binding votes. If you’re unsure, you can specify that your vote is non-binding. To read more about voting in the Apache framework, checkout the [Voting](https://www.apache.org/foundation/voting.html) information page on the Apache foundation’s website. From 26f64d8df2af80a1d51917a0fe8c13d74e3578a2 Mon Sep 17 00:00:00 2001 From: "Jonas(Rushan) Jiang" Date: Thu, 23 Mar 2023 12:38:56 -0400 Subject: [PATCH 408/642] Python: Add conversion from iceberg table scan to Ray dataset (#7148) --- Makefile | 4 +- dev/provision.py | 30 +++ mkdocs/docs/api.md | 48 +++++ mkdocs/docs/index.md | 2 + poetry.lock | 393 +++++++++++++++++++++++++++++++++++- pyiceberg/table/__init__.py | 6 + pyproject.toml | 7 + tests/test_integration.py | 38 ++++ 8 files changed, 518 insertions(+), 10 deletions(-) diff --git a/Makefile b/Makefile index c693ab7fd7..ea0a3e82a8 100644 --- a/Makefile +++ b/Makefile @@ -17,7 +17,7 @@ install: pip install poetry - poetry install -E pyarrow -E hive -E s3fs -E glue -E adlfs -E duckdb + poetry install -E pyarrow -E hive -E s3fs -E glue -E adlfs -E duckdb -E ray check-license: ./dev/check-license @@ -36,7 +36,7 @@ test-integration: docker-compose -f dev/docker-compose-integration.yml kill docker-compose -f dev/docker-compose-integration.yml build docker-compose -f dev/docker-compose-integration.yml up -d - sleep 20 + sleep 30 poetry run pytest tests/ -m integration ${PYTEST_ARGS} test-adlfs: diff --git a/dev/provision.py b/dev/provision.py index ec84e87a23..1e6f5b7319 100644 --- a/dev/provision.py +++ b/dev/provision.py @@ -17,6 +17,7 @@ import time from pyspark.sql import SparkSession +from pyspark.sql.functions import current_date, date_add, expr spark = SparkSession.builder.getOrCreate() @@ -56,6 +57,12 @@ """ ) +spark.sql( + """ + DROP TABLE IF EXISTS test_null_nan_rewritten; +""" +) + spark.sql( """ CREATE TABLE test_null_nan_rewritten @@ -94,5 +101,28 @@ """ ) +all_types_dataframe = ( + spark.range(0, 5, 1, 5) + .withColumnRenamed("id", "longCol") + .withColumn("intCol", expr("CAST(longCol AS INT)")) + .withColumn("floatCol", expr("CAST(longCol AS FLOAT)")) + .withColumn("doubleCol", expr("CAST(longCol AS DOUBLE)")) + .withColumn("dateCol", date_add(current_date(), 1)) + .withColumn("timestampCol", expr("TO_TIMESTAMP(dateCol)")) + .withColumn("stringCol", expr("CAST(dateCol AS STRING)")) + .withColumn("booleanCol", expr("longCol > 5")) + .withColumn("binaryCol", expr("CAST(longCol AS BINARY)")) + .withColumn("byteCol", expr("CAST(longCol AS BYTE)")) + .withColumn("decimalCol", expr("CAST(longCol AS DECIMAL(10, 2))")) + .withColumn("shortCol", expr("CAST(longCol AS SHORT)")) + .withColumn("mapCol", expr("MAP(longCol, decimalCol)")) + .withColumn("arrayCol", expr("ARRAY(longCol)")) + .withColumn("structCol", expr("STRUCT(mapCol, arrayCol)")) +) + +all_types_dataframe.writeTo("default.test_all_types").tableProperty("format-version", "2").partitionedBy( + "intCol" +).createOrReplace() + while True: time.sleep(1) diff --git a/mkdocs/docs/api.md b/mkdocs/docs/api.md index 302c6fd88a..9516516859 100644 --- a/mkdocs/docs/api.md +++ b/mkdocs/docs/api.md @@ -369,3 +369,51 @@ print( (datetime.timedelta(seconds=1581),), ] ``` + +### Ray + +!!! note "Requirements" + This requires [Ray to be installed](index.md). + +A table scan can also be converted into a Ray dataset: + +```python +ray_dataset = table.scan( + row_filter=GreaterThanOrEqual("trip_distance", 10.0), + selected_fields=("VendorID", "tpep_pickup_datetime", "tpep_dropoff_datetime"), +).to_ray() +``` + +This will return a Ray dataset: + +``` +Dataset( + num_blocks=1, + num_rows=1168798, + schema={ + VendorID: int64, + tpep_pickup_datetime: timestamp[us, tz=UTC], + tpep_dropoff_datetime: timestamp[us, tz=UTC] + } +) +``` + +Using [Ray Dataset API](https://docs.ray.io/en/latest/data/api/dataset.html) to interact with the dataset: + +```python +print( + ray_dataset.take(2) +) +[ + { + 'VendorID': 2, + 'tpep_pickup_datetime': datetime.datetime(2008, 12, 31, 23, 23, 50, tzinfo=), + 'tpep_dropoff_datetime': datetime.datetime(2009, 1, 1, 0, 34, 31, tzinfo=) + }, + { + 'VendorID': 2, + 'tpep_pickup_datetime': datetime.datetime(2008, 12, 31, 23, 5, 3, tzinfo=), + 'tpep_dropoff_datetime': datetime.datetime(2009, 1, 1, 16, 10, 18, tzinfo=) + } +] +``` diff --git a/mkdocs/docs/index.md b/mkdocs/docs/index.md index c35196813f..650075b8d3 100644 --- a/mkdocs/docs/index.md +++ b/mkdocs/docs/index.md @@ -49,7 +49,9 @@ You can mix and match optional dependencies depending on your needs: | glue | Support for AWS Glue | | dynamodb | Support for AWS DynamoDB | | pyarrow | PyArrow as a FileIO implementation to interact with the object store | +| pandas | Installs both PyArrow and Pandas | | duckdb | Installs both PyArrow and DuckDB | +| ray | Installs PyArrow, Pandas, and Ray | | s3fs | S3FS as a FileIO implementation to interact with the object store | | adlfs | ADLFS as a FileIO implementation to interact with the object store | | snappy | Support for snappy Avro compression | diff --git a/poetry.lock b/poetry.lock index bf2b9ad15c..d94d0ee155 100644 --- a/poetry.lock +++ b/poetry.lock @@ -1,4 +1,4 @@ -# This file is automatically @generated by Poetry and should not be changed by hand. +# This file is automatically @generated by Poetry 1.4.0 and should not be changed by hand. [[package]] name = "adal" @@ -713,7 +713,7 @@ tox = ["tox"] name = "distlib" version = "0.3.6" description = "Distribution utilities" -category = "dev" +category = "main" optional = false python-versions = "*" files = [ @@ -856,7 +856,7 @@ zstandard = ["zstandard"] name = "filelock" version = "3.9.0" description = "A platform independent file lock." -category = "dev" +category = "main" optional = false python-versions = ">=3.7" files = [ @@ -987,6 +987,125 @@ smb = ["smbprotocol"] ssh = ["paramiko"] tqdm = ["tqdm"] +[[package]] +name = "grpcio" +version = "1.49.1" +description = "HTTP/2-based RPC framework" +category = "main" +optional = true +python-versions = ">=3.7" +files = [ + {file = "grpcio-1.49.1-cp310-cp310-linux_armv7l.whl", hash = "sha256:fd86040232e805b8e6378b2348c928490ee595b058ce9aaa27ed8e4b0f172b20"}, + {file = "grpcio-1.49.1-cp310-cp310-macosx_10_10_x86_64.whl", hash = "sha256:6fd0c9cede9552bf00f8c5791d257d5bf3790d7057b26c59df08be5e7a1e021d"}, + {file = "grpcio-1.49.1-cp310-cp310-manylinux_2_17_aarch64.whl", hash = "sha256:d0d402e158d4e84e49c158cb5204119d55e1baf363ee98d6cb5dce321c3a065d"}, + {file = "grpcio-1.49.1-cp310-cp310-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:822ceec743d42a627e64ea266059a62d214c5a3cdfcd0d7fe2b7a8e4e82527c7"}, + {file = "grpcio-1.49.1-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:2106d9c16527f0a85e2eea6e6b91a74fc99579c60dd810d8690843ea02bc0f5f"}, + {file = "grpcio-1.49.1-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:52dd02b7e7868233c571b49bc38ebd347c3bb1ff8907bb0cb74cb5f00c790afc"}, + {file = "grpcio-1.49.1-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:120fecba2ec5d14b5a15d11063b39783fda8dc8d24addd83196acb6582cabd9b"}, + {file = "grpcio-1.49.1-cp310-cp310-win32.whl", hash = "sha256:f1a3b88e3c53c1a6e6bed635ec1bbb92201bb6a1f2db186179f7f3f244829788"}, + {file = "grpcio-1.49.1-cp310-cp310-win_amd64.whl", hash = "sha256:a7d0017b92d3850abea87c1bdec6ea41104e71c77bca44c3e17f175c6700af62"}, + {file = "grpcio-1.49.1-cp311-cp311-linux_armv7l.whl", hash = "sha256:9fb17ff8c0d56099ac6ebfa84f670c5a62228d6b5c695cf21c02160c2ac1446b"}, + {file = "grpcio-1.49.1-cp311-cp311-macosx_10_10_x86_64.whl", hash = "sha256:075f2d06e3db6b48a2157a1bcd52d6cbdca980dd18988fe6afdb41795d51625f"}, + {file = "grpcio-1.49.1-cp311-cp311-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:46d93a1b4572b461a227f1db6b8d35a88952db1c47e5fadcf8b8a2f0e1dd9201"}, + {file = "grpcio-1.49.1-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:dc79b2b37d779ac42341ddef40ad5bf0966a64af412c89fc2b062e3ddabb093f"}, + {file = "grpcio-1.49.1-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:5f8b3a971c7820ea9878f3fd70086240a36aeee15d1b7e9ecbc2743b0e785568"}, + {file = "grpcio-1.49.1-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:49b301740cf5bc8fed4fee4c877570189ae3951432d79fa8e524b09353659811"}, + {file = "grpcio-1.49.1-cp311-cp311-win32.whl", hash = "sha256:1c66a25afc6c71d357867b341da594a5587db5849b48f4b7d5908d236bb62ede"}, + {file = "grpcio-1.49.1-cp311-cp311-win_amd64.whl", hash = "sha256:6b6c3a95d27846f4145d6967899b3ab25fffc6ae99544415e1adcacef84842d2"}, + {file = "grpcio-1.49.1-cp37-cp37m-linux_armv7l.whl", hash = "sha256:1cc400c8a2173d1c042997d98a9563e12d9bb3fb6ad36b7f355bc77c7663b8af"}, + {file = "grpcio-1.49.1-cp37-cp37m-macosx_10_10_x86_64.whl", hash = "sha256:34f736bd4d0deae90015c0e383885b431444fe6b6c591dea288173df20603146"}, + {file = "grpcio-1.49.1-cp37-cp37m-manylinux_2_17_aarch64.whl", hash = "sha256:196082b9c89ebf0961dcd77cb114bed8171964c8e3063b9da2fb33536a6938ed"}, + {file = "grpcio-1.49.1-cp37-cp37m-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:8c9f89c42749890618cd3c2464e1fbf88446e3d2f67f1e334c8e5db2f3272bbd"}, + {file = "grpcio-1.49.1-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:64419cb8a5b612cdb1550c2fd4acbb7d4fb263556cf4625f25522337e461509e"}, + {file = "grpcio-1.49.1-cp37-cp37m-musllinux_1_1_i686.whl", hash = "sha256:8a5272061826e6164f96e3255405ef6f73b88fd3e8bef464c7d061af8585ac62"}, + {file = "grpcio-1.49.1-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:ea9d0172445241ad7cb49577314e39d0af2c5267395b3561d7ced5d70458a9f3"}, + {file = "grpcio-1.49.1-cp37-cp37m-win32.whl", hash = "sha256:2070e87d95991473244c72d96d13596c751cb35558e11f5df5414981e7ed2492"}, + {file = "grpcio-1.49.1-cp37-cp37m-win_amd64.whl", hash = "sha256:4fcedcab49baaa9db4a2d240ac81f2d57eb0052b1c6a9501b46b8ae912720fbf"}, + {file = "grpcio-1.49.1-cp38-cp38-linux_armv7l.whl", hash = "sha256:afbb3475cf7f4f7d380c2ca37ee826e51974f3e2665613996a91d6a58583a534"}, + {file = "grpcio-1.49.1-cp38-cp38-macosx_10_10_x86_64.whl", hash = "sha256:a4f9ba141380abde6c3adc1727f21529137a2552002243fa87c41a07e528245c"}, + {file = "grpcio-1.49.1-cp38-cp38-manylinux_2_17_aarch64.whl", hash = "sha256:cf0a1fb18a7204b9c44623dfbd1465b363236ce70c7a4ed30402f9f60d8b743b"}, + {file = "grpcio-1.49.1-cp38-cp38-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:17bb6fe72784b630728c6cff9c9d10ccc3b6d04e85da6e0a7b27fb1d135fac62"}, + {file = "grpcio-1.49.1-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:18305d5a082d1593b005a895c10041f833b16788e88b02bb81061f5ebcc465df"}, + {file = "grpcio-1.49.1-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:b6a1b39e59ac5a3067794a0e498911cf2e37e4b19ee9e9977dc5e7051714f13f"}, + {file = "grpcio-1.49.1-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:0e20d59aafc086b1cc68400463bddda6e41d3e5ed30851d1e2e0f6a2e7e342d3"}, + {file = "grpcio-1.49.1-cp38-cp38-win32.whl", hash = "sha256:e1e83233d4680863a421f3ee4a7a9b80d33cd27ee9ed7593bc93f6128302d3f2"}, + {file = "grpcio-1.49.1-cp38-cp38-win_amd64.whl", hash = "sha256:221d42c654d2a41fa31323216279c73ed17d92f533bc140a3390cc1bd78bf63c"}, + {file = "grpcio-1.49.1-cp39-cp39-linux_armv7l.whl", hash = "sha256:fa9e6e61391e99708ac87fc3436f6b7b9c6b845dc4639b406e5e61901e1aacde"}, + {file = "grpcio-1.49.1-cp39-cp39-macosx_10_10_x86_64.whl", hash = "sha256:9b449e966ef518ce9c860d21f8afe0b0f055220d95bc710301752ac1db96dd6a"}, + {file = "grpcio-1.49.1-cp39-cp39-manylinux_2_17_aarch64.whl", hash = "sha256:aa34d2ad9f24e47fa9a3172801c676e4037d862247e39030165fe83821a7aafd"}, + {file = "grpcio-1.49.1-cp39-cp39-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:5207f4eed1b775d264fcfe379d8541e1c43b878f2b63c0698f8f5c56c40f3d68"}, + {file = "grpcio-1.49.1-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:0b24a74651438d45619ac67004638856f76cc13d78b7478f2457754cbcb1c8ad"}, + {file = "grpcio-1.49.1-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:fe763781669790dc8b9618e7e677c839c87eae6cf28b655ee1fa69ae04eea03f"}, + {file = "grpcio-1.49.1-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:2f2ff7ba0f8f431f32d4b4bc3a3713426949d3533b08466c4ff1b2b475932ca8"}, + {file = "grpcio-1.49.1-cp39-cp39-win32.whl", hash = "sha256:08ff74aec8ff457a89b97152d36cb811dcc1d17cd5a92a65933524e363327394"}, + {file = "grpcio-1.49.1-cp39-cp39-win_amd64.whl", hash = "sha256:274ffbb39717918c514b35176510ae9be06e1d93121e84d50b350861dcb9a705"}, + {file = "grpcio-1.49.1.tar.gz", hash = "sha256:d4725fc9ec8e8822906ae26bb26f5546891aa7fbc3443de970cc556d43a5c99f"}, +] + +[package.dependencies] +six = ">=1.5.2" + +[package.extras] +protobuf = ["grpcio-tools (>=1.49.1)"] + +[[package]] +name = "grpcio" +version = "1.51.3" +description = "HTTP/2-based RPC framework" +category = "main" +optional = true +python-versions = ">=3.7" +files = [ + {file = "grpcio-1.51.3-cp310-cp310-linux_armv7l.whl", hash = "sha256:f601aaeae18dab81930fb8d4f916b0da21e89bb4b5f7367ef793f46b4a76b7b0"}, + {file = "grpcio-1.51.3-cp310-cp310-macosx_12_0_universal2.whl", hash = "sha256:eef0450a4b5ed11feab639bf3eb1b6e23d0efa9b911bf7b06fb60e14f5f8a585"}, + {file = "grpcio-1.51.3-cp310-cp310-manylinux_2_17_aarch64.whl", hash = "sha256:82b0ad8ac825d4bb31bff9f638557c045f4a6d824d84b21e893968286f88246b"}, + {file = "grpcio-1.51.3-cp310-cp310-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:3667c06e37d6cd461afdd51cefe6537702f3d1dc5ff4cac07e88d8b4795dc16f"}, + {file = "grpcio-1.51.3-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:3709048fe0aa23dda09b3e69849a12055790171dab9e399a72ea8f9dfbf9ac80"}, + {file = "grpcio-1.51.3-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:200d69857f9910f7458b39b9bcf83ee4a180591b40146ba9e49314e3a7419313"}, + {file = "grpcio-1.51.3-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:cd9a5e68e79c5f031500e67793048a90209711e0854a9ddee8a3ce51728de4e5"}, + {file = "grpcio-1.51.3-cp310-cp310-win32.whl", hash = "sha256:6604f614016127ae10969176bbf12eb0e03d2fb3d643f050b3b69e160d144fb4"}, + {file = "grpcio-1.51.3-cp310-cp310-win_amd64.whl", hash = "sha256:e95c7ccd4c5807adef1602005513bf7c7d14e5a41daebcf9d8d30d8bf51b8f81"}, + {file = "grpcio-1.51.3-cp311-cp311-linux_armv7l.whl", hash = "sha256:5e77ee138100f0bb55cbd147840f87ee6241dbd25f09ea7cd8afe7efff323449"}, + {file = "grpcio-1.51.3-cp311-cp311-macosx_10_10_universal2.whl", hash = "sha256:68a7514b754e38e8de9075f7bb4dee919919515ec68628c43a894027e40ddec4"}, + {file = "grpcio-1.51.3-cp311-cp311-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:3c1b9f8afa62ff265d86a4747a2990ec5a96e4efce5d5888f245a682d66eca47"}, + {file = "grpcio-1.51.3-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:8de30f0b417744288cec65ec8cf84b8a57995cf7f1e84ccad2704d93f05d0aae"}, + {file = "grpcio-1.51.3-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:b69c7adc7ed60da1cb1b502853db61f453fc745f940cbcc25eb97c99965d8f41"}, + {file = "grpcio-1.51.3-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:d81528ffe0e973dc840ec73a4132fd18b8203ad129d7410155d951a0a7e4f5d0"}, + {file = "grpcio-1.51.3-cp311-cp311-win32.whl", hash = "sha256:040eb421613b57c696063abde405916dd830203c184c9000fc8c3b3b3c950325"}, + {file = "grpcio-1.51.3-cp311-cp311-win_amd64.whl", hash = "sha256:2a8e17286c4240137d933b8ca506465472248b4ce0fe46f3404459e708b65b68"}, + {file = "grpcio-1.51.3-cp37-cp37m-linux_armv7l.whl", hash = "sha256:d5cd1389669a847555df54177b911d9ff6f17345b2a6f19388707b7a9f724c88"}, + {file = "grpcio-1.51.3-cp37-cp37m-macosx_10_10_universal2.whl", hash = "sha256:be1bf35ce82cdbcac14e39d5102d8de4079a1c1a6a06b68e41fcd9ef64f9dd28"}, + {file = "grpcio-1.51.3-cp37-cp37m-manylinux_2_17_aarch64.whl", hash = "sha256:5eed34994c095e2bf7194ffac7381c6068b057ef1e69f8f08db77771350a7566"}, + {file = "grpcio-1.51.3-cp37-cp37m-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:3f9a7d88082b2a17ae7bd3c2354d13bab0453899e0851733f6afa6918373f476"}, + {file = "grpcio-1.51.3-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:36c8abbc5f837111e7bd619612eedc223c290b0903b952ce0c7b00840ea70f14"}, + {file = "grpcio-1.51.3-cp37-cp37m-musllinux_1_1_i686.whl", hash = "sha256:165b05af77e6aecb4210ae7663e25acf234ba78a7c1c157fa5f2efeb0d6ec53c"}, + {file = "grpcio-1.51.3-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:54e36c2ee304ff15f2bfbdc43d2b56c63331c52d818c364e5b5214e5bc2ad9f6"}, + {file = "grpcio-1.51.3-cp37-cp37m-win32.whl", hash = "sha256:cd0daac21d9ef5e033a5100c1d3aa055bbed28bfcf070b12d8058045c4e821b1"}, + {file = "grpcio-1.51.3-cp37-cp37m-win_amd64.whl", hash = "sha256:2fdd6333ce96435408565a9dbbd446212cd5d62e4d26f6a3c0feb1e3c35f1cc8"}, + {file = "grpcio-1.51.3-cp38-cp38-linux_armv7l.whl", hash = "sha256:54b0c29bdd9a3b1e1b61443ab152f060fc719f1c083127ab08d03fac5efd51be"}, + {file = "grpcio-1.51.3-cp38-cp38-macosx_10_10_universal2.whl", hash = "sha256:ffaaf7e93fcb437356b5a4b23bf36e8a3d0221399ff77fd057e4bc77776a24be"}, + {file = "grpcio-1.51.3-cp38-cp38-manylinux_2_17_aarch64.whl", hash = "sha256:eafbe7501a3268d05f2e450e1ddaffb950d842a8620c13ec328b501d25d2e2c3"}, + {file = "grpcio-1.51.3-cp38-cp38-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:881ecb34feabf31c6b3b9bbbddd1a5b57e69f805041e5a2c6c562a28574f71c4"}, + {file = "grpcio-1.51.3-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:e860a3222139b41d430939bbec2ec9c3f6c740938bf7a04471a9a8caaa965a2e"}, + {file = "grpcio-1.51.3-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:49ede0528e9dac7e8a9fe30b16c73b630ddd9a576bf4b675eb6b0c53ee5ca00f"}, + {file = "grpcio-1.51.3-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:6972b009638b40a448d10e1bc18e2223143b8a7aa20d7def0d78dd4af4126d12"}, + {file = "grpcio-1.51.3-cp38-cp38-win32.whl", hash = "sha256:5694448256e3cdfe5bd358f1574a3f2f51afa20cc834713c4b9788d60b7cc646"}, + {file = "grpcio-1.51.3-cp38-cp38-win_amd64.whl", hash = "sha256:3ea4341efe603b049e8c9a5f13c696ca37fcdf8a23ca35f650428ad3606381d9"}, + {file = "grpcio-1.51.3-cp39-cp39-linux_armv7l.whl", hash = "sha256:6c677581ce129f5fa228b8f418cee10bd28dd449f3a544ea73c8ba590ee49d0b"}, + {file = "grpcio-1.51.3-cp39-cp39-macosx_10_10_universal2.whl", hash = "sha256:30e09b5e0531685e176f49679b6a3b190762cc225f4565e55a899f5e14b3aa62"}, + {file = "grpcio-1.51.3-cp39-cp39-manylinux_2_17_aarch64.whl", hash = "sha256:c831f31336e81243f85b6daff3e5e8a123302ce0ea1f2726ad752fd7a59f3aee"}, + {file = "grpcio-1.51.3-cp39-cp39-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:2cd2e4cefb724cab1ba2df4b7535a9980531b9ec51b4dbb5f137a1f3a3754ef0"}, + {file = "grpcio-1.51.3-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:f7a0d0bf44438869d307f85a54f25a896ad6b4b0ca12370f76892ad732928d87"}, + {file = "grpcio-1.51.3-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:c02abd55409bfb293371554adf6a4401197ec2133dd97727c01180889014ba4d"}, + {file = "grpcio-1.51.3-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:2f8ff75e61e1227ba7a3f16b2eadbcc11d0a54096d52ab75a6b88cfbe56f55d1"}, + {file = "grpcio-1.51.3-cp39-cp39-win32.whl", hash = "sha256:6c99a73a6260bdf844b2e5ddad02dcd530310f80e1fa72c300fa19c1c7496962"}, + {file = "grpcio-1.51.3-cp39-cp39-win_amd64.whl", hash = "sha256:22bdfac4f7f27acdd4da359b5e7e1973dc74bf1ed406729b07d0759fde2f064b"}, + {file = "grpcio-1.51.3.tar.gz", hash = "sha256:be7b2265b7527bb12109a7727581e274170766d5b3c9258d4e466f4872522d7a"}, +] + +[package.extras] +protobuf = ["grpcio-tools (>=1.51.3)"] + [[package]] name = "identify" version = "2.5.18" @@ -1034,6 +1153,25 @@ docs = ["furo", "jaraco.packaging (>=9)", "jaraco.tidelift (>=1.4)", "rst.linker perf = ["ipython"] testing = ["flake8 (<5)", "flufl.flake8", "importlib-resources (>=1.3)", "packaging", "pyfakefs", "pytest (>=6)", "pytest-black (>=0.3.7)", "pytest-checkdocs (>=2.4)", "pytest-cov", "pytest-enabler (>=1.3)", "pytest-flake8", "pytest-mypy (>=0.9.1)", "pytest-perf (>=0.9.2)"] +[[package]] +name = "importlib-resources" +version = "5.12.0" +description = "Read resources from Python packages" +category = "main" +optional = true +python-versions = ">=3.7" +files = [ + {file = "importlib_resources-5.12.0-py3-none-any.whl", hash = "sha256:7b1deeebbf351c7578e09bf2f63fa2ce8b5ffec296e0d349139d43cca061a81a"}, + {file = "importlib_resources-5.12.0.tar.gz", hash = "sha256:4be82589bf5c1d7999aedf2a45159d10cb3ca4f19b2271f8792bc8e6da7b22f6"}, +] + +[package.dependencies] +zipp = {version = ">=3.1.0", markers = "python_version < \"3.10\""} + +[package.extras] +docs = ["furo", "jaraco.packaging (>=9)", "jaraco.tidelift (>=1.4)", "rst.linker (>=1.9)", "sphinx (>=3.5)", "sphinx-lint"] +testing = ["flake8 (<5)", "pytest (>=6)", "pytest-black (>=0.3.7)", "pytest-checkdocs (>=2.4)", "pytest-cov", "pytest-enabler (>=1.3)", "pytest-flake8", "pytest-mypy (>=0.9.1)"] + [[package]] name = "iniconfig" version = "2.0.0" @@ -1091,6 +1229,28 @@ files = [ {file = "jmespath-1.0.1.tar.gz", hash = "sha256:90261b206d6defd58fdd5e85f478bf633a2901798906be2ad389150c5c60edbe"}, ] +[[package]] +name = "jsonschema" +version = "4.17.3" +description = "An implementation of JSON Schema validation for Python" +category = "main" +optional = true +python-versions = ">=3.7" +files = [ + {file = "jsonschema-4.17.3-py3-none-any.whl", hash = "sha256:a870ad254da1a8ca84b6a2905cac29d265f805acc57af304784962a2aa6508f6"}, + {file = "jsonschema-4.17.3.tar.gz", hash = "sha256:0f864437ab8b6076ba6707453ef8f98a6a0d512a80e93f8abdb676f737ecb60d"}, +] + +[package.dependencies] +attrs = ">=17.4.0" +importlib-resources = {version = ">=1.4.0", markers = "python_version < \"3.9\""} +pkgutil-resolve-name = {version = ">=1.3.10", markers = "python_version < \"3.9\""} +pyrsistent = ">=0.14.0,<0.17.0 || >0.17.0,<0.17.1 || >0.17.1,<0.17.2 || >0.17.2" + +[package.extras] +format = ["fqdn", "idna", "isoduration", "jsonpointer (>1.13)", "rfc3339-validator", "rfc3987", "uri-template", "webcolors (>=1.11)"] +format-nongpl = ["fqdn", "idna", "isoduration", "jsonpointer (>1.13)", "rfc3339-validator", "rfc3986-validator (>0.1.0)", "uri-template", "webcolors (>=1.11)"] + [[package]] name = "markdown-it-py" version = "2.2.0" @@ -1318,6 +1478,79 @@ portalocker = [ {version = ">=1.6,<3", markers = "python_version >= \"3.5\" and platform_system == \"Windows\""}, ] +[[package]] +name = "msgpack" +version = "1.0.5" +description = "MessagePack serializer" +category = "main" +optional = true +python-versions = "*" +files = [ + {file = "msgpack-1.0.5-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:525228efd79bb831cf6830a732e2e80bc1b05436b086d4264814b4b2955b2fa9"}, + {file = "msgpack-1.0.5-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:4f8d8b3bf1ff2672567d6b5c725a1b347fe838b912772aa8ae2bf70338d5a198"}, + {file = "msgpack-1.0.5-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:cdc793c50be3f01106245a61b739328f7dccc2c648b501e237f0699fe1395b81"}, + {file = "msgpack-1.0.5-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:5cb47c21a8a65b165ce29f2bec852790cbc04936f502966768e4aae9fa763cb7"}, + {file = "msgpack-1.0.5-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:e42b9594cc3bf4d838d67d6ed62b9e59e201862a25e9a157019e171fbe672dd3"}, + {file = "msgpack-1.0.5-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:55b56a24893105dc52c1253649b60f475f36b3aa0fc66115bffafb624d7cb30b"}, + {file = "msgpack-1.0.5-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:1967f6129fc50a43bfe0951c35acbb729be89a55d849fab7686004da85103f1c"}, + {file = "msgpack-1.0.5-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:20a97bf595a232c3ee6d57ddaadd5453d174a52594bf9c21d10407e2a2d9b3bd"}, + {file = "msgpack-1.0.5-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:d25dd59bbbbb996eacf7be6b4ad082ed7eacc4e8f3d2df1ba43822da9bfa122a"}, + {file = "msgpack-1.0.5-cp310-cp310-win32.whl", hash = "sha256:382b2c77589331f2cb80b67cc058c00f225e19827dbc818d700f61513ab47bea"}, + {file = "msgpack-1.0.5-cp310-cp310-win_amd64.whl", hash = "sha256:4867aa2df9e2a5fa5f76d7d5565d25ec76e84c106b55509e78c1ede0f152659a"}, + {file = "msgpack-1.0.5-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:9f5ae84c5c8a857ec44dc180a8b0cc08238e021f57abdf51a8182e915e6299f0"}, + {file = "msgpack-1.0.5-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:9e6ca5d5699bcd89ae605c150aee83b5321f2115695e741b99618f4856c50898"}, + {file = "msgpack-1.0.5-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:5494ea30d517a3576749cad32fa27f7585c65f5f38309c88c6d137877fa28a5a"}, + {file = "msgpack-1.0.5-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:1ab2f3331cb1b54165976a9d976cb251a83183631c88076613c6c780f0d6e45a"}, + {file = "msgpack-1.0.5-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:28592e20bbb1620848256ebc105fc420436af59515793ed27d5c77a217477705"}, + {file = "msgpack-1.0.5-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:fe5c63197c55bce6385d9aee16c4d0641684628f63ace85f73571e65ad1c1e8d"}, + {file = "msgpack-1.0.5-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:ed40e926fa2f297e8a653c954b732f125ef97bdd4c889f243182299de27e2aa9"}, + {file = "msgpack-1.0.5-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:b2de4c1c0538dcb7010902a2b97f4e00fc4ddf2c8cda9749af0e594d3b7fa3d7"}, + {file = "msgpack-1.0.5-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:bf22a83f973b50f9d38e55c6aade04c41ddda19b00c4ebc558930d78eecc64ed"}, + {file = "msgpack-1.0.5-cp311-cp311-win32.whl", hash = "sha256:c396e2cc213d12ce017b686e0f53497f94f8ba2b24799c25d913d46c08ec422c"}, + {file = "msgpack-1.0.5-cp311-cp311-win_amd64.whl", hash = "sha256:6c4c68d87497f66f96d50142a2b73b97972130d93677ce930718f68828b382e2"}, + {file = "msgpack-1.0.5-cp36-cp36m-macosx_10_9_x86_64.whl", hash = "sha256:a2b031c2e9b9af485d5e3c4520f4220d74f4d222a5b8dc8c1a3ab9448ca79c57"}, + {file = "msgpack-1.0.5-cp36-cp36m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:4f837b93669ce4336e24d08286c38761132bc7ab29782727f8557e1eb21b2080"}, + {file = "msgpack-1.0.5-cp36-cp36m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:b1d46dfe3832660f53b13b925d4e0fa1432b00f5f7210eb3ad3bb9a13c6204a6"}, + {file = "msgpack-1.0.5-cp36-cp36m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:366c9a7b9057e1547f4ad51d8facad8b406bab69c7d72c0eb6f529cf76d4b85f"}, + {file = "msgpack-1.0.5-cp36-cp36m-musllinux_1_1_aarch64.whl", hash = "sha256:4c075728a1095efd0634a7dccb06204919a2f67d1893b6aa8e00497258bf926c"}, + {file = "msgpack-1.0.5-cp36-cp36m-musllinux_1_1_i686.whl", hash = "sha256:f933bbda5a3ee63b8834179096923b094b76f0c7a73c1cfe8f07ad608c58844b"}, + {file = "msgpack-1.0.5-cp36-cp36m-musllinux_1_1_x86_64.whl", hash = "sha256:36961b0568c36027c76e2ae3ca1132e35123dcec0706c4b7992683cc26c1320c"}, + {file = "msgpack-1.0.5-cp36-cp36m-win32.whl", hash = "sha256:b5ef2f015b95f912c2fcab19c36814963b5463f1fb9049846994b007962743e9"}, + {file = "msgpack-1.0.5-cp36-cp36m-win_amd64.whl", hash = "sha256:288e32b47e67f7b171f86b030e527e302c91bd3f40fd9033483f2cacc37f327a"}, + {file = "msgpack-1.0.5-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:137850656634abddfb88236008339fdaba3178f4751b28f270d2ebe77a563b6c"}, + {file = "msgpack-1.0.5-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:0c05a4a96585525916b109bb85f8cb6511db1c6f5b9d9cbcbc940dc6b4be944b"}, + {file = "msgpack-1.0.5-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:56a62ec00b636583e5cb6ad313bbed36bb7ead5fa3a3e38938503142c72cba4f"}, + {file = "msgpack-1.0.5-cp37-cp37m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:ef8108f8dedf204bb7b42994abf93882da1159728a2d4c5e82012edd92c9da9f"}, + {file = "msgpack-1.0.5-cp37-cp37m-musllinux_1_1_aarch64.whl", hash = "sha256:1835c84d65f46900920b3708f5ba829fb19b1096c1800ad60bae8418652a951d"}, + {file = "msgpack-1.0.5-cp37-cp37m-musllinux_1_1_i686.whl", hash = "sha256:e57916ef1bd0fee4f21c4600e9d1da352d8816b52a599c46460e93a6e9f17086"}, + {file = "msgpack-1.0.5-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:17358523b85973e5f242ad74aa4712b7ee560715562554aa2134d96e7aa4cbbf"}, + {file = "msgpack-1.0.5-cp37-cp37m-win32.whl", hash = "sha256:cb5aaa8c17760909ec6cb15e744c3ebc2ca8918e727216e79607b7bbce9c8f77"}, + {file = "msgpack-1.0.5-cp37-cp37m-win_amd64.whl", hash = "sha256:ab31e908d8424d55601ad7075e471b7d0140d4d3dd3272daf39c5c19d936bd82"}, + {file = "msgpack-1.0.5-cp38-cp38-macosx_10_9_universal2.whl", hash = "sha256:b72d0698f86e8d9ddf9442bdedec15b71df3598199ba33322d9711a19f08145c"}, + {file = "msgpack-1.0.5-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:379026812e49258016dd84ad79ac8446922234d498058ae1d415f04b522d5b2d"}, + {file = "msgpack-1.0.5-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:332360ff25469c346a1c5e47cbe2a725517919892eda5cfaffe6046656f0b7bb"}, + {file = "msgpack-1.0.5-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:476a8fe8fae289fdf273d6d2a6cb6e35b5a58541693e8f9f019bfe990a51e4ba"}, + {file = "msgpack-1.0.5-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:a9985b214f33311df47e274eb788a5893a761d025e2b92c723ba4c63936b69b1"}, + {file = "msgpack-1.0.5-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:48296af57cdb1d885843afd73c4656be5c76c0c6328db3440c9601a98f303d87"}, + {file = "msgpack-1.0.5-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:addab7e2e1fcc04bd08e4eb631c2a90960c340e40dfc4a5e24d2ff0d5a3b3edb"}, + {file = "msgpack-1.0.5-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:916723458c25dfb77ff07f4c66aed34e47503b2eb3188b3adbec8d8aa6e00f48"}, + {file = "msgpack-1.0.5-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:821c7e677cc6acf0fd3f7ac664c98803827ae6de594a9f99563e48c5a2f27eb0"}, + {file = "msgpack-1.0.5-cp38-cp38-win32.whl", hash = "sha256:1c0f7c47f0087ffda62961d425e4407961a7ffd2aa004c81b9c07d9269512f6e"}, + {file = "msgpack-1.0.5-cp38-cp38-win_amd64.whl", hash = "sha256:bae7de2026cbfe3782c8b78b0db9cbfc5455e079f1937cb0ab8d133496ac55e1"}, + {file = "msgpack-1.0.5-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:20c784e66b613c7f16f632e7b5e8a1651aa5702463d61394671ba07b2fc9e025"}, + {file = "msgpack-1.0.5-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:266fa4202c0eb94d26822d9bfd7af25d1e2c088927fe8de9033d929dd5ba24c5"}, + {file = "msgpack-1.0.5-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:18334484eafc2b1aa47a6d42427da7fa8f2ab3d60b674120bce7a895a0a85bdd"}, + {file = "msgpack-1.0.5-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:57e1f3528bd95cc44684beda696f74d3aaa8a5e58c816214b9046512240ef437"}, + {file = "msgpack-1.0.5-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:586d0d636f9a628ddc6a17bfd45aa5b5efaf1606d2b60fa5d87b8986326e933f"}, + {file = "msgpack-1.0.5-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:a740fa0e4087a734455f0fc3abf5e746004c9da72fbd541e9b113013c8dc3282"}, + {file = "msgpack-1.0.5-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:3055b0455e45810820db1f29d900bf39466df96ddca11dfa6d074fa47054376d"}, + {file = "msgpack-1.0.5-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:a61215eac016f391129a013c9e46f3ab308db5f5ec9f25811e811f96962599a8"}, + {file = "msgpack-1.0.5-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:362d9655cd369b08fda06b6657a303eb7172d5279997abe094512e919cf74b11"}, + {file = "msgpack-1.0.5-cp39-cp39-win32.whl", hash = "sha256:ac9dd47af78cae935901a9a500104e2dea2e253207c924cc95de149606dc43cc"}, + {file = "msgpack-1.0.5-cp39-cp39-win_amd64.whl", hash = "sha256:06f5174b5f8ed0ed919da0e62cbd4ffde676a374aba4020034da05fab67b9164"}, + {file = "msgpack-1.0.5.tar.gz", hash = "sha256:c075544284eadc5cddc70f4757331d99dcbc16b2bbd4849d15f8aae4cf36d31c"}, +] + [[package]] name = "msrest" version = "0.7.1" @@ -1498,7 +1731,7 @@ signedtoken = ["cryptography (>=3.0.0)", "pyjwt (>=2.0.0,<3)"] name = "packaging" version = "23.0" description = "Core utilities for Python packages" -category = "dev" +category = "main" optional = false python-versions = ">=3.7" files = [ @@ -1555,11 +1788,23 @@ pytz = ">=2020.1" [package.extras] test = ["hypothesis (>=5.5.3)", "pytest (>=6.0)", "pytest-xdist (>=1.31)"] +[[package]] +name = "pkgutil-resolve-name" +version = "1.3.10" +description = "Resolve a name to an object." +category = "main" +optional = true +python-versions = ">=3.6" +files = [ + {file = "pkgutil_resolve_name-1.3.10-py3-none-any.whl", hash = "sha256:ca27cc078d25c5ad71a9de0a7a330146c4e014c2462d9af19c6b828280649c5e"}, + {file = "pkgutil_resolve_name-1.3.10.tar.gz", hash = "sha256:357d6c9e6a755653cfd78893817c0853af365dd51ec97f3d358a819373bbd174"}, +] + [[package]] name = "platformdirs" version = "3.0.0" description = "A small Python package for determining appropriate platform-specific dirs, e.g. a \"user data dir\"." -category = "dev" +category = "main" optional = false python-versions = ">=3.7" files = [ @@ -1626,6 +1871,29 @@ nodeenv = ">=0.11.1" pyyaml = ">=5.1" virtualenv = ">=20.10.0" +[[package]] +name = "protobuf" +version = "4.22.1" +description = "" +category = "main" +optional = true +python-versions = ">=3.7" +files = [ + {file = "protobuf-4.22.1-cp310-abi3-win32.whl", hash = "sha256:85aa9acc5a777adc0c21b449dafbc40d9a0b6413ff3a4f77ef9df194be7f975b"}, + {file = "protobuf-4.22.1-cp310-abi3-win_amd64.whl", hash = "sha256:8bc971d76c03f1dd49f18115b002254f2ddb2d4b143c583bb860b796bb0d399e"}, + {file = "protobuf-4.22.1-cp37-abi3-macosx_10_9_universal2.whl", hash = "sha256:5917412347e1da08ce2939eb5cd60650dfb1a9ab4606a415b9278a1041fb4d19"}, + {file = "protobuf-4.22.1-cp37-abi3-manylinux2014_aarch64.whl", hash = "sha256:9e12e2810e7d297dbce3c129ae5e912ffd94240b050d33f9ecf023f35563b14f"}, + {file = "protobuf-4.22.1-cp37-abi3-manylinux2014_x86_64.whl", hash = "sha256:953fc7904ef46900262a26374b28c2864610b60cdc8b272f864e22143f8373c4"}, + {file = "protobuf-4.22.1-cp37-cp37m-win32.whl", hash = "sha256:6e100f7bc787cd0a0ae58dbf0ab8bbf1ee7953f862b89148b6cf5436d5e9eaa1"}, + {file = "protobuf-4.22.1-cp37-cp37m-win_amd64.whl", hash = "sha256:87a6393fa634f294bf24d1cfe9fdd6bb605cbc247af81b9b10c4c0f12dfce4b3"}, + {file = "protobuf-4.22.1-cp38-cp38-win32.whl", hash = "sha256:e3fb58076bdb550e75db06ace2a8b3879d4c4f7ec9dd86e4254656118f4a78d7"}, + {file = "protobuf-4.22.1-cp38-cp38-win_amd64.whl", hash = "sha256:651113695bc2e5678b799ee5d906b5d3613f4ccfa61b12252cfceb6404558af0"}, + {file = "protobuf-4.22.1-cp39-cp39-win32.whl", hash = "sha256:67b7d19da0fda2733702c2299fd1ef6cb4b3d99f09263eacaf1aa151d9d05f02"}, + {file = "protobuf-4.22.1-cp39-cp39-win_amd64.whl", hash = "sha256:b8700792f88e59ccecfa246fa48f689d6eee6900eddd486cdae908ff706c482b"}, + {file = "protobuf-4.22.1-py3-none-any.whl", hash = "sha256:3e19dcf4adbf608924d3486ece469dd4f4f2cf7d2649900f0efcd1a84e8fd3ba"}, + {file = "protobuf-4.22.1.tar.gz", hash = "sha256:dce7a55d501c31ecf688adb2f6c3f763cf11bc0be815d1946a84d74772ab07a7"}, +] + [[package]] name = "pyarrow" version = "11.0.0" @@ -1795,6 +2063,43 @@ files = [ [package.dependencies] tomli = {version = ">=1.1.0", markers = "python_version < \"3.11\""} +[[package]] +name = "pyrsistent" +version = "0.19.3" +description = "Persistent/Functional/Immutable data structures" +category = "main" +optional = true +python-versions = ">=3.7" +files = [ + {file = "pyrsistent-0.19.3-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:20460ac0ea439a3e79caa1dbd560344b64ed75e85d8703943e0b66c2a6150e4a"}, + {file = "pyrsistent-0.19.3-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:4c18264cb84b5e68e7085a43723f9e4c1fd1d935ab240ce02c0324a8e01ccb64"}, + {file = "pyrsistent-0.19.3-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:4b774f9288dda8d425adb6544e5903f1fb6c273ab3128a355c6b972b7df39dcf"}, + {file = "pyrsistent-0.19.3-cp310-cp310-win32.whl", hash = "sha256:5a474fb80f5e0d6c9394d8db0fc19e90fa540b82ee52dba7d246a7791712f74a"}, + {file = "pyrsistent-0.19.3-cp310-cp310-win_amd64.whl", hash = "sha256:49c32f216c17148695ca0e02a5c521e28a4ee6c5089f97e34fe24163113722da"}, + {file = "pyrsistent-0.19.3-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:f0774bf48631f3a20471dd7c5989657b639fd2d285b861237ea9e82c36a415a9"}, + {file = "pyrsistent-0.19.3-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:3ab2204234c0ecd8b9368dbd6a53e83c3d4f3cab10ecaf6d0e772f456c442393"}, + {file = "pyrsistent-0.19.3-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:e42296a09e83028b3476f7073fcb69ffebac0e66dbbfd1bd847d61f74db30f19"}, + {file = "pyrsistent-0.19.3-cp311-cp311-win32.whl", hash = "sha256:64220c429e42a7150f4bfd280f6f4bb2850f95956bde93c6fda1b70507af6ef3"}, + {file = "pyrsistent-0.19.3-cp311-cp311-win_amd64.whl", hash = "sha256:016ad1afadf318eb7911baa24b049909f7f3bb2c5b1ed7b6a8f21db21ea3faa8"}, + {file = "pyrsistent-0.19.3-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:c4db1bd596fefd66b296a3d5d943c94f4fac5bcd13e99bffe2ba6a759d959a28"}, + {file = "pyrsistent-0.19.3-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:aeda827381f5e5d65cced3024126529ddc4289d944f75e090572c77ceb19adbf"}, + {file = "pyrsistent-0.19.3-cp37-cp37m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:42ac0b2f44607eb92ae88609eda931a4f0dfa03038c44c772e07f43e738bcac9"}, + {file = "pyrsistent-0.19.3-cp37-cp37m-win32.whl", hash = "sha256:e8f2b814a3dc6225964fa03d8582c6e0b6650d68a232df41e3cc1b66a5d2f8d1"}, + {file = "pyrsistent-0.19.3-cp37-cp37m-win_amd64.whl", hash = "sha256:c9bb60a40a0ab9aba40a59f68214eed5a29c6274c83b2cc206a359c4a89fa41b"}, + {file = "pyrsistent-0.19.3-cp38-cp38-macosx_10_9_universal2.whl", hash = "sha256:a2471f3f8693101975b1ff85ffd19bb7ca7dd7c38f8a81701f67d6b4f97b87d8"}, + {file = "pyrsistent-0.19.3-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:cc5d149f31706762c1f8bda2e8c4f8fead6e80312e3692619a75301d3dbb819a"}, + {file = "pyrsistent-0.19.3-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:3311cb4237a341aa52ab8448c27e3a9931e2ee09561ad150ba94e4cfd3fc888c"}, + {file = "pyrsistent-0.19.3-cp38-cp38-win32.whl", hash = "sha256:f0e7c4b2f77593871e918be000b96c8107da48444d57005b6a6bc61fb4331b2c"}, + {file = "pyrsistent-0.19.3-cp38-cp38-win_amd64.whl", hash = "sha256:c147257a92374fde8498491f53ffa8f4822cd70c0d85037e09028e478cababb7"}, + {file = "pyrsistent-0.19.3-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:b735e538f74ec31378f5a1e3886a26d2ca6351106b4dfde376a26fc32a044edc"}, + {file = "pyrsistent-0.19.3-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:99abb85579e2165bd8522f0c0138864da97847875ecbd45f3e7e2af569bfc6f2"}, + {file = "pyrsistent-0.19.3-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:3a8cb235fa6d3fd7aae6a4f1429bbb1fec1577d978098da1252f0489937786f3"}, + {file = "pyrsistent-0.19.3-cp39-cp39-win32.whl", hash = "sha256:c74bed51f9b41c48366a286395c67f4e894374306b197e62810e0fdaf2364da2"}, + {file = "pyrsistent-0.19.3-cp39-cp39-win_amd64.whl", hash = "sha256:878433581fc23e906d947a6814336eee031a00e6defba224234169ae3d3d6a98"}, + {file = "pyrsistent-0.19.3-py3-none-any.whl", hash = "sha256:ccf0d6bd208f8111179f0c26fdf84ed7c3891982f2edaeae7422575f47e66b64"}, + {file = "pyrsistent-0.19.3.tar.gz", hash = "sha256:1a2994773706bbb4995c31a97bc94f1418314923bd1048c6d964837040376440"}, +] + [[package]] name = "pytest" version = "7.2.2" @@ -1999,6 +2304,77 @@ files = [ {file = "PyYAML-6.0.tar.gz", hash = "sha256:68fb519c14306fec9720a2a5b45bc9f0c8d1b9c72adf45c37baedfcd949c35a2"}, ] +[[package]] +name = "ray" +version = "2.3.0" +description = "Ray provides a simple, universal API for building distributed applications." +category = "main" +optional = true +python-versions = "*" +files = [ + {file = "ray-2.3.0-cp310-cp310-macosx_10_15_universal2.whl", hash = "sha256:7a6a36682eaf22836da02a2ecf8e999f15707674cad5ed753b0efef497b1be1b"}, + {file = "ray-2.3.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:58c0d6574873c8276d29f2fc5473590c86da7b53eb7e747e47a3cc707797c37b"}, + {file = "ray-2.3.0-cp310-cp310-manylinux2014_aarch64.whl", hash = "sha256:f46ba421694e396bd34ae70c19032eb22926ecb58366175ab80b22de9b4b8b7f"}, + {file = "ray-2.3.0-cp310-cp310-manylinux2014_x86_64.whl", hash = "sha256:b19d381d425259c80b7a9b14aac9d898637ce262d91baeb9569999de3b043967"}, + {file = "ray-2.3.0-cp310-cp310-win_amd64.whl", hash = "sha256:f496b31cb58434e72f91fca2ddbf9622318d3b6b4b298ff8c45234a5fd1ca954"}, + {file = "ray-2.3.0-cp311-cp311-manylinux2014_aarch64.whl", hash = "sha256:1d2c72c1a9053a4eeae4db2cb2f1a73a4a2efa39cfe19c7306975c20f9a46653"}, + {file = "ray-2.3.0-cp311-cp311-manylinux2014_x86_64.whl", hash = "sha256:d86126b6ab5113a3be35281d5699bad731c8f90e02cf6561057017df9312f0a5"}, + {file = "ray-2.3.0-cp36-cp36m-macosx_10_15_intel.whl", hash = "sha256:288af5d1e7738512563f7a84961aad37c231a0c7d781d65a253294f55c23352b"}, + {file = "ray-2.3.0-cp36-cp36m-manylinux2014_aarch64.whl", hash = "sha256:1c4710a43168b0c3361e4bd5d15b1ec51bebcb7b68bf69ff0cc3a7b9049c8134"}, + {file = "ray-2.3.0-cp36-cp36m-manylinux2014_x86_64.whl", hash = "sha256:f9a79a510c262f38ac6a5a2603bf944f87081cfd5ccf59901585ef4fa20278ee"}, + {file = "ray-2.3.0-cp37-cp37m-macosx_10_15_intel.whl", hash = "sha256:15836d39eba4e4c209e0df0dbd5e3944dcf48755dbbe27847e9d05a23bffc20d"}, + {file = "ray-2.3.0-cp37-cp37m-manylinux2014_aarch64.whl", hash = "sha256:aa31c48e78f40e61a5fcdca69b660c1bff798aad2372e8623d0c1bc08fe177fa"}, + {file = "ray-2.3.0-cp37-cp37m-manylinux2014_x86_64.whl", hash = "sha256:93fb9ddfc11b9bade5f36e36428781bb4c1b56d8d66274ec832150c95123be64"}, + {file = "ray-2.3.0-cp37-cp37m-win_amd64.whl", hash = "sha256:fdc2c46ed34906a10b7c48bca952d7709e5c5d401ab3d1250fc67ba90b314616"}, + {file = "ray-2.3.0-cp38-cp38-macosx_10_15_x86_64.whl", hash = "sha256:2bd65a572d638ec9e100eb22a3089f4759a9e831ac9d9d8e3cc0ecb619e013b1"}, + {file = "ray-2.3.0-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:cb9f18c041ed40e814f8d17d5d99813023b88c57ae5c89800d564804dbdf4757"}, + {file = "ray-2.3.0-cp38-cp38-manylinux2014_aarch64.whl", hash = "sha256:dfc4ec7d4f4181731b61cafdb9c4f10fd535b891cd471fc1050bc8e1951df8e4"}, + {file = "ray-2.3.0-cp38-cp38-manylinux2014_x86_64.whl", hash = "sha256:dd5a7fd409f9653dfecd7cfad58a7e8104b03feca313ed1c16969331c5f0e422"}, + {file = "ray-2.3.0-cp38-cp38-win_amd64.whl", hash = "sha256:fc095984dc27a1775474f2074902d027fd7132b61f3d089e7954636c307f8714"}, + {file = "ray-2.3.0-cp39-cp39-macosx_10_15_x86_64.whl", hash = "sha256:ddcb1ebbb348be78d57fc8898855e344e02295a07ecedb7f10859785a1b23bdc"}, + {file = "ray-2.3.0-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:37168bbe809352bcba40a860da72b704209cde67c8c1c8d1509e91e896a57ce0"}, + {file = "ray-2.3.0-cp39-cp39-manylinux2014_aarch64.whl", hash = "sha256:5fb4d75c9c0a9f50bb7d413f3031ab5bcd20b1517b19908c9d60cb50ceb601e7"}, + {file = "ray-2.3.0-cp39-cp39-manylinux2014_x86_64.whl", hash = "sha256:596811ae6b41e50f038e0a7cdf0a5320151508d74bac84795bbccad4f6adc6a9"}, + {file = "ray-2.3.0-cp39-cp39-win_amd64.whl", hash = "sha256:6e13a541e621f58caa77592c912f96baac3b57737db4e87cc15d3ff6c94a4168"}, +] + +[package.dependencies] +aiosignal = "*" +attrs = "*" +click = ">=7.0" +filelock = "*" +frozenlist = "*" +grpcio = [ + {version = ">=1.32.0", markers = "python_version < \"3.10\" and sys_platform != \"darwin\""}, + {version = ">=1.32.0,<=1.49.1", markers = "python_version < \"3.10\" and sys_platform == \"darwin\""}, + {version = ">=1.42.0", markers = "python_version >= \"3.10\" and sys_platform != \"darwin\""}, + {version = ">=1.42.0,<=1.49.1", markers = "python_version >= \"3.10\" and sys_platform == \"darwin\""}, +] +jsonschema = "*" +msgpack = ">=1.0.0,<2.0.0" +numpy = [ + {version = ">=1.16", markers = "python_version < \"3.9\""}, + {version = ">=1.19.3", markers = "python_version >= \"3.9\""}, +] +packaging = {version = "*", markers = "python_version >= \"3.10\""} +protobuf = ">=3.15.3,<3.19.5 || >3.19.5" +pyyaml = "*" +requests = "*" +virtualenv = ">=20.0.24" + +[package.extras] +air = ["aiohttp (>=3.7)", "aiohttp-cors", "aiorwlock", "colorful", "fastapi", "fsspec", "gpustat (>=1.0.0)", "numpy (>=1.20)", "opencensus", "pandas", "pandas (>=1.3)", "prometheus-client (>=0.7.1)", "py-spy (>=0.2.0)", "pyarrow (>=6.0.1)", "pydantic", "requests", "smart-open", "starlette", "tabulate", "tensorboardX (>=1.9)", "uvicorn"] +all = ["aiohttp (>=3.7)", "aiohttp-cors", "aiorwlock", "colorful", "dm-tree", "fastapi", "fsspec", "gpustat (>=1.0.0)", "gymnasium (==0.26.3)", "kubernetes", "lz4", "numpy (>=1.20)", "opencensus", "opentelemetry-api", "opentelemetry-exporter-otlp", "opentelemetry-sdk", "pandas", "pandas (>=1.3)", "prometheus-client (>=0.7.1)", "py-spy (>=0.2.0)", "pyarrow (>=6.0.1)", "pydantic", "pyyaml", "ray-cpp (==2.3.0)", "requests", "rich", "scikit-image", "scipy", "smart-open", "starlette", "tabulate", "tensorboardX (>=1.9)", "typer", "urllib3", "uvicorn"] +cpp = ["ray-cpp (==2.3.0)"] +data = ["fsspec", "numpy (>=1.20)", "pandas (>=1.3)", "pyarrow (>=6.0.1)"] +default = ["aiohttp (>=3.7)", "aiohttp-cors", "colorful", "gpustat (>=1.0.0)", "opencensus", "prometheus-client (>=0.7.1)", "py-spy (>=0.2.0)", "pydantic", "requests", "smart-open"] +k8s = ["kubernetes", "urllib3"] +observability = ["opentelemetry-api", "opentelemetry-exporter-otlp", "opentelemetry-sdk"] +rllib = ["dm-tree", "gymnasium (==0.26.3)", "lz4", "pandas", "pyyaml", "requests", "rich", "scikit-image", "scipy", "tabulate", "tensorboardX (>=1.9)", "typer"] +serve = ["aiohttp (>=3.7)", "aiohttp-cors", "aiorwlock", "colorful", "fastapi", "gpustat (>=1.0.0)", "opencensus", "prometheus-client (>=0.7.1)", "py-spy (>=0.2.0)", "pydantic", "requests", "smart-open", "starlette", "uvicorn"] +train = ["pandas", "requests", "tabulate", "tensorboardX (>=1.9)"] +tune = ["pandas", "requests", "tabulate", "tensorboardX (>=1.9)"] + [[package]] name = "requests" version = "2.28.2" @@ -2257,7 +2633,7 @@ socks = ["PySocks (>=1.5.6,!=1.5.7,<2.0)"] name = "virtualenv" version = "20.19.0" description = "Virtual Python Environment builder" -category = "dev" +category = "main" optional = false python-versions = ">=3.7" files = [ @@ -2470,7 +2846,7 @@ multidict = ">=4.0" name = "zipp" version = "3.13.0" description = "Backport of pathlib-compatible object wrapper for zip files" -category = "dev" +category = "main" optional = false python-versions = ">=3.7" files = [ @@ -2557,10 +2933,11 @@ glue = ["boto3"] hive = ["thrift"] pandas = ["pandas", "pyarrow"] pyarrow = ["pyarrow"] +ray = ["pandas", "pyarrow", "ray"] s3fs = ["s3fs"] snappy = ["python-snappy"] [metadata] lock-version = "2.0" python-versions = "^3.8" -content-hash = "690692d9d932f3d1b5fe540aea1f4bcf7dbcc12fedb0c6f1ab343440d8a1abfc" +content-hash = "ab04e48ec7502d48a72c8f773a510230b8ffca1824623d7bfbc6a7e36f4c57a6" diff --git a/pyiceberg/table/__init__.py b/pyiceberg/table/__init__.py index 89f3af847b..ef53087fee 100644 --- a/pyiceberg/table/__init__.py +++ b/pyiceberg/table/__init__.py @@ -66,6 +66,7 @@ if TYPE_CHECKING: import pandas as pd import pyarrow as pa + import ray from duckdb import DuckDBPyConnection @@ -415,3 +416,8 @@ def to_duckdb(self, table_name: str, connection: Optional[DuckDBPyConnection] = con.register(table_name, self.to_arrow()) return con + + def to_ray(self) -> ray.data.dataset.Dataset: + import ray + + return ray.data.from_arrow(self.to_arrow()) diff --git a/pyproject.toml b/pyproject.toml index f4c4f6981e..8f94e57b9d 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -68,6 +68,8 @@ pandas = { version = ">=1.4.4,<=1.5.3", optional = true } duckdb = { version = ">=0.6.0,<=0.7.1", optional = true } +ray = { version = ">=2.0.0,<=2.3.0", optional = true } + python-snappy = { version = "0.6.1", optional = true } thrift = { version = "0.16.0", optional = true } @@ -100,6 +102,7 @@ build-backend = "poetry.core.masonry.api" pyarrow = ["pyarrow"] pandas = ["pandas", "pyarrow"] duckdb = ["duckdb", "pyarrow"] +ray = ["ray", "pyarrow", "pandas"] snappy = ["python-snappy"] hive = ["thrift"] s3fs = ["s3fs"] @@ -237,6 +240,10 @@ ignore_missing_imports = true module = "duckdb.*" ignore_missing_imports = true +[[tool.mypy.overrides]] +module = "ray.*" +ignore_missing_imports = true + [[tool.mypy.overrides]] module = "pyparsing.*" ignore_missing_imports = true diff --git a/tests/test_integration.py b/tests/test_integration.py index 3d498f0ec8..653f803527 100644 --- a/tests/test_integration.py +++ b/tests/test_integration.py @@ -49,6 +49,11 @@ def table_test_null_nan_rewritten(catalog: Catalog) -> Table: return catalog.load_table("default.test_null_nan_rewritten") +@pytest.fixture() +def table_test_all_types(catalog: Catalog) -> Table: + return catalog.load_table("default.test_all_types") + + @pytest.mark.integration def test_pyarrow_nan(table_test_null_nan: Table) -> None: arrow_table = table_test_null_nan.scan(row_filter=IsNaN("col_numeric"), selected_fields=("idx", "col_numeric")).to_arrow() @@ -80,3 +85,36 @@ def test_duckdb_nan(table_test_null_nan_rewritten: Table) -> None: result = con.query("SELECT idx, col_numeric FROM table_test_null_nan WHERE isnan(col_numeric)").fetchone() assert result[0] == 1 assert math.isnan(result[1]) + + +@pytest.mark.integration +def test_ray_nan(table_test_null_nan_rewritten: Table) -> None: + ray_dataset = table_test_null_nan_rewritten.scan().to_ray() + assert ray_dataset.count() == 3 + assert math.isnan(ray_dataset.take()[0]["col_numeric"]) + + +@pytest.mark.integration +def test_ray_nan_rewritten(table_test_null_nan_rewritten: Table) -> None: + ray_dataset = table_test_null_nan_rewritten.scan( + row_filter=IsNaN("col_numeric"), selected_fields=("idx", "col_numeric") + ).to_ray() + assert ray_dataset.count() == 1 + assert ray_dataset.take()[0]["idx"] == 1 + assert math.isnan(ray_dataset.take()[0]["col_numeric"]) + + +@pytest.mark.integration +@pytest.mark.skip(reason="Fixing issues with NaN's: https://github.com/apache/arrow/issues/34162") +def test_ray_not_nan_count(table_test_null_nan_rewritten: Table) -> None: + ray_dataset = table_test_null_nan_rewritten.scan(row_filter=NotNaN("col_numeric"), selected_fields=("idx",)).to_ray() + print(ray_dataset.take()) + assert ray_dataset.count() == 2 + + +@pytest.mark.integration +def test_ray_all_types(table_test_all_types: Table) -> None: + ray_dataset = table_test_all_types.scan().to_ray() + pandas_dataframe = table_test_all_types.scan().to_pandas() + assert ray_dataset.count() == pandas_dataframe.shape[0] + assert pandas_dataframe.equals(ray_dataset.to_pandas()) From e8e62bd3fb44c01f065064de8b284488664e474d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20R=C3=BCckert=20Garc=C3=ADa?= Date: Fri, 24 Mar 2023 09:30:53 +0100 Subject: [PATCH 409/642] Python: Add limit to table scan (#7163) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * Python: Add support for ORC Creates fragments based on the FileFormat. Blocked by: https://github.com/apache/iceberg/pull/6997 * Revert * TableScan add limit * pyarrow limit number of rows fetched from files if limit is set * add tests for scan limit * python ci rebuild container if changes on python/dev/ * remove support for ORC * remove unused imports * increase sleep before running tests * update python docs to include limit in table query * docs fix format --------- Co-authored-by: Fokko Driesprong Co-authored-by: Daniel Rückert García --- dev/provision.py | 33 ++++++++++++++++++++ mkdocs/docs/api.md | 3 +- pyiceberg/files.py | 8 ++--- pyiceberg/io/pyarrow.py | 60 ++++++++++++++++++++++++++----------- pyiceberg/manifest.py | 6 ++++ pyiceberg/table/__init__.py | 15 ++++++++-- tests/test_integration.py | 17 +++++++++++ 7 files changed, 118 insertions(+), 24 deletions(-) diff --git a/dev/provision.py b/dev/provision.py index 1e6f5b7319..81bd094c58 100644 --- a/dev/provision.py +++ b/dev/provision.py @@ -71,6 +71,39 @@ """ ) +spark.sql( + """ + DROP TABLE IF EXISTS test_limit; +""" +) + +spark.sql( + """ + CREATE TABLE test_limit + USING iceberg + AS SELECT + 1 AS idx + UNION ALL SELECT + 2 AS idx + UNION ALL SELECT + 3 AS idx + UNION ALL SELECT + 4 AS idx + UNION ALL SELECT + 5 AS idx + UNION ALL SELECT + 6 AS idx + UNION ALL SELECT + 7 AS idx + UNION ALL SELECT + 8 AS idx + UNION ALL SELECT + 9 AS idx + UNION ALL SELECT + 10 AS idx + """ +) + spark.sql( """ DROP TABLE IF EXISTS test_deletes; diff --git a/mkdocs/docs/api.md b/mkdocs/docs/api.md index 9516516859..1cb26714ad 100644 --- a/mkdocs/docs/api.md +++ b/mkdocs/docs/api.md @@ -281,7 +281,7 @@ Table( ## Query a table -To query a table, a table scan is needed. A table scan accepts a filter, columns and optionally a snapshot ID: +To query a table, a table scan is needed. A table scan accepts a filter, columns and optionally a limit and a snapshot ID: ```python from pyiceberg.catalog import load_catalog @@ -293,6 +293,7 @@ table = catalog.load_table("nyc.taxis") scan = table.scan( row_filter=GreaterThanOrEqual("trip_distance", 10.0), selected_fields=("VendorID", "tpep_pickup_datetime", "tpep_dropoff_datetime"), + limit=100, ) # Or filter using a string predicate diff --git a/pyiceberg/files.py b/pyiceberg/files.py index b832bbbfe7..df19f37e0c 100644 --- a/pyiceberg/files.py +++ b/pyiceberg/files.py @@ -28,7 +28,7 @@ class FileContentType(Enum): class FileFormat(Enum): """An enum that includes all possible formats for an Iceberg data file""" - ORC = auto() - PARQUET = auto() - AVRO = auto() - METADATA = auto() + ORC = "ORC" + PARQUET = "PARQUET" + AVRO = "AVRO" + METADATA = "METADATA" diff --git a/pyiceberg/io/pyarrow.py b/pyiceberg/io/pyarrow.py index 07b59258dd..ae81733968 100644 --- a/pyiceberg/io/pyarrow.py +++ b/pyiceberg/io/pyarrow.py @@ -24,9 +24,11 @@ """ from __future__ import annotations +import multiprocessing import os from functools import lru_cache from multiprocessing.pool import ThreadPool +from multiprocessing.sharedctypes import Synchronized from typing import ( TYPE_CHECKING, Any, @@ -42,7 +44,7 @@ import pyarrow as pa import pyarrow.compute as pc -import pyarrow.parquet as pq +import pyarrow.dataset as ds from pyarrow.fs import ( FileInfo, FileSystem, @@ -491,14 +493,19 @@ def _file_to_table( projected_schema: Schema, projected_field_ids: Set[int], case_sensitive: bool, + rows_counter: Synchronized[int], + limit: Optional[int] = None, ) -> Optional[pa.Table]: - _, path = PyArrowFileIO.parse_location(task.file.file_path) + if limit and rows_counter.value >= limit: + return None - # Get the schema - with fs.open_input_file(path) as fout: - parquet_schema = pq.read_schema(fout) + _, path = PyArrowFileIO.parse_location(task.file.file_path) + arrow_format = ds.ParquetFileFormat(pre_buffer=True, buffer_size=(ONE_MEGABYTE * 8)) + with fs.open_input_file(path) as fin: + fragment = arrow_format.make_fragment(fin) + physical_schema = fragment.physical_schema schema_raw = None - if metadata := parquet_schema.metadata: + if metadata := physical_schema.metadata: schema_raw = metadata.get(ICEBERG_SCHEMA) if schema_raw is None: raise ValueError( @@ -517,15 +524,22 @@ def _file_to_table( if file_schema is None: raise ValueError(f"Missing Iceberg schema in Metadata for file: {path}") - arrow_table = pq.read_table( - source=fout, - schema=parquet_schema, - pre_buffer=True, - buffer_size=8 * ONE_MEGABYTE, - filters=pyarrow_filter, + fragment_scanner = ds.Scanner.from_fragment( + fragment=fragment, + schema=physical_schema, + filter=pyarrow_filter, columns=[col.name for col in file_project_schema.columns], ) + if limit: + arrow_table = fragment_scanner.head(limit) + with rows_counter.get_lock(): + if rows_counter.value >= limit: + return None + rows_counter.value += len(arrow_table) + else: + arrow_table = fragment_scanner.to_table() + # If there is no data, we don't have to go through the schema if len(arrow_table) > 0: return to_requested_schema(projected_schema, file_project_schema, arrow_table) @@ -534,7 +548,12 @@ def _file_to_table( def project_table( - tasks: Iterable[FileScanTask], table: Table, row_filter: BooleanExpression, projected_schema: Schema, case_sensitive: bool + tasks: Iterable[FileScanTask], + table: Table, + row_filter: BooleanExpression, + projected_schema: Schema, + case_sensitive: bool, + limit: Optional[int] = None, ) -> pa.Table: """Resolves the right columns based on the identifier @@ -570,23 +589,30 @@ def project_table( id for id in projected_schema.field_ids if not isinstance(projected_schema.find_type(id), (MapType, ListType)) }.union(extract_field_ids(bound_row_filter)) + rows_counter = multiprocessing.Value("i", 0) + with ThreadPool() as pool: tables = [ table for table in pool.starmap( func=_file_to_table, - iterable=[(fs, task, bound_row_filter, projected_schema, projected_field_ids, case_sensitive) for task in tasks], + iterable=[ + (fs, task, bound_row_filter, projected_schema, projected_field_ids, case_sensitive, rows_counter, limit) + for task in tasks + ], chunksize=None, # we could use this to control how to materialize the generator of tasks (we should also make the expression above lazy) ) if table is not None ] if len(tables) > 1: - return pa.concat_tables(tables) + final_table = pa.concat_tables(tables) elif len(tables) == 1: - return tables[0] + final_table = tables[0] else: - return pa.Table.from_batches([], schema=schema_to_pyarrow(projected_schema)) + final_table = pa.Table.from_batches([], schema=schema_to_pyarrow(projected_schema)) + + return final_table.slice(0, limit) def to_requested_schema(requested_schema: Schema, file_schema: Schema, table: pa.Table) -> pa.Table: diff --git a/pyiceberg/manifest.py b/pyiceberg/manifest.py index 757f3bd016..942c582b51 100644 --- a/pyiceberg/manifest.py +++ b/pyiceberg/manifest.py @@ -177,6 +177,12 @@ class DataFile(Record): sort_order_id: Optional[int] spec_id: Optional[int] + def __setattr__(self, name: str, value: Any) -> None: + # The file_format is written as a string, so we need to cast it to the Enum + if name == "file_format": + value = FileFormat[value] + super().__setattr__(name, value) + def __init__(self, *data: Any, **named_data: Any) -> None: super().__init__(*data, **{"struct": DATA_FILE_TYPE, **named_data}) diff --git a/pyiceberg/table/__init__.py b/pyiceberg/table/__init__.py index ef53087fee..8f19d93acf 100644 --- a/pyiceberg/table/__init__.py +++ b/pyiceberg/table/__init__.py @@ -100,6 +100,7 @@ def scan( case_sensitive: bool = True, snapshot_id: Optional[int] = None, options: Properties = EMPTY_DICT, + limit: Optional[int] = None, ) -> DataScan: return DataScan( table=self, @@ -108,6 +109,7 @@ def scan( case_sensitive=case_sensitive, snapshot_id=snapshot_id, options=options, + limit=limit, ) def schema(self) -> Schema: @@ -220,6 +222,7 @@ class TableScan(ABC): case_sensitive: bool snapshot_id: Optional[int] options: Properties + limit: Optional[int] def __init__( self, @@ -229,6 +232,7 @@ def __init__( case_sensitive: bool = True, snapshot_id: Optional[int] = None, options: Properties = EMPTY_DICT, + limit: Optional[int] = None, ): self.table = table self.row_filter = _parse_row_filter(row_filter) @@ -236,6 +240,7 @@ def __init__( self.case_sensitive = case_sensitive self.snapshot_id = snapshot_id self.options = options + self.limit = limit def snapshot(self) -> Optional[Snapshot]: if self.snapshot_id: @@ -336,8 +341,9 @@ def __init__( case_sensitive: bool = True, snapshot_id: Optional[int] = None, options: Properties = EMPTY_DICT, + limit: Optional[int] = None, ): - super().__init__(table, row_filter, selected_fields, case_sensitive, snapshot_id, options) + super().__init__(table, row_filter, selected_fields, case_sensitive, snapshot_id, options, limit) def _build_partition_projection(self, spec_id: int) -> BooleanExpression: project = inclusive_projection(self.table.schema(), self.table.specs()[spec_id]) @@ -403,7 +409,12 @@ def to_arrow(self) -> pa.Table: from pyiceberg.io.pyarrow import project_table return project_table( - self.plan_files(), self.table, self.row_filter, self.projection(), case_sensitive=self.case_sensitive + self.plan_files(), + self.table, + self.row_filter, + self.projection(), + case_sensitive=self.case_sensitive, + limit=self.limit, ) def to_pandas(self, **kwargs: Any) -> pd.DataFrame: diff --git a/tests/test_integration.py b/tests/test_integration.py index 653f803527..3eb24cd48e 100644 --- a/tests/test_integration.py +++ b/tests/test_integration.py @@ -49,6 +49,11 @@ def table_test_null_nan_rewritten(catalog: Catalog) -> Table: return catalog.load_table("default.test_null_nan_rewritten") +@pytest.fixture() +def table_test_limit(catalog: Catalog) -> Table: + return catalog.load_table("default.test_limit") + + @pytest.fixture() def table_test_all_types(catalog: Catalog) -> Table: return catalog.load_table("default.test_all_types") @@ -87,6 +92,18 @@ def test_duckdb_nan(table_test_null_nan_rewritten: Table) -> None: assert math.isnan(result[1]) +@pytest.mark.integration +def test_pyarrow_limit(table_test_limit: Table) -> None: + limited_result = table_test_limit.scan(selected_fields=("idx",), limit=1).to_arrow() + assert len(limited_result) == 1 + + empty_result = table_test_limit.scan(selected_fields=("idx",), limit=0).to_arrow() + assert len(empty_result) == 0 + + full_result = table_test_limit.scan(selected_fields=("idx",), limit=999).to_arrow() + assert len(full_result) == 10 + + @pytest.mark.integration def test_ray_nan(table_test_null_nan_rewritten: Table) -> None: ray_dataset = table_test_null_nan_rewritten.scan().to_ray() From 0d86db5a4f627f9e73e8662e5df963dc6956f7b8 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Sun, 26 Mar 2023 07:57:04 +0200 Subject: [PATCH 410/642] Build: Bump moto from 4.1.4 to 4.1.5 in /python (#7203) --- poetry.lock | 26 +++++++++++++------------- pyproject.toml | 2 +- 2 files changed, 14 insertions(+), 14 deletions(-) diff --git a/poetry.lock b/poetry.lock index d94d0ee155..215dc2950d 100644 --- a/poetry.lock +++ b/poetry.lock @@ -1,4 +1,4 @@ -# This file is automatically @generated by Poetry 1.4.0 and should not be changed by hand. +# This file is automatically @generated by Poetry and should not be changed by hand. [[package]] name = "adal" @@ -1394,14 +1394,14 @@ files = [ [[package]] name = "moto" -version = "4.1.4" +version = "4.1.5" description = "" category = "dev" optional = false python-versions = ">=3.7" files = [ - {file = "moto-4.1.4-py2.py3-none-any.whl", hash = "sha256:f9bf72aec6aea49ebb1a46c8096b2c9629be58ecca3ecd8b51f781fea78148e2"}, - {file = "moto-4.1.4.tar.gz", hash = "sha256:304cb19eee0019cd13f7d87ca590a4694677469c383ab8f8439fcb6717c47037"}, + {file = "moto-4.1.5-py2.py3-none-any.whl", hash = "sha256:363577f7a0cdf639852420f6ba5caa9aa3c90a688feae6315f8ee4bf324b8c27"}, + {file = "moto-4.1.5.tar.gz", hash = "sha256:63542b7b9f307b00fae460b42d15cf9346de3ad3b1287fba38fc68f3c05e4da4"}, ] [package.dependencies] @@ -1416,17 +1416,17 @@ werkzeug = ">=0.5,<2.2.0 || >2.2.0,<2.2.1 || >2.2.1" xmltodict = "*" [package.extras] -all = ["PyYAML (>=5.1)", "aws-xray-sdk (>=0.93,!=0.96)", "cfn-lint (>=0.40.0)", "docker (>=2.5.1)", "ecdsa (!=0.15)", "graphql-core", "jsondiff (>=1.1.2)", "openapi-spec-validator (>=0.2.8)", "pyparsing (>=3.0.7)", "python-jose[cryptography] (>=3.1.0,<4.0.0)", "setuptools", "sshpubkeys (>=3.1.0)"] +all = ["PyYAML (>=5.1)", "aws-xray-sdk (>=0.93,!=0.96)", "cfn-lint (>=0.40.0)", "docker (>=3.0.0)", "ecdsa (!=0.15)", "graphql-core", "jsondiff (>=1.1.2)", "openapi-spec-validator (>=0.2.8)", "pyparsing (>=3.0.7)", "python-jose[cryptography] (>=3.1.0,<4.0.0)", "setuptools", "sshpubkeys (>=3.1.0)"] apigateway = ["PyYAML (>=5.1)", "ecdsa (!=0.15)", "openapi-spec-validator (>=0.2.8)", "python-jose[cryptography] (>=3.1.0,<4.0.0)"] apigatewayv2 = ["PyYAML (>=5.1)"] appsync = ["graphql-core"] -awslambda = ["docker (>=2.5.1)"] -batch = ["docker (>=2.5.1)"] -cloudformation = ["PyYAML (>=5.1)", "aws-xray-sdk (>=0.93,!=0.96)", "cfn-lint (>=0.40.0)", "docker (>=2.5.1)", "ecdsa (!=0.15)", "graphql-core", "jsondiff (>=1.1.2)", "openapi-spec-validator (>=0.2.8)", "pyparsing (>=3.0.7)", "python-jose[cryptography] (>=3.1.0,<4.0.0)", "setuptools", "sshpubkeys (>=3.1.0)"] +awslambda = ["docker (>=3.0.0)"] +batch = ["docker (>=3.0.0)"] +cloudformation = ["PyYAML (>=5.1)", "aws-xray-sdk (>=0.93,!=0.96)", "cfn-lint (>=0.40.0)", "docker (>=3.0.0)", "ecdsa (!=0.15)", "graphql-core", "jsondiff (>=1.1.2)", "openapi-spec-validator (>=0.2.8)", "pyparsing (>=3.0.7)", "python-jose[cryptography] (>=3.1.0,<4.0.0)", "setuptools", "sshpubkeys (>=3.1.0)"] cognitoidp = ["ecdsa (!=0.15)", "python-jose[cryptography] (>=3.1.0,<4.0.0)"] ds = ["sshpubkeys (>=3.1.0)"] -dynamodb = ["docker (>=2.5.1)"] -dynamodbstreams = ["docker (>=2.5.1)"] +dynamodb = ["docker (>=3.0.0)"] +dynamodbstreams = ["docker (>=3.0.0)"] ebs = ["sshpubkeys (>=3.1.0)"] ec2 = ["sshpubkeys (>=3.1.0)"] efs = ["sshpubkeys (>=3.1.0)"] @@ -1435,7 +1435,7 @@ glue = ["pyparsing (>=3.0.7)"] iotdata = ["jsondiff (>=1.1.2)"] route53resolver = ["sshpubkeys (>=3.1.0)"] s3 = ["PyYAML (>=5.1)"] -server = ["PyYAML (>=5.1)", "aws-xray-sdk (>=0.93,!=0.96)", "cfn-lint (>=0.40.0)", "docker (>=2.5.1)", "ecdsa (!=0.15)", "flask (!=2.2.0,!=2.2.1)", "flask-cors", "graphql-core", "jsondiff (>=1.1.2)", "openapi-spec-validator (>=0.2.8)", "pyparsing (>=3.0.7)", "python-jose[cryptography] (>=3.1.0,<4.0.0)", "setuptools", "sshpubkeys (>=3.1.0)"] +server = ["PyYAML (>=5.1)", "aws-xray-sdk (>=0.93,!=0.96)", "cfn-lint (>=0.40.0)", "docker (>=3.0.0)", "ecdsa (!=0.15)", "flask (!=2.2.0,!=2.2.1)", "flask-cors", "graphql-core", "jsondiff (>=1.1.2)", "openapi-spec-validator (>=0.2.8)", "pyparsing (>=3.0.7)", "python-jose[cryptography] (>=3.1.0,<4.0.0)", "setuptools", "sshpubkeys (>=3.1.0)"] ssm = ["PyYAML (>=5.1)"] xray = ["aws-xray-sdk (>=0.93,!=0.96)", "setuptools"] @@ -2933,11 +2933,11 @@ glue = ["boto3"] hive = ["thrift"] pandas = ["pandas", "pyarrow"] pyarrow = ["pyarrow"] -ray = ["pandas", "pyarrow", "ray"] +ray = ["ray", "pyarrow", "pandas"] s3fs = ["s3fs"] snappy = ["python-snappy"] [metadata] lock-version = "2.0" python-versions = "^3.8" -content-hash = "ab04e48ec7502d48a72c8f773a510230b8ffca1824623d7bfbc6a7e36f4c57a6" +content-hash = "84081baf2f0abde7b94c62dc71838648c272eeceb92e8bc7976348ea576cb398" diff --git a/pyproject.toml b/pyproject.toml index 8f94e57b9d..2d724c856b 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -87,7 +87,7 @@ pre-commit = "3.2.0" fastavro = "1.7.3" coverage = { version = "^7.2.2", extras = ["toml"] } requests-mock = "1.10.0" -moto = "^4.1.4" +moto = "^4.1.5" typing-extensions = '4.5.0' [tool.poetry.scripts] From 2410878507c0f5c3aa967354d20064e24d0396f0 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Sun, 26 Mar 2023 08:27:51 +0200 Subject: [PATCH 411/642] Build: Bump pre-commit from 3.2.0 to 3.2.1 in /python (#7204) --- poetry.lock | 8 ++++---- pyproject.toml | 2 +- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/poetry.lock b/poetry.lock index 215dc2950d..13423647f3 100644 --- a/poetry.lock +++ b/poetry.lock @@ -1854,14 +1854,14 @@ tests = ["pytest (>=5.4.1)", "pytest-cov (>=2.8.1)", "pytest-mypy (>=0.8.0)", "p [[package]] name = "pre-commit" -version = "3.2.0" +version = "3.2.1" description = "A framework for managing and maintaining multi-language pre-commit hooks." category = "dev" optional = false python-versions = ">=3.8" files = [ - {file = "pre_commit-3.2.0-py2.py3-none-any.whl", hash = "sha256:f712d3688102e13c8e66b7d7dbd8934a6dda157e58635d89f7d6fecdca39ce8a"}, - {file = "pre_commit-3.2.0.tar.gz", hash = "sha256:818f0d998059934d0f81bb3667e3ccdc32da6ed7ccaac33e43dc231561ddaaa9"}, + {file = "pre_commit-3.2.1-py2.py3-none-any.whl", hash = "sha256:a06a7fcce7f420047a71213c175714216498b49ebc81fe106f7716ca265f5bb6"}, + {file = "pre_commit-3.2.1.tar.gz", hash = "sha256:b5aee7d75dbba21ee161ba641b01e7ae10c5b91967ebf7b2ab0dfae12d07e1f1"}, ] [package.dependencies] @@ -2940,4 +2940,4 @@ snappy = ["python-snappy"] [metadata] lock-version = "2.0" python-versions = "^3.8" -content-hash = "84081baf2f0abde7b94c62dc71838648c272eeceb92e8bc7976348ea576cb398" +content-hash = "80c4e8c22a73338b6dc6706398cfab2f7b401baeae16935a4a79ce0822960353" diff --git a/pyproject.toml b/pyproject.toml index 2d724c856b..f92d7821aa 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -83,7 +83,7 @@ adlfs = { version = ">=2022.8.2,<=2023.1.0", optional = true } [tool.poetry.dev-dependencies] pytest = "7.2.2" pytest-checkdocs = "2.9.0" -pre-commit = "3.2.0" +pre-commit = "3.2.1" fastavro = "1.7.3" coverage = { version = "^7.2.2", extras = ["toml"] } requests-mock = "1.10.0" From 52d18ab83e20df196a6e58e8c97f08ecc54e1541 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Sun, 26 Mar 2023 11:51:38 +0200 Subject: [PATCH 412/642] Build: Bump pydantic from 1.10.6 to 1.10.7 in /python (#7205) --- poetry.lock | 76 +++++++++++++++++++++++++------------------------- pyproject.toml | 2 +- 2 files changed, 39 insertions(+), 39 deletions(-) diff --git a/poetry.lock b/poetry.lock index 13423647f3..38b7077960 100644 --- a/poetry.lock +++ b/poetry.lock @@ -1946,48 +1946,48 @@ files = [ [[package]] name = "pydantic" -version = "1.10.6" +version = "1.10.7" description = "Data validation and settings management using python type hints" category = "main" optional = false python-versions = ">=3.7" files = [ - {file = "pydantic-1.10.6-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:f9289065611c48147c1dd1fd344e9d57ab45f1d99b0fb26c51f1cf72cd9bcd31"}, - {file = "pydantic-1.10.6-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:8c32b6bba301490d9bb2bf5f631907803135e8085b6aa3e5fe5a770d46dd0160"}, - {file = "pydantic-1.10.6-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:fd9b9e98068fa1068edfc9eabde70a7132017bdd4f362f8b4fd0abed79c33083"}, - {file = "pydantic-1.10.6-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:1c84583b9df62522829cbc46e2b22e0ec11445625b5acd70c5681ce09c9b11c4"}, - {file = "pydantic-1.10.6-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:b41822064585fea56d0116aa431fbd5137ce69dfe837b599e310034171996084"}, - {file = "pydantic-1.10.6-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:61f1f08adfaa9cc02e0cbc94f478140385cbd52d5b3c5a657c2fceb15de8d1fb"}, - {file = "pydantic-1.10.6-cp310-cp310-win_amd64.whl", hash = "sha256:32937835e525d92c98a1512218db4eed9ddc8f4ee2a78382d77f54341972c0e7"}, - {file = "pydantic-1.10.6-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:bbd5c531b22928e63d0cb1868dee76123456e1de2f1cb45879e9e7a3f3f1779b"}, - {file = "pydantic-1.10.6-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:e277bd18339177daa62a294256869bbe84df1fb592be2716ec62627bb8d7c81d"}, - {file = "pydantic-1.10.6-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:89f15277d720aa57e173954d237628a8d304896364b9de745dcb722f584812c7"}, - {file = "pydantic-1.10.6-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:b243b564cea2576725e77aeeda54e3e0229a168bc587d536cd69941e6797543d"}, - {file = "pydantic-1.10.6-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:3ce13a558b484c9ae48a6a7c184b1ba0e5588c5525482681db418268e5f86186"}, - {file = "pydantic-1.10.6-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:3ac1cd4deed871dfe0c5f63721e29debf03e2deefa41b3ed5eb5f5df287c7b70"}, - {file = "pydantic-1.10.6-cp311-cp311-win_amd64.whl", hash = "sha256:b1eb6610330a1dfba9ce142ada792f26bbef1255b75f538196a39e9e90388bf4"}, - {file = "pydantic-1.10.6-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:4ca83739c1263a044ec8b79df4eefc34bbac87191f0a513d00dd47d46e307a65"}, - {file = "pydantic-1.10.6-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:ea4e2a7cb409951988e79a469f609bba998a576e6d7b9791ae5d1e0619e1c0f2"}, - {file = "pydantic-1.10.6-cp37-cp37m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:53de12b4608290992a943801d7756f18a37b7aee284b9ffa794ee8ea8153f8e2"}, - {file = "pydantic-1.10.6-cp37-cp37m-musllinux_1_1_i686.whl", hash = "sha256:60184e80aac3b56933c71c48d6181e630b0fbc61ae455a63322a66a23c14731a"}, - {file = "pydantic-1.10.6-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:415a3f719ce518e95a92effc7ee30118a25c3d032455d13e121e3840985f2efd"}, - {file = "pydantic-1.10.6-cp37-cp37m-win_amd64.whl", hash = "sha256:72cb30894a34d3a7ab6d959b45a70abac8a2a93b6480fc5a7bfbd9c935bdc4fb"}, - {file = "pydantic-1.10.6-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:3091d2eaeda25391405e36c2fc2ed102b48bac4b384d42b2267310abae350ca6"}, - {file = "pydantic-1.10.6-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:751f008cd2afe812a781fd6aa2fb66c620ca2e1a13b6a2152b1ad51553cb4b77"}, - {file = "pydantic-1.10.6-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:12e837fd320dd30bd625be1b101e3b62edc096a49835392dcf418f1a5ac2b832"}, - {file = "pydantic-1.10.6-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:587d92831d0115874d766b1f5fddcdde0c5b6c60f8c6111a394078ec227fca6d"}, - {file = "pydantic-1.10.6-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:476f6674303ae7965730a382a8e8d7fae18b8004b7b69a56c3d8fa93968aa21c"}, - {file = "pydantic-1.10.6-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:3a2be0a0f32c83265fd71a45027201e1278beaa82ea88ea5b345eea6afa9ac7f"}, - {file = "pydantic-1.10.6-cp38-cp38-win_amd64.whl", hash = "sha256:0abd9c60eee6201b853b6c4be104edfba4f8f6c5f3623f8e1dba90634d63eb35"}, - {file = "pydantic-1.10.6-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:6195ca908045054dd2d57eb9c39a5fe86409968b8040de8c2240186da0769da7"}, - {file = "pydantic-1.10.6-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:43cdeca8d30de9a897440e3fb8866f827c4c31f6c73838e3a01a14b03b067b1d"}, - {file = "pydantic-1.10.6-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:4c19eb5163167489cb1e0161ae9220dadd4fc609a42649e7e84a8fa8fff7a80f"}, - {file = "pydantic-1.10.6-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:012c99a9c0d18cfde7469aa1ebff922e24b0c706d03ead96940f5465f2c9cf62"}, - {file = "pydantic-1.10.6-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:528dcf7ec49fb5a84bf6fe346c1cc3c55b0e7603c2123881996ca3ad79db5bfc"}, - {file = "pydantic-1.10.6-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:163e79386c3547c49366e959d01e37fc30252285a70619ffc1b10ede4758250a"}, - {file = "pydantic-1.10.6-cp39-cp39-win_amd64.whl", hash = "sha256:189318051c3d57821f7233ecc94708767dd67687a614a4e8f92b4a020d4ffd06"}, - {file = "pydantic-1.10.6-py3-none-any.whl", hash = "sha256:acc6783751ac9c9bc4680379edd6d286468a1dc8d7d9906cd6f1186ed682b2b0"}, - {file = "pydantic-1.10.6.tar.gz", hash = "sha256:cf95adb0d1671fc38d8c43dd921ad5814a735e7d9b4d9e437c088002863854fd"}, + {file = "pydantic-1.10.7-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:e79e999e539872e903767c417c897e729e015872040e56b96e67968c3b918b2d"}, + {file = "pydantic-1.10.7-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:01aea3a42c13f2602b7ecbbea484a98169fb568ebd9e247593ea05f01b884b2e"}, + {file = "pydantic-1.10.7-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:516f1ed9bc2406a0467dd777afc636c7091d71f214d5e413d64fef45174cfc7a"}, + {file = "pydantic-1.10.7-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:ae150a63564929c675d7f2303008d88426a0add46efd76c3fc797cd71cb1b46f"}, + {file = "pydantic-1.10.7-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:ecbbc51391248116c0a055899e6c3e7ffbb11fb5e2a4cd6f2d0b93272118a209"}, + {file = "pydantic-1.10.7-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:f4a2b50e2b03d5776e7f21af73e2070e1b5c0d0df255a827e7c632962f8315af"}, + {file = "pydantic-1.10.7-cp310-cp310-win_amd64.whl", hash = "sha256:a7cd2251439988b413cb0a985c4ed82b6c6aac382dbaff53ae03c4b23a70e80a"}, + {file = "pydantic-1.10.7-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:68792151e174a4aa9e9fc1b4e653e65a354a2fa0fed169f7b3d09902ad2cb6f1"}, + {file = "pydantic-1.10.7-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:dfe2507b8ef209da71b6fb5f4e597b50c5a34b78d7e857c4f8f3115effaef5fe"}, + {file = "pydantic-1.10.7-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:10a86d8c8db68086f1e30a530f7d5f83eb0685e632e411dbbcf2d5c0150e8dcd"}, + {file = "pydantic-1.10.7-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:d75ae19d2a3dbb146b6f324031c24f8a3f52ff5d6a9f22f0683694b3afcb16fb"}, + {file = "pydantic-1.10.7-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:464855a7ff7f2cc2cf537ecc421291b9132aa9c79aef44e917ad711b4a93163b"}, + {file = "pydantic-1.10.7-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:193924c563fae6ddcb71d3f06fa153866423ac1b793a47936656e806b64e24ca"}, + {file = "pydantic-1.10.7-cp311-cp311-win_amd64.whl", hash = "sha256:b4a849d10f211389502059c33332e91327bc154acc1845f375a99eca3afa802d"}, + {file = "pydantic-1.10.7-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:cc1dde4e50a5fc1336ee0581c1612215bc64ed6d28d2c7c6f25d2fe3e7c3e918"}, + {file = "pydantic-1.10.7-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:e0cfe895a504c060e5d36b287ee696e2fdad02d89e0d895f83037245218a87fe"}, + {file = "pydantic-1.10.7-cp37-cp37m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:670bb4683ad1e48b0ecb06f0cfe2178dcf74ff27921cdf1606e527d2617a81ee"}, + {file = "pydantic-1.10.7-cp37-cp37m-musllinux_1_1_i686.whl", hash = "sha256:950ce33857841f9a337ce07ddf46bc84e1c4946d2a3bba18f8280297157a3fd1"}, + {file = "pydantic-1.10.7-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:c15582f9055fbc1bfe50266a19771bbbef33dd28c45e78afbe1996fd70966c2a"}, + {file = "pydantic-1.10.7-cp37-cp37m-win_amd64.whl", hash = "sha256:82dffb306dd20bd5268fd6379bc4bfe75242a9c2b79fec58e1041fbbdb1f7914"}, + {file = "pydantic-1.10.7-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:8c7f51861d73e8b9ddcb9916ae7ac39fb52761d9ea0df41128e81e2ba42886cd"}, + {file = "pydantic-1.10.7-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:6434b49c0b03a51021ade5c4daa7d70c98f7a79e95b551201fff682fc1661245"}, + {file = "pydantic-1.10.7-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:64d34ab766fa056df49013bb6e79921a0265204c071984e75a09cbceacbbdd5d"}, + {file = "pydantic-1.10.7-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:701daea9ffe9d26f97b52f1d157e0d4121644f0fcf80b443248434958fd03dc3"}, + {file = "pydantic-1.10.7-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:cf135c46099ff3f919d2150a948ce94b9ce545598ef2c6c7bf55dca98a304b52"}, + {file = "pydantic-1.10.7-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:b0f85904f73161817b80781cc150f8b906d521fa11e3cdabae19a581c3606209"}, + {file = "pydantic-1.10.7-cp38-cp38-win_amd64.whl", hash = "sha256:9f6f0fd68d73257ad6685419478c5aece46432f4bdd8d32c7345f1986496171e"}, + {file = "pydantic-1.10.7-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:c230c0d8a322276d6e7b88c3f7ce885f9ed16e0910354510e0bae84d54991143"}, + {file = "pydantic-1.10.7-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:976cae77ba6a49d80f461fd8bba183ff7ba79f44aa5cfa82f1346b5626542f8e"}, + {file = "pydantic-1.10.7-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:7d45fc99d64af9aaf7e308054a0067fdcd87ffe974f2442312372dfa66e1001d"}, + {file = "pydantic-1.10.7-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:d2a5ebb48958754d386195fe9e9c5106f11275867051bf017a8059410e9abf1f"}, + {file = "pydantic-1.10.7-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:abfb7d4a7cd5cc4e1d1887c43503a7c5dd608eadf8bc615413fc498d3e4645cd"}, + {file = "pydantic-1.10.7-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:80b1fab4deb08a8292d15e43a6edccdffa5377a36a4597bb545b93e79c5ff0a5"}, + {file = "pydantic-1.10.7-cp39-cp39-win_amd64.whl", hash = "sha256:d71e69699498b020ea198468e2480a2f1e7433e32a3a99760058c6520e2bea7e"}, + {file = "pydantic-1.10.7-py3-none-any.whl", hash = "sha256:0cd181f1d0b1d00e2b705f1bf1ac7799a2d938cce3376b8007df62b29be3c2c6"}, + {file = "pydantic-1.10.7.tar.gz", hash = "sha256:cfc83c0678b6ba51b0532bea66860617c4cd4251ecf76e9846fa5a9f3454e97e"}, ] [package.dependencies] @@ -2940,4 +2940,4 @@ snappy = ["python-snappy"] [metadata] lock-version = "2.0" python-versions = "^3.8" -content-hash = "80c4e8c22a73338b6dc6706398cfab2f7b401baeae16935a4a79ce0822960353" +content-hash = "a8f4da6d60917f374c291ed0bf8cc221793c6da0e929622f80e2e9db82316d80" diff --git a/pyproject.toml b/pyproject.toml index f92d7821aa..4e5c0555bd 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -55,7 +55,7 @@ click = "8.1.3" rich = ">=13.0.0,<=13.3.2" pyyaml = ">=5.4.0,<=6.0.0" -pydantic = "1.10.6" +pydantic = "1.10.7" fsspec = ">=2022.8.2,<=2023.1.0" pyparsing = "3.0.9" From bc1299bb9dac991b36b4dd57ff4e9c3959d897be Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Sun, 26 Mar 2023 12:37:50 +0200 Subject: [PATCH 413/642] Build: Bump ray from 2.3.0 to 2.3.1 in /python (#7206) --- poetry.lock | 54 ++++++++++++++++++++++++-------------------------- pyproject.toml | 2 +- 2 files changed, 27 insertions(+), 29 deletions(-) diff --git a/poetry.lock b/poetry.lock index 38b7077960..1920780687 100644 --- a/poetry.lock +++ b/poetry.lock @@ -2306,36 +2306,34 @@ files = [ [[package]] name = "ray" -version = "2.3.0" +version = "2.3.1" description = "Ray provides a simple, universal API for building distributed applications." category = "main" optional = true python-versions = "*" files = [ - {file = "ray-2.3.0-cp310-cp310-macosx_10_15_universal2.whl", hash = "sha256:7a6a36682eaf22836da02a2ecf8e999f15707674cad5ed753b0efef497b1be1b"}, - {file = "ray-2.3.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:58c0d6574873c8276d29f2fc5473590c86da7b53eb7e747e47a3cc707797c37b"}, - {file = "ray-2.3.0-cp310-cp310-manylinux2014_aarch64.whl", hash = "sha256:f46ba421694e396bd34ae70c19032eb22926ecb58366175ab80b22de9b4b8b7f"}, - {file = "ray-2.3.0-cp310-cp310-manylinux2014_x86_64.whl", hash = "sha256:b19d381d425259c80b7a9b14aac9d898637ce262d91baeb9569999de3b043967"}, - {file = "ray-2.3.0-cp310-cp310-win_amd64.whl", hash = "sha256:f496b31cb58434e72f91fca2ddbf9622318d3b6b4b298ff8c45234a5fd1ca954"}, - {file = "ray-2.3.0-cp311-cp311-manylinux2014_aarch64.whl", hash = "sha256:1d2c72c1a9053a4eeae4db2cb2f1a73a4a2efa39cfe19c7306975c20f9a46653"}, - {file = "ray-2.3.0-cp311-cp311-manylinux2014_x86_64.whl", hash = "sha256:d86126b6ab5113a3be35281d5699bad731c8f90e02cf6561057017df9312f0a5"}, - {file = "ray-2.3.0-cp36-cp36m-macosx_10_15_intel.whl", hash = "sha256:288af5d1e7738512563f7a84961aad37c231a0c7d781d65a253294f55c23352b"}, - {file = "ray-2.3.0-cp36-cp36m-manylinux2014_aarch64.whl", hash = "sha256:1c4710a43168b0c3361e4bd5d15b1ec51bebcb7b68bf69ff0cc3a7b9049c8134"}, - {file = "ray-2.3.0-cp36-cp36m-manylinux2014_x86_64.whl", hash = "sha256:f9a79a510c262f38ac6a5a2603bf944f87081cfd5ccf59901585ef4fa20278ee"}, - {file = "ray-2.3.0-cp37-cp37m-macosx_10_15_intel.whl", hash = "sha256:15836d39eba4e4c209e0df0dbd5e3944dcf48755dbbe27847e9d05a23bffc20d"}, - {file = "ray-2.3.0-cp37-cp37m-manylinux2014_aarch64.whl", hash = "sha256:aa31c48e78f40e61a5fcdca69b660c1bff798aad2372e8623d0c1bc08fe177fa"}, - {file = "ray-2.3.0-cp37-cp37m-manylinux2014_x86_64.whl", hash = "sha256:93fb9ddfc11b9bade5f36e36428781bb4c1b56d8d66274ec832150c95123be64"}, - {file = "ray-2.3.0-cp37-cp37m-win_amd64.whl", hash = "sha256:fdc2c46ed34906a10b7c48bca952d7709e5c5d401ab3d1250fc67ba90b314616"}, - {file = "ray-2.3.0-cp38-cp38-macosx_10_15_x86_64.whl", hash = "sha256:2bd65a572d638ec9e100eb22a3089f4759a9e831ac9d9d8e3cc0ecb619e013b1"}, - {file = "ray-2.3.0-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:cb9f18c041ed40e814f8d17d5d99813023b88c57ae5c89800d564804dbdf4757"}, - {file = "ray-2.3.0-cp38-cp38-manylinux2014_aarch64.whl", hash = "sha256:dfc4ec7d4f4181731b61cafdb9c4f10fd535b891cd471fc1050bc8e1951df8e4"}, - {file = "ray-2.3.0-cp38-cp38-manylinux2014_x86_64.whl", hash = "sha256:dd5a7fd409f9653dfecd7cfad58a7e8104b03feca313ed1c16969331c5f0e422"}, - {file = "ray-2.3.0-cp38-cp38-win_amd64.whl", hash = "sha256:fc095984dc27a1775474f2074902d027fd7132b61f3d089e7954636c307f8714"}, - {file = "ray-2.3.0-cp39-cp39-macosx_10_15_x86_64.whl", hash = "sha256:ddcb1ebbb348be78d57fc8898855e344e02295a07ecedb7f10859785a1b23bdc"}, - {file = "ray-2.3.0-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:37168bbe809352bcba40a860da72b704209cde67c8c1c8d1509e91e896a57ce0"}, - {file = "ray-2.3.0-cp39-cp39-manylinux2014_aarch64.whl", hash = "sha256:5fb4d75c9c0a9f50bb7d413f3031ab5bcd20b1517b19908c9d60cb50ceb601e7"}, - {file = "ray-2.3.0-cp39-cp39-manylinux2014_x86_64.whl", hash = "sha256:596811ae6b41e50f038e0a7cdf0a5320151508d74bac84795bbccad4f6adc6a9"}, - {file = "ray-2.3.0-cp39-cp39-win_amd64.whl", hash = "sha256:6e13a541e621f58caa77592c912f96baac3b57737db4e87cc15d3ff6c94a4168"}, + {file = "ray-2.3.1-cp310-cp310-macosx_10_15_universal2.whl", hash = "sha256:9b62148078c4d64014a70e42d689fca5fb612040dc6bf2ba145f9e83857eb995"}, + {file = "ray-2.3.1-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:6f951ab6f0dc1321488c74e845540807b1cb5037b860528d83a7f0efeefaf56a"}, + {file = "ray-2.3.1-cp310-cp310-manylinux2014_aarch64.whl", hash = "sha256:72676a520c870a6bf21c02c33e1e7e61e75370701c0e5dc515c8767ff7bae2c8"}, + {file = "ray-2.3.1-cp310-cp310-manylinux2014_x86_64.whl", hash = "sha256:21d5e9ffa026593d4f1820c852e0dd2f2cfc880ba9b619370b218991bd96a42d"}, + {file = "ray-2.3.1-cp310-cp310-win_amd64.whl", hash = "sha256:b1484bf06d2e7bdb7b306467c62ecaac1f409d2553297c53751761ccdeb5a585"}, + {file = "ray-2.3.1-cp311-cp311-manylinux2014_aarch64.whl", hash = "sha256:8262daa5fc18733cc71b2200e7ad83f08263966e1d802a1d4fde59525c79c21f"}, + {file = "ray-2.3.1-cp311-cp311-manylinux2014_x86_64.whl", hash = "sha256:3436c4acbf90fd3d6a5ca14cbce6236f3c6e50bf9e939635d21a99e8cf3aca12"}, + {file = "ray-2.3.1-cp36-cp36m-manylinux2014_aarch64.whl", hash = "sha256:25967bc63f9052ae96b584926938efe7f299c56ea1b17b0f2e58a5ead994620f"}, + {file = "ray-2.3.1-cp36-cp36m-manylinux2014_x86_64.whl", hash = "sha256:c33154bceec74521e303cd8145e166bc07e6a6f3e90ac41478d00ae8e91e753e"}, + {file = "ray-2.3.1-cp37-cp37m-manylinux2014_aarch64.whl", hash = "sha256:5c6a4224bdd98d4dff9209bdc65a372f769de2d0de32c9edb1d25f72588486d5"}, + {file = "ray-2.3.1-cp37-cp37m-manylinux2014_x86_64.whl", hash = "sha256:7b5dc72a3dab1d3944ac7c8dd1026e8386d93af970dfcdc2d27925dd55d3136e"}, + {file = "ray-2.3.1-cp37-cp37m-win_amd64.whl", hash = "sha256:164596dc5113e86151bb24efefd18a830d7dbbd3aa566f025e310e0f9c095fd1"}, + {file = "ray-2.3.1-cp38-cp38-macosx_10_15_x86_64.whl", hash = "sha256:8e411155ab66ddf2b4e24088de7957c0ad5383ff11400eef2d7a35e6091d309b"}, + {file = "ray-2.3.1-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:1d9ed35a480ba4f92f0cae5cd653d2865e1e37c68321ed631b74bfa87ddafc1a"}, + {file = "ray-2.3.1-cp38-cp38-manylinux2014_aarch64.whl", hash = "sha256:3090f0be7e80ea25128b2caef7afa4a8e7cf73fd38a88e43c8d64c272d506a01"}, + {file = "ray-2.3.1-cp38-cp38-manylinux2014_x86_64.whl", hash = "sha256:4d362eb55c6ead4ced1946e3aaf0e92fb545ee8bee511f7bf1b97d5a8b7b490e"}, + {file = "ray-2.3.1-cp38-cp38-win_amd64.whl", hash = "sha256:47103766f1c40c74543d1c8e66567a7a19d31ecfc7ad2a0ac67c1ca80a338913"}, + {file = "ray-2.3.1-cp39-cp39-macosx_10_15_x86_64.whl", hash = "sha256:385573f198698c770e88b56505354f1b6677e305ad5caf04c6f5becbe61bf58b"}, + {file = "ray-2.3.1-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:75254f37067f18733afe93880b9847170770271db6be06b5f3df1151e4c2eb61"}, + {file = "ray-2.3.1-cp39-cp39-manylinux2014_aarch64.whl", hash = "sha256:8d0e97d976ec0cc12e13925fc3aaeb4077841d49194ccbdb6bca83d36214454d"}, + {file = "ray-2.3.1-cp39-cp39-manylinux2014_x86_64.whl", hash = "sha256:55c181d3921f886a0833800c2392d14fb58486fc008c6bae7cbde668dd54d43b"}, + {file = "ray-2.3.1-cp39-cp39-win_amd64.whl", hash = "sha256:165f7560c9d04c129ad8f1f1da1232f726bc046381f3b552d3468aaeb802adf9"}, ] [package.dependencies] @@ -2364,8 +2362,8 @@ virtualenv = ">=20.0.24" [package.extras] air = ["aiohttp (>=3.7)", "aiohttp-cors", "aiorwlock", "colorful", "fastapi", "fsspec", "gpustat (>=1.0.0)", "numpy (>=1.20)", "opencensus", "pandas", "pandas (>=1.3)", "prometheus-client (>=0.7.1)", "py-spy (>=0.2.0)", "pyarrow (>=6.0.1)", "pydantic", "requests", "smart-open", "starlette", "tabulate", "tensorboardX (>=1.9)", "uvicorn"] -all = ["aiohttp (>=3.7)", "aiohttp-cors", "aiorwlock", "colorful", "dm-tree", "fastapi", "fsspec", "gpustat (>=1.0.0)", "gymnasium (==0.26.3)", "kubernetes", "lz4", "numpy (>=1.20)", "opencensus", "opentelemetry-api", "opentelemetry-exporter-otlp", "opentelemetry-sdk", "pandas", "pandas (>=1.3)", "prometheus-client (>=0.7.1)", "py-spy (>=0.2.0)", "pyarrow (>=6.0.1)", "pydantic", "pyyaml", "ray-cpp (==2.3.0)", "requests", "rich", "scikit-image", "scipy", "smart-open", "starlette", "tabulate", "tensorboardX (>=1.9)", "typer", "urllib3", "uvicorn"] -cpp = ["ray-cpp (==2.3.0)"] +all = ["aiohttp (>=3.7)", "aiohttp-cors", "aiorwlock", "colorful", "dm-tree", "fastapi", "fsspec", "gpustat (>=1.0.0)", "gymnasium (==0.26.3)", "kubernetes", "lz4", "numpy (>=1.20)", "opencensus", "opentelemetry-api", "opentelemetry-exporter-otlp", "opentelemetry-sdk", "pandas", "pandas (>=1.3)", "prometheus-client (>=0.7.1)", "py-spy (>=0.2.0)", "pyarrow (>=6.0.1)", "pydantic", "pyyaml", "ray-cpp (==2.3.1)", "requests", "rich", "scikit-image", "scipy", "smart-open", "starlette", "tabulate", "tensorboardX (>=1.9)", "typer", "urllib3", "uvicorn"] +cpp = ["ray-cpp (==2.3.1)"] data = ["fsspec", "numpy (>=1.20)", "pandas (>=1.3)", "pyarrow (>=6.0.1)"] default = ["aiohttp (>=3.7)", "aiohttp-cors", "colorful", "gpustat (>=1.0.0)", "opencensus", "prometheus-client (>=0.7.1)", "py-spy (>=0.2.0)", "pydantic", "requests", "smart-open"] k8s = ["kubernetes", "urllib3"] @@ -2940,4 +2938,4 @@ snappy = ["python-snappy"] [metadata] lock-version = "2.0" python-versions = "^3.8" -content-hash = "a8f4da6d60917f374c291ed0bf8cc221793c6da0e929622f80e2e9db82316d80" +content-hash = "45caab46bbbb5bdf0bf328c789ec912493933138f02271244e74a71417fba3fe" diff --git a/pyproject.toml b/pyproject.toml index 4e5c0555bd..b218d81796 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -68,7 +68,7 @@ pandas = { version = ">=1.4.4,<=1.5.3", optional = true } duckdb = { version = ">=0.6.0,<=0.7.1", optional = true } -ray = { version = ">=2.0.0,<=2.3.0", optional = true } +ray = { version = ">=2.0.0,<=2.3.1", optional = true } python-snappy = { version = "0.6.1", optional = true } From fafc624670d59d77e1480a8f0346f4ee2e96bd87 Mon Sep 17 00:00:00 2001 From: Deepyaman Datta Date: Fri, 31 Mar 2023 07:49:34 -0400 Subject: [PATCH 414/642] Python: Relax the pin for `fsspec` implementations (#7242) --- poetry.lock | 693 ++++++++++++++++++++++--------------------------- pyproject.toml | 6 +- 2 files changed, 313 insertions(+), 386 deletions(-) diff --git a/poetry.lock b/poetry.lock index 1920780687..47cce87990 100644 --- a/poetry.lock +++ b/poetry.lock @@ -1,4 +1,4 @@ -# This file is automatically @generated by Poetry and should not be changed by hand. +# This file is automatically @generated by Poetry 1.4.1 and should not be changed by hand. [[package]] name = "adal" @@ -291,20 +291,24 @@ six = ">=1.12.0" [[package]] name = "azure-storage-blob" -version = "12.14.1" +version = "12.15.0" description = "Microsoft Azure Blob Storage Client Library for Python" category = "main" optional = true python-versions = ">=3.7" files = [ - {file = "azure-storage-blob-12.14.1.zip", hash = "sha256:860d4d82985a4bfc7d3271e71275af330f54f330a754355435a7ba749ccde997"}, - {file = "azure_storage_blob-12.14.1-py3-none-any.whl", hash = "sha256:52b84658e8df7853a3cf1c563814655b5028b979b2a87905b92aa6bb30be240e"}, + {file = "azure-storage-blob-12.15.0.zip", hash = "sha256:f8b8d582492740ab16744455408342fb8e4c8897b64a8a3fc31743844722c2f2"}, + {file = "azure_storage_blob-12.15.0-py3-none-any.whl", hash = "sha256:08d8807c577c63a436740627927c1a03a97c963efc29af5c818aed906590e1cf"}, ] [package.dependencies] -azure-core = ">=1.24.2,<2.0.0" +azure-core = ">=1.26.0,<2.0.0" cryptography = ">=2.1.4" -msrest = ">=0.7.1" +isodate = ">=0.6.1" +typing-extensions = ">=4.0.1" + +[package.extras] +aio = ["azure-core[aio] (>=1.26.0,<2.0.0)"] [[package]] name = "boto3" @@ -473,100 +477,87 @@ files = [ [[package]] name = "charset-normalizer" -version = "3.0.1" +version = "3.1.0" description = "The Real First Universal Charset Detector. Open, modern and actively maintained alternative to Chardet." category = "main" optional = false -python-versions = "*" +python-versions = ">=3.7.0" files = [ - {file = "charset-normalizer-3.0.1.tar.gz", hash = "sha256:ebea339af930f8ca5d7a699b921106c6e29c617fe9606fa7baa043c1cdae326f"}, - {file = "charset_normalizer-3.0.1-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:88600c72ef7587fe1708fd242b385b6ed4b8904976d5da0893e31df8b3480cb6"}, - {file = "charset_normalizer-3.0.1-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:c75ffc45f25324e68ab238cb4b5c0a38cd1c3d7f1fb1f72b5541de469e2247db"}, - {file = "charset_normalizer-3.0.1-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:db72b07027db150f468fbada4d85b3b2729a3db39178abf5c543b784c1254539"}, - {file = "charset_normalizer-3.0.1-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:62595ab75873d50d57323a91dd03e6966eb79c41fa834b7a1661ed043b2d404d"}, - {file = "charset_normalizer-3.0.1-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:ff6f3db31555657f3163b15a6b7c6938d08df7adbfc9dd13d9d19edad678f1e8"}, - {file = "charset_normalizer-3.0.1-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:772b87914ff1152b92a197ef4ea40efe27a378606c39446ded52c8f80f79702e"}, - {file = "charset_normalizer-3.0.1-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:70990b9c51340e4044cfc394a81f614f3f90d41397104d226f21e66de668730d"}, - {file = "charset_normalizer-3.0.1-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:292d5e8ba896bbfd6334b096e34bffb56161c81408d6d036a7dfa6929cff8783"}, - {file = "charset_normalizer-3.0.1-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:2edb64ee7bf1ed524a1da60cdcd2e1f6e2b4f66ef7c077680739f1641f62f555"}, - {file = "charset_normalizer-3.0.1-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:31a9ddf4718d10ae04d9b18801bd776693487cbb57d74cc3458a7673f6f34639"}, - {file = "charset_normalizer-3.0.1-cp310-cp310-musllinux_1_1_ppc64le.whl", hash = "sha256:44ba614de5361b3e5278e1241fda3dc1838deed864b50a10d7ce92983797fa76"}, - {file = "charset_normalizer-3.0.1-cp310-cp310-musllinux_1_1_s390x.whl", hash = "sha256:12db3b2c533c23ab812c2b25934f60383361f8a376ae272665f8e48b88e8e1c6"}, - {file = "charset_normalizer-3.0.1-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:c512accbd6ff0270939b9ac214b84fb5ada5f0409c44298361b2f5e13f9aed9e"}, - {file = "charset_normalizer-3.0.1-cp310-cp310-win32.whl", hash = "sha256:502218f52498a36d6bf5ea77081844017bf7982cdbe521ad85e64cabee1b608b"}, - {file = "charset_normalizer-3.0.1-cp310-cp310-win_amd64.whl", hash = "sha256:601f36512f9e28f029d9481bdaf8e89e5148ac5d89cffd3b05cd533eeb423b59"}, - {file = "charset_normalizer-3.0.1-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:0298eafff88c99982a4cf66ba2efa1128e4ddaca0b05eec4c456bbc7db691d8d"}, - {file = "charset_normalizer-3.0.1-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:a8d0fc946c784ff7f7c3742310cc8a57c5c6dc31631269876a88b809dbeff3d3"}, - {file = "charset_normalizer-3.0.1-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:87701167f2a5c930b403e9756fab1d31d4d4da52856143b609e30a1ce7160f3c"}, - {file = "charset_normalizer-3.0.1-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:14e76c0f23218b8f46c4d87018ca2e441535aed3632ca134b10239dfb6dadd6b"}, - {file = "charset_normalizer-3.0.1-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:0c0a590235ccd933d9892c627dec5bc7511ce6ad6c1011fdf5b11363022746c1"}, - {file = "charset_normalizer-3.0.1-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:8c7fe7afa480e3e82eed58e0ca89f751cd14d767638e2550c77a92a9e749c317"}, - {file = "charset_normalizer-3.0.1-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:79909e27e8e4fcc9db4addea88aa63f6423ebb171db091fb4373e3312cb6d603"}, - {file = "charset_normalizer-3.0.1-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:8ac7b6a045b814cf0c47f3623d21ebd88b3e8cf216a14790b455ea7ff0135d18"}, - {file = "charset_normalizer-3.0.1-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:72966d1b297c741541ca8cf1223ff262a6febe52481af742036a0b296e35fa5a"}, - {file = "charset_normalizer-3.0.1-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:f9d0c5c045a3ca9bedfc35dca8526798eb91a07aa7a2c0fee134c6c6f321cbd7"}, - {file = "charset_normalizer-3.0.1-cp311-cp311-musllinux_1_1_ppc64le.whl", hash = "sha256:5995f0164fa7df59db4746112fec3f49c461dd6b31b841873443bdb077c13cfc"}, - {file = "charset_normalizer-3.0.1-cp311-cp311-musllinux_1_1_s390x.whl", hash = "sha256:4a8fcf28c05c1f6d7e177a9a46a1c52798bfe2ad80681d275b10dcf317deaf0b"}, - {file = "charset_normalizer-3.0.1-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:761e8904c07ad053d285670f36dd94e1b6ab7f16ce62b9805c475b7aa1cffde6"}, - {file = "charset_normalizer-3.0.1-cp311-cp311-win32.whl", hash = "sha256:71140351489970dfe5e60fc621ada3e0f41104a5eddaca47a7acb3c1b851d6d3"}, - {file = "charset_normalizer-3.0.1-cp311-cp311-win_amd64.whl", hash = "sha256:9ab77acb98eba3fd2a85cd160851816bfce6871d944d885febf012713f06659c"}, - {file = "charset_normalizer-3.0.1-cp36-cp36m-macosx_10_9_x86_64.whl", hash = "sha256:84c3990934bae40ea69a82034912ffe5a62c60bbf6ec5bc9691419641d7d5c9a"}, - {file = "charset_normalizer-3.0.1-cp36-cp36m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:74292fc76c905c0ef095fe11e188a32ebd03bc38f3f3e9bcb85e4e6db177b7ea"}, - {file = "charset_normalizer-3.0.1-cp36-cp36m-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:c95a03c79bbe30eec3ec2b7f076074f4281526724c8685a42872974ef4d36b72"}, - {file = "charset_normalizer-3.0.1-cp36-cp36m-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:f4c39b0e3eac288fedc2b43055cfc2ca7a60362d0e5e87a637beac5d801ef478"}, - {file = "charset_normalizer-3.0.1-cp36-cp36m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:df2c707231459e8a4028eabcd3cfc827befd635b3ef72eada84ab13b52e1574d"}, - {file = "charset_normalizer-3.0.1-cp36-cp36m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:93ad6d87ac18e2a90b0fe89df7c65263b9a99a0eb98f0a3d2e079f12a0735837"}, - {file = "charset_normalizer-3.0.1-cp36-cp36m-musllinux_1_1_aarch64.whl", hash = "sha256:59e5686dd847347e55dffcc191a96622f016bc0ad89105e24c14e0d6305acbc6"}, - {file = "charset_normalizer-3.0.1-cp36-cp36m-musllinux_1_1_i686.whl", hash = "sha256:cd6056167405314a4dc3c173943f11249fa0f1b204f8b51ed4bde1a9cd1834dc"}, - {file = "charset_normalizer-3.0.1-cp36-cp36m-musllinux_1_1_ppc64le.whl", hash = "sha256:083c8d17153ecb403e5e1eb76a7ef4babfc2c48d58899c98fcaa04833e7a2f9a"}, - {file = "charset_normalizer-3.0.1-cp36-cp36m-musllinux_1_1_s390x.whl", hash = "sha256:f5057856d21e7586765171eac8b9fc3f7d44ef39425f85dbcccb13b3ebea806c"}, - {file = "charset_normalizer-3.0.1-cp36-cp36m-musllinux_1_1_x86_64.whl", hash = "sha256:7eb33a30d75562222b64f569c642ff3dc6689e09adda43a082208397f016c39a"}, - {file = "charset_normalizer-3.0.1-cp36-cp36m-win32.whl", hash = "sha256:95dea361dd73757c6f1c0a1480ac499952c16ac83f7f5f4f84f0658a01b8ef41"}, - {file = "charset_normalizer-3.0.1-cp36-cp36m-win_amd64.whl", hash = "sha256:eaa379fcd227ca235d04152ca6704c7cb55564116f8bc52545ff357628e10602"}, - {file = "charset_normalizer-3.0.1-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:3e45867f1f2ab0711d60c6c71746ac53537f1684baa699f4f668d4c6f6ce8e14"}, - {file = "charset_normalizer-3.0.1-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:cadaeaba78750d58d3cc6ac4d1fd867da6fc73c88156b7a3212a3cd4819d679d"}, - {file = "charset_normalizer-3.0.1-cp37-cp37m-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:911d8a40b2bef5b8bbae2e36a0b103f142ac53557ab421dc16ac4aafee6f53dc"}, - {file = "charset_normalizer-3.0.1-cp37-cp37m-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:503e65837c71b875ecdd733877d852adbc465bd82c768a067badd953bf1bc5a3"}, - {file = "charset_normalizer-3.0.1-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:a60332922359f920193b1d4826953c507a877b523b2395ad7bc716ddd386d866"}, - {file = "charset_normalizer-3.0.1-cp37-cp37m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:16a8663d6e281208d78806dbe14ee9903715361cf81f6d4309944e4d1e59ac5b"}, - {file = "charset_normalizer-3.0.1-cp37-cp37m-musllinux_1_1_aarch64.whl", hash = "sha256:a16418ecf1329f71df119e8a65f3aa68004a3f9383821edcb20f0702934d8087"}, - {file = "charset_normalizer-3.0.1-cp37-cp37m-musllinux_1_1_i686.whl", hash = "sha256:9d9153257a3f70d5f69edf2325357251ed20f772b12e593f3b3377b5f78e7ef8"}, - {file = "charset_normalizer-3.0.1-cp37-cp37m-musllinux_1_1_ppc64le.whl", hash = "sha256:02a51034802cbf38db3f89c66fb5d2ec57e6fe7ef2f4a44d070a593c3688667b"}, - {file = "charset_normalizer-3.0.1-cp37-cp37m-musllinux_1_1_s390x.whl", hash = "sha256:2e396d70bc4ef5325b72b593a72c8979999aa52fb8bcf03f701c1b03e1166918"}, - {file = "charset_normalizer-3.0.1-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:11b53acf2411c3b09e6af37e4b9005cba376c872503c8f28218c7243582df45d"}, - {file = "charset_normalizer-3.0.1-cp37-cp37m-win32.whl", hash = "sha256:0bf2dae5291758b6f84cf923bfaa285632816007db0330002fa1de38bfcb7154"}, - {file = "charset_normalizer-3.0.1-cp37-cp37m-win_amd64.whl", hash = "sha256:2c03cc56021a4bd59be889c2b9257dae13bf55041a3372d3295416f86b295fb5"}, - {file = "charset_normalizer-3.0.1-cp38-cp38-macosx_10_9_universal2.whl", hash = "sha256:024e606be3ed92216e2b6952ed859d86b4cfa52cd5bc5f050e7dc28f9b43ec42"}, - {file = "charset_normalizer-3.0.1-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:4b0d02d7102dd0f997580b51edc4cebcf2ab6397a7edf89f1c73b586c614272c"}, - {file = "charset_normalizer-3.0.1-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:358a7c4cb8ba9b46c453b1dd8d9e431452d5249072e4f56cfda3149f6ab1405e"}, - {file = "charset_normalizer-3.0.1-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:81d6741ab457d14fdedc215516665050f3822d3e56508921cc7239f8c8e66a58"}, - {file = "charset_normalizer-3.0.1-cp38-cp38-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:8b8af03d2e37866d023ad0ddea594edefc31e827fee64f8de5611a1dbc373174"}, - {file = "charset_normalizer-3.0.1-cp38-cp38-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:9cf4e8ad252f7c38dd1f676b46514f92dc0ebeb0db5552f5f403509705e24753"}, - {file = "charset_normalizer-3.0.1-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:e696f0dd336161fca9adbb846875d40752e6eba585843c768935ba5c9960722b"}, - {file = "charset_normalizer-3.0.1-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:c22d3fe05ce11d3671297dc8973267daa0f938b93ec716e12e0f6dee81591dc1"}, - {file = "charset_normalizer-3.0.1-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:109487860ef6a328f3eec66f2bf78b0b72400280d8f8ea05f69c51644ba6521a"}, - {file = "charset_normalizer-3.0.1-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:37f8febc8ec50c14f3ec9637505f28e58d4f66752207ea177c1d67df25da5aed"}, - {file = "charset_normalizer-3.0.1-cp38-cp38-musllinux_1_1_ppc64le.whl", hash = "sha256:f97e83fa6c25693c7a35de154681fcc257c1c41b38beb0304b9c4d2d9e164479"}, - {file = "charset_normalizer-3.0.1-cp38-cp38-musllinux_1_1_s390x.whl", hash = "sha256:a152f5f33d64a6be73f1d30c9cc82dfc73cec6477ec268e7c6e4c7d23c2d2291"}, - {file = "charset_normalizer-3.0.1-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:39049da0ffb96c8cbb65cbf5c5f3ca3168990adf3551bd1dee10c48fce8ae820"}, - {file = "charset_normalizer-3.0.1-cp38-cp38-win32.whl", hash = "sha256:4457ea6774b5611f4bed5eaa5df55f70abde42364d498c5134b7ef4c6958e20e"}, - {file = "charset_normalizer-3.0.1-cp38-cp38-win_amd64.whl", hash = "sha256:e62164b50f84e20601c1ff8eb55620d2ad25fb81b59e3cd776a1902527a788af"}, - {file = "charset_normalizer-3.0.1-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:8eade758719add78ec36dc13201483f8e9b5d940329285edcd5f70c0a9edbd7f"}, - {file = "charset_normalizer-3.0.1-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:8499ca8f4502af841f68135133d8258f7b32a53a1d594aa98cc52013fff55678"}, - {file = "charset_normalizer-3.0.1-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:3fc1c4a2ffd64890aebdb3f97e1278b0cc72579a08ca4de8cd2c04799a3a22be"}, - {file = "charset_normalizer-3.0.1-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:00d3ffdaafe92a5dc603cb9bd5111aaa36dfa187c8285c543be562e61b755f6b"}, - {file = "charset_normalizer-3.0.1-cp39-cp39-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:c2ac1b08635a8cd4e0cbeaf6f5e922085908d48eb05d44c5ae9eabab148512ca"}, - {file = "charset_normalizer-3.0.1-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:f6f45710b4459401609ebebdbcfb34515da4fc2aa886f95107f556ac69a9147e"}, - {file = "charset_normalizer-3.0.1-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:3ae1de54a77dc0d6d5fcf623290af4266412a7c4be0b1ff7444394f03f5c54e3"}, - {file = "charset_normalizer-3.0.1-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:3b590df687e3c5ee0deef9fc8c547d81986d9a1b56073d82de008744452d6541"}, - {file = "charset_normalizer-3.0.1-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:ab5de034a886f616a5668aa5d098af2b5385ed70142090e2a31bcbd0af0fdb3d"}, - {file = "charset_normalizer-3.0.1-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:9cb3032517f1627cc012dbc80a8ec976ae76d93ea2b5feaa9d2a5b8882597579"}, - {file = "charset_normalizer-3.0.1-cp39-cp39-musllinux_1_1_ppc64le.whl", hash = "sha256:608862a7bf6957f2333fc54ab4399e405baad0163dc9f8d99cb236816db169d4"}, - {file = "charset_normalizer-3.0.1-cp39-cp39-musllinux_1_1_s390x.whl", hash = "sha256:0f438ae3532723fb6ead77e7c604be7c8374094ef4ee2c5e03a3a17f1fca256c"}, - {file = "charset_normalizer-3.0.1-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:356541bf4381fa35856dafa6a965916e54bed415ad8a24ee6de6e37deccf2786"}, - {file = "charset_normalizer-3.0.1-cp39-cp39-win32.whl", hash = "sha256:39cf9ed17fe3b1bc81f33c9ceb6ce67683ee7526e65fde1447c772afc54a1bb8"}, - {file = "charset_normalizer-3.0.1-cp39-cp39-win_amd64.whl", hash = "sha256:0a11e971ed097d24c534c037d298ad32c6ce81a45736d31e0ff0ad37ab437d59"}, - {file = "charset_normalizer-3.0.1-py3-none-any.whl", hash = "sha256:7e189e2e1d3ed2f4aebabd2d5b0f931e883676e51c7624826e0a4e5fe8a0bf24"}, + {file = "charset-normalizer-3.1.0.tar.gz", hash = "sha256:34e0a2f9c370eb95597aae63bf85eb5e96826d81e3dcf88b8886012906f509b5"}, + {file = "charset_normalizer-3.1.0-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:e0ac8959c929593fee38da1c2b64ee9778733cdf03c482c9ff1d508b6b593b2b"}, + {file = "charset_normalizer-3.1.0-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:d7fc3fca01da18fbabe4625d64bb612b533533ed10045a2ac3dd194bfa656b60"}, + {file = "charset_normalizer-3.1.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:04eefcee095f58eaabe6dc3cc2262f3bcd776d2c67005880894f447b3f2cb9c1"}, + {file = "charset_normalizer-3.1.0-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:20064ead0717cf9a73a6d1e779b23d149b53daf971169289ed2ed43a71e8d3b0"}, + {file = "charset_normalizer-3.1.0-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:1435ae15108b1cb6fffbcea2af3d468683b7afed0169ad718451f8db5d1aff6f"}, + {file = "charset_normalizer-3.1.0-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:c84132a54c750fda57729d1e2599bb598f5fa0344085dbde5003ba429a4798c0"}, + {file = "charset_normalizer-3.1.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:75f2568b4189dda1c567339b48cba4ac7384accb9c2a7ed655cd86b04055c795"}, + {file = "charset_normalizer-3.1.0-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:11d3bcb7be35e7b1bba2c23beedac81ee893ac9871d0ba79effc7fc01167db6c"}, + {file = "charset_normalizer-3.1.0-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:891cf9b48776b5c61c700b55a598621fdb7b1e301a550365571e9624f270c203"}, + {file = "charset_normalizer-3.1.0-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:5f008525e02908b20e04707a4f704cd286d94718f48bb33edddc7d7b584dddc1"}, + {file = "charset_normalizer-3.1.0-cp310-cp310-musllinux_1_1_ppc64le.whl", hash = "sha256:b06f0d3bf045158d2fb8837c5785fe9ff9b8c93358be64461a1089f5da983137"}, + {file = "charset_normalizer-3.1.0-cp310-cp310-musllinux_1_1_s390x.whl", hash = "sha256:49919f8400b5e49e961f320c735388ee686a62327e773fa5b3ce6721f7e785ce"}, + {file = "charset_normalizer-3.1.0-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:22908891a380d50738e1f978667536f6c6b526a2064156203d418f4856d6e86a"}, + {file = "charset_normalizer-3.1.0-cp310-cp310-win32.whl", hash = "sha256:12d1a39aa6b8c6f6248bb54550efcc1c38ce0d8096a146638fd4738e42284448"}, + {file = "charset_normalizer-3.1.0-cp310-cp310-win_amd64.whl", hash = "sha256:65ed923f84a6844de5fd29726b888e58c62820e0769b76565480e1fdc3d062f8"}, + {file = "charset_normalizer-3.1.0-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:9a3267620866c9d17b959a84dd0bd2d45719b817245e49371ead79ed4f710d19"}, + {file = "charset_normalizer-3.1.0-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:6734e606355834f13445b6adc38b53c0fd45f1a56a9ba06c2058f86893ae8017"}, + {file = "charset_normalizer-3.1.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:f8303414c7b03f794347ad062c0516cee0e15f7a612abd0ce1e25caf6ceb47df"}, + {file = "charset_normalizer-3.1.0-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:aaf53a6cebad0eae578f062c7d462155eada9c172bd8c4d250b8c1d8eb7f916a"}, + {file = "charset_normalizer-3.1.0-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:3dc5b6a8ecfdc5748a7e429782598e4f17ef378e3e272eeb1340ea57c9109f41"}, + {file = "charset_normalizer-3.1.0-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:e1b25e3ad6c909f398df8921780d6a3d120d8c09466720226fc621605b6f92b1"}, + {file = "charset_normalizer-3.1.0-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:0ca564606d2caafb0abe6d1b5311c2649e8071eb241b2d64e75a0d0065107e62"}, + {file = "charset_normalizer-3.1.0-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:b82fab78e0b1329e183a65260581de4375f619167478dddab510c6c6fb04d9b6"}, + {file = "charset_normalizer-3.1.0-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:bd7163182133c0c7701b25e604cf1611c0d87712e56e88e7ee5d72deab3e76b5"}, + {file = "charset_normalizer-3.1.0-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:11d117e6c63e8f495412d37e7dc2e2fff09c34b2d09dbe2bee3c6229577818be"}, + {file = "charset_normalizer-3.1.0-cp311-cp311-musllinux_1_1_ppc64le.whl", hash = "sha256:cf6511efa4801b9b38dc5546d7547d5b5c6ef4b081c60b23e4d941d0eba9cbeb"}, + {file = "charset_normalizer-3.1.0-cp311-cp311-musllinux_1_1_s390x.whl", hash = "sha256:abc1185d79f47c0a7aaf7e2412a0eb2c03b724581139193d2d82b3ad8cbb00ac"}, + {file = "charset_normalizer-3.1.0-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:cb7b2ab0188829593b9de646545175547a70d9a6e2b63bf2cd87a0a391599324"}, + {file = "charset_normalizer-3.1.0-cp311-cp311-win32.whl", hash = "sha256:c36bcbc0d5174a80d6cccf43a0ecaca44e81d25be4b7f90f0ed7bcfbb5a00909"}, + {file = "charset_normalizer-3.1.0-cp311-cp311-win_amd64.whl", hash = "sha256:cca4def576f47a09a943666b8f829606bcb17e2bc2d5911a46c8f8da45f56755"}, + {file = "charset_normalizer-3.1.0-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:0c95f12b74681e9ae127728f7e5409cbbef9cd914d5896ef238cc779b8152373"}, + {file = "charset_normalizer-3.1.0-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:fca62a8301b605b954ad2e9c3666f9d97f63872aa4efcae5492baca2056b74ab"}, + {file = "charset_normalizer-3.1.0-cp37-cp37m-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:ac0aa6cd53ab9a31d397f8303f92c42f534693528fafbdb997c82bae6e477ad9"}, + {file = "charset_normalizer-3.1.0-cp37-cp37m-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:c3af8e0f07399d3176b179f2e2634c3ce9c1301379a6b8c9c9aeecd481da494f"}, + {file = "charset_normalizer-3.1.0-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:3a5fc78f9e3f501a1614a98f7c54d3969f3ad9bba8ba3d9b438c3bc5d047dd28"}, + {file = "charset_normalizer-3.1.0-cp37-cp37m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:628c985afb2c7d27a4800bfb609e03985aaecb42f955049957814e0491d4006d"}, + {file = "charset_normalizer-3.1.0-cp37-cp37m-musllinux_1_1_aarch64.whl", hash = "sha256:74db0052d985cf37fa111828d0dd230776ac99c740e1a758ad99094be4f1803d"}, + {file = "charset_normalizer-3.1.0-cp37-cp37m-musllinux_1_1_i686.whl", hash = "sha256:1e8fcdd8f672a1c4fc8d0bd3a2b576b152d2a349782d1eb0f6b8e52e9954731d"}, + {file = "charset_normalizer-3.1.0-cp37-cp37m-musllinux_1_1_ppc64le.whl", hash = "sha256:04afa6387e2b282cf78ff3dbce20f0cc071c12dc8f685bd40960cc68644cfea6"}, + {file = "charset_normalizer-3.1.0-cp37-cp37m-musllinux_1_1_s390x.whl", hash = "sha256:dd5653e67b149503c68c4018bf07e42eeed6b4e956b24c00ccdf93ac79cdff84"}, + {file = "charset_normalizer-3.1.0-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:d2686f91611f9e17f4548dbf050e75b079bbc2a82be565832bc8ea9047b61c8c"}, + {file = "charset_normalizer-3.1.0-cp37-cp37m-win32.whl", hash = "sha256:4155b51ae05ed47199dc5b2a4e62abccb274cee6b01da5b895099b61b1982974"}, + {file = "charset_normalizer-3.1.0-cp37-cp37m-win_amd64.whl", hash = "sha256:322102cdf1ab682ecc7d9b1c5eed4ec59657a65e1c146a0da342b78f4112db23"}, + {file = "charset_normalizer-3.1.0-cp38-cp38-macosx_10_9_universal2.whl", hash = "sha256:e633940f28c1e913615fd624fcdd72fdba807bf53ea6925d6a588e84e1151531"}, + {file = "charset_normalizer-3.1.0-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:3a06f32c9634a8705f4ca9946d667609f52cf130d5548881401f1eb2c39b1e2c"}, + {file = "charset_normalizer-3.1.0-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:7381c66e0561c5757ffe616af869b916c8b4e42b367ab29fedc98481d1e74e14"}, + {file = "charset_normalizer-3.1.0-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:3573d376454d956553c356df45bb824262c397c6e26ce43e8203c4c540ee0acb"}, + {file = "charset_normalizer-3.1.0-cp38-cp38-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:e89df2958e5159b811af9ff0f92614dabf4ff617c03a4c1c6ff53bf1c399e0e1"}, + {file = "charset_normalizer-3.1.0-cp38-cp38-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:78cacd03e79d009d95635e7d6ff12c21eb89b894c354bd2b2ed0b4763373693b"}, + {file = "charset_normalizer-3.1.0-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:de5695a6f1d8340b12a5d6d4484290ee74d61e467c39ff03b39e30df62cf83a0"}, + {file = "charset_normalizer-3.1.0-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:1c60b9c202d00052183c9be85e5eaf18a4ada0a47d188a83c8f5c5b23252f649"}, + {file = "charset_normalizer-3.1.0-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:f645caaf0008bacf349875a974220f1f1da349c5dbe7c4ec93048cdc785a3326"}, + {file = "charset_normalizer-3.1.0-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:ea9f9c6034ea2d93d9147818f17c2a0860d41b71c38b9ce4d55f21b6f9165a11"}, + {file = "charset_normalizer-3.1.0-cp38-cp38-musllinux_1_1_ppc64le.whl", hash = "sha256:80d1543d58bd3d6c271b66abf454d437a438dff01c3e62fdbcd68f2a11310d4b"}, + {file = "charset_normalizer-3.1.0-cp38-cp38-musllinux_1_1_s390x.whl", hash = "sha256:73dc03a6a7e30b7edc5b01b601e53e7fc924b04e1835e8e407c12c037e81adbd"}, + {file = "charset_normalizer-3.1.0-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:6f5c2e7bc8a4bf7c426599765b1bd33217ec84023033672c1e9a8b35eaeaaaf8"}, + {file = "charset_normalizer-3.1.0-cp38-cp38-win32.whl", hash = "sha256:12a2b561af122e3d94cdb97fe6fb2bb2b82cef0cdca131646fdb940a1eda04f0"}, + {file = "charset_normalizer-3.1.0-cp38-cp38-win_amd64.whl", hash = "sha256:3160a0fd9754aab7d47f95a6b63ab355388d890163eb03b2d2b87ab0a30cfa59"}, + {file = "charset_normalizer-3.1.0-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:38e812a197bf8e71a59fe55b757a84c1f946d0ac114acafaafaf21667a7e169e"}, + {file = "charset_normalizer-3.1.0-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:6baf0baf0d5d265fa7944feb9f7451cc316bfe30e8df1a61b1bb08577c554f31"}, + {file = "charset_normalizer-3.1.0-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:8f25e17ab3039b05f762b0a55ae0b3632b2e073d9c8fc88e89aca31a6198e88f"}, + {file = "charset_normalizer-3.1.0-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:3747443b6a904001473370d7810aa19c3a180ccd52a7157aacc264a5ac79265e"}, + {file = "charset_normalizer-3.1.0-cp39-cp39-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:b116502087ce8a6b7a5f1814568ccbd0e9f6cfd99948aa59b0e241dc57cf739f"}, + {file = "charset_normalizer-3.1.0-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:d16fd5252f883eb074ca55cb622bc0bee49b979ae4e8639fff6ca3ff44f9f854"}, + {file = "charset_normalizer-3.1.0-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:21fa558996782fc226b529fdd2ed7866c2c6ec91cee82735c98a197fae39f706"}, + {file = "charset_normalizer-3.1.0-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:6f6c7a8a57e9405cad7485f4c9d3172ae486cfef1344b5ddd8e5239582d7355e"}, + {file = "charset_normalizer-3.1.0-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:ac3775e3311661d4adace3697a52ac0bab17edd166087d493b52d4f4f553f9f0"}, + {file = "charset_normalizer-3.1.0-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:10c93628d7497c81686e8e5e557aafa78f230cd9e77dd0c40032ef90c18f2230"}, + {file = "charset_normalizer-3.1.0-cp39-cp39-musllinux_1_1_ppc64le.whl", hash = "sha256:6f4f4668e1831850ebcc2fd0b1cd11721947b6dc7c00bf1c6bd3c929ae14f2c7"}, + {file = "charset_normalizer-3.1.0-cp39-cp39-musllinux_1_1_s390x.whl", hash = "sha256:0be65ccf618c1e7ac9b849c315cc2e8a8751d9cfdaa43027d4f6624bd587ab7e"}, + {file = "charset_normalizer-3.1.0-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:53d0a3fa5f8af98a1e261de6a3943ca631c526635eb5817a87a59d9a57ebf48f"}, + {file = "charset_normalizer-3.1.0-cp39-cp39-win32.whl", hash = "sha256:a04f86f41a8916fe45ac5024ec477f41f886b3c435da2d4e3d2709b22ab02af1"}, + {file = "charset_normalizer-3.1.0-cp39-cp39-win_amd64.whl", hash = "sha256:830d2948a5ec37c386d3170c483063798d7879037492540f10a475e3fd6f244b"}, + {file = "charset_normalizer-3.1.0-py3-none-any.whl", hash = "sha256:3d9098b479e78c85080c98e1e35ff40b4a31d8953102bb0fd7d1b6f8a2111a3d"}, ] [[package]] @@ -665,35 +656,31 @@ toml = ["tomli"] [[package]] name = "cryptography" -version = "39.0.1" +version = "40.0.1" description = "cryptography is a package which provides cryptographic recipes and primitives to Python developers." category = "main" optional = false python-versions = ">=3.6" files = [ - {file = "cryptography-39.0.1-cp36-abi3-macosx_10_12_universal2.whl", hash = "sha256:6687ef6d0a6497e2b58e7c5b852b53f62142cfa7cd1555795758934da363a965"}, - {file = "cryptography-39.0.1-cp36-abi3-macosx_10_12_x86_64.whl", hash = "sha256:706843b48f9a3f9b9911979761c91541e3d90db1ca905fd63fee540a217698bc"}, - {file = "cryptography-39.0.1-cp36-abi3-manylinux_2_17_aarch64.manylinux2014_aarch64.manylinux_2_24_aarch64.whl", hash = "sha256:5d2d8b87a490bfcd407ed9d49093793d0f75198a35e6eb1a923ce1ee86c62b41"}, - {file = "cryptography-39.0.1-cp36-abi3-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:83e17b26de248c33f3acffb922748151d71827d6021d98c70e6c1a25ddd78505"}, - {file = "cryptography-39.0.1-cp36-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:e124352fd3db36a9d4a21c1aa27fd5d051e621845cb87fb851c08f4f75ce8be6"}, - {file = "cryptography-39.0.1-cp36-abi3-manylinux_2_24_x86_64.whl", hash = "sha256:5aa67414fcdfa22cf052e640cb5ddc461924a045cacf325cd164e65312d99502"}, - {file = "cryptography-39.0.1-cp36-abi3-manylinux_2_28_aarch64.whl", hash = "sha256:35f7c7d015d474f4011e859e93e789c87d21f6f4880ebdc29896a60403328f1f"}, - {file = "cryptography-39.0.1-cp36-abi3-manylinux_2_28_x86_64.whl", hash = "sha256:f24077a3b5298a5a06a8e0536e3ea9ec60e4c7ac486755e5fb6e6ea9b3500106"}, - {file = "cryptography-39.0.1-cp36-abi3-musllinux_1_1_aarch64.whl", hash = "sha256:f0c64d1bd842ca2633e74a1a28033d139368ad959872533b1bab8c80e8240a0c"}, - {file = "cryptography-39.0.1-cp36-abi3-musllinux_1_1_x86_64.whl", hash = "sha256:0f8da300b5c8af9f98111ffd512910bc792b4c77392a9523624680f7956a99d4"}, - {file = "cryptography-39.0.1-cp36-abi3-win32.whl", hash = "sha256:fe913f20024eb2cb2f323e42a64bdf2911bb9738a15dba7d3cce48151034e3a8"}, - {file = "cryptography-39.0.1-cp36-abi3-win_amd64.whl", hash = "sha256:ced4e447ae29ca194449a3f1ce132ded8fcab06971ef5f618605aacaa612beac"}, - {file = "cryptography-39.0.1-pp38-pypy38_pp73-macosx_10_12_x86_64.whl", hash = "sha256:807ce09d4434881ca3a7594733669bd834f5b2c6d5c7e36f8c00f691887042ad"}, - {file = "cryptography-39.0.1-pp38-pypy38_pp73-manylinux_2_28_aarch64.whl", hash = "sha256:c5caeb8188c24888c90b5108a441c106f7faa4c4c075a2bcae438c6e8ca73cef"}, - {file = "cryptography-39.0.1-pp38-pypy38_pp73-manylinux_2_28_x86_64.whl", hash = "sha256:4789d1e3e257965e960232345002262ede4d094d1a19f4d3b52e48d4d8f3b885"}, - {file = "cryptography-39.0.1-pp38-pypy38_pp73-win_amd64.whl", hash = "sha256:96f1157a7c08b5b189b16b47bc9db2332269d6680a196341bf30046330d15388"}, - {file = "cryptography-39.0.1-pp39-pypy39_pp73-macosx_10_12_x86_64.whl", hash = "sha256:e422abdec8b5fa8462aa016786680720d78bdce7a30c652b7fadf83a4ba35336"}, - {file = "cryptography-39.0.1-pp39-pypy39_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.manylinux_2_24_aarch64.whl", hash = "sha256:b0afd054cd42f3d213bf82c629efb1ee5f22eba35bf0eec88ea9ea7304f511a2"}, - {file = "cryptography-39.0.1-pp39-pypy39_pp73-manylinux_2_24_x86_64.whl", hash = "sha256:6f8ba7f0328b79f08bdacc3e4e66fb4d7aab0c3584e0bd41328dce5262e26b2e"}, - {file = "cryptography-39.0.1-pp39-pypy39_pp73-manylinux_2_28_aarch64.whl", hash = "sha256:ef8b72fa70b348724ff1218267e7f7375b8de4e8194d1636ee60510aae104cd0"}, - {file = "cryptography-39.0.1-pp39-pypy39_pp73-manylinux_2_28_x86_64.whl", hash = "sha256:aec5a6c9864be7df2240c382740fcf3b96928c46604eaa7f3091f58b878c0bb6"}, - {file = "cryptography-39.0.1-pp39-pypy39_pp73-win_amd64.whl", hash = "sha256:fdd188c8a6ef8769f148f88f859884507b954cc64db6b52f66ef199bb9ad660a"}, - {file = "cryptography-39.0.1.tar.gz", hash = "sha256:d1f6198ee6d9148405e49887803907fe8962a23e6c6f83ea7d98f1c0de375695"}, + {file = "cryptography-40.0.1-cp36-abi3-macosx_10_12_universal2.whl", hash = "sha256:918cb89086c7d98b1b86b9fdb70c712e5a9325ba6f7d7cfb509e784e0cfc6917"}, + {file = "cryptography-40.0.1-cp36-abi3-macosx_10_12_x86_64.whl", hash = "sha256:9618a87212cb5200500e304e43691111570e1f10ec3f35569fdfcd17e28fd797"}, + {file = "cryptography-40.0.1-cp36-abi3-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:3a4805a4ca729d65570a1b7cac84eac1e431085d40387b7d3bbaa47e39890b88"}, + {file = "cryptography-40.0.1-cp36-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:63dac2d25c47f12a7b8aa60e528bfb3c51c5a6c5a9f7c86987909c6c79765554"}, + {file = "cryptography-40.0.1-cp36-abi3-manylinux_2_28_aarch64.whl", hash = "sha256:0a4e3406cfed6b1f6d6e87ed243363652b2586b2d917b0609ca4f97072994405"}, + {file = "cryptography-40.0.1-cp36-abi3-manylinux_2_28_x86_64.whl", hash = "sha256:1e0af458515d5e4028aad75f3bb3fe7a31e46ad920648cd59b64d3da842e4356"}, + {file = "cryptography-40.0.1-cp36-abi3-musllinux_1_1_aarch64.whl", hash = "sha256:d8aa3609d337ad85e4eb9bb0f8bcf6e4409bfb86e706efa9a027912169e89122"}, + {file = "cryptography-40.0.1-cp36-abi3-musllinux_1_1_x86_64.whl", hash = "sha256:cf91e428c51ef692b82ce786583e214f58392399cf65c341bc7301d096fa3ba2"}, + {file = "cryptography-40.0.1-cp36-abi3-win32.whl", hash = "sha256:650883cc064297ef3676b1db1b7b1df6081794c4ada96fa457253c4cc40f97db"}, + {file = "cryptography-40.0.1-cp36-abi3-win_amd64.whl", hash = "sha256:a805a7bce4a77d51696410005b3e85ae2839bad9aa38894afc0aa99d8e0c3160"}, + {file = "cryptography-40.0.1-pp38-pypy38_pp73-macosx_10_12_x86_64.whl", hash = "sha256:cd033d74067d8928ef00a6b1327c8ea0452523967ca4463666eeba65ca350d4c"}, + {file = "cryptography-40.0.1-pp38-pypy38_pp73-manylinux_2_28_aarch64.whl", hash = "sha256:d36bbeb99704aabefdca5aee4eba04455d7a27ceabd16f3b3ba9bdcc31da86c4"}, + {file = "cryptography-40.0.1-pp38-pypy38_pp73-manylinux_2_28_x86_64.whl", hash = "sha256:32057d3d0ab7d4453778367ca43e99ddb711770477c4f072a51b3ca69602780a"}, + {file = "cryptography-40.0.1-pp38-pypy38_pp73-win_amd64.whl", hash = "sha256:f5d7b79fa56bc29580faafc2ff736ce05ba31feaa9d4735048b0de7d9ceb2b94"}, + {file = "cryptography-40.0.1-pp39-pypy39_pp73-macosx_10_12_x86_64.whl", hash = "sha256:7c872413353c70e0263a9368c4993710070e70ab3e5318d85510cc91cce77e7c"}, + {file = "cryptography-40.0.1-pp39-pypy39_pp73-manylinux_2_28_aarch64.whl", hash = "sha256:28d63d75bf7ae4045b10de5413fb1d6338616e79015999ad9cf6fc538f772d41"}, + {file = "cryptography-40.0.1-pp39-pypy39_pp73-manylinux_2_28_x86_64.whl", hash = "sha256:6f2bbd72f717ce33100e6467572abaedc61f1acb87b8d546001328d7f466b778"}, + {file = "cryptography-40.0.1-pp39-pypy39_pp73-win_amd64.whl", hash = "sha256:cc3a621076d824d75ab1e1e530e66e7e8564e357dd723f2533225d40fe35c60c"}, + {file = "cryptography-40.0.1.tar.gz", hash = "sha256:2803f2f8b1e95f614419926c7e6f55d828afc614ca5ed61543877ae668cc3472"}, ] [package.dependencies] @@ -702,10 +689,10 @@ cffi = ">=1.12" [package.extras] docs = ["sphinx (>=5.3.0)", "sphinx-rtd-theme (>=1.1.1)"] docstest = ["pyenchant (>=1.6.11)", "sphinxcontrib-spelling (>=4.0.1)", "twine (>=1.12.0)"] -pep8test = ["black", "check-manifest", "mypy", "ruff", "types-pytz", "types-requests"] +pep8test = ["black", "check-manifest", "mypy", "ruff"] sdist = ["setuptools-rust (>=0.11.4)"] ssh = ["bcrypt (>=3.1.5)"] -test = ["hypothesis (>=1.11.4,!=3.79.2)", "iso8601", "pretend", "pytest (>=6.2.0)", "pytest-benchmark", "pytest-cov", "pytest-shard (>=0.1.2)", "pytest-subtests", "pytest-xdist", "pytz"] +test = ["iso8601", "pretend", "pytest (>=6.2.0)", "pytest-benchmark", "pytest-cov", "pytest-shard (>=0.1.2)", "pytest-subtests", "pytest-xdist"] test-randomorder = ["pytest-randomly"] tox = ["tox"] @@ -792,14 +779,14 @@ files = [ [[package]] name = "exceptiongroup" -version = "1.1.0" +version = "1.1.1" description = "Backport of PEP 654 (exception groups)" category = "dev" optional = false python-versions = ">=3.7" files = [ - {file = "exceptiongroup-1.1.0-py3-none-any.whl", hash = "sha256:327cbda3da756e2de031a3107b81ab7b3770a602c4d16ca618298c526f4bec1e"}, - {file = "exceptiongroup-1.1.0.tar.gz", hash = "sha256:bcb67d800a4497e1b404c2dd44fca47d3b7a5e5433dbab67f96c1a685cdfdf23"}, + {file = "exceptiongroup-1.1.1-py3-none-any.whl", hash = "sha256:232c37c63e4f682982c8b6459f33a8981039e5fb8756b2074364e5055c498c9e"}, + {file = "exceptiongroup-1.1.1.tar.gz", hash = "sha256:d484c3090ba2889ae2928419117447a14daf3c1231d5e30d0aae34f354f01785"}, ] [package.extras] @@ -854,19 +841,19 @@ zstandard = ["zstandard"] [[package]] name = "filelock" -version = "3.9.0" +version = "3.10.7" description = "A platform independent file lock." category = "main" optional = false python-versions = ">=3.7" files = [ - {file = "filelock-3.9.0-py3-none-any.whl", hash = "sha256:f58d535af89bb9ad5cd4df046f741f8553a418c01a7856bf0d173bbc9f6bd16d"}, - {file = "filelock-3.9.0.tar.gz", hash = "sha256:7b319f24340b51f55a2bf7a12ac0755a9b03e718311dac567a0f4f7fabd2f5de"}, + {file = "filelock-3.10.7-py3-none-any.whl", hash = "sha256:bde48477b15fde2c7e5a0713cbe72721cb5a5ad32ee0b8f419907960b9d75536"}, + {file = "filelock-3.10.7.tar.gz", hash = "sha256:892be14aa8efc01673b5ed6589dbccb95f9a8596f0507e232626155495c18105"}, ] [package.extras] -docs = ["furo (>=2022.12.7)", "sphinx (>=5.3)", "sphinx-autodoc-typehints (>=1.19.5)"] -testing = ["covdefaults (>=2.2.2)", "coverage (>=7.0.1)", "pytest (>=7.2)", "pytest-cov (>=4)", "pytest-timeout (>=2.1)"] +docs = ["furo (>=2022.12.7)", "sphinx (>=6.1.3)", "sphinx-autodoc-typehints (>=1.22,!=1.23.4)"] +testing = ["covdefaults (>=2.3)", "coverage (>=7.2.2)", "diff-cover (>=7.5)", "pytest (>=7.2.2)", "pytest-cov (>=4)", "pytest-mock (>=3.10)", "pytest-timeout (>=2.1)"] [[package]] name = "frozenlist" @@ -954,14 +941,14 @@ files = [ [[package]] name = "fsspec" -version = "2023.1.0" +version = "2023.3.0" description = "File-system specification" category = "main" optional = false -python-versions = ">=3.7" +python-versions = ">=3.8" files = [ - {file = "fsspec-2023.1.0-py3-none-any.whl", hash = "sha256:b833e2e541e9e8cde0ab549414187871243177feb3d344f9d27b25a93f5d8139"}, - {file = "fsspec-2023.1.0.tar.gz", hash = "sha256:fbae7f20ff801eb5f7d0bedf81f25c787c0dfac5e982d98fa3884a9cde2b5411"}, + {file = "fsspec-2023.3.0-py3-none-any.whl", hash = "sha256:bf57215e19dbfa4fe7edae53040cc1deef825e3b1605cca9a8d2c2fadd2328a0"}, + {file = "fsspec-2023.3.0.tar.gz", hash = "sha256:24e635549a590d74c6c18274ddd3ffab4753341753e923408b1904eaabafe04d"}, ] [package.extras] @@ -970,7 +957,6 @@ adl = ["adlfs"] arrow = ["pyarrow (>=1)"] dask = ["dask", "distributed"] dropbox = ["dropbox", "dropboxdrivefs", "requests"] -entrypoints = ["importlib-metadata"] fuse = ["fusepy"] gcs = ["gcsfs"] git = ["pygit2"] @@ -1050,72 +1036,72 @@ protobuf = ["grpcio-tools (>=1.49.1)"] [[package]] name = "grpcio" -version = "1.51.3" +version = "1.53.0" description = "HTTP/2-based RPC framework" category = "main" optional = true python-versions = ">=3.7" files = [ - {file = "grpcio-1.51.3-cp310-cp310-linux_armv7l.whl", hash = "sha256:f601aaeae18dab81930fb8d4f916b0da21e89bb4b5f7367ef793f46b4a76b7b0"}, - {file = "grpcio-1.51.3-cp310-cp310-macosx_12_0_universal2.whl", hash = "sha256:eef0450a4b5ed11feab639bf3eb1b6e23d0efa9b911bf7b06fb60e14f5f8a585"}, - {file = "grpcio-1.51.3-cp310-cp310-manylinux_2_17_aarch64.whl", hash = "sha256:82b0ad8ac825d4bb31bff9f638557c045f4a6d824d84b21e893968286f88246b"}, - {file = "grpcio-1.51.3-cp310-cp310-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:3667c06e37d6cd461afdd51cefe6537702f3d1dc5ff4cac07e88d8b4795dc16f"}, - {file = "grpcio-1.51.3-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:3709048fe0aa23dda09b3e69849a12055790171dab9e399a72ea8f9dfbf9ac80"}, - {file = "grpcio-1.51.3-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:200d69857f9910f7458b39b9bcf83ee4a180591b40146ba9e49314e3a7419313"}, - {file = "grpcio-1.51.3-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:cd9a5e68e79c5f031500e67793048a90209711e0854a9ddee8a3ce51728de4e5"}, - {file = "grpcio-1.51.3-cp310-cp310-win32.whl", hash = "sha256:6604f614016127ae10969176bbf12eb0e03d2fb3d643f050b3b69e160d144fb4"}, - {file = "grpcio-1.51.3-cp310-cp310-win_amd64.whl", hash = "sha256:e95c7ccd4c5807adef1602005513bf7c7d14e5a41daebcf9d8d30d8bf51b8f81"}, - {file = "grpcio-1.51.3-cp311-cp311-linux_armv7l.whl", hash = "sha256:5e77ee138100f0bb55cbd147840f87ee6241dbd25f09ea7cd8afe7efff323449"}, - {file = "grpcio-1.51.3-cp311-cp311-macosx_10_10_universal2.whl", hash = "sha256:68a7514b754e38e8de9075f7bb4dee919919515ec68628c43a894027e40ddec4"}, - {file = "grpcio-1.51.3-cp311-cp311-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:3c1b9f8afa62ff265d86a4747a2990ec5a96e4efce5d5888f245a682d66eca47"}, - {file = "grpcio-1.51.3-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:8de30f0b417744288cec65ec8cf84b8a57995cf7f1e84ccad2704d93f05d0aae"}, - {file = "grpcio-1.51.3-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:b69c7adc7ed60da1cb1b502853db61f453fc745f940cbcc25eb97c99965d8f41"}, - {file = "grpcio-1.51.3-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:d81528ffe0e973dc840ec73a4132fd18b8203ad129d7410155d951a0a7e4f5d0"}, - {file = "grpcio-1.51.3-cp311-cp311-win32.whl", hash = "sha256:040eb421613b57c696063abde405916dd830203c184c9000fc8c3b3b3c950325"}, - {file = "grpcio-1.51.3-cp311-cp311-win_amd64.whl", hash = "sha256:2a8e17286c4240137d933b8ca506465472248b4ce0fe46f3404459e708b65b68"}, - {file = "grpcio-1.51.3-cp37-cp37m-linux_armv7l.whl", hash = "sha256:d5cd1389669a847555df54177b911d9ff6f17345b2a6f19388707b7a9f724c88"}, - {file = "grpcio-1.51.3-cp37-cp37m-macosx_10_10_universal2.whl", hash = "sha256:be1bf35ce82cdbcac14e39d5102d8de4079a1c1a6a06b68e41fcd9ef64f9dd28"}, - {file = "grpcio-1.51.3-cp37-cp37m-manylinux_2_17_aarch64.whl", hash = "sha256:5eed34994c095e2bf7194ffac7381c6068b057ef1e69f8f08db77771350a7566"}, - {file = "grpcio-1.51.3-cp37-cp37m-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:3f9a7d88082b2a17ae7bd3c2354d13bab0453899e0851733f6afa6918373f476"}, - {file = "grpcio-1.51.3-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:36c8abbc5f837111e7bd619612eedc223c290b0903b952ce0c7b00840ea70f14"}, - {file = "grpcio-1.51.3-cp37-cp37m-musllinux_1_1_i686.whl", hash = "sha256:165b05af77e6aecb4210ae7663e25acf234ba78a7c1c157fa5f2efeb0d6ec53c"}, - {file = "grpcio-1.51.3-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:54e36c2ee304ff15f2bfbdc43d2b56c63331c52d818c364e5b5214e5bc2ad9f6"}, - {file = "grpcio-1.51.3-cp37-cp37m-win32.whl", hash = "sha256:cd0daac21d9ef5e033a5100c1d3aa055bbed28bfcf070b12d8058045c4e821b1"}, - {file = "grpcio-1.51.3-cp37-cp37m-win_amd64.whl", hash = "sha256:2fdd6333ce96435408565a9dbbd446212cd5d62e4d26f6a3c0feb1e3c35f1cc8"}, - {file = "grpcio-1.51.3-cp38-cp38-linux_armv7l.whl", hash = "sha256:54b0c29bdd9a3b1e1b61443ab152f060fc719f1c083127ab08d03fac5efd51be"}, - {file = "grpcio-1.51.3-cp38-cp38-macosx_10_10_universal2.whl", hash = "sha256:ffaaf7e93fcb437356b5a4b23bf36e8a3d0221399ff77fd057e4bc77776a24be"}, - {file = "grpcio-1.51.3-cp38-cp38-manylinux_2_17_aarch64.whl", hash = "sha256:eafbe7501a3268d05f2e450e1ddaffb950d842a8620c13ec328b501d25d2e2c3"}, - {file = "grpcio-1.51.3-cp38-cp38-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:881ecb34feabf31c6b3b9bbbddd1a5b57e69f805041e5a2c6c562a28574f71c4"}, - {file = "grpcio-1.51.3-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:e860a3222139b41d430939bbec2ec9c3f6c740938bf7a04471a9a8caaa965a2e"}, - {file = "grpcio-1.51.3-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:49ede0528e9dac7e8a9fe30b16c73b630ddd9a576bf4b675eb6b0c53ee5ca00f"}, - {file = "grpcio-1.51.3-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:6972b009638b40a448d10e1bc18e2223143b8a7aa20d7def0d78dd4af4126d12"}, - {file = "grpcio-1.51.3-cp38-cp38-win32.whl", hash = "sha256:5694448256e3cdfe5bd358f1574a3f2f51afa20cc834713c4b9788d60b7cc646"}, - {file = "grpcio-1.51.3-cp38-cp38-win_amd64.whl", hash = "sha256:3ea4341efe603b049e8c9a5f13c696ca37fcdf8a23ca35f650428ad3606381d9"}, - {file = "grpcio-1.51.3-cp39-cp39-linux_armv7l.whl", hash = "sha256:6c677581ce129f5fa228b8f418cee10bd28dd449f3a544ea73c8ba590ee49d0b"}, - {file = "grpcio-1.51.3-cp39-cp39-macosx_10_10_universal2.whl", hash = "sha256:30e09b5e0531685e176f49679b6a3b190762cc225f4565e55a899f5e14b3aa62"}, - {file = "grpcio-1.51.3-cp39-cp39-manylinux_2_17_aarch64.whl", hash = "sha256:c831f31336e81243f85b6daff3e5e8a123302ce0ea1f2726ad752fd7a59f3aee"}, - {file = "grpcio-1.51.3-cp39-cp39-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:2cd2e4cefb724cab1ba2df4b7535a9980531b9ec51b4dbb5f137a1f3a3754ef0"}, - {file = "grpcio-1.51.3-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:f7a0d0bf44438869d307f85a54f25a896ad6b4b0ca12370f76892ad732928d87"}, - {file = "grpcio-1.51.3-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:c02abd55409bfb293371554adf6a4401197ec2133dd97727c01180889014ba4d"}, - {file = "grpcio-1.51.3-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:2f8ff75e61e1227ba7a3f16b2eadbcc11d0a54096d52ab75a6b88cfbe56f55d1"}, - {file = "grpcio-1.51.3-cp39-cp39-win32.whl", hash = "sha256:6c99a73a6260bdf844b2e5ddad02dcd530310f80e1fa72c300fa19c1c7496962"}, - {file = "grpcio-1.51.3-cp39-cp39-win_amd64.whl", hash = "sha256:22bdfac4f7f27acdd4da359b5e7e1973dc74bf1ed406729b07d0759fde2f064b"}, - {file = "grpcio-1.51.3.tar.gz", hash = "sha256:be7b2265b7527bb12109a7727581e274170766d5b3c9258d4e466f4872522d7a"}, + {file = "grpcio-1.53.0-cp310-cp310-linux_armv7l.whl", hash = "sha256:752d2949b40e12e6ad3ed8cc552a65b54d226504f6b1fb67cab2ccee502cc06f"}, + {file = "grpcio-1.53.0-cp310-cp310-macosx_12_0_universal2.whl", hash = "sha256:8a48fd3a7222be226bb86b7b413ad248f17f3101a524018cdc4562eeae1eb2a3"}, + {file = "grpcio-1.53.0-cp310-cp310-manylinux_2_17_aarch64.whl", hash = "sha256:f3e837d29f0e1b9d6e7b29d569e2e9b0da61889e41879832ea15569c251c303a"}, + {file = "grpcio-1.53.0-cp310-cp310-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:aef7d30242409c3aa5839b501e877e453a2c8d3759ca8230dd5a21cda029f046"}, + {file = "grpcio-1.53.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:e6f90698b5d1c5dd7b3236cd1fa959d7b80e17923f918d5be020b65f1c78b173"}, + {file = "grpcio-1.53.0-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:a96c3c7f564b263c5d7c0e49a337166c8611e89c4c919f66dba7b9a84abad137"}, + {file = "grpcio-1.53.0-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:ee81349411648d1abc94095c68cd25e3c2812e4e0367f9a9355be1e804a5135c"}, + {file = "grpcio-1.53.0-cp310-cp310-win32.whl", hash = "sha256:fdc6191587de410a184550d4143e2b24a14df495c86ca15e59508710681690ac"}, + {file = "grpcio-1.53.0-cp310-cp310-win_amd64.whl", hash = "sha256:658ffe1e39171be00490db5bd3b966f79634ac4215a1eb9a85c6cd6783bf7f6e"}, + {file = "grpcio-1.53.0-cp311-cp311-linux_armv7l.whl", hash = "sha256:1b172e6d497191940c4b8d75b53de82dc252e15b61de2951d577ec5b43316b29"}, + {file = "grpcio-1.53.0-cp311-cp311-macosx_10_10_universal2.whl", hash = "sha256:82434ba3a5935e47908bc861ce1ebc43c2edfc1001d235d6e31e5d3ed55815f7"}, + {file = "grpcio-1.53.0-cp311-cp311-manylinux_2_17_aarch64.whl", hash = "sha256:1c734a2d4843e4e14ececf5600c3c4750990ec319e1299db7e4f0d02c25c1467"}, + {file = "grpcio-1.53.0-cp311-cp311-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:b6a2ead3de3b2d53119d473aa2f224030257ef33af1e4ddabd4afee1dea5f04c"}, + {file = "grpcio-1.53.0-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:a34d6e905f071f9b945cabbcc776e2055de1fdb59cd13683d9aa0a8f265b5bf9"}, + {file = "grpcio-1.53.0-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:eaf8e3b97caaf9415227a3c6ca5aa8d800fecadd526538d2bf8f11af783f1550"}, + {file = "grpcio-1.53.0-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:da95778d37be8e4e9afca771a83424f892296f5dfb2a100eda2571a1d8bbc0dc"}, + {file = "grpcio-1.53.0-cp311-cp311-win32.whl", hash = "sha256:e4f513d63df6336fd84b74b701f17d1bb3b64e9d78a6ed5b5e8a198bbbe8bbfa"}, + {file = "grpcio-1.53.0-cp311-cp311-win_amd64.whl", hash = "sha256:ddb2511fbbb440ed9e5c9a4b9b870f2ed649b7715859fd6f2ebc585ee85c0364"}, + {file = "grpcio-1.53.0-cp37-cp37m-linux_armv7l.whl", hash = "sha256:2a912397eb8d23c177d6d64e3c8bc46b8a1c7680b090d9f13a640b104aaec77c"}, + {file = "grpcio-1.53.0-cp37-cp37m-macosx_10_10_universal2.whl", hash = "sha256:55930c56b8f5b347d6c8c609cc341949a97e176c90f5cbb01d148d778f3bbd23"}, + {file = "grpcio-1.53.0-cp37-cp37m-manylinux_2_17_aarch64.whl", hash = "sha256:6601d812105583948ab9c6e403a7e2dba6e387cc678c010e74f2d6d589d1d1b3"}, + {file = "grpcio-1.53.0-cp37-cp37m-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:c705e0c21acb0e8478a00e7e773ad0ecdb34bd0e4adc282d3d2f51ba3961aac7"}, + {file = "grpcio-1.53.0-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:ba074af9ca268ad7b05d3fc2b920b5fb3c083da94ab63637aaf67f4f71ecb755"}, + {file = "grpcio-1.53.0-cp37-cp37m-musllinux_1_1_i686.whl", hash = "sha256:14817de09317dd7d3fbc8272864288320739973ef0f4b56bf2c0032349da8cdf"}, + {file = "grpcio-1.53.0-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:c7ad9fbedb93f331c2e9054e202e95cf825b885811f1bcbbdfdc301e451442db"}, + {file = "grpcio-1.53.0-cp37-cp37m-win_amd64.whl", hash = "sha256:dad5b302a4c21c604d88a5d441973f320134e6ff6a84ecef9c1139e5ffd466f6"}, + {file = "grpcio-1.53.0-cp38-cp38-linux_armv7l.whl", hash = "sha256:fa8eaac75d3107e3f5465f2c9e3bbd13db21790c6e45b7de1756eba16b050aca"}, + {file = "grpcio-1.53.0-cp38-cp38-macosx_10_10_universal2.whl", hash = "sha256:104a2210edd3776c38448b4f76c2f16e527adafbde171fc72a8a32976c20abc7"}, + {file = "grpcio-1.53.0-cp38-cp38-manylinux_2_17_aarch64.whl", hash = "sha256:dbc1ba968639c1d23476f75c356e549e7bbf2d8d6688717dcab5290e88e8482b"}, + {file = "grpcio-1.53.0-cp38-cp38-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:95952d3fe795b06af29bb8ec7bbf3342cdd867fc17b77cc25e6733d23fa6c519"}, + {file = "grpcio-1.53.0-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:f144a790f14c51b8a8e591eb5af40507ffee45ea6b818c2482f0457fec2e1a2e"}, + {file = "grpcio-1.53.0-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:0698c094688a2dd4c7c2f2c0e3e142cac439a64d1cef6904c97f6cde38ba422f"}, + {file = "grpcio-1.53.0-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:6b6d60b0958be711bab047e9f4df5dbbc40367955f8651232bfdcdd21450b9ab"}, + {file = "grpcio-1.53.0-cp38-cp38-win32.whl", hash = "sha256:1948539ce78805d4e6256ab0e048ec793956d54787dc9d6777df71c1d19c7f81"}, + {file = "grpcio-1.53.0-cp38-cp38-win_amd64.whl", hash = "sha256:df9ba1183b3f649210788cf80c239041dddcb375d6142d8bccafcfdf549522cd"}, + {file = "grpcio-1.53.0-cp39-cp39-linux_armv7l.whl", hash = "sha256:19caa5b7282a89b799e63776ff602bb39604f7ca98db6df27e2de06756ae86c3"}, + {file = "grpcio-1.53.0-cp39-cp39-macosx_10_10_universal2.whl", hash = "sha256:b5bd026ac928c96cc23149e6ef79183125542062eb6d1ccec34c0a37e02255e7"}, + {file = "grpcio-1.53.0-cp39-cp39-manylinux_2_17_aarch64.whl", hash = "sha256:7dc8584ca6c015ad82e186e82f4c0fe977394588f66b8ecfc4ec873285314619"}, + {file = "grpcio-1.53.0-cp39-cp39-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:2eddaae8af625e45b5c8500dcca1043264d751a6872cde2eda5022df8a336959"}, + {file = "grpcio-1.53.0-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:c5fb6f3d7824696c1c9f2ad36ddb080ba5a86f2d929ef712d511b4d9972d3d27"}, + {file = "grpcio-1.53.0-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:8270d1dc2c98ab57e6dbf36fa187db8df4c036f04a398e5d5e25b4e01a766d70"}, + {file = "grpcio-1.53.0-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:976a7f24eb213e8429cab78d5e120500dfcdeb01041f1f5a77b17b9101902615"}, + {file = "grpcio-1.53.0-cp39-cp39-win32.whl", hash = "sha256:9c84a481451e7174f3a764a44150f93b041ab51045aa33d7b5b68b6979114e48"}, + {file = "grpcio-1.53.0-cp39-cp39-win_amd64.whl", hash = "sha256:6beb84f83360ff29a3654f43f251ec11b809dcb5524b698d711550243debd289"}, + {file = "grpcio-1.53.0.tar.gz", hash = "sha256:a4952899b4931a6ba12951f9a141ef3e74ff8a6ec9aa2dc602afa40f63595e33"}, ] [package.extras] -protobuf = ["grpcio-tools (>=1.51.3)"] +protobuf = ["grpcio-tools (>=1.53.0)"] [[package]] name = "identify" -version = "2.5.18" +version = "2.5.22" description = "File identification library for Python" category = "dev" optional = false python-versions = ">=3.7" files = [ - {file = "identify-2.5.18-py2.py3-none-any.whl", hash = "sha256:93aac7ecf2f6abf879b8f29a8002d3c6de7086b8c28d88e1ad15045a15ab63f9"}, - {file = "identify-2.5.18.tar.gz", hash = "sha256:89e144fa560cc4cffb6ef2ab5e9fb18ed9f9b3cb054384bab4b95c12f6c309fe"}, + {file = "identify-2.5.22-py2.py3-none-any.whl", hash = "sha256:f0faad595a4687053669c112004178149f6c326db71ee999ae4636685753ad2f"}, + {file = "identify-2.5.22.tar.gz", hash = "sha256:f7a93d6cf98e29bd07663c60728e7a4057615068d7a639d132dc883b2d54d31e"}, ] [package.extras] @@ -1135,14 +1121,14 @@ files = [ [[package]] name = "importlib-metadata" -version = "6.0.0" +version = "6.1.0" description = "Read metadata from Python packages" category = "dev" optional = false python-versions = ">=3.7" files = [ - {file = "importlib_metadata-6.0.0-py3-none-any.whl", hash = "sha256:7efb448ec9a5e313a57655d35aa54cd3e01b7e1fbcf72dce1bf06119420f5bad"}, - {file = "importlib_metadata-6.0.0.tar.gz", hash = "sha256:e354bedeb60efa6affdcc8ae121b73544a7aa74156d047311948f6d711cd378d"}, + {file = "importlib_metadata-6.1.0-py3-none-any.whl", hash = "sha256:ff80f3b5394912eb1b108fcfd444dc78b7f1f3e16b16188054bd01cb9cb86f09"}, + {file = "importlib_metadata-6.1.0.tar.gz", hash = "sha256:43ce9281e097583d758c2c708c4376371261a02c34682491a8e98352365aad20"}, ] [package.dependencies] @@ -1394,14 +1380,14 @@ files = [ [[package]] name = "moto" -version = "4.1.5" +version = "4.1.6" description = "" category = "dev" optional = false python-versions = ">=3.7" files = [ - {file = "moto-4.1.5-py2.py3-none-any.whl", hash = "sha256:363577f7a0cdf639852420f6ba5caa9aa3c90a688feae6315f8ee4bf324b8c27"}, - {file = "moto-4.1.5.tar.gz", hash = "sha256:63542b7b9f307b00fae460b42d15cf9346de3ad3b1287fba38fc68f3c05e4da4"}, + {file = "moto-4.1.6-py2.py3-none-any.whl", hash = "sha256:cfe398a1f6e317d061c47c3d2dd8c6893f3eb49154984a7cbb8bcd4ba517d67d"}, + {file = "moto-4.1.6.tar.gz", hash = "sha256:fdcc2731212ca050a28b2bc83e87628294bcbd55cb4f4c4692f972023fb1e7e6"}, ] [package.dependencies] @@ -1416,13 +1402,13 @@ werkzeug = ">=0.5,<2.2.0 || >2.2.0,<2.2.1 || >2.2.1" xmltodict = "*" [package.extras] -all = ["PyYAML (>=5.1)", "aws-xray-sdk (>=0.93,!=0.96)", "cfn-lint (>=0.40.0)", "docker (>=3.0.0)", "ecdsa (!=0.15)", "graphql-core", "jsondiff (>=1.1.2)", "openapi-spec-validator (>=0.2.8)", "pyparsing (>=3.0.7)", "python-jose[cryptography] (>=3.1.0,<4.0.0)", "setuptools", "sshpubkeys (>=3.1.0)"] +all = ["PyYAML (>=5.1)", "aws-xray-sdk (>=0.93,!=0.96)", "cfn-lint (>=0.40.0)", "docker (>=3.0.0)", "ecdsa (!=0.15)", "graphql-core", "jsondiff (>=1.1.2)", "openapi-spec-validator (>=0.2.8)", "py-partiql-parser (==0.1.0)", "pyparsing (>=3.0.7)", "python-jose[cryptography] (>=3.1.0,<4.0.0)", "setuptools", "sshpubkeys (>=3.1.0)"] apigateway = ["PyYAML (>=5.1)", "ecdsa (!=0.15)", "openapi-spec-validator (>=0.2.8)", "python-jose[cryptography] (>=3.1.0,<4.0.0)"] apigatewayv2 = ["PyYAML (>=5.1)"] appsync = ["graphql-core"] awslambda = ["docker (>=3.0.0)"] batch = ["docker (>=3.0.0)"] -cloudformation = ["PyYAML (>=5.1)", "aws-xray-sdk (>=0.93,!=0.96)", "cfn-lint (>=0.40.0)", "docker (>=3.0.0)", "ecdsa (!=0.15)", "graphql-core", "jsondiff (>=1.1.2)", "openapi-spec-validator (>=0.2.8)", "pyparsing (>=3.0.7)", "python-jose[cryptography] (>=3.1.0,<4.0.0)", "setuptools", "sshpubkeys (>=3.1.0)"] +cloudformation = ["PyYAML (>=5.1)", "aws-xray-sdk (>=0.93,!=0.96)", "cfn-lint (>=0.40.0)", "docker (>=3.0.0)", "ecdsa (!=0.15)", "graphql-core", "jsondiff (>=1.1.2)", "openapi-spec-validator (>=0.2.8)", "py-partiql-parser (==0.1.0)", "pyparsing (>=3.0.7)", "python-jose[cryptography] (>=3.1.0,<4.0.0)", "setuptools", "sshpubkeys (>=3.1.0)"] cognitoidp = ["ecdsa (!=0.15)", "python-jose[cryptography] (>=3.1.0,<4.0.0)"] ds = ["sshpubkeys (>=3.1.0)"] dynamodb = ["docker (>=3.0.0)"] @@ -1434,8 +1420,8 @@ eks = ["sshpubkeys (>=3.1.0)"] glue = ["pyparsing (>=3.0.7)"] iotdata = ["jsondiff (>=1.1.2)"] route53resolver = ["sshpubkeys (>=3.1.0)"] -s3 = ["PyYAML (>=5.1)"] -server = ["PyYAML (>=5.1)", "aws-xray-sdk (>=0.93,!=0.96)", "cfn-lint (>=0.40.0)", "docker (>=3.0.0)", "ecdsa (!=0.15)", "flask (!=2.2.0,!=2.2.1)", "flask-cors", "graphql-core", "jsondiff (>=1.1.2)", "openapi-spec-validator (>=0.2.8)", "pyparsing (>=3.0.7)", "python-jose[cryptography] (>=3.1.0,<4.0.0)", "setuptools", "sshpubkeys (>=3.1.0)"] +s3 = ["PyYAML (>=5.1)", "py-partiql-parser (==0.1.0)"] +server = ["PyYAML (>=5.1)", "aws-xray-sdk (>=0.93,!=0.96)", "cfn-lint (>=0.40.0)", "docker (>=3.0.0)", "ecdsa (!=0.15)", "flask (!=2.2.0,!=2.2.1)", "flask-cors", "graphql-core", "jsondiff (>=1.1.2)", "openapi-spec-validator (>=0.2.8)", "py-partiql-parser (==0.1.0)", "pyparsing (>=3.0.7)", "python-jose[cryptography] (>=3.1.0,<4.0.0)", "setuptools", "sshpubkeys (>=3.1.0)"] ssm = ["PyYAML (>=5.1)"] xray = ["aws-xray-sdk (>=0.93,!=0.96)", "setuptools"] @@ -1551,28 +1537,6 @@ files = [ {file = "msgpack-1.0.5.tar.gz", hash = "sha256:c075544284eadc5cddc70f4757331d99dcbc16b2bbd4849d15f8aae4cf36d31c"}, ] -[[package]] -name = "msrest" -version = "0.7.1" -description = "AutoRest swagger generator Python client runtime." -category = "main" -optional = true -python-versions = ">=3.6" -files = [ - {file = "msrest-0.7.1-py3-none-any.whl", hash = "sha256:21120a810e1233e5e6cc7fe40b474eeb4ec6f757a15d7cf86702c369f9567c32"}, - {file = "msrest-0.7.1.zip", hash = "sha256:6e7661f46f3afd88b75667b7187a92829924446c7ea1d169be8c4bb7eeb788b9"}, -] - -[package.dependencies] -azure-core = ">=1.24.0" -certifi = ">=2017.4.17" -isodate = ">=0.6.0" -requests = ">=2.16,<3.0" -requests-oauthlib = ">=0.5.0" - -[package.extras] -async = ["aiodns", "aiohttp (>=3.0)"] - [[package]] name = "multidict" version = "6.0.4" @@ -1710,23 +1674,6 @@ files = [ {file = "numpy-1.24.2.tar.gz", hash = "sha256:003a9f530e880cb2cd177cba1af7220b9aa42def9c4afc2a2fc3ee6be7eb2b22"}, ] -[[package]] -name = "oauthlib" -version = "3.2.2" -description = "A generic, spec-compliant, thorough implementation of the OAuth request-signing logic" -category = "main" -optional = true -python-versions = ">=3.6" -files = [ - {file = "oauthlib-3.2.2-py3-none-any.whl", hash = "sha256:8139f29aac13e25d502680e9e19963e83f16838d48a0d71c287fe40e7067fbca"}, - {file = "oauthlib-3.2.2.tar.gz", hash = "sha256:9859c40929662bec5d64f34d01c99e093149682a3f38915dc0655d5a633dd918"}, -] - -[package.extras] -rsa = ["cryptography (>=3.0.0)"] -signals = ["blinker (>=1.4.0)"] -signedtoken = ["cryptography (>=3.0.0)", "pyjwt (>=2.0.0,<3)"] - [[package]] name = "packaging" version = "23.0" @@ -1802,19 +1749,19 @@ files = [ [[package]] name = "platformdirs" -version = "3.0.0" +version = "3.2.0" description = "A small Python package for determining appropriate platform-specific dirs, e.g. a \"user data dir\"." category = "main" optional = false python-versions = ">=3.7" files = [ - {file = "platformdirs-3.0.0-py3-none-any.whl", hash = "sha256:b1d5eb14f221506f50d6604a561f4c5786d9e80355219694a1b244bcd96f4567"}, - {file = "platformdirs-3.0.0.tar.gz", hash = "sha256:8a1228abb1ef82d788f74139988b137e78692984ec7b08eaa6c65f1723af28f9"}, + {file = "platformdirs-3.2.0-py3-none-any.whl", hash = "sha256:ebe11c0d7a805086e99506aa331612429a72ca7cd52a1f0d277dc4adc20cb10e"}, + {file = "platformdirs-3.2.0.tar.gz", hash = "sha256:d5b638ca397f25f979350ff789db335903d7ea010ab28903f57b27e1b16c2b08"}, ] [package.extras] docs = ["furo (>=2022.12.7)", "proselint (>=0.13)", "sphinx (>=6.1.3)", "sphinx-autodoc-typehints (>=1.22,!=1.23.4)"] -test = ["appdirs (==1.4.4)", "covdefaults (>=2.2.2)", "pytest (>=7.2.1)", "pytest-cov (>=4)", "pytest-mock (>=3.10)"] +test = ["appdirs (==1.4.4)", "covdefaults (>=2.3)", "pytest (>=7.2.2)", "pytest-cov (>=4)", "pytest-mock (>=3.10)"] [[package]] name = "pluggy" @@ -2220,38 +2167,38 @@ files = [ [[package]] name = "pytz" -version = "2022.7.1" +version = "2023.3" description = "World timezone definitions, modern and historical" category = "main" optional = true python-versions = "*" files = [ - {file = "pytz-2022.7.1-py2.py3-none-any.whl", hash = "sha256:78f4f37d8198e0627c5f1143240bb0206b8691d8d7ac6d78fee88b78733f8c4a"}, - {file = "pytz-2022.7.1.tar.gz", hash = "sha256:01a0681c4b9684a28304615eba55d1ab31ae00bf68ec157ec3708a8182dbbcd0"}, + {file = "pytz-2023.3-py2.py3-none-any.whl", hash = "sha256:a151b3abb88eda1d4e34a9814df37de2a80e301e68ba0fd856fb9b46bfbbbffb"}, + {file = "pytz-2023.3.tar.gz", hash = "sha256:1d8ce29db189191fb55338ee6d0387d82ab59f3d00eac103412d64e0ebd0c588"}, ] [[package]] name = "pywin32" -version = "305" +version = "306" description = "Python for Window Extensions" category = "main" optional = true python-versions = "*" files = [ - {file = "pywin32-305-cp310-cp310-win32.whl", hash = "sha256:421f6cd86e84bbb696d54563c48014b12a23ef95a14e0bdba526be756d89f116"}, - {file = "pywin32-305-cp310-cp310-win_amd64.whl", hash = "sha256:73e819c6bed89f44ff1d690498c0a811948f73777e5f97c494c152b850fad478"}, - {file = "pywin32-305-cp310-cp310-win_arm64.whl", hash = "sha256:742eb905ce2187133a29365b428e6c3b9001d79accdc30aa8969afba1d8470f4"}, - {file = "pywin32-305-cp311-cp311-win32.whl", hash = "sha256:19ca459cd2e66c0e2cc9a09d589f71d827f26d47fe4a9d09175f6aa0256b51c2"}, - {file = "pywin32-305-cp311-cp311-win_amd64.whl", hash = "sha256:326f42ab4cfff56e77e3e595aeaf6c216712bbdd91e464d167c6434b28d65990"}, - {file = "pywin32-305-cp311-cp311-win_arm64.whl", hash = "sha256:4ecd404b2c6eceaca52f8b2e3e91b2187850a1ad3f8b746d0796a98b4cea04db"}, - {file = "pywin32-305-cp36-cp36m-win32.whl", hash = "sha256:48d8b1659284f3c17b68587af047d110d8c44837736b8932c034091683e05863"}, - {file = "pywin32-305-cp36-cp36m-win_amd64.whl", hash = "sha256:13362cc5aa93c2beaf489c9c9017c793722aeb56d3e5166dadd5ef82da021fe1"}, - {file = "pywin32-305-cp37-cp37m-win32.whl", hash = "sha256:a55db448124d1c1484df22fa8bbcbc45c64da5e6eae74ab095b9ea62e6d00496"}, - {file = "pywin32-305-cp37-cp37m-win_amd64.whl", hash = "sha256:109f98980bfb27e78f4df8a51a8198e10b0f347257d1e265bb1a32993d0c973d"}, - {file = "pywin32-305-cp38-cp38-win32.whl", hash = "sha256:9dd98384da775afa009bc04863426cb30596fd78c6f8e4e2e5bbf4edf8029504"}, - {file = "pywin32-305-cp38-cp38-win_amd64.whl", hash = "sha256:56d7a9c6e1a6835f521788f53b5af7912090674bb84ef5611663ee1595860fc7"}, - {file = "pywin32-305-cp39-cp39-win32.whl", hash = "sha256:9d968c677ac4d5cbdaa62fd3014ab241718e619d8e36ef8e11fb930515a1e918"}, - {file = "pywin32-305-cp39-cp39-win_amd64.whl", hash = "sha256:50768c6b7c3f0b38b7fb14dd4104da93ebced5f1a50dc0e834594bff6fbe1271"}, + {file = "pywin32-306-cp310-cp310-win32.whl", hash = "sha256:06d3420a5155ba65f0b72f2699b5bacf3109f36acbe8923765c22938a69dfc8d"}, + {file = "pywin32-306-cp310-cp310-win_amd64.whl", hash = "sha256:84f4471dbca1887ea3803d8848a1616429ac94a4a8d05f4bc9c5dcfd42ca99c8"}, + {file = "pywin32-306-cp311-cp311-win32.whl", hash = "sha256:e65028133d15b64d2ed8f06dd9fbc268352478d4f9289e69c190ecd6818b6407"}, + {file = "pywin32-306-cp311-cp311-win_amd64.whl", hash = "sha256:a7639f51c184c0272e93f244eb24dafca9b1855707d94c192d4a0b4c01e1100e"}, + {file = "pywin32-306-cp311-cp311-win_arm64.whl", hash = "sha256:70dba0c913d19f942a2db25217d9a1b726c278f483a919f1abfed79c9cf64d3a"}, + {file = "pywin32-306-cp312-cp312-win32.whl", hash = "sha256:383229d515657f4e3ed1343da8be101000562bf514591ff383ae940cad65458b"}, + {file = "pywin32-306-cp312-cp312-win_amd64.whl", hash = "sha256:37257794c1ad39ee9be652da0462dc2e394c8159dfd913a8a4e8eb6fd346da0e"}, + {file = "pywin32-306-cp312-cp312-win_arm64.whl", hash = "sha256:5821ec52f6d321aa59e2db7e0a35b997de60c201943557d108af9d4ae1ec7040"}, + {file = "pywin32-306-cp37-cp37m-win32.whl", hash = "sha256:1c73ea9a0d2283d889001998059f5eaaba3b6238f767c9cf2833b13e6a685f65"}, + {file = "pywin32-306-cp37-cp37m-win_amd64.whl", hash = "sha256:72c5f621542d7bdd4fdb716227be0dd3f8565c11b280be6315b06ace35487d36"}, + {file = "pywin32-306-cp38-cp38-win32.whl", hash = "sha256:e4c092e2589b5cf0d365849e73e02c391c1349958c5ac3e9d5ccb9a28e017b3a"}, + {file = "pywin32-306-cp38-cp38-win_amd64.whl", hash = "sha256:e8ac1ae3601bee6ca9f7cb4b5363bf1c0badb935ef243c4733ff9a393b1690c0"}, + {file = "pywin32-306-cp39-cp39-win32.whl", hash = "sha256:e25fd5b485b55ac9c057f67d94bc203f3f6595078d1fb3b458c9c28b7153a802"}, + {file = "pywin32-306-cp39-cp39-win_amd64.whl", hash = "sha256:39b61c15272833b5c329a2989999dcae836b1eed650252ab1b7bfbe1d59f30f4"}, ] [[package]] @@ -2415,45 +2362,26 @@ six = "*" fixture = ["fixtures"] test = ["fixtures", "mock", "purl", "pytest", "requests-futures", "sphinx", "testrepository (>=0.0.18)", "testtools"] -[[package]] -name = "requests-oauthlib" -version = "1.3.1" -description = "OAuthlib authentication support for Requests." -category = "main" -optional = true -python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*" -files = [ - {file = "requests-oauthlib-1.3.1.tar.gz", hash = "sha256:75beac4a47881eeb94d5ea5d6ad31ef88856affe2332b9aafb52c6452ccf0d7a"}, - {file = "requests_oauthlib-1.3.1-py2.py3-none-any.whl", hash = "sha256:2577c501a2fb8d05a304c09d090d6e47c306fef15809d102b327cf8364bddab5"}, -] - -[package.dependencies] -oauthlib = ">=3.0.0" -requests = ">=2.0.0" - -[package.extras] -rsa = ["oauthlib[signedtoken] (>=3.0.0)"] - [[package]] name = "responses" -version = "0.22.0" +version = "0.23.1" description = "A utility library for mocking out the `requests` Python library." category = "dev" optional = false python-versions = ">=3.7" files = [ - {file = "responses-0.22.0-py3-none-any.whl", hash = "sha256:dcf294d204d14c436fddcc74caefdbc5764795a40ff4e6a7740ed8ddbf3294be"}, - {file = "responses-0.22.0.tar.gz", hash = "sha256:396acb2a13d25297789a5866b4881cf4e46ffd49cc26c43ab1117f40b973102e"}, + {file = "responses-0.23.1-py3-none-any.whl", hash = "sha256:8a3a5915713483bf353b6f4079ba8b2a29029d1d1090a503c70b0dc5d9d0c7bd"}, + {file = "responses-0.23.1.tar.gz", hash = "sha256:c4d9aa9fc888188f0c673eff79a8dadbe2e75b7fe879dc80a221a06e0a68138f"}, ] [package.dependencies] +pyyaml = "*" requests = ">=2.22.0,<3.0" -toml = "*" -types-toml = "*" +types-PyYAML = "*" urllib3 = ">=1.25.10" [package.extras] -tests = ["coverage (>=6.0.0)", "flake8", "mypy", "pytest (>=7.0.0)", "pytest-asyncio", "pytest-cov", "pytest-httpserver", "types-requests"] +tests = ["coverage (>=6.0.0)", "flake8", "mypy", "pytest (>=7.0.0)", "pytest-asyncio", "pytest-cov", "pytest-httpserver", "tomli", "tomli-w", "types-requests"] [[package]] name = "rich" @@ -2477,20 +2405,20 @@ jupyter = ["ipywidgets (>=7.5.1,<9)"] [[package]] name = "s3fs" -version = "2023.1.0" +version = "2023.3.0" description = "Convenient Filesystem interface over S3" category = "main" optional = true -python-versions = ">= 3.7" +python-versions = ">= 3.8" files = [ - {file = "s3fs-2023.1.0-py3-none-any.whl", hash = "sha256:a549ae518fff4388bd6fca5248575c29f521e7e39efcd2bfab476651701fd114"}, - {file = "s3fs-2023.1.0.tar.gz", hash = "sha256:8b2e28372423e93f26312208a9272e22a962ddd0a79d63f9a68693b6af5ff187"}, + {file = "s3fs-2023.3.0-py3-none-any.whl", hash = "sha256:8858cc9a03a00eaf81f8641f080d1765953181bdc0d242df682941171e9214c3"}, + {file = "s3fs-2023.3.0.tar.gz", hash = "sha256:5c154ded38e3c3d8f0eba7fec2706f29b41e0c395f19fbfef3ba8e70c685b049"}, ] [package.dependencies] aiobotocore = ">=2.4.2,<2.5.0" aiohttp = "<4.0.0a0 || >4.0.0a0,<4.0.0a1 || >4.0.0a1" -fsspec = "2023.1.0" +fsspec = "2023.3.0" [package.extras] awscli = ["aiobotocore[awscli] (>=2.4.2,<2.5.0)"] @@ -2516,14 +2444,14 @@ crt = ["botocore[crt] (>=1.20.29,<2.0a.0)"] [[package]] name = "setuptools" -version = "67.3.2" +version = "67.6.1" description = "Easily download, build, install, upgrade, and uninstall Python packages" category = "dev" optional = false python-versions = ">=3.7" files = [ - {file = "setuptools-67.3.2-py3-none-any.whl", hash = "sha256:bb6d8e508de562768f2027902929f8523932fcd1fb784e6d573d2cafac995a48"}, - {file = "setuptools-67.3.2.tar.gz", hash = "sha256:95f00380ef2ffa41d9bba85d95b27689d923c93dfbafed4aecd7cf988a25e012"}, + {file = "setuptools-67.6.1-py3-none-any.whl", hash = "sha256:e728ca814a823bf7bf60162daf9db95b93d532948c4c0bea762ce62f60189078"}, + {file = "setuptools-67.6.1.tar.gz", hash = "sha256:257de92a9d50a60b8e22abfcbb771571fde0dbf3ec234463212027a4eeecbe9a"}, ] [package.extras] @@ -2562,18 +2490,6 @@ all = ["tornado (>=4.0)", "twisted"] tornado = ["tornado (>=4.0)"] twisted = ["twisted"] -[[package]] -name = "toml" -version = "0.10.2" -description = "Python Library for Tom's Obvious, Minimal Language" -category = "dev" -optional = false -python-versions = ">=2.6, !=3.0.*, !=3.1.*, !=3.2.*" -files = [ - {file = "toml-0.10.2-py2.py3-none-any.whl", hash = "sha256:806143ae5bfb6a3c6e736a764057db0e6a0e05e338b5630894a5f779cabb4f9b"}, - {file = "toml-0.10.2.tar.gz", hash = "sha256:b3bda1d108d5dd99f4a20d24d9c348e91c4db7ab1b749200bded2f839ccbe68f"}, -] - [[package]] name = "tomli" version = "2.0.1" @@ -2587,15 +2503,15 @@ files = [ ] [[package]] -name = "types-toml" -version = "0.10.8.4" -description = "Typing stubs for toml" +name = "types-pyyaml" +version = "6.0.12.9" +description = "Typing stubs for PyYAML" category = "dev" optional = false python-versions = "*" files = [ - {file = "types-toml-0.10.8.4.tar.gz", hash = "sha256:c8748dd225b28eb80ce712e2d7d61b57599815e7b48d07ef53df51ed148fa6b1"}, - {file = "types_toml-0.10.8.4-py3-none-any.whl", hash = "sha256:306b1bb8b5bbc5f1b60387dbcc4b489e79f8490ce20e93af5f422a68b470d94b"}, + {file = "types-PyYAML-6.0.12.9.tar.gz", hash = "sha256:c51b1bd6d99ddf0aa2884a7a328810ebf70a4262c292195d3f4f9a0005f9eeb6"}, + {file = "types_PyYAML-6.0.12.9-py3-none-any.whl", hash = "sha256:5aed5aa66bd2d2e158f75dda22b059570ede988559f030cf294871d3b647e3e8"}, ] [[package]] @@ -2612,14 +2528,14 @@ files = [ [[package]] name = "urllib3" -version = "1.26.14" +version = "1.26.15" description = "HTTP library with thread-safe connection pooling, file post, and more." category = "main" optional = false python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*, !=3.5.*" files = [ - {file = "urllib3-1.26.14-py2.py3-none-any.whl", hash = "sha256:75edcdc2f7d85b137124a6c3c9fc3933cdeaa12ecb9a6a959f22797a0feca7e1"}, - {file = "urllib3-1.26.14.tar.gz", hash = "sha256:076907bf8fd355cde77728471316625a4d2f7e713c125f51953bb5b3eecf4f72"}, + {file = "urllib3-1.26.15-py2.py3-none-any.whl", hash = "sha256:aa751d169e23c7479ce47a0cb0da579e3ede798f994f5816a74e4f4500dcea42"}, + {file = "urllib3-1.26.15.tar.gz", hash = "sha256:8a388717b9476f934a21484e8c8e61875ab60644d29b9b39e11e4b9dc1c6b305"}, ] [package.extras] @@ -2629,14 +2545,14 @@ socks = ["PySocks (>=1.5.6,!=1.5.7,<2.0)"] [[package]] name = "virtualenv" -version = "20.19.0" +version = "20.21.0" description = "Virtual Python Environment builder" category = "main" optional = false python-versions = ">=3.7" files = [ - {file = "virtualenv-20.19.0-py3-none-any.whl", hash = "sha256:54eb59e7352b573aa04d53f80fc9736ed0ad5143af445a1e539aada6eb947dd1"}, - {file = "virtualenv-20.19.0.tar.gz", hash = "sha256:37a640ba82ed40b226599c522d411e4be5edb339a0c0de030c0dc7b646d61590"}, + {file = "virtualenv-20.21.0-py3-none-any.whl", hash = "sha256:31712f8f2a17bd06234fa97fdf19609e789dd4e3e4bf108c3da71d710651adbc"}, + {file = "virtualenv-20.21.0.tar.gz", hash = "sha256:f50e3e60f990a0757c9b68333c9fdaa72d7188caa417f96af9e52407831a3b68"}, ] [package.dependencies] @@ -2668,76 +2584,87 @@ watchdog = ["watchdog"] [[package]] name = "wrapt" -version = "1.14.1" +version = "1.15.0" description = "Module for decorators, wrappers and monkey patching." category = "main" optional = true python-versions = "!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,!=3.4.*,>=2.7" files = [ - {file = "wrapt-1.14.1-cp27-cp27m-macosx_10_9_x86_64.whl", hash = "sha256:1b376b3f4896e7930f1f772ac4b064ac12598d1c38d04907e696cc4d794b43d3"}, - {file = "wrapt-1.14.1-cp27-cp27m-manylinux1_i686.whl", hash = "sha256:903500616422a40a98a5a3c4ff4ed9d0066f3b4c951fa286018ecdf0750194ef"}, - {file = "wrapt-1.14.1-cp27-cp27m-manylinux1_x86_64.whl", hash = "sha256:5a9a0d155deafd9448baff28c08e150d9b24ff010e899311ddd63c45c2445e28"}, - {file = "wrapt-1.14.1-cp27-cp27m-manylinux2010_i686.whl", hash = "sha256:ddaea91abf8b0d13443f6dac52e89051a5063c7d014710dcb4d4abb2ff811a59"}, - {file = "wrapt-1.14.1-cp27-cp27m-manylinux2010_x86_64.whl", hash = "sha256:36f582d0c6bc99d5f39cd3ac2a9062e57f3cf606ade29a0a0d6b323462f4dd87"}, - {file = "wrapt-1.14.1-cp27-cp27mu-manylinux1_i686.whl", hash = "sha256:7ef58fb89674095bfc57c4069e95d7a31cfdc0939e2a579882ac7d55aadfd2a1"}, - {file = "wrapt-1.14.1-cp27-cp27mu-manylinux1_x86_64.whl", hash = "sha256:e2f83e18fe2f4c9e7db597e988f72712c0c3676d337d8b101f6758107c42425b"}, - {file = "wrapt-1.14.1-cp27-cp27mu-manylinux2010_i686.whl", hash = "sha256:ee2b1b1769f6707a8a445162ea16dddf74285c3964f605877a20e38545c3c462"}, - {file = "wrapt-1.14.1-cp27-cp27mu-manylinux2010_x86_64.whl", hash = "sha256:833b58d5d0b7e5b9832869f039203389ac7cbf01765639c7309fd50ef619e0b1"}, - {file = "wrapt-1.14.1-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:80bb5c256f1415f747011dc3604b59bc1f91c6e7150bd7db03b19170ee06b320"}, - {file = "wrapt-1.14.1-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:07f7a7d0f388028b2df1d916e94bbb40624c59b48ecc6cbc232546706fac74c2"}, - {file = "wrapt-1.14.1-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:02b41b633c6261feff8ddd8d11c711df6842aba629fdd3da10249a53211a72c4"}, - {file = "wrapt-1.14.1-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:2fe803deacd09a233e4762a1adcea5db5d31e6be577a43352936179d14d90069"}, - {file = "wrapt-1.14.1-cp310-cp310-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:257fd78c513e0fb5cdbe058c27a0624c9884e735bbd131935fd49e9fe719d310"}, - {file = "wrapt-1.14.1-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:4fcc4649dc762cddacd193e6b55bc02edca674067f5f98166d7713b193932b7f"}, - {file = "wrapt-1.14.1-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:11871514607b15cfeb87c547a49bca19fde402f32e2b1c24a632506c0a756656"}, - {file = "wrapt-1.14.1-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:8ad85f7f4e20964db4daadcab70b47ab05c7c1cf2a7c1e51087bfaa83831854c"}, - {file = "wrapt-1.14.1-cp310-cp310-win32.whl", hash = "sha256:a9a52172be0b5aae932bef82a79ec0a0ce87288c7d132946d645eba03f0ad8a8"}, - {file = "wrapt-1.14.1-cp310-cp310-win_amd64.whl", hash = "sha256:6d323e1554b3d22cfc03cd3243b5bb815a51f5249fdcbb86fda4bf62bab9e164"}, - {file = "wrapt-1.14.1-cp35-cp35m-manylinux1_i686.whl", hash = "sha256:43ca3bbbe97af00f49efb06e352eae40434ca9d915906f77def219b88e85d907"}, - {file = "wrapt-1.14.1-cp35-cp35m-manylinux1_x86_64.whl", hash = "sha256:6b1a564e6cb69922c7fe3a678b9f9a3c54e72b469875aa8018f18b4d1dd1adf3"}, - {file = "wrapt-1.14.1-cp35-cp35m-manylinux2010_i686.whl", hash = "sha256:00b6d4ea20a906c0ca56d84f93065b398ab74b927a7a3dbd470f6fc503f95dc3"}, - {file = "wrapt-1.14.1-cp35-cp35m-manylinux2010_x86_64.whl", hash = "sha256:a85d2b46be66a71bedde836d9e41859879cc54a2a04fad1191eb50c2066f6e9d"}, - {file = "wrapt-1.14.1-cp35-cp35m-win32.whl", hash = "sha256:dbcda74c67263139358f4d188ae5faae95c30929281bc6866d00573783c422b7"}, - {file = "wrapt-1.14.1-cp35-cp35m-win_amd64.whl", hash = "sha256:b21bb4c09ffabfa0e85e3a6b623e19b80e7acd709b9f91452b8297ace2a8ab00"}, - {file = "wrapt-1.14.1-cp36-cp36m-macosx_10_9_x86_64.whl", hash = "sha256:9e0fd32e0148dd5dea6af5fee42beb949098564cc23211a88d799e434255a1f4"}, - {file = "wrapt-1.14.1-cp36-cp36m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:9736af4641846491aedb3c3f56b9bc5568d92b0692303b5a305301a95dfd38b1"}, - {file = "wrapt-1.14.1-cp36-cp36m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:5b02d65b9ccf0ef6c34cba6cf5bf2aab1bb2f49c6090bafeecc9cd81ad4ea1c1"}, - {file = "wrapt-1.14.1-cp36-cp36m-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:21ac0156c4b089b330b7666db40feee30a5d52634cc4560e1905d6529a3897ff"}, - {file = "wrapt-1.14.1-cp36-cp36m-musllinux_1_1_aarch64.whl", hash = "sha256:9f3e6f9e05148ff90002b884fbc2a86bd303ae847e472f44ecc06c2cd2fcdb2d"}, - {file = "wrapt-1.14.1-cp36-cp36m-musllinux_1_1_i686.whl", hash = "sha256:6e743de5e9c3d1b7185870f480587b75b1cb604832e380d64f9504a0535912d1"}, - {file = "wrapt-1.14.1-cp36-cp36m-musllinux_1_1_x86_64.whl", hash = "sha256:d79d7d5dc8a32b7093e81e97dad755127ff77bcc899e845f41bf71747af0c569"}, - {file = "wrapt-1.14.1-cp36-cp36m-win32.whl", hash = "sha256:81b19725065dcb43df02b37e03278c011a09e49757287dca60c5aecdd5a0b8ed"}, - {file = "wrapt-1.14.1-cp36-cp36m-win_amd64.whl", hash = "sha256:b014c23646a467558be7da3d6b9fa409b2c567d2110599b7cf9a0c5992b3b471"}, - {file = "wrapt-1.14.1-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:88bd7b6bd70a5b6803c1abf6bca012f7ed963e58c68d76ee20b9d751c74a3248"}, - {file = "wrapt-1.14.1-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:b5901a312f4d14c59918c221323068fad0540e34324925c8475263841dbdfe68"}, - {file = "wrapt-1.14.1-cp37-cp37m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:d77c85fedff92cf788face9bfa3ebaa364448ebb1d765302e9af11bf449ca36d"}, - {file = "wrapt-1.14.1-cp37-cp37m-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:8d649d616e5c6a678b26d15ece345354f7c2286acd6db868e65fcc5ff7c24a77"}, - {file = "wrapt-1.14.1-cp37-cp37m-musllinux_1_1_aarch64.whl", hash = "sha256:7d2872609603cb35ca513d7404a94d6d608fc13211563571117046c9d2bcc3d7"}, - {file = "wrapt-1.14.1-cp37-cp37m-musllinux_1_1_i686.whl", hash = "sha256:ee6acae74a2b91865910eef5e7de37dc6895ad96fa23603d1d27ea69df545015"}, - {file = "wrapt-1.14.1-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:2b39d38039a1fdad98c87279b48bc5dce2c0ca0d73483b12cb72aa9609278e8a"}, - {file = "wrapt-1.14.1-cp37-cp37m-win32.whl", hash = "sha256:60db23fa423575eeb65ea430cee741acb7c26a1365d103f7b0f6ec412b893853"}, - {file = "wrapt-1.14.1-cp37-cp37m-win_amd64.whl", hash = "sha256:709fe01086a55cf79d20f741f39325018f4df051ef39fe921b1ebe780a66184c"}, - {file = "wrapt-1.14.1-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:8c0ce1e99116d5ab21355d8ebe53d9460366704ea38ae4d9f6933188f327b456"}, - {file = "wrapt-1.14.1-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:e3fb1677c720409d5f671e39bac6c9e0e422584e5f518bfd50aa4cbbea02433f"}, - {file = "wrapt-1.14.1-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:642c2e7a804fcf18c222e1060df25fc210b9c58db7c91416fb055897fc27e8cc"}, - {file = "wrapt-1.14.1-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:7b7c050ae976e286906dd3f26009e117eb000fb2cf3533398c5ad9ccc86867b1"}, - {file = "wrapt-1.14.1-cp38-cp38-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:ef3f72c9666bba2bab70d2a8b79f2c6d2c1a42a7f7e2b0ec83bb2f9e383950af"}, - {file = "wrapt-1.14.1-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:01c205616a89d09827986bc4e859bcabd64f5a0662a7fe95e0d359424e0e071b"}, - {file = "wrapt-1.14.1-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:5a0f54ce2c092aaf439813735584b9537cad479575a09892b8352fea5e988dc0"}, - {file = "wrapt-1.14.1-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:2cf71233a0ed05ccdabe209c606fe0bac7379fdcf687f39b944420d2a09fdb57"}, - {file = "wrapt-1.14.1-cp38-cp38-win32.whl", hash = "sha256:aa31fdcc33fef9eb2552cbcbfee7773d5a6792c137b359e82879c101e98584c5"}, - {file = "wrapt-1.14.1-cp38-cp38-win_amd64.whl", hash = "sha256:d1967f46ea8f2db647c786e78d8cc7e4313dbd1b0aca360592d8027b8508e24d"}, - {file = "wrapt-1.14.1-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:3232822c7d98d23895ccc443bbdf57c7412c5a65996c30442ebe6ed3df335383"}, - {file = "wrapt-1.14.1-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:988635d122aaf2bdcef9e795435662bcd65b02f4f4c1ae37fbee7401c440b3a7"}, - {file = "wrapt-1.14.1-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:9cca3c2cdadb362116235fdbd411735de4328c61425b0aa9f872fd76d02c4e86"}, - {file = "wrapt-1.14.1-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:d52a25136894c63de15a35bc0bdc5adb4b0e173b9c0d07a2be9d3ca64a332735"}, - {file = "wrapt-1.14.1-cp39-cp39-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:40e7bc81c9e2b2734ea4bc1aceb8a8f0ceaac7c5299bc5d69e37c44d9081d43b"}, - {file = "wrapt-1.14.1-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:b9b7a708dd92306328117d8c4b62e2194d00c365f18eff11a9b53c6f923b01e3"}, - {file = "wrapt-1.14.1-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:6a9a25751acb379b466ff6be78a315e2b439d4c94c1e99cb7266d40a537995d3"}, - {file = "wrapt-1.14.1-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:34aa51c45f28ba7f12accd624225e2b1e5a3a45206aa191f6f9aac931d9d56fe"}, - {file = "wrapt-1.14.1-cp39-cp39-win32.whl", hash = "sha256:dee0ce50c6a2dd9056c20db781e9c1cfd33e77d2d569f5d1d9321c641bb903d5"}, - {file = "wrapt-1.14.1-cp39-cp39-win_amd64.whl", hash = "sha256:dee60e1de1898bde3b238f18340eec6148986da0455d8ba7848d50470a7a32fb"}, - {file = "wrapt-1.14.1.tar.gz", hash = "sha256:380a85cf89e0e69b7cfbe2ea9f765f004ff419f34194018a6827ac0e3edfed4d"}, + {file = "wrapt-1.15.0-cp27-cp27m-macosx_10_9_x86_64.whl", hash = "sha256:ca1cccf838cd28d5a0883b342474c630ac48cac5df0ee6eacc9c7290f76b11c1"}, + {file = "wrapt-1.15.0-cp27-cp27m-manylinux1_i686.whl", hash = "sha256:e826aadda3cae59295b95343db8f3d965fb31059da7de01ee8d1c40a60398b29"}, + {file = "wrapt-1.15.0-cp27-cp27m-manylinux1_x86_64.whl", hash = "sha256:5fc8e02f5984a55d2c653f5fea93531e9836abbd84342c1d1e17abc4a15084c2"}, + {file = "wrapt-1.15.0-cp27-cp27m-manylinux2010_i686.whl", hash = "sha256:96e25c8603a155559231c19c0349245eeb4ac0096fe3c1d0be5c47e075bd4f46"}, + {file = "wrapt-1.15.0-cp27-cp27m-manylinux2010_x86_64.whl", hash = "sha256:40737a081d7497efea35ab9304b829b857f21558acfc7b3272f908d33b0d9d4c"}, + {file = "wrapt-1.15.0-cp27-cp27mu-manylinux1_i686.whl", hash = "sha256:f87ec75864c37c4c6cb908d282e1969e79763e0d9becdfe9fe5473b7bb1e5f09"}, + {file = "wrapt-1.15.0-cp27-cp27mu-manylinux1_x86_64.whl", hash = "sha256:1286eb30261894e4c70d124d44b7fd07825340869945c79d05bda53a40caa079"}, + {file = "wrapt-1.15.0-cp27-cp27mu-manylinux2010_i686.whl", hash = "sha256:493d389a2b63c88ad56cdc35d0fa5752daac56ca755805b1b0c530f785767d5e"}, + {file = "wrapt-1.15.0-cp27-cp27mu-manylinux2010_x86_64.whl", hash = "sha256:58d7a75d731e8c63614222bcb21dd992b4ab01a399f1f09dd82af17bbfc2368a"}, + {file = "wrapt-1.15.0-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:21f6d9a0d5b3a207cdf7acf8e58d7d13d463e639f0c7e01d82cdb671e6cb7923"}, + {file = "wrapt-1.15.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:ce42618f67741d4697684e501ef02f29e758a123aa2d669e2d964ff734ee00ee"}, + {file = "wrapt-1.15.0-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:41d07d029dd4157ae27beab04d22b8e261eddfc6ecd64ff7000b10dc8b3a5727"}, + {file = "wrapt-1.15.0-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:54accd4b8bc202966bafafd16e69da9d5640ff92389d33d28555c5fd4f25ccb7"}, + {file = "wrapt-1.15.0-cp310-cp310-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:2fbfbca668dd15b744418265a9607baa970c347eefd0db6a518aaf0cfbd153c0"}, + {file = "wrapt-1.15.0-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:76e9c727a874b4856d11a32fb0b389afc61ce8aaf281ada613713ddeadd1cfec"}, + {file = "wrapt-1.15.0-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:e20076a211cd6f9b44a6be58f7eeafa7ab5720eb796975d0c03f05b47d89eb90"}, + {file = "wrapt-1.15.0-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:a74d56552ddbde46c246b5b89199cb3fd182f9c346c784e1a93e4dc3f5ec9975"}, + {file = "wrapt-1.15.0-cp310-cp310-win32.whl", hash = "sha256:26458da5653aa5b3d8dc8b24192f574a58984c749401f98fff994d41d3f08da1"}, + {file = "wrapt-1.15.0-cp310-cp310-win_amd64.whl", hash = "sha256:75760a47c06b5974aa5e01949bf7e66d2af4d08cb8c1d6516af5e39595397f5e"}, + {file = "wrapt-1.15.0-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:ba1711cda2d30634a7e452fc79eabcadaffedf241ff206db2ee93dd2c89a60e7"}, + {file = "wrapt-1.15.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:56374914b132c702aa9aa9959c550004b8847148f95e1b824772d453ac204a72"}, + {file = "wrapt-1.15.0-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a89ce3fd220ff144bd9d54da333ec0de0399b52c9ac3d2ce34b569cf1a5748fb"}, + {file = "wrapt-1.15.0-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:3bbe623731d03b186b3d6b0d6f51865bf598587c38d6f7b0be2e27414f7f214e"}, + {file = "wrapt-1.15.0-cp311-cp311-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:3abbe948c3cbde2689370a262a8d04e32ec2dd4f27103669a45c6929bcdbfe7c"}, + {file = "wrapt-1.15.0-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:b67b819628e3b748fd3c2192c15fb951f549d0f47c0449af0764d7647302fda3"}, + {file = "wrapt-1.15.0-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:7eebcdbe3677e58dd4c0e03b4f2cfa346ed4049687d839adad68cc38bb559c92"}, + {file = "wrapt-1.15.0-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:74934ebd71950e3db69960a7da29204f89624dde411afbfb3b4858c1409b1e98"}, + {file = "wrapt-1.15.0-cp311-cp311-win32.whl", hash = "sha256:bd84395aab8e4d36263cd1b9308cd504f6cf713b7d6d3ce25ea55670baec5416"}, + {file = "wrapt-1.15.0-cp311-cp311-win_amd64.whl", hash = "sha256:a487f72a25904e2b4bbc0817ce7a8de94363bd7e79890510174da9d901c38705"}, + {file = "wrapt-1.15.0-cp35-cp35m-manylinux1_i686.whl", hash = "sha256:4ff0d20f2e670800d3ed2b220d40984162089a6e2c9646fdb09b85e6f9a8fc29"}, + {file = "wrapt-1.15.0-cp35-cp35m-manylinux1_x86_64.whl", hash = "sha256:9ed6aa0726b9b60911f4aed8ec5b8dd7bf3491476015819f56473ffaef8959bd"}, + {file = "wrapt-1.15.0-cp35-cp35m-manylinux2010_i686.whl", hash = "sha256:896689fddba4f23ef7c718279e42f8834041a21342d95e56922e1c10c0cc7afb"}, + {file = "wrapt-1.15.0-cp35-cp35m-manylinux2010_x86_64.whl", hash = "sha256:75669d77bb2c071333417617a235324a1618dba66f82a750362eccbe5b61d248"}, + {file = "wrapt-1.15.0-cp35-cp35m-win32.whl", hash = "sha256:fbec11614dba0424ca72f4e8ba3c420dba07b4a7c206c8c8e4e73f2e98f4c559"}, + {file = "wrapt-1.15.0-cp35-cp35m-win_amd64.whl", hash = "sha256:fd69666217b62fa5d7c6aa88e507493a34dec4fa20c5bd925e4bc12fce586639"}, + {file = "wrapt-1.15.0-cp36-cp36m-macosx_10_9_x86_64.whl", hash = "sha256:b0724f05c396b0a4c36a3226c31648385deb6a65d8992644c12a4963c70326ba"}, + {file = "wrapt-1.15.0-cp36-cp36m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:bbeccb1aa40ab88cd29e6c7d8585582c99548f55f9b2581dfc5ba68c59a85752"}, + {file = "wrapt-1.15.0-cp36-cp36m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:38adf7198f8f154502883242f9fe7333ab05a5b02de7d83aa2d88ea621f13364"}, + {file = "wrapt-1.15.0-cp36-cp36m-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:578383d740457fa790fdf85e6d346fda1416a40549fe8db08e5e9bd281c6a475"}, + {file = "wrapt-1.15.0-cp36-cp36m-musllinux_1_1_aarch64.whl", hash = "sha256:a4cbb9ff5795cd66f0066bdf5947f170f5d63a9274f99bdbca02fd973adcf2a8"}, + {file = "wrapt-1.15.0-cp36-cp36m-musllinux_1_1_i686.whl", hash = "sha256:af5bd9ccb188f6a5fdda9f1f09d9f4c86cc8a539bd48a0bfdc97723970348418"}, + {file = "wrapt-1.15.0-cp36-cp36m-musllinux_1_1_x86_64.whl", hash = "sha256:b56d5519e470d3f2fe4aa7585f0632b060d532d0696c5bdfb5e8319e1d0f69a2"}, + {file = "wrapt-1.15.0-cp36-cp36m-win32.whl", hash = "sha256:77d4c1b881076c3ba173484dfa53d3582c1c8ff1f914c6461ab70c8428b796c1"}, + {file = "wrapt-1.15.0-cp36-cp36m-win_amd64.whl", hash = "sha256:077ff0d1f9d9e4ce6476c1a924a3332452c1406e59d90a2cf24aeb29eeac9420"}, + {file = "wrapt-1.15.0-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:5c5aa28df055697d7c37d2099a7bc09f559d5053c3349b1ad0c39000e611d317"}, + {file = "wrapt-1.15.0-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:3a8564f283394634a7a7054b7983e47dbf39c07712d7b177b37e03f2467a024e"}, + {file = "wrapt-1.15.0-cp37-cp37m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:780c82a41dc493b62fc5884fb1d3a3b81106642c5c5c78d6a0d4cbe96d62ba7e"}, + {file = "wrapt-1.15.0-cp37-cp37m-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:e169e957c33576f47e21864cf3fc9ff47c223a4ebca8960079b8bd36cb014fd0"}, + {file = "wrapt-1.15.0-cp37-cp37m-musllinux_1_1_aarch64.whl", hash = "sha256:b02f21c1e2074943312d03d243ac4388319f2456576b2c6023041c4d57cd7019"}, + {file = "wrapt-1.15.0-cp37-cp37m-musllinux_1_1_i686.whl", hash = "sha256:f2e69b3ed24544b0d3dbe2c5c0ba5153ce50dcebb576fdc4696d52aa22db6034"}, + {file = "wrapt-1.15.0-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:d787272ed958a05b2c86311d3a4135d3c2aeea4fc655705f074130aa57d71653"}, + {file = "wrapt-1.15.0-cp37-cp37m-win32.whl", hash = "sha256:02fce1852f755f44f95af51f69d22e45080102e9d00258053b79367d07af39c0"}, + {file = "wrapt-1.15.0-cp37-cp37m-win_amd64.whl", hash = "sha256:abd52a09d03adf9c763d706df707c343293d5d106aea53483e0ec8d9e310ad5e"}, + {file = "wrapt-1.15.0-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:cdb4f085756c96a3af04e6eca7f08b1345e94b53af8921b25c72f096e704e145"}, + {file = "wrapt-1.15.0-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:230ae493696a371f1dbffaad3dafbb742a4d27a0afd2b1aecebe52b740167e7f"}, + {file = "wrapt-1.15.0-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:63424c681923b9f3bfbc5e3205aafe790904053d42ddcc08542181a30a7a51bd"}, + {file = "wrapt-1.15.0-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:d6bcbfc99f55655c3d93feb7ef3800bd5bbe963a755687cbf1f490a71fb7794b"}, + {file = "wrapt-1.15.0-cp38-cp38-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:c99f4309f5145b93eca6e35ac1a988f0dc0a7ccf9ccdcd78d3c0adf57224e62f"}, + {file = "wrapt-1.15.0-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:b130fe77361d6771ecf5a219d8e0817d61b236b7d8b37cc045172e574ed219e6"}, + {file = "wrapt-1.15.0-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:96177eb5645b1c6985f5c11d03fc2dbda9ad24ec0f3a46dcce91445747e15094"}, + {file = "wrapt-1.15.0-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:d5fe3e099cf07d0fb5a1e23d399e5d4d1ca3e6dfcbe5c8570ccff3e9208274f7"}, + {file = "wrapt-1.15.0-cp38-cp38-win32.whl", hash = "sha256:abd8f36c99512755b8456047b7be10372fca271bf1467a1caa88db991e7c421b"}, + {file = "wrapt-1.15.0-cp38-cp38-win_amd64.whl", hash = "sha256:b06fa97478a5f478fb05e1980980a7cdf2712015493b44d0c87606c1513ed5b1"}, + {file = "wrapt-1.15.0-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:2e51de54d4fb8fb50d6ee8327f9828306a959ae394d3e01a1ba8b2f937747d86"}, + {file = "wrapt-1.15.0-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:0970ddb69bba00670e58955f8019bec4a42d1785db3faa043c33d81de2bf843c"}, + {file = "wrapt-1.15.0-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:76407ab327158c510f44ded207e2f76b657303e17cb7a572ffe2f5a8a48aa04d"}, + {file = "wrapt-1.15.0-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:cd525e0e52a5ff16653a3fc9e3dd827981917d34996600bbc34c05d048ca35cc"}, + {file = "wrapt-1.15.0-cp39-cp39-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:9d37ac69edc5614b90516807de32d08cb8e7b12260a285ee330955604ed9dd29"}, + {file = "wrapt-1.15.0-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:078e2a1a86544e644a68422f881c48b84fef6d18f8c7a957ffd3f2e0a74a0d4a"}, + {file = "wrapt-1.15.0-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:2cf56d0e237280baed46f0b5316661da892565ff58309d4d2ed7dba763d984b8"}, + {file = "wrapt-1.15.0-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:7dc0713bf81287a00516ef43137273b23ee414fe41a3c14be10dd95ed98a2df9"}, + {file = "wrapt-1.15.0-cp39-cp39-win32.whl", hash = "sha256:46ed616d5fb42f98630ed70c3529541408166c22cdfd4540b88d5f21006b0eff"}, + {file = "wrapt-1.15.0-cp39-cp39-win_amd64.whl", hash = "sha256:eef4d64c650f33347c1f9266fa5ae001440b232ad9b98f1f43dfe7a79435c0a6"}, + {file = "wrapt-1.15.0-py3-none-any.whl", hash = "sha256:64b1df0f83706b4ef4cfb4fb0e4c2669100fd7ecacfb59e091fad300d4e04640"}, + {file = "wrapt-1.15.0.tar.gz", hash = "sha256:d06730c6aed78cee4126234cf2d071e01b44b915e725a6cb439a879ec9754a3a"}, ] [[package]] @@ -2842,19 +2769,19 @@ multidict = ">=4.0" [[package]] name = "zipp" -version = "3.13.0" +version = "3.15.0" description = "Backport of pathlib-compatible object wrapper for zip files" category = "main" optional = false python-versions = ">=3.7" files = [ - {file = "zipp-3.13.0-py3-none-any.whl", hash = "sha256:e8b2a36ea17df80ffe9e2c4fda3f693c3dad6df1697d3cd3af232db680950b0b"}, - {file = "zipp-3.13.0.tar.gz", hash = "sha256:23f70e964bc11a34cef175bc90ba2914e1e4545ea1e3e2f67c079671883f9cb6"}, + {file = "zipp-3.15.0-py3-none-any.whl", hash = "sha256:48904fc76a60e542af151aded95726c1a5c34ed43ab4134b597665c86d7ad556"}, + {file = "zipp-3.15.0.tar.gz", hash = "sha256:112929ad649da941c23de50f356a2b5570c954b65150642bccdd66bf194d224b"}, ] [package.extras] docs = ["furo", "jaraco.packaging (>=9)", "jaraco.tidelift (>=1.4)", "rst.linker (>=1.9)", "sphinx (>=3.5)", "sphinx-lint"] -testing = ["flake8 (<5)", "func-timeout", "jaraco.functools", "jaraco.itertools", "more-itertools", "pytest (>=6)", "pytest-black (>=0.3.7)", "pytest-checkdocs (>=2.4)", "pytest-cov", "pytest-enabler (>=1.3)", "pytest-flake8", "pytest-mypy (>=0.9.1)"] +testing = ["big-O", "flake8 (<5)", "jaraco.functools", "jaraco.itertools", "more-itertools", "pytest (>=6)", "pytest-black (>=0.3.7)", "pytest-checkdocs (>=2.4)", "pytest-cov", "pytest-enabler (>=1.3)", "pytest-flake8", "pytest-mypy (>=0.9.1)"] [[package]] name = "zstandard" @@ -2931,11 +2858,11 @@ glue = ["boto3"] hive = ["thrift"] pandas = ["pandas", "pyarrow"] pyarrow = ["pyarrow"] -ray = ["ray", "pyarrow", "pandas"] +ray = ["pandas", "pyarrow", "ray"] s3fs = ["s3fs"] snappy = ["python-snappy"] [metadata] lock-version = "2.0" python-versions = "^3.8" -content-hash = "45caab46bbbb5bdf0bf328c789ec912493933138f02271244e74a71417fba3fe" +content-hash = "92346bcef2c67fbc23288fa3fec1dc93af96518265ac57f744596d615c6fde8a" diff --git a/pyproject.toml b/pyproject.toml index b218d81796..9ff9bafc37 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -56,7 +56,7 @@ rich = ">=13.0.0,<=13.3.2" pyyaml = ">=5.4.0,<=6.0.0" pydantic = "1.10.7" -fsspec = ">=2022.8.2,<=2023.1.0" +fsspec = ">=2022.8.2,<=2023.3.0" pyparsing = "3.0.9" @@ -77,8 +77,8 @@ thrift = { version = "0.16.0", optional = true } boto3 = { version = "1.24.59", optional = true } # The versions of the fsspec implementations should run in sync with fsspec above -s3fs = { version = ">=2022.8.2,<=2023.1.0", optional = true } -adlfs = { version = ">=2022.8.2,<=2023.1.0", optional = true } +s3fs = { version = ">=2022.8.2,<=2023.3.0", optional = true } +adlfs = { version = ">=2022.8.2,<=2023.3.0", optional = true } [tool.poetry.dev-dependencies] pytest = "7.2.2" From 6f051b210932971fde25a986990059e2c34b1f9f Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Sun, 2 Apr 2023 13:26:17 +0200 Subject: [PATCH 415/642] Build: Bump rich from 13.3.2 to 13.3.3 in /python (#7266) --- poetry.lock | 12 ++++++------ pyproject.toml | 2 +- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/poetry.lock b/poetry.lock index 47cce87990..64d470151e 100644 --- a/poetry.lock +++ b/poetry.lock @@ -1,4 +1,4 @@ -# This file is automatically @generated by Poetry 1.4.1 and should not be changed by hand. +# This file is automatically @generated by Poetry and should not be changed by hand. [[package]] name = "adal" @@ -2385,14 +2385,14 @@ tests = ["coverage (>=6.0.0)", "flake8", "mypy", "pytest (>=7.0.0)", "pytest-asy [[package]] name = "rich" -version = "13.3.2" +version = "13.3.3" description = "Render rich text, tables, progress bars, syntax highlighting, markdown and more to the terminal" category = "main" optional = false python-versions = ">=3.7.0" files = [ - {file = "rich-13.3.2-py3-none-any.whl", hash = "sha256:a104f37270bf677148d8acb07d33be1569eeee87e2d1beb286a4e9113caf6f2f"}, - {file = "rich-13.3.2.tar.gz", hash = "sha256:91954fe80cfb7985727a467ca98a7618e5dd15178cc2da10f553b36a93859001"}, + {file = "rich-13.3.3-py3-none-any.whl", hash = "sha256:540c7d6d26a1178e8e8b37e9ba44573a3cd1464ff6348b99ee7061b95d1c6333"}, + {file = "rich-13.3.3.tar.gz", hash = "sha256:dc84400a9d842b3a9c5ff74addd8eb798d155f36c1c91303888e0a66850d2a15"}, ] [package.dependencies] @@ -2858,11 +2858,11 @@ glue = ["boto3"] hive = ["thrift"] pandas = ["pandas", "pyarrow"] pyarrow = ["pyarrow"] -ray = ["pandas", "pyarrow", "ray"] +ray = ["ray", "pyarrow", "pandas"] s3fs = ["s3fs"] snappy = ["python-snappy"] [metadata] lock-version = "2.0" python-versions = "^3.8" -content-hash = "92346bcef2c67fbc23288fa3fec1dc93af96518265ac57f744596d615c6fde8a" +content-hash = "ee6d76a660838ead91e0775e9649f4682cf2c8ddca9be7d71cae9eb467ed607a" diff --git a/pyproject.toml b/pyproject.toml index 9ff9bafc37..61c6403640 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -52,7 +52,7 @@ python = "^3.8" mmhash3 = "3.0.1" requests = ">=2.28.1,<=2.28.2" click = "8.1.3" -rich = ">=13.0.0,<=13.3.2" +rich = ">=13.0.0,<=13.3.3" pyyaml = ">=5.4.0,<=6.0.0" pydantic = "1.10.7" From a1c68cfef2c60ff164061b4e597e85a42b43f529 Mon Sep 17 00:00:00 2001 From: Deepyaman Datta Date: Sat, 8 Apr 2023 00:28:59 -0700 Subject: [PATCH 416/642] Python: Use Prettier to format TOML (#7248) * Python: Use Prettier to format TOML and more files * Python: Make Prettier skip the mkdocs admonoitions --- .pre-commit-config.yaml | 24 +++++++++----- dev/docker-compose-azurite.yml | 2 +- dev/docker-compose-integration.yml | 2 +- dev/docker-compose.yml | 2 +- mkdocs/docs/api.md | 18 +++++++++-- mkdocs/docs/configuration.md | 24 +++++++------- mkdocs/docs/contributing.md | 4 +-- mkdocs/docs/feature-support.md | 30 +++++++++--------- mkdocs/docs/index.md | 2 +- mkdocs/docs/verify-release.md | 6 +++- mkdocs/mkdocs.yml | 24 +++++++------- pyproject.toml | 50 ++++++++++-------------------- 12 files changed, 99 insertions(+), 89 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index c4a6fdbaad..d6642ff87c 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -36,12 +36,21 @@ repos: rev: v5.10.1 hooks: - id: isort - args: [ --settings-path=python/pyproject.toml ] + args: [--settings-path=python/pyproject.toml] - repo: https://github.com/pre-commit/mirrors-mypy rev: v0.991 hooks: - id: mypy - args: [ --install-types, --non-interactive, --config=python/pyproject.toml] + args: + [--install-types, --non-interactive, --config=python/pyproject.toml] + - repo: https://github.com/pre-commit/mirrors-prettier + rev: v2.7.1 + hooks: + - id: prettier + args: [--plugin=prettier-plugin-toml] + additional_dependencies: + - prettier@2.7.1 + - prettier-plugin-toml@0.3.1 - repo: https://github.com/hadialqattan/pycln rev: v2.1.3 hooks: @@ -51,18 +60,19 @@ repos: rev: v3.3.1 hooks: - id: pyupgrade - args: [ --py38-plus, --keep-runtime-typing ] + args: [--py38-plus, --keep-runtime-typing] - repo: https://github.com/pycqa/pylint rev: v2.16.0 hooks: - id: pylint - args: [ --rcfile=python/pylintrc ] + args: [--rcfile=python/pylintrc] - repo: https://github.com/pycqa/flake8 - rev: '6.0.0' + rev: "6.0.0" hooks: - id: flake8 - args: [ "--ignore=E501,W503,E203,B024" ] - additional_dependencies: [ flake8-bugbear==22.12.6, flake8-comprehensions==3.10.1 ] + args: ["--ignore=E501,W503,E203,B024"] + additional_dependencies: + [flake8-bugbear==22.12.6, flake8-comprehensions==3.10.1] - repo: https://github.com/executablebooks/mdformat rev: 0.7.16 hooks: diff --git a/dev/docker-compose-azurite.yml b/dev/docker-compose-azurite.yml index 60f4ff4f94..9be491d896 100644 --- a/dev/docker-compose-azurite.yml +++ b/dev/docker-compose-azurite.yml @@ -23,4 +23,4 @@ services: hostname: azurite ports: - 10000:10000 - command: [ "azurite-blob", "--loose", "--blobHost", "0.0.0.0" ] + command: ["azurite-blob", "--loose", "--blobHost", "0.0.0.0"] diff --git a/dev/docker-compose-integration.yml b/dev/docker-compose-integration.yml index 562a3c7f46..658bd698c9 100644 --- a/dev/docker-compose-integration.yml +++ b/dev/docker-compose-integration.yml @@ -66,7 +66,7 @@ services: ports: - 9001:9001 - 9000:9000 - command: [ "server", "/data", "--console-address", ":9001" ] + command: ["server", "/data", "--console-address", ":9001"] mc: depends_on: - minio diff --git a/dev/docker-compose.yml b/dev/docker-compose.yml index 423ea62f04..817f05b56c 100644 --- a/dev/docker-compose.yml +++ b/dev/docker-compose.yml @@ -27,7 +27,7 @@ services: ports: - 9001:9001 - 9000:9000 - command: [ "server", "/data", "--console-address", ":9001" ] + command: ["server", "/data", "--console-address", ":9001"] mc: depends_on: - minio diff --git a/mkdocs/docs/api.md b/mkdocs/docs/api.md index 1cb26714ad..bd6b3d3d6f 100644 --- a/mkdocs/docs/api.md +++ b/mkdocs/docs/api.md @@ -307,15 +307,21 @@ scan = table.scan( The low level API `plan_files` methods returns a set of tasks that provide the files that might contain matching rows: ```json -['s3a://warehouse/wh/nyc/taxis/data/00003-4-42464649-92dd-41ad-b83b-dea1a2fe4b58-00001.parquet'] +[ + "s3a://warehouse/wh/nyc/taxis/data/00003-4-42464649-92dd-41ad-b83b-dea1a2fe4b58-00001.parquet" +] ``` In this case it is up to the engine itself to filter the file itself. Below, `to_arrow()` and `to_duckdb()` that already do this for you. ### Apache Arrow + + !!! note "Requirements" - This requires [PyArrow to be installed](index.md) + This requires [PyArrow to be installed](index.md). + + Using PyIceberg it is filter out data from a huge table and pull it into a PyArrow table: @@ -343,9 +349,13 @@ This will only pull in the files that that might contain matching rows. ### DuckDB + + !!! note "Requirements" This requires [DuckDB to be installed](index.md). + + A table scan can also be converted into a in-memory DuckDB table: ```python @@ -373,9 +383,13 @@ print( ### Ray + + !!! note "Requirements" This requires [Ray to be installed](index.md). + + A table scan can also be converted into a Ray dataset: ```python diff --git a/mkdocs/docs/configuration.md b/mkdocs/docs/configuration.md index 1ea034ed2d..60d46fba44 100644 --- a/mkdocs/docs/configuration.md +++ b/mkdocs/docs/configuration.md @@ -47,16 +47,16 @@ Iceberg works with the concept of a FileIO which is a pluggable module for readi You can also set the FileIO explicitly: -| Key | Example | Description | -|----------------------|----------------------------------|-------------------------------------------------------------------------------------------------| -| py-io-impl | pyiceberg.io.fsspec.FsspecFileIO | Sets the FileIO explicitly to an implementation, and will fail explicitly if it can't be loaded | +| Key | Example | Description | +| ---------- | -------------------------------- | ----------------------------------------------------------------------------------------------- | +| py-io-impl | pyiceberg.io.fsspec.FsspecFileIO | Sets the FileIO explicitly to an implementation, and will fail explicitly if it can't be loaded | For the FileIO there are several configuration options available: ### S3 | Key | Example | Description | -|----------------------|---------------------|-----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| +| -------------------- | ------------------- | --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | | s3.endpoint | https://10.0.19.25/ | Configure an alternative endpoint of the S3 service for the FileIO to access. This could be used to use S3FileIO with any s3-compatible object storage service that has a different endpoint, or access a private S3 endpoint in a virtual private cloud. | | s3.access-key-id | admin | Configure the static secret access key used to access the FileIO. | | s3.secret-access-key | password | Configure the static session token used to access the FileIO. | @@ -65,15 +65,15 @@ For the FileIO there are several configuration options available: ### Azure Data lake -| Key | Example | Description | -|-------------------------|-------------------------------------------------------------------------------------------|--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| +| Key | Example | Description | +| ----------------------- | ----------------------------------------------------------------------------------------- | -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | | adlfs.connection-string | AccountName=devstoreaccount1;AccountKey=Eby8vdM02xNOcqF...;BlobEndpoint=http://localhost/ | A [connection string](https://learn.microsoft.com/en-us/azure/storage/common/storage-configure-connection-string). This could be used to use FileIO with any adlfs-compatible object storage service that has a different endpoint (like [azurite](https://github.com/azure/azurite)). | -| adlfs.account-name | devstoreaccount1 | The account that you want to connect to | -| adlfs.account-key | Eby8vdM02xNOcqF... | The key to authentication against the account. | -| adlfs.sas-token | NuHOuuzdQN7VRM%2FOpOeqBlawRCA845IY05h9eu1Yte4%3D | The shared access signature | -| adlfs.tenant-id | ad667be4-b811-11ed-afa1-0242ac120002 | The tenant-id | -| adlfs.client-id | ad667be4-b811-11ed-afa1-0242ac120002 | The client-id | -| adlfs.client-secret | oCA3R6P\*ka#oa1Sms2J74z... | The client-secret | +| adlfs.account-name | devstoreaccount1 | The account that you want to connect to | +| adlfs.account-key | Eby8vdM02xNOcqF... | The key to authentication against the account. | +| adlfs.sas-token | NuHOuuzdQN7VRM%2FOpOeqBlawRCA845IY05h9eu1Yte4%3D | The shared access signature | +| adlfs.tenant-id | ad667be4-b811-11ed-afa1-0242ac120002 | The tenant-id | +| adlfs.client-id | ad667be4-b811-11ed-afa1-0242ac120002 | The client-id | +| adlfs.client-secret | oCA3R6P\*ka#oa1Sms2J74z... | The client-secret | ## REST Catalog diff --git a/mkdocs/docs/contributing.md b/mkdocs/docs/contributing.md index a170ab1ae8..47bc4421d6 100644 --- a/mkdocs/docs/contributing.md +++ b/mkdocs/docs/contributing.md @@ -88,13 +88,13 @@ make test-adlfs To pass additional arguments to pytest, you can use `PYTEST_ARGS`. -*Run pytest in verbose mode* +_Run pytest in verbose mode_ ```sh make test PYTEST_ARGS="-v" ``` -*Run pytest with pdb enabled* +_Run pytest with pdb enabled_ ```sh make test PYTEST_ARGS="--pdb" diff --git a/mkdocs/docs/feature-support.md b/mkdocs/docs/feature-support.md index 366dfbe0c5..d562a0065c 100644 --- a/mkdocs/docs/feature-support.md +++ b/mkdocs/docs/feature-support.md @@ -21,21 +21,21 @@ The goal is that the python library will provide a functional, performant subset ## Metadata -| Operation | Java | Python | -|:-------------------------|:-----:|:------:| -| Get Schema | X | X | -| Get Snapshots | X | X | -| Plan Scan | X | X | -| Plan Scan for Snapshot | X | X | -| Update Current Snapshot | X | | -| Create Table | X | X | -| Rename Table | X | X | -| Drop Table | X | X | -| Alter Table | X | | -| Set Table Properties | X | | -| Create Namespace | X | X | -| Drop Namespace | X | X | -| Set Namespace Properties | X | X | +| Operation | Java | Python | +| :----------------------- | :--: | :----: | +| Get Schema | X | X | +| Get Snapshots | X | X | +| Plan Scan | X | X | +| Plan Scan for Snapshot | X | X | +| Update Current Snapshot | X | | +| Create Table | X | X | +| Rename Table | X | X | +| Drop Table | X | X | +| Alter Table | X | | +| Set Table Properties | X | | +| Create Namespace | X | X | +| Drop Namespace | X | X | +| Set Namespace Properties | X | X | ## Types diff --git a/mkdocs/docs/index.md b/mkdocs/docs/index.md index 650075b8d3..2f739d6afd 100644 --- a/mkdocs/docs/index.md +++ b/mkdocs/docs/index.md @@ -44,7 +44,7 @@ pip3 install -e ".[s3fs,hive]" You can mix and match optional dependencies depending on your needs: | Key | Description: | -|----------|----------------------------------------------------------------------| +| -------- | -------------------------------------------------------------------- | | hive | Support for the Hive metastore | | glue | Support for AWS Glue | | dynamodb | Support for AWS DynamoDB | diff --git a/mkdocs/docs/verify-release.md b/mkdocs/docs/verify-release.md index 962d0ba111..d21a2e42cc 100644 --- a/mkdocs/docs/verify-release.md +++ b/mkdocs/docs/verify-release.md @@ -73,9 +73,13 @@ Run RAT checks to validate license header: This section explains how to run the tests of the source distribution. + + !!! note "Clean environment" To make sure that your environment is fresh is to run the tests in a new Docker container: - `docker run -t -i -v $(pwd):/pyiceberg/ python:3.9 bash`. And change directory: `cd /pyiceberg/`. + `docker run -t -i -v $(pwd):/pyiceberg/ python:3.9 bash`. And change directory: `cd /pyiceberg/`. + + First step is to install the package: diff --git a/mkdocs/mkdocs.yml b/mkdocs/mkdocs.yml index 522a45342e..d9904da21d 100644 --- a/mkdocs/mkdocs.yml +++ b/mkdocs/mkdocs.yml @@ -18,17 +18,17 @@ site_name: PyIceberg site_url: https://py.iceberg.apache.org/ nav: - - Home: index.md - - Configuration: configuration.md - - CLI: cli.md - - API: api.md - - Contributing: - - Contributing: contributing.md - - Feature support: feature-support.md - - Releases: - - Verify a release: verify-release.md - - How to release: how-to-release.md + - Home: index.md + - Configuration: configuration.md + - CLI: cli.md + - API: api.md + - Contributing: + - Contributing: contributing.md + - Feature support: feature-support.md + - Releases: + - Verify a release: verify-release.md + - How to release: how-to-release.md theme: - name: readthedocs + name: readthedocs markdown_extensions: - - admonition + - admonition diff --git a/pyproject.toml b/pyproject.toml index 61c6403640..800bb5676b 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -14,7 +14,6 @@ # KIND, either express or implied. See the License for the # specific language governing permissions and limitations # under the License. - [tool.poetry] name = "pyiceberg" version = "0.3.0" @@ -24,28 +23,23 @@ repository = "https://github.com/apache/iceberg/" description = "Apache Iceberg is an open table format for huge analytic datasets" authors = ["Apache Software Foundation "] license = "Apache License 2.0" - classifiers = [ - "License :: OSI Approved :: Apache Software License", - "Operating System :: OS Independent", - "Programming Language :: Python :: 3.8", - "Programming Language :: Python :: 3.9", - "Programming Language :: Python :: 3.10", - "Programming Language :: Python :: 3.11" + "License :: OSI Approved :: Apache Software License", + "Operating System :: OS Independent", + "Programming Language :: Python :: 3.8", + "Programming Language :: Python :: 3.9", + "Programming Language :: Python :: 3.10", + "Programming Language :: Python :: 3.11" ] - packages = [ - { include = "pyiceberg" }, - { from = "vendor", include = "fb303" }, - { from = "vendor", include = "hive_metastore" }, - { include = "tests", format = "sdist" }, - { include = "Makefile", format = "sdist" }, - { include = "NOTICE", format = ["sdist", "wheel"] } -] - -include = [ - {path = "dev", format = "sdist"} + { include = "pyiceberg" }, + { from = "vendor", include = "fb303" }, + { from = "vendor", include = "hive_metastore" }, + { include = "tests", format = "sdist" }, + { include = "Makefile", format = "sdist" }, + { include = "NOTICE", format = ["sdist", "wheel"] } ] +include = [{ path = "dev", format = "sdist" }] [tool.poetry.dependencies] python = "^3.8" @@ -54,28 +48,17 @@ requests = ">=2.28.1,<=2.28.2" click = "8.1.3" rich = ">=13.0.0,<=13.3.3" pyyaml = ">=5.4.0,<=6.0.0" - pydantic = "1.10.7" fsspec = ">=2022.8.2,<=2023.3.0" - pyparsing = "3.0.9" - zstandard = "0.20.0" - pyarrow = { version = ">=8.0.0,<=11.0.0", optional = true } - pandas = { version = ">=1.4.4,<=1.5.3", optional = true } - duckdb = { version = ">=0.6.0,<=0.7.1", optional = true } - ray = { version = ">=2.0.0,<=2.3.1", optional = true } - python-snappy = { version = "0.6.1", optional = true } - thrift = { version = "0.16.0", optional = true } - boto3 = { version = "1.24.59", optional = true } - # The versions of the fsspec implementations should run in sync with fsspec above s3fs = { version = ">=2022.8.2,<=2023.3.0", optional = true } adlfs = { version = ">=2022.8.2,<=2023.3.0", optional = true } @@ -93,7 +76,6 @@ typing-extensions = '4.5.0' [tool.poetry.scripts] pyiceberg = "pyiceberg.cli.console:run" - [build-system] requires = ["poetry-core>=1.0.0"] build-backend = "poetry.core.masonry.api" @@ -112,9 +94,9 @@ dynamodb = ["boto3"] [tool.pytest.ini_options] markers = [ - "s3: marks a test as requiring access to s3 compliant storage (use with --aws-access-key-id, --aws-secret-access-key, and --endpoint-url args)", - "adlfs: marks a test as requiring access to adlfs compliant storage (use with --adlfs.account-name, --adlfs.account-key, and --adlfs.endpoint args)", - "integration: marks integration tests against Apache Spark" + "s3: marks a test as requiring access to s3 compliant storage (use with --aws-access-key-id, --aws-secret-access-key, and --endpoint-url args)", + "adlfs: marks a test as requiring access to adlfs compliant storage (use with --adlfs.account-name, --adlfs.account-key, and --adlfs.endpoint args)", + "integration: marks integration tests against Apache Spark" ] [tool.black] From 6ce27bdc260d7eb3cb025f6153e51bc6834f806d Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Sun, 9 Apr 2023 08:05:09 +0200 Subject: [PATCH 417/642] Build: Bump coverage from 7.2.2 to 7.2.3 in /python (#7308) --- poetry.lock | 106 ++++++++++++++++++++++++------------------------- pyproject.toml | 2 +- 2 files changed, 54 insertions(+), 54 deletions(-) diff --git a/poetry.lock b/poetry.lock index 64d470151e..17adf0fa6c 100644 --- a/poetry.lock +++ b/poetry.lock @@ -589,63 +589,63 @@ files = [ [[package]] name = "coverage" -version = "7.2.2" +version = "7.2.3" description = "Code coverage measurement for Python" category = "dev" optional = false python-versions = ">=3.7" files = [ - {file = "coverage-7.2.2-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:c90e73bdecb7b0d1cea65a08cb41e9d672ac6d7995603d6465ed4914b98b9ad7"}, - {file = "coverage-7.2.2-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:e2926b8abedf750c2ecf5035c07515770944acf02e1c46ab08f6348d24c5f94d"}, - {file = "coverage-7.2.2-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:57b77b9099f172804e695a40ebaa374f79e4fb8b92f3e167f66facbf92e8e7f5"}, - {file = "coverage-7.2.2-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:efe1c0adad110bf0ad7fb59f833880e489a61e39d699d37249bdf42f80590169"}, - {file = "coverage-7.2.2-cp310-cp310-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:2199988e0bc8325d941b209f4fd1c6fa007024b1442c5576f1a32ca2e48941e6"}, - {file = "coverage-7.2.2-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:81f63e0fb74effd5be736cfe07d710307cc0a3ccb8f4741f7f053c057615a137"}, - {file = "coverage-7.2.2-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:186e0fc9cf497365036d51d4d2ab76113fb74f729bd25da0975daab2e107fd90"}, - {file = "coverage-7.2.2-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:420f94a35e3e00a2b43ad5740f935358e24478354ce41c99407cddd283be00d2"}, - {file = "coverage-7.2.2-cp310-cp310-win32.whl", hash = "sha256:38004671848b5745bb05d4d621526fca30cee164db42a1f185615f39dc997292"}, - {file = "coverage-7.2.2-cp310-cp310-win_amd64.whl", hash = "sha256:0ce383d5f56d0729d2dd40e53fe3afeb8f2237244b0975e1427bfb2cf0d32bab"}, - {file = "coverage-7.2.2-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:3eb55b7b26389dd4f8ae911ba9bc8c027411163839dea4c8b8be54c4ee9ae10b"}, - {file = "coverage-7.2.2-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:d2b96123a453a2d7f3995ddb9f28d01fd112319a7a4d5ca99796a7ff43f02af5"}, - {file = "coverage-7.2.2-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:299bc75cb2a41e6741b5e470b8c9fb78d931edbd0cd009c58e5c84de57c06731"}, - {file = "coverage-7.2.2-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:5e1df45c23d4230e3d56d04414f9057eba501f78db60d4eeecfcb940501b08fd"}, - {file = "coverage-7.2.2-cp311-cp311-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:006ed5582e9cbc8115d2e22d6d2144a0725db542f654d9d4fda86793832f873d"}, - {file = "coverage-7.2.2-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:d683d230b5774816e7d784d7ed8444f2a40e7a450e5720d58af593cb0b94a212"}, - {file = "coverage-7.2.2-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:8efb48fa743d1c1a65ee8787b5b552681610f06c40a40b7ef94a5b517d885c54"}, - {file = "coverage-7.2.2-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:4c752d5264053a7cf2fe81c9e14f8a4fb261370a7bb344c2a011836a96fb3f57"}, - {file = "coverage-7.2.2-cp311-cp311-win32.whl", hash = "sha256:55272f33da9a5d7cccd3774aeca7a01e500a614eaea2a77091e9be000ecd401d"}, - {file = "coverage-7.2.2-cp311-cp311-win_amd64.whl", hash = "sha256:92ebc1619650409da324d001b3a36f14f63644c7f0a588e331f3b0f67491f512"}, - {file = "coverage-7.2.2-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:5afdad4cc4cc199fdf3e18088812edcf8f4c5a3c8e6cb69127513ad4cb7471a9"}, - {file = "coverage-7.2.2-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:0484d9dd1e6f481b24070c87561c8d7151bdd8b044c93ac99faafd01f695c78e"}, - {file = "coverage-7.2.2-cp37-cp37m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:d530191aa9c66ab4f190be8ac8cc7cfd8f4f3217da379606f3dd4e3d83feba69"}, - {file = "coverage-7.2.2-cp37-cp37m-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:4ac0f522c3b6109c4b764ffec71bf04ebc0523e926ca7cbe6c5ac88f84faced0"}, - {file = "coverage-7.2.2-cp37-cp37m-musllinux_1_1_aarch64.whl", hash = "sha256:ba279aae162b20444881fc3ed4e4f934c1cf8620f3dab3b531480cf602c76b7f"}, - {file = "coverage-7.2.2-cp37-cp37m-musllinux_1_1_i686.whl", hash = "sha256:53d0fd4c17175aded9c633e319360d41a1f3c6e352ba94edcb0fa5167e2bad67"}, - {file = "coverage-7.2.2-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:8c99cb7c26a3039a8a4ee3ca1efdde471e61b4837108847fb7d5be7789ed8fd9"}, - {file = "coverage-7.2.2-cp37-cp37m-win32.whl", hash = "sha256:5cc0783844c84af2522e3a99b9b761a979a3ef10fb87fc4048d1ee174e18a7d8"}, - {file = "coverage-7.2.2-cp37-cp37m-win_amd64.whl", hash = "sha256:817295f06eacdc8623dc4df7d8b49cea65925030d4e1e2a7c7218380c0072c25"}, - {file = "coverage-7.2.2-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:6146910231ece63facfc5984234ad1b06a36cecc9fd0c028e59ac7c9b18c38c6"}, - {file = "coverage-7.2.2-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:387fb46cb8e53ba7304d80aadca5dca84a2fbf6fe3faf6951d8cf2d46485d1e5"}, - {file = "coverage-7.2.2-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:046936ab032a2810dcaafd39cc4ef6dd295df1a7cbead08fe996d4765fca9fe4"}, - {file = "coverage-7.2.2-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:e627dee428a176ffb13697a2c4318d3f60b2ccdde3acdc9b3f304206ec130ccd"}, - {file = "coverage-7.2.2-cp38-cp38-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:4fa54fb483decc45f94011898727802309a109d89446a3c76387d016057d2c84"}, - {file = "coverage-7.2.2-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:3668291b50b69a0c1ef9f462c7df2c235da3c4073f49543b01e7eb1dee7dd540"}, - {file = "coverage-7.2.2-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:7c20b731211261dc9739bbe080c579a1835b0c2d9b274e5fcd903c3a7821cf88"}, - {file = "coverage-7.2.2-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:5764e1f7471cb8f64b8cda0554f3d4c4085ae4b417bfeab236799863703e5de2"}, - {file = "coverage-7.2.2-cp38-cp38-win32.whl", hash = "sha256:4f01911c010122f49a3e9bdc730eccc66f9b72bd410a3a9d3cb8448bb50d65d3"}, - {file = "coverage-7.2.2-cp38-cp38-win_amd64.whl", hash = "sha256:c448b5c9e3df5448a362208b8d4b9ed85305528313fca1b479f14f9fe0d873b8"}, - {file = "coverage-7.2.2-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:bfe7085783cda55e53510482fa7b5efc761fad1abe4d653b32710eb548ebdd2d"}, - {file = "coverage-7.2.2-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:9d22e94e6dc86de981b1b684b342bec5e331401599ce652900ec59db52940005"}, - {file = "coverage-7.2.2-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:507e4720791977934bba016101579b8c500fb21c5fa3cd4cf256477331ddd988"}, - {file = "coverage-7.2.2-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:bc4803779f0e4b06a2361f666e76f5c2e3715e8e379889d02251ec911befd149"}, - {file = "coverage-7.2.2-cp39-cp39-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:db8c2c5ace167fd25ab5dd732714c51d4633f58bac21fb0ff63b0349f62755a8"}, - {file = "coverage-7.2.2-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:4f68ee32d7c4164f1e2c8797535a6d0a3733355f5861e0f667e37df2d4b07140"}, - {file = "coverage-7.2.2-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:d52f0a114b6a58305b11a5cdecd42b2e7f1ec77eb20e2b33969d702feafdd016"}, - {file = "coverage-7.2.2-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:797aad79e7b6182cb49c08cc5d2f7aa7b2128133b0926060d0a8889ac43843be"}, - {file = "coverage-7.2.2-cp39-cp39-win32.whl", hash = "sha256:db45eec1dfccdadb179b0f9ca616872c6f700d23945ecc8f21bb105d74b1c5fc"}, - {file = "coverage-7.2.2-cp39-cp39-win_amd64.whl", hash = "sha256:8dbe2647bf58d2c5a6c5bcc685f23b5f371909a5624e9f5cd51436d6a9f6c6ef"}, - {file = "coverage-7.2.2-pp37.pp38.pp39-none-any.whl", hash = "sha256:872d6ce1f5be73f05bea4df498c140b9e7ee5418bfa2cc8204e7f9b817caa968"}, - {file = "coverage-7.2.2.tar.gz", hash = "sha256:36dd42da34fe94ed98c39887b86db9d06777b1c8f860520e21126a75507024f2"}, + {file = "coverage-7.2.3-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:e58c0d41d336569d63d1b113bd573db8363bc4146f39444125b7f8060e4e04f5"}, + {file = "coverage-7.2.3-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:344e714bd0fe921fc72d97404ebbdbf9127bac0ca1ff66d7b79efc143cf7c0c4"}, + {file = "coverage-7.2.3-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:974bc90d6f6c1e59ceb1516ab00cf1cdfbb2e555795d49fa9571d611f449bcb2"}, + {file = "coverage-7.2.3-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:0743b0035d4b0e32bc1df5de70fba3059662ace5b9a2a86a9f894cfe66569013"}, + {file = "coverage-7.2.3-cp310-cp310-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:5d0391fb4cfc171ce40437f67eb050a340fdbd0f9f49d6353a387f1b7f9dd4fa"}, + {file = "coverage-7.2.3-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:4a42e1eff0ca9a7cb7dc9ecda41dfc7cbc17cb1d02117214be0561bd1134772b"}, + {file = "coverage-7.2.3-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:be19931a8dcbe6ab464f3339966856996b12a00f9fe53f346ab3be872d03e257"}, + {file = "coverage-7.2.3-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:72fcae5bcac3333a4cf3b8f34eec99cea1187acd55af723bcbd559adfdcb5535"}, + {file = "coverage-7.2.3-cp310-cp310-win32.whl", hash = "sha256:aeae2aa38395b18106e552833f2a50c27ea0000122bde421c31d11ed7e6f9c91"}, + {file = "coverage-7.2.3-cp310-cp310-win_amd64.whl", hash = "sha256:83957d349838a636e768251c7e9979e899a569794b44c3728eaebd11d848e58e"}, + {file = "coverage-7.2.3-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:dfd393094cd82ceb9b40df4c77976015a314b267d498268a076e940fe7be6b79"}, + {file = "coverage-7.2.3-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:182eb9ac3f2b4874a1f41b78b87db20b66da6b9cdc32737fbbf4fea0c35b23fc"}, + {file = "coverage-7.2.3-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:1bb1e77a9a311346294621be905ea8a2c30d3ad371fc15bb72e98bfcfae532df"}, + {file = "coverage-7.2.3-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:ca0f34363e2634deffd390a0fef1aa99168ae9ed2af01af4a1f5865e362f8623"}, + {file = "coverage-7.2.3-cp311-cp311-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:55416d7385774285b6e2a5feca0af9652f7f444a4fa3d29d8ab052fafef9d00d"}, + {file = "coverage-7.2.3-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:06ddd9c0249a0546997fdda5a30fbcb40f23926df0a874a60a8a185bc3a87d93"}, + {file = "coverage-7.2.3-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:fff5aaa6becf2c6a1699ae6a39e2e6fb0672c2d42eca8eb0cafa91cf2e9bd312"}, + {file = "coverage-7.2.3-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:ea53151d87c52e98133eb8ac78f1206498c015849662ca8dc246255265d9c3c4"}, + {file = "coverage-7.2.3-cp311-cp311-win32.whl", hash = "sha256:8f6c930fd70d91ddee53194e93029e3ef2aabe26725aa3c2753df057e296b925"}, + {file = "coverage-7.2.3-cp311-cp311-win_amd64.whl", hash = "sha256:fa546d66639d69aa967bf08156eb8c9d0cd6f6de84be9e8c9819f52ad499c910"}, + {file = "coverage-7.2.3-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:b2317d5ed777bf5a033e83d4f1389fd4ef045763141d8f10eb09a7035cee774c"}, + {file = "coverage-7.2.3-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:be9824c1c874b73b96288c6d3de793bf7f3a597770205068c6163ea1f326e8b9"}, + {file = "coverage-7.2.3-cp37-cp37m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:2c3b2803e730dc2797a017335827e9da6da0e84c745ce0f552e66400abdfb9a1"}, + {file = "coverage-7.2.3-cp37-cp37m-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:8f69770f5ca1994cb32c38965e95f57504d3aea96b6c024624fdd5bb1aa494a1"}, + {file = "coverage-7.2.3-cp37-cp37m-musllinux_1_1_aarch64.whl", hash = "sha256:1127b16220f7bfb3f1049ed4a62d26d81970a723544e8252db0efde853268e21"}, + {file = "coverage-7.2.3-cp37-cp37m-musllinux_1_1_i686.whl", hash = "sha256:aa784405f0c640940595fa0f14064d8e84aff0b0f762fa18393e2760a2cf5841"}, + {file = "coverage-7.2.3-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:3146b8e16fa60427e03884301bf8209221f5761ac754ee6b267642a2fd354c48"}, + {file = "coverage-7.2.3-cp37-cp37m-win32.whl", hash = "sha256:1fd78b911aea9cec3b7e1e2622c8018d51c0d2bbcf8faaf53c2497eb114911c1"}, + {file = "coverage-7.2.3-cp37-cp37m-win_amd64.whl", hash = "sha256:0f3736a5d34e091b0a611964c6262fd68ca4363df56185902528f0b75dbb9c1f"}, + {file = "coverage-7.2.3-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:981b4df72c93e3bc04478153df516d385317628bd9c10be699c93c26ddcca8ab"}, + {file = "coverage-7.2.3-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:c0045f8f23a5fb30b2eb3b8a83664d8dc4fb58faddf8155d7109166adb9f2040"}, + {file = "coverage-7.2.3-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:f760073fcf8f3d6933178d67754f4f2d4e924e321f4bb0dcef0424ca0215eba1"}, + {file = "coverage-7.2.3-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:c86bd45d1659b1ae3d0ba1909326b03598affbc9ed71520e0ff8c31a993ad911"}, + {file = "coverage-7.2.3-cp38-cp38-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:172db976ae6327ed4728e2507daf8a4de73c7cc89796483e0a9198fd2e47b462"}, + {file = "coverage-7.2.3-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:d2a3a6146fe9319926e1d477842ca2a63fe99af5ae690b1f5c11e6af074a6b5c"}, + {file = "coverage-7.2.3-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:f649dd53833b495c3ebd04d6eec58479454a1784987af8afb77540d6c1767abd"}, + {file = "coverage-7.2.3-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:7c4ed4e9f3b123aa403ab424430b426a1992e6f4c8fd3cb56ea520446e04d152"}, + {file = "coverage-7.2.3-cp38-cp38-win32.whl", hash = "sha256:eb0edc3ce9760d2f21637766c3aa04822030e7451981ce569a1b3456b7053f22"}, + {file = "coverage-7.2.3-cp38-cp38-win_amd64.whl", hash = "sha256:63cdeaac4ae85a179a8d6bc09b77b564c096250d759eed343a89d91bce8b6367"}, + {file = "coverage-7.2.3-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:20d1a2a76bb4eb00e4d36b9699f9b7aba93271c9c29220ad4c6a9581a0320235"}, + {file = "coverage-7.2.3-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:4ea748802cc0de4de92ef8244dd84ffd793bd2e7be784cd8394d557a3c751e21"}, + {file = "coverage-7.2.3-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:21b154aba06df42e4b96fc915512ab39595105f6c483991287021ed95776d934"}, + {file = "coverage-7.2.3-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:fd214917cabdd6f673a29d708574e9fbdb892cb77eb426d0eae3490d95ca7859"}, + {file = "coverage-7.2.3-cp39-cp39-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:2c2e58e45fe53fab81f85474e5d4d226eeab0f27b45aa062856c89389da2f0d9"}, + {file = "coverage-7.2.3-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:87ecc7c9a1a9f912e306997ffee020297ccb5ea388421fe62a2a02747e4d5539"}, + {file = "coverage-7.2.3-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:387065e420aed3c71b61af7e82c7b6bc1c592f7e3c7a66e9f78dd178699da4fe"}, + {file = "coverage-7.2.3-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:ea3f5bc91d7d457da7d48c7a732beaf79d0c8131df3ab278e6bba6297e23c6c4"}, + {file = "coverage-7.2.3-cp39-cp39-win32.whl", hash = "sha256:ae7863a1d8db6a014b6f2ff9c1582ab1aad55a6d25bac19710a8df68921b6e30"}, + {file = "coverage-7.2.3-cp39-cp39-win_amd64.whl", hash = "sha256:3f04becd4fcda03c0160d0da9c8f0c246bc78f2f7af0feea1ec0930e7c93fa4a"}, + {file = "coverage-7.2.3-pp37.pp38.pp39-none-any.whl", hash = "sha256:965ee3e782c7892befc25575fa171b521d33798132692df428a09efacaffe8d0"}, + {file = "coverage-7.2.3.tar.gz", hash = "sha256:d298c2815fa4891edd9abe5ad6e6cb4207104c7dd9fd13aea3fdebf6f9b91259"}, ] [package.dependencies] @@ -2865,4 +2865,4 @@ snappy = ["python-snappy"] [metadata] lock-version = "2.0" python-versions = "^3.8" -content-hash = "ee6d76a660838ead91e0775e9649f4682cf2c8ddca9be7d71cae9eb467ed607a" +content-hash = "d2b0ed1bfbe9a0f978e6c70f55ca9cc0015766036020604e4eb34350e1563e71" diff --git a/pyproject.toml b/pyproject.toml index 800bb5676b..4006287ff7 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -68,7 +68,7 @@ pytest = "7.2.2" pytest-checkdocs = "2.9.0" pre-commit = "3.2.1" fastavro = "1.7.3" -coverage = { version = "^7.2.2", extras = ["toml"] } +coverage = { version = "^7.2.3", extras = ["toml"] } requests-mock = "1.10.0" moto = "^4.1.5" typing-extensions = '4.5.0' From 5b2a8517c641568970fe050f5a6a5129a19c6a35 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Sun, 9 Apr 2023 08:19:27 +0200 Subject: [PATCH 418/642] Build: Bump pytest from 7.2.2 to 7.3.0 in /python (#7306) --- poetry.lock | 13 ++++++------- pyproject.toml | 2 +- 2 files changed, 7 insertions(+), 8 deletions(-) diff --git a/poetry.lock b/poetry.lock index 17adf0fa6c..bfd75c7ebb 100644 --- a/poetry.lock +++ b/poetry.lock @@ -219,7 +219,7 @@ name = "attrs" version = "22.2.0" description = "Classes Without Boilerplate" category = "main" -optional = false +optional = true python-versions = ">=3.6" files = [ {file = "attrs-22.2.0-py3-none-any.whl", hash = "sha256:29e95c7f6778868dbd49170f98f8818f78f3dc5e0e37c0b1f474e3561b240836"}, @@ -2049,18 +2049,17 @@ files = [ [[package]] name = "pytest" -version = "7.2.2" +version = "7.3.0" description = "pytest: simple powerful testing with Python" category = "dev" optional = false python-versions = ">=3.7" files = [ - {file = "pytest-7.2.2-py3-none-any.whl", hash = "sha256:130328f552dcfac0b1cec75c12e3f005619dc5f874f0a06e8ff7263f0ee6225e"}, - {file = "pytest-7.2.2.tar.gz", hash = "sha256:c99ab0c73aceb050f68929bc93af19ab6db0558791c6a0715723abe9d0ade9d4"}, + {file = "pytest-7.3.0-py3-none-any.whl", hash = "sha256:933051fa1bfbd38a21e73c3960cebdad4cf59483ddba7696c48509727e17f201"}, + {file = "pytest-7.3.0.tar.gz", hash = "sha256:58ecc27ebf0ea643ebfdf7fb1249335da761a00c9f955bcd922349bcb68ee57d"}, ] [package.dependencies] -attrs = ">=19.2.0" colorama = {version = "*", markers = "sys_platform == \"win32\""} exceptiongroup = {version = ">=1.0.0rc8", markers = "python_version < \"3.11\""} iniconfig = "*" @@ -2069,7 +2068,7 @@ pluggy = ">=0.12,<2.0" tomli = {version = ">=1.0.0", markers = "python_version < \"3.11\""} [package.extras] -testing = ["argcomplete", "hypothesis (>=3.56)", "mock", "nose", "pygments (>=2.7.2)", "requests", "xmlschema"] +testing = ["argcomplete", "attrs (>=19.2.0)", "hypothesis (>=3.56)", "mock", "nose", "pygments (>=2.7.2)", "requests", "xmlschema"] [[package]] name = "pytest-checkdocs" @@ -2865,4 +2864,4 @@ snappy = ["python-snappy"] [metadata] lock-version = "2.0" python-versions = "^3.8" -content-hash = "d2b0ed1bfbe9a0f978e6c70f55ca9cc0015766036020604e4eb34350e1563e71" +content-hash = "a105342130313735379d3540558fab4f8145ee46fe1939051388c0442bdb602e" diff --git a/pyproject.toml b/pyproject.toml index 4006287ff7..968e6b92dd 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -64,7 +64,7 @@ s3fs = { version = ">=2022.8.2,<=2023.3.0", optional = true } adlfs = { version = ">=2022.8.2,<=2023.3.0", optional = true } [tool.poetry.dev-dependencies] -pytest = "7.2.2" +pytest = "7.3.0" pytest-checkdocs = "2.9.0" pre-commit = "3.2.1" fastavro = "1.7.3" From 127a2781ff141111fb2420da4863a12bf7cccb25 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Sun, 9 Apr 2023 08:38:46 +0200 Subject: [PATCH 419/642] Build: Bump pre-commit from 3.2.1 to 3.2.2 in /python (#7307) --- poetry.lock | 8 ++++---- pyproject.toml | 2 +- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/poetry.lock b/poetry.lock index bfd75c7ebb..4ff683ee1f 100644 --- a/poetry.lock +++ b/poetry.lock @@ -1801,14 +1801,14 @@ tests = ["pytest (>=5.4.1)", "pytest-cov (>=2.8.1)", "pytest-mypy (>=0.8.0)", "p [[package]] name = "pre-commit" -version = "3.2.1" +version = "3.2.2" description = "A framework for managing and maintaining multi-language pre-commit hooks." category = "dev" optional = false python-versions = ">=3.8" files = [ - {file = "pre_commit-3.2.1-py2.py3-none-any.whl", hash = "sha256:a06a7fcce7f420047a71213c175714216498b49ebc81fe106f7716ca265f5bb6"}, - {file = "pre_commit-3.2.1.tar.gz", hash = "sha256:b5aee7d75dbba21ee161ba641b01e7ae10c5b91967ebf7b2ab0dfae12d07e1f1"}, + {file = "pre_commit-3.2.2-py2.py3-none-any.whl", hash = "sha256:0b4210aea813fe81144e87c5a291f09ea66f199f367fa1df41b55e1d26e1e2b4"}, + {file = "pre_commit-3.2.2.tar.gz", hash = "sha256:5b808fcbda4afbccf6d6633a56663fed35b6c2bc08096fd3d47ce197ac351d9d"}, ] [package.dependencies] @@ -2864,4 +2864,4 @@ snappy = ["python-snappy"] [metadata] lock-version = "2.0" python-versions = "^3.8" -content-hash = "a105342130313735379d3540558fab4f8145ee46fe1939051388c0442bdb602e" +content-hash = "62fde4b500c7314d03d31514d316aa2f26fc3362f710437e44e3795b2771d818" diff --git a/pyproject.toml b/pyproject.toml index 968e6b92dd..8e3d2cdeea 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -66,7 +66,7 @@ adlfs = { version = ">=2022.8.2,<=2023.3.0", optional = true } [tool.poetry.dev-dependencies] pytest = "7.3.0" pytest-checkdocs = "2.9.0" -pre-commit = "3.2.1" +pre-commit = "3.2.2" fastavro = "1.7.3" coverage = { version = "^7.2.3", extras = ["toml"] } requests-mock = "1.10.0" From ba4782801e93fcabad66e5510129af2a71d0dbe0 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Sun, 16 Apr 2023 20:34:23 +0200 Subject: [PATCH 420/642] Build: Bump rich from 13.3.3 to 13.3.4 in /python (#7354) Bumps [rich](https://github.com/Textualize/rich) from 13.3.3 to 13.3.4. - [Release notes](https://github.com/Textualize/rich/releases) - [Changelog](https://github.com/Textualize/rich/blob/master/CHANGELOG.md) - [Commits](https://github.com/Textualize/rich/compare/v13.3.3...v13.3.4) --- updated-dependencies: - dependency-name: rich dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- poetry.lock | 8 ++++---- pyproject.toml | 2 +- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/poetry.lock b/poetry.lock index 4ff683ee1f..d0be2a65a2 100644 --- a/poetry.lock +++ b/poetry.lock @@ -2384,14 +2384,14 @@ tests = ["coverage (>=6.0.0)", "flake8", "mypy", "pytest (>=7.0.0)", "pytest-asy [[package]] name = "rich" -version = "13.3.3" +version = "13.3.4" description = "Render rich text, tables, progress bars, syntax highlighting, markdown and more to the terminal" category = "main" optional = false python-versions = ">=3.7.0" files = [ - {file = "rich-13.3.3-py3-none-any.whl", hash = "sha256:540c7d6d26a1178e8e8b37e9ba44573a3cd1464ff6348b99ee7061b95d1c6333"}, - {file = "rich-13.3.3.tar.gz", hash = "sha256:dc84400a9d842b3a9c5ff74addd8eb798d155f36c1c91303888e0a66850d2a15"}, + {file = "rich-13.3.4-py3-none-any.whl", hash = "sha256:22b74cae0278fd5086ff44144d3813be1cedc9115bdfabbfefd86400cb88b20a"}, + {file = "rich-13.3.4.tar.gz", hash = "sha256:b5d573e13605423ec80bdd0cd5f8541f7844a0e71a13f74cf454ccb2f490708b"}, ] [package.dependencies] @@ -2864,4 +2864,4 @@ snappy = ["python-snappy"] [metadata] lock-version = "2.0" python-versions = "^3.8" -content-hash = "62fde4b500c7314d03d31514d316aa2f26fc3362f710437e44e3795b2771d818" +content-hash = "21f010d640df620775b6acb97d9dabe9219b507e4939d636ca497c81f98750bb" diff --git a/pyproject.toml b/pyproject.toml index 8e3d2cdeea..7bb50476b0 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -46,7 +46,7 @@ python = "^3.8" mmhash3 = "3.0.1" requests = ">=2.28.1,<=2.28.2" click = "8.1.3" -rich = ">=13.0.0,<=13.3.3" +rich = ">=13.0.0,<=13.3.4" pyyaml = ">=5.4.0,<=6.0.0" pydantic = "1.10.7" fsspec = ">=2022.8.2,<=2023.3.0" From 60ce2e39c4f947b62de0634860e33877e572b104 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Sun, 16 Apr 2023 21:18:40 +0200 Subject: [PATCH 421/642] Build: Bump pytest from 7.3.0 to 7.3.1 in /python (#7355) Bumps [pytest](https://github.com/pytest-dev/pytest) from 7.3.0 to 7.3.1. - [Release notes](https://github.com/pytest-dev/pytest/releases) - [Changelog](https://github.com/pytest-dev/pytest/blob/main/CHANGELOG.rst) - [Commits](https://github.com/pytest-dev/pytest/compare/7.3.0...7.3.1) --- updated-dependencies: - dependency-name: pytest dependency-type: direct:development update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- poetry.lock | 8 ++++---- pyproject.toml | 2 +- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/poetry.lock b/poetry.lock index d0be2a65a2..a8f6db6476 100644 --- a/poetry.lock +++ b/poetry.lock @@ -2049,14 +2049,14 @@ files = [ [[package]] name = "pytest" -version = "7.3.0" +version = "7.3.1" description = "pytest: simple powerful testing with Python" category = "dev" optional = false python-versions = ">=3.7" files = [ - {file = "pytest-7.3.0-py3-none-any.whl", hash = "sha256:933051fa1bfbd38a21e73c3960cebdad4cf59483ddba7696c48509727e17f201"}, - {file = "pytest-7.3.0.tar.gz", hash = "sha256:58ecc27ebf0ea643ebfdf7fb1249335da761a00c9f955bcd922349bcb68ee57d"}, + {file = "pytest-7.3.1-py3-none-any.whl", hash = "sha256:3799fa815351fea3a5e96ac7e503a96fa51cc9942c3753cda7651b93c1cfa362"}, + {file = "pytest-7.3.1.tar.gz", hash = "sha256:434afafd78b1d78ed0addf160ad2b77a30d35d4bdf8af234fe621919d9ed15e3"}, ] [package.dependencies] @@ -2864,4 +2864,4 @@ snappy = ["python-snappy"] [metadata] lock-version = "2.0" python-versions = "^3.8" -content-hash = "21f010d640df620775b6acb97d9dabe9219b507e4939d636ca497c81f98750bb" +content-hash = "8bcbe587f8926a0616510a20563b0351b27cd3d280fe48094fa3eccde19e5033" diff --git a/pyproject.toml b/pyproject.toml index 7bb50476b0..e2a9831ba6 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -64,7 +64,7 @@ s3fs = { version = ">=2022.8.2,<=2023.3.0", optional = true } adlfs = { version = ">=2022.8.2,<=2023.3.0", optional = true } [tool.poetry.dev-dependencies] -pytest = "7.3.0" +pytest = "7.3.1" pytest-checkdocs = "2.9.0" pre-commit = "3.2.2" fastavro = "1.7.3" From 85afe6e63c86e13090e5ab929f86d4d3928601b8 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Sun, 16 Apr 2023 21:30:24 +0200 Subject: [PATCH 422/642] Build: Bump moto from 4.1.6 to 4.1.7 in /python (#7356) Bumps [moto](https://github.com/getmoto/moto) from 4.1.6 to 4.1.7. - [Release notes](https://github.com/getmoto/moto/releases) - [Changelog](https://github.com/getmoto/moto/blob/master/CHANGELOG.md) - [Commits](https://github.com/getmoto/moto/compare/4.1.6...4.1.7) --- updated-dependencies: - dependency-name: moto dependency-type: direct:development update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- poetry.lock | 8 ++++---- pyproject.toml | 2 +- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/poetry.lock b/poetry.lock index a8f6db6476..0195b1e67d 100644 --- a/poetry.lock +++ b/poetry.lock @@ -1380,14 +1380,14 @@ files = [ [[package]] name = "moto" -version = "4.1.6" +version = "4.1.7" description = "" category = "dev" optional = false python-versions = ">=3.7" files = [ - {file = "moto-4.1.6-py2.py3-none-any.whl", hash = "sha256:cfe398a1f6e317d061c47c3d2dd8c6893f3eb49154984a7cbb8bcd4ba517d67d"}, - {file = "moto-4.1.6.tar.gz", hash = "sha256:fdcc2731212ca050a28b2bc83e87628294bcbd55cb4f4c4692f972023fb1e7e6"}, + {file = "moto-4.1.7-py2.py3-none-any.whl", hash = "sha256:56de986179f79920f59243bc532e03a7039d24a5ee5aec2eb3b666dcd23d6262"}, + {file = "moto-4.1.7.tar.gz", hash = "sha256:fb9a7615f744da4ea7f154ff8e79782b19781344a6356ca4c0d6217c1237d379"}, ] [package.dependencies] @@ -2864,4 +2864,4 @@ snappy = ["python-snappy"] [metadata] lock-version = "2.0" python-versions = "^3.8" -content-hash = "8bcbe587f8926a0616510a20563b0351b27cd3d280fe48094fa3eccde19e5033" +content-hash = "d5f45c6a4c6766572a6aa9277526f294a8db399d0c73dfa9c3476542279badf9" diff --git a/pyproject.toml b/pyproject.toml index e2a9831ba6..01ee7361be 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -70,7 +70,7 @@ pre-commit = "3.2.2" fastavro = "1.7.3" coverage = { version = "^7.2.3", extras = ["toml"] } requests-mock = "1.10.0" -moto = "^4.1.5" +moto = "^4.1.7" typing-extensions = '4.5.0' [tool.poetry.scripts] From 0d15082201863ebbc7fe11ad453d2b53a30c53ca Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Sun, 23 Apr 2023 20:14:06 +0200 Subject: [PATCH 423/642] Build: Bump zstandard from 0.20.0 to 0.21.0 in /python (#7411) Bumps [zstandard](https://github.com/indygreg/python-zstandard) from 0.20.0 to 0.21.0. - [Release notes](https://github.com/indygreg/python-zstandard/releases) - [Changelog](https://github.com/indygreg/python-zstandard/blob/main/docs/news.rst) - [Commits](https://github.com/indygreg/python-zstandard/compare/0.20.0...0.21.0) --- updated-dependencies: - dependency-name: zstandard dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- poetry.lock | 101 +++++++++++++++++++++++-------------------------- pyproject.toml | 2 +- 2 files changed, 48 insertions(+), 55 deletions(-) diff --git a/poetry.lock b/poetry.lock index 0195b1e67d..7fefd9fd5b 100644 --- a/poetry.lock +++ b/poetry.lock @@ -2267,6 +2267,7 @@ files = [ {file = "ray-2.3.1-cp311-cp311-manylinux2014_x86_64.whl", hash = "sha256:3436c4acbf90fd3d6a5ca14cbce6236f3c6e50bf9e939635d21a99e8cf3aca12"}, {file = "ray-2.3.1-cp36-cp36m-manylinux2014_aarch64.whl", hash = "sha256:25967bc63f9052ae96b584926938efe7f299c56ea1b17b0f2e58a5ead994620f"}, {file = "ray-2.3.1-cp36-cp36m-manylinux2014_x86_64.whl", hash = "sha256:c33154bceec74521e303cd8145e166bc07e6a6f3e90ac41478d00ae8e91e753e"}, + {file = "ray-2.3.1-cp37-cp37m-macosx_10_15_x86_64.whl", hash = "sha256:97b70cab20b1a192e9ccffcffa6d4ad0ad682965cb5d10d95e364636817915fb"}, {file = "ray-2.3.1-cp37-cp37m-manylinux2014_aarch64.whl", hash = "sha256:5c6a4224bdd98d4dff9209bdc65a372f769de2d0de32c9edb1d25f72588486d5"}, {file = "ray-2.3.1-cp37-cp37m-manylinux2014_x86_64.whl", hash = "sha256:7b5dc72a3dab1d3944ac7c8dd1026e8386d93af970dfcdc2d27925dd55d3136e"}, {file = "ray-2.3.1-cp37-cp37m-win_amd64.whl", hash = "sha256:164596dc5113e86151bb24efefd18a830d7dbbd3aa566f025e310e0f9c095fd1"}, @@ -2784,63 +2785,55 @@ testing = ["big-O", "flake8 (<5)", "jaraco.functools", "jaraco.itertools", "more [[package]] name = "zstandard" -version = "0.20.0" +version = "0.21.0" description = "Zstandard bindings for Python" category = "main" optional = false -python-versions = ">=3.6" +python-versions = ">=3.7" files = [ - {file = "zstandard-0.20.0-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:c4efa051799703dc37c072e22af1f0e4c77069a78fb37caf70e26414c738ca1d"}, - {file = "zstandard-0.20.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:f847701d77371d90783c0ce6cfdb7ebde4053882c2aaba7255c70ae3c3eb7af0"}, - {file = "zstandard-0.20.0-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:0aa4d178560d7ee32092ddfd415c2cdc6ab5ddce9554985c75f1a019a0ff4c55"}, - {file = "zstandard-0.20.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:0488f2a238b4560828b3a595f3337daac4d3725c2a1637ffe2a0d187c091da59"}, - {file = "zstandard-0.20.0-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_12_i686.manylinux2010_i686.whl", hash = "sha256:cd0aa9a043c38901925ae1bba49e1e638f2d9c3cdf1b8000868993c642deb7f2"}, - {file = "zstandard-0.20.0-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:cdd769da7add8498658d881ce0eeb4c35ea1baac62e24c5a030c50f859f29724"}, - {file = "zstandard-0.20.0-cp310-cp310-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:9aea3c7bab4276212e5ac63d28e6bd72a79ff058d57e06926dfe30a52451d943"}, - {file = "zstandard-0.20.0-cp310-cp310-win32.whl", hash = "sha256:0d213353d58ad37fb5070314b156fb983b4d680ed5f3fce76ab013484cf3cf12"}, - {file = "zstandard-0.20.0-cp310-cp310-win_amd64.whl", hash = "sha256:d08459f7f7748398a6cc65eb7f88aa7ef5731097be2ddfba544be4b558acd900"}, - {file = "zstandard-0.20.0-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:c1929afea64da48ec59eca9055d7ec7e5955801489ac40ac2a19dde19e7edad9"}, - {file = "zstandard-0.20.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:b6d718f1b7cd30adb02c2a46dde0f25a84a9de8865126e0fff7d0162332d6b92"}, - {file = "zstandard-0.20.0-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:5499d65d4a1978dccf0a9c2c0d12415e16d4995ffad7a0bc4f72cc66691cf9f2"}, - {file = "zstandard-0.20.0-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:302a31400de0280f17c4ce67a73444a7a069f228db64048e4ce555cd0c02fbc4"}, - {file = "zstandard-0.20.0-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:39ae788dcdc404c07ef7aac9b11925185ea0831b985db0bbc43f95acdbd1c2ce"}, - {file = "zstandard-0.20.0-cp311-cp311-win32.whl", hash = "sha256:e3f6887d2bdfb5752d5544860bd6b778e53ebfaf4ab6c3f9d7fd388445429d41"}, - {file = "zstandard-0.20.0-cp311-cp311-win_amd64.whl", hash = "sha256:4abf9a9e0841b844736d1ae8ead2b583d2cd212815eab15391b702bde17477a7"}, - {file = "zstandard-0.20.0-cp36-cp36m-macosx_10_9_x86_64.whl", hash = "sha256:dc47cec184e66953f635254e5381df8a22012a2308168c069230b1a95079ccd0"}, - {file = "zstandard-0.20.0-cp36-cp36m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:84c1dae0c0a21eea245b5691286fe6470dc797d5e86e0c26b57a3afd1e750b48"}, - {file = "zstandard-0.20.0-cp36-cp36m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:059316f07e39b7214cd9eed565d26ab239035d2c76835deeff381995f7a27ba8"}, - {file = "zstandard-0.20.0-cp36-cp36m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_12_i686.manylinux2010_i686.whl", hash = "sha256:9aca916724d0802d3e70dc68adeff893efece01dffe7252ee3ae0053f1f1990f"}, - {file = "zstandard-0.20.0-cp36-cp36m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:b07f391fd85e3d07514c05fb40c5573b398d0063ab2bada6eb09949ec6004772"}, - {file = "zstandard-0.20.0-cp36-cp36m-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:2adf65cfce73ce94ef4c482f6cc01f08ddf5e1ca0c1ec95f2b63840f9e4c226c"}, - {file = "zstandard-0.20.0-cp36-cp36m-win32.whl", hash = "sha256:ee2a1510e06dfc7706ea9afad363efe222818a1eafa59abc32d9bbcd8465fba7"}, - {file = "zstandard-0.20.0-cp36-cp36m-win_amd64.whl", hash = "sha256:29699746fae2760d3963a4ffb603968e77da55150ee0a3326c0569f4e35f319f"}, - {file = "zstandard-0.20.0-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:78fb35d07423f25efd0fc90d0d4710ae83cfc86443a32192b0c6cb8475ec79a5"}, - {file = "zstandard-0.20.0-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:40466adfa071f58bfa448d90f9623d6aff67c6d86de6fc60be47a26388f6c74d"}, - {file = "zstandard-0.20.0-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:ba86f931bf925e9561ccd6cb978acb163e38c425990927feb38be10c894fa937"}, - {file = "zstandard-0.20.0-cp37-cp37m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_12_i686.manylinux2010_i686.whl", hash = "sha256:b671b75ae88139b1dd022fa4aa66ba419abd66f98869af55a342cb9257a1831e"}, - {file = "zstandard-0.20.0-cp37-cp37m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:cc98c8bcaa07150d3f5d7c4bd264eaa4fdd4a4dfb8fd3f9d62565ae5c4aba227"}, - {file = "zstandard-0.20.0-cp37-cp37m-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:0b815dec62e2d5a1bf7a373388f2616f21a27047b9b999de328bca7462033708"}, - {file = "zstandard-0.20.0-cp37-cp37m-win32.whl", hash = "sha256:5a3578b182c21b8af3c49619eb4cd0b9127fa60791e621b34217d65209722002"}, - {file = "zstandard-0.20.0-cp37-cp37m-win_amd64.whl", hash = "sha256:f1ba6bbd28ad926d130f0af8016f3a2930baa013c2128cfff46ca76432f50669"}, - {file = "zstandard-0.20.0-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:b0f556c74c6f0f481b61d917e48c341cdfbb80cc3391511345aed4ce6fb52fdc"}, - {file = "zstandard-0.20.0-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:862ad0a5c94670f2bd6f64fff671bd2045af5f4ed428a3f2f69fa5e52483f86a"}, - {file = "zstandard-0.20.0-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a56036c08645aa6041d435a50103428f0682effdc67f5038de47cea5e4221d6f"}, - {file = "zstandard-0.20.0-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:4af5d1891eebef430038ea4981957d31b1eb70aca14b906660c3ac1c3e7a8612"}, - {file = "zstandard-0.20.0-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_12_i686.manylinux2010_i686.whl", hash = "sha256:489959e2d52f7f1fe8ea275fecde6911d454df465265bf3ec51b3e755e769a5e"}, - {file = "zstandard-0.20.0-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:7041efe3a93d0975d2ad16451720932e8a3d164be8521bfd0873b27ac917b77a"}, - {file = "zstandard-0.20.0-cp38-cp38-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:c28c7441638c472bfb794f424bd560a22c7afce764cd99196e8d70fbc4d14e85"}, - {file = "zstandard-0.20.0-cp38-cp38-win32.whl", hash = "sha256:ba4bb4c5a0cac802ff485fa1e57f7763df5efa0ad4ee10c2693ecc5a018d2c1a"}, - {file = "zstandard-0.20.0-cp38-cp38-win_amd64.whl", hash = "sha256:a5efe366bf0545a1a5a917787659b445ba16442ae4093f102204f42a9da1ecbc"}, - {file = "zstandard-0.20.0-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:79c3058ccbe1fa37356a73c9d3c0475ec935ab528f5b76d56fc002a5a23407c7"}, - {file = "zstandard-0.20.0-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:39cbaf8fe3fa3515d35fb790465db4dc1ff45e58e1e00cbaf8b714e85437f039"}, - {file = "zstandard-0.20.0-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:f199d58f3fd7dfa0d447bc255ff22571f2e4e5e5748bfd1c41370454723cb053"}, - {file = "zstandard-0.20.0-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:0f32a8f3a697ef87e67c0d0c0673b245babee6682b2c95e46eb30208ffb720bd"}, - {file = "zstandard-0.20.0-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_12_i686.manylinux2010_i686.whl", hash = "sha256:4a3c36284c219a4d2694e52b2582fe5d5f0ecaf94a22cf0ea959b527dbd8a2a6"}, - {file = "zstandard-0.20.0-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:2eeb9e1ecd48ac1d352608bfe0dc1ed78a397698035a1796cf72f0c9d905d219"}, - {file = "zstandard-0.20.0-cp39-cp39-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:6179808ebd1ebc42b1e2f221a23c28a22d3bc8f79209ae4a3cc114693c380bff"}, - {file = "zstandard-0.20.0-cp39-cp39-win32.whl", hash = "sha256:afbcd2ed0c1145e24dd3df8440a429688a1614b83424bc871371b176bed429f9"}, - {file = "zstandard-0.20.0-cp39-cp39-win_amd64.whl", hash = "sha256:e6b4de1ba2f3028fafa0d82222d1e91b729334c8d65fbf04290c65c09d7457e1"}, - {file = "zstandard-0.20.0.tar.gz", hash = "sha256:613daadd72c71b1488742cafb2c3b381c39d0c9bb8c6cc157aa2d5ea45cc2efc"}, + {file = "zstandard-0.21.0-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:649a67643257e3b2cff1c0a73130609679a5673bf389564bc6d4b164d822a7ce"}, + {file = "zstandard-0.21.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:144a4fe4be2e747bf9c646deab212666e39048faa4372abb6a250dab0f347a29"}, + {file = "zstandard-0.21.0-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:b72060402524ab91e075881f6b6b3f37ab715663313030d0ce983da44960a86f"}, + {file = "zstandard-0.21.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:8257752b97134477fb4e413529edaa04fc0457361d304c1319573de00ba796b1"}, + {file = "zstandard-0.21.0-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_12_i686.manylinux2010_i686.whl", hash = "sha256:c053b7c4cbf71cc26808ed67ae955836232f7638444d709bfc302d3e499364fa"}, + {file = "zstandard-0.21.0-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:2769730c13638e08b7a983b32cb67775650024632cd0476bf1ba0e6360f5ac7d"}, + {file = "zstandard-0.21.0-cp310-cp310-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:7d3bc4de588b987f3934ca79140e226785d7b5e47e31756761e48644a45a6766"}, + {file = "zstandard-0.21.0-cp310-cp310-win32.whl", hash = "sha256:67829fdb82e7393ca68e543894cd0581a79243cc4ec74a836c305c70a5943f07"}, + {file = "zstandard-0.21.0-cp310-cp310-win_amd64.whl", hash = "sha256:e6048a287f8d2d6e8bc67f6b42a766c61923641dd4022b7fd3f7439e17ba5a4d"}, + {file = "zstandard-0.21.0-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:7f2afab2c727b6a3d466faee6974a7dad0d9991241c498e7317e5ccf53dbc766"}, + {file = "zstandard-0.21.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:ff0852da2abe86326b20abae912d0367878dd0854b8931897d44cfeb18985472"}, + {file = "zstandard-0.21.0-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:d12fa383e315b62630bd407477d750ec96a0f438447d0e6e496ab67b8b451d39"}, + {file = "zstandard-0.21.0-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:f1b9703fe2e6b6811886c44052647df7c37478af1b4a1a9078585806f42e5b15"}, + {file = "zstandard-0.21.0-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:df28aa5c241f59a7ab524f8ad8bb75d9a23f7ed9d501b0fed6d40ec3064784e8"}, + {file = "zstandard-0.21.0-cp311-cp311-win32.whl", hash = "sha256:0aad6090ac164a9d237d096c8af241b8dcd015524ac6dbec1330092dba151657"}, + {file = "zstandard-0.21.0-cp311-cp311-win_amd64.whl", hash = "sha256:48b6233b5c4cacb7afb0ee6b4f91820afbb6c0e3ae0fa10abbc20000acdf4f11"}, + {file = "zstandard-0.21.0-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:e7d560ce14fd209db6adacce8908244503a009c6c39eee0c10f138996cd66d3e"}, + {file = "zstandard-0.21.0-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:1e6e131a4df2eb6f64961cea6f979cdff22d6e0d5516feb0d09492c8fd36f3bc"}, + {file = "zstandard-0.21.0-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:e1e0c62a67ff425927898cf43da2cf6b852289ebcc2054514ea9bf121bec10a5"}, + {file = "zstandard-0.21.0-cp37-cp37m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_12_i686.manylinux2010_i686.whl", hash = "sha256:1545fb9cb93e043351d0cb2ee73fa0ab32e61298968667bb924aac166278c3fc"}, + {file = "zstandard-0.21.0-cp37-cp37m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:fe6c821eb6870f81d73bf10e5deed80edcac1e63fbc40610e61f340723fd5f7c"}, + {file = "zstandard-0.21.0-cp37-cp37m-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:ddb086ea3b915e50f6604be93f4f64f168d3fc3cef3585bb9a375d5834392d4f"}, + {file = "zstandard-0.21.0-cp37-cp37m-win32.whl", hash = "sha256:57ac078ad7333c9db7a74804684099c4c77f98971c151cee18d17a12649bc25c"}, + {file = "zstandard-0.21.0-cp37-cp37m-win_amd64.whl", hash = "sha256:1243b01fb7926a5a0417120c57d4c28b25a0200284af0525fddba812d575f605"}, + {file = "zstandard-0.21.0-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:ea68b1ba4f9678ac3d3e370d96442a6332d431e5050223626bdce748692226ea"}, + {file = "zstandard-0.21.0-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:8070c1cdb4587a8aa038638acda3bd97c43c59e1e31705f2766d5576b329e97c"}, + {file = "zstandard-0.21.0-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:4af612c96599b17e4930fe58bffd6514e6c25509d120f4eae6031b7595912f85"}, + {file = "zstandard-0.21.0-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:cff891e37b167bc477f35562cda1248acc115dbafbea4f3af54ec70821090965"}, + {file = "zstandard-0.21.0-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_12_i686.manylinux2010_i686.whl", hash = "sha256:a9fec02ce2b38e8b2e86079ff0b912445495e8ab0b137f9c0505f88ad0d61296"}, + {file = "zstandard-0.21.0-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:0bdbe350691dec3078b187b8304e6a9c4d9db3eb2d50ab5b1d748533e746d099"}, + {file = "zstandard-0.21.0-cp38-cp38-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:b69cccd06a4a0a1d9fb3ec9a97600055cf03030ed7048d4bcb88c574f7895773"}, + {file = "zstandard-0.21.0-cp38-cp38-win32.whl", hash = "sha256:9980489f066a391c5572bc7dc471e903fb134e0b0001ea9b1d3eff85af0a6f1b"}, + {file = "zstandard-0.21.0-cp38-cp38-win_amd64.whl", hash = "sha256:0e1e94a9d9e35dc04bf90055e914077c80b1e0c15454cc5419e82529d3e70728"}, + {file = "zstandard-0.21.0-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:d2d61675b2a73edcef5e327e38eb62bdfc89009960f0e3991eae5cc3d54718de"}, + {file = "zstandard-0.21.0-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:25fbfef672ad798afab12e8fd204d122fca3bc8e2dcb0a2ba73bf0a0ac0f5f07"}, + {file = "zstandard-0.21.0-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:62957069a7c2626ae80023998757e27bd28d933b165c487ab6f83ad3337f773d"}, + {file = "zstandard-0.21.0-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:14e10ed461e4807471075d4b7a2af51f5234c8f1e2a0c1d37d5ca49aaaad49e8"}, + {file = "zstandard-0.21.0-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_12_i686.manylinux2010_i686.whl", hash = "sha256:9cff89a036c639a6a9299bf19e16bfb9ac7def9a7634c52c257166db09d950e7"}, + {file = "zstandard-0.21.0-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:52b2b5e3e7670bd25835e0e0730a236f2b0df87672d99d3bf4bf87248aa659fb"}, + {file = "zstandard-0.21.0-cp39-cp39-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:b1367da0dde8ae5040ef0413fb57b5baeac39d8931c70536d5f013b11d3fc3a5"}, + {file = "zstandard-0.21.0-cp39-cp39-win32.whl", hash = "sha256:db62cbe7a965e68ad2217a056107cc43d41764c66c895be05cf9c8b19578ce9c"}, + {file = "zstandard-0.21.0-cp39-cp39-win_amd64.whl", hash = "sha256:a8d200617d5c876221304b0e3fe43307adde291b4a897e7b0617a61611dfff6a"}, + {file = "zstandard-0.21.0.tar.gz", hash = "sha256:f08e3a10d01a247877e4cb61a82a319ea746c356a3786558bed2481e6c405546"}, ] [package.dependencies] @@ -2864,4 +2857,4 @@ snappy = ["python-snappy"] [metadata] lock-version = "2.0" python-versions = "^3.8" -content-hash = "d5f45c6a4c6766572a6aa9277526f294a8db399d0c73dfa9c3476542279badf9" +content-hash = "d982e711b62f55f0f01252f7f4abe2d47a025b3a539a240dc9bc3aba031405e9" diff --git a/pyproject.toml b/pyproject.toml index 01ee7361be..828d20c7e4 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -51,7 +51,7 @@ pyyaml = ">=5.4.0,<=6.0.0" pydantic = "1.10.7" fsspec = ">=2022.8.2,<=2023.3.0" pyparsing = "3.0.9" -zstandard = "0.20.0" +zstandard = "0.21.0" pyarrow = { version = ">=8.0.0,<=11.0.0", optional = true } pandas = { version = ">=1.4.4,<=1.5.3", optional = true } duckdb = { version = ">=0.6.0,<=0.7.1", optional = true } From 920be6cd4f4fd831edcf59809a0a5427e28f1dc4 Mon Sep 17 00:00:00 2001 From: Luigi Cerone Date: Mon, 24 Apr 2023 10:03:09 +0200 Subject: [PATCH 424/642] Python: Add mkdocstrings (#7108) * Add mkdocstrings * Add mkdocs theme * Add theme to mkdocs and fix pages gen * Fix missing license * Fix prettier --- .pre-commit-config.yaml | 2 + mkdocs/docs/SUMMARY.md | 31 ++++++++++ mkdocs/docs/api.md | 5 ++ .../docs/assets/images/iceberg-logo-icon.png | Bin 0 -> 17608 bytes mkdocs/docs/cli.md | 5 ++ mkdocs/docs/configuration.md | 5 ++ mkdocs/docs/contributing.md | 5 ++ mkdocs/docs/feature-support.md | 5 ++ mkdocs/docs/index.md | 5 ++ mkdocs/gen_doc_stubs.py | 55 ++++++++++++++++++ mkdocs/mkdocs.yml | 47 +++++++++++---- mkdocs/requirements.txt | 9 +++ 12 files changed, 162 insertions(+), 12 deletions(-) create mode 100644 mkdocs/docs/SUMMARY.md create mode 100644 mkdocs/docs/assets/images/iceberg-logo-icon.png create mode 100644 mkdocs/gen_doc_stubs.py diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index d6642ff87c..2abac29d27 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -82,3 +82,5 @@ repos: - mdformat-config - mdformat-beautysh - mdformat-admon + - mdformat-mkdocs + - mdformat-frontmatter diff --git a/mkdocs/docs/SUMMARY.md b/mkdocs/docs/SUMMARY.md new file mode 100644 index 0000000000..46383823ec --- /dev/null +++ b/mkdocs/docs/SUMMARY.md @@ -0,0 +1,31 @@ + + + + +- [Home](index.md) +- [Configuration](configuration.md) +- [CLI](cli.md) +- [API](api.md) +- [Contributing](contributing.md) +- [Feature support](feature-support.md) +- Releases + - [Verify a release](verify-release.md) + - [How to release](how-to-release.md) +- [Code Reference](reference/) + + diff --git a/mkdocs/docs/api.md b/mkdocs/docs/api.md index bd6b3d3d6f..ddd5ca180f 100644 --- a/mkdocs/docs/api.md +++ b/mkdocs/docs/api.md @@ -1,3 +1,8 @@ +--- +hide: + - navigation +--- +